From c603366b527fcd10640ef86a009e471db3866e7d Mon Sep 17 00:00:00 2001 From: Junrou Nishida Date: Sat, 20 Nov 2021 18:48:46 +0900 Subject: [PATCH] build!: MediaPipe v0.8.8 (#356) * build: upgrade mediapipe to 0.8.8 * fix: resolve compile errors * build: upgrade unity (2020.3.23f1) * doc: update Unity and MediaPipe version --- .../Runtime/Scripts/Gpu/GlTexture.cs | 6 - .../NativeMethods/Gpu/GlTexture_Unsafe.cs | 3 - .../Tests/EditMode/Gpu/GlTextureTest.cs | 11 -- Packages/manifest.json | 18 +- Packages/packages-lock.json | 47 +++-- ProjectSettings/ProjectVersion.txt | 4 +- README.md | 2 +- WORKSPACE | 76 +++++--- mediapipe_api/gpu/gl_calculator_helper.cc | 7 - mediapipe_api/gpu/gl_calculator_helper.h | 1 - third_party/mediapipe_extension.diff | 181 +++--------------- third_party/mediapipe_workaround.diff | 27 +-- 12 files changed, 125 insertions(+), 258 deletions(-) diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Gpu/GlTexture.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Gpu/GlTexture.cs index bb118bed0..ef9487c94 100644 --- a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Gpu/GlTexture.cs +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Gpu/GlTexture.cs @@ -16,12 +16,6 @@ public GlTexture() : base() this.ptr = ptr; } - public GlTexture(uint name, int width, int height) : base() - { - UnsafeNativeMethods.mp_GlTexture__ui_i_i(name, width, height, out var ptr).Assert(); - this.ptr = ptr; - } - public GlTexture(IntPtr ptr, bool isOwner = true) : base(ptr, isOwner) { } protected override void DeleteMpPtr() diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Gpu/GlTexture_Unsafe.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Gpu/GlTexture_Unsafe.cs index c1c78d315..ced7bd542 100644 --- a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Gpu/GlTexture_Unsafe.cs +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Gpu/GlTexture_Unsafe.cs @@ -14,9 +14,6 @@ internal static partial class UnsafeNativeMethods [DllImport(MediaPipeLibrary, ExactSpelling = true)] public static extern MpReturnCode mp_GlTexture__(out IntPtr glTexture); - [DllImport(MediaPipeLibrary, ExactSpelling = true)] - public static extern MpReturnCode mp_GlTexture__ui_i_i(uint name, int width, int height, out IntPtr glTexture); - [DllImport(MediaPipeLibrary, ExactSpelling = true)] public static extern void mp_GlTexture__delete(IntPtr glTexture); diff --git a/Packages/com.github.homuler.mediapipe/Tests/EditMode/Gpu/GlTextureTest.cs b/Packages/com.github.homuler.mediapipe/Tests/EditMode/Gpu/GlTextureTest.cs index 14ab5f7a0..acbc314c3 100644 --- a/Packages/com.github.homuler.mediapipe/Tests/EditMode/Gpu/GlTextureTest.cs +++ b/Packages/com.github.homuler.mediapipe/Tests/EditMode/Gpu/GlTextureTest.cs @@ -21,17 +21,6 @@ public void Ctor_ShouldInstantiateGlTexture_When_CalledWithNoArguments() Assert.AreEqual(glTexture.height, 0); } } - - [Test, GpuOnly] - public void Ctor_ShouldInstantiateGlTexture_When_CalledWithNameAndSize() - { - using (var glTexture = new GlTexture(1, 100, 100)) - { - Assert.AreEqual(glTexture.name, 1); - Assert.AreEqual(glTexture.width, 100); - Assert.AreEqual(glTexture.height, 100); - } - } #endregion #region #isDisposed diff --git a/Packages/manifest.json b/Packages/manifest.json index 71547cf37..9d3c82a1f 100644 --- a/Packages/manifest.json +++ b/Packages/manifest.json @@ -1,21 +1,21 @@ { "dependencies": { - "com.unity.2d.animation": "5.0.4", + "com.unity.2d.animation": "5.0.9", "com.unity.2d.pixel-perfect": "4.0.1", - "com.unity.2d.psdimporter": "4.0.2", + "com.unity.2d.psdimporter": "4.1.2", "com.unity.2d.sprite": "1.0.0", - "com.unity.2d.spriteshape": "5.1.1", + "com.unity.2d.spriteshape": "5.1.5", "com.unity.2d.tilemap": "1.0.0", - "com.unity.collab-proxy": "1.3.9", + "com.unity.collab-proxy": "1.15.1", "com.unity.ide.rider": "2.0.7", - "com.unity.ide.visualstudio": "2.0.7", - "com.unity.ide.vscode": "1.2.3", + "com.unity.ide.visualstudio": "2.0.11", + "com.unity.ide.vscode": "1.2.4", "com.unity.mathematics": "1.2.1", "com.unity.settings-manager": "1.0.3", - "com.unity.test-framework": "1.1.24", + "com.unity.test-framework": "1.1.29", "com.unity.testtools.codecoverage": "1.0.0", - "com.unity.textmeshpro": "3.0.1", - "com.unity.timeline": "1.4.7", + "com.unity.textmeshpro": "3.0.6", + "com.unity.timeline": "1.4.8", "com.unity.toolchain.linux-x86_64": "0.1.18-preview", "com.unity.toolchain.macos-x86_64-linux-x86_64": "0.1.22-preview", "com.unity.ugui": "1.0.0", diff --git a/Packages/packages-lock.json b/Packages/packages-lock.json index 8c359be66..1a1ca0b83 100644 --- a/Packages/packages-lock.json +++ b/Packages/packages-lock.json @@ -7,7 +7,7 @@ "dependencies": {} }, "com.unity.2d.animation": { - "version": "5.0.4", + "version": "5.0.9", "depth": 0, "source": "registry", "dependencies": { @@ -30,7 +30,7 @@ "url": "https://packages.unity.com" }, "com.unity.2d.path": { - "version": "4.0.1", + "version": "4.0.2", "depth": 1, "source": "registry", "dependencies": {}, @@ -44,12 +44,12 @@ "url": "https://packages.unity.com" }, "com.unity.2d.psdimporter": { - "version": "4.0.2", + "version": "4.1.2", "depth": 0, "source": "registry", "dependencies": { - "com.unity.2d.common": "4.0.2", - "com.unity.2d.animation": "5.0.2", + "com.unity.2d.common": "4.0.3", + "com.unity.2d.animation": "5.0.9", "com.unity.2d.sprite": "1.0.0" }, "url": "https://packages.unity.com" @@ -61,13 +61,13 @@ "dependencies": {} }, "com.unity.2d.spriteshape": { - "version": "5.1.1", + "version": "5.1.5", "depth": 0, "source": "registry", "dependencies": { "com.unity.mathematics": "1.1.0", "com.unity.2d.common": "4.0.3", - "com.unity.2d.path": "4.0.1", + "com.unity.2d.path": "4.0.2", "com.unity.modules.physics2d": "1.0.0" }, "url": "https://packages.unity.com" @@ -79,10 +79,13 @@ "dependencies": {} }, "com.unity.collab-proxy": { - "version": "1.3.9", + "version": "1.15.1", "depth": 0, "source": "registry", - "dependencies": {}, + "dependencies": { + "com.unity.nuget.newtonsoft-json": "2.0.0", + "com.unity.services.core": "1.0.1" + }, "url": "https://packages.unity.com" }, "com.unity.ext.nunit": { @@ -102,7 +105,7 @@ "url": "https://packages.unity.com" }, "com.unity.ide.visualstudio": { - "version": "2.0.7", + "version": "2.0.11", "depth": 0, "source": "registry", "dependencies": { @@ -111,7 +114,7 @@ "url": "https://packages.unity.com" }, "com.unity.ide.vscode": { - "version": "1.2.3", + "version": "1.2.4", "depth": 0, "source": "registry", "dependencies": {}, @@ -124,6 +127,22 @@ "dependencies": {}, "url": "https://packages.unity.com" }, + "com.unity.nuget.newtonsoft-json": { + "version": "2.0.0", + "depth": 1, + "source": "registry", + "dependencies": {}, + "url": "https://packages.unity.com" + }, + "com.unity.services.core": { + "version": "1.0.1", + "depth": 1, + "source": "registry", + "dependencies": { + "com.unity.modules.unitywebrequest": "1.0.0" + }, + "url": "https://packages.unity.com" + }, "com.unity.settings-manager": { "version": "1.0.3", "depth": 0, @@ -148,7 +167,7 @@ "url": "https://packages.unity.com" }, "com.unity.test-framework": { - "version": "1.1.24", + "version": "1.1.29", "depth": 0, "source": "registry", "dependencies": { @@ -169,7 +188,7 @@ "url": "https://packages.unity.com" }, "com.unity.textmeshpro": { - "version": "3.0.1", + "version": "3.0.6", "depth": 0, "source": "registry", "dependencies": { @@ -178,7 +197,7 @@ "url": "https://packages.unity.com" }, "com.unity.timeline": { - "version": "1.4.7", + "version": "1.4.8", "depth": 0, "source": "registry", "dependencies": { diff --git a/ProjectSettings/ProjectVersion.txt b/ProjectSettings/ProjectVersion.txt index 4e87e307f..24993f714 100644 --- a/ProjectSettings/ProjectVersion.txt +++ b/ProjectSettings/ProjectVersion.txt @@ -1,2 +1,2 @@ -m_EditorVersion: 2020.3.8f1 -m_EditorVersionWithRevision: 2020.3.8f1 (507919d4fff5) +m_EditorVersion: 2020.3.23f1 +m_EditorVersionWithRevision: 2020.3.23f1 (c5d91304a876) diff --git a/README.md b/README.md index 411770e40..9fb1cfcd7 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # MediaPipe Unity Plugin -This is a Unity (2020.3.8f1) Plugin to use MediaPipe (0.8.6). +This is a Unity (2020.3.23f1) Plugin to use MediaPipe (0.8.8). ## Prerequisites diff --git a/WORKSPACE b/WORKSPACE index 1462d0ebd..0cd0e96b2 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -43,7 +43,7 @@ rules_pkg_dependencies() new_local_repository( name = "unity", build_file = "@//third_party:unity.BUILD", - path = "/path/to/unity/2020.3.8f1", + path = "/path/to/unity/2020.3.23f1", ) # mediapipe @@ -59,12 +59,12 @@ http_archive( "@//third_party:mediapipe_model_path.diff", "@//third_party:mediapipe_extension.diff", ], - sha256 = "3fa1548474d8ef943361804a9f86a3edaa00b0976b9d58fd3b41092d339053c6", - strip_prefix = "mediapipe-0.8.6", - urls = ["https://github.com/google/mediapipe/archive/v0.8.6.tar.gz"], + sha256 = "d20c4c7a957c70be6018c704a5cea3c61314df22e4f88b452c19472f0227a48f", + strip_prefix = "mediapipe-0.8.8", + urls = ["https://github.com/google/mediapipe/archive/v0.8.8.tar.gz"], ) -# ABSL cpp library lts_2020_09_23 +# ABSL cpp library lts_2021_03_24, patch 2. http_archive( name = "com_google_absl", patch_args = [ @@ -74,10 +74,10 @@ http_archive( patches = [ "@com_google_mediapipe//third_party:com_google_absl_f863b622fe13612433fdf43f76547d5edda0c93001.diff", ], - sha256 = "b3744a4f7a249d5eaf2309daad597631ce77ea62e0fc6abffbab4b4c3dc0fc08", - strip_prefix = "abseil-cpp-20200923", + sha256 = "59b862f50e710277f8ede96f083a5bb8d7c9595376146838b9580be90374ee1f", + strip_prefix = "abseil-cpp-20210324.2", urls = [ - "https://github.com/abseil/abseil-cpp/archive/20200923.tar.gz", + "https://github.com/abseil/abseil-cpp/archive/refs/tags/20210324.2.tar.gz", ], ) @@ -99,19 +99,12 @@ load("@rules_foreign_cc//foreign_cc:repositories.bzl", "rules_foreign_cc_depende rules_foreign_cc_dependencies() # GoogleTest/GoogleMock framework. Used by most unit-tests. -# Last updated 2020-06-30. +# Last updated 2021-07-02. http_archive( name = "com_google_googletest", - patch_args = [ - "-p1", - ], - patches = [ - # fix for https://github.com/google/googletest/issues/2817 - "@com_google_mediapipe//third_party:com_google_googletest_9d580ea80592189e6d44fa35bcf9cdea8bf620d6.diff", - ], - sha256 = "04a1751f94244307cebe695a69cc945f9387a80b0ef1af21394a490697c5c895", - strip_prefix = "googletest-aee0f9d9b5b87796ee8a0ab26b7587ec30e8858e", - urls = ["https://github.com/google/googletest/archive/aee0f9d9b5b87796ee8a0ab26b7587ec30e8858e.zip"], + sha256 = "de682ea824bfffba05b4e33b67431c247397d6175962534305136aa06f92e049", + strip_prefix = "googletest-4ec4cd23f486bf70efcc5d2caa40f24368f752e3", + urls = ["https://github.com/google/googletest/archive/4ec4cd23f486bf70efcc5d2caa40f24368f752e3.zip"], ) # Google Benchmark library. @@ -211,11 +204,11 @@ http_archive( http_archive( name = "pybind11", build_file = "@pybind11_bazel//:pybind11.BUILD", - sha256 = "1eed57bc6863190e35637290f97a20c81cfe4d9090ac0a24f3bbf08f265eb71d", - strip_prefix = "pybind11-2.4.3", + sha256 = "616d1c42e4cf14fa27b2a4ff759d7d7b33006fdc5ad8fd603bb2c22622f27020", + strip_prefix = "pybind11-2.7.1", urls = [ - "https://storage.googleapis.com/mirror.tensorflow.org/github.com/pybind/pybind11/archive/v2.4.3.tar.gz", - "https://github.com/pybind/pybind11/archive/v2.4.3.tar.gz", + "https://storage.googleapis.com/mirror.tensorflow.org/github.com/pybind/pybind11/archive/v2.7.1.tar.gz", + "https://github.com/pybind/pybind11/archive/v2.7.1.tar.gz", ], ) @@ -389,7 +382,10 @@ load("@rules_jvm_external//:defs.bzl", "maven_install") maven_install( artifacts = [ "androidx.concurrent:concurrent-futures:1.0.0-alpha03", - "androidx.lifecycle:lifecycle-common:2.2.0", + "androidx.lifecycle:lifecycle-common:2.3.1", + "androidx.activity:activity:1.2.2", + "androidx.exifinterface:exifinterface:1.3.3", + "androidx.fragment:fragment:1.3.4", "androidx.annotation:annotation:aar:1.1.0", "androidx.appcompat:appcompat:aar:1.1.0-rc01", "androidx.camera:camera-core:1.0.0-beta10", @@ -404,9 +400,9 @@ maven_install( "com.google.android.material:material:aar:1.0.0-rc01", "com.google.auto.value:auto-value:1.8.1", "com.google.auto.value:auto-value-annotations:1.8.1", - "com.google.code.findbugs:jsr305:3.0.2", - "com.google.flogger:flogger-system-backend:0.3.1", - "com.google.flogger:flogger:0.3.1", + "com.google.code.findbugs:jsr305:latest.release", + "com.google.flogger:flogger-system-backend:0.6", + "com.google.flogger:flogger:0.6", "com.google.guava:guava:27.0.1-android", "com.google.guava:listenablefuture:1.0", "junit:junit:4.12", @@ -434,10 +430,10 @@ http_archive( ) # Tensorflow repo should always go after the other external dependencies. -# 2021-06-07 -_TENSORFLOW_GIT_COMMIT = "700533808e6016dc458bb2eeecfca4babfc482ec" +# 2021-07-29 +_TENSORFLOW_GIT_COMMIT = "52a2905cbc21034766c08041933053178c5d10e3" -_TENSORFLOW_SHA256 = "b6edd7f4039bfc19f3e77594ecff558ba620091d0dc48181484b3d9085026126" +_TENSORFLOW_SHA256 = "06d4691bcdb700f3275fa0971a1585221c2b9f3dffe867963be565a6643d7f56" http_archive( name = "org_tensorflow", @@ -447,6 +443,8 @@ http_archive( patches = [ "@com_google_mediapipe//third_party:org_tensorflow_compatibility_fixes.diff", "@com_google_mediapipe//third_party:org_tensorflow_objc_cxx17.diff", + # Diff is generated with a script, don't update it manually. + "@com_google_mediapipe//third_party:org_tensorflow_custom_ops.diff", "@//third_party:tensorflow_python_path.diff", ], sha256 = _TENSORFLOW_SHA256, @@ -463,3 +461,21 @@ tf_workspace3() load("@org_tensorflow//tensorflow:workspace2.bzl", "tf_workspace2") tf_workspace2() + +# Edge TPU +http_archive( + name = "libedgetpu", + sha256 = "14d5527a943a25bc648c28a9961f954f70ba4d79c0a9ca5ae226e1831d72fe80", + strip_prefix = "libedgetpu-3164995622300286ef2bb14d7fdc2792dae045b7", + urls = [ + "https://github.com/google-coral/libedgetpu/archive/3164995622300286ef2bb14d7fdc2792dae045b7.tar.gz", + ], +) + +load("@libedgetpu//:workspace.bzl", "libedgetpu_dependencies") + +libedgetpu_dependencies() + +load("@coral_crosstool//:configure.bzl", "cc_crosstool") + +cc_crosstool(name = "crosstool") diff --git a/mediapipe_api/gpu/gl_calculator_helper.cc b/mediapipe_api/gpu/gl_calculator_helper.cc index b1a99894a..5d37e2f94 100644 --- a/mediapipe_api/gpu/gl_calculator_helper.cc +++ b/mediapipe_api/gpu/gl_calculator_helper.cc @@ -97,13 +97,6 @@ MpReturnCode mp_GlTexture__(mediapipe::GlTexture** gl_texture_out) { CATCH_EXCEPTION } -MpReturnCode mp_GlTexture__ui_i_i(GLuint name, int width, int height, mediapipe::GlTexture** gl_texture_out) { - TRY - *gl_texture_out = new mediapipe::GlTexture{name, width, height}; - RETURN_CODE(MpReturnCode::Success); - CATCH_EXCEPTION -} - void mp_GlTexture__delete(mediapipe::GlTexture* gl_texture) { delete gl_texture; } int mp_GlTexture__width(mediapipe::GlTexture* gl_texture) { return gl_texture->width(); } diff --git a/mediapipe_api/gpu/gl_calculator_helper.h b/mediapipe_api/gpu/gl_calculator_helper.h index d59e0f029..8b8be0650 100644 --- a/mediapipe_api/gpu/gl_calculator_helper.h +++ b/mediapipe_api/gpu/gl_calculator_helper.h @@ -47,7 +47,6 @@ MP_CAPI(bool) mp_GlCalculatorHelper__Initialized(mediapipe::GlCalculatorHelper* /** GlTexture API */ MP_CAPI(MpReturnCode) mp_GlTexture__(mediapipe::GlTexture** gl_texture_out); -MP_CAPI(MpReturnCode) mp_GlTexture__ui_i_i(GLuint name, int width, int height, mediapipe::GlTexture** gl_texture_out); MP_CAPI(void) mp_GlTexture__delete(mediapipe::GlTexture* gl_texture); MP_CAPI(int) mp_GlTexture__width(mediapipe::GlTexture* gl_texture); MP_CAPI(int) mp_GlTexture__height(mediapipe::GlTexture* gl_texture); diff --git a/third_party/mediapipe_extension.diff b/third_party/mediapipe_extension.diff index 2dceaa7e4..e078851fb 100644 --- a/third_party/mediapipe_extension.diff +++ b/third_party/mediapipe_extension.diff @@ -1,135 +1,8 @@ -diff --git a/mediapipe/framework/calculator_graph.cc b/mediapipe/framework/calculator_graph.cc -index 2c34112..3b8ab93 100644 ---- a/mediapipe/framework/calculator_graph.cc -+++ b/mediapipe/framework/calculator_graph.cc -@@ -432,7 +432,7 @@ absl::Status CalculatorGraph::ObserveOutputStream( - } - - absl::StatusOr CalculatorGraph::AddOutputStreamPoller( -- const std::string& stream_name) { -+ const std::string& stream_name, bool observe_timestamp_bounds) { - RET_CHECK(initialized_).SetNoLogging() - << "CalculatorGraph is not initialized."; - int output_stream_index = validated_graph_->OutputStreamIndex(stream_name); -@@ -446,7 +446,7 @@ absl::StatusOr CalculatorGraph::AddOutputStreamPoller( - stream_name, &any_packet_type_, - std::bind(&CalculatorGraph::UpdateThrottledNodes, this, - std::placeholders::_1, std::placeholders::_2), -- &output_stream_managers_[output_stream_index])); -+ &output_stream_managers_[output_stream_index], observe_timestamp_bounds)); - OutputStreamPoller poller(internal_poller); - graph_output_streams_.push_back(std::move(internal_poller)); - return std::move(poller); -diff --git a/mediapipe/framework/calculator_graph.h b/mediapipe/framework/calculator_graph.h -index 77a1ee5..81beec2 100644 ---- a/mediapipe/framework/calculator_graph.h -+++ b/mediapipe/framework/calculator_graph.h -@@ -164,7 +164,8 @@ class CalculatorGraph { - // polling API for accessing a stream's output. Should only be called before - // Run() or StartRun(). For asynchronous output, use ObserveOutputStream. See - // also the helpers in tool/sink.h. -- StatusOrPoller AddOutputStreamPoller(const std::string& stream_name); -+ StatusOrPoller AddOutputStreamPoller(const std::string& stream_name, -+ bool observe_timestamp_bounds = false); - - // Gets output side packet by name after the graph is done. However, base - // packets (generated by PacketGenerators) can be retrieved before -diff --git a/mediapipe/framework/graph_output_stream.cc b/mediapipe/framework/graph_output_stream.cc -index 6639bb8..8235d3c 100644 ---- a/mediapipe/framework/graph_output_stream.cc -+++ b/mediapipe/framework/graph_output_stream.cc -@@ -125,9 +125,10 @@ absl::Status OutputStreamObserver::Notify() { - absl::Status OutputStreamPollerImpl::Initialize( - const std::string& stream_name, const PacketType* packet_type, - std::function queue_size_callback, -- OutputStreamManager* output_stream_manager) { -+ OutputStreamManager* output_stream_manager, bool observe_timestamp_bounds) { - MP_RETURN_IF_ERROR(GraphOutputStream::Initialize(stream_name, packet_type, -- output_stream_manager)); -+ output_stream_manager, -+ observe_timestamp_bounds)); - input_stream_handler_->SetQueueSizeCallbacks(queue_size_callback, - queue_size_callback); - return absl::OkStatus(); -@@ -176,12 +177,16 @@ void OutputStreamPollerImpl::NotifyError() { - bool OutputStreamPollerImpl::Next(Packet* packet) { - CHECK(packet); - bool empty_queue = true; -+ bool observed_timestamp_bound_change = false; - Timestamp min_timestamp = Timestamp::Unset(); - mutex_.Lock(); - while (true) { - min_timestamp = input_stream_->MinTimestampOrBound(&empty_queue); -+ observed_timestamp_bound_change = -+ input_stream_handler_->ProcessTimestampBounds() && -+ prev_output_ts_ < min_timestamp.PreviousAllowedInStream(); - if (graph_has_error_ || !empty_queue || -- min_timestamp == Timestamp::Done()) { -+ min_timestamp == Timestamp::Done() || observed_timestamp_bound_change) { - break; - } else { - handler_condvar_.Wait(&mutex_); -@@ -191,17 +196,26 @@ bool OutputStreamPollerImpl::Next(Packet* packet) { - mutex_.Unlock(); - return false; - } -+ if (empty_queue) { -+ prev_output_ts_ = min_timestamp.PreviousAllowedInStream(); -+ } else { -+ prev_output_ts_ = min_timestamp; -+ } - mutex_.Unlock(); - if (min_timestamp == Timestamp::Done()) { - return false; - } -- int num_packets_dropped = 0; -- bool stream_is_done = false; -- *packet = input_stream_->PopPacketAtTimestamp( -- min_timestamp, &num_packets_dropped, &stream_is_done); -- CHECK_EQ(num_packets_dropped, 0) -- << absl::Substitute("Dropped $0 packet(s) on input stream \"$1\".", -- num_packets_dropped, input_stream_->Name()); -+ if (!empty_queue) { -+ int num_packets_dropped = 0; -+ bool stream_is_done = false; -+ *packet = input_stream_->PopPacketAtTimestamp( -+ min_timestamp, &num_packets_dropped, &stream_is_done); -+ CHECK_EQ(num_packets_dropped, 0) -+ << absl::Substitute("Dropped $0 packet(s) on input stream \"$1\".", -+ num_packets_dropped, input_stream_->Name()); -+ } else if (observed_timestamp_bound_change) { -+ *packet = Packet().At(Timestamp(min_timestamp.PreviousAllowedInStream())); -+ } - return true; - } - -diff --git a/mediapipe/framework/graph_output_stream.h b/mediapipe/framework/graph_output_stream.h -index 393407a..1110a10 100644 ---- a/mediapipe/framework/graph_output_stream.h -+++ b/mediapipe/framework/graph_output_stream.h -@@ -143,7 +143,8 @@ class OutputStreamPollerImpl : public GraphOutputStream { - absl::Status Initialize( - const std::string& stream_name, const PacketType* packet_type, - std::function queue_size_callback, -- OutputStreamManager* output_stream_manager); -+ OutputStreamManager* output_stream_manager, -+ bool observe_timestamp_bounds = false); - - void PrepareForRun(std::function notification_callback, - std::function error_callback) override; -@@ -170,6 +171,7 @@ class OutputStreamPollerImpl : public GraphOutputStream { - absl::Mutex mutex_; - absl::CondVar handler_condvar_ ABSL_GUARDED_BY(mutex_); - bool graph_has_error_ ABSL_GUARDED_BY(mutex_); -+ Timestamp prev_output_ts_ ABSL_GUARDED_BY(mutex_) = Timestamp::Min(); - }; - - } // namespace internal diff --git a/mediapipe/gpu/gl_calculator_helper.cc b/mediapipe/gpu/gl_calculator_helper.cc -index aa708e7..c146f56 100644 +index efd822a..c6da56d 100644 --- a/mediapipe/gpu/gl_calculator_helper.cc +++ b/mediapipe/gpu/gl_calculator_helper.cc -@@ -137,6 +137,10 @@ GlTexture GlCalculatorHelper::CreateDestinationTexture(int output_width, +@@ -130,6 +130,10 @@ GlTexture GlCalculatorHelper::CreateDestinationTexture(int output_width, return impl_->CreateDestinationTexture(output_width, output_height, format); } @@ -141,10 +14,10 @@ index aa708e7..c146f56 100644 return impl_->GetGlContext(); } diff --git a/mediapipe/gpu/gl_calculator_helper.h b/mediapipe/gpu/gl_calculator_helper.h -index b5cc699..e0aa081 100644 +index d6bd718..d2dec4b 100644 --- a/mediapipe/gpu/gl_calculator_helper.h +++ b/mediapipe/gpu/gl_calculator_helper.h -@@ -144,6 +144,8 @@ class GlCalculatorHelper { +@@ -134,6 +134,8 @@ class GlCalculatorHelper { int output_width, int output_height, GpuBufferFormat format = GpuBufferFormat::kBGRA32); @@ -154,7 +27,7 @@ index b5cc699..e0aa081 100644 GLuint framebuffer() const; diff --git a/mediapipe/gpu/gl_calculator_helper_impl.h b/mediapipe/gpu/gl_calculator_helper_impl.h -index 4381111..cdabc45 100644 +index 40b53f5..3214043 100644 --- a/mediapipe/gpu/gl_calculator_helper_impl.h +++ b/mediapipe/gpu/gl_calculator_helper_impl.h @@ -49,6 +49,7 @@ class GlCalculatorHelperImpl { @@ -166,10 +39,10 @@ index 4381111..cdabc45 100644 GLuint framebuffer() const { return framebuffer_; } void BindFramebuffer(const GlTexture& dst); diff --git a/mediapipe/gpu/gl_calculator_helper_impl_common.cc b/mediapipe/gpu/gl_calculator_helper_impl_common.cc -index fb2685b..af7ed96 100644 +index fef2550..67f7b43 100644 --- a/mediapipe/gpu/gl_calculator_helper_impl_common.cc +++ b/mediapipe/gpu/gl_calculator_helper_impl_common.cc -@@ -196,14 +196,18 @@ GlTextureBufferSharedPtr GlCalculatorHelperImpl::MakeGlTextureBuffer( +@@ -146,14 +146,18 @@ std::unique_ptr GlTexture::GetFrame() const { GlTexture GlCalculatorHelperImpl::CreateDestinationTexture( int width, int height, GpuBufferFormat format) { @@ -186,14 +59,14 @@ index fb2685b..af7ed96 100644 - GpuBuffer buffer = - gpu_resources_.gpu_buffer_pool().GetBuffer(width, height, format); -- GlTexture texture = MapGpuBuffer(buffer, 0); +- GlTexture texture = MapGpuBuffer(buffer, 0, false); - -+ GlTexture texture = MapGpuBuffer(gpu_buffer, 0); ++ GlTexture texture = MapGpuBuffer(gpu_buffer, 0, false); return texture; } diff --git a/mediapipe/gpu/gl_scaler_calculator.cc b/mediapipe/gpu/gl_scaler_calculator.cc -index 6191876..19a8cf8 100644 +index fa06c88..0d0bd0c 100644 --- a/mediapipe/gpu/gl_scaler_calculator.cc +++ b/mediapipe/gpu/gl_scaler_calculator.cc @@ -12,6 +12,8 @@ @@ -205,7 +78,7 @@ index 6191876..19a8cf8 100644 #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/formats/image.h" #include "mediapipe/framework/port/ret_check.h" -@@ -61,6 +63,7 @@ using Image = mediapipe::Image; +@@ -68,6 +70,7 @@ using Image = mediapipe::Image; // existing calculator options, depending on field merge_fields. // OUTPUT_DIMENSIONS: the output width and height in pixels. // ROTATION: the counterclockwise rotation angle in degrees. @@ -213,7 +86,7 @@ index 6191876..19a8cf8 100644 // These can also be specified as options. // To enable horizontal or vertical flip, specify them in options. // The flipping is applied after rotation. -@@ -85,6 +88,7 @@ class GlScalerCalculator : public CalculatorBase { +@@ -92,6 +95,7 @@ class GlScalerCalculator : public CalculatorBase { private: GlCalculatorHelper helper_; @@ -221,17 +94,17 @@ index 6191876..19a8cf8 100644 int dst_width_ = 0; int dst_height_ = 0; float dst_scale_ = -1.f; -@@ -121,6 +125,9 @@ absl::Status GlScalerCalculator::GetContract(CalculatorContract* cc) { +@@ -128,6 +132,9 @@ absl::Status GlScalerCalculator::GetContract(CalculatorContract* cc) { } MP_RETURN_IF_ERROR(GlCalculatorHelper::UpdateContract(cc)); + if (cc->InputSidePackets().HasTag("DESTINATION")) { + cc->InputSidePackets().Tag("DESTINATION").Set(); + } - if (cc->InputSidePackets().HasTag("OPTIONS")) { - cc->InputSidePackets().Tag("OPTIONS").Set(); + if (cc->InputSidePackets().HasTag(kOptionsTag)) { + cc->InputSidePackets().Tag(kOptionsTag).Set(); } -@@ -187,6 +194,11 @@ absl::Status GlScalerCalculator::Open(CalculatorContext* cc) { +@@ -194,6 +201,11 @@ absl::Status GlScalerCalculator::Open(CalculatorContext* cc) { dst_width_ = dimensions[0]; dst_height_ = dimensions[1]; } @@ -240,19 +113,19 @@ index 6191876..19a8cf8 100644 + dst_width_ = dst_buffer_.width(); + dst_height_ = dst_buffer_.height(); + } - if (cc->InputSidePackets().HasTag("ROTATION")) { - rotation_ccw = cc->InputSidePackets().Tag("ROTATION").Get(); + if (cc->InputSidePackets().HasTag(kRotationTag)) { + rotation_ccw = cc->InputSidePackets().Tag(kRotationTag).Get(); } -@@ -197,7 +209,7 @@ absl::Status GlScalerCalculator::Open(CalculatorContext* cc) { +@@ -204,7 +216,7 @@ absl::Status GlScalerCalculator::Open(CalculatorContext* cc) { } absl::Status GlScalerCalculator::Process(CalculatorContext* cc) { -- if (cc->Inputs().HasTag("OUTPUT_DIMENSIONS")) { -+ if (!dst_buffer_ && cc->Inputs().HasTag("OUTPUT_DIMENSIONS")) { - if (cc->Inputs().Tag("OUTPUT_DIMENSIONS").IsEmpty()) { +- if (cc->Inputs().HasTag(kOutputDimensionsTag)) { ++ if (!dst_buffer_ && cc->Inputs().HasTag(kOutputDimensionsTag)) { + if (cc->Inputs().Tag(kOutputDimensionsTag).IsEmpty()) { // OUTPUT_DIMENSIONS input stream is specified, but value is missing. return absl::OkStatus(); -@@ -279,9 +291,18 @@ absl::Status GlScalerCalculator::Process(CalculatorContext* cc) { +@@ -286,9 +298,18 @@ absl::Status GlScalerCalculator::Process(CalculatorContext* cc) { MakePacket(left_right_padding).At(cc->InputTimestamp())); } @@ -275,10 +148,10 @@ index 6191876..19a8cf8 100644 glActiveTexture(GL_TEXTURE1); glBindTexture(src1.target(), src1.name()); diff --git a/mediapipe/modules/objectron/objectron_gpu.pbtxt b/mediapipe/modules/objectron/objectron_gpu.pbtxt -index 16187de..a7703f5 100644 +index 7ef2b67..83856d5 100644 --- a/mediapipe/modules/objectron/objectron_gpu.pbtxt +++ b/mediapipe/modules/objectron/objectron_gpu.pbtxt -@@ -11,6 +11,10 @@ input_side_packet: "MAX_NUM_OBJECTS:max_num_objects" +@@ -14,6 +14,10 @@ input_side_packet: "USE_PREV_LANDMARKS:use_prev_landmarks" # Collection of detected 3D objects, represented as a FrameAnnotation. output_stream: "FRAME_ANNOTATION:detected_objects" @@ -287,5 +160,5 @@ index 16187de..a7703f5 100644 +# Crop rectangles derived from bounding box landmarks. +output_stream: "NORM_RECTS:box_rects" - # Defines whether landmarks from the previous video frame should be used to help - # predict landmarks on the current video frame. + # When the optional input side packet "use_prev_landmarks" is either absent or + # set to true, uses the landmarks on the previous image to help localize diff --git a/third_party/mediapipe_workaround.diff b/third_party/mediapipe_workaround.diff index e03a2fe0f..56fef50ec 100644 --- a/third_party/mediapipe_workaround.diff +++ b/third_party/mediapipe_workaround.diff @@ -1,8 +1,8 @@ diff --git a/mediapipe/calculators/tensor/BUILD b/mediapipe/calculators/tensor/BUILD -index 71be05f..5dffa2f 100644 +index 72c2f51..79929e0 100644 --- a/mediapipe/calculators/tensor/BUILD +++ b/mediapipe/calculators/tensor/BUILD -@@ -87,7 +87,12 @@ cc_library( +@@ -92,7 +92,12 @@ cc_library( "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_buffer", "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_program", "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_shader", @@ -16,19 +16,6 @@ index 71be05f..5dffa2f 100644 alwayslink = 1, ) -diff --git a/mediapipe/calculators/tensor/tensors_to_detections_calculator.cc b/mediapipe/calculators/tensor/tensors_to_detections_calculator.cc -index f161127..a44deb8 100644 ---- a/mediapipe/calculators/tensor/tensors_to_detections_calculator.cc -+++ b/mediapipe/calculators/tensor/tensors_to_detections_calculator.cc -@@ -670,7 +670,7 @@ absl::Status TensorsToDetectionsCalculator::ConvertToDetections( - detection_boxes[box_offset + 2], detection_boxes[box_offset + 3], - detection_scores[i], detection_classes[i], options_.flip_vertically()); - const auto& bbox = detection.location_data().relative_bounding_box(); -- if (bbox.width() < 0 || bbox.height() < 0) { -+ if (bbox.width() < 0 || bbox.height() < 0 || std::isnan(bbox.width()) || std::isnan(bbox.height())) { - // Decoded detection boxes could have negative values for width/height due - // to model prediction. Filter out those boxes since some downstream - // calculators may assume non-negative values. (b/171391719) diff --git a/mediapipe/graphs/instant_motion_tracking/calculators/matrices_manager_calculator.cc b/mediapipe/graphs/instant_motion_tracking/calculators/matrices_manager_calculator.cc index c003135..edcd239 100644 --- a/mediapipe/graphs/instant_motion_tracking/calculators/matrices_manager_calculator.cc @@ -234,10 +221,10 @@ index cbacdb7..f40cf91 100644 float y; // [0.0-1.0] float z; // Centered around 1.0 [current_scale = z * initial_scale] diff --git a/mediapipe/modules/objectron/calculators/BUILD b/mediapipe/modules/objectron/calculators/BUILD -index 66150c5..b3a14a9 100644 +index fb75eb3..0eee4ff 100644 --- a/mediapipe/modules/objectron/calculators/BUILD +++ b/mediapipe/modules/objectron/calculators/BUILD -@@ -181,6 +181,15 @@ cc_library( +@@ -199,6 +199,15 @@ cc_library( hdrs = [ "tensor_util.h", ], @@ -253,7 +240,7 @@ index 66150c5..b3a14a9 100644 deps = [ "//mediapipe/framework/formats:tensor", "//mediapipe/framework/port:logging", -@@ -296,6 +305,15 @@ cc_library( +@@ -314,6 +323,15 @@ cc_library( name = "lift_2d_frame_annotation_to_3d_calculator", srcs = ["lift_2d_frame_annotation_to_3d_calculator.cc"], visibility = ["//visibility:public"], @@ -270,7 +257,7 @@ index 66150c5..b3a14a9 100644 ":annotation_cc_proto", ":belief_decoder_config_cc_proto", diff --git a/mediapipe/util/tflite/tflite_gpu_runner.cc b/mediapipe/util/tflite/tflite_gpu_runner.cc -index 5445f02..0758efe 100644 +index 5445f02..3f0eed7 100644 --- a/mediapipe/util/tflite/tflite_gpu_runner.cc +++ b/mediapipe/util/tflite/tflite_gpu_runner.cc @@ -213,8 +213,10 @@ absl::Status TFLiteGPURunner::InitializeOpenCL( @@ -280,7 +267,7 @@ index 5445f02..0758efe 100644 -#endif return absl::OkStatus(); +#else -+ return mediapipe::UnimplementedError("Currently only Android is supported"); ++ return mediapipe::UnimplementedError("OpenCL is currently only supported on Android"); +#endif }