fix: synchronize the input image and the output annotation (#359)

* fix: synchronize annotations and images * fix: image is upside down on iOS devices
homuler · Nov 27, 2021 · 47598ce · 47598ce
1 parent 06423ca
commit 47598ce
Show file tree

Hide file tree

Showing 12 changed files with 47 additions and 11 deletions.
diff --git a/Assets/Mediapipe/Samples/Common/Scripts/Solution.cs b/Assets/Mediapipe/Samples/Common/Scripts/Solution.cs
@@ -77,6 +77,8 @@ protected static void SetupScreen(RawImage screen, ImageSource imageSource)
       {
         screen.uvRect = new UnityEngine.Rect(0, 1, 1, -1);
       }
+
+      screen.texture = imageSource.GetCurrentTexture();
     }
 
     protected static void SetupAnnotationController<T>(AnnotationController<T> annotationController, ImageSource imageSource, bool expectedToBeMirrored = false) where T : HierarchicalAnnotation
@@ -106,5 +108,15 @@ protected static void ReadFromImageSource(ImageSource imageSource, TextureFrame
         textureFrame.ReadTextureFromOnCPU(sourceTexture);
       }
     }
+
+    protected static void UpdateScreenSync(RawImage screen, TextureFrame textureFrame)
+    {
+      if (!(screen.texture is Texture2D))
+      {
+        screen.texture = new Texture2D(textureFrame.width, textureFrame.height, TextureFormat.RGBA32, false);
+        screen.uvRect = new UnityEngine.Rect(0, 0, 1, 1);
+      }
+      textureFrame.CopyTexture(screen.texture);
+    }
   }
 }
diff --git a/Assets/Mediapipe/Samples/Scenes/Box Tracking/BoxTrackingSolution.cs b/Assets/Mediapipe/Samples/Scenes/Box Tracking/BoxTrackingSolution.cs
@@ -72,7 +72,6 @@ private IEnumerator Run()
       }
       // NOTE: The _screen will be resized later, keeping the aspect ratio.
       SetupScreen(_screen, imageSource);
-      _screen.texture = imageSource.GetCurrentTexture();
 
       Logger.LogInfo(TAG, $"Running Mode = {runningMode}");
 
@@ -114,6 +113,9 @@ private IEnumerator Run()
 
         if (runningMode == RunningMode.Sync)
         {
+          // TODO: copy texture before `textureFrame` is released
+          UpdateScreenSync(_screen, textureFrame);
+
           // When running synchronously, wait for the outputs here (blocks the main thread).
           var trackedDetections = _graphRunner.FetchNextValue();
           _trackedDetectionsAnnotationController.DrawNow(trackedDetections);

diff --git a/Assets/Mediapipe/Samples/Scenes/Face Detection/FaceDetectionSolution.cs b/Assets/Mediapipe/Samples/Scenes/Face Detection/FaceDetectionSolution.cs
@@ -78,7 +78,6 @@ private IEnumerator Run()
       }
       // NOTE: The _screen will be resized later, keeping the aspect ratio.
       SetupScreen(_screen, imageSource);
-      _screen.texture = imageSource.GetCurrentTexture();
 
       Logger.LogInfo(TAG, $"Model Selection = {modelType}");
       Logger.LogInfo(TAG, $"Running Mode = {runningMode}");
@@ -121,6 +120,9 @@ private IEnumerator Run()
 
         if (runningMode == RunningMode.Sync)
         {
+          // TODO: copy texture before `textureFrame` is released
+          UpdateScreenSync(_screen, textureFrame);
+
           // When running synchronously, wait for the outputs here (blocks the main thread).
           var detections = _graphRunner.FetchNextValue();
           _faceDetectionsAnnotationController.DrawNow(detections);

diff --git a/Assets/Mediapipe/Samples/Scenes/Face Mesh/FaceMeshSolution.cs b/Assets/Mediapipe/Samples/Scenes/Face Mesh/FaceMeshSolution.cs
@@ -87,7 +87,6 @@ private IEnumerator Run()
       }
       // NOTE: The _screen will be resized later, keeping the aspect ratio.
       SetupScreen(_screen, imageSource);
-      _screen.texture = imageSource.GetCurrentTexture();
 
       Logger.LogInfo(TAG, $"Max Num Faces = {maxNumFaces}");
       Logger.LogInfo(TAG, $"Refine Landmarks = {refineLandmarks}");
@@ -138,6 +137,9 @@ private IEnumerator Run()
 
         if (runningMode == RunningMode.Sync)
         {
+          // TODO: copy texture before `textureFrame` is released
+          UpdateScreenSync(_screen, textureFrame);
+
           // When running synchronously, wait for the outputs here (blocks the main thread).
           var value = _graphRunner.FetchNextValue();
           _faceDetectionsAnnotationController.DrawNow(value.faceDetections);

diff --git a/Assets/Mediapipe/Samples/Scenes/Hair Segmentation/HairSegmentationSolution.cs b/Assets/Mediapipe/Samples/Scenes/Hair Segmentation/HairSegmentationSolution.cs
@@ -71,7 +71,6 @@ private IEnumerator Run()
       }
       // NOTE: The _screen will be resized later, keeping the aspect ratio.
       SetupScreen(_screen, imageSource);
-      _screen.texture = imageSource.GetCurrentTexture();
 
       Logger.LogInfo(TAG, $"Running Mode = {runningMode}");
 
@@ -114,6 +113,9 @@ private IEnumerator Run()
 
         if (runningMode == RunningMode.Sync)
         {
+          // TODO: copy texture before `textureFrame` is released
+          UpdateScreenSync(_screen, textureFrame);
+
           // When running synchronously, wait for the outputs here (blocks the main thread).
           var hairMask = _graphRunner.FetchNextValue();
           _hairMaskAnnotationController.DrawNow(hairMask);

diff --git a/Assets/Mediapipe/Samples/Scenes/Hand Tracking/HandTrackingSolution.cs b/Assets/Mediapipe/Samples/Scenes/Hand Tracking/HandTrackingSolution.cs
@@ -81,7 +81,6 @@ private IEnumerator Run()
       }
       // NOTE: The _screen will be resized later, keeping the aspect ratio.
       SetupScreen(_screen, imageSource);
-      _screen.texture = imageSource.GetCurrentTexture();
 
       Logger.LogInfo(TAG, $"Max Num Hands = {maxNumHands}");
       Logger.LogInfo(TAG, $"Running Mode = {runningMode}");
@@ -132,6 +131,9 @@ private IEnumerator Run()
 
         if (runningMode == RunningMode.Sync)
         {
+          // TODO: copy texture before `textureFrame` is released
+          UpdateScreenSync(_screen, textureFrame);
+
           // When running synchronously, wait for the outputs here (blocks the main thread).
           var value = _graphRunner.FetchNextValue();
           _palmDetectionsAnnotationController.DrawNow(value.palmDetections);

diff --git a/Assets/Mediapipe/Samples/Scenes/Holistic/HolisticTrackingSolution.cs b/Assets/Mediapipe/Samples/Scenes/Holistic/HolisticTrackingSolution.cs
@@ -92,7 +92,7 @@ private IEnumerator Run()
       }
       // NOTE: The _screen will be resized later, keeping the aspect ratio.
       SetupScreen(_screen, imageSource);
-      _screen.texture = imageSource.GetCurrentTexture();
+
       _worldAnnotationArea.localEulerAngles = imageSource.rotation.Reverse().GetEulerAngles();
 
       Logger.LogInfo(TAG, $"Model Complexity = {modelComplexity}");
@@ -148,6 +148,9 @@ private IEnumerator Run()
 
         if (runningMode == RunningMode.Sync)
         {
+          // TODO: copy texture before `textureFrame` is released
+          UpdateScreenSync(_screen, textureFrame);
+
           // When running synchronously, wait for the outputs here (blocks the main thread).
           var value = _graphRunner.FetchNextValue();
           _poseDetectionAnnotationController.DrawNow(value.poseDetection);

diff --git a/Assets/Mediapipe/Samples/Scenes/Instant Motion Tracking/InstantMotionTrackingSolution.cs b/Assets/Mediapipe/Samples/Scenes/Instant Motion Tracking/InstantMotionTrackingSolution.cs
@@ -91,7 +91,6 @@ private IEnumerator Run()
       }
       // NOTE: The _screen will be resized later, keeping the aspect ratio.
       SetupScreen(_screen, imageSource);
-      _screen.texture = imageSource.GetCurrentTexture();
 
       Logger.LogInfo(TAG, $"Running Mode = {runningMode}");
 
@@ -136,6 +135,9 @@ private IEnumerator Run()
 
         if (runningMode == RunningMode.Sync)
         {
+          // TODO: copy texture before `textureFrame` is released
+          UpdateScreenSync(_screen, textureFrame);
+
           // When running synchronously, wait for the outputs here (blocks the main thread).
           var value = _graphRunner.FetchNextValue();
           _trackedAnchorDataAnnotationController.DrawNow(value);

diff --git a/Assets/Mediapipe/Samples/Scenes/Iris Tracking/IrisTrackingSolution.cs b/Assets/Mediapipe/Samples/Scenes/Iris Tracking/IrisTrackingSolution.cs
@@ -74,7 +74,6 @@ private IEnumerator Run()
       }
       // NOTE: The _screen will be resized later, keeping the aspect ratio.
       SetupScreen(_screen, imageSource);
-      _screen.texture = imageSource.GetCurrentTexture();
 
       Logger.LogInfo(TAG, $"Running Mode = {runningMode}");
 
@@ -120,6 +119,9 @@ private IEnumerator Run()
 
         if (runningMode == RunningMode.Sync)
         {
+          // TODO: copy texture before `textureFrame` is released
+          UpdateScreenSync(_screen, textureFrame);
+
           // When running synchronously, wait for the outputs here (blocks the main thread).
           var value = _graphRunner.FetchNextValue();
           _faceDetectionsAnnotationController.DrawNow(value.faceDetections);

diff --git a/Assets/Mediapipe/Samples/Scenes/Object Detection/ObjectDetectionSolution.cs b/Assets/Mediapipe/Samples/Scenes/Object Detection/ObjectDetectionSolution.cs
@@ -72,7 +72,6 @@ private IEnumerator Run()
       }
       // NOTE: The _screen will be resized later, keeping the aspect ratio.
       SetupScreen(_screen, imageSource);
-      _screen.texture = imageSource.GetCurrentTexture();
 
       Logger.LogInfo(TAG, $"Running Mode = {runningMode}");
 
@@ -114,6 +113,9 @@ private IEnumerator Run()
 
         if (runningMode == RunningMode.Sync)
         {
+          // TODO: copy texture before `textureFrame` is released
+          UpdateScreenSync(_screen, textureFrame);
+
           // When running synchronously, wait for the outputs here (blocks the main thread).
           var detections = _graphRunner.FetchNextDetections();
           _outputDetectionsAnnotationController.DrawNow(detections);

diff --git a/Assets/Mediapipe/Samples/Scenes/Objectron/ObjectronSolution.cs b/Assets/Mediapipe/Samples/Scenes/Objectron/ObjectronSolution.cs
@@ -86,7 +86,6 @@ private IEnumerator Run()
       }
       // NOTE: The _screen will be resized later, keeping the aspect ratio.
       SetupScreen(_screen, imageSource);
-      _screen.texture = imageSource.GetCurrentTexture();
 
       Logger.LogInfo(TAG, $"Category = {category}");
       Logger.LogInfo(TAG, $"Max Num Objects = {maxNumObjects}");
@@ -137,6 +136,9 @@ private IEnumerator Run()
 
         if (runningMode == RunningMode.Sync)
         {
+          // TODO: copy texture before `textureFrame` is released
+          UpdateScreenSync(_screen, textureFrame);
+
           // When running synchronously, wait for the outputs here (blocks the main thread).
           var value = _graphRunner.FetchNextValue();
           _liftedObjectsAnnotationController.DrawNow(value.liftedObjects);

diff --git a/Assets/Mediapipe/Samples/Scenes/Pose Tracking/PoseTrackingSolution.cs b/Assets/Mediapipe/Samples/Scenes/Pose Tracking/PoseTrackingSolution.cs
@@ -87,7 +87,7 @@ private IEnumerator Run()
       }
       // NOTE: The _screen will be resized later, keeping the aspect ratio.
       SetupScreen(_screen, imageSource);
-      _screen.texture = imageSource.GetCurrentTexture();
+
       _worldAnnotationArea.localEulerAngles = imageSource.rotation.Reverse().GetEulerAngles();
 
       Logger.LogInfo(TAG, $"Model Complexity = {modelComplexity}");
@@ -138,6 +138,9 @@ private IEnumerator Run()
 
         if (runningMode == RunningMode.Sync)
         {
+          // TODO: copy texture before `textureFrame` is released
+          UpdateScreenSync(_screen, textureFrame);
+
           // When running synchronously, wait for the outputs here (blocks the main thread).
           var value = _graphRunner.FetchNextValue();
           _poseDetectionAnnotationController.DrawNow(value.poseDetection);