wip: pass TextureFrame

homuler · Dec 19, 2020 · 0eb161f · 0eb161f
1 parent 9723843
commit 0eb161f
Show file tree

Hide file tree

Showing 23 changed files with 168 additions and 94 deletions.
diff --git a/Assets/MediaPipe/Examples/Scenes/DesktopGPU.unity b/Assets/MediaPipe/Examples/Scenes/DesktopGPU.unity
@@ -824,6 +824,7 @@ GameObject:
   - component: {fileID: 496037461}
   - component: {fileID: 496037460}
   - component: {fileID: 496037459}
+  - component: {fileID: 496037464}
   m_Layer: 0
   m_Name: WebCamScreen
   m_TagString: Untagged
@@ -846,7 +847,7 @@ MonoBehaviour:
   DefaultWidth: 640
   DefaultHeight: 480
   FPS: 30
-  focalLengthPx: 2
+  DefaultFocalLengthPx: 2
 --- !u!64 &496037460
 MeshCollider:
   m_ObjectHideFlags: 0
@@ -922,6 +923,19 @@ Transform:
   m_Father: {fileID: 0}
   m_RootOrder: 2
   m_LocalEulerAnglesHint: {x: 90, y: 180, z: 0}
+--- !u!114 &496037464
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 496037458}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: d5da564da19cb6b7d8e4f97f269edc5d, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  poolSize: 20
 --- !u!1 &624144232
 GameObject:
   m_ObjectHideFlags: 0

diff --git a/Assets/MediaPipe/Examples/Scripts/DefaultGraphOnCPU.cs b/Assets/MediaPipe/Examples/Scripts/DefaultGraphOnCPU.cs
@@ -35,12 +35,12 @@ public override Status StartRun() {
     return graph.StartRun();
   }
 
-  public override void RenderOutput(WebCamScreenController screenController, PixelData pixelData) {
+  public override void RenderOutput(WebCamScreenController screenController, TextureFrame textureFrame) {
     var texture = screenController.GetScreen();
 
     if (!outputStreamPoller.Next(outputPacket)) {
       Debug.LogWarning("Failed to fetch an output packet, rendering the input image");
-      texture.SetPixels32(pixelData.Colors);
+      texture.SetPixels32(textureFrame.GetPixels32());
     } else {
       texture.SetPixels32(outputPacket.Get().GetColor32s());
     }

diff --git a/Assets/MediaPipe/Examples/Scripts/DefaultGraphOnGPU.cs b/Assets/MediaPipe/Examples/Scripts/DefaultGraphOnGPU.cs
@@ -35,12 +35,13 @@ public override Status StartRun() {
     return graph.StartRun();
   }
 
-  public override void RenderOutput(WebCamScreenController screenController, PixelData pixelData) {
+  public override void RenderOutput(WebCamScreenController screenController, TextureFrame textureFrame) {
     var texture = screenController.GetScreen();
+    var pixels = textureFrame.GetPixels32();
 
     if (!outputStreamPoller.Next(outputPacket)) {
       Debug.LogWarning("Failed to fetch an output packet, rendering the input image");
-      texture.SetPixels32(pixelData.Colors);
+      texture.SetPixels32(pixels);
       texture.Apply();
       return;
     }
@@ -70,7 +71,7 @@ public override void RenderOutput(WebCamScreenController screenController, Pixel
       texture.SetPixels32(outputFrame.GetColor32s());
     } else {
       Debug.LogError(status.ToString());
-      texture.SetPixels32(pixelData.Colors);
+      texture.SetPixels32(pixels);
     }
 
     texture.Apply();

diff --git a/Assets/MediaPipe/Examples/Scripts/DemoGraph.cs b/Assets/MediaPipe/Examples/Scripts/DemoGraph.cs
@@ -3,7 +3,7 @@
 using System.Collections.Generic;
 using UnityEngine;
 
-public abstract class DemoGraph : MonoBehaviour, IDemoGraph<PixelData> {
+public abstract class DemoGraph : MonoBehaviour, IDemoGraph<TextureFrame> {
   [SerializeField] protected TextAsset config = null;
 
   protected const string inputStream = "input_video";
@@ -34,32 +34,53 @@ public void Initialize(GpuResources gpuResources, GlCalculatorHelper gpuHelper)
   /// <summary>
   ///   Convert <paramref name="colors" /> to a packet and send it to the input stream.
   /// </summary>
-  public Status PushInput(PixelData pixelData) {
+  public Status PushInput(TextureFrame textureFrame) {
     var timestamp = new Timestamp(System.Environment.TickCount & System.Int32.MaxValue);
-    var imageFrame = ImageFrame.FromPixels32(pixelData.Colors, pixelData.Width, pixelData.Height, true);
+    ImageFrame imageFrame = null;
 
     if (!IsGpuEnabled()) {
+      imageFrame = CopyPixelsFrom(textureFrame);
       var packet = new ImageFramePacket(imageFrame, timestamp);
 
       return graph.AddPacketToInputStream(inputStream, packet);
     }
 
-    var status = gpuHelper.RunInGlContext(() => {
-      var texture = gpuHelper.CreateSourceTexture(imageFrame);
-      var gpuFrame = texture.GetGpuBufferFrame();
+    #if UNITY_ANDROID
+      var glTextureName = textureFrame.GetNativeTexturePtr();
 
-      Gl.Flush();
-      texture.Release();
+      return gpuHelper.RunInGlContext(() => {
+        var glContext = GlContext.GetCurrent();
+        var glTextureBuffer = new GlTextureBuffer((UInt32)glTextureName, textureFrame.width, textureFrame.height,
+                                                  textureFrame.gpuBufferformat, textureFrame.OnRelease, glContext);
+        var gpuBuffer = new GpuBuffer(glTextureBuffer);
+        var texture = gpuHelper.CreateSourceTexture(gpuBuffer);
+        var gpuFrame = texture.GetGpuBufferFrame();
 
-      return graph.AddPacketToInputStream(inputStream, new GpuBufferPacket(gpuFrame, timestamp));
-    });
+        Gl.Flush();
+        texture.Release();
 
-    imageFrame.Dispose();
+        return graph.AddPacketToInputStream(inputStream, new GpuBufferPacket(gpuBuffer, timestamp));
+      });
+    #else
+      imageFrame = CopyPixelsFrom(textureFrame);
 
-    return status;
+      return gpuHelper.RunInGlContext(() => {
+        var texture = gpuHelper.CreateSourceTexture(imageFrame);
+        var gpuBuffer = texture.GetGpuBufferFrame();
+
+        Gl.Flush();
+        texture.Release();
+
+        return graph.AddPacketToInputStream(inputStream, new GpuBufferPacket(gpuBuffer, timestamp));
+      });
+    #endif
+  }
+
+  private ImageFrame CopyPixelsFrom(TextureFrame textureFrame) {
+    return ImageFrame.FromPixels32(textureFrame.GetPixels32(), textureFrame.width, textureFrame.height, true);
   }
 
-  public abstract void RenderOutput(WebCamScreenController screenController, PixelData pixelData);
+  public abstract void RenderOutput(WebCamScreenController screenController, TextureFrame textureFrame);
 
   public void Stop() {
     if (graph != null) {

diff --git a/Assets/MediaPipe/Examples/Scripts/FaceDetection/FaceDetectionGraph.cs b/Assets/MediaPipe/Examples/Scripts/FaceDetection/FaceDetectionGraph.cs
@@ -20,13 +20,11 @@ public override Status StartRun() {
     return graph.StartRun();
   }
 
-  public override void RenderOutput(WebCamScreenController screenController, PixelData pixelData) {
-    var detections =  FetchNextFaceDetectionsPresence() ? FetchNextFaceDetections() : new List<Detection>();
+  public override void RenderOutput(WebCamScreenController screenController, TextureFrame textureFrame) {
+    var detections = FetchNextFaceDetectionsPresence() ? FetchNextFaceDetections() : new List<Detection>();
     RenderAnnotation(screenController, detections);
 
-    var texture = screenController.GetScreen();
-    texture.SetPixels32(pixelData.Colors);
-    texture.Apply();
+    screenController.DrawScreen(textureFrame);
   }
 
   private bool FetchNextFaceDetectionsPresence() {

diff --git a/Assets/MediaPipe/Examples/Scripts/FaceMesh/FaceMeshGraph.cs b/Assets/MediaPipe/Examples/Scripts/FaceMesh/FaceMeshGraph.cs
@@ -49,12 +49,12 @@ public override Status StartRun() {
     return graph.StartRun(sidePacket);
   }
 
-  public override void RenderOutput(WebCamScreenController screenController, PixelData pixelData) {
+  public override void RenderOutput(WebCamScreenController screenController, TextureFrame textureFrame) {
     var faceMeshValue = FetchNextFaceMeshValue();
     RenderAnnotation(screenController, faceMeshValue);
 
     var texture = screenController.GetScreen();
-    texture.SetPixels32(pixelData.Colors);
+    texture.SetPixels32(textureFrame.GetPixels32());
     texture.Apply();
   }
 

diff --git a/Assets/MediaPipe/Examples/Scripts/HairSegmentation/HairSegmentationGraph.cs b/Assets/MediaPipe/Examples/Scripts/HairSegmentation/HairSegmentationGraph.cs
@@ -13,11 +13,11 @@ public override Status StartRun() {
     return graph.StartRun();
   }
 
-  public override void RenderOutput(WebCamScreenController screenController, PixelData pixelData) {
+  public override void RenderOutput(WebCamScreenController screenController, TextureFrame textureFrame) {
     var hairMask = FetchNextHairMask();
     var texture = screenController.GetScreen();
 
-    texture.SetPixels32(pixelData.Colors);
+    texture.SetPixels32(textureFrame.GetPixels32());
     RenderAnnotation(screenController, hairMask);
 
     texture.Apply();

diff --git a/Assets/MediaPipe/Examples/Scripts/HandTracking/HandTrackingGraph.cs b/Assets/MediaPipe/Examples/Scripts/HandTracking/HandTrackingGraph.cs
@@ -53,12 +53,12 @@ public override Status StartRun() {
     return graph.StartRun(sidePacket);
   }
 
-  public override void RenderOutput(WebCamScreenController screenController, PixelData pixelData) {
+  public override void RenderOutput(WebCamScreenController screenController, TextureFrame textureFrame) {
     var handTrackingValue = FetchNextHandTrackingValue();
     RenderAnnotation(screenController, handTrackingValue);
 
     var texture = screenController.GetScreen();
-    texture.SetPixels32(pixelData.Colors);
+    texture.SetPixels32(textureFrame.GetPixels32());
 
     texture.Apply();
   }

diff --git a/Assets/MediaPipe/Examples/Scripts/IrisTracking/IrisTrackingGraph.cs b/Assets/MediaPipe/Examples/Scripts/IrisTracking/IrisTrackingGraph.cs
@@ -42,12 +42,12 @@ public override Status StartRun() {
     return graph.StartRun();
   }
 
-  public override void RenderOutput(WebCamScreenController screenController, PixelData pixelData) {
+  public override void RenderOutput(WebCamScreenController screenController, TextureFrame textureFrame) {
     var faceMeshValue = FetchNextIrisTrackingValue();
     RenderAnnotation(screenController, faceMeshValue);
 
     var texture = screenController.GetScreen();
-    texture.SetPixels32(pixelData.Colors);
+    texture.SetPixels32(textureFrame.GetPixels32());
     texture.Apply();
   }
 

diff --git a/Assets/MediaPipe/Examples/Scripts/ObjectDetection/ObjectDetectionGraph.cs b/Assets/MediaPipe/Examples/Scripts/ObjectDetection/ObjectDetectionGraph.cs
@@ -14,12 +14,12 @@ public override Status StartRun() {
     return graph.StartRun();
   }
 
-  public override void RenderOutput(WebCamScreenController screenController, PixelData pixelData) {
+  public override void RenderOutput(WebCamScreenController screenController, TextureFrame textureFrame) {
     var detections = FetchNextOutputDetections();
     RenderAnnotation(screenController, detections);
 
     var texture = screenController.GetScreen();
-    texture.SetPixels32(pixelData.Colors);
+    texture.SetPixels32(textureFrame.GetPixels32());
     texture.Apply();
   }
 

diff --git a/Assets/MediaPipe/Examples/Scripts/PixelData.cs b/Assets/MediaPipe/Examples/Scripts/PixelData.cs
diff --git a/Assets/MediaPipe/Examples/Scripts/PoseTracking/PoseTrackingGraph.cs b/Assets/MediaPipe/Examples/Scripts/PoseTracking/PoseTrackingGraph.cs
@@ -34,12 +34,12 @@ public override Status StartRun() {
     return graph.StartRun();
   }
 
-  public override void RenderOutput(WebCamScreenController screenController, PixelData pixelData) {
+  public override void RenderOutput(WebCamScreenController screenController, TextureFrame textureFrame) {
     var poseTrackingValue = FetchNextPoseTrackingValue();
     RenderAnnotation(screenController, poseTrackingValue);
 
     var texture = screenController.GetScreen();
-    texture.SetPixels32(pixelData.Colors);
+    texture.SetPixels32(textureFrame.GetPixels32());
 
     texture.Apply();
   }

diff --git a/Assets/MediaPipe/Examples/Scripts/ResourceManager/TextureFrame.cs b/Assets/MediaPipe/Examples/Scripts/ResourceManager/TextureFrame.cs
@@ -3,8 +3,6 @@
 using System.Runtime.InteropServices;
 using UnityEngine;
 
-using MpGlSyncToken = System.IntPtr;
-
 public class TextureFrame {
   private Texture2D texture;
   private GCHandle releaseCallbackHandle;
@@ -52,10 +50,9 @@ public GpuBufferFormat gpuBufferformat {
     }
   }
 
-  public void OnRelease(MpGlSyncToken ptr) {
+  public void OnRelease(IntPtr ptr) {
+    var token = new GlSyncPoint(ptr);
     Debug.Log("OnRelease");
-    var glSyncToken = new GlSyncToken(ptr);
-
-    glSyncToken.Wait();
+    token.Wait();
   }
 }
diff --git a/Assets/MediaPipe/Examples/Scripts/SceneDirector.cs b/Assets/MediaPipe/Examples/Scripts/SceneDirector.cs
@@ -11,7 +11,6 @@ public class SceneDirector : MonoBehaviour {
   [SerializeField] bool useGPU = true;
 
   GameObject webCamScreen;
-  WebCamDevice? webCamDevice;
   GameObject graphPrefab;
   GameObject graphContainer;
   Coroutine graphRunner;
@@ -38,8 +37,7 @@ async void Start() {
     webCamScreen = GameObject.Find("WebCamScreen");
 
     if (useGPU) {
-      gpuResources = GpuResources.Create().ConsumeValueOrDie();
-
+      gpuResources = GpuResources.Create(Egl.getCurrentContext()).ConsumeValueOrDie();
       gpuHelper = new GlCalculatorHelper();
       gpuHelper.InitializeForTest(gpuResources);
     }
@@ -66,8 +64,6 @@ void OnDisable() {
   }
 
   public void ChangeWebCamDevice(WebCamDevice? webCamDevice) {
-    this.webCamDevice = webCamDevice;
-
     webCamScreen.GetComponent<WebCamScreenController>().ResetScreen(webCamDevice);
   }
 
@@ -104,7 +100,7 @@ IEnumerator RunGraph() {
       waitFrame--;
 
       var isGraphPrefabPresent = graphPrefab != null;
-      var isWebCamPlaying = webCamScreenController.IsPlaying();
+      var isWebCamPlaying = webCamScreenController.isPlaying;
 
       if (!isGraphPrefabPresent && waitFrame % 10 == 0) {
         Debug.Log($"Waiting for a graph");
@@ -121,8 +117,8 @@ IEnumerator RunGraph() {
       Debug.LogWarning("No graph is set. Stopping...");
       yield break;
     }
-
-    if (!webCamScreenController.IsPlaying()) {
+    
+    if (!webCamScreenController.isPlaying) {
       Debug.LogWarning("WebCamDevice is not working. Stopping...");
       yield break;
     }
@@ -141,7 +137,7 @@ IEnumerator RunGraph() {
     }
 
     graphContainer = Instantiate(graphPrefab);
-    var graph = graphContainer.GetComponent<IDemoGraph<PixelData>>();
+    var graph = graphContainer.GetComponent<IDemoGraph<TextureFrame>>();
 
     if (useGPU) {
       graph.Initialize(gpuResources, gpuHelper);
@@ -154,15 +150,20 @@ IEnumerator RunGraph() {
     while (true) {
       yield return new WaitForEndOfFrame();
 
-      if (!webCamScreenController.IsPlaying()) {
+      if (!webCamScreenController.isPlaying) {
         Debug.LogWarning("WebCam is not working");
         break;
       }
 
-      var pixelData = webCamScreenController.GetPixelData();
+      var nextFrameRequest = webCamScreenController.RequestNextFrame();
+      yield return nextFrameRequest;
+
+      var nextFrame = nextFrameRequest.textureFrame;
+
+      graph.PushInput(nextFrame).AssertOk();
+      graph.RenderOutput(webCamScreenController, nextFrame);
 
-      graph.PushInput(pixelData).AssertOk();
-      graph.RenderOutput(webCamScreenController, pixelData);
+      webCamScreenController.OnReleaseFrame(nextFrame);
     }
   }
 }