feat(example): render external textures directly

homuler · Dec 28, 2020 · ebbf0ed · ebbf0ed
1 parent 1643620
commit ebbf0ed
Show file tree

Hide file tree

Showing 3 changed files with 41 additions and 53 deletions.
diff --git a/Assets/MediaPipe/Examples/Scripts/OfficialDemo/OfficialDemoCPU.cs b/Assets/MediaPipe/Examples/Scripts/OfficialDemo/OfficialDemoCPU.cs
@@ -31,11 +31,11 @@ public class OfficialDemoCPU : DemoGraph {
 
   public override void RenderOutput(WebCamScreenController screenController, TextureFrame textureFrame) {
     lock (outputImageLock) {
-      if (outputImage != null) {
-        screenController.DrawScreen(outputImage);
-        outputImage.Dispose();
-        outputImage = null;
-      }
+      if (outputImage == null) { return; }
+
+      screenController.DrawScreen(outputImage);
+      outputImage.Dispose();
+      outputImage = null;
     }
   }
 

diff --git a/Assets/MediaPipe/Examples/Scripts/OfficialDemo/OfficialDemoGPU.cs b/Assets/MediaPipe/Examples/Scripts/OfficialDemo/OfficialDemoGPU.cs
@@ -5,16 +5,16 @@
 public class OfficialDemoGPU : DemoGraph {
   private const string outputStream = "output_video";
 
-  private readonly object outputImageLock = new object();
-  private GpuBuffer outputImage;
-  private GCHandle outputVideoCallbackHandle;
+  private OutputStreamPoller<GpuBuffer> outputStreamPoller;
+  private GpuBufferPacket outputPacket;
 
   private SidePacket sidePacket;
 
   public override Status StartRun() {
     Debug.LogWarning("This graph is for testing official examples. You can customize the graph by editing `official_demo_gpu.txt` (default is `hand_tracking_mobile.pbtxt`)");
 
-    graph.ObserveOutputStream<GpuBufferPacket, GpuBuffer>(outputStream, OutputVideoCallback, out outputVideoCallbackHandle).AssertOk();
+    outputStreamPoller = graph.AddOutputStreamPoller<GpuBuffer>(outputStream).ConsumeValueOrDie();
+    outputPacket = new GpuBufferPacket();
 
     sidePacket = new SidePacket();
     sidePacket.Emplace("num_hands", new IntPacket(2));
@@ -23,58 +23,41 @@ public class OfficialDemoGPU : DemoGraph {
   }
 
   public override void RenderOutput(WebCamScreenController screenController, TextureFrame textureFrame) {
-    ImageFrame imageFrame = null;
-
-    lock (outputImageLock) {
-      if (outputImage == null) { return; }
-
-      var status = gpuHelper.RunInGlContext(() => {
-        var gpuBufferFormat = outputImage.Format();
-        var sourceTexture = gpuHelper.CreateSourceTexture(outputImage);
+    if (!outputStreamPoller.Next(outputPacket)) {
+      Debug.LogWarning("Failed to fetch an output packet, rendering the input image");
+      screenController.DrawScreen(textureFrame);
+      return;
+    }
 
-        imageFrame = new ImageFrame(
-          gpuBufferFormat.ImageFormatFor(), outputImage.Width(), outputImage.Height(), ImageFrame.kGlDefaultAlignmentBoundary);
+    using (var gpuBuffer = outputPacket.Get()) {
+      #if UNITY_ANDROID
+        // OpenGL ES
+        screenController.DrawScreen(gpuBuffer);
+      #else
+        ImageFrame imageFrame = null;
 
-        gpuHelper.BindFramebuffer(sourceTexture);
-        var info = gpuBufferFormat.GlTextureInfoFor(0);
+        gpuHelper.RunInGlContext(() => {
+          var gpuBufferFormat = gpuBuffer.Format();
+          var sourceTexture = gpuHelper.CreateSourceTexture(gpuBuffer);
 
-        Gl.ReadPixels(0, 0, sourceTexture.width, sourceTexture.height, info.glFormat, info.glType, imageFrame.MutablePixelData());
-        Gl.Flush();
+          imageFrame = new ImageFrame(
+            gpuBufferFormat.ImageFormatFor(), gpuBuffer.Width(), gpuBuffer.Height(), ImageFrame.kGlDefaultAlignmentBoundary);
 
-        sourceTexture.Release();
+          gpuHelper.BindFramebuffer(sourceTexture);
+          var info = gpuBufferFormat.GlTextureInfoFor(0);
 
-        return Status.Ok(false);
-      });
+          Gl.ReadPixels(0, 0, sourceTexture.width, sourceTexture.height, info.glFormat, info.glType, imageFrame.MutablePixelData());
+          Gl.Flush();
 
-      outputImage.Dispose();
-      outputImage = null;
+          sourceTexture.Release();
 
-      if (!status.ok) {
-        Debug.LogError(status.ToString());
-        return;
-      }
-    }
+          return Status.Ok(false);
+        }).AssertOk();
 
-    if (imageFrame != null) { /// always true
-      screenController.DrawScreen(imageFrame);
-    }
-  }
-
-  private Status OutputVideoCallback(GpuBufferPacket packet) {
-    return gpuHelper.RunInGlContext(() => {
-      var statusOrGpuBuffer = packet.Consume();
-
-      if (statusOrGpuBuffer.ok) {
-        lock (outputImageLock) {
-          if (outputImage != null) {
-            outputImage.Dispose();
-          }
-
-          outputImage = statusOrGpuBuffer.ConsumeValueOrDie();
+        if (imageFrame != null) { // always true
+          screenController.DrawScreen(imageFrame);
         }
-      }
-
-      return Status.Ok();
-    });
+      #endif
+    }
   }
 }
diff --git a/Assets/MediaPipe/Examples/Scripts/WebCamScreenController.cs b/Assets/MediaPipe/Examples/Scripts/WebCamScreenController.cs
@@ -91,6 +91,11 @@ public class WebCamScreenController : MonoBehaviour {
     outputTexture.Apply();
   }
 
+  public void DrawScreen(GpuBuffer gpuBuffer) {
+    // TODO: create an external texture
+    outputTexture.UpdateExternalTexture((IntPtr)gpuBuffer.GetGlTextureBuffer().Name());
+  }
+
   public TextureFramePool.TextureFrameRequest RequestNextFrame() {
     return WebCamTextureFramePool.Instance.RequestNextTextureFrame((TextureFrame textureFrame) => {
       textureFrame.CopyTextureFrom(webCamTexture);