fix: PacketPresenceCalculator does not work with PoseTracking

homuler · Aug 27, 2021 · d58ab72 · d58ab72
1 parent b8ea5c1
commit d58ab72
Show file tree

Hide file tree

Showing 6 changed files with 26 additions and 80 deletions.
diff --git a/Assets/Mediapipe/Samples/Graphs/PoseTracking/Resources/pose_tracking_android.txt b/Assets/Mediapipe/Samples/Graphs/PoseTracking/Resources/pose_tracking_android.txt
@@ -17,7 +17,6 @@
 # CHANGES:
 #   - Add ImageTransformationCalculator and rotate the input
 #   - Remove PoseRendererGpu
-#   - Add PacketPresenceCalculators
 
 # MediaPipe graph that performs upper-body pose tracking with TensorFlow Lite on GPU.
 
@@ -26,8 +25,6 @@ input_stream: "input_video"
 output_stream: "pose_landmarks"
 output_stream: "pose_world_landmarks"
 output_stream: "pose_detection"
-output_stream: "pose_landmarks_presence"
-output_stream: "pose_detection_presence"
 
 # Throttles the images flowing downstream for flow control. It passes through
 # the very first incoming image unaltered, and waits for downstream nodes
@@ -42,7 +39,7 @@ output_stream: "pose_detection_presence"
 node {
   calculator: "FlowLimiterCalculator"
   input_stream: "input_video"
-  input_stream: "FINISHED:pose_detection_presence"
+  input_stream: "FINISHED:pose_landmarks"
   input_stream_info: {
     tag_index: "FINISHED"
     back_edge: true
@@ -70,15 +67,3 @@ node {
   output_stream: "DETECTION:pose_detection"
   output_stream: "ROI_FROM_LANDMARKS:roi_from_landmarks"
 }
-
-node {
-  calculator: "PacketPresenceCalculator"
-  input_stream: "PACKET:pose_landmarks"
-  output_stream: "PRESENCE:pose_landmarks_presence"
-}
-
-node {
-  calculator: "PacketPresenceCalculator"
-  input_stream: "PACKET:pose_detection"
-  output_stream: "PRESENCE:pose_detection_presence"
-}
diff --git a/Assets/Mediapipe/Samples/Graphs/PoseTracking/Resources/pose_tracking_desktop_cpu.txt b/Assets/Mediapipe/Samples/Graphs/PoseTracking/Resources/pose_tracking_desktop_cpu.txt
@@ -17,7 +17,6 @@
 # CHANGES:
 #   - Add ImageTransformationCalculator and rotate the input
 #   - Remove PoseRendererCpu
-#   - Add PacketPresenceCalculators
 
 # MediaPipe graph that performs upper-body pose tracking with TensorFlow Lite on CPU.
 
@@ -26,8 +25,6 @@ input_stream: "input_video"
 output_stream: "pose_landmarks"
 output_stream: "pose_world_landmarks"
 output_stream: "pose_detection"
-output_stream: "pose_landmarks_presence"
-output_stream: "pose_detection_presence"
 
 # Throttles the images flowing downstream for flow control. It passes through
 # the very first incoming image unaltered, and waits for downstream nodes
@@ -42,7 +39,7 @@ output_stream: "pose_detection_presence"
 node {
   calculator: "FlowLimiterCalculator"
   input_stream: "input_video"
-  input_stream: "FINISHED:pose_detection_presence"
+  input_stream: "FINISHED:pose_landmarks"
   input_stream_info: {
     tag_index: "FINISHED"
     back_edge: true
@@ -70,15 +67,3 @@ node {
   output_stream: "DETECTION:pose_detection"
   output_stream: "ROI_FROM_LANDMARKS:roi_from_landmarks"
 }
-
-node {
-  calculator: "PacketPresenceCalculator"
-  input_stream: "PACKET:pose_landmarks"
-  output_stream: "PRESENCE:pose_landmarks_presence"
-}
-
-node {
-  calculator: "PacketPresenceCalculator"
-  input_stream: "PACKET:pose_detection"
-  output_stream: "PRESENCE:pose_detection_presence"
-}
diff --git a/Assets/Mediapipe/Samples/Graphs/PoseTracking/Resources/pose_tracking_desktop_gpu.txt b/Assets/Mediapipe/Samples/Graphs/PoseTracking/Resources/pose_tracking_desktop_gpu.txt
@@ -18,7 +18,6 @@
 #   - `input_video` is ImageFrame (ImageFrameToGpuBufferCalculator converts it into GpuBuffer)
 #   - Add ImageTransformationCalculator and rotate the input
 #   - Remove PoseRendererGpu
-#   - Add PacketPresenceCalculators
 
 # MediaPipe graph that performs upper-body pose tracking with TensorFlow Lite on GPU.
 
@@ -27,8 +26,6 @@ input_stream: "input_video"
 output_stream: "pose_landmarks"
 output_stream: "pose_world_landmarks"
 output_stream: "pose_detection"
-output_stream: "pose_landmarks_presence"
-output_stream: "pose_detection_presence"
 
 # Throttles the images flowing downstream for flow control. It passes through
 # the very first incoming image unaltered, and waits for downstream nodes
@@ -43,7 +40,7 @@ output_stream: "pose_detection_presence"
 node {
   calculator: "FlowLimiterCalculator"
   input_stream: "input_video"
-  input_stream: "FINISHED:pose_detection_presence"
+  input_stream: "FINISHED:pose_landmarks"
   input_stream_info: {
     tag_index: "FINISHED"
     back_edge: true
@@ -77,15 +74,3 @@ node {
   output_stream: "DETECTION:pose_detection"
   output_stream: "ROI_FROM_LANDMARKS:roi_from_landmarks"
 }
-
-node {
-  calculator: "PacketPresenceCalculator"
-  input_stream: "PACKET:pose_landmarks"
-  output_stream: "PRESENCE:pose_landmarks_presence"
-}
-
-node {
-  calculator: "PacketPresenceCalculator"
-  input_stream: "PACKET:pose_detection"
-  output_stream: "PRESENCE:pose_detection_presence"
-}
diff --git a/Assets/Mediapipe/Samples/Graphs/PoseTracking/Scripts/PoseTrackingGraph.cs b/Assets/Mediapipe/Samples/Graphs/PoseTracking/Scripts/PoseTrackingGraph.cs
@@ -23,14 +23,6 @@ enum ModelComplexity {
   private OutputStreamPoller<Detection> poseDetectionStreamPoller;
   private DetectionPacket poseDetectionPacket;
 
-  private const string poseLandmarksPresenceStream = "pose_landmarks_presence";
-  private OutputStreamPoller<bool> poseLandmarksPresenceStreamPoller;
-  private BoolPacket poseLandmarksPresencePacket;
-
-  private const string poseDetectionPresenceStream = "pose_detection_presence";
-  private OutputStreamPoller<bool> poseDetectionPresenceStreamPoller;
-  private BoolPacket poseDetectionPresencePacket;
-
   private SidePacket sidePacket;
 
   public override Status StartRun() {
@@ -43,12 +35,6 @@ enum ModelComplexity {
     poseDetectionStreamPoller = graph.AddOutputStreamPoller<Detection>(poseDetectionStream).Value();
     poseDetectionPacket = new DetectionPacket();
 
-    poseLandmarksPresenceStreamPoller = graph.AddOutputStreamPoller<bool>(poseLandmarksPresenceStream).Value();
-    poseLandmarksPresencePacket = new BoolPacket();
-
-    poseDetectionPresenceStreamPoller = graph.AddOutputStreamPoller<bool>(poseDetectionPresenceStream).Value();
-    poseDetectionPresencePacket = new BoolPacket();
-
     sidePacket = new SidePacket();
     sidePacket.Emplace("model_complexity", new IntPacket((int)modelComplexity));
     sidePacket.Emplace("smooth_landmarks", new BoolPacket(smoothLandmarks));
@@ -64,18 +50,16 @@ enum ModelComplexity {
   }
 
   private PoseTrackingValue FetchNextPoseTrackingValue() {
-    if (!FetchNextPoseLandmarksPresence()) {
-      return new PoseTrackingValue();
+    NormalizedLandmarkList poseLandmarks = null;
+    if (poseLandmarksStreamPoller.QueueSize() > 0) {
+      poseLandmarks = FetchNextPoseLandmarks();
     }
 
-    var poseLandmarks = FetchNextPoseLandmarks();
-
-    if (!FetchNextPoseDetectionPresence()) {
-      return new PoseTrackingValue(poseLandmarks);
+    Detection poseDetection = null;
+    if (poseDetectionStreamPoller.QueueSize() > 0) {
+      poseDetection = FetchNextPoseDetection();
     }
 
-    var poseDetection = FetchNextPoseDetection();
-
     return new PoseTrackingValue(poseLandmarks, poseDetection);
   }
 
@@ -91,14 +75,6 @@ enum ModelComplexity {
     return FetchNext(poseDetectionStreamPoller, poseDetectionPacket, poseDetectionStream);
   }
 
-  private bool FetchNextPoseLandmarksPresence() {
-    return FetchNext(poseLandmarksPresenceStreamPoller, poseLandmarksPresencePacket, poseLandmarksPresenceStream);
-  }
-
-  private bool FetchNextPoseDetectionPresence() {
-    return FetchNext(poseDetectionPresenceStreamPoller, poseDetectionPresencePacket, poseDetectionPresenceStream);
-  }
-
   private void RenderAnnotation(WebCamScreenController screenController, PoseTrackingValue value) {
     // NOTE: input image is flipped
     GetComponent<PoseTrackingAnnotationController>().Draw(screenController.transform, value.PoseLandmarkList, value.PoseDetection, true);

diff --git a/Assets/Mediapipe/Samples/Graphs/PoseTracking/Scripts/PoseTrackingValue.cs b/Assets/Mediapipe/Samples/Graphs/PoseTracking/Scripts/PoseTrackingValue.cs
@@ -5,8 +5,8 @@ class PoseTrackingValue {
   public readonly Detection PoseDetection;
 
   public PoseTrackingValue(NormalizedLandmarkList landmarkList, Detection detection) {
-    PoseLandmarkList = landmarkList;
-    PoseDetection = detection;
+    PoseLandmarkList = landmarkList == null ? new NormalizedLandmarkList() : landmarkList;
+    PoseDetection = detection == null ? new Detection() : detection;
   }
 
   public PoseTrackingValue(NormalizedLandmarkList landmarkList) : this(landmarkList, new Detection()) {}

diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/OutputStreamPoller.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/OutputStreamPoller.cs
@@ -14,5 +14,20 @@ public class OutputStreamPoller<T> : MpResourceHandle {
       GC.KeepAlive(this);
       return result;
     }
+
+    public void Reset() {
+      UnsafeNativeMethods.mp_OutputStreamPoller__Reset(mpPtr).Assert();
+    }
+
+    public void SetMaxQueueSize(int queueSize) {
+      UnsafeNativeMethods.mp_OutputStreamPoller__SetMaxQueueSize(mpPtr, queueSize).Assert();
+    }
+
+    public int QueueSize() {
+      UnsafeNativeMethods.mp_OutputStreamPoller__QueueSize(mpPtr, out var result).Assert();
+
+      GC.KeepAlive(this);
+      return result;
+    }
   }
 }