From abbade082cbed35209f5c28ca5790f6c59c5e045 Mon Sep 17 00:00:00 2001 From: homuler Date: Sun, 11 Jul 2021 17:03:47 +0900 Subject: [PATCH] feat(example): enable to get pose world landmarks --- .../Holistic/Resources/holistic_android.txt | 2 ++ .../Resources/holistic_desktop_cpu.txt | 2 ++ .../Resources/holistic_desktop_gpu.txt | 2 ++ .../Graphs/Holistic/Scripts/HolisticGraph.cs | 11 +++++++++ .../Resources/pose_tracking_android.txt | 2 ++ .../Resources/pose_tracking_desktop_cpu.txt | 2 ++ .../Resources/pose_tracking_desktop_gpu.txt | 2 ++ .../PoseTracking/Scripts/PoseTrackingGraph.cs | 11 +++++++++ .../Framework/Packet/LandmarkListPacket.cs | 22 ++++++++++++++++++ .../Packet/LandmarkListPacket.cs.meta | 11 +++++++++ .../Packet/LandmarkListVectorPacket.cs | 23 +++++++++++++++++++ .../Packet/LandmarkListVectorPacket.cs.meta | 11 +++++++++ 12 files changed, 101 insertions(+) create mode 100644 Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/LandmarkListPacket.cs create mode 100644 Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/LandmarkListPacket.cs.meta create mode 100644 Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/LandmarkListVectorPacket.cs create mode 100644 Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/LandmarkListVectorPacket.cs.meta diff --git a/Assets/Mediapipe/Samples/Graphs/Holistic/Resources/holistic_android.txt b/Assets/Mediapipe/Samples/Graphs/Holistic/Resources/holistic_android.txt index 4072cb947..8250fa424 100644 --- a/Assets/Mediapipe/Samples/Graphs/Holistic/Resources/holistic_android.txt +++ b/Assets/Mediapipe/Samples/Graphs/Holistic/Resources/holistic_android.txt @@ -29,6 +29,7 @@ input_stream: "input_video" input_side_packet: "enable_iris_detection" output_stream: "pose_landmarks" +output_stream: "pose_world_landmarks" output_stream: "pose_roi" output_stream: "pose_detection" output_stream: "face_landmarks" @@ -88,6 +89,7 @@ node { calculator: "HolisticLandmarkGpu" input_stream: "IMAGE:transformed_input_video" output_stream: "POSE_LANDMARKS:pose_landmarks" + output_stream: "WORLD_LANDMARKS:pose_world_landmarks" output_stream: "POSE_ROI:pose_roi" output_stream: "POSE_DETECTION:pose_detection" output_stream: "FACE_LANDMARKS:face_landmarks_orig" diff --git a/Assets/Mediapipe/Samples/Graphs/Holistic/Resources/holistic_desktop_cpu.txt b/Assets/Mediapipe/Samples/Graphs/Holistic/Resources/holistic_desktop_cpu.txt index 35abba100..f7bd0f21a 100644 --- a/Assets/Mediapipe/Samples/Graphs/Holistic/Resources/holistic_desktop_cpu.txt +++ b/Assets/Mediapipe/Samples/Graphs/Holistic/Resources/holistic_desktop_cpu.txt @@ -29,6 +29,7 @@ input_stream: "input_video" input_side_packet: "enable_iris_detection" output_stream: "pose_landmarks" +output_stream: "pose_world_landmarks" output_stream: "pose_roi" output_stream: "pose_detection" output_stream: "face_landmarks" @@ -88,6 +89,7 @@ node { calculator: "HolisticLandmarkCpu" input_stream: "IMAGE:transformed_input_video" output_stream: "POSE_LANDMARKS:pose_landmarks" + output_stream: "WORLD_LANDMARKS:pose_world_landmarks" output_stream: "POSE_ROI:pose_roi" output_stream: "POSE_DETECTION:pose_detection" output_stream: "FACE_LANDMARKS:face_landmarks_orig" diff --git a/Assets/Mediapipe/Samples/Graphs/Holistic/Resources/holistic_desktop_gpu.txt b/Assets/Mediapipe/Samples/Graphs/Holistic/Resources/holistic_desktop_gpu.txt index 7d6397762..d4f0adf46 100644 --- a/Assets/Mediapipe/Samples/Graphs/Holistic/Resources/holistic_desktop_gpu.txt +++ b/Assets/Mediapipe/Samples/Graphs/Holistic/Resources/holistic_desktop_gpu.txt @@ -30,6 +30,7 @@ input_stream: "input_video" input_side_packet: "enable_iris_detection" output_stream: "pose_landmarks" +output_stream: "pose_world_landmarks" output_stream: "pose_roi" output_stream: "pose_detection" output_stream: "face_landmarks" @@ -95,6 +96,7 @@ node { calculator: "HolisticLandmarkGpu" input_stream: "IMAGE:transformed_input_video" output_stream: "POSE_LANDMARKS:pose_landmarks" + output_stream: "WORLD_LANDMARKS:pose_world_landmarks" output_stream: "POSE_ROI:pose_roi" output_stream: "POSE_DETECTION:pose_detection" output_stream: "FACE_LANDMARKS:face_landmarks_orig" diff --git a/Assets/Mediapipe/Samples/Graphs/Holistic/Scripts/HolisticGraph.cs b/Assets/Mediapipe/Samples/Graphs/Holistic/Scripts/HolisticGraph.cs index 98100cf5c..ee2c3b2c1 100644 --- a/Assets/Mediapipe/Samples/Graphs/Holistic/Scripts/HolisticGraph.cs +++ b/Assets/Mediapipe/Samples/Graphs/Holistic/Scripts/HolisticGraph.cs @@ -16,6 +16,10 @@ enum ModelComplexity { private OutputStreamPoller poseLandmarksStreamPoller; private NormalizedLandmarkListPacket poseLandmarksPacket; + private const string poseWorldLandmarksStream = "pose_world_landmarks"; + private OutputStreamPoller poseWorldLandmarksStreamPoller; + private LandmarkListPacket poseWorldLandmarksPacket; + private const string poseRoiStream = "pose_roi"; private OutputStreamPoller poseRoiStreamPoller; private NormalizedRectPacket poseRoiPacket; @@ -66,6 +70,9 @@ public override Status StartRun() { poseLandmarksStreamPoller = graph.AddOutputStreamPoller(poseLandmarksStream).Value(); poseLandmarksPacket = new NormalizedLandmarkListPacket(); + poseWorldLandmarksStreamPoller = graph.AddOutputStreamPoller(poseWorldLandmarksStream).Value(); + poseWorldLandmarksPacket = new LandmarkListPacket(); + poseRoiStreamPoller = graph.AddOutputStreamPoller(poseRoiStream).Value(); poseRoiPacket = new NormalizedRectPacket(); @@ -136,6 +143,10 @@ private NormalizedLandmarkList FetchNextPoseLandmarks() { return FetchNext(poseLandmarksStreamPoller, poseLandmarksPacket, poseLandmarksStream); } + private LandmarkList FetchNextPoseWorldLandmarks() { + return FetchNext(poseWorldLandmarksStreamPoller, poseWorldLandmarksPacket, poseWorldLandmarksStream); + } + private NormalizedRect FetchNextPoseRoi() { return FetchNext(poseRoiStreamPoller, poseRoiPacket, poseRoiStream); } diff --git a/Assets/Mediapipe/Samples/Graphs/PoseTracking/Resources/pose_tracking_android.txt b/Assets/Mediapipe/Samples/Graphs/PoseTracking/Resources/pose_tracking_android.txt index 049869796..4e3b3445d 100644 --- a/Assets/Mediapipe/Samples/Graphs/PoseTracking/Resources/pose_tracking_android.txt +++ b/Assets/Mediapipe/Samples/Graphs/PoseTracking/Resources/pose_tracking_android.txt @@ -24,6 +24,7 @@ input_stream: "input_video" output_stream: "pose_landmarks" +output_stream: "pose_world_landmarks" output_stream: "pose_detection" output_stream: "pose_landmarks_presence" output_stream: "pose_detection_presence" @@ -65,6 +66,7 @@ node { calculator: "PoseLandmarkGpu" input_stream: "IMAGE:transformed_input_video" output_stream: "LANDMARKS:pose_landmarks" + output_stream: "WORLD_LANDMARKS:pose_world_landmarks" output_stream: "DETECTION:pose_detection" output_stream: "ROI_FROM_LANDMARKS:roi_from_landmarks" } diff --git a/Assets/Mediapipe/Samples/Graphs/PoseTracking/Resources/pose_tracking_desktop_cpu.txt b/Assets/Mediapipe/Samples/Graphs/PoseTracking/Resources/pose_tracking_desktop_cpu.txt index 5d0dd3228..a2ce2b052 100644 --- a/Assets/Mediapipe/Samples/Graphs/PoseTracking/Resources/pose_tracking_desktop_cpu.txt +++ b/Assets/Mediapipe/Samples/Graphs/PoseTracking/Resources/pose_tracking_desktop_cpu.txt @@ -24,6 +24,7 @@ input_stream: "input_video" output_stream: "pose_landmarks" +output_stream: "pose_world_landmarks" output_stream: "pose_detection" output_stream: "pose_landmarks_presence" output_stream: "pose_detection_presence" @@ -65,6 +66,7 @@ node { calculator: "PoseLandmarkCpu" input_stream: "IMAGE:transformed_input_video" output_stream: "LANDMARKS:pose_landmarks" + output_stream: "WORLD_LANDMARKS:pose_world_landmarks" output_stream: "DETECTION:pose_detection" output_stream: "ROI_FROM_LANDMARKS:roi_from_landmarks" } diff --git a/Assets/Mediapipe/Samples/Graphs/PoseTracking/Resources/pose_tracking_desktop_gpu.txt b/Assets/Mediapipe/Samples/Graphs/PoseTracking/Resources/pose_tracking_desktop_gpu.txt index bfe384010..e673a13f6 100644 --- a/Assets/Mediapipe/Samples/Graphs/PoseTracking/Resources/pose_tracking_desktop_gpu.txt +++ b/Assets/Mediapipe/Samples/Graphs/PoseTracking/Resources/pose_tracking_desktop_gpu.txt @@ -25,6 +25,7 @@ input_stream: "input_video" output_stream: "pose_landmarks" +output_stream: "pose_world_landmarks" output_stream: "pose_detection" output_stream: "pose_landmarks_presence" output_stream: "pose_detection_presence" @@ -72,6 +73,7 @@ node { calculator: "PoseLandmarkGpu" input_stream: "IMAGE:transformed_input_video" output_stream: "LANDMARKS:pose_landmarks" + output_stream: "WORLD_LANDMARKS:pose_world_landmarks" output_stream: "DETECTION:pose_detection" output_stream: "ROI_FROM_LANDMARKS:roi_from_landmarks" } diff --git a/Assets/Mediapipe/Samples/Graphs/PoseTracking/Scripts/PoseTrackingGraph.cs b/Assets/Mediapipe/Samples/Graphs/PoseTracking/Scripts/PoseTrackingGraph.cs index 4c6fb5771..20030ac03 100644 --- a/Assets/Mediapipe/Samples/Graphs/PoseTracking/Scripts/PoseTrackingGraph.cs +++ b/Assets/Mediapipe/Samples/Graphs/PoseTracking/Scripts/PoseTrackingGraph.cs @@ -15,6 +15,10 @@ enum ModelComplexity { private OutputStreamPoller poseLandmarksStreamPoller; private NormalizedLandmarkListPacket poseLandmarksPacket; + private const string poseWorldLandmarksStream = "pose_world_landmarks"; + private OutputStreamPoller poseWorldLandmarksStreamPoller; + private LandmarkListPacket poseWorldLandmarksPacket; + private const string poseDetectionStream = "pose_detection"; private OutputStreamPoller poseDetectionStreamPoller; private DetectionPacket poseDetectionPacket; @@ -33,6 +37,9 @@ public override Status StartRun() { poseLandmarksStreamPoller = graph.AddOutputStreamPoller(poseLandmarksStream).Value(); poseLandmarksPacket = new NormalizedLandmarkListPacket(); + poseWorldLandmarksStreamPoller = graph.AddOutputStreamPoller(poseWorldLandmarksStream).Value(); + poseWorldLandmarksPacket = new LandmarkListPacket(); + poseDetectionStreamPoller = graph.AddOutputStreamPoller(poseDetectionStream).Value(); poseDetectionPacket = new DetectionPacket(); @@ -76,6 +83,10 @@ private NormalizedLandmarkList FetchNextPoseLandmarks() { return FetchNext(poseLandmarksStreamPoller, poseLandmarksPacket, poseLandmarksStream); } + private LandmarkList FetchNextPoseWorldLandmarks() { + return FetchNext(poseWorldLandmarksStreamPoller, poseWorldLandmarksPacket, poseWorldLandmarksStream); + } + private Detection FetchNextPoseDetection() { return FetchNext(poseDetectionStreamPoller, poseDetectionPacket, poseDetectionStream); } diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/LandmarkListPacket.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/LandmarkListPacket.cs new file mode 100644 index 000000000..f92493620 --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/LandmarkListPacket.cs @@ -0,0 +1,22 @@ +using System; + +namespace Mediapipe { + public class LandmarkListPacket : Packet { + public LandmarkListPacket() : base() {} + public LandmarkListPacket(IntPtr ptr, bool isOwner = true) : base(ptr, isOwner) {} + + public override LandmarkList Get() { + UnsafeNativeMethods.mp_Packet__GetLandmarkList(mpPtr, out var serializedProtoPtr).Assert(); + GC.KeepAlive(this); + + var landmarkList = Protobuf.DeserializeProto(serializedProtoPtr, LandmarkList.Parser); + UnsafeNativeMethods.mp_api_SerializedProto__delete(serializedProtoPtr); + + return landmarkList; + } + + public override StatusOr Consume() { + throw new NotSupportedException(); + } + } +} diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/LandmarkListPacket.cs.meta b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/LandmarkListPacket.cs.meta new file mode 100644 index 000000000..5cf7140cd --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/LandmarkListPacket.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: a52c4850b3d6f4fff840b03da689e307 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/LandmarkListVectorPacket.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/LandmarkListVectorPacket.cs new file mode 100644 index 000000000..b0b73fe13 --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/LandmarkListVectorPacket.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; + +namespace Mediapipe { + public class LandmarkListVectorPacket : Packet> { + public LandmarkListVectorPacket() : base() {} + public LandmarkListVectorPacket(IntPtr ptr, bool isOwner = true) : base(ptr, isOwner) {} + + public override List Get() { + UnsafeNativeMethods.mp_Packet__GetLandmarkListVector(mpPtr, out var serializedProtoVectorPtr).Assert(); + GC.KeepAlive(this); + + var landmarkLists = Protobuf.DeserializeProtoVector(serializedProtoVectorPtr, LandmarkList.Parser); + UnsafeNativeMethods.mp_api_SerializedProtoVector__delete(serializedProtoVectorPtr); + + return landmarkLists; + } + + public override StatusOr> Consume() { + throw new NotSupportedException(); + } + } +} diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/LandmarkListVectorPacket.cs.meta b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/LandmarkListVectorPacket.cs.meta new file mode 100644 index 000000000..a0f1bea4b --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/LandmarkListVectorPacket.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 5221b835adb8927dc84cb8c5f3af2f8d +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: