feat: marshal NormalizedLandmarks (#1088)

* feat: marshal NormalizedLandmarks directly

* add NativeLandmarks

Packages/com.github.homuler.mediapipe/Runtime/Scripts/Extension/ListExtension.cs

@@ -0,0 +1,27 @@
+// Copyright (c) 2023 homuler
+//
+// Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file or at
+// https://opensource.org/licenses/MIT.
+
+using System.Collections.Generic;
+
+namespace Mediapipe
+{
+  internal static class ListExtension
+  {
+    public static void ResizeTo<T>(this List<T> list, int size)
+    {
+      if (list.Count > size)
+      {
+        list.RemoveRange(size, list.Count - size);
+      }
+
+      var count = size - list.Count;
+      for (var i = 0; i < count; i++)
+      {
+        list.Add(default);
+      }
+    }
+  }
+}

Packages/com.github.homuler.mediapipe/Runtime/Scripts/Marshal/NativeLandmark.cs

@@ -0,0 +1,101 @@
+// Copyright (c) 2023 homuler
+//
+// Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file or at
+// https://opensource.org/licenses/MIT.
+
+using System;
+using System.Runtime.InteropServices;
+
+namespace Mediapipe
+{
+  [StructLayout(LayoutKind.Sequential)]
+  internal readonly struct NativeLandmark
+  {
+    public readonly float x;
+    public readonly float y;
+    public readonly float z;
+
+    [MarshalAs(UnmanagedType.I1)]
+    public readonly bool hasVisibility;
+    public readonly float visibility;
+
+    [MarshalAs(UnmanagedType.I1)]
+    public readonly bool hasPresence;
+    public readonly float presence;
+
+    private readonly IntPtr _name;
+
+    public string name => Marshal.PtrToStringAnsi(_name);
+  }
+
+  [StructLayout(LayoutKind.Sequential)]
+  internal readonly struct NativeNormalizedLandmark
+  {
+    public readonly float x;
+    public readonly float y;
+    public readonly float z;
+
+    [MarshalAs(UnmanagedType.I1)]
+    public readonly bool hasVisibility;
+    public readonly float visibility;
+
+    [MarshalAs(UnmanagedType.I1)]
+    public readonly bool hasPresence;
+    public readonly float presence;
+
+    private readonly IntPtr _name;
+
+    public string name => Marshal.PtrToStringAnsi(_name);
+  }
+
+  [StructLayout(LayoutKind.Sequential)]
+  internal readonly struct NativeLandmarks
+  {
+    private readonly IntPtr _landmarks;
+    public readonly uint landmarksCount;
+
+    public ReadOnlySpan<NativeLandmark> AsReadOnlySpan()
+    {
+      unsafe
+      {
+        return new ReadOnlySpan<NativeLandmark>((NativeLandmark*)_landmarks, (int)landmarksCount);
+      }
+    }
+  }
+
+  [StructLayout(LayoutKind.Sequential)]
+  internal readonly struct NativeNormalizedLandmarks
+  {
+    private readonly IntPtr _landmarks;
+    public readonly uint landmarksCount;
+
+    public ReadOnlySpan<NativeNormalizedLandmark> AsReadOnlySpan()
+    {
+      unsafe
+      {
+        return new ReadOnlySpan<NativeNormalizedLandmark>((NativeNormalizedLandmark*)_landmarks, (int)landmarksCount);
+      }
+    }
+  }
+
+  [StructLayout(LayoutKind.Sequential)]
+  internal readonly struct NativeNormalizedLandmarksArray
+  {
+    private readonly IntPtr _data;
+    public readonly int size;
+
+    public void Dispose()
+    {
+      UnsafeNativeMethods.mp_api_NormalizedLandmarksArray__delete(_data, size);
+    }
+
+    public ReadOnlySpan<NativeNormalizedLandmarks> AsReadOnlySpan()
+    {
+      unsafe
+      {
+        return new ReadOnlySpan<NativeNormalizedLandmarks>((NativeNormalizedLandmarks*)_data, size);
+      }
+    }
+  }
+}

Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Tasks/Components/Containers_Unsafe.cs

@@ -0,0 +1,20 @@
+// Copyright (c) 2023 homuler
+//
+// Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file or at
+// https://opensource.org/licenses/MIT.
+
+using System;
+using System.Runtime.InteropServices;
+
+namespace Mediapipe
+{
+  internal static partial class UnsafeNativeMethods
+  {
+    [DllImport(MediaPipeLibrary, ExactSpelling = true)]
+    public static extern MpReturnCode mp_Packet__GetNormalizedLandmarksVector(IntPtr packet, out NativeNormalizedLandmarksArray value);
+
+    [DllImport(MediaPipeLibrary, ExactSpelling = true)]
+    public static extern void mp_api_NormalizedLandmarksArray__delete(IntPtr data, int size);
+  }
+}

Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/Landmark.cs

@@ -6,6 +6,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Runtime.InteropServices;
 
 // TODO: use System.MathF
 using Mathf = UnityEngine.Mathf;
@@ -105,6 +106,17 @@ public override string ToString()
     public readonly float? presence;
     public readonly string name;
 
+    internal NormalizedLandmark(NativeNormalizedLandmark nativeLandmark) : this(
+      nativeLandmark.x, nativeLandmark.y, nativeLandmark.z,
+#pragma warning disable IDE0004 // for Unity 2020.3.x
+      nativeLandmark.hasVisibility ? (float?)nativeLandmark.visibility : null,
+      nativeLandmark.hasPresence ? (float?)nativeLandmark.presence : null,
+#pragma warning restore IDE0004 // for Unity 2020.3.x
+      nativeLandmark.name
+    )
+    {
+    }
+
     internal NormalizedLandmark(float x, float y, float z, float? visibility, float? presence) : this(x, y, z, visibility, presence, null)
     {
     }
@@ -180,9 +192,9 @@ public static Landmarks CreateFrom(LandmarkList proto)
   /// </summary>
   public readonly struct NormalizedLandmarks
   {
-    public readonly IReadOnlyList<NormalizedLandmark> landmarks;
+    public readonly List<NormalizedLandmark> landmarks;
 
-    internal NormalizedLandmarks(IReadOnlyList<NormalizedLandmark> landmarks)
+    internal NormalizedLandmarks(List<NormalizedLandmark> landmarks)
     {
       this.landmarks = landmarks;
     }
@@ -197,6 +209,34 @@ public static NormalizedLandmarks CreateFrom(NormalizedLandmarkList proto)
       return new NormalizedLandmarks(landmarks);
     }
 
+    internal static void Copy(NativeNormalizedLandmarks source, ref NormalizedLandmarks destination)
+    {
+      var landmarks = destination.landmarks ?? new List<NormalizedLandmark>((int)source.landmarksCount);
+      landmarks.Clear();
+
+      foreach (var nativeLandmark in source.AsReadOnlySpan())
+      {
+        landmarks.Add(new NormalizedLandmark(nativeLandmark));
+      }
+      destination = new NormalizedLandmarks(landmarks);
+    }
+
     public override string ToString() => $"{{ \"landmarks\": {Util.Format(landmarks)} }}";
   }
+
+  internal static class NativeNormalizedLandmarksArrayExtension
+  {
+    public static void FillWith(this List<NormalizedLandmarks> target, NativeNormalizedLandmarksArray source)
+    {
+      target.ResizeTo(source.size);
+
+      var i = 0;
+      foreach (var nativeLandmarks in source.AsReadOnlySpan())
+      {
+        var landmarks = target[i];
+        NormalizedLandmarks.Copy(nativeLandmarks, ref landmarks);
+        target[i] = landmarks;
+      }
+    }
+  }
 }

Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/PacketExtension.cs

@@ -0,0 +1,20 @@
+// Copyright (c) 2023 homuler
+//
+// Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file or at
+// https://opensource.org/licenses/MIT.
+
+using System.Collections.Generic;
+
+namespace Mediapipe.Tasks.Components.Containers
+{
+  public static class PacketExtension
+  {
+    public static void GetNormalizedLandmarksList(this Packet packet, List<NormalizedLandmarks> outs)
+    {
+      UnsafeNativeMethods.mp_Packet__GetNormalizedLandmarksVector(packet.mpPtr, out var landmarksArray).Assert();
+      outs.FillWith(landmarksArray);
+      landmarksArray.Dispose();
+    }
+  }
+}

mediapipe_api/BUILD

@@ -164,6 +164,7 @@ cc_library(
         "//mediapipe_api/framework/formats:matrix_data",
         "//mediapipe_api/framework/formats:rect",
         "//mediapipe_api/framework/port:logging",
+        "//mediapipe_api/tasks/c/components/containers:landmark",
         "//mediapipe_api/tasks/cc/vision/face_geometry/proto:face_geometry",
         "//mediapipe_api/tasks/cc/core:task_runner",
         "//mediapipe_api/util:resource_util",

mediapipe_api/tasks/c/components/containers/BUILD

@@ -0,0 +1,39 @@
+# Copyright (c) 2023 homuler
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+
+package(
+    default_visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "detection_result",
+    srcs = ["detection_result.cc"],
+    hdrs = ["detection_result.h"],
+    deps = [
+        "//mediapipe_api:common",
+        "@com_google_mediapipe//mediapipe/framework:packet",
+        "@com_google_mediapipe//mediapipe/framework/formats:detection_cc_proto",
+        "@com_google_mediapipe//mediapipe/tasks/c/components/containers:detection_result",
+        "@com_google_mediapipe//mediapipe/tasks/c/components/containers:detection_result_converter",
+        "@com_google_mediapipe//mediapipe/tasks/cc/components/containers:detection_result",
+    ],
+    alwayslink = True,
+)
+
+cc_library(
+    name = "landmark",
+    srcs = ["landmark.cc"],
+    hdrs = ["landmark.h"],
+    deps = [
+        "//mediapipe_api:common",
+        "@com_google_mediapipe//mediapipe/framework:packet",
+        "@com_google_mediapipe//mediapipe/framework/formats:landmark_cc_proto",
+        "@com_google_mediapipe//mediapipe/tasks/c/components/containers:landmark",
+        "@com_google_mediapipe//mediapipe/tasks/c/components/containers:landmark_converter",
+        "@com_google_mediapipe//mediapipe/tasks/cc/components/containers:landmark",
+    ],
+    alwayslink = True,
+)

mediapipe_api/tasks/c/components/containers/landmark.cc

@@ -0,0 +1,27 @@
+#include "mediapipe_api/tasks/c/components/containers/landmark.h"
+
+MpReturnCode mp_Packet__GetNormalizedLandmarksVector(mediapipe::Packet* packet, mp_api::StructArray<NormalizedLandmarks>* value_out) {
+  TRY_ALL
+    // get std::vector<NormalizedLandmarkList> and convert it to NormalizedLandmarks*
+    auto proto_vec = packet->Get<std::vector<mediapipe::NormalizedLandmarkList>>();
+    auto vec_size = proto_vec.size();
+    auto data = new NormalizedLandmarks[vec_size];
+
+    for (auto i = 0; i < vec_size; ++i) {
+      auto landmarks = mediapipe::tasks::components::containers::ConvertToNormalizedLandmarks(proto_vec[i]);
+      mediapipe::tasks::c::components::containers::CppConvertToNormalizedLandmarks(landmarks.landmarks, &data[i]);
+    }
+
+    value_out->data = data;
+    value_out->size = static_cast<int>(vec_size);
+    RETURN_CODE(MpReturnCode::Success);
+  CATCH_ALL
+}
+
+void mp_api_NormalizedLandmarksArray__delete(NormalizedLandmarks* data, int size) {
+  auto landmarks = data;
+  for (auto i = 0; i < size; ++i) {
+    mediapipe::tasks::c::components::containers::CppCloseNormalizedLandmarks(landmarks++);
+  }
+  delete[] data;
+}