diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Protobuf/Tasks/Components.meta b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Protobuf/Tasks/Components.meta new file mode 100644 index 000000000..74e090c0e --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Protobuf/Tasks/Components.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 50ccf9d7e48df805e955c2dc12872777 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Protobuf/Tasks/Components/Containers.meta b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Protobuf/Tasks/Components/Containers.meta new file mode 100644 index 000000000..c55ae193b --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Protobuf/Tasks/Components/Containers.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: e2d6e04f9469d6adfbea445760363119 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Protobuf/Tasks/Components/Containers/Proto.meta b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Protobuf/Tasks/Components/Containers/Proto.meta new file mode 100644 index 000000000..08ef3a551 --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Protobuf/Tasks/Components/Containers/Proto.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 71b03f61e580a90a4a979f0feaf38faf +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Protobuf/Tasks/Components/Containers/Proto/Classifications.cs.meta b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Protobuf/Tasks/Components/Containers/Proto/Classifications.cs.meta new file mode 100644 index 000000000..71e149dea --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Protobuf/Tasks/Components/Containers/Proto/Classifications.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 5583dd4786615740b99787f355436440 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/ClassificationResult.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/ClassificationResult.cs new file mode 100644 index 000000000..ddd9b3861 --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/ClassificationResult.cs @@ -0,0 +1,106 @@ +// Copyright (c) 2023 homuler +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file or at +// https://opensource.org/licenses/MIT. + +using System.Collections.Generic; + +namespace Mediapipe.Tasks.Components.Containers +{ + /// + /// Defines classification results for a given classifier head. + /// + public readonly struct Classifications + { + /// + /// The array of predicted categories, usually sorted by descending scores, + /// e.g. from high to low probability. + /// + public readonly IReadOnlyList categories; + /// + /// The index of the classifier head (i.e. output tensor) these categories + /// refer to. This is useful for multi-head models. + /// + public readonly int headIndex; + /// + /// The optional name of the classifier head, as provided in the TFLite Model + /// Metadata [1] if present. This is useful for multi-head models. + /// + /// [1]: https://www.tensorflow.org/lite/convert/metadata + /// + public readonly string headName; + + internal Classifications(IReadOnlyList categories, int headIndex, string headName) + { + this.categories = categories; + this.headIndex = headIndex; + this.headName = headName; + } + + public static Classifications CreateFrom(Proto.Classifications proto) + { + var categories = new List(proto.ClassificationList.Classification.Count); + foreach (var classification in proto.ClassificationList.Classification) + { + categories.Add(Category.CreateFrom(classification)); + } + return new Classifications(categories, proto.HeadIndex, proto.HasHeadName ? proto.HeadName : null); + } + + public static Classifications CreateFrom(ClassificationList proto, int headIndex = 0, string headName = null) + { + var categories = new List(proto.Classification.Count); + foreach (var classification in proto.Classification) + { + categories.Add(Category.CreateFrom(classification)); + } + return new Classifications(categories, headIndex, headName); + } + + public override string ToString() + => $"{{ \"categories\": {Util.Format(categories)}, \"headIndex\": {headIndex}, \"headName\": {Util.Format(headName)} }}"; + } + + /// + /// Defines classification results of a model. + /// + public readonly struct ClassificationResult + { + /// + /// The classification results for each head of the model. + /// + public readonly IReadOnlyList classifications; + + /// + /// The optional timestamp (in milliseconds) of the start of the chunk of data + /// corresponding to these results. + /// + /// This is only used for classification on time series (e.g. audio + /// classification). In these use cases, the amount of data to process might + /// exceed the maximum size that the model can process: to solve this, the + /// input data is split into multiple chunks starting at different timestamps. + /// + public readonly long? timestampMs; + + internal ClassificationResult(IReadOnlyList classifications, long? timestampMs) + { + this.classifications = classifications; + this.timestampMs = timestampMs; + } + + public static ClassificationResult CreateFrom(Proto.ClassificationResult proto) + { + var classifications = new List(proto.Classifications.Count); + foreach (var classification in proto.Classifications) + { + classifications.Add(Classifications.CreateFrom(classification)); + } +#pragma warning disable IDE0004 // for Unity 2020.3.x + return new ClassificationResult(classifications, proto.HasTimestampMs ? (long?)proto.TimestampMs : null); +#pragma warning restore IDE0004 // for Unity 2020.3.x + } + + public override string ToString() => $"{{ \"classifications\": {Util.Format(classifications)}, \"timestampMs\": {Util.Format(timestampMs)} }}"; + } +} diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/ClassificationResult.cs.meta b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/ClassificationResult.cs.meta new file mode 100644 index 000000000..95fea8597 --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/ClassificationResult.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 85fd33c29fb9f25da97555929d28154b +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/DetectionResult.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/DetectionResult.cs index e82a00141..27483a766 100644 --- a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/DetectionResult.cs +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/DetectionResult.cs @@ -5,7 +5,6 @@ // https://opensource.org/licenses/MIT. using System.Collections.Generic; -using System.Linq; namespace Mediapipe.Tasks.Components.Containers { @@ -81,11 +80,7 @@ public static Detection CreateFrom(Mediapipe.Detection proto) } public override string ToString() - { - var categoriesStr = $"[{string.Join(", ", categories.Select(category => category.ToString()))}]"; - var keypointsStr = keypoints == null ? "null" : $"[{string.Join(", ", keypoints.Select(keypoint => keypoint.ToString()))}]"; - return $"{{\"categories\": {categoriesStr}, \"boundingBox\": {boundingBox}, \"keypoints\": {keypointsStr}}}"; - } + => $"{{ \"categories\": {Util.Format(categories)}, \"boundingBox\": {boundingBox}, \"keypoints\": {Util.Format(keypoints)} }}"; } /// @@ -113,10 +108,6 @@ public static DetectionResult CreateFrom(IReadOnlyList dete return new DetectionResult(detections); } - public override string ToString() - { - var detectionsStr = string.Join(", ", detections.Select(detection => detection.ToString())); - return $"{{ \"detections\": [{detectionsStr}] }}"; - } + public override string ToString() => $"{{ \"detections\": {Util.Format(detections)} }}"; } } diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/Keypoint.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/Keypoint.cs index 5c5dd0e6a..4f3f7af48 100644 --- a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/Keypoint.cs +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/Keypoint.cs @@ -36,10 +36,6 @@ internal NormalizedKeypoint(float x, float y, string label, float? score) this.score = score; } - public override string ToString() - { - var scoreStr = score == null ? "null" : $"{score}"; - return $"{{ \"x\": {x}, \"y\": {y}, \"label\": \"{label}\", \"score\": {scoreStr} }}"; - } + public override string ToString() => $"{{ \"x\": {x}, \"y\": {y}, \"label\": \"{label}\", \"score\": {Util.Format(score)} }}"; } } diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/Landmark.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/Landmark.cs new file mode 100644 index 000000000..6ca8ac2b2 --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/Landmark.cs @@ -0,0 +1,202 @@ +// Copyright (c) 2023 homuler +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file or at +// https://opensource.org/licenses/MIT. + +using System; +using System.Collections.Generic; + +// TODO: use System.MathF +using Mathf = UnityEngine.Mathf; + +namespace Mediapipe.Tasks.Components.Containers +{ + /// + /// Landmark represents a point in 3D space with x, y, z coordinates. The + /// landmark coordinates are in meters. z represents the landmark depth, and the + /// smaller the value the closer the world landmark is to the camera. + /// + public readonly struct Landmark : IEquatable + { + private const float _LandmarkTolerance = 1e-6f; + + public readonly float x; + public readonly float y; + public readonly float z; + /// + /// Landmark visibility. Should stay unset if not supported. + /// Float score of whether landmark is visible or occluded by other objects. + /// Landmark considered as invisible also if it is not present on the screen + /// (out of scene bounds). Depending on the model, visibility value is either a + /// sigmoid or an argument of sigmoid. + /// + public readonly float? visibility; + /// + /// Landmark presence. Should stay unset if not supported. + /// Float score of whether landmark is present on the scene (located within + /// scene bounds). Depending on the model, presence value is either a result of + /// sigmoid or an argument of sigmoid function to get landmark presence + /// probability. + /// + public readonly float? presence; + /// + /// Landmark name. Should stay unset if not supported. + /// + public readonly string name; + + internal Landmark(float x, float y, float z, float? visibility, float? presence) : this(x, y, z, visibility, presence, null) + { + } + + internal Landmark(float x, float y, float z, float? visibility, float? presence, string name) + { + this.x = x; + this.y = y; + this.z = z; + this.visibility = visibility; + this.presence = presence; + this.name = name; + } + +#nullable enable + public override bool Equals(object? obj) => obj is Landmark other && Equals(other); +#nullable disable + + bool IEquatable.Equals(Landmark other) + { + return Mathf.Abs(x - other.x) < _LandmarkTolerance && + Mathf.Abs(y - other.y) < _LandmarkTolerance && + Mathf.Abs(z - other.z) < _LandmarkTolerance; + } + + // TODO: use HashCode.Combine + public override int GetHashCode() => Tuple.Create(x, y, z).GetHashCode(); + public static bool operator ==(in Landmark lhs, in Landmark rhs) => lhs.Equals(rhs); + public static bool operator !=(in Landmark lhs, in Landmark rhs) => !(lhs == rhs); + + public static Landmark CreateFrom(Mediapipe.Landmark proto) + { + return new Landmark( + proto.X, proto.Y, proto.Z, +#pragma warning disable IDE0004 // for Unity 2020.3.x + proto.HasVisibility ? (float?)proto.Visibility : null, + proto.HasPresence ? (float?)proto.Presence : null +#pragma warning restore IDE0004 // for Unity 2020.3.x + ); + } + + public override string ToString() + => $"{{ \"x\": {x}, \"y\": {y}, \"z\": {z}, \"visibility\": {Util.Format(visibility)}, \"presence\": {Util.Format(presence)}, \"name\": \"{name}\" }}"; + } + + /// + /// A normalized version of above Landmark struct. All coordinates should be + /// within [0, 1]. + /// + public readonly struct NormalizedLandmark : IEquatable + { + private const float _LandmarkTolerance = 1e-6f; + + public readonly float x; + public readonly float y; + public readonly float z; + public readonly float? visibility; + public readonly float? presence; + public readonly string name; + + internal NormalizedLandmark(float x, float y, float z, float? visibility, float? presence) : this(x, y, z, visibility, presence, null) + { + } + + internal NormalizedLandmark(float x, float y, float z, float? visibility, float? presence, string name) + { + this.x = x; + this.y = y; + this.z = z; + this.visibility = visibility; + this.presence = presence; + this.name = name; + } + +#nullable enable + public override bool Equals(object? obj) => obj is NormalizedLandmark other && Equals(other); +#nullable disable + + bool IEquatable.Equals(NormalizedLandmark other) + { + return Mathf.Abs(x - other.x) < _LandmarkTolerance && + Mathf.Abs(y - other.y) < _LandmarkTolerance && + Mathf.Abs(z - other.z) < _LandmarkTolerance; + } + + // TODO: use HashCode.Combine + public override int GetHashCode() => Tuple.Create(x, y, z).GetHashCode(); + public static bool operator ==(in NormalizedLandmark lhs, in NormalizedLandmark rhs) => lhs.Equals(rhs); + public static bool operator !=(in NormalizedLandmark lhs, in NormalizedLandmark rhs) => !(lhs == rhs); + + public static NormalizedLandmark CreateFrom(Mediapipe.NormalizedLandmark proto) + { + return new NormalizedLandmark( + proto.X, proto.Y, proto.Z, +#pragma warning disable IDE0004 // for Unity 2020.3.x + proto.HasVisibility ? (float?)proto.Visibility : null, + proto.HasPresence ? (float?)proto.Presence : null +#pragma warning restore IDE0004 // for Unity 2020.3.x + ); + } + + public override string ToString() + => $"{{ \"x\": {x}, \"y\": {y}, \"z\": {z}, \"visibility\": {Util.Format(visibility)}, \"presence\": {Util.Format(presence)}, \"name\": \"{name}\" }}"; + } + + /// + /// A list of Landmarks. + /// + public readonly struct Landmarks + { + public readonly IReadOnlyList landmarks; + + internal Landmarks(IReadOnlyList landmarks) + { + this.landmarks = landmarks; + } + + public static Landmarks CreateFrom(LandmarkList proto) + { + var landmarks = new List(proto.Landmark.Count); + foreach (var landmark in proto.Landmark) + { + landmarks.Add(Landmark.CreateFrom(landmark)); + } + return new Landmarks(landmarks); + } + + public override string ToString() => $"{{ \"landmarks\": {Util.Format(landmarks)} }}"; + } + + /// + /// A list of NormalizedLandmarks. + /// + public readonly struct NormalizedLandmarks + { + public readonly IReadOnlyList landmarks; + + internal NormalizedLandmarks(IReadOnlyList landmarks) + { + this.landmarks = landmarks; + } + + public static NormalizedLandmarks CreateFrom(NormalizedLandmarkList proto) + { + var landmarks = new List(proto.Landmark.Count); + foreach (var landmark in proto.Landmark) + { + landmarks.Add(NormalizedLandmark.CreateFrom(landmark)); + } + return new NormalizedLandmarks(landmarks); + } + + public override string ToString() => $"{{ \"landmarks\": {Util.Format(landmarks)} }}"; + } +} diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/Landmark.cs.meta b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/Landmark.cs.meta new file mode 100644 index 000000000..d2fa20162 --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/Landmark.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 487d9694dfbc620c3b322c7383f0e2c1 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/Util.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/Util.cs new file mode 100644 index 000000000..90de52b3c --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/Util.cs @@ -0,0 +1,28 @@ +// Copyright (c) 2023 homuler +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file or at +// https://opensource.org/licenses/MIT. + +using System.Collections.Generic; +using System.Linq; + +namespace Mediapipe.Tasks.Components.Containers +{ + internal static class Util + { + public static string Format(T value) => value == null ? "null" : $"{value}"; + + public static string Format(string value) => value == null ? "null" : $"\"{value}\""; + + public static string Format(IReadOnlyList list) + { + if (list == null) + { + return "null"; + } + var str = string.Join(", ", list.Select(x => x.ToString())); + return $"[{str}]"; + } + } +} diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/Util.cs.meta b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/Util.cs.meta new file mode 100644 index 000000000..46bb82f86 --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/Util.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: ece2cfb1f67737de98e0a8bcf4f81b7a +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/mediapipe_api/BUILD b/mediapipe_api/BUILD index b7fac9417..cfdde9a75 100644 --- a/mediapipe_api/BUILD +++ b/mediapipe_api/BUILD @@ -562,6 +562,7 @@ pkg_zip( "//mediapipe_api/modules/face_detection:proto_srcs", "//mediapipe_api/modules/holistic_landmark/calculators:proto_srcs", "//mediapipe_api/tasks/cc/core/proto:proto_srcs", + "//mediapipe_api/tasks/cc/components/containers/proto:proto_srcs", "//mediapipe_api/tasks/cc/vision/face_detector/proto:proto_srcs", "//mediapipe_api/tasks/cc/vision/face_geometry/proto:proto_srcs", "//mediapipe_api/tasks/cc/vision/face_geometry/calculators:proto_srcs", diff --git a/mediapipe_api/tasks/cc/components/containers/proto/BUILD b/mediapipe_api/tasks/cc/components/containers/proto/BUILD new file mode 100644 index 000000000..a3ddfb9fb --- /dev/null +++ b/mediapipe_api/tasks/cc/components/containers/proto/BUILD @@ -0,0 +1,27 @@ +# Copyright (c) 2021 homuler +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. + +load("@rules_pkg//pkg:mappings.bzl", "pkg_files") +load("//mediapipe_api:csharp_proto_src.bzl", "csharp_proto_src") + +package(default_visibility = ["//visibility:public"]) + +pkg_files( + name = "proto_srcs", + srcs = [ + ":classifications_cs", + ], + prefix = "Tasks/Components/Containers/Proto", +) + +csharp_proto_src( + name = "classifications_cs", + proto_src = "mediapipe/tasks/cc/components/containers/proto/classifications.proto", + deps = [ + "@com_google_mediapipe//mediapipe/tasks/cc/components/containers/proto:protos_src", + "@com_google_mediapipe//mediapipe/framework/formats:protos_src", + ], +) diff --git a/third_party/mediapipe_visibility.diff b/third_party/mediapipe_visibility.diff index 88faad0b5..2e566316d 100644 --- a/third_party/mediapipe_visibility.diff +++ b/third_party/mediapipe_visibility.diff @@ -333,6 +333,20 @@ index 2e33ebf6..76ed790e 100644 + srcs = glob(["*.proto"]), + visibility = ["//visibility:public"], +) +diff --git a/mediapipe/tasks/cc/components/containers/proto/BUILD b/mediapipe/tasks/cc/components/containers/proto/BUILD +index 66255aed..8dab4bbc 100644 +--- a/mediapipe/tasks/cc/components/containers/proto/BUILD ++++ b/mediapipe/tasks/cc/components/containers/proto/BUILD +@@ -42,3 +42,9 @@ mediapipe_proto_library( + "//mediapipe/framework/formats:rect_proto", + ], + ) ++ ++filegroup( ++ name = "protos_src", ++ srcs = glob(["*.proto"]), ++ visibility = ["//visibility:public"], ++) diff --git a/mediapipe/tasks/cc/core/BUILD b/mediapipe/tasks/cc/core/BUILD index dad9cdf1..aeaed482 100644 --- a/mediapipe/tasks/cc/core/BUILD