From 3eb66c8bf01883032f3305c2ae19ef8170c2187f Mon Sep 17 00:00:00 2001 From: Junrou Nishida Date: Sat, 29 Jul 2023 20:15:01 +0900 Subject: [PATCH] feat: implement Components.Containers.* (#972) --- .../Tasks/Components/Containers/Category.cs | 55 ++++++++ .../Components/Containers/Category.cs.meta | 11 ++ .../Components/Containers/DetectionResult.cs | 122 ++++++++++++++++++ .../Containers/DetectionResult.cs.meta | 11 ++ .../Tasks/Components/Containers/Keypoint.cs | 45 +++++++ .../Components/Containers/Keypoint.cs.meta | 11 ++ .../Tasks/Components/Containers/Rect.cs | 25 +++- 7 files changed, 279 insertions(+), 1 deletion(-) create mode 100644 Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/Category.cs create mode 100644 Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/Category.cs.meta create mode 100644 Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/DetectionResult.cs create mode 100644 Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/DetectionResult.cs.meta create mode 100644 Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/Keypoint.cs create mode 100644 Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/Keypoint.cs.meta diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/Category.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/Category.cs new file mode 100644 index 000000000..d370fcdbd --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/Category.cs @@ -0,0 +1,55 @@ +// Copyright (c) 2023 homuler +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file or at +// https://opensource.org/licenses/MIT. + +namespace Mediapipe.Tasks.Components.Containers +{ + /// + /// Defines a single classification result. + /// + /// The label maps packed into the TFLite Model Metadata [1] are used to populate + /// the 'category_name' and 'display_name' fields. + /// + /// [1]: https://www.tensorflow.org/lite/convert/metadata + /// + public readonly struct Category + { + /// + /// The index of the category in the classification model output. + /// + public readonly int index; + /// + /// The score for this category, e.g. (but not necessarily) a probability in [0,1]. + /// + public readonly float score; + /// + /// The optional ID for the category, read from the label map packed in the + /// TFLite Model Metadata if present. Not necessarily human-readable. + /// + public readonly string categoryName; + /// + /// The optional human-readable name for the category, read from the label map + /// packed in the TFLite Model Metadata if present. + /// + public readonly string displayName; + + internal Category(int index, float score, string categoryName, string displayName) + { + this.index = index; + this.score = score; + this.categoryName = categoryName; + this.displayName = displayName; + } + + public static Category CreateFrom(Classification proto) + { + var categoryName = proto.HasLabel ? proto.Label : null; + var displayName = proto.HasDisplayName ? proto.DisplayName : null; + return new Category(proto.Index, proto.Score, categoryName, displayName); + } + + public override string ToString() => $"{{ \"index\": {index}, \"score\": {score}, \"categoryName\": \"{categoryName}\", \"displayName\": \"{displayName}\" }}"; + } +} diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/Category.cs.meta b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/Category.cs.meta new file mode 100644 index 000000000..e5a18ae08 --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/Category.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 22d456f303c2dd853affaca39ecd65d6 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/DetectionResult.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/DetectionResult.cs new file mode 100644 index 000000000..e82a00141 --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/DetectionResult.cs @@ -0,0 +1,122 @@ +// Copyright (c) 2023 homuler +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file or at +// https://opensource.org/licenses/MIT. + +using System.Collections.Generic; +using System.Linq; + +namespace Mediapipe.Tasks.Components.Containers +{ + /// + /// Represents one detected object in the object detector's results. + /// + public readonly struct Detection + { + private const int _DefaultCategoryIndex = -1; + + /// + /// A list of objects. + /// + public readonly IReadOnlyList categories; + /// + /// The bounding box location. + /// + public readonly Rect boundingBox; + /// + /// Optional list of keypoints associated with the detection. Keypoints + /// represent interesting points related to the detection. For example, the + /// keypoints represent the eye, ear and mouth from face detection model. Or + /// in the template matching detection, e.g. KNIFT, they can represent the + /// feature points for template matching. + /// + public readonly IReadOnlyList keypoints; + + internal Detection(IReadOnlyList categories, Rect boundingBox, IReadOnlyList keypoints) + { + this.categories = categories; + this.boundingBox = boundingBox; + this.keypoints = keypoints; + } + + public static Detection CreateFrom(Mediapipe.Detection proto) + { + var categories = new List(proto.Score.Count); + for (var idx = 0; idx < proto.Score.Count; idx++) + { + categories.Add(new Category( + proto.LabelId.Count > idx ? proto.LabelId[idx] : _DefaultCategoryIndex, + proto.Score[idx], + proto.Label.Count > idx ? proto.Label[idx] : "", + proto.DisplayName.Count > idx ? proto.DisplayName[idx] : "" + )); + } + + var boundingBox = proto.LocationData != null ? new Rect( + proto.LocationData.BoundingBox.Xmin, + proto.LocationData.BoundingBox.Ymin, + proto.LocationData.BoundingBox.Xmin + proto.LocationData.BoundingBox.Width, + proto.LocationData.BoundingBox.Ymin + proto.LocationData.BoundingBox.Height + ) : new Rect(0, 0, 0, 0); + + List keypoints = null; + if (proto.LocationData.RelativeKeypoints.Count > 0) + { + keypoints = new List(proto.LocationData.RelativeKeypoints.Count); + foreach (var keypoint in proto.LocationData.RelativeKeypoints) + { + keypoints.Add(new NormalizedKeypoint( + keypoint.X, + keypoint.Y, + keypoint.HasKeypointLabel ? keypoint.KeypointLabel : null, +#pragma warning disable IDE0004 // for Unity 2020.3.x + keypoint.HasScore ? (float?)keypoint.Score : null +#pragma warning restore IDE0004 + )); + } + } + + return new Detection(categories, boundingBox, keypoints); + } + + public override string ToString() + { + var categoriesStr = $"[{string.Join(", ", categories.Select(category => category.ToString()))}]"; + var keypointsStr = keypoints == null ? "null" : $"[{string.Join(", ", keypoints.Select(keypoint => keypoint.ToString()))}]"; + return $"{{\"categories\": {categoriesStr}, \"boundingBox\": {boundingBox}, \"keypoints\": {keypointsStr}}}"; + } + } + + /// + /// Represents the list of detected objects. + /// + public readonly struct DetectionResult + { + /// + /// A list of objects. + /// + public readonly IReadOnlyList detections; + + internal DetectionResult(IReadOnlyList detections) + { + this.detections = detections; + } + + public static DetectionResult CreateFrom(IReadOnlyList detectionsProto) + { + var detections = new List(detectionsProto.Count); + foreach (var detectionProto in detectionsProto) + { + detections.Add(Detection.CreateFrom(detectionProto)); + } + return new DetectionResult(detections); + } + + public override string ToString() + { + var detectionsStr = string.Join(", ", detections.Select(detection => detection.ToString())); + return $"{{ \"detections\": [{detectionsStr}] }}"; + } + } +} diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/DetectionResult.cs.meta b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/DetectionResult.cs.meta new file mode 100644 index 000000000..f3cbefc92 --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/DetectionResult.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: c308f4879af9249c5845a14ab75607f7 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/Keypoint.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/Keypoint.cs new file mode 100644 index 000000000..5c5dd0e6a --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/Keypoint.cs @@ -0,0 +1,45 @@ +// Copyright (c) 2023 homuler +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file or at +// https://opensource.org/licenses/MIT. + +namespace Mediapipe.Tasks.Components.Containers +{ + /// + /// A keypoint, defined by the coordinates (x, y), normalized by the image dimensions. + /// + public readonly struct NormalizedKeypoint + { + /// + /// x in normalized image coordinates. + /// + public readonly float x; + /// + /// y in normalized image coordinates. + /// + public readonly float y; + /// + /// optional label of the keypoint. + /// + public readonly string label; + /// + /// optional score of the keypoint. + /// + public readonly float? score; + + internal NormalizedKeypoint(float x, float y, string label, float? score) + { + this.x = x; + this.y = y; + this.label = label; + this.score = score; + } + + public override string ToString() + { + var scoreStr = score == null ? "null" : $"{score}"; + return $"{{ \"x\": {x}, \"y\": {y}, \"label\": \"{label}\", \"score\": {scoreStr} }}"; + } + } +} diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/Keypoint.cs.meta b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/Keypoint.cs.meta new file mode 100644 index 000000000..57638d858 --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/Keypoint.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 600a82a9dcc9b4b2f808c01ffc6b8a85 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/Rect.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/Rect.cs index 2174a8e69..d8c2b5f08 100644 --- a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/Rect.cs +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Tasks/Components/Containers/Rect.cs @@ -1,4 +1,4 @@ -// Copyright (c) 2021 homuler +// Copyright (c) 2023 homuler // // Use of this source code is governed by an MIT-style // license that can be found in the LICENSE file or at @@ -11,6 +11,27 @@ namespace Mediapipe.Tasks.Components.Containers { + /// + /// Defines a rectangle, used e.g. as part of detection results or as input region-of-interest. + /// + public readonly struct Rect + { + public readonly int left; + public readonly int top; + public readonly int right; + public readonly int bottom; + + internal Rect(int left, int top, int right, int bottom) + { + this.left = left; + this.top = top; + this.right = right; + this.bottom = bottom; + } + + public override string ToString() => $"{{ \"left\": {left}, \"top\": {top}, \"right\": {right}, \"bottom\": {bottom} }}"; + } + /// /// A rectangle, used as part of detection results or as input region-of-interest. /// @@ -43,5 +64,7 @@ bool IEquatable.Equals(RectF other) public override int GetHashCode() => Tuple.Create(left, top, right, bottom).GetHashCode(); public static bool operator ==(RectF lhs, RectF rhs) => lhs.Equals(rhs); public static bool operator !=(RectF lhs, RectF rhs) => !(lhs == rhs); + + public override string ToString() => $"{{ \"left\": {left}, \"top\": {top}, \"right\": {right}, \"bottom\": {bottom} }}"; } }