diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Formats/Matrix.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Formats/Matrix.cs new file mode 100644 index 000000000..b91de3269 --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Formats/Matrix.cs @@ -0,0 +1,56 @@ +// Copyright (c) 2023 homuler +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file or at +// https://opensource.org/licenses/MIT. + +using System; + +namespace Mediapipe +{ + public readonly struct Matrix + { + public enum Layout + { + ColMajor = 0, + RowMajor = 1, + } + + internal readonly float[] data; + public readonly int rows; + public readonly int cols; + internal readonly Layout layout; + + public Matrix(float[] data, int rows, int cols, Layout layout = Layout.ColMajor) + { + if (rows * cols != data.Length) + { + throw new ArgumentException($"Matrix size mismatch ({rows}x{cols} != {data.Length})"); + } + + this.data = data; + this.rows = rows; + this.cols = cols; + this.layout = layout; + } + + internal Matrix(NativeMatrix nativeMatrix) : this(nativeMatrix.AsReadOnlySpan().ToArray(), nativeMatrix.rows, nativeMatrix.cols, nativeMatrix.layout == 0 ? Layout.ColMajor : Layout.RowMajor) + { } + + internal static void Copy(NativeMatrix source, ref Matrix destination) + { + if (destination.rows != source.rows || destination.cols != source.cols) + { + throw new ArgumentException($"Matrix size mismatch ({source.rows}x{source.cols} != {destination.rows}x{destination.cols})"); + } + + source.AsReadOnlySpan().CopyTo(destination.data); + var layout = source.layout == 0 ? Layout.ColMajor : Layout.RowMajor; + + destination = new Matrix(destination.data, source.rows, source.cols, layout); + } + + public readonly bool isColMajor => layout == Layout.ColMajor; + public readonly bool isRowMajor => layout == Layout.RowMajor; + } +} diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Formats/Matrix.cs.meta b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Formats/Matrix.cs.meta new file mode 100644 index 000000000..9cd617573 --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Formats/Matrix.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 2af00eb0822a21721994e0ecc5880364 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet.cs index d31c40d78..a0c1a7a0f 100644 --- a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet.cs +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet.cs @@ -248,6 +248,56 @@ public static Packet CreateIntAt(int value, long timestampMicrosec) return new Packet(ptr, true); } + /// + /// Create a Matrix Packet. + /// + public static Packet CreateColMajorMatrix(float[] data, int row, int col) + { + UnsafeNativeMethods.mp__MakeColMajorMatrixPacket__Pf_i_i(data, row, col, out var ptr).Assert(); + + return new Packet(ptr, true); + } + + /// + /// Create a Matrix Packet. + /// + public static Packet CreateColMajorMatrix(Matrix value) + { + if (!value.isColMajor) + { + throw new ArgumentException("Matrix must be col-major"); + } + return CreateColMajorMatrix(value.data, value.rows, value.cols); + } + + /// + /// Create a Matrix Packet. + /// + /// + /// The timestamp of the packet. + /// + public static Packet CreateColMajorMatrixAt(float[] data, int row, int col, long timestampMicrosec) + { + UnsafeNativeMethods.mp__MakeColMajorMatrixPacket_At__Pf_i_i_ll(data, row, col, timestampMicrosec, out var ptr).Assert(); + + return new Packet(ptr, true); + } + + /// + /// Create a Matrix Packet. + /// + /// + /// The timestamp of the packet. + /// + public static Packet CreateColMajorMatrixAt(Matrix value, long timestampMicrosec) + { + if (!value.isColMajor) + { + throw new ArgumentException("Matrix must be col-major"); + } + return CreateColMajorMatrixAt(value.data, value.rows, value.cols, timestampMicrosec); + } + /// /// Create a MediaPipe protobuf message Packet. /// diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/PacketGetterExtension.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/PacketGetterExtension.cs index 195512845..d8df65255 100644 --- a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/PacketGetterExtension.cs +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/PacketGetterExtension.cs @@ -355,6 +355,43 @@ public static int Get(this Packet packet) [Obsolete("Use Get instead")] public static int GetInt(this Packet packet) => Get(packet); + /// + /// Get the content of the as a . + /// + /// + /// On some platforms (e.g. Windows), it will abort the process when should be thrown. + /// + /// + /// The to be filled with the content of the . + /// + /// + /// If the doesn't contain a mediapipe::Matrix data. + /// + public static void Get(this Packet packet, ref Matrix value) + { + UnsafeNativeMethods.mp_Packet__GetMpMatrix(packet.mpPtr, out var nativeMatrix).Assert(); + GC.KeepAlive(packet); + + Matrix.Copy(nativeMatrix, ref value); + } + + /// + /// Get the content of the as a . + /// + /// + /// On some platforms (e.g. Windows), it will abort the process when should be thrown. + /// + /// + /// If the doesn't contain a mediapipe::Matrix data. + /// + public static Matrix Get(this Packet packet) + { + UnsafeNativeMethods.mp_Packet__GetMpMatrix(packet.mpPtr, out var nativeMatrix).Assert(); + GC.KeepAlive(packet); + + return new Matrix(nativeMatrix); + } + /// /// Get the content of the as a proto message. /// diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/PacketValidatorExtension.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/PacketValidatorExtension.cs index 5c0e78530..9eae75f3e 100644 --- a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/PacketValidatorExtension.cs +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/PacketValidatorExtension.cs @@ -182,6 +182,20 @@ public static void Validate(this Packet packet) [Obsolete("Use Validate instead")] public static void ValidateAsInt(this Packet packet) => Validate(packet); + /// + /// Validate if the content of the is mediapipe::Matrix. + /// + /// + /// If the doesn't contain mediapipe::Matrix. + /// + public static void Validate(this Packet packet) + { + UnsafeNativeMethods.mp_Packet__ValidateAsMatrix(packet.mpPtr, out var statusPtr).Assert(); + GC.KeepAlive(packet); + + Status.UnsafeAssertOk(statusPtr); + } + /// /// Validate if the content of the is a proto message. /// diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Marshal/NativeMatrix.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Marshal/NativeMatrix.cs new file mode 100644 index 000000000..1974dd8c4 --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Marshal/NativeMatrix.cs @@ -0,0 +1,33 @@ +// Copyright (c) 2023 homuler +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file or at +// https://opensource.org/licenses/MIT. + +using System; +using System.Runtime.InteropServices; + +namespace Mediapipe +{ + [StructLayout(LayoutKind.Sequential)] + internal readonly struct NativeMatrix + { + private readonly IntPtr _data; + public readonly int rows; + public readonly int cols; + public readonly int layout; + + public void Dispose() + { + UnsafeNativeMethods.mp_api_Matrix__delete(this); + } + + public ReadOnlySpan AsReadOnlySpan() + { + unsafe + { + return new ReadOnlySpan((float*)_data, rows * cols); + } + } + } +} diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Marshal/NativeMatrix.cs.meta b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Marshal/NativeMatrix.cs.meta new file mode 100644 index 000000000..537bdabf6 --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Marshal/NativeMatrix.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: a999021701cb86452815895b78ddd2ff +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/Matrix_Unsafe.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/Matrix_Unsafe.cs index bfc16aec5..d28ea7959 100644 --- a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/Matrix_Unsafe.cs +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/Matrix_Unsafe.cs @@ -13,17 +13,19 @@ internal static partial class UnsafeNativeMethods { #region Packet [DllImport(MediaPipeLibrary, ExactSpelling = true)] - public static extern MpReturnCode mp__MakeMatrixPacket__PKc_i(byte[] serializedMatrixData, int size, out IntPtr packet_out); + public static extern MpReturnCode mp_Packet__ValidateAsMatrix(IntPtr packet, out IntPtr status); [DllImport(MediaPipeLibrary, ExactSpelling = true)] - public static extern MpReturnCode mp__MakeMatrixPacket_At__PKc_i_Rt(byte[] serializedMatrixData, int size, IntPtr timestamp, out IntPtr packet_out); + public static extern MpReturnCode mp_Packet__GetMpMatrix(IntPtr packet, out NativeMatrix matrix); [DllImport(MediaPipeLibrary, ExactSpelling = true)] - public static extern MpReturnCode mp_Packet__ValidateAsMatrix(IntPtr packet, out IntPtr status); + public static extern MpReturnCode mp__MakeColMajorMatrixPacket__Pf_i_i(float[] data, int rows, int cols, out IntPtr packet_out); [DllImport(MediaPipeLibrary, ExactSpelling = true)] - public static extern MpReturnCode mp_Packet__GetMatrix(IntPtr packet, out SerializedProto serializedProto); + public static extern MpReturnCode mp__MakeColMajorMatrixPacket_At__Pf_i_i_ll(float[] data, int rows, int cols, long timestampMicrosec, out IntPtr packet_out); + [DllImport(MediaPipeLibrary, ExactSpelling = true)] + public static extern void mp_api_Matrix__delete(NativeMatrix matrix); #endregion } } diff --git a/Packages/com.github.homuler.mediapipe/Tests/EditMode/Framework/PacketTest.cs b/Packages/com.github.homuler.mediapipe/Tests/EditMode/Framework/PacketTest.cs index 3eff9176b..78c59577f 100644 --- a/Packages/com.github.homuler.mediapipe/Tests/EditMode/Framework/PacketTest.cs +++ b/Packages/com.github.homuler.mediapipe/Tests/EditMode/Framework/PacketTest.cs @@ -321,6 +321,44 @@ public void CreateIntAt_ShouldReturnNewIntPacket(int value) } #endregion + #region Matrix + [Test] + public void CreateColMajorMatrix_ShouldReturnNewMatrixPacket() + { + var value = new Matrix(new float[] { 1, 2, 3, 4, 5, 6 }, 2, 3); + using var packet = Packet.CreateColMajorMatrix(value); + + Assert.DoesNotThrow(packet.Validate); + + var result = packet.Get(); + Assert.AreEqual(value.data, result.data); + Assert.AreEqual(value.rows, result.rows); + Assert.AreEqual(value.cols, result.cols); + Assert.AreEqual(value.layout, result.layout); + + using var unsetTimestamp = Timestamp.Unset(); + Assert.AreEqual(unsetTimestamp.Microseconds(), packet.TimestampMicroseconds()); + } + + [Test] + public void CreateColMajorMatrixAt_ShouldReturnNewMatrixPacket() + { + var value = new Matrix(new float[] { 1, 2, 3, 4, 5, 6 }, 2, 3); + var timestamp = 1; + using var packet = Packet.CreateColMajorMatrixAt(value, timestamp); + + Assert.DoesNotThrow(packet.Validate); + + var result = packet.Get(); + Assert.AreEqual(value.data, result.data); + Assert.AreEqual(value.rows, result.rows); + Assert.AreEqual(value.cols, result.cols); + Assert.AreEqual(value.layout, result.layout); + + Assert.AreEqual(timestamp, packet.TimestampMicroseconds()); + } + #endregion + #region Proto [Test] public void CreateProto_ShouldReturnNewProtoPacket() diff --git a/mediapipe_api/BUILD b/mediapipe_api/BUILD index 94b42a45c..e40e81f24 100644 --- a/mediapipe_api/BUILD +++ b/mediapipe_api/BUILD @@ -161,7 +161,7 @@ cc_library( "//mediapipe_api/framework/formats:image", "//mediapipe_api/framework/formats:image_frame", "//mediapipe_api/framework/formats:landmark", - "//mediapipe_api/framework/formats:matrix_data", + "//mediapipe_api/framework/formats:matrix", "//mediapipe_api/framework/formats:rect", "//mediapipe_api/framework/port:logging", "//mediapipe_api/tasks/c/components/containers:classification_result", diff --git a/mediapipe_api/framework/formats/BUILD b/mediapipe_api/framework/formats/BUILD index 640fb1b4a..ff05d5526 100644 --- a/mediapipe_api/framework/formats/BUILD +++ b/mediapipe_api/framework/formats/BUILD @@ -77,9 +77,9 @@ cc_library( ) cc_library( - name = "matrix_data", - srcs = ["matrix_data.cc"], - hdrs = ["matrix_data.h"], + name = "matrix", + srcs = ["matrix.cc"], + hdrs = ["matrix.h"], deps = [ "//mediapipe_api:common", "//mediapipe_api/external/absl:status", diff --git a/mediapipe_api/framework/formats/matrix.cc b/mediapipe_api/framework/formats/matrix.cc new file mode 100644 index 000000000..0520196f1 --- /dev/null +++ b/mediapipe_api/framework/formats/matrix.cc @@ -0,0 +1,60 @@ +// Copyright (c) 2023 homuler +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file or at +// https://opensource.org/licenses/MIT. + +#include "mediapipe_api/framework/formats/matrix.h" + + +MpReturnCode mp__MakeColMajorMatrixPacket__Pf_i_i(float* pcm_data, int rows, int cols, mediapipe::Packet** packet_out) { + TRY + Eigen::Map m(pcm_data, rows, cols); + + *packet_out = new mediapipe::Packet{mediapipe::MakePacket(m)}; + RETURN_CODE(MpReturnCode::Success); + CATCH_EXCEPTION +} + +MpReturnCode mp__MakeColMajorMatrixPacket_At__Pf_i_i_ll(float* pcm_data, int rows, int cols, int64 timestamp_microsec, mediapipe::Packet** packet_out) { + TRY + Eigen::Map m(pcm_data, rows, cols); + + *packet_out = new mediapipe::Packet{mediapipe::MakePacket(m).At(mediapipe::Timestamp(timestamp_microsec))}; + RETURN_CODE(MpReturnCode::Success); + CATCH_EXCEPTION +} + +MpReturnCode mp_Packet__GetMpMatrix(mediapipe::Packet* packet, mp_api::Matrix* value_out) { + TRY + auto matrix = packet->Get(); + auto rows = matrix.rows(); + auto cols = matrix.cols(); + auto data = matrix.data(); + auto len = rows * cols; + + value_out->rows = rows; + value_out->cols = cols; + if (matrix.IsRowMajor) { + value_out->layout = mp_api::rowMajor; + } else { + value_out->layout = mp_api::colMajor; + } + value_out->data = new float[len]; + memcpy(value_out->data, data, len * sizeof(float)); + + RETURN_CODE(MpReturnCode::Success); + CATCH_EXCEPTION +} + +MpReturnCode mp_Packet__ValidateAsMatrix(mediapipe::Packet* packet, absl::Status** status_out) { + TRY + *status_out = new absl::Status{packet->ValidateAsType()}; + RETURN_CODE(MpReturnCode::Success); + CATCH_EXCEPTION +} + + +void mp_api_Matrix__delete(mp_api::Matrix matrix) { + delete[] matrix.data; +} diff --git a/mediapipe_api/framework/formats/matrix.h b/mediapipe_api/framework/formats/matrix.h new file mode 100644 index 000000000..9f64fbdf8 --- /dev/null +++ b/mediapipe_api/framework/formats/matrix.h @@ -0,0 +1,39 @@ +// Copyright (c) 2021 homuler +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file or at +// https://opensource.org/licenses/MIT. + +#ifndef MEDIAPIPE_API_FRAMEWORK_FORMATS_MATRIX_H_ +#define MEDIAPIPE_API_FRAMEWORK_FORMATS_MATRIX_H_ + +#include "mediapipe/framework/formats/matrix.h" +#include "mediapipe_api/common.h" +#include "mediapipe_api/external/protobuf.h" +#include "mediapipe_api/framework/packet.h" + +namespace mp_api { + constexpr int colMajor = 0; + constexpr int rowMajor = 1; + + struct Matrix { + float* data; + int rows; + int cols; + int layout; + }; +} + +extern "C" { + +MP_CAPI(MpReturnCode) mp__MakeColMajorMatrixPacket__Pf_i_i(float* pcm_data, int rows, int cols, mediapipe::Packet** packet_out); +MP_CAPI(MpReturnCode) mp__MakeColMajorMatrixPacket_At__Pf_i_i_ll(float* pcm_data, int rows, int cols, int64 timestamp_microsec, mediapipe::Packet** packet_out); + +MP_CAPI(MpReturnCode) mp_Packet__GetMpMatrix(mediapipe::Packet* packet, mp_api::Matrix* value_out); +MP_CAPI(MpReturnCode) mp_Packet__ValidateAsMatrix(mediapipe::Packet* packet, absl::Status** status_out); + +MP_CAPI(void) mp_api_Matrix__delete(mp_api::Matrix matrix); + +} // extern "C" + +#endif // MEDIAPIPE_API_FRAMEWORK_FORMATS_MATRIX_H_ diff --git a/mediapipe_api/framework/formats/matrix_data.cc b/mediapipe_api/framework/formats/matrix_data.cc deleted file mode 100644 index b3e7b7578..000000000 --- a/mediapipe_api/framework/formats/matrix_data.cc +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright (c) 2021 homuler -// -// Use of this source code is governed by an MIT-style -// license that can be found in the LICENSE file or at -// https://opensource.org/licenses/MIT. - -#include "mediapipe_api/framework/formats/matrix_data.h" - -MpReturnCode mp__MakeMatrixPacket__PKc_i(const char* serialized_matrix_data, int size, mediapipe::Packet** packet_out) { - TRY - auto matrix_data = ParseFromStringAsProto(serialized_matrix_data, size); - - mediapipe::Matrix matrix; - mediapipe::MatrixFromMatrixDataProto(matrix_data, &matrix); - - *packet_out = new mediapipe::Packet{mediapipe::MakePacket(matrix)}; - RETURN_CODE(MpReturnCode::Success); - CATCH_EXCEPTION -} - -MpReturnCode mp__MakeMatrixPacket_At__PKc_i_Rt(const char* serialized_matrix_data, int size, mediapipe::Timestamp* timestamp, mediapipe::Packet** packet_out) { - TRY - auto matrix_data = ParseFromStringAsProto(serialized_matrix_data, size); - - mediapipe::Matrix matrix; - mediapipe::MatrixFromMatrixDataProto(matrix_data, &matrix); - - *packet_out = new mediapipe::Packet{mediapipe::MakePacket(matrix).At(*timestamp)}; - RETURN_CODE(MpReturnCode::Success); - CATCH_EXCEPTION -} - -MP_CAPI(MpReturnCode) mp_Packet__GetMatrix(mediapipe::Packet* packet, mp_api::SerializedProto* value_out) { - TRY - mediapipe::MatrixData matrix_data; - auto matrix = packet->Get(); - mediapipe::MatrixDataProtoFromMatrix(matrix, &matrix_data); - - SerializeProto(matrix_data, value_out); - - RETURN_CODE(MpReturnCode::Success); - CATCH_EXCEPTION -} - -MP_CAPI(MpReturnCode) mp_Packet__ValidateAsMatrix(mediapipe::Packet* packet, absl::Status** status_out) { - TRY - *status_out = new absl::Status{packet->ValidateAsType()}; - RETURN_CODE(MpReturnCode::Success); - CATCH_EXCEPTION -} diff --git a/mediapipe_api/framework/formats/matrix_data.h b/mediapipe_api/framework/formats/matrix_data.h deleted file mode 100644 index 6580093f2..000000000 --- a/mediapipe_api/framework/formats/matrix_data.h +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright (c) 2021 homuler -// -// Use of this source code is governed by an MIT-style -// license that can be found in the LICENSE file or at -// https://opensource.org/licenses/MIT. - -#ifndef MEDIAPIPE_API_FRAMEWORK_FORMATS_MATRIX_DATA_H_ -#define MEDIAPIPE_API_FRAMEWORK_FORMATS_MATRIX_DATA_H_ - -#include "mediapipe/framework/formats/matrix.h" -#include "mediapipe_api/common.h" -#include "mediapipe_api/external/protobuf.h" -#include "mediapipe_api/framework/packet.h" - -extern "C" { - -MP_CAPI(MpReturnCode) mp__MakeMatrixPacket__PKc_i(const char* matrix_data_serialized, int size, mediapipe::Packet** packet_out); -MP_CAPI(MpReturnCode) mp__MakeMatrixPacket_At__PKc_i_Rt(const char* matrix_data_serialized, int size, mediapipe::Timestamp* timestamp, - mediapipe::Packet** packet_out); -MP_CAPI(MpReturnCode) mp_Packet__GetMatrix(mediapipe::Packet* packet, mp_api::SerializedProto* value_out); -MP_CAPI(MpReturnCode) mp_Packet__ValidateAsMatrix(mediapipe::Packet* packet, absl::Status** status_out); - -} // extern "C" - -#endif // MEDIAPIPE_API_FRAMEWORK_FORMATS_MATRIX_DATA_H_