diff --git a/Assets/MediaPipeUnity/Samples/Common/Scripts/StartSceneController.cs b/Assets/MediaPipeUnity/Samples/Common/Scripts/StartSceneController.cs index e112ff2cc..a7fb6e5a1 100644 --- a/Assets/MediaPipeUnity/Samples/Common/Scripts/StartSceneController.cs +++ b/Assets/MediaPipeUnity/Samples/Common/Scripts/StartSceneController.cs @@ -6,7 +6,6 @@ using System.Collections; using UnityEngine; -using UnityEngine.UI; using UnityEngine.SceneManagement; namespace Mediapipe.Unity @@ -15,7 +14,7 @@ public class StartSceneController : MonoBehaviour { private const string _TAG = nameof(Bootstrap); - [SerializeField] private Image _screen; + [SerializeField] private UnityEngine.UI.Image _screen; [SerializeField] private GameObject _consolePrefab; private IEnumerator Start() diff --git a/Assets/MediaPipeUnity/Samples/UI/Scripts/SolutionMenu.cs b/Assets/MediaPipeUnity/Samples/UI/Scripts/SolutionMenu.cs index 46c01029e..259b3ab77 100644 --- a/Assets/MediaPipeUnity/Samples/UI/Scripts/SolutionMenu.cs +++ b/Assets/MediaPipeUnity/Samples/UI/Scripts/SolutionMenu.cs @@ -70,7 +70,7 @@ private Button GetButtonInRow(Transform row, int index) private void HideButton(Button button) { - var image = button.gameObject.GetComponent(); + var image = button.gameObject.GetComponent(); image.color = new Color(0, 0, 0, 0); button.transform.GetComponentInChildren().text = null; } diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Formats/Image.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Formats/Image.cs new file mode 100644 index 000000000..2abab40b6 --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Formats/Image.cs @@ -0,0 +1,154 @@ +// Copyright (c) 2023 homuler +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file or at +// https://opensource.org/licenses/MIT. + +using System; +using Unity.Collections; +using Unity.Collections.LowLevel.Unsafe; + +namespace Mediapipe +{ + public class Image : MpResourceHandle + { + public Image(IntPtr imagePtr, bool isOwner = true) : base(imagePtr, isOwner) { } + + public Image(ImageFormat.Types.Format format, int width, int height, int widthStep, IntPtr pixelData, ImageFrame.Deleter deleter) : base() + { + UnsafeNativeMethods.mp_Image__ui_i_i_i_Pui8_PF(format, width, height, widthStep, pixelData, deleter, out var ptr).Assert(); + this.ptr = ptr; + } + + public unsafe Image(ImageFormat.Types.Format format, int width, int height, int widthStep, NativeArray pixelData, ImageFrame.Deleter deleter) + : this(format, width, height, widthStep, (IntPtr)NativeArrayUnsafeUtility.GetUnsafeReadOnlyPtr(pixelData), deleter) + { } + + /// + /// Initialize an . + /// + /// + /// won't be released if the instance is disposed of.
+ /// It's useful when: + /// + /// + /// You can reuse the memory allocated to . + /// + /// + /// You've not allocated the memory (e.g. ). + /// + /// + ///
+ public Image(ImageFormat.Types.Format format, int width, int height, int widthStep, NativeArray pixelData) + : this(format, width, height, widthStep, pixelData, ImageFrame.VoidDeleter) + { } + +#if UNITY_EDITOR_LINUX || UNITY_STANDLONE_LINUX || UNITY_ANDROID + public Image(uint target, uint name, int width, int height, GpuBufferFormat format, GlTextureBuffer.DeletionCallback callback, GlContext glContext) : base() + { + UnsafeNativeMethods.mp_Image__ui_ui_i_i_ui_PF_PSgc(target, name, width, height, format, callback, glContext.sharedPtr, out var ptr).Assert(); + this.ptr = ptr; + } + + public Image(uint name, int width, int height, GpuBufferFormat format, GlTextureBuffer.DeletionCallback callback, GlContext glContext) : + this(Gl.GL_TEXTURE_2D, name, width, height, format, callback, glContext) + { } +#endif + + protected override void DeleteMpPtr() + { + UnsafeNativeMethods.mp_Image__delete(ptr); + } + + public int Width() + { + var ret = SafeNativeMethods.mp_Image__width(mpPtr); + + GC.KeepAlive(this); + return ret; + } + + public int Height() + { + var ret = SafeNativeMethods.mp_Image__height(mpPtr); + + GC.KeepAlive(this); + return ret; + } + + public int Channels() + { + var ret = SafeNativeMethods.mp_Image__channels(mpPtr); + + GC.KeepAlive(this); + return ret; + } + + public int Step() + { + var ret = SafeNativeMethods.mp_Image__step(mpPtr); + + GC.KeepAlive(this); + return ret; + } + + public bool UsesGpu() + { + var ret = SafeNativeMethods.mp_Image__UsesGpu(mpPtr); + + GC.KeepAlive(this); + return ret; + } + + public ImageFormat.Types.Format ImageFormat() + { + var ret = SafeNativeMethods.mp_Image__image_format(mpPtr); + + GC.KeepAlive(this); + return ret; + } + + public GpuBufferFormat Format() + { + var ret = SafeNativeMethods.mp_Image__format(mpPtr); + + GC.KeepAlive(this); + return ret; + } + + public bool ConvertToCpu() + { + UnsafeNativeMethods.mp_Image__ConvertToCpu(mpPtr, out var result).Assert(); + + GC.KeepAlive(this); + return result; + } + + public bool ConvertToGpu() + { + UnsafeNativeMethods.mp_Image__ConvertToGpu(mpPtr, out var result).Assert(); + + GC.KeepAlive(this); + return result; + } + } + + public class PixelWriteLock : MpResourceHandle + { + public PixelWriteLock(Image image) : base() + { + UnsafeNativeMethods.mp_PixelWriteLock__RI(image.mpPtr, out var ptr).Assert(); + this.ptr = ptr; + } + + protected override void DeleteMpPtr() + { + UnsafeNativeMethods.mp_PixelWriteLock__delete(ptr); + } + + public IntPtr Pixels() + { + return SafeNativeMethods.mp_PixelWriteLock__Pixels(mpPtr); + } + } +} diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Formats/Image.cs.meta b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Formats/Image.cs.meta new file mode 100644 index 000000000..d2911bb79 --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Formats/Image.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: df36c61aea905ded78928ff2a8869f50 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Formats/ImageFrame.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Formats/ImageFrame.cs index 57bfd2084..2111adb22 100644 --- a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Formats/ImageFrame.cs +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Formats/ImageFrame.cs @@ -71,7 +71,7 @@ protected override void DeleteMpPtr() } [AOT.MonoPInvokeCallback(typeof(Deleter))] - private static void VoidDeleter(IntPtr _) { } + internal static void VoidDeleter(IntPtr _) { } /// /// The number of channels for a . diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/ImagePacket.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/ImagePacket.cs new file mode 100644 index 000000000..fbae1f4bd --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/ImagePacket.cs @@ -0,0 +1,68 @@ +// Copyright (c) 2023 homuler +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file or at +// https://opensource.org/licenses/MIT. + +using System; + +namespace Mediapipe +{ + public class ImagePacket : Packet + { + /// + /// Creates an empty instance. + /// + public ImagePacket() : base(true) { } + + [UnityEngine.Scripting.Preserve] + public ImagePacket(IntPtr ptr, bool isOwner = true) : base(ptr, isOwner) { } + + public ImagePacket(Image image) : base() + { + UnsafeNativeMethods.mp__MakeImagePacket__Pif(image.mpPtr, out var ptr).Assert(); + image.Dispose(); // respect move semantics + + this.ptr = ptr; + } + + public ImagePacket(Image image, Timestamp timestamp) : base() + { + UnsafeNativeMethods.mp__MakeImagePacket_At__Pif_Rt(image.mpPtr, timestamp.mpPtr, out var ptr).Assert(); + GC.KeepAlive(timestamp); + image.Dispose(); // respect move semantics + + this.ptr = ptr; + } + + public ImagePacket At(Timestamp timestamp) + { + return At(timestamp); + } + + public override Image Get() + { + UnsafeNativeMethods.mp_Packet__GetImage(mpPtr, out var imagePtr).Assert(); + + GC.KeepAlive(this); + return new Image(imagePtr, false); + } + + public override Image Consume() + { + UnsafeNativeMethods.mp_Packet__ConsumeImage(mpPtr, out var statusPtr, out var imagePtr).Assert(); + + GC.KeepAlive(this); + AssertStatusOk(statusPtr); + return new Image(imagePtr, true); + } + + public override Status ValidateAsType() + { + UnsafeNativeMethods.mp_Packet__ValidateAsImage(mpPtr, out var statusPtr).Assert(); + + GC.KeepAlive(this); + return new Status(statusPtr); + } + } +} diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/ImagePacket.cs.meta b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/ImagePacket.cs.meta new file mode 100644 index 000000000..7c72a901e --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Packet/ImagePacket.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 0c3ef771d0a5039b1b9073de07802ed9 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/Image_Safe.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/Image_Safe.cs new file mode 100644 index 000000000..5f09657ec --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/Image_Safe.cs @@ -0,0 +1,43 @@ +// Copyright (c) 2021 homuler +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file or at +// https://opensource.org/licenses/MIT. + +using System; +using System.Diagnostics.Contracts; +using System.Runtime.InteropServices; + +namespace Mediapipe +{ + internal static partial class SafeNativeMethods + { + [Pure, DllImport(MediaPipeLibrary, ExactSpelling = true)] + public static extern int mp_Image__width(IntPtr image); + + [Pure, DllImport(MediaPipeLibrary, ExactSpelling = true)] + public static extern int mp_Image__height(IntPtr image); + + [Pure, DllImport(MediaPipeLibrary, ExactSpelling = true)] + public static extern int mp_Image__channels(IntPtr image); + + [Pure, DllImport(MediaPipeLibrary, ExactSpelling = true)] + public static extern int mp_Image__step(IntPtr image); + + [Pure, DllImport(MediaPipeLibrary, ExactSpelling = true)] + [return: MarshalAs(UnmanagedType.I1)] + public static extern bool mp_Image__UsesGpu(IntPtr image); + + [Pure, DllImport(MediaPipeLibrary, ExactSpelling = true)] + public static extern ImageFormat.Types.Format mp_Image__image_format(IntPtr image); + + [Pure, DllImport(MediaPipeLibrary, ExactSpelling = true)] + public static extern GpuBufferFormat mp_Image__format(IntPtr image); + + #region PixelWriteLock + [Pure, DllImport(MediaPipeLibrary, ExactSpelling = true)] + [return: MarshalAs(UnmanagedType.I1)] + public static extern IntPtr mp_PixelWriteLock__Pixels(IntPtr pixelWriteLock); + #endregion + } +} diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/Image_Safe.cs.meta b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/Image_Safe.cs.meta new file mode 100644 index 000000000..ce7f3eab9 --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/Image_Safe.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: dff907bdc68ea6bd3aa06cb59e4f35dc +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/Image_Unsafe.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/Image_Unsafe.cs new file mode 100644 index 000000000..f13096aa2 --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/Image_Unsafe.cs @@ -0,0 +1,61 @@ +// Copyright (c) 2023 homuler +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file or at +// https://opensource.org/licenses/MIT. + +using System; +using System.Runtime.InteropServices; + +namespace Mediapipe +{ + internal static partial class UnsafeNativeMethods + { + [DllImport(MediaPipeLibrary, ExactSpelling = true)] + public static extern MpReturnCode mp_Image__ui_i_i_i_Pui8_PF( + ImageFormat.Types.Format format, int width, int height, int widthStep, IntPtr pixelData, + [MarshalAs(UnmanagedType.FunctionPtr)] ImageFrame.Deleter deleter, out IntPtr image); + +#if UNITY_EDITOR_LINUX || UNITY_STANDALONE_LINUX || UNITY_ANDROID + [DllImport(MediaPipeLibrary, ExactSpelling = true)] + public static extern MpReturnCode mp_Image__ui_ui_i_i_ui_PF_PSgc( + uint target, uint name, int width, int height, GpuBufferFormat format, + [MarshalAs(UnmanagedType.FunctionPtr)] GlTextureBuffer.DeletionCallback deletionCallback, + IntPtr producerContext, out IntPtr image); +#endif + + [DllImport(MediaPipeLibrary, ExactSpelling = true)] + public static extern void mp_Image__delete(IntPtr image); + + [DllImport(MediaPipeLibrary, ExactSpelling = true)] + public static extern MpReturnCode mp_Image__ConvertToCpu(IntPtr image, [MarshalAs(UnmanagedType.I1)] out bool result); + + [DllImport(MediaPipeLibrary, ExactSpelling = true)] + public static extern MpReturnCode mp_Image__ConvertToGpu(IntPtr image, [MarshalAs(UnmanagedType.I1)] out bool result); + + #region PixelWriteLock + [DllImport(MediaPipeLibrary, ExactSpelling = true)] + public static extern MpReturnCode mp_PixelWriteLock__RI(IntPtr image, out IntPtr pixelWriteLock); + + [DllImport(MediaPipeLibrary, ExactSpelling = true)] + public static extern void mp_PixelWriteLock__delete(IntPtr pixelWriteLock); + #endregion + + #region Packet + [DllImport(MediaPipeLibrary, ExactSpelling = true)] + public static extern MpReturnCode mp__MakeImagePacket__Pif(IntPtr image, out IntPtr packet); + + [DllImport(MediaPipeLibrary, ExactSpelling = true)] + public static extern MpReturnCode mp__MakeImagePacket_At__Pif_Rt(IntPtr image, IntPtr timestamp, out IntPtr packet); + + [DllImport(MediaPipeLibrary, ExactSpelling = true)] + public static extern MpReturnCode mp_Packet__ConsumeImage(IntPtr packet, out IntPtr status, out IntPtr image); + + [DllImport(MediaPipeLibrary, ExactSpelling = true)] + public static extern MpReturnCode mp_Packet__GetImage(IntPtr packet, out IntPtr image); + + [DllImport(MediaPipeLibrary, ExactSpelling = true)] + public static extern MpReturnCode mp_Packet__ValidateAsImage(IntPtr packet, out IntPtr status); + #endregion + } +} diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/Image_Unsafe.cs.meta b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/Image_Unsafe.cs.meta new file mode 100644 index 000000000..1eef5d605 --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/PInvoke/NativeMethods/Framework/Format/Image_Unsafe.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 9c513829d76a62e18842ed576e268772 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/com.github.homuler.mediapipe/Tests/EditMode/Framework/Format/ImageTest.cs b/Packages/com.github.homuler.mediapipe/Tests/EditMode/Framework/Format/ImageTest.cs new file mode 100644 index 000000000..ccd1b43e7 --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Tests/EditMode/Framework/Format/ImageTest.cs @@ -0,0 +1,223 @@ +// Copyright (c) 2023 homuler +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file or at +// https://opensource.org/licenses/MIT. + +using NUnit.Framework; +using System; +using System.Runtime.InteropServices; +using Unity.Collections; +using UnityEngine; + +namespace Mediapipe.Tests +{ + public class ImageTest + { + #region Constructor + [Test] + public void Ctor_ShouldInstantiateCpuImage() + { + var pixelData = BuildPixelData(); + + using (var image = new Image(ImageFormat.Types.Format.Srgba, 4, 2, 16, pixelData)) + { + Assert.AreEqual(ImageFormat.Types.Format.Srgba, image.ImageFormat()); + Assert.AreEqual(GpuBufferFormat.kBGRA32, image.Format()); + Assert.AreEqual(4, image.Width()); + Assert.AreEqual(2, image.Height()); + Assert.AreEqual(4, image.Channels()); + Assert.AreEqual(16, image.Step()); + Assert.False(image.UsesGpu()); + } + } + +#if UNITY_EDITOR_LINUX || UNITY_STANDALONE_LINUX || UNITY_ANDROID + [Test, GpuOnly] + public void Ctor_ShouldInstantiateGpuImage() + { + var texture = new Texture2D(4, 2, TextureFormat.RGBA32, false); + using (var image = new Image((uint)texture.GetNativeTexturePtr(), 4, 2, GpuBufferFormat.kBGRA32, OnRelease, GetGlContext())) + { + Assert.AreEqual(ImageFormat.Types.Format.Srgba, image.ImageFormat()); + Assert.AreEqual(GpuBufferFormat.kBGRA32, image.Format()); + Assert.AreEqual(4, image.Width()); + Assert.AreEqual(2, image.Height()); + Assert.AreEqual(4, image.Channels()); + Assert.AreEqual(16, image.Step()); + Assert.True(image.UsesGpu()); + } + } +#endif + #endregion + + #region ConvertToCpu + [Test] + public void ConvertToCpu_ShouldReturnTrue_When_ImageIsOnCpu() + { + var pixelData = BuildPixelData(); + + using (var image = new Image(ImageFormat.Types.Format.Srgba, 4, 2, 16, pixelData)) + { + Assert.False(image.UsesGpu()); + Assert.True(image.ConvertToCpu()); + } + } + +#if UNITY_EDITOR_LINUX || UNITY_STANDALONE_LINUX || UNITY_ANDROID + [Test, GpuOnly] + public void ConvertToCpu_ShouldReturnTrue_When_ImageIsOnGpu() + { + var texture = new Texture2D(4, 2, TextureFormat.RGBA32, false); + using (var image = new Image((uint)texture.GetNativeTexturePtr(), 4, 2, GpuBufferFormat.kBGRA32, OnRelease, GetGlContext())) + { + Assert.True(image.UsesGpu()); + Assert.True(image.ConvertToCpu()); + } + } +#endif + #endregion + + #region ConvertToGpu + [Test, GpuOnly] + public void ConvertToGpu_ShouldReturnTrue_If_GpuIsSuppored() + { + var pixelData = BuildPixelData(); + + using (var image = new Image(ImageFormat.Types.Format.Srgba, 4, 2, 16, pixelData)) + { + Assert.False(image.UsesGpu()); + + RunInGlContext(() => { Assert.True(image.ConvertToGpu()); }); + } + } + +#if UNITY_EDITOR_LINUX || UNITY_STANDALONE_LINUX || UNITY_ANDROID + [Test, GpuOnly] + public void ConvertToGpu_ShouldReturnTrue_When_ImageIsOnGpu() + { + var texture = new Texture2D(4, 2, TextureFormat.RGBA32, false); + RunInGlContext(() => + { + var glContext = GlContext.GetCurrent(); + using (var image = new Image((uint)texture.GetNativeTexturePtr(), 4, 2, GpuBufferFormat.kBGRA32, OnRelease, glContext)) + { + Assert.True(image.UsesGpu()); + Assert.True(image.ConvertToGpu()); + } + }); + } +#endif + #endregion + + #region PixelWriteLock + [Test] + public void PixelWriteLock_CanBeInitializedRepeatedly() + { + var pixelData = BuildPixelData(); + + using (var image = new Image(ImageFormat.Types.Format.Srgba, 2, 2, 8, pixelData)) + { + var readLock = new PixelWriteLock(image); + readLock.Dispose(); + readLock = new PixelWriteLock(image); + readLock.Dispose(); + } + } + + [Test] + public void Pixels_ShouldReturnPixelData_When_ImageIsOnCpu() + { + var bytes = new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }; + var pixelData = BuildPixelData(bytes); + + using (var image = new Image(ImageFormat.Types.Format.Srgba, 2, 2, 8, pixelData)) + { + Assert.False(image.UsesGpu()); + + using (var readLock = new PixelWriteLock(image)) + { + var ptr = readLock.Pixels(); + var outs = new byte[16]; + Marshal.Copy(ptr, outs, 0, 16); + Assert.AreEqual(bytes, outs); + } + } + } + + [Test, GpuOnly] + public void Pixels_ShouldReturnPixelData_When_ImageIsOnGpu() + { + var bytes = new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }; + var pixelData = BuildPixelData(bytes); + + using (var image = new Image(ImageFormat.Types.Format.Srgba, 4, 2, 16, pixelData)) + { + Assert.False(image.UsesGpu()); + + RunInGlContext(() => + { + Assert.True(image.ConvertToGpu()); + Assert.True(image.UsesGpu()); + + using (var readLock = new PixelWriteLock(image)) + { + var ptr = readLock.Pixels(); + var outs = new byte[16]; + Marshal.Copy(ptr, outs, 0, 16); + Assert.AreEqual(bytes, outs); + } + }); + } + } + #endregion + + private NativeArray BuildPixelData() + { + var srcBytes = new byte[] { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + }; + return BuildPixelData(srcBytes); + } + + private NativeArray BuildPixelData(byte[] bytes) + { + var pixelData = new NativeArray(bytes.Length, Allocator.Temp, NativeArrayOptions.UninitializedMemory); + pixelData.CopyFrom(bytes); + + return pixelData; + } + + private GlContext GetGlContext() + { + using (var glCalculatorHelper = new GlCalculatorHelper()) + { + glCalculatorHelper.InitializeForTest(GpuResources.Create()); + + return glCalculatorHelper.GetGlContext(); + } + } + + private static void RunInGlContext(Action action) + { + using (var glCalculatorHelper = new GlCalculatorHelper()) + { + glCalculatorHelper.InitializeForTest(GpuResources.Create()); + + var status = glCalculatorHelper.RunInGlContext(() => + { + action(); + }); + status.Dispose(); + } + } + + [AOT.MonoPInvokeCallback(typeof(GlTextureBuffer.DeletionCallback))] + private static void OnRelease(uint _, IntPtr syncTokenPtr) + { + var _glSyncToken = syncTokenPtr == IntPtr.Zero ? null : new GlSyncPoint(syncTokenPtr); + _glSyncToken?.Dispose(); + } + } +} diff --git a/Packages/com.github.homuler.mediapipe/Tests/EditMode/Framework/Format/ImageTest.cs.meta b/Packages/com.github.homuler.mediapipe/Tests/EditMode/Framework/Format/ImageTest.cs.meta new file mode 100644 index 000000000..984b4c91c --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Tests/EditMode/Framework/Format/ImageTest.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: afe29f972a1810a48b9c8836b504543b +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/com.github.homuler.mediapipe/Tests/EditMode/Framework/Packet/ImageFramePacketTest.cs b/Packages/com.github.homuler.mediapipe/Tests/EditMode/Framework/Packet/ImageFramePacketTest.cs index b34dd2e25..b67d65103 100644 --- a/Packages/com.github.homuler.mediapipe/Tests/EditMode/Framework/Packet/ImageFramePacketTest.cs +++ b/Packages/com.github.homuler.mediapipe/Tests/EditMode/Framework/Packet/ImageFramePacketTest.cs @@ -113,6 +113,7 @@ public void Get_ShouldThrowMediaPipeException_When_DataIsEmpty() } } + [Test] public void Get_ShouldReturnImageFrame_When_DataIsNotEmpty() { using (var packet = new ImageFramePacket(new ImageFrame(ImageFormat.Types.Format.Sbgra, 10, 10))) diff --git a/Packages/com.github.homuler.mediapipe/Tests/EditMode/Framework/Packet/ImagePacketTest.cs b/Packages/com.github.homuler.mediapipe/Tests/EditMode/Framework/Packet/ImagePacketTest.cs new file mode 100644 index 000000000..77a2322b7 --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Tests/EditMode/Framework/Packet/ImagePacketTest.cs @@ -0,0 +1,166 @@ +// Copyright (c) 2023 homuler +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file or at +// https://opensource.org/licenses/MIT. + +using NUnit.Framework; +using Unity.Collections; + +namespace Mediapipe.Tests +{ + public class ImagePacketTest + { + #region Constructor + [Test] + public void Ctor_ShouldInstantiatePacket_When_CalledWithNoArguments() + { + using (var packet = new ImagePacket()) + { + Assert.AreEqual(Status.StatusCode.Internal, packet.ValidateAsType().Code()); + var exception = Assert.Throws(() => { _ = packet.Consume(); }); + Assert.AreEqual(Status.StatusCode.Internal, exception.statusCode); + Assert.AreEqual(Timestamp.Unset(), packet.Timestamp()); + } + } + + [Test] + public void Ctor_ShouldInstantiatePacket_When_CalledWithValue() + { + var srcImage = BuildSRGBAImage(); + + using (var packet = new ImagePacket(srcImage)) + { + Assert.True(srcImage.isDisposed); + Assert.True(packet.ValidateAsType().Ok()); + Assert.AreEqual(Timestamp.Unset(), packet.Timestamp()); + + var image = packet.Consume(); + Assert.AreEqual(ImageFormat.Types.Format.Srgba, image.ImageFormat()); + } + } + + [Test] + public void Ctor_ShouldInstantiatePacket_When_CalledWithValueAndTimestamp() + { + var srcImage = BuildSRGBAImage(); + + using (var timestamp = new Timestamp(1)) + { + using (var packet = new ImagePacket(srcImage, timestamp)) + { + Assert.True(srcImage.isDisposed); + Assert.True(packet.ValidateAsType().Ok()); + + var image = packet.Consume(); + Assert.AreEqual(ImageFormat.Types.Format.Srgba, image.ImageFormat()); + Assert.AreEqual(timestamp, packet.Timestamp()); + } + } + } + #endregion + + #region #isDisposed + [Test] + public void IsDisposed_ShouldReturnFalse_When_NotDisposedYet() + { + using (var packet = new ImagePacket()) + { + Assert.False(packet.isDisposed); + } + } + + [Test] + public void IsDisposed_ShouldReturnTrue_When_AlreadyDisposed() + { + var packet = new ImagePacket(); + packet.Dispose(); + + Assert.True(packet.isDisposed); + } + #endregion + + #region #At + [Test] + public void At_ShouldReturnNewPacketWithTimestamp() + { + using (var timestamp = new Timestamp(1)) + { + var packet = new ImagePacket(BuildSRGBAImage()).At(timestamp); + Assert.AreEqual(timestamp, packet.Timestamp()); + + using (var newTimestamp = new Timestamp(2)) + { + var newPacket = packet.At(newTimestamp); + Assert.AreEqual(newTimestamp, newPacket.Timestamp()); + } + + Assert.AreEqual(timestamp, packet.Timestamp()); + } + } + #endregion + + #region #Get + [Test, SignalAbort] + public void Get_ShouldThrowMediaPipeException_When_DataIsEmpty() + { + using (var packet = new ImagePacket()) + { +#pragma warning disable IDE0058 + Assert.Throws(() => { packet.Get(); }); +#pragma warning restore IDE0058 + } + } + + [Test] + public void Get_ShouldReturnImage_When_DataIsNotEmpty() + { + using (var packet = new ImagePacket(BuildSRGBAImage())) + { + Assert.False(packet.IsEmpty()); + using (var image = packet.Get()) + { + Assert.AreEqual(ImageFormat.Types.Format.Srgba, image.ImageFormat()); + } + Assert.False(packet.IsEmpty()); + } + } + #endregion + + #region #Consume + [Test] + public void Consume_ShouldReturnImage() + { + using (var packet = new ImagePacket(BuildSRGBAImage())) + { + Assert.False(packet.IsEmpty()); + using (var image = packet.Consume()) + { + Assert.AreEqual(ImageFormat.Types.Format.Srgba, image.ImageFormat()); + } + Assert.True(packet.IsEmpty()); + } + } + #endregion + + #region #ValidateAsType + [Test] + public void ValidateAsType_ShouldReturnOk_When_ValueIsSet() + { + using (var packet = new ImagePacket(BuildSRGBAImage())) + { + Assert.True(packet.ValidateAsType().Ok()); + } + } + #endregion + + private Image BuildSRGBAImage() + { + var bytes = new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }; + var pixelData = new NativeArray(bytes.Length, Allocator.Temp, NativeArrayOptions.UninitializedMemory); + pixelData.CopyFrom(bytes); + + return new Image(ImageFormat.Types.Format.Srgba, 4, 2, 16, pixelData); + } + } +} diff --git a/Packages/com.github.homuler.mediapipe/Tests/EditMode/Framework/Packet/ImagePacketTest.cs.meta b/Packages/com.github.homuler.mediapipe/Tests/EditMode/Framework/Packet/ImagePacketTest.cs.meta new file mode 100644 index 000000000..06dbc8ad4 --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Tests/EditMode/Framework/Packet/ImagePacketTest.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 169fd9ca0b144ce5598f26936d4d7de4 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/mediapipe_api/BUILD b/mediapipe_api/BUILD index f99bcfc63..fd3726b9e 100644 --- a/mediapipe_api/BUILD +++ b/mediapipe_api/BUILD @@ -157,6 +157,7 @@ cc_library( "//mediapipe_api/framework:validated_graph_config", "//mediapipe_api/framework/formats:classification", "//mediapipe_api/framework/formats:detection", + "//mediapipe_api/framework/formats:image", "//mediapipe_api/framework/formats:image_frame", "//mediapipe_api/framework/formats:landmark", "//mediapipe_api/framework/formats:matrix_data", diff --git a/mediapipe_api/framework/formats/BUILD b/mediapipe_api/framework/formats/BUILD index cbcf694a0..640fb1b4a 100644 --- a/mediapipe_api/framework/formats/BUILD +++ b/mediapipe_api/framework/formats/BUILD @@ -37,6 +37,19 @@ cc_library( alwayslink = True, ) +cc_library( + name = "image", + srcs = ["image.cc"], + hdrs = ["image.h"], + deps = [ + "//mediapipe_api:common", + "//mediapipe_api/framework:packet", + "@com_google_absl//absl/status", + "@com_google_mediapipe//mediapipe/framework/formats:image", + ], + alwayslink = True, +) + cc_library( name = "image_frame", srcs = ["image_frame.cc"], diff --git a/mediapipe_api/framework/formats/image.cc b/mediapipe_api/framework/formats/image.cc new file mode 100644 index 000000000..3a9d041ab --- /dev/null +++ b/mediapipe_api/framework/formats/image.cc @@ -0,0 +1,127 @@ +#include "mediapipe_api/framework/formats/image.h" + +MpReturnCode mp_Image__ui_i_i_i_Pui8_PF(mediapipe::ImageFormat::Format format, int width, int height, int width_step, uint8* pixel_data, + Deleter* deleter, mediapipe::Image** image_out) { + TRY_ALL + *image_out = new mediapipe::Image{std::make_shared(format, width, height, width_step, pixel_data, deleter)}; + RETURN_CODE(MpReturnCode::Success); + CATCH_ALL +} + +#if !MEDIAPIPE_DISABLE_GPU + +#if !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER +MpReturnCode mp_Image__ui_ui_i_i_ui_PF_PSgc(GLenum target, GLuint name, int width, int height, mediapipe::GpuBufferFormat format, + GlTextureBufferDeletionCallback* deletion_callback, + std::shared_ptr* producer_context, + mediapipe::Image** image_out) { + TRY_ALL + auto callback = [name, deletion_callback](mediapipe::GlSyncToken token) -> void { + deletion_callback(name, new mediapipe::GlSyncToken{token}); + }; + *image_out = new mediapipe::Image{std::make_shared( + GL_TEXTURE_2D, + name, + width, + height, + format, + callback, + *producer_context + )}; + RETURN_CODE(MpReturnCode::Success); + CATCH_ALL +} +#endif + +#endif + +void mp_Image__delete(mediapipe::Image* image) { + delete image; +} + +int mp_Image__width(mediapipe::Image* image) { + return image->width(); +} + +int mp_Image__height(mediapipe::Image* image) { + return image->height(); +} + +int mp_Image__channels(mediapipe::Image* image) { + return image->channels(); +} + +int mp_Image__step(mediapipe::Image* image) { + return image->step(); +} + +bool mp_Image__UsesGpu(mediapipe::Image* image) { + return image->UsesGpu(); +} + +mediapipe::ImageFormat::Format mp_Image__image_format(mediapipe::Image* image) { + return image->image_format(); +} + +mediapipe::GpuBufferFormat mp_Image__format(mediapipe::Image* image) { + return image->format(); +} + +MpReturnCode mp_Image__ConvertToCpu(mediapipe::Image* image, bool* result_out) { + TRY_ALL + *result_out = image->ConvertToCpu(); + RETURN_CODE(MpReturnCode::Success); + CATCH_ALL +} + +MpReturnCode mp_Image__ConvertToGpu(mediapipe::Image* image, bool* result_out) { + TRY_ALL + *result_out = image->ConvertToGpu(); + RETURN_CODE(MpReturnCode::Success); + CATCH_ALL +} + +MpReturnCode mp_PixelWriteLock__RI(mediapipe::Image* image, mediapipe::PixelWriteLock** pixel_Write_lock_out) { + TRY + *pixel_Write_lock_out = new mediapipe::PixelWriteLock{image}; + RETURN_CODE(MpReturnCode::Success); + CATCH_EXCEPTION +} + +void mp_PixelWriteLock__delete(mediapipe::PixelWriteLock* pixel_Write_lock) { + delete pixel_Write_lock; +} + +uint8* mp_PixelWriteLock__Pixels(mediapipe::PixelWriteLock* pixel_read_lock) { + return pixel_read_lock->Pixels(); +} + +// Packet API +MpReturnCode mp__MakeImagePacket__Pif(mediapipe::Image* image, mediapipe::Packet** packet_out) { + TRY_ALL + *packet_out = new mediapipe::Packet{mediapipe::MakePacket(std::move(*image))}; + RETURN_CODE(MpReturnCode::Success); + CATCH_ALL +} + +MpReturnCode mp__MakeImagePacket_At__Pif_Rt(mediapipe::Image* image, mediapipe::Timestamp* timestamp, mediapipe::Packet** packet_out) { + TRY_ALL + *packet_out = new mediapipe::Packet{mediapipe::MakePacket(std::move(*image)).At(*timestamp)}; + RETURN_CODE(MpReturnCode::Success); + CATCH_ALL +} + +MpReturnCode mp_Packet__ConsumeImage(mediapipe::Packet* packet, absl::Status **status_out, mediapipe::Image** value_out) { + return mp_Packet__Consume(packet, status_out, value_out); +} + +MpReturnCode mp_Packet__GetImage(mediapipe::Packet* packet, const mediapipe::Image** value_out) { + return mp_Packet__Get(packet, value_out); +} + +MpReturnCode mp_Packet__ValidateAsImage(mediapipe::Packet* packet, absl::Status** status_out) { + TRY + *status_out = new absl::Status{packet->ValidateAsType()}; + RETURN_CODE(MpReturnCode::Success); + CATCH_EXCEPTION +} diff --git a/mediapipe_api/framework/formats/image.h b/mediapipe_api/framework/formats/image.h new file mode 100644 index 000000000..6dd89d27f --- /dev/null +++ b/mediapipe_api/framework/formats/image.h @@ -0,0 +1,70 @@ +// Copyright (c) 2023 homuler +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file or at +// https://opensource.org/licenses/MIT. + +#ifndef MEDIAPIPE_API_FRAMEWORK_FORMATS_IMAGE_H_ +#define MEDIAPIPE_API_FRAMEWORK_FORMATS_IMAGE_H_ + +#include +#include + +#include "absl/status/status.h" +#include "mediapipe/framework/formats/image.h" +#include "mediapipe_api/framework/packet.h" +#include "mediapipe_api/common.h" + +#if !MEDIAPIPE_DISABLE_GPU + +#if !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER // OSX, use GL textures. +#include "mediapipe/gpu/gl_texture_buffer.h" +#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER + +#endif // !MEDIAPIPE_DISABLE_GPU + +extern "C" { + +typedef void(Deleter)(uint8*); + +MP_CAPI(MpReturnCode) mp_Image__ui_i_i_i_Pui8_PF(mediapipe::ImageFormat::Format format, int width, int height, int width_step, uint8* pixel_data, + Deleter* deleter, mediapipe::Image** image_out); + +#if !MEDIAPIPE_DISABLE_GPU + +#if !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER +typedef void GlTextureBufferDeletionCallback(GLuint name, std::shared_ptr* sync_token); + +MP_CAPI(MpReturnCode) mp_Image__ui_ui_i_i_ui_PF_PSgc(GLenum target, GLuint name, int width, int height, mediapipe::GpuBufferFormat format, + GlTextureBufferDeletionCallback* deletion_callback, + std::shared_ptr* producer_context, + mediapipe::Image** image_out); +#endif + +#endif + +MP_CAPI(void) mp_Image__delete(mediapipe::Image* image); +MP_CAPI(int) mp_Image__width(mediapipe::Image* image); +MP_CAPI(int) mp_Image__height(mediapipe::Image* image); +MP_CAPI(int) mp_Image__channels(mediapipe::Image* image); +MP_CAPI(int) mp_Image__step(mediapipe::Image* image); +MP_CAPI(bool) mp_Image__UsesGpu(mediapipe::Image* image); +MP_CAPI(mediapipe::ImageFormat::Format) mp_Image__image_format(mediapipe::Image* image); +MP_CAPI(mediapipe::GpuBufferFormat) mp_Image__format(mediapipe::Image* image); +MP_CAPI(MpReturnCode) mp_Image__ConvertToCpu(mediapipe::Image* image, bool* result_out); +MP_CAPI(MpReturnCode) mp_Image__ConvertToGpu(mediapipe::Image* image, bool* result_out); + +MP_CAPI(MpReturnCode) mp_PixelWriteLock__RI(mediapipe::Image* image, mediapipe::PixelWriteLock** pixel_Write_lock_out); +MP_CAPI(void) mp_PixelWriteLock__delete(mediapipe::PixelWriteLock* pixel_Write_lock); +MP_CAPI(uint8*) mp_PixelWriteLock__Pixels(mediapipe::PixelWriteLock* pixel_read_lock); + +// Packet API +MP_CAPI(MpReturnCode) mp__MakeImagePacket__Pif(mediapipe::Image* image, mediapipe::Packet** packet_out); +MP_CAPI(MpReturnCode) mp__MakeImagePacket_At__Pif_Rt(mediapipe::Image* image, mediapipe::Timestamp* timestamp, mediapipe::Packet** packet_out); +MP_CAPI(MpReturnCode) mp_Packet__ConsumeImage(mediapipe::Packet* packet, absl::Status **status_out, mediapipe::Image** value_out); +MP_CAPI(MpReturnCode) mp_Packet__GetImage(mediapipe::Packet* packet, const mediapipe::Image** value_out); +MP_CAPI(MpReturnCode) mp_Packet__ValidateAsImage(mediapipe::Packet* packet, absl::Status** status_out); + +} // extern "C" + +#endif // MEDIAPIPE_API_FRAMEWORK_FORMATS_IMAGE_H_