Skip to content

Commit

Permalink
feat: build Image/ImageFrame from Texture2D (#1083)
Browse files Browse the repository at this point in the history 
* refactor: add ImageFormatExtension

* feat: build Image/ImageFrame from Texture2D

* refactor: format.NumberOfChannels

* refactor: implement ByteDepth as a extension method

* refactor: implement ChannelSize as a extension method

* refactor: use new constructors in TextureFrame
  • Loading branch information
@homuler
homuler committed Jan 1, 2024
1 parent deeb9f0 commit bcce0e9
Show file tree
Hide file tree
Showing 7 changed files with 160 additions and 122 deletions.
121 changes: 121 additions & 0 deletions Packages/com.github.homuler.mediapipe/Runtime/Scripts/Extension/ImageFormatExtension.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
// Copyright (c) 2023 homuler
//
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file or at
// https://opensource.org/licenses/MIT.

namespace Mediapipe
{
public static class ImageFormatExtension
{
/// <returns>
/// The number of channels for a <paramref name="format" />.
/// If channels don't make sense in the <paramref name="format" />, returns <c>0</c>.
/// </returns>
/// <remarks>
/// Unlike the original implementation, this API won't signal SIGABRT.
/// </remarks>
public static int NumberOfChannels(this ImageFormat.Types.Format format)
{
switch (format)
{
case ImageFormat.Types.Format.Srgb:
case ImageFormat.Types.Format.Srgb48:
return 3;
case ImageFormat.Types.Format.Srgba:
case ImageFormat.Types.Format.Srgba64:
case ImageFormat.Types.Format.Sbgra:
return 4;
case ImageFormat.Types.Format.Gray8:
case ImageFormat.Types.Format.Gray16:
return 1;
case ImageFormat.Types.Format.Vec32F1:
return 1;
case ImageFormat.Types.Format.Vec32F2:
return 2;
case ImageFormat.Types.Format.Vec32F4:
return 4;
case ImageFormat.Types.Format.Lab8:
return 3;
case ImageFormat.Types.Format.Ycbcr420P:
case ImageFormat.Types.Format.Ycbcr420P10:
case ImageFormat.Types.Format.Unknown:
default:
return 0;
}
}

/// <returns>
/// The depth of each channel in bytes for a <paramref name="format" />.
/// If channels don't make sense in the <paramref name="format" />, returns <c>0</c>.
/// </returns>
/// <remarks>
/// Unlike the original implementation, this API won't signal SIGABRT.
/// </remarks>
public static int ByteDepth(this ImageFormat.Types.Format format)
{
switch (format)
{
case ImageFormat.Types.Format.Srgb:
case ImageFormat.Types.Format.Srgba:
case ImageFormat.Types.Format.Sbgra:
return 1;
case ImageFormat.Types.Format.Srgb48:
case ImageFormat.Types.Format.Srgba64:
return 2;
case ImageFormat.Types.Format.Gray8:
return 1;
case ImageFormat.Types.Format.Gray16:
return 2;
case ImageFormat.Types.Format.Vec32F1:
case ImageFormat.Types.Format.Vec32F2:
case ImageFormat.Types.Format.Vec32F4:
return 4;
case ImageFormat.Types.Format.Lab8:
return 1;
case ImageFormat.Types.Format.Ycbcr420P:
case ImageFormat.Types.Format.Ycbcr420P10:
case ImageFormat.Types.Format.Unknown:
default:
return 0;
}
}

/// <returns>
/// The channel size for a <paramref name="format" />.
/// If channels don't make sense in the <paramref name="format" />, returns <c>0</c>.
/// </returns>
/// <remarks>
/// Unlike the original implementation, this API won't signal SIGABRT.
/// </remarks>
public static int ChannelSize(this ImageFormat.Types.Format format)
{
switch (format)
{
case ImageFormat.Types.Format.Srgb:
case ImageFormat.Types.Format.Srgba:
case ImageFormat.Types.Format.Sbgra:
return sizeof(byte);
case ImageFormat.Types.Format.Srgb48:
case ImageFormat.Types.Format.Srgba64:
return sizeof(ushort);
case ImageFormat.Types.Format.Gray8:
return sizeof(byte);
case ImageFormat.Types.Format.Gray16:
return sizeof(ushort);
case ImageFormat.Types.Format.Vec32F1:
case ImageFormat.Types.Format.Vec32F2:
case ImageFormat.Types.Format.Vec32F4:
// sizeof float may be wrong since it's platform-dependent, but we assume that it's constant across all supported platforms.
return sizeof(float);
case ImageFormat.Types.Format.Lab8:
return sizeof(byte);
case ImageFormat.Types.Format.Ycbcr420P:
case ImageFormat.Types.Format.Ycbcr420P10:
case ImageFormat.Types.Format.Unknown:
default:
return 0;
}
}
}
}
11 changes: 11 additions & 0 deletions Packages/com.github.homuler.mediapipe/Runtime/Scripts/Extension/ImageFormatExtension.cs.meta
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 6 additions & 0 deletions Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Formats/Image.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
using System;
using Unity.Collections;
using Unity.Collections.LowLevel.Unsafe;
using UnityEngine;

namespace Mediapipe
{
Expand Down Expand Up @@ -43,6 +44,11 @@ public Image(ImageFormat.Types.Format format, int width, int height, int widthSt
: this(format, width, height, widthStep, pixelData, _VoidDeleter)
{ }

// TODO: detect format from the texture
public Image(ImageFormat.Types.Format format, Texture2D texture) :
this(format, texture.width, texture.height, format.NumberOfChannels() * texture.width, texture.GetRawTextureData<byte>())
{ }

#if UNITY_EDITOR_LINUX || UNITY_STANDLONE_LINUX || UNITY_ANDROID
public Image(uint target, uint name, int width, int height, GpuBufferFormat format, GlTextureBuffer.DeletionCallback callback, GlContext glContext) : base()
{
Expand Down
118 changes: 9 additions & 109 deletions Packages/com.github.homuler.mediapipe/Runtime/Scripts/Framework/Formats/ImageFrame.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
using System;
using Unity.Collections;
using Unity.Collections.LowLevel.Unsafe;
using UnityEngine;

namespace Mediapipe
{
Expand Down Expand Up @@ -65,6 +66,11 @@ public ImageFrame(ImageFormat.Types.Format format, int width, int height, int wi
: this(format, width, height, widthStep, pixelData, _VoidDeleter)
{ }

// TODO: detect format from the texture
public ImageFrame(ImageFormat.Types.Format format, Texture2D texture) :
this(format, texture.width, texture.height, format.NumberOfChannels() * texture.width, texture.GetRawTextureData<byte>())
{ }

protected override void DeleteMpPtr()
{
UnsafeNativeMethods.mp_ImageFrame__delete(ptr);
Expand All @@ -75,112 +81,6 @@ protected override void DeleteMpPtr()
[AOT.MonoPInvokeCallback(typeof(Deleter))]
internal static void VoidDeleter(IntPtr _) { }

/// <returns>
/// The number of channels for a <paramref name="format" />.
/// If channels don't make sense in the <paramref name="format" />, returns <c>0</c>.
/// </returns>
/// <remarks>
/// Unlike the original implementation, this API won't signal SIGABRT.
/// </remarks>
public static int NumberOfChannelsForFormat(ImageFormat.Types.Format format)
{
switch (format)
{
case ImageFormat.Types.Format.Srgb:
case ImageFormat.Types.Format.Srgb48:
return 3;
case ImageFormat.Types.Format.Srgba:
case ImageFormat.Types.Format.Srgba64:
case ImageFormat.Types.Format.Sbgra:
return 4;
case ImageFormat.Types.Format.Gray8:
case ImageFormat.Types.Format.Gray16:
return 1;
case ImageFormat.Types.Format.Vec32F1:
return 1;
case ImageFormat.Types.Format.Vec32F2:
return 2;
case ImageFormat.Types.Format.Lab8:
return 3;
case ImageFormat.Types.Format.Ycbcr420P:
case ImageFormat.Types.Format.Ycbcr420P10:
case ImageFormat.Types.Format.Unknown:
default:
return 0;
}
}

/// <returns>
/// The channel size for a <paramref name="format" />.
/// If channels don't make sense in the <paramref name="format" />, returns <c>0</c>.
/// </returns>
/// <remarks>
/// Unlike the original implementation, this API won't signal SIGABRT.
/// </remarks>
public static int ChannelSizeForFormat(ImageFormat.Types.Format format)
{
switch (format)
{
case ImageFormat.Types.Format.Srgb:
case ImageFormat.Types.Format.Srgba:
case ImageFormat.Types.Format.Sbgra:
return sizeof(byte);
case ImageFormat.Types.Format.Srgb48:
case ImageFormat.Types.Format.Srgba64:
return sizeof(ushort);
case ImageFormat.Types.Format.Gray8:
return sizeof(byte);
case ImageFormat.Types.Format.Gray16:
return sizeof(ushort);
case ImageFormat.Types.Format.Vec32F1:
case ImageFormat.Types.Format.Vec32F2:
// sizeof float may be wrong since it's platform-dependent, but we assume that it's constant across all supported platforms.
return sizeof(float);
case ImageFormat.Types.Format.Lab8:
return sizeof(byte);
case ImageFormat.Types.Format.Ycbcr420P:
case ImageFormat.Types.Format.Ycbcr420P10:
case ImageFormat.Types.Format.Unknown:
default:
return 0;
}
}

/// <returns>
/// The depth of each channel in bytes for a <paramref name="format" />.
/// If channels don't make sense in the <paramref name="format" />, returns <c>0</c>.
/// </returns>
/// <remarks>
/// Unlike the original implementation, this API won't signal SIGABRT.
/// </remarks>
public static int ByteDepthForFormat(ImageFormat.Types.Format format)
{
switch (format)
{
case ImageFormat.Types.Format.Srgb:
case ImageFormat.Types.Format.Srgba:
case ImageFormat.Types.Format.Sbgra:
return 1;
case ImageFormat.Types.Format.Srgb48:
case ImageFormat.Types.Format.Srgba64:
return 2;
case ImageFormat.Types.Format.Gray8:
return 1;
case ImageFormat.Types.Format.Gray16:
return 2;
case ImageFormat.Types.Format.Vec32F1:
case ImageFormat.Types.Format.Vec32F2:
return 4;
case ImageFormat.Types.Format.Lab8:
return 1;
case ImageFormat.Types.Format.Ycbcr420P:
case ImageFormat.Types.Format.Ycbcr420P10:
case ImageFormat.Types.Format.Unknown:
default:
return 0;
}
}

public bool IsEmpty()
{
return SafeNativeMethods.mp_ImageFrame__IsEmpty(mpPtr);
Expand Down Expand Up @@ -223,7 +123,7 @@ public int Height()
/// </remarks>
public int ChannelSize()
{
return ChannelSizeForFormat(Format());
return Format().ChannelSize();
}

/// <returns>
Expand All @@ -235,7 +135,7 @@ public int ChannelSize()
/// </remarks>
public int NumberOfChannels()
{
return NumberOfChannelsForFormat(Format());
return Format().NumberOfChannels();
}

/// <returns>
Expand All @@ -247,7 +147,7 @@ public int NumberOfChannels()
/// </remarks>
public int ByteDepth()
{
return ByteDepthForFormat(Format());
return Format().ByteDepth();
}

public int WidthStep()
Expand Down
4 changes: 2 additions & 2 deletions Packages/com.github.homuler.mediapipe/Runtime/Scripts/Unity/Experimental/TextureFrame.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -222,9 +222,9 @@ public IntPtr GetNativeTexturePtr()

public Guid GetInstanceID() => _instanceId;

public ImageFrame BuildImageFrame() => new ImageFrame(imageFormat, width, height, 4 * width, GetRawTextureData<byte>());
public ImageFrame BuildImageFrame() => new ImageFrame(imageFormat, _texture);

public Image BuildCPUImage() => new Image(imageFormat, width, height, 4 * width, GetRawTextureData<byte>());
public Image BuildCPUImage() => new Image(imageFormat, _texture);

public GpuBuffer BuildGpuBuffer(GlContext glContext)
{
Expand Down
Loading

0 comments on commit bcce0e9

Please sign in to comment.