Skip to content

Commit

Permalink
refactor!: define callbacks as static methods for IL2CPP
Browse files Browse the repository at this point in the history
  • Loading branch information
@homuler
homuler committed Jan 16, 2021
1 parent 9658aec commit 0172e01
Show file tree
Hide file tree
Showing 12 changed files with 153 additions and 138 deletions.
74 changes: 39 additions & 35 deletions Assets/MediaPipe/Examples/Scripts/DemoGraph.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,15 @@ public abstract class DemoGraph : MonoBehaviour, IDemoGraph<TextureFrame> {
[SerializeField] protected TextAsset config = null;

protected const string inputStream = "input_video";
protected CalculatorGraph graph;
protected GlCalculatorHelper gpuHelper;
protected static CalculatorGraph graph;
protected static GlCalculatorHelper gpuHelper;

#if UNITY_ANDROID
static readonly object frameLock = new object();
static Timestamp currentTimestamp;
static TextureFrame currentTextureFrame;
static IntPtr currentTextureName;
#endif

protected virtual void OnDestroy() {
Stop();
Expand All @@ -26,57 +33,54 @@ public abstract class DemoGraph : MonoBehaviour, IDemoGraph<TextureFrame> {
this.Initialize();

graph.SetGpuResources(gpuResources).AssertOk();
this.gpuHelper = gpuHelper;
DemoGraph.gpuHelper = gpuHelper;
}

public abstract Status StartRun();
public virtual Status StartRun(Texture texture) {
return StartRun();
}

/// <summary>
/// Convert <paramref name="colors" /> to a packet and send it to the input stream.
/// </summary>
public Status PushInput(TextureFrame textureFrame) {
var timestamp = new Timestamp(System.Environment.TickCount & System.Int32.MaxValue);
ImageFrame imageFrame = null;

if (!IsGpuEnabled()) {
imageFrame = new ImageFrame(
ImageFormat.Format.SRGBA, textureFrame.width, textureFrame.height, 4 * textureFrame.width, textureFrame.GetRawNativeByteArray());
textureFrame.Release();
var packet = new ImageFramePacket(imageFrame, timestamp);
#if !UNITY_ANDROID
var imageFrame = new ImageFrame(
ImageFormat.Format.SRGBA, textureFrame.width, textureFrame.height, 4 * textureFrame.width, textureFrame.GetRawNativeByteArray());
textureFrame.Release();
var packet = new ImageFramePacket(imageFrame, timestamp);

return graph.AddPacketToInputStream(inputStream, packet);
#else
lock (frameLock) {
currentTimestamp = timestamp;
currentTextureFrame = textureFrame;
currentTextureName = textureFrame.GetNativeTexturePtr();

return graph.AddPacketToInputStream(inputStream, packet);
return gpuHelper.RunInGlContext(PushInputInGlContext);
}
#endif
}

#if UNITY_ANDROID
var glTextureName = textureFrame.GetNativeTexturePtr();

return gpuHelper.RunInGlContext(() => {
/// <remarks>
/// <see cref="currentTimestamp" />, <see cref="currentTextureFrame" /> and <see cref="currentTextureName" /> must be set before calling.
/// </remarks>
[AOT.MonoPInvokeCallback(typeof(GlCalculatorHelper.NativeGlStatusFunction))]
static IntPtr PushInputInGlContext() {
try {
var glContext = GlContext.GetCurrent();
var glTextureBuffer = new GlTextureBuffer((UInt32)glTextureName, textureFrame.width, textureFrame.height,
textureFrame.gpuBufferformat, textureFrame.OnRelease, glContext);
var glTextureBuffer = new GlTextureBuffer((UInt32)currentTextureName, currentTextureFrame.width, currentTextureFrame.height,
currentTextureFrame.gpuBufferformat, currentTextureFrame.OnRelease, glContext);
var gpuBuffer = new GpuBuffer(glTextureBuffer);

return graph.AddPacketToInputStream(inputStream, new GpuBufferPacket(gpuBuffer, timestamp));
});
#else
imageFrame = new ImageFrame(
ImageFormat.Format.SRGBA, textureFrame.width, textureFrame.height, 4 * textureFrame.width, textureFrame.GetRawNativeByteArray());
textureFrame.Release();

return gpuHelper.RunInGlContext(() => {
var texture = gpuHelper.CreateSourceTexture(imageFrame);
var gpuBuffer = texture.GetGpuBufferFrame();
Gl.Flush();
texture.Release();
return graph.AddPacketToInputStream(inputStream, new GpuBufferPacket(gpuBuffer, timestamp));
});
#endif
// TODO: ensure the returned status won't be garbage collected prematurely.
return graph.AddPacketToInputStream(inputStream, new GpuBufferPacket(gpuBuffer, currentTimestamp)).mpPtr;
} catch (Exception e) {
return Status.FailedPrecondition(e.ToString()).mpPtr;
}
}
#endif

public abstract void RenderOutput(WebCamScreenController screenController, TextureFrame textureFrame);

Expand Down
27 changes: 21 additions & 6 deletions Assets/MediaPipe/Examples/Scripts/ResourceManager/AssetBundleManager.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,26 @@
using System.Threading.Tasks;
using UnityEngine;

/// <summary>
/// Sample implementation of ResourceManager, that reads files from AssetBundle.
/// </summary>
public sealed class AssetBundleManager : ResourceManager {
private static readonly Lazy<AssetBundleManager> lazy = new Lazy<AssetBundleManager>(() => new AssetBundleManager());
public static AssetBundleManager Instance { get { return lazy.Value; } }

private readonly static string CacheRootPath = Path.Combine(Application.persistentDataPath, "Cache");
private readonly static string ModelCacheRootPath = Path.Combine(CacheRootPath, "MediaPipe", "Models");
private readonly static string AssetBundlePath = Path.Combine(Application.streamingAssetsPath, "AssetBundles", "mediapipe", "models");

public override CacheFilePathResolver cacheFilePathResolver {
get { return CacheFileFromAsset; }
}

public override ReadFileHandler readFileHandler {
get { return ReadFile; }
}

private AssetBundleManager() : base() {
private AssetBundleManager() {
if (!Directory.Exists(ModelCacheRootPath)) {
Directory.CreateDirectory(ModelCacheRootPath);
}
Expand All @@ -20,7 +33,7 @@ public sealed class AssetBundleManager : ResourceManager {
public async Task LoadAllAssetsAsync() {
Debug.Log("Starting to load assets");

var bundleLoadReq = await AssetBundle.LoadFromFileAsync(Path.Combine(Application.streamingAssetsPath, "AssetBundles", "mediapipe", "models"));
var bundleLoadReq = await AssetBundle.LoadFromFileAsync(AssetBundlePath);

if (bundleLoadReq.assetBundle == null) {
Debug.LogError("Failed to load the AssetBundle");
Expand All @@ -38,7 +51,8 @@ public sealed class AssetBundleManager : ResourceManager {
Debug.Log("Loaded all assets");
}

protected override string CacheFileFromAsset(string assetPath) {
[AOT.MonoPInvokeCallback(typeof(CacheFilePathResolver))]
static string CacheFileFromAsset(string assetPath) {
var assetName = Path.GetFileNameWithoutExtension(assetPath);
var cachePath = CachePathFor(assetName);

Expand All @@ -49,7 +63,8 @@ public sealed class AssetBundleManager : ResourceManager {
return null;
}

protected override bool ReadFile(string path, IntPtr dst) {
[AOT.MonoPInvokeCallback(typeof(ReadFileHandler))]
static bool ReadFile(string path, IntPtr dst) {
var cachePath = CacheFileFromAsset(path);

if (!File.Exists(cachePath)) {
Expand All @@ -64,11 +79,11 @@ public sealed class AssetBundleManager : ResourceManager {
return true;
}

private string CachePathFor(string assetName) {
static string CachePathFor(string assetName) {
return Path.Combine(ModelCacheRootPath, assetName);
}

private async Task WriteCacheFileAsync(TextAsset asset) {
async Task WriteCacheFileAsync(TextAsset asset) {
var path = CachePathFor(asset.name);
var bytes = asset.bytes;
Debug.Log($"Saving {asset.name} to {path} (length={bytes.Length})");
Expand Down
18 changes: 13 additions & 5 deletions Assets/MediaPipe/Examples/Scripts/ResourceManager/LocalAssetManager.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,22 @@ public sealed class LocalAssetManager : ResourceManager {
public static LocalAssetManager Instance { get { return lazy.Value; } }
private readonly static string ModelRootPath = Path.Combine(Application.dataPath, "MediaPipe", "SDK", "Models");

private LocalAssetManager() : base() {}
public override CacheFilePathResolver cacheFilePathResolver {
get { return CacheFileFromAsset; }
}

public override ReadFileHandler readFileHandler {
get { return ReadFile; }
}

private LocalAssetManager() {}

/// <summary>dummy method</summary>
public async Task LoadAllAssetsAsync() {
await Task.CompletedTask;
}

protected override string CacheFileFromAsset(string assetPath) {
static string CacheFileFromAsset(string assetPath) {
var assetName = GetAssetName(assetPath);
var localPath = GetLocalFilePath(assetName);

Expand All @@ -31,7 +39,7 @@ public sealed class LocalAssetManager : ResourceManager {
return null;
}

protected override bool ReadFile(string path, IntPtr dst) {
static bool ReadFile(string path, IntPtr dst) {
try {
Debug.Log(path);
var localPath = CacheFileFromAsset(path);
Expand All @@ -48,14 +56,14 @@ public sealed class LocalAssetManager : ResourceManager {
}
}

private string GetAssetName(string assetPath) {
static string GetAssetName(string assetPath) {
var assetName = Path.GetFileNameWithoutExtension(assetPath);
var extension = Path.GetExtension(assetPath);

return (extension == ".tflite" || extension == ".bytes") ? $"{assetName}.bytes" : $"{assetName}.txt";
}

private string GetLocalFilePath(string assetName) {
static string GetLocalFilePath(string assetName) {
return Path.Combine(ModelRootPath, assetName);
}
}
8 changes: 0 additions & 8 deletions Assets/MediaPipe/Examples/Scripts/ResourceManager/TextureFrame.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,20 +12,12 @@ public class TextureFrame {
public int height { get; private set; }

public GlTextureBuffer.DeletionCallback OnRelease;
private GCHandle deletionCallbackHandle;

public TextureFrame(int width, int height, GlTextureBuffer.DeletionCallback OnRelease) {
texture = new Texture2D(width, height, TextureFormat.RGBA32, false);
this.width = width;
this.height = height;
this.OnRelease = OnRelease;
deletionCallbackHandle = GCHandle.Alloc(this.OnRelease, GCHandleType.Pinned);
}

~TextureFrame() {
if (deletionCallbackHandle.IsAllocated) {
deletionCallbackHandle.Free();
}
}

public void CopyTexture(Texture dst) {
Expand Down
19 changes: 10 additions & 9 deletions Assets/MediaPipe/Examples/Scripts/ResourceManager/TextureFramePool.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,16 +45,17 @@ public class TextureFramePool : MonoSingleton<TextureFramePool> {
return new TextureFrameRequest(this, callback);
}

private void OnTextureFrameRelease(UInt64 textureName, IntPtr syncTokenPtr) {
lock(((ICollection)textureFramesInUse).SyncRoot) {
if (!textureFramesInUse.TryGetValue(textureName, out var textureFrame)) {
[AOT.MonoPInvokeCallback(typeof(GlTextureBuffer.DeletionCallback))]
private static void OnTextureFrameRelease(UInt64 textureName, IntPtr syncTokenPtr) {
lock(((ICollection)Instance.textureFramesInUse).SyncRoot) {
if (!Instance.textureFramesInUse.TryGetValue(textureName, out var textureFrame)) {
Debug.LogWarning("The released texture does not belong to the pool");
return;
}

textureFramesInUse.Remove(textureName);
Instance.textureFramesInUse.Remove(textureName);

if (frameCount > poolSize || IsStale(textureFrame)) {
if (Instance.frameCount > Instance.poolSize || IsStale(textureFrame)) {
return;
}

Expand All @@ -63,13 +64,13 @@ public class TextureFramePool : MonoSingleton<TextureFramePool> {
glSyncToken.Wait();
}
}
availableTextureFrames.Enqueue(textureFrame);
Instance.availableTextureFrames.Enqueue(textureFrame);
}
}

private bool IsStale(TextureFrame textureFrame) {
lock(dimensionLock) {
return textureFrame.width != textureWidth || textureFrame.height != textureHeight;
private static bool IsStale(TextureFrame textureFrame) {
lock(Instance.dimensionLock) {
return textureFrame.width != Instance.textureWidth || textureFrame.height != Instance.textureHeight;
}
}

Expand Down
46 changes: 19 additions & 27 deletions Assets/MediaPipe/Examples/Scripts/SceneDirector.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ public class SceneDirector : MonoBehaviour {
bool IsAssetLoadFailed = false;

delegate void PluginCallback(int eventId);
GCHandle InitializeGpuHelperHandle;
IntPtr currentContext = IntPtr.Zero;

void OnEnable() {
var nameForGlog = Path.Combine(Application.dataPath, "MediaPipePlugin");
Expand All @@ -37,36 +37,23 @@ public class SceneDirector : MonoBehaviour {
Glog.Initialize(nameForGlog, logDir);
}

void InitializeGpuHelper(int eventId) {
#if UNITY_ANDROID
// context is EGL_NO_CONTEXT if the graphics API is not OpenGL ES
var context = Egl.getCurrentContext();

if (context == IntPtr.Zero) {
Debug.LogWarning("No EGL Context Found");
} else {
Debug.Log($"EGL Context Found ({context})");
}

gpuResources = GpuResources.Create(context).ConsumeValueOrDie();
#else
gpuResources = GpuResources.Create().ConsumeValueOrDie();
#endif

gpuHelper = new GlCalculatorHelper();
gpuHelper.InitializeForTest(gpuResources);
void GetCurrentContext(int eventId) {
currentContext = Egl.getCurrentContext();
}
#endif

async void Start() {
webCamScreen = GameObject.Find("WebCamScreen");

#if UNITY_ANDROID
if (useGPU) {
PluginCallback gpuHelperInitializer = InitializeGpuHelper;
InitializeGpuHelperHandle = GCHandle.Alloc(gpuHelperInitializer, GCHandleType.Pinned);
PluginCallback callback = GetCurrentContext;

var fp = Marshal.GetFunctionPointerForDelegate(gpuHelperInitializer);
var fp = Marshal.GetFunctionPointerForDelegate(callback);
GL.IssuePluginEvent(fp, 1);
}
#endif

#if UNITY_EDITOR
var resourceManager = LocalAssetManager.Instance;
Expand All @@ -87,11 +74,6 @@ public class SceneDirector : MonoBehaviour {

void OnDisable() {
StopGraph();

if (InitializeGpuHelperHandle.IsAllocated) {
InitializeGpuHelperHandle.Free();
}

Glog.Shutdown();
}

Expand Down Expand Up @@ -172,7 +154,17 @@ public class SceneDirector : MonoBehaviour {
var graph = graphContainer.GetComponent<IDemoGraph<TextureFrame>>();

if (useGPU) {
// TODO: have to wait for gpuHelper to be initialized.
// TODO: have to wait for currentContext to be initialized.
if (currentContext == IntPtr.Zero) {
Debug.LogWarning("No EGL Context Found");
} else {
Debug.Log($"EGL Context Found ({currentContext})");
}

gpuResources = GpuResources.Create(currentContext).ConsumeValueOrDie();
gpuHelper = new GlCalculatorHelper();
gpuHelper.InitializeForTest(gpuResources);

graph.Initialize(gpuResources, gpuHelper);
} else {
graph.Initialize();
Expand Down
7 changes: 4 additions & 3 deletions Assets/MediaPipe/SDK/Scripts/External/Protobuf.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,13 +36,14 @@ class Protobuf {
}

public delegate void ProtobufLogHandler(int level, string filename, int line, string message);
private static readonly ProtobufLogHandler protobufLogHandler = LogProtobufMessage;
static readonly ProtobufLogHandler protobufLogHandler = LogProtobufMessage;

private static void LogProtobufMessage(int level, string filename, int line, string message) {
[AOT.MonoPInvokeCallback(typeof(ProtobufLogHandler))]
static void LogProtobufMessage(int level, string filename, int line, string message) {
Debug.Log($"[libprotobuf {FormatProtobufLogLevel(level)} {filename}:{line}] {message}");
}

private static string FormatProtobufLogLevel(int level) {
static string FormatProtobufLogLevel(int level) {
switch (level) {
case 1: return "WARNING";
case 2: return "ERROR";
Expand Down
Loading

0 comments on commit 0172e01

Please sign in to comment.