Skip to content

Commit

Permalink
fix: wait for WebCamTexture to be initialized
Browse files Browse the repository at this point in the history
  • Loading branch information
@homuler
homuler committed Jan 16, 2021
1 parent 5e2574d commit 8fd204c
Show file tree
Hide file tree
Showing 2 changed files with 147 additions and 56 deletions.
141 changes: 95 additions & 46 deletions Assets/MediaPipe/Examples/Scripts/SceneDirector.cs
View file
Original file line number Diff line number Diff line change
@@ -1,25 +1,30 @@
using Mediapipe;
using System;
using System.IO;
using System.Collections;
using System.Runtime.InteropServices;

using Mediapipe;
using UnityEngine;

#if UNITY_ANDROID
using System.Runtime.InteropServices;
#endif

using Directory = System.IO.Directory;

public class SceneDirector : MonoBehaviour {
[SerializeField] bool useGPU = true;

readonly object graphLock = new object();
WebCamDevice? webCamDevice;
GameObject webCamScreen;
Coroutine cameraSetupCoroutine;
GameObject graphPrefab;
GameObject graphContainer;
Coroutine graphRunner;

GpuResources gpuResources;
GlCalculatorHelper gpuHelper;

const int MAX_WAIT_FRAME = 500;

const int MAX_WAIT_FRAME = 1000;
bool IsAssetLoaded = false;
bool IsAssetLoadFailed = false;

Expand Down Expand Up @@ -73,22 +78,49 @@ public class SceneDirector : MonoBehaviour {
}

void OnDisable() {
StopGraph();
DestroyGraph();
StopCamera();
Glog.Shutdown();
}

public void ChangeWebCamDevice(WebCamDevice? webCamDevice) {
webCamScreen.GetComponent<WebCamScreenController>().ResetScreen(webCamDevice);
lock (graphLock) {
ResetCamera(webCamDevice);

if (graphPrefab != null) {
StopGraph();
StartGraph();
}
}
}

void ResetCamera(WebCamDevice? webCamDevice) {
StopCamera();
cameraSetupCoroutine = StartCoroutine(webCamScreen.GetComponent<WebCamScreenController>().ResetScreen(webCamDevice));
this.webCamDevice = webCamDevice;
}

void StopCamera() {
if (cameraSetupCoroutine != null) {
StopCoroutine(cameraSetupCoroutine);
cameraSetupCoroutine = null;
}
}

public void ChangeGraph(GameObject graphPrefab) {
StopGraph();
this.graphPrefab = graphPrefab;
StartGraph();
lock (graphLock) {
DestroyGraph();
this.graphPrefab = graphPrefab;

if (webCamDevice != null) {
StartGraph();
}
}
}

void StartGraph() {
if (graphRunner != null) {
Debug.Log("The graph is already running");
return;
}

Expand All @@ -100,56 +132,39 @@ public class SceneDirector : MonoBehaviour {
StopCoroutine(graphRunner);
graphRunner = null;
}
}

void DestroyGraph() {
StopGraph();

if (graphContainer != null) {
Destroy(graphContainer);;
Destroy(graphContainer);
}
}

IEnumerator RunGraph() {
var webCamScreenController = webCamScreen.GetComponent<WebCamScreenController>();
var waitFrame = MAX_WAIT_FRAME;

yield return new WaitWhile(() => {
waitFrame--;
var isGraphPrefabPresent = graphPrefab != null;
var isWebCamPlaying = webCamScreenController.isPlaying;
if (!isGraphPrefabPresent && waitFrame % 10 == 0) {
Debug.Log($"Waiting for a graph");
}
yield return WaitForAssets();

if (!isWebCamPlaying && waitFrame % 10 == 0) {
Debug.Log($"Waiting for a WebCamDevice");
}
if (IsAssetLoadFailed) {
Debug.LogError("Failed to load assets. Stopping...");
yield break;
}

return (!isGraphPrefabPresent || !isWebCamPlaying) && waitFrame > 0;
});
yield return WaitForGraph();

if (graphPrefab == null) {
Debug.LogWarning("No graph is set. Stopping...");
yield break;
}

var webCamScreenController = webCamScreen.GetComponent<WebCamScreenController>();
yield return WaitForCamera(webCamScreenController);

if (!webCamScreenController.isPlaying) {
Debug.LogWarning("WebCamDevice is not working. Stopping...");
yield break;
}

webCamScreenController.InitScreen();

if (!IsAssetLoaded && !IsAssetLoadFailed) {
Debug.Log("Waiting for assets to be loaded...");
}

yield return new WaitUntil(() => IsAssetLoaded || IsAssetLoadFailed);

if (IsAssetLoadFailed) {
Debug.LogError("Failed to load assets. Stopping...");
yield break;
}

graphContainer = Instantiate(graphPrefab);
var graph = graphContainer.GetComponent<IDemoGraph<TextureFrame>>();

Expand All @@ -175,11 +190,6 @@ public class SceneDirector : MonoBehaviour {
while (true) {
yield return new WaitForEndOfFrame();

if (!webCamScreenController.isPlaying) {
Debug.LogWarning("WebCam is not working");
break;
}

var nextFrameRequest = webCamScreenController.RequestNextFrame();
yield return nextFrameRequest;

Expand All @@ -189,4 +199,43 @@ public class SceneDirector : MonoBehaviour {
graph.RenderOutput(webCamScreenController, nextFrame);
}
}

IEnumerator WaitForAssets() {
if (!IsAssetLoaded && !IsAssetLoadFailed) {
Debug.Log("Waiting for assets to be loaded...");
}

yield return new WaitUntil(() => IsAssetLoaded || IsAssetLoadFailed);
}

IEnumerator WaitForGraph() {
var waitFrame = MAX_WAIT_FRAME;

yield return new WaitUntil(() => {
waitFrame--;
var isGraphPrefabPresent = graphPrefab != null;
if (!isGraphPrefabPresent && waitFrame % 50 == 0) {
Debug.Log($"Waiting for a graph");
}
return isGraphPrefabPresent || waitFrame < 0;
});
}

IEnumerator WaitForCamera(WebCamScreenController webCamScreenController) {
var waitFrame = MAX_WAIT_FRAME;

yield return new WaitUntil(() => {
waitFrame--;
var isWebCamPlaying = webCamScreenController.isPlaying;
if (!isWebCamPlaying && waitFrame % 50 == 0) {
Debug.Log($"Waiting for a WebCamDevice");
}
return isWebCamPlaying || waitFrame < 0;
});
}
}
62 changes: 52 additions & 10 deletions Assets/MediaPipe/Examples/Scripts/WebCamScreenController.cs
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,19 @@
using Mediapipe;
using System;
using System.Diagnostics;
using System.Collections;
using UnityEngine;

using Debug = UnityEngine.Debug;
#if UNITY_ANDROID
using UnityEngine.Android;
#endif

public class WebCamScreenController : MonoBehaviour {
[SerializeField] int Width = 640;
[SerializeField] int Height = 480;
[SerializeField] int FPS = 30;
[SerializeField] float FocalLengthPx = 2.0f; /// TODO: calculate it from webCamDevice info if possible.
private const int TEXTURE_SIZE_THRESHOLD = 50;
private const int MAX_FRAMES_TO_BE_INITIALIZED = 500;

private WebCamDevice webCamDevice;
private WebCamTexture webCamTexture;
Expand All @@ -28,20 +31,40 @@ public class WebCamScreenController : MonoBehaviour {
}
}

/// TODO: use Coroutine and call InitScreen here
public void ResetScreen(WebCamDevice? device) {
private bool isWebCamReady {
get {
return isWebCamTextureInitialized && pixelData != null;
}
}

public IEnumerator ResetScreen(WebCamDevice? device) {
#if UNITY_ANDROID
if (!Permission.HasUserAuthorizedPermission(Permission.Camera)) {
Permission.RequestUserPermission(Permission.Camera);
}
#elif UNITY_IOS
if (!Application.HasUserAuthorization(UserAuthorization.WebCam)) {
yield return Application.RequestUserAuthorization(UserAuthorization.WebCam);
}
#endif

if (isPlaying) {
webCamTexture.Stop();
webCamTexture = null;
pixelData = null;
}

if (!Application.HasUserAuthorization(UserAuthorization.WebCam)) {
Debug.LogWarning("Not permitted to use Camera");
yield break;
}

if (device is WebCamDevice deviceValue) {
webCamDevice = deviceValue;
} else {
return;
yield break;
}

/// TODO: call Application.RequestUserAuthorization
webCamTexture = new WebCamTexture(webCamDevice.name, Width, Height, FPS);
WebCamTextureFramePool.Instance.SetDimension(Width, Height);

Expand All @@ -50,11 +73,20 @@ public class WebCamScreenController : MonoBehaviour {
Debug.Log($"WebCamTexture Graphics Format: {webCamTexture.graphicsFormat}");
} catch (Exception e) {
Debug.LogWarning(e.ToString());
return;
yield break;
}

var waitFrame = MAX_FRAMES_TO_BE_INITIALIZED;

yield return new WaitUntil(() => {
return isWebCamTextureInitialized || --waitFrame < 0;
});

if (!isWebCamTextureInitialized) {
Debug.LogError("Failed to initialize WebCamTexture");
yield break;
}
}

public void InitScreen() {
Renderer renderer = GetComponent<Renderer>();
outputTexture = new Texture2D(webCamTexture.width, webCamTexture.height, TextureFormat.RGBA32, false);
renderer.material.mainTexture = outputTexture;
Expand All @@ -79,22 +111,30 @@ public class WebCamScreenController : MonoBehaviour {
}

public void DrawScreen(Color32[] colors) {
if (!isWebCamReady) { return; }

// TODO: size assertion
outputTexture.SetPixels32(colors);
outputTexture.Apply();
}

public void DrawScreen(TextureFrame src) {
if (!isWebCamReady) { return; }

// TODO: size assertion
src.CopyTexture(outputTexture);
}

public void DrawScreen(ImageFrame imageFrame) {
if (!isWebCamReady) { return; }

outputTexture.LoadRawTextureData(imageFrame.MutablePixelData(), imageFrame.PixelDataSize());
outputTexture.Apply();
}

public void DrawScreen(GpuBuffer gpuBuffer) {
if (!isWebCamReady) { return; }

#if UNITY_STANDALONE_LINUX || UNITY_STANDALONE_OSX || UNITY_ANDROID
// TODO: create an external texture
outputTexture.UpdateExternalTexture((IntPtr)gpuBuffer.GetGlTextureBuffer().Name());
Expand All @@ -105,7 +145,9 @@ public class WebCamScreenController : MonoBehaviour {

public TextureFramePool.TextureFrameRequest RequestNextFrame() {
return WebCamTextureFramePool.Instance.RequestNextTextureFrame((TextureFrame textureFrame) => {
textureFrame.CopyTextureFrom(webCamTexture);
if (isPlaying) {
textureFrame.CopyTextureFrom(webCamTexture);
}
});
}

Expand Down

0 comments on commit 8fd204c

Please sign in to comment.