Skip to content

Commit

Permalink
feat(sample): support refineLandmarks option (Face Mesh)
Browse files Browse the repository at this point in the history
  • Loading branch information
@homuler
homuler committed Nov 22, 2021
1 parent c603366 commit a76194f
Show file tree
Hide file tree
Showing 19 changed files with 877 additions and 142 deletions.
506 changes: 504 additions & 2 deletions Assets/Mediapipe/Samples/Scenes/Face Mesh/Face Mesh Config Window.prefab
View file

Large diffs are not rendered by default.

121 changes: 120 additions & 1 deletion Assets/Mediapipe/Samples/Scenes/Face Mesh/Face Mesh.unity
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ RenderSettings:
m_ReflectionIntensity: 1
m_CustomReflection: {fileID: 0}
m_Sun: {fileID: 0}
m_IndirectSpecularColor: {r: 0.4465791, g: 0.4964133, b: 0.57481784, a: 1}
m_IndirectSpecularColor: {r: 0.44402248, g: 0.49316555, b: 0.5722324, a: 1}
m_UseRadianceAmbientProbe: 0
--- !u!157 &3
LightmapSettings:
Expand Down Expand Up @@ -254,6 +254,7 @@ MonoBehaviour:
_openGlEsConfig: {fileID: 4900000, guid: b84acd1bb05b70ac19d12ec8ba95ae8d, type: 3}
_timeoutMicrosec: 0
maxNumFaces: 1
withAttention: 1
OnFaceDetectionsOutput:
m_PersistentCalls:
m_Calls: []
Expand All @@ -263,6 +264,9 @@ MonoBehaviour:
OnFaceRectsFromLandmarksOutput:
m_PersistentCalls:
m_Calls: []
OnFaceRectsFromDetectionsOutput:
m_PersistentCalls:
m_Calls: []
--- !u!114 &275874969
MonoBehaviour:
m_ObjectHideFlags: 0
Expand All @@ -279,6 +283,7 @@ MonoBehaviour:
_faceDetectionsAnnotationController: {fileID: 1324426577}
_multiFaceLandmarksAnnotationController: {fileID: 1324426576}
_faceRectsFromLandmarksAnnotationController: {fileID: 1324426575}
_faceRectsFromDetectionsAnnotationController: {fileID: 1324426579}
_graphRunner: {fileID: 275874968}
_textureFramePool: {fileID: 275874970}
runningMode: 0
Expand Down Expand Up @@ -789,6 +794,107 @@ Transform:
m_Father: {fileID: 0}
m_RootOrder: 0
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
--- !u!1001 &1302709924
PrefabInstance:
m_ObjectHideFlags: 0
serializedVersion: 2
m_Modification:
m_TransformParent: {fileID: 1324426573}
m_Modifications:
- target: {fileID: 1620015869145370096, guid: 8fcc7ea504479f06a9334ee2faa42b57,
type: 3}
propertyPath: m_Name
value: FaceRectsFromDetections Annotation
objectReference: {fileID: 0}
- target: {fileID: 1620015869145370098, guid: 8fcc7ea504479f06a9334ee2faa42b57,
type: 3}
propertyPath: _color.a
value: 1
objectReference: {fileID: 0}
- target: {fileID: 1620015869145370098, guid: 8fcc7ea504479f06a9334ee2faa42b57,
type: 3}
propertyPath: _color.b
value: 0.5
objectReference: {fileID: 0}
- target: {fileID: 1620015869145370098, guid: 8fcc7ea504479f06a9334ee2faa42b57,
type: 3}
propertyPath: _color.g
value: 0
objectReference: {fileID: 0}
- target: {fileID: 1620015869145370098, guid: 8fcc7ea504479f06a9334ee2faa42b57,
type: 3}
propertyPath: _color.r
value: 0.5
objectReference: {fileID: 0}
- target: {fileID: 1620015869145370099, guid: 8fcc7ea504479f06a9334ee2faa42b57,
type: 3}
propertyPath: m_RootOrder
value: 3
objectReference: {fileID: 0}
- target: {fileID: 1620015869145370099, guid: 8fcc7ea504479f06a9334ee2faa42b57,
type: 3}
propertyPath: m_LocalPosition.x
value: 0
objectReference: {fileID: 0}
- target: {fileID: 1620015869145370099, guid: 8fcc7ea504479f06a9334ee2faa42b57,
type: 3}
propertyPath: m_LocalPosition.y
value: 0
objectReference: {fileID: 0}
- target: {fileID: 1620015869145370099, guid: 8fcc7ea504479f06a9334ee2faa42b57,
type: 3}
propertyPath: m_LocalPosition.z
value: 0
objectReference: {fileID: 0}
- target: {fileID: 1620015869145370099, guid: 8fcc7ea504479f06a9334ee2faa42b57,
type: 3}
propertyPath: m_LocalRotation.w
value: 1
objectReference: {fileID: 0}
- target: {fileID: 1620015869145370099, guid: 8fcc7ea504479f06a9334ee2faa42b57,
type: 3}
propertyPath: m_LocalRotation.x
value: 0
objectReference: {fileID: 0}
- target: {fileID: 1620015869145370099, guid: 8fcc7ea504479f06a9334ee2faa42b57,
type: 3}
propertyPath: m_LocalRotation.y
value: 0
objectReference: {fileID: 0}
- target: {fileID: 1620015869145370099, guid: 8fcc7ea504479f06a9334ee2faa42b57,
type: 3}
propertyPath: m_LocalRotation.z
value: 0
objectReference: {fileID: 0}
- target: {fileID: 1620015869145370099, guid: 8fcc7ea504479f06a9334ee2faa42b57,
type: 3}
propertyPath: m_LocalEulerAnglesHint.x
value: 0
objectReference: {fileID: 0}
- target: {fileID: 1620015869145370099, guid: 8fcc7ea504479f06a9334ee2faa42b57,
type: 3}
propertyPath: m_LocalEulerAnglesHint.y
value: 0
objectReference: {fileID: 0}
- target: {fileID: 1620015869145370099, guid: 8fcc7ea504479f06a9334ee2faa42b57,
type: 3}
propertyPath: m_LocalEulerAnglesHint.z
value: 0
objectReference: {fileID: 0}
m_RemovedComponents: []
m_SourcePrefab: {fileID: 100100000, guid: 8fcc7ea504479f06a9334ee2faa42b57, type: 3}
--- !u!114 &1302709925 stripped
MonoBehaviour:
m_CorrespondingSourceObject: {fileID: 1620015869145370098, guid: 8fcc7ea504479f06a9334ee2faa42b57,
type: 3}
m_PrefabInstance: {fileID: 1302709924}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 0}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: ff689dc19c0db10608af875e2c24ade9, type: 3}
m_Name:
m_EditorClassIdentifier:
--- !u!1001 &1324426572
PrefabInstance:
m_ObjectHideFlags: 0
Expand Down Expand Up @@ -973,6 +1079,19 @@ MonoBehaviour:
m_Script: {fileID: 11500000, guid: 1344c3c82d62a2a41a3576d8abb8e3ea, type: 3}
m_Name:
m_EditorClassIdentifier:
--- !u!114 &1324426579
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1324426574}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: 4f4190c4421f92d5187d2ebdc88a9594, type: 3}
m_Name:
m_EditorClassIdentifier:
annotation: {fileID: 1302709925}
--- !u!1001 &1329911427
PrefabInstance:
m_ObjectHideFlags: 0
Expand Down
16 changes: 16 additions & 0 deletions Assets/Mediapipe/Samples/Scenes/Face Mesh/FaceMeshConfig.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,13 @@ namespace Mediapipe.Unity.FaceMesh.UI
public class FaceMeshConfig : ModalContents
{
private const string _MaxNumFacesPath = "Scroll View/Viewport/Contents/Max Num Faces/InputField";
private const string _RefineLandmarksPath = "Scroll View/Viewport/Contents/Refine Landmarks/Toggle";
private const string _RunningModePath = "Scroll View/Viewport/Contents/Running Mode/Dropdown";
private const string _TimeoutMillisecPath = "Scroll View/Viewport/Contents/Timeout Millisec/InputField";

private FaceMeshSolution _solution;
private InputField _maxNumFacesInput;
private Toggle _refineLandmarksInput;
private Dropdown _runningModeInput;
private InputField _timeoutMillisecInput;

Expand All @@ -45,6 +47,12 @@ public void UpdateMaxNumFaces()
}
}

public void ToggleRefineLandmarks()
{
_solution.refineLandmarks = _refineLandmarksInput.isOn;
_isChanged = true;
}

public void SwitchRunningMode()
{
_solution.runningMode = (RunningMode)_runningModeInput.value;
Expand All @@ -63,6 +71,7 @@ public void SetTimeoutMillisec()
private void InitializeContents()
{
InitializeMaxNumFaces();
InitializeRefineLandmarks();
InitializeRunningMode();
InitializeTimeoutMillisec();
}
Expand All @@ -74,6 +83,13 @@ private void InitializeMaxNumFaces()
_maxNumFacesInput.onEndEdit.AddListener(delegate { UpdateMaxNumFaces(); });
}

private void InitializeRefineLandmarks()
{
_refineLandmarksInput = gameObject.transform.Find(_RefineLandmarksPath).gameObject.GetComponent<Toggle>();
_refineLandmarksInput.isOn = _solution.refineLandmarks;
_refineLandmarksInput.onValueChanged.AddListener(delegate { ToggleRefineLandmarks(); });
}

private void InitializeRunningMode()
{
_runningModeInput = gameObject.transform.Find(_RunningModePath).gameObject.GetComponent<Dropdown>();
Expand Down
31 changes: 29 additions & 2 deletions Assets/Mediapipe/Samples/Scenes/Face Mesh/FaceMeshGraph.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,25 +13,30 @@ namespace Mediapipe.Unity.FaceMesh
public class FaceMeshGraph : GraphRunner
{
public int maxNumFaces = 1;
public bool refineLandmarks = true;
#pragma warning disable IDE1006 // UnityEvent is PascalCase
public UnityEvent<List<Detection>> OnFaceDetectionsOutput = new UnityEvent<List<Detection>>();
public UnityEvent<List<NormalizedLandmarkList>> OnMultiFaceLandmarksOutput = new UnityEvent<List<NormalizedLandmarkList>>();
public UnityEvent<List<NormalizedRect>> OnFaceRectsFromLandmarksOutput = new UnityEvent<List<NormalizedRect>>();
public UnityEvent<List<NormalizedRect>> OnFaceRectsFromDetectionsOutput = new UnityEvent<List<NormalizedRect>>();
#pragma warning restore IDE1006

private const string _InputStreamName = "input_video";

private const string _FaceDetectionsStreamName = "face_detections";
private const string _MultiFaceLandmarksStreamName = "multi_face_landmarks";
private const string _FaceRectsFromLandmarksStreamName = "face_rects_from_landmarks";
private const string _FaceRectsFromDetectionsStreamName = "face_rects_from_detections";

private OutputStream<DetectionVectorPacket, List<Detection>> _faceDetectionsStream;
private OutputStream<NormalizedLandmarkListVectorPacket, List<NormalizedLandmarkList>> _multiFaceLandmarksStream;
private OutputStream<NormalizedRectVectorPacket, List<NormalizedRect>> _faceRectsFromLandmarksStream;
private OutputStream<NormalizedRectVectorPacket, List<NormalizedRect>> _faceRectsFromDetectionsStream;

protected long prevFaceDetectionsMicrosec = 0;
protected long prevMultiFaceLandmarksMicrosec = 0;
protected long prevFaceRectsFromLandmarksMicrosec = 0;
protected long prevFaceRectsFromDetectionsMicrosec = 0;

public override Status StartRun(ImageSource imageSource)
{
Expand All @@ -40,6 +45,7 @@ public override Status StartRun(ImageSource imageSource)
_faceDetectionsStream.StartPolling(true).AssertOk();
_multiFaceLandmarksStream.StartPolling(true).AssertOk();
_faceRectsFromLandmarksStream.StartPolling(true).AssertOk();
_faceRectsFromDetectionsStream.StartPolling(true).AssertOk();

return calculatorGraph.StartRun(BuildSidePacket(imageSource));
}
Expand All @@ -51,6 +57,7 @@ public Status StartRunAsync(ImageSource imageSource)
_faceDetectionsStream.AddListener(FaceDetectionsCallback, true).AssertOk();
_multiFaceLandmarksStream.AddListener(MultiFaceLandmarksCallback, true).AssertOk();
_faceRectsFromLandmarksStream.AddListener(FaceRectsFromLandmarksCallback, true).AssertOk();
_faceRectsFromDetectionsStream.AddListener(FaceRectsFromDetectionsCallback, true).AssertOk();

return calculatorGraph.StartRun(BuildSidePacket(imageSource));
}
Expand All @@ -61,6 +68,7 @@ public override void Stop()
OnFaceDetectionsOutput.RemoveAllListeners();
OnMultiFaceLandmarksOutput.RemoveAllListeners();
OnFaceRectsFromLandmarksOutput.RemoveAllListeners();
OnFaceRectsFromDetectionsOutput.RemoveAllListeners();
}

public Status AddTextureFrameToInputStream(TextureFrame textureFrame)
Expand All @@ -73,12 +81,14 @@ public FaceMeshValue FetchNextValue()
var _ = _faceDetectionsStream.TryGetNext(out var faceDetections);
_ = _multiFaceLandmarksStream.TryGetNext(out var multiFaceLandmarks);
_ = _faceRectsFromLandmarksStream.TryGetNext(out var faceRectsFromLandmarks);
_ = _faceRectsFromDetectionsStream.TryGetNext(out var faceRectsFromDetections);

OnFaceDetectionsOutput.Invoke(faceDetections);
OnMultiFaceLandmarksOutput.Invoke(multiFaceLandmarks);
OnFaceRectsFromLandmarksOutput.Invoke(faceRectsFromLandmarks);
OnFaceRectsFromDetectionsOutput.Invoke(faceRectsFromDetections);

return new FaceMeshValue(faceDetections, multiFaceLandmarks, faceRectsFromLandmarks);
return new FaceMeshValue(faceDetections, multiFaceLandmarks, faceRectsFromLandmarks, faceRectsFromDetections);
}

[AOT.MonoPInvokeCallback(typeof(CalculatorGraph.NativePacketCallback))]
Expand Down Expand Up @@ -126,18 +136,34 @@ private static IntPtr FaceRectsFromLandmarksCallback(IntPtr graphPtr, IntPtr pac
}).mpPtr;
}

[AOT.MonoPInvokeCallback(typeof(CalculatorGraph.NativePacketCallback))]
private static IntPtr FaceRectsFromDetectionsCallback(IntPtr graphPtr, IntPtr packetPtr)
{
return InvokeIfGraphRunnerFound<FaceMeshGraph>(graphPtr, packetPtr, (faceMeshGraph, ptr) =>
{
using (var packet = new NormalizedRectVectorPacket(ptr, false))
{
if (faceMeshGraph.TryGetPacketValue(packet, ref faceMeshGraph.prevFaceRectsFromDetectionsMicrosec, out var value))
{
faceMeshGraph.OnFaceRectsFromDetectionsOutput.Invoke(value);
}
}
}).mpPtr;
}

protected void InitializeOutputStreams()
{
_faceDetectionsStream = new OutputStream<DetectionVectorPacket, List<Detection>>(calculatorGraph, _FaceDetectionsStreamName);
_multiFaceLandmarksStream = new OutputStream<NormalizedLandmarkListVectorPacket, List<NormalizedLandmarkList>>(calculatorGraph, _MultiFaceLandmarksStreamName);
_faceRectsFromLandmarksStream = new OutputStream<NormalizedRectVectorPacket, List<NormalizedRect>>(calculatorGraph, _FaceRectsFromLandmarksStreamName);
_faceRectsFromDetectionsStream = new OutputStream<NormalizedRectVectorPacket, List<NormalizedRect>>(calculatorGraph, _FaceRectsFromDetectionsStreamName);
}

protected override IList<WaitForResult> RequestDependentAssets()
{
return new List<WaitForResult> {
WaitForAsset("face_detection_short_range.bytes"),
WaitForAsset("face_landmark.bytes"),
WaitForAsset(refineLandmarks ? "face_landmark_with_attention.bytes" : "face_landmark.bytes"),
};
}

Expand All @@ -147,6 +173,7 @@ private SidePacket BuildSidePacket(ImageSource imageSource)

SetImageTransformationOptions(sidePacket, imageSource);
sidePacket.Emplace("num_faces", new IntPacket(maxNumFaces));
sidePacket.Emplace("with_attention", new BoolPacket(refineLandmarks));

return sidePacket;
}
Expand Down
15 changes: 15 additions & 0 deletions Assets/Mediapipe/Samples/Scenes/Face Mesh/FaceMeshSolution.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ public class FaceMeshSolution : Solution
[SerializeField] private DetectionListAnnotationController _faceDetectionsAnnotationController;
[SerializeField] private MultiFaceLandmarkListAnnotationController _multiFaceLandmarksAnnotationController;
[SerializeField] private NormalizedRectListAnnotationController _faceRectsFromLandmarksAnnotationController;
[SerializeField] private NormalizedRectListAnnotationController _faceRectsFromDetectionsAnnotationController;
[SerializeField] private FaceMeshGraph _graphRunner;
[SerializeField] private TextureFramePool _textureFramePool;

Expand All @@ -30,6 +31,12 @@ public int maxNumFaces
set => _graphRunner.maxNumFaces = value;
}

public bool refineLandmarks
{
get => _graphRunner.refineLandmarks;
set => _graphRunner.refineLandmarks = value;
}

public long timeoutMillisec
{
get => _graphRunner.timeoutMillisec;
Expand Down Expand Up @@ -83,6 +90,7 @@ private IEnumerator Run()
_screen.texture = imageSource.GetCurrentTexture();

Logger.LogInfo(TAG, $"Max Num Faces = {maxNumFaces}");
Logger.LogInfo(TAG, $"Refine Landmarks = {refineLandmarks}");
Logger.LogInfo(TAG, $"Running Mode = {runningMode}");

// Wait for completion of loading of dependent files, etc.
Expand All @@ -98,6 +106,7 @@ private IEnumerator Run()
_graphRunner.OnFaceDetectionsOutput.AddListener(OnFaceDetectionsOutput);
_graphRunner.OnMultiFaceLandmarksOutput.AddListener(OnMultiFaceLandmarksOutput);
_graphRunner.OnFaceRectsFromLandmarksOutput.AddListener(OnFaceRectsFromLandmarksOutput);
_graphRunner.OnFaceRectsFromDetectionsOutput.AddListener(OnFaceRectsFromDetectionsOutput);
_graphRunner.StartRunAsync(imageSource).AssertOk();
}
else
Expand All @@ -112,6 +121,7 @@ private IEnumerator Run()
SetupAnnotationController(_faceDetectionsAnnotationController, imageSource);
SetupAnnotationController(_faceRectsFromLandmarksAnnotationController, imageSource);
SetupAnnotationController(_multiFaceLandmarksAnnotationController, imageSource);
SetupAnnotationController(_faceRectsFromDetectionsAnnotationController, imageSource);

while (true)
{
Expand Down Expand Up @@ -153,5 +163,10 @@ private void OnFaceRectsFromLandmarksOutput(List<NormalizedRect> faceRectsFromLa
{
_faceRectsFromLandmarksAnnotationController.DrawLater(faceRectsFromLandmarks);
}

private void OnFaceRectsFromDetectionsOutput(List<NormalizedRect> faceRectsFromDetections)
{
_faceRectsFromDetectionsAnnotationController.DrawLater(faceRectsFromDetections);
}
}
}
4 changes: 3 additions & 1 deletion Assets/Mediapipe/Samples/Scenes/Face Mesh/FaceMeshValue.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,14 @@ public class FaceMeshValue
public readonly List<Detection> faceDetections;
public readonly List<NormalizedLandmarkList> multiFaceLandmarks;
public readonly List<NormalizedRect> faceRectsFromLandmarks;
public readonly List<NormalizedRect> faceRectsFromDetections;

public FaceMeshValue(List<Detection> faceDetections, List<NormalizedLandmarkList> multiFaceLandmarks, List<NormalizedRect> faceRectsFromLandmarks)
public FaceMeshValue(List<Detection> faceDetections, List<NormalizedLandmarkList> multiFaceLandmarks, List<NormalizedRect> faceRectsFromLandmarks, List<NormalizedRect> faceRectsFromDetections)
{
this.faceDetections = faceDetections;
this.multiFaceLandmarks = multiFaceLandmarks;
this.faceRectsFromLandmarks = faceRectsFromLandmarks;
this.faceRectsFromDetections = faceRectsFromDetections;
}
}
}
Loading

0 comments on commit a76194f

Please sign in to comment.