From a76194f413093f45c9fb35d3c423b6af76c80d0d Mon Sep 17 00:00:00 2001 From: homuler Date: Mon, 22 Nov 2021 11:35:38 +0900 Subject: [PATCH] feat(sample): support refineLandmarks option (Face Mesh) --- .../Face Mesh/Face Mesh Config Window.prefab | 506 +++++++++++++++++- .../Samples/Scenes/Face Mesh/Face Mesh.unity | 121 ++++- .../Scenes/Face Mesh/FaceMeshConfig.cs | 16 + .../Samples/Scenes/Face Mesh/FaceMeshGraph.cs | 31 +- .../Scenes/Face Mesh/FaceMeshSolution.cs | 15 + .../Samples/Scenes/Face Mesh/FaceMeshValue.cs | 4 +- .../Scenes/Face Mesh/face_mesh_cpu.txt | 2 + .../Scenes/Face Mesh/face_mesh_gpu.txt | 2 + .../Scenes/Face Mesh/face_mesh_opengles.txt | 2 + .../Scenes/Iris Tracking/Iris Tracking.unity | 4 +- .../Iris Tracking/IrisTrackingSolution.cs | 2 +- .../Multi FaceLandmarkList Annotation.prefab | 14 +- .../face_landmark_with_attention.bytes.meta | 7 + .../FaceLandmarkListAnnotationController.cs | 5 +- .../FaceLandmarkListWithIrisAnnotation.cs | 64 ++- ...andmarkListWithIrisAnnotationController.cs | 76 --- ...rkListWithIrisAnnotationController.cs.meta | 11 - .../MultiFaceLandmarkListAnnotation.cs | 136 +++-- mediapipe_api/BUILD | 1 + 19 files changed, 877 insertions(+), 142 deletions(-) create mode 100644 Packages/com.github.homuler.mediapipe/Runtime/Resources/face_landmark_with_attention.bytes.meta delete mode 100644 Packages/com.github.homuler.mediapipe/Runtime/Scripts/Unity/Annotation/FaceLandmarkListWithIrisAnnotationController.cs delete mode 100644 Packages/com.github.homuler.mediapipe/Runtime/Scripts/Unity/Annotation/FaceLandmarkListWithIrisAnnotationController.cs.meta diff --git a/Assets/Mediapipe/Samples/Scenes/Face Mesh/Face Mesh Config Window.prefab b/Assets/Mediapipe/Samples/Scenes/Face Mesh/Face Mesh Config Window.prefab index a2a0be104..8e52d7197 100644 --- a/Assets/Mediapipe/Samples/Scenes/Face Mesh/Face Mesh Config Window.prefab +++ b/Assets/Mediapipe/Samples/Scenes/Face Mesh/Face Mesh Config Window.prefab @@ -581,6 +581,81 @@ RectTransform: m_AnchoredPosition: {x: 0, y: 0} m_SizeDelta: {x: -20, y: -20} m_Pivot: {x: 0.5, y: 0.5} +--- !u!1 &2059363814091093479 +GameObject: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + serializedVersion: 6 + m_Component: + - component: {fileID: 880546811728870616} + - component: {fileID: 5708894169854237517} + - component: {fileID: 5775392906338624159} + m_Layer: 5 + m_Name: Checkmark + m_TagString: Untagged + m_Icon: {fileID: 0} + m_NavMeshLayer: 0 + m_StaticEditorFlags: 0 + m_IsActive: 1 +--- !u!224 &880546811728870616 +RectTransform: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 2059363814091093479} + m_LocalRotation: {x: -0, y: -0, z: -0, w: 1} + m_LocalPosition: {x: 0, y: 0, z: 0} + m_LocalScale: {x: 1, y: 1, z: 1} + m_Children: [] + m_Father: {fileID: 4352924266239188134} + m_RootOrder: 0 + m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} + m_AnchorMin: {x: 0.5, y: 0.5} + m_AnchorMax: {x: 0.5, y: 0.5} + m_AnchoredPosition: {x: 0, y: 0} + m_SizeDelta: {x: 64, y: 64} + m_Pivot: {x: 0.5, y: 0.5} +--- !u!222 &5708894169854237517 +CanvasRenderer: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 2059363814091093479} + m_CullTransparentMesh: 1 +--- !u!114 &5775392906338624159 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 2059363814091093479} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: fe87c0e1cc204ed48ad3b37840f39efc, type: 3} + m_Name: + m_EditorClassIdentifier: + m_Material: {fileID: 0} + m_Color: {r: 1, g: 1, b: 1, a: 1} + m_RaycastTarget: 1 + m_RaycastPadding: {x: 0, y: 0, z: 0, w: 0} + m_Maskable: 1 + m_OnCullStateChanged: + m_PersistentCalls: + m_Calls: [] + m_Sprite: {fileID: 10901, guid: 0000000000000000f000000000000000, type: 0} + m_Type: 0 + m_PreserveAspect: 0 + m_FillCenter: 1 + m_FillMethod: 4 + m_FillAmount: 1 + m_FillClockwise: 1 + m_FillOrigin: 0 + m_UseSpriteMesh: 0 + m_PixelsPerUnitMultiplier: 1 --- !u!1 &2358664826247471437 GameObject: m_ObjectHideFlags: 0 @@ -656,6 +731,85 @@ MonoBehaviour: m_FillOrigin: 0 m_UseSpriteMesh: 0 m_PixelsPerUnitMultiplier: 1 +--- !u!1 &2536184223038791301 +GameObject: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + serializedVersion: 6 + m_Component: + - component: {fileID: 8169629189900129759} + - component: {fileID: 224053162250289337} + - component: {fileID: 719175656437001407} + m_Layer: 5 + m_Name: Label + m_TagString: Untagged + m_Icon: {fileID: 0} + m_NavMeshLayer: 0 + m_StaticEditorFlags: 0 + m_IsActive: 1 +--- !u!224 &8169629189900129759 +RectTransform: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 2536184223038791301} + m_LocalRotation: {x: -0, y: -0, z: -0, w: 1} + m_LocalPosition: {x: 0, y: 0, z: 0} + m_LocalScale: {x: 1, y: 1, z: 1} + m_Children: [] + m_Father: {fileID: 4601895220429817723} + m_RootOrder: 1 + m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} + m_AnchorMin: {x: 0, y: 0} + m_AnchorMax: {x: 1, y: 1} + m_AnchoredPosition: {x: 9, y: -0.5} + m_SizeDelta: {x: -28, y: -3} + m_Pivot: {x: 0.5, y: 0.5} +--- !u!222 &224053162250289337 +CanvasRenderer: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 2536184223038791301} + m_CullTransparentMesh: 1 +--- !u!114 &719175656437001407 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 2536184223038791301} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: 5f7201a12d95ffc409449d95f23cf332, type: 3} + m_Name: + m_EditorClassIdentifier: + m_Material: {fileID: 0} + m_Color: {r: 0.19607843, g: 0.19607843, b: 0.19607843, a: 1} + m_RaycastTarget: 1 + m_RaycastPadding: {x: 0, y: 0, z: 0, w: 0} + m_Maskable: 1 + m_OnCullStateChanged: + m_PersistentCalls: + m_Calls: [] + m_FontData: + m_Font: {fileID: 10102, guid: 0000000000000000e000000000000000, type: 0} + m_FontSize: 48 + m_FontStyle: 0 + m_BestFit: 0 + m_MinSize: 4 + m_MaxSize: 49 + m_Alignment: 0 + m_AlignByGeometry: 0 + m_RichText: 1 + m_HorizontalOverflow: 0 + m_VerticalOverflow: 0 + m_LineSpacing: 1 + m_Text: --- !u!1 &2622189582878093186 GameObject: m_ObjectHideFlags: 0 @@ -1323,6 +1477,7 @@ RectTransform: m_Children: - {fileID: 9098513968449949306} - {fileID: 4825301634575857686} + - {fileID: 7053335182989981517} - {fileID: 1423722328293772700} - {fileID: 2901740110241252406} m_Father: {fileID: 8898856628389366212} @@ -1772,6 +1927,70 @@ MonoBehaviour: m_FlexibleWidth: 3 m_FlexibleHeight: -1 m_LayoutPriority: 1 +--- !u!1 &4342929391286843849 +GameObject: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + serializedVersion: 6 + m_Component: + - component: {fileID: 7053335182989981517} + - component: {fileID: 1930882000882463570} + m_Layer: 5 + m_Name: Refine Landmarks + m_TagString: Untagged + m_Icon: {fileID: 0} + m_NavMeshLayer: 0 + m_StaticEditorFlags: 0 + m_IsActive: 1 +--- !u!224 &7053335182989981517 +RectTransform: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 4342929391286843849} + m_LocalRotation: {x: 0, y: 0, z: 0, w: 1} + m_LocalPosition: {x: 0, y: 0, z: 0} + m_LocalScale: {x: 1, y: 1, z: 1} + m_Children: + - {fileID: 2571345137534825987} + - {fileID: 4601895220429817723} + m_Father: {fileID: 7710130227850707868} + m_RootOrder: 2 + m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} + m_AnchorMin: {x: 0, y: 0} + m_AnchorMax: {x: 0, y: 0} + m_AnchoredPosition: {x: 0, y: 0} + m_SizeDelta: {x: 0, y: 0} + m_Pivot: {x: 0, y: 1} +--- !u!114 &1930882000882463570 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 4342929391286843849} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: 30649d3a9faa99c48a7b1166b86bf2a0, type: 3} + m_Name: + m_EditorClassIdentifier: + m_Padding: + m_Left: 24 + m_Right: 0 + m_Top: 0 + m_Bottom: 0 + m_ChildAlignment: 3 + m_Spacing: 12 + m_ChildForceExpandWidth: 0 + m_ChildForceExpandHeight: 1 + m_ChildControlWidth: 1 + m_ChildControlHeight: 0 + m_ChildScaleWidth: 0 + m_ChildScaleHeight: 0 + m_ReverseArrangement: 0 --- !u!1 &4861348491489321247 GameObject: m_ObjectHideFlags: 0 @@ -2560,7 +2779,7 @@ RectTransform: - {fileID: 7368247365119313596} - {fileID: 916782750758033152} m_Father: {fileID: 7710130227850707868} - m_RootOrder: 2 + m_RootOrder: 3 m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} m_AnchorMin: {x: 0, y: 0} m_AnchorMax: {x: 0, y: 0} @@ -2660,7 +2879,7 @@ RectTransform: - {fileID: 583459496656910013} - {fileID: 4742019235859864154} m_Father: {fileID: 7710130227850707868} - m_RootOrder: 3 + m_RootOrder: 4 m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} m_AnchorMin: {x: 0, y: 0} m_AnchorMax: {x: 0, y: 0} @@ -2693,6 +2912,82 @@ MonoBehaviour: m_ChildScaleWidth: 0 m_ChildScaleHeight: 0 m_ReverseArrangement: 0 +--- !u!1 &7256557690221712702 +GameObject: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + serializedVersion: 6 + m_Component: + - component: {fileID: 4352924266239188134} + - component: {fileID: 7559410552337441118} + - component: {fileID: 8283409977520513000} + m_Layer: 5 + m_Name: Background + m_TagString: Untagged + m_Icon: {fileID: 0} + m_NavMeshLayer: 0 + m_StaticEditorFlags: 0 + m_IsActive: 1 +--- !u!224 &4352924266239188134 +RectTransform: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 7256557690221712702} + m_LocalRotation: {x: -0, y: -0, z: -0, w: 1} + m_LocalPosition: {x: 0, y: 0, z: 0} + m_LocalScale: {x: 1, y: 1, z: 1} + m_Children: + - {fileID: 880546811728870616} + m_Father: {fileID: 4601895220429817723} + m_RootOrder: 0 + m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} + m_AnchorMin: {x: 0, y: 1} + m_AnchorMax: {x: 0, y: 1} + m_AnchoredPosition: {x: 0, y: 0} + m_SizeDelta: {x: 64, y: 64} + m_Pivot: {x: 0, y: 1} +--- !u!222 &7559410552337441118 +CanvasRenderer: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 7256557690221712702} + m_CullTransparentMesh: 1 +--- !u!114 &8283409977520513000 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 7256557690221712702} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: fe87c0e1cc204ed48ad3b37840f39efc, type: 3} + m_Name: + m_EditorClassIdentifier: + m_Material: {fileID: 0} + m_Color: {r: 1, g: 1, b: 1, a: 1} + m_RaycastTarget: 1 + m_RaycastPadding: {x: 0, y: 0, z: 0, w: 0} + m_Maskable: 1 + m_OnCullStateChanged: + m_PersistentCalls: + m_Calls: [] + m_Sprite: {fileID: 10905, guid: 0000000000000000f000000000000000, type: 0} + m_Type: 1 + m_PreserveAspect: 0 + m_FillCenter: 1 + m_FillMethod: 4 + m_FillAmount: 1 + m_FillClockwise: 1 + m_FillOrigin: 0 + m_UseSpriteMesh: 0 + m_PixelsPerUnitMultiplier: 1 --- !u!1 &7335675734870366313 GameObject: m_ObjectHideFlags: 0 @@ -2780,6 +3075,106 @@ MonoBehaviour: m_PersistentCalls: m_Calls: [] m_IsOn: 1 +--- !u!1 &7461601173453612696 +GameObject: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + serializedVersion: 6 + m_Component: + - component: {fileID: 2571345137534825987} + - component: {fileID: 7874882839420124399} + - component: {fileID: 7675377581133039747} + - component: {fileID: 2639817751445096445} + m_Layer: 5 + m_Name: Text + m_TagString: Untagged + m_Icon: {fileID: 0} + m_NavMeshLayer: 0 + m_StaticEditorFlags: 0 + m_IsActive: 1 +--- !u!224 &2571345137534825987 +RectTransform: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 7461601173453612696} + m_LocalRotation: {x: 0, y: 0, z: 0, w: 1} + m_LocalPosition: {x: 0, y: 0, z: 0} + m_LocalScale: {x: 1, y: 1, z: 1} + m_Children: [] + m_Father: {fileID: 7053335182989981517} + m_RootOrder: 0 + m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} + m_AnchorMin: {x: 0, y: 0} + m_AnchorMax: {x: 0, y: 0} + m_AnchoredPosition: {x: 0, y: 0} + m_SizeDelta: {x: 0, y: 72} + m_Pivot: {x: 0.5, y: 0.5} +--- !u!222 &7874882839420124399 +CanvasRenderer: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 7461601173453612696} + m_CullTransparentMesh: 1 +--- !u!114 &7675377581133039747 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 7461601173453612696} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: 5f7201a12d95ffc409449d95f23cf332, type: 3} + m_Name: + m_EditorClassIdentifier: + m_Material: {fileID: 0} + m_Color: {r: 0.19607843, g: 0.19607843, b: 0.19607843, a: 1} + m_RaycastTarget: 1 + m_RaycastPadding: {x: 0, y: 0, z: 0, w: 0} + m_Maskable: 1 + m_OnCullStateChanged: + m_PersistentCalls: + m_Calls: [] + m_FontData: + m_Font: {fileID: 10102, guid: 0000000000000000e000000000000000, type: 0} + m_FontSize: 48 + m_FontStyle: 0 + m_BestFit: 0 + m_MinSize: 4 + m_MaxSize: 300 + m_Alignment: 3 + m_AlignByGeometry: 0 + m_RichText: 1 + m_HorizontalOverflow: 0 + m_VerticalOverflow: 0 + m_LineSpacing: 1 + m_Text: Refine Landmarks +--- !u!114 &2639817751445096445 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 7461601173453612696} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: 306cc8c2b49d7114eaa3623786fc2126, type: 3} + m_Name: + m_EditorClassIdentifier: + m_IgnoreLayout: 0 + m_MinWidth: -1 + m_MinHeight: -1 + m_PreferredWidth: 800 + m_PreferredHeight: -1 + m_FlexibleWidth: 1 + m_FlexibleHeight: -1 + m_LayoutPriority: 1 --- !u!1 &7876729585577833804 GameObject: m_ObjectHideFlags: 0 @@ -3407,6 +3802,113 @@ MonoBehaviour: m_FlexibleWidth: 1 m_FlexibleHeight: -1 m_LayoutPriority: 1 +--- !u!1 &8877587778501423737 +GameObject: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + serializedVersion: 6 + m_Component: + - component: {fileID: 4601895220429817723} + - component: {fileID: 6099402187400410094} + - component: {fileID: 7245224572394472681} + m_Layer: 5 + m_Name: Toggle + m_TagString: Untagged + m_Icon: {fileID: 0} + m_NavMeshLayer: 0 + m_StaticEditorFlags: 0 + m_IsActive: 1 +--- !u!224 &4601895220429817723 +RectTransform: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 8877587778501423737} + m_LocalRotation: {x: 0, y: 0, z: 0, w: 1} + m_LocalPosition: {x: 0, y: 0, z: 0} + m_LocalScale: {x: 1, y: 1, z: 1} + m_Children: + - {fileID: 4352924266239188134} + - {fileID: 8169629189900129759} + m_Father: {fileID: 7053335182989981517} + m_RootOrder: 1 + m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} + m_AnchorMin: {x: 0, y: 0} + m_AnchorMax: {x: 0, y: 0} + m_AnchoredPosition: {x: 0, y: 0} + m_SizeDelta: {x: 0, y: 72} + m_Pivot: {x: 0.5, y: 0.5} +--- !u!114 &6099402187400410094 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 8877587778501423737} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: 9085046f02f69544eb97fd06b6048fe2, type: 3} + m_Name: + m_EditorClassIdentifier: + m_Navigation: + m_Mode: 3 + m_WrapAround: 0 + m_SelectOnUp: {fileID: 0} + m_SelectOnDown: {fileID: 0} + m_SelectOnLeft: {fileID: 0} + m_SelectOnRight: {fileID: 0} + m_Transition: 1 + m_Colors: + m_NormalColor: {r: 1, g: 1, b: 1, a: 1} + m_HighlightedColor: {r: 0.9607843, g: 0.9607843, b: 0.9607843, a: 1} + m_PressedColor: {r: 0.78431374, g: 0.78431374, b: 0.78431374, a: 1} + m_SelectedColor: {r: 0.9607843, g: 0.9607843, b: 0.9607843, a: 1} + m_DisabledColor: {r: 0.78431374, g: 0.78431374, b: 0.78431374, a: 0.5019608} + m_ColorMultiplier: 1 + m_FadeDuration: 0.1 + m_SpriteState: + m_HighlightedSprite: {fileID: 0} + m_PressedSprite: {fileID: 0} + m_SelectedSprite: {fileID: 0} + m_DisabledSprite: {fileID: 0} + m_AnimationTriggers: + m_NormalTrigger: Normal + m_HighlightedTrigger: Highlighted + m_PressedTrigger: Pressed + m_SelectedTrigger: Selected + m_DisabledTrigger: Disabled + m_Interactable: 1 + m_TargetGraphic: {fileID: 8283409977520513000} + toggleTransition: 1 + graphic: {fileID: 5775392906338624159} + m_Group: {fileID: 0} + onValueChanged: + m_PersistentCalls: + m_Calls: [] + m_IsOn: 1 +--- !u!114 &7245224572394472681 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 8877587778501423737} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: 306cc8c2b49d7114eaa3623786fc2126, type: 3} + m_Name: + m_EditorClassIdentifier: + m_IgnoreLayout: 0 + m_MinWidth: -1 + m_MinHeight: -1 + m_PreferredWidth: 800 + m_PreferredHeight: -1 + m_FlexibleWidth: 3 + m_FlexibleHeight: -1 + m_LayoutPriority: 1 --- !u!1 &9142691146296850947 GameObject: m_ObjectHideFlags: 0 diff --git a/Assets/Mediapipe/Samples/Scenes/Face Mesh/Face Mesh.unity b/Assets/Mediapipe/Samples/Scenes/Face Mesh/Face Mesh.unity index 55c73b610..0d7dab0cd 100644 --- a/Assets/Mediapipe/Samples/Scenes/Face Mesh/Face Mesh.unity +++ b/Assets/Mediapipe/Samples/Scenes/Face Mesh/Face Mesh.unity @@ -38,7 +38,7 @@ RenderSettings: m_ReflectionIntensity: 1 m_CustomReflection: {fileID: 0} m_Sun: {fileID: 0} - m_IndirectSpecularColor: {r: 0.4465791, g: 0.4964133, b: 0.57481784, a: 1} + m_IndirectSpecularColor: {r: 0.44402248, g: 0.49316555, b: 0.5722324, a: 1} m_UseRadianceAmbientProbe: 0 --- !u!157 &3 LightmapSettings: @@ -254,6 +254,7 @@ MonoBehaviour: _openGlEsConfig: {fileID: 4900000, guid: b84acd1bb05b70ac19d12ec8ba95ae8d, type: 3} _timeoutMicrosec: 0 maxNumFaces: 1 + withAttention: 1 OnFaceDetectionsOutput: m_PersistentCalls: m_Calls: [] @@ -263,6 +264,9 @@ MonoBehaviour: OnFaceRectsFromLandmarksOutput: m_PersistentCalls: m_Calls: [] + OnFaceRectsFromDetectionsOutput: + m_PersistentCalls: + m_Calls: [] --- !u!114 &275874969 MonoBehaviour: m_ObjectHideFlags: 0 @@ -279,6 +283,7 @@ MonoBehaviour: _faceDetectionsAnnotationController: {fileID: 1324426577} _multiFaceLandmarksAnnotationController: {fileID: 1324426576} _faceRectsFromLandmarksAnnotationController: {fileID: 1324426575} + _faceRectsFromDetectionsAnnotationController: {fileID: 1324426579} _graphRunner: {fileID: 275874968} _textureFramePool: {fileID: 275874970} runningMode: 0 @@ -789,6 +794,107 @@ Transform: m_Father: {fileID: 0} m_RootOrder: 0 m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} +--- !u!1001 &1302709924 +PrefabInstance: + m_ObjectHideFlags: 0 + serializedVersion: 2 + m_Modification: + m_TransformParent: {fileID: 1324426573} + m_Modifications: + - target: {fileID: 1620015869145370096, guid: 8fcc7ea504479f06a9334ee2faa42b57, + type: 3} + propertyPath: m_Name + value: FaceRectsFromDetections Annotation + objectReference: {fileID: 0} + - target: {fileID: 1620015869145370098, guid: 8fcc7ea504479f06a9334ee2faa42b57, + type: 3} + propertyPath: _color.a + value: 1 + objectReference: {fileID: 0} + - target: {fileID: 1620015869145370098, guid: 8fcc7ea504479f06a9334ee2faa42b57, + type: 3} + propertyPath: _color.b + value: 0.5 + objectReference: {fileID: 0} + - target: {fileID: 1620015869145370098, guid: 8fcc7ea504479f06a9334ee2faa42b57, + type: 3} + propertyPath: _color.g + value: 0 + objectReference: {fileID: 0} + - target: {fileID: 1620015869145370098, guid: 8fcc7ea504479f06a9334ee2faa42b57, + type: 3} + propertyPath: _color.r + value: 0.5 + objectReference: {fileID: 0} + - target: {fileID: 1620015869145370099, guid: 8fcc7ea504479f06a9334ee2faa42b57, + type: 3} + propertyPath: m_RootOrder + value: 3 + objectReference: {fileID: 0} + - target: {fileID: 1620015869145370099, guid: 8fcc7ea504479f06a9334ee2faa42b57, + type: 3} + propertyPath: m_LocalPosition.x + value: 0 + objectReference: {fileID: 0} + - target: {fileID: 1620015869145370099, guid: 8fcc7ea504479f06a9334ee2faa42b57, + type: 3} + propertyPath: m_LocalPosition.y + value: 0 + objectReference: {fileID: 0} + - target: {fileID: 1620015869145370099, guid: 8fcc7ea504479f06a9334ee2faa42b57, + type: 3} + propertyPath: m_LocalPosition.z + value: 0 + objectReference: {fileID: 0} + - target: {fileID: 1620015869145370099, guid: 8fcc7ea504479f06a9334ee2faa42b57, + type: 3} + propertyPath: m_LocalRotation.w + value: 1 + objectReference: {fileID: 0} + - target: {fileID: 1620015869145370099, guid: 8fcc7ea504479f06a9334ee2faa42b57, + type: 3} + propertyPath: m_LocalRotation.x + value: 0 + objectReference: {fileID: 0} + - target: {fileID: 1620015869145370099, guid: 8fcc7ea504479f06a9334ee2faa42b57, + type: 3} + propertyPath: m_LocalRotation.y + value: 0 + objectReference: {fileID: 0} + - target: {fileID: 1620015869145370099, guid: 8fcc7ea504479f06a9334ee2faa42b57, + type: 3} + propertyPath: m_LocalRotation.z + value: 0 + objectReference: {fileID: 0} + - target: {fileID: 1620015869145370099, guid: 8fcc7ea504479f06a9334ee2faa42b57, + type: 3} + propertyPath: m_LocalEulerAnglesHint.x + value: 0 + objectReference: {fileID: 0} + - target: {fileID: 1620015869145370099, guid: 8fcc7ea504479f06a9334ee2faa42b57, + type: 3} + propertyPath: m_LocalEulerAnglesHint.y + value: 0 + objectReference: {fileID: 0} + - target: {fileID: 1620015869145370099, guid: 8fcc7ea504479f06a9334ee2faa42b57, + type: 3} + propertyPath: m_LocalEulerAnglesHint.z + value: 0 + objectReference: {fileID: 0} + m_RemovedComponents: [] + m_SourcePrefab: {fileID: 100100000, guid: 8fcc7ea504479f06a9334ee2faa42b57, type: 3} +--- !u!114 &1302709925 stripped +MonoBehaviour: + m_CorrespondingSourceObject: {fileID: 1620015869145370098, guid: 8fcc7ea504479f06a9334ee2faa42b57, + type: 3} + m_PrefabInstance: {fileID: 1302709924} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 0} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: ff689dc19c0db10608af875e2c24ade9, type: 3} + m_Name: + m_EditorClassIdentifier: --- !u!1001 &1324426572 PrefabInstance: m_ObjectHideFlags: 0 @@ -973,6 +1079,19 @@ MonoBehaviour: m_Script: {fileID: 11500000, guid: 1344c3c82d62a2a41a3576d8abb8e3ea, type: 3} m_Name: m_EditorClassIdentifier: +--- !u!114 &1324426579 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 1324426574} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: 4f4190c4421f92d5187d2ebdc88a9594, type: 3} + m_Name: + m_EditorClassIdentifier: + annotation: {fileID: 1302709925} --- !u!1001 &1329911427 PrefabInstance: m_ObjectHideFlags: 0 diff --git a/Assets/Mediapipe/Samples/Scenes/Face Mesh/FaceMeshConfig.cs b/Assets/Mediapipe/Samples/Scenes/Face Mesh/FaceMeshConfig.cs index 8137f1b26..08411fa2d 100644 --- a/Assets/Mediapipe/Samples/Scenes/Face Mesh/FaceMeshConfig.cs +++ b/Assets/Mediapipe/Samples/Scenes/Face Mesh/FaceMeshConfig.cs @@ -15,11 +15,13 @@ namespace Mediapipe.Unity.FaceMesh.UI public class FaceMeshConfig : ModalContents { private const string _MaxNumFacesPath = "Scroll View/Viewport/Contents/Max Num Faces/InputField"; + private const string _RefineLandmarksPath = "Scroll View/Viewport/Contents/Refine Landmarks/Toggle"; private const string _RunningModePath = "Scroll View/Viewport/Contents/Running Mode/Dropdown"; private const string _TimeoutMillisecPath = "Scroll View/Viewport/Contents/Timeout Millisec/InputField"; private FaceMeshSolution _solution; private InputField _maxNumFacesInput; + private Toggle _refineLandmarksInput; private Dropdown _runningModeInput; private InputField _timeoutMillisecInput; @@ -45,6 +47,12 @@ public void UpdateMaxNumFaces() } } + public void ToggleRefineLandmarks() + { + _solution.refineLandmarks = _refineLandmarksInput.isOn; + _isChanged = true; + } + public void SwitchRunningMode() { _solution.runningMode = (RunningMode)_runningModeInput.value; @@ -63,6 +71,7 @@ public void SetTimeoutMillisec() private void InitializeContents() { InitializeMaxNumFaces(); + InitializeRefineLandmarks(); InitializeRunningMode(); InitializeTimeoutMillisec(); } @@ -74,6 +83,13 @@ private void InitializeMaxNumFaces() _maxNumFacesInput.onEndEdit.AddListener(delegate { UpdateMaxNumFaces(); }); } + private void InitializeRefineLandmarks() + { + _refineLandmarksInput = gameObject.transform.Find(_RefineLandmarksPath).gameObject.GetComponent(); + _refineLandmarksInput.isOn = _solution.refineLandmarks; + _refineLandmarksInput.onValueChanged.AddListener(delegate { ToggleRefineLandmarks(); }); + } + private void InitializeRunningMode() { _runningModeInput = gameObject.transform.Find(_RunningModePath).gameObject.GetComponent(); diff --git a/Assets/Mediapipe/Samples/Scenes/Face Mesh/FaceMeshGraph.cs b/Assets/Mediapipe/Samples/Scenes/Face Mesh/FaceMeshGraph.cs index a6600c703..f979e1d84 100644 --- a/Assets/Mediapipe/Samples/Scenes/Face Mesh/FaceMeshGraph.cs +++ b/Assets/Mediapipe/Samples/Scenes/Face Mesh/FaceMeshGraph.cs @@ -13,10 +13,12 @@ namespace Mediapipe.Unity.FaceMesh public class FaceMeshGraph : GraphRunner { public int maxNumFaces = 1; + public bool refineLandmarks = true; #pragma warning disable IDE1006 // UnityEvent is PascalCase public UnityEvent> OnFaceDetectionsOutput = new UnityEvent>(); public UnityEvent> OnMultiFaceLandmarksOutput = new UnityEvent>(); public UnityEvent> OnFaceRectsFromLandmarksOutput = new UnityEvent>(); + public UnityEvent> OnFaceRectsFromDetectionsOutput = new UnityEvent>(); #pragma warning restore IDE1006 private const string _InputStreamName = "input_video"; @@ -24,14 +26,17 @@ public class FaceMeshGraph : GraphRunner private const string _FaceDetectionsStreamName = "face_detections"; private const string _MultiFaceLandmarksStreamName = "multi_face_landmarks"; private const string _FaceRectsFromLandmarksStreamName = "face_rects_from_landmarks"; + private const string _FaceRectsFromDetectionsStreamName = "face_rects_from_detections"; private OutputStream> _faceDetectionsStream; private OutputStream> _multiFaceLandmarksStream; private OutputStream> _faceRectsFromLandmarksStream; + private OutputStream> _faceRectsFromDetectionsStream; protected long prevFaceDetectionsMicrosec = 0; protected long prevMultiFaceLandmarksMicrosec = 0; protected long prevFaceRectsFromLandmarksMicrosec = 0; + protected long prevFaceRectsFromDetectionsMicrosec = 0; public override Status StartRun(ImageSource imageSource) { @@ -40,6 +45,7 @@ public override Status StartRun(ImageSource imageSource) _faceDetectionsStream.StartPolling(true).AssertOk(); _multiFaceLandmarksStream.StartPolling(true).AssertOk(); _faceRectsFromLandmarksStream.StartPolling(true).AssertOk(); + _faceRectsFromDetectionsStream.StartPolling(true).AssertOk(); return calculatorGraph.StartRun(BuildSidePacket(imageSource)); } @@ -51,6 +57,7 @@ public Status StartRunAsync(ImageSource imageSource) _faceDetectionsStream.AddListener(FaceDetectionsCallback, true).AssertOk(); _multiFaceLandmarksStream.AddListener(MultiFaceLandmarksCallback, true).AssertOk(); _faceRectsFromLandmarksStream.AddListener(FaceRectsFromLandmarksCallback, true).AssertOk(); + _faceRectsFromDetectionsStream.AddListener(FaceRectsFromDetectionsCallback, true).AssertOk(); return calculatorGraph.StartRun(BuildSidePacket(imageSource)); } @@ -61,6 +68,7 @@ public override void Stop() OnFaceDetectionsOutput.RemoveAllListeners(); OnMultiFaceLandmarksOutput.RemoveAllListeners(); OnFaceRectsFromLandmarksOutput.RemoveAllListeners(); + OnFaceRectsFromDetectionsOutput.RemoveAllListeners(); } public Status AddTextureFrameToInputStream(TextureFrame textureFrame) @@ -73,12 +81,14 @@ public FaceMeshValue FetchNextValue() var _ = _faceDetectionsStream.TryGetNext(out var faceDetections); _ = _multiFaceLandmarksStream.TryGetNext(out var multiFaceLandmarks); _ = _faceRectsFromLandmarksStream.TryGetNext(out var faceRectsFromLandmarks); + _ = _faceRectsFromDetectionsStream.TryGetNext(out var faceRectsFromDetections); OnFaceDetectionsOutput.Invoke(faceDetections); OnMultiFaceLandmarksOutput.Invoke(multiFaceLandmarks); OnFaceRectsFromLandmarksOutput.Invoke(faceRectsFromLandmarks); + OnFaceRectsFromDetectionsOutput.Invoke(faceRectsFromDetections); - return new FaceMeshValue(faceDetections, multiFaceLandmarks, faceRectsFromLandmarks); + return new FaceMeshValue(faceDetections, multiFaceLandmarks, faceRectsFromLandmarks, faceRectsFromDetections); } [AOT.MonoPInvokeCallback(typeof(CalculatorGraph.NativePacketCallback))] @@ -126,18 +136,34 @@ private static IntPtr FaceRectsFromLandmarksCallback(IntPtr graphPtr, IntPtr pac }).mpPtr; } + [AOT.MonoPInvokeCallback(typeof(CalculatorGraph.NativePacketCallback))] + private static IntPtr FaceRectsFromDetectionsCallback(IntPtr graphPtr, IntPtr packetPtr) + { + return InvokeIfGraphRunnerFound(graphPtr, packetPtr, (faceMeshGraph, ptr) => + { + using (var packet = new NormalizedRectVectorPacket(ptr, false)) + { + if (faceMeshGraph.TryGetPacketValue(packet, ref faceMeshGraph.prevFaceRectsFromDetectionsMicrosec, out var value)) + { + faceMeshGraph.OnFaceRectsFromDetectionsOutput.Invoke(value); + } + } + }).mpPtr; + } + protected void InitializeOutputStreams() { _faceDetectionsStream = new OutputStream>(calculatorGraph, _FaceDetectionsStreamName); _multiFaceLandmarksStream = new OutputStream>(calculatorGraph, _MultiFaceLandmarksStreamName); _faceRectsFromLandmarksStream = new OutputStream>(calculatorGraph, _FaceRectsFromLandmarksStreamName); + _faceRectsFromDetectionsStream = new OutputStream>(calculatorGraph, _FaceRectsFromDetectionsStreamName); } protected override IList RequestDependentAssets() { return new List { WaitForAsset("face_detection_short_range.bytes"), - WaitForAsset("face_landmark.bytes"), + WaitForAsset(refineLandmarks ? "face_landmark_with_attention.bytes" : "face_landmark.bytes"), }; } @@ -147,6 +173,7 @@ private SidePacket BuildSidePacket(ImageSource imageSource) SetImageTransformationOptions(sidePacket, imageSource); sidePacket.Emplace("num_faces", new IntPacket(maxNumFaces)); + sidePacket.Emplace("with_attention", new BoolPacket(refineLandmarks)); return sidePacket; } diff --git a/Assets/Mediapipe/Samples/Scenes/Face Mesh/FaceMeshSolution.cs b/Assets/Mediapipe/Samples/Scenes/Face Mesh/FaceMeshSolution.cs index 16f24da8b..b50ef1287 100644 --- a/Assets/Mediapipe/Samples/Scenes/Face Mesh/FaceMeshSolution.cs +++ b/Assets/Mediapipe/Samples/Scenes/Face Mesh/FaceMeshSolution.cs @@ -17,6 +17,7 @@ public class FaceMeshSolution : Solution [SerializeField] private DetectionListAnnotationController _faceDetectionsAnnotationController; [SerializeField] private MultiFaceLandmarkListAnnotationController _multiFaceLandmarksAnnotationController; [SerializeField] private NormalizedRectListAnnotationController _faceRectsFromLandmarksAnnotationController; + [SerializeField] private NormalizedRectListAnnotationController _faceRectsFromDetectionsAnnotationController; [SerializeField] private FaceMeshGraph _graphRunner; [SerializeField] private TextureFramePool _textureFramePool; @@ -30,6 +31,12 @@ public int maxNumFaces set => _graphRunner.maxNumFaces = value; } + public bool refineLandmarks + { + get => _graphRunner.refineLandmarks; + set => _graphRunner.refineLandmarks = value; + } + public long timeoutMillisec { get => _graphRunner.timeoutMillisec; @@ -83,6 +90,7 @@ private IEnumerator Run() _screen.texture = imageSource.GetCurrentTexture(); Logger.LogInfo(TAG, $"Max Num Faces = {maxNumFaces}"); + Logger.LogInfo(TAG, $"Refine Landmarks = {refineLandmarks}"); Logger.LogInfo(TAG, $"Running Mode = {runningMode}"); // Wait for completion of loading of dependent files, etc. @@ -98,6 +106,7 @@ private IEnumerator Run() _graphRunner.OnFaceDetectionsOutput.AddListener(OnFaceDetectionsOutput); _graphRunner.OnMultiFaceLandmarksOutput.AddListener(OnMultiFaceLandmarksOutput); _graphRunner.OnFaceRectsFromLandmarksOutput.AddListener(OnFaceRectsFromLandmarksOutput); + _graphRunner.OnFaceRectsFromDetectionsOutput.AddListener(OnFaceRectsFromDetectionsOutput); _graphRunner.StartRunAsync(imageSource).AssertOk(); } else @@ -112,6 +121,7 @@ private IEnumerator Run() SetupAnnotationController(_faceDetectionsAnnotationController, imageSource); SetupAnnotationController(_faceRectsFromLandmarksAnnotationController, imageSource); SetupAnnotationController(_multiFaceLandmarksAnnotationController, imageSource); + SetupAnnotationController(_faceRectsFromDetectionsAnnotationController, imageSource); while (true) { @@ -153,5 +163,10 @@ private void OnFaceRectsFromLandmarksOutput(List faceRectsFromLa { _faceRectsFromLandmarksAnnotationController.DrawLater(faceRectsFromLandmarks); } + + private void OnFaceRectsFromDetectionsOutput(List faceRectsFromDetections) + { + _faceRectsFromDetectionsAnnotationController.DrawLater(faceRectsFromDetections); + } } } diff --git a/Assets/Mediapipe/Samples/Scenes/Face Mesh/FaceMeshValue.cs b/Assets/Mediapipe/Samples/Scenes/Face Mesh/FaceMeshValue.cs index 11390326b..02dc46c29 100644 --- a/Assets/Mediapipe/Samples/Scenes/Face Mesh/FaceMeshValue.cs +++ b/Assets/Mediapipe/Samples/Scenes/Face Mesh/FaceMeshValue.cs @@ -13,12 +13,14 @@ public class FaceMeshValue public readonly List faceDetections; public readonly List multiFaceLandmarks; public readonly List faceRectsFromLandmarks; + public readonly List faceRectsFromDetections; - public FaceMeshValue(List faceDetections, List multiFaceLandmarks, List faceRectsFromLandmarks) + public FaceMeshValue(List faceDetections, List multiFaceLandmarks, List faceRectsFromLandmarks, List faceRectsFromDetections) { this.faceDetections = faceDetections; this.multiFaceLandmarks = multiFaceLandmarks; this.faceRectsFromLandmarks = faceRectsFromLandmarks; + this.faceRectsFromDetections = faceRectsFromDetections; } } } diff --git a/Assets/Mediapipe/Samples/Scenes/Face Mesh/face_mesh_cpu.txt b/Assets/Mediapipe/Samples/Scenes/Face Mesh/face_mesh_cpu.txt index edbc8421e..bb15008c2 100644 --- a/Assets/Mediapipe/Samples/Scenes/Face Mesh/face_mesh_cpu.txt +++ b/Assets/Mediapipe/Samples/Scenes/Face Mesh/face_mesh_cpu.txt @@ -32,6 +32,7 @@ output_stream: "multi_face_landmarks" output_stream: "face_rects_from_landmarks" output_stream: "face_detections" +output_stream: "face_rects_from_detections" # Throttles the images flowing downstream for flow control. It passes through # the very first incoming image unaltered, and waits for downstream nodes @@ -68,6 +69,7 @@ node { calculator: "FaceLandmarkFrontCpu" input_stream: "IMAGE:transformed_input_video" input_side_packet: "NUM_FACES:num_faces" + input_side_packet: "WITH_ATTENTION:with_attention" output_stream: "LANDMARKS:multi_face_landmarks" output_stream: "ROIS_FROM_LANDMARKS:face_rects_from_landmarks" output_stream: "DETECTIONS:face_detections" diff --git a/Assets/Mediapipe/Samples/Scenes/Face Mesh/face_mesh_gpu.txt b/Assets/Mediapipe/Samples/Scenes/Face Mesh/face_mesh_gpu.txt index 259e5523b..1a6793b59 100644 --- a/Assets/Mediapipe/Samples/Scenes/Face Mesh/face_mesh_gpu.txt +++ b/Assets/Mediapipe/Samples/Scenes/Face Mesh/face_mesh_gpu.txt @@ -33,6 +33,7 @@ output_stream: "multi_face_landmarks" output_stream: "face_rects_from_landmarks" output_stream: "face_detections" +output_stream: "face_rects_from_detections" # Throttles the images flowing downstream for flow control. It passes through # the very first incoming image unaltered, and waits for downstream nodes @@ -75,6 +76,7 @@ node { calculator: "FaceLandmarkFrontGpu" input_stream: "IMAGE:transformed_input_video" input_side_packet: "NUM_FACES:num_faces" + input_side_packet: "WITH_ATTENTION:with_attention" output_stream: "LANDMARKS:multi_face_landmarks" output_stream: "ROIS_FROM_LANDMARKS:face_rects_from_landmarks" output_stream: "DETECTIONS:face_detections" diff --git a/Assets/Mediapipe/Samples/Scenes/Face Mesh/face_mesh_opengles.txt b/Assets/Mediapipe/Samples/Scenes/Face Mesh/face_mesh_opengles.txt index ee4ecc3b6..070f67d32 100644 --- a/Assets/Mediapipe/Samples/Scenes/Face Mesh/face_mesh_opengles.txt +++ b/Assets/Mediapipe/Samples/Scenes/Face Mesh/face_mesh_opengles.txt @@ -32,6 +32,7 @@ output_stream: "multi_face_landmarks" output_stream: "face_rects_from_landmarks" output_stream: "face_detections" +output_stream: "face_rects_from_detections" # Throttles the images flowing downstream for flow control. It passes through # the very first incoming image unaltered, and waits for downstream nodes @@ -68,6 +69,7 @@ node { calculator: "FaceLandmarkFrontGpu" input_stream: "IMAGE:transformed_input_video" input_side_packet: "NUM_FACES:num_faces" + input_side_packet: "WITH_ATTENTION:with_attention" output_stream: "LANDMARKS:multi_face_landmarks" output_stream: "ROIS_FROM_LANDMARKS:face_rects_from_landmarks" output_stream: "DETECTIONS:face_detections" diff --git a/Assets/Mediapipe/Samples/Scenes/Iris Tracking/Iris Tracking.unity b/Assets/Mediapipe/Samples/Scenes/Iris Tracking/Iris Tracking.unity index 06fb3cfae..24d8baa3b 100644 --- a/Assets/Mediapipe/Samples/Scenes/Iris Tracking/Iris Tracking.unity +++ b/Assets/Mediapipe/Samples/Scenes/Iris Tracking/Iris Tracking.unity @@ -38,7 +38,7 @@ RenderSettings: m_ReflectionIntensity: 1 m_CustomReflection: {fileID: 0} m_Sun: {fileID: 0} - m_IndirectSpecularColor: {r: 0.4465791, g: 0.4964133, b: 0.57481784, a: 1} + m_IndirectSpecularColor: {r: 0.44402248, g: 0.49316555, b: 0.5722324, a: 1} m_UseRadianceAmbientProbe: 0 --- !u!157 &3 LightmapSettings: @@ -953,7 +953,7 @@ MonoBehaviour: m_GameObject: {fileID: 1228562676} m_Enabled: 1 m_EditorHideFlags: 0 - m_Script: {fileID: 11500000, guid: 58fd8fa4464d8ded49d2e5965318022c, type: 3} + m_Script: {fileID: 11500000, guid: b1fe9c61c434938249e2c9d067a337a5, type: 3} m_Name: m_EditorClassIdentifier: annotation: {fileID: 985505889} diff --git a/Assets/Mediapipe/Samples/Scenes/Iris Tracking/IrisTrackingSolution.cs b/Assets/Mediapipe/Samples/Scenes/Iris Tracking/IrisTrackingSolution.cs index e1b532462..3a51f13d3 100644 --- a/Assets/Mediapipe/Samples/Scenes/Iris Tracking/IrisTrackingSolution.cs +++ b/Assets/Mediapipe/Samples/Scenes/Iris Tracking/IrisTrackingSolution.cs @@ -16,7 +16,7 @@ public class IrisTrackingSolution : Solution [SerializeField] private RawImage _screen; [SerializeField] private DetectionListAnnotationController _faceDetectionsAnnotationController; [SerializeField] private NormalizedRectAnnotationController _faceRectAnnotationController; - [SerializeField] private FaceLandmarkListWithIrisAnnotationController _faceLandmarksWithIrisAnnotationController; + [SerializeField] private FaceLandmarkListAnnotationController _faceLandmarksWithIrisAnnotationController; [SerializeField] private IrisTrackingGraph _graphRunner; [SerializeField] private TextureFramePool _textureFramePool; diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Objects/Multi FaceLandmarkList Annotation.prefab b/Packages/com.github.homuler.mediapipe/Runtime/Objects/Multi FaceLandmarkList Annotation.prefab index 6669f9b4c..801325b9b 100644 --- a/Packages/com.github.homuler.mediapipe/Runtime/Objects/Multi FaceLandmarkList Annotation.prefab +++ b/Packages/com.github.homuler.mediapipe/Runtime/Objects/Multi FaceLandmarkList Annotation.prefab @@ -43,9 +43,13 @@ MonoBehaviour: m_Script: {fileID: 11500000, guid: d1ec0e202f29d7ee28cccba68415d95b, type: 3} m_Name: m_EditorClassIdentifier: - _annotationPrefab: {fileID: 7273100862690186831, guid: 16074ce928b0558829296ab0d9ccadf3, + _annotationPrefab: {fileID: 7477175687846711451, guid: c0d008e23161282b39199515fa2c0da4, type: 3} - landmarkColor: {r: 0, g: 1, b: 0, a: 1} - landmarkRadius: 10 - connectionColor: {r: 1, g: 0, b: 0, a: 1} - connectionWidth: 1 + _faceLandmarkColor: {r: 0, g: 1, b: 0, a: 1} + _irisLandmarkColor: {r: 1, g: 0.92156863, b: 0.015686275, a: 1} + _faceLandmarkRadius: 10 + _irisLandmarkRadius: 10 + _faceConnectionColor: {r: 1, g: 0, b: 0, a: 1} + _irisCircleColor: {r: 0, g: 0, b: 1, a: 1} + _faceConnectionWidth: 1 + _irisCircleWidth: 1 diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Resources/face_landmark_with_attention.bytes.meta b/Packages/com.github.homuler.mediapipe/Runtime/Resources/face_landmark_with_attention.bytes.meta new file mode 100644 index 000000000..b8bc1505a --- /dev/null +++ b/Packages/com.github.homuler.mediapipe/Runtime/Resources/face_landmark_with_attention.bytes.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 1cf05e630aa9cd6f38522c83264115ee +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Unity/Annotation/FaceLandmarkListAnnotationController.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Unity/Annotation/FaceLandmarkListAnnotationController.cs index 0fa96f932..6d9735609 100644 --- a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Unity/Annotation/FaceLandmarkListAnnotationController.cs +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Unity/Annotation/FaceLandmarkListAnnotationController.cs @@ -9,9 +9,10 @@ namespace Mediapipe.Unity { - public class FaceLandmarkListAnnotationController : AnnotationController + public class FaceLandmarkListAnnotationController : AnnotationController { [SerializeField] private bool _visualizeZ = false; + [SerializeField] private int _circleVertices = 128; private IList _currentTarget; @@ -39,7 +40,7 @@ public void DrawLater(NormalizedLandmarkList target) protected override void SyncNow() { isStale = false; - annotation.Draw(_currentTarget, _visualizeZ); + annotation.Draw(_currentTarget, _visualizeZ, _circleVertices); } } } diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Unity/Annotation/FaceLandmarkListWithIrisAnnotation.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Unity/Annotation/FaceLandmarkListWithIrisAnnotation.cs index f8afcd4ed..0a63ca3d7 100644 --- a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Unity/Annotation/FaceLandmarkListWithIrisAnnotation.cs +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Unity/Annotation/FaceLandmarkListWithIrisAnnotation.cs @@ -40,12 +40,66 @@ public override RotationAngle rotationAngle } } + public void SetFaceLandmarkColor(Color color) + { + _faceLandmarkListAnnotation.SetLandmarkColor(color); + } + + public void SetIrisLandmarkColor(Color color) + { + _leftIrisLandmarkListAnnotation.SetLandmarkColor(color); + _rightIrisLandmarkListAnnotation.SetLandmarkColor(color); + } + + public void SetFaceLandmarkRadius(float radius) + { + _faceLandmarkListAnnotation.SetLandmarkRadius(radius); + } + + public void SetIrisLandmarkRadius(float radius) + { + _leftIrisLandmarkListAnnotation.SetLandmarkRadius(radius); + _rightIrisLandmarkListAnnotation.SetLandmarkRadius(radius); + } + + public void SetFaceConnectionColor(Color color) + { + _faceLandmarkListAnnotation.SetConnectionColor(color); + } + + public void SetFaceConnectionWidth(float width) + { + _faceLandmarkListAnnotation.SetConnectionWidth(width); + } + + public void SetIrisCircleColor(Color color) + { + _leftIrisLandmarkListAnnotation.SetCircleColor(color); + _rightIrisLandmarkListAnnotation.SetCircleColor(color); + } + + public void SetIrisCircleWidth(float width) + { + _leftIrisLandmarkListAnnotation.SetCircleWidth(width); + _rightIrisLandmarkListAnnotation.SetCircleWidth(width); + } + + public void Draw(IList target, bool visualizeZ = false, int circleVertices = 128) + { + var (faceLandmarks, leftLandmarks, rightLandmarks) = PartitionLandmarkList(target); + DrawFaceLandmarkList(faceLandmarks, visualizeZ); + DrawLeftIrisLandmarkList(leftLandmarks, visualizeZ, circleVertices); + DrawRightIrisLandmarkList(rightLandmarks, visualizeZ, circleVertices); + } + + public void Draw(NormalizedLandmarkList target, bool visualizeZ = false, int circleVertices = 128) + { + Draw(target.Landmark, visualizeZ, circleVertices); + } + public void DrawFaceLandmarkList(IList target, bool visualizeZ = false) { - if (ActivateFor(target)) - { - _faceLandmarkListAnnotation.Draw(target, visualizeZ); - } + _faceLandmarkListAnnotation.Draw(target, visualizeZ); } public void DrawLeftIrisLandmarkList(IList target, bool visualizeZ = false, int circleVertices = 128) @@ -60,7 +114,7 @@ public void DrawRightIrisLandmarkList(IList target, bool vis _rightIrisLandmarkListAnnotation.Draw(target, visualizeZ, circleVertices); } - public static (IList, IList, IList) PartitionLandmarkList(IList landmarks) + private static (IList, IList, IList) PartitionLandmarkList(IList landmarks) { if (landmarks == null) { diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Unity/Annotation/FaceLandmarkListWithIrisAnnotationController.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Unity/Annotation/FaceLandmarkListWithIrisAnnotationController.cs deleted file mode 100644 index 086edac79..000000000 --- a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Unity/Annotation/FaceLandmarkListWithIrisAnnotationController.cs +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright (c) 2021 homuler -// -// Use of this source code is governed by an MIT-style -// license that can be found in the LICENSE file or at -// https://opensource.org/licenses/MIT. - -using System.Collections.Generic; -using UnityEngine; - -namespace Mediapipe.Unity -{ - public class FaceLandmarkListWithIrisAnnotationController : AnnotationController - { - [SerializeField] private bool _visualizeZ = false; - [SerializeField] private int _circleVertices = 128; - - private IList _currentFaceLandmarkList; - private IList _currentLeftIrisLandmarkList; - private IList _currentRightIrisLandmarkList; - - public void DrawNow(IList faceLandmarkList, IList leftIrisLandmarkList, IList rightIrisLandmarkList) - { - _currentFaceLandmarkList = faceLandmarkList; - _currentLeftIrisLandmarkList = leftIrisLandmarkList; - _currentRightIrisLandmarkList = rightIrisLandmarkList; - SyncNow(); - } - - public void DrawNow(IList target) - { - var (faceLandmarkList, leftIrisLandmarkList, rightIrisLandmarkList) = FaceLandmarkListWithIrisAnnotation.PartitionLandmarkList(target); - DrawNow(faceLandmarkList, leftIrisLandmarkList, rightIrisLandmarkList); - } - - public void DrawNow(NormalizedLandmarkList target) - { - DrawNow(target?.Landmark); - } - - public void DrawLater(IList target) - { - var (faceLandmarkList, leftIrisLandmarkList, rightIrisLandmarkList) = FaceLandmarkListWithIrisAnnotation.PartitionLandmarkList(target); - DrawFaceLandmarkListLater(faceLandmarkList); - DrawLeftIrisLandmarkListLater(leftIrisLandmarkList); - DrawRightIrisLandmarkListLater(rightIrisLandmarkList); - } - - public void DrawLater(NormalizedLandmarkList target) - { - DrawLater(target?.Landmark); - } - - public void DrawFaceLandmarkListLater(IList faceLandmarkList) - { - UpdateCurrentTarget(faceLandmarkList, ref _currentFaceLandmarkList); - } - - public void DrawLeftIrisLandmarkListLater(IList leftIrisLandmarkList) - { - UpdateCurrentTarget(leftIrisLandmarkList, ref _currentLeftIrisLandmarkList); - } - - public void DrawRightIrisLandmarkListLater(IList rightIrisLandmarkList) - { - UpdateCurrentTarget(rightIrisLandmarkList, ref _currentRightIrisLandmarkList); - } - - protected override void SyncNow() - { - isStale = false; - annotation.DrawFaceLandmarkList(_currentFaceLandmarkList, _visualizeZ); - annotation.DrawLeftIrisLandmarkList(_currentLeftIrisLandmarkList, _visualizeZ, _circleVertices); - annotation.DrawRightIrisLandmarkList(_currentRightIrisLandmarkList, _visualizeZ, _circleVertices); - } - } -} diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Unity/Annotation/FaceLandmarkListWithIrisAnnotationController.cs.meta b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Unity/Annotation/FaceLandmarkListWithIrisAnnotationController.cs.meta deleted file mode 100644 index b4e9083dc..000000000 --- a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Unity/Annotation/FaceLandmarkListWithIrisAnnotationController.cs.meta +++ /dev/null @@ -1,11 +0,0 @@ -fileFormatVersion: 2 -guid: 58fd8fa4464d8ded49d2e5965318022c -MonoImporter: - externalObjects: {} - serializedVersion: 2 - defaultReferences: [] - executionOrder: 0 - icon: {instanceID: 0} - userData: - assetBundleName: - assetBundleVariant: diff --git a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Unity/Annotation/MultiFaceLandmarkListAnnotation.cs b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Unity/Annotation/MultiFaceLandmarkListAnnotation.cs index 6836c20cd..bbc532a89 100644 --- a/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Unity/Annotation/MultiFaceLandmarkListAnnotation.cs +++ b/Packages/com.github.homuler.mediapipe/Runtime/Scripts/Unity/Annotation/MultiFaceLandmarkListAnnotation.cs @@ -9,43 +9,75 @@ namespace Mediapipe.Unity { - public sealed class MultiFaceLandmarkListAnnotation : ListAnnotation + public sealed class MultiFaceLandmarkListAnnotation : ListAnnotation { - [SerializeField] private Color _landmarkColor = Color.green; - [SerializeField] private float _landmarkRadius = 10.0f; - [SerializeField] private Color _connectionColor = Color.red; - [SerializeField, Range(0, 1)] private float _connectionWidth = 1.0f; + [SerializeField] private Color _faceLandmarkColor = Color.green; + [SerializeField] private Color _irisLandmarkColor = Color.yellow; + [SerializeField] private float _faceLandmarkRadius = 10.0f; + [SerializeField] private float _irisLandmarkRadius = 10.0f; + [SerializeField] private Color _faceConnectionColor = Color.red; + [SerializeField] private Color _irisCircleColor = Color.blue; + [SerializeField, Range(0, 1)] private float _faceConnectionWidth = 1.0f; + [SerializeField, Range(0, 1)] private float _irisCircleWidth = 1.0f; private void OnValidate() { - ApplyLandmarkColor(_landmarkColor); - ApplyLandmarkRadius(_landmarkRadius); - ApplyConnectionColor(_connectionColor); - ApplyConnectionWidth(_connectionWidth); + ApplyFaceLandmarkColor(_faceLandmarkColor); + ApplyIrisLandmarkColor(_irisLandmarkColor); + ApplyFaceLandmarkRadius(_faceLandmarkRadius); + ApplyIrisLandmarkRadius(_irisLandmarkRadius); + ApplyFaceConnectionColor(_faceConnectionColor); + ApplyIrisCircleColor(_irisCircleColor); + ApplyFaceConnectionWidth(_faceConnectionWidth); + ApplyIrisCircleWidth(_irisCircleWidth); } - public void SetLandmarkRadius(float landmarkRadius) + public void SetFaceLandmarkRadius(float radius) { - _landmarkRadius = landmarkRadius; - ApplyLandmarkRadius(_landmarkRadius); + _faceLandmarkRadius = radius; + ApplyFaceLandmarkRadius(_faceLandmarkRadius); } - public void SetLandmarkColor(Color landmarkColor) + public void SetIrisLandmarkRadius(float radius) { - _landmarkColor = landmarkColor; - ApplyLandmarkColor(_landmarkColor); + _irisLandmarkRadius = radius; + ApplyIrisLandmarkRadius(_irisLandmarkRadius); } - public void SetConnectionWidth(float connectionWidth) + public void SetFaceLandmarkColor(Color color) { - _connectionWidth = connectionWidth; - ApplyConnectionWidth(_connectionWidth); + _faceLandmarkColor = color; + ApplyFaceLandmarkColor(_faceLandmarkColor); } - public void SetConnectionColor(Color connectionColor) + public void SetIrisLandmarkColor(Color color) { - _connectionColor = connectionColor; - ApplyConnectionColor(_connectionColor); + _irisLandmarkColor = color; + ApplyIrisLandmarkColor(_irisLandmarkColor); + } + + public void SetFaceConnectionWidth(float width) + { + _faceConnectionWidth = width; + ApplyFaceConnectionWidth(_faceConnectionWidth); + } + + public void SetFaceConnectionColor(Color color) + { + _faceConnectionColor = color; + ApplyFaceConnectionColor(_faceConnectionColor); + } + + public void SetIrisCircleWidth(float width) + { + _irisCircleWidth = width; + ApplyIrisCircleWidth(_irisCircleWidth); + } + + public void SetIrisCircleColor(Color color) + { + _irisCircleColor = color; + ApplyIrisCircleColor(_irisCircleColor); } public void Draw(IList targets, bool visualizeZ = false) @@ -59,45 +91,81 @@ public void Draw(IList targets, bool visualizeZ = false) } } - protected override FaceLandmarkListAnnotation InstantiateChild(bool isActive = true) + protected override FaceLandmarkListWithIrisAnnotation InstantiateChild(bool isActive = true) { var annotation = base.InstantiateChild(isActive); - annotation.SetLandmarkRadius(_landmarkRadius); - annotation.SetLandmarkColor(_landmarkColor); - annotation.SetConnectionWidth(_connectionWidth); - annotation.SetConnectionColor(_connectionColor); + annotation.SetFaceLandmarkRadius(_faceLandmarkRadius); + annotation.SetIrisLandmarkRadius(_irisLandmarkRadius); + annotation.SetFaceLandmarkColor(_faceLandmarkColor); + annotation.SetIrisLandmarkColor(_irisLandmarkColor); + annotation.SetFaceConnectionWidth(_faceConnectionWidth); + annotation.SetFaceConnectionColor(_faceConnectionColor); + annotation.SetIrisCircleWidth(_irisCircleWidth); + annotation.SetIrisCircleColor(_irisCircleColor); return annotation; } - private void ApplyLandmarkRadius(float landmarkRadius) + private void ApplyFaceLandmarkRadius(float radius) + { + foreach (var faceLandmarkList in children) + { + if (faceLandmarkList != null) { faceLandmarkList.SetFaceLandmarkRadius(radius); } + } + } + + private void ApplyIrisLandmarkRadius(float radius) + { + foreach (var faceLandmarkList in children) + { + if (faceLandmarkList != null) { faceLandmarkList.SetIrisLandmarkRadius(radius); } + } + } + + private void ApplyFaceLandmarkColor(Color color) + { + foreach (var faceLandmarkList in children) + { + if (faceLandmarkList != null) { faceLandmarkList.SetFaceLandmarkColor(color); } + } + } + + private void ApplyIrisLandmarkColor(Color color) + { + foreach (var faceLandmarkList in children) + { + if (faceLandmarkList != null) { faceLandmarkList.SetIrisLandmarkColor(color); } + } + } + + private void ApplyFaceConnectionWidth(float width) { foreach (var faceLandmarkList in children) { - if (faceLandmarkList != null) { faceLandmarkList.SetLandmarkRadius(landmarkRadius); } + if (faceLandmarkList != null) { faceLandmarkList.SetFaceConnectionWidth(width); } } } - private void ApplyLandmarkColor(Color landmarkColor) + private void ApplyFaceConnectionColor(Color color) { foreach (var faceLandmarkList in children) { - if (faceLandmarkList != null) { faceLandmarkList.SetLandmarkColor(landmarkColor); } + if (faceLandmarkList != null) { faceLandmarkList.SetFaceConnectionColor(color); } } } - private void ApplyConnectionWidth(float connectionWidth) + private void ApplyIrisCircleWidth(float width) { foreach (var faceLandmarkList in children) { - if (faceLandmarkList != null) { faceLandmarkList.SetConnectionWidth(connectionWidth); } + if (faceLandmarkList != null) { faceLandmarkList.SetIrisCircleWidth(width); } } } - private void ApplyConnectionColor(Color connectionColor) + private void ApplyIrisCircleColor(Color color) { foreach (var faceLandmarkList in children) { - if (faceLandmarkList != null) { faceLandmarkList.SetConnectionColor(connectionColor); } + if (faceLandmarkList != null) { faceLandmarkList.SetIrisCircleColor(color); } } } } diff --git a/mediapipe_api/BUILD b/mediapipe_api/BUILD index e7ada6fc9..93e519949 100644 --- a/mediapipe_api/BUILD +++ b/mediapipe_api/BUILD @@ -152,6 +152,7 @@ pkg_asset( "@com_google_mediapipe//mediapipe/modules/face_detection:face_detection_full_range_sparse.tflite", "@com_google_mediapipe//mediapipe/modules/face_detection:face_detection_short_range.tflite", "@com_google_mediapipe//mediapipe/modules/face_landmark:face_landmark.tflite", + "@com_google_mediapipe//mediapipe/modules/face_landmark:face_landmark_with_attention.tflite", "@com_google_mediapipe//mediapipe/modules/hand_landmark:hand_landmark.tflite", "@com_google_mediapipe//mediapipe/modules/hand_landmark:handedness.txt", "@com_google_mediapipe//mediapipe/modules/holistic_landmark:hand_recrop.tflite",