Skip to content

Commit

Permalink
perf!: implement ImageFrame#TryReadChannel and MaskShader (#507)
Browse files Browse the repository at this point in the history 
* implement MaskShader

* feat!(plugin): implement channel-related methods in a safe way

* remove unused APIs

* remove unnecessary SignalAbort flags

* feat!: ImageFrame#TryReadChannel

* implement ImageFrame#TryReadChannelNormalized

* perf(sample): render annotation using MaskShader in Hair Segmentation

* change namespace + set MinValue

* refactor: remove the screen only for MaskAnnotation

* create a screen for MaskAnnotation again

* feat: change the options of MaskAnnotation at runtime

* TryReadChannel receives `isVerticallyFlipped`

* release GraphicsBuffer

* revert MaskShader

* rotate output mask

* consider isHorizontallyFlipped

* remove code for debug
  • Loading branch information
@homuler
homuler committed Apr 7, 2022
1 parent f3a0642 commit 05adcb5
Show file tree
Hide file tree
Showing 20 changed files with 1,891 additions and 366 deletions.
105 changes: 26 additions & 79 deletions Assets/Mediapipe/Samples/Scenes/Hair Segmentation/Hair Segmentation.unity
View file
Original file line number Diff line number Diff line change
Expand Up @@ -242,12 +242,18 @@ PrefabInstance:
objectReference: {fileID: 0}
m_RemovedComponents: []
m_SourcePrefab: {fileID: 100100000, guid: 5713b959e3c77a58fb258133fc8e4aef, type: 3}
--- !u!224 &107400409 stripped
RectTransform:
m_CorrespondingSourceObject: {fileID: 3259285889726014650, guid: 5713b959e3c77a58fb258133fc8e4aef,
--- !u!114 &107400409 stripped
MonoBehaviour:
m_CorrespondingSourceObject: {fileID: 3259285889726014649, guid: 5713b959e3c77a58fb258133fc8e4aef,
type: 3}
m_PrefabInstance: {fileID: 107400408}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 0}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: 1344c3c82d62a2a41a3576d8abb8e3ea, type: 3}
m_Name:
m_EditorClassIdentifier:
--- !u!4 &107400410 stripped
Transform:
m_CorrespondingSourceObject: {fileID: 7074087083388479136, guid: 5713b959e3c77a58fb258133fc8e4aef,
Expand Down Expand Up @@ -275,8 +281,6 @@ MonoBehaviour:
annotation: {fileID: 1452360049}
_maskWidth: 512
_maskHeight: 512
_minAlpha: 0.9
_maxAlpha: 1
--- !u!114 &107400413 stripped
MonoBehaviour:
m_CorrespondingSourceObject: {fileID: 2020366215714292840, guid: 5713b959e3c77a58fb258133fc8e4aef,
Expand Down Expand Up @@ -508,12 +512,27 @@ PrefabInstance:
type: 3}
propertyPath: screen
value:
objectReference: {fileID: 1625703806}
objectReference: {fileID: 0}
- target: {fileID: 5850198979010014858, guid: 24d349d9c78db0f3d94d4657029f4c5e,
type: 3}
propertyPath: _screen
value:
objectReference: {fileID: 1625703806}
objectReference: {fileID: 107400409}
- target: {fileID: 5850198979010014858, guid: 24d349d9c78db0f3d94d4657029f4c5e,
type: 3}
propertyPath: _maskShader
value:
objectReference: {fileID: 4800000, guid: 7adfa9ebe754989caa35dbb2c5526d1c, type: 3}
- target: {fileID: 5850198979010014858, guid: 24d349d9c78db0f3d94d4657029f4c5e,
type: 3}
propertyPath: _maskTexture
value:
objectReference: {fileID: 0}
- target: {fileID: 5850198979010014858, guid: 24d349d9c78db0f3d94d4657029f4c5e,
type: 3}
propertyPath: _minConfidence
value: 0.275
objectReference: {fileID: 0}
m_RemovedComponents: []
m_SourcePrefab: {fileID: 100100000, guid: 24d349d9c78db0f3d94d4657029f4c5e, type: 3}
--- !u!114 &1452360049 stripped
Expand All @@ -528,78 +547,6 @@ MonoBehaviour:
m_Script: {fileID: 11500000, guid: f993d16dcccfcecb6892ad3cc2ea76c8, type: 3}
m_Name:
m_EditorClassIdentifier:
--- !u!1 &1625703804
GameObject:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
serializedVersion: 6
m_Component:
- component: {fileID: 1625703805}
- component: {fileID: 1625703807}
- component: {fileID: 1625703806}
m_Layer: 5
m_Name: HairMask Screen
m_TagString: Untagged
m_Icon: {fileID: 0}
m_NavMeshLayer: 0
m_StaticEditorFlags: 0
m_IsActive: 1
--- !u!224 &1625703805
RectTransform:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1625703804}
m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
m_LocalPosition: {x: 0, y: 0, z: 0}
m_LocalScale: {x: 1, y: 1, z: 1}
m_Children: []
m_Father: {fileID: 107400409}
m_RootOrder: 1
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
m_AnchorMin: {x: 0, y: 0}
m_AnchorMax: {x: 1, y: 1}
m_AnchoredPosition: {x: 0, y: 0}
m_SizeDelta: {x: 0, y: 0}
m_Pivot: {x: 0.5, y: 0.5}
--- !u!114 &1625703806
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1625703804}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: 1344c3c82d62a2a41a3576d8abb8e3ea, type: 3}
m_Name:
m_EditorClassIdentifier:
m_Material: {fileID: 0}
m_Color: {r: 1, g: 1, b: 1, a: 0}
m_RaycastTarget: 1
m_RaycastPadding: {x: 0, y: 0, z: 0, w: 0}
m_Maskable: 1
m_OnCullStateChanged:
m_PersistentCalls:
m_Calls: []
m_Texture: {fileID: 0}
m_UVRect:
serializedVersion: 2
x: 0
y: 0
width: 1
height: 1
--- !u!222 &1625703807
CanvasRenderer:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1625703804}
m_CullTransparentMesh: 1
--- !u!1 &1806680259
GameObject:
m_ObjectHideFlags: 0
Expand Down
19 changes: 18 additions & 1 deletion Assets/Mediapipe/Samples/Scenes/Hair Segmentation/HairSegmentationGraph.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,9 +100,26 @@ private SidePacket BuildSidePacket(ImageSource imageSource)
var sidePacket = new SidePacket();

SetImageTransformationOptions(sidePacket, imageSource);
var outputRotation = imageSource.isHorizontallyFlipped ? imageSource.rotation.Reverse() : imageSource.rotation;

// TODO: refactoring
// The orientation of the output image must match that of the input image.
var isInverted = CoordinateSystem.ImageCoordinate.IsInverted(imageSource.rotation);
var outputRotation = imageSource.rotation;
var outputHorizontallyFlipped = !isInverted && imageSource.isHorizontallyFlipped;
var outputVerticallyFlipped = imageSource.isVerticallyFlipped ^ (isInverted && imageSource.isHorizontallyFlipped);

if ((outputHorizontallyFlipped && outputVerticallyFlipped) || outputRotation == RotationAngle.Rotation180)
{
outputRotation = outputRotation.Add(RotationAngle.Rotation180);
outputHorizontallyFlipped = !outputHorizontallyFlipped;
outputVerticallyFlipped = !outputVerticallyFlipped;
}

sidePacket.Emplace("output_rotation", new IntPacket((int)outputRotation));
sidePacket.Emplace("output_horizontally_flipped", new BoolPacket(outputHorizontallyFlipped));
sidePacket.Emplace("output_vertically_flipped", new BoolPacket(outputVerticallyFlipped));

Logger.LogDebug($"output_rotation = {outputRotation}, output_horizontally_flipped = {outputHorizontallyFlipped}, output_vertically_flipped = {outputVerticallyFlipped}");
return sidePacket;
}
}
Expand Down
4 changes: 3 additions & 1 deletion Assets/Mediapipe/Samples/Scenes/Hair Segmentation/hair_segmentation_cpu.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,5 +165,7 @@ node: {
calculator: "ImageTransformationCalculator"
input_stream: "IMAGE:hair_mask_rotated"
input_side_packet: "ROTATION_DEGREES:output_rotation"
input_side_packet: "FLIP_HORIZONTALLY:output_horizontally_flipped"
input_side_packet: "FLIP_VERTICALLY:output_vertically_flipped"
output_stream: "IMAGE:hair_mask"
}
}
2 changes: 2 additions & 0 deletions Assets/Mediapipe/Samples/Scenes/Hair Segmentation/hair_segmentation_gpu.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,8 @@ node: {
calculator: "ImageTransformationCalculator"
input_stream: "IMAGE_GPU:hair_mask_gpu"
input_side_packet: "ROTATION_DEGREES:output_rotation"
input_side_packet: "FLIP_HORIZONTALLY:output_horizontally_flipped"
input_side_packet: "FLIP_VERTICALLY:output_vertically_flipped"
output_stream: "IMAGE_GPU:hair_mask_unrotated_gpu"
}

Expand Down
2 changes: 2 additions & 0 deletions Assets/Mediapipe/Samples/Scenes/Hair Segmentation/hair_segmentation_opengles.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,8 @@ node: {
calculator: "ImageTransformationCalculator"
input_stream: "IMAGE_GPU:hair_mask_gpu"
input_side_packet: "ROTATION_DEGREES:output_rotation"
input_side_packet: "FLIP_HORIZONTALLY:output_horizontally_flipped"
input_side_packet: "FLIP_VERTICALLY:output_vertically_flipped"
output_stream: "IMAGE_GPU:hair_mask_unrotated_gpu"
}

Expand Down
Loading

0 comments on commit 05adcb5

Please sign in to comment.