diff --git a/.gitignore b/.gitignore index 86308dbc9..10a8aa91a 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ bazel-* Assets/StreamingAssets* Assets/MediaPipe/SDK/Resources/* !Assets/MediaPipe/SDK/Resources/*.meta +Assets/MediaPipe/SDK/Resources/object_detection_3d.bytes.meta Assets/MediaPipe/SDK/Plugins/opencv_world*.dll.meta Assets/MediaPipe/SDK/Plugins/Protobuf* Assets/MediaPipe/SDK/Scripts/Protobuf/**/*.cs diff --git a/Assets/MediaPipe/Examples/Graphs/BoxTracking/Resources/box_tracking_android.txt b/Assets/MediaPipe/Examples/Graphs/BoxTracking/Resources/box_tracking_android.txt index ab29f1be1..fa57e6931 100644 --- a/Assets/MediaPipe/Examples/Graphs/BoxTracking/Resources/box_tracking_android.txt +++ b/Assets/MediaPipe/Examples/Graphs/BoxTracking/Resources/box_tracking_android.txt @@ -16,7 +16,7 @@ # # CHANGES: # - Add ImageTransformationCalculator and rotate the input -# - Add GlScalerCalculator and flips the output +# - Add GlScalerCalculator and flip the output # MediaPipe graph that performs object detection and tracking with TensorFlow # Lite on GPU. diff --git a/Assets/MediaPipe/Examples/Graphs/InstantMotionTracking/Resources/instant_motion_tracking_gpu.txt b/Assets/MediaPipe/Examples/Graphs/InstantMotionTracking/Resources/instant_motion_tracking_gpu.txt index 497298d9d..5aa9fa7ec 100644 --- a/Assets/MediaPipe/Examples/Graphs/InstantMotionTracking/Resources/instant_motion_tracking_gpu.txt +++ b/Assets/MediaPipe/Examples/Graphs/InstantMotionTracking/Resources/instant_motion_tracking_gpu.txt @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Copied from mediapipe/graphs/instant_motion_tracking/instant_motion_tracking.pbtxt +# Copied from mediapipe/graphs/object_detection_3d/object_occlusion_tracking.pbtxt # # CHANGES: # - Remove GIF related inputs diff --git a/Assets/MediaPipe/Examples/Graphs/ObjectDetection3d.meta b/Assets/MediaPipe/Examples/Graphs/ObjectDetection3d.meta new file mode 100644 index 000000000..307f4cc62 --- /dev/null +++ b/Assets/MediaPipe/Examples/Graphs/ObjectDetection3d.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: a86f58a484db2a80b96da1c08ecb96e6 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/MediaPipe/Examples/Graphs/ObjectDetection3d/Objects.meta b/Assets/MediaPipe/Examples/Graphs/ObjectDetection3d/Objects.meta new file mode 100644 index 000000000..475221ce6 --- /dev/null +++ b/Assets/MediaPipe/Examples/Graphs/ObjectDetection3d/Objects.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 9062d3b0bd298661384381cc49f55bfe +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/MediaPipe/Examples/Graphs/ObjectDetection3d/Objects/ObjectDetection3d.prefab b/Assets/MediaPipe/Examples/Graphs/ObjectDetection3d/Objects/ObjectDetection3d.prefab new file mode 100644 index 000000000..473c75883 --- /dev/null +++ b/Assets/MediaPipe/Examples/Graphs/ObjectDetection3d/Objects/ObjectDetection3d.prefab @@ -0,0 +1,52 @@ +%YAML 1.1 +%TAG !u! tag:unity3d.com,2011: +--- !u!1 &8245872100598249146 +GameObject: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + serializedVersion: 6 + m_Component: + - component: {fileID: 8245872100598249144} + - component: {fileID: 6244794291530564790} + m_Layer: 0 + m_Name: ObjectDetection3d + m_TagString: Untagged + m_Icon: {fileID: 0} + m_NavMeshLayer: 0 + m_StaticEditorFlags: 0 + m_IsActive: 1 +--- !u!4 &8245872100598249144 +Transform: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 8245872100598249146} + m_LocalRotation: {x: 0, y: 0, z: 0, w: 1} + m_LocalPosition: {x: 0, y: 0, z: -10} + m_LocalScale: {x: 1, y: 1, z: 1} + m_Children: [] + m_Father: {fileID: 0} + m_RootOrder: 0 + m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} +--- !u!114 &6244794291530564790 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 8245872100598249146} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: a1ced5119e2040cbc9cf2235b2cca548, type: 3} + m_Name: + m_EditorClassIdentifier: + gpuConfig: {fileID: 4900000, guid: 339e94fa26caa7779bba22281ecb7cef, type: 3} + cpuConfig: {fileID: 0} + androidConfig: {fileID: 4900000, guid: ef2e1f6b03bf3077d8e6ff3378c95501, type: 3} + category: 3 + maxNumObjects: 5 + objTextureAsset: {fileID: 4900000, guid: ff3b4f7354b931c37b0cfb5ff65dec1c, type: 3} + boxTextureAsset: {fileID: 4900000, guid: 12957b0bde1a6c4a18fb9173ddebaa73, type: 3} diff --git a/Assets/MediaPipe/Examples/Graphs/ObjectDetection3d/Objects/ObjectDetection3d.prefab.meta b/Assets/MediaPipe/Examples/Graphs/ObjectDetection3d/Objects/ObjectDetection3d.prefab.meta new file mode 100644 index 000000000..bcef71b46 --- /dev/null +++ b/Assets/MediaPipe/Examples/Graphs/ObjectDetection3d/Objects/ObjectDetection3d.prefab.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: e062f09bfc63ada65a687ade46bef7ff +PrefabImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/MediaPipe/Examples/Graphs/ObjectDetection3d/Resources.meta b/Assets/MediaPipe/Examples/Graphs/ObjectDetection3d/Resources.meta new file mode 100644 index 000000000..53616fef4 --- /dev/null +++ b/Assets/MediaPipe/Examples/Graphs/ObjectDetection3d/Resources.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 5012a5d0f3496ff5eab03425dc7bf471 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/MediaPipe/Examples/Graphs/ObjectDetection3d/Resources/object_detection_3d_android.txt b/Assets/MediaPipe/Examples/Graphs/ObjectDetection3d/Resources/object_detection_3d_android.txt new file mode 100644 index 000000000..b56e7e224 --- /dev/null +++ b/Assets/MediaPipe/Examples/Graphs/ObjectDetection3d/Resources/object_detection_3d_android.txt @@ -0,0 +1,166 @@ +# Copyright 2019 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copied from mediapipe/graphs/object_detection_3d/object_occlusion_tracking.pbtxt +# +# CHANGES: +# - Add ImageTransformationCalculator and flip the input +# - Add GlScalerCalculator and flip the output + +# MediaPipe graph that performs box tracking with TensorFlow Lite on GPU. + +# Images coming into and out of the graph. +input_stream: "input_video" +input_stream: "WIDTH:input_width" +input_stream: "HEIGHT:input_height" +input_side_packet: "LABELS_CSV:allowed_labels" +input_side_packet: "MODEL_SCALE:model_scale" +input_side_packet: "MODEL_TRANSFORMATION:model_transformation" +input_side_packet: "TEXTURE:box_texture" +input_side_packet: "MAX_NUM_OBJECTS:max_num_objects" +input_side_packet: "ANIMATION_ASSET:box_asset_name" +input_side_packet: "MASK_TEXTURE:obj_texture" +input_side_packet: "MASK_ASSET:obj_asset_name" +output_stream: "output_video" + +# Throttles the images flowing downstream for flow control. It passes through +# the very first incoming image unaltered, and waits for downstream nodes +# (calculators and subgraphs) in the graph to finish their tasks before it +# passes through another image. All images that come in while waiting are +# dropped, limiting the number of in-flight images in most part of the graph to +# 1. This prevents the downstream nodes from queuing up incoming images and data +# excessively, which leads to increased latency and memory usage, unwanted in +# real-time mobile applications. It also eliminates unnecessarily computation, +# e.g., the output produced by a node may get dropped downstream if the +# subsequent nodes are still busy processing previous inputs. +node { + calculator: "FlowLimiterCalculator" + input_stream: "input_video" + input_stream: "FINISHED:output_video" + input_stream_info: { + tag_index: "FINISHED" + back_edge: true + } + output_stream: "throttled_input_video" +} + +# Flip vertically because the input image is aligned from bottom-left to top-right. +node: { + calculator: "ImageTransformationCalculator" + input_stream: "IMAGE_GPU:throttled_input_video" + output_stream: "IMAGE_GPU:transformed_input_video" + node_options: { + [type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] { + flip_vertically: true + } + } +} + +# Crops the image from the center to the size WIDTHxHEIGHT. +node: { + calculator: "ImageCroppingCalculator" + input_stream: "IMAGE_GPU:transformed_input_video" + output_stream: "IMAGE_GPU:throttled_input_video_3x4" + input_stream: "WIDTH:input_width" + input_stream: "HEIGHT:input_height" + node_options: { + [type.googleapis.com/mediapipe.ImageCroppingCalculatorOptions] { + border_mode: BORDER_REPLICATE + } + } +} + +node { + calculator: "ObjectronGpuSubgraph" + input_stream: "IMAGE_GPU:throttled_input_video_3x4" + input_side_packet: "LABELS_CSV:allowed_labels" + input_side_packet: "MAX_NUM_OBJECTS:max_num_objects" + output_stream: "FRAME_ANNOTATION:lifted_objects" +} + +# The rendering nodes: +# We are rendering two meshes: 1) a 3D bounding box, which we overlay directly +# on the texture, and 2) a virtual object, which we use as an occlusion mask. +# These models are designed using different tools, so we supply a transformation +# to bring both of them to the Objectron's coordinate system. + +# Creates a model matrices for the tracked object given the lifted 3D points. +# This calculator does two things: 1) Estimates object's pose (orientation, +# translation, and scale) from the 3D vertices, and +# 2) bring the object from the objectron's coordinate system to the renderer +# (OpenGL) coordinate system. Since the final goal is to render a mesh file on +# top of the object, we also supply a transformation to bring the mesh to the +# objectron's coordinate system, and rescale mesh to the unit size. +node { + calculator: "AnnotationsToModelMatricesCalculator" + input_stream: "ANNOTATIONS:lifted_objects" + output_stream: "MODEL_MATRICES:model_matrices" + node_options: { + [type.googleapis.com/mediapipe.AnnotationsToModelMatricesCalculatorOptions] { + # Re-scale the CAD model to the size of a unit box + model_scale: [0.04, 0.04, 0.04] + # Bring the box CAD model to objectron's coordinate system. This + # is equivalent of -pi/2 rotation along the y-axis (right-hand rule): + # Eigen::AngleAxisf(-M_PI / 2., Eigen::Vector3f::UnitY()) + model_transformation: [0.0, 0.0, -1.0, 0.0] + model_transformation: [0.0, 1.0, 0.0, 0.0] + model_transformation: [1.0, 0.0, 0.0, 0.0] + model_transformation: [0.0, 0.0, 0.0, 1.0] + } + } +} + +# Compute the model matrices for the CAD model of the virtual object, to be used +# as an occlusion mask. The model will be rendered at the exact same location as +# the bounding box. +node { + calculator: "AnnotationsToModelMatricesCalculator" + input_stream: "ANNOTATIONS:lifted_objects" + input_side_packet: "MODEL_SCALE:model_scale" + input_side_packet: "MODEL_TRANSFORMATION:model_transformation" + output_stream: "MODEL_MATRICES:mask_model_matrices" +} + +# Render everything together. First we render the 3D bounding box animation, +# then we render the occlusion mask. +node: { + calculator: "GlAnimationOverlayCalculator" + input_stream: "VIDEO:throttled_input_video_3x4" + input_stream: "MODEL_MATRICES:model_matrices" + input_stream: "MASK_MODEL_MATRICES:mask_model_matrices" + output_stream: "output_video_raw" + input_side_packet: "TEXTURE:box_texture" + input_side_packet: "ANIMATION_ASSET:box_asset_name" + input_side_packet: "MASK_TEXTURE:obj_texture" + input_side_packet: "MASK_ASSET:obj_asset_name" + node_options: { + [type.googleapis.com/mediapipe.GlAnimationOverlayCalculatorOptions] { + aspect_ratio: 0.75 + vertical_fov_degrees: 70. + animation_speed_fps: 25 + } + } +} + +# Flip vertically because the output image is aligned from top-left to bottom-right. +node: { + calculator: "GlScalerCalculator" + input_stream: "VIDEO:output_video_raw" + output_stream: "VIDEO:output_video" + node_options: { + [type.googleapis.com/mediapipe.GlScalerCalculatorOptions] { + flip_vertical: true + } + } +} diff --git a/Assets/MediaPipe/Examples/Graphs/ObjectDetection3d/Resources/object_detection_3d_android.txt.meta b/Assets/MediaPipe/Examples/Graphs/ObjectDetection3d/Resources/object_detection_3d_android.txt.meta new file mode 100644 index 000000000..8d3777c57 --- /dev/null +++ b/Assets/MediaPipe/Examples/Graphs/ObjectDetection3d/Resources/object_detection_3d_android.txt.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: ef2e1f6b03bf3077d8e6ff3378c95501 +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/MediaPipe/Examples/Graphs/ObjectDetection3d/Resources/object_detection_3d_gpu.txt b/Assets/MediaPipe/Examples/Graphs/ObjectDetection3d/Resources/object_detection_3d_gpu.txt new file mode 100644 index 000000000..072d6d2a6 --- /dev/null +++ b/Assets/MediaPipe/Examples/Graphs/ObjectDetection3d/Resources/object_detection_3d_gpu.txt @@ -0,0 +1,180 @@ +# Copyright 2019 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copied from mediapipe/graphs/object_detection_3d/object_occlusion_tracking.pbtxt +# +# CHANGES: +# - `input_video` is ImageFrame (ImageFrameToGpuBufferCalculator converts it into GpuBuffer) +# - Add ImageTransformationCalculator and flip the input +# - Add GlScalerCalculator and flip the output +# - `output_video` is ImageFrame + +# MediaPipe graph that performs box tracking with TensorFlow Lite on GPU. + +# Images coming into and out of the graph. +input_stream: "input_video" +input_stream: "WIDTH:input_width" +input_stream: "HEIGHT:input_height" +input_side_packet: "LABELS_CSV:allowed_labels" +input_side_packet: "MODEL_SCALE:model_scale" +input_side_packet: "MODEL_TRANSFORMATION:model_transformation" +input_side_packet: "TEXTURE:box_texture" +input_side_packet: "MAX_NUM_OBJECTS:max_num_objects" +input_side_packet: "ANIMATION_ASSET:box_asset_name" +input_side_packet: "MASK_TEXTURE:obj_texture" +input_side_packet: "MASK_ASSET:obj_asset_name" +output_stream: "output_video" + +# Throttles the images flowing downstream for flow control. It passes through +# the very first incoming image unaltered, and waits for downstream nodes +# (calculators and subgraphs) in the graph to finish their tasks before it +# passes through another image. All images that come in while waiting are +# dropped, limiting the number of in-flight images in most part of the graph to +# 1. This prevents the downstream nodes from queuing up incoming images and data +# excessively, which leads to increased latency and memory usage, unwanted in +# real-time mobile applications. It also eliminates unnecessarily computation, +# e.g., the output produced by a node may get dropped downstream if the +# subsequent nodes are still busy processing previous inputs. +node { + calculator: "FlowLimiterCalculator" + input_stream: "input_video" + input_stream: "FINISHED:output_video" + input_stream_info: { + tag_index: "FINISHED" + back_edge: true + } + output_stream: "throttled_input_video" +} + +node: { + calculator: "ImageFrameToGpuBufferCalculator" + input_stream: "throttled_input_video" + output_stream: "throttled_input_video_gpu" +} + +# Flip vertically because the input image is aligned from bottom-left to top-right. +node: { + calculator: "ImageTransformationCalculator" + input_stream: "IMAGE_GPU:throttled_input_video_gpu" + output_stream: "IMAGE_GPU:transformed_input_video" + node_options: { + [type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] { + flip_vertically: true + } + } +} + +# Crops the image from the center to the size WIDTHxHEIGHT. +node: { + calculator: "ImageCroppingCalculator" + input_stream: "IMAGE_GPU:transformed_input_video" + output_stream: "IMAGE_GPU:throttled_input_video_3x4" + input_stream: "WIDTH:input_width" + input_stream: "HEIGHT:input_height" + node_options: { + [type.googleapis.com/mediapipe.ImageCroppingCalculatorOptions] { + border_mode: BORDER_REPLICATE + } + } +} + +node { + calculator: "ObjectronGpuSubgraph" + input_stream: "IMAGE_GPU:throttled_input_video_3x4" + input_side_packet: "LABELS_CSV:allowed_labels" + input_side_packet: "MAX_NUM_OBJECTS:max_num_objects" + output_stream: "FRAME_ANNOTATION:lifted_objects" +} + +# The rendering nodes: +# We are rendering two meshes: 1) a 3D bounding box, which we overlay directly +# on the texture, and 2) a virtual object, which we use as an occlusion mask. +# These models are designed using different tools, so we supply a transformation +# to bring both of them to the Objectron's coordinate system. + +# Creates a model matrices for the tracked object given the lifted 3D points. +# This calculator does two things: 1) Estimates object's pose (orientation, +# translation, and scale) from the 3D vertices, and +# 2) bring the object from the objectron's coordinate system to the renderer +# (OpenGL) coordinate system. Since the final goal is to render a mesh file on +# top of the object, we also supply a transformation to bring the mesh to the +# objectron's coordinate system, and rescale mesh to the unit size. +node { + calculator: "AnnotationsToModelMatricesCalculator" + input_stream: "ANNOTATIONS:lifted_objects" + output_stream: "MODEL_MATRICES:model_matrices" + node_options: { + [type.googleapis.com/mediapipe.AnnotationsToModelMatricesCalculatorOptions] { + # Re-scale the CAD model to the size of a unit box + model_scale: [0.04, 0.04, 0.04] + # Bring the box CAD model to objectron's coordinate system. This + # is equivalent of -pi/2 rotation along the y-axis (right-hand rule): + # Eigen::AngleAxisf(-M_PI / 2., Eigen::Vector3f::UnitY()) + model_transformation: [0.0, 0.0, -1.0, 0.0] + model_transformation: [0.0, 1.0, 0.0, 0.0] + model_transformation: [1.0, 0.0, 0.0, 0.0] + model_transformation: [0.0, 0.0, 0.0, 1.0] + } + } +} + +# Compute the model matrices for the CAD model of the virtual object, to be used +# as an occlusion mask. The model will be rendered at the exact same location as +# the bounding box. +node { + calculator: "AnnotationsToModelMatricesCalculator" + input_stream: "ANNOTATIONS:lifted_objects" + input_side_packet: "MODEL_SCALE:model_scale" + input_side_packet: "MODEL_TRANSFORMATION:model_transformation" + output_stream: "MODEL_MATRICES:mask_model_matrices" +} + +# Render everything together. First we render the 3D bounding box animation, +# then we render the occlusion mask. +node: { + calculator: "GlAnimationOverlayCalculator" + input_stream: "VIDEO:throttled_input_video_3x4" + input_stream: "MODEL_MATRICES:model_matrices" + input_stream: "MASK_MODEL_MATRICES:mask_model_matrices" + output_stream: "output_video_raw" + input_side_packet: "TEXTURE:box_texture" + input_side_packet: "ANIMATION_ASSET:box_asset_name" + input_side_packet: "MASK_TEXTURE:obj_texture" + input_side_packet: "MASK_ASSET:obj_asset_name" + node_options: { + [type.googleapis.com/mediapipe.GlAnimationOverlayCalculatorOptions] { + aspect_ratio: 0.75 + vertical_fov_degrees: 70. + animation_speed_fps: 25 + } + } +} + +# Flip vertically because the output image is aligned from top-left to bottom-right. +node: { + calculator: "GlScalerCalculator" + input_stream: "VIDEO:output_video_raw" + output_stream: "VIDEO:output_video_gpu" + node_options: { + [type.googleapis.com/mediapipe.GlScalerCalculatorOptions] { + flip_vertical: true + } + } +} + +node: { + calculator: "GpuBufferToImageFrameCalculator" + input_stream: "output_video_gpu" + output_stream: "output_video" +} diff --git a/Assets/MediaPipe/Examples/Graphs/ObjectDetection3d/Resources/object_detection_3d_gpu.txt.meta b/Assets/MediaPipe/Examples/Graphs/ObjectDetection3d/Resources/object_detection_3d_gpu.txt.meta new file mode 100644 index 000000000..3f5285cd2 --- /dev/null +++ b/Assets/MediaPipe/Examples/Graphs/ObjectDetection3d/Resources/object_detection_3d_gpu.txt.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 339e94fa26caa7779bba22281ecb7cef +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/MediaPipe/Examples/Graphs/ObjectDetection3d/Scripts.meta b/Assets/MediaPipe/Examples/Graphs/ObjectDetection3d/Scripts.meta new file mode 100644 index 000000000..49a3b5ea2 --- /dev/null +++ b/Assets/MediaPipe/Examples/Graphs/ObjectDetection3d/Scripts.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 82ee1cbc30ccd70299fab2a6ecc80b00 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/MediaPipe/Examples/Graphs/ObjectDetection3d/Scripts/ObjectDetection3dGraph.cs b/Assets/MediaPipe/Examples/Graphs/ObjectDetection3d/Scripts/ObjectDetection3dGraph.cs new file mode 100644 index 000000000..bec6faac1 --- /dev/null +++ b/Assets/MediaPipe/Examples/Graphs/ObjectDetection3d/Scripts/ObjectDetection3dGraph.cs @@ -0,0 +1,187 @@ +using Mediapipe; +using UnityEngine; + +public class ObjectDetection3dGraph : OfficialDemoGraph { + [SerializeField] Category category; + [SerializeField] int maxNumObjects = 5; + + [SerializeField] TextAsset objTextureAsset = null; + [SerializeField] TextAsset boxTextureAsset = null; + + enum Category { + Camera, + Chair, + Cup, + Sneaker, + }; + + public override Status StartRun(Texture texture) { + stopwatch.Start(); + + sidePacket = new SidePacket(); + sidePacket.Emplace("allowed_labels", new StringPacket(GetAllowedLabels(category))); + sidePacket.Emplace("model_scale", new FloatArrayPacket(GetModelScale(category))); + sidePacket.Emplace("model_transformation", new FloatArrayPacket(GetModelTransformation(category))); + sidePacket.Emplace("box_texture", new ImageFramePacket(GetImageFrameFromImage(boxTextureAsset))); + sidePacket.Emplace("max_num_objects", new IntPacket(maxNumObjects)); + sidePacket.Emplace("box_asset_name", new StringPacket("box.obj.bytes")); + sidePacket.Emplace("obj_texture", new ImageFramePacket(GetImageFrameFromImage(objTextureAsset))); + sidePacket.Emplace("obj_asset_name", new StringPacket(GetObjAssetName(category))); + +#if UNITY_ANDROID && !UNITY_EDITOR + SetupOutputPacket(texture); + sidePacket.Emplace(destinationBufferName, outputPacket); + + return graph.StartRun(sidePacket); +#else + return StartRun(); +#endif + } + + public override Status PushInput(TextureFrame textureFrame) { + base.PushInput(textureFrame).AssertOk(); + + graph.AddPacketToInputStream("input_width", new IntPacket(textureFrame.width, currentTimestamp)).AssertOk(); + return graph.AddPacketToInputStream("input_height", new IntPacket(textureFrame.height, currentTimestamp)); + } + + ImageFrame GetImageFrameFromImage(TextAsset image) { + var tempTexture = new Texture2D(1, 1); + tempTexture.LoadImage(image.bytes); + + // ensure that the texture format is RGBA32 + var texture = new Texture2D(tempTexture.width, tempTexture.height, TextureFormat.RGBA32, false); + texture.SetPixels32(tempTexture.GetPixels32()); + // flip the image vertically to align pixels from top-left to bottom-right + FlipTexture2D(texture); + texture.Apply(); + + return new ImageFrame(ImageFormat.Format.SRGBA, texture.width, texture.height, 4 * texture.width, texture.GetRawTextureData()); + } + + void FlipTexture2D(Texture2D texture) { + var src = texture.GetPixels32(); + var dest = new Color32[src.Length]; + + for (var i = 0; i < texture.height; i++) { + var srcIdx = i * texture.width; + var destIdx = (texture.height - 1 - i) * texture.width; + System.Array.Copy(src, srcIdx, dest, destIdx, texture.width); + } + + texture.SetPixels32(dest); + } + + string GetAllowedLabels(Category category) { + switch (category) { + case Category.Camera: { + return "Camera"; + } + case Category.Chair: { + return "Chair"; + } + case Category.Cup: { + return "Coffee cup,Mug"; + } + default: { + return "Footwear"; + } + } + } + + float[] GetModelScale(Category category) { + switch (category) { + case Category.Camera: { + return new float[] { 250, 250, 250 }; + } + case Category.Chair: { + return new float[] { 0.1f, 0.05f, 0.1f }; + } + case Category.Cup: { + return new float[] { 500, 500, 500 }; + } + default: { + return new float[] { 0.25f, 0.25f, 0.12f }; + } + } + } + + float[] GetModelTransformation(Category category) { + switch (category) { + case Category.Camera: { + return new float[] { + 1.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 1.0f, 0.0f, + 0.0f, -1.0f, 0.0f, -0.0015f, + 0.0f, 0.0f, 0.0f, 1.0f, + }; + } + case Category.Chair: { + return new float[] { + 1.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 1.0f, 0.0f, -10.0f, + 0.0f, 0.0f, -1.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 1.0f, + }; + } + case Category.Cup: { + return new float[] { + 1.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 1.0f, -0.001f, + 0.0f, -1.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 1.0f, + }; + } + default: { + return new float[] { + 1.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 1.0f, 0.0f, + 0.0f, -1.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 1.0f, + }; + } + } + } + + string GetObjAssetName(Category category) { + switch (category) { + case Category.Camera: { + return "camera.obj.bytes"; + } + case Category.Chair: { + return "chair.obj.bytes"; + } + case Category.Cup: { + return "cup.obj.bytes"; + } + default: { + return "sneaker.obj.bytes"; + } + } + } + + string GetModelAssetName(Category category) { + switch (category) { + case Category.Camera: { + return "object_detection_3d_camera.bytes"; + } + case Category.Chair: { + return "object_detection_3d_chair.bytes"; + } + case Category.Cup: { + return "object_detection_3d_chair.bytes"; + } + default: { + return "object_detection_3d_sneakers.bytes"; + } + } + } + + protected override void PrepareDependentAssets() { + PrepareDependentAsset("object_detection_ssd_mobilenetv2_oidv4_fp16.bytes"); + PrepareDependentAsset("object_detection_oidv4_labelmap.txt"); + PrepareDependentAsset("box.obj.bytes"); + PrepareDependentAsset(GetObjAssetName(category)); + PrepareDependentAsset(GetModelAssetName(category), "object_detection_3d.bytes", true); + } +} diff --git a/Assets/MediaPipe/Examples/Graphs/ObjectDetection3d/Scripts/ObjectDetection3dGraph.cs.meta b/Assets/MediaPipe/Examples/Graphs/ObjectDetection3d/Scripts/ObjectDetection3dGraph.cs.meta new file mode 100644 index 000000000..555e92114 --- /dev/null +++ b/Assets/MediaPipe/Examples/Graphs/ObjectDetection3d/Scripts/ObjectDetection3dGraph.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: a1ced5119e2040cbc9cf2235b2cca548 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/MediaPipe/Examples/Scenes/DesktopDemo.unity b/Assets/MediaPipe/Examples/Scenes/DesktopDemo.unity index 97e825bf2..6cf4abe10 100644 --- a/Assets/MediaPipe/Examples/Scenes/DesktopDemo.unity +++ b/Assets/MediaPipe/Examples/Scenes/DesktopDemo.unity @@ -1238,6 +1238,8 @@ MonoBehaviour: type: 3} objectDetectionGraph: {fileID: 8958252977655340068, guid: d5971d2eb607f463cb377c8460592f82, type: 3} + objectDetection3dGraph: {fileID: 8245872100598249146, guid: e062f09bfc63ada65a687ade46bef7ff, + type: 3} boxTrackingGraph: {fileID: 6989980473500709381, guid: 11c74e9e95a2b0ef3b180784b721735e, type: 3} instantMotionTrackingGraph: {fileID: 4459170324269958257, guid: dc998ba0475a9049eadb1dedd8bbf7dd, diff --git a/Assets/MediaPipe/Examples/Scripts/DemoGraph.cs b/Assets/MediaPipe/Examples/Scripts/DemoGraph.cs index 504c64e90..9a3fe3f6d 100644 --- a/Assets/MediaPipe/Examples/Scripts/DemoGraph.cs +++ b/Assets/MediaPipe/Examples/Scripts/DemoGraph.cs @@ -188,11 +188,11 @@ protected Timestamp GetCurrentTimestamp() { protected virtual void PrepareDependentAssets() {} - protected void PrepareDependentAsset(string assetName, string uniqueKey) { - resourceManager.GetComponent().PrepareAsset(assetName, uniqueKey); + protected void PrepareDependentAsset(string assetName, string uniqueKey, bool overwrite = false) { + resourceManager.GetComponent().PrepareAsset(assetName, uniqueKey, overwrite); } - protected void PrepareDependentAsset(string assetName) { - PrepareDependentAsset(assetName, assetName); + protected void PrepareDependentAsset(string assetName, bool overwrite = false) { + PrepareDependentAsset(assetName, assetName, overwrite); } } diff --git a/Assets/MediaPipe/Examples/Scripts/GraphSelectorController.cs b/Assets/MediaPipe/Examples/Scripts/GraphSelectorController.cs index 9bedc1777..835f6e134 100644 --- a/Assets/MediaPipe/Examples/Scripts/GraphSelectorController.cs +++ b/Assets/MediaPipe/Examples/Scripts/GraphSelectorController.cs @@ -13,6 +13,7 @@ public class GraphSelectorController : MonoBehaviour { [SerializeField] GameObject holisticGraph = null; [SerializeField] GameObject hairSegmentationGraph = null; [SerializeField] GameObject objectDetectionGraph = null; + [SerializeField] GameObject objectDetection3dGraph = null; [SerializeField] GameObject boxTrackingGraph = null; [SerializeField] GameObject instantMotionTrackingGraph = null; [SerializeField] GameObject officialDemoGraph = null; @@ -42,6 +43,7 @@ void InitializeOptions() { AddGraph("Hair Segmentation", hairSegmentationGraph); #endif AddGraph("Object Detection", objectDetectionGraph); + AddGraph("Object Detection 3d", objectDetection3dGraph); AddGraph("Box Tracking", boxTrackingGraph); AddGraph("Instant Motion Tracking", instantMotionTrackingGraph); AddGraph("Official Demo", officialDemoGraph); diff --git a/Assets/MediaPipe/Examples/Scripts/ResourceManager/AssetBundleLoader.cs b/Assets/MediaPipe/Examples/Scripts/ResourceManager/AssetBundleLoader.cs index 3fffdee37..5f3b887a0 100644 --- a/Assets/MediaPipe/Examples/Scripts/ResourceManager/AssetBundleLoader.cs +++ b/Assets/MediaPipe/Examples/Scripts/ResourceManager/AssetBundleLoader.cs @@ -9,13 +9,11 @@ void OnDestroy() { ((AssetBundleManager)resourceManager).ClearAllCacheFiles(); } - public override void PrepareAsset(string name, string uniqueKey) { - if (!resourceManager.IsPrepared(uniqueKey)) { - resourceManager.PrepareAsset(name, uniqueKey); - } + public override void PrepareAsset(string name, string uniqueKey, bool overwrite = false) { + resourceManager.PrepareAsset(name, uniqueKey, overwrite); } - public override void PrepareAsset(string name) { - PrepareAsset(name, name); + public override void PrepareAsset(string name, bool overwrite = false) { + PrepareAsset(name, name, overwrite); } } diff --git a/Assets/MediaPipe/Examples/Scripts/ResourceManager/AssetLoader.cs b/Assets/MediaPipe/Examples/Scripts/ResourceManager/AssetLoader.cs index d0c5b1e74..f3ce0a8f6 100644 --- a/Assets/MediaPipe/Examples/Scripts/ResourceManager/AssetLoader.cs +++ b/Assets/MediaPipe/Examples/Scripts/ResourceManager/AssetLoader.cs @@ -4,7 +4,7 @@ public abstract class AssetLoader : MonoBehaviour { protected ResourceManager resourceManager; - public abstract void PrepareAsset(string name, string uniqueKey); + public abstract void PrepareAsset(string name, string uniqueKey, bool overwrite = false); - public abstract void PrepareAsset(string name); + public abstract void PrepareAsset(string name, bool overwrite = false); } diff --git a/Assets/MediaPipe/Examples/Scripts/ResourceManager/LocalAssetLoader.cs b/Assets/MediaPipe/Examples/Scripts/ResourceManager/LocalAssetLoader.cs index 398b53d28..7863e05c0 100644 --- a/Assets/MediaPipe/Examples/Scripts/ResourceManager/LocalAssetLoader.cs +++ b/Assets/MediaPipe/Examples/Scripts/ResourceManager/LocalAssetLoader.cs @@ -5,11 +5,11 @@ void Start() { resourceManager = new Mediapipe.LocalAssetManager(); } - public override void PrepareAsset(string name, string uniqueKey) { - resourceManager.PrepareAsset(name, uniqueKey); + public override void PrepareAsset(string name, string uniqueKey, bool overwrite = false) { + resourceManager.PrepareAsset(name, uniqueKey, overwrite); } - public override void PrepareAsset(string name) { - resourceManager.PrepareAsset(name, name); + public override void PrepareAsset(string name, bool overwrite = false) { + resourceManager.PrepareAsset(name, name, overwrite); } } diff --git a/Assets/MediaPipe/SDK/Resources/box.obj.bytes.meta b/Assets/MediaPipe/SDK/Resources/box.obj.bytes.meta new file mode 100644 index 000000000..64b164bd1 --- /dev/null +++ b/Assets/MediaPipe/SDK/Resources/box.obj.bytes.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 1c907be3d670982b1a52e5f96a30d195 +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: mediapipe + assetBundleVariant: diff --git a/Assets/MediaPipe/SDK/Resources/camera.obj.bytes.meta b/Assets/MediaPipe/SDK/Resources/camera.obj.bytes.meta new file mode 100644 index 000000000..58f846e65 --- /dev/null +++ b/Assets/MediaPipe/SDK/Resources/camera.obj.bytes.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 540f26fdba63a25228c673b4176568af +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: mediapipe + assetBundleVariant: diff --git a/Assets/MediaPipe/SDK/Resources/camera_texture.bytes.meta b/Assets/MediaPipe/SDK/Resources/camera_texture.bytes.meta new file mode 100644 index 000000000..8593faff4 --- /dev/null +++ b/Assets/MediaPipe/SDK/Resources/camera_texture.bytes.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: cb17ea23c2685464a91a6e6b96c5b1e7 +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/MediaPipe/SDK/Resources/chair.obj.bytes.meta b/Assets/MediaPipe/SDK/Resources/chair.obj.bytes.meta new file mode 100644 index 000000000..ed7905084 --- /dev/null +++ b/Assets/MediaPipe/SDK/Resources/chair.obj.bytes.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 781e97f32a4776555a1ea06d9d716527 +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: mediapipe + assetBundleVariant: diff --git a/Assets/MediaPipe/SDK/Resources/chair_texture.bytes.meta b/Assets/MediaPipe/SDK/Resources/chair_texture.bytes.meta new file mode 100644 index 000000000..13ea14d78 --- /dev/null +++ b/Assets/MediaPipe/SDK/Resources/chair_texture.bytes.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: dc5b3d281530f7ce2b6dffd6bcce5dfa +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/MediaPipe/SDK/Resources/classic_colors.bytes.meta b/Assets/MediaPipe/SDK/Resources/classic_colors.bytes.meta new file mode 100644 index 000000000..4fb6017c7 --- /dev/null +++ b/Assets/MediaPipe/SDK/Resources/classic_colors.bytes.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 12957b0bde1a6c4a18fb9173ddebaa73 +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/MediaPipe/SDK/Resources/cup.obj.bytes.meta b/Assets/MediaPipe/SDK/Resources/cup.obj.bytes.meta new file mode 100644 index 000000000..26fbe3fdd --- /dev/null +++ b/Assets/MediaPipe/SDK/Resources/cup.obj.bytes.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 8579edd37c0968ca1a1df9659f1d1a43 +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: mediapipe + assetBundleVariant: diff --git a/Assets/MediaPipe/SDK/Resources/cup_texture.bytes.meta b/Assets/MediaPipe/SDK/Resources/cup_texture.bytes.meta new file mode 100644 index 000000000..107e7e911 --- /dev/null +++ b/Assets/MediaPipe/SDK/Resources/cup_texture.bytes.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 264ec0dfca3549566a49e2dc32aba37f +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/MediaPipe/SDK/Resources/object_detection_3d_camera.bytes.meta b/Assets/MediaPipe/SDK/Resources/object_detection_3d_camera.bytes.meta new file mode 100644 index 000000000..99bc31362 --- /dev/null +++ b/Assets/MediaPipe/SDK/Resources/object_detection_3d_camera.bytes.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 25f7f8d6c78414aa6b47573191f3cb13 +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: mediapipe + assetBundleVariant: diff --git a/Assets/MediaPipe/SDK/Resources/object_detection_3d_chair.bytes.meta b/Assets/MediaPipe/SDK/Resources/object_detection_3d_chair.bytes.meta new file mode 100644 index 000000000..6383aeace --- /dev/null +++ b/Assets/MediaPipe/SDK/Resources/object_detection_3d_chair.bytes.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 393afd72cbd0d7781ab071f3d6fb38fe +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: mediapipe + assetBundleVariant: diff --git a/Assets/MediaPipe/SDK/Resources/object_detection_3d_chair_1stage.bytes.meta b/Assets/MediaPipe/SDK/Resources/object_detection_3d_chair_1stage.bytes.meta new file mode 100644 index 000000000..43a3cce4c --- /dev/null +++ b/Assets/MediaPipe/SDK/Resources/object_detection_3d_chair_1stage.bytes.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 53a5a9537d1a5ea59a5ee2dd6f154af2 +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: mediapipe + assetBundleVariant: diff --git a/Assets/MediaPipe/SDK/Resources/object_detection_3d_cup.bytes.meta b/Assets/MediaPipe/SDK/Resources/object_detection_3d_cup.bytes.meta new file mode 100644 index 000000000..325b87856 --- /dev/null +++ b/Assets/MediaPipe/SDK/Resources/object_detection_3d_cup.bytes.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 5cde808ca06049fbcbdbf351c0579181 +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: mediapipe + assetBundleVariant: diff --git a/Assets/MediaPipe/SDK/Resources/object_detection_3d_sneakers.bytes.meta b/Assets/MediaPipe/SDK/Resources/object_detection_3d_sneakers.bytes.meta new file mode 100644 index 000000000..7af5ce69c --- /dev/null +++ b/Assets/MediaPipe/SDK/Resources/object_detection_3d_sneakers.bytes.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: f05d0f79da53b48858ba17062530313b +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: mediapipe + assetBundleVariant: diff --git a/Assets/MediaPipe/SDK/Resources/object_detection_3d_sneakers_1stage.bytes.meta b/Assets/MediaPipe/SDK/Resources/object_detection_3d_sneakers_1stage.bytes.meta new file mode 100644 index 000000000..9ebfbdf50 --- /dev/null +++ b/Assets/MediaPipe/SDK/Resources/object_detection_3d_sneakers_1stage.bytes.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 290ea52a92f4d637289ebae96906985b +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: mediapipe + assetBundleVariant: diff --git a/Assets/MediaPipe/SDK/Resources/object_detection_oidv4_labelmap.txt.meta b/Assets/MediaPipe/SDK/Resources/object_detection_oidv4_labelmap.txt.meta new file mode 100644 index 000000000..f68f0d843 --- /dev/null +++ b/Assets/MediaPipe/SDK/Resources/object_detection_oidv4_labelmap.txt.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 5c852a0cff35fa9d1b6598816e80af6e +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: mediapipe + assetBundleVariant: diff --git a/Assets/MediaPipe/SDK/Resources/object_detection_ssd_mobilenetv2_oidv4_fp16.bytes.meta b/Assets/MediaPipe/SDK/Resources/object_detection_ssd_mobilenetv2_oidv4_fp16.bytes.meta new file mode 100644 index 000000000..5f5a3ff52 --- /dev/null +++ b/Assets/MediaPipe/SDK/Resources/object_detection_ssd_mobilenetv2_oidv4_fp16.bytes.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: c800c44315d9fd7958d61b5cda0fa33f +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: mediapipe + assetBundleVariant: diff --git a/Assets/MediaPipe/SDK/Resources/sneaker.obj.bytes.meta b/Assets/MediaPipe/SDK/Resources/sneaker.obj.bytes.meta new file mode 100644 index 000000000..665955ae6 --- /dev/null +++ b/Assets/MediaPipe/SDK/Resources/sneaker.obj.bytes.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 626c627a682f87cc98179e281e64276a +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: mediapipe + assetBundleVariant: diff --git a/Assets/MediaPipe/SDK/Resources/sneaker_texture.bytes.meta b/Assets/MediaPipe/SDK/Resources/sneaker_texture.bytes.meta new file mode 100644 index 000000000..68344da35 --- /dev/null +++ b/Assets/MediaPipe/SDK/Resources/sneaker_texture.bytes.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: ff3b4f7354b931c37b0cfb5ff65dec1c +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/MediaPipe/SDK/Scripts/Util/AssetBundleManager.cs b/Assets/MediaPipe/SDK/Scripts/Util/AssetBundleManager.cs index cc202bab7..b86a664c9 100644 --- a/Assets/MediaPipe/SDK/Scripts/Util/AssetBundleManager.cs +++ b/Assets/MediaPipe/SDK/Scripts/Util/AssetBundleManager.cs @@ -1,6 +1,5 @@ using System; using System.IO; -using System.Linq; using System.Threading.Tasks; using UnityEngine; @@ -89,7 +88,12 @@ public override void PrepareAsset(string name, string uniqueKey, bool overwrite if (assetBundle == null) { LoadAssetBundle(); } + var asset = assetBundle.LoadAsset(name); + if (asset == null) { + throw new IOException($"Failed to load {name} from {assetBundle.name}"); + } + WriteCacheFile(asset, uniqueKey, overwrite); } @@ -108,7 +112,7 @@ public override async Task PrepareAssetAsync(string name, string uniqueKey, bool [AOT.MonoPInvokeCallback(typeof(CacheFilePathResolver))] static string CacheFileFromAsset(string assetPath) { - var assetName = GetAssetName(assetPath); + var assetName = GetAssetNameFromPath(assetPath); var cachePath = GetCacheFilePathFor(assetName); if (File.Exists(cachePath)) { @@ -136,13 +140,6 @@ protected static bool ReadFile(string path, IntPtr dst) { } } - static string GetAssetName(string assetPath) { - var assetName = Path.GetFileNameWithoutExtension(assetPath); - var extension = Path.GetExtension(assetPath); - - return extension == ".tflite" ? $"{assetName}.bytes" : $"{assetName}{extension}"; - } - static string GetCacheFilePathFor(string assetName) { return Path.Combine(_CacheRootPath, assetName); } @@ -151,7 +148,8 @@ void WriteCacheFile(TextAsset asset, string uniqueKey, bool overwrite) { var cachePath = GetCacheFilePathFor(uniqueKey); if (!overwrite && File.Exists(cachePath)) { - throw new IOException($"{cachePath} exists"); + Debug.Log($"{cachePath} already exists"); + return; } if (!Directory.Exists(CacheRootPath)) { @@ -163,24 +161,23 @@ void WriteCacheFile(TextAsset asset, string uniqueKey, bool overwrite) { } async Task WriteCacheFileAsync(TextAsset asset, string uniqueKey, bool overwrite) { + var cachePath = GetCacheFilePathFor(uniqueKey); + + if (!overwrite && File.Exists(cachePath)) { + Debug.Log($"{cachePath} already exists"); + return; + } + if (!Directory.Exists(CacheRootPath)) { Directory.CreateDirectory(CacheRootPath); } - var cachePath = GetCacheFilePathFor(uniqueKey); var bytes = asset.bytes; + var mode = overwrite ? FileMode.Create : FileMode.CreateNew; - FileStream sourceStream = null; - - try { - var mode = overwrite ? FileMode.Create : FileMode.CreateNew; - sourceStream = new FileStream(cachePath, mode, FileAccess.Write, FileShare.None, bufferSize: 4096, useAsync: true); + using (var sourceStream = new FileStream(cachePath, mode, FileAccess.Write, FileShare.None, bufferSize: 4096, useAsync: true)) { await sourceStream.WriteAsync(bytes, 0, bytes.Length); Debug.Log($"{asset.name} is saved to {cachePath} (length={bytes.Length})"); - } finally { - if (sourceStream != null) { - sourceStream.Close(); - } } } } diff --git a/Assets/MediaPipe/SDK/Scripts/Util/LocalAssetManager.cs b/Assets/MediaPipe/SDK/Scripts/Util/LocalAssetManager.cs index fac3b41c3..a65f4bdf0 100644 --- a/Assets/MediaPipe/SDK/Scripts/Util/LocalAssetManager.cs +++ b/Assets/MediaPipe/SDK/Scripts/Util/LocalAssetManager.cs @@ -34,8 +34,20 @@ public override void PrepareAsset(string name, string uniqueKey, bool overwrite var destFilePath = GetCacheFilePathFor(uniqueKey); - if (sourceFilePath != destFilePath) { - File.Copy(sourceFilePath, destFilePath, overwrite); + if (sourceFilePath == destFilePath) { + return; + } + + if (!overwrite && File.Exists(destFilePath)) { + Debug.Log($"{destFilePath} already exists"); + return; + } + + using (var sourceStream = File.OpenRead(sourceFilePath)) { + using (var destStream = File.Open(destFilePath, overwrite ? FileMode.Create : FileMode.CreateNew, FileAccess.Write)) { + sourceStream.CopyTo(destStream); + Debug.Log($"{sourceFilePath} is copied to {destFilePath}"); + } } } @@ -52,27 +64,16 @@ public override async Task PrepareAssetAsync(string name, string uniqueKey, bool return; } - FileStream sourceStream = null; - FileStream destStream = null; - - try { - sourceStream = File.OpenRead(sourceFilePath); - destStream = File.Open(destFilePath, overwrite ? FileMode.Create : FileMode.CreateNew); - - await sourceStream.CopyToAsync(destStream); - } finally { - if (sourceStream != null) { - sourceStream.Close(); - } - if (destStream != null) { - destStream.Close(); + using (var sourceStream = File.OpenRead(sourceFilePath)) { + using (var destStream = File.Open(destFilePath, overwrite ? FileMode.Create : FileMode.CreateNew)) { + await sourceStream.CopyToAsync(destStream); } } } [AOT.MonoPInvokeCallback(typeof(CacheFilePathResolver))] protected static string CacheFileFromAsset(string assetPath) { - var assetName = GetAssetName(assetPath); + var assetName = GetAssetNameFromPath(assetPath); var cachePath = GetCacheFilePathFor(assetName); if (File.Exists(cachePath)) { @@ -100,14 +101,6 @@ protected static bool ReadFile(string path, IntPtr dst) { } } - static string GetAssetName(string assetPath) { - // assume that each file name is unique if its extension is ignored. - var assetName = Path.GetFileNameWithoutExtension(assetPath); - var extension = Path.GetExtension(assetPath); - - return extension == ".tflite" ? $"{assetName}.bytes" : $"{assetName}{extension}"; - } - static string GetCacheFilePathFor(string assetName) { return Path.Combine(ResourceRootPath, assetName); } diff --git a/Assets/MediaPipe/SDK/Scripts/Util/ResourceManager.cs b/Assets/MediaPipe/SDK/Scripts/Util/ResourceManager.cs index 5e6d50e07..270c769e0 100644 --- a/Assets/MediaPipe/SDK/Scripts/Util/ResourceManager.cs +++ b/Assets/MediaPipe/SDK/Scripts/Util/ResourceManager.cs @@ -1,4 +1,5 @@ using System; +using System.IO; using System.Threading; using System.Threading.Tasks; @@ -81,5 +82,22 @@ public void PrepareAsset(string name, bool overwrite = true) { public Task PrepareAssetAsync(string name, bool overwrite = true) { return PrepareAssetAsync(name, name, overwrite); } + + protected static string GetAssetNameFromPath(string assetPath) { + var assetName = Path.GetFileNameWithoutExtension(assetPath); + var extension = Path.GetExtension(assetPath); + + switch (extension) { + case ".tflite": { + return $"{assetName}.bytes"; + } + case ".pbtxt": { + return $"{assetName}.txt"; + } + default: { + return $"{assetName}{extension}"; + } + } + } } } diff --git a/C/mediapipe_api/BUILD b/C/mediapipe_api/BUILD index 9071e5002..37007f0b2 100644 --- a/C/mediapipe_api/BUILD +++ b/C/mediapipe_api/BUILD @@ -91,6 +91,7 @@ cc_library( "@com_google_mediapipe//mediapipe/graphs/holistic_tracking:holistic_tracking_cpu_graph_deps", "@com_google_mediapipe//mediapipe/graphs/iris_tracking:iris_tracking_cpu_deps", "@com_google_mediapipe//mediapipe/graphs/object_detection:desktop_tflite_calculators", + "@com_google_mediapipe//mediapipe/graphs/object_detection_3d:desktop_cpu_calculators", "@com_google_mediapipe//mediapipe/graphs/pose_tracking:upper_body_pose_tracking_cpu_deps", "@com_google_mediapipe//mediapipe/graphs/tracking:desktop_calculators", ], @@ -109,6 +110,7 @@ cc_library( "@com_google_mediapipe//mediapipe/graphs/instant_motion_tracking:instant_motion_tracking_deps", "@com_google_mediapipe//mediapipe/graphs/iris_tracking:iris_tracking_gpu_deps", "@com_google_mediapipe//mediapipe/graphs/object_detection:mobile_calculators", + "@com_google_mediapipe//mediapipe/graphs/object_detection_3d:mobile_calculators", "@com_google_mediapipe//mediapipe/graphs/pose_tracking:upper_body_pose_tracking_gpu_deps", "@com_google_mediapipe//mediapipe/graphs/tracking:mobile_calculators", ], @@ -141,9 +143,18 @@ pkg_asset( name = "mediapipe_assets", srcs = [ "//mediapipe_api/graphs/instant_motion_tracking:assets", + "//mediapipe_api/graphs/object_detection_3d:assets", "@com_google_mediapipe//mediapipe/models:face_detection_front.tflite", "@com_google_mediapipe//mediapipe/models:face_landmark.tflite", "@com_google_mediapipe//mediapipe/models:hair_segmentation.tflite", + "@com_google_mediapipe//mediapipe/models:object_detection_3d_camera.tflite", + "@com_google_mediapipe//mediapipe/models:object_detection_3d_chair.tflite", + "@com_google_mediapipe//mediapipe/models:object_detection_3d_chair_1stage.tflite", + "@com_google_mediapipe//mediapipe/models:object_detection_3d_cup.tflite", + "@com_google_mediapipe//mediapipe/models:object_detection_3d_sneakers.tflite", + "@com_google_mediapipe//mediapipe/models:object_detection_3d_sneakers_1stage.tflite", + "@com_google_mediapipe//mediapipe/models:object_detection_oidv4_labelmap.pbtxt", + "@com_google_mediapipe//mediapipe/models:object_detection_ssd_mobilenetv2_oidv4_fp16.tflite", "@com_google_mediapipe//mediapipe/models:ssdlite_object_detection.tflite", "@com_google_mediapipe//mediapipe/models:ssdlite_object_detection_labelmap.txt", "@com_google_mediapipe//mediapipe/modules/hand_landmark:hand_landmark.tflite", diff --git a/C/mediapipe_api/graphs/object_detection_3d/BUILD b/C/mediapipe_api/graphs/object_detection_3d/BUILD new file mode 100644 index 000000000..27f8513aa --- /dev/null +++ b/C/mediapipe_api/graphs/object_detection_3d/BUILD @@ -0,0 +1,69 @@ +package( + default_visibility = ["//visibility:public"], +) + +load("//mediapipe_api:import_model.bzl", "copy_file") + +copy_file( + name = "camera_texture", + src = "@com_google_mediapipe//mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/camera:texture.jpg", + out = "camera_texture.jpg", +) + +copy_file( + name = "chair_texture", + src = "@com_google_mediapipe//mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/chair:texture.jpg", + out = "chair_texture.jpg", +) + +copy_file( + name = "cup_texture", + src = "@com_google_mediapipe//mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/cup:texture.jpg", + out = "cup_texture.jpg", +) + +copy_file( + name = "sneaker_texture", + src = "@com_google_mediapipe//mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/sneaker:texture.jpg", + out = "sneaker_texture.jpg", +) + +copy_file( + name = "camera_mesh", + src = "@com_google_mediapipe//mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/camera:model.obj.uuu", + out = "camera.obj.uuu", +) + +copy_file( + name = "chair_mesh", + src = "@com_google_mediapipe//mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/chair:model.obj.uuu", + out = "chair.obj.uuu", +) + +copy_file( + name = "cup_mesh", + src = "@com_google_mediapipe//mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/cup:model.obj.uuu", + out = "cup.obj.uuu", +) + +copy_file( + name = "sneaker_mesh", + src = "@com_google_mediapipe//mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/sneaker:model.obj.uuu", + out = "sneaker.obj.uuu", +) + +filegroup( + name = "assets", + srcs = [ + ":sneaker_texture", + ":chair_texture", + ":cup_texture", + ":camera_texture", + ":sneaker_mesh", + ":chair_mesh", + ":cup_mesh", + ":camera_mesh", + "@com_google_mediapipe//mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets:box.obj.uuu", + "@com_google_mediapipe//mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets:classic_colors.png", + ], +) diff --git a/C/mediapipe_api/import_model.bzl b/C/mediapipe_api/import_model.bzl index 8467d463c..0cd32aee0 100644 --- a/C/mediapipe_api/import_model.bzl +++ b/C/mediapipe_api/import_model.bzl @@ -1,5 +1,13 @@ load("@rules_pkg//:pkg.bzl", "pkg_zip") +def copy_file(name, src, out): + native.genrule( + name = name, + srcs = [src], + outs = [out], + cmd = "cp $< $@" + ) + def pkg_asset(name, srcs = [], **kwargs): """Package MediaPipe assets This task renames asset files so that they can be added to an AssetBundle (e.g. x.tflte -> x.bytes) and zip them. @@ -22,8 +30,12 @@ def _normalize_exts_impl(ctx): output_files = [] for src in ctx.files.srcs: - if src.extension in ctx.attr.bytes_exts: - dest = ctx.actions.declare_file(src.path[:-1 * len(src.extension)] + "bytes") + ext = "bytes" if src.extension in ctx.attr.bytes_exts else ("txt" if src.extension in ctx.attr.txt_exts else src.extension) + + if ext == src.extension: + output_files.append(src) + else: + dest = ctx.actions.declare_file(src.path[:-1 * len(src.extension)] + ext) ctx.actions.run_shell( inputs = [src], outputs = [dest], @@ -32,8 +44,6 @@ def _normalize_exts_impl(ctx): progress_message = "Copying %s to %s...".format(src.path, dest.path), ) output_files.append(dest) - else: - output_files.append(src) return [ DefaultInfo(files = depset(output_files)), @@ -43,6 +53,7 @@ _normalize_exts = rule( implementation = _normalize_exts_impl, attrs = { "srcs": attr.label_list(allow_files = True), - "bytes_exts": attr.string_list(default = ["jpg", "tflite", "uuu"]), + "bytes_exts": attr.string_list(default = ["jpg", "png", "tflite", "uuu"]), + "txt_exts": attr.string_list(default = ["pbtxt"]), }, ) diff --git a/ProjectSettings/ProjectSettings.asset b/ProjectSettings/ProjectSettings.asset index c5d3b8833..4f6462f6b 100644 --- a/ProjectSettings/ProjectSettings.asset +++ b/ProjectSettings/ProjectSettings.asset @@ -264,7 +264,7 @@ PlayerSettings: banner: {fileID: 0} androidGamepadSupportLevel: 0 AndroidValidateAppBundleSize: 1 - AndroidAppBundleSizeToValidate: 200 + AndroidAppBundleSizeToValidate: 300 m_BuildTargetIcons: [] m_BuildTargetPlatformIcons: - m_BuildTarget: Android