feat(example): Objectron

.gitignore

@@ -21,6 +21,7 @@ bazel-*
 Assets/StreamingAssets*
 Assets/MediaPipe/SDK/Resources/*
 !Assets/MediaPipe/SDK/Resources/*.meta
+Assets/MediaPipe/SDK/Resources/object_detection_3d.bytes.meta
 Assets/MediaPipe/SDK/Plugins/opencv_world*.dll.meta
 Assets/MediaPipe/SDK/Plugins/Protobuf*
 Assets/MediaPipe/SDK/Scripts/Protobuf/**/*.cs

Assets/MediaPipe/Examples/Graphs/BoxTracking/Resources/box_tracking_android.txt

@@ -16,7 +16,7 @@
 #
 # CHANGES:
 #   - Add ImageTransformationCalculator and rotate the input
-#   - Add GlScalerCalculator and flips the output
+#   - Add GlScalerCalculator and flip the output
 
 # MediaPipe graph that performs object detection and tracking with TensorFlow
 # Lite on GPU.

Assets/MediaPipe/Examples/Graphs/InstantMotionTracking/Resources/instant_motion_tracking_gpu.txt

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Copied from mediapipe/graphs/instant_motion_tracking/instant_motion_tracking.pbtxt
+# Copied from mediapipe/graphs/object_detection_3d/object_occlusion_tracking.pbtxt
 #
 # CHANGES:
 #   - Remove GIF related inputs

Assets/MediaPipe/Examples/Graphs/ObjectDetection3d/Objects/ObjectDetection3d.prefab

@@ -0,0 +1,52 @@
+%YAML 1.1
+%TAG !u! tag:unity3d.com,2011:
+--- !u!1 &8245872100598249146
+GameObject:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  serializedVersion: 6
+  m_Component:
+  - component: {fileID: 8245872100598249144}
+  - component: {fileID: 6244794291530564790}
+  m_Layer: 0
+  m_Name: ObjectDetection3d
+  m_TagString: Untagged
+  m_Icon: {fileID: 0}
+  m_NavMeshLayer: 0
+  m_StaticEditorFlags: 0
+  m_IsActive: 1
+--- !u!4 &8245872100598249144
+Transform:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 8245872100598249146}
+  m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
+  m_LocalPosition: {x: 0, y: 0, z: -10}
+  m_LocalScale: {x: 1, y: 1, z: 1}
+  m_Children: []
+  m_Father: {fileID: 0}
+  m_RootOrder: 0
+  m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
+--- !u!114 &6244794291530564790
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 8245872100598249146}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: a1ced5119e2040cbc9cf2235b2cca548, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  gpuConfig: {fileID: 4900000, guid: 339e94fa26caa7779bba22281ecb7cef, type: 3}
+  cpuConfig: {fileID: 0}
+  androidConfig: {fileID: 4900000, guid: ef2e1f6b03bf3077d8e6ff3378c95501, type: 3}
+  category: 3
+  maxNumObjects: 5
+  objTextureAsset: {fileID: 4900000, guid: ff3b4f7354b931c37b0cfb5ff65dec1c, type: 3}
+  boxTextureAsset: {fileID: 4900000, guid: 12957b0bde1a6c4a18fb9173ddebaa73, type: 3}

Assets/MediaPipe/Examples/Graphs/ObjectDetection3d/Resources/object_detection_3d_android.txt

@@ -0,0 +1,166 @@
+# Copyright 2019 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Copied from mediapipe/graphs/object_detection_3d/object_occlusion_tracking.pbtxt
+#
+# CHANGES:
+#   - Add ImageTransformationCalculator and flip the input
+#   - Add GlScalerCalculator and flip the output
+
+# MediaPipe graph that performs box tracking with TensorFlow Lite on GPU.
+
+# Images coming into and out of the graph.
+input_stream: "input_video"
+input_stream: "WIDTH:input_width"
+input_stream: "HEIGHT:input_height"
+input_side_packet: "LABELS_CSV:allowed_labels"
+input_side_packet: "MODEL_SCALE:model_scale"
+input_side_packet: "MODEL_TRANSFORMATION:model_transformation"
+input_side_packet: "TEXTURE:box_texture"
+input_side_packet: "MAX_NUM_OBJECTS:max_num_objects"
+input_side_packet: "ANIMATION_ASSET:box_asset_name"
+input_side_packet: "MASK_TEXTURE:obj_texture"
+input_side_packet: "MASK_ASSET:obj_asset_name"
+output_stream: "output_video"
+
+# Throttles the images flowing downstream for flow control. It passes through
+# the very first incoming image unaltered, and waits for downstream nodes
+# (calculators and subgraphs) in the graph to finish their tasks before it
+# passes through another image. All images that come in while waiting are
+# dropped, limiting the number of in-flight images in most part of the graph to
+# 1. This prevents the downstream nodes from queuing up incoming images and data
+# excessively, which leads to increased latency and memory usage, unwanted in
+# real-time mobile applications. It also eliminates unnecessarily computation,
+# e.g., the output produced by a node may get dropped downstream if the
+# subsequent nodes are still busy processing previous inputs.
+node {
+  calculator: "FlowLimiterCalculator"
+  input_stream: "input_video"
+  input_stream: "FINISHED:output_video"
+  input_stream_info: {
+    tag_index: "FINISHED"
+    back_edge: true
+  }
+  output_stream: "throttled_input_video"
+}
+
+# Flip vertically because the input image is aligned from bottom-left to top-right.
+node: {
+  calculator: "ImageTransformationCalculator"
+  input_stream: "IMAGE_GPU:throttled_input_video"
+  output_stream: "IMAGE_GPU:transformed_input_video"
+  node_options: {
+    [type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] {
+      flip_vertically: true
+    }
+  }
+}
+
+# Crops the image from the center to the size WIDTHxHEIGHT.
+node: {
+  calculator: "ImageCroppingCalculator"
+  input_stream: "IMAGE_GPU:transformed_input_video"
+  output_stream: "IMAGE_GPU:throttled_input_video_3x4"
+  input_stream: "WIDTH:input_width"
+  input_stream: "HEIGHT:input_height"
+  node_options: {
+    [type.googleapis.com/mediapipe.ImageCroppingCalculatorOptions] {
+      border_mode: BORDER_REPLICATE
+    }
+  }
+}
+
+node {
+  calculator: "ObjectronGpuSubgraph"
+  input_stream: "IMAGE_GPU:throttled_input_video_3x4"
+  input_side_packet: "LABELS_CSV:allowed_labels"
+  input_side_packet: "MAX_NUM_OBJECTS:max_num_objects"
+  output_stream: "FRAME_ANNOTATION:lifted_objects"
+}
+
+# The rendering nodes:
+# We are rendering two meshes: 1) a 3D bounding box, which we overlay directly
+# on the texture, and 2) a virtual object, which we use as an occlusion mask.
+# These models are designed using different tools, so we supply a transformation
+# to bring both of them to the Objectron's coordinate system.
+
+# Creates a model matrices for the tracked object given the lifted 3D points.
+# This calculator does two things: 1) Estimates object's pose (orientation,
+# translation, and scale) from the 3D vertices, and
+# 2) bring the object from the objectron's coordinate system to the renderer
+# (OpenGL) coordinate system. Since the final goal is to render a mesh file on
+# top of the object, we also supply a transformation to bring the mesh to the
+# objectron's coordinate system, and rescale mesh to the unit size.
+node {
+  calculator: "AnnotationsToModelMatricesCalculator"
+  input_stream: "ANNOTATIONS:lifted_objects"
+  output_stream: "MODEL_MATRICES:model_matrices"
+  node_options: {
+    [type.googleapis.com/mediapipe.AnnotationsToModelMatricesCalculatorOptions] {
+      # Re-scale the CAD model to the size of a unit box
+      model_scale: [0.04, 0.04, 0.04]
+      # Bring the box CAD model to objectron's coordinate system. This
+      # is equivalent of -pi/2 rotation along the y-axis (right-hand rule):
+      # Eigen::AngleAxisf(-M_PI / 2., Eigen::Vector3f::UnitY())
+      model_transformation: [0.0,  0.0, -1.0,  0.0]
+      model_transformation: [0.0,  1.0,  0.0,  0.0]
+      model_transformation: [1.0,  0.0,  0.0,  0.0]
+      model_transformation: [0.0,  0.0,  0.0,  1.0]
+    }
+  }
+}
+
+# Compute the model matrices for the CAD model of the virtual object, to be used
+# as an occlusion mask. The model will be rendered at the exact same location as
+# the bounding box.
+node {
+  calculator: "AnnotationsToModelMatricesCalculator"
+  input_stream: "ANNOTATIONS:lifted_objects"
+  input_side_packet: "MODEL_SCALE:model_scale"
+  input_side_packet: "MODEL_TRANSFORMATION:model_transformation"
+  output_stream: "MODEL_MATRICES:mask_model_matrices"
+}
+
+# Render everything together. First we render the 3D bounding box animation,
+# then we render the occlusion mask.
+node: {
+  calculator: "GlAnimationOverlayCalculator"
+  input_stream: "VIDEO:throttled_input_video_3x4"
+  input_stream: "MODEL_MATRICES:model_matrices"
+  input_stream: "MASK_MODEL_MATRICES:mask_model_matrices"
+  output_stream: "output_video_raw"
+  input_side_packet: "TEXTURE:box_texture"
+  input_side_packet: "ANIMATION_ASSET:box_asset_name"
+  input_side_packet: "MASK_TEXTURE:obj_texture"
+  input_side_packet: "MASK_ASSET:obj_asset_name"
+  node_options: {
+    [type.googleapis.com/mediapipe.GlAnimationOverlayCalculatorOptions] {
+      aspect_ratio: 0.75
+      vertical_fov_degrees: 70.
+      animation_speed_fps: 25
+    }
+  }
+}
+
+# Flip vertically because the output image is aligned from top-left to bottom-right.
+node: {
+  calculator: "GlScalerCalculator"
+  input_stream: "VIDEO:output_video_raw"
+  output_stream: "VIDEO:output_video"
+  node_options: {
+    [type.googleapis.com/mediapipe.GlScalerCalculatorOptions] {
+      flip_vertical: true
+    }
+  }
+}