Skip to content

Commit

Permalink
refactor!: coordinate transform and other helper methods (#569)
Browse files Browse the repository at this point in the history 
* refactor: GetLocalPosition -> ImageToLocalPoint

* test ImageNormalizedToPoint

* revise ImageCoordinate I/Fs

* format comment

* refactor: GetLocalPosition -> CameraToPoint, RealWorldToPoint

* feat: GetApproximateQuaternion

* CameraToRealWorld

* fix rotation bugs

* typo
  • Loading branch information
@homuler
homuler committed May 16, 2022
1 parent dcf992d commit 527d3b1
Show file tree
Hide file tree
Showing 13 changed files with 935 additions and 283 deletions.
2 changes: 1 addition & 1 deletion Assets/MediaPipeUnity/Samples/Scenes/Instant Motion Tracking/Anchor3dAnnotation.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ public void Draw(Anchor3d? target, Quaternion rotation, Vector3 cameraPosition,
if (ActivateFor(target))
{
var anchor3d = (Anchor3d)target;
var anchor2dPosition = GetAnnotationLayer().GetLocalPosition(anchor3d, rotationAngle, isMirrored);
var anchor2dPosition = GetScreenRect().GetPoint(anchor3d, rotationAngle, isMirrored);
var anchor3dPosition = GetAnchorPositionInRay(anchor2dPosition, anchor3d.z * defaultDepth, cameraPosition);

_pointAnnotation.Draw(anchor2dPosition);
Expand Down
2 changes: 1 addition & 1 deletion ...ts/MediaPipeUnity/Samples/Scenes/Instant Motion Tracking/InstantMotionTrackingSolution.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ private void Update()
if (RectTransformUtility.ScreenPointToLocalPointInRectangle(rectTransform, Input.mousePosition, Camera.main, out var localPoint))
{
var isMirrored = ImageSourceProvider.ImageSource.isFrontFacing ^ ImageSourceProvider.ImageSource.isHorizontallyFlipped;
var normalizedPoint = rectTransform.GetNormalizedPosition(localPoint, graphRunner.rotation, isMirrored);
var normalizedPoint = rectTransform.rect.PointToImageNormalized(localPoint, graphRunner.rotation, isMirrored);
graphRunner.ResetAnchor(normalizedPoint.x, normalizedPoint.y);
_trackedAnchorDataAnnotationController.ResetAnchor();
}
Expand Down
2 changes: 1 addition & 1 deletion ...ages/com.github.homuler.mediapipe/Runtime/Scripts/Unity/Annotation/DetectionAnnotation.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ public void Draw(Detection target, float threshold = 0.0f)

// Assume that location data's format is always RelativeBoundingBox
// TODO: fix if there are cases where this assumption is not correct.
var rectVertices = GetAnnotationLayer().GetRectVertices(target.LocationData.RelativeBoundingBox, rotationAngle, isMirrored);
var rectVertices = GetScreenRect().GetRectVertices(target.LocationData.RelativeBoundingBox, rotationAngle, isMirrored);
_locationDataAnnotation.SetColor(GetColor(score, Mathf.Clamp(threshold, 0.0f, 1.0f)));
_locationDataAnnotation.Draw(rectVertices);

Expand Down
6 changes: 6 additions & 0 deletions ...s/com.github.homuler.mediapipe/Runtime/Scripts/Unity/Annotation/HierarchicalAnnotation.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ public interface IHierachicalAnnotation
IHierachicalAnnotation root { get; }
Transform transform { get; }
RectTransform GetAnnotationLayer();
UnityEngine.Rect GetScreenRect();
}

public abstract class HierarchicalAnnotation : MonoBehaviour, IHierachicalAnnotation
Expand All @@ -37,6 +38,11 @@ public RectTransform GetAnnotationLayer()
return root.transform.parent.gameObject.GetComponent<RectTransform>();
}

public UnityEngine.Rect GetScreenRect()
{
return GetAnnotationLayer().rect;
}

public bool isActive => gameObject.activeSelf;
public bool isActiveInHierarchy => gameObject.activeInHierarchy;

Expand Down
18 changes: 9 additions & 9 deletions ...m.github.homuler.mediapipe/Runtime/Scripts/Unity/Annotation/IrisLandmarkListAnnotation.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,13 +65,13 @@ public void Draw(IList<NormalizedLandmark> target, bool visualizeZ = false, int
{
_landmarkListAnnotation.Draw(target, visualizeZ);

var rectTransform = GetAnnotationLayer();
var center = rectTransform.GetLocalPosition(target[0], rotationAngle, isMirrored);
var rect = GetScreenRect();
var center = rect.GetPoint(target[0], rotationAngle, isMirrored);
if (!visualizeZ)
{
center.z = 0.0f;
}
var radius = CalculateRadius(rectTransform, target);
var radius = CalculateRadius(rect, target);
_circleAnnotation.Draw(center, radius, vertices);
}
}
Expand All @@ -81,17 +81,17 @@ public void Draw(NormalizedLandmarkList target, bool visualizeZ = false, int ver
Draw(target?.Landmark, visualizeZ, vertices);
}

private float CalculateRadius(RectTransform rectTransform, IList<NormalizedLandmark> target)
private float CalculateRadius(UnityEngine.Rect rect, IList<NormalizedLandmark> target)
{
var r1 = CalculateDistance(rectTransform, target[1], target[3]);
var r2 = CalculateDistance(rectTransform, target[2], target[4]);
var r1 = CalculateDistance(rect, target[1], target[3]);
var r2 = CalculateDistance(rect, target[2], target[4]);
return (r1 + r2) / 4;
}

private float CalculateDistance(RectTransform rectTransform, NormalizedLandmark a, NormalizedLandmark b)
private float CalculateDistance(UnityEngine.Rect rect, NormalizedLandmark a, NormalizedLandmark b)
{
var aPos = rectTransform.GetLocalPosition(a, rotationAngle, isMirrored);
var bPos = rectTransform.GetLocalPosition(b, rotationAngle, isMirrored);
var aPos = rect.GetPoint(a, rotationAngle, isMirrored);
var bPos = rect.GetPoint(b, rotationAngle, isMirrored);
aPos.z = 0.0f;
bPos.z = 0.0f;
return Vector3.Distance(aPos, bPos);
Expand Down
10 changes: 5 additions & 5 deletions Packages/com.github.homuler.mediapipe/Runtime/Scripts/Unity/Annotation/PointAnnotation.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ public void Draw(Landmark target, Vector3 scale, bool visualizeZ = true)
{
if (ActivateFor(target))
{
var position = GetAnnotationLayer().GetLocalPosition(target, scale, rotationAngle, isMirrored);
var position = GetScreenRect().GetPoint(target, scale, rotationAngle, isMirrored);
if (!visualizeZ)
{
position.z = 0.0f;
Expand All @@ -66,7 +66,7 @@ public void Draw(NormalizedLandmark target, bool visualizeZ = true)
{
if (ActivateFor(target))
{
var position = GetAnnotationLayer().GetLocalPosition(target, rotationAngle, isMirrored);
var position = GetScreenRect().GetPoint(target, rotationAngle, isMirrored);
if (!visualizeZ)
{
position.z = 0.0f;
Expand All @@ -79,7 +79,7 @@ public void Draw(NormalizedPoint2D target)
{
if (ActivateFor(target))
{
var position = GetAnnotationLayer().GetLocalPosition(target, rotationAngle, isMirrored);
var position = GetScreenRect().GetPoint(target, rotationAngle, isMirrored);
transform.localPosition = position;
}
}
Expand All @@ -88,7 +88,7 @@ public void Draw(Point3D target, Vector2 focalLength, Vector2 principalPoint, fl
{
if (ActivateFor(target))
{
var position = GetAnnotationLayer().GetLocalPosition(target, focalLength, principalPoint, zScale, rotationAngle, isMirrored);
var position = GetScreenRect().GetPoint(target, focalLength, principalPoint, zScale, rotationAngle, isMirrored);
if (!visualizeZ)
{
position.z = 0.0f;
Expand All @@ -113,7 +113,7 @@ public void Draw(mplt.RelativeKeypoint target, float threshold = 0.0f)
{
if (ActivateFor(target))
{
Draw(GetAnnotationLayer().GetLocalPosition(target, rotationAngle, isMirrored));
Draw(GetScreenRect().GetPoint(target, rotationAngle, isMirrored));
SetColor(GetColor(target.Score, threshold));
}
}
Expand Down
14 changes: 7 additions & 7 deletions ...ages/com.github.homuler.mediapipe/Runtime/Scripts/Unity/Annotation/RectangleAnnotation.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,36 +58,36 @@ public void Draw(Vector3[] positions)
_lineRenderer.SetPositions(positions ?? _EmptyPositions);
}

public void Draw(Rect target, Vector2 imageSize)
public void Draw(Rect target, Vector2Int imageSize)
{
if (ActivateFor(target))
{
Draw(GetAnnotationLayer().GetRectVertices(target, imageSize, rotationAngle, isMirrored));
Draw(GetScreenRect().GetRectVertices(target, imageSize, rotationAngle, isMirrored));
}
}

public void Draw(NormalizedRect target)
{
if (ActivateFor(target))
{
Draw(GetAnnotationLayer().GetRectVertices(target, rotationAngle, isMirrored));
Draw(GetScreenRect().GetRectVertices(target, rotationAngle, isMirrored));
}
}

public void Draw(LocationData target, Vector2 imageSize)
public void Draw(LocationData target, Vector2Int imageSize)
{
if (ActivateFor(target))
{
switch (target.Format)
{
case mplt.Format.BoundingBox:
{
Draw(GetAnnotationLayer().GetRectVertices(target.BoundingBox, imageSize, rotationAngle, isMirrored));
Draw(GetScreenRect().GetRectVertices(target.BoundingBox, imageSize, rotationAngle, isMirrored));
break;
}
case mplt.Format.RelativeBoundingBox:
{
Draw(GetAnnotationLayer().GetRectVertices(target.RelativeBoundingBox, rotationAngle, isMirrored));
Draw(GetScreenRect().GetRectVertices(target.RelativeBoundingBox, rotationAngle, isMirrored));
break;
}
case mplt.Format.Global:
Expand All @@ -108,7 +108,7 @@ public void Draw(LocationData target)
{
case mplt.Format.RelativeBoundingBox:
{
Draw(GetAnnotationLayer().GetRectVertices(target.RelativeBoundingBox, rotationAngle, isMirrored));
Draw(GetScreenRect().GetRectVertices(target.RelativeBoundingBox, rotationAngle, isMirrored));
break;
}
case mplt.Format.BoundingBox:
Expand Down
2 changes: 1 addition & 1 deletion .../com.github.homuler.mediapipe/Runtime/Scripts/Unity/Annotation/RectangleListAnnotation.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ public void SetLineWidth(float lineWidth)
ApplyLineWidth(_lineWidth);
}

public void Draw(IList<Rect> targets, Vector2 imageSize)
public void Draw(IList<Rect> targets, Vector2Int imageSize)
{
if (ActivateFor(targets))
{
Expand Down
35 changes: 10 additions & 25 deletions ...ages/com.github.homuler.mediapipe/Runtime/Scripts/Unity/Annotation/TransformAnnotation.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,34 +38,25 @@ public void SetArrowWidth(float arrowWidth)
public void Draw(Quaternion rotation, Vector3 scale, bool visualizeZ = true)
{
var q = Quaternion.Euler(0, 0, -(int)rotationAngle);
DrawArrow(_xArrow, q * rotation * Vector3.right, scale.x, visualizeZ);
DrawArrow(_yArrow, q * rotation * Vector3.up, scale.y, visualizeZ);
DrawArrow(_zArrow, q * rotation * Vector3.forward, scale.z, visualizeZ);
DrawArrow(_xArrow, scale.x * (q * rotation * Vector3.right), visualizeZ);
DrawArrow(_yArrow, scale.y * (q * rotation * Vector3.up), visualizeZ);
DrawArrow(_zArrow, scale.z * (q * rotation * Vector3.forward), visualizeZ);
}

public void Draw(ObjectAnnotation target, Vector3 position, float arrowLengthScale = 1.0f, bool visualizeZ = true)
{
origin = position;

var isInverted = CameraCoordinate.IsInverted(rotationAngle);
var (xScale, yScale) = isInverted ? (target.Scale[1], target.Scale[0]) : (target.Scale[0], target.Scale[1]);
var zScale = target.Scale[2];
// convert from right-handed to left-handed
var isXReversed = CameraCoordinate.IsXReversed(rotationAngle, isMirrored);
var isYReversed = CameraCoordinate.IsYReversed(rotationAngle, isMirrored);
var rotation = target.Rotation;
var xDir = GetDirection(rotation[0], rotation[3], rotation[6], isXReversed, isYReversed, isInverted);
var yDir = GetDirection(rotation[1], rotation[4], rotation[7], isXReversed, isYReversed, isInverted);
var zDir = GetDirection(rotation[2], rotation[5], rotation[8], isXReversed, isYReversed, isInverted);
DrawArrow(_xArrow, xDir, (isMirrored ? -1 : 1) * arrowLengthScale * xScale, visualizeZ);
DrawArrow(_yArrow, yDir, arrowLengthScale * yScale, visualizeZ);
DrawArrow(_zArrow, zDir, -arrowLengthScale * zScale, visualizeZ);
var (xDir, yDir, zDir) = CameraCoordinate.GetDirections(target, rotationAngle, isMirrored);
DrawArrow(_xArrow, arrowLengthScale * xDir, visualizeZ);
DrawArrow(_yArrow, arrowLengthScale * yDir, visualizeZ);
DrawArrow(_zArrow, arrowLengthScale * zDir, visualizeZ);
}

private void DrawArrow(Arrow arrow, Vector3 normalizedDirection, float scale, bool visualizeZ)
private void DrawArrow(Arrow arrow, Vector3 vec, bool visualizeZ)
{
var direction = Mathf.Sign(scale) * normalizedDirection;
var magnitude = Mathf.Abs(scale);
var magnitude = vec.magnitude;
var direction = vec.normalized;

if (!visualizeZ)
{
Expand All @@ -76,11 +67,5 @@ private void DrawArrow(Arrow arrow, Vector3 normalizedDirection, float scale, bo
arrow.direction = direction;
arrow.magnitude = magnitude;
}

private Vector3 GetDirection(float x, float y, float z, bool isXReversed, bool isYReversed, bool isInverted)
{
var dir = isInverted ? new Vector3(y, x, z) : new Vector3(x, y, z);
return Vector3.Scale(dir, new Vector3(isXReversed ? -1 : 1, isYReversed ? -1 : 1, -1));
}
}
}
Loading

0 comments on commit 527d3b1

Please sign in to comment.