SkAR-Java: finger painting first implementation

SkAR Java: drawing planes as paths


FREEZE.unindexed

Bug: skia:
Change-Id: I2a7a8534fc1c35c4b8d6054bc1d6e682ffabc47a
Reviewed-on: https://skia-review.googlesource.com/141827
Reviewed-by: Mike Reed <reed@google.com>

4 files changed, 180 insertions, 2 deletions

diff --git a/platform_tools/android/apps/skar_java/src/main/AndroidManifest.xml b/platform_tools/android/apps/skar_java/src/main/AndroidManifest.xml
index 6f93cbf7a2..3ee3cd9cd7 100644
--- a/platform_tools/android/apps/skar_java/src/main/AndroidManifest.xml
+++ b/platform_tools/android/apps/skar_java/src/main/AndroidManifest.xml
@@ -46,5 +46,9 @@
     <!-- This tag indicates that this application requires ARCore.  This results in the Google Play
          Store downloading and installing ARCore along with the application. -->
     <meta-data android:name="com.google.ar.core" android:value="required" />
+    <meta-data
+        android:name="com.google.ar.core.min_apk_version"
+        tools:replace="android:value"
+        android:value="180226000" />
   </application>
 </manifest>
diff --git a/platform_tools/android/apps/skar_java/src/main/java/com/google/ar/core/examples/java/helloskar/DrawManager.java b/platform_tools/android/apps/skar_java/src/main/java/com/google/ar/core/examples/java/helloskar/DrawManager.java
index 7e5ac29384..020df5ea33 100644
--- a/platform_tools/android/apps/skar_java/src/main/java/com/google/ar/core/examples/java/helloskar/DrawManager.java
+++ b/platform_tools/android/apps/skar_java/src/main/java/com/google/ar/core/examples/java/helloskar/DrawManager.java
@@ -1,12 +1,14 @@
 package com.google.ar.core.examples.java.helloskar;
 
 import android.content.Context;
+import android.content.pm.ApplicationInfo;
 import android.graphics.Bitmap;
 import android.graphics.BitmapFactory;
 import android.graphics.BitmapShader;
 import android.graphics.Canvas;
 import android.graphics.Color;
 import android.graphics.ColorFilter;
+import android.graphics.LinearGradient;
 import android.graphics.Paint;
 import android.graphics.Path;
 import android.graphics.PointF;
@@ -15,10 +17,13 @@ import android.graphics.PorterDuffColorFilter;
 import android.graphics.RectF;
 import android.graphics.Shader;
 import android.opengl.Matrix;
+import android.os.Build;
+
 import com.google.ar.core.Plane;
 import com.google.ar.core.PointCloud;
 import com.google.ar.core.Pose;
 import com.google.ar.core.TrackingState;
+import com.google.skar.SkARFingerPainting;
 import com.google.skar.SkARMatrix;
 import com.google.skar.SkARUtil;
 import java.io.IOException;
@@ -40,6 +45,7 @@ public class DrawManager {
     private ColorFilter lightFilter;
     private BitmapShader planeShader;
     public ArrayList<float[]> modelMatrices = new ArrayList<>();
+    public SkARFingerPainting fingerPainting = new SkARFingerPainting();
 
     public void updateViewport(float width, float height) {
         viewportWidth = width;
@@ -58,6 +64,10 @@ public class DrawManager {
         lightFilter = SkARUtil.createLightCorrectionColorFilter(colorCorr);
     }
 
+    public void updateFingerPainting(PointF p) {
+        fingerPainting.addPoint(p);
+    }
+
     // Sample function for drawing a circle
     public void drawCircle(Canvas canvas) {
         if (modelMatrices.isEmpty()) {
@@ -129,6 +139,41 @@ public class DrawManager {
         canvas.restore();
     }
 
+    public void drawFingerPainting(Canvas canvas) {
+        if (fingerPainting.path.isEmpty()) {
+            return;
+        }
+
+        // Get finger painting model matrix
+        float[] m = fingerPainting.getModelMatrix();
+        float[] model = new float[16];
+        Matrix.setIdentityM(model, 0);
+        Matrix.translateM(model, 0, m[12], m[13], m[14]);
+
+        float[] initRot = SkARMatrix.createXYtoXZRotationMatrix();
+
+        // Matrix = mvpv
+        float[][] matrices = {initRot, model, viewMatrix, projectionMatrix, SkARMatrix.createViewportMatrix(viewportWidth, viewportHeight)};
+        android.graphics.Matrix mvpv = SkARMatrix.createMatrixFrom4x4(SkARMatrix.multiplyMatrices4x4(matrices));
+
+        // Set up paint
+        Paint p = new Paint();
+        p.setColor(Color.GREEN);
+        p.setStyle(Paint.Style.STROKE);
+        p.setStrokeWidth(10f);
+        p.setAlpha(120);
+
+        // Build destination path by transforming source path
+        Path pathDst = new Path();
+        fingerPainting.path.transform(mvpv, pathDst);
+
+        // Draw dest path
+        canvas.save();
+        canvas.setMatrix(new android.graphics.Matrix());
+        canvas.drawPath(pathDst, p);
+        canvas.restore();
+    }
+
     // Sample function for drawing the AR point cloud
     public void drawPointCloud(Canvas canvas, PointCloud cloud) {
         FloatBuffer points = cloud.getPoints();
@@ -224,7 +269,7 @@ public class DrawManager {
 
         // Shader local matrix
         android.graphics.Matrix lm = new android.graphics.Matrix();
-        lm.setScale(0.0005f, 0.0005f);
+        lm.setScale(0.00005f, 0.00005f);
         lm.postConcat(mvpv);
         planeShader.setLocalMatrix(lm);
 
diff --git a/platform_tools/android/apps/skar_java/src/main/java/com/google/ar/core/examples/java/helloskar/HelloSkARActivity.java b/platform_tools/android/apps/skar_java/src/main/java/com/google/ar/core/examples/java/helloskar/HelloSkARActivity.java
index 7bd4085ff8..b62513684e 100644
--- a/platform_tools/android/apps/skar_java/src/main/java/com/google/ar/core/examples/java/helloskar/HelloSkARActivity.java
+++ b/platform_tools/android/apps/skar_java/src/main/java/com/google/ar/core/examples/java/helloskar/HelloSkARActivity.java
@@ -20,14 +20,17 @@ import android.animation.PropertyValuesHolder;
 import android.animation.ValueAnimator;
 import android.graphics.Canvas;
 import android.graphics.Color;
+import android.graphics.PointF;
 import android.graphics.PorterDuff;
 import android.opengl.GLES20;
 import android.opengl.GLSurfaceView;
+import android.opengl.Matrix;
 import android.os.Bundle;
 import android.support.v7.app.AppCompatActivity;
 import android.util.Log;
 import android.view.MotionEvent;
 import android.view.SurfaceHolder;
+import android.view.View;
 import android.widget.Toast;
 
 import com.google.ar.core.Anchor;
@@ -54,6 +57,7 @@ import com.google.ar.core.exceptions.UnavailableArcoreNotInstalledException;
 import com.google.ar.core.exceptions.UnavailableDeviceNotCompatibleException;
 import com.google.ar.core.exceptions.UnavailableSdkTooOldException;
 import com.google.ar.core.exceptions.UnavailableUserDeclinedInstallationException;
+import com.google.skar.SkARMatrix;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -72,6 +76,8 @@ public class HelloSkARActivity extends AppCompatActivity implements GLSurfaceVie
 
     //Simple SurfaceView used to draw 2D objects on top of the GLSurfaceView
     private ARSurfaceView arSurfaceView;
+    private Canvas canvas;
+    private SurfaceHolder holder;
 
     //GLSurfaceView used to draw 3D objects & camera input
     private GLSurfaceView glSurfaceView;
@@ -93,6 +99,11 @@ public class HelloSkARActivity extends AppCompatActivity implements GLSurfaceVie
     // Temporary matrix allocated here to reduce number of allocations for each frame.
     private final float[] anchorMatrix = new float[16];
 
+    private final float[] back = new float[16];
+
+    PointF previousEvent;
+    android.graphics.Matrix inverted;
+
     // Anchors created from taps used for object placing.
     private final ArrayList<Anchor> anchors = new ArrayList<>();
 
@@ -108,6 +119,7 @@ public class HelloSkARActivity extends AppCompatActivity implements GLSurfaceVie
         arSurfaceView = findViewById(R.id.arsurfaceview);
         glSurfaceView = findViewById(R.id.glsurfaceview);
         arSurfaceView.bringToFront();
+        arSurfaceView.setLayerType(View.LAYER_TYPE_HARDWARE, null);
         displayRotationHelper = new DisplayRotationHelper(/*context=*/ this);
 
         // Set up tap listener.
@@ -259,6 +271,9 @@ public class HelloSkARActivity extends AppCompatActivity implements GLSurfaceVie
 
     @Override
     public void onDrawFrame(GL10 gl) {
+        canvas = null;
+        holder = null;
+
         // Clear screen to notify driver it should not load any pixels from previous frame.
         GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT | GLES20.GL_DEPTH_BUFFER_BIT);
 
@@ -269,6 +284,7 @@ public class HelloSkARActivity extends AppCompatActivity implements GLSurfaceVie
         // the video background can be properly adjusted.
         displayRotationHelper.updateSessionIfNeeded(session);
 
+
         try {
             session.setCameraTextureName(backgroundRenderer.getTextureId());
             Frame frame = session.update();
@@ -297,6 +313,54 @@ public class HelloSkARActivity extends AppCompatActivity implements GLSurfaceVie
                 }
             }
 
+            MotionEvent holdTap = tapHelper.holdPoll();
+            if (holdTap != null && camera.getTrackingState() == TrackingState.TRACKING) {
+                for (HitResult hit : frame.hitTest(holdTap)) {
+                    // Check if any plane was hit, and if it was hit inside the plane polygon
+                    Trackable trackable = hit.getTrackable();
+                    // Creates an anchor if a plane or an oriented point was hit.
+                    if ((trackable instanceof Plane
+                            && ((Plane) trackable).isPoseInPolygon(hit.getHitPose())
+                            && (DrawManager.calculateDistanceToPlane(hit.getHitPose(), camera.getPose())
+                            > 0))
+                            || (trackable instanceof Point
+                            && ((Point) trackable).getOrientationMode()
+                            == OrientationMode.ESTIMATED_SURFACE_NORMAL)) {
+
+                        float[] pt = {hit.getHitPose().tx(), hit.getHitPose().tz()};
+
+                        if (drawManager.fingerPainting.isEmpty()) {
+                            float[] originalPt = {pt[0], pt[1]};
+
+                            // Get model matrix of first point
+                            float[] m = new float[16];
+                            hit.getHitPose().toMatrix(m, 0);
+                            drawManager.fingerPainting.setModelMatrix(m); //M0
+
+                            // Construct the inverse matrix + the translation to the origin
+                            float[] inv = new float[16];
+                            hit.getHitPose().toMatrix(inv, 0);
+                            Matrix.invertM(inv, 0, inv, 0);
+                            drawManager.fingerPainting.setInverseModelMatrix(inv);
+                            //inverted = SkARMatrix.createMatrixFrom4x4(inv); //M0 -1
+
+                            // Map hit location using the raw inverse matrix
+                            drawManager.fingerPainting.getInverseModelMatrix().mapPoints(originalPt);
+
+                            // Translate the point back to the origin, and update the inverse matrix
+                            Matrix.translateM(inv, 0, -originalPt[0], -originalPt[1], 0);
+                            drawManager.fingerPainting.setInverseModelMatrix(inv);
+                            //inverted = SkARMatrix.createMatrixFrom4x4(inv);
+                        }
+
+                        drawManager.fingerPainting.getInverseModelMatrix().mapPoints(pt);
+                        PointF newPoint = new PointF(pt[0], pt[1]);
+                        drawManager.fingerPainting.addPoint(newPoint);
+                        break;
+                    }
+                }
+            }
+
             // Draw background with OpenGL.
             // TODO: possibly find a way to extract texture and draw on Canvas
             backgroundRenderer.draw(frame);
@@ -350,12 +414,17 @@ public class HelloSkARActivity extends AppCompatActivity implements GLSurfaceVie
                 // Draw models
                 drawModels(canvas);
 
+                // Draw finger painting
+                drawFingerPainting(canvas);
+
                 // Unlock canvas
                 holder.unlockCanvasAndPost(canvas);
             }
-
         } catch (Throwable t) {
             // Avoid crashing the application due to unhandled exceptions.
+            if (holder != null && canvas != null) {
+                holder.unlockCanvasAndPost(canvas);
+            }
             Log.e(TAG, "Exception on the OpenGL thread", t);
         }
     }
@@ -385,4 +454,8 @@ public class HelloSkARActivity extends AppCompatActivity implements GLSurfaceVie
             drawManager.drawText(canvas, "HelloSkAR");
         }
     }
+
+    private void drawFingerPainting(Canvas canvas) {
+        drawManager.drawFingerPainting(canvas);
+    }
 }
diff --git a/platform_tools/android/apps/skar_java/src/main/java/com/google/skar/SkARFingerPainting.java b/platform_tools/android/apps/skar_java/src/main/java/com/google/skar/SkARFingerPainting.java
new file mode 100644
index 0000000000..6d3d0582a4
--- /dev/null
+++ b/platform_tools/android/apps/skar_java/src/main/java/com/google/skar/SkARFingerPainting.java
@@ -0,0 +1,56 @@
+package com.google.skar;
+
+import android.graphics.Matrix;
+import android.graphics.Path;
+import android.graphics.PointF;
+
+public class SkARFingerPainting { ;
+    public Path path = new Path();
+
+    private int numberOfPoints = 0;
+
+    // Holds the model matrix of the first point added to such that the path can be drawn at the
+    // model location (i.e on the Plane)
+    private float[] modelMatrix;
+
+    // Holds the inverse model matrix of the first point that was added such that the path is drawn
+    // first at (0, 0)
+    private float[] inverseModelMatrix;
+
+    public SkARFingerPainting() {}
+
+    // Adds another point to the path in Local space (i.e apply InverseModelMatrix to points located
+    // in Global space (e.g hit positions acquired through hit tests)
+    public void addPoint(PointF p) {
+        if (numberOfPoints == 0) {
+            path.moveTo(p.x, p.y);
+        } else {
+            path.lineTo(p.x, p.y);
+        }
+        numberOfPoints++;
+    }
+
+    public boolean isEmpty() {
+        return numberOfPoints == 0;
+    }
+
+    public float[] getModelMatrix() {
+        return modelMatrix;
+    }
+
+    public float[] getRawInverseModelMatrix() {
+        return inverseModelMatrix;
+    }
+
+    public Matrix getInverseModelMatrix() {
+        return SkARMatrix.createMatrixFrom4x4(inverseModelMatrix);
+    }
+
+    public void setModelMatrix(float[] m) {
+        modelMatrix = m;
+    }
+
+    public void setInverseModelMatrix(float[] m) {
+        inverseModelMatrix = m;
+    }
+}