From a44a8a16a8e801c08ee407cfaaafd892d5c2c407 Mon Sep 17 00:00:00 2001 From: ziadb Date: Tue, 17 Jul 2018 15:30:55 -0400 Subject: SkAR-Java: finger painting first implementation SkAR Java: drawing planes as paths FREEZE.unindexed Bug: skia: Change-Id: I2a7a8534fc1c35c4b8d6054bc1d6e682ffabc47a Reviewed-on: https://skia-review.googlesource.com/141827 Reviewed-by: Mike Reed --- .../apps/skar_java/src/main/AndroidManifest.xml | 4 ++ .../core/examples/java/helloskar/DrawManager.java | 47 +++++++++++++- .../examples/java/helloskar/HelloSkARActivity.java | 75 +++++++++++++++++++++- .../java/com/google/skar/SkARFingerPainting.java | 56 ++++++++++++++++ 4 files changed, 180 insertions(+), 2 deletions(-) create mode 100644 platform_tools/android/apps/skar_java/src/main/java/com/google/skar/SkARFingerPainting.java (limited to 'platform_tools') diff --git a/platform_tools/android/apps/skar_java/src/main/AndroidManifest.xml b/platform_tools/android/apps/skar_java/src/main/AndroidManifest.xml index 6f93cbf7a2..3ee3cd9cd7 100644 --- a/platform_tools/android/apps/skar_java/src/main/AndroidManifest.xml +++ b/platform_tools/android/apps/skar_java/src/main/AndroidManifest.xml @@ -46,5 +46,9 @@ + diff --git a/platform_tools/android/apps/skar_java/src/main/java/com/google/ar/core/examples/java/helloskar/DrawManager.java b/platform_tools/android/apps/skar_java/src/main/java/com/google/ar/core/examples/java/helloskar/DrawManager.java index 7e5ac29384..020df5ea33 100644 --- a/platform_tools/android/apps/skar_java/src/main/java/com/google/ar/core/examples/java/helloskar/DrawManager.java +++ b/platform_tools/android/apps/skar_java/src/main/java/com/google/ar/core/examples/java/helloskar/DrawManager.java @@ -1,12 +1,14 @@ package com.google.ar.core.examples.java.helloskar; import android.content.Context; +import android.content.pm.ApplicationInfo; import android.graphics.Bitmap; import android.graphics.BitmapFactory; import android.graphics.BitmapShader; import android.graphics.Canvas; import android.graphics.Color; import android.graphics.ColorFilter; +import android.graphics.LinearGradient; import android.graphics.Paint; import android.graphics.Path; import android.graphics.PointF; @@ -15,10 +17,13 @@ import android.graphics.PorterDuffColorFilter; import android.graphics.RectF; import android.graphics.Shader; import android.opengl.Matrix; +import android.os.Build; + import com.google.ar.core.Plane; import com.google.ar.core.PointCloud; import com.google.ar.core.Pose; import com.google.ar.core.TrackingState; +import com.google.skar.SkARFingerPainting; import com.google.skar.SkARMatrix; import com.google.skar.SkARUtil; import java.io.IOException; @@ -40,6 +45,7 @@ public class DrawManager { private ColorFilter lightFilter; private BitmapShader planeShader; public ArrayList modelMatrices = new ArrayList<>(); + public SkARFingerPainting fingerPainting = new SkARFingerPainting(); public void updateViewport(float width, float height) { viewportWidth = width; @@ -58,6 +64,10 @@ public class DrawManager { lightFilter = SkARUtil.createLightCorrectionColorFilter(colorCorr); } + public void updateFingerPainting(PointF p) { + fingerPainting.addPoint(p); + } + // Sample function for drawing a circle public void drawCircle(Canvas canvas) { if (modelMatrices.isEmpty()) { @@ -129,6 +139,41 @@ public class DrawManager { canvas.restore(); } + public void drawFingerPainting(Canvas canvas) { + if (fingerPainting.path.isEmpty()) { + return; + } + + // Get finger painting model matrix + float[] m = fingerPainting.getModelMatrix(); + float[] model = new float[16]; + Matrix.setIdentityM(model, 0); + Matrix.translateM(model, 0, m[12], m[13], m[14]); + + float[] initRot = SkARMatrix.createXYtoXZRotationMatrix(); + + // Matrix = mvpv + float[][] matrices = {initRot, model, viewMatrix, projectionMatrix, SkARMatrix.createViewportMatrix(viewportWidth, viewportHeight)}; + android.graphics.Matrix mvpv = SkARMatrix.createMatrixFrom4x4(SkARMatrix.multiplyMatrices4x4(matrices)); + + // Set up paint + Paint p = new Paint(); + p.setColor(Color.GREEN); + p.setStyle(Paint.Style.STROKE); + p.setStrokeWidth(10f); + p.setAlpha(120); + + // Build destination path by transforming source path + Path pathDst = new Path(); + fingerPainting.path.transform(mvpv, pathDst); + + // Draw dest path + canvas.save(); + canvas.setMatrix(new android.graphics.Matrix()); + canvas.drawPath(pathDst, p); + canvas.restore(); + } + // Sample function for drawing the AR point cloud public void drawPointCloud(Canvas canvas, PointCloud cloud) { FloatBuffer points = cloud.getPoints(); @@ -224,7 +269,7 @@ public class DrawManager { // Shader local matrix android.graphics.Matrix lm = new android.graphics.Matrix(); - lm.setScale(0.0005f, 0.0005f); + lm.setScale(0.00005f, 0.00005f); lm.postConcat(mvpv); planeShader.setLocalMatrix(lm); diff --git a/platform_tools/android/apps/skar_java/src/main/java/com/google/ar/core/examples/java/helloskar/HelloSkARActivity.java b/platform_tools/android/apps/skar_java/src/main/java/com/google/ar/core/examples/java/helloskar/HelloSkARActivity.java index 7bd4085ff8..b62513684e 100644 --- a/platform_tools/android/apps/skar_java/src/main/java/com/google/ar/core/examples/java/helloskar/HelloSkARActivity.java +++ b/platform_tools/android/apps/skar_java/src/main/java/com/google/ar/core/examples/java/helloskar/HelloSkARActivity.java @@ -20,14 +20,17 @@ import android.animation.PropertyValuesHolder; import android.animation.ValueAnimator; import android.graphics.Canvas; import android.graphics.Color; +import android.graphics.PointF; import android.graphics.PorterDuff; import android.opengl.GLES20; import android.opengl.GLSurfaceView; +import android.opengl.Matrix; import android.os.Bundle; import android.support.v7.app.AppCompatActivity; import android.util.Log; import android.view.MotionEvent; import android.view.SurfaceHolder; +import android.view.View; import android.widget.Toast; import com.google.ar.core.Anchor; @@ -54,6 +57,7 @@ import com.google.ar.core.exceptions.UnavailableArcoreNotInstalledException; import com.google.ar.core.exceptions.UnavailableDeviceNotCompatibleException; import com.google.ar.core.exceptions.UnavailableSdkTooOldException; import com.google.ar.core.exceptions.UnavailableUserDeclinedInstallationException; +import com.google.skar.SkARMatrix; import java.io.IOException; import java.util.ArrayList; @@ -72,6 +76,8 @@ public class HelloSkARActivity extends AppCompatActivity implements GLSurfaceVie //Simple SurfaceView used to draw 2D objects on top of the GLSurfaceView private ARSurfaceView arSurfaceView; + private Canvas canvas; + private SurfaceHolder holder; //GLSurfaceView used to draw 3D objects & camera input private GLSurfaceView glSurfaceView; @@ -93,6 +99,11 @@ public class HelloSkARActivity extends AppCompatActivity implements GLSurfaceVie // Temporary matrix allocated here to reduce number of allocations for each frame. private final float[] anchorMatrix = new float[16]; + private final float[] back = new float[16]; + + PointF previousEvent; + android.graphics.Matrix inverted; + // Anchors created from taps used for object placing. private final ArrayList anchors = new ArrayList<>(); @@ -108,6 +119,7 @@ public class HelloSkARActivity extends AppCompatActivity implements GLSurfaceVie arSurfaceView = findViewById(R.id.arsurfaceview); glSurfaceView = findViewById(R.id.glsurfaceview); arSurfaceView.bringToFront(); + arSurfaceView.setLayerType(View.LAYER_TYPE_HARDWARE, null); displayRotationHelper = new DisplayRotationHelper(/*context=*/ this); // Set up tap listener. @@ -259,6 +271,9 @@ public class HelloSkARActivity extends AppCompatActivity implements GLSurfaceVie @Override public void onDrawFrame(GL10 gl) { + canvas = null; + holder = null; + // Clear screen to notify driver it should not load any pixels from previous frame. GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT | GLES20.GL_DEPTH_BUFFER_BIT); @@ -269,6 +284,7 @@ public class HelloSkARActivity extends AppCompatActivity implements GLSurfaceVie // the video background can be properly adjusted. displayRotationHelper.updateSessionIfNeeded(session); + try { session.setCameraTextureName(backgroundRenderer.getTextureId()); Frame frame = session.update(); @@ -297,6 +313,54 @@ public class HelloSkARActivity extends AppCompatActivity implements GLSurfaceVie } } + MotionEvent holdTap = tapHelper.holdPoll(); + if (holdTap != null && camera.getTrackingState() == TrackingState.TRACKING) { + for (HitResult hit : frame.hitTest(holdTap)) { + // Check if any plane was hit, and if it was hit inside the plane polygon + Trackable trackable = hit.getTrackable(); + // Creates an anchor if a plane or an oriented point was hit. + if ((trackable instanceof Plane + && ((Plane) trackable).isPoseInPolygon(hit.getHitPose()) + && (DrawManager.calculateDistanceToPlane(hit.getHitPose(), camera.getPose()) + > 0)) + || (trackable instanceof Point + && ((Point) trackable).getOrientationMode() + == OrientationMode.ESTIMATED_SURFACE_NORMAL)) { + + float[] pt = {hit.getHitPose().tx(), hit.getHitPose().tz()}; + + if (drawManager.fingerPainting.isEmpty()) { + float[] originalPt = {pt[0], pt[1]}; + + // Get model matrix of first point + float[] m = new float[16]; + hit.getHitPose().toMatrix(m, 0); + drawManager.fingerPainting.setModelMatrix(m); //M0 + + // Construct the inverse matrix + the translation to the origin + float[] inv = new float[16]; + hit.getHitPose().toMatrix(inv, 0); + Matrix.invertM(inv, 0, inv, 0); + drawManager.fingerPainting.setInverseModelMatrix(inv); + //inverted = SkARMatrix.createMatrixFrom4x4(inv); //M0 -1 + + // Map hit location using the raw inverse matrix + drawManager.fingerPainting.getInverseModelMatrix().mapPoints(originalPt); + + // Translate the point back to the origin, and update the inverse matrix + Matrix.translateM(inv, 0, -originalPt[0], -originalPt[1], 0); + drawManager.fingerPainting.setInverseModelMatrix(inv); + //inverted = SkARMatrix.createMatrixFrom4x4(inv); + } + + drawManager.fingerPainting.getInverseModelMatrix().mapPoints(pt); + PointF newPoint = new PointF(pt[0], pt[1]); + drawManager.fingerPainting.addPoint(newPoint); + break; + } + } + } + // Draw background with OpenGL. // TODO: possibly find a way to extract texture and draw on Canvas backgroundRenderer.draw(frame); @@ -350,12 +414,17 @@ public class HelloSkARActivity extends AppCompatActivity implements GLSurfaceVie // Draw models drawModels(canvas); + // Draw finger painting + drawFingerPainting(canvas); + // Unlock canvas holder.unlockCanvasAndPost(canvas); } - } catch (Throwable t) { // Avoid crashing the application due to unhandled exceptions. + if (holder != null && canvas != null) { + holder.unlockCanvasAndPost(canvas); + } Log.e(TAG, "Exception on the OpenGL thread", t); } } @@ -385,4 +454,8 @@ public class HelloSkARActivity extends AppCompatActivity implements GLSurfaceVie drawManager.drawText(canvas, "HelloSkAR"); } } + + private void drawFingerPainting(Canvas canvas) { + drawManager.drawFingerPainting(canvas); + } } diff --git a/platform_tools/android/apps/skar_java/src/main/java/com/google/skar/SkARFingerPainting.java b/platform_tools/android/apps/skar_java/src/main/java/com/google/skar/SkARFingerPainting.java new file mode 100644 index 0000000000..6d3d0582a4 --- /dev/null +++ b/platform_tools/android/apps/skar_java/src/main/java/com/google/skar/SkARFingerPainting.java @@ -0,0 +1,56 @@ +package com.google.skar; + +import android.graphics.Matrix; +import android.graphics.Path; +import android.graphics.PointF; + +public class SkARFingerPainting { ; + public Path path = new Path(); + + private int numberOfPoints = 0; + + // Holds the model matrix of the first point added to such that the path can be drawn at the + // model location (i.e on the Plane) + private float[] modelMatrix; + + // Holds the inverse model matrix of the first point that was added such that the path is drawn + // first at (0, 0) + private float[] inverseModelMatrix; + + public SkARFingerPainting() {} + + // Adds another point to the path in Local space (i.e apply InverseModelMatrix to points located + // in Global space (e.g hit positions acquired through hit tests) + public void addPoint(PointF p) { + if (numberOfPoints == 0) { + path.moveTo(p.x, p.y); + } else { + path.lineTo(p.x, p.y); + } + numberOfPoints++; + } + + public boolean isEmpty() { + return numberOfPoints == 0; + } + + public float[] getModelMatrix() { + return modelMatrix; + } + + public float[] getRawInverseModelMatrix() { + return inverseModelMatrix; + } + + public Matrix getInverseModelMatrix() { + return SkARMatrix.createMatrixFrom4x4(inverseModelMatrix); + } + + public void setModelMatrix(float[] m) { + modelMatrix = m; + } + + public void setInverseModelMatrix(float[] m) { + inverseModelMatrix = m; + } +} -- cgit v1.2.3