From bb5b7588d67aa3ed24f1b44ded654215e1818b2f Mon Sep 17 00:00:00 2001 From: ziadb Date: Mon, 2 Jul 2018 11:12:27 -0400 Subject: Java-only version of SkAR: drawing on android Canvas Bug: skia: Change-Id: I3b85fac93a2854d1575f71554e2a7da66e8a6a6f Reviewed-on: https://skia-review.googlesource.com/138920 Reviewed-by: Mike Reed --- platform_tools/android/apps/settings.gradle | 1 + platform_tools/android/apps/skar_java/build.gradle | 42 +++ .../android/apps/skar_java/proguard-rules.pro | 17 + .../apps/skar_java/src/main/AndroidManifest.xml | 50 +++ .../skar_java/src/main/assets/models/trigrid.png | Bin 0 -> 37354 bytes .../skar_java/src/main/assets/shaders/plane.frag | 31 ++ .../skar_java/src/main/assets/shaders/plane.vert | 40 +++ .../src/main/assets/shaders/point_cloud.frag | 21 ++ .../src/main/assets/shaders/point_cloud.vert | 28 ++ .../src/main/assets/shaders/screenquad.frag | 24 ++ .../src/main/assets/shaders/screenquad.vert | 24 ++ .../examples/java/helloskar/ARSurfaceView.java | 45 +++ .../core/examples/java/helloskar/DrawManager.java | 106 ++++++ .../examples/java/helloskar/HelloSkARActivity.java | 355 +++++++++++++++++++++ .../src/main/java/com/google/skar/SkARMatrix.java | 211 ++++++++++++ .../src/main/java/com/google/skar/SkARUtil.java | 31 ++ .../src/main/res/drawable-xxhdpi/ic_launcher.png | Bin 0 -> 21226 bytes .../src/main/res/layout/activity_main.xml | 39 +++ .../apps/skar_java/src/main/res/values/strings.xml | 18 ++ .../apps/skar_java/src/main/res/values/styles.xml | 35 ++ 20 files changed, 1118 insertions(+) create mode 100644 platform_tools/android/apps/skar_java/build.gradle create mode 100644 platform_tools/android/apps/skar_java/proguard-rules.pro create mode 100644 platform_tools/android/apps/skar_java/src/main/AndroidManifest.xml create mode 100644 platform_tools/android/apps/skar_java/src/main/assets/models/trigrid.png create mode 100644 platform_tools/android/apps/skar_java/src/main/assets/shaders/plane.frag create mode 100644 platform_tools/android/apps/skar_java/src/main/assets/shaders/plane.vert create mode 100644 platform_tools/android/apps/skar_java/src/main/assets/shaders/point_cloud.frag create mode 100644 platform_tools/android/apps/skar_java/src/main/assets/shaders/point_cloud.vert create mode 100644 platform_tools/android/apps/skar_java/src/main/assets/shaders/screenquad.frag create mode 100644 platform_tools/android/apps/skar_java/src/main/assets/shaders/screenquad.vert create mode 100644 platform_tools/android/apps/skar_java/src/main/java/com/google/ar/core/examples/java/helloskar/ARSurfaceView.java create mode 100644 platform_tools/android/apps/skar_java/src/main/java/com/google/ar/core/examples/java/helloskar/DrawManager.java create mode 100644 platform_tools/android/apps/skar_java/src/main/java/com/google/ar/core/examples/java/helloskar/HelloSkARActivity.java create mode 100644 platform_tools/android/apps/skar_java/src/main/java/com/google/skar/SkARMatrix.java create mode 100644 platform_tools/android/apps/skar_java/src/main/java/com/google/skar/SkARUtil.java create mode 100644 platform_tools/android/apps/skar_java/src/main/res/drawable-xxhdpi/ic_launcher.png create mode 100644 platform_tools/android/apps/skar_java/src/main/res/layout/activity_main.xml create mode 100644 platform_tools/android/apps/skar_java/src/main/res/values/strings.xml create mode 100644 platform_tools/android/apps/skar_java/src/main/res/values/styles.xml (limited to 'platform_tools/android') diff --git a/platform_tools/android/apps/settings.gradle b/platform_tools/android/apps/settings.gradle index 148e1ac2af..72774bb32b 100644 --- a/platform_tools/android/apps/settings.gradle +++ b/platform_tools/android/apps/settings.gradle @@ -1,3 +1,4 @@ include ':viewer' include ':skqp' include ':arcore' //must build out directory first: bin/gn gen out/arm64 --args='ndk="NDKPATH" target_cpu="ABI" is_component_build=true' +include ':skar_java' \ No newline at end of file diff --git a/platform_tools/android/apps/skar_java/build.gradle b/platform_tools/android/apps/skar_java/build.gradle new file mode 100644 index 0000000000..fd85e24230 --- /dev/null +++ b/platform_tools/android/apps/skar_java/build.gradle @@ -0,0 +1,42 @@ +apply plugin: 'com.android.application' + +android { + compileSdkVersion 27 + defaultConfig { + applicationId "com.google.ar.core.examples.java.helloar" + + // 24 is the minimum since ARCore only works with 24 and higher. + minSdkVersion 26 + targetSdkVersion 27 + versionCode 1 + versionName "1.0" + } + + testOptions { + unitTests.returnDefaultValues = false + } + + buildTypes { + release { + minifyEnabled false + proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro' + } + } +} + +dependencies { + // ARCore library + implementation 'com.google.ar:core:1.2.0' + + // Obj - a simple Wavefront OBJ file loader + // https://github.com/javagl/Obj + implementation 'de.javagl:obj:0.2.1' + + implementation 'com.android.support:appcompat-v7:27.0.2' + implementation 'com.android.support:design:27.0.2' + + // Required -- JUnit 4 framework + testImplementation 'junit:junit:4.12' + // Optional -- Mockito framework + testImplementation 'org.mockito:mockito-core:1.10.19' +} diff --git a/platform_tools/android/apps/skar_java/proguard-rules.pro b/platform_tools/android/apps/skar_java/proguard-rules.pro new file mode 100644 index 0000000000..45dc58a590 --- /dev/null +++ b/platform_tools/android/apps/skar_java/proguard-rules.pro @@ -0,0 +1,17 @@ +# Add project specific ProGuard rules here. +# By default, the flags in this file are appended to flags specified +# in /opt/android-sdk/tools/proguard/proguard-android.txt +# You can edit the include path and order by changing the proguardFiles +# directive in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# Add any project specific keep options here: + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} diff --git a/platform_tools/android/apps/skar_java/src/main/AndroidManifest.xml b/platform_tools/android/apps/skar_java/src/main/AndroidManifest.xml new file mode 100644 index 0000000000..6f93cbf7a2 --- /dev/null +++ b/platform_tools/android/apps/skar_java/src/main/AndroidManifest.xml @@ -0,0 +1,50 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/platform_tools/android/apps/skar_java/src/main/assets/models/trigrid.png b/platform_tools/android/apps/skar_java/src/main/assets/models/trigrid.png new file mode 100644 index 0000000000..05cbe6e52f Binary files /dev/null and b/platform_tools/android/apps/skar_java/src/main/assets/models/trigrid.png differ diff --git a/platform_tools/android/apps/skar_java/src/main/assets/shaders/plane.frag b/platform_tools/android/apps/skar_java/src/main/assets/shaders/plane.frag new file mode 100644 index 0000000000..d0a4708895 --- /dev/null +++ b/platform_tools/android/apps/skar_java/src/main/assets/shaders/plane.frag @@ -0,0 +1,31 @@ +/* + * Copyright 2017 Google Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; +uniform sampler2D u_Texture; +uniform vec4 u_dotColor; +uniform vec4 u_lineColor; +uniform vec4 u_gridControl; // dotThreshold, lineThreshold, lineFadeShrink, occlusionShrink +varying vec3 v_TexCoordAlpha; + +void main() { + vec4 control = texture2D(u_Texture, v_TexCoordAlpha.xy); + float dotScale = v_TexCoordAlpha.z; + float lineFade = max(0.0, u_gridControl.z * v_TexCoordAlpha.z - (u_gridControl.z - 1.0)); + vec3 color = (control.r * dotScale > u_gridControl.x) ? u_dotColor.rgb + : (control.g > u_gridControl.y) ? u_lineColor.rgb * lineFade + : (u_lineColor.rgb * 0.25 * lineFade) ; + gl_FragColor = vec4(color, v_TexCoordAlpha.z * u_gridControl.w); +} diff --git a/platform_tools/android/apps/skar_java/src/main/assets/shaders/plane.vert b/platform_tools/android/apps/skar_java/src/main/assets/shaders/plane.vert new file mode 100644 index 0000000000..9ac5a6db27 --- /dev/null +++ b/platform_tools/android/apps/skar_java/src/main/assets/shaders/plane.vert @@ -0,0 +1,40 @@ +/* + * Copyright 2017 Google Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +uniform mat4 u_Model; +uniform mat4 u_ModelViewProjection; +uniform mat2 u_PlaneUvMatrix; +uniform vec3 u_Normal; + +attribute vec3 a_XZPositionAlpha; // (x, z, alpha) + +varying vec3 v_TexCoordAlpha; + +void main() { + vec4 local_pos = vec4(a_XZPositionAlpha.x, 0.0, a_XZPositionAlpha.y, 1.0); + vec4 world_pos = u_Model * local_pos; + + // Construct two vectors that are orthogonal to the normal. + // This arbitrary choice is not co-linear with either horizontal + // or vertical plane normals. + const vec3 arbitrary = vec3(1.0, 1.0, 0.0); + vec3 vec_u = normalize(cross(u_Normal, arbitrary)); + vec3 vec_v = normalize(cross(u_Normal, vec_u)); + + // Project vertices in world frame onto vec_u and vec_v. + vec2 uv = vec2(dot(world_pos.xyz, vec_u), dot(world_pos.xyz, vec_v)); + v_TexCoordAlpha = vec3(u_PlaneUvMatrix * uv, a_XZPositionAlpha.z); + gl_Position = u_ModelViewProjection * local_pos; +} diff --git a/platform_tools/android/apps/skar_java/src/main/assets/shaders/point_cloud.frag b/platform_tools/android/apps/skar_java/src/main/assets/shaders/point_cloud.frag new file mode 100644 index 0000000000..463d0526e4 --- /dev/null +++ b/platform_tools/android/apps/skar_java/src/main/assets/shaders/point_cloud.frag @@ -0,0 +1,21 @@ +/* + * Copyright 2017 Google Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision mediump float; +varying vec4 v_Color; + +void main() { + gl_FragColor = v_Color; +} diff --git a/platform_tools/android/apps/skar_java/src/main/assets/shaders/point_cloud.vert b/platform_tools/android/apps/skar_java/src/main/assets/shaders/point_cloud.vert new file mode 100644 index 0000000000..627fc1a6f3 --- /dev/null +++ b/platform_tools/android/apps/skar_java/src/main/assets/shaders/point_cloud.vert @@ -0,0 +1,28 @@ +/* + * Copyright 2017 Google Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +uniform mat4 u_ModelViewProjection; +uniform vec4 u_Color; +uniform float u_PointSize; + +attribute vec4 a_Position; + +varying vec4 v_Color; + +void main() { + v_Color = u_Color; + gl_Position = u_ModelViewProjection * vec4(a_Position.xyz, 1.0); + gl_PointSize = u_PointSize; +} diff --git a/platform_tools/android/apps/skar_java/src/main/assets/shaders/screenquad.frag b/platform_tools/android/apps/skar_java/src/main/assets/shaders/screenquad.frag new file mode 100644 index 0000000000..800d723a80 --- /dev/null +++ b/platform_tools/android/apps/skar_java/src/main/assets/shaders/screenquad.frag @@ -0,0 +1,24 @@ +/* + * Copyright 2017 Google Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#extension GL_OES_EGL_image_external : require + +precision mediump float; +varying vec2 v_TexCoord; +uniform samplerExternalOES sTexture; + + +void main() { + gl_FragColor = texture2D(sTexture, v_TexCoord); +} diff --git a/platform_tools/android/apps/skar_java/src/main/assets/shaders/screenquad.vert b/platform_tools/android/apps/skar_java/src/main/assets/shaders/screenquad.vert new file mode 100644 index 0000000000..01c93e3d48 --- /dev/null +++ b/platform_tools/android/apps/skar_java/src/main/assets/shaders/screenquad.vert @@ -0,0 +1,24 @@ +/* + * Copyright 2017 Google Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +attribute vec4 a_Position; +attribute vec2 a_TexCoord; + +varying vec2 v_TexCoord; + +void main() { + gl_Position = a_Position; + v_TexCoord = a_TexCoord; +} diff --git a/platform_tools/android/apps/skar_java/src/main/java/com/google/ar/core/examples/java/helloskar/ARSurfaceView.java b/platform_tools/android/apps/skar_java/src/main/java/com/google/ar/core/examples/java/helloskar/ARSurfaceView.java new file mode 100644 index 0000000000..eaa336cc9e --- /dev/null +++ b/platform_tools/android/apps/skar_java/src/main/java/com/google/ar/core/examples/java/helloskar/ARSurfaceView.java @@ -0,0 +1,45 @@ +package com.google.ar.core.examples.java.helloskar; + +import android.content.Context; +import android.graphics.Color; +import android.graphics.PixelFormat; +import android.util.AttributeSet; +import android.view.SurfaceHolder; +import android.view.SurfaceView; + +/** + * SurfaceView that is overlayed on top of a GLSurfaceView. All 2D drawings can be done on this + * surface. + */ +public class ARSurfaceView extends SurfaceView implements SurfaceHolder.Callback { + + boolean running; + + public ARSurfaceView(Context context, AttributeSet attrs) { + super(context, attrs); + + SurfaceHolder holder = getHolder(); + this.setBackgroundColor(Color.TRANSPARENT); + this.setZOrderOnTop(true); + holder.setFormat(PixelFormat.TRANSPARENT); + holder.addCallback(this); + } + + public boolean isRunning() { + return running; + } + + @Override + public void surfaceCreated(SurfaceHolder holder) { + running = true; + } + + @Override + public void surfaceDestroyed(SurfaceHolder holder) { + running = false; + } + + @Override + public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) { + } +} diff --git a/platform_tools/android/apps/skar_java/src/main/java/com/google/ar/core/examples/java/helloskar/DrawManager.java b/platform_tools/android/apps/skar_java/src/main/java/com/google/ar/core/examples/java/helloskar/DrawManager.java new file mode 100644 index 0000000000..bf85a767c2 --- /dev/null +++ b/platform_tools/android/apps/skar_java/src/main/java/com/google/ar/core/examples/java/helloskar/DrawManager.java @@ -0,0 +1,106 @@ +package com.google.ar.core.examples.java.helloskar; + +import android.graphics.Canvas; +import android.graphics.ColorFilter; +import android.graphics.Paint; +import android.graphics.RectF; + +import com.google.skar.SkARMatrix; +import com.google.skar.SkARUtil; + +import java.util.ArrayList; + +public class DrawManager { + private float[] projectionMatrix = new float[16]; + private float[] viewMatrix = new float[16]; + private float[] viewportMatrix = new float[16]; + private ColorFilter lightFilter; + public ArrayList modelMatrices = new ArrayList<>(); + + public void updateViewportMatrix(float width, float height) { + viewportMatrix = SkARMatrix.createViewportMatrix(width, height); + } + + public void updateProjectionMatrix(float[] projectionMatrix) { + this.projectionMatrix = projectionMatrix; + } + + public void updateViewMatrix(float[] viewMatrix) { + this.viewMatrix = viewMatrix; + } + + public void updateLightColorFilter(float[] colorCorr) { + lightFilter = SkARUtil.createLightCorrectionColorFilter(colorCorr); + } + + public void drawCircle(Canvas canvas) { + if (modelMatrices.isEmpty()) { + return; + } + Paint p = new Paint(); + p.setColorFilter(lightFilter); + p.setARGB(180, 100, 0, 0); + + canvas.save(); + canvas.setMatrix(SkARMatrix.createPerspectiveMatrix(modelMatrices.get(0), + viewMatrix, projectionMatrix, viewportMatrix)); + canvas.drawCircle(0, 0, 0.1f, p); + canvas.restore(); + } + + public void drawRect(Canvas canvas) { + if (modelMatrices.isEmpty()) { + return; + } + Paint p = new Paint(); + p.setColorFilter(lightFilter); + p.setARGB(180, 0, 0, 255); + canvas.save(); + canvas.setMatrix(SkARMatrix.createPerspectiveMatrix(modelMatrices.get(0), + viewMatrix, projectionMatrix, viewportMatrix)); + RectF rect = new RectF(0, 0, 0.2f, 0.2f); + canvas.drawRect(rect, p); + canvas.restore(); + } + + public void drawText(Canvas canvas, String text) { + if (modelMatrices.isEmpty()) { + return; + } + Paint p = new Paint(); + float textSize = 100; + p.setColorFilter(lightFilter); + p.setARGB(255, 0, 255, 0); + p.setTextSize(textSize); + + + float[] scaleMatrix = getTextScaleMatrix(textSize); + float[] rotateMatrix = createXYtoXZRotationMatrix(); + float[] actualModel = new float[16]; + android.opengl.Matrix.setIdentityM(actualModel, 0); + + android.opengl.Matrix.multiplyMM(actualModel, 0, scaleMatrix, 0, rotateMatrix, 0); + android.opengl.Matrix.multiplyMM(actualModel, 0, modelMatrices.get(0), 0, actualModel, 0); + + canvas.save(); + canvas.setMatrix(SkARMatrix.createPerspectiveMatrix(actualModel, + viewMatrix, projectionMatrix, viewportMatrix, false)); + canvas.drawText(text, 0, 0, p); + canvas.restore(); + } + + private float[] getTextScaleMatrix(float size) { + float scaleFactor = 1 / (size * 10); + float[] initScale = new float[16]; + android.opengl.Matrix.setIdentityM(initScale, 0); + android.opengl.Matrix.scaleM(initScale, 0, scaleFactor, scaleFactor, scaleFactor); + return initScale; + } + + private float[] createXYtoXZRotationMatrix() { + float[] skiaRotation = new float[16]; + android.opengl.Matrix.setIdentityM(skiaRotation, 0); + android.opengl.Matrix.rotateM(skiaRotation, 0, 90, 1, 0, 0); + return skiaRotation; + } +} diff --git a/platform_tools/android/apps/skar_java/src/main/java/com/google/ar/core/examples/java/helloskar/HelloSkARActivity.java b/platform_tools/android/apps/skar_java/src/main/java/com/google/ar/core/examples/java/helloskar/HelloSkARActivity.java new file mode 100644 index 0000000000..a8ccb53478 --- /dev/null +++ b/platform_tools/android/apps/skar_java/src/main/java/com/google/ar/core/examples/java/helloskar/HelloSkARActivity.java @@ -0,0 +1,355 @@ +/* + * Copyright 2017 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.ar.core.examples.java.helloskar; + +import android.graphics.Canvas; +import android.graphics.Color; +import android.graphics.PorterDuff; +import android.opengl.GLES20; +import android.opengl.GLSurfaceView; +import android.os.Bundle; +import android.support.v7.app.AppCompatActivity; +import android.util.Log; +import android.view.MotionEvent; +import android.view.SurfaceHolder; +import android.widget.Toast; + +import com.google.ar.core.Anchor; +import com.google.ar.core.ArCoreApk; +import com.google.ar.core.Camera; +import com.google.ar.core.Frame; +import com.google.ar.core.HitResult; +import com.google.ar.core.Plane; +import com.google.ar.core.Point; +import com.google.ar.core.Point.OrientationMode; +import com.google.ar.core.PointCloud; +import com.google.ar.core.Session; +import com.google.ar.core.Trackable; +import com.google.ar.core.TrackingState; +import com.google.ar.core.examples.java.common.helpers.CameraPermissionHelper; +import com.google.ar.core.examples.java.common.helpers.DisplayRotationHelper; +import com.google.ar.core.examples.java.common.helpers.FullScreenHelper; +import com.google.ar.core.examples.java.common.helpers.SnackbarHelper; +import com.google.ar.core.examples.java.common.helpers.TapHelper; +import com.google.ar.core.examples.java.common.rendering.BackgroundRenderer; +import com.google.ar.core.examples.java.common.rendering.PlaneRenderer; +import com.google.ar.core.examples.java.common.rendering.PointCloudRenderer; +import com.google.ar.core.exceptions.CameraNotAvailableException; +import com.google.ar.core.exceptions.UnavailableApkTooOldException; +import com.google.ar.core.exceptions.UnavailableArcoreNotInstalledException; +import com.google.ar.core.exceptions.UnavailableDeviceNotCompatibleException; +import com.google.ar.core.exceptions.UnavailableSdkTooOldException; +import com.google.ar.core.exceptions.UnavailableUserDeclinedInstallationException; + +import java.io.IOException; +import java.util.ArrayList; + +import javax.microedition.khronos.egl.EGLConfig; +import javax.microedition.khronos.opengles.GL10; + +/** + * This is a simple example that shows how to create an augmented reality (AR) application using the + * ARCore API. The application will display any detected planes and will allow the user to tap on a + * plane to place 2D objects + */ +public class HelloSkARActivity extends AppCompatActivity implements GLSurfaceView.Renderer { + private static final String TAG = HelloSkARActivity.class.getSimpleName(); + + //Simple SurfaceView used to draw 2D objects on top of the GLSurfaceView + private ARSurfaceView arSurfaceView; + + //GLSurfaceView used to draw 3D objects & camera input + private GLSurfaceView glSurfaceView; + + //ARSession + private Session session; + + private boolean installRequested; + private final SnackbarHelper messageSnackbarHelper = new SnackbarHelper(); + private DisplayRotationHelper displayRotationHelper; + private TapHelper tapHelper; + + //Renderers + private final BackgroundRenderer backgroundRenderer = new BackgroundRenderer(); + private final PlaneRenderer planeRenderer = new PlaneRenderer(); + private final PointCloudRenderer pointCloudRenderer = new PointCloudRenderer(); + + //2D Renderer + private DrawManager drawManager = new DrawManager(); + + // Temporary matrix allocated here to reduce number of allocations for each frame. + private final float[] anchorMatrix = new float[16]; + + // Anchors created from taps used for object placing. + private final ArrayList anchors = new ArrayList<>(); + + @Override + protected void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + setContentView(R.layout.activity_main); + arSurfaceView = findViewById(R.id.arsurfaceview); + glSurfaceView = findViewById(R.id.glsurfaceview); + arSurfaceView.bringToFront(); + displayRotationHelper = new DisplayRotationHelper(/*context=*/ this); + + // Set up tap listener. + tapHelper = new TapHelper(/*context=*/ this); + glSurfaceView.setOnTouchListener(tapHelper); + + // Set up renderer. + glSurfaceView.setPreserveEGLContextOnPause(true); + glSurfaceView.setEGLContextClientVersion(2); + glSurfaceView.setEGLConfigChooser(8, 8, 8, 8, 16, 0); // Alpha used for plane blending. + glSurfaceView.setRenderer(this); + glSurfaceView.setRenderMode(GLSurfaceView.RENDERMODE_CONTINUOUSLY); + + installRequested = false; + } + + @Override + protected void onResume() { + super.onResume(); + + if (session == null) { + Exception exception = null; + String message = null; + try { + switch (ArCoreApk.getInstance().requestInstall(this, !installRequested)) { + case INSTALL_REQUESTED: + installRequested = true; + return; + case INSTALLED: + break; + } + + // ARCore requires camera permissions to operate. If we did not yet obtain runtime + // permission on Android M and above, now is a good time to ask the user for it. + if (!CameraPermissionHelper.hasCameraPermission(this)) { + CameraPermissionHelper.requestCameraPermission(this); + return; + } + + // Create the session. + session = new Session(/* context= */ this); + + } catch (UnavailableArcoreNotInstalledException + | UnavailableUserDeclinedInstallationException e) { + message = "Please install ARCore"; + exception = e; + } catch (UnavailableApkTooOldException e) { + message = "Please update ARCore"; + exception = e; + } catch (UnavailableSdkTooOldException e) { + message = "Please update this app"; + exception = e; + } catch (UnavailableDeviceNotCompatibleException e) { + message = "This device does not support AR"; + exception = e; + } catch (Exception e) { + message = "Failed to create AR session"; + exception = e; + } + + if (message != null) { + messageSnackbarHelper.showError(this, message); + Log.e(TAG, "Exception creating session", exception); + return; + } + } + + // Note that order matters - see the note in onPause(), the reverse applies here. + try { + session.resume(); + } catch (CameraNotAvailableException e) { + messageSnackbarHelper.showError(this, "Camera not available. Please restart the app."); + session = null; + return; + } + + glSurfaceView.onResume(); + displayRotationHelper.onResume(); + messageSnackbarHelper.showMessage(this, "Searching for surfaces..."); + } + + @Override + public void onPause() { + super.onPause(); + if (session != null) { + displayRotationHelper.onPause(); + glSurfaceView.onPause(); + session.pause(); + } + } + + @Override + public void onRequestPermissionsResult(int requestCode, String[] permissions, int[] results) { + if (!CameraPermissionHelper.hasCameraPermission(this)) { + Toast.makeText(this, "Camera permission is needed to run this application", Toast.LENGTH_LONG) + .show(); + if (!CameraPermissionHelper.shouldShowRequestPermissionRationale(this)) { + // Permission denied with checking "Do not ask again". + CameraPermissionHelper.launchPermissionSettings(this); + } + finish(); + } + } + + @Override + public void onWindowFocusChanged(boolean hasFocus) { + super.onWindowFocusChanged(hasFocus); + FullScreenHelper.setFullScreenOnWindowFocusChanged(this, hasFocus); + } + + /************** GLSurfaceView Methods ****************************/ + @Override + public void onSurfaceCreated(GL10 gl, EGLConfig config) { + GLES20.glClearColor(0.1f, 0.1f, 0.1f, 1.0f); + + // Prepare the rendering objects. This involves reading shaders, so may throw an IOException. + try { + // Create the texture and pass it to ARCore session to be filled during update(). + backgroundRenderer.createOnGlThread(/*context=*/ this); + planeRenderer.createOnGlThread(/*context=*/ this, "models/trigrid.png"); + pointCloudRenderer.createOnGlThread(/*context=*/ this); + } catch (IOException e) { + Log.e(TAG, "Failed to read an asset file", e); + } + } + + @Override + public void onSurfaceChanged(GL10 gl, int width, int height) { + displayRotationHelper.onSurfaceChanged(width, height); + GLES20.glViewport(0, 0, width, height); + + // Send viewport information to 2D AR drawing manager + drawManager.updateViewportMatrix(width, height); + } + + @Override + public void onDrawFrame(GL10 gl) { + // Clear screen to notify driver it should not load any pixels from previous frame. + GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT | GLES20.GL_DEPTH_BUFFER_BIT); + + if (session == null) { + return; + } + // Notify ARCore session that the view size changed so that the perspective matrix and + // the video background can be properly adjusted. + displayRotationHelper.updateSessionIfNeeded(session); + + try { + session.setCameraTextureName(backgroundRenderer.getTextureId()); + Frame frame = session.update(); + Camera camera = frame.getCamera(); + + MotionEvent tap = tapHelper.poll(); + if (tap != null && camera.getTrackingState() == TrackingState.TRACKING) { + for (HitResult hit : frame.hitTest(tap)) { + // Check if any plane was hit, and if it was hit inside the plane polygon + Trackable trackable = hit.getTrackable(); + // Creates an anchor if a plane or an oriented point was hit. + if ((trackable instanceof Plane + && ((Plane) trackable).isPoseInPolygon(hit.getHitPose()) + && (PlaneRenderer.calculateDistanceToPlane(hit.getHitPose(), camera.getPose()) + > 0)) + || (trackable instanceof Point + && ((Point) trackable).getOrientationMode() + == OrientationMode.ESTIMATED_SURFACE_NORMAL)) { + if (anchors.size() >= 20) { + anchors.get(0).detach(); + anchors.remove(0); + } + anchors.add(hit.createAnchor()); + break; + } + } + } + + // Draw background. + backgroundRenderer.draw(frame); + + // If not tracking, don't draw objects + if (camera.getTrackingState() == TrackingState.PAUSED) { + return; + } + + // Get projection matrix. + float[] projmtx = new float[16]; + camera.getProjectionMatrix(projmtx, 0, 0.1f, 100.0f); + drawManager.updateProjectionMatrix(projmtx); + + // Get camera matrix and draw. + float[] viewmtx = new float[16]; + camera.getViewMatrix(viewmtx, 0); + drawManager.updateViewMatrix(viewmtx); + + final float[] colorCorrectionRgba = new float[4]; + frame.getLightEstimate().getColorCorrection(colorCorrectionRgba, 0); + drawManager.updateLightColorFilter(colorCorrectionRgba); + + PointCloud pointCloud = frame.acquirePointCloud(); + pointCloudRenderer.update(pointCloud); + pointCloudRenderer.draw(viewmtx, projmtx); + pointCloud.release(); + + // Check if we detected at least one plane. If so, hide the loading message. + if (messageSnackbarHelper.isShowing()) { + for (Plane plane : session.getAllTrackables(Plane.class)) { + if (plane.getType() == com.google.ar.core.Plane.Type.HORIZONTAL_UPWARD_FACING + && plane.getTrackingState() == TrackingState.TRACKING) { + messageSnackbarHelper.hide(this); + break; + } + } + } + // Visualize planes. + planeRenderer.drawPlanes( + session.getAllTrackables(Plane.class), camera.getDisplayOrientedPose(), projmtx); + + // Draw models using Canvas + if (arSurfaceView.isRunning()) { + drawModels(); + } + + + } catch (Throwable t) { + // Avoid crashing the application due to unhandled exceptions. + Log.e(TAG, "Exception on the OpenGL thread", t); + } + } + + private void drawModels() { + SurfaceHolder holder = arSurfaceView.getHolder(); + Canvas canvas = holder.lockHardwareCanvas(); + canvas.drawColor(Color.TRANSPARENT, PorterDuff.Mode.CLEAR); + for (Anchor anchor : anchors) { + if (anchor.getTrackingState() != TrackingState.TRACKING) { + continue; + } + // Get the current pose of an Anchor in world space. The Anchor pose is updated + // during calls to session.update() as ARCore refines its estimate of the world. + anchor.getPose().toMatrix(anchorMatrix, 0); + drawManager.modelMatrices.add(0, anchorMatrix); + + drawManager.drawRect(canvas); + drawManager.drawCircle(canvas); + drawManager.drawText(canvas, "HelloSkAR"); + } + holder.unlockCanvasAndPost(canvas); + + } +} diff --git a/platform_tools/android/apps/skar_java/src/main/java/com/google/skar/SkARMatrix.java b/platform_tools/android/apps/skar_java/src/main/java/com/google/skar/SkARMatrix.java new file mode 100644 index 0000000000..fc9333c6df --- /dev/null +++ b/platform_tools/android/apps/skar_java/src/main/java/com/google/skar/SkARMatrix.java @@ -0,0 +1,211 @@ +package com.google.skar; + +import android.graphics.Matrix; + +/** + * Provides static methods for matrix manipulation. Input matrices are assumed to be 4x4 + * android.opengl.Matrix types. Output matrices are 3x3 android.graphics.Matrix types. + * The main use of this class is to be able to get a Matrix for a Canvas that applies perspective + * to 2D objects + */ + +public class SkARMatrix { + /** + * Returns an android.graphics.Matrix that can be used on a Canvas to draw a 2D object in + * perspective. Objects will be rotated towards the XZ plane. Undefined behavior when any of + * the matrices are not of size 16, or are null. + * + * @param model 4x4 model matrix of the object to be drawn (global/world) + * @param view 4x4 camera view matrix (brings objects to camera origin and orientation) + * @param projection 4x4 projection matrix + * @param viewport 4x4 viewport matrix + * @return 3x3 matrix that puts a 2D objects in perspective on a Canvas + */ + + public static Matrix createPerspectiveMatrix(float[] model, float[] view, float[] projection, + float[] viewport) { + float[] skiaRotation = createXYtoXZRotationMatrix(); + float[][] matrices = {skiaRotation, model, view, projection, viewport}; + return createMatrixFrom4x4Array(matrices); + } + + /** + * Returns an android.graphics.Matrix that can be used on a Canvas to draw a 2D object in + * perspective. Undefined behavior when any of the matrices are not of size 16, or are null. + * + * @param model 4x4 model matrix of the object to be drawn (global/world) + * @param view 4x4 camera view matrix (brings objects to camera origin and orientation) + * @param projection 4x4 projection matrix + * @param viewport 4x4 viewport matrix + * @param rotatePlane specifies if object should be from the XY plane to the XZ plane + * @return 3x3 matrix that puts a 2D objects in perspective on a Canvas + */ + + public static Matrix createPerspectiveMatrix(float[] model, float[] view, float[] projection, + float[] viewport, boolean rotatePlane) { + if (rotatePlane) { + return createPerspectiveMatrix(model, view, projection, viewport); + } else { + float[][] matrices = {model, view, projection, viewport}; + return createMatrixFrom4x4Array(matrices); + } + } + + /** + * Returns an android.graphics.Matrix that can be used on a Canvas to draw a 2D object in + * perspective. Undefined behavior when any of the matrices are not of size 16, or are null. + * + * @param model 4x4 model matrix of the object to be drawn (global/world) + * @param view 4x4 camera view matrix (brings objects to camera origin and orientation) + * @param projection 4x4 projection matrix + * @param viewPortWidth width of viewport of GLSurfaceView + * @param viewPortHeight height of viewport of GLSurfaceView + * @param rotatePlane specifies if object should be from the XY plane to the XZ plane + * @return 3x3 matrix that puts a 2D objects in perspective on a Canvas + */ + + public static Matrix createPerspectiveMatrix(float[] model, float[] view, float[] projection, + float viewPortWidth, float viewPortHeight, boolean rotatePlane) { + if (rotatePlane) { + return createPerspectiveMatrix(model, view, projection, viewPortWidth, viewPortHeight); + } else { + float[] viewPort = createViewportMatrix(viewPortWidth, viewPortHeight); + float[][] matrices = {model, view, projection, viewPort}; + return createMatrixFrom4x4Array(matrices); + } + } + + /** + * Returns an android.graphics.Matrix that can be used on a Canvas to draw a 2D object in + * perspective. Object will be rotated towards the XZ plane. Undefined behavior when any of + * the matrices are not of size 16, or are null. + * + * @param model 4x4 model matrix of the object to be drawn (global/world) + * @param view 4x4 camera view matrix (brings objects to camera origin and orientation) + * @param projection 4x4 projection matrix + * @param viewPortWidth width of viewport of GLSurfaceView + * @param viewPortHeight height of viewport of GLSurfaceView + * @return 3x3 matrix that puts a 2D objects in perspective on a Canvas + */ + + public static Matrix createPerspectiveMatrix(float[] model, float[] view, float[] projection, + float viewPortWidth, float viewPortHeight) { + float[] viewPort = createViewportMatrix(viewPortWidth, viewPortHeight); + float[] skiaRotation = createXYtoXZRotationMatrix(); + float[][] matrices = {skiaRotation, model, view, projection, viewPort}; + return createMatrixFrom4x4Array(matrices); + } + + /** + * Returns a 16-float matrix in column-major order that represents a viewport matrix given + * the width and height of the viewport. + * + * @param width width of viewport + * @param height height of viewport + */ + + public static float[] createViewportMatrix(float width, float height) { + float[] viewPort = new float[16]; + android.opengl.Matrix.setIdentityM(viewPort, 0); + android.opengl.Matrix.translateM(viewPort, 0, width / 2, height / 2, 0); + android.opengl.Matrix.scaleM(viewPort, 0, width / 2, -height / 2, 0); + return viewPort; + } + + /** + * Returns a 16-float matrix in column-major order that is used to rotate objects from the XY plane + * to the XZ plane. This is useful given that objects drawn on the Canvas are on the XY plane. + * In order to get objects to appear as if they are sticking on planes/ceilings/walls, we need + * to rotate them from the XY plane to the XZ plane. + */ + + private static float[] createXYtoXZRotationMatrix() { + float[] rotation = new float[16]; + android.opengl.Matrix.setIdentityM(rotation, 0); + android.opengl.Matrix.rotateM(rotation, 0, 90, 1, 0, 0); + return rotation; + } + + /** + * Returns an android.graphics.Matrix resulting from a 9-float matrix array in row-major order. + * Undefined behavior when the array is not of size 9 or is null. + * + * @param m3 9-float matrix array in row-major order + */ + + public static Matrix createMatrixFrom3x3(float[] m3) { + Matrix m = new Matrix(); + m.setValues(m3); + return m; + } + + /** + * Returns an android.graphics.Matrix resulting from a 16-float matrix array in column-major order + * Undefined behavior when the array is not of size 16 or is null. + * + * @param m4 + */ + + public static Matrix createMatrixFrom4x4(float[] m4) { + float[] m3 = matrix4x4ToMatrix3x3(m4); + return createMatrixFrom3x3(m3); + } + + /** + * Returns an android.graphics.Matrix resulting from the concatenation of 16-float matrices + * in column-major order from left to right. + * e.g: m4Array = {m1, m2, m3} --> returns m = m3 * m2 * m1 + * Undefined behavior when the array is empty, null, or contains arrays not of size 9 (or null) + * + * @param m4Array array of 16-float matrices in column-major order + */ + + public static Matrix createMatrixFrom4x4Array(float[][] m4Array) { + float[] result = multiplyMatrices4x4(m4Array); + return createMatrixFrom4x4(result); + } + + /** + * Returns a 9-float matrix in row-major order given a 16-float matrix in column-major order. + * This will drop the Z column and row. + * Undefined behavior when the array is not of size 9 or is null. + * + * @param m4 16-float matrix in column-major order + */ + + private static float[] matrix4x4ToMatrix3x3(float[] m4) { + float[] m3 = new float[9]; + + int j = 0; + for (int i = 0; i < 7; i = i + 3) { + if (j == 2) { + j++; //skip row #3 + } + m3[i] = m4[j]; + m3[i + 1] = m4[j + 4]; + m3[i + 2] = m4[j + 12]; + j++; + } + return m3; + } + + /** + * Returns a 16-float matrix in column-major order resulting from the multiplication of matrices. + * e.g: m4Array = {m1, m2, m3} --> returns m = m3 * m2 * m1 + * Undefined behavior when the array is empty, null, or contains arrays not of size 9 (or null) + * + * @param m4Array array of 16-float matrices in column-major order + */ + + private static float[] multiplyMatrices4x4(float[][] m4Array) { + float[] result = new float[16]; + android.opengl.Matrix.setIdentityM(result, 0); + float[] rhs = result; + for (int i = 0; i < m4Array.length; i++) { + float[] lhs = m4Array[i]; + android.opengl.Matrix.multiplyMM(result, 0, lhs, 0, rhs, 0); + rhs = result; + } + return result; + } +} diff --git a/platform_tools/android/apps/skar_java/src/main/java/com/google/skar/SkARUtil.java b/platform_tools/android/apps/skar_java/src/main/java/com/google/skar/SkARUtil.java new file mode 100644 index 0000000000..42a221af86 --- /dev/null +++ b/platform_tools/android/apps/skar_java/src/main/java/com/google/skar/SkARUtil.java @@ -0,0 +1,31 @@ +package com.google.skar; + +import android.graphics.ColorFilter; +import android.graphics.ColorMatrix; +import android.graphics.ColorMatrixColorFilter; + +import java.util.Arrays; + +public class SkARUtil { + + private static final float MIDDLE_GRAY_GAMMA = 0.466f; + + /** + * Returns a ColorFilter that can be used on a Paint to apply color correction effects + * as documented by ARCore in + * LightEstimate + * + * @param colorCorr output array of + * getColorCorrection() + * @return ColorFilter with effects applied + */ + public static ColorFilter createLightCorrectionColorFilter(float[] colorCorr) { + float[] colorCorrCopy = Arrays.copyOf(colorCorr, 4); + for (int i = 0; i < 3; i++) { + colorCorrCopy[i] *= colorCorrCopy[3] / MIDDLE_GRAY_GAMMA; + } + ColorMatrix m = new ColorMatrix(); + m.setScale(colorCorrCopy[0], colorCorrCopy[1], colorCorrCopy[2], 1); + return new ColorMatrixColorFilter(m); + } +} diff --git a/platform_tools/android/apps/skar_java/src/main/res/drawable-xxhdpi/ic_launcher.png b/platform_tools/android/apps/skar_java/src/main/res/drawable-xxhdpi/ic_launcher.png new file mode 100644 index 0000000000..3f691da039 Binary files /dev/null and b/platform_tools/android/apps/skar_java/src/main/res/drawable-xxhdpi/ic_launcher.png differ diff --git a/platform_tools/android/apps/skar_java/src/main/res/layout/activity_main.xml b/platform_tools/android/apps/skar_java/src/main/res/layout/activity_main.xml new file mode 100644 index 0000000000..2c46a04e91 --- /dev/null +++ b/platform_tools/android/apps/skar_java/src/main/res/layout/activity_main.xml @@ -0,0 +1,39 @@ + + + + + + + + + + + + + diff --git a/platform_tools/android/apps/skar_java/src/main/res/values/strings.xml b/platform_tools/android/apps/skar_java/src/main/res/values/strings.xml new file mode 100644 index 0000000000..ed91c5d4f0 --- /dev/null +++ b/platform_tools/android/apps/skar_java/src/main/res/values/strings.xml @@ -0,0 +1,18 @@ + + + HelloSkAR Java + diff --git a/platform_tools/android/apps/skar_java/src/main/res/values/styles.xml b/platform_tools/android/apps/skar_java/src/main/res/values/styles.xml new file mode 100644 index 0000000000..59cf7e9ffb --- /dev/null +++ b/platform_tools/android/apps/skar_java/src/main/res/values/styles.xml @@ -0,0 +1,35 @@ + + + + + + + + + + -- cgit v1.2.3