SkAR-Java: finger painting first implementation
SkAR Java: drawing planes as paths
FREEZE.unindexed
Bug: skia:
Change-Id: I2a7a8534fc1c35c4b8d6054bc1d6e682ffabc47a
Reviewed-on: https://skia-review.googlesource.com/141827
Reviewed-by: Mike Reed <reed@google.com>
diff --git a/platform_tools/android/apps/skar_java/src/main/AndroidManifest.xml b/platform_tools/android/apps/skar_java/src/main/AndroidManifest.xml
index 6f93cbf..3ee3cd9 100644
--- a/platform_tools/android/apps/skar_java/src/main/AndroidManifest.xml
+++ b/platform_tools/android/apps/skar_java/src/main/AndroidManifest.xml
@@ -46,5 +46,9 @@
<!-- This tag indicates that this application requires ARCore. This results in the Google Play
Store downloading and installing ARCore along with the application. -->
<meta-data android:name="com.google.ar.core" android:value="required" />
+ <meta-data
+ android:name="com.google.ar.core.min_apk_version"
+ tools:replace="android:value"
+ android:value="180226000" />
</application>
</manifest>
diff --git a/platform_tools/android/apps/skar_java/src/main/java/com/google/ar/core/examples/java/helloskar/DrawManager.java b/platform_tools/android/apps/skar_java/src/main/java/com/google/ar/core/examples/java/helloskar/DrawManager.java
index 7e5ac29..020df5e 100644
--- a/platform_tools/android/apps/skar_java/src/main/java/com/google/ar/core/examples/java/helloskar/DrawManager.java
+++ b/platform_tools/android/apps/skar_java/src/main/java/com/google/ar/core/examples/java/helloskar/DrawManager.java
@@ -1,12 +1,14 @@
package com.google.ar.core.examples.java.helloskar;
import android.content.Context;
+import android.content.pm.ApplicationInfo;
import android.graphics.Bitmap;
import android.graphics.BitmapFactory;
import android.graphics.BitmapShader;
import android.graphics.Canvas;
import android.graphics.Color;
import android.graphics.ColorFilter;
+import android.graphics.LinearGradient;
import android.graphics.Paint;
import android.graphics.Path;
import android.graphics.PointF;
@@ -15,10 +17,13 @@
import android.graphics.RectF;
import android.graphics.Shader;
import android.opengl.Matrix;
+import android.os.Build;
+
import com.google.ar.core.Plane;
import com.google.ar.core.PointCloud;
import com.google.ar.core.Pose;
import com.google.ar.core.TrackingState;
+import com.google.skar.SkARFingerPainting;
import com.google.skar.SkARMatrix;
import com.google.skar.SkARUtil;
import java.io.IOException;
@@ -40,6 +45,7 @@
private ColorFilter lightFilter;
private BitmapShader planeShader;
public ArrayList<float[]> modelMatrices = new ArrayList<>();
+ public SkARFingerPainting fingerPainting = new SkARFingerPainting();
public void updateViewport(float width, float height) {
viewportWidth = width;
@@ -58,6 +64,10 @@
lightFilter = SkARUtil.createLightCorrectionColorFilter(colorCorr);
}
+ public void updateFingerPainting(PointF p) {
+ fingerPainting.addPoint(p);
+ }
+
// Sample function for drawing a circle
public void drawCircle(Canvas canvas) {
if (modelMatrices.isEmpty()) {
@@ -129,6 +139,41 @@
canvas.restore();
}
+ public void drawFingerPainting(Canvas canvas) {
+ if (fingerPainting.path.isEmpty()) {
+ return;
+ }
+
+ // Get finger painting model matrix
+ float[] m = fingerPainting.getModelMatrix();
+ float[] model = new float[16];
+ Matrix.setIdentityM(model, 0);
+ Matrix.translateM(model, 0, m[12], m[13], m[14]);
+
+ float[] initRot = SkARMatrix.createXYtoXZRotationMatrix();
+
+ // Matrix = mvpv
+ float[][] matrices = {initRot, model, viewMatrix, projectionMatrix, SkARMatrix.createViewportMatrix(viewportWidth, viewportHeight)};
+ android.graphics.Matrix mvpv = SkARMatrix.createMatrixFrom4x4(SkARMatrix.multiplyMatrices4x4(matrices));
+
+ // Set up paint
+ Paint p = new Paint();
+ p.setColor(Color.GREEN);
+ p.setStyle(Paint.Style.STROKE);
+ p.setStrokeWidth(10f);
+ p.setAlpha(120);
+
+ // Build destination path by transforming source path
+ Path pathDst = new Path();
+ fingerPainting.path.transform(mvpv, pathDst);
+
+ // Draw dest path
+ canvas.save();
+ canvas.setMatrix(new android.graphics.Matrix());
+ canvas.drawPath(pathDst, p);
+ canvas.restore();
+ }
+
// Sample function for drawing the AR point cloud
public void drawPointCloud(Canvas canvas, PointCloud cloud) {
FloatBuffer points = cloud.getPoints();
@@ -224,7 +269,7 @@
// Shader local matrix
android.graphics.Matrix lm = new android.graphics.Matrix();
- lm.setScale(0.0005f, 0.0005f);
+ lm.setScale(0.00005f, 0.00005f);
lm.postConcat(mvpv);
planeShader.setLocalMatrix(lm);
diff --git a/platform_tools/android/apps/skar_java/src/main/java/com/google/ar/core/examples/java/helloskar/HelloSkARActivity.java b/platform_tools/android/apps/skar_java/src/main/java/com/google/ar/core/examples/java/helloskar/HelloSkARActivity.java
index 7bd4085..b625136 100644
--- a/platform_tools/android/apps/skar_java/src/main/java/com/google/ar/core/examples/java/helloskar/HelloSkARActivity.java
+++ b/platform_tools/android/apps/skar_java/src/main/java/com/google/ar/core/examples/java/helloskar/HelloSkARActivity.java
@@ -20,14 +20,17 @@
import android.animation.ValueAnimator;
import android.graphics.Canvas;
import android.graphics.Color;
+import android.graphics.PointF;
import android.graphics.PorterDuff;
import android.opengl.GLES20;
import android.opengl.GLSurfaceView;
+import android.opengl.Matrix;
import android.os.Bundle;
import android.support.v7.app.AppCompatActivity;
import android.util.Log;
import android.view.MotionEvent;
import android.view.SurfaceHolder;
+import android.view.View;
import android.widget.Toast;
import com.google.ar.core.Anchor;
@@ -54,6 +57,7 @@
import com.google.ar.core.exceptions.UnavailableDeviceNotCompatibleException;
import com.google.ar.core.exceptions.UnavailableSdkTooOldException;
import com.google.ar.core.exceptions.UnavailableUserDeclinedInstallationException;
+import com.google.skar.SkARMatrix;
import java.io.IOException;
import java.util.ArrayList;
@@ -72,6 +76,8 @@
//Simple SurfaceView used to draw 2D objects on top of the GLSurfaceView
private ARSurfaceView arSurfaceView;
+ private Canvas canvas;
+ private SurfaceHolder holder;
//GLSurfaceView used to draw 3D objects & camera input
private GLSurfaceView glSurfaceView;
@@ -93,6 +99,11 @@
// Temporary matrix allocated here to reduce number of allocations for each frame.
private final float[] anchorMatrix = new float[16];
+ private final float[] back = new float[16];
+
+ PointF previousEvent;
+ android.graphics.Matrix inverted;
+
// Anchors created from taps used for object placing.
private final ArrayList<Anchor> anchors = new ArrayList<>();
@@ -108,6 +119,7 @@
arSurfaceView = findViewById(R.id.arsurfaceview);
glSurfaceView = findViewById(R.id.glsurfaceview);
arSurfaceView.bringToFront();
+ arSurfaceView.setLayerType(View.LAYER_TYPE_HARDWARE, null);
displayRotationHelper = new DisplayRotationHelper(/*context=*/ this);
// Set up tap listener.
@@ -259,6 +271,9 @@
@Override
public void onDrawFrame(GL10 gl) {
+ canvas = null;
+ holder = null;
+
// Clear screen to notify driver it should not load any pixels from previous frame.
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT | GLES20.GL_DEPTH_BUFFER_BIT);
@@ -269,6 +284,7 @@
// the video background can be properly adjusted.
displayRotationHelper.updateSessionIfNeeded(session);
+
try {
session.setCameraTextureName(backgroundRenderer.getTextureId());
Frame frame = session.update();
@@ -297,6 +313,54 @@
}
}
+ MotionEvent holdTap = tapHelper.holdPoll();
+ if (holdTap != null && camera.getTrackingState() == TrackingState.TRACKING) {
+ for (HitResult hit : frame.hitTest(holdTap)) {
+ // Check if any plane was hit, and if it was hit inside the plane polygon
+ Trackable trackable = hit.getTrackable();
+ // Creates an anchor if a plane or an oriented point was hit.
+ if ((trackable instanceof Plane
+ && ((Plane) trackable).isPoseInPolygon(hit.getHitPose())
+ && (DrawManager.calculateDistanceToPlane(hit.getHitPose(), camera.getPose())
+ > 0))
+ || (trackable instanceof Point
+ && ((Point) trackable).getOrientationMode()
+ == OrientationMode.ESTIMATED_SURFACE_NORMAL)) {
+
+ float[] pt = {hit.getHitPose().tx(), hit.getHitPose().tz()};
+
+ if (drawManager.fingerPainting.isEmpty()) {
+ float[] originalPt = {pt[0], pt[1]};
+
+ // Get model matrix of first point
+ float[] m = new float[16];
+ hit.getHitPose().toMatrix(m, 0);
+ drawManager.fingerPainting.setModelMatrix(m); //M0
+
+ // Construct the inverse matrix + the translation to the origin
+ float[] inv = new float[16];
+ hit.getHitPose().toMatrix(inv, 0);
+ Matrix.invertM(inv, 0, inv, 0);
+ drawManager.fingerPainting.setInverseModelMatrix(inv);
+ //inverted = SkARMatrix.createMatrixFrom4x4(inv); //M0 -1
+
+ // Map hit location using the raw inverse matrix
+ drawManager.fingerPainting.getInverseModelMatrix().mapPoints(originalPt);
+
+ // Translate the point back to the origin, and update the inverse matrix
+ Matrix.translateM(inv, 0, -originalPt[0], -originalPt[1], 0);
+ drawManager.fingerPainting.setInverseModelMatrix(inv);
+ //inverted = SkARMatrix.createMatrixFrom4x4(inv);
+ }
+
+ drawManager.fingerPainting.getInverseModelMatrix().mapPoints(pt);
+ PointF newPoint = new PointF(pt[0], pt[1]);
+ drawManager.fingerPainting.addPoint(newPoint);
+ break;
+ }
+ }
+ }
+
// Draw background with OpenGL.
// TODO: possibly find a way to extract texture and draw on Canvas
backgroundRenderer.draw(frame);
@@ -350,12 +414,17 @@
// Draw models
drawModels(canvas);
+ // Draw finger painting
+ drawFingerPainting(canvas);
+
// Unlock canvas
holder.unlockCanvasAndPost(canvas);
}
-
} catch (Throwable t) {
// Avoid crashing the application due to unhandled exceptions.
+ if (holder != null && canvas != null) {
+ holder.unlockCanvasAndPost(canvas);
+ }
Log.e(TAG, "Exception on the OpenGL thread", t);
}
}
@@ -385,4 +454,8 @@
drawManager.drawText(canvas, "HelloSkAR");
}
}
+
+ private void drawFingerPainting(Canvas canvas) {
+ drawManager.drawFingerPainting(canvas);
+ }
}
diff --git a/platform_tools/android/apps/skar_java/src/main/java/com/google/skar/SkARFingerPainting.java b/platform_tools/android/apps/skar_java/src/main/java/com/google/skar/SkARFingerPainting.java
new file mode 100644
index 0000000..6d3d058
--- /dev/null
+++ b/platform_tools/android/apps/skar_java/src/main/java/com/google/skar/SkARFingerPainting.java
@@ -0,0 +1,56 @@
+package com.google.skar;
+
+import android.graphics.Matrix;
+import android.graphics.Path;
+import android.graphics.PointF;
+
+public class SkARFingerPainting { ;
+ public Path path = new Path();
+
+ private int numberOfPoints = 0;
+
+ // Holds the model matrix of the first point added to such that the path can be drawn at the
+ // model location (i.e on the Plane)
+ private float[] modelMatrix;
+
+ // Holds the inverse model matrix of the first point that was added such that the path is drawn
+ // first at (0, 0)
+ private float[] inverseModelMatrix;
+
+ public SkARFingerPainting() {}
+
+ // Adds another point to the path in Local space (i.e apply InverseModelMatrix to points located
+ // in Global space (e.g hit positions acquired through hit tests)
+ public void addPoint(PointF p) {
+ if (numberOfPoints == 0) {
+ path.moveTo(p.x, p.y);
+ } else {
+ path.lineTo(p.x, p.y);
+ }
+ numberOfPoints++;
+ }
+
+ public boolean isEmpty() {
+ return numberOfPoints == 0;
+ }
+
+ public float[] getModelMatrix() {
+ return modelMatrix;
+ }
+
+ public float[] getRawInverseModelMatrix() {
+ return inverseModelMatrix;
+ }
+
+ public Matrix getInverseModelMatrix() {
+ return SkARMatrix.createMatrixFrom4x4(inverseModelMatrix);
+ }
+
+ public void setModelMatrix(float[] m) {
+ modelMatrix = m;
+ }
+
+ public void setInverseModelMatrix(float[] m) {
+ inverseModelMatrix = m;
+ }
+}