From 6426c4efd825e80579c478a61f66579c1c3bcef1 Mon Sep 17 00:00:00 2001 From: Andrew Harp Date: Wed, 11 Jan 2017 14:05:52 -0800 Subject: Android: add image stylization example demo based on "A Learned Representation For Artistic Style" Change: 144247143 --- WORKSPACE | 7 + tensorflow/examples/android/AndroidManifest.xml | 11 +- tensorflow/examples/android/BUILD | 1 + tensorflow/examples/android/README.md | 4 + .../examples/android/bin/AndroidManifest.xml | 64 ++ .../layout/camera_connection_fragment_stylize.xml | 51 ++ .../examples/android/res/values/base-strings.xml | 1 + .../src/org/tensorflow/demo/StylizeActivity.java | 662 +++++++++++++++++++++ 8 files changed, 800 insertions(+), 1 deletion(-) create mode 100644 tensorflow/examples/android/bin/AndroidManifest.xml create mode 100644 tensorflow/examples/android/res/layout/camera_connection_fragment_stylize.xml create mode 100644 tensorflow/examples/android/src/org/tensorflow/demo/StylizeActivity.java diff --git a/WORKSPACE b/WORKSPACE index 9d3622878c..e0931512f4 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -38,6 +38,13 @@ new_http_archive( sha256 = "b4c178fd6236dcf0a20d25d07c45eebe85281263978c6a6f1dfc49d75befc45f" ) +new_http_archive( + name = "stylize", + build_file = "models.BUILD", + url = "https://storage.googleapis.com/download.tensorflow.org/models/stylize_v1.zip", + sha256 = "3d374a730aef330424a356a8d4f04d8a54277c425e274ecb7d9c83aa912c6bfa" +) + # TENSORBOARD_BOWER_AUTOGENERATED_BELOW_THIS_LINE_DO_NOT_EDIT new_http_archive( diff --git a/tensorflow/examples/android/AndroidManifest.xml b/tensorflow/examples/android/AndroidManifest.xml index e388734564..9f229d8b9d 100644 --- a/tensorflow/examples/android/AndroidManifest.xml +++ b/tensorflow/examples/android/AndroidManifest.xml @@ -41,7 +41,7 @@ - + @@ -50,6 +50,15 @@ + + + + + + + diff --git a/tensorflow/examples/android/BUILD b/tensorflow/examples/android/BUILD index 3ba3a494ab..0c1cea5fc3 100644 --- a/tensorflow/examples/android/BUILD +++ b/tensorflow/examples/android/BUILD @@ -66,6 +66,7 @@ android_binary( "//tensorflow/examples/android/assets:asset_files", "@inception5h//:model_files", "@mobile_multibox//:model_files", + "@stylize//:model_files", ], assets_dir = "", custom_package = "org.tensorflow.demo", diff --git a/tensorflow/examples/android/README.md b/tensorflow/examples/android/README.md index 79f543fb74..fbbe9f276b 100644 --- a/tensorflow/examples/android/README.md +++ b/tensorflow/examples/android/README.md @@ -22,6 +22,10 @@ existing application. Demonstrates a model based on [Scalable Object Detection using Deep Neural Networks](https://arxiv.org/abs/1312.2249) to localize and track people in the camera preview in real-time. +3. [TF Stylize](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/android/src/org/tensorflow/demo/StylizeActivity.java): + Uses a model based on [A Learned Representation For Artistic Style] + (https://arxiv.org/abs/1610.07629) to restyle the camera preview image + to that of a number of different artists. ## Prebuilt APK: diff --git a/tensorflow/examples/android/bin/AndroidManifest.xml b/tensorflow/examples/android/bin/AndroidManifest.xml new file mode 100644 index 0000000000..d4792bc482 --- /dev/null +++ b/tensorflow/examples/android/bin/AndroidManifest.xml @@ -0,0 +1,64 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tensorflow/examples/android/res/layout/camera_connection_fragment_stylize.xml b/tensorflow/examples/android/res/layout/camera_connection_fragment_stylize.xml new file mode 100644 index 0000000000..1cdb24cab0 --- /dev/null +++ b/tensorflow/examples/android/res/layout/camera_connection_fragment_stylize.xml @@ -0,0 +1,51 @@ + + + + + + + + + + + + diff --git a/tensorflow/examples/android/res/values/base-strings.xml b/tensorflow/examples/android/res/values/base-strings.xml index f6c57d5030..56edb55def 100644 --- a/tensorflow/examples/android/res/values/base-strings.xml +++ b/tensorflow/examples/android/res/values/base-strings.xml @@ -19,4 +19,5 @@ TensorFlow Demo TF Classify TF Detect + TF Stylize diff --git a/tensorflow/examples/android/src/org/tensorflow/demo/StylizeActivity.java b/tensorflow/examples/android/src/org/tensorflow/demo/StylizeActivity.java new file mode 100644 index 0000000000..8a3c7a4ef9 --- /dev/null +++ b/tensorflow/examples/android/src/org/tensorflow/demo/StylizeActivity.java @@ -0,0 +1,662 @@ +/* + * Copyright 2017 The TensorFlow Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.tensorflow.demo; + +import android.content.Context; +import android.content.res.AssetManager; +import android.graphics.Bitmap; +import android.graphics.Bitmap.Config; +import android.graphics.BitmapFactory; +import android.graphics.Canvas; +import android.graphics.Color; +import android.graphics.Matrix; +import android.graphics.Paint; +import android.graphics.Paint.Style; +import android.graphics.Rect; +import android.graphics.Typeface; +import android.media.Image; +import android.media.Image.Plane; +import android.media.ImageReader; +import android.media.ImageReader.OnImageAvailableListener; +import android.os.Bundle; +import android.os.SystemClock; +import android.os.Trace; +import android.util.Size; +import android.util.TypedValue; +import android.view.Display; +import android.view.MotionEvent; +import android.view.View; +import android.view.View.OnClickListener; +import android.view.View.OnTouchListener; +import android.view.ViewGroup; +import android.widget.BaseAdapter; +import android.widget.Button; +import android.widget.GridView; +import android.widget.ImageView; +import android.widget.Toast; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Vector; +import org.tensorflow.contrib.android.TensorFlowInferenceInterface; +import org.tensorflow.demo.OverlayView.DrawCallback; +import org.tensorflow.demo.env.BorderedText; +import org.tensorflow.demo.env.ImageUtils; +import org.tensorflow.demo.env.Logger; +import org.tensorflow.demo.R; + +/** + * Sample activity that stylizes the camera preview according to "A Learned Representation For + * Artistic Style" (https://arxiv.org/abs/1610.07629) + */ +public class StylizeActivity extends CameraActivity implements OnImageAvailableListener { + static { + System.loadLibrary("tensorflow_demo"); + } + + private static final Logger LOGGER = new Logger(); + + private static final String MODEL_FILE = "file:///android_asset/stylize_quantized.pb"; + private static final String INPUT_NODE = "input:0"; + private static final String STYLE_NODE = "style_num:0"; + private static final String OUTPUT_NODE = "transformer/expand/conv3/conv/Sigmoid"; + private static final int NUM_STYLES = 26; + + private static final boolean SAVE_PREVIEW_BITMAP = false; + + // Whether to actively manipulate non-selected sliders so that sum of activations always appears + // to be 1.0. The actual style input tensor will be normalized to sum to 1.0 regardless. + private static final boolean NORMALIZE_SLIDERS = true; + + private static final float TEXT_SIZE_DIP = 12; + + private static final boolean DEBUG_MODEL = false; + + private static final int[] SIZES = {32, 48, 64, 96, 128, 192, 256, 384, 512, 768, 1024}; + + // Start at a medium size, but let the user step up through smaller sizes so they don't get + // immediately stuck processing a large image. + private int desiredSizeIndex = -1; + private int desiredSize = 256; + private int initializedSize = 0; + + private Integer sensorOrientation; + + private int previewWidth = 0; + private int previewHeight = 0; + private byte[][] yuvBytes; + private int[] rgbBytes = null; + private Bitmap rgbFrameBitmap = null; + private Bitmap croppedBitmap = null; + + private final float[] styleVals = new float[NUM_STYLES]; + private int[] intValues; + private float[] floatValues; + + private int frameNum = 0; + + private Bitmap cropCopyBitmap; + private Bitmap textureCopyBitmap; + + private boolean computing = false; + + private Matrix frameToCropTransform; + private Matrix cropToFrameTransform; + + private BorderedText borderedText; + + private long lastProcessingTimeMs; + + private TensorFlowInferenceInterface inferenceInterface; + + private int lastOtherStyle = 1; + + private boolean allZero = false; + + private ImageGridAdapter adapter; + private GridView grid; + + private final OnTouchListener gridTouchAdapter = + new OnTouchListener() { + ImageSlider slider = null; + + @Override + public boolean onTouch(final View v, final MotionEvent event) { + switch (event.getActionMasked()) { + case MotionEvent.ACTION_DOWN: + for (int i = 0; i < NUM_STYLES; ++i) { + final ImageSlider child = adapter.items[i]; + final Rect rect = new Rect(); + child.getHitRect(rect); + if (rect.contains((int) event.getX(), (int) event.getY())) { + slider = child; + slider.setHilighted(true); + } + } + break; + + case MotionEvent.ACTION_MOVE: + if (slider != null) { + final Rect rect = new Rect(); + slider.getHitRect(rect); + + final float newSliderVal = + (float) + Math.min( + 1.0, + Math.max( + 0.0, 1.0 - (event.getY() - slider.getTop()) / slider.getHeight())); + + setStyle(slider, newSliderVal); + } + break; + + case MotionEvent.ACTION_UP: + if (slider != null) { + slider.setHilighted(false); + slider = null; + } + break; + } + return true; + } + }; + + @Override + public void onCreate(final Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + } + + @Override + protected int getLayoutId() { + return R.layout.camera_connection_fragment_stylize; + } + + @Override + protected int getDesiredPreviewFrameSize() { + return SIZES[SIZES.length - 1]; + } + + public static Bitmap getBitmapFromAsset(final Context context, final String filePath) { + final AssetManager assetManager = context.getAssets(); + + Bitmap bitmap = null; + try { + final InputStream inputStream = assetManager.open(filePath); + bitmap = BitmapFactory.decodeStream(inputStream); + } catch (final IOException e) { + LOGGER.e("Error opening bitmap!", e); + } + + return bitmap; + } + + private class ImageSlider extends ImageView { + private float value = 0.0f; + private boolean hilighted = false; + + private final Paint boxPaint; + private final Paint linePaint; + + public ImageSlider(final Context context) { + super(context); + value = 0.0f; + + boxPaint = new Paint(); + boxPaint.setColor(Color.BLACK); + boxPaint.setAlpha(128); + + linePaint = new Paint(); + linePaint.setColor(Color.WHITE); + linePaint.setStrokeWidth(10.0f); + linePaint.setStyle(Style.STROKE); + } + + @Override + public void onDraw(final Canvas canvas) { + super.onDraw(canvas); + final float y = (1.0f - value) * canvas.getHeight(); + + // If all sliders are zero, don't bother shading anything. + if (!allZero) { + canvas.drawRect(0, 0, canvas.getWidth(), y, boxPaint); + } + + if (value > 0.0f) { + canvas.drawLine(0, y, canvas.getWidth(), y, linePaint); + } + + if (hilighted) { + canvas.drawRect(0, 0, getWidth(), getHeight(), linePaint); + } + } + + @Override + protected void onMeasure(final int widthMeasureSpec, final int heightMeasureSpec) { + super.onMeasure(widthMeasureSpec, heightMeasureSpec); + setMeasuredDimension(getMeasuredWidth(), getMeasuredWidth()); + } + + public void setValue(final float value) { + this.value = value; + postInvalidate(); + } + + public void setHilighted(final boolean highlighted) { + this.hilighted = highlighted; + this.postInvalidate(); + } + } + + private class ImageGridAdapter extends BaseAdapter { + final ImageSlider[] items = new ImageSlider[NUM_STYLES]; + final ArrayList