Merge "Second Pass at Integration of ImageBackend" into ub-camera-haleakala
diff --git a/src/com/android/camera/processing/ImageBackend.java b/src/com/android/camera/processing/ImageBackend.java
new file mode 100644
index 0000000..04a0240
--- /dev/null
+++ b/src/com/android/camera/processing/ImageBackend.java
@@ -0,0 +1,613 @@
+/*
+ * Copyright (C) 2014 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.camera.processing;
+
+import com.android.camera.one.v2.camera2proxy.ImageProxy;
+import com.android.camera.processing.TaskImageContainer.ProcessingPriority;
+import com.android.camera.debug.Log;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.ReentrantLock;
+
+/**
+ * This ImageBackend is created for the purpose of creating a task-running infrastructure that has
+ * two-level of priority and doing the book-keeping to keep track of tasks that use Android Images.
+ * Android.media.images are critical system resources that MUST be properly managed in order to
+ * maintain camera application performance. Android.media.images are merely Java handles to regions
+ * of physically contiguous memory used by the camera hardware as a destination for imaging data. In
+ * general, this physically contiguous memory is not counted as an application resource, but as a
+ * system resources held by the application and does NOT count against the limits of application
+ * memory. The performance pressures of both computing and memory resources must often be
+ * prioritized in releasing Android.media.images in a timely manner. In order to properly balance
+ * these concerns, most image processing requested should be routed through this object. This object
+ * is also responsible for releasing Android.media image as soon as possible, so as not to stall the
+ * camera hardware subsystem. Image that reserve these images are a subclass of the basic Java
+ * Runnable with a few conditions placed upon their run() implementation:
+ * <ol>
+ * <li>The task will try to release the image as early as possible by calling the
+ * releaseSemaphoreReference as soon as a reference to the original image is no longer required.</li>
+ * <li>A set of tasks that require ImageData must only happen on the first receiveImage call.
+ * receiveImage must only be called once per image.</li>
+ * <li>However, the submitted tasks may spawn new tasks via the appendTask with any image that have
+ * had a task submitted, but NOT released via releaseSemaphoreReference.</li>
+ * <li>Computation that is dependent on multiple images should be written into this task framework
+ * in a distributed manner where image task can be computed independently and join their results to
+ * a common shared object.This style of implementation allows for the earliest release of Android
+ * Images while honoring the resources priorities set by this class. See the Lucky shot
+ * implementation for a concrete example for this shared object and its respective task
+ * {@link TaskLuckyShotSession} {@link LuckyShotSession}</li>
+ * </ol>
+ */
+public class ImageBackend implements ImageConsumer {
+
+ protected static final int FAST_THREAD_PRIORITY = Thread.MAX_PRIORITY;
+
+ protected static final int SLOW_THREAD_PRIORITY = Thread.NORM_PRIORITY;
+
+ protected static final int NUM_THREADS_FAST = 2;
+
+ protected static final int NUM_THREADS_SLOW = 2;
+
+ protected final SimpleCache mSimpleCache;
+
+ protected final Map<ImageProxy, ImageReleaseProtocol> mImageSemaphoreMap;
+
+ protected final ExecutorService mThreadPoolFast;
+
+ protected final ExecutorService mThreadPoolSlow;
+
+ private final static Log.Tag TAG = new Log.Tag("ImageBackend");
+
+ // Some invariants to know that we're keeping track of everything
+ // that reflect the state of mImageSemaphoreMap
+ private int mOutstandingImageRefs = 0;
+
+ private int mOutstandingImageOpened = 0;
+
+ private int mOutstandingImageClosed = 0;
+
+ // Objects that may be registered to this objects events.
+ private ImageProcessorProxyListener mProxyListener = null;
+
+ // Default constructor, values are conservatively targeted to the Nexus 6
+ public ImageBackend() {
+ mThreadPoolFast = Executors.newFixedThreadPool(NUM_THREADS_FAST, new FastThreadFactory());
+ mThreadPoolSlow = Executors.newFixedThreadPool(NUM_THREADS_SLOW, new SlowThreadFactory());
+ mProxyListener = new ImageProcessorProxyListener();
+ mImageSemaphoreMap = new HashMap<ImageProxy, ImageReleaseProtocol>();
+ mSimpleCache = new SimpleCache(NUM_THREADS_SLOW);
+ }
+
+ /**
+ * Direct Injection Constructor for Testing purposes.
+ *
+ * @param fastService Service where Tasks of FAST Priority are placed.
+ * @param slowService Service where Tasks of SLOW Priority are placed.
+ * @param imageProcessorProxyListener iamge proxy listener to be used
+ */
+ public ImageBackend(ExecutorService fastService, ExecutorService slowService,
+ ImageProcessorProxyListener imageProcessorProxyListener) {
+ mThreadPoolFast = fastService;
+ mThreadPoolSlow = slowService;
+ mProxyListener = imageProcessorProxyListener;
+ mImageSemaphoreMap = new HashMap<ImageProxy, ImageReleaseProtocol>();
+ mSimpleCache = new SimpleCache(NUM_THREADS_SLOW);
+ }
+
+ /**
+ * Simple getter for the simple cache functionality associated with this instantiation. Needs to
+ * be accessed by the tasks in order to get/return memory. TODO: Replace with something better.
+ *
+ * @return cache object that implements a simple memory pool for this object.
+ */
+ public SimpleCache getCache() {
+ return mSimpleCache;
+ }
+
+ /**
+ * Simple getting for the associated listener object associated with this instantiation that
+ * handles registration of events listeners.
+ *
+ * @return listener proxy that handles events messaging for this object.
+ */
+ public ImageProcessorProxyListener getProxyListener() {
+ return mProxyListener;
+ }
+
+ public void setCameraImageProcessorListener(ImageProcessorListener listener) {
+ this.mProxyListener.registerListener(listener, null);
+ }
+
+ /**
+ * Wrapper function for all log messages created by this object. Default implementation is to
+ * send messages to the Android logger. For test purposes, this method can be overridden to
+ * avoid "Stub!" Runtime exceptions in Unit Tests.
+ */
+ public void logWrapper(String message) {
+ Log.e(TAG, message);
+ }
+
+ /**
+ * @return Number of Image references currently held by this instance
+ */
+ @Override
+ public int numberOfReservedOpenImages() {
+ synchronized (mImageSemaphoreMap) {
+ // since mOutstandingImageOpened, mOutstandingImageClosed reflect
+ // the historical state of mImageSemaphoreMap, we need to lock on
+ // before we return a value.
+ return mOutstandingImageOpened - mOutstandingImageClosed;
+ }
+ }
+
+ /**
+ * Signals the ImageBackend that a tasks has released a reference to the image. Imagebackend
+ * determines whether all references have been released and applies its specified release
+ * protocol of closing image and/or unblocking the caller. Should ONLY be called by the tasks
+ * running on this class.
+ *
+ * @param img the image to be released by the task.
+ * @param executor the executor on which the image close is run. if null, image close is run by
+ * the calling thread (usually the main task thread).
+ */
+ public void releaseSemaphoreReference(final ImageProxy img, Executor executor) {
+ synchronized (mImageSemaphoreMap) {
+ ImageReleaseProtocol protocol = mImageSemaphoreMap.get(img);
+ if (protocol == null || protocol.getCount() <= 0) {
+ // That means task implementation has allowed an unbalanced
+ // semaphore release.
+ throw new RuntimeException(
+ "ERROR: Task implementation did NOT balance its release.");
+ }
+
+ // Normal operation from here.
+ protocol.addCount(-1);
+ mOutstandingImageRefs--;
+ logWrapper("Ref release. Total refs = " + mOutstandingImageRefs);
+ if (protocol.getCount() == 0) {
+ // Image is ready to be released
+ // Remove the image from the map so that it may be submitted
+ // again.
+ mImageSemaphoreMap.remove(img);
+
+ // Conditionally close the image, specified by initial
+ // receiveImage call
+ if (protocol.closeOnRelease) {
+ closeImageExecutorSafe(img, executor);
+ }
+
+ // Conditionally signal the blocking thread to go.
+ if (protocol.blockUntilRelease) {
+ protocol.signal();
+ }
+ } else {
+ // Image is still being held by other tasks.
+ // Otherwise, update the semaphore
+ mImageSemaphoreMap.put(img, protocol);
+ }
+ }
+ }
+
+ /**
+ * Spawns dependent tasks from internal implementation of a task. If a dependent task does NOT
+ * require the image reference, it should be passed a null pointer as an image reference. In
+ * general, this method should be called after the task has completed its own computations, but
+ * before it has released its own image reference (via the releaseSemaphoreReference call).
+ *
+ * @param tasks The set of tasks to be run
+ * @return whether tasks are successfully submitted.
+ */
+ public boolean appendTasks(ImageProxy img, Set<TaskImageContainer> tasks) {
+ // Make sure that referred images are all the same, if it exists.
+ // And count how image references need to be kept track of.
+ int countImageRefs = numPropagatedImageReferences(img, tasks);
+
+ if (img != null) {
+ // If you're still holding onto the reference, make sure you keep
+ // count
+ incrementSemaphoreReferenceCount(img, countImageRefs);
+ }
+
+ scheduleTasks(tasks);
+ return true;
+ }
+
+ /**
+ * Spawns a single dependent task from internal implementation of a task.
+ *
+ * @param task The task to be run
+ * @return whether tasks are successfully submitted.
+ */
+ public boolean appendTasks(ImageProxy img, TaskImageContainer task) {
+ Set<TaskImageContainer> tasks = new HashSet<TaskImageContainer>(1);
+ tasks.add(task);
+ return appendTasks(img, tasks);
+ }
+
+ /**
+ * Implements that top-level image single task submission that is defined by the ImageConsumer
+ * interface.
+ *
+ * @param img Image required by the task
+ * @param task Task to be run
+ * @param blockUntilImageRelease If true, call blocks until the object img is no longer referred
+ * by any task. If false, call is non-blocking
+ * @param closeOnImageRelease If true, images is closed when the object img is is no longer
+ * referred by any task. If false, After an image is submitted, it should never be
+ * submitted again to the interface until all tasks and their spawned tasks are
+ * finished.
+ */
+ @Override
+ public boolean receiveImage(ImageProxy img, TaskImageContainer task,
+ boolean blockUntilImageRelease, boolean closeOnImageRelease)
+ throws InterruptedException {
+ Set<TaskImageContainer> passTasks = new HashSet<TaskImageContainer>(1);
+ passTasks.add(task);
+ return receiveImage(img, passTasks, blockUntilImageRelease, closeOnImageRelease);
+ }
+
+ /**
+ * Implements that top-level image single task submission that is defined by the ImageConsumer
+ * interface.
+ *
+ * @param img Image required by the task
+ * @param tasks A set of Tasks to be run
+ * @param blockUntilImageRelease If true, call blocks until the object img is no longer referred
+ * by any task. If false, call is non-blocking
+ * @param closeOnImageRelease If true, images is closed when the object img is is no longer
+ * referred by any task. If false, After an image is submitted, it should never be
+ * submitted again to the interface until all tasks and their spawned tasks are
+ * finished.
+ * @return whether the blocking completed properly
+ */
+ @Override
+ public boolean receiveImage(ImageProxy img, Set<TaskImageContainer> tasks,
+ boolean blockUntilImageRelease, boolean closeOnImageRelease)
+ throws InterruptedException {
+
+ // Short circuit if no tasks submitted.
+ if (tasks == null || tasks.size() <= 0) {
+ return false;
+ }
+
+ if (img == null) {
+ // TODO: Determine whether you need to be so strict at the top level
+ throw new RuntimeException("ERROR: Initial call must reference valid Image!");
+ }
+
+ // Make sure that referred images are all the same, if it exists.
+ // And count how image references need to be kept track of.
+ int countImageRefs = numPropagatedImageReferences(img, tasks);
+
+ // Set the semaphore, given that the number of tasks that need to be
+ // scheduled
+ // and the boolean flags for imaging closing and thread blocking
+ ImageReleaseProtocol protocol = setSemaphoreReferenceCount(img, countImageRefs,
+ blockUntilImageRelease, closeOnImageRelease);
+
+ // Put the tasks on their respective queues.
+ scheduleTasks(tasks);
+
+ // Implement blocking if required
+ if (protocol.blockUntilRelease) {
+ protocol.block();
+ }
+
+ return true;
+ }
+
+ /**
+ * Implements that top-level image task submission short-cut that is defined by the
+ * ImageConsumer interface.
+ *
+ * @param img Image required by the task
+ * @param executor Executor to run events and image closes, in case of control leakage
+ * @param processingFlags Magical bit vector that specifies jobs to be run After an image is
+ * submitted, it should never be submitted again to the interface until all tasks and
+ * their spawned tasks are finished.
+ */
+ @Override
+ public boolean receiveImage(ImageProxy img, Executor executor,
+ Set<ImageTaskFlags> processingFlags) throws InterruptedException {
+
+ Set<TaskImageContainer> tasksToExecute = new HashSet<TaskImageContainer>();
+
+ if (img == null) {
+ // No data to process, just pure message.
+ return true;
+ }
+
+ // Now add the pre-mixed versions of the tasks.
+
+ if (processingFlags.contains(ImageTaskFlags.COMPRESS_IMAGE_TO_JPEG)
+ || processingFlags.contains(ImageTaskFlags.WRITE_IMAGE_TO_DISK)) {
+ // Add this type of task to the appropriate queue.
+ tasksToExecute.add(new TaskCompressImageToJpeg(img, executor, this));
+ }
+
+ if (processingFlags.contains(ImageTaskFlags.CONVERT_IMAGE_TO_RGB_PREVIEW)) {
+ // Add this type of task to the appropriate queue.
+ tasksToExecute.add(new TaskConvertImageToRGBPreview(img, executor, this, 160, 100));
+ }
+
+ if (processingFlags.contains(ImageTaskFlags.WRITE_IMAGE_TO_DISK)) {
+ // Add this type of task to the appropriate queue.
+ // Has a dependency as well on the result JPEG_COMPRESSION
+ // TODO: Put disk writing implementation within the framework.
+ }
+
+ receiveImage(img, tasksToExecute,
+ processingFlags.contains(ImageTaskFlags.BLOCK_UNTIL_IMAGE_RELEASE),
+ processingFlags.contains(ImageTaskFlags.CLOSE_IMAGE_ON_RELEASE));
+
+ return true;
+ }
+
+ /**
+ * Factory functions, in case, you want some shake and bake functionality.
+ */
+ public TaskConvertImageToRGBPreview createTaskConvertImageToRGBPreview(ImageProxy imageProxy,
+ Executor executor, ImageBackend imageBackend, int targetWidth, int targetHeight) {
+ return new TaskConvertImageToRGBPreview(imageProxy, executor, imageBackend, targetWidth,
+ targetHeight);
+ }
+
+ public TaskCompressImageToJpeg createTaskCompressImageToJpeg(ImageProxy imageProxy,
+ Executor executor, ImageBackend imageBackend) {
+ return new TaskCompressImageToJpeg(imageProxy, executor, imageBackend);
+ }
+
+ /**
+ * Blocks and waits for all tasks to complete.
+ */
+ @Override
+ public void shutdown() {
+ mThreadPoolSlow.shutdown();
+ mThreadPoolFast.shutdown();
+ }
+
+ /**
+ * Puts the tasks on the specified queue. May be more complicated in the future.
+ *
+ * @param tasks The set of tasks to be run
+ */
+ protected void scheduleTasks(Set<TaskImageContainer> tasks) {
+ for (TaskImageContainer task : tasks) {
+ if (task.getProcessingPriority() == ProcessingPriority.FAST) {
+ mThreadPoolFast.execute(task);
+ } else {
+ mThreadPoolSlow.execute(task);
+ }
+ }
+ }
+
+ /**
+ * Initializes the semaphore count for the image
+ *
+ * @return The protocol object that keeps tracks of the image reference count and actions to be
+ * taken on release.
+ */
+ protected ImageReleaseProtocol setSemaphoreReferenceCount(ImageProxy img, int count,
+ boolean blockUntilRelease, boolean closeOnRelease) throws RuntimeException {
+ synchronized (mImageSemaphoreMap) {
+ if (mImageSemaphoreMap.get(img) != null) {
+ throw new RuntimeException(
+ "ERROR: Rewriting of Semaphore Lock. Image references may not freed properly");
+ }
+
+ // Create the new booking-keeping object.
+ ImageReleaseProtocol protocol = new ImageReleaseProtocol(blockUntilRelease,
+ closeOnRelease);
+ protocol.count = count;
+
+ mImageSemaphoreMap.put(img, protocol);
+ mOutstandingImageRefs += count;
+ mOutstandingImageOpened++;
+ logWrapper("Received an opened image: " + mOutstandingImageOpened + "/"
+ + mOutstandingImageClosed);
+ logWrapper("Setting an image reference count of " + count + " Total refs = "
+ + mOutstandingImageRefs);
+ return protocol;
+ }
+ }
+
+ /**
+ * Increments the semaphore count for the image. Should ONLY be internally via appendTasks by
+ * internal tasks. Otherwise, image references could get out of whack.
+ *
+ * @return Number of Image references currently held by this instance
+ */
+ protected void incrementSemaphoreReferenceCount(ImageProxy img, int count)
+ throws RuntimeException {
+ synchronized (mImageSemaphoreMap) {
+ ImageReleaseProtocol protocol = mImageSemaphoreMap.get(img);
+ if (mImageSemaphoreMap.get(img) == null) {
+ throw new RuntimeException(
+ "Image Reference has already been released or has never been held.");
+ }
+
+ protocol.addCount(count);
+ mImageSemaphoreMap.put(img, protocol);
+
+ mOutstandingImageRefs += count;
+ }
+ }
+
+ /**
+ * Close an Image with a executor if it's available and does the proper booking keeping on the
+ * object.
+ *
+ * @param img Image to be closed
+ * @param executor Executor to be used, if executor is null, the close is run on the task thread
+ */
+ private void closeImageExecutorSafe(final ImageProxy img, Executor executor) {
+ Runnable closeTask = new Runnable() {
+ @Override
+ public void run() {
+ img.close();
+ mOutstandingImageClosed++;
+ logWrapper("Release of image occurred. Good fun. " + "Total Images Open/Closed = "
+ + mOutstandingImageOpened + "/" + mOutstandingImageClosed);
+ }
+ };
+ if (executor == null) {
+ // Just run it on the main thread.
+ closeTask.run();
+ } else {
+ executor.execute(closeTask);
+ }
+ }
+
+ /**
+ * Calculates the number of new Image references in a set of dependent tasks. Checks to make
+ * sure no new image references are being introduced.
+ *
+ * @param tasks The set of dependent tasks to be run
+ */
+ private int numPropagatedImageReferences(ImageProxy img, Set<TaskImageContainer> tasks)
+ throws RuntimeException {
+ int countImageRefs = 0;
+ for (TaskImageContainer task : tasks) {
+ if (task.mImageProxy != null && task.mImageProxy != img) {
+ throw new RuntimeException("ERROR: Spawned tasks cannot reference new images!");
+ }
+
+ if (task.mImageProxy != null) {
+ countImageRefs++;
+ }
+ }
+
+ return countImageRefs;
+ }
+
+ /**
+ * A simple tuple class to keep track of image reference, and whether to block and/or close on
+ * final image release. Instantiated on every task submission call.
+ */
+ static private class ImageReleaseProtocol {
+
+ public final boolean blockUntilRelease;
+
+ public final boolean closeOnRelease;
+
+ private int count;
+
+ private final ReentrantLock mLock = new ReentrantLock();
+
+ private Condition mSignal;
+
+ // TODO: Backport to Reentrant lock
+ public void setCount(int value) {
+ mLock.lock();
+ count = value;
+ mLock.unlock();
+ }
+
+ public int getCount() {
+ int value;
+ mLock.lock();
+ value = count;
+ mLock.unlock();
+ return value;
+ }
+
+ public void addCount(int value) {
+ mLock.lock();
+ count += value;
+ mLock.unlock();
+ }
+
+ ImageReleaseProtocol(boolean block, boolean close) {
+ blockUntilRelease = block;
+ closeOnRelease = close;
+ count = 0;
+ mSignal = mLock.newCondition();
+ }
+
+ public void block() throws InterruptedException {
+ mLock.lock();
+ try {
+ while (count != 0) {
+ // Spin to deal with spurious signals.
+ mSignal.await();
+ }
+ mLock.unlock();
+ } catch (InterruptedException e) {
+ // TODO: on interruption, figure out what to do.
+ throw (e);
+ }
+ }
+
+ public void signal() {
+ mLock.lock();
+ mSignal.signal();
+ mLock.unlock();
+ }
+
+ }
+
+ // Thread factories for a default constructor
+ private class FastThreadFactory implements ThreadFactory {
+
+ public Thread newThread(Runnable r) {
+ Thread t = new Thread(r);
+ t.setPriority(FAST_THREAD_PRIORITY);
+ return t;
+ }
+ }
+
+ private class SlowThreadFactory implements ThreadFactory {
+
+ public Thread newThread(Runnable r) {
+ Thread t = new Thread(r);
+ t.setPriority(SLOW_THREAD_PRIORITY);
+ return t;
+ }
+ }
+
+ // TODO: Remove with a better implementation. Just to avoid
+ // the GC not getting rid of elements. Should be hooked up to properly
+ // implemented memory pool.
+ public class SimpleCache extends ArrayList<byte[]> {
+
+ public SimpleCache(int numEntries) {
+ super(numEntries);
+ }
+
+ public synchronized void cacheSave(byte[] mem) {
+ add(mem);
+ }
+
+ public synchronized byte[] cacheGet() {
+ if (size() < 1) {
+ return null;
+ } else {
+ return mSimpleCache.remove(0);
+ }
+ }
+ }
+
+}
diff --git a/src/com/android/camera/processing/ImageConsumer.java b/src/com/android/camera/processing/ImageConsumer.java
new file mode 100644
index 0000000..964e16e
--- /dev/null
+++ b/src/com/android/camera/processing/ImageConsumer.java
@@ -0,0 +1,102 @@
+/*
+ * Copyright (C) 2014 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.camera.processing;
+
+import com.android.camera.one.v2.camera2proxy.ImageProxy;
+
+import java.util.Set;
+import java.util.concurrent.Executor;
+
+/**
+ * Defines interface between an ImageBackend object and a simplified camera object that merely
+ * delivers images to the backend. After the tasks and Android image is submitted to the
+ * ImageConsumer, the responsibility to close on the Android image object as early as possible is
+ * transferred to the implementation. Whether an image can be submitted again for process is up to
+ * the implementation of the consumer.
+ */
+public interface ImageConsumer {
+
+ /**
+ * Provides the basic functionality of camera processing via an easy-to-use method call.
+ *
+ * @param img The Image to be Processed
+ * @param executor The executor on which to execute events and image close
+ * @param processingFlags Bit vector comprised of logically ORed TASK_FLAG* constants
+ */
+ public boolean receiveImage(ImageProxy img, Executor executor,
+ Set<ImageTaskFlags> processingFlags) throws InterruptedException;
+
+ /**
+ * Provides the basic functionality of camera processing via a more general- purpose method
+ * call. Tasks can be extended off of the TaskImageContainer, or created from factory method
+ * provided by implementor.
+ *
+ * @param img The Image to be Processed
+ * @param sharedTask Set of Tasks to be run on the given image
+ * @param blockOnImageRelease If true, call blocks until the object img is no longer referred by
+ * any task. If false, call is non-blocking
+ * @param closeOnImageRelease If true, images is closed when the object img is is no longer
+ * referred by any task. If false,
+ * @return Whether the blocking completed properly. If false, there may be a need to clean up
+ * image closes manually.
+ */
+ public boolean receiveImage(ImageProxy img, TaskImageContainer sharedTask,
+ boolean blockOnImageRelease, boolean closeOnImageRelease) throws InterruptedException;
+
+ /**
+ * Provides the basic functionality of camera processing via a more general- purpose method
+ * call. Tasks can be extended off of the TaskImageContainer, or created from factory method
+ * provided by implementor.
+ *
+ * @param img The Image to be Processed
+ * @param sharedTasks Set of tasks to be run on the given image
+ * @param blockOnImageRelease If true, call blocks until the object img is no longer referred by
+ * any task. If false, call is non-blocking
+ * @param closeOnImageRelease If true, images is closed when the object img is is no longer
+ * referred by any task. If false, close is not called on release
+ * @return Whether the blocking completed properly. If false, there may be a need to clean up
+ * image closes manually.
+ */
+ public boolean receiveImage(ImageProxy img, Set<TaskImageContainer> sharedTasks,
+ boolean blockOnImageRelease, boolean closeOnImageRelease) throws InterruptedException;
+
+ /**
+ * Returns the number of images that are currently being referred by the consumer component.
+ *
+ * @return Number of images that are currently being referred by the consumer
+ */
+ public int numberOfReservedOpenImages();
+
+ /**
+ * Shutdown all tasks by blocking on tasks to be completed.
+ */
+ public void shutdown();
+
+ /**
+ * Getter to the object that manages the ListenerEvents. Reigster listeners to this object.
+ */
+ public ImageProcessorProxyListener getProxyListener();
+
+ // Current jobs that should be able to be tagged to an image.
+ public enum ImageTaskFlags {
+ COMPRESS_IMAGE_TO_JPEG,
+ CONVERT_IMAGE_TO_RGB_PREVIEW,
+ WRITE_IMAGE_TO_DISK,
+ BLOCK_UNTIL_IMAGE_RELEASE,
+ CLOSE_IMAGE_ON_RELEASE
+ }
+}
diff --git a/src/com/android/camera/processing/ImageProcessorListener.java b/src/com/android/camera/processing/ImageProcessorListener.java
new file mode 100644
index 0000000..291c5a9
--- /dev/null
+++ b/src/com/android/camera/processing/ImageProcessorListener.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright (C) 2014 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.camera.processing;
+
+import android.net.Uri;
+
+/**
+ * Defines the interactions between the Tasks that are running on ImageBackend and other subsystems
+ * (such as UI) which need to update and listen for said event (such as preview completition).
+ */
+public interface ImageProcessorListener {
+
+ /*
+ * !!!!PLACEHOLDER IMPLEMENTATION!!!! Unclear what the best pattern for listeners, given the
+ * types of tasks and the return types For now, I've gone with a minimal interface that is not
+ * currently plumbed for error handling.
+ */
+
+ /**
+ * Called when a task starts running.
+ *
+ * @param task Task specification that includes unique content id
+ */
+ public void onStart(TaskImageContainer.TaskInfo task);
+
+ /**
+ * Called when compressed image is complete and is ready for further processing
+ *
+ * @param task Task specification that includes unique content id
+ * @param payload Byte array that contains the compressed image data
+ */
+ public void onResultCompressed(TaskImageContainer.TaskInfo task,
+ TaskImageContainer.CompressedPayload payload);
+
+ /**
+ * Called when uncompressed image conversion is done and is ready for further processing
+ *
+ * @param task Task specification that includes unique content id
+ * @param payload 32-bit Integer array that contains the uncompressed image data
+ */
+ public void onResultUncompressed(TaskImageContainer.TaskInfo task,
+ TaskImageContainer.UncompressedPayload payload);
+
+ /**
+ * Called when image has been written to disk and ready for further processing via uri.
+ *
+ * @param task Task specification that includes unique content id
+ * @param uri Uri that serves as handle to image written to disk
+ */
+ public void onResultUri(TaskImageContainer.TaskInfo task, Uri uri);
+
+ // TODO: Figure out how to best error handling interface
+ // public void onError(TaskImageContainer.JobInfo task);
+}
diff --git a/src/com/android/camera/processing/ImageProcessorProxyListener.java b/src/com/android/camera/processing/ImageProcessorProxyListener.java
new file mode 100644
index 0000000..4bf63df
--- /dev/null
+++ b/src/com/android/camera/processing/ImageProcessorProxyListener.java
@@ -0,0 +1,122 @@
+/*
+ * Copyright (C) 2014 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.camera.processing;
+
+import android.media.Image;
+import android.net.Uri;
+
+import com.android.camera.debug.Log;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+/**
+ * Implements the ability for the object to send events to multiple listeners in a thread-safe
+ * manner. Also, listeners can also filter messages based on the a specific image result. TODO:
+ * Replace this object with a more generic listener classes.
+ */
+public class ImageProcessorProxyListener implements ImageProcessorListener {
+
+ private final static Log.Tag TAG = new Log.Tag("IProxyListener");
+
+ private List<ImageProcessorListener> mRegisteredListeners = null;
+
+ private HashMap<ImageProcessorListener, Long> mImageFilter = null;
+
+ ImageProcessorProxyListener() {
+ mRegisteredListeners = new ArrayList<ImageProcessorListener>();
+ mImageFilter = new HashMap<ImageProcessorListener, Long>();
+ }
+
+ // TODO: Return the state of the paired thing in processing
+ public List<TaskImageContainer.TaskImage> registerListener(ImageProcessorListener listener,
+ Image image) {
+ synchronized (mRegisteredListeners) {
+ mRegisteredListeners.add(listener);
+ if (image == null) {
+ mImageFilter.put(listener, null);
+ } else {
+ mImageFilter.put(listener, image.getTimestamp());
+ }
+ }
+
+ // TODO: return an array that encapsulated the current jobs that are
+ // running.
+ return null;
+ }
+
+ private List<ImageProcessorListener> filteredListeners(long imageId) {
+ List<ImageProcessorListener> filteredList = new ArrayList<ImageProcessorListener>();
+
+ for (ImageProcessorListener l : mRegisteredListeners) {
+ if (mImageFilter.get(l) == null || mImageFilter.get(l) == imageId) {
+ filteredList.add(l);
+ }
+ }
+
+ return filteredList;
+ }
+
+ public void unregisterListener(ImageProcessorListener listener) {
+ synchronized (mRegisteredListeners) {
+ if (mRegisteredListeners.contains(listener)) {
+ mRegisteredListeners.remove(listener);
+ mImageFilter.remove(listener);
+ Log.e(TAG, "There are " + mRegisteredListeners.size() + " listeners after removal");
+ } else {
+ Log.e(TAG, "Couldn't find listener. There are " + mRegisteredListeners.size()
+ + " listeners after removal");
+ }
+ }
+ }
+
+ public void onStart(TaskImageContainer.TaskInfo job) {
+ synchronized (mRegisteredListeners) {
+ for (ImageProcessorListener l : filteredListeners(job.contentId)) {
+ l.onStart(job);
+ }
+ }
+ }
+
+ public void onResultCompressed(TaskImageContainer.TaskInfo job,
+ TaskImageContainer.CompressedPayload payload) {
+ synchronized (mRegisteredListeners) {
+ for (ImageProcessorListener l : filteredListeners(job.contentId)) {
+ l.onResultCompressed(job, payload);
+ }
+ }
+ }
+
+ public void onResultUncompressed(TaskImageContainer.TaskInfo job,
+ TaskImageContainer.UncompressedPayload payload) {
+ synchronized (mRegisteredListeners) {
+ for (ImageProcessorListener l : filteredListeners(job.contentId)) {
+ l.onResultUncompressed(job, payload);
+ }
+ }
+ }
+
+ public void onResultUri(TaskImageContainer.TaskInfo job, Uri uri) {
+ synchronized (mRegisteredListeners) {
+ for (ImageProcessorListener l : filteredListeners(job.contentId)) {
+ l.onResultUri(job, uri);
+ }
+ }
+ }
+
+}
diff --git a/src/com/android/camera/processing/TaskChainedCompressImageToJpeg.java b/src/com/android/camera/processing/TaskChainedCompressImageToJpeg.java
new file mode 100644
index 0000000..b2f6e3b
--- /dev/null
+++ b/src/com/android/camera/processing/TaskChainedCompressImageToJpeg.java
@@ -0,0 +1,95 @@
+/*
+ * Copyright (C) 2014 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.camera.processing;
+
+import android.graphics.ImageFormat;
+
+import com.android.camera.app.OrientationManager;
+import com.android.camera.debug.Log;
+import com.android.camera.one.v2.camera2proxy.ImageProxy;
+
+import java.util.concurrent.Executor;
+
+/**
+ * Implements the conversion of a YUV_420_888 image to compressed JPEG byte array,
+ * as two separate tasks: the first to copy from the image to NV21 memory layout, and the second
+ * to convert the image into JPEG, using the built-in Android compressor.
+ */
+class TaskChainedCompressImageToJpeg extends TaskJpegEncode {
+ private final static Log.Tag TAG = new Log.Tag("TaskChainJpg");
+
+ TaskChainedCompressImageToJpeg(ImageProxy imageProxy, Executor executor,
+ ImageBackend imageBackend) {
+ super(imageProxy, executor, imageBackend, ProcessingPriority.SLOW);
+ }
+
+ private void logWrapper(String message) {
+ // Do nothing.
+ }
+
+
+ @Override
+ public void run() {
+ ImageProxy img = mImageProxy;
+
+ // TODO: Pass in the orientation for processing as well.
+ final TaskImage inputImage = new TaskImage(OrientationManager.DeviceOrientation.CLOCKWISE_0,
+ img.getWidth(),img.getHeight(),img.getFormat());
+ final TaskImage resultImage = new TaskImage(OrientationManager.DeviceOrientation.CLOCKWISE_0,
+ img.getWidth(),img.getHeight(),ImageFormat.JPEG);
+
+ onStart(mId,inputImage,resultImage);
+
+ int[] strides = new int[3];
+ // Do the byte copy
+ strides[0] = img.getPlanes()[0].getRowStride() / img.getPlanes()[0].getPixelStride();
+ strides[1] = 2 * img.getPlanes()[1].getRowStride() / img.getPlanes()[1].getPixelStride();
+ strides[2] = 2 * img.getPlanes()[2].getRowStride() / img.getPlanes()[2].getPixelStride();
+
+ byte[] dataCopy = mImageBackend.getCache().cacheGet();
+ if (dataCopy == null) {
+ dataCopy = convertYUV420ImageToPackedNV21(img);
+ } else {
+ convertYUV420ImageToPackedNV21(img, dataCopy);
+ }
+
+ // Release the image now that you have a usable copy
+ mImageBackend.releaseSemaphoreReference(img, mExecutor);
+
+ final byte[] chainedDataCopy = dataCopy;
+ final int[] chainedStrides = strides;
+
+ // This task drops the image reference.
+ TaskImageContainer chainedTask = new TaskJpegEncode(this, ProcessingPriority.SLOW) {
+
+ @Override
+ public void run() {
+ // Image is closed by now. Do NOT reference image directly.
+ byte[] compressedData = convertNv21toJpeg(chainedDataCopy,
+ resultImage.height, resultImage.width, chainedStrides);
+ onJpegEncodeDone(mId, inputImage, resultImage, compressedData);
+ mImageBackend.getCache().cacheSave(chainedDataCopy);
+ logWrapper("Finished off a chained task now that image is released.");
+ }
+ };
+
+ // Passed null, since the image has already been released.
+ mImageBackend.appendTasks(null, chainedTask);
+ logWrapper("Kicking off a chained task now that image is released.");
+
+ }
+}
diff --git a/src/com/android/camera/processing/TaskCompressImageToJpeg.java b/src/com/android/camera/processing/TaskCompressImageToJpeg.java
new file mode 100644
index 0000000..db588ab
--- /dev/null
+++ b/src/com/android/camera/processing/TaskCompressImageToJpeg.java
@@ -0,0 +1,80 @@
+/*
+ * Copyright (C) 2014 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.camera.processing;
+
+import android.graphics.ImageFormat;
+
+import com.android.camera.app.OrientationManager;
+import com.android.camera.debug.Log;
+import com.android.camera.one.v2.camera2proxy.ImageProxy;
+
+import java.util.concurrent.Executor;
+
+/**
+ * Implements the conversion of a YUV_420_888 image to compressed JPEG byte array,
+ * as a single two-step task: the first to copy from the image to NV21 memory layout,
+ * and the second to convert the image into JPEG, using the built-in Android compressor.
+ */
+public class TaskCompressImageToJpeg extends TaskJpegEncode {
+
+ TaskCompressImageToJpeg(ImageProxy imageProxy, Executor executor, ImageBackend imageBackend) {
+ super(imageProxy, executor, imageBackend, ProcessingPriority.SLOW);
+ }
+
+ private void logWrapper(String message) {
+ // Do nothing.
+ }
+
+ @Override
+ public void run() {
+ ImageProxy img = mImageProxy;
+
+ // TODO: Pass in the orientation for processing as well.
+ final TaskImage inputImage = new TaskImage(
+ OrientationManager.DeviceOrientation.CLOCKWISE_0,
+ img.getWidth(),img.getHeight(),img.getFormat());
+ final TaskImage resultImage = new TaskImage(
+ OrientationManager.DeviceOrientation.CLOCKWISE_0,
+ img.getWidth(),img.getHeight(),ImageFormat.JPEG);
+
+ onStart(mId,inputImage,resultImage);
+
+ logWrapper("TIMER_END Full-size YUV buffer available, w=" + img.getWidth() + " h="
+ + img.getHeight() + " of format " + img.getFormat() + " (35==YUV_420_888)");
+ int[] strides = new int[3];
+ // Do the byte copy
+ strides[0] = img.getPlanes()[0].getRowStride() / img.getPlanes()[0].getPixelStride();
+ strides[1] = 2 * img.getPlanes()[1].getRowStride() / img.getPlanes()[1].getPixelStride();
+ strides[2] = 2 * img.getPlanes()[2].getRowStride() / img.getPlanes()[2].getPixelStride();
+
+ byte[] dataCopy = mImageBackend.getCache().cacheGet();
+ if (dataCopy == null) {
+ dataCopy = convertYUV420ImageToPackedNV21(img);
+ } else {
+ convertYUV420ImageToPackedNV21(img, dataCopy);
+ }
+
+ // Release the image now that you have a usable copy
+ mImageBackend.releaseSemaphoreReference(img, mExecutor);
+
+ byte[] compressedData = convertNv21toJpeg(dataCopy, inputImage.width, inputImage.height,
+ strides);
+ mImageBackend.getCache().cacheSave(dataCopy);
+ onJpegEncodeDone(mId,inputImage,resultImage,compressedData);
+
+ }
+}
diff --git a/src/com/android/camera/processing/TaskConvertImageToRGBPreview.java b/src/com/android/camera/processing/TaskConvertImageToRGBPreview.java
new file mode 100644
index 0000000..2ee55cc
--- /dev/null
+++ b/src/com/android/camera/processing/TaskConvertImageToRGBPreview.java
@@ -0,0 +1,433 @@
+/*
+ * Copyright (C) 2014 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.camera.processing;
+
+import com.android.camera.app.OrientationManager;
+import com.android.camera.debug.Log;
+import com.android.camera.one.v2.camera2proxy.ImageProxy;
+
+import java.nio.ByteBuffer;
+import java.util.concurrent.Executor;
+
+/**
+ * Implements the conversion of a YUV_420_888 image to subsampled image inscribed in a circle.
+ */
+public class TaskConvertImageToRGBPreview extends TaskImageContainer {
+ // 24 bit-vector to be written for images that are out of bounds.
+ public final static int OUT_OF_BOUNDS_COLOR = 0x00000000;
+
+ protected final static Log.Tag TAG = new Log.Tag("TaskRGBPreview");
+
+ private int mTargetHeight;
+
+ private int mTargetWidth;
+
+ TaskConvertImageToRGBPreview(ImageProxy imageProxy, Executor executor,
+ ImageBackend imageBackend, int targetWidth, int targetHeight) {
+ super(imageProxy, executor, imageBackend, ProcessingPriority.FAST);
+ mTargetWidth = targetWidth;
+ mTargetHeight = targetHeight;
+ }
+
+ private void logWrapper(String message) {
+ // Do nothing.
+ }
+
+ /**
+ * Simple helper function
+ */
+ private int quantizeBy2(int value) {
+ return (value / 2) * 2;
+ }
+
+ /**
+ * Way to calculate the resultant image sizes of inscribed circles:
+ * colorInscribedDataCircleFromYuvImage, dummyColorInscribedDataCircleFromYuvImage,
+ * colorDataCircleFromYuvImage
+ *
+ * @param height height of the input image
+ * @param width width of the input image
+ * @return height/width of the resultant square image TODO: Refactor functions in question to
+ * return the image size as a tuple for these functions, or re-use an general purpose
+ * holder object.
+ */
+ protected int inscribedCircleRadius(int width, int height) {
+ return (Math.min(height, width) / 2) + 1;
+ }
+
+ /**
+ * Calculates the best integer subsample from a given height and width to a target width and
+ * height It is assumed that the exact scaling will be done with the Android Bitmap framework;
+ * this subsample value is to best convert raw images into the lowest resolution raw images in
+ * visually lossless manner without changing the aspect ratio or creating subsample artifacts.
+ *
+ * @param height height of the input image
+ * @param width width of the input image
+ * @param targetWidth width of the image as it will be seen on the screen in raw pixels
+ * @param targetHeight height of the image as it will be seen on the screen in raw pixels
+ * @returns inscribed image as ARGB_8888
+ */
+ protected int calculateBestSubsampleFactor(int height, int width, int targetWidth,
+ int targetHeight) {
+ int maxSubsample = Math.min(height / targetHeight, width / targetWidth);
+ if (maxSubsample < 1) {
+ return 1;
+ }
+
+ // Make sure the resultant image width/height is divisible by 2 to
+ // account
+ // for chroma subsampled images such as YUV
+ for (int i = maxSubsample; i >= 1; i--) {
+ if (((height % (2 * i) == 0) && (width % (2 * i) == 0))) {
+ return i;
+ }
+ }
+
+ return 1; // If all fails, don't do the subsample.
+ }
+
+ /**
+ * Converts an Android Image to a inscribed circle bitmap of ARGB_8888 in a super-optimized loop
+ * unroll. Guarantees only one subsampled pass over the YUV data. This version of the function
+ * should be used in production and also feathers the edges with 50% alpha on its edges. NOTE:
+ * To get the size of the resultant bitmap, you need to call inscribedCircleRadius(w, h) outside
+ * of this function. Runs in ~10-15ms for 4K image with a subsample of 13. TODO: Implement
+ * horizontal alpha feathering of the edge of the image, if necessary.
+ *
+ * @param img YUV420_888 Image to convert
+ * @param subsample width/height subsample factor
+ * @returns inscribed image as ARGB_8888
+ */
+ protected int[] colorInscribedDataCircleFromYuvImage(ImageProxy img, int subsample) {
+ int w = img.getWidth() / subsample;
+ int h = img.getHeight() / subsample;
+ int r = inscribedCircleRadius(w, h);
+
+ int inscribedXMin;
+ int inscribedXMax;
+ int inscribedYMin;
+ int inscribedYMax;
+
+ // Set up input read boundaries.
+ if (w > h) {
+ // since we're 2x2 blocks we need to quantize these values by 2
+ inscribedXMin = quantizeBy2(w / 2 - r);
+ inscribedXMax = quantizeBy2(w / 2 + r);
+ inscribedYMin = 0;
+ inscribedYMax = h;
+ } else {
+ inscribedXMin = 0;
+ inscribedXMax = w;
+ // since we're 2x2 blocks we need to quantize these values by 2
+ inscribedYMin = quantizeBy2(h / 2 - r);
+ inscribedYMax = quantizeBy2(h / 2 + r);
+ }
+
+ ByteBuffer buf0 = img.getPlanes()[0].getBuffer();
+ ByteBuffer bufU = img.getPlanes()[1].getBuffer(); // Downsampled by 2
+ ByteBuffer bufV = img.getPlanes()[2].getBuffer(); // Downsampled by 2
+ int yByteStride = img.getPlanes()[0].getRowStride() * subsample;
+ int uByteStride = img.getPlanes()[1].getRowStride() * subsample;
+ int vByteStride = img.getPlanes()[2].getRowStride() * subsample;
+ int yPixelStride = img.getPlanes()[0].getPixelStride() * subsample;
+ int uPixelStride = img.getPlanes()[1].getPixelStride() * subsample;
+ int vPixelStride = img.getPlanes()[2].getPixelStride() * subsample;
+ int outputPixelStride = r * 2;
+ int centerY = h / 2;
+ int centerX = w / 2;
+
+ int len = r * r * 4;
+ int[] colors = new int[len];
+ int alpha = 255 << 24;
+
+ /*
+ * Quick n' Dirty YUV to RGB conversion R = Y + 1.403V' G = Y - 0.344U' - 0.714V' B = Y +
+ * 1.770U'
+ */
+
+ Log.v(TAG, "TIMER_BEGIN Starting Native Java YUV420-to-RGB Quick n' Dirty Conversion 4");
+ Log.v(TAG, "\t Y-Plane Size=" + w + "x" + h);
+ Log.v(TAG,
+ "\t U-Plane Size=" + img.getPlanes()[1].getRowStride() + " Pixel Stride="
+ + img.getPlanes()[1].getPixelStride());
+ Log.v(TAG,
+ "\t V-Plane Size=" + img.getPlanes()[2].getRowStride() + " Pixel Stride="
+ + img.getPlanes()[2].getPixelStride());
+ // Take in vertical lines by factor of two because of the u/v component
+ // subsample
+ for (int j = inscribedYMin; j < inscribedYMax; j += 2) {
+ int offsetY = j * yByteStride + inscribedXMin;
+ int offsetColor = (j - inscribedYMin) * (outputPixelStride);
+ int offsetU = (j / 2) * (uByteStride) + (inscribedXMin);
+ int offsetV = (j / 2) * (vByteStride) + (inscribedXMin);
+ // Parametrize the circle boundaries w.r.t. the y component.
+ // Find the subsequence of pixels we need for each horizontal raster
+ // line.
+ int circleHalfWidth0 =
+ (int)(Math.sqrt((float)(r * r - (j - centerY) * (j - centerY))) + 0.5f);
+ int circleMin0 = centerX - (circleHalfWidth0);
+ int circleMax0 = centerX + circleHalfWidth0;
+ int circleHalfWidth1 = (int)(Math.sqrt((float)(r * r - (j + 1 - centerY)
+ * (j + 1 - centerY))) + 0.5f);
+ int circleMin1 = centerX - (circleHalfWidth1);
+ int circleMax1 = centerX + circleHalfWidth1;
+
+ // Take in horizontal lines by factor of two because of the u/v
+ // component subsample
+ // and everything as 2x2 blocks.
+ for (int i = inscribedXMin; i < inscribedXMax; i += 2, offsetY += 2 * yPixelStride,
+ offsetColor += 2, offsetU += uPixelStride, offsetV += vPixelStride) {
+ // Note i and j are in terms of pixels of the subsampled image
+ // offsetY, offsetU, and offsetV are in terms of bytes of the
+ // image
+ // offsetColor, output_pixel stride are in terms of the packed
+ // output image
+ if ((i > circleMax0 && i > circleMax1) || (i + 1 < circleMin0 && i < circleMin1)) {
+ colors[offsetColor] = OUT_OF_BOUNDS_COLOR;
+ colors[offsetColor + 1] = OUT_OF_BOUNDS_COLOR;
+ colors[offsetColor + outputPixelStride] = OUT_OF_BOUNDS_COLOR;
+ colors[offsetColor + outputPixelStride + 1] = OUT_OF_BOUNDS_COLOR;
+ continue;
+ }
+
+ // calculate the RGB component of the u/v channels and use it
+ // for all pixels in the 2x2 block
+ int u = (int)(bufU.get(offsetU) & 255) - 128;
+ int v = (int)(bufV.get(offsetV) & 255) - 128;
+ int redDiff = (v * 45) >> 4;
+ int greenDiff = 0 - ((u * 11 + v * 22) >> 4);
+ int blueDiff = (u * 58) >> 4;
+
+ if (i > circleMax0 || i < circleMin0) {
+ colors[offsetColor] = OUT_OF_BOUNDS_COLOR;
+ } else {
+ // Do a little alpha feathering on the edges
+ int alpha00 = (i == circleMax0 || i == circleMin0) ? (128 << 24) : (255 << 24);
+
+ int y00 = (int)(buf0.get(offsetY) & 255);
+
+ int green00 = y00 + greenDiff;
+ int blue00 = y00 + blueDiff;
+ int red00 = y00 + redDiff;
+
+ // Get the railing correct
+ if (green00 < 0) {
+ green00 = 0;
+ }
+ if (red00 < 0) {
+ red00 = 0;
+ }
+ if (blue00 < 0) {
+ blue00 = 0;
+ }
+
+ if (green00 > 255) {
+ green00 = 255;
+ }
+ if (red00 > 255) {
+ red00 = 255;
+ }
+ if (blue00 > 255) {
+ blue00 = 255;
+ }
+
+ colors[offsetColor] = (red00 & 255) << 16 | (green00 & 255) << 8
+ | (blue00 & 255) | alpha00;
+ }
+
+ if (i + 1 > circleMax0 || i + 1 < circleMin0) {
+ colors[offsetColor + 1] = OUT_OF_BOUNDS_COLOR;
+ } else {
+ int alpha01 = ((i + 1) == circleMax0 || (i + 1) == circleMin0) ? (128 << 24)
+ : (255 << 24);
+ int y01 = (int)(buf0.get(offsetY + yPixelStride) & 255);
+ int green01 = y01 + greenDiff;
+ int blue01 = y01 + blueDiff;
+ int red01 = y01 + redDiff;
+
+ // Get the railing correct
+ if (green01 < 0) {
+ green01 = 0;
+ }
+ if (red01 < 0) {
+ red01 = 0;
+ }
+ if (blue01 < 0) {
+ blue01 = 0;
+ }
+
+ if (green01 > 255) {
+ green01 = 255;
+ }
+ if (red01 > 255) {
+ red01 = 255;
+ }
+ if (blue01 > 255) {
+ blue01 = 255;
+ }
+ colors[offsetColor + 1] = (red01 & 255) << 16 | (green01 & 255) << 8
+ | (blue01 & 255) | alpha01;
+ }
+
+ if (i > circleMax1 || i < circleMin1) {
+ colors[offsetColor + outputPixelStride] = OUT_OF_BOUNDS_COLOR;
+ } else {
+ int alpha10 = (i == circleMax1 || i == circleMin1) ? (128 << 24) : (255 << 24);
+ int y10 = (int)(buf0.get(offsetY + yByteStride) & 255);
+ int green10 = y10 + greenDiff;
+ int blue10 = y10 + blueDiff;
+ int red10 = y10 + redDiff;
+
+ // Get the railing correct
+ if (green10 < 0) {
+ green10 = 0;
+ }
+ if (red10 < 0) {
+ red10 = 0;
+ }
+ if (blue10 < 0) {
+ blue10 = 0;
+ }
+ if (green10 > 255) {
+ green10 = 255;
+ }
+ if (red10 > 255) {
+ red10 = 255;
+ }
+ if (blue10 > 255) {
+ blue10 = 255;
+ }
+
+ colors[offsetColor + outputPixelStride] = (red10 & 255) << 16
+ | (green10 & 255) << 8 | (blue10 & 255) | alpha10;
+ }
+
+ if (i + 1 > circleMax1 || i + 1 < circleMin1) {
+ colors[offsetColor + outputPixelStride + 1] = OUT_OF_BOUNDS_COLOR;
+ } else {
+ int alpha11 = ((i + 1) == circleMax1 || (i + 1) == circleMin1) ? (128 << 24)
+ : (255 << 24);
+ int y11 = (int)(buf0.get(offsetY + yByteStride + yPixelStride) & 255);
+ int green11 = y11 + greenDiff;
+ int blue11 = y11 + blueDiff;
+ int red11 = y11 + redDiff;
+
+ // Get the railing correct
+ if (green11 < 0) {
+ green11 = 0;
+ }
+ if (red11 < 0) {
+ red11 = 0;
+ }
+ if (blue11 < 0) {
+ blue11 = 0;
+ }
+
+ if (green11 > 255) {
+ green11 = 255;
+ }
+
+ if (red11 > 255) {
+ red11 = 255;
+ }
+ if (blue11 > 255) {
+ blue11 = 255;
+ }
+ colors[offsetColor + outputPixelStride + 1] = (red11 & 255) << 16
+ | (green11 & 255) << 8 | (blue11 & 255) | alpha11;
+ }
+
+ }
+ }
+ Log.v(TAG, "TIMER_END Starting Native Java YUV420-to-RGB Quick n' Dirty Conversion 4");
+
+ return colors;
+ }
+
+ /**
+ * DEBUG IMAGE FUNCTION Converts an Android Image to a inscribed circle bitmap, currently wired
+ * to the test pattern. Will subsample and optimize the image given a target resolution.
+ *
+ * @param img YUV420_888 Image to convert
+ * @param subsample width/height subsample factor
+ * @returns inscribed image as ARGB_8888
+ */
+ protected int[] dummyColorInscribedDataCircleFromYuvImage(ImageProxy img, int subsample) {
+ Log.e(TAG, "RUNNING DUMMY dummyColorInscribedDataCircleFromYuvImage");
+ int w = img.getWidth() / subsample;
+ int h = img.getHeight() / subsample;
+ int r = inscribedCircleRadius(w, h);
+ int len = r * r * 4;
+ int[] colors = new int[len];
+
+ // Make a fun test pattern.
+ for (int i = 0; i < len; i++) {
+ int x = i % (2 * r);
+ int y = i / (2 * r);
+ colors[i] = (255 << 24) | ((x & 255) << 16) | ((y & 255) << 8);
+ }
+
+ return colors;
+ }
+
+ @Override
+ public void run() {
+ ImageProxy img = mImageProxy;
+
+ // TODO: Pass in the orientation for processing as well.
+ final TaskImage inputImage = new TaskImage(
+ OrientationManager.DeviceOrientation.CLOCKWISE_0, img.getWidth(), img.getHeight(),
+ img.getFormat());
+ final int subsample = calculateBestSubsampleFactor(inputImage.width, inputImage.height,
+ mTargetWidth, mTargetHeight);
+ final int radius = inscribedCircleRadius(inputImage.width / subsample, inputImage.height
+ / subsample);
+
+ final TaskImage resultImage = new TaskImage(
+ OrientationManager.DeviceOrientation.CLOCKWISE_0, radius * 2, radius * 2,
+ TaskImage.EXTRA_USER_DEFINED_FORMAT_ARGB_8888);
+
+ onStart(mId, inputImage, resultImage);
+
+ logWrapper("TIMER_END Rendering preview YUV buffer available, w=" + img.getWidth()
+ / subsample + " h=" + img.getHeight() / subsample + " of subsample " + subsample);
+
+ // For dummy version, use
+ // dummyColorInscribedDataCircleFromYuvImage
+ final int[] convertedImage = colorInscribedDataCircleFromYuvImage(img, subsample);
+ // Signal backend that reference has been released
+ mImageBackend.releaseSemaphoreReference(img, mExecutor);
+
+ onPreviewDone(resultImage, inputImage, convertedImage);
+
+ }
+
+ /**
+ * Wraps the onResultUncompressed listener function
+ *
+ * @param resultImage Image specification of result image
+ * @param inputImage Image specification of the input image
+ * @param colors Uncompressed data buffer
+ */
+ public void onPreviewDone(TaskImage resultImage, TaskImage inputImage, int[] colors) {
+ TaskInfo job = new TaskInfo(mId, inputImage, resultImage);
+ final ImageProcessorListener listener = mImageBackend.getProxyListener();
+
+ listener.onResultUncompressed(job, new UncompressedPayload(colors));
+ }
+
+}
diff --git a/src/com/android/camera/processing/TaskImageContainer.java b/src/com/android/camera/processing/TaskImageContainer.java
new file mode 100644
index 0000000..1ffb23a
--- /dev/null
+++ b/src/com/android/camera/processing/TaskImageContainer.java
@@ -0,0 +1,174 @@
+/*
+ * Copyright (C) 2014 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.camera.processing;
+
+import com.android.camera.app.OrientationManager;
+import com.android.camera.debug.Log;
+import com.android.camera.one.v2.camera2proxy.ImageProxy;
+
+import java.util.concurrent.Executor;
+
+/**
+ * TaskImageContainer are the base class of tasks that wish to run with the ImageBackend class. It
+ * contains the basic information required to interact with the ImageBackend class and the ability
+ * to identify itself to the UI backend for updates on its progress.
+ */
+public abstract class TaskImageContainer implements Runnable {
+
+
+ /**
+ * Simple helper class to encapsulate uncompressed payloads. Could be more complex in
+ * the future.
+ */
+ static public class UncompressedPayload {
+ final public int [] data;
+ UncompressedPayload(int [] passData) {
+ data = passData;
+ }
+ }
+
+
+ /**
+ * Simple helper class to encapsulate compressed payloads. Could be more complex in
+ * the future.
+ */
+ static public class CompressedPayload {
+ final public byte [] data;
+ CompressedPayload(byte [] passData) {
+ data = passData;
+ }
+ }
+
+ /**
+ * Simple helper class to encapsulate all necessary image information that is carried with the
+ * data to processing, so that tasks derived off of TaskImageContainer can properly coordinate
+ * and optimize its computation.
+ */
+ static public class TaskImage {
+ // Addendum to Android-defined image-format
+ public final static int EXTRA_USER_DEFINED_FORMAT_ARGB_8888 = -1;
+
+ // Minimal required knowledge for the image specification.
+ final OrientationManager.DeviceOrientation orientation;
+
+ final int height;
+
+ final int width;
+
+ final int format;
+
+ TaskImage(OrientationManager.DeviceOrientation anOrientation, int aWidth, int aHeight,
+ int aFormat) {
+ orientation = anOrientation;
+ height = aWidth;
+ width = aHeight;
+ format = aFormat;
+ }
+
+ }
+
+ /**
+ * Simple helper class to encapsulate input and resultant image specification.
+ * TasksImageContainer classes can be uniquely identified by triplet of its content (currently,
+ * the global timestamp of when the object was taken), the image specification of the input and
+ * the desired output image specification.
+ */
+ static public class TaskInfo {
+ // The unique Id of the image being processed.
+ public final long contentId;
+
+ public final TaskImage input;
+
+ public final TaskImage result;
+
+ TaskInfo(long aContentId, TaskImage inputSpec, TaskImage outputSpec) {
+ contentId = aContentId;
+ input = inputSpec;
+ result = outputSpec;
+ }
+
+ }
+
+ public enum ProcessingPriority {
+ FAST, SLOW
+ }
+
+ protected final static Log.Tag TAG = new Log.Tag("TaskImgContain");
+
+ final protected ImageBackend mImageBackend;
+
+ final protected Executor mExecutor;
+
+ final protected long mId;
+
+ final protected ProcessingPriority mProcessingPriority;
+
+ final protected ImageProxy mImageProxy;
+
+ /**
+ * Constructor when releasing the image reference.
+ *
+ * @param otherTask the original task that is spawning this task.
+ * @param processingPriority Priority that the derived task will run at.
+ */
+ public TaskImageContainer(TaskImageContainer otherTask, ProcessingPriority processingPriority) {
+ this.mId = otherTask.mId;
+ this.mExecutor = otherTask.mExecutor;
+ this.mImageBackend = otherTask.mImageBackend;
+ this.mProcessingPriority = processingPriority;
+ mImageProxy = null;
+ }
+
+ /**
+ * Constructor to use when keeping the image reference.
+ *
+ * @param image Image reference that needs to be released.
+ * @param Executor Executor to run the event handling
+ * @param imageBackend a reference to the ImageBackend, in case, you need to spawn other tasks
+ * @param preferredLane Priority that the derived task will run at
+ */
+ public TaskImageContainer(ImageProxy image, Executor Executor, ImageBackend imageBackend,
+ ProcessingPriority preferredLane) {
+ mImageProxy = image;
+ this.mId = image.getTimestamp();
+ this.mExecutor = Executor;
+ this.mImageBackend = imageBackend;
+ this.mProcessingPriority = preferredLane;
+ }
+
+ /**
+ * Basic listener function to signal ImageBackend that task has started.
+ *
+ * @param id Id for image content
+ * @param input Image specification for task input
+ * @param result Image specification for task result
+ */
+ public void onStart(long id, TaskImage input, TaskImage result) {
+ TaskInfo job = new TaskInfo(id, input, result);
+ final ImageProcessorListener listener = mImageBackend.getProxyListener();
+ listener.onStart(job);
+ }
+
+ /**
+ * Getter for Processing Priority
+ *
+ * @return Processing Priority associated with the task.
+ */
+ public ProcessingPriority getProcessingPriority() {
+ return mProcessingPriority;
+ }
+}
diff --git a/src/com/android/camera/processing/TaskJpegEncode.java b/src/com/android/camera/processing/TaskJpegEncode.java
new file mode 100644
index 0000000..1f7ea96
--- /dev/null
+++ b/src/com/android/camera/processing/TaskJpegEncode.java
@@ -0,0 +1,141 @@
+/*
+ * Copyright (C) 2014 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.camera.processing;
+
+import android.graphics.ImageFormat;
+import android.graphics.Rect;
+import android.graphics.YuvImage;
+
+import com.android.camera.debug.Log;
+import com.android.camera.one.v2.camera2proxy.ImageProxy;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.concurrent.Executor;
+
+/**
+ * TaskImageContainer are the base class of tasks that wish to run with the
+ * ImageBackend class. It contains the basic information required to interact
+ * with the ImageBackend class and the ability to identify itself to the UI
+ * backend for updates on its progress.
+ */
+public abstract class TaskJpegEncode extends TaskImageContainer {
+
+ protected final static Log.Tag TAG = new Log.Tag("TaskJpegEnc");
+
+ public TaskJpegEncode(TaskImageContainer otherTask, ProcessingPriority processingPriority) {
+ super(otherTask, processingPriority);
+ }
+
+ public TaskJpegEncode(ImageProxy image, Executor executor, ImageBackend imageBackend,
+ TaskImageContainer.ProcessingPriority preferredLane) {
+ super(image, executor, imageBackend, preferredLane);
+ }
+
+ /**
+ * Converts the YUV420_888 Image into a packed NV21 of a single byte array, suitable for JPEG
+ * compression by the method convertNv21toJpeg. This version will allocate its own byte buffer
+ * memory.
+ *
+ * @param img image to be converted
+ * @return byte array of NV21 packed image
+ */
+ public byte[] convertYUV420ImageToPackedNV21(ImageProxy img) {
+ ByteBuffer y_buffer = img.getPlanes()[0].getBuffer();
+ ByteBuffer u_buffer = img.getPlanes()[1].getBuffer();
+ ByteBuffer v_buffer = img.getPlanes()[2].getBuffer();
+ byte[] dataCopy = new byte[y_buffer.capacity() + u_buffer.capacity() + v_buffer.capacity()];
+
+ return convertYUV420ImageToPackedNV21(img, dataCopy);
+ }
+
+ /**
+ * Converts the YUV420_888 Image into a packed NV21 of a single byte array, suitable for JPEG
+ * compression by the method convertNv21toJpeg. Creates a memory block with the y component at
+ * the head and interleaves the u,v components following the y component. Caller is responsible
+ * to allocate a large enough buffer for results.
+ *
+ * @param img image to be converted
+ * @param dataCopy buffer to write NV21 packed image
+ * @return byte array of NV21 packed image
+ */
+ public byte[] convertYUV420ImageToPackedNV21(ImageProxy img, byte[] dataCopy) {
+ // Get all the relevant information and then release the image.
+ final int w = img.getWidth();
+ final int h = img.getHeight();
+
+ ByteBuffer y_buffer = img.getPlanes()[0].getBuffer();
+ ByteBuffer u_buffer = img.getPlanes()[1].getBuffer();
+ ByteBuffer v_buffer = img.getPlanes()[2].getBuffer();
+
+ for (int i = 0; i < y_buffer.capacity(); i++) {
+ dataCopy[i] = (byte)(y_buffer.get(i) & 255);
+ }
+
+ int color_pixel_stride = img.getPlanes()[1].getPixelStride();
+ int data_offset = y_buffer.capacity();
+ for (int i = 0; i < u_buffer.capacity() / color_pixel_stride; i++) {
+ dataCopy[data_offset + 2 * i] = v_buffer.get(i * color_pixel_stride);
+ dataCopy[data_offset + 2 * i + 1] = u_buffer.get(i * color_pixel_stride);
+ }
+
+ return dataCopy;
+ }
+
+ /**
+ * Wraps the Android built-in YUV to Jpeg conversion routine. Pass in a valid NV21 image and get
+ * back a compressed JPEG buffer. A good default JPEG compression implementation that should be
+ * supported on all platforms.
+ *
+ * @param data_copy byte buffer that contains the NV21 image
+ * @param w width of NV21 image
+ * @param h height of N21 image
+ * @return byte array of compressed JPEG image
+ */
+ public byte[] convertNv21toJpeg(byte[] data_copy, int w, int h, int[] strides) {
+ Log.e(TAG, "TIMER_BEGIN NV21 to Jpeg Conversion.");
+ YuvImage yuvImage = new YuvImage(data_copy, ImageFormat.NV21, w, h, strides);
+
+ ByteArrayOutputStream postViewBytes = new ByteArrayOutputStream();
+
+ yuvImage.compressToJpeg(new Rect(0, 0, w, h), 90, postViewBytes);
+ try {
+ postViewBytes.flush();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ Log.e(TAG, "TIMER_END NV21 to Jpeg Conversion.");
+ return postViewBytes.toByteArray();
+ }
+
+ /**
+ * Wraps the onResultCompressed listener for ease of use.
+ *
+ * @param id Unique content id
+ * @param input Specification of image input size
+ * @param result Specification of resultant input size
+ * @param data Container for uncompressed data that represents image
+ */
+ public void onJpegEncodeDone(long id, TaskImage input, TaskImage result, byte[] data) {
+ TaskInfo job = new TaskInfo(id, input, result);
+ final ImageProcessorListener listener = mImageBackend.getProxyListener();
+ listener.onResultCompressed(job, new CompressedPayload(data));
+ }
+
+}
diff --git a/src/com/android/camera/processing/TaskWriteImageToDisk.java b/src/com/android/camera/processing/TaskWriteImageToDisk.java
new file mode 100644
index 0000000..bb39cbf
--- /dev/null
+++ b/src/com/android/camera/processing/TaskWriteImageToDisk.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright (C) 2014 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.camera.processing;
+
+import com.android.camera.debug.Log;
+import com.android.camera.one.v2.camera2proxy.ImageProxy;
+
+import java.util.concurrent.Executor;
+
+/**
+ * Placeholder for writing an image to disk.
+ */
+public class TaskWriteImageToDisk extends TaskImageContainer {
+
+ public TaskWriteImageToDisk(ImageProxy imageProxy, Executor executor,
+ ImageBackend imageBackend) {
+ super(imageProxy, executor, imageBackend, ProcessingPriority.SLOW);
+ }
+
+ @Override
+ public void run() {
+ // TODO: Make a dependency on JPEG_COMPRESSION
+
+ }
+}