ImageReader/Writer: Add ImageWriter and Opaque ImageReader
ImageWriter/Reader API change, including below changes
* Interface for Opaque ImageReader
* ImageWriter Interface and implementation
* Image class minor update to support ImageWriter opaque ImageReader
detach/attach interface are defined but yet to be implemented.
Change-Id: Ic7c0d2df73c80b1a81a7316d8c4556bf7703c309
diff --git a/media/java/android/media/Image.java b/media/java/android/media/Image.java
index 53ab264..9d07492 100644
--- a/media/java/android/media/Image.java
+++ b/media/java/android/media/Image.java
@@ -115,14 +115,49 @@
/**
* Get the timestamp associated with this frame.
* <p>
- * The timestamp is measured in nanoseconds, and is monotonically
- * increasing. However, the zero point and whether the timestamp can be
- * compared against other sources of time or images depend on the source of
- * this image.
+ * The timestamp is measured in nanoseconds, and is normally monotonically
+ * increasing. However, the behavior of the timestamp depends on the source
+ * of this image. See {@link android.hardware.Camera Camera},
+ * {@link android.hardware.camera2.CameraDevice CameraDevice}, {@link MediaPlayer} and
+ * {@link MediaCodec} for more details.
* </p>
*/
public abstract long getTimestamp();
+ /**
+ * Set the timestamp associated with this frame.
+ * <p>
+ * The timestamp is measured in nanoseconds, and is normally monotonically
+ * increasing. However, However, the behavior of the timestamp depends on
+ * the destination of this image. See {@link android.hardware.Camera Camera}
+ * , {@link android.hardware.camera2.CameraDevice CameraDevice},
+ * {@link MediaPlayer} and {@link MediaCodec} for more details.
+ * </p>
+ * <p>
+ * For images dequeued from {@link ImageWriter} via
+ * {@link ImageWriter#dequeueInputImage()}, it's up to the application to
+ * set the timestamps correctly before sending them back to the
+ * {@link ImageWriter}, or the timestamp will be generated automatically when
+ * {@link ImageWriter#queueInputImage queueInputImage()} is called.
+ * </p>
+ *
+ * @param timestamp The timestamp to be set for this image.
+ */
+ public void setTimestamp(long timestamp) {
+ return;
+ }
+
+ /**
+ * <p>Check if the image is opaque.</p>
+ *
+ * <p>The pixel data of opaque images are not accessible to the application,
+ * and therefore {@link #getPlanes} will return an empty array for an opaque image.
+ * </p>
+ */
+ public boolean isOpaque() {
+ return false;
+ }
+
private Rect mCropRect;
/**
@@ -155,7 +190,10 @@
/**
* Get the array of pixel planes for this Image. The number of planes is
- * determined by the format of the Image.
+ * determined by the format of the Image. The application will get an
+ * empty array if the image is opaque because the opaque image pixel data
+ * is not directly accessible. The application can check if an image is
+ * opaque by calling {@link Image#isOpaque}.
*/
public abstract Plane[] getPlanes();
@@ -164,14 +202,54 @@
* <p>
* After calling this method, calling any methods on this {@code Image} will
* result in an {@link IllegalStateException}, and attempting to read from
- * {@link ByteBuffer ByteBuffers} returned by an earlier
- * {@link Plane#getBuffer} call will have undefined behavior.
+ * or write to {@link ByteBuffer ByteBuffers} returned by an earlier
+ * {@link Plane#getBuffer} call will have undefined behavior. If the image
+ * was obtained from {@link ImageWriter} via
+ * {@link ImageWriter#dequeueInputImage()}, after calling this method, any
+ * image data filled by the application will be lost and the image will be
+ * returned to {@link ImageWriter} for reuse. Images given to
+ * {@link ImageWriter#queueInputImage queueInputImage()} are automatically
+ * closed.
* </p>
*/
@Override
public abstract void close();
/**
+ * <p>
+ * Check if the image can be attached to a new owner (e.g. {@link ImageWriter}).
+ * </p>
+ * <p>
+ * This is a package private method that is only used internally.
+ * </p>
+ *
+ * @return true if the image is attachable to a new owner, false if the image is still attached
+ * to its current owner, or the image is a stand-alone image and is not attachable to
+ * a new owner.
+ */
+ boolean isAttachable() {
+ return false;
+ }
+
+ /**
+ * <p>
+ * Get the owner of the {@link Image}.
+ * </p>
+ * <p>
+ * The owner of an {@link Image} could be {@link ImageReader}, {@link ImageWriter},
+ * {@link MediaCodec} etc. This method returns the owner that produces this image, or null
+ * if the image is stand-alone image or the owner is unknown.
+ * </p>
+ * <p>
+ * This is a package private method that is only used internally.
+ * </p>
+ *
+ * @return The owner of the Image.
+ */
+ Object getOwner() {
+ return null;
+ }
+ /**
* <p>A single color plane of image data.</p>
*
* <p>The number and meaning of the planes in an Image are determined by the
diff --git a/media/java/android/media/ImageReader.java b/media/java/android/media/ImageReader.java
index 18ffe12..b2f7a20 100644
--- a/media/java/android/media/ImageReader.java
+++ b/media/java/android/media/ImageReader.java
@@ -27,6 +27,7 @@
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.NioUtils;
+import java.util.concurrent.atomic.AtomicBoolean;
/**
* <p>The ImageReader class allows direct application access to image data
@@ -34,7 +35,7 @@
*
* <p>Several Android media API classes accept Surface objects as targets to
* render to, including {@link MediaPlayer}, {@link MediaCodec},
- * {@link android.hardware.camera2.CameraDevice}, and
+ * {@link android.hardware.camera2.CameraDevice}, {@link ImageWriter} and
* {@link android.renderscript.Allocation RenderScript Allocations}. The image
* sizes and formats that can be used with each source vary, and should be
* checked in the documentation for the specific API.</p>
@@ -97,10 +98,60 @@
* @see Image
*/
public static ImageReader newInstance(int width, int height, int format, int maxImages) {
+ if (format == PixelFormat.OPAQUE) {
+ throw new IllegalArgumentException("To obtain an opaque ImageReader, please use"
+ + " newOpaqueInstance rather than newInstance");
+ }
return new ImageReader(width, height, format, maxImages);
}
/**
+ * <p>
+ * Create a new opaque reader for images of the desired size.
+ * </p>
+ * <p>
+ * An opaque {@link ImageReader} produces images that are not directly
+ * accessible by the application. The application can still acquire images
+ * from an opaque image reader, and send them to the
+ * {@link android.hardware.camera2.CameraDevice camera} for reprocessing via
+ * {@link ImageWriter} interface. However, the {@link Image#getPlanes()
+ * getPlanes()} will return an empty array for opaque images. The
+ * application can check if an existing reader is an opaque reader by
+ * calling {@link #isOpaque()}.
+ * </p>
+ * <p>
+ * The {@code maxImages} parameter determines the maximum number of
+ * {@link Image} objects that can be be acquired from the
+ * {@code ImageReader} simultaneously. Requesting more buffers will use up
+ * more memory, so it is important to use only the minimum number necessary.
+ * </p>
+ * <p>
+ * The valid sizes and formats depend on the source of the image data.
+ * </p>
+ * <p>
+ * Opaque ImageReaders are more efficient to use when application access to
+ * image data is not necessary, comparing to ImageReaders using a non-opaque
+ * format such as {@link ImageFormat#YUV_420_888 YUV_420_888}.
+ * </p>
+ *
+ * @param width The default width in pixels of the Images that this reader
+ * will produce.
+ * @param height The default height in pixels of the Images that this reader
+ * will produce.
+ * @param maxImages The maximum number of images the user will want to
+ * access simultaneously. This should be as small as possible to
+ * limit memory use. Once maxImages Images are obtained by the
+ * user, one of them has to be released before a new Image will
+ * become available for access through
+ * {@link #acquireLatestImage()} or {@link #acquireNextImage()}.
+ * Must be greater than 0.
+ * @see Image
+ */
+ public static ImageReader newOpaqueInstance(int width, int height, int maxImages) {
+ return new ImageReader(width, height, PixelFormat.OPAQUE, maxImages);
+ }
+
+ /**
* @hide
*/
protected ImageReader(int width, int height, int format, int maxImages) {
@@ -197,6 +248,23 @@
}
/**
+ * <p>
+ * Check if the {@link ImageReader} is an opaque reader.
+ * </p>
+ * <p>
+ * An opaque image reader produces opaque images, see {@link Image#isOpaque}
+ * for more details.
+ * </p>
+ *
+ * @return true if the ImageReader is opaque.
+ * @see Image#isOpaque
+ * @see ImageReader#newOpaqueInstance
+ */
+ public boolean isOpaque() {
+ return mFormat == PixelFormat.OPAQUE;
+ }
+
+ /**
* <p>Get a {@link Surface} that can be used to produce {@link Image Images} for this
* {@code ImageReader}.</p>
*
@@ -457,6 +525,58 @@
}
/**
+ * <p>
+ * Remove the ownership of this image from the ImageReader.
+ * </p>
+ * <p>
+ * After this call, the ImageReader no longer owns this image, and the image
+ * ownership can be transfered to another entity like {@link ImageWriter}
+ * via {@link ImageWriter#queueInputImage}. It's up to the new owner to
+ * release the resources held by this image. For example, if the ownership
+ * of this image is transfered to an {@link ImageWriter}, the image will be
+ * freed by the ImageWriter after the image data consumption is done.
+ * </p>
+ * <p>
+ * This method can be used to achieve zero buffer copy for use cases like
+ * {@link android.hardware.camera2.CameraDevice Camera2 API} OPAQUE and YUV
+ * reprocessing, where the application can select an output image from
+ * {@link ImageReader} and transfer this image directly to
+ * {@link ImageWriter}, where this image can be consumed by camera directly.
+ * For OPAQUE reprocessing, this is the only way to send input buffers to
+ * the {@link android.hardware.camera2.CameraDevice camera} for
+ * reprocessing.
+ * </p>
+ * <p>
+ * This is a package private method that is only used internally.
+ * </p>
+ *
+ * @param image The image to be detached from this ImageReader.
+ * @throws IllegalStateException If the ImageReader or image have been
+ * closed, or the has been detached, or has not yet been
+ * acquired.
+ */
+ void detachImage(Image image) {
+ if (image == null) {
+ throw new IllegalArgumentException("input image must not be null");
+ }
+ if (!isImageOwnedbyMe(image)) {
+ throw new IllegalArgumentException("Trying to detach an image that is not owned by"
+ + " this ImageReader");
+ }
+
+ SurfaceImage si = (SurfaceImage) image;
+ if (!si.isImageValid()) {
+ throw new IllegalStateException("Image is no longer valid");
+ }
+ if (si.isAttachable()) {
+ throw new IllegalStateException("Image was already detached from this ImageReader");
+ }
+
+ nativeDetachImage(image);
+ si.setDetached(true);
+ }
+
+ /**
* Only a subset of the formats defined in
* {@link android.graphics.ImageFormat ImageFormat} and
* {@link android.graphics.PixelFormat PixelFormat} are supported by
@@ -487,12 +607,22 @@
case ImageFormat.DEPTH16:
case ImageFormat.DEPTH_POINT_CLOUD:
return 1;
+ case PixelFormat.OPAQUE:
+ return 0;
default:
throw new UnsupportedOperationException(
String.format("Invalid format specified %d", mFormat));
}
}
+ private boolean isImageOwnedbyMe(Image image) {
+ if (!(image instanceof SurfaceImage)) {
+ return false;
+ }
+ SurfaceImage si = (SurfaceImage) image;
+ return si.getReader() == this;
+ }
+
/**
* Called from Native code when an Event happens.
*
@@ -561,7 +691,11 @@
@Override
public void close() {
if (mIsImageValid) {
- ImageReader.this.releaseImage(this);
+ if (!mIsDetached.get()) {
+ // For detached images, the new owner is responsible for
+ // releasing the resources
+ ImageReader.this.releaseImage(this);
+ }
}
}
@@ -614,6 +748,15 @@
}
@Override
+ public void setTimestamp(long timestampNs) {
+ if (mIsImageValid) {
+ mTimestamp = timestampNs;
+ } else {
+ throw new IllegalStateException("Image is already released");
+ }
+ }
+
+ @Override
public Plane[] getPlanes() {
if (mIsImageValid) {
// Shallow copy is fine.
@@ -624,6 +767,11 @@
}
@Override
+ public boolean isOpaque() {
+ return mFormat == PixelFormat.OPAQUE;
+ }
+
+ @Override
protected final void finalize() throws Throwable {
try {
close();
@@ -632,6 +780,20 @@
}
}
+ @Override
+ boolean isAttachable() {
+ return mIsDetached.get();
+ }
+
+ @Override
+ ImageReader getOwner() {
+ return ImageReader.this;
+ }
+
+ private void setDetached(boolean detached) {
+ mIsDetached.getAndSet(detached);
+ }
+
private void setImageValid(boolean isValid) {
mIsImageValid = isValid;
}
@@ -734,6 +896,8 @@
private boolean mIsImageValid;
private int mHeight = -1;
private int mWidth = -1;
+ // If this image is detached from the ImageReader.
+ private AtomicBoolean mIsDetached = new AtomicBoolean(false);
private synchronized native ByteBuffer nativeImageGetBuffer(int idx, int readerFormat);
private synchronized native SurfacePlane nativeCreatePlane(int idx, int readerFormat);
@@ -746,6 +910,7 @@
private synchronized native void nativeClose();
private synchronized native void nativeReleaseImage(Image i);
private synchronized native Surface nativeGetSurface();
+ private synchronized native void nativeDetachImage(Image i);
/**
* @return A return code {@code ACQUIRE_*}
diff --git a/media/java/android/media/ImageUtils.java b/media/java/android/media/ImageUtils.java
new file mode 100644
index 0000000..89313bf3
--- /dev/null
+++ b/media/java/android/media/ImageUtils.java
@@ -0,0 +1,120 @@
+/*
+ * Copyright 2015 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.media;
+
+import android.graphics.ImageFormat;
+import android.graphics.PixelFormat;
+import android.media.Image.Plane;
+import android.util.Size;
+
+import java.nio.ByteBuffer;
+
+/**
+ * Package private utility class for hosting commonly used Image related methods.
+ */
+class ImageUtils {
+
+ /**
+ * Only a subset of the formats defined in
+ * {@link android.graphics.ImageFormat ImageFormat} and
+ * {@link android.graphics.PixelFormat PixelFormat} are supported by
+ * ImageReader. When reading RGB data from a surface, the formats defined in
+ * {@link android.graphics.PixelFormat PixelFormat} can be used; when
+ * reading YUV, JPEG or raw sensor data (for example, from the camera or video
+ * decoder), formats from {@link android.graphics.ImageFormat ImageFormat}
+ * are used.
+ */
+ public static int getNumPlanesForFormat(int format) {
+ switch (format) {
+ case ImageFormat.YV12:
+ case ImageFormat.YUV_420_888:
+ case ImageFormat.NV21:
+ return 3;
+ case ImageFormat.NV16:
+ return 2;
+ case PixelFormat.RGB_565:
+ case PixelFormat.RGBA_8888:
+ case PixelFormat.RGBX_8888:
+ case PixelFormat.RGB_888:
+ case ImageFormat.JPEG:
+ case ImageFormat.YUY2:
+ case ImageFormat.Y8:
+ case ImageFormat.Y16:
+ case ImageFormat.RAW_SENSOR:
+ case ImageFormat.RAW10:
+ return 1;
+ case PixelFormat.OPAQUE:
+ return 0;
+ default:
+ throw new UnsupportedOperationException(
+ String.format("Invalid format specified %d", format));
+ }
+ }
+
+ /**
+ * <p>
+ * Copy source image data to destination Image.
+ * </p>
+ * <p>
+ * Only support the copy between two non-opaque images with same properties
+ * (format, size, etc.). The data from the source image will be copied to
+ * the byteBuffers from the destination Image starting from position zero,
+ * and the destination image will be rewound to zero after copy is done.
+ * </p>
+ *
+ * @param src The source image to be copied from.
+ * @param dst The destination image to be copied to.
+ * @throws IllegalArgumentException If the source and destination images
+ * have different format, or one of the images is not copyable.
+ */
+ public static void imageCopy(Image src, Image dst) {
+ if (src == null || dst == null) {
+ throw new IllegalArgumentException("Images should be non-null");
+ }
+ if (src.getFormat() != dst.getFormat()) {
+ throw new IllegalArgumentException("Src and dst images should have the same format");
+ }
+ if (src.isOpaque() || dst.isOpaque()) {
+ throw new IllegalArgumentException("Opaque image is not copyable");
+ }
+ if (!(dst.getOwner() instanceof ImageWriter)) {
+ throw new IllegalArgumentException("Destination image is not from ImageWriter. Only"
+ + " the images from ImageWriter are writable");
+ }
+ Size srcSize = new Size(src.getWidth(), src.getHeight());
+ Size dstSize = new Size(dst.getWidth(), dst.getHeight());
+ if (!srcSize.equals(dstSize)) {
+ throw new IllegalArgumentException("source image size " + srcSize + " is different"
+ + " with " + "destination image size " + dstSize);
+ }
+
+ Plane[] srcPlanes = src.getPlanes();
+ Plane[] dstPlanes = dst.getPlanes();
+ ByteBuffer srcBuffer = null;
+ ByteBuffer dstBuffer = null;
+ for (int i = 0; i < srcPlanes.length; i++) {
+ srcBuffer = srcPlanes[i].getBuffer();
+ int srcPos = srcBuffer.position();
+ srcBuffer.rewind();
+ dstBuffer = dstPlanes[i].getBuffer();
+ dstBuffer.rewind();
+ dstBuffer.put(srcBuffer);
+ srcBuffer.position(srcPos);
+ dstBuffer.rewind();
+ }
+ }
+}
diff --git a/media/java/android/media/ImageWriter.java b/media/java/android/media/ImageWriter.java
new file mode 100644
index 0000000..20389a39
--- /dev/null
+++ b/media/java/android/media/ImageWriter.java
@@ -0,0 +1,798 @@
+/*
+ * Copyright 2015 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.media;
+
+import android.graphics.PixelFormat;
+import android.graphics.Rect;
+import android.os.Handler;
+import android.os.Looper;
+import android.os.Message;
+import android.view.Surface;
+
+import java.lang.ref.WeakReference;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.NioUtils;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * <p>
+ * The ImageWriter class allows an application to produce Image data into a
+ * {@link android.view.Surface}, and have it be consumed by another component like
+ * {@link android.hardware.camera2.CameraDevice CameraDevice}.
+ * </p>
+ * <p>
+ * Several Android API classes can provide input {@link android.view.Surface
+ * Surface} objects for ImageWriter to produce data into, including
+ * {@link MediaCodec MediaCodec} (encoder),
+ * {@link android.hardware.camera2.CameraDevice CameraDevice} (reprocessing
+ * input), {@link ImageReader}, etc.
+ * </p>
+ * <p>
+ * The input Image data is encapsulated in {@link Image} objects. To produce
+ * Image data into a destination {@link android.view.Surface Surface}, the
+ * application can get an input Image via {@link #dequeueInputImage} then write
+ * Image data into it. Multiple such {@link Image} objects can be dequeued at
+ * the same time and queued back in any order, up to the number specified by the
+ * {@code maxImages} constructor parameter.
+ * </p>
+ * <p>
+ * If the application already has an Image from {@link ImageReader}, the
+ * application can directly queue this Image into ImageWriter (via
+ * {@link #queueInputImage}), potentially with zero buffer copies. For the opaque
+ * Images produced by an opaque ImageReader (created by
+ * {@link ImageReader#newOpaqueInstance}), this is the only way to send Image
+ * data to ImageWriter, as the Image data aren't accessible by the application.
+ * </p>
+ * Once new input Images are queued into an ImageWriter, it's up to the downstream
+ * components (e.g. {@link ImageReader} or
+ * {@link android.hardware.camera2.CameraDevice}) to consume the Images. If the
+ * downstream components cannot consume the Images at least as fast as the
+ * ImageWriter production rate, the {@link #dequeueInputImage} call will eventually
+ * block and the application will have to drop input frames. </p>
+ */
+public class ImageWriter implements AutoCloseable {
+ private final Object mListenerLock = new Object();
+ private ImageListener mListener;
+ private ListenerHandler mListenerHandler;
+ private long mNativeContext;
+
+ // Field below is used by native code, do not access or modify.
+ private int mWriterFormat;
+
+ private final int mMaxImages;
+ // Keep track of the currently attached Image; or an attached Image that is
+ // released will be removed from this list.
+ private List<Image> mAttachedImages = new ArrayList<Image>();
+ private List<Image> mDequeuedImages = new ArrayList<Image>();
+
+ /**
+ * <p>
+ * Create a new ImageWriter.
+ * </p>
+ * <p>
+ * The {@code maxImages} parameter determines the maximum number of
+ * {@link Image} objects that can be be dequeued from the
+ * {@code ImageWriter} simultaneously. Requesting more buffers will use up
+ * more memory, so it is important to use only the minimum number necessary.
+ * </p>
+ * <p>
+ * The input Image size and format depend on the Surface that is provided by
+ * the downstream consumer end-point.
+ * </p>
+ *
+ * @param surface The destination Surface this writer produces Image data
+ * into.
+ * @param maxImages The maximum number of Images the user will want to
+ * access simultaneously for producing Image data. This should be
+ * as small as possible to limit memory use. Once maxImages
+ * Images are dequeued by the user, one of them has to be queued
+ * back before a new Image can be dequeued for access via
+ * {@link #dequeueInputImage()}.
+ * @return a new ImageWriter instance.
+ */
+ public static ImageWriter newInstance(Surface surface, int maxImages) {
+ return new ImageWriter(surface, maxImages);
+ }
+
+ /**
+ * @hide
+ */
+ protected ImageWriter(Surface surface, int maxImages) {
+ if (surface == null || maxImages < 1) {
+ throw new IllegalArgumentException("Illegal input argument: surface " + surface
+ + ", maxImages: " + maxImages);
+ }
+
+ mMaxImages = maxImages;
+ // Note that the underlying BufferQueue is working in synchronous mode
+ // to avoid dropping any buffers.
+ mNativeContext = nativeInit(new WeakReference<ImageWriter>(this), surface, maxImages);
+ }
+
+ /**
+ * <p>
+ * Maximum number of Images that can be dequeued from the ImageWriter
+ * simultaneously (for example, with {@link #dequeueInputImage()}).
+ * </p>
+ * <p>
+ * An Image is considered dequeued after it's returned by
+ * {@link #dequeueInputImage()} from ImageWriter, and until the Image is
+ * sent back to ImageWriter via {@link #queueInputImage}, or
+ * {@link Image#close()}.
+ * </p>
+ * <p>
+ * Attempting to dequeue more than {@code maxImages} concurrently will
+ * result in the {@link #dequeueInputImage()} function throwing an
+ * {@link IllegalStateException}.
+ * </p>
+ *
+ * @return Maximum number of Images that can be dequeued from this
+ * ImageWriter.
+ * @see #dequeueInputImage
+ * @see #queueInputImage
+ * @see Image#close
+ */
+ public int getMaxImages() {
+ return mMaxImages;
+ }
+
+ /**
+ * <p>
+ * Dequeue the next available input Image for the application to produce
+ * data into.
+ * </p>
+ * <p>
+ * This method requests a new input Image from ImageWriter. The application
+ * owns this Image after this call. Once the application fills the Image
+ * data, it is expected to return this Image back to ImageWriter for
+ * downstream consumer components (e.g.
+ * {@link android.hardware.camera2.CameraDevice}) to consume. The Image can
+ * be returned to ImageWriter via {@link #queueInputImage} or
+ * {@link Image#close()}.
+ * </p>
+ * <p>
+ * This call will block if all available input images have been filled by
+ * the application and the downstream consumer has not yet consumed any.
+ * When an Image is consumed by the downstream consumer, an
+ * {@link ImageListener#onInputImageReleased} callback will be fired, which
+ * indicates that there is one input Image available. It is recommended to
+ * dequeue next Image only after this callback is fired, in the steady state.
+ * </p>
+ *
+ * @return The next available input Image from this ImageWriter.
+ * @throws IllegalStateException if {@code maxImages} Images are currently
+ * dequeued.
+ * @see #queueInputImage
+ * @see Image#close
+ */
+ public Image dequeueInputImage() {
+ if (mDequeuedImages.size() >= mMaxImages) {
+ throw new IllegalStateException("Already dequeued max number of Images " + mMaxImages);
+ }
+ WriterSurfaceImage newImage = new WriterSurfaceImage(this);
+ nativeDequeueInputImage(mNativeContext, newImage);
+ mDequeuedImages.add(newImage);
+ newImage.setImageValid(true);
+ return newImage;
+ }
+
+ /**
+ * <p>
+ * Queue an input {@link Image} back to ImageWriter for the downstream
+ * consumer to access.
+ * </p>
+ * <p>
+ * The input {@link Image} could be from ImageReader (acquired via
+ * {@link ImageReader#acquireNextImage} or
+ * {@link ImageReader#acquireLatestImage}), or from this ImageWriter
+ * (acquired via {@link #dequeueInputImage}). In the former case, the Image
+ * data will be moved to this ImageWriter. Note that the Image properties
+ * (size, format, strides, etc.) must be the same as the properties of the
+ * images dequeued from this ImageWriter, or this method will throw an
+ * {@link IllegalArgumentException}. In the latter case, the application has
+ * filled the input image with data. This method then passes the filled
+ * buffer to the downstream consumer. In both cases, it's up to the caller
+ * to ensure that the Image timestamp (in nanoseconds) is correctly set, as
+ * the downstream component may want to use it to indicate the Image data
+ * capture time.
+ * </p>
+ * <p>
+ * Passing in a non-opaque Image may result in a memory copy, which also
+ * requires a free input Image from this ImageWriter as the destination. In
+ * this case, this call will block, as {@link #dequeueInputImage} does, if
+ * there are no free Images available. To be safe, the application should ensure
+ * that there is at least one free Image available in this ImageWriter before calling
+ * this method.
+ * </p>
+ * <p>
+ * After this call, the input Image is no longer valid for further access,
+ * as if the Image is {@link Image#close closed}. Attempting to access the
+ * {@link ByteBuffer ByteBuffers} returned by an earlier
+ * {@link Image.Plane#getBuffer Plane#getBuffer} call will result in an
+ * {@link IllegalStateException}.
+ * </p>
+ *
+ * @param image The Image to be queued back to ImageWriter for future
+ * consumption.
+ * @see #dequeueInputImage()
+ */
+ public void queueInputImage(Image image) {
+ if (image == null) {
+ throw new IllegalArgumentException("image shouldn't be null");
+ }
+ boolean ownedByMe = isImageOwnedByMe(image);
+ if (ownedByMe && !(((WriterSurfaceImage) image).isImageValid())) {
+ throw new IllegalStateException("Image from ImageWriter is invalid");
+ }
+
+ // For images from other components, need to detach first, then attach.
+ if (!ownedByMe) {
+ if (!(image.getOwner() instanceof ImageReader)) {
+ throw new IllegalArgumentException("Only images from ImageReader can be queued to"
+ + " ImageWriter, other image source is not supported yet!");
+ }
+
+ ImageReader prevOwner = (ImageReader) image.getOwner();
+ // Only do the image attach for opaque images for now. Do the image
+ // copy for other formats. TODO: use attach for other formats to
+ // improve the performance, and fall back to copy when attach/detach fails.
+ if (image.isOpaque()) {
+ prevOwner.detachImage(image);
+ attachInputImage(image);
+ } else {
+ Image inputImage = dequeueInputImage();
+ inputImage.setTimestamp(image.getTimestamp());
+ inputImage.setCropRect(image.getCropRect());
+ ImageUtils.imageCopy(image, inputImage);
+ image.close();
+ image = inputImage;
+ ownedByMe = true;
+ }
+ }
+
+ Rect crop = image.getCropRect();
+ nativeQueueInputImage(mNativeContext, image, image.getTimestamp(), crop.left, crop.top,
+ crop.right, crop.bottom);
+
+ /**
+ * Only remove and cleanup the Images that are owned by this
+ * ImageWriter. Images detached from other owners are only
+ * temporarily owned by this ImageWriter and will be detached immediately
+ * after they are released by downstream consumers, so there is no need to
+ * keep track of them in mDequeuedImages.
+ */
+ if (ownedByMe) {
+ mDequeuedImages.remove(image);
+ WriterSurfaceImage wi = (WriterSurfaceImage) image;
+ wi.clearSurfacePlanes();
+ wi.setImageValid(false);
+ } else {
+ // This clears the native reference held by the original owner. When
+ // this Image is detached later by this ImageWriter, the native
+ // memory won't be leaked.
+ image.close();
+ }
+ }
+
+ /**
+ * ImageWriter callback interface, used to to asynchronously notify the
+ * application of various ImageWriter events.
+ */
+ public interface ImageListener {
+ /**
+ * <p>
+ * Callback that is called when an input Image is released back to
+ * ImageWriter after the data consumption.
+ * </p>
+ * <p>
+ * The client can use this callback to indicate either an input Image is
+ * available to fill data into, or the input Image is returned and freed
+ * if it was attached from other components (e.g. an
+ * {@link ImageReader}). For the latter case, the ownership of the Image
+ * will be automatically removed by ImageWriter right before this
+ * callback is fired.
+ * </p>
+ *
+ * @param writer the ImageWriter the callback is associated with.
+ * @see ImageWriter
+ * @see Image
+ */
+ // TODO: the semantics is confusion, does't tell which buffer is
+ // released if an application is doing queueInputImage with a mix of
+ // buffers from dequeueInputImage and from an ImageReader. see b/19872821
+ void onInputImageReleased(ImageWriter writer);
+ }
+
+ /**
+ * Register a listener to be invoked when an input Image is returned to
+ * the ImageWriter.
+ *
+ * @param listener The listener that will be run.
+ * @param handler The handler on which the listener should be invoked, or
+ * null if the listener should be invoked on the calling thread's
+ * looper.
+ * @throws IllegalArgumentException If no handler specified and the calling
+ * thread has no looper.
+ */
+ public void setImageListener(ImageListener listener, Handler handler) {
+ synchronized (mListenerLock) {
+ if (listener != null) {
+ Looper looper = handler != null ? handler.getLooper() : Looper.myLooper();
+ if (looper == null) {
+ throw new IllegalArgumentException(
+ "handler is null but the current thread is not a looper");
+ }
+ if (mListenerHandler == null || mListenerHandler.getLooper() != looper) {
+ mListenerHandler = new ListenerHandler(looper);
+ }
+ mListener = listener;
+ } else {
+ mListener = null;
+ mListenerHandler = null;
+ }
+ }
+ }
+
+ /**
+ * Free up all the resources associated with this ImageWriter.
+ * <p>
+ * After calling this method, this ImageWriter cannot be used. Calling any
+ * methods on this ImageWriter and Images previously provided by
+ * {@link #dequeueInputImage()} will result in an
+ * {@link IllegalStateException}, and attempting to write into
+ * {@link ByteBuffer ByteBuffers} returned by an earlier
+ * {@link Image.Plane#getBuffer Plane#getBuffer} call will have undefined
+ * behavior.
+ * </p>
+ */
+ @Override
+ public void close() {
+ setImageListener(null, null);
+ for (Image image : mDequeuedImages) {
+ image.close();
+ }
+ mDequeuedImages.clear();
+ nativeClose(mNativeContext);
+ mNativeContext = 0;
+ }
+
+ @Override
+ protected void finalize() throws Throwable {
+ try {
+ close();
+ } finally {
+ super.finalize();
+ }
+ }
+
+ /**
+ * Get the ImageWriter format.
+ * <p>
+ * This format may be different than the Image format returned by
+ * {@link Image#getFormat()}
+ * </p>
+ *
+ * @return The ImageWriter format.
+ */
+ int getFormat() {
+ return mWriterFormat;
+ }
+
+
+ /**
+ * <p>
+ * Attach input Image to this ImageWriter.
+ * </p>
+ * <p>
+ * When an Image is from an opaque source (e.g. an opaque ImageReader created
+ * by {@link ImageReader#newOpaqueInstance}), or the source Image is so large
+ * that copying its data is too expensive, this method can be used to
+ * migrate the source Image into ImageWriter without a data copy. The source
+ * Image must be detached from its previous owner already, or this call will
+ * throw an {@link IllegalStateException}.
+ * </p>
+ * <p>
+ * After this call, the ImageWriter takes ownership of this Image.
+ * This ownership will be automatically removed from this writer after the
+ * consumer releases this Image, that is, after
+ * {@link ImageListener#onInputImageReleased}. The caller is
+ * responsible for closing this Image through {@link Image#close()} to free up
+ * the resources held by this Image.
+ * </p>
+ *
+ * @param image The source Image to be attached and queued into this
+ * ImageWriter for downstream consumer to use.
+ * @throws IllegalStateException if the Image is not detached from its
+ * previous owner, or the Image is already attached to this
+ * ImageWriter, or the source Image is invalid.
+ */
+ private void attachInputImage(Image image) {
+ if (image == null) {
+ throw new IllegalArgumentException("image shouldn't be null");
+ }
+ if (isImageOwnedByMe(image)) {
+ throw new IllegalArgumentException(
+ "Can not attach an image that is owned ImageWriter already");
+ }
+ /**
+ * Throw ISE if the image is not attachable, which means that it is
+ * either owned by other entity now, or completely non-attachable (some
+ * stand-alone images are not backed by native gralloc buffer, thus not
+ * attachable).
+ */
+ if (!image.isAttachable()) {
+ throw new IllegalStateException("Image was not detached from last owner, or image "
+ + " is not detachable");
+ }
+ if (mAttachedImages.contains(image)) {
+ throw new IllegalStateException("Image was already attached to ImageWritter");
+ }
+
+ // TODO: what if attach failed, throw RTE or detach a slot then attach?
+ // need do some cleanup to make sure no orphaned
+ // buffer caused leak.
+ nativeAttachImage(mNativeContext, image);
+ mAttachedImages.add(image);
+ }
+
+ /**
+ * This custom handler runs asynchronously so callbacks don't get queued
+ * behind UI messages.
+ */
+ private final class ListenerHandler extends Handler {
+ public ListenerHandler(Looper looper) {
+ super(looper, null, true /* async */);
+ }
+
+ @Override
+ public void handleMessage(Message msg) {
+ ImageListener listener;
+ synchronized (mListenerLock) {
+ listener = mListener;
+ }
+ // TODO: detach Image from ImageWriter and remove the Image from
+ // mAttachedImage list.
+ if (listener != null) {
+ listener.onInputImageReleased(ImageWriter.this);
+ }
+ }
+ }
+
+ /**
+ * Called from Native code when an Event happens. This may be called from an
+ * arbitrary Binder thread, so access to the ImageWriter must be
+ * synchronized appropriately.
+ */
+ private static void postEventFromNative(Object selfRef) {
+ @SuppressWarnings("unchecked")
+ WeakReference<ImageWriter> weakSelf = (WeakReference<ImageWriter>) selfRef;
+ final ImageWriter iw = weakSelf.get();
+ if (iw == null) {
+ return;
+ }
+
+ final Handler handler;
+ synchronized (iw.mListenerLock) {
+ handler = iw.mListenerHandler;
+ }
+ if (handler != null) {
+ handler.sendEmptyMessage(0);
+ }
+ }
+
+ /**
+ * <p>
+ * Abort the Images that were dequeued from this ImageWriter, and return
+ * them to this writer for reuse.
+ * </p>
+ * <p>
+ * This method is used for the cases where the application dequeued the
+ * Image, may have filled the data, but does not want the downstream
+ * component to consume it. The Image will be returned to this ImageWriter
+ * for reuse after this call, and the ImageWriter will immediately have an
+ * Image available to be dequeued. This aborted Image will be invisible to
+ * the downstream consumer, as if nothing happened.
+ * </p>
+ *
+ * @param image The Image to be aborted.
+ * @see #dequeueInputImage()
+ * @see Image#close()
+ */
+ private void abortImage(Image image) {
+ if (image == null) {
+ throw new IllegalArgumentException("image shouldn't be null");
+ }
+
+ if (!mDequeuedImages.contains(image)) {
+ throw new IllegalStateException("It is illegal to abort some image that is not"
+ + " dequeued yet");
+ }
+
+ WriterSurfaceImage wi = (WriterSurfaceImage) image;
+
+ if (!wi.isImageValid()) {
+ throw new IllegalStateException("Image is invalid");
+ }
+
+ /**
+ * We only need abort Images that are owned and dequeued by ImageWriter.
+ * For attached Images, no need to abort, as there are only two cases:
+ * attached + queued successfully, and attach failed. Neither of the
+ * cases need abort.
+ */
+ cancelImage(mNativeContext,image);
+ mDequeuedImages.remove(image);
+ wi.clearSurfacePlanes();
+ wi.setImageValid(false);
+ }
+
+ private boolean isImageOwnedByMe(Image image) {
+ if (!(image instanceof WriterSurfaceImage)) {
+ return false;
+ }
+ WriterSurfaceImage wi = (WriterSurfaceImage) image;
+ if (wi.getOwner() != this) {
+ return false;
+ }
+
+ return true;
+ }
+
+ private static class WriterSurfaceImage extends android.media.Image {
+ private ImageWriter mOwner;
+ private AtomicBoolean mIsImageValid = new AtomicBoolean(false);
+ // This field is used by native code, do not access or modify.
+ private long mNativeBuffer;
+ private int mNativeFenceFd = -1;
+ private SurfacePlane[] mPlanes;
+ private int mHeight = -1;
+ private int mWidth = -1;
+ private int mFormat = -1;
+ // When this default timestamp is used, timestamp for the input Image
+ // will be generated automatically when queueInputBuffer is called.
+ private final long DEFAULT_TIMESTAMP = Long.MIN_VALUE;
+ private long mTimestamp = DEFAULT_TIMESTAMP;
+
+ public WriterSurfaceImage(ImageWriter writer) {
+ mOwner = writer;
+ }
+
+ @Override
+ public int getFormat() {
+ if (!mIsImageValid.get()) {
+ throw new IllegalStateException("Image is already released");
+ }
+ if (mFormat == -1) {
+ mFormat = nativeGetFormat();
+ }
+ return mFormat;
+ }
+
+ @Override
+ public int getWidth() {
+ if (!mIsImageValid.get()) {
+ throw new IllegalStateException("Image is already released");
+ }
+
+ if (mWidth == -1) {
+ mWidth = nativeGetWidth();
+ }
+
+ return mWidth;
+ }
+
+ @Override
+ public int getHeight() {
+ if (!mIsImageValid.get()) {
+ throw new IllegalStateException("Image is already released");
+ }
+
+ if (mHeight == -1) {
+ mHeight = nativeGetHeight();
+ }
+
+ return mHeight;
+ }
+
+ @Override
+ public long getTimestamp() {
+ if (!mIsImageValid.get()) {
+ throw new IllegalStateException("Image is already released");
+ }
+
+ return mTimestamp;
+ }
+
+ @Override
+ public void setTimestamp(long timestamp) {
+ if (!mIsImageValid.get()) {
+ throw new IllegalStateException("Image is already released");
+ }
+
+ mTimestamp = timestamp;
+ }
+
+ @Override
+ public boolean isOpaque() {
+ if (!mIsImageValid.get()) {
+ throw new IllegalStateException("Image is already released");
+ }
+
+ return getFormat() == PixelFormat.OPAQUE;
+ }
+
+ @Override
+ public Plane[] getPlanes() {
+ if (!mIsImageValid.get()) {
+ throw new IllegalStateException("Image is already released");
+ }
+
+ if (mPlanes == null) {
+ int numPlanes = ImageUtils.getNumPlanesForFormat(getFormat());
+ mPlanes = nativeCreatePlanes(numPlanes, getOwner().getFormat());
+ }
+
+ return mPlanes.clone();
+ }
+
+ @Override
+ boolean isAttachable() {
+ if (!mIsImageValid.get()) {
+ throw new IllegalStateException("Image is already released");
+ }
+ // Don't allow Image to be detached from ImageWriter for now, as no
+ // detach API is exposed.
+ return false;
+ }
+
+ @Override
+ ImageWriter getOwner() {
+ return mOwner;
+ }
+
+ @Override
+ public void close() {
+ if (mIsImageValid.get()) {
+ getOwner().abortImage(this);
+ }
+ }
+
+ @Override
+ protected final void finalize() throws Throwable {
+ try {
+ close();
+ } finally {
+ super.finalize();
+ }
+ }
+
+ private boolean isImageValid() {
+ return mIsImageValid.get();
+ }
+
+ private void setImageValid(boolean isValid) {
+ mIsImageValid.getAndSet(isValid);
+ }
+
+ private void clearSurfacePlanes() {
+ if (mIsImageValid.get()) {
+ for (int i = 0; i < mPlanes.length; i++) {
+ if (mPlanes[i] != null) {
+ mPlanes[i].clearBuffer();
+ mPlanes[i] = null;
+ }
+ }
+ }
+ }
+
+ private class SurfacePlane extends android.media.Image.Plane {
+ private ByteBuffer mBuffer;
+ final private int mPixelStride;
+ final private int mRowStride;
+
+ // SurfacePlane instance is created by native code when a new
+ // SurfaceImage is created
+ private SurfacePlane(int rowStride, int pixelStride, ByteBuffer buffer) {
+ mRowStride = rowStride;
+ mPixelStride = pixelStride;
+ mBuffer = buffer;
+ /**
+ * Set the byteBuffer order according to host endianness (native
+ * order), otherwise, the byteBuffer order defaults to
+ * ByteOrder.BIG_ENDIAN.
+ */
+ mBuffer.order(ByteOrder.nativeOrder());
+ }
+
+ @Override
+ public int getRowStride() {
+ if (WriterSurfaceImage.this.isImageValid() == false) {
+ throw new IllegalStateException("Image is already released");
+ }
+ return mRowStride;
+ }
+
+ @Override
+ public int getPixelStride() {
+ if (WriterSurfaceImage.this.isImageValid() == false) {
+ throw new IllegalStateException("Image is already released");
+ }
+ return mPixelStride;
+ }
+
+ @Override
+ public ByteBuffer getBuffer() {
+ if (WriterSurfaceImage.this.isImageValid() == false) {
+ throw new IllegalStateException("Image is already released");
+ }
+
+ return mBuffer;
+ }
+
+ private void clearBuffer() {
+ // Need null check first, as the getBuffer() may not be called
+ // before an Image is closed.
+ if (mBuffer == null) {
+ return;
+ }
+
+ if (mBuffer.isDirect()) {
+ NioUtils.freeDirectBuffer(mBuffer);
+ }
+ mBuffer = null;
+ }
+
+ }
+
+ // this will create the SurfacePlane object and fill the information
+ private synchronized native SurfacePlane[] nativeCreatePlanes(int numPlanes, int writerFmt);
+
+ private synchronized native int nativeGetWidth();
+
+ private synchronized native int nativeGetHeight();
+
+ private synchronized native int nativeGetFormat();
+ }
+
+ // Native implemented ImageWriter methods.
+ private synchronized native long nativeInit(Object weakSelf, Surface surface, int maxImgs);
+
+ private synchronized native void nativeClose(long nativeCtx);
+
+ private synchronized native void nativeAttachImage(long nativeCtx, Image image);
+
+ private synchronized native void nativeDequeueInputImage(long nativeCtx, Image wi);
+
+ private synchronized native void nativeQueueInputImage(long nativeCtx, Image image,
+ long timestampNs, int left, int top, int right, int bottom);
+
+ private synchronized native void cancelImage(long nativeCtx, Image image);
+
+ /**
+ * We use a class initializer to allow the native code to cache some field
+ * offsets.
+ */
+ private static native void nativeClassInit();
+
+ static {
+ System.loadLibrary("media_jni");
+ nativeClassInit();
+ }
+}
diff --git a/media/jni/Android.mk b/media/jni/Android.mk
index 4ebbe26..dae57a8 100644
--- a/media/jni/Android.mk
+++ b/media/jni/Android.mk
@@ -2,6 +2,7 @@
include $(CLEAR_VARS)
LOCAL_SRC_FILES:= \
+ android_media_ImageWriter.cpp \
android_media_ImageReader.cpp \
android_media_MediaCrypto.cpp \
android_media_MediaCodec.cpp \
diff --git a/media/jni/android_media_ImageReader.cpp b/media/jni/android_media_ImageReader.cpp
index b247493..9fc7e8e 100644
--- a/media/jni/android_media_ImageReader.cpp
+++ b/media/jni/android_media_ImageReader.cpp
@@ -860,6 +860,25 @@
return ACQUIRE_SUCCESS;
}
+static void ImageReader_detachImage(JNIEnv* env, jobject thiz, jobject image) {
+ ALOGV("%s:", __FUNCTION__);
+ JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz);
+ if (ctx == NULL) {
+ jniThrowException(env, "java/lang/IllegalStateException", "ImageReader was already closed");
+ return;
+ }
+
+ // CpuConsumer* consumer = ctx->getCpuConsumer();
+ CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, image);
+ if (!buffer) {
+ ALOGW("Image already released!!!");
+ return;
+ }
+
+ // TODO: need implement
+ jniThrowRuntimeException(env, "nativeDetachImage is not implemented yet!!!");
+}
+
static jobject ImageReader_getSurface(JNIEnv* env, jobject thiz)
{
ALOGV("%s: ", __FUNCTION__);
@@ -961,6 +980,7 @@
{"nativeReleaseImage", "(Landroid/media/Image;)V", (void*)ImageReader_imageRelease },
{"nativeImageSetup", "(Landroid/media/Image;)I", (void*)ImageReader_imageSetup },
{"nativeGetSurface", "()Landroid/view/Surface;", (void*)ImageReader_getSurface },
+ {"nativeDetachImage", "(Landroid/media/Image;)V", (void*)ImageReader_detachImage },
};
static JNINativeMethod gImageMethods[] = {
diff --git a/media/jni/android_media_ImageWriter.cpp b/media/jni/android_media_ImageWriter.cpp
new file mode 100644
index 0000000..1433c79
--- /dev/null
+++ b/media/jni/android_media_ImageWriter.cpp
@@ -0,0 +1,1014 @@
+/*
+ * Copyright 2015 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+//#define LOG_NDEBUG 0
+#define LOG_TAG "ImageWriter_JNI"
+#include <utils/Log.h>
+#include <utils/String8.h>
+
+#include <gui/IProducerListener.h>
+#include <gui/Surface.h>
+#include <gui/CpuConsumer.h>
+#include <android_runtime/AndroidRuntime.h>
+#include <android_runtime/android_view_Surface.h>
+#include <camera3.h>
+
+#include <jni.h>
+#include <JNIHelp.h>
+
+#include <stdint.h>
+#include <inttypes.h>
+
+#define ALIGN(x, mask) ( ((x) + (mask) - 1) & ~((mask) - 1) )
+
+#define IMAGE_BUFFER_JNI_ID "mNativeBuffer"
+
+// ----------------------------------------------------------------------------
+
+using namespace android;
+
+enum {
+ IMAGE_WRITER_MAX_NUM_PLANES = 3,
+};
+
+static struct {
+ jmethodID postEventFromNative;
+ jfieldID mWriterFormat;
+} gImageWriterClassInfo;
+
+static struct {
+ jfieldID mNativeBuffer;
+ jfieldID mNativeFenceFd;
+ jfieldID mPlanes;
+} gSurfaceImageClassInfo;
+
+static struct {
+ jclass clazz;
+ jmethodID ctor;
+} gSurfacePlaneClassInfo;
+
+typedef CpuConsumer::LockedBuffer LockedImage;
+
+// ----------------------------------------------------------------------------
+
+class JNIImageWriterContext : public BnProducerListener {
+public:
+ JNIImageWriterContext(JNIEnv* env, jobject weakThiz, jclass clazz);
+
+ virtual ~JNIImageWriterContext();
+
+ // Implementation of IProducerListener, used to notify the ImageWriter that the consumer
+ // has returned a buffer and it is ready for ImageWriter to dequeue.
+ virtual void onBufferReleased();
+
+ void setProducer(const sp<ANativeWindow>& producer) { mProducer = producer; }
+ ANativeWindow* getProducer() { return mProducer.get(); }
+
+ void setBufferFormat(int format) { mFormat = format; }
+ int getBufferFormat() { return mFormat; }
+
+ void setBufferWidth(int width) { mWidth = width; }
+ int getBufferWidth() { return mWidth; }
+
+ void setBufferHeight(int height) { mHeight = height; }
+ int getBufferHeight() { return mHeight; }
+
+private:
+ static JNIEnv* getJNIEnv(bool* needsDetach);
+ static void detachJNI();
+
+ sp<ANativeWindow> mProducer;
+ jobject mWeakThiz;
+ jclass mClazz;
+ int mFormat;
+ int mWidth;
+ int mHeight;
+};
+
+JNIImageWriterContext::JNIImageWriterContext(JNIEnv* env, jobject weakThiz, jclass clazz) :
+ mWeakThiz(env->NewGlobalRef(weakThiz)),
+ mClazz((jclass)env->NewGlobalRef(clazz)),
+ mFormat(0),
+ mWidth(-1),
+ mHeight(-1) {
+}
+
+JNIImageWriterContext::~JNIImageWriterContext() {
+ ALOGV("%s", __FUNCTION__);
+ bool needsDetach = false;
+ JNIEnv* env = getJNIEnv(&needsDetach);
+ if (env != NULL) {
+ env->DeleteGlobalRef(mWeakThiz);
+ env->DeleteGlobalRef(mClazz);
+ } else {
+ ALOGW("leaking JNI object references");
+ }
+ if (needsDetach) {
+ detachJNI();
+ }
+
+ mProducer.clear();
+}
+
+JNIEnv* JNIImageWriterContext::getJNIEnv(bool* needsDetach) {
+ ALOGV("%s", __FUNCTION__);
+ LOG_ALWAYS_FATAL_IF(needsDetach == NULL, "needsDetach is null!!!");
+ *needsDetach = false;
+ JNIEnv* env = AndroidRuntime::getJNIEnv();
+ if (env == NULL) {
+ JavaVMAttachArgs args = {JNI_VERSION_1_4, NULL, NULL};
+ JavaVM* vm = AndroidRuntime::getJavaVM();
+ int result = vm->AttachCurrentThread(&env, (void*) &args);
+ if (result != JNI_OK) {
+ ALOGE("thread attach failed: %#x", result);
+ return NULL;
+ }
+ *needsDetach = true;
+ }
+ return env;
+}
+
+void JNIImageWriterContext::detachJNI() {
+ ALOGV("%s", __FUNCTION__);
+ JavaVM* vm = AndroidRuntime::getJavaVM();
+ int result = vm->DetachCurrentThread();
+ if (result != JNI_OK) {
+ ALOGE("thread detach failed: %#x", result);
+ }
+}
+
+void JNIImageWriterContext::onBufferReleased() {
+ ALOGV("%s: buffer released", __FUNCTION__);
+ bool needsDetach = false;
+ JNIEnv* env = getJNIEnv(&needsDetach);
+ if (env != NULL) {
+ env->CallStaticVoidMethod(mClazz, gImageWriterClassInfo.postEventFromNative, mWeakThiz);
+ } else {
+ ALOGW("onBufferReleased event will not posted");
+ }
+ if (needsDetach) {
+ detachJNI();
+ }
+}
+
+// ----------------------------------------------------------------------------
+
+extern "C" {
+
+// -------------------------------Private method declarations--------------
+
+static bool isWritable(int format);
+static bool isPossiblyYUV(PixelFormat format);
+static void Image_setNativeContext(JNIEnv* env, jobject thiz,
+ sp<GraphicBuffer> buffer, int fenceFd);
+static void Image_getNativeContext(JNIEnv* env, jobject thiz,
+ GraphicBuffer** buffer, int* fenceFd);
+static void Image_unlockIfLocked(JNIEnv* env, jobject thiz);
+
+// --------------------------ImageWriter methods---------------------------------------
+
+static void ImageWriter_classInit(JNIEnv* env, jclass clazz) {
+ ALOGV("%s:", __FUNCTION__);
+ jclass imageClazz = env->FindClass("android/media/ImageWriter$WriterSurfaceImage");
+ LOG_ALWAYS_FATAL_IF(imageClazz == NULL,
+ "can't find android/media/ImageWriter$WriterSurfaceImage");
+ gSurfaceImageClassInfo.mNativeBuffer = env->GetFieldID(
+ imageClazz, IMAGE_BUFFER_JNI_ID, "J");
+ LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mNativeBuffer == NULL,
+ "can't find android/media/ImageWriter$WriterSurfaceImage.%s", IMAGE_BUFFER_JNI_ID);
+
+ gSurfaceImageClassInfo.mNativeFenceFd = env->GetFieldID(
+ imageClazz, "mNativeFenceFd", "I");
+ LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mNativeFenceFd == NULL,
+ "can't find android/media/ImageWriter$WriterSurfaceImage.mNativeFenceFd");
+
+ gSurfaceImageClassInfo.mPlanes = env->GetFieldID(
+ imageClazz, "mPlanes", "[Landroid/media/ImageWriter$WriterSurfaceImage$SurfacePlane;");
+ LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mPlanes == NULL,
+ "can't find android/media/ImageWriter$WriterSurfaceImage.mPlanes");
+
+ gImageWriterClassInfo.postEventFromNative = env->GetStaticMethodID(
+ clazz, "postEventFromNative", "(Ljava/lang/Object;)V");
+ LOG_ALWAYS_FATAL_IF(gImageWriterClassInfo.postEventFromNative == NULL,
+ "can't find android/media/ImageWriter.postEventFromNative");
+
+ gImageWriterClassInfo.mWriterFormat = env->GetFieldID(
+ clazz, "mWriterFormat", "I");
+ LOG_ALWAYS_FATAL_IF(gImageWriterClassInfo.mWriterFormat == NULL,
+ "can't find android/media/ImageWriter.mWriterFormat");
+
+ jclass planeClazz = env->FindClass("android/media/ImageWriter$WriterSurfaceImage$SurfacePlane");
+ LOG_ALWAYS_FATAL_IF(planeClazz == NULL, "Can not find SurfacePlane class");
+ // FindClass only gives a local reference of jclass object.
+ gSurfacePlaneClassInfo.clazz = (jclass) env->NewGlobalRef(planeClazz);
+ gSurfacePlaneClassInfo.ctor = env->GetMethodID(gSurfacePlaneClassInfo.clazz, "<init>",
+ "(Landroid/media/ImageWriter$WriterSurfaceImage;IILjava/nio/ByteBuffer;)V");
+ LOG_ALWAYS_FATAL_IF(gSurfacePlaneClassInfo.ctor == NULL,
+ "Can not find SurfacePlane constructor");
+}
+
+static jlong ImageWriter_init(JNIEnv* env, jobject thiz, jobject weakThiz, jobject jsurface,
+ jint maxImages) {
+ status_t res;
+
+ ALOGV("%s: maxImages:%d", __FUNCTION__, maxImages);
+
+ sp<Surface> surface(android_view_Surface_getSurface(env, jsurface));
+ if (surface == NULL) {
+ jniThrowException(env,
+ "java/lang/IllegalArgumentException",
+ "The surface has been released");
+ return 0;
+ }
+ sp<IGraphicBufferProducer> bufferProducer = surface->getIGraphicBufferProducer();
+
+ jclass clazz = env->GetObjectClass(thiz);
+ if (clazz == NULL) {
+ jniThrowRuntimeException(env, "Can't find android/graphics/ImageWriter");
+ return 0;
+ }
+ sp<JNIImageWriterContext> ctx(new JNIImageWriterContext(env, weakThiz, clazz));
+
+ sp<Surface> producer = new Surface(bufferProducer, /*controlledByApp*/false);
+ ctx->setProducer(producer);
+ /**
+ * NATIVE_WINDOW_API_CPU isn't a good choice here, as it makes the bufferQueue not connectable
+ * after disconnect. MEDIA or CAMERA are treated the same internally. The producer listener
+ * will be cleared after disconnect call.
+ */
+ producer->connect(/*api*/NATIVE_WINDOW_API_CAMERA, /*listener*/ctx);
+ jlong nativeCtx = reinterpret_cast<jlong>(ctx.get());
+
+ // Get the dimension and format of the producer.
+ sp<ANativeWindow> anw = producer;
+ int32_t width, height, format;
+ if ((res = anw->query(anw.get(), NATIVE_WINDOW_WIDTH, &width)) != OK) {
+ ALOGE("%s: Query Surface width failed: %s (%d)", __FUNCTION__, strerror(-res), res);
+ jniThrowRuntimeException(env, "Failed to query Surface width");
+ return 0;
+ }
+ ctx->setBufferWidth(width);
+
+ if ((res = anw->query(anw.get(), NATIVE_WINDOW_HEIGHT, &height)) != OK) {
+ ALOGE("%s: Query Surface height failed: %s (%d)", __FUNCTION__, strerror(-res), res);
+ jniThrowRuntimeException(env, "Failed to query Surface height");
+ return 0;
+ }
+ ctx->setBufferHeight(height);
+
+ if ((res = anw->query(anw.get(), NATIVE_WINDOW_FORMAT, &format)) != OK) {
+ ALOGE("%s: Query Surface format failed: %s (%d)", __FUNCTION__, strerror(-res), res);
+ jniThrowRuntimeException(env, "Failed to query Surface format");
+ return 0;
+ }
+ ctx->setBufferFormat(format);
+ env->SetIntField(thiz, gImageWriterClassInfo.mWriterFormat, reinterpret_cast<jint>(format));
+
+
+ if (isWritable(format)) {
+ res = native_window_set_usage(anw.get(), GRALLOC_USAGE_SW_WRITE_OFTEN);
+ if (res != OK) {
+ ALOGE("%s: Configure usage %08x for format %08x failed: %s (%d)",
+ __FUNCTION__, GRALLOC_USAGE_SW_WRITE_OFTEN, format, strerror(-res), res);
+ jniThrowRuntimeException(env, "Failed to SW_WRITE_OFTEN configure usage");
+ return 0;
+ }
+ }
+
+ int minUndequeuedBufferCount = 0;
+ res = anw->query(anw.get(),
+ NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS, &minUndequeuedBufferCount);
+ if (res != OK) {
+ ALOGE("%s: Query producer undequeued buffer count failed: %s (%d)",
+ __FUNCTION__, strerror(-res), res);
+ jniThrowRuntimeException(env, "Query producer undequeued buffer count failed");
+ return 0;
+ }
+
+ size_t totalBufferCount = maxImages + minUndequeuedBufferCount;
+ res = native_window_set_buffer_count(anw.get(), totalBufferCount);
+ if (res != OK) {
+ ALOGE("%s: Set buffer count failed: %s (%d)", __FUNCTION__, strerror(-res), res);
+ jniThrowRuntimeException(env, "Set buffer count failed");
+ return 0;
+ }
+
+ if (ctx != 0) {
+ ctx->incStrong((void*)ImageWriter_init);
+ }
+ return nativeCtx;
+}
+
+static void ImageWriter_dequeueImage(JNIEnv* env, jobject thiz, jlong nativeCtx, jobject image) {
+ ALOGV("%s", __FUNCTION__);
+ JNIImageWriterContext* const ctx = reinterpret_cast<JNIImageWriterContext *>(nativeCtx);
+ if (ctx == NULL || thiz == NULL) {
+ jniThrowException(env, "java/lang/IllegalStateException",
+ "ImageWriterContext is not initialized");
+ return;
+ }
+
+ sp<ANativeWindow> anw = ctx->getProducer();
+ android_native_buffer_t *anb = NULL;
+ int fenceFd = -1;
+ status_t res = anw->dequeueBuffer(anw.get(), &anb, &fenceFd);
+ if (res != OK) {
+ // TODO: handle different error cases here.
+ ALOGE("%s: Set buffer count failed: %s (%d)", __FUNCTION__, strerror(-res), res);
+ jniThrowRuntimeException(env, "dequeue buffer failed");
+ return;
+ }
+ // New GraphicBuffer object doesn't own the handle, thus the native buffer
+ // won't be freed when this object is destroyed.
+ sp<GraphicBuffer> buffer(new GraphicBuffer(anb, /*keepOwnership*/false));
+
+ // Note that:
+ // 1. No need to lock buffer now, will only lock it when the first getPlanes() is called.
+ // 2. Fence will be saved to mNativeFenceFd, and will consumed by lock/queue/cancel buffer
+ // later.
+ // 3. need use lockAsync here, as it will handle the dequeued fence for us automatically.
+
+ // Finally, set the native info into image object.
+ Image_setNativeContext(env, image, buffer, fenceFd);
+}
+
+static void ImageWriter_close(JNIEnv* env, jobject thiz, jlong nativeCtx) {
+ ALOGV("%s:", __FUNCTION__);
+ JNIImageWriterContext* const ctx = reinterpret_cast<JNIImageWriterContext *>(nativeCtx);
+ if (ctx == NULL || thiz == NULL) {
+ jniThrowException(env, "java/lang/IllegalStateException",
+ "ImageWriterContext is not initialized");
+ return;
+ }
+
+ ANativeWindow* producer = ctx->getProducer();
+ if (producer != NULL) {
+ /**
+ * NATIVE_WINDOW_API_CPU isn't a good choice here, as it makes the bufferQueue not
+ * connectable after disconnect. MEDIA or CAMERA are treated the same internally.
+ * The producer listener will be cleared after disconnect call.
+ */
+ status_t res = native_window_api_disconnect(producer, /*api*/NATIVE_WINDOW_API_CAMERA);
+ /**
+ * This is not an error. if client calling process dies, the window will
+ * also die and all calls to it will return DEAD_OBJECT, thus it's already
+ * "disconnected"
+ */
+ if (res == DEAD_OBJECT) {
+ ALOGW("%s: While disconnecting ImageWriter from native window, the"
+ " native window died already", __FUNCTION__);
+ } else if (res != OK) {
+ ALOGE("%s: native window disconnect failed: %s (%d)",
+ __FUNCTION__, strerror(-res), res);
+ jniThrowRuntimeException(env, "Native window disconnect failed");
+ return;
+ }
+ }
+
+ ctx->decStrong((void*)ImageWriter_init);
+}
+
+static void ImageWriter_cancelImage(JNIEnv* env, jobject thiz, jlong nativeCtx, jobject image) {
+ ALOGV("%s", __FUNCTION__);
+ JNIImageWriterContext* const ctx = reinterpret_cast<JNIImageWriterContext *>(nativeCtx);
+ if (ctx == NULL || thiz == NULL) {
+ jniThrowException(env, "java/lang/IllegalStateException",
+ "ImageWriterContext is not initialized");
+ return;
+ }
+
+ sp<ANativeWindow> anw = ctx->getProducer();
+
+ GraphicBuffer *buffer = NULL;
+ int fenceFd = -1;
+ Image_getNativeContext(env, image, &buffer, &fenceFd);
+ if (buffer == NULL) {
+ jniThrowException(env, "java/lang/IllegalStateException",
+ "Image is not initialized");
+ return;
+ }
+
+ // Unlock the image if it was locked
+ Image_unlockIfLocked(env, image);
+
+ anw->cancelBuffer(anw.get(), buffer, fenceFd);
+
+ Image_setNativeContext(env, image, NULL, -1);
+}
+
+static void ImageWriter_queueImage(JNIEnv* env, jobject thiz, jlong nativeCtx, jobject image,
+ jlong timestampNs, jint left, jint top, jint right, jint bottom) {
+ ALOGV("%s", __FUNCTION__);
+ JNIImageWriterContext* const ctx = reinterpret_cast<JNIImageWriterContext *>(nativeCtx);
+ if (ctx == NULL || thiz == NULL) {
+ jniThrowException(env, "java/lang/IllegalStateException",
+ "ImageWriterContext is not initialized");
+ return;
+ }
+
+ status_t res = OK;
+ sp<ANativeWindow> anw = ctx->getProducer();
+
+ GraphicBuffer *buffer = NULL;
+ int fenceFd = -1;
+ Image_getNativeContext(env, image, &buffer, &fenceFd);
+ if (buffer == NULL) {
+ jniThrowException(env, "java/lang/IllegalStateException",
+ "Image is not initialized");
+ return;
+ }
+
+ // Unlock image if it was locked.
+ Image_unlockIfLocked(env, image);
+
+ // Set timestamp
+ ALOGV("timestamp to be queued: %lld", timestampNs);
+ res = native_window_set_buffers_timestamp(anw.get(), timestampNs);
+ if (res != OK) {
+ jniThrowRuntimeException(env, "Set timestamp failed");
+ return;
+ }
+
+ // Set crop
+ android_native_rect_t cropRect;
+ cropRect.left = left;
+ cropRect.top = top;
+ cropRect.right = right;
+ cropRect.bottom = bottom;
+ res = native_window_set_crop(anw.get(), &cropRect);
+ if (res != OK) {
+ jniThrowRuntimeException(env, "Set crop rect failed");
+ return;
+ }
+
+ // Finally, queue input buffer
+ res = anw->queueBuffer(anw.get(), buffer, fenceFd);
+ if (res != OK) {
+ jniThrowRuntimeException(env, "Queue input buffer failed");
+ return;
+ }
+
+ Image_setNativeContext(env, image, NULL, -1);
+}
+
+static void ImageWriter_attachImage(JNIEnv* env, jobject thiz, jlong nativeCtx, jobject image) {
+ ALOGV("%s", __FUNCTION__);
+ JNIImageWriterContext* const ctx = reinterpret_cast<JNIImageWriterContext *>(nativeCtx);
+ if (ctx == NULL || thiz == NULL) {
+ jniThrowException(env, "java/lang/IllegalStateException",
+ "ImageWriterContext is not initialized");
+ return;
+ }
+
+ sp<ANativeWindow> anw = ctx->getProducer();
+
+ GraphicBuffer *buffer = NULL;
+ int fenceFd = -1;
+ Image_getNativeContext(env, image, &buffer, &fenceFd);
+ if (buffer == NULL) {
+ jniThrowException(env, "java/lang/IllegalStateException",
+ "Image is not initialized");
+ return;
+ }
+
+ // TODO: need implement
+ jniThrowRuntimeException(env, "nativeAttachImage is not implement yet!!!");
+}
+
+// --------------------------Image methods---------------------------------------
+
+static void Image_getNativeContext(JNIEnv* env, jobject thiz,
+ GraphicBuffer** buffer, int* fenceFd) {
+ ALOGV("%s", __FUNCTION__);
+ if (buffer != NULL) {
+ GraphicBuffer *gb = reinterpret_cast<GraphicBuffer *>
+ (env->GetLongField(thiz, gSurfaceImageClassInfo.mNativeBuffer));
+ *buffer = gb;
+ }
+
+ if (fenceFd != NULL) {
+ *fenceFd = reinterpret_cast<jint>(env->GetIntField(
+ thiz, gSurfaceImageClassInfo.mNativeFenceFd));
+ }
+}
+
+static void Image_setNativeContext(JNIEnv* env, jobject thiz,
+ sp<GraphicBuffer> buffer, int fenceFd) {
+ ALOGV("%s:", __FUNCTION__);
+ GraphicBuffer* p = NULL;
+ Image_getNativeContext(env, thiz, &p, /*fenceFd*/NULL);
+ if (buffer != 0) {
+ buffer->incStrong((void*)Image_setNativeContext);
+ }
+ if (p) {
+ p->decStrong((void*)Image_setNativeContext);
+ }
+ env->SetLongField(thiz, gSurfaceImageClassInfo.mNativeBuffer,
+ reinterpret_cast<jlong>(buffer.get()));
+
+ env->SetIntField(thiz, gSurfaceImageClassInfo.mNativeFenceFd, reinterpret_cast<jint>(fenceFd));
+}
+
+static void Image_unlockIfLocked(JNIEnv* env, jobject thiz) {
+ ALOGV("%s", __FUNCTION__);
+ GraphicBuffer* buffer;
+ Image_getNativeContext(env, thiz, &buffer, NULL);
+ if (buffer == NULL) {
+ jniThrowException(env, "java/lang/IllegalStateException",
+ "Image is not initialized");
+ return;
+ }
+
+ // Is locked?
+ bool isLocked = false;
+ jobject planes = env->GetObjectField(thiz, gSurfaceImageClassInfo.mPlanes);
+ isLocked = (planes != NULL);
+ if (isLocked) {
+ // no need to use fence here, as we it will be consumed by either concel or queue buffer.
+ status_t res = buffer->unlock();
+ if (res != OK) {
+ jniThrowRuntimeException(env, "unlock buffer failed");
+ }
+ ALOGV("Successfully unlocked the image");
+ }
+}
+
+static jint Image_getWidth(JNIEnv* env, jobject thiz) {
+ ALOGV("%s", __FUNCTION__);
+ GraphicBuffer* buffer;
+ Image_getNativeContext(env, thiz, &buffer, NULL);
+ if (buffer == NULL) {
+ jniThrowException(env, "java/lang/IllegalStateException",
+ "Image is not initialized");
+ return -1;
+ }
+
+ return buffer->getWidth();
+}
+
+static jint Image_getHeight(JNIEnv* env, jobject thiz) {
+ ALOGV("%s", __FUNCTION__);
+ GraphicBuffer* buffer;
+ Image_getNativeContext(env, thiz, &buffer, NULL);
+ if (buffer == NULL) {
+ jniThrowException(env, "java/lang/IllegalStateException",
+ "Image is not initialized");
+ return -1;
+ }
+
+ return buffer->getHeight();
+}
+
+// Some formats like JPEG defined with different values between android.graphics.ImageFormat and
+// graphics.h, need convert to the one defined in graphics.h here.
+static int Image_getPixelFormat(JNIEnv* env, int format) {
+ int jpegFormat;
+ jfieldID fid;
+
+ ALOGV("%s: format = 0x%x", __FUNCTION__, format);
+
+ jclass imageFormatClazz = env->FindClass("android/graphics/ImageFormat");
+ ALOG_ASSERT(imageFormatClazz != NULL);
+
+ fid = env->GetStaticFieldID(imageFormatClazz, "JPEG", "I");
+ jpegFormat = env->GetStaticIntField(imageFormatClazz, fid);
+
+ // Translate the JPEG to BLOB for camera purpose.
+ if (format == jpegFormat) {
+ format = HAL_PIXEL_FORMAT_BLOB;
+ }
+
+ return format;
+}
+
+static jint Image_getFormat(JNIEnv* env, jobject thiz) {
+ ALOGV("%s", __FUNCTION__);
+ GraphicBuffer* buffer;
+ Image_getNativeContext(env, thiz, &buffer, NULL);
+ if (buffer == NULL) {
+ jniThrowException(env, "java/lang/IllegalStateException",
+ "Image is not initialized");
+ return 0;
+ }
+
+ return Image_getPixelFormat(env, buffer->getPixelFormat());
+}
+
+static void Image_setFenceFd(JNIEnv* env, jobject thiz, int fenceFd) {
+ ALOGV("%s:", __FUNCTION__);
+ env->SetIntField(thiz, gSurfaceImageClassInfo.mNativeFenceFd, reinterpret_cast<jint>(fenceFd));
+}
+
+static void Image_getLockedImage(JNIEnv* env, jobject thiz, LockedImage *image) {
+ ALOGV("%s", __FUNCTION__);
+ GraphicBuffer* buffer;
+ int fenceFd = -1;
+ Image_getNativeContext(env, thiz, &buffer, &fenceFd);
+ if (buffer == NULL) {
+ jniThrowException(env, "java/lang/IllegalStateException",
+ "Image is not initialized");
+ return;
+ }
+
+ void* pData = NULL;
+ android_ycbcr ycbcr = android_ycbcr();
+ status_t res;
+ int format = Image_getFormat(env, thiz);
+ int flexFormat = format;
+ if (isPossiblyYUV(format)) {
+ // ImageWriter doesn't use crop by itself, app sets it, use the no crop version.
+ res = buffer->lockAsyncYCbCr(GRALLOC_USAGE_SW_WRITE_OFTEN, &ycbcr, fenceFd);
+ // Clear the fenceFd as it is already consumed by lock call.
+ Image_setFenceFd(env, thiz, /*fenceFd*/-1);
+ if (res != OK) {
+ jniThrowRuntimeException(env, "lockAsyncYCbCr failed for YUV buffer");
+ return;
+ }
+ pData = ycbcr.y;
+ flexFormat = HAL_PIXEL_FORMAT_YCbCr_420_888;
+ }
+
+ // lockAsyncYCbCr for YUV is unsuccessful.
+ if (pData == NULL) {
+ res = buffer->lockAsync(GRALLOC_USAGE_SW_WRITE_OFTEN, &pData, fenceFd);
+ if (res != OK) {
+ jniThrowRuntimeException(env, "lockAsync failed");
+ return;
+ }
+ }
+
+ image->data = reinterpret_cast<uint8_t*>(pData);
+ image->width = buffer->getWidth();
+ image->height = buffer->getHeight();
+ image->format = format;
+ image->flexFormat = flexFormat;
+ image->stride = (ycbcr.y != NULL) ? static_cast<uint32_t>(ycbcr.ystride) : buffer->getStride();
+
+ image->dataCb = reinterpret_cast<uint8_t*>(ycbcr.cb);
+ image->dataCr = reinterpret_cast<uint8_t*>(ycbcr.cr);
+ image->chromaStride = static_cast<uint32_t>(ycbcr.cstride);
+ image->chromaStep = static_cast<uint32_t>(ycbcr.chroma_step);
+ ALOGV("Successfully locked the image");
+ // crop, transform, scalingMode, timestamp, and frameNumber should be set by producer,
+ // and we don't set them here.
+}
+
+static bool usingRGBAToJpegOverride(int32_t bufferFormat, int32_t writerCtxFormat) {
+ return writerCtxFormat == HAL_PIXEL_FORMAT_BLOB && bufferFormat == HAL_PIXEL_FORMAT_RGBA_8888;
+}
+
+static int32_t applyFormatOverrides(int32_t bufferFormat, int32_t writerCtxFormat)
+{
+ // Using HAL_PIXEL_FORMAT_RGBA_8888 gralloc buffers containing JPEGs to get around SW
+ // write limitations for some platforms (b/17379185).
+ if (usingRGBAToJpegOverride(bufferFormat, writerCtxFormat)) {
+ return HAL_PIXEL_FORMAT_BLOB;
+ }
+ return bufferFormat;
+}
+
+static uint32_t Image_getJpegSize(LockedImage* buffer, bool usingRGBAOverride) {
+ ALOGV("%s", __FUNCTION__);
+ ALOG_ASSERT(buffer != NULL, "Input buffer is NULL!!!");
+ uint32_t size = 0;
+ uint32_t width = buffer->width;
+ uint8_t* jpegBuffer = buffer->data;
+
+ if (usingRGBAOverride) {
+ width = (buffer->width + buffer->stride * (buffer->height - 1)) * 4;
+ }
+
+ // First check for JPEG transport header at the end of the buffer
+ uint8_t* header = jpegBuffer + (width - sizeof(struct camera3_jpeg_blob));
+ struct camera3_jpeg_blob *blob = (struct camera3_jpeg_blob*)(header);
+ if (blob->jpeg_blob_id == CAMERA3_JPEG_BLOB_ID) {
+ size = blob->jpeg_size;
+ ALOGV("%s: Jpeg size = %d", __FUNCTION__, size);
+ }
+
+ // failed to find size, default to whole buffer
+ if (size == 0) {
+ /*
+ * This is a problem because not including the JPEG header
+ * means that in certain rare situations a regular JPEG blob
+ * will be misidentified as having a header, in which case
+ * we will get a garbage size value.
+ */
+ ALOGW("%s: No JPEG header detected, defaulting to size=width=%d",
+ __FUNCTION__, width);
+ size = width;
+ }
+
+ return size;
+}
+
+static void Image_getLockedImageInfo(JNIEnv* env, LockedImage* buffer, int idx,
+ int32_t writerFormat, uint8_t **base, uint32_t *size, int *pixelStride, int *rowStride) {
+ ALOGV("%s", __FUNCTION__);
+ ALOG_ASSERT(buffer != NULL, "Input buffer is NULL!!!");
+ ALOG_ASSERT(base != NULL, "base is NULL!!!");
+ ALOG_ASSERT(size != NULL, "size is NULL!!!");
+ ALOG_ASSERT(pixelStride != NULL, "pixelStride is NULL!!!");
+ ALOG_ASSERT(rowStride != NULL, "rowStride is NULL!!!");
+ ALOG_ASSERT((idx < IMAGE_WRITER_MAX_NUM_PLANES) && (idx >= 0));
+
+ ALOGV("%s: buffer: %p", __FUNCTION__, buffer);
+
+ uint32_t dataSize, ySize, cSize, cStride;
+ uint32_t pStride = 0, rStride = 0;
+ uint8_t *cb, *cr;
+ uint8_t *pData = NULL;
+ int bytesPerPixel = 0;
+
+ dataSize = ySize = cSize = cStride = 0;
+ int32_t fmt = buffer->flexFormat;
+
+ bool usingRGBAOverride = usingRGBAToJpegOverride(fmt, writerFormat);
+ fmt = applyFormatOverrides(fmt, writerFormat);
+ switch (fmt) {
+ case HAL_PIXEL_FORMAT_YCbCr_420_888:
+ pData =
+ (idx == 0) ?
+ buffer->data :
+ (idx == 1) ?
+ buffer->dataCb :
+ buffer->dataCr;
+ // only map until last pixel
+ if (idx == 0) {
+ pStride = 1;
+ rStride = buffer->stride;
+ dataSize = buffer->stride * (buffer->height - 1) + buffer->width;
+ } else {
+ pStride = buffer->chromaStep;
+ rStride = buffer->chromaStride;
+ dataSize = buffer->chromaStride * (buffer->height / 2 - 1) +
+ buffer->chromaStep * (buffer->width / 2 - 1) + 1;
+ }
+ break;
+ // NV21
+ case HAL_PIXEL_FORMAT_YCrCb_420_SP:
+ cr = buffer->data + (buffer->stride * buffer->height);
+ cb = cr + 1;
+ // only map until last pixel
+ ySize = buffer->width * (buffer->height - 1) + buffer->width;
+ cSize = buffer->width * (buffer->height / 2 - 1) + buffer->width - 1;
+
+ pData =
+ (idx == 0) ?
+ buffer->data :
+ (idx == 1) ?
+ cb:
+ cr;
+
+ dataSize = (idx == 0) ? ySize : cSize;
+ pStride = (idx == 0) ? 1 : 2;
+ rStride = buffer->width;
+ break;
+ case HAL_PIXEL_FORMAT_YV12:
+ // Y and C stride need to be 16 pixel aligned.
+ LOG_ALWAYS_FATAL_IF(buffer->stride % 16,
+ "Stride is not 16 pixel aligned %d", buffer->stride);
+
+ ySize = buffer->stride * buffer->height;
+ cStride = ALIGN(buffer->stride / 2, 16);
+ cr = buffer->data + ySize;
+ cSize = cStride * buffer->height / 2;
+ cb = cr + cSize;
+
+ pData =
+ (idx == 0) ?
+ buffer->data :
+ (idx == 1) ?
+ cb :
+ cr;
+ dataSize = (idx == 0) ? ySize : cSize;
+ pStride = 1;
+ rStride = (idx == 0) ? buffer->stride : ALIGN(buffer->stride / 2, 16);
+ break;
+ case HAL_PIXEL_FORMAT_Y8:
+ // Single plane, 8bpp.
+ ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
+
+ pData = buffer->data;
+ dataSize = buffer->stride * buffer->height;
+ pStride = 1;
+ rStride = buffer->stride;
+ break;
+ case HAL_PIXEL_FORMAT_Y16:
+ bytesPerPixel = 2;
+ // Single plane, 16bpp, strides are specified in pixels, not in bytes
+ ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
+
+ pData = buffer->data;
+ dataSize = buffer->stride * buffer->height * bytesPerPixel;
+ pStride = bytesPerPixel;
+ rStride = buffer->stride * 2;
+ break;
+ case HAL_PIXEL_FORMAT_BLOB:
+ // Used for JPEG data, height must be 1, width == size, single plane.
+ ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
+ ALOG_ASSERT(buffer->height == 1, "JPEG should has height value %d", buffer->height);
+
+ pData = buffer->data;
+ dataSize = Image_getJpegSize(buffer, usingRGBAOverride);
+ pStride = bytesPerPixel;
+ rowStride = 0;
+ break;
+ case HAL_PIXEL_FORMAT_RAW16:
+ // Single plane 16bpp bayer data.
+ bytesPerPixel = 2;
+ ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
+ pData = buffer->data;
+ dataSize = buffer->stride * buffer->height * bytesPerPixel;
+ pStride = bytesPerPixel;
+ rStride = buffer->stride * 2;
+ break;
+ case HAL_PIXEL_FORMAT_RAW10:
+ // Single plane 10bpp bayer data.
+ ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
+ LOG_ALWAYS_FATAL_IF(buffer->width % 4,
+ "Width is not multiple of 4 %d", buffer->width);
+ LOG_ALWAYS_FATAL_IF(buffer->height % 2,
+ "Height is not even %d", buffer->height);
+ LOG_ALWAYS_FATAL_IF(buffer->stride < (buffer->width * 10 / 8),
+ "stride (%d) should be at least %d",
+ buffer->stride, buffer->width * 10 / 8);
+ pData = buffer->data;
+ dataSize = buffer->stride * buffer->height;
+ pStride = 0;
+ rStride = buffer->stride;
+ break;
+ case HAL_PIXEL_FORMAT_RGBA_8888:
+ case HAL_PIXEL_FORMAT_RGBX_8888:
+ // Single plane, 32bpp.
+ bytesPerPixel = 4;
+ ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
+ pData = buffer->data;
+ dataSize = buffer->stride * buffer->height * bytesPerPixel;
+ pStride = bytesPerPixel;
+ rStride = buffer->stride * 4;
+ break;
+ case HAL_PIXEL_FORMAT_RGB_565:
+ // Single plane, 16bpp.
+ bytesPerPixel = 2;
+ ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
+ pData = buffer->data;
+ dataSize = buffer->stride * buffer->height * bytesPerPixel;
+ pStride = bytesPerPixel;
+ rStride = buffer->stride * 2;
+ break;
+ case HAL_PIXEL_FORMAT_RGB_888:
+ // Single plane, 24bpp.
+ bytesPerPixel = 3;
+ ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
+ pData = buffer->data;
+ dataSize = buffer->stride * buffer->height * bytesPerPixel;
+ pStride = bytesPerPixel;
+ rStride = buffer->stride * 3;
+ break;
+ default:
+ jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException",
+ "Pixel format: 0x%x is unsupported", fmt);
+ break;
+ }
+
+ *base = pData;
+ *size = dataSize;
+ *pixelStride = pStride;
+ *rowStride = rStride;
+}
+
+static jobjectArray Image_createSurfacePlanes(JNIEnv* env, jobject thiz,
+ int numPlanes, int writerFormat) {
+ ALOGV("%s: create SurfacePlane array with size %d", __FUNCTION__, numPlanes);
+ int rowStride, pixelStride;
+ uint8_t *pData;
+ uint32_t dataSize;
+ jobject byteBuffer;
+
+ int format = Image_getFormat(env, thiz);
+ if (!isWritable(format) && numPlanes > 0) {
+ String8 msg;
+ msg.appendFormat("Format 0x%x is opaque, thus not writable, the number of planes (%d)"
+ " must be 0", format, numPlanes);
+ jniThrowException(env, "java/lang/IllegalArgumentException", msg.string());
+ return NULL;
+ }
+
+ jobjectArray surfacePlanes = env->NewObjectArray(numPlanes, gSurfacePlaneClassInfo.clazz,
+ /*initial_element*/NULL);
+ if (surfacePlanes == NULL) {
+ jniThrowRuntimeException(env, "Failed to create SurfacePlane arrays,"
+ " probably out of memory");
+ return NULL;
+ }
+
+ // Buildup buffer info: rowStride, pixelStride and byteBuffers.
+ LockedImage lockedImg = LockedImage();
+ Image_getLockedImage(env, thiz, &lockedImg);
+
+ // Create all SurfacePlanes
+ writerFormat = Image_getPixelFormat(env, writerFormat);
+ for (int i = 0; i < numPlanes; i++) {
+ Image_getLockedImageInfo(env, &lockedImg, i, writerFormat,
+ &pData, &dataSize, &pixelStride, &rowStride);
+ byteBuffer = env->NewDirectByteBuffer(pData, dataSize);
+ if ((byteBuffer == NULL) && (env->ExceptionCheck() == false)) {
+ jniThrowException(env, "java/lang/IllegalStateException",
+ "Failed to allocate ByteBuffer");
+ return NULL;
+ }
+
+ // Finally, create this SurfacePlane.
+ jobject surfacePlane = env->NewObject(gSurfacePlaneClassInfo.clazz,
+ gSurfacePlaneClassInfo.ctor, thiz, rowStride, pixelStride, byteBuffer);
+ env->SetObjectArrayElement(surfacePlanes, i, surfacePlane);
+ }
+
+ return surfacePlanes;
+}
+
+// -------------------------------Private convenience methods--------------------
+
+// Check if buffer with this format is writable. Generally speaking, the opaque formats
+// like IMPLEMENTATION_DEFINED is not writable, as the actual buffer formats and layouts
+// are unknown to frameworks.
+static bool isWritable(int format) {
+ // Assume all other formats are writable.
+ return !(format == HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED ||
+ format == HAL_PIXEL_FORMAT_RAW_OPAQUE);
+}
+
+static bool isPossiblyYUV(PixelFormat format) {
+ switch (static_cast<int>(format)) {
+ case HAL_PIXEL_FORMAT_RGBA_8888:
+ case HAL_PIXEL_FORMAT_RGBX_8888:
+ case HAL_PIXEL_FORMAT_RGB_888:
+ case HAL_PIXEL_FORMAT_RGB_565:
+ case HAL_PIXEL_FORMAT_BGRA_8888:
+ case HAL_PIXEL_FORMAT_Y8:
+ case HAL_PIXEL_FORMAT_Y16:
+ case HAL_PIXEL_FORMAT_RAW16:
+ case HAL_PIXEL_FORMAT_RAW10:
+ case HAL_PIXEL_FORMAT_RAW_OPAQUE:
+ case HAL_PIXEL_FORMAT_BLOB:
+ case HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED:
+ return false;
+
+ case HAL_PIXEL_FORMAT_YV12:
+ case HAL_PIXEL_FORMAT_YCbCr_420_888:
+ case HAL_PIXEL_FORMAT_YCbCr_422_SP:
+ case HAL_PIXEL_FORMAT_YCrCb_420_SP:
+ case HAL_PIXEL_FORMAT_YCbCr_422_I:
+ default:
+ return true;
+ }
+}
+
+} // extern "C"
+
+// ----------------------------------------------------------------------------
+
+static JNINativeMethod gImageWriterMethods[] = {
+ {"nativeClassInit", "()V", (void*)ImageWriter_classInit },
+ {"nativeInit", "(Ljava/lang/Object;Landroid/view/Surface;I)J",
+ (void*)ImageWriter_init },
+ {"nativeClose", "(J)V", (void*)ImageWriter_close },
+ {"nativeAttachImage", "(JLandroid/media/Image;)V", (void*)ImageWriter_attachImage },
+ {"nativeDequeueInputImage", "(JLandroid/media/Image;)V", (void*)ImageWriter_dequeueImage },
+ {"nativeQueueInputImage", "(JLandroid/media/Image;JIIII)V", (void*)ImageWriter_queueImage },
+ {"cancelImage", "(JLandroid/media/Image;)V", (void*)ImageWriter_cancelImage },
+};
+
+static JNINativeMethod gImageMethods[] = {
+ {"nativeCreatePlanes", "(II)[Landroid/media/ImageWriter$WriterSurfaceImage$SurfacePlane;",
+ (void*)Image_createSurfacePlanes },
+ {"nativeGetWidth", "()I", (void*)Image_getWidth },
+ {"nativeGetHeight", "()I", (void*)Image_getHeight },
+ {"nativeGetFormat", "()I", (void*)Image_getFormat },
+};
+
+int register_android_media_ImageWriter(JNIEnv *env) {
+
+ int ret1 = AndroidRuntime::registerNativeMethods(env,
+ "android/media/ImageWriter", gImageWriterMethods, NELEM(gImageWriterMethods));
+
+ int ret2 = AndroidRuntime::registerNativeMethods(env,
+ "android/media/ImageWriter$WriterSurfaceImage", gImageMethods, NELEM(gImageMethods));
+
+ return (ret1 || ret2);
+}
+
diff --git a/media/jni/android_media_MediaPlayer.cpp b/media/jni/android_media_MediaPlayer.cpp
index 55643f7..b748f3a 100644
--- a/media/jni/android_media_MediaPlayer.cpp
+++ b/media/jni/android_media_MediaPlayer.cpp
@@ -914,8 +914,8 @@
return AndroidRuntime::registerNativeMethods(env,
"android/media/MediaPlayer", gMethods, NELEM(gMethods));
}
-
extern int register_android_media_ImageReader(JNIEnv *env);
+extern int register_android_media_ImageWriter(JNIEnv *env);
extern int register_android_media_Crypto(JNIEnv *env);
extern int register_android_media_Drm(JNIEnv *env);
extern int register_android_media_MediaCodec(JNIEnv *env);
@@ -944,6 +944,11 @@
}
assert(env != NULL);
+ if (register_android_media_ImageWriter(env) != JNI_OK) {
+ ALOGE("ERROR: ImageWriter native registration failed");
+ goto bail;
+ }
+
if (register_android_media_ImageReader(env) < 0) {
ALOGE("ERROR: ImageReader native registration failed");
goto bail;