ImageReader/Writer: implement opaque format operations
Implement attach/detach for image reader and writer.
Bug: 19872821
Change-Id: Ib45a054c6be0b56b370fa8d709b47b0298ba5ea7
diff --git a/api/current.txt b/api/current.txt
index c6eff6c..29c39c8 100644
--- a/api/current.txt
+++ b/api/current.txt
@@ -14912,6 +14912,7 @@
public class ImageWriter implements java.lang.AutoCloseable {
method public void close();
method public android.media.Image dequeueInputImage();
+ method public int getFormat();
method public int getMaxImages();
method public static android.media.ImageWriter newInstance(android.view.Surface, int);
method public void queueInputImage(android.media.Image);
diff --git a/api/system-current.txt b/api/system-current.txt
index 88892a3..8f3a20e 100644
--- a/api/system-current.txt
+++ b/api/system-current.txt
@@ -16120,6 +16120,7 @@
public class ImageWriter implements java.lang.AutoCloseable {
method public void close();
method public android.media.Image dequeueInputImage();
+ method public int getFormat();
method public int getMaxImages();
method public static android.media.ImageWriter newInstance(android.view.Surface, int);
method public void queueInputImage(android.media.Image);
diff --git a/media/java/android/media/Image.java b/media/java/android/media/Image.java
index 9d07492..9ae468a 100644
--- a/media/java/android/media/Image.java
+++ b/media/java/android/media/Image.java
@@ -249,6 +249,22 @@
Object getOwner() {
return null;
}
+
+ /**
+ * Get native context (buffer pointer) associated with this image.
+ * <p>
+ * This is a package private method that is only used internally. It can be
+ * used to get the native buffer pointer and passed to native, which may be
+ * passed to {@link ImageWriter#attachAndQueueInputImage} to avoid a reverse
+ * JNI call.
+ * </p>
+ *
+ * @return native context associated with this Image.
+ */
+ long getNativeContext() {
+ return 0;
+ }
+
/**
* <p>A single color plane of image data.</p>
*
diff --git a/media/java/android/media/ImageReader.java b/media/java/android/media/ImageReader.java
index b2f7a20..3d8f9a0 100644
--- a/media/java/android/media/ImageReader.java
+++ b/media/java/android/media/ImageReader.java
@@ -98,7 +98,7 @@
* @see Image
*/
public static ImageReader newInstance(int width, int height, int format, int maxImages) {
- if (format == PixelFormat.OPAQUE) {
+ if (format == ImageFormat.PRIVATE) {
throw new IllegalArgumentException("To obtain an opaque ImageReader, please use"
+ " newOpaqueInstance rather than newInstance");
}
@@ -148,7 +148,7 @@
* @see Image
*/
public static ImageReader newOpaqueInstance(int width, int height, int maxImages) {
- return new ImageReader(width, height, PixelFormat.OPAQUE, maxImages);
+ return new ImageReader(width, height, ImageFormat.PRIVATE, maxImages);
}
/**
@@ -261,7 +261,7 @@
* @see ImageReader#newOpaqueInstance
*/
public boolean isOpaque() {
- return mFormat == PixelFormat.OPAQUE;
+ return mFormat == ImageFormat.PRIVATE;
}
/**
@@ -343,7 +343,7 @@
* @hide
*/
public Image acquireNextImageNoThrowISE() {
- SurfaceImage si = new SurfaceImage();
+ SurfaceImage si = new SurfaceImage(mFormat);
return acquireNextSurfaceImage(si) == ACQUIRE_SUCCESS ? si : null;
}
@@ -408,7 +408,9 @@
* @see #acquireLatestImage
*/
public Image acquireNextImage() {
- SurfaceImage si = new SurfaceImage();
+ // Initialize with reader format, but can be overwritten by native if the image
+ // format is different from the reader format.
+ SurfaceImage si = new SurfaceImage(mFormat);
int status = acquireNextSurfaceImage(si);
switch (status) {
@@ -565,9 +567,8 @@
}
SurfaceImage si = (SurfaceImage) image;
- if (!si.isImageValid()) {
- throw new IllegalStateException("Image is no longer valid");
- }
+ si.throwISEIfImageIsInvalid();
+
if (si.isAttachable()) {
throw new IllegalStateException("Image was already detached from this ImageReader");
}
@@ -607,7 +608,7 @@
case ImageFormat.DEPTH16:
case ImageFormat.DEPTH_POINT_CLOUD:
return 1;
- case PixelFormat.OPAQUE:
+ case ImageFormat.PRIVATE:
return 0;
default:
throw new UnsupportedOperationException(
@@ -684,18 +685,15 @@
}
private class SurfaceImage extends android.media.Image {
- public SurfaceImage() {
+ public SurfaceImage(int format) {
mIsImageValid = false;
+ mFormat = format;
}
@Override
public void close() {
if (mIsImageValid) {
- if (!mIsDetached.get()) {
- // For detached images, the new owner is responsible for
- // releasing the resources
- ImageReader.this.releaseImage(this);
- }
+ ImageReader.this.releaseImage(this);
}
}
@@ -705,70 +703,53 @@
@Override
public int getFormat() {
- if (mIsImageValid) {
- return ImageReader.this.mFormat;
- } else {
- throw new IllegalStateException("Image is already released");
- }
+ throwISEIfImageIsInvalid();
+ return mFormat;
}
@Override
public int getWidth() {
- if (mIsImageValid) {
- if (mWidth == -1) {
- mWidth = (getFormat() == ImageFormat.JPEG) ? ImageReader.this.getWidth() :
- nativeGetWidth();
- }
- return mWidth;
- } else {
- throw new IllegalStateException("Image is already released");
+ throwISEIfImageIsInvalid();
+ if (mWidth == -1) {
+ mWidth = (getFormat() == ImageFormat.JPEG) ? ImageReader.this.getWidth() :
+ nativeGetWidth(mFormat);
}
+ return mWidth;
}
@Override
public int getHeight() {
- if (mIsImageValid) {
- if (mHeight == -1) {
- mHeight = (getFormat() == ImageFormat.JPEG) ? ImageReader.this.getHeight() :
- nativeGetHeight();
- }
- return mHeight;
- } else {
- throw new IllegalStateException("Image is already released");
+ throwISEIfImageIsInvalid();
+ if (mHeight == -1) {
+ mHeight = (getFormat() == ImageFormat.JPEG) ? ImageReader.this.getHeight() :
+ nativeGetHeight(mFormat);
}
+ return mHeight;
}
@Override
public long getTimestamp() {
- if (mIsImageValid) {
- return mTimestamp;
- } else {
- throw new IllegalStateException("Image is already released");
- }
+ throwISEIfImageIsInvalid();
+ return mTimestamp;
}
@Override
public void setTimestamp(long timestampNs) {
- if (mIsImageValid) {
- mTimestamp = timestampNs;
- } else {
- throw new IllegalStateException("Image is already released");
- }
+ throwISEIfImageIsInvalid();
+ mTimestamp = timestampNs;
}
@Override
public Plane[] getPlanes() {
- if (mIsImageValid) {
- // Shallow copy is fine.
- return mPlanes.clone();
- } else {
- throw new IllegalStateException("Image is already released");
- }
+ throwISEIfImageIsInvalid();
+ // Shallow copy is fine.
+ return mPlanes.clone();
}
@Override
public boolean isOpaque() {
- return mFormat == PixelFormat.OPAQUE;
+ throwISEIfImageIsInvalid();
+ return mFormat == ImageFormat.PRIVATE;
}
@Override
@@ -782,15 +763,24 @@
@Override
boolean isAttachable() {
+ throwISEIfImageIsInvalid();
return mIsDetached.get();
}
@Override
ImageReader getOwner() {
+ throwISEIfImageIsInvalid();
return ImageReader.this;
}
+ @Override
+ long getNativeContext() {
+ throwISEIfImageIsInvalid();
+ return mNativeBuffer;
+ }
+
private void setDetached(boolean detached) {
+ throwISEIfImageIsInvalid();
mIsDetached.getAndSet(detached);
}
@@ -798,8 +788,10 @@
mIsImageValid = isValid;
}
- private boolean isImageValid() {
- return mIsImageValid;
+ private void throwISEIfImageIsInvalid() {
+ if (!mIsImageValid) {
+ throw new IllegalStateException("Image is already closed");
+ }
}
private void clearSurfacePlanes() {
@@ -829,9 +821,7 @@
@Override
public ByteBuffer getBuffer() {
- if (SurfaceImage.this.isImageValid() == false) {
- throw new IllegalStateException("Image is already released");
- }
+ SurfaceImage.this.throwISEIfImageIsInvalid();
if (mBuffer != null) {
return mBuffer;
} else {
@@ -845,20 +835,14 @@
@Override
public int getPixelStride() {
- if (SurfaceImage.this.isImageValid()) {
- return mPixelStride;
- } else {
- throw new IllegalStateException("Image is already released");
- }
+ SurfaceImage.this.throwISEIfImageIsInvalid();
+ return mPixelStride;
}
@Override
public int getRowStride() {
- if (SurfaceImage.this.isImageValid()) {
- return mRowStride;
- } else {
- throw new IllegalStateException("Image is already released");
- }
+ SurfaceImage.this.throwISEIfImageIsInvalid();
+ return mRowStride;
}
private void clearBuffer() {
@@ -885,7 +869,7 @@
* This field is used to keep track of native object and used by native code only.
* Don't modify.
*/
- private long mLockedBuffer;
+ private long mNativeBuffer;
/**
* This field is set by native code during nativeImageSetup().
@@ -896,13 +880,14 @@
private boolean mIsImageValid;
private int mHeight = -1;
private int mWidth = -1;
+ private int mFormat = ImageFormat.UNKNOWN;
// If this image is detached from the ImageReader.
private AtomicBoolean mIsDetached = new AtomicBoolean(false);
private synchronized native ByteBuffer nativeImageGetBuffer(int idx, int readerFormat);
private synchronized native SurfacePlane nativeCreatePlane(int idx, int readerFormat);
- private synchronized native int nativeGetWidth();
- private synchronized native int nativeGetHeight();
+ private synchronized native int nativeGetWidth(int format);
+ private synchronized native int nativeGetHeight(int format);
}
private synchronized native void nativeInit(Object weakSelf, int w, int h,
@@ -910,7 +895,7 @@
private synchronized native void nativeClose();
private synchronized native void nativeReleaseImage(Image i);
private synchronized native Surface nativeGetSurface();
- private synchronized native void nativeDetachImage(Image i);
+ private synchronized native int nativeDetachImage(Image i);
/**
* @return A return code {@code ACQUIRE_*}
diff --git a/media/java/android/media/ImageWriter.java b/media/java/android/media/ImageWriter.java
index 20389a39..c18b463 100644
--- a/media/java/android/media/ImageWriter.java
+++ b/media/java/android/media/ImageWriter.java
@@ -16,7 +16,7 @@
package android.media;
-import android.graphics.PixelFormat;
+import android.graphics.ImageFormat;
import android.graphics.Rect;
import android.os.Handler;
import android.os.Looper;
@@ -77,9 +77,7 @@
private int mWriterFormat;
private final int mMaxImages;
- // Keep track of the currently attached Image; or an attached Image that is
- // released will be removed from this list.
- private List<Image> mAttachedImages = new ArrayList<Image>();
+ // Keep track of the currently dequeued Image.
private List<Image> mDequeuedImages = new ArrayList<Image>();
/**
@@ -168,21 +166,38 @@
* {@link Image#close()}.
* </p>
* <p>
- * This call will block if all available input images have been filled by
+ * This call will block if all available input images have been queued by
* the application and the downstream consumer has not yet consumed any.
- * When an Image is consumed by the downstream consumer, an
+ * When an Image is consumed by the downstream consumer and released, an
* {@link ImageListener#onInputImageReleased} callback will be fired, which
- * indicates that there is one input Image available. It is recommended to
- * dequeue next Image only after this callback is fired, in the steady state.
+ * indicates that there is one input Image available. For non-opaque formats
+ * (({@link ImageWriter#getFormat()} != {@link ImageFormat#PRIVATE})), it is
+ * recommended to dequeue the next Image only after this callback is fired,
+ * in the steady state.
+ * </p>
+ * <p>
+ * If the ImageWriter is opaque ({@link ImageWriter#getFormat()} ==
+ * {@link ImageFormat#PRIVATE}), the image buffer is inaccessible to
+ * the application, and calling this method will result in an
+ * {@link IllegalStateException}. Instead, the application should acquire
+ * opaque images from some other component (e.g. an opaque
+ * {@link ImageReader}), and queue them directly to this ImageWriter via the
+ * {@link ImageWriter#queueInputImage queueInputImage()} method.
* </p>
*
* @return The next available input Image from this ImageWriter.
* @throws IllegalStateException if {@code maxImages} Images are currently
- * dequeued.
+ * dequeued, or the ImageWriter is opaque.
* @see #queueInputImage
* @see Image#close
*/
public Image dequeueInputImage() {
+ if (mWriterFormat == ImageFormat.PRIVATE) {
+ throw new IllegalStateException(
+ "Opaque ImageWriter doesn't support this operation since opaque images are"
+ + " inaccessible to the application!");
+ }
+
if (mDequeuedImages.size() >= mMaxImages) {
throw new IllegalStateException("Already dequeued max number of Images " + mMaxImages);
}
@@ -214,12 +229,19 @@
* capture time.
* </p>
* <p>
- * Passing in a non-opaque Image may result in a memory copy, which also
- * requires a free input Image from this ImageWriter as the destination. In
- * this case, this call will block, as {@link #dequeueInputImage} does, if
- * there are no free Images available. To be safe, the application should ensure
- * that there is at least one free Image available in this ImageWriter before calling
- * this method.
+ * After this method is called and the downstream consumer consumes and
+ * releases the Image, an {@link ImageListener#onInputImageReleased
+ * onInputImageReleased()} callback will fire. The application can use this
+ * callback to avoid sending Images faster than the downstream consumer
+ * processing rate in steady state.
+ * </p>
+ * <p>
+ * Passing in an Image from some other component (e.g. an
+ * {@link ImageReader}) requires a free input Image from this ImageWriter as
+ * the destination. In this case, this call will block, as
+ * {@link #dequeueInputImage} does, if there are no free Images available.
+ * To avoid blocking, the application should ensure that there is at least
+ * one free Image available in this ImageWriter before calling this method.
* </p>
* <p>
* After this call, the input Image is no longer valid for further access,
@@ -252,10 +274,17 @@
ImageReader prevOwner = (ImageReader) image.getOwner();
// Only do the image attach for opaque images for now. Do the image
// copy for other formats. TODO: use attach for other formats to
- // improve the performance, and fall back to copy when attach/detach fails.
+ // improve the performance, and fall back to copy when attach/detach
+ // fails. Right now, detach is guaranteed to fail as the buffer is
+ // locked when ImageReader#acquireNextImage is called. See bug 19962027.
if (image.isOpaque()) {
prevOwner.detachImage(image);
- attachInputImage(image);
+ attachAndQueueInputImage(image);
+ // This clears the native reference held by the original owner.
+ // When this Image is detached later by this ImageWriter, the
+ // native memory won't be leaked.
+ image.close();
+ return;
} else {
Image inputImage = dequeueInputImage();
inputImage.setTimestamp(image.getTimestamp());
@@ -273,25 +302,35 @@
/**
* Only remove and cleanup the Images that are owned by this
- * ImageWriter. Images detached from other owners are only
- * temporarily owned by this ImageWriter and will be detached immediately
- * after they are released by downstream consumers, so there is no need to
- * keep track of them in mDequeuedImages.
+ * ImageWriter. Images detached from other owners are only temporarily
+ * owned by this ImageWriter and will be detached immediately after they
+ * are released by downstream consumers, so there is no need to keep
+ * track of them in mDequeuedImages.
*/
if (ownedByMe) {
mDequeuedImages.remove(image);
+ // Do not call close here, as close is essentially cancel image.
WriterSurfaceImage wi = (WriterSurfaceImage) image;
wi.clearSurfacePlanes();
wi.setImageValid(false);
- } else {
- // This clears the native reference held by the original owner. When
- // this Image is detached later by this ImageWriter, the native
- // memory won't be leaked.
- image.close();
}
}
/**
+ * Get the ImageWriter format.
+ * <p>
+ * This format may be different than the Image format returned by
+ * {@link Image#getFormat()}. However, if the ImageWriter is opaque (format
+ * == {@link ImageFormat#PRIVATE}) , the images from it will also be opaque.
+ * </p>
+ *
+ * @return The ImageWriter format.
+ */
+ public int getFormat() {
+ return mWriterFormat;
+ }
+
+ /**
* ImageWriter callback interface, used to to asynchronously notify the
* application of various ImageWriter events.
*/
@@ -302,27 +341,33 @@
* ImageWriter after the data consumption.
* </p>
* <p>
- * The client can use this callback to indicate either an input Image is
- * available to fill data into, or the input Image is returned and freed
- * if it was attached from other components (e.g. an
- * {@link ImageReader}). For the latter case, the ownership of the Image
- * will be automatically removed by ImageWriter right before this
- * callback is fired.
+ * The client can use this callback to be notified that an input Image
+ * has been consumed and released by the downstream consumer. More
+ * specifically, this callback will be fired for below cases:
+ * <li>The application dequeues an input Image via the
+ * {@link ImageWriter#dequeueInputImage dequeueInputImage()} method,
+ * uses it, and then queues it back to this ImageWriter via the
+ * {@link ImageWriter#queueInputImage queueInputImage()} method. After
+ * the downstream consumer uses and releases this image to this
+ * ImageWriter, this callback will be fired. This image will be
+ * available to be dequeued after this callback.</li>
+ * <li>The application obtains an Image from some other component (e.g.
+ * an {@link ImageReader}), uses it, and then queues it to this
+ * ImageWriter via {@link ImageWriter#queueInputImage queueInputImage()}.
+ * After the downstream consumer uses and releases this image to this
+ * ImageWriter, this callback will be fired.</li>
* </p>
*
* @param writer the ImageWriter the callback is associated with.
* @see ImageWriter
* @see Image
*/
- // TODO: the semantics is confusion, does't tell which buffer is
- // released if an application is doing queueInputImage with a mix of
- // buffers from dequeueInputImage and from an ImageReader. see b/19872821
void onInputImageReleased(ImageWriter writer);
}
/**
- * Register a listener to be invoked when an input Image is returned to
- * the ImageWriter.
+ * Register a listener to be invoked when an input Image is returned to the
+ * ImageWriter.
*
* @param listener The listener that will be run.
* @param handler The handler on which the listener should be invoked, or
@@ -383,38 +428,25 @@
}
/**
- * Get the ImageWriter format.
* <p>
- * This format may be different than the Image format returned by
- * {@link Image#getFormat()}
- * </p>
- *
- * @return The ImageWriter format.
- */
- int getFormat() {
- return mWriterFormat;
- }
-
-
- /**
- * <p>
- * Attach input Image to this ImageWriter.
+ * Attach and queue input Image to this ImageWriter.
* </p>
* <p>
- * When an Image is from an opaque source (e.g. an opaque ImageReader created
- * by {@link ImageReader#newOpaqueInstance}), or the source Image is so large
- * that copying its data is too expensive, this method can be used to
- * migrate the source Image into ImageWriter without a data copy. The source
- * Image must be detached from its previous owner already, or this call will
- * throw an {@link IllegalStateException}.
+ * When an Image is from an opaque source (e.g. an opaque ImageReader
+ * created by {@link ImageReader#newOpaqueInstance}), or the source Image is
+ * so large that copying its data is too expensive, this method can be used
+ * to migrate the source Image into ImageWriter without a data copy, and
+ * then queue it to this ImageWriter. The source Image must be detached from
+ * its previous owner already, or this call will throw an
+ * {@link IllegalStateException}.
* </p>
* <p>
- * After this call, the ImageWriter takes ownership of this Image.
- * This ownership will be automatically removed from this writer after the
+ * After this call, the ImageWriter takes ownership of this Image. This
+ * ownership will automatically be removed from this writer after the
* consumer releases this Image, that is, after
- * {@link ImageListener#onInputImageReleased}. The caller is
- * responsible for closing this Image through {@link Image#close()} to free up
- * the resources held by this Image.
+ * {@link ImageListener#onInputImageReleased}. The caller is responsible for
+ * closing this Image through {@link Image#close()} to free up the resources
+ * held by this Image.
* </p>
*
* @param image The source Image to be attached and queued into this
@@ -423,7 +455,7 @@
* previous owner, or the Image is already attached to this
* ImageWriter, or the source Image is invalid.
*/
- private void attachInputImage(Image image) {
+ private void attachAndQueueInputImage(Image image) {
if (image == null) {
throw new IllegalArgumentException("image shouldn't be null");
}
@@ -441,15 +473,13 @@
throw new IllegalStateException("Image was not detached from last owner, or image "
+ " is not detachable");
}
- if (mAttachedImages.contains(image)) {
- throw new IllegalStateException("Image was already attached to ImageWritter");
- }
// TODO: what if attach failed, throw RTE or detach a slot then attach?
// need do some cleanup to make sure no orphaned
// buffer caused leak.
- nativeAttachImage(mNativeContext, image);
- mAttachedImages.add(image);
+ Rect crop = image.getCropRect();
+ nativeAttachAndQueueImage(mNativeContext, image.getNativeContext(), image.getFormat(),
+ image.getTimestamp(), crop.left, crop.top, crop.right, crop.bottom);
}
/**
@@ -467,8 +497,6 @@
synchronized (mListenerLock) {
listener = mListener;
}
- // TODO: detach Image from ImageWriter and remove the Image from
- // mAttachedImage list.
if (listener != null) {
listener.onInputImageReleased(ImageWriter.this);
}
@@ -537,7 +565,7 @@
* attached + queued successfully, and attach failed. Neither of the
* cases need abort.
*/
- cancelImage(mNativeContext,image);
+ cancelImage(mNativeContext, image);
mDequeuedImages.remove(image);
wi.clearSurfacePlanes();
wi.setImageValid(false);
@@ -635,7 +663,7 @@
throw new IllegalStateException("Image is already released");
}
- return getFormat() == PixelFormat.OPAQUE;
+ return getFormat() == ImageFormat.PRIVATE;
}
@Override
@@ -668,6 +696,11 @@
}
@Override
+ long getNativeContext() {
+ return mNativeBuffer;
+ }
+
+ @Override
public void close() {
if (mIsImageValid.get()) {
getOwner().abortImage(this);
@@ -776,13 +809,15 @@
private synchronized native void nativeClose(long nativeCtx);
- private synchronized native void nativeAttachImage(long nativeCtx, Image image);
-
private synchronized native void nativeDequeueInputImage(long nativeCtx, Image wi);
private synchronized native void nativeQueueInputImage(long nativeCtx, Image image,
long timestampNs, int left, int top, int right, int bottom);
+ private synchronized native int nativeAttachAndQueueImage(long nativeCtx,
+ long imageNativeBuffer, int imageFormat, long timestampNs, int left,
+ int top, int right, int bottom);
+
private synchronized native void cancelImage(long nativeCtx, Image image);
/**
diff --git a/media/jni/android_media_ImageReader.cpp b/media/jni/android_media_ImageReader.cpp
index 708c083..043e20b 100644
--- a/media/jni/android_media_ImageReader.cpp
+++ b/media/jni/android_media_ImageReader.cpp
@@ -24,6 +24,7 @@
#include <cstdio>
#include <gui/CpuConsumer.h>
+#include <gui/BufferItemConsumer.h>
#include <gui/Surface.h>
#include <camera3.h>
@@ -39,7 +40,7 @@
#define ALIGN(x, mask) ( ((x) + (mask) - 1) & ~((mask) - 1) )
#define ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID "mNativeContext"
-#define ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID "mLockedBuffer"
+#define ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID "mNativeBuffer"
#define ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID "mTimestamp"
// ----------------------------------------------------------------------------
@@ -62,7 +63,7 @@
} gImageReaderClassInfo;
static struct {
- jfieldID mLockedBuffer;
+ jfieldID mNativeBuffer;
jfieldID mTimestamp;
} gSurfaceImageClassInfo;
@@ -73,7 +74,7 @@
// ----------------------------------------------------------------------------
-class JNIImageReaderContext : public CpuConsumer::FrameAvailableListener
+class JNIImageReaderContext : public ConsumerBase::FrameAvailableListener
{
public:
JNIImageReaderContext(JNIEnv* env, jobject weakThiz, jclass clazz, int maxImages);
@@ -83,12 +84,19 @@
virtual void onFrameAvailable(const BufferItem& item);
CpuConsumer::LockedBuffer* getLockedBuffer();
-
void returnLockedBuffer(CpuConsumer::LockedBuffer* buffer);
+ BufferItem* getOpaqueBuffer();
+ void returnOpaqueBuffer(BufferItem* buffer);
+
void setCpuConsumer(const sp<CpuConsumer>& consumer) { mConsumer = consumer; }
CpuConsumer* getCpuConsumer() { return mConsumer.get(); }
+ void setOpaqueConsumer(const sp<BufferItemConsumer>& consumer) { mOpaqueConsumer = consumer; }
+ BufferItemConsumer* getOpaqueConsumer() { return mOpaqueConsumer.get(); }
+ // This is the only opaque format exposed in the ImageFormat public API.
+ bool isOpaque() { return mFormat == HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED; }
+
void setProducer(const sp<IGraphicBufferProducer>& producer) { mProducer = producer; }
IGraphicBufferProducer* getProducer() { return mProducer.get(); }
@@ -109,7 +117,9 @@
static void detachJNI();
List<CpuConsumer::LockedBuffer*> mBuffers;
+ List<BufferItem*> mOpaqueBuffers;
sp<CpuConsumer> mConsumer;
+ sp<BufferItemConsumer> mOpaqueConsumer;
sp<IGraphicBufferProducer> mProducer;
jobject mWeakThiz;
jclass mClazz;
@@ -125,7 +135,9 @@
mClazz((jclass)env->NewGlobalRef(clazz)) {
for (int i = 0; i < maxImages; i++) {
CpuConsumer::LockedBuffer *buffer = new CpuConsumer::LockedBuffer;
+ BufferItem* opaqueBuffer = new BufferItem;
mBuffers.push_back(buffer);
+ mOpaqueBuffers.push_back(opaqueBuffer);
}
}
@@ -169,6 +181,21 @@
mBuffers.push_back(buffer);
}
+BufferItem* JNIImageReaderContext::getOpaqueBuffer() {
+ if (mOpaqueBuffers.empty()) {
+ return NULL;
+ }
+ // Return an opaque buffer pointer and remove it from the list
+ List<BufferItem*>::iterator it = mOpaqueBuffers.begin();
+ BufferItem* buffer = *it;
+ mOpaqueBuffers.erase(it);
+ return buffer;
+}
+
+void JNIImageReaderContext::returnOpaqueBuffer(BufferItem* buffer) {
+ mOpaqueBuffers.push_back(buffer);
+}
+
JNIImageReaderContext::~JNIImageReaderContext() {
bool needsDetach = false;
JNIEnv* env = getJNIEnv(&needsDetach);
@@ -187,8 +214,20 @@
it != mBuffers.end(); it++) {
delete *it;
}
+
+ // Delete opaque buffers
+ for (List<BufferItem *>::iterator it = mOpaqueBuffers.begin();
+ it != mOpaqueBuffers.end(); it++) {
+ delete *it;
+ }
+
mBuffers.clear();
- mConsumer.clear();
+ if (mConsumer != 0) {
+ mConsumer.clear();
+ }
+ if (mOpaqueConsumer != 0) {
+ mOpaqueConsumer.clear();
+ }
}
void JNIImageReaderContext::onFrameAvailable(const BufferItem& /*item*/)
@@ -210,6 +249,11 @@
extern "C" {
+static bool isFormatOpaque(int format) {
+ // Only treat IMPLEMENTATION_DEFINED as an opaque format for now.
+ return format == HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED;
+}
+
static JNIImageReaderContext* ImageReader_getContext(JNIEnv* env, jobject thiz)
{
JNIImageReaderContext *ctx;
@@ -226,6 +270,13 @@
jniThrowRuntimeException(env, "ImageReaderContext is not initialized");
return NULL;
}
+
+ if (ctx->isOpaque()) {
+ jniThrowException(env, "java/lang/IllegalStateException",
+ "Opaque ImageReader doesn't support this method");
+ return NULL;
+ }
+
return ctx->getCpuConsumer();
}
@@ -237,6 +288,7 @@
jniThrowRuntimeException(env, "ImageReaderContext is not initialized");
return NULL;
}
+
return ctx->getProducer();
}
@@ -258,13 +310,19 @@
static CpuConsumer::LockedBuffer* Image_getLockedBuffer(JNIEnv* env, jobject image)
{
return reinterpret_cast<CpuConsumer::LockedBuffer*>(
- env->GetLongField(image, gSurfaceImageClassInfo.mLockedBuffer));
+ env->GetLongField(image, gSurfaceImageClassInfo.mNativeBuffer));
}
static void Image_setBuffer(JNIEnv* env, jobject thiz,
const CpuConsumer::LockedBuffer* buffer)
{
- env->SetLongField(thiz, gSurfaceImageClassInfo.mLockedBuffer, reinterpret_cast<jlong>(buffer));
+ env->SetLongField(thiz, gSurfaceImageClassInfo.mNativeBuffer, reinterpret_cast<jlong>(buffer));
+}
+
+static void Image_setOpaqueBuffer(JNIEnv* env, jobject thiz,
+ const BufferItem* buffer)
+{
+ env->SetLongField(thiz, gSurfaceImageClassInfo.mNativeBuffer, reinterpret_cast<jlong>(buffer));
}
static uint32_t Image_getJpegSize(CpuConsumer::LockedBuffer* buffer, bool usingRGBAOverride)
@@ -633,6 +691,52 @@
return buffer->height;
}
+// --------------------------Methods for opaque Image and ImageReader----------
+
+static BufferItemConsumer* ImageReader_getOpaqueConsumer(JNIEnv* env, jobject thiz)
+{
+ ALOGV("%s:", __FUNCTION__);
+ JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz);
+ if (ctx == NULL) {
+ jniThrowRuntimeException(env, "ImageReaderContext is not initialized");
+ return NULL;
+ }
+
+ if (!ctx->isOpaque()) {
+ jniThrowException(env, "java/lang/IllegalStateException",
+ "Non-opaque ImageReader doesn't support this method");
+ }
+
+ return ctx->getOpaqueConsumer();
+}
+
+static BufferItem* Image_getOpaqueBuffer(JNIEnv* env, jobject image)
+{
+ return reinterpret_cast<BufferItem*>(
+ env->GetLongField(image, gSurfaceImageClassInfo.mNativeBuffer));
+}
+
+static int Image_getOpaqueBufferWidth(BufferItem* buffer) {
+ if (buffer == NULL) return -1;
+
+ if (!buffer->mCrop.isEmpty()) {
+ return buffer->mCrop.getWidth();
+ }
+ return buffer->mGraphicBuffer->getWidth();
+}
+
+static int Image_getOpaqueBufferHeight(BufferItem* buffer) {
+ if (buffer == NULL) return -1;
+
+ if (!buffer->mCrop.isEmpty()) {
+ return buffer->mCrop.getHeight();
+ }
+
+ return buffer->mGraphicBuffer->getHeight();
+}
+
+
+
// ----------------------------------------------------------------------------
static void ImageReader_classInit(JNIEnv* env, jclass clazz)
@@ -642,9 +746,9 @@
jclass imageClazz = env->FindClass("android/media/ImageReader$SurfaceImage");
LOG_ALWAYS_FATAL_IF(imageClazz == NULL,
"can't find android/graphics/ImageReader$SurfaceImage");
- gSurfaceImageClassInfo.mLockedBuffer = env->GetFieldID(
+ gSurfaceImageClassInfo.mNativeBuffer = env->GetFieldID(
imageClazz, ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID, "J");
- LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mLockedBuffer == NULL,
+ LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mNativeBuffer == NULL,
"can't find android/graphics/ImageReader.%s",
ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID);
@@ -691,24 +795,42 @@
nativeDataspace = android_view_Surface_mapPublicFormatToHalDataspace(
publicFormat);
- sp<IGraphicBufferProducer> gbProducer;
- sp<IGraphicBufferConsumer> gbConsumer;
- BufferQueue::createBufferQueue(&gbProducer, &gbConsumer);
- sp<CpuConsumer> consumer = new CpuConsumer(gbConsumer, maxImages,
- /*controlledByApp*/true);
- // TODO: throw dvm exOutOfMemoryError?
- if (consumer == NULL) {
- jniThrowRuntimeException(env, "Failed to allocate native CpuConsumer");
- return;
- }
-
jclass clazz = env->GetObjectClass(thiz);
if (clazz == NULL) {
jniThrowRuntimeException(env, "Can't find android/graphics/ImageReader");
return;
}
sp<JNIImageReaderContext> ctx(new JNIImageReaderContext(env, weakThiz, clazz, maxImages));
- ctx->setCpuConsumer(consumer);
+
+ sp<IGraphicBufferProducer> gbProducer;
+ sp<IGraphicBufferConsumer> gbConsumer;
+ BufferQueue::createBufferQueue(&gbProducer, &gbConsumer);
+ sp<ConsumerBase> consumer;
+ sp<CpuConsumer> cpuConsumer;
+ sp<BufferItemConsumer> opaqueConsumer;
+ if (isFormatOpaque(nativeFormat)) {
+ // Use the SW_READ_NEVER usage to tell producer that this format is not for preview or video
+ // encoding. The only possibility will be ZSL output.
+ opaqueConsumer =
+ new BufferItemConsumer(gbConsumer, GRALLOC_USAGE_SW_READ_NEVER, maxImages,
+ /*controlledByApp*/true);
+ if (opaqueConsumer == NULL) {
+ jniThrowRuntimeException(env, "Failed to allocate native opaque consumer");
+ return;
+ }
+ ctx->setOpaqueConsumer(opaqueConsumer);
+ consumer = opaqueConsumer;
+ } else {
+ cpuConsumer = new CpuConsumer(gbConsumer, maxImages, /*controlledByApp*/true);
+ // TODO: throw dvm exOutOfMemoryError?
+ if (cpuConsumer == NULL) {
+ jniThrowRuntimeException(env, "Failed to allocate native CpuConsumer");
+ return;
+ }
+ ctx->setCpuConsumer(cpuConsumer);
+ consumer = cpuConsumer;
+ }
+
ctx->setProducer(gbProducer);
consumer->setFrameAvailableListener(ctx);
ImageReader_setNativeContext(env, thiz, ctx);
@@ -718,23 +840,42 @@
ctx->setBufferHeight(height);
// Set the width/height/format/dataspace to the CpuConsumer
- res = consumer->setDefaultBufferSize(width, height);
- if (res != OK) {
- jniThrowException(env, "java/lang/IllegalStateException",
- "Failed to set CpuConsumer buffer size");
- return;
+ // TODO: below code can be simplified once b/19977701 is fixed.
+ if (isFormatOpaque(nativeFormat)) {
+ res = opaqueConsumer->setDefaultBufferSize(width, height);
+ if (res != OK) {
+ jniThrowException(env, "java/lang/IllegalStateException",
+ "Failed to set opaque consumer buffer size");
+ return;
+ }
+ res = opaqueConsumer->setDefaultBufferFormat(nativeFormat);
+ if (res != OK) {
+ jniThrowException(env, "java/lang/IllegalStateException",
+ "Failed to set opaque consumer buffer format");
+ }
+ res = opaqueConsumer->setDefaultBufferDataSpace(nativeDataspace);
+ if (res != OK) {
+ jniThrowException(env, "java/lang/IllegalStateException",
+ "Failed to set opaque consumer buffer dataSpace");
+ }
+ } else {
+ res = cpuConsumer->setDefaultBufferSize(width, height);
+ if (res != OK) {
+ jniThrowException(env, "java/lang/IllegalStateException",
+ "Failed to set CpuConsumer buffer size");
+ return;
+ }
+ res = cpuConsumer->setDefaultBufferFormat(nativeFormat);
+ if (res != OK) {
+ jniThrowException(env, "java/lang/IllegalStateException",
+ "Failed to set CpuConsumer buffer format");
+ }
+ res = cpuConsumer->setDefaultBufferDataSpace(nativeDataspace);
+ if (res != OK) {
+ jniThrowException(env, "java/lang/IllegalStateException",
+ "Failed to set CpuConsumer buffer dataSpace");
+ }
}
- res = consumer->setDefaultBufferFormat(nativeFormat);
- if (res != OK) {
- jniThrowException(env, "java/lang/IllegalStateException",
- "Failed to set CpuConsumer buffer format");
- }
- res = consumer->setDefaultBufferDataSpace(nativeDataspace);
- if (res != OK) {
- jniThrowException(env, "java/lang/IllegalStateException",
- "Failed to set CpuConsumer buffer dataSpace");
- }
-
}
static void ImageReader_close(JNIEnv* env, jobject thiz)
@@ -747,7 +888,13 @@
return;
}
- CpuConsumer* consumer = ImageReader_getCpuConsumer(env, thiz);
+ ConsumerBase* consumer = NULL;
+ if (ctx->isOpaque()) {
+ consumer = ImageReader_getOpaqueConsumer(env, thiz);
+ } else {
+ consumer = ImageReader_getCpuConsumer(env, thiz);
+ }
+
if (consumer != NULL) {
consumer->abandon();
consumer->setFrameAvailableListener(NULL);
@@ -764,27 +911,66 @@
return;
}
- CpuConsumer* consumer = ctx->getCpuConsumer();
- CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, image);
- if (!buffer) {
- ALOGW("Image already released!!!");
- return;
+ if (ctx->isOpaque()) {
+ BufferItemConsumer* opaqueConsumer = ctx->getOpaqueConsumer();
+ BufferItem* opaqueBuffer = Image_getOpaqueBuffer(env, image);
+ opaqueConsumer->releaseBuffer(*opaqueBuffer); // Not using fence for now.
+ Image_setOpaqueBuffer(env, image, NULL);
+ ctx->returnOpaqueBuffer(opaqueBuffer);
+ ALOGV("%s: Opaque Image has been released", __FUNCTION__);
+ } else {
+ CpuConsumer* consumer = ctx->getCpuConsumer();
+ CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, image);
+ if (!buffer) {
+ ALOGW("Image already released!!!");
+ return;
+ }
+ consumer->unlockBuffer(*buffer);
+ Image_setBuffer(env, image, NULL);
+ ctx->returnLockedBuffer(buffer);
+ ALOGV("%s: Image (format: 0x%x) has been released", __FUNCTION__, ctx->getBufferFormat());
}
- consumer->unlockBuffer(*buffer);
- Image_setBuffer(env, image, NULL);
- ctx->returnLockedBuffer(buffer);
}
-static jint ImageReader_imageSetup(JNIEnv* env, jobject thiz,
- jobject image)
-{
+static jint ImageReader_opaqueImageSetup(JNIEnv* env, JNIImageReaderContext* ctx, jobject image) {
ALOGV("%s:", __FUNCTION__);
- JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz);
- if (ctx == NULL) {
+ if (ctx == NULL || !ctx->isOpaque()) {
jniThrowRuntimeException(env, "ImageReaderContext is not initialized");
return -1;
}
+ BufferItemConsumer* opaqueConsumer = ctx->getOpaqueConsumer();
+ BufferItem* buffer = ctx->getOpaqueBuffer();
+ if (buffer == NULL) {
+ ALOGW("Unable to acquire a buffer item, very likely client tried to acquire more than"
+ " maxImages buffers");
+ return ACQUIRE_MAX_IMAGES;
+ }
+
+ status_t res = opaqueConsumer->acquireBuffer(buffer, 0);
+ if (res != OK) {
+ ctx->returnOpaqueBuffer(buffer);
+ if (res == INVALID_OPERATION) {
+ // Max number of images were already acquired.
+ ALOGE("%s: Max number of buffers allowed are already acquired : %s (%d)",
+ __FUNCTION__, strerror(-res), res);
+ return ACQUIRE_MAX_IMAGES;
+ } else {
+ ALOGE("%s: Acquire image failed with error: %s (%d)",
+ __FUNCTION__, strerror(-res), res);
+ return ACQUIRE_NO_BUFFERS;
+ }
+ }
+
+ // Set SurfaceImage instance member variables
+ Image_setOpaqueBuffer(env, image, buffer);
+ env->SetLongField(image, gSurfaceImageClassInfo.mTimestamp,
+ static_cast<jlong>(buffer->mTimestamp));
+
+ return ACQUIRE_SUCCESS;
+}
+
+static jint ImageReader_lockedImageSetup(JNIEnv* env, JNIImageReaderContext* ctx, jobject image) {
CpuConsumer* consumer = ctx->getCpuConsumer();
CpuConsumer::LockedBuffer* buffer = ctx->getLockedBuffer();
if (buffer == NULL) {
@@ -877,23 +1063,55 @@
return ACQUIRE_SUCCESS;
}
-static void ImageReader_detachImage(JNIEnv* env, jobject thiz, jobject image) {
+static jint ImageReader_imageSetup(JNIEnv* env, jobject thiz, jobject image) {
+ ALOGV("%s:", __FUNCTION__);
+ JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz);
+ if (ctx == NULL) {
+ jniThrowRuntimeException(env, "ImageReaderContext is not initialized");
+ return -1;
+ }
+
+ if (ctx->isOpaque()) {
+ return ImageReader_opaqueImageSetup(env, ctx, image);
+ } else {
+ return ImageReader_lockedImageSetup(env, ctx, image);
+ }
+}
+
+static jint ImageReader_detachImage(JNIEnv* env, jobject thiz, jobject image) {
ALOGV("%s:", __FUNCTION__);
JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz);
if (ctx == NULL) {
jniThrowException(env, "java/lang/IllegalStateException", "ImageReader was already closed");
- return;
+ return -1;
}
- // CpuConsumer* consumer = ctx->getCpuConsumer();
- CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, image);
- if (!buffer) {
- ALOGW("Image already released!!!");
- return;
+ status_t res = OK;
+ if (!ctx->isOpaque()) {
+ // TODO: Non-Opaque format detach is not implemented yet.
+ jniThrowRuntimeException(env,
+ "nativeDetachImage is not implemented yet for non-opaque format !!!");
+ return -1;
}
- // TODO: need implement
- jniThrowRuntimeException(env, "nativeDetachImage is not implemented yet!!!");
+ BufferItemConsumer* opaqueConsumer = ctx->getOpaqueConsumer();
+ BufferItem* opaqueBuffer = Image_getOpaqueBuffer(env, image);
+ if (!opaqueBuffer) {
+ ALOGE(
+ "Opaque Image already released and can not be detached from ImageReader!!!");
+ jniThrowException(env, "java/lang/IllegalStateException",
+ "Opaque Image detach from ImageReader failed: buffer was already released");
+ return -1;
+ }
+
+ res = opaqueConsumer->detachBuffer(opaqueBuffer->mSlot);
+ if (res != OK) {
+ ALOGE("Opaque Image detach failed: %s (%d)!!!", strerror(-res), res);
+ jniThrowRuntimeException(env,
+ "nativeDetachImage failed for opaque image!!!");
+ return res;
+ }
+ return OK;
}
static jobject ImageReader_getSurface(JNIEnv* env, jobject thiz)
@@ -914,8 +1132,15 @@
{
int rowStride, pixelStride;
PublicFormat publicReaderFormat = static_cast<PublicFormat>(readerFormat);
+ int halReaderFormat = android_view_Surface_mapPublicFormatToHalFormat(
+ publicReaderFormat);
ALOGV("%s: buffer index: %d", __FUNCTION__, idx);
+ if (isFormatOpaque(halReaderFormat)) {
+ jniThrowException(env, "java/lang/IllegalStateException",
+ "Opaque images from Opaque ImageReader do not have any planes");
+ return NULL;
+ }
CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, thiz);
@@ -924,9 +1149,6 @@
jniThrowException(env, "java/lang/IllegalStateException", "Image was released");
}
- int halReaderFormat = android_view_Surface_mapPublicFormatToHalFormat(
- publicReaderFormat);
-
rowStride = Image_imageGetRowStride(env, buffer, idx, halReaderFormat);
pixelStride = Image_imageGetPixelStride(env, buffer, idx, halReaderFormat);
@@ -942,18 +1164,23 @@
uint32_t size = 0;
jobject byteBuffer;
PublicFormat readerPublicFormat = static_cast<PublicFormat>(readerFormat);
+ int readerHalFormat = android_view_Surface_mapPublicFormatToHalFormat(
+ readerPublicFormat);
ALOGV("%s: buffer index: %d", __FUNCTION__, idx);
+ if (isFormatOpaque(readerHalFormat)) {
+ jniThrowException(env, "java/lang/IllegalStateException",
+ "Opaque images from Opaque ImageReader do not have any plane");
+ return NULL;
+ }
+
CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, thiz);
if (buffer == NULL) {
jniThrowException(env, "java/lang/IllegalStateException", "Image was released");
}
- int readerHalFormat = android_view_Surface_mapPublicFormatToHalFormat(
- readerPublicFormat);
-
// Create byteBuffer from native buffer
Image_getLockedBufferInfo(env, buffer, idx, &base, &size, readerHalFormat);
@@ -973,19 +1200,28 @@
return byteBuffer;
}
-static jint Image_getWidth(JNIEnv* env, jobject thiz)
+static jint Image_getWidth(JNIEnv* env, jobject thiz, jint format)
{
- CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, thiz);
- return Image_getBufferWidth(buffer);
+ if (isFormatOpaque(format)) {
+ BufferItem* opaqueBuffer = Image_getOpaqueBuffer(env, thiz);
+ return Image_getOpaqueBufferWidth(opaqueBuffer);
+ } else {
+ CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, thiz);
+ return Image_getBufferWidth(buffer);
+ }
}
-static jint Image_getHeight(JNIEnv* env, jobject thiz)
+static jint Image_getHeight(JNIEnv* env, jobject thiz, jint format)
{
- CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, thiz);
- return Image_getBufferHeight(buffer);
+ if (isFormatOpaque(format)) {
+ BufferItem* opaqueBuffer = Image_getOpaqueBuffer(env, thiz);
+ return Image_getOpaqueBufferHeight(opaqueBuffer);
+ } else {
+ CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, thiz);
+ return Image_getBufferHeight(buffer);
+ }
}
-
} // extern "C"
// ----------------------------------------------------------------------------
@@ -997,15 +1233,15 @@
{"nativeReleaseImage", "(Landroid/media/Image;)V", (void*)ImageReader_imageRelease },
{"nativeImageSetup", "(Landroid/media/Image;)I", (void*)ImageReader_imageSetup },
{"nativeGetSurface", "()Landroid/view/Surface;", (void*)ImageReader_getSurface },
- {"nativeDetachImage", "(Landroid/media/Image;)V", (void*)ImageReader_detachImage },
+ {"nativeDetachImage", "(Landroid/media/Image;)I", (void*)ImageReader_detachImage },
};
static JNINativeMethod gImageMethods[] = {
{"nativeImageGetBuffer", "(II)Ljava/nio/ByteBuffer;", (void*)Image_getByteBuffer },
{"nativeCreatePlane", "(II)Landroid/media/ImageReader$SurfaceImage$SurfacePlane;",
(void*)Image_createSurfacePlane },
- {"nativeGetWidth", "()I", (void*)Image_getWidth },
- {"nativeGetHeight", "()I", (void*)Image_getHeight },
+ {"nativeGetWidth", "(I)I", (void*)Image_getWidth },
+ {"nativeGetHeight", "(I)I", (void*)Image_getHeight },
};
int register_android_media_ImageReader(JNIEnv *env) {
diff --git a/media/jni/android_media_ImageWriter.cpp b/media/jni/android_media_ImageWriter.cpp
index d10df3e..d2c614e 100644
--- a/media/jni/android_media_ImageWriter.cpp
+++ b/media/jni/android_media_ImageWriter.cpp
@@ -74,8 +74,8 @@
// has returned a buffer and it is ready for ImageWriter to dequeue.
virtual void onBufferReleased();
- void setProducer(const sp<ANativeWindow>& producer) { mProducer = producer; }
- ANativeWindow* getProducer() { return mProducer.get(); }
+ void setProducer(const sp<Surface>& producer) { mProducer = producer; }
+ Surface* getProducer() { return mProducer.get(); }
void setBufferFormat(int format) { mFormat = format; }
int getBufferFormat() { return mFormat; }
@@ -90,7 +90,7 @@
static JNIEnv* getJNIEnv(bool* needsDetach);
static void detachJNI();
- sp<ANativeWindow> mProducer;
+ sp<Surface> mProducer;
jobject mWeakThiz;
jclass mClazz;
int mFormat;
@@ -155,10 +155,21 @@
bool needsDetach = false;
JNIEnv* env = getJNIEnv(&needsDetach);
if (env != NULL) {
+ // Detach the buffer every time when a buffer consumption is done,
+ // need let this callback give a BufferItem, then only detach if it was attached to this
+ // Writer. Do the detach unconditionally for opaque format now. see b/19977520
+ if (mFormat == HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED) {
+ sp<Fence> fence;
+ ANativeWindowBuffer* buffer;
+ ALOGV("%s: One buffer is detached", __FUNCTION__);
+ mProducer->detachNextBuffer(&buffer, &fence);
+ }
+
env->CallStaticVoidMethod(mClazz, gImageWriterClassInfo.postEventFromNative, mWeakThiz);
} else {
ALOGW("onBufferReleased event will not posted");
}
+
if (needsDetach) {
detachJNI();
}
@@ -170,13 +181,13 @@
// -------------------------------Private method declarations--------------
-static bool isWritable(int format);
static bool isPossiblyYUV(PixelFormat format);
static void Image_setNativeContext(JNIEnv* env, jobject thiz,
sp<GraphicBuffer> buffer, int fenceFd);
static void Image_getNativeContext(JNIEnv* env, jobject thiz,
GraphicBuffer** buffer, int* fenceFd);
static void Image_unlockIfLocked(JNIEnv* env, jobject thiz);
+static bool isFormatOpaque(int format);
// --------------------------ImageWriter methods---------------------------------------
@@ -278,7 +289,7 @@
env->SetIntField(thiz, gImageWriterClassInfo.mWriterFormat, reinterpret_cast<jint>(format));
- if (isWritable(format)) {
+ if (!isFormatOpaque(format)) {
res = native_window_set_usage(anw.get(), GRALLOC_USAGE_SW_WRITE_OFTEN);
if (res != OK) {
ALOGE("%s: Configure usage %08x for format %08x failed: %s (%d)",
@@ -461,31 +472,87 @@
return;
}
+ // Clear the image native context: end of this image's lifecycle in public API.
Image_setNativeContext(env, image, NULL, -1);
}
-static void ImageWriter_attachImage(JNIEnv* env, jobject thiz, jlong nativeCtx, jobject image) {
+static jint ImageWriter_attachAndQueueImage(JNIEnv* env, jobject thiz, jlong nativeCtx,
+ jlong nativeBuffer, jint imageFormat, jlong timestampNs, jint left, jint top,
+ jint right, jint bottom) {
ALOGV("%s", __FUNCTION__);
JNIImageWriterContext* const ctx = reinterpret_cast<JNIImageWriterContext *>(nativeCtx);
if (ctx == NULL || thiz == NULL) {
jniThrowException(env, "java/lang/IllegalStateException",
"ImageWriterContext is not initialized");
- return;
+ return -1;
}
- sp<ANativeWindow> anw = ctx->getProducer();
+ sp<Surface> surface = ctx->getProducer();
+ status_t res = OK;
+ if (!isFormatOpaque(imageFormat)) {
+ // TODO: need implement, see b/19962027
+ jniThrowRuntimeException(env,
+ "nativeAttachImage for non-opaque image is not implement yet!!!");
+ return -1;
+ }
- GraphicBuffer *buffer = NULL;
- int fenceFd = -1;
- Image_getNativeContext(env, image, &buffer, &fenceFd);
- if (buffer == NULL) {
+ if (!isFormatOpaque(ctx->getBufferFormat())) {
jniThrowException(env, "java/lang/IllegalStateException",
- "Image is not initialized");
- return;
+ "Trying to attach an opaque image into a non-opaque ImageWriter");
+ return -1;
}
- // TODO: need implement
- jniThrowRuntimeException(env, "nativeAttachImage is not implement yet!!!");
+ // Image is guaranteed to be from ImageReader at this point, so it is safe to
+ // cast to BufferItem pointer.
+ BufferItem* opaqueBuffer = reinterpret_cast<BufferItem*>(nativeBuffer);
+ if (opaqueBuffer == NULL) {
+ jniThrowException(env, "java/lang/IllegalStateException",
+ "Image is not initialized or already closed");
+ return -1;
+ }
+
+ // Step 1. Attach Image
+ res = surface->attachBuffer(opaqueBuffer->mGraphicBuffer.get());
+ if (res != OK) {
+ // TODO: handle different error case separately.
+ ALOGE("Attach image failed: %s (%d)", strerror(-res), res);
+ jniThrowRuntimeException(env, "nativeAttachImage failed!!!");
+ return res;
+ }
+ sp < ANativeWindow > anw = surface;
+
+ // Step 2. Set timestamp and crop. Note that we do not need unlock the image because
+ // it was not locked.
+ ALOGV("timestamp to be queued: %" PRId64, timestampNs);
+ res = native_window_set_buffers_timestamp(anw.get(), timestampNs);
+ if (res != OK) {
+ jniThrowRuntimeException(env, "Set timestamp failed");
+ return res;
+ }
+
+ android_native_rect_t cropRect;
+ cropRect.left = left;
+ cropRect.top = top;
+ cropRect.right = right;
+ cropRect.bottom = bottom;
+ res = native_window_set_crop(anw.get(), &cropRect);
+ if (res != OK) {
+ jniThrowRuntimeException(env, "Set crop rect failed");
+ return res;
+ }
+
+ // Step 3. Queue Image.
+ res = anw->queueBuffer(anw.get(), opaqueBuffer->mGraphicBuffer.get(), /*fenceFd*/
+ -1);
+ if (res != OK) {
+ jniThrowRuntimeException(env, "Queue input buffer failed");
+ return res;
+ }
+
+ // Do not set the image native context. Since it would overwrite the existing native context
+ // of the image that is from ImageReader, the subsequent image close will run into issues.
+
+ return res;
}
// --------------------------Image methods---------------------------------------
@@ -534,10 +601,13 @@
// Is locked?
bool isLocked = false;
- jobject planes = env->GetObjectField(thiz, gSurfaceImageClassInfo.mPlanes);
+ jobject planes = NULL;
+ if (!isFormatOpaque(buffer->getPixelFormat())) {
+ planes = env->GetObjectField(thiz, gSurfaceImageClassInfo.mPlanes);
+ }
isLocked = (planes != NULL);
if (isLocked) {
- // no need to use fence here, as we it will be consumed by either concel or queue buffer.
+ // no need to use fence here, as we it will be consumed by either cancel or queue buffer.
status_t res = buffer->unlock();
if (res != OK) {
jniThrowRuntimeException(env, "unlock buffer failed");
@@ -900,7 +970,7 @@
jobject byteBuffer;
int format = Image_getFormat(env, thiz);
- if (!isWritable(format) && numPlanes > 0) {
+ if (isFormatOpaque(format) && numPlanes > 0) {
String8 msg;
msg.appendFormat("Format 0x%x is opaque, thus not writable, the number of planes (%d)"
" must be 0", format, numPlanes);
@@ -915,6 +985,9 @@
" probably out of memory");
return NULL;
}
+ if (isFormatOpaque(format)) {
+ return surfacePlanes;
+ }
// Buildup buffer info: rowStride, pixelStride and byteBuffers.
LockedImage lockedImg = LockedImage();
@@ -943,13 +1016,9 @@
// -------------------------------Private convenience methods--------------------
-// Check if buffer with this format is writable. Generally speaking, the opaque formats
-// like IMPLEMENTATION_DEFINED is not writable, as the actual buffer formats and layouts
-// are unknown to frameworks.
-static bool isWritable(int format) {
- // Assume all other formats are writable.
- return !(format == HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED ||
- format == HAL_PIXEL_FORMAT_RAW_OPAQUE);
+static bool isFormatOpaque(int format) {
+ // Only treat IMPLEMENTATION_DEFINED as an opaque format for now.
+ return format == HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED;
}
static bool isPossiblyYUV(PixelFormat format) {
@@ -986,8 +1055,8 @@
{"nativeClassInit", "()V", (void*)ImageWriter_classInit },
{"nativeInit", "(Ljava/lang/Object;Landroid/view/Surface;I)J",
(void*)ImageWriter_init },
- {"nativeClose", "(J)V", (void*)ImageWriter_close },
- {"nativeAttachImage", "(JLandroid/media/Image;)V", (void*)ImageWriter_attachImage },
+ {"nativeClose", "(J)V", (void*)ImageWriter_close },
+ {"nativeAttachAndQueueImage", "(JJIJIIII)I", (void*)ImageWriter_attachAndQueueImage },
{"nativeDequeueInputImage", "(JLandroid/media/Image;)V", (void*)ImageWriter_dequeueImage },
{"nativeQueueInputImage", "(JLandroid/media/Image;JIIII)V", (void*)ImageWriter_queueImage },
{"cancelImage", "(JLandroid/media/Image;)V", (void*)ImageWriter_cancelImage },