ImageReader: Implementation of ImageReader and Image APIs
Used for direct image data access from producer like camera or video decoder.
Bug: 9254294
Change-Id: I1853af03f4487ac3585d86202f6140854471fa89
diff --git a/api/current.txt b/api/current.txt
index 8d11dd0..2d92651 100644
--- a/api/current.txt
+++ b/api/current.txt
@@ -11816,26 +11816,24 @@
field public static final int EULER_Z = 2; // 0x2
}
- public abstract class Image implements java.lang.AutoCloseable {
- ctor public Image();
+ public abstract interface Image implements java.lang.AutoCloseable {
method public abstract void close();
- method protected final void finalize();
- method public int getFormat();
- method public int getHeight();
- method public android.media.Image.Plane[] getPlanes();
- method public long getTimestamp();
- method public int getWidth();
+ method public abstract int getFormat();
+ method public abstract int getHeight();
+ method public abstract android.media.Image.Plane[] getPlanes();
+ method public abstract long getTimestamp();
+ method public abstract int getWidth();
}
- public static final class Image.Plane {
- ctor public Image.Plane();
- method public java.nio.ByteBuffer getBuffer();
- method public int getPixelStride();
- method public int getRowStride();
+ public static abstract interface Image.Plane {
+ method public abstract java.nio.ByteBuffer getBuffer();
+ method public abstract int getPixelStride();
+ method public abstract int getRowStride();
}
- public final class ImageReader {
+ public final class ImageReader implements java.lang.AutoCloseable {
ctor public ImageReader(int, int, int, int);
+ method public void close();
method public int getHeight();
method public int getImageFormat();
method public int getMaxImages();
@@ -11843,7 +11841,7 @@
method public android.view.Surface getSurface();
method public int getWidth();
method public void releaseImage(android.media.Image);
- method public void setOnImageAvailableListener(android.media.ImageReader.OnImageAvailableListener);
+ method public void setImageAvailableListener(android.media.ImageReader.OnImageAvailableListener, android.os.Handler);
}
public static abstract interface ImageReader.OnImageAvailableListener {
diff --git a/media/java/android/media/Image.java b/media/java/android/media/Image.java
index eb94346..f55756c 100644
--- a/media/java/android/media/Image.java
+++ b/media/java/android/media/Image.java
@@ -44,7 +44,7 @@
*
* @see ImageReader
*/
-public abstract class Image implements AutoCloseable {
+public interface Image extends AutoCloseable {
/**
* Get the format for this image. This format determines the number of
* ByteBuffers needed to represent the image, and the general layout of the
@@ -87,25 +87,19 @@
*
* @see android.graphics.ImageFormat
*/
- public int getFormat() {
- return ImageFormat.UNKNOWN;
- }
+ public int getFormat();
/**
* The width of the image in pixels. For formats where some color channels
* are subsampled, this is the width of the largest-resolution plane.
*/
- public int getWidth() {
- return 0;
- }
+ public int getWidth();
/**
* The height of the image in pixels. For formats where some color channels
* are subsampled, this is the height of the largest-resolution plane.
*/
- public int getHeight() {
- return 0;
- }
+ public int getHeight();
/**
* Get the timestamp associated with this frame. The timestamp is measured
@@ -113,17 +107,13 @@
* and whether the timestamp can be compared against other sources of time
* or images depend on the source of this image.
*/
- public long getTimestamp() {
- return 0;
- }
+ public long getTimestamp();
/**
* Get the array of pixel planes for this Image. The number of planes is
* determined by the format of the Image.
*/
- public Plane[] getPlanes() {
- return null;
- }
+ public Plane[] getPlanes();
/**
* Free up this frame for reuse. After calling this method, calling any
@@ -131,11 +121,7 @@
* attempting to read from ByteBuffers returned by an earlier
* {@code Plane#getBuffer} call will have undefined behavior.
*/
- public abstract void close();
-
- protected final void finalize() {
- close();
- }
+ public void close();
/**
* <p>A single color plane of image data.</p>
@@ -148,17 +134,14 @@
*
* @see #getFormat
*/
- public static final class Plane {
+ public interface Plane {
/**
* <p>The row stride for this color plane, in bytes.
*
* <p>This is the distance between the start of two consecutive rows of
* pixels in the image.</p>
*/
- public int getRowStride() {
- return 0;
- }
-
+ public int getRowStride();
/**
* <p>The distance between adjacent pixel samples, in bytes.</p>
*
@@ -166,19 +149,14 @@
* of pixels. It may be larger than the size of a single pixel to
* account for interleaved image data or padded formats.</p>
*/
- public int getPixelStride() {
- return 0;
- }
-
+ public int getPixelStride();
/**
* <p>Get a set of direct {@link java.nio.ByteBuffer byte buffers}
* containing the frame data.</p>
*
* @return the byte buffer containing the image data for this plane.
*/
- public ByteBuffer getBuffer() {
- return null;
- }
+ public ByteBuffer getBuffer();
}
}
diff --git a/media/java/android/media/ImageReader.java b/media/java/android/media/ImageReader.java
index 9384c14..8f09b54 100644
--- a/media/java/android/media/ImageReader.java
+++ b/media/java/android/media/ImageReader.java
@@ -16,8 +16,15 @@
package android.media;
+import android.graphics.ImageFormat;
+import android.os.Handler;
+import android.os.Looper;
+import android.os.Message;
import android.view.Surface;
-import java.lang.AutoCloseable;
+
+import java.lang.ref.WeakReference;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
/**
* <p>The ImageReader class allows direct application access to image data
@@ -39,7 +46,7 @@
* ImageReader does not obtain and release Images at a rate equal to the
* production rate.</p>
*/
-public final class ImageReader {
+public final class ImageReader implements AutoCloseable {
/**
* <p>Create a new reader for images of the desired size and format.</p>
@@ -62,7 +69,7 @@
* access simultaneously. This should be as small as possible to limit
* memory use. Once maxImages Images are obtained by the user, one of them
* has to be released before a new Image will become available for access
- * through getImage(). Must be greater than 0.
+ * through getNextImage(). Must be greater than 0.
*
* @see Image
*/
@@ -80,6 +87,12 @@
throw new IllegalArgumentException(
"Maximum outstanding image count must be at least 1");
}
+
+ mNumPlanes = getNumPlanesFromFormat();
+
+ nativeInit(new WeakReference<ImageReader>(this), width, height, format, maxImages);
+
+ mSurface = nativeGetSurface();
}
public int getWidth() {
@@ -111,7 +124,7 @@
* @return A Surface to use for a drawing target for various APIs.
*/
public Surface getSurface() {
- return null;
+ return mSurface;
}
/**
@@ -122,6 +135,13 @@
* available.
*/
public Image getNextImage() {
+ SurfaceImage si = new SurfaceImage();
+ if (nativeImageSetup(si)) {
+ // create SurfacePlane objects
+ si.createSurfacePlanes();
+ si.setImageValid(true);
+ return si;
+ }
return null;
}
@@ -138,34 +158,307 @@
throw new IllegalArgumentException(
"This image was not produced by this ImageReader");
}
+
+ si.clearSurfacePlanes();
+ nativeReleaseImage(i);
+ si.setImageValid(false);
}
- public void setOnImageAvailableListener(OnImageAvailableListener l) {
- mImageListener = l;
+ /**
+ * Register a listener to be invoked when a new image becomes available
+ * from the ImageReader.
+ * @param listener the listener that will be run
+ * @param handler The handler on which the listener should be invoked, or null
+ * if the listener should be invoked on the calling thread's looper.
+ */
+ public void setImageAvailableListener(OnImageAvailableListener listener, Handler handler) {
+ mImageListener = listener;
+
+ Looper looper;
+ mHandler = handler;
+ if (mHandler == null) {
+ if ((looper = Looper.myLooper()) != null) {
+ mHandler = new Handler();
+ } else {
+ throw new IllegalArgumentException(
+ "Looper doesn't exist in the calling thread");
+ }
+ }
}
+ /**
+ * Callback interface for being notified that a new image is available.
+ * The onImageAvailable is called per image basis, that is, callback fires for every new frame
+ * available from ImageReader.
+ */
public interface OnImageAvailableListener {
+ /**
+ * Callback that is called when a new image is available from ImageReader.
+ * @param reader the ImageReader the callback is associated with.
+ * @see ImageReader
+ * @see Image
+ */
void onImageAvailable(ImageReader reader);
}
+ /**
+ * Free up all the resources associated with this ImageReader. After
+ * Calling this method, this ImageReader can not be used. calling
+ * any methods on this ImageReader and Images previously provided by {@link #getNextImage}
+ * will result in an IllegalStateException, and attempting to read from
+ * ByteBuffers returned by an earlier {@code Plane#getBuffer} call will
+ * have undefined behavior.
+ */
+ @Override
+ public void close() {
+ nativeClose();
+ }
+
+ @Override
+ protected void finalize() throws Throwable {
+ try {
+ close();
+ } finally {
+ super.finalize();
+ }
+ }
+
+ private int getNumPlanesFromFormat() {
+ switch (mFormat) {
+ case ImageFormat.YV12:
+ case ImageFormat.YUV_420_888:
+ case ImageFormat.NV21:
+ return 3;
+ case ImageFormat.NV16:
+ return 2;
+ case ImageFormat.RGB_565:
+ case ImageFormat.JPEG:
+ case ImageFormat.YUY2:
+ case ImageFormat.Y8:
+ case ImageFormat.Y16:
+ case ImageFormat.RAW_SENSOR:
+ return 1;
+ default:
+ throw new UnsupportedOperationException(
+ String.format("Invalid format specified %d", mFormat));
+ }
+ }
+
+ /**
+ * Called from Native code when an Event happens.
+ */
+ private static void postEventFromNative(Object selfRef) {
+ WeakReference weakSelf = (WeakReference)selfRef;
+ final ImageReader ir = (ImageReader)weakSelf.get();
+ if (ir == null) {
+ return;
+ }
+
+ if (ir.mHandler != null) {
+ ir.mHandler.post(new Runnable() {
+ @Override
+ public void run() {
+ ir.mImageListener.onImageAvailable(ir);
+ }
+ });
+ }
+ }
+
private final int mWidth;
private final int mHeight;
private final int mFormat;
private final int mMaxImages;
+ private final int mNumPlanes;
+ private final Surface mSurface;
+ private Handler mHandler;
private OnImageAvailableListener mImageListener;
- private class SurfaceImage extends android.media.Image {
+ /**
+ * This field is used by native code, do not access or modify.
+ */
+ private long mNativeContext;
+
+ private class SurfaceImage implements android.media.Image {
public SurfaceImage() {
+ mIsImageValid = false;
}
@Override
public void close() {
- ImageReader.this.releaseImage(this);
+ if (mIsImageValid) {
+ ImageReader.this.releaseImage(this);
+ }
}
public ImageReader getReader() {
return ImageReader.this;
}
+
+ @Override
+ public int getFormat() {
+ if (mIsImageValid) {
+ return ImageReader.this.mFormat;
+ } else {
+ throw new IllegalStateException("Image is already released");
+ }
+ }
+
+ @Override
+ public int getWidth() {
+ if (mIsImageValid) {
+ return ImageReader.this.mWidth;
+ } else {
+ throw new IllegalStateException("Image is already released");
+ }
+ }
+
+ @Override
+ public int getHeight() {
+ if (mIsImageValid) {
+ return ImageReader.this.mHeight;
+ } else {
+ throw new IllegalStateException("Image is already released");
+ }
+ }
+
+ @Override
+ public long getTimestamp() {
+ if (mIsImageValid) {
+ return mTimestamp;
+ } else {
+ throw new IllegalStateException("Image is already released");
+ }
+ }
+
+ @Override
+ public Plane[] getPlanes() {
+ if (mIsImageValid) {
+ // Shallow copy is fine.
+ return mPlanes.clone();
+ } else {
+ throw new IllegalStateException("Image is already released");
+ }
+ }
+
+ @Override
+ protected final void finalize() throws Throwable {
+ try {
+ close();
+ } finally {
+ super.finalize();
+ }
+ }
+
+ private void setImageValid(boolean isValid) {
+ mIsImageValid = isValid;
+ }
+
+ private boolean isImageValid() {
+ return mIsImageValid;
+ }
+
+ private void clearSurfacePlanes() {
+ if (mIsImageValid) {
+ for (int i = 0; i < mPlanes.length; i++) {
+ if (mPlanes[i] != null) {
+ mPlanes[i].clearBuffer();
+ mPlanes[i] = null;
+ }
+ }
+ }
+ }
+
+ private void createSurfacePlanes() {
+ mPlanes = new SurfacePlane[ImageReader.this.mNumPlanes];
+ for (int i = 0; i < ImageReader.this.mNumPlanes; i++) {
+ mPlanes[i] = nativeCreatePlane(i);
+ }
+ }
+ private class SurfacePlane implements android.media.Image.Plane {
+ // SurfacePlane instance is created by native code when a new SurfaceImage is created
+ private SurfacePlane(int index, int rowStride, int pixelStride) {
+ mIndex = index;
+ mRowStride = rowStride;
+ mPixelStride = pixelStride;
+ }
+
+ @Override
+ public ByteBuffer getBuffer() {
+ if (SurfaceImage.this.isImageValid() == false) {
+ throw new IllegalStateException("Image is already released");
+ }
+ if (mBuffer != null) {
+ return mBuffer;
+ } else {
+ mBuffer = SurfaceImage.this.nativeImageGetBuffer(mIndex);
+ // Set the byteBuffer order according to host endianness (native order),
+ // otherwise, the byteBuffer order defaults to ByteOrder.BIG_ENDIAN.
+ return mBuffer.order(ByteOrder.nativeOrder());
+ }
+ }
+
+ @Override
+ public int getPixelStride() {
+ if (SurfaceImage.this.isImageValid()) {
+ return mPixelStride;
+ } else {
+ throw new IllegalStateException("Image is already released");
+ }
+ }
+
+ @Override
+ public int getRowStride() {
+ if (SurfaceImage.this.isImageValid()) {
+ return mRowStride;
+ } else {
+ throw new IllegalStateException("Image is already released");
+ }
+ }
+
+ private void clearBuffer() {
+ mBuffer = null;
+ }
+
+ final private int mIndex;
+ final private int mPixelStride;
+ final private int mRowStride;
+
+ private ByteBuffer mBuffer;
+ }
+
+ /**
+ * This field is used to keep track of native object and used by native code only.
+ * Don't modify.
+ */
+ private long mLockedBuffer;
+
+ /**
+ * This field is set by native code during nativeImageSetup().
+ */
+ private long mTimestamp;
+
+ private SurfacePlane[] mPlanes;
+ private boolean mIsImageValid;
+
+ private synchronized native ByteBuffer nativeImageGetBuffer(int idx);
+ private synchronized native SurfacePlane nativeCreatePlane(int idx);
+ }
+
+ private synchronized native void nativeInit(Object weakSelf, int w, int h,
+ int fmt, int maxImgs);
+ private synchronized native void nativeClose();
+ private synchronized native void nativeReleaseImage(Image i);
+ private synchronized native Surface nativeGetSurface();
+ private synchronized native boolean nativeImageSetup(Image i);
+
+ /*
+ * We use a class initializer to allow the native code to cache some
+ * field offsets.
+ */
+ private static native void nativeClassInit();
+ static {
+ System.loadLibrary("media_jni");
+ nativeClassInit();
}
}
diff --git a/media/jni/Android.mk b/media/jni/Android.mk
index 416a2a1..01b3174 100644
--- a/media/jni/Android.mk
+++ b/media/jni/Android.mk
@@ -2,6 +2,7 @@
include $(CLEAR_VARS)
LOCAL_SRC_FILES:= \
+ android_media_ImageReader.cpp \
android_media_MediaCrypto.cpp \
android_media_MediaCodec.cpp \
android_media_MediaCodecList.cpp \
diff --git a/media/jni/android_media_ImageReader.cpp b/media/jni/android_media_ImageReader.cpp
new file mode 100644
index 0000000..bdb07a6
--- /dev/null
+++ b/media/jni/android_media_ImageReader.cpp
@@ -0,0 +1,747 @@
+/*
+ * Copyright 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+//#define LOG_NDEBUG 0
+#define LOG_TAG "ImageReader_JNI"
+#include <utils/Log.h>
+#include <utils/misc.h>
+#include <utils/List.h>
+
+#include <cstdio>
+
+#include <gui/CpuConsumer.h>
+#include <gui/Surface.h>
+
+#include <android_runtime/AndroidRuntime.h>
+#include <android_runtime/android_view_Surface.h>
+
+#include <jni.h>
+#include <JNIHelp.h>
+
+#define ALIGN(x, mask) ( ((x) + (mask) - 1) & ~((mask) - 1) )
+
+#define ANDROID_MEDIA_IMAGEREADER_JNI_ID "mCpuConsumer"
+#define ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID "mNativeContext"
+#define ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID "mLockedBuffer"
+#define ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID "mTimestamp"
+
+// ----------------------------------------------------------------------------
+
+using namespace android;
+
+enum {
+ IMAGE_READER_MAX_NUM_PLANES = 3,
+};
+
+struct fields_t {
+ // For ImageReader class
+ jfieldID imageReaderContext;
+ jmethodID postEvent;
+ // For SurfaceImage class
+ jfieldID buffer;
+ jfieldID timeStamp;
+};
+
+struct classInfo_t {
+ jclass clazz;
+ jmethodID ctor;
+};
+
+static fields_t fields;
+static classInfo_t surfPlaneClassInfo;
+
+// ----------------------------------------------------------------------------
+
+class JNIImageReaderContext : public CpuConsumer::FrameAvailableListener
+{
+public:
+ JNIImageReaderContext(JNIEnv* env, jobject weakThiz, jclass clazz, int maxImages);
+
+ virtual ~JNIImageReaderContext();
+
+ virtual void onFrameAvailable();
+
+ CpuConsumer::LockedBuffer* getLockedBuffer();
+
+ void returnLockedBuffer(CpuConsumer::LockedBuffer* buffer);
+
+ CpuConsumer* getCpuConsumer() { return mConsumer.get(); }
+
+ void setCpuConsumer(sp<CpuConsumer> consumer) { mConsumer = consumer; }
+
+ void setBufferFormat(int format) { mFormat = format; }
+ int getBufferFormat() { return mFormat; }
+
+ void setBufferWidth(int width) { mWidth = width; }
+ int getBufferWidth() { return mWidth; }
+
+ void setBufferHeight(int height) { mHeight = height; }
+ int getBufferHeight() { return mHeight; }
+
+private:
+ static JNIEnv* getJNIEnv(bool* needsDetach);
+ static void detachJNI();
+
+ List<CpuConsumer::LockedBuffer*> mBuffers;
+ sp<CpuConsumer> mConsumer;
+ jobject mWeakThiz;
+ jclass mClazz;
+ int mFormat;
+ int mWidth;
+ int mHeight;
+};
+
+JNIImageReaderContext::JNIImageReaderContext(JNIEnv* env,
+ jobject weakThiz, jclass clazz, int maxImages) :
+ mWeakThiz(env->NewGlobalRef(weakThiz)),
+ mClazz((jclass)env->NewGlobalRef(clazz)) {
+ for (int i = 0; i < maxImages; i++) {
+ CpuConsumer::LockedBuffer *buffer = new CpuConsumer::LockedBuffer;
+ mBuffers.push_back(buffer);
+ }
+}
+
+JNIEnv* JNIImageReaderContext::getJNIEnv(bool* needsDetach) {
+ LOG_ALWAYS_FATAL_IF(needsDetach == NULL, "needsDetach is null!!!");
+ *needsDetach = false;
+ JNIEnv* env = AndroidRuntime::getJNIEnv();
+ if (env == NULL) {
+ JavaVMAttachArgs args = {JNI_VERSION_1_4, NULL, NULL};
+ JavaVM* vm = AndroidRuntime::getJavaVM();
+ int result = vm->AttachCurrentThread(&env, (void*) &args);
+ if (result != JNI_OK) {
+ ALOGE("thread attach failed: %#x", result);
+ return NULL;
+ }
+ *needsDetach = true;
+ }
+ return env;
+}
+
+void JNIImageReaderContext::detachJNI() {
+ JavaVM* vm = AndroidRuntime::getJavaVM();
+ int result = vm->DetachCurrentThread();
+ if (result != JNI_OK) {
+ ALOGE("thread detach failed: %#x", result);
+ }
+}
+
+CpuConsumer::LockedBuffer* JNIImageReaderContext::getLockedBuffer() {
+ if (mBuffers.empty()) {
+ return NULL;
+ }
+ // Return a LockedBuffer pointer and remove it from the list
+ List<CpuConsumer::LockedBuffer*>::iterator it = mBuffers.begin();
+ CpuConsumer::LockedBuffer* buffer = *it;
+ mBuffers.erase(it);
+ return buffer;
+}
+
+void JNIImageReaderContext::returnLockedBuffer(CpuConsumer::LockedBuffer * buffer) {
+ mBuffers.push_back(buffer);
+}
+
+JNIImageReaderContext::~JNIImageReaderContext() {
+ bool needsDetach = false;
+ JNIEnv* env = getJNIEnv(&needsDetach);
+ if (env != NULL) {
+ env->DeleteGlobalRef(mWeakThiz);
+ env->DeleteGlobalRef(mClazz);
+ } else {
+ ALOGW("leaking JNI object references");
+ }
+ if (needsDetach) {
+ detachJNI();
+ }
+
+ // Delete LockedBuffers
+ for (List<CpuConsumer::LockedBuffer *>::iterator it = mBuffers.begin();
+ it != mBuffers.end(); it++) {
+ delete *it;
+ }
+ mBuffers.clear();
+ mConsumer.clear();
+}
+
+void JNIImageReaderContext::onFrameAvailable()
+{
+ ALOGV("%s: frame available", __FUNCTION__);
+ bool needsDetach = false;
+ JNIEnv* env = getJNIEnv(&needsDetach);
+ if (env != NULL) {
+ env->CallStaticVoidMethod(mClazz, fields.postEvent, mWeakThiz);
+ } else {
+ ALOGW("onFrameAvailable event will not posted");
+ }
+ if (needsDetach) {
+ detachJNI();
+ }
+}
+
+// ----------------------------------------------------------------------------
+
+extern "C" {
+
+static JNIImageReaderContext* ImageReader_getContext(JNIEnv* env, jobject thiz)
+{
+ JNIImageReaderContext *ctx;
+ ctx = reinterpret_cast<JNIImageReaderContext *>
+ (env->GetLongField(thiz, fields.imageReaderContext));
+ return ctx;
+}
+
+static CpuConsumer* ImageReader_getCpuConsumer(JNIEnv* env, jobject thiz)
+{
+ ALOGV("%s:", __FUNCTION__);
+ JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz);
+ if (ctx == NULL) {
+ jniThrowRuntimeException(env, "ImageReaderContext is not initialized");
+ return NULL;
+ }
+ return ctx->getCpuConsumer();
+}
+
+static void ImageReader_setNativeContext(JNIEnv* env,
+ jobject thiz, sp<JNIImageReaderContext> ctx)
+{
+ ALOGV("%s:", __FUNCTION__);
+ JNIImageReaderContext* const p = ImageReader_getContext(env, thiz);
+ if (ctx != 0) {
+ ctx->incStrong((void*)ImageReader_setNativeContext);
+ }
+ if (p) {
+ p->decStrong((void*)ImageReader_setNativeContext);
+ }
+ env->SetLongField(thiz, fields.imageReaderContext, reinterpret_cast<jlong>(ctx.get()));
+}
+
+static CpuConsumer::LockedBuffer* Image_getLockedBuffer(JNIEnv* env, jobject image)
+{
+ return reinterpret_cast<CpuConsumer::LockedBuffer*>(env->GetLongField(image, fields.buffer));
+}
+
+static void Image_setBuffer(JNIEnv* env, jobject thiz,
+ const CpuConsumer::LockedBuffer* buffer)
+{
+ env->SetLongField(thiz, fields.buffer, reinterpret_cast<jlong>(buffer));
+}
+
+// Some formats like JPEG defined with different values between android.graphics.ImageFormat and
+// graphics.h, need convert to the one defined in graphics.h here.
+static int Image_getPixelFormat(JNIEnv* env, int format)
+{
+ int jpegFormat, rawSensorFormat;
+ jfieldID fid;
+
+ ALOGV("%s: format = 0x%x", __FUNCTION__, format);
+
+ jclass imageFormatClazz = env->FindClass("android/graphics/ImageFormat");
+ ALOG_ASSERT(imageFormatClazz != NULL);
+
+ fid = env->GetStaticFieldID(imageFormatClazz, "JPEG", "I");
+ jpegFormat = env->GetStaticIntField(imageFormatClazz, fid);
+ fid = env->GetStaticFieldID(imageFormatClazz, "RAW_SENSOR", "I");
+ rawSensorFormat = env->GetStaticIntField(imageFormatClazz, fid);
+
+ // Translate the JPEG to BLOB for camera purpose, an add more if more mismatch is found.
+ if (format == jpegFormat) {
+ format = HAL_PIXEL_FORMAT_BLOB;
+ }
+ // Same thing for RAW_SENSOR format
+ if (format == rawSensorFormat) {
+ format = HAL_PIXEL_FORMAT_RAW_SENSOR;
+ }
+
+ return format;
+}
+
+static void Image_getLockedBufferInfo(JNIEnv* env, CpuConsumer::LockedBuffer* buffer, int idx,
+ uint8_t **base, uint32_t *size)
+{
+ ALOG_ASSERT(buffer != NULL, "Input buffer is NULL!!!");
+ ALOG_ASSERT(base != NULL, "base is NULL!!!");
+ ALOG_ASSERT(size != NULL, "size is NULL!!!");
+ ALOG_ASSERT((idx < IMAGE_READER_MAX_NUM_PLANES) && (idx >= 0));
+
+ ALOGV("%s: buffer: 0x%p", __FUNCTION__, buffer);
+
+ uint32_t dataSize, ySize, cSize, cStride;
+ uint8_t *cb, *cr;
+ uint8_t *pData = NULL;
+
+ dataSize = ySize = cSize = cStride = 0;
+ int32_t fmt = buffer->format;
+ switch (fmt) {
+ case HAL_PIXEL_FORMAT_YCbCr_420_888:
+ pData =
+ (idx == 0) ?
+ buffer->data :
+ (idx == 1) ?
+ buffer->dataCb :
+ buffer->dataCr;
+ if (idx == 0) {
+ dataSize = buffer->stride * buffer->height;
+ } else {
+ dataSize = buffer->chromaStride * buffer->height / 2;
+ }
+ break;
+ // NV21
+ case HAL_PIXEL_FORMAT_YCrCb_420_SP:
+ cr = buffer->data + (buffer->stride * buffer->height);
+ cb = cr + 1;
+ ySize = buffer->width * buffer->height;
+ cSize = buffer->width * buffer->height / 2;
+
+ pData =
+ (idx == 0) ?
+ buffer->data :
+ (idx == 1) ?
+ cb:
+ cr;
+
+ dataSize = (idx == 0) ? ySize : cSize;
+ break;
+ case HAL_PIXEL_FORMAT_YV12:
+ // Y and C stride need to be 16 pixel aligned.
+ LOG_ALWAYS_FATAL_IF(buffer->stride % 16,
+ "Stride is not 16 pixel aligned %d", buffer->stride);
+
+ ySize = buffer->stride * buffer->height;
+ cStride = ALIGN(buffer->stride / 2, 16);
+ cr = buffer->data + ySize;
+ cSize = cStride * buffer->height / 2;
+ cb = cr + cSize;
+
+ pData =
+ (idx == 0) ?
+ buffer->data :
+ (idx == 1) ?
+ cb :
+ cr;
+ dataSize = (idx == 0) ? ySize : cSize;
+ break;
+ case HAL_PIXEL_FORMAT_Y8:
+ // Single plane, 8bpp.
+ ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
+
+ pData = buffer->data;
+ dataSize = buffer->stride * buffer->height;
+ break;
+ case HAL_PIXEL_FORMAT_Y16:
+ // Single plane, 16bpp, strides are specified in pixels, not in bytes
+ ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
+
+ pData = buffer->data;
+ dataSize = buffer->stride * buffer->height * 2;
+ break;
+ case HAL_PIXEL_FORMAT_BLOB:
+ // Used for JPEG data, height must be 1, width == size, single plane.
+ ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
+ ALOG_ASSERT(buffer->height == 1, "JPEG should has height value %d", buffer->height);
+
+ pData = buffer->data;
+ dataSize = buffer->width;
+ break;
+ case HAL_PIXEL_FORMAT_RAW_SENSOR:
+ // Single plane 16bpp bayer data.
+ ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
+ pData = buffer->data;
+ dataSize = buffer->width * 2 * buffer->height;
+ break;
+ default:
+ jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException",
+ "Pixel format: 0x%x is unsupported", fmt);
+ break;
+ }
+
+ *base = pData;
+ *size = dataSize;
+}
+
+static jint Image_imageGetPixelStride(JNIEnv* env, CpuConsumer::LockedBuffer* buffer, int idx)
+{
+ ALOGV("%s: buffer index: %d", __FUNCTION__, idx);
+ ALOG_ASSERT((idx < IMAGE_READER_MAX_NUM_PLANES) && (idx >= 0), "Index is out of range:%d", idx);
+
+ int pixelStride = 0;
+ ALOG_ASSERT(buffer != NULL, "buffer is NULL");
+
+ int32_t fmt = buffer->format;
+ switch (fmt) {
+ case HAL_PIXEL_FORMAT_YCbCr_420_888:
+ pixelStride = (idx == 0) ? 1 : buffer->chromaStep;
+ break;
+ case HAL_PIXEL_FORMAT_YCrCb_420_SP:
+ pixelStride = (idx == 0) ? 1 : 2;
+ break;
+ case HAL_PIXEL_FORMAT_Y8:
+ // Single plane 8bpp data.
+ ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
+ pixelStride;
+ break;
+ case HAL_PIXEL_FORMAT_YV12:
+ pixelStride = 1;
+ break;
+ case HAL_PIXEL_FORMAT_BLOB:
+ // Used for JPEG data, single plane, row and pixel strides are 0
+ ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
+ pixelStride = 0;
+ break;
+ case HAL_PIXEL_FORMAT_Y16:
+ case HAL_PIXEL_FORMAT_RAW_SENSOR:
+ // Single plane 16bpp data.
+ ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
+ pixelStride = 2;
+ break;
+ default:
+ jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException",
+ "Pixel format: 0x%x is unsupported", fmt);
+ break;
+ }
+
+ return pixelStride;
+}
+
+static jint Image_imageGetRowStride(JNIEnv* env, CpuConsumer::LockedBuffer* buffer, int idx)
+{
+ ALOGV("%s: buffer index: %d", __FUNCTION__, idx);
+ ALOG_ASSERT((idx < IMAGE_READER_MAX_NUM_PLANES) && (idx >= 0));
+
+ int rowStride = 0;
+ ALOG_ASSERT(buffer != NULL, "buffer is NULL");
+
+ int32_t fmt = buffer->format;
+
+ switch (fmt) {
+ case HAL_PIXEL_FORMAT_YCbCr_420_888:
+ rowStride = (idx == 0) ? buffer->stride : buffer->chromaStride;
+ break;
+ case HAL_PIXEL_FORMAT_YCrCb_420_SP:
+ rowStride = buffer->width;
+ break;
+ case HAL_PIXEL_FORMAT_YV12:
+ LOG_ALWAYS_FATAL_IF(buffer->stride % 16,
+ "Stride is not 16 pixel aligned %d", buffer->stride);
+ rowStride = (idx == 0) ? buffer->stride : ALIGN(buffer->stride / 2, 16);
+ break;
+ case HAL_PIXEL_FORMAT_BLOB:
+ // Used for JPEG data, single plane, row and pixel strides are 0
+ ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
+ rowStride = 0;
+ break;
+ case HAL_PIXEL_FORMAT_Y8:
+ ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
+ LOG_ALWAYS_FATAL_IF(buffer->stride % 16,
+ "Stride is not 16 pixel aligned %d", buffer->stride);
+ rowStride = buffer->stride;
+ break;
+ case HAL_PIXEL_FORMAT_Y16:
+ case HAL_PIXEL_FORMAT_RAW_SENSOR:
+ // In native side, strides are specified in pixels, not in bytes.
+ // Single plane 16bpp bayer data. even width/height,
+ // row stride multiple of 16 pixels (32 bytes)
+ ALOG_ASSERT(idx == 0, "Wrong index: %d", idx);
+ LOG_ALWAYS_FATAL_IF(buffer->stride % 16,
+ "Stride is not 16 pixel aligned %d", buffer->stride);
+ rowStride = buffer->stride * 2;
+ break;
+ default:
+ ALOGE("%s Pixel format: 0x%x is unsupported", __FUNCTION__, fmt);
+ jniThrowException(env, "java/lang/UnsupportedOperationException",
+ "unsupported buffer format");
+ break;
+ }
+
+ return rowStride;
+}
+
+// ----------------------------------------------------------------------------
+
+static void ImageReader_classInit(JNIEnv* env, jclass clazz)
+{
+ ALOGV("%s:", __FUNCTION__);
+
+ jclass imageClazz = env->FindClass("android/media/ImageReader$SurfaceImage");
+ LOG_ALWAYS_FATAL_IF(imageClazz == NULL,
+ "can't find android/graphics/ImageReader$SurfaceImage");
+ fields.buffer = env->GetFieldID(imageClazz, ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID, "J");
+ LOG_ALWAYS_FATAL_IF(fields.buffer == NULL,
+ "can't find android/graphics/ImageReader.%s",
+ ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID);
+
+ fields.timeStamp = env->GetFieldID(imageClazz, ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID, "J");
+ LOG_ALWAYS_FATAL_IF(fields.timeStamp == NULL,
+ "can't find android/graphics/ImageReader.%s",
+ ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID);
+
+ fields.imageReaderContext = env->GetFieldID(clazz, ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID, "J");
+ LOG_ALWAYS_FATAL_IF(fields.imageReaderContext == NULL,
+ "can't find android/graphics/ImageReader.%s",
+ ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID);
+
+ fields.postEvent = env->GetStaticMethodID(clazz, "postEventFromNative",
+ "(Ljava/lang/Object;)V");
+ LOG_ALWAYS_FATAL_IF(fields.postEvent == NULL,
+ "can't find android/graphics/ImageReader.postEventFromNative");
+
+ jclass planeClazz = env->FindClass("android/media/ImageReader$SurfaceImage$SurfacePlane");
+ LOG_ALWAYS_FATAL_IF(planeClazz == NULL, "Can not find SurfacePlane class");
+ // FindClass only gives a local reference of jclass object.
+ surfPlaneClassInfo.clazz = (jclass) env->NewGlobalRef(planeClazz);
+ surfPlaneClassInfo.ctor = env->GetMethodID(surfPlaneClassInfo.clazz, "<init>",
+ "(Landroid/media/ImageReader$SurfaceImage;III)V");
+ LOG_ALWAYS_FATAL_IF(surfPlaneClassInfo.ctor == NULL, "Can not find SurfacePlane constructor");
+}
+
+static void ImageReader_init(JNIEnv* env, jobject thiz, jobject weakThiz,
+ jint width, jint height, jint format, jint maxImages)
+{
+ status_t res;
+ int nativeFormat;
+
+ ALOGV("%s: width:%d, height: %d, format: 0x%x, maxImages:%d",
+ __FUNCTION__, width, height, format, maxImages);
+
+ nativeFormat = Image_getPixelFormat(env, format);
+
+ sp<CpuConsumer> consumer = new CpuConsumer(maxImages);
+ // TODO: throw dvm exOutOfMemoryError?
+ if (consumer == NULL) {
+ jniThrowRuntimeException(env, "Failed to allocate native CpuConsumer");
+ return;
+ }
+
+ jclass clazz = env->GetObjectClass(thiz);
+ if (clazz == NULL) {
+ jniThrowRuntimeException(env, "Can't find android/graphics/ImageReader");
+ return;
+ }
+ sp<JNIImageReaderContext> ctx(new JNIImageReaderContext(env, weakThiz, clazz, maxImages));
+ ctx->setCpuConsumer(consumer);
+ consumer->setFrameAvailableListener(ctx);
+ ImageReader_setNativeContext(env, thiz, ctx);
+ ctx->setBufferFormat(nativeFormat);
+ ctx->setBufferWidth(width);
+ ctx->setBufferHeight(height);
+
+ // Set the width/height/format to the CpuConsumer
+ res = consumer->setDefaultBufferSize(width, height);
+ if (res != OK) {
+ jniThrowException(env, "java/lang/IllegalStateException",
+ "Failed to set CpuConsumer buffer size");
+ return;
+ }
+ res = consumer->setDefaultBufferFormat(nativeFormat);
+ if (res != OK) {
+ jniThrowException(env, "java/lang/IllegalStateException",
+ "Failed to set CpuConsumer buffer format");
+ }
+}
+
+static void ImageReader_close(JNIEnv* env, jobject thiz)
+{
+ ALOGV("%s:", __FUNCTION__);
+
+ JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz);
+ if (ctx == NULL) {
+ // ImageReader is already closed.
+ return;
+ }
+
+ CpuConsumer* consumer = ImageReader_getCpuConsumer(env, thiz);
+ if (consumer != NULL) {
+ consumer->abandon();
+ consumer->setFrameAvailableListener(NULL);
+ }
+ ImageReader_setNativeContext(env, thiz, NULL);
+}
+
+static void ImageReader_imageRelease(JNIEnv* env, jobject thiz, jobject image)
+{
+ ALOGV("%s:", __FUNCTION__);
+ JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz);
+ if (ctx == NULL) {
+ ALOGW("ImageReader#close called before Image#close, consider calling Image#close first");
+ return;
+ }
+
+ CpuConsumer* consumer = ctx->getCpuConsumer();
+ CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, image);
+ if (!buffer) {
+ ALOGW("Image already released!!!");
+ return;
+ }
+ consumer->unlockBuffer(*buffer);
+ Image_setBuffer(env, image, NULL);
+ ctx->returnLockedBuffer(buffer);
+}
+
+static jboolean ImageReader_imageSetup(JNIEnv* env, jobject thiz,
+ jobject image)
+{
+ ALOGV("%s:", __FUNCTION__);
+ JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz);
+ if (ctx == NULL) {
+ jniThrowRuntimeException(env, "ImageReaderContext is not initialized");
+ return false;
+ }
+
+ CpuConsumer* consumer = ctx->getCpuConsumer();
+ CpuConsumer::LockedBuffer* buffer = ctx->getLockedBuffer();
+ if (buffer == NULL) {
+ ALOGE("Unable to acquire a lockedBuffer, very likely client tries to lock more than"
+ "maxImages buffers");
+ return false;
+ }
+ status_t res = consumer->lockNextBuffer(buffer);
+ if (res != NO_ERROR) {
+ ALOGE("%s Fail to lockNextBuffer with error: 0x%x ", __FUNCTION__, res);
+ return false;
+ }
+
+
+ // Check if the left-top corner of the crop rect is origin, we currently assume this point is
+ // zero, will revist this once this assumption turns out problematic.
+ Point lt = buffer->crop.leftTop();
+ if (lt.x != 0 || lt.y != 0) {
+ ALOGE("crop left: %d, top = %d", lt.x, lt.y);
+ jniThrowException(env, "java/lang/UnsupportedOperationException",
+ "crop left top corner need to at origin");
+ }
+
+ // Check if the producer buffer configurations match what ImageReader configured.
+ // We want to fail for the very first image because this case is too bad.
+ int outputWidth = buffer->crop.getWidth() + 1;
+ int outputHeight = buffer->crop.getHeight() + 1;
+ int imageReaderWidth = ctx->getBufferWidth();
+ int imageReaderHeight = ctx->getBufferHeight();
+ if ((imageReaderWidth != outputWidth) ||
+ (imageReaderHeight != outputHeight)) {
+ // Spew warning for now, since MediaCodec decoder has a bug to setup the right crop
+ // TODO: make it throw exception once the decoder bug is fixed.
+ ALOGW("Producer buffer size: %dx%d, doesn't match ImageReader configured size: %dx%d",
+ outputWidth, outputHeight, imageReaderWidth, imageReaderHeight);
+ }
+
+ if (ctx->getBufferFormat() != buffer->format) {
+ // Return the buffer to the queue.
+ consumer->unlockBuffer(*buffer);
+ ctx->returnLockedBuffer(buffer);
+
+ // Throw exception
+ ALOGE("Producer output buffer format: 0x%x, ImageReader configured format: 0x%x",
+ buffer->format, ctx->getBufferFormat());
+ jniThrowException(env, "java/lang/UnsupportedOperationException",
+ "The producer output buffer configuration doesn't match the ImageReader"
+ "configured");
+ return false;
+ }
+ // Set SurfaceImage instance member variables
+ Image_setBuffer(env, image, buffer);
+ env->SetLongField(image, fields.timeStamp, static_cast<jlong>(buffer->timestamp));
+
+ return true;
+}
+
+static jobject ImageReader_getSurface(JNIEnv* env, jobject thiz)
+{
+ ALOGV("%s: ", __FUNCTION__);
+
+ CpuConsumer* consumer = ImageReader_getCpuConsumer(env, thiz);
+ if (consumer == NULL) {
+ jniThrowRuntimeException(env, "CpuConsumer is uninitialized");
+ return NULL;
+ }
+
+ // Wrap the IGBP in a Java-language Surface.
+ return android_view_Surface_createFromIGraphicBufferProducer(env,
+ consumer->getProducerInterface());
+}
+
+static jobject Image_createSurfacePlane(JNIEnv* env, jobject thiz, int idx)
+{
+ int rowStride, pixelStride;
+ ALOGV("%s: buffer index: %d", __FUNCTION__, idx);
+
+ CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, thiz);
+
+ ALOG_ASSERT(buffer != NULL);
+ if (buffer == NULL) {
+ jniThrowException(env, "java/lang/IllegalStateException", "Image was released");
+ }
+ rowStride = Image_imageGetRowStride(env, buffer, idx);
+ pixelStride = Image_imageGetPixelStride(env, buffer, idx);
+
+ jobject surfPlaneObj = env->NewObject(surfPlaneClassInfo.clazz, surfPlaneClassInfo.ctor,
+ thiz, idx, rowStride, pixelStride);
+
+ return surfPlaneObj;
+}
+
+static jobject Image_getByteBuffer(JNIEnv* env, jobject thiz, int idx)
+{
+ uint8_t *base = NULL;
+ uint32_t size = 0;
+ jobject byteBuffer;
+
+ ALOGV("%s: buffer index: %d", __FUNCTION__, idx);
+
+ CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, thiz);
+
+ if (buffer == NULL) {
+ jniThrowException(env, "java/lang/IllegalStateException", "Image was released");
+ }
+
+ // Create byteBuffer from native buffer
+ Image_getLockedBufferInfo(env, buffer, idx, &base, &size);
+ byteBuffer = env->NewDirectByteBuffer(base, size);
+ // TODO: throw dvm exOutOfMemoryError?
+ if ((byteBuffer == NULL) && (env->ExceptionCheck() == false)) {
+ jniThrowException(env, "java/lang/IllegalStateException", "Failed to allocate ByteBuffer");
+ }
+
+ return byteBuffer;
+}
+
+} // extern "C"
+
+// ----------------------------------------------------------------------------
+
+static JNINativeMethod gImageReaderMethods[] = {
+ {"nativeClassInit", "()V", (void*)ImageReader_classInit },
+ {"nativeInit", "(Ljava/lang/Object;IIII)V", (void*)ImageReader_init },
+ {"nativeClose", "()V", (void*)ImageReader_close },
+ {"nativeReleaseImage", "(Landroid/media/Image;)V", (void*)ImageReader_imageRelease },
+ {"nativeImageSetup", "(Landroid/media/Image;)Z", (void*)ImageReader_imageSetup },
+ {"nativeGetSurface", "()Landroid/view/Surface;", (void*)ImageReader_getSurface },
+};
+
+static JNINativeMethod gImageMethods[] = {
+ {"nativeImageGetBuffer", "(I)Ljava/nio/ByteBuffer;", (void*)Image_getByteBuffer },
+ {"nativeCreatePlane", "(I)Landroid/media/ImageReader$SurfaceImage$SurfacePlane;",
+ (void*)Image_createSurfacePlane },
+};
+
+int register_android_media_ImageReader(JNIEnv *env) {
+
+ int ret1 = AndroidRuntime::registerNativeMethods(env,
+ "android/media/ImageReader", gImageReaderMethods, NELEM(gImageReaderMethods));
+
+ int ret2 = AndroidRuntime::registerNativeMethods(env,
+ "android/media/ImageReader$SurfaceImage", gImageMethods, NELEM(gImageMethods));
+
+ return (ret1 || ret2);
+}
diff --git a/media/jni/android_media_MediaPlayer.cpp b/media/jni/android_media_MediaPlayer.cpp
index 0f78649..9b66c06 100644
--- a/media/jni/android_media_MediaPlayer.cpp
+++ b/media/jni/android_media_MediaPlayer.cpp
@@ -896,6 +896,7 @@
"android/media/MediaPlayer", gMethods, NELEM(gMethods));
}
+extern int register_android_media_ImageReader(JNIEnv *env);
extern int register_android_media_Crypto(JNIEnv *env);
extern int register_android_media_Drm(JNIEnv *env);
extern int register_android_media_MediaCodec(JNIEnv *env);
@@ -923,6 +924,11 @@
}
assert(env != NULL);
+ if (register_android_media_ImageReader(env) < 0) {
+ ALOGE("ERROR: ImageReader native registration failed");
+ goto bail;
+ }
+
if (register_android_media_MediaPlayer(env) < 0) {
ALOGE("ERROR: MediaPlayer native registration failed\n");
goto bail;