Implement MediaCodec.getImage methods
Bug: 10706245
Change-Id: Icbac5538a27ffdb53d974e2e1f8dc5afe02fb391
diff --git a/media/java/android/media/MediaCodec.java b/media/java/android/media/MediaCodec.java
index f84c383..275d9b2 100644
--- a/media/java/android/media/MediaCodec.java
+++ b/media/java/android/media/MediaCodec.java
@@ -16,7 +16,10 @@
package android.media;
+import android.graphics.ImageFormat;
+import android.graphics.Rect;
import android.media.Image;
+import android.media.Image.Plane;
import android.media.MediaCodecInfo;
import android.media.MediaCodecList;
import android.media.MediaCrypto;
@@ -29,6 +32,7 @@
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.nio.ReadOnlyBufferException;
import java.util.Arrays;
import java.util.Map;
@@ -1537,4 +1541,175 @@
}
private long mNativeContext;
+
+ /** @hide */
+ public static class MediaImage extends Image {
+ private final boolean mIsReadOnly;
+ private boolean mIsValid;
+ private final int mWidth;
+ private final int mHeight;
+ private final int mFormat;
+ private long mTimestamp;
+ private final Plane[] mPlanes;
+ private final ByteBuffer mBuffer;
+ private final ByteBuffer mInfo;
+ private final int mXOffset;
+ private final int mYOffset;
+
+ private final static int TYPE_YUV = 1;
+
+ public int getFormat() {
+ checkValid();
+ return mFormat;
+ }
+
+ public int getHeight() {
+ checkValid();
+ return mHeight;
+ }
+
+ public int getWidth() {
+ checkValid();
+ return mWidth;
+ }
+
+ public long getTimestamp() {
+ checkValid();
+ return mTimestamp;
+ }
+
+ public Plane[] getPlanes() {
+ checkValid();
+ return Arrays.copyOf(mPlanes, mPlanes.length);
+ }
+
+ public void close() {
+ if (mIsValid) {
+ java.nio.NioUtils.freeDirectBuffer(mBuffer);
+ mIsValid = false;
+ }
+ }
+
+ /**
+ * Set the crop rectangle associated with this frame.
+ * <p>
+ * The crop rectangle specifies the region of valid pixels in the image,
+ * using coordinates in the largest-resolution plane.
+ */
+ public void setCropRect(Rect cropRect) {
+ if (mIsReadOnly) {
+ throw new ReadOnlyBufferException();
+ }
+ super.setCropRect(cropRect);
+ }
+
+ private void checkValid() {
+ if (!mIsValid) {
+ throw new IllegalStateException("Image is already released");
+ }
+ }
+
+ private int readInt(ByteBuffer buffer, boolean asLong) {
+ if (asLong) {
+ return (int)buffer.getLong();
+ } else {
+ return buffer.getInt();
+ }
+ }
+
+ public MediaImage(
+ ByteBuffer buffer, ByteBuffer info, boolean readOnly,
+ long timestamp, int xOffset, int yOffset, Rect cropRect) {
+ mFormat = ImageFormat.YUV_420_888;
+ mTimestamp = timestamp;
+ mIsValid = true;
+ mIsReadOnly = buffer.isReadOnly();
+ mBuffer = buffer.duplicate();
+ if (cropRect != null) {
+ cropRect.offset(-xOffset, -yOffset);
+ }
+ mCropRect = cropRect;
+
+ // save offsets and info
+ mXOffset = xOffset;
+ mYOffset = yOffset;
+ mInfo = info;
+
+ // read media-info. the size of media info can be 80 or 156 depending on
+ // whether it was created on a 32- or 64-bit process. See MediaImage
+ if (info.remaining() == 80 || info.remaining() == 156) {
+ boolean sizeIsLong = info.remaining() == 156;
+ int type = info.getInt();
+ if (type != TYPE_YUV) {
+ throw new UnsupportedOperationException("unsupported type: " + type);
+ }
+ int numPlanes = readInt(info, sizeIsLong);
+ if (numPlanes != 3) {
+ throw new RuntimeException("unexpected number of planes: " + numPlanes);
+ }
+ mWidth = readInt(info, sizeIsLong);
+ mHeight = readInt(info, sizeIsLong);
+ if (mWidth < 1 || mHeight < 1) {
+ throw new UnsupportedOperationException(
+ "unsupported size: " + mWidth + "x" + mHeight);
+ }
+ int bitDepth = readInt(info, sizeIsLong);
+ if (bitDepth != 8) {
+ throw new UnsupportedOperationException("unsupported bit depth: " + bitDepth);
+ }
+ mPlanes = new MediaPlane[numPlanes];
+ for (int ix = 0; ix < numPlanes; ix++) {
+ int planeOffset = readInt(info, sizeIsLong);
+ int colInc = readInt(info, sizeIsLong);
+ int rowInc = readInt(info, sizeIsLong);
+ int horiz = readInt(info, sizeIsLong);
+ int vert = readInt(info, sizeIsLong);
+ if (horiz != vert || horiz != (ix == 0 ? 1 : 2)) {
+ throw new UnsupportedOperationException("unexpected subsampling: "
+ + horiz + "x" + vert + " on plane " + ix);
+ }
+
+ buffer.clear();
+ buffer.position(mBuffer.position() + planeOffset
+ + (xOffset / horiz) * colInc + (yOffset / vert) * rowInc);
+ buffer.limit(buffer.position() + Utils.divUp(bitDepth, 8)
+ + (mHeight / vert - 1) * rowInc + (mWidth / horiz - 1) * colInc);
+ mPlanes[ix] = new MediaPlane(buffer.slice(), rowInc, colInc);
+ }
+ } else {
+ throw new UnsupportedOperationException(
+ "unsupported info length: " + info.remaining());
+ }
+ }
+
+ private class MediaPlane extends Plane {
+ public MediaPlane(ByteBuffer buffer, int rowInc, int colInc) {
+ mData = buffer;
+ mRowInc = rowInc;
+ mColInc = colInc;
+ }
+
+ @Override
+ public int getRowStride() {
+ checkValid();
+ return mRowInc;
+ }
+
+ @Override
+ public int getPixelStride() {
+ checkValid();
+ return mColInc;
+ }
+
+ @Override
+ public ByteBuffer getBuffer() {
+ checkValid();
+ return mData;
+ }
+
+ private final int mRowInc;
+ private final int mColInc;
+ private final ByteBuffer mData;
+ }
+ }
}
diff --git a/media/jni/android_media_MediaCodec.cpp b/media/jni/android_media_MediaCodec.cpp
index 04ff098..d033f76 100644
--- a/media/jni/android_media_MediaCodec.cpp
+++ b/media/jni/android_media_MediaCodec.cpp
@@ -90,6 +90,8 @@
mClass = (jclass)env->NewGlobalRef(clazz);
mObject = env->NewWeakGlobalRef(thiz);
+ cacheJavaObjects(env);
+
mLooper = new ALooper;
mLooper->setName("MediaCodec_looper");
@@ -105,6 +107,45 @@
}
}
+void JMediaCodec::cacheJavaObjects(JNIEnv *env) {
+ jclass clazz = (jclass)env->FindClass("java/nio/ByteBuffer");
+ mByteBufferClass = (jclass)env->NewGlobalRef(clazz);
+ CHECK(mByteBufferClass != NULL);
+
+ ScopedLocalRef<jclass> byteOrderClass(
+ env, env->FindClass("java/nio/ByteOrder"));
+ CHECK(byteOrderClass.get() != NULL);
+
+ jmethodID nativeOrderID = env->GetStaticMethodID(
+ byteOrderClass.get(), "nativeOrder", "()Ljava/nio/ByteOrder;");
+ CHECK(nativeOrderID != NULL);
+
+ jobject nativeByteOrderObj =
+ env->CallStaticObjectMethod(byteOrderClass.get(), nativeOrderID);
+ mNativeByteOrderObj = env->NewGlobalRef(nativeByteOrderObj);
+ CHECK(mNativeByteOrderObj != NULL);
+ env->DeleteLocalRef(nativeByteOrderObj);
+ nativeByteOrderObj = NULL;
+
+ mByteBufferOrderMethodID = env->GetMethodID(
+ mByteBufferClass,
+ "order",
+ "(Ljava/nio/ByteOrder;)Ljava/nio/ByteBuffer;");
+ CHECK(mByteBufferOrderMethodID != NULL);
+
+ mByteBufferAsReadOnlyBufferMethodID = env->GetMethodID(
+ mByteBufferClass, "asReadOnlyBuffer", "()Ljava/nio/ByteBuffer;");
+ CHECK(mByteBufferAsReadOnlyBufferMethodID != NULL);
+
+ mByteBufferPositionMethodID = env->GetMethodID(
+ mByteBufferClass, "position", "(I)Ljava/nio/Buffer;");
+ CHECK(mByteBufferPositionMethodID != NULL);
+
+ mByteBufferLimitMethodID = env->GetMethodID(
+ mByteBufferClass, "limit", "(I)Ljava/nio/Buffer;");
+ CHECK(mByteBufferLimitMethodID != NULL);
+}
+
status_t JMediaCodec::initCheck() const {
return mCodec != NULL ? OK : NO_INIT;
}
@@ -148,6 +189,19 @@
mObject = NULL;
env->DeleteGlobalRef(mClass);
mClass = NULL;
+ deleteJavaObjects(env);
+}
+
+void JMediaCodec::deleteJavaObjects(JNIEnv *env) {
+ env->DeleteGlobalRef(mByteBufferClass);
+ mByteBufferClass = NULL;
+ env->DeleteGlobalRef(mNativeByteOrderObj);
+ mNativeByteOrderObj = NULL;
+
+ mByteBufferOrderMethodID = NULL;
+ mByteBufferAsReadOnlyBufferMethodID = NULL;
+ mByteBufferPositionMethodID = NULL;
+ mByteBufferLimitMethodID = NULL;
}
status_t JMediaCodec::setCallback(jobject cb) {
@@ -298,80 +352,69 @@
return err;
}
- ScopedLocalRef<jclass> byteBufferClass(
- env, env->FindClass("java/nio/ByteBuffer"));
-
- CHECK(byteBufferClass.get() != NULL);
-
- jmethodID orderID = env->GetMethodID(
- byteBufferClass.get(),
- "order",
- "(Ljava/nio/ByteOrder;)Ljava/nio/ByteBuffer;");
-
- CHECK(orderID != NULL);
-
- jmethodID asReadOnlyBufferID = env->GetMethodID(
- byteBufferClass.get(), "asReadOnlyBuffer", "()Ljava/nio/ByteBuffer;");
-
- CHECK(asReadOnlyBufferID != NULL);
-
- ScopedLocalRef<jclass> byteOrderClass(
- env, env->FindClass("java/nio/ByteOrder"));
-
- CHECK(byteOrderClass.get() != NULL);
-
- jmethodID nativeOrderID = env->GetStaticMethodID(
- byteOrderClass.get(), "nativeOrder", "()Ljava/nio/ByteOrder;");
- CHECK(nativeOrderID != NULL);
-
- jobject nativeByteOrderObj =
- env->CallStaticObjectMethod(byteOrderClass.get(), nativeOrderID);
- CHECK(nativeByteOrderObj != NULL);
-
*bufArray = (jobjectArray)env->NewObjectArray(
- buffers.size(), byteBufferClass.get(), NULL);
+ buffers.size(), mByteBufferClass, NULL);
if (*bufArray == NULL) {
- env->DeleteLocalRef(nativeByteOrderObj);
return NO_MEMORY;
}
for (size_t i = 0; i < buffers.size(); ++i) {
const sp<ABuffer> &buffer = buffers.itemAt(i);
- // if this is an ABuffer that doesn't actually hold any accessible memory,
- // use a null ByteBuffer
- if (buffer->base() == NULL) {
- continue;
+ jobject byteBuffer = NULL;
+ err = createByteBufferFromABuffer(
+ env, !input /* readOnly */, true /* clearBuffer */, buffer, &byteBuffer);
+ if (err != OK) {
+ return err;
}
- jobject byteBuffer =
- env->NewDirectByteBuffer(
- buffer->base(),
- buffer->capacity());
- if (!input && byteBuffer != NULL) {
- jobject readOnlyBuffer = env->CallObjectMethod(
- byteBuffer, asReadOnlyBufferID);
+ if (byteBuffer != NULL) {
+ env->SetObjectArrayElement(
+ *bufArray, i, byteBuffer);
+
env->DeleteLocalRef(byteBuffer);
- byteBuffer = readOnlyBuffer;
+ byteBuffer = NULL;
}
- if (byteBuffer == NULL) {
- env->DeleteLocalRef(nativeByteOrderObj);
- return NO_MEMORY;
- }
- jobject me = env->CallObjectMethod(
- byteBuffer, orderID, nativeByteOrderObj);
- env->DeleteLocalRef(me);
- me = NULL;
-
- env->SetObjectArrayElement(
- *bufArray, i, byteBuffer);
-
- env->DeleteLocalRef(byteBuffer);
- byteBuffer = NULL;
}
- env->DeleteLocalRef(nativeByteOrderObj);
- nativeByteOrderObj = NULL;
+ return OK;
+}
+// static
+status_t JMediaCodec::createByteBufferFromABuffer(
+ JNIEnv *env, bool readOnly, bool clearBuffer, const sp<ABuffer> &buffer,
+ jobject *buf) const {
+ // if this is an ABuffer that doesn't actually hold any accessible memory,
+ // use a null ByteBuffer
+ *buf = NULL;
+ if (buffer->base() == NULL) {
+ return OK;
+ }
+
+ jobject byteBuffer =
+ env->NewDirectByteBuffer(buffer->base(), buffer->capacity());
+ if (readOnly && byteBuffer != NULL) {
+ jobject readOnlyBuffer = env->CallObjectMethod(
+ byteBuffer, mByteBufferAsReadOnlyBufferMethodID);
+ env->DeleteLocalRef(byteBuffer);
+ byteBuffer = readOnlyBuffer;
+ }
+ if (byteBuffer == NULL) {
+ return NO_MEMORY;
+ }
+ jobject me = env->CallObjectMethod(
+ byteBuffer, mByteBufferOrderMethodID, mNativeByteOrderObj);
+ env->DeleteLocalRef(me);
+ me = env->CallObjectMethod(
+ byteBuffer, mByteBufferLimitMethodID,
+ clearBuffer ? buffer->capacity() : (buffer->offset() + buffer->size()));
+ env->DeleteLocalRef(me);
+ me = env->CallObjectMethod(
+ byteBuffer, mByteBufferPositionMethodID,
+ clearBuffer ? 0 : buffer->offset());
+ env->DeleteLocalRef(me);
+ me = NULL;
+
+ *buf = byteBuffer;
return OK;
}
@@ -388,85 +431,8 @@
return err;
}
- ScopedLocalRef<jclass> byteBufferClass(
- env, env->FindClass("java/nio/ByteBuffer"));
-
- CHECK(byteBufferClass.get() != NULL);
-
- jmethodID orderID = env->GetMethodID(
- byteBufferClass.get(),
- "order",
- "(Ljava/nio/ByteOrder;)Ljava/nio/ByteBuffer;");
-
- CHECK(orderID != NULL);
-
- jmethodID asReadOnlyBufferID = env->GetMethodID(
- byteBufferClass.get(), "asReadOnlyBuffer", "()Ljava/nio/ByteBuffer;");
-
- CHECK(asReadOnlyBufferID != NULL);
-
- jmethodID positionID = env->GetMethodID(
- byteBufferClass.get(), "position", "(I)Ljava/nio/Buffer;");
-
- CHECK(positionID != NULL);
-
- jmethodID limitID = env->GetMethodID(
- byteBufferClass.get(), "limit", "(I)Ljava/nio/Buffer;");
-
- CHECK(limitID != NULL);
-
- ScopedLocalRef<jclass> byteOrderClass(
- env, env->FindClass("java/nio/ByteOrder"));
-
- CHECK(byteOrderClass.get() != NULL);
-
- jmethodID nativeOrderID = env->GetStaticMethodID(
- byteOrderClass.get(), "nativeOrder", "()Ljava/nio/ByteOrder;");
- CHECK(nativeOrderID != NULL);
-
- jobject nativeByteOrderObj =
- env->CallStaticObjectMethod(byteOrderClass.get(), nativeOrderID);
- CHECK(nativeByteOrderObj != NULL);
-
- // if this is an ABuffer that doesn't actually hold any accessible memory,
- // use a null ByteBuffer
- if (buffer->base() == NULL) {
- *buf = NULL;
- return OK;
- }
-
- jobject byteBuffer =
- env->NewDirectByteBuffer(
- buffer->base(),
- buffer->capacity());
- if (!input && byteBuffer != NULL) {
- jobject readOnlyBuffer = env->CallObjectMethod(
- byteBuffer, asReadOnlyBufferID);
- env->DeleteLocalRef(byteBuffer);
- byteBuffer = readOnlyBuffer;
- }
- if (byteBuffer == NULL) {
- env->DeleteLocalRef(nativeByteOrderObj);
- return NO_MEMORY;
- }
- jobject me = env->CallObjectMethod(
- byteBuffer, orderID, nativeByteOrderObj);
- env->DeleteLocalRef(me);
- me = env->CallObjectMethod(
- byteBuffer, limitID,
- input ? buffer->capacity() : (buffer->offset() + buffer->size()));
- env->DeleteLocalRef(me);
- me = env->CallObjectMethod(
- byteBuffer, positionID,
- input ? 0 : buffer->offset());
- env->DeleteLocalRef(me);
- me = NULL;
-
- env->DeleteLocalRef(nativeByteOrderObj);
- nativeByteOrderObj = NULL;
-
- *buf = byteBuffer;
- return OK;
+ return createByteBufferFromABuffer(
+ env, !input /* readOnly */, input /* clearBuffer */, buffer, buf);
}
status_t JMediaCodec::getImage(
@@ -490,15 +456,80 @@
}
// check if buffer is an image
- AString imageData;
- if (!buffer->meta()->findString("image-data", &imageData)) {
+ sp<ABuffer> imageData;
+ if (!buffer->meta()->findBuffer("image-data", &imageData)) {
return OK;
}
+ int64_t timestamp = 0;
+ if (!input && buffer->meta()->findInt64("timeUs", ×tamp)) {
+ timestamp *= 1000; // adjust to ns
+ }
+
+ jobject byteBuffer = NULL;
+ err = createByteBufferFromABuffer(
+ env, !input /* readOnly */, input /* clearBuffer */, buffer, &byteBuffer);
+ if (err != OK) {
+ return OK;
+ }
+
+ jobject infoBuffer = NULL;
+ err = createByteBufferFromABuffer(
+ env, true /* readOnly */, true /* clearBuffer */, imageData, &infoBuffer);
+ if (err != OK) {
+ env->DeleteLocalRef(byteBuffer);
+ byteBuffer = NULL;
+ return OK;
+ }
+
+ jobject cropRect = NULL;
+ int32_t left, top, right, bottom;
+ if (buffer->meta()->findRect("crop-rect", &left, &top, &right, &bottom)) {
+ ScopedLocalRef<jclass> rectClazz(
+ env, env->FindClass("android/graphics/Rect"));
+ CHECK(rectClazz.get() != NULL);
+
+ jmethodID rectConstructID = env->GetMethodID(
+ rectClazz.get(), "<init>", "(IIII)V");
+
+ cropRect = env->NewObject(
+ rectClazz.get(), rectConstructID, left, top, right + 1, bottom + 1);
+ }
+
+ ScopedLocalRef<jclass> imageClazz(
+ env, env->FindClass("android/media/MediaCodec$MediaImage"));
+ CHECK(imageClazz.get() != NULL);
+
+ jmethodID imageConstructID = env->GetMethodID(imageClazz.get(), "<init>",
+ "(Ljava/nio/ByteBuffer;Ljava/nio/ByteBuffer;ZJIILandroid/graphics/Rect;)V");
+
+ *buf = env->NewObject(imageClazz.get(), imageConstructID,
+ byteBuffer, infoBuffer,
+ (jboolean)!input /* readOnly */,
+ (jlong)timestamp,
+ (jint)0 /* xOffset */, (jint)0 /* yOffset */, cropRect);
+
+ // if MediaImage creation fails, return null
+ if (env->ExceptionCheck()) {
+ env->ExceptionDescribe();
+ env->ExceptionClear();
+ *buf = NULL;
+ }
+
+ if (cropRect != NULL) {
+ env->DeleteLocalRef(cropRect);
+ cropRect = NULL;
+ }
+
+ env->DeleteLocalRef(byteBuffer);
+ byteBuffer = NULL;
+
+ env->DeleteLocalRef(infoBuffer);
+ infoBuffer = NULL;
+
return OK;
}
-
status_t JMediaCodec::getName(JNIEnv *env, jstring *nameStr) const {
AString name;
diff --git a/media/jni/android_media_MediaCodec.h b/media/jni/android_media_MediaCodec.h
index dbccb0f..f84a16a 100644
--- a/media/jni/android_media_MediaCodec.h
+++ b/media/jni/android_media_MediaCodec.h
@@ -26,6 +26,7 @@
namespace android {
+struct ABuffer;
struct ALooper;
struct AMessage;
struct AString;
@@ -121,11 +122,26 @@
jweak mObject;
sp<Surface> mSurfaceTextureClient;
+ // java objects cached
+ jclass mByteBufferClass;
+ jobject mNativeByteOrderObj;
+ jmethodID mByteBufferOrderMethodID;
+ jmethodID mByteBufferPositionMethodID;
+ jmethodID mByteBufferLimitMethodID;
+ jmethodID mByteBufferAsReadOnlyBufferMethodID;
+
sp<ALooper> mLooper;
sp<MediaCodec> mCodec;
sp<AMessage> mCallbackNotification;
+ status_t createByteBufferFromABuffer(
+ JNIEnv *env, bool readOnly, bool clearBuffer, const sp<ABuffer> &buffer,
+ jobject *buf) const;
+
+ void cacheJavaObjects(JNIEnv *env);
+ void deleteJavaObjects(JNIEnv *env);
+
DISALLOW_EVIL_CONSTRUCTORS(JMediaCodec);
};