Add DEPTH image formats, support in ImageReader
- Add an explicit mapping between public ImageFormat/
PixelFormat enums and internal HAL format/dataspace.
- Add DEPTH16 and DEPTH_POINT_CLOUD formats
- Wire up mapping layer to ImageReader to support depth
formats
Change-Id: I8197eccef900cc91baddcfcb934ccd4d8c972eff
diff --git a/api/current.txt b/api/current.txt
index b22dd1c1..612d16d 100644
--- a/api/current.txt
+++ b/api/current.txt
@@ -10973,6 +10973,8 @@
public class ImageFormat {
ctor public ImageFormat();
method public static int getBitsPerPixel(int);
+ field public static final int DEPTH16 = 1144402265; // 0x44363159
+ field public static final int DEPTH_POINT_CLOUD = 257; // 0x101
field public static final int JPEG = 256; // 0x100
field public static final int NV16 = 16; // 0x10
field public static final int NV21 = 17; // 0x11
diff --git a/api/system-current.txt b/api/system-current.txt
index 2b8ce98..4315d1d 100644
--- a/api/system-current.txt
+++ b/api/system-current.txt
@@ -11248,6 +11248,8 @@
public class ImageFormat {
ctor public ImageFormat();
method public static int getBitsPerPixel(int);
+ field public static final int DEPTH16 = 1144402265; // 0x44363159
+ field public static final int DEPTH_POINT_CLOUD = 257; // 0x101
field public static final int JPEG = 256; // 0x100
field public static final int NV16 = 16; // 0x10
field public static final int NV21 = 17; // 0x11
diff --git a/core/jni/android_view_Surface.cpp b/core/jni/android_view_Surface.cpp
index bfa0534..39064ed 100644
--- a/core/jni/android_view_Surface.cpp
+++ b/core/jni/android_view_Surface.cpp
@@ -130,6 +130,100 @@
return surfaceObj;
}
+int android_view_Surface_mapPublicFormatToHalFormat(PublicFormat f) {
+
+ switch(f) {
+ case PublicFormat::JPEG:
+ case PublicFormat::DEPTH_POINT_CLOUD:
+ return HAL_PIXEL_FORMAT_BLOB;
+ case PublicFormat::DEPTH16:
+ return HAL_PIXEL_FORMAT_Y16;
+ case PublicFormat::RAW_SENSOR:
+ return HAL_PIXEL_FORMAT_RAW16;
+ default:
+ // Most formats map 1:1
+ return static_cast<int>(f);
+ }
+}
+
+android_dataspace android_view_Surface_mapPublicFormatToHalDataspace(
+ PublicFormat f) {
+ switch(f) {
+ case PublicFormat::JPEG:
+ return HAL_DATASPACE_JFIF;
+ case PublicFormat::DEPTH_POINT_CLOUD:
+ case PublicFormat::DEPTH16:
+ return HAL_DATASPACE_DEPTH;
+ case PublicFormat::RAW_SENSOR:
+ case PublicFormat::RAW10:
+ return HAL_DATASPACE_ARBITRARY;
+ case PublicFormat::YUV_420_888:
+ case PublicFormat::NV21:
+ case PublicFormat::YV12:
+ return HAL_DATASPACE_JFIF;
+ default:
+ // Most formats map to UNKNOWN
+ return HAL_DATASPACE_UNKNOWN;
+ }
+}
+
+PublicFormat android_view_Surface_mapHalFormatDataspaceToPublicFormat(
+ int format, android_dataspace dataSpace) {
+ switch(format) {
+ case HAL_PIXEL_FORMAT_RGBA_8888:
+ case HAL_PIXEL_FORMAT_RGBX_8888:
+ case HAL_PIXEL_FORMAT_RGB_888:
+ case HAL_PIXEL_FORMAT_RGB_565:
+ case HAL_PIXEL_FORMAT_Y8:
+ case HAL_PIXEL_FORMAT_RAW10:
+ case HAL_PIXEL_FORMAT_YCbCr_420_888:
+ case HAL_PIXEL_FORMAT_YV12:
+ // Enums overlap in both name and value
+ return static_cast<PublicFormat>(format);
+ case HAL_PIXEL_FORMAT_RAW16:
+ // Name differs, though value is the same
+ return PublicFormat::RAW_SENSOR;
+ case HAL_PIXEL_FORMAT_YCbCr_422_SP:
+ // Name differs, though the value is the same
+ return PublicFormat::NV16;
+ case HAL_PIXEL_FORMAT_YCrCb_420_SP:
+ // Name differs, though the value is the same
+ return PublicFormat::NV21;
+ case HAL_PIXEL_FORMAT_YCbCr_422_I:
+ // Name differs, though the value is the same
+ return PublicFormat::YUY2;
+ case HAL_PIXEL_FORMAT_Y16:
+ // Dataspace-dependent
+ switch (dataSpace) {
+ case HAL_DATASPACE_DEPTH:
+ return PublicFormat::DEPTH16;
+ default:
+ // Assume non-depth Y16 is just Y16.
+ return PublicFormat::Y16;
+ }
+ break;
+ case HAL_PIXEL_FORMAT_BLOB:
+ // Dataspace-dependent
+ switch (dataSpace) {
+ case HAL_DATASPACE_DEPTH:
+ return PublicFormat::DEPTH_POINT_CLOUD;
+ case HAL_DATASPACE_JFIF:
+ return PublicFormat::JPEG;
+ default:
+ // Assume otherwise-marked blobs are also JPEG
+ return PublicFormat::JPEG;
+ }
+ break;
+ case HAL_PIXEL_FORMAT_BGRA_8888:
+ case HAL_PIXEL_FORMAT_RAW_OPAQUE:
+ case HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED:
+ // Not defined in public API
+ return PublicFormat::UNKNOWN;
+
+ default:
+ return PublicFormat::UNKNOWN;
+ }
+}
// ----------------------------------------------------------------------------
static inline bool isSurfaceValid(const sp<Surface>& sur) {
diff --git a/graphics/java/android/graphics/ImageFormat.java b/graphics/java/android/graphics/ImageFormat.java
index 3efb9c0..49c4247 100644
--- a/graphics/java/android/graphics/ImageFormat.java
+++ b/graphics/java/android/graphics/ImageFormat.java
@@ -356,6 +356,38 @@
public static final int RAW10 = 0x25;
/**
+ * Android dense depth image format.
+ *
+ * Each pixel is 16 bits, representing a depth ranging measurement from
+ * a depth camera or similar sensor.
+ *
+ * <p>This format assumes
+ * <ul>
+ * <li>an even width</li>
+ * <li>an even height</li>
+ * <li>a horizontal stride multiple of 16 pixels</li>
+ * </ul>
+ * </p>
+ *
+ * <pre> y_size = stride * height </pre>
+ *
+ * When produced by a camera, the units are millimeters.
+ */
+ public static final int DEPTH16 = 0x44363159;
+
+ /**
+ * Android sparse depth point cloud format.
+ *
+ * <p>A variable-length list of 3D points, with each point represented
+ * by a triple of floats.</p>
+ *
+ * <p>The number of points is {@code (size of the buffer in bytes) / 12}.
+ *
+ * The coordinate system and units depend on the source of the point cloud data.
+ */
+ public static final int DEPTH_POINT_CLOUD = 0x101;
+
+ /**
* Use this function to retrieve the number of bits per pixel of an
* ImageFormat.
*
@@ -376,6 +408,7 @@
case Y8:
return 8;
case Y16:
+ case DEPTH16:
return 16;
case NV21:
return 12;
@@ -412,6 +445,8 @@
case YUV_420_888:
case RAW_SENSOR:
case RAW10:
+ case DEPTH16:
+ case DEPTH_POINT_CLOUD:
return true;
}
diff --git a/include/android_runtime/android_view_Surface.h b/include/android_runtime/android_view_Surface.h
index 53e8b49..a6836a8 100644
--- a/include/android_runtime/android_view_Surface.h
+++ b/include/android_runtime/android_view_Surface.h
@@ -26,6 +26,33 @@
class Surface;
class IGraphicBufferProducer;
+/**
+ * Enum mirroring the public API definitions for image and pixel formats.
+ * Some of these are hidden in the public API
+ *
+ * Keep up to date with android.graphics.ImageFormat and
+ * android.graphics.PixelFormat
+ */
+enum class PublicFormat {
+ UNKNOWN = 0x0,
+ RGBA_8888 = 0x1,
+ RGBX_8888 = 0x2,
+ RGB_888 = 0x3,
+ RGB_565 = 0x4,
+ NV16 = 0x10,
+ NV21 = 0x11,
+ YUY2 = 0x14,
+ RAW_SENSOR = 0x20,
+ YUV_420_888 = 0x23,
+ RAW10 = 0x25,
+ JPEG = 0x100,
+ DEPTH_POINT_CLOUD = 0x101,
+ YV12 = 0x32315659,
+ Y8 = 0x20203859, // @hide
+ Y16 = 0x20363159, // @hide
+ DEPTH16 = 0x44363159
+};
+
/* Gets the underlying ANativeWindow for a Surface. */
extern sp<ANativeWindow> android_view_Surface_getNativeWindow(
JNIEnv* env, jobject surfaceObj);
@@ -40,6 +67,21 @@
extern jobject android_view_Surface_createFromIGraphicBufferProducer(JNIEnv* env,
const sp<IGraphicBufferProducer>& bufferProducer);
+/* Convert from android.graphics.ImageFormat/PixelFormat enums to graphics.h HAL
+ * format */
+extern int android_view_Surface_mapPublicFormatToHalFormat(PublicFormat f);
+
+/* Convert from android.graphics.ImageFormat/PixelFormat enums to graphics.h HAL
+ * dataspace */
+extern android_dataspace android_view_Surface_mapPublicFormatToHalDataspace(
+ PublicFormat f);
+
+/* Convert from HAL format, dataspace pair to
+ * android.graphics.ImageFormat/PixelFormat.
+ * For unknown/unspecified pairs, returns PublicFormat::UNKNOWN */
+extern PublicFormat android_view_Surface_mapHalFormatDataspaceToPublicFormat(
+ int format, android_dataspace dataSpace);
+
} // namespace android
#endif // _ANDROID_VIEW_SURFACE_H
diff --git a/media/java/android/media/ImageReader.java b/media/java/android/media/ImageReader.java
index 8d6a588..824a7ad 100644
--- a/media/java/android/media/ImageReader.java
+++ b/media/java/android/media/ImageReader.java
@@ -483,6 +483,8 @@
case ImageFormat.Y16:
case ImageFormat.RAW_SENSOR:
case ImageFormat.RAW10:
+ case ImageFormat.DEPTH16:
+ case ImageFormat.DEPTH_POINT_CLOUD:
return 1;
default:
throw new UnsupportedOperationException(
diff --git a/media/jni/android_media_ImageReader.cpp b/media/jni/android_media_ImageReader.cpp
index cf69b8f..b247493 100644
--- a/media/jni/android_media_ImageReader.cpp
+++ b/media/jni/android_media_ImageReader.cpp
@@ -95,6 +95,9 @@
void setBufferFormat(int format) { mFormat = format; }
int getBufferFormat() { return mFormat; }
+ void setBufferDataspace(android_dataspace dataSpace) { mDataSpace = dataSpace; }
+ android_dataspace getBufferDataspace() { return mDataSpace; }
+
void setBufferWidth(int width) { mWidth = width; }
int getBufferWidth() { return mWidth; }
@@ -111,6 +114,7 @@
jobject mWeakThiz;
jclass mClazz;
int mFormat;
+ android_dataspace mDataSpace;
int mWidth;
int mHeight;
};
@@ -263,29 +267,6 @@
env->SetLongField(thiz, gSurfaceImageClassInfo.mLockedBuffer, reinterpret_cast<jlong>(buffer));
}
-// Some formats like JPEG defined with different values between android.graphics.ImageFormat and
-// graphics.h, need convert to the one defined in graphics.h here.
-static int Image_getPixelFormat(JNIEnv* env, int format)
-{
- int jpegFormat;
- jfieldID fid;
-
- ALOGV("%s: format = 0x%x", __FUNCTION__, format);
-
- jclass imageFormatClazz = env->FindClass("android/graphics/ImageFormat");
- ALOG_ASSERT(imageFormatClazz != NULL);
-
- fid = env->GetStaticFieldID(imageFormatClazz, "JPEG", "I");
- jpegFormat = env->GetStaticIntField(imageFormatClazz, fid);
-
- // Translate the JPEG to BLOB for camera purpose.
- if (format == jpegFormat) {
- format = HAL_PIXEL_FORMAT_BLOB;
- }
-
- return format;
-}
-
static uint32_t Image_getJpegSize(CpuConsumer::LockedBuffer* buffer, bool usingRGBAOverride)
{
ALOG_ASSERT(buffer != NULL, "Input buffer is NULL!!!");
@@ -483,7 +464,7 @@
}
static jint Image_imageGetPixelStride(JNIEnv* env, CpuConsumer::LockedBuffer* buffer, int idx,
- int32_t readerFormat)
+ int32_t halReaderFormat)
{
ALOGV("%s: buffer index: %d", __FUNCTION__, idx);
ALOG_ASSERT((idx < IMAGE_READER_MAX_NUM_PLANES) && (idx >= 0), "Index is out of range:%d", idx);
@@ -493,7 +474,7 @@
int32_t fmt = buffer->flexFormat;
- fmt = applyFormatOverrides(fmt, readerFormat);
+ fmt = applyFormatOverrides(fmt, halReaderFormat);
switch (fmt) {
case HAL_PIXEL_FORMAT_YCbCr_420_888:
@@ -543,7 +524,7 @@
}
static jint Image_imageGetRowStride(JNIEnv* env, CpuConsumer::LockedBuffer* buffer, int idx,
- int32_t readerFormat)
+ int32_t halReaderFormat)
{
ALOGV("%s: buffer index: %d", __FUNCTION__, idx);
ALOG_ASSERT((idx < IMAGE_READER_MAX_NUM_PLANES) && (idx >= 0));
@@ -553,7 +534,7 @@
int32_t fmt = buffer->flexFormat;
- fmt = applyFormatOverrides(fmt, readerFormat);
+ fmt = applyFormatOverrides(fmt, halReaderFormat);
switch (fmt) {
case HAL_PIXEL_FORMAT_YCbCr_420_888:
@@ -682,11 +663,16 @@
{
status_t res;
int nativeFormat;
+ android_dataspace nativeDataspace;
ALOGV("%s: width:%d, height: %d, format: 0x%x, maxImages:%d",
__FUNCTION__, width, height, format, maxImages);
- nativeFormat = Image_getPixelFormat(env, format);
+ PublicFormat publicFormat = static_cast<PublicFormat>(format);
+ nativeFormat = android_view_Surface_mapPublicFormatToHalFormat(
+ publicFormat);
+ nativeDataspace = android_view_Surface_mapPublicFormatToHalDataspace(
+ publicFormat);
sp<IGraphicBufferProducer> gbProducer;
sp<IGraphicBufferConsumer> gbConsumer;
@@ -710,10 +696,11 @@
consumer->setFrameAvailableListener(ctx);
ImageReader_setNativeContext(env, thiz, ctx);
ctx->setBufferFormat(nativeFormat);
+ ctx->setBufferDataspace(nativeDataspace);
ctx->setBufferWidth(width);
ctx->setBufferHeight(height);
- // Set the width/height/format to the CpuConsumer
+ // Set the width/height/format/dataspace to the CpuConsumer
res = consumer->setDefaultBufferSize(width, height);
if (res != OK) {
jniThrowException(env, "java/lang/IllegalStateException",
@@ -725,6 +712,12 @@
jniThrowException(env, "java/lang/IllegalStateException",
"Failed to set CpuConsumer buffer format");
}
+ res = consumer->setDefaultBufferDataSpace(nativeDataspace);
+ if (res != OK) {
+ jniThrowException(env, "java/lang/IllegalStateException",
+ "Failed to set CpuConsumer buffer dataSpace");
+ }
+
}
static void ImageReader_close(JNIEnv* env, jobject thiz)
@@ -884,6 +877,8 @@
static jobject Image_createSurfacePlane(JNIEnv* env, jobject thiz, int idx, int readerFormat)
{
int rowStride, pixelStride;
+ PublicFormat publicReaderFormat = static_cast<PublicFormat>(readerFormat);
+
ALOGV("%s: buffer index: %d", __FUNCTION__, idx);
CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, thiz);
@@ -893,10 +888,11 @@
jniThrowException(env, "java/lang/IllegalStateException", "Image was released");
}
- readerFormat = Image_getPixelFormat(env, readerFormat);
+ int halReaderFormat = android_view_Surface_mapPublicFormatToHalFormat(
+ publicReaderFormat);
- rowStride = Image_imageGetRowStride(env, buffer, idx, readerFormat);
- pixelStride = Image_imageGetPixelStride(env, buffer, idx, readerFormat);
+ rowStride = Image_imageGetRowStride(env, buffer, idx, halReaderFormat);
+ pixelStride = Image_imageGetPixelStride(env, buffer, idx, halReaderFormat);
jobject surfPlaneObj = env->NewObject(gSurfacePlaneClassInfo.clazz,
gSurfacePlaneClassInfo.ctor, thiz, idx, rowStride, pixelStride);
@@ -909,6 +905,7 @@
uint8_t *base = NULL;
uint32_t size = 0;
jobject byteBuffer;
+ PublicFormat readerPublicFormat = static_cast<PublicFormat>(readerFormat);
ALOGV("%s: buffer index: %d", __FUNCTION__, idx);
@@ -918,10 +915,11 @@
jniThrowException(env, "java/lang/IllegalStateException", "Image was released");
}
- readerFormat = Image_getPixelFormat(env, readerFormat);
+ int readerHalFormat = android_view_Surface_mapPublicFormatToHalFormat(
+ readerPublicFormat);
// Create byteBuffer from native buffer
- Image_getLockedBufferInfo(env, buffer, idx, &base, &size, readerFormat);
+ Image_getLockedBufferInfo(env, buffer, idx, &base, &size, readerHalFormat);
if (size > static_cast<uint32_t>(INT32_MAX)) {
// Byte buffer have 'int capacity', so check the range