Add support for timestamps into SurfaceTexture.
API addition: The timestamps are represented as nanoseconds from some
arbitrary time point. Like the SurfaceTexture transform matrix, the
timestamp retrieved by getTimestamp is for the last frame sent to the
GL texture using updateTexImage().
Camera HAL change: Expect vendors to set these timestamps using
native_window_set_buffers_timestamp(). For now, they are
autogenerated by SurfaceTextureClient if set_buffers_timestamp() is
never called, but such timing is likely not accurate enough to pass a
CTS test.
bug:3300707
Change-Id: Ife131a0c2a826ac27342e11b8a6c42ff49e1bea7
diff --git a/core/java/android/hardware/Camera.java b/core/java/android/hardware/Camera.java
index 97f0e1b..ed2b205 100644
--- a/core/java/android/hardware/Camera.java
+++ b/core/java/android/hardware/Camera.java
@@ -410,8 +410,9 @@
/**
* Starts capturing and drawing preview frames to the screen.
- * Preview will not actually start until a surface is supplied with
- * {@link #setPreviewDisplay(SurfaceHolder)}.
+ * Preview will not actually start until a surface is supplied
+ * with {@link #setPreviewDisplay(SurfaceHolder)} or
+ * {@link #setPreviewTexture(SurfaceTexture)}.
*
* <p>If {@link #setPreviewCallback(Camera.PreviewCallback)},
* {@link #setOneShotPreviewCallback(Camera.PreviewCallback)}, or
diff --git a/core/jni/android/graphics/SurfaceTexture.cpp b/core/jni/android/graphics/SurfaceTexture.cpp
index c4e5878..2f70190 100644
--- a/core/jni/android/graphics/SurfaceTexture.cpp
+++ b/core/jni/android/graphics/SurfaceTexture.cpp
@@ -171,6 +171,12 @@
env->ReleaseFloatArrayElements(jmtx, mtx, 0);
}
+static jlong SurfaceTexture_getTimestamp(JNIEnv* env, jobject thiz)
+{
+ sp<SurfaceTexture> surfaceTexture(SurfaceTexture_getSurfaceTexture(env, thiz));
+ return surfaceTexture->getTimestamp();
+}
+
// ----------------------------------------------------------------------------
const char* const kSurfaceTextureClassPathName = "android/graphics/SurfaceTexture";
@@ -178,9 +184,10 @@
static JNINativeMethod gSurfaceTextureMethods[] = {
{"nativeClassInit", "()V", (void*)SurfaceTexture_classInit },
{"nativeInit", "(ILjava/lang/Object;)V", (void*)SurfaceTexture_init },
- {"nativeFinalize", "()V", (void*)SurfaceTexture_finalize },
+ {"nativeFinalize", "()V", (void*)SurfaceTexture_finalize },
{"nativeUpdateTexImage", "()V", (void*)SurfaceTexture_updateTexImage },
{"nativeGetTransformMatrix", "([F)V", (void*)SurfaceTexture_getTransformMatrix },
+ {"nativeGetTimestamp", "()J", (void*)SurfaceTexture_getTimestamp }
};
int register_android_graphics_SurfaceTexture(JNIEnv* env)
diff --git a/graphics/java/android/graphics/SurfaceTexture.java b/graphics/java/android/graphics/SurfaceTexture.java
index 970b207..b8327a8 100644
--- a/graphics/java/android/graphics/SurfaceTexture.java
+++ b/graphics/java/android/graphics/SurfaceTexture.java
@@ -144,6 +144,21 @@
nativeGetTransformMatrix(mtx);
}
+ /**
+ * Retrieve the timestamp associated with the texture image set by the most recent call to
+ * updateTexImage.
+ *
+ * This timestamp is in nanoseconds, and is guaranteed to be monotonically increasing. The
+ * specific meaning and zero point of the timestamp depends on the source providing images to
+ * the SurfaceTexture. Unless otherwise specified by the image source, timestamps cannot
+ * generally be compared across SurfaceTexture instances, or across multiple program
+ * invocations. It is mostly useful for determining time offsets between subsequent frames.
+ * @hide
+ */
+ public long getTimestamp() {
+ return nativeGetTimestamp();
+ }
+
protected void finalize() throws Throwable {
try {
nativeFinalize();
@@ -182,6 +197,7 @@
private native void nativeInit(int texName, Object weakSelf);
private native void nativeFinalize();
private native void nativeGetTransformMatrix(float[] mtx);
+ private native long nativeGetTimestamp();
private native void nativeUpdateTexImage();
/*
diff --git a/include/gui/ISurfaceTexture.h b/include/gui/ISurfaceTexture.h
index 168310c..6ed3c6f 100644
--- a/include/gui/ISurfaceTexture.h
+++ b/include/gui/ISurfaceTexture.h
@@ -62,8 +62,11 @@
// contents of the buffer associated with slot and transfers ownership of
// that slot back to the server. It is not valid to call queueBuffer on a
// slot that is not owned by the client or one for which a buffer associated
- // via requestBuffer.
- virtual status_t queueBuffer(int slot) = 0;
+ // via requestBuffer. In addition, a timestamp must be provided by the
+ // client for this buffer. The timestamp is measured in nanoseconds, and
+ // must be monotonically increasing. Its other properties (zero point, etc)
+ // are client-dependent, and should be documented by the client.
+ virtual status_t queueBuffer(int slot, int64_t timestamp) = 0;
// cancelBuffer indicates that the client does not wish to fill in the
// buffer associated with slot and transfers ownership of the slot back to
diff --git a/include/gui/SurfaceTexture.h b/include/gui/SurfaceTexture.h
index 9bf38f7..afa64d3 100644
--- a/include/gui/SurfaceTexture.h
+++ b/include/gui/SurfaceTexture.h
@@ -66,7 +66,12 @@
// unmodified.
virtual status_t dequeueBuffer(int *buf);
- virtual status_t queueBuffer(int buf);
+ // queueBuffer returns a filled buffer to the SurfaceTexture. In addition, a
+ // timestamp must be provided for the buffer. The timestamp is in
+ // nanoseconds, and must be monotonically increasing. Its other semantics
+ // (zero point, etc) are client-dependent and should be documented by the
+ // client.
+ virtual status_t queueBuffer(int buf, int64_t timestamp);
virtual void cancelBuffer(int buf);
virtual status_t setCrop(const Rect& reg);
virtual status_t setTransform(uint32_t transform);
@@ -98,6 +103,14 @@
// functions.
void getTransformMatrix(float mtx[16]);
+ // getTimestamp retrieves the timestamp associated with the texture image
+ // set by the most recent call to updateTexImage.
+ //
+ // The timestamp is in nanoseconds, and is monotonically increasing. Its
+ // other semantics (zero point, etc) are source-dependent and should be
+ // documented by the source.
+ int64_t getTimestamp();
+
// setFrameAvailableListener sets the listener object that will be notified
// when a new frame becomes available.
void setFrameAvailableListener(const sp<FrameAvailableListener>& l);
@@ -172,6 +185,10 @@
// gets set to mLastQueuedTransform each time updateTexImage is called.
uint32_t mCurrentTransform;
+ // mCurrentTimestamp is the timestamp for the current texture. It
+ // gets set to mLastQueuedTimestamp each time updateTexImage is called.
+ int64_t mCurrentTimestamp;
+
// mLastQueued is the buffer slot index of the most recently enqueued buffer.
// At construction time it is initialized to INVALID_BUFFER_SLOT, and is
// updated each time queueBuffer is called.
@@ -187,6 +204,10 @@
// queueBuffer gets called.
uint32_t mLastQueuedTransform;
+ // mLastQueuedTimestamp is the timestamp for the buffer that was most
+ // recently queued. This gets set by queueBuffer.
+ int64_t mLastQueuedTimestamp;
+
// mNextCrop is the crop rectangle that will be used for the next buffer
// that gets queued. It is set by calling setCrop.
Rect mNextCrop;
diff --git a/include/gui/SurfaceTextureClient.h b/include/gui/SurfaceTextureClient.h
index 7992105..df82bf2 100644
--- a/include/gui/SurfaceTextureClient.h
+++ b/include/gui/SurfaceTextureClient.h
@@ -63,6 +63,7 @@
int dispatchSetBufferCount(va_list args);
int dispatchSetBuffersGeometry(va_list args);
int dispatchSetBuffersTransform(va_list args);
+ int dispatchSetBuffersTimestamp(va_list args);
int dispatchSetCrop(va_list args);
int dispatchSetUsage(va_list args);
@@ -71,6 +72,7 @@
int setBufferCount(int bufferCount);
int setBuffersGeometry(int w, int h, int format);
int setBuffersTransform(int transform);
+ int setBuffersTimestamp(int64_t timestamp);
int setCrop(Rect const* rect);
int setUsage(uint32_t reqUsage);
@@ -114,6 +116,11 @@
// at the next deuque operation. It is initialized to 0.
uint32_t mReqUsage;
+ // mTimestamp is the timestamp that will be used for the next buffer queue
+ // operation. It defaults to NATIVE_WINDOW_TIMESTAMP_AUTO, which means that
+ // a timestamp is auto-generated when queueBuffer is called.
+ int64_t mTimestamp;
+
// mMutex is the mutex used to prevent concurrent access to the member
// variables of SurfaceTexture objects. It must be locked whenever the
// member variables are accessed.
diff --git a/include/surfaceflinger/Surface.h b/include/surfaceflinger/Surface.h
index 9e0b5bb..a59d9e5 100644
--- a/include/surfaceflinger/Surface.h
+++ b/include/surfaceflinger/Surface.h
@@ -226,7 +226,8 @@
int dispatch_set_buffer_count(va_list args);
int dispatch_set_buffers_geometry(va_list args);
int dispatch_set_buffers_transform(va_list args);
-
+ int dispatch_set_buffers_timestamp(va_list args);
+
void setUsage(uint32_t reqUsage);
int connect(int api);
int disconnect(int api);
@@ -234,6 +235,7 @@
int setBufferCount(int bufferCount);
int setBuffersGeometry(int w, int h, int format);
int setBuffersTransform(int transform);
+ int setBuffersTimestamp(int64_t timestamp);
/*
* private stuff...
diff --git a/include/ui/egl/android_natives.h b/include/ui/egl/android_natives.h
index 0fc1ddf..0a6e4fb 100644
--- a/include/ui/egl/android_natives.h
+++ b/include/ui/egl/android_natives.h
@@ -57,7 +57,7 @@
{
/* a magic value defined by the actual EGL native type */
int magic;
-
+
/* the sizeof() of the actual EGL native type */
int version;
@@ -129,6 +129,7 @@
NATIVE_WINDOW_SET_BUFFER_COUNT,
NATIVE_WINDOW_SET_BUFFERS_GEOMETRY,
NATIVE_WINDOW_SET_BUFFERS_TRANSFORM,
+ NATIVE_WINDOW_SET_BUFFERS_TIMESTAMP,
};
/* parameter for NATIVE_WINDOW_[DIS]CONNECT */
@@ -157,7 +158,15 @@
NATIVE_WINDOW_SURFACE_TEXTURE_CLIENT, // SurfaceTextureClient
};
-struct ANativeWindow
+/* parameter for NATIVE_WINDOW_SET_BUFFERS_TIMESTAMP
+ *
+ * Special timestamp value to indicate that timestamps should be auto-generated
+ * by the native window when queueBuffer is called. This is equal to INT64_MIN,
+ * defined directly to avoid problems with C99/C++ inclusion of stdint.h.
+ */
+const int64_t NATIVE_WINDOW_TIMESTAMP_AUTO = (-9223372036854775807LL-1);
+
+struct ANativeWindow
{
#ifdef __cplusplus
ANativeWindow()
@@ -262,7 +271,8 @@
* NATIVE_WINDOW_SET_BUFFER_COUNT
* NATIVE_WINDOW_SET_BUFFERS_GEOMETRY
* NATIVE_WINDOW_SET_BUFFERS_TRANSFORM
- *
+ * NATIVE_WINDOW_SET_BUFFERS_TIMESTAMP
+ *
*/
int (*perform)(struct ANativeWindow* window,
@@ -389,6 +399,22 @@
transform);
}
+/*
+ * native_window_set_buffers_timestamp(..., int64_t timestamp)
+ * All buffers queued after this call will be associated with the timestamp
+ * parameter specified. If the timestamp is set to NATIVE_WINDOW_TIMESTAMP_AUTO
+ * (the default), timestamps will be generated automatically when queueBuffer is
+ * called. The timestamp is measured in nanoseconds, and must be monotonically
+ * increasing.
+ */
+static inline int native_window_set_buffers_timestamp(
+ ANativeWindow* window,
+ int64_t timestamp)
+{
+ return window->perform(window, NATIVE_WINDOW_SET_BUFFERS_TIMESTAMP,
+ timestamp);
+}
+
// ---------------------------------------------------------------------------
/* FIXME: this is legacy for pixmaps */
diff --git a/libs/gui/ISurfaceTexture.cpp b/libs/gui/ISurfaceTexture.cpp
index d661fd5..bc14ad5 100644
--- a/libs/gui/ISurfaceTexture.cpp
+++ b/libs/gui/ISurfaceTexture.cpp
@@ -88,10 +88,11 @@
return result;
}
- virtual status_t queueBuffer(int buf) {
+ virtual status_t queueBuffer(int buf, int64_t timestamp) {
Parcel data, reply;
data.writeInterfaceToken(ISurfaceTexture::getInterfaceDescriptor());
data.writeInt32(buf);
+ data.writeInt64(timestamp);
remote()->transact(QUEUE_BUFFER, data, &reply);
status_t result = reply.readInt32();
return result;
@@ -174,7 +175,8 @@
case QUEUE_BUFFER: {
CHECK_INTERFACE(ISurfaceTexture, data, reply);
int buf = data.readInt32();
- status_t result = queueBuffer(buf);
+ int64_t timestamp = data.readInt64();
+ status_t result = queueBuffer(buf, timestamp);
reply->writeInt32(result);
return NO_ERROR;
} break;
@@ -196,7 +198,6 @@
return NO_ERROR;
} break;
case SET_TRANSFORM: {
- Rect reg;
CHECK_INTERFACE(ISurfaceTexture, data, reply);
uint32_t transform = data.readInt32();
status_t result = setTransform(transform);
diff --git a/libs/gui/SurfaceTexture.cpp b/libs/gui/SurfaceTexture.cpp
index 5c6d71b..cdaca47 100644
--- a/libs/gui/SurfaceTexture.cpp
+++ b/libs/gui/SurfaceTexture.cpp
@@ -76,9 +76,15 @@
static void mtxMul(float out[16], const float a[16], const float b[16]);
SurfaceTexture::SurfaceTexture(GLuint tex) :
- mBufferCount(MIN_BUFFER_SLOTS), mCurrentTexture(INVALID_BUFFER_SLOT),
- mCurrentTransform(0), mLastQueued(INVALID_BUFFER_SLOT),
- mLastQueuedTransform(0), mNextTransform(0), mTexName(tex) {
+ mBufferCount(MIN_BUFFER_SLOTS),
+ mCurrentTexture(INVALID_BUFFER_SLOT),
+ mCurrentTransform(0),
+ mCurrentTimestamp(0),
+ mLastQueued(INVALID_BUFFER_SLOT),
+ mLastQueuedTransform(0),
+ mLastQueuedTimestamp(0),
+ mNextTransform(0),
+ mTexName(tex) {
LOGV("SurfaceTexture::SurfaceTexture");
for (int i = 0; i < NUM_BUFFER_SLOTS; i++) {
mSlots[i].mEglImage = EGL_NO_IMAGE_KHR;
@@ -153,7 +159,7 @@
return OK;
}
-status_t SurfaceTexture::queueBuffer(int buf) {
+status_t SurfaceTexture::queueBuffer(int buf, int64_t timestamp) {
LOGV("SurfaceTexture::queueBuffer");
Mutex::Autolock lock(mMutex);
if (buf < 0 || mBufferCount <= buf) {
@@ -172,6 +178,7 @@
mLastQueued = buf;
mLastQueuedCrop = mNextCrop;
mLastQueuedTransform = mNextTransform;
+ mLastQueuedTimestamp = timestamp;
if (mFrameAvailableListener != 0) {
mFrameAvailableListener->onFrameAvailable();
}
@@ -246,12 +253,13 @@
mCurrentTextureBuf = mSlots[mCurrentTexture].mGraphicBuffer;
mCurrentCrop = mLastQueuedCrop;
mCurrentTransform = mLastQueuedTransform;
+ mCurrentTimestamp = mLastQueuedTimestamp;
}
return OK;
}
void SurfaceTexture::getTransformMatrix(float mtx[16]) {
- LOGV("SurfaceTexture::updateTexImage");
+ LOGV("SurfaceTexture::getTransformMatrix");
Mutex::Autolock lock(mMutex);
float xform[16];
@@ -342,6 +350,12 @@
mtxMul(mtx, mtxFlipV, mtxBeforeFlipV);
}
+nsecs_t SurfaceTexture::getTimestamp() {
+ LOGV("SurfaceTexture::getTimestamp");
+ Mutex::Autolock lock(mMutex);
+ return mCurrentTimestamp;
+}
+
void SurfaceTexture::setFrameAvailableListener(
const sp<FrameAvailableListener>& l) {
LOGV("SurfaceTexture::setFrameAvailableListener");
diff --git a/libs/gui/SurfaceTextureClient.cpp b/libs/gui/SurfaceTextureClient.cpp
index 7f1d9cb..a4812d0 100644
--- a/libs/gui/SurfaceTextureClient.cpp
+++ b/libs/gui/SurfaceTextureClient.cpp
@@ -26,7 +26,8 @@
SurfaceTextureClient::SurfaceTextureClient(
const sp<ISurfaceTexture>& surfaceTexture):
mSurfaceTexture(surfaceTexture), mAllocator(0), mReqWidth(1),
- mReqHeight(1), mReqFormat(DEFAULT_FORMAT), mReqUsage(0), mMutex() {
+ mReqHeight(1), mReqFormat(DEFAULT_FORMAT), mReqUsage(0),
+ mTimestamp(NATIVE_WINDOW_TIMESTAMP_AUTO), mMutex() {
// Initialize the ANativeWindow function pointers.
ANativeWindow::setSwapInterval = setSwapInterval;
ANativeWindow::dequeueBuffer = dequeueBuffer;
@@ -135,9 +136,17 @@
int SurfaceTextureClient::queueBuffer(android_native_buffer_t* buffer) {
LOGV("SurfaceTextureClient::queueBuffer");
Mutex::Autolock lock(mMutex);
+ int64_t timestamp;
+ if (mTimestamp == NATIVE_WINDOW_TIMESTAMP_AUTO) {
+ timestamp = systemTime(SYSTEM_TIME_MONOTONIC);
+ LOGV("SurfaceTextureClient::queueBuffer making up timestamp: %.2f ms",
+ timestamp / 1000000.f);
+ } else {
+ timestamp = mTimestamp;
+ }
for (int i = 0; i < NUM_BUFFER_SLOTS; i++) {
if (mSlots[i]->handle == buffer->handle) {
- return mSurfaceTexture->queueBuffer(i);
+ return mSurfaceTexture->queueBuffer(i, timestamp);
}
}
LOGE("queueBuffer: unknown buffer queued");
@@ -196,6 +205,9 @@
case NATIVE_WINDOW_SET_BUFFERS_TRANSFORM:
res = dispatchSetBuffersTransform(args);
break;
+ case NATIVE_WINDOW_SET_BUFFERS_TIMESTAMP:
+ res = dispatchSetBuffersTimestamp(args);
+ break;
default:
res = NAME_NOT_FOUND;
break;
@@ -240,6 +252,11 @@
return setBuffersTransform(transform);
}
+int SurfaceTextureClient::dispatchSetBuffersTimestamp(va_list args) {
+ int64_t timestamp = va_arg(args, int64_t);
+ return setBuffersTimestamp(timestamp);
+}
+
int SurfaceTextureClient::connect(int api) {
LOGV("SurfaceTextureClient::connect");
// XXX: Implement this!
@@ -323,6 +340,14 @@
return err;
}
+int SurfaceTextureClient::setBuffersTimestamp(int64_t timestamp)
+{
+ LOGV("SurfaceTextureClient::setBuffersTimestamp");
+ Mutex::Autolock lock(mMutex);
+ mTimestamp = timestamp;
+ return NO_ERROR;
+}
+
void SurfaceTextureClient::freeAllBuffers() {
for (int i = 0; i < NUM_BUFFER_SLOTS; i++) {
mSlots[i] = 0;
diff --git a/libs/surfaceflinger_client/Surface.cpp b/libs/surfaceflinger_client/Surface.cpp
index 21d509a..0dfbf01 100644
--- a/libs/surfaceflinger_client/Surface.cpp
+++ b/libs/surfaceflinger_client/Surface.cpp
@@ -753,6 +753,9 @@
case NATIVE_WINDOW_SET_BUFFERS_TRANSFORM:
res = dispatch_set_buffers_transform( args );
break;
+ case NATIVE_WINDOW_SET_BUFFERS_TIMESTAMP:
+ res = dispatch_set_buffers_timestamp( args );
+ break;
default:
res = NAME_NOT_FOUND;
break;
@@ -792,6 +795,11 @@
return setBuffersTransform(transform);
}
+int Surface::dispatch_set_buffers_timestamp(va_list args) {
+ int64_t timestamp = va_arg(args, int64_t);
+ return setBuffersTimestamp(timestamp);
+}
+
void Surface::setUsage(uint32_t reqUsage)
{
Mutex::Autolock _l(mSurfaceLock);
@@ -910,6 +918,13 @@
return NO_ERROR;
}
+int Surface::setBuffersTimestamp(int64_t timestamp)
+{
+ // Surface doesn't really have anything meaningful to do with timestamps
+ // so they'll just be dropped here.
+ return NO_ERROR;
+}
+
// ----------------------------------------------------------------------------
int Surface::getConnectedApi() const
diff --git a/media/java/android/media/MediaPlayer.java b/media/java/android/media/MediaPlayer.java
index 8b29ea84..fcbe59e 100644
--- a/media/java/android/media/MediaPlayer.java
+++ b/media/java/android/media/MediaPlayer.java
@@ -623,6 +623,11 @@
* being played. Note that if a SurfaceTexture is used, the value
* set via setScreenOnWhilePlaying has no effect.
*
+ * The timestamps provided by {@link SurfaceTexture#getTimestamp()} for a
+ * SurfaceTexture set as the video sink have an unspecified zero point,
+ * and cannot be directly compared between different media sources or different
+ * instances of the same media source, or across multiple runs of the same
+ * program.
* @hide
*/
public void setTexture(SurfaceTexture st) {