Merge "Optionally repeat the previously submitted frame to the encoder" into klp-dev
diff --git a/include/media/IOMX.h b/include/media/IOMX.h
index 6d116f0..db9093a 100644
--- a/include/media/IOMX.h
+++ b/include/media/IOMX.h
@@ -137,6 +137,7 @@
enum InternalOptionType {
INTERNAL_OPTION_SUSPEND, // data is a bool
+ INTERNAL_OPTION_REPEAT_PREVIOUS_FRAME_DELAY, // data is an int64_t
};
virtual status_t setInternalOption(
node_id node,
diff --git a/include/media/stagefright/ACodec.h b/include/media/stagefright/ACodec.h
index 6bf83dd..41542ec 100644
--- a/include/media/stagefright/ACodec.h
+++ b/include/media/stagefright/ACodec.h
@@ -199,6 +199,8 @@
bool mStoreMetaDataInOutputBuffers;
int32_t mMetaDataBuffersToSubmit;
+ int64_t mRepeatFrameDelayUs;
+
status_t setCyclicIntraMacroblockRefresh(const sp<AMessage> &msg, int32_t mode);
status_t allocateBuffersOnPort(OMX_U32 portIndex);
status_t freeBuffersOnPort(OMX_U32 portIndex);
diff --git a/media/libstagefright/ACodec.cpp b/media/libstagefright/ACodec.cpp
index 36549d1..7b0bce0 100644
--- a/media/libstagefright/ACodec.cpp
+++ b/media/libstagefright/ACodec.cpp
@@ -369,7 +369,8 @@
mChannelMask(0),
mDequeueCounter(0),
mStoreMetaDataInOutputBuffers(false),
- mMetaDataBuffersToSubmit(0) {
+ mMetaDataBuffersToSubmit(0),
+ mRepeatFrameDelayUs(-1ll) {
mUninitializedState = new UninitializedState(this);
mLoadedState = new LoadedState(this);
mLoadedToIdleState = new LoadedToIdleState(this);
@@ -1089,6 +1090,12 @@
} else {
mUseMetadataOnEncoderOutput = enable;
}
+
+ if (!msg->findInt64(
+ "repeat-previous-frame-after",
+ &mRepeatFrameDelayUs)) {
+ mRepeatFrameDelayUs = -1ll;
+ }
}
// Always try to enable dynamic output buffers on native surface
@@ -3611,6 +3618,7 @@
mCodec->mDequeueCounter = 0;
mCodec->mMetaDataBuffersToSubmit = 0;
+ mCodec->mRepeatFrameDelayUs = -1ll;
if (mCodec->mShutdownInProgress) {
bool keepComponentAllocated = mCodec->mKeepComponentAllocated;
@@ -3742,6 +3750,23 @@
err = mCodec->mOMX->createInputSurface(mCodec->mNode, kPortIndexInput,
&bufferProducer);
+
+ if (err == OK && mCodec->mRepeatFrameDelayUs > 0ll) {
+ err = mCodec->mOMX->setInternalOption(
+ mCodec->mNode,
+ kPortIndexInput,
+ IOMX::INTERNAL_OPTION_REPEAT_PREVIOUS_FRAME_DELAY,
+ &mCodec->mRepeatFrameDelayUs,
+ sizeof(mCodec->mRepeatFrameDelayUs));
+
+ if (err != OK) {
+ ALOGE("[%s] Unable to configure option to repeat previous "
+ "frames (err %d)",
+ mCodec->mComponentName.c_str(),
+ err);
+ }
+ }
+
if (err == OK) {
notify->setObject("input-surface",
new BufferProducerWrapper(bufferProducer));
diff --git a/media/libstagefright/omx/GraphicBufferSource.cpp b/media/libstagefright/omx/GraphicBufferSource.cpp
index 325ffcf..cf43e94 100644
--- a/media/libstagefright/omx/GraphicBufferSource.cpp
+++ b/media/libstagefright/omx/GraphicBufferSource.cpp
@@ -22,6 +22,7 @@
#include <OMX_Core.h>
#include <media/stagefright/foundation/ADebug.h>
+#include <media/stagefright/foundation/AMessage.h>
#include <media/hardware/MetadataBufferType.h>
#include <ui/GraphicBuffer.h>
@@ -39,7 +40,13 @@
mSuspended(false),
mNumFramesAvailable(0),
mEndOfStream(false),
- mEndOfStreamSent(false) {
+ mEndOfStreamSent(false),
+ mRepeatAfterUs(-1ll),
+ mRepeatLastFrameGeneration(0),
+ mLatestSubmittedBufferId(-1),
+ mLatestSubmittedBufferFrameNum(0),
+ mLatestSubmittedBufferUseCount(0),
+ mRepeatBufferDeferred(false) {
ALOGV("GraphicBufferSource w=%u h=%u c=%u",
bufferWidth, bufferHeight, bufferCount);
@@ -123,6 +130,22 @@
if (mEndOfStream && mNumFramesAvailable == 0) {
submitEndOfInputStream_l();
}
+
+ if (mRepeatAfterUs > 0ll && mLooper == NULL) {
+ mReflector = new AHandlerReflector<GraphicBufferSource>(this);
+
+ mLooper = new ALooper;
+ mLooper->registerHandler(mReflector);
+ mLooper->start();
+
+ if (mLatestSubmittedBufferId >= 0) {
+ sp<AMessage> msg =
+ new AMessage(kWhatRepeatLastFrame, mReflector->id());
+
+ msg->setInt32("generation", ++mRepeatLastFrameGeneration);
+ msg->post(mRepeatAfterUs);
+ }
+ }
}
void GraphicBufferSource::omxLoaded(){
@@ -132,6 +155,14 @@
ALOGW("Dropped back down to Loaded without Executing");
}
+ if (mLooper != NULL) {
+ mLooper->unregisterHandler(mReflector->id());
+ mReflector.clear();
+
+ mLooper->stop();
+ mLooper.clear();
+ }
+
ALOGV("--> loaded; avail=%d eos=%d eosSent=%d",
mNumFramesAvailable, mEndOfStream, mEndOfStreamSent);
@@ -211,8 +242,12 @@
ALOGV("cbi %d matches bq slot %d, handle=%p",
cbi, id, mBufferSlot[id]->handle);
- mBufferQueue->releaseBuffer(id, codecBuffer.mFrameNumber,
- EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, Fence::NO_FENCE);
+ if (id == mLatestSubmittedBufferId) {
+ CHECK_GT(mLatestSubmittedBufferUseCount--, 0);
+ } else {
+ mBufferQueue->releaseBuffer(id, codecBuffer.mFrameNumber,
+ EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, Fence::NO_FENCE);
+ }
} else {
ALOGV("codecBufferEmptied: no match for emptied buffer in cbi %d",
cbi);
@@ -232,7 +267,16 @@
// send that.
ALOGV("buffer freed, EOS pending");
submitEndOfInputStream_l();
+ } else if (mRepeatBufferDeferred) {
+ bool success = repeatLatestSubmittedBuffer_l();
+ if (success) {
+ ALOGV("deferred repeatLatestSubmittedBuffer_l SUCCESS");
+ } else {
+ ALOGV("deferred repeatLatestSubmittedBuffer_l FAILURE");
+ }
+ mRepeatBufferDeferred = false;
}
+
return;
}
@@ -264,6 +308,16 @@
}
mSuspended = false;
+
+ if (mExecuting && mNumFramesAvailable == 0 && mRepeatBufferDeferred) {
+ if (repeatLatestSubmittedBuffer_l()) {
+ ALOGV("suspend/deferred repeatLatestSubmittedBuffer_l SUCCESS");
+
+ mRepeatBufferDeferred = false;
+ } else {
+ ALOGV("suspend/deferred repeatLatestSubmittedBuffer_l FAILURE");
+ }
+ }
}
bool GraphicBufferSource::fillCodecBuffer_l() {
@@ -318,11 +372,68 @@
EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, Fence::NO_FENCE);
} else {
ALOGV("buffer submitted (bq %d, cbi %d)", item.mBuf, cbi);
+ setLatestSubmittedBuffer_l(item);
}
return true;
}
+bool GraphicBufferSource::repeatLatestSubmittedBuffer_l() {
+ CHECK(mExecuting && mNumFramesAvailable == 0);
+
+ if (mLatestSubmittedBufferId < 0 || mSuspended) {
+ return false;
+ }
+
+ int cbi = findAvailableCodecBuffer_l();
+ if (cbi < 0) {
+ // No buffers available, bail.
+ ALOGV("repeatLatestSubmittedBuffer_l: no codec buffers.");
+ return false;
+ }
+
+ BufferQueue::BufferItem item;
+ item.mBuf = mLatestSubmittedBufferId;
+ item.mFrameNumber = mLatestSubmittedBufferFrameNum;
+
+ status_t err = submitBuffer_l(item, cbi);
+
+ if (err != OK) {
+ return false;
+ }
+
+ ++mLatestSubmittedBufferUseCount;
+
+ return true;
+}
+
+void GraphicBufferSource::setLatestSubmittedBuffer_l(
+ const BufferQueue::BufferItem &item) {
+ ALOGV("setLatestSubmittedBuffer_l");
+
+ if (mLatestSubmittedBufferId >= 0) {
+ if (mLatestSubmittedBufferUseCount == 0) {
+ mBufferQueue->releaseBuffer(
+ mLatestSubmittedBufferId,
+ mLatestSubmittedBufferFrameNum,
+ EGL_NO_DISPLAY,
+ EGL_NO_SYNC_KHR,
+ Fence::NO_FENCE);
+ }
+ }
+
+ mLatestSubmittedBufferId = item.mBuf;
+ mLatestSubmittedBufferFrameNum = item.mFrameNumber;
+ mLatestSubmittedBufferUseCount = 1;
+ mRepeatBufferDeferred = false;
+
+ if (mReflector != NULL) {
+ sp<AMessage> msg = new AMessage(kWhatRepeatLastFrame, mReflector->id());
+ msg->setInt32("generation", ++mRepeatLastFrameGeneration);
+ msg->post(mRepeatAfterUs);
+ }
+}
+
status_t GraphicBufferSource::signalEndOfInputStream() {
Mutex::Autolock autoLock(mMutex);
ALOGV("signalEndOfInputStream: exec=%d avail=%d eos=%d",
@@ -470,6 +581,9 @@
mNumFramesAvailable++;
+ mRepeatBufferDeferred = false;
+ ++mRepeatLastFrameGeneration;
+
if (mExecuting) {
fillCodecBuffer_l();
}
@@ -495,4 +609,51 @@
}
}
+status_t GraphicBufferSource::setRepeatPreviousFrameDelayUs(
+ int64_t repeatAfterUs) {
+ Mutex::Autolock autoLock(mMutex);
+
+ if (mExecuting || repeatAfterUs <= 0ll) {
+ return INVALID_OPERATION;
+ }
+
+ mRepeatAfterUs = repeatAfterUs;
+
+ return OK;
+}
+
+void GraphicBufferSource::onMessageReceived(const sp<AMessage> &msg) {
+ switch (msg->what()) {
+ case kWhatRepeatLastFrame:
+ {
+ Mutex::Autolock autoLock(mMutex);
+
+ int32_t generation;
+ CHECK(msg->findInt32("generation", &generation));
+
+ if (generation != mRepeatLastFrameGeneration) {
+ // stale
+ break;
+ }
+
+ if (!mExecuting || mNumFramesAvailable > 0) {
+ break;
+ }
+
+ bool success = repeatLatestSubmittedBuffer_l();
+
+ if (success) {
+ ALOGV("repeatLatestSubmittedBuffer_l SUCCESS");
+ } else {
+ ALOGV("repeatLatestSubmittedBuffer_l FAILURE");
+ mRepeatBufferDeferred = true;
+ }
+ break;
+ }
+
+ default:
+ TRESPASS();
+ }
+}
+
} // namespace android
diff --git a/media/libstagefright/omx/GraphicBufferSource.h b/media/libstagefright/omx/GraphicBufferSource.h
index ac73770..244a843 100644
--- a/media/libstagefright/omx/GraphicBufferSource.h
+++ b/media/libstagefright/omx/GraphicBufferSource.h
@@ -25,6 +25,8 @@
#include <OMX_Core.h>
#include "../include/OMXNodeInstance.h"
#include <media/stagefright/foundation/ABase.h>
+#include <media/stagefright/foundation/AHandlerReflector.h>
+#include <media/stagefright/foundation/ALooper.h>
namespace android {
@@ -89,6 +91,15 @@
// in the BufferQueue) will be discarded until the suspension is lifted.
void suspend(bool suspend);
+ // Specifies the interval after which we requeue the buffer previously
+ // queued to the encoder. This is useful in the case of surface flinger
+ // providing the input surface if the resulting encoded stream is to
+ // be displayed "live". If we were not to push through the extra frame
+ // the decoder on the remote end would be unable to decode the latest frame.
+ // This API must be called before transitioning the encoder to "executing"
+ // state and once this behaviour is specified it cannot be reset.
+ status_t setRepeatPreviousFrameDelayUs(int64_t repeatAfterUs);
+
protected:
// BufferQueue::ConsumerListener interface, called when a new frame of
// data is available. If we're executing and a codec buffer is
@@ -147,6 +158,9 @@
// doing anything if we don't have a codec buffer available.
void submitEndOfInputStream_l();
+ void setLatestSubmittedBuffer_l(const BufferQueue::BufferItem &item);
+ bool repeatLatestSubmittedBuffer_l();
+
// Lock, covers all member variables.
mutable Mutex mMutex;
@@ -181,6 +195,30 @@
// Tracks codec buffers.
Vector<CodecBuffer> mCodecBuffers;
+ ////
+ friend class AHandlerReflector<GraphicBufferSource>;
+
+ enum {
+ kWhatRepeatLastFrame,
+ };
+
+ int64_t mRepeatAfterUs;
+
+ sp<ALooper> mLooper;
+ sp<AHandlerReflector<GraphicBufferSource> > mReflector;
+
+ int32_t mRepeatLastFrameGeneration;
+
+ int mLatestSubmittedBufferId;
+ uint64_t mLatestSubmittedBufferFrameNum;
+ int32_t mLatestSubmittedBufferUseCount;
+
+ // The previously submitted buffer should've been repeated but
+ // no codec buffer was available at the time.
+ bool mRepeatBufferDeferred;
+
+ void onMessageReceived(const sp<AMessage> &msg);
+
DISALLOW_EVIL_CONSTRUCTORS(GraphicBufferSource);
};
diff --git a/media/libstagefright/omx/OMXNodeInstance.cpp b/media/libstagefright/omx/OMXNodeInstance.cpp
index 8d100f1..ef683a0 100644
--- a/media/libstagefright/omx/OMXNodeInstance.cpp
+++ b/media/libstagefright/omx/OMXNodeInstance.cpp
@@ -809,6 +809,7 @@
size_t size) {
switch (type) {
case IOMX::INTERNAL_OPTION_SUSPEND:
+ case IOMX::INTERNAL_OPTION_REPEAT_PREVIOUS_FRAME_DELAY:
{
const sp<GraphicBufferSource> &bufferSource =
getGraphicBufferSource();
@@ -817,12 +818,22 @@
return ERROR_UNSUPPORTED;
}
- if (size != sizeof(bool)) {
- return INVALID_OPERATION;
- }
+ if (type == IOMX::INTERNAL_OPTION_SUSPEND) {
+ if (size != sizeof(bool)) {
+ return INVALID_OPERATION;
+ }
- bool suspend = *(bool *)data;
- bufferSource->suspend(suspend);
+ bool suspend = *(bool *)data;
+ bufferSource->suspend(suspend);
+ } else {
+ if (size != sizeof(int64_t)) {
+ return INVALID_OPERATION;
+ }
+
+ int64_t delayUs = *(int64_t *)data;
+
+ return bufferSource->setRepeatPreviousFrameDelayUs(delayUs);
+ }
return OK;
}