Allows the authoring engine to skip frame.
This is 1st part of the work to allow audio and video resync if
we found out that audio and video are out of sync during authoring
- also fixed a problem in AACEncoder::read() where the buffer acquired
from the buffer group does not release when error out at
reading from source.
Change-Id: I8a2740097fcfdf85e6178869afeb9f3687a99118
diff --git a/media/libstagefright/AudioSource.cpp b/media/libstagefright/AudioSource.cpp
index 6031797..50c0edc 100644
--- a/media/libstagefright/AudioSource.cpp
+++ b/media/libstagefright/AudioSource.cpp
@@ -137,53 +137,76 @@
MediaBuffer *buffer;
CHECK_EQ(mGroup->acquire_buffer(&buffer), OK);
- uint32_t numFramesRecorded;
- mRecord->getPosition(&numFramesRecorded);
- int64_t latency = mRecord->latency() * 1000;
+ while (mStarted) {
+ uint32_t numFramesRecorded;
+ mRecord->getPosition(&numFramesRecorded);
+ int64_t latency = mRecord->latency() * 1000;
- int64_t readTime = systemTime() / 1000;
- if (numFramesRecorded == 0) {
- // Initial delay
- if (mStartTimeUs > 0) {
- mStartTimeUs = readTime - mStartTimeUs;
+ int64_t readTime = systemTime() / 1000;
+
+ if (numFramesRecorded == 0) {
+ // Initial delay
+ if (mStartTimeUs > 0) {
+ mStartTimeUs = readTime - mStartTimeUs;
+ } else {
+ mStartTimeUs += latency;
+ }
+ }
+
+ ssize_t n = 0;
+ if (mCollectStats) {
+ n = mRecord->read(buffer->data(), buffer->size());
+ int64_t endTime = systemTime() / 1000;
+ mTotalReadTimeUs += (endTime - readTime);
+ if (n >= 0) {
+ mTotalReadBytes += n;
+ }
} else {
- mStartTimeUs += latency;
+ n = mRecord->read(buffer->data(), buffer->size());
}
- }
- ssize_t n = 0;
- if (mCollectStats) {
- n = mRecord->read(buffer->data(), buffer->size());
- int64_t endTime = systemTime() / 1000;
- mTotalReadTimeUs += (endTime - readTime);
- if (n >= 0) {
- mTotalReadBytes += n;
+ if (n < 0) {
+ buffer->release();
+ buffer = NULL;
+
+ return (status_t)n;
}
- } else {
- n = mRecord->read(buffer->data(), buffer->size());
+
+ uint32_t sampleRate = mRecord->getSampleRate();
+ int64_t timestampUs = (1000000LL * numFramesRecorded) / sampleRate +
+ mStartTimeUs;
+ int64_t skipFrameUs;
+ if (!options || !options->getSkipFrame(&skipFrameUs)) {
+ skipFrameUs = timestampUs; // Don't skip frame
+ }
+
+ if (skipFrameUs > timestampUs) {
+ // Safe guard against the abuse of the kSkipFrame_Option.
+ if (skipFrameUs - timestampUs >= 1E6) {
+ LOGE("Frame skipping requested is way too long: %lld us",
+ skipFrameUs - timestampUs);
+ buffer->release();
+ return UNKNOWN_ERROR;
+ }
+ LOGV("skipFrame: %lld us > timestamp: %lld us, samples %d",
+ skipFrameUs, timestampUs, numFramesRecorded);
+ continue;
+ }
+
+ if (mTrackMaxAmplitude) {
+ trackMaxAmplitude((int16_t *) buffer->data(), n >> 1);
+ }
+
+ buffer->meta_data()->setInt64(kKeyTime, timestampUs);
+ LOGV("initial delay: %lld, sample rate: %d, timestamp: %lld",
+ mStartTimeUs, sampleRate, timestampUs);
+
+ buffer->set_range(0, n);
+
+ *out = buffer;
+ return OK;
}
- if (n < 0) {
- buffer->release();
- buffer = NULL;
-
- return (status_t)n;
- }
-
- if (mTrackMaxAmplitude) {
- trackMaxAmplitude((int16_t *) buffer->data(), n >> 1);
- }
-
- uint32_t sampleRate = mRecord->getSampleRate();
- int64_t timestampUs = (1000000LL * numFramesRecorded) / sampleRate + mStartTimeUs;
- buffer->meta_data()->setInt64(kKeyTime, timestampUs);
- LOGV("initial delay: %lld, sample rate: %d, timestamp: %lld",
- mStartTimeUs, sampleRate, timestampUs);
-
- buffer->set_range(0, n);
-
- *out = buffer;
-
return OK;
}
diff --git a/media/libstagefright/CameraSource.cpp b/media/libstagefright/CameraSource.cpp
index 4cc176a..9c48daf 100644
--- a/media/libstagefright/CameraSource.cpp
+++ b/media/libstagefright/CameraSource.cpp
@@ -277,23 +277,44 @@
{
Mutex::Autolock autoLock(mLock);
- while (mStarted && mFramesReceived.empty()) {
- mFrameAvailableCondition.wait(mLock);
- }
- if (!mStarted) {
- return OK;
- }
- frame = *mFramesReceived.begin();
- mFramesReceived.erase(mFramesReceived.begin());
+ while (mStarted) {
+ while(mFramesReceived.empty()) {
+ mFrameAvailableCondition.wait(mLock);
+ }
- frameTime = *mFrameTimes.begin();
- mFrameTimes.erase(mFrameTimes.begin());
+ if (!mStarted) {
+ return OK;
+ }
- mFramesBeingEncoded.push_back(frame);
- *buffer = new MediaBuffer(frame->pointer(), frame->size());
- (*buffer)->setObserver(this);
- (*buffer)->add_ref();
- (*buffer)->meta_data()->setInt64(kKeyTime, frameTime);
+ frame = *mFramesReceived.begin();
+ mFramesReceived.erase(mFramesReceived.begin());
+
+ frameTime = *mFrameTimes.begin();
+ mFrameTimes.erase(mFrameTimes.begin());
+ int64_t skipTimeUs;
+ if (!options || !options->getSkipFrame(&skipTimeUs)) {
+ skipTimeUs = frameTime;
+ }
+ if (skipTimeUs > frameTime) {
+ LOGV("skipTimeUs: %lld us > frameTime: %lld us",
+ skipTimeUs, frameTime);
+ releaseOneRecordingFrame(frame);
+ ++mNumFramesDropped;
+ // Safeguard against the abuse of the kSkipFrame_Option.
+ if (skipTimeUs - frameTime >= 1E6) {
+ LOGE("Frame skipping requested is way too long: %lld us",
+ skipTimeUs - frameTime);
+ return UNKNOWN_ERROR;
+ }
+ } else {
+ mFramesBeingEncoded.push_back(frame);
+ *buffer = new MediaBuffer(frame->pointer(), frame->size());
+ (*buffer)->setObserver(this);
+ (*buffer)->add_ref();
+ (*buffer)->meta_data()->setInt64(kKeyTime, frameTime);
+ return OK;
+ }
+ }
}
return OK;
}
diff --git a/media/libstagefright/MediaSource.cpp b/media/libstagefright/MediaSource.cpp
index fd0e79c..b4ef338 100644
--- a/media/libstagefright/MediaSource.cpp
+++ b/media/libstagefright/MediaSource.cpp
@@ -32,6 +32,7 @@
mOptions = 0;
mSeekTimeUs = 0;
mLatenessUs = 0;
+ mSkipFrameUntilTimeUs = 0;
}
void MediaSource::ReadOptions::setSeekTo(int64_t time_us, SeekMode mode) {
@@ -53,6 +54,21 @@
return (mOptions & kSeekTo_Option) != 0;
}
+void MediaSource::ReadOptions::clearSkipFrame() {
+ mOptions &= ~kSkipFrame_Option;
+ mSkipFrameUntilTimeUs = 0;
+}
+
+void MediaSource::ReadOptions::setSkipFrame(int64_t timeUs) {
+ mOptions |= kSkipFrame_Option;
+ mSkipFrameUntilTimeUs = timeUs;
+}
+
+bool MediaSource::ReadOptions::getSkipFrame(int64_t *timeUs) const {
+ *timeUs = mSkipFrameUntilTimeUs;
+ return (mOptions & kSkipFrame_Option) != 0;
+}
+
void MediaSource::ReadOptions::setLateBy(int64_t lateness_us) {
mLatenessUs = lateness_us;
}
diff --git a/media/libstagefright/OMXCodec.cpp b/media/libstagefright/OMXCodec.cpp
index 1b63083..3fba2d9 100644
--- a/media/libstagefright/OMXCodec.cpp
+++ b/media/libstagefright/OMXCodec.cpp
@@ -1274,6 +1274,7 @@
mSeekTimeUs(-1),
mSeekMode(ReadOptions::SEEK_CLOSEST_SYNC),
mTargetTimeUs(-1),
+ mSkipTimeUs(-1),
mLeftOverBuffer(NULL),
mPaused(false) {
mPortStatus[kPortIndexInput] = ENABLED;
@@ -2200,13 +2201,15 @@
int32_t n = 0;
for (;;) {
MediaBuffer *srcBuffer;
+ MediaSource::ReadOptions options;
+ if (mSkipTimeUs >= 0) {
+ options.setSkipFrame(mSkipTimeUs);
+ }
if (mSeekTimeUs >= 0) {
if (mLeftOverBuffer) {
mLeftOverBuffer->release();
mLeftOverBuffer = NULL;
}
-
- MediaSource::ReadOptions options;
options.setSeekTo(mSeekTimeUs, mSeekMode);
mSeekTimeUs = -1;
@@ -2231,7 +2234,7 @@
err = OK;
} else {
- err = mSource->read(&srcBuffer);
+ err = mSource->read(&srcBuffer, &options);
}
if (err != OK) {
@@ -2830,6 +2833,12 @@
if (options && options->getSeekTo(&seekTimeUs, &seekMode)) {
seeking = true;
}
+ int64_t skipTimeUs;
+ if (options && options->getSkipFrame(&skipTimeUs)) {
+ mSkipTimeUs = skipTimeUs;
+ } else {
+ mSkipTimeUs = -1;
+ }
if (mInitialBufferSubmit) {
mInitialBufferSubmit = false;
diff --git a/media/libstagefright/codecs/aacenc/AACEncoder.cpp b/media/libstagefright/codecs/aacenc/AACEncoder.cpp
index e8235c2..052c354 100644
--- a/media/libstagefright/codecs/aacenc/AACEncoder.cpp
+++ b/media/libstagefright/codecs/aacenc/AACEncoder.cpp
@@ -224,6 +224,7 @@
if (mInputBuffer == NULL) {
if (mSource->read(&mInputBuffer, options) != OK) {
if (mNumInputSamples == 0) {
+ buffer->release();
return ERROR_END_OF_STREAM;
}
memset(&mInputFrame[mNumInputSamples],