Snap for 7842835 from 5e994b906a08e24633e016f010a3a4b21a7a5804 to sc-v2-release
Change-Id: I1787ad4ee6a3aa3b0c6fec7ef631c19e5d714cbe
diff --git a/README.md b/README.md
index fca0e51..a381864 100644
--- a/README.md
+++ b/README.md
@@ -10,7 +10,7 @@
v4l2\_codec2 project provides a component implementation of Codec2 framework,
the next-generation codec framework. The component implementation delegates the
-request to the driver via V4L2 API.
+request to the driver via the V4L2 API.
## Quick Start Guide
@@ -104,11 +104,31 @@
<MediaCodecs>
<Encoders>
<MediaCodec name="c2.v4l2.avc.encoder" type="video/avc">
- <Limit name="size" min="32x32" max="4096x4096" />
+ <Limit name="size" min="32x32" max="1920x1088" />
<Limit name="alignment" value="2x2" />
<Limit name="block-size" value="16x16" />
- <Limit name="blocks-per-second" range="1-983040" />
- <Limit name="bitrate" range="1-40000000" />
+ <Limit name="blocks-per-second" range="1-244800" />
+ <Limit name="bitrate" range="1-12000000" />
+ <Limit name="concurrent-instances" max="8" />
+ <Limit name="performance-point-1280x720" range="30-30" />
+ </MediaCodec>
+
+ <MediaCodec name="c2.v4l2.vp8.encoder" type="video/x-vnd.on2.vp8">
+ <Limit name="size" min="32x32" max="1920x1088" />
+ <Limit name="alignment" value="2x2" />
+ <Limit name="block-size" value="16x16" />
+ <Limit name="blocks-per-second" range="1-244800" />
+ <Limit name="bitrate" range="1-12000000" />
+ <Limit name="concurrent-instances" max="8" />
+ <Limit name="performance-point-1280x720" range="30-30" />
+ </MediaCodec>
+
+ <MediaCodec name="c2.v4l2.vp9.encoder" type="video/x-vnd.on2.vp9">
+ <Limit name="size" min="32x32" max="1920x1088" />
+ <Limit name="alignment" value="2x2" />
+ <Limit name="block-size" value="16x16" />
+ <Limit name="blocks-per-second" range="1-244800" />
+ <Limit name="bitrate" range="1-12000000" />
<Limit name="concurrent-instances" max="8" />
<Limit name="performance-point-1280x720" range="30-30" />
</MediaCodec>
@@ -288,3 +308,59 @@
}
```
+## V4L2 Encoder
+
+### Supported Codecs
+
+Currently the V4L2 encoder has support for the H.264, VP8 and VP9 codecs. Codec
+selection can be done by selecting the encoder with the appropriate name.
+
+- H26: *c2.v4l2.avc.encoder*
+- VP8: *c2.v4l2.vp8.encoder*
+- VP9: *c2.v4l2.vp9.encoder*
+
+### Supported Parameters:
+
+The following parameters are static and can not be changed at run-time:
+
+- *C2_PARAMKEY_PICTURE_SIZE*: This parameter can be used to configure the
+resolution of the encoded video stream.
+- *C2_PARAMKEY_PROFILE_LEVEL*: This parameter can be used to adjust the desired
+profile level. When using the H.264 codec the profile level might be adjusted to
+conform to the minimum requirements for the specified bitrate and framerate.
+- *C2_PARAMKEY_SYNC_FRAME_INTERVAL*: This parameter can be used to configure the
+desired time between subsequent key frames in microseconds.
+- *C2_PARAMKEY_BITRATE_MODE*: This parameter can be used to switch between
+constant (*C2Config::BITRATE_CONST*) and variable (*C2Config::BITRATE_VARIABLE*)
+bitrate modes. When using CBR the encoder will try to maintain a constant
+bitrate. This mode is preferable for video conferencing where maintaining a
+stable bitrate is more important than quality. When using VBR the encoder will
+be allowed to dynamically adjust the bitrate to maintain a constant quality. As
+the mediacodec framework does not provide facilities to configure the peak
+bitrate when using VBR, it is currently always set to 2x the target bitrate.
+
+The following parameters are dynamic, and can be freely adjusted at run-time:
+
+- *C2_PARAMKEY_BITRATE*: Use this parameter to specify the desired bitrate.
+- *C2_PARAMKEY_FRAME_RATE*: This parameter can be used to configure the desired
+framerate. Note that the encoder will automatically try to adjust the framerate
+if the timestamps on the input video frames don't match the configured
+framerate.
+- *C2_PARAMKEY_REQUEST_SYNC_FRAME*: This parameter can be used to request
+additional key frames in addition to the periodic ones requested through the
+*C2_PARAMKEY_SYNC_FRAME_INTERVAL* parameter.
+
+### Supported Input Pixel Formats:
+
+The V4L2 encoder supports various input pixel formats, however frames are
+currently always passed to the V4L2 encoder in the NV12 format. If a video frame
+using a different pixel format is passed to the encoder, format conversion will
+be performed to convert the frame to the NV12 format.
+
+### Additional Features:
+
+To improve the resilience of H.264 video streams when data is missing, SPS and
+PPS NAL units are prepended to IDR frames by enabling the
+*V4L2_CID_MPEG_VIDEO_PREPEND_SPSPPS_TO_IDR* control. If the V4L2 driver does not
+support this control the encoder will manually cache and prepend SPS and PPS NAL
+units.
diff --git a/common/EncodeHelpers.cpp b/common/EncodeHelpers.cpp
index 4575197..c07281f 100644
--- a/common/EncodeHelpers.cpp
+++ b/common/EncodeHelpers.cpp
@@ -18,6 +18,32 @@
namespace android {
+namespace {
+
+// Android frameworks needs 4 bytes start code.
+constexpr uint8_t kH264StartCode[] = {0x00, 0x00, 0x00, 0x01};
+constexpr size_t kH264StartCodeSize = 4;
+
+// Copy an H.264 NAL unit with size |srcSize| (without a start code) into a buffer with size
+// |dstSize|. An H.264 start code is prepended to the NAL unit. After copying |dst| is adjusted to
+// point to the address immediately following the copied data, and the |dstSize| is updated to
+// reflect the remaining destination buffer size.
+bool copyNALUPrependingStartCode(const uint8_t* src, size_t srcSize, uint8_t** dst,
+ size_t* dstSize) {
+ size_t naluSize = srcSize + kH264StartCodeSize;
+ if (naluSize > *dstSize) {
+ ALOGE("Couldn't copy NAL unit, not enough space in destination buffer");
+ return false;
+ }
+ memcpy(*dst, kH264StartCode, kH264StartCodeSize);
+ memcpy(*dst + kH264StartCodeSize, src, srcSize);
+ *dst += naluSize;
+ *dstSize -= naluSize;
+ return true;
+}
+
+} // namespace
+
uint8_t c2LevelToV4L2Level(C2Config::level_t level) {
switch (level) {
case C2Config::LEVEL_AVC_1:
@@ -84,41 +110,101 @@
return ycbcr;
}
-void extractCSDInfo(std::unique_ptr<C2StreamInitDataInfo::output>* const csd, const uint8_t* data,
- size_t length) {
- // Android frameworks needs 4 bytes start code.
- constexpr uint8_t kStartCode[] = {0x00, 0x00, 0x00, 0x01};
- constexpr int kStartCodeLength = 4;
+bool extractSPSPPS(const uint8_t* data, size_t length, std::vector<uint8_t>* sps,
+ std::vector<uint8_t>* pps) {
+ bool foundSPS = false;
+ bool foundPPS = false;
+ NalParser parser(data, length);
+ while (!(foundSPS && foundPPS) && parser.locateNextNal()) {
+ switch (parser.type()) {
+ case NalParser::kSPSType:
+ sps->resize(parser.length());
+ memcpy(sps->data(), parser.data(), parser.length());
+ foundSPS = true;
+ break;
+ case NalParser::kPPSType:
+ pps->resize(parser.length());
+ memcpy(pps->data(), parser.data(), parser.length());
+ foundPPS = true;
+ break;
+ }
+ }
+ return foundSPS && foundPPS;
+}
+bool extractCSDInfo(std::unique_ptr<C2StreamInitDataInfo::output>* const csd, const uint8_t* data,
+ size_t length) {
csd->reset();
- // Temporarily allocate a byte array to copy codec config data. This should be freed after
- // codec config data extraction is done.
- auto tmpConfigData = std::make_unique<uint8_t[]>(length);
- uint8_t* tmpOutput = tmpConfigData.get();
- uint8_t* tmpConfigDataEnd = tmpOutput + length;
-
- NalParser parser(data, length);
- while (parser.locateNextNal()) {
- if (parser.length() == 0) continue;
- uint8_t nalType = parser.type();
- ALOGV("find next NAL: type=%d, length=%zu", nalType, parser.length());
- if (nalType != NalParser::kSPSType && nalType != NalParser::kPPSType) continue;
-
- if (tmpOutput + kStartCodeLength + parser.length() > tmpConfigDataEnd) {
- ALOGE("Buffer overflow on extracting codec config data (length=%zu)", length);
- return;
- }
- std::memcpy(tmpOutput, kStartCode, kStartCodeLength);
- tmpOutput += kStartCodeLength;
- std::memcpy(tmpOutput, parser.data(), parser.length());
- tmpOutput += parser.length();
+ std::vector<uint8_t> sps;
+ std::vector<uint8_t> pps;
+ if (!extractSPSPPS(data, length, &sps, &pps)) {
+ return false;
}
- size_t configDataLength = tmpOutput - tmpConfigData.get();
+ size_t configDataLength = sps.size() + pps.size() + (2u * kH264StartCodeSize);
ALOGV("Extracted codec config data: length=%zu", configDataLength);
+
*csd = C2StreamInitDataInfo::output::AllocUnique(configDataLength, 0u);
- std::memcpy((*csd)->m.value, tmpConfigData.get(), configDataLength);
+ uint8_t* csdBuffer = (*csd)->m.value;
+ return copyNALUPrependingStartCode(sps.data(), sps.size(), &csdBuffer, &configDataLength) &&
+ copyNALUPrependingStartCode(pps.data(), pps.size(), &csdBuffer, &configDataLength);
+}
+
+size_t prependSPSPPSToIDR(const uint8_t* src, size_t srcSize, uint8_t* dst, size_t dstSize,
+ std::vector<uint8_t>* sps, std::vector<uint8_t>* pps) {
+ bool foundStreamParams = false;
+ size_t remainingDstSize = dstSize;
+ NalParser parser(src, srcSize);
+ while (parser.locateNextNal()) {
+ switch (parser.type()) {
+ case NalParser::kSPSType:
+ // SPS found, copy to cache.
+ ALOGV("Found SPS (length %zu)", parser.length());
+ sps->resize(parser.length());
+ memcpy(sps->data(), parser.data(), parser.length());
+ foundStreamParams = true;
+ break;
+ case NalParser::kPPSType:
+ // PPS found, copy to cache.
+ ALOGV("Found PPS (length %zu)", parser.length());
+ pps->resize(parser.length());
+ memcpy(pps->data(), parser.data(), parser.length());
+ foundStreamParams = true;
+ break;
+ case NalParser::kIDRType:
+ ALOGV("Found IDR (length %zu)", parser.length());
+ if (foundStreamParams) {
+ ALOGV("Not injecting SPS and PPS before IDR, already present");
+ break;
+ }
+
+ // Prepend the cached SPS and PPS to the IDR NAL unit.
+ if (sps->empty() || pps->empty()) {
+ ALOGE("No cached SPS or PPS NAL unit available to inject before IDR");
+ return 0;
+ }
+ if (!copyNALUPrependingStartCode(sps->data(), sps->size(), &dst, &remainingDstSize)) {
+ ALOGE("Not enough space to inject SPS NAL unit before IDR");
+ return 0;
+ }
+ if (!copyNALUPrependingStartCode(pps->data(), pps->size(), &dst, &remainingDstSize)) {
+ ALOGE("Not enough space to inject PPS NAL unit before IDR");
+ return 0;
+ }
+
+ ALOGV("Stream header injected before IDR");
+ break;
+ }
+
+ // Copy the NAL unit to the new output buffer.
+ if (!copyNALUPrependingStartCode(parser.data(), parser.length(), &dst, &remainingDstSize)) {
+ ALOGE("NAL unit does not fit in the provided output buffer");
+ return 0;
+ }
+ }
+
+ return dstSize - remainingDstSize;
}
} // namespace android
diff --git a/common/V4L2Device.cpp b/common/V4L2Device.cpp
index d4fa7f6..1efb4e3 100644
--- a/common/V4L2Device.cpp
+++ b/common/V4L2Device.cpp
@@ -1439,6 +1439,27 @@
}
// static
+v4l2_mpeg_video_bitrate_mode V4L2Device::C2BitrateModeToV4L2BitrateMode(
+ C2Config::bitrate_mode_t bitrateMode) {
+ switch (bitrateMode) {
+ case C2Config::bitrate_mode_t::BITRATE_CONST_SKIP_ALLOWED:
+ ALOGW("BITRATE_CONST_SKIP_ALLOWED not supported, defaulting to BITRATE_CONST");
+ FALLTHROUGH;
+ case C2Config::bitrate_mode_t::BITRATE_CONST:
+ return V4L2_MPEG_VIDEO_BITRATE_MODE_CBR;
+ case C2Config::bitrate_mode_t::BITRATE_VARIABLE_SKIP_ALLOWED:
+ ALOGW("BITRATE_VARIABLE_SKIP_ALLOWED not supported, defaulting to BITRATE_VARIABLE");
+ FALLTHROUGH;
+ case C2Config::bitrate_mode_t::BITRATE_VARIABLE:
+ return V4L2_MPEG_VIDEO_BITRATE_MODE_VBR;
+ default:
+ ALOGW("Unsupported bitrate mode %u, defaulting to BITRATE_VARIABLE",
+ static_cast<uint32_t>(bitrateMode));
+ return V4L2_MPEG_VIDEO_BITRATE_MODE_VBR;
+ }
+}
+
+// static
ui::Size V4L2Device::allocatedSizeFromV4L2Format(const struct v4l2_format& format) {
ui::Size codedSize;
ui::Size visibleSize;
diff --git a/common/include/v4l2_codec2/common/EncodeHelpers.h b/common/include/v4l2_codec2/common/EncodeHelpers.h
index bfbdd05..832ac91 100644
--- a/common/include/v4l2_codec2/common/EncodeHelpers.h
+++ b/common/include/v4l2_codec2/common/EncodeHelpers.h
@@ -42,12 +42,24 @@
// Get the specified graphics block in YCbCr format.
android_ycbcr getGraphicBlockInfo(const C2ConstGraphicBlock& block);
+// Try to extract SPS and PPS NAL units from the specified H.264 |data| stream. If found the data
+// will be copied (after resizing) into the provided |sps| and |pps| buffers. Returns whether
+// extraction was successful.
+bool extractSPSPPS(const uint8_t* data, size_t length, std::vector<uint8_t>* sps,
+ std::vector<uint8_t>* pps);
+
// When encoding a video the codec-specific data (CSD; e.g. SPS and PPS for H264 encoding) will be
// concatenated to the first encoded slice. This function extracts the CSD out of the bitstream and
-// stores it into |csd|.
-void extractCSDInfo(std::unique_ptr<C2StreamInitDataInfo::output>* const csd, const uint8_t* data,
+// stores it into |csd|. Returns whether extracting CSD info was successful.
+bool extractCSDInfo(std::unique_ptr<C2StreamInitDataInfo::output>* const csd, const uint8_t* data,
size_t length);
+// Prepend the specified |sps| and |pps| NAL units (without start codes) to the H.264 |data| stream.
+// The result is copied into |dst|. The provided |sps| and |pps| data will be updated if an SPS or
+// PPS NAL unit is encountered. Returns the size of the new data, will be 0 if an error occurred.
+size_t prependSPSPPSToIDR(const uint8_t* src, size_t srcSize, uint8_t* dst, size_t dstSize,
+ std::vector<uint8_t>* sps, std::vector<uint8_t>* pps);
+
} // namespace android
#endif // ANDROID_V4L2_CODEC2_COMMON_HELPERS_H
diff --git a/common/include/v4l2_codec2/common/NalParser.h b/common/include/v4l2_codec2/common/NalParser.h
index 657627c..ec8a876 100644
--- a/common/include/v4l2_codec2/common/NalParser.h
+++ b/common/include/v4l2_codec2/common/NalParser.h
@@ -12,6 +12,8 @@
// Helper class to parse H264 NAL units from data.
class NalParser {
public:
+ // Type of a IDR Slice NAL unit.
+ static constexpr uint8_t kIDRType = 5;
// Type of a SPS NAL unit.
static constexpr uint8_t kSPSType = 7;
// Type of a PPS NAL unit.
diff --git a/common/include/v4l2_codec2/common/V4L2Device.h b/common/include/v4l2_codec2/common/V4L2Device.h
index b4c909c..77d7ddb 100644
--- a/common/include/v4l2_codec2/common/V4L2Device.h
+++ b/common/include/v4l2_codec2/common/V4L2Device.h
@@ -349,6 +349,8 @@
// Convert required H264 profile and level to V4L2 enums.
static int32_t c2ProfileToV4L2H264Profile(C2Config::profile_t profile);
static int32_t h264LevelIdcToV4L2H264Level(uint8_t levelIdc);
+ static v4l2_mpeg_video_bitrate_mode C2BitrateModeToV4L2BitrateMode(
+ C2Config::bitrate_mode_t bitrateMode);
// Converts v4l2_memory to a string.
static const char* v4L2MemoryToString(const v4l2_memory memory);
diff --git a/components/V4L2DecodeComponent.cpp b/components/V4L2DecodeComponent.cpp
index 1b68c2e..456f3c4 100644
--- a/components/V4L2DecodeComponent.cpp
+++ b/components/V4L2DecodeComponent.cpp
@@ -202,7 +202,6 @@
}
mDecoderTaskRunner = mDecoderThread.task_runner();
mWeakThis = mWeakThisFactory.GetWeakPtr();
- mStdWeakThis = weak_from_this();
c2_status_t status = C2_CORRUPTED;
::base::WaitableEvent done;
@@ -261,7 +260,7 @@
ALOGV("%s()", __func__);
ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
- auto sharedThis = mStdWeakThis.lock();
+ auto sharedThis = weak_from_this().lock();
if (sharedThis == nullptr) {
ALOGE("%s(): V4L2DecodeComponent instance is destroyed.", __func__);
return nullptr;
@@ -348,7 +347,6 @@
ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
mWeakThisFactory.InvalidateWeakPtrs();
- mStdWeakThis.reset();
mDecoder = nullptr;
}
@@ -494,9 +492,8 @@
}
}
- std::unique_ptr<BitstreamBuffer> buffer =
- std::make_unique<BitstreamBuffer>(bitstreamId, linearBlock.handle()->data[0],
- linearBlock.offset(), linearBlock.size());
+ std::unique_ptr<ConstBitstreamBuffer> buffer = std::make_unique<ConstBitstreamBuffer>(
+ bitstreamId, linearBlock, linearBlock.offset(), linearBlock.size());
if (!buffer) {
reportError(C2_CORRUPTED);
return;
@@ -699,12 +696,6 @@
ALOGV("%s(work=%llu)", __func__, work->input.ordinal.frameIndex.peekull());
ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
- auto sharedThis = mStdWeakThis.lock();
- if (sharedThis == nullptr) {
- ALOGE("%s(): V4L2DecodeComponent instance is destroyed.", __func__);
- return false;
- }
-
if (!mListener) {
ALOGE("mListener is nullptr, setListener_vb() not called?");
return false;
@@ -712,7 +703,7 @@
std::list<std::unique_ptr<C2Work>> finishedWorks;
finishedWorks.emplace_back(std::move(work));
- mListener->onWorkDone_nb(std::move(sharedThis), std::move(finishedWorks));
+ mListener->onWorkDone_nb(weak_from_this(), std::move(finishedWorks));
return true;
}
@@ -749,12 +740,6 @@
ALOGV("%s()", __func__);
ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
- auto sharedThis = mStdWeakThis.lock();
- if (sharedThis == nullptr) {
- ALOGE("%s(): V4L2DecodeComponent instance is destroyed.", __func__);
- return;
- }
-
std::list<std::unique_ptr<C2Work>> abandonedWorks;
while (!mPendingWorks.empty()) {
abandonedWorks.emplace_back(std::move(mPendingWorks.front()));
@@ -778,7 +763,7 @@
ALOGE("mListener is nullptr, setListener_vb() not called?");
return;
}
- mListener->onWorkDone_nb(std::move(sharedThis), std::move(abandonedWorks));
+ mListener->onWorkDone_nb(weak_from_this(), std::move(abandonedWorks));
}
}
@@ -852,12 +837,6 @@
ALOGE("%s(error=%u)", __func__, static_cast<uint32_t>(error));
ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
- auto sharedThis = mStdWeakThis.lock();
- if (sharedThis == nullptr) {
- ALOGE("%s(): V4L2DecodeComponent instance is destroyed.", __func__);
- return;
- }
-
if (mComponentState.load() == ComponentState::ERROR) return;
mComponentState.store(ComponentState::ERROR);
@@ -865,7 +844,7 @@
ALOGE("mListener is nullptr, setListener_vb() not called?");
return;
}
- mListener->onError_nb(std::move(sharedThis), static_cast<uint32_t>(error));
+ mListener->onError_nb(weak_from_this(), static_cast<uint32_t>(error));
}
c2_status_t V4L2DecodeComponent::announce_nb(const std::vector<C2WorkOutline>& /* items */) {
diff --git a/components/V4L2Decoder.cpp b/components/V4L2Decoder.cpp
index 3bad19a..18d62d2 100644
--- a/components/V4L2Decoder.cpp
+++ b/components/V4L2Decoder.cpp
@@ -192,7 +192,7 @@
return true;
}
-void V4L2Decoder::decode(std::unique_ptr<BitstreamBuffer> buffer, DecodeCB decodeCb) {
+void V4L2Decoder::decode(std::unique_ptr<ConstBitstreamBuffer> buffer, DecodeCB decodeCb) {
ALOGV("%s(id=%d)", __func__, buffer->id);
ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence());
@@ -300,7 +300,7 @@
inputBuffer->setPlaneDataOffset(0, request.buffer->offset);
inputBuffer->setPlaneBytesUsed(0, request.buffer->offset + request.buffer->size);
std::vector<int> fds;
- fds.push_back(std::move(request.buffer->dmabuf_fd));
+ fds.push_back(std::move(request.buffer->dmabuf.handle()->data[0]));
if (!std::move(*inputBuffer).queueDMABuf(fds)) {
ALOGE("%s(): Failed to QBUF to input queue, bitstreamId=%d", __func__, bitstreamId);
onError();
diff --git a/components/V4L2EncodeComponent.cpp b/components/V4L2EncodeComponent.cpp
index 459ac85..a1b46ab 100644
--- a/components/V4L2EncodeComponent.cpp
+++ b/components/V4L2EncodeComponent.cpp
@@ -40,6 +40,9 @@
const VideoPixelFormat kInputPixelFormat = VideoPixelFormat::NV12;
+// The peak bitrate in function of the target bitrate, used when the bitrate mode is VBR.
+constexpr uint32_t kPeakBitrateMultiplier = 2u;
+
// Get the video frame layout from the specified |inputBlock|.
// TODO(dstaessens): Clean up code extracting layout from a C2GraphicBlock.
std::optional<std::vector<VideoFramePlane>> getVideoFrameLayout(const C2ConstGraphicBlock& block,
@@ -186,6 +189,12 @@
format, index, timestamp);
}
+// Check whether the specified |profile| is an H.264 profile.
+bool IsH264Profile(C2Config::profile_t profile) {
+ return (profile >= C2Config::PROFILE_AVC_BASELINE &&
+ profile <= C2Config::PROFILE_AVC_ENHANCED_MULTIVIEW_DEPTH_HIGH);
+}
+
} // namespace
// static
@@ -617,14 +626,16 @@
ALOG_ASSERT(!mInputFormatConverter);
ALOG_ASSERT(!mEncoder);
- mCSDSubmitted = false;
+ mLastFrameTime = std::nullopt;
// Get the requested profile and level.
C2Config::profile_t outputProfile = mInterface->getOutputProfile();
+ // CSD only needs to be extracted when using an H.264 profile.
+ mExtractCSD = IsH264Profile(outputProfile);
+
std::optional<uint8_t> h264Level;
- if (outputProfile >= C2Config::PROFILE_AVC_BASELINE &&
- outputProfile <= C2Config::PROFILE_AVC_ENHANCED_MULTIVIEW_DEPTH_HIGH) {
+ if (IsH264Profile(outputProfile)) {
h264Level = c2LevelToV4L2Level(mInterface->getOutputLevel());
}
@@ -638,9 +649,15 @@
return false;
}
+ // Get the requested bitrate mode and bitrate. The C2 framework doesn't offer a parameter to
+ // configure the peak bitrate, so we use a multiple of the target bitrate.
+ mBitrateMode = mInterface->getBitrateMode();
+ mBitrate = mInterface->getBitrate();
+
mEncoder = V4L2Encoder::create(
outputProfile, h264Level, mInterface->getInputVisibleSize(), *stride,
- mInterface->getKeyFramePeriod(),
+ mInterface->getKeyFramePeriod(), mBitrateMode, mBitrate,
+ mBitrate * kPeakBitrateMultiplier,
::base::BindRepeating(&V4L2EncodeComponent::fetchOutputBlock, mWeakThis),
::base::BindRepeating(&V4L2EncodeComponent::onInputBufferDone, mWeakThis),
::base::BindRepeating(&V4L2EncodeComponent::onOutputBufferDone, mWeakThis),
@@ -670,19 +687,10 @@
ALOGV("%s()", __func__);
ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence());
- // Query the interface for the encoding parameters requested by the codec 2.0 framework.
- C2StreamBitrateInfo::output bitrateInfo;
- C2StreamFrameRateInfo::output framerateInfo;
- c2_status_t status =
- mInterface->query({&bitrateInfo, &framerateInfo}, {}, C2_DONT_BLOCK, nullptr);
- if (status != C2_OK) {
- ALOGE("Failed to query interface for encoding parameters (error code: %d)", status);
- reportError(status);
- return false;
- }
-
- // Ask device to change bitrate if it's different from the currently configured bitrate.
- uint32_t bitrate = bitrateInfo.value;
+ // Ask device to change bitrate if it's different from the currently configured bitrate. The C2
+ // framework doesn't offer a parameter to configure the peak bitrate, so we'll use a multiple of
+ // the target bitrate here. The peak bitrate is only used if the bitrate mode is set to VBR.
+ uint32_t bitrate = mInterface->getBitrate();
if (mBitrate != bitrate) {
ALOG_ASSERT(bitrate > 0u);
ALOGV("Setting bitrate to %u", bitrate);
@@ -691,10 +699,17 @@
return false;
}
mBitrate = bitrate;
+
+ if (mBitrateMode == C2Config::BITRATE_VARIABLE) {
+ ALOGV("Setting peak bitrate to %u", bitrate * kPeakBitrateMultiplier);
+ // TODO(b/190336806): Our stack doesn't support dynamic peak bitrate changes yet, ignore
+ // errors for now.
+ mEncoder->setPeakBitrate(bitrate * kPeakBitrateMultiplier);
+ }
}
// Ask device to change framerate if it's different from the currently configured framerate.
- uint32_t framerate = static_cast<uint32_t>(std::round(framerateInfo.value));
+ uint32_t framerate = static_cast<uint32_t>(std::round(mInterface->getFramerate()));
if (mFramerate != framerate) {
ALOG_ASSERT(framerate > 0u);
ALOGV("Setting framerate to %u", framerate);
@@ -709,7 +724,7 @@
// Check whether an explicit key frame was requested, if so reset the key frame counter to
// immediately request a key frame.
C2StreamRequestSyncFrameTuning::output requestKeyFrame;
- status = mInterface->query({&requestKeyFrame}, {}, C2_DONT_BLOCK, nullptr);
+ c2_status_t status = mInterface->query({&requestKeyFrame}, {}, C2_DONT_BLOCK, nullptr);
if (status != C2_OK) {
ALOGE("Failed to query interface for key frame request (error code: %d)", status);
reportError(status);
@@ -738,6 +753,18 @@
ALOGV("Encoding input block (index: %" PRIu64 ", timestamp: %" PRId64 ", size: %dx%d)", index,
timestamp, block.width(), block.height());
+ // Dynamically adjust framerate based on the frame's timestamp if required.
+ constexpr int64_t kMaxFramerateDiff = 5;
+ if (mLastFrameTime && (timestamp > *mLastFrameTime)) {
+ int64_t newFramerate =
+ static_cast<int64_t>(std::round(1000000.0 / (timestamp - *mLastFrameTime)));
+ if (abs(mFramerate - newFramerate) > kMaxFramerateDiff) {
+ ALOGV("Adjusting framerate to %" PRId64 " based on frame timestamps", newFramerate);
+ mInterface->setFramerate(static_cast<uint32_t>(newFramerate));
+ }
+ }
+ mLastFrameTime = timestamp;
+
// Update dynamic encoding parameters (bitrate, framerate, key frame) if requested.
if (!updateEncodingParameters()) return false;
@@ -785,7 +812,7 @@
mWorkQueue.pop_front();
}
if (!abortedWorkItems.empty()) {
- mListener->onWorkDone_nb(shared_from_this(), std::move(abortedWorkItems));
+ mListener->onWorkDone_nb(weak_from_this(), std::move(abortedWorkItems));
}
}
@@ -803,13 +830,7 @@
reportError(status);
}
- // Store a reference to the block to keep the fds alive.
- int fd = block->handle()->data[0];
- ALOG_ASSERT(!mOutputBuffersMap[fd]);
- mOutputBuffersMap[fd] = std::move(block);
-
- // TODO(dstaessens) Store the C2LinearBlock directly into the BitstreamBuffer.
- *buffer = std::make_unique<BitstreamBuffer>(fd, fd, 0, size);
+ *buffer = std::make_unique<BitstreamBuffer>(std::move(block), 0, size);
}
void V4L2EncodeComponent::onInputBufferDone(uint64_t index) {
@@ -859,22 +880,19 @@
ALOGV("%s(): output buffer done (timestamp: %" PRId64 ", size: %zu, keyframe: %d)", __func__,
timestamp, dataSize, keyFrame);
ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence());
+ ALOG_ASSERT(buffer->dmabuf);
- std::shared_ptr<C2LinearBlock> outputBlock = std::move(mOutputBuffersMap[buffer->id]);
- mOutputBuffersMap.erase(buffer->id);
- ALOG_ASSERT(outputBlock);
-
- C2ConstLinearBlock constBlock = outputBlock->share(outputBlock->offset(), dataSize, C2Fence());
+ C2ConstLinearBlock constBlock =
+ buffer->dmabuf->share(buffer->dmabuf->offset(), dataSize, C2Fence());
// If no CSD (content-specific-data, e.g. SPS for H.264) has been submitted yet, we expect this
// output block to contain CSD. We only submit the CSD once, even if it's attached to each key
// frame.
- if (!mCSDSubmitted) {
+ if (mExtractCSD) {
ALOGV("No CSD submitted yet, extracting CSD");
std::unique_ptr<C2StreamInitDataInfo::output> csd;
C2ReadView view = constBlock.map().get();
- extractCSDInfo(&csd, view.data(), view.capacity());
- if (!csd) {
+ if (!extractCSDInfo(&csd, view.data(), view.capacity())) {
ALOGE("Failed to extract CSD");
reportError(C2_CORRUPTED);
return;
@@ -884,7 +902,7 @@
LOG_ASSERT(!mWorkQueue.empty());
C2Work* work = mWorkQueue.front().get();
work->worklets.front()->output.configUpdate.push_back(std::move(csd));
- mCSDSubmitted = true;
+ mExtractCSD = false;
}
// Get the work item associated with the timestamp.
@@ -993,15 +1011,23 @@
std::list<std::unique_ptr<C2Work>> finishedWorkList;
finishedWorkList.emplace_back(std::move(work));
- mListener->onWorkDone_nb(shared_from_this(), std::move(finishedWorkList));
+ mListener->onWorkDone_nb(weak_from_this(), std::move(finishedWorkList));
}
bool V4L2EncodeComponent::getBlockPool() {
+ ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence());
+
+ auto sharedThis = weak_from_this().lock();
+ if (!sharedThis) {
+ ALOGI("%s(): V4L2EncodeComponent instance is already destroyed", __func__);
+ return false;
+ }
+
C2BlockPool::local_id_t poolId = mInterface->getBlockPoolId();
if (poolId == C2BlockPool::BASIC_LINEAR) {
ALOGW("Using unoptimized linear block pool");
}
- c2_status_t status = GetCodec2BlockPool(poolId, shared_from_this(), &mOutputBlockPool);
+ c2_status_t status = GetCodec2BlockPool(poolId, std::move(sharedThis), &mOutputBlockPool);
if (status != C2_OK || !mOutputBlockPool) {
ALOGE("Failed to get output block pool, error: %d", status);
return false;
@@ -1017,7 +1043,7 @@
std::lock_guard<std::mutex> lock(mComponentLock);
if (mComponentState != ComponentState::ERROR) {
setComponentState(ComponentState::ERROR);
- mListener->onError_nb(shared_from_this(), static_cast<uint32_t>(error));
+ mListener->onError_nb(weak_from_this(), static_cast<uint32_t>(error));
}
}
diff --git a/components/V4L2EncodeInterface.cpp b/components/V4L2EncodeInterface.cpp
index 7f0fb39..03d8c37 100644
--- a/components/V4L2EncodeInterface.cpp
+++ b/components/V4L2EncodeInterface.cpp
@@ -310,6 +310,15 @@
.withSetter(Setter<decltype(*mBitrate)>::StrictValueWithNoDeps)
.build());
+ addParameter(
+ DefineParam(mBitrateMode, C2_PARAMKEY_BITRATE_MODE)
+ .withDefault(new C2StreamBitrateModeTuning::output(0u, C2Config::BITRATE_CONST))
+ .withFields(
+ {C2F(mBitrateMode, value)
+ .oneOf({C2Config::BITRATE_CONST, C2Config::BITRATE_VARIABLE})})
+ .withSetter(Setter<decltype(*mBitrateMode)>::StrictValueWithNoDeps)
+ .build());
+
std::string outputMime;
if (getCodecFromComponentName(name) == VideoCodec::H264) {
outputMime = MEDIA_MIMETYPE_VIDEO_AVC;
@@ -329,8 +338,8 @@
C2Config::LEVEL_AVC_2_1, C2Config::LEVEL_AVC_2_2,
C2Config::LEVEL_AVC_3, C2Config::LEVEL_AVC_3_1,
C2Config::LEVEL_AVC_3_2, C2Config::LEVEL_AVC_4,
- C2Config::LEVEL_AVC_4_1, C2Config::LEVEL_AVC_5,
- C2Config::LEVEL_AVC_5_1})})
+ C2Config::LEVEL_AVC_4_1, C2Config::LEVEL_AVC_4_2,
+ C2Config::LEVEL_AVC_5, C2Config::LEVEL_AVC_5_1})})
.withSetter(H264ProfileLevelSetter, mInputVisibleSize, mFrameRate, mBitrate)
.build());
} else if (getCodecFromComponentName(name) == VideoCodec::VP8) {
diff --git a/components/V4L2Encoder.cpp b/components/V4L2Encoder.cpp
index 67d54c8..cd20cb5 100644
--- a/components/V4L2Encoder.cpp
+++ b/components/V4L2Encoder.cpp
@@ -17,6 +17,7 @@
#include <log/log.h>
#include <ui/Rect.h>
+#include <v4l2_codec2/common/EncodeHelpers.h>
#include <v4l2_codec2/common/Fourcc.h>
#include <v4l2_codec2/common/V4L2Device.h>
#include <v4l2_codec2/components/BitstreamBuffer.h>
@@ -54,6 +55,7 @@
std::unique_ptr<VideoEncoder> V4L2Encoder::create(
C2Config::profile_t outputProfile, std::optional<uint8_t> level,
const ui::Size& visibleSize, uint32_t stride, uint32_t keyFramePeriod,
+ C2Config::bitrate_mode_t bitrateMode, uint32_t bitrate, std::optional<uint32_t> peakBitrate,
FetchOutputBufferCB fetchOutputBufferCb, InputBufferDoneCB inputBufferDoneCb,
OutputBufferDoneCB outputBufferDoneCb, DrainDoneCB drainDoneCb, ErrorCB errorCb,
scoped_refptr<::base::SequencedTaskRunner> taskRunner) {
@@ -62,7 +64,8 @@
std::unique_ptr<V4L2Encoder> encoder = ::base::WrapUnique<V4L2Encoder>(new V4L2Encoder(
std::move(taskRunner), std::move(fetchOutputBufferCb), std::move(inputBufferDoneCb),
std::move(outputBufferDoneCb), std::move(drainDoneCb), std::move(errorCb)));
- if (!encoder->initialize(outputProfile, level, visibleSize, stride, keyFramePeriod)) {
+ if (!encoder->initialize(outputProfile, level, visibleSize, stride, keyFramePeriod, bitrateMode,
+ bitrate, peakBitrate)) {
return nullptr;
}
return encoder;
@@ -161,6 +164,19 @@
return true;
}
+bool V4L2Encoder::setPeakBitrate(uint32_t peakBitrate) {
+ ALOGV("%s()", __func__);
+ ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence());
+
+ if (!mDevice->setExtCtrls(V4L2_CTRL_CLASS_MPEG,
+ {V4L2ExtCtrl(V4L2_CID_MPEG_VIDEO_BITRATE_PEAK, peakBitrate)})) {
+ // TODO(b/190336806): Our stack doesn't support dynamic peak bitrate changes yet, ignore
+ // errors for now.
+ ALOGW("Setting peak bitrate to %u failed", peakBitrate);
+ }
+ return true;
+}
+
bool V4L2Encoder::setFramerate(uint32_t framerate) {
ALOGV("%s()", __func__);
ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence());
@@ -189,8 +205,9 @@
}
bool V4L2Encoder::initialize(C2Config::profile_t outputProfile, std::optional<uint8_t> level,
- const ui::Size& visibleSize, uint32_t stride,
- uint32_t keyFramePeriod) {
+ const ui::Size& visibleSize, uint32_t stride, uint32_t keyFramePeriod,
+ C2Config::bitrate_mode_t bitrateMode, uint32_t bitrate,
+ std::optional<uint32_t> peakBitrate) {
ALOGV("%s()", __func__);
ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence());
ALOG_ASSERT(keyFramePeriod > 0);
@@ -238,6 +255,12 @@
return false;
}
+ // Configure the requested bitrate mode and bitrate on the device.
+ if (!configureBitrateMode(bitrateMode) || !setBitrate(bitrate)) return false;
+
+ // If the bitrate mode is VBR we also need to configure the peak bitrate on the device.
+ if ((bitrateMode == C2Config::BITRATE_VARIABLE) && !setPeakBitrate(*peakBitrate)) return false;
+
// First try to configure the specified output format, as changing the output format can affect
// the configured input format.
if (!configureOutputFormat(outputProfile)) return false;
@@ -594,17 +617,18 @@
std::optional<const uint8_t> outputH264Level) {
// When encoding H.264 we want to prepend SPS and PPS to each IDR for resilience. Some
// devices support this through the V4L2_CID_MPEG_VIDEO_PREPEND_SPSPPS_TO_IDR control.
- // TODO(b/161495502): V4L2_CID_MPEG_VIDEO_PREPEND_SPSPPS_TO_IDR is currently not supported
- // yet, just log a warning if the operation was unsuccessful for now.
+ // Otherwise we have to cache the latest SPS and PPS and inject these manually.
if (mDevice->isCtrlExposed(V4L2_CID_MPEG_VIDEO_PREPEND_SPSPPS_TO_IDR)) {
if (!mDevice->setExtCtrls(V4L2_CTRL_CLASS_MPEG,
{V4L2ExtCtrl(V4L2_CID_MPEG_VIDEO_PREPEND_SPSPPS_TO_IDR, 1)})) {
ALOGE("Failed to configure device to prepend SPS and PPS to each IDR");
return false;
}
+ mInjectParamsBeforeIDR = false;
ALOGV("Device supports prepending SPS and PPS to each IDR");
} else {
- ALOGW("Device doesn't support prepending SPS and PPS to IDR");
+ mInjectParamsBeforeIDR = true;
+ ALOGV("Device doesn't support prepending SPS and PPS to IDR, injecting manually.");
}
std::vector<V4L2ExtCtrl> h264Ctrls;
@@ -637,6 +661,21 @@
return true;
}
+bool V4L2Encoder::configureBitrateMode(C2Config::bitrate_mode_t bitrateMode) {
+ ALOGV("%s()", __func__);
+ ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence());
+
+ v4l2_mpeg_video_bitrate_mode v4l2BitrateMode =
+ V4L2Device::C2BitrateModeToV4L2BitrateMode(bitrateMode);
+ if (!mDevice->setExtCtrls(V4L2_CTRL_CLASS_MPEG,
+ {V4L2ExtCtrl(V4L2_CID_MPEG_VIDEO_BITRATE_MODE, v4l2BitrateMode)})) {
+ // TODO(b/190336806): Our stack doesn't support bitrate mode changes yet. We default to CBR
+ // which is currently the only supported mode so we can safely ignore this for now.
+ ALOGW("Setting bitrate mode to %u failed", v4l2BitrateMode);
+ }
+ return true;
+}
+
bool V4L2Encoder::startDevicePoll() {
ALOGV("%s()", __func__);
ALOG_ASSERT(mTaskRunner->RunsTasksInCurrentSequence());
@@ -780,7 +819,7 @@
size_t bufferId = buffer->bufferId();
std::vector<int> fds;
- fds.push_back(bitstreamBuffer->dmabuf_fd);
+ fds.push_back(bitstreamBuffer->dmabuf->handle()->data[0]);
if (!std::move(*buffer).queueDMABuf(fds)) {
ALOGE("Failed to queue output buffer using QueueDMABuf");
onError();
@@ -875,11 +914,51 @@
return false;
}
- std::unique_ptr<BitstreamBuffer> bitstream_buffer =
+ std::unique_ptr<BitstreamBuffer> bitstreamBuffer =
std::move(mOutputBuffers[buffer->bufferId()]);
if (encodedDataSize > 0) {
- mOutputBufferDoneCb.Run(encodedDataSize, timestamp.InMicroseconds(), buffer->isKeyframe(),
- std::move(bitstream_buffer));
+ if (!mInjectParamsBeforeIDR) {
+ // No need to inject SPS or PPS before IDR frames, we can just return the buffer as-is.
+ mOutputBufferDoneCb.Run(encodedDataSize, timestamp.InMicroseconds(),
+ buffer->isKeyframe(), std::move(bitstreamBuffer));
+ } else if (!buffer->isKeyframe()) {
+ // We need to inject SPS and PPS before IDR frames, but this frame is not a key frame.
+ // We can return the buffer as-is, but need to update our SPS and PPS cache if required.
+ C2ConstLinearBlock constBlock = bitstreamBuffer->dmabuf->share(
+ bitstreamBuffer->dmabuf->offset(), encodedDataSize, C2Fence());
+ C2ReadView readView = constBlock.map().get();
+ extractSPSPPS(readView.data(), encodedDataSize, &mCachedSPS, &mCachedPPS);
+ mOutputBufferDoneCb.Run(encodedDataSize, timestamp.InMicroseconds(),
+ buffer->isKeyframe(), std::move(bitstreamBuffer));
+ } else {
+ // We need to inject our cached SPS and PPS NAL units to the IDR frame. It's possible
+ // this frame already has SPS and PPS NAL units attached, in which case we only need to
+ // update our cached SPS and PPS.
+ C2ConstLinearBlock constBlock = bitstreamBuffer->dmabuf->share(
+ bitstreamBuffer->dmabuf->offset(), encodedDataSize, C2Fence());
+ C2ReadView readView = constBlock.map().get();
+
+ // Allocate a new buffer to copy the data with prepended SPS and PPS into.
+ std::unique_ptr<BitstreamBuffer> prependedBitstreamBuffer;
+ mFetchOutputBufferCb.Run(mOutputBufferSize, &prependedBitstreamBuffer);
+ if (!prependedBitstreamBuffer) {
+ ALOGE("Failed to fetch output block");
+ onError();
+ return false;
+ }
+ C2WriteView writeView = prependedBitstreamBuffer->dmabuf->map().get();
+
+ // If there is not enough space in the output buffer just return the original buffer.
+ size_t newSize = prependSPSPPSToIDR(readView.data(), encodedDataSize, writeView.data(),
+ writeView.size(), &mCachedSPS, &mCachedPPS);
+ if (newSize > 0) {
+ mOutputBufferDoneCb.Run(newSize, timestamp.InMicroseconds(), buffer->isKeyframe(),
+ std::move(prependedBitstreamBuffer));
+ } else {
+ mOutputBufferDoneCb.Run(encodedDataSize, timestamp.InMicroseconds(),
+ buffer->isKeyframe(), std::move(bitstreamBuffer));
+ }
+ }
}
// If the buffer is marked as last and we were flushing the encoder, flushing is now done.
diff --git a/components/include/v4l2_codec2/components/BitstreamBuffer.h b/components/include/v4l2_codec2/components/BitstreamBuffer.h
index d61e4f9..97cb203 100644
--- a/components/include/v4l2_codec2/components/BitstreamBuffer.h
+++ b/components/include/v4l2_codec2/components/BitstreamBuffer.h
@@ -7,18 +7,30 @@
#include <stdint.h>
+#include <C2Buffer.h>
+
namespace android {
-// The BitstreamBuffer class can be used to store encoded video data.
-// Note: The BitstreamBuffer does not take ownership of the data. The file descriptor is not
-// duplicated and the caller is responsible for keeping the data alive.
-struct BitstreamBuffer {
- BitstreamBuffer(const int32_t id, int dmabuf_fd, const size_t offset, const size_t size)
- : id(id), dmabuf_fd(dmabuf_fd), offset(offset), size(size) {}
- ~BitstreamBuffer() = default;
+// The ConstBitstreamBuffer class can be used to store non-modifiable encoded video data.
+struct ConstBitstreamBuffer {
+ ConstBitstreamBuffer(const int32_t id, C2ConstLinearBlock dmabuf, const size_t offset,
+ const size_t size)
+ : id(id), dmabuf(std::move(dmabuf)), offset(offset), size(size) {}
+ ~ConstBitstreamBuffer() = default;
const int32_t id;
- int dmabuf_fd;
+ C2ConstLinearBlock dmabuf;
+ const size_t offset;
+ const size_t size;
+};
+
+// The BitstreamBuffer class can be used to store modifiable encoded video data.
+struct BitstreamBuffer {
+ BitstreamBuffer(std::shared_ptr<C2LinearBlock> dmabuf, const size_t offset, const size_t size)
+ : dmabuf(std::move(dmabuf)), offset(offset), size(size) {}
+ ~BitstreamBuffer() = default;
+
+ std::shared_ptr<C2LinearBlock> dmabuf;
const size_t offset;
const size_t size;
};
diff --git a/components/include/v4l2_codec2/components/V4L2DecodeComponent.h b/components/include/v4l2_codec2/components/V4L2DecodeComponent.h
index 1e98118..962f7d6 100644
--- a/components/include/v4l2_codec2/components/V4L2DecodeComponent.h
+++ b/components/include/v4l2_codec2/components/V4L2DecodeComponent.h
@@ -140,9 +140,6 @@
::base::Thread mDecoderThread{"V4L2DecodeComponentDecoderThread"};
scoped_refptr<::base::SequencedTaskRunner> mDecoderTaskRunner;
- // Hold a weak_ptr of |*this| when |mDecoderThread| is running.
- std::weak_ptr<V4L2DecodeComponent> mStdWeakThis;
-
::base::WeakPtrFactory<V4L2DecodeComponent> mWeakThisFactory{this};
::base::WeakPtr<V4L2DecodeComponent> mWeakThis;
};
diff --git a/components/include/v4l2_codec2/components/V4L2Decoder.h b/components/include/v4l2_codec2/components/V4L2Decoder.h
index 12c2768..2ecb3bd 100644
--- a/components/include/v4l2_codec2/components/V4L2Decoder.h
+++ b/components/include/v4l2_codec2/components/V4L2Decoder.h
@@ -31,7 +31,7 @@
scoped_refptr<::base::SequencedTaskRunner> taskRunner);
~V4L2Decoder() override;
- void decode(std::unique_ptr<BitstreamBuffer> buffer, DecodeCB decodeCb) override;
+ void decode(std::unique_ptr<ConstBitstreamBuffer> buffer, DecodeCB decodeCb) override;
void drain(DecodeCB drainCb) override;
void flush() override;
@@ -45,12 +45,12 @@
static const char* StateToString(State state);
struct DecodeRequest {
- DecodeRequest(std::unique_ptr<BitstreamBuffer> buffer, DecodeCB decodeCb)
+ DecodeRequest(std::unique_ptr<ConstBitstreamBuffer> buffer, DecodeCB decodeCb)
: buffer(std::move(buffer)), decodeCb(std::move(decodeCb)) {}
DecodeRequest(DecodeRequest&&) = default;
~DecodeRequest() = default;
- std::unique_ptr<BitstreamBuffer> buffer; // nullptr means Drain
+ std::unique_ptr<ConstBitstreamBuffer> buffer; // nullptr means Drain
DecodeCB decodeCb;
};
diff --git a/components/include/v4l2_codec2/components/V4L2EncodeComponent.h b/components/include/v4l2_codec2/components/V4L2EncodeComponent.h
index 4665ffa..0b150e4 100644
--- a/components/include/v4l2_codec2/components/V4L2EncodeComponent.h
+++ b/components/include/v4l2_codec2/components/V4L2EncodeComponent.h
@@ -155,18 +155,19 @@
// The bitrate currently configured on the v4l2 device.
uint32_t mBitrate = 0;
+ // The bitrate mode currently configured on the v4l2 device.
+ C2Config::bitrate_mode_t mBitrateMode = C2Config::BITRATE_CONST;
// The framerate currently configured on the v4l2 device.
uint32_t mFramerate = 0;
+ // The timestamp of the last frame encoded, used to dynamically adjust the framerate.
+ std::optional<int64_t> mLastFrameTime;
- // Whether we extracted and submitted CSD (codec-specific data, e.g. H.264 SPS) to the framework.
- bool mCSDSubmitted = false;
+ // Whether we need to extract and submit CSD (codec-specific data, e.g. H.264 SPS).
+ bool mExtractCSD = false;
// The queue of encode work items currently being processed.
std::deque<std::unique_ptr<C2Work>> mWorkQueue;
- // Map of buffer ids and associated C2LinearBlock buffers. The buffer's fds are used as id.
- std::unordered_map<int32_t, std::shared_ptr<C2LinearBlock>> mOutputBuffersMap;
-
// The output block pool.
std::shared_ptr<C2BlockPool> mOutputBlockPool;
diff --git a/components/include/v4l2_codec2/components/V4L2EncodeInterface.h b/components/include/v4l2_codec2/components/V4L2EncodeInterface.h
index 2efbfcc..fefebf0 100644
--- a/components/include/v4l2_codec2/components/V4L2EncodeInterface.h
+++ b/components/include/v4l2_codec2/components/V4L2EncodeInterface.h
@@ -39,8 +39,18 @@
return ui::Size(mInputVisibleSize->width, mInputVisibleSize->height);
}
C2BlockPool::local_id_t getBlockPoolId() const { return mOutputBlockPoolIds->m.values[0]; }
+
// Get sync key-frame period in frames.
uint32_t getKeyFramePeriod() const;
+ // Get the requested bitrate mode.
+ C2Config::bitrate_mode_t getBitrateMode() const { return mBitrateMode->value; }
+ // Get the requested bitrate.
+ uint32_t getBitrate() const { return mBitrate->value; }
+ // Get the requested framerate.
+ float getFramerate() const { return mFrameRate->value; }
+
+ // Request changing the framerate to the specified value.
+ void setFramerate(uint32_t framerate) { mFrameRate->value = framerate; }
protected:
void Initialize(const C2String& name);
@@ -95,6 +105,8 @@
// The requested bitrate of the encoded output stream, in bits per second.
std::shared_ptr<C2StreamBitrateInfo::output> mBitrate;
+ // The requested bitrate mode.
+ std::shared_ptr<C2StreamBitrateModeTuning::output> mBitrateMode;
// The requested framerate, in frames per second.
std::shared_ptr<C2StreamFrameRateInfo::output> mFrameRate;
// The switch-type parameter that will be set to true while client requests keyframe. It
diff --git a/components/include/v4l2_codec2/components/V4L2Encoder.h b/components/include/v4l2_codec2/components/V4L2Encoder.h
index 5abee8f..d7b55c0 100644
--- a/components/include/v4l2_codec2/components/V4L2Encoder.h
+++ b/components/include/v4l2_codec2/components/V4L2Encoder.h
@@ -33,9 +33,10 @@
static std::unique_ptr<VideoEncoder> create(
C2Config::profile_t profile, std::optional<uint8_t> level, const ui::Size& visibleSize,
- uint32_t stride, uint32_t keyFramePeriod, FetchOutputBufferCB fetchOutputBufferCb,
- InputBufferDoneCB inputBufferDoneCb, OutputBufferDoneCB outputBufferDoneCb,
- DrainDoneCB drainDoneCb, ErrorCB errorCb,
+ uint32_t stride, uint32_t keyFramePeriod, C2Config::bitrate_mode_t bitrateMode,
+ uint32_t bitrate, std::optional<uint32_t> peakBitrate,
+ FetchOutputBufferCB fetchOutputBufferCb, InputBufferDoneCB inputBufferDoneCb,
+ OutputBufferDoneCB outputBufferDoneCb, DrainDoneCB drainDoneCb, ErrorCB errorCb,
scoped_refptr<::base::SequencedTaskRunner> taskRunner);
~V4L2Encoder() override;
@@ -44,6 +45,7 @@
void flush() override;
bool setBitrate(uint32_t bitrate) override;
+ bool setPeakBitrate(uint32_t peakBitrate) override;
bool setFramerate(uint32_t framerate) override;
void requestKeyframe() override;
@@ -80,7 +82,9 @@
// Initialize the V4L2 encoder for specified parameters.
bool initialize(C2Config::profile_t outputProfile, std::optional<uint8_t> level,
- const ui::Size& visibleSize, uint32_t stride, uint32_t keyFramePeriod);
+ const ui::Size& visibleSize, uint32_t stride, uint32_t keyFramePeriod,
+ C2Config::bitrate_mode_t bitrateMode, uint32_t bitrate,
+ std::optional<uint32_t> peakBitrate);
// Handle the next encode request on the queue.
void handleEncodeRequest();
@@ -101,6 +105,8 @@
// Configure required and optional H.264 controls on the V4L2 device.
bool configureH264(C2Config::profile_t outputProfile,
std::optional<const uint8_t> outputH264Level);
+ // Configure the specified bitrate mode on the V4L2 device.
+ bool configureBitrateMode(C2Config::bitrate_mode_t bitrateMode);
// Attempt to start the V4L2 device poller.
bool startDevicePoll();
@@ -158,6 +164,12 @@
// Key frame counter, a key frame will be requested each time it reaches zero.
uint32_t mKeyFrameCounter = 0;
+ // Whether we need to manually cache and prepend SPS and PPS to IDR frames.
+ bool mInjectParamsBeforeIDR = false;
+ // The latest cached SPS and PPS (without H.264 start code).
+ std::vector<uint8_t> mCachedSPS;
+ std::vector<uint8_t> mCachedPPS;
+
// The V4L2 device and associated queues used to interact with the device.
scoped_refptr<V4L2Device> mDevice;
scoped_refptr<V4L2Queue> mInputQueue;
diff --git a/components/include/v4l2_codec2/components/VideoDecoder.h b/components/include/v4l2_codec2/components/VideoDecoder.h
index 9a48562..5b2da41 100644
--- a/components/include/v4l2_codec2/components/VideoDecoder.h
+++ b/components/include/v4l2_codec2/components/VideoDecoder.h
@@ -34,7 +34,7 @@
virtual ~VideoDecoder();
- virtual void decode(std::unique_ptr<BitstreamBuffer> buffer, DecodeCB decodeCb) = 0;
+ virtual void decode(std::unique_ptr<ConstBitstreamBuffer> buffer, DecodeCB decodeCb) = 0;
virtual void drain(DecodeCB drainCb) = 0;
virtual void flush() = 0;
};
diff --git a/components/include/v4l2_codec2/components/VideoEncoder.h b/components/include/v4l2_codec2/components/VideoEncoder.h
index 46bcad1..5f23541 100644
--- a/components/include/v4l2_codec2/components/VideoEncoder.h
+++ b/components/include/v4l2_codec2/components/VideoEncoder.h
@@ -64,8 +64,12 @@
// Flush the encoder, pending drain operations will be aborted.
virtual void flush() = 0;
- // Set the bitrate to the specified value, will affect all non-processed frames.
+ // Set the target bitrate to the specified value, will affect all non-processed frames.
virtual bool setBitrate(uint32_t bitrate) = 0;
+ // Set the peak bitrate to the specified value. The peak bitrate must be larger or equal to the
+ // target bitrate and is ignored if the bitrate mode is constant.
+ virtual bool setPeakBitrate(uint32_t peakBitrate) = 0;
+
// Set the framerate to the specified value, will affect all non-processed frames.
virtual bool setFramerate(uint32_t framerate) = 0;
// Request the next frame encoded to be a key frame, will affect the next non-processed frame.