mflodman | 351424e | 2017-08-10 02:43:14 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 11 | #include "api/video_codecs/video_encoder.h" |
mflodman | 351424e | 2017-08-10 02:43:14 -0700 | [diff] [blame] | 12 | |
Yves Gerey | 988cc08 | 2018-10-23 12:03:01 +0200 | [diff] [blame] | 13 | #include <string.h> |
Henrik Boström | b0f2e0c | 2020-03-06 13:32:03 +0100 | [diff] [blame^] | 14 | #include <algorithm> |
Yves Gerey | 988cc08 | 2018-10-23 12:03:01 +0200 | [diff] [blame] | 15 | |
| 16 | #include "rtc_base/checks.h" |
Erik Språng | 7968530 | 2019-11-27 17:26:58 +0100 | [diff] [blame] | 17 | #include "rtc_base/strings/string_builder.h" |
Yves Gerey | 988cc08 | 2018-10-23 12:03:01 +0200 | [diff] [blame] | 18 | |
mflodman | 351424e | 2017-08-10 02:43:14 -0700 | [diff] [blame] | 19 | namespace webrtc { |
| 20 | |
| 21 | // TODO(mflodman): Add default complexity for VP9 and VP9. |
| 22 | VideoCodecVP8 VideoEncoder::GetDefaultVp8Settings() { |
| 23 | VideoCodecVP8 vp8_settings; |
| 24 | memset(&vp8_settings, 0, sizeof(vp8_settings)); |
| 25 | |
mflodman | 351424e | 2017-08-10 02:43:14 -0700 | [diff] [blame] | 26 | vp8_settings.numberOfTemporalLayers = 1; |
| 27 | vp8_settings.denoisingOn = true; |
mflodman | 351424e | 2017-08-10 02:43:14 -0700 | [diff] [blame] | 28 | vp8_settings.automaticResizeOn = false; |
| 29 | vp8_settings.frameDroppingOn = true; |
| 30 | vp8_settings.keyFrameInterval = 3000; |
| 31 | |
| 32 | return vp8_settings; |
| 33 | } |
| 34 | |
| 35 | VideoCodecVP9 VideoEncoder::GetDefaultVp9Settings() { |
| 36 | VideoCodecVP9 vp9_settings; |
| 37 | memset(&vp9_settings, 0, sizeof(vp9_settings)); |
| 38 | |
mflodman | 351424e | 2017-08-10 02:43:14 -0700 | [diff] [blame] | 39 | vp9_settings.numberOfTemporalLayers = 1; |
| 40 | vp9_settings.denoisingOn = true; |
| 41 | vp9_settings.frameDroppingOn = true; |
| 42 | vp9_settings.keyFrameInterval = 3000; |
| 43 | vp9_settings.adaptiveQpMode = true; |
| 44 | vp9_settings.automaticResizeOn = true; |
| 45 | vp9_settings.numberOfSpatialLayers = 1; |
| 46 | vp9_settings.flexibleMode = false; |
Sergey Silkin | 6a8f30e | 2018-04-26 11:03:49 +0200 | [diff] [blame] | 47 | vp9_settings.interLayerPred = InterLayerPredMode::kOn; |
mflodman | 351424e | 2017-08-10 02:43:14 -0700 | [diff] [blame] | 48 | |
| 49 | return vp9_settings; |
| 50 | } |
| 51 | |
| 52 | VideoCodecH264 VideoEncoder::GetDefaultH264Settings() { |
| 53 | VideoCodecH264 h264_settings; |
| 54 | memset(&h264_settings, 0, sizeof(h264_settings)); |
| 55 | |
| 56 | h264_settings.frameDroppingOn = true; |
| 57 | h264_settings.keyFrameInterval = 3000; |
Johnny Lee | 1a1c52b | 2019-02-08 14:25:40 -0500 | [diff] [blame] | 58 | h264_settings.numberOfTemporalLayers = 1; |
mflodman | 351424e | 2017-08-10 02:43:14 -0700 | [diff] [blame] | 59 | |
| 60 | return h264_settings; |
| 61 | } |
| 62 | |
Niels Möller | 225c787 | 2018-02-22 15:03:53 +0100 | [diff] [blame] | 63 | VideoEncoder::ScalingSettings::ScalingSettings() = default; |
mflodman | 351424e | 2017-08-10 02:43:14 -0700 | [diff] [blame] | 64 | |
Niels Möller | 225c787 | 2018-02-22 15:03:53 +0100 | [diff] [blame] | 65 | VideoEncoder::ScalingSettings::ScalingSettings(KOff) : ScalingSettings() {} |
| 66 | |
| 67 | VideoEncoder::ScalingSettings::ScalingSettings(int low, int high) |
| 68 | : thresholds(QpThresholds(low, high)) {} |
| 69 | |
| 70 | VideoEncoder::ScalingSettings::ScalingSettings(int low, |
asapersson | 142fcc9 | 2017-08-17 08:58:54 -0700 | [diff] [blame] | 71 | int high, |
| 72 | int min_pixels) |
Niels Möller | 225c787 | 2018-02-22 15:03:53 +0100 | [diff] [blame] | 73 | : thresholds(QpThresholds(low, high)), min_pixels_per_frame(min_pixels) {} |
asapersson | 142fcc9 | 2017-08-17 08:58:54 -0700 | [diff] [blame] | 74 | |
Niels Möller | 225c787 | 2018-02-22 15:03:53 +0100 | [diff] [blame] | 75 | VideoEncoder::ScalingSettings::ScalingSettings(const ScalingSettings&) = |
| 76 | default; |
mflodman | 351424e | 2017-08-10 02:43:14 -0700 | [diff] [blame] | 77 | |
| 78 | VideoEncoder::ScalingSettings::~ScalingSettings() {} |
| 79 | |
Niels Möller | 225c787 | 2018-02-22 15:03:53 +0100 | [diff] [blame] | 80 | // static |
| 81 | constexpr VideoEncoder::ScalingSettings::KOff |
| 82 | VideoEncoder::ScalingSettings::kOff; |
Erik Språng | dbdd839 | 2019-01-17 15:27:50 +0100 | [diff] [blame] | 83 | // static |
| 84 | constexpr uint8_t VideoEncoder::EncoderInfo::kMaxFramerateFraction; |
mflodman | 351424e | 2017-08-10 02:43:14 -0700 | [diff] [blame] | 85 | |
Erik Språng | 7968530 | 2019-11-27 17:26:58 +0100 | [diff] [blame] | 86 | bool VideoEncoder::ResolutionBitrateLimits::operator==( |
| 87 | const ResolutionBitrateLimits& rhs) const { |
| 88 | return frame_size_pixels == rhs.frame_size_pixels && |
| 89 | min_start_bitrate_bps == rhs.min_start_bitrate_bps && |
| 90 | min_bitrate_bps == rhs.min_bitrate_bps && |
| 91 | max_bitrate_bps == rhs.max_bitrate_bps; |
| 92 | } |
| 93 | |
Erik Språng | e2fd86a | 2018-10-24 11:32:39 +0200 | [diff] [blame] | 94 | VideoEncoder::EncoderInfo::EncoderInfo() |
| 95 | : scaling_settings(VideoEncoder::ScalingSettings::kOff), |
Rasmus Brandt | 5cad55b | 2019-12-19 09:47:11 +0100 | [diff] [blame] | 96 | requested_resolution_alignment(1), |
Erik Språng | e2fd86a | 2018-10-24 11:32:39 +0200 | [diff] [blame] | 97 | supports_native_handle(false), |
Erik Språng | d3438aa | 2018-11-08 16:56:43 +0100 | [diff] [blame] | 98 | implementation_name("unknown"), |
Mirta Dvornicic | 897a991 | 2018-11-30 13:12:21 +0100 | [diff] [blame] | 99 | has_trusted_rate_controller(false), |
Mirta Dvornicic | ccc1b57 | 2019-01-15 12:42:18 +0100 | [diff] [blame] | 100 | is_hardware_accelerated(true), |
Erik Språng | dbdd839 | 2019-01-17 15:27:50 +0100 | [diff] [blame] | 101 | has_internal_source(false), |
| 102 | fps_allocation{absl::InlinedVector<uint8_t, kMaxTemporalStreams>( |
| 103 | 1, |
Erik Språng | f4e0c29 | 2019-10-01 18:50:03 +0200 | [diff] [blame] | 104 | kMaxFramerateFraction)}, |
| 105 | supports_simulcast(false) {} |
Mirta Dvornicic | 897a991 | 2018-11-30 13:12:21 +0100 | [diff] [blame] | 106 | |
| 107 | VideoEncoder::EncoderInfo::EncoderInfo(const EncoderInfo&) = default; |
Erik Språng | e2fd86a | 2018-10-24 11:32:39 +0200 | [diff] [blame] | 108 | |
Erik Språng | e2fd86a | 2018-10-24 11:32:39 +0200 | [diff] [blame] | 109 | VideoEncoder::EncoderInfo::~EncoderInfo() = default; |
| 110 | |
Erik Språng | 7968530 | 2019-11-27 17:26:58 +0100 | [diff] [blame] | 111 | std::string VideoEncoder::EncoderInfo::ToString() const { |
| 112 | char string_buf[2048]; |
| 113 | rtc::SimpleStringBuilder oss(string_buf); |
| 114 | |
| 115 | oss << "EncoderInfo { " |
Jonas Olsson | b2b2031 | 2020-01-14 12:11:31 +0100 | [diff] [blame] | 116 | "ScalingSettings { "; |
Erik Språng | 7968530 | 2019-11-27 17:26:58 +0100 | [diff] [blame] | 117 | if (scaling_settings.thresholds) { |
| 118 | oss << "Thresholds { " |
Jonas Olsson | b2b2031 | 2020-01-14 12:11:31 +0100 | [diff] [blame] | 119 | "low = " |
| 120 | << scaling_settings.thresholds->low |
Erik Språng | 7968530 | 2019-11-27 17:26:58 +0100 | [diff] [blame] | 121 | << ", high = " << scaling_settings.thresholds->high << "}, "; |
| 122 | } |
| 123 | oss << "min_pixels_per_frame = " << scaling_settings.min_pixels_per_frame |
| 124 | << " }"; |
Rasmus Brandt | 5cad55b | 2019-12-19 09:47:11 +0100 | [diff] [blame] | 125 | oss << ", requested_resolution_alignment = " << requested_resolution_alignment |
| 126 | << ", supports_native_handle = " << supports_native_handle |
Jonas Olsson | b2b2031 | 2020-01-14 12:11:31 +0100 | [diff] [blame] | 127 | << ", implementation_name = '" << implementation_name |
| 128 | << "'" |
| 129 | ", has_trusted_rate_controller = " |
| 130 | << has_trusted_rate_controller |
Erik Språng | 7968530 | 2019-11-27 17:26:58 +0100 | [diff] [blame] | 131 | << ", is_hardware_accelerated = " << is_hardware_accelerated |
| 132 | << ", has_internal_source = " << has_internal_source |
| 133 | << ", fps_allocation = ["; |
| 134 | bool first = true; |
| 135 | for (size_t i = 0; i < fps_allocation->size(); ++i) { |
| 136 | if (!first) { |
| 137 | oss << ", "; |
| 138 | } |
| 139 | const absl::InlinedVector<uint8_t, kMaxTemporalStreams>& fractions = |
| 140 | fps_allocation[i]; |
| 141 | if (!fractions.empty()) { |
| 142 | first = false; |
| 143 | oss << "[ "; |
| 144 | for (size_t i = 0; i < fractions.size(); ++i) { |
| 145 | if (i > 0) { |
| 146 | oss << ", "; |
| 147 | } |
| 148 | oss << (static_cast<double>(fractions[i]) / kMaxFramerateFraction); |
| 149 | } |
| 150 | oss << "] "; |
| 151 | } |
| 152 | } |
| 153 | oss << "]"; |
| 154 | oss << ", resolution_bitrate_limits = ["; |
| 155 | for (size_t i = 0; i < resolution_bitrate_limits.size(); ++i) { |
| 156 | if (i > 0) { |
| 157 | oss << ", "; |
| 158 | } |
| 159 | ResolutionBitrateLimits l = resolution_bitrate_limits[i]; |
| 160 | oss << "Limits { " |
Jonas Olsson | b2b2031 | 2020-01-14 12:11:31 +0100 | [diff] [blame] | 161 | "frame_size_pixels = " |
| 162 | << l.frame_size_pixels |
Erik Språng | 7968530 | 2019-11-27 17:26:58 +0100 | [diff] [blame] | 163 | << ", min_start_bitrate_bps = " << l.min_start_bitrate_bps |
| 164 | << ", min_bitrate_bps = " << l.min_bitrate_bps |
| 165 | << ", max_bitrate_bps = " << l.max_bitrate_bps << "} "; |
| 166 | } |
| 167 | oss << "] " |
Jonas Olsson | b2b2031 | 2020-01-14 12:11:31 +0100 | [diff] [blame] | 168 | ", supports_simulcast = " |
| 169 | << supports_simulcast << "}"; |
Erik Språng | 7968530 | 2019-11-27 17:26:58 +0100 | [diff] [blame] | 170 | return oss.str(); |
| 171 | } |
| 172 | |
| 173 | bool VideoEncoder::EncoderInfo::operator==(const EncoderInfo& rhs) const { |
| 174 | if (scaling_settings.thresholds.has_value() != |
| 175 | rhs.scaling_settings.thresholds.has_value()) { |
| 176 | return false; |
| 177 | } |
| 178 | if (scaling_settings.thresholds.has_value()) { |
| 179 | QpThresholds l = *scaling_settings.thresholds; |
| 180 | QpThresholds r = *rhs.scaling_settings.thresholds; |
| 181 | if (l.low != r.low || l.high != r.high) { |
| 182 | return false; |
| 183 | } |
| 184 | } |
| 185 | if (scaling_settings.min_pixels_per_frame != |
| 186 | rhs.scaling_settings.min_pixels_per_frame) { |
| 187 | return false; |
| 188 | } |
| 189 | |
| 190 | if (supports_native_handle != rhs.supports_native_handle || |
| 191 | implementation_name != rhs.implementation_name || |
| 192 | has_trusted_rate_controller != rhs.has_trusted_rate_controller || |
| 193 | is_hardware_accelerated != rhs.is_hardware_accelerated || |
| 194 | has_internal_source != rhs.has_internal_source) { |
| 195 | return false; |
| 196 | } |
| 197 | |
| 198 | for (size_t i = 0; i < kMaxSpatialLayers; ++i) { |
| 199 | if (fps_allocation[i] != rhs.fps_allocation[i]) { |
| 200 | return false; |
| 201 | } |
| 202 | } |
| 203 | |
| 204 | if (resolution_bitrate_limits != rhs.resolution_bitrate_limits || |
| 205 | supports_simulcast != rhs.supports_simulcast) { |
| 206 | return false; |
| 207 | } |
| 208 | |
| 209 | return true; |
| 210 | } |
| 211 | |
Henrik Boström | b0f2e0c | 2020-03-06 13:32:03 +0100 | [diff] [blame^] | 212 | absl::optional<VideoEncoder::ResolutionBitrateLimits> |
| 213 | VideoEncoder::EncoderInfo::GetEncoderBitrateLimitsForResolution( |
| 214 | int frame_size_pixels) const { |
| 215 | std::vector<ResolutionBitrateLimits> bitrate_limits = |
| 216 | resolution_bitrate_limits; |
| 217 | |
| 218 | // Sort the list of bitrate limits by resolution. |
| 219 | sort(bitrate_limits.begin(), bitrate_limits.end(), |
| 220 | [](const ResolutionBitrateLimits& lhs, |
| 221 | const ResolutionBitrateLimits& rhs) { |
| 222 | return lhs.frame_size_pixels < rhs.frame_size_pixels; |
| 223 | }); |
| 224 | |
| 225 | for (size_t i = 0; i < bitrate_limits.size(); ++i) { |
| 226 | RTC_DCHECK_GE(bitrate_limits[i].min_bitrate_bps, 0); |
| 227 | RTC_DCHECK_GE(bitrate_limits[i].min_start_bitrate_bps, 0); |
| 228 | RTC_DCHECK_GE(bitrate_limits[i].max_bitrate_bps, |
| 229 | bitrate_limits[i].min_bitrate_bps); |
| 230 | if (i > 0) { |
| 231 | // The bitrate limits aren't expected to decrease with resolution. |
| 232 | RTC_DCHECK_GE(bitrate_limits[i].min_bitrate_bps, |
| 233 | bitrate_limits[i - 1].min_bitrate_bps); |
| 234 | RTC_DCHECK_GE(bitrate_limits[i].min_start_bitrate_bps, |
| 235 | bitrate_limits[i - 1].min_start_bitrate_bps); |
| 236 | RTC_DCHECK_GE(bitrate_limits[i].max_bitrate_bps, |
| 237 | bitrate_limits[i - 1].max_bitrate_bps); |
| 238 | } |
| 239 | |
| 240 | if (bitrate_limits[i].frame_size_pixels >= frame_size_pixels) { |
| 241 | return absl::optional<ResolutionBitrateLimits>(bitrate_limits[i]); |
| 242 | } |
| 243 | } |
| 244 | |
| 245 | return absl::nullopt; |
| 246 | } |
| 247 | |
Erik Språng | 4c6ca30 | 2019-04-08 15:14:01 +0200 | [diff] [blame] | 248 | VideoEncoder::RateControlParameters::RateControlParameters() |
| 249 | : bitrate(VideoBitrateAllocation()), |
| 250 | framerate_fps(0.0), |
| 251 | bandwidth_allocation(DataRate::Zero()) {} |
| 252 | |
| 253 | VideoEncoder::RateControlParameters::RateControlParameters( |
| 254 | const VideoBitrateAllocation& bitrate, |
Erik Språng | 16cb8f5 | 2019-04-12 13:59:09 +0200 | [diff] [blame] | 255 | double framerate_fps) |
| 256 | : bitrate(bitrate), |
| 257 | framerate_fps(framerate_fps), |
Danil Chapovalov | cad3e0e | 2020-02-17 18:46:07 +0100 | [diff] [blame] | 258 | bandwidth_allocation(DataRate::BitsPerSec(bitrate.get_sum_bps())) {} |
Erik Språng | 16cb8f5 | 2019-04-12 13:59:09 +0200 | [diff] [blame] | 259 | |
| 260 | VideoEncoder::RateControlParameters::RateControlParameters( |
| 261 | const VideoBitrateAllocation& bitrate, |
Erik Språng | 4c6ca30 | 2019-04-08 15:14:01 +0200 | [diff] [blame] | 262 | double framerate_fps, |
| 263 | DataRate bandwidth_allocation) |
| 264 | : bitrate(bitrate), |
| 265 | framerate_fps(framerate_fps), |
| 266 | bandwidth_allocation(bandwidth_allocation) {} |
| 267 | |
Evan Shrubsole | 7c079f6 | 2019-09-26 09:55:03 +0200 | [diff] [blame] | 268 | bool VideoEncoder::RateControlParameters::operator==( |
| 269 | const VideoEncoder::RateControlParameters& rhs) const { |
| 270 | return std::tie(bitrate, framerate_fps, bandwidth_allocation) == |
| 271 | std::tie(rhs.bitrate, rhs.framerate_fps, rhs.bandwidth_allocation); |
| 272 | } |
| 273 | |
| 274 | bool VideoEncoder::RateControlParameters::operator!=( |
| 275 | const VideoEncoder::RateControlParameters& rhs) const { |
| 276 | return !(rhs == *this); |
| 277 | } |
| 278 | |
Erik Språng | 4c6ca30 | 2019-04-08 15:14:01 +0200 | [diff] [blame] | 279 | VideoEncoder::RateControlParameters::~RateControlParameters() = default; |
| 280 | |
Elad Alon | 8f01c4e | 2019-06-28 15:19:43 +0200 | [diff] [blame] | 281 | void VideoEncoder::SetFecControllerOverride( |
| 282 | FecControllerOverride* fec_controller_override) {} |
| 283 | |
Elad Alon | 370f93a | 2019-06-11 14:57:57 +0200 | [diff] [blame] | 284 | int32_t VideoEncoder::InitEncode(const VideoCodec* codec_settings, |
| 285 | int32_t number_of_cores, |
| 286 | size_t max_payload_size) { |
| 287 | const VideoEncoder::Capabilities capabilities(/* loss_notification= */ false); |
| 288 | const VideoEncoder::Settings settings(capabilities, number_of_cores, |
| 289 | max_payload_size); |
| 290 | // In theory, this and the other version of InitEncode() could end up calling |
| 291 | // each other in a loop until we get a stack overflow. |
| 292 | // In practice, any subclass of VideoEncoder would overload at least one |
| 293 | // of these, and we have a TODO in the header file to make this pure virtual. |
| 294 | return InitEncode(codec_settings, settings); |
| 295 | } |
| 296 | |
| 297 | int VideoEncoder::InitEncode(const VideoCodec* codec_settings, |
| 298 | const VideoEncoder::Settings& settings) { |
| 299 | // In theory, this and the other version of InitEncode() could end up calling |
| 300 | // each other in a loop until we get a stack overflow. |
| 301 | // In practice, any subclass of VideoEncoder would overload at least one |
| 302 | // of these, and we have a TODO in the header file to make this pure virtual. |
| 303 | return InitEncode(codec_settings, settings.number_of_cores, |
| 304 | settings.max_payload_size); |
| 305 | } |
| 306 | |
Elad Alon | cde8ab2 | 2019-03-20 11:56:20 +0100 | [diff] [blame] | 307 | void VideoEncoder::OnPacketLossRateUpdate(float packet_loss_rate) {} |
| 308 | |
| 309 | void VideoEncoder::OnRttUpdate(int64_t rtt_ms) {} |
| 310 | |
Elad Alon | 6c371ca | 2019-04-04 12:28:51 +0200 | [diff] [blame] | 311 | void VideoEncoder::OnLossNotification( |
| 312 | const LossNotification& loss_notification) {} |
| 313 | |
Erik Språng | 6ed4f14 | 2018-11-26 13:42:39 +0100 | [diff] [blame] | 314 | // TODO(webrtc:9722): Remove and make pure virtual. |
Erik Språng | e2fd86a | 2018-10-24 11:32:39 +0200 | [diff] [blame] | 315 | VideoEncoder::EncoderInfo VideoEncoder::GetEncoderInfo() const { |
Erik Språng | 6ed4f14 | 2018-11-26 13:42:39 +0100 | [diff] [blame] | 316 | return EncoderInfo(); |
Erik Språng | e2fd86a | 2018-10-24 11:32:39 +0200 | [diff] [blame] | 317 | } |
Erik Språng | 6ed4f14 | 2018-11-26 13:42:39 +0100 | [diff] [blame] | 318 | |
mflodman | 351424e | 2017-08-10 02:43:14 -0700 | [diff] [blame] | 319 | } // namespace webrtc |