Henrik Boström | 382cc6d | 2020-01-07 10:15:04 +0100 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2019 The WebRTC Project Authors. All rights reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
| 11 | #ifndef CALL_ADAPTATION_RESOURCE_ADAPTATION_MODULE_INTERFACE_H_ |
| 12 | #define CALL_ADAPTATION_RESOURCE_ADAPTATION_MODULE_INTERFACE_H_ |
| 13 | |
Henrik Boström | ede69c0 | 2020-01-21 17:45:35 +0100 | [diff] [blame] | 14 | #include "absl/types/optional.h" |
Henrik Boström | a3d4252 | 2020-01-16 13:55:29 +0100 | [diff] [blame] | 15 | #include "api/rtp_parameters.h" |
Henrik Boström | d4578ae | 2020-01-22 16:16:04 +0100 | [diff] [blame] | 16 | #include "api/video/video_frame.h" |
Henrik Boström | b0f2e0c | 2020-03-06 13:32:03 +0100 | [diff] [blame^] | 17 | #include "call/adaptation/encoder_settings.h" |
Evan Shrubsole | aa6fbc1 | 2020-02-25 16:26:01 +0100 | [diff] [blame] | 18 | #include "call/adaptation/resource.h" |
Henrik Boström | ce0ea49 | 2020-01-13 11:27:18 +0100 | [diff] [blame] | 19 | #include "call/adaptation/video_source_restrictions.h" |
Henrik Boström | d238200 | 2020-01-10 15:44:01 +0100 | [diff] [blame] | 20 | |
Henrik Boström | 382cc6d | 2020-01-07 10:15:04 +0100 | [diff] [blame] | 21 | namespace webrtc { |
| 22 | |
Henrik Boström | d238200 | 2020-01-10 15:44:01 +0100 | [diff] [blame] | 23 | // The listener is responsible for carrying out the reconfiguration of the video |
| 24 | // source such that the VideoSourceRestrictions are fulfilled. |
| 25 | class ResourceAdaptationModuleListener { |
| 26 | public: |
| 27 | virtual ~ResourceAdaptationModuleListener(); |
| 28 | |
| 29 | // TODO(hbos): When we support the muli-stream use case, the arguments need to |
| 30 | // specify which video stream's source needs to be reconfigured. |
| 31 | virtual void OnVideoSourceRestrictionsUpdated( |
| 32 | VideoSourceRestrictions restrictions) = 0; |
| 33 | }; |
| 34 | |
Henrik Boström | 382cc6d | 2020-01-07 10:15:04 +0100 | [diff] [blame] | 35 | // Responsible for reconfiguring encoded streams based on resource consumption, |
| 36 | // such as scaling down resolution or frame rate when CPU is overused. This |
| 37 | // interface is meant to be injectable into VideoStreamEncoder. |
| 38 | // |
| 39 | // [UNDER CONSTRUCTION] This interface is work-in-progress. In the future it |
| 40 | // needs to be able to handle all the necessary input and output for resource |
| 41 | // adaptation decision making. |
| 42 | // |
| 43 | // TODO(https://crbug.com/webrtc/11222): Make this interface feature-complete so |
| 44 | // that a module (such as OveruseFrameDetectorResourceAdaptationModule) is fully |
| 45 | // operational through this abstract interface. |
| 46 | class ResourceAdaptationModuleInterface { |
| 47 | public: |
| 48 | virtual ~ResourceAdaptationModuleInterface(); |
| 49 | |
| 50 | // TODO(hbos): When input/output of the module is adequetly handled by this |
| 51 | // interface, these methods need to say which stream to start/stop, enabling |
| 52 | // multi-stream aware implementations of ResourceAdaptationModuleInterface. We |
| 53 | // don't want to do this before we have the right interfaces (e.g. if we pass |
| 54 | // in a VideoStreamEncoder here directly then have a dependency on a different |
| 55 | // build target). For the multi-stream use case we may consider making |
| 56 | // ResourceAdaptationModuleInterface reference counted. |
Henrik Boström | a3d4252 | 2020-01-16 13:55:29 +0100 | [diff] [blame] | 57 | virtual void StartResourceAdaptation( |
Henrik Boström | d238200 | 2020-01-10 15:44:01 +0100 | [diff] [blame] | 58 | ResourceAdaptationModuleListener* adaptation_listener) = 0; |
Henrik Boström | a3d4252 | 2020-01-16 13:55:29 +0100 | [diff] [blame] | 59 | virtual void StopResourceAdaptation() = 0; |
| 60 | |
Evan Shrubsole | aa6fbc1 | 2020-02-25 16:26:01 +0100 | [diff] [blame] | 61 | // The resource must out-live the module. |
| 62 | virtual void AddResource(Resource* resource) = 0; |
| 63 | |
Henrik Boström | a3d4252 | 2020-01-16 13:55:29 +0100 | [diff] [blame] | 64 | // The following methods are callable whether or not adaption is started. |
| 65 | |
| 66 | // Informs the module whether we have input video. By default, the module must |
| 67 | // assume the value is false. |
| 68 | virtual void SetHasInputVideo(bool has_input_video) = 0; |
| 69 | virtual void SetDegradationPreference( |
| 70 | DegradationPreference degradation_preference) = 0; |
Henrik Boström | 4bab2fc | 2020-01-21 11:18:06 +0100 | [diff] [blame] | 71 | virtual void SetEncoderSettings(EncoderSettings encoder_settings) = 0; |
Evan Shrubsole | 7c3a1fc | 2020-02-04 16:26:38 +0100 | [diff] [blame] | 72 | // TODO(bugs.webrtc.org/11222): This function shouldn't be needed, start |
| 73 | // bitrates should be apart of the constructor ideally. See the comment on |
| 74 | // VideoStreamEncoderInterface::SetStartBitrate. |
| 75 | virtual void SetStartBitrate(DataRate start_bitrate) = 0; |
| 76 | virtual void SetTargetBitrate(DataRate target_bitrate) = 0; |
Evan Shrubsole | e331a12 | 2020-02-05 13:30:33 +0100 | [diff] [blame] | 77 | // The encoder rates are the target encoder bitrate distributed across spatial |
| 78 | // and temporal layers. This may be different than target bitrate depending on |
| 79 | // encoder configuration, e.g. if we can encode at desired quality in less |
| 80 | // than the allowed target bitrate or if the encoder has not been initialized |
| 81 | // yet. |
| 82 | virtual void SetEncoderRates( |
| 83 | const VideoEncoder::RateControlParameters& encoder_rates) = 0; |
Henrik Boström | d4578ae | 2020-01-22 16:16:04 +0100 | [diff] [blame] | 84 | |
| 85 | // The following methods correspond to the pipeline that a frame goes through. |
| 86 | // Note that if the encoder is parallelized, multiple frames may be processed |
| 87 | // in parallel and methods may be invoked in unexpected orders. |
| 88 | // |
| 89 | // The implementation must not retain VideoFrames. Doing so may keep video |
| 90 | // frame buffers alive - this may even stall encoding. |
| 91 | // TODO(hbos): Can we replace VideoFrame with a different struct, maybe width |
| 92 | // and height is enough, and some sort of way to identify it at each step? |
| 93 | |
| 94 | // 1. A frame is delivered to the encoder, e.g. from the camera. Next up: it |
| 95 | // may get dropped or it may get encoded, see OnFrameDroppedDueToSize() and |
| 96 | // OnEncodeStarted(). |
| 97 | virtual void OnFrame(const VideoFrame& frame) = 0; |
| 98 | // 2.i) An input frame was dropped because its resolution is too big (e.g. for |
| 99 | // the target bitrate). This frame will not continue through the rest of the |
| 100 | // pipeline. The module should adapt down in resolution to avoid subsequent |
| 101 | // frames getting dropped for the same reason. |
| 102 | // TODO(hbos): If we take frame rate into account perhaps it would be valid to |
| 103 | // adapt down in frame rate as well. |
| 104 | virtual void OnFrameDroppedDueToSize() = 0; |
Mirko Bonadei | 2e161c4 | 2020-02-20 08:45:01 +0000 | [diff] [blame] | 105 | // 2.ii) If the frame will not be dropped due to size then signal that it may |
| 106 | // get encoded. However the frame is not guaranteed to be encoded right away |
| 107 | // or ever (for example if encoding is paused). |
| 108 | // TODO(eshr): Try replace OnMaybeEncodeFrame and merge behaviour into |
| 109 | // EncodeStarted. |
| 110 | // TODO(eshr): Try to merge OnFrame, OnFrameDroppedDueToSize, and |
| 111 | // OnMaybeEncode frame into one method. |
| 112 | virtual void OnMaybeEncodeFrame() = 0; |
| 113 | // 2.iii) An input frame is about to be encoded. It may have been cropped and |
Henrik Boström | d4578ae | 2020-01-22 16:16:04 +0100 | [diff] [blame] | 114 | // have different dimensions than what was observed at OnFrame(). Next |
| 115 | // up: encoding completes or fails, see OnEncodeCompleted(). There is |
| 116 | // currently no signal for encode failure. |
| 117 | virtual void OnEncodeStarted(const VideoFrame& cropped_frame, |
| 118 | int64_t time_when_first_seen_us) = 0; |
Evan Shrubsole | c809e8b | 2020-01-31 15:36:35 +0100 | [diff] [blame] | 119 | // 3.i) The frame has successfully completed encoding. Next up: The encoded |
Henrik Boström | d4578ae | 2020-01-22 16:16:04 +0100 | [diff] [blame] | 120 | // frame is dropped or packetized and sent over the network. There is |
| 121 | // currently no signal what happens beyond this point. |
Evan Shrubsole | bfe3ef8 | 2020-01-30 14:29:35 +0100 | [diff] [blame] | 122 | virtual void OnEncodeCompleted(const EncodedImage& encoded_image, |
Henrik Boström | d4578ae | 2020-01-22 16:16:04 +0100 | [diff] [blame] | 123 | int64_t time_sent_in_us, |
Henrik Boström | d4578ae | 2020-01-22 16:16:04 +0100 | [diff] [blame] | 124 | absl::optional<int> encode_duration_us) = 0; |
Evan Shrubsole | c809e8b | 2020-01-31 15:36:35 +0100 | [diff] [blame] | 125 | // A frame was dropped at any point in the pipeline. This may come from |
| 126 | // the encoder, or elsewhere, like a frame dropper or frame size check. |
| 127 | virtual void OnFrameDropped(EncodedImageCallback::DropReason reason) = 0; |
Henrik Boström | 382cc6d | 2020-01-07 10:15:04 +0100 | [diff] [blame] | 128 | }; |
| 129 | |
| 130 | } // namespace webrtc |
| 131 | |
| 132 | #endif // CALL_ADAPTATION_RESOURCE_ADAPTATION_MODULE_INTERFACE_H_ |