mflodman@webrtc.org | e6168f5 | 2013-06-26 11:23:01 +0000 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 11 | #ifndef VIDEO_OVERUSE_FRAME_DETECTOR_H_ |
| 12 | #define VIDEO_OVERUSE_FRAME_DETECTOR_H_ |
mflodman@webrtc.org | e6168f5 | 2013-06-26 11:23:01 +0000 | [diff] [blame] | 13 | |
Peter Boström | e449915 | 2016-02-05 11:13:28 +0100 | [diff] [blame] | 14 | #include <list> |
kwiberg | 27f982b | 2016-03-01 11:52:33 -0800 | [diff] [blame] | 15 | #include <memory> |
Peter Boström | e449915 | 2016-02-05 11:13:28 +0100 | [diff] [blame] | 16 | |
Danil Chapovalov | b9b146c | 2018-06-15 12:28:07 +0200 | [diff] [blame] | 17 | #include "absl/types/optional.h" |
Henrik Boström | 07b17df | 2020-01-15 11:42:12 +0100 | [diff] [blame] | 18 | #include "api/task_queue/task_queue_base.h" |
Niels Möller | 213618e | 2018-07-24 09:29:58 +0200 | [diff] [blame] | 19 | #include "api/video/video_stream_encoder_observer.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 20 | #include "modules/video_coding/utility/quality_scaler.h" |
Steve Anton | 10542f2 | 2019-01-11 09:11:00 -0800 | [diff] [blame] | 21 | #include "rtc_base/constructor_magic.h" |
Niels Möller | eea9288 | 2019-04-25 08:44:04 +0200 | [diff] [blame] | 22 | #include "rtc_base/experiments/field_trial_parser.h" |
Niels Möller | 7dc26b7 | 2017-12-06 10:27:48 +0100 | [diff] [blame] | 23 | #include "rtc_base/numerics/exp_filter.h" |
Sebastian Jansson | b55015e | 2019-04-09 13:44:04 +0200 | [diff] [blame] | 24 | #include "rtc_base/synchronization/sequence_checker.h" |
Sebastian Jansson | ecb6897 | 2019-01-18 10:30:54 +0100 | [diff] [blame] | 25 | #include "rtc_base/task_utils/repeating_task.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 26 | #include "rtc_base/thread_annotations.h" |
mflodman@webrtc.org | e6168f5 | 2013-06-26 11:23:01 +0000 | [diff] [blame] | 27 | |
| 28 | namespace webrtc { |
| 29 | |
Peter Boström | e449915 | 2016-02-05 11:13:28 +0100 | [diff] [blame] | 30 | class VideoFrame; |
Peter Boström | 300eeb6 | 2015-05-12 16:51:11 +0200 | [diff] [blame] | 31 | |
Peter Boström | 300eeb6 | 2015-05-12 16:51:11 +0200 | [diff] [blame] | 32 | struct CpuOveruseOptions { |
torbjorng | 448468d | 2016-02-10 08:11:57 -0800 | [diff] [blame] | 33 | CpuOveruseOptions(); |
Peter Boström | 300eeb6 | 2015-05-12 16:51:11 +0200 | [diff] [blame] | 34 | |
Peter Boström | 300eeb6 | 2015-05-12 16:51:11 +0200 | [diff] [blame] | 35 | int low_encode_usage_threshold_percent; // Threshold for triggering underuse. |
kjellander@webrtc.org | 0fcaf99 | 2015-11-26 15:24:52 +0100 | [diff] [blame] | 36 | int high_encode_usage_threshold_percent; // Threshold for triggering overuse. |
Peter Boström | 300eeb6 | 2015-05-12 16:51:11 +0200 | [diff] [blame] | 37 | // General settings. |
| 38 | int frame_timeout_interval_ms; // The maximum allowed interval between two |
| 39 | // frames before resetting estimations. |
Yves Gerey | 665174f | 2018-06-19 15:03:05 +0200 | [diff] [blame] | 40 | int min_frame_samples; // The minimum number of frames required. |
Peter Boström | 300eeb6 | 2015-05-12 16:51:11 +0200 | [diff] [blame] | 41 | int min_process_count; // The number of initial process times required before |
| 42 | // triggering an overuse/underuse. |
kjellander@webrtc.org | 0fcaf99 | 2015-11-26 15:24:52 +0100 | [diff] [blame] | 43 | int high_threshold_consecutive_count; // The number of consecutive checks |
| 44 | // above the high threshold before |
| 45 | // triggering an overuse. |
Niels Möller | 83dbeac | 2017-12-14 16:39:44 +0100 | [diff] [blame] | 46 | // New estimator enabled if this is set non-zero. |
| 47 | int filter_time_ms; // Time constant for averaging |
Peter Boström | 300eeb6 | 2015-05-12 16:51:11 +0200 | [diff] [blame] | 48 | }; |
| 49 | |
Åsa Persson | 746210f | 2015-09-08 10:52:42 +0200 | [diff] [blame] | 50 | // Use to detect system overuse based on the send-side processing time of |
perkj | d52063f | 2016-09-07 06:32:18 -0700 | [diff] [blame] | 51 | // incoming frames. All methods must be called on a single task queue but it can |
| 52 | // be created and destroyed on an arbitrary thread. |
| 53 | // OveruseFrameDetector::StartCheckForOveruse must be called to periodically |
| 54 | // check for overuse. |
| 55 | class OveruseFrameDetector { |
mflodman@webrtc.org | e6168f5 | 2013-06-26 11:23:01 +0000 | [diff] [blame] | 56 | public: |
Niels Möller | d1f7eb6 | 2018-03-28 16:40:58 +0200 | [diff] [blame] | 57 | explicit OveruseFrameDetector(CpuOveruseMetricsObserver* metrics_observer); |
sprang | fda496a | 2017-06-15 04:21:07 -0700 | [diff] [blame] | 58 | virtual ~OveruseFrameDetector(); |
mflodman@webrtc.org | e6168f5 | 2013-06-26 11:23:01 +0000 | [diff] [blame] | 59 | |
perkj | d52063f | 2016-09-07 06:32:18 -0700 | [diff] [blame] | 60 | // Start to periodically check for overuse. |
Henrik Boström | 07b17df | 2020-01-15 11:42:12 +0100 | [diff] [blame] | 61 | void StartCheckForOveruse(TaskQueueBase* task_queue_base, |
Sebastian Jansson | cda86dd | 2019-03-11 17:26:36 +0100 | [diff] [blame] | 62 | const CpuOveruseOptions& options, |
Niels Möller | d1f7eb6 | 2018-03-28 16:40:58 +0200 | [diff] [blame] | 63 | AdaptationObserverInterface* overuse_observer); |
perkj | d52063f | 2016-09-07 06:32:18 -0700 | [diff] [blame] | 64 | |
| 65 | // StopCheckForOveruse must be called before destruction if |
| 66 | // StartCheckForOveruse has been called. |
| 67 | void StopCheckForOveruse(); |
| 68 | |
Niels Möller | 7dc26b7 | 2017-12-06 10:27:48 +0100 | [diff] [blame] | 69 | // Defines the current maximum framerate targeted by the capturer. This is |
| 70 | // used to make sure the encode usage percent doesn't drop unduly if the |
| 71 | // capturer has quiet periods (for instance caused by screen capturers with |
| 72 | // variable capture rate depending on content updates), otherwise we might |
| 73 | // experience adaptation toggling. |
| 74 | virtual void OnTargetFramerateUpdated(int framerate_fps); |
mflodman@webrtc.org | e6168f5 | 2013-06-26 11:23:01 +0000 | [diff] [blame] | 75 | |
Niels Möller | 7dc26b7 | 2017-12-06 10:27:48 +0100 | [diff] [blame] | 76 | // Called for each captured frame. |
| 77 | void FrameCaptured(const VideoFrame& frame, int64_t time_when_first_seen_us); |
| 78 | |
| 79 | // Called for each sent frame. |
Niels Möller | 83dbeac | 2017-12-14 16:39:44 +0100 | [diff] [blame] | 80 | void FrameSent(uint32_t timestamp, |
| 81 | int64_t time_sent_in_us, |
| 82 | int64_t capture_time_us, |
Danil Chapovalov | b9b146c | 2018-06-15 12:28:07 +0200 | [diff] [blame] | 83 | absl::optional<int> encode_duration_us); |
asapersson@webrtc.org | b24d335 | 2013-11-20 13:51:40 +0000 | [diff] [blame] | 84 | |
Niels Möller | 904f869 | 2017-12-07 11:22:39 +0100 | [diff] [blame] | 85 | // Interface for cpu load estimation. Intended for internal use only. |
| 86 | class ProcessingUsage { |
| 87 | public: |
| 88 | virtual void Reset() = 0; |
| 89 | virtual void SetMaxSampleDiffMs(float diff_ms) = 0; |
Niels Möller | e08cf3a | 2017-12-07 15:23:58 +0100 | [diff] [blame] | 90 | virtual void FrameCaptured(const VideoFrame& frame, |
| 91 | int64_t time_when_first_seen_us, |
| 92 | int64_t last_capture_time_us) = 0; |
| 93 | // Returns encode_time in us, if there's a new measurement. |
Danil Chapovalov | b9b146c | 2018-06-15 12:28:07 +0200 | [diff] [blame] | 94 | virtual absl::optional<int> FrameSent( |
Niels Möller | 83dbeac | 2017-12-14 16:39:44 +0100 | [diff] [blame] | 95 | // These two argument used by old estimator. |
| 96 | uint32_t timestamp, |
| 97 | int64_t time_sent_in_us, |
| 98 | // And these two by the new estimator. |
| 99 | int64_t capture_time_us, |
Danil Chapovalov | b9b146c | 2018-06-15 12:28:07 +0200 | [diff] [blame] | 100 | absl::optional<int> encode_duration_us) = 0; |
Niels Möller | e08cf3a | 2017-12-07 15:23:58 +0100 | [diff] [blame] | 101 | |
Niels Möller | 904f869 | 2017-12-07 11:22:39 +0100 | [diff] [blame] | 102 | virtual int Value() = 0; |
| 103 | virtual ~ProcessingUsage() = default; |
| 104 | }; |
| 105 | |
perkj | d52063f | 2016-09-07 06:32:18 -0700 | [diff] [blame] | 106 | protected: |
Niels Möller | 73f29cb | 2018-01-31 16:09:31 +0100 | [diff] [blame] | 107 | // Protected for test purposes. |
| 108 | void CheckForOveruse(AdaptationObserverInterface* overuse_observer); |
Niels Möller | d1f7eb6 | 2018-03-28 16:40:58 +0200 | [diff] [blame] | 109 | void SetOptions(const CpuOveruseOptions& options); |
mflodman@webrtc.org | e6168f5 | 2013-06-26 11:23:01 +0000 | [diff] [blame] | 110 | |
Niels Möller | 4db138e | 2018-04-19 09:04:13 +0200 | [diff] [blame] | 111 | CpuOveruseOptions options_; |
| 112 | |
mflodman@webrtc.org | e6168f5 | 2013-06-26 11:23:01 +0000 | [diff] [blame] | 113 | private: |
perkj | d52063f | 2016-09-07 06:32:18 -0700 | [diff] [blame] | 114 | void EncodedFrameTimeMeasured(int encode_duration_ms); |
Niels Möller | 213618e | 2018-07-24 09:29:58 +0200 | [diff] [blame] | 115 | bool IsOverusing(int encode_usage_percent); |
| 116 | bool IsUnderusing(int encode_usage_percent, int64_t time_now); |
mflodman@webrtc.org | e6168f5 | 2013-06-26 11:23:01 +0000 | [diff] [blame] | 117 | |
perkj | d52063f | 2016-09-07 06:32:18 -0700 | [diff] [blame] | 118 | bool FrameTimeoutDetected(int64_t now) const; |
| 119 | bool FrameSizeChanged(int num_pixels) const; |
asapersson@webrtc.org | 8a8c3ef | 2014-03-20 13:15:01 +0000 | [diff] [blame] | 120 | |
Niels Möller | 7dc26b7 | 2017-12-06 10:27:48 +0100 | [diff] [blame] | 121 | void ResetAll(int num_pixels); |
asapersson@webrtc.org | b60346e | 2014-02-17 19:02:15 +0000 | [diff] [blame] | 122 | |
Niels Möller | 904f869 | 2017-12-07 11:22:39 +0100 | [diff] [blame] | 123 | static std::unique_ptr<ProcessingUsage> CreateProcessingUsage( |
Niels Möller | 6b642f7 | 2017-12-08 14:11:14 +0100 | [diff] [blame] | 124 | const CpuOveruseOptions& options); |
sprang | c5d62e2 | 2017-04-02 23:53:04 -0700 | [diff] [blame] | 125 | |
Sebastian Jansson | b55015e | 2019-04-09 13:44:04 +0200 | [diff] [blame] | 126 | SequenceChecker task_checker_; |
perkj | d52063f | 2016-09-07 06:32:18 -0700 | [diff] [blame] | 127 | // Owned by the task queue from where StartCheckForOveruse is called. |
Sebastian Jansson | ecb6897 | 2019-01-18 10:30:54 +0100 | [diff] [blame] | 128 | RepeatingTaskHandle check_overuse_task_ RTC_GUARDED_BY(task_checker_); |
mflodman@webrtc.org | e6168f5 | 2013-06-26 11:23:01 +0000 | [diff] [blame] | 129 | |
pbos@webrtc.org | 3e6e271 | 2015-02-26 12:19:31 +0000 | [diff] [blame] | 130 | // Stats metrics. |
| 131 | CpuOveruseMetricsObserver* const metrics_observer_; |
Niels Möller | 213618e | 2018-07-24 09:29:58 +0200 | [diff] [blame] | 132 | absl::optional<int> encode_usage_percent_ RTC_GUARDED_BY(task_checker_); |
mflodman@webrtc.org | e6168f5 | 2013-06-26 11:23:01 +0000 | [diff] [blame] | 133 | |
danilchap | a37de39 | 2017-09-09 04:17:22 -0700 | [diff] [blame] | 134 | int64_t num_process_times_ RTC_GUARDED_BY(task_checker_); |
perkj | d52063f | 2016-09-07 06:32:18 -0700 | [diff] [blame] | 135 | |
danilchap | a37de39 | 2017-09-09 04:17:22 -0700 | [diff] [blame] | 136 | int64_t last_capture_time_us_ RTC_GUARDED_BY(task_checker_); |
mflodman@webrtc.org | e6168f5 | 2013-06-26 11:23:01 +0000 | [diff] [blame] | 137 | |
asapersson | 74d85e1 | 2015-09-24 00:53:32 -0700 | [diff] [blame] | 138 | // Number of pixels of last captured frame. |
danilchap | a37de39 | 2017-09-09 04:17:22 -0700 | [diff] [blame] | 139 | int num_pixels_ RTC_GUARDED_BY(task_checker_); |
Niels Möller | 7dc26b7 | 2017-12-06 10:27:48 +0100 | [diff] [blame] | 140 | int max_framerate_ RTC_GUARDED_BY(task_checker_); |
danilchap | a37de39 | 2017-09-09 04:17:22 -0700 | [diff] [blame] | 141 | int64_t last_overuse_time_ms_ RTC_GUARDED_BY(task_checker_); |
| 142 | int checks_above_threshold_ RTC_GUARDED_BY(task_checker_); |
| 143 | int num_overuse_detections_ RTC_GUARDED_BY(task_checker_); |
| 144 | int64_t last_rampup_time_ms_ RTC_GUARDED_BY(task_checker_); |
| 145 | bool in_quick_rampup_ RTC_GUARDED_BY(task_checker_); |
| 146 | int current_rampup_delay_ms_ RTC_GUARDED_BY(task_checker_); |
mflodman@webrtc.org | e6168f5 | 2013-06-26 11:23:01 +0000 | [diff] [blame] | 147 | |
Niels Möller | d1f7eb6 | 2018-03-28 16:40:58 +0200 | [diff] [blame] | 148 | std::unique_ptr<ProcessingUsage> usage_ RTC_PT_GUARDED_BY(task_checker_); |
asapersson@webrtc.org | b24d335 | 2013-11-20 13:51:40 +0000 | [diff] [blame] | 149 | |
Niels Möller | eea9288 | 2019-04-25 08:44:04 +0200 | [diff] [blame] | 150 | // If set by field trial, overrides CpuOveruseOptions::filter_time_ms. |
| 151 | FieldTrialOptional<TimeDelta> filter_time_constant_{"tau"}; |
| 152 | |
henrikg | 3c089d7 | 2015-09-16 05:37:44 -0700 | [diff] [blame] | 153 | RTC_DISALLOW_COPY_AND_ASSIGN(OveruseFrameDetector); |
mflodman@webrtc.org | e6168f5 | 2013-06-26 11:23:01 +0000 | [diff] [blame] | 154 | }; |
| 155 | |
| 156 | } // namespace webrtc |
| 157 | |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 158 | #endif // VIDEO_OVERUSE_FRAME_DETECTOR_H_ |