Sebastian Jansson | d4c5d63 | 2018-07-10 12:57:37 +0200 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2018 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | #ifndef VIDEO_VIDEO_ANALYZER_H_ |
| 11 | #define VIDEO_VIDEO_ANALYZER_H_ |
| 12 | |
| 13 | #include <deque> |
| 14 | #include <map> |
| 15 | #include <memory> |
| 16 | #include <string> |
| 17 | #include <vector> |
| 18 | |
Niels Möller | 1c931c4 | 2018-12-18 16:08:11 +0100 | [diff] [blame] | 19 | #include "api/video/video_source_interface.h" |
Yves Gerey | 79e9f4b | 2019-04-13 18:59:53 +0200 | [diff] [blame] | 20 | #include "rtc_base/numerics/running_statistics.h" |
Bjorn Terelius | 5c2f1f0 | 2019-01-16 17:45:05 +0100 | [diff] [blame] | 21 | #include "rtc_base/time_utils.h" |
Sebastian Jansson | d4c5d63 | 2018-07-10 12:57:37 +0200 | [diff] [blame] | 22 | #include "test/layer_filtering_transport.h" |
| 23 | #include "test/rtp_file_writer.h" |
Sebastian Jansson | d4c5d63 | 2018-07-10 12:57:37 +0200 | [diff] [blame] | 24 | |
| 25 | namespace webrtc { |
| 26 | |
| 27 | class VideoAnalyzer : public PacketReceiver, |
| 28 | public Transport, |
Niels Möller | 88be972 | 2018-10-10 10:58:52 +0200 | [diff] [blame] | 29 | public rtc::VideoSinkInterface<VideoFrame> { |
Sebastian Jansson | d4c5d63 | 2018-07-10 12:57:37 +0200 | [diff] [blame] | 30 | public: |
Yves Gerey | 79e9f4b | 2019-04-13 18:59:53 +0200 | [diff] [blame] | 31 | using Statistics = RunningStatistics<double>; |
| 32 | |
Sebastian Jansson | d4c5d63 | 2018-07-10 12:57:37 +0200 | [diff] [blame] | 33 | VideoAnalyzer(test::LayerFilteringTransport* transport, |
| 34 | const std::string& test_label, |
| 35 | double avg_psnr_threshold, |
| 36 | double avg_ssim_threshold, |
| 37 | int duration_frames, |
| 38 | FILE* graph_data_output_file, |
| 39 | const std::string& graph_title, |
| 40 | uint32_t ssrc_to_analyze, |
| 41 | uint32_t rtx_ssrc_to_analyze, |
| 42 | size_t selected_stream, |
| 43 | int selected_sl, |
| 44 | int selected_tl, |
| 45 | bool is_quick_test_enabled, |
| 46 | Clock* clock, |
Artem Titov | ff7730d | 2019-04-02 13:46:53 +0200 | [diff] [blame] | 47 | std::string rtp_dump_name, |
| 48 | test::SingleThreadedTaskQueueForTesting* task_queue); |
Sebastian Jansson | d4c5d63 | 2018-07-10 12:57:37 +0200 | [diff] [blame] | 49 | ~VideoAnalyzer(); |
| 50 | |
| 51 | virtual void SetReceiver(PacketReceiver* receiver); |
Niels Möller | 1c931c4 | 2018-12-18 16:08:11 +0100 | [diff] [blame] | 52 | void SetSource(rtc::VideoSourceInterface<VideoFrame>* video_source, |
Sebastian Jansson | f1f363f | 2018-08-13 14:24:58 +0200 | [diff] [blame] | 53 | bool respect_sink_wants); |
Sebastian Jansson | d4c5d63 | 2018-07-10 12:57:37 +0200 | [diff] [blame] | 54 | void SetCall(Call* call); |
| 55 | void SetSendStream(VideoSendStream* stream); |
| 56 | void SetReceiveStream(VideoReceiveStream* stream); |
Christoffer Rodbro | c2a0288 | 2018-08-07 14:10:56 +0200 | [diff] [blame] | 57 | void SetAudioReceiveStream(AudioReceiveStream* recv_stream); |
| 58 | |
Sebastian Jansson | d4c5d63 | 2018-07-10 12:57:37 +0200 | [diff] [blame] | 59 | rtc::VideoSinkInterface<VideoFrame>* InputInterface(); |
| 60 | rtc::VideoSourceInterface<VideoFrame>* OutputInterface(); |
| 61 | |
| 62 | DeliveryStatus DeliverPacket(MediaType media_type, |
| 63 | rtc::CopyOnWriteBuffer packet, |
Niels Möller | 7008287 | 2018-08-07 11:03:12 +0200 | [diff] [blame] | 64 | int64_t packet_time_us) override; |
Sebastian Jansson | d4c5d63 | 2018-07-10 12:57:37 +0200 | [diff] [blame] | 65 | |
| 66 | void PreEncodeOnFrame(const VideoFrame& video_frame); |
Niels Möller | 88be972 | 2018-10-10 10:58:52 +0200 | [diff] [blame] | 67 | void PostEncodeOnFrame(size_t stream_id, uint32_t timestamp); |
Sebastian Jansson | d4c5d63 | 2018-07-10 12:57:37 +0200 | [diff] [blame] | 68 | |
| 69 | bool SendRtp(const uint8_t* packet, |
| 70 | size_t length, |
| 71 | const PacketOptions& options) override; |
| 72 | |
| 73 | bool SendRtcp(const uint8_t* packet, size_t length) override; |
| 74 | void OnFrame(const VideoFrame& video_frame) override; |
| 75 | void Wait(); |
| 76 | |
Sebastian Jansson | d4c5d63 | 2018-07-10 12:57:37 +0200 | [diff] [blame] | 77 | void StartMeasuringCpuProcessTime(); |
| 78 | void StopMeasuringCpuProcessTime(); |
| 79 | void StartExcludingCpuThreadTime(); |
| 80 | void StopExcludingCpuThreadTime(); |
| 81 | double GetCpuUsagePercent(); |
| 82 | |
| 83 | test::LayerFilteringTransport* const transport_; |
| 84 | PacketReceiver* receiver_; |
| 85 | |
| 86 | private: |
| 87 | struct FrameComparison { |
| 88 | FrameComparison(); |
| 89 | FrameComparison(const VideoFrame& reference, |
| 90 | const VideoFrame& render, |
| 91 | bool dropped, |
| 92 | int64_t input_time_ms, |
| 93 | int64_t send_time_ms, |
| 94 | int64_t recv_time_ms, |
| 95 | int64_t render_time_ms, |
| 96 | size_t encoded_frame_size); |
| 97 | FrameComparison(bool dropped, |
| 98 | int64_t input_time_ms, |
| 99 | int64_t send_time_ms, |
| 100 | int64_t recv_time_ms, |
| 101 | int64_t render_time_ms, |
| 102 | size_t encoded_frame_size); |
| 103 | |
| 104 | absl::optional<VideoFrame> reference; |
| 105 | absl::optional<VideoFrame> render; |
| 106 | bool dropped; |
| 107 | int64_t input_time_ms; |
| 108 | int64_t send_time_ms; |
| 109 | int64_t recv_time_ms; |
| 110 | int64_t render_time_ms; |
| 111 | size_t encoded_frame_size; |
| 112 | }; |
| 113 | |
| 114 | struct Sample { |
| 115 | Sample(int dropped, |
| 116 | int64_t input_time_ms, |
| 117 | int64_t send_time_ms, |
| 118 | int64_t recv_time_ms, |
| 119 | int64_t render_time_ms, |
| 120 | size_t encoded_frame_size, |
| 121 | double psnr, |
| 122 | double ssim); |
| 123 | |
| 124 | int dropped; |
| 125 | int64_t input_time_ms; |
| 126 | int64_t send_time_ms; |
| 127 | int64_t recv_time_ms; |
| 128 | int64_t render_time_ms; |
| 129 | size_t encoded_frame_size; |
| 130 | double psnr; |
| 131 | double ssim; |
| 132 | }; |
| 133 | |
Sebastian Jansson | d4c5d63 | 2018-07-10 12:57:37 +0200 | [diff] [blame] | 134 | // Implements VideoSinkInterface to receive captured frames from a |
| 135 | // FrameGeneratorCapturer. Implements VideoSourceInterface to be able to act |
| 136 | // as a source to VideoSendStream. |
| 137 | // It forwards all input frames to the VideoAnalyzer for later comparison and |
| 138 | // forwards the captured frames to the VideoSendStream. |
| 139 | class CapturedFrameForwarder : public rtc::VideoSinkInterface<VideoFrame>, |
| 140 | public rtc::VideoSourceInterface<VideoFrame> { |
| 141 | public: |
Ilya Nikolaevskiy | 6957abe | 2019-01-29 16:33:04 +0100 | [diff] [blame] | 142 | CapturedFrameForwarder(VideoAnalyzer* analyzer, |
| 143 | Clock* clock, |
Ilya Nikolaevskiy | 85fc325 | 2019-02-11 10:41:50 +0100 | [diff] [blame] | 144 | int frames_to_process); |
Niels Möller | 1c931c4 | 2018-12-18 16:08:11 +0100 | [diff] [blame] | 145 | void SetSource(rtc::VideoSourceInterface<VideoFrame>* video_source); |
Sebastian Jansson | d4c5d63 | 2018-07-10 12:57:37 +0200 | [diff] [blame] | 146 | |
| 147 | private: |
| 148 | void OnFrame(const VideoFrame& video_frame) override; |
| 149 | |
| 150 | // Called when |send_stream_.SetSource()| is called. |
| 151 | void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink, |
| 152 | const rtc::VideoSinkWants& wants) override; |
| 153 | |
| 154 | // Called by |send_stream_| when |send_stream_.SetSource()| is called. |
| 155 | void RemoveSink(rtc::VideoSinkInterface<VideoFrame>* sink) override; |
| 156 | |
| 157 | VideoAnalyzer* const analyzer_; |
| 158 | rtc::CriticalSection crit_; |
| 159 | rtc::VideoSinkInterface<VideoFrame>* send_stream_input_ |
| 160 | RTC_GUARDED_BY(crit_); |
Niels Möller | 1c931c4 | 2018-12-18 16:08:11 +0100 | [diff] [blame] | 161 | VideoSourceInterface<VideoFrame>* video_source_; |
Sebastian Jansson | d4c5d63 | 2018-07-10 12:57:37 +0200 | [diff] [blame] | 162 | Clock* clock_; |
Ilya Nikolaevskiy | 6957abe | 2019-01-29 16:33:04 +0100 | [diff] [blame] | 163 | int captured_frames_ RTC_GUARDED_BY(crit_); |
Yves Gerey | 0c67c80 | 2019-08-01 17:45:54 +0200 | [diff] [blame^] | 164 | const int frames_to_process_; |
Sebastian Jansson | d4c5d63 | 2018-07-10 12:57:37 +0200 | [diff] [blame] | 165 | }; |
| 166 | |
| 167 | struct FrameWithPsnr { |
| 168 | double psnr; |
| 169 | VideoFrame frame; |
| 170 | }; |
| 171 | |
| 172 | bool IsInSelectedSpatialAndTemporalLayer(const uint8_t* packet, |
| 173 | size_t length, |
| 174 | const RTPHeader& header); |
| 175 | |
| 176 | void AddFrameComparison(const VideoFrame& reference, |
| 177 | const VideoFrame& render, |
| 178 | bool dropped, |
| 179 | int64_t render_time_ms) |
| 180 | RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); |
| 181 | |
Sebastian Jansson | d4c5d63 | 2018-07-10 12:57:37 +0200 | [diff] [blame] | 182 | void PollStats(); |
Niels Möller | 4731f00 | 2019-05-03 09:34:24 +0200 | [diff] [blame] | 183 | static void FrameComparisonThread(void* obj); |
Sebastian Jansson | d4c5d63 | 2018-07-10 12:57:37 +0200 | [diff] [blame] | 184 | bool CompareFrames(); |
| 185 | bool PopComparison(FrameComparison* comparison); |
| 186 | // Increment counter for number of frames received for comparison. |
| 187 | void FrameRecorded(); |
| 188 | // Returns true if all frames to be compared have been taken from the queue. |
| 189 | bool AllFramesRecorded(); |
| 190 | // Increase count of number of frames processed. Returns true if this was the |
| 191 | // last frame to be processed. |
| 192 | bool FrameProcessed(); |
| 193 | void PrintResults(); |
| 194 | void PerformFrameComparison(const FrameComparison& comparison); |
Yves Gerey | 79e9f4b | 2019-04-13 18:59:53 +0200 | [diff] [blame] | 195 | void PrintResult(const char* result_type, Statistics stats, const char* unit); |
Johannes Kron | a1b99b3 | 2019-07-30 15:08:16 +0200 | [diff] [blame] | 196 | void PrintResultWithExternalMean(const char* result_type, |
| 197 | double mean, |
| 198 | Statistics stats, |
| 199 | const char* unit); |
Sebastian Jansson | d4c5d63 | 2018-07-10 12:57:37 +0200 | [diff] [blame] | 200 | void PrintSamplesToFile(void); |
Sebastian Jansson | d4c5d63 | 2018-07-10 12:57:37 +0200 | [diff] [blame] | 201 | void AddCapturedFrameForComparison(const VideoFrame& video_frame); |
| 202 | |
| 203 | Call* call_; |
| 204 | VideoSendStream* send_stream_; |
| 205 | VideoReceiveStream* receive_stream_; |
Christoffer Rodbro | c2a0288 | 2018-08-07 14:10:56 +0200 | [diff] [blame] | 206 | AudioReceiveStream* audio_receive_stream_; |
Sebastian Jansson | d4c5d63 | 2018-07-10 12:57:37 +0200 | [diff] [blame] | 207 | CapturedFrameForwarder captured_frame_forwarder_; |
| 208 | const std::string test_label_; |
| 209 | FILE* const graph_data_output_file_; |
| 210 | const std::string graph_title_; |
| 211 | const uint32_t ssrc_to_analyze_; |
| 212 | const uint32_t rtx_ssrc_to_analyze_; |
| 213 | const size_t selected_stream_; |
| 214 | const int selected_sl_; |
| 215 | const int selected_tl_; |
Sebastian Jansson | d4c5d63 | 2018-07-10 12:57:37 +0200 | [diff] [blame] | 216 | |
| 217 | rtc::CriticalSection comparison_lock_; |
| 218 | std::vector<Sample> samples_ RTC_GUARDED_BY(comparison_lock_); |
Yves Gerey | 79e9f4b | 2019-04-13 18:59:53 +0200 | [diff] [blame] | 219 | Statistics sender_time_ RTC_GUARDED_BY(comparison_lock_); |
| 220 | Statistics receiver_time_ RTC_GUARDED_BY(comparison_lock_); |
| 221 | Statistics network_time_ RTC_GUARDED_BY(comparison_lock_); |
| 222 | Statistics psnr_ RTC_GUARDED_BY(comparison_lock_); |
| 223 | Statistics ssim_ RTC_GUARDED_BY(comparison_lock_); |
| 224 | Statistics end_to_end_ RTC_GUARDED_BY(comparison_lock_); |
| 225 | Statistics rendered_delta_ RTC_GUARDED_BY(comparison_lock_); |
| 226 | Statistics encoded_frame_size_ RTC_GUARDED_BY(comparison_lock_); |
| 227 | Statistics encode_frame_rate_ RTC_GUARDED_BY(comparison_lock_); |
| 228 | Statistics encode_time_ms_ RTC_GUARDED_BY(comparison_lock_); |
| 229 | Statistics encode_usage_percent_ RTC_GUARDED_BY(comparison_lock_); |
Johannes Kron | a1b99b3 | 2019-07-30 15:08:16 +0200 | [diff] [blame] | 230 | double mean_decode_time_ms_ RTC_GUARDED_BY(comparison_lock_); |
Yves Gerey | 79e9f4b | 2019-04-13 18:59:53 +0200 | [diff] [blame] | 231 | Statistics decode_time_ms_ RTC_GUARDED_BY(comparison_lock_); |
| 232 | Statistics decode_time_max_ms_ RTC_GUARDED_BY(comparison_lock_); |
| 233 | Statistics media_bitrate_bps_ RTC_GUARDED_BY(comparison_lock_); |
| 234 | Statistics fec_bitrate_bps_ RTC_GUARDED_BY(comparison_lock_); |
| 235 | Statistics send_bandwidth_bps_ RTC_GUARDED_BY(comparison_lock_); |
| 236 | Statistics memory_usage_ RTC_GUARDED_BY(comparison_lock_); |
Yves Gerey | 79e9f4b | 2019-04-13 18:59:53 +0200 | [diff] [blame] | 237 | Statistics audio_expand_rate_ RTC_GUARDED_BY(comparison_lock_); |
| 238 | Statistics audio_accelerate_rate_ RTC_GUARDED_BY(comparison_lock_); |
| 239 | Statistics audio_jitter_buffer_ms_ RTC_GUARDED_BY(comparison_lock_); |
| 240 | Statistics pixels_ RTC_GUARDED_BY(comparison_lock_); |
Sebastian Jansson | d4c5d63 | 2018-07-10 12:57:37 +0200 | [diff] [blame] | 241 | // Rendered frame with worst PSNR is saved for further analysis. |
| 242 | absl::optional<FrameWithPsnr> worst_frame_ RTC_GUARDED_BY(comparison_lock_); |
Elad Alon | 8c513c7 | 2019-05-07 21:22:24 +0200 | [diff] [blame] | 243 | // Freeze metrics. |
| 244 | Statistics time_between_freezes_ RTC_GUARDED_BY(comparison_lock_); |
| 245 | uint32_t freeze_count_ RTC_GUARDED_BY(comparison_lock_); |
| 246 | uint32_t total_freezes_duration_ms_ RTC_GUARDED_BY(comparison_lock_); |
| 247 | uint32_t total_frames_duration_ms_ RTC_GUARDED_BY(comparison_lock_); |
| 248 | double sum_squared_frame_durations_ RTC_GUARDED_BY(comparison_lock_); |
Sebastian Jansson | d4c5d63 | 2018-07-10 12:57:37 +0200 | [diff] [blame] | 249 | |
Elad Alon | 58e0657 | 2019-05-08 15:34:24 +0200 | [diff] [blame] | 250 | double decode_frame_rate_ RTC_GUARDED_BY(comparison_lock_); |
| 251 | double render_frame_rate_ RTC_GUARDED_BY(comparison_lock_); |
| 252 | |
Sebastian Jansson | d4c5d63 | 2018-07-10 12:57:37 +0200 | [diff] [blame] | 253 | size_t last_fec_bytes_; |
| 254 | |
Yves Gerey | 0c67c80 | 2019-08-01 17:45:54 +0200 | [diff] [blame^] | 255 | rtc::CriticalSection crit_; |
Sebastian Jansson | d4c5d63 | 2018-07-10 12:57:37 +0200 | [diff] [blame] | 256 | const int frames_to_process_; |
Yves Gerey | 0c67c80 | 2019-08-01 17:45:54 +0200 | [diff] [blame^] | 257 | int frames_recorded_ RTC_GUARDED_BY(comparison_lock_); |
| 258 | int frames_processed_ RTC_GUARDED_BY(comparison_lock_); |
| 259 | int captured_frames_ RTC_GUARDED_BY(comparison_lock_); |
| 260 | int dropped_frames_ RTC_GUARDED_BY(comparison_lock_); |
| 261 | int dropped_frames_before_first_encode_ RTC_GUARDED_BY(crit_); |
| 262 | int dropped_frames_before_rendering_ RTC_GUARDED_BY(crit_); |
| 263 | int64_t last_render_time_ RTC_GUARDED_BY(comparison_lock_); |
| 264 | int64_t last_render_delta_ms_ RTC_GUARDED_BY(comparison_lock_); |
| 265 | int64_t last_unfreeze_time_ms_ RTC_GUARDED_BY(comparison_lock_); |
| 266 | uint32_t rtp_timestamp_delta_ RTC_GUARDED_BY(crit_); |
Sebastian Jansson | d4c5d63 | 2018-07-10 12:57:37 +0200 | [diff] [blame] | 267 | |
| 268 | rtc::CriticalSection cpu_measurement_lock_; |
| 269 | int64_t cpu_time_ RTC_GUARDED_BY(cpu_measurement_lock_); |
| 270 | int64_t wallclock_time_ RTC_GUARDED_BY(cpu_measurement_lock_); |
| 271 | |
Sebastian Jansson | d4c5d63 | 2018-07-10 12:57:37 +0200 | [diff] [blame] | 272 | std::deque<VideoFrame> frames_ RTC_GUARDED_BY(crit_); |
| 273 | absl::optional<VideoFrame> last_rendered_frame_ RTC_GUARDED_BY(crit_); |
| 274 | rtc::TimestampWrapAroundHandler wrap_handler_ RTC_GUARDED_BY(crit_); |
| 275 | std::map<int64_t, int64_t> send_times_ RTC_GUARDED_BY(crit_); |
| 276 | std::map<int64_t, int64_t> recv_times_ RTC_GUARDED_BY(crit_); |
| 277 | std::map<int64_t, size_t> encoded_frame_sizes_ RTC_GUARDED_BY(crit_); |
| 278 | absl::optional<uint32_t> first_encoded_timestamp_ RTC_GUARDED_BY(crit_); |
| 279 | absl::optional<uint32_t> first_sent_timestamp_ RTC_GUARDED_BY(crit_); |
| 280 | const double avg_psnr_threshold_; |
| 281 | const double avg_ssim_threshold_; |
| 282 | bool is_quick_test_enabled_; |
| 283 | |
| 284 | std::vector<rtc::PlatformThread*> comparison_thread_pool_; |
Sebastian Jansson | d4c5d63 | 2018-07-10 12:57:37 +0200 | [diff] [blame] | 285 | rtc::Event comparison_available_event_; |
| 286 | std::deque<FrameComparison> comparisons_ RTC_GUARDED_BY(comparison_lock_); |
Niels Möller | 4731f00 | 2019-05-03 09:34:24 +0200 | [diff] [blame] | 287 | bool quit_ RTC_GUARDED_BY(comparison_lock_); |
Sebastian Jansson | d4c5d63 | 2018-07-10 12:57:37 +0200 | [diff] [blame] | 288 | rtc::Event done_; |
Artem Titov | ff7730d | 2019-04-02 13:46:53 +0200 | [diff] [blame] | 289 | test::SingleThreadedTaskQueueForTesting::TaskId stats_polling_task_id_ |
| 290 | RTC_GUARDED_BY(comparison_lock_); |
| 291 | bool stop_stats_poller_ RTC_GUARDED_BY(comparison_lock_); |
Sebastian Jansson | d4c5d63 | 2018-07-10 12:57:37 +0200 | [diff] [blame] | 292 | |
| 293 | std::unique_ptr<test::RtpFileWriter> rtp_file_writer_; |
| 294 | Clock* const clock_; |
| 295 | const int64_t start_ms_; |
Artem Titov | ff7730d | 2019-04-02 13:46:53 +0200 | [diff] [blame] | 296 | test::SingleThreadedTaskQueueForTesting* task_queue_; |
Sebastian Jansson | d4c5d63 | 2018-07-10 12:57:37 +0200 | [diff] [blame] | 297 | }; |
| 298 | |
| 299 | } // namespace webrtc |
| 300 | #endif // VIDEO_VIDEO_ANALYZER_H_ |