media/base/mediachannel.h

/*
 *  Copyright (c) 2004 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#ifndef MEDIA_BASE_MEDIACHANNEL_H_
#define MEDIA_BASE_MEDIACHANNEL_H_

#include <map>
#include <memory>
#include <string>
#include <vector>

#include "api/audio_codecs/audio_encoder.h"
#include "api/optional.h"
#include "api/rtpparameters.h"
#include "api/rtpreceiverinterface.h"
#include "api/video/video_timing.h"
#include "call/video_config.h"
#include "media/base/codec.h"
#include "media/base/mediaconstants.h"
#include "media/base/streamparams.h"
#include "media/base/videosinkinterface.h"
#include "media/base/videosourceinterface.h"
#include "modules/audio_processing/include/audio_processing_statistics.h"
#include "rtc_base/basictypes.h"
#include "rtc_base/buffer.h"
#include "rtc_base/copyonwritebuffer.h"
#include "rtc_base/dscp.h"
#include "rtc_base/logging.h"
#include "rtc_base/networkroute.h"
#include "rtc_base/sigslot.h"
#include "rtc_base/socket.h"
#include "rtc_base/window.h"
// TODO(juberti): re-evaluate this include
#include "pc/audiomonitor.h"

namespace rtc {
class RateLimiter;
class Timing;
}

namespace webrtc {
class AudioSinkInterface;
class VideoFrame;
}

namespace cricket {

class AudioSource;
class VideoCapturer;
struct RtpHeader;
struct VideoFormat;

const int kScreencastDefaultFps = 5;

template <class T>
static std::string ToStringIfSet(const char* key, const rtc::Optional<T>& val) {
  std::string str;
  if (val) {
    str = key;
    str += ": ";
    str += val ? rtc::ToString(*val) : "";
    str += ", ";
  }
  return str;
}

template <class T>
static std::string VectorToString(const std::vector<T>& vals) {
    std::ostringstream ost;
    ost << "[";
    for (size_t i = 0; i < vals.size(); ++i) {
      if (i > 0) {
        ost << ", ";
      }
      ost << vals[i].ToString();
    }
    ost << "]";
    return ost.str();
}

// Construction-time settings, passed on when creating
// MediaChannels.
struct MediaConfig {
  // Set DSCP value on packets. This flag comes from the
  // PeerConnection constraint 'googDscp'.
  bool enable_dscp = false;

  // Video-specific config.
  struct Video {
    // Enable WebRTC CPU Overuse Detection. This flag comes from the
    // PeerConnection constraint 'googCpuOveruseDetection'.
    bool enable_cpu_overuse_detection = true;

    // Enable WebRTC suspension of video. No video frames will be sent
    // when the bitrate is below the configured minimum bitrate. This
    // flag comes from the PeerConnection constraint
    // 'googSuspendBelowMinBitrate', and WebRtcVideoChannel copies it
    // to VideoSendStream::Config::suspend_below_min_bitrate.
    bool suspend_below_min_bitrate = false;

    // Set to true if the renderer has an algorithm of frame selection.
    // If the value is true, then WebRTC will hand over a frame as soon as
    // possible without delay, and rendering smoothness is completely the duty
    // of the renderer;
    // If the value is false, then WebRTC is responsible to delay frame release
    // in order to increase rendering smoothness.
    //
    // This flag comes from PeerConnection's RtcConfiguration, but is
    // currently only set by the command line flag
    // 'disable-rtc-smoothness-algorithm'.
    // WebRtcVideoChannel::AddRecvStream copies it to the created
    // WebRtcVideoReceiveStream, where it is returned by the
    // SmoothsRenderedFrames method. This method is used by the
    // VideoReceiveStream, where the value is passed on to the
    // IncomingVideoStream constructor.
    bool disable_prerenderer_smoothing = false;

    // Enables periodic bandwidth probing in application-limited region.
    bool periodic_alr_bandwidth_probing = false;
  } video;

  bool operator==(const MediaConfig& o) const {
    return enable_dscp == o.enable_dscp &&
           video.enable_cpu_overuse_detection ==
               o.video.enable_cpu_overuse_detection &&
           video.suspend_below_min_bitrate ==
               o.video.suspend_below_min_bitrate &&
           video.disable_prerenderer_smoothing ==
               o.video.disable_prerenderer_smoothing &&
           video.periodic_alr_bandwidth_probing ==
               o.video.periodic_alr_bandwidth_probing;
  }

  bool operator!=(const MediaConfig& o) const { return !(*this == o); }
};

// Options that can be applied to a VoiceMediaChannel or a VoiceMediaEngine.
// Used to be flags, but that makes it hard to selectively apply options.
// We are moving all of the setting of options to structs like this,
// but some things currently still use flags.
struct AudioOptions {
  void SetAll(const AudioOptions& change) {
    SetFrom(&echo_cancellation, change.echo_cancellation);
    SetFrom(&auto_gain_control, change.auto_gain_control);
    SetFrom(&noise_suppression, change.noise_suppression);
    SetFrom(&highpass_filter, change.highpass_filter);
    SetFrom(&stereo_swapping, change.stereo_swapping);
    SetFrom(&audio_jitter_buffer_max_packets,
            change.audio_jitter_buffer_max_packets);
    SetFrom(&audio_jitter_buffer_fast_accelerate,
            change.audio_jitter_buffer_fast_accelerate);
    SetFrom(&typing_detection, change.typing_detection);
    SetFrom(&aecm_generate_comfort_noise, change.aecm_generate_comfort_noise);
    SetFrom(&adjust_agc_delta, change.adjust_agc_delta);
    SetFrom(&experimental_agc, change.experimental_agc);
    SetFrom(&extended_filter_aec, change.extended_filter_aec);
    SetFrom(&delay_agnostic_aec, change.delay_agnostic_aec);
    SetFrom(&experimental_ns, change.experimental_ns);
    SetFrom(&intelligibility_enhancer, change.intelligibility_enhancer);
    SetFrom(&level_control, change.level_control);
    SetFrom(&residual_echo_detector, change.residual_echo_detector);
    SetFrom(&tx_agc_target_dbov, change.tx_agc_target_dbov);
    SetFrom(&tx_agc_digital_compression_gain,
            change.tx_agc_digital_compression_gain);
    SetFrom(&tx_agc_limiter, change.tx_agc_limiter);
    SetFrom(&combined_audio_video_bwe, change.combined_audio_video_bwe);
    SetFrom(&audio_network_adaptor, change.audio_network_adaptor);
    SetFrom(&audio_network_adaptor_config, change.audio_network_adaptor_config);
    SetFrom(&level_control_initial_peak_level_dbfs,
            change.level_control_initial_peak_level_dbfs);
  }

  bool operator==(const AudioOptions& o) const {
    return echo_cancellation == o.echo_cancellation &&
           auto_gain_control == o.auto_gain_control &&
           noise_suppression == o.noise_suppression &&
           highpass_filter == o.highpass_filter &&
           stereo_swapping == o.stereo_swapping &&
           audio_jitter_buffer_max_packets ==
               o.audio_jitter_buffer_max_packets &&
           audio_jitter_buffer_fast_accelerate ==
               o.audio_jitter_buffer_fast_accelerate &&
           typing_detection == o.typing_detection &&
           aecm_generate_comfort_noise == o.aecm_generate_comfort_noise &&
           experimental_agc == o.experimental_agc &&
           extended_filter_aec == o.extended_filter_aec &&
           delay_agnostic_aec == o.delay_agnostic_aec &&
           experimental_ns == o.experimental_ns &&
           intelligibility_enhancer == o.intelligibility_enhancer &&
           level_control == o.level_control &&
           residual_echo_detector == o.residual_echo_detector &&
           adjust_agc_delta == o.adjust_agc_delta &&
           tx_agc_target_dbov == o.tx_agc_target_dbov &&
           tx_agc_digital_compression_gain ==
               o.tx_agc_digital_compression_gain &&
           tx_agc_limiter == o.tx_agc_limiter &&
           combined_audio_video_bwe == o.combined_audio_video_bwe &&
           audio_network_adaptor == o.audio_network_adaptor &&
           audio_network_adaptor_config == o.audio_network_adaptor_config &&
           level_control_initial_peak_level_dbfs ==
               o.level_control_initial_peak_level_dbfs;
  }
  bool operator!=(const AudioOptions& o) const { return !(*this == o); }

  std::string ToString() const {
    std::ostringstream ost;
    ost << "AudioOptions {";
    ost << ToStringIfSet("aec", echo_cancellation);
    ost << ToStringIfSet("agc", auto_gain_control);
    ost << ToStringIfSet("ns", noise_suppression);
    ost << ToStringIfSet("hf", highpass_filter);
    ost << ToStringIfSet("swap", stereo_swapping);
    ost << ToStringIfSet("audio_jitter_buffer_max_packets",
                         audio_jitter_buffer_max_packets);
    ost << ToStringIfSet("audio_jitter_buffer_fast_accelerate",
                         audio_jitter_buffer_fast_accelerate);
    ost << ToStringIfSet("typing", typing_detection);
    ost << ToStringIfSet("comfort_noise", aecm_generate_comfort_noise);
    ost << ToStringIfSet("agc_delta", adjust_agc_delta);
    ost << ToStringIfSet("experimental_agc", experimental_agc);
    ost << ToStringIfSet("extended_filter_aec", extended_filter_aec);
    ost << ToStringIfSet("delay_agnostic_aec", delay_agnostic_aec);
    ost << ToStringIfSet("experimental_ns", experimental_ns);
    ost << ToStringIfSet("intelligibility_enhancer", intelligibility_enhancer);
    ost << ToStringIfSet("level_control", level_control);
    ost << ToStringIfSet("level_control_initial_peak_level_dbfs",
                         level_control_initial_peak_level_dbfs);
    ost << ToStringIfSet("residual_echo_detector", residual_echo_detector);
    ost << ToStringIfSet("tx_agc_target_dbov", tx_agc_target_dbov);
    ost << ToStringIfSet("tx_agc_digital_compression_gain",
        tx_agc_digital_compression_gain);
    ost << ToStringIfSet("tx_agc_limiter", tx_agc_limiter);
    ost << ToStringIfSet("combined_audio_video_bwe", combined_audio_video_bwe);
    ost << ToStringIfSet("audio_network_adaptor", audio_network_adaptor);
    // The adaptor config is a serialized proto buffer and therefore not human
    // readable. So we comment out the following line.
    // ost << ToStringIfSet("audio_network_adaptor_config",
    //     audio_network_adaptor_config);
    ost << "}";
    return ost.str();
  }

  // Audio processing that attempts to filter away the output signal from
  // later inbound pickup.
  rtc::Optional<bool> echo_cancellation;
  // Audio processing to adjust the sensitivity of the local mic dynamically.
  rtc::Optional<bool> auto_gain_control;
  // Audio processing to filter out background noise.
  rtc::Optional<bool> noise_suppression;
  // Audio processing to remove background noise of lower frequencies.
  rtc::Optional<bool> highpass_filter;
  // Audio processing to swap the left and right channels.
  rtc::Optional<bool> stereo_swapping;
  // Audio receiver jitter buffer (NetEq) max capacity in number of packets.
  rtc::Optional<int> audio_jitter_buffer_max_packets;
  // Audio receiver jitter buffer (NetEq) fast accelerate mode.
  rtc::Optional<bool> audio_jitter_buffer_fast_accelerate;
  // Audio processing to detect typing.
  rtc::Optional<bool> typing_detection;
  rtc::Optional<bool> aecm_generate_comfort_noise;
  rtc::Optional<int> adjust_agc_delta;
  rtc::Optional<bool> experimental_agc;
  rtc::Optional<bool> extended_filter_aec;
  rtc::Optional<bool> delay_agnostic_aec;
  rtc::Optional<bool> experimental_ns;
  rtc::Optional<bool> intelligibility_enhancer;
  rtc::Optional<bool> level_control;
  // Specifies an optional initialization value for the level controller.
  rtc::Optional<float> level_control_initial_peak_level_dbfs;
  // Note that tx_agc_* only applies to non-experimental AGC.
  rtc::Optional<bool> residual_echo_detector;
  rtc::Optional<uint16_t> tx_agc_target_dbov;
  rtc::Optional<uint16_t> tx_agc_digital_compression_gain;
  rtc::Optional<bool> tx_agc_limiter;
  // Enable combined audio+bandwidth BWE.
  // TODO(pthatcher): This flag is set from the
  // "googCombinedAudioVideoBwe", but not used anywhere. So delete it,
  // and check if any other AudioOptions members are unused.
  rtc::Optional<bool> combined_audio_video_bwe;
  // Enable audio network adaptor.
  rtc::Optional<bool> audio_network_adaptor;
  // Config string for audio network adaptor.
  rtc::Optional<std::string> audio_network_adaptor_config;

 private:
  template <typename T>
  static void SetFrom(rtc::Optional<T>* s, const rtc::Optional<T>& o) {
    if (o) {
      *s = o;
    }
  }
};

// Options that can be applied to a VideoMediaChannel or a VideoMediaEngine.
// Used to be flags, but that makes it hard to selectively apply options.
// We are moving all of the setting of options to structs like this,
// but some things currently still use flags.
struct VideoOptions {
  void SetAll(const VideoOptions& change) {
    SetFrom(&video_noise_reduction, change.video_noise_reduction);
    SetFrom(&screencast_min_bitrate_kbps, change.screencast_min_bitrate_kbps);
    SetFrom(&is_screencast, change.is_screencast);
  }

  bool operator==(const VideoOptions& o) const {
    return video_noise_reduction == o.video_noise_reduction &&
           screencast_min_bitrate_kbps == o.screencast_min_bitrate_kbps &&
           is_screencast == o.is_screencast;
  }
  bool operator!=(const VideoOptions& o) const { return !(*this == o); }

  std::string ToString() const {
    std::ostringstream ost;
    ost << "VideoOptions {";
    ost << ToStringIfSet("noise reduction", video_noise_reduction);
    ost << ToStringIfSet("screencast min bitrate kbps",
                         screencast_min_bitrate_kbps);
    ost << ToStringIfSet("is_screencast ", is_screencast);
    ost << "}";
    return ost.str();
  }

  // Enable denoising? This flag comes from the getUserMedia
  // constraint 'googNoiseReduction', and WebRtcVideoEngine passes it
  // on to the codec options. Disabled by default.
  rtc::Optional<bool> video_noise_reduction;
  // Force screencast to use a minimum bitrate. This flag comes from
  // the PeerConnection constraint 'googScreencastMinBitrate'. It is
  // copied to the encoder config by WebRtcVideoChannel.
  rtc::Optional<int> screencast_min_bitrate_kbps;
  // Set by screencast sources. Implies selection of encoding settings
  // suitable for screencast. Most likely not the right way to do
  // things, e.g., screencast of a text document and screencast of a
  // youtube video have different needs.
  rtc::Optional<bool> is_screencast;

 private:
  template <typename T>
  static void SetFrom(rtc::Optional<T>* s, const rtc::Optional<T>& o) {
    if (o) {
      *s = o;
    }
  }
};

// TODO(isheriff): Remove this once client usage is fixed to use RtpExtension.
struct RtpHeaderExtension {
  RtpHeaderExtension() : id(0) {}
  RtpHeaderExtension(const std::string& uri, int id) : uri(uri), id(id) {}

  std::string ToString() const {
    std::ostringstream ost;
    ost << "{";
    ost << "uri: " << uri;
    ost << ", id: " << id;
    ost << "}";
    return ost.str();
  }

  std::string uri;
  int id;
};

class MediaChannel : public sigslot::has_slots<> {
 public:
  class NetworkInterface {
   public:
    enum SocketType { ST_RTP, ST_RTCP };
    virtual bool SendPacket(rtc::CopyOnWriteBuffer* packet,
                            const rtc::PacketOptions& options) = 0;
    virtual bool SendRtcp(rtc::CopyOnWriteBuffer* packet,
                          const rtc::PacketOptions& options) = 0;
    virtual int SetOption(SocketType type, rtc::Socket::Option opt,
                          int option) = 0;
    virtual ~NetworkInterface() {}
  };

  explicit MediaChannel(const MediaConfig& config)
      : enable_dscp_(config.enable_dscp), network_interface_(NULL) {}
  MediaChannel() : enable_dscp_(false), network_interface_(NULL) {}
  virtual ~MediaChannel() {}

  // Sets the abstract interface class for sending RTP/RTCP data.
  virtual void SetInterface(NetworkInterface *iface) {
    rtc::CritScope cs(&network_interface_crit_);
    network_interface_ = iface;
    SetDscp(enable_dscp_ ? PreferredDscp() : rtc::DSCP_DEFAULT);
  }
  virtual rtc::DiffServCodePoint PreferredDscp() const {
    return rtc::DSCP_DEFAULT;
  }
  // Called when a RTP packet is received.
  virtual void OnPacketReceived(rtc::CopyOnWriteBuffer* packet,
                                const rtc::PacketTime& packet_time) = 0;
  // Called when a RTCP packet is received.
  virtual void OnRtcpReceived(rtc::CopyOnWriteBuffer* packet,
                              const rtc::PacketTime& packet_time) = 0;
  // Called when the socket's ability to send has changed.
  virtual void OnReadyToSend(bool ready) = 0;
  // Called when the network route used for sending packets changed.
  virtual void OnNetworkRouteChanged(
      const std::string& transport_name,
      const rtc::NetworkRoute& network_route) = 0;
  // Creates a new outgoing media stream with SSRCs and CNAME as described
  // by sp.
  virtual bool AddSendStream(const StreamParams& sp) = 0;
  // Removes an outgoing media stream.
  // ssrc must be the first SSRC of the media stream if the stream uses
  // multiple SSRCs.
  virtual bool RemoveSendStream(uint32_t ssrc) = 0;
  // Creates a new incoming media stream with SSRCs and CNAME as described
  // by sp.
  virtual bool AddRecvStream(const StreamParams& sp) = 0;
  // Removes an incoming media stream.
  // ssrc must be the first SSRC of the media stream if the stream uses
  // multiple SSRCs.
  virtual bool RemoveRecvStream(uint32_t ssrc) = 0;

  // Returns the absoulte sendtime extension id value from media channel.
  virtual int GetRtpSendTimeExtnId() const {
    return -1;
  }

  // Base method to send packet using NetworkInterface.
  bool SendPacket(rtc::CopyOnWriteBuffer* packet,
                  const rtc::PacketOptions& options) {
    return DoSendPacket(packet, false, options);
  }

  bool SendRtcp(rtc::CopyOnWriteBuffer* packet,
                const rtc::PacketOptions& options) {
    return DoSendPacket(packet, true, options);
  }

  int SetOption(NetworkInterface::SocketType type,
                rtc::Socket::Option opt,
                int option) {
    rtc::CritScope cs(&network_interface_crit_);
    if (!network_interface_)
      return -1;

    return network_interface_->SetOption(type, opt, option);
  }

 private:
  // This method sets DSCP |value| on both RTP and RTCP channels.
  int SetDscp(rtc::DiffServCodePoint value) {
    int ret;
    ret = SetOption(NetworkInterface::ST_RTP,
                    rtc::Socket::OPT_DSCP,
                    value);
    if (ret == 0) {
      ret = SetOption(NetworkInterface::ST_RTCP,
                      rtc::Socket::OPT_DSCP,
                      value);
    }
    return ret;
  }

  bool DoSendPacket(rtc::CopyOnWriteBuffer* packet,
                    bool rtcp,
                    const rtc::PacketOptions& options) {
    rtc::CritScope cs(&network_interface_crit_);
    if (!network_interface_)
      return false;

    return (!rtcp) ? network_interface_->SendPacket(packet, options)
                   : network_interface_->SendRtcp(packet, options);
  }

  const bool enable_dscp_;
  // |network_interface_| can be accessed from the worker_thread and
  // from any MediaEngine threads. This critical section is to protect accessing
  // of network_interface_ object.
  rtc::CriticalSection network_interface_crit_;
  NetworkInterface* network_interface_;
};

// The stats information is structured as follows:
// Media are represented by either MediaSenderInfo or MediaReceiverInfo.
// Media contains a vector of SSRC infos that are exclusively used by this
// media. (SSRCs shared between media streams can't be represented.)

// Information about an SSRC.
// This data may be locally recorded, or received in an RTCP SR or RR.
struct SsrcSenderInfo {
  SsrcSenderInfo()
      : ssrc(0),
    timestamp(0) {
  }
  uint32_t ssrc;
  double timestamp;  // NTP timestamp, represented as seconds since epoch.
};

struct SsrcReceiverInfo {
  SsrcReceiverInfo()
      : ssrc(0),
        timestamp(0) {
  }
  uint32_t ssrc;
  double timestamp;
};

struct MediaSenderInfo {
  MediaSenderInfo()
      : bytes_sent(0),
        packets_sent(0),
        packets_lost(0),
        fraction_lost(0.0),
        rtt_ms(0) {
  }
  void add_ssrc(const SsrcSenderInfo& stat) {
    local_stats.push_back(stat);
  }
  // Temporary utility function for call sites that only provide SSRC.
  // As more info is added into SsrcSenderInfo, this function should go away.
  void add_ssrc(uint32_t ssrc) {
    SsrcSenderInfo stat;
    stat.ssrc = ssrc;
    add_ssrc(stat);
  }
  // Utility accessor for clients that are only interested in ssrc numbers.
  std::vector<uint32_t> ssrcs() const {
    std::vector<uint32_t> retval;
    for (std::vector<SsrcSenderInfo>::const_iterator it = local_stats.begin();
         it != local_stats.end(); ++it) {
      retval.push_back(it->ssrc);
    }
    return retval;
  }
  // Utility accessor for clients that make the assumption only one ssrc
  // exists per media.
  // This will eventually go away.
  uint32_t ssrc() const {
    if (local_stats.size() > 0) {
      return local_stats[0].ssrc;
    } else {
      return 0;
    }
  }
  int64_t bytes_sent;
  int packets_sent;
  int packets_lost;
  float fraction_lost;
  int64_t rtt_ms;
  std::string codec_name;
  rtc::Optional<int> codec_payload_type;
  std::vector<SsrcSenderInfo> local_stats;
  std::vector<SsrcReceiverInfo> remote_stats;
};

struct MediaReceiverInfo {
  MediaReceiverInfo()
      : bytes_rcvd(0),
        packets_rcvd(0),
        packets_lost(0),
        fraction_lost(0.0) {
  }
  void add_ssrc(const SsrcReceiverInfo& stat) {
    local_stats.push_back(stat);
  }
  // Temporary utility function for call sites that only provide SSRC.
  // As more info is added into SsrcSenderInfo, this function should go away.
  void add_ssrc(uint32_t ssrc) {
    SsrcReceiverInfo stat;
    stat.ssrc = ssrc;
    add_ssrc(stat);
  }
  std::vector<uint32_t> ssrcs() const {
    std::vector<uint32_t> retval;
    for (std::vector<SsrcReceiverInfo>::const_iterator it = local_stats.begin();
         it != local_stats.end(); ++it) {
      retval.push_back(it->ssrc);
    }
    return retval;
  }
  // Utility accessor for clients that make the assumption only one ssrc
  // exists per media.
  // This will eventually go away.
  uint32_t ssrc() const {
    if (local_stats.size() > 0) {
      return local_stats[0].ssrc;
    } else {
      return 0;
    }
  }

  int64_t bytes_rcvd;
  int packets_rcvd;
  int packets_lost;
  float fraction_lost;
  std::string codec_name;
  rtc::Optional<int> codec_payload_type;
  std::vector<SsrcReceiverInfo> local_stats;
  std::vector<SsrcSenderInfo> remote_stats;
};

struct VoiceSenderInfo : public MediaSenderInfo {
  VoiceSenderInfo()
      : ext_seqnum(0),
        jitter_ms(0),
        audio_level(0),
        total_input_energy(0.0),
        total_input_duration(0.0),
        aec_quality_min(0.0),
        echo_delay_median_ms(0),
        echo_delay_std_ms(0),
        echo_return_loss(0),
        echo_return_loss_enhancement(0),
        residual_echo_likelihood(0.0f),
        residual_echo_likelihood_recent_max(0.0f),
        typing_noise_detected(false) {}

  int ext_seqnum;
  int jitter_ms;
  int audio_level;
  // See description of "totalAudioEnergy" in the WebRTC stats spec:
  // https://w3c.github.io/webrtc-stats/#dom-rtcmediastreamtrackstats-totalaudioenergy
  double total_input_energy;
  double total_input_duration;
  // TODO(bugs.webrtc.org/8572): Remove APM stats from this struct, since they
  // are no longer needed now that we have apm_statistics.
  float aec_quality_min;
  int echo_delay_median_ms;
  int echo_delay_std_ms;
  int echo_return_loss;
  int echo_return_loss_enhancement;
  float residual_echo_likelihood;
  float residual_echo_likelihood_recent_max;
  bool typing_noise_detected;
  webrtc::ANAStats ana_statistics;
  webrtc::AudioProcessingStats apm_statistics;
};

struct VoiceReceiverInfo : public MediaReceiverInfo {
  VoiceReceiverInfo()
      : ext_seqnum(0),
        jitter_ms(0),
        jitter_buffer_ms(0),
        jitter_buffer_preferred_ms(0),
        delay_estimate_ms(0),
        audio_level(0),
        total_output_energy(0.0),
        total_samples_received(0),
        total_output_duration(0.0),
        concealed_samples(0),
        concealment_events(0),
        jitter_buffer_delay_seconds(0),
        expand_rate(0),
        speech_expand_rate(0),
        secondary_decoded_rate(0),
        secondary_discarded_rate(0),
        accelerate_rate(0),
        preemptive_expand_rate(0),
        decoding_calls_to_silence_generator(0),
        decoding_calls_to_neteq(0),
        decoding_normal(0),
        decoding_plc(0),
        decoding_cng(0),
        decoding_plc_cng(0),
        decoding_muted_output(0),
        capture_start_ntp_time_ms(-1) {}

  int ext_seqnum;
  int jitter_ms;
  int jitter_buffer_ms;
  int jitter_buffer_preferred_ms;
  int delay_estimate_ms;
  int audio_level;
  // Stats below correspond to similarly-named fields in the WebRTC stats spec.
  // https://w3c.github.io/webrtc-stats/#dom-rtcmediastreamtrackstats
  double total_output_energy;
  uint64_t total_samples_received;
  double total_output_duration;
  uint64_t concealed_samples;
  uint64_t concealment_events;
  double jitter_buffer_delay_seconds;
  // Stats below DO NOT correspond directly to anything in the WebRTC stats
  // fraction of synthesized audio inserted through expansion.
  float expand_rate;
  // fraction of synthesized speech inserted through expansion.
  float speech_expand_rate;
  // fraction of data out of secondary decoding, including FEC and RED.
  float secondary_decoded_rate;
  // Fraction of secondary data, including FEC and RED, that is discarded.
  // Discarding of secondary data can be caused by the reception of the primary
  // data, obsoleting the secondary data. It can also be caused by early
  // or late arrival of secondary data. This metric is the percentage of
  // discarded secondary data since last query of receiver info.
  float secondary_discarded_rate;
  // Fraction of data removed through time compression.
  float accelerate_rate;
  // Fraction of data inserted through time stretching.
  float preemptive_expand_rate;
  int decoding_calls_to_silence_generator;
  int decoding_calls_to_neteq;
  int decoding_normal;
  int decoding_plc;
  int decoding_cng;
  int decoding_plc_cng;
  int decoding_muted_output;
  // Estimated capture start time in NTP time in ms.
  int64_t capture_start_ntp_time_ms;
};

struct VideoSenderInfo : public MediaSenderInfo {
  VideoSenderInfo()
      : packets_cached(0),
        firs_rcvd(0),
        plis_rcvd(0),
        nacks_rcvd(0),
        send_frame_width(0),
        send_frame_height(0),
        framerate_input(0),
        framerate_sent(0),
        nominal_bitrate(0),
        preferred_bitrate(0),
        adapt_reason(0),
        adapt_changes(0),
        avg_encode_ms(0),
        encode_usage_percent(0),
        frames_encoded(0),
        has_entered_low_resolution(false),
        content_type(webrtc::VideoContentType::UNSPECIFIED) {}

  std::vector<SsrcGroup> ssrc_groups;
  // TODO(hbos): Move this to |VideoMediaInfo::send_codecs|?
  std::string encoder_implementation_name;
  int packets_cached;
  int firs_rcvd;
  int plis_rcvd;
  int nacks_rcvd;
  int send_frame_width;
  int send_frame_height;
  int framerate_input;
  int framerate_sent;
  int nominal_bitrate;
  int preferred_bitrate;
  int adapt_reason;
  int adapt_changes;
  int avg_encode_ms;
  int encode_usage_percent;
  uint32_t frames_encoded;
  bool has_entered_low_resolution;
  rtc::Optional<uint64_t> qp_sum;
  webrtc::VideoContentType content_type;
};

struct VideoReceiverInfo : public MediaReceiverInfo {
  VideoReceiverInfo()
      : packets_concealed(0),
        firs_sent(0),
        plis_sent(0),
        nacks_sent(0),
        frame_width(0),
        frame_height(0),
        framerate_rcvd(0),
        framerate_decoded(0),
        framerate_output(0),
        framerate_render_input(0),
        framerate_render_output(0),
        frames_received(0),
        frames_decoded(0),
        frames_rendered(0),
        interframe_delay_max_ms(-1),
        content_type(webrtc::VideoContentType::UNSPECIFIED),
        decode_ms(0),
        max_decode_ms(0),
        jitter_buffer_ms(0),
        min_playout_delay_ms(0),
        render_delay_ms(0),
        target_delay_ms(0),
        current_delay_ms(0),
        capture_start_ntp_time_ms(-1) {}

  std::vector<SsrcGroup> ssrc_groups;
  // TODO(hbos): Move this to |VideoMediaInfo::receive_codecs|?
  std::string decoder_implementation_name;
  int packets_concealed;
  int firs_sent;
  int plis_sent;
  int nacks_sent;
  int frame_width;
  int frame_height;
  int framerate_rcvd;
  int framerate_decoded;
  int framerate_output;
  // Framerate as sent to the renderer.
  int framerate_render_input;
  // Framerate that the renderer reports.
  int framerate_render_output;
  uint32_t frames_received;
  uint32_t frames_decoded;
  uint32_t frames_rendered;
  rtc::Optional<uint64_t> qp_sum;
  int64_t interframe_delay_max_ms;

  webrtc::VideoContentType content_type;

  // All stats below are gathered per-VideoReceiver, but some will be correlated
  // across MediaStreamTracks.  NOTE(hta): when sinking stats into per-SSRC
  // structures, reflect this in the new layout.

  // Current frame decode latency.
  int decode_ms;
  // Maximum observed frame decode latency.
  int max_decode_ms;
  // Jitter (network-related) latency.
  int jitter_buffer_ms;
  // Requested minimum playout latency.
  int min_playout_delay_ms;
  // Requested latency to account for rendering delay.
  int render_delay_ms;
  // Target overall delay: network+decode+render, accounting for
  // min_playout_delay_ms.
  int target_delay_ms;
  // Current overall delay, possibly ramping towards target_delay_ms.
  int current_delay_ms;

  // Estimated capture start time in NTP time in ms.
  int64_t capture_start_ntp_time_ms;

  // Timing frame info: all important timestamps for a full lifetime of a
  // single 'timing frame'.
  rtc::Optional<webrtc::TimingFrameInfo> timing_frame_info;
};

struct DataSenderInfo : public MediaSenderInfo {
  DataSenderInfo()
      : ssrc(0) {
  }

  uint32_t ssrc;
};

struct DataReceiverInfo : public MediaReceiverInfo {
  DataReceiverInfo()
      : ssrc(0) {
  }

  uint32_t ssrc;
};

struct BandwidthEstimationInfo {
  BandwidthEstimationInfo()
      : available_send_bandwidth(0),
        available_recv_bandwidth(0),
        target_enc_bitrate(0),
        actual_enc_bitrate(0),
        retransmit_bitrate(0),
        transmit_bitrate(0),
        bucket_delay(0) {
  }

  int available_send_bandwidth;
  int available_recv_bandwidth;
  int target_enc_bitrate;
  int actual_enc_bitrate;
  int retransmit_bitrate;
  int transmit_bitrate;
  int64_t bucket_delay;
};

// Maps from payload type to |RtpCodecParameters|.
typedef std::map<int, webrtc::RtpCodecParameters> RtpCodecParametersMap;

struct VoiceMediaInfo {
  void Clear() {
    senders.clear();
    receivers.clear();
    send_codecs.clear();
    receive_codecs.clear();
  }
  std::vector<VoiceSenderInfo> senders;
  std::vector<VoiceReceiverInfo> receivers;
  RtpCodecParametersMap send_codecs;
  RtpCodecParametersMap receive_codecs;
};

struct VideoMediaInfo {
  void Clear() {
    senders.clear();
    receivers.clear();
    bw_estimations.clear();
    send_codecs.clear();
    receive_codecs.clear();
  }
  std::vector<VideoSenderInfo> senders;
  std::vector<VideoReceiverInfo> receivers;
  // Deprecated.
  // TODO(holmer): Remove once upstream projects no longer use this.
  std::vector<BandwidthEstimationInfo> bw_estimations;
  RtpCodecParametersMap send_codecs;
  RtpCodecParametersMap receive_codecs;
};

struct DataMediaInfo {
  void Clear() {
    senders.clear();
    receivers.clear();
  }
  std::vector<DataSenderInfo> senders;
  std::vector<DataReceiverInfo> receivers;
};

struct RtcpParameters {
  bool reduced_size = false;
};

template <class Codec>
struct RtpParameters {
  virtual std::string ToString() const {
    std::ostringstream ost;
    ost << "{";
    ost << "codecs: " << VectorToString(codecs) << ", ";
    ost << "extensions: " << VectorToString(extensions);
    ost << "}";
    return ost.str();
  }

  std::vector<Codec> codecs;
  std::vector<webrtc::RtpExtension> extensions;
  // TODO(pthatcher): Add streams.
  RtcpParameters rtcp;
  virtual ~RtpParameters() = default;
};

// TODO(deadbeef): Rename to RtpSenderParameters, since they're intended to
// encapsulate all the parameters needed for an RtpSender.
template <class Codec>
struct RtpSendParameters : RtpParameters<Codec> {
  std::string ToString() const override {
    std::ostringstream ost;
    ost << "{";
    ost << "codecs: " << VectorToString(this->codecs) << ", ";
    ost << "extensions: " << VectorToString(this->extensions) << ", ";
    ost << "max_bandwidth_bps: " << max_bandwidth_bps << ", ";
    ost << "}";
    return ost.str();
  }

  int max_bandwidth_bps = -1;
};

struct AudioSendParameters : RtpSendParameters<AudioCodec> {
  std::string ToString() const override {
    std::ostringstream ost;
    ost << "{";
    ost << "codecs: " << VectorToString(this->codecs) << ", ";
    ost << "extensions: " << VectorToString(this->extensions) << ", ";
    ost << "max_bandwidth_bps: " << max_bandwidth_bps << ", ";
    ost << "options: " << options.ToString();
    ost << "}";
    return ost.str();
  }

  AudioOptions options;
};

struct AudioRecvParameters : RtpParameters<AudioCodec> {
};

class VoiceMediaChannel : public MediaChannel {
 public:
  enum Error {
    ERROR_NONE = 0,                       // No error.
    ERROR_OTHER,                          // Other errors.
    ERROR_REC_DEVICE_OPEN_FAILED = 100,   // Could not open mic.
    ERROR_REC_DEVICE_MUTED,               // Mic was muted by OS.
    ERROR_REC_DEVICE_SILENT,              // No background noise picked up.
    ERROR_REC_DEVICE_SATURATION,          // Mic input is clipping.
    ERROR_REC_DEVICE_REMOVED,             // Mic was removed while active.
    ERROR_REC_RUNTIME_ERROR,              // Processing is encountering errors.
    ERROR_REC_SRTP_ERROR,                 // Generic SRTP failure.
    ERROR_REC_SRTP_AUTH_FAILED,           // Failed to authenticate packets.
    ERROR_REC_TYPING_NOISE_DETECTED,      // Typing noise is detected.
    ERROR_PLAY_DEVICE_OPEN_FAILED = 200,  // Could not open playout.
    ERROR_PLAY_DEVICE_MUTED,              // Playout muted by OS.
    ERROR_PLAY_DEVICE_REMOVED,            // Playout removed while active.
    ERROR_PLAY_RUNTIME_ERROR,             // Errors in voice processing.
    ERROR_PLAY_SRTP_ERROR,                // Generic SRTP failure.
    ERROR_PLAY_SRTP_AUTH_FAILED,          // Failed to authenticate packets.
    ERROR_PLAY_SRTP_REPLAY,               // Packet replay detected.
  };

  VoiceMediaChannel() {}
  explicit VoiceMediaChannel(const MediaConfig& config)
      : MediaChannel(config) {}
  virtual ~VoiceMediaChannel() {}
  virtual bool SetSendParameters(const AudioSendParameters& params) = 0;
  virtual bool SetRecvParameters(const AudioRecvParameters& params) = 0;
  virtual webrtc::RtpParameters GetRtpSendParameters(uint32_t ssrc) const = 0;
  virtual bool SetRtpSendParameters(
      uint32_t ssrc,
      const webrtc::RtpParameters& parameters) = 0;
  // Get the receive parameters for the incoming stream identified by |ssrc|.
  // If |ssrc| is 0, retrieve the receive parameters for the default receive
  // stream, which is used when SSRCs are not signaled. Note that calling with
  // an |ssrc| of 0 will return encoding parameters with an unset |ssrc|
  // member.
  virtual webrtc::RtpParameters GetRtpReceiveParameters(
      uint32_t ssrc) const = 0;
  virtual bool SetRtpReceiveParameters(
      uint32_t ssrc,
      const webrtc::RtpParameters& parameters) = 0;
  // Starts or stops playout of received audio.
  virtual void SetPlayout(bool playout) = 0;
  // Starts or stops sending (and potentially capture) of local audio.
  virtual void SetSend(bool send) = 0;
  // Configure stream for sending.
  virtual bool SetAudioSend(uint32_t ssrc,
                            bool enable,
                            const AudioOptions* options,
                            AudioSource* source) = 0;
  // Gets current energy levels for all incoming streams.
  virtual bool GetActiveStreams(AudioInfo::StreamList* actives) = 0;
  // Get the current energy level of the stream sent to the speaker.
  virtual int GetOutputLevel() = 0;
  // Set speaker output volume of the specified ssrc.
  virtual bool SetOutputVolume(uint32_t ssrc, double volume) = 0;
  // Returns if the telephone-event has been negotiated.
  virtual bool CanInsertDtmf() = 0;
  // Send a DTMF |event|. The DTMF out-of-band signal will be used.
  // The |ssrc| should be either 0 or a valid send stream ssrc.
  // The valid value for the |event| are 0 to 15 which corresponding to
  // DTMF event 0-9, *, #, A-D.
  virtual bool InsertDtmf(uint32_t ssrc, int event, int duration) = 0;
  // Gets quality stats for the channel.
  virtual bool GetStats(VoiceMediaInfo* info) = 0;

  virtual void SetRawAudioSink(
      uint32_t ssrc,
      std::unique_ptr<webrtc::AudioSinkInterface> sink) = 0;

  virtual std::vector<webrtc::RtpSource> GetSources(uint32_t ssrc) const = 0;
};

// TODO(deadbeef): Rename to VideoSenderParameters, since they're intended to
// encapsulate all the parameters needed for a video RtpSender.
struct VideoSendParameters : RtpSendParameters<VideoCodec> {
  // Use conference mode? This flag comes from the remote
  // description's SDP line 'a=x-google-flag:conference', copied over
  // by VideoChannel::SetRemoteContent_w, and ultimately used by
  // conference mode screencast logic in
  // WebRtcVideoChannel::WebRtcVideoSendStream::CreateVideoEncoderConfig.
  // The special screencast behaviour is disabled by default.
  bool conference_mode = false;
};

// TODO(deadbeef): Rename to VideoReceiverParameters, since they're intended to
// encapsulate all the parameters needed for a video RtpReceiver.
struct VideoRecvParameters : RtpParameters<VideoCodec> {
};

class VideoMediaChannel : public MediaChannel {
 public:
  enum Error {
    ERROR_NONE = 0,                       // No error.
    ERROR_OTHER,                          // Other errors.
    ERROR_REC_DEVICE_OPEN_FAILED = 100,   // Could not open camera.
    ERROR_REC_DEVICE_NO_DEVICE,           // No camera.
    ERROR_REC_DEVICE_IN_USE,              // Device is in already use.
    ERROR_REC_DEVICE_REMOVED,             // Device is removed.
    ERROR_REC_SRTP_ERROR,                 // Generic sender SRTP failure.
    ERROR_REC_SRTP_AUTH_FAILED,           // Failed to authenticate packets.
    ERROR_REC_CPU_MAX_CANT_DOWNGRADE,     // Can't downgrade capture anymore.
    ERROR_PLAY_SRTP_ERROR = 200,          // Generic receiver SRTP failure.
    ERROR_PLAY_SRTP_AUTH_FAILED,          // Failed to authenticate packets.
    ERROR_PLAY_SRTP_REPLAY,               // Packet replay detected.
  };

  VideoMediaChannel() {}
  explicit VideoMediaChannel(const MediaConfig& config)
      : MediaChannel(config) {}
  virtual ~VideoMediaChannel() {}

  virtual bool SetSendParameters(const VideoSendParameters& params) = 0;
  virtual bool SetRecvParameters(const VideoRecvParameters& params) = 0;
  virtual webrtc::RtpParameters GetRtpSendParameters(uint32_t ssrc) const = 0;
  virtual bool SetRtpSendParameters(
      uint32_t ssrc,
      const webrtc::RtpParameters& parameters) = 0;
  // Get the receive parameters for the incoming stream identified by |ssrc|.
  // If |ssrc| is 0, retrieve the receive parameters for the default receive
  // stream, which is used when SSRCs are not signaled. Note that calling with
  // an |ssrc| of 0 will return encoding parameters with an unset |ssrc|
  // member.
  virtual webrtc::RtpParameters GetRtpReceiveParameters(
      uint32_t ssrc) const = 0;
  virtual bool SetRtpReceiveParameters(
      uint32_t ssrc,
      const webrtc::RtpParameters& parameters) = 0;
  // Gets the currently set codecs/payload types to be used for outgoing media.
  virtual bool GetSendCodec(VideoCodec* send_codec) = 0;
  // Starts or stops transmission (and potentially capture) of local video.
  virtual bool SetSend(bool send) = 0;
  // Configure stream for sending and register a source.
  // The |ssrc| must correspond to a registered send stream.
  virtual bool SetVideoSend(
      uint32_t ssrc,
      bool enable,
      const VideoOptions* options,
      rtc::VideoSourceInterface<webrtc::VideoFrame>* source) = 0;
  // Sets the sink object to be used for the specified stream.
  // If SSRC is 0, the sink is used for the 'default' stream.
  virtual bool SetSink(uint32_t ssrc,
                       rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) = 0;
  // This fills the "bitrate parts" (rtx, video bitrate) of the
  // BandwidthEstimationInfo, since that part that isn't possible to get
  // through webrtc::Call::GetStats, as they are statistics of the send
  // streams.
  // TODO(holmer): We should change this so that either BWE graphs doesn't
  // need access to bitrates of the streams, or change the (RTC)StatsCollector
  // so that it's getting the send stream stats separately by calling
  // GetStats(), and merges with BandwidthEstimationInfo by itself.
  virtual void FillBitrateInfo(BandwidthEstimationInfo* bwe_info) = 0;
  // Gets quality stats for the channel.
  virtual bool GetStats(VideoMediaInfo* info) = 0;
};

enum DataMessageType {
  // Chrome-Internal use only.  See SctpDataMediaChannel for the actual PPID
  // values.
  DMT_NONE = 0,
  DMT_CONTROL = 1,
  DMT_BINARY = 2,
  DMT_TEXT = 3,
};

// Info about data received in DataMediaChannel.  For use in
// DataMediaChannel::SignalDataReceived and in all of the signals that
// signal fires, on up the chain.
struct ReceiveDataParams {
  // The in-packet stream indentifier.
  // RTP data channels use SSRCs, SCTP data channels use SIDs.
  union {
    uint32_t ssrc;
    int sid;
  };
  // The type of message (binary, text, or control).
  DataMessageType type;
  // A per-stream value incremented per packet in the stream.
  int seq_num;
  // A per-stream value monotonically increasing with time.
  int timestamp;

  ReceiveDataParams() : sid(0), type(DMT_TEXT), seq_num(0), timestamp(0) {}
};

struct SendDataParams {
  // The in-packet stream indentifier.
  // RTP data channels use SSRCs, SCTP data channels use SIDs.
  union {
    uint32_t ssrc;
    int sid;
  };
  // The type of message (binary, text, or control).
  DataMessageType type;

  // For SCTP, whether to send messages flagged as ordered or not.
  // If false, messages can be received out of order.
  bool ordered;
  // For SCTP, whether the messages are sent reliably or not.
  // If false, messages may be lost.
  bool reliable;
  // For SCTP, if reliable == false, provide partial reliability by
  // resending up to this many times.  Either count or millis
  // is supported, not both at the same time.
  int max_rtx_count;
  // For SCTP, if reliable == false, provide partial reliability by
  // resending for up to this many milliseconds.  Either count or millis
  // is supported, not both at the same time.
  int max_rtx_ms;

  SendDataParams()
      : sid(0),
        type(DMT_TEXT),
        // TODO(pthatcher): Make these true by default?
        ordered(false),
        reliable(false),
        max_rtx_count(0),
        max_rtx_ms(0) {}
};

enum SendDataResult { SDR_SUCCESS, SDR_ERROR, SDR_BLOCK };

struct DataSendParameters : RtpSendParameters<DataCodec> {
  std::string ToString() const {
    std::ostringstream ost;
    // Options and extensions aren't used.
    ost << "{";
    ost << "codecs: " << VectorToString(codecs) << ", ";
    ost << "max_bandwidth_bps: " << max_bandwidth_bps;
    ost << "}";
    return ost.str();
  }
};

struct DataRecvParameters : RtpParameters<DataCodec> {
};

class DataMediaChannel : public MediaChannel {
 public:
  enum Error {
    ERROR_NONE = 0,                       // No error.
    ERROR_OTHER,                          // Other errors.
    ERROR_SEND_SRTP_ERROR = 200,          // Generic SRTP failure.
    ERROR_SEND_SRTP_AUTH_FAILED,          // Failed to authenticate packets.
    ERROR_RECV_SRTP_ERROR,                // Generic SRTP failure.
    ERROR_RECV_SRTP_AUTH_FAILED,          // Failed to authenticate packets.
    ERROR_RECV_SRTP_REPLAY,               // Packet replay detected.
  };

  DataMediaChannel() {}
  explicit DataMediaChannel(const MediaConfig& config) : MediaChannel(config) {}
  virtual ~DataMediaChannel() {}

  virtual bool SetSendParameters(const DataSendParameters& params) = 0;
  virtual bool SetRecvParameters(const DataRecvParameters& params) = 0;

  // TODO(pthatcher): Implement this.
  virtual bool GetStats(DataMediaInfo* info) { return true; }

  virtual bool SetSend(bool send) = 0;
  virtual bool SetReceive(bool receive) = 0;

  virtual void OnNetworkRouteChanged(const std::string& transport_name,
                                     const rtc::NetworkRoute& network_route) {}

  virtual bool SendData(
      const SendDataParams& params,
      const rtc::CopyOnWriteBuffer& payload,
      SendDataResult* result = NULL) = 0;
  // Signals when data is received (params, data, len)
  sigslot::signal3<const ReceiveDataParams&,
                   const char*,
                   size_t> SignalDataReceived;
  // Signal when the media channel is ready to send the stream. Arguments are:
  //     writable(bool)
  sigslot::signal1<bool> SignalReadyToSend;
};

}  // namespace cricket

#endif  // MEDIA_BASE_MEDIACHANNEL_H_