diff --git a/call/adaptation/resource_adaptation_module_interface.h b/call/adaptation/resource_adaptation_module_interface.h
index 074857d..4f4a302 100644
--- a/call/adaptation/resource_adaptation_module_interface.h
+++ b/call/adaptation/resource_adaptation_module_interface.h
@@ -113,7 +113,13 @@
   // TODO(hbos): If we take frame rate into account perhaps it would be valid to
   // adapt down in frame rate as well.
   virtual void OnFrameDroppedDueToSize() = 0;
-  // 2.ii) An input frame is about to be encoded. It may have been cropped and
+  // 2.ii) If the frame will not be dropped due to size then signal that it may
+  // get encoded. However the frame is not guaranteed to be encoded right away
+  // or ever (for example if encoding is paused).
+  // TODO(eshr): Try replace OnMaybeEncodeFrame and merge behaviour into
+  // EncodeStarted.
+  virtual void OnMaybeEncodeFrame() = 0;
+  // 2.iii) An input frame is about to be encoded. It may have been cropped and
   // have different dimensions than what was observed at OnFrame(). Next
   // up: encoding completes or fails, see OnEncodeCompleted(). There is
   // currently no signal for encode failure.
diff --git a/video/overuse_frame_detector_resource_adaptation_module.cc b/video/overuse_frame_detector_resource_adaptation_module.cc
index 4403857..c67e47e 100644
--- a/video/overuse_frame_detector_resource_adaptation_module.cc
+++ b/video/overuse_frame_detector_resource_adaptation_module.cc
@@ -66,6 +66,10 @@
   return source_restrictions;
 }
 
+// The maximum number of frames to drop at beginning of stream
+// to try and achieve desired bitrate.
+const int kMaxInitialFramedrop = 4;
+
 }  // namespace
 
 // VideoSourceRestrictor is responsible for keeping track of current
@@ -361,7 +365,8 @@
       target_bitrate_bps_(absl::nullopt),
       quality_scaler_(nullptr),
       encoder_settings_(absl::nullopt),
-      encoder_stats_observer_(encoder_stats_observer) {
+      encoder_stats_observer_(encoder_stats_observer),
+      initial_framedrop_(0) {
   RTC_DCHECK(adaptation_listener_);
   RTC_DCHECK(overuse_detector_);
   RTC_DCHECK(encoder_stats_observer_);
@@ -459,6 +464,7 @@
           AdaptationObserverInterface::AdaptReason::kQuality) > res_count) {
     encoder_stats_observer_->OnInitialQualityResolutionAdaptDown();
   }
+  ++initial_framedrop_;
 }
 
 void OveruseFrameDetectorResourceAdaptationModule::OnEncodeStarted(
@@ -499,13 +505,29 @@
   }
 }
 
+void OveruseFrameDetectorResourceAdaptationModule::OnMaybeEncodeFrame() {
+  initial_framedrop_ = kMaxInitialFramedrop;
+}
+
+bool OveruseFrameDetectorResourceAdaptationModule::DropInitialFrames() const {
+  return initial_framedrop_ < kMaxInitialFramedrop;
+}
+
+void OveruseFrameDetectorResourceAdaptationModule::ResetInitialFrameDropping() {
+  initial_framedrop_ = 0;
+}
+
 void OveruseFrameDetectorResourceAdaptationModule::UpdateQualityScalerSettings(
     absl::optional<VideoEncoder::QpThresholds> qp_thresholds) {
   if (qp_thresholds.has_value()) {
     quality_scaler_ =
         std::make_unique<QualityScaler>(this, qp_thresholds.value());
+    // Restart frame drops due to size.
+    initial_framedrop_ = 0;
   } else {
     quality_scaler_ = nullptr;
+    // Quality scaling disabled so we shouldn't drop initial frames.
+    initial_framedrop_ = kMaxInitialFramedrop;
   }
 }
 
diff --git a/video/overuse_frame_detector_resource_adaptation_module.h b/video/overuse_frame_detector_resource_adaptation_module.h
index 21c055b..379e1ad 100644
--- a/video/overuse_frame_detector_resource_adaptation_module.h
+++ b/video/overuse_frame_detector_resource_adaptation_module.h
@@ -79,12 +79,16 @@
 
   void OnFrame(const VideoFrame& frame) override;
   void OnFrameDroppedDueToSize() override;
+  void OnMaybeEncodeFrame() override;
   void OnEncodeStarted(const VideoFrame& cropped_frame,
                        int64_t time_when_first_seen_us) override;
   void OnEncodeCompleted(const EncodedImage& encoded_image,
                          int64_t time_sent_in_us,
                          absl::optional<int> encode_duration_us) override;
   void OnFrameDropped(EncodedImageCallback::DropReason reason) override;
+  bool DropInitialFrames() const;
+  // TODO(eshr): Remove once all qp-scaling is in this class.
+  void ResetInitialFrameDropping();
 
   // Use nullopt to disable quality scaling.
   void UpdateQualityScalerSettings(
@@ -210,6 +214,8 @@
   std::unique_ptr<QualityScaler> quality_scaler_;
   absl::optional<EncoderSettings> encoder_settings_;
   VideoStreamEncoderObserver* const encoder_stats_observer_;
+  // Counts how many frames we've dropped in the initial framedrop phase.
+  int initial_framedrop_;
 };
 
 }  // namespace webrtc
diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc
index cf71271..e1fdb8c 100644
--- a/video/video_stream_encoder.cc
+++ b/video/video_stream_encoder.cc
@@ -52,10 +52,6 @@
 
 constexpr char kFrameDropperFieldTrial[] = "WebRTC-FrameDropper";
 
-// The maximum number of frames to drop at beginning of stream
-// to try and achieve desired bitrate.
-const int kMaxInitialFramedrop = 4;
-
 // Averaging window spanning 90 frames at default 30fps, matching old media
 // optimization module defaults.
 const int64_t kFrameRateAvergingWindowSizeMs = (1000 / 30) * 90;
@@ -258,7 +254,6 @@
     TaskQueueFactory* task_queue_factory)
     : shutdown_event_(true /* manual_reset */, false),
       number_of_cores_(number_of_cores),
-      initial_framedrop_(0),
       quality_rampup_done_(false),
       quality_rampup_experiment_(QualityRampupExperiment::ParseSettings()),
       quality_scaling_experiment_enabled_(QualityScalingExperiment::Enabled()),
@@ -798,11 +793,9 @@
       resource_adaptation_module_->UpdateQualityScalerSettings(
           experimental_thresholds ? *experimental_thresholds
                                   : *(scaling_settings.thresholds));
-      initial_framedrop_ = 0;
     }
   } else {
     resource_adaptation_module_->UpdateQualityScalerSettings(absl::nullopt);
-    initial_framedrop_ = kMaxInitialFramedrop;
   }
 
   QualityScaler* quality_scaler = resource_adaptation_module_->quality_scaler();
@@ -1112,7 +1105,6 @@
   if (DropDueToSize(video_frame.size())) {
     RTC_LOG(LS_INFO) << "Dropping frame. Too large for target bitrate.";
     resource_adaptation_module_->OnFrameDroppedDueToSize();
-    ++initial_framedrop_;
     // Storing references to a native buffer risks blocking frame capture.
     if (video_frame.video_frame_buffer()->type() !=
         VideoFrameBuffer::Type::kNative) {
@@ -1126,7 +1118,7 @@
     }
     return;
   }
-  initial_framedrop_ = kMaxInitialFramedrop;
+  resource_adaptation_module_->OnMaybeEncodeFrame();
 
   if (!quality_rampup_done_ && TryQualityRampup(now_ms) &&
       resource_adaptation_module_->GetConstAdaptCounter().ResolutionCount(
@@ -1608,7 +1600,7 @@
       RTC_LOG(LS_INFO) << "Reset initial_framedrop_. Start bitrate: "
                        << set_start_bitrate_bps_
                        << ", target bitrate: " << target_bitrate.bps();
-      initial_framedrop_ = 0;
+      resource_adaptation_module_->ResetInitialFrameDropping();
       has_seen_first_bwe_drop_ = true;
     }
   }
@@ -1648,7 +1640,7 @@
 }
 
 bool VideoStreamEncoder::DropDueToSize(uint32_t pixel_count) const {
-  if (initial_framedrop_ >= kMaxInitialFramedrop ||
+  if (!resource_adaptation_module_->DropInitialFrames() ||
       !encoder_target_bitrate_bps_.has_value()) {
     return false;
   }
diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h
index 97a4a66..d0b037d 100644
--- a/video/video_stream_encoder.h
+++ b/video/video_stream_encoder.h
@@ -213,8 +213,6 @@
   rtc::Event shutdown_event_;
 
   const uint32_t number_of_cores_;
-  // Counts how many frames we've dropped in the initial framedrop phase.
-  int initial_framedrop_;
   bool quality_rampup_done_ RTC_GUARDED_BY(&encoder_queue_);
   QualityRampupExperiment quality_rampup_experiment_
       RTC_GUARDED_BY(&encoder_queue_);