Add new consecutiveDroppedFrames callback

This is similar to the `droppedFrames` callback, but represents the number of consecutive frames that we dropped before rendering a frame or seeking or stopping the renderer.

While we already have a `maxConsecutiveDroppedFrame` available in the `DecoderCounters`, this doesn't provide enough visibility into the actual statistics of dropped frames.

If we get 200 dropped frames and a `maxConsecutive` of 20, we don't know if we dropped 20 frames in a row once and then dropped a single frame 180 times or if we dropped 20 frames 10 times.

We could add some code on our `OnDroppedFrames` callback to estimate if two calls are for consecutive frames, but that seems very fragile.

Specifying when to invoke the callback is controlled by `minConsecutiveDroppedFramesToNotify` similar to the `maxDroppedFramesToNotify` but that would only notify if more than X consecutive frames were dropped.

Adding support for both `MediaCodecVideoRenderer` and `DecoderVideoRenderer`.

Author: khouzam

libraries/decoder_av1/src/main/java/androidx/media3/decoder/av1/Libgav1VideoRenderer.java

@@ -78,17 +78,22 @@ public class Libgav1VideoRenderer extends DecoderVideoRenderer {
    * @param eventListener A listener of events. May be null if delivery of events is not required.
    * @param maxDroppedFramesToNotify The maximum number of frames that can be dropped between
    *     invocations of {@link VideoRendererEventListener#onDroppedFrames(int, long)}.
+   * @param minConsecutiveDroppedFramesToNotify The minimum number of consecutive frames that must
+   *     be dropped for {@link VideoRendererEventListener#onConsecutiveDroppedFrames(int, long)} to
+   *     be called.
    */
   public Libgav1VideoRenderer(
       long allowedJoiningTimeMs,
       @Nullable Handler eventHandler,
       @Nullable VideoRendererEventListener eventListener,
-      int maxDroppedFramesToNotify) {
+      int maxDroppedFramesToNotify,
+      int minConsecutiveDroppedFramesToNotify) {
     this(
         allowedJoiningTimeMs,
         eventHandler,
         eventListener,
         maxDroppedFramesToNotify,
+        minConsecutiveDroppedFramesToNotify,
         THREAD_COUNT_AUTODETECT,
         DEFAULT_NUM_OF_INPUT_BUFFERS,
         DEFAULT_NUM_OF_OUTPUT_BUFFERS);
@@ -104,6 +109,9 @@ public Libgav1VideoRenderer(
    * @param eventListener A listener of events. May be null if delivery of events is not required.
    * @param maxDroppedFramesToNotify The maximum number of frames that can be dropped between
    *     invocations of {@link VideoRendererEventListener#onDroppedFrames(int, long)}.
+   * @param minConsecutiveDroppedFramesToNotify The minimum number of consecutive frames that must
+   *     be dropped for {@link VideoRendererEventListener#onConsecutiveDroppedFrames(int, long)} to
+   *     be called.
    * @param threads Number of threads libgav1 will use to decode. If {@link
    *     #THREAD_COUNT_AUTODETECT} is passed, then the number of threads to use is autodetected
    *     based on CPU capabilities.
@@ -115,10 +123,16 @@ public Libgav1VideoRenderer(
       @Nullable Handler eventHandler,
       @Nullable VideoRendererEventListener eventListener,
       int maxDroppedFramesToNotify,
+      int minConsecutiveDroppedFramesToNotify,
       int threads,
       int numInputBuffers,
       int numOutputBuffers) {
-    super(allowedJoiningTimeMs, eventHandler, eventListener, maxDroppedFramesToNotify);
+    super(
+      allowedJoiningTimeMs,
+      eventHandler,
+      eventListener,
+      maxDroppedFramesToNotify,
+      minConsecutiveDroppedFramesToNotify);
     this.threads = threads;
     this.numInputBuffers = numInputBuffers;
     this.numOutputBuffers = numOutputBuffers;

libraries/decoder_ffmpeg/src/main/java/androidx/media3/decoder/ffmpeg/ExperimentalFfmpegVideoRenderer.java

@@ -57,13 +57,22 @@ public final class ExperimentalFfmpegVideoRenderer extends DecoderVideoRenderer
    * @param eventListener A listener of events. May be null if delivery of events is not required.
    * @param maxDroppedFramesToNotify The maximum number of frames that can be dropped between
    *     invocations of {@link VideoRendererEventListener#onDroppedFrames(int, long)}.
+   * @param minConsecutiveDroppedFramesToNotify The minimum number of consecutive frames that must
+   *     be dropped for {@link VideoRendererEventListener#onConsecutiveDroppedFrames(int, long)} to
+   *     be called.
    */
   public ExperimentalFfmpegVideoRenderer(
       long allowedJoiningTimeMs,
       @Nullable Handler eventHandler,
       @Nullable VideoRendererEventListener eventListener,
-      int maxDroppedFramesToNotify) {
-    super(allowedJoiningTimeMs, eventHandler, eventListener, maxDroppedFramesToNotify);
+      int maxDroppedFramesToNotify,
+      int minConsecutiveDroppedFramesToNotify) {
+    super(
+      allowedJoiningTimeMs,
+      eventHandler,
+      eventListener,
+      maxDroppedFramesToNotify,
+      minConsecutiveDroppedFramesToNotify);
     // TODO: Implement.
   }
 

libraries/decoder_vp9/src/main/java/androidx/media3/decoder/vp9/LibvpxVideoRenderer.java

@@ -65,7 +65,7 @@ public class LibvpxVideoRenderer extends DecoderVideoRenderer {
    *     can attempt to seamlessly join an ongoing playback.
    */
   public LibvpxVideoRenderer(long allowedJoiningTimeMs) {
-    this(allowedJoiningTimeMs, null, null, 0);
+    this(allowedJoiningTimeMs, null, null, 0, 0);
   }
 
   /**
@@ -78,17 +78,22 @@ public LibvpxVideoRenderer(long allowedJoiningTimeMs) {
    * @param eventListener A listener of events. May be null if delivery of events is not required.
    * @param maxDroppedFramesToNotify The maximum number of frames that can be dropped between
    *     invocations of {@link VideoRendererEventListener#onDroppedFrames(int, long)}.
+   * @param minConsecutiveDroppedFramesToNotify The minimum number of consecutive frames that must
+   *     be dropped for {@link VideoRendererEventListener#onConsecutiveDroppedFrames(int, long)} to
+   *     be called.
    */
   public LibvpxVideoRenderer(
       long allowedJoiningTimeMs,
       @Nullable Handler eventHandler,
       @Nullable VideoRendererEventListener eventListener,
-      int maxDroppedFramesToNotify) {
+      int maxDroppedFramesToNotify,
+      int minConsecutiveDroppedFramesToNotify) {
     this(
         allowedJoiningTimeMs,
         eventHandler,
         eventListener,
         maxDroppedFramesToNotify,
+        minConsecutiveDroppedFramesToNotify,
         getRuntime().availableProcessors(),
         /* numInputBuffers= */ 4,
         /* numOutputBuffers= */ 4);
@@ -104,6 +109,9 @@ public LibvpxVideoRenderer(
    * @param eventListener A listener of events. May be null if delivery of events is not required.
    * @param maxDroppedFramesToNotify The maximum number of frames that can be dropped between
    *     invocations of {@link VideoRendererEventListener#onDroppedFrames(int, long)}.
+   * @param minConsecutiveDroppedFramesToNotify The minimum number of consecutive frames that must
+   *     be dropped for {@link VideoRendererEventListener#onConsecutiveDroppedFrames(int, long)} to
+   *     be called.
    * @param threads Number of threads libvpx will use to decode.
    * @param numInputBuffers Number of input buffers.
    * @param numOutputBuffers Number of output buffers.
@@ -113,10 +121,16 @@ public LibvpxVideoRenderer(
       @Nullable Handler eventHandler,
       @Nullable VideoRendererEventListener eventListener,
       int maxDroppedFramesToNotify,
+      int minConsecutiveDroppedFramesToNotify,
       int threads,
       int numInputBuffers,
       int numOutputBuffers) {
-    super(allowedJoiningTimeMs, eventHandler, eventListener, maxDroppedFramesToNotify);
+    super(
+      allowedJoiningTimeMs,
+      eventHandler,
+      eventListener,
+      maxDroppedFramesToNotify,
+      minConsecutiveDroppedFramesToNotify);
     this.threads = threads;
     this.numInputBuffers = numInputBuffers;
     this.numOutputBuffers = numOutputBuffers;

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/DefaultRenderersFactory.java

@@ -96,6 +96,12 @@ public class DefaultRenderersFactory implements RenderersFactory {
    */
   public static final int MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY = 50;
 
+  /**
+   * The minimum number of consecutive video frames that would be dropped to
+   * consider invoking {@link VideoRendererEventListener#onConsecutiveDroppedFrames(int, long)}.
+   */
+  public static final int MIN_CONSECUTIVE_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY = 5;
+
   private static final String TAG = "DefaultRenderersFactory";
 
   private final Context context;
@@ -347,7 +353,8 @@ protected void buildVideoRenderers(
             enableDecoderFallback,
             eventHandler,
             eventListener,
-            MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY);
+            MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY,
+            MIN_CONSECUTIVE_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY);
     out.add(videoRenderer);
 
     if (extensionRendererMode == EXTENSION_RENDERER_MODE_OFF) {
@@ -366,14 +373,16 @@ protected void buildVideoRenderers(
               long.class,
               android.os.Handler.class,
               androidx.media3.exoplayer.video.VideoRendererEventListener.class,
+              int.class,
               int.class);
       Renderer renderer =
           (Renderer)
               constructor.newInstance(
                   allowedVideoJoiningTimeMs,
                   eventHandler,
                   eventListener,
-                  MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY);
+                  MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY,
+                  MIN_CONSECUTIVE_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY);
       out.add(extensionRendererIndex++, renderer);
       Log.i(TAG, "Loaded LibvpxVideoRenderer.");
     } catch (ClassNotFoundException e) {
@@ -391,14 +400,16 @@ protected void buildVideoRenderers(
               long.class,
               android.os.Handler.class,
               androidx.media3.exoplayer.video.VideoRendererEventListener.class,
+              int.class,
               int.class);
       Renderer renderer =
           (Renderer)
               constructor.newInstance(
                   allowedVideoJoiningTimeMs,
                   eventHandler,
                   eventListener,
-                  MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY);
+                  MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY,
+                  MIN_CONSECUTIVE_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY);
       out.add(extensionRendererIndex++, renderer);
       Log.i(TAG, "Loaded Libgav1VideoRenderer.");
     } catch (ClassNotFoundException e) {
@@ -417,14 +428,16 @@ protected void buildVideoRenderers(
               long.class,
               android.os.Handler.class,
               androidx.media3.exoplayer.video.VideoRendererEventListener.class,
+              int.class,
               int.class);
       Renderer renderer =
           (Renderer)
               constructor.newInstance(
                   allowedVideoJoiningTimeMs,
                   eventHandler,
                   eventListener,
-                  MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY);
+                  MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY,
+                  MIN_CONSECUTIVE_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY);
       out.add(extensionRendererIndex++, renderer);
       Log.i(TAG, "Loaded FfmpegVideoRenderer.");
     } catch (ClassNotFoundException e) {

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImpl.java

@@ -3033,6 +3033,11 @@ public void onDroppedFrames(int count, long elapsed) {
       analyticsCollector.onDroppedFrames(count, elapsed);
     }
 
+    @Override
+    public void onConsecutiveDroppedFrames(int count, long elapsed) {
+      analyticsCollector.onConsecutiveDroppedFrames(count, elapsed);
+    }
+
     @Override
     public void onVideoSizeChanged(VideoSize newVideoSize) {
       videoSize = newVideoSize;

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/analytics/AnalyticsCollector.java

@@ -250,6 +250,18 @@ void onVideoInputFormatChanged(
    */
   void onDroppedFrames(int count, long elapsedMs);
 
+  /**
+   * Called to report the number of consecutive frames dropped by the video renderer. Consecutive
+   * dropped frames are reported once a frame is renderered after a period in which more frames were
+   * dropped consecutively than the specified threshold.
+   *
+   * @param count The number of consecutive dropped frames.
+   * @param elapsedMs The duration in milliseconds over which the consecutive frames were dropped.
+   *     This duration is timed from when the first frame was dropped, until the time the renderer
+   *     succesfully rendered a frame or the rendered was interrupted (stopped, seeked, disabled).
+   */
+  void onConsecutiveDroppedFrames(int count, long elapsedMs);
+
   /**
    * Called when a video decoder is released.
    *

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/analytics/AnalyticsListener.java

@@ -235,7 +235,8 @@ public int size() {
     EVENT_VIDEO_CODEC_ERROR,
     EVENT_AUDIO_TRACK_INITIALIZED,
     EVENT_AUDIO_TRACK_RELEASED,
-    EVENT_RENDERER_READY_CHANGED
+    EVENT_RENDERER_READY_CHANGED,
+    EVENT_CONSECUTIVE_DROPPED_VIDEO_FRAMES,
   })
   @interface EventFlags {}
 
@@ -448,6 +449,9 @@ public int size() {
   /** A renderer changed its readiness for playback. */
   @UnstableApi int EVENT_RENDERER_READY_CHANGED = 1033;
 
+  /** Consecutive video frames have been dropped. */
+  @UnstableApi int EVENT_CONSECUTIVE_DROPPED_VIDEO_FRAMES = 1034;
+
   /** Time information of an event. */
   @UnstableApi
   final class EventTime {
@@ -1244,6 +1248,18 @@ default void onVideoInputFormatChanged(
   @UnstableApi
   default void onDroppedVideoFrames(EventTime eventTime, int droppedFrames, long elapsedMs) {}
 
+  /**
+   * Called after consecutive video frames have been dropped.
+   *
+   * @param eventTime The event time.
+   * @param consecutiveDroppedFrames The number of consecutive frames that have been dropped before the
+   *     last rendered frame.
+   * @param elapsedMs The duration in milliseconds over which the frames were dropped. This duration
+   *     is timed from the first dropped framed in the sequence.
+   */
+  @UnstableApi
+  default void onConsecutiveDroppedVideoFrames(EventTime eventTime, int droppedFrames, long elapsedMs) {}
+
   /**
    * Called when a video renderer releases a decoder.
    *

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/analytics/DefaultAnalyticsCollector.java

@@ -340,6 +340,15 @@ public final void onDroppedFrames(int count, long elapsedMs) {
         listener -> listener.onDroppedVideoFrames(eventTime, count, elapsedMs));
   }
 
+  @Override
+  public final void onConsecutiveDroppedFrames(int count, long elapsedMs) {
+    EventTime eventTime = generatePlayingMediaPeriodEventTime();
+    sendEvent(
+        eventTime,
+        AnalyticsListener.EVENT_CONSECUTIVE_DROPPED_VIDEO_FRAMES,
+        listener -> listener.onConsecutiveDroppedVideoFrames(eventTime, count, elapsedMs));
+  }
+
   @Override
   public final void onVideoDecoderReleased(String decoderName) {
     EventTime eventTime = generateReadingMediaPeriodEventTime();

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/util/EventLogger.java

@@ -451,6 +451,13 @@ public void onDroppedVideoFrames(EventTime eventTime, int droppedFrames, long el
     logd(eventTime, "droppedFrames", Integer.toString(droppedFrames));
   }
 
+  @UnstableApi
+  @Override
+  public void onConsecutiveDroppedVideoFrames(
+      EventTime eventTime, int consecutiveDroppedFrames, long elapsedMs) {
+    logd(eventTime, "consecutiveDroppedFrames", Integer.toString(consecutiveDroppedFrames));
+  }
+
   @UnstableApi
   @Override
   public void onVideoDecoderReleased(EventTime eventTime, String decoderName) {

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/DecoderVideoRenderer.java

@@ -113,6 +113,7 @@ public abstract class DecoderVideoRenderer extends BaseRenderer {
 
   private final long allowedJoiningTimeMs;
   private final int maxDroppedFramesToNotify;
+  private final int minConsecutiveDroppedFramesToNotify;
   private final EventDispatcher eventDispatcher;
   private final TimedValueQueue<Format> formatQueue;
   private final DecoderInputBuffer flagsOnlyBuffer;
@@ -150,6 +151,7 @@ public abstract class DecoderVideoRenderer extends BaseRenderer {
 
   private long droppedFrameAccumulationStartTimeMs;
   private int droppedFrames;
+  private long consecutiveDroppedFrameAccumulationStartTimeMs;
   private int consecutiveDroppedFrameCount;
   private int buffersInCodecCount;
   private long lastRenderTimeUs;
@@ -165,15 +167,20 @@ public abstract class DecoderVideoRenderer extends BaseRenderer {
    * @param eventListener A listener of events. May be null if delivery of events is not required.
    * @param maxDroppedFramesToNotify The maximum number of frames that can be dropped between
    *     invocations of {@link VideoRendererEventListener#onDroppedFrames(int, long)}.
+   * @param minConsecutiveDroppedFramesToNotify The minimum number of consecutive frames that must
+   *     be dropped for {@link VideoRendererEventListener#onConsecutiveDroppedFrames(int, long)} to
+   *     be called.
    */
   protected DecoderVideoRenderer(
       long allowedJoiningTimeMs,
       @Nullable Handler eventHandler,
       @Nullable VideoRendererEventListener eventListener,
-      int maxDroppedFramesToNotify) {
+      int maxDroppedFramesToNotify,
+      int minConsecutiveDroppedFramesToNotify) {
     super(C.TRACK_TYPE_VIDEO);
     this.allowedJoiningTimeMs = allowedJoiningTimeMs;
     this.maxDroppedFramesToNotify = maxDroppedFramesToNotify;
+    this.minConsecutiveDroppedFramesToNotify = minConsecutiveDroppedFramesToNotify;
     joiningDeadlineMs = C.TIME_UNSET;
     formatQueue = new TimedValueQueue<>();
     flagsOnlyBuffer = DecoderInputBuffer.newNoDataInstance();
@@ -296,7 +303,7 @@ protected void onPositionReset(long positionUs, boolean joining) throws ExoPlayb
     outputStreamEnded = false;
     lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED);
     initialPositionUs = C.TIME_UNSET;
-    consecutiveDroppedFrameCount = 0;
+    maybeNotifyConsecutiveDroppedFrames();
     if (decoder != null) {
       flushDecoder();
     }
@@ -319,6 +326,7 @@ protected void onStarted() {
   protected void onStopped() {
     joiningDeadlineMs = C.TIME_UNSET;
     maybeNotifyDroppedFrames();
+    maybeNotifyConsecutiveDroppedFrames();
   }
 
   @Override
@@ -603,8 +611,8 @@ protected void renderOutputBuffer(
       } else {
         renderOutputBufferToSurface(outputBuffer, checkNotNull(outputSurface));
       }
-      consecutiveDroppedFrameCount = 0;
       decoderCounters.renderedOutputBufferCount++;
+      maybeNotifyConsecutiveDroppedFrames();
       maybeNotifyRenderedFirstFrame();
     }
   }
@@ -996,6 +1004,17 @@ private void maybeNotifyDroppedFrames() {
     }
   }
 
+  private void maybeNotifyConsecutiveDroppedFrames() {
+    if (consecutiveDroppedFrameCount > 0
+        && consecutiveDroppedFrameCount >= minConsecutiveDroppedFramesToNotify) {
+      long elapsedMs = SystemClock.elapsedRealtime() - consecutiveDroppedFrameAccumulationStartTimeMs;
+      eventDispatcher.consecutiveDroppedFrames(consecutiveDroppedFrameCount, elapsedMs);
+    }
+    // Always reset the counter to 0, even if the threshold is not reached.
+    consecutiveDroppedFrameCount = 0;
+    consecutiveDroppedFrameAccumulationStartTimeMs = SystemClock.elapsedRealtime();
+  }
+
   private static boolean isBufferLate(long earlyUs) {
     // Class a buffer as late if it should have been presented more than 30 ms ago.
     return earlyUs < -30000;