MediaCodecVideoRenderer skips decoder inputs unused as reference

During a seek, or when playing a media with clipped start,
MCVR encounters preroll decode-only buffers that are not rendered.
Use C.BUFFER_FLAG_NO_OTHER_SAMPLE_DEPENDS_ON_THIS to determine
whether a decode-only buffer is unused as reference.
These buffers can be dropped before the decoder.
When this optimization is triggered, increment
decoderCounters.skippedInputBufferCount.

Tested in ExoPlayer demo app on "One hour frame counter (MP4)"
after enabling extractorsFactory.setMp4ExtractorFlags(
    FLAG_READ_WITHIN_GOP_SAMPLE_DEPENDENCIES);
Observe: "sib" increases on each seek.
PiperOrigin-RevId: 650566216

Author: ychaparov

RELEASENOTES.md

@@ -19,6 +19,8 @@
     *   Add `PreloadMediaSource.PreloadControl.onPreloadError` to allow
         `PreloadMediaSource.PreloadControl` implementations to take actions when
         error occurs.
+    *   `MediaCodecVideoRenderer` avoids decoding samples that are neither
+        rendered nor used as reference by other samples.
 *   Transformer:
     *   Add `SurfaceAssetLoader`, which supports queueing video data to
         Transformer via a `Surface`.

libraries/common/src/main/java/androidx/media3/common/C.java

@@ -620,7 +620,7 @@ private C() {}
    * <ul>
    *   <li>{@link #BUFFER_FLAG_KEY_FRAME}
    *   <li>{@link #BUFFER_FLAG_END_OF_STREAM}
-   *   <li>{@link #BUFFER_FLAG_NO_OTHER_SAMPLE_DEPENDS_ON_THIS}
+   *   <li>{@link #BUFFER_FLAG_NOT_DEPENDED_ON}
    *   <li>{@link #BUFFER_FLAG_FIRST_SAMPLE}
    *   <li>{@link #BUFFER_FLAG_LAST_SAMPLE}
    *   <li>{@link #BUFFER_FLAG_ENCRYPTED}
@@ -635,7 +635,7 @@ private C() {}
       value = {
         BUFFER_FLAG_KEY_FRAME,
         BUFFER_FLAG_END_OF_STREAM,
-        BUFFER_FLAG_NO_OTHER_SAMPLE_DEPENDS_ON_THIS,
+        BUFFER_FLAG_NOT_DEPENDED_ON,
         BUFFER_FLAG_FIRST_SAMPLE,
         BUFFER_FLAG_HAS_SUPPLEMENTAL_DATA,
         BUFFER_FLAG_LAST_SAMPLE,
@@ -650,9 +650,8 @@ private C() {}
   @UnstableApi
   public static final int BUFFER_FLAG_END_OF_STREAM = MediaCodec.BUFFER_FLAG_END_OF_STREAM;
 
-  /** Indicates that future buffers do not depend on the data in this buffer. */
-  @UnstableApi
-  public static final int BUFFER_FLAG_NO_OTHER_SAMPLE_DEPENDS_ON_THIS = 1 << 26; // 0x04000000
+  /** Indicates that no other buffers depend on the data in this buffer. */
+  @UnstableApi public static final int BUFFER_FLAG_NOT_DEPENDED_ON = 1 << 26; // 0x04000000
 
   /** Indicates that a buffer is known to contain the first media sample of the stream. */
   @UnstableApi public static final int BUFFER_FLAG_FIRST_SAMPLE = 1 << 27; // 0x08000000

libraries/decoder/src/main/java/androidx/media3/decoder/Buffer.java

@@ -60,6 +60,11 @@ public final boolean hasSupplementalData() {
     return getFlag(C.BUFFER_FLAG_HAS_SUPPLEMENTAL_DATA);
   }
 
+  /** Returns whether the {@link C#BUFFER_FLAG_NOT_DEPENDED_ON} flag is set. */
+  public final boolean notDependedOn() {
+    return getFlag(C.BUFFER_FLAG_NOT_DEPENDED_ON);
+  }
+
   /**
    * Replaces this buffer's flags with {@code flags}.
    *

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java

@@ -1452,6 +1452,12 @@ private boolean feedInputBuffer() throws ExoPlaybackException {
       return true;
     }
 
+    if (shouldSkipDecoderInputBuffer(buffer)) {
+      buffer.clear();
+      decoderCounters.skippedInputBufferCount += 1;
+      return true;
+    }
+
     boolean bufferEncrypted = buffer.isEncrypted();
     if (bufferEncrypted) {
       buffer.cryptoInfo.increaseClearDataFirstSubSampleBy(adaptiveReconfigurationBytes);
@@ -1754,6 +1760,18 @@ protected int getCodecBufferFlags(DecoderInputBuffer buffer) {
     return 0;
   }
 
+  /**
+   * Returns whether the input buffer should be skipped before the decoder.
+   *
+   * <p>This can be used to skip decoding of buffers that are not depended on during seeking. See
+   * {@link C#BUFFER_FLAG_NOT_DEPENDED_ON}.
+   *
+   * @param buffer The input buffer.
+   */
+  protected boolean shouldSkipDecoderInputBuffer(DecoderInputBuffer buffer) {
+    return false;
+  }
+
   /**
    * Returns the presentation time of the last buffer in the stream.
    *

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java

@@ -1247,7 +1247,7 @@ protected void onQueueInputBuffer(DecoderInputBuffer buffer) throws ExoPlaybackE
 
   @Override
   protected int getCodecBufferFlags(DecoderInputBuffer buffer) {
-    if (Util.SDK_INT >= 34 && tunneling && buffer.timeUs < getLastResetPositionUs()) {
+    if (Util.SDK_INT >= 34 && tunneling && isBufferBeforeStartTime(buffer)) {
       // The buffer likely needs to be dropped because its timestamp is less than the start time.
       // We can't decide to do this after decoding because we won't get the buffer back from the
       // codec in tunneling mode. This may not work perfectly, e.g. when the codec is doing frame
@@ -1257,6 +1257,27 @@ protected int getCodecBufferFlags(DecoderInputBuffer buffer) {
     return 0;
   }
 
+  @Override
+  protected boolean shouldSkipDecoderInputBuffer(DecoderInputBuffer buffer) {
+    // TODO: b/351164714 - Do not apply this optimization for buffers with timestamp near
+    //  the media duration.
+    if (hasReadStreamToEnd() || buffer.isLastSample()) {
+      // Last buffer is always decoded.
+      return false;
+    }
+    if (buffer.isEncrypted()) {
+      // Commonly used decryption algorithms require updating the initialization vector for each
+      // block processed. Skipping input buffers before the decoder is not allowed.
+      return false;
+    }
+    // Skip buffers without sample dependencies that won't be rendered.
+    return isBufferBeforeStartTime(buffer) && buffer.notDependedOn();
+  }
+
+  private boolean isBufferBeforeStartTime(DecoderInputBuffer buffer) {
+    return buffer.timeUs < getLastResetPositionUs();
+  }
+
   @Override
   protected void onOutputFormatChanged(Format format, @Nullable MediaFormat mediaFormat) {
     @Nullable MediaCodecAdapter codec = getCodec();

libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java

@@ -356,6 +356,73 @@ public void render_withBufferLimitEqualToNumberOfSamples_rendersLastFrameAfterEn
     assertThat(argumentDecoderCounters.getValue().skippedOutputBufferCount).isEqualTo(2);
   }
 
+  @Test
+  public void render_withoutSampleDependencies_rendersLastFrameAfterEndOfStream() throws Exception {
+    ArgumentCaptor<DecoderCounters> argumentDecoderCounters =
+        ArgumentCaptor.forClass(DecoderCounters.class);
+    FakeSampleStream fakeSampleStream =
+        new FakeSampleStream(
+            new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024),
+            /* mediaSourceEventDispatcher= */ null,
+            DrmSessionManager.DRM_UNSUPPORTED,
+            new DrmSessionEventListener.EventDispatcher(),
+            /* initialFormat= */ VIDEO_H264,
+            ImmutableList.of(
+                oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), // First buffer.
+                // Second buffer will be skipped before decoder during a seek.
+                oneByteSample(/* timeUs= */ 10_000, C.BUFFER_FLAG_NOT_DEPENDED_ON),
+                // Last buffer without sample dependencies will be rendered.
+                oneByteSample(/* timeUs= */ 20_000, C.BUFFER_FLAG_NOT_DEPENDED_ON),
+                END_OF_STREAM_ITEM));
+    fakeSampleStream.writeData(/* startPositionUs= */ 0);
+    // Seek to time after samples.
+    fakeSampleStream.seekToUs(30_000, /* allowTimeBeyondBuffer= */ true);
+    mediaCodecVideoRenderer =
+        new MediaCodecVideoRenderer(
+            ApplicationProvider.getApplicationContext(),
+            new ForwardingSynchronousMediaCodecAdapterWithBufferLimit.Factory(/* bufferLimit= */ 3),
+            mediaCodecSelector,
+            /* allowedJoiningTimeMs= */ 0,
+            /* enableDecoderFallback= */ false,
+            /* eventHandler= */ new Handler(testMainLooper),
+            eventListener,
+            /* maxDroppedFramesToNotify= */ 1);
+    mediaCodecVideoRenderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT);
+    mediaCodecVideoRenderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface);
+    mediaCodecVideoRenderer.enable(
+        RendererConfiguration.DEFAULT,
+        new Format[] {VIDEO_H264},
+        fakeSampleStream,
+        /* positionUs= */ 0,
+        /* joining= */ false,
+        /* mayRenderStartOfStream= */ true,
+        /* startPositionUs= */ 30_000,
+        /* offsetUs= */ 0,
+        new MediaSource.MediaPeriodId(new Object()));
+
+    mediaCodecVideoRenderer.start();
+    mediaCodecVideoRenderer.setCurrentStreamFinal();
+    mediaCodecVideoRenderer.render(0, SystemClock.elapsedRealtime() * 1000);
+    // Call to render has reads all samples including the END_OF_STREAM_ITEM because the
+    // previous sample is skipped before decoding.
+    assertThat(mediaCodecVideoRenderer.hasReadStreamToEnd()).isTrue();
+    int posUs = 30_000;
+    while (!mediaCodecVideoRenderer.isEnded()) {
+      mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000);
+      posUs += 40_000;
+    }
+    shadowOf(testMainLooper).idle();
+
+    verify(eventListener).onRenderedFirstFrame(eq(surface), /* renderTimeMs= */ anyLong());
+    verify(eventListener).onVideoEnabled(argumentDecoderCounters.capture());
+    // First key frame is decoded and skipped as an output buffer.
+    assertThat(argumentDecoderCounters.getValue().skippedOutputBufferCount).isEqualTo(1);
+    // Second frame is skipped before the decoder, as an input buffer.
+    assertThat(argumentDecoderCounters.getValue().skippedInputBufferCount).isEqualTo(1);
+    // Last frame is rendered.
+    assertThat(argumentDecoderCounters.getValue().renderedOutputBufferCount).isEqualTo(1);
+  }
+
   @Test
   public void
       render_withClippingMediaPeriodAndBufferContainingLastAndClippingSamples_rendersLastFrame()

libraries/extractor/src/main/java/androidx/media3/extractor/mp4/FragmentedMp4Extractor.java

@@ -135,7 +135,7 @@ public static ExtractorsFactory newFactory(SubtitleParser.Factory subtitleParser
 
   /**
    * Flag to extract additional sample dependency information, and mark output buffers with {@link
-   * C#BUFFER_FLAG_NO_OTHER_SAMPLE_DEPENDS_ON_THIS}.
+   * C#BUFFER_FLAG_NOT_DEPENDED_ON}.
    *
    * <p>This class always marks the samples at the start of each group of picture (GOP) with {@link
    * C#BUFFER_FLAG_KEY_FRAME}. Usually, key frames can be decoded independently, without depending
@@ -1661,7 +1661,7 @@ private boolean readSample(ExtractorInput input) throws IOException {
 
     @C.BufferFlags int sampleFlags = trackBundle.getCurrentSampleFlags();
     if ((flags & FLAG_READ_WITHIN_GOP_SAMPLE_DEPENDENCIES) != 0 && !isSampleDependedOn) {
-      sampleFlags |= C.BUFFER_FLAG_NO_OTHER_SAMPLE_DEPENDS_ON_THIS;
+      sampleFlags |= C.BUFFER_FLAG_NOT_DEPENDED_ON;
     }
 
     // Encryption data.

libraries/extractor/src/main/java/androidx/media3/extractor/mp4/Mp4Extractor.java

@@ -125,7 +125,7 @@ public static ExtractorsFactory newFactory(SubtitleParser.Factory subtitleParser
 
   /**
    * Flag to extract additional sample dependency information, and mark output buffers with {@link
-   * C#BUFFER_FLAG_NO_OTHER_SAMPLE_DEPENDS_ON_THIS}.
+   * C#BUFFER_FLAG_NOT_DEPENDED_ON}.
    *
    * <p>This class always marks the samples at the start of each group of picture (GOP) with {@link
    * C#BUFFER_FLAG_KEY_FRAME}. Usually, key frames can be decoded independently, without depending
@@ -804,7 +804,7 @@ private int readSample(ExtractorInput input, PositionHolder positionHolder) thro
     long timeUs = track.sampleTable.timestampsUs[sampleIndex];
     @C.BufferFlags int sampleFlags = track.sampleTable.flags[sampleIndex];
     if (!isSampleDependedOn) {
-      sampleFlags |= C.BUFFER_FLAG_NO_OTHER_SAMPLE_DEPENDS_ON_THIS;
+      sampleFlags |= C.BUFFER_FLAG_NOT_DEPENDED_ON;
     }
     if (trueHdSampleRechunker != null) {
       trueHdSampleRechunker.sampleMetadata(