Skip to content

Commit 6193f7c

Browse files
ychaparovcopybara-github
ychaparov
authored and
copybara-github
committed
Generate static HDR metadata when using DefaultEncoderFactory
* Select an encoder that supports HDR editing. * Set KEY_PROFILE to an HDR10 option * Use DecodeOneFrameUtil test util to return the MediaCodec format, which includes HDR_STATIC_INFO PiperOrigin-RevId: 702752639
1 parent 3e94bd6 commit 6193f7c

File tree

5 files changed

+152
-29
lines changed

5 files changed

+152
-29
lines changed

Diff for: ‎RELEASENOTES.md

+1
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
`VideoFrameProcessor.Listener.onInputStreamRegistered` to use `Format`.
3030
* Add support for transmuxing into alternative backwards compatible
3131
formats.
32+
* Generate HDR static metadata when using `DefaultEncoderFactory`.
3233
* Extractors:
3334
* MP3: Don't stop playback early when a `VBRI` frame's table of contents
3435
doesn't cover all the MP3 data in a file

Diff for: ‎libraries/test_utils/src/main/java/androidx/media3/test/utils/DecodeOneFrameUtil.java

+23-3
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818

1919
import static androidx.media3.common.util.Assertions.checkNotNull;
2020
import static androidx.media3.common.util.MediaFormatUtil.createMediaFormatFromFormat;
21+
import static androidx.media3.common.util.Util.postOrRun;
2122
import static androidx.media3.test.utils.TestUtil.buildAssetUri;
2223
import static androidx.test.core.app.ApplicationProvider.getApplicationContext;
2324

@@ -70,13 +71,34 @@ public interface Listener {
7071
@SuppressWarnings("CatchingUnchecked")
7172
public static void decodeOneAssetFileFrame(
7273
String assetFilePath, Listener listener, Surface surface) throws Exception {
74+
decodeOneMediaItemFrame(MediaItem.fromUri(buildAssetUri(assetFilePath)), listener, surface);
75+
}
76+
77+
/**
78+
* Reads and decodes one frame synchronously from the {@code mediaItem} and renders it to the
79+
* {@code surface}.
80+
*
81+
* <p>This method blocks until the frame has been rendered to the {@code surface}.
82+
*
83+
* @param mediaItem The {@link MediaItem} from which to decode a frame.
84+
* @param listener A {@link Listener} implementation.
85+
* @param surface The {@link Surface} to render the decoded frame to.
86+
*/
87+
public static void decodeOneMediaItemFrame(
88+
MediaItem mediaItem, Listener listener, Surface surface) throws Exception {
7389
Context context = getApplicationContext();
7490
AtomicReference<@NullableType Exception> unexpectedExceptionReference = new AtomicReference<>();
7591
AtomicReference<@NullableType PlaybackException> playbackExceptionReference =
7692
new AtomicReference<>();
7793
ConditionVariable firstFrameRenderedOrError = new ConditionVariable();
7894

7995
ExoPlayer exoPlayer = new ExoPlayer.Builder(context).build();
96+
postOrRun(
97+
new Handler(exoPlayer.getApplicationLooper()),
98+
() ->
99+
exoPlayer.setVideoFrameMetadataListener(
100+
(presentationTimeUs, releaseTimeNs, format, mediaFormat) ->
101+
listener.onFrameDecoded(checkNotNull(mediaFormat))));
80102
Handler handler = new Handler(exoPlayer.getApplicationLooper());
81103
AnalyticsListener analyticsListener =
82104
new AnalyticsListener() {
@@ -93,8 +115,6 @@ public void onRenderedFirstFrame(EventTime eventTime, Object output, long render
93115
if (exoPlayer.isReleased()) {
94116
return;
95117
}
96-
listener.onFrameDecoded(
97-
createMediaFormatFromFormat(checkNotNull(exoPlayer.getVideoFormat())));
98118
firstFrameRenderedOrError.open();
99119
}
100120

@@ -115,7 +135,7 @@ public void onEvents(Player player, Events events) {
115135
try {
116136
exoPlayer.setVideoSurface(surface);
117137
exoPlayer.addAnalyticsListener(analyticsListener);
118-
exoPlayer.setMediaItem(MediaItem.fromUri(buildAssetUri(assetFilePath)));
138+
exoPlayer.setMediaItem(mediaItem);
119139
exoPlayer.setPlayWhenReady(false);
120140
exoPlayer.prepare();
121141
// Catch all exceptions to report. Exceptions thrown here and not caught will not

Diff for: ‎libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/HdrEditingTest.java

+64-6
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
package androidx.media3.transformer.mh;
1717

1818
import static androidx.media3.effect.DefaultVideoFrameProcessor.WORKING_COLOR_SPACE_ORIGINAL;
19+
import static androidx.media3.test.utils.DecodeOneFrameUtil.decodeOneMediaItemFrame;
1920
import static androidx.media3.test.utils.TestUtil.retrieveTrackFormat;
2021
import static androidx.media3.transformer.AndroidTestUtil.FORCE_TRANSCODE_VIDEO_EFFECTS;
2122
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_1080P_5_SECOND_HLG10;
@@ -30,16 +31,21 @@
3031
import static androidx.media3.transformer.mh.HdrCapabilitiesUtil.assumeDeviceSupportsHdrEditing;
3132
import static androidx.test.core.app.ApplicationProvider.getApplicationContext;
3233
import static com.google.common.truth.Truth.assertThat;
34+
import static java.util.Collections.max;
3335

3436
import android.content.Context;
37+
import android.media.MediaFormat;
3538
import android.net.Uri;
39+
import android.view.Surface;
3640
import androidx.annotation.Nullable;
3741
import androidx.media3.common.C;
3842
import androidx.media3.common.Format;
3943
import androidx.media3.common.MediaItem;
4044
import androidx.media3.common.MimeTypes;
4145
import androidx.media3.common.util.Util;
4246
import androidx.media3.effect.DefaultVideoFrameProcessor;
47+
import androidx.media3.exoplayer.video.PlaceholderSurface;
48+
import androidx.media3.test.utils.DecodeOneFrameUtil;
4349
import androidx.media3.transformer.Composition;
4450
import androidx.media3.transformer.EditedMediaItem;
4551
import androidx.media3.transformer.EncoderUtil;
@@ -50,8 +56,13 @@
5056
import androidx.media3.transformer.TransformerAndroidTestRunner;
5157
import androidx.test.core.app.ApplicationProvider;
5258
import androidx.test.ext.junit.runners.AndroidJUnit4;
59+
import java.nio.ByteBuffer;
60+
import java.util.ArrayList;
61+
import java.util.List;
5362
import java.util.Objects;
5463
import java.util.concurrent.atomic.AtomicBoolean;
64+
import java.util.concurrent.atomic.AtomicReference;
65+
import org.junit.After;
5566
import org.junit.AssumptionViolatedException;
5667
import org.junit.Before;
5768
import org.junit.Rule;
@@ -69,12 +80,20 @@ public final class HdrEditingTest {
6980
@Rule public final TestName testName = new TestName();
7081

7182
private String testId;
83+
@Nullable private Surface placeholderSurface;
7284

7385
@Before
7486
public void setUpTestId() {
7587
testId = testName.getMethodName();
7688
}
7789

90+
@After
91+
public void tearDown() {
92+
if (placeholderSurface != null) {
93+
placeholderSurface.release();
94+
}
95+
}
96+
7897
@Test
7998
public void export_transmuxHdr10File() throws Exception {
8099
Context context = ApplicationProvider.getApplicationContext();
@@ -154,12 +173,12 @@ public void exportAndTranscode_hdr10File_whenHdrEditingIsSupported() throws Exce
154173
new TransformerAndroidTestRunner.Builder(context, transformer)
155174
.build()
156175
.run(testId, editedMediaItem);
157-
@C.ColorTransfer
158-
int actualColorTransfer =
159-
retrieveTrackFormat(context, exportTestResult.filePath, C.TRACK_TYPE_VIDEO)
160-
.colorInfo
161-
.colorTransfer;
162-
assertThat(actualColorTransfer).isEqualTo(C.COLOR_TRANSFER_ST2084);
176+
MediaFormat mediaFormat = getVideoMediaFormatFromDecoder(context, exportTestResult.filePath);
177+
ByteBuffer hdrStaticInfo = mediaFormat.getByteBuffer(MediaFormat.KEY_HDR_STATIC_INFO);
178+
179+
assertThat(max(byteList(hdrStaticInfo))).isAtLeast((byte) 1);
180+
assertThat(mediaFormat.getInteger(MediaFormat.KEY_COLOR_TRANSFER))
181+
.isEqualTo(MediaFormat.COLOR_TRANSFER_ST2084);
163182
}
164183

165184
@Test
@@ -246,10 +265,14 @@ public void exportAndTranscode_dolbyVisionFile_whenHdrEditingIsSupported() throw
246265
new TransformerAndroidTestRunner.Builder(context, transformer)
247266
.build()
248267
.run(testId, editedMediaItem);
268+
MediaFormat mediaFormat = getVideoMediaFormatFromDecoder(context, exportTestResult.filePath);
269+
ByteBuffer hdrStaticInfo = mediaFormat.getByteBuffer(MediaFormat.KEY_HDR_STATIC_INFO);
270+
249271
Format outputFormat =
250272
retrieveTrackFormat(context, exportTestResult.filePath, C.TRACK_TYPE_VIDEO);
251273
assertThat(outputFormat.colorInfo.colorTransfer).isEqualTo(C.COLOR_TRANSFER_ST2084);
252274
assertThat(outputFormat.sampleMimeType).isEqualTo(MimeTypes.VIDEO_H265);
275+
assertThat(max(byteList(hdrStaticInfo))).isAtLeast((byte) 1);
253276
}
254277

255278
@Test
@@ -401,4 +424,39 @@ public void onFallbackApplied(
401424
throw exception;
402425
}
403426
}
427+
428+
private static List<Byte> byteList(ByteBuffer buffer) {
429+
ArrayList<Byte> outputBytes = new ArrayList<>();
430+
while (buffer.hasRemaining()) {
431+
outputBytes.add(buffer.get());
432+
}
433+
return outputBytes;
434+
}
435+
436+
/**
437+
* Returns the {@link MediaFormat} corresponding to the video track in {@code filePath}.
438+
*
439+
* <p>HDR metadata is optional in both the container and bitstream. Return the {@link MediaFormat}
440+
* produced by the decoder which should include any metadata from either container or bitstream.
441+
*/
442+
private MediaFormat getVideoMediaFormatFromDecoder(Context context, String filePath)
443+
throws Exception {
444+
AtomicReference<MediaFormat> decodedFrameFormat = new AtomicReference<>();
445+
if (placeholderSurface == null) {
446+
placeholderSurface = PlaceholderSurface.newInstance(context, false);
447+
}
448+
decodeOneMediaItemFrame(
449+
MediaItem.fromUri(filePath),
450+
new DecodeOneFrameUtil.Listener() {
451+
@Override
452+
public void onContainerExtracted(MediaFormat mediaFormat) {}
453+
454+
@Override
455+
public void onFrameDecoded(MediaFormat mediaFormat) {
456+
decodedFrameFormat.set(mediaFormat);
457+
}
458+
},
459+
placeholderSurface);
460+
return decodedFrameFormat.get();
461+
}
404462
}

Diff for: ‎libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java

+30-1
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import static androidx.media3.common.util.Assertions.checkStateNotNull;
2424
import static androidx.media3.common.util.MediaFormatUtil.createMediaFormatFromFormat;
2525
import static androidx.media3.common.util.Util.SDK_INT;
26+
import static androidx.media3.transformer.EncoderUtil.getCodecProfilesForHdrFormat;
2627
import static java.lang.Math.abs;
2728
import static java.lang.Math.floor;
2829
import static java.lang.Math.max;
@@ -318,6 +319,10 @@ public DefaultCodec createForVideoEncoding(Format format) throws ExportException
318319
// the values.
319320
mediaFormat.setInteger(MediaFormat.KEY_PROFILE, supportedVideoEncoderSettings.profile);
320321
mediaFormat.setInteger(MediaFormat.KEY_LEVEL, supportedVideoEncoderSettings.level);
322+
} else if (SDK_INT >= 24 && ColorInfo.isTransferHdr(format.colorInfo)) {
323+
ImmutableList<Integer> codecProfilesForHdrFormat =
324+
getCodecProfilesForHdrFormat(mimeType, checkNotNull(format.colorInfo).colorTransfer);
325+
mediaFormat.setInteger(MediaFormat.KEY_PROFILE, codecProfilesForHdrFormat.get(0));
321326
}
322327

323328
if (mimeType.equals(MimeTypes.VIDEO_H264)) {
@@ -417,6 +422,13 @@ private static VideoEncoderQueryResult findEncoderWithClosestSupportedFormat(
417422
filteredEncoderInfos.get(0), requestedFormat, videoEncoderSettings);
418423
}
419424

425+
filteredEncoderInfos =
426+
filterEncodersByHdrEditingSupport(
427+
filteredEncoderInfos, mimeType, requestedFormat.colorInfo);
428+
if (filteredEncoderInfos.isEmpty()) {
429+
return null;
430+
}
431+
420432
filteredEncoderInfos =
421433
filterEncodersByResolution(
422434
filteredEncoderInfos, mimeType, requestedFormat.width, requestedFormat.height);
@@ -542,6 +554,23 @@ private static ImmutableList<MediaCodecInfo> filterEncodersByBitrateMode(
542554
: Integer.MAX_VALUE); // Drops encoder.
543555
}
544556

557+
/**
558+
* Returns a list of encoders that support the requested {@link ColorInfo#colorTransfer}, or all
559+
* input encoders if HDR editing is not needed.
560+
*/
561+
private static ImmutableList<MediaCodecInfo> filterEncodersByHdrEditingSupport(
562+
List<MediaCodecInfo> encoders, String mimeType, @Nullable ColorInfo colorInfo) {
563+
if (Util.SDK_INT < 33 || !ColorInfo.isTransferHdr(colorInfo)) {
564+
return ImmutableList.copyOf(encoders);
565+
}
566+
return filterEncoders(
567+
encoders,
568+
/* cost= */ (encoderInfo) ->
569+
EncoderUtil.isHdrEditingSupported(encoderInfo, mimeType, checkNotNull(colorInfo))
570+
? 0
571+
: Integer.MAX_VALUE); // Drops encoder.
572+
}
573+
545574
private static final class VideoEncoderQueryResult {
546575
public final MediaCodecInfo encoder;
547576
public final Format supportedFormat;
@@ -614,7 +643,7 @@ private static void adjustMediaFormatForH264EncoderSettings(
614643
if (colorInfo != null) {
615644
int colorTransfer = colorInfo.colorTransfer;
616645
ImmutableList<Integer> codecProfiles =
617-
EncoderUtil.getCodecProfilesForHdrFormat(mimeType, colorTransfer);
646+
getCodecProfilesForHdrFormat(mimeType, colorTransfer);
618647
if (!codecProfiles.isEmpty()) {
619648
// Default to the most compatible profile, which is first in the list.
620649
expectedEncodingProfile = codecProfiles.get(0);

Diff for: ‎libraries/transformer/src/main/java/androidx/media3/transformer/EncoderUtil.java

+34-19
Original file line numberDiff line numberDiff line change
@@ -90,36 +90,51 @@ public static ImmutableList<MediaCodecInfo> getSupportedEncodersForHdrEditing(
9090
}
9191

9292
ImmutableList<MediaCodecInfo> encoders = getSupportedEncoders(mimeType);
93-
ImmutableList<Integer> allowedColorProfiles =
94-
getCodecProfilesForHdrFormat(mimeType, colorInfo.colorTransfer);
9593
ImmutableList.Builder<MediaCodecInfo> resultBuilder = new ImmutableList.Builder<>();
9694
for (int i = 0; i < encoders.size(); i++) {
9795
MediaCodecInfo mediaCodecInfo = encoders.get(i);
9896
if (mediaCodecInfo.isAlias()) {
9997
continue;
10098
}
101-
boolean hasNeededHdrSupport =
102-
isFeatureSupported(
103-
mediaCodecInfo, mimeType, MediaCodecInfo.CodecCapabilities.FEATURE_HdrEditing)
104-
|| (colorInfo.colorTransfer == C.COLOR_TRANSFER_HLG
105-
&& Util.SDK_INT >= 35
106-
&& isFeatureSupported(
107-
mediaCodecInfo,
108-
mimeType,
109-
MediaCodecInfo.CodecCapabilities.FEATURE_HlgEditing));
110-
if (!hasNeededHdrSupport) {
111-
continue;
112-
}
113-
for (MediaCodecInfo.CodecProfileLevel codecProfileLevel :
114-
mediaCodecInfo.getCapabilitiesForType(mimeType).profileLevels) {
115-
if (allowedColorProfiles.contains(codecProfileLevel.profile)) {
116-
resultBuilder.add(mediaCodecInfo);
117-
}
99+
if (isHdrEditingSupported(mediaCodecInfo, mimeType, colorInfo)) {
100+
resultBuilder.add(mediaCodecInfo);
118101
}
119102
}
120103
return resultBuilder.build();
121104
}
122105

106+
/**
107+
* Returns whether HDR editing with the given {@linkplain ColorInfo color transfer} is supported
108+
* by the given {@linkplain MediaCodecInfo encoder}.
109+
*
110+
* @param mediaCodecInfo The encoder.
111+
* @param mimeType The MIME type of the video stream.
112+
* @param colorInfo The color info.
113+
*/
114+
@RequiresApi(33)
115+
public static boolean isHdrEditingSupported(
116+
MediaCodecInfo mediaCodecInfo, String mimeType, ColorInfo colorInfo) {
117+
ImmutableList<Integer> allowedColorProfiles =
118+
getCodecProfilesForHdrFormat(mimeType, colorInfo.colorTransfer);
119+
boolean hasNeededHdrSupport =
120+
isFeatureSupported(
121+
mediaCodecInfo, mimeType, MediaCodecInfo.CodecCapabilities.FEATURE_HdrEditing)
122+
|| (colorInfo.colorTransfer == C.COLOR_TRANSFER_HLG
123+
&& Util.SDK_INT >= 35
124+
&& isFeatureSupported(
125+
mediaCodecInfo, mimeType, MediaCodecInfo.CodecCapabilities.FEATURE_HlgEditing));
126+
if (!hasNeededHdrSupport) {
127+
return false;
128+
}
129+
for (MediaCodecInfo.CodecProfileLevel codecProfileLevel :
130+
mediaCodecInfo.getCapabilitiesForType(mimeType).profileLevels) {
131+
if (allowedColorProfiles.contains(codecProfileLevel.profile)) {
132+
return true;
133+
}
134+
}
135+
return false;
136+
}
137+
123138
/**
124139
* Returns the {@linkplain MediaCodecInfo.CodecProfileLevel#profile profile} constants that can be
125140
* used to encode the given HDR format, if supported by the device (this method does not check

0 commit comments

Comments
 (0)