Fetch encoded QP from MediaCodec encoders
It is a part of "encoding statistics" feature [1] available in Android SDK 33. Local testing revealed that for HW VP8/9 encoders we get QP in range [0,64] which is not what WebRTC quality scaler expects. Exclude VP8/9 encoders for now.
[1] https://developer.android.com/reference/android/media/MediaFormat#VIDEO_ENCODING_STATISTICS_LEVEL_1
Bug: webrtc:15015
Change-Id: I8af2fd96afb34e18cb3e2cc3562b10149324c16e
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/298306
Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#39722}
diff --git a/sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java b/sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java
index b5f7629..bd01b7d 100644
--- a/sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java
+++ b/sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java
@@ -16,6 +16,7 @@
import android.media.MediaCodec;
import android.media.MediaCodecInfo;
+import android.media.MediaCodecInfo.CodecCapabilities;
import android.media.MediaFormat;
import android.opengl.GLES20;
import android.os.Build;
@@ -162,6 +163,9 @@
// value to send exceptions thrown during release back to the encoder thread.
@Nullable private volatile Exception shutdownException;
+ // True if collection of encoding statistics is enabled.
+ private boolean isEncodingStatisticsEnabled;
+
/**
* Creates a new HardwareVideoEncoder with the given codecName, codecType, colorFormat, key frame
* intervals, and bitrateAdjuster.
@@ -226,6 +230,8 @@
nextPresentationTimestampUs = 0;
lastKeyFrameNs = -1;
+ isEncodingStatisticsEnabled = false;
+
try {
codec = mediaCodecWrapperFactory.createByCodecName(codecName);
} catch (IOException | IllegalArgumentException e) {
@@ -258,6 +264,13 @@
Logging.w(TAG, "Unknown profile level id: " + profileLevelId);
}
}
+
+ if (isEncodingStatisticsSupported()) {
+ format.setInteger(MediaFormat.KEY_VIDEO_ENCODING_STATISTICS_LEVEL,
+ MediaFormat.VIDEO_ENCODING_STATISTICS_LEVEL_1);
+ isEncodingStatisticsEnabled = true;
+ }
+
Logging.d(TAG, "Format: " + format);
codec.configure(
format, null /* surface */, null /* crypto */, MediaCodec.CONFIGURE_FLAG_ENCODE);
@@ -606,21 +619,30 @@
outputBuffersBusyCount.increment();
EncodedImage.Builder builder = outputBuilders.poll();
- EncodedImage encodedImage = builder
- .setBuffer(frameBuffer,
- () -> {
- // This callback should not throw any exceptions since
- // it may be called on an arbitrary thread.
- // Check bug webrtc:11230 for more details.
- try {
- codec.releaseOutputBuffer(index, false);
- } catch (Exception e) {
- Logging.e(TAG, "releaseOutputBuffer failed", e);
- }
- outputBuffersBusyCount.decrement();
- })
- .setFrameType(frameType)
- .createEncodedImage();
+ builder
+ .setBuffer(frameBuffer,
+ () -> {
+ // This callback should not throw any exceptions since
+ // it may be called on an arbitrary thread.
+ // Check bug webrtc:11230 for more details.
+ try {
+ codec.releaseOutputBuffer(index, false);
+ } catch (Exception e) {
+ Logging.e(TAG, "releaseOutputBuffer failed", e);
+ }
+ outputBuffersBusyCount.decrement();
+ })
+ .setFrameType(frameType);
+
+ if (isEncodingStatisticsEnabled) {
+ MediaFormat format = codec.getOutputFormat(index);
+ if (format != null && format.containsKey(MediaFormat.KEY_VIDEO_QP_AVERAGE)) {
+ int qp = format.getInteger(MediaFormat.KEY_VIDEO_QP_AVERAGE);
+ builder.setQp(qp);
+ }
+ }
+
+ EncodedImage encodedImage = builder.createEncodedImage();
// TODO(mellem): Set codec-specific info.
callback.onEncodedFrame(encodedImage, new CodecSpecificInfo());
// Note that the callback may have retained the image.
@@ -685,6 +707,29 @@
return height;
}
+ protected boolean isEncodingStatisticsSupported() {
+ // WebRTC quality scaler, which adjusts resolution and/or frame rate based on encoded QP,
+ // expects QP to be in native bitstream range for given codec. Native QP range for VP8 is
+ // [0, 127] and for VP9 is [0, 255]. MediaCodec VP8 and VP9 encoders (perhaps not all)
+ // return QP in range [0, 64], which is libvpx API specific range. Due to this mismatch we
+ // can't use QP feedback from these codecs.
+ if (codecType == VideoCodecMimeType.VP8 || codecType == VideoCodecMimeType.VP9) {
+ return false;
+ }
+
+ MediaCodecInfo codecInfo = codec.getCodecInfo();
+ if (codecInfo == null) {
+ return false;
+ }
+
+ CodecCapabilities codecCaps = codecInfo.getCapabilitiesForType(codecType.mimeType());
+ if (codecCaps == null) {
+ return false;
+ }
+
+ return codecCaps.isFeatureSupported(CodecCapabilities.FEATURE_EncodingStatistics);
+ }
+
// Visible for testing.
protected void fillInputBuffer(ByteBuffer buffer, VideoFrame.Buffer videoFrameBuffer) {
yuvFormat.fillBuffer(buffer, videoFrameBuffer, stride, sliceHeight);
diff --git a/sdk/android/src/java/org/webrtc/MediaCodecWrapper.java b/sdk/android/src/java/org/webrtc/MediaCodecWrapper.java
index 60c853d..6abdbfe 100644
--- a/sdk/android/src/java/org/webrtc/MediaCodecWrapper.java
+++ b/sdk/android/src/java/org/webrtc/MediaCodecWrapper.java
@@ -11,6 +11,7 @@
package org.webrtc;
import android.media.MediaCodec;
+import android.media.MediaCodecInfo;
import android.media.MediaCrypto;
import android.media.MediaFormat;
import android.os.Bundle;
@@ -52,4 +53,8 @@
Surface createInputSurface();
void setParameters(Bundle params);
+
+ MediaCodecInfo getCodecInfo();
+
+ MediaFormat getOutputFormat(int index);
}
diff --git a/sdk/android/src/java/org/webrtc/MediaCodecWrapperFactoryImpl.java b/sdk/android/src/java/org/webrtc/MediaCodecWrapperFactoryImpl.java
index 2ba62ac..56ab21f 100644
--- a/sdk/android/src/java/org/webrtc/MediaCodecWrapperFactoryImpl.java
+++ b/sdk/android/src/java/org/webrtc/MediaCodecWrapperFactoryImpl.java
@@ -12,6 +12,7 @@
import android.media.MediaCodec;
import android.media.MediaCodec.BufferInfo;
+import android.media.MediaCodecInfo;
import android.media.MediaCrypto;
import android.media.MediaFormat;
import android.os.Bundle;
@@ -106,6 +107,16 @@
public void setParameters(Bundle params) {
mediaCodec.setParameters(params);
}
+
+ @Override
+ public MediaCodecInfo getCodecInfo() {
+ return mediaCodec.getCodecInfo();
+ }
+
+ @Override
+ public MediaFormat getOutputFormat(int index) {
+ return mediaCodec.getOutputFormat(index);
+ }
}
@Override
diff --git a/sdk/android/tests/src/org/webrtc/FakeMediaCodecWrapper.java b/sdk/android/tests/src/org/webrtc/FakeMediaCodecWrapper.java
index fb7aba4..5e2a1f4 100644
--- a/sdk/android/tests/src/org/webrtc/FakeMediaCodecWrapper.java
+++ b/sdk/android/tests/src/org/webrtc/FakeMediaCodecWrapper.java
@@ -12,6 +12,7 @@
import android.graphics.SurfaceTexture;
import android.media.MediaCodec;
+import android.media.MediaCodecInfo;
import android.media.MediaCodecInfo.CodecCapabilities;
import android.media.MediaCrypto;
import android.media.MediaFormat;
@@ -318,4 +319,14 @@
@Override
public void setParameters(Bundle params) {}
+
+ @Override
+ public MediaCodecInfo getCodecInfo() {
+ return null;
+ }
+
+ @Override
+ public MediaFormat getOutputFormat(int index) {
+ return outputFormat;
+ }
}
diff --git a/sdk/android/tests/src/org/webrtc/HardwareVideoEncoderTest.java b/sdk/android/tests/src/org/webrtc/HardwareVideoEncoderTest.java
index bd4a642..36bfb20 100644
--- a/sdk/android/tests/src/org/webrtc/HardwareVideoEncoderTest.java
+++ b/sdk/android/tests/src/org/webrtc/HardwareVideoEncoderTest.java
@@ -15,6 +15,7 @@
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
@@ -22,6 +23,7 @@
import android.media.MediaCodec;
import android.media.MediaCodecInfo;
import android.media.MediaFormat;
+import android.os.Build.VERSION_CODES;
import android.os.Bundle;
import androidx.test.runner.AndroidJUnit4;
import java.nio.ByteBuffer;
@@ -69,13 +71,16 @@
private static class TestEncoder extends HardwareVideoEncoder {
private final Object deliverEncodedImageLock = new Object();
private boolean deliverEncodedImageDone = true;
+ private boolean isEncodingStatisticsSupported;
TestEncoder(MediaCodecWrapperFactory mediaCodecWrapperFactory, String codecName,
VideoCodecMimeType codecType, Integer surfaceColorFormat, Integer yuvColorFormat,
Map<String, String> params, int keyFrameIntervalSec, int forceKeyFrameIntervalMs,
- BitrateAdjuster bitrateAdjuster, EglBase14.Context sharedContext) {
+ BitrateAdjuster bitrateAdjuster, EglBase14.Context sharedContext,
+ boolean isEncodingStatisticsSupported) {
super(mediaCodecWrapperFactory, codecName, codecType, surfaceColorFormat, yuvColorFormat,
params, keyFrameIntervalSec, forceKeyFrameIntervalMs, bitrateAdjuster, sharedContext);
+ this.isEncodingStatisticsSupported = isEncodingStatisticsSupported;
}
public void waitDeliverEncodedImage() throws InterruptedException {
@@ -118,11 +123,17 @@
buffer.flip();
i420Buffer.release();
}
+
+ @Override
+ protected boolean isEncodingStatisticsSupported() {
+ return isEncodingStatisticsSupported;
+ }
}
private class TestEncoderBuilder {
private VideoCodecMimeType codecType = VideoCodecMimeType.VP8;
private BitrateAdjuster bitrateAdjuster = new BaseBitrateAdjuster();
+ private boolean isEncodingStatisticsSupported;
public TestEncoderBuilder setCodecType(VideoCodecMimeType codecType) {
this.codecType = codecType;
@@ -134,6 +145,12 @@
return this;
}
+ public TestEncoderBuilder SetIsEncodingStatisticsSupported(
+ boolean isEncodingStatisticsSupported) {
+ this.isEncodingStatisticsSupported = isEncodingStatisticsSupported;
+ return this;
+ }
+
public TestEncoder build() {
return new TestEncoder((String name)
-> fakeMediaCodecWrapper,
@@ -143,7 +160,7 @@
/* params= */ new HashMap<>(),
/* keyFrameIntervalSec= */ 0,
/* forceKeyFrameIntervalMs= */ 0, bitrateAdjuster,
- /* sharedContext= */ null);
+ /* sharedContext= */ null, isEncodingStatisticsSupported);
}
}
@@ -194,6 +211,76 @@
}
@Test
+ public void encodingStatistics_unsupported_disabled() throws InterruptedException {
+ TestEncoder encoder = new TestEncoderBuilder().SetIsEncodingStatisticsSupported(false).build();
+
+ assertThat(encoder.initEncode(TEST_ENCODER_SETTINGS, mockEncoderCallback))
+ .isEqualTo(VideoCodecStatus.OK);
+
+ MediaFormat configuredFormat = fakeMediaCodecWrapper.getConfiguredFormat();
+ assertThat(configuredFormat).isNotNull();
+ assertThat(configuredFormat.containsKey(MediaFormat.KEY_VIDEO_ENCODING_STATISTICS_LEVEL))
+ .isFalse();
+
+ // Verify that QP is not set in encoded frame even if reported by MediaCodec.
+ MediaFormat outputFormat = new MediaFormat();
+ outputFormat.setInteger(MediaFormat.KEY_VIDEO_QP_AVERAGE, 123);
+ doReturn(outputFormat).when(fakeMediaCodecWrapper).getOutputFormat(anyInt());
+
+ encoder.encode(createTestVideoFrame(/* timestampNs= */ 42), ENCODE_INFO_KEY_FRAME);
+
+ fakeMediaCodecWrapper.addOutputData(CodecTestHelper.generateRandomData(100),
+ /* presentationTimestampUs= */ 0,
+ /* flags= */ MediaCodec.BUFFER_FLAG_SYNC_FRAME);
+
+ encoder.waitDeliverEncodedImage();
+
+ ArgumentCaptor<EncodedImage> videoFrameCaptor = ArgumentCaptor.forClass(EncodedImage.class);
+ verify(mockEncoderCallback)
+ .onEncodedFrame(videoFrameCaptor.capture(), any(CodecSpecificInfo.class));
+
+ EncodedImage videoFrame = videoFrameCaptor.getValue();
+ assertThat(videoFrame).isNotNull();
+ assertThat(videoFrame.qp).isNull();
+ }
+
+ @Test
+ public void encodingStatistics_supported_enabled() throws InterruptedException {
+ TestEncoder encoder = new TestEncoderBuilder().SetIsEncodingStatisticsSupported(true).build();
+
+ assertThat(encoder.initEncode(TEST_ENCODER_SETTINGS, mockEncoderCallback))
+ .isEqualTo(VideoCodecStatus.OK);
+
+ MediaFormat configuredFormat = fakeMediaCodecWrapper.getConfiguredFormat();
+ assertThat(configuredFormat).isNotNull();
+ assertThat(configuredFormat.containsKey(MediaFormat.KEY_VIDEO_ENCODING_STATISTICS_LEVEL))
+ .isTrue();
+ assertThat(configuredFormat.getInteger(MediaFormat.KEY_VIDEO_ENCODING_STATISTICS_LEVEL))
+ .isEqualTo(MediaFormat.VIDEO_ENCODING_STATISTICS_LEVEL_1);
+
+ // Verify that QP is set in encoded frame.
+ MediaFormat outputFormat = new MediaFormat();
+ outputFormat.setInteger(MediaFormat.KEY_VIDEO_QP_AVERAGE, 123);
+ doReturn(outputFormat).when(fakeMediaCodecWrapper).getOutputFormat(anyInt());
+
+ encoder.encode(createTestVideoFrame(/* timestampNs= */ 42), ENCODE_INFO_KEY_FRAME);
+
+ fakeMediaCodecWrapper.addOutputData(CodecTestHelper.generateRandomData(100),
+ /* presentationTimestampUs= */ 0,
+ /* flags= */ MediaCodec.BUFFER_FLAG_SYNC_FRAME);
+
+ encoder.waitDeliverEncodedImage();
+
+ ArgumentCaptor<EncodedImage> videoFrameCaptor = ArgumentCaptor.forClass(EncodedImage.class);
+ verify(mockEncoderCallback)
+ .onEncodedFrame(videoFrameCaptor.capture(), any(CodecSpecificInfo.class));
+
+ EncodedImage videoFrame = videoFrameCaptor.getValue();
+ assertThat(videoFrame).isNotNull();
+ assertThat(videoFrame.qp).isEqualTo(123);
+ }
+
+ @Test
public void testEncodeByteBuffer() {
// Set-up.
HardwareVideoEncoder encoder = new TestEncoderBuilder().build();
