admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fetch encoded QP from MediaCodec encoders

Browse Source 
It is a part of "encoding statistics" feature [1] available in Android SDK 33. Local testing revealed that for HW VP8/9 encoders we get QP in range [0,64] which is not what WebRTC quality scaler expects. Exclude VP8/9 encoders for now.

[1] https://developer.android.com/reference/android/media/MediaFormat#VIDEO_ENCODING_STATISTICS_LEVEL_1

Bug: webrtc:15015
Change-Id: I8af2fd96afb34e18cb3e2cc3562b10149324c16e
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/298306
Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#39722}
This commit is contained in:
Sergey Silkin 2023-03-30 09:30:50 +00:00 committed by WebRTC LUCI CQ
parent 77158ace75
commit 910b225d82
5 changed files with 176 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java
View File

@ -16,6 +16,7 @@ import static android.media.MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_CBR;
import android.media.MediaCodec;
import android.media.MediaCodecInfo;
import android.media.MediaCodecInfo.CodecCapabilities;
import android.media.MediaFormat;
import android.opengl.GLES20;
import android.os.Build;
@ -162,6 +163,9 @@ class HardwareVideoEncoder implements VideoEncoder {
// value to send exceptions thrown during release back to the encoder thread.
@Nullable private volatile Exception shutdownException;
// True if collection of encoding statistics is enabled.
private boolean isEncodingStatisticsEnabled;
/**
* Creates a new HardwareVideoEncoder with the given codecName, codecType, colorFormat, key frame
* intervals, and bitrateAdjuster.
@ -226,6 +230,8 @@ class HardwareVideoEncoder implements VideoEncoder {
nextPresentationTimestampUs = 0;
lastKeyFrameNs = -1;
isEncodingStatisticsEnabled = false;
try {
codec = mediaCodecWrapperFactory.createByCodecName(codecName);
} catch (IOException | IllegalArgumentException e) {
@ -258,6 +264,13 @@ class HardwareVideoEncoder implements VideoEncoder {
Logging.w(TAG, "Unknown profile level id: " + profileLevelId);
}
}
if (isEncodingStatisticsSupported()) {
format.setInteger(MediaFormat.KEY_VIDEO_ENCODING_STATISTICS_LEVEL,
MediaFormat.VIDEO_ENCODING_STATISTICS_LEVEL_1);
isEncodingStatisticsEnabled = true;
}
Logging.d(TAG, "Format: " + format);
codec.configure(
format, null /* surface */, null /* crypto */, MediaCodec.CONFIGURE_FLAG_ENCODE);
@ -606,21 +619,30 @@ class HardwareVideoEncoder implements VideoEncoder {
outputBuffersBusyCount.increment();
EncodedImage.Builder builder = outputBuilders.poll();
EncodedImage encodedImage = builder
.setBuffer(frameBuffer,
() -> {
// This callback should not throw any exceptions since
// it may be called on an arbitrary thread.
// Check bug webrtc:11230 for more details.
try {
codec.releaseOutputBuffer(index, false);
} catch (Exception e) {
Logging.e(TAG, "releaseOutputBuffer failed", e);
}
outputBuffersBusyCount.decrement();
})
.setFrameType(frameType)
.createEncodedImage();
builder
.setBuffer(frameBuffer,
() -> {
// This callback should not throw any exceptions since
// it may be called on an arbitrary thread.
// Check bug webrtc:11230 for more details.
try {
codec.releaseOutputBuffer(index, false);
} catch (Exception e) {
Logging.e(TAG, "releaseOutputBuffer failed", e);
}
outputBuffersBusyCount.decrement();
})
.setFrameType(frameType);
if (isEncodingStatisticsEnabled) {
MediaFormat format = codec.getOutputFormat(index);
if (format != null && format.containsKey(MediaFormat.KEY_VIDEO_QP_AVERAGE)) {
int qp = format.getInteger(MediaFormat.KEY_VIDEO_QP_AVERAGE);
builder.setQp(qp);
}
}
EncodedImage encodedImage = builder.createEncodedImage();
// TODO(mellem): Set codec-specific info.
callback.onEncodedFrame(encodedImage, new CodecSpecificInfo());
// Note that the callback may have retained the image.
@ -685,6 +707,29 @@ class HardwareVideoEncoder implements VideoEncoder {
return height;
}
protected boolean isEncodingStatisticsSupported() {
// WebRTC quality scaler, which adjusts resolution and/or frame rate based on encoded QP,
// expects QP to be in native bitstream range for given codec. Native QP range for VP8 is
// [0, 127] and for VP9 is [0, 255]. MediaCodec VP8 and VP9 encoders (perhaps not all)
// return QP in range [0, 64], which is libvpx API specific range. Due to this mismatch we
// can't use QP feedback from these codecs.
if (codecType == VideoCodecMimeType.VP8 || codecType == VideoCodecMimeType.VP9) {
return false;
}
MediaCodecInfo codecInfo = codec.getCodecInfo();
if (codecInfo == null) {
return false;
}
CodecCapabilities codecCaps = codecInfo.getCapabilitiesForType(codecType.mimeType());
if (codecCaps == null) {
return false;
}
return codecCaps.isFeatureSupported(CodecCapabilities.FEATURE_EncodingStatistics);
}
// Visible for testing.
protected void fillInputBuffer(ByteBuffer buffer, VideoFrame.Buffer videoFrameBuffer) {
yuvFormat.fillBuffer(buffer, videoFrameBuffer, stride, sliceHeight);

5
sdk/android/src/java/org/webrtc/MediaCodecWrapper.java
View File

@ -11,6 +11,7 @@
package org.webrtc;
import android.media.MediaCodec;
import android.media.MediaCodecInfo;
import android.media.MediaCrypto;
import android.media.MediaFormat;
import android.os.Bundle;
@ -52,4 +53,8 @@ interface MediaCodecWrapper {
Surface createInputSurface();
void setParameters(Bundle params);
MediaCodecInfo getCodecInfo();
MediaFormat getOutputFormat(int index);
}

11
sdk/android/src/java/org/webrtc/MediaCodecWrapperFactoryImpl.java
View File

@ -12,6 +12,7 @@ package org.webrtc;
import android.media.MediaCodec;
import android.media.MediaCodec.BufferInfo;
import android.media.MediaCodecInfo;
import android.media.MediaCrypto;
import android.media.MediaFormat;
import android.os.Bundle;
@ -106,6 +107,16 @@ class MediaCodecWrapperFactoryImpl implements MediaCodecWrapperFactory {
public void setParameters(Bundle params) {
mediaCodec.setParameters(params);
}
@Override
public MediaCodecInfo getCodecInfo() {
return mediaCodec.getCodecInfo();
}
@Override
public MediaFormat getOutputFormat(int index) {
return mediaCodec.getOutputFormat(index);
}
}
@Override

11
sdk/android/tests/src/org/webrtc/FakeMediaCodecWrapper.java
View File

@ -12,6 +12,7 @@ package org.webrtc;
import android.graphics.SurfaceTexture;
import android.media.MediaCodec;
import android.media.MediaCodecInfo;
import android.media.MediaCodecInfo.CodecCapabilities;
import android.media.MediaCrypto;
import android.media.MediaFormat;
@ -318,4 +319,14 @@ public class FakeMediaCodecWrapper implements MediaCodecWrapper {
@Override
public void setParameters(Bundle params) {}
@Override
public MediaCodecInfo getCodecInfo() {
return null;
}
@Override
public MediaFormat getOutputFormat(int index) {
return outputFormat;
}
}

91
sdk/android/tests/src/org/webrtc/HardwareVideoEncoderTest.java
View File

@ -15,6 +15,7 @@ import static java.util.concurrent.TimeUnit.SECONDS;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
@ -22,6 +23,7 @@ import static org.mockito.Mockito.verify;
import android.media.MediaCodec;
import android.media.MediaCodecInfo;
import android.media.MediaFormat;
import android.os.Build.VERSION_CODES;
import android.os.Bundle;
import androidx.test.runner.AndroidJUnit4;
import java.nio.ByteBuffer;
@ -69,13 +71,16 @@ public class HardwareVideoEncoderTest {
private static class TestEncoder extends HardwareVideoEncoder {
private final Object deliverEncodedImageLock = new Object();
private boolean deliverEncodedImageDone = true;
private boolean isEncodingStatisticsSupported;
TestEncoder(MediaCodecWrapperFactory mediaCodecWrapperFactory, String codecName,
VideoCodecMimeType codecType, Integer surfaceColorFormat, Integer yuvColorFormat,
Map<String, String> params, int keyFrameIntervalSec, int forceKeyFrameIntervalMs,
BitrateAdjuster bitrateAdjuster, EglBase14.Context sharedContext) {
BitrateAdjuster bitrateAdjuster, EglBase14.Context sharedContext,
boolean isEncodingStatisticsSupported) {
super(mediaCodecWrapperFactory, codecName, codecType, surfaceColorFormat, yuvColorFormat,
params, keyFrameIntervalSec, forceKeyFrameIntervalMs, bitrateAdjuster, sharedContext);
this.isEncodingStatisticsSupported = isEncodingStatisticsSupported;
}
public void waitDeliverEncodedImage() throws InterruptedException {
@ -118,11 +123,17 @@ public class HardwareVideoEncoderTest {
buffer.flip();
i420Buffer.release();
}
@Override
protected boolean isEncodingStatisticsSupported() {
return isEncodingStatisticsSupported;
}
}
private class TestEncoderBuilder {
private VideoCodecMimeType codecType = VideoCodecMimeType.VP8;
private BitrateAdjuster bitrateAdjuster = new BaseBitrateAdjuster();
private boolean isEncodingStatisticsSupported;
public TestEncoderBuilder setCodecType(VideoCodecMimeType codecType) {
this.codecType = codecType;
@ -134,6 +145,12 @@ public class HardwareVideoEncoderTest {
return this;
}
public TestEncoderBuilder SetIsEncodingStatisticsSupported(
boolean isEncodingStatisticsSupported) {
this.isEncodingStatisticsSupported = isEncodingStatisticsSupported;
return this;
}
public TestEncoder build() {
return new TestEncoder((String name)
-> fakeMediaCodecWrapper,
@ -143,7 +160,7 @@ public class HardwareVideoEncoderTest {
/* params= */ new HashMap<>(),
/* keyFrameIntervalSec= */ 0,
/* forceKeyFrameIntervalMs= */ 0, bitrateAdjuster,
/* sharedContext= */ null);
/* sharedContext= */ null, isEncodingStatisticsSupported);
}
}
@ -193,6 +210,76 @@ public class HardwareVideoEncoderTest {
.isEqualTo(MediaCodec.CONFIGURE_FLAG_ENCODE);
}
@Test
public void encodingStatistics_unsupported_disabled() throws InterruptedException {
TestEncoder encoder = new TestEncoderBuilder().SetIsEncodingStatisticsSupported(false).build();
assertThat(encoder.initEncode(TEST_ENCODER_SETTINGS, mockEncoderCallback))
.isEqualTo(VideoCodecStatus.OK);
MediaFormat configuredFormat = fakeMediaCodecWrapper.getConfiguredFormat();
assertThat(configuredFormat).isNotNull();
assertThat(configuredFormat.containsKey(MediaFormat.KEY_VIDEO_ENCODING_STATISTICS_LEVEL))
.isFalse();
// Verify that QP is not set in encoded frame even if reported by MediaCodec.
MediaFormat outputFormat = new MediaFormat();
outputFormat.setInteger(MediaFormat.KEY_VIDEO_QP_AVERAGE, 123);
doReturn(outputFormat).when(fakeMediaCodecWrapper).getOutputFormat(anyInt());
encoder.encode(createTestVideoFrame(/* timestampNs= */ 42), ENCODE_INFO_KEY_FRAME);
fakeMediaCodecWrapper.addOutputData(CodecTestHelper.generateRandomData(100),
/* presentationTimestampUs= */ 0,
/* flags= */ MediaCodec.BUFFER_FLAG_SYNC_FRAME);
encoder.waitDeliverEncodedImage();
ArgumentCaptor<EncodedImage> videoFrameCaptor = ArgumentCaptor.forClass(EncodedImage.class);
verify(mockEncoderCallback)
.onEncodedFrame(videoFrameCaptor.capture(), any(CodecSpecificInfo.class));
EncodedImage videoFrame = videoFrameCaptor.getValue();
assertThat(videoFrame).isNotNull();
assertThat(videoFrame.qp).isNull();
}
@Test
public void encodingStatistics_supported_enabled() throws InterruptedException {
TestEncoder encoder = new TestEncoderBuilder().SetIsEncodingStatisticsSupported(true).build();
assertThat(encoder.initEncode(TEST_ENCODER_SETTINGS, mockEncoderCallback))
.isEqualTo(VideoCodecStatus.OK);
MediaFormat configuredFormat = fakeMediaCodecWrapper.getConfiguredFormat();
assertThat(configuredFormat).isNotNull();
assertThat(configuredFormat.containsKey(MediaFormat.KEY_VIDEO_ENCODING_STATISTICS_LEVEL))
.isTrue();
assertThat(configuredFormat.getInteger(MediaFormat.KEY_VIDEO_ENCODING_STATISTICS_LEVEL))
.isEqualTo(MediaFormat.VIDEO_ENCODING_STATISTICS_LEVEL_1);
// Verify that QP is set in encoded frame.
MediaFormat outputFormat = new MediaFormat();
outputFormat.setInteger(MediaFormat.KEY_VIDEO_QP_AVERAGE, 123);
doReturn(outputFormat).when(fakeMediaCodecWrapper).getOutputFormat(anyInt());
encoder.encode(createTestVideoFrame(/* timestampNs= */ 42), ENCODE_INFO_KEY_FRAME);
fakeMediaCodecWrapper.addOutputData(CodecTestHelper.generateRandomData(100),
/* presentationTimestampUs= */ 0,
/* flags= */ MediaCodec.BUFFER_FLAG_SYNC_FRAME);
encoder.waitDeliverEncodedImage();
ArgumentCaptor<EncodedImage> videoFrameCaptor = ArgumentCaptor.forClass(EncodedImage.class);
verify(mockEncoderCallback)
.onEncodedFrame(videoFrameCaptor.capture(), any(CodecSpecificInfo.class));
EncodedImage videoFrame = videoFrameCaptor.getValue();
assertThat(videoFrame).isNotNull();
assertThat(videoFrame.qp).isEqualTo(123);
}
@Test
public void testEncodeByteBuffer() {
// Set-up.