diff --git a/sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java b/sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java index b5f7629230..bd01b7d0f3 100644 --- a/sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java +++ b/sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java @@ -16,6 +16,7 @@ import static android.media.MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_CBR; import android.media.MediaCodec; import android.media.MediaCodecInfo; +import android.media.MediaCodecInfo.CodecCapabilities; import android.media.MediaFormat; import android.opengl.GLES20; import android.os.Build; @@ -162,6 +163,9 @@ class HardwareVideoEncoder implements VideoEncoder { // value to send exceptions thrown during release back to the encoder thread. @Nullable private volatile Exception shutdownException; + // True if collection of encoding statistics is enabled. + private boolean isEncodingStatisticsEnabled; + /** * Creates a new HardwareVideoEncoder with the given codecName, codecType, colorFormat, key frame * intervals, and bitrateAdjuster. @@ -226,6 +230,8 @@ class HardwareVideoEncoder implements VideoEncoder { nextPresentationTimestampUs = 0; lastKeyFrameNs = -1; + isEncodingStatisticsEnabled = false; + try { codec = mediaCodecWrapperFactory.createByCodecName(codecName); } catch (IOException | IllegalArgumentException e) { @@ -258,6 +264,13 @@ class HardwareVideoEncoder implements VideoEncoder { Logging.w(TAG, "Unknown profile level id: " + profileLevelId); } } + + if (isEncodingStatisticsSupported()) { + format.setInteger(MediaFormat.KEY_VIDEO_ENCODING_STATISTICS_LEVEL, + MediaFormat.VIDEO_ENCODING_STATISTICS_LEVEL_1); + isEncodingStatisticsEnabled = true; + } + Logging.d(TAG, "Format: " + format); codec.configure( format, null /* surface */, null /* crypto */, MediaCodec.CONFIGURE_FLAG_ENCODE); @@ -606,21 +619,30 @@ class HardwareVideoEncoder implements VideoEncoder { outputBuffersBusyCount.increment(); EncodedImage.Builder builder = outputBuilders.poll(); - EncodedImage encodedImage = builder - .setBuffer(frameBuffer, - () -> { - // This callback should not throw any exceptions since - // it may be called on an arbitrary thread. - // Check bug webrtc:11230 for more details. - try { - codec.releaseOutputBuffer(index, false); - } catch (Exception e) { - Logging.e(TAG, "releaseOutputBuffer failed", e); - } - outputBuffersBusyCount.decrement(); - }) - .setFrameType(frameType) - .createEncodedImage(); + builder + .setBuffer(frameBuffer, + () -> { + // This callback should not throw any exceptions since + // it may be called on an arbitrary thread. + // Check bug webrtc:11230 for more details. + try { + codec.releaseOutputBuffer(index, false); + } catch (Exception e) { + Logging.e(TAG, "releaseOutputBuffer failed", e); + } + outputBuffersBusyCount.decrement(); + }) + .setFrameType(frameType); + + if (isEncodingStatisticsEnabled) { + MediaFormat format = codec.getOutputFormat(index); + if (format != null && format.containsKey(MediaFormat.KEY_VIDEO_QP_AVERAGE)) { + int qp = format.getInteger(MediaFormat.KEY_VIDEO_QP_AVERAGE); + builder.setQp(qp); + } + } + + EncodedImage encodedImage = builder.createEncodedImage(); // TODO(mellem): Set codec-specific info. callback.onEncodedFrame(encodedImage, new CodecSpecificInfo()); // Note that the callback may have retained the image. @@ -685,6 +707,29 @@ class HardwareVideoEncoder implements VideoEncoder { return height; } + protected boolean isEncodingStatisticsSupported() { + // WebRTC quality scaler, which adjusts resolution and/or frame rate based on encoded QP, + // expects QP to be in native bitstream range for given codec. Native QP range for VP8 is + // [0, 127] and for VP9 is [0, 255]. MediaCodec VP8 and VP9 encoders (perhaps not all) + // return QP in range [0, 64], which is libvpx API specific range. Due to this mismatch we + // can't use QP feedback from these codecs. + if (codecType == VideoCodecMimeType.VP8 || codecType == VideoCodecMimeType.VP9) { + return false; + } + + MediaCodecInfo codecInfo = codec.getCodecInfo(); + if (codecInfo == null) { + return false; + } + + CodecCapabilities codecCaps = codecInfo.getCapabilitiesForType(codecType.mimeType()); + if (codecCaps == null) { + return false; + } + + return codecCaps.isFeatureSupported(CodecCapabilities.FEATURE_EncodingStatistics); + } + // Visible for testing. protected void fillInputBuffer(ByteBuffer buffer, VideoFrame.Buffer videoFrameBuffer) { yuvFormat.fillBuffer(buffer, videoFrameBuffer, stride, sliceHeight); diff --git a/sdk/android/src/java/org/webrtc/MediaCodecWrapper.java b/sdk/android/src/java/org/webrtc/MediaCodecWrapper.java index 60c853df35..6abdbfe6cd 100644 --- a/sdk/android/src/java/org/webrtc/MediaCodecWrapper.java +++ b/sdk/android/src/java/org/webrtc/MediaCodecWrapper.java @@ -11,6 +11,7 @@ package org.webrtc; import android.media.MediaCodec; +import android.media.MediaCodecInfo; import android.media.MediaCrypto; import android.media.MediaFormat; import android.os.Bundle; @@ -52,4 +53,8 @@ interface MediaCodecWrapper { Surface createInputSurface(); void setParameters(Bundle params); + + MediaCodecInfo getCodecInfo(); + + MediaFormat getOutputFormat(int index); } diff --git a/sdk/android/src/java/org/webrtc/MediaCodecWrapperFactoryImpl.java b/sdk/android/src/java/org/webrtc/MediaCodecWrapperFactoryImpl.java index 2ba62ac7d6..56ab21fbb9 100644 --- a/sdk/android/src/java/org/webrtc/MediaCodecWrapperFactoryImpl.java +++ b/sdk/android/src/java/org/webrtc/MediaCodecWrapperFactoryImpl.java @@ -12,6 +12,7 @@ package org.webrtc; import android.media.MediaCodec; import android.media.MediaCodec.BufferInfo; +import android.media.MediaCodecInfo; import android.media.MediaCrypto; import android.media.MediaFormat; import android.os.Bundle; @@ -106,6 +107,16 @@ class MediaCodecWrapperFactoryImpl implements MediaCodecWrapperFactory { public void setParameters(Bundle params) { mediaCodec.setParameters(params); } + + @Override + public MediaCodecInfo getCodecInfo() { + return mediaCodec.getCodecInfo(); + } + + @Override + public MediaFormat getOutputFormat(int index) { + return mediaCodec.getOutputFormat(index); + } } @Override diff --git a/sdk/android/tests/src/org/webrtc/FakeMediaCodecWrapper.java b/sdk/android/tests/src/org/webrtc/FakeMediaCodecWrapper.java index fb7aba4700..5e2a1f40d4 100644 --- a/sdk/android/tests/src/org/webrtc/FakeMediaCodecWrapper.java +++ b/sdk/android/tests/src/org/webrtc/FakeMediaCodecWrapper.java @@ -12,6 +12,7 @@ package org.webrtc; import android.graphics.SurfaceTexture; import android.media.MediaCodec; +import android.media.MediaCodecInfo; import android.media.MediaCodecInfo.CodecCapabilities; import android.media.MediaCrypto; import android.media.MediaFormat; @@ -318,4 +319,14 @@ public class FakeMediaCodecWrapper implements MediaCodecWrapper { @Override public void setParameters(Bundle params) {} + + @Override + public MediaCodecInfo getCodecInfo() { + return null; + } + + @Override + public MediaFormat getOutputFormat(int index) { + return outputFormat; + } } diff --git a/sdk/android/tests/src/org/webrtc/HardwareVideoEncoderTest.java b/sdk/android/tests/src/org/webrtc/HardwareVideoEncoderTest.java index bd4a642f00..36bfb20036 100644 --- a/sdk/android/tests/src/org/webrtc/HardwareVideoEncoderTest.java +++ b/sdk/android/tests/src/org/webrtc/HardwareVideoEncoderTest.java @@ -15,6 +15,7 @@ import static java.util.concurrent.TimeUnit.SECONDS; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -22,6 +23,7 @@ import static org.mockito.Mockito.verify; import android.media.MediaCodec; import android.media.MediaCodecInfo; import android.media.MediaFormat; +import android.os.Build.VERSION_CODES; import android.os.Bundle; import androidx.test.runner.AndroidJUnit4; import java.nio.ByteBuffer; @@ -69,13 +71,16 @@ public class HardwareVideoEncoderTest { private static class TestEncoder extends HardwareVideoEncoder { private final Object deliverEncodedImageLock = new Object(); private boolean deliverEncodedImageDone = true; + private boolean isEncodingStatisticsSupported; TestEncoder(MediaCodecWrapperFactory mediaCodecWrapperFactory, String codecName, VideoCodecMimeType codecType, Integer surfaceColorFormat, Integer yuvColorFormat, Map params, int keyFrameIntervalSec, int forceKeyFrameIntervalMs, - BitrateAdjuster bitrateAdjuster, EglBase14.Context sharedContext) { + BitrateAdjuster bitrateAdjuster, EglBase14.Context sharedContext, + boolean isEncodingStatisticsSupported) { super(mediaCodecWrapperFactory, codecName, codecType, surfaceColorFormat, yuvColorFormat, params, keyFrameIntervalSec, forceKeyFrameIntervalMs, bitrateAdjuster, sharedContext); + this.isEncodingStatisticsSupported = isEncodingStatisticsSupported; } public void waitDeliverEncodedImage() throws InterruptedException { @@ -118,11 +123,17 @@ public class HardwareVideoEncoderTest { buffer.flip(); i420Buffer.release(); } + + @Override + protected boolean isEncodingStatisticsSupported() { + return isEncodingStatisticsSupported; + } } private class TestEncoderBuilder { private VideoCodecMimeType codecType = VideoCodecMimeType.VP8; private BitrateAdjuster bitrateAdjuster = new BaseBitrateAdjuster(); + private boolean isEncodingStatisticsSupported; public TestEncoderBuilder setCodecType(VideoCodecMimeType codecType) { this.codecType = codecType; @@ -134,6 +145,12 @@ public class HardwareVideoEncoderTest { return this; } + public TestEncoderBuilder SetIsEncodingStatisticsSupported( + boolean isEncodingStatisticsSupported) { + this.isEncodingStatisticsSupported = isEncodingStatisticsSupported; + return this; + } + public TestEncoder build() { return new TestEncoder((String name) -> fakeMediaCodecWrapper, @@ -143,7 +160,7 @@ public class HardwareVideoEncoderTest { /* params= */ new HashMap<>(), /* keyFrameIntervalSec= */ 0, /* forceKeyFrameIntervalMs= */ 0, bitrateAdjuster, - /* sharedContext= */ null); + /* sharedContext= */ null, isEncodingStatisticsSupported); } } @@ -193,6 +210,76 @@ public class HardwareVideoEncoderTest { .isEqualTo(MediaCodec.CONFIGURE_FLAG_ENCODE); } + @Test + public void encodingStatistics_unsupported_disabled() throws InterruptedException { + TestEncoder encoder = new TestEncoderBuilder().SetIsEncodingStatisticsSupported(false).build(); + + assertThat(encoder.initEncode(TEST_ENCODER_SETTINGS, mockEncoderCallback)) + .isEqualTo(VideoCodecStatus.OK); + + MediaFormat configuredFormat = fakeMediaCodecWrapper.getConfiguredFormat(); + assertThat(configuredFormat).isNotNull(); + assertThat(configuredFormat.containsKey(MediaFormat.KEY_VIDEO_ENCODING_STATISTICS_LEVEL)) + .isFalse(); + + // Verify that QP is not set in encoded frame even if reported by MediaCodec. + MediaFormat outputFormat = new MediaFormat(); + outputFormat.setInteger(MediaFormat.KEY_VIDEO_QP_AVERAGE, 123); + doReturn(outputFormat).when(fakeMediaCodecWrapper).getOutputFormat(anyInt()); + + encoder.encode(createTestVideoFrame(/* timestampNs= */ 42), ENCODE_INFO_KEY_FRAME); + + fakeMediaCodecWrapper.addOutputData(CodecTestHelper.generateRandomData(100), + /* presentationTimestampUs= */ 0, + /* flags= */ MediaCodec.BUFFER_FLAG_SYNC_FRAME); + + encoder.waitDeliverEncodedImage(); + + ArgumentCaptor videoFrameCaptor = ArgumentCaptor.forClass(EncodedImage.class); + verify(mockEncoderCallback) + .onEncodedFrame(videoFrameCaptor.capture(), any(CodecSpecificInfo.class)); + + EncodedImage videoFrame = videoFrameCaptor.getValue(); + assertThat(videoFrame).isNotNull(); + assertThat(videoFrame.qp).isNull(); + } + + @Test + public void encodingStatistics_supported_enabled() throws InterruptedException { + TestEncoder encoder = new TestEncoderBuilder().SetIsEncodingStatisticsSupported(true).build(); + + assertThat(encoder.initEncode(TEST_ENCODER_SETTINGS, mockEncoderCallback)) + .isEqualTo(VideoCodecStatus.OK); + + MediaFormat configuredFormat = fakeMediaCodecWrapper.getConfiguredFormat(); + assertThat(configuredFormat).isNotNull(); + assertThat(configuredFormat.containsKey(MediaFormat.KEY_VIDEO_ENCODING_STATISTICS_LEVEL)) + .isTrue(); + assertThat(configuredFormat.getInteger(MediaFormat.KEY_VIDEO_ENCODING_STATISTICS_LEVEL)) + .isEqualTo(MediaFormat.VIDEO_ENCODING_STATISTICS_LEVEL_1); + + // Verify that QP is set in encoded frame. + MediaFormat outputFormat = new MediaFormat(); + outputFormat.setInteger(MediaFormat.KEY_VIDEO_QP_AVERAGE, 123); + doReturn(outputFormat).when(fakeMediaCodecWrapper).getOutputFormat(anyInt()); + + encoder.encode(createTestVideoFrame(/* timestampNs= */ 42), ENCODE_INFO_KEY_FRAME); + + fakeMediaCodecWrapper.addOutputData(CodecTestHelper.generateRandomData(100), + /* presentationTimestampUs= */ 0, + /* flags= */ MediaCodec.BUFFER_FLAG_SYNC_FRAME); + + encoder.waitDeliverEncodedImage(); + + ArgumentCaptor videoFrameCaptor = ArgumentCaptor.forClass(EncodedImage.class); + verify(mockEncoderCallback) + .onEncodedFrame(videoFrameCaptor.capture(), any(CodecSpecificInfo.class)); + + EncodedImage videoFrame = videoFrameCaptor.getValue(); + assertThat(videoFrame).isNotNull(); + assertThat(videoFrame.qp).isEqualTo(123); + } + @Test public void testEncodeByteBuffer() { // Set-up.