From 5a79d28eba61aea39558a492fb4c0ff4fef427ba Mon Sep 17 00:00:00 2001 From: Jiwon Jung Date: Fri, 8 Oct 2021 16:41:01 +0900 Subject: [PATCH] Require 16x16 alignment when using HardwareVideoEncoder for encoding. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It seems the Android CTS tests only verify that 16x16 aligned resolutions are supported. This change checks the validity of input frame's size when initialing or encoding processes are about to start using H/W MediaCodec. This change has additional APIs to retrieve |requested_resolution_alignment| and |apply_alignment_to_all_simulcast_layers| from JAVA VideoEncoder class and its inherited classes. HardwareVideoEncoder using MediaCodec has values of 16 and true for above variables. Bug: webrtc:13089 Change-Id: I0c4ebf94eb36da29c2e384a3edf85b82e779b7f9 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/229460 Reviewed-by: Sergey Silkin Reviewed-by: Åsa Persson Commit-Queue: Sergey Silkin Cr-Commit-Position: refs/heads/main@{#35169} --- .../codecs/test/videocodec_test_mediacodec.cc | 2 +- sdk/android/api/org/webrtc/VideoEncoder.java | 39 +++++++++++ ...ndroidVideoDecoderInstrumentationTest.java | 24 +++++-- .../org/webrtc/HardwareVideoEncoderTest.java | 70 +++++++++++++++++-- .../org/webrtc/FakeVideoEncoder.java | 2 + .../java/org/webrtc/HardwareVideoEncoder.java | 25 +++++++ sdk/android/src/jni/video_encoder_wrapper.cc | 26 +++++++ sdk/android/src/jni/video_encoder_wrapper.h | 2 + 8 files changed, 178 insertions(+), 12 deletions(-) diff --git a/modules/video_coding/codecs/test/videocodec_test_mediacodec.cc b/modules/video_coding/codecs/test/videocodec_test_mediacodec.cc index 978fd8856f..3cc87a9db9 100644 --- a/modules/video_coding/codecs/test/videocodec_test_mediacodec.cc +++ b/modules/video_coding/codecs/test/videocodec_test_mediacodec.cc @@ -120,7 +120,7 @@ TEST(VideoCodecTestMediaCodec, ForemanMixedRes100kbpsVp8H264) { const std::vector codecs = {cricket::kVp8CodecName, cricket::kH264CodecName}; const std::vector> resolutions = { - {128, 96}, {160, 120}, {176, 144}, {240, 136}, {320, 240}, {480, 272}}; + {128, 96}, {176, 144}, {320, 240}, {480, 272}}; const std::vector rate_profiles = { {100, kForemanFramerateFps, 0}}; const std::vector quality_thresholds = { diff --git a/sdk/android/api/org/webrtc/VideoEncoder.java b/sdk/android/api/org/webrtc/VideoEncoder.java index 89a9bfd1c1..0d8cf830ae 100644 --- a/sdk/android/api/org/webrtc/VideoEncoder.java +++ b/sdk/android/api/org/webrtc/VideoEncoder.java @@ -258,6 +258,39 @@ public interface VideoEncoder { } } + /** + * Metadata about the Encoder. + */ + public class EncoderInfo { + /** + * The width and height of the incoming video frames should be divisible by + * |requested_resolution_alignment| + */ + public final int requestedResolutionAlignment; + + /** + * Same as above but if true, each simulcast layer should also be divisible by + * |requested_resolution_alignment|. + */ + public final boolean applyAlignmentToAllSimulcastLayers; + + public EncoderInfo( + int requestedResolutionAlignment, boolean applyAlignmentToAllSimulcastLayers) { + this.requestedResolutionAlignment = requestedResolutionAlignment; + this.applyAlignmentToAllSimulcastLayers = applyAlignmentToAllSimulcastLayers; + } + + @CalledByNative("EncoderInfo") + public int getRequestedResolutionAlignment() { + return requestedResolutionAlignment; + } + + @CalledByNative("EncoderInfo") + public boolean getApplyAlignmentToAllSimulcastLayers() { + return applyAlignmentToAllSimulcastLayers; + } + } + public interface Callback { /** * Old encoders assume that the byte buffer held by `frame` is not accessed after the call to @@ -343,4 +376,10 @@ public interface VideoEncoder { * called from arbitrary thread. */ @CalledByNative String getImplementationName(); + + @CalledByNative + default EncoderInfo getEncoderInfo() { + return new EncoderInfo( + /* requestedResolutionAlignment= */ 1, /* applyAlignmentToAllSimulcastLayers= */ false); + } } diff --git a/sdk/android/instrumentationtests/src/org/webrtc/AndroidVideoDecoderInstrumentationTest.java b/sdk/android/instrumentationtests/src/org/webrtc/AndroidVideoDecoderInstrumentationTest.java index 3c9e0f7431..37b9b3a749 100644 --- a/sdk/android/instrumentationtests/src/org/webrtc/AndroidVideoDecoderInstrumentationTest.java +++ b/sdk/android/instrumentationtests/src/org/webrtc/AndroidVideoDecoderInstrumentationTest.java @@ -73,14 +73,17 @@ public final class AndroidVideoDecoderInstrumentationTest { private static final boolean ENABLE_INTEL_VP8_ENCODER = true; private static final boolean ENABLE_H264_HIGH_PROFILE = true; - private static final VideoEncoder.Settings ENCODER_SETTINGS = - new VideoEncoder.Settings(1 /* core */, TEST_FRAME_WIDTH, TEST_FRAME_HEIGHT, 300 /* kbps */, - 30 /* fps */, 1 /* numberOfSimulcastStreams */, true /* automaticResizeOn */, - /* capabilities= */ new VideoEncoder.Capabilities(false /* lossNotification */)); + private static final VideoEncoder.Settings ENCODER_SETTINGS = new VideoEncoder.Settings( + 1 /* core */, + getAlignedNumber(TEST_FRAME_WIDTH, HardwareVideoEncoderTest.getPixelAlignmentRequired()), + getAlignedNumber(TEST_FRAME_HEIGHT, HardwareVideoEncoderTest.getPixelAlignmentRequired()), + 300 /* kbps */, 30 /* fps */, 1 /* numberOfSimulcastStreams */, true /* automaticResizeOn */, + /* capabilities= */ new VideoEncoder.Capabilities(false /* lossNotification */)); private static final int DECODE_TIMEOUT_MS = 1000; - private static final VideoDecoder.Settings SETTINGS = - new VideoDecoder.Settings(1 /* core */, TEST_FRAME_WIDTH, TEST_FRAME_HEIGHT); + private static final VideoDecoder.Settings SETTINGS = new VideoDecoder.Settings(1 /* core */, + getAlignedNumber(TEST_FRAME_WIDTH, HardwareVideoEncoderTest.getPixelAlignmentRequired()), + getAlignedNumber(TEST_FRAME_HEIGHT, HardwareVideoEncoderTest.getPixelAlignmentRequired())); private static class MockDecodeCallback implements VideoDecoder.Callback { private BlockingQueue frameQueue = new LinkedBlockingQueue<>(); @@ -116,7 +119,10 @@ public final class AndroidVideoDecoderInstrumentationTest { private static VideoFrame.I420Buffer[] generateTestFrames() { VideoFrame.I420Buffer[] result = new VideoFrame.I420Buffer[TEST_FRAME_COUNT]; for (int i = 0; i < TEST_FRAME_COUNT; i++) { - result[i] = JavaI420Buffer.allocate(TEST_FRAME_WIDTH, TEST_FRAME_HEIGHT); + result[i] = JavaI420Buffer.allocate( + getAlignedNumber(TEST_FRAME_WIDTH, HardwareVideoEncoderTest.getPixelAlignmentRequired()), + getAlignedNumber( + TEST_FRAME_HEIGHT, HardwareVideoEncoderTest.getPixelAlignmentRequired())); // TODO(sakal): Generate content for the test frames. } return result; @@ -156,6 +162,10 @@ public final class AndroidVideoDecoderInstrumentationTest { assertEquals(VideoCodecStatus.OK, encoder.release()); } + private static int getAlignedNumber(int number, int alignment) { + return (number / alignment) * alignment; + } + @Before public void setUp() { NativeLibrary.initialize(new NativeLibrary.DefaultLoader(), TestConstants.NATIVE_LIBRARY); diff --git a/sdk/android/instrumentationtests/src/org/webrtc/HardwareVideoEncoderTest.java b/sdk/android/instrumentationtests/src/org/webrtc/HardwareVideoEncoderTest.java index 1542a39808..3a61b6edc2 100644 --- a/sdk/android/instrumentationtests/src/org/webrtc/HardwareVideoEncoderTest.java +++ b/sdk/android/instrumentationtests/src/org/webrtc/HardwareVideoEncoderTest.java @@ -11,6 +11,7 @@ package org.webrtc; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -75,6 +76,8 @@ public class HardwareVideoEncoderTest { private static final int NUM_TEST_FRAMES = 10; private static final int NUM_ENCODE_TRIES = 100; private static final int ENCODE_RETRY_SLEEP_MS = 1; + private static final int PIXEL_ALIGNMENT_REQUIRED = 16; + private static final boolean APPLY_ALIGNMENT_TO_ALL_SIMULCAST_LAYERS = false; // # Mock classes /** @@ -322,7 +325,7 @@ public class HardwareVideoEncoderTest { return useTextures ? generateTextureFrame(width, height) : generateI420Frame(width, height); } - static void testEncodeFrame( + static VideoCodecStatus testEncodeFrame( VideoEncoder encoder, VideoFrame frame, VideoEncoder.EncodeInfo info) { int numTries = 0; @@ -332,8 +335,10 @@ public class HardwareVideoEncoderTest { final VideoCodecStatus returnValue = encoder.encode(frame, info); switch (returnValue) { - case OK: - return; // Success + case OK: // Success + // Fall through + case ERR_SIZE: // Wrong size + return returnValue; case NO_OUTPUT: if (numTries >= NUM_ENCODE_TRIES) { fail("encoder.encode keeps returning NO_OUTPUT"); @@ -350,6 +355,14 @@ public class HardwareVideoEncoderTest { } } + private static int getAlignedNumber(int number, int alignment) { + return (number / alignment) * alignment; + } + + public static int getPixelAlignmentRequired() { + return PIXEL_ALIGNMENT_REQUIRED; + } + // # Tests @Before public void setUp() { @@ -446,11 +459,60 @@ public class HardwareVideoEncoderTest { callback.assertFrameEncoded(frame); frame.release(); - frame = generateFrame(SETTINGS.width / 4, SETTINGS.height / 4); + // Android MediaCodec only guarantees of proper operation with 16-pixel-aligned input frame. + // Force the size of input frame with the greatest multiple of 16 below the original size. + frame = generateFrame(getAlignedNumber(SETTINGS.width / 4, PIXEL_ALIGNMENT_REQUIRED), + getAlignedNumber(SETTINGS.height / 4, PIXEL_ALIGNMENT_REQUIRED)); testEncodeFrame(encoder, frame, info); callback.assertFrameEncoded(frame); frame.release(); assertEquals(VideoCodecStatus.OK, encoder.release()); } + + @Test + @SmallTest + public void testEncodeAlignmentCheck() { + VideoEncoder encoder = createEncoder(); + org.webrtc.HardwareVideoEncoderTest.MockEncoderCallback callback = + new org.webrtc.HardwareVideoEncoderTest.MockEncoderCallback(); + assertEquals(VideoCodecStatus.OK, encoder.initEncode(SETTINGS, callback)); + + VideoFrame frame; + VideoEncoder.EncodeInfo info = new VideoEncoder.EncodeInfo( + new EncodedImage.FrameType[] {EncodedImage.FrameType.VideoFrameDelta}); + + frame = generateFrame(SETTINGS.width / 2, SETTINGS.height / 2); + assertEquals(VideoCodecStatus.OK, testEncodeFrame(encoder, frame, info)); + frame.release(); + + // Android MediaCodec only guarantees of proper operation with 16-pixel-aligned input frame. + // Following input frame with non-aligned size would return ERR_SIZE. + frame = generateFrame(SETTINGS.width / 4, SETTINGS.height / 4); + assertNotEquals(VideoCodecStatus.OK, testEncodeFrame(encoder, frame, info)); + frame.release(); + + // Since our encoder has returned with an error, we reinitialize the encoder. + assertEquals(VideoCodecStatus.OK, encoder.release()); + assertEquals(VideoCodecStatus.OK, encoder.initEncode(SETTINGS, callback)); + + frame = generateFrame(getAlignedNumber(SETTINGS.width / 4, PIXEL_ALIGNMENT_REQUIRED), + getAlignedNumber(SETTINGS.height / 4, PIXEL_ALIGNMENT_REQUIRED)); + assertEquals(VideoCodecStatus.OK, testEncodeFrame(encoder, frame, info)); + frame.release(); + + assertEquals(VideoCodecStatus.OK, encoder.release()); + } + + @Test + @SmallTest + public void testGetEncoderInfo() { + VideoEncoder encoder = createEncoder(); + assertEquals(VideoCodecStatus.OK, encoder.initEncode(SETTINGS, null)); + VideoEncoder.EncoderInfo info = encoder.getEncoderInfo(); + assertEquals(PIXEL_ALIGNMENT_REQUIRED, info.getRequestedResolutionAlignment()); + assertEquals( + APPLY_ALIGNMENT_TO_ALL_SIMULCAST_LAYERS, info.getApplyAlignmentToAllSimulcastLayers()); + assertEquals(VideoCodecStatus.OK, encoder.release()); + } } diff --git a/sdk/android/native_unittests/org/webrtc/FakeVideoEncoder.java b/sdk/android/native_unittests/org/webrtc/FakeVideoEncoder.java index 8359b49c5f..513f145518 100644 --- a/sdk/android/native_unittests/org/webrtc/FakeVideoEncoder.java +++ b/sdk/android/native_unittests/org/webrtc/FakeVideoEncoder.java @@ -10,6 +10,8 @@ package org.webrtc; +import org.webrtc.VideoEncoder; + /** * An implementation of VideoEncoder that is used for testing of functionalities of * VideoEncoderWrapper. diff --git a/sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java b/sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java index 5c2721ac69..7e59d2134d 100644 --- a/sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java +++ b/sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java @@ -54,6 +54,9 @@ class HardwareVideoEncoder implements VideoEncoder { private static final int MEDIA_CODEC_RELEASE_TIMEOUT_MS = 5000; private static final int DEQUEUE_OUTPUT_BUFFER_TIMEOUT_US = 100000; + // Size of the input frames should be multiple of 16 for the H/W encoder. + private static final int REQUIRED_RESOLUTION_ALIGNMENT = 16; + /** * Keeps track of the number of output buffers that have been passed down the pipeline and not yet * released. We need to wait for this to go down to zero before operations invalidating the output @@ -207,6 +210,12 @@ class HardwareVideoEncoder implements VideoEncoder { this.callback = callback; automaticResizeOn = settings.automaticResizeOn; + + if (settings.width % REQUIRED_RESOLUTION_ALIGNMENT != 0 + || settings.height % REQUIRED_RESOLUTION_ALIGNMENT != 0) { + Logging.e(TAG, "MediaCodec is only tested with resolutions that are 16x16 aligned."); + return VideoCodecStatus.ERR_SIZE; + } this.width = settings.width; this.height = settings.height; useSurfaceMode = canUseSurface(); @@ -498,12 +507,28 @@ class HardwareVideoEncoder implements VideoEncoder { return "HWEncoder"; } + @Override + public EncoderInfo getEncoderInfo() { + // Since our MediaCodec is guaranteed to encode 16-pixel-aligned frames only, we set alignment + // value to be 16. Additionally, this encoder produces a single stream. So it should not require + // alignment for all layers. + return new EncoderInfo( + /* requestedResolutionAlignment= */ REQUIRED_RESOLUTION_ALIGNMENT, + /* applyAlignmentToAllSimulcastLayers= */ false); + } + private VideoCodecStatus resetCodec(int newWidth, int newHeight, boolean newUseSurfaceMode) { encodeThreadChecker.checkIsOnValidThread(); VideoCodecStatus status = release(); if (status != VideoCodecStatus.OK) { return status; } + + if (newWidth % REQUIRED_RESOLUTION_ALIGNMENT != 0 + || newHeight % REQUIRED_RESOLUTION_ALIGNMENT != 0) { + Logging.e(TAG, "MediaCodec is only tested with resolutions that are 16x16 aligned."); + return VideoCodecStatus.ERR_SIZE; + } width = newWidth; height = newHeight; useSurfaceMode = newUseSurfaceMode; diff --git a/sdk/android/src/jni/video_encoder_wrapper.cc b/sdk/android/src/jni/video_encoder_wrapper.cc index 5c585c8b16..b919db7f3d 100644 --- a/sdk/android/src/jni/video_encoder_wrapper.cc +++ b/sdk/android/src/jni/video_encoder_wrapper.cc @@ -113,6 +113,12 @@ void VideoEncoderWrapper::UpdateEncoderInfo(JNIEnv* jni) { encoder_info_.resolution_bitrate_limits = JavaToNativeResolutionBitrateLimits( jni, Java_VideoEncoder_getResolutionBitrateLimits(jni, encoder_)); + + EncoderInfo info = GetEncoderInfoInternal(jni); + encoder_info_.requested_resolution_alignment = + info.requested_resolution_alignment; + encoder_info_.apply_alignment_to_all_simulcast_layers = + info.apply_alignment_to_all_simulcast_layers; } int32_t VideoEncoderWrapper::RegisterEncodeCompleteCallback( @@ -230,6 +236,26 @@ VideoEncoderWrapper::GetScalingSettingsInternal(JNIEnv* jni) const { } } +VideoEncoder::EncoderInfo VideoEncoderWrapper::GetEncoderInfoInternal( + JNIEnv* jni) const { + ScopedJavaLocalRef j_encoder_info = + Java_VideoEncoder_getEncoderInfo(jni, encoder_); + + jint requested_resolution_alignment = + Java_EncoderInfo_getRequestedResolutionAlignment(jni, j_encoder_info); + + jboolean apply_alignment_to_all_simulcast_layers = + Java_EncoderInfo_getApplyAlignmentToAllSimulcastLayers(jni, + j_encoder_info); + + VideoEncoder::EncoderInfo info; + info.requested_resolution_alignment = requested_resolution_alignment; + info.apply_alignment_to_all_simulcast_layers = + apply_alignment_to_all_simulcast_layers; + + return info; +} + void VideoEncoderWrapper::OnEncodedFrame( JNIEnv* jni, const JavaRef& j_encoded_image) { diff --git a/sdk/android/src/jni/video_encoder_wrapper.h b/sdk/android/src/jni/video_encoder_wrapper.h index 9cf5c5a4de..5c5aab7588 100644 --- a/sdk/android/src/jni/video_encoder_wrapper.h +++ b/sdk/android/src/jni/video_encoder_wrapper.h @@ -87,6 +87,8 @@ class VideoEncoderWrapper : public VideoEncoder { std::vector GetResolutionBitrateLimits( JNIEnv* jni) const; + VideoEncoder::EncoderInfo GetEncoderInfoInternal(JNIEnv* jni) const; + const ScopedJavaGlobalRef encoder_; const ScopedJavaGlobalRef int_array_class_;