diff --git a/modules/video_coding/codecs/test/videocodec_test_mediacodec.cc b/modules/video_coding/codecs/test/videocodec_test_mediacodec.cc index 978fd8856f..3cc87a9db9 100644 --- a/modules/video_coding/codecs/test/videocodec_test_mediacodec.cc +++ b/modules/video_coding/codecs/test/videocodec_test_mediacodec.cc @@ -120,7 +120,7 @@ TEST(VideoCodecTestMediaCodec, ForemanMixedRes100kbpsVp8H264) { const std::vector codecs = {cricket::kVp8CodecName, cricket::kH264CodecName}; const std::vector> resolutions = { - {128, 96}, {160, 120}, {176, 144}, {240, 136}, {320, 240}, {480, 272}}; + {128, 96}, {176, 144}, {320, 240}, {480, 272}}; const std::vector rate_profiles = { {100, kForemanFramerateFps, 0}}; const std::vector quality_thresholds = { diff --git a/sdk/android/api/org/webrtc/VideoEncoder.java b/sdk/android/api/org/webrtc/VideoEncoder.java index 89a9bfd1c1..0d8cf830ae 100644 --- a/sdk/android/api/org/webrtc/VideoEncoder.java +++ b/sdk/android/api/org/webrtc/VideoEncoder.java @@ -258,6 +258,39 @@ public interface VideoEncoder { } } + /** + * Metadata about the Encoder. + */ + public class EncoderInfo { + /** + * The width and height of the incoming video frames should be divisible by + * |requested_resolution_alignment| + */ + public final int requestedResolutionAlignment; + + /** + * Same as above but if true, each simulcast layer should also be divisible by + * |requested_resolution_alignment|. + */ + public final boolean applyAlignmentToAllSimulcastLayers; + + public EncoderInfo( + int requestedResolutionAlignment, boolean applyAlignmentToAllSimulcastLayers) { + this.requestedResolutionAlignment = requestedResolutionAlignment; + this.applyAlignmentToAllSimulcastLayers = applyAlignmentToAllSimulcastLayers; + } + + @CalledByNative("EncoderInfo") + public int getRequestedResolutionAlignment() { + return requestedResolutionAlignment; + } + + @CalledByNative("EncoderInfo") + public boolean getApplyAlignmentToAllSimulcastLayers() { + return applyAlignmentToAllSimulcastLayers; + } + } + public interface Callback { /** * Old encoders assume that the byte buffer held by `frame` is not accessed after the call to @@ -343,4 +376,10 @@ public interface VideoEncoder { * called from arbitrary thread. */ @CalledByNative String getImplementationName(); + + @CalledByNative + default EncoderInfo getEncoderInfo() { + return new EncoderInfo( + /* requestedResolutionAlignment= */ 1, /* applyAlignmentToAllSimulcastLayers= */ false); + } } diff --git a/sdk/android/instrumentationtests/src/org/webrtc/AndroidVideoDecoderInstrumentationTest.java b/sdk/android/instrumentationtests/src/org/webrtc/AndroidVideoDecoderInstrumentationTest.java index 3c9e0f7431..37b9b3a749 100644 --- a/sdk/android/instrumentationtests/src/org/webrtc/AndroidVideoDecoderInstrumentationTest.java +++ b/sdk/android/instrumentationtests/src/org/webrtc/AndroidVideoDecoderInstrumentationTest.java @@ -73,14 +73,17 @@ public final class AndroidVideoDecoderInstrumentationTest { private static final boolean ENABLE_INTEL_VP8_ENCODER = true; private static final boolean ENABLE_H264_HIGH_PROFILE = true; - private static final VideoEncoder.Settings ENCODER_SETTINGS = - new VideoEncoder.Settings(1 /* core */, TEST_FRAME_WIDTH, TEST_FRAME_HEIGHT, 300 /* kbps */, - 30 /* fps */, 1 /* numberOfSimulcastStreams */, true /* automaticResizeOn */, - /* capabilities= */ new VideoEncoder.Capabilities(false /* lossNotification */)); + private static final VideoEncoder.Settings ENCODER_SETTINGS = new VideoEncoder.Settings( + 1 /* core */, + getAlignedNumber(TEST_FRAME_WIDTH, HardwareVideoEncoderTest.getPixelAlignmentRequired()), + getAlignedNumber(TEST_FRAME_HEIGHT, HardwareVideoEncoderTest.getPixelAlignmentRequired()), + 300 /* kbps */, 30 /* fps */, 1 /* numberOfSimulcastStreams */, true /* automaticResizeOn */, + /* capabilities= */ new VideoEncoder.Capabilities(false /* lossNotification */)); private static final int DECODE_TIMEOUT_MS = 1000; - private static final VideoDecoder.Settings SETTINGS = - new VideoDecoder.Settings(1 /* core */, TEST_FRAME_WIDTH, TEST_FRAME_HEIGHT); + private static final VideoDecoder.Settings SETTINGS = new VideoDecoder.Settings(1 /* core */, + getAlignedNumber(TEST_FRAME_WIDTH, HardwareVideoEncoderTest.getPixelAlignmentRequired()), + getAlignedNumber(TEST_FRAME_HEIGHT, HardwareVideoEncoderTest.getPixelAlignmentRequired())); private static class MockDecodeCallback implements VideoDecoder.Callback { private BlockingQueue frameQueue = new LinkedBlockingQueue<>(); @@ -116,7 +119,10 @@ public final class AndroidVideoDecoderInstrumentationTest { private static VideoFrame.I420Buffer[] generateTestFrames() { VideoFrame.I420Buffer[] result = new VideoFrame.I420Buffer[TEST_FRAME_COUNT]; for (int i = 0; i < TEST_FRAME_COUNT; i++) { - result[i] = JavaI420Buffer.allocate(TEST_FRAME_WIDTH, TEST_FRAME_HEIGHT); + result[i] = JavaI420Buffer.allocate( + getAlignedNumber(TEST_FRAME_WIDTH, HardwareVideoEncoderTest.getPixelAlignmentRequired()), + getAlignedNumber( + TEST_FRAME_HEIGHT, HardwareVideoEncoderTest.getPixelAlignmentRequired())); // TODO(sakal): Generate content for the test frames. } return result; @@ -156,6 +162,10 @@ public final class AndroidVideoDecoderInstrumentationTest { assertEquals(VideoCodecStatus.OK, encoder.release()); } + private static int getAlignedNumber(int number, int alignment) { + return (number / alignment) * alignment; + } + @Before public void setUp() { NativeLibrary.initialize(new NativeLibrary.DefaultLoader(), TestConstants.NATIVE_LIBRARY); diff --git a/sdk/android/instrumentationtests/src/org/webrtc/HardwareVideoEncoderTest.java b/sdk/android/instrumentationtests/src/org/webrtc/HardwareVideoEncoderTest.java index 1542a39808..3a61b6edc2 100644 --- a/sdk/android/instrumentationtests/src/org/webrtc/HardwareVideoEncoderTest.java +++ b/sdk/android/instrumentationtests/src/org/webrtc/HardwareVideoEncoderTest.java @@ -11,6 +11,7 @@ package org.webrtc; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -75,6 +76,8 @@ public class HardwareVideoEncoderTest { private static final int NUM_TEST_FRAMES = 10; private static final int NUM_ENCODE_TRIES = 100; private static final int ENCODE_RETRY_SLEEP_MS = 1; + private static final int PIXEL_ALIGNMENT_REQUIRED = 16; + private static final boolean APPLY_ALIGNMENT_TO_ALL_SIMULCAST_LAYERS = false; // # Mock classes /** @@ -322,7 +325,7 @@ public class HardwareVideoEncoderTest { return useTextures ? generateTextureFrame(width, height) : generateI420Frame(width, height); } - static void testEncodeFrame( + static VideoCodecStatus testEncodeFrame( VideoEncoder encoder, VideoFrame frame, VideoEncoder.EncodeInfo info) { int numTries = 0; @@ -332,8 +335,10 @@ public class HardwareVideoEncoderTest { final VideoCodecStatus returnValue = encoder.encode(frame, info); switch (returnValue) { - case OK: - return; // Success + case OK: // Success + // Fall through + case ERR_SIZE: // Wrong size + return returnValue; case NO_OUTPUT: if (numTries >= NUM_ENCODE_TRIES) { fail("encoder.encode keeps returning NO_OUTPUT"); @@ -350,6 +355,14 @@ public class HardwareVideoEncoderTest { } } + private static int getAlignedNumber(int number, int alignment) { + return (number / alignment) * alignment; + } + + public static int getPixelAlignmentRequired() { + return PIXEL_ALIGNMENT_REQUIRED; + } + // # Tests @Before public void setUp() { @@ -446,11 +459,60 @@ public class HardwareVideoEncoderTest { callback.assertFrameEncoded(frame); frame.release(); - frame = generateFrame(SETTINGS.width / 4, SETTINGS.height / 4); + // Android MediaCodec only guarantees of proper operation with 16-pixel-aligned input frame. + // Force the size of input frame with the greatest multiple of 16 below the original size. + frame = generateFrame(getAlignedNumber(SETTINGS.width / 4, PIXEL_ALIGNMENT_REQUIRED), + getAlignedNumber(SETTINGS.height / 4, PIXEL_ALIGNMENT_REQUIRED)); testEncodeFrame(encoder, frame, info); callback.assertFrameEncoded(frame); frame.release(); assertEquals(VideoCodecStatus.OK, encoder.release()); } + + @Test + @SmallTest + public void testEncodeAlignmentCheck() { + VideoEncoder encoder = createEncoder(); + org.webrtc.HardwareVideoEncoderTest.MockEncoderCallback callback = + new org.webrtc.HardwareVideoEncoderTest.MockEncoderCallback(); + assertEquals(VideoCodecStatus.OK, encoder.initEncode(SETTINGS, callback)); + + VideoFrame frame; + VideoEncoder.EncodeInfo info = new VideoEncoder.EncodeInfo( + new EncodedImage.FrameType[] {EncodedImage.FrameType.VideoFrameDelta}); + + frame = generateFrame(SETTINGS.width / 2, SETTINGS.height / 2); + assertEquals(VideoCodecStatus.OK, testEncodeFrame(encoder, frame, info)); + frame.release(); + + // Android MediaCodec only guarantees of proper operation with 16-pixel-aligned input frame. + // Following input frame with non-aligned size would return ERR_SIZE. + frame = generateFrame(SETTINGS.width / 4, SETTINGS.height / 4); + assertNotEquals(VideoCodecStatus.OK, testEncodeFrame(encoder, frame, info)); + frame.release(); + + // Since our encoder has returned with an error, we reinitialize the encoder. + assertEquals(VideoCodecStatus.OK, encoder.release()); + assertEquals(VideoCodecStatus.OK, encoder.initEncode(SETTINGS, callback)); + + frame = generateFrame(getAlignedNumber(SETTINGS.width / 4, PIXEL_ALIGNMENT_REQUIRED), + getAlignedNumber(SETTINGS.height / 4, PIXEL_ALIGNMENT_REQUIRED)); + assertEquals(VideoCodecStatus.OK, testEncodeFrame(encoder, frame, info)); + frame.release(); + + assertEquals(VideoCodecStatus.OK, encoder.release()); + } + + @Test + @SmallTest + public void testGetEncoderInfo() { + VideoEncoder encoder = createEncoder(); + assertEquals(VideoCodecStatus.OK, encoder.initEncode(SETTINGS, null)); + VideoEncoder.EncoderInfo info = encoder.getEncoderInfo(); + assertEquals(PIXEL_ALIGNMENT_REQUIRED, info.getRequestedResolutionAlignment()); + assertEquals( + APPLY_ALIGNMENT_TO_ALL_SIMULCAST_LAYERS, info.getApplyAlignmentToAllSimulcastLayers()); + assertEquals(VideoCodecStatus.OK, encoder.release()); + } } diff --git a/sdk/android/native_unittests/org/webrtc/FakeVideoEncoder.java b/sdk/android/native_unittests/org/webrtc/FakeVideoEncoder.java index 8359b49c5f..513f145518 100644 --- a/sdk/android/native_unittests/org/webrtc/FakeVideoEncoder.java +++ b/sdk/android/native_unittests/org/webrtc/FakeVideoEncoder.java @@ -10,6 +10,8 @@ package org.webrtc; +import org.webrtc.VideoEncoder; + /** * An implementation of VideoEncoder that is used for testing of functionalities of * VideoEncoderWrapper. diff --git a/sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java b/sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java index 5c2721ac69..7e59d2134d 100644 --- a/sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java +++ b/sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java @@ -54,6 +54,9 @@ class HardwareVideoEncoder implements VideoEncoder { private static final int MEDIA_CODEC_RELEASE_TIMEOUT_MS = 5000; private static final int DEQUEUE_OUTPUT_BUFFER_TIMEOUT_US = 100000; + // Size of the input frames should be multiple of 16 for the H/W encoder. + private static final int REQUIRED_RESOLUTION_ALIGNMENT = 16; + /** * Keeps track of the number of output buffers that have been passed down the pipeline and not yet * released. We need to wait for this to go down to zero before operations invalidating the output @@ -207,6 +210,12 @@ class HardwareVideoEncoder implements VideoEncoder { this.callback = callback; automaticResizeOn = settings.automaticResizeOn; + + if (settings.width % REQUIRED_RESOLUTION_ALIGNMENT != 0 + || settings.height % REQUIRED_RESOLUTION_ALIGNMENT != 0) { + Logging.e(TAG, "MediaCodec is only tested with resolutions that are 16x16 aligned."); + return VideoCodecStatus.ERR_SIZE; + } this.width = settings.width; this.height = settings.height; useSurfaceMode = canUseSurface(); @@ -498,12 +507,28 @@ class HardwareVideoEncoder implements VideoEncoder { return "HWEncoder"; } + @Override + public EncoderInfo getEncoderInfo() { + // Since our MediaCodec is guaranteed to encode 16-pixel-aligned frames only, we set alignment + // value to be 16. Additionally, this encoder produces a single stream. So it should not require + // alignment for all layers. + return new EncoderInfo( + /* requestedResolutionAlignment= */ REQUIRED_RESOLUTION_ALIGNMENT, + /* applyAlignmentToAllSimulcastLayers= */ false); + } + private VideoCodecStatus resetCodec(int newWidth, int newHeight, boolean newUseSurfaceMode) { encodeThreadChecker.checkIsOnValidThread(); VideoCodecStatus status = release(); if (status != VideoCodecStatus.OK) { return status; } + + if (newWidth % REQUIRED_RESOLUTION_ALIGNMENT != 0 + || newHeight % REQUIRED_RESOLUTION_ALIGNMENT != 0) { + Logging.e(TAG, "MediaCodec is only tested with resolutions that are 16x16 aligned."); + return VideoCodecStatus.ERR_SIZE; + } width = newWidth; height = newHeight; useSurfaceMode = newUseSurfaceMode; diff --git a/sdk/android/src/jni/video_encoder_wrapper.cc b/sdk/android/src/jni/video_encoder_wrapper.cc index 5c585c8b16..b919db7f3d 100644 --- a/sdk/android/src/jni/video_encoder_wrapper.cc +++ b/sdk/android/src/jni/video_encoder_wrapper.cc @@ -113,6 +113,12 @@ void VideoEncoderWrapper::UpdateEncoderInfo(JNIEnv* jni) { encoder_info_.resolution_bitrate_limits = JavaToNativeResolutionBitrateLimits( jni, Java_VideoEncoder_getResolutionBitrateLimits(jni, encoder_)); + + EncoderInfo info = GetEncoderInfoInternal(jni); + encoder_info_.requested_resolution_alignment = + info.requested_resolution_alignment; + encoder_info_.apply_alignment_to_all_simulcast_layers = + info.apply_alignment_to_all_simulcast_layers; } int32_t VideoEncoderWrapper::RegisterEncodeCompleteCallback( @@ -230,6 +236,26 @@ VideoEncoderWrapper::GetScalingSettingsInternal(JNIEnv* jni) const { } } +VideoEncoder::EncoderInfo VideoEncoderWrapper::GetEncoderInfoInternal( + JNIEnv* jni) const { + ScopedJavaLocalRef j_encoder_info = + Java_VideoEncoder_getEncoderInfo(jni, encoder_); + + jint requested_resolution_alignment = + Java_EncoderInfo_getRequestedResolutionAlignment(jni, j_encoder_info); + + jboolean apply_alignment_to_all_simulcast_layers = + Java_EncoderInfo_getApplyAlignmentToAllSimulcastLayers(jni, + j_encoder_info); + + VideoEncoder::EncoderInfo info; + info.requested_resolution_alignment = requested_resolution_alignment; + info.apply_alignment_to_all_simulcast_layers = + apply_alignment_to_all_simulcast_layers; + + return info; +} + void VideoEncoderWrapper::OnEncodedFrame( JNIEnv* jni, const JavaRef& j_encoded_image) { diff --git a/sdk/android/src/jni/video_encoder_wrapper.h b/sdk/android/src/jni/video_encoder_wrapper.h index 9cf5c5a4de..5c5aab7588 100644 --- a/sdk/android/src/jni/video_encoder_wrapper.h +++ b/sdk/android/src/jni/video_encoder_wrapper.h @@ -87,6 +87,8 @@ class VideoEncoderWrapper : public VideoEncoder { std::vector GetResolutionBitrateLimits( JNIEnv* jni) const; + VideoEncoder::EncoderInfo GetEncoderInfoInternal(JNIEnv* jni) const; + const ScopedJavaGlobalRef encoder_; const ScopedJavaGlobalRef int_array_class_;