Reland "Call native codec factories from Android ones."

This is a reland of commit 937a59268e2ae56a58f648fba827444f7beb4466

Check if codec requested in createEncoder/Decoder is supported and return null if not.

Original change's description:
> Call native codec factories from Android ones.
>
> Android video codec factories are expected to be synchronised with the native ones in terms on supported codecs. But before this change there were differences:
>
> 1. Native decoder factory keeps AV1 support behind RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY while Android decoder factory advertises AV1 unconditionally;
>
> 2. Native encoder factory advertises AV1 if RTC_USE_LIBAOM_AV1_ENCODER is enabled while Android encoder factory never advertises AV1.
>
> This CL synchronises the codecs set in Android factories with that of native factories by calling native factories from Android ones.
>
> Bug: webrtc:13573, b/257272020
> Change-Id: I99d801eda0c5f3400bac222b9b08d719f1a6ed72
> Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/282240
> Reviewed-by: Rasmus Brandt <brandtr@webrtc.org>
> Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
> Reviewed-by: Xavier Lepaul‎ <xalep@webrtc.org>
> Cr-Commit-Position: refs/heads/main@{#38583}

Bug: webrtc:13573, b/257272020
Change-Id: Ida7bb9a2954b836a07ad560de29c1f8088264b77
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/282802
Reviewed-by: Xavier Lepaul‎ <xalep@webrtc.org>
Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#38607}
This commit is contained in:
Sergey Silkin 2022-11-09 14:45:42 +00:00 committed by WebRTC LUCI CQ
parent dd7dc25a30
commit c97651cbb4
8 changed files with 330 additions and 103 deletions

View File

@ -885,9 +885,20 @@ if (current_os == "linux" || is_android) {
rtc_library("swcodecs_jni") { rtc_library("swcodecs_jni") {
visibility = [ "*" ] visibility = [ "*" ]
allow_poison = [ "software_video_codecs" ] allow_poison = [ "software_video_codecs" ]
sources = [
"src/jni/software_video_decoder_factory.cc",
"src/jni/software_video_encoder_factory.cc",
]
deps = [ deps = [
":base_jni",
":generated_swcodecs_jni",
":libvpx_vp8_jni", ":libvpx_vp8_jni",
":libvpx_vp9_jni", ":libvpx_vp9_jni",
":native_api_jni",
":video_jni",
"../../api/video_codecs:builtin_video_decoder_factory",
"../../api/video_codecs:builtin_video_encoder_factory",
"../../api/video_codecs:video_codecs_api",
] ]
} }
@ -1353,6 +1364,16 @@ if (current_os == "linux" || is_android) {
jni_generator_include = "//sdk/android/src/jni/jni_generator_helper.h" jni_generator_include = "//sdk/android/src/jni/jni_generator_helper.h"
} }
generate_jni("generated_swcodecs_jni") {
sources = [
"api/org/webrtc/SoftwareVideoDecoderFactory.java",
"api/org/webrtc/SoftwareVideoEncoderFactory.java",
]
namespace = "webrtc::jni"
jni_generator_include = "//sdk/android/src/jni/jni_generator_helper.h"
}
generate_jni("generated_peerconnection_jni") { generate_jni("generated_peerconnection_jni") {
sources = [ sources = [
"api/org/webrtc/AddIceObserver.java", "api/org/webrtc/AddIceObserver.java",
@ -1471,6 +1492,8 @@ if (is_android) {
"instrumentationtests/src/org/webrtc/RtcCertificatePemTest.java", "instrumentationtests/src/org/webrtc/RtcCertificatePemTest.java",
"instrumentationtests/src/org/webrtc/RtpSenderTest.java", "instrumentationtests/src/org/webrtc/RtpSenderTest.java",
"instrumentationtests/src/org/webrtc/RtpTransceiverTest.java", "instrumentationtests/src/org/webrtc/RtpTransceiverTest.java",
"instrumentationtests/src/org/webrtc/SoftwareVideoDecoderFactoryTest.java",
"instrumentationtests/src/org/webrtc/SoftwareVideoEncoderFactoryTest.java",
"instrumentationtests/src/org/webrtc/SurfaceTextureHelperTest.java", "instrumentationtests/src/org/webrtc/SurfaceTextureHelperTest.java",
"instrumentationtests/src/org/webrtc/SurfaceViewRendererOnMeasureTest.java", "instrumentationtests/src/org/webrtc/SurfaceViewRendererOnMeasureTest.java",
"instrumentationtests/src/org/webrtc/TestConstants.java", "instrumentationtests/src/org/webrtc/TestConstants.java",

View File

@ -11,45 +11,43 @@
package org.webrtc; package org.webrtc;
import androidx.annotation.Nullable; import androidx.annotation.Nullable;
import java.util.ArrayList; import java.util.Arrays;
import java.util.HashMap;
import java.util.List; import java.util.List;
public class SoftwareVideoDecoderFactory implements VideoDecoderFactory { public class SoftwareVideoDecoderFactory implements VideoDecoderFactory {
private static final String TAG = "SoftwareVideoDecoderFactory";
private final long nativeFactory;
public SoftwareVideoDecoderFactory() {
this.nativeFactory = nativeCreateFactory();
}
@Nullable @Nullable
@Override @Override
public VideoDecoder createDecoder(VideoCodecInfo codecInfo) { public VideoDecoder createDecoder(VideoCodecInfo info) {
String codecName = codecInfo.getName(); long nativeDecoder = nativeCreateDecoder(nativeFactory, info);
if (nativeDecoder == 0) {
if (codecName.equalsIgnoreCase(VideoCodecMimeType.VP8.name())) { Logging.w(TAG, "Trying to create decoder for unsupported format. " + info);
return new LibvpxVp8Decoder();
}
if (codecName.equalsIgnoreCase(VideoCodecMimeType.VP9.name())
&& LibvpxVp9Decoder.nativeIsSupported()) {
return new LibvpxVp9Decoder();
}
if (codecName.equalsIgnoreCase(VideoCodecMimeType.AV1.name())) {
return new Dav1dDecoder();
}
return null; return null;
} }
return new WrappedNativeVideoDecoder() {
@Override
public long createNativeVideoDecoder() {
return nativeDecoder;
}
};
}
@Override @Override
public VideoCodecInfo[] getSupportedCodecs() { public VideoCodecInfo[] getSupportedCodecs() {
return supportedCodecs(); return nativeGetSupportedCodecs(nativeFactory).toArray(new VideoCodecInfo[0]);
} }
static VideoCodecInfo[] supportedCodecs() { private static native long nativeCreateFactory();
List<VideoCodecInfo> codecs = new ArrayList<VideoCodecInfo>();
codecs.add(new VideoCodecInfo(VideoCodecMimeType.VP8.name(), new HashMap<>())); private static native long nativeCreateDecoder(long factory, VideoCodecInfo videoCodecInfo);
if (LibvpxVp9Decoder.nativeIsSupported()) {
codecs.add(new VideoCodecInfo(VideoCodecMimeType.VP9.name(), new HashMap<>()));
}
codecs.add(new VideoCodecInfo(VideoCodecMimeType.AV1.name(), new HashMap<>())); private static native List<VideoCodecInfo> nativeGetSupportedCodecs(long factory);
return codecs.toArray(new VideoCodecInfo[codecs.size()]);
}
} }

View File

@ -11,40 +11,48 @@
package org.webrtc; package org.webrtc;
import androidx.annotation.Nullable; import androidx.annotation.Nullable;
import java.util.ArrayList; import java.util.Arrays;
import java.util.HashMap;
import java.util.List; import java.util.List;
public class SoftwareVideoEncoderFactory implements VideoEncoderFactory { public class SoftwareVideoEncoderFactory implements VideoEncoderFactory {
private static final String TAG = "SoftwareVideoEncoderFactory";
private final long nativeFactory;
public SoftwareVideoEncoderFactory() {
this.nativeFactory = nativeCreateFactory();
}
@Nullable @Nullable
@Override @Override
public VideoEncoder createEncoder(VideoCodecInfo codecInfo) { public VideoEncoder createEncoder(VideoCodecInfo info) {
String codecName = codecInfo.getName(); long nativeEncoder = nativeCreateEncoder(nativeFactory, info);
if (nativeEncoder == 0) {
if (codecName.equalsIgnoreCase(VideoCodecMimeType.VP8.name())) { Logging.w(TAG, "Trying to create encoder for unsupported format. " + info);
return new LibvpxVp8Encoder();
}
if (codecName.equalsIgnoreCase(VideoCodecMimeType.VP9.name())
&& LibvpxVp9Encoder.nativeIsSupported()) {
return new LibvpxVp9Encoder();
}
return null; return null;
} }
return new WrappedNativeVideoEncoder() {
@Override
public long createNativeVideoEncoder() {
return nativeEncoder;
}
@Override
public boolean isHardwareEncoder() {
return false;
}
};
}
@Override @Override
public VideoCodecInfo[] getSupportedCodecs() { public VideoCodecInfo[] getSupportedCodecs() {
return supportedCodecs(); return nativeGetSupportedCodecs(nativeFactory).toArray(new VideoCodecInfo[0]);
} }
static VideoCodecInfo[] supportedCodecs() { private static native long nativeCreateFactory();
List<VideoCodecInfo> codecs = new ArrayList<VideoCodecInfo>();
codecs.add(new VideoCodecInfo(VideoCodecMimeType.VP8.name(), new HashMap<>())); private static native long nativeCreateEncoder(long factory, VideoCodecInfo videoCodecInfo);
if (LibvpxVp9Encoder.nativeIsSupported()) {
codecs.add(new VideoCodecInfo(VideoCodecMimeType.VP9.name(), new HashMap<>()));
}
return codecs.toArray(new VideoCodecInfo[codecs.size()]); private static native List<VideoCodecInfo> nativeGetSupportedCodecs(long factory);
}
} }

View File

@ -22,27 +22,10 @@ import org.junit.Test;
/** Unit tests for {@link DefaultVideoEncoderFactory}. */ /** Unit tests for {@link DefaultVideoEncoderFactory}. */
public class DefaultVideoEncoderFactoryTest { public class DefaultVideoEncoderFactoryTest {
static class CustomHardwareVideoEncoderFactory implements VideoEncoderFactory { static class CustomHardwareVideoEncoderFactory implements VideoEncoderFactory {
private ArrayList<VideoCodecInfo> codecs = new ArrayList<>(); private VideoCodecInfo supportedCodec;
public CustomHardwareVideoEncoderFactory(boolean includeVP8, boolean includeH264High) { public CustomHardwareVideoEncoderFactory(VideoCodecInfo supportedCodec) {
if (includeVP8) { this.supportedCodec = supportedCodec;
codecs.add(new VideoCodecInfo("VP8", new HashMap<>()));
}
codecs.add(new VideoCodecInfo("VP9", new HashMap<>()));
HashMap<String, String> baselineParams = new HashMap<String, String>();
baselineParams.put("profile-level-id", "42e01f");
baselineParams.put("level-asymmetry-allowed", "1");
baselineParams.put("packetization-mode", "1");
codecs.add(new VideoCodecInfo("H264", baselineParams));
if (includeH264High) {
HashMap<String, String> highParams = new HashMap<String, String>();
highParams.put("profile-level-id", "640c1f");
highParams.put("level-asymmetry-allowed", "1");
highParams.put("packetization-mode", "1");
codecs.add(new VideoCodecInfo("H264", highParams));
}
} }
@Override @Override
@ -52,7 +35,7 @@ public class DefaultVideoEncoderFactoryTest {
@Override @Override
public VideoCodecInfo[] getSupportedCodecs() { public VideoCodecInfo[] getSupportedCodecs() {
return codecs.toArray(new VideoCodecInfo[codecs.size()]); return new VideoCodecInfo[] {supportedCodec};
} }
} }
@ -63,44 +46,32 @@ public class DefaultVideoEncoderFactoryTest {
@SmallTest @SmallTest
@Test @Test
public void testGetSupportedCodecsWithHardwareH264HighProfile() { public void getSupportedCodecs_hwVp8SameParamsAsSwVp8_oneVp8() {
VideoEncoderFactory hwFactory = new CustomHardwareVideoEncoderFactory(true, true); VideoCodecInfo hwVp8Encoder = new VideoCodecInfo("VP8", new HashMap<>());
DefaultVideoEncoderFactory dvef = new DefaultVideoEncoderFactory(hwFactory); VideoEncoderFactory hwFactory = new CustomHardwareVideoEncoderFactory(hwVp8Encoder);
VideoCodecInfo[] videoCodecs = dvef.getSupportedCodecs(); DefaultVideoEncoderFactory defFactory = new DefaultVideoEncoderFactory(hwFactory);
assertEquals(4, videoCodecs.length); VideoCodecInfo[] supportedCodecs = defFactory.getSupportedCodecs();
assertEquals("VP8", videoCodecs[0].name); assertEquals(3, supportedCodecs.length);
assertEquals("VP9", videoCodecs[1].name); assertEquals("VP8", supportedCodecs[0].name);
assertEquals("H264", videoCodecs[2].name); assertEquals("AV1", supportedCodecs[1].name);
assertEquals("42e01f", videoCodecs[2].params.get("profile-level-id")); assertEquals("VP9", supportedCodecs[2].name);
assertEquals("H264", videoCodecs[3].name);
assertEquals("640c1f", videoCodecs[3].params.get("profile-level-id"));
} }
@SmallTest @SmallTest
@Test @Test
public void testGetSupportedCodecsWithoutHardwareH264HighProfile() { public void getSupportedCodecs_hwVp8WithDifferentParams_twoVp8() {
VideoEncoderFactory hwFactory = new CustomHardwareVideoEncoderFactory(true, false); VideoCodecInfo hwVp8Encoder = new VideoCodecInfo("VP8", new HashMap<String, String>() {
DefaultVideoEncoderFactory dvef = new DefaultVideoEncoderFactory(hwFactory); { put("param", "value"); }
VideoCodecInfo[] videoCodecs = dvef.getSupportedCodecs(); });
assertEquals(3, videoCodecs.length); VideoEncoderFactory hwFactory = new CustomHardwareVideoEncoderFactory(hwVp8Encoder);
assertEquals("VP8", videoCodecs[0].name); DefaultVideoEncoderFactory defFactory = new DefaultVideoEncoderFactory(hwFactory);
assertEquals("VP9", videoCodecs[1].name); VideoCodecInfo[] supportedCodecs = defFactory.getSupportedCodecs();
assertEquals("H264", videoCodecs[2].name); assertEquals(4, supportedCodecs.length);
assertEquals("42e01f", videoCodecs[2].params.get("profile-level-id")); assertEquals("VP8", supportedCodecs[0].name);
} assertEquals("AV1", supportedCodecs[1].name);
assertEquals("VP9", supportedCodecs[2].name);
@SmallTest assertEquals("VP8", supportedCodecs[3].name);
@Test assertEquals(1, supportedCodecs[3].params.size());
public void testGetSupportedCodecsWithoutHardwareVP8() { assertEquals("value", supportedCodecs[3].params.get("param"));
VideoEncoderFactory hwFactory = new CustomHardwareVideoEncoderFactory(false, true);
DefaultVideoEncoderFactory dvef = new DefaultVideoEncoderFactory(hwFactory);
VideoCodecInfo[] videoCodecs = dvef.getSupportedCodecs();
assertEquals(4, videoCodecs.length);
assertEquals("VP8", videoCodecs[0].name);
assertEquals("VP9", videoCodecs[1].name);
assertEquals("H264", videoCodecs[2].name);
assertEquals("42e01f", videoCodecs[2].params.get("profile-level-id"));
assertEquals("H264", videoCodecs[3].name);
assertEquals("640c1f", videoCodecs[3].params.get("profile-level-id"));
} }
} }

View File

@ -0,0 +1,62 @@
/*
* Copyright 2022 The WebRTC Project Authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
package org.webrtc;
import static com.google.common.truth.Truth.assertThat;
import androidx.annotation.Nullable;
import androidx.test.filters.SmallTest;
import java.util.HashMap;
import org.junit.Before;
import org.junit.Test;
/** Unit tests for {@link SoftwareVideoDecoderFactory}. */
public class SoftwareVideoDecoderFactoryTest {
@Before
public void setUp() {
NativeLibrary.initialize(new NativeLibrary.DefaultLoader(), TestConstants.NATIVE_LIBRARY);
}
@SmallTest
@Test
public void getSupportedCodecs_returnsDefaultCodecs() {
VideoDecoderFactory factory = new SoftwareVideoDecoderFactory();
VideoCodecInfo[] codecs = factory.getSupportedCodecs();
assertThat(codecs.length).isEqualTo(6);
assertThat(codecs[0].name).isEqualTo("VP8");
assertThat(codecs[1].name).isEqualTo("VP9");
assertThat(codecs[2].name).isEqualTo("VP9");
assertThat(codecs[3].name).isEqualTo("VP9");
assertThat(codecs[4].name).isEqualTo("AV1");
assertThat(codecs[5].name).isEqualTo("AV1");
}
@SmallTest
@Test
public void createDecoder_supportedCodec_returnsNotNull() {
VideoDecoderFactory factory = new SoftwareVideoDecoderFactory();
VideoCodecInfo[] codecs = factory.getSupportedCodecs();
assertThat(codecs.length).isGreaterThan(0);
for (VideoCodecInfo codec : codecs) {
VideoDecoder decoder = factory.createDecoder(codec);
assertThat(decoder).isNotNull();
}
}
@SmallTest
@Test
public void createDecoder_unsupportedCodec_returnsNull() {
VideoDecoderFactory factory = new SoftwareVideoDecoderFactory();
VideoCodecInfo codec = new VideoCodecInfo("unsupported", new HashMap<String, String>());
VideoDecoder decoder = factory.createDecoder(codec);
assertThat(decoder).isNull();
}
}

View File

@ -0,0 +1,59 @@
/*
* Copyright 2022 The WebRTC Project Authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
package org.webrtc;
import static com.google.common.truth.Truth.assertThat;
import androidx.annotation.Nullable;
import androidx.test.filters.SmallTest;
import java.util.HashMap;
import org.junit.Before;
import org.junit.Test;
/** Unit tests for {@link SoftwareVideoEncoderFactory}. */
public class SoftwareVideoEncoderFactoryTest {
@Before
public void setUp() {
NativeLibrary.initialize(new NativeLibrary.DefaultLoader(), TestConstants.NATIVE_LIBRARY);
}
@SmallTest
@Test
public void getSupportedCodecs_returnsDefaultCodecs() {
VideoEncoderFactory factory = new SoftwareVideoEncoderFactory();
VideoCodecInfo[] codecs = factory.getSupportedCodecs();
assertThat(codecs.length).isEqualTo(3);
assertThat(codecs[0].name).isEqualTo("VP8");
assertThat(codecs[1].name).isEqualTo("AV1");
assertThat(codecs[2].name).isEqualTo("VP9");
}
@SmallTest
@Test
public void createEncoder_supportedCodec_returnsNotNull() {
VideoEncoderFactory factory = new SoftwareVideoEncoderFactory();
VideoCodecInfo[] codecs = factory.getSupportedCodecs();
assertThat(codecs.length).isGreaterThan(0);
for (VideoCodecInfo codec : codecs) {
VideoEncoder encoder = factory.createEncoder(codec);
assertThat(encoder).isNotNull();
}
}
@SmallTest
@Test
public void createEncoder_unsupportedCodec_returnsNull() {
VideoEncoderFactory factory = new SoftwareVideoEncoderFactory();
VideoCodecInfo codec = new VideoCodecInfo("unsupported", new HashMap<String, String>());
VideoEncoder encoder = factory.createEncoder(codec);
assertThat(encoder).isNull();
}
}

View File

@ -0,0 +1,53 @@
/*
* Copyright 2022 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "api/video_codecs/builtin_video_decoder_factory.h"
#include "api/video_codecs/video_decoder.h"
#include "sdk/android/generated_swcodecs_jni/SoftwareVideoDecoderFactory_jni.h"
#include "sdk/android/native_api/jni/java_types.h"
#include "sdk/android/src/jni/jni_helpers.h"
#include "sdk/android/src/jni/video_codec_info.h"
namespace webrtc {
namespace jni {
static jlong JNI_SoftwareVideoDecoderFactory_CreateFactory(JNIEnv* env) {
return webrtc::NativeToJavaPointer(
CreateBuiltinVideoDecoderFactory().release());
}
static jlong JNI_SoftwareVideoDecoderFactory_CreateDecoder(
JNIEnv* env,
jlong j_factory,
const webrtc::JavaParamRef<jobject>& j_video_codec_info) {
auto* const native_factory =
reinterpret_cast<webrtc::VideoDecoderFactory*>(j_factory);
const auto video_format =
webrtc::jni::VideoCodecInfoToSdpVideoFormat(env, j_video_codec_info);
auto decoder = native_factory->CreateVideoDecoder(video_format);
if (decoder == nullptr) {
return 0;
}
return webrtc::NativeToJavaPointer(decoder.release());
}
static webrtc::ScopedJavaLocalRef<jobject>
JNI_SoftwareVideoDecoderFactory_GetSupportedCodecs(JNIEnv* env,
jlong j_factory) {
auto* const native_factory =
reinterpret_cast<webrtc::VideoDecoderFactory*>(j_factory);
return webrtc::NativeToJavaList(env, native_factory->GetSupportedFormats(),
&webrtc::jni::SdpVideoFormatToVideoCodecInfo);
}
} // namespace jni
} // namespace webrtc

View File

@ -0,0 +1,53 @@
/*
* Copyright 2022 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "api/video_codecs/builtin_video_encoder_factory.h"
#include "api/video_codecs/video_encoder.h"
#include "sdk/android/generated_swcodecs_jni/SoftwareVideoEncoderFactory_jni.h"
#include "sdk/android/native_api/jni/java_types.h"
#include "sdk/android/src/jni/jni_helpers.h"
#include "sdk/android/src/jni/video_codec_info.h"
namespace webrtc {
namespace jni {
static jlong JNI_SoftwareVideoEncoderFactory_CreateFactory(JNIEnv* env) {
return webrtc::NativeToJavaPointer(
CreateBuiltinVideoEncoderFactory().release());
}
static jlong JNI_SoftwareVideoEncoderFactory_CreateEncoder(
JNIEnv* env,
jlong j_factory,
const webrtc::JavaParamRef<jobject>& j_video_codec_info) {
auto* const native_factory =
reinterpret_cast<webrtc::VideoEncoderFactory*>(j_factory);
const auto video_format =
webrtc::jni::VideoCodecInfoToSdpVideoFormat(env, j_video_codec_info);
auto encoder = native_factory->CreateVideoEncoder(video_format);
if (encoder == nullptr) {
return 0;
}
return webrtc::NativeToJavaPointer(encoder.release());
}
static webrtc::ScopedJavaLocalRef<jobject>
JNI_SoftwareVideoEncoderFactory_GetSupportedCodecs(JNIEnv* env,
jlong j_factory) {
auto* const native_factory =
reinterpret_cast<webrtc::VideoEncoderFactory*>(j_factory);
return webrtc::NativeToJavaList(env, native_factory->GetSupportedFormats(),
&webrtc::jni::SdpVideoFormatToVideoCodecInfo);
}
} // namespace jni
} // namespace webrtc