Call native codec factories from Android ones.

Android video codec factories are expected to be synchronised with the native ones in terms on supported codecs. But before this change there were differences:

1. Native decoder factory keeps AV1 support behind RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY while Android decoder factory advertises AV1 unconditionally;

2. Native encoder factory advertises AV1 if RTC_USE_LIBAOM_AV1_ENCODER is enabled while Android encoder factory never advertises AV1.

This CL synchronises the codecs set in Android factories with that of native factories by calling native factories from Android ones.

Bug: webrtc:13573, b/257272020
Change-Id: I99d801eda0c5f3400bac222b9b08d719f1a6ed72
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/282240
Reviewed-by: Rasmus Brandt <brandtr@webrtc.org>
Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
Reviewed-by: Xavier Lepaul‎ <xalep@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#38583}

sdk/android/BUILD.gn

@@ -885,9 +885,20 @@ if (current_os == "linux" || is_android) {
   rtc_library("swcodecs_jni") {
     visibility = [ "*" ]
     allow_poison = [ "software_video_codecs" ]
+    sources = [
+      "src/jni/software_video_decoder_factory.cc",
+      "src/jni/software_video_encoder_factory.cc",
+    ]
     deps = [
+      ":base_jni",
+      ":generated_swcodecs_jni",
       ":libvpx_vp8_jni",
       ":libvpx_vp9_jni",
+      ":native_api_jni",
+      ":video_jni",
+      "../../api/video_codecs:builtin_video_decoder_factory",
+      "../../api/video_codecs:builtin_video_encoder_factory",
+      "../../api/video_codecs:video_codecs_api",
     ]
   }
 
@@ -1353,6 +1364,16 @@ if (current_os == "linux" || is_android) {
     jni_generator_include = "//sdk/android/src/jni/jni_generator_helper.h"
   }
 
+  generate_jni("generated_swcodecs_jni") {
+    sources = [
+      "api/org/webrtc/SoftwareVideoDecoderFactory.java",
+      "api/org/webrtc/SoftwareVideoEncoderFactory.java",
+    ]
+
+    namespace = "webrtc::jni"
+    jni_generator_include = "//sdk/android/src/jni/jni_generator_helper.h"
+  }
+
   generate_jni("generated_peerconnection_jni") {
     sources = [
       "api/org/webrtc/AddIceObserver.java",
@@ -1471,6 +1492,8 @@ if (is_android) {
         "instrumentationtests/src/org/webrtc/RtcCertificatePemTest.java",
         "instrumentationtests/src/org/webrtc/RtpSenderTest.java",
         "instrumentationtests/src/org/webrtc/RtpTransceiverTest.java",
+        "instrumentationtests/src/org/webrtc/SoftwareVideoDecoderFactoryTest.java",
+        "instrumentationtests/src/org/webrtc/SoftwareVideoEncoderFactoryTest.java",
         "instrumentationtests/src/org/webrtc/SurfaceTextureHelperTest.java",
         "instrumentationtests/src/org/webrtc/SurfaceViewRendererOnMeasureTest.java",
         "instrumentationtests/src/org/webrtc/TestConstants.java",

sdk/android/api/org/webrtc/SoftwareVideoDecoderFactory.java

@@ -16,40 +16,33 @@ import java.util.HashMap;
 import java.util.List;
 
 public class SoftwareVideoDecoderFactory implements VideoDecoderFactory {
+  private static final String TAG = "SoftwareVideoDecoderFactory";
+
+  private final long nativeFactory;
+
+  public SoftwareVideoDecoderFactory() {
+    this.nativeFactory = nativeCreateFactory();
+  }
+
   @Nullable
   @Override
-  public VideoDecoder createDecoder(VideoCodecInfo codecInfo) {
-    String codecName = codecInfo.getName();
-
-    if (codecName.equalsIgnoreCase(VideoCodecMimeType.VP8.name())) {
-      return new LibvpxVp8Decoder();
+  public VideoDecoder createDecoder(VideoCodecInfo info) {
+    return new WrappedNativeVideoDecoder() {
+      @Override
+      public long createNativeVideoDecoder() {
+        return nativeCreateDecoder(nativeFactory, info);
       }
-    if (codecName.equalsIgnoreCase(VideoCodecMimeType.VP9.name())
-        && LibvpxVp9Decoder.nativeIsSupported()) {
-      return new LibvpxVp9Decoder();
-    }
-    if (codecName.equalsIgnoreCase(VideoCodecMimeType.AV1.name())) {
-      return new Dav1dDecoder();
-    }
-
-    return null;
+    };
   }
 
   @Override
   public VideoCodecInfo[] getSupportedCodecs() {
-    return supportedCodecs();
+    return nativeGetSupportedCodecs(nativeFactory).toArray(new VideoCodecInfo[0]);
   }
 
-  static VideoCodecInfo[] supportedCodecs() {
-    List<VideoCodecInfo> codecs = new ArrayList<VideoCodecInfo>();
+  private static native long nativeCreateFactory();
 
-    codecs.add(new VideoCodecInfo(VideoCodecMimeType.VP8.name(), new HashMap<>()));
-    if (LibvpxVp9Decoder.nativeIsSupported()) {
-      codecs.add(new VideoCodecInfo(VideoCodecMimeType.VP9.name(), new HashMap<>()));
-    }
+  private static native long nativeCreateDecoder(long factory, VideoCodecInfo videoCodecInfo);
 
-    codecs.add(new VideoCodecInfo(VideoCodecMimeType.AV1.name(), new HashMap<>()));
-
-    return codecs.toArray(new VideoCodecInfo[codecs.size()]);
-  }
+  private static native List<VideoCodecInfo> nativeGetSupportedCodecs(long factory);
 }

sdk/android/api/org/webrtc/SoftwareVideoEncoderFactory.java

@@ -16,35 +16,38 @@ import java.util.HashMap;
 import java.util.List;
 
 public class SoftwareVideoEncoderFactory implements VideoEncoderFactory {
+  private static final String TAG = "SoftwareVideoEncoderFactory";
+
+  private final long nativeFactory;
+
+  public SoftwareVideoEncoderFactory() {
+    this.nativeFactory = nativeCreateFactory();
+  }
+
   @Nullable
   @Override
-  public VideoEncoder createEncoder(VideoCodecInfo codecInfo) {
-    String codecName = codecInfo.getName();
-
-    if (codecName.equalsIgnoreCase(VideoCodecMimeType.VP8.name())) {
-      return new LibvpxVp8Encoder();
-    }
-    if (codecName.equalsIgnoreCase(VideoCodecMimeType.VP9.name())
-        && LibvpxVp9Encoder.nativeIsSupported()) {
-      return new LibvpxVp9Encoder();
+  public VideoEncoder createEncoder(VideoCodecInfo info) {
+    return new WrappedNativeVideoEncoder() {
+      @Override
+      public long createNativeVideoEncoder() {
+        return nativeCreateEncoder(nativeFactory, info);
       }
 
-    return null;
+      @Override
+      public boolean isHardwareEncoder() {
+        return false;
+      }
+    };
   }
 
   @Override
   public VideoCodecInfo[] getSupportedCodecs() {
-    return supportedCodecs();
+    return nativeGetSupportedCodecs(nativeFactory).toArray(new VideoCodecInfo[0]);
   }
 
-  static VideoCodecInfo[] supportedCodecs() {
-    List<VideoCodecInfo> codecs = new ArrayList<VideoCodecInfo>();
+  private static native long nativeCreateFactory();
 
-    codecs.add(new VideoCodecInfo(VideoCodecMimeType.VP8.name(), new HashMap<>()));
-    if (LibvpxVp9Encoder.nativeIsSupported()) {
-      codecs.add(new VideoCodecInfo(VideoCodecMimeType.VP9.name(), new HashMap<>()));
-    }
+  private static native long nativeCreateEncoder(long factory, VideoCodecInfo videoCodecInfo);
 
-    return codecs.toArray(new VideoCodecInfo[codecs.size()]);
-  }
+  private static native List<VideoCodecInfo> nativeGetSupportedCodecs(long factory);
 }

sdk/android/instrumentationtests/src/org/webrtc/DefaultVideoEncoderFactoryTest.java

@@ -22,27 +22,10 @@ import org.junit.Test;
 /** Unit tests for {@link DefaultVideoEncoderFactory}. */
 public class DefaultVideoEncoderFactoryTest {
   static class CustomHardwareVideoEncoderFactory implements VideoEncoderFactory {
-    private ArrayList<VideoCodecInfo> codecs = new ArrayList<>();
+    private VideoCodecInfo supportedCodec;
 
-    public CustomHardwareVideoEncoderFactory(boolean includeVP8, boolean includeH264High) {
-      if (includeVP8) {
-        codecs.add(new VideoCodecInfo("VP8", new HashMap<>()));
-      }
-      codecs.add(new VideoCodecInfo("VP9", new HashMap<>()));
-
-      HashMap<String, String> baselineParams = new HashMap<String, String>();
-      baselineParams.put("profile-level-id", "42e01f");
-      baselineParams.put("level-asymmetry-allowed", "1");
-      baselineParams.put("packetization-mode", "1");
-      codecs.add(new VideoCodecInfo("H264", baselineParams));
-
-      if (includeH264High) {
-        HashMap<String, String> highParams = new HashMap<String, String>();
-        highParams.put("profile-level-id", "640c1f");
-        highParams.put("level-asymmetry-allowed", "1");
-        highParams.put("packetization-mode", "1");
-        codecs.add(new VideoCodecInfo("H264", highParams));
-      }
+    public CustomHardwareVideoEncoderFactory(VideoCodecInfo supportedCodec) {
+      this.supportedCodec = supportedCodec;
     }
 
     @Override
@@ -52,7 +35,7 @@ public class DefaultVideoEncoderFactoryTest {
 
     @Override
     public VideoCodecInfo[] getSupportedCodecs() {
-      return codecs.toArray(new VideoCodecInfo[codecs.size()]);
+      return new VideoCodecInfo[] {supportedCodec};
     }
   }
 
@@ -63,44 +46,32 @@ public class DefaultVideoEncoderFactoryTest {
 
   @SmallTest
   @Test
-  public void testGetSupportedCodecsWithHardwareH264HighProfile() {
-    VideoEncoderFactory hwFactory = new CustomHardwareVideoEncoderFactory(true, true);
-    DefaultVideoEncoderFactory dvef = new DefaultVideoEncoderFactory(hwFactory);
-    VideoCodecInfo[] videoCodecs = dvef.getSupportedCodecs();
-    assertEquals(4, videoCodecs.length);
-    assertEquals("VP8", videoCodecs[0].name);
-    assertEquals("VP9", videoCodecs[1].name);
-    assertEquals("H264", videoCodecs[2].name);
-    assertEquals("42e01f", videoCodecs[2].params.get("profile-level-id"));
-    assertEquals("H264", videoCodecs[3].name);
-    assertEquals("640c1f", videoCodecs[3].params.get("profile-level-id"));
+  public void getSupportedCodecs_hwVp8SameParamsAsSwVp8_oneVp8() {
+    VideoCodecInfo hwVp8Encoder = new VideoCodecInfo("VP8", new HashMap<>());
+    VideoEncoderFactory hwFactory = new CustomHardwareVideoEncoderFactory(hwVp8Encoder);
+    DefaultVideoEncoderFactory defFactory = new DefaultVideoEncoderFactory(hwFactory);
+    VideoCodecInfo[] supportedCodecs = defFactory.getSupportedCodecs();
+    assertEquals(3, supportedCodecs.length);
+    assertEquals("VP8", supportedCodecs[0].name);
+    assertEquals("AV1", supportedCodecs[1].name);
+    assertEquals("VP9", supportedCodecs[2].name);
   }
 
   @SmallTest
   @Test
-  public void testGetSupportedCodecsWithoutHardwareH264HighProfile() {
-    VideoEncoderFactory hwFactory = new CustomHardwareVideoEncoderFactory(true, false);
-    DefaultVideoEncoderFactory dvef = new DefaultVideoEncoderFactory(hwFactory);
-    VideoCodecInfo[] videoCodecs = dvef.getSupportedCodecs();
-    assertEquals(3, videoCodecs.length);
-    assertEquals("VP8", videoCodecs[0].name);
-    assertEquals("VP9", videoCodecs[1].name);
-    assertEquals("H264", videoCodecs[2].name);
-    assertEquals("42e01f", videoCodecs[2].params.get("profile-level-id"));
-  }
-
-  @SmallTest
-  @Test
-  public void testGetSupportedCodecsWithoutHardwareVP8() {
-    VideoEncoderFactory hwFactory = new CustomHardwareVideoEncoderFactory(false, true);
-    DefaultVideoEncoderFactory dvef = new DefaultVideoEncoderFactory(hwFactory);
-    VideoCodecInfo[] videoCodecs = dvef.getSupportedCodecs();
-    assertEquals(4, videoCodecs.length);
-    assertEquals("VP8", videoCodecs[0].name);
-    assertEquals("VP9", videoCodecs[1].name);
-    assertEquals("H264", videoCodecs[2].name);
-    assertEquals("42e01f", videoCodecs[2].params.get("profile-level-id"));
-    assertEquals("H264", videoCodecs[3].name);
-    assertEquals("640c1f", videoCodecs[3].params.get("profile-level-id"));
+  public void getSupportedCodecs_hwVp8WithDifferentParams_twoVp8() {
+    VideoCodecInfo hwVp8Encoder = new VideoCodecInfo("VP8", new HashMap<String, String>() {
+      { put("param", "value"); }
+    });
+    VideoEncoderFactory hwFactory = new CustomHardwareVideoEncoderFactory(hwVp8Encoder);
+    DefaultVideoEncoderFactory defFactory = new DefaultVideoEncoderFactory(hwFactory);
+    VideoCodecInfo[] supportedCodecs = defFactory.getSupportedCodecs();
+    assertEquals(4, supportedCodecs.length);
+    assertEquals("VP8", supportedCodecs[0].name);
+    assertEquals("AV1", supportedCodecs[1].name);
+    assertEquals("VP9", supportedCodecs[2].name);
+    assertEquals("VP8", supportedCodecs[3].name);
+    assertEquals(1, supportedCodecs[3].params.size());
+    assertEquals("value", supportedCodecs[3].params.get("param"));
   }
 }

sdk/android/instrumentationtests/src/org/webrtc/SoftwareVideoDecoderFactoryTest.java

@@ -0,0 +1,42 @@
+/*
+ *  Copyright 2022 The WebRTC Project Authors. All rights reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+package org.webrtc;
+
+import static org.junit.Assert.assertEquals;
+
+import androidx.annotation.Nullable;
+import androidx.test.filters.SmallTest;
+import java.util.ArrayList;
+import java.util.HashMap;
+import org.junit.Before;
+import org.junit.Test;
+
+/** Unit tests for {@link SoftwareVideoDecoderFactory}. */
+public class SoftwareVideoDecoderFactoryTest {
+  @Before
+  public void setUp() {
+    NativeLibrary.initialize(new NativeLibrary.DefaultLoader(), TestConstants.NATIVE_LIBRARY);
+  }
+
+  @SmallTest
+  @Test
+  public void getSupportedCodecs_returnsDefaultCodecs() {
+    VideoDecoderFactory factory = new SoftwareVideoDecoderFactory();
+    VideoCodecInfo[] codecs = factory.getSupportedCodecs();
+    assertEquals(6, codecs.length);
+    assertEquals("VP8", codecs[0].name);
+    assertEquals("VP9", codecs[1].name);
+    assertEquals("VP9", codecs[2].name);
+    assertEquals("VP9", codecs[3].name);
+    assertEquals("AV1", codecs[4].name);
+    assertEquals("AV1", codecs[5].name);
+  }
+}

sdk/android/instrumentationtests/src/org/webrtc/SoftwareVideoEncoderFactoryTest.java

@@ -0,0 +1,39 @@
+/*
+ *  Copyright 2022 The WebRTC Project Authors. All rights reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+package org.webrtc;
+
+import static org.junit.Assert.assertEquals;
+
+import androidx.annotation.Nullable;
+import androidx.test.filters.SmallTest;
+import java.util.ArrayList;
+import java.util.HashMap;
+import org.junit.Before;
+import org.junit.Test;
+
+/** Unit tests for {@link SoftwareVideoEncoderFactory}. */
+public class SoftwareVideoEncoderFactoryTest {
+  @Before
+  public void setUp() {
+    NativeLibrary.initialize(new NativeLibrary.DefaultLoader(), TestConstants.NATIVE_LIBRARY);
+  }
+
+  @SmallTest
+  @Test
+  public void getSupportedCodecs_returnsDefaultCodecs() {
+    VideoEncoderFactory factory = new SoftwareVideoEncoderFactory();
+    VideoCodecInfo[] codecs = factory.getSupportedCodecs();
+    assertEquals(3, codecs.length);
+    assertEquals("VP8", codecs[0].name);
+    assertEquals("AV1", codecs[1].name);
+    assertEquals("VP9", codecs[2].name);
+  }
+}

sdk/android/src/jni/software_video_decoder_factory.cc

@@ -0,0 +1,50 @@
+/*
+ *  Copyright 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "api/video_codecs/builtin_video_decoder_factory.h"
+#include "api/video_codecs/video_decoder.h"
+#include "sdk/android/generated_swcodecs_jni/SoftwareVideoDecoderFactory_jni.h"
+#include "sdk/android/native_api/jni/java_types.h"
+#include "sdk/android/src/jni/jni_helpers.h"
+#include "sdk/android/src/jni/video_codec_info.h"
+
+namespace webrtc {
+namespace jni {
+
+static jlong JNI_SoftwareVideoDecoderFactory_CreateFactory(JNIEnv* env) {
+  return webrtc::NativeToJavaPointer(
+      CreateBuiltinVideoDecoderFactory().release());
+}
+
+static jlong JNI_SoftwareVideoDecoderFactory_CreateDecoder(
+    JNIEnv* env,
+    jlong j_factory,
+    const webrtc::JavaParamRef<jobject>& j_video_codec_info) {
+  auto* const native_factory =
+      reinterpret_cast<webrtc::VideoDecoderFactory*>(j_factory);
+  const auto video_format =
+      webrtc::jni::VideoCodecInfoToSdpVideoFormat(env, j_video_codec_info);
+
+  return webrtc::NativeToJavaPointer(
+      native_factory->CreateVideoDecoder(video_format).release());
+}
+
+static webrtc::ScopedJavaLocalRef<jobject>
+JNI_SoftwareVideoDecoderFactory_GetSupportedCodecs(JNIEnv* env,
+                                                   jlong j_factory) {
+  auto* const native_factory =
+      reinterpret_cast<webrtc::VideoDecoderFactory*>(j_factory);
+
+  return webrtc::NativeToJavaList(env, native_factory->GetSupportedFormats(),
+                                  &webrtc::jni::SdpVideoFormatToVideoCodecInfo);
+}
+
+}  // namespace jni
+}  // namespace webrtc

sdk/android/src/jni/software_video_encoder_factory.cc

@@ -0,0 +1,50 @@
+/*
+ *  Copyright 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "api/video_codecs/builtin_video_encoder_factory.h"
+#include "api/video_codecs/video_encoder.h"
+#include "sdk/android/generated_swcodecs_jni/SoftwareVideoEncoderFactory_jni.h"
+#include "sdk/android/native_api/jni/java_types.h"
+#include "sdk/android/src/jni/jni_helpers.h"
+#include "sdk/android/src/jni/video_codec_info.h"
+
+namespace webrtc {
+namespace jni {
+
+static jlong JNI_SoftwareVideoEncoderFactory_CreateFactory(JNIEnv* env) {
+  return webrtc::NativeToJavaPointer(
+      CreateBuiltinVideoEncoderFactory().release());
+}
+
+static jlong JNI_SoftwareVideoEncoderFactory_CreateEncoder(
+    JNIEnv* env,
+    jlong j_factory,
+    const webrtc::JavaParamRef<jobject>& j_video_codec_info) {
+  auto* const native_factory =
+      reinterpret_cast<webrtc::VideoEncoderFactory*>(j_factory);
+  const auto video_format =
+      webrtc::jni::VideoCodecInfoToSdpVideoFormat(env, j_video_codec_info);
+
+  return webrtc::NativeToJavaPointer(
+      native_factory->CreateVideoEncoder(video_format).release());
+}
+
+static webrtc::ScopedJavaLocalRef<jobject>
+JNI_SoftwareVideoEncoderFactory_GetSupportedCodecs(JNIEnv* env,
+                                                   jlong j_factory) {
+  auto* const native_factory =
+      reinterpret_cast<webrtc::VideoEncoderFactory*>(j_factory);
+
+  return webrtc::NativeToJavaList(env, native_factory->GetSupportedFormats(),
+                                  &webrtc::jni::SdpVideoFormatToVideoCodecInfo);
+}
+
+}  // namespace jni
+}  // namespace webrtc