Call native codec factories from Android ones.

Android video codec factories are expected to be synchronised with the native ones in terms on supported codecs. But before this change there were differences:

1. Native decoder factory keeps AV1 support behind RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY while Android decoder factory advertises AV1 unconditionally;

2. Native encoder factory advertises AV1 if RTC_USE_LIBAOM_AV1_ENCODER is enabled while Android encoder factory never advertises AV1.

This CL synchronises the codecs set in Android factories with that of native factories by calling native factories from Android ones.

Bug: webrtc:13573, b/257272020
Change-Id: I99d801eda0c5f3400bac222b9b08d719f1a6ed72
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/282240
Reviewed-by: Rasmus Brandt <brandtr@webrtc.org>
Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
Reviewed-by: Xavier Lepaul‎ <xalep@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#38583}

sdk/android/BUILD.gn

@@ -885,9 +885,20 @@ if (current_os == "linux" || is_android) {
   rtc_library("swcodecs_jni") {
     visibility = [ "*" ]
     allow_poison = [ "software_video_codecs" ]
+    sources = [
+      "src/jni/software_video_decoder_factory.cc",
+      "src/jni/software_video_encoder_factory.cc",
+    ]
     deps = [
+      ":base_jni",
+      ":generated_swcodecs_jni",
       ":libvpx_vp8_jni",
       ":libvpx_vp9_jni",
+      ":native_api_jni",
+      ":video_jni",
+      "../../api/video_codecs:builtin_video_decoder_factory",
+      "../../api/video_codecs:builtin_video_encoder_factory",
+      "../../api/video_codecs:video_codecs_api",
     ]
   }
 
@@ -1353,6 +1364,16 @@ if (current_os == "linux" || is_android) {
     jni_generator_include = "//sdk/android/src/jni/jni_generator_helper.h"
   }
 
+  generate_jni("generated_swcodecs_jni") {
+    sources = [
+      "api/org/webrtc/SoftwareVideoDecoderFactory.java",
+      "api/org/webrtc/SoftwareVideoEncoderFactory.java",
+    ]
+
+    namespace = "webrtc::jni"
+    jni_generator_include = "//sdk/android/src/jni/jni_generator_helper.h"
+  }
+
   generate_jni("generated_peerconnection_jni") {
     sources = [
       "api/org/webrtc/AddIceObserver.java",
@@ -1471,6 +1492,8 @@ if (is_android) {
         "instrumentationtests/src/org/webrtc/RtcCertificatePemTest.java",
         "instrumentationtests/src/org/webrtc/RtpSenderTest.java",
         "instrumentationtests/src/org/webrtc/RtpTransceiverTest.java",
+        "instrumentationtests/src/org/webrtc/SoftwareVideoDecoderFactoryTest.java",
+        "instrumentationtests/src/org/webrtc/SoftwareVideoEncoderFactoryTest.java",
         "instrumentationtests/src/org/webrtc/SurfaceTextureHelperTest.java",
         "instrumentationtests/src/org/webrtc/SurfaceViewRendererOnMeasureTest.java",
         "instrumentationtests/src/org/webrtc/TestConstants.java",

sdk/android/api/org/webrtc/SoftwareVideoDecoderFactory.java

@@ -16,40 +16,33 @@ import java.util.HashMap;
 import java.util.List;
 
 public class SoftwareVideoDecoderFactory implements VideoDecoderFactory {
+  private static final String TAG = "SoftwareVideoDecoderFactory";
+
+  private final long nativeFactory;
+
+  public SoftwareVideoDecoderFactory() {
+    this.nativeFactory = nativeCreateFactory();
+  }
+
   @Nullable
   @Override
-  public VideoDecoder createDecoder(VideoCodecInfo codecInfo) {
+  public VideoDecoder createDecoder(VideoCodecInfo info) {
-    String codecName = codecInfo.getName();
+    return new WrappedNativeVideoDecoder() {
-
+      @Override
-    if (codecName.equalsIgnoreCase(VideoCodecMimeType.VP8.name())) {
+      public long createNativeVideoDecoder() {
-      return new LibvpxVp8Decoder();
+        return nativeCreateDecoder(nativeFactory, info);
-    }
+      }
-    if (codecName.equalsIgnoreCase(VideoCodecMimeType.VP9.name())
+    };
-        && LibvpxVp9Decoder.nativeIsSupported()) {
-      return new LibvpxVp9Decoder();
-    }
-    if (codecName.equalsIgnoreCase(VideoCodecMimeType.AV1.name())) {
-      return new Dav1dDecoder();
-    }
-
-    return null;
   }
 
   @Override
   public VideoCodecInfo[] getSupportedCodecs() {
-    return supportedCodecs();
+    return nativeGetSupportedCodecs(nativeFactory).toArray(new VideoCodecInfo[0]);
   }
 
-  static VideoCodecInfo[] supportedCodecs() {
+  private static native long nativeCreateFactory();
-    List<VideoCodecInfo> codecs = new ArrayList<VideoCodecInfo>();
 
-    codecs.add(new VideoCodecInfo(VideoCodecMimeType.VP8.name(), new HashMap<>()));
+  private static native long nativeCreateDecoder(long factory, VideoCodecInfo videoCodecInfo);
-    if (LibvpxVp9Decoder.nativeIsSupported()) {
-      codecs.add(new VideoCodecInfo(VideoCodecMimeType.VP9.name(), new HashMap<>()));
-    }
 
-    codecs.add(new VideoCodecInfo(VideoCodecMimeType.AV1.name(), new HashMap<>()));
+  private static native List<VideoCodecInfo> nativeGetSupportedCodecs(long factory);
-
-    return codecs.toArray(new VideoCodecInfo[codecs.size()]);
-  }
 }

sdk/android/api/org/webrtc/SoftwareVideoEncoderFactory.java

@@ -16,35 +16,38 @@ import java.util.HashMap;
 import java.util.List;
 
 public class SoftwareVideoEncoderFactory implements VideoEncoderFactory {
+  private static final String TAG = "SoftwareVideoEncoderFactory";
+
+  private final long nativeFactory;
+
+  public SoftwareVideoEncoderFactory() {
+    this.nativeFactory = nativeCreateFactory();
+  }
+
   @Nullable
   @Override
-  public VideoEncoder createEncoder(VideoCodecInfo codecInfo) {
+  public VideoEncoder createEncoder(VideoCodecInfo info) {
-    String codecName = codecInfo.getName();
+    return new WrappedNativeVideoEncoder() {
+      @Override
+      public long createNativeVideoEncoder() {
+        return nativeCreateEncoder(nativeFactory, info);
+      }
 
-    if (codecName.equalsIgnoreCase(VideoCodecMimeType.VP8.name())) {
+      @Override
-      return new LibvpxVp8Encoder();
+      public boolean isHardwareEncoder() {
-    }
+        return false;
-    if (codecName.equalsIgnoreCase(VideoCodecMimeType.VP9.name())
+      }
-        && LibvpxVp9Encoder.nativeIsSupported()) {
+    };
-      return new LibvpxVp9Encoder();
-    }
-
-    return null;
   }
 
   @Override
   public VideoCodecInfo[] getSupportedCodecs() {
-    return supportedCodecs();
+    return nativeGetSupportedCodecs(nativeFactory).toArray(new VideoCodecInfo[0]);
   }
 
-  static VideoCodecInfo[] supportedCodecs() {
+  private static native long nativeCreateFactory();
-    List<VideoCodecInfo> codecs = new ArrayList<VideoCodecInfo>();
 
-    codecs.add(new VideoCodecInfo(VideoCodecMimeType.VP8.name(), new HashMap<>()));
+  private static native long nativeCreateEncoder(long factory, VideoCodecInfo videoCodecInfo);
-    if (LibvpxVp9Encoder.nativeIsSupported()) {
-      codecs.add(new VideoCodecInfo(VideoCodecMimeType.VP9.name(), new HashMap<>()));
-    }
 
-    return codecs.toArray(new VideoCodecInfo[codecs.size()]);
+  private static native List<VideoCodecInfo> nativeGetSupportedCodecs(long factory);
-  }
 }

sdk/android/instrumentationtests/src/org/webrtc/DefaultVideoEncoderFactoryTest.java

@@ -22,27 +22,10 @@ import org.junit.Test;
 /** Unit tests for {@link DefaultVideoEncoderFactory}. */
 public class DefaultVideoEncoderFactoryTest {
   static class CustomHardwareVideoEncoderFactory implements VideoEncoderFactory {
-    private ArrayList<VideoCodecInfo> codecs = new ArrayList<>();
+    private VideoCodecInfo supportedCodec;
 
-    public CustomHardwareVideoEncoderFactory(boolean includeVP8, boolean includeH264High) {
+    public CustomHardwareVideoEncoderFactory(VideoCodecInfo supportedCodec) {
-      if (includeVP8) {
+      this.supportedCodec = supportedCodec;
-        codecs.add(new VideoCodecInfo("VP8", new HashMap<>()));
-      }
-      codecs.add(new VideoCodecInfo("VP9", new HashMap<>()));
-
-      HashMap<String, String> baselineParams = new HashMap<String, String>();
-      baselineParams.put("profile-level-id", "42e01f");
-      baselineParams.put("level-asymmetry-allowed", "1");
-      baselineParams.put("packetization-mode", "1");
-      codecs.add(new VideoCodecInfo("H264", baselineParams));
-
-      if (includeH264High) {
-        HashMap<String, String> highParams = new HashMap<String, String>();
-        highParams.put("profile-level-id", "640c1f");
-        highParams.put("level-asymmetry-allowed", "1");
-        highParams.put("packetization-mode", "1");
-        codecs.add(new VideoCodecInfo("H264", highParams));
-      }
     }
 
     @Override
@@ -52,7 +35,7 @@ public class DefaultVideoEncoderFactoryTest {
 
     @Override
     public VideoCodecInfo[] getSupportedCodecs() {
-      return codecs.toArray(new VideoCodecInfo[codecs.size()]);
+      return new VideoCodecInfo[] {supportedCodec};
     }
   }
 
@@ -63,44 +46,32 @@ public class DefaultVideoEncoderFactoryTest {
 
   @SmallTest
   @Test
-  public void testGetSupportedCodecsWithHardwareH264HighProfile() {
+  public void getSupportedCodecs_hwVp8SameParamsAsSwVp8_oneVp8() {
-    VideoEncoderFactory hwFactory = new CustomHardwareVideoEncoderFactory(true, true);
+    VideoCodecInfo hwVp8Encoder = new VideoCodecInfo("VP8", new HashMap<>());
-    DefaultVideoEncoderFactory dvef = new DefaultVideoEncoderFactory(hwFactory);
+    VideoEncoderFactory hwFactory = new CustomHardwareVideoEncoderFactory(hwVp8Encoder);
-    VideoCodecInfo[] videoCodecs = dvef.getSupportedCodecs();
+    DefaultVideoEncoderFactory defFactory = new DefaultVideoEncoderFactory(hwFactory);
-    assertEquals(4, videoCodecs.length);
+    VideoCodecInfo[] supportedCodecs = defFactory.getSupportedCodecs();
-    assertEquals("VP8", videoCodecs[0].name);
+    assertEquals(3, supportedCodecs.length);
-    assertEquals("VP9", videoCodecs[1].name);
+    assertEquals("VP8", supportedCodecs[0].name);
-    assertEquals("H264", videoCodecs[2].name);
+    assertEquals("AV1", supportedCodecs[1].name);
-    assertEquals("42e01f", videoCodecs[2].params.get("profile-level-id"));
+    assertEquals("VP9", supportedCodecs[2].name);
-    assertEquals("H264", videoCodecs[3].name);
-    assertEquals("640c1f", videoCodecs[3].params.get("profile-level-id"));
   }
 
   @SmallTest
   @Test
-  public void testGetSupportedCodecsWithoutHardwareH264HighProfile() {
+  public void getSupportedCodecs_hwVp8WithDifferentParams_twoVp8() {
-    VideoEncoderFactory hwFactory = new CustomHardwareVideoEncoderFactory(true, false);
+    VideoCodecInfo hwVp8Encoder = new VideoCodecInfo("VP8", new HashMap<String, String>() {
-    DefaultVideoEncoderFactory dvef = new DefaultVideoEncoderFactory(hwFactory);
+      { put("param", "value"); }
-    VideoCodecInfo[] videoCodecs = dvef.getSupportedCodecs();
+    });
-    assertEquals(3, videoCodecs.length);
+    VideoEncoderFactory hwFactory = new CustomHardwareVideoEncoderFactory(hwVp8Encoder);
-    assertEquals("VP8", videoCodecs[0].name);
+    DefaultVideoEncoderFactory defFactory = new DefaultVideoEncoderFactory(hwFactory);
-    assertEquals("VP9", videoCodecs[1].name);
+    VideoCodecInfo[] supportedCodecs = defFactory.getSupportedCodecs();
-    assertEquals("H264", videoCodecs[2].name);
+    assertEquals(4, supportedCodecs.length);
-    assertEquals("42e01f", videoCodecs[2].params.get("profile-level-id"));
+    assertEquals("VP8", supportedCodecs[0].name);
-  }
+    assertEquals("AV1", supportedCodecs[1].name);
-
+    assertEquals("VP9", supportedCodecs[2].name);
-  @SmallTest
+    assertEquals("VP8", supportedCodecs[3].name);
-  @Test
+    assertEquals(1, supportedCodecs[3].params.size());
-  public void testGetSupportedCodecsWithoutHardwareVP8() {
+    assertEquals("value", supportedCodecs[3].params.get("param"));
-    VideoEncoderFactory hwFactory = new CustomHardwareVideoEncoderFactory(false, true);
-    DefaultVideoEncoderFactory dvef = new DefaultVideoEncoderFactory(hwFactory);
-    VideoCodecInfo[] videoCodecs = dvef.getSupportedCodecs();
-    assertEquals(4, videoCodecs.length);
-    assertEquals("VP8", videoCodecs[0].name);
-    assertEquals("VP9", videoCodecs[1].name);
-    assertEquals("H264", videoCodecs[2].name);
-    assertEquals("42e01f", videoCodecs[2].params.get("profile-level-id"));
-    assertEquals("H264", videoCodecs[3].name);
-    assertEquals("640c1f", videoCodecs[3].params.get("profile-level-id"));
   }
 }

sdk/android/instrumentationtests/src/org/webrtc/SoftwareVideoDecoderFactoryTest.java

@@ -0,0 +1,42 @@
+/*
+ *  Copyright 2022 The WebRTC Project Authors. All rights reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+package org.webrtc;
+
+import static org.junit.Assert.assertEquals;
+
+import androidx.annotation.Nullable;
+import androidx.test.filters.SmallTest;
+import java.util.ArrayList;
+import java.util.HashMap;
+import org.junit.Before;
+import org.junit.Test;
+
+/** Unit tests for {@link SoftwareVideoDecoderFactory}. */
+public class SoftwareVideoDecoderFactoryTest {
+  @Before
+  public void setUp() {
+    NativeLibrary.initialize(new NativeLibrary.DefaultLoader(), TestConstants.NATIVE_LIBRARY);
+  }
+
+  @SmallTest
+  @Test
+  public void getSupportedCodecs_returnsDefaultCodecs() {
+    VideoDecoderFactory factory = new SoftwareVideoDecoderFactory();
+    VideoCodecInfo[] codecs = factory.getSupportedCodecs();
+    assertEquals(6, codecs.length);
+    assertEquals("VP8", codecs[0].name);
+    assertEquals("VP9", codecs[1].name);
+    assertEquals("VP9", codecs[2].name);
+    assertEquals("VP9", codecs[3].name);
+    assertEquals("AV1", codecs[4].name);
+    assertEquals("AV1", codecs[5].name);
+  }
+}

sdk/android/instrumentationtests/src/org/webrtc/SoftwareVideoEncoderFactoryTest.java

@@ -0,0 +1,39 @@
+/*
+ *  Copyright 2022 The WebRTC Project Authors. All rights reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+package org.webrtc;
+
+import static org.junit.Assert.assertEquals;
+
+import androidx.annotation.Nullable;
+import androidx.test.filters.SmallTest;
+import java.util.ArrayList;
+import java.util.HashMap;
+import org.junit.Before;
+import org.junit.Test;
+
+/** Unit tests for {@link SoftwareVideoEncoderFactory}. */
+public class SoftwareVideoEncoderFactoryTest {
+  @Before
+  public void setUp() {
+    NativeLibrary.initialize(new NativeLibrary.DefaultLoader(), TestConstants.NATIVE_LIBRARY);
+  }
+
+  @SmallTest
+  @Test
+  public void getSupportedCodecs_returnsDefaultCodecs() {
+    VideoEncoderFactory factory = new SoftwareVideoEncoderFactory();
+    VideoCodecInfo[] codecs = factory.getSupportedCodecs();
+    assertEquals(3, codecs.length);
+    assertEquals("VP8", codecs[0].name);
+    assertEquals("AV1", codecs[1].name);
+    assertEquals("VP9", codecs[2].name);
+  }
+}

sdk/android/src/jni/software_video_decoder_factory.cc

@@ -0,0 +1,50 @@
+/*
+ *  Copyright 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "api/video_codecs/builtin_video_decoder_factory.h"
+#include "api/video_codecs/video_decoder.h"
+#include "sdk/android/generated_swcodecs_jni/SoftwareVideoDecoderFactory_jni.h"
+#include "sdk/android/native_api/jni/java_types.h"
+#include "sdk/android/src/jni/jni_helpers.h"
+#include "sdk/android/src/jni/video_codec_info.h"
+
+namespace webrtc {
+namespace jni {
+
+static jlong JNI_SoftwareVideoDecoderFactory_CreateFactory(JNIEnv* env) {
+  return webrtc::NativeToJavaPointer(
+      CreateBuiltinVideoDecoderFactory().release());
+}
+
+static jlong JNI_SoftwareVideoDecoderFactory_CreateDecoder(
+    JNIEnv* env,
+    jlong j_factory,
+    const webrtc::JavaParamRef<jobject>& j_video_codec_info) {
+  auto* const native_factory =
+      reinterpret_cast<webrtc::VideoDecoderFactory*>(j_factory);
+  const auto video_format =
+      webrtc::jni::VideoCodecInfoToSdpVideoFormat(env, j_video_codec_info);
+
+  return webrtc::NativeToJavaPointer(
+      native_factory->CreateVideoDecoder(video_format).release());
+}
+
+static webrtc::ScopedJavaLocalRef<jobject>
+JNI_SoftwareVideoDecoderFactory_GetSupportedCodecs(JNIEnv* env,
+                                                   jlong j_factory) {
+  auto* const native_factory =
+      reinterpret_cast<webrtc::VideoDecoderFactory*>(j_factory);
+
+  return webrtc::NativeToJavaList(env, native_factory->GetSupportedFormats(),
+                                  &webrtc::jni::SdpVideoFormatToVideoCodecInfo);
+}
+
+}  // namespace jni
+}  // namespace webrtc

sdk/android/src/jni/software_video_encoder_factory.cc

@@ -0,0 +1,50 @@
+/*
+ *  Copyright 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "api/video_codecs/builtin_video_encoder_factory.h"
+#include "api/video_codecs/video_encoder.h"
+#include "sdk/android/generated_swcodecs_jni/SoftwareVideoEncoderFactory_jni.h"
+#include "sdk/android/native_api/jni/java_types.h"
+#include "sdk/android/src/jni/jni_helpers.h"
+#include "sdk/android/src/jni/video_codec_info.h"
+
+namespace webrtc {
+namespace jni {
+
+static jlong JNI_SoftwareVideoEncoderFactory_CreateFactory(JNIEnv* env) {
+  return webrtc::NativeToJavaPointer(
+      CreateBuiltinVideoEncoderFactory().release());
+}
+
+static jlong JNI_SoftwareVideoEncoderFactory_CreateEncoder(
+    JNIEnv* env,
+    jlong j_factory,
+    const webrtc::JavaParamRef<jobject>& j_video_codec_info) {
+  auto* const native_factory =
+      reinterpret_cast<webrtc::VideoEncoderFactory*>(j_factory);
+  const auto video_format =
+      webrtc::jni::VideoCodecInfoToSdpVideoFormat(env, j_video_codec_info);
+
+  return webrtc::NativeToJavaPointer(
+      native_factory->CreateVideoEncoder(video_format).release());
+}
+
+static webrtc::ScopedJavaLocalRef<jobject>
+JNI_SoftwareVideoEncoderFactory_GetSupportedCodecs(JNIEnv* env,
+                                                   jlong j_factory) {
+  auto* const native_factory =
+      reinterpret_cast<webrtc::VideoEncoderFactory*>(j_factory);
+
+  return webrtc::NativeToJavaList(env, native_factory->GetSupportedFormats(),
+                                  &webrtc::jni::SdpVideoFormatToVideoCodecInfo);
+}
+
+}  // namespace jni
+}  // namespace webrtc