Implement Injectable Audio Codecs for the Java SDK.

Support Injectable Audio Codecs from the Java SDK.
The PeerConnectionFactory.Builder defaults to
BuiltinAudio(Encoder|Decoder)Factory, but other implementations are
permitted via the Audio(Encoder|Decoder)FactoryFactory interface.

Bug: webrtc:9916
Change-Id: I61ad4a6e57666bc1be79daf5f40b129e0eacad84
Reviewed-on: https://webrtc-review.googlesource.com/c/107711
Commit-Queue: Lennart Kolmodin <kolmodin@webrtc.org>
Reviewed-by: Karl Wiberg <kwiberg@webrtc.org>
Reviewed-by: Sami Kalliomäki <sakal@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#25478}

sdk/android/BUILD.gn

@@ -38,6 +38,7 @@ if (is_android) {
     deps = [
       ":audio_api_java",
       ":base_java",
+      ":builtin_audio_codecs_java",
       ":camera_java",
       ":default_video_codec_factory_java",
       ":filevideo_java",
@@ -110,6 +111,7 @@ if (is_android) {
     public_deps = [  # no-presubmit-check TODO(webrtc:8603)
       ":audio_jni",
       ":base_jni",
+      ":builtin_audio_codecs_jni",
       ":java_audio_device_module_jni",
       ":media_jni",
       ":peerconnection_jni",
@@ -204,7 +206,11 @@ if (is_android) {
   }
 
   rtc_android_library("audio_api_java") {
-    java_files = [ "api/org/webrtc/audio/AudioDeviceModule.java" ]
+    java_files = [
+      "api/org/webrtc/audio/AudioDeviceModule.java",
+      "api/org/webrtc/AudioDecoderFactoryFactory.java",
+      "api/org/webrtc/AudioEncoderFactoryFactory.java",
+    ]
 
     deps = [
       ":base_java",
@@ -320,6 +326,7 @@ if (is_android) {
     deps = [
       ":audio_api_java",
       ":base_java",
+      ":builtin_audio_codecs_java",
       ":default_video_codec_factory_java",
       ":logging_java",
       ":swcodecs_java",
@@ -429,6 +436,17 @@ if (is_android) {
     ]
   }
 
+  rtc_android_library("builtin_audio_codecs_java") {
+    java_files = [
+      "api/org/webrtc/BuiltinAudioDecoderFactoryFactory.java",
+      "api/org/webrtc/BuiltinAudioEncoderFactoryFactory.java",
+    ]
+
+    deps = [
+      ":audio_api_java",
+    ]
+  }
+
   rtc_android_library("screencapturer_java") {
     java_files = [ "api/org/webrtc/ScreenCapturerAndroid.java" ]
 
@@ -514,6 +532,7 @@ if (is_android) {
 
     deps = [
       ":base_jni",
+      ":builtin_audio_codecs_jni",
       "../../api/audio_codecs:builtin_audio_decoder_factory",
       "../../api/audio_codecs:builtin_audio_encoder_factory",
       "../../modules/audio_processing:audio_processing",
@@ -521,6 +540,22 @@ if (is_android) {
     ]
   }
 
+  rtc_static_library("builtin_audio_codecs_jni") {
+    sources = [
+      "src/jni/builtinaudiodecoderfactoryfactory.cc",
+      "src/jni/builtinaudioencoderfactoryfactory.cc",
+    ]
+
+    deps = [
+      ":base_jni",
+      ":generated_builtin_audio_codecs_jni",
+      ":native_api_jni",
+      "../../api/audio_codecs:builtin_audio_decoder_factory",
+      "../../api/audio_codecs:builtin_audio_encoder_factory",
+      "../../rtc_base:rtc_base_approved",
+    ]
+  }
+
   rtc_static_library("video_jni") {
     sources = []
     deps = [
@@ -1216,6 +1251,16 @@ if (is_android) {
     jni_generator_include = "//sdk/android/src/jni/jni_generator_helper.h"
   }
 
+  generate_jni("generated_builtin_audio_codecs_jni") {
+    sources = [
+      "api/org/webrtc/BuiltinAudioDecoderFactoryFactory.java",
+      "api/org/webrtc/BuiltinAudioEncoderFactoryFactory.java",
+    ]
+    jni_package = ""
+    namespace = "webrtc::jni"
+    jni_generator_include = "//sdk/android/src/jni/jni_generator_helper.h"
+  }
+
   # Generated JNI for native API targets
 
   generate_jni("generated_native_api_jni") {
@@ -1268,6 +1313,7 @@ if (is_android) {
 
       java_files = [
         "instrumentationtests/src/org/webrtc/AndroidVideoDecoderInstrumentationTest.java",
+        "instrumentationtests/src/org/webrtc/BuiltinAudioCodecsFactoryFactoryTest.java",
         "instrumentationtests/src/org/webrtc/Camera1CapturerUsingByteBufferTest.java",
         "instrumentationtests/src/org/webrtc/Camera1CapturerUsingTextureTest.java",
         "instrumentationtests/src/org/webrtc/Camera2CapturerTest.java",

sdk/android/api/org/webrtc/AudioDecoderFactoryFactory.java

@@ -0,0 +1,21 @@
+/*
+ *  Copyright 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+package org.webrtc;
+
+/**
+ * Implementations of this interface can create a native {@code webrtc::AudioDecoderFactory}.
+ */
+public interface AudioDecoderFactoryFactory {
+  /**
+   * Returns a pointer to a {@code webrtc::AudioDecoderFactory}. The caller takes ownership.
+   */
+  long createNativeAudioDecoderFactory();
+}

sdk/android/api/org/webrtc/AudioEncoderFactoryFactory.java

@@ -0,0 +1,21 @@
+/*
+ *  Copyright 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+package org.webrtc;
+
+/**
+ * Implementations of this interface can create a native {@code webrtc::AudioEncoderFactory}.
+ */
+public interface AudioEncoderFactoryFactory {
+  /**
+   * Returns a pointer to a {@code webrtc::AudioEncoderFactory}. The caller takes ownership.
+   */
+  long createNativeAudioEncoderFactory();
+}

sdk/android/api/org/webrtc/BuiltinAudioDecoderFactoryFactory.java

@@ -0,0 +1,23 @@
+/*
+ *  Copyright 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+package org.webrtc;
+
+/**
+ * Creates a native {@code webrtc::AudioDecoderFactory} with the builtin audio decoders.
+ */
+public class BuiltinAudioDecoderFactoryFactory implements AudioDecoderFactoryFactory {
+  @Override
+  public long createNativeAudioDecoderFactory() {
+    return nativeCreateBuiltinAudioDecoderFactory();
+  }
+
+  private static native long nativeCreateBuiltinAudioDecoderFactory();
+}

sdk/android/api/org/webrtc/BuiltinAudioEncoderFactoryFactory.java

@@ -0,0 +1,23 @@
+/*
+ *  Copyright 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+package org.webrtc;
+
+/**
+ * This class creates a native {@code webrtc::AudioEncoderFactory} with the builtin audio encoders.
+ */
+public class BuiltinAudioEncoderFactoryFactory implements AudioEncoderFactoryFactory {
+  @Override
+  public long createNativeAudioEncoderFactory() {
+    return nativeCreateBuiltinAudioEncoderFactory();
+  }
+
+  private static native long nativeCreateBuiltinAudioEncoderFactory();
+}

sdk/android/api/org/webrtc/PeerConnectionFactory.java

@@ -181,13 +181,17 @@ public class PeerConnectionFactory {
   }
 
   public static class Builder {
-    private @Nullable Options options;
+    @Nullable private Options options;
-    private @Nullable AudioDeviceModule audioDeviceModule = new LegacyAudioDeviceModule();
+    @Nullable private AudioDeviceModule audioDeviceModule = new LegacyAudioDeviceModule();
-    private @Nullable VideoEncoderFactory encoderFactory;
+    private AudioEncoderFactoryFactory audioEncoderFactoryFactory =
-    private @Nullable VideoDecoderFactory decoderFactory;
+        new BuiltinAudioEncoderFactoryFactory();
-    private @Nullable AudioProcessingFactory audioProcessingFactory;
+    private AudioDecoderFactoryFactory audioDecoderFactoryFactory =
-    private @Nullable FecControllerFactoryFactoryInterface fecControllerFactoryFactory;
+        new BuiltinAudioDecoderFactoryFactory();
-    private @Nullable MediaTransportFactoryFactory mediaTransportFactoryFactory;
+    @Nullable private VideoEncoderFactory videoEncoderFactory;
+    @Nullable private VideoDecoderFactory videoDecoderFactory;
+    @Nullable private AudioProcessingFactory audioProcessingFactory;
+    @Nullable private FecControllerFactoryFactoryInterface fecControllerFactoryFactory;
+    @Nullable private MediaTransportFactoryFactory mediaTransportFactoryFactory;
 
     private Builder() {}
 
@@ -201,13 +205,33 @@ public class PeerConnectionFactory {
       return this;
     }
 
-    public Builder setVideoEncoderFactory(VideoEncoderFactory encoderFactory) {
+    public Builder setAudioEncoderFactoryFactory(
-      this.encoderFactory = encoderFactory;
+        AudioEncoderFactoryFactory audioEncoderFactoryFactory) {
+      if (audioEncoderFactoryFactory == null) {
+        throw new IllegalArgumentException(
+            "PeerConnectionFactory.Builder does not accept a null AudioEncoderFactoryFactory.");
+      }
+      this.audioEncoderFactoryFactory = audioEncoderFactoryFactory;
       return this;
     }
 
-    public Builder setVideoDecoderFactory(VideoDecoderFactory decoderFactory) {
+    public Builder setAudioDecoderFactoryFactory(
-      this.decoderFactory = decoderFactory;
+        AudioDecoderFactoryFactory audioDecoderFactoryFactory) {
+      if (audioDecoderFactoryFactory == null) {
+        throw new IllegalArgumentException(
+            "PeerConnectionFactory.Builder does not accept a null AudioDecoderFactoryFactory.");
+      }
+      this.audioDecoderFactoryFactory = audioDecoderFactoryFactory;
+      return this;
+    }
+
+    public Builder setVideoEncoderFactory(VideoEncoderFactory videoEncoderFactory) {
+      this.videoEncoderFactory = videoEncoderFactory;
+      return this;
+    }
+
+    public Builder setVideoDecoderFactory(VideoDecoderFactory videoDecoderFactory) {
+      this.videoDecoderFactory = videoDecoderFactory;
       return this;
     }
 
@@ -234,7 +258,8 @@ public class PeerConnectionFactory {
     }
 
     public PeerConnectionFactory createPeerConnectionFactory() {
-      return new PeerConnectionFactory(options, audioDeviceModule, encoderFactory, decoderFactory,
+      return new PeerConnectionFactory(options, audioDeviceModule, audioEncoderFactoryFactory,
+          audioDecoderFactoryFactory, videoEncoderFactory, videoDecoderFactory,
           audioProcessingFactory, fecControllerFactoryFactory, mediaTransportFactoryFactory);
     }
   }
@@ -314,14 +339,19 @@ public class PeerConnectionFactory {
   }
 
   private PeerConnectionFactory(Options options, @Nullable AudioDeviceModule audioDeviceModule,
-      @Nullable VideoEncoderFactory encoderFactory, @Nullable VideoDecoderFactory decoderFactory,
+      AudioEncoderFactoryFactory audioEncoderFactoryFactory,
+      AudioDecoderFactoryFactory audioDecoderFactoryFactory,
+      @Nullable VideoEncoderFactory videoEncoderFactory,
+      @Nullable VideoDecoderFactory videoDecoderFactory,
       @Nullable AudioProcessingFactory audioProcessingFactory,
       @Nullable FecControllerFactoryFactoryInterface fecControllerFactoryFactory,
       @Nullable MediaTransportFactoryFactory mediaTransportFactoryFactory) {
     checkInitializeHasBeenCalled();
     nativeFactory = nativeCreatePeerConnectionFactory(ContextUtils.getApplicationContext(), options,
         audioDeviceModule == null ? 0 : audioDeviceModule.getNativeAudioDeviceModulePointer(),
-        encoderFactory, decoderFactory,
+        audioEncoderFactoryFactory.createNativeAudioEncoderFactory(),
+        audioDecoderFactoryFactory.createNativeAudioDecoderFactory(), videoEncoderFactory,
+        videoDecoderFactory,
         audioProcessingFactory == null ? 0 : audioProcessingFactory.createNative(),
         fecControllerFactoryFactory == null ? 0 : fecControllerFactoryFactory.createNative(),
         mediaTransportFactoryFactory == null
@@ -527,10 +557,12 @@ public class PeerConnectionFactory {
   private static native void nativeShutdownInternalTracer();
   private static native boolean nativeStartInternalTracingCapture(String tracingFilename);
   private static native void nativeStopInternalTracingCapture();
+
   private static native long nativeCreatePeerConnectionFactory(Context context, Options options,
-      long nativeAudioDeviceModule, VideoEncoderFactory encoderFactory,
+      long nativeAudioDeviceModule, long audioEncoderFactory, long audioDecoderFactory,
-      VideoDecoderFactory decoderFactory, long nativeAudioProcessor,
+      VideoEncoderFactory encoderFactory, VideoDecoderFactory decoderFactory,
-      long nativeFecControllerFactory, long mediaTransportFactory);
+      long nativeAudioProcessor, long nativeFecControllerFactory, long mediaTransportFactory);
+
   private static native long nativeCreatePeerConnection(long factory,
       PeerConnection.RTCConfiguration rtcConfig, MediaConstraints constraints, long nativeObserver,
       SSLCertificateVerifier sslCertificateVerifier);

sdk/android/instrumentationtests/src/org/webrtc/BuiltinAudioCodecsFactoryFactoryTest.java

@@ -0,0 +1,58 @@
+/*
+ *  Copyright 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+package org.webrtc;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import android.support.test.filters.SmallTest;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+public final class BuiltinAudioCodecsFactoryFactoryTest {
+  @Before
+  public void setUp() {
+    System.loadLibrary(TestConstants.NATIVE_LIBRARY);
+  }
+
+  @Test
+  @SmallTest
+  public void testAudioEncoderFactoryFactoryTest() throws Exception {
+    BuiltinAudioEncoderFactoryFactory factory = new BuiltinAudioEncoderFactoryFactory();
+    long aef = 0;
+    try {
+      aef = factory.createNativeAudioEncoderFactory();
+      assertThat(aef).isNotEqualTo(0);
+    } finally {
+      if (aef != 0) {
+        JniCommon.nativeReleaseRef(aef);
+      }
+    }
+  }
+
+  @Test
+  @SmallTest
+  public void testAudioDecoderFactoryFactoryTest() throws Exception {
+    BuiltinAudioDecoderFactoryFactory factory = new BuiltinAudioDecoderFactoryFactory();
+    long adf = 0;
+    try {
+      adf = factory.createNativeAudioDecoderFactory();
+      assertThat(adf).isNotEqualTo(0);
+    } finally {
+      if (adf != 0) {
+        JniCommon.nativeReleaseRef(adf);
+      }
+    }
+  }
+}

sdk/android/src/jni/builtinaudiodecoderfactoryfactory.cc

@@ -0,0 +1,28 @@
+/*
+ *  Copyright 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "sdk/android/generated_builtin_audio_codecs_jni/jni/BuiltinAudioDecoderFactoryFactory_jni.h"
+#include "sdk/android/native_api/jni/java_types.h"
+#include "sdk/android/src/jni/jni_helpers.h"
+
+#include "api/audio_codecs/builtin_audio_decoder_factory.h"
+
+namespace webrtc {
+namespace jni {
+
+static jlong
+JNI_BuiltinAudioDecoderFactoryFactory_CreateBuiltinAudioDecoderFactory(
+    JNIEnv* env,
+    const JavaParamRef<jclass>& jcaller) {
+  return NativeToJavaPointer(CreateBuiltinAudioDecoderFactory().release());
+}
+
+}  // namespace jni
+}  // namespace webrtc

sdk/android/src/jni/builtinaudioencoderfactoryfactory.cc

@@ -0,0 +1,28 @@
+/*
+ *  Copyright 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "sdk/android/generated_builtin_audio_codecs_jni/jni/BuiltinAudioEncoderFactoryFactory_jni.h"
+#include "sdk/android/native_api/jni/java_types.h"
+#include "sdk/android/src/jni/jni_helpers.h"
+
+#include "api/audio_codecs/builtin_audio_encoder_factory.h"
+
+namespace webrtc {
+namespace jni {
+
+static jlong
+JNI_BuiltinAudioEncoderFactoryFactory_CreateBuiltinAudioEncoderFactory(
+    JNIEnv* env,
+    const JavaParamRef<jclass>& jcaller) {
+  return NativeToJavaPointer(CreateBuiltinAudioEncoderFactory().release());
+}
+
+}  // namespace jni
+}  // namespace webrtc

sdk/android/src/jni/pc/peerconnectionfactory.cc

@@ -212,6 +212,8 @@ jlong CreatePeerConnectionFactoryForJava(
     const JavaParamRef<jobject>& jcontext,
     const JavaParamRef<jobject>& joptions,
     rtc::scoped_refptr<AudioDeviceModule> audio_device_module,
+    rtc::scoped_refptr<AudioEncoderFactory> audio_encoder_factory,
+    rtc::scoped_refptr<AudioDecoderFactory> audio_decoder_factory,
     const JavaParamRef<jobject>& jencoder_factory,
     const JavaParamRef<jobject>& jdecoder_factory,
     rtc::scoped_refptr<AudioProcessing> audio_processor,
@@ -238,8 +240,6 @@ jlong CreatePeerConnectionFactoryForJava(
   RTC_CHECK(signaling_thread->Start()) << "Failed to start thread";
 
   rtc::NetworkMonitorFactory* network_monitor_factory = nullptr;
-  auto audio_encoder_factory = CreateAudioEncoderFactory();
-  auto audio_decoder_factory = CreateAudioDecoderFactory();
 
   PeerConnectionFactoryInterface::Options options;
   bool has_options = !joptions.is_null();
@@ -299,6 +299,8 @@ static jlong JNI_PeerConnectionFactory_CreatePeerConnectionFactory(
     const JavaParamRef<jobject>& jcontext,
     const JavaParamRef<jobject>& joptions,
     jlong native_audio_device_module,
+    jlong native_audio_encoder_factory,
+    jlong native_audio_decoder_factory,
     const JavaParamRef<jobject>& jencoder_factory,
     const JavaParamRef<jobject>& jdecoder_factory,
     jlong native_audio_processor,
@@ -306,6 +308,18 @@ static jlong JNI_PeerConnectionFactory_CreatePeerConnectionFactory(
     jlong native_media_transport_factory) {
   rtc::scoped_refptr<AudioProcessing> audio_processor =
       reinterpret_cast<AudioProcessing*>(native_audio_processor);
+  AudioEncoderFactory* audio_encoder_factory_ptr =
+      reinterpret_cast<AudioEncoderFactory*>(native_audio_encoder_factory);
+  rtc::scoped_refptr<AudioEncoderFactory> audio_encoder_factory(
+      audio_encoder_factory_ptr);
+  // Release the caller's reference count.
+  audio_encoder_factory->Release();
+  AudioDecoderFactory* audio_decoder_factory_ptr =
+      reinterpret_cast<AudioDecoderFactory*>(native_audio_decoder_factory);
+  rtc::scoped_refptr<AudioDecoderFactory> audio_decoder_factory(
+      audio_decoder_factory_ptr);
+  // Release the caller's reference count.
+  audio_decoder_factory->Release();
   std::unique_ptr<FecControllerFactoryInterface> fec_controller_factory(
       reinterpret_cast<FecControllerFactoryInterface*>(
           native_fec_controller_factory));
@@ -314,7 +328,8 @@ static jlong JNI_PeerConnectionFactory_CreatePeerConnectionFactory(
   return CreatePeerConnectionFactoryForJava(
       jni, jcontext, joptions,
       reinterpret_cast<AudioDeviceModule*>(native_audio_device_module),
-      jencoder_factory, jdecoder_factory,
+      audio_encoder_factory, audio_decoder_factory, jencoder_factory,
+      jdecoder_factory,
       audio_processor ? audio_processor : CreateAudioProcessing(),
       std::move(fec_controller_factory), std::move(media_transport_factory));
 }