diff --git a/webrtc/api/java/jni/androidmediadecoder_jni.cc b/webrtc/api/java/jni/androidmediadecoder_jni.cc index 7c303e8fc3..43c195fac3 100644 --- a/webrtc/api/java/jni/androidmediadecoder_jni.cc +++ b/webrtc/api/java/jni/androidmediadecoder_jni.cc @@ -99,6 +99,7 @@ class MediaCodecVideoDecoder : public webrtc::VideoDecoder, void CheckOnCodecThread(); int32_t InitDecodeOnCodecThread(); + int32_t ResetDecodeOnCodecThread(); int32_t ReleaseOnCodecThread(); int32_t DecodeOnCodecThread(const EncodedImage& inputImage); // Deliver any outputs pending in the MediaCodec to our |callback_| and return @@ -106,6 +107,7 @@ class MediaCodecVideoDecoder : public webrtc::VideoDecoder, bool DeliverPendingOutputs(JNIEnv* jni, int dequeue_timeout_us); int32_t ProcessHWErrorOnCodecThread(); void EnableFrameLogOnWarning(); + void ResetVariables(); // Type of video codec. VideoCodecType codecType_; @@ -139,6 +141,7 @@ class MediaCodecVideoDecoder : public webrtc::VideoDecoder, ScopedGlobalRef j_media_codec_video_decoder_class_; ScopedGlobalRef j_media_codec_video_decoder_; jmethodID j_init_decode_method_; + jmethodID j_reset_method_; jmethodID j_release_method_; jmethodID j_dequeue_input_buffer_method_; jmethodID j_queue_input_buffer_method_; @@ -200,6 +203,8 @@ MediaCodecVideoDecoder::MediaCodecVideoDecoder( jni, *j_media_codec_video_decoder_class_, "initDecode", "(Lorg/webrtc/MediaCodecVideoDecoder$VideoCodecType;" "IILorg/webrtc/SurfaceTextureHelper;)Z"); + j_reset_method_ = + GetMethodID(jni, *j_media_codec_video_decoder_class_, "reset", "(II)V"); j_release_method_ = GetMethodID(jni, *j_media_codec_video_decoder_class_, "release", "()V"); j_dequeue_input_buffer_method_ = GetMethodID( @@ -306,6 +311,20 @@ int32_t MediaCodecVideoDecoder::InitDecode(const VideoCodec* inst, Bind(&MediaCodecVideoDecoder::InitDecodeOnCodecThread, this)); } +void MediaCodecVideoDecoder::ResetVariables() { + CheckOnCodecThread(); + + key_frame_required_ = true; + frames_received_ = 0; + frames_decoded_ = 0; + frames_decoded_logged_ = kMaxDecodedLogFrames; + start_time_ms_ = GetCurrentTimeMs(); + current_frames_ = 0; + current_bytes_ = 0; + current_decoding_time_ms_ = 0; + current_delay_time_ms_ = 0; +} + int32_t MediaCodecVideoDecoder::InitDecodeOnCodecThread() { CheckOnCodecThread(); JNIEnv* jni = AttachCurrentThreadIfNeeded(); @@ -322,11 +341,7 @@ int32_t MediaCodecVideoDecoder::InitDecodeOnCodecThread() { return WEBRTC_VIDEO_CODEC_ERROR; } - // Always start with a complete key frame. - key_frame_required_ = true; - frames_received_ = 0; - frames_decoded_ = 0; - frames_decoded_logged_ = kMaxDecodedLogFrames; + ResetVariables(); jobject java_surface_texture_helper_ = nullptr; if (use_surface_) { @@ -352,6 +367,7 @@ int32_t MediaCodecVideoDecoder::InitDecodeOnCodecThread() { codec_.width, codec_.height, java_surface_texture_helper_); + if (CheckException(jni) || !success) { ALOGE << "Codec initialization error - fallback to SW codec."; sw_fallback_required_ = true; @@ -372,16 +388,11 @@ int32_t MediaCodecVideoDecoder::InitDecodeOnCodecThread() { default: max_pending_frames_ = 0; } - start_time_ms_ = GetCurrentTimeMs(); - current_frames_ = 0; - current_bytes_ = 0; - current_decoding_time_ms_ = 0; - current_delay_time_ms_ = 0; + ALOGD << "Maximum amount of pending frames: " << max_pending_frames_; jobjectArray input_buffers = (jobjectArray)GetObjectField( jni, *j_media_codec_video_decoder_, j_input_buffers_field_); size_t num_input_buffers = jni->GetArrayLength(input_buffers); - ALOGD << "Maximum amount of pending frames: " << max_pending_frames_; input_buffers_.resize(num_input_buffers); for (size_t i = 0; i < num_input_buffers; ++i) { input_buffers_[i] = @@ -398,6 +409,37 @@ int32_t MediaCodecVideoDecoder::InitDecodeOnCodecThread() { return WEBRTC_VIDEO_CODEC_OK; } +int32_t MediaCodecVideoDecoder::ResetDecodeOnCodecThread() { + CheckOnCodecThread(); + JNIEnv* jni = AttachCurrentThreadIfNeeded(); + ScopedLocalRefFrame local_ref_frame(jni); + ALOGD << "ResetDecodeOnCodecThread Type: " << (int)codecType_ << ". " + << codec_.width << " x " << codec_.height; + ALOGD << " Frames received: " << frames_received_ << + ". Frames decoded: " << frames_decoded_; + + inited_ = false; + rtc::MessageQueueManager::Clear(this); + ResetVariables(); + + jni->CallVoidMethod( + *j_media_codec_video_decoder_, + j_reset_method_, + codec_.width, + codec_.height); + + if (CheckException(jni)) { + ALOGE << "Soft reset error - fallback to SW codec."; + sw_fallback_required_ = true; + return WEBRTC_VIDEO_CODEC_ERROR; + } + inited_ = true; + + codec_thread_->PostDelayed(kMediaCodecPollMs, this); + + return WEBRTC_VIDEO_CODEC_OK; +} + int32_t MediaCodecVideoDecoder::Release() { ALOGD << "DecoderRelease request"; return codec_thread_->Invoke( @@ -493,9 +535,22 @@ int32_t MediaCodecVideoDecoder::Decode( if ((inputImage._encodedWidth * inputImage._encodedHeight > 0) && (inputImage._encodedWidth != codec_.width || inputImage._encodedHeight != codec_.height)) { + ALOGW << "Input resolution changed from " << + codec_.width << " x " << codec_.height << " to " << + inputImage._encodedWidth << " x " << inputImage._encodedHeight; codec_.width = inputImage._encodedWidth; codec_.height = inputImage._encodedHeight; - int32_t ret = InitDecode(&codec_, 1); + int32_t ret; + if (use_surface_ && codecType_ == kVideoCodecVP8) { + // Soft codec reset - only for VP8 and surface decoding. + // TODO(glaznev): try to use similar approach for H.264 + // and buffer decoding. + ret = codec_thread_->Invoke(Bind( + &MediaCodecVideoDecoder::ResetDecodeOnCodecThread, this)); + } else { + // Hard codec reset. + ret = InitDecode(&codec_, 1); + } if (ret < 0) { ALOGE << "InitDecode failure: " << ret << " - fallback to SW codec"; sw_fallback_required_ = true; @@ -531,7 +586,9 @@ int32_t MediaCodecVideoDecoder::DecodeOnCodecThread( // Try to drain the decoder and wait until output is not too // much behind the input. - if (frames_received_ > frames_decoded_ + max_pending_frames_) { + if (codecType_ == kVideoCodecH264 && + frames_received_ > frames_decoded_ + max_pending_frames_) { + // Print warning for H.264 only - for VP8/VP9 one frame delay is ok. ALOGW << "Decoder is too far behind. Try to drain. Received: " << frames_received_ << ". Decoded: " << frames_decoded_; EnableFrameLogOnWarning(); diff --git a/webrtc/api/java/src/org/webrtc/MediaCodecVideoDecoder.java b/webrtc/api/java/src/org/webrtc/MediaCodecVideoDecoder.java index fe2c7dc468..6a3e710769 100644 --- a/webrtc/api/java/src/org/webrtc/MediaCodecVideoDecoder.java +++ b/webrtc/api/java/src/org/webrtc/MediaCodecVideoDecoder.java @@ -239,12 +239,14 @@ public class MediaCodecVideoDecoder { // Pass null in |surfaceTextureHelper| to configure the codec for ByteBuffer output. private boolean initDecode( - VideoCodecType type, int width, int height, SurfaceTextureHelper surfaceTextureHelper) { + VideoCodecType type, int width, int height, + SurfaceTextureHelper surfaceTextureHelper) { if (mediaCodecThread != null) { - throw new RuntimeException("Forgot to release()?"); + throw new RuntimeException("initDecode: Forgot to release()?"); } - useSurface = (surfaceTextureHelper != null); + String mime = null; + useSurface = (surfaceTextureHelper != null); String[] supportedCodecPrefixes = null; if (type == VideoCodecType.VIDEO_CODEC_VP8) { mime = VP8_MIME_TYPE; @@ -256,15 +258,17 @@ public class MediaCodecVideoDecoder { mime = H264_MIME_TYPE; supportedCodecPrefixes = supportedH264HwCodecPrefixes; } else { - throw new RuntimeException("Non supported codec " + type); + throw new RuntimeException("initDecode: Non-supported codec " + type); } DecoderProperties properties = findDecoder(mime, supportedCodecPrefixes); if (properties == null) { throw new RuntimeException("Cannot find HW decoder for " + type); } + Logging.d(TAG, "Java initDecode: " + type + " : "+ width + " x " + height + ". Color: 0x" + Integer.toHexString(properties.colorFormat) + ". Use Surface: " + useSurface); + runningInstance = this; // Decoder is now running and can be queried for stack traces. mediaCodecThread = Thread.currentThread(); try { @@ -283,14 +287,14 @@ public class MediaCodecVideoDecoder { format.setInteger(MediaFormat.KEY_COLOR_FORMAT, properties.colorFormat); } Logging.d(TAG, " Format: " + format); - mediaCodec = - MediaCodecVideoEncoder.createByCodecName(properties.codecName); + mediaCodec = MediaCodecVideoEncoder.createByCodecName(properties.codecName); if (mediaCodec == null) { Logging.e(TAG, "Can not create media decoder"); return false; } mediaCodec.configure(format, surface, null, 0); mediaCodec.start(); + colorFormat = properties.colorFormat; outputBuffers = mediaCodec.getOutputBuffers(); inputBuffers = mediaCodec.getInputBuffers(); @@ -307,6 +311,24 @@ public class MediaCodecVideoDecoder { } } + // Resets the decoder so it can start decoding frames with new resolution. + // Flushes MediaCodec and clears decoder output buffers. + private void reset(int width, int height) { + if (mediaCodecThread == null || mediaCodec == null) { + throw new RuntimeException("Incorrect reset call for non-initialized decoder."); + } + Logging.d(TAG, "Java reset: " + width + " x " + height); + + mediaCodec.flush(); + + this.width = width; + this.height = height; + decodeStartTimeMs.clear(); + dequeuedSurfaceOutputBuffers.clear(); + hasDecodedFirstFrame = false; + droppedFrames = 0; + } + private void release() { Logging.d(TAG, "Java releaseDecoder. Total number of dropped frames: " + droppedFrames); checkOnMediaCodecThread(); diff --git a/webrtc/examples/androidtests/src/org/appspot/apprtc/test/PeerConnectionClientTest.java b/webrtc/examples/androidtests/src/org/appspot/apprtc/test/PeerConnectionClientTest.java index e5fd9526af..08f5b91d23 100644 --- a/webrtc/examples/androidtests/src/org/appspot/apprtc/test/PeerConnectionClientTest.java +++ b/webrtc/examples/androidtests/src/org/appspot/apprtc/test/PeerConnectionClientTest.java @@ -40,6 +40,7 @@ public class PeerConnectionClientTest extends InstrumentationTestCase private static final int WAIT_TIMEOUT = 7000; private static final int CAMERA_SWITCH_ATTEMPTS = 3; private static final int VIDEO_RESTART_ATTEMPTS = 3; + private static final int CAPTURE_FORMAT_CHANGE_ATTEMPTS = 3; private static final int VIDEO_RESTART_TIMEOUT = 500; private static final int EXPECTED_VIDEO_FRAMES = 10; private static final String VIDEO_CODEC_VP8 = "VP8"; @@ -49,6 +50,12 @@ public class PeerConnectionClientTest extends InstrumentationTestCase private static final String LOCAL_RENDERER_NAME = "Local renderer"; private static final String REMOTE_RENDERER_NAME = "Remote renderer"; + private static final int MAX_VIDEO_FPS = 30; + private static final int WIDTH_VGA = 640; + private static final int HEIGHT_VGA = 480; + private static final int WIDTH_QVGA = 320; + private static final int HEIGHT_QVGA = 240; + // The peer connection client is assumed to be thread safe in itself; the // reference is written by the test thread and read by worker threads. private volatile PeerConnectionClient pcClient; @@ -551,4 +558,55 @@ public class PeerConnectionClientTest extends InstrumentationTestCase assertTrue(waitForPeerConnectionClosed(WAIT_TIMEOUT)); Log.d(TAG, "testVideoSourceRestart done."); } + + // Checks if capture format can be changed on fly and decoder can be reset properly. + public void testCaptureFormatChange() throws InterruptedException { + Log.d(TAG, "testCaptureFormatChange"); + loopback = true; + + MockRenderer localRenderer = new MockRenderer(EXPECTED_VIDEO_FRAMES, LOCAL_RENDERER_NAME); + MockRenderer remoteRenderer = new MockRenderer(EXPECTED_VIDEO_FRAMES, REMOTE_RENDERER_NAME); + + pcClient = createPeerConnectionClient( + localRenderer, remoteRenderer, createParametersForVideoCall(VIDEO_CODEC_VP8, false), null); + + // Wait for local SDP, rename it to answer and set as remote SDP. + assertTrue("Local SDP was not set.", waitForLocalSDP(WAIT_TIMEOUT)); + SessionDescription remoteSdp = new SessionDescription( + SessionDescription.Type.fromCanonicalForm("answer"), + localSdp.description); + pcClient.setRemoteDescription(remoteSdp); + + // Wait for ICE connection. + assertTrue("ICE connection failure.", waitForIceConnected(ICE_CONNECTION_WAIT_TIMEOUT)); + + // Check that local and remote video frames were rendered. + assertTrue("Local video frames were not rendered before camera resolution change.", + localRenderer.waitForFramesRendered(WAIT_TIMEOUT)); + assertTrue("Remote video frames were not rendered before camera resolution change.", + remoteRenderer.waitForFramesRendered(WAIT_TIMEOUT)); + + // Change capture output format a few times. + for (int i = 0; i < 2 * CAPTURE_FORMAT_CHANGE_ATTEMPTS; i++) { + if (i % 2 == 0) { + pcClient.changeCaptureFormat(WIDTH_VGA, HEIGHT_VGA, MAX_VIDEO_FPS); + } else { + pcClient.changeCaptureFormat(WIDTH_QVGA, HEIGHT_QVGA, MAX_VIDEO_FPS); + } + + // Reset video renders and check that local and remote video frames + // were rendered after capture format change. + localRenderer.reset(EXPECTED_VIDEO_FRAMES); + remoteRenderer.reset(EXPECTED_VIDEO_FRAMES); + assertTrue("Local video frames were not rendered after capture format change.", + localRenderer.waitForFramesRendered(WAIT_TIMEOUT)); + assertTrue("Remote video frames were not rendered after capture format change.", + remoteRenderer.waitForFramesRendered(WAIT_TIMEOUT)); + } + + pcClient.close(); + assertTrue(waitForPeerConnectionClosed(WAIT_TIMEOUT)); + Log.d(TAG, "testCaptureFormatChange done."); + } + }