From 66492210e57ee8efce2ad4d45a8781df1fcaa5e3 Mon Sep 17 00:00:00 2001 From: nisse Date: Wed, 21 Sep 2016 02:09:53 -0700 Subject: [PATCH] Revert of Delete VideoFrameFactory, CapturedFrame, and related code. (patchset #9 id:160001 of https://codereview.webrtc.org/2262443003/ ) Reason for revert: Breaks downstream testcode, still using CapturedFrame. Original issue's description: > Delete VideoFrameFactory, CapturedFrame, and related code. > > BUG=webrtc:5682 > > Committed: https://crrev.com/66ac50e58c790624d51ede10ae438cbadbca9d2e > Cr-Commit-Position: refs/heads/master@{#14315} TBR=pthatcher@webrtc.org,perkj@webrtc.org # Skipping CQ checks because original CL landed less than 1 days ago. NOPRESUBMIT=true NOTREECHECKS=true NOTRY=true BUG=webrtc:5682 Review-Url: https://codereview.webrtc.org/2357113002 Cr-Commit-Position: refs/heads/master@{#14320} --- webrtc/media/BUILD.gn | 5 + webrtc/media/base/fakevideocapturer.h | 65 +++++---- webrtc/media/base/videoadapter_unittest.cc | 33 ++--- webrtc/media/base/videocapturer.cc | 125 +++++++++++++++--- webrtc/media/base/videocapturer.h | 55 +++++++- webrtc/media/base/videocapturer_unittest.cc | 16 +-- webrtc/media/base/videoframefactory.cc | 51 +++++++ webrtc/media/base/videoframefactory.h | 63 +++++++++ webrtc/media/engine/webrtcvideocapturer.cc | 9 +- webrtc/media/engine/webrtcvideoengine2.h | 1 + webrtc/media/engine/webrtcvideoframe.cc | 8 ++ webrtc/media/engine/webrtcvideoframe.h | 15 ++- .../media/engine/webrtcvideoframe_unittest.cc | 72 ++++++++++ .../media/engine/webrtcvideoframefactory.cc | 30 +++++ webrtc/media/engine/webrtcvideoframefactory.h | 35 +++++ .../webrtcvideoframefactory_unittest.cc | 109 +++++++++++++++ webrtc/media/media.gyp | 4 + 17 files changed, 615 insertions(+), 81 deletions(-) create mode 100644 webrtc/media/base/videoframefactory.cc create mode 100644 webrtc/media/base/videoframefactory.h create mode 100644 webrtc/media/engine/webrtcvideoframefactory.cc create mode 100644 webrtc/media/engine/webrtcvideoframefactory.h create mode 100644 webrtc/media/engine/webrtcvideoframefactory_unittest.cc diff --git a/webrtc/media/BUILD.gn b/webrtc/media/BUILD.gn index 14a3e2f6f5..6179f67d6b 100644 --- a/webrtc/media/BUILD.gn +++ b/webrtc/media/BUILD.gn @@ -84,6 +84,8 @@ rtc_source_set("rtc_media") { "base/videocommon.h", "base/videoframe.cc", "base/videoframe.h", + "base/videoframefactory.cc", + "base/videoframefactory.h", "base/videorenderer.h", "base/videosourcebase.cc", "base/videosourcebase.h", @@ -106,6 +108,8 @@ rtc_source_set("rtc_media") { "engine/webrtcvideoengine2.h", "engine/webrtcvideoframe.cc", "engine/webrtcvideoframe.h", + "engine/webrtcvideoframefactory.cc", + "engine/webrtcvideoframefactory.h", "engine/webrtcvoe.h", "engine/webrtcvoiceengine.cc", "engine/webrtcvoiceengine.h", @@ -319,6 +323,7 @@ if (rtc_include_tests) { "engine/webrtcvideocapturer_unittest.cc", "engine/webrtcvideoengine2_unittest.cc", "engine/webrtcvideoframe_unittest.cc", + "engine/webrtcvideoframefactory_unittest.cc", "engine/webrtcvoiceengine_unittest.cc", "sctp/sctpdataengine_unittest.cc", ] diff --git a/webrtc/media/base/fakevideocapturer.h b/webrtc/media/base/fakevideocapturer.h index 0f2d8edc30..8ba56f1e6f 100644 --- a/webrtc/media/base/fakevideocapturer.h +++ b/webrtc/media/base/fakevideocapturer.h @@ -20,6 +20,9 @@ #include "webrtc/media/base/videocapturer.h" #include "webrtc/media/base/videocommon.h" #include "webrtc/media/base/videoframe.h" +#ifdef HAVE_WEBRTC_VIDEO +#include "webrtc/media/engine/webrtcvideoframefactory.h" +#endif namespace cricket { @@ -32,6 +35,9 @@ class FakeVideoCapturer : public cricket::VideoCapturer { next_timestamp_(rtc::kNumNanosecsPerMillisec), is_screencast_(is_screencast), rotation_(webrtc::kVideoRotation_0) { +#ifdef HAVE_WEBRTC_VIDEO + set_frame_factory(new cricket::WebRtcVideoFrameFactory()); +#endif // Default supported formats. Use ResetSupportedFormats to over write. std::vector formats; formats.push_back(cricket::VideoFormat(1280, 720, @@ -75,37 +81,46 @@ class FakeVideoCapturer : public cricket::VideoCapturer { if (!running_) { return false; } - RTC_CHECK(fourcc == FOURCC_I420); - RTC_CHECK(width > 0); - RTC_CHECK(height > 0); - - int adapted_width; - int adapted_height; - int crop_width; - int crop_height; - int crop_x; - int crop_y; - - // TODO(nisse): It's a bit silly to have this logic in a fake - // class. Child classes of VideoCapturer are expected to call - // AdaptFrame, and the test case - // VideoCapturerTest.SinkWantsMaxPixelAndMaxPixelCountStepUp - // depends on this. - if (AdaptFrame(width, height, 0, 0, &adapted_width, &adapted_height, - &crop_width, &crop_height, &crop_x, &crop_y, nullptr)) { - rtc::scoped_refptr buffer( - webrtc::I420Buffer::Create(adapted_width, adapted_height)); - buffer->InitializeData(); - - OnFrame(WebRtcVideoFrame(buffer, rotation_, - next_timestamp_ / rtc::kNumNanosecsPerMicrosec), - width, height); + // Currently, |fourcc| is always I420 or ARGB. + uint32_t size = 0u; + if (fourcc == cricket::FOURCC_ARGB) { + size = width * 4 * height; + } else if (fourcc == cricket::FOURCC_I420) { + size = width * height + 2 * ((width + 1) / 2) * ((height + 1) / 2); + } else { + return false; // Unsupported FOURCC. } + if (size == 0u) { + return false; // Width and/or Height were zero. + } + + cricket::CapturedFrame frame; + frame.width = width; + frame.height = height; + frame.fourcc = fourcc; + frame.data_size = size; + frame.time_stamp = initial_timestamp_ + next_timestamp_; next_timestamp_ += timestamp_interval; + std::unique_ptr data(new char[size]); + frame.data = data.get(); + // Copy something non-zero into the buffer so Validate wont complain that + // the frame is all duplicate. + memset(frame.data, 1, size / 2); + memset(reinterpret_cast(frame.data) + (size / 2), 2, + size - (size / 2)); + memcpy(frame.data, reinterpret_cast(&fourcc), 4); + frame.rotation = rotation_; + // TODO(zhurunz): SignalFrameCaptured carry returned value to be able to + // capture results from downstream. + SignalFrameCaptured(this, &frame); return true; } + void SignalCapturedFrame(cricket::CapturedFrame* frame) { + SignalFrameCaptured(this, frame); + } + sigslot::signal1 SignalDestroyed; cricket::CaptureState Start(const cricket::VideoFormat& format) override { diff --git a/webrtc/media/base/videoadapter_unittest.cc b/webrtc/media/base/videoadapter_unittest.cc index 444ef54886..e805679890 100644 --- a/webrtc/media/base/videoadapter_unittest.cc +++ b/webrtc/media/base/videoadapter_unittest.cc @@ -31,18 +31,18 @@ class VideoAdapterTest : public testing::Test { capture_format_.interval = VideoFormat::FpsToInterval(30); listener_.reset(new VideoCapturerListener(&adapter_)); - capturer_->AddOrUpdateSink(listener_.get(), rtc::VideoSinkWants()); + capturer_->SignalFrameCaptured.connect( + listener_.get(), &VideoCapturerListener::OnFrameCaptured); } virtual void TearDown() { // Explicitly disconnect the VideoCapturer before to avoid data races // (frames delivered to VideoCapturerListener while it's being destructed). - capturer_->RemoveSink(listener_.get()); + capturer_->SignalFrameCaptured.disconnect_all(); } protected: - class VideoCapturerListener - : public rtc::VideoSinkInterface { + class VideoCapturerListener: public sigslot::has_slots<> { public: struct Stats { int captured_frames; @@ -62,18 +62,19 @@ class VideoAdapterTest : public testing::Test { last_adapt_was_no_op_(false) { } - void OnFrame(const cricket::VideoFrame& frame) { + void OnFrameCaptured(VideoCapturer* capturer, + const CapturedFrame* captured_frame) { rtc::CritScope lock(&crit_); - const int in_width = frame.width(); - const int in_height = frame.height(); + const int in_width = captured_frame->width; + const int in_height = abs(captured_frame->height); int cropped_width; int cropped_height; int out_width; int out_height; - if (video_adapter_->AdaptFrameResolution( - in_width, in_height, - frame.timestamp_us() * rtc::kNumNanosecsPerMicrosec, - &cropped_width, &cropped_height, &out_width, &out_height)) { + if (video_adapter_->AdaptFrameResolution(in_width, in_height, + captured_frame->time_stamp, + &cropped_width, &cropped_height, + &out_width, &out_height)) { cropped_width_ = cropped_width; cropped_height_ = cropped_height; out_width_ = out_width; @@ -182,7 +183,7 @@ TEST_F(VideoAdapterTest, AdaptFramerateToHalf) { capturer_->CaptureFrame(); EXPECT_GE(listener_->GetStats().captured_frames, 2); - EXPECT_EQ(1, listener_->GetStats().dropped_frames); + EXPECT_EQ(0, listener_->GetStats().dropped_frames); capturer_->CaptureFrame(); EXPECT_GE(listener_->GetStats().captured_frames, 3); @@ -190,7 +191,7 @@ TEST_F(VideoAdapterTest, AdaptFramerateToHalf) { capturer_->CaptureFrame(); EXPECT_GE(listener_->GetStats().captured_frames, 4); - EXPECT_EQ(2, listener_->GetStats().dropped_frames); + EXPECT_EQ(1, listener_->GetStats().dropped_frames); capturer_->CaptureFrame(); EXPECT_GE(listener_->GetStats().captured_frames, 5); @@ -198,7 +199,7 @@ TEST_F(VideoAdapterTest, AdaptFramerateToHalf) { capturer_->CaptureFrame(); EXPECT_GE(listener_->GetStats().captured_frames, 6); - EXPECT_EQ(3, listener_->GetStats().dropped_frames); + EXPECT_EQ(2, listener_->GetStats().dropped_frames); capturer_->CaptureFrame(); EXPECT_GE(listener_->GetStats().captured_frames, 7); @@ -206,7 +207,7 @@ TEST_F(VideoAdapterTest, AdaptFramerateToHalf) { capturer_->CaptureFrame(); EXPECT_GE(listener_->GetStats().captured_frames, 8); - EXPECT_EQ(4, listener_->GetStats().dropped_frames); + EXPECT_EQ(3, listener_->GetStats().dropped_frames); capturer_->CaptureFrame(); EXPECT_GE(listener_->GetStats().captured_frames, 9); @@ -214,7 +215,7 @@ TEST_F(VideoAdapterTest, AdaptFramerateToHalf) { capturer_->CaptureFrame(); EXPECT_GE(listener_->GetStats().captured_frames, 10); - EXPECT_EQ(5, listener_->GetStats().dropped_frames); + EXPECT_EQ(4, listener_->GetStats().dropped_frames); } // Adapt the frame rate to be two thirds of the capture rate at the beginning. diff --git a/webrtc/media/base/videocapturer.cc b/webrtc/media/base/videocapturer.cc index 1d81d49eb8..c340760468 100644 --- a/webrtc/media/base/videocapturer.cc +++ b/webrtc/media/base/videocapturer.cc @@ -18,7 +18,9 @@ #include "webrtc/base/common.h" #include "webrtc/base/logging.h" #include "webrtc/base/systeminfo.h" +#include "webrtc/media/base/videoframefactory.h" #include "webrtc/media/engine/webrtcvideoframe.h" +#include "webrtc/media/engine/webrtcvideoframefactory.h" namespace cricket { @@ -31,6 +33,29 @@ static const int kYU12Penalty = 16; // Needs to be higher than MJPG index. } // namespace +///////////////////////////////////////////////////////////////////// +// Implementation of struct CapturedFrame +///////////////////////////////////////////////////////////////////// +CapturedFrame::CapturedFrame() + : width(0), + height(0), + fourcc(0), + pixel_width(0), + pixel_height(0), + time_stamp(0), + data_size(0), + rotation(webrtc::kVideoRotation_0), + data(NULL) {} + +// TODO(fbarchard): Remove this function once lmimediaengine stops using it. +bool CapturedFrame::GetDataSize(uint32_t* size) const { + if (!size || data_size == CapturedFrame::kUnknownDataSize) { + return false; + } + *size = data_size; + return true; +} + ///////////////////////////////////////////////////////////////////// // Implementation of class VideoCapturer ///////////////////////////////////////////////////////////////////// @@ -42,9 +67,16 @@ VideoCapturer::VideoCapturer() : apply_rotation_(false) { void VideoCapturer::Construct() { enable_camera_list_ = false; capture_state_ = CS_STOPPED; + SignalFrameCaptured.connect(this, &VideoCapturer::OnFrameCaptured); scaled_width_ = 0; scaled_height_ = 0; enable_video_adapter_ = true; + // There are lots of video capturers out there that don't call + // set_frame_factory. We can either go change all of them, or we + // can set this default. + // TODO(pthatcher): Remove this hack and require the frame factory + // to be passed in the constructor. + set_frame_factory(new WebRtcVideoFrameFactory()); } const std::vector* VideoCapturer::GetSupportedFormats() const { @@ -120,6 +152,29 @@ void VideoCapturer::ConstrainSupportedFormats(const VideoFormat& max_format) { UpdateFilteredSupportedFormats(); } +std::string VideoCapturer::ToString(const CapturedFrame* captured_frame) const { + std::string fourcc_name = GetFourccName(captured_frame->fourcc) + " "; + for (std::string::const_iterator i = fourcc_name.begin(); + i < fourcc_name.end(); ++i) { + // Test character is printable; Avoid isprint() which asserts on negatives. + if (*i < 32 || *i >= 127) { + fourcc_name = ""; + break; + } + } + + std::ostringstream ss; + ss << fourcc_name << captured_frame->width << "x" << captured_frame->height; + return ss.str(); +} + +void VideoCapturer::set_frame_factory(VideoFrameFactory* frame_factory) { + frame_factory_.reset(frame_factory); + if (frame_factory) { + frame_factory->SetApplyRotation(apply_rotation_); + } +} + bool VideoCapturer::GetInputSize(int* width, int* height) { rtc::CritScope cs(&frame_stats_crit_); if (!input_size_valid_) { @@ -149,6 +204,9 @@ void VideoCapturer::AddOrUpdateSink( void VideoCapturer::OnSinkWantsChanged(const rtc::VideoSinkWants& wants) { RTC_DCHECK(thread_checker_.CalledOnValidThread()); apply_rotation_ = wants.rotation_applied; + if (frame_factory_) { + frame_factory_->SetApplyRotation(apply_rotation_); + } if (video_adapter()) { video_adapter()->OnResolutionRequest(wants.max_pixel_count, @@ -196,31 +254,54 @@ bool VideoCapturer::AdaptFrame(int width, return true; } +void VideoCapturer::OnFrameCaptured(VideoCapturer*, + const CapturedFrame* captured_frame) { + int out_width; + int out_height; + int crop_width; + int crop_height; + int crop_x; + int crop_y; + + // TODO(nisse): We don't do timestamp translation on this input + // path. It seems straight-forward to enable translation, but that + // breaks the WebRtcVideoEngine2Test.PropagatesInputFrameTimestamp + // test. Probably not worth the effort to fix, instead, try to + // delete or refactor all code using VideoFrameFactory and + // SignalCapturedFrame. + if (!AdaptFrame(captured_frame->width, captured_frame->height, + captured_frame->time_stamp / rtc::kNumNanosecsPerMicrosec, + 0, + &out_width, &out_height, + &crop_width, &crop_height, &crop_x, &crop_y, nullptr)) { + return; + } + + if (!frame_factory_) { + LOG(LS_ERROR) << "No video frame factory."; + return; + } + + // TODO(nisse): Reorganize frame factory methods. crop_x and crop_y + // are ignored for now. + std::unique_ptr adapted_frame(frame_factory_->CreateAliasedFrame( + captured_frame, crop_width, crop_height, out_width, out_height)); + + if (!adapted_frame) { + // TODO(fbarchard): LOG more information about captured frame attributes. + LOG(LS_ERROR) << "Couldn't convert to I420! " + << "From " << ToString(captured_frame) << " To " + << out_width << " x " << out_height; + return; + } + + OnFrame(*adapted_frame, captured_frame->width, captured_frame->height); +} + void VideoCapturer::OnFrame(const VideoFrame& frame, int orig_width, int orig_height) { - // For a child class which implements rotation itself, we should - // always have apply_rotation_ == false or frame.rotation() == 0. - // Except possibly during races where apply_rotation_ is changed - // mid-stream. - if (apply_rotation_ && frame.rotation() != webrtc::kVideoRotation_0) { - rtc::scoped_refptr buffer( - frame.video_frame_buffer()); - if (buffer->native_handle()) { - // Sources producing native frames must handle apply_rotation - // themselves. But even if they do, we may occasionally end up - // in this case, for frames in flight at the time - // applied_rotation is set to true. In that case, we just drop - // the frame. - LOG(LS_WARNING) << "Native frame requiring rotation. Discarding."; - return; - } - broadcaster_.OnFrame(WebRtcVideoFrame( - webrtc::I420Buffer::Rotate(buffer, frame.rotation()), - webrtc::kVideoRotation_0, frame.timestamp_us())); - } else { - broadcaster_.OnFrame(frame); - } + broadcaster_.OnFrame(frame); UpdateInputSize(orig_width, orig_height); } diff --git a/webrtc/media/base/videocapturer.h b/webrtc/media/base/videocapturer.h index bb25d2a678..eb868d9877 100644 --- a/webrtc/media/base/videocapturer.h +++ b/webrtc/media/base/videocapturer.h @@ -28,6 +28,7 @@ #include "webrtc/media/base/videoadapter.h" #include "webrtc/media/base/videobroadcaster.h" #include "webrtc/media/base/videocommon.h" +#include "webrtc/media/base/videoframefactory.h" namespace cricket { @@ -44,6 +45,38 @@ enum CaptureState { class VideoFrame; +struct CapturedFrame { + static const uint32_t kFrameHeaderSize = 40; // Size from width to data_size. + static const uint32_t kUnknownDataSize = 0xFFFFFFFF; + + CapturedFrame(); + + // Get the number of bytes of the frame data. If data_size is known, return + // it directly. Otherwise, calculate the size based on width, height, and + // fourcc. Return true if succeeded. + bool GetDataSize(uint32_t* size) const; + + // The width and height of the captured frame could be different from those + // of VideoFormat. Once the first frame is captured, the width, height, + // fourcc, pixel_width, and pixel_height should keep the same over frames. + int width; // in number of pixels + int height; // in number of pixels + uint32_t fourcc; // compression + uint32_t pixel_width; // width of a pixel, default is 1 + uint32_t pixel_height; // height of a pixel, default is 1 + int64_t time_stamp; // timestamp of when the frame was captured, in unix + // time with nanosecond units. + uint32_t data_size; // number of bytes of the frame data + + webrtc::VideoRotation rotation; // rotation in degrees of the frame. + + void* data; // pointer to the frame data. This object allocates the + // memory or points to an existing memory. + + private: + RTC_DISALLOW_COPY_AND_ASSIGN(CapturedFrame); +}; + // VideoCapturer is an abstract class that defines the interfaces for video // capturing. The subclasses implement the video capturer for various types of // capturers and various platforms. @@ -171,6 +204,13 @@ class VideoCapturer : public sigslot::has_slots<>, // Signal all capture state changes that are not a direct result of calling // Start(). sigslot::signal2 SignalStateChange; + // Frame callbacks are multithreaded to allow disconnect and connect to be + // called concurrently. It also ensures that it is safe to call disconnect + // at any time which is needed since the signal may be called from an + // unmarshalled thread owned by the VideoCapturer. + // Signal the captured frame to downstream. + sigslot::signal2 SignalFrameCaptured; // If true, run video adaptation. By default, video adaptation is enabled // and users must call video_adapter()->OnOutputFormatRequest() @@ -180,6 +220,9 @@ class VideoCapturer : public sigslot::has_slots<>, enable_video_adapter_ = enable_video_adapter; } + // Takes ownership. + void set_frame_factory(VideoFrameFactory* frame_factory); + bool GetInputSize(int* width, int* height); // Implements VideoSourceInterface @@ -220,6 +263,10 @@ class VideoCapturer : public sigslot::has_slots<>, int* crop_y, int64_t* translated_camera_time_us); + // Callback attached to SignalFrameCaptured where SignalVideoFrames is called. + void OnFrameCaptured(VideoCapturer* video_capturer, + const CapturedFrame* captured_frame); + // Called when a frame has been captured and converted to a // VideoFrame. OnFrame can be called directly by an implementation // that does not use SignalFrameCaptured or OnFrameCaptured. The @@ -244,6 +291,7 @@ class VideoCapturer : public sigslot::has_slots<>, } void SetSupportedFormats(const std::vector& formats); + VideoFrameFactory* frame_factory() { return frame_factory_.get(); } private: void Construct(); @@ -253,6 +301,9 @@ class VideoCapturer : public sigslot::has_slots<>, int64_t GetFormatDistance(const VideoFormat& desired, const VideoFormat& supported); + // Convert captured frame to readable string for LOG messages. + std::string ToString(const CapturedFrame* frame) const; + // Updates filtered_supported_formats_ so that it contains the formats in // supported_formats_ that fulfill all applied restrictions. void UpdateFilteredSupportedFormats(); @@ -264,6 +315,7 @@ class VideoCapturer : public sigslot::has_slots<>, rtc::ThreadChecker thread_checker_; std::string id_; CaptureState capture_state_; + std::unique_ptr frame_factory_; std::unique_ptr capture_format_; std::vector supported_formats_; std::unique_ptr max_format_; @@ -283,8 +335,7 @@ class VideoCapturer : public sigslot::has_slots<>, int input_width_ GUARDED_BY(frame_stats_crit_); int input_height_ GUARDED_BY(frame_stats_crit_); - // Whether capturer should apply rotation to the frame before - // passing it on to the registered sinks. + // Whether capturer should apply rotation to the frame before signaling it. bool apply_rotation_; // State for the timestamp translation. diff --git a/webrtc/media/base/videocapturer_unittest.cc b/webrtc/media/base/videocapturer_unittest.cc index 821de5838c..25230b5118 100644 --- a/webrtc/media/base/videocapturer_unittest.cc +++ b/webrtc/media/base/videocapturer_unittest.cc @@ -91,14 +91,14 @@ TEST_F(VideoCapturerTest, ScreencastScaledOddWidth) { std::vector formats; formats.push_back(cricket::VideoFormat(kWidth, kHeight, - cricket::VideoFormat::FpsToInterval(5), - cricket::FOURCC_I420)); + cricket::VideoFormat::FpsToInterval(5), cricket::FOURCC_ARGB)); capturer_->ResetSupportedFormats(formats); - EXPECT_EQ(cricket::CS_RUNNING, - capturer_->Start(cricket::VideoFormat( - kWidth, kHeight, cricket::VideoFormat::FpsToInterval(30), - cricket::FOURCC_ANY))); + EXPECT_EQ(cricket::CS_RUNNING, capturer_->Start(cricket::VideoFormat( + kWidth, + kHeight, + cricket::VideoFormat::FpsToInterval(30), + cricket::FOURCC_ARGB))); EXPECT_TRUE(capturer_->IsRunning()); EXPECT_EQ(0, renderer_.num_rendered_frames()); EXPECT_TRUE(capturer_->CaptureFrame()); @@ -245,10 +245,6 @@ TEST_F(VideoCapturerTest, TestRotationAppliedBySourceWhenDifferentWants) { EXPECT_EQ(webrtc::kVideoRotation_0, renderer2.rotation()); } -// TODO(nisse): This test doesn't quite fit here. It tests two things: -// Aggregation of VideoSinkWants, which is the responsibility of -// VideoBroadcaster, and translation of VideoSinkWants to actual -// resolution, which is the responsibility of the VideoAdapter. TEST_F(VideoCapturerTest, SinkWantsMaxPixelAndMaxPixelCountStepUp) { EXPECT_EQ(cricket::CS_RUNNING, capturer_->Start(cricket::VideoFormat( diff --git a/webrtc/media/base/videoframefactory.cc b/webrtc/media/base/videoframefactory.cc new file mode 100644 index 0000000000..a4eeece148 --- /dev/null +++ b/webrtc/media/base/videoframefactory.cc @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/media/base/videoframefactory.h" + +#include +#include "webrtc/media/base/videocapturer.h" + +namespace cricket { + +VideoFrame* VideoFrameFactory::CreateAliasedFrame( + const CapturedFrame* input_frame, + int cropped_input_width, + int cropped_input_height, + int output_width, + int output_height) const { + std::unique_ptr cropped_input_frame(CreateAliasedFrame( + input_frame, cropped_input_width, cropped_input_height)); + if (!cropped_input_frame) + return nullptr; + + if (cropped_input_width == output_width && + cropped_input_height == output_height) { + // No scaling needed. + return cropped_input_frame.release(); + } + + // If the frame is rotated, we need to switch the width and height. + if (apply_rotation_ && + (input_frame->rotation == webrtc::kVideoRotation_90 || + input_frame->rotation == webrtc::kVideoRotation_270)) { + std::swap(output_width, output_height); + } + + rtc::scoped_refptr scaled_buffer( + pool_.CreateBuffer(output_width, output_height)); + scaled_buffer->CropAndScaleFrom(cropped_input_frame->video_frame_buffer()); + + return new WebRtcVideoFrame(scaled_buffer, cropped_input_frame->rotation(), + cropped_input_frame->timestamp_us(), + cropped_input_frame->transport_frame_id()); +} + +} // namespace cricket diff --git a/webrtc/media/base/videoframefactory.h b/webrtc/media/base/videoframefactory.h new file mode 100644 index 0000000000..985199f646 --- /dev/null +++ b/webrtc/media/base/videoframefactory.h @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MEDIA_BASE_VIDEOFRAMEFACTORY_H_ +#define WEBRTC_MEDIA_BASE_VIDEOFRAMEFACTORY_H_ + +#include + +#include "webrtc/common_video/include/i420_buffer_pool.h" +#include "webrtc/media/base/videoframe.h" + +namespace cricket { + +struct CapturedFrame; +class VideoFrame; + +// Creates cricket::VideoFrames, or a subclass of cricket::VideoFrame +// depending on the subclass of VideoFrameFactory. +class VideoFrameFactory { + public: + VideoFrameFactory() : apply_rotation_(false) {} + virtual ~VideoFrameFactory() {} + + // The returned frame aliases the aliased_frame if the input color + // space allows for aliasing, otherwise a color conversion will + // occur. Returns NULL if conversion fails. + + // The returned frame will be a center crop of |input_frame| with + // size |cropped_width| x |cropped_height|. + virtual VideoFrame* CreateAliasedFrame(const CapturedFrame* input_frame, + int cropped_width, + int cropped_height) const = 0; + + // The returned frame will be a center crop of |input_frame| with size + // |cropped_width| x |cropped_height|, scaled to |output_width| x + // |output_height|. + virtual VideoFrame* CreateAliasedFrame(const CapturedFrame* input_frame, + int cropped_input_width, + int cropped_input_height, + int output_width, + int output_height) const; + + void SetApplyRotation(bool enable) { apply_rotation_ = enable; } + + protected: + bool apply_rotation_; + + private: + // An internal pool to avoid reallocations. It is mutable because it + // does not affect behaviour, only performance. + mutable webrtc::I420BufferPool pool_; +}; + +} // namespace cricket + +#endif // WEBRTC_MEDIA_BASE_VIDEOFRAMEFACTORY_H_ diff --git a/webrtc/media/engine/webrtcvideocapturer.cc b/webrtc/media/engine/webrtcvideocapturer.cc index 2b1b62bedd..ba44b1324e 100644 --- a/webrtc/media/engine/webrtcvideocapturer.cc +++ b/webrtc/media/engine/webrtcvideocapturer.cc @@ -19,6 +19,7 @@ #include "webrtc/base/thread.h" #include "webrtc/base/timeutils.h" #include "webrtc/media/engine/webrtcvideoframe.h" +#include "webrtc/media/engine/webrtcvideoframefactory.h" #include "webrtc/base/win32.h" // Need this to #include the impl files. #include "webrtc/modules/video_capture/video_capture_factory.h" @@ -112,14 +113,18 @@ WebRtcVideoCapturer::WebRtcVideoCapturer() module_(nullptr), captured_frames_(0), start_thread_(nullptr), - async_invoker_(nullptr) {} + async_invoker_(nullptr) { + set_frame_factory(new WebRtcVideoFrameFactory()); +} WebRtcVideoCapturer::WebRtcVideoCapturer(WebRtcVcmFactoryInterface* factory) : factory_(factory), module_(nullptr), captured_frames_(0), start_thread_(nullptr), - async_invoker_(nullptr) {} + async_invoker_(nullptr) { + set_frame_factory(new WebRtcVideoFrameFactory()); +} WebRtcVideoCapturer::~WebRtcVideoCapturer() {} diff --git a/webrtc/media/engine/webrtcvideoengine2.h b/webrtc/media/engine/webrtcvideoengine2.h index e568c1e37d..b8d17e8105 100644 --- a/webrtc/media/engine/webrtcvideoengine2.h +++ b/webrtc/media/engine/webrtcvideoengine2.h @@ -60,6 +60,7 @@ class WebRtcVideoChannelSendInfo; class WebRtcVoiceEngine; class WebRtcVoiceMediaChannel; +struct CapturedFrame; struct Device; // Exposed here for unittests. diff --git a/webrtc/media/engine/webrtcvideoframe.cc b/webrtc/media/engine/webrtcvideoframe.cc index 2a9bbcc98f..7b5a68064f 100644 --- a/webrtc/media/engine/webrtcvideoframe.cc +++ b/webrtc/media/engine/webrtcvideoframe.cc @@ -66,6 +66,14 @@ bool WebRtcVideoFrame::Init(uint32_t format, true /*apply_rotation*/); } +bool WebRtcVideoFrame::Init(const CapturedFrame* frame, int dw, int dh, + bool apply_rotation) { + return Reset(frame->fourcc, frame->width, frame->height, dw, dh, + static_cast(frame->data), frame->data_size, + frame->time_stamp / rtc::kNumNanosecsPerMicrosec, + frame->rotation, apply_rotation); +} + int WebRtcVideoFrame::width() const { return video_frame_buffer_ ? video_frame_buffer_->width() : 0; } diff --git a/webrtc/media/engine/webrtcvideoframe.h b/webrtc/media/engine/webrtcvideoframe.h index 79ad571236..d0034e27cd 100644 --- a/webrtc/media/engine/webrtcvideoframe.h +++ b/webrtc/media/engine/webrtcvideoframe.h @@ -22,6 +22,8 @@ namespace cricket { +struct CapturedFrame; + // TODO(nisse): This class will be deleted when the cricket::VideoFrame and // webrtc::VideoFrame classes are merged. See // https://bugs.chromium.org/p/webrtc/issues/detail?id=5682. Try to use only the @@ -53,10 +55,6 @@ class WebRtcVideoFrame : public VideoFrame { ~WebRtcVideoFrame(); - // TODO(nisse): Init (and its helpers Reset and Validate) are used - // only by the LoadFrame function used in the VideoFrame unittests. - // Rewrite tests, and delete this function. - // Creates a frame from a raw sample with FourCC "format" and size "w" x "h". // "h" can be negative indicating a vertically flipped image. // "dh" is destination height if cropping is desired and is always positive. @@ -71,6 +69,15 @@ class WebRtcVideoFrame : public VideoFrame { int64_t timestamp_ns, webrtc::VideoRotation rotation); + // TODO(nisse): We're moving to have all timestamps use the same + // time scale as rtc::TimeMicros. However, this method is used by + // WebRtcVideoFrameFactory::CreateAliasedFrame this code path + // currently does not conform to the new timestamp conventions and + // may use the camera's own clock instead. It's unclear if this + // should be fixed, or if instead all of the VideoFrameFactory + // abstraction should be eliminated. + bool Init(const CapturedFrame* frame, int dw, int dh, bool apply_rotation); + void InitToEmptyBuffer(int w, int h); int width() const override; diff --git a/webrtc/media/engine/webrtcvideoframe_unittest.cc b/webrtc/media/engine/webrtcvideoframe_unittest.cc index 2385e44ab2..3743e87899 100644 --- a/webrtc/media/engine/webrtcvideoframe_unittest.cc +++ b/webrtc/media/engine/webrtcvideoframe_unittest.cc @@ -22,6 +22,52 @@ class WebRtcVideoFrameTest : public VideoFrameTest { public: WebRtcVideoFrameTest() {} + void TestInit(int cropped_width, int cropped_height, + webrtc::VideoRotation frame_rotation, + bool apply_rotation) { + const int frame_width = 1920; + const int frame_height = 1080; + + // Build the CapturedFrame. + CapturedFrame captured_frame; + captured_frame.fourcc = FOURCC_I420; + captured_frame.time_stamp = rtc::TimeNanos(); + captured_frame.rotation = frame_rotation; + captured_frame.width = frame_width; + captured_frame.height = frame_height; + captured_frame.data_size = (frame_width * frame_height) + + ((frame_width + 1) / 2) * ((frame_height + 1) / 2) * 2; + std::unique_ptr captured_frame_buffer( + new uint8_t[captured_frame.data_size]); + // Initialize memory to satisfy DrMemory tests. + memset(captured_frame_buffer.get(), 0, captured_frame.data_size); + captured_frame.data = captured_frame_buffer.get(); + + // Create the new frame from the CapturedFrame. + WebRtcVideoFrame frame; + EXPECT_TRUE( + frame.Init(&captured_frame, cropped_width, cropped_height, + apply_rotation)); + + // Verify the new frame. + EXPECT_EQ(captured_frame.time_stamp / rtc::kNumNanosecsPerMicrosec, + frame.timestamp_us()); + if (apply_rotation) + EXPECT_EQ(webrtc::kVideoRotation_0, frame.rotation()); + else + EXPECT_EQ(frame_rotation, frame.rotation()); + // If |apply_rotation| and the frame rotation is 90 or 270, width and + // height are flipped. + if (apply_rotation && (frame_rotation == webrtc::kVideoRotation_90 + || frame_rotation == webrtc::kVideoRotation_270)) { + EXPECT_EQ(cropped_width, frame.height()); + EXPECT_EQ(cropped_height, frame.width()); + } else { + EXPECT_EQ(cropped_width, frame.width()); + EXPECT_EQ(cropped_height, frame.height()); + } + } + void SetFrameRotation(WebRtcVideoFrame* frame, webrtc::VideoRotation rotation) { frame->rotation_ = rotation; @@ -108,6 +154,32 @@ TEST_WEBRTCVIDEOFRAME(ValidateI420HugeSize) // TEST_WEBRTCVIDEOFRAME(ConvertToI422Buffer) // TEST_WEBRTCVIDEOFRAME(ConstructARGBBlackWhitePixel) +// These functions test implementation-specific details. +// Tests the Init function with different cropped size. +TEST_F(WebRtcVideoFrameTest, InitEvenSize) { + TestInit(640, 360, webrtc::kVideoRotation_0, true); +} + +TEST_F(WebRtcVideoFrameTest, InitOddWidth) { + TestInit(601, 480, webrtc::kVideoRotation_0, true); +} + +TEST_F(WebRtcVideoFrameTest, InitOddHeight) { + TestInit(360, 765, webrtc::kVideoRotation_0, true); +} + +TEST_F(WebRtcVideoFrameTest, InitOddWidthHeight) { + TestInit(355, 1021, webrtc::kVideoRotation_0, true); +} + +TEST_F(WebRtcVideoFrameTest, InitRotated90ApplyRotation) { + TestInit(640, 360, webrtc::kVideoRotation_90, true); +} + +TEST_F(WebRtcVideoFrameTest, InitRotated90DontApplyRotation) { + TestInit(640, 360, webrtc::kVideoRotation_90, false); +} + TEST_F(WebRtcVideoFrameTest, TextureInitialValues) { webrtc::test::FakeNativeHandle* dummy_handle = new webrtc::test::FakeNativeHandle(); diff --git a/webrtc/media/engine/webrtcvideoframefactory.cc b/webrtc/media/engine/webrtcvideoframefactory.cc new file mode 100644 index 0000000000..2c08c63129 --- /dev/null +++ b/webrtc/media/engine/webrtcvideoframefactory.cc @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "webrtc/base/logging.h" +#include "webrtc/media/engine/webrtcvideoframe.h" +#include "webrtc/media/engine/webrtcvideoframefactory.h" + +namespace cricket { + +VideoFrame* WebRtcVideoFrameFactory::CreateAliasedFrame( + const CapturedFrame* aliased_frame, int width, int height) const { + std::unique_ptr frame(new WebRtcVideoFrame()); + if (!frame->Init(aliased_frame, width, height, apply_rotation_)) { + LOG(LS_ERROR) << + "Failed to create WebRtcVideoFrame in CreateAliasedFrame."; + return NULL; + } + return frame.release(); +} + +} // namespace cricket diff --git a/webrtc/media/engine/webrtcvideoframefactory.h b/webrtc/media/engine/webrtcvideoframefactory.h new file mode 100644 index 0000000000..39d813ff75 --- /dev/null +++ b/webrtc/media/engine/webrtcvideoframefactory.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MEDIA_ENGINE_WEBRTCVIDEOFRAMEFACTORY_H_ +#define WEBRTC_MEDIA_ENGINE_WEBRTCVIDEOFRAMEFACTORY_H_ + +#include "webrtc/media/base/videoframefactory.h" + +namespace cricket { + +struct CapturedFrame; + +// Creates instances of cricket::WebRtcVideoFrame. +class WebRtcVideoFrameFactory : public VideoFrameFactory { + public: + // Note: Overriding a method name overrides all overloaded versions. + // Without this using-declaration, we would hide the 5-argument + // method we want to inherit. + using VideoFrameFactory::CreateAliasedFrame; + + VideoFrame* CreateAliasedFrame(const CapturedFrame* aliased_frame, + int width, + int height) const override; +}; + +} // namespace cricket + +#endif // WEBRTC_MEDIA_ENGINE_WEBRTCVIDEOFRAMEFACTORY_H_ diff --git a/webrtc/media/engine/webrtcvideoframefactory_unittest.cc b/webrtc/media/engine/webrtcvideoframefactory_unittest.cc new file mode 100644 index 0000000000..197784b54a --- /dev/null +++ b/webrtc/media/engine/webrtcvideoframefactory_unittest.cc @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include + +#include "webrtc/base/gunit.h" +#include "webrtc/media/base/videocapturer.h" +#include "webrtc/media/engine/webrtcvideoframe.h" +#include "webrtc/media/engine/webrtcvideoframefactory.h" + +class WebRtcVideoFrameFactoryTest : public testing::Test { + public: + WebRtcVideoFrameFactoryTest() {} + + void InitFrame(webrtc::VideoRotation frame_rotation) { + const int frame_width = 1920; + const int frame_height = 1080; + + // Build the CapturedFrame. + captured_frame_.fourcc = cricket::FOURCC_I420; + captured_frame_.pixel_width = 1; + captured_frame_.pixel_height = 1; + captured_frame_.time_stamp = rtc::TimeNanos(); + captured_frame_.rotation = frame_rotation; + captured_frame_.width = frame_width; + captured_frame_.height = frame_height; + captured_frame_.data_size = + (frame_width * frame_height) + + ((frame_width + 1) / 2) * ((frame_height + 1) / 2) * 2; + captured_frame_buffer_.reset(new uint8_t[captured_frame_.data_size]); + // Initialize memory to satisfy DrMemory tests. + memset(captured_frame_buffer_.get(), 0, captured_frame_.data_size); + captured_frame_.data = captured_frame_buffer_.get(); + } + + void VerifyFrame(cricket::VideoFrame* dest_frame, + webrtc::VideoRotation src_rotation, + int src_width, + int src_height, + bool apply_rotation) { + if (!apply_rotation) { + EXPECT_EQ(dest_frame->rotation(), src_rotation); + EXPECT_EQ(dest_frame->width(), src_width); + EXPECT_EQ(dest_frame->height(), src_height); + } else { + EXPECT_EQ(dest_frame->rotation(), webrtc::kVideoRotation_0); + if (src_rotation == webrtc::kVideoRotation_90 || + src_rotation == webrtc::kVideoRotation_270) { + EXPECT_EQ(dest_frame->width(), src_height); + EXPECT_EQ(dest_frame->height(), src_width); + } else { + EXPECT_EQ(dest_frame->width(), src_width); + EXPECT_EQ(dest_frame->height(), src_height); + } + } + } + + void TestCreateAliasedFrame(bool apply_rotation) { + cricket::VideoFrameFactory& factory = factory_; + factory.SetApplyRotation(apply_rotation); + InitFrame(webrtc::kVideoRotation_270); + const cricket::CapturedFrame& captured_frame = get_captured_frame(); + // Create the new frame from the CapturedFrame. + std::unique_ptr frame; + int new_width = captured_frame.width / 2; + int new_height = captured_frame.height / 2; + frame.reset(factory.CreateAliasedFrame(&captured_frame, new_width, + new_height, new_width, new_height)); + VerifyFrame(frame.get(), webrtc::kVideoRotation_270, new_width, new_height, + apply_rotation); + + frame.reset(factory.CreateAliasedFrame( + &captured_frame, new_width, new_height, new_width / 2, new_height / 2)); + VerifyFrame(frame.get(), webrtc::kVideoRotation_270, new_width / 2, + new_height / 2, apply_rotation); + + // Reset the frame first so it's exclusive hence we could go through the + // StretchToFrame code path in CreateAliasedFrame. + frame.reset(); + frame.reset(factory.CreateAliasedFrame( + &captured_frame, new_width, new_height, new_width / 2, new_height / 2)); + VerifyFrame(frame.get(), webrtc::kVideoRotation_270, new_width / 2, + new_height / 2, apply_rotation); + } + + const cricket::CapturedFrame& get_captured_frame() { return captured_frame_; } + + private: + cricket::CapturedFrame captured_frame_; + std::unique_ptr captured_frame_buffer_; + cricket::WebRtcVideoFrameFactory factory_; +}; + +TEST_F(WebRtcVideoFrameFactoryTest, NoApplyRotation) { + TestCreateAliasedFrame(false); +} + +TEST_F(WebRtcVideoFrameFactoryTest, ApplyRotation) { + TestCreateAliasedFrame(true); +} diff --git a/webrtc/media/media.gyp b/webrtc/media/media.gyp index 9528441c32..9a9fbbb831 100644 --- a/webrtc/media/media.gyp +++ b/webrtc/media/media.gyp @@ -63,6 +63,8 @@ 'base/videocommon.h', 'base/videoframe.cc', 'base/videoframe.h', + 'base/videoframefactory.cc', + 'base/videoframefactory.h', 'base/videosourcebase.cc', 'base/videosourcebase.h', 'devices/videorendererfactory.h', @@ -85,6 +87,8 @@ 'engine/webrtcvideoengine2.h', 'engine/webrtcvideoframe.cc', 'engine/webrtcvideoframe.h', + 'engine/webrtcvideoframefactory.cc', + 'engine/webrtcvideoframefactory.h', 'engine/webrtcvoe.h', 'engine/webrtcvoiceengine.cc', 'engine/webrtcvoiceengine.h',