diff --git a/webrtc/api/java/jni/androidmediaencoder_jni.cc b/webrtc/api/java/jni/androidmediaencoder_jni.cc index a240b25429..b76cfe7155 100644 --- a/webrtc/api/java/jni/androidmediaencoder_jni.cc +++ b/webrtc/api/java/jni/androidmediaencoder_jni.cc @@ -670,7 +670,7 @@ int32_t MediaCodecVideoEncoder::EncodeOnCodecThread( VideoFrame input_frame = frame; if (scale_) { // Check framerate before spatial resolution change. - quality_scaler_.OnEncodeFrame(frame); + quality_scaler_.OnEncodeFrame(frame.width(), frame.height()); const webrtc::QualityScaler::Resolution scaled_resolution = quality_scaler_.GetScaledResolution(); if (scaled_resolution.width != frame.width() || @@ -684,7 +684,8 @@ int32_t MediaCodecVideoEncoder::EncodeOnCodecThread( webrtc::kVideoRotation_0)); input_frame.set_video_frame_buffer(scaled_buffer); } else { - input_frame = quality_scaler_.GetScaledFrame(frame); + input_frame.set_video_frame_buffer( + quality_scaler_.GetScaledBuffer(frame.video_frame_buffer())); } } } diff --git a/webrtc/api/java/jni/androidvideocapturer_jni.cc b/webrtc/api/java/jni/androidvideocapturer_jni.cc index 82d8c8e194..4f3d64b97a 100644 --- a/webrtc/api/java/jni/androidvideocapturer_jni.cc +++ b/webrtc/api/java/jni/androidvideocapturer_jni.cc @@ -13,7 +13,6 @@ #include "webrtc/api/java/jni/native_handle_impl.h" #include "webrtc/api/java/jni/surfacetexturehelper_jni.h" #include "third_party/libyuv/include/libyuv/convert.h" -#include "third_party/libyuv/include/libyuv/scale.h" #include "webrtc/base/bind.h" namespace webrtc_jni { @@ -223,25 +222,11 @@ void AndroidVideoCapturerJni::OnMemoryBufferFrame(void* video_frame, crop_width, crop_height, static_cast( capturer_->apply_rotation() ? rotation : 0)); - if (adapted_width != rotated_width || adapted_height != rotated_height) { - rtc::scoped_refptr scaled = - post_scale_pool_.CreateBuffer(adapted_width, adapted_height); - // TODO(nisse): This should be done by some Scale method in - // I420Buffer, but we can't do that right now, since - // I420BufferPool uses a wrapper object. - if (libyuv::I420Scale(buffer->DataY(), buffer->StrideY(), - buffer->DataU(), buffer->StrideU(), - buffer->DataV(), buffer->StrideV(), - rotated_width, rotated_height, - scaled->MutableDataY(), scaled->StrideY(), - scaled->MutableDataU(), scaled->StrideU(), - scaled->MutableDataV(), scaled->StrideV(), - adapted_width, adapted_height, - libyuv::kFilterBox) < 0) { - LOG(LS_WARNING) << "I420Scale failed"; - return; - } - buffer = scaled; + if (adapted_width != buffer->width() || adapted_height != buffer->height()) { + rtc::scoped_refptr scaled_buffer( + post_scale_pool_.CreateBuffer(adapted_width, adapted_height)); + scaled_buffer->ScaleFrom(buffer); + buffer = scaled_buffer; } // TODO(nisse): Use microsecond time instead. capturer_->OnFrame(cricket::WebRtcVideoFrame( diff --git a/webrtc/common_video/BUILD.gn b/webrtc/common_video/BUILD.gn index cb3e3904ed..d8e6bbfe0f 100644 --- a/webrtc/common_video/BUILD.gn +++ b/webrtc/common_video/BUILD.gn @@ -34,9 +34,7 @@ source_set("common_video") { "include/incoming_video_stream.h", "include/video_frame_buffer.h", "incoming_video_stream.cc", - "libyuv/include/scaler.h", "libyuv/include/webrtc_libyuv.h", - "libyuv/scaler.cc", "libyuv/webrtc_libyuv.cc", "video_frame.cc", "video_frame_buffer.cc", @@ -94,7 +92,6 @@ if (rtc_include_tests) { "i420_buffer_pool_unittest.cc", "i420_video_frame_unittest.cc", "libyuv/libyuv_unittest.cc", - "libyuv/scaler_unittest.cc", ] configs += [ "..:common_config" ] diff --git a/webrtc/common_video/common_video.gyp b/webrtc/common_video/common_video.gyp index e3b6fa4caa..b1f460418a 100644 --- a/webrtc/common_video/common_video.gyp +++ b/webrtc/common_video/common_video.gyp @@ -69,9 +69,7 @@ 'include/i420_buffer_pool.h', 'include/incoming_video_stream.h', 'include/video_frame_buffer.h', - 'libyuv/include/scaler.h', 'libyuv/include/webrtc_libyuv.h', - 'libyuv/scaler.cc', 'libyuv/webrtc_libyuv.cc', 'video_frame_buffer.cc', 'video_render_frames.cc', diff --git a/webrtc/common_video/common_video_unittests.gyp b/webrtc/common_video/common_video_unittests.gyp index cb6102fe7c..fb7a7431a0 100644 --- a/webrtc/common_video/common_video_unittests.gyp +++ b/webrtc/common_video/common_video_unittests.gyp @@ -27,7 +27,6 @@ 'i420_buffer_pool_unittest.cc', 'i420_video_frame_unittest.cc', 'libyuv/libyuv_unittest.cc', - 'libyuv/scaler_unittest.cc', ], # Disable warnings to enable Win64 build, issue 1323. 'msvs_disabled_warnings': [ diff --git a/webrtc/common_video/i420_video_frame_unittest.cc b/webrtc/common_video/i420_video_frame_unittest.cc index 5144c2715f..c2037c1b63 100644 --- a/webrtc/common_video/i420_video_frame_unittest.cc +++ b/webrtc/common_video/i420_video_frame_unittest.cc @@ -19,7 +19,70 @@ namespace webrtc { -int ExpectedSize(int plane_stride, int image_height, PlaneType type); +namespace { + +int ExpectedSize(int plane_stride, int image_height, PlaneType type) { + if (type == kYPlane) + return plane_stride * image_height; + return plane_stride * ((image_height + 1) / 2); +} + +rtc::scoped_refptr CreateGradient(int width, int height) { + rtc::scoped_refptr buffer( + new rtc::RefCountedObject(width, height)); + // Initialize with gradient, Y = 128(x/w + y/h), U = 256 x/w, V = 256 y/h + for (int x = 0; x < width; x++) { + for (int y = 0; y < height; y++) { + buffer->MutableDataY()[x + y * width] = + 128 * (x * height + y * width) / (width * height); + } + } + int chroma_width = (width + 1) / 2; + int chroma_height = (height + 1) / 2; + for (int x = 0; x < chroma_width; x++) { + for (int y = 0; y < chroma_height; y++) { + buffer->MutableDataU()[x + y * chroma_width] = + 255 * x / (chroma_width - 1); + buffer->MutableDataV()[x + y * chroma_width] = + 255 * y / (chroma_height - 1); + } + } + return buffer; +} + +// The offsets and sizes describe the rectangle extracted from the +// original (gradient) frame, in relative coordinates where the +// original frame correspond to the unit square, 0.0 <= x, y < 1.0. +void CheckCrop(webrtc::VideoFrameBuffer* frame, + double offset_x, + double offset_y, + double rel_width, + double rel_height) { + int width = frame->width(); + int height = frame->height(); + // Check that pixel values in the corners match the gradient used + // for initialization. + for (int i = 0; i < 2; i++) { + for (int j = 0; j < 2; j++) { + // Pixel coordinates of the corner. + int x = i * (width - 1); + int y = j * (height - 1); + // Relative coordinates, range 0.0 - 1.0 correspond to the + // size of the uncropped input frame. + double orig_x = offset_x + i * rel_width; + double orig_y = offset_y + j * rel_height; + + EXPECT_NEAR(frame->DataY()[x + y * frame->StrideY()] / 256.0, + (orig_x + orig_y) / 2, 0.02); + EXPECT_NEAR(frame->DataU()[x / 2 + (y / 2) * frame->StrideU()] / 256.0, + orig_x, 0.02); + EXPECT_NEAR(frame->DataV()[x / 2 + (y / 2) * frame->StrideV()] / 256.0, + orig_y, 0.02); + } + } +} + +} // namespace TEST(TestVideoFrame, InitialValues) { VideoFrame frame; @@ -241,4 +304,70 @@ TEST(TestI420FrameBuffer, Copy) { EXPECT_TRUE(test::FrameBufsEqual(buf1, buf2)); } +TEST(TestI420FrameBuffer, Scale) { + rtc::scoped_refptr buf = CreateGradient(200, 100); + + // Pure scaling, no cropping. + rtc::scoped_refptr scaled_buffer( + new rtc::RefCountedObject(150, 75)); + + scaled_buffer->ScaleFrom(buf); + CheckCrop(scaled_buffer, 0.0, 0.0, 1.0, 1.0); +} + +TEST(TestI420FrameBuffer, CropXCenter) { + rtc::scoped_refptr buf = CreateGradient(200, 100); + + // Pure center cropping, no scaling. + rtc::scoped_refptr scaled_buffer( + new rtc::RefCountedObject(100, 100)); + + scaled_buffer->CropAndScaleFrom(buf, 50, 0, 100, 100); + CheckCrop(scaled_buffer, 0.25, 0.0, 0.5, 1.0); +} + +TEST(TestI420FrameBuffer, CropXNotCenter) { + rtc::scoped_refptr buf = CreateGradient(200, 100); + + // Non-center cropping, no scaling. + rtc::scoped_refptr scaled_buffer( + new rtc::RefCountedObject(100, 100)); + + scaled_buffer->CropAndScaleFrom(buf, 25, 0, 100, 100); + CheckCrop(scaled_buffer, 0.125, 0.0, 0.5, 1.0); +} + +TEST(TestI420FrameBuffer, CropYCenter) { + rtc::scoped_refptr buf = CreateGradient(100, 200); + + // Pure center cropping, no scaling. + rtc::scoped_refptr scaled_buffer( + new rtc::RefCountedObject(100, 100)); + + scaled_buffer->CropAndScaleFrom(buf, 0, 50, 100, 100); + CheckCrop(scaled_buffer, 0.0, 0.25, 1.0, 0.5); +} + +TEST(TestI420FrameBuffer, CropYNotCenter) { + rtc::scoped_refptr buf = CreateGradient(100, 200); + + // Non-center cropping, no scaling. + rtc::scoped_refptr scaled_buffer( + new rtc::RefCountedObject(100, 100)); + + scaled_buffer->CropAndScaleFrom(buf, 0, 25, 100, 100); + CheckCrop(scaled_buffer, 0.0, 0.125, 1.0, 0.5); +} + +TEST(TestI420FrameBuffer, CropAndScale16x9) { + rtc::scoped_refptr buf = CreateGradient(640, 480); + + // Center crop to 640 x 360 (16/9 aspect), then scale down by 2. + rtc::scoped_refptr scaled_buffer( + new rtc::RefCountedObject(320, 180)); + + scaled_buffer->CropAndScaleFrom(buf); + CheckCrop(scaled_buffer, 0.0, 0.125, 1.0, 0.75); +} + } // namespace webrtc diff --git a/webrtc/common_video/include/video_frame_buffer.h b/webrtc/common_video/include/video_frame_buffer.h index 890922f21b..dc51164450 100644 --- a/webrtc/common_video/include/video_frame_buffer.h +++ b/webrtc/common_video/include/video_frame_buffer.h @@ -119,6 +119,21 @@ class I420Buffer : public VideoFrameBuffer { static rtc::scoped_refptr Copy( const rtc::scoped_refptr& buffer); + // Scale the cropped area of |src| to the size of |this| buffer, and + // write the result into |this|. + void CropAndScaleFrom(const rtc::scoped_refptr& src, + int offset_x, + int offset_y, + int crop_width, + int crop_height); + + // The common case of a center crop, when needed to adjust the + // aspect ratio without distorting the image. + void CropAndScaleFrom(const rtc::scoped_refptr& src); + + // Scale all of |src| to the size of |this| buffer, with no cropping. + void ScaleFrom(const rtc::scoped_refptr& src); + protected: ~I420Buffer() override; @@ -196,13 +211,6 @@ class WrappedI420Buffer : public webrtc::VideoFrameBuffer { rtc::Callback0 no_longer_used_cb_; }; -// Helper function to crop |buffer| without making a deep copy. May only be used -// for non-native frames. -rtc::scoped_refptr ShallowCenterCrop( - const rtc::scoped_refptr& buffer, - int cropped_width, - int cropped_height); - } // namespace webrtc #endif // WEBRTC_COMMON_VIDEO_INCLUDE_VIDEO_FRAME_BUFFER_H_ diff --git a/webrtc/common_video/libyuv/include/scaler.h b/webrtc/common_video/libyuv/include/scaler.h deleted file mode 100644 index 6e3e05da0f..0000000000 --- a/webrtc/common_video/libyuv/include/scaler.h +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -/* - * Interface to the LibYuv scaling functionality - */ - -#ifndef WEBRTC_COMMON_VIDEO_LIBYUV_INCLUDE_SCALER_H_ -#define WEBRTC_COMMON_VIDEO_LIBYUV_INCLUDE_SCALER_H_ - -#include "webrtc/common_video/include/i420_buffer_pool.h" -#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" -#include "webrtc/typedefs.h" -#include "webrtc/video_frame.h" - -namespace webrtc { - -// Supported scaling types -// Note: Must have the same values as libyuv::FilterMode. -enum ScaleMethod { - kScalePoint, // no interpolation - kFilterLinear, - kScaleBilinear, - kScaleBox -}; - -class Scaler { - public: - Scaler(); - ~Scaler(); - - // Set interpolation properties: - // - // Return value: 0 - OK - // -1 - parameter error - int Set(int src_width, int src_height, - int dst_width, int dst_height, - VideoType src_video_type, VideoType dst_video_type, - ScaleMethod method); - - // Scale frame - // Memory is allocated by this object and recycled using |buffer_pool_|. - // Return value: 0 - OK, - // -1 - parameter error - // -2 - scaler not set - int Scale(const VideoFrame& src_frame, VideoFrame* dst_frame); - - private: - // Determine if the VideoTypes are currently supported. - bool SupportedVideoType(VideoType src_video_type, - VideoType dst_video_type); - - ScaleMethod method_; - int src_width_; - int src_height_; - int dst_width_; - int dst_height_; - bool set_; - I420BufferPool buffer_pool_; -}; - -} // namespace webrtc - -#endif // WEBRTC_COMMON_VIDEO_LIBYUV_INCLUDE_SCALER_H_ diff --git a/webrtc/common_video/libyuv/include/webrtc_libyuv.h b/webrtc/common_video/libyuv/include/webrtc_libyuv.h index ad417ea0e9..ec3720e442 100644 --- a/webrtc/common_video/libyuv/include/webrtc_libyuv.h +++ b/webrtc/common_video/libyuv/include/webrtc_libyuv.h @@ -69,13 +69,17 @@ size_t CalcBufferSize(VideoType type, int width, int height); // Return value: 0 if OK, < 0 otherwise. int PrintVideoFrame(const VideoFrame& frame, FILE* file); -// Extract buffer from VideoFrame (consecutive planes, no stride) +// Extract buffer from VideoFrame or VideoFrameBuffer (consecutive +// planes, no stride) // Input: // - frame : Reference to video frame. // - size : pointer to the size of the allocated buffer. If size is // insufficient, an error will be returned. // - buffer : Pointer to buffer // Return value: length of buffer if OK, < 0 otherwise. +int ExtractBuffer(const rtc::scoped_refptr& input_frame, + size_t size, + uint8_t* buffer); int ExtractBuffer(const VideoFrame& input_frame, size_t size, uint8_t* buffer); // Convert To I420 // Input: diff --git a/webrtc/common_video/libyuv/scaler.cc b/webrtc/common_video/libyuv/scaler.cc deleted file mode 100644 index 6e683c003b..0000000000 --- a/webrtc/common_video/libyuv/scaler.cc +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "webrtc/common_video/libyuv/include/scaler.h" - -#include - -// NOTE(ajm): Path provided by gyp. -#include "libyuv.h" // NOLINT - -namespace webrtc { - -Scaler::Scaler() - : method_(kScaleBox), - src_width_(0), - src_height_(0), - dst_width_(0), - dst_height_(0), - set_(false) {} - -Scaler::~Scaler() {} - -int Scaler::Set(int src_width, int src_height, - int dst_width, int dst_height, - VideoType src_video_type, VideoType dst_video_type, - ScaleMethod method) { - set_ = false; - if (src_width < 1 || src_height < 1 || dst_width < 1 || dst_height < 1) - return -1; - - if (!SupportedVideoType(src_video_type, dst_video_type)) - return -1; - - src_width_ = src_width; - src_height_ = src_height; - dst_width_ = dst_width; - dst_height_ = dst_height; - method_ = method; - set_ = true; - return 0; -} - -// TODO(nisse): Should work with VideoFrameBuffer instead. -int Scaler::Scale(const VideoFrame& src_frame, VideoFrame* dst_frame) { - assert(dst_frame); - if (src_frame.IsZeroSize()) - return -1; - if (!set_) - return -2; - - // Making sure that destination frame is of sufficient size. - dst_frame->set_video_frame_buffer( - buffer_pool_.CreateBuffer(dst_width_, dst_height_)); - - // We want to preserve aspect ratio instead of stretching the frame. - // Therefore, we need to crop the source frame. Calculate the largest center - // aligned region of the source frame that can be used. - const int cropped_src_width = - std::min(src_width_, dst_width_ * src_height_ / dst_height_); - const int cropped_src_height = - std::min(src_height_, dst_height_ * src_width_ / dst_width_); - // Make sure the offsets are even to avoid rounding errors for the U/V planes. - const int src_offset_x = ((src_width_ - cropped_src_width) / 2) & ~1; - const int src_offset_y = ((src_height_ - cropped_src_height) / 2) & ~1; - - const uint8_t* y_ptr = - src_frame.video_frame_buffer()->DataY() + - src_offset_y * src_frame.video_frame_buffer()->StrideY() + - src_offset_x; - const uint8_t* u_ptr = - src_frame.video_frame_buffer()->DataU() + - src_offset_y / 2 * src_frame.video_frame_buffer()->StrideU() + - src_offset_x / 2; - const uint8_t* v_ptr = - src_frame.video_frame_buffer()->DataV() + - src_offset_y / 2 * src_frame.video_frame_buffer()->StrideV() + - src_offset_x / 2; - - return libyuv::I420Scale( - y_ptr, - src_frame.video_frame_buffer()->StrideY(), - u_ptr, - src_frame.video_frame_buffer()->StrideU(), - v_ptr, - src_frame.video_frame_buffer()->StrideV(), - cropped_src_width, cropped_src_height, - dst_frame->video_frame_buffer()->MutableDataY(), - dst_frame->video_frame_buffer()->StrideY(), - dst_frame->video_frame_buffer()->MutableDataU(), - dst_frame->video_frame_buffer()->StrideU(), - dst_frame->video_frame_buffer()->MutableDataV(), - dst_frame->video_frame_buffer()->StrideV(), - dst_width_, dst_height_, - libyuv::FilterMode(method_)); -} - -bool Scaler::SupportedVideoType(VideoType src_video_type, - VideoType dst_video_type) { - if (src_video_type != dst_video_type) - return false; - - if ((src_video_type == kI420) || (src_video_type == kIYUV) || - (src_video_type == kYV12)) - return true; - - return false; -} - -} // namespace webrtc diff --git a/webrtc/common_video/libyuv/scaler_unittest.cc b/webrtc/common_video/libyuv/scaler_unittest.cc deleted file mode 100644 index 29a0a7403d..0000000000 --- a/webrtc/common_video/libyuv/scaler_unittest.cc +++ /dev/null @@ -1,399 +0,0 @@ -/* - * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include -#include - -#include - -#include "testing/gtest/include/gtest/gtest.h" -#include "webrtc/base/timeutils.h" -#include "webrtc/common_video/libyuv/include/scaler.h" -#include "webrtc/test/testsupport/fileutils.h" - -namespace webrtc { - -class TestScaler : public ::testing::Test { - protected: - TestScaler(); - virtual void SetUp(); - virtual void TearDown(); - - void ScaleSequence(ScaleMethod method, - FILE* source_file, std::string out_name, - int src_width, int src_height, - int dst_width, int dst_height); - // Computes the sequence average PSNR between an input sequence in - // |input_file| and an output sequence with filename |out_name|. |width| and - // |height| are the frame sizes of both sequences. - double ComputeAvgSequencePSNR(FILE* input_file, std::string out_name, - int width, int height); - - Scaler test_scaler_; - FILE* source_file_; - VideoFrame test_frame_; - const int width_; - const int half_width_; - const int height_; - const int half_height_; - const int size_y_; - const int size_uv_; - const size_t frame_length_; -}; - -TestScaler::TestScaler() - : source_file_(NULL), - width_(352), - half_width_(width_ / 2), - height_(288), - half_height_(height_ / 2), - size_y_(width_ * height_), - size_uv_(half_width_ * half_height_), - frame_length_(CalcBufferSize(kI420, width_, height_)) { -} - -void TestScaler::SetUp() { - const std::string input_file_name = - webrtc::test::ResourcePath("foreman_cif", "yuv"); - source_file_ = fopen(input_file_name.c_str(), "rb"); - ASSERT_TRUE(source_file_ != NULL) << "Cannot read file: "<< - input_file_name << "\n"; - test_frame_.CreateEmptyFrame(width_, height_, - width_, half_width_, half_width_); -} - -void TestScaler::TearDown() { - if (source_file_ != NULL) { - ASSERT_EQ(0, fclose(source_file_)); - } - source_file_ = NULL; -} - -TEST_F(TestScaler, ScaleWithoutSettingValues) { - EXPECT_EQ(-2, test_scaler_.Scale(test_frame_, &test_frame_)); -} - -TEST_F(TestScaler, ScaleBadInitialValues) { - EXPECT_EQ(-1, test_scaler_.Set(0, 288, 352, 288, kI420, kI420, kScalePoint)); - EXPECT_EQ(-1, test_scaler_.Set(704, 0, 352, 288, kI420, kI420, kScaleBox)); - EXPECT_EQ(-1, test_scaler_.Set(704, 576, 352, 0, kI420, kI420, - kScaleBilinear)); - EXPECT_EQ(-1, test_scaler_.Set(704, 576, 0, 288, kI420, kI420, kScalePoint)); -} - -TEST_F(TestScaler, ScaleSendingNullSourcePointer) { - VideoFrame null_src_frame; - EXPECT_EQ(-1, test_scaler_.Scale(null_src_frame, &test_frame_)); -} - -TEST_F(TestScaler, ScaleSendingBufferTooSmall) { - // Sending a buffer which is too small (should reallocate and update size) - EXPECT_EQ(0, test_scaler_.Set(width_, height_, - half_width_, half_height_, - kI420, kI420, - kScalePoint)); - VideoFrame test_frame2; - std::unique_ptr orig_buffer(new uint8_t[frame_length_]); - EXPECT_GT(fread(orig_buffer.get(), 1, frame_length_, source_file_), 0U); - test_frame_.CreateFrame(orig_buffer.get(), - orig_buffer.get() + size_y_, - orig_buffer.get() + size_y_ + size_uv_, - width_, height_, - width_, half_width_, half_width_, - kVideoRotation_0); - EXPECT_EQ(0, test_scaler_.Scale(test_frame_, &test_frame2)); - EXPECT_GT(width_ * height_, test_frame2.allocated_size(kYPlane)); - EXPECT_GT(size_uv_, test_frame2.allocated_size(kUPlane)); - EXPECT_GT(size_uv_, test_frame2.allocated_size(kVPlane)); - EXPECT_EQ(half_width_, test_frame2.width()); - EXPECT_EQ(half_height_, test_frame2.height()); -} - -// TODO(mikhal): Converge the test into one function that accepts the method. -#if defined(WEBRTC_ANDROID) -#define MAYBE_PointScaleTest DISABLED_PointScaleTest -#else -#define MAYBE_PointScaleTest PointScaleTest -#endif -TEST_F(TestScaler, MAYBE_PointScaleTest) { - double avg_psnr; - FILE* source_file2; - ScaleMethod method = kScalePoint; - std::string out_name = webrtc::test::OutputPath() + - "LibYuvTest_PointScale_176_144.yuv"; - ScaleSequence(method, - source_file_, out_name, - width_, height_, - half_width_, half_height_); - // Upsample back up and check PSNR. - source_file2 = fopen(out_name.c_str(), "rb"); - out_name = webrtc::test::OutputPath() + "LibYuvTest_PointScale_352_288_" - "upfrom_176_144.yuv"; - ScaleSequence(method, - source_file2, out_name, - 176, 144, - 352, 288); - avg_psnr = ComputeAvgSequencePSNR(source_file_, out_name, width_, height_); - printf("PSNR for scaling from: %d %d, down/up to: %d %d, and back to " - "original size: %f \n", width_, height_, 176, 144, avg_psnr); - // Average PSNR for lower bound in assert is ~0.1dB lower than the actual - // average PSNR under same conditions. - ASSERT_GT(avg_psnr, 27.9); - ASSERT_EQ(0, fclose(source_file2)); - out_name = webrtc::test::OutputPath() + "LibYuvTest_PointScale_320_240.yuv"; - ScaleSequence(method, - source_file_, out_name, - width_, height_, - 320, 240); - out_name = webrtc::test::OutputPath() + "LibYuvTest_PointScale_704_576.yuv"; - ScaleSequence(method, - source_file_, out_name, - width_, height_, - width_ * 2, height_ * 2); - out_name = webrtc::test::OutputPath() + "LibYuvTest_PointScale_300_200.yuv"; - ScaleSequence(method, - source_file_, out_name, - width_, height_, - 300, 200); - out_name = webrtc::test::OutputPath() + "LibYuvTest_PointScale_400_300.yuv"; - ScaleSequence(method, - source_file_, out_name, - width_, height_, - 400, 300); - // Down-sample to odd size frame and scale back up. - out_name = webrtc::test::OutputPath() + "LibYuvTest_PointScale_282_231.yuv"; - ScaleSequence(method, - source_file_, out_name, - width_, height_, - 282, 231); - source_file2 = fopen(out_name.c_str(), "rb"); - out_name = webrtc::test::OutputPath() + "LibYuvTest_PointScale_352_288_" - "upfrom_282_231.yuv"; - ScaleSequence(method, - source_file2, out_name, - 282, 231, - 352, 288); - avg_psnr = ComputeAvgSequencePSNR(source_file_, out_name, width_, height_); - printf("PSNR for scaling from: %d %d, down/up to: %d %d, and back to " - "original size: %f \n", width_, height_, 282, 231, avg_psnr); - // Average PSNR for lower bound in assert is ~0.1dB lower than the actual - // average PSNR under same conditions. - ASSERT_GT(avg_psnr, 25.8); - ASSERT_EQ(0, fclose(source_file2)); -} - -#if defined(WEBRTC_ANDROID) -#define MAYBE_BilinearScaleTest DISABLED_BiLinearScaleTest -#else -#define MAYBE_BilinearScaleTest BiLinearScaleTest -#endif -TEST_F(TestScaler, MAYBE_BiLinearScaleTest) { - double avg_psnr; - FILE* source_file2; - ScaleMethod method = kScaleBilinear; - std::string out_name = webrtc::test::OutputPath() + - "LibYuvTest_BilinearScale_176_144.yuv"; - ScaleSequence(method, - source_file_, out_name, - width_, height_, - width_ / 2, height_ / 2); - // Up-sample back up and check PSNR. - source_file2 = fopen(out_name.c_str(), "rb"); - out_name = webrtc::test::OutputPath() + "LibYuvTest_BilinearScale_352_288_" - "upfrom_176_144.yuv"; - ScaleSequence(method, - source_file2, out_name, - 176, 144, - 352, 288); - avg_psnr = ComputeAvgSequencePSNR(source_file_, out_name, width_, height_); - printf("PSNR for scaling from: %d %d, down/up to: %d %d, and back to " - "original size: %f \n", width_, height_, 176, 144, avg_psnr); - // Average PSNR for lower bound in assert is ~0.1dB lower than the actual - // average PSNR under same conditions. - ASSERT_GT(avg_psnr, 27.5); - ComputeAvgSequencePSNR(source_file_, out_name, width_, height_); - ASSERT_EQ(0, fclose(source_file2)); - out_name = webrtc::test::OutputPath() + - "LibYuvTest_BilinearScale_320_240.yuv"; - ScaleSequence(method, - source_file_, out_name, - width_, height_, - 320, 240); - out_name = webrtc::test::OutputPath() + - "LibYuvTest_BilinearScale_704_576.yuv"; - ScaleSequence(method, - source_file_, out_name, - width_, height_, - width_ * 2, height_ * 2); - out_name = webrtc::test::OutputPath() + - "LibYuvTest_BilinearScale_300_200.yuv"; - ScaleSequence(method, - source_file_, out_name, - width_, height_, - 300, 200); - out_name = webrtc::test::OutputPath() + - "LibYuvTest_BilinearScale_400_300.yuv"; - ScaleSequence(method, - source_file_, out_name, - width_, height_, - 400, 300); -} - -#if defined(WEBRTC_ANDROID) -#define MAYBE_BoxScaleTest DISABLED_BoxScaleTest -#else -#define MAYBE_BoxScaleTest BoxScaleTest -#endif -TEST_F(TestScaler, MAYBE_BoxScaleTest) { - double avg_psnr; - FILE* source_file2; - ScaleMethod method = kScaleBox; - std::string out_name = webrtc::test::OutputPath() + - "LibYuvTest_BoxScale_176_144.yuv"; - ScaleSequence(method, - source_file_, out_name, - width_, height_, - width_ / 2, height_ / 2); - // Up-sample back up and check PSNR. - source_file2 = fopen(out_name.c_str(), "rb"); - out_name = webrtc::test::OutputPath() + "LibYuvTest_BoxScale_352_288_" - "upfrom_176_144.yuv"; - ScaleSequence(method, - source_file2, out_name, - 176, 144, - 352, 288); - avg_psnr = ComputeAvgSequencePSNR(source_file_, out_name, width_, height_); - printf("PSNR for scaling from: %d %d, down/up to: %d %d, and back to " - "original size: %f \n", width_, height_, 176, 144, avg_psnr); - // Average PSNR for lower bound in assert is ~0.1dB lower than the actual - // average PSNR under same conditions. - ASSERT_GT(avg_psnr, 27.5); - ASSERT_EQ(0, fclose(source_file2)); - out_name = webrtc::test::OutputPath() + "LibYuvTest_BoxScale_320_240.yuv"; - ScaleSequence(method, - source_file_, out_name, - width_, height_, - 320, 240); - out_name = webrtc::test::OutputPath() + "LibYuvTest_BoxScale_704_576.yuv"; - ScaleSequence(method, - source_file_, out_name, - width_, height_, - width_ * 2, height_ * 2); - out_name = webrtc::test::OutputPath() + "LibYuvTest_BoxScale_300_200.yuv"; - ScaleSequence(method, - source_file_, out_name, - width_, height_, - 300, 200); - out_name = webrtc::test::OutputPath() + "LibYuvTest_BoxScale_400_300.yuv"; - ScaleSequence(method, - source_file_, out_name, - width_, height_, - 400, 300); -} - -double TestScaler::ComputeAvgSequencePSNR(FILE* input_file, - std::string out_name, - int width, int height) { - FILE* output_file; - output_file = fopen(out_name.c_str(), "rb"); - assert(output_file != NULL); - rewind(input_file); - rewind(output_file); - - size_t required_size = CalcBufferSize(kI420, width, height); - uint8_t* input_buffer = new uint8_t[required_size]; - uint8_t* output_buffer = new uint8_t[required_size]; - - int frame_count = 0; - double avg_psnr = 0; - VideoFrame in_frame, out_frame; - const int half_width = (width + 1) / 2; - in_frame.CreateEmptyFrame(width, height, width, half_width, half_width); - out_frame.CreateEmptyFrame(width, height, width, half_width, half_width); - while (feof(input_file) == 0) { - if (fread(input_buffer, 1, required_size, input_file) != required_size) { - break; - } - if (fread(output_buffer, 1, required_size, output_file) != required_size) { - break; - } - frame_count++; - EXPECT_EQ(0, ConvertToI420(kI420, input_buffer, 0, 0, width, height, - required_size, kVideoRotation_0, &in_frame)); - EXPECT_EQ(0, ConvertToI420(kI420, output_buffer, 0, 0, width, height, - required_size, kVideoRotation_0, &out_frame)); - double psnr = I420PSNR(&in_frame, &out_frame); - avg_psnr += psnr; - } - avg_psnr = avg_psnr / frame_count; - assert(0 == fclose(output_file)); - delete [] input_buffer; - delete [] output_buffer; - return avg_psnr; -} - -// TODO(mikhal): Move part to a separate scale test. -void TestScaler::ScaleSequence(ScaleMethod method, - FILE* source_file, std::string out_name, - int src_width, int src_height, - int dst_width, int dst_height) { - FILE* output_file; - EXPECT_EQ(0, test_scaler_.Set(src_width, src_height, - dst_width, dst_height, - kI420, kI420, method)); - - output_file = fopen(out_name.c_str(), "wb"); - ASSERT_TRUE(output_file != NULL); - - rewind(source_file); - - VideoFrame input_frame; - VideoFrame output_frame; - int64_t start_clock, total_clock; - total_clock = 0; - int frame_count = 0; - size_t src_required_size = CalcBufferSize(kI420, src_width, src_height); - std::unique_ptr frame_buffer(new uint8_t[src_required_size]); - int size_y = src_width * src_height; - int size_uv = ((src_width + 1) / 2) * ((src_height + 1) / 2); - - // Running through entire sequence. - while (feof(source_file) == 0) { - if (fread(frame_buffer.get(), 1, src_required_size, source_file) != - src_required_size) - break; - - input_frame.CreateFrame(frame_buffer.get(), - frame_buffer.get() + size_y, - frame_buffer.get() + size_y + size_uv, - src_width, src_height, - src_width, (src_width + 1) / 2, - (src_width + 1) / 2, - kVideoRotation_0); - - start_clock = rtc::TimeMillis(); - EXPECT_EQ(0, test_scaler_.Scale(input_frame, &output_frame)); - total_clock += rtc::TimeMillis() - start_clock; - if (PrintVideoFrame(output_frame, output_file) < 0) { - return; - } - frame_count++; - } - - if (frame_count) { - printf("Scaling[%d %d] => [%d %d]: ", - src_width, src_height, dst_width, dst_height); - printf("Average time per frame[ms]: %.2lf\n", - (static_cast(total_clock) / frame_count)); - } - ASSERT_EQ(0, fclose(output_file)); -} - -} // namespace webrtc diff --git a/webrtc/common_video/libyuv/webrtc_libyuv.cc b/webrtc/common_video/libyuv/webrtc_libyuv.cc index 298c4960b1..44577e9ac8 100644 --- a/webrtc/common_video/libyuv/webrtc_libyuv.cc +++ b/webrtc/common_video/libyuv/webrtc_libyuv.cc @@ -130,27 +130,28 @@ int PrintVideoFrame(const VideoFrame& frame, FILE* file) { return 0; } -int ExtractBuffer(const VideoFrame& input_frame, size_t size, uint8_t* buffer) { +int ExtractBuffer(const rtc::scoped_refptr& input_frame, + size_t size, + uint8_t* buffer) { assert(buffer); - if (input_frame.IsZeroSize()) + if (!input_frame) return -1; - size_t length = - CalcBufferSize(kI420, input_frame.width(), input_frame.height()); + int width = input_frame->width(); + int height = input_frame->height(); + size_t length = CalcBufferSize(kI420, width, height); if (size < length) { return -1; } - int width = input_frame.video_frame_buffer()->width(); - int height = input_frame.video_frame_buffer()->height(); int chroma_width = (width + 1) / 2; int chroma_height = (height + 1) / 2; - libyuv::I420Copy(input_frame.video_frame_buffer()->DataY(), - input_frame.video_frame_buffer()->StrideY(), - input_frame.video_frame_buffer()->DataU(), - input_frame.video_frame_buffer()->StrideU(), - input_frame.video_frame_buffer()->DataV(), - input_frame.video_frame_buffer()->StrideV(), + libyuv::I420Copy(input_frame->DataY(), + input_frame->StrideY(), + input_frame->DataU(), + input_frame->StrideU(), + input_frame->DataV(), + input_frame->StrideV(), buffer, width, buffer + width*height, chroma_width, buffer + width*height + chroma_width*chroma_height, @@ -160,6 +161,9 @@ int ExtractBuffer(const VideoFrame& input_frame, size_t size, uint8_t* buffer) { return static_cast(length); } +int ExtractBuffer(const VideoFrame& input_frame, size_t size, uint8_t* buffer) { + return ExtractBuffer(input_frame.video_frame_buffer(), size, buffer); +} int ConvertNV12ToRGB565(const uint8_t* src_frame, uint8_t* dst_frame, diff --git a/webrtc/common_video/video_frame.cc b/webrtc/common_video/video_frame.cc index 978dde15b7..bfac3a680d 100644 --- a/webrtc/common_video/video_frame.cc +++ b/webrtc/common_video/video_frame.cc @@ -23,12 +23,6 @@ namespace webrtc { // to optimized bitstream readers. See avcodec_decode_video2. const size_t EncodedImage::kBufferPaddingBytesH264 = 8; -int ExpectedSize(int plane_stride, int image_height, PlaneType type) { - if (type == kYPlane) - return plane_stride * image_height; - return plane_stride * ((image_height + 1) / 2); -} - VideoFrame::VideoFrame() : video_frame_buffer_(nullptr), timestamp_(0), diff --git a/webrtc/common_video/video_frame_buffer.cc b/webrtc/common_video/video_frame_buffer.cc index 27aea3efc5..2fb3540131 100644 --- a/webrtc/common_video/video_frame_buffer.cc +++ b/webrtc/common_video/video_frame_buffer.cc @@ -8,11 +8,14 @@ * be found in the AUTHORS file in the root of the source tree. */ +#include + #include "webrtc/common_video/include/video_frame_buffer.h" #include "webrtc/base/checks.h" #include "webrtc/base/keep_ref_until_done.h" #include "libyuv/convert.h" +#include "libyuv/scale.h" // Aligning pointer to 64 bytes for improved performance, e.g. use SIMD. static const int kBufferAlignment = 64; @@ -208,6 +211,60 @@ void I420Buffer::SetToBlack() { 0, 128, 128) == 0); } +void I420Buffer::CropAndScaleFrom( + const rtc::scoped_refptr& src, + int offset_x, + int offset_y, + int crop_width, + int crop_height) { + RTC_CHECK_LE(crop_width, src->width()); + RTC_CHECK_LE(crop_height, src->height()); + RTC_CHECK_LE(crop_width + offset_x, src->width()); + RTC_CHECK_LE(crop_height + offset_y, src->height()); + RTC_CHECK_GE(offset_x, 0); + RTC_CHECK_GE(offset_y, 0); + + // Make sure offset is even so that u/v plane becomes aligned. + const int uv_offset_x = offset_x / 2; + const int uv_offset_y = offset_y / 2; + offset_x = uv_offset_x * 2; + offset_y = uv_offset_y * 2; + + const uint8_t* y_plane = + src->DataY() + src->StrideY() * offset_y + offset_x; + const uint8_t* u_plane = + src->DataU() + src->StrideU() * uv_offset_y + uv_offset_x; + const uint8_t* v_plane = + src->DataV() + src->StrideV() * uv_offset_y + uv_offset_x; + int res = libyuv::I420Scale(y_plane, src->StrideY(), + u_plane, src->StrideU(), + v_plane, src->StrideV(), + crop_width, crop_height, + MutableDataY(), StrideY(), + MutableDataU(), StrideU(), + MutableDataV(), StrideV(), + width(), height(), libyuv::kFilterBox); + + RTC_DCHECK_EQ(res, 0); +} + +void I420Buffer::CropAndScaleFrom( + const rtc::scoped_refptr& src) { + const int crop_width = + std::min(src->width(), width() * src->height() / height()); + const int crop_height = + std::min(src->height(), height() * src->width() / width()); + + CropAndScaleFrom( + src, + (src->width() - crop_width) / 2, (src->height() - crop_height) / 2, + crop_width, crop_height); +} + +void I420Buffer::ScaleFrom(const rtc::scoped_refptr& src) { + CropAndScaleFrom(src, 0, 0, src->width(), src->height()); +} + NativeHandleBuffer::NativeHandleBuffer(void* native_handle, int width, int height) @@ -316,35 +373,4 @@ rtc::scoped_refptr WrappedI420Buffer::NativeToI420Buffer() { return nullptr; } -rtc::scoped_refptr ShallowCenterCrop( - const rtc::scoped_refptr& buffer, - int cropped_width, - int cropped_height) { - RTC_CHECK(buffer->native_handle() == nullptr); - RTC_CHECK_LE(cropped_width, buffer->width()); - RTC_CHECK_LE(cropped_height, buffer->height()); - if (buffer->width() == cropped_width && buffer->height() == cropped_height) - return buffer; - - // Center crop to |cropped_width| x |cropped_height|. - // Make sure offset is even so that u/v plane becomes aligned. - const int uv_offset_x = (buffer->width() - cropped_width) / 4; - const int uv_offset_y = (buffer->height() - cropped_height) / 4; - const int offset_x = uv_offset_x * 2; - const int offset_y = uv_offset_y * 2; - - const uint8_t* y_plane = buffer->DataY() + - buffer->StrideY() * offset_y + offset_x; - const uint8_t* u_plane = buffer->DataU() + - buffer->StrideU() * uv_offset_y + uv_offset_x; - const uint8_t* v_plane = buffer->DataV() + - buffer->StrideV() * uv_offset_y + uv_offset_x; - return new rtc::RefCountedObject( - cropped_width, cropped_height, - y_plane, buffer->StrideY(), - u_plane, buffer->StrideU(), - v_plane, buffer->StrideV(), - rtc::KeepRefUntilDone(buffer)); -} - } // namespace webrtc diff --git a/webrtc/media/base/videoframe.cc b/webrtc/media/base/videoframe.cc index 40cced1154..6e0a59d82b 100644 --- a/webrtc/media/base/videoframe.cc +++ b/webrtc/media/base/videoframe.cc @@ -22,32 +22,6 @@ namespace cricket { -// Round to 2 pixels because Chroma channels are half size. -#define ROUNDTO2(v) (v & ~1) - -bool VideoFrame::CopyToPlanes(uint8_t* dst_y, - uint8_t* dst_u, - uint8_t* dst_v, - int32_t dst_pitch_y, - int32_t dst_pitch_u, - int32_t dst_pitch_v) const { - const rtc::scoped_refptr& buffer = - video_frame_buffer(); - if (!buffer) { - LOG(LS_ERROR) << "NULL video buffer."; - return false; - } - int32_t src_width = width(); - int32_t src_height = height(); - return libyuv::I420Copy(buffer->DataY(), buffer->StrideY(), - buffer->DataU(), buffer->StrideU(), - buffer->DataV(), buffer->StrideV(), - dst_y, dst_pitch_y, - dst_u, dst_pitch_u, - dst_v, dst_pitch_v, - src_width, src_height) == 0; -} - size_t VideoFrame::ConvertToRgbBuffer(uint32_t to_fourcc, uint8_t* buffer, size_t size, @@ -69,85 +43,6 @@ size_t VideoFrame::ConvertToRgbBuffer(uint32_t to_fourcc, return needed; } -// TODO(fbarchard): Handle odd width/height with rounding. -// TODO(nisse): If method is kept, switch to using int instead of -// size_t and int32_t. -void VideoFrame::StretchToPlanes(uint8_t* dst_y, - uint8_t* dst_u, - uint8_t* dst_v, - int32_t dst_pitch_y, - int32_t dst_pitch_u, - int32_t dst_pitch_v, - size_t dst_width, - size_t dst_height, - bool interpolate, - bool vert_crop) const { - if (!video_frame_buffer()) { - LOG(LS_ERROR) << "NULL frame buffer."; - return; - } - - size_t src_width = width(); - size_t src_height = height(); - if (dst_width == src_width && dst_height == src_height) { - CopyToPlanes(dst_y, dst_u, dst_v, dst_pitch_y, dst_pitch_u, dst_pitch_v); - return; - } - const uint8_t* src_y = video_frame_buffer()->DataY(); - const uint8_t* src_u = video_frame_buffer()->DataU(); - const uint8_t* src_v = video_frame_buffer()->DataV(); - - if (vert_crop) { - // Adjust the input width:height ratio to be the same as the output ratio. - if (src_width * dst_height > src_height * dst_width) { - // Reduce the input width, but keep size/position aligned for YuvScaler - src_width = ROUNDTO2(src_height * dst_width / dst_height); - int32_t iwidth_offset = ROUNDTO2((width() - src_width) / 2); - src_y += iwidth_offset; - src_u += iwidth_offset / 2; - src_v += iwidth_offset / 2; - } else if (src_width * dst_height < src_height * dst_width) { - // Reduce the input height. - src_height = src_width * dst_height / dst_width; - int32_t iheight_offset = - static_cast((height() - src_height) >> 2); - iheight_offset <<= 1; // Ensure that iheight_offset is even. - src_y += iheight_offset * video_frame_buffer()->StrideY(); - src_u += iheight_offset / 2 * video_frame_buffer()->StrideU(); - src_v += iheight_offset / 2 * video_frame_buffer()->StrideV(); - } - } - - // Scale to the output I420 frame. - libyuv::Scale(src_y, src_u, src_v, video_frame_buffer()->StrideY(), - video_frame_buffer()->StrideU(), - video_frame_buffer()->StrideV(), - static_cast(src_width), static_cast(src_height), - dst_y, dst_u, dst_v, dst_pitch_y, dst_pitch_u, dst_pitch_v, - static_cast(dst_width), static_cast(dst_height), - interpolate); -} - -void VideoFrame::StretchToFrame(VideoFrame* dst, - bool interpolate, bool vert_crop) const { - if (!dst) { - LOG(LS_ERROR) << "NULL dst pointer."; - return; - } - - StretchToPlanes(dst->video_frame_buffer()->MutableDataY(), - dst->video_frame_buffer()->MutableDataU(), - dst->video_frame_buffer()->MutableDataV(), - dst->video_frame_buffer()->StrideY(), - dst->video_frame_buffer()->StrideU(), - dst->video_frame_buffer()->StrideV(), - dst->width(), dst->height(), - interpolate, vert_crop); - dst->SetTimeStamp(GetTimeStamp()); - // Stretched frame should have the same rotation as the source. - dst->set_rotation(rotation()); -} - static const size_t kMaxSampleSize = 1000000000u; // Returns whether a sample is valid. bool VideoFrame::Validate(uint32_t fourcc, diff --git a/webrtc/media/base/videoframe.h b/webrtc/media/base/videoframe.h index c0d9e6658e..4015f906f9 100644 --- a/webrtc/media/base/videoframe.h +++ b/webrtc/media/base/videoframe.h @@ -69,28 +69,6 @@ class VideoFrame { size_t size, int stride_rgb) const; - // Writes the frame into the given planes, stretched to the given width and - // height. The parameter "interpolate" controls whether to interpolate or just - // take the nearest-point. The parameter "crop" controls whether to crop this - // frame to the aspect ratio of the given dimensions before stretching. - virtual void StretchToPlanes(uint8_t* y, - uint8_t* u, - uint8_t* v, - int32_t pitchY, - int32_t pitchU, - int32_t pitchV, - size_t width, - size_t height, - bool interpolate, - bool crop) const; - - // Writes the frame into the target VideoFrame, stretched to the size of that - // frame. The parameter "interpolate" controls whether to interpolate or just - // take the nearest-point. The parameter "crop" controls whether to crop this - // frame to the aspect ratio of the target frame before stretching. - virtual void StretchToFrame(VideoFrame *target, bool interpolate, - bool crop) const; - // Tests if sample is valid. Returns true if valid. static bool Validate(uint32_t fourcc, int w, @@ -99,17 +77,6 @@ class VideoFrame { size_t sample_size); protected: - // Writes the frame into the given planes, stretched to the given width and - // height. The parameter "interpolate" controls whether to interpolate or just - // take the nearest-point. The parameter "crop" controls whether to crop this - // frame to the aspect ratio of the given dimensions before stretching. - virtual bool CopyToPlanes(uint8_t* dst_y, - uint8_t* dst_u, - uint8_t* dst_v, - int32_t dst_pitch_y, - int32_t dst_pitch_u, - int32_t dst_pitch_v) const; - // Creates an empty frame. virtual VideoFrame* CreateEmptyFrame(int w, int h, diff --git a/webrtc/media/base/videoframe_unittest.h b/webrtc/media/base/videoframe_unittest.h index 720634b9d1..a79492d106 100644 --- a/webrtc/media/base/videoframe_unittest.h +++ b/webrtc/media/base/videoframe_unittest.h @@ -1811,33 +1811,6 @@ class VideoFrameTest : public testing::Test { EXPECT_EQ(const_source->video_frame_buffer(), target->video_frame_buffer()); } - void StretchToFrame() { - // Create the source frame as a black frame. - rtc::scoped_refptr buffer( - new rtc::RefCountedObject(kWidth * 2, kHeight * 2)); - - buffer->SetToBlack(); - T source(buffer, 0, webrtc::kVideoRotation_0); - - EXPECT_TRUE(IsSize(source, kWidth * 2, kHeight * 2)); - - // Create the target frame by loading from a file. - T target1; - ASSERT_TRUE(LoadFrameNoRepeat(&target1)); - EXPECT_FALSE(IsBlack(target1)); - - // Stretch and check if the stretched target is black. - source.StretchToFrame(&target1, true, false); - EXPECT_TRUE(IsBlack(target1)); - - // Crop and stretch and check if the stretched target is black. - T target2; - ASSERT_TRUE(LoadFrameNoRepeat(&target2)); - source.StretchToFrame(&target2, true, true); - EXPECT_TRUE(IsBlack(target2)); - EXPECT_EQ(source.GetTimeStamp(), target2.GetTimeStamp()); - } - int repeat_; }; diff --git a/webrtc/media/base/videoframefactory.cc b/webrtc/media/base/videoframefactory.cc index 0a7dfc22d4..7b89ad6315 100644 --- a/webrtc/media/base/videoframefactory.cc +++ b/webrtc/media/base/videoframefactory.cc @@ -39,13 +39,12 @@ VideoFrame* VideoFrameFactory::CreateAliasedFrame( std::swap(output_width, output_height); } - std::unique_ptr output_frame(new WebRtcVideoFrame( - pool_.CreateBuffer(output_width, output_height), - cropped_input_frame->rotation(), - cropped_input_frame->timestamp_us())); + rtc::scoped_refptr scaled_buffer( + pool_.CreateBuffer(output_width, output_height)); + scaled_buffer->CropAndScaleFrom(cropped_input_frame->video_frame_buffer()); - cropped_input_frame->StretchToFrame(output_frame.get(), true, true); - return output_frame.release(); + return new WebRtcVideoFrame(scaled_buffer, cropped_input_frame->rotation(), + cropped_input_frame->timestamp_us()); } } // namespace cricket diff --git a/webrtc/media/engine/webrtcvideoframe_unittest.cc b/webrtc/media/engine/webrtcvideoframe_unittest.cc index 6d1d810934..2c6f56cb25 100644 --- a/webrtc/media/engine/webrtcvideoframe_unittest.cc +++ b/webrtc/media/engine/webrtcvideoframe_unittest.cc @@ -245,7 +245,6 @@ TEST_WEBRTCVIDEOFRAME(ConvertFromUYVYBufferInverted) // TEST_WEBRTCVIDEOFRAME(ConvertToI422Buffer) // TEST_WEBRTCVIDEOFRAME(ConstructARGBBlackWhitePixel) -TEST_WEBRTCVIDEOFRAME(StretchToFrame) TEST_WEBRTCVIDEOFRAME(Copy) TEST_WEBRTCVIDEOFRAME(CopyIsRef) diff --git a/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.cc b/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.cc index 6dcb5d57a1..a7bd5e2040 100644 --- a/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.cc +++ b/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.cc @@ -145,15 +145,16 @@ struct FrameEncodeParams { // We receive I420Frames as input, but we need to feed CVPixelBuffers into the // encoder. This performs the copy and format conversion. // TODO(tkchin): See if encoder will accept i420 frames and compare performance. -bool CopyVideoFrameToPixelBuffer(const webrtc::VideoFrame& frame, - CVPixelBufferRef pixel_buffer) { +bool CopyVideoFrameToPixelBuffer( + const rtc::scoped_refptr& frame, + CVPixelBufferRef pixel_buffer) { RTC_DCHECK(pixel_buffer); RTC_DCHECK(CVPixelBufferGetPixelFormatType(pixel_buffer) == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange); RTC_DCHECK(CVPixelBufferGetHeightOfPlane(pixel_buffer, 0) == - static_cast(frame.height())); + static_cast(frame->height())); RTC_DCHECK(CVPixelBufferGetWidthOfPlane(pixel_buffer, 0) == - static_cast(frame.width())); + static_cast(frame->width())); CVReturn cvRet = CVPixelBufferLockBaseAddress(pixel_buffer, 0); if (cvRet != kCVReturnSuccess) { @@ -168,14 +169,11 @@ bool CopyVideoFrameToPixelBuffer(const webrtc::VideoFrame& frame, int dst_stride_uv = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, 1); // Convert I420 to NV12. int ret = libyuv::I420ToNV12( - frame.video_frame_buffer()->DataY(), - frame.video_frame_buffer()->StrideY(), - frame.video_frame_buffer()->DataU(), - frame.video_frame_buffer()->StrideU(), - frame.video_frame_buffer()->DataV(), - frame.video_frame_buffer()->StrideV(), + frame->DataY(), frame->StrideY(), + frame->DataU(), frame->StrideU(), + frame->DataV(), frame->StrideV(), dst_y, dst_stride_y, dst_uv, dst_stride_uv, - frame.width(), frame.height()); + frame->width(), frame->height()); CVPixelBufferUnlockBaseAddress(pixel_buffer, 0); if (ret) { LOG(LS_ERROR) << "Error converting I420 VideoFrame to NV12 :" << ret; @@ -247,11 +245,12 @@ int H264VideoToolboxEncoder::InitEncode(const VideoCodec* codec_settings, return ResetCompressionSession(); } -const VideoFrame& H264VideoToolboxEncoder::GetScaledFrameOnEncode( - const VideoFrame& frame) { +rtc::scoped_refptr +H264VideoToolboxEncoder::GetScaledBufferOnEncode( + const rtc::scoped_refptr& frame) { rtc::CritScope lock(&quality_scaler_crit_); - quality_scaler_.OnEncodeFrame(frame); - return quality_scaler_.GetScaledFrame(frame); + quality_scaler_.OnEncodeFrame(frame->width(), frame->height()); + return quality_scaler_.GetScaledBuffer(frame); } int H264VideoToolboxEncoder::Encode( @@ -270,11 +269,12 @@ int H264VideoToolboxEncoder::Encode( } #endif bool is_keyframe_required = false; - const VideoFrame& input_image = GetScaledFrameOnEncode(frame); + rtc::scoped_refptr input_image( + GetScaledBufferOnEncode(frame.video_frame_buffer())); - if (input_image.width() != width_ || input_image.height() != height_) { - width_ = input_image.width(); - height_ = input_image.height(); + if (input_image->width() != width_ || input_image->height() != height_) { + width_ = input_image->width(); + height_ = input_image->height(); int ret = ResetCompressionSession(); if (ret < 0) return ret; @@ -327,7 +327,7 @@ int H264VideoToolboxEncoder::Encode( } CMTime presentation_time_stamp = - CMTimeMake(input_image.render_time_ms(), 1000); + CMTimeMake(frame.render_time_ms(), 1000); CFDictionaryRef frame_properties = nullptr; if (is_keyframe_required) { CFTypeRef keys[] = {kVTEncodeFrameOptionKey_ForceKeyFrame}; @@ -336,8 +336,8 @@ int H264VideoToolboxEncoder::Encode( } std::unique_ptr encode_params; encode_params.reset(new internal::FrameEncodeParams( - this, codec_specific_info, width_, height_, input_image.render_time_ms(), - input_image.timestamp(), input_image.rotation())); + this, codec_specific_info, width_, height_, frame.render_time_ms(), + frame.timestamp(), frame.rotation())); // Update the bitrate if needed. SetBitrateBps(bitrate_adjuster_.GetAdjustedBitrateBps()); diff --git a/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.h b/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.h index 5dfcf9f60a..c8ae495eb3 100644 --- a/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.h +++ b/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.h @@ -70,7 +70,8 @@ class H264VideoToolboxEncoder : public H264Encoder { int ResetCompressionSession(); void ConfigureCompressionSession(); void DestroyCompressionSession(); - const VideoFrame& GetScaledFrameOnEncode(const VideoFrame& frame); + rtc::scoped_refptr GetScaledBufferOnEncode( + const rtc::scoped_refptr& frame); void SetBitrateBps(uint32_t bitrate_bps); void SetEncoderBitrateBps(uint32_t bitrate_bps); diff --git a/webrtc/modules/video_coding/codecs/test/videoprocessor.cc b/webrtc/modules/video_coding/codecs/test/videoprocessor.cc index 9a9a0ddf16..1f7d738dc5 100644 --- a/webrtc/modules/video_coding/codecs/test/videoprocessor.cc +++ b/webrtc/modules/video_coding/codecs/test/videoprocessor.cc @@ -66,8 +66,7 @@ VideoProcessorImpl::VideoProcessorImpl(webrtc::VideoEncoder* encoder, num_dropped_frames_(0), num_spatial_resizes_(0), last_encoder_frame_width_(0), - last_encoder_frame_height_(0), - scaler_() { + last_encoder_frame_height_(0) { assert(encoder); assert(decoder); assert(frame_reader); @@ -335,23 +334,16 @@ void VideoProcessorImpl::FrameDecoded(const VideoFrame& image) { // upsample back to original size: needed for PSNR and SSIM computations. if (image.width() != config_.codec_settings->width || image.height() != config_.codec_settings->height) { - VideoFrame up_image; - int ret_val = scaler_.Set( - image.width(), image.height(), config_.codec_settings->width, - config_.codec_settings->height, kI420, kI420, kScaleBilinear); - assert(ret_val >= 0); - if (ret_val < 0) { - fprintf(stderr, "Failed to set scalar for frame: %d, return code: %d\n", - frame_number, ret_val); - } - ret_val = scaler_.Scale(image, &up_image); - assert(ret_val >= 0); - if (ret_val < 0) { - fprintf(stderr, "Failed to scale frame: %d, return code: %d\n", - frame_number, ret_val); - } + rtc::scoped_refptr up_image( + new rtc::RefCountedObject(config_.codec_settings->width, + config_.codec_settings->height)); + + // Should be the same aspect ratio, no cropping needed. + up_image->ScaleFrom(image.video_frame_buffer()); + // TODO(mikhal): Extracting the buffer for now - need to update test. - size_t length = CalcBufferSize(kI420, up_image.width(), up_image.height()); + size_t length = + CalcBufferSize(kI420, up_image->width(), up_image->height()); std::unique_ptr image_buffer(new uint8_t[length]); int extracted_length = ExtractBuffer(up_image, length, image_buffer.get()); assert(extracted_length > 0); diff --git a/webrtc/modules/video_coding/codecs/test/videoprocessor.h b/webrtc/modules/video_coding/codecs/test/videoprocessor.h index f0322dd67b..af843298ba 100644 --- a/webrtc/modules/video_coding/codecs/test/videoprocessor.h +++ b/webrtc/modules/video_coding/codecs/test/videoprocessor.h @@ -14,7 +14,6 @@ #include #include "webrtc/base/checks.h" -#include "webrtc/common_video/libyuv/include/scaler.h" #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" #include "webrtc/modules/video_coding/include/video_codec_interface.h" #include "webrtc/modules/video_coding/codecs/test/packet_manipulator.h" @@ -219,7 +218,6 @@ class VideoProcessorImpl : public VideoProcessor { int num_spatial_resizes_; int last_encoder_frame_width_; int last_encoder_frame_height_; - Scaler scaler_; // Statistics double bit_rate_factor_; // multiply frame length with this to get bit rate diff --git a/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc b/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc index 9f361dc626..706e57a0e5 100644 --- a/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc +++ b/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc @@ -525,8 +525,8 @@ class VideoProcessorIntegrationTest : public testing::Test { EXPECT_GT(psnr_result.min, quality_metrics.minimum_min_psnr); EXPECT_GT(ssim_result.average, quality_metrics.minimum_avg_ssim); EXPECT_GT(ssim_result.min, quality_metrics.minimum_min_ssim); - if (!remove(config_.output_filename.c_str())) { - fprintf(stderr, "Failed to remove temporary file!"); + if (remove(config_.output_filename.c_str()) < 0) { + fprintf(stderr, "Failed to remove temporary file!\n"); } } }; diff --git a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc index 4c9a37cbf8..ee30cfa983 100644 --- a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc +++ b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc @@ -729,40 +729,40 @@ int VP8EncoderImpl::Encode(const VideoFrame& frame, if (encoded_complete_callback_ == NULL) return WEBRTC_VIDEO_CODEC_UNINITIALIZED; - if (quality_scaler_enabled_) - quality_scaler_.OnEncodeFrame(frame); - const VideoFrame& input_image = - quality_scaler_enabled_ ? quality_scaler_.GetScaledFrame(frame) : frame; + rtc::scoped_refptr input_image = frame.video_frame_buffer(); - if (quality_scaler_enabled_ && (input_image.width() != codec_.width || - input_image.height() != codec_.height)) { - int ret = UpdateCodecFrameSize(input_image); - if (ret < 0) - return ret; + if (quality_scaler_enabled_) { + quality_scaler_.OnEncodeFrame(frame.width(), frame.height()); + input_image = quality_scaler_.GetScaledBuffer(input_image); + + if (input_image->width() != codec_.width || + input_image->height() != codec_.height) { + int ret = + UpdateCodecFrameSize(input_image->width(), input_image->height()); + if (ret < 0) + return ret; + } } // Since we are extracting raw pointers from |input_image| to // |raw_images_[0]|, the resolution of these frames must match. Note that // |input_image| might be scaled from |frame|. In that case, the resolution of // |raw_images_[0]| should have been updated in UpdateCodecFrameSize. - RTC_DCHECK_EQ(input_image.width(), static_cast(raw_images_[0].d_w)); - RTC_DCHECK_EQ(input_image.height(), static_cast(raw_images_[0].d_h)); + RTC_DCHECK_EQ(input_image->width(), static_cast(raw_images_[0].d_w)); + RTC_DCHECK_EQ(input_image->height(), static_cast(raw_images_[0].d_h)); // Image in vpx_image_t format. // Input image is const. VP8's raw image is not defined as const. raw_images_[0].planes[VPX_PLANE_Y] = - const_cast(input_image.video_frame_buffer()->DataY()); + const_cast(input_image->DataY()); raw_images_[0].planes[VPX_PLANE_U] = - const_cast(input_image.video_frame_buffer()->DataU()); + const_cast(input_image->DataU()); raw_images_[0].planes[VPX_PLANE_V] = - const_cast(input_image.video_frame_buffer()->DataV()); + const_cast(input_image->DataV()); - raw_images_[0].stride[VPX_PLANE_Y] = - input_image.video_frame_buffer()->StrideY(); - raw_images_[0].stride[VPX_PLANE_U] = - input_image.video_frame_buffer()->StrideU(); - raw_images_[0].stride[VPX_PLANE_V] = - input_image.video_frame_buffer()->StrideV(); + raw_images_[0].stride[VPX_PLANE_Y] = input_image->StrideY(); + raw_images_[0].stride[VPX_PLANE_U] = input_image->StrideU(); + raw_images_[0].stride[VPX_PLANE_V] = input_image->StrideV(); for (size_t i = 1; i < encoders_.size(); ++i) { // Scale the image down a number of times by downsampling factor @@ -781,7 +781,7 @@ int VP8EncoderImpl::Encode(const VideoFrame& frame, } vpx_enc_frame_flags_t flags[kMaxSimulcastStreams]; for (size_t i = 0; i < encoders_.size(); ++i) { - int ret = temporal_layers_[i]->EncodeFlags(input_image.timestamp()); + int ret = temporal_layers_[i]->EncodeFlags(frame.timestamp()); if (ret < 0) { // Drop this frame. return WEBRTC_VIDEO_CODEC_OK; @@ -833,11 +833,11 @@ int VP8EncoderImpl::Encode(const VideoFrame& frame, rps_.ReceivedRPSI(codec_specific_info->codecSpecific.VP8.pictureIdRPSI); } if (codec_specific_info->codecSpecific.VP8.hasReceivedSLI) { - sendRefresh = rps_.ReceivedSLI(input_image.timestamp()); + sendRefresh = rps_.ReceivedSLI(frame.timestamp()); } for (size_t i = 0; i < encoders_.size(); ++i) { flags[i] = rps_.EncodeFlags(picture_id_[i], sendRefresh, - input_image.timestamp()); + frame.timestamp()); } } else { if (codec_specific_info->codecSpecific.VP8.hasReceivedRPSI) { @@ -905,17 +905,18 @@ int VP8EncoderImpl::Encode(const VideoFrame& frame, if (error) return WEBRTC_VIDEO_CODEC_ERROR; timestamp_ += duration; - return GetEncodedPartitions(input_image, only_predict_from_key_frame); + // Examines frame timestamps only. + return GetEncodedPartitions(frame, only_predict_from_key_frame); } // TODO(pbos): Make sure this works for properly for >1 encoders. -int VP8EncoderImpl::UpdateCodecFrameSize(const VideoFrame& input_image) { - codec_.width = input_image.width(); - codec_.height = input_image.height(); +int VP8EncoderImpl::UpdateCodecFrameSize(int width, int height) { + codec_.width = width; + codec_.height = height; if (codec_.numberOfSimulcastStreams <= 1) { // For now scaling is only used for single-layer streams. - codec_.simulcastStream[0].width = input_image.width(); - codec_.simulcastStream[0].height = input_image.height(); + codec_.simulcastStream[0].width = width; + codec_.simulcastStream[0].height = height; } // Update the cpu_speed setting for resolution change. vpx_codec_control(&(encoders_[0]), VP8E_SET_CPUUSED, diff --git a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.h b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.h index 0ec01426c1..a9079b2a7f 100644 --- a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.h +++ b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.h @@ -75,7 +75,7 @@ class VP8EncoderImpl : public VP8Encoder { int InitAndSetControlSettings(); // Update frame size for codec. - int UpdateCodecFrameSize(const VideoFrame& input_image); + int UpdateCodecFrameSize(int width, int height); void PopulateCodecSpecific(CodecSpecificInfo* codec_specific, const vpx_codec_cx_pkt& pkt, diff --git a/webrtc/modules/video_coding/utility/moving_average.h b/webrtc/modules/video_coding/utility/moving_average.h index 494bfd51fb..cdad50f0f9 100644 --- a/webrtc/modules/video_coding/utility/moving_average.h +++ b/webrtc/modules/video_coding/utility/moving_average.h @@ -11,6 +11,8 @@ #ifndef WEBRTC_MODULES_VIDEO_CODING_UTILITY_MOVING_AVERAGE_H_ #define WEBRTC_MODULES_VIDEO_CODING_UTILITY_MOVING_AVERAGE_H_ +#include + #include #include "webrtc/typedefs.h" diff --git a/webrtc/modules/video_coding/utility/quality_scaler.cc b/webrtc/modules/video_coding/utility/quality_scaler.cc index bb60ee036e..541d0d431f 100644 --- a/webrtc/modules/video_coding/utility/quality_scaler.cc +++ b/webrtc/modules/video_coding/utility/quality_scaler.cc @@ -7,6 +7,7 @@ * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ + #include "webrtc/modules/video_coding/utility/quality_scaler.h" namespace webrtc { @@ -94,11 +95,11 @@ void QualityScaler::ReportDroppedFrame() { framedrop_percent_.AddSample(100); } -void QualityScaler::OnEncodeFrame(const VideoFrame& frame) { +void QualityScaler::OnEncodeFrame(int width, int height) { // Should be set through InitEncode -> Should be set by now. - assert(low_qp_threshold_ >= 0); - assert(num_samples_upscale_ > 0); - assert(num_samples_downscale_ > 0); + RTC_DCHECK_GE(low_qp_threshold_, 0); + RTC_DCHECK_GT(num_samples_upscale_, 0u); + RTC_DCHECK_GT(num_samples_downscale_, 0u); // Update scale factor. int avg_drop = 0; @@ -113,38 +114,31 @@ void QualityScaler::OnEncodeFrame(const VideoFrame& frame) { avg_qp <= low_qp_threshold_) { AdjustScale(true); } - UpdateTargetResolution(frame.width(), frame.height()); + UpdateTargetResolution(width, height); } QualityScaler::Resolution QualityScaler::GetScaledResolution() const { return res_; } -const VideoFrame& QualityScaler::GetScaledFrame(const VideoFrame& frame) { +rtc::scoped_refptr QualityScaler::GetScaledBuffer( + const rtc::scoped_refptr& frame) { Resolution res = GetScaledResolution(); - if (res.width == frame.width()) + int src_width = frame->width(); + int src_height = frame->height(); + + if (res.width == src_width && res.height == src_height) return frame; + rtc::scoped_refptr scaled_buffer = + pool_.CreateBuffer(res.width, res.height); - scaler_.Set(frame.width(), frame.height(), res.width, res.height, kI420, - kI420, kScaleBox); - if (scaler_.Scale(frame, &scaled_frame_) != 0) - return frame; + scaled_buffer->ScaleFrom(frame); - // TODO(perkj): Refactor the scaler to not own |scaled_frame|. VideoFrame are - // just thin wrappers so instead the scaler should return a - // rtc::scoped_refptr and a new VideoFrame be created with - // the meta data from |frame|. That way we would not have to set all these - // meta data. - scaled_frame_.set_ntp_time_ms(frame.ntp_time_ms()); - scaled_frame_.set_timestamp(frame.timestamp()); - scaled_frame_.set_render_time_ms(frame.render_time_ms()); - scaled_frame_.set_rotation(frame.rotation()); - - return scaled_frame_; + return scaled_buffer; } void QualityScaler::UpdateTargetResolution(int frame_width, int frame_height) { - assert(downscale_shift_ >= 0); + RTC_DCHECK_GE(downscale_shift_, 0); int shifts_performed = 0; for (int shift = downscale_shift_; shift > 0 && (frame_width / 2 >= kMinDownscaleDimension) && diff --git a/webrtc/modules/video_coding/utility/quality_scaler.h b/webrtc/modules/video_coding/utility/quality_scaler.h index fe70393c21..7176d4966e 100644 --- a/webrtc/modules/video_coding/utility/quality_scaler.h +++ b/webrtc/modules/video_coding/utility/quality_scaler.h @@ -11,7 +11,7 @@ #ifndef WEBRTC_MODULES_VIDEO_CODING_UTILITY_QUALITY_SCALER_H_ #define WEBRTC_MODULES_VIDEO_CODING_UTILITY_QUALITY_SCALER_H_ -#include "webrtc/common_video/libyuv/include/scaler.h" +#include "webrtc/common_video/include/i420_buffer_pool.h" #include "webrtc/modules/video_coding/utility/moving_average.h" namespace webrtc { @@ -32,9 +32,10 @@ class QualityScaler { void ReportFramerate(int framerate); void ReportQP(int qp); void ReportDroppedFrame(); - void OnEncodeFrame(const VideoFrame& frame); + void OnEncodeFrame(int width, int height); Resolution GetScaledResolution() const; - const VideoFrame& GetScaledFrame(const VideoFrame& frame); + rtc::scoped_refptr GetScaledBuffer( + const rtc::scoped_refptr& frame); int downscale_shift() const { return downscale_shift_; } // QP is obtained from VP8-bitstream for HW, so the QP corresponds to the @@ -52,8 +53,7 @@ class QualityScaler { void ClearSamples(); void UpdateSampleCounts(); - Scaler scaler_; - VideoFrame scaled_frame_; + I420BufferPool pool_; size_t num_samples_downscale_; size_t num_samples_upscale_; diff --git a/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc b/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc index fdec081c90..abe44ffa0d 100644 --- a/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc +++ b/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc @@ -16,7 +16,6 @@ namespace webrtc { namespace { static const int kNumSeconds = 10; static const int kWidth = 1920; -static const int kHalfWidth = kWidth / 2; static const int kHeight = 1080; static const int kFramerate = 30; static const int kLowQp = 15; @@ -43,14 +42,14 @@ class QualityScalerTest : public ::testing::Test { }; QualityScalerTest() { - input_frame_.CreateEmptyFrame(kWidth, kHeight, kWidth, kHalfWidth, - kHalfWidth); + input_frame_ = rtc::scoped_refptr( + new rtc::RefCountedObject(kWidth, kHeight)); qs_.Init(kLowQpThreshold, kHighQp, 0, 0, 0, kFramerate); - qs_.OnEncodeFrame(input_frame_); + qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); } bool TriggerScale(ScaleDirection scale_direction) { - qs_.OnEncodeFrame(input_frame_); + qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); int initial_width = qs_.GetScaledResolution().width; for (int i = 0; i < kFramerate * kNumSeconds; ++i) { switch (scale_direction) { @@ -67,7 +66,7 @@ class QualityScalerTest : public ::testing::Test { qs_.ReportQP(kHighQp + 1); break; } - qs_.OnEncodeFrame(input_frame_); + qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); if (qs_.GetScaledResolution().width != initial_width) return true; } @@ -76,16 +75,17 @@ class QualityScalerTest : public ::testing::Test { } void ExpectOriginalFrame() { - EXPECT_EQ(&input_frame_, &qs_.GetScaledFrame(input_frame_)) + EXPECT_EQ(input_frame_, qs_.GetScaledBuffer(input_frame_)) << "Using scaled frame instead of original input."; } void ExpectScaleUsingReportedResolution() { - qs_.OnEncodeFrame(input_frame_); + qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); QualityScaler::Resolution res = qs_.GetScaledResolution(); - const VideoFrame& scaled_frame = qs_.GetScaledFrame(input_frame_); - EXPECT_EQ(res.width, scaled_frame.width()); - EXPECT_EQ(res.height, scaled_frame.height()); + rtc::scoped_refptr scaled_frame = + qs_.GetScaledBuffer(input_frame_); + EXPECT_EQ(res.width, scaled_frame->width()); + EXPECT_EQ(res.height, scaled_frame->height()); } void ContinuouslyDownscalesByHalfDimensionsAndBackUp(); @@ -98,7 +98,7 @@ class QualityScalerTest : public ::testing::Test { int end_height); QualityScaler qs_; - VideoFrame input_frame_; + rtc::scoped_refptr input_frame_; }; TEST_F(QualityScalerTest, UsesOriginalFrameInitially) { @@ -106,34 +106,34 @@ TEST_F(QualityScalerTest, UsesOriginalFrameInitially) { } TEST_F(QualityScalerTest, ReportsOriginalResolutionInitially) { - qs_.OnEncodeFrame(input_frame_); + qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); QualityScaler::Resolution res = qs_.GetScaledResolution(); - EXPECT_EQ(input_frame_.width(), res.width); - EXPECT_EQ(input_frame_.height(), res.height); + EXPECT_EQ(input_frame_->width(), res.width); + EXPECT_EQ(input_frame_->height(), res.height); } TEST_F(QualityScalerTest, DownscalesAfterContinuousFramedrop) { EXPECT_TRUE(TriggerScale(kScaleDown)) << "No downscale within " << kNumSeconds << " seconds."; QualityScaler::Resolution res = qs_.GetScaledResolution(); - EXPECT_LT(res.width, input_frame_.width()); - EXPECT_LT(res.height, input_frame_.height()); + EXPECT_LT(res.width, input_frame_->width()); + EXPECT_LT(res.height, input_frame_->height()); } TEST_F(QualityScalerTest, KeepsScaleAtHighQp) { EXPECT_FALSE(TriggerScale(kKeepScaleAtHighQp)) << "Downscale at high threshold which should keep scale."; QualityScaler::Resolution res = qs_.GetScaledResolution(); - EXPECT_EQ(res.width, input_frame_.width()); - EXPECT_EQ(res.height, input_frame_.height()); + EXPECT_EQ(res.width, input_frame_->width()); + EXPECT_EQ(res.height, input_frame_->height()); } TEST_F(QualityScalerTest, DownscalesAboveHighQp) { EXPECT_TRUE(TriggerScale(kScaleDownAboveHighQp)) << "No downscale within " << kNumSeconds << " seconds."; QualityScaler::Resolution res = qs_.GetScaledResolution(); - EXPECT_LT(res.width, input_frame_.width()); - EXPECT_LT(res.height, input_frame_.height()); + EXPECT_LT(res.width, input_frame_->width()); + EXPECT_LT(res.height, input_frame_->height()); } TEST_F(QualityScalerTest, DownscalesAfterTwoThirdsFramedrop) { @@ -141,8 +141,8 @@ TEST_F(QualityScalerTest, DownscalesAfterTwoThirdsFramedrop) { qs_.ReportQP(kNormalQp); qs_.ReportDroppedFrame(); qs_.ReportDroppedFrame(); - qs_.OnEncodeFrame(input_frame_); - if (qs_.GetScaledResolution().width < input_frame_.width()) + qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); + if (qs_.GetScaledResolution().width < input_frame_->width()) return; } @@ -152,8 +152,8 @@ TEST_F(QualityScalerTest, DownscalesAfterTwoThirdsFramedrop) { TEST_F(QualityScalerTest, DoesNotDownscaleOnNormalQp) { for (int i = 0; i < kFramerate * kNumSeconds; ++i) { qs_.ReportQP(kNormalQp); - qs_.OnEncodeFrame(input_frame_); - ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution().width) + qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); + ASSERT_EQ(input_frame_->width(), qs_.GetScaledResolution().width) << "Unexpected scale on half framedrop."; } } @@ -161,33 +161,33 @@ TEST_F(QualityScalerTest, DoesNotDownscaleOnNormalQp) { TEST_F(QualityScalerTest, DoesNotDownscaleAfterHalfFramedrop) { for (int i = 0; i < kFramerate * kNumSeconds / 2; ++i) { qs_.ReportQP(kNormalQp); - qs_.OnEncodeFrame(input_frame_); - ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution().width) + qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); + ASSERT_EQ(input_frame_->width(), qs_.GetScaledResolution().width) << "Unexpected scale on half framedrop."; qs_.ReportDroppedFrame(); - qs_.OnEncodeFrame(input_frame_); - ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution().width) + qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); + ASSERT_EQ(input_frame_->width(), qs_.GetScaledResolution().width) << "Unexpected scale on half framedrop."; } } void QualityScalerTest::ContinuouslyDownscalesByHalfDimensionsAndBackUp() { - const int initial_min_dimension = input_frame_.width() < input_frame_.height() - ? input_frame_.width() - : input_frame_.height(); + const int initial_min_dimension = + input_frame_->width() < input_frame_->height() ? input_frame_->width() + : input_frame_->height(); int min_dimension = initial_min_dimension; int current_shift = 0; // Drop all frames to force-trigger downscaling. while (min_dimension >= 2 * kMinDownscaleDimension) { EXPECT_TRUE(TriggerScale(kScaleDown)) << "No downscale within " << kNumSeconds << " seconds."; - qs_.OnEncodeFrame(input_frame_); + qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); QualityScaler::Resolution res = qs_.GetScaledResolution(); min_dimension = res.width < res.height ? res.width : res.height; ++current_shift; - ASSERT_EQ(input_frame_.width() >> current_shift, res.width); - ASSERT_EQ(input_frame_.height() >> current_shift, res.height); + ASSERT_EQ(input_frame_->width() >> current_shift, res.width); + ASSERT_EQ(input_frame_->height() >> current_shift, res.height); ExpectScaleUsingReportedResolution(); } @@ -195,12 +195,12 @@ void QualityScalerTest::ContinuouslyDownscalesByHalfDimensionsAndBackUp() { while (min_dimension < initial_min_dimension) { EXPECT_TRUE(TriggerScale(kScaleUp)) << "No upscale within " << kNumSeconds << " seconds."; - qs_.OnEncodeFrame(input_frame_); + qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); QualityScaler::Resolution res = qs_.GetScaledResolution(); min_dimension = res.width < res.height ? res.width : res.height; --current_shift; - ASSERT_EQ(input_frame_.width() >> current_shift, res.width); - ASSERT_EQ(input_frame_.height() >> current_shift, res.height); + ASSERT_EQ(input_frame_->width() >> current_shift, res.width); + ASSERT_EQ(input_frame_->height() >> current_shift, res.height); ExpectScaleUsingReportedResolution(); } @@ -218,21 +218,20 @@ TEST_F(QualityScalerTest, ContinuouslyDownscalesByHalfDimensionsAndBackUp) { TEST_F(QualityScalerTest, ContinuouslyDownscalesOddResolutionsByHalfDimensionsAndBackUp) { const int kOddWidth = 517; - const int kHalfOddWidth = (kOddWidth + 1) / 2; const int kOddHeight = 1239; - input_frame_.CreateEmptyFrame(kOddWidth, kOddHeight, kOddWidth, kHalfOddWidth, - kHalfOddWidth); + input_frame_ = rtc::scoped_refptr( + new rtc::RefCountedObject(kOddWidth, kOddHeight)); ContinuouslyDownscalesByHalfDimensionsAndBackUp(); } void QualityScalerTest::DoesNotDownscaleFrameDimensions(int width, int height) { - input_frame_.CreateEmptyFrame(width, height, width, (width + 1) / 2, - (width + 1) / 2); + input_frame_ = rtc::scoped_refptr( + new rtc::RefCountedObject(width, height)); for (int i = 0; i < kFramerate * kNumSeconds; ++i) { qs_.ReportDroppedFrame(); - qs_.OnEncodeFrame(input_frame_); - ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution().width) + qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); + ASSERT_EQ(input_frame_->width(), qs_.GetScaledResolution().width) << "Unexpected scale of minimal-size frame."; } } @@ -263,11 +262,11 @@ TEST_F(QualityScalerTest, DownscaleToVgaOnLowInitialBitrate) { static const int kWidth720p = 1280; static const int kHeight720p = 720; static const int kInitialBitrateKbps = 300; - input_frame_.CreateEmptyFrame(kWidth720p, kHeight720p, kWidth720p, - kWidth720p / 2, kWidth720p / 2); + input_frame_ = rtc::scoped_refptr( + new rtc::RefCountedObject(kWidth720p, kHeight720p)); qs_.Init(kLowQpThreshold, kDisabledBadQpThreshold, kInitialBitrateKbps, kWidth720p, kHeight720p, kFramerate); - qs_.OnEncodeFrame(input_frame_); + qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); int init_width = qs_.GetScaledResolution().width; int init_height = qs_.GetScaledResolution().height; EXPECT_EQ(640, init_width); @@ -278,11 +277,11 @@ TEST_F(QualityScalerTest, DownscaleToQvgaOnLowerInitialBitrate) { static const int kWidth720p = 1280; static const int kHeight720p = 720; static const int kInitialBitrateKbps = 200; - input_frame_.CreateEmptyFrame(kWidth720p, kHeight720p, kWidth720p, - kWidth720p / 2, kWidth720p / 2); + input_frame_ = rtc::scoped_refptr( + new rtc::RefCountedObject(kWidth720p, kHeight720p)); qs_.Init(kLowQpThreshold, kDisabledBadQpThreshold, kInitialBitrateKbps, kWidth720p, kHeight720p, kFramerate); - qs_.OnEncodeFrame(input_frame_); + qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); int init_width = qs_.GetScaledResolution().width; int init_height = qs_.GetScaledResolution().height; EXPECT_EQ(320, init_width); @@ -291,13 +290,13 @@ TEST_F(QualityScalerTest, DownscaleToQvgaOnLowerInitialBitrate) { TEST_F(QualityScalerTest, DownscaleAfterMeasuredSecondsThenSlowerBackUp) { qs_.Init(kLowQpThreshold, kHighQp, 0, kWidth, kHeight, kFramerate); - qs_.OnEncodeFrame(input_frame_); + qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); QualityScaler::Resolution initial_res = qs_.GetScaledResolution(); // Should not downscale if less than kMeasureSecondsDownscale seconds passed. for (int i = 0; i < kFramerate * kMeasureSecondsDownscale - 1; ++i) { qs_.ReportQP(kHighQp + 1); - qs_.OnEncodeFrame(input_frame_); + qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); } EXPECT_EQ(initial_res.width, qs_.GetScaledResolution().width); EXPECT_EQ(initial_res.height, qs_.GetScaledResolution().height); @@ -305,7 +304,7 @@ TEST_F(QualityScalerTest, DownscaleAfterMeasuredSecondsThenSlowerBackUp) { // Should downscale if more than kMeasureSecondsDownscale seconds passed (add // last frame). qs_.ReportQP(kHighQp + 1); - qs_.OnEncodeFrame(input_frame_); + qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); EXPECT_GT(initial_res.width, qs_.GetScaledResolution().width); EXPECT_GT(initial_res.height, qs_.GetScaledResolution().height); @@ -313,7 +312,7 @@ TEST_F(QualityScalerTest, DownscaleAfterMeasuredSecondsThenSlowerBackUp) { // we saw issues initially (have already gone down). for (int i = 0; i < kFramerate * kMeasureSecondsUpscale - 1; ++i) { qs_.ReportQP(kLowQp); - qs_.OnEncodeFrame(input_frame_); + qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); } EXPECT_GT(initial_res.width, qs_.GetScaledResolution().width); EXPECT_GT(initial_res.height, qs_.GetScaledResolution().height); @@ -321,7 +320,7 @@ TEST_F(QualityScalerTest, DownscaleAfterMeasuredSecondsThenSlowerBackUp) { // Should upscale (back to initial) if kMeasureSecondsUpscale seconds passed // (add last frame). qs_.ReportQP(kLowQp); - qs_.OnEncodeFrame(input_frame_); + qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); EXPECT_EQ(initial_res.width, qs_.GetScaledResolution().width); EXPECT_EQ(initial_res.height, qs_.GetScaledResolution().height); } @@ -329,13 +328,13 @@ TEST_F(QualityScalerTest, DownscaleAfterMeasuredSecondsThenSlowerBackUp) { TEST_F(QualityScalerTest, UpscaleQuicklyInitiallyAfterMeasuredSeconds) { qs_.Init(kLowQpThreshold, kHighQp, kLowInitialBitrateKbps, kWidth, kHeight, kFramerate); - qs_.OnEncodeFrame(input_frame_); + qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); QualityScaler::Resolution initial_res = qs_.GetScaledResolution(); // Should not upscale if less than kMeasureSecondsFastUpscale seconds passed. for (int i = 0; i < kFramerate * kMeasureSecondsFastUpscale - 1; ++i) { qs_.ReportQP(kLowQp); - qs_.OnEncodeFrame(input_frame_); + qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); } EXPECT_EQ(initial_res.width, qs_.GetScaledResolution().width); EXPECT_EQ(initial_res.height, qs_.GetScaledResolution().height); @@ -343,7 +342,7 @@ TEST_F(QualityScalerTest, UpscaleQuicklyInitiallyAfterMeasuredSeconds) { // Should upscale if kMeasureSecondsFastUpscale seconds passed (add last // frame). qs_.ReportQP(kLowQp); - qs_.OnEncodeFrame(input_frame_); + qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); EXPECT_LT(initial_res.width, qs_.GetScaledResolution().width); EXPECT_LT(initial_res.height, qs_.GetScaledResolution().height); } @@ -354,8 +353,8 @@ void QualityScalerTest::DownscaleEndsAt(int input_width, int end_height) { // Create a frame with 2x expected end width/height to verify that we can // scale down to expected end width/height. - input_frame_.CreateEmptyFrame(input_width, input_height, input_width, - (input_width + 1) / 2, (input_width + 1) / 2); + input_frame_ = rtc::scoped_refptr( + new rtc::RefCountedObject(input_width, input_height)); int last_width = input_width; int last_height = input_height; diff --git a/webrtc/modules/video_processing/spatial_resampler.cc b/webrtc/modules/video_processing/spatial_resampler.cc index cdbe0efac1..74a570fb07 100644 --- a/webrtc/modules/video_processing/spatial_resampler.cc +++ b/webrtc/modules/video_processing/spatial_resampler.cc @@ -13,10 +13,7 @@ namespace webrtc { VPMSimpleSpatialResampler::VPMSimpleSpatialResampler() - : resampling_mode_(kFastRescaling), - target_width_(0), - target_height_(0), - scaler_() {} + : resampling_mode_(kFastRescaling), target_width_(0), target_height_(0) {} VPMSimpleSpatialResampler::~VPMSimpleSpatialResampler() {} @@ -56,26 +53,17 @@ int32_t VPMSimpleSpatialResampler::ResampleFrame(const VideoFrame& inFrame, return VPM_OK; } - // Setting scaler - // TODO(mikhal/marpan): Should we allow for setting the filter mode in - // _scale.Set() with |resampling_mode_|? - int ret_val = 0; - ret_val = scaler_.Set(inFrame.width(), inFrame.height(), target_width_, - target_height_, kI420, kI420, kScaleBox); - if (ret_val < 0) - return ret_val; + rtc::scoped_refptr scaled_buffer( + buffer_pool_.CreateBuffer(target_width_, target_height_)); - ret_val = scaler_.Scale(inFrame, outFrame); + scaled_buffer->CropAndScaleFrom(inFrame.video_frame_buffer()); + outFrame->set_video_frame_buffer(scaled_buffer); // Setting time parameters to the output frame. - // Timestamp will be reset in Scale call above, so we should set it after. outFrame->set_timestamp(inFrame.timestamp()); outFrame->set_render_time_ms(inFrame.render_time_ms()); - if (ret_val == 0) - return VPM_OK; - else - return VPM_SCALE_ERROR; + return VPM_OK; } int32_t VPMSimpleSpatialResampler::TargetHeight() { diff --git a/webrtc/modules/video_processing/spatial_resampler.h b/webrtc/modules/video_processing/spatial_resampler.h index 51820e24e5..66f7bbc887 100644 --- a/webrtc/modules/video_processing/spatial_resampler.h +++ b/webrtc/modules/video_processing/spatial_resampler.h @@ -16,8 +16,8 @@ #include "webrtc/modules/include/module_common_types.h" #include "webrtc/modules/video_processing/include/video_processing_defines.h" -#include "webrtc/common_video/libyuv/include/scaler.h" -#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" +#include "webrtc/common_video/include/i420_buffer_pool.h" +#include "webrtc/video_frame.h" namespace webrtc { @@ -52,7 +52,7 @@ class VPMSimpleSpatialResampler : public VPMSpatialResampler { VideoFrameResampling resampling_mode_; int32_t target_width_; int32_t target_height_; - Scaler scaler_; + I420BufferPool buffer_pool_; }; } // namespace webrtc diff --git a/webrtc/modules/video_processing/video_denoiser.cc b/webrtc/modules/video_processing/video_denoiser.cc index f116f882cd..a7aa7d3b60 100644 --- a/webrtc/modules/video_processing/video_denoiser.cc +++ b/webrtc/modules/video_processing/video_denoiser.cc @@ -8,7 +8,6 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "webrtc/common_video/libyuv/include/scaler.h" #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" #include "webrtc/modules/video_processing/video_denoiser.h"