admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Reland of Make QualityScaler more responsive to downgrades. (patchset #1 id:1 of https://codereview.webrtc.org/1880103002/ )

Browse Source 
Reason for revert:
Regressed behavior is actually desirable (go down to 360p instead of producing super-bad 720p).

Original issue's description:
> Revert of Make QualityScaler more responsive to downgrades. (patchset #3 id:40001 of https://codereview.webrtc.org/1830593003/ )
>
> Reason for revert:
> Speculative revert: want to see if this causes the regression in https://crbug.com/602621
>
> Original issue's description:
> > Make QualityScaler more responsive to downgrades.
> >
> > Permits going from HD to QVGA in 6 seconds instead of 10. Also adds
> > windows for going up quickly in the beginning of a call (before any
> > downscaling happens due to bad quality).
> >
> > BUG=webrtc:5678
> > R=glaznev@webrtc.org, stefan@webrtc.org
> >
> > Committed: https://crrev.com/85829fd90cc4e7a91c9857921b19e8fc126aeb60
> > Cr-Commit-Position: refs/heads/master@{#12219}
>
> TBR=glaznev@webrtc.org,stefan@webrtc.org,pbos@webrtc.org
> # Not skipping CQ checks because original CL landed more than 1 days ago.
> BUG=webrtc:5678
> NOTRY=true
>
> Committed: https://crrev.com/19b4fecf08e3fe215e431a260fb673553c15e569
> Cr-Commit-Position: refs/heads/master@{#12331}

TBR=glaznev@webrtc.org,stefan@webrtc.org,phoglund@webrtc.org
# Skipping CQ checks because original CL landed less than 1 days ago.
NOPRESUBMIT=true
NOTREECHECKS=true
NOTRY=true
BUG=chromium:602621, webrtc:5678

Review URL: https://codereview.webrtc.org/1887493003

Cr-Commit-Position: refs/heads/master@{#12341}
This commit is contained in:
pbos 2016-04-13 02:51:02 -07:00 committed by Commit bot
parent afaae0d151
commit cbac40d321
5 changed files with 146 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
webrtc/api/java/jni/androidmediaencoder_jni.cc
View File

@ -380,15 +380,17 @@ int32_t MediaCodecVideoEncoder::InitEncode(
const int kLowQpThreshold = 29;
const int kBadQpThreshold = 90;
quality_scaler_.Init(kLowQpThreshold, kBadQpThreshold, false,
codec_settings->startBitrate,
codec_settings->width, codec_settings->height);
codec_settings->startBitrate, codec_settings->width,
codec_settings->height,
codec_settings->maxFramerate);
} else if (codecType_ == kVideoCodecH264) {
// H264 QP is in the range [0, 51].
const int kLowQpThreshold = 21;
const int kBadQpThreshold = 33;
quality_scaler_.Init(kLowQpThreshold, kBadQpThreshold, false,
codec_settings->startBitrate,
codec_settings->width, codec_settings->height);
codec_settings->startBitrate, codec_settings->width,
codec_settings->height,
codec_settings->maxFramerate);
} else {
// When adding codec support to additional hardware codecs, also configure
// their QP thresholds for scaling.
@ -396,7 +398,6 @@ int32_t MediaCodecVideoEncoder::InitEncode(
scale_ = false;
}
quality_scaler_.SetMinResolution(kMinDimension, kMinDimension);
quality_scaler_.ReportFramerate(codec_settings->maxFramerate);
QualityScaler::Resolution res = quality_scaler_.GetScaledResolution();
init_width = std::max(res.width, kMinDimension);
init_height = std::max(res.height, kMinDimension);

4
webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc
View File

@ -604,8 +604,8 @@ int VP8EncoderImpl::InitEncode(const VideoCodec* inst,
// TODO(glaznev/sprang): consider passing codec initial bitrate to quality
// scaler to avoid starting with HD for low initial bitrates.
quality_scaler_.Init(codec_.qpMax / QualityScaler::kDefaultLowQpDenominator,
kDisabledBadQpThreshold, false, 0, 0, 0);
quality_scaler_.ReportFramerate(codec_.maxFramerate);
kDisabledBadQpThreshold, false, 0, 0, 0,
codec_.maxFramerate);
// Only apply scaling to improve for single-layer streams. The scaling metrics
// use frame drops as a signal and is only applicable when we drop frames.

49
webrtc/modules/video_coding/utility/quality_scaler.cc
View File

@ -11,8 +11,11 @@
namespace webrtc {
static const int kMinFps = 10;
static const int kMeasureSeconds = 5;
static const int kMinFps = 5;
static const int kMeasureSecondsDownscale = 3;
// Threshold constant used until first downscale (to permit fast rampup).
static const int kMeasureSecondsFastUpscale = 2;
static const int kMeasureSecondsUpscale = 5;
static const int kFramedropPercentThreshold = 60;
static const int kHdResolutionThreshold = 700 * 500;
static const int kHdBitrateThresholdKbps = 500;
@ -23,9 +26,7 @@ const int QualityScaler::kDefaultLowQpDenominator = 3;
const int QualityScaler::kDefaultMinDownscaleDimension = 90;
QualityScaler::QualityScaler()
: num_samples_(0),
low_qp_threshold_(-1),
downscale_shift_(0),
: low_qp_threshold_(-1),
framerate_down_(false),
min_width_(kDefaultMinDownscaleDimension),
min_height_(kDefaultMinDownscaleDimension) {}
@ -35,12 +36,17 @@ void QualityScaler::Init(int low_qp_threshold,
bool use_framerate_reduction,
int initial_bitrate_kbps,
int width,
int height) {
int height,
int fps) {
ClearSamples();
low_qp_threshold_ = low_qp_threshold;
high_qp_threshold_ = high_qp_threshold;
use_framerate_reduction_ = use_framerate_reduction;
downscale_shift_ = 0;
// Use a faster window for upscaling initially (but be more graceful later).
// This enables faster initial rampups without risking strong up-down
// behavior later.
measure_seconds_upscale_ = kMeasureSecondsFastUpscale;
const int init_width = width;
const int init_height = height;
// TODO(glaznev): Investigate using thresholds for other resolutions
@ -55,6 +61,7 @@ void QualityScaler::Init(int low_qp_threshold,
}
}
UpdateTargetResolution(init_width, init_height);
ReportFramerate(fps);
target_framerate_ = -1;
}
@ -65,14 +72,14 @@ void QualityScaler::SetMinResolution(int min_width, int min_height) {
// Report framerate(fps) to estimate # of samples.
void QualityScaler::ReportFramerate(int framerate) {
num_samples_ = static_cast<size_t>(
kMeasureSeconds * (framerate < kMinFps ? kMinFps : framerate));
framerate_ = framerate;
UpdateSampleCounts();
}
void QualityScaler::ReportQP(int qp) {
framedrop_percent_.AddSample(0);
average_qp_.AddSample(qp);
average_qp_downscale_.AddSample(qp);
average_qp_upscale_.AddSample(qp);
}
void QualityScaler::ReportDroppedFrame() {
@ -82,7 +89,8 @@ void QualityScaler::ReportDroppedFrame() {
void QualityScaler::OnEncodeFrame(const VideoFrame& frame) {
// Should be set through InitEncode -> Should be set by now.
assert(low_qp_threshold_ >= 0);
assert(num_samples_ > 0);
assert(num_samples_upscale_ > 0);
assert(num_samples_downscale_ > 0);
// Update scale factor.
int avg_drop = 0;
@ -90,9 +98,9 @@ void QualityScaler::OnEncodeFrame(const VideoFrame& frame) {
// When encoder consistently overshoots, framerate reduction and spatial
// resizing will be triggered to get a smoother video.
if ((framedrop_percent_.GetAverage(num_samples_, &avg_drop) &&
if ((framedrop_percent_.GetAverage(num_samples_downscale_, &avg_drop) &&
avg_drop >= kFramedropPercentThreshold) ||
(average_qp_.GetAverage(num_samples_, &avg_qp) &&
(average_qp_downscale_.GetAverage(num_samples_downscale_, &avg_qp) &&
avg_qp > high_qp_threshold_)) {
// Reducing frame rate before spatial resolution change.
// Reduce frame rate only when it is above a certain number.
@ -107,7 +115,7 @@ void QualityScaler::OnEncodeFrame(const VideoFrame& frame) {
} else {
AdjustScale(false);
}
} else if (average_qp_.GetAverage(num_samples_, &avg_qp) &&
} else if (average_qp_upscale_.GetAverage(num_samples_upscale_, &avg_qp) &&
avg_qp <= low_qp_threshold_) {
if (use_framerate_reduction_ && framerate_down_) {
target_framerate_ = -1;
@ -160,13 +168,26 @@ void QualityScaler::UpdateTargetResolution(int frame_width, int frame_height) {
void QualityScaler::ClearSamples() {
framedrop_percent_.Reset();
average_qp_.Reset();
average_qp_downscale_.Reset();
average_qp_upscale_.Reset();
}
void QualityScaler::UpdateSampleCounts() {
num_samples_downscale_ = static_cast<size_t>(
kMeasureSecondsDownscale * (framerate_ < kMinFps ? kMinFps : framerate_));
num_samples_upscale_ = static_cast<size_t>(
measure_seconds_upscale_ * (framerate_ < kMinFps ? kMinFps : framerate_));
}
void QualityScaler::AdjustScale(bool up) {
downscale_shift_ += up ? -1 : 1;
if (downscale_shift_ < 0)
downscale_shift_ = 0;
if (!up) {
// Hit first downscale, start using a slower threshold for going up.
measure_seconds_upscale_ = kMeasureSecondsUpscale;
UpdateSampleCounts();
}
ClearSamples();
}

12
webrtc/modules/video_coding/utility/quality_scaler.h
View File

@ -30,7 +30,8 @@ class QualityScaler {
bool use_framerate_reduction,
int initial_bitrate_kbps,
int width,
int height);
int height,
int fps);
void SetMinResolution(int min_width, int min_height);
void ReportFramerate(int framerate);
void ReportQP(int qp);
@ -46,17 +47,22 @@ class QualityScaler {
void AdjustScale(bool up);
void UpdateTargetResolution(int frame_width, int frame_height);
void ClearSamples();
void UpdateSampleCounts();
Scaler scaler_;
VideoFrame scaled_frame_;
size_t num_samples_;
size_t num_samples_downscale_;
size_t num_samples_upscale_;
int measure_seconds_upscale_;
MovingAverage<int> average_qp_upscale_;
MovingAverage<int> average_qp_downscale_;
int framerate_;
int target_framerate_;
int low_qp_threshold_;
int high_qp_threshold_;
MovingAverage<int> framedrop_percent_;
MovingAverage<int> average_qp_;
Resolution res_;
int downscale_shift_;

113
webrtc/modules/video_coding/utility/quality_scaler_unittest.cc
View File

@ -27,6 +27,11 @@ static const int kHighQp = 40;
static const int kMaxQp = 56;
static const int kDisabledBadQpThreshold = kMaxQp + 1;
static const int kLowInitialBitrateKbps = 300;
// These values need to be in sync with corresponding constants
// in quality_scaler.cc
static const int kMeasureSecondsDownscale = 3;
static const int kMeasureSecondsFastUpscale = 2;
static const int kMeasureSecondsUpscale = 5;
} // namespace
class QualityScalerTest : public ::testing::Test {
@ -51,8 +56,7 @@ class QualityScalerTest : public ::testing::Test {
input_frame_.CreateEmptyFrame(kWidth, kHeight, kWidth, kHalfWidth,
kHalfWidth);
qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator, kHighQp, false,
0, 0, 0);
qs_.ReportFramerate(kFramerate);
0, 0, 0, kFramerate);
qs_.OnEncodeFrame(input_frame_);
}
@ -104,7 +108,8 @@ class QualityScalerTest : public ::testing::Test {
int initial_framerate);
void VerifyQualityAdaptation(int initial_framerate,
int seconds,
int seconds_downscale,
int seconds_upscale,
bool expect_spatial_resize,
bool expect_framerate_reduction);
@ -298,18 +303,19 @@ QualityScalerTest::Resolution QualityScalerTest::TriggerResolutionChange(
void QualityScalerTest::VerifyQualityAdaptation(
int initial_framerate,
int seconds,
int seconds_downscale,
int seconds_upscale,
bool expect_spatial_resize,
bool expect_framerate_reduction) {
qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator,
kDisabledBadQpThreshold, true, 0, 0, 0);
kDisabledBadQpThreshold, true, 0, 0, 0, initial_framerate);
qs_.OnEncodeFrame(input_frame_);
int init_width = qs_.GetScaledResolution().width;
int init_height = qs_.GetScaledResolution().height;
// Test reducing framerate by dropping frame continuously.
QualityScalerTest::Resolution res =
TriggerResolutionChange(kDropFrame, seconds, initial_framerate);
TriggerResolutionChange(kDropFrame, seconds_downscale, initial_framerate);
if (expect_framerate_reduction) {
EXPECT_LT(res.framerate, initial_framerate);
@ -327,35 +333,40 @@ void QualityScalerTest::VerifyQualityAdaptation(
}
// The "seconds * 1.5" is to ensure spatial resolution to recover.
// For example, in 10 seconds test, framerate reduction happens in the first
// 5 seconds from 30fps to 15fps and causes the buffer size to be half of the
// original one. Then it will take only 75 samples to downscale (twice in 150
// For example, in 6 seconds test, framerate reduction happens in the first
// 3 seconds from 30fps to 15fps and causes the buffer size to be half of the
// original one. Then it will take only 45 samples to downscale (twice in 90
// samples). So to recover the resolution changes, we need more than 10
// seconds (i.e, seconds * 1.5). This is because the framerate increases
// before spatial size recovers, so it will take 150 samples to recover
// spatial size (300 for twice).
res = TriggerResolutionChange(kReportLowQP, seconds * 1.5, initial_framerate);
// seconds (i.e, seconds_upscale * 1.5). This is because the framerate
// increases before spatial size recovers, so it will take 150 samples to
// recover spatial size (300 for twice).
res = TriggerResolutionChange(kReportLowQP, seconds_upscale * 1.5,
initial_framerate);
EXPECT_EQ(-1, res.framerate);
EXPECT_EQ(init_width, res.width);
EXPECT_EQ(init_height, res.height);
}
// In 5 seconds test, only framerate adjusting should happen.
// In 3 seconds test, only framerate adjusting should happen and 5 second
// upscaling duration, only a framerate adjusting should happen.
TEST_F(QualityScalerTest, ChangeFramerateOnly) {
VerifyQualityAdaptation(kFramerate, 5, false, true);
VerifyQualityAdaptation(kFramerate, kMeasureSecondsDownscale,
kMeasureSecondsUpscale, false, true);
}
// In 10 seconds test, framerate adjusting and scaling are both
// In 6 seconds test, framerate adjusting and scaling are both
// triggered, it shows that scaling would happen after framerate
// adjusting.
TEST_F(QualityScalerTest, ChangeFramerateAndSpatialSize) {
VerifyQualityAdaptation(kFramerate, 10, true, true);
VerifyQualityAdaptation(kFramerate, kMeasureSecondsDownscale * 2,
kMeasureSecondsUpscale * 2, true, true);
}
// When starting from a low framerate, only spatial size will be changed.
TEST_F(QualityScalerTest, ChangeSpatialSizeOnly) {
qs_.ReportFramerate(kFramerate >> 1);
VerifyQualityAdaptation(kFramerate >> 1, 10, true, false);
VerifyQualityAdaptation(kFramerate >> 1, kMeasureSecondsDownscale * 2,
kMeasureSecondsUpscale * 2, true, false);
}
TEST_F(QualityScalerTest, DoesNotDownscaleBelow2xDefaultMinDimensionsWidth) {
@ -371,7 +382,7 @@ TEST_F(QualityScalerTest, DoesNotDownscaleBelow2xDefaultMinDimensionsHeight) {
TEST_F(QualityScalerTest, DownscaleToVgaOnLowInitialBitrate) {
qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator,
kDisabledBadQpThreshold, true,
kLowInitialBitrateKbps, kWidth, kHeight);
kLowInitialBitrateKbps, kWidth, kHeight, kFramerate);
qs_.OnEncodeFrame(input_frame_);
int init_width = qs_.GetScaledResolution().width;
int init_height = qs_.GetScaledResolution().height;
@ -379,6 +390,70 @@ TEST_F(QualityScalerTest, DownscaleToVgaOnLowInitialBitrate) {
EXPECT_LE(init_height, kHeightVga);
}
TEST_F(QualityScalerTest, DownscaleAfterMeasuredSecondsThenSlowerBackUp) {
QualityScalerTest::Resolution initial_res;
qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator, kHighQp, false, 0,
kWidth, kHeight, kFramerate);
qs_.OnEncodeFrame(input_frame_);
initial_res.width = qs_.GetScaledResolution().width;
initial_res.height = qs_.GetScaledResolution().height;
// Should not downscale if less than kMeasureSecondsDownscale seconds passed.
for (int i = 0; i < kFramerate * kMeasureSecondsDownscale - 1; ++i) {
qs_.ReportQP(kHighQp + 1);
qs_.OnEncodeFrame(input_frame_);
}
EXPECT_EQ(initial_res.width, qs_.GetScaledResolution().width);
EXPECT_EQ(initial_res.height, qs_.GetScaledResolution().height);
// Should downscale if more than kMeasureSecondsDownscale seconds passed (add
// last frame).
qs_.ReportQP(kHighQp + 1);
qs_.OnEncodeFrame(input_frame_);
EXPECT_GT(initial_res.width, qs_.GetScaledResolution().width);
EXPECT_GT(initial_res.height, qs_.GetScaledResolution().height);
// Should not upscale if less than kMeasureSecondsUpscale seconds passed since
// we saw issues initially (have already gone down).
for (int i = 0; i < kFramerate * kMeasureSecondsUpscale - 1; ++i) {
qs_.ReportQP(kLowQp);
qs_.OnEncodeFrame(input_frame_);
}
EXPECT_GT(initial_res.width, qs_.GetScaledResolution().width);
EXPECT_GT(initial_res.height, qs_.GetScaledResolution().height);
// Should upscale (back to initial) if kMeasureSecondsUpscale seconds passed
// (add last frame).
qs_.ReportQP(kLowQp);
qs_.OnEncodeFrame(input_frame_);
EXPECT_EQ(initial_res.width, qs_.GetScaledResolution().width);
EXPECT_EQ(initial_res.height, qs_.GetScaledResolution().height);
}
TEST_F(QualityScalerTest, UpscaleQuicklyInitiallyAfterMeasuredSeconds) {
QualityScalerTest::Resolution initial_res;
qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator, kHighQp, false,
kLowInitialBitrateKbps, kWidth, kHeight, kFramerate);
qs_.OnEncodeFrame(input_frame_);
initial_res.width = qs_.GetScaledResolution().width;
initial_res.height = qs_.GetScaledResolution().height;
// Should not upscale if less than kMeasureSecondsFastUpscale seconds passed.
for (int i = 0; i < kFramerate * kMeasureSecondsFastUpscale - 1; ++i) {
qs_.ReportQP(kLowQp);
qs_.OnEncodeFrame(input_frame_);
}
EXPECT_EQ(initial_res.width, qs_.GetScaledResolution().width);
EXPECT_EQ(initial_res.height, qs_.GetScaledResolution().height);
// Should upscale if kMeasureSecondsFastUpscale seconds passed (add last
// frame).
qs_.ReportQP(kLowQp);
qs_.OnEncodeFrame(input_frame_);
EXPECT_LT(initial_res.width, qs_.GetScaledResolution().width);
EXPECT_LT(initial_res.height, qs_.GetScaledResolution().height);
}
void QualityScalerTest::DownscaleEndsAt(int input_width,
int input_height,
int end_width,