admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Reland "Parse encoded frame QP if not provided by encoder"

Browse Source 
This reverts commit 727d2afc4330efebc904e0e4f366e885d7b08787.

Reason for revert: Use thread-safe wrapper for H264 parser.

Original change's description:
> Revert "Parse encoded frame QP if not provided by encoder"
>
> This reverts commit 8639673f0c098efc294a7593fa3bd98e28ab7508.
>
> Reason for revert: linux_tsan fails https://ci.chromium.org/ui/p/webrtc/builders/ci/Linux%20Tsan%20v2/25329/overview
>
> Original change's description:
> > Parse encoded frame QP if not provided by encoder
> >
> > Bug: webrtc:12542
> > Change-Id: Ic70b46e226f158db7a478a9f20e1f940804febba
> > Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/210966
> > Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
> > Reviewed-by: Åsa Persson <asapersson@webrtc.org>
> > Cr-Commit-Position: refs/heads/master@{#33434}
>
> TBR=asapersson@webrtc.org,ssilkin@webrtc.org
>
> Change-Id: Ie251d8f70f8e87fd86b63730aefd2ef3f941e4bb
> No-Presubmit: true
> No-Tree-Checks: true
> No-Try: true
> Bug: webrtc:12542
> Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/211355
> Reviewed-by: Sergey Silkin <ssilkin@webrtc.org>
> Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
> Cr-Commit-Position: refs/heads/master@{#33441}

# Not skipping CQ checks because this is a reland.

Bug: webrtc:12542
Change-Id: Ib7601fd6f2f26bceddbea2b4ba54d67a281f3a59
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/211660
Reviewed-by: Sergey Silkin <ssilkin@webrtc.org>
Reviewed-by: Åsa Persson <asapersson@webrtc.org>
Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#33458}
This commit is contained in:
Sergey Silkin 2021-03-15 10:12:57 +01:00 committed by Commit Bot
parent b6bc357a1e
commit 0e42cf703b
7 changed files with 318 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
modules/video_coding/BUILD.gn
View File

@ -323,6 +323,8 @@ rtc_library("video_coding_utility") {
"utility/ivf_file_reader.h",
"utility/ivf_file_writer.cc",
"utility/ivf_file_writer.h",
"utility/qp_parser.cc",
"utility/qp_parser.h",
"utility/quality_scaler.cc",
"utility/quality_scaler.h",
"utility/simulcast_rate_allocator.cc",
@ -344,6 +346,7 @@ rtc_library("video_coding_utility") {
"../../api/video:video_adaptation",
"../../api/video:video_bitrate_allocation",
"../../api/video:video_bitrate_allocator",
"../../api/video:video_codec_constants",
"../../api/video:video_frame",
"../../api/video_codecs:video_codecs_api",
"../../common_video",
@ -357,6 +360,7 @@ rtc_library("video_coding_utility") {
"../../rtc_base/experiments:quality_scaling_experiment",
"../../rtc_base/experiments:rate_control_settings",
"../../rtc_base/experiments:stable_target_rate_experiment",
"../../rtc_base/synchronization:mutex",
"../../rtc_base/system:arch",
"../../rtc_base/system:file_wrapper",
"../../rtc_base/system:no_unique_address",
@ -969,6 +973,7 @@ if (rtc_include_tests) {
"utility/framerate_controller_unittest.cc",
"utility/ivf_file_reader_unittest.cc",
"utility/ivf_file_writer_unittest.cc",
"utility/qp_parser_unittest.cc",
"utility/quality_scaler_unittest.cc",
"utility/simulcast_rate_allocator_unittest.cc",
"video_codec_initializer_unittest.cc",

53
modules/video_coding/utility/qp_parser.cc Normal file
View File

@ -0,0 +1,53 @@
/*
* Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/utility/qp_parser.h"
#include "modules/video_coding/utility/vp8_header_parser.h"
#include "modules/video_coding/utility/vp9_uncompressed_header_parser.h"
namespace webrtc {
absl::optional<uint32_t> QpParser::Parse(VideoCodecType codec_type,
size_t spatial_idx,
const uint8_t* frame_data,
size_t frame_size) {
if (frame_data == nullptr || frame_size == 0 ||
spatial_idx >= kMaxSimulcastStreams) {
return absl::nullopt;
}
if (codec_type == kVideoCodecVP8) {
int qp = -1;
if (vp8::GetQp(frame_data, frame_size, &qp)) {
return qp;
}
} else if (codec_type == kVideoCodecVP9) {
int qp = -1;
if (vp9::GetQp(frame_data, frame_size, &qp)) {
return qp;
}
} else if (codec_type == kVideoCodecH264) {
return h264_parsers_[spatial_idx].Parse(frame_data, frame_size);
}
return absl::nullopt;
}
absl::optional<uint32_t> QpParser::H264QpParser::Parse(
const uint8_t* frame_data,
size_t frame_size) {
MutexLock lock(&mutex_);
bitstream_parser_.ParseBitstream(
rtc::ArrayView<const uint8_t>(frame_data, frame_size));
return bitstream_parser_.GetLastSliceQp();
}
} // namespace webrtc

45
modules/video_coding/utility/qp_parser.h Normal file
View File

@ -0,0 +1,45 @@
/*
* Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_VIDEO_CODING_UTILITY_QP_PARSER_H_
#define MODULES_VIDEO_CODING_UTILITY_QP_PARSER_H_
#include "absl/types/optional.h"
#include "api/video/video_codec_constants.h"
#include "api/video/video_codec_type.h"
#include "common_video/h264/h264_bitstream_parser.h"
#include "rtc_base/synchronization/mutex.h"
namespace webrtc {
class QpParser {
public:
absl::optional<uint32_t> Parse(VideoCodecType codec_type,
size_t spatial_idx,
const uint8_t* frame_data,
size_t frame_size);
private:
// A thread safe wrapper for H264 bitstream parser.
class H264QpParser {
public:
absl::optional<uint32_t> Parse(const uint8_t* frame_data,
size_t frame_size);
private:
Mutex mutex_;
H264BitstreamParser bitstream_parser_ RTC_GUARDED_BY(mutex_);
};
H264QpParser h264_parsers_[kMaxSimulcastStreams];
};
} // namespace webrtc
#endif // MODULES_VIDEO_CODING_UTILITY_QP_PARSER_H_

118
modules/video_coding/utility/qp_parser_unittest.cc Normal file
View File

@ -0,0 +1,118 @@
/*
* Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/utility/qp_parser.h"
#include <stddef.h>
#include "test/gtest.h"
namespace webrtc {
namespace {
// ffmpeg -s 16x16 -f rawvideo -pix_fmt rgb24 -r 30 -i /dev/zero -c:v libvpx
// -qmin 20 -qmax 20 -crf 20 -frames:v 1 -y out.ivf
const uint8_t kCodedFrameVp8Qp25[] = {
0x10, 0x02, 0x00, 0x9d, 0x01, 0x2a, 0x10, 0x00, 0x10, 0x00,
0x02, 0x47, 0x08, 0x85, 0x85, 0x88, 0x85, 0x84, 0x88, 0x0c,
0x82, 0x00, 0x0c, 0x0d, 0x60, 0x00, 0xfe, 0xfc, 0x5c, 0xd0};
// ffmpeg -s 16x16 -f rawvideo -pix_fmt rgb24 -r 30 -i /dev/zero -c:v libvpx-vp9
// -qmin 24 -qmax 24 -crf 24 -frames:v 1 -y out.ivf
const uint8_t kCodedFrameVp9Qp96[] = {
0xa2, 0x49, 0x83, 0x42, 0xe0, 0x00, 0xf0, 0x00, 0xf6, 0x00,
0x38, 0x24, 0x1c, 0x18, 0xc0, 0x00, 0x00, 0x30, 0x70, 0x00,
0x00, 0x4a, 0xa7, 0xff, 0xfc, 0xb9, 0x01, 0xbf, 0xff, 0xff,
0x97, 0x20, 0xdb, 0xff, 0xff, 0xcb, 0x90, 0x5d, 0x40};
// ffmpeg -s 16x16 -f rawvideo -pix_fmt yuv420p -r 30 -i /dev/zero -c:v libx264
// -qmin 38 -qmax 38 -crf 38 -profile:v baseline -frames:v 2 -y out.264
const uint8_t kCodedFrameH264SpsPpsIdrQp38[] = {
0x00, 0x00, 0x00, 0x01, 0x67, 0x42, 0xc0, 0x0a, 0xd9, 0x1e, 0x84,
0x00, 0x00, 0x03, 0x00, 0x04, 0x00, 0x00, 0x03, 0x00, 0xf0, 0x3c,
0x48, 0x99, 0x20, 0x00, 0x00, 0x00, 0x01, 0x68, 0xcb, 0x80, 0xc4,
0xb2, 0x00, 0x00, 0x01, 0x65, 0x88, 0x84, 0xf1, 0x18, 0xa0, 0x00,
0x20, 0x5b, 0x1c, 0x00, 0x04, 0x07, 0xe3, 0x80, 0x00, 0x80, 0xfe};
const uint8_t kCodedFrameH264SpsPpsIdrQp49[] = {
0x00, 0x00, 0x00, 0x01, 0x67, 0x42, 0xc0, 0x0a, 0xd9, 0x1e, 0x84,
0x00, 0x00, 0x03, 0x00, 0x04, 0x00, 0x00, 0x03, 0x00, 0xf0, 0x3c,
0x48, 0x99, 0x20, 0x00, 0x00, 0x00, 0x01, 0x68, 0xcb, 0x80, 0x5d,
0x2c, 0x80, 0x00, 0x00, 0x01, 0x65, 0x88, 0x84, 0xf1, 0x18, 0xa0,
0x00, 0x5e, 0x38, 0x00, 0x08, 0x03, 0xc7, 0x00, 0x01, 0x00, 0x7c};
const uint8_t kCodedFrameH264InterSliceQpDelta0[] = {0x00, 0x00, 0x00, 0x01,
0x41, 0x9a, 0x39, 0xea};
} // namespace
TEST(QpParserTest, ParseQpVp8) {
QpParser parser;
absl::optional<uint32_t> qp = parser.Parse(
kVideoCodecVP8, 0, kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25));
EXPECT_EQ(qp, 25u);
}
TEST(QpParserTest, ParseQpVp9) {
QpParser parser;
absl::optional<uint32_t> qp = parser.Parse(
kVideoCodecVP9, 0, kCodedFrameVp9Qp96, sizeof(kCodedFrameVp9Qp96));
EXPECT_EQ(qp, 96u);
}
TEST(QpParserTest, ParseQpH264) {
QpParser parser;
absl::optional<uint32_t> qp = parser.Parse(
VideoCodecType::kVideoCodecH264, 0, kCodedFrameH264SpsPpsIdrQp38,
sizeof(kCodedFrameH264SpsPpsIdrQp38));
EXPECT_EQ(qp, 38u);
qp = parser.Parse(kVideoCodecH264, 1, kCodedFrameH264SpsPpsIdrQp49,
sizeof(kCodedFrameH264SpsPpsIdrQp49));
EXPECT_EQ(qp, 49u);
qp = parser.Parse(kVideoCodecH264, 0, kCodedFrameH264InterSliceQpDelta0,
sizeof(kCodedFrameH264InterSliceQpDelta0));
EXPECT_EQ(qp, 38u);
qp = parser.Parse(kVideoCodecH264, 1, kCodedFrameH264InterSliceQpDelta0,
sizeof(kCodedFrameH264InterSliceQpDelta0));
EXPECT_EQ(qp, 49u);
}
TEST(QpParserTest, ParseQpUnsupportedCodecType) {
QpParser parser;
absl::optional<uint32_t> qp = parser.Parse(
kVideoCodecGeneric, 0, kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25));
EXPECT_FALSE(qp.has_value());
}
TEST(QpParserTest, ParseQpNullData) {
QpParser parser;
absl::optional<uint32_t> qp = parser.Parse(kVideoCodecVP8, 0, nullptr, 100);
EXPECT_FALSE(qp.has_value());
}
TEST(QpParserTest, ParseQpEmptyData) {
QpParser parser;
absl::optional<uint32_t> qp =
parser.Parse(kVideoCodecVP8, 0, kCodedFrameVp8Qp25, 0);
EXPECT_FALSE(qp.has_value());
}
TEST(QpParserTest, ParseQpSpatialIdxExceedsMax) {
QpParser parser;
absl::optional<uint32_t> qp =
parser.Parse(kVideoCodecVP8, kMaxSimulcastStreams, kCodedFrameVp8Qp25,
sizeof(kCodedFrameVp8Qp25));
EXPECT_FALSE(qp.has_value());
}
} // namespace webrtc

21
video/video_stream_encoder.cc
View File

@ -656,6 +656,8 @@ VideoStreamEncoder::VideoStreamEncoder(
/*source=*/nullptr),
default_limits_allowed_(
!field_trial::IsEnabled("WebRTC-DefaultBitrateLimitsKillSwitch")),
qp_parsing_allowed_(
!field_trial::IsEnabled("WebRTC-QpParsingKillSwitch")),
encoder_queue_(task_queue_factory->CreateTaskQueue(
"EncoderQueue",
TaskQueueFactory::Priority::NORMAL)) {
@ -1855,6 +1857,18 @@ EncodedImageCallback::Result VideoStreamEncoder::OnEncodedImage(
frame_encode_metadata_writer_.UpdateBitstream(codec_specific_info,
&image_copy);
VideoCodecType codec_type = codec_specific_info
? codec_specific_info->codecType
: VideoCodecType::kVideoCodecGeneric;
if (image_copy.qp_ < 0 && qp_parsing_allowed_) {
// Parse encoded frame QP if that was not provided by encoder.
image_copy.qp_ = qp_parser_
.Parse(codec_type, spatial_idx, image_copy.data(),
image_copy.size())
.value_or(-1);
}
// Piggyback ALR experiment group id and simulcast id into the content type.
const uint8_t experiment_id =
experiment_groups_[videocontenttypehelpers::IsScreenshare(
@ -1877,12 +1891,9 @@ EncodedImageCallback::Result VideoStreamEncoder::OnEncodedImage(
// Post a task because |send_codec_| requires |encoder_queue_| lock.
unsigned int image_width = image_copy._encodedWidth;
unsigned int image_height = image_copy._encodedHeight;
VideoCodecType codec = codec_specific_info
? codec_specific_info->codecType
: VideoCodecType::kVideoCodecGeneric;
encoder_queue_.PostTask([this, codec, image_width, image_height] {
encoder_queue_.PostTask([this, codec_type, image_width, image_height] {
RTC_DCHECK_RUN_ON(&encoder_queue_);
if (codec == VideoCodecType::kVideoCodecVP9 &&
if (codec_type == VideoCodecType::kVideoCodecVP9 &&
send_codec_.VP9()->automaticResizeOn) {
unsigned int expected_width = send_codec_.width;
unsigned int expected_height = send_codec_.height;

6
video/video_stream_encoder.h
View File

@ -34,6 +34,7 @@
#include "call/adaptation/video_source_restrictions.h"
#include "call/adaptation/video_stream_input_state_provider.h"
#include "modules/video_coding/utility/frame_dropper.h"
#include "modules/video_coding/utility/qp_parser.h"
#include "rtc_base/experiments/rate_control_settings.h"
#include "rtc_base/numerics/exp_filter.h"
#include "rtc_base/race_checker.h"
@ -407,6 +408,11 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface,
// Default bitrate limits in EncoderInfoSettings allowed.
const bool default_limits_allowed_;
// QP parser is used to extract QP value from encoded frame when that is not
// provided by encoder.
QpParser qp_parser_;
const bool qp_parsing_allowed_;
// Public methods are proxied to the task queues. The queues must be destroyed
// first to make sure no tasks run that use other members.
rtc::TaskQueue encoder_queue_;

80
video/video_stream_encoder_unittest.cc
View File

@ -97,6 +97,11 @@ uint8_t optimal_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
0x05, 0x03, 0xC7, 0xE0, 0x1B,
0x41, 0x10, 0x8D, 0x00};
const uint8_t kCodedFrameVp8Qp25[] = {
0x10, 0x02, 0x00, 0x9d, 0x01, 0x2a, 0x10, 0x00, 0x10, 0x00,
0x02, 0x47, 0x08, 0x85, 0x85, 0x88, 0x85, 0x84, 0x88, 0x0c,
0x82, 0x00, 0x0c, 0x0d, 0x60, 0x00, 0xfe, 0xfc, 0x5c, 0xd0};
class TestBuffer : public webrtc::I420Buffer {
public:
TestBuffer(rtc::Event* event, int width, int height)
@ -960,9 +965,10 @@ class VideoStreamEncoderTest : public ::testing::Test {
FakeEncoder::Encode(input_image, &frame_type);
}
void InjectEncodedImage(const EncodedImage& image) {
void InjectEncodedImage(const EncodedImage& image,
const CodecSpecificInfo* codec_specific_info) {
MutexLock lock(&local_mutex_);
encoded_image_callback_->OnEncodedImage(image, nullptr);
encoded_image_callback_->OnEncodedImage(image, codec_specific_info);
}
void SetEncodedImageData(
@ -1248,6 +1254,11 @@ class VideoStreamEncoderTest : public ::testing::Test {
return last_capture_time_ms_;
}
const EncodedImage& GetLastEncodedImage() {
MutexLock lock(&mutex_);
return last_encoded_image_;
}
std::vector<uint8_t> GetLastEncodedImageData() {
MutexLock lock(&mutex_);
return std::move(last_encoded_image_data_);
@ -1279,6 +1290,7 @@ class VideoStreamEncoderTest : public ::testing::Test {
const CodecSpecificInfo* codec_specific_info) override {
MutexLock lock(&mutex_);
EXPECT_TRUE(expect_frames_);
last_encoded_image_ = EncodedImage(encoded_image);
last_encoded_image_data_ = std::vector<uint8_t>(
encoded_image.data(), encoded_image.data() + encoded_image.size());
uint32_t timestamp = encoded_image.Timestamp();
@ -1337,6 +1349,7 @@ class VideoStreamEncoderTest : public ::testing::Test {
mutable Mutex mutex_;
TestEncoder* test_encoder_;
rtc::Event encoded_frame_event_;
EncodedImage last_encoded_image_;
std::vector<uint8_t> last_encoded_image_data_;
uint32_t last_timestamp_ = 0;
int64_t last_capture_time_ms_ = 0;
@ -7135,14 +7148,12 @@ TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
int64_t timestamp = 1;
EncodedImage image;
image.SetEncodedData(
EncodedImageBuffer::Create(kTargetBitrateBps / kDefaultFramerate / 8));
image.capture_time_ms_ = ++timestamp;
image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
const int64_t kEncodeFinishDelayMs = 10;
image.timing_.encode_start_ms = timestamp;
image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
fake_encoder_.InjectEncodedImage(image);
fake_encoder_.InjectEncodedImage(image, /*codec_specific_info=*/nullptr);
// Wait for frame without incrementing clock.
EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
// Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
@ -7803,4 +7814,63 @@ TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSimulcast) {
video_stream_encoder_->Stop();
}
TEST_F(VideoStreamEncoderTest, QpPresent_QpKept) {
// Enable encoder source to force encoder reconfig.
encoder_factory_.SetHasInternalSource(true);
ResetEncoder("VP8", 1, 1, 1, false);
// Set QP on encoded frame and pass the frame to encode complete callback.
// Since QP is present QP parsing won't be triggered and the original value
// should be kept.
EncodedImage encoded_image;
encoded_image.qp_ = 123;
encoded_image.SetEncodedData(EncodedImageBuffer::Create(
kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
CodecSpecificInfo codec_info;
codec_info.codecType = kVideoCodecVP8;
fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 123);
video_stream_encoder_->Stop();
}
TEST_F(VideoStreamEncoderTest, QpAbsent_QpParsed) {
// Enable encoder source to force encoder reconfig.
encoder_factory_.SetHasInternalSource(true);
ResetEncoder("VP8", 1, 1, 1, false);
// Pass an encoded frame without QP to encode complete callback. QP should be
// parsed and set.
EncodedImage encoded_image;
encoded_image.qp_ = -1;
encoded_image.SetEncodedData(EncodedImageBuffer::Create(
kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
CodecSpecificInfo codec_info;
codec_info.codecType = kVideoCodecVP8;
fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 25);
video_stream_encoder_->Stop();
}
TEST_F(VideoStreamEncoderTest, QpAbsentParsingDisabled_QpAbsent) {
webrtc::test::ScopedFieldTrials field_trials(
"WebRTC-QpParsingKillSwitch/Enabled/");
// Enable encoder source to force encoder reconfig.
encoder_factory_.SetHasInternalSource(true);
ResetEncoder("VP8", 1, 1, 1, false);
EncodedImage encoded_image;
encoded_image.qp_ = -1;
encoded_image.SetEncodedData(EncodedImageBuffer::Create(
kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
CodecSpecificInfo codec_info;
codec_info.codecType = kVideoCodecVP8;
fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
EXPECT_EQ(sink_.GetLastEncodedImage().qp_, -1);
video_stream_encoder_->Stop();
}
} // namespace webrtc