From 32232e92f31663ea0bb4b57e671a4e4f53bf313d Mon Sep 17 00:00:00 2001 From: Artem Titov Date: Wed, 20 Feb 2019 21:13:14 +0100 Subject: [PATCH] Add spatial layers support to video analyze pipeline. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To support analyze of spatial layers we will continue sending them into the network on encoder side, but will mark which should be then discarded and which should be processed. On decoder side we will drop layers, if they should be discarded and decode only parts, that should be processed. Bug: webrtc:10138 Change-Id: Ic8b8fe7787674c0ec49b879fcc29e54e8e3d787f Reviewed-on: https://webrtc-review.googlesource.com/c/123185 Reviewed-by: Peter Slatala Reviewed-by: Erik Språng Reviewed-by: Ilya Nikolaevskiy Commit-Queue: Artem Titov Cr-Commit-Position: refs/heads/master@{#26784} --- test/pc/e2e/BUILD.gn | 54 ++--- ...=> default_encoded_image_data_injector.cc} | 53 +++-- ... => default_encoded_image_data_injector.h} | 51 +++-- ...lt_encoded_image_data_injector_unittest.cc | 191 +++++++++++++++++ ...ault_encoded_image_id_injector_unittest.cc | 136 ------------ .../video/default_video_quality_analyzer.cc | 7 + .../video/default_video_quality_analyzer.h | 1 + .../video/encoded_image_data_injector.h | 60 ++++++ .../video/encoded_image_id_injector.h | 54 ----- .../video/example_video_quality_analyzer.cc | 12 ++ .../video/example_video_quality_analyzer.h | 3 + .../video/quality_analyzing_video_decoder.cc | 9 +- .../video/quality_analyzing_video_decoder.h | 14 +- .../video/quality_analyzing_video_encoder.cc | 100 ++++++++- .../video/quality_analyzing_video_encoder.h | 77 ++++++- ...le_process_encoded_image_data_injector.cc} | 47 +++-- ...gle_process_encoded_image_data_injector.h} | 48 +++-- ...ss_encoded_image_data_injector_unittest.cc | 198 ++++++++++++++++++ ...cess_encoded_image_id_injector_unittest.cc | 141 ------------- ...video_quality_analyzer_injection_helper.cc | 12 +- .../video_quality_analyzer_injection_helper.h | 15 +- .../api/peerconnection_quality_test_fixture.h | 13 ++ .../api/video_quality_analyzer_interface.h | 2 + test/pc/e2e/peer_connection_quality_test.cc | 2 +- test/pc/e2e/peer_connection_quality_test.h | 4 +- test/pc/e2e/test_peer.cc | 31 ++- test/pc/e2e/test_peer.h | 2 +- 27 files changed, 851 insertions(+), 486 deletions(-) rename test/pc/e2e/analyzer/video/{default_encoded_image_id_injector.cc => default_encoded_image_data_injector.cc} (70%) rename test/pc/e2e/analyzer/video/{default_encoded_image_id_injector.h => default_encoded_image_data_injector.h} (65%) create mode 100644 test/pc/e2e/analyzer/video/default_encoded_image_data_injector_unittest.cc delete mode 100644 test/pc/e2e/analyzer/video/default_encoded_image_id_injector_unittest.cc create mode 100644 test/pc/e2e/analyzer/video/encoded_image_data_injector.h delete mode 100644 test/pc/e2e/analyzer/video/encoded_image_id_injector.h rename test/pc/e2e/analyzer/video/{single_process_encoded_image_id_injector.cc => single_process_encoded_image_data_injector.cc} (63%) rename test/pc/e2e/analyzer/video/{single_process_encoded_image_id_injector.h => single_process_encoded_image_data_injector.h} (59%) create mode 100644 test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector_unittest.cc delete mode 100644 test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector_unittest.cc diff --git a/test/pc/e2e/BUILD.gn b/test/pc/e2e/BUILD.gn index 4e47ab599e..ebf69c8bcd 100644 --- a/test/pc/e2e/BUILD.gn +++ b/test/pc/e2e/BUILD.gn @@ -12,13 +12,13 @@ group("e2e") { testonly = true deps = [ - ":default_encoded_image_id_injector", - ":encoded_image_id_injector_api", + ":default_encoded_image_data_injector", + ":encoded_image_data_injector_api", ":example_video_quality_analyzer", ":id_generator", ":quality_analyzing_video_decoder", ":quality_analyzing_video_encoder", - ":single_process_encoded_image_id_injector", + ":single_process_encoded_image_data_injector", ] if (rtc_include_tests) { deps += [ @@ -34,17 +34,17 @@ if (rtc_include_tests) { testonly = true deps = [ - ":default_encoded_image_id_injector_unittest", + ":default_encoded_image_data_injector_unittest", ":peer_connection_e2e_smoke_test", - ":single_process_encoded_image_id_injector_unittest", + ":single_process_encoded_image_data_injector_unittest", ] } } -rtc_source_set("encoded_image_id_injector_api") { +rtc_source_set("encoded_image_data_injector_api") { visibility = [ "*" ] sources = [ - "analyzer/video/encoded_image_id_injector.h", + "analyzer/video/encoded_image_data_injector.h", ] deps = [ @@ -52,15 +52,15 @@ rtc_source_set("encoded_image_id_injector_api") { ] } -rtc_source_set("default_encoded_image_id_injector") { +rtc_source_set("default_encoded_image_data_injector") { visibility = [ "*" ] sources = [ - "analyzer/video/default_encoded_image_id_injector.cc", - "analyzer/video/default_encoded_image_id_injector.h", + "analyzer/video/default_encoded_image_data_injector.cc", + "analyzer/video/default_encoded_image_data_injector.h", ] deps = [ - ":encoded_image_id_injector_api", + ":encoded_image_data_injector_api", "../../../api/video:encoded_image", "../../../rtc_base:checks", "../../../rtc_base:criticalsection", @@ -68,15 +68,15 @@ rtc_source_set("default_encoded_image_id_injector") { ] } -rtc_source_set("single_process_encoded_image_id_injector") { +rtc_source_set("single_process_encoded_image_data_injector") { visibility = [ "*" ] sources = [ - "analyzer/video/single_process_encoded_image_id_injector.cc", - "analyzer/video/single_process_encoded_image_id_injector.h", + "analyzer/video/single_process_encoded_image_data_injector.cc", + "analyzer/video/single_process_encoded_image_data_injector.h", ] deps = [ - ":encoded_image_id_injector_api", + ":encoded_image_data_injector_api", "../../../api/video:encoded_image", "../../../rtc_base:checks", "../../../rtc_base:criticalsection", @@ -100,7 +100,7 @@ rtc_source_set("quality_analyzing_video_decoder") { "analyzer/video/quality_analyzing_video_decoder.h", ] deps = [ - ":encoded_image_id_injector_api", + ":encoded_image_data_injector_api", ":id_generator", "../../../api/video:encoded_image", "../../../api/video:video_frame", @@ -121,7 +121,7 @@ rtc_source_set("quality_analyzing_video_encoder") { "analyzer/video/quality_analyzing_video_encoder.h", ] deps = [ - ":encoded_image_id_injector_api", + ":encoded_image_data_injector_api", ":id_generator", "../../../api/video:encoded_image", "../../../api/video:video_frame", @@ -143,7 +143,7 @@ if (rtc_include_tests) { "analyzer/video/video_quality_analyzer_injection_helper.h", ] deps = [ - ":encoded_image_id_injector_api", + ":encoded_image_data_injector_api", ":id_generator", ":quality_analyzing_video_decoder", ":quality_analyzing_video_encoder", @@ -164,8 +164,8 @@ if (rtc_include_tests) { "test_peer.h", ] deps = [ - ":default_encoded_image_id_injector", - ":encoded_image_id_injector_api", + ":default_encoded_image_data_injector", + ":encoded_image_data_injector_api", ":example_video_quality_analyzer", ":video_quality_analyzer_injection_helper", "../../../api:array_view", @@ -206,7 +206,7 @@ if (rtc_include_tests) { ] deps = [ ":example_video_quality_analyzer", - ":single_process_encoded_image_id_injector", + ":single_process_encoded_image_data_injector", ":test_peer", ":video_quality_analyzer_injection_helper", "../../../api:libjingle_peerconnection_api", @@ -231,26 +231,26 @@ if (rtc_include_tests) { } } - rtc_source_set("single_process_encoded_image_id_injector_unittest") { + rtc_source_set("single_process_encoded_image_data_injector_unittest") { testonly = true sources = [ - "analyzer/video/single_process_encoded_image_id_injector_unittest.cc", + "analyzer/video/single_process_encoded_image_data_injector_unittest.cc", ] deps = [ - ":single_process_encoded_image_id_injector", + ":single_process_encoded_image_data_injector", "../../../api/video:encoded_image", "../../../rtc_base:rtc_base_approved", "../../../test:test_support", ] } - rtc_source_set("default_encoded_image_id_injector_unittest") { + rtc_source_set("default_encoded_image_data_injector_unittest") { testonly = true sources = [ - "analyzer/video/default_encoded_image_id_injector_unittest.cc", + "analyzer/video/default_encoded_image_data_injector_unittest.cc", ] deps = [ - ":default_encoded_image_id_injector", + ":default_encoded_image_data_injector", "../../../api/video:encoded_image", "../../../rtc_base:rtc_base_approved", "../../../test:test_support", diff --git a/test/pc/e2e/analyzer/video/default_encoded_image_id_injector.cc b/test/pc/e2e/analyzer/video/default_encoded_image_data_injector.cc similarity index 70% rename from test/pc/e2e/analyzer/video/default_encoded_image_id_injector.cc rename to test/pc/e2e/analyzer/video/default_encoded_image_data_injector.cc index f52325ece6..0820c4d939 100644 --- a/test/pc/e2e/analyzer/video/default_encoded_image_id_injector.cc +++ b/test/pc/e2e/analyzer/video/default_encoded_image_data_injector.cc @@ -8,7 +8,7 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "test/pc/e2e/analyzer/video/default_encoded_image_id_injector.h" +#include "test/pc/e2e/analyzer/video/default_encoded_image_data_injector.h" #include @@ -32,18 +32,20 @@ constexpr size_t kBuffersPoolPerCodingEntity = 256; } // namespace -DefaultEncodedImageIdInjector::DefaultEncodedImageIdInjector() { +DefaultEncodedImageDataInjector::DefaultEncodedImageDataInjector() { for (size_t i = 0; i < kPreInitCodingEntitiesCount * kBuffersPoolPerCodingEntity; ++i) { bufs_pool_.push_back( absl::make_unique>(kInitialBufferSize)); } } -DefaultEncodedImageIdInjector::~DefaultEncodedImageIdInjector() = default; +DefaultEncodedImageDataInjector::~DefaultEncodedImageDataInjector() = default; -EncodedImage DefaultEncodedImageIdInjector::InjectId(uint16_t id, - const EncodedImage& source, - int coding_entity_id) { +EncodedImage DefaultEncodedImageDataInjector::InjectData( + uint16_t id, + bool discard, + const EncodedImage& source, + int coding_entity_id) { ExtendIfRequired(coding_entity_id); EncodedImage out = source; @@ -57,14 +59,18 @@ EncodedImage DefaultEncodedImageIdInjector::InjectId(uint16_t id, source.size()); out.data()[0] = id & 0x00ff; out.data()[1] = (id & 0xff00) >> 8; - out.data()[2] = source.size() & 0x00ff; - out.data()[3] = (source.size() & 0xff00) >> 8; - out.data()[4] = (source.size() & 0xff00) >> 16; - out.data()[5] = (source.size() & 0xff00) >> 24; + out.data()[2] = source.size() & 0x000000ff; + out.data()[3] = (source.size() & 0x0000ff00) >> 8; + out.data()[4] = (source.size() & 0x00ff0000) >> 16; + out.data()[5] = (source.size() & 0xff000000) >> 24; + + // We will store discard flag in the high bit of high byte of the size. + RTC_CHECK_LT(source.size(), 1U << 31) << "High bit is already in use"; + out.data()[5] = out.data()[5] | ((discard ? 1 : 0) << 7); return out; } -EncodedImageWithId DefaultEncodedImageIdInjector::ExtractId( +EncodedImageExtractionResult DefaultEncodedImageDataInjector::ExtractData( const EncodedImage& source, int coding_entity_id) { ExtendIfRequired(coding_entity_id); @@ -79,6 +85,7 @@ EncodedImageWithId DefaultEncodedImageIdInjector::ExtractId( size_t source_pos = 0; size_t out_pos = 0; absl::optional id = absl::nullopt; + bool discard = true; while (source_pos < source.size()) { RTC_CHECK_LE(source_pos + kEncodedImageBufferExpansion, source.size()); uint16_t next_id = @@ -89,21 +96,29 @@ EncodedImageWithId DefaultEncodedImageIdInjector::ExtractId( id = next_id; uint32_t length = source.data()[source_pos + 2] + (source.data()[source_pos + 3] << 8) + - (source.data()[source_pos + 3] << 16) + - (source.data()[source_pos + 3] << 24); + (source.data()[source_pos + 4] << 16) + + ((source.data()[source_pos + 5] << 24) & 0b01111111); + bool current_discard = (source.data()[source_pos + 5] & 0b10000000) != 0; RTC_CHECK_LE(source_pos + kEncodedImageBufferExpansion + length, source.size()); - memcpy(&out.data()[out_pos], - &source.data()[source_pos + kEncodedImageBufferExpansion], length); + if (!current_discard) { + // Copy next encoded image payload from concatenated buffer only if it is + // not discarded. + memcpy(&out.data()[out_pos], + &source.data()[source_pos + kEncodedImageBufferExpansion], length); + out_pos += length; + } source_pos += length + kEncodedImageBufferExpansion; - out_pos += length; + // Extraction result is discarded only if all encoded partitions are + // discarded. + discard = discard && current_discard; } out.set_size(out_pos); - return EncodedImageWithId{id.value(), out}; + return EncodedImageExtractionResult{id.value(), out, discard}; } -void DefaultEncodedImageIdInjector::ExtendIfRequired(int coding_entity_id) { +void DefaultEncodedImageDataInjector::ExtendIfRequired(int coding_entity_id) { rtc::CritScope crit(&lock_); if (coding_entities_.find(coding_entity_id) != coding_entities_.end()) { // This entity is already known for this injector, so buffers are allocated. @@ -124,7 +139,7 @@ void DefaultEncodedImageIdInjector::ExtendIfRequired(int coding_entity_id) { } } -std::vector* DefaultEncodedImageIdInjector::NextBuffer() { +std::vector* DefaultEncodedImageDataInjector::NextBuffer() { rtc::CritScope crit(&lock_); // Get buffer from the front of the queue, return it to the caller and // put in the back diff --git a/test/pc/e2e/analyzer/video/default_encoded_image_id_injector.h b/test/pc/e2e/analyzer/video/default_encoded_image_data_injector.h similarity index 65% rename from test/pc/e2e/analyzer/video/default_encoded_image_id_injector.h rename to test/pc/e2e/analyzer/video/default_encoded_image_data_injector.h index 6f272714cf..989877df7e 100644 --- a/test/pc/e2e/analyzer/video/default_encoded_image_id_injector.h +++ b/test/pc/e2e/analyzer/video/default_encoded_image_data_injector.h @@ -8,8 +8,8 @@ * be found in the AUTHORS file in the root of the source tree. */ -#ifndef TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_ENCODED_IMAGE_ID_INJECTOR_H_ -#define TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_ENCODED_IMAGE_ID_INJECTOR_H_ +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_ENCODED_IMAGE_DATA_INJECTOR_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_ENCODED_IMAGE_DATA_INJECTOR_H_ #include #include @@ -20,20 +20,21 @@ #include "api/video/encoded_image.h" #include "rtc_base/critical_section.h" -#include "test/pc/e2e/analyzer/video/encoded_image_id_injector.h" +#include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h" namespace webrtc { namespace test { -// Injects frame id into EncodedImage payload buffer. The payload buffer will -// be prepended in the injector with 2 bytes frame id and 4 bytes original -// buffer length. It is assumed, that frame's data can't be more then 2^32 -// bytes. In the decoder, frame id will be extracted and the length will be used -// to restore original buffer. +// Injects frame id and discard flag into EncodedImage payload buffer. The +// payload buffer will be prepended in the injector with 2 bytes frame id and 4 +// bytes original buffer length. Discarded flag will be put into the highest bit +// of the length. It is assumed, that frame's data can't be more then 2^31 +// bytes. In the decoder, frame id and discard flag will be extracted and the +// length will be used to restore original buffer. // // The data in the EncodedImage on encoder side after injection will look like // this: -// 4 bytes frame length +// 4 bytes frame length + discard flag // _ _ _↓_ _ _ _________________ // | | | original buffer | // ¯↑¯ ¯ ¯ ¯ ¯ ¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯ @@ -48,10 +49,12 @@ namespace test { // 1. pos = 0 // 2. Extract id from buf[pos] and buf[pos + 1] // 3. Extract length from buf[pos + 2]..buf[pos + 5] -// 4. Copy |length| bytes starting from buf[pos + 6] to output buffer. -// 5. pos = pos + length + 6 -// 6. If pos < buf.length - go to the step 2. -// Also it will be checked, that all extracted ids are equals. +// 4. Extract discard flag from length highest bit. +// 5. If discard flag is False, copy |length| bytes starting from buf[pos + 6] +// to output buffer. +// 6. pos = pos + length + 6 +// 7. If pos < buf.length - go to the step 2. +// Also it will check, that all extracted ids are equals. // // Because EncodedImage doesn't take ownership of its buffer, injector will keep // ownership of the buffers that will be used for EncodedImages with injected @@ -67,17 +70,19 @@ namespace test { // preallocate buffers for 2 coding entities, so 512 buffers with initial size // 2KB. If in some point of time bigger buffer will be required, it will be also // extended. -class DefaultEncodedImageIdInjector : public EncodedImageIdInjector, - public EncodedImageIdExtractor { +class DefaultEncodedImageDataInjector : public EncodedImageDataInjector, + public EncodedImageDataExtractor { public: - DefaultEncodedImageIdInjector(); - ~DefaultEncodedImageIdInjector() override; + DefaultEncodedImageDataInjector(); + ~DefaultEncodedImageDataInjector() override; - EncodedImage InjectId(uint16_t id, - const EncodedImage& source, - int coding_entity_id) override; - EncodedImageWithId ExtractId(const EncodedImage& source, - int coding_entity_id) override; + // TODO(titovartem) add support for discard injection and update the doc. + EncodedImage InjectData(uint16_t id, + bool discard, + const EncodedImage& source, + int coding_entity_id) override; + EncodedImageExtractionResult ExtractData(const EncodedImage& source, + int coding_entity_id) override; private: void ExtendIfRequired(int coding_entity_id) RTC_LOCKS_EXCLUDED(lock_); @@ -99,4 +104,4 @@ class DefaultEncodedImageIdInjector : public EncodedImageIdInjector, } // namespace test } // namespace webrtc -#endif // TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_ENCODED_IMAGE_ID_INJECTOR_H_ +#endif // TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_ENCODED_IMAGE_DATA_INJECTOR_H_ diff --git a/test/pc/e2e/analyzer/video/default_encoded_image_data_injector_unittest.cc b/test/pc/e2e/analyzer/video/default_encoded_image_data_injector_unittest.cc new file mode 100644 index 0000000000..8dc341e33a --- /dev/null +++ b/test/pc/e2e/analyzer/video/default_encoded_image_data_injector_unittest.cc @@ -0,0 +1,191 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/default_encoded_image_data_injector.h" + +#include + +#include "api/video/encoded_image.h" +#include "rtc_base/buffer.h" +#include "test/gtest.h" + +namespace webrtc { +namespace test { +namespace { + +rtc::Buffer CreateBufferOfSizeNFilledWithValuesFromX(size_t n, uint8_t x) { + rtc::Buffer buffer(n); + for (size_t i = 0; i < n; ++i) { + buffer[i] = static_cast(x + i); + } + return buffer; +} + +} // namespace + +TEST(DefaultEncodedImageDataInjector, InjectExtractDiscardFalse) { + DefaultEncodedImageDataInjector injector; + + rtc::Buffer buffer = CreateBufferOfSizeNFilledWithValuesFromX(10, 1); + + EncodedImage source(buffer.data(), 10, 10); + source.SetTimestamp(123456789); + + EncodedImageExtractionResult out = + injector.ExtractData(injector.InjectData(512, false, source, 1), 2); + EXPECT_EQ(out.id, 512); + EXPECT_FALSE(out.discard); + EXPECT_EQ(out.image.size(), 10ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out.image.data()[i], i + 1); + } +} + +TEST(DefaultEncodedImageDataInjector, InjectExtractDiscardTrue) { + DefaultEncodedImageDataInjector injector; + + rtc::Buffer buffer = CreateBufferOfSizeNFilledWithValuesFromX(10, 1); + + EncodedImage source(buffer.data(), 10, 10); + source.SetTimestamp(123456789); + + EncodedImageExtractionResult out = + injector.ExtractData(injector.InjectData(512, true, source, 1), 2); + EXPECT_EQ(out.id, 512); + EXPECT_TRUE(out.discard); + EXPECT_EQ(out.image.size(), 0ul); +} + +TEST(DefaultEncodedImageDataInjector, Inject3Extract3) { + DefaultEncodedImageDataInjector injector; + + rtc::Buffer buffer1 = CreateBufferOfSizeNFilledWithValuesFromX(10, 1); + rtc::Buffer buffer2 = CreateBufferOfSizeNFilledWithValuesFromX(10, 11); + rtc::Buffer buffer3 = CreateBufferOfSizeNFilledWithValuesFromX(10, 21); + + // 1st frame + EncodedImage source1(buffer1.data(), 10, 10); + source1.SetTimestamp(123456710); + // 2nd frame 1st spatial layer + EncodedImage source2(buffer2.data(), 10, 10); + source2.SetTimestamp(123456720); + // 2nd frame 2nd spatial layer + EncodedImage source3(buffer3.data(), 10, 10); + source3.SetTimestamp(123456720); + + EncodedImage intermediate1 = injector.InjectData(510, false, source1, 1); + EncodedImage intermediate2 = injector.InjectData(520, true, source2, 1); + EncodedImage intermediate3 = injector.InjectData(520, false, source3, 1); + + // Extract ids in different order. + EncodedImageExtractionResult out3 = injector.ExtractData(intermediate3, 2); + EncodedImageExtractionResult out1 = injector.ExtractData(intermediate1, 2); + EncodedImageExtractionResult out2 = injector.ExtractData(intermediate2, 2); + + EXPECT_EQ(out1.id, 510); + EXPECT_FALSE(out1.discard); + EXPECT_EQ(out1.image.size(), 10ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out1.image.data()[i], i + 1); + } + EXPECT_EQ(out2.id, 520); + EXPECT_TRUE(out2.discard); + EXPECT_EQ(out2.image.size(), 0ul); + EXPECT_EQ(out3.id, 520); + EXPECT_FALSE(out3.discard); + EXPECT_EQ(out3.image.size(), 10ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out3.image.data()[i], i + 21); + } +} + +TEST(DefaultEncodedImageDataInjector, InjectExtractFromConcatenated) { + DefaultEncodedImageDataInjector injector; + + rtc::Buffer buffer1 = CreateBufferOfSizeNFilledWithValuesFromX(10, 1); + rtc::Buffer buffer2 = CreateBufferOfSizeNFilledWithValuesFromX(10, 11); + rtc::Buffer buffer3 = CreateBufferOfSizeNFilledWithValuesFromX(10, 21); + + EncodedImage source1(buffer1.data(), 10, 10); + source1.SetTimestamp(123456710); + EncodedImage source2(buffer2.data(), 10, 10); + source2.SetTimestamp(123456710); + EncodedImage source3(buffer3.data(), 10, 10); + source3.SetTimestamp(123456710); + + // Inject id into 3 images with same frame id. + EncodedImage intermediate1 = injector.InjectData(512, false, source1, 1); + EncodedImage intermediate2 = injector.InjectData(512, true, source2, 1); + EncodedImage intermediate3 = injector.InjectData(512, false, source3, 1); + + // Concatenate them into single encoded image, like it can be done in jitter + // buffer. + size_t concatenated_length = + intermediate1.size() + intermediate2.size() + intermediate3.size(); + rtc::Buffer concatenated_buffer; + concatenated_buffer.AppendData(intermediate1.data(), intermediate1.size()); + concatenated_buffer.AppendData(intermediate2.data(), intermediate2.size()); + concatenated_buffer.AppendData(intermediate3.data(), intermediate3.size()); + EncodedImage concatenated(concatenated_buffer.data(), concatenated_length, + concatenated_length); + + // Extract frame id from concatenated image + EncodedImageExtractionResult out = injector.ExtractData(concatenated, 2); + + EXPECT_EQ(out.id, 512); + EXPECT_FALSE(out.discard); + EXPECT_EQ(out.image.size(), 2 * 10ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out.image.data()[i], i + 1); + EXPECT_EQ(out.image.data()[i + 10], i + 21); + } +} + +TEST(DefaultEncodedImageDataInjector, + InjectExtractFromConcatenatedAllDiscarded) { + DefaultEncodedImageDataInjector injector; + + rtc::Buffer buffer1 = CreateBufferOfSizeNFilledWithValuesFromX(10, 1); + rtc::Buffer buffer2 = CreateBufferOfSizeNFilledWithValuesFromX(10, 11); + rtc::Buffer buffer3 = CreateBufferOfSizeNFilledWithValuesFromX(10, 21); + + EncodedImage source1(buffer1.data(), 10, 10); + source1.SetTimestamp(123456710); + EncodedImage source2(buffer2.data(), 10, 10); + source2.SetTimestamp(123456710); + EncodedImage source3(buffer3.data(), 10, 10); + source3.SetTimestamp(123456710); + + // Inject id into 3 images with same frame id. + EncodedImage intermediate1 = injector.InjectData(512, true, source1, 1); + EncodedImage intermediate2 = injector.InjectData(512, true, source2, 1); + EncodedImage intermediate3 = injector.InjectData(512, true, source3, 1); + + // Concatenate them into single encoded image, like it can be done in jitter + // buffer. + size_t concatenated_length = + intermediate1.size() + intermediate2.size() + intermediate3.size(); + rtc::Buffer concatenated_buffer; + concatenated_buffer.AppendData(intermediate1.data(), intermediate1.size()); + concatenated_buffer.AppendData(intermediate2.data(), intermediate2.size()); + concatenated_buffer.AppendData(intermediate3.data(), intermediate3.size()); + EncodedImage concatenated(concatenated_buffer.data(), concatenated_length, + concatenated_length); + + // Extract frame id from concatenated image + EncodedImageExtractionResult out = injector.ExtractData(concatenated, 2); + + EXPECT_EQ(out.id, 512); + EXPECT_TRUE(out.discard); + EXPECT_EQ(out.image.size(), 0ul); +} + +} // namespace test +} // namespace webrtc diff --git a/test/pc/e2e/analyzer/video/default_encoded_image_id_injector_unittest.cc b/test/pc/e2e/analyzer/video/default_encoded_image_id_injector_unittest.cc deleted file mode 100644 index e3e50e599e..0000000000 --- a/test/pc/e2e/analyzer/video/default_encoded_image_id_injector_unittest.cc +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "test/pc/e2e/analyzer/video/default_encoded_image_id_injector.h" - -#include - -#include "api/video/encoded_image.h" -#include "rtc_base/buffer.h" -#include "test/gtest.h" - -namespace webrtc { -namespace test { -namespace { - -rtc::Buffer CreateBufferOfSizeNFilledWithValuesFromX(size_t n, uint8_t x) { - rtc::Buffer buffer(n); - for (size_t i = 0; i < n; ++i) { - buffer[i] = static_cast(x + i); - } - return buffer; -} - -} // namespace - -TEST(DefaultEncodedImageIdInjector, InjectExtract) { - DefaultEncodedImageIdInjector injector; - - rtc::Buffer buffer = CreateBufferOfSizeNFilledWithValuesFromX(10, 1); - - EncodedImage source(buffer.data(), 10, 10); - source.SetTimestamp(123456789); - - EncodedImageWithId out = - injector.ExtractId(injector.InjectId(512, source, 1), 2); - ASSERT_EQ(out.id, 512); - ASSERT_EQ(out.image.size(), 10ul); - for (int i = 0; i < 10; ++i) { - ASSERT_EQ(out.image.data()[i], i + 1); - } -} - -TEST(DefaultEncodedImageIdInjector, Inject3Extract3) { - DefaultEncodedImageIdInjector injector; - - rtc::Buffer buffer1 = CreateBufferOfSizeNFilledWithValuesFromX(10, 1); - rtc::Buffer buffer2 = CreateBufferOfSizeNFilledWithValuesFromX(10, 11); - rtc::Buffer buffer3 = CreateBufferOfSizeNFilledWithValuesFromX(10, 21); - - // 1st frame - EncodedImage source1(buffer1.data(), 10, 10); - source1.SetTimestamp(123456710); - // 2nd frame 1st spatial layer - EncodedImage source2(buffer2.data(), 10, 10); - source2.SetTimestamp(123456720); - // 2nd frame 2nd spatial layer - EncodedImage source3(buffer3.data(), 10, 10); - source3.SetTimestamp(123456720); - - EncodedImage intermediate1 = injector.InjectId(510, source1, 1); - EncodedImage intermediate2 = injector.InjectId(520, source2, 1); - EncodedImage intermediate3 = injector.InjectId(520, source3, 1); - - // Extract ids in different order. - EncodedImageWithId out3 = injector.ExtractId(intermediate3, 2); - EncodedImageWithId out1 = injector.ExtractId(intermediate1, 2); - EncodedImageWithId out2 = injector.ExtractId(intermediate2, 2); - - ASSERT_EQ(out1.id, 510); - ASSERT_EQ(out1.image.size(), 10ul); - for (int i = 0; i < 10; ++i) { - ASSERT_EQ(out1.image.data()[i], i + 1); - } - ASSERT_EQ(out2.id, 520); - ASSERT_EQ(out2.image.size(), 10ul); - for (int i = 0; i < 10; ++i) { - ASSERT_EQ(out2.image.data()[i], i + 11); - } - ASSERT_EQ(out3.id, 520); - ASSERT_EQ(out3.image.size(), 10ul); - for (int i = 0; i < 10; ++i) { - ASSERT_EQ(out3.image.data()[i], i + 21); - } -} - -TEST(DefaultEncodedImageIdInjector, InjectExtractFromConcatenated) { - DefaultEncodedImageIdInjector injector; - - rtc::Buffer buffer1 = CreateBufferOfSizeNFilledWithValuesFromX(10, 1); - rtc::Buffer buffer2 = CreateBufferOfSizeNFilledWithValuesFromX(10, 11); - rtc::Buffer buffer3 = CreateBufferOfSizeNFilledWithValuesFromX(10, 21); - - EncodedImage source1(buffer1.data(), 10, 10); - source1.SetTimestamp(123456710); - EncodedImage source2(buffer2.data(), 10, 10); - source2.SetTimestamp(123456710); - EncodedImage source3(buffer3.data(), 10, 10); - source3.SetTimestamp(123456710); - - // Inject id into 3 images with same frame id. - EncodedImage intermediate1 = injector.InjectId(512, source1, 1); - EncodedImage intermediate2 = injector.InjectId(512, source2, 1); - EncodedImage intermediate3 = injector.InjectId(512, source3, 1); - - // Concatenate them into single encoded image, like it can be done in jitter - // buffer. - size_t concatenated_length = - intermediate1.size() + intermediate2.size() + intermediate3.size(); - rtc::Buffer concatenated_buffer; - concatenated_buffer.AppendData(intermediate1.data(), intermediate1.size()); - concatenated_buffer.AppendData(intermediate2.data(), intermediate2.size()); - concatenated_buffer.AppendData(intermediate3.data(), intermediate3.size()); - EncodedImage concatenated(concatenated_buffer.data(), concatenated_length, - concatenated_length); - - // Extract frame id from concatenated image - EncodedImageWithId out = injector.ExtractId(concatenated, 2); - - ASSERT_EQ(out.id, 512); - ASSERT_EQ(out.image.size(), 3 * 10ul); - for (int i = 0; i < 10; ++i) { - ASSERT_EQ(out.image.data()[i], i + 1); - ASSERT_EQ(out.image.data()[i + 10], i + 11); - ASSERT_EQ(out.image.data()[i + 20], i + 21); - } -} - -} // namespace test -} // namespace webrtc diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc index e4fcd209f5..2c91351bd7 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc @@ -299,6 +299,13 @@ void DefaultVideoQualityAnalyzer::Stop() { ReportResults(); } +std::string DefaultVideoQualityAnalyzer::GetStreamLabel(uint16_t frame_id) { + rtc::CritScope crit1(&lock_); + auto it = frame_stats_.find(frame_id); + RTC_DCHECK(it != frame_stats_.end()) << "Unknown frame_id=" << frame_id; + return it->second.stream_label; +} + std::set DefaultVideoQualityAnalyzer::GetKnownVideoStreams() const { rtc::CritScope crit2(&comparison_lock_); diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h index 570253debc..07c37ea62d 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h @@ -136,6 +136,7 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { void OnEncoderError(const VideoFrame& frame, int32_t error_code) override; void OnDecoderError(uint16_t frame_id, int32_t error_code) override; void Stop() override; + std::string GetStreamLabel(uint16_t frame_id) override; // Returns set of stream labels, that were met during test call. std::set GetKnownVideoStreams() const; diff --git a/test/pc/e2e/analyzer/video/encoded_image_data_injector.h b/test/pc/e2e/analyzer/video/encoded_image_data_injector.h new file mode 100644 index 0000000000..bf541ba826 --- /dev/null +++ b/test/pc/e2e/analyzer/video/encoded_image_data_injector.h @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_ENCODED_IMAGE_DATA_INJECTOR_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_ENCODED_IMAGE_DATA_INJECTOR_H_ + +#include +#include + +#include "api/video/encoded_image.h" + +namespace webrtc { +namespace test { + +// Injects frame id into EncodedImage on encoder side +class EncodedImageDataInjector { + public: + virtual ~EncodedImageDataInjector() = default; + + // Return encoded image with specified |id| and |discard| flag injected into + // its payload. |discard| flag mean does analyzing decoder should discard this + // encoded image because it belongs to unnecessary simulcast stream or spatial + // layer. |coding_entity_id| is unique id of decoder or encoder. + virtual EncodedImage InjectData(uint16_t id, + bool discard, + const EncodedImage& source, + int coding_entity_id) = 0; +}; + +struct EncodedImageExtractionResult { + uint16_t id; + EncodedImage image; + // Is true if encoded image should be discarded. It is used to filter out + // unnecessary spatial layers and simulcast streams. + bool discard; +}; + +// Extracts frame id from EncodedImage on decoder side. +class EncodedImageDataExtractor { + public: + virtual ~EncodedImageDataExtractor() = default; + + // Returns encoded image id, extracted from payload and also encoded image + // with its original payload. For concatenated spatial layers it should be the + // same id. |coding_entity_id| is unique id of decoder or encoder. + virtual EncodedImageExtractionResult ExtractData(const EncodedImage& source, + int coding_entity_id) = 0; +}; + +} // namespace test +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_ENCODED_IMAGE_DATA_INJECTOR_H_ diff --git a/test/pc/e2e/analyzer/video/encoded_image_id_injector.h b/test/pc/e2e/analyzer/video/encoded_image_id_injector.h deleted file mode 100644 index 7f53d4e876..0000000000 --- a/test/pc/e2e/analyzer/video/encoded_image_id_injector.h +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef TEST_PC_E2E_ANALYZER_VIDEO_ENCODED_IMAGE_ID_INJECTOR_H_ -#define TEST_PC_E2E_ANALYZER_VIDEO_ENCODED_IMAGE_ID_INJECTOR_H_ - -#include -#include - -#include "api/video/encoded_image.h" - -namespace webrtc { -namespace test { - -// Injects frame id into EncodedImage on encoder side -class EncodedImageIdInjector { - public: - virtual ~EncodedImageIdInjector() = default; - - // Return encoded image with specified |id| injected into its payload. - // |coding_entity_id| is unique id of decoder or encoder. - virtual EncodedImage InjectId(uint16_t id, - const EncodedImage& source, - int coding_entity_id) = 0; -}; - -struct EncodedImageWithId { - uint16_t id; - EncodedImage image; -}; - -// Extracts frame id from EncodedImage on decoder side. -class EncodedImageIdExtractor { - public: - virtual ~EncodedImageIdExtractor() = default; - - // Returns encoded image id, extracted from payload and also encoded image - // with its original payload. For concatenated spatial layers it should be the - // same id. |coding_entity_id| is unique id of decoder or encoder. - virtual EncodedImageWithId ExtractId(const EncodedImage& source, - int coding_entity_id) = 0; -}; - -} // namespace test -} // namespace webrtc - -#endif // TEST_PC_E2E_ANALYZER_VIDEO_ENCODED_IMAGE_ID_INJECTOR_H_ diff --git a/test/pc/e2e/analyzer/video/example_video_quality_analyzer.cc b/test/pc/e2e/analyzer/video/example_video_quality_analyzer.cc index 07b31752ad..00c0a5b718 100644 --- a/test/pc/e2e/analyzer/video/example_video_quality_analyzer.cc +++ b/test/pc/e2e/analyzer/video/example_video_quality_analyzer.cc @@ -28,12 +28,16 @@ uint16_t ExampleVideoQualityAnalyzer::OnFrameCaptured( auto it = frames_in_flight_.find(frame_id); if (it == frames_in_flight_.end()) { frames_in_flight_.insert(frame_id); + frames_to_stream_label_.insert({frame_id, stream_label}); } else { RTC_LOG(WARNING) << "Meet new frame with the same id: " << frame_id << ". Assumes old one as dropped"; // We needn't insert frame to frames_in_flight_, because it is already // there. ++frames_dropped_; + auto stream_it = frames_to_stream_label_.find(frame_id); + RTC_CHECK(stream_it != frames_to_stream_label_.end()); + stream_it->second = stream_label; } ++frames_captured_; return frame_id; @@ -95,6 +99,14 @@ void ExampleVideoQualityAnalyzer::Stop() { frames_dropped_ += frames_in_flight_.size(); } +std::string ExampleVideoQualityAnalyzer::GetStreamLabel(uint16_t frame_id) { + rtc::CritScope crit(&lock_); + auto it = frames_to_stream_label_.find(frame_id); + RTC_DCHECK(it != frames_to_stream_label_.end()) + << "Unknown frame_id=" << frame_id; + return it->second; +} + uint64_t ExampleVideoQualityAnalyzer::frames_captured() const { rtc::CritScope crit(&lock_); return frames_captured_; diff --git a/test/pc/e2e/analyzer/video/example_video_quality_analyzer.h b/test/pc/e2e/analyzer/video/example_video_quality_analyzer.h index 4070b4d63b..e8a2fa4ebb 100644 --- a/test/pc/e2e/analyzer/video/example_video_quality_analyzer.h +++ b/test/pc/e2e/analyzer/video/example_video_quality_analyzer.h @@ -12,6 +12,7 @@ #define TEST_PC_E2E_ANALYZER_VIDEO_EXAMPLE_VIDEO_QUALITY_ANALYZER_H_ #include +#include #include #include @@ -48,6 +49,7 @@ class ExampleVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { void OnEncoderError(const VideoFrame& frame, int32_t error_code) override; void OnDecoderError(uint16_t frame_id, int32_t error_code) override; void Stop() override; + std::string GetStreamLabel(uint16_t frame_id) override; uint64_t frames_captured() const; uint64_t frames_sent() const; @@ -66,6 +68,7 @@ class ExampleVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { // need to keep them to correctly determine dropped frames and also correctly // process frame id overlap. std::set frames_in_flight_ RTC_GUARDED_BY(lock_); + std::map frames_to_stream_label_ RTC_GUARDED_BY(lock_); uint16_t next_frame_id_ RTC_GUARDED_BY(lock_) = 0; uint64_t frames_captured_ RTC_GUARDED_BY(lock_) = 0; uint64_t frames_sent_ RTC_GUARDED_BY(lock_) = 0; diff --git a/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc b/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc index 6d48e4d12b..13e44bf27d 100644 --- a/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc +++ b/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc @@ -25,7 +25,7 @@ namespace test { QualityAnalyzingVideoDecoder::QualityAnalyzingVideoDecoder( int id, std::unique_ptr delegate, - EncodedImageIdExtractor* extractor, + EncodedImageDataExtractor* extractor, VideoQualityAnalyzerInterface* analyzer) : id_(id), implementation_name_("AnalyzingDecoder-" + @@ -53,7 +53,10 @@ int32_t QualityAnalyzingVideoDecoder::Decode( // owner of original buffer will be responsible for deleting it, or extractor // can create a new buffer. In such case extractor will be responsible for // deleting it. - EncodedImageWithId out = extractor_->ExtractId(input_image, id_); + EncodedImageExtractionResult out = extractor_->ExtractData(input_image, id_); + + // TODO(titovartem) add support for simulcast. + RTC_CHECK(!out.discard) << "Simulcast is not supported yet"; EncodedImage* origin_image; { @@ -195,7 +198,7 @@ void QualityAnalyzingVideoDecoder::OnFrameDecoded( QualityAnalyzingVideoDecoderFactory::QualityAnalyzingVideoDecoderFactory( std::unique_ptr delegate, IdGenerator* id_generator, - EncodedImageIdExtractor* extractor, + EncodedImageDataExtractor* extractor, VideoQualityAnalyzerInterface* analyzer) : delegate_(std::move(delegate)), id_generator_(id_generator), diff --git a/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h b/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h index 2fc69b426d..1f887f7620 100644 --- a/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h +++ b/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h @@ -22,7 +22,7 @@ #include "api/video_codecs/video_decoder.h" #include "api/video_codecs/video_decoder_factory.h" #include "rtc_base/critical_section.h" -#include "test/pc/e2e/analyzer/video/encoded_image_id_injector.h" +#include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h" #include "test/pc/e2e/analyzer/video/id_generator.h" #include "test/pc/e2e/api/video_quality_analyzer_interface.h" @@ -51,10 +51,10 @@ class QualityAnalyzingVideoDecoder : public VideoDecoder { public: // Creates analyzing decoder. |id| is unique coding entity id, that will // be used to distinguish all encoders and decoders inside - // EncodedImageIdInjector and EncodedImageIdExtracor. + // EncodedImageDataInjector and EncodedImageIdExtracor. QualityAnalyzingVideoDecoder(int id, std::unique_ptr delegate, - EncodedImageIdExtractor* extractor, + EncodedImageDataExtractor* extractor, VideoQualityAnalyzerInterface* analyzer); ~QualityAnalyzingVideoDecoder() override; @@ -101,7 +101,7 @@ class QualityAnalyzingVideoDecoder : public VideoDecoder { const int id_; const std::string implementation_name_; std::unique_ptr delegate_; - EncodedImageIdExtractor* const extractor_; + EncodedImageDataExtractor* const extractor_; VideoQualityAnalyzerInterface* const analyzer_; std::unique_ptr analyzing_callback_; @@ -112,7 +112,7 @@ class QualityAnalyzingVideoDecoder : public VideoDecoder { std::map timestamp_to_frame_id_ RTC_GUARDED_BY(lock_); // Stores currently being decoded images by frame id. Because - // EncodedImageIdExtractor can create new copy on EncodedImage we need to + // EncodedImageDataExtractor can create new copy on EncodedImage we need to // ensure, that this image won't be deleted during async decoding. To do it // all images are putted into this map and removed from here inside callback. std::map decoding_images_ RTC_GUARDED_BY(lock_); @@ -126,7 +126,7 @@ class QualityAnalyzingVideoDecoderFactory : public VideoDecoderFactory { QualityAnalyzingVideoDecoderFactory( std::unique_ptr delegate, IdGenerator* id_generator, - EncodedImageIdExtractor* extractor, + EncodedImageDataExtractor* extractor, VideoQualityAnalyzerInterface* analyzer); ~QualityAnalyzingVideoDecoderFactory() override; @@ -141,7 +141,7 @@ class QualityAnalyzingVideoDecoderFactory : public VideoDecoderFactory { private: std::unique_ptr delegate_; IdGenerator* const id_generator_; - EncodedImageIdExtractor* const extractor_; + EncodedImageDataExtractor* const extractor_; VideoQualityAnalyzerInterface* const analyzer_; }; diff --git a/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc b/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc index 55061cab86..95cf98aca8 100644 --- a/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc +++ b/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc @@ -13,7 +13,9 @@ #include #include "absl/memory/memory.h" +#include "api/video/video_codec_type.h" #include "modules/video_coding/include/video_error_codes.h" +#include "rtc_base/critical_section.h" #include "rtc_base/logging.h" namespace webrtc { @@ -27,10 +29,12 @@ constexpr size_t kMaxFrameInPipelineCount = 1000; QualityAnalyzingVideoEncoder::QualityAnalyzingVideoEncoder( int id, std::unique_ptr delegate, - EncodedImageIdInjector* injector, + std::map> stream_required_spatial_index, + EncodedImageDataInjector* injector, VideoQualityAnalyzerInterface* analyzer) : id_(id), delegate_(std::move(delegate)), + stream_required_spatial_index_(std::move(stream_required_spatial_index)), injector_(injector), analyzer_(analyzer) {} QualityAnalyzingVideoEncoder::~QualityAnalyzingVideoEncoder() = default; @@ -39,6 +43,29 @@ int32_t QualityAnalyzingVideoEncoder::InitEncode( const VideoCodec* codec_settings, int32_t number_of_cores, size_t max_payload_size) { + rtc::CritScope crit(&lock_); + mode_ = SimulcastMode::kNormal; + if (codec_settings->codecType == kVideoCodecVP9) { + if (codec_settings->VP9().numberOfSpatialLayers > 1) { + switch (codec_settings->VP9().interLayerPred) { + case InterLayerPredMode::kOn: + mode_ = SimulcastMode::kSVC; + break; + case InterLayerPredMode::kOnKeyPic: + mode_ = SimulcastMode::kKSVC; + break; + case InterLayerPredMode::kOff: + mode_ = SimulcastMode::kSimulcast; + break; + default: + RTC_NOTREACHED() << "Unknown codec_settings->VP9().interLayerPred"; + break; + } + } + } + if (codec_settings->numberOfSimulcastStreams > 1) { + mode_ = SimulcastMode::kSimulcast; + } return delegate_->InitEncode(codec_settings, number_of_cores, max_payload_size); } @@ -109,7 +136,7 @@ VideoEncoder::EncoderInfo QualityAnalyzingVideoEncoder::GetEncoderInfo() const { } // It is assumed, that encoded callback will be always invoked with encoded -// images that correspond to the frames if the same sequence, that frames +// images that correspond to the frames in the same sequence, that frames // arrived. In other words, assume we have frames F1, F2 and F3 and they have // corresponding encoded images I1, I2 and I3. In such case if we will call // encode first with F1, then with F2 and then with F3, then encoder callback @@ -126,6 +153,7 @@ EncodedImageCallback::Result QualityAnalyzingVideoEncoder::OnEncodedImage( const CodecSpecificInfo* codec_specific_info, const RTPFragmentationHeader* fragmentation) { uint16_t frame_id; + bool discard = false; { rtc::CritScope crit(&lock_); std::pair timestamp_frame_id; @@ -155,15 +183,22 @@ EncodedImageCallback::Result QualityAnalyzingVideoEncoder::OnEncodedImage( EncodedImageCallback::Result::Error::OK); } frame_id = timestamp_frame_id.second; + + discard = ShouldDiscard(frame_id, encoded_image); } - analyzer_->OnFrameEncoded(frame_id, encoded_image); + if (!discard) { + // Analyzer should see only encoded images, that weren't discarded. + analyzer_->OnFrameEncoded(frame_id, encoded_image); + } - // Image id injector injects frame id into provided EncodedImage and returns - // the image with a) modified original buffer (in such case the current owner - // of the buffer will be responsible for deleting it) or b) a new buffer (in - // such case injector will be responsible for deleting it). - const EncodedImage& image = injector_->InjectId(frame_id, encoded_image, id_); + // Image data injector injects frame id and discard flag into provided + // EncodedImage and returns the image with a) modified original buffer (in + // such case the current owner of the buffer will be responsible for deleting + // it) or b) a new buffer (in such case injector will be responsible for + // deleting it). + const EncodedImage& image = + injector_->InjectData(frame_id, discard, encoded_image, id_); { rtc::CritScope crit(&lock_); RTC_DCHECK(delegate_callback_); @@ -180,12 +215,57 @@ void QualityAnalyzingVideoEncoder::OnDroppedFrame( delegate_callback_->OnDroppedFrame(reason); } +bool QualityAnalyzingVideoEncoder::ShouldDiscard( + uint16_t frame_id, + const EncodedImage& encoded_image) { + std::string stream_label = analyzer_->GetStreamLabel(frame_id); + absl::optional required_spatial_index = + stream_required_spatial_index_[stream_label]; + if (required_spatial_index) { + RTC_CHECK(encoded_image.SpatialIndex()) + << "Specific spatial layer/simulcast stream requested for track, but " + "now spatial layers/simulcast streams produced by encoder. " + "stream_label=" + << stream_label + << "; required_spatial_index=" << *required_spatial_index; + RTC_CHECK(mode_ != SimulcastMode::kNormal) + << "Analyzing encoder is in kNormal " + "mode, but spatial layer/simulcast " + "stream met."; + if (mode_ == SimulcastMode::kSimulcast) { + // In simulcast mode only encoded images with required spatial index are + // interested, so all others have to be discarded. + return *encoded_image.SpatialIndex() != *required_spatial_index; + } else if (mode_ == SimulcastMode::kSVC) { + // In SVC mode encoded images with spatial indexes that are equal or + // less than required one are interesting, so all above have to be + // discarded. + return *encoded_image.SpatialIndex() > *required_spatial_index; + } else if (mode_ == SimulcastMode::kKSVC) { + // In KSVC mode for key frame encoded images with spatial indexes that + // are equal or less than required one are interesting, so all above + // have to be discarded. For other frames only required spatial index + // is interesting, so all others have to be discarded. + if (encoded_image._frameType == FrameType::kVideoFrameKey) { + return *encoded_image.SpatialIndex() > *required_spatial_index; + } else { + return *encoded_image.SpatialIndex() != *required_spatial_index; + } + } else { + RTC_NOTREACHED() << "Unsupported encoder mode"; + } + } + return false; +} + QualityAnalyzingVideoEncoderFactory::QualityAnalyzingVideoEncoderFactory( std::unique_ptr delegate, + std::map> stream_required_spatial_index, IdGenerator* id_generator, - EncodedImageIdInjector* injector, + EncodedImageDataInjector* injector, VideoQualityAnalyzerInterface* analyzer) : delegate_(std::move(delegate)), + stream_required_spatial_index_(std::move(stream_required_spatial_index)), id_generator_(id_generator), injector_(injector), analyzer_(analyzer) {} @@ -208,7 +288,7 @@ QualityAnalyzingVideoEncoderFactory::CreateVideoEncoder( const SdpVideoFormat& format) { return absl::make_unique( id_generator_->GetNextId(), delegate_->CreateVideoEncoder(format), - injector_, analyzer_); + stream_required_spatial_index_, injector_, analyzer_); } } // namespace test diff --git a/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h b/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h index 56b3334dcf..b59c81cfe4 100644 --- a/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h +++ b/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h @@ -22,7 +22,7 @@ #include "api/video_codecs/video_encoder.h" #include "api/video_codecs/video_encoder_factory.h" #include "rtc_base/critical_section.h" -#include "test/pc/e2e/analyzer/video/encoded_image_id_injector.h" +#include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h" #include "test/pc/e2e/analyzer/video/id_generator.h" #include "test/pc/e2e/api/video_quality_analyzer_interface.h" @@ -41,7 +41,7 @@ namespace test { // // When origin encoder encodes the image it will call quality encoder's special // callback, where video analyzer will be called again and then frame id will be -// injected into EncodedImage with passed EncodedImageIdInjector. Then new +// injected into EncodedImage with passed EncodedImageDataInjector. Then new // EncodedImage will be passed to origin callback, provided by user. // // Quality encoder registers its own callback in origin encoder at the same @@ -51,11 +51,13 @@ class QualityAnalyzingVideoEncoder : public VideoEncoder, public: // Creates analyzing encoder. |id| is unique coding entity id, that will // be used to distinguish all encoders and decoders inside - // EncodedImageIdInjector and EncodedImageIdExtracor. - QualityAnalyzingVideoEncoder(int id, - std::unique_ptr delegate, - EncodedImageIdInjector* injector, - VideoQualityAnalyzerInterface* analyzer); + // EncodedImageDataInjector and EncodedImageIdExtracor. + QualityAnalyzingVideoEncoder( + int id, + std::unique_ptr delegate, + std::map> stream_required_spatial_index, + EncodedImageDataInjector* injector, + VideoQualityAnalyzerInterface* analyzer); ~QualityAnalyzingVideoEncoder() override; // Methods of VideoEncoder interface. @@ -81,9 +83,61 @@ class QualityAnalyzingVideoEncoder : public VideoEncoder, void OnDroppedFrame(DropReason reason) override; private: + enum SimulcastMode { + // In this mode encoder assumes not more than 1 encoded image per video + // frame + kNormal, + + // Next modes are to test video conference behavior. For conference sender + // will send multiple spatial layers/simulcast streams for single video + // track and there is some Selective Forwarding Unit (SFU), that forwards + // only best one, that will pass through downlink to the receiver. + // + // Here this behavior will be partly emulated. Sender will send all spatial + // layers/simulcast streams and then some of them will be filtered out on + // the receiver side. During test setup user can specify which spatial + // layer/simulcast stream is required, what will simulated which spatial + // layer/simulcast stream will be chosen by SFU in the real world. Then + // sender will mark encoded images for all spatial layers above required or + // all simulcast streams except required as to be discarded and on receiver + // side they will be discarded in quality analyzing decoder and won't be + // passed into delegate decoder. + // + // If the sender for some reasons won't send specified spatial layer, then + // receiver still will fall back on lower spatial layers. But for simulcast + // streams if required one won't be sent, receiver will assume all frames + // in that period as dropped and will experience video freeze. + // + // Test based on this simulation will be used to evaluate video quality + // of concrete spatial layers/simulcast streams and also check distribution + // of bandwidth between spatial layers/simulcast streams by BWE. + + // In this mode encoder assumes that for each frame simulcast encoded + // images will be produced. So all simulcast streams except required will + // be marked as to be discarded in decoder and won't reach video quality + // analyzer. + kSimulcast, + // In this mode encoder assumes that for each frame encoded images for + // different spatial layers will be produced. So all spatial layers above + // required will be marked to be discarded in decoder and won't reach + // video quality analyzer. + kSVC, + // In this mode encoder assumes that for each frame encoded images for + // different spatial layers will be produced. Compared to kSVC mode + // spatial layers that are above required will be marked to be discarded + // only for key frames and for regular frames all except required spatial + // layer will be marked as to be discarded in decoder and won't reach video + // quality analyzer. + kKSVC + }; + + bool ShouldDiscard(uint16_t frame_id, const EncodedImage& encoded_image) + RTC_EXCLUSIVE_LOCKS_REQUIRED(lock_); + const int id_; std::unique_ptr delegate_; - EncodedImageIdInjector* const injector_; + std::map> stream_required_spatial_index_; + EncodedImageDataInjector* const injector_; VideoQualityAnalyzerInterface* const analyzer_; // VideoEncoder interface assumes async delivery of encoded images. @@ -91,6 +145,7 @@ class QualityAnalyzingVideoEncoder : public VideoEncoder, // from received VideoFrame to resulted EncodedImage. rtc::CriticalSection lock_; + SimulcastMode mode_ RTC_GUARDED_BY(lock_); EncodedImageCallback* delegate_callback_ RTC_GUARDED_BY(lock_); std::list> timestamp_to_frame_id_list_ RTC_GUARDED_BY(lock_); @@ -103,8 +158,9 @@ class QualityAnalyzingVideoEncoderFactory : public VideoEncoderFactory { public: QualityAnalyzingVideoEncoderFactory( std::unique_ptr delegate, + std::map> stream_required_spatial_index, IdGenerator* id_generator, - EncodedImageIdInjector* injector, + EncodedImageDataInjector* injector, VideoQualityAnalyzerInterface* analyzer); ~QualityAnalyzingVideoEncoderFactory() override; @@ -117,8 +173,9 @@ class QualityAnalyzingVideoEncoderFactory : public VideoEncoderFactory { private: std::unique_ptr delegate_; + std::map> stream_required_spatial_index_; IdGenerator* const id_generator_; - EncodedImageIdInjector* const injector_; + EncodedImageDataInjector* const injector_; VideoQualityAnalyzerInterface* const analyzer_; }; diff --git a/test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector.cc b/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.cc similarity index 63% rename from test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector.cc rename to test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.cc index 105ee8eacd..89be698508 100644 --- a/test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector.cc +++ b/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.cc @@ -8,7 +8,7 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector.h" +#include "test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h" #include @@ -26,19 +26,21 @@ constexpr size_t kUsedBufferSize = 3; } // namespace -SingleProcessEncodedImageIdInjector::SingleProcessEncodedImageIdInjector() = - default; -SingleProcessEncodedImageIdInjector::~SingleProcessEncodedImageIdInjector() = +SingleProcessEncodedImageDataInjector::SingleProcessEncodedImageDataInjector() = default; +SingleProcessEncodedImageDataInjector:: + ~SingleProcessEncodedImageDataInjector() = default; -EncodedImage SingleProcessEncodedImageIdInjector::InjectId( +EncodedImage SingleProcessEncodedImageDataInjector::InjectData( uint16_t id, + bool discard, const EncodedImage& source, int coding_entity_id) { RTC_CHECK(source.size() >= kUsedBufferSize); ExtractionInfo info; info.length = source.size(); + info.discard = discard; memcpy(info.origin_data, source.data(), kUsedBufferSize); { rtc::CritScope crit(&lock_); @@ -55,18 +57,24 @@ EncodedImage SingleProcessEncodedImageIdInjector::InjectId( return out; } -EncodedImageWithId SingleProcessEncodedImageIdInjector::ExtractId( +EncodedImageExtractionResult SingleProcessEncodedImageDataInjector::ExtractData( const EncodedImage& source, int coding_entity_id) { EncodedImage out = source; + // Both |source| and |out| image will share the same buffer for payload or + // out will have a copy for it, so we can operate on the |out| buffer only. + uint8_t* buffer = out.data(); + size_t size = out.size(); + size_t pos = 0; absl::optional id = absl::nullopt; - while (pos < source.size()) { + bool discard = true; + while (pos < size) { // Extract frame id from first 2 bytes of the payload. - uint16_t next_id = source.data()[pos] + (source.data()[pos + 1] << 8); + uint16_t next_id = buffer[pos] + (buffer[pos + 1] << 8); // Extract frame sub id from second 2 byte of the payload. - uint16_t sub_id = source.data()[pos + 2]; + uint16_t sub_id = buffer[pos + 2]; RTC_CHECK(!id || id.value() == next_id) << "Different frames encoded into single encoded image: " << id.value() @@ -87,17 +95,28 @@ EncodedImageWithId SingleProcessEncodedImageIdInjector::ExtractId( ext_vector_it->second.infos.erase(info_it); } - memcpy(&out.data()[pos], info.origin_data, kUsedBufferSize); - pos += info.length; + if (info.discard) { + // If this encoded image is marked to be discarded - erase it's payload + // from the buffer. + memmove(&buffer[pos], &buffer[pos + info.length], + size - pos - info.length); + size -= info.length; + } else { + memmove(&buffer[pos], info.origin_data, kUsedBufferSize); + pos += info.length; + } + // We need to discard encoded image only if all concatenated encoded images + // have to be discarded. + discard = discard & info.discard; } out.set_size(pos); - return EncodedImageWithId{id.value(), out}; + return EncodedImageExtractionResult{id.value(), out, discard}; } -SingleProcessEncodedImageIdInjector::ExtractionInfoVector:: +SingleProcessEncodedImageDataInjector::ExtractionInfoVector:: ExtractionInfoVector() = default; -SingleProcessEncodedImageIdInjector::ExtractionInfoVector:: +SingleProcessEncodedImageDataInjector::ExtractionInfoVector:: ~ExtractionInfoVector() = default; } // namespace test diff --git a/test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector.h b/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h similarity index 59% rename from test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector.h rename to test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h index 94e1c01cfd..c5d47c7d4b 100644 --- a/test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector.h +++ b/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h @@ -8,8 +8,8 @@ * be found in the AUTHORS file in the root of the source tree. */ -#ifndef TEST_PC_E2E_ANALYZER_VIDEO_SINGLE_PROCESS_ENCODED_IMAGE_ID_INJECTOR_H_ -#define TEST_PC_E2E_ANALYZER_VIDEO_SINGLE_PROCESS_ENCODED_IMAGE_ID_INJECTOR_H_ +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_SINGLE_PROCESS_ENCODED_IMAGE_DATA_INJECTOR_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_SINGLE_PROCESS_ENCODED_IMAGE_DATA_INJECTOR_H_ #include #include @@ -19,36 +19,39 @@ #include "api/video/encoded_image.h" #include "rtc_base/critical_section.h" -#include "test/pc/e2e/analyzer/video/encoded_image_id_injector.h" +#include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h" namespace webrtc { namespace test { // Based on assumption that all call participants are in the same OS process -// and uses same QualityAnalyzingVideoContext to obtain EncodedImageIdInjector. +// and uses same QualityAnalyzingVideoContext to obtain +// EncodedImageDataInjector. // -// To inject frame id into EncodedImage injector uses first 2 bytes of -// EncodedImage payload. Then it uses 3rd byte for frame sub id, that is -// required to distinguish different spatial layers. The origin data from these -// 3 bytes will be stored inside injector's internal storage and then will be -// restored during extraction phase. +// To inject frame id and discard flag into EncodedImage injector uses first 2 +// bytes of EncodedImage payload. Then it uses 3rd byte for frame sub id, that +// is required to distinguish different spatial layers. The origin data from +// these 3 bytes will be stored inside injector's internal storage and then will +// be restored during extraction phase. // // This injector won't add any extra overhead into EncodedImage payload and // support frames with any size of payload. Also assumes that every EncodedImage // payload size is greater or equals to 3 bytes -class SingleProcessEncodedImageIdInjector : public EncodedImageIdInjector, - public EncodedImageIdExtractor { +class SingleProcessEncodedImageDataInjector : public EncodedImageDataInjector, + public EncodedImageDataExtractor { public: - SingleProcessEncodedImageIdInjector(); - ~SingleProcessEncodedImageIdInjector() override; + SingleProcessEncodedImageDataInjector(); + ~SingleProcessEncodedImageDataInjector() override; - // Id will be injected into EncodedImage buffer directly. This buffer won't - // be fully copied, so |source| image buffer will be also changed. - EncodedImage InjectId(uint16_t id, - const EncodedImage& source, - int coding_entity_id) override; - EncodedImageWithId ExtractId(const EncodedImage& source, - int coding_entity_id) override; + // Id and discard flag will be injected into EncodedImage buffer directly. + // This buffer won't be fully copied, so |source| image buffer will be also + // changed. + EncodedImage InjectData(uint16_t id, + bool discard, + const EncodedImage& source, + int coding_entity_id) override; + EncodedImageExtractionResult ExtractData(const EncodedImage& source, + int coding_entity_id) override; private: // Contains data required to extract frame id from EncodedImage and restore @@ -58,6 +61,9 @@ class SingleProcessEncodedImageIdInjector : public EncodedImageIdInjector, uint8_t sub_id; // Length of the origin buffer encoded image. size_t length; + // Flag to show is this encoded images should be discarded by analyzing + // decoder because of not required spatial layer/simulcast stream. + bool discard; // Data from first 3 bytes of origin encoded image's payload. uint8_t origin_data[3]; }; @@ -84,4 +90,4 @@ class SingleProcessEncodedImageIdInjector : public EncodedImageIdInjector, } // namespace test } // namespace webrtc -#endif // TEST_PC_E2E_ANALYZER_VIDEO_SINGLE_PROCESS_ENCODED_IMAGE_ID_INJECTOR_H_ +#endif // TEST_PC_E2E_ANALYZER_VIDEO_SINGLE_PROCESS_ENCODED_IMAGE_DATA_INJECTOR_H_ diff --git a/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector_unittest.cc b/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector_unittest.cc new file mode 100644 index 0000000000..636877df40 --- /dev/null +++ b/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector_unittest.cc @@ -0,0 +1,198 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h" + +#include + +#include "api/video/encoded_image.h" +#include "rtc_base/buffer.h" +#include "test/gtest.h" + +namespace webrtc { +namespace test { +namespace { + +rtc::Buffer CreateBufferOfSizeNFilledWithValuesFromX(size_t n, uint8_t x) { + rtc::Buffer buffer(n); + for (size_t i = 0; i < n; ++i) { + buffer[i] = static_cast(x + i); + } + return buffer; +} + +} // namespace + +TEST(SingleProcessEncodedImageDataInjector, InjectExtractDiscardFalse) { + SingleProcessEncodedImageDataInjector injector; + + rtc::Buffer buffer = CreateBufferOfSizeNFilledWithValuesFromX(10, 1); + + EncodedImage source(buffer.data(), 10, 10); + source.SetTimestamp(123456789); + + EncodedImageExtractionResult out = + injector.ExtractData(injector.InjectData(512, false, source, 1), 2); + EXPECT_EQ(out.id, 512); + EXPECT_FALSE(out.discard); + EXPECT_EQ(out.image.size(), 10ul); + EXPECT_EQ(out.image.capacity(), 10ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out.image.data()[i], i + 1); + } +} + +TEST(SingleProcessEncodedImageDataInjector, InjectExtractDiscardTrue) { + SingleProcessEncodedImageDataInjector injector; + + rtc::Buffer buffer = CreateBufferOfSizeNFilledWithValuesFromX(10, 1); + + EncodedImage source(buffer.data(), 10, 10); + source.SetTimestamp(123456789); + + EncodedImageExtractionResult out = + injector.ExtractData(injector.InjectData(512, true, source, 1), 2); + EXPECT_EQ(out.id, 512); + EXPECT_TRUE(out.discard); + EXPECT_EQ(out.image.size(), 0ul); + EXPECT_EQ(out.image.capacity(), 10ul); +} + +TEST(SingleProcessEncodedImageDataInjector, Inject3Extract3) { + SingleProcessEncodedImageDataInjector injector; + + rtc::Buffer buffer1 = CreateBufferOfSizeNFilledWithValuesFromX(10, 1); + rtc::Buffer buffer2 = CreateBufferOfSizeNFilledWithValuesFromX(10, 11); + rtc::Buffer buffer3 = CreateBufferOfSizeNFilledWithValuesFromX(10, 21); + + // 1st frame + EncodedImage source1(buffer1.data(), 10, 10); + source1.SetTimestamp(123456710); + // 2nd frame 1st spatial layer + EncodedImage source2(buffer2.data(), 10, 10); + source2.SetTimestamp(123456720); + // 2nd frame 2nd spatial layer + EncodedImage source3(buffer3.data(), 10, 10); + source3.SetTimestamp(123456720); + + EncodedImage intermediate1 = injector.InjectData(510, false, source1, 1); + EncodedImage intermediate2 = injector.InjectData(520, true, source2, 1); + EncodedImage intermediate3 = injector.InjectData(520, false, source3, 1); + + // Extract ids in different order. + EncodedImageExtractionResult out3 = injector.ExtractData(intermediate3, 2); + EncodedImageExtractionResult out1 = injector.ExtractData(intermediate1, 2); + EncodedImageExtractionResult out2 = injector.ExtractData(intermediate2, 2); + + EXPECT_EQ(out1.id, 510); + EXPECT_FALSE(out1.discard); + EXPECT_EQ(out1.image.size(), 10ul); + EXPECT_EQ(out1.image.capacity(), 10ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out1.image.data()[i], i + 1); + } + EXPECT_EQ(out2.id, 520); + EXPECT_TRUE(out2.discard); + EXPECT_EQ(out2.image.size(), 0ul); + EXPECT_EQ(out2.image.capacity(), 10ul); + EXPECT_EQ(out3.id, 520); + EXPECT_FALSE(out3.discard); + EXPECT_EQ(out3.image.size(), 10ul); + EXPECT_EQ(out3.image.capacity(), 10ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out3.image.data()[i], i + 21); + } +} + +TEST(SingleProcessEncodedImageDataInjector, InjectExtractFromConcatenated) { + SingleProcessEncodedImageDataInjector injector; + + rtc::Buffer buffer1 = CreateBufferOfSizeNFilledWithValuesFromX(10, 1); + rtc::Buffer buffer2 = CreateBufferOfSizeNFilledWithValuesFromX(10, 11); + rtc::Buffer buffer3 = CreateBufferOfSizeNFilledWithValuesFromX(10, 21); + + EncodedImage source1(buffer1.data(), 10, 10); + source1.SetTimestamp(123456710); + EncodedImage source2(buffer2.data(), 10, 10); + source2.SetTimestamp(123456710); + EncodedImage source3(buffer3.data(), 10, 10); + source3.SetTimestamp(123456710); + + // Inject id into 3 images with same frame id. + EncodedImage intermediate1 = injector.InjectData(512, false, source1, 1); + EncodedImage intermediate2 = injector.InjectData(512, true, source2, 1); + EncodedImage intermediate3 = injector.InjectData(512, false, source3, 1); + + // Concatenate them into single encoded image, like it can be done in jitter + // buffer. + size_t concatenated_length = + intermediate1.size() + intermediate2.size() + intermediate3.size(); + rtc::Buffer concatenated_buffer; + concatenated_buffer.AppendData(intermediate1.data(), intermediate1.size()); + concatenated_buffer.AppendData(intermediate2.data(), intermediate2.size()); + concatenated_buffer.AppendData(intermediate3.data(), intermediate3.size()); + EncodedImage concatenated(concatenated_buffer.data(), concatenated_length, + concatenated_length); + + // Extract frame id from concatenated image + EncodedImageExtractionResult out = injector.ExtractData(concatenated, 2); + + EXPECT_EQ(out.id, 512); + EXPECT_FALSE(out.discard); + EXPECT_EQ(out.image.size(), 2 * 10ul); + EXPECT_EQ(out.image.capacity(), 3 * 10ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out.image.data()[i], i + 1); + EXPECT_EQ(out.image.data()[i + 10], i + 21); + } +} + +TEST(SingleProcessEncodedImageDataInjector, + InjectExtractFromConcatenatedAllDiscarded) { + SingleProcessEncodedImageDataInjector injector; + + rtc::Buffer buffer1 = CreateBufferOfSizeNFilledWithValuesFromX(10, 1); + rtc::Buffer buffer2 = CreateBufferOfSizeNFilledWithValuesFromX(10, 11); + rtc::Buffer buffer3 = CreateBufferOfSizeNFilledWithValuesFromX(10, 21); + + EncodedImage source1(buffer1.data(), 10, 10); + source1.SetTimestamp(123456710); + EncodedImage source2(buffer2.data(), 10, 10); + source2.SetTimestamp(123456710); + EncodedImage source3(buffer3.data(), 10, 10); + source3.SetTimestamp(123456710); + + // Inject id into 3 images with same frame id. + EncodedImage intermediate1 = injector.InjectData(512, true, source1, 1); + EncodedImage intermediate2 = injector.InjectData(512, true, source2, 1); + EncodedImage intermediate3 = injector.InjectData(512, true, source3, 1); + + // Concatenate them into single encoded image, like it can be done in jitter + // buffer. + size_t concatenated_length = + intermediate1.size() + intermediate2.size() + intermediate3.size(); + rtc::Buffer concatenated_buffer; + concatenated_buffer.AppendData(intermediate1.data(), intermediate1.size()); + concatenated_buffer.AppendData(intermediate2.data(), intermediate2.size()); + concatenated_buffer.AppendData(intermediate3.data(), intermediate3.size()); + EncodedImage concatenated(concatenated_buffer.data(), concatenated_length, + concatenated_length); + + // Extract frame id from concatenated image + EncodedImageExtractionResult out = injector.ExtractData(concatenated, 2); + + EXPECT_EQ(out.id, 512); + EXPECT_TRUE(out.discard); + EXPECT_EQ(out.image.size(), 0ul); + EXPECT_EQ(out.image.capacity(), 3 * 10ul); +} + +} // namespace test +} // namespace webrtc diff --git a/test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector_unittest.cc b/test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector_unittest.cc deleted file mode 100644 index a554c025db..0000000000 --- a/test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector_unittest.cc +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector.h" - -#include - -#include "api/video/encoded_image.h" -#include "rtc_base/buffer.h" -#include "test/gtest.h" - -namespace webrtc { -namespace test { -namespace { - -rtc::Buffer CreateBufferOfSizeNFilledWithValuesFromX(size_t n, uint8_t x) { - rtc::Buffer buffer(n); - for (size_t i = 0; i < n; ++i) { - buffer[i] = static_cast(x + i); - } - return buffer; -} - -} // namespace - -TEST(SingleProcessEncodedImageIdInjector, InjectExtract) { - SingleProcessEncodedImageIdInjector injector; - - rtc::Buffer buffer = CreateBufferOfSizeNFilledWithValuesFromX(10, 1); - - EncodedImage source(buffer.data(), 10, 10); - source.SetTimestamp(123456789); - - EncodedImageWithId out = - injector.ExtractId(injector.InjectId(512, source, 1), 2); - ASSERT_EQ(out.id, 512); - ASSERT_EQ(out.image.size(), 10ul); - ASSERT_EQ(out.image.capacity(), 10ul); - for (int i = 0; i < 10; ++i) { - ASSERT_EQ(out.image.data()[i], i + 1); - } -} - -TEST(SingleProcessEncodedImageIdInjector, Inject3Extract3) { - SingleProcessEncodedImageIdInjector injector; - - rtc::Buffer buffer1 = CreateBufferOfSizeNFilledWithValuesFromX(10, 1); - rtc::Buffer buffer2 = CreateBufferOfSizeNFilledWithValuesFromX(10, 11); - rtc::Buffer buffer3 = CreateBufferOfSizeNFilledWithValuesFromX(10, 21); - - // 1st frame - EncodedImage source1(buffer1.data(), 10, 10); - source1.SetTimestamp(123456710); - // 2nd frame 1st spatial layer - EncodedImage source2(buffer2.data(), 10, 10); - source2.SetTimestamp(123456720); - // 2nd frame 2nd spatial layer - EncodedImage source3(buffer3.data(), 10, 10); - source3.SetTimestamp(123456720); - - EncodedImage intermediate1 = injector.InjectId(510, source1, 1); - EncodedImage intermediate2 = injector.InjectId(520, source2, 1); - EncodedImage intermediate3 = injector.InjectId(520, source3, 1); - - // Extract ids in different order. - EncodedImageWithId out3 = injector.ExtractId(intermediate3, 2); - EncodedImageWithId out1 = injector.ExtractId(intermediate1, 2); - EncodedImageWithId out2 = injector.ExtractId(intermediate2, 2); - - ASSERT_EQ(out1.id, 510); - ASSERT_EQ(out1.image.size(), 10ul); - ASSERT_EQ(out1.image.capacity(), 10ul); - for (int i = 0; i < 10; ++i) { - ASSERT_EQ(out1.image.data()[i], i + 1); - } - ASSERT_EQ(out2.id, 520); - ASSERT_EQ(out2.image.size(), 10ul); - ASSERT_EQ(out2.image.capacity(), 10ul); - for (int i = 0; i < 10; ++i) { - ASSERT_EQ(out2.image.data()[i], i + 11); - } - ASSERT_EQ(out3.id, 520); - ASSERT_EQ(out3.image.size(), 10ul); - ASSERT_EQ(out3.image.capacity(), 10ul); - for (int i = 0; i < 10; ++i) { - ASSERT_EQ(out3.image.data()[i], i + 21); - } -} - -TEST(SingleProcessEncodedImageIdInjector, InjectExtractFromConcatenated) { - SingleProcessEncodedImageIdInjector injector; - - rtc::Buffer buffer1 = CreateBufferOfSizeNFilledWithValuesFromX(10, 1); - rtc::Buffer buffer2 = CreateBufferOfSizeNFilledWithValuesFromX(10, 11); - rtc::Buffer buffer3 = CreateBufferOfSizeNFilledWithValuesFromX(10, 21); - - EncodedImage source1(buffer1.data(), 10, 10); - source1.SetTimestamp(123456710); - EncodedImage source2(buffer2.data(), 10, 10); - source2.SetTimestamp(123456710); - EncodedImage source3(buffer3.data(), 10, 10); - source3.SetTimestamp(123456710); - - // Inject id into 3 images with same frame id. - EncodedImage intermediate1 = injector.InjectId(512, source1, 1); - EncodedImage intermediate2 = injector.InjectId(512, source2, 1); - EncodedImage intermediate3 = injector.InjectId(512, source3, 1); - - // Concatenate them into single encoded image, like it can be done in jitter - // buffer. - size_t concatenated_length = - intermediate1.size() + intermediate2.size() + intermediate3.size(); - rtc::Buffer concatenated_buffer; - concatenated_buffer.AppendData(intermediate1.data(), intermediate1.size()); - concatenated_buffer.AppendData(intermediate2.data(), intermediate2.size()); - concatenated_buffer.AppendData(intermediate3.data(), intermediate3.size()); - EncodedImage concatenated(concatenated_buffer.data(), concatenated_length, - concatenated_length); - - // Extract frame id from concatenated image - EncodedImageWithId out = injector.ExtractId(concatenated, 2); - - ASSERT_EQ(out.id, 512); - ASSERT_EQ(out.image.size(), 3 * 10ul); - ASSERT_EQ(out.image.capacity(), 3 * 10ul); - for (int i = 0; i < 10; ++i) { - ASSERT_EQ(out.image.data()[i], i + 1); - ASSERT_EQ(out.image.data()[i + 10], i + 11); - ASSERT_EQ(out.image.data()[i + 20], i + 21); - } -} - -} // namespace test -} // namespace webrtc diff --git a/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc b/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc index 815505c810..df2893ac8c 100644 --- a/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc +++ b/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc @@ -88,8 +88,8 @@ class AnalyzingVideoSink : public rtc::VideoSinkInterface { VideoQualityAnalyzerInjectionHelper::VideoQualityAnalyzerInjectionHelper( std::unique_ptr analyzer, - EncodedImageIdInjector* injector, - EncodedImageIdExtractor* extractor) + EncodedImageDataInjector* injector, + EncodedImageDataExtractor* extractor) : analyzer_(std::move(analyzer)), injector_(injector), extractor_(extractor), @@ -102,10 +102,12 @@ VideoQualityAnalyzerInjectionHelper::~VideoQualityAnalyzerInjectionHelper() = std::unique_ptr VideoQualityAnalyzerInjectionHelper::WrapVideoEncoderFactory( - std::unique_ptr delegate) const { + std::unique_ptr delegate, + std::map> stream_required_spatial_index) + const { return absl::make_unique( - std::move(delegate), encoding_entities_id_generator_.get(), injector_, - analyzer_.get()); + std::move(delegate), std::move(stream_required_spatial_index), + encoding_entities_id_generator_.get(), injector_, analyzer_.get()); } std::unique_ptr diff --git a/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h b/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h index 880723f88f..1908b51874 100644 --- a/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h +++ b/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h @@ -11,6 +11,7 @@ #ifndef TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_QUALITY_ANALYZER_INJECTION_HELPER_H_ #define TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_QUALITY_ANALYZER_INJECTION_HELPER_H_ +#include #include #include @@ -19,7 +20,7 @@ #include "api/video_codecs/video_decoder_factory.h" #include "api/video_codecs/video_encoder_factory.h" #include "test/frame_generator.h" -#include "test/pc/e2e/analyzer/video/encoded_image_id_injector.h" +#include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h" #include "test/pc/e2e/analyzer/video/id_generator.h" #include "test/pc/e2e/api/video_quality_analyzer_interface.h" #include "test/testsupport/video_frame_writer.h" @@ -33,14 +34,16 @@ class VideoQualityAnalyzerInjectionHelper { public: VideoQualityAnalyzerInjectionHelper( std::unique_ptr analyzer, - EncodedImageIdInjector* injector, - EncodedImageIdExtractor* extractor); + EncodedImageDataInjector* injector, + EncodedImageDataExtractor* extractor); ~VideoQualityAnalyzerInjectionHelper(); // Wraps video encoder factory to give video quality analyzer access to frames // before encoding and encoded images after. std::unique_ptr WrapVideoEncoderFactory( - std::unique_ptr delegate) const; + std::unique_ptr delegate, + std::map> stream_required_spatial_index) + const; // Wraps video decoder factory to give video quality analyzer access to // received encoded images and frames, that were decoded from them. std::unique_ptr WrapVideoDecoderFactory( @@ -66,8 +69,8 @@ class VideoQualityAnalyzerInjectionHelper { private: std::unique_ptr analyzer_; - EncodedImageIdInjector* injector_; - EncodedImageIdExtractor* extractor_; + EncodedImageDataInjector* injector_; + EncodedImageDataExtractor* extractor_; std::unique_ptr> encoding_entities_id_generator_; }; diff --git a/test/pc/e2e/api/peerconnection_quality_test_fixture.h b/test/pc/e2e/api/peerconnection_quality_test_fixture.h index 79354a283c..4ed1a5bbd8 100644 --- a/test/pc/e2e/api/peerconnection_quality_test_fixture.h +++ b/test/pc/e2e/api/peerconnection_quality_test_fixture.h @@ -125,6 +125,19 @@ class PeerConnectionE2EQualityTestFixture { absl::optional input_file_name; // If specified screen share video stream will be created as input. absl::optional screen_share_config; + // Specifies spatial index of the video stream to analyze. + // There are 3 cases: + // 1. |target_spatial_index| omitted: in such case it will be assumed that + // video stream has not spatial layers and simulcast streams. + // 2. |target_spatial_index| presented and simulcast encoder is used: + // in such case |target_spatial_index| will specify the index of + // simulcast strem, that should be analyzed. Other streams will be + // dropped. + // 3. |target_spatial_index| presented and SVP encoder is used: + // in such case |target_spatial_index| will specify the top interesting + // spatial layer and all layers bellow, including target one will be + // processed. All layers above target one will be dropped. + absl::optional target_spatial_index; // If specified the input stream will be also copied to specified file. // It is actually one of the test's output file, which contains copy of what // was captured during the test for this video stream on sender side. diff --git a/test/pc/e2e/api/video_quality_analyzer_interface.h b/test/pc/e2e/api/video_quality_analyzer_interface.h index c112e09b66..4a7096f31c 100644 --- a/test/pc/e2e/api/video_quality_analyzer_interface.h +++ b/test/pc/e2e/api/video_quality_analyzer_interface.h @@ -93,6 +93,8 @@ class VideoQualityAnalyzerInterface { // Tells analyzer that analysis complete and it should calculate final // statistics. virtual void Stop() {} + + virtual std::string GetStreamLabel(uint16_t frame_id) = 0; }; } // namespace webrtc diff --git a/test/pc/e2e/peer_connection_quality_test.cc b/test/pc/e2e/peer_connection_quality_test.cc index d2d8551b4c..c7bd0a657d 100644 --- a/test/pc/e2e/peer_connection_quality_test.cc +++ b/test/pc/e2e/peer_connection_quality_test.cc @@ -101,7 +101,7 @@ PeerConnectionE2EQualityTest::PeerConnectionE2EQualityTest( video_quality_analyzer = absl::make_unique(); } encoded_image_id_controller_ = - absl::make_unique(); + absl::make_unique(); video_quality_analyzer_injection_helper_ = absl::make_unique( std::move(video_quality_analyzer), encoded_image_id_controller_.get(), diff --git a/test/pc/e2e/peer_connection_quality_test.h b/test/pc/e2e/peer_connection_quality_test.h index f401b639cc..eda6db3822 100644 --- a/test/pc/e2e/peer_connection_quality_test.h +++ b/test/pc/e2e/peer_connection_quality_test.h @@ -18,7 +18,7 @@ #include "rtc_base/task_queue.h" #include "rtc_base/thread.h" #include "system_wrappers/include/clock.h" -#include "test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector.h" +#include "test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h" #include "test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h" #include "test/pc/e2e/api/peerconnection_quality_test_fixture.h" #include "test/pc/e2e/test_peer.h" @@ -81,7 +81,7 @@ class PeerConnectionE2EQualityTest Clock* const clock_; std::unique_ptr video_quality_analyzer_injection_helper_; - std::unique_ptr + std::unique_ptr encoded_image_id_controller_; std::unique_ptr alice_; diff --git a/test/pc/e2e/test_peer.cc b/test/pc/e2e/test_peer.cc index d0e7e5a597..5dac263e29 100644 --- a/test/pc/e2e/test_peer.cc +++ b/test/pc/e2e/test_peer.cc @@ -29,7 +29,7 @@ #include "rtc_base/location.h" #include "rtc_base/network.h" #include "test/frame_generator_capturer.h" -#include "test/pc/e2e/analyzer/video/default_encoded_image_id_injector.h" +#include "test/pc/e2e/analyzer/video/default_encoded_image_data_injector.h" #include "test/pc/e2e/analyzer/video/example_video_quality_analyzer.h" #include "test/testsupport/copy_to_file_audio_capturer.h" @@ -122,7 +122,8 @@ rtc::scoped_refptr CreateAudioDeviceModule( std::unique_ptr CreateVideoEncoderFactory( PeerConnectionFactoryComponents* pcf_dependencies, - VideoQualityAnalyzerInjectionHelper* video_analyzer_helper) { + VideoQualityAnalyzerInjectionHelper* video_analyzer_helper, + std::map> stream_required_spatial_index) { std::unique_ptr video_encoder_factory; if (pcf_dependencies->video_encoder_factory != nullptr) { video_encoder_factory = std::move(pcf_dependencies->video_encoder_factory); @@ -130,7 +131,8 @@ std::unique_ptr CreateVideoEncoderFactory( video_encoder_factory = CreateBuiltinVideoEncoderFactory(); } return video_analyzer_helper->WrapVideoEncoderFactory( - std::move(video_encoder_factory)); + std::move(video_encoder_factory), + std::move(stream_required_spatial_index)); } std::unique_ptr CreateVideoDecoderFactory( @@ -149,13 +151,15 @@ std::unique_ptr CreateVideoDecoderFactory( std::unique_ptr CreateMediaEngine( PeerConnectionFactoryComponents* pcf_dependencies, absl::optional audio_config, + std::map> stream_required_spatial_index, VideoQualityAnalyzerInjectionHelper* video_analyzer_helper, absl::optional audio_output_file_name) { rtc::scoped_refptr adm = CreateAudioDeviceModule( std::move(audio_config), std::move(audio_output_file_name)); std::unique_ptr video_encoder_factory = - CreateVideoEncoderFactory(pcf_dependencies, video_analyzer_helper); + CreateVideoEncoderFactory(pcf_dependencies, video_analyzer_helper, + std::move(stream_required_spatial_index)); std::unique_ptr video_decoder_factory = CreateVideoDecoderFactory(pcf_dependencies, video_analyzer_helper); @@ -173,6 +177,7 @@ std::unique_ptr CreateMediaEngine( PeerConnectionFactoryDependencies CreatePCFDependencies( std::unique_ptr pcf_dependencies, absl::optional audio_config, + std::map> stream_required_spatial_index, VideoQualityAnalyzerInjectionHelper* video_analyzer_helper, rtc::Thread* network_thread, rtc::Thread* signaling_thread, @@ -181,7 +186,8 @@ PeerConnectionFactoryDependencies CreatePCFDependencies( pcf_deps.network_thread = network_thread; pcf_deps.signaling_thread = signaling_thread; pcf_deps.media_engine = CreateMediaEngine( - pcf_dependencies.get(), std::move(audio_config), video_analyzer_helper, + pcf_dependencies.get(), std::move(audio_config), + std::move(stream_required_spatial_index), video_analyzer_helper, std::move(audio_output_file_name)); pcf_deps.call_factory = std::move(pcf_dependencies->call_factory); @@ -254,10 +260,23 @@ std::unique_ptr TestPeer::CreateTestPeer( SetMandatoryEntities(components.get()); params->rtc_configuration.sdp_semantics = SdpSemantics::kUnifiedPlan; + std::map> stream_required_spatial_index; + for (auto& video_config : params->video_configs) { + // Stream label should be set by fixture implementation here. + RTC_DCHECK(video_config.stream_label); + bool res = stream_required_spatial_index + .insert({*video_config.stream_label, + video_config.target_spatial_index}) + .second; + RTC_DCHECK(res) << "Duplicate video_config.stream_label=" + << *video_config.stream_label; + } + // Create peer connection factory. PeerConnectionFactoryDependencies pcf_deps = CreatePCFDependencies( std::move(components->pcf_dependencies), params->audio_config, - video_analyzer_helper, components->network_thread, signaling_thread, + std::move(stream_required_spatial_index), video_analyzer_helper, + components->network_thread, signaling_thread, std::move(audio_output_file_name)); rtc::scoped_refptr pcf = CreateModularPeerConnectionFactory(std::move(pcf_deps)); diff --git a/test/pc/e2e/test_peer.h b/test/pc/e2e/test_peer.h index 3ee96c22ef..5f88de27f4 100644 --- a/test/pc/e2e/test_peer.h +++ b/test/pc/e2e/test_peer.h @@ -22,7 +22,7 @@ #include "pc/test/mock_peer_connection_observers.h" #include "rtc_base/network.h" #include "rtc_base/thread.h" -#include "test/pc/e2e/analyzer/video/encoded_image_id_injector.h" +#include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h" #include "test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h" #include "test/pc/e2e/api/peerconnection_quality_test_fixture.h"