diff --git a/test/pc/e2e/BUILD.gn b/test/pc/e2e/BUILD.gn index 4e47ab599e..ebf69c8bcd 100644 --- a/test/pc/e2e/BUILD.gn +++ b/test/pc/e2e/BUILD.gn @@ -12,13 +12,13 @@ group("e2e") { testonly = true deps = [ - ":default_encoded_image_id_injector", - ":encoded_image_id_injector_api", + ":default_encoded_image_data_injector", + ":encoded_image_data_injector_api", ":example_video_quality_analyzer", ":id_generator", ":quality_analyzing_video_decoder", ":quality_analyzing_video_encoder", - ":single_process_encoded_image_id_injector", + ":single_process_encoded_image_data_injector", ] if (rtc_include_tests) { deps += [ @@ -34,17 +34,17 @@ if (rtc_include_tests) { testonly = true deps = [ - ":default_encoded_image_id_injector_unittest", + ":default_encoded_image_data_injector_unittest", ":peer_connection_e2e_smoke_test", - ":single_process_encoded_image_id_injector_unittest", + ":single_process_encoded_image_data_injector_unittest", ] } } -rtc_source_set("encoded_image_id_injector_api") { +rtc_source_set("encoded_image_data_injector_api") { visibility = [ "*" ] sources = [ - "analyzer/video/encoded_image_id_injector.h", + "analyzer/video/encoded_image_data_injector.h", ] deps = [ @@ -52,15 +52,15 @@ rtc_source_set("encoded_image_id_injector_api") { ] } -rtc_source_set("default_encoded_image_id_injector") { +rtc_source_set("default_encoded_image_data_injector") { visibility = [ "*" ] sources = [ - "analyzer/video/default_encoded_image_id_injector.cc", - "analyzer/video/default_encoded_image_id_injector.h", + "analyzer/video/default_encoded_image_data_injector.cc", + "analyzer/video/default_encoded_image_data_injector.h", ] deps = [ - ":encoded_image_id_injector_api", + ":encoded_image_data_injector_api", "../../../api/video:encoded_image", "../../../rtc_base:checks", "../../../rtc_base:criticalsection", @@ -68,15 +68,15 @@ rtc_source_set("default_encoded_image_id_injector") { ] } -rtc_source_set("single_process_encoded_image_id_injector") { +rtc_source_set("single_process_encoded_image_data_injector") { visibility = [ "*" ] sources = [ - "analyzer/video/single_process_encoded_image_id_injector.cc", - "analyzer/video/single_process_encoded_image_id_injector.h", + "analyzer/video/single_process_encoded_image_data_injector.cc", + "analyzer/video/single_process_encoded_image_data_injector.h", ] deps = [ - ":encoded_image_id_injector_api", + ":encoded_image_data_injector_api", "../../../api/video:encoded_image", "../../../rtc_base:checks", "../../../rtc_base:criticalsection", @@ -100,7 +100,7 @@ rtc_source_set("quality_analyzing_video_decoder") { "analyzer/video/quality_analyzing_video_decoder.h", ] deps = [ - ":encoded_image_id_injector_api", + ":encoded_image_data_injector_api", ":id_generator", "../../../api/video:encoded_image", "../../../api/video:video_frame", @@ -121,7 +121,7 @@ rtc_source_set("quality_analyzing_video_encoder") { "analyzer/video/quality_analyzing_video_encoder.h", ] deps = [ - ":encoded_image_id_injector_api", + ":encoded_image_data_injector_api", ":id_generator", "../../../api/video:encoded_image", "../../../api/video:video_frame", @@ -143,7 +143,7 @@ if (rtc_include_tests) { "analyzer/video/video_quality_analyzer_injection_helper.h", ] deps = [ - ":encoded_image_id_injector_api", + ":encoded_image_data_injector_api", ":id_generator", ":quality_analyzing_video_decoder", ":quality_analyzing_video_encoder", @@ -164,8 +164,8 @@ if (rtc_include_tests) { "test_peer.h", ] deps = [ - ":default_encoded_image_id_injector", - ":encoded_image_id_injector_api", + ":default_encoded_image_data_injector", + ":encoded_image_data_injector_api", ":example_video_quality_analyzer", ":video_quality_analyzer_injection_helper", "../../../api:array_view", @@ -206,7 +206,7 @@ if (rtc_include_tests) { ] deps = [ ":example_video_quality_analyzer", - ":single_process_encoded_image_id_injector", + ":single_process_encoded_image_data_injector", ":test_peer", ":video_quality_analyzer_injection_helper", "../../../api:libjingle_peerconnection_api", @@ -231,26 +231,26 @@ if (rtc_include_tests) { } } - rtc_source_set("single_process_encoded_image_id_injector_unittest") { + rtc_source_set("single_process_encoded_image_data_injector_unittest") { testonly = true sources = [ - "analyzer/video/single_process_encoded_image_id_injector_unittest.cc", + "analyzer/video/single_process_encoded_image_data_injector_unittest.cc", ] deps = [ - ":single_process_encoded_image_id_injector", + ":single_process_encoded_image_data_injector", "../../../api/video:encoded_image", "../../../rtc_base:rtc_base_approved", "../../../test:test_support", ] } - rtc_source_set("default_encoded_image_id_injector_unittest") { + rtc_source_set("default_encoded_image_data_injector_unittest") { testonly = true sources = [ - "analyzer/video/default_encoded_image_id_injector_unittest.cc", + "analyzer/video/default_encoded_image_data_injector_unittest.cc", ] deps = [ - ":default_encoded_image_id_injector", + ":default_encoded_image_data_injector", "../../../api/video:encoded_image", "../../../rtc_base:rtc_base_approved", "../../../test:test_support", diff --git a/test/pc/e2e/analyzer/video/default_encoded_image_id_injector.cc b/test/pc/e2e/analyzer/video/default_encoded_image_data_injector.cc similarity index 70% rename from test/pc/e2e/analyzer/video/default_encoded_image_id_injector.cc rename to test/pc/e2e/analyzer/video/default_encoded_image_data_injector.cc index f52325ece6..0820c4d939 100644 --- a/test/pc/e2e/analyzer/video/default_encoded_image_id_injector.cc +++ b/test/pc/e2e/analyzer/video/default_encoded_image_data_injector.cc @@ -8,7 +8,7 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "test/pc/e2e/analyzer/video/default_encoded_image_id_injector.h" +#include "test/pc/e2e/analyzer/video/default_encoded_image_data_injector.h" #include @@ -32,18 +32,20 @@ constexpr size_t kBuffersPoolPerCodingEntity = 256; } // namespace -DefaultEncodedImageIdInjector::DefaultEncodedImageIdInjector() { +DefaultEncodedImageDataInjector::DefaultEncodedImageDataInjector() { for (size_t i = 0; i < kPreInitCodingEntitiesCount * kBuffersPoolPerCodingEntity; ++i) { bufs_pool_.push_back( absl::make_unique>(kInitialBufferSize)); } } -DefaultEncodedImageIdInjector::~DefaultEncodedImageIdInjector() = default; +DefaultEncodedImageDataInjector::~DefaultEncodedImageDataInjector() = default; -EncodedImage DefaultEncodedImageIdInjector::InjectId(uint16_t id, - const EncodedImage& source, - int coding_entity_id) { +EncodedImage DefaultEncodedImageDataInjector::InjectData( + uint16_t id, + bool discard, + const EncodedImage& source, + int coding_entity_id) { ExtendIfRequired(coding_entity_id); EncodedImage out = source; @@ -57,14 +59,18 @@ EncodedImage DefaultEncodedImageIdInjector::InjectId(uint16_t id, source.size()); out.data()[0] = id & 0x00ff; out.data()[1] = (id & 0xff00) >> 8; - out.data()[2] = source.size() & 0x00ff; - out.data()[3] = (source.size() & 0xff00) >> 8; - out.data()[4] = (source.size() & 0xff00) >> 16; - out.data()[5] = (source.size() & 0xff00) >> 24; + out.data()[2] = source.size() & 0x000000ff; + out.data()[3] = (source.size() & 0x0000ff00) >> 8; + out.data()[4] = (source.size() & 0x00ff0000) >> 16; + out.data()[5] = (source.size() & 0xff000000) >> 24; + + // We will store discard flag in the high bit of high byte of the size. + RTC_CHECK_LT(source.size(), 1U << 31) << "High bit is already in use"; + out.data()[5] = out.data()[5] | ((discard ? 1 : 0) << 7); return out; } -EncodedImageWithId DefaultEncodedImageIdInjector::ExtractId( +EncodedImageExtractionResult DefaultEncodedImageDataInjector::ExtractData( const EncodedImage& source, int coding_entity_id) { ExtendIfRequired(coding_entity_id); @@ -79,6 +85,7 @@ EncodedImageWithId DefaultEncodedImageIdInjector::ExtractId( size_t source_pos = 0; size_t out_pos = 0; absl::optional id = absl::nullopt; + bool discard = true; while (source_pos < source.size()) { RTC_CHECK_LE(source_pos + kEncodedImageBufferExpansion, source.size()); uint16_t next_id = @@ -89,21 +96,29 @@ EncodedImageWithId DefaultEncodedImageIdInjector::ExtractId( id = next_id; uint32_t length = source.data()[source_pos + 2] + (source.data()[source_pos + 3] << 8) + - (source.data()[source_pos + 3] << 16) + - (source.data()[source_pos + 3] << 24); + (source.data()[source_pos + 4] << 16) + + ((source.data()[source_pos + 5] << 24) & 0b01111111); + bool current_discard = (source.data()[source_pos + 5] & 0b10000000) != 0; RTC_CHECK_LE(source_pos + kEncodedImageBufferExpansion + length, source.size()); - memcpy(&out.data()[out_pos], - &source.data()[source_pos + kEncodedImageBufferExpansion], length); + if (!current_discard) { + // Copy next encoded image payload from concatenated buffer only if it is + // not discarded. + memcpy(&out.data()[out_pos], + &source.data()[source_pos + kEncodedImageBufferExpansion], length); + out_pos += length; + } source_pos += length + kEncodedImageBufferExpansion; - out_pos += length; + // Extraction result is discarded only if all encoded partitions are + // discarded. + discard = discard && current_discard; } out.set_size(out_pos); - return EncodedImageWithId{id.value(), out}; + return EncodedImageExtractionResult{id.value(), out, discard}; } -void DefaultEncodedImageIdInjector::ExtendIfRequired(int coding_entity_id) { +void DefaultEncodedImageDataInjector::ExtendIfRequired(int coding_entity_id) { rtc::CritScope crit(&lock_); if (coding_entities_.find(coding_entity_id) != coding_entities_.end()) { // This entity is already known for this injector, so buffers are allocated. @@ -124,7 +139,7 @@ void DefaultEncodedImageIdInjector::ExtendIfRequired(int coding_entity_id) { } } -std::vector* DefaultEncodedImageIdInjector::NextBuffer() { +std::vector* DefaultEncodedImageDataInjector::NextBuffer() { rtc::CritScope crit(&lock_); // Get buffer from the front of the queue, return it to the caller and // put in the back diff --git a/test/pc/e2e/analyzer/video/default_encoded_image_id_injector.h b/test/pc/e2e/analyzer/video/default_encoded_image_data_injector.h similarity index 65% rename from test/pc/e2e/analyzer/video/default_encoded_image_id_injector.h rename to test/pc/e2e/analyzer/video/default_encoded_image_data_injector.h index 6f272714cf..989877df7e 100644 --- a/test/pc/e2e/analyzer/video/default_encoded_image_id_injector.h +++ b/test/pc/e2e/analyzer/video/default_encoded_image_data_injector.h @@ -8,8 +8,8 @@ * be found in the AUTHORS file in the root of the source tree. */ -#ifndef TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_ENCODED_IMAGE_ID_INJECTOR_H_ -#define TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_ENCODED_IMAGE_ID_INJECTOR_H_ +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_ENCODED_IMAGE_DATA_INJECTOR_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_ENCODED_IMAGE_DATA_INJECTOR_H_ #include #include @@ -20,20 +20,21 @@ #include "api/video/encoded_image.h" #include "rtc_base/critical_section.h" -#include "test/pc/e2e/analyzer/video/encoded_image_id_injector.h" +#include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h" namespace webrtc { namespace test { -// Injects frame id into EncodedImage payload buffer. The payload buffer will -// be prepended in the injector with 2 bytes frame id and 4 bytes original -// buffer length. It is assumed, that frame's data can't be more then 2^32 -// bytes. In the decoder, frame id will be extracted and the length will be used -// to restore original buffer. +// Injects frame id and discard flag into EncodedImage payload buffer. The +// payload buffer will be prepended in the injector with 2 bytes frame id and 4 +// bytes original buffer length. Discarded flag will be put into the highest bit +// of the length. It is assumed, that frame's data can't be more then 2^31 +// bytes. In the decoder, frame id and discard flag will be extracted and the +// length will be used to restore original buffer. // // The data in the EncodedImage on encoder side after injection will look like // this: -// 4 bytes frame length +// 4 bytes frame length + discard flag // _ _ _↓_ _ _ _________________ // | | | original buffer | // ¯↑¯ ¯ ¯ ¯ ¯ ¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯ @@ -48,10 +49,12 @@ namespace test { // 1. pos = 0 // 2. Extract id from buf[pos] and buf[pos + 1] // 3. Extract length from buf[pos + 2]..buf[pos + 5] -// 4. Copy |length| bytes starting from buf[pos + 6] to output buffer. -// 5. pos = pos + length + 6 -// 6. If pos < buf.length - go to the step 2. -// Also it will be checked, that all extracted ids are equals. +// 4. Extract discard flag from length highest bit. +// 5. If discard flag is False, copy |length| bytes starting from buf[pos + 6] +// to output buffer. +// 6. pos = pos + length + 6 +// 7. If pos < buf.length - go to the step 2. +// Also it will check, that all extracted ids are equals. // // Because EncodedImage doesn't take ownership of its buffer, injector will keep // ownership of the buffers that will be used for EncodedImages with injected @@ -67,17 +70,19 @@ namespace test { // preallocate buffers for 2 coding entities, so 512 buffers with initial size // 2KB. If in some point of time bigger buffer will be required, it will be also // extended. -class DefaultEncodedImageIdInjector : public EncodedImageIdInjector, - public EncodedImageIdExtractor { +class DefaultEncodedImageDataInjector : public EncodedImageDataInjector, + public EncodedImageDataExtractor { public: - DefaultEncodedImageIdInjector(); - ~DefaultEncodedImageIdInjector() override; + DefaultEncodedImageDataInjector(); + ~DefaultEncodedImageDataInjector() override; - EncodedImage InjectId(uint16_t id, - const EncodedImage& source, - int coding_entity_id) override; - EncodedImageWithId ExtractId(const EncodedImage& source, - int coding_entity_id) override; + // TODO(titovartem) add support for discard injection and update the doc. + EncodedImage InjectData(uint16_t id, + bool discard, + const EncodedImage& source, + int coding_entity_id) override; + EncodedImageExtractionResult ExtractData(const EncodedImage& source, + int coding_entity_id) override; private: void ExtendIfRequired(int coding_entity_id) RTC_LOCKS_EXCLUDED(lock_); @@ -99,4 +104,4 @@ class DefaultEncodedImageIdInjector : public EncodedImageIdInjector, } // namespace test } // namespace webrtc -#endif // TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_ENCODED_IMAGE_ID_INJECTOR_H_ +#endif // TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_ENCODED_IMAGE_DATA_INJECTOR_H_ diff --git a/test/pc/e2e/analyzer/video/default_encoded_image_data_injector_unittest.cc b/test/pc/e2e/analyzer/video/default_encoded_image_data_injector_unittest.cc new file mode 100644 index 0000000000..8dc341e33a --- /dev/null +++ b/test/pc/e2e/analyzer/video/default_encoded_image_data_injector_unittest.cc @@ -0,0 +1,191 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/default_encoded_image_data_injector.h" + +#include + +#include "api/video/encoded_image.h" +#include "rtc_base/buffer.h" +#include "test/gtest.h" + +namespace webrtc { +namespace test { +namespace { + +rtc::Buffer CreateBufferOfSizeNFilledWithValuesFromX(size_t n, uint8_t x) { + rtc::Buffer buffer(n); + for (size_t i = 0; i < n; ++i) { + buffer[i] = static_cast(x + i); + } + return buffer; +} + +} // namespace + +TEST(DefaultEncodedImageDataInjector, InjectExtractDiscardFalse) { + DefaultEncodedImageDataInjector injector; + + rtc::Buffer buffer = CreateBufferOfSizeNFilledWithValuesFromX(10, 1); + + EncodedImage source(buffer.data(), 10, 10); + source.SetTimestamp(123456789); + + EncodedImageExtractionResult out = + injector.ExtractData(injector.InjectData(512, false, source, 1), 2); + EXPECT_EQ(out.id, 512); + EXPECT_FALSE(out.discard); + EXPECT_EQ(out.image.size(), 10ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out.image.data()[i], i + 1); + } +} + +TEST(DefaultEncodedImageDataInjector, InjectExtractDiscardTrue) { + DefaultEncodedImageDataInjector injector; + + rtc::Buffer buffer = CreateBufferOfSizeNFilledWithValuesFromX(10, 1); + + EncodedImage source(buffer.data(), 10, 10); + source.SetTimestamp(123456789); + + EncodedImageExtractionResult out = + injector.ExtractData(injector.InjectData(512, true, source, 1), 2); + EXPECT_EQ(out.id, 512); + EXPECT_TRUE(out.discard); + EXPECT_EQ(out.image.size(), 0ul); +} + +TEST(DefaultEncodedImageDataInjector, Inject3Extract3) { + DefaultEncodedImageDataInjector injector; + + rtc::Buffer buffer1 = CreateBufferOfSizeNFilledWithValuesFromX(10, 1); + rtc::Buffer buffer2 = CreateBufferOfSizeNFilledWithValuesFromX(10, 11); + rtc::Buffer buffer3 = CreateBufferOfSizeNFilledWithValuesFromX(10, 21); + + // 1st frame + EncodedImage source1(buffer1.data(), 10, 10); + source1.SetTimestamp(123456710); + // 2nd frame 1st spatial layer + EncodedImage source2(buffer2.data(), 10, 10); + source2.SetTimestamp(123456720); + // 2nd frame 2nd spatial layer + EncodedImage source3(buffer3.data(), 10, 10); + source3.SetTimestamp(123456720); + + EncodedImage intermediate1 = injector.InjectData(510, false, source1, 1); + EncodedImage intermediate2 = injector.InjectData(520, true, source2, 1); + EncodedImage intermediate3 = injector.InjectData(520, false, source3, 1); + + // Extract ids in different order. + EncodedImageExtractionResult out3 = injector.ExtractData(intermediate3, 2); + EncodedImageExtractionResult out1 = injector.ExtractData(intermediate1, 2); + EncodedImageExtractionResult out2 = injector.ExtractData(intermediate2, 2); + + EXPECT_EQ(out1.id, 510); + EXPECT_FALSE(out1.discard); + EXPECT_EQ(out1.image.size(), 10ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out1.image.data()[i], i + 1); + } + EXPECT_EQ(out2.id, 520); + EXPECT_TRUE(out2.discard); + EXPECT_EQ(out2.image.size(), 0ul); + EXPECT_EQ(out3.id, 520); + EXPECT_FALSE(out3.discard); + EXPECT_EQ(out3.image.size(), 10ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out3.image.data()[i], i + 21); + } +} + +TEST(DefaultEncodedImageDataInjector, InjectExtractFromConcatenated) { + DefaultEncodedImageDataInjector injector; + + rtc::Buffer buffer1 = CreateBufferOfSizeNFilledWithValuesFromX(10, 1); + rtc::Buffer buffer2 = CreateBufferOfSizeNFilledWithValuesFromX(10, 11); + rtc::Buffer buffer3 = CreateBufferOfSizeNFilledWithValuesFromX(10, 21); + + EncodedImage source1(buffer1.data(), 10, 10); + source1.SetTimestamp(123456710); + EncodedImage source2(buffer2.data(), 10, 10); + source2.SetTimestamp(123456710); + EncodedImage source3(buffer3.data(), 10, 10); + source3.SetTimestamp(123456710); + + // Inject id into 3 images with same frame id. + EncodedImage intermediate1 = injector.InjectData(512, false, source1, 1); + EncodedImage intermediate2 = injector.InjectData(512, true, source2, 1); + EncodedImage intermediate3 = injector.InjectData(512, false, source3, 1); + + // Concatenate them into single encoded image, like it can be done in jitter + // buffer. + size_t concatenated_length = + intermediate1.size() + intermediate2.size() + intermediate3.size(); + rtc::Buffer concatenated_buffer; + concatenated_buffer.AppendData(intermediate1.data(), intermediate1.size()); + concatenated_buffer.AppendData(intermediate2.data(), intermediate2.size()); + concatenated_buffer.AppendData(intermediate3.data(), intermediate3.size()); + EncodedImage concatenated(concatenated_buffer.data(), concatenated_length, + concatenated_length); + + // Extract frame id from concatenated image + EncodedImageExtractionResult out = injector.ExtractData(concatenated, 2); + + EXPECT_EQ(out.id, 512); + EXPECT_FALSE(out.discard); + EXPECT_EQ(out.image.size(), 2 * 10ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out.image.data()[i], i + 1); + EXPECT_EQ(out.image.data()[i + 10], i + 21); + } +} + +TEST(DefaultEncodedImageDataInjector, + InjectExtractFromConcatenatedAllDiscarded) { + DefaultEncodedImageDataInjector injector; + + rtc::Buffer buffer1 = CreateBufferOfSizeNFilledWithValuesFromX(10, 1); + rtc::Buffer buffer2 = CreateBufferOfSizeNFilledWithValuesFromX(10, 11); + rtc::Buffer buffer3 = CreateBufferOfSizeNFilledWithValuesFromX(10, 21); + + EncodedImage source1(buffer1.data(), 10, 10); + source1.SetTimestamp(123456710); + EncodedImage source2(buffer2.data(), 10, 10); + source2.SetTimestamp(123456710); + EncodedImage source3(buffer3.data(), 10, 10); + source3.SetTimestamp(123456710); + + // Inject id into 3 images with same frame id. + EncodedImage intermediate1 = injector.InjectData(512, true, source1, 1); + EncodedImage intermediate2 = injector.InjectData(512, true, source2, 1); + EncodedImage intermediate3 = injector.InjectData(512, true, source3, 1); + + // Concatenate them into single encoded image, like it can be done in jitter + // buffer. + size_t concatenated_length = + intermediate1.size() + intermediate2.size() + intermediate3.size(); + rtc::Buffer concatenated_buffer; + concatenated_buffer.AppendData(intermediate1.data(), intermediate1.size()); + concatenated_buffer.AppendData(intermediate2.data(), intermediate2.size()); + concatenated_buffer.AppendData(intermediate3.data(), intermediate3.size()); + EncodedImage concatenated(concatenated_buffer.data(), concatenated_length, + concatenated_length); + + // Extract frame id from concatenated image + EncodedImageExtractionResult out = injector.ExtractData(concatenated, 2); + + EXPECT_EQ(out.id, 512); + EXPECT_TRUE(out.discard); + EXPECT_EQ(out.image.size(), 0ul); +} + +} // namespace test +} // namespace webrtc diff --git a/test/pc/e2e/analyzer/video/default_encoded_image_id_injector_unittest.cc b/test/pc/e2e/analyzer/video/default_encoded_image_id_injector_unittest.cc deleted file mode 100644 index e3e50e599e..0000000000 --- a/test/pc/e2e/analyzer/video/default_encoded_image_id_injector_unittest.cc +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "test/pc/e2e/analyzer/video/default_encoded_image_id_injector.h" - -#include - -#include "api/video/encoded_image.h" -#include "rtc_base/buffer.h" -#include "test/gtest.h" - -namespace webrtc { -namespace test { -namespace { - -rtc::Buffer CreateBufferOfSizeNFilledWithValuesFromX(size_t n, uint8_t x) { - rtc::Buffer buffer(n); - for (size_t i = 0; i < n; ++i) { - buffer[i] = static_cast(x + i); - } - return buffer; -} - -} // namespace - -TEST(DefaultEncodedImageIdInjector, InjectExtract) { - DefaultEncodedImageIdInjector injector; - - rtc::Buffer buffer = CreateBufferOfSizeNFilledWithValuesFromX(10, 1); - - EncodedImage source(buffer.data(), 10, 10); - source.SetTimestamp(123456789); - - EncodedImageWithId out = - injector.ExtractId(injector.InjectId(512, source, 1), 2); - ASSERT_EQ(out.id, 512); - ASSERT_EQ(out.image.size(), 10ul); - for (int i = 0; i < 10; ++i) { - ASSERT_EQ(out.image.data()[i], i + 1); - } -} - -TEST(DefaultEncodedImageIdInjector, Inject3Extract3) { - DefaultEncodedImageIdInjector injector; - - rtc::Buffer buffer1 = CreateBufferOfSizeNFilledWithValuesFromX(10, 1); - rtc::Buffer buffer2 = CreateBufferOfSizeNFilledWithValuesFromX(10, 11); - rtc::Buffer buffer3 = CreateBufferOfSizeNFilledWithValuesFromX(10, 21); - - // 1st frame - EncodedImage source1(buffer1.data(), 10, 10); - source1.SetTimestamp(123456710); - // 2nd frame 1st spatial layer - EncodedImage source2(buffer2.data(), 10, 10); - source2.SetTimestamp(123456720); - // 2nd frame 2nd spatial layer - EncodedImage source3(buffer3.data(), 10, 10); - source3.SetTimestamp(123456720); - - EncodedImage intermediate1 = injector.InjectId(510, source1, 1); - EncodedImage intermediate2 = injector.InjectId(520, source2, 1); - EncodedImage intermediate3 = injector.InjectId(520, source3, 1); - - // Extract ids in different order. - EncodedImageWithId out3 = injector.ExtractId(intermediate3, 2); - EncodedImageWithId out1 = injector.ExtractId(intermediate1, 2); - EncodedImageWithId out2 = injector.ExtractId(intermediate2, 2); - - ASSERT_EQ(out1.id, 510); - ASSERT_EQ(out1.image.size(), 10ul); - for (int i = 0; i < 10; ++i) { - ASSERT_EQ(out1.image.data()[i], i + 1); - } - ASSERT_EQ(out2.id, 520); - ASSERT_EQ(out2.image.size(), 10ul); - for (int i = 0; i < 10; ++i) { - ASSERT_EQ(out2.image.data()[i], i + 11); - } - ASSERT_EQ(out3.id, 520); - ASSERT_EQ(out3.image.size(), 10ul); - for (int i = 0; i < 10; ++i) { - ASSERT_EQ(out3.image.data()[i], i + 21); - } -} - -TEST(DefaultEncodedImageIdInjector, InjectExtractFromConcatenated) { - DefaultEncodedImageIdInjector injector; - - rtc::Buffer buffer1 = CreateBufferOfSizeNFilledWithValuesFromX(10, 1); - rtc::Buffer buffer2 = CreateBufferOfSizeNFilledWithValuesFromX(10, 11); - rtc::Buffer buffer3 = CreateBufferOfSizeNFilledWithValuesFromX(10, 21); - - EncodedImage source1(buffer1.data(), 10, 10); - source1.SetTimestamp(123456710); - EncodedImage source2(buffer2.data(), 10, 10); - source2.SetTimestamp(123456710); - EncodedImage source3(buffer3.data(), 10, 10); - source3.SetTimestamp(123456710); - - // Inject id into 3 images with same frame id. - EncodedImage intermediate1 = injector.InjectId(512, source1, 1); - EncodedImage intermediate2 = injector.InjectId(512, source2, 1); - EncodedImage intermediate3 = injector.InjectId(512, source3, 1); - - // Concatenate them into single encoded image, like it can be done in jitter - // buffer. - size_t concatenated_length = - intermediate1.size() + intermediate2.size() + intermediate3.size(); - rtc::Buffer concatenated_buffer; - concatenated_buffer.AppendData(intermediate1.data(), intermediate1.size()); - concatenated_buffer.AppendData(intermediate2.data(), intermediate2.size()); - concatenated_buffer.AppendData(intermediate3.data(), intermediate3.size()); - EncodedImage concatenated(concatenated_buffer.data(), concatenated_length, - concatenated_length); - - // Extract frame id from concatenated image - EncodedImageWithId out = injector.ExtractId(concatenated, 2); - - ASSERT_EQ(out.id, 512); - ASSERT_EQ(out.image.size(), 3 * 10ul); - for (int i = 0; i < 10; ++i) { - ASSERT_EQ(out.image.data()[i], i + 1); - ASSERT_EQ(out.image.data()[i + 10], i + 11); - ASSERT_EQ(out.image.data()[i + 20], i + 21); - } -} - -} // namespace test -} // namespace webrtc diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc index e4fcd209f5..2c91351bd7 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc @@ -299,6 +299,13 @@ void DefaultVideoQualityAnalyzer::Stop() { ReportResults(); } +std::string DefaultVideoQualityAnalyzer::GetStreamLabel(uint16_t frame_id) { + rtc::CritScope crit1(&lock_); + auto it = frame_stats_.find(frame_id); + RTC_DCHECK(it != frame_stats_.end()) << "Unknown frame_id=" << frame_id; + return it->second.stream_label; +} + std::set DefaultVideoQualityAnalyzer::GetKnownVideoStreams() const { rtc::CritScope crit2(&comparison_lock_); diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h index 570253debc..07c37ea62d 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h @@ -136,6 +136,7 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { void OnEncoderError(const VideoFrame& frame, int32_t error_code) override; void OnDecoderError(uint16_t frame_id, int32_t error_code) override; void Stop() override; + std::string GetStreamLabel(uint16_t frame_id) override; // Returns set of stream labels, that were met during test call. std::set GetKnownVideoStreams() const; diff --git a/test/pc/e2e/analyzer/video/encoded_image_data_injector.h b/test/pc/e2e/analyzer/video/encoded_image_data_injector.h new file mode 100644 index 0000000000..bf541ba826 --- /dev/null +++ b/test/pc/e2e/analyzer/video/encoded_image_data_injector.h @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_ENCODED_IMAGE_DATA_INJECTOR_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_ENCODED_IMAGE_DATA_INJECTOR_H_ + +#include +#include + +#include "api/video/encoded_image.h" + +namespace webrtc { +namespace test { + +// Injects frame id into EncodedImage on encoder side +class EncodedImageDataInjector { + public: + virtual ~EncodedImageDataInjector() = default; + + // Return encoded image with specified |id| and |discard| flag injected into + // its payload. |discard| flag mean does analyzing decoder should discard this + // encoded image because it belongs to unnecessary simulcast stream or spatial + // layer. |coding_entity_id| is unique id of decoder or encoder. + virtual EncodedImage InjectData(uint16_t id, + bool discard, + const EncodedImage& source, + int coding_entity_id) = 0; +}; + +struct EncodedImageExtractionResult { + uint16_t id; + EncodedImage image; + // Is true if encoded image should be discarded. It is used to filter out + // unnecessary spatial layers and simulcast streams. + bool discard; +}; + +// Extracts frame id from EncodedImage on decoder side. +class EncodedImageDataExtractor { + public: + virtual ~EncodedImageDataExtractor() = default; + + // Returns encoded image id, extracted from payload and also encoded image + // with its original payload. For concatenated spatial layers it should be the + // same id. |coding_entity_id| is unique id of decoder or encoder. + virtual EncodedImageExtractionResult ExtractData(const EncodedImage& source, + int coding_entity_id) = 0; +}; + +} // namespace test +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_ENCODED_IMAGE_DATA_INJECTOR_H_ diff --git a/test/pc/e2e/analyzer/video/encoded_image_id_injector.h b/test/pc/e2e/analyzer/video/encoded_image_id_injector.h deleted file mode 100644 index 7f53d4e876..0000000000 --- a/test/pc/e2e/analyzer/video/encoded_image_id_injector.h +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef TEST_PC_E2E_ANALYZER_VIDEO_ENCODED_IMAGE_ID_INJECTOR_H_ -#define TEST_PC_E2E_ANALYZER_VIDEO_ENCODED_IMAGE_ID_INJECTOR_H_ - -#include -#include - -#include "api/video/encoded_image.h" - -namespace webrtc { -namespace test { - -// Injects frame id into EncodedImage on encoder side -class EncodedImageIdInjector { - public: - virtual ~EncodedImageIdInjector() = default; - - // Return encoded image with specified |id| injected into its payload. - // |coding_entity_id| is unique id of decoder or encoder. - virtual EncodedImage InjectId(uint16_t id, - const EncodedImage& source, - int coding_entity_id) = 0; -}; - -struct EncodedImageWithId { - uint16_t id; - EncodedImage image; -}; - -// Extracts frame id from EncodedImage on decoder side. -class EncodedImageIdExtractor { - public: - virtual ~EncodedImageIdExtractor() = default; - - // Returns encoded image id, extracted from payload and also encoded image - // with its original payload. For concatenated spatial layers it should be the - // same id. |coding_entity_id| is unique id of decoder or encoder. - virtual EncodedImageWithId ExtractId(const EncodedImage& source, - int coding_entity_id) = 0; -}; - -} // namespace test -} // namespace webrtc - -#endif // TEST_PC_E2E_ANALYZER_VIDEO_ENCODED_IMAGE_ID_INJECTOR_H_ diff --git a/test/pc/e2e/analyzer/video/example_video_quality_analyzer.cc b/test/pc/e2e/analyzer/video/example_video_quality_analyzer.cc index 07b31752ad..00c0a5b718 100644 --- a/test/pc/e2e/analyzer/video/example_video_quality_analyzer.cc +++ b/test/pc/e2e/analyzer/video/example_video_quality_analyzer.cc @@ -28,12 +28,16 @@ uint16_t ExampleVideoQualityAnalyzer::OnFrameCaptured( auto it = frames_in_flight_.find(frame_id); if (it == frames_in_flight_.end()) { frames_in_flight_.insert(frame_id); + frames_to_stream_label_.insert({frame_id, stream_label}); } else { RTC_LOG(WARNING) << "Meet new frame with the same id: " << frame_id << ". Assumes old one as dropped"; // We needn't insert frame to frames_in_flight_, because it is already // there. ++frames_dropped_; + auto stream_it = frames_to_stream_label_.find(frame_id); + RTC_CHECK(stream_it != frames_to_stream_label_.end()); + stream_it->second = stream_label; } ++frames_captured_; return frame_id; @@ -95,6 +99,14 @@ void ExampleVideoQualityAnalyzer::Stop() { frames_dropped_ += frames_in_flight_.size(); } +std::string ExampleVideoQualityAnalyzer::GetStreamLabel(uint16_t frame_id) { + rtc::CritScope crit(&lock_); + auto it = frames_to_stream_label_.find(frame_id); + RTC_DCHECK(it != frames_to_stream_label_.end()) + << "Unknown frame_id=" << frame_id; + return it->second; +} + uint64_t ExampleVideoQualityAnalyzer::frames_captured() const { rtc::CritScope crit(&lock_); return frames_captured_; diff --git a/test/pc/e2e/analyzer/video/example_video_quality_analyzer.h b/test/pc/e2e/analyzer/video/example_video_quality_analyzer.h index 4070b4d63b..e8a2fa4ebb 100644 --- a/test/pc/e2e/analyzer/video/example_video_quality_analyzer.h +++ b/test/pc/e2e/analyzer/video/example_video_quality_analyzer.h @@ -12,6 +12,7 @@ #define TEST_PC_E2E_ANALYZER_VIDEO_EXAMPLE_VIDEO_QUALITY_ANALYZER_H_ #include +#include #include #include @@ -48,6 +49,7 @@ class ExampleVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { void OnEncoderError(const VideoFrame& frame, int32_t error_code) override; void OnDecoderError(uint16_t frame_id, int32_t error_code) override; void Stop() override; + std::string GetStreamLabel(uint16_t frame_id) override; uint64_t frames_captured() const; uint64_t frames_sent() const; @@ -66,6 +68,7 @@ class ExampleVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { // need to keep them to correctly determine dropped frames and also correctly // process frame id overlap. std::set frames_in_flight_ RTC_GUARDED_BY(lock_); + std::map frames_to_stream_label_ RTC_GUARDED_BY(lock_); uint16_t next_frame_id_ RTC_GUARDED_BY(lock_) = 0; uint64_t frames_captured_ RTC_GUARDED_BY(lock_) = 0; uint64_t frames_sent_ RTC_GUARDED_BY(lock_) = 0; diff --git a/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc b/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc index 6d48e4d12b..13e44bf27d 100644 --- a/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc +++ b/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc @@ -25,7 +25,7 @@ namespace test { QualityAnalyzingVideoDecoder::QualityAnalyzingVideoDecoder( int id, std::unique_ptr delegate, - EncodedImageIdExtractor* extractor, + EncodedImageDataExtractor* extractor, VideoQualityAnalyzerInterface* analyzer) : id_(id), implementation_name_("AnalyzingDecoder-" + @@ -53,7 +53,10 @@ int32_t QualityAnalyzingVideoDecoder::Decode( // owner of original buffer will be responsible for deleting it, or extractor // can create a new buffer. In such case extractor will be responsible for // deleting it. - EncodedImageWithId out = extractor_->ExtractId(input_image, id_); + EncodedImageExtractionResult out = extractor_->ExtractData(input_image, id_); + + // TODO(titovartem) add support for simulcast. + RTC_CHECK(!out.discard) << "Simulcast is not supported yet"; EncodedImage* origin_image; { @@ -195,7 +198,7 @@ void QualityAnalyzingVideoDecoder::OnFrameDecoded( QualityAnalyzingVideoDecoderFactory::QualityAnalyzingVideoDecoderFactory( std::unique_ptr delegate, IdGenerator* id_generator, - EncodedImageIdExtractor* extractor, + EncodedImageDataExtractor* extractor, VideoQualityAnalyzerInterface* analyzer) : delegate_(std::move(delegate)), id_generator_(id_generator), diff --git a/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h b/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h index 2fc69b426d..1f887f7620 100644 --- a/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h +++ b/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h @@ -22,7 +22,7 @@ #include "api/video_codecs/video_decoder.h" #include "api/video_codecs/video_decoder_factory.h" #include "rtc_base/critical_section.h" -#include "test/pc/e2e/analyzer/video/encoded_image_id_injector.h" +#include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h" #include "test/pc/e2e/analyzer/video/id_generator.h" #include "test/pc/e2e/api/video_quality_analyzer_interface.h" @@ -51,10 +51,10 @@ class QualityAnalyzingVideoDecoder : public VideoDecoder { public: // Creates analyzing decoder. |id| is unique coding entity id, that will // be used to distinguish all encoders and decoders inside - // EncodedImageIdInjector and EncodedImageIdExtracor. + // EncodedImageDataInjector and EncodedImageIdExtracor. QualityAnalyzingVideoDecoder(int id, std::unique_ptr delegate, - EncodedImageIdExtractor* extractor, + EncodedImageDataExtractor* extractor, VideoQualityAnalyzerInterface* analyzer); ~QualityAnalyzingVideoDecoder() override; @@ -101,7 +101,7 @@ class QualityAnalyzingVideoDecoder : public VideoDecoder { const int id_; const std::string implementation_name_; std::unique_ptr delegate_; - EncodedImageIdExtractor* const extractor_; + EncodedImageDataExtractor* const extractor_; VideoQualityAnalyzerInterface* const analyzer_; std::unique_ptr analyzing_callback_; @@ -112,7 +112,7 @@ class QualityAnalyzingVideoDecoder : public VideoDecoder { std::map timestamp_to_frame_id_ RTC_GUARDED_BY(lock_); // Stores currently being decoded images by frame id. Because - // EncodedImageIdExtractor can create new copy on EncodedImage we need to + // EncodedImageDataExtractor can create new copy on EncodedImage we need to // ensure, that this image won't be deleted during async decoding. To do it // all images are putted into this map and removed from here inside callback. std::map decoding_images_ RTC_GUARDED_BY(lock_); @@ -126,7 +126,7 @@ class QualityAnalyzingVideoDecoderFactory : public VideoDecoderFactory { QualityAnalyzingVideoDecoderFactory( std::unique_ptr delegate, IdGenerator* id_generator, - EncodedImageIdExtractor* extractor, + EncodedImageDataExtractor* extractor, VideoQualityAnalyzerInterface* analyzer); ~QualityAnalyzingVideoDecoderFactory() override; @@ -141,7 +141,7 @@ class QualityAnalyzingVideoDecoderFactory : public VideoDecoderFactory { private: std::unique_ptr delegate_; IdGenerator* const id_generator_; - EncodedImageIdExtractor* const extractor_; + EncodedImageDataExtractor* const extractor_; VideoQualityAnalyzerInterface* const analyzer_; }; diff --git a/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc b/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc index 55061cab86..95cf98aca8 100644 --- a/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc +++ b/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc @@ -13,7 +13,9 @@ #include #include "absl/memory/memory.h" +#include "api/video/video_codec_type.h" #include "modules/video_coding/include/video_error_codes.h" +#include "rtc_base/critical_section.h" #include "rtc_base/logging.h" namespace webrtc { @@ -27,10 +29,12 @@ constexpr size_t kMaxFrameInPipelineCount = 1000; QualityAnalyzingVideoEncoder::QualityAnalyzingVideoEncoder( int id, std::unique_ptr delegate, - EncodedImageIdInjector* injector, + std::map> stream_required_spatial_index, + EncodedImageDataInjector* injector, VideoQualityAnalyzerInterface* analyzer) : id_(id), delegate_(std::move(delegate)), + stream_required_spatial_index_(std::move(stream_required_spatial_index)), injector_(injector), analyzer_(analyzer) {} QualityAnalyzingVideoEncoder::~QualityAnalyzingVideoEncoder() = default; @@ -39,6 +43,29 @@ int32_t QualityAnalyzingVideoEncoder::InitEncode( const VideoCodec* codec_settings, int32_t number_of_cores, size_t max_payload_size) { + rtc::CritScope crit(&lock_); + mode_ = SimulcastMode::kNormal; + if (codec_settings->codecType == kVideoCodecVP9) { + if (codec_settings->VP9().numberOfSpatialLayers > 1) { + switch (codec_settings->VP9().interLayerPred) { + case InterLayerPredMode::kOn: + mode_ = SimulcastMode::kSVC; + break; + case InterLayerPredMode::kOnKeyPic: + mode_ = SimulcastMode::kKSVC; + break; + case InterLayerPredMode::kOff: + mode_ = SimulcastMode::kSimulcast; + break; + default: + RTC_NOTREACHED() << "Unknown codec_settings->VP9().interLayerPred"; + break; + } + } + } + if (codec_settings->numberOfSimulcastStreams > 1) { + mode_ = SimulcastMode::kSimulcast; + } return delegate_->InitEncode(codec_settings, number_of_cores, max_payload_size); } @@ -109,7 +136,7 @@ VideoEncoder::EncoderInfo QualityAnalyzingVideoEncoder::GetEncoderInfo() const { } // It is assumed, that encoded callback will be always invoked with encoded -// images that correspond to the frames if the same sequence, that frames +// images that correspond to the frames in the same sequence, that frames // arrived. In other words, assume we have frames F1, F2 and F3 and they have // corresponding encoded images I1, I2 and I3. In such case if we will call // encode first with F1, then with F2 and then with F3, then encoder callback @@ -126,6 +153,7 @@ EncodedImageCallback::Result QualityAnalyzingVideoEncoder::OnEncodedImage( const CodecSpecificInfo* codec_specific_info, const RTPFragmentationHeader* fragmentation) { uint16_t frame_id; + bool discard = false; { rtc::CritScope crit(&lock_); std::pair timestamp_frame_id; @@ -155,15 +183,22 @@ EncodedImageCallback::Result QualityAnalyzingVideoEncoder::OnEncodedImage( EncodedImageCallback::Result::Error::OK); } frame_id = timestamp_frame_id.second; + + discard = ShouldDiscard(frame_id, encoded_image); } - analyzer_->OnFrameEncoded(frame_id, encoded_image); + if (!discard) { + // Analyzer should see only encoded images, that weren't discarded. + analyzer_->OnFrameEncoded(frame_id, encoded_image); + } - // Image id injector injects frame id into provided EncodedImage and returns - // the image with a) modified original buffer (in such case the current owner - // of the buffer will be responsible for deleting it) or b) a new buffer (in - // such case injector will be responsible for deleting it). - const EncodedImage& image = injector_->InjectId(frame_id, encoded_image, id_); + // Image data injector injects frame id and discard flag into provided + // EncodedImage and returns the image with a) modified original buffer (in + // such case the current owner of the buffer will be responsible for deleting + // it) or b) a new buffer (in such case injector will be responsible for + // deleting it). + const EncodedImage& image = + injector_->InjectData(frame_id, discard, encoded_image, id_); { rtc::CritScope crit(&lock_); RTC_DCHECK(delegate_callback_); @@ -180,12 +215,57 @@ void QualityAnalyzingVideoEncoder::OnDroppedFrame( delegate_callback_->OnDroppedFrame(reason); } +bool QualityAnalyzingVideoEncoder::ShouldDiscard( + uint16_t frame_id, + const EncodedImage& encoded_image) { + std::string stream_label = analyzer_->GetStreamLabel(frame_id); + absl::optional required_spatial_index = + stream_required_spatial_index_[stream_label]; + if (required_spatial_index) { + RTC_CHECK(encoded_image.SpatialIndex()) + << "Specific spatial layer/simulcast stream requested for track, but " + "now spatial layers/simulcast streams produced by encoder. " + "stream_label=" + << stream_label + << "; required_spatial_index=" << *required_spatial_index; + RTC_CHECK(mode_ != SimulcastMode::kNormal) + << "Analyzing encoder is in kNormal " + "mode, but spatial layer/simulcast " + "stream met."; + if (mode_ == SimulcastMode::kSimulcast) { + // In simulcast mode only encoded images with required spatial index are + // interested, so all others have to be discarded. + return *encoded_image.SpatialIndex() != *required_spatial_index; + } else if (mode_ == SimulcastMode::kSVC) { + // In SVC mode encoded images with spatial indexes that are equal or + // less than required one are interesting, so all above have to be + // discarded. + return *encoded_image.SpatialIndex() > *required_spatial_index; + } else if (mode_ == SimulcastMode::kKSVC) { + // In KSVC mode for key frame encoded images with spatial indexes that + // are equal or less than required one are interesting, so all above + // have to be discarded. For other frames only required spatial index + // is interesting, so all others have to be discarded. + if (encoded_image._frameType == FrameType::kVideoFrameKey) { + return *encoded_image.SpatialIndex() > *required_spatial_index; + } else { + return *encoded_image.SpatialIndex() != *required_spatial_index; + } + } else { + RTC_NOTREACHED() << "Unsupported encoder mode"; + } + } + return false; +} + QualityAnalyzingVideoEncoderFactory::QualityAnalyzingVideoEncoderFactory( std::unique_ptr delegate, + std::map> stream_required_spatial_index, IdGenerator* id_generator, - EncodedImageIdInjector* injector, + EncodedImageDataInjector* injector, VideoQualityAnalyzerInterface* analyzer) : delegate_(std::move(delegate)), + stream_required_spatial_index_(std::move(stream_required_spatial_index)), id_generator_(id_generator), injector_(injector), analyzer_(analyzer) {} @@ -208,7 +288,7 @@ QualityAnalyzingVideoEncoderFactory::CreateVideoEncoder( const SdpVideoFormat& format) { return absl::make_unique( id_generator_->GetNextId(), delegate_->CreateVideoEncoder(format), - injector_, analyzer_); + stream_required_spatial_index_, injector_, analyzer_); } } // namespace test diff --git a/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h b/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h index 56b3334dcf..b59c81cfe4 100644 --- a/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h +++ b/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h @@ -22,7 +22,7 @@ #include "api/video_codecs/video_encoder.h" #include "api/video_codecs/video_encoder_factory.h" #include "rtc_base/critical_section.h" -#include "test/pc/e2e/analyzer/video/encoded_image_id_injector.h" +#include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h" #include "test/pc/e2e/analyzer/video/id_generator.h" #include "test/pc/e2e/api/video_quality_analyzer_interface.h" @@ -41,7 +41,7 @@ namespace test { // // When origin encoder encodes the image it will call quality encoder's special // callback, where video analyzer will be called again and then frame id will be -// injected into EncodedImage with passed EncodedImageIdInjector. Then new +// injected into EncodedImage with passed EncodedImageDataInjector. Then new // EncodedImage will be passed to origin callback, provided by user. // // Quality encoder registers its own callback in origin encoder at the same @@ -51,11 +51,13 @@ class QualityAnalyzingVideoEncoder : public VideoEncoder, public: // Creates analyzing encoder. |id| is unique coding entity id, that will // be used to distinguish all encoders and decoders inside - // EncodedImageIdInjector and EncodedImageIdExtracor. - QualityAnalyzingVideoEncoder(int id, - std::unique_ptr delegate, - EncodedImageIdInjector* injector, - VideoQualityAnalyzerInterface* analyzer); + // EncodedImageDataInjector and EncodedImageIdExtracor. + QualityAnalyzingVideoEncoder( + int id, + std::unique_ptr delegate, + std::map> stream_required_spatial_index, + EncodedImageDataInjector* injector, + VideoQualityAnalyzerInterface* analyzer); ~QualityAnalyzingVideoEncoder() override; // Methods of VideoEncoder interface. @@ -81,9 +83,61 @@ class QualityAnalyzingVideoEncoder : public VideoEncoder, void OnDroppedFrame(DropReason reason) override; private: + enum SimulcastMode { + // In this mode encoder assumes not more than 1 encoded image per video + // frame + kNormal, + + // Next modes are to test video conference behavior. For conference sender + // will send multiple spatial layers/simulcast streams for single video + // track and there is some Selective Forwarding Unit (SFU), that forwards + // only best one, that will pass through downlink to the receiver. + // + // Here this behavior will be partly emulated. Sender will send all spatial + // layers/simulcast streams and then some of them will be filtered out on + // the receiver side. During test setup user can specify which spatial + // layer/simulcast stream is required, what will simulated which spatial + // layer/simulcast stream will be chosen by SFU in the real world. Then + // sender will mark encoded images for all spatial layers above required or + // all simulcast streams except required as to be discarded and on receiver + // side they will be discarded in quality analyzing decoder and won't be + // passed into delegate decoder. + // + // If the sender for some reasons won't send specified spatial layer, then + // receiver still will fall back on lower spatial layers. But for simulcast + // streams if required one won't be sent, receiver will assume all frames + // in that period as dropped and will experience video freeze. + // + // Test based on this simulation will be used to evaluate video quality + // of concrete spatial layers/simulcast streams and also check distribution + // of bandwidth between spatial layers/simulcast streams by BWE. + + // In this mode encoder assumes that for each frame simulcast encoded + // images will be produced. So all simulcast streams except required will + // be marked as to be discarded in decoder and won't reach video quality + // analyzer. + kSimulcast, + // In this mode encoder assumes that for each frame encoded images for + // different spatial layers will be produced. So all spatial layers above + // required will be marked to be discarded in decoder and won't reach + // video quality analyzer. + kSVC, + // In this mode encoder assumes that for each frame encoded images for + // different spatial layers will be produced. Compared to kSVC mode + // spatial layers that are above required will be marked to be discarded + // only for key frames and for regular frames all except required spatial + // layer will be marked as to be discarded in decoder and won't reach video + // quality analyzer. + kKSVC + }; + + bool ShouldDiscard(uint16_t frame_id, const EncodedImage& encoded_image) + RTC_EXCLUSIVE_LOCKS_REQUIRED(lock_); + const int id_; std::unique_ptr delegate_; - EncodedImageIdInjector* const injector_; + std::map> stream_required_spatial_index_; + EncodedImageDataInjector* const injector_; VideoQualityAnalyzerInterface* const analyzer_; // VideoEncoder interface assumes async delivery of encoded images. @@ -91,6 +145,7 @@ class QualityAnalyzingVideoEncoder : public VideoEncoder, // from received VideoFrame to resulted EncodedImage. rtc::CriticalSection lock_; + SimulcastMode mode_ RTC_GUARDED_BY(lock_); EncodedImageCallback* delegate_callback_ RTC_GUARDED_BY(lock_); std::list> timestamp_to_frame_id_list_ RTC_GUARDED_BY(lock_); @@ -103,8 +158,9 @@ class QualityAnalyzingVideoEncoderFactory : public VideoEncoderFactory { public: QualityAnalyzingVideoEncoderFactory( std::unique_ptr delegate, + std::map> stream_required_spatial_index, IdGenerator* id_generator, - EncodedImageIdInjector* injector, + EncodedImageDataInjector* injector, VideoQualityAnalyzerInterface* analyzer); ~QualityAnalyzingVideoEncoderFactory() override; @@ -117,8 +173,9 @@ class QualityAnalyzingVideoEncoderFactory : public VideoEncoderFactory { private: std::unique_ptr delegate_; + std::map> stream_required_spatial_index_; IdGenerator* const id_generator_; - EncodedImageIdInjector* const injector_; + EncodedImageDataInjector* const injector_; VideoQualityAnalyzerInterface* const analyzer_; }; diff --git a/test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector.cc b/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.cc similarity index 63% rename from test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector.cc rename to test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.cc index 105ee8eacd..89be698508 100644 --- a/test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector.cc +++ b/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.cc @@ -8,7 +8,7 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector.h" +#include "test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h" #include @@ -26,19 +26,21 @@ constexpr size_t kUsedBufferSize = 3; } // namespace -SingleProcessEncodedImageIdInjector::SingleProcessEncodedImageIdInjector() = - default; -SingleProcessEncodedImageIdInjector::~SingleProcessEncodedImageIdInjector() = +SingleProcessEncodedImageDataInjector::SingleProcessEncodedImageDataInjector() = default; +SingleProcessEncodedImageDataInjector:: + ~SingleProcessEncodedImageDataInjector() = default; -EncodedImage SingleProcessEncodedImageIdInjector::InjectId( +EncodedImage SingleProcessEncodedImageDataInjector::InjectData( uint16_t id, + bool discard, const EncodedImage& source, int coding_entity_id) { RTC_CHECK(source.size() >= kUsedBufferSize); ExtractionInfo info; info.length = source.size(); + info.discard = discard; memcpy(info.origin_data, source.data(), kUsedBufferSize); { rtc::CritScope crit(&lock_); @@ -55,18 +57,24 @@ EncodedImage SingleProcessEncodedImageIdInjector::InjectId( return out; } -EncodedImageWithId SingleProcessEncodedImageIdInjector::ExtractId( +EncodedImageExtractionResult SingleProcessEncodedImageDataInjector::ExtractData( const EncodedImage& source, int coding_entity_id) { EncodedImage out = source; + // Both |source| and |out| image will share the same buffer for payload or + // out will have a copy for it, so we can operate on the |out| buffer only. + uint8_t* buffer = out.data(); + size_t size = out.size(); + size_t pos = 0; absl::optional id = absl::nullopt; - while (pos < source.size()) { + bool discard = true; + while (pos < size) { // Extract frame id from first 2 bytes of the payload. - uint16_t next_id = source.data()[pos] + (source.data()[pos + 1] << 8); + uint16_t next_id = buffer[pos] + (buffer[pos + 1] << 8); // Extract frame sub id from second 2 byte of the payload. - uint16_t sub_id = source.data()[pos + 2]; + uint16_t sub_id = buffer[pos + 2]; RTC_CHECK(!id || id.value() == next_id) << "Different frames encoded into single encoded image: " << id.value() @@ -87,17 +95,28 @@ EncodedImageWithId SingleProcessEncodedImageIdInjector::ExtractId( ext_vector_it->second.infos.erase(info_it); } - memcpy(&out.data()[pos], info.origin_data, kUsedBufferSize); - pos += info.length; + if (info.discard) { + // If this encoded image is marked to be discarded - erase it's payload + // from the buffer. + memmove(&buffer[pos], &buffer[pos + info.length], + size - pos - info.length); + size -= info.length; + } else { + memmove(&buffer[pos], info.origin_data, kUsedBufferSize); + pos += info.length; + } + // We need to discard encoded image only if all concatenated encoded images + // have to be discarded. + discard = discard & info.discard; } out.set_size(pos); - return EncodedImageWithId{id.value(), out}; + return EncodedImageExtractionResult{id.value(), out, discard}; } -SingleProcessEncodedImageIdInjector::ExtractionInfoVector:: +SingleProcessEncodedImageDataInjector::ExtractionInfoVector:: ExtractionInfoVector() = default; -SingleProcessEncodedImageIdInjector::ExtractionInfoVector:: +SingleProcessEncodedImageDataInjector::ExtractionInfoVector:: ~ExtractionInfoVector() = default; } // namespace test diff --git a/test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector.h b/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h similarity index 59% rename from test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector.h rename to test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h index 94e1c01cfd..c5d47c7d4b 100644 --- a/test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector.h +++ b/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h @@ -8,8 +8,8 @@ * be found in the AUTHORS file in the root of the source tree. */ -#ifndef TEST_PC_E2E_ANALYZER_VIDEO_SINGLE_PROCESS_ENCODED_IMAGE_ID_INJECTOR_H_ -#define TEST_PC_E2E_ANALYZER_VIDEO_SINGLE_PROCESS_ENCODED_IMAGE_ID_INJECTOR_H_ +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_SINGLE_PROCESS_ENCODED_IMAGE_DATA_INJECTOR_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_SINGLE_PROCESS_ENCODED_IMAGE_DATA_INJECTOR_H_ #include #include @@ -19,36 +19,39 @@ #include "api/video/encoded_image.h" #include "rtc_base/critical_section.h" -#include "test/pc/e2e/analyzer/video/encoded_image_id_injector.h" +#include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h" namespace webrtc { namespace test { // Based on assumption that all call participants are in the same OS process -// and uses same QualityAnalyzingVideoContext to obtain EncodedImageIdInjector. +// and uses same QualityAnalyzingVideoContext to obtain +// EncodedImageDataInjector. // -// To inject frame id into EncodedImage injector uses first 2 bytes of -// EncodedImage payload. Then it uses 3rd byte for frame sub id, that is -// required to distinguish different spatial layers. The origin data from these -// 3 bytes will be stored inside injector's internal storage and then will be -// restored during extraction phase. +// To inject frame id and discard flag into EncodedImage injector uses first 2 +// bytes of EncodedImage payload. Then it uses 3rd byte for frame sub id, that +// is required to distinguish different spatial layers. The origin data from +// these 3 bytes will be stored inside injector's internal storage and then will +// be restored during extraction phase. // // This injector won't add any extra overhead into EncodedImage payload and // support frames with any size of payload. Also assumes that every EncodedImage // payload size is greater or equals to 3 bytes -class SingleProcessEncodedImageIdInjector : public EncodedImageIdInjector, - public EncodedImageIdExtractor { +class SingleProcessEncodedImageDataInjector : public EncodedImageDataInjector, + public EncodedImageDataExtractor { public: - SingleProcessEncodedImageIdInjector(); - ~SingleProcessEncodedImageIdInjector() override; + SingleProcessEncodedImageDataInjector(); + ~SingleProcessEncodedImageDataInjector() override; - // Id will be injected into EncodedImage buffer directly. This buffer won't - // be fully copied, so |source| image buffer will be also changed. - EncodedImage InjectId(uint16_t id, - const EncodedImage& source, - int coding_entity_id) override; - EncodedImageWithId ExtractId(const EncodedImage& source, - int coding_entity_id) override; + // Id and discard flag will be injected into EncodedImage buffer directly. + // This buffer won't be fully copied, so |source| image buffer will be also + // changed. + EncodedImage InjectData(uint16_t id, + bool discard, + const EncodedImage& source, + int coding_entity_id) override; + EncodedImageExtractionResult ExtractData(const EncodedImage& source, + int coding_entity_id) override; private: // Contains data required to extract frame id from EncodedImage and restore @@ -58,6 +61,9 @@ class SingleProcessEncodedImageIdInjector : public EncodedImageIdInjector, uint8_t sub_id; // Length of the origin buffer encoded image. size_t length; + // Flag to show is this encoded images should be discarded by analyzing + // decoder because of not required spatial layer/simulcast stream. + bool discard; // Data from first 3 bytes of origin encoded image's payload. uint8_t origin_data[3]; }; @@ -84,4 +90,4 @@ class SingleProcessEncodedImageIdInjector : public EncodedImageIdInjector, } // namespace test } // namespace webrtc -#endif // TEST_PC_E2E_ANALYZER_VIDEO_SINGLE_PROCESS_ENCODED_IMAGE_ID_INJECTOR_H_ +#endif // TEST_PC_E2E_ANALYZER_VIDEO_SINGLE_PROCESS_ENCODED_IMAGE_DATA_INJECTOR_H_ diff --git a/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector_unittest.cc b/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector_unittest.cc new file mode 100644 index 0000000000..636877df40 --- /dev/null +++ b/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector_unittest.cc @@ -0,0 +1,198 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h" + +#include + +#include "api/video/encoded_image.h" +#include "rtc_base/buffer.h" +#include "test/gtest.h" + +namespace webrtc { +namespace test { +namespace { + +rtc::Buffer CreateBufferOfSizeNFilledWithValuesFromX(size_t n, uint8_t x) { + rtc::Buffer buffer(n); + for (size_t i = 0; i < n; ++i) { + buffer[i] = static_cast(x + i); + } + return buffer; +} + +} // namespace + +TEST(SingleProcessEncodedImageDataInjector, InjectExtractDiscardFalse) { + SingleProcessEncodedImageDataInjector injector; + + rtc::Buffer buffer = CreateBufferOfSizeNFilledWithValuesFromX(10, 1); + + EncodedImage source(buffer.data(), 10, 10); + source.SetTimestamp(123456789); + + EncodedImageExtractionResult out = + injector.ExtractData(injector.InjectData(512, false, source, 1), 2); + EXPECT_EQ(out.id, 512); + EXPECT_FALSE(out.discard); + EXPECT_EQ(out.image.size(), 10ul); + EXPECT_EQ(out.image.capacity(), 10ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out.image.data()[i], i + 1); + } +} + +TEST(SingleProcessEncodedImageDataInjector, InjectExtractDiscardTrue) { + SingleProcessEncodedImageDataInjector injector; + + rtc::Buffer buffer = CreateBufferOfSizeNFilledWithValuesFromX(10, 1); + + EncodedImage source(buffer.data(), 10, 10); + source.SetTimestamp(123456789); + + EncodedImageExtractionResult out = + injector.ExtractData(injector.InjectData(512, true, source, 1), 2); + EXPECT_EQ(out.id, 512); + EXPECT_TRUE(out.discard); + EXPECT_EQ(out.image.size(), 0ul); + EXPECT_EQ(out.image.capacity(), 10ul); +} + +TEST(SingleProcessEncodedImageDataInjector, Inject3Extract3) { + SingleProcessEncodedImageDataInjector injector; + + rtc::Buffer buffer1 = CreateBufferOfSizeNFilledWithValuesFromX(10, 1); + rtc::Buffer buffer2 = CreateBufferOfSizeNFilledWithValuesFromX(10, 11); + rtc::Buffer buffer3 = CreateBufferOfSizeNFilledWithValuesFromX(10, 21); + + // 1st frame + EncodedImage source1(buffer1.data(), 10, 10); + source1.SetTimestamp(123456710); + // 2nd frame 1st spatial layer + EncodedImage source2(buffer2.data(), 10, 10); + source2.SetTimestamp(123456720); + // 2nd frame 2nd spatial layer + EncodedImage source3(buffer3.data(), 10, 10); + source3.SetTimestamp(123456720); + + EncodedImage intermediate1 = injector.InjectData(510, false, source1, 1); + EncodedImage intermediate2 = injector.InjectData(520, true, source2, 1); + EncodedImage intermediate3 = injector.InjectData(520, false, source3, 1); + + // Extract ids in different order. + EncodedImageExtractionResult out3 = injector.ExtractData(intermediate3, 2); + EncodedImageExtractionResult out1 = injector.ExtractData(intermediate1, 2); + EncodedImageExtractionResult out2 = injector.ExtractData(intermediate2, 2); + + EXPECT_EQ(out1.id, 510); + EXPECT_FALSE(out1.discard); + EXPECT_EQ(out1.image.size(), 10ul); + EXPECT_EQ(out1.image.capacity(), 10ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out1.image.data()[i], i + 1); + } + EXPECT_EQ(out2.id, 520); + EXPECT_TRUE(out2.discard); + EXPECT_EQ(out2.image.size(), 0ul); + EXPECT_EQ(out2.image.capacity(), 10ul); + EXPECT_EQ(out3.id, 520); + EXPECT_FALSE(out3.discard); + EXPECT_EQ(out3.image.size(), 10ul); + EXPECT_EQ(out3.image.capacity(), 10ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out3.image.data()[i], i + 21); + } +} + +TEST(SingleProcessEncodedImageDataInjector, InjectExtractFromConcatenated) { + SingleProcessEncodedImageDataInjector injector; + + rtc::Buffer buffer1 = CreateBufferOfSizeNFilledWithValuesFromX(10, 1); + rtc::Buffer buffer2 = CreateBufferOfSizeNFilledWithValuesFromX(10, 11); + rtc::Buffer buffer3 = CreateBufferOfSizeNFilledWithValuesFromX(10, 21); + + EncodedImage source1(buffer1.data(), 10, 10); + source1.SetTimestamp(123456710); + EncodedImage source2(buffer2.data(), 10, 10); + source2.SetTimestamp(123456710); + EncodedImage source3(buffer3.data(), 10, 10); + source3.SetTimestamp(123456710); + + // Inject id into 3 images with same frame id. + EncodedImage intermediate1 = injector.InjectData(512, false, source1, 1); + EncodedImage intermediate2 = injector.InjectData(512, true, source2, 1); + EncodedImage intermediate3 = injector.InjectData(512, false, source3, 1); + + // Concatenate them into single encoded image, like it can be done in jitter + // buffer. + size_t concatenated_length = + intermediate1.size() + intermediate2.size() + intermediate3.size(); + rtc::Buffer concatenated_buffer; + concatenated_buffer.AppendData(intermediate1.data(), intermediate1.size()); + concatenated_buffer.AppendData(intermediate2.data(), intermediate2.size()); + concatenated_buffer.AppendData(intermediate3.data(), intermediate3.size()); + EncodedImage concatenated(concatenated_buffer.data(), concatenated_length, + concatenated_length); + + // Extract frame id from concatenated image + EncodedImageExtractionResult out = injector.ExtractData(concatenated, 2); + + EXPECT_EQ(out.id, 512); + EXPECT_FALSE(out.discard); + EXPECT_EQ(out.image.size(), 2 * 10ul); + EXPECT_EQ(out.image.capacity(), 3 * 10ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out.image.data()[i], i + 1); + EXPECT_EQ(out.image.data()[i + 10], i + 21); + } +} + +TEST(SingleProcessEncodedImageDataInjector, + InjectExtractFromConcatenatedAllDiscarded) { + SingleProcessEncodedImageDataInjector injector; + + rtc::Buffer buffer1 = CreateBufferOfSizeNFilledWithValuesFromX(10, 1); + rtc::Buffer buffer2 = CreateBufferOfSizeNFilledWithValuesFromX(10, 11); + rtc::Buffer buffer3 = CreateBufferOfSizeNFilledWithValuesFromX(10, 21); + + EncodedImage source1(buffer1.data(), 10, 10); + source1.SetTimestamp(123456710); + EncodedImage source2(buffer2.data(), 10, 10); + source2.SetTimestamp(123456710); + EncodedImage source3(buffer3.data(), 10, 10); + source3.SetTimestamp(123456710); + + // Inject id into 3 images with same frame id. + EncodedImage intermediate1 = injector.InjectData(512, true, source1, 1); + EncodedImage intermediate2 = injector.InjectData(512, true, source2, 1); + EncodedImage intermediate3 = injector.InjectData(512, true, source3, 1); + + // Concatenate them into single encoded image, like it can be done in jitter + // buffer. + size_t concatenated_length = + intermediate1.size() + intermediate2.size() + intermediate3.size(); + rtc::Buffer concatenated_buffer; + concatenated_buffer.AppendData(intermediate1.data(), intermediate1.size()); + concatenated_buffer.AppendData(intermediate2.data(), intermediate2.size()); + concatenated_buffer.AppendData(intermediate3.data(), intermediate3.size()); + EncodedImage concatenated(concatenated_buffer.data(), concatenated_length, + concatenated_length); + + // Extract frame id from concatenated image + EncodedImageExtractionResult out = injector.ExtractData(concatenated, 2); + + EXPECT_EQ(out.id, 512); + EXPECT_TRUE(out.discard); + EXPECT_EQ(out.image.size(), 0ul); + EXPECT_EQ(out.image.capacity(), 3 * 10ul); +} + +} // namespace test +} // namespace webrtc diff --git a/test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector_unittest.cc b/test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector_unittest.cc deleted file mode 100644 index a554c025db..0000000000 --- a/test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector_unittest.cc +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector.h" - -#include - -#include "api/video/encoded_image.h" -#include "rtc_base/buffer.h" -#include "test/gtest.h" - -namespace webrtc { -namespace test { -namespace { - -rtc::Buffer CreateBufferOfSizeNFilledWithValuesFromX(size_t n, uint8_t x) { - rtc::Buffer buffer(n); - for (size_t i = 0; i < n; ++i) { - buffer[i] = static_cast(x + i); - } - return buffer; -} - -} // namespace - -TEST(SingleProcessEncodedImageIdInjector, InjectExtract) { - SingleProcessEncodedImageIdInjector injector; - - rtc::Buffer buffer = CreateBufferOfSizeNFilledWithValuesFromX(10, 1); - - EncodedImage source(buffer.data(), 10, 10); - source.SetTimestamp(123456789); - - EncodedImageWithId out = - injector.ExtractId(injector.InjectId(512, source, 1), 2); - ASSERT_EQ(out.id, 512); - ASSERT_EQ(out.image.size(), 10ul); - ASSERT_EQ(out.image.capacity(), 10ul); - for (int i = 0; i < 10; ++i) { - ASSERT_EQ(out.image.data()[i], i + 1); - } -} - -TEST(SingleProcessEncodedImageIdInjector, Inject3Extract3) { - SingleProcessEncodedImageIdInjector injector; - - rtc::Buffer buffer1 = CreateBufferOfSizeNFilledWithValuesFromX(10, 1); - rtc::Buffer buffer2 = CreateBufferOfSizeNFilledWithValuesFromX(10, 11); - rtc::Buffer buffer3 = CreateBufferOfSizeNFilledWithValuesFromX(10, 21); - - // 1st frame - EncodedImage source1(buffer1.data(), 10, 10); - source1.SetTimestamp(123456710); - // 2nd frame 1st spatial layer - EncodedImage source2(buffer2.data(), 10, 10); - source2.SetTimestamp(123456720); - // 2nd frame 2nd spatial layer - EncodedImage source3(buffer3.data(), 10, 10); - source3.SetTimestamp(123456720); - - EncodedImage intermediate1 = injector.InjectId(510, source1, 1); - EncodedImage intermediate2 = injector.InjectId(520, source2, 1); - EncodedImage intermediate3 = injector.InjectId(520, source3, 1); - - // Extract ids in different order. - EncodedImageWithId out3 = injector.ExtractId(intermediate3, 2); - EncodedImageWithId out1 = injector.ExtractId(intermediate1, 2); - EncodedImageWithId out2 = injector.ExtractId(intermediate2, 2); - - ASSERT_EQ(out1.id, 510); - ASSERT_EQ(out1.image.size(), 10ul); - ASSERT_EQ(out1.image.capacity(), 10ul); - for (int i = 0; i < 10; ++i) { - ASSERT_EQ(out1.image.data()[i], i + 1); - } - ASSERT_EQ(out2.id, 520); - ASSERT_EQ(out2.image.size(), 10ul); - ASSERT_EQ(out2.image.capacity(), 10ul); - for (int i = 0; i < 10; ++i) { - ASSERT_EQ(out2.image.data()[i], i + 11); - } - ASSERT_EQ(out3.id, 520); - ASSERT_EQ(out3.image.size(), 10ul); - ASSERT_EQ(out3.image.capacity(), 10ul); - for (int i = 0; i < 10; ++i) { - ASSERT_EQ(out3.image.data()[i], i + 21); - } -} - -TEST(SingleProcessEncodedImageIdInjector, InjectExtractFromConcatenated) { - SingleProcessEncodedImageIdInjector injector; - - rtc::Buffer buffer1 = CreateBufferOfSizeNFilledWithValuesFromX(10, 1); - rtc::Buffer buffer2 = CreateBufferOfSizeNFilledWithValuesFromX(10, 11); - rtc::Buffer buffer3 = CreateBufferOfSizeNFilledWithValuesFromX(10, 21); - - EncodedImage source1(buffer1.data(), 10, 10); - source1.SetTimestamp(123456710); - EncodedImage source2(buffer2.data(), 10, 10); - source2.SetTimestamp(123456710); - EncodedImage source3(buffer3.data(), 10, 10); - source3.SetTimestamp(123456710); - - // Inject id into 3 images with same frame id. - EncodedImage intermediate1 = injector.InjectId(512, source1, 1); - EncodedImage intermediate2 = injector.InjectId(512, source2, 1); - EncodedImage intermediate3 = injector.InjectId(512, source3, 1); - - // Concatenate them into single encoded image, like it can be done in jitter - // buffer. - size_t concatenated_length = - intermediate1.size() + intermediate2.size() + intermediate3.size(); - rtc::Buffer concatenated_buffer; - concatenated_buffer.AppendData(intermediate1.data(), intermediate1.size()); - concatenated_buffer.AppendData(intermediate2.data(), intermediate2.size()); - concatenated_buffer.AppendData(intermediate3.data(), intermediate3.size()); - EncodedImage concatenated(concatenated_buffer.data(), concatenated_length, - concatenated_length); - - // Extract frame id from concatenated image - EncodedImageWithId out = injector.ExtractId(concatenated, 2); - - ASSERT_EQ(out.id, 512); - ASSERT_EQ(out.image.size(), 3 * 10ul); - ASSERT_EQ(out.image.capacity(), 3 * 10ul); - for (int i = 0; i < 10; ++i) { - ASSERT_EQ(out.image.data()[i], i + 1); - ASSERT_EQ(out.image.data()[i + 10], i + 11); - ASSERT_EQ(out.image.data()[i + 20], i + 21); - } -} - -} // namespace test -} // namespace webrtc diff --git a/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc b/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc index 815505c810..df2893ac8c 100644 --- a/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc +++ b/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc @@ -88,8 +88,8 @@ class AnalyzingVideoSink : public rtc::VideoSinkInterface { VideoQualityAnalyzerInjectionHelper::VideoQualityAnalyzerInjectionHelper( std::unique_ptr analyzer, - EncodedImageIdInjector* injector, - EncodedImageIdExtractor* extractor) + EncodedImageDataInjector* injector, + EncodedImageDataExtractor* extractor) : analyzer_(std::move(analyzer)), injector_(injector), extractor_(extractor), @@ -102,10 +102,12 @@ VideoQualityAnalyzerInjectionHelper::~VideoQualityAnalyzerInjectionHelper() = std::unique_ptr VideoQualityAnalyzerInjectionHelper::WrapVideoEncoderFactory( - std::unique_ptr delegate) const { + std::unique_ptr delegate, + std::map> stream_required_spatial_index) + const { return absl::make_unique( - std::move(delegate), encoding_entities_id_generator_.get(), injector_, - analyzer_.get()); + std::move(delegate), std::move(stream_required_spatial_index), + encoding_entities_id_generator_.get(), injector_, analyzer_.get()); } std::unique_ptr diff --git a/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h b/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h index 880723f88f..1908b51874 100644 --- a/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h +++ b/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h @@ -11,6 +11,7 @@ #ifndef TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_QUALITY_ANALYZER_INJECTION_HELPER_H_ #define TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_QUALITY_ANALYZER_INJECTION_HELPER_H_ +#include #include #include @@ -19,7 +20,7 @@ #include "api/video_codecs/video_decoder_factory.h" #include "api/video_codecs/video_encoder_factory.h" #include "test/frame_generator.h" -#include "test/pc/e2e/analyzer/video/encoded_image_id_injector.h" +#include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h" #include "test/pc/e2e/analyzer/video/id_generator.h" #include "test/pc/e2e/api/video_quality_analyzer_interface.h" #include "test/testsupport/video_frame_writer.h" @@ -33,14 +34,16 @@ class VideoQualityAnalyzerInjectionHelper { public: VideoQualityAnalyzerInjectionHelper( std::unique_ptr analyzer, - EncodedImageIdInjector* injector, - EncodedImageIdExtractor* extractor); + EncodedImageDataInjector* injector, + EncodedImageDataExtractor* extractor); ~VideoQualityAnalyzerInjectionHelper(); // Wraps video encoder factory to give video quality analyzer access to frames // before encoding and encoded images after. std::unique_ptr WrapVideoEncoderFactory( - std::unique_ptr delegate) const; + std::unique_ptr delegate, + std::map> stream_required_spatial_index) + const; // Wraps video decoder factory to give video quality analyzer access to // received encoded images and frames, that were decoded from them. std::unique_ptr WrapVideoDecoderFactory( @@ -66,8 +69,8 @@ class VideoQualityAnalyzerInjectionHelper { private: std::unique_ptr analyzer_; - EncodedImageIdInjector* injector_; - EncodedImageIdExtractor* extractor_; + EncodedImageDataInjector* injector_; + EncodedImageDataExtractor* extractor_; std::unique_ptr> encoding_entities_id_generator_; }; diff --git a/test/pc/e2e/api/peerconnection_quality_test_fixture.h b/test/pc/e2e/api/peerconnection_quality_test_fixture.h index 79354a283c..4ed1a5bbd8 100644 --- a/test/pc/e2e/api/peerconnection_quality_test_fixture.h +++ b/test/pc/e2e/api/peerconnection_quality_test_fixture.h @@ -125,6 +125,19 @@ class PeerConnectionE2EQualityTestFixture { absl::optional input_file_name; // If specified screen share video stream will be created as input. absl::optional screen_share_config; + // Specifies spatial index of the video stream to analyze. + // There are 3 cases: + // 1. |target_spatial_index| omitted: in such case it will be assumed that + // video stream has not spatial layers and simulcast streams. + // 2. |target_spatial_index| presented and simulcast encoder is used: + // in such case |target_spatial_index| will specify the index of + // simulcast strem, that should be analyzed. Other streams will be + // dropped. + // 3. |target_spatial_index| presented and SVP encoder is used: + // in such case |target_spatial_index| will specify the top interesting + // spatial layer and all layers bellow, including target one will be + // processed. All layers above target one will be dropped. + absl::optional target_spatial_index; // If specified the input stream will be also copied to specified file. // It is actually one of the test's output file, which contains copy of what // was captured during the test for this video stream on sender side. diff --git a/test/pc/e2e/api/video_quality_analyzer_interface.h b/test/pc/e2e/api/video_quality_analyzer_interface.h index c112e09b66..4a7096f31c 100644 --- a/test/pc/e2e/api/video_quality_analyzer_interface.h +++ b/test/pc/e2e/api/video_quality_analyzer_interface.h @@ -93,6 +93,8 @@ class VideoQualityAnalyzerInterface { // Tells analyzer that analysis complete and it should calculate final // statistics. virtual void Stop() {} + + virtual std::string GetStreamLabel(uint16_t frame_id) = 0; }; } // namespace webrtc diff --git a/test/pc/e2e/peer_connection_quality_test.cc b/test/pc/e2e/peer_connection_quality_test.cc index d2d8551b4c..c7bd0a657d 100644 --- a/test/pc/e2e/peer_connection_quality_test.cc +++ b/test/pc/e2e/peer_connection_quality_test.cc @@ -101,7 +101,7 @@ PeerConnectionE2EQualityTest::PeerConnectionE2EQualityTest( video_quality_analyzer = absl::make_unique(); } encoded_image_id_controller_ = - absl::make_unique(); + absl::make_unique(); video_quality_analyzer_injection_helper_ = absl::make_unique( std::move(video_quality_analyzer), encoded_image_id_controller_.get(), diff --git a/test/pc/e2e/peer_connection_quality_test.h b/test/pc/e2e/peer_connection_quality_test.h index f401b639cc..eda6db3822 100644 --- a/test/pc/e2e/peer_connection_quality_test.h +++ b/test/pc/e2e/peer_connection_quality_test.h @@ -18,7 +18,7 @@ #include "rtc_base/task_queue.h" #include "rtc_base/thread.h" #include "system_wrappers/include/clock.h" -#include "test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector.h" +#include "test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h" #include "test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h" #include "test/pc/e2e/api/peerconnection_quality_test_fixture.h" #include "test/pc/e2e/test_peer.h" @@ -81,7 +81,7 @@ class PeerConnectionE2EQualityTest Clock* const clock_; std::unique_ptr video_quality_analyzer_injection_helper_; - std::unique_ptr + std::unique_ptr encoded_image_id_controller_; std::unique_ptr alice_; diff --git a/test/pc/e2e/test_peer.cc b/test/pc/e2e/test_peer.cc index d0e7e5a597..5dac263e29 100644 --- a/test/pc/e2e/test_peer.cc +++ b/test/pc/e2e/test_peer.cc @@ -29,7 +29,7 @@ #include "rtc_base/location.h" #include "rtc_base/network.h" #include "test/frame_generator_capturer.h" -#include "test/pc/e2e/analyzer/video/default_encoded_image_id_injector.h" +#include "test/pc/e2e/analyzer/video/default_encoded_image_data_injector.h" #include "test/pc/e2e/analyzer/video/example_video_quality_analyzer.h" #include "test/testsupport/copy_to_file_audio_capturer.h" @@ -122,7 +122,8 @@ rtc::scoped_refptr CreateAudioDeviceModule( std::unique_ptr CreateVideoEncoderFactory( PeerConnectionFactoryComponents* pcf_dependencies, - VideoQualityAnalyzerInjectionHelper* video_analyzer_helper) { + VideoQualityAnalyzerInjectionHelper* video_analyzer_helper, + std::map> stream_required_spatial_index) { std::unique_ptr video_encoder_factory; if (pcf_dependencies->video_encoder_factory != nullptr) { video_encoder_factory = std::move(pcf_dependencies->video_encoder_factory); @@ -130,7 +131,8 @@ std::unique_ptr CreateVideoEncoderFactory( video_encoder_factory = CreateBuiltinVideoEncoderFactory(); } return video_analyzer_helper->WrapVideoEncoderFactory( - std::move(video_encoder_factory)); + std::move(video_encoder_factory), + std::move(stream_required_spatial_index)); } std::unique_ptr CreateVideoDecoderFactory( @@ -149,13 +151,15 @@ std::unique_ptr CreateVideoDecoderFactory( std::unique_ptr CreateMediaEngine( PeerConnectionFactoryComponents* pcf_dependencies, absl::optional audio_config, + std::map> stream_required_spatial_index, VideoQualityAnalyzerInjectionHelper* video_analyzer_helper, absl::optional audio_output_file_name) { rtc::scoped_refptr adm = CreateAudioDeviceModule( std::move(audio_config), std::move(audio_output_file_name)); std::unique_ptr video_encoder_factory = - CreateVideoEncoderFactory(pcf_dependencies, video_analyzer_helper); + CreateVideoEncoderFactory(pcf_dependencies, video_analyzer_helper, + std::move(stream_required_spatial_index)); std::unique_ptr video_decoder_factory = CreateVideoDecoderFactory(pcf_dependencies, video_analyzer_helper); @@ -173,6 +177,7 @@ std::unique_ptr CreateMediaEngine( PeerConnectionFactoryDependencies CreatePCFDependencies( std::unique_ptr pcf_dependencies, absl::optional audio_config, + std::map> stream_required_spatial_index, VideoQualityAnalyzerInjectionHelper* video_analyzer_helper, rtc::Thread* network_thread, rtc::Thread* signaling_thread, @@ -181,7 +186,8 @@ PeerConnectionFactoryDependencies CreatePCFDependencies( pcf_deps.network_thread = network_thread; pcf_deps.signaling_thread = signaling_thread; pcf_deps.media_engine = CreateMediaEngine( - pcf_dependencies.get(), std::move(audio_config), video_analyzer_helper, + pcf_dependencies.get(), std::move(audio_config), + std::move(stream_required_spatial_index), video_analyzer_helper, std::move(audio_output_file_name)); pcf_deps.call_factory = std::move(pcf_dependencies->call_factory); @@ -254,10 +260,23 @@ std::unique_ptr TestPeer::CreateTestPeer( SetMandatoryEntities(components.get()); params->rtc_configuration.sdp_semantics = SdpSemantics::kUnifiedPlan; + std::map> stream_required_spatial_index; + for (auto& video_config : params->video_configs) { + // Stream label should be set by fixture implementation here. + RTC_DCHECK(video_config.stream_label); + bool res = stream_required_spatial_index + .insert({*video_config.stream_label, + video_config.target_spatial_index}) + .second; + RTC_DCHECK(res) << "Duplicate video_config.stream_label=" + << *video_config.stream_label; + } + // Create peer connection factory. PeerConnectionFactoryDependencies pcf_deps = CreatePCFDependencies( std::move(components->pcf_dependencies), params->audio_config, - video_analyzer_helper, components->network_thread, signaling_thread, + std::move(stream_required_spatial_index), video_analyzer_helper, + components->network_thread, signaling_thread, std::move(audio_output_file_name)); rtc::scoped_refptr pcf = CreateModularPeerConnectionFactory(std::move(pcf_deps)); diff --git a/test/pc/e2e/test_peer.h b/test/pc/e2e/test_peer.h index 3ee96c22ef..5f88de27f4 100644 --- a/test/pc/e2e/test_peer.h +++ b/test/pc/e2e/test_peer.h @@ -22,7 +22,7 @@ #include "pc/test/mock_peer_connection_observers.h" #include "rtc_base/network.h" #include "rtc_base/thread.h" -#include "test/pc/e2e/analyzer/video/encoded_image_id_injector.h" +#include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h" #include "test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h" #include "test/pc/e2e/api/peerconnection_quality_test_fixture.h"