From 1d4aa36988d9557bde0d36f14308fbf8d066be07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Terelius?= Date: Fri, 1 Oct 2021 13:45:46 +0200 Subject: [PATCH] Allow encoding string fields in new event log format. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Return parse results as a StatusOr containing views to values owned by the parser. Bug: webrtc:11933 Change-Id: Icf26b9cb651d1e9244c764c3ec1fdb66abfc9e08 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/233740 Reviewed-by: Sebastian Jansson Commit-Queue: Björn Terelius Cr-Commit-Position: refs/heads/main@{#35129} --- .../events/rtc_event_field_encoding.cc | 32 ++ .../events/rtc_event_field_encoding.h | 3 + .../events/rtc_event_field_encoding_parser.cc | 285 +++++++++++------- .../events/rtc_event_field_encoding_parser.h | 105 +++++-- .../rtc_event_field_encoding_unittest.cc | 155 +++++++--- .../events/rtc_event_field_extraction.h | 31 +- 6 files changed, 439 insertions(+), 172 deletions(-) diff --git a/logging/rtc_event_log/events/rtc_event_field_encoding.cc b/logging/rtc_event_log/events/rtc_event_field_encoding.cc index 7598587775..db407ff71c 100644 --- a/logging/rtc_event_log/events/rtc_event_field_encoding.cc +++ b/logging/rtc_event_log/events/rtc_event_field_encoding.cc @@ -229,6 +229,38 @@ void EventEncoder::EncodeField(const FieldParameters& params, } } +void EventEncoder::EncodeField(const FieldParameters& params, + const std::vector& values) { + RTC_DCHECK_EQ(values.size(), batch_size_); + + if (values.size() == 0) { + // If all values for a particular field is empty/nullopt, + // then we completely skip the field even if the the batch is non-empty. + return; + } + + // Write the field tag. + RTC_CHECK_NE(params.field_id, FieldParameters::kTimestampField); + RTC_DCHECK_LE(params.field_id, std::numeric_limits::max() >> 3); + RTC_DCHECK_EQ(params.field_type, FieldType::kString); + uint64_t field_tag = params.field_id << 3; + field_tag += static_cast(params.field_type); + encoded_fields_.push_back(EncodeVarInt(field_tag)); + + if (values.size() > 1) { + // If multiple values in the batch, write the encoding + // parameters. (Values >0 reserved for future use.) + uint64_t encoding_params = 0; + encoded_fields_.push_back(EncodeVarInt(encoding_params)); + } + + // Write the strings as (length, data) pairs. + for (absl::string_view s : values) { + encoded_fields_.push_back(EncodeVarInt(s.size())); + encoded_fields_.push_back(std::string(s)); + } +} + std::string EventEncoder::AsString() { std::string encoded_event; diff --git a/logging/rtc_event_log/events/rtc_event_field_encoding.h b/logging/rtc_event_log/events/rtc_event_field_encoding.h index d70644ebb8..8376a8b8a5 100644 --- a/logging/rtc_event_log/events/rtc_event_field_encoding.h +++ b/logging/rtc_event_log/events/rtc_event_field_encoding.h @@ -77,6 +77,9 @@ class EventEncoder { void EncodeField(const FieldParameters& params, const ValuesWithPositions& values); + void EncodeField(const FieldParameters& params, + const std::vector& values); + std::string AsString(); private: diff --git a/logging/rtc_event_log/events/rtc_event_field_encoding_parser.cc b/logging/rtc_event_log/events/rtc_event_field_encoding_parser.cc index 0eb88dd3d8..ea7ea6e804 100644 --- a/logging/rtc_event_log/events/rtc_event_field_encoding_parser.cc +++ b/logging/rtc_event_log/events/rtc_event_field_encoding_parser.cc @@ -74,13 +74,10 @@ uint64_t EventParser::ReadVarInt() { return output; } -uint64_t EventParser::ReadOptionalValuePositions(std::vector* positions) { - if (!positions) { - return CountAndIgnoreOptionalValuePositions(); - } - +uint64_t EventParser::ReadOptionalValuePositions() { + RTC_DCHECK(positions_.empty()); size_t bits_to_read = NumEventsInBatch(); - RTC_DCHECK(positions->empty()); + positions_.reserve(bits_to_read); if (pending_data_.size() * 8 < bits_to_read) { SetError(); return 0; @@ -88,7 +85,7 @@ uint64_t EventParser::ReadOptionalValuePositions(std::vector* positions) { BitstreamReader reader(pending_data_); for (size_t i = 0; i < bits_to_read; i++) { - positions->push_back(reader.ReadBit()); + positions_.push_back(reader.ReadBit()); } if (!reader.Ok()) { SetError(); @@ -96,30 +93,7 @@ uint64_t EventParser::ReadOptionalValuePositions(std::vector* positions) { } size_t num_existing_values = - std::count(positions->begin(), positions->end(), true); - pending_data_ = pending_data_.substr((bits_to_read + 7) / 8); - return num_existing_values; -} - -uint64_t EventParser::CountAndIgnoreOptionalValuePositions() { - size_t bits_to_read = NumEventsInBatch(); - if (pending_data_.size() * 8 < bits_to_read) { - SetError(); - return 0; - } - - BitstreamReader reader(pending_data_); - size_t num_existing_values = 0; - for (size_t i = 0; i < bits_to_read; i++) { - if (reader.ReadBit()) { - ++num_existing_values; - } - } - if (!reader.Ok()) { - SetError(); - return 0; - } - + std::count(positions_.begin(), positions_.end(), 1); pending_data_ = pending_data_.substr((bits_to_read + 7) / 8); return num_existing_values; } @@ -144,12 +118,10 @@ uint64_t EventParser::ReadSingleValue(FieldType field_type) { void EventParser::ReadDeltasAndPopulateValues( FixedLengthEncodingParametersV3 params, uint64_t num_deltas, - uint64_t base, - std::vector* values) { - RTC_CHECK(values != nullptr); - RTC_DCHECK(values->empty()); - values->reserve(num_deltas + 1); - values->push_back(base); + uint64_t base) { + RTC_DCHECK(values_.empty()); + values_.reserve(num_deltas + 1); + values_.push_back(base); if (pending_data_.size() * 8 < num_deltas * params.delta_bit_width()) { SetError(); @@ -174,7 +146,7 @@ void EventParser::ReadDeltasAndPopulateValues( } else { value = (value + delta) & params.value_mask(); } - values->push_back(value); + values_.push_back(value); } if (!reader.Ok()) { @@ -201,11 +173,111 @@ RtcEventLogParseStatus EventParser::Initialize(absl::string_view s, return RtcEventLogParseStatus::Success(); } -RtcEventLogParseStatus EventParser::ParseField(const FieldParameters& params, - std::vector* values, - std::vector* positions) { - RTC_CHECK(values != nullptr); +RtcEventLogParseStatus EventParser::ParseNumericFieldInternal( + uint64_t value_bit_width, + FieldType field_type) { + RTC_DCHECK(values_.empty()); + RTC_DCHECK(positions_.empty()); + if (num_events_ == 1) { + // Just a single value in the batch. + uint64_t base = ReadSingleValue(field_type); + if (!Ok()) { + return RtcEventLogParseStatus::Error("Failed to read value", __FILE__, + __LINE__); + } + positions_.push_back(true); + values_.push_back(base); + } else { + // Delta compressed batch. + // Read delta header. + uint64_t header_value = ReadVarInt(); + if (!Ok()) + return RtcEventLogParseStatus::Error("Failed to read delta header", + __FILE__, __LINE__); + // NB: value_bit_width may be incorrect for the field, if this isn't the + // field we are looking for. + absl::optional delta_header = + FixedLengthEncodingParametersV3::ParseDeltaHeader(header_value, + value_bit_width); + if (!delta_header.has_value()) { + return RtcEventLogParseStatus::Error("Failed to parse delta header", + __FILE__, __LINE__); + } + + uint64_t num_existing_deltas = NumEventsInBatch() - 1; + if (delta_header->values_optional()) { + size_t num_nonempty_values = ReadOptionalValuePositions(); + if (!Ok()) { + return RtcEventLogParseStatus::Error( + "Failed to read positions of optional values", __FILE__, __LINE__); + } + if (num_nonempty_values < 1 || NumEventsInBatch() < num_nonempty_values) { + return RtcEventLogParseStatus::Error( + "Expected at least one non_empty value", __FILE__, __LINE__); + } + num_existing_deltas = num_nonempty_values - 1; + } else { + // All elements in the batch have values. + positions_.assign(NumEventsInBatch(), 1u); + } + + // Read base. + uint64_t base = ReadSingleValue(field_type); + if (!Ok()) { + return RtcEventLogParseStatus::Error("Failed to read value", __FILE__, + __LINE__); + } + + if (delta_header->values_equal()) { + // Duplicate the base value num_existing_deltas times. + values_.assign(num_existing_deltas + 1, base); + } else { + // Read deltas; ceil(num_existing_deltas*delta_width/8) bits + ReadDeltasAndPopulateValues(delta_header.value(), num_existing_deltas, + base); + if (!Ok()) { + return RtcEventLogParseStatus::Error("Failed to decode deltas", + __FILE__, __LINE__); + } + } + } + return RtcEventLogParseStatus::Success(); +} + +RtcEventLogParseStatus EventParser::ParseStringFieldInternal() { + RTC_DCHECK(strings_.empty()); + if (num_events_ > 1) { + // String encoding params reserved for future use. + uint64_t encoding_params = ReadVarInt(); + if (!Ok()) { + return RtcEventLogParseStatus::Error("Failed to read string encoding", + __FILE__, __LINE__); + } + if (encoding_params != 0) { + return RtcEventLogParseStatus::Error( + "Unrecognized string encoding parameters", __FILE__, __LINE__); + } + } + strings_.reserve(num_events_); + for (uint64_t i = 0; i < num_events_; ++i) { + // Just a single value in the batch. + uint64_t size = ReadVarInt(); + if (!Ok()) { + return RtcEventLogParseStatus::Error("Failed to read string size", + __FILE__, __LINE__); + } + if (size > pending_data_.size()) { + return RtcEventLogParseStatus::Error("String size exceeds remaining data", + __FILE__, __LINE__); + } + strings_.push_back(pending_data_.substr(0, size)); + pending_data_ = pending_data_.substr(size); + } + return RtcEventLogParseStatus::Success(); +} + +RtcEventLogParseStatus EventParser::ParseField(const FieldParameters& params) { // Verify that the event parses fields in increasing order. if (params.field_id == FieldParameters::kTimestampField) { RTC_DCHECK_EQ(last_field_id_, FieldParameters::kTimestampField); @@ -224,10 +296,7 @@ RtcEventLogParseStatus EventParser::ParseField(const FieldParameters& params, // params.field_id doesn't exist. while (!pending_data_.empty()) { absl::string_view field_start = pending_data_; - values->clear(); - if (positions) { - positions->clear(); - } + ClearTemporaries(); // Read tag for non-positional fields. if (params.field_id != FieldParameters::kTimestampField) { @@ -252,72 +321,15 @@ RtcEventLogParseStatus EventParser::ParseField(const FieldParameters& params, return RtcEventLogParseStatus::Success(); } - if (num_events_ == 1) { - // Just a single value in the batch. - uint64_t base = ReadSingleValue(field_type); - if (!Ok()) - return RtcEventLogParseStatus::Error("Failed to read value", __FILE__, - __LINE__); - if (positions) { - positions->push_back(true); + if (field_type == FieldType::kString) { + auto status = ParseStringFieldInternal(); + if (!status.ok()) { + return status; } - values->push_back(base); } else { - // Delta compressed batch. - // Read delta header. - uint64_t header_value = ReadVarInt(); - if (!Ok()) - return RtcEventLogParseStatus::Error("Failed to read delta header", - __FILE__, __LINE__); - // NB: value_width may be incorrect for the field, if this isn't the field - // we are looking for. - absl::optional delta_header = - FixedLengthEncodingParametersV3::ParseDeltaHeader(header_value, - params.value_width); - if (!delta_header.has_value()) { - return RtcEventLogParseStatus::Error("Failed to parse delta header", - __FILE__, __LINE__); - } - - uint64_t num_existing_deltas = NumEventsInBatch() - 1; - if (delta_header->values_optional()) { - size_t num_nonempty_values = ReadOptionalValuePositions(positions); - if (!Ok()) { - return RtcEventLogParseStatus::Error( - "Failed to read positions of optional values", __FILE__, - __LINE__); - } - if (num_nonempty_values < 1 || - NumEventsInBatch() < num_nonempty_values) { - return RtcEventLogParseStatus::Error( - "Expected at least one non_empty value", __FILE__, __LINE__); - } - num_existing_deltas = num_nonempty_values - 1; - } else { - // All elements in the batch have values. - if (positions) { - positions->assign(NumEventsInBatch(), true); - } - } - - // Read base. - uint64_t base = ReadSingleValue(field_type); - if (!Ok()) { - return RtcEventLogParseStatus::Error("Failed to read value", __FILE__, - __LINE__); - } - - if (delta_header->values_equal()) { - // Duplicate the base value num_existing_deltas times. - values->assign(num_existing_deltas + 1, base); - } else { - // Read deltas; ceil(num_existing_deltas*delta_width/8) bits - ReadDeltasAndPopulateValues(delta_header.value(), num_existing_deltas, - base, values); - if (!Ok()) { - return RtcEventLogParseStatus::Error("Failed to decode deltas", - __FILE__, __LINE__); - } + auto status = ParseNumericFieldInternal(params.value_width, field_type); + if (!status.ok()) { + return status; } } @@ -328,11 +340,56 @@ RtcEventLogParseStatus EventParser::ParseField(const FieldParameters& params, } // Field not found because the event ended. - values->clear(); - if (positions) { - positions->clear(); - } + ClearTemporaries(); return RtcEventLogParseStatus::Success(); } +RtcEventLogParseStatusOr> +EventParser::ParseStringField(const FieldParameters& params, + bool required_field) { + using StatusOr = RtcEventLogParseStatusOr>; + RTC_DCHECK_EQ(params.field_type, FieldType::kString); + auto status = ParseField(params); + if (!status.ok()) + return StatusOr(status); + rtc::ArrayView strings = GetStrings(); + if (required_field && strings.size() != NumEventsInBatch()) { + return StatusOr::Error("Required string field not found", __FILE__, + __LINE__); + } + return StatusOr(strings); +} + +RtcEventLogParseStatusOr> +EventParser::ParseNumericField(const FieldParameters& params, + bool required_field) { + using StatusOr = RtcEventLogParseStatusOr>; + RTC_DCHECK_NE(params.field_type, FieldType::kString); + auto status = ParseField(params); + if (!status.ok()) + return StatusOr(status); + rtc::ArrayView values = GetValues(); + if (required_field && values.size() != NumEventsInBatch()) { + return StatusOr::Error("Required numerical field not found", __FILE__, + __LINE__); + } + return StatusOr(values); +} + +RtcEventLogParseStatusOr +EventParser::ParseOptionalNumericField(const FieldParameters& params, + bool required_field) { + using StatusOr = RtcEventLogParseStatusOr; + RTC_DCHECK_NE(params.field_type, FieldType::kString); + auto status = ParseField(params); + if (!status.ok()) + return StatusOr(status); + ValueAndPostionView view{GetValues(), GetPositions()}; + if (required_field && view.positions.size() != NumEventsInBatch()) { + return StatusOr::Error("Required numerical field not found", __FILE__, + __LINE__); + } + return StatusOr(view); +} + } // namespace webrtc diff --git a/logging/rtc_event_log/events/rtc_event_field_encoding_parser.h b/logging/rtc_event_log/events/rtc_event_field_encoding_parser.h index a1cf2b4319..f1af5db44a 100644 --- a/logging/rtc_event_log/events/rtc_event_field_encoding_parser.h +++ b/logging/rtc_event_log/events/rtc_event_field_encoding_parser.h @@ -24,6 +24,9 @@ // If/when we start using absl::Status in WebRTC, consider // whether payloads would be an appropriate alternative. class RtcEventLogParseStatus { + template + friend class RtcEventLogParseStatusOr; + public: static RtcEventLogParseStatus Success() { return RtcEventLogParseStatus(); } static RtcEventLogParseStatus Error(std::string error, @@ -33,6 +36,7 @@ class RtcEventLogParseStatus { } bool ok() const { return error_.empty(); } + std::string message() const { return error_; } private: @@ -43,10 +47,52 @@ class RtcEventLogParseStatus { std::string error_; }; +template +class RtcEventLogParseStatusOr { + public: + explicit RtcEventLogParseStatusOr(RtcEventLogParseStatus status) + : status_(status), value_() {} + explicit RtcEventLogParseStatusOr(const T& value) + : status_(), value_(value) {} + + bool ok() const { return status_.ok(); } + + std::string message() const { return status_.message(); } + + const T& value() const { + RTC_DCHECK(ok()); + return value_; + } + + T& value() { + RTC_DCHECK(ok()); + return value_; + } + + static RtcEventLogParseStatusOr Error(std::string error, + std::string file, + int line) { + return RtcEventLogParseStatusOr(error, file, line); + } + + private: + RtcEventLogParseStatusOr() : status_() {} + RtcEventLogParseStatusOr(std::string error, std::string file, int line) + : status_(error, file, line), value_() {} + + RtcEventLogParseStatus status_; + T value_; +}; + namespace webrtc { class EventParser { public: + struct ValueAndPostionView { + rtc::ArrayView values; + rtc::ArrayView positions; + }; + EventParser() = default; // N.B: This method stores a abls::string_view into the string to be @@ -55,17 +101,18 @@ class EventParser { RtcEventLogParseStatus Initialize(absl::string_view s, bool batched); // Attempts to parse the field specified by `params`, skipping past - // other fields that may occur before it. Returns - // RtcEventLogParseStatus::Success() and populates `values` (and `positions`) - // if the field is found. Returns RtcEventLogParseStatus::Success() and clears - // `values` (and `positions`) if the field doesn't exist. Returns a - // RtcEventLogParseStatus::Error if the log is incomplete, malformed or - // otherwise can't be parsed. `values` and `positions` are pure out-parameters - // that allow the caller to reuse the same temporary storage for all fields. - // Any previous content in the out parameters is cleared. - RtcEventLogParseStatus ParseField(const FieldParameters& params, - std::vector* values, - std::vector* positions = nullptr); + // other fields that may occur before it. If 'required_field == true', + // then failing to find the field is an error, otherwise the functions + // return success, but with an empty view of values. + RtcEventLogParseStatusOr> ParseStringField( + const FieldParameters& params, + bool required_field = true); + RtcEventLogParseStatusOr> ParseNumericField( + const FieldParameters& params, + bool required_field = true); + RtcEventLogParseStatusOr ParseOptionalNumericField( + const FieldParameters& params, + bool required_field = true); // Number of events in a batch. uint64_t NumEventsInBatch() const { return num_events_; } @@ -79,23 +126,47 @@ class EventParser { uint64_t ReadLittleEndian(uint8_t bytes); uint64_t ReadVarInt(); uint64_t ReadSingleValue(FieldType field_type); - uint64_t ReadOptionalValuePositions(std::vector* positions); - uint64_t CountAndIgnoreOptionalValuePositions(); + uint64_t ReadOptionalValuePositions(); void ReadDeltasAndPopulateValues(FixedLengthEncodingParametersV3 params, uint64_t num_deltas, - const uint64_t base, - std::vector* values); + const uint64_t base); + RtcEventLogParseStatus ParseNumericFieldInternal(uint64_t value_bit_width, + FieldType field_type); + RtcEventLogParseStatus ParseStringFieldInternal(); + + // Attempts to parse the field specified by `params`, skipping past + // other fields that may occur before it. Returns + // RtcEventLogParseStatus::Success() and populates `values_` (and + // `positions_`) if the field is found. Returns + // RtcEventLogParseStatus::Success() and clears `values_` (and `positions_`) + // if the field doesn't exist. Returns a RtcEventLogParseStatus::Error() if + // the log is incomplete, malformed or otherwise can't be parsed. + RtcEventLogParseStatus ParseField(const FieldParameters& params); void SetError() { error_ = true; } bool Ok() const { return !error_; } - // String to be consumed. - absl::string_view pending_data_; + rtc::ArrayView GetValues() { return values_; } + rtc::ArrayView GetPositions() { return positions_; } + rtc::ArrayView GetStrings() { return strings_; } + + void ClearTemporaries() { + positions_.clear(); + values_.clear(); + strings_.clear(); + } // Tracks whether an error has occurred in one of the helper // functions above. bool error_ = false; + // Temporary storage for result. + std::vector positions_; + std::vector values_; + std::vector strings_; + + // String to be consumed. + absl::string_view pending_data_; uint64_t num_events_ = 1; uint64_t last_field_id_ = FieldParameters::kTimestampField; }; diff --git a/logging/rtc_event_log/events/rtc_event_field_encoding_unittest.cc b/logging/rtc_event_log/events/rtc_event_field_encoding_unittest.cc index a993e7bbe2..3ff0114c31 100644 --- a/logging/rtc_event_log/events/rtc_event_field_encoding_unittest.cc +++ b/logging/rtc_event_log/events/rtc_event_field_encoding_unittest.cc @@ -116,6 +116,26 @@ size_t ExpectedEncodingSize(const FieldParameters& params, return tag_size + base_size + delta_header_size + positions_size + delta_size; } +size_t ExpectedStringEncodingSize(const FieldParameters& params, + const std::vector& values) { + EXPECT_EQ(params.field_type, FieldType::kString); + uint64_t numeric_field_type = static_cast(params.field_type); + RTC_DCHECK_LT(numeric_field_type, 1u << 3); + size_t tag_size = + ExpectedVarIntSize((params.field_id << 3) + numeric_field_type); + + size_t expected_size = tag_size; + if (values.size() > 1) { + // VarInt encoding header reserved for future use. Currently always 0. + expected_size += 1; + } + for (const auto& s : values) { + expected_size += ExpectedVarIntSize(s.size()); + expected_size += s.size(); + } + return expected_size; +} + } // namespace class RtcTestEvent final : public RtcEvent { @@ -137,7 +157,8 @@ class RtcTestEvent final : public RtcEvent { uint64_t unsigned64, absl::optional optional_signed32, absl::optional optional_signed64, - uint32_t wrapping21) + uint32_t wrapping21, + std::string string) : b_(b), signed32_(signed32), unsigned32_(unsigned32), @@ -145,7 +166,8 @@ class RtcTestEvent final : public RtcEvent { unsigned64_(unsigned64), optional_signed32_(optional_signed32), optional_signed64_(optional_signed64), - wrapping21_(wrapping21) {} + wrapping21_(wrapping21), + string_(string) {} ~RtcTestEvent() override = default; Type GetType() const override { return static_cast(4711); } @@ -170,6 +192,8 @@ class RtcTestEvent final : public RtcEvent { FieldType::kVarInt, 64}; static constexpr FieldParameters wrapping21_params{"wrapping21", 9, FieldType::kFixed32, 21}; + static constexpr FieldParameters string_params{ + "string", 10, FieldType::kString, /*value_width = */ 0}; static constexpr Type kType = static_cast(4711); @@ -181,6 +205,7 @@ class RtcTestEvent final : public RtcEvent { const absl::optional optional_signed32_ = absl::nullopt; const absl::optional optional_signed64_ = absl::nullopt; const uint32_t wrapping21_ = 0; + const std::string string_; }; constexpr EventParameters RtcTestEvent::event_params; @@ -194,6 +219,7 @@ constexpr FieldParameters RtcTestEvent::unsigned64_params; constexpr FieldParameters RtcTestEvent::optional32_params; constexpr FieldParameters RtcTestEvent::optional64_params; constexpr FieldParameters RtcTestEvent::wrapping21_params; +constexpr FieldParameters RtcTestEvent::string_params; constexpr RtcEvent::Type RtcTestEvent::kType; @@ -209,7 +235,8 @@ class RtcEventFieldTest : public ::testing::Test { const std::vector& unsigned64_values, const std::vector>& optional32_values, const std::vector>& optional64_values, - const std::vector& wrapping21_values) { + const std::vector& wrapping21_values, + const std::vector& string_values) { size_t size = bool_values.size(); RTC_CHECK_EQ(signed32_values.size(), size); RTC_CHECK_EQ(unsigned32_values.size(), size); @@ -218,12 +245,13 @@ class RtcEventFieldTest : public ::testing::Test { RTC_CHECK_EQ(optional32_values.size(), size); RTC_CHECK_EQ(optional64_values.size(), size); RTC_CHECK_EQ(wrapping21_values.size(), size); + RTC_CHECK_EQ(string_values.size(), size); for (size_t i = 0; i < size; i++) { batch_.push_back(new RtcTestEvent( bool_values[i], signed32_values[i], unsigned32_values[i], signed64_values[i], unsigned64_values[i], optional32_values[i], - optional64_values[i], wrapping21_values[i])); + optional64_values[i], wrapping21_values[i], string_values[i])); } } @@ -252,15 +280,33 @@ class RtcEventFieldTest : public ::testing::Test { } void ParseAndVerifyTimestamps() { - std::vector values; - auto status = parser_.ParseField(RtcTestEvent::timestamp_params, &values); - ASSERT_TRUE(status.ok()) << status.message().c_str(); - ASSERT_EQ(values.size(), batch_.size()); + auto result = parser_.ParseNumericField(RtcTestEvent::timestamp_params); + ASSERT_TRUE(result.ok()) << result.message().c_str(); + ASSERT_EQ(result.value().size(), batch_.size()); for (size_t i = 0; i < batch_.size(); i++) { - EXPECT_EQ(values[i], static_cast(batch_[i]->timestamp_ms())); + EXPECT_EQ(result.value()[i], + static_cast(batch_[i]->timestamp_ms())); } } + void ParseAndVerifyStringField( + const FieldParameters& params, + const std::vector& expected_values, + size_t expected_skipped_bytes = 0) { + size_t expected_size = ExpectedStringEncodingSize(params, expected_values) + + expected_skipped_bytes; + size_t size_before = parser_.RemainingBytes(); + auto result = parser_.ParseStringField(params); + ASSERT_TRUE(result.ok()) << result.message().c_str(); + ASSERT_EQ(result.value().size(), expected_values.size()); + for (size_t i = 0; i < expected_values.size(); i++) { + EXPECT_EQ(result.value()[i], expected_values[i]); + } + size_t size_after = parser_.RemainingBytes(); + EXPECT_EQ(size_before - size_after, expected_size) + << " for field " << params.name; + } + template void ParseAndVerifyField(const FieldParameters& params, const std::vector& expected_values, @@ -269,13 +315,13 @@ class RtcEventFieldTest : public ::testing::Test { size_t expected_size = ExpectedEncodingSize(params, expected_values, expected_bits_per_delta) + expected_skipped_bytes; - std::vector values; size_t size_before = parser_.RemainingBytes(); - auto status = parser_.ParseField(params, &values); - ASSERT_TRUE(status.ok()) << status.message().c_str(); - ASSERT_EQ(values.size(), expected_values.size()); + auto result = parser_.ParseNumericField(params); + ASSERT_TRUE(result.ok()) << result.message().c_str(); + ASSERT_EQ(result.value().size(), expected_values.size()); for (size_t i = 0; i < expected_values.size(); i++) { - EXPECT_EQ(DecodeFromUnsignedToType(values[i]), expected_values[i]); + EXPECT_EQ(DecodeFromUnsignedToType(result.value()[i]), + expected_values[i]); } size_t size_after = parser_.RemainingBytes(); EXPECT_EQ(size_before - size_after, expected_size) @@ -291,15 +337,13 @@ class RtcEventFieldTest : public ::testing::Test { size_t expected_size = ExpectedEncodingSize(params, expected_values, expected_bits_per_delta) + expected_skipped_bytes; - std::vector positions; - positions.reserve(expected_values.size()); - std::vector values; - values.reserve(expected_values.size()); size_t size_before = parser_.RemainingBytes(); - auto status = parser_.ParseField(params, &values, &positions); - ASSERT_TRUE(status.ok()) << status.message().c_str(); - auto value_it = values.begin(); + auto result = parser_.ParseOptionalNumericField(params); + ASSERT_TRUE(result.ok()) << result.message().c_str(); + rtc::ArrayView values = result.value().values; + rtc::ArrayView positions = result.value().positions; ASSERT_EQ(positions.size(), expected_values.size()); + auto value_it = values.begin(); for (size_t i = 0; i < expected_values.size(); i++) { if (positions[i]) { ASSERT_NE(value_it, values.end()); @@ -317,17 +361,17 @@ class RtcEventFieldTest : public ::testing::Test { } void ParseAndVerifyMissingField(const FieldParameters& params) { - std::vector values{4711}; - auto status = parser_.ParseField(params, &values); - ASSERT_TRUE(status.ok()) << status.message().c_str(); - EXPECT_EQ(values.size(), 0u); + auto result = parser_.ParseNumericField(params, /*required_field=*/false); + ASSERT_TRUE(result.ok()) << result.message().c_str(); + EXPECT_EQ(result.value().size(), 0u); } void ParseAndVerifyMissingOptionalField(const FieldParameters& params) { - std::vector positions{true, false}; - std::vector values{4711}; - auto status = parser_.ParseField(params, &values, &positions); - ASSERT_TRUE(status.ok()) << status.message().c_str(); + auto result = + parser_.ParseOptionalNumericField(params, /*required_field=*/false); + ASSERT_TRUE(result.ok()) << result.message().c_str(); + rtc::ArrayView values = result.value().values; + rtc::ArrayView positions = result.value().positions; EXPECT_EQ(positions.size(), 0u); EXPECT_EQ(values.size(), 0u); } @@ -359,10 +403,11 @@ TEST_F(RtcEventFieldTest, Singleton) { std::vector> optional32_values = {kInt32Min}; std::vector> optional64_values = {kInt64Max}; std::vector wrapping21_values = {(1 << 21) - 1}; + std::vector string_values = {"foo"}; CreateFullEvents(bool_values, signed32_values, unsigned32_values, signed64_values, unsigned64_values, optional32_values, - optional64_values, wrapping21_values); + optional64_values, wrapping21_values, string_values); EventEncoder encoder(RtcTestEvent::event_params, batch_); encoder.EncodeField(RtcTestEvent::bool_params, @@ -386,6 +431,8 @@ TEST_F(RtcEventFieldTest, Singleton) { encoder.EncodeField( RtcTestEvent::wrapping21_params, ExtractRtcEventMember(batch_, &RtcTestEvent::wrapping21_)); + encoder.EncodeField(RtcTestEvent::string_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::string_)); std::string s = encoder.AsString(); // Optional debug printing @@ -409,6 +456,8 @@ TEST_F(RtcEventFieldTest, Singleton) { optional64_values, /*no deltas*/ 0); ParseAndVerifyField(RtcTestEvent::wrapping21_params, wrapping21_values, /*no deltas*/ 0); + ParseAndVerifyStringField(RtcTestEvent::string_params, string_values); + EXPECT_EQ(parser_.RemainingBytes(), 0u); } TEST_F(RtcEventFieldTest, EqualElements) { @@ -426,10 +475,11 @@ TEST_F(RtcEventFieldTest, EqualElements) { kInt64Max, kInt64Max, kInt64Max, kInt64Max}; std::vector wrapping21_values = {(1 << 21) - 1, (1 << 21) - 1, (1 << 21) - 1, (1 << 21) - 1}; + std::vector string_values = {"foo", "foo", "foo", "foo"}; CreateFullEvents(bool_values, signed32_values, unsigned32_values, signed64_values, unsigned64_values, optional32_values, - optional64_values, wrapping21_values); + optional64_values, wrapping21_values, string_values); EventEncoder encoder(RtcTestEvent::event_params, batch_); encoder.EncodeField(RtcTestEvent::bool_params, ExtractRtcEventMember(batch_, &RtcTestEvent::b_)); @@ -452,6 +502,8 @@ TEST_F(RtcEventFieldTest, EqualElements) { encoder.EncodeField( RtcTestEvent::wrapping21_params, ExtractRtcEventMember(batch_, &RtcTestEvent::wrapping21_)); + encoder.EncodeField(RtcTestEvent::string_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::string_)); std::string s = encoder.AsString(); // Optional debug printing @@ -475,6 +527,8 @@ TEST_F(RtcEventFieldTest, EqualElements) { optional64_values, /*no deltas*/ 0); ParseAndVerifyField(RtcTestEvent::wrapping21_params, wrapping21_values, /*no deltas*/ 0); + ParseAndVerifyStringField(RtcTestEvent::string_params, string_values); + EXPECT_EQ(parser_.RemainingBytes(), 0u); } TEST_F(RtcEventFieldTest, Increasing) { @@ -490,10 +544,12 @@ TEST_F(RtcEventFieldTest, Increasing) { kInt64Max - 1, kInt64Max, kInt64Min, kInt64Min + 1}; std::vector wrapping21_values = {(1 << 21) - 2, (1 << 21) - 1, 0, 1}; + std::vector string_values = { + "", "a", "bc", "def"}; // No special compression of strings. CreateFullEvents(bool_values, signed32_values, unsigned32_values, signed64_values, unsigned64_values, optional32_values, - optional64_values, wrapping21_values); + optional64_values, wrapping21_values, string_values); EventEncoder encoder(RtcTestEvent::event_params, batch_); encoder.EncodeField(RtcTestEvent::bool_params, @@ -517,6 +573,8 @@ TEST_F(RtcEventFieldTest, Increasing) { encoder.EncodeField( RtcTestEvent::wrapping21_params, ExtractRtcEventMember(batch_, &RtcTestEvent::wrapping21_)); + encoder.EncodeField(RtcTestEvent::string_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::string_)); std::string s = encoder.AsString(); // Optional debug printing @@ -540,6 +598,8 @@ TEST_F(RtcEventFieldTest, Increasing) { optional64_values, /*delta bits*/ 1); ParseAndVerifyField(RtcTestEvent::wrapping21_params, wrapping21_values, /*delta bits*/ 1); + ParseAndVerifyStringField(RtcTestEvent::string_params, string_values); + EXPECT_EQ(parser_.RemainingBytes(), 0u); } TEST_F(RtcEventFieldTest, Decreasing) { @@ -555,10 +615,12 @@ TEST_F(RtcEventFieldTest, Decreasing) { kInt64Min + 1, kInt64Min, kInt64Max, kInt64Max - 1}; std::vector wrapping21_values = {1, 0, (1 << 21) - 1, (1 << 21) - 2}; + std::vector string_values = { + "def", "bc", "a", ""}; // No special compression of strings. CreateFullEvents(bool_values, signed32_values, unsigned32_values, signed64_values, unsigned64_values, optional32_values, - optional64_values, wrapping21_values); + optional64_values, wrapping21_values, string_values); EventEncoder encoder(RtcTestEvent::event_params, batch_); encoder.EncodeField(RtcTestEvent::bool_params, @@ -582,6 +644,8 @@ TEST_F(RtcEventFieldTest, Decreasing) { encoder.EncodeField( RtcTestEvent::wrapping21_params, ExtractRtcEventMember(batch_, &RtcTestEvent::wrapping21_)); + encoder.EncodeField(RtcTestEvent::string_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::string_)); std::string s = encoder.AsString(); // Optional debug printing @@ -605,6 +669,8 @@ TEST_F(RtcEventFieldTest, Decreasing) { optional64_values, /*delta bits*/ 1); ParseAndVerifyField(RtcTestEvent::wrapping21_params, wrapping21_values, /*delta bits*/ 1); + ParseAndVerifyStringField(RtcTestEvent::string_params, string_values); + EXPECT_EQ(parser_.RemainingBytes(), 0u); } TEST_F(RtcEventFieldTest, SkipsDeprecatedFields) { @@ -620,6 +686,7 @@ TEST_F(RtcEventFieldTest, SkipsDeprecatedFields) { std::vector> optional64_values = {kInt64Min / 2, kInt64Max / 2}; std::vector wrapping21_values = {0, 1 << 20}; + std::vector string_values = {"foo", "bar"}; size_t signed32_encoding_size = /*tag*/ 1 + /* varint base*/ 5 + /* delta_header*/ 1 + /*deltas*/ 4; @@ -630,7 +697,7 @@ TEST_F(RtcEventFieldTest, SkipsDeprecatedFields) { CreateFullEvents(bool_values, signed32_values, unsigned32_values, signed64_values, unsigned64_values, optional32_values, - optional64_values, wrapping21_values); + optional64_values, wrapping21_values, string_values); EventEncoder encoder(RtcTestEvent::event_params, batch_); encoder.EncodeField(RtcTestEvent::bool_params, @@ -654,6 +721,8 @@ TEST_F(RtcEventFieldTest, SkipsDeprecatedFields) { encoder.EncodeField( RtcTestEvent::wrapping21_params, ExtractRtcEventMember(batch_, &RtcTestEvent::wrapping21_)); + encoder.EncodeField(RtcTestEvent::string_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::string_)); std::string s = encoder.AsString(); // Optional debug printing @@ -666,7 +735,8 @@ TEST_F(RtcEventFieldTest, SkipsDeprecatedFields) { // Skips parsing the `signed32_values`. The following unsigned fields should // still be found. ParseAndVerifyField(RtcTestEvent::unsigned32_params, unsigned32_values, - /*delta_bits=*/31, signed32_encoding_size); + /*delta_bits=*/31, + /*expected_skipped_bytes=*/signed32_encoding_size); // Skips parsing the `signed64_values`. The following unsigned fields should // still be found. ParseAndVerifyField(RtcTestEvent::unsigned64_params, unsigned64_values, @@ -678,6 +748,8 @@ TEST_F(RtcEventFieldTest, SkipsDeprecatedFields) { /*delta_bits=*/63, optional32_encoding_size); ParseAndVerifyField(RtcTestEvent::wrapping21_params, wrapping21_values, /*delta_bits=*/20); + ParseAndVerifyStringField(RtcTestEvent::string_params, string_values); + EXPECT_EQ(parser_.RemainingBytes(), 0u); } TEST_F(RtcEventFieldTest, SkipsMissingFields) { @@ -693,10 +765,11 @@ TEST_F(RtcEventFieldTest, SkipsMissingFields) { std::vector> optional64_values = {kInt64Min / 2, kInt64Max / 2}; std::vector wrapping21_values = {0, 1 << 20}; + std::vector string_values = {"foo", "foo"}; CreateFullEvents(bool_values, signed32_values, unsigned32_values, signed64_values, unsigned64_values, optional32_values, - optional64_values, wrapping21_values); + optional64_values, wrapping21_values, string_values); EventEncoder encoder(RtcTestEvent::event_params, batch_); // Skip encoding the `bool_values`. @@ -713,6 +786,8 @@ TEST_F(RtcEventFieldTest, SkipsMissingFields) { encoder.EncodeField( RtcTestEvent::wrapping21_params, ExtractRtcEventMember(batch_, &RtcTestEvent::wrapping21_)); + encoder.EncodeField(RtcTestEvent::string_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::string_)); std::string s = encoder.AsString(); // Optional debug printing @@ -732,6 +807,8 @@ TEST_F(RtcEventFieldTest, SkipsMissingFields) { ParseAndVerifyMissingOptionalField(RtcTestEvent::optional64_params); ParseAndVerifyField(RtcTestEvent::wrapping21_params, wrapping21_values, /*delta_bits=*/20); + ParseAndVerifyStringField(RtcTestEvent::string_params, string_values); + EXPECT_EQ(parser_.RemainingBytes(), 0u); } TEST_F(RtcEventFieldTest, OptionalFields) { @@ -744,7 +821,7 @@ TEST_F(RtcEventFieldTest, OptionalFields) { for (size_t i = 0; i < optional32_values.size(); i++) { batch_.push_back(new RtcTestEvent(0, 0, 0, 0, 0, optional32_values[i], optional64_values[i], - wrapping21_values[i])); + wrapping21_values[i], "")); } EventEncoder encoder(RtcTestEvent::event_params, batch_); @@ -770,6 +847,7 @@ TEST_F(RtcEventFieldTest, OptionalFields) { optional64_values, /*delta bits*/ 1); ParseAndVerifyField(RtcTestEvent::wrapping21_params, wrapping21_values, /*delta bits*/ 2); + EXPECT_EQ(parser_.RemainingBytes(), 0u); } TEST_F(RtcEventFieldTest, AllNulloptTreatedAsMissing) { @@ -781,7 +859,7 @@ TEST_F(RtcEventFieldTest, AllNulloptTreatedAsMissing) { for (size_t i = 0; i < optional32_values.size(); i++) { batch_.push_back(new RtcTestEvent(0, 0, 0, 0, 0, optional32_values[i], - optional64_values[i], 0)); + optional64_values[i], 0, "")); } EventEncoder encoder(RtcTestEvent::event_params, batch_); @@ -801,6 +879,7 @@ TEST_F(RtcEventFieldTest, AllNulloptTreatedAsMissing) { ParseAndVerifyMissingOptionalField(RtcTestEvent::optional32_params); ParseAndVerifyOptionalField(RtcTestEvent::optional64_params, optional64_values, /*delta_bits=*/1); + EXPECT_EQ(parser_.RemainingBytes(), 0u); } } // namespace webrtc diff --git a/logging/rtc_event_log/events/rtc_event_field_extraction.h b/logging/rtc_event_log/events/rtc_event_field_extraction.h index a0e8ff01c7..8cd020fe16 100644 --- a/logging/rtc_event_log/events/rtc_event_field_extraction.h +++ b/logging/rtc_event_log/events/rtc_event_field_extraction.h @@ -116,13 +116,27 @@ ValuesWithPositions ExtractRtcEventMember(rtc::ArrayView batch, return result; } +template +std::vector ExtractRtcEventMember( + rtc::ArrayView batch, + const std::string E::*member) { + std::vector values; + values.reserve(batch.size()); + for (const RtcEvent* event : batch) { + RTC_CHECK_EQ(event->GetType(), E::kType); + absl::string_view str = static_cast(event)->*member; + values.push_back(str); + } + return values; +} + // Inverse of the ExtractRtcEventMember function used when parsing // a log. Uses a vector of values to populate a specific field in a // vector of structs. template ::value, bool> = true> -void PopulateRtcEventMember(const std::vector& values, +void PopulateRtcEventMember(const rtc::ArrayView values, T E::*member, rtc::ArrayView output) { size_t batch_size = values.size(); @@ -136,8 +150,8 @@ void PopulateRtcEventMember(const std::vector& values, template ::value, bool> = true> -void PopulateRtcEventMember(const std::vector& positions, - const std::vector& values, +void PopulateRtcEventMember(const rtc::ArrayView positions, + const rtc::ArrayView values, absl::optional E::*member, rtc::ArrayView output) { size_t batch_size = positions.size(); @@ -156,6 +170,17 @@ void PopulateRtcEventMember(const std::vector& positions, RTC_CHECK(value_it == values.end()); } +template +void PopulateRtcEventMember(const rtc::ArrayView values, + std::string E::*member, + rtc::ArrayView output) { + size_t batch_size = values.size(); + RTC_CHECK_EQ(output.size(), batch_size); + for (size_t i = 0; i < batch_size; ++i) { + output[i].*member = values[i]; + } +} + } // namespace webrtc #endif // LOGGING_RTC_EVENT_LOG_EVENTS_RTC_EVENT_FIELD_EXTRACTION_H_