diff --git a/logging/BUILD.gn b/logging/BUILD.gn index 68f6092ade..eae92de5b2 100644 --- a/logging/BUILD.gn +++ b/logging/BUILD.gn @@ -34,6 +34,12 @@ rtc_source_set("rtc_event_log_api") { rtc_library("rtc_event_field") { sources = [ + "rtc_event_log/events/fixed_length_encoding_parameters_v3.cc", + "rtc_event_log/events/fixed_length_encoding_parameters_v3.h", + "rtc_event_log/events/rtc_event_field_encoding.cc", + "rtc_event_log/events/rtc_event_field_encoding.h", + "rtc_event_log/events/rtc_event_field_encoding_parser.cc", + "rtc_event_log/events/rtc_event_field_encoding_parser.h", "rtc_event_log/events/rtc_event_field_extraction.cc", "rtc_event_log/events/rtc_event_field_extraction.h", ] @@ -42,12 +48,12 @@ rtc_library("rtc_event_field") { ":rtc_event_number_encodings", "../api:array_view", "../api/rtc_event_log", + "../rtc_base:bitstream_reader", "../rtc_base:checks", "../rtc_base:logging", "../rtc_base:rtc_base_approved", ] absl_deps = [ - "//third_party/abseil-cpp/absl/memory", "//third_party/abseil-cpp/absl/strings", "//third_party/abseil-cpp/absl/types:optional", ] @@ -414,6 +420,7 @@ if (rtc_enable_protobuf) { "rtc_event_log/encoder/delta_encoding_unittest.cc", "rtc_event_log/encoder/rtc_event_log_encoder_common_unittest.cc", "rtc_event_log/encoder/rtc_event_log_encoder_unittest.cc", + "rtc_event_log/events/rtc_event_field_encoding_unittest.cc", "rtc_event_log/events/rtc_event_field_extraction_unittest.cc", "rtc_event_log/rtc_event_log_unittest.cc", "rtc_event_log/rtc_event_log_unittest_helper.cc", diff --git a/logging/rtc_event_log/events/fixed_length_encoding_parameters_v3.cc b/logging/rtc_event_log/events/fixed_length_encoding_parameters_v3.cc new file mode 100644 index 0000000000..0c93e6226d --- /dev/null +++ b/logging/rtc_event_log/events/fixed_length_encoding_parameters_v3.cc @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "logging/rtc_event_log/events/fixed_length_encoding_parameters_v3.h" + +#include + +#include "absl/types/optional.h" +#include "api/array_view.h" +#include "logging/rtc_event_log/events/rtc_event_field_extraction.h" +#include "rtc_base/logging.h" + +using webrtc_event_logging::MaxUnsignedValueOfBitWidth; +using webrtc_event_logging::SignedBitWidth; +using webrtc_event_logging::UnsignedBitWidth; +using webrtc_event_logging::UnsignedDelta; + +namespace webrtc { + +FixedLengthEncodingParametersV3 +FixedLengthEncodingParametersV3::CalculateParameters( + uint64_t base, + const rtc::ArrayView values, + uint64_t value_bit_width, + bool values_optional) { + // As a special case, if all of the elements are identical to the base + // we just encode the base value with a special delta header. + if (std::all_of(values.cbegin(), values.cend(), + [base](uint64_t val) { return val == base; })) { + // Delta header with signed=true and delta_bitwidth=64 + return FixedLengthEncodingParametersV3(/*delta_bit_width=*/64, + /*signed_deltas=*/true, + values_optional, value_bit_width); + } + + const uint64_t bit_mask = MaxUnsignedValueOfBitWidth(value_bit_width); + + // Calculate the bitwidth required to encode all deltas when using a + // unsigned or signed represenation, respectively. For the unsigned + // representation, we just track the largest delta. For the signed + // representation, we have two possibilities for each delta; either + // going "forward" (i.e. current - previous) or "backwards" + // (i.e. previous - current) where both values are calculated with + // wrap around. We then track the largest positive and negative + // magnitude across the batch, assuming that we choose the smaller + // delta for each element. + uint64_t max_unsigned_delta = 0; + uint64_t max_positive_signed_delta = 0; + uint64_t min_negative_signed_delta = 0; + uint64_t prev = base; + for (uint64_t current : values) { + uint64_t positive_delta = UnsignedDelta(prev, current, bit_mask); + uint64_t negative_delta = UnsignedDelta(current, prev, bit_mask); + + max_unsigned_delta = std::max(max_unsigned_delta, positive_delta); + + if (positive_delta < negative_delta) { + max_positive_signed_delta = + std::max(max_positive_signed_delta, positive_delta); + } else { + min_negative_signed_delta = + std::max(min_negative_signed_delta, negative_delta); + } + + prev = current; + } + + // We now know the largest unsigned delta and the largest magnitudes of + // positive and negative signed deltas. Get the bitwidths required for + // each of the two encodings. + const uint64_t unsigned_delta_bit_width = + UnsignedBitWidth(max_unsigned_delta); + const uint64_t signed_delta_bit_width = + SignedBitWidth(max_positive_signed_delta, min_negative_signed_delta); + + // Note: Preference for unsigned if the two have the same width (efficiency). + bool use_signed_deltas = signed_delta_bit_width < unsigned_delta_bit_width; + uint64_t delta_bit_width = + use_signed_deltas ? signed_delta_bit_width : unsigned_delta_bit_width; + + // use_signed_deltas && delta_bit_width==64 is reserved for "all values + // equal". + RTC_DCHECK(!use_signed_deltas || delta_bit_width < 64); + + RTC_DCHECK(ValidParameters(delta_bit_width, use_signed_deltas, + values_optional, value_bit_width)); + return FixedLengthEncodingParametersV3(delta_bit_width, use_signed_deltas, + values_optional, value_bit_width); +} + +uint64_t FixedLengthEncodingParametersV3::DeltaHeaderAsInt() const { + uint64_t header = delta_bit_width_ - 1; + RTC_CHECK_LT(header, 1u << 6); + if (signed_deltas_) { + header += 1u << 6; + } + RTC_CHECK_LT(header, 1u << 7); + if (values_optional_) { + header += 1u << 7; + } + return header; +} + +absl::optional +FixedLengthEncodingParametersV3::ParseDeltaHeader(uint64_t header, + uint64_t value_bit_width) { + uint64_t delta_bit_width = (header & ((1u << 6) - 1)) + 1; + bool signed_deltas = header & (1u << 6); + bool values_optional = header & (1u << 7); + + if (header >= (1u << 8)) { + RTC_LOG(LS_ERROR) << "Failed to parse delta header; unread bits remaining."; + return absl::nullopt; + } + + if (!ValidParameters(delta_bit_width, signed_deltas, values_optional, + value_bit_width)) { + RTC_LOG(LS_ERROR) << "Failed to parse delta header. Invalid combination of " + "values: delta_bit_width=" + << delta_bit_width << " signed_deltas=" << signed_deltas + << " values_optional=" << values_optional + << " value_bit_width=" << value_bit_width; + return absl::nullopt; + } + + return FixedLengthEncodingParametersV3(delta_bit_width, signed_deltas, + values_optional, value_bit_width); +} + +} // namespace webrtc diff --git a/logging/rtc_event_log/events/fixed_length_encoding_parameters_v3.h b/logging/rtc_event_log/events/fixed_length_encoding_parameters_v3.h new file mode 100644 index 0000000000..59d633ced7 --- /dev/null +++ b/logging/rtc_event_log/events/fixed_length_encoding_parameters_v3.h @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef LOGGING_RTC_EVENT_LOG_EVENTS_FIXED_LENGTH_ENCODING_PARAMETERS_V3_H_ +#define LOGGING_RTC_EVENT_LOG_EVENTS_FIXED_LENGTH_ENCODING_PARAMETERS_V3_H_ + +#include "absl/types/optional.h" +#include "api/array_view.h" +#include "logging/rtc_event_log/events/rtc_event_field_extraction.h" + +namespace webrtc { + +// Parameters for fixed-size delta-encoding/decoding. +// These are tailored for the sequence which will be encoded (e.g. widths). +class FixedLengthEncodingParametersV3 final { + public: + static bool ValidParameters(uint64_t delta_bit_width, + bool signed_deltas, + bool values_optional, + uint64_t value_bit_width) { + return (1 <= delta_bit_width && delta_bit_width <= 64 && + 1 <= value_bit_width && value_bit_width <= 64 && + (delta_bit_width <= value_bit_width || + (signed_deltas && delta_bit_width == 64))); + } + + static FixedLengthEncodingParametersV3 CalculateParameters( + uint64_t base, + const rtc::ArrayView values, + uint64_t value_bit_width, + bool values_optional); + static absl::optional ParseDeltaHeader( + uint64_t header, + uint64_t value_bit_width); + + uint64_t DeltaHeaderAsInt() const; + + // Number of bits necessary to hold the widest(*) of the deltas between the + // values in the sequence. + // (*) - Widest might not be the largest, if signed deltas are used. + uint64_t delta_bit_width() const { return delta_bit_width_; } + + // Whether deltas are signed. + bool signed_deltas() const { return signed_deltas_; } + + // Whether the values of the sequence are optional. That is, it may be + // that some of them do not have a value (not even a sentinel value indicating + // invalidity). + bool values_optional() const { return values_optional_; } + + // Whether all values are equal. 64-bit signed deltas are assumed to not + // occur, since those could equally well be represented using 64 bit unsigned + // deltas. + bool values_equal() const { + return delta_bit_width() == 64 && signed_deltas(); + } + + // Number of bits necessary to hold the largest value in the sequence. + uint64_t value_bit_width() const { return value_bit_width_; } + + // Masks where only the bits relevant to the deltas/values are turned on. + uint64_t delta_mask() const { return delta_mask_; } + uint64_t value_mask() const { return value_mask_; } + + private: + FixedLengthEncodingParametersV3(uint64_t delta_bit_width, + bool signed_deltas, + bool values_optional, + uint64_t value_bit_width) + : delta_bit_width_(delta_bit_width), + signed_deltas_(signed_deltas), + values_optional_(values_optional), + value_bit_width_(value_bit_width), + delta_mask_( + webrtc_event_logging::MaxUnsignedValueOfBitWidth(delta_bit_width_)), + value_mask_(webrtc_event_logging::MaxUnsignedValueOfBitWidth( + value_bit_width_)) {} + + uint64_t delta_bit_width_; + bool signed_deltas_; + bool values_optional_; + uint64_t value_bit_width_; + + uint64_t delta_mask_; + uint64_t value_mask_; +}; + +} // namespace webrtc +#endif // LOGGING_RTC_EVENT_LOG_EVENTS_FIXED_LENGTH_ENCODING_PARAMETERS_V3_H_ diff --git a/logging/rtc_event_log/events/rtc_event_field_encoding.cc b/logging/rtc_event_log/events/rtc_event_field_encoding.cc new file mode 100644 index 0000000000..7598587775 --- /dev/null +++ b/logging/rtc_event_log/events/rtc_event_field_encoding.cc @@ -0,0 +1,265 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "logging/rtc_event_log/events/rtc_event_field_encoding.h" + +#include +#include +#include +#include + +#include "logging/rtc_event_log/encoder/bit_writer.h" +#include "logging/rtc_event_log/encoder/var_int.h" +#include "logging/rtc_event_log/events/rtc_event_field_extraction.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +using webrtc_event_logging::UnsignedDelta; + +namespace { + +std::string SerializeLittleEndian(uint64_t value, uint8_t bytes) { + RTC_DCHECK_LE(bytes, sizeof(uint64_t)); + RTC_DCHECK_GE(bytes, 1); + if (bytes < sizeof(uint64_t)) { + // Note that shifting a 64-bit value by 64 (or more) bits is undefined. + RTC_DCHECK_EQ(value >> (8 * bytes), 0); + } + std::string output(bytes, 0); + // Getting a non-const pointer to the representation. See e.g. + // https://en.cppreference.com/w/cpp/string/basic_string: + // "The elements of a basic_string are stored contiguously, + // that is, [...] a pointer to s[0] can be passed to functions + // that expect a pointer to the first element of a null-terminated + // CharT[] array." + uint8_t* p = reinterpret_cast(&output[0]); +#ifdef WEBRTC_ARCH_LITTLE_ENDIAN + memcpy(p, &value, bytes); +#else + while (bytes > 0) { + *p = static_cast(value & 0xFF); + value >>= 8; + ++p; + --bytes; + } +#endif // WEBRTC_ARCH_LITTLE_ENDIAN + return output; +} + +} // namespace + +namespace webrtc { + +std::string EncodeOptionalValuePositions(std::vector positions) { + BitWriter writer((positions.size() + 7) / 8); + for (bool position : positions) { + writer.WriteBits(position ? 1u : 0u, 1); + } + return writer.GetString(); +} + +std::string EncodeSingleValue(uint64_t value, FieldType field_type) { + switch (field_type) { + case FieldType::kFixed8: + return SerializeLittleEndian(value, /*bytes=*/1); + case FieldType::kFixed32: + return SerializeLittleEndian(value, /*bytes=*/4); + case FieldType::kFixed64: + return SerializeLittleEndian(value, /*bytes=*/8); + case FieldType::kVarInt: + return EncodeVarInt(value); + case FieldType::kString: + RTC_NOTREACHED(); + return std::string(); + } +} + +absl::optional ConvertFieldType(uint64_t value) { + switch (value) { + case static_cast(FieldType::kFixed8): + return FieldType::kFixed8; + case static_cast(FieldType::kFixed32): + return FieldType::kFixed32; + case static_cast(FieldType::kFixed64): + return FieldType::kFixed64; + case static_cast(FieldType::kVarInt): + return FieldType::kVarInt; + case static_cast(FieldType::kString): + return FieldType::kString; + default: + return absl::nullopt; + } +} + +std::string EncodeDeltasV3(FixedLengthEncodingParametersV3 params, + uint64_t base, + rtc::ArrayView values) { + size_t outputbound = (values.size() * params.delta_bit_width() + 7) / 8; + BitWriter writer(outputbound); + + uint64_t previous = base; + for (uint64_t value : values) { + if (params.signed_deltas()) { + uint64_t positive_delta = + UnsignedDelta(previous, value, params.value_mask()); + uint64_t negative_delta = + UnsignedDelta(value, previous, params.value_mask()); + uint64_t delta; + if (positive_delta <= negative_delta) { + delta = positive_delta; + } else { + // Compute the two's complement representation of a negative + // delta, in a field width params_.delta_mask(). + RTC_DCHECK_GE(params.delta_mask(), negative_delta); + RTC_DCHECK_LT(params.delta_mask() - negative_delta, + params.delta_mask()); + delta = params.delta_mask() - negative_delta + 1; + RTC_DCHECK_LE(delta, params.delta_mask()); + } + writer.WriteBits(delta, params.delta_bit_width()); + } else { + uint64_t delta = UnsignedDelta(previous, value, params.value_mask()); + writer.WriteBits(delta, params.delta_bit_width()); + } + previous = value; + } + + return writer.GetString(); +} + +EventEncoder::EventEncoder(EventParameters params, + rtc::ArrayView batch) { + batch_size_ = batch.size(); + if (!batch.empty()) { + // Encode event type. + uint32_t batched = batch.size() > 1 ? 1 : 0; + event_tag_ = (static_cast(params.id) << 1) + batched; + + // Event tag and number of encoded bytes will be filled in when the + // encoding is finalized in AsString(). + + // Encode number of events in batch + if (batched) { + encoded_fields_.push_back(EncodeVarInt(batch.size())); + } + + // Encode timestamp + std::vector timestamps; + timestamps.reserve(batch.size()); + for (const RtcEvent* event : batch) { + timestamps.push_back(EncodeAsUnsigned(event->timestamp_ms())); + } + constexpr FieldParameters timestamp_params{"timestamp_ms", + FieldParameters::kTimestampField, + FieldType::kVarInt, 64}; + EncodeField(timestamp_params, timestamps); + } +} + +void EventEncoder::EncodeField(const FieldParameters& params, + const ValuesWithPositions& values) { + return EncodeField(params, values.values, &values.position_mask); +} + +void EventEncoder::EncodeField(const FieldParameters& params, + const std::vector& values, + const std::vector* positions) { + if (positions) { + RTC_DCHECK_EQ(positions->size(), batch_size_); + RTC_DCHECK_LE(values.size(), batch_size_); + } else { + RTC_DCHECK_EQ(values.size(), batch_size_); + } + + if (values.size() == 0) { + // If all values for a particular field is empty/nullopt, + // then we completely skip the field even if the the batch is non-empty. + return; + } + + // We know that each event starts with the varint encoded timestamp, + // so we omit that field tag (field id + field type). In all other + // cases, we write the field tag. + if (params.field_id != FieldParameters::kTimestampField) { + RTC_DCHECK_LE(params.field_id, std::numeric_limits::max() >> 3); + uint64_t field_tag = params.field_id << 3; + field_tag += static_cast(params.field_type); + encoded_fields_.push_back(EncodeVarInt(field_tag)); + } + + RTC_CHECK_GE(values.size(), 1); + if (batch_size_ == 1) { + encoded_fields_.push_back(EncodeSingleValue(values[0], params.field_type)); + return; + } + + const bool values_optional = values.size() != batch_size_; + + // Compute delta parameters + rtc::ArrayView all_values(values); + uint64_t base = values[0]; + rtc::ArrayView remaining_values(all_values.subview(1)); + + FixedLengthEncodingParametersV3 delta_params = + FixedLengthEncodingParametersV3::CalculateParameters( + base, remaining_values, params.value_width, values_optional); + + encoded_fields_.push_back(EncodeVarInt(delta_params.DeltaHeaderAsInt())); + + if (values_optional) { + RTC_CHECK(positions); + encoded_fields_.push_back(EncodeOptionalValuePositions(*positions)); + } + // Base element, encoded as uint8, uint32, uint64 or varint + encoded_fields_.push_back(EncodeSingleValue(base, params.field_type)); + + // If all (existing) values are equal to the base, then we can skip + // writing the all-zero deltas, and instead infer those from the delta + // header. + if (!delta_params.values_equal()) { + encoded_fields_.push_back( + EncodeDeltasV3(delta_params, base, remaining_values)); + } +} + +std::string EventEncoder::AsString() { + std::string encoded_event; + + if (batch_size_ == 0) { + RTC_DCHECK_EQ(encoded_fields_.size(), 0); + return encoded_event; + } + + // Compute size of encoded fields. + size_t total_fields_size = 0; + for (const std::string& s : encoded_fields_) { + total_fields_size += s.size(); + } + + constexpr size_t kExpectedMaxEventTagBytes = 4; + constexpr size_t kExpectedMaxSizeEncodingBytes = 4; + encoded_event.reserve(kExpectedMaxEventTagBytes + + kExpectedMaxSizeEncodingBytes + total_fields_size); + + // Encode event tag (event id and whether batch or single event). + encoded_event.append(EncodeVarInt(event_tag_)); + + // Encode size of the remaining fields. + encoded_event.append(EncodeVarInt(total_fields_size)); + + // Append encoded fields. + for (const std::string& s : encoded_fields_) { + encoded_event.append(s); + } + + return encoded_event; +} + +} // namespace webrtc diff --git a/logging/rtc_event_log/events/rtc_event_field_encoding.h b/logging/rtc_event_log/events/rtc_event_field_encoding.h new file mode 100644 index 0000000000..d70644ebb8 --- /dev/null +++ b/logging/rtc_event_log/events/rtc_event_field_encoding.h @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef LOGGING_RTC_EVENT_LOG_EVENTS_RTC_EVENT_FIELD_ENCODING_H_ +#define LOGGING_RTC_EVENT_LOG_EVENTS_RTC_EVENT_FIELD_ENCODING_H_ + +#include +#include + +#include "absl/types/optional.h" +#include "api/array_view.h" +#include "api/rtc_event_log/rtc_event.h" +#include "logging/rtc_event_log/encoder/rtc_event_log_encoder_common.h" +#include "logging/rtc_event_log/events/fixed_length_encoding_parameters_v3.h" +#include "logging/rtc_event_log/events/rtc_event_field_extraction.h" +#include "rtc_base/logging.h" + +namespace webrtc { + +// To maintain backwards compatibility with past (or future) logs, +// the constants in this enum must not be changed. +// New field types with numerical IDs 5-7 can be added, but old +// parsers will fail to parse events containing the new fields. +enum class FieldType : uint8_t { + kFixed8 = 0, + kFixed32 = 1, + kFixed64 = 2, + kVarInt = 3, + kString = 4, +}; + +// EventParameters map an event name to a numerical ID. +struct EventParameters { + // The name is primarily used for debugging purposes. + const char* const name; + // + const RtcEvent::Type id; +}; + +// FieldParameters define the encoding for a field. +struct FieldParameters { + // The name is primarily used for debugging purposes. + const char* const name; + // Numerical ID for the field. Must be strictly greater than 0, + // and unique within each event type. + const uint64_t field_id; + // Encoding type for the base (i.e. non-delta) field in a batch. + const FieldType field_type; + // Number of bits after which wrap-around occurs. In most cases, + // this should be the number of bits in the field data type, i.e. + // 8 for an uint8_t, 32 for a int32_t and so on. However, `value_width` + // can be used to achieve a more efficient encoding if it is known + // that the field uses a smaller number of bits. For example, a + // 15-bit counter could set `value_width` to 15 even if the data is + // actually stored in a uint32_t. + const uint64_t value_width; + // Field ID 0 is reserved for timestamps. + static constexpr uint64_t kTimestampField = 0; +}; + +// The EventEncoder is used to encode a batch of events. +class EventEncoder { + public: + EventEncoder(EventParameters params, rtc::ArrayView batch); + + void EncodeField(const FieldParameters& params, + const std::vector& values, + const std::vector* positions = nullptr); + + void EncodeField(const FieldParameters& params, + const ValuesWithPositions& values); + + std::string AsString(); + + private: + size_t batch_size_; + uint32_t event_tag_; + std::vector encoded_fields_; +}; + +std::string EncodeSingleValue(uint64_t value, FieldType field_type); +std::string EncodeDeltasV3(FixedLengthEncodingParametersV3 params, + uint64_t base, + rtc::ArrayView values); + +} // namespace webrtc +#endif // LOGGING_RTC_EVENT_LOG_EVENTS_RTC_EVENT_FIELD_ENCODING_H_ diff --git a/logging/rtc_event_log/events/rtc_event_field_encoding_parser.cc b/logging/rtc_event_log/events/rtc_event_field_encoding_parser.cc new file mode 100644 index 0000000000..0eb88dd3d8 --- /dev/null +++ b/logging/rtc_event_log/events/rtc_event_field_encoding_parser.cc @@ -0,0 +1,338 @@ + +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "logging/rtc_event_log/events/rtc_event_field_encoding_parser.h" + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "logging/rtc_event_log/encoder/var_int.h" +#include "logging/rtc_event_log/events/rtc_event_field_encoding.h" +#include "rtc_base/bitstream_reader.h" +#include "rtc_base/checks.h" + +namespace { +absl::optional ConvertFieldType(uint64_t value) { + switch (value) { + case static_cast(webrtc::FieldType::kFixed8): + return webrtc::FieldType::kFixed8; + case static_cast(webrtc::FieldType::kFixed32): + return webrtc::FieldType::kFixed32; + case static_cast(webrtc::FieldType::kFixed64): + return webrtc::FieldType::kFixed64; + case static_cast(webrtc::FieldType::kVarInt): + return webrtc::FieldType::kVarInt; + case static_cast(webrtc::FieldType::kString): + return webrtc::FieldType::kString; + default: + return absl::nullopt; + } +} +} // namespace + +namespace webrtc { + +uint64_t EventParser::ReadLittleEndian(uint8_t bytes) { + RTC_DCHECK_LE(bytes, sizeof(uint64_t)); + RTC_DCHECK_GE(bytes, 1); + + uint64_t value = 0; + + if (bytes > pending_data_.length()) { + SetError(); + return value; + } + + const uint8_t* p = reinterpret_cast(pending_data_.data()); + unsigned int shift = 0; + uint8_t remaining = bytes; + while (remaining > 0) { + value += (static_cast(*p) << shift); + shift += 8; + ++p; + --remaining; + } + + pending_data_ = pending_data_.substr(bytes); + return value; +} + +uint64_t EventParser::ReadVarInt() { + uint64_t output = 0; + bool success; + std::tie(success, pending_data_) = DecodeVarInt(pending_data_, &output); + if (!success) { + SetError(); + } + return output; +} + +uint64_t EventParser::ReadOptionalValuePositions(std::vector* positions) { + if (!positions) { + return CountAndIgnoreOptionalValuePositions(); + } + + size_t bits_to_read = NumEventsInBatch(); + RTC_DCHECK(positions->empty()); + if (pending_data_.size() * 8 < bits_to_read) { + SetError(); + return 0; + } + + BitstreamReader reader(pending_data_); + for (size_t i = 0; i < bits_to_read; i++) { + positions->push_back(reader.ReadBit()); + } + if (!reader.Ok()) { + SetError(); + return 0; + } + + size_t num_existing_values = + std::count(positions->begin(), positions->end(), true); + pending_data_ = pending_data_.substr((bits_to_read + 7) / 8); + return num_existing_values; +} + +uint64_t EventParser::CountAndIgnoreOptionalValuePositions() { + size_t bits_to_read = NumEventsInBatch(); + if (pending_data_.size() * 8 < bits_to_read) { + SetError(); + return 0; + } + + BitstreamReader reader(pending_data_); + size_t num_existing_values = 0; + for (size_t i = 0; i < bits_to_read; i++) { + if (reader.ReadBit()) { + ++num_existing_values; + } + } + if (!reader.Ok()) { + SetError(); + return 0; + } + + pending_data_ = pending_data_.substr((bits_to_read + 7) / 8); + return num_existing_values; +} + +uint64_t EventParser::ReadSingleValue(FieldType field_type) { + switch (field_type) { + case FieldType::kFixed8: + return ReadLittleEndian(/*bytes=*/1); + case FieldType::kFixed32: + return ReadLittleEndian(/*bytes=*/4); + case FieldType::kFixed64: + return ReadLittleEndian(/*bytes=*/8); + case FieldType::kVarInt: + return ReadVarInt(); + case FieldType::kString: + RTC_NOTREACHED(); + SetError(); + return 0; + } +} + +void EventParser::ReadDeltasAndPopulateValues( + FixedLengthEncodingParametersV3 params, + uint64_t num_deltas, + uint64_t base, + std::vector* values) { + RTC_CHECK(values != nullptr); + RTC_DCHECK(values->empty()); + values->reserve(num_deltas + 1); + values->push_back(base); + + if (pending_data_.size() * 8 < num_deltas * params.delta_bit_width()) { + SetError(); + return; + } + + BitstreamReader reader(pending_data_); + const uint64_t top_bit = static_cast(1) + << (params.delta_bit_width() - 1); + + uint64_t value = base; + for (uint64_t i = 0; i < num_deltas; ++i) { + uint64_t delta = reader.ReadBits(params.delta_bit_width()); + RTC_DCHECK_LE(value, webrtc_event_logging::MaxUnsignedValueOfBitWidth( + params.value_bit_width())); + RTC_DCHECK_LE(delta, webrtc_event_logging::MaxUnsignedValueOfBitWidth( + params.delta_bit_width())); + bool negative_delta = params.signed_deltas() && ((delta & top_bit) != 0); + if (negative_delta) { + uint64_t delta_abs = (~delta & params.delta_mask()) + 1; + value = (value - delta_abs) & params.value_mask(); + } else { + value = (value + delta) & params.value_mask(); + } + values->push_back(value); + } + + if (!reader.Ok()) { + SetError(); + return; + } + + pending_data_ = + pending_data_.substr((num_deltas * params.delta_bit_width() + 7) / 8); +} + +RtcEventLogParseStatus EventParser::Initialize(absl::string_view s, + bool batched) { + pending_data_ = s; + num_events_ = 1; + + if (batched) { + num_events_ = ReadVarInt(); + if (!Ok()) { + return RtcEventLogParseStatus::Error( + "Failed to read number of events in batch.", __FILE__, __LINE__); + } + } + return RtcEventLogParseStatus::Success(); +} + +RtcEventLogParseStatus EventParser::ParseField(const FieldParameters& params, + std::vector* values, + std::vector* positions) { + RTC_CHECK(values != nullptr); + + // Verify that the event parses fields in increasing order. + if (params.field_id == FieldParameters::kTimestampField) { + RTC_DCHECK_EQ(last_field_id_, FieldParameters::kTimestampField); + } else { + RTC_DCHECK_GT(params.field_id, last_field_id_); + } + last_field_id_ = params.field_id; + + // Initialization for positional fields that don't encode field ID and type. + uint64_t field_id = params.field_id; + FieldType field_type = params.field_type; + + // Fields are encoded in increasing field_id order. + // Skip unknown fields with field_id < params.field_id until we either + // find params.field_id or a field with higher id, in which case we know that + // params.field_id doesn't exist. + while (!pending_data_.empty()) { + absl::string_view field_start = pending_data_; + values->clear(); + if (positions) { + positions->clear(); + } + + // Read tag for non-positional fields. + if (params.field_id != FieldParameters::kTimestampField) { + uint64_t field_tag = ReadVarInt(); + if (!Ok()) + return RtcEventLogParseStatus::Error("Failed to read field tag", + __FILE__, __LINE__); + // Split tag into field ID and field type. + field_id = field_tag >> 3; + absl::optional conversion = ConvertFieldType(field_tag & 7u); + if (!conversion.has_value()) + return RtcEventLogParseStatus::Error("Failed to parse field type", + __FILE__, __LINE__); + field_type = conversion.value(); + } + + if (field_id > params.field_id) { + // We've passed all fields with ids less than or equal to what we are + // looking for. Reset pending_data_ to first field with id higher than + // params.field_id, since we didn't find the field we were looking for. + pending_data_ = field_start; + return RtcEventLogParseStatus::Success(); + } + + if (num_events_ == 1) { + // Just a single value in the batch. + uint64_t base = ReadSingleValue(field_type); + if (!Ok()) + return RtcEventLogParseStatus::Error("Failed to read value", __FILE__, + __LINE__); + if (positions) { + positions->push_back(true); + } + values->push_back(base); + } else { + // Delta compressed batch. + // Read delta header. + uint64_t header_value = ReadVarInt(); + if (!Ok()) + return RtcEventLogParseStatus::Error("Failed to read delta header", + __FILE__, __LINE__); + // NB: value_width may be incorrect for the field, if this isn't the field + // we are looking for. + absl::optional delta_header = + FixedLengthEncodingParametersV3::ParseDeltaHeader(header_value, + params.value_width); + if (!delta_header.has_value()) { + return RtcEventLogParseStatus::Error("Failed to parse delta header", + __FILE__, __LINE__); + } + + uint64_t num_existing_deltas = NumEventsInBatch() - 1; + if (delta_header->values_optional()) { + size_t num_nonempty_values = ReadOptionalValuePositions(positions); + if (!Ok()) { + return RtcEventLogParseStatus::Error( + "Failed to read positions of optional values", __FILE__, + __LINE__); + } + if (num_nonempty_values < 1 || + NumEventsInBatch() < num_nonempty_values) { + return RtcEventLogParseStatus::Error( + "Expected at least one non_empty value", __FILE__, __LINE__); + } + num_existing_deltas = num_nonempty_values - 1; + } else { + // All elements in the batch have values. + if (positions) { + positions->assign(NumEventsInBatch(), true); + } + } + + // Read base. + uint64_t base = ReadSingleValue(field_type); + if (!Ok()) { + return RtcEventLogParseStatus::Error("Failed to read value", __FILE__, + __LINE__); + } + + if (delta_header->values_equal()) { + // Duplicate the base value num_existing_deltas times. + values->assign(num_existing_deltas + 1, base); + } else { + // Read deltas; ceil(num_existing_deltas*delta_width/8) bits + ReadDeltasAndPopulateValues(delta_header.value(), num_existing_deltas, + base, values); + if (!Ok()) { + return RtcEventLogParseStatus::Error("Failed to decode deltas", + __FILE__, __LINE__); + } + } + } + + if (field_id == params.field_id) { + // The field we're looking for has been found and values populated. + return RtcEventLogParseStatus::Success(); + } + } + + // Field not found because the event ended. + values->clear(); + if (positions) { + positions->clear(); + } + return RtcEventLogParseStatus::Success(); +} + +} // namespace webrtc diff --git a/logging/rtc_event_log/events/rtc_event_field_encoding_parser.h b/logging/rtc_event_log/events/rtc_event_field_encoding_parser.h new file mode 100644 index 0000000000..a1cf2b4319 --- /dev/null +++ b/logging/rtc_event_log/events/rtc_event_field_encoding_parser.h @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef LOGGING_RTC_EVENT_LOG_EVENTS_RTC_EVENT_FIELD_ENCODING_PARSER_H_ +#define LOGGING_RTC_EVENT_LOG_EVENTS_RTC_EVENT_FIELD_ENCODING_PARSER_H_ + +#include +#include + +#include "logging/rtc_event_log/events/rtc_event_field_encoding.h" + +// TODO(terelius): Compared to a generic 'Status' class, this +// class allows us additional information about the context +// in which the error occurred. This is currently limited to +// the source location (file and line), but we plan on adding +// information about the event and field name being parsed. +// If/when we start using absl::Status in WebRTC, consider +// whether payloads would be an appropriate alternative. +class RtcEventLogParseStatus { + public: + static RtcEventLogParseStatus Success() { return RtcEventLogParseStatus(); } + static RtcEventLogParseStatus Error(std::string error, + std::string file, + int line) { + return RtcEventLogParseStatus(error, file, line); + } + + bool ok() const { return error_.empty(); } + std::string message() const { return error_; } + + private: + RtcEventLogParseStatus() : error_() {} + RtcEventLogParseStatus(std::string error, std::string file, int line) + : error_(error + " (" + file + ": " + std::to_string(line) + ")") {} + + std::string error_; +}; + +namespace webrtc { + +class EventParser { + public: + EventParser() = default; + + // N.B: This method stores a abls::string_view into the string to be + // parsed. The caller is responsible for ensuring that the actual string + // remains unmodified and outlives the EventParser. + RtcEventLogParseStatus Initialize(absl::string_view s, bool batched); + + // Attempts to parse the field specified by `params`, skipping past + // other fields that may occur before it. Returns + // RtcEventLogParseStatus::Success() and populates `values` (and `positions`) + // if the field is found. Returns RtcEventLogParseStatus::Success() and clears + // `values` (and `positions`) if the field doesn't exist. Returns a + // RtcEventLogParseStatus::Error if the log is incomplete, malformed or + // otherwise can't be parsed. `values` and `positions` are pure out-parameters + // that allow the caller to reuse the same temporary storage for all fields. + // Any previous content in the out parameters is cleared. + RtcEventLogParseStatus ParseField(const FieldParameters& params, + std::vector* values, + std::vector* positions = nullptr); + + // Number of events in a batch. + uint64_t NumEventsInBatch() const { return num_events_; } + + // Bytes remaining in `pending_data_`. Assuming there are no unknown + // fields, BytesRemaining() should return 0 when all known fields + // in the event have been parsed. + size_t RemainingBytes() const { return pending_data_.size(); } + + private: + uint64_t ReadLittleEndian(uint8_t bytes); + uint64_t ReadVarInt(); + uint64_t ReadSingleValue(FieldType field_type); + uint64_t ReadOptionalValuePositions(std::vector* positions); + uint64_t CountAndIgnoreOptionalValuePositions(); + void ReadDeltasAndPopulateValues(FixedLengthEncodingParametersV3 params, + uint64_t num_deltas, + const uint64_t base, + std::vector* values); + + void SetError() { error_ = true; } + bool Ok() const { return !error_; } + + // String to be consumed. + absl::string_view pending_data_; + + // Tracks whether an error has occurred in one of the helper + // functions above. + bool error_ = false; + + uint64_t num_events_ = 1; + uint64_t last_field_id_ = FieldParameters::kTimestampField; +}; + +} // namespace webrtc +#endif // LOGGING_RTC_EVENT_LOG_EVENTS_RTC_EVENT_FIELD_ENCODING_PARSER_H_ diff --git a/logging/rtc_event_log/events/rtc_event_field_encoding_unittest.cc b/logging/rtc_event_log/events/rtc_event_field_encoding_unittest.cc new file mode 100644 index 0000000000..a993e7bbe2 --- /dev/null +++ b/logging/rtc_event_log/events/rtc_event_field_encoding_unittest.cc @@ -0,0 +1,806 @@ +/* Copyright (c) 2021 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "logging/rtc_event_log/events/rtc_event_field_encoding.h" + +#include +#include +#include + +#include "api/rtc_event_log/rtc_event.h" +#include "logging/rtc_event_log/encoder/var_int.h" +#include "logging/rtc_event_log/events/rtc_event_field_encoding_parser.h" +#include "test/gtest.h" + +namespace webrtc { + +namespace { +constexpr int32_t kInt32Max = std::numeric_limits::max(); +constexpr int32_t kInt32Min = std::numeric_limits::min(); +constexpr uint32_t kUint32Max = std::numeric_limits::max(); +constexpr int64_t kInt64Max = std::numeric_limits::max(); +constexpr int64_t kInt64Min = std::numeric_limits::min(); +constexpr uint64_t kUint64Max = std::numeric_limits::max(); + +template ::value, bool> = true> +size_t ExpectedVarIntSize(T value) { + size_t bytes = 0; + uint64_t x = EncodeAsUnsigned(value); + do { + ++bytes; + x = x >> 7; + } while (x > 0); + return bytes; +} + +template ::value, bool> = true> +size_t ExpectedBaseValueSize(const FieldParameters& params, T value) { + switch (params.field_type) { + case FieldType::kFixed8: + return 1; + case FieldType::kFixed32: + return 4; + case FieldType::kFixed64: + return 8; + case FieldType::kVarInt: + return ExpectedVarIntSize(value); + default: + break; + } + RTC_NOTREACHED(); + return 0; +} + +template ::value, bool> = true> +size_t ExpectedEncodingSize(const FieldParameters& params, + const std::vector& v, + size_t expected_bits_per_delta) { + if (v.size() == 0) + return 0; + + uint64_t numeric_field_type = static_cast(params.field_type); + RTC_DCHECK_LT(numeric_field_type, 1u << 3); + size_t tag_size = + ExpectedVarIntSize((params.field_id << 3) + numeric_field_type); + T base = v[0]; + size_t base_size = ExpectedBaseValueSize(params, base); + if (v.size() == 1) + return tag_size + base_size; + + size_t delta_header_size = 1; + // Check if there is an element *not* equal to base. + if (std::all_of(v.begin(), v.end(), [base](T x) { return x == base; })) { + return tag_size + base_size + delta_header_size; + } + + size_t delta_size = ((v.size() - 1) * expected_bits_per_delta + 7) / 8; + return tag_size + base_size + delta_header_size + delta_size; +} + +template ::value, bool> = true> +size_t ExpectedEncodingSize(const FieldParameters& params, + const std::vector>& v, + size_t expected_bits_per_delta) { + size_t num_existing_values = + v.size() - std::count(v.begin(), v.end(), absl::nullopt); + auto first_existing_value = std::find_if( + v.begin(), v.end(), [](absl::optional x) { return x.has_value(); }); + if (num_existing_values == 0) + return 0; + + uint64_t numeric_field_type = static_cast(params.field_type); + RTC_DCHECK_LT(numeric_field_type, 1u << 3); + size_t tag_size = + ExpectedVarIntSize((params.field_id << 3) + numeric_field_type); + T base = first_existing_value->value(); + size_t base_size = ExpectedBaseValueSize(params, base); + if (num_existing_values == 1 && v.size() == 1) + return tag_size + base_size; + + size_t delta_header_size = (num_existing_values == v.size() ? 1 : 2); + size_t positions_size = + (num_existing_values == v.size() ? 0 : (v.size() + 7) / 8); + // Check if there is an element *not* equal to base. + if (std::all_of(v.begin(), v.end(), + [base](absl::optional x) { return x == base; })) { + return tag_size + base_size + delta_header_size + positions_size; + } + + size_t delta_size = + ((num_existing_values - 1) * expected_bits_per_delta + 7) / 8; + return tag_size + base_size + delta_header_size + positions_size + delta_size; +} + +} // namespace + +class RtcTestEvent final : public RtcEvent { + public: + RtcTestEvent(bool b, + int32_t signed32, + uint32_t unsigned32, + int64_t signed64, + uint64_t unsigned64) + : b_(b), + signed32_(signed32), + unsigned32_(unsigned32), + signed64_(signed64), + unsigned64_(unsigned64) {} + RtcTestEvent(bool b, + int32_t signed32, + uint32_t unsigned32, + int64_t signed64, + uint64_t unsigned64, + absl::optional optional_signed32, + absl::optional optional_signed64, + uint32_t wrapping21) + : b_(b), + signed32_(signed32), + unsigned32_(unsigned32), + signed64_(signed64), + unsigned64_(unsigned64), + optional_signed32_(optional_signed32), + optional_signed64_(optional_signed64), + wrapping21_(wrapping21) {} + ~RtcTestEvent() override = default; + + Type GetType() const override { return static_cast(4711); } + bool IsConfigEvent() const override { return false; } + + static constexpr EventParameters event_params{ + "TestEvent", static_cast(4711)}; + static constexpr FieldParameters timestamp_params{ + "timestamp_ms", FieldParameters::kTimestampField, FieldType::kVarInt, 64}; + static constexpr FieldParameters bool_params{"b", 2, FieldType::kFixed8, 1}; + static constexpr FieldParameters signed32_params{"signed32", 3, + FieldType::kVarInt, 32}; + static constexpr FieldParameters unsigned32_params{"unsigned32", 4, + FieldType::kFixed32, 32}; + static constexpr FieldParameters signed64_params{"signed64", 5, + FieldType::kFixed64, 64}; + static constexpr FieldParameters unsigned64_params{"unsigned64", 6, + FieldType::kVarInt, 64}; + static constexpr FieldParameters optional32_params{"optional_signed32", 7, + FieldType::kFixed32, 32}; + static constexpr FieldParameters optional64_params{"optional_signed64", 8, + FieldType::kVarInt, 64}; + static constexpr FieldParameters wrapping21_params{"wrapping21", 9, + FieldType::kFixed32, 21}; + + static constexpr Type kType = static_cast(4711); + + const bool b_; + const int32_t signed32_; + const uint32_t unsigned32_; + const int64_t signed64_; + const uint64_t unsigned64_; + const absl::optional optional_signed32_ = absl::nullopt; + const absl::optional optional_signed64_ = absl::nullopt; + const uint32_t wrapping21_ = 0; +}; + +constexpr EventParameters RtcTestEvent::event_params; +constexpr FieldParameters RtcTestEvent::timestamp_params; +constexpr FieldParameters RtcTestEvent::bool_params; +constexpr FieldParameters RtcTestEvent::signed32_params; +constexpr FieldParameters RtcTestEvent::unsigned32_params; +constexpr FieldParameters RtcTestEvent::signed64_params; +constexpr FieldParameters RtcTestEvent::unsigned64_params; + +constexpr FieldParameters RtcTestEvent::optional32_params; +constexpr FieldParameters RtcTestEvent::optional64_params; +constexpr FieldParameters RtcTestEvent::wrapping21_params; + +constexpr RtcEvent::Type RtcTestEvent::kType; + +class RtcEventFieldTest : public ::testing::Test { + protected: + void SetUp() override {} + + void CreateFullEvents( + const std::vector& bool_values, + const std::vector& signed32_values, + const std::vector& unsigned32_values, + const std::vector& signed64_values, + const std::vector& unsigned64_values, + const std::vector>& optional32_values, + const std::vector>& optional64_values, + const std::vector& wrapping21_values) { + size_t size = bool_values.size(); + RTC_CHECK_EQ(signed32_values.size(), size); + RTC_CHECK_EQ(unsigned32_values.size(), size); + RTC_CHECK_EQ(signed64_values.size(), size); + RTC_CHECK_EQ(unsigned64_values.size(), size); + RTC_CHECK_EQ(optional32_values.size(), size); + RTC_CHECK_EQ(optional64_values.size(), size); + RTC_CHECK_EQ(wrapping21_values.size(), size); + + for (size_t i = 0; i < size; i++) { + batch_.push_back(new RtcTestEvent( + bool_values[i], signed32_values[i], unsigned32_values[i], + signed64_values[i], unsigned64_values[i], optional32_values[i], + optional64_values[i], wrapping21_values[i])); + } + } + + void PrintBytes(const std::string& s) { + for (auto c : s) { + fprintf(stderr, "%d ", static_cast(c)); + } + fprintf(stderr, "\n"); + } + + void ParseEventHeader(absl::string_view encoded_event) { + uint64_t event_tag; + bool success; + std::tie(success, encoded_event) = DecodeVarInt(encoded_event, &event_tag); + ASSERT_TRUE(success); + uint64_t event_id = event_tag >> 1; + ASSERT_EQ(event_id, static_cast(RtcTestEvent::event_params.id)); + bool batched = event_tag & 1u; + ASSERT_EQ(batched, batch_.size() > 1u); + + uint64_t size; + std::tie(success, encoded_event) = DecodeVarInt(encoded_event, &size); + ASSERT_EQ(encoded_event.size(), size); + + ASSERT_TRUE(parser_.Initialize(encoded_event, batched).ok()); + } + + void ParseAndVerifyTimestamps() { + std::vector values; + auto status = parser_.ParseField(RtcTestEvent::timestamp_params, &values); + ASSERT_TRUE(status.ok()) << status.message().c_str(); + ASSERT_EQ(values.size(), batch_.size()); + for (size_t i = 0; i < batch_.size(); i++) { + EXPECT_EQ(values[i], static_cast(batch_[i]->timestamp_ms())); + } + } + + template + void ParseAndVerifyField(const FieldParameters& params, + const std::vector& expected_values, + size_t expected_bits_per_delta, + size_t expected_skipped_bytes = 0) { + size_t expected_size = + ExpectedEncodingSize(params, expected_values, expected_bits_per_delta) + + expected_skipped_bytes; + std::vector values; + size_t size_before = parser_.RemainingBytes(); + auto status = parser_.ParseField(params, &values); + ASSERT_TRUE(status.ok()) << status.message().c_str(); + ASSERT_EQ(values.size(), expected_values.size()); + for (size_t i = 0; i < expected_values.size(); i++) { + EXPECT_EQ(DecodeFromUnsignedToType(values[i]), expected_values[i]); + } + size_t size_after = parser_.RemainingBytes(); + EXPECT_EQ(size_before - size_after, expected_size) + << " for field " << params.name; + } + + template + void ParseAndVerifyOptionalField( + const FieldParameters& params, + const std::vector>& expected_values, + size_t expected_bits_per_delta, + size_t expected_skipped_bytes = 0) { + size_t expected_size = + ExpectedEncodingSize(params, expected_values, expected_bits_per_delta) + + expected_skipped_bytes; + std::vector positions; + positions.reserve(expected_values.size()); + std::vector values; + values.reserve(expected_values.size()); + size_t size_before = parser_.RemainingBytes(); + auto status = parser_.ParseField(params, &values, &positions); + ASSERT_TRUE(status.ok()) << status.message().c_str(); + auto value_it = values.begin(); + ASSERT_EQ(positions.size(), expected_values.size()); + for (size_t i = 0; i < expected_values.size(); i++) { + if (positions[i]) { + ASSERT_NE(value_it, values.end()); + ASSERT_TRUE(expected_values[i].has_value()); + EXPECT_EQ(DecodeFromUnsignedToType(*value_it), + expected_values[i].value()); + ++value_it; + } else { + EXPECT_EQ(absl::nullopt, expected_values[i]); + } + } + EXPECT_EQ(value_it, values.end()); + size_t size_after = parser_.RemainingBytes(); + EXPECT_EQ(size_before - size_after, expected_size); + } + + void ParseAndVerifyMissingField(const FieldParameters& params) { + std::vector values{4711}; + auto status = parser_.ParseField(params, &values); + ASSERT_TRUE(status.ok()) << status.message().c_str(); + EXPECT_EQ(values.size(), 0u); + } + + void ParseAndVerifyMissingOptionalField(const FieldParameters& params) { + std::vector positions{true, false}; + std::vector values{4711}; + auto status = parser_.ParseField(params, &values, &positions); + ASSERT_TRUE(status.ok()) << status.message().c_str(); + EXPECT_EQ(positions.size(), 0u); + EXPECT_EQ(values.size(), 0u); + } + + void TearDown() override { + for (const RtcEvent* event : batch_) { + delete event; + } + } + + std::vector batch_; + EventParser parser_; +}; + +TEST_F(RtcEventFieldTest, EmptyList) { + EventEncoder encoder(RtcTestEvent::event_params, batch_); + encoder.EncodeField(RtcTestEvent::bool_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::b_)); + std::string s = encoder.AsString(); + EXPECT_TRUE(s.empty()); +} + +TEST_F(RtcEventFieldTest, Singleton) { + std::vector bool_values = {true}; + std::vector signed32_values = {-2}; + std::vector unsigned32_values = {123456789}; + std::vector signed64_values = {-9876543210}; + std::vector unsigned64_values = {9876543210}; + std::vector> optional32_values = {kInt32Min}; + std::vector> optional64_values = {kInt64Max}; + std::vector wrapping21_values = {(1 << 21) - 1}; + + CreateFullEvents(bool_values, signed32_values, unsigned32_values, + signed64_values, unsigned64_values, optional32_values, + optional64_values, wrapping21_values); + + EventEncoder encoder(RtcTestEvent::event_params, batch_); + encoder.EncodeField(RtcTestEvent::bool_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::b_)); + encoder.EncodeField(RtcTestEvent::signed32_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::signed32_)); + encoder.EncodeField( + RtcTestEvent::unsigned32_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::unsigned32_)); + encoder.EncodeField(RtcTestEvent::signed64_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::signed64_)); + encoder.EncodeField( + RtcTestEvent::unsigned64_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::unsigned64_)); + encoder.EncodeField( + RtcTestEvent::optional32_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::optional_signed32_)); + encoder.EncodeField( + RtcTestEvent::optional64_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::optional_signed64_)); + encoder.EncodeField( + RtcTestEvent::wrapping21_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::wrapping21_)); + std::string s = encoder.AsString(); + + // Optional debug printing + // PrintBytes(s); + + ParseEventHeader(s); + ParseAndVerifyTimestamps(); + ParseAndVerifyField(RtcTestEvent::bool_params, bool_values, + /*no deltas*/ 0); + ParseAndVerifyField(RtcTestEvent::signed32_params, signed32_values, + /*no deltas*/ 0); + ParseAndVerifyField(RtcTestEvent::unsigned32_params, unsigned32_values, + /*no deltas*/ 0); + ParseAndVerifyField(RtcTestEvent::signed64_params, signed64_values, + /*no deltas*/ 0); + ParseAndVerifyField(RtcTestEvent::unsigned64_params, unsigned64_values, + /*no deltas*/ 0); + ParseAndVerifyOptionalField(RtcTestEvent::optional32_params, + optional32_values, /*no deltas*/ 0); + ParseAndVerifyOptionalField(RtcTestEvent::optional64_params, + optional64_values, /*no deltas*/ 0); + ParseAndVerifyField(RtcTestEvent::wrapping21_params, wrapping21_values, + /*no deltas*/ 0); +} + +TEST_F(RtcEventFieldTest, EqualElements) { + std::vector bool_values = {true, true, true, true}; + std::vector signed32_values = {-2, -2, -2, -2}; + std::vector unsigned32_values = {123456789, 123456789, 123456789, + 123456789}; + std::vector signed64_values = {-9876543210, -9876543210, -9876543210, + -9876543210}; + std::vector unsigned64_values = {9876543210, 9876543210, 9876543210, + 9876543210}; + std::vector> optional32_values = { + kInt32Min, kInt32Min, kInt32Min, kInt32Min}; + std::vector> optional64_values = { + kInt64Max, kInt64Max, kInt64Max, kInt64Max}; + std::vector wrapping21_values = {(1 << 21) - 1, (1 << 21) - 1, + (1 << 21) - 1, (1 << 21) - 1}; + + CreateFullEvents(bool_values, signed32_values, unsigned32_values, + signed64_values, unsigned64_values, optional32_values, + optional64_values, wrapping21_values); + EventEncoder encoder(RtcTestEvent::event_params, batch_); + encoder.EncodeField(RtcTestEvent::bool_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::b_)); + encoder.EncodeField(RtcTestEvent::signed32_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::signed32_)); + encoder.EncodeField( + RtcTestEvent::unsigned32_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::unsigned32_)); + encoder.EncodeField(RtcTestEvent::signed64_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::signed64_)); + encoder.EncodeField( + RtcTestEvent::unsigned64_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::unsigned64_)); + encoder.EncodeField( + RtcTestEvent::optional32_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::optional_signed32_)); + encoder.EncodeField( + RtcTestEvent::optional64_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::optional_signed64_)); + encoder.EncodeField( + RtcTestEvent::wrapping21_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::wrapping21_)); + std::string s = encoder.AsString(); + + // Optional debug printing + // PrintBytes(s); + + ParseEventHeader(s); + ParseAndVerifyTimestamps(); + ParseAndVerifyField(RtcTestEvent::bool_params, bool_values, + /*no deltas*/ 0); + ParseAndVerifyField(RtcTestEvent::signed32_params, signed32_values, + /*no deltas*/ 0); + ParseAndVerifyField(RtcTestEvent::unsigned32_params, unsigned32_values, + /*no deltas*/ 0); + ParseAndVerifyField(RtcTestEvent::signed64_params, signed64_values, + /*no deltas*/ 0); + ParseAndVerifyField(RtcTestEvent::unsigned64_params, unsigned64_values, + /*no deltas*/ 0); + ParseAndVerifyOptionalField(RtcTestEvent::optional32_params, + optional32_values, /*no deltas*/ 0); + ParseAndVerifyOptionalField(RtcTestEvent::optional64_params, + optional64_values, /*no deltas*/ 0); + ParseAndVerifyField(RtcTestEvent::wrapping21_params, wrapping21_values, + /*no deltas*/ 0); +} + +TEST_F(RtcEventFieldTest, Increasing) { + std::vector bool_values = {false, true, false, true}; + std::vector signed32_values = {-2, -1, 0, 1}; + std::vector unsigned32_values = {kUint32Max - 1, kUint32Max, 0, 1}; + std::vector signed64_values = {kInt64Max - 1, kInt64Max, kInt64Min, + kInt64Min + 1}; + std::vector unsigned64_values = {kUint64Max - 1, kUint64Max, 0, 1}; + std::vector> optional32_values = { + kInt32Max - 1, kInt32Max, kInt32Min, kInt32Min + 1}; + std::vector> optional64_values = { + kInt64Max - 1, kInt64Max, kInt64Min, kInt64Min + 1}; + std::vector wrapping21_values = {(1 << 21) - 2, (1 << 21) - 1, 0, + 1}; + + CreateFullEvents(bool_values, signed32_values, unsigned32_values, + signed64_values, unsigned64_values, optional32_values, + optional64_values, wrapping21_values); + + EventEncoder encoder(RtcTestEvent::event_params, batch_); + encoder.EncodeField(RtcTestEvent::bool_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::b_)); + encoder.EncodeField(RtcTestEvent::signed32_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::signed32_)); + encoder.EncodeField( + RtcTestEvent::unsigned32_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::unsigned32_)); + encoder.EncodeField(RtcTestEvent::signed64_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::signed64_)); + encoder.EncodeField( + RtcTestEvent::unsigned64_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::unsigned64_)); + encoder.EncodeField( + RtcTestEvent::optional32_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::optional_signed32_)); + encoder.EncodeField( + RtcTestEvent::optional64_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::optional_signed64_)); + encoder.EncodeField( + RtcTestEvent::wrapping21_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::wrapping21_)); + std::string s = encoder.AsString(); + + // Optional debug printing + // PrintBytes(s); + + ParseEventHeader(s); + ParseAndVerifyTimestamps(); + ParseAndVerifyField(RtcTestEvent::bool_params, bool_values, + /*delta bits*/ 1); + ParseAndVerifyField(RtcTestEvent::signed32_params, signed32_values, + /*delta bits*/ 1); + ParseAndVerifyField(RtcTestEvent::unsigned32_params, unsigned32_values, + /*delta bits*/ 1); + ParseAndVerifyField(RtcTestEvent::signed64_params, signed64_values, + /*delta bits*/ 1); + ParseAndVerifyField(RtcTestEvent::unsigned64_params, unsigned64_values, + /*delta bits*/ 1); + ParseAndVerifyOptionalField(RtcTestEvent::optional32_params, + optional32_values, /*delta bits*/ 1); + ParseAndVerifyOptionalField(RtcTestEvent::optional64_params, + optional64_values, /*delta bits*/ 1); + ParseAndVerifyField(RtcTestEvent::wrapping21_params, wrapping21_values, + /*delta bits*/ 1); +} + +TEST_F(RtcEventFieldTest, Decreasing) { + std::vector bool_values = {true, false, true, false}; + std::vector signed32_values = {2, 1, 0, -1}; + std::vector unsigned32_values = {1, 0, kUint32Max, kUint32Max - 1}; + std::vector signed64_values = {kInt64Min + 1, kInt64Min, kInt64Max, + kInt64Max - 1}; + std::vector unsigned64_values = {1, 0, kUint64Max, kUint64Max - 1}; + std::vector> optional32_values = { + kInt32Min + 1, kInt32Min, kInt32Max, kInt32Max - 1}; + std::vector> optional64_values = { + kInt64Min + 1, kInt64Min, kInt64Max, kInt64Max - 1}; + std::vector wrapping21_values = {1, 0, (1 << 21) - 1, + (1 << 21) - 2}; + + CreateFullEvents(bool_values, signed32_values, unsigned32_values, + signed64_values, unsigned64_values, optional32_values, + optional64_values, wrapping21_values); + + EventEncoder encoder(RtcTestEvent::event_params, batch_); + encoder.EncodeField(RtcTestEvent::bool_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::b_)); + encoder.EncodeField(RtcTestEvent::signed32_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::signed32_)); + encoder.EncodeField( + RtcTestEvent::unsigned32_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::unsigned32_)); + encoder.EncodeField(RtcTestEvent::signed64_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::signed64_)); + encoder.EncodeField( + RtcTestEvent::unsigned64_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::unsigned64_)); + encoder.EncodeField( + RtcTestEvent::optional32_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::optional_signed32_)); + encoder.EncodeField( + RtcTestEvent::optional64_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::optional_signed64_)); + encoder.EncodeField( + RtcTestEvent::wrapping21_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::wrapping21_)); + std::string s = encoder.AsString(); + + // Optional debug printing + // PrintBytes(s); + + ParseEventHeader(s); + ParseAndVerifyTimestamps(); + ParseAndVerifyField(RtcTestEvent::bool_params, bool_values, + /*delta bits*/ 1); + ParseAndVerifyField(RtcTestEvent::signed32_params, signed32_values, + /*delta bits*/ 1); + ParseAndVerifyField(RtcTestEvent::unsigned32_params, unsigned32_values, + /*delta bits*/ 1); + ParseAndVerifyField(RtcTestEvent::signed64_params, signed64_values, + /*delta bits*/ 1); + ParseAndVerifyField(RtcTestEvent::unsigned64_params, unsigned64_values, + /*delta bits*/ 1); + ParseAndVerifyOptionalField(RtcTestEvent::optional32_params, + optional32_values, /*delta bits*/ 1); + ParseAndVerifyOptionalField(RtcTestEvent::optional64_params, + optional64_values, /*delta bits*/ 1); + ParseAndVerifyField(RtcTestEvent::wrapping21_params, wrapping21_values, + /*delta bits*/ 1); +} + +TEST_F(RtcEventFieldTest, SkipsDeprecatedFields) { + // Expect parser to skip fields it doesn't recognize, but find subsequent + // fields. + std::vector bool_values = {true, false}; + std::vector signed32_values = {kInt32Min / 2, kInt32Max / 2}; + std::vector unsigned32_values = {0, kUint32Max / 2}; + std::vector signed64_values = {kInt64Min / 2, kInt64Max / 2}; + std::vector unsigned64_values = {0, kUint64Max / 2}; + std::vector> optional32_values = {kInt32Max / 2, + kInt32Min / 2}; + std::vector> optional64_values = {kInt64Min / 2, + kInt64Max / 2}; + std::vector wrapping21_values = {0, 1 << 20}; + + size_t signed32_encoding_size = + /*tag*/ 1 + /* varint base*/ 5 + /* delta_header*/ 1 + /*deltas*/ 4; + size_t signed64_encoding_size = + /*tag*/ 1 + /* fixed64 base*/ 8 + /* delta_header*/ 1 + /*deltas*/ 8; + size_t optional32_encoding_size = + /*tag*/ 1 + /* fixed32 base*/ 4 + /* delta_header*/ 1 + /*deltas*/ 4; + + CreateFullEvents(bool_values, signed32_values, unsigned32_values, + signed64_values, unsigned64_values, optional32_values, + optional64_values, wrapping21_values); + + EventEncoder encoder(RtcTestEvent::event_params, batch_); + encoder.EncodeField(RtcTestEvent::bool_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::b_)); + encoder.EncodeField(RtcTestEvent::signed32_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::signed32_)); + encoder.EncodeField( + RtcTestEvent::unsigned32_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::unsigned32_)); + encoder.EncodeField(RtcTestEvent::signed64_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::signed64_)); + encoder.EncodeField( + RtcTestEvent::unsigned64_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::unsigned64_)); + encoder.EncodeField( + RtcTestEvent::optional32_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::optional_signed32_)); + encoder.EncodeField( + RtcTestEvent::optional64_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::optional_signed64_)); + encoder.EncodeField( + RtcTestEvent::wrapping21_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::wrapping21_)); + std::string s = encoder.AsString(); + + // Optional debug printing + // PrintBytes(s); + + ParseEventHeader(s); + ParseAndVerifyTimestamps(); + ParseAndVerifyField(RtcTestEvent::bool_params, bool_values, + /*delta_bits=*/1); + // Skips parsing the `signed32_values`. The following unsigned fields should + // still be found. + ParseAndVerifyField(RtcTestEvent::unsigned32_params, unsigned32_values, + /*delta_bits=*/31, signed32_encoding_size); + // Skips parsing the `signed64_values`. The following unsigned fields should + // still be found. + ParseAndVerifyField(RtcTestEvent::unsigned64_params, unsigned64_values, + /*delta_bits=*/63, signed64_encoding_size); + // Skips parsing the `optional32_values`. The following unsigned fields should + // still be found. + ParseAndVerifyOptionalField(RtcTestEvent::optional64_params, + optional64_values, + /*delta_bits=*/63, optional32_encoding_size); + ParseAndVerifyField(RtcTestEvent::wrapping21_params, wrapping21_values, + /*delta_bits=*/20); +} + +TEST_F(RtcEventFieldTest, SkipsMissingFields) { + // Expect parsing of missing field to succeed but return an empty list. + + std::vector bool_values = {true, false}; + std::vector signed32_values = {kInt32Min / 2, kInt32Max / 2}; + std::vector unsigned32_values = {0, kUint32Max / 2}; + std::vector signed64_values = {kInt64Min / 2, kInt64Max / 2}; + std::vector unsigned64_values = {0, kUint64Max / 2}; + std::vector> optional32_values = {kInt32Max / 2, + kInt32Min / 2}; + std::vector> optional64_values = {kInt64Min / 2, + kInt64Max / 2}; + std::vector wrapping21_values = {0, 1 << 20}; + + CreateFullEvents(bool_values, signed32_values, unsigned32_values, + signed64_values, unsigned64_values, optional32_values, + optional64_values, wrapping21_values); + + EventEncoder encoder(RtcTestEvent::event_params, batch_); + // Skip encoding the `bool_values`. + encoder.EncodeField(RtcTestEvent::signed32_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::signed32_)); + // Skip encoding the `unsigned32_values`. + encoder.EncodeField(RtcTestEvent::signed64_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::signed64_)); + // Skip encoding the `unsigned64_values`. + encoder.EncodeField( + RtcTestEvent::optional32_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::optional_signed32_)); + // Skip encoding the `optional64_values`. + encoder.EncodeField( + RtcTestEvent::wrapping21_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::wrapping21_)); + std::string s = encoder.AsString(); + + // Optional debug printing + // PrintBytes(s); + + ParseEventHeader(s); + ParseAndVerifyTimestamps(); + ParseAndVerifyMissingField(RtcTestEvent::bool_params); + ParseAndVerifyField(RtcTestEvent::signed32_params, signed32_values, + /*delta_bits=*/31); + ParseAndVerifyMissingField(RtcTestEvent::unsigned32_params); + ParseAndVerifyField(RtcTestEvent::signed64_params, signed64_values, + /*delta_bits=*/63); + ParseAndVerifyMissingField(RtcTestEvent::unsigned64_params); + ParseAndVerifyOptionalField(RtcTestEvent::optional32_params, + optional32_values, /*delta_bits=*/31); + ParseAndVerifyMissingOptionalField(RtcTestEvent::optional64_params); + ParseAndVerifyField(RtcTestEvent::wrapping21_params, wrapping21_values, + /*delta_bits=*/20); +} + +TEST_F(RtcEventFieldTest, OptionalFields) { + std::vector> optional32_values = { + 2, absl::nullopt, 4, absl::nullopt, 6, absl::nullopt}; + std::vector> optional64_values = { + absl::nullopt, 1024, absl::nullopt, 1025, absl::nullopt, 1026}; + std::vector wrapping21_values = {(1 << 21) - 3, 0, 2, 5, 5, 6}; + + for (size_t i = 0; i < optional32_values.size(); i++) { + batch_.push_back(new RtcTestEvent(0, 0, 0, 0, 0, optional32_values[i], + optional64_values[i], + wrapping21_values[i])); + } + + EventEncoder encoder(RtcTestEvent::event_params, batch_); + encoder.EncodeField( + RtcTestEvent::optional32_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::optional_signed32_)); + encoder.EncodeField( + RtcTestEvent::optional64_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::optional_signed64_)); + encoder.EncodeField( + RtcTestEvent::wrapping21_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::wrapping21_)); + std::string s = encoder.AsString(); + + // Optional debug output + // PrintBytes(s); + + ParseEventHeader(s); + ParseAndVerifyTimestamps(); + ParseAndVerifyOptionalField(RtcTestEvent::optional32_params, + optional32_values, /*delta bits*/ 2); + ParseAndVerifyOptionalField(RtcTestEvent::optional64_params, + optional64_values, /*delta bits*/ 1); + ParseAndVerifyField(RtcTestEvent::wrapping21_params, wrapping21_values, + /*delta bits*/ 2); +} + +TEST_F(RtcEventFieldTest, AllNulloptTreatedAsMissing) { + std::vector> optional32_values = { + absl::nullopt, absl::nullopt, absl::nullopt, + absl::nullopt, absl::nullopt, absl::nullopt}; + std::vector> optional64_values = { + absl::nullopt, 1024, absl::nullopt, 1025, absl::nullopt, 1026}; + + for (size_t i = 0; i < optional32_values.size(); i++) { + batch_.push_back(new RtcTestEvent(0, 0, 0, 0, 0, optional32_values[i], + optional64_values[i], 0)); + } + + EventEncoder encoder(RtcTestEvent::event_params, batch_); + encoder.EncodeField( + RtcTestEvent::optional32_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::optional_signed32_)); + encoder.EncodeField( + RtcTestEvent::optional64_params, + ExtractRtcEventMember(batch_, &RtcTestEvent::optional_signed64_)); + std::string s = encoder.AsString(); + + // Optional debug output + // PrintBytes(s); + + ParseEventHeader(s); + ParseAndVerifyTimestamps(); + ParseAndVerifyMissingOptionalField(RtcTestEvent::optional32_params); + ParseAndVerifyOptionalField(RtcTestEvent::optional64_params, + optional64_values, /*delta_bits=*/1); +} + +} // namespace webrtc