Add encoding for numeric RTC event fields.

Bug: webrtc:11933
Change-Id: I5fe98c6753547b2c096d8e97870a7f9ce90b7b8b
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/230703
Reviewed-by: Sebastian Jansson <srte@webrtc.org>
Commit-Queue: Björn Terelius <terelius@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#35126}
This commit is contained in:
Björn Terelius 2021-09-27 18:30:44 +02:00 committed by WebRTC LUCI CQ
parent a654e07d11
commit fe903d5eab
8 changed files with 1848 additions and 1 deletions

View File

@ -34,6 +34,12 @@ rtc_source_set("rtc_event_log_api") {
rtc_library("rtc_event_field") {
sources = [
"rtc_event_log/events/fixed_length_encoding_parameters_v3.cc",
"rtc_event_log/events/fixed_length_encoding_parameters_v3.h",
"rtc_event_log/events/rtc_event_field_encoding.cc",
"rtc_event_log/events/rtc_event_field_encoding.h",
"rtc_event_log/events/rtc_event_field_encoding_parser.cc",
"rtc_event_log/events/rtc_event_field_encoding_parser.h",
"rtc_event_log/events/rtc_event_field_extraction.cc",
"rtc_event_log/events/rtc_event_field_extraction.h",
]
@ -42,12 +48,12 @@ rtc_library("rtc_event_field") {
":rtc_event_number_encodings",
"../api:array_view",
"../api/rtc_event_log",
"../rtc_base:bitstream_reader",
"../rtc_base:checks",
"../rtc_base:logging",
"../rtc_base:rtc_base_approved",
]
absl_deps = [
"//third_party/abseil-cpp/absl/memory",
"//third_party/abseil-cpp/absl/strings",
"//third_party/abseil-cpp/absl/types:optional",
]
@ -414,6 +420,7 @@ if (rtc_enable_protobuf) {
"rtc_event_log/encoder/delta_encoding_unittest.cc",
"rtc_event_log/encoder/rtc_event_log_encoder_common_unittest.cc",
"rtc_event_log/encoder/rtc_event_log_encoder_unittest.cc",
"rtc_event_log/events/rtc_event_field_encoding_unittest.cc",
"rtc_event_log/events/rtc_event_field_extraction_unittest.cc",
"rtc_event_log/rtc_event_log_unittest.cc",
"rtc_event_log/rtc_event_log_unittest_helper.cc",

View File

@ -0,0 +1,137 @@
/*
* Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "logging/rtc_event_log/events/fixed_length_encoding_parameters_v3.h"
#include <algorithm>
#include "absl/types/optional.h"
#include "api/array_view.h"
#include "logging/rtc_event_log/events/rtc_event_field_extraction.h"
#include "rtc_base/logging.h"
using webrtc_event_logging::MaxUnsignedValueOfBitWidth;
using webrtc_event_logging::SignedBitWidth;
using webrtc_event_logging::UnsignedBitWidth;
using webrtc_event_logging::UnsignedDelta;
namespace webrtc {
FixedLengthEncodingParametersV3
FixedLengthEncodingParametersV3::CalculateParameters(
uint64_t base,
const rtc::ArrayView<const uint64_t> values,
uint64_t value_bit_width,
bool values_optional) {
// As a special case, if all of the elements are identical to the base
// we just encode the base value with a special delta header.
if (std::all_of(values.cbegin(), values.cend(),
[base](uint64_t val) { return val == base; })) {
// Delta header with signed=true and delta_bitwidth=64
return FixedLengthEncodingParametersV3(/*delta_bit_width=*/64,
/*signed_deltas=*/true,
values_optional, value_bit_width);
}
const uint64_t bit_mask = MaxUnsignedValueOfBitWidth(value_bit_width);
// Calculate the bitwidth required to encode all deltas when using a
// unsigned or signed represenation, respectively. For the unsigned
// representation, we just track the largest delta. For the signed
// representation, we have two possibilities for each delta; either
// going "forward" (i.e. current - previous) or "backwards"
// (i.e. previous - current) where both values are calculated with
// wrap around. We then track the largest positive and negative
// magnitude across the batch, assuming that we choose the smaller
// delta for each element.
uint64_t max_unsigned_delta = 0;
uint64_t max_positive_signed_delta = 0;
uint64_t min_negative_signed_delta = 0;
uint64_t prev = base;
for (uint64_t current : values) {
uint64_t positive_delta = UnsignedDelta(prev, current, bit_mask);
uint64_t negative_delta = UnsignedDelta(current, prev, bit_mask);
max_unsigned_delta = std::max(max_unsigned_delta, positive_delta);
if (positive_delta < negative_delta) {
max_positive_signed_delta =
std::max(max_positive_signed_delta, positive_delta);
} else {
min_negative_signed_delta =
std::max(min_negative_signed_delta, negative_delta);
}
prev = current;
}
// We now know the largest unsigned delta and the largest magnitudes of
// positive and negative signed deltas. Get the bitwidths required for
// each of the two encodings.
const uint64_t unsigned_delta_bit_width =
UnsignedBitWidth(max_unsigned_delta);
const uint64_t signed_delta_bit_width =
SignedBitWidth(max_positive_signed_delta, min_negative_signed_delta);
// Note: Preference for unsigned if the two have the same width (efficiency).
bool use_signed_deltas = signed_delta_bit_width < unsigned_delta_bit_width;
uint64_t delta_bit_width =
use_signed_deltas ? signed_delta_bit_width : unsigned_delta_bit_width;
// use_signed_deltas && delta_bit_width==64 is reserved for "all values
// equal".
RTC_DCHECK(!use_signed_deltas || delta_bit_width < 64);
RTC_DCHECK(ValidParameters(delta_bit_width, use_signed_deltas,
values_optional, value_bit_width));
return FixedLengthEncodingParametersV3(delta_bit_width, use_signed_deltas,
values_optional, value_bit_width);
}
uint64_t FixedLengthEncodingParametersV3::DeltaHeaderAsInt() const {
uint64_t header = delta_bit_width_ - 1;
RTC_CHECK_LT(header, 1u << 6);
if (signed_deltas_) {
header += 1u << 6;
}
RTC_CHECK_LT(header, 1u << 7);
if (values_optional_) {
header += 1u << 7;
}
return header;
}
absl::optional<FixedLengthEncodingParametersV3>
FixedLengthEncodingParametersV3::ParseDeltaHeader(uint64_t header,
uint64_t value_bit_width) {
uint64_t delta_bit_width = (header & ((1u << 6) - 1)) + 1;
bool signed_deltas = header & (1u << 6);
bool values_optional = header & (1u << 7);
if (header >= (1u << 8)) {
RTC_LOG(LS_ERROR) << "Failed to parse delta header; unread bits remaining.";
return absl::nullopt;
}
if (!ValidParameters(delta_bit_width, signed_deltas, values_optional,
value_bit_width)) {
RTC_LOG(LS_ERROR) << "Failed to parse delta header. Invalid combination of "
"values: delta_bit_width="
<< delta_bit_width << " signed_deltas=" << signed_deltas
<< " values_optional=" << values_optional
<< " value_bit_width=" << value_bit_width;
return absl::nullopt;
}
return FixedLengthEncodingParametersV3(delta_bit_width, signed_deltas,
values_optional, value_bit_width);
}
} // namespace webrtc

View File

@ -0,0 +1,96 @@
/*
* Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef LOGGING_RTC_EVENT_LOG_EVENTS_FIXED_LENGTH_ENCODING_PARAMETERS_V3_H_
#define LOGGING_RTC_EVENT_LOG_EVENTS_FIXED_LENGTH_ENCODING_PARAMETERS_V3_H_
#include "absl/types/optional.h"
#include "api/array_view.h"
#include "logging/rtc_event_log/events/rtc_event_field_extraction.h"
namespace webrtc {
// Parameters for fixed-size delta-encoding/decoding.
// These are tailored for the sequence which will be encoded (e.g. widths).
class FixedLengthEncodingParametersV3 final {
public:
static bool ValidParameters(uint64_t delta_bit_width,
bool signed_deltas,
bool values_optional,
uint64_t value_bit_width) {
return (1 <= delta_bit_width && delta_bit_width <= 64 &&
1 <= value_bit_width && value_bit_width <= 64 &&
(delta_bit_width <= value_bit_width ||
(signed_deltas && delta_bit_width == 64)));
}
static FixedLengthEncodingParametersV3 CalculateParameters(
uint64_t base,
const rtc::ArrayView<const uint64_t> values,
uint64_t value_bit_width,
bool values_optional);
static absl::optional<FixedLengthEncodingParametersV3> ParseDeltaHeader(
uint64_t header,
uint64_t value_bit_width);
uint64_t DeltaHeaderAsInt() const;
// Number of bits necessary to hold the widest(*) of the deltas between the
// values in the sequence.
// (*) - Widest might not be the largest, if signed deltas are used.
uint64_t delta_bit_width() const { return delta_bit_width_; }
// Whether deltas are signed.
bool signed_deltas() const { return signed_deltas_; }
// Whether the values of the sequence are optional. That is, it may be
// that some of them do not have a value (not even a sentinel value indicating
// invalidity).
bool values_optional() const { return values_optional_; }
// Whether all values are equal. 64-bit signed deltas are assumed to not
// occur, since those could equally well be represented using 64 bit unsigned
// deltas.
bool values_equal() const {
return delta_bit_width() == 64 && signed_deltas();
}
// Number of bits necessary to hold the largest value in the sequence.
uint64_t value_bit_width() const { return value_bit_width_; }
// Masks where only the bits relevant to the deltas/values are turned on.
uint64_t delta_mask() const { return delta_mask_; }
uint64_t value_mask() const { return value_mask_; }
private:
FixedLengthEncodingParametersV3(uint64_t delta_bit_width,
bool signed_deltas,
bool values_optional,
uint64_t value_bit_width)
: delta_bit_width_(delta_bit_width),
signed_deltas_(signed_deltas),
values_optional_(values_optional),
value_bit_width_(value_bit_width),
delta_mask_(
webrtc_event_logging::MaxUnsignedValueOfBitWidth(delta_bit_width_)),
value_mask_(webrtc_event_logging::MaxUnsignedValueOfBitWidth(
value_bit_width_)) {}
uint64_t delta_bit_width_;
bool signed_deltas_;
bool values_optional_;
uint64_t value_bit_width_;
uint64_t delta_mask_;
uint64_t value_mask_;
};
} // namespace webrtc
#endif // LOGGING_RTC_EVENT_LOG_EVENTS_FIXED_LENGTH_ENCODING_PARAMETERS_V3_H_

View File

@ -0,0 +1,265 @@
/*
* Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "logging/rtc_event_log/events/rtc_event_field_encoding.h"
#include <algorithm>
#include <limits>
#include <memory>
#include <utility>
#include "logging/rtc_event_log/encoder/bit_writer.h"
#include "logging/rtc_event_log/encoder/var_int.h"
#include "logging/rtc_event_log/events/rtc_event_field_extraction.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
using webrtc_event_logging::UnsignedDelta;
namespace {
std::string SerializeLittleEndian(uint64_t value, uint8_t bytes) {
RTC_DCHECK_LE(bytes, sizeof(uint64_t));
RTC_DCHECK_GE(bytes, 1);
if (bytes < sizeof(uint64_t)) {
// Note that shifting a 64-bit value by 64 (or more) bits is undefined.
RTC_DCHECK_EQ(value >> (8 * bytes), 0);
}
std::string output(bytes, 0);
// Getting a non-const pointer to the representation. See e.g.
// https://en.cppreference.com/w/cpp/string/basic_string:
// "The elements of a basic_string are stored contiguously,
// that is, [...] a pointer to s[0] can be passed to functions
// that expect a pointer to the first element of a null-terminated
// CharT[] array."
uint8_t* p = reinterpret_cast<uint8_t*>(&output[0]);
#ifdef WEBRTC_ARCH_LITTLE_ENDIAN
memcpy(p, &value, bytes);
#else
while (bytes > 0) {
*p = static_cast<uint8_t>(value & 0xFF);
value >>= 8;
++p;
--bytes;
}
#endif // WEBRTC_ARCH_LITTLE_ENDIAN
return output;
}
} // namespace
namespace webrtc {
std::string EncodeOptionalValuePositions(std::vector<bool> positions) {
BitWriter writer((positions.size() + 7) / 8);
for (bool position : positions) {
writer.WriteBits(position ? 1u : 0u, 1);
}
return writer.GetString();
}
std::string EncodeSingleValue(uint64_t value, FieldType field_type) {
switch (field_type) {
case FieldType::kFixed8:
return SerializeLittleEndian(value, /*bytes=*/1);
case FieldType::kFixed32:
return SerializeLittleEndian(value, /*bytes=*/4);
case FieldType::kFixed64:
return SerializeLittleEndian(value, /*bytes=*/8);
case FieldType::kVarInt:
return EncodeVarInt(value);
case FieldType::kString:
RTC_NOTREACHED();
return std::string();
}
}
absl::optional<FieldType> ConvertFieldType(uint64_t value) {
switch (value) {
case static_cast<uint64_t>(FieldType::kFixed8):
return FieldType::kFixed8;
case static_cast<uint64_t>(FieldType::kFixed32):
return FieldType::kFixed32;
case static_cast<uint64_t>(FieldType::kFixed64):
return FieldType::kFixed64;
case static_cast<uint64_t>(FieldType::kVarInt):
return FieldType::kVarInt;
case static_cast<uint64_t>(FieldType::kString):
return FieldType::kString;
default:
return absl::nullopt;
}
}
std::string EncodeDeltasV3(FixedLengthEncodingParametersV3 params,
uint64_t base,
rtc::ArrayView<const uint64_t> values) {
size_t outputbound = (values.size() * params.delta_bit_width() + 7) / 8;
BitWriter writer(outputbound);
uint64_t previous = base;
for (uint64_t value : values) {
if (params.signed_deltas()) {
uint64_t positive_delta =
UnsignedDelta(previous, value, params.value_mask());
uint64_t negative_delta =
UnsignedDelta(value, previous, params.value_mask());
uint64_t delta;
if (positive_delta <= negative_delta) {
delta = positive_delta;
} else {
// Compute the two's complement representation of a negative
// delta, in a field width params_.delta_mask().
RTC_DCHECK_GE(params.delta_mask(), negative_delta);
RTC_DCHECK_LT(params.delta_mask() - negative_delta,
params.delta_mask());
delta = params.delta_mask() - negative_delta + 1;
RTC_DCHECK_LE(delta, params.delta_mask());
}
writer.WriteBits(delta, params.delta_bit_width());
} else {
uint64_t delta = UnsignedDelta(previous, value, params.value_mask());
writer.WriteBits(delta, params.delta_bit_width());
}
previous = value;
}
return writer.GetString();
}
EventEncoder::EventEncoder(EventParameters params,
rtc::ArrayView<const RtcEvent*> batch) {
batch_size_ = batch.size();
if (!batch.empty()) {
// Encode event type.
uint32_t batched = batch.size() > 1 ? 1 : 0;
event_tag_ = (static_cast<uint32_t>(params.id) << 1) + batched;
// Event tag and number of encoded bytes will be filled in when the
// encoding is finalized in AsString().
// Encode number of events in batch
if (batched) {
encoded_fields_.push_back(EncodeVarInt(batch.size()));
}
// Encode timestamp
std::vector<uint64_t> timestamps;
timestamps.reserve(batch.size());
for (const RtcEvent* event : batch) {
timestamps.push_back(EncodeAsUnsigned(event->timestamp_ms()));
}
constexpr FieldParameters timestamp_params{"timestamp_ms",
FieldParameters::kTimestampField,
FieldType::kVarInt, 64};
EncodeField(timestamp_params, timestamps);
}
}
void EventEncoder::EncodeField(const FieldParameters& params,
const ValuesWithPositions& values) {
return EncodeField(params, values.values, &values.position_mask);
}
void EventEncoder::EncodeField(const FieldParameters& params,
const std::vector<uint64_t>& values,
const std::vector<bool>* positions) {
if (positions) {
RTC_DCHECK_EQ(positions->size(), batch_size_);
RTC_DCHECK_LE(values.size(), batch_size_);
} else {
RTC_DCHECK_EQ(values.size(), batch_size_);
}
if (values.size() == 0) {
// If all values for a particular field is empty/nullopt,
// then we completely skip the field even if the the batch is non-empty.
return;
}
// We know that each event starts with the varint encoded timestamp,
// so we omit that field tag (field id + field type). In all other
// cases, we write the field tag.
if (params.field_id != FieldParameters::kTimestampField) {
RTC_DCHECK_LE(params.field_id, std::numeric_limits<uint64_t>::max() >> 3);
uint64_t field_tag = params.field_id << 3;
field_tag += static_cast<uint64_t>(params.field_type);
encoded_fields_.push_back(EncodeVarInt(field_tag));
}
RTC_CHECK_GE(values.size(), 1);
if (batch_size_ == 1) {
encoded_fields_.push_back(EncodeSingleValue(values[0], params.field_type));
return;
}
const bool values_optional = values.size() != batch_size_;
// Compute delta parameters
rtc::ArrayView<const uint64_t> all_values(values);
uint64_t base = values[0];
rtc::ArrayView<const uint64_t> remaining_values(all_values.subview(1));
FixedLengthEncodingParametersV3 delta_params =
FixedLengthEncodingParametersV3::CalculateParameters(
base, remaining_values, params.value_width, values_optional);
encoded_fields_.push_back(EncodeVarInt(delta_params.DeltaHeaderAsInt()));
if (values_optional) {
RTC_CHECK(positions);
encoded_fields_.push_back(EncodeOptionalValuePositions(*positions));
}
// Base element, encoded as uint8, uint32, uint64 or varint
encoded_fields_.push_back(EncodeSingleValue(base, params.field_type));
// If all (existing) values are equal to the base, then we can skip
// writing the all-zero deltas, and instead infer those from the delta
// header.
if (!delta_params.values_equal()) {
encoded_fields_.push_back(
EncodeDeltasV3(delta_params, base, remaining_values));
}
}
std::string EventEncoder::AsString() {
std::string encoded_event;
if (batch_size_ == 0) {
RTC_DCHECK_EQ(encoded_fields_.size(), 0);
return encoded_event;
}
// Compute size of encoded fields.
size_t total_fields_size = 0;
for (const std::string& s : encoded_fields_) {
total_fields_size += s.size();
}
constexpr size_t kExpectedMaxEventTagBytes = 4;
constexpr size_t kExpectedMaxSizeEncodingBytes = 4;
encoded_event.reserve(kExpectedMaxEventTagBytes +
kExpectedMaxSizeEncodingBytes + total_fields_size);
// Encode event tag (event id and whether batch or single event).
encoded_event.append(EncodeVarInt(event_tag_));
// Encode size of the remaining fields.
encoded_event.append(EncodeVarInt(total_fields_size));
// Append encoded fields.
for (const std::string& s : encoded_fields_) {
encoded_event.append(s);
}
return encoded_event;
}
} // namespace webrtc

View File

@ -0,0 +1,94 @@
/*
* Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef LOGGING_RTC_EVENT_LOG_EVENTS_RTC_EVENT_FIELD_ENCODING_H_
#define LOGGING_RTC_EVENT_LOG_EVENTS_RTC_EVENT_FIELD_ENCODING_H_
#include <string>
#include <vector>
#include "absl/types/optional.h"
#include "api/array_view.h"
#include "api/rtc_event_log/rtc_event.h"
#include "logging/rtc_event_log/encoder/rtc_event_log_encoder_common.h"
#include "logging/rtc_event_log/events/fixed_length_encoding_parameters_v3.h"
#include "logging/rtc_event_log/events/rtc_event_field_extraction.h"
#include "rtc_base/logging.h"
namespace webrtc {
// To maintain backwards compatibility with past (or future) logs,
// the constants in this enum must not be changed.
// New field types with numerical IDs 5-7 can be added, but old
// parsers will fail to parse events containing the new fields.
enum class FieldType : uint8_t {
kFixed8 = 0,
kFixed32 = 1,
kFixed64 = 2,
kVarInt = 3,
kString = 4,
};
// EventParameters map an event name to a numerical ID.
struct EventParameters {
// The name is primarily used for debugging purposes.
const char* const name;
//
const RtcEvent::Type id;
};
// FieldParameters define the encoding for a field.
struct FieldParameters {
// The name is primarily used for debugging purposes.
const char* const name;
// Numerical ID for the field. Must be strictly greater than 0,
// and unique within each event type.
const uint64_t field_id;
// Encoding type for the base (i.e. non-delta) field in a batch.
const FieldType field_type;
// Number of bits after which wrap-around occurs. In most cases,
// this should be the number of bits in the field data type, i.e.
// 8 for an uint8_t, 32 for a int32_t and so on. However, `value_width`
// can be used to achieve a more efficient encoding if it is known
// that the field uses a smaller number of bits. For example, a
// 15-bit counter could set `value_width` to 15 even if the data is
// actually stored in a uint32_t.
const uint64_t value_width;
// Field ID 0 is reserved for timestamps.
static constexpr uint64_t kTimestampField = 0;
};
// The EventEncoder is used to encode a batch of events.
class EventEncoder {
public:
EventEncoder(EventParameters params, rtc::ArrayView<const RtcEvent*> batch);
void EncodeField(const FieldParameters& params,
const std::vector<uint64_t>& values,
const std::vector<bool>* positions = nullptr);
void EncodeField(const FieldParameters& params,
const ValuesWithPositions& values);
std::string AsString();
private:
size_t batch_size_;
uint32_t event_tag_;
std::vector<std::string> encoded_fields_;
};
std::string EncodeSingleValue(uint64_t value, FieldType field_type);
std::string EncodeDeltasV3(FixedLengthEncodingParametersV3 params,
uint64_t base,
rtc::ArrayView<const uint64_t> values);
} // namespace webrtc
#endif // LOGGING_RTC_EVENT_LOG_EVENTS_RTC_EVENT_FIELD_ENCODING_H_

View File

@ -0,0 +1,338 @@
/*
* Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "logging/rtc_event_log/events/rtc_event_field_encoding_parser.h"
#include "absl/strings/string_view.h"
#include "absl/types/optional.h"
#include "logging/rtc_event_log/encoder/var_int.h"
#include "logging/rtc_event_log/events/rtc_event_field_encoding.h"
#include "rtc_base/bitstream_reader.h"
#include "rtc_base/checks.h"
namespace {
absl::optional<webrtc::FieldType> ConvertFieldType(uint64_t value) {
switch (value) {
case static_cast<uint64_t>(webrtc::FieldType::kFixed8):
return webrtc::FieldType::kFixed8;
case static_cast<uint64_t>(webrtc::FieldType::kFixed32):
return webrtc::FieldType::kFixed32;
case static_cast<uint64_t>(webrtc::FieldType::kFixed64):
return webrtc::FieldType::kFixed64;
case static_cast<uint64_t>(webrtc::FieldType::kVarInt):
return webrtc::FieldType::kVarInt;
case static_cast<uint64_t>(webrtc::FieldType::kString):
return webrtc::FieldType::kString;
default:
return absl::nullopt;
}
}
} // namespace
namespace webrtc {
uint64_t EventParser::ReadLittleEndian(uint8_t bytes) {
RTC_DCHECK_LE(bytes, sizeof(uint64_t));
RTC_DCHECK_GE(bytes, 1);
uint64_t value = 0;
if (bytes > pending_data_.length()) {
SetError();
return value;
}
const uint8_t* p = reinterpret_cast<const uint8_t*>(pending_data_.data());
unsigned int shift = 0;
uint8_t remaining = bytes;
while (remaining > 0) {
value += (static_cast<uint64_t>(*p) << shift);
shift += 8;
++p;
--remaining;
}
pending_data_ = pending_data_.substr(bytes);
return value;
}
uint64_t EventParser::ReadVarInt() {
uint64_t output = 0;
bool success;
std::tie(success, pending_data_) = DecodeVarInt(pending_data_, &output);
if (!success) {
SetError();
}
return output;
}
uint64_t EventParser::ReadOptionalValuePositions(std::vector<bool>* positions) {
if (!positions) {
return CountAndIgnoreOptionalValuePositions();
}
size_t bits_to_read = NumEventsInBatch();
RTC_DCHECK(positions->empty());
if (pending_data_.size() * 8 < bits_to_read) {
SetError();
return 0;
}
BitstreamReader reader(pending_data_);
for (size_t i = 0; i < bits_to_read; i++) {
positions->push_back(reader.ReadBit());
}
if (!reader.Ok()) {
SetError();
return 0;
}
size_t num_existing_values =
std::count(positions->begin(), positions->end(), true);
pending_data_ = pending_data_.substr((bits_to_read + 7) / 8);
return num_existing_values;
}
uint64_t EventParser::CountAndIgnoreOptionalValuePositions() {
size_t bits_to_read = NumEventsInBatch();
if (pending_data_.size() * 8 < bits_to_read) {
SetError();
return 0;
}
BitstreamReader reader(pending_data_);
size_t num_existing_values = 0;
for (size_t i = 0; i < bits_to_read; i++) {
if (reader.ReadBit()) {
++num_existing_values;
}
}
if (!reader.Ok()) {
SetError();
return 0;
}
pending_data_ = pending_data_.substr((bits_to_read + 7) / 8);
return num_existing_values;
}
uint64_t EventParser::ReadSingleValue(FieldType field_type) {
switch (field_type) {
case FieldType::kFixed8:
return ReadLittleEndian(/*bytes=*/1);
case FieldType::kFixed32:
return ReadLittleEndian(/*bytes=*/4);
case FieldType::kFixed64:
return ReadLittleEndian(/*bytes=*/8);
case FieldType::kVarInt:
return ReadVarInt();
case FieldType::kString:
RTC_NOTREACHED();
SetError();
return 0;
}
}
void EventParser::ReadDeltasAndPopulateValues(
FixedLengthEncodingParametersV3 params,
uint64_t num_deltas,
uint64_t base,
std::vector<uint64_t>* values) {
RTC_CHECK(values != nullptr);
RTC_DCHECK(values->empty());
values->reserve(num_deltas + 1);
values->push_back(base);
if (pending_data_.size() * 8 < num_deltas * params.delta_bit_width()) {
SetError();
return;
}
BitstreamReader reader(pending_data_);
const uint64_t top_bit = static_cast<uint64_t>(1)
<< (params.delta_bit_width() - 1);
uint64_t value = base;
for (uint64_t i = 0; i < num_deltas; ++i) {
uint64_t delta = reader.ReadBits(params.delta_bit_width());
RTC_DCHECK_LE(value, webrtc_event_logging::MaxUnsignedValueOfBitWidth(
params.value_bit_width()));
RTC_DCHECK_LE(delta, webrtc_event_logging::MaxUnsignedValueOfBitWidth(
params.delta_bit_width()));
bool negative_delta = params.signed_deltas() && ((delta & top_bit) != 0);
if (negative_delta) {
uint64_t delta_abs = (~delta & params.delta_mask()) + 1;
value = (value - delta_abs) & params.value_mask();
} else {
value = (value + delta) & params.value_mask();
}
values->push_back(value);
}
if (!reader.Ok()) {
SetError();
return;
}
pending_data_ =
pending_data_.substr((num_deltas * params.delta_bit_width() + 7) / 8);
}
RtcEventLogParseStatus EventParser::Initialize(absl::string_view s,
bool batched) {
pending_data_ = s;
num_events_ = 1;
if (batched) {
num_events_ = ReadVarInt();
if (!Ok()) {
return RtcEventLogParseStatus::Error(
"Failed to read number of events in batch.", __FILE__, __LINE__);
}
}
return RtcEventLogParseStatus::Success();
}
RtcEventLogParseStatus EventParser::ParseField(const FieldParameters& params,
std::vector<uint64_t>* values,
std::vector<bool>* positions) {
RTC_CHECK(values != nullptr);
// Verify that the event parses fields in increasing order.
if (params.field_id == FieldParameters::kTimestampField) {
RTC_DCHECK_EQ(last_field_id_, FieldParameters::kTimestampField);
} else {
RTC_DCHECK_GT(params.field_id, last_field_id_);
}
last_field_id_ = params.field_id;
// Initialization for positional fields that don't encode field ID and type.
uint64_t field_id = params.field_id;
FieldType field_type = params.field_type;
// Fields are encoded in increasing field_id order.
// Skip unknown fields with field_id < params.field_id until we either
// find params.field_id or a field with higher id, in which case we know that
// params.field_id doesn't exist.
while (!pending_data_.empty()) {
absl::string_view field_start = pending_data_;
values->clear();
if (positions) {
positions->clear();
}
// Read tag for non-positional fields.
if (params.field_id != FieldParameters::kTimestampField) {
uint64_t field_tag = ReadVarInt();
if (!Ok())
return RtcEventLogParseStatus::Error("Failed to read field tag",
__FILE__, __LINE__);
// Split tag into field ID and field type.
field_id = field_tag >> 3;
absl::optional<FieldType> conversion = ConvertFieldType(field_tag & 7u);
if (!conversion.has_value())
return RtcEventLogParseStatus::Error("Failed to parse field type",
__FILE__, __LINE__);
field_type = conversion.value();
}
if (field_id > params.field_id) {
// We've passed all fields with ids less than or equal to what we are
// looking for. Reset pending_data_ to first field with id higher than
// params.field_id, since we didn't find the field we were looking for.
pending_data_ = field_start;
return RtcEventLogParseStatus::Success();
}
if (num_events_ == 1) {
// Just a single value in the batch.
uint64_t base = ReadSingleValue(field_type);
if (!Ok())
return RtcEventLogParseStatus::Error("Failed to read value", __FILE__,
__LINE__);
if (positions) {
positions->push_back(true);
}
values->push_back(base);
} else {
// Delta compressed batch.
// Read delta header.
uint64_t header_value = ReadVarInt();
if (!Ok())
return RtcEventLogParseStatus::Error("Failed to read delta header",
__FILE__, __LINE__);
// NB: value_width may be incorrect for the field, if this isn't the field
// we are looking for.
absl::optional<FixedLengthEncodingParametersV3> delta_header =
FixedLengthEncodingParametersV3::ParseDeltaHeader(header_value,
params.value_width);
if (!delta_header.has_value()) {
return RtcEventLogParseStatus::Error("Failed to parse delta header",
__FILE__, __LINE__);
}
uint64_t num_existing_deltas = NumEventsInBatch() - 1;
if (delta_header->values_optional()) {
size_t num_nonempty_values = ReadOptionalValuePositions(positions);
if (!Ok()) {
return RtcEventLogParseStatus::Error(
"Failed to read positions of optional values", __FILE__,
__LINE__);
}
if (num_nonempty_values < 1 ||
NumEventsInBatch() < num_nonempty_values) {
return RtcEventLogParseStatus::Error(
"Expected at least one non_empty value", __FILE__, __LINE__);
}
num_existing_deltas = num_nonempty_values - 1;
} else {
// All elements in the batch have values.
if (positions) {
positions->assign(NumEventsInBatch(), true);
}
}
// Read base.
uint64_t base = ReadSingleValue(field_type);
if (!Ok()) {
return RtcEventLogParseStatus::Error("Failed to read value", __FILE__,
__LINE__);
}
if (delta_header->values_equal()) {
// Duplicate the base value num_existing_deltas times.
values->assign(num_existing_deltas + 1, base);
} else {
// Read deltas; ceil(num_existing_deltas*delta_width/8) bits
ReadDeltasAndPopulateValues(delta_header.value(), num_existing_deltas,
base, values);
if (!Ok()) {
return RtcEventLogParseStatus::Error("Failed to decode deltas",
__FILE__, __LINE__);
}
}
}
if (field_id == params.field_id) {
// The field we're looking for has been found and values populated.
return RtcEventLogParseStatus::Success();
}
}
// Field not found because the event ended.
values->clear();
if (positions) {
positions->clear();
}
return RtcEventLogParseStatus::Success();
}
} // namespace webrtc

View File

@ -0,0 +1,104 @@
/*
* Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef LOGGING_RTC_EVENT_LOG_EVENTS_RTC_EVENT_FIELD_ENCODING_PARSER_H_
#define LOGGING_RTC_EVENT_LOG_EVENTS_RTC_EVENT_FIELD_ENCODING_PARSER_H_
#include <string>
#include <vector>
#include "logging/rtc_event_log/events/rtc_event_field_encoding.h"
// TODO(terelius): Compared to a generic 'Status' class, this
// class allows us additional information about the context
// in which the error occurred. This is currently limited to
// the source location (file and line), but we plan on adding
// information about the event and field name being parsed.
// If/when we start using absl::Status in WebRTC, consider
// whether payloads would be an appropriate alternative.
class RtcEventLogParseStatus {
public:
static RtcEventLogParseStatus Success() { return RtcEventLogParseStatus(); }
static RtcEventLogParseStatus Error(std::string error,
std::string file,
int line) {
return RtcEventLogParseStatus(error, file, line);
}
bool ok() const { return error_.empty(); }
std::string message() const { return error_; }
private:
RtcEventLogParseStatus() : error_() {}
RtcEventLogParseStatus(std::string error, std::string file, int line)
: error_(error + " (" + file + ": " + std::to_string(line) + ")") {}
std::string error_;
};
namespace webrtc {
class EventParser {
public:
EventParser() = default;
// N.B: This method stores a abls::string_view into the string to be
// parsed. The caller is responsible for ensuring that the actual string
// remains unmodified and outlives the EventParser.
RtcEventLogParseStatus Initialize(absl::string_view s, bool batched);
// Attempts to parse the field specified by `params`, skipping past
// other fields that may occur before it. Returns
// RtcEventLogParseStatus::Success() and populates `values` (and `positions`)
// if the field is found. Returns RtcEventLogParseStatus::Success() and clears
// `values` (and `positions`) if the field doesn't exist. Returns a
// RtcEventLogParseStatus::Error if the log is incomplete, malformed or
// otherwise can't be parsed. `values` and `positions` are pure out-parameters
// that allow the caller to reuse the same temporary storage for all fields.
// Any previous content in the out parameters is cleared.
RtcEventLogParseStatus ParseField(const FieldParameters& params,
std::vector<uint64_t>* values,
std::vector<bool>* positions = nullptr);
// Number of events in a batch.
uint64_t NumEventsInBatch() const { return num_events_; }
// Bytes remaining in `pending_data_`. Assuming there are no unknown
// fields, BytesRemaining() should return 0 when all known fields
// in the event have been parsed.
size_t RemainingBytes() const { return pending_data_.size(); }
private:
uint64_t ReadLittleEndian(uint8_t bytes);
uint64_t ReadVarInt();
uint64_t ReadSingleValue(FieldType field_type);
uint64_t ReadOptionalValuePositions(std::vector<bool>* positions);
uint64_t CountAndIgnoreOptionalValuePositions();
void ReadDeltasAndPopulateValues(FixedLengthEncodingParametersV3 params,
uint64_t num_deltas,
const uint64_t base,
std::vector<uint64_t>* values);
void SetError() { error_ = true; }
bool Ok() const { return !error_; }
// String to be consumed.
absl::string_view pending_data_;
// Tracks whether an error has occurred in one of the helper
// functions above.
bool error_ = false;
uint64_t num_events_ = 1;
uint64_t last_field_id_ = FieldParameters::kTimestampField;
};
} // namespace webrtc
#endif // LOGGING_RTC_EVENT_LOG_EVENTS_RTC_EVENT_FIELD_ENCODING_PARSER_H_

View File

@ -0,0 +1,806 @@
/* Copyright (c) 2021 The WebRTC Project Authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "logging/rtc_event_log/events/rtc_event_field_encoding.h"
#include <limits>
#include <memory>
#include <string>
#include "api/rtc_event_log/rtc_event.h"
#include "logging/rtc_event_log/encoder/var_int.h"
#include "logging/rtc_event_log/events/rtc_event_field_encoding_parser.h"
#include "test/gtest.h"
namespace webrtc {
namespace {
constexpr int32_t kInt32Max = std::numeric_limits<int32_t>::max();
constexpr int32_t kInt32Min = std::numeric_limits<int32_t>::min();
constexpr uint32_t kUint32Max = std::numeric_limits<uint32_t>::max();
constexpr int64_t kInt64Max = std::numeric_limits<int64_t>::max();
constexpr int64_t kInt64Min = std::numeric_limits<int64_t>::min();
constexpr uint64_t kUint64Max = std::numeric_limits<uint64_t>::max();
template <typename T, std::enable_if_t<std::is_integral<T>::value, bool> = true>
size_t ExpectedVarIntSize(T value) {
size_t bytes = 0;
uint64_t x = EncodeAsUnsigned(value);
do {
++bytes;
x = x >> 7;
} while (x > 0);
return bytes;
}
template <typename T, std::enable_if_t<std::is_integral<T>::value, bool> = true>
size_t ExpectedBaseValueSize(const FieldParameters& params, T value) {
switch (params.field_type) {
case FieldType::kFixed8:
return 1;
case FieldType::kFixed32:
return 4;
case FieldType::kFixed64:
return 8;
case FieldType::kVarInt:
return ExpectedVarIntSize(value);
default:
break;
}
RTC_NOTREACHED();
return 0;
}
template <typename T, std::enable_if_t<std::is_integral<T>::value, bool> = true>
size_t ExpectedEncodingSize(const FieldParameters& params,
const std::vector<T>& v,
size_t expected_bits_per_delta) {
if (v.size() == 0)
return 0;
uint64_t numeric_field_type = static_cast<uint64_t>(params.field_type);
RTC_DCHECK_LT(numeric_field_type, 1u << 3);
size_t tag_size =
ExpectedVarIntSize((params.field_id << 3) + numeric_field_type);
T base = v[0];
size_t base_size = ExpectedBaseValueSize(params, base);
if (v.size() == 1)
return tag_size + base_size;
size_t delta_header_size = 1;
// Check if there is an element *not* equal to base.
if (std::all_of(v.begin(), v.end(), [base](T x) { return x == base; })) {
return tag_size + base_size + delta_header_size;
}
size_t delta_size = ((v.size() - 1) * expected_bits_per_delta + 7) / 8;
return tag_size + base_size + delta_header_size + delta_size;
}
template <typename T, std::enable_if_t<std::is_integral<T>::value, bool> = true>
size_t ExpectedEncodingSize(const FieldParameters& params,
const std::vector<absl::optional<T>>& v,
size_t expected_bits_per_delta) {
size_t num_existing_values =
v.size() - std::count(v.begin(), v.end(), absl::nullopt);
auto first_existing_value = std::find_if(
v.begin(), v.end(), [](absl::optional<T> x) { return x.has_value(); });
if (num_existing_values == 0)
return 0;
uint64_t numeric_field_type = static_cast<uint64_t>(params.field_type);
RTC_DCHECK_LT(numeric_field_type, 1u << 3);
size_t tag_size =
ExpectedVarIntSize((params.field_id << 3) + numeric_field_type);
T base = first_existing_value->value();
size_t base_size = ExpectedBaseValueSize(params, base);
if (num_existing_values == 1 && v.size() == 1)
return tag_size + base_size;
size_t delta_header_size = (num_existing_values == v.size() ? 1 : 2);
size_t positions_size =
(num_existing_values == v.size() ? 0 : (v.size() + 7) / 8);
// Check if there is an element *not* equal to base.
if (std::all_of(v.begin(), v.end(),
[base](absl::optional<T> x) { return x == base; })) {
return tag_size + base_size + delta_header_size + positions_size;
}
size_t delta_size =
((num_existing_values - 1) * expected_bits_per_delta + 7) / 8;
return tag_size + base_size + delta_header_size + positions_size + delta_size;
}
} // namespace
class RtcTestEvent final : public RtcEvent {
public:
RtcTestEvent(bool b,
int32_t signed32,
uint32_t unsigned32,
int64_t signed64,
uint64_t unsigned64)
: b_(b),
signed32_(signed32),
unsigned32_(unsigned32),
signed64_(signed64),
unsigned64_(unsigned64) {}
RtcTestEvent(bool b,
int32_t signed32,
uint32_t unsigned32,
int64_t signed64,
uint64_t unsigned64,
absl::optional<int32_t> optional_signed32,
absl::optional<int64_t> optional_signed64,
uint32_t wrapping21)
: b_(b),
signed32_(signed32),
unsigned32_(unsigned32),
signed64_(signed64),
unsigned64_(unsigned64),
optional_signed32_(optional_signed32),
optional_signed64_(optional_signed64),
wrapping21_(wrapping21) {}
~RtcTestEvent() override = default;
Type GetType() const override { return static_cast<Type>(4711); }
bool IsConfigEvent() const override { return false; }
static constexpr EventParameters event_params{
"TestEvent", static_cast<RtcEvent::Type>(4711)};
static constexpr FieldParameters timestamp_params{
"timestamp_ms", FieldParameters::kTimestampField, FieldType::kVarInt, 64};
static constexpr FieldParameters bool_params{"b", 2, FieldType::kFixed8, 1};
static constexpr FieldParameters signed32_params{"signed32", 3,
FieldType::kVarInt, 32};
static constexpr FieldParameters unsigned32_params{"unsigned32", 4,
FieldType::kFixed32, 32};
static constexpr FieldParameters signed64_params{"signed64", 5,
FieldType::kFixed64, 64};
static constexpr FieldParameters unsigned64_params{"unsigned64", 6,
FieldType::kVarInt, 64};
static constexpr FieldParameters optional32_params{"optional_signed32", 7,
FieldType::kFixed32, 32};
static constexpr FieldParameters optional64_params{"optional_signed64", 8,
FieldType::kVarInt, 64};
static constexpr FieldParameters wrapping21_params{"wrapping21", 9,
FieldType::kFixed32, 21};
static constexpr Type kType = static_cast<RtcEvent::Type>(4711);
const bool b_;
const int32_t signed32_;
const uint32_t unsigned32_;
const int64_t signed64_;
const uint64_t unsigned64_;
const absl::optional<int32_t> optional_signed32_ = absl::nullopt;
const absl::optional<int64_t> optional_signed64_ = absl::nullopt;
const uint32_t wrapping21_ = 0;
};
constexpr EventParameters RtcTestEvent::event_params;
constexpr FieldParameters RtcTestEvent::timestamp_params;
constexpr FieldParameters RtcTestEvent::bool_params;
constexpr FieldParameters RtcTestEvent::signed32_params;
constexpr FieldParameters RtcTestEvent::unsigned32_params;
constexpr FieldParameters RtcTestEvent::signed64_params;
constexpr FieldParameters RtcTestEvent::unsigned64_params;
constexpr FieldParameters RtcTestEvent::optional32_params;
constexpr FieldParameters RtcTestEvent::optional64_params;
constexpr FieldParameters RtcTestEvent::wrapping21_params;
constexpr RtcEvent::Type RtcTestEvent::kType;
class RtcEventFieldTest : public ::testing::Test {
protected:
void SetUp() override {}
void CreateFullEvents(
const std::vector<bool>& bool_values,
const std::vector<int32_t>& signed32_values,
const std::vector<uint32_t>& unsigned32_values,
const std::vector<int64_t>& signed64_values,
const std::vector<uint64_t>& unsigned64_values,
const std::vector<absl::optional<int32_t>>& optional32_values,
const std::vector<absl::optional<int64_t>>& optional64_values,
const std::vector<uint32_t>& wrapping21_values) {
size_t size = bool_values.size();
RTC_CHECK_EQ(signed32_values.size(), size);
RTC_CHECK_EQ(unsigned32_values.size(), size);
RTC_CHECK_EQ(signed64_values.size(), size);
RTC_CHECK_EQ(unsigned64_values.size(), size);
RTC_CHECK_EQ(optional32_values.size(), size);
RTC_CHECK_EQ(optional64_values.size(), size);
RTC_CHECK_EQ(wrapping21_values.size(), size);
for (size_t i = 0; i < size; i++) {
batch_.push_back(new RtcTestEvent(
bool_values[i], signed32_values[i], unsigned32_values[i],
signed64_values[i], unsigned64_values[i], optional32_values[i],
optional64_values[i], wrapping21_values[i]));
}
}
void PrintBytes(const std::string& s) {
for (auto c : s) {
fprintf(stderr, "%d ", static_cast<uint8_t>(c));
}
fprintf(stderr, "\n");
}
void ParseEventHeader(absl::string_view encoded_event) {
uint64_t event_tag;
bool success;
std::tie(success, encoded_event) = DecodeVarInt(encoded_event, &event_tag);
ASSERT_TRUE(success);
uint64_t event_id = event_tag >> 1;
ASSERT_EQ(event_id, static_cast<uint64_t>(RtcTestEvent::event_params.id));
bool batched = event_tag & 1u;
ASSERT_EQ(batched, batch_.size() > 1u);
uint64_t size;
std::tie(success, encoded_event) = DecodeVarInt(encoded_event, &size);
ASSERT_EQ(encoded_event.size(), size);
ASSERT_TRUE(parser_.Initialize(encoded_event, batched).ok());
}
void ParseAndVerifyTimestamps() {
std::vector<uint64_t> values;
auto status = parser_.ParseField(RtcTestEvent::timestamp_params, &values);
ASSERT_TRUE(status.ok()) << status.message().c_str();
ASSERT_EQ(values.size(), batch_.size());
for (size_t i = 0; i < batch_.size(); i++) {
EXPECT_EQ(values[i], static_cast<uint64_t>(batch_[i]->timestamp_ms()));
}
}
template <typename T>
void ParseAndVerifyField(const FieldParameters& params,
const std::vector<T>& expected_values,
size_t expected_bits_per_delta,
size_t expected_skipped_bytes = 0) {
size_t expected_size =
ExpectedEncodingSize(params, expected_values, expected_bits_per_delta) +
expected_skipped_bytes;
std::vector<uint64_t> values;
size_t size_before = parser_.RemainingBytes();
auto status = parser_.ParseField(params, &values);
ASSERT_TRUE(status.ok()) << status.message().c_str();
ASSERT_EQ(values.size(), expected_values.size());
for (size_t i = 0; i < expected_values.size(); i++) {
EXPECT_EQ(DecodeFromUnsignedToType<T>(values[i]), expected_values[i]);
}
size_t size_after = parser_.RemainingBytes();
EXPECT_EQ(size_before - size_after, expected_size)
<< " for field " << params.name;
}
template <typename T>
void ParseAndVerifyOptionalField(
const FieldParameters& params,
const std::vector<absl::optional<T>>& expected_values,
size_t expected_bits_per_delta,
size_t expected_skipped_bytes = 0) {
size_t expected_size =
ExpectedEncodingSize(params, expected_values, expected_bits_per_delta) +
expected_skipped_bytes;
std::vector<bool> positions;
positions.reserve(expected_values.size());
std::vector<uint64_t> values;
values.reserve(expected_values.size());
size_t size_before = parser_.RemainingBytes();
auto status = parser_.ParseField(params, &values, &positions);
ASSERT_TRUE(status.ok()) << status.message().c_str();
auto value_it = values.begin();
ASSERT_EQ(positions.size(), expected_values.size());
for (size_t i = 0; i < expected_values.size(); i++) {
if (positions[i]) {
ASSERT_NE(value_it, values.end());
ASSERT_TRUE(expected_values[i].has_value());
EXPECT_EQ(DecodeFromUnsignedToType<T>(*value_it),
expected_values[i].value());
++value_it;
} else {
EXPECT_EQ(absl::nullopt, expected_values[i]);
}
}
EXPECT_EQ(value_it, values.end());
size_t size_after = parser_.RemainingBytes();
EXPECT_EQ(size_before - size_after, expected_size);
}
void ParseAndVerifyMissingField(const FieldParameters& params) {
std::vector<uint64_t> values{4711};
auto status = parser_.ParseField(params, &values);
ASSERT_TRUE(status.ok()) << status.message().c_str();
EXPECT_EQ(values.size(), 0u);
}
void ParseAndVerifyMissingOptionalField(const FieldParameters& params) {
std::vector<bool> positions{true, false};
std::vector<uint64_t> values{4711};
auto status = parser_.ParseField(params, &values, &positions);
ASSERT_TRUE(status.ok()) << status.message().c_str();
EXPECT_EQ(positions.size(), 0u);
EXPECT_EQ(values.size(), 0u);
}
void TearDown() override {
for (const RtcEvent* event : batch_) {
delete event;
}
}
std::vector<const RtcEvent*> batch_;
EventParser parser_;
};
TEST_F(RtcEventFieldTest, EmptyList) {
EventEncoder encoder(RtcTestEvent::event_params, batch_);
encoder.EncodeField(RtcTestEvent::bool_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::b_));
std::string s = encoder.AsString();
EXPECT_TRUE(s.empty());
}
TEST_F(RtcEventFieldTest, Singleton) {
std::vector<bool> bool_values = {true};
std::vector<int32_t> signed32_values = {-2};
std::vector<uint32_t> unsigned32_values = {123456789};
std::vector<int64_t> signed64_values = {-9876543210};
std::vector<uint64_t> unsigned64_values = {9876543210};
std::vector<absl::optional<int32_t>> optional32_values = {kInt32Min};
std::vector<absl::optional<int64_t>> optional64_values = {kInt64Max};
std::vector<uint32_t> wrapping21_values = {(1 << 21) - 1};
CreateFullEvents(bool_values, signed32_values, unsigned32_values,
signed64_values, unsigned64_values, optional32_values,
optional64_values, wrapping21_values);
EventEncoder encoder(RtcTestEvent::event_params, batch_);
encoder.EncodeField(RtcTestEvent::bool_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::b_));
encoder.EncodeField(RtcTestEvent::signed32_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::signed32_));
encoder.EncodeField(
RtcTestEvent::unsigned32_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::unsigned32_));
encoder.EncodeField(RtcTestEvent::signed64_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::signed64_));
encoder.EncodeField(
RtcTestEvent::unsigned64_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::unsigned64_));
encoder.EncodeField(
RtcTestEvent::optional32_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::optional_signed32_));
encoder.EncodeField(
RtcTestEvent::optional64_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::optional_signed64_));
encoder.EncodeField(
RtcTestEvent::wrapping21_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::wrapping21_));
std::string s = encoder.AsString();
// Optional debug printing
// PrintBytes(s);
ParseEventHeader(s);
ParseAndVerifyTimestamps();
ParseAndVerifyField(RtcTestEvent::bool_params, bool_values,
/*no deltas*/ 0);
ParseAndVerifyField(RtcTestEvent::signed32_params, signed32_values,
/*no deltas*/ 0);
ParseAndVerifyField(RtcTestEvent::unsigned32_params, unsigned32_values,
/*no deltas*/ 0);
ParseAndVerifyField(RtcTestEvent::signed64_params, signed64_values,
/*no deltas*/ 0);
ParseAndVerifyField(RtcTestEvent::unsigned64_params, unsigned64_values,
/*no deltas*/ 0);
ParseAndVerifyOptionalField(RtcTestEvent::optional32_params,
optional32_values, /*no deltas*/ 0);
ParseAndVerifyOptionalField(RtcTestEvent::optional64_params,
optional64_values, /*no deltas*/ 0);
ParseAndVerifyField(RtcTestEvent::wrapping21_params, wrapping21_values,
/*no deltas*/ 0);
}
TEST_F(RtcEventFieldTest, EqualElements) {
std::vector<bool> bool_values = {true, true, true, true};
std::vector<int32_t> signed32_values = {-2, -2, -2, -2};
std::vector<uint32_t> unsigned32_values = {123456789, 123456789, 123456789,
123456789};
std::vector<int64_t> signed64_values = {-9876543210, -9876543210, -9876543210,
-9876543210};
std::vector<uint64_t> unsigned64_values = {9876543210, 9876543210, 9876543210,
9876543210};
std::vector<absl::optional<int32_t>> optional32_values = {
kInt32Min, kInt32Min, kInt32Min, kInt32Min};
std::vector<absl::optional<int64_t>> optional64_values = {
kInt64Max, kInt64Max, kInt64Max, kInt64Max};
std::vector<uint32_t> wrapping21_values = {(1 << 21) - 1, (1 << 21) - 1,
(1 << 21) - 1, (1 << 21) - 1};
CreateFullEvents(bool_values, signed32_values, unsigned32_values,
signed64_values, unsigned64_values, optional32_values,
optional64_values, wrapping21_values);
EventEncoder encoder(RtcTestEvent::event_params, batch_);
encoder.EncodeField(RtcTestEvent::bool_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::b_));
encoder.EncodeField(RtcTestEvent::signed32_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::signed32_));
encoder.EncodeField(
RtcTestEvent::unsigned32_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::unsigned32_));
encoder.EncodeField(RtcTestEvent::signed64_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::signed64_));
encoder.EncodeField(
RtcTestEvent::unsigned64_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::unsigned64_));
encoder.EncodeField(
RtcTestEvent::optional32_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::optional_signed32_));
encoder.EncodeField(
RtcTestEvent::optional64_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::optional_signed64_));
encoder.EncodeField(
RtcTestEvent::wrapping21_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::wrapping21_));
std::string s = encoder.AsString();
// Optional debug printing
// PrintBytes(s);
ParseEventHeader(s);
ParseAndVerifyTimestamps();
ParseAndVerifyField(RtcTestEvent::bool_params, bool_values,
/*no deltas*/ 0);
ParseAndVerifyField(RtcTestEvent::signed32_params, signed32_values,
/*no deltas*/ 0);
ParseAndVerifyField(RtcTestEvent::unsigned32_params, unsigned32_values,
/*no deltas*/ 0);
ParseAndVerifyField(RtcTestEvent::signed64_params, signed64_values,
/*no deltas*/ 0);
ParseAndVerifyField(RtcTestEvent::unsigned64_params, unsigned64_values,
/*no deltas*/ 0);
ParseAndVerifyOptionalField(RtcTestEvent::optional32_params,
optional32_values, /*no deltas*/ 0);
ParseAndVerifyOptionalField(RtcTestEvent::optional64_params,
optional64_values, /*no deltas*/ 0);
ParseAndVerifyField(RtcTestEvent::wrapping21_params, wrapping21_values,
/*no deltas*/ 0);
}
TEST_F(RtcEventFieldTest, Increasing) {
std::vector<bool> bool_values = {false, true, false, true};
std::vector<int32_t> signed32_values = {-2, -1, 0, 1};
std::vector<uint32_t> unsigned32_values = {kUint32Max - 1, kUint32Max, 0, 1};
std::vector<int64_t> signed64_values = {kInt64Max - 1, kInt64Max, kInt64Min,
kInt64Min + 1};
std::vector<uint64_t> unsigned64_values = {kUint64Max - 1, kUint64Max, 0, 1};
std::vector<absl::optional<int32_t>> optional32_values = {
kInt32Max - 1, kInt32Max, kInt32Min, kInt32Min + 1};
std::vector<absl::optional<int64_t>> optional64_values = {
kInt64Max - 1, kInt64Max, kInt64Min, kInt64Min + 1};
std::vector<uint32_t> wrapping21_values = {(1 << 21) - 2, (1 << 21) - 1, 0,
1};
CreateFullEvents(bool_values, signed32_values, unsigned32_values,
signed64_values, unsigned64_values, optional32_values,
optional64_values, wrapping21_values);
EventEncoder encoder(RtcTestEvent::event_params, batch_);
encoder.EncodeField(RtcTestEvent::bool_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::b_));
encoder.EncodeField(RtcTestEvent::signed32_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::signed32_));
encoder.EncodeField(
RtcTestEvent::unsigned32_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::unsigned32_));
encoder.EncodeField(RtcTestEvent::signed64_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::signed64_));
encoder.EncodeField(
RtcTestEvent::unsigned64_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::unsigned64_));
encoder.EncodeField(
RtcTestEvent::optional32_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::optional_signed32_));
encoder.EncodeField(
RtcTestEvent::optional64_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::optional_signed64_));
encoder.EncodeField(
RtcTestEvent::wrapping21_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::wrapping21_));
std::string s = encoder.AsString();
// Optional debug printing
// PrintBytes(s);
ParseEventHeader(s);
ParseAndVerifyTimestamps();
ParseAndVerifyField(RtcTestEvent::bool_params, bool_values,
/*delta bits*/ 1);
ParseAndVerifyField(RtcTestEvent::signed32_params, signed32_values,
/*delta bits*/ 1);
ParseAndVerifyField(RtcTestEvent::unsigned32_params, unsigned32_values,
/*delta bits*/ 1);
ParseAndVerifyField(RtcTestEvent::signed64_params, signed64_values,
/*delta bits*/ 1);
ParseAndVerifyField(RtcTestEvent::unsigned64_params, unsigned64_values,
/*delta bits*/ 1);
ParseAndVerifyOptionalField(RtcTestEvent::optional32_params,
optional32_values, /*delta bits*/ 1);
ParseAndVerifyOptionalField(RtcTestEvent::optional64_params,
optional64_values, /*delta bits*/ 1);
ParseAndVerifyField(RtcTestEvent::wrapping21_params, wrapping21_values,
/*delta bits*/ 1);
}
TEST_F(RtcEventFieldTest, Decreasing) {
std::vector<bool> bool_values = {true, false, true, false};
std::vector<int32_t> signed32_values = {2, 1, 0, -1};
std::vector<uint32_t> unsigned32_values = {1, 0, kUint32Max, kUint32Max - 1};
std::vector<int64_t> signed64_values = {kInt64Min + 1, kInt64Min, kInt64Max,
kInt64Max - 1};
std::vector<uint64_t> unsigned64_values = {1, 0, kUint64Max, kUint64Max - 1};
std::vector<absl::optional<int32_t>> optional32_values = {
kInt32Min + 1, kInt32Min, kInt32Max, kInt32Max - 1};
std::vector<absl::optional<int64_t>> optional64_values = {
kInt64Min + 1, kInt64Min, kInt64Max, kInt64Max - 1};
std::vector<uint32_t> wrapping21_values = {1, 0, (1 << 21) - 1,
(1 << 21) - 2};
CreateFullEvents(bool_values, signed32_values, unsigned32_values,
signed64_values, unsigned64_values, optional32_values,
optional64_values, wrapping21_values);
EventEncoder encoder(RtcTestEvent::event_params, batch_);
encoder.EncodeField(RtcTestEvent::bool_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::b_));
encoder.EncodeField(RtcTestEvent::signed32_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::signed32_));
encoder.EncodeField(
RtcTestEvent::unsigned32_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::unsigned32_));
encoder.EncodeField(RtcTestEvent::signed64_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::signed64_));
encoder.EncodeField(
RtcTestEvent::unsigned64_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::unsigned64_));
encoder.EncodeField(
RtcTestEvent::optional32_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::optional_signed32_));
encoder.EncodeField(
RtcTestEvent::optional64_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::optional_signed64_));
encoder.EncodeField(
RtcTestEvent::wrapping21_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::wrapping21_));
std::string s = encoder.AsString();
// Optional debug printing
// PrintBytes(s);
ParseEventHeader(s);
ParseAndVerifyTimestamps();
ParseAndVerifyField(RtcTestEvent::bool_params, bool_values,
/*delta bits*/ 1);
ParseAndVerifyField(RtcTestEvent::signed32_params, signed32_values,
/*delta bits*/ 1);
ParseAndVerifyField(RtcTestEvent::unsigned32_params, unsigned32_values,
/*delta bits*/ 1);
ParseAndVerifyField(RtcTestEvent::signed64_params, signed64_values,
/*delta bits*/ 1);
ParseAndVerifyField(RtcTestEvent::unsigned64_params, unsigned64_values,
/*delta bits*/ 1);
ParseAndVerifyOptionalField(RtcTestEvent::optional32_params,
optional32_values, /*delta bits*/ 1);
ParseAndVerifyOptionalField(RtcTestEvent::optional64_params,
optional64_values, /*delta bits*/ 1);
ParseAndVerifyField(RtcTestEvent::wrapping21_params, wrapping21_values,
/*delta bits*/ 1);
}
TEST_F(RtcEventFieldTest, SkipsDeprecatedFields) {
// Expect parser to skip fields it doesn't recognize, but find subsequent
// fields.
std::vector<bool> bool_values = {true, false};
std::vector<int32_t> signed32_values = {kInt32Min / 2, kInt32Max / 2};
std::vector<uint32_t> unsigned32_values = {0, kUint32Max / 2};
std::vector<int64_t> signed64_values = {kInt64Min / 2, kInt64Max / 2};
std::vector<uint64_t> unsigned64_values = {0, kUint64Max / 2};
std::vector<absl::optional<int32_t>> optional32_values = {kInt32Max / 2,
kInt32Min / 2};
std::vector<absl::optional<int64_t>> optional64_values = {kInt64Min / 2,
kInt64Max / 2};
std::vector<uint32_t> wrapping21_values = {0, 1 << 20};
size_t signed32_encoding_size =
/*tag*/ 1 + /* varint base*/ 5 + /* delta_header*/ 1 + /*deltas*/ 4;
size_t signed64_encoding_size =
/*tag*/ 1 + /* fixed64 base*/ 8 + /* delta_header*/ 1 + /*deltas*/ 8;
size_t optional32_encoding_size =
/*tag*/ 1 + /* fixed32 base*/ 4 + /* delta_header*/ 1 + /*deltas*/ 4;
CreateFullEvents(bool_values, signed32_values, unsigned32_values,
signed64_values, unsigned64_values, optional32_values,
optional64_values, wrapping21_values);
EventEncoder encoder(RtcTestEvent::event_params, batch_);
encoder.EncodeField(RtcTestEvent::bool_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::b_));
encoder.EncodeField(RtcTestEvent::signed32_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::signed32_));
encoder.EncodeField(
RtcTestEvent::unsigned32_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::unsigned32_));
encoder.EncodeField(RtcTestEvent::signed64_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::signed64_));
encoder.EncodeField(
RtcTestEvent::unsigned64_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::unsigned64_));
encoder.EncodeField(
RtcTestEvent::optional32_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::optional_signed32_));
encoder.EncodeField(
RtcTestEvent::optional64_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::optional_signed64_));
encoder.EncodeField(
RtcTestEvent::wrapping21_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::wrapping21_));
std::string s = encoder.AsString();
// Optional debug printing
// PrintBytes(s);
ParseEventHeader(s);
ParseAndVerifyTimestamps();
ParseAndVerifyField(RtcTestEvent::bool_params, bool_values,
/*delta_bits=*/1);
// Skips parsing the `signed32_values`. The following unsigned fields should
// still be found.
ParseAndVerifyField(RtcTestEvent::unsigned32_params, unsigned32_values,
/*delta_bits=*/31, signed32_encoding_size);
// Skips parsing the `signed64_values`. The following unsigned fields should
// still be found.
ParseAndVerifyField(RtcTestEvent::unsigned64_params, unsigned64_values,
/*delta_bits=*/63, signed64_encoding_size);
// Skips parsing the `optional32_values`. The following unsigned fields should
// still be found.
ParseAndVerifyOptionalField(RtcTestEvent::optional64_params,
optional64_values,
/*delta_bits=*/63, optional32_encoding_size);
ParseAndVerifyField(RtcTestEvent::wrapping21_params, wrapping21_values,
/*delta_bits=*/20);
}
TEST_F(RtcEventFieldTest, SkipsMissingFields) {
// Expect parsing of missing field to succeed but return an empty list.
std::vector<bool> bool_values = {true, false};
std::vector<int32_t> signed32_values = {kInt32Min / 2, kInt32Max / 2};
std::vector<uint32_t> unsigned32_values = {0, kUint32Max / 2};
std::vector<int64_t> signed64_values = {kInt64Min / 2, kInt64Max / 2};
std::vector<uint64_t> unsigned64_values = {0, kUint64Max / 2};
std::vector<absl::optional<int32_t>> optional32_values = {kInt32Max / 2,
kInt32Min / 2};
std::vector<absl::optional<int64_t>> optional64_values = {kInt64Min / 2,
kInt64Max / 2};
std::vector<uint32_t> wrapping21_values = {0, 1 << 20};
CreateFullEvents(bool_values, signed32_values, unsigned32_values,
signed64_values, unsigned64_values, optional32_values,
optional64_values, wrapping21_values);
EventEncoder encoder(RtcTestEvent::event_params, batch_);
// Skip encoding the `bool_values`.
encoder.EncodeField(RtcTestEvent::signed32_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::signed32_));
// Skip encoding the `unsigned32_values`.
encoder.EncodeField(RtcTestEvent::signed64_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::signed64_));
// Skip encoding the `unsigned64_values`.
encoder.EncodeField(
RtcTestEvent::optional32_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::optional_signed32_));
// Skip encoding the `optional64_values`.
encoder.EncodeField(
RtcTestEvent::wrapping21_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::wrapping21_));
std::string s = encoder.AsString();
// Optional debug printing
// PrintBytes(s);
ParseEventHeader(s);
ParseAndVerifyTimestamps();
ParseAndVerifyMissingField(RtcTestEvent::bool_params);
ParseAndVerifyField(RtcTestEvent::signed32_params, signed32_values,
/*delta_bits=*/31);
ParseAndVerifyMissingField(RtcTestEvent::unsigned32_params);
ParseAndVerifyField(RtcTestEvent::signed64_params, signed64_values,
/*delta_bits=*/63);
ParseAndVerifyMissingField(RtcTestEvent::unsigned64_params);
ParseAndVerifyOptionalField(RtcTestEvent::optional32_params,
optional32_values, /*delta_bits=*/31);
ParseAndVerifyMissingOptionalField(RtcTestEvent::optional64_params);
ParseAndVerifyField(RtcTestEvent::wrapping21_params, wrapping21_values,
/*delta_bits=*/20);
}
TEST_F(RtcEventFieldTest, OptionalFields) {
std::vector<absl::optional<int32_t>> optional32_values = {
2, absl::nullopt, 4, absl::nullopt, 6, absl::nullopt};
std::vector<absl::optional<int64_t>> optional64_values = {
absl::nullopt, 1024, absl::nullopt, 1025, absl::nullopt, 1026};
std::vector<uint32_t> wrapping21_values = {(1 << 21) - 3, 0, 2, 5, 5, 6};
for (size_t i = 0; i < optional32_values.size(); i++) {
batch_.push_back(new RtcTestEvent(0, 0, 0, 0, 0, optional32_values[i],
optional64_values[i],
wrapping21_values[i]));
}
EventEncoder encoder(RtcTestEvent::event_params, batch_);
encoder.EncodeField(
RtcTestEvent::optional32_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::optional_signed32_));
encoder.EncodeField(
RtcTestEvent::optional64_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::optional_signed64_));
encoder.EncodeField(
RtcTestEvent::wrapping21_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::wrapping21_));
std::string s = encoder.AsString();
// Optional debug output
// PrintBytes(s);
ParseEventHeader(s);
ParseAndVerifyTimestamps();
ParseAndVerifyOptionalField(RtcTestEvent::optional32_params,
optional32_values, /*delta bits*/ 2);
ParseAndVerifyOptionalField(RtcTestEvent::optional64_params,
optional64_values, /*delta bits*/ 1);
ParseAndVerifyField(RtcTestEvent::wrapping21_params, wrapping21_values,
/*delta bits*/ 2);
}
TEST_F(RtcEventFieldTest, AllNulloptTreatedAsMissing) {
std::vector<absl::optional<int32_t>> optional32_values = {
absl::nullopt, absl::nullopt, absl::nullopt,
absl::nullopt, absl::nullopt, absl::nullopt};
std::vector<absl::optional<int64_t>> optional64_values = {
absl::nullopt, 1024, absl::nullopt, 1025, absl::nullopt, 1026};
for (size_t i = 0; i < optional32_values.size(); i++) {
batch_.push_back(new RtcTestEvent(0, 0, 0, 0, 0, optional32_values[i],
optional64_values[i], 0));
}
EventEncoder encoder(RtcTestEvent::event_params, batch_);
encoder.EncodeField(
RtcTestEvent::optional32_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::optional_signed32_));
encoder.EncodeField(
RtcTestEvent::optional64_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::optional_signed64_));
std::string s = encoder.AsString();
// Optional debug output
// PrintBytes(s);
ParseEventHeader(s);
ParseAndVerifyTimestamps();
ParseAndVerifyMissingOptionalField(RtcTestEvent::optional32_params);
ParseAndVerifyOptionalField(RtcTestEvent::optional64_params,
optional64_values, /*delta_bits=*/1);
}
} // namespace webrtc