admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Allow encoding string fields in new event log format.

Browse Source 
Return parse results as a StatusOr containing views to values owned by the parser.

Bug: webrtc:11933
Change-Id: Icf26b9cb651d1e9244c764c3ec1fdb66abfc9e08
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/233740
Reviewed-by: Sebastian Jansson <srte@webrtc.org>
Commit-Queue: Björn Terelius <terelius@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#35129}
This commit is contained in:
Björn Terelius 2021-10-01 13:45:46 +02:00 committed by WebRTC LUCI CQ
parent ba4d870acf
commit 1d4aa36988
6 changed files with 439 additions and 172 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
logging/rtc_event_log/events/rtc_event_field_encoding.cc
View File

@ -229,6 +229,38 @@ void EventEncoder::EncodeField(const FieldParameters& params,
}
}
void EventEncoder::EncodeField(const FieldParameters& params,
const std::vector<absl::string_view>& values) {
RTC_DCHECK_EQ(values.size(), batch_size_);
if (values.size() == 0) {
// If all values for a particular field is empty/nullopt,
// then we completely skip the field even if the the batch is non-empty.
return;
}
// Write the field tag.
RTC_CHECK_NE(params.field_id, FieldParameters::kTimestampField);
RTC_DCHECK_LE(params.field_id, std::numeric_limits<uint64_t>::max() >> 3);
RTC_DCHECK_EQ(params.field_type, FieldType::kString);
uint64_t field_tag = params.field_id << 3;
field_tag += static_cast<uint64_t>(params.field_type);
encoded_fields_.push_back(EncodeVarInt(field_tag));
if (values.size() > 1) {
// If multiple values in the batch, write the encoding
// parameters. (Values >0 reserved for future use.)
uint64_t encoding_params = 0;
encoded_fields_.push_back(EncodeVarInt(encoding_params));
}
// Write the strings as (length, data) pairs.
for (absl::string_view s : values) {
encoded_fields_.push_back(EncodeVarInt(s.size()));
encoded_fields_.push_back(std::string(s));
}
}
std::string EventEncoder::AsString() {
std::string encoded_event;

3
logging/rtc_event_log/events/rtc_event_field_encoding.h
View File

@ -77,6 +77,9 @@ class EventEncoder {
void EncodeField(const FieldParameters& params,
const ValuesWithPositions& values);
void EncodeField(const FieldParameters& params,
const std::vector<absl::string_view>& values);
std::string AsString();
private:

285
logging/rtc_event_log/events/rtc_event_field_encoding_parser.cc
View File

@ -74,13 +74,10 @@ uint64_t EventParser::ReadVarInt() {
return output;
}
uint64_t EventParser::ReadOptionalValuePositions(std::vector<bool>* positions) {
if (!positions) {
return CountAndIgnoreOptionalValuePositions();
}
uint64_t EventParser::ReadOptionalValuePositions() {
RTC_DCHECK(positions_.empty());
size_t bits_to_read = NumEventsInBatch();
RTC_DCHECK(positions->empty());
positions_.reserve(bits_to_read);
if (pending_data_.size() * 8 < bits_to_read) {
SetError();
return 0;
@ -88,7 +85,7 @@ uint64_t EventParser::ReadOptionalValuePositions(std::vector<bool>* positions) {
BitstreamReader reader(pending_data_);
for (size_t i = 0; i < bits_to_read; i++) {
positions->push_back(reader.ReadBit());
positions_.push_back(reader.ReadBit());
}
if (!reader.Ok()) {
SetError();
@ -96,30 +93,7 @@ uint64_t EventParser::ReadOptionalValuePositions(std::vector<bool>* positions) {
}
size_t num_existing_values =
std::count(positions->begin(), positions->end(), true);
pending_data_ = pending_data_.substr((bits_to_read + 7) / 8);
return num_existing_values;
}
uint64_t EventParser::CountAndIgnoreOptionalValuePositions() {
size_t bits_to_read = NumEventsInBatch();
if (pending_data_.size() * 8 < bits_to_read) {
SetError();
return 0;
}
BitstreamReader reader(pending_data_);
size_t num_existing_values = 0;
for (size_t i = 0; i < bits_to_read; i++) {
if (reader.ReadBit()) {
++num_existing_values;
}
}
if (!reader.Ok()) {
SetError();
return 0;
}
std::count(positions_.begin(), positions_.end(), 1);
pending_data_ = pending_data_.substr((bits_to_read + 7) / 8);
return num_existing_values;
}
@ -144,12 +118,10 @@ uint64_t EventParser::ReadSingleValue(FieldType field_type) {
void EventParser::ReadDeltasAndPopulateValues(
FixedLengthEncodingParametersV3 params,
uint64_t num_deltas,
uint64_t base,
std::vector<uint64_t>* values) {
RTC_CHECK(values != nullptr);
RTC_DCHECK(values->empty());
values->reserve(num_deltas + 1);
values->push_back(base);
uint64_t base) {
RTC_DCHECK(values_.empty());
values_.reserve(num_deltas + 1);
values_.push_back(base);
if (pending_data_.size() * 8 < num_deltas * params.delta_bit_width()) {
SetError();
@ -174,7 +146,7 @@ void EventParser::ReadDeltasAndPopulateValues(
} else {
value = (value + delta) & params.value_mask();
}
values->push_back(value);
values_.push_back(value);
}
if (!reader.Ok()) {
@ -201,11 +173,111 @@ RtcEventLogParseStatus EventParser::Initialize(absl::string_view s,
return RtcEventLogParseStatus::Success();
}
RtcEventLogParseStatus EventParser::ParseField(const FieldParameters& params,
std::vector<uint64_t>* values,
std::vector<bool>* positions) {
RTC_CHECK(values != nullptr);
RtcEventLogParseStatus EventParser::ParseNumericFieldInternal(
uint64_t value_bit_width,
FieldType field_type) {
RTC_DCHECK(values_.empty());
RTC_DCHECK(positions_.empty());
if (num_events_ == 1) {
// Just a single value in the batch.
uint64_t base = ReadSingleValue(field_type);
if (!Ok()) {
return RtcEventLogParseStatus::Error("Failed to read value", __FILE__,
__LINE__);
}
positions_.push_back(true);
values_.push_back(base);
} else {
// Delta compressed batch.
// Read delta header.
uint64_t header_value = ReadVarInt();
if (!Ok())
return RtcEventLogParseStatus::Error("Failed to read delta header",
__FILE__, __LINE__);
// NB: value_bit_width may be incorrect for the field, if this isn't the
// field we are looking for.
absl::optional<FixedLengthEncodingParametersV3> delta_header =
FixedLengthEncodingParametersV3::ParseDeltaHeader(header_value,
value_bit_width);
if (!delta_header.has_value()) {
return RtcEventLogParseStatus::Error("Failed to parse delta header",
__FILE__, __LINE__);
}
uint64_t num_existing_deltas = NumEventsInBatch() - 1;
if (delta_header->values_optional()) {
size_t num_nonempty_values = ReadOptionalValuePositions();
if (!Ok()) {
return RtcEventLogParseStatus::Error(
"Failed to read positions of optional values", __FILE__, __LINE__);
}
if (num_nonempty_values < 1 || NumEventsInBatch() < num_nonempty_values) {
return RtcEventLogParseStatus::Error(
"Expected at least one non_empty value", __FILE__, __LINE__);
}
num_existing_deltas = num_nonempty_values - 1;
} else {
// All elements in the batch have values.
positions_.assign(NumEventsInBatch(), 1u);
}
// Read base.
uint64_t base = ReadSingleValue(field_type);
if (!Ok()) {
return RtcEventLogParseStatus::Error("Failed to read value", __FILE__,
__LINE__);
}
if (delta_header->values_equal()) {
// Duplicate the base value num_existing_deltas times.
values_.assign(num_existing_deltas + 1, base);
} else {
// Read deltas; ceil(num_existing_deltas*delta_width/8) bits
ReadDeltasAndPopulateValues(delta_header.value(), num_existing_deltas,
base);
if (!Ok()) {
return RtcEventLogParseStatus::Error("Failed to decode deltas",
__FILE__, __LINE__);
}
}
}
return RtcEventLogParseStatus::Success();
}
RtcEventLogParseStatus EventParser::ParseStringFieldInternal() {
RTC_DCHECK(strings_.empty());
if (num_events_ > 1) {
// String encoding params reserved for future use.
uint64_t encoding_params = ReadVarInt();
if (!Ok()) {
return RtcEventLogParseStatus::Error("Failed to read string encoding",
__FILE__, __LINE__);
}
if (encoding_params != 0) {
return RtcEventLogParseStatus::Error(
"Unrecognized string encoding parameters", __FILE__, __LINE__);
}
}
strings_.reserve(num_events_);
for (uint64_t i = 0; i < num_events_; ++i) {
// Just a single value in the batch.
uint64_t size = ReadVarInt();
if (!Ok()) {
return RtcEventLogParseStatus::Error("Failed to read string size",
__FILE__, __LINE__);
}
if (size > pending_data_.size()) {
return RtcEventLogParseStatus::Error("String size exceeds remaining data",
__FILE__, __LINE__);
}
strings_.push_back(pending_data_.substr(0, size));
pending_data_ = pending_data_.substr(size);
}
return RtcEventLogParseStatus::Success();
}
RtcEventLogParseStatus EventParser::ParseField(const FieldParameters& params) {
// Verify that the event parses fields in increasing order.
if (params.field_id == FieldParameters::kTimestampField) {
RTC_DCHECK_EQ(last_field_id_, FieldParameters::kTimestampField);
@ -224,10 +296,7 @@ RtcEventLogParseStatus EventParser::ParseField(const FieldParameters& params,
// params.field_id doesn't exist.
while (!pending_data_.empty()) {
absl::string_view field_start = pending_data_;
values->clear();
if (positions) {
positions->clear();
}
ClearTemporaries();
// Read tag for non-positional fields.
if (params.field_id != FieldParameters::kTimestampField) {
@ -252,72 +321,15 @@ RtcEventLogParseStatus EventParser::ParseField(const FieldParameters& params,
return RtcEventLogParseStatus::Success();
}
if (num_events_ == 1) {
// Just a single value in the batch.
uint64_t base = ReadSingleValue(field_type);
if (!Ok())
return RtcEventLogParseStatus::Error("Failed to read value", __FILE__,
__LINE__);
if (positions) {
positions->push_back(true);
if (field_type == FieldType::kString) {
auto status = ParseStringFieldInternal();
if (!status.ok()) {
return status;
}
values->push_back(base);
} else {
// Delta compressed batch.
// Read delta header.
uint64_t header_value = ReadVarInt();
if (!Ok())
return RtcEventLogParseStatus::Error("Failed to read delta header",
__FILE__, __LINE__);
// NB: value_width may be incorrect for the field, if this isn't the field
// we are looking for.
absl::optional<FixedLengthEncodingParametersV3> delta_header =
FixedLengthEncodingParametersV3::ParseDeltaHeader(header_value,
params.value_width);
if (!delta_header.has_value()) {
return RtcEventLogParseStatus::Error("Failed to parse delta header",
__FILE__, __LINE__);
}
uint64_t num_existing_deltas = NumEventsInBatch() - 1;
if (delta_header->values_optional()) {
size_t num_nonempty_values = ReadOptionalValuePositions(positions);
if (!Ok()) {
return RtcEventLogParseStatus::Error(
"Failed to read positions of optional values", __FILE__,
__LINE__);
}
if (num_nonempty_values < 1 ||
NumEventsInBatch() < num_nonempty_values) {
return RtcEventLogParseStatus::Error(
"Expected at least one non_empty value", __FILE__, __LINE__);
}
num_existing_deltas = num_nonempty_values - 1;
} else {
// All elements in the batch have values.
if (positions) {
positions->assign(NumEventsInBatch(), true);
}
}
// Read base.
uint64_t base = ReadSingleValue(field_type);
if (!Ok()) {
return RtcEventLogParseStatus::Error("Failed to read value", __FILE__,
__LINE__);
}
if (delta_header->values_equal()) {
// Duplicate the base value num_existing_deltas times.
values->assign(num_existing_deltas + 1, base);
} else {
// Read deltas; ceil(num_existing_deltas*delta_width/8) bits
ReadDeltasAndPopulateValues(delta_header.value(), num_existing_deltas,
base, values);
if (!Ok()) {
return RtcEventLogParseStatus::Error("Failed to decode deltas",
__FILE__, __LINE__);
}
auto status = ParseNumericFieldInternal(params.value_width, field_type);
if (!status.ok()) {
return status;
}
}
@ -328,11 +340,56 @@ RtcEventLogParseStatus EventParser::ParseField(const FieldParameters& params,
}
// Field not found because the event ended.
values->clear();
if (positions) {
positions->clear();
}
ClearTemporaries();
return RtcEventLogParseStatus::Success();
}
RtcEventLogParseStatusOr<rtc::ArrayView<absl::string_view>>
EventParser::ParseStringField(const FieldParameters& params,
bool required_field) {
using StatusOr = RtcEventLogParseStatusOr<rtc::ArrayView<absl::string_view>>;
RTC_DCHECK_EQ(params.field_type, FieldType::kString);
auto status = ParseField(params);
if (!status.ok())
return StatusOr(status);
rtc::ArrayView<absl::string_view> strings = GetStrings();
if (required_field && strings.size() != NumEventsInBatch()) {
return StatusOr::Error("Required string field not found", __FILE__,
__LINE__);
}
return StatusOr(strings);
}
RtcEventLogParseStatusOr<rtc::ArrayView<uint64_t>>
EventParser::ParseNumericField(const FieldParameters& params,
bool required_field) {
using StatusOr = RtcEventLogParseStatusOr<rtc::ArrayView<uint64_t>>;
RTC_DCHECK_NE(params.field_type, FieldType::kString);
auto status = ParseField(params);
if (!status.ok())
return StatusOr(status);
rtc::ArrayView<uint64_t> values = GetValues();
if (required_field && values.size() != NumEventsInBatch()) {
return StatusOr::Error("Required numerical field not found", __FILE__,
__LINE__);
}
return StatusOr(values);
}
RtcEventLogParseStatusOr<EventParser::ValueAndPostionView>
EventParser::ParseOptionalNumericField(const FieldParameters& params,
bool required_field) {
using StatusOr = RtcEventLogParseStatusOr<ValueAndPostionView>;
RTC_DCHECK_NE(params.field_type, FieldType::kString);
auto status = ParseField(params);
if (!status.ok())
return StatusOr(status);
ValueAndPostionView view{GetValues(), GetPositions()};
if (required_field && view.positions.size() != NumEventsInBatch()) {
return StatusOr::Error("Required numerical field not found", __FILE__,
__LINE__);
}
return StatusOr(view);
}
} // namespace webrtc

105
logging/rtc_event_log/events/rtc_event_field_encoding_parser.h
View File

@ -24,6 +24,9 @@
// If/when we start using absl::Status in WebRTC, consider
// whether payloads would be an appropriate alternative.
class RtcEventLogParseStatus {
template <typename T>
friend class RtcEventLogParseStatusOr;
public:
static RtcEventLogParseStatus Success() { return RtcEventLogParseStatus(); }
static RtcEventLogParseStatus Error(std::string error,
@ -33,6 +36,7 @@ class RtcEventLogParseStatus {
}
bool ok() const { return error_.empty(); }
std::string message() const { return error_; }
private:
@ -43,10 +47,52 @@ class RtcEventLogParseStatus {
std::string error_;
};
template <typename T>
class RtcEventLogParseStatusOr {
public:
explicit RtcEventLogParseStatusOr(RtcEventLogParseStatus status)
: status_(status), value_() {}
explicit RtcEventLogParseStatusOr(const T& value)
: status_(), value_(value) {}
bool ok() const { return status_.ok(); }
std::string message() const { return status_.message(); }
const T& value() const {
RTC_DCHECK(ok());
return value_;
}
T& value() {
RTC_DCHECK(ok());
return value_;
}
static RtcEventLogParseStatusOr Error(std::string error,
std::string file,
int line) {
return RtcEventLogParseStatusOr(error, file, line);
}
private:
RtcEventLogParseStatusOr() : status_() {}
RtcEventLogParseStatusOr(std::string error, std::string file, int line)
: status_(error, file, line), value_() {}
RtcEventLogParseStatus status_;
T value_;
};
namespace webrtc {
class EventParser {
public:
struct ValueAndPostionView {
rtc::ArrayView<uint64_t> values;
rtc::ArrayView<uint8_t> positions;
};
EventParser() = default;
// N.B: This method stores a abls::string_view into the string to be
@ -55,17 +101,18 @@ class EventParser {
RtcEventLogParseStatus Initialize(absl::string_view s, bool batched);
// Attempts to parse the field specified by `params`, skipping past
// other fields that may occur before it. Returns
// RtcEventLogParseStatus::Success() and populates `values` (and `positions`)
// if the field is found. Returns RtcEventLogParseStatus::Success() and clears
// `values` (and `positions`) if the field doesn't exist. Returns a
// RtcEventLogParseStatus::Error if the log is incomplete, malformed or
// otherwise can't be parsed. `values` and `positions` are pure out-parameters
// that allow the caller to reuse the same temporary storage for all fields.
// Any previous content in the out parameters is cleared.
RtcEventLogParseStatus ParseField(const FieldParameters& params,
std::vector<uint64_t>* values,
std::vector<bool>* positions = nullptr);
// other fields that may occur before it. If 'required_field == true',
// then failing to find the field is an error, otherwise the functions
// return success, but with an empty view of values.
RtcEventLogParseStatusOr<rtc::ArrayView<absl::string_view>> ParseStringField(
const FieldParameters& params,
bool required_field = true);
RtcEventLogParseStatusOr<rtc::ArrayView<uint64_t>> ParseNumericField(
const FieldParameters& params,
bool required_field = true);
RtcEventLogParseStatusOr<ValueAndPostionView> ParseOptionalNumericField(
const FieldParameters& params,
bool required_field = true);
// Number of events in a batch.
uint64_t NumEventsInBatch() const { return num_events_; }
@ -79,23 +126,47 @@ class EventParser {
uint64_t ReadLittleEndian(uint8_t bytes);
uint64_t ReadVarInt();
uint64_t ReadSingleValue(FieldType field_type);
uint64_t ReadOptionalValuePositions(std::vector<bool>* positions);
uint64_t CountAndIgnoreOptionalValuePositions();
uint64_t ReadOptionalValuePositions();
void ReadDeltasAndPopulateValues(FixedLengthEncodingParametersV3 params,
uint64_t num_deltas,
const uint64_t base,
std::vector<uint64_t>* values);
const uint64_t base);
RtcEventLogParseStatus ParseNumericFieldInternal(uint64_t value_bit_width,
FieldType field_type);
RtcEventLogParseStatus ParseStringFieldInternal();
// Attempts to parse the field specified by `params`, skipping past
// other fields that may occur before it. Returns
// RtcEventLogParseStatus::Success() and populates `values_` (and
// `positions_`) if the field is found. Returns
// RtcEventLogParseStatus::Success() and clears `values_` (and `positions_`)
// if the field doesn't exist. Returns a RtcEventLogParseStatus::Error() if
// the log is incomplete, malformed or otherwise can't be parsed.
RtcEventLogParseStatus ParseField(const FieldParameters& params);
void SetError() { error_ = true; }
bool Ok() const { return !error_; }
// String to be consumed.
absl::string_view pending_data_;
rtc::ArrayView<uint64_t> GetValues() { return values_; }
rtc::ArrayView<uint8_t> GetPositions() { return positions_; }
rtc::ArrayView<absl::string_view> GetStrings() { return strings_; }
void ClearTemporaries() {
positions_.clear();
values_.clear();
strings_.clear();
}
// Tracks whether an error has occurred in one of the helper
// functions above.
bool error_ = false;
// Temporary storage for result.
std::vector<uint8_t> positions_;
std::vector<uint64_t> values_;
std::vector<absl::string_view> strings_;
// String to be consumed.
absl::string_view pending_data_;
uint64_t num_events_ = 1;
uint64_t last_field_id_ = FieldParameters::kTimestampField;
};

155
logging/rtc_event_log/events/rtc_event_field_encoding_unittest.cc
View File

@ -116,6 +116,26 @@ size_t ExpectedEncodingSize(const FieldParameters& params,
return tag_size + base_size + delta_header_size + positions_size + delta_size;
}
size_t ExpectedStringEncodingSize(const FieldParameters& params,
const std::vector<std::string>& values) {
EXPECT_EQ(params.field_type, FieldType::kString);
uint64_t numeric_field_type = static_cast<uint64_t>(params.field_type);
RTC_DCHECK_LT(numeric_field_type, 1u << 3);
size_t tag_size =
ExpectedVarIntSize((params.field_id << 3) + numeric_field_type);
size_t expected_size = tag_size;
if (values.size() > 1) {
// VarInt encoding header reserved for future use. Currently always 0.
expected_size += 1;
}
for (const auto& s : values) {
expected_size += ExpectedVarIntSize(s.size());
expected_size += s.size();
}
return expected_size;
}
} // namespace
class RtcTestEvent final : public RtcEvent {
@ -137,7 +157,8 @@ class RtcTestEvent final : public RtcEvent {
uint64_t unsigned64,
absl::optional<int32_t> optional_signed32,
absl::optional<int64_t> optional_signed64,
uint32_t wrapping21)
uint32_t wrapping21,
std::string string)
: b_(b),
signed32_(signed32),
unsigned32_(unsigned32),
@ -145,7 +166,8 @@ class RtcTestEvent final : public RtcEvent {
unsigned64_(unsigned64),
optional_signed32_(optional_signed32),
optional_signed64_(optional_signed64),
wrapping21_(wrapping21) {}
wrapping21_(wrapping21),
string_(string) {}
~RtcTestEvent() override = default;
Type GetType() const override { return static_cast<Type>(4711); }
@ -170,6 +192,8 @@ class RtcTestEvent final : public RtcEvent {
FieldType::kVarInt, 64};
static constexpr FieldParameters wrapping21_params{"wrapping21", 9,
FieldType::kFixed32, 21};
static constexpr FieldParameters string_params{
"string", 10, FieldType::kString, /*value_width = */ 0};
static constexpr Type kType = static_cast<RtcEvent::Type>(4711);
@ -181,6 +205,7 @@ class RtcTestEvent final : public RtcEvent {
const absl::optional<int32_t> optional_signed32_ = absl::nullopt;
const absl::optional<int64_t> optional_signed64_ = absl::nullopt;
const uint32_t wrapping21_ = 0;
const std::string string_;
};
constexpr EventParameters RtcTestEvent::event_params;
@ -194,6 +219,7 @@ constexpr FieldParameters RtcTestEvent::unsigned64_params;
constexpr FieldParameters RtcTestEvent::optional32_params;
constexpr FieldParameters RtcTestEvent::optional64_params;
constexpr FieldParameters RtcTestEvent::wrapping21_params;
constexpr FieldParameters RtcTestEvent::string_params;
constexpr RtcEvent::Type RtcTestEvent::kType;
@ -209,7 +235,8 @@ class RtcEventFieldTest : public ::testing::Test {
const std::vector<uint64_t>& unsigned64_values,
const std::vector<absl::optional<int32_t>>& optional32_values,
const std::vector<absl::optional<int64_t>>& optional64_values,
const std::vector<uint32_t>& wrapping21_values) {
const std::vector<uint32_t>& wrapping21_values,
const std::vector<std::string>& string_values) {
size_t size = bool_values.size();
RTC_CHECK_EQ(signed32_values.size(), size);
RTC_CHECK_EQ(unsigned32_values.size(), size);
@ -218,12 +245,13 @@ class RtcEventFieldTest : public ::testing::Test {
RTC_CHECK_EQ(optional32_values.size(), size);
RTC_CHECK_EQ(optional64_values.size(), size);
RTC_CHECK_EQ(wrapping21_values.size(), size);
RTC_CHECK_EQ(string_values.size(), size);
for (size_t i = 0; i < size; i++) {
batch_.push_back(new RtcTestEvent(
bool_values[i], signed32_values[i], unsigned32_values[i],
signed64_values[i], unsigned64_values[i], optional32_values[i],
optional64_values[i], wrapping21_values[i]));
optional64_values[i], wrapping21_values[i], string_values[i]));
}
}
@ -252,15 +280,33 @@ class RtcEventFieldTest : public ::testing::Test {
}
void ParseAndVerifyTimestamps() {
std::vector<uint64_t> values;
auto status = parser_.ParseField(RtcTestEvent::timestamp_params, &values);
ASSERT_TRUE(status.ok()) << status.message().c_str();
ASSERT_EQ(values.size(), batch_.size());
auto result = parser_.ParseNumericField(RtcTestEvent::timestamp_params);
ASSERT_TRUE(result.ok()) << result.message().c_str();
ASSERT_EQ(result.value().size(), batch_.size());
for (size_t i = 0; i < batch_.size(); i++) {
EXPECT_EQ(values[i], static_cast<uint64_t>(batch_[i]->timestamp_ms()));
EXPECT_EQ(result.value()[i],
static_cast<uint64_t>(batch_[i]->timestamp_ms()));
}
}
void ParseAndVerifyStringField(
const FieldParameters& params,
const std::vector<std::string>& expected_values,
size_t expected_skipped_bytes = 0) {
size_t expected_size = ExpectedStringEncodingSize(params, expected_values) +
expected_skipped_bytes;
size_t size_before = parser_.RemainingBytes();
auto result = parser_.ParseStringField(params);
ASSERT_TRUE(result.ok()) << result.message().c_str();
ASSERT_EQ(result.value().size(), expected_values.size());
for (size_t i = 0; i < expected_values.size(); i++) {
EXPECT_EQ(result.value()[i], expected_values[i]);
}
size_t size_after = parser_.RemainingBytes();
EXPECT_EQ(size_before - size_after, expected_size)
<< " for field " << params.name;
}
template <typename T>
void ParseAndVerifyField(const FieldParameters& params,
const std::vector<T>& expected_values,
@ -269,13 +315,13 @@ class RtcEventFieldTest : public ::testing::Test {
size_t expected_size =
ExpectedEncodingSize(params, expected_values, expected_bits_per_delta) +
expected_skipped_bytes;
std::vector<uint64_t> values;
size_t size_before = parser_.RemainingBytes();
auto status = parser_.ParseField(params, &values);
ASSERT_TRUE(status.ok()) << status.message().c_str();
ASSERT_EQ(values.size(), expected_values.size());
auto result = parser_.ParseNumericField(params);
ASSERT_TRUE(result.ok()) << result.message().c_str();
ASSERT_EQ(result.value().size(), expected_values.size());
for (size_t i = 0; i < expected_values.size(); i++) {
EXPECT_EQ(DecodeFromUnsignedToType<T>(values[i]), expected_values[i]);
EXPECT_EQ(DecodeFromUnsignedToType<T>(result.value()[i]),
expected_values[i]);
}
size_t size_after = parser_.RemainingBytes();
EXPECT_EQ(size_before - size_after, expected_size)
@ -291,15 +337,13 @@ class RtcEventFieldTest : public ::testing::Test {
size_t expected_size =
ExpectedEncodingSize(params, expected_values, expected_bits_per_delta) +
expected_skipped_bytes;
std::vector<bool> positions;
positions.reserve(expected_values.size());
std::vector<uint64_t> values;
values.reserve(expected_values.size());
size_t size_before = parser_.RemainingBytes();
auto status = parser_.ParseField(params, &values, &positions);
ASSERT_TRUE(status.ok()) << status.message().c_str();
auto value_it = values.begin();
auto result = parser_.ParseOptionalNumericField(params);
ASSERT_TRUE(result.ok()) << result.message().c_str();
rtc::ArrayView<uint64_t> values = result.value().values;
rtc::ArrayView<uint8_t> positions = result.value().positions;
ASSERT_EQ(positions.size(), expected_values.size());
auto value_it = values.begin();
for (size_t i = 0; i < expected_values.size(); i++) {
if (positions[i]) {
ASSERT_NE(value_it, values.end());
@ -317,17 +361,17 @@ class RtcEventFieldTest : public ::testing::Test {
}
void ParseAndVerifyMissingField(const FieldParameters& params) {
std::vector<uint64_t> values{4711};
auto status = parser_.ParseField(params, &values);
ASSERT_TRUE(status.ok()) << status.message().c_str();
EXPECT_EQ(values.size(), 0u);
auto result = parser_.ParseNumericField(params, /*required_field=*/false);
ASSERT_TRUE(result.ok()) << result.message().c_str();
EXPECT_EQ(result.value().size(), 0u);
}
void ParseAndVerifyMissingOptionalField(const FieldParameters& params) {
std::vector<bool> positions{true, false};
std::vector<uint64_t> values{4711};
auto status = parser_.ParseField(params, &values, &positions);
ASSERT_TRUE(status.ok()) << status.message().c_str();
auto result =
parser_.ParseOptionalNumericField(params, /*required_field=*/false);
ASSERT_TRUE(result.ok()) << result.message().c_str();
rtc::ArrayView<uint64_t> values = result.value().values;
rtc::ArrayView<uint8_t> positions = result.value().positions;
EXPECT_EQ(positions.size(), 0u);
EXPECT_EQ(values.size(), 0u);
}
@ -359,10 +403,11 @@ TEST_F(RtcEventFieldTest, Singleton) {
std::vector<absl::optional<int32_t>> optional32_values = {kInt32Min};
std::vector<absl::optional<int64_t>> optional64_values = {kInt64Max};
std::vector<uint32_t> wrapping21_values = {(1 << 21) - 1};
std::vector<std::string> string_values = {"foo"};
CreateFullEvents(bool_values, signed32_values, unsigned32_values,
signed64_values, unsigned64_values, optional32_values,
optional64_values, wrapping21_values);
optional64_values, wrapping21_values, string_values);
EventEncoder encoder(RtcTestEvent::event_params, batch_);
encoder.EncodeField(RtcTestEvent::bool_params,
@ -386,6 +431,8 @@ TEST_F(RtcEventFieldTest, Singleton) {
encoder.EncodeField(
RtcTestEvent::wrapping21_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::wrapping21_));
encoder.EncodeField(RtcTestEvent::string_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::string_));
std::string s = encoder.AsString();
// Optional debug printing
@ -409,6 +456,8 @@ TEST_F(RtcEventFieldTest, Singleton) {
optional64_values, /*no deltas*/ 0);
ParseAndVerifyField(RtcTestEvent::wrapping21_params, wrapping21_values,
/*no deltas*/ 0);
ParseAndVerifyStringField(RtcTestEvent::string_params, string_values);
EXPECT_EQ(parser_.RemainingBytes(), 0u);
}
TEST_F(RtcEventFieldTest, EqualElements) {
@ -426,10 +475,11 @@ TEST_F(RtcEventFieldTest, EqualElements) {
kInt64Max, kInt64Max, kInt64Max, kInt64Max};
std::vector<uint32_t> wrapping21_values = {(1 << 21) - 1, (1 << 21) - 1,
(1 << 21) - 1, (1 << 21) - 1};
std::vector<std::string> string_values = {"foo", "foo", "foo", "foo"};
CreateFullEvents(bool_values, signed32_values, unsigned32_values,
signed64_values, unsigned64_values, optional32_values,
optional64_values, wrapping21_values);
optional64_values, wrapping21_values, string_values);
EventEncoder encoder(RtcTestEvent::event_params, batch_);
encoder.EncodeField(RtcTestEvent::bool_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::b_));
@ -452,6 +502,8 @@ TEST_F(RtcEventFieldTest, EqualElements) {
encoder.EncodeField(
RtcTestEvent::wrapping21_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::wrapping21_));
encoder.EncodeField(RtcTestEvent::string_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::string_));
std::string s = encoder.AsString();
// Optional debug printing
@ -475,6 +527,8 @@ TEST_F(RtcEventFieldTest, EqualElements) {
optional64_values, /*no deltas*/ 0);
ParseAndVerifyField(RtcTestEvent::wrapping21_params, wrapping21_values,
/*no deltas*/ 0);
ParseAndVerifyStringField(RtcTestEvent::string_params, string_values);
EXPECT_EQ(parser_.RemainingBytes(), 0u);
}
TEST_F(RtcEventFieldTest, Increasing) {
@ -490,10 +544,12 @@ TEST_F(RtcEventFieldTest, Increasing) {
kInt64Max - 1, kInt64Max, kInt64Min, kInt64Min + 1};
std::vector<uint32_t> wrapping21_values = {(1 << 21) - 2, (1 << 21) - 1, 0,
1};
std::vector<std::string> string_values = {
"", "a", "bc", "def"}; // No special compression of strings.
CreateFullEvents(bool_values, signed32_values, unsigned32_values,
signed64_values, unsigned64_values, optional32_values,
optional64_values, wrapping21_values);
optional64_values, wrapping21_values, string_values);
EventEncoder encoder(RtcTestEvent::event_params, batch_);
encoder.EncodeField(RtcTestEvent::bool_params,
@ -517,6 +573,8 @@ TEST_F(RtcEventFieldTest, Increasing) {
encoder.EncodeField(
RtcTestEvent::wrapping21_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::wrapping21_));
encoder.EncodeField(RtcTestEvent::string_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::string_));
std::string s = encoder.AsString();
// Optional debug printing
@ -540,6 +598,8 @@ TEST_F(RtcEventFieldTest, Increasing) {
optional64_values, /*delta bits*/ 1);
ParseAndVerifyField(RtcTestEvent::wrapping21_params, wrapping21_values,
/*delta bits*/ 1);
ParseAndVerifyStringField(RtcTestEvent::string_params, string_values);
EXPECT_EQ(parser_.RemainingBytes(), 0u);
}
TEST_F(RtcEventFieldTest, Decreasing) {
@ -555,10 +615,12 @@ TEST_F(RtcEventFieldTest, Decreasing) {
kInt64Min + 1, kInt64Min, kInt64Max, kInt64Max - 1};
std::vector<uint32_t> wrapping21_values = {1, 0, (1 << 21) - 1,
(1 << 21) - 2};
std::vector<std::string> string_values = {
"def", "bc", "a", ""}; // No special compression of strings.
CreateFullEvents(bool_values, signed32_values, unsigned32_values,
signed64_values, unsigned64_values, optional32_values,
optional64_values, wrapping21_values);
optional64_values, wrapping21_values, string_values);
EventEncoder encoder(RtcTestEvent::event_params, batch_);
encoder.EncodeField(RtcTestEvent::bool_params,
@ -582,6 +644,8 @@ TEST_F(RtcEventFieldTest, Decreasing) {
encoder.EncodeField(
RtcTestEvent::wrapping21_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::wrapping21_));
encoder.EncodeField(RtcTestEvent::string_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::string_));
std::string s = encoder.AsString();
// Optional debug printing
@ -605,6 +669,8 @@ TEST_F(RtcEventFieldTest, Decreasing) {
optional64_values, /*delta bits*/ 1);
ParseAndVerifyField(RtcTestEvent::wrapping21_params, wrapping21_values,
/*delta bits*/ 1);
ParseAndVerifyStringField(RtcTestEvent::string_params, string_values);
EXPECT_EQ(parser_.RemainingBytes(), 0u);
}
TEST_F(RtcEventFieldTest, SkipsDeprecatedFields) {
@ -620,6 +686,7 @@ TEST_F(RtcEventFieldTest, SkipsDeprecatedFields) {
std::vector<absl::optional<int64_t>> optional64_values = {kInt64Min / 2,
kInt64Max / 2};
std::vector<uint32_t> wrapping21_values = {0, 1 << 20};
std::vector<std::string> string_values = {"foo", "bar"};
size_t signed32_encoding_size =
/*tag*/ 1 + /* varint base*/ 5 + /* delta_header*/ 1 + /*deltas*/ 4;
@ -630,7 +697,7 @@ TEST_F(RtcEventFieldTest, SkipsDeprecatedFields) {
CreateFullEvents(bool_values, signed32_values, unsigned32_values,
signed64_values, unsigned64_values, optional32_values,
optional64_values, wrapping21_values);
optional64_values, wrapping21_values, string_values);
EventEncoder encoder(RtcTestEvent::event_params, batch_);
encoder.EncodeField(RtcTestEvent::bool_params,
@ -654,6 +721,8 @@ TEST_F(RtcEventFieldTest, SkipsDeprecatedFields) {
encoder.EncodeField(
RtcTestEvent::wrapping21_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::wrapping21_));
encoder.EncodeField(RtcTestEvent::string_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::string_));
std::string s = encoder.AsString();
// Optional debug printing
@ -666,7 +735,8 @@ TEST_F(RtcEventFieldTest, SkipsDeprecatedFields) {
// Skips parsing the `signed32_values`. The following unsigned fields should
// still be found.
ParseAndVerifyField(RtcTestEvent::unsigned32_params, unsigned32_values,
/*delta_bits=*/31, signed32_encoding_size);
/*delta_bits=*/31,
/*expected_skipped_bytes=*/signed32_encoding_size);
// Skips parsing the `signed64_values`. The following unsigned fields should
// still be found.
ParseAndVerifyField(RtcTestEvent::unsigned64_params, unsigned64_values,
@ -678,6 +748,8 @@ TEST_F(RtcEventFieldTest, SkipsDeprecatedFields) {
/*delta_bits=*/63, optional32_encoding_size);
ParseAndVerifyField(RtcTestEvent::wrapping21_params, wrapping21_values,
/*delta_bits=*/20);
ParseAndVerifyStringField(RtcTestEvent::string_params, string_values);
EXPECT_EQ(parser_.RemainingBytes(), 0u);
}
TEST_F(RtcEventFieldTest, SkipsMissingFields) {
@ -693,10 +765,11 @@ TEST_F(RtcEventFieldTest, SkipsMissingFields) {
std::vector<absl::optional<int64_t>> optional64_values = {kInt64Min / 2,
kInt64Max / 2};
std::vector<uint32_t> wrapping21_values = {0, 1 << 20};
std::vector<std::string> string_values = {"foo", "foo"};
CreateFullEvents(bool_values, signed32_values, unsigned32_values,
signed64_values, unsigned64_values, optional32_values,
optional64_values, wrapping21_values);
optional64_values, wrapping21_values, string_values);
EventEncoder encoder(RtcTestEvent::event_params, batch_);
// Skip encoding the `bool_values`.
@ -713,6 +786,8 @@ TEST_F(RtcEventFieldTest, SkipsMissingFields) {
encoder.EncodeField(
RtcTestEvent::wrapping21_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::wrapping21_));
encoder.EncodeField(RtcTestEvent::string_params,
ExtractRtcEventMember(batch_, &RtcTestEvent::string_));
std::string s = encoder.AsString();
// Optional debug printing
@ -732,6 +807,8 @@ TEST_F(RtcEventFieldTest, SkipsMissingFields) {
ParseAndVerifyMissingOptionalField(RtcTestEvent::optional64_params);
ParseAndVerifyField(RtcTestEvent::wrapping21_params, wrapping21_values,
/*delta_bits=*/20);
ParseAndVerifyStringField(RtcTestEvent::string_params, string_values);
EXPECT_EQ(parser_.RemainingBytes(), 0u);
}
TEST_F(RtcEventFieldTest, OptionalFields) {
@ -744,7 +821,7 @@ TEST_F(RtcEventFieldTest, OptionalFields) {
for (size_t i = 0; i < optional32_values.size(); i++) {
batch_.push_back(new RtcTestEvent(0, 0, 0, 0, 0, optional32_values[i],
optional64_values[i],
wrapping21_values[i]));
wrapping21_values[i], ""));
}
EventEncoder encoder(RtcTestEvent::event_params, batch_);
@ -770,6 +847,7 @@ TEST_F(RtcEventFieldTest, OptionalFields) {
optional64_values, /*delta bits*/ 1);
ParseAndVerifyField(RtcTestEvent::wrapping21_params, wrapping21_values,
/*delta bits*/ 2);
EXPECT_EQ(parser_.RemainingBytes(), 0u);
}
TEST_F(RtcEventFieldTest, AllNulloptTreatedAsMissing) {
@ -781,7 +859,7 @@ TEST_F(RtcEventFieldTest, AllNulloptTreatedAsMissing) {
for (size_t i = 0; i < optional32_values.size(); i++) {
batch_.push_back(new RtcTestEvent(0, 0, 0, 0, 0, optional32_values[i],
optional64_values[i], 0));
optional64_values[i], 0, ""));
}
EventEncoder encoder(RtcTestEvent::event_params, batch_);
@ -801,6 +879,7 @@ TEST_F(RtcEventFieldTest, AllNulloptTreatedAsMissing) {
ParseAndVerifyMissingOptionalField(RtcTestEvent::optional32_params);
ParseAndVerifyOptionalField(RtcTestEvent::optional64_params,
optional64_values, /*delta_bits=*/1);
EXPECT_EQ(parser_.RemainingBytes(), 0u);
}
} // namespace webrtc

31
logging/rtc_event_log/events/rtc_event_field_extraction.h
View File

@ -116,13 +116,27 @@ ValuesWithPositions ExtractRtcEventMember(rtc::ArrayView<const RtcEvent*> batch,
return result;
}
template <typename E>
std::vector<absl::string_view> ExtractRtcEventMember(
rtc::ArrayView<const RtcEvent*> batch,
const std::string E::*member) {
std::vector<absl::string_view> values;
values.reserve(batch.size());
for (const RtcEvent* event : batch) {
RTC_CHECK_EQ(event->GetType(), E::kType);
absl::string_view str = static_cast<const E*>(event)->*member;
values.push_back(str);
}
return values;
}
// Inverse of the ExtractRtcEventMember function used when parsing
// a log. Uses a vector of values to populate a specific field in a
// vector of structs.
template <typename T,
typename E,
std::enable_if_t<std::is_integral<T>::value, bool> = true>
void PopulateRtcEventMember(const std::vector<uint64_t>& values,
void PopulateRtcEventMember(const rtc::ArrayView<uint64_t> values,
T E::*member,
rtc::ArrayView<E> output) {
size_t batch_size = values.size();
@ -136,8 +150,8 @@ void PopulateRtcEventMember(const std::vector<uint64_t>& values,
template <typename T,
typename E,
std::enable_if_t<std::is_integral<T>::value, bool> = true>
void PopulateRtcEventMember(const std::vector<bool>& positions,
const std::vector<uint64_t>& values,
void PopulateRtcEventMember(const rtc::ArrayView<uint8_t> positions,
const rtc::ArrayView<uint64_t> values,
absl::optional<T> E::*member,
rtc::ArrayView<E> output) {
size_t batch_size = positions.size();
@ -156,6 +170,17 @@ void PopulateRtcEventMember(const std::vector<bool>& positions,
RTC_CHECK(value_it == values.end());
}
template <typename E>
void PopulateRtcEventMember(const rtc::ArrayView<absl::string_view> values,
std::string E::*member,
rtc::ArrayView<E> output) {
size_t batch_size = values.size();
RTC_CHECK_EQ(output.size(), batch_size);
for (size_t i = 0; i < batch_size; ++i) {
output[i].*member = values[i];
}
}
} // namespace webrtc
#endif // LOGGING_RTC_EVENT_LOG_EVENTS_RTC_EVENT_FIELD_EXTRACTION_H_