admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Prepare for new encoding of RTC event log numeric fields.

Browse Source 
Bug: webrtc:11933
Change-Id: I32e59059ea6166b2fc089d9d19d3ab3829c2190e
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/228942
Commit-Queue: Björn Terelius <terelius@webrtc.org>
Reviewed-by: Sebastian Jansson <srte@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#35071}
This commit is contained in:
Björn Terelius 2021-09-22 19:37:22 +02:00 committed by WebRTC LUCI CQ
parent a551bd1dc5
commit 98952d7f0f
7 changed files with 484 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
logging/BUILD.gn
View File

@ -32,6 +32,27 @@ rtc_source_set("rtc_event_log_api") {
deps = [ "../api/rtc_event_log" ]
}
rtc_library("rtc_event_field") {
sources = [
"rtc_event_log/events/rtc_event_field_extraction.cc",
"rtc_event_log/events/rtc_event_field_extraction.h",
]
deps = [
":rtc_event_number_encodings",
"../api:array_view",
"../api/rtc_event_log",
"../rtc_base:checks",
"../rtc_base:logging",
"../rtc_base:rtc_base_approved",
]
absl_deps = [
"//third_party/abseil-cpp/absl/memory",
"//third_party/abseil-cpp/absl/strings",
"//third_party/abseil-cpp/absl/types:optional",
]
}
rtc_library("rtc_stream_config") {
sources = [
"rtc_event_log/rtc_stream_config.cc",
@ -190,14 +211,10 @@ rtc_library("rtc_event_video") {
absl_deps = [ "//third_party/abseil-cpp/absl/memory" ]
}
# TODO(eladalon): Break down into (1) encoder and (2) decoder; we don't need
# the decoder code in the WebRTC library, only in unit tests and tools.
rtc_library("rtc_event_log_impl_encoder") {
rtc_library("rtc_event_number_encodings") {
sources = [
"rtc_event_log/encoder/blob_encoding.cc",
"rtc_event_log/encoder/blob_encoding.h",
"rtc_event_log/encoder/delta_encoding.cc",
"rtc_event_log/encoder/delta_encoding.h",
"rtc_event_log/encoder/bit_writer.cc",
"rtc_event_log/encoder/bit_writer.h",
"rtc_event_log/encoder/rtc_event_log_encoder_common.cc",
"rtc_event_log/encoder/rtc_event_log_encoder_common.h",
"rtc_event_log/encoder/var_int.cc",
@ -207,6 +224,33 @@ rtc_library("rtc_event_log_impl_encoder") {
defines = []
deps = [
"../rtc_base:bitstream_reader",
"../rtc_base:checks",
"../rtc_base:ignore_wundef",
"../rtc_base:macromagic",
"../rtc_base:rtc_base_approved",
]
absl_deps = [
"//third_party/abseil-cpp/absl/memory",
"//third_party/abseil-cpp/absl/strings",
"//third_party/abseil-cpp/absl/types:optional",
]
}
# TODO(eladalon): Break down into (1) encoder and (2) decoder; we don't need
# the decoder code in the WebRTC library, only in unit tests and tools.
rtc_library("rtc_event_log_impl_encoder") {
sources = [
"rtc_event_log/encoder/blob_encoding.cc",
"rtc_event_log/encoder/blob_encoding.h",
"rtc_event_log/encoder/delta_encoding.cc",
"rtc_event_log/encoder/delta_encoding.h",
]
defines = []
deps = [
":rtc_event_number_encodings",
"../api:rtp_headers",
"../api:rtp_parameters",
"../api/transport:network_control",
@ -330,6 +374,7 @@ if (rtc_enable_protobuf) {
":rtc_event_log2_proto",
":rtc_event_log_impl_encoder",
":rtc_event_log_proto",
":rtc_event_number_encodings",
":rtc_event_pacing",
":rtc_event_rtp_rtcp",
":rtc_event_video",
@ -369,6 +414,7 @@ if (rtc_enable_protobuf) {
"rtc_event_log/encoder/delta_encoding_unittest.cc",
"rtc_event_log/encoder/rtc_event_log_encoder_common_unittest.cc",
"rtc_event_log/encoder/rtc_event_log_encoder_unittest.cc",
"rtc_event_log/events/rtc_event_field_extraction_unittest.cc",
"rtc_event_log/rtc_event_log_unittest.cc",
"rtc_event_log/rtc_event_log_unittest_helper.cc",
"rtc_event_log/rtc_event_log_unittest_helper.h",
@ -378,12 +424,14 @@ if (rtc_enable_protobuf) {
":ice_log",
":rtc_event_audio",
":rtc_event_bwe",
":rtc_event_field",
":rtc_event_frame_events",
":rtc_event_generic_packet_events",
":rtc_event_log2_proto",
":rtc_event_log_impl_encoder",
":rtc_event_log_parser",
":rtc_event_log_proto",
":rtc_event_number_encodings",
":rtc_event_pacing",
":rtc_event_rtp_rtcp",
":rtc_event_video",

49
logging/rtc_event_log/encoder/bit_writer.cc Normal file
View File

@ -0,0 +1,49 @@
/*
* Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "logging/rtc_event_log/encoder/bit_writer.h"
namespace webrtc {
namespace {
size_t BitsToBytes(size_t bits) {
return (bits / 8) + (bits % 8 > 0 ? 1 : 0);
}
} // namespace
void BitWriter::WriteBits(uint64_t val, size_t bit_count) {
RTC_DCHECK(valid_);
const bool success = bit_writer_.WriteBits(val, bit_count);
RTC_DCHECK(success);
written_bits_ += bit_count;
}
void BitWriter::WriteBits(absl::string_view input) {
RTC_DCHECK(valid_);
for (char c : input) {
WriteBits(static_cast<unsigned char>(c), CHAR_BIT);
}
}
// Returns everything that was written so far.
// Nothing more may be written after this is called.
std::string BitWriter::GetString() {
RTC_DCHECK(valid_);
valid_ = false;
buffer_.resize(BitsToBytes(written_bits_));
written_bits_ = 0;
std::string result;
std::swap(buffer_, result);
return result;
}
} // namespace webrtc

61
logging/rtc_event_log/encoder/bit_writer.h Normal file
View File

@ -0,0 +1,61 @@
/*
* Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef LOGGING_RTC_EVENT_LOG_ENCODER_BIT_WRITER_H_
#define LOGGING_RTC_EVENT_LOG_ENCODER_BIT_WRITER_H_
#include <stddef.h>
#include <stdint.h>
#include <string>
#include <utility>
#include "absl/strings/string_view.h"
#include "rtc_base/bit_buffer.h"
#include "rtc_base/checks.h"
#include "rtc_base/constructor_magic.h"
namespace webrtc {
// Wrap BitBufferWriter and extend its functionality by (1) keeping track of
// the number of bits written and (2) owning its buffer.
class BitWriter final {
public:
explicit BitWriter(size_t byte_count)
: buffer_(byte_count, '\0'),
bit_writer_(reinterpret_cast<uint8_t*>(&buffer_[0]), buffer_.size()),
written_bits_(0),
valid_(true) {
RTC_DCHECK_GT(byte_count, 0);
}
void WriteBits(uint64_t val, size_t bit_count);
void WriteBits(absl::string_view input);
// Returns everything that was written so far.
// Nothing more may be written after this is called.
std::string GetString();
private:
std::string buffer_;
rtc::BitBufferWriter bit_writer_;
// Note: Counting bits instead of bytes wraps around earlier than it has to,
// which means the maximum length is lower than it could be. We don't expect
// to go anywhere near the limit, though, so this is good enough.
size_t written_bits_;
bool valid_;
RTC_DISALLOW_COPY_AND_ASSIGN(BitWriter);
};
} // namespace webrtc
#endif // LOGGING_RTC_EVENT_LOG_ENCODER_BIT_WRITER_H_

53
logging/rtc_event_log/encoder/delta_encoding.cc
View File

@ -16,6 +16,7 @@
#include <utility>
#include "absl/memory/memory.h"
#include "logging/rtc_event_log/encoder/bit_writer.h"
#include "logging/rtc_event_log/encoder/var_int.h"
#include "rtc_base/bit_buffer.h"
#include "rtc_base/bitstream_reader.h"
@ -107,58 +108,6 @@ constexpr bool kDefaultSignedDeltas = false;
constexpr bool kDefaultValuesOptional = false;
constexpr uint64_t kDefaultValueWidthBits = 64;
// Wrap BitBufferWriter and extend its functionality by (1) keeping track of
// the number of bits written and (2) owning its buffer.
class BitWriter final {
public:
explicit BitWriter(size_t byte_count)
: buffer_(byte_count, '\0'),
bit_writer_(reinterpret_cast<uint8_t*>(&buffer_[0]), buffer_.size()),
written_bits_(0),
valid_(true) {
RTC_DCHECK_GT(byte_count, 0);
}
void WriteBits(uint64_t val, size_t bit_count) {
RTC_DCHECK(valid_);
const bool success = bit_writer_.WriteBits(val, bit_count);
RTC_DCHECK(success);
written_bits_ += bit_count;
}
void WriteBits(const std::string& input) {
RTC_DCHECK(valid_);
for (std::string::value_type c : input) {
WriteBits(c, 8 * sizeof(std::string::value_type));
}
}
// Returns everything that was written so far.
// Nothing more may be written after this is called.
std::string GetString() {
RTC_DCHECK(valid_);
valid_ = false;
buffer_.resize(BitsToBytes(written_bits_));
written_bits_ = 0;
std::string result;
std::swap(buffer_, result);
return result;
}
private:
std::string buffer_;
rtc::BitBufferWriter bit_writer_;
// Note: Counting bits instead of bytes wraps around earlier than it has to,
// which means the maximum length is lower than it could be. We don't expect
// to go anywhere near the limit, though, so this is good enough.
size_t written_bits_;
bool valid_;
RTC_DISALLOW_COPY_AND_ASSIGN(BitWriter);
};
// Parameters for fixed-size delta-encoding/decoding.
// These are tailored for the sequence which will be encoded (e.g. widths).
class FixedLengthEncodingParameters final {

60
logging/rtc_event_log/events/rtc_event_field_extraction.cc Normal file
View File

@ -0,0 +1,60 @@
/*
* Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "logging/rtc_event_log/events/rtc_event_field_extraction.h"
#include <algorithm>
#include <limits>
#include "rtc_base/checks.h"
namespace webrtc_event_logging {
// The bitwidth required to encode values in the range
// [0, `max_pos_magnitude`] using an unsigned representation.
uint8_t UnsignedBitWidth(uint64_t max_magnitude) {
uint8_t required_bits = 1;
while (max_magnitude >>= 1) {
++required_bits;
}
return required_bits;
}
// The bitwidth required to encode signed values in the range
// [-`max_neg_magnitude`, `max_pos_magnitude`] using a signed
// 2-complement representation.
uint8_t SignedBitWidth(uint64_t max_pos_magnitude, uint64_t max_neg_magnitude) {
const uint8_t bitwidth_positive =
max_pos_magnitude > 0 ? UnsignedBitWidth(max_pos_magnitude) : 0;
const uint8_t bitwidth_negative =
(max_neg_magnitude > 1) ? UnsignedBitWidth(max_neg_magnitude - 1) : 0;
return 1 + std::max(bitwidth_positive, bitwidth_negative);
}
// Return the maximum integer of a given bit width.
uint64_t MaxUnsignedValueOfBitWidth(uint64_t bit_width) {
RTC_DCHECK_GE(bit_width, 1);
RTC_DCHECK_LE(bit_width, 64);
return (bit_width == 64) ? std::numeric_limits<uint64_t>::max()
: ((static_cast<uint64_t>(1) << bit_width) - 1);
}
// Computes the delta between `previous` and `current`, under the assumption
// that `bit_mask` is the largest value before wrap-around occurs. The bitmask
// must be of the form 2^x-1. (We use the wrap-around to more efficiently
// compress counters that wrap around at different bit widths than the
// backing C++ data type.)
uint64_t UnsignedDelta(uint64_t previous, uint64_t current, uint64_t bit_mask) {
RTC_DCHECK_LE(previous, bit_mask);
RTC_DCHECK_LE(current, bit_mask);
return (current - previous) & bit_mask;
}
} // namespace webrtc_event_logging

161
logging/rtc_event_log/events/rtc_event_field_extraction.h Normal file
View File

@ -0,0 +1,161 @@
/*
* Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef LOGGING_RTC_EVENT_LOG_EVENTS_RTC_EVENT_FIELD_EXTRACTION_H_
#define LOGGING_RTC_EVENT_LOG_EVENTS_RTC_EVENT_FIELD_EXTRACTION_H_
#include <string>
#include <vector>
#include "absl/types/optional.h"
#include "api/array_view.h"
#include "api/rtc_event_log/rtc_event.h"
#include "logging/rtc_event_log/encoder/rtc_event_log_encoder_common.h"
#include "rtc_base/logging.h"
namespace webrtc_event_logging {
uint8_t UnsignedBitWidth(uint64_t max_magnitude);
uint8_t SignedBitWidth(uint64_t max_pos_magnitude, uint64_t max_neg_magnitude);
uint64_t MaxUnsignedValueOfBitWidth(uint64_t bit_width);
uint64_t UnsignedDelta(uint64_t previous, uint64_t current, uint64_t bit_mask);
} // namespace webrtc_event_logging
namespace webrtc {
template <typename T, std::enable_if_t<std::is_signed<T>::value, bool> = true>
uint64_t EncodeAsUnsigned(T value) {
return webrtc_event_logging::ToUnsigned(value);
}
template <typename T, std::enable_if_t<std::is_unsigned<T>::value, bool> = true>
uint64_t EncodeAsUnsigned(T value) {
return static_cast<uint64_t>(value);
}
template <typename T, std::enable_if_t<std::is_signed<T>::value, bool> = true>
T DecodeFromUnsignedToType(uint64_t value) {
T signed_value = 0;
bool success = webrtc_event_logging::ToSigned<T>(value, &signed_value);
if (!success) {
RTC_LOG(LS_ERROR) << "Failed to convert " << value << "to signed type.";
// TODO(terelius): Propagate error?
}
return signed_value;
}
template <typename T, std::enable_if_t<std::is_unsigned<T>::value, bool> = true>
T DecodeFromUnsignedToType(uint64_t value) {
// TODO(terelius): Check range?
return static_cast<T>(value);
}
// Given a batch of RtcEvents and a member pointer, extract that
// member from each event in the batch. Signed integer members are
// encoded as unsigned, and the bitsize increased so the result can
// represented as a std::vector<uin64_t>.
// This is intended to be used in conjuction with
// EventEncoder::EncodeField to encode a batch of events as follows:
// auto values = ExtractRtcEventMember(batch, RtcEventFoo::timestamp_ms);
// encoder.EncodeField(timestamp_params, values)
template <typename T,
typename E,
std::enable_if_t<std::is_integral<T>::value, bool> = true>
std::vector<uint64_t> ExtractRtcEventMember(
rtc::ArrayView<const RtcEvent*> batch,
const T E::*member) {
std::vector<uint64_t> values;
values.reserve(batch.size());
for (const RtcEvent* event : batch) {
RTC_CHECK_EQ(event->GetType(), E::kType);
T value = static_cast<const E*>(event)->*member;
values.push_back(EncodeAsUnsigned(value));
}
return values;
}
// Represents a vector<optional<uint64_t>> optional_values
// as a bit-vector `position_mask` which identifies the positions
// of existing values, and a (potentially shorter)
// `vector<uint64_t> values` containing the actual values.
// The bit vector is constructed such that position_mask[i]
// is true iff optional_values[i] has a value, and `values.size()`
// is equal to the number of set bits in `position_mask`.
struct ValuesWithPositions {
std::vector<bool> position_mask;
std::vector<uint64_t> values;
};
// Same as above but for optional fields. It returns a struct
// containing a vector of positions in addition to the vector of values.
// The vector `positions` has the same length as the batch where
// `positions[i] == true` iff the batch[i]->member has a value.
// The values vector only contains the values that exists, so it
// may be shorter than the batch.
template <typename T,
typename E,
std::enable_if_t<std::is_integral<T>::value, bool> = true>
ValuesWithPositions ExtractRtcEventMember(rtc::ArrayView<const RtcEvent*> batch,
const absl::optional<T> E::*member) {
ValuesWithPositions result;
result.position_mask.reserve(batch.size());
result.values.reserve(batch.size());
for (const RtcEvent* event : batch) {
RTC_CHECK_EQ(event->GetType(), E::kType);
absl::optional<T> field = static_cast<const E*>(event)->*member;
result.position_mask.push_back(field.has_value());
if (field.has_value()) {
result.values.push_back(EncodeAsUnsigned(field.value()));
}
}
return result;
}
// Inverse of the ExtractRtcEventMember function used when parsing
// a log. Uses a vector of values to populate a specific field in a
// vector of structs.
template <typename T,
typename E,
std::enable_if_t<std::is_integral<T>::value, bool> = true>
void PopulateRtcEventMember(const std::vector<uint64_t>& values,
T E::*member,
rtc::ArrayView<E> output) {
size_t batch_size = values.size();
RTC_CHECK_EQ(output.size(), batch_size);
for (size_t i = 0; i < batch_size; ++i) {
output[i].*member = DecodeFromUnsignedToType<T>(values[i]);
}
}
// Same as above, but for optional fields.
template <typename T,
typename E,
std::enable_if_t<std::is_integral<T>::value, bool> = true>
void PopulateRtcEventMember(const std::vector<bool>& positions,
const std::vector<uint64_t>& values,
absl::optional<T> E::*member,
rtc::ArrayView<E> output) {
size_t batch_size = positions.size();
RTC_CHECK_EQ(output.size(), batch_size);
RTC_CHECK_LE(values.size(), batch_size);
auto value_it = values.begin();
for (size_t i = 0; i < batch_size; ++i) {
if (positions[i]) {
RTC_CHECK(value_it != values.end());
output[i].*member = DecodeFromUnsignedToType<T>(value_it);
++value_it;
} else {
output[i].*member = absl::nullopt;
}
}
RTC_CHECK(value_it == values.end());
}
} // namespace webrtc
#endif // LOGGING_RTC_EVENT_LOG_EVENTS_RTC_EVENT_FIELD_EXTRACTION_H_

97
logging/rtc_event_log/events/rtc_event_field_extraction_unittest.cc Normal file
View File

@ -0,0 +1,97 @@
/* Copyright (c) 2021 The WebRTC Project Authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "logging/rtc_event_log/events/rtc_event_field_extraction.h"
#include "rtc_base/random.h"
#include "test/gtest.h"
namespace webrtc {
TEST(UnsignedBitWidthTest, SmallValues) {
EXPECT_EQ(webrtc_event_logging::UnsignedBitWidth(0), 1u);
EXPECT_EQ(webrtc_event_logging::UnsignedBitWidth(1), 1u);
EXPECT_EQ(webrtc_event_logging::UnsignedBitWidth(2), 2u);
EXPECT_EQ(webrtc_event_logging::UnsignedBitWidth(3), 2u);
EXPECT_EQ(webrtc_event_logging::UnsignedBitWidth(4), 3u);
EXPECT_EQ(webrtc_event_logging::UnsignedBitWidth(5), 3u);
EXPECT_EQ(webrtc_event_logging::UnsignedBitWidth(6), 3u);
EXPECT_EQ(webrtc_event_logging::UnsignedBitWidth(7), 3u);
}
TEST(UnsignedBitWidthTest, PowersOfTwo) {
EXPECT_EQ(webrtc_event_logging::UnsignedBitWidth(0), 1u);
for (unsigned i = 0; i < 64; i++) {
uint64_t x = 1;
x = x << i;
EXPECT_EQ(webrtc_event_logging::UnsignedBitWidth(x), i + 1);
}
}
TEST(UnsignedBitWidthTest, PowersOfTwoMinusOne) {
for (unsigned i = 1; i < 64; i++) {
uint64_t x = 1;
x = (x << i) - 1;
EXPECT_EQ(webrtc_event_logging::UnsignedBitWidth(x), i);
}
uint64_t x = ~static_cast<uint64_t>(0);
EXPECT_EQ(webrtc_event_logging::UnsignedBitWidth(x), 64u);
}
TEST(UnsignedBitWidthTest, RandomInputs) {
Random rand(12345);
for (unsigned i = 0; i < 64; i++) {
uint64_t x = 1;
x = x << i;
uint64_t high = rand.Rand<uint32_t>();
uint64_t low = rand.Rand<uint32_t>();
x += ((high << 32) + low) % x;
EXPECT_EQ(webrtc_event_logging::UnsignedBitWidth(x), i + 1);
}
}
TEST(SignedBitWidthTest, SignedBitWidth) {
EXPECT_EQ(webrtc_event_logging::SignedBitWidth(0, 1), 1u);
EXPECT_EQ(webrtc_event_logging::SignedBitWidth(1, 0), 2u);
EXPECT_EQ(webrtc_event_logging::SignedBitWidth(1, 2), 2u);
EXPECT_EQ(webrtc_event_logging::SignedBitWidth(1, 128), 8u);
EXPECT_EQ(webrtc_event_logging::SignedBitWidth(127, 1), 8u);
EXPECT_EQ(webrtc_event_logging::SignedBitWidth(127, 128), 8u);
EXPECT_EQ(webrtc_event_logging::SignedBitWidth(1, 129), 9u);
EXPECT_EQ(webrtc_event_logging::SignedBitWidth(128, 1), 9u);
}
TEST(MaxUnsignedValueOfBitWidthTest, MaxUnsignedValueOfBitWidth) {
EXPECT_EQ(webrtc_event_logging::MaxUnsignedValueOfBitWidth(1), 0x01u);
EXPECT_EQ(webrtc_event_logging::MaxUnsignedValueOfBitWidth(6), 0x3Fu);
EXPECT_EQ(webrtc_event_logging::MaxUnsignedValueOfBitWidth(8), 0xFFu);
EXPECT_EQ(webrtc_event_logging::MaxUnsignedValueOfBitWidth(32), 0xFFFFFFFFu);
}
TEST(EncodeAsUnsignedTest, NegativeValues) {
// Negative values are converted as if cast to unsigned type of
// the same bitsize using 2-complement representation.
int16_t x = -1;
EXPECT_EQ(EncodeAsUnsigned(x), static_cast<uint64_t>(0xFFFF));
int64_t y = -1;
EXPECT_EQ(EncodeAsUnsigned(y), static_cast<uint64_t>(0xFFFFFFFFFFFFFFFFull));
}
TEST(EncodeAsUnsignedTest, PositiveValues) {
// Postive values are unchanged.
int16_t x = 42;
EXPECT_EQ(EncodeAsUnsigned(x), static_cast<uint64_t>(42));
int64_t y = 42;
EXPECT_EQ(EncodeAsUnsigned(y), static_cast<uint64_t>(42));
}
} // namespace webrtc