diff --git a/logging/BUILD.gn b/logging/BUILD.gn index 200a37795e..68f6092ade 100644 --- a/logging/BUILD.gn +++ b/logging/BUILD.gn @@ -32,6 +32,27 @@ rtc_source_set("rtc_event_log_api") { deps = [ "../api/rtc_event_log" ] } +rtc_library("rtc_event_field") { + sources = [ + "rtc_event_log/events/rtc_event_field_extraction.cc", + "rtc_event_log/events/rtc_event_field_extraction.h", + ] + + deps = [ + ":rtc_event_number_encodings", + "../api:array_view", + "../api/rtc_event_log", + "../rtc_base:checks", + "../rtc_base:logging", + "../rtc_base:rtc_base_approved", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/memory", + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] +} + rtc_library("rtc_stream_config") { sources = [ "rtc_event_log/rtc_stream_config.cc", @@ -190,14 +211,10 @@ rtc_library("rtc_event_video") { absl_deps = [ "//third_party/abseil-cpp/absl/memory" ] } -# TODO(eladalon): Break down into (1) encoder and (2) decoder; we don't need -# the decoder code in the WebRTC library, only in unit tests and tools. -rtc_library("rtc_event_log_impl_encoder") { +rtc_library("rtc_event_number_encodings") { sources = [ - "rtc_event_log/encoder/blob_encoding.cc", - "rtc_event_log/encoder/blob_encoding.h", - "rtc_event_log/encoder/delta_encoding.cc", - "rtc_event_log/encoder/delta_encoding.h", + "rtc_event_log/encoder/bit_writer.cc", + "rtc_event_log/encoder/bit_writer.h", "rtc_event_log/encoder/rtc_event_log_encoder_common.cc", "rtc_event_log/encoder/rtc_event_log_encoder_common.h", "rtc_event_log/encoder/var_int.cc", @@ -207,6 +224,33 @@ rtc_library("rtc_event_log_impl_encoder") { defines = [] deps = [ + "../rtc_base:bitstream_reader", + "../rtc_base:checks", + "../rtc_base:ignore_wundef", + "../rtc_base:macromagic", + "../rtc_base:rtc_base_approved", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/memory", + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] +} + +# TODO(eladalon): Break down into (1) encoder and (2) decoder; we don't need +# the decoder code in the WebRTC library, only in unit tests and tools. +rtc_library("rtc_event_log_impl_encoder") { + sources = [ + "rtc_event_log/encoder/blob_encoding.cc", + "rtc_event_log/encoder/blob_encoding.h", + "rtc_event_log/encoder/delta_encoding.cc", + "rtc_event_log/encoder/delta_encoding.h", + ] + + defines = [] + + deps = [ + ":rtc_event_number_encodings", "../api:rtp_headers", "../api:rtp_parameters", "../api/transport:network_control", @@ -330,6 +374,7 @@ if (rtc_enable_protobuf) { ":rtc_event_log2_proto", ":rtc_event_log_impl_encoder", ":rtc_event_log_proto", + ":rtc_event_number_encodings", ":rtc_event_pacing", ":rtc_event_rtp_rtcp", ":rtc_event_video", @@ -369,6 +414,7 @@ if (rtc_enable_protobuf) { "rtc_event_log/encoder/delta_encoding_unittest.cc", "rtc_event_log/encoder/rtc_event_log_encoder_common_unittest.cc", "rtc_event_log/encoder/rtc_event_log_encoder_unittest.cc", + "rtc_event_log/events/rtc_event_field_extraction_unittest.cc", "rtc_event_log/rtc_event_log_unittest.cc", "rtc_event_log/rtc_event_log_unittest_helper.cc", "rtc_event_log/rtc_event_log_unittest_helper.h", @@ -378,12 +424,14 @@ if (rtc_enable_protobuf) { ":ice_log", ":rtc_event_audio", ":rtc_event_bwe", + ":rtc_event_field", ":rtc_event_frame_events", ":rtc_event_generic_packet_events", ":rtc_event_log2_proto", ":rtc_event_log_impl_encoder", ":rtc_event_log_parser", ":rtc_event_log_proto", + ":rtc_event_number_encodings", ":rtc_event_pacing", ":rtc_event_rtp_rtcp", ":rtc_event_video", diff --git a/logging/rtc_event_log/encoder/bit_writer.cc b/logging/rtc_event_log/encoder/bit_writer.cc new file mode 100644 index 0000000000..e8748d3db3 --- /dev/null +++ b/logging/rtc_event_log/encoder/bit_writer.cc @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "logging/rtc_event_log/encoder/bit_writer.h" + +namespace webrtc { + +namespace { +size_t BitsToBytes(size_t bits) { + return (bits / 8) + (bits % 8 > 0 ? 1 : 0); +} +} // namespace + +void BitWriter::WriteBits(uint64_t val, size_t bit_count) { + RTC_DCHECK(valid_); + const bool success = bit_writer_.WriteBits(val, bit_count); + RTC_DCHECK(success); + written_bits_ += bit_count; +} + +void BitWriter::WriteBits(absl::string_view input) { + RTC_DCHECK(valid_); + for (char c : input) { + WriteBits(static_cast(c), CHAR_BIT); + } +} + +// Returns everything that was written so far. +// Nothing more may be written after this is called. +std::string BitWriter::GetString() { + RTC_DCHECK(valid_); + valid_ = false; + + buffer_.resize(BitsToBytes(written_bits_)); + written_bits_ = 0; + + std::string result; + std::swap(buffer_, result); + return result; +} + +} // namespace webrtc diff --git a/logging/rtc_event_log/encoder/bit_writer.h b/logging/rtc_event_log/encoder/bit_writer.h new file mode 100644 index 0000000000..85340c380d --- /dev/null +++ b/logging/rtc_event_log/encoder/bit_writer.h @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef LOGGING_RTC_EVENT_LOG_ENCODER_BIT_WRITER_H_ +#define LOGGING_RTC_EVENT_LOG_ENCODER_BIT_WRITER_H_ + +#include +#include + +#include +#include + +#include "absl/strings/string_view.h" +#include "rtc_base/bit_buffer.h" +#include "rtc_base/checks.h" +#include "rtc_base/constructor_magic.h" + +namespace webrtc { + +// Wrap BitBufferWriter and extend its functionality by (1) keeping track of +// the number of bits written and (2) owning its buffer. +class BitWriter final { + public: + explicit BitWriter(size_t byte_count) + : buffer_(byte_count, '\0'), + bit_writer_(reinterpret_cast(&buffer_[0]), buffer_.size()), + written_bits_(0), + valid_(true) { + RTC_DCHECK_GT(byte_count, 0); + } + + void WriteBits(uint64_t val, size_t bit_count); + + void WriteBits(absl::string_view input); + + // Returns everything that was written so far. + // Nothing more may be written after this is called. + std::string GetString(); + + private: + std::string buffer_; + rtc::BitBufferWriter bit_writer_; + // Note: Counting bits instead of bytes wraps around earlier than it has to, + // which means the maximum length is lower than it could be. We don't expect + // to go anywhere near the limit, though, so this is good enough. + size_t written_bits_; + bool valid_; + + RTC_DISALLOW_COPY_AND_ASSIGN(BitWriter); +}; + +} // namespace webrtc + +#endif // LOGGING_RTC_EVENT_LOG_ENCODER_BIT_WRITER_H_ diff --git a/logging/rtc_event_log/encoder/delta_encoding.cc b/logging/rtc_event_log/encoder/delta_encoding.cc index 437392f929..a96d3a7dc2 100644 --- a/logging/rtc_event_log/encoder/delta_encoding.cc +++ b/logging/rtc_event_log/encoder/delta_encoding.cc @@ -16,6 +16,7 @@ #include #include "absl/memory/memory.h" +#include "logging/rtc_event_log/encoder/bit_writer.h" #include "logging/rtc_event_log/encoder/var_int.h" #include "rtc_base/bit_buffer.h" #include "rtc_base/bitstream_reader.h" @@ -107,58 +108,6 @@ constexpr bool kDefaultSignedDeltas = false; constexpr bool kDefaultValuesOptional = false; constexpr uint64_t kDefaultValueWidthBits = 64; -// Wrap BitBufferWriter and extend its functionality by (1) keeping track of -// the number of bits written and (2) owning its buffer. -class BitWriter final { - public: - explicit BitWriter(size_t byte_count) - : buffer_(byte_count, '\0'), - bit_writer_(reinterpret_cast(&buffer_[0]), buffer_.size()), - written_bits_(0), - valid_(true) { - RTC_DCHECK_GT(byte_count, 0); - } - - void WriteBits(uint64_t val, size_t bit_count) { - RTC_DCHECK(valid_); - const bool success = bit_writer_.WriteBits(val, bit_count); - RTC_DCHECK(success); - written_bits_ += bit_count; - } - - void WriteBits(const std::string& input) { - RTC_DCHECK(valid_); - for (std::string::value_type c : input) { - WriteBits(c, 8 * sizeof(std::string::value_type)); - } - } - - // Returns everything that was written so far. - // Nothing more may be written after this is called. - std::string GetString() { - RTC_DCHECK(valid_); - valid_ = false; - - buffer_.resize(BitsToBytes(written_bits_)); - written_bits_ = 0; - - std::string result; - std::swap(buffer_, result); - return result; - } - - private: - std::string buffer_; - rtc::BitBufferWriter bit_writer_; - // Note: Counting bits instead of bytes wraps around earlier than it has to, - // which means the maximum length is lower than it could be. We don't expect - // to go anywhere near the limit, though, so this is good enough. - size_t written_bits_; - bool valid_; - - RTC_DISALLOW_COPY_AND_ASSIGN(BitWriter); -}; - // Parameters for fixed-size delta-encoding/decoding. // These are tailored for the sequence which will be encoded (e.g. widths). class FixedLengthEncodingParameters final { diff --git a/logging/rtc_event_log/events/rtc_event_field_extraction.cc b/logging/rtc_event_log/events/rtc_event_field_extraction.cc new file mode 100644 index 0000000000..99f0b3697c --- /dev/null +++ b/logging/rtc_event_log/events/rtc_event_field_extraction.cc @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "logging/rtc_event_log/events/rtc_event_field_extraction.h" + +#include +#include + +#include "rtc_base/checks.h" + +namespace webrtc_event_logging { + +// The bitwidth required to encode values in the range +// [0, `max_pos_magnitude`] using an unsigned representation. +uint8_t UnsignedBitWidth(uint64_t max_magnitude) { + uint8_t required_bits = 1; + while (max_magnitude >>= 1) { + ++required_bits; + } + return required_bits; +} + +// The bitwidth required to encode signed values in the range +// [-`max_neg_magnitude`, `max_pos_magnitude`] using a signed +// 2-complement representation. +uint8_t SignedBitWidth(uint64_t max_pos_magnitude, uint64_t max_neg_magnitude) { + const uint8_t bitwidth_positive = + max_pos_magnitude > 0 ? UnsignedBitWidth(max_pos_magnitude) : 0; + const uint8_t bitwidth_negative = + (max_neg_magnitude > 1) ? UnsignedBitWidth(max_neg_magnitude - 1) : 0; + return 1 + std::max(bitwidth_positive, bitwidth_negative); +} + +// Return the maximum integer of a given bit width. +uint64_t MaxUnsignedValueOfBitWidth(uint64_t bit_width) { + RTC_DCHECK_GE(bit_width, 1); + RTC_DCHECK_LE(bit_width, 64); + return (bit_width == 64) ? std::numeric_limits::max() + : ((static_cast(1) << bit_width) - 1); +} + +// Computes the delta between `previous` and `current`, under the assumption +// that `bit_mask` is the largest value before wrap-around occurs. The bitmask +// must be of the form 2^x-1. (We use the wrap-around to more efficiently +// compress counters that wrap around at different bit widths than the +// backing C++ data type.) +uint64_t UnsignedDelta(uint64_t previous, uint64_t current, uint64_t bit_mask) { + RTC_DCHECK_LE(previous, bit_mask); + RTC_DCHECK_LE(current, bit_mask); + return (current - previous) & bit_mask; +} + +} // namespace webrtc_event_logging diff --git a/logging/rtc_event_log/events/rtc_event_field_extraction.h b/logging/rtc_event_log/events/rtc_event_field_extraction.h new file mode 100644 index 0000000000..a0e8ff01c7 --- /dev/null +++ b/logging/rtc_event_log/events/rtc_event_field_extraction.h @@ -0,0 +1,161 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef LOGGING_RTC_EVENT_LOG_EVENTS_RTC_EVENT_FIELD_EXTRACTION_H_ +#define LOGGING_RTC_EVENT_LOG_EVENTS_RTC_EVENT_FIELD_EXTRACTION_H_ + +#include +#include + +#include "absl/types/optional.h" +#include "api/array_view.h" +#include "api/rtc_event_log/rtc_event.h" +#include "logging/rtc_event_log/encoder/rtc_event_log_encoder_common.h" +#include "rtc_base/logging.h" + +namespace webrtc_event_logging { +uint8_t UnsignedBitWidth(uint64_t max_magnitude); +uint8_t SignedBitWidth(uint64_t max_pos_magnitude, uint64_t max_neg_magnitude); +uint64_t MaxUnsignedValueOfBitWidth(uint64_t bit_width); +uint64_t UnsignedDelta(uint64_t previous, uint64_t current, uint64_t bit_mask); +} // namespace webrtc_event_logging + +namespace webrtc { +template ::value, bool> = true> +uint64_t EncodeAsUnsigned(T value) { + return webrtc_event_logging::ToUnsigned(value); +} + +template ::value, bool> = true> +uint64_t EncodeAsUnsigned(T value) { + return static_cast(value); +} + +template ::value, bool> = true> +T DecodeFromUnsignedToType(uint64_t value) { + T signed_value = 0; + bool success = webrtc_event_logging::ToSigned(value, &signed_value); + if (!success) { + RTC_LOG(LS_ERROR) << "Failed to convert " << value << "to signed type."; + // TODO(terelius): Propagate error? + } + return signed_value; +} + +template ::value, bool> = true> +T DecodeFromUnsignedToType(uint64_t value) { + // TODO(terelius): Check range? + return static_cast(value); +} + +// Given a batch of RtcEvents and a member pointer, extract that +// member from each event in the batch. Signed integer members are +// encoded as unsigned, and the bitsize increased so the result can +// represented as a std::vector. +// This is intended to be used in conjuction with +// EventEncoder::EncodeField to encode a batch of events as follows: +// auto values = ExtractRtcEventMember(batch, RtcEventFoo::timestamp_ms); +// encoder.EncodeField(timestamp_params, values) +template ::value, bool> = true> +std::vector ExtractRtcEventMember( + rtc::ArrayView batch, + const T E::*member) { + std::vector values; + values.reserve(batch.size()); + for (const RtcEvent* event : batch) { + RTC_CHECK_EQ(event->GetType(), E::kType); + T value = static_cast(event)->*member; + values.push_back(EncodeAsUnsigned(value)); + } + return values; +} + +// Represents a vector> optional_values +// as a bit-vector `position_mask` which identifies the positions +// of existing values, and a (potentially shorter) +// `vector values` containing the actual values. +// The bit vector is constructed such that position_mask[i] +// is true iff optional_values[i] has a value, and `values.size()` +// is equal to the number of set bits in `position_mask`. +struct ValuesWithPositions { + std::vector position_mask; + std::vector values; +}; + +// Same as above but for optional fields. It returns a struct +// containing a vector of positions in addition to the vector of values. +// The vector `positions` has the same length as the batch where +// `positions[i] == true` iff the batch[i]->member has a value. +// The values vector only contains the values that exists, so it +// may be shorter than the batch. +template ::value, bool> = true> +ValuesWithPositions ExtractRtcEventMember(rtc::ArrayView batch, + const absl::optional E::*member) { + ValuesWithPositions result; + result.position_mask.reserve(batch.size()); + result.values.reserve(batch.size()); + for (const RtcEvent* event : batch) { + RTC_CHECK_EQ(event->GetType(), E::kType); + absl::optional field = static_cast(event)->*member; + result.position_mask.push_back(field.has_value()); + if (field.has_value()) { + result.values.push_back(EncodeAsUnsigned(field.value())); + } + } + return result; +} + +// Inverse of the ExtractRtcEventMember function used when parsing +// a log. Uses a vector of values to populate a specific field in a +// vector of structs. +template ::value, bool> = true> +void PopulateRtcEventMember(const std::vector& values, + T E::*member, + rtc::ArrayView output) { + size_t batch_size = values.size(); + RTC_CHECK_EQ(output.size(), batch_size); + for (size_t i = 0; i < batch_size; ++i) { + output[i].*member = DecodeFromUnsignedToType(values[i]); + } +} + +// Same as above, but for optional fields. +template ::value, bool> = true> +void PopulateRtcEventMember(const std::vector& positions, + const std::vector& values, + absl::optional E::*member, + rtc::ArrayView output) { + size_t batch_size = positions.size(); + RTC_CHECK_EQ(output.size(), batch_size); + RTC_CHECK_LE(values.size(), batch_size); + auto value_it = values.begin(); + for (size_t i = 0; i < batch_size; ++i) { + if (positions[i]) { + RTC_CHECK(value_it != values.end()); + output[i].*member = DecodeFromUnsignedToType(value_it); + ++value_it; + } else { + output[i].*member = absl::nullopt; + } + } + RTC_CHECK(value_it == values.end()); +} + +} // namespace webrtc + +#endif // LOGGING_RTC_EVENT_LOG_EVENTS_RTC_EVENT_FIELD_EXTRACTION_H_ diff --git a/logging/rtc_event_log/events/rtc_event_field_extraction_unittest.cc b/logging/rtc_event_log/events/rtc_event_field_extraction_unittest.cc new file mode 100644 index 0000000000..f9fb993af0 --- /dev/null +++ b/logging/rtc_event_log/events/rtc_event_field_extraction_unittest.cc @@ -0,0 +1,97 @@ +/* Copyright (c) 2021 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "logging/rtc_event_log/events/rtc_event_field_extraction.h" + +#include "rtc_base/random.h" +#include "test/gtest.h" + +namespace webrtc { + +TEST(UnsignedBitWidthTest, SmallValues) { + EXPECT_EQ(webrtc_event_logging::UnsignedBitWidth(0), 1u); + EXPECT_EQ(webrtc_event_logging::UnsignedBitWidth(1), 1u); + EXPECT_EQ(webrtc_event_logging::UnsignedBitWidth(2), 2u); + EXPECT_EQ(webrtc_event_logging::UnsignedBitWidth(3), 2u); + EXPECT_EQ(webrtc_event_logging::UnsignedBitWidth(4), 3u); + EXPECT_EQ(webrtc_event_logging::UnsignedBitWidth(5), 3u); + EXPECT_EQ(webrtc_event_logging::UnsignedBitWidth(6), 3u); + EXPECT_EQ(webrtc_event_logging::UnsignedBitWidth(7), 3u); +} + +TEST(UnsignedBitWidthTest, PowersOfTwo) { + EXPECT_EQ(webrtc_event_logging::UnsignedBitWidth(0), 1u); + + for (unsigned i = 0; i < 64; i++) { + uint64_t x = 1; + x = x << i; + EXPECT_EQ(webrtc_event_logging::UnsignedBitWidth(x), i + 1); + } +} + +TEST(UnsignedBitWidthTest, PowersOfTwoMinusOne) { + for (unsigned i = 1; i < 64; i++) { + uint64_t x = 1; + x = (x << i) - 1; + EXPECT_EQ(webrtc_event_logging::UnsignedBitWidth(x), i); + } + + uint64_t x = ~static_cast(0); + EXPECT_EQ(webrtc_event_logging::UnsignedBitWidth(x), 64u); +} + +TEST(UnsignedBitWidthTest, RandomInputs) { + Random rand(12345); + + for (unsigned i = 0; i < 64; i++) { + uint64_t x = 1; + x = x << i; + uint64_t high = rand.Rand(); + uint64_t low = rand.Rand(); + x += ((high << 32) + low) % x; + EXPECT_EQ(webrtc_event_logging::UnsignedBitWidth(x), i + 1); + } +} + +TEST(SignedBitWidthTest, SignedBitWidth) { + EXPECT_EQ(webrtc_event_logging::SignedBitWidth(0, 1), 1u); + EXPECT_EQ(webrtc_event_logging::SignedBitWidth(1, 0), 2u); + EXPECT_EQ(webrtc_event_logging::SignedBitWidth(1, 2), 2u); + EXPECT_EQ(webrtc_event_logging::SignedBitWidth(1, 128), 8u); + EXPECT_EQ(webrtc_event_logging::SignedBitWidth(127, 1), 8u); + EXPECT_EQ(webrtc_event_logging::SignedBitWidth(127, 128), 8u); + EXPECT_EQ(webrtc_event_logging::SignedBitWidth(1, 129), 9u); + EXPECT_EQ(webrtc_event_logging::SignedBitWidth(128, 1), 9u); +} + +TEST(MaxUnsignedValueOfBitWidthTest, MaxUnsignedValueOfBitWidth) { + EXPECT_EQ(webrtc_event_logging::MaxUnsignedValueOfBitWidth(1), 0x01u); + EXPECT_EQ(webrtc_event_logging::MaxUnsignedValueOfBitWidth(6), 0x3Fu); + EXPECT_EQ(webrtc_event_logging::MaxUnsignedValueOfBitWidth(8), 0xFFu); + EXPECT_EQ(webrtc_event_logging::MaxUnsignedValueOfBitWidth(32), 0xFFFFFFFFu); +} + +TEST(EncodeAsUnsignedTest, NegativeValues) { + // Negative values are converted as if cast to unsigned type of + // the same bitsize using 2-complement representation. + int16_t x = -1; + EXPECT_EQ(EncodeAsUnsigned(x), static_cast(0xFFFF)); + int64_t y = -1; + EXPECT_EQ(EncodeAsUnsigned(y), static_cast(0xFFFFFFFFFFFFFFFFull)); +} + +TEST(EncodeAsUnsignedTest, PositiveValues) { + // Postive values are unchanged. + int16_t x = 42; + EXPECT_EQ(EncodeAsUnsigned(x), static_cast(42)); + int64_t y = 42; + EXPECT_EQ(EncodeAsUnsigned(y), static_cast(42)); +} + +} // namespace webrtc