diff --git a/logging/BUILD.gn b/logging/BUILD.gn index ac0b720f33..571ddf8c30 100644 --- a/logging/BUILD.gn +++ b/logging/BUILD.gn @@ -183,6 +183,7 @@ rtc_static_library("rtc_event_log_impl_encoder") { "../rtc_base:rtc_base_approved", "//third_party/abseil-cpp/absl/memory", "//third_party/abseil-cpp/absl/strings:strings", + "//third_party/abseil-cpp/absl/types:optional", ] if (rtc_enable_protobuf) { diff --git a/logging/rtc_event_log/encoder/delta_encoding.cc b/logging/rtc_event_log/encoder/delta_encoding.cc index 2da4ca74cf..755eb41ec8 100644 --- a/logging/rtc_event_log/encoder/delta_encoding.cc +++ b/logging/rtc_event_log/encoder/delta_encoding.cc @@ -27,12 +27,19 @@ namespace { // TODO(eladalon): Only build the decoder in tools and unit tests. +bool g_force_unsigned_for_testing = false; +bool g_force_signed_for_testing = false; + size_t BitsToBytes(size_t bits) { return (bits / 8) + (bits % 8 > 0 ? 1 : 0); } // TODO(eladalon): Replace by something more efficient. -uint64_t BitWidth(uint64_t input) { +uint64_t UnsignedBitWidth(uint64_t input, bool zero_val_as_zero_width = false) { + if (zero_val_as_zero_width && input == 0) { + return 0; + } + uint64_t width = 0; do { // input == 0 -> width == 1 width += 1; @@ -41,13 +48,22 @@ uint64_t BitWidth(uint64_t input) { return width; } +uint64_t SignedBitWidth(uint64_t max_pos_magnitude, + uint64_t max_neg_magnitude) { + const uint64_t bitwidth_pos = UnsignedBitWidth(max_pos_magnitude, true); + const uint64_t bitwidth_neg = + (max_neg_magnitude > 0) ? UnsignedBitWidth(max_neg_magnitude - 1, true) + : 0; + return 1 + std::max(bitwidth_pos, bitwidth_neg); +} + // Return the maximum integer of a given bit width. // Examples: -// MaxValueOfBitWidth(1) = 0x01 -// MaxValueOfBitWidth(6) = 0x3f -// MaxValueOfBitWidth(8) = 0xff -// MaxValueOfBitWidth(32) = 0xffffffff -uint64_t MaxValueOfBitWidth(uint64_t bit_width) { +// MaxUnsignedValueOfBitWidth(1) = 0x01 +// MaxUnsignedValueOfBitWidth(6) = 0x3f +// MaxUnsignedValueOfBitWidth(8) = 0xff +// MaxUnsignedValueOfBitWidth(32) = 0xffffffff +uint64_t MaxUnsignedValueOfBitWidth(uint64_t bit_width) { RTC_DCHECK_GE(bit_width, 1); RTC_DCHECK_LE(bit_width, 64); return (bit_width == 64) ? std::numeric_limits::max() @@ -55,18 +71,9 @@ uint64_t MaxValueOfBitWidth(uint64_t bit_width) { } // Computes the delta between |previous| and |current|, under the assumption -// that wrap-around occurs after width |bit_width| is exceeded. -uint64_t ComputeDelta(uint64_t previous, uint64_t current, uint64_t width) { - RTC_DCHECK(width == 64 || current < (static_cast(1) << width)); - RTC_DCHECK(width == 64 || previous < (static_cast(1) << width)); - - if (current >= previous) { - // Simply "walk" forward. - return current - previous; - } else { // previous > current - // "Walk" until the max value, one more step to 0, then to |current|. - return (MaxValueOfBitWidth(width) - previous) + 1 + current; - } +// that wrap-around occurs after |width| is exceeded. +uint64_t UnsignedDelta(uint64_t previous, uint64_t current, uint64_t bit_mask) { + return (current - previous) & bit_mask; } // Determines the encoding type (e.g. fixed-size encoding). @@ -87,7 +94,7 @@ constexpr size_t kBitsInHeaderForEncodingType = 2; constexpr size_t kBitsInHeaderForDeltaWidthBits = 6; constexpr size_t kBitsInHeaderForSignedDeltas = 1; constexpr size_t kBitsInHeaderForValuesOptional = 1; -constexpr size_t kBitsInHeaderForOriginalWidthBits = 6; +constexpr size_t kBitsInHeaderForValueWidthBits = 6; static_assert(static_cast(EncodingType::kNumberOfEncodingTypes) <= 1 << kBitsInHeaderForEncodingType, @@ -96,7 +103,7 @@ static_assert(static_cast(EncodingType::kNumberOfEncodingTypes) <= // Default values for when the encoding header does not specify explicitly. constexpr bool kDefaultSignedDeltas = false; constexpr bool kDefaultValuesOptional = false; -constexpr uint64_t kDefaultOriginalWidthBits = 64; +constexpr uint64_t kDefaultValueWidthBits = 64; // Wrap BitBufferWriter and extend its functionality by (1) keeping track of // the number of bits written and (2) owning its buffer. @@ -145,33 +152,66 @@ class BitWriter final { // Parameters for fixed-size delta-encoding/decoding. // These are tailored for the sequence which will be encoded (e.g. widths). -struct FixedLengthEncodingParameters final { +class FixedLengthEncodingParameters final { + public: + static bool ValidParameters(uint64_t delta_width_bits, + bool signed_deltas, + bool values_optional, + uint64_t value_width_bits) { + return (1 <= delta_width_bits && delta_width_bits <= 64 && + 1 <= value_width_bits && value_width_bits <= 64 && + delta_width_bits <= value_width_bits); + } + FixedLengthEncodingParameters(uint64_t delta_width_bits, bool signed_deltas, bool values_optional, - uint64_t original_width_bits) - : delta_width_bits(delta_width_bits), - signed_deltas(signed_deltas), - values_optional(values_optional), - original_width_bits(original_width_bits) {} + uint64_t value_width_bits) + : delta_width_bits_(delta_width_bits), + signed_deltas_(signed_deltas), + values_optional_(values_optional), + value_width_bits_(value_width_bits), + delta_mask_(MaxUnsignedValueOfBitWidth(delta_width_bits_)), + value_mask_(MaxUnsignedValueOfBitWidth(value_width_bits_)) { + RTC_DCHECK(ValidParameters(delta_width_bits, signed_deltas, values_optional, + value_width_bits)); + } // Number of bits necessary to hold the widest(*) of the deltas between the // values in the sequence. // (*) - Widest might not be the largest, if signed deltas are used. - uint64_t delta_width_bits; + uint64_t delta_width_bits() const { return delta_width_bits_; } // Whether deltas are signed. - // TODO(eladalon): Add support for signed deltas. - bool signed_deltas; + bool signed_deltas() const { return signed_deltas_; } // Whether the values of the sequence are optional. That is, it may be // that some of them do not have a value (not even a sentinel value indicating // invalidity). // TODO(eladalon): Add support for optional elements. - bool values_optional; + bool values_optional() const { return values_optional_; } // Number of bits necessary to hold the largest value in the sequence. - uint64_t original_width_bits; + uint64_t value_width_bits() const { return value_width_bits_; } + + // Masks where only the bits relevant to the deltas/values are turned on. + uint64_t delta_mask() const { return delta_mask_; } + uint64_t value_mask() const { return value_mask_; } + + void SetSignedDeltas(bool signed_deltas) { signed_deltas_ = signed_deltas; } + void SetDeltaWidthBits(uint64_t delta_width_bits) { + delta_width_bits_ = delta_width_bits; + delta_mask_ = MaxUnsignedValueOfBitWidth(delta_width_bits); + } + + private: + uint64_t delta_width_bits_; // Normally const, but mutable in tests. + bool signed_deltas_; // Normally const, but mutable in tests. + const bool values_optional_; + const uint64_t value_width_bits_; + + uint64_t delta_mask_; // Normally const, but mutable in tests. + const uint64_t value_mask_; }; // Performs delta-encoding of a single (non-empty) sequence of values, using @@ -190,6 +230,21 @@ class FixedLengthDeltaEncoder final { const std::vector& values); private: + // Calculate min/max values of unsigned/signed deltas, given the bit width + // of all the values in the series. + static void CalculateMinAndMaxDeltas(uint64_t base, + const std::vector& values, + uint64_t bit_width, + uint64_t* max_unsigned_delta, + uint64_t* max_pos_signed_delta, + uint64_t* min_neg_signed_delta); + + // No effect outside of unit tests. + // In unit tests, may lead to forcing signed/unsigned deltas, etc. + static void ConsiderTestOverrides(FixedLengthEncodingParameters* params, + uint64_t delta_width_bits_signed, + uint64_t delta_width_bits_unsigned); + // FixedLengthDeltaEncoder objects are to be created by EncodeDeltas() and // released by it before it returns. They're mostly a convenient way to // avoid having to pass a lot of state between different functions. @@ -213,6 +268,8 @@ class FixedLengthDeltaEncoder final { // Encode a given delta into the stream. void EncodeDelta(uint64_t previous, uint64_t current); + void EncodeUnsignedDelta(uint64_t previous, uint64_t current); + void EncodeSignedDelta(uint64_t previous, uint64_t current); // The parameters according to which encoding will be done (width of // fields, whether signed deltas should be used, etc.) @@ -251,16 +308,14 @@ std::string FixedLengthDeltaEncoder::EncodeDeltas( // If the sequence is non-decreasing, it may be assumed to have width = 64; // there's no reason to encode the actual max width in the encoding header. - const uint64_t original_width_bits = - non_decreasing ? 64 : BitWidth(max_value_including_base); + const uint64_t value_width_bits = + non_decreasing ? 64 : UnsignedBitWidth(max_value_including_base); - uint64_t max_unsigned_delta = - ComputeDelta(base, values[0], original_width_bits); - for (size_t i = 1; i < values.size(); ++i) { - const uint64_t unsigned_delta = - ComputeDelta(values[i - 1], values[i], original_width_bits); - max_unsigned_delta = std::max(unsigned_delta, max_unsigned_delta); - } + uint64_t max_unsigned_delta; + uint64_t max_pos_signed_delta; + uint64_t min_neg_signed_delta; + CalculateMinAndMaxDeltas(base, values, value_width_bits, &max_unsigned_delta, + &max_pos_signed_delta, &min_neg_signed_delta); // We indicate the special case of all values being equal to the base with // the empty string. @@ -270,11 +325,10 @@ std::string FixedLengthDeltaEncoder::EncodeDeltas( return std::string(); } - const uint64_t delta_width_bits_unsigned = BitWidth(max_unsigned_delta); - // This will prevent the use of signed deltas, by always assuming - // they will not provide value over unsigned. - // TODO(eladalon): Add support for signed deltas. - const uint64_t delta_width_bits_signed = 64; + const uint64_t delta_width_bits_unsigned = + UnsignedBitWidth(max_unsigned_delta); + const uint64_t delta_width_bits_signed = + SignedBitWidth(max_pos_signed_delta, min_neg_signed_delta); // Note: Preference for unsigned if the two have the same width (efficiency). const bool signed_deltas = @@ -285,21 +339,75 @@ std::string FixedLengthDeltaEncoder::EncodeDeltas( const bool values_optional = false; FixedLengthEncodingParameters params(delta_width_bits, signed_deltas, - values_optional, original_width_bits); + values_optional, value_width_bits); + + // No effect in production. + ConsiderTestOverrides(¶ms, delta_width_bits_signed, + delta_width_bits_unsigned); + FixedLengthDeltaEncoder encoder(params, base, values); return encoder.Encode(); } +void FixedLengthDeltaEncoder::CalculateMinAndMaxDeltas( + uint64_t base, + const std::vector& values, + uint64_t bit_width, + uint64_t* max_unsigned_delta_out, + uint64_t* max_pos_signed_delta_out, + uint64_t* min_neg_signed_delta_out) { + RTC_DCHECK(!values.empty()); + RTC_DCHECK(max_unsigned_delta_out); + RTC_DCHECK(max_pos_signed_delta_out); + RTC_DCHECK(min_neg_signed_delta_out); + + const uint64_t bit_mask = MaxUnsignedValueOfBitWidth(bit_width); + + uint64_t max_unsigned_delta = 0; + uint64_t max_pos_signed_delta = 0; + uint64_t min_neg_signed_delta = 0; + + uint64_t prev = base; + for (size_t i = 0; i < values.size(); ++i) { + const uint64_t forward_delta = UnsignedDelta(prev, values[i], bit_mask); + const uint64_t backward_delta = UnsignedDelta(values[i], prev, bit_mask); + + max_unsigned_delta = std::max(max_unsigned_delta, forward_delta); + + if (forward_delta < backward_delta) { + max_pos_signed_delta = std::max(max_pos_signed_delta, forward_delta); + } else { + min_neg_signed_delta = std::max(min_neg_signed_delta, backward_delta); + } + + prev = values[i]; + } + + *max_unsigned_delta_out = max_unsigned_delta; + *max_pos_signed_delta_out = max_pos_signed_delta; + *min_neg_signed_delta_out = min_neg_signed_delta; +} + +void FixedLengthDeltaEncoder::ConsiderTestOverrides( + FixedLengthEncodingParameters* params, + uint64_t delta_width_bits_signed, + uint64_t delta_width_bits_unsigned) { + if (g_force_unsigned_for_testing) { + params->SetDeltaWidthBits(delta_width_bits_unsigned); + params->SetSignedDeltas(false); + } else if (g_force_signed_for_testing) { + params->SetDeltaWidthBits(delta_width_bits_signed); + params->SetSignedDeltas(true); + } else { + // Unchanged. + } +} + FixedLengthDeltaEncoder::FixedLengthDeltaEncoder( const FixedLengthEncodingParameters& params, uint64_t base, const std::vector& values) : params_(params), base_(base), values_(values) { - RTC_DCHECK_GE(params_.delta_width_bits, 1); - RTC_DCHECK_LE(params_.delta_width_bits, 64); - RTC_DCHECK_GE(params_.original_width_bits, 1); - RTC_DCHECK_LE(params_.original_width_bits, 64); - RTC_DCHECK_LE(params_.delta_width_bits, params_.original_width_bits); RTC_DCHECK(!values_.empty()); writer_ = absl::make_unique(OutputLengthBytes()); } @@ -322,31 +430,31 @@ size_t FixedLengthDeltaEncoder::OutputLengthBytes() const { } size_t FixedLengthDeltaEncoder::HeaderLengthBits() const { - if (params_.signed_deltas == kDefaultSignedDeltas && - params_.values_optional == kDefaultValuesOptional && - params_.original_width_bits == kDefaultOriginalWidthBits) { + if (params_.signed_deltas() == kDefaultSignedDeltas && + params_.values_optional() == kDefaultValuesOptional && + params_.value_width_bits() == kDefaultValueWidthBits) { return kBitsInHeaderForEncodingType + kBitsInHeaderForDeltaWidthBits; } else { return kBitsInHeaderForEncodingType + kBitsInHeaderForDeltaWidthBits + kBitsInHeaderForSignedDeltas + kBitsInHeaderForValuesOptional + - kBitsInHeaderForOriginalWidthBits; + kBitsInHeaderForValueWidthBits; } } size_t FixedLengthDeltaEncoder::EncodedDeltasLengthBits() const { // TODO(eladalon): When optional values are supported, iterate over the // deltas to check the exact cost of each. - RTC_DCHECK(!params_.values_optional); - return values_.size() * params_.delta_width_bits; + RTC_DCHECK(!params_.values_optional()); + return values_.size() * params_.delta_width_bits(); } void FixedLengthDeltaEncoder::EncodeHeader() { RTC_DCHECK(writer_); const EncodingType encoding_type = - (params_.original_width_bits == kDefaultOriginalWidthBits && - params_.signed_deltas == kDefaultSignedDeltas && - params_.values_optional == kDefaultValuesOptional) + (params_.value_width_bits() == kDefaultValueWidthBits && + params_.signed_deltas() == kDefaultSignedDeltas && + params_.values_optional() == kDefaultValuesOptional) ? EncodingType::kFixedSizeUnsignedDeltasNoEarlyWrapNoOpt : EncodingType::kFixedSizeSignedDeltasEarlyWrapAndOptSupported; @@ -356,26 +464,59 @@ void FixedLengthDeltaEncoder::EncodeHeader() { // Note: Since it's meaningless for a field to be of width 0, when it comes // to fields that relate widths, we encode width 1 as 0, width 2 as 1, - writer_->WriteBits(params_.delta_width_bits - 1, + writer_->WriteBits(params_.delta_width_bits() - 1, kBitsInHeaderForDeltaWidthBits); if (encoding_type == EncodingType::kFixedSizeUnsignedDeltasNoEarlyWrapNoOpt) { return; } - writer_->WriteBits(static_cast(params_.signed_deltas), + writer_->WriteBits(static_cast(params_.signed_deltas()), kBitsInHeaderForSignedDeltas); - writer_->WriteBits(static_cast(params_.values_optional), + writer_->WriteBits(static_cast(params_.values_optional()), kBitsInHeaderForValuesOptional); - writer_->WriteBits(params_.original_width_bits - 1, - kBitsInHeaderForOriginalWidthBits); + writer_->WriteBits(params_.value_width_bits() - 1, + kBitsInHeaderForValueWidthBits); } void FixedLengthDeltaEncoder::EncodeDelta(uint64_t previous, uint64_t current) { + if (params_.signed_deltas()) { + EncodeSignedDelta(previous, current); + } else { + EncodeUnsignedDelta(previous, current); + } +} + +void FixedLengthDeltaEncoder::EncodeUnsignedDelta(uint64_t previous, + uint64_t current) { RTC_DCHECK(writer_); - writer_->WriteBits( - ComputeDelta(previous, current, params_.original_width_bits), - params_.delta_width_bits); + const uint64_t delta = UnsignedDelta(previous, current, params_.value_mask()); + writer_->WriteBits(delta, params_.delta_width_bits()); +} + +void FixedLengthDeltaEncoder::EncodeSignedDelta(uint64_t previous, + uint64_t current) { + RTC_DCHECK(writer_); + + const uint64_t forward_delta = + UnsignedDelta(previous, current, params_.value_mask()); + const uint64_t backward_delta = + UnsignedDelta(current, previous, params_.value_mask()); + + uint64_t delta; + if (forward_delta <= backward_delta) { + delta = forward_delta; + } else { + // Compute the unsigned representation of a negative delta. + // This is the two's complement representation of this negative value, + // when deltas are of width params_.delta_mask(). + RTC_DCHECK_GE(params_.delta_mask(), backward_delta); + RTC_DCHECK_LT(params_.delta_mask() - backward_delta, params_.delta_mask()); + delta = params_.delta_mask() - backward_delta + 1; + RTC_DCHECK_LE(delta, params_.delta_mask()); + } + + writer_->WriteBits(delta, params_.delta_width_bits()); } // Perform decoding of a a delta-encoded stream, extracting the original @@ -434,6 +575,10 @@ class FixedLengthDeltaDecoder final { // values' width, as specified by the aforementioned encoding parameters. uint64_t ApplyDelta(uint64_t base, uint64_t delta) const; + // Helpers for ApplyDelta(). + uint64_t ApplyUnsignedDelta(uint64_t base, uint64_t delta) const; + uint64_t ApplySignedDelta(uint64_t base, uint64_t delta) const; + // Reader of the input stream to be decoded. Does not own that buffer. // See comment above ctor for details. const std::unique_ptr reader_; @@ -450,11 +595,6 @@ class FixedLengthDeltaDecoder final { // The number of values to be known to be decoded. const size_t num_of_deltas_; - // Bit mask corresponding to |params_.original_width_bits|. That is, the bits - // necessary for encoding all of the values in the encoded sequence are on. - // Used as an optimization. - const uint64_t value_mask_; - RTC_DISALLOW_COPY_AND_ASSIGN(FixedLengthDeltaDecoder); }; @@ -524,24 +664,20 @@ std::unique_ptr FixedLengthDeltaDecoder::Create( const uint64_t delta_width_bits = read_buffer + 1; // See encoding for +1's rationale. - // signed_deltas, values_optional, original_width_bits + // signed_deltas, values_optional, value_width_bits bool signed_deltas; bool values_optional; - uint64_t original_width_bits; + uint64_t value_width_bits; if (encoding == EncodingType::kFixedSizeUnsignedDeltasNoEarlyWrapNoOpt) { signed_deltas = kDefaultSignedDeltas; values_optional = kDefaultValuesOptional; - original_width_bits = kDefaultOriginalWidthBits; + value_width_bits = kDefaultValueWidthBits; } else { // signed_deltas if (!reader->ReadBits(&read_buffer, kBitsInHeaderForSignedDeltas)) { return nullptr; } signed_deltas = rtc::dchecked_cast(read_buffer); - if (signed_deltas) { - RTC_LOG(LS_WARNING) << "Not implemented."; - return nullptr; - } // values_optional if (!reader->ReadBits(&read_buffer, kBitsInHeaderForValuesOptional)) { @@ -554,16 +690,25 @@ std::unique_ptr FixedLengthDeltaDecoder::Create( return nullptr; } - // original_width_bits - if (!reader->ReadBits(&read_buffer, kBitsInHeaderForOriginalWidthBits)) { + // value_width_bits + if (!reader->ReadBits(&read_buffer, kBitsInHeaderForValueWidthBits)) { return nullptr; } - RTC_DCHECK_LE(read_buffer, 64 - 1); // See encoding for -1's rationale. - original_width_bits = read_buffer + 1; // See encoding for +1's rationale. + RTC_DCHECK_LE(read_buffer, 64 - 1); // See encoding for -1's rationale. + value_width_bits = read_buffer + 1; // See encoding for +1's rationale. + } + + // Note: Because of the way the parameters are read, it is not possible + // for illegal values to be read. We check nevertheless, in case the code + // changes in the future in a way that breaks this promise. + if (!FixedLengthEncodingParameters::ValidParameters( + delta_width_bits, signed_deltas, values_optional, value_width_bits)) { + RTC_LOG(LS_WARNING) << "Corrupt log; illegal encoding parameters."; + return nullptr; } FixedLengthEncodingParameters params(delta_width_bits, signed_deltas, - values_optional, original_width_bits); + values_optional, value_width_bits); return absl::WrapUnique(new FixedLengthDeltaDecoder(std::move(reader), params, base, num_of_deltas)); } @@ -576,13 +721,10 @@ FixedLengthDeltaDecoder::FixedLengthDeltaDecoder( : reader_(std::move(reader)), params_(params), base_(base), - num_of_deltas_(num_of_deltas), - value_mask_(MaxValueOfBitWidth(params_.original_width_bits)) { + num_of_deltas_(num_of_deltas) { RTC_DCHECK(reader_); - // TODO(eladalon): Support signed deltas. - RTC_DCHECK(!params.signed_deltas) << "Not implemented."; // TODO(eladalon): Support optional values. - RTC_DCHECK(!params.values_optional) << "Not implemented."; + RTC_DCHECK(!params.values_optional()) << "Not implemented."; } std::vector FixedLengthDeltaDecoder::Decode() { @@ -603,15 +745,15 @@ std::vector FixedLengthDeltaDecoder::Decode() { bool FixedLengthDeltaDecoder::ParseDelta(uint64_t* delta) { RTC_DCHECK(reader_); - RTC_DCHECK(!params_.signed_deltas) << "Not implemented."; // Reminder. - RTC_DCHECK(!params_.values_optional) << "Not implemented."; // Reminder. + RTC_DCHECK(!params_.values_optional()) << "Not implemented."; // Reminder. // BitBuffer and BitBufferWriter read/write higher bits before lower bits. const size_t lower_bit_count = - std::min(params_.delta_width_bits, 32u); - const size_t higher_bit_count = - (params_.delta_width_bits <= 32u) ? 0 : params_.delta_width_bits - 32u; + std::min(params_.delta_width_bits(), 32u); + const size_t higher_bit_count = (params_.delta_width_bits() <= 32u) + ? 0 + : params_.delta_width_bits() - 32u; uint32_t lower_bits; uint32_t higher_bits; @@ -639,13 +781,37 @@ bool FixedLengthDeltaDecoder::ParseDelta(uint64_t* delta) { uint64_t FixedLengthDeltaDecoder::ApplyDelta(uint64_t base, uint64_t delta) const { - RTC_DCHECK(!params_.signed_deltas) << "Not implemented."; // Reminder. - RTC_DCHECK(!params_.values_optional) << "Not implemented."; // Reminder. - RTC_DCHECK_LE(base, MaxValueOfBitWidth(params_.original_width_bits)); - RTC_DCHECK_LE(delta, MaxValueOfBitWidth(params_.delta_width_bits)); - RTC_DCHECK_LE(params_.delta_width_bits, - params_.original_width_bits); // Reminder. - return (base + delta) & value_mask_; + RTC_DCHECK(!params_.values_optional()) << "Not implemented."; // Reminder. + RTC_DCHECK_LE(base, MaxUnsignedValueOfBitWidth(params_.value_width_bits())); + RTC_DCHECK_LE(delta, MaxUnsignedValueOfBitWidth(params_.delta_width_bits())); + return params_.signed_deltas() ? ApplySignedDelta(base, delta) + : ApplyUnsignedDelta(base, delta); +} + +uint64_t FixedLengthDeltaDecoder::ApplyUnsignedDelta(uint64_t base, + uint64_t delta) const { + // Note: May still be used if signed deltas used. + RTC_DCHECK_LE(base, MaxUnsignedValueOfBitWidth(params_.value_width_bits())); + RTC_DCHECK_LE(delta, MaxUnsignedValueOfBitWidth(params_.delta_width_bits())); + return (base + delta) & params_.value_mask(); +} + +uint64_t FixedLengthDeltaDecoder::ApplySignedDelta(uint64_t base, + uint64_t delta) const { + RTC_DCHECK(params_.signed_deltas()); + RTC_DCHECK_LE(base, MaxUnsignedValueOfBitWidth(params_.value_width_bits())); + RTC_DCHECK_LE(delta, MaxUnsignedValueOfBitWidth(params_.delta_width_bits())); + + const uint64_t top_bit = static_cast(1) + << (params_.delta_width_bits() - 1); + + const bool positive_delta = ((delta & top_bit) == 0); + if (positive_delta) { + return ApplyUnsignedDelta(base, delta); + } + + const uint64_t delta_abs = (~delta & params_.delta_mask()) + 1; + return (base - delta_abs) & params_.value_mask(); } } // namespace @@ -676,4 +842,14 @@ std::vector DecodeDeltas(const std::string& input, return std::vector(); } +void SetFixedLengthEncoderDeltaSignednessForTesting(bool signedness) { + g_force_unsigned_for_testing = !signedness; + g_force_signed_for_testing = signedness; +} + +void UnsetFixedLengthEncoderDeltaSignednessForTesting() { + g_force_unsigned_for_testing = false; + g_force_signed_for_testing = false; +} + } // namespace webrtc diff --git a/logging/rtc_event_log/encoder/delta_encoding_unittest.cc b/logging/rtc_event_log/encoder/delta_encoding_unittest.cc index c318674bc3..d5535177ce 100644 --- a/logging/rtc_event_log/encoder/delta_encoding_unittest.cc +++ b/logging/rtc_event_log/encoder/delta_encoding_unittest.cc @@ -16,14 +16,36 @@ #include #include +#include "absl/types/optional.h" #include "rtc_base/arraysize.h" #include "rtc_base/checks.h" #include "rtc_base/random.h" #include "test/gtest.h" namespace webrtc { + +void SetFixedLengthEncoderDeltaSignednessForTesting(bool signedness); +void UnsetFixedLengthEncoderDeltaSignednessForTesting(); + namespace { +enum class DeltaSignedness { kNoOverride, kForceUnsigned, kForceSigned }; + +void MaybeSetSignedness(DeltaSignedness signedness) { + switch (signedness) { + case DeltaSignedness::kNoOverride: + UnsetFixedLengthEncoderDeltaSignednessForTesting(); + return; + case DeltaSignedness::kForceUnsigned: + SetFixedLengthEncoderDeltaSignednessForTesting(false); + return; + case DeltaSignedness::kForceSigned: + SetFixedLengthEncoderDeltaSignednessForTesting(true); + return; + } + RTC_NOTREACHED(); +} + uint64_t RandomWithMaxBitWidth(Random* prng, uint64_t max_width) { RTC_DCHECK_GE(max_width, 1u); RTC_DCHECK_LE(max_width, 64u); @@ -95,21 +117,42 @@ std::vector CreateSequenceByDeltas( } size_t EncodingLengthUpperBound(size_t delta_max_bit_width, - size_t num_of_deltas) { - constexpr size_t kSmallestHeaderSizeBytes = 1; - return delta_max_bit_width * num_of_deltas + kSmallestHeaderSizeBytes; + size_t num_of_deltas, + DeltaSignedness signedness_override) { + absl::optional smallest_header_size_bytes; + switch (signedness_override) { + case DeltaSignedness::kNoOverride: + case DeltaSignedness::kForceUnsigned: + smallest_header_size_bytes = 1; + break; + case DeltaSignedness::kForceSigned: + smallest_header_size_bytes = 2; + break; + } + RTC_DCHECK(smallest_header_size_bytes); + + return delta_max_bit_width * num_of_deltas + *smallest_header_size_bytes; } // Tests of the delta encoding, parameterized by the number of values // in the sequence created by the test. -class DeltaEncodingTest : public ::testing::TestWithParam { +class DeltaEncodingTest + : public ::testing::TestWithParam> { public: + DeltaEncodingTest() + : signedness_(std::get<0>(GetParam())), + num_of_values_(std::get<1>(GetParam())) { + MaybeSetSignedness(signedness_); + } ~DeltaEncodingTest() override = default; + + const DeltaSignedness signedness_; + const uint64_t num_of_values_; }; TEST_P(DeltaEncodingTest, AllValuesEqualToBaseValue) { const uint64_t base = 3432; - std::vector values(GetParam()); + std::vector values(num_of_values_); std::fill(values.begin(), values.end(), base); std::string encoded; TestEncodingAndDecoding(base, values, &encoded); @@ -122,7 +165,7 @@ TEST_P(DeltaEncodingTest, AllValuesEqualToBaseValue) { TEST_P(DeltaEncodingTest, MinDeltaNoWrapAround) { const uint64_t base = 3432; - const auto values = CreateSequenceByFirstValue(base + 1, GetParam()); + const auto values = CreateSequenceByFirstValue(base + 1, num_of_values_); ASSERT_GT(values[values.size() - 1], base) << "Sanity; must not wrap around"; TestEncodingAndDecoding(base, values); @@ -132,7 +175,8 @@ TEST_P(DeltaEncodingTest, BigDeltaNoWrapAround) { const uint64_t kBigDelta = 132828; const uint64_t base = 3432; - const auto values = CreateSequenceByFirstValue(base + kBigDelta, GetParam()); + const auto values = + CreateSequenceByFirstValue(base + kBigDelta, num_of_values_); ASSERT_GT(values[values.size() - 1], base) << "Sanity; must not wrap around"; TestEncodingAndDecoding(base, values); @@ -142,7 +186,7 @@ TEST_P(DeltaEncodingTest, MaxDeltaNoWrapAround) { const uint64_t base = 3432; const auto values = CreateSequenceByLastValue( - std::numeric_limits::max(), GetParam()); + std::numeric_limits::max(), num_of_values_); ASSERT_GT(values[values.size() - 1], base) << "Sanity; must not wrap around"; TestEncodingAndDecoding(base, values); @@ -151,20 +195,20 @@ TEST_P(DeltaEncodingTest, MaxDeltaNoWrapAround) { TEST_P(DeltaEncodingTest, SmallDeltaWithWrapAroundComparedToBase) { const uint64_t base = std::numeric_limits::max(); - const auto values = CreateSequenceByDeltas(base, {1, 10, 3}, GetParam()); + const auto values = CreateSequenceByDeltas(base, {1, 10, 3}, num_of_values_); ASSERT_LT(values[values.size() - 1], base) << "Sanity; must wrap around"; TestEncodingAndDecoding(base, values); } TEST_P(DeltaEncodingTest, SmallDeltaWithWrapAroundInValueSequence) { - if (GetParam() == 1) { + if (num_of_values_ == 1) { return; // Inapplicable. } const uint64_t base = std::numeric_limits::max() - 2; - const auto values = CreateSequenceByDeltas(base, {1, 10, 3}, GetParam()); + const auto values = CreateSequenceByDeltas(base, {1, 10, 3}, num_of_values_); ASSERT_LT(values[values.size() - 1], values[0]) << "Sanity; must wrap around"; TestEncodingAndDecoding(base, values); @@ -179,14 +223,15 @@ TEST_P(DeltaEncodingTest, BigDeltaWithWrapAroundComparedToBase) { const uint64_t kBigDelta = 132828; const uint64_t base = std::numeric_limits::max() - kBigDelta + 3; - const auto values = CreateSequenceByFirstValue(base + kBigDelta, GetParam()); + const auto values = + CreateSequenceByFirstValue(base + kBigDelta, num_of_values_); ASSERT_LT(values[values.size() - 1], base) << "Sanity; must wrap around"; TestEncodingAndDecoding(base, values); } TEST_P(DeltaEncodingTest, BigDeltaWithWrapAroundInValueSequence) { - if (GetParam() == 1) { + if (num_of_values_ == 1) { return; // Inapplicable. } @@ -194,7 +239,7 @@ TEST_P(DeltaEncodingTest, BigDeltaWithWrapAroundInValueSequence) { const uint64_t base = std::numeric_limits::max() - kBigDelta + 3; const auto values = CreateSequenceByFirstValue( - std::numeric_limits::max(), GetParam()); + std::numeric_limits::max(), num_of_values_); ASSERT_LT(values[values.size() - 1], base) << "Sanity; must wrap around"; TestEncodingAndDecoding(base, values); @@ -205,27 +250,27 @@ TEST_P(DeltaEncodingTest, BigDeltaWithWrapAroundInValueSequence) { TEST_P(DeltaEncodingTest, MaxDeltaWithWrapAroundComparedToBase) { const uint64_t base = 3432; - const auto values = CreateSequenceByFirstValue(base - 1, GetParam()); + const auto values = CreateSequenceByFirstValue(base - 1, num_of_values_); TestEncodingAndDecoding(base, values); } TEST_P(DeltaEncodingTest, MaxDeltaWithWrapAroundInValueSequence) { - if (GetParam() == 1) { + if (num_of_values_ == 1) { return; // Inapplicable. } const uint64_t base = 3432; const auto values = CreateSequenceByDeltas( - base, {0, std::numeric_limits::max(), 3}, GetParam()); + base, {0, std::numeric_limits::max(), 3}, num_of_values_); ASSERT_LT(values[1], base) << "Sanity; must wrap around"; TestEncodingAndDecoding(base, values); } -// If GetParam() == 1, a zero delta will yield an empty string; that's already -// covered by AllValuesEqualToBaseValue, but it doesn't hurt to test again. -// For all other cases, we have a new test. +// If num_of_values_ == 1, a zero delta will yield an empty string; that's +// already covered by AllValuesEqualToBaseValue, but it doesn't hurt to test +// again. For all other cases, we have a new test. TEST_P(DeltaEncodingTest, ZeroDelta) { const uint64_t base = 3432; @@ -233,26 +278,46 @@ TEST_P(DeltaEncodingTest, ZeroDelta) { // consecutive zeros. const std::vector deltas = {0, 312, 11, 1, 1, 0, 0, 12, 400321, 3, 3, 12, 5, 0, 6}; - const auto values = CreateSequenceByDeltas(base, deltas, GetParam()); + const auto values = CreateSequenceByDeltas(base, deltas, num_of_values_); TestEncodingAndDecoding(base, values); } -INSTANTIATE_TEST_CASE_P(NumberOfValuesInSequence, - DeltaEncodingTest, - ::testing::Values(1, 2, 100, 10000)); +INSTANTIATE_TEST_CASE_P( + SignednessOverrideAndNumberOfValuesInSequence, + DeltaEncodingTest, + ::testing::Combine(::testing::Values(DeltaSignedness::kNoOverride, + DeltaSignedness::kForceUnsigned, + DeltaSignedness::kForceSigned), + ::testing::Values(1, 2, 100, 10000))); // Tests over the quality of the compression (as opposed to its correctness). // Not to be confused with tests of runtime efficiency. class DeltaEncodingCompressionQualityTest - : public ::testing::TestWithParam> { + : public ::testing::TestWithParam< + std::tuple> { public: DeltaEncodingCompressionQualityTest() - : delta_max_bit_width_(std::get<0>(GetParam())), - num_of_values_(std::get<1>(GetParam())) {} + : signedness_(std::get<0>(GetParam())), + delta_max_bit_width_(std::get<1>(GetParam())), + num_of_values_(std::get<2>(GetParam())) { + MaybeSetSignedness(signedness_); + } ~DeltaEncodingCompressionQualityTest() override = default; + // Running with the same seed for all variants would make all tests start + // with the same sequence; avoid this by making the seed different. + uint64_t Seed() const { + constexpr uint64_t non_zero_base_seed = 3012; + // Multiply everything but |non_zero_base_seed| by different prime numbers + // to produce unique results. + return non_zero_base_seed + 2 * static_cast(signedness_) + + 3 * delta_max_bit_width_ + 5 * delta_max_bit_width_ + + 7 * num_of_values_; + } + + const DeltaSignedness signedness_; const uint64_t delta_max_bit_width_; const uint64_t num_of_values_; }; @@ -261,12 +326,6 @@ class DeltaEncodingCompressionQualityTest // matter to compression performance; only the deltas matter. TEST_P(DeltaEncodingCompressionQualityTest, BaseDoesNotAffectEfficiencyIfNoWrapAround) { - Random prng(3012); - std::vector deltas(num_of_values_); - for (size_t i = 0; i < deltas.size(); ++i) { - deltas[i] = RandomWithMaxBitWidth(&prng, delta_max_bit_width_); - } - // 1. Bases which will not produce a wrap-around. // 2. The last base - 0xffffffffffffffff - does cause a wrap-around, but // that still works, because the width is 64 anyway, and does not @@ -274,6 +333,46 @@ TEST_P(DeltaEncodingCompressionQualityTest, const uint64_t bases[] = {0, 0x55, 0xffffffff, std::numeric_limits::max()}; + const size_t kIntendedWrapAroundBaseIndex = arraysize(bases); + + std::vector deltas(num_of_values_); + + // Allows us to make sure that the deltas do not produce a wrap-around. + uint64_t last_element[arraysize(bases)]; + memcpy(last_element, bases, sizeof(bases)); + + // Avoid empty |deltas| due to first element causing wrap-around. + deltas[0] = 1; + for (size_t i = 0; i < arraysize(last_element); ++i) { + last_element[i] += 1; + } + + Random prng(Seed()); + + for (size_t i = 1; i < deltas.size(); ++i) { + const uint64_t delta = RandomWithMaxBitWidth(&prng, delta_max_bit_width_); + + bool wrap_around = false; + for (size_t j = 0; j < arraysize(last_element); ++j) { + if (j == kIntendedWrapAroundBaseIndex) { + continue; + } + + last_element[j] += delta; + if (last_element[j] < bases[j]) { + wrap_around = true; + break; + } + } + + if (wrap_around) { + deltas.resize(i); + break; + } + + deltas[i] = delta; + } + std::string encodings[arraysize(bases)]; for (size_t i = 0; i < arraysize(bases); ++i) { @@ -284,7 +383,8 @@ TEST_P(DeltaEncodingCompressionQualityTest, // the encoding/decoding, though that is not the test's focus. TestEncodingAndDecoding(bases[i], values, &encodings[i]); EXPECT_LE(encodings[i].length(), - EncodingLengthUpperBound(delta_max_bit_width_, num_of_values_)); + EncodingLengthUpperBound(delta_max_bit_width_, num_of_values_, + signedness_)); } // Test focus - all of the encodings should be the same, as they are based @@ -295,23 +395,42 @@ TEST_P(DeltaEncodingCompressionQualityTest, } INSTANTIATE_TEST_CASE_P( - DeltaMaxBitWidthAndNumberOfValuesInSequence, + SignednessOverrideAndDeltaMaxBitWidthAndNumberOfValuesInSequence, DeltaEncodingCompressionQualityTest, ::testing::Combine( + ::testing::Values(DeltaSignedness::kNoOverride, + DeltaSignedness::kForceUnsigned, + DeltaSignedness::kForceSigned), ::testing::Values(1, 4, 8, 15, 16, 17, 31, 32, 33, 63, 64), ::testing::Values(1, 2, 100, 10000))); // Similar to DeltaEncodingTest, but instead of semi-surgically producing // specific cases, produce large amount of semi-realistic inputs. class DeltaEncodingFuzzerLikeTest - : public ::testing::TestWithParam> { + : public ::testing::TestWithParam< + std::tuple> { public: DeltaEncodingFuzzerLikeTest() - : delta_max_bit_width_(std::get<0>(GetParam())), - num_of_values_(std::get<1>(GetParam())) {} + : signedness_(std::get<0>(GetParam())), + delta_max_bit_width_(std::get<1>(GetParam())), + num_of_values_(std::get<2>(GetParam())) { + MaybeSetSignedness(signedness_); + } ~DeltaEncodingFuzzerLikeTest() override = default; + // Running with the same seed for all variants would make all tests start + // with the same sequence; avoid this by making the seed different. + uint64_t Seed() const { + constexpr uint64_t non_zero_base_seed = 1983; + // Multiply everything but |non_zero_base_seed| by different prime numbers + // to produce unique results. + return non_zero_base_seed + 2 * static_cast(signedness_) + + 3 * delta_max_bit_width_ + 5 * delta_max_bit_width_ + + 7 * num_of_values_; + } + + const DeltaSignedness signedness_; const uint64_t delta_max_bit_width_; const uint64_t num_of_values_; }; @@ -319,7 +438,7 @@ class DeltaEncodingFuzzerLikeTest TEST_P(DeltaEncodingFuzzerLikeTest, Test) { const uint64_t base = 3432; - Random prng(1983); + Random prng(Seed()); std::vector deltas(num_of_values_); for (size_t i = 0; i < deltas.size(); ++i) { deltas[i] = RandomWithMaxBitWidth(&prng, delta_max_bit_width_); @@ -331,11 +450,75 @@ TEST_P(DeltaEncodingFuzzerLikeTest, Test) { } INSTANTIATE_TEST_CASE_P( - DeltaMaxBitWidthAndNumberOfValuesInSequence, + SignednessOverrideAndDeltaMaxBitWidthAndNumberOfValuesInSequence, DeltaEncodingFuzzerLikeTest, ::testing::Combine( + ::testing::Values(DeltaSignedness::kNoOverride, + DeltaSignedness::kForceUnsigned, + DeltaSignedness::kForceSigned), ::testing::Values(1, 4, 8, 15, 16, 17, 31, 32, 33, 63, 64), ::testing::Values(1, 2, 100, 10000))); +class DeltaEncodingSpecificEdgeCasesTest + : public ::testing::TestWithParam< + std::tuple> { + public: + DeltaEncodingSpecificEdgeCasesTest() { + UnsetFixedLengthEncoderDeltaSignednessForTesting(); + } + + ~DeltaEncodingSpecificEdgeCasesTest() override = default; +}; + +// This case is special because it produces identical forward/backward deltas. +TEST_F(DeltaEncodingSpecificEdgeCasesTest, SignedDeltaWithOnlyTopBitOn) { + MaybeSetSignedness(DeltaSignedness::kForceSigned); + + const uint64_t base = 3432; + + const uint64_t delta = static_cast(1) << 63; + const std::vector values = {base + delta}; + + TestEncodingAndDecoding(base, values); +} + +TEST_F(DeltaEncodingSpecificEdgeCasesTest, MaximumUnsignedDelta) { + MaybeSetSignedness(DeltaSignedness::kForceUnsigned); + + const uint64_t base = (static_cast(1) << 63) + 0x123; + + const std::vector values = {base - 1}; + + TestEncodingAndDecoding(base, values); +} + +// Check that, if all deltas are set to -1, things still work. +TEST_P(DeltaEncodingSpecificEdgeCasesTest, ReverseSequence) { + MaybeSetSignedness(std::get<0>(GetParam())); + const uint64_t width = std::get<1>(GetParam()); + const bool wrap_around = std::get<2>(GetParam()); + + const uint64_t value_mask = (width == 64) + ? std::numeric_limits::max() + : ((static_cast(1) << width) - 1); + + const uint64_t base = wrap_around ? 1u : (0xf82d3 & value_mask); + const std::vector values = {(base - 1u) & value_mask, + (base - 2u) & value_mask, + (base - 3u) & value_mask}; + + TestEncodingAndDecoding(base, values); +} + +INSTANTIATE_TEST_CASE_P( + _, + DeltaEncodingSpecificEdgeCasesTest, + ::testing::Combine( + ::testing::Values(DeltaSignedness::kNoOverride, + DeltaSignedness::kForceUnsigned, + DeltaSignedness::kForceSigned), + ::testing::Values(1, 4, 8, 15, 16, 17, 31, 32, 33, 63, 64), + ::testing::Bool())); + } // namespace } // namespace webrtc