admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add support for signed deltas in FixedLengthDeltaEncoder

Browse Source 
Signed deltas can yield a more efficient encoding when the encoded
sequence sometimes moves backwards.

Bug: webrtc:8111
Change-Id: Ib1a50192851214ccc3f2bd7eaf88f4be97e4beb0
Reviewed-on: https://webrtc-review.googlesource.com/c/100423
Commit-Queue: Elad Alon <eladalon@webrtc.org>
Reviewed-by: Björn Terelius <terelius@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#25324}
This commit is contained in:
Elad Alon 2018-10-23 18:24:10 +02:00 committed by Commit Bot
parent 4b31cf571f
commit 73f3917e89
3 changed files with 502 additions and 142 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
logging/BUILD.gn
View File

@ -183,6 +183,7 @@ rtc_static_library("rtc_event_log_impl_encoder") {
"../rtc_base:rtc_base_approved",
"//third_party/abseil-cpp/absl/memory",
"//third_party/abseil-cpp/absl/strings:strings",
"//third_party/abseil-cpp/absl/types:optional",
]
if (rtc_enable_protobuf) {

378
logging/rtc_event_log/encoder/delta_encoding.cc
View File

@ -27,12 +27,19 @@ namespace {
// TODO(eladalon): Only build the decoder in tools and unit tests.
bool g_force_unsigned_for_testing = false;
bool g_force_signed_for_testing = false;
size_t BitsToBytes(size_t bits) {
return (bits / 8) + (bits % 8 > 0 ? 1 : 0);
}
// TODO(eladalon): Replace by something more efficient.
uint64_t BitWidth(uint64_t input) {
uint64_t UnsignedBitWidth(uint64_t input, bool zero_val_as_zero_width = false) {
if (zero_val_as_zero_width && input == 0) {
return 0;
}
uint64_t width = 0;
do { // input == 0 -> width == 1
width += 1;
@ -41,13 +48,22 @@ uint64_t BitWidth(uint64_t input) {
return width;
}
uint64_t SignedBitWidth(uint64_t max_pos_magnitude,
uint64_t max_neg_magnitude) {
const uint64_t bitwidth_pos = UnsignedBitWidth(max_pos_magnitude, true);
const uint64_t bitwidth_neg =
(max_neg_magnitude > 0) ? UnsignedBitWidth(max_neg_magnitude - 1, true)
: 0;
return 1 + std::max(bitwidth_pos, bitwidth_neg);
}
// Return the maximum integer of a given bit width.
// Examples:
// MaxValueOfBitWidth(1) = 0x01
// MaxValueOfBitWidth(6) = 0x3f
// MaxValueOfBitWidth(8) = 0xff
// MaxValueOfBitWidth(32) = 0xffffffff
uint64_t MaxValueOfBitWidth(uint64_t bit_width) {
// MaxUnsignedValueOfBitWidth(1) = 0x01
// MaxUnsignedValueOfBitWidth(6) = 0x3f
// MaxUnsignedValueOfBitWidth(8) = 0xff
// MaxUnsignedValueOfBitWidth(32) = 0xffffffff
uint64_t MaxUnsignedValueOfBitWidth(uint64_t bit_width) {
RTC_DCHECK_GE(bit_width, 1);
RTC_DCHECK_LE(bit_width, 64);
return (bit_width == 64) ? std::numeric_limits<uint64_t>::max()
@ -55,18 +71,9 @@ uint64_t MaxValueOfBitWidth(uint64_t bit_width) {
}
// Computes the delta between |previous| and |current|, under the assumption
// that wrap-around occurs after width |bit_width| is exceeded.
uint64_t ComputeDelta(uint64_t previous, uint64_t current, uint64_t width) {
RTC_DCHECK(width == 64 || current < (static_cast<uint64_t>(1) << width));
RTC_DCHECK(width == 64 || previous < (static_cast<uint64_t>(1) << width));
if (current >= previous) {
// Simply "walk" forward.
return current - previous;
} else { // previous > current
// "Walk" until the max value, one more step to 0, then to |current|.
return (MaxValueOfBitWidth(width) - previous) + 1 + current;
}
// that wrap-around occurs after |width| is exceeded.
uint64_t UnsignedDelta(uint64_t previous, uint64_t current, uint64_t bit_mask) {
return (current - previous) & bit_mask;
}
// Determines the encoding type (e.g. fixed-size encoding).
@ -87,7 +94,7 @@ constexpr size_t kBitsInHeaderForEncodingType = 2;
constexpr size_t kBitsInHeaderForDeltaWidthBits = 6;
constexpr size_t kBitsInHeaderForSignedDeltas = 1;
constexpr size_t kBitsInHeaderForValuesOptional = 1;
constexpr size_t kBitsInHeaderForOriginalWidthBits = 6;
constexpr size_t kBitsInHeaderForValueWidthBits = 6;
static_assert(static_cast<size_t>(EncodingType::kNumberOfEncodingTypes) <=
1 << kBitsInHeaderForEncodingType,
@ -96,7 +103,7 @@ static_assert(static_cast<size_t>(EncodingType::kNumberOfEncodingTypes) <=
// Default values for when the encoding header does not specify explicitly.
constexpr bool kDefaultSignedDeltas = false;
constexpr bool kDefaultValuesOptional = false;
constexpr uint64_t kDefaultOriginalWidthBits = 64;
constexpr uint64_t kDefaultValueWidthBits = 64;
// Wrap BitBufferWriter and extend its functionality by (1) keeping track of
// the number of bits written and (2) owning its buffer.
@ -145,33 +152,66 @@ class BitWriter final {
// Parameters for fixed-size delta-encoding/decoding.
// These are tailored for the sequence which will be encoded (e.g. widths).
struct FixedLengthEncodingParameters final {
class FixedLengthEncodingParameters final {
public:
static bool ValidParameters(uint64_t delta_width_bits,
bool signed_deltas,
bool values_optional,
uint64_t value_width_bits) {
return (1 <= delta_width_bits && delta_width_bits <= 64 &&
1 <= value_width_bits && value_width_bits <= 64 &&
delta_width_bits <= value_width_bits);
}
FixedLengthEncodingParameters(uint64_t delta_width_bits,
bool signed_deltas,
bool values_optional,
uint64_t original_width_bits)
: delta_width_bits(delta_width_bits),
signed_deltas(signed_deltas),
values_optional(values_optional),
original_width_bits(original_width_bits) {}
uint64_t value_width_bits)
: delta_width_bits_(delta_width_bits),
signed_deltas_(signed_deltas),
values_optional_(values_optional),
value_width_bits_(value_width_bits),
delta_mask_(MaxUnsignedValueOfBitWidth(delta_width_bits_)),
value_mask_(MaxUnsignedValueOfBitWidth(value_width_bits_)) {
RTC_DCHECK(ValidParameters(delta_width_bits, signed_deltas, values_optional,
value_width_bits));
}
// Number of bits necessary to hold the widest(*) of the deltas between the
// values in the sequence.
// (*) - Widest might not be the largest, if signed deltas are used.
uint64_t delta_width_bits;
uint64_t delta_width_bits() const { return delta_width_bits_; }
// Whether deltas are signed.
// TODO(eladalon): Add support for signed deltas.
bool signed_deltas;
bool signed_deltas() const { return signed_deltas_; }
// Whether the values of the sequence are optional. That is, it may be
// that some of them do not have a value (not even a sentinel value indicating
// invalidity).
// TODO(eladalon): Add support for optional elements.
bool values_optional;
bool values_optional() const { return values_optional_; }
// Number of bits necessary to hold the largest value in the sequence.
uint64_t original_width_bits;
uint64_t value_width_bits() const { return value_width_bits_; }
// Masks where only the bits relevant to the deltas/values are turned on.
uint64_t delta_mask() const { return delta_mask_; }
uint64_t value_mask() const { return value_mask_; }
void SetSignedDeltas(bool signed_deltas) { signed_deltas_ = signed_deltas; }
void SetDeltaWidthBits(uint64_t delta_width_bits) {
delta_width_bits_ = delta_width_bits;
delta_mask_ = MaxUnsignedValueOfBitWidth(delta_width_bits);
}
private:
uint64_t delta_width_bits_; // Normally const, but mutable in tests.
bool signed_deltas_; // Normally const, but mutable in tests.
const bool values_optional_;
const uint64_t value_width_bits_;
uint64_t delta_mask_; // Normally const, but mutable in tests.
const uint64_t value_mask_;
};
// Performs delta-encoding of a single (non-empty) sequence of values, using
@ -190,6 +230,21 @@ class FixedLengthDeltaEncoder final {
const std::vector<uint64_t>& values);
private:
// Calculate min/max values of unsigned/signed deltas, given the bit width
// of all the values in the series.
static void CalculateMinAndMaxDeltas(uint64_t base,
const std::vector<uint64_t>& values,
uint64_t bit_width,
uint64_t* max_unsigned_delta,
uint64_t* max_pos_signed_delta,
uint64_t* min_neg_signed_delta);
// No effect outside of unit tests.
// In unit tests, may lead to forcing signed/unsigned deltas, etc.
static void ConsiderTestOverrides(FixedLengthEncodingParameters* params,
uint64_t delta_width_bits_signed,
uint64_t delta_width_bits_unsigned);
// FixedLengthDeltaEncoder objects are to be created by EncodeDeltas() and
// released by it before it returns. They're mostly a convenient way to
// avoid having to pass a lot of state between different functions.
@ -213,6 +268,8 @@ class FixedLengthDeltaEncoder final {
// Encode a given delta into the stream.
void EncodeDelta(uint64_t previous, uint64_t current);
void EncodeUnsignedDelta(uint64_t previous, uint64_t current);
void EncodeSignedDelta(uint64_t previous, uint64_t current);
// The parameters according to which encoding will be done (width of
// fields, whether signed deltas should be used, etc.)
@ -251,16 +308,14 @@ std::string FixedLengthDeltaEncoder::EncodeDeltas(
// If the sequence is non-decreasing, it may be assumed to have width = 64;
// there's no reason to encode the actual max width in the encoding header.
const uint64_t original_width_bits =
non_decreasing ? 64 : BitWidth(max_value_including_base);
const uint64_t value_width_bits =
non_decreasing ? 64 : UnsignedBitWidth(max_value_including_base);
uint64_t max_unsigned_delta =
ComputeDelta(base, values[0], original_width_bits);
for (size_t i = 1; i < values.size(); ++i) {
const uint64_t unsigned_delta =
ComputeDelta(values[i - 1], values[i], original_width_bits);
max_unsigned_delta = std::max(unsigned_delta, max_unsigned_delta);
}
uint64_t max_unsigned_delta;
uint64_t max_pos_signed_delta;
uint64_t min_neg_signed_delta;
CalculateMinAndMaxDeltas(base, values, value_width_bits, &max_unsigned_delta,
&max_pos_signed_delta, &min_neg_signed_delta);
// We indicate the special case of all values being equal to the base with
// the empty string.
@ -270,11 +325,10 @@ std::string FixedLengthDeltaEncoder::EncodeDeltas(
return std::string();
}
const uint64_t delta_width_bits_unsigned = BitWidth(max_unsigned_delta);
// This will prevent the use of signed deltas, by always assuming
// they will not provide value over unsigned.
// TODO(eladalon): Add support for signed deltas.
const uint64_t delta_width_bits_signed = 64;
const uint64_t delta_width_bits_unsigned =
UnsignedBitWidth(max_unsigned_delta);
const uint64_t delta_width_bits_signed =
SignedBitWidth(max_pos_signed_delta, min_neg_signed_delta);
// Note: Preference for unsigned if the two have the same width (efficiency).
const bool signed_deltas =
@ -285,21 +339,75 @@ std::string FixedLengthDeltaEncoder::EncodeDeltas(
const bool values_optional = false;
FixedLengthEncodingParameters params(delta_width_bits, signed_deltas,
values_optional, original_width_bits);
values_optional, value_width_bits);
// No effect in production.
ConsiderTestOverrides(&params, delta_width_bits_signed,
delta_width_bits_unsigned);
FixedLengthDeltaEncoder encoder(params, base, values);
return encoder.Encode();
}
void FixedLengthDeltaEncoder::CalculateMinAndMaxDeltas(
uint64_t base,
const std::vector<uint64_t>& values,
uint64_t bit_width,
uint64_t* max_unsigned_delta_out,
uint64_t* max_pos_signed_delta_out,
uint64_t* min_neg_signed_delta_out) {
RTC_DCHECK(!values.empty());
RTC_DCHECK(max_unsigned_delta_out);
RTC_DCHECK(max_pos_signed_delta_out);
RTC_DCHECK(min_neg_signed_delta_out);
const uint64_t bit_mask = MaxUnsignedValueOfBitWidth(bit_width);
uint64_t max_unsigned_delta = 0;
uint64_t max_pos_signed_delta = 0;
uint64_t min_neg_signed_delta = 0;
uint64_t prev = base;
for (size_t i = 0; i < values.size(); ++i) {
const uint64_t forward_delta = UnsignedDelta(prev, values[i], bit_mask);
const uint64_t backward_delta = UnsignedDelta(values[i], prev, bit_mask);
max_unsigned_delta = std::max(max_unsigned_delta, forward_delta);
if (forward_delta < backward_delta) {
max_pos_signed_delta = std::max(max_pos_signed_delta, forward_delta);
} else {
min_neg_signed_delta = std::max(min_neg_signed_delta, backward_delta);
}
prev = values[i];
}
*max_unsigned_delta_out = max_unsigned_delta;
*max_pos_signed_delta_out = max_pos_signed_delta;
*min_neg_signed_delta_out = min_neg_signed_delta;
}
void FixedLengthDeltaEncoder::ConsiderTestOverrides(
FixedLengthEncodingParameters* params,
uint64_t delta_width_bits_signed,
uint64_t delta_width_bits_unsigned) {
if (g_force_unsigned_for_testing) {
params->SetDeltaWidthBits(delta_width_bits_unsigned);
params->SetSignedDeltas(false);
} else if (g_force_signed_for_testing) {
params->SetDeltaWidthBits(delta_width_bits_signed);
params->SetSignedDeltas(true);
} else {
// Unchanged.
}
}
FixedLengthDeltaEncoder::FixedLengthDeltaEncoder(
const FixedLengthEncodingParameters& params,
uint64_t base,
const std::vector<uint64_t>& values)
: params_(params), base_(base), values_(values) {
RTC_DCHECK_GE(params_.delta_width_bits, 1);
RTC_DCHECK_LE(params_.delta_width_bits, 64);
RTC_DCHECK_GE(params_.original_width_bits, 1);
RTC_DCHECK_LE(params_.original_width_bits, 64);
RTC_DCHECK_LE(params_.delta_width_bits, params_.original_width_bits);
RTC_DCHECK(!values_.empty());
writer_ = absl::make_unique<BitWriter>(OutputLengthBytes());
}
@ -322,31 +430,31 @@ size_t FixedLengthDeltaEncoder::OutputLengthBytes() const {
}
size_t FixedLengthDeltaEncoder::HeaderLengthBits() const {
if (params_.signed_deltas == kDefaultSignedDeltas &&
params_.values_optional == kDefaultValuesOptional &&
params_.original_width_bits == kDefaultOriginalWidthBits) {
if (params_.signed_deltas() == kDefaultSignedDeltas &&
params_.values_optional() == kDefaultValuesOptional &&
params_.value_width_bits() == kDefaultValueWidthBits) {
return kBitsInHeaderForEncodingType + kBitsInHeaderForDeltaWidthBits;
} else {
return kBitsInHeaderForEncodingType + kBitsInHeaderForDeltaWidthBits +
kBitsInHeaderForSignedDeltas + kBitsInHeaderForValuesOptional +
kBitsInHeaderForOriginalWidthBits;
kBitsInHeaderForValueWidthBits;
}
}
size_t FixedLengthDeltaEncoder::EncodedDeltasLengthBits() const {
// TODO(eladalon): When optional values are supported, iterate over the
// deltas to check the exact cost of each.
RTC_DCHECK(!params_.values_optional);
return values_.size() * params_.delta_width_bits;
RTC_DCHECK(!params_.values_optional());
return values_.size() * params_.delta_width_bits();
}
void FixedLengthDeltaEncoder::EncodeHeader() {
RTC_DCHECK(writer_);
const EncodingType encoding_type =
(params_.original_width_bits == kDefaultOriginalWidthBits &&
params_.signed_deltas == kDefaultSignedDeltas &&
params_.values_optional == kDefaultValuesOptional)
(params_.value_width_bits() == kDefaultValueWidthBits &&
params_.signed_deltas() == kDefaultSignedDeltas &&
params_.values_optional() == kDefaultValuesOptional)
? EncodingType::kFixedSizeUnsignedDeltasNoEarlyWrapNoOpt
: EncodingType::kFixedSizeSignedDeltasEarlyWrapAndOptSupported;
@ -356,26 +464,59 @@ void FixedLengthDeltaEncoder::EncodeHeader() {
// Note: Since it's meaningless for a field to be of width 0, when it comes
// to fields that relate widths, we encode width 1 as 0, width 2 as 1,
writer_->WriteBits(params_.delta_width_bits - 1,
writer_->WriteBits(params_.delta_width_bits() - 1,
kBitsInHeaderForDeltaWidthBits);
if (encoding_type == EncodingType::kFixedSizeUnsignedDeltasNoEarlyWrapNoOpt) {
return;
}
writer_->WriteBits(static_cast<uint64_t>(params_.signed_deltas),
writer_->WriteBits(static_cast<uint64_t>(params_.signed_deltas()),
kBitsInHeaderForSignedDeltas);
writer_->WriteBits(static_cast<uint64_t>(params_.values_optional),
writer_->WriteBits(static_cast<uint64_t>(params_.values_optional()),
kBitsInHeaderForValuesOptional);
writer_->WriteBits(params_.original_width_bits - 1,
kBitsInHeaderForOriginalWidthBits);
writer_->WriteBits(params_.value_width_bits() - 1,
kBitsInHeaderForValueWidthBits);
}
void FixedLengthDeltaEncoder::EncodeDelta(uint64_t previous, uint64_t current) {
if (params_.signed_deltas()) {
EncodeSignedDelta(previous, current);
} else {
EncodeUnsignedDelta(previous, current);
}
}
void FixedLengthDeltaEncoder::EncodeUnsignedDelta(uint64_t previous,
uint64_t current) {
RTC_DCHECK(writer_);
writer_->WriteBits(
ComputeDelta(previous, current, params_.original_width_bits),
params_.delta_width_bits);
const uint64_t delta = UnsignedDelta(previous, current, params_.value_mask());
writer_->WriteBits(delta, params_.delta_width_bits());
}
void FixedLengthDeltaEncoder::EncodeSignedDelta(uint64_t previous,
uint64_t current) {
RTC_DCHECK(writer_);
const uint64_t forward_delta =
UnsignedDelta(previous, current, params_.value_mask());
const uint64_t backward_delta =
UnsignedDelta(current, previous, params_.value_mask());
uint64_t delta;
if (forward_delta <= backward_delta) {
delta = forward_delta;
} else {
// Compute the unsigned representation of a negative delta.
// This is the two's complement representation of this negative value,
// when deltas are of width params_.delta_mask().
RTC_DCHECK_GE(params_.delta_mask(), backward_delta);
RTC_DCHECK_LT(params_.delta_mask() - backward_delta, params_.delta_mask());
delta = params_.delta_mask() - backward_delta + 1;
RTC_DCHECK_LE(delta, params_.delta_mask());
}
writer_->WriteBits(delta, params_.delta_width_bits());
}
// Perform decoding of a a delta-encoded stream, extracting the original
@ -434,6 +575,10 @@ class FixedLengthDeltaDecoder final {
// values' width, as specified by the aforementioned encoding parameters.
uint64_t ApplyDelta(uint64_t base, uint64_t delta) const;
// Helpers for ApplyDelta().
uint64_t ApplyUnsignedDelta(uint64_t base, uint64_t delta) const;
uint64_t ApplySignedDelta(uint64_t base, uint64_t delta) const;
// Reader of the input stream to be decoded. Does not own that buffer.
// See comment above ctor for details.
const std::unique_ptr<rtc::BitBuffer> reader_;
@ -450,11 +595,6 @@ class FixedLengthDeltaDecoder final {
// The number of values to be known to be decoded.
const size_t num_of_deltas_;
// Bit mask corresponding to |params_.original_width_bits|. That is, the bits
// necessary for encoding all of the values in the encoded sequence are on.
// Used as an optimization.
const uint64_t value_mask_;
RTC_DISALLOW_COPY_AND_ASSIGN(FixedLengthDeltaDecoder);
};
@ -524,24 +664,20 @@ std::unique_ptr<FixedLengthDeltaDecoder> FixedLengthDeltaDecoder::Create(
const uint64_t delta_width_bits =
read_buffer + 1; // See encoding for +1's rationale.
// signed_deltas, values_optional, original_width_bits
// signed_deltas, values_optional, value_width_bits
bool signed_deltas;
bool values_optional;
uint64_t original_width_bits;
uint64_t value_width_bits;
if (encoding == EncodingType::kFixedSizeUnsignedDeltasNoEarlyWrapNoOpt) {
signed_deltas = kDefaultSignedDeltas;
values_optional = kDefaultValuesOptional;
original_width_bits = kDefaultOriginalWidthBits;
value_width_bits = kDefaultValueWidthBits;
} else {
// signed_deltas
if (!reader->ReadBits(&read_buffer, kBitsInHeaderForSignedDeltas)) {
return nullptr;
}
signed_deltas = rtc::dchecked_cast<bool>(read_buffer);
if (signed_deltas) {
RTC_LOG(LS_WARNING) << "Not implemented.";
return nullptr;
}
// values_optional
if (!reader->ReadBits(&read_buffer, kBitsInHeaderForValuesOptional)) {
@ -554,16 +690,25 @@ std::unique_ptr<FixedLengthDeltaDecoder> FixedLengthDeltaDecoder::Create(
return nullptr;
}
// original_width_bits
if (!reader->ReadBits(&read_buffer, kBitsInHeaderForOriginalWidthBits)) {
// value_width_bits
if (!reader->ReadBits(&read_buffer, kBitsInHeaderForValueWidthBits)) {
return nullptr;
}
RTC_DCHECK_LE(read_buffer, 64 - 1); // See encoding for -1's rationale.
original_width_bits = read_buffer + 1; // See encoding for +1's rationale.
value_width_bits = read_buffer + 1; // See encoding for +1's rationale.
}
// Note: Because of the way the parameters are read, it is not possible
// for illegal values to be read. We check nevertheless, in case the code
// changes in the future in a way that breaks this promise.
if (!FixedLengthEncodingParameters::ValidParameters(
delta_width_bits, signed_deltas, values_optional, value_width_bits)) {
RTC_LOG(LS_WARNING) << "Corrupt log; illegal encoding parameters.";
return nullptr;
}
FixedLengthEncodingParameters params(delta_width_bits, signed_deltas,
values_optional, original_width_bits);
values_optional, value_width_bits);
return absl::WrapUnique(new FixedLengthDeltaDecoder(std::move(reader), params,
base, num_of_deltas));
}
@ -576,13 +721,10 @@ FixedLengthDeltaDecoder::FixedLengthDeltaDecoder(
: reader_(std::move(reader)),
params_(params),
base_(base),
num_of_deltas_(num_of_deltas),
value_mask_(MaxValueOfBitWidth(params_.original_width_bits)) {
num_of_deltas_(num_of_deltas) {
RTC_DCHECK(reader_);
// TODO(eladalon): Support signed deltas.
RTC_DCHECK(!params.signed_deltas) << "Not implemented.";
// TODO(eladalon): Support optional values.
RTC_DCHECK(!params.values_optional) << "Not implemented.";
RTC_DCHECK(!params.values_optional()) << "Not implemented.";
}
std::vector<uint64_t> FixedLengthDeltaDecoder::Decode() {
@ -603,15 +745,15 @@ std::vector<uint64_t> FixedLengthDeltaDecoder::Decode() {
bool FixedLengthDeltaDecoder::ParseDelta(uint64_t* delta) {
RTC_DCHECK(reader_);
RTC_DCHECK(!params_.signed_deltas) << "Not implemented."; // Reminder.
RTC_DCHECK(!params_.values_optional) << "Not implemented."; // Reminder.
RTC_DCHECK(!params_.values_optional()) << "Not implemented."; // Reminder.
// BitBuffer and BitBufferWriter read/write higher bits before lower bits.
const size_t lower_bit_count =
std::min<uint64_t>(params_.delta_width_bits, 32u);
const size_t higher_bit_count =
(params_.delta_width_bits <= 32u) ? 0 : params_.delta_width_bits - 32u;
std::min<uint64_t>(params_.delta_width_bits(), 32u);
const size_t higher_bit_count = (params_.delta_width_bits() <= 32u)
? 0
: params_.delta_width_bits() - 32u;
uint32_t lower_bits;
uint32_t higher_bits;
@ -639,13 +781,37 @@ bool FixedLengthDeltaDecoder::ParseDelta(uint64_t* delta) {
uint64_t FixedLengthDeltaDecoder::ApplyDelta(uint64_t base,
uint64_t delta) const {
RTC_DCHECK(!params_.signed_deltas) << "Not implemented."; // Reminder.
RTC_DCHECK(!params_.values_optional) << "Not implemented."; // Reminder.
RTC_DCHECK_LE(base, MaxValueOfBitWidth(params_.original_width_bits));
RTC_DCHECK_LE(delta, MaxValueOfBitWidth(params_.delta_width_bits));
RTC_DCHECK_LE(params_.delta_width_bits,
params_.original_width_bits); // Reminder.
return (base + delta) & value_mask_;
RTC_DCHECK(!params_.values_optional()) << "Not implemented."; // Reminder.
RTC_DCHECK_LE(base, MaxUnsignedValueOfBitWidth(params_.value_width_bits()));
RTC_DCHECK_LE(delta, MaxUnsignedValueOfBitWidth(params_.delta_width_bits()));
return params_.signed_deltas() ? ApplySignedDelta(base, delta)
: ApplyUnsignedDelta(base, delta);
}
uint64_t FixedLengthDeltaDecoder::ApplyUnsignedDelta(uint64_t base,
uint64_t delta) const {
// Note: May still be used if signed deltas used.
RTC_DCHECK_LE(base, MaxUnsignedValueOfBitWidth(params_.value_width_bits()));
RTC_DCHECK_LE(delta, MaxUnsignedValueOfBitWidth(params_.delta_width_bits()));
return (base + delta) & params_.value_mask();
}
uint64_t FixedLengthDeltaDecoder::ApplySignedDelta(uint64_t base,
uint64_t delta) const {
RTC_DCHECK(params_.signed_deltas());
RTC_DCHECK_LE(base, MaxUnsignedValueOfBitWidth(params_.value_width_bits()));
RTC_DCHECK_LE(delta, MaxUnsignedValueOfBitWidth(params_.delta_width_bits()));
const uint64_t top_bit = static_cast<uint64_t>(1)
<< (params_.delta_width_bits() - 1);
const bool positive_delta = ((delta & top_bit) == 0);
if (positive_delta) {
return ApplyUnsignedDelta(base, delta);
}
const uint64_t delta_abs = (~delta & params_.delta_mask()) + 1;
return (base - delta_abs) & params_.value_mask();
}
} // namespace
@ -676,4 +842,14 @@ std::vector<uint64_t> DecodeDeltas(const std::string& input,
return std::vector<uint64_t>();
}
void SetFixedLengthEncoderDeltaSignednessForTesting(bool signedness) {
g_force_unsigned_for_testing = !signedness;
g_force_signed_for_testing = signedness;
}
void UnsetFixedLengthEncoderDeltaSignednessForTesting() {
g_force_unsigned_for_testing = false;
g_force_signed_for_testing = false;
}
} // namespace webrtc

261
logging/rtc_event_log/encoder/delta_encoding_unittest.cc
View File

@ -16,14 +16,36 @@
#include <tuple>
#include <vector>
#include "absl/types/optional.h"
#include "rtc_base/arraysize.h"
#include "rtc_base/checks.h"
#include "rtc_base/random.h"
#include "test/gtest.h"
namespace webrtc {
void SetFixedLengthEncoderDeltaSignednessForTesting(bool signedness);
void UnsetFixedLengthEncoderDeltaSignednessForTesting();
namespace {
enum class DeltaSignedness { kNoOverride, kForceUnsigned, kForceSigned };
void MaybeSetSignedness(DeltaSignedness signedness) {
switch (signedness) {
case DeltaSignedness::kNoOverride:
UnsetFixedLengthEncoderDeltaSignednessForTesting();
return;
case DeltaSignedness::kForceUnsigned:
SetFixedLengthEncoderDeltaSignednessForTesting(false);
return;
case DeltaSignedness::kForceSigned:
SetFixedLengthEncoderDeltaSignednessForTesting(true);
return;
}
RTC_NOTREACHED();
}
uint64_t RandomWithMaxBitWidth(Random* prng, uint64_t max_width) {
RTC_DCHECK_GE(max_width, 1u);
RTC_DCHECK_LE(max_width, 64u);
@ -95,21 +117,42 @@ std::vector<uint64_t> CreateSequenceByDeltas(
}
size_t EncodingLengthUpperBound(size_t delta_max_bit_width,
size_t num_of_deltas) {
constexpr size_t kSmallestHeaderSizeBytes = 1;
return delta_max_bit_width * num_of_deltas + kSmallestHeaderSizeBytes;
size_t num_of_deltas,
DeltaSignedness signedness_override) {
absl::optional<size_t> smallest_header_size_bytes;
switch (signedness_override) {
case DeltaSignedness::kNoOverride:
case DeltaSignedness::kForceUnsigned:
smallest_header_size_bytes = 1;
break;
case DeltaSignedness::kForceSigned:
smallest_header_size_bytes = 2;
break;
}
RTC_DCHECK(smallest_header_size_bytes);
return delta_max_bit_width * num_of_deltas + *smallest_header_size_bytes;
}
// Tests of the delta encoding, parameterized by the number of values
// in the sequence created by the test.
class DeltaEncodingTest : public ::testing::TestWithParam<size_t> {
class DeltaEncodingTest
: public ::testing::TestWithParam<std::tuple<DeltaSignedness, size_t>> {
public:
DeltaEncodingTest()
: signedness_(std::get<0>(GetParam())),
num_of_values_(std::get<1>(GetParam())) {
MaybeSetSignedness(signedness_);
}
~DeltaEncodingTest() override = default;
const DeltaSignedness signedness_;
const uint64_t num_of_values_;
};
TEST_P(DeltaEncodingTest, AllValuesEqualToBaseValue) {
const uint64_t base = 3432;
std::vector<uint64_t> values(GetParam());
std::vector<uint64_t> values(num_of_values_);
std::fill(values.begin(), values.end(), base);
std::string encoded;
TestEncodingAndDecoding(base, values, &encoded);
@ -122,7 +165,7 @@ TEST_P(DeltaEncodingTest, AllValuesEqualToBaseValue) {
TEST_P(DeltaEncodingTest, MinDeltaNoWrapAround) {
const uint64_t base = 3432;
const auto values = CreateSequenceByFirstValue(base + 1, GetParam());
const auto values = CreateSequenceByFirstValue(base + 1, num_of_values_);
ASSERT_GT(values[values.size() - 1], base) << "Sanity; must not wrap around";
TestEncodingAndDecoding(base, values);
@ -132,7 +175,8 @@ TEST_P(DeltaEncodingTest, BigDeltaNoWrapAround) {
const uint64_t kBigDelta = 132828;
const uint64_t base = 3432;
const auto values = CreateSequenceByFirstValue(base + kBigDelta, GetParam());
const auto values =
CreateSequenceByFirstValue(base + kBigDelta, num_of_values_);
ASSERT_GT(values[values.size() - 1], base) << "Sanity; must not wrap around";
TestEncodingAndDecoding(base, values);
@ -142,7 +186,7 @@ TEST_P(DeltaEncodingTest, MaxDeltaNoWrapAround) {
const uint64_t base = 3432;
const auto values = CreateSequenceByLastValue(
std::numeric_limits<uint64_t>::max(), GetParam());
std::numeric_limits<uint64_t>::max(), num_of_values_);
ASSERT_GT(values[values.size() - 1], base) << "Sanity; must not wrap around";
TestEncodingAndDecoding(base, values);
@ -151,20 +195,20 @@ TEST_P(DeltaEncodingTest, MaxDeltaNoWrapAround) {
TEST_P(DeltaEncodingTest, SmallDeltaWithWrapAroundComparedToBase) {
const uint64_t base = std::numeric_limits<uint64_t>::max();
const auto values = CreateSequenceByDeltas(base, {1, 10, 3}, GetParam());
const auto values = CreateSequenceByDeltas(base, {1, 10, 3}, num_of_values_);
ASSERT_LT(values[values.size() - 1], base) << "Sanity; must wrap around";
TestEncodingAndDecoding(base, values);
}
TEST_P(DeltaEncodingTest, SmallDeltaWithWrapAroundInValueSequence) {
if (GetParam() == 1) {
if (num_of_values_ == 1) {
return; // Inapplicable.
}
const uint64_t base = std::numeric_limits<uint64_t>::max() - 2;
const auto values = CreateSequenceByDeltas(base, {1, 10, 3}, GetParam());
const auto values = CreateSequenceByDeltas(base, {1, 10, 3}, num_of_values_);
ASSERT_LT(values[values.size() - 1], values[0]) << "Sanity; must wrap around";
TestEncodingAndDecoding(base, values);
@ -179,14 +223,15 @@ TEST_P(DeltaEncodingTest, BigDeltaWithWrapAroundComparedToBase) {
const uint64_t kBigDelta = 132828;
const uint64_t base = std::numeric_limits<uint64_t>::max() - kBigDelta + 3;
const auto values = CreateSequenceByFirstValue(base + kBigDelta, GetParam());
const auto values =
CreateSequenceByFirstValue(base + kBigDelta, num_of_values_);
ASSERT_LT(values[values.size() - 1], base) << "Sanity; must wrap around";
TestEncodingAndDecoding(base, values);
}
TEST_P(DeltaEncodingTest, BigDeltaWithWrapAroundInValueSequence) {
if (GetParam() == 1) {
if (num_of_values_ == 1) {
return; // Inapplicable.
}
@ -194,7 +239,7 @@ TEST_P(DeltaEncodingTest, BigDeltaWithWrapAroundInValueSequence) {
const uint64_t base = std::numeric_limits<uint64_t>::max() - kBigDelta + 3;
const auto values = CreateSequenceByFirstValue(
std::numeric_limits<uint64_t>::max(), GetParam());
std::numeric_limits<uint64_t>::max(), num_of_values_);
ASSERT_LT(values[values.size() - 1], base) << "Sanity; must wrap around";
TestEncodingAndDecoding(base, values);
@ -205,27 +250,27 @@ TEST_P(DeltaEncodingTest, BigDeltaWithWrapAroundInValueSequence) {
TEST_P(DeltaEncodingTest, MaxDeltaWithWrapAroundComparedToBase) {
const uint64_t base = 3432;
const auto values = CreateSequenceByFirstValue(base - 1, GetParam());
const auto values = CreateSequenceByFirstValue(base - 1, num_of_values_);
TestEncodingAndDecoding(base, values);
}
TEST_P(DeltaEncodingTest, MaxDeltaWithWrapAroundInValueSequence) {
if (GetParam() == 1) {
if (num_of_values_ == 1) {
return; // Inapplicable.
}
const uint64_t base = 3432;
const auto values = CreateSequenceByDeltas(
base, {0, std::numeric_limits<uint64_t>::max(), 3}, GetParam());
base, {0, std::numeric_limits<uint64_t>::max(), 3}, num_of_values_);
ASSERT_LT(values[1], base) << "Sanity; must wrap around";
TestEncodingAndDecoding(base, values);
}
// If GetParam() == 1, a zero delta will yield an empty string; that's already
// covered by AllValuesEqualToBaseValue, but it doesn't hurt to test again.
// For all other cases, we have a new test.
// If num_of_values_ == 1, a zero delta will yield an empty string; that's
// already covered by AllValuesEqualToBaseValue, but it doesn't hurt to test
// again. For all other cases, we have a new test.
TEST_P(DeltaEncodingTest, ZeroDelta) {
const uint64_t base = 3432;
@ -233,26 +278,46 @@ TEST_P(DeltaEncodingTest, ZeroDelta) {
// consecutive zeros.
const std::vector<uint64_t> deltas = {0, 312, 11, 1, 1, 0, 0, 12,
400321, 3, 3, 12, 5, 0, 6};
const auto values = CreateSequenceByDeltas(base, deltas, GetParam());
const auto values = CreateSequenceByDeltas(base, deltas, num_of_values_);
TestEncodingAndDecoding(base, values);
}
INSTANTIATE_TEST_CASE_P(NumberOfValuesInSequence,
INSTANTIATE_TEST_CASE_P(
SignednessOverrideAndNumberOfValuesInSequence,
DeltaEncodingTest,
::testing::Values(1, 2, 100, 10000));
::testing::Combine(::testing::Values(DeltaSignedness::kNoOverride,
DeltaSignedness::kForceUnsigned,
DeltaSignedness::kForceSigned),
::testing::Values(1, 2, 100, 10000)));
// Tests over the quality of the compression (as opposed to its correctness).
// Not to be confused with tests of runtime efficiency.
class DeltaEncodingCompressionQualityTest
: public ::testing::TestWithParam<std::tuple<uint64_t, uint64_t>> {
: public ::testing::TestWithParam<
std::tuple<DeltaSignedness, uint64_t, uint64_t>> {
public:
DeltaEncodingCompressionQualityTest()
: delta_max_bit_width_(std::get<0>(GetParam())),
num_of_values_(std::get<1>(GetParam())) {}
: signedness_(std::get<0>(GetParam())),
delta_max_bit_width_(std::get<1>(GetParam())),
num_of_values_(std::get<2>(GetParam())) {
MaybeSetSignedness(signedness_);
}
~DeltaEncodingCompressionQualityTest() override = default;
// Running with the same seed for all variants would make all tests start
// with the same sequence; avoid this by making the seed different.
uint64_t Seed() const {
constexpr uint64_t non_zero_base_seed = 3012;
// Multiply everything but |non_zero_base_seed| by different prime numbers
// to produce unique results.
return non_zero_base_seed + 2 * static_cast<uint64_t>(signedness_) +
3 * delta_max_bit_width_ + 5 * delta_max_bit_width_ +
7 * num_of_values_;
}
const DeltaSignedness signedness_;
const uint64_t delta_max_bit_width_;
const uint64_t num_of_values_;
};
@ -261,12 +326,6 @@ class DeltaEncodingCompressionQualityTest
// matter to compression performance; only the deltas matter.
TEST_P(DeltaEncodingCompressionQualityTest,
BaseDoesNotAffectEfficiencyIfNoWrapAround) {
Random prng(3012);
std::vector<uint64_t> deltas(num_of_values_);
for (size_t i = 0; i < deltas.size(); ++i) {
deltas[i] = RandomWithMaxBitWidth(&prng, delta_max_bit_width_);
}
// 1. Bases which will not produce a wrap-around.
// 2. The last base - 0xffffffffffffffff - does cause a wrap-around, but
// that still works, because the width is 64 anyway, and does not
@ -274,6 +333,46 @@ TEST_P(DeltaEncodingCompressionQualityTest,
const uint64_t bases[] = {0, 0x55, 0xffffffff,
std::numeric_limits<uint64_t>::max()};
const size_t kIntendedWrapAroundBaseIndex = arraysize(bases);
std::vector<uint64_t> deltas(num_of_values_);
// Allows us to make sure that the deltas do not produce a wrap-around.
uint64_t last_element[arraysize(bases)];
memcpy(last_element, bases, sizeof(bases));
// Avoid empty |deltas| due to first element causing wrap-around.
deltas[0] = 1;
for (size_t i = 0; i < arraysize(last_element); ++i) {
last_element[i] += 1;
}
Random prng(Seed());
for (size_t i = 1; i < deltas.size(); ++i) {
const uint64_t delta = RandomWithMaxBitWidth(&prng, delta_max_bit_width_);
bool wrap_around = false;
for (size_t j = 0; j < arraysize(last_element); ++j) {
if (j == kIntendedWrapAroundBaseIndex) {
continue;
}
last_element[j] += delta;
if (last_element[j] < bases[j]) {
wrap_around = true;
break;
}
}
if (wrap_around) {
deltas.resize(i);
break;
}
deltas[i] = delta;
}
std::string encodings[arraysize(bases)];
for (size_t i = 0; i < arraysize(bases); ++i) {
@ -284,7 +383,8 @@ TEST_P(DeltaEncodingCompressionQualityTest,
// the encoding/decoding, though that is not the test's focus.
TestEncodingAndDecoding(bases[i], values, &encodings[i]);
EXPECT_LE(encodings[i].length(),
EncodingLengthUpperBound(delta_max_bit_width_, num_of_values_));
EncodingLengthUpperBound(delta_max_bit_width_, num_of_values_,
signedness_));
}
// Test focus - all of the encodings should be the same, as they are based
@ -295,23 +395,42 @@ TEST_P(DeltaEncodingCompressionQualityTest,
}
INSTANTIATE_TEST_CASE_P(
DeltaMaxBitWidthAndNumberOfValuesInSequence,
SignednessOverrideAndDeltaMaxBitWidthAndNumberOfValuesInSequence,
DeltaEncodingCompressionQualityTest,
::testing::Combine(
::testing::Values(DeltaSignedness::kNoOverride,
DeltaSignedness::kForceUnsigned,
DeltaSignedness::kForceSigned),
::testing::Values(1, 4, 8, 15, 16, 17, 31, 32, 33, 63, 64),
::testing::Values(1, 2, 100, 10000)));
// Similar to DeltaEncodingTest, but instead of semi-surgically producing
// specific cases, produce large amount of semi-realistic inputs.
class DeltaEncodingFuzzerLikeTest
: public ::testing::TestWithParam<std::tuple<uint64_t, uint64_t>> {
: public ::testing::TestWithParam<
std::tuple<DeltaSignedness, uint64_t, uint64_t>> {
public:
DeltaEncodingFuzzerLikeTest()
: delta_max_bit_width_(std::get<0>(GetParam())),
num_of_values_(std::get<1>(GetParam())) {}
: signedness_(std::get<0>(GetParam())),
delta_max_bit_width_(std::get<1>(GetParam())),
num_of_values_(std::get<2>(GetParam())) {
MaybeSetSignedness(signedness_);
}
~DeltaEncodingFuzzerLikeTest() override = default;
// Running with the same seed for all variants would make all tests start
// with the same sequence; avoid this by making the seed different.
uint64_t Seed() const {
constexpr uint64_t non_zero_base_seed = 1983;
// Multiply everything but |non_zero_base_seed| by different prime numbers
// to produce unique results.
return non_zero_base_seed + 2 * static_cast<uint64_t>(signedness_) +
3 * delta_max_bit_width_ + 5 * delta_max_bit_width_ +
7 * num_of_values_;
}
const DeltaSignedness signedness_;
const uint64_t delta_max_bit_width_;
const uint64_t num_of_values_;
};
@ -319,7 +438,7 @@ class DeltaEncodingFuzzerLikeTest
TEST_P(DeltaEncodingFuzzerLikeTest, Test) {
const uint64_t base = 3432;
Random prng(1983);
Random prng(Seed());
std::vector<uint64_t> deltas(num_of_values_);
for (size_t i = 0; i < deltas.size(); ++i) {
deltas[i] = RandomWithMaxBitWidth(&prng, delta_max_bit_width_);
@ -331,11 +450,75 @@ TEST_P(DeltaEncodingFuzzerLikeTest, Test) {
}
INSTANTIATE_TEST_CASE_P(
DeltaMaxBitWidthAndNumberOfValuesInSequence,
SignednessOverrideAndDeltaMaxBitWidthAndNumberOfValuesInSequence,
DeltaEncodingFuzzerLikeTest,
::testing::Combine(
::testing::Values(DeltaSignedness::kNoOverride,
DeltaSignedness::kForceUnsigned,
DeltaSignedness::kForceSigned),
::testing::Values(1, 4, 8, 15, 16, 17, 31, 32, 33, 63, 64),
::testing::Values(1, 2, 100, 10000)));
class DeltaEncodingSpecificEdgeCasesTest
: public ::testing::TestWithParam<
std::tuple<DeltaSignedness, uint64_t, bool>> {
public:
DeltaEncodingSpecificEdgeCasesTest() {
UnsetFixedLengthEncoderDeltaSignednessForTesting();
}
~DeltaEncodingSpecificEdgeCasesTest() override = default;
};
// This case is special because it produces identical forward/backward deltas.
TEST_F(DeltaEncodingSpecificEdgeCasesTest, SignedDeltaWithOnlyTopBitOn) {
MaybeSetSignedness(DeltaSignedness::kForceSigned);
const uint64_t base = 3432;
const uint64_t delta = static_cast<uint64_t>(1) << 63;
const std::vector<uint64_t> values = {base + delta};
TestEncodingAndDecoding(base, values);
}
TEST_F(DeltaEncodingSpecificEdgeCasesTest, MaximumUnsignedDelta) {
MaybeSetSignedness(DeltaSignedness::kForceUnsigned);
const uint64_t base = (static_cast<uint64_t>(1) << 63) + 0x123;
const std::vector<uint64_t> values = {base - 1};
TestEncodingAndDecoding(base, values);
}
// Check that, if all deltas are set to -1, things still work.
TEST_P(DeltaEncodingSpecificEdgeCasesTest, ReverseSequence) {
MaybeSetSignedness(std::get<0>(GetParam()));
const uint64_t width = std::get<1>(GetParam());
const bool wrap_around = std::get<2>(GetParam());
const uint64_t value_mask = (width == 64)
? std::numeric_limits<uint64_t>::max()
: ((static_cast<uint64_t>(1) << width) - 1);
const uint64_t base = wrap_around ? 1u : (0xf82d3 & value_mask);
const std::vector<uint64_t> values = {(base - 1u) & value_mask,
(base - 2u) & value_mask,
(base - 3u) & value_mask};
TestEncodingAndDecoding(base, values);
}
INSTANTIATE_TEST_CASE_P(
_,
DeltaEncodingSpecificEdgeCasesTest,
::testing::Combine(
::testing::Values(DeltaSignedness::kNoOverride,
DeltaSignedness::kForceUnsigned,
DeltaSignedness::kForceSigned),
::testing::Values(1, 4, 8, 15, 16, 17, 31, 32, 33, 63, 64),
::testing::Bool()));
} // namespace
} // namespace webrtc