Add support for signed deltas in FixedLengthDeltaEncoder
Signed deltas can yield a more efficient encoding when the encoded
sequence sometimes moves backwards.
Bug: webrtc:8111
Change-Id: Ib1a50192851214ccc3f2bd7eaf88f4be97e4beb0
Reviewed-on: https://webrtc-review.googlesource.com/c/100423
Commit-Queue: Elad Alon <eladalon@webrtc.org>
Reviewed-by: Björn Terelius <terelius@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#25324}diff --git a/logging/BUILD.gn b/logging/BUILD.gn
index ac0b720..571ddf8 100644
--- a/logging/BUILD.gn
+++ b/logging/BUILD.gn
@@ -183,6 +183,7 @@
"../rtc_base:rtc_base_approved",
"//third_party/abseil-cpp/absl/memory",
"//third_party/abseil-cpp/absl/strings:strings",
+ "//third_party/abseil-cpp/absl/types:optional",
]
if (rtc_enable_protobuf) {
diff --git a/logging/rtc_event_log/encoder/delta_encoding.cc b/logging/rtc_event_log/encoder/delta_encoding.cc
index 2da4ca7..755eb41 100644
--- a/logging/rtc_event_log/encoder/delta_encoding.cc
+++ b/logging/rtc_event_log/encoder/delta_encoding.cc
@@ -27,12 +27,19 @@
// TODO(eladalon): Only build the decoder in tools and unit tests.
+bool g_force_unsigned_for_testing = false;
+bool g_force_signed_for_testing = false;
+
size_t BitsToBytes(size_t bits) {
return (bits / 8) + (bits % 8 > 0 ? 1 : 0);
}
// TODO(eladalon): Replace by something more efficient.
-uint64_t BitWidth(uint64_t input) {
+uint64_t UnsignedBitWidth(uint64_t input, bool zero_val_as_zero_width = false) {
+ if (zero_val_as_zero_width && input == 0) {
+ return 0;
+ }
+
uint64_t width = 0;
do { // input == 0 -> width == 1
width += 1;
@@ -41,13 +48,22 @@
return width;
}
+uint64_t SignedBitWidth(uint64_t max_pos_magnitude,
+ uint64_t max_neg_magnitude) {
+ const uint64_t bitwidth_pos = UnsignedBitWidth(max_pos_magnitude, true);
+ const uint64_t bitwidth_neg =
+ (max_neg_magnitude > 0) ? UnsignedBitWidth(max_neg_magnitude - 1, true)
+ : 0;
+ return 1 + std::max(bitwidth_pos, bitwidth_neg);
+}
+
// Return the maximum integer of a given bit width.
// Examples:
-// MaxValueOfBitWidth(1) = 0x01
-// MaxValueOfBitWidth(6) = 0x3f
-// MaxValueOfBitWidth(8) = 0xff
-// MaxValueOfBitWidth(32) = 0xffffffff
-uint64_t MaxValueOfBitWidth(uint64_t bit_width) {
+// MaxUnsignedValueOfBitWidth(1) = 0x01
+// MaxUnsignedValueOfBitWidth(6) = 0x3f
+// MaxUnsignedValueOfBitWidth(8) = 0xff
+// MaxUnsignedValueOfBitWidth(32) = 0xffffffff
+uint64_t MaxUnsignedValueOfBitWidth(uint64_t bit_width) {
RTC_DCHECK_GE(bit_width, 1);
RTC_DCHECK_LE(bit_width, 64);
return (bit_width == 64) ? std::numeric_limits<uint64_t>::max()
@@ -55,18 +71,9 @@
}
// Computes the delta between |previous| and |current|, under the assumption
-// that wrap-around occurs after width |bit_width| is exceeded.
-uint64_t ComputeDelta(uint64_t previous, uint64_t current, uint64_t width) {
- RTC_DCHECK(width == 64 || current < (static_cast<uint64_t>(1) << width));
- RTC_DCHECK(width == 64 || previous < (static_cast<uint64_t>(1) << width));
-
- if (current >= previous) {
- // Simply "walk" forward.
- return current - previous;
- } else { // previous > current
- // "Walk" until the max value, one more step to 0, then to |current|.
- return (MaxValueOfBitWidth(width) - previous) + 1 + current;
- }
+// that wrap-around occurs after |width| is exceeded.
+uint64_t UnsignedDelta(uint64_t previous, uint64_t current, uint64_t bit_mask) {
+ return (current - previous) & bit_mask;
}
// Determines the encoding type (e.g. fixed-size encoding).
@@ -87,7 +94,7 @@
constexpr size_t kBitsInHeaderForDeltaWidthBits = 6;
constexpr size_t kBitsInHeaderForSignedDeltas = 1;
constexpr size_t kBitsInHeaderForValuesOptional = 1;
-constexpr size_t kBitsInHeaderForOriginalWidthBits = 6;
+constexpr size_t kBitsInHeaderForValueWidthBits = 6;
static_assert(static_cast<size_t>(EncodingType::kNumberOfEncodingTypes) <=
1 << kBitsInHeaderForEncodingType,
@@ -96,7 +103,7 @@
// Default values for when the encoding header does not specify explicitly.
constexpr bool kDefaultSignedDeltas = false;
constexpr bool kDefaultValuesOptional = false;
-constexpr uint64_t kDefaultOriginalWidthBits = 64;
+constexpr uint64_t kDefaultValueWidthBits = 64;
// Wrap BitBufferWriter and extend its functionality by (1) keeping track of
// the number of bits written and (2) owning its buffer.
@@ -145,33 +152,66 @@
// Parameters for fixed-size delta-encoding/decoding.
// These are tailored for the sequence which will be encoded (e.g. widths).
-struct FixedLengthEncodingParameters final {
+class FixedLengthEncodingParameters final {
+ public:
+ static bool ValidParameters(uint64_t delta_width_bits,
+ bool signed_deltas,
+ bool values_optional,
+ uint64_t value_width_bits) {
+ return (1 <= delta_width_bits && delta_width_bits <= 64 &&
+ 1 <= value_width_bits && value_width_bits <= 64 &&
+ delta_width_bits <= value_width_bits);
+ }
+
FixedLengthEncodingParameters(uint64_t delta_width_bits,
bool signed_deltas,
bool values_optional,
- uint64_t original_width_bits)
- : delta_width_bits(delta_width_bits),
- signed_deltas(signed_deltas),
- values_optional(values_optional),
- original_width_bits(original_width_bits) {}
+ uint64_t value_width_bits)
+ : delta_width_bits_(delta_width_bits),
+ signed_deltas_(signed_deltas),
+ values_optional_(values_optional),
+ value_width_bits_(value_width_bits),
+ delta_mask_(MaxUnsignedValueOfBitWidth(delta_width_bits_)),
+ value_mask_(MaxUnsignedValueOfBitWidth(value_width_bits_)) {
+ RTC_DCHECK(ValidParameters(delta_width_bits, signed_deltas, values_optional,
+ value_width_bits));
+ }
// Number of bits necessary to hold the widest(*) of the deltas between the
// values in the sequence.
// (*) - Widest might not be the largest, if signed deltas are used.
- uint64_t delta_width_bits;
+ uint64_t delta_width_bits() const { return delta_width_bits_; }
// Whether deltas are signed.
- // TODO(eladalon): Add support for signed deltas.
- bool signed_deltas;
+ bool signed_deltas() const { return signed_deltas_; }
// Whether the values of the sequence are optional. That is, it may be
// that some of them do not have a value (not even a sentinel value indicating
// invalidity).
// TODO(eladalon): Add support for optional elements.
- bool values_optional;
+ bool values_optional() const { return values_optional_; }
// Number of bits necessary to hold the largest value in the sequence.
- uint64_t original_width_bits;
+ uint64_t value_width_bits() const { return value_width_bits_; }
+
+ // Masks where only the bits relevant to the deltas/values are turned on.
+ uint64_t delta_mask() const { return delta_mask_; }
+ uint64_t value_mask() const { return value_mask_; }
+
+ void SetSignedDeltas(bool signed_deltas) { signed_deltas_ = signed_deltas; }
+ void SetDeltaWidthBits(uint64_t delta_width_bits) {
+ delta_width_bits_ = delta_width_bits;
+ delta_mask_ = MaxUnsignedValueOfBitWidth(delta_width_bits);
+ }
+
+ private:
+ uint64_t delta_width_bits_; // Normally const, but mutable in tests.
+ bool signed_deltas_; // Normally const, but mutable in tests.
+ const bool values_optional_;
+ const uint64_t value_width_bits_;
+
+ uint64_t delta_mask_; // Normally const, but mutable in tests.
+ const uint64_t value_mask_;
};
// Performs delta-encoding of a single (non-empty) sequence of values, using
@@ -190,6 +230,21 @@
const std::vector<uint64_t>& values);
private:
+ // Calculate min/max values of unsigned/signed deltas, given the bit width
+ // of all the values in the series.
+ static void CalculateMinAndMaxDeltas(uint64_t base,
+ const std::vector<uint64_t>& values,
+ uint64_t bit_width,
+ uint64_t* max_unsigned_delta,
+ uint64_t* max_pos_signed_delta,
+ uint64_t* min_neg_signed_delta);
+
+ // No effect outside of unit tests.
+ // In unit tests, may lead to forcing signed/unsigned deltas, etc.
+ static void ConsiderTestOverrides(FixedLengthEncodingParameters* params,
+ uint64_t delta_width_bits_signed,
+ uint64_t delta_width_bits_unsigned);
+
// FixedLengthDeltaEncoder objects are to be created by EncodeDeltas() and
// released by it before it returns. They're mostly a convenient way to
// avoid having to pass a lot of state between different functions.
@@ -213,6 +268,8 @@
// Encode a given delta into the stream.
void EncodeDelta(uint64_t previous, uint64_t current);
+ void EncodeUnsignedDelta(uint64_t previous, uint64_t current);
+ void EncodeSignedDelta(uint64_t previous, uint64_t current);
// The parameters according to which encoding will be done (width of
// fields, whether signed deltas should be used, etc.)
@@ -251,16 +308,14 @@
// If the sequence is non-decreasing, it may be assumed to have width = 64;
// there's no reason to encode the actual max width in the encoding header.
- const uint64_t original_width_bits =
- non_decreasing ? 64 : BitWidth(max_value_including_base);
+ const uint64_t value_width_bits =
+ non_decreasing ? 64 : UnsignedBitWidth(max_value_including_base);
- uint64_t max_unsigned_delta =
- ComputeDelta(base, values[0], original_width_bits);
- for (size_t i = 1; i < values.size(); ++i) {
- const uint64_t unsigned_delta =
- ComputeDelta(values[i - 1], values[i], original_width_bits);
- max_unsigned_delta = std::max(unsigned_delta, max_unsigned_delta);
- }
+ uint64_t max_unsigned_delta;
+ uint64_t max_pos_signed_delta;
+ uint64_t min_neg_signed_delta;
+ CalculateMinAndMaxDeltas(base, values, value_width_bits, &max_unsigned_delta,
+ &max_pos_signed_delta, &min_neg_signed_delta);
// We indicate the special case of all values being equal to the base with
// the empty string.
@@ -270,11 +325,10 @@
return std::string();
}
- const uint64_t delta_width_bits_unsigned = BitWidth(max_unsigned_delta);
- // This will prevent the use of signed deltas, by always assuming
- // they will not provide value over unsigned.
- // TODO(eladalon): Add support for signed deltas.
- const uint64_t delta_width_bits_signed = 64;
+ const uint64_t delta_width_bits_unsigned =
+ UnsignedBitWidth(max_unsigned_delta);
+ const uint64_t delta_width_bits_signed =
+ SignedBitWidth(max_pos_signed_delta, min_neg_signed_delta);
// Note: Preference for unsigned if the two have the same width (efficiency).
const bool signed_deltas =
@@ -285,21 +339,75 @@
const bool values_optional = false;
FixedLengthEncodingParameters params(delta_width_bits, signed_deltas,
- values_optional, original_width_bits);
+ values_optional, value_width_bits);
+
+ // No effect in production.
+ ConsiderTestOverrides(¶ms, delta_width_bits_signed,
+ delta_width_bits_unsigned);
+
FixedLengthDeltaEncoder encoder(params, base, values);
return encoder.Encode();
}
+void FixedLengthDeltaEncoder::CalculateMinAndMaxDeltas(
+ uint64_t base,
+ const std::vector<uint64_t>& values,
+ uint64_t bit_width,
+ uint64_t* max_unsigned_delta_out,
+ uint64_t* max_pos_signed_delta_out,
+ uint64_t* min_neg_signed_delta_out) {
+ RTC_DCHECK(!values.empty());
+ RTC_DCHECK(max_unsigned_delta_out);
+ RTC_DCHECK(max_pos_signed_delta_out);
+ RTC_DCHECK(min_neg_signed_delta_out);
+
+ const uint64_t bit_mask = MaxUnsignedValueOfBitWidth(bit_width);
+
+ uint64_t max_unsigned_delta = 0;
+ uint64_t max_pos_signed_delta = 0;
+ uint64_t min_neg_signed_delta = 0;
+
+ uint64_t prev = base;
+ for (size_t i = 0; i < values.size(); ++i) {
+ const uint64_t forward_delta = UnsignedDelta(prev, values[i], bit_mask);
+ const uint64_t backward_delta = UnsignedDelta(values[i], prev, bit_mask);
+
+ max_unsigned_delta = std::max(max_unsigned_delta, forward_delta);
+
+ if (forward_delta < backward_delta) {
+ max_pos_signed_delta = std::max(max_pos_signed_delta, forward_delta);
+ } else {
+ min_neg_signed_delta = std::max(min_neg_signed_delta, backward_delta);
+ }
+
+ prev = values[i];
+ }
+
+ *max_unsigned_delta_out = max_unsigned_delta;
+ *max_pos_signed_delta_out = max_pos_signed_delta;
+ *min_neg_signed_delta_out = min_neg_signed_delta;
+}
+
+void FixedLengthDeltaEncoder::ConsiderTestOverrides(
+ FixedLengthEncodingParameters* params,
+ uint64_t delta_width_bits_signed,
+ uint64_t delta_width_bits_unsigned) {
+ if (g_force_unsigned_for_testing) {
+ params->SetDeltaWidthBits(delta_width_bits_unsigned);
+ params->SetSignedDeltas(false);
+ } else if (g_force_signed_for_testing) {
+ params->SetDeltaWidthBits(delta_width_bits_signed);
+ params->SetSignedDeltas(true);
+ } else {
+ // Unchanged.
+ }
+}
+
FixedLengthDeltaEncoder::FixedLengthDeltaEncoder(
const FixedLengthEncodingParameters& params,
uint64_t base,
const std::vector<uint64_t>& values)
: params_(params), base_(base), values_(values) {
- RTC_DCHECK_GE(params_.delta_width_bits, 1);
- RTC_DCHECK_LE(params_.delta_width_bits, 64);
- RTC_DCHECK_GE(params_.original_width_bits, 1);
- RTC_DCHECK_LE(params_.original_width_bits, 64);
- RTC_DCHECK_LE(params_.delta_width_bits, params_.original_width_bits);
RTC_DCHECK(!values_.empty());
writer_ = absl::make_unique<BitWriter>(OutputLengthBytes());
}
@@ -322,31 +430,31 @@
}
size_t FixedLengthDeltaEncoder::HeaderLengthBits() const {
- if (params_.signed_deltas == kDefaultSignedDeltas &&
- params_.values_optional == kDefaultValuesOptional &&
- params_.original_width_bits == kDefaultOriginalWidthBits) {
+ if (params_.signed_deltas() == kDefaultSignedDeltas &&
+ params_.values_optional() == kDefaultValuesOptional &&
+ params_.value_width_bits() == kDefaultValueWidthBits) {
return kBitsInHeaderForEncodingType + kBitsInHeaderForDeltaWidthBits;
} else {
return kBitsInHeaderForEncodingType + kBitsInHeaderForDeltaWidthBits +
kBitsInHeaderForSignedDeltas + kBitsInHeaderForValuesOptional +
- kBitsInHeaderForOriginalWidthBits;
+ kBitsInHeaderForValueWidthBits;
}
}
size_t FixedLengthDeltaEncoder::EncodedDeltasLengthBits() const {
// TODO(eladalon): When optional values are supported, iterate over the
// deltas to check the exact cost of each.
- RTC_DCHECK(!params_.values_optional);
- return values_.size() * params_.delta_width_bits;
+ RTC_DCHECK(!params_.values_optional());
+ return values_.size() * params_.delta_width_bits();
}
void FixedLengthDeltaEncoder::EncodeHeader() {
RTC_DCHECK(writer_);
const EncodingType encoding_type =
- (params_.original_width_bits == kDefaultOriginalWidthBits &&
- params_.signed_deltas == kDefaultSignedDeltas &&
- params_.values_optional == kDefaultValuesOptional)
+ (params_.value_width_bits() == kDefaultValueWidthBits &&
+ params_.signed_deltas() == kDefaultSignedDeltas &&
+ params_.values_optional() == kDefaultValuesOptional)
? EncodingType::kFixedSizeUnsignedDeltasNoEarlyWrapNoOpt
: EncodingType::kFixedSizeSignedDeltasEarlyWrapAndOptSupported;
@@ -356,26 +464,59 @@
// Note: Since it's meaningless for a field to be of width 0, when it comes
// to fields that relate widths, we encode width 1 as 0, width 2 as 1,
- writer_->WriteBits(params_.delta_width_bits - 1,
+ writer_->WriteBits(params_.delta_width_bits() - 1,
kBitsInHeaderForDeltaWidthBits);
if (encoding_type == EncodingType::kFixedSizeUnsignedDeltasNoEarlyWrapNoOpt) {
return;
}
- writer_->WriteBits(static_cast<uint64_t>(params_.signed_deltas),
+ writer_->WriteBits(static_cast<uint64_t>(params_.signed_deltas()),
kBitsInHeaderForSignedDeltas);
- writer_->WriteBits(static_cast<uint64_t>(params_.values_optional),
+ writer_->WriteBits(static_cast<uint64_t>(params_.values_optional()),
kBitsInHeaderForValuesOptional);
- writer_->WriteBits(params_.original_width_bits - 1,
- kBitsInHeaderForOriginalWidthBits);
+ writer_->WriteBits(params_.value_width_bits() - 1,
+ kBitsInHeaderForValueWidthBits);
}
void FixedLengthDeltaEncoder::EncodeDelta(uint64_t previous, uint64_t current) {
+ if (params_.signed_deltas()) {
+ EncodeSignedDelta(previous, current);
+ } else {
+ EncodeUnsignedDelta(previous, current);
+ }
+}
+
+void FixedLengthDeltaEncoder::EncodeUnsignedDelta(uint64_t previous,
+ uint64_t current) {
RTC_DCHECK(writer_);
- writer_->WriteBits(
- ComputeDelta(previous, current, params_.original_width_bits),
- params_.delta_width_bits);
+ const uint64_t delta = UnsignedDelta(previous, current, params_.value_mask());
+ writer_->WriteBits(delta, params_.delta_width_bits());
+}
+
+void FixedLengthDeltaEncoder::EncodeSignedDelta(uint64_t previous,
+ uint64_t current) {
+ RTC_DCHECK(writer_);
+
+ const uint64_t forward_delta =
+ UnsignedDelta(previous, current, params_.value_mask());
+ const uint64_t backward_delta =
+ UnsignedDelta(current, previous, params_.value_mask());
+
+ uint64_t delta;
+ if (forward_delta <= backward_delta) {
+ delta = forward_delta;
+ } else {
+ // Compute the unsigned representation of a negative delta.
+ // This is the two's complement representation of this negative value,
+ // when deltas are of width params_.delta_mask().
+ RTC_DCHECK_GE(params_.delta_mask(), backward_delta);
+ RTC_DCHECK_LT(params_.delta_mask() - backward_delta, params_.delta_mask());
+ delta = params_.delta_mask() - backward_delta + 1;
+ RTC_DCHECK_LE(delta, params_.delta_mask());
+ }
+
+ writer_->WriteBits(delta, params_.delta_width_bits());
}
// Perform decoding of a a delta-encoded stream, extracting the original
@@ -434,6 +575,10 @@
// values' width, as specified by the aforementioned encoding parameters.
uint64_t ApplyDelta(uint64_t base, uint64_t delta) const;
+ // Helpers for ApplyDelta().
+ uint64_t ApplyUnsignedDelta(uint64_t base, uint64_t delta) const;
+ uint64_t ApplySignedDelta(uint64_t base, uint64_t delta) const;
+
// Reader of the input stream to be decoded. Does not own that buffer.
// See comment above ctor for details.
const std::unique_ptr<rtc::BitBuffer> reader_;
@@ -450,11 +595,6 @@
// The number of values to be known to be decoded.
const size_t num_of_deltas_;
- // Bit mask corresponding to |params_.original_width_bits|. That is, the bits
- // necessary for encoding all of the values in the encoded sequence are on.
- // Used as an optimization.
- const uint64_t value_mask_;
-
RTC_DISALLOW_COPY_AND_ASSIGN(FixedLengthDeltaDecoder);
};
@@ -524,24 +664,20 @@
const uint64_t delta_width_bits =
read_buffer + 1; // See encoding for +1's rationale.
- // signed_deltas, values_optional, original_width_bits
+ // signed_deltas, values_optional, value_width_bits
bool signed_deltas;
bool values_optional;
- uint64_t original_width_bits;
+ uint64_t value_width_bits;
if (encoding == EncodingType::kFixedSizeUnsignedDeltasNoEarlyWrapNoOpt) {
signed_deltas = kDefaultSignedDeltas;
values_optional = kDefaultValuesOptional;
- original_width_bits = kDefaultOriginalWidthBits;
+ value_width_bits = kDefaultValueWidthBits;
} else {
// signed_deltas
if (!reader->ReadBits(&read_buffer, kBitsInHeaderForSignedDeltas)) {
return nullptr;
}
signed_deltas = rtc::dchecked_cast<bool>(read_buffer);
- if (signed_deltas) {
- RTC_LOG(LS_WARNING) << "Not implemented.";
- return nullptr;
- }
// values_optional
if (!reader->ReadBits(&read_buffer, kBitsInHeaderForValuesOptional)) {
@@ -554,16 +690,25 @@
return nullptr;
}
- // original_width_bits
- if (!reader->ReadBits(&read_buffer, kBitsInHeaderForOriginalWidthBits)) {
+ // value_width_bits
+ if (!reader->ReadBits(&read_buffer, kBitsInHeaderForValueWidthBits)) {
return nullptr;
}
- RTC_DCHECK_LE(read_buffer, 64 - 1); // See encoding for -1's rationale.
- original_width_bits = read_buffer + 1; // See encoding for +1's rationale.
+ RTC_DCHECK_LE(read_buffer, 64 - 1); // See encoding for -1's rationale.
+ value_width_bits = read_buffer + 1; // See encoding for +1's rationale.
+ }
+
+ // Note: Because of the way the parameters are read, it is not possible
+ // for illegal values to be read. We check nevertheless, in case the code
+ // changes in the future in a way that breaks this promise.
+ if (!FixedLengthEncodingParameters::ValidParameters(
+ delta_width_bits, signed_deltas, values_optional, value_width_bits)) {
+ RTC_LOG(LS_WARNING) << "Corrupt log; illegal encoding parameters.";
+ return nullptr;
}
FixedLengthEncodingParameters params(delta_width_bits, signed_deltas,
- values_optional, original_width_bits);
+ values_optional, value_width_bits);
return absl::WrapUnique(new FixedLengthDeltaDecoder(std::move(reader), params,
base, num_of_deltas));
}
@@ -576,13 +721,10 @@
: reader_(std::move(reader)),
params_(params),
base_(base),
- num_of_deltas_(num_of_deltas),
- value_mask_(MaxValueOfBitWidth(params_.original_width_bits)) {
+ num_of_deltas_(num_of_deltas) {
RTC_DCHECK(reader_);
- // TODO(eladalon): Support signed deltas.
- RTC_DCHECK(!params.signed_deltas) << "Not implemented.";
// TODO(eladalon): Support optional values.
- RTC_DCHECK(!params.values_optional) << "Not implemented.";
+ RTC_DCHECK(!params.values_optional()) << "Not implemented.";
}
std::vector<uint64_t> FixedLengthDeltaDecoder::Decode() {
@@ -603,15 +745,15 @@
bool FixedLengthDeltaDecoder::ParseDelta(uint64_t* delta) {
RTC_DCHECK(reader_);
- RTC_DCHECK(!params_.signed_deltas) << "Not implemented."; // Reminder.
- RTC_DCHECK(!params_.values_optional) << "Not implemented."; // Reminder.
+ RTC_DCHECK(!params_.values_optional()) << "Not implemented."; // Reminder.
// BitBuffer and BitBufferWriter read/write higher bits before lower bits.
const size_t lower_bit_count =
- std::min<uint64_t>(params_.delta_width_bits, 32u);
- const size_t higher_bit_count =
- (params_.delta_width_bits <= 32u) ? 0 : params_.delta_width_bits - 32u;
+ std::min<uint64_t>(params_.delta_width_bits(), 32u);
+ const size_t higher_bit_count = (params_.delta_width_bits() <= 32u)
+ ? 0
+ : params_.delta_width_bits() - 32u;
uint32_t lower_bits;
uint32_t higher_bits;
@@ -639,13 +781,37 @@
uint64_t FixedLengthDeltaDecoder::ApplyDelta(uint64_t base,
uint64_t delta) const {
- RTC_DCHECK(!params_.signed_deltas) << "Not implemented."; // Reminder.
- RTC_DCHECK(!params_.values_optional) << "Not implemented."; // Reminder.
- RTC_DCHECK_LE(base, MaxValueOfBitWidth(params_.original_width_bits));
- RTC_DCHECK_LE(delta, MaxValueOfBitWidth(params_.delta_width_bits));
- RTC_DCHECK_LE(params_.delta_width_bits,
- params_.original_width_bits); // Reminder.
- return (base + delta) & value_mask_;
+ RTC_DCHECK(!params_.values_optional()) << "Not implemented."; // Reminder.
+ RTC_DCHECK_LE(base, MaxUnsignedValueOfBitWidth(params_.value_width_bits()));
+ RTC_DCHECK_LE(delta, MaxUnsignedValueOfBitWidth(params_.delta_width_bits()));
+ return params_.signed_deltas() ? ApplySignedDelta(base, delta)
+ : ApplyUnsignedDelta(base, delta);
+}
+
+uint64_t FixedLengthDeltaDecoder::ApplyUnsignedDelta(uint64_t base,
+ uint64_t delta) const {
+ // Note: May still be used if signed deltas used.
+ RTC_DCHECK_LE(base, MaxUnsignedValueOfBitWidth(params_.value_width_bits()));
+ RTC_DCHECK_LE(delta, MaxUnsignedValueOfBitWidth(params_.delta_width_bits()));
+ return (base + delta) & params_.value_mask();
+}
+
+uint64_t FixedLengthDeltaDecoder::ApplySignedDelta(uint64_t base,
+ uint64_t delta) const {
+ RTC_DCHECK(params_.signed_deltas());
+ RTC_DCHECK_LE(base, MaxUnsignedValueOfBitWidth(params_.value_width_bits()));
+ RTC_DCHECK_LE(delta, MaxUnsignedValueOfBitWidth(params_.delta_width_bits()));
+
+ const uint64_t top_bit = static_cast<uint64_t>(1)
+ << (params_.delta_width_bits() - 1);
+
+ const bool positive_delta = ((delta & top_bit) == 0);
+ if (positive_delta) {
+ return ApplyUnsignedDelta(base, delta);
+ }
+
+ const uint64_t delta_abs = (~delta & params_.delta_mask()) + 1;
+ return (base - delta_abs) & params_.value_mask();
}
} // namespace
@@ -676,4 +842,14 @@
return std::vector<uint64_t>();
}
+void SetFixedLengthEncoderDeltaSignednessForTesting(bool signedness) {
+ g_force_unsigned_for_testing = !signedness;
+ g_force_signed_for_testing = signedness;
+}
+
+void UnsetFixedLengthEncoderDeltaSignednessForTesting() {
+ g_force_unsigned_for_testing = false;
+ g_force_signed_for_testing = false;
+}
+
} // namespace webrtc
diff --git a/logging/rtc_event_log/encoder/delta_encoding_unittest.cc b/logging/rtc_event_log/encoder/delta_encoding_unittest.cc
index c318674..d553517 100644
--- a/logging/rtc_event_log/encoder/delta_encoding_unittest.cc
+++ b/logging/rtc_event_log/encoder/delta_encoding_unittest.cc
@@ -16,14 +16,36 @@
#include <tuple>
#include <vector>
+#include "absl/types/optional.h"
#include "rtc_base/arraysize.h"
#include "rtc_base/checks.h"
#include "rtc_base/random.h"
#include "test/gtest.h"
namespace webrtc {
+
+void SetFixedLengthEncoderDeltaSignednessForTesting(bool signedness);
+void UnsetFixedLengthEncoderDeltaSignednessForTesting();
+
namespace {
+enum class DeltaSignedness { kNoOverride, kForceUnsigned, kForceSigned };
+
+void MaybeSetSignedness(DeltaSignedness signedness) {
+ switch (signedness) {
+ case DeltaSignedness::kNoOverride:
+ UnsetFixedLengthEncoderDeltaSignednessForTesting();
+ return;
+ case DeltaSignedness::kForceUnsigned:
+ SetFixedLengthEncoderDeltaSignednessForTesting(false);
+ return;
+ case DeltaSignedness::kForceSigned:
+ SetFixedLengthEncoderDeltaSignednessForTesting(true);
+ return;
+ }
+ RTC_NOTREACHED();
+}
+
uint64_t RandomWithMaxBitWidth(Random* prng, uint64_t max_width) {
RTC_DCHECK_GE(max_width, 1u);
RTC_DCHECK_LE(max_width, 64u);
@@ -95,21 +117,42 @@
}
size_t EncodingLengthUpperBound(size_t delta_max_bit_width,
- size_t num_of_deltas) {
- constexpr size_t kSmallestHeaderSizeBytes = 1;
- return delta_max_bit_width * num_of_deltas + kSmallestHeaderSizeBytes;
+ size_t num_of_deltas,
+ DeltaSignedness signedness_override) {
+ absl::optional<size_t> smallest_header_size_bytes;
+ switch (signedness_override) {
+ case DeltaSignedness::kNoOverride:
+ case DeltaSignedness::kForceUnsigned:
+ smallest_header_size_bytes = 1;
+ break;
+ case DeltaSignedness::kForceSigned:
+ smallest_header_size_bytes = 2;
+ break;
+ }
+ RTC_DCHECK(smallest_header_size_bytes);
+
+ return delta_max_bit_width * num_of_deltas + *smallest_header_size_bytes;
}
// Tests of the delta encoding, parameterized by the number of values
// in the sequence created by the test.
-class DeltaEncodingTest : public ::testing::TestWithParam<size_t> {
+class DeltaEncodingTest
+ : public ::testing::TestWithParam<std::tuple<DeltaSignedness, size_t>> {
public:
+ DeltaEncodingTest()
+ : signedness_(std::get<0>(GetParam())),
+ num_of_values_(std::get<1>(GetParam())) {
+ MaybeSetSignedness(signedness_);
+ }
~DeltaEncodingTest() override = default;
+
+ const DeltaSignedness signedness_;
+ const uint64_t num_of_values_;
};
TEST_P(DeltaEncodingTest, AllValuesEqualToBaseValue) {
const uint64_t base = 3432;
- std::vector<uint64_t> values(GetParam());
+ std::vector<uint64_t> values(num_of_values_);
std::fill(values.begin(), values.end(), base);
std::string encoded;
TestEncodingAndDecoding(base, values, &encoded);
@@ -122,7 +165,7 @@
TEST_P(DeltaEncodingTest, MinDeltaNoWrapAround) {
const uint64_t base = 3432;
- const auto values = CreateSequenceByFirstValue(base + 1, GetParam());
+ const auto values = CreateSequenceByFirstValue(base + 1, num_of_values_);
ASSERT_GT(values[values.size() - 1], base) << "Sanity; must not wrap around";
TestEncodingAndDecoding(base, values);
@@ -132,7 +175,8 @@
const uint64_t kBigDelta = 132828;
const uint64_t base = 3432;
- const auto values = CreateSequenceByFirstValue(base + kBigDelta, GetParam());
+ const auto values =
+ CreateSequenceByFirstValue(base + kBigDelta, num_of_values_);
ASSERT_GT(values[values.size() - 1], base) << "Sanity; must not wrap around";
TestEncodingAndDecoding(base, values);
@@ -142,7 +186,7 @@
const uint64_t base = 3432;
const auto values = CreateSequenceByLastValue(
- std::numeric_limits<uint64_t>::max(), GetParam());
+ std::numeric_limits<uint64_t>::max(), num_of_values_);
ASSERT_GT(values[values.size() - 1], base) << "Sanity; must not wrap around";
TestEncodingAndDecoding(base, values);
@@ -151,20 +195,20 @@
TEST_P(DeltaEncodingTest, SmallDeltaWithWrapAroundComparedToBase) {
const uint64_t base = std::numeric_limits<uint64_t>::max();
- const auto values = CreateSequenceByDeltas(base, {1, 10, 3}, GetParam());
+ const auto values = CreateSequenceByDeltas(base, {1, 10, 3}, num_of_values_);
ASSERT_LT(values[values.size() - 1], base) << "Sanity; must wrap around";
TestEncodingAndDecoding(base, values);
}
TEST_P(DeltaEncodingTest, SmallDeltaWithWrapAroundInValueSequence) {
- if (GetParam() == 1) {
+ if (num_of_values_ == 1) {
return; // Inapplicable.
}
const uint64_t base = std::numeric_limits<uint64_t>::max() - 2;
- const auto values = CreateSequenceByDeltas(base, {1, 10, 3}, GetParam());
+ const auto values = CreateSequenceByDeltas(base, {1, 10, 3}, num_of_values_);
ASSERT_LT(values[values.size() - 1], values[0]) << "Sanity; must wrap around";
TestEncodingAndDecoding(base, values);
@@ -179,14 +223,15 @@
const uint64_t kBigDelta = 132828;
const uint64_t base = std::numeric_limits<uint64_t>::max() - kBigDelta + 3;
- const auto values = CreateSequenceByFirstValue(base + kBigDelta, GetParam());
+ const auto values =
+ CreateSequenceByFirstValue(base + kBigDelta, num_of_values_);
ASSERT_LT(values[values.size() - 1], base) << "Sanity; must wrap around";
TestEncodingAndDecoding(base, values);
}
TEST_P(DeltaEncodingTest, BigDeltaWithWrapAroundInValueSequence) {
- if (GetParam() == 1) {
+ if (num_of_values_ == 1) {
return; // Inapplicable.
}
@@ -194,7 +239,7 @@
const uint64_t base = std::numeric_limits<uint64_t>::max() - kBigDelta + 3;
const auto values = CreateSequenceByFirstValue(
- std::numeric_limits<uint64_t>::max(), GetParam());
+ std::numeric_limits<uint64_t>::max(), num_of_values_);
ASSERT_LT(values[values.size() - 1], base) << "Sanity; must wrap around";
TestEncodingAndDecoding(base, values);
@@ -205,27 +250,27 @@
TEST_P(DeltaEncodingTest, MaxDeltaWithWrapAroundComparedToBase) {
const uint64_t base = 3432;
- const auto values = CreateSequenceByFirstValue(base - 1, GetParam());
+ const auto values = CreateSequenceByFirstValue(base - 1, num_of_values_);
TestEncodingAndDecoding(base, values);
}
TEST_P(DeltaEncodingTest, MaxDeltaWithWrapAroundInValueSequence) {
- if (GetParam() == 1) {
+ if (num_of_values_ == 1) {
return; // Inapplicable.
}
const uint64_t base = 3432;
const auto values = CreateSequenceByDeltas(
- base, {0, std::numeric_limits<uint64_t>::max(), 3}, GetParam());
+ base, {0, std::numeric_limits<uint64_t>::max(), 3}, num_of_values_);
ASSERT_LT(values[1], base) << "Sanity; must wrap around";
TestEncodingAndDecoding(base, values);
}
-// If GetParam() == 1, a zero delta will yield an empty string; that's already
-// covered by AllValuesEqualToBaseValue, but it doesn't hurt to test again.
-// For all other cases, we have a new test.
+// If num_of_values_ == 1, a zero delta will yield an empty string; that's
+// already covered by AllValuesEqualToBaseValue, but it doesn't hurt to test
+// again. For all other cases, we have a new test.
TEST_P(DeltaEncodingTest, ZeroDelta) {
const uint64_t base = 3432;
@@ -233,26 +278,46 @@
// consecutive zeros.
const std::vector<uint64_t> deltas = {0, 312, 11, 1, 1, 0, 0, 12,
400321, 3, 3, 12, 5, 0, 6};
- const auto values = CreateSequenceByDeltas(base, deltas, GetParam());
+ const auto values = CreateSequenceByDeltas(base, deltas, num_of_values_);
TestEncodingAndDecoding(base, values);
}
-INSTANTIATE_TEST_CASE_P(NumberOfValuesInSequence,
- DeltaEncodingTest,
- ::testing::Values(1, 2, 100, 10000));
+INSTANTIATE_TEST_CASE_P(
+ SignednessOverrideAndNumberOfValuesInSequence,
+ DeltaEncodingTest,
+ ::testing::Combine(::testing::Values(DeltaSignedness::kNoOverride,
+ DeltaSignedness::kForceUnsigned,
+ DeltaSignedness::kForceSigned),
+ ::testing::Values(1, 2, 100, 10000)));
// Tests over the quality of the compression (as opposed to its correctness).
// Not to be confused with tests of runtime efficiency.
class DeltaEncodingCompressionQualityTest
- : public ::testing::TestWithParam<std::tuple<uint64_t, uint64_t>> {
+ : public ::testing::TestWithParam<
+ std::tuple<DeltaSignedness, uint64_t, uint64_t>> {
public:
DeltaEncodingCompressionQualityTest()
- : delta_max_bit_width_(std::get<0>(GetParam())),
- num_of_values_(std::get<1>(GetParam())) {}
+ : signedness_(std::get<0>(GetParam())),
+ delta_max_bit_width_(std::get<1>(GetParam())),
+ num_of_values_(std::get<2>(GetParam())) {
+ MaybeSetSignedness(signedness_);
+ }
~DeltaEncodingCompressionQualityTest() override = default;
+ // Running with the same seed for all variants would make all tests start
+ // with the same sequence; avoid this by making the seed different.
+ uint64_t Seed() const {
+ constexpr uint64_t non_zero_base_seed = 3012;
+ // Multiply everything but |non_zero_base_seed| by different prime numbers
+ // to produce unique results.
+ return non_zero_base_seed + 2 * static_cast<uint64_t>(signedness_) +
+ 3 * delta_max_bit_width_ + 5 * delta_max_bit_width_ +
+ 7 * num_of_values_;
+ }
+
+ const DeltaSignedness signedness_;
const uint64_t delta_max_bit_width_;
const uint64_t num_of_values_;
};
@@ -261,12 +326,6 @@
// matter to compression performance; only the deltas matter.
TEST_P(DeltaEncodingCompressionQualityTest,
BaseDoesNotAffectEfficiencyIfNoWrapAround) {
- Random prng(3012);
- std::vector<uint64_t> deltas(num_of_values_);
- for (size_t i = 0; i < deltas.size(); ++i) {
- deltas[i] = RandomWithMaxBitWidth(&prng, delta_max_bit_width_);
- }
-
// 1. Bases which will not produce a wrap-around.
// 2. The last base - 0xffffffffffffffff - does cause a wrap-around, but
// that still works, because the width is 64 anyway, and does not
@@ -274,6 +333,46 @@
const uint64_t bases[] = {0, 0x55, 0xffffffff,
std::numeric_limits<uint64_t>::max()};
+ const size_t kIntendedWrapAroundBaseIndex = arraysize(bases);
+
+ std::vector<uint64_t> deltas(num_of_values_);
+
+ // Allows us to make sure that the deltas do not produce a wrap-around.
+ uint64_t last_element[arraysize(bases)];
+ memcpy(last_element, bases, sizeof(bases));
+
+ // Avoid empty |deltas| due to first element causing wrap-around.
+ deltas[0] = 1;
+ for (size_t i = 0; i < arraysize(last_element); ++i) {
+ last_element[i] += 1;
+ }
+
+ Random prng(Seed());
+
+ for (size_t i = 1; i < deltas.size(); ++i) {
+ const uint64_t delta = RandomWithMaxBitWidth(&prng, delta_max_bit_width_);
+
+ bool wrap_around = false;
+ for (size_t j = 0; j < arraysize(last_element); ++j) {
+ if (j == kIntendedWrapAroundBaseIndex) {
+ continue;
+ }
+
+ last_element[j] += delta;
+ if (last_element[j] < bases[j]) {
+ wrap_around = true;
+ break;
+ }
+ }
+
+ if (wrap_around) {
+ deltas.resize(i);
+ break;
+ }
+
+ deltas[i] = delta;
+ }
+
std::string encodings[arraysize(bases)];
for (size_t i = 0; i < arraysize(bases); ++i) {
@@ -284,7 +383,8 @@
// the encoding/decoding, though that is not the test's focus.
TestEncodingAndDecoding(bases[i], values, &encodings[i]);
EXPECT_LE(encodings[i].length(),
- EncodingLengthUpperBound(delta_max_bit_width_, num_of_values_));
+ EncodingLengthUpperBound(delta_max_bit_width_, num_of_values_,
+ signedness_));
}
// Test focus - all of the encodings should be the same, as they are based
@@ -295,23 +395,42 @@
}
INSTANTIATE_TEST_CASE_P(
- DeltaMaxBitWidthAndNumberOfValuesInSequence,
+ SignednessOverrideAndDeltaMaxBitWidthAndNumberOfValuesInSequence,
DeltaEncodingCompressionQualityTest,
::testing::Combine(
+ ::testing::Values(DeltaSignedness::kNoOverride,
+ DeltaSignedness::kForceUnsigned,
+ DeltaSignedness::kForceSigned),
::testing::Values(1, 4, 8, 15, 16, 17, 31, 32, 33, 63, 64),
::testing::Values(1, 2, 100, 10000)));
// Similar to DeltaEncodingTest, but instead of semi-surgically producing
// specific cases, produce large amount of semi-realistic inputs.
class DeltaEncodingFuzzerLikeTest
- : public ::testing::TestWithParam<std::tuple<uint64_t, uint64_t>> {
+ : public ::testing::TestWithParam<
+ std::tuple<DeltaSignedness, uint64_t, uint64_t>> {
public:
DeltaEncodingFuzzerLikeTest()
- : delta_max_bit_width_(std::get<0>(GetParam())),
- num_of_values_(std::get<1>(GetParam())) {}
+ : signedness_(std::get<0>(GetParam())),
+ delta_max_bit_width_(std::get<1>(GetParam())),
+ num_of_values_(std::get<2>(GetParam())) {
+ MaybeSetSignedness(signedness_);
+ }
~DeltaEncodingFuzzerLikeTest() override = default;
+ // Running with the same seed for all variants would make all tests start
+ // with the same sequence; avoid this by making the seed different.
+ uint64_t Seed() const {
+ constexpr uint64_t non_zero_base_seed = 1983;
+ // Multiply everything but |non_zero_base_seed| by different prime numbers
+ // to produce unique results.
+ return non_zero_base_seed + 2 * static_cast<uint64_t>(signedness_) +
+ 3 * delta_max_bit_width_ + 5 * delta_max_bit_width_ +
+ 7 * num_of_values_;
+ }
+
+ const DeltaSignedness signedness_;
const uint64_t delta_max_bit_width_;
const uint64_t num_of_values_;
};
@@ -319,7 +438,7 @@
TEST_P(DeltaEncodingFuzzerLikeTest, Test) {
const uint64_t base = 3432;
- Random prng(1983);
+ Random prng(Seed());
std::vector<uint64_t> deltas(num_of_values_);
for (size_t i = 0; i < deltas.size(); ++i) {
deltas[i] = RandomWithMaxBitWidth(&prng, delta_max_bit_width_);
@@ -331,11 +450,75 @@
}
INSTANTIATE_TEST_CASE_P(
- DeltaMaxBitWidthAndNumberOfValuesInSequence,
+ SignednessOverrideAndDeltaMaxBitWidthAndNumberOfValuesInSequence,
DeltaEncodingFuzzerLikeTest,
::testing::Combine(
+ ::testing::Values(DeltaSignedness::kNoOverride,
+ DeltaSignedness::kForceUnsigned,
+ DeltaSignedness::kForceSigned),
::testing::Values(1, 4, 8, 15, 16, 17, 31, 32, 33, 63, 64),
::testing::Values(1, 2, 100, 10000)));
+class DeltaEncodingSpecificEdgeCasesTest
+ : public ::testing::TestWithParam<
+ std::tuple<DeltaSignedness, uint64_t, bool>> {
+ public:
+ DeltaEncodingSpecificEdgeCasesTest() {
+ UnsetFixedLengthEncoderDeltaSignednessForTesting();
+ }
+
+ ~DeltaEncodingSpecificEdgeCasesTest() override = default;
+};
+
+// This case is special because it produces identical forward/backward deltas.
+TEST_F(DeltaEncodingSpecificEdgeCasesTest, SignedDeltaWithOnlyTopBitOn) {
+ MaybeSetSignedness(DeltaSignedness::kForceSigned);
+
+ const uint64_t base = 3432;
+
+ const uint64_t delta = static_cast<uint64_t>(1) << 63;
+ const std::vector<uint64_t> values = {base + delta};
+
+ TestEncodingAndDecoding(base, values);
+}
+
+TEST_F(DeltaEncodingSpecificEdgeCasesTest, MaximumUnsignedDelta) {
+ MaybeSetSignedness(DeltaSignedness::kForceUnsigned);
+
+ const uint64_t base = (static_cast<uint64_t>(1) << 63) + 0x123;
+
+ const std::vector<uint64_t> values = {base - 1};
+
+ TestEncodingAndDecoding(base, values);
+}
+
+// Check that, if all deltas are set to -1, things still work.
+TEST_P(DeltaEncodingSpecificEdgeCasesTest, ReverseSequence) {
+ MaybeSetSignedness(std::get<0>(GetParam()));
+ const uint64_t width = std::get<1>(GetParam());
+ const bool wrap_around = std::get<2>(GetParam());
+
+ const uint64_t value_mask = (width == 64)
+ ? std::numeric_limits<uint64_t>::max()
+ : ((static_cast<uint64_t>(1) << width) - 1);
+
+ const uint64_t base = wrap_around ? 1u : (0xf82d3 & value_mask);
+ const std::vector<uint64_t> values = {(base - 1u) & value_mask,
+ (base - 2u) & value_mask,
+ (base - 3u) & value_mask};
+
+ TestEncodingAndDecoding(base, values);
+}
+
+INSTANTIATE_TEST_CASE_P(
+ _,
+ DeltaEncodingSpecificEdgeCasesTest,
+ ::testing::Combine(
+ ::testing::Values(DeltaSignedness::kNoOverride,
+ DeltaSignedness::kForceUnsigned,
+ DeltaSignedness::kForceSigned),
+ ::testing::Values(1, 4, 8, 15, 16, 17, 31, 32, 33, 63, 64),
+ ::testing::Bool()));
+
} // namespace
} // namespace webrtc
