From d164dafc07ba4b9f57f139da9500fffa4de1fcb1 Mon Sep 17 00:00:00 2001 From: The Fern Date: Sun, 5 Feb 2023 21:21:41 +0100 Subject: [PATCH 1/3] Add variable length encoding to array_traits Add serialize_traits specialization for bool Add unit tests for bool trait Fix some issues with integral_traits --- include/bitstream/traits/array_traits.h | 117 ++++++++++++++++----- include/bitstream/traits/bool_trait.h | 34 ++++++ include/bitstream/traits/integral_traits.h | 80 +++++++------- test/src/test/serialize_array_traits.cpp | 17 +-- test/src/test/serialize_bool_test.cpp | 62 +++++++++++ 5 files changed, 241 insertions(+), 69 deletions(-) create mode 100644 include/bitstream/traits/bool_trait.h create mode 100644 test/src/test/serialize_bool_test.cpp diff --git a/include/bitstream/traits/array_traits.h b/include/bitstream/traits/array_traits.h index fec8dfe..eb06ba7 100644 --- a/include/bitstream/traits/array_traits.h +++ b/include/bitstream/traits/array_traits.h @@ -6,6 +6,9 @@ #include "../stream/bit_reader.h" #include "../stream/bit_writer.h" +#include "../traits/bool_trait.h" +#include "../traits/integral_traits.h" + #include namespace bitstream @@ -16,49 +19,113 @@ namespace bitstream template struct serialize_traits> { + private: + template + static bool serialize_difference(Stream& stream, int& previous, int& current, uint32_t& difference) + { + bool use_bits; + if (Stream::writing) + use_bits = difference <= Max; + BS_ASSERT(stream.serialize(use_bits)); + if (use_bits) + { + using bounded_trait = bounded_int; + + BS_ASSERT(stream.serialize(difference)); + if (Stream::reading) + current = previous + difference; + previous = current; + return true; + } + + return false; + } + + template + static bool serialize_index(Stream& stream, int& previous, int& current, int max_size) + { + uint32_t difference; + if (Stream::writing) + { + BS_ASSERT(previous < current); + difference = current - previous; + BS_ASSERT(difference > 0); + } + + // +1 (1 bit) + bool plus_one; + if (Stream::writing) + plus_one = difference == 1; + BS_ASSERT(stream.serialize(plus_one)); + if (plus_one) + { + if (Stream::reading) + current = previous + 1; + previous = current; + return true; + } + + // [+2,5] -> [0,3] (2 bits) + if (serialize_difference<2, 5>(stream, previous, current, difference)) + return true; + + // [6,13] -> [0,7] (3 bits) + if (serialize_difference<6, 13>(stream, previous, current, difference)) + return true; + + // [14,29] -> [0,15] (4 bits) + if (serialize_difference<14, 29>(stream, previous, current, difference)) + return true; + + // [30,61] -> [0,31] (5 bits) + if (serialize_difference<30, 61>(stream, previous, current, difference)) + return true; + + // [62,125] -> [0,63] (6 bits) + if (serialize_difference<62, 125>(stream, previous, current, difference)) + return true; + + // [126,MaxObjects+1] + BS_ASSERT(stream.serialize(difference, 126, max_size)); + if (Stream::reading) + current = previous + difference; + previous = current; + return true; + } + + public: template - static bool serialize(bit_writer& writer, T* values, uint32_t max_size, Compare compare, Args&&... args) noexcept + static bool serialize(bit_writer& writer, T* values, int max_size, Compare compare, Args&&... args) noexcept { - uint32_t prev_index = 0; - for (uint32_t i = 0; i < max_size; i++) + int prev_index = -1; + for (int index = 0; index < max_size; index++) { - if (!compare(values[i])) + if (!compare(values[index])) continue; - uint32_t num_bits = utility::bits_in_range(prev_index, max_size); - - uint32_t index = i - prev_index; - BS_ASSERT(writer.serialize_bits(index, num_bits)); - - BS_ASSERT(writer.serialize(values[i], std::forward(args)...)); + BS_ASSERT(serialize_index(writer, prev_index, index, max_size)); - prev_index = i; + BS_ASSERT(writer.serialize(values[index], std::forward(args)...)); } - uint32_t num_bits = utility::bits_in_range(prev_index, max_size); - - BS_ASSERT(writer.serialize_bits(max_size - prev_index, num_bits)); + BS_ASSERT(serialize_index(writer, prev_index, max_size, max_size)); return true; } template - static bool serialize(bit_reader& reader, T* values, uint32_t max_size, Compare compare, Args&&... args) noexcept + static bool serialize(bit_reader& reader, T* values, int max_size, Compare compare, Args&&... args) noexcept { - uint32_t prev_index = 0; - for (uint32_t i = 0; i <= max_size; i++) + int prev_index = -1; + int index = 0; + while (true) { - uint32_t num_bits = utility::bits_in_range(prev_index, max_size); - - uint32_t index; - BS_ASSERT(reader.serialize_bits(index, num_bits)); + BS_ASSERT(serialize_index(reader, prev_index, index, max_size)); - if (index + prev_index == max_size) + if (index == max_size) break; - BS_ASSERT(reader.serialize(values[index + prev_index], std::forward(args)...)); - - prev_index += index; + BS_ASSERT(reader.serialize(values[index], std::forward(args)...)); } return true; diff --git a/include/bitstream/traits/bool_trait.h b/include/bitstream/traits/bool_trait.h new file mode 100644 index 0000000..6375337 --- /dev/null +++ b/include/bitstream/traits/bool_trait.h @@ -0,0 +1,34 @@ +#pragma once +#include "../utility/assert.h" + +#include "../stream/serialize_traits.h" +#include "../stream/bit_reader.h" +#include "../stream/bit_writer.h" + +namespace bitstream +{ + /** + * @brief A trait used to serialize a boolean as a single bit + */ + template<> + struct serialize_traits + { + static bool serialize(bit_writer& writer, bool value) noexcept + { + uint32_t unsigned_value = value; + + return writer.serialize_bits(unsigned_value, 1U); + } + + static bool serialize(bit_reader& reader, bool& value) noexcept + { + uint32_t unsigned_value; + + BS_ASSERT(reader.serialize_bits(unsigned_value, 1U)); + + value = unsigned_value; + + return true; + } + }; +} \ No newline at end of file diff --git a/include/bitstream/traits/integral_traits.h b/include/bitstream/traits/integral_traits.h index 2e0d366..7c4ef8a 100644 --- a/include/bitstream/traits/integral_traits.h +++ b/include/bitstream/traits/integral_traits.h @@ -27,32 +27,35 @@ namespace bitstream template struct serialize_traits>> { - static bool serialize(bit_writer& writer, const T& value, T min = (std::numeric_limits::min)(), T max = (std::numeric_limits::max)()) noexcept + static bool serialize(bit_writer& writer, T value, T min = (std::numeric_limits::min)(), T max = (std::numeric_limits::max)()) noexcept { BS_ASSERT(min < max); - BS_ASSERT(value >= min && value < max); + BS_ASSERT(value >= min && value <= max); - int num_bits = static_cast(utility::bits_in_range(min, max)); + uint32_t num_bits = utility::bits_in_range(min, max); BS_ASSERT(num_bits <= sizeof(T) * 8); if constexpr (sizeof(T) > 4) { - // If the given range is bigger than a word (32 bits) - uint32_t unsigned_value = static_cast(value - min); - BS_ASSERT(writer.serialize_bits(unsigned_value, 32)); + if (num_bits > 32) + { + // If the given range is bigger than a word (32 bits) + uint32_t unsigned_value = static_cast(value - min); + BS_ASSERT(writer.serialize_bits(unsigned_value, 32)); - unsigned_value = static_cast((value - min) >> 32); - BS_ASSERT(writer.serialize_bits(unsigned_value, num_bits - 32)); - } - else - { - // If the given range is smaller than or equal to a word (32 bits) - uint32_t unsigned_value = static_cast(value - min); - BS_ASSERT(writer.serialize_bits(unsigned_value, num_bits)); + unsigned_value = static_cast((value - min) >> 32); + BS_ASSERT(writer.serialize_bits(unsigned_value, num_bits - 32)); + + return true; + } } + // If the given range is smaller than or equal to a word (32 bits) + uint32_t unsigned_value = static_cast(value - min); + BS_ASSERT(writer.serialize_bits(unsigned_value, num_bits)); + return true; } @@ -60,34 +63,39 @@ namespace bitstream { BS_ASSERT(min < max); - int num_bits = static_cast(utility::bits_in_range(min, max)); + uint32_t num_bits = utility::bits_in_range(min, max); BS_ASSERT(num_bits <= sizeof(T) * 8); if constexpr (sizeof(T) > 4) { - // If the given range is bigger than a word (32 bits) - value = 0; - uint32_t unsigned_value; + if (num_bits > 32) + { + // If the given range is bigger than a word (32 bits) + value = 0; + uint32_t unsigned_value; - BS_ASSERT(reader.serialize_bits(unsigned_value, 32)); - value |= static_cast(unsigned_value); + BS_ASSERT(reader.serialize_bits(unsigned_value, 32)); + value |= static_cast(unsigned_value); - BS_ASSERT(reader.serialize_bits(unsigned_value, num_bits - 32)); - value |= static_cast(unsigned_value) << 32; + BS_ASSERT(reader.serialize_bits(unsigned_value, num_bits - 32)); + value |= static_cast(unsigned_value) << 32; - value += min; - } - else - { - // If the given range is smaller than or equal to a word (32 bits) - uint32_t unsigned_value; - BS_ASSERT(reader.serialize_bits(unsigned_value, num_bits)); + value += min; - value = static_cast(unsigned_value) + min; + BS_ASSERT(value >= min && value <= max); + + return true; + } } + + // If the given range is smaller than or equal to a word (32 bits) + uint32_t unsigned_value; + BS_ASSERT(reader.serialize_bits(unsigned_value, num_bits)); + + value = static_cast(unsigned_value) + min; - BS_ASSERT(value >= min && value < max); + BS_ASSERT(value >= min && value <= max); return true; } @@ -104,13 +112,13 @@ namespace bitstream template struct serialize_traits, typename std::enable_if_t>> { - static bool serialize(bit_writer& writer, const T& value) noexcept + static bool serialize(bit_writer& writer, T value) noexcept { static_assert(Min < Max); - BS_ASSERT(value >= Min && value < Max); + BS_ASSERT(value >= Min && value <= Max); - constexpr int num_bits = static_cast(utility::bits_in_range(Min, Max)); + constexpr uint32_t num_bits = utility::bits_in_range(Min, Max); static_assert(num_bits <= sizeof(T) * 8); @@ -137,7 +145,7 @@ namespace bitstream { static_assert(Min < Max); - constexpr int num_bits = static_cast(utility::bits_in_range(Min, Max)); + constexpr uint32_t num_bits = utility::bits_in_range(Min, Max); static_assert(num_bits <= sizeof(T) * 8); @@ -164,7 +172,7 @@ namespace bitstream value = static_cast(unsigned_value) + Min; } - BS_ASSERT(value >= Min && value < Max); + BS_ASSERT(value >= Min && value <= Max); return true; } diff --git a/test/src/test/serialize_array_traits.cpp b/test/src/test/serialize_array_traits.cpp index 53d3312..0bffab6 100644 --- a/test/src/test/serialize_array_traits.cpp +++ b/test/src/test/serialize_array_traits.cpp @@ -13,33 +13,34 @@ namespace bitstream::test::traits { using trait = array_subset>; - // Test half precision float - uint32_t values_in[5] + // Test array subset + uint32_t values_in[6] { 10, 21, 42, 99, + 420, 1337 }; + auto compare = [](uint32_t value) { return value != 21 && value != 42 && value != 99; }; + uint8_t buffer[16]{ 0 }; bit_writer writer(buffer, 16); - auto compare = [](uint32_t value) { return value != 21 && value != 99; }; - - BS_TEST_ASSERT(writer.serialize(values_in, 5U, compare)); // Use bounded_int for writing + BS_TEST_ASSERT(writer.serialize(values_in, 6, compare)); // Use bounded_int for writing uint32_t num_bytes = writer.flush(); BS_TEST_ASSERT_OPERATION(num_bytes, == , 6); - uint32_t values_out[5]; + uint32_t values_out[6]; bit_reader reader(std::move(writer)); - BS_TEST_ASSERT(reader.serialize>(values_out, 5U, compare, 0U, 2048U)); // Use min, max arguments for reading + BS_TEST_ASSERT(reader.serialize>(values_out, 6, compare, 0U, 2048U)); // Use min, max arguments for reading - for (int i = 0; i < 5; i++) + for (int i = 0; i < 6; i++) { if (!compare(values_in[i])) continue; diff --git a/test/src/test/serialize_bool_test.cpp b/test/src/test/serialize_bool_test.cpp new file mode 100644 index 0000000..d5ee3ab --- /dev/null +++ b/test/src/test/serialize_bool_test.cpp @@ -0,0 +1,62 @@ +#include "test_assert.h" +#include "test.h" + +#include +#include + +#include + +namespace bitstream::test::traits +{ + BS_ADD_TEST(test_serialize_bool_aligned) + { + // Test bools + bool value = true; + + // Write a char array, but make sure the word count isn't whole + uint8_t buffer[4]; + bit_writer writer(buffer, 4); + + BS_TEST_ASSERT(writer.serialize(value)); + uint32_t num_bytes = writer.flush(); + + BS_TEST_ASSERT_OPERATION(num_bytes, == , 1); + + // Read the bool back and validate + bool out_value; + bit_reader reader(std::move(writer)); + + BS_TEST_ASSERT(reader.serialize(out_value)); + + BS_TEST_ASSERT_OPERATION(out_value, == , value); + } + + BS_ADD_TEST(test_serialize_bool_misaligned) + { + // Test bools + uint32_t padding = 17; + bool value = true; + + // Write a char array, but make sure the word count isn't whole + uint8_t buffer[4]; + bit_writer writer(buffer, 4); + + BS_TEST_ASSERT(writer.serialize_bits(padding, 5U)); + BS_TEST_ASSERT(writer.serialize(value)); + uint32_t num_bytes = writer.flush(); + + BS_TEST_ASSERT_OPERATION(num_bytes, == , 1); + + // Read the bool back and validate + uint32_t out_padding; + bool out_value; + bit_reader reader(std::move(writer)); + + BS_TEST_ASSERT(reader.serialize_bits(out_padding, 5U)); + BS_TEST_ASSERT(reader.serialize(out_value)); + + BS_TEST_ASSERT_OPERATION(out_padding, == , padding); + BS_TEST_ASSERT_OPERATION(out_value, == , value); + //BS_TEST_ASSERT_OPERATION(*reinterpret_cast(&out_value), == , 1U); + } +} \ No newline at end of file From 9032fdb4da4b0dfbc5c782c93e9206660711db36 Mon Sep 17 00:00:00 2001 From: The Fern Date: Mon, 6 Feb 2023 22:27:50 +0100 Subject: [PATCH 2/3] Attempt to fix issue in array_traits on linux --- include/bitstream/traits/array_traits.h | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/include/bitstream/traits/array_traits.h b/include/bitstream/traits/array_traits.h index eb06ba7..612991f 100644 --- a/include/bitstream/traits/array_traits.h +++ b/include/bitstream/traits/array_traits.h @@ -1,6 +1,5 @@ #pragma once #include "../utility/assert.h" -#include "../utility/bits.h" #include "../stream/serialize_traits.h" #include "../stream/bit_reader.h" @@ -26,12 +25,14 @@ namespace bitstream bool use_bits; if (Stream::writing) use_bits = difference <= Max; - BS_ASSERT(stream.serialize(use_bits)); + if (!stream.serialize(use_bits)) + return false; if (use_bits) { using bounded_trait = bounded_int; - BS_ASSERT(stream.serialize(difference)); + if (!stream.serialize(difference)) + return false; if (Stream::reading) current = previous + difference; previous = current; @@ -56,7 +57,8 @@ namespace bitstream bool plus_one; if (Stream::writing) plus_one = difference == 1; - BS_ASSERT(stream.serialize(plus_one)); + if (!stream.serialize(plus_one)) + return false; if (plus_one) { if (Stream::reading) @@ -86,7 +88,8 @@ namespace bitstream return true; // [126,MaxObjects+1] - BS_ASSERT(stream.serialize(difference, 126, max_size)); + if (!stream.serialize(difference, 126, max_size)) + return false; if (Stream::reading) current = previous + difference; previous = current; From 3e428db5b70e7eec2eeb7d6339bb138833a606ac Mon Sep 17 00:00:00 2001 From: The Fern Date: Mon, 6 Feb 2023 22:31:28 +0100 Subject: [PATCH 3/3] Add template expression to array_traits fix --- include/bitstream/traits/array_traits.h | 12 ++++-------- test/src/test/serialize_array_traits.cpp | 1 + 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/include/bitstream/traits/array_traits.h b/include/bitstream/traits/array_traits.h index 612991f..578214e 100644 --- a/include/bitstream/traits/array_traits.h +++ b/include/bitstream/traits/array_traits.h @@ -25,14 +25,12 @@ namespace bitstream bool use_bits; if (Stream::writing) use_bits = difference <= Max; - if (!stream.serialize(use_bits)) - return false; + BS_ASSERT(stream.template serialize(use_bits)); if (use_bits) { using bounded_trait = bounded_int; - if (!stream.serialize(difference)) - return false; + BS_ASSERT(stream.template serialize(difference)); if (Stream::reading) current = previous + difference; previous = current; @@ -57,8 +55,7 @@ namespace bitstream bool plus_one; if (Stream::writing) plus_one = difference == 1; - if (!stream.serialize(plus_one)) - return false; + BS_ASSERT(stream.template serialize(plus_one)); if (plus_one) { if (Stream::reading) @@ -88,8 +85,7 @@ namespace bitstream return true; // [126,MaxObjects+1] - if (!stream.serialize(difference, 126, max_size)) - return false; + BS_ASSERT(stream.template serialize(difference, 126, max_size)); if (Stream::reading) current = previous + difference; previous = current; diff --git a/test/src/test/serialize_array_traits.cpp b/test/src/test/serialize_array_traits.cpp index 0bffab6..4a922fd 100644 --- a/test/src/test/serialize_array_traits.cpp +++ b/test/src/test/serialize_array_traits.cpp @@ -5,6 +5,7 @@ #include #include +#include #include namespace bitstream::test::traits