catarrh/cdc-file-transfer
1
0
Fork 0
mirror of https://github.com/google/cdc-file-transfer.git synced 2026-09-13 01:10:44 +03:00
Code Issues Packages Projects Releases Wiki Activity

Change fastcdc to a better and simpler algorithm. (#79)

Browse Source 
This CL changes the chunking algorithm from "normalized chunking" to
simple "regression chunking", and changes the has criteria from
'hash&mask' to 'hash<=threshold'. These are all ideas taken from
testing and analysis done at
  https://github.com/dbaarda/rollsum-chunking/blob/master/RESULTS.rst
Regression chunking was introduced in
  https://www.usenix.org/system/files/conference/atc12/atc12-final293.pdf

The algorithm uses an arbitrary number of regressions using power-of-2
regression target lengths. This means we can use a simple bitmask for
the regression hash criteria.

Regression chunking yields high deduplication rates even for lower max
chunk sizes, so that the cdc_stream max chunk can be reduced to 512K
from 1024K. This fixes potential latency spikes from large chunks.
This commit is contained in:
Donovan Baarda
2023-02-09 01:06:41 +11:00
committed by GitHub
parent 24906eb36e
commit fcc4cbc3f3
10 changed files with 121 additions and 331 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fastcdc/fastcdc_test.cc
+8 -142
View File
@@ -25,160 +25,26 @@ namespace fastcdc {
// - data like {1, 1, 1, ...} results in a continuously all-ones rolling hash,
// thus is never identified as a chunk boundary.
static const uint64_t testgear64[256]{0, 1}; // 0, 1, 0, 0, 0, ...
static constexpr uint32_t test_mask_stages = 5;
static constexpr uint32_t test_mask_lshift = 1;
template <uint32_t mask_stages = test_mask_stages,
uint32_t mask_lshift = test_mask_lshift>
using TestChunker = ChunkerTmpl<uint64_t, testgear64, mask_stages, mask_lshift>;
// Returns the number of bits set to 1 in the given mask.
uint32_t BitCount(uint64_t mask) {
uint32_t count = 0;
for (; mask; mask >>= 1) {
count += mask & 1u;
}
return count;
}
template <const uint64_t gear[256] = testgear64>
using TestChunker = Chunker64<gear>;
class ChunkerTest : public ::testing::Test {
public:
ChunkerTest() {}
protected:
template <uint32_t mask_stages>
static void ValidateStagesTmpl(const Config& cfg);
template <uint32_t mask_lshift>
static void ValidateLshiftTmpl(const Config& cfg);
};
template <uint32_t mask_stages>
void ChunkerTest::ValidateStagesTmpl(const Config& cfg) {
TestChunker<mask_stages> chunker(cfg, nullptr);
EXPECT_EQ(chunker.StagesCount(), mask_stages);
for (uint32_t i = 1; i < chunker.StagesCount(); i++) {
auto prev_stg = chunker.Stage(i - 1);
auto stg = chunker.Stage(i);
EXPECT_LT(prev_stg.barrier, stg.barrier)
<< "Stage " << i + 1 << " of " << mask_stages
<< ": barriers should be at increasing positions";
if (prev_stg.mask > 1) {
EXPECT_EQ(BitCount(prev_stg.mask), BitCount(stg.mask) + 1)
<< "Stage " << i + 1 << " of " << mask_stages
<< ": number of bits in adjacent stages should differ by 1";
} else {
EXPECT_EQ(1, BitCount(stg.mask))
<< "Stage " << i + 1 << " of " << mask_stages
<< ": number of bits in last bitmasks should be 1";
}
}
EXPECT_EQ(chunker.Stage(mask_stages - 1).barrier, cfg.max_size)
<< "final stage barrier must match the maximum chunk size";
}
// Tests that the stages to apply different bitmasks are initialized properly
TEST_F(ChunkerTest, ValidateStages) {
// Tests that the threshold for hash comparison is set correctly.
TEST_F(ChunkerTest, ValidateThreshold) {
// Sizes: 128/256/512 bytes
Config cfg(128, 256, 512);
ValidateStagesTmpl<1>(cfg);
ValidateStagesTmpl<2>(cfg);
ValidateStagesTmpl<3>(cfg);
ValidateStagesTmpl<4>(cfg);
ValidateStagesTmpl<5>(cfg);
ValidateStagesTmpl<6>(cfg);
ValidateStagesTmpl<7>(cfg);
ValidateStagesTmpl<8>(cfg);
// Sizes: 128/256/512 KiB
cfg = Config(128 << 10, 256 << 10, 512 << 10);
ValidateStagesTmpl<1>(cfg);
ValidateStagesTmpl<2>(cfg);
ValidateStagesTmpl<3>(cfg);
ValidateStagesTmpl<4>(cfg);
ValidateStagesTmpl<5>(cfg);
ValidateStagesTmpl<6>(cfg);
ValidateStagesTmpl<7>(cfg);
ValidateStagesTmpl<8>(cfg);
ValidateStagesTmpl<16>(cfg);
ValidateStagesTmpl<32>(cfg);
ValidateStagesTmpl<64>(cfg);
// Sizes: 128/256/512 MiB
cfg = Config(128 << 20, 256 << 20, 512 << 20);
ValidateStagesTmpl<1>(cfg);
ValidateStagesTmpl<2>(cfg);
ValidateStagesTmpl<3>(cfg);
ValidateStagesTmpl<4>(cfg);
ValidateStagesTmpl<5>(cfg);
ValidateStagesTmpl<6>(cfg);
ValidateStagesTmpl<7>(cfg);
ValidateStagesTmpl<8>(cfg);
ValidateStagesTmpl<16>(cfg);
ValidateStagesTmpl<32>(cfg);
ValidateStagesTmpl<64>(cfg);
// Sizes: 0/512/1024 KiB
cfg = Config(0, 512 << 10, 1024 << 10);
ValidateStagesTmpl<1>(cfg);
ValidateStagesTmpl<2>(cfg);
ValidateStagesTmpl<3>(cfg);
ValidateStagesTmpl<4>(cfg);
ValidateStagesTmpl<5>(cfg);
ValidateStagesTmpl<6>(cfg);
ValidateStagesTmpl<7>(cfg);
ValidateStagesTmpl<8>(cfg);
ValidateStagesTmpl<16>(cfg);
ValidateStagesTmpl<32>(cfg);
ValidateStagesTmpl<64>(cfg);
// Sizes: 0/512/1024 MiB
cfg = Config(0, 512 << 20, 1024 << 20);
ValidateStagesTmpl<1>(cfg);
ValidateStagesTmpl<2>(cfg);
ValidateStagesTmpl<3>(cfg);
ValidateStagesTmpl<4>(cfg);
ValidateStagesTmpl<5>(cfg);
ValidateStagesTmpl<6>(cfg);
ValidateStagesTmpl<7>(cfg);
ValidateStagesTmpl<8>(cfg);
ValidateStagesTmpl<16>(cfg);
ValidateStagesTmpl<32>(cfg);
ValidateStagesTmpl<64>(cfg);
}
template <uint32_t mask_lshift>
void ChunkerTest::ValidateLshiftTmpl(const Config& cfg) {
TestChunker<1, mask_lshift> chunker(cfg, nullptr);
uint64_t mask = chunker.Stage(0).mask;
uint64_t expected = BitCount(mask);
EXPECT_GE(expected, 1) << "no bits were set in the bit mask for lshift "
<< mask_lshift;
// Compare no. of all 1-bits to no. of 1-bits with the given shift amount.
uint32_t actual = 0;
for (; mask; mask >>= mask_lshift) {
actual += mask & 1u;
}
EXPECT_EQ(expected, actual)
<< "number of bits set is different with lshift " << mask_lshift;
}
// Tests that the bitmasks for each stage honor the mask_lshift template
// parameter correctly.
TEST_F(ChunkerTest, ValidateLshift) {
Config cfg(32, 64, 128);
ValidateLshiftTmpl<1>(cfg);
ValidateLshiftTmpl<2>(cfg);
ValidateLshiftTmpl<3>(cfg);
ValidateLshiftTmpl<4>(cfg);
ValidateLshiftTmpl<5>(cfg);
TestChunker<> chunker(cfg, nullptr);
EXPECT_EQ(0x1fc07f01fc07f01, chunker.Threshold());
}
// Tests that the minimum chunk size is not undercut.
TEST_F(ChunkerTest, MinChunkSize) {
Config cfg(32, 64, 128);
Config cfg(64, 96, 128);
std::vector<size_t> chunk_sizes;
TestChunker<> chunker(cfg, [&](const uint8_t* /* data */, size_t len) {
chunk_sizes.push_back(len);
@@ -187,7 +53,7 @@ TEST_F(ChunkerTest, MinChunkSize) {
std::vector<uint8_t> data(cfg.max_size, 0);
chunker.Process(data.data(), data.size());
chunker.Finalize();
EXPECT_EQ(chunk_sizes.size(), 4);
EXPECT_EQ(chunk_sizes.size(), 2);
for (size_t size : chunk_sizes) {
EXPECT_EQ(size, cfg.min_size);
}