catarrh/cdc-file-transfer
1
0
Fork 0
mirror of https://github.com/google/cdc-file-transfer.git synced 2026-09-13 01:10:44 +03:00
Code Issues Packages Projects Releases Wiki Activity

Change fastcdc to a better and simpler algorithm. (#79)

Browse Source 
This CL changes the chunking algorithm from "normalized chunking" to
simple "regression chunking", and changes the has criteria from
'hash&mask' to 'hash<=threshold'. These are all ideas taken from
testing and analysis done at
  https://github.com/dbaarda/rollsum-chunking/blob/master/RESULTS.rst
Regression chunking was introduced in
  https://www.usenix.org/system/files/conference/atc12/atc12-final293.pdf

The algorithm uses an arbitrary number of regressions using power-of-2
regression target lengths. This means we can use a simple bitmask for
the regression hash criteria.

Regression chunking yields high deduplication rates even for lower max
chunk sizes, so that the cdc_stream max chunk can be reduced to 512K
from 1024K. This fixes potential latency spikes from large chunks.
This commit is contained in:
Donovan Baarda
2023-02-09 01:06:41 +11:00
committed by GitHub
parent 24906eb36e
commit fcc4cbc3f3
10 changed files with 121 additions and 331 deletions
Download Patch File Download Diff File Expand all files Collapse all files
manifest/manifest_test_base.cc
+12 -5
View File
@@ -201,8 +201,11 @@ bool ManifestTestBase::InProgress(const ContentIdProto& manifest_id,
void ManifestTestBase::ValidateChunkLookup(const std::string& rel_path,
bool expect_contained) {
Buffer file;
EXPECT_OK(path::ReadFile(path::Join(cfg_.src_dir, rel_path), &file));
uint64_t offset = 0;
auto handler = [&offset, &rel_path, file_chunks = &file_chunks_,
auto handler = [&file, &offset, &rel_path, file_chunks = &file_chunks_,
expect_contained](const void* data, size_t size) {
ContentIdProto id = ContentId::FromArray(data, size);
@@ -214,8 +217,14 @@ void ManifestTestBase::ValidateChunkLookup(const std::string& rel_path,
expect_contained);
if (expect_contained) {
EXPECT_EQ(lookup_path, rel_path);
EXPECT_EQ(lookup_offset, offset);
EXPECT_EQ(lookup_size, size);
// The offset can be ambiguous since the file might contain duplicate
// data. Make sure that the actual data is the same.
EXPECT_LE(offset + size, file.size());
EXPECT_LE(lookup_offset + size, file.size());
EXPECT_EQ(memcmp(file.data() + offset, file.data() + lookup_offset, size),
0);
}
offset += size;
@@ -224,9 +233,7 @@ void ManifestTestBase::ValidateChunkLookup(const std::string& rel_path,
cfg_.max_chunk_size);
fastcdc::Chunker chunker(cdc_cfg, handler);
Buffer b;
EXPECT_OK(path::ReadFile(path::Join(cfg_.src_dir, rel_path), &b));
chunker.Process(reinterpret_cast<uint8_t*>(b.data()), b.size());
chunker.Process(reinterpret_cast<uint8_t*>(file.data()), file.size());
chunker.Finalize();
}