[cdc_rsync] Improve throughput for local copies (#74)

On Windows, fclose() seems to be very expensive for large files, where
closing a 1 GB file takes up to 5 seconds. This CL calls fclose() in
background threads. This tremendously improves local syncs, e.g.
copying a 4.5 GB, 300 files data set takes only 7 seconds instead of
30 seconds.

Also increases the buffer size for copying from 16K to 128K (better
throughput for local copies), and adds a timestamp to debug and
verbose console logs (useful when comparing client and server logs).
This commit is contained in:
Lutz Justen
2023-01-31 16:33:03 +01:00
committed by GitHub
parent 1200b34316
commit 5a909bb443
9 changed files with 275 additions and 73 deletions
+32
View File
@@ -151,5 +151,37 @@ TEST_F(ThreadpoolTest, GetCompletedTask) {
EXPECT_EQ(completed_task.get(), task);
}
TEST_F(ThreadpoolTest, SetTaskCompletedCallback) {
auto task_func = [](Task::IsCancelledPredicate) { /* empty */ };
Semaphore task_finished(0);
Threadpool pool(1);
std::atomic_bool finished = false;
pool.SetTaskCompletedCallback(
[&task_finished, &finished](std::unique_ptr<Task> task) {
finished = true;
task_finished.Signal();
});
pool.QueueTask(std::make_unique<TestTask>(task_func));
task_finished.Wait();
EXPECT_TRUE(finished);
EXPECT_FALSE(pool.TryGetCompletedTask());
}
TEST_F(ThreadpoolTest, WaitForQueuedTasksAtMost) {
Semaphore task_signal(0);
auto task_func = [&task_signal](Task::IsCancelledPredicate) {
task_signal.Wait();
};
Threadpool pool(1);
pool.QueueTask(std::make_unique<TestTask>(task_func));
pool.QueueTask(std::make_unique<TestTask>(task_func));
EXPECT_FALSE(pool.WaitForQueuedTasksAtMost(1, absl::Milliseconds(10)));
task_signal.Signal();
EXPECT_TRUE(pool.WaitForQueuedTasksAtMost(1, absl::Milliseconds(5000)));
EXPECT_EQ(pool.NumQueuedTasks(), 1);
task_signal.Signal();
}
} // namespace
} // namespace cdc_ft