[cdc_stream] Use ephemeral ports (#100)

Instead of running netstat/ss on local and remote systems, just bind
with port 0 to find an ephemeral port. This is much more robust,
simpler and a bit faster. Since the remote port is only known after
running cdc_fuse_fs, port forwarding has to be set up after running
cdc_fuse_fs.

.github/workflows/create_release.yml

@@ -14,6 +14,30 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
+      - name: Use last commit hash as build version for the developer build.
+        if: "startsWith(github.ref, 'refs/heads/')"
+        run: echo "build_version=${GITHUB_SHA}" >> $GITHUB_ENV
+
+      - name: Use tag name as build version for the release build.
+        if: startsWith(github.ref, 'refs/tags/v')
+        run: echo "build_version=${GITHUB_REF#refs/tags/v}" >> $GITHUB_ENV
+
+      # This flow should not be used for pull requests. However the section
+      # below might be useful for testing purposes.
+      - name: Use last commit hash as build version for the pull request
+        if: startsWith(github.ref, 'refs/pull')
+        run: echo "build_version=${GITHUB_SHA}" >> $GITHUB_ENV
+
+      - name: Replace CDC_BUILD_VERSION
+        run: |
+          if grep -q "DCDC_BUILD_VERSION=DEV" "common/BUILD"; then
+              sed -i 's/DCDC_BUILD_VERSION=DEV/DCDC_BUILD_VERSION=${{ env.build_version }}/g' common/BUILD
+          else
+              echo "CDC_BUILD_VERSION was moved out from common/BUILD file."
+              echo "Please edit create_release.yaml workflow."
+              exit 1
+          fi
+
       - name: Initialize submodules
         run: git submodule update --init --recursive
 
@@ -39,7 +63,6 @@ jobs:
                      --test_output=errors --local_test_jobs=1  \
                      -- //... -//third_party/... -//cdc_rsync_server:file_finder_test
 
-      # The artifact collector doesn't like the fact that bazel-bin is a symlink.
       - name: Copy artifacts
         run: |
           mkdir artifacts
@@ -58,6 +81,40 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
+      - name: Use last commit hash as build version for the developer build.
+        if: "startsWith(github.ref, 'refs/heads/')"
+        run: |
+          $build_version="${{ github.sha }}"
+          echo "build_version=$build_version" >> $env:GITHUB_ENV
+
+      - name: Use tag name as build version for the release build.
+        if: "startsWith(github.ref, 'refs/tags/v')"
+        run: |
+          $build_version="${{ github.ref }}".replace("refs/tags/v", "")
+          echo "build_version=$build_version" >> $env:GITHUB_ENV
+
+      # This flow should not be used for pull requests. However the section
+      # below might be useful for testing purposes.
+      - name: Use last commit hash as build version for the pull request
+        if: startsWith(github.ref, 'refs/pull')
+        run: |
+          $build_version="${{ github.sha }}"
+          echo "build_version=$build_version" >> $env:GITHUB_ENV
+
+      - name: Replace CDC_BUILD_VERSION
+        run: |
+          $cdc_version = Select-String -Path common/BUILD -Pattern "DCDC_BUILD_VERSION=DEV"
+          if ($cdc_version -ne $null) {
+              $build_file = Get-Content -path common/BUILD -Raw
+              $build_file = $build_file -replace 'DCDC_BUILD_VERSION=DEV','DCDC_BUILD_VERSION=${{ env.build_version }}'
+              $build_file | Set-Content -Path common/BUILD
+          }
+          else {
+              Write-Host "CDC_BUILD_VERSION was moved out from common/BUILD file."
+              Write-Host "Please edit create_release.yaml workflow."
+              exit 1
+          }
+
       - name: Initialize submodules
         run: git submodule update --init --recursive
 
@@ -94,14 +151,16 @@ jobs:
             //manifest/... `
             //metrics/...
 
-      # The artifact collector doesn't like the fact that bazel-bin is a symlink.
       - name: Copy artifacts
         run: |
           mkdir artifacts
+          mkdir artifacts\docs
           cp bazel-bin/cdc_rsync/cdc_rsync.exe artifacts
+          cp bazel-bin/cdc_rsync_server/cdc_rsync_server.exe artifacts
           cp bazel-bin/cdc_stream/cdc_stream.exe artifacts
           cp LICENSE artifacts
           cp README.md artifacts
+          cp docs\* artifacts\docs
 
       - name: Upload artifacts
         uses: actions/upload-artifact@v3
@@ -119,7 +178,10 @@ jobs:
       - name: Zip binaries
         run: |
           # The ref resolves to "main" for latest and e.g. "v0.1.0" for tagged.
-          BINARIES_ZIP_NAME=cdc-file-transfer-binaries-${GITHUB_REF#refs/*/}-x64.zip
+          REF=${GITHUB_REF#refs/*/}
+          # For pull requests, this is e.g. '70/merge', so replace / with -.
+          REF=${REF/\//-}
+          BINARIES_ZIP_NAME=cdc-file-transfer-binaries-$REF-x64.zip
           echo "BINARIES_ZIP_NAME=$BINARIES_ZIP_NAME" >> $GITHUB_ENV
           zip -j $BINARIES_ZIP_NAME Windows-Artifacts/* Linux-Artifacts/*
 

.github/workflows/lint.yml

@@ -3,7 +3,7 @@ name: Lint
 on:
   push:
     branches:
-      - master
+      - main
   pull_request:
 
 jobs:

README.md

@@ -1,8 +1,8 @@
 # CDC File Transfer
 
-Born from the ashes of Stadia, this repository contains tools for synching and
+Born from the ashes of Stadia, this repository contains tools for syncing and
-streaming files from Windows to Linux. They are based on Content Defined
+streaming files from Windows to Windows or Linux. The tools are based on Content
-Chunking (CDC), in particular
+Defined Chunking (CDC), in particular
 [FastCDC](https://www.usenix.org/conference/atc16/technical-sessions/presentation/xia),
 to split up files into chunks.
 
@@ -38,27 +38,103 @@ version of the files available in the target directory.
 </p>
 
 The remote diffing algorithm is based on CDC. In our tests, it is up to 30x
-faster than the one used in rsync (1500 MB/s vs 50 MB/s).
+faster than the one used in `rsync` (1500 MB/s vs 50 MB/s).
 
-The following chart shows a comparison of `cdc_rsync` and Linux rsync running
+The following chart shows a comparison of `cdc_rsync` and Linux `rsync` running
 under Cygwin on Windows. The test data consists of 58 development builds
 of some game provided to us for evaluation purposes. The builds are 40-45 GB
 large. For this experiment, we uploaded the first build, then synced the second
 build with each of the two tools and measured the time. For example, syncing
-from build 1 to build 2 took 210 seconds with the Linux rsync, but only 75
+from build 1 to build 2 took 210 seconds with the Cygwin `rsync`, but only 75
 seconds with `cdc_rsync`. The three outliers are probably feature drops from
 another development branch, where the delta was much higher. Overall,
-`cdc_rsync` syncs files about **3 times faster** than Linux rsync.
+`cdc_rsync` syncs files about **3 times faster** than Cygwin `rsync`.
 
 <p align="center">
   <img src="docs/cdc_rsync_vs_cygwin_rsync.png" alt="Comparison of cdc_rsync and Linux rsync running in Cygwin" width="753" />
 </p>
 
+We also ran the experiment with the native Linux `rsync`, i.e syncing Linux to
+Linux, to rule out issues with Cygwin. Linux `rsync` performed on average 35%
+worse than Cygwin `rsync`, which can be attributed to CPU differences. We did
+not include it in the figure because of this, but you can find it
+[here](docs/cdc_rsync_vs_cygwin_rsync_vs_linux_rsync.png).
+
+### How does it work and why is it faster?
+
+The standard Linux `rsync` splits a file into fixed-size chunks of typically
+several KB.
+
+<p align="center">
+  <img src="docs/fixed_size_chunks.png" alt="Linux rsync uses fixed size chunks" width="258" />
+</p>
+
+If the file is modified in the middle, e.g. by inserting `xxxx` after `567`,
+this usually means that <span style="color: red">the modified chunks as well as
+all subsequent chunks</span> change.
+
+<p align="center">
+  <img src="docs/fixed_size_chunks_inserted.png" alt="Fixed size chunks after inserting data" width="301" />
+</p>
+
+The standard `rsync` algorithm hashes the chunks of the remote "old" file
+and sends the hashes to the local device. The local device then figures out 
+which parts of the "new" file matches known chunks.
+
+<p align="center">
+  <img src="docs/linux_rsync_animation.gif" alt="Syncing a file with the standard Linux rsync" width="855" />
+  <br>
+  Standard rsync algorithm
+</p>
+
+This is a simplification. The actual algorithm is more complicated and uses
+two hashes, a weak rolling hash and a strong hash, see
+[here](https://rsync.samba.org/tech_report/) for a great overview. What makes
+`rsync` relatively slow is the "no match" situation where the rolling hash does
+not match any remote hash, and the algorithm has to roll the hash forward and
+perform a hash map lookup for each byte. `rsync` goes to
+[great lengths](https://github.com/librsync/librsync/blob/master/src/hashtable.h)
+optimizing lookups.
+
+`cdc_rsync` does not use fixed-size chunks, but instead variable-size,
+content-defined chunks. That means, chunk boundaries are determined by the
+*local content* of the file, in practice a 64 byte sliding window. For more
+details, see
+[the FastCDC paper](https://www.usenix.org/conference/atc16/technical-sessions/presentation/xia)
+or take a look at [our implementation](fastcdc/fastcdc.h).
+
+<p align="center">
+  <img src="docs/variable_size_chunks.png" alt="cdc_rsync uses variable, content-defined size chunks" width="260" />
+</p>
+
+If the file is modified in the middle, only <span style="color: red">the modified
+chunks</span>, but not <span style="color: #38761d">subsequent chunks</span>
+change (unless they are less than 64 bytes away from the modifications).
+
+<p align="center">
+  <img src="docs/variable_size_chunks_inserted.png" alt="Content-defined chunks after inserting data" width="314" />
+</p>
+
+Computing the chunk boundaries is cheap and involves only a left-shift, a memory
+lookup, an `add` and an `and` operation for each input byte. This is cheaper
+than the hash map lookup for the standard `rsync` algorithm.
+
+Because of this, the `cdc_rsync` algorithm is faster than the standard
+`rsync`. It is also simpler. Since chunk boundaries move along with insertions
+or deletions, the task to match local and remote hashes is a trivial set
+difference operation. It does not involve a per-byte hash map lookup.
+
+<p align="center">
+  <img src="docs/cdc_rsync_animation.gif" alt="Syncing a file with cdc_rsync" width="857" />
+  <br>
+  cdc_rsync algorithm
+</p>
+
 ## CDC Stream
 
-`cdc_stream` is a tool to stream files and directories from a Windows machine to a
+`cdc_stream` is a tool to stream files and directories from a Windows machine to
-Linux device. Conceptually, it is similar to [sshfs](https://github.com/libfuse/sshfs),
+a Linux device. Conceptually, it is similar to
-but it is optimized for read speed.
+[sshfs](https://github.com/libfuse/sshfs), but it is optimized for read speed.
 * It caches streamed data on the Linux device.
 * If a file is re-read on Linux after it changed on Windows, only the
   differences are streamed again. The rest is read from the cache.
@@ -85,18 +161,48 @@ In one case, the game is streamed via `sshfs`, in the other case we use
   <img src="docs/cdc_stream_vs_sshfs.png" alt="Comparison of cdc_stream and sshfs" width="752" />
 </p>
 
+# Supported Platforms
+
+| `cdc_rsync`                  | From                 | To                   |
+|:-----------------------------|:--------------------:|:--------------------:|
+| Windows x86_64               | &check;              | &check; <sup>1</sup> |
+| Ubuntu 22.04 x86_64          | &cross; <sup>2</sup> | &check;              |
+| Ubuntu 22.04 aarch64         | &cross;              | &cross;              |
+| macOS 13 x86_64 <sup>3</sup> | &cross;              | &cross;              |
+| macOS 13 aarch64 <sup>3</sup>| &cross;              | &cross;              |
+
+| `cdc_stream`                 | From                 | To                   |
+|:-----------------------------|:--------------------:|:--------------------:|
+| Windows x86_64               | &check;              | &cross;              |
+| Ubuntu 22.04 x86_64          | &cross;              | &check;              |
+| Ubuntu 22.04 aarch64         | &cross;              | &cross;              |
+| macOS 13 x86_64 <sup>3</sup> | &cross;              | &cross;              |
+| macOS 13 aarch64 <sup>3</sup>| &cross;              | &cross;              |
+
+<span style="font-size: 0.8rem">
+
+<sup>1</sup> Only local syncs, e.g. `cdc_rsync C:\src\* C:\dst`. Support for
+remote syncs is being added, see
+[#61](https://github.com/google/cdc-file-transfer/issues/61).  
+<sup>2</sup> See [#56](https://github.com/google/cdc-file-transfer/issues/56).  
+<sup>3</sup> See [#62](https://github.com/google/cdc-file-transfer/issues/62).
+
+</span>
+
 # Getting Started
 
 Download the precompiled binaries from the
-[latest release](https://github.com/google/cdc-file-transfer/releases).
+[latest release](https://github.com/google/cdc-file-transfer/releases) to a
-We currently provide Linux binaries compiled on
+Windows device and unzip them. The Linux binaries are automatically deployed
+to `~/.cache/cdc-file-transfer` by the Windows tools. There is no need to manually
+deploy them. We currently provide Linux binaries compiled on
 [Github's latest Ubuntu](https://github.com/actions/runner-images) version.
 If the binaries work for you, you can skip the following two sections.
 
 Alternatively, the project can be built from source. Some binaries have to be
 built on Windows, some on Linux.
 
-## Prerequisites
+## Prerequisites for Building
 
 To build the tools from source, the following steps have to be executed on
 **both Windows and Linux**.
@@ -114,61 +220,59 @@ To build the tools from source, the following steps have to be executed on
   git submodule update --init --recursive
   ```
 
-Finally, install an SSH client on the Windows device if not present.
+Finally, install an SSH client on the Windows machine if not present.
-The file transfer tools require `ssh.exe` and `scp.exe`.
+The file transfer tools require `ssh.exe` and `sftp.exe`.
 
 ## Building
 
-The two tools can be built and used independently.
+The two tools CDC RSync and CDC Stream can be built and used independently.
 
 ### CDC RSync
 
-* Build Linux components
+* On a Linux device, build the Linux components
   ```
   bazel build --config linux --compilation_mode=opt --linkopt=-Wl,--strip-all --copt=-fdata-sections --copt=-ffunction-sections --linkopt=-Wl,--gc-sections //cdc_rsync_server
   ```
-* Build Windows components
+* On a Windows device, build the Windows components
   ```
   bazel build --config windows --compilation_mode=opt --copt=/GL //cdc_rsync
   ```
 * Copy the Linux build output file `cdc_rsync_server` from 
-  `bazel-bin/cdc_rsync_server` on the Linux system to `bazel-bin\cdc_rsync`
+  `bazel-bin/cdc_rsync_server` to `bazel-bin\cdc_rsync` on the Windows machine.
-  on the Windows machine.
 
 ### CDC Stream
 
-* Build Linux components
+* On a Linux device, build the Linux components
   ```
   bazel build --config linux --compilation_mode=opt --linkopt=-Wl,--strip-all --copt=-fdata-sections --copt=-ffunction-sections --linkopt=-Wl,--gc-sections //cdc_fuse_fs
   ```
-* Build Windows components
+* On a Windows device, build the Windows components
   ```
   bazel build --config windows --compilation_mode=opt --copt=/GL //cdc_stream
   ```
 * Copy the Linux build output files `cdc_fuse_fs` and `libfuse.so` from 
-  `bazel-bin/cdc_fuse_fs` on the Linux system to `bazel-bin\cdc_stream`
+  `bazel-bin/cdc_fuse_fs` to `bazel-bin\cdc_stream` on the Windows machine.
-  on the Windows machine.
 
 ## Usage
 
-The tools require a setup where you can use SSH and SCP from the Windows machine
+The tools require a setup where you can use SSH and SFTP from the Windows
-to the Linux device without entering a password, e.g. by using key-based
+machine to the Linux device without entering a password, e.g. by using key-based
 authentication.
 
-### Configuring SSH and SCP
+### Configuring SSH and SFTP
 
-By default, the tools search `ssh.exe` and `scp.exe` from the path environment
+By default, the tools search `ssh.exe` and `sftp.exe` from the path environment
 variable. If you can run the following commands in a Windows cmd without
 entering your password, you are all set:
 ```
 ssh user@linux.device.com
-scp somefile.txt user@linux.device.com:
+sftp user@linux.device.com
 ```
 Here, `user` is the Linux user and `linux.device.com` is the Linux host to
 SSH into or copy the file to.
 
 If additional arguments are required, it is recommended to provide an SSH config
-file. By default, both `ssh.exe` and `scp.exe` use the file at
+file. By default, both `ssh.exe` and `sftp.exe` use the file at
 `%USERPROFILE%\.ssh\config` on Windows, if it exists. A possible config file
 that sets a username, a port, an identity file and a known host file could look
 as follows:
@@ -180,21 +284,21 @@ Host linux_device
 	IdentityFile C:\path\to\id_rsa
 	UserKnownHostsFile C:\path\to\known_hosts
 ```
-If `ssh.exe` or `scp.exe` cannot be found, you can specify the full paths via
+If `ssh.exe` or `sftp.exe` cannot be found, you can specify the full paths via
-the command line arguments `--ssh-command` and `--scp-command` for `cdc_rsync`
+the command line arguments `--ssh-command` and `--sftp-command` for `cdc_rsync`
 and `cdc_stream start` (see below), or set the environment variables
-`CDC_SSH_COMMAND` and `CDC_SCP_COMMAND`, e.g.
+`CDC_SSH_COMMAND` and `CDC_SFTP_COMMAND`, e.g.
 ```
 set CDC_SSH_COMMAND="C:\path with space\to\ssh.exe"
-set CDC_SCP_COMMAND="C:\path with space\to\scp.exe"
+set CDC_SFTP_COMMAND="C:\path with space\to\sftp.exe"
 ```
 Note that you can also specify SSH configuration via the environment variables
 instead of using a config file:
 ```
 set CDC_SSH_COMMAND=C:\path\to\ssh.exe -p 12345 -i C:\path\to\id_rsa -oUserKnownHostsFile=C:\path\to\known_hosts
-set CDC_SCP_COMMAND=C:\path\to\scp.exe -P 12345 -i C:\path\to\id_rsa -oUserKnownHostsFile=C:\path\to\known_hosts
+set CDC_SFTP_COMMAND=C:\path\to\sftp.exe -P 12345 -i C:\path\to\id_rsa -oUserKnownHostsFile=C:\path\to\known_hosts
 ```
-Note the small `-p` for `ssh.exe` and the capital `-P` for `scp.exe`.
+Note the small `-p` for `ssh.exe` and the capital `-P` for `sftp.exe`.
 
 #### Google Specific
 
@@ -202,7 +306,7 @@ For Google internal usage, set the following environment variables to enable SSH
 authentication using a Google security key:
 ```
 set CDC_SSH_COMMAND=C:\gnubby\bin\ssh.exe
-set CDC_SCP_COMMAND=C:\gnubby\bin\scp.exe
+set CDC_SFTP_COMMAND=C:\gnubby\bin\sftp.exe
 ```
 Note that you will have to touch the security key multiple times during the
 first run. Subsequent runs only require a single touch.
@@ -228,6 +332,10 @@ To get per file progress, add `-v`:
 ```
 cdc_rsync C:\path\to\assets\* user@linux.device.com:~/assets -vr
 ```
+The tool also supports local syncs:
+```
+cdc_rsync C:\path\to\assets\* C:\path\to\destination -vr
+```
 
 ### CDC Stream
 
@@ -276,7 +384,7 @@ instead of to the file.
 `cdc_rsync` always logs to the console. To increase log verbosity, pass `-vvv`
 for debug logs or `-vvvv` for verbose logs.
 
-For both sync and stream, the debug logs contain all SSH and SCP commands that
+For both sync and stream, the debug logs contain all SSH and SFTP commands that
 are attempted to run, which is very useful for troubleshooting. If a command
 fails unexpectedly, copy it and run it in isolation. Pass `-vv` or `-vvv` for
 additional debug output.

absl.natvis

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<AutoVisualizer xmlns="http://schemas.microsoft.com/vstudio/debugger/natvis/2010">
+  <Type Name="absl::flat_hash_map&lt;*&gt;">
+    <DisplayString Condition="size_ == 0">empty</DisplayString>
+    <DisplayString>{{ size={size_} }}</DisplayString>
+    <Expand>
+      <Item Name="[size]" ExcludeView="simple">size_</Item>
+      <Item Name="[capacity]" ExcludeView="simple">capacity_</Item>
+      <CustomListItems MaxItemsPerView="5000">
+        <Variable Name="iSlot" InitialValue="0" />
+        <Size>size_</Size>
+        <Loop>
+          <!-- bool IsFull(ctrl_t c) { return c >= 0; } -->
+          <If Condition="ctrl_[iSlot] &gt;= 0">
+            <Item>slots_[iSlot]</Item>
+          </If>
+          <Exec>iSlot++</Exec>
+          <Break Condition="iSlot == capacity_" />
+        </Loop>
+      </CustomListItems>
+    </Expand>
+  </Type>
+  <Type Name="absl::container_internal::map_slot_type&lt;*&gt;">
+    <DisplayString>{value.first}:{value.second}</DisplayString>
+    <Expand>
+      <Item Name="[key]" ExcludeView="simple">value.first</Item>
+      <Item Name="[value]" ExcludeView="simple">value.second</Item>
+    </Expand>
+  </Type>
+</AutoVisualizer>

all_files.vcxitems

@@ -15,9 +15,12 @@
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="$(MSBuildThisFileDirectory)absl_helper\jedec_size_flag.cc" />
-    <ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync\base\socket.cc" />
+    <ClCompile Include="$(MSBuildThisFileDirectory)cdc_fuse_fs\mock_config_stream_client.cc" />
+    <ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync\server_arch.cc" />
+    <ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync\server_arch_test.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_stream\asset_stream_config.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_stream\asset_stream_server.cc" />
+    <ClCompile Include="$(MSBuildThisFileDirectory)cdc_stream\background_service_client.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_stream\background_service_impl.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_stream\base_command.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_stream\cdc_fuse_manager.cc" />
@@ -35,8 +38,11 @@
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_stream\start_command.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_stream\start_service_command.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_stream\stop_service_command.cc" />
-    <ClCompile Include="$(MSBuildThisFileDirectory)common\port_range_parser.cc" />
+    <ClCompile Include="$(MSBuildThisFileDirectory)fastcdc\fastcdc_test.cc" />
-    <ClCompile Include="$(MSBuildThisFileDirectory)common\port_range_parser_test.cc" />
+    <ClCompile Include="$(MSBuildThisFileDirectory)manifest\pending_assets_queue.cc" />
+    <ClInclude Include="$(MSBuildThisFileDirectory)cdc_fuse_fs\mock_config_stream_client.h" />
+    <ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync\server_arch.h" />
+    <ClInclude Include="$(MSBuildThisFileDirectory)cdc_stream\background_service_client.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)cdc_stream\stop_command.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_stream\testing_asset_stream_server.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_fuse_fs\asset.cc" />
@@ -49,13 +55,19 @@
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_fuse_fs\mock_libfuse.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_indexer\indexer.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_indexer\main.cc" />
+    <ClCompile Include="$(MSBuildThisFileDirectory)common\ansi_filter.cc" />
+    <ClCompile Include="$(MSBuildThisFileDirectory)common\ansi_filter_test.cc" />
+    <ClCompile Include="$(MSBuildThisFileDirectory)common\arch_type.cc" />
+    <ClCompile Include="$(MSBuildThisFileDirectory)common\arch_type_test.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)common\buffer.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)common\buffer_test.cc" />
+    <ClCompile Include="$(MSBuildThisFileDirectory)common\client_socket.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)common\clock.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)common\dir_iter.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)common\dir_iter_test.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)common\errno_mapping.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)common\errno_mapping_test.cc" />
+    <ClCompile Include="$(MSBuildThisFileDirectory)common\fake_socket.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)common\file_watcher_win.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)common\file_watcher_win_test.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)common\gamelet_component.cc" />
@@ -68,6 +80,8 @@
     <ClCompile Include="$(MSBuildThisFileDirectory)common\path_test.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)common\port_manager_test.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)common\port_manager_win.cc" />
+    <ClCompile Include="$(MSBuildThisFileDirectory)common\port_range_parser.cc" />
+    <ClCompile Include="$(MSBuildThisFileDirectory)common\port_range_parser_test.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)common\process_test.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)common\process_win.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)common\remote_util.cc" />
@@ -77,6 +91,8 @@
     <ClCompile Include="$(MSBuildThisFileDirectory)common\sdk_util_test.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)common\semaphore.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)common\semaphore_test.cc" />
+    <ClCompile Include="$(MSBuildThisFileDirectory)common\server_socket.cc" />
+    <ClCompile Include="$(MSBuildThisFileDirectory)common\socket.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)common\stats_collector.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)common\status.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)common\stopwatch.cc" />
@@ -102,10 +118,8 @@
     <ClCompile Include="$(MSBuildThisFileDirectory)data_store\mem_data_store_test.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync\base\cdc_interface.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync\base\cdc_interface_test.cc" />
-    <ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync\base\fake_socket.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync\base\message_pump.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync\base\message_pump_test.cc" />
-    <ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync\client_socket.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync\file_finder_and_sender.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync\file_finder_and_sender_test.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync\cdc_rsync_client.cc" />
@@ -126,7 +140,6 @@
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync_server\file_finder_test.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync_server\cdc_rsync_server.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync_server\main.cc" />
-    <ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync_server\server_socket.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync_server\unzstd_stream.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)manifest\asset_builder.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)manifest\content_id.cc" />
@@ -147,7 +160,12 @@
     <ClCompile Include="$(MSBuildThisFileDirectory)metrics\messages_test.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)metrics\metrics.cc" />
     <ClInclude Include="$(MSBuildThisFileDirectory)cdc_stream\stop_service_command.h" />
+    <ClInclude Include="$(MSBuildThisFileDirectory)common\ansi_filter.h" />
+    <ClInclude Include="$(MSBuildThisFileDirectory)common\arch_type.h" />
+    <ClInclude Include="$(MSBuildThisFileDirectory)common\build_version.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)common\port_range_parser.h" />
+    <ClInclude Include="$(MSBuildThisFileDirectory)fastcdc\fastcdc.h" />
+    <ClInclude Include="$(MSBuildThisFileDirectory)manifest\pending_assets_queue.h" />
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="$(MSBuildThisFileDirectory)absl_helper\jedec_size_flag.h" />
@@ -177,9 +195,11 @@
     <ClInclude Include="$(MSBuildThisFileDirectory)cdc_fuse_fs\mock_libfuse.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)cdc_indexer\indexer.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)common\buffer.h" />
+    <ClInclude Include="$(MSBuildThisFileDirectory)common\client_socket.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)common\clock.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)common\dir_iter.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)common\errno_mapping.h" />
+    <ClInclude Include="$(MSBuildThisFileDirectory)common\fake_socket.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)common\file_watcher_win.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)common\gamelet_component.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)common\grpc_status.h" />
@@ -193,6 +213,8 @@
     <ClInclude Include="$(MSBuildThisFileDirectory)common\scoped_handle_win.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)common\sdk_util.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)common\semaphore.h" />
+    <ClInclude Include="$(MSBuildThisFileDirectory)common\server_socket.h" />
+    <ClInclude Include="$(MSBuildThisFileDirectory)common\socket.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)common\stats_collector.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)common\status.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)common\status_macros.h" />
@@ -212,12 +234,9 @@
     <ClInclude Include="$(MSBuildThisFileDirectory)data_store\grpc_reader.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)data_store\mem_data_store.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync\base\cdc_interface.h" />
-    <ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync\base\fake_socket.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync\base\message_pump.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync\base\server_exit_code.h" />
-    <ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync\base\socket.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync\client_file_info.h" />
-    <ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync\client_socket.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync\file_finder_and_sender.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync\cdc_rsync_client.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync\parallel_file_opener.h" />
@@ -229,7 +248,6 @@
     <ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync_server\file_finder.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync_server\file_info.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync_server\cdc_rsync_server.h" />
-    <ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync_server\server_socket.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync_server\unzstd_stream.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)manifest\asset_builder.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)manifest\content_id.h" />
@@ -263,6 +281,7 @@
     <None Include="$(MSBuildThisFileDirectory)cdc_rsync\README.md" />
     <None Include="$(MSBuildThisFileDirectory)cdc_rsync_server\BUILD" />
     <None Include="$(MSBuildThisFileDirectory)cdc_rsync_tests\BUILD" />
+    <None Include="$(MSBuildThisFileDirectory)fastcdc\BUILD" />
     <None Include="$(MSBuildThisFileDirectory)manifest\BUILD" />
     <None Include="$(MSBuildThisFileDirectory)manifest_cli\BUILD" />
     <None Include="$(MSBuildThisFileDirectory)metrics\BUILD" />
@@ -282,6 +301,7 @@
     <None Include="$(MSBuildThisFileDirectory)WORKSPACE" />
   </ItemGroup>
   <ItemGroup>
+    <Natvis Include="$(MSBuildThisFileDirectory)absl.natvis" />
     <Natvis Include="$(MSBuildThisFileDirectory)manifest.natvis" />
     <Natvis Include="$(MSBuildThisFileDirectory)protobuf.natvis" />
   </ItemGroup>

cdc_fuse_fs/BUILD

@@ -9,8 +9,10 @@ cc_binary(
         ":cdc_fuse_fs_lib",
         ":constants",
         "//absl_helper:jedec_size_flag",
+        "//common:client_socket",
         "//common:gamelet_component",
         "//common:log",
+        "//common:server_socket",
         "//data_store:data_provider",
         "//data_store:disk_data_store",
         "//data_store:grpc_reader",

cdc_fuse_fs/constants.h

@@ -19,15 +19,22 @@
 
 namespace cdc_ft {
 
-// FUSE prints this to stdout when the binary timestamp and file size match the
+// FUSE prints
-// file on the workstation.
+//   Port 12345 cdc_fuse_fs is up-to-date
-static constexpr char kFuseUpToDate[] = "cdc_fuse_fs is up-to-date";
+// to stdout when its version matches the version (=build version or
+// size/timestamp for DEV builds) on the local device. The port is the gRPC port
+// that FUSE will try to connect to.
+static constexpr char kFusePortPrefix[] = "Port ";
+static constexpr char kFuseUpToDate[] = " cdc_fuse_fs is up-to-date";
 
-// FUSE prints this to stdout when the binary timestamp or file size does not
+// FUSE prints this to stdout when its version does not match the version on the
-// match the file on the workstation. It indicates that the binary has to be
+// local device. It indicates that the binary has to be redeployed.
-// redeployed.
 static constexpr char kFuseNotUpToDate[] = "cdc_fuse_fs is not up-to-date";
 
+// FUSE prints this to stdout when it can connect to its port. This means that
+// port forwarding has finished setting up, and startup is finished.
+static constexpr char kFuseConnected[] = "cdc_fuse_fs is connected";
+
 }  // namespace cdc_ft
 
 #endif  // CDC_FUSE_FS_CONSTANTS_H_

cdc_fuse_fs/main.cc

@@ -21,9 +21,11 @@
 #include "cdc_fuse_fs/cdc_fuse_fs.h"
 #include "cdc_fuse_fs/config_stream_client.h"
 #include "cdc_fuse_fs/constants.h"
+#include "common/client_socket.h"
 #include "common/gamelet_component.h"
 #include "common/log.h"
 #include "common/path.h"
+#include "common/server_socket.h"
 #include "data_store/data_provider.h"
 #include "data_store/disk_data_store.h"
 #include "data_store/grpc_reader.h"
@@ -37,6 +39,8 @@ namespace {
 constexpr char kFuseFilename[] = "cdc_fuse_fs";
 constexpr char kLibFuseFilename[] = "libfuse.so";
 
+constexpr absl::Duration kConnectionTimeout = absl::Seconds(60);
+
 bool IsUpToDate(const std::string& components_arg) {
   // Components are expected to reside in the same dir as the executable.
   std::string component_dir;
@@ -107,7 +111,6 @@ ABSL_FLAG(
     "Whitespace-separated triples filename, size and timestamp of the "
     "workstation version of this binary and dependencies. Used for a fast "
     "up-to-date check.");
-ABSL_FLAG(uint16_t, port, 0, "Port to connect to on localhost");
 ABSL_FLAG(cdc_ft::JedecSize, prefetch_size, cdc_ft::JedecSize(512 << 10),
           "Additional data to request from the server when a FUSE read of "
           "maximum size is detected. This amount is added to the original "
@@ -138,7 +141,6 @@ int main(int argc, char* argv[]) {
   std::vector<char*> mount_args = absl::ParseCommandLine(argc, argv);
   std::string instance = absl::GetFlag(FLAGS_instance);
   std::string components = absl::GetFlag(FLAGS_components);
-  uint16_t port = absl::GetFlag(FLAGS_port);
   std::string cache_dir = absl::GetFlag(FLAGS_cache_dir);
   int cache_dir_levels = absl::GetFlag(FLAGS_cache_dir_levels);
   int verbosity = absl::GetFlag(FLAGS_verbosity);
@@ -159,7 +161,18 @@ int main(int argc, char* argv[]) {
     printf("%s\n", cdc_ft::kFuseNotUpToDate);
     return 0;
   }
-  printf("%s\n", cdc_ft::kFuseUpToDate);
+
+  // Find an available port.
+  absl::StatusOr<int> port_or = cdc_ft::ServerSocket::FindAvailablePort();
+  if (!port_or.ok()) {
+    LOG_ERROR("Failed to find available port: %s\n",
+              port_or.status().ToString());
+    return 1;
+  }
+  int port = *port_or;
+
+  // Write marker for the server.
+  printf("%s%i%s\n", cdc_ft::kFusePortPrefix, port, cdc_ft::kFuseUpToDate);
   fflush(stdout);
 
   // Create mount dir if it doesn't exist yet.
@@ -189,6 +202,18 @@ int main(int argc, char* argv[]) {
   store.value()->SetCapacity(cache_capacity);
   LOG_INFO("Caching chunks in '%s'", store.value()->RootDir());
 
+  // Wait for port forwarding to listen to |port|.
+  status =
+      cdc_ft::ClientSocket::WaitForConnection(port, cdc_ft::kConnectionTimeout);
+  if (!status.ok()) {
+    LOG_ERROR("Failed to connect to port %i: %s", port, status.ToString());
+    return static_cast<int>(status.code());
+  }
+
+  // Write another marker for the server.
+  printf("%s\n", cdc_ft::kFuseConnected);
+  fflush(stdout);
+
   // Start a gRpc client.
   std::string client_address = absl::StrFormat("localhost:%u", port);
   grpc::ChannelArguments channel_args;

cdc_indexer/README.md

@@ -14,7 +14,7 @@ experimentation. See the file `indexer.h` for preprocessor macros that can be
 enabled, for example:
 
 ```
-bazel build -c opt --copt=-DCDC_GEAR_TABLE=1 //cdc_indexer
+bazel build -c opt --copt=-DCDC_GEAR_BITS=32 //cdc_indexer
 ```
 
 At the end of the operation, the indexer outputs a summary of the results such
@@ -25,7 +25,7 @@ as the following:
 Operation succeeded.
 
 Chunk size (min/avg/max): 128 KB / 256 KB / 1024 KB  |  Threads: 12
-gear_table: 64 bit  |  mask_s: 0x49249249249249  |  mask_l: 0x1249249249
+gear_table: 64 bit  |  threshold: 0x7fffc0001fff
            Duration:           00:03
         Total files:               2
        Total chunks:           39203

cdc_indexer/indexer.cc

@@ -140,8 +140,7 @@ Indexer::Impl::Impl(const IndexerConfig& cfg,
   fastcdc::Config ccfg(cfg_.min_chunk_size, cfg_.avg_chunk_size,
                        cfg_.max_chunk_size);
   Indexer::Chunker chunker(ccfg, nullptr);
-  cfg_.mask_s = chunker.Stage(0).mask;
+  cfg_.threshold = chunker.Threshold();
-  cfg_.mask_l = chunker.Stage(chunker.StagesCount() - 1).mask;
   // Collect inputs.
   for (auto it = inputs.begin(); it != inputs.end(); ++it) {
     inputs_.push(*it);
@@ -368,8 +367,7 @@ IndexerConfig::IndexerConfig()
       max_chunk_size(0),
       max_chunk_size_step(0),
       num_threads(0),
-      mask_s(0),
+      threshold(0) {}
-      mask_l(0) {}
 
 Indexer::Indexer() : impl_(nullptr) {}
 

cdc_indexer/indexer.h

@@ -27,16 +27,10 @@
 #include "fastcdc/fastcdc.h"
 
 // Compile-time parameters for the FastCDC algorithm.
-#define CDC_GEAR_32BIT 1
+#define CDC_GEAR_32BIT 32
-#define CDC_GEAR_64BIT 2
+#define CDC_GEAR_64BIT 64
-#ifndef CDC_GEAR_TABLE
+#ifndef CDC_GEAR_BITS
-#define CDC_GEAR_TABLE CDC_GEAR_64BIT
+#define CDC_GEAR_BITS CDC_GEAR_64BIT
-#endif
-#ifndef CDC_MASK_STAGES
-#define CDC_MASK_STAGES 7
-#endif
-#ifndef CDC_MASK_BIT_LSHIFT_AMOUNT
-#define CDC_MASK_BIT_LSHIFT_AMOUNT 3
 #endif
 
 namespace cdc_ft {
@@ -66,23 +60,20 @@ struct IndexerConfig {
   uint32_t num_threads;
   // Which hash function to use.
   HashType hash_type;
-  // The masks will be populated by the indexer, setting them here has no
+  // The threshold will be populated by the indexer, setting it here has no
-  // effect. They are in this struct so that they can be conveniently accessed
+  // effect. It is in this struct so that it can be conveniently accessed
-  // when printing the operation summary (and since they are derived from the
+  // when printing the operation summary (and since it is derived from the
-  // configuration, they are technically part of it).
+  // configuration, it is technically part of it).
-  uint64_t mask_s;
+  uint64_t threshold;
-  uint64_t mask_l;
 };
 
 class Indexer {
  public:
   using hash_t = std::string;
-#if CDC_GEAR_TABLE == CDC_GEAR_32BIT
+#if CDC_GEAR_BITS == CDC_GEAR_32BIT
-  typedef fastcdc::Chunker32<CDC_MASK_STAGES, CDC_MASK_BIT_LSHIFT_AMOUNT>
+  typedef fastcdc::Chunker32<> Chunker;
-      Chunker;
+#elif CDC_GEAR_BITS == CDC_GEAR_64BIT
-#elif CDC_GEAR_TABLE == CDC_GEAR_64BIT
+  typedef fastcdc::Chunker64<> Chunker;
-  typedef fastcdc::Chunker64<CDC_MASK_STAGES, CDC_MASK_BIT_LSHIFT_AMOUNT>
-      Chunker;
 #else
 #error "Unknown gear table"
 #endif

cdc_indexer/main.cc

@@ -64,9 +64,9 @@ namespace {
 
 const char* GearTable() {
   // The following macros are defined in indexer.h.
-#if CDC_GEAR_TABLE == CDC_GEAR_32BIT
+#if CDC_GEAR_BITS == CDC_GEAR_32BIT
   return "32 bit";
-#elif CDC_GEAR_TABLE == CDC_GEAR_64BIT
+#elif CDC_GEAR_BITS == CDC_GEAR_64BIT
   return "64 bit";
 #else
 #error "Unknown gear table"
@@ -165,9 +165,8 @@ void ShowSummary(const IndexerConfig& cfg, const Indexer::OpStats& stats,
             << HumanBytes(cfg.max_chunk_size)
             << "  |  Hash: " << HashTypeToString(cfg.hash_type)
             << "  |  Threads: " << cfg.num_threads << std::endl;
-  std::cout << "gear_table: " << GearTable() << "  |  mask_s: 0x" << std::hex
+  std::cout << "gear_table: " << GearTable() << "  |  threshold: 0x" << std::hex
-            << cfg.mask_s << "  |  mask_l: 0x" << cfg.mask_l << std::dec
+            << cfg.threshold << std::dec << std::endl;
-            << std::endl;
   std::cout << std::setw(title_w) << "Duration:" << std::setw(num_w)
             << HumanDuration(elapsed) << std::endl;
   std::cout << std::setw(title_w) << "Total files:" << std::setw(num_w)
@@ -279,11 +278,10 @@ absl::Status WriteResultsFile(const std::string& filepath,
 
   path::FileCloser closer(fout);
 
-  static constexpr int num_columns = 15;
+  static constexpr int num_columns = 14;
   static const char* columns[num_columns] = {
       "gear_table",
-      "mask_s",
+      "threshold",
-      "mask_l",
       "Min chunk size [KiB]",
       "Avg chunk size [KiB]",
       "Max chunk size [KiB]",
@@ -332,7 +330,7 @@ absl::Status WriteResultsFile(const std::string& filepath,
   // Write user-supplied description
   if (!description.empty()) std::fprintf(fout, "%s,", description.c_str());
   // Write chunking params.
-  std::fprintf(fout, "%s,0x%zx,0x%zx,", GearTable(), cfg.mask_s, cfg.mask_l);
+  std::fprintf(fout, "%s,0x%zx,", GearTable(), cfg.threshold);
   std::fprintf(fout, "%zu,%zu,%zu,", cfg.min_chunk_size >> 10,
                cfg.avg_chunk_size >> 10, cfg.max_chunk_size >> 10);
   // Write speed, files, chunks.

cdc_rsync/BUILD

@@ -18,19 +18,6 @@ cc_library(
     hdrs = ["client_file_info.h"],
 )
 
-cc_library(
-    name = "client_socket",
-    srcs = ["client_socket.cc"],
-    hdrs = ["client_socket.h"],
-    target_compatible_with = ["@platforms//os:windows"],
-    deps = [
-        "//cdc_rsync/base:socket",
-        "//common:log",
-        "//common:status",
-        "//common:util",
-    ],
-)
-
 cc_library(
     name = "file_finder_and_sender",
     srcs = ["file_finder_and_sender.cc"],
@@ -54,8 +41,8 @@ cc_test(
     data = ["testdata/root.txt"] + glob(["testdata/file_finder_and_sender/**"]),
     deps = [
         ":file_finder_and_sender",
-        "//cdc_rsync/base:fake_socket",
         "//cdc_rsync/protos:messages_cc_proto",
+        "//common:fake_socket",
         "//common:status_test_macros",
         "//common:test_main",
         "@com_google_googletest//:gtest",
@@ -67,32 +54,27 @@ cc_library(
     name = "cdc_rsync_client",
     srcs = ["cdc_rsync_client.cc"],
     hdrs = ["cdc_rsync_client.h"],
-    linkopts = select({
-        "//tools:windows": [
-            "/DEFAULTLIB:Ws2_32.lib",  # Sockets, e.g. recv, send, WSA*.
-        ],
-        "//conditions:default": [],
-    }),
     target_compatible_with = ["@platforms//os:windows"],
     deps = [
-        ":client_socket",
         ":file_finder_and_sender",
         ":parallel_file_opener",
         ":progress_tracker",
+        ":server_arch",
         ":zstd_stream",
         "//cdc_rsync/base:cdc_interface",
         "//cdc_rsync/base:message_pump",
         "//cdc_rsync/base:server_exit_code",
-        "//cdc_rsync/base:socket",
         "//cdc_rsync/protos:messages_cc_proto",
+        "//common:client_socket",
         "//common:gamelet_component",
         "//common:log",
         "//common:path",
         "//common:path_filter",
         "//common:platform",
-        "//common:port_manager",
         "//common:process",
         "//common:remote_util",
+        "//common:server_socket",
+        "//common:socket",
         "//common:status",
         "//common:status_macros",
         "//common:threadpool",
@@ -130,6 +112,7 @@ cc_library(
     hdrs = ["params.h"],
     deps = [
         ":cdc_rsync_client",
+        "//common:build_version",
         "//common:port_range_parser",
         "@com_github_zstd//:zstd",
         "@com_google_absl//absl/status",
@@ -172,13 +155,37 @@ cc_test(
     ],
 )
 
+cc_library(
+    name = "server_arch",
+    srcs = ["server_arch.cc"],
+    hdrs = ["server_arch.h"],
+    deps = [
+        "//common:ansi_filter",
+        "//common:arch_type",
+        "//common:path",
+        "//common:remote_util",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_test(
+    name = "server_arch_test",
+    srcs = ["server_arch_test.cc"],
+    deps = [
+        ":server_arch",
+        "//common:test_main",
+        "@com_google_absl//absl/strings",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
 cc_library(
     name = "zstd_stream",
     srcs = ["zstd_stream.cc"],
     hdrs = ["zstd_stream.h"],
     deps = [
-        ":client_socket",
         "//common:buffer",
+        "//common:client_socket",
         "//common:status",
         "//common:status_macros",
         "//common:stopwatch",
@@ -191,8 +198,8 @@ cc_test(
     srcs = ["zstd_stream_test.cc"],
     deps = [
         ":zstd_stream",
-        "//cdc_rsync/base:fake_socket",
         "//cdc_rsync_server:unzstd_stream",
+        "//common:fake_socket",
         "//common:status_test_macros",
         "//common:test_main",
         "@com_github_zstd//:zstd",

cdc_rsync/base/BUILD

@@ -28,31 +28,21 @@ cc_test(
     data = ["testdata/root.txt"] + glob(["testdata/cdc_interface/**"]),
     deps = [
         ":cdc_interface",
-        ":fake_socket",
+        "//common:fake_socket",
         "//common:status_test_macros",
         "//common:test_main",
         "@com_google_googletest//:gtest",
     ],
 )
 
-cc_library(
-    name = "fake_socket",
-    srcs = ["fake_socket.cc"],
-    hdrs = ["fake_socket.h"],
-    deps = [
-        "//cdc_rsync/base:socket",
-        "@com_google_absl//absl/status",
-    ],
-)
-
 cc_library(
     name = "message_pump",
     srcs = ["message_pump.cc"],
     hdrs = ["message_pump.h"],
     deps = [
-        ":socket",
         "//common:buffer",
         "//common:log",
+        "//common:socket",
         "//common:status",
         "@com_google_absl//absl/status",
         "@com_google_absl//absl/strings:str_format",
@@ -64,9 +54,9 @@ cc_test(
     name = "message_pump_test",
     srcs = ["message_pump_test.cc"],
     deps = [
-        ":fake_socket",
         ":message_pump",
         "//cdc_rsync/protos:messages_cc_proto",
+        "//common:fake_socket",
         "//common:status_test_macros",
         "//common:test_main",
         "@com_google_googletest//:gtest",
@@ -78,19 +68,6 @@ cc_library(
     hdrs = ["server_exit_code.h"],
 )
 
-cc_library(
-    name = "socket",
-    srcs = ["socket.cc"],
-    hdrs = ["socket.h"],
-    deps = [
-        "//common:log",
-        "//common:platform",
-        "//common:status",
-        "//common:util",
-        "@com_google_absl//absl/status",
-    ],
-)
-
 filegroup(
     name = "all_test_sources",
     srcs = glob(["*_test.cc"]),

cdc_rsync/base/cdc_interface_test.cc

@@ -17,8 +17,8 @@
 #include <cstdio>
 #include <fstream>
 
-#include "cdc_rsync/base/fake_socket.h"
 #include "cdc_rsync/base/message_pump.h"
+#include "common/fake_socket.h"
 #include "common/log.h"
 #include "common/path.h"
 #include "common/status_test_macros.h"

cdc_rsync/base/message_pump.cc

@@ -16,9 +16,9 @@
 
 #include "absl/status/status.h"
 #include "absl/strings/str_format.h"
-#include "cdc_rsync/base/socket.h"
 #include "common/buffer.h"
 #include "common/log.h"
+#include "common/socket.h"
 #include "common/status.h"
 #include "google/protobuf/message_lite.h"
 

cdc_rsync/base/message_pump_test.cc

@@ -14,8 +14,8 @@
 
 #include "cdc_rsync/base/message_pump.h"
 
-#include "cdc_rsync/base/fake_socket.h"
 #include "cdc_rsync/protos/messages.pb.h"
+#include "common/fake_socket.h"
 #include "common/log.h"
 #include "common/status.h"
 #include "common/status_test_macros.h"

cdc_rsync/cdc_rsync.vcxproj

@@ -66,20 +66,21 @@
   </ItemDefinitionGroup>
   <!-- Bazel setup -->
   <PropertyGroup>
-    <BazelTargets>//cdc_rsync</BazelTargets>
+    <BazelTargets>//cdc_rsync //cdc_rsync_server</BazelTargets>
     <BazelOutputFile>cdc_rsync.exe</BazelOutputFile>
-    <BazelIncludePaths>..\;..\third_party\absl;..\bazel-cdc-file-transfer\external\com_github_blake3\c;..\bazel-stadia-file-transfer\external\com_github_zstd\lib;..\third_party\googletest\googletest\include;..\bazel-cdc-file-transfer\external\com_google_protobuf\src;$(VC_IncludePath);$(WindowsSDK_IncludePath)</BazelIncludePaths>
+    <BazelIncludePaths>..\;..\third_party\absl;..\bazel-cdc-file-transfer\external\com_github_blake3\c;..\bazel-cdc-file-transfer\external\com_github_zstd;..\third_party\googletest\googletest\include;..\bazel-cdc-file-transfer\external\com_google_protobuf\src;..\bazel-cdc-file-transfer\external\com_github_grpc_grpc\include;..\bazel-bin;$(VC_IncludePath);$(WindowsSDK_IncludePath)</BazelIncludePaths>
   </PropertyGroup>
   <Import Project="..\NMakeBazelProject.targets" />
   <!-- For some reason, msbuild doesn't include this file, so copy it explicitly. -->
-  <!-- TODO: Reenable once we can cross-compile these.
+  <!-- TODO: Reenable copying the Linux file once we can cross-compile these. -->
   <PropertyGroup>
-    <CdcRsyncServerFile>$(SolutionDir)bazel-out\k8-$(BazelCompilationMode)\bin\cdc_rsync_server\cdc_rsync_server</CdcRsyncServerFile>
+    <!-- <CdcRsyncLinuxServerFile>$(SolutionDir)bazel-out\k8-$(BazelCompilationMode)\bin\cdc_rsync_server\cdc_rsync_server</CdcRsyncLinuxServerFile> -->
+    <CdcRsyncWindowsServerFile>$(SolutionDir)bazel-out\x64_windows-$(BazelCompilationMode)\bin\cdc_rsync_server\cdc_rsync_server.exe</CdcRsyncWindowsServerFile>
   </PropertyGroup>
-  <Target Name="CopyServer" Inputs="$(CdcRsyncServerFile)" Outputs="$(OutDir)cdc_rsync_server" AfterTargets="Build">
+  <Target Name="CopyServer" Inputs="$(CdcRsyncLinuxServerFile);$(CdcRsyncWindowsServerFile)" Outputs="$(OutDir)cdc_rsync_server;$(OutDir)cdc_rsync_server.exe" AfterTargets="Build">
-    <Copy SourceFiles="$(CdcRsyncServerFile)" DestinationFiles="$(OutDir)cdc_rsync_server" />
+    <!-- <Copy SourceFiles="$(CdcRsyncLinuxServerFile)" DestinationFiles="$(OutDir)cdc_rsync_server" SkipUnchangedFiles="true" /> -->
+    <Copy SourceFiles="$(CdcRsyncWindowsServerFile)" DestinationFiles="$(OutDir)cdc_rsync_server.exe" SkipUnchangedFiles="true" />
   </Target>
-  -->
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>

cdc_rsync/cdc_rsync_client.cc

@@ -20,16 +20,19 @@
 #include "cdc_rsync/base/message_pump.h"
 #include "cdc_rsync/base/server_exit_code.h"
 #include "cdc_rsync/client_file_info.h"
-#include "cdc_rsync/client_socket.h"
 #include "cdc_rsync/file_finder_and_sender.h"
 #include "cdc_rsync/parallel_file_opener.h"
 #include "cdc_rsync/progress_tracker.h"
 #include "cdc_rsync/protos/messages.pb.h"
+#include "cdc_rsync/server_arch.h"
 #include "cdc_rsync/zstd_stream.h"
+#include "common/client_socket.h"
 #include "common/gamelet_component.h"
 #include "common/log.h"
 #include "common/path.h"
 #include "common/process.h"
+#include "common/remote_util.h"
+#include "common/server_socket.h"
 #include "common/status.h"
 #include "common/status_macros.h"
 #include "common/stopwatch.h"
@@ -44,9 +47,6 @@ constexpr int kExitCodeCouldNotExecute = 126;
 // Bash exit code if binary was not found.
 constexpr int kExitCodeNotFound = 127;
 
-constexpr char kCdcServerFilename[] = "cdc_rsync_server";
-constexpr char kRemoteToolsBinDir[] = "~/.cache/cdc-file-transfer/bin/";
-
 SetOptionsRequest::FilterRule::Type ToProtoType(PathFilter::Rule::Type type) {
   switch (type) {
     case PathFilter::Rule::Type::kInclude:
@@ -98,19 +98,20 @@ CdcRsyncClient::CdcRsyncClient(const Options& options,
     : options_(options),
       sources_(std::move(sources)),
       destination_(std::move(destination)),
-      remote_util_(std::move(user_host), options.verbosity, options.quiet,
-                   &process_factory_,
-                   /*forward_output_to_log=*/false),
-      port_manager_("cdc_rsync_ports_f77bcdfe-368c-4c45-9f01-230c5e7e2132",
-                    options.forward_port_first, options.forward_port_last,
-                    &process_factory_, &remote_util_),
       printer_(options.quiet, Util::IsTTY() && !options.json),
       progress_(&printer_, options.verbosity, options.json) {
+  // If there is no |user_host|, we sync files locally!
+  if (!user_host.empty()) {
+    remote_util_ =
+        std::make_unique<RemoteUtil>(std::move(user_host), options.verbosity,
+                                     options.quiet, &process_factory_,
+                                     /*forward_output_to_log=*/false);
     if (!options_.ssh_command.empty()) {
-    remote_util_.SetSshCommand(options_.ssh_command);
+      remote_util_->SetSshCommand(options_.ssh_command);
+    }
+    if (!options_.sftp_command.empty()) {
+      remote_util_->SetSftpCommand(options_.sftp_command);
     }
-  if (!options_.scp_command.empty()) {
-    remote_util_.SetScpCommand(options_.scp_command);
   }
 }
 
@@ -120,16 +121,42 @@ CdcRsyncClient::~CdcRsyncClient() {
 }
 
 absl::Status CdcRsyncClient::Run() {
+  // For local syncs, cdc_rsync_server runs on this machine. For remote syncs,
+  // guess the architecture of the device that runs cdc_rsync_server from the
+  // destination path, e.g. "C:\path\to\dest" strongly indicates Windows.
+  ServerArch server_arch = IsRemoteConnection()
+                               ? ServerArch::GuessFromDestination(destination_)
+                               : ServerArch::DetectFromLocalDevice();
+
   // Start the server process.
-  absl::Status status = StartServer();
+  absl::Status status = StartServer(server_arch);
+  if (HasTag(status, Tag::kDeployServer) && server_arch.IsGuess() &&
+      server_exit_code_ != kServerExitCodeOutOfDate) {
+    // Server couldn't be run, e.g. not found or failed to start.
+    // Check whether we guessed the arch type wrong and try again.
+    // Note that in case of a local sync, or if the server actively reported
+    // that it's out-dated, there's no need to detect the arch.
+    LOG_DEBUG(
+        "Failed to start server, retrying after detecting remote arch: %s",
+        status.ToString());
+    const ArchType old_type = server_arch.GetType();
+    ASSIGN_OR_RETURN(server_arch,
+                     ServerArch::DetectFromRemoteDevice(remote_util_.get()));
+    if (server_arch.GetType() != old_type) {
+      LOG_DEBUG("Guessed server arch type wrong, guessed %s, actual %s.",
+                GetArchTypeStr(old_type), server_arch.GetTypeStr());
+      status = StartServer(server_arch);
+    }
+  }
+
   if (HasTag(status, Tag::kDeployServer)) {
     // Gamelet components are not deployed or out-dated. Deploy and retry.
-    status = DeployServer();
+    status = DeployServer(server_arch);
     if (!status.ok()) {
       return WrapStatus(status, "Failed to deploy server");
     }
 
-    status = StartServer();
+    status = StartServer(server_arch);
   }
   if (!status.ok()) {
     return WrapStatus(status, "Failed to start server");
@@ -161,7 +188,7 @@ absl::Status CdcRsyncClient::Run() {
   return status;
 }
 
-absl::Status CdcRsyncClient::StartServer() {
+absl::Status CdcRsyncClient::StartServer(const ServerArch& arch) {
   assert(!server_process_);
 
   // Components are expected to reside in the same dir as the executable.
@@ -173,55 +200,41 @@ absl::Status CdcRsyncClient::StartServer() {
 
   std::vector<GameletComponent> components;
   status = GameletComponent::Get(
-      {path::Join(component_dir, kCdcServerFilename)}, &components);
+      {path::Join(component_dir, arch.CdcServerFilename())}, &components);
   if (!status.ok()) {
     return MakeStatus(
         "Required instance component not found. Make sure the file "
-        "cdc_rsync_server resides in the same folder as cdc_rsync.exe.");
+        "%s resides in the same folder as %s.",
+        arch.CdcServerFilename(), ServerArch::CdcRsyncFilename());
   }
   std::string component_args = GameletComponent::ToCommandLineArgs(components);
 
-  // Find available local and remote ports for port forwarding.
+  ProcessStartInfo start_info;
-  // If only one port is in the given range, try that without checking.
-  int port = options_.forward_port_first;
-  if (options_.forward_port_first < options_.forward_port_last) {
-    absl::StatusOr<int> port_res =
-        port_manager_.ReservePort(options_.connection_timeout_sec);
-    constexpr char kErrorMsg[] = "Failed to find available port";
-    if (absl::IsDeadlineExceeded(port_res.status())) {
-      // Server didn't respond in time.
-      return SetTag(WrapStatus(port_res.status(), kErrorMsg),
-                    Tag::kConnectionTimeout);
-    }
-    if (absl::IsResourceExhausted(port_res.status()))
-      return SetTag(WrapStatus(port_res.status(), kErrorMsg),
-                    Tag::kAddressInUse);
-    if (!port_res.ok())
-      return WrapStatus(port_res.status(), "Failed to find available port");
-    port = *port_res;
-  }
-
-  std::string remote_server_path =
-      std::string(kRemoteToolsBinDir) + kCdcServerFilename;
-  // Test existence manually to prevent misleading bash output message
-  // "bash: .../cdc_rsync_server: No such file or directory".
-  // Also create the bin dir because otherwise scp below might fail.
-  std::string remote_command =
-      absl::StrFormat("mkdir -p %s; if [ ! -f %s ]; then exit %i; fi; %s %i %s",
-                      kRemoteToolsBinDir, remote_server_path, kExitCodeNotFound,
-                      remote_server_path, port, component_args);
-  ProcessStartInfo start_info =
-      remote_util_.BuildProcessStartInfoForSshPortForwardAndCommand(
-          port, port, false, remote_command);
   start_info.name = "cdc_rsync_server";
 
+  if (IsRemoteConnection()) {
+    // Run cdc_rsync_server on the remote instance.
+    std::string remote_command =
+        arch.GetStartServerCommand(kExitCodeNotFound, component_args);
+    assert(remote_util_);
+    start_info = remote_util_->BuildProcessStartInfoForSsh(remote_command,
+                                                           arch.GetType());
+  } else {
+    // Run cdc_rsync_server locally.
+    std::string exe_dir;
+    RETURN_IF_ERROR(path::GetExeDir(&exe_dir), "Failed to get exe directory");
+
+    std::string server_path = path::Join(exe_dir, arch.CdcServerFilename());
+    start_info.command = absl::StrFormat("%s %s", server_path, component_args);
+  }
+
   // Capture stdout, but forward to stdout for debugging purposes.
   start_info.stdout_handler = [this](const char* data, size_t /*data_size*/) {
     return HandleServerOutput(data);
   };
 
-  std::unique_ptr<Process> process = process_factory_.Create(start_info);
+  std::unique_ptr<Process> srv_process = process_factory_.Create(start_info);
-  status = process->Start();
+  status = srv_process->Start();
   if (!status.ok()) {
     return WrapStatus(status, "Failed to start cdc_rsync_server process");
   }
@@ -229,17 +242,17 @@ absl::Status CdcRsyncClient::StartServer() {
   // Wait until the server process is listening.
   Stopwatch timeout_timer;
   bool is_timeout = false;
-  auto detect_listening_or_timeout = [is_listening = &is_server_listening_,
+  auto detect_listening_or_timeout = [port = &server_listen_port_,
                                       timeout = options_.connection_timeout_sec,
                                       &timeout_timer, &is_timeout]() -> bool {
     is_timeout = timeout_timer.ElapsedSeconds() > timeout;
-    return *is_listening || is_timeout;
+    return *port != 0 || is_timeout;
   };
-  status = process->RunUntil(detect_listening_or_timeout);
+  status = srv_process->RunUntil(detect_listening_or_timeout);
   if (!status.ok()) {
     // Some internal process error. Note that this does NOT mean that
     // cdc_rsync_server does not exist. In that case, the ssh process exits with
-    // code 127.
+    // code kExitCodeNotFound.
     return status;
   }
   if (is_timeout) {
@@ -247,15 +260,21 @@ absl::Status CdcRsyncClient::StartServer() {
                   Tag::kConnectionTimeout);
   }
 
-  if (process->HasExited()) {
+  if (srv_process->HasExited()) {
     // Don't re-deploy for code > kServerExitCodeOutOfDate, which means that the
     // out-of-date check already passed on the server.
-    server_exit_code_ = process->ExitCode();
+    server_exit_code_ = srv_process->ExitCode();
     if (server_exit_code_ > kServerExitCodeOutOfDate &&
         server_exit_code_ <= kServerExitCodeMax) {
       return GetServerExitStatus(server_exit_code_, server_error_);
     }
 
+    // Don't re-deploy if we're not copying to a remote device. We can start
+    // cdc_rsync_server from the original location directly.
+    if (!IsRemoteConnection()) {
+      return GetServerExitStatus(server_exit_code_, server_error_);
+    }
+
     // Server exited before it started listening, most likely because of
     // outdated components (code kServerExitCodeOutOfDate) or because the server
     // wasn't deployed at all yet (code kExitCodeNotFound). Instruct caller
@@ -263,19 +282,30 @@ absl::Status CdcRsyncClient::StartServer() {
     return SetTag(MakeStatus("Redeploy server"), Tag::kDeployServer);
   }
 
-  status = Socket::Initialize();
+  // Start up sockets.
-  if (!status.ok()) {
+  RETURN_IF_ERROR(Socket::Initialize(), "Failed to initialize sockets");
-    return WrapStatus(status, "Failed to initialize sockets");
-  }
   socket_finalizer_ = std::make_unique<SocketFinalizer>();
 
-  assert(is_server_listening_);
+  // Now that we know which port the server is using, set up port forwarding.
-  status = socket_.Connect(port);
+  std::unique_ptr<Process> fwd_process;
-  if (!status.ok()) {
+  int local_port = server_listen_port_;
-    return WrapStatus(status, "Failed to initialize connection");
+  if (IsRemoteConnection()) {
+    ASSIGN_OR_RETURN(local_port, ServerSocket::FindAvailablePort());
+    ProcessStartInfo start_info =
+        remote_util_->BuildProcessStartInfoForSshPortForward(
+            local_port, server_listen_port_, /*reverse=*/false);
+    start_info.forward_output_to_log = true;
+    fwd_process = process_factory_.Create(start_info);
+    RETURN_IF_ERROR(fwd_process->Start(),
+                    "Failed to start cdc_rsync_server process");
   }
 
-  server_process_ = std::move(process);
+  // Wait for port forwarding to be up.
+  RETURN_IF_ERROR(ClientSocket::WaitForConnection(
+      local_port, absl::Seconds(options_.connection_timeout_sec)));
+
+  server_process_ = std::move(srv_process);
+  port_forwarding_process_ = std::move(fwd_process);
   message_pump_.StartMessagePump();
   return absl::OkStatus();
 }
@@ -296,6 +326,7 @@ absl::Status CdcRsyncClient::StopServer() {
 
   server_exit_code_ = server_process_->ExitCode();
   server_process_.reset();
+  port_forwarding_process_.reset();
   return absl::OkStatus();
 }
 
@@ -327,10 +358,29 @@ absl::Status CdcRsyncClient::HandleServerOutput(const char* data) {
   }
 
   printer_.Print(stdout_data, false, Util::GetConsoleWidth());
-  if (!is_server_listening_) {
+  if (server_listen_port_ == 0) {
     server_output_.append(stdout_data);
-    is_server_listening_ =
+
-        server_output_.find("Server is listening") != std::string::npos;
+    // Parse port from "Port <n>: Server is listening".
+    size_t listening_pos = server_output_.find("Server is listening");
+    if (listening_pos != std::string::npos) {
+      // Search backwards until we find "Port ".
+      constexpr char port_key[] = "Port ";
+      size_t port_pos = server_output_.rfind(port_key, listening_pos);
+      if (port_pos == std::string::npos) {
+        return MakeStatus("Failed to find 'Port' marker in server output '%s'",
+                          server_output_);
+      }
+      assert(listening_pos > port_pos);
+      server_listen_port_ = atoi(
+          server_output_
+              .substr(port_pos + strlen(port_key), listening_pos - port_pos)
+              .c_str());
+      if (server_listen_port_ == 0) {
+        return MakeStatus("Failed to parse port from server output '%s'",
+                          server_output_);
+      }
+    }
   }
 
   return absl::OkStatus();
@@ -394,8 +444,10 @@ absl::Status CdcRsyncClient::Sync() {
   return status;
 }
 
-absl::Status CdcRsyncClient::DeployServer() {
+absl::Status CdcRsyncClient::DeployServer(const ServerArch& arch) {
   assert(!server_process_);
+  assert(remote_util_);
+  assert(IsRemoteConnection());
 
   std::string exe_dir;
   absl::Status status = path::GetExeDir(&exe_dir);
@@ -415,32 +467,10 @@ absl::Status CdcRsyncClient::DeployServer() {
   }
   printer_.Print(deploy_msg, true, Util::GetConsoleWidth());
 
-  // scp cdc_rsync_server to a temp location on the gamelet.
+  // sftp cdc_rsync_server to the target.
-  std::string remoteServerTmpPath =
+  std::string commands = arch.GetDeploySftpCommands();
-      absl::StrFormat("%s%s.%s", kRemoteToolsBinDir, kCdcServerFilename,
+  RETURN_IF_ERROR(remote_util_->Sftp(commands, exe_dir, /*compress=*/false),
-                      Util::GenerateUniqueId());
+                  "Failed to deploy cdc_rsync_server");
-  std::string localServerPath = path::Join(exe_dir, kCdcServerFilename);
-  status = remote_util_.Scp({localServerPath}, remoteServerTmpPath,
-                            /*compress=*/true);
-  if (!status.ok()) {
-    return WrapStatus(status, "Failed to copy cdc_rsync_server to instance");
-  }
-
-  // Do 3 things in one SSH command, to save time:
-  // - Make the old cdc_rsync_server writable (if it exists).
-  // - Make the new cdc_rsync_server executable.
-  // - Replace the old cdc_rsync_server by the new one.
-  std::string old_path = RemoteUtil::QuoteForWindows(
-      std::string(kRemoteToolsBinDir) + kCdcServerFilename);
-  std::string new_path = RemoteUtil::QuoteForWindows(remoteServerTmpPath);
-  std::string replace_cmd = absl::StrFormat(
-      " ([ ! -f %s ] || chmod u+w %s) && chmod a+x %s && mv %s %s", old_path,
-      old_path, new_path, new_path, old_path);
-  status = remote_util_.Run(replace_cmd, "chmod && chmod && mv");
-  if (!status.ok()) {
-    return WrapStatus(status,
-                      "Failed to replace old cdc_rsync_server by new one");
-  }
 
   return absl::OkStatus();
 }
@@ -617,7 +647,7 @@ absl::Status CdcRsyncClient::SendMissingFiles() {
 
   ParallelFileOpener file_opener(&files_, missing_file_indices_);
 
-  constexpr size_t kBufferSize = 16000;
+  constexpr size_t kBufferSize = 128 * 1024;
   for (uint32_t server_index = 0; server_index < missing_file_indices_.size();
        ++server_index) {
     uint32_t client_index = missing_file_indices_[server_index];
@@ -779,9 +809,9 @@ absl::Status CdcRsyncClient::StopCompressionStream() {
   message_pump_.FlushOutgoingQueue();
   message_pump_.RedirectOutput(nullptr);
 
-  // Flush compression stream and reset.
+  // Finish compression stream and reset.
-  RETURN_IF_ERROR(compression_stream_->Flush(),
+  RETURN_IF_ERROR(compression_stream_->Finish(),
-                  "Failed to flush compression stream");
+                  "Failed to finish compression stream");
   compression_stream_.reset();
 
   // Wait for the server ack. This must be done before sending more data.

cdc_rsync/cdc_rsync_client.h

@@ -21,16 +21,18 @@
 #include <vector>
 
 #include "absl/status/status.h"
+#include "absl/status/statusor.h"
 #include "cdc_rsync/base/message_pump.h"
-#include "cdc_rsync/client_socket.h"
 #include "cdc_rsync/progress_tracker.h"
+#include "common/client_socket.h"
 #include "common/path_filter.h"
-#include "common/port_manager.h"
+#include "common/process.h"
-#include "common/remote_util.h"
 
 namespace cdc_ft {
 
 class Process;
+class RemoteUtil;
+class ServerArch;
 class ZstdStream;
 
 class CdcRsyncClient {
@@ -50,13 +52,15 @@ class CdcRsyncClient {
     std::string copy_dest;
     int compress_level = 6;
     int connection_timeout_sec = 10;
-    int forward_port_first = 44450;
-    int forward_port_last = 44459;
     std::string ssh_command;
-    std::string scp_command;
+    std::string sftp_command;
     std::string sources_dir;  // Base dir for files loaded for --files-from.
     PathFilter filter;
 
+    // Backwards compatibility for switching from scp to sftp.
+    // Used internally, do not use.
+    std::string deprecated_scp_command;
+
     // Compression level 0 is invalid.
     static constexpr int kMinCompressLevel = -5;
     static constexpr int kMaxCompressLevel = 22;
@@ -73,7 +77,7 @@ class CdcRsyncClient {
  private:
   // Starts the server process. If the method returns a status with tag
   // |kTagDeployServer|, Run() calls DeployServer() and tries again.
-  absl::Status StartServer();
+  absl::Status StartServer(const ServerArch& arch);
 
   // Stops the server process.
   absl::Status StopServer();
@@ -85,7 +89,7 @@ class CdcRsyncClient {
   absl::Status Sync();
 
   // Copies all gamelet components to the gamelet.
-  absl::Status DeployServer();
+  absl::Status DeployServer(const ServerArch& arch);
 
   // Sends relevant options to the server.
   absl::Status SendOptions();
@@ -117,12 +121,14 @@ class CdcRsyncClient {
   // Stops the zstd compression stream.
   absl::Status StopCompressionStream();
 
+  // Returns true if the target is a remote target.
+  bool IsRemoteConnection() const { return remote_util_ != nullptr; }
+
   Options options_;
   std::vector<std::string> sources_;
   const std::string destination_;
   WinProcessFactory process_factory_;
-  RemoteUtil remote_util_;
+  std::unique_ptr<RemoteUtil> remote_util_;
-  PortManager port_manager_;
   std::unique_ptr<SocketFinalizer> socket_finalizer_;
   ClientSocket socket_;
   MessagePump message_pump_{&socket_, MessagePump::PacketReceivedDelegate()};
@@ -131,10 +137,11 @@ class CdcRsyncClient {
   std::unique_ptr<ZstdStream> compression_stream_;
 
   std::unique_ptr<Process> server_process_;
+  std::unique_ptr<Process> port_forwarding_process_;
   std::string server_output_;  // Written in a background thread. Do not access
   std::string server_error_;   // while the server process is active.
   int server_exit_code_ = 0;
-  std::atomic_bool is_server_listening_{false};
+  std::atomic_int server_listen_port_{0};
   bool is_server_error_ = false;
 
   // All source files found on the client.

cdc_rsync/file_finder_and_sender_test.cc

@@ -18,8 +18,8 @@
 
 #include "absl/strings/str_format.h"
 #include "absl/strings/str_join.h"
-#include "cdc_rsync/base/fake_socket.h"
 #include "cdc_rsync/base/message_pump.h"
+#include "common/fake_socket.h"
 #include "common/log.h"
 #include "common/path.h"
 #include "common/path_filter.h"

cdc_rsync/main.cc

@@ -68,8 +68,8 @@ ReturnCode TagToMessage(cdc_ft::Tag tag,
 
     case cdc_ft::Tag::kDeployServer:
       *msg =
-          "Failed to deploy the instance components for unknown reasons. "
+          "Failed to deploy or run the instance components for unknown "
-          "Please report this issue.";
+          "reasons. Please report this issue.";
       return ReturnCode::kDeployFailed;
 
     case cdc_ft::Tag::kConnectionTimeout:

cdc_rsync/params.cc

@@ -19,8 +19,10 @@
 #include "absl/status/status.h"
 #include "absl/strings/str_format.h"
 #include "absl/strings/str_split.h"
+#include "common/build_version.h"
 #include "common/path.h"
 #include "common/port_range_parser.h"
+#include "common/remote_util.h"
 #include "lib/zstd.h"
 
 namespace cdc_ft {
@@ -37,22 +39,22 @@ void PrintError(const absl::FormatSpec<Args...>& format, Args... args) {
 enum class OptionResult { kConsumedKey, kConsumedKeyValue, kError };
 
 const char kHelpText[] =
-    R"(Copy local files to a gamelet
+    R"(
-
+Matching files are skipped based on file size and modified time. For partially
-Synchronizes local files and files on a gamelet. Matching files are skipped.
+matching files only the differences are transferred. The destination directory
-For partially matching files only the deltas are transferred.
+can be the same Windows machine or a remote Windows or Linux device.
 
 Usage:
-  cdc_rsync [options] source [source]... [user@]host:destination
+  cdc_rsync [options] source [source]... [[user@]host:]destination
 
 Parameters:
-  source                    Local file or directory to be copied
+  source                    Local file or directory to be copied or synced
   user                      Remote SSH user name
   host                      Remote host or IP address
-  destination               Remote destination directory
+  destination               Local or remote destination directory
 
 Options:
-    --contimeout sec        Gamelet connection timeout in seconds (default: 10)
+    --contimeout sec        Remote connection timeout in seconds (default: 10)
 -q, --quiet                 Quiet mode, only print errors
 -v, --verbose               Increase output verbosity
     --json                  Print JSON progress
@@ -75,22 +77,23 @@ Options:
     --ssh-command <cmd>     Path and arguments of ssh command to use, e.g.
                             "C:\path\to\ssh.exe -p 12345 -i id_rsa -oUserKnownHostsFile=known_hosts"
                             Can also be specified by the CDC_SSH_COMMAND environment variable.
-    --scp-command <cmd>     Path and arguments of scp command to use, e.g.
+    --sftp-command <cmd>    Path and arguments of sftp command to use, e.g.
-                            "C:\path\to\scp.exe -P 12345 -i id_rsa -oUserKnownHostsFile=known_hosts"
+                            "C:\path\to\sftp.exe -P 12345 -i id_rsa -oUserKnownHostsFile=known_hosts"
-                            Can also be specified by the CDC_SCP_COMMAND environment variable.
+                            Can also be specified by the CDC_SFTP_COMMAND environment variable.
-    --forward-port <port>   TCP port or range used for SSH port forwarding (default: 44450-44459).
+-h, --help                  Help for cdc_rsync
-                            If a range is specified, searches for available ports (slower).
--h  --help                  Help for cdc_rsync
 )";
 
 constexpr char kSshCommandEnvVar[] = "CDC_SSH_COMMAND";
 constexpr char kScpCommandEnvVar[] = "CDC_SCP_COMMAND";
+constexpr char kSftpCommandEnvVar[] = "CDC_SFTP_COMMAND";
 
 // Populates some parameters from environment variables.
 void PopulateFromEnvVars(Parameters* parameters) {
   path::GetEnv(kSshCommandEnvVar, &parameters->options.ssh_command)
       .IgnoreError();
-  path::GetEnv(kScpCommandEnvVar, &parameters->options.scp_command)
+  path::GetEnv(kScpCommandEnvVar, &parameters->options.deprecated_scp_command)
+      .IgnoreError();
+  path::GetEnv(kSftpCommandEnvVar, &parameters->options.sftp_command)
       .IgnoreError();
 }
 
@@ -287,21 +290,22 @@ OptionResult HandleParameter(const std::string& key, const char* value,
   }
 
   if (key == "scp-command") {
+    // Backwards compatibility. Note that this flag is hidden from the help.
     if (!ValidateValue(key, value)) return OptionResult::kError;
-    params->options.scp_command = value;
+    params->options.deprecated_scp_command = value;
+    return OptionResult::kConsumedKeyValue;
+  }
+
+  if (key == "sftp-command") {
+    if (!ValidateValue(key, value)) return OptionResult::kError;
+    params->options.sftp_command = value;
     return OptionResult::kConsumedKeyValue;
   }
 
   if (key == "forward-port") {
-    if (!ValidateValue(key, value)) return OptionResult::kError;
+    // This param is no longer needed. Just print a warning for backwards
-    uint16_t first, last;
+    // compatibility.
-    if (!port_range::Parse(value, &first, &last)) {
+    std::cout << "--forward-port argument no longer needed" << std::endl;
-      PrintError("Failed to parse %s=%s, expected <port> or <port1>-<port2>",
-                 key, value);
-      return OptionResult::kError;
-    }
-    params->options.forward_port_first = first;
-    params->options.forward_port_last = last;
     return OptionResult::kConsumedKeyValue;
   }
 
@@ -311,6 +315,8 @@ OptionResult HandleParameter(const std::string& key, const char* value,
 
 bool ValidateParameters(const Parameters& params, bool help) {
   if (help) {
+    std::cout << "cdc_rsync - Synchronize files and directories. Version: "
+              << BUILD_VERSION << "\n";
     std::cout << kHelpText;
     return false;
   }
@@ -364,14 +370,6 @@ bool ValidateParameters(const Parameters& params, bool help) {
     return false;
   }
 
-  if (params.user_host.empty()) {
-    PrintError(
-        "No remote host specified in destination '%s'. "
-        "Expected [user@]host:destination.",
-        params.destination);
-    return false;
-  }
-
   return true;
 }
 
@@ -397,16 +395,15 @@ bool CheckOptionResult(OptionResult result, const std::string& name,
 // afterward and |user_host| is |user@foo.com|. Does not touch Windows drives,
 // e.g. C:\foo.
 void PopUserHost(std::string* destination, std::string* user_host) {
+  user_host->clear();
+
+  // Don't mistake the C part of C:\foo or \\share\C:\foo as user/host.
+  if (!path::GetDrivePrefix(*destination).empty()) return;
+
   std::vector<std::string> parts =
       absl::StrSplit(*destination, absl::MaxSplits(':', 1));
   if (parts.size() < 2) return;
 
-  // Don't mistake the C part of C:\foo as user/host.
-  if (parts[0].size() == 1 && toupper(parts[0][0]) >= 'A' &&
-      toupper(parts[0][0]) <= 'Z') {
-    return;
-  }
-
   *user_host = parts[0];
   *destination = parts[1];
 }
@@ -491,6 +488,27 @@ bool Parse(int argc, const char* const* argv, Parameters* parameters) {
 
   PopUserHost(&parameters->destination, &parameters->user_host);
 
+  // Backwards compabitility after switching to sftp. Convert scp to sftp
+  // command Note that this flag is hidden from the help.
+  if (parameters->options.sftp_command.empty() &&
+      !parameters->options.deprecated_scp_command.empty()) {
+    LOG_WARNING(
+        "The CDC_SCP_COMMAND environment variable and the --scp-command flag "
+        "are deprecated. Please set CDC_SFTP_COMMAND or --sftp-command "
+        "instead.");
+
+    parameters->options.sftp_command = RemoteUtil::ScpToSftpCommand(
+        parameters->options.deprecated_scp_command);
+    if (!parameters->options.sftp_command.empty()) {
+      LOG_WARNING("Converted scp command '%s' to sftp command '%s'.",
+                  parameters->options.deprecated_scp_command,
+                  parameters->options.sftp_command);
+    } else {
+      LOG_WARNING("Failed to convert scp command '%s' to sftp command.",
+                  parameters->options.deprecated_scp_command);
+    }
+  }
+
   if (!ValidateParameters(*parameters, help)) {
     return false;
   }

cdc_rsync/params_test.cc

@@ -32,6 +32,10 @@ constexpr char kUserHostDst[] = "user@host:destination";
 constexpr char kUserHost[] = "user@host";
 constexpr char kDst[] = "destination";
 
+constexpr char kSshCommandEnvVar[] = "CDC_SSH_COMMAND";
+constexpr char kScpCommandEnvVar[] = "CDC_SCP_COMMAND";
+constexpr char kSftpCommandEnvVar[] = "CDC_SFTP_COMMAND";
+
 class TestLog : public Log {
  public:
   explicit TestLog() : Log(LogLevel::kInfo) {}
@@ -60,6 +64,11 @@ class ParamsTest : public ::testing::Test {
   void TearDown() override {
     std::cout.rdbuf(prev_stdout_);
     std::cerr.rdbuf(prev_stderr_);
+
+    // Clear env. They seem to be sticky sometimes and leak into other tests.
+    path::SetEnv(kSshCommandEnvVar, "");
+    path::SetEnv(kScpCommandEnvVar, "");
+    path::SetEnv(kSftpCommandEnvVar, "");
   }
 
  protected:
@@ -152,24 +161,57 @@ TEST_F(ParamsTest, ParseFailsOnContimeoutEqualsNoValue) {
   ExpectError(NeedsValueError("contimeout"));
 }
 
-TEST_F(ParamsTest, ParseSucceedsWithSshScpCommands) {
+TEST_F(ParamsTest, ParseSucceedsWithSshSftpCommands) {
-  const char* argv[] = {"cdc_rsync.exe",        kSrc,
+  const char* argv[] = {
-                        kUserHostDst,           "--ssh-command=sshcmd",
+      "cdc_rsync.exe",          kSrc, kUserHostDst, "--ssh-command=sshcmd",
-                        "--scp-command=scpcmd", NULL};
+      "--sftp-command=sftpcmd", NULL};
   EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
-  EXPECT_EQ(parameters_.options.scp_command, "scpcmd");
+  EXPECT_EQ(parameters_.options.sftp_command, "sftpcmd");
   EXPECT_EQ(parameters_.options.ssh_command, "sshcmd");
 }
 
-TEST_F(ParamsTest, ParseSucceedsWithSshScpCommandsByEnvVars) {
+TEST_F(ParamsTest, ParseSucceedsWithSshSftpCommandsByEnvVars) {
-  EXPECT_OK(path::SetEnv("CDC_SSH_COMMAND", "sshcmd"));
+  EXPECT_OK(path::SetEnv(kSshCommandEnvVar, "sshcmd"));
-  EXPECT_OK(path::SetEnv("CDC_SCP_COMMAND", "scpcmd"));
+  EXPECT_OK(path::SetEnv(kSftpCommandEnvVar, "sftpcmd"));
   const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst, NULL};
   EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
-  EXPECT_EQ(parameters_.options.scp_command, "scpcmd");
+  EXPECT_EQ(parameters_.options.sftp_command, "sftpcmd");
   EXPECT_EQ(parameters_.options.ssh_command, "sshcmd");
 }
 
+TEST_F(ParamsTest, ParseSucceedsWithScpCommandFallback) {
+  const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst,
+                        "--scp-command=C:\\scp.exe foo", NULL};
+  EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
+  EXPECT_EQ(parameters_.options.sftp_command, "C:\\sftp.exe foo");
+}
+
+TEST_F(ParamsTest, ParseSucceedsWithScpCommandFallbackByEnvVar) {
+  EXPECT_OK(path::SetEnv(kScpCommandEnvVar, "C:\\scp.exe foo"));
+  const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst, NULL};
+  EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
+  EXPECT_EQ(parameters_.options.sftp_command, "C:\\sftp.exe foo");
+}
+
+TEST_F(ParamsTest, ParseSucceedsWithSftpOverwritingScp) {
+  const char* argv[] = {"cdc_rsync.exe",
+                        kSrc,
+                        kUserHostDst,
+                        "--scp-command=C:\\scp.exe foo",
+                        "--sftp-command=sftpcmd",
+                        NULL};
+  EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
+  EXPECT_EQ(parameters_.options.sftp_command, "sftpcmd");
+}
+
+TEST_F(ParamsTest, ParseSucceedsWithSftpEnvVarOverwritingScp) {
+  EXPECT_OK(path::SetEnv(kSftpCommandEnvVar, "sftpcmd"));
+  const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst,
+                        "--scp-command=C:\\scp.exe foo", NULL};
+  EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
+  EXPECT_EQ(parameters_.options.sftp_command, "sftpcmd");
+}
+
 TEST_F(ParamsTest, ParseSucceedsWithNoSshCommand) {
   const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst,
                         "--ssh-command=", NULL};
@@ -178,26 +220,33 @@ TEST_F(ParamsTest, ParseSucceedsWithNoSshCommand) {
   ExpectError(NeedsValueError("ssh-command"));
 }
 
-TEST_F(ParamsTest, ParseSucceedsWithNoScpCommand) {
+TEST_F(ParamsTest, ParseSucceedsWithNoSftpCommand) {
-  const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst, "--scp-command",
+  const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst, "--sftp-command",
                         NULL};
   EXPECT_FALSE(
       Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
-  ExpectError(NeedsValueError("scp-command"));
+  ExpectError(NeedsValueError("sftp-command"));
 }
 
-TEST_F(ParamsTest, ParseFailsOnNoUserHost) {
+TEST_F(ParamsTest, ParseSucceedsOnNoUserHost) {
   const char* argv[] = {"cdc_rsync.exe", kSrc, kDst, NULL};
-  EXPECT_FALSE(
+  EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
-      Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
-  ExpectError("No remote host specified");
 }
 
-TEST_F(ParamsTest, ParseDoesNotThinkCIsAHost) {
+TEST_F(ParamsTest, ParseDoesNotThinkDriveIsAHost) {
   const char* argv[] = {"cdc_rsync.exe", kSrc, "C:\\foo", NULL};
-  EXPECT_FALSE(
+  EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
-      Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
+  EXPECT_TRUE(parameters_.user_host.empty());
-  ExpectError("No remote host specified");
+
+  const char* argv2[] = {"cdc_rsync.exe", kSrc, "\\\\.\\C:\\foo", NULL};
+  EXPECT_TRUE(
+      Parse(static_cast<int>(std::size(argv2)) - 1, argv, &parameters_));
+  EXPECT_TRUE(parameters_.user_host.empty());
+
+  const char* argv3[] = {"cdc_rsync.exe", kSrc, "\\\\?\\C:\\foo", NULL};
+  EXPECT_TRUE(
+      Parse(static_cast<int>(std::size(argv3)) - 1, argv, &parameters_));
+  EXPECT_TRUE(parameters_.user_host.empty());
 }
 
 TEST_F(ParamsTest, ParseWithoutParametersFailsOnMissingSourceAndDestination) {
@@ -536,40 +585,6 @@ TEST_F(ParamsTest, IncludeExcludeMixed_ProperOrder) {
   ExpectNoError();
 }
 
-TEST_F(ParamsTest, ForwardPort_Single) {
-  const char* argv[] = {"cdc_rsync.exe", "--forward-port=65535", kSrc,
-                        kUserHostDst, NULL};
-  EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
-  EXPECT_EQ(parameters_.options.forward_port_first, 65535);
-  EXPECT_EQ(parameters_.options.forward_port_last, 65535);
-  ExpectNoError();
-}
-
-TEST_F(ParamsTest, ForwardPort_Range) {
-  const char* argv[] = {
-      "cdc_rsync.exe", "--forward-port", "1-2", kSrc, kUserHostDst, NULL};
-  EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
-  EXPECT_EQ(parameters_.options.forward_port_first, 1);
-  EXPECT_EQ(parameters_.options.forward_port_last, 2);
-  ExpectNoError();
-}
-
-TEST_F(ParamsTest, ForwardPort_NoValue) {
-  const char* argv[] = {"cdc_rsync.exe", "--forward-port=", kSrc, kUserHostDst,
-                        NULL};
-  EXPECT_FALSE(
-      Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
-  ExpectError(NeedsValueError("forward-port"));
-}
-
-TEST_F(ParamsTest, ForwardPort_BadValueTooSmall) {
-  const char* argv[] = {"cdc_rsync.exe", "--forward-port=0", kSrc, kUserHostDst,
-                        NULL};
-  EXPECT_FALSE(
-      Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
-  ExpectError("Failed to parse");
-}
-
 }  // namespace
 }  // namespace params
 }  // namespace cdc_ft

cdc_rsync/server_arch.cc

@@ -0,0 +1,271 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "cdc_rsync/server_arch.h"
+
+#include <filesystem>
+
+#include "absl/strings/match.h"
+#include "absl/strings/str_format.h"
+#include "absl/strings/str_split.h"
+#include "common/ansi_filter.h"
+#include "common/path.h"
+#include "common/remote_util.h"
+#include "common/status_macros.h"
+#include "common/util.h"
+
+namespace cdc_ft {
+namespace {
+
+constexpr char kErrorArchTypeUnhandled[] = "arch_type_unhandled";
+constexpr char kUnsupportedArchErrorFmt[] =
+    "Unsupported remote device architecture '%s'. If you think this is a "
+    "bug, or if this combination should be supported, please file a bug at "
+    "https://github.com/google/cdc-file-transfer.";
+
+absl::StatusOr<ArchType> GetArchTypeFromUname(const std::string& uname_out) {
+  // uname_out is "KERNEL MACHINE"
+  // Possible values for KERNEL: Linux (not sure what else).
+  // Possible values for MACHINE:
+  // https://stackoverflow.com/questions/45125516/possible-values-for-uname-m
+  // Relevant for us: x86_64, aarch64.
+  if (absl::StartsWith(uname_out, "Linux ")) {
+    // Linux kernel. Check CPU type.
+    if (absl::StrContains(uname_out, "x86_64")) {
+      return ArchType::kLinux_x86_64;
+    }
+  }
+
+  if (absl::StartsWith(uname_out, "MSYS_")) {
+    // Windows machine that happens to have Cygwin/MSYS on it. Check CPU type.
+    if (absl::StrContains(uname_out, "x86_64")) {
+      return ArchType::kWindows_x86_64;
+    }
+  }
+
+  return absl::UnimplementedError(
+      absl::StrFormat(kUnsupportedArchErrorFmt, uname_out));
+}
+
+absl::StatusOr<ArchType> GetArchTypeFromWinProcArch(
+    const std::string& arch_out) {
+  // Possible values: AMD64, IA64, ARM64, x86
+  if (absl::StrContains(arch_out, "AMD64")) {
+    return ArchType::kWindows_x86_64;
+  }
+
+  return absl::UnimplementedError(
+      absl::StrFormat(kUnsupportedArchErrorFmt, arch_out));
+}
+
+}  // namespace
+
+// static
+ServerArch ServerArch::GuessFromDestination(const std::string& destination) {
+  // Path starting with ~ or / -> Linux.
+  if (absl::StartsWith(destination, "~") ||
+      absl::StartsWith(destination, "/")) {
+    LOG_DEBUG("Guessed server arch type Linux based on ~ or /");
+    return ServerArch(ArchType::kLinux_x86_64, /*is_guess=*/true);
+  }
+
+  // Path starting with C: etc. -> Windows.
+  if (!path::GetDrivePrefix(destination).empty()) {
+    LOG_DEBUG("Guessed server arch type Windows based on drive prefix");
+    return ServerArch(ArchType::kWindows_x86_64, /*is_guess=*/true);
+  }
+
+  // Path with only / -> Linux.
+  if (absl::StrContains(destination, "/") &&
+      !absl::StrContains(destination, "\\")) {
+    LOG_DEBUG("Guessed server arch type Linux based on forward slashes");
+    return ServerArch(ArchType::kLinux_x86_64, /*is_guess=*/true);
+  }
+
+  // Path with only \\ -> Windows.
+  if (absl::StrContains(destination, "\\") &&
+      !absl::StrContains(destination, "/")) {
+    LOG_DEBUG("Guessed server arch type Windows based on backslashes");
+    return ServerArch(ArchType::kWindows_x86_64, /*is_guess=*/true);
+  }
+
+  // Default to Linux.
+  LOG_DEBUG("Guessed server arch type Linux as default");
+  return ServerArch(ArchType::kLinux_x86_64, /*is_guess=*/true);
+}
+
+// static
+ServerArch ServerArch::DetectFromLocalDevice() {
+  LOG_DEBUG("Detected local device type %s",
+            GetArchTypeStr(GetLocalArchType()));
+  return ServerArch(GetLocalArchType(), /*is_guess=*/false);
+}
+
+// static
+absl::StatusOr<ServerArch> ServerArch::DetectFromRemoteDevice(
+    RemoteUtil* remote_util) {
+  assert(remote_util);
+
+  // Run uname, assuming it's a Linux machine.
+  std::string uname_out;
+  std::string linux_cmd = "uname -sm";
+  absl::Status status = remote_util->RunWithCapture(
+      linux_cmd, "uname", &uname_out, nullptr, ArchType::kLinux_x86_64);
+  if (status.ok()) {
+    // Running uname on Windows, assuming it's Linux, leads to tons of ANSI
+    // escape sequences in the output. Remove them to at least get some readable
+    // output.
+    uname_out = absl::StripAsciiWhitespace(
+        ansi_filter::RemoveEscapeSequences(uname_out));
+
+    LOG_DEBUG("Uname returned '%s'", uname_out);
+    absl::StatusOr<ArchType> type = GetArchTypeFromUname(uname_out);
+    if (type.ok()) {
+      LOG_DEBUG("Detected server arch type '%s' from uname",
+                GetArchTypeStr(*type));
+      return ServerArch(*type, /*is_guess=*/false);
+    }
+    status = type.status();
+  }
+  LOG_DEBUG(
+      "Failed to detect arch type from uname; this is expected if the remote "
+      "machine is not Linux; will try Windows next: %s",
+      status.ToString());
+
+  // Check %PROCESSOR_ARCHITECTURE%, assuming it's a Windows machine.
+  // Note: That space after PROCESSOR_ARCHITECTURE is important or else Windows
+  //       command magic interprets quotes as part of the string.
+  std::string arch_out;
+  std::string windows_cmd = "\"cmd /C set PROCESSOR_ARCHITECTURE \"";
+  status = remote_util->RunWithCapture(windows_cmd,
+                                       "set PROCESSOR_ARCHITECTURE", &arch_out,
+                                       nullptr, ArchType::kWindows_x86_64);
+  if (status.ok()) {
+    LOG_DEBUG("PROCESSOR_ARCHITECTURE is '%s'", arch_out);
+    absl::StatusOr<ArchType> type = GetArchTypeFromWinProcArch(arch_out);
+    if (type.ok()) {
+      LOG_DEBUG("Detected server arch type '%s' from PROCESSOR_ARCHITECTURE",
+                GetArchTypeStr(*type));
+      return ServerArch(*type, /*is_guess=*/false);
+    }
+    status = type.status();
+  }
+  LOG_DEBUG("Failed to detect arch type from PROCESSOR_ARCHITECTURE: %s",
+            status.ToString());
+
+  return absl::InternalError("Failed to detect remote architecture");
+}
+
+// static
+std::string ServerArch::CdcRsyncFilename() {
+  switch (GetLocalArchType()) {
+    case ArchType::kWindows_x86_64:
+      return "cdc_rsync.exe";
+    case ArchType::kLinux_x86_64:
+      return "cdc_rsync";
+    default:
+      assert(!kErrorArchTypeUnhandled);
+      return std::string();
+  }
+}
+
+ServerArch::ServerArch(ArchType type, bool is_guess)
+    : type_(type), is_guess_(is_guess) {}
+
+ServerArch::~ServerArch() {}
+
+const char* ServerArch::GetTypeStr() const { return GetArchTypeStr(type_); }
+
+std::string ServerArch::CdcServerFilename() const {
+  switch (type_) {
+    case ArchType::kWindows_x86_64:
+      return "cdc_rsync_server.exe";
+    case ArchType::kLinux_x86_64:
+      return "cdc_rsync_server";
+    default:
+      assert(!kErrorArchTypeUnhandled);
+      return std::string();
+  }
+}
+
+std::string ServerArch::RemoteToolsBinDir() const {
+  if (IsWindowsArchType(type_)) {
+    return "AppData\\Roaming\\cdc-file-transfer\\bin\\";
+  }
+
+  if (IsLinuxArchType(type_)) {
+    return ".cache/cdc-file-transfer/bin/";
+  }
+
+  assert(!kErrorArchTypeUnhandled);
+  return std::string();
+}
+
+std::string ServerArch::GetStartServerCommand(int exit_code_not_found,
+                                              const std::string& args) const {
+  std::string server_path = RemoteToolsBinDir() + CdcServerFilename();
+
+  if (IsWindowsArchType(type_)) {
+    // TODO(ljusten): On Windows, ssh does not seem to forward the Powershell
+    // exit code (exit_code_not_found) to the process. However, that's really
+    // a minor issue and means we display "Deploying server..." instead of
+    // "Server not deployed. Deploying...";
+    return RemoteUtil::QuoteForWindows(
+        absl::StrFormat("powershell -Command "
+                        "Set-StrictMode -Version 2; "
+                        "$ErrorActionPreference = 'Stop'; "
+                        "if (-not (Test-Path -Path '%s')) { "
+                        "  exit %i; "
+                        "} "
+                        "%s %s",
+                        server_path, exit_code_not_found, server_path, args));
+  }
+
+  if (IsLinuxArchType(type_)) {
+    return absl::StrFormat("if [ ! -f %s ]; then exit %i; fi; %s %s",
+                           server_path, exit_code_not_found, server_path, args);
+  }
+
+  assert(!kErrorArchTypeUnhandled);
+  return std::string();
+}
+
+std::string ServerArch::GetDeploySftpCommands() const {
+  std::string commands;
+
+  // Create the remote tools bin dir if it doesn't exist yet.
+  // This assumes that sftp's remote startup directory is the home directory.
+  const std::string server_dir = path::ToUnix(RemoteToolsBinDir());
+  std::vector<std::string> dir_parts =
+      absl::StrSplit(server_dir, '/', absl::SkipEmpty());
+  for (const std::string& dir : dir_parts) {
+    // Use -mkdir to ignore errors if the directory already exists.
+    commands += absl::StrFormat("-mkdir %s\ncd %s\n", dir, dir);
+  }
+
+  // Copy the server binary to a temp location. This assumes that sftp's local
+  // startup directory is cdc_rsync's exe dir.
+  const std::string server_file = CdcServerFilename();
+  const std::string server_temp_file = server_file + Util::GenerateUniqueId();
+  commands += absl::StrFormat("put %s %s\n", server_file, server_temp_file);
+
+  // Restore permissions in case they changed and propagate temp file.
+  commands += absl::StrFormat("-chmod 755 %s\n", server_file);
+  commands += absl::StrFormat("chmod 755 %s\n", server_temp_file);
+  commands += absl::StrFormat("rename %s %s\n", server_temp_file, server_file);
+
+  return commands;
+}
+
+}  // namespace cdc_ft

cdc_rsync/server_arch.h

@@ -0,0 +1,97 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef CDC_RSYNC_SERVER_ARCH_H_
+#define CDC_RSYNC_SERVER_ARCH_H_
+
+#include <string>
+
+#include "absl/status/statusor.h"
+#include "common/arch_type.h"
+
+namespace cdc_ft {
+
+class RemoteUtil;
+
+// Abstracts all architecture specifics of cdc_rsync_server deployment.
+// Comes in two flavors, "guessed" and "detected". Guesses are used as an
+// optimization. For instance, if one syncs to C:\some\path, it's clearly a
+// Windows machine and we can skip detection.
+class ServerArch {
+ public:
+  // Guesses the arch type based on the destination path, e.g. path starting
+  // with C: indicate Windows. This is a guessed type. It may be wrong. For
+  // instance, if destination is just a single folder like "foo", the method
+  // defaults to Type::kLinux.
+  static ServerArch GuessFromDestination(const std::string& destination);
+
+  // Returns the arch type that matches the current process's type.
+  // This is not a guessed type, it is reliable.
+  static ServerArch DetectFromLocalDevice();
+
+  // Creates an  by properly detecting it on the remote device.
+  // This is more costly than guessing, but it is reliable.
+  static absl::StatusOr<ServerArch> DetectFromRemoteDevice(
+      RemoteUtil* remote_util);
+
+  // Returns the (local!) arch specific filename of cdc_rsync[.exe].
+  static std::string CdcRsyncFilename();
+
+  ServerArch(ArchType type, bool is_guess);
+  ~ServerArch();
+
+  // Accessor for the arch type.
+  ArchType GetType() const { return type_; }
+
+  // Returns the type as a human readable string.
+  const char* GetTypeStr() const;
+
+  // Returns true if the type was guessed and not detected.
+  bool IsGuess() const { return is_guess_; }
+
+  // Returns the arch-specific filename of cdc_rsync_server[.exe].
+  std::string CdcServerFilename() const;
+
+  // Returns the arch-specific directory where cdc_rsync_server is deployed.
+  std::string RemoteToolsBinDir() const;
+
+  // Returns an arch-specific SSH shell command that gets invoked in order to
+  // start cdc_rsync_server. The command
+  // - returns |exit_code_not_found| if cdc_rsync_server does not exist (to
+  //   prevent the confusing bash output message
+  //     "bash: .../cdc_rsync_server: No such file or directory"), and
+  // - runs the server with the provided |args|.
+  std::string GetStartServerCommand(int exit_code_not_found,
+                                    const std::string& args) const;
+
+  // Returns an arch-specific SFTP command sequence that deploys the server
+  // component on the target gets invoked after
+  // cdc_rsync_server has been copied to a temp location. The commands
+  // - create the cdc-file-transfer/bin folder if it doesn't exist yet,
+  // - make the old cdc_rsync_server writable if it exists,
+  // - copy cdc_rsync_server to a temp location,
+  // - make the new cdc_rsync_server executable (Linux only) and
+  // - replaces the existing cdc_rsync_server by the temp one.
+  std::string GetDeploySftpCommands() const;
+
+ private:
+  ArchType type_;
+  bool is_guess_ = false;
+};
+
+}  // namespace cdc_ft
+
+#endif  // CDC_RSYNC_SERVER_ARCH_H_

cdc_rsync/server_arch_test.cc

@@ -0,0 +1,104 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "cdc_rsync/server_arch.h"
+
+#include "absl/strings/match.h"
+#include "gtest/gtest.h"
+
+namespace cdc_ft {
+namespace {
+
+constexpr auto kLinux = ArchType::kLinux_x86_64;
+constexpr auto kWindows = ArchType::kWindows_x86_64;
+
+constexpr bool kNoGuess = false;
+
+TEST(ServerArchTest, GuessesLinuxIfPathStartsWithSlashOrTilde) {
+  EXPECT_EQ(ServerArch::GuessFromDestination("/linux/path").GetType(), kLinux);
+  EXPECT_EQ(ServerArch::GuessFromDestination("/linux\\path").GetType(), kLinux);
+  EXPECT_EQ(ServerArch::GuessFromDestination("~/linux/path").GetType(), kLinux);
+  EXPECT_EQ(ServerArch::GuessFromDestination("~/linux\\path").GetType(),
+            kLinux);
+  EXPECT_EQ(ServerArch::GuessFromDestination("~\\linux\\path").GetType(),
+            kLinux);
+}
+
+TEST(ServerArchTest, GuessesWindowsIfPathStartsWithDrive) {
+  EXPECT_EQ(ServerArch::GuessFromDestination("C:\\win\\path").GetType(),
+            kWindows);
+  EXPECT_EQ(ServerArch::GuessFromDestination("D:win").GetType(), kWindows);
+  EXPECT_EQ(ServerArch::GuessFromDestination("Z:\\win/path").GetType(),
+            kWindows);
+}
+
+TEST(ServerArchTest, GuessesLinuxIfPathOnlyHasForwardSlashes) {
+  EXPECT_EQ(ServerArch::GuessFromDestination("linux/path").GetType(), kLinux);
+}
+
+TEST(ServerArchTest, GuessesWindowsIfPathOnlyHasBackSlashes) {
+  EXPECT_EQ(ServerArch::GuessFromDestination("\\win\\path").GetType(),
+            kWindows);
+}
+
+TEST(ServerArchTest, GuessesLinuxByDefault) {
+  EXPECT_EQ(ServerArch::GuessFromDestination("/mixed\\path").GetType(), kLinux);
+  EXPECT_EQ(ServerArch::GuessFromDestination("/mixed\\path").GetType(), kLinux);
+  EXPECT_EQ(ServerArch::GuessFromDestination("C\\linux/path").GetType(),
+            kLinux);
+  EXPECT_EQ(ServerArch::GuessFromDestination("").GetType(), kLinux);
+}
+
+TEST(ServerArchTest, IsGuess) {
+  EXPECT_TRUE(ServerArch::GuessFromDestination("foo").IsGuess());
+  EXPECT_FALSE(ServerArch::DetectFromLocalDevice().IsGuess());
+}
+
+TEST(ServerArchTest, CdcServerFilename) {
+  EXPECT_FALSE(absl::StrContains(
+      ServerArch(kLinux, kNoGuess).CdcServerFilename(), "exe"));
+  EXPECT_TRUE(absl::StrContains(
+      ServerArch(kWindows, kNoGuess).CdcServerFilename(), "exe"));
+}
+
+TEST(ServerArchTest, RemoteToolsBinDir) {
+  const std::string linux_dir =
+      ServerArch(kLinux, kNoGuess).RemoteToolsBinDir();
+  EXPECT_TRUE(absl::StrContains(linux_dir, ".cache/"));
+
+  std::string win_dir = ServerArch(kWindows, kNoGuess).RemoteToolsBinDir();
+  EXPECT_TRUE(absl::StrContains(win_dir, "AppData\\Roaming\\"));
+}
+
+TEST(ServerArchTest, GetStartServerCommand) {
+  std::string cmd =
+      ServerArch(kWindows, kNoGuess).GetStartServerCommand(123, "foo bar");
+  EXPECT_TRUE(absl::StrContains(cmd, "123"));
+  EXPECT_TRUE(absl::StrContains(cmd, "cdc_rsync_server.exe foo bar"));
+
+  cmd = ServerArch(kLinux, kNoGuess).GetStartServerCommand(123, "foo bar");
+  EXPECT_TRUE(absl::StrContains(cmd, "123"));
+  EXPECT_TRUE(absl::StrContains(cmd, "cdc_rsync_server foo bar"));
+}
+
+TEST(ServerArchTest, GetDeployReplaceCommand) {
+  std::string cmd = ServerArch(kWindows, kNoGuess).GetDeploySftpCommands();
+  EXPECT_TRUE(absl::StrContains(cmd, "cdc_rsync_server.exe"));
+
+  cmd = ServerArch(kLinux, kNoGuess).GetDeploySftpCommands();
+  EXPECT_TRUE(absl::StrContains(cmd, "cdc_rsync_server"));
+}
+
+}  // namespace
+}  // namespace cdc_ft

cdc_rsync/zstd_stream.cc

@@ -27,22 +27,19 @@ namespace {
 // trigger a flush. This happens when files with no changes are diff'ed (this
 // produces very low volume data). Flushing prevents that the server gets stale
 // and becomes overwhelmed later.
-constexpr absl::Duration kMinCompressPeriod = absl::Milliseconds(500);
+constexpr absl::Duration kDefaultAutoFlushPeriod = absl::Milliseconds(500);
 
 }  // namespace
 
 ZstdStream::ZstdStream(Socket* socket, int level, uint32_t num_threads)
-    : socket_(socket), cctx_(nullptr) {
+    : socket_(socket),
+      cctx_(nullptr),
+      auto_flush_period_(kDefaultAutoFlushPeriod) {
   status_ = WrapStatus(Initialize(level, num_threads),
                        "Failed to initialize stream compressor");
 }
 
 ZstdStream::~ZstdStream() {
-  if (cctx_) {
-    ZSTD_freeCCtx(cctx_);
-    cctx_ = nullptr;
-  }
-
   {
     absl::MutexLock lock(&mutex_);
     shutdown_ = true;
@@ -50,6 +47,11 @@ ZstdStream::~ZstdStream() {
   if (compressor_thread_.joinable()) {
     compressor_thread_.join();
   }
+
+  if (cctx_) {
+    ZSTD_freeCCtx(cctx_);
+    cctx_ = nullptr;
+  }
 }
 
 absl::Status ZstdStream::Write(const void* data, size_t size) {
@@ -79,7 +81,7 @@ absl::Status ZstdStream::Write(const void* data, size_t size) {
   return absl::OkStatus();
 }
 
-absl::Status ZstdStream::Flush() {
+absl::Status ZstdStream::Finish() {
   absl::MutexLock lock(&mutex_);
   if (!status_.ok()) return status_;
 
@@ -134,7 +136,7 @@ void ZstdStream::ThreadCompressorMain() {
              in_buffer_.size() == in_buffer_.capacity();
     };
     bool flush =
-        !mutex_.AwaitWithTimeout(absl::Condition(&cond), kMinCompressPeriod);
+        !mutex_.AwaitWithTimeout(absl::Condition(&cond), auto_flush_period_);
     if (shutdown_) {
       return;
     }
@@ -144,7 +146,7 @@ void ZstdStream::ThreadCompressorMain() {
     const ZSTD_EndDirective mode = last_chunk_ ? ZSTD_e_end
                                    : flush     ? ZSTD_e_flush
                                                : ZSTD_e_continue;
-    LOG_DEBUG("Compressing %u bytes (mode=%s)", in_buffer_.size(),
+    LOG_VERBOSE("Compressing %u bytes (mode=%s)", in_buffer_.size(),
                 mode == ZSTD_e_end     ? "end"
                 : mode == ZSTD_e_flush ? "flush"
                                        : "continue");

cdc_rsync/zstd_stream.h

@@ -21,8 +21,8 @@
 
 #include "absl/status/status.h"
 #include "absl/synchronization/mutex.h"
-#include "cdc_rsync/base/socket.h"
 #include "common/buffer.h"
+#include "common/socket.h"
 #include "lib/zstd.h"
 
 namespace cdc_ft {
@@ -36,8 +36,13 @@ class ZstdStream {
   // Sends the given |data| to the compressor.
   absl::Status Write(const void* data, size_t size) ABSL_LOCKS_EXCLUDED(mutex_);
 
-  // Flushes all remaining data and sends the compressed data to the socket.
+  // Finishes the stream and flushes all remaining data.
-  absl::Status Flush() ABSL_LOCKS_EXCLUDED(mutex_);
+  absl::Status Finish() ABSL_LOCKS_EXCLUDED(mutex_);
+
+  // Flushes internal buffers if no new data is written for longer than this
+  // time. This makes sure that no data is stuck in the pipeline if no new input
+  // is available. Default is 500 ms.
+  void AutoFlushAfter(absl::Duration dur) { auto_flush_period_ = dur; }
 
  private:
   // Initializes the compressor and related data.
@@ -58,6 +63,8 @@ class ZstdStream {
   bool last_chunk_sent_ ABSL_GUARDED_BY(mutex_) = false;
   absl::Status status_ ABSL_GUARDED_BY(mutex_);
   std::thread compressor_thread_;
+
+  absl::Duration auto_flush_period_;
 };
 
 }  // namespace cdc_ft

cdc_rsync/zstd_stream_test.cc

@@ -14,8 +14,8 @@
 
 #include "cdc_rsync/zstd_stream.h"
 
-#include "cdc_rsync/base/fake_socket.h"
 #include "cdc_rsync_server/unzstd_stream.h"
+#include "common/fake_socket.h"
 #include "common/status_test_macros.h"
 #include "gtest/gtest.h"
 
@@ -32,7 +32,7 @@ class ZstdStreamTest : public ::testing::Test {
 TEST_F(ZstdStreamTest, Small) {
   const std::string want = "Lorem ipsum gibberisulum foobarberis";
   EXPECT_OK(cstream_.Write(want.data(), want.size()));
-  EXPECT_OK(cstream_.Flush());
+  EXPECT_OK(cstream_.Finish());
 
   Buffer buff(1024);
   size_t bytes_read;
@@ -43,6 +43,52 @@ TEST_F(ZstdStreamTest, Small) {
   EXPECT_EQ(got, want);
 }
 
+TEST_F(ZstdStreamTest, AutoFlushesAfterTimeout) {
+  const std::string want = "Lorem ipsum gibberisulum foobarberis";
+  cstream_.AutoFlushAfter(absl::Milliseconds(10));
+  EXPECT_OK(cstream_.Write(want.data(), want.size()));
+  // Note: No flush! cstream_ will compress and send the data after 10 ms.
+
+  // Only read as much data as we have written, or else dstream_.Read() will
+  // expect more data.
+  Buffer buff(want.size());
+  size_t bytes_read;
+  bool eof = false;
+  EXPECT_OK(dstream_.Read(buff.data(), buff.size(), &bytes_read, &eof));
+  EXPECT_FALSE(eof);
+  std::string got(buff.data(), bytes_read);
+  EXPECT_EQ(got, want);
+}
+
+// Regression test for an issue in UnzstdStream, where the reader tried to read
+// from the socket even though the output data was still available in internal
+// buffers.
+TEST_F(ZstdStreamTest, DeliversOutputBeforeReadingNewData) {
+  const std::string want1 = "I want";
+  const std::string want2 = "to eat cookies";
+  cstream_.AutoFlushAfter(absl::Milliseconds(10));
+  EXPECT_OK(cstream_.Write(want1.data(), want1.size()));
+  EXPECT_OK(cstream_.Write(want2.data(), want2.size()));
+  // Note: No flush! cstream_ will compress and send the data after 10 ms.
+
+  Buffer buff1(want1.size());
+  Buffer buff2(want2.size());
+  size_t bytes_read1, bytes_read2;
+  bool eof1 = false, eof2 = false;
+  EXPECT_OK(dstream_.Read(buff1.data(), buff1.size(), &bytes_read1, &eof1));
+
+  // There was a bug in dstream_.Read(), where the method would first try to
+  // read new input before uncompressing data, even though the data was already
+  // present in internal buffers.
+  EXPECT_OK(dstream_.Read(buff2.data(), buff2.size(), &bytes_read2, &eof2));
+  EXPECT_FALSE(eof1);
+  EXPECT_FALSE(eof2);
+  std::string got1(buff1.data(), bytes_read1);
+  std::string got2(buff2.data(), bytes_read2);
+  EXPECT_EQ(got1, want1);
+  EXPECT_EQ(got2, want2);
+}
+
 TEST_F(ZstdStreamTest, Large) {
   Buffer want(1024 * 1024 * 10 + 12345);
   constexpr uint64_t prime = 919393;
@@ -55,7 +101,7 @@ TEST_F(ZstdStreamTest, Large) {
     size_t size = std::min<size_t>(kChunkSize, want.size() - pos);
     EXPECT_OK(cstream_.Write(want.data() + pos, size));
   }
-  EXPECT_OK(cstream_.Flush());
+  EXPECT_OK(cstream_.Finish());
 
   bool eof = false;
   Buffer buff(128 * 1024);

cdc_rsync_server/BUILD

@@ -22,7 +22,7 @@ cc_test(
     srcs = ["file_deleter_and_sender_test.cc"],
     deps = [
         ":file_deleter_and_sender",
-        "//cdc_rsync/base:fake_socket",
+        "//common:fake_socket",
         "//common:status_test_macros",
         "//common:test_main",
         "@com_google_googletest//:gtest",
@@ -95,15 +95,16 @@ cc_binary(
         ":file_diff_generator",
         ":file_finder",
         ":file_info",
-        ":server_socket",
         ":unzstd_stream",
         "//cdc_rsync/base:cdc_interface",
         "//cdc_rsync/base:message_pump",
         "//cdc_rsync/base:server_exit_code",
+        "//common:build_version",
         "//common:clock",
         "//common:gamelet_component",
         "//common:log",
         "//common:path_filter",
+        "//common:server_socket",
         "//common:status",
         "//common:stopwatch",
         "//common:threadpool",
@@ -123,32 +124,13 @@ cc_library(
     hdrs = ["file_info.h"],
 )
 
-cc_library(
-    name = "server_socket",
-    srcs = ["server_socket.cc"],
-    hdrs = ["server_socket.h"],
-    linkopts = select({
-        "//tools:windows": [
-            "/DEFAULTLIB:Ws2_32.lib",  # Sockets, e.g. recv, send, WSA*.
-        ],
-        "//conditions:default": [],
-    }),
-    deps = [
-        "//cdc_rsync/base:socket",
-        "//common:log",
-        "//common:status",
-        "//common:util",
-        "@com_google_absl//absl/status",
-    ],
-)
-
 cc_library(
     name = "unzstd_stream",
     srcs = ["unzstd_stream.cc"],
     hdrs = ["unzstd_stream.h"],
     deps = [
         "//cdc_rsync/base:message_pump",
-        "//cdc_rsync/base:socket",
+        "//common:socket",
         "//common:status",
         "@com_github_zstd//:zstd",
         "@com_google_absl//absl/status",

cdc_rsync_server/cdc_rsync_server.cc

@@ -19,11 +19,12 @@
 #include "cdc_rsync/protos/messages.pb.h"
 #include "cdc_rsync_server/file_deleter_and_sender.h"
 #include "cdc_rsync_server/file_finder.h"
-#include "cdc_rsync_server/server_socket.h"
 #include "cdc_rsync_server/unzstd_stream.h"
 #include "common/log.h"
 #include "common/path.h"
+#include "common/server_socket.h"
 #include "common/status.h"
+#include "common/status_macros.h"
 #include "common/stopwatch.h"
 #include "common/threadpool.h"
 #include "common/util.h"
@@ -32,9 +33,22 @@ namespace cdc_ft {
 
 namespace {
 
+// Number of files for which to call fclose() and finalize files in parallel.
+constexpr size_t kNumFinalizerThreads = 8;
+
+// Max 16 files in the patcher and finalizer queues to prevent that too many
+// files are open concurrently.
+constexpr size_t kMaxQueueSize = 16;
+
 // Suffix for the patched file created from the basis file and the diff.
 constexpr char kIntermediatePathSuffix[] = ".__cdc_rsync_temp__";
 
+#if PLATFORM_WINDOWS
+constexpr char kServerFilename[] = "cdc_rsync_server.exe";
+#elif PLATFORM_LINUX
+constexpr char kServerFilename[] = "cdc_rsync_server";
+#endif
+
 uint16_t kExecutableBits =
     path::MODE_IXUSR | path::MODE_IXGRP | path::MODE_IXOTH;
 
@@ -43,6 +57,9 @@ uint16_t kExecutableBits =
 // |target_filepath| match, writes an intermediate file and replaces
 // the file at |target_filepath| with the intermediate file when all data has
 // been received.
+// Each PatchTask is queued twice, once to create the patched file, and once in
+// a different thread pool to close and finalize the patched file. This is
+// because fclose() can take a long time to finish, so it could block patching.
 class PatchTask : public Task {
  public:
   PatchTask(const std::string& base_filepath,
@@ -51,38 +68,69 @@ class PatchTask : public Task {
       : base_filepath_(base_filepath),
         target_filepath_(target_filepath),
         file_(file),
-        cdc_(cdc) {}
+        cdc_(cdc),
+        need_intermediate_file_(target_filepath_ == base_filepath_),
+        patched_filepath_(target_filepath_ == base_filepath_
+                              ? base_filepath_ + kIntermediatePathSuffix
+                              : target_filepath_) {}
 
   virtual ~PatchTask() = default;
 
+  PatchTask(const PatchTask& other) = delete;
+  PatchTask& operator=(const PatchTask& other) = delete;
+
   const ChangedFileInfo& File() const { return file_; }
 
   const absl::Status& Status() const { return status_; }
 
   // Task:
   void ThreadRun(IsCancelledPredicate is_cancelled) override {
-    bool need_intermediate_file = target_filepath_ == base_filepath_;
+    // Each PatchTask is queued twice, once to apply the patch and once to
-    std::string patched_filepath =
+    // close and finalize the patched file.
-        need_intermediate_file ? base_filepath_ + kIntermediatePathSuffix
+    switch (state_) {
-                               : target_filepath_;
+      case State::kPatching:
-
+        Patch();
-    absl::StatusOr<FILE*> patched_file = path::OpenFile(patched_filepath, "wb");
+        state_ = State::kFinalizing;
-    if (!patched_file.ok()) {
+        break;
-      status_ = patched_file.status();
+      case State::kFinalizing:
-      return;
+        Finalize();
+        state_ = State::kDone;
+        break;
+      default:
+        assert(!"Invalid state");
+    }
   }
 
+ private:
+  void Patch() {
+    absl::StatusOr<FILE*> patched_fp = path::OpenFile(patched_filepath_, "wb");
+    if (!patched_fp.ok()) {
+      status_ = patched_fp.status();
+      return;
+    }
+    patched_fp_ = *patched_fp;
+
     // Receive diff stream from server and apply.
-    bool is_executable = false;
+    status_ =
-    status_ = cdc_->ReceiveDiffAndPatch(base_filepath_, *patched_file,
+        cdc_->ReceiveDiffAndPatch(base_filepath_, patched_fp_, &is_executable_);
-                                        &is_executable);
+
-    fclose(*patched_file);
+    // The file is closed by Finalize() in a separate thread pool since fclose()
+    // takes a while on some systems.
+  }
+
+  void Finalize() {
+    if (patched_fp_) {
+      fclose(patched_fp_);
+      patched_fp_ = nullptr;
+    }
+
     if (!status_.ok()) {
+      // Some error occurred during Patch().
       return;
     }
 
     // These bits are OR'ed on top of the mode bits.
-    uint16_t mode_or_bits = is_executable ? kExecutableBits : 0;
+    uint16_t mode_or_bits = is_executable_ ? kExecutableBits : 0;
 
     // Store mode from the original base path.
     path::Stats stats;
@@ -93,13 +141,13 @@ class PatchTask : public Task {
       return;
     }
 
-    if (need_intermediate_file) {
+    if (need_intermediate_file_) {
       // Replace |base_filepath_| (==|target_filepath_|) by the intermediate
       // file |patched_filepath|.
-      status_ = path::ReplaceFile(target_filepath_, patched_filepath);
+      status_ = path::ReplaceFile(target_filepath_, patched_filepath_);
       if (!status_.ok()) {
         status_ = WrapStatus(status_, "ReplaceFile() for '%s' by '%s' failed",
-                             base_filepath_, patched_filepath);
+                             base_filepath_, patched_filepath_);
         return;
       }
     } else {
@@ -120,11 +168,82 @@ class PatchTask : public Task {
     status_ = path::SetFileTime(target_filepath_, file_.client_modified_time);
   }
 
+  const std::string base_filepath_;
+  const std::string target_filepath_;
+  const ChangedFileInfo file_;
+  CdcInterface* const cdc_;
+  const bool need_intermediate_file_ = false;
+  const std::string patched_filepath_;
+
+  FILE* patched_fp_ = nullptr;
+  bool is_executable_ = false;
+  absl::Status status_;
+
+  // This task is queued twice, once to patch and once to close and finalize the
+  // patched file.
+  enum class State { kPatching, kFinalizing, kDone };
+  State state_ = State::kPatching;
+};
+
+// Background task that closes a file and sets the mtime and perms. This is done
+// in the background since closing a file might block for a long time.
+class FinalizeCopiedFileTask : public Task {
+ public:
+  // Finalize |file| with given path |filepath|. |status| is the status from
+  // writing the file. On error, the file is only closed.
+  FinalizeCopiedFileTask(FILE* fp, FileInfo file, std::string filepath,
+                         bool is_executable, absl::Status status)
+      : fp_(fp),
+        file_(std::move(file)),
+        filepath_(std::move(filepath)),
+        is_executable_(is_executable),
+        status_(status) {}
+  virtual ~FinalizeCopiedFileTask() = default;
+
+  FinalizeCopiedFileTask(const FinalizeCopiedFileTask& other) = delete;
+  FinalizeCopiedFileTask& operator=(const FinalizeCopiedFileTask& other) =
+      delete;
+
+  const absl::Status& Status() const { return status_; }
+
+  // Task:
+  void ThreadRun(IsCancelledPredicate is_cancelled) override {
+    assert(fp_);
+    fclose(fp_);
+
+    if (!status_.ok()) {
+      // Writing the file failed, nothing to finalize.
+      status_ = WrapStatus(status_, "Failed to write file %s", filepath_);
+      return;
+    }
+
+    // Set file write time.
+    status_ = path::SetFileTime(filepath_, file_.modified_time);
+    if (!status_.ok()) {
+      status_ =
+          WrapStatus(status_, "Failed to set file mod time for %s", filepath_);
+      return;
+    }
+
+    // Set executable bit, but just print warnings as it's not critical.
+    if (is_executable_) {
+      path::Stats stats;
+      status_ = path::GetStats(filepath_, &stats);
+      if (status_.ok()) {
+        status_ = path::ChangeMode(filepath_, stats.mode | kExecutableBits);
+      }
+      if (!status_.ok()) {
+        LOG_WARNING("Failed to set executable bit on '%s': %s", filepath_,
+                    status_.ToString());
+      }
+    }
+  }
+
  private:
-  std::string base_filepath_;
+  FILE* const fp_ = nullptr;
-  std::string target_filepath_;
+  const FileInfo file_;
-  ChangedFileInfo file_;
+  const std::string filepath_;
-  CdcInterface* cdc_;
+  const bool is_executable_;
   absl::Status status_;
 };
 
@@ -160,8 +279,8 @@ bool CdcRsyncServer::CheckComponents(
   }
 
   std::vector<GameletComponent> our_components;
-  status = GameletComponent::Get(
+  status = GameletComponent::Get({path::Join(component_dir, kServerFilename)},
-      {path::Join(component_dir, "cdc_rsync_server")}, &our_components);
+                                 &our_components);
   if (!status.ok() || components != our_components) {
     return false;
   }
@@ -169,28 +288,24 @@ bool CdcRsyncServer::CheckComponents(
   return true;
 }
 
-absl::Status CdcRsyncServer::Run(int port) {
+absl::Status CdcRsyncServer::Run() {
-  absl::Status status = Socket::Initialize();
+  RETURN_IF_ERROR(Socket::Initialize(), "Failed to initialize sockets");
-  if (!status.ok()) {
-    return WrapStatus(status, "Failed to initialize sockets");
-  }
   socket_finalizer_ = std::make_unique<SocketFinalizer>();
 
   socket_ = std::make_unique<ServerSocket>();
-  status = socket_->StartListening(port);
+  int port;
-  if (!status.ok()) {
+  ASSIGN_OR_RETURN(port, socket_->StartListening(0),
-    return WrapStatus(status, "Failed to start listening on port %i", port);
+                   "Failed to start listening for connections");
-  }
   LOG_INFO("cdc_rsync_server listening on port %i", port);
 
   // This is the marker for the client, so it knows it can connect.
-  printf("Server is listening\n");
+  // Print port first so the client can easily parse it when it sees "Server is
+  // listening" without dealing with half-transmitted data.
+  printf("Port %i: Server is listening\n", port);
   fflush(stdout);
 
-  status = socket_->WaitForConnection();
+  RETURN_IF_ERROR(socket_->WaitForConnection(),
-  if (!status.ok()) {
+                  "Failed to establish a connection");
-    return WrapStatus(status, "Failed to establish a connection");
-  }
 
   message_pump_ = std::make_unique<MessagePump>(
       socket_.get(),
@@ -198,7 +313,7 @@ absl::Status CdcRsyncServer::Run(int port) {
   message_pump_->StartMessagePump();
 
   LOG_INFO("Client connected. Starting to sync.");
-  status = Sync();
+  absl::Status status = Sync();
   if (!status.ok()) {
     socket_->ShutdownSendingEnd().IgnoreError();
     return status;
@@ -487,7 +602,7 @@ absl::Status CdcRsyncServer::CreateMissingDirs() {
 template <typename T>
 absl::Status CdcRsyncServer::SendFileIndices(const char* file_type,
                                              const std::vector<T>& files) {
-  LOG_INFO("Sending indices of missing files to client");
+  LOG_INFO("Sending indices of %s files to client", file_type);
   constexpr char error_fmt[] = "Failed to send indices of %s files.";
 
   AddFileIndicesResponse response;
@@ -544,6 +659,8 @@ absl::Status CdcRsyncServer::HandleSendMissingFileData() {
     }
   }
 
+  Threadpool finalize_pool(kNumFinalizerThreads);
+
   for (uint32_t server_index = 0; server_index < diff_.missing_files.size();
        server_index++) {
     const FileInfo& file = diff_.missing_files[server_index];
@@ -563,8 +680,8 @@ absl::Status CdcRsyncServer::HandleSendMissingFileData() {
                         request.server_index(), server_index);
     }
 
-    // Verify that there is no directory existing with the same name.
+    // Remove |filepath| if it is a directory.
-    if (path::Exists(filepath) && path::DirExists(filepath)) {
+    if (path::DirExists(filepath)) {
       assert(!diff_.extraneous_dirs.empty());
       status = path::RemoveFile(filepath);
       if (!status.ok()) {
@@ -613,27 +730,25 @@ absl::Status CdcRsyncServer::HandleSendMissingFileData() {
     }
 
     status = path::StreamWriteFileContents(*fp, handler);
-    fclose(*fp);
+    finalize_pool.QueueTask(std::make_unique<FinalizeCopiedFileTask>(
-    if (!status.ok()) {
+        *fp, file, filepath, is_executable, status));
-      return WrapStatus(status, "Failed to write file %s", filepath);
+    finalize_pool.WaitForQueuedTasksAtMost(kMaxQueueSize);
+
+    // Drain finalize pool for the last file.
+    if (server_index + 1 == diff_.missing_files.size()) {
+      finalize_pool.Wait();
     }
 
-    // Set file write time.
+    // Check the results of completed tasks.
-    status = path::SetFileTime(filepath, file.modified_time);
+    for (std::unique_ptr<Task> task = finalize_pool.TryGetCompletedTask();
-    if (!status.ok()) {
+         task != nullptr; task = finalize_pool.TryGetCompletedTask()) {
-      return WrapStatus(status, "Failed to set file mod time for %s", filepath);
+      const FinalizeCopiedFileTask* finalize_task =
-    }
+          static_cast<FinalizeCopiedFileTask*>(task.get());
-
+      if (!finalize_task->Status().ok()) {
-    // Set executable bit, but just print warnings as it's not critical.
+        // Close and finish files that have already been copied, so we don't
-    if (is_executable) {
+        // discard several already copied files because one failed.
-      path::Stats stats;
+        finalize_pool.Wait();
-      status = path::GetStats(filepath, &stats);
+        return finalize_task->Status();
-      if (status.ok()) {
-        status = path::ChangeMode(filepath, stats.mode | kExecutableBits);
-      }
-      if (!status.ok()) {
-        LOG_WARNING("Failed to set executable bit on '%s': %s", filepath,
-                    status.ToString());
       }
     }
   }
@@ -674,9 +789,20 @@ absl::Status CdcRsyncServer::SyncChangedFiles() {
   // Pipeline sending signatures and patching files:
   // MAIN THREAD:       Send signatures to client.
   //                    Only sends to the socket.
-  // WORKER THREAD: Receive diffs from client and patch file.
+  // PATCHER THREAD:    Receive diffs from client and create patch file.
   //                    Only reads from the socket.
-  Threadpool pool(1);
+  // FINALIZER THREADS: Close patched files and finalize them.
+  Threadpool patch_pool(1);
+  Threadpool finalize_pool(kNumFinalizerThreads);
+
+  // Forward finished patch task immediately to finalize pool.
+  patch_pool.SetTaskCompletedCallback(
+      [&finalize_pool](std::unique_ptr<Task> task) {
+        // Spin if there are too many outstanding tasks, in order to limit the
+        // max number of outstanding tasks.
+        finalize_pool.QueueTask(std::move(task));
+        finalize_pool.WaitForQueuedTasksAtMost(kMaxQueueSize);
+      });
 
   for (uint32_t server_index = 0; server_index < diff_.changed_files.size();
        server_index++) {
@@ -702,25 +828,28 @@ absl::Status CdcRsyncServer::SyncChangedFiles() {
     }
 
     // Queue patching task.
-    pool.QueueTask(std::make_unique<PatchTask>(base_filepath, target_filepath,
+    patch_pool.QueueTask(std::make_unique<PatchTask>(
-                                               file, &cdc));
+        base_filepath, target_filepath, file, &cdc));
 
-    // Wait for the last file to finish.
+    // Drain pools for the last file.
     if (server_index + 1 == diff_.changed_files.size()) {
-      pool.Wait();
+      patch_pool.Wait();
+      finalize_pool.Wait();
     }
 
     // Check the results of completed tasks.
-    std::unique_ptr<Task> task = pool.TryGetCompletedTask();
+    for (std::unique_ptr<Task> task = finalize_pool.TryGetCompletedTask();
-    while (task) {
+         task != nullptr; task = finalize_pool.TryGetCompletedTask()) {
-      PatchTask* patch_task = static_cast<PatchTask*>(task.get());
+      const PatchTask* patch_task = static_cast<PatchTask*>(task.get());
       const std::string& task_path = patch_task->File().filepath;
       if (!patch_task->Status().ok()) {
+        // Close and finish files that have already been synced, so we don't
+        // discard several already synced files because one failed.
+        finalize_pool.Wait();
         return WrapStatus(patch_task->Status(), "Failed to patch file '%s'",
                           task_path);
       }
       LOG_INFO("Finished patching file %s", task_path.c_str());
-      task = pool.TryGetCompletedTask();
     }
   }
 

cdc_rsync_server/cdc_rsync_server.h

@@ -43,9 +43,10 @@ class CdcRsyncServer {
   // up-to-date by checking their sizes and timestamps.
   bool CheckComponents(const std::vector<GameletComponent>& components);
 
-  // Listens to |port|, accepts a connection from the client and runs the rsync
+  // Listens to any available port, accepts a connection from the client and
-  // procedure.
+  // runs the rsync procedure. Prints "Port <n>: Server is listening" to stdout,
-  absl::Status Run(int port);
+  // so the client can retrieve the selected port.
+  absl::Status Run();
 
   // Returns the verbosity sent from the client. 0 by default.
   int GetVerbosity() const { return verbosity_; }

cdc_rsync_server/cdc_rsync_server.vcxproj

@@ -49,7 +49,7 @@
   <PropertyGroup>
     <BazelTargets>//cdc_rsync_server:cdc_rsync_server</BazelTargets>
     <BazelOutputFile>cdc_rsync_server</BazelOutputFile>
-    <BazelIncludePaths>..\;..\third_party\absl;..\bazel-cdc-file-transfer\external\com_github_blake3\c;..\bazel-stadia-file-transfer\external\com_github_zstd\lib;..\third_party\googletest\googletest\include;..\bazel-cdc-file-transfer\external\com_google_protobuf\src</BazelIncludePaths>
+    <BazelIncludePaths>..\;..\third_party\absl;..\bazel-cdc-file-transfer\external\com_github_blake3\c;..\bazel-cdc-file-transfer\external\com_github_zstd;..\third_party\googletest\googletest\include;..\bazel-cdc-file-transfer\external\com_google_protobuf\src;..\bazel-bin</BazelIncludePaths>
   </PropertyGroup>
   <Import Project="..\NMakeBazelProject.targets" />
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />

cdc_rsync_server/file_deleter_and_sender_test.cc

@@ -14,8 +14,8 @@
 
 #include "cdc_rsync_server/file_deleter_and_sender.h"
 
-#include "cdc_rsync/base/fake_socket.h"
 #include "cdc_rsync/base/message_pump.h"
+#include "common/fake_socket.h"
 #include "common/log.h"
 #include "common/path.h"
 #include "common/status_test_macros.h"

cdc_rsync_server/main.cc

@@ -14,6 +14,7 @@
 
 #include "cdc_rsync/base/server_exit_code.h"
 #include "cdc_rsync_server/cdc_rsync_server.h"
+#include "common/build_version.h"
 #include "common/gamelet_component.h"
 #include "common/log.h"
 #include "common/status.h"
@@ -60,33 +61,33 @@ ServerExitCode GetExitCode(const absl::Status& status) {
 }  // namespace cdc_ft
 
 int main(int argc, const char** argv) {
-  if (argc < 2) {
+  if (argc < 5) {
-    printf("Usage: cdc_rsync_server <port> cdc_rsync_server <size> <time> \n");
+    printf(R"("cdc_rsync_server - Remote component of cdc_rsync. Version: %s
-    printf("       where <size> and <time> are the file size and modified\n");
+
-    printf("       timestamp (Unix epoch) of the corresponding component.\n");
+Usage: cdc_rsync_server <build_version> cdc_rsync_server <size> <modified_time>
-    return cdc_ft::kServerExitCodeGenericStartup;
+       <build_version>            build version embedded in the component
-  }
+       <size>                     file size of the component
+       <modified_time>            timestamp (Unix epoch) of the component)",
+           BUILD_VERSION);
 
-  int port = atoi(argv[1]);
-  if (port == 0) {
-    SendErrorMessage(absl::StrFormat("Invalid port '%s'", argv[1]).c_str());
     return cdc_ft::kServerExitCodeGenericStartup;
   }
 
   // The rest is expected to be sets of gamelet component info consisting of
-  // (filename, filesize, modified_time). This is used check whether the
+  // (version, filename, size, modified_time). This is used check whether the
   // components are up-to-date.
   std::vector<cdc_ft::GameletComponent> components =
-      cdc_ft::GameletComponent::FromCommandLineArgs(argc - 2, argv + 2);
+      cdc_ft::GameletComponent::FromCommandLineArgs(argc - 1, argv + 1);
 
   cdc_ft::Log::Initialize(
       std::make_unique<cdc_ft::ConsoleLog>(cdc_ft::LogLevel::kWarning));
   cdc_ft::CdcRsyncServer server;
+
   if (!server.CheckComponents(components)) {
     return cdc_ft::kServerExitCodeOutOfDate;
   }
 
-  absl::Status status = server.Run(port);
+  absl::Status status = server.Run();
   if (status.ok()) {
     return 0;
   }

cdc_rsync_server/unzstd_stream.cc

@@ -14,7 +14,7 @@
 
 #include "cdc_rsync_server/unzstd_stream.h"
 
-#include "cdc_rsync/base/socket.h"
+#include "common/socket.h"
 #include "common/status.h"
 
 namespace cdc_ft {
@@ -41,20 +41,6 @@ absl::Status UnzstdStream::Read(void* out_buffer, size_t out_size,
 
   ZSTD_outBuffer output = {out_buffer, out_size, 0};
   while (output.pos < output.size && !*eof) {
-    if (input_.pos == input_.size) {
-      // Read more compressed input data.
-      // Allow partial reads since the stream could end any time.
-      size_t in_size;
-      absl::Status status =
-          socket_->Receive(in_buffer_.data(), in_buffer_.size(),
-                           /*allow_partial_read=*/true, &in_size);
-      if (!status.ok()) {
-        return WrapStatus(status, "socket_->ReceiveEx() failed");
-      }
-      input_.pos = 0;
-      input_.size = in_size;
-    }
-
     // Decompress.
     size_t ret = ZSTD_decompressStream(dctx_, &output, &input_);
     if (ZSTD_isError(ret)) {
@@ -67,6 +53,20 @@ absl::Status UnzstdStream::Read(void* out_buffer, size_t out_size,
       return MakeStatus("EOF with %u bytes input data available",
                         input_.size - input_.pos);
     }
+
+    if (input_.pos == input_.size && output.pos < output.size && !*eof) {
+      // Read more compressed input data.
+      // Allow partial reads since the stream could end any time.
+      size_t in_size;
+      absl::Status status =
+          socket_->Receive(in_buffer_.data(), in_buffer_.size(),
+                           /*allow_partial_read=*/true, &in_size);
+      if (!status.ok()) {
+        return WrapStatus(status, "socket_->ReceiveEx() failed");
+      }
+      input_.pos = 0;
+      input_.size = in_size;
+    }
   }
 
   // Output buffer is full or eof.

cdc_stream/BUILD

@@ -214,6 +214,7 @@ cc_library(
         "//common:process",
         "//common:remote_util",
         "//common:sdk_util",
+        "//common:server_socket",
         "//common:status_macros",
         "//common:stopwatch",
         "//data_store:disk_data_store",

cdc_stream/asset_stream_config.cc

@@ -50,19 +50,14 @@ void AssetStreamConfig::RegisterCommandLineFlags(lyra::command& cmd,
                              "asset stream service, default: " +
                              std::to_string(service_port_)));
 
-  session_cfg_.forward_port_first = MultiSession::kDefaultForwardPortFirst;
+  cmd.add_argument(lyra::opt(base_command.PortRangeParser(
-  session_cfg_.forward_port_last = MultiSession::kDefaultForwardPortLast;
+                                 "--forward-port",
-  cmd.add_argument(
+                                 &session_cfg_.deprecated_forward_port_first,
-      lyra::opt(base_command.PortRangeParser("--forward-port",
+                                 &session_cfg_.deprecated_forward_port_last),
-                                             &session_cfg_.forward_port_first,
-                                             &session_cfg_.forward_port_last),
                              "port")
                        .name("--forward-port")
-          .help("TCP port or range used for SSH port forwarding, default: " +
+                       .help("[Deprecated, ignored] TCP port or range used for "
-                std::to_string(MultiSession::kDefaultForwardPortFirst) + "-" +
+                             "SSH port forwarding"));
-                std::to_string(MultiSession::kDefaultForwardPortLast) +
-                ". If a range is specified, searches for available ports "
-                "(slower)."));
 
   session_cfg_.verbosity = kDefaultVerbosity;
   cmd.add_argument(lyra::opt(session_cfg_.verbosity, "num")
@@ -157,9 +152,9 @@ void AssetStreamConfig::RegisterCommandLineFlags(lyra::command& cmd,
                 "connection to the host. See also --dev-src-dir."));
 
   cmd.add_argument(
-      lyra::opt(dev_target_.scp_command, "cmd")
+      lyra::opt(dev_target_.sftp_command, "cmd")
-          .name("--dev-scp-command")
+          .name("--dev-sftp-command")
-          .help("Scp command and extra flags to use for the "
+          .help("Sftp command and extra flags to use for the "
                 "connection to the host. See also --dev-src-dir."));
 
   cmd.add_argument(
@@ -190,8 +185,6 @@ absl::Status AssetStreamConfig::LoadFromFile(const std::string& path) {
   } while (0)
 
   ASSIGN_VAR(service_port_, "service-port", Int);
-  ASSIGN_VAR(session_cfg_.forward_port_first, "forward-port-first", Int);
-  ASSIGN_VAR(session_cfg_.forward_port_last, "forward-port-last", Int);
   ASSIGN_VAR(session_cfg_.verbosity, "verbosity", Int);
   ASSIGN_VAR(session_cfg_.fuse_debug, "debug", Bool);
   ASSIGN_VAR(session_cfg_.fuse_singlethreaded, "singlethreaded", Bool);
@@ -231,8 +224,6 @@ absl::Status AssetStreamConfig::LoadFromFile(const std::string& path) {
 std::string AssetStreamConfig::ToString() {
   std::ostringstream ss;
   ss << "service-port                 = " << service_port_ << std::endl;
-  ss << "forward-port                 = " << session_cfg_.forward_port_first
-     << "-" << session_cfg_.forward_port_last << std::endl;
   ss << "verbosity                    = " << session_cfg_.verbosity
      << std::endl;
   ss << "debug                        = " << session_cfg_.fuse_debug
@@ -258,7 +249,7 @@ std::string AssetStreamConfig::ToString() {
   ss << "dev-user-host                = " << dev_target_.user_host << std::endl;
   ss << "dev-ssh-command              = " << dev_target_.ssh_command
      << std::endl;
-  ss << "dev-scp-command              = " << dev_target_.scp_command
+  ss << "dev-sftp-command             = " << dev_target_.sftp_command
      << std::endl;
   ss << "dev-mount-dir                = " << dev_target_.mount_dir << std::endl;
   return ss.str();

cdc_stream/cdc_fuse_manager.cc

@@ -16,6 +16,7 @@
 
 #include "absl/strings/match.h"
 #include "absl/strings/str_format.h"
+#include "absl/strings/str_split.h"
 #include "cdc_fuse_fs/constants.h"
 #include "common/gamelet_component.h"
 #include "common/log.h"
@@ -30,11 +31,29 @@ constexpr char kExeFilename[] = "cdc_stream.exe";
 constexpr char kFuseFilename[] = "cdc_fuse_fs";
 constexpr char kLibFuseFilename[] = "libfuse.so";
 constexpr char kFuseStdoutPrefix[] = "cdc_fuse_fs_stdout";
-constexpr char kRemoteToolsBinDir[] = "~/.cache/cdc-file-transfer/bin/";
+constexpr char kRemoteToolsBinDir[] = ".cache/cdc-file-transfer/bin/";
 
 // Cache directory on the gamelet to store data chunks.
 constexpr char kCacheDir[] = "~/.cache/cdc-file-transfer/chunks";
 
+// Parses the port from the FUSE stdout when FUSE is up-to-date. In that case,
+// the expected stdout is similar to "Port 12345 cdc_fuse_fs is up-to-date".
+absl::StatusOr<int> ParsePort(const std::string& fuse_stdout) {
+  // Search backwards until we find "Port ".
+  size_t port_pos = fuse_stdout.find(kFusePortPrefix);
+  if (port_pos == std::string::npos) {
+    return MakeStatus("Failed to find '%s' marker in server output '%s'",
+                      kFusePortPrefix, fuse_stdout);
+  }
+  int port =
+      atoi(fuse_stdout.substr(port_pos + strlen(kFusePortPrefix)).c_str());
+  if (port == 0) {
+    return MakeStatus("Failed to parse port from server output '%s'",
+                      fuse_stdout);
+  }
+  return port;
+}
+
 }  // namespace
 
 CdcFuseManager::CdcFuseManager(std::string instance,
@@ -54,32 +73,34 @@ absl::Status CdcFuseManager::Deploy() {
   std::string exe_dir;
   RETURN_IF_ERROR(path::GetExeDir(&exe_dir), "Failed to get exe directory");
 
-  // Set the cwd to the exe dir and pass the filenames to scp. Otherwise, some
+  // Create the remote tools bin dir if it doesn't exist yet.
-  // scp implementations can get confused and create the wrong remote filenames.
+  // This assumes that sftp's remote startup directory is the home directory.
-  path::SetCwd(exe_dir);
+  std::vector<std::string> dir_parts =
+      absl::StrSplit(kRemoteToolsBinDir, '/', absl::SkipEmpty());
+  std::string sftp_commands;
+  for (const std::string& dir : dir_parts) {
+    // Use -mkdir to ignore errors if the directory already exists.
+    sftp_commands += absl::StrFormat("-mkdir %s\ncd %s\n", dir, dir);
+  }
 
-  // Copy FUSE to the gamelet.
+  sftp_commands += absl::StrFormat("put %s\n", kFuseFilename);
-  LOG_DEBUG("Copying FUSE");
+  sftp_commands += absl::StrFormat("put %s\n", kLibFuseFilename);
-  RETURN_IF_ERROR(remote_util_->Scp({kFuseFilename, kLibFuseFilename},
+  sftp_commands += absl::StrFormat("chmod 755 %s\n", kFuseFilename);
-                                    kRemoteToolsBinDir, /*compress=*/false),
-                  "Failed to copy FUSE to gamelet");
-  LOG_DEBUG("Copying FUSE succeeded");
 
-  // Make FUSE executable. Note that sync does it automatically.
+  LOG_DEBUG("Deploying FUSE");
-  LOG_DEBUG("Making FUSE executable");
+  RETURN_IF_ERROR(
-  std::string remotePath = path::JoinUnix(kRemoteToolsBinDir, kFuseFilename);
+      remote_util_->Sftp(sftp_commands, exe_dir, /*compress=*/false),
-  RETURN_IF_ERROR(remote_util_->Chmod("a+x", remotePath),
+      "Failed to deploy FUSE");
-                  "Failed to set executable flag on FUSE");
+  LOG_DEBUG("Deploying FUSE succeeded");
-  LOG_DEBUG("Making FUSE succeeded");
 
   return absl::OkStatus();
 }
 
 absl::Status CdcFuseManager::Start(const std::string& mount_dir,
-                                   uint16_t local_port, uint16_t remote_port,
+                                   uint16_t local_port, int verbosity,
-                                   int verbosity, bool debug,
+                                   bool debug, bool singlethreaded,
-                                   bool singlethreaded, bool enable_stats,
+                                   bool enable_stats, bool check,
-                                   bool check, uint64_t cache_capacity,
+                                   uint64_t cache_capacity,
                                    uint32_t cleanup_timeout_sec,
                                    uint32_t access_idle_timeout_sec) {
   assert(!fuse_process_);
@@ -104,72 +125,111 @@ absl::Status CdcFuseManager::Start(const std::string& mount_dir,
   // Build the remote command.
   std::string remotePath = path::JoinUnix(kRemoteToolsBinDir, kFuseFilename);
   std::string remote_command = absl::StrFormat(
-      "mkdir -p %s; LD_LIBRARY_PATH=%s %s "
+      "LD_LIBRARY_PATH=%s %s "
       "--instance=%s "
-      "--components=%s --port=%i --cache_dir=%s "
+      "--components=%s --cache_dir=%s "
       "--verbosity=%i --cleanup_timeout=%i --access_idle_timeout=%i --stats=%i "
       "--check=%i --cache_capacity=%u -- -o allow_root -o ro -o nonempty -o "
       "auto_unmount %s%s%s",
-      kRemoteToolsBinDir, kRemoteToolsBinDir, remotePath,
+      kRemoteToolsBinDir, remotePath, RemoteUtil::QuoteForSsh(instance_),
-      RemoteUtil::QuoteForSsh(instance_),
+      RemoteUtil::QuoteForSsh(component_args), kCacheDir, verbosity,
-      RemoteUtil::QuoteForSsh(component_args), remote_port, kCacheDir,
+      cleanup_timeout_sec, access_idle_timeout_sec, enable_stats, check,
-      verbosity, cleanup_timeout_sec, access_idle_timeout_sec, enable_stats,
+      cache_capacity, debug ? "-d " : "", singlethreaded ? "-s " : "",
-      check, cache_capacity, debug ? "-d " : "", singlethreaded ? "-s " : "",
       RemoteUtil::QuoteForSsh(mount_dir));
 
   bool needs_deploy = false;
-  RETURN_IF_ERROR(
+  int remote_port;
-      RunFuseProcess(local_port, remote_port, remote_command, &needs_deploy));
+  ASSIGN_OR_RETURN(remote_port, RunFuseProcess(remote_command, &needs_deploy));
   if (needs_deploy) {
     // Deploy and try again.
     RETURN_IF_ERROR(Deploy());
-    RETURN_IF_ERROR(
+    ASSIGN_OR_RETURN(remote_port,
-        RunFuseProcess(local_port, remote_port, remote_command, &needs_deploy));
+                     RunFuseProcess(remote_command, &needs_deploy));
   }
 
+  // Start port forwarding.
+  RETURN_IF_ERROR(RunPortForwardingProcess(local_port, remote_port));
+
+  // Wait until port forwarding is up and FUSE can connect to |remote_port|.
+  RETURN_IF_ERROR(WaitForFuseConnected());
+
   return absl::OkStatus();
 }
 
-absl::Status CdcFuseManager::RunFuseProcess(uint16_t local_port,
+absl::StatusOr<int> CdcFuseManager::RunFuseProcess(
-                                            uint16_t remote_port,
+    const std::string& remote_command, bool* needs_deploy) {
-                                            const std::string& remote_command,
-                                            bool* needs_deploy) {
   assert(!fuse_process_);
   assert(needs_deploy);
   *needs_deploy = false;
 
   LOG_DEBUG("Running FUSE process");
-  ProcessStartInfo start_info =
+  ProcessStartInfo start_info = remote_util_->BuildProcessStartInfoForSsh(
-      remote_util_->BuildProcessStartInfoForSshPortForwardAndCommand(
+      remote_command, ArchType::kLinux_x86_64);
-          local_port, remote_port, true, remote_command);
   start_info.name = kFuseFilename;
 
   // Capture stdout to determine whether a deploy is required.
   fuse_stdout_.clear();
-  fuse_startup_finished_ = false;
+  fuse_port_ = 0;
-  start_info.stdout_handler = [this, needs_deploy](const char* data,
+  fuse_not_up_to_date_ = false;
-                                                   size_t size) {
+  fuse_update_check_finished_ = false;
-    return HandleFuseStdout(data, size, needs_deploy);
+  fuse_connected_ = false;
+  start_info.stdout_handler = [this](const char* data, size_t size) {
+    return HandleFuseStdout(data, size);
   };
   fuse_process_ = process_factory_->Create(start_info);
   RETURN_IF_ERROR(fuse_process_->Start(), "Failed to start FUSE process");
   LOG_DEBUG("FUSE process started. Waiting for startup to finish.");
 
   // Run until process exits or startup finishes.
-  auto startup_finished = [this]() { return fuse_startup_finished_.load(); };
+  RETURN_IF_ERROR(fuse_process_->RunUntil(
-  RETURN_IF_ERROR(fuse_process_->RunUntil(startup_finished),
+                      [this]() { return fuse_update_check_finished_.load(); }),
                   "Failed to run FUSE process");
-  LOG_DEBUG("FUSE process startup complete.");
+  LOG_DEBUG("FUSE process update check complete.");
 
   // If the FUSE process exited before it could perform its up-to-date check, it
   // most likely happens because the binary does not exist and needs to be
   // deployed.
-  *needs_deploy |= !fuse_startup_finished_ && fuse_process_->HasExited() &&
+  *needs_deploy = fuse_not_up_to_date_ ||
-                   fuse_process_->ExitCode() != 0;
+                  (!fuse_update_check_finished_ && fuse_process_->HasExited() &&
+                   fuse_process_->ExitCode() != 0);
   if (*needs_deploy) {
     LOG_DEBUG("FUSE needs to be (re-)deployed.");
     fuse_process_.reset();
+  }
+
+  return fuse_port_;
+}
+
+absl::Status CdcFuseManager::RunPortForwardingProcess(int local_port,
+                                                      int remote_port) {
+  assert(fuse_process_);
+  assert(!forwarding_process_);
+
+  LOG_DEBUG(
+      "Running reverse port forwarding process, local port %i, remote port %i",
+      local_port, remote_port);
+  ProcessStartInfo start_info =
+      remote_util_->BuildProcessStartInfoForSshPortForward(
+          local_port, remote_port, /*reverse=*/true);
+  forwarding_process_ = process_factory_->Create(start_info);
+  RETURN_IF_ERROR(forwarding_process_->Start(),
+                  "Failed to start port forwarding process");
+
   return absl::OkStatus();
+}
+
+absl::Status CdcFuseManager::WaitForFuseConnected() {
+  assert(fuse_process_);
+  assert(forwarding_process_);
+
+  RETURN_IF_ERROR(
+      fuse_process_->RunUntil([this]() { return fuse_connected_.load(); }),
+      "Failed to run FUSE process");
+  LOG_DEBUG("FUSE process connected.");
+
+  if (!fuse_connected_ && fuse_process_->HasExited()) {
+    return MakeStatus("FUSE exited during startup with code %u",
+                      fuse_process_->ExitCode());
   }
 
   return absl::OkStatus();
@@ -180,30 +240,37 @@ absl::Status CdcFuseManager::Stop() {
     return absl::OkStatus();
   }
 
-  LOG_DEBUG("Terminating FUSE process");
+  LOG_DEBUG("Terminating FUSE and port forwarding processes");
   absl::Status status = fuse_process_->Terminate();
+  status.Update(forwarding_process_->Terminate());
   fuse_process_.reset();
+  forwarding_process_.reset();
   return status;
 }
 
 bool CdcFuseManager::IsHealthy() const {
-  return fuse_process_ && !fuse_process_->HasExited();
+  return fuse_process_ && !fuse_process_->HasExited() && forwarding_process_ &&
+         !forwarding_process_->HasExited();
 }
 
-absl::Status CdcFuseManager::HandleFuseStdout(const char* data, size_t size,
+absl::Status CdcFuseManager::HandleFuseStdout(const char* data, size_t size) {
-                                              bool* needs_deploy) {
-  assert(needs_deploy);
-
   // Don't capture stdout beyond startup.
-  if (!fuse_startup_finished_) {
+  if (!fuse_connected_) {
     fuse_stdout_.append(data, size);
-    // The gamelet component prints some magic strings to stdout to indicate
+
+    // The remote component prints some magic strings to stdout to indicate
     // whether it's up-to-date.
     if (absl::StrContains(fuse_stdout_, kFuseUpToDate)) {
-      fuse_startup_finished_ = true;
+      ASSIGN_OR_RETURN(fuse_port_, ParsePort(fuse_stdout_));
+      fuse_update_check_finished_ = true;
     } else if (absl::StrContains(fuse_stdout_, kFuseNotUpToDate)) {
-      fuse_startup_finished_ = true;
+      fuse_not_up_to_date_ = true;
-      *needs_deploy = true;
+      fuse_update_check_finished_ = true;
+    }
+
+    // It also prints stuff when it can connect to its port.
+    if (absl::StrContains(fuse_stdout_, kFuseConnected)) {
+      fuse_connected_ = true;
     }
   }
 

cdc_stream/cdc_fuse_manager.h

@@ -18,6 +18,7 @@
 #define CDC_STREAM_CDC_FUSE_MANAGER_H_
 
 #include "absl/status/status.h"
+#include "absl/status/statusor.h"
 #include "common/remote_util.h"
 
 namespace cdc_ft {
@@ -26,7 +27,7 @@ class Process;
 class ProcessFactory;
 class RemoteUtil;
 
-// Manages the gamelet-side CDC FUSE filesystem process.
+// Manages the remote CDC FUSE filesystem process.
 class CdcFuseManager {
  public:
   CdcFuseManager(std::string instance, ProcessFactory* process_factory,
@@ -36,11 +37,10 @@ class CdcFuseManager {
   CdcFuseManager(CdcFuseManager&) = delete;
   CdcFuseManager& operator=(CdcFuseManager&) = delete;
 
-  // Starts the CDC FUSE and establishes a reverse SSH tunnel from the gamelet's
+  // Starts the remote CDC FUSE process. Deploys the binary if necessary.
-  // |remote_port| to the workstation's |local_port|. Deploys the binary if
-  // necessary.
   //
   // |mount_dir| is the remote directory where to mount the FUSE.
+  // |local_port| is the local port used for gRPC connections to the FUSE.
   // |verbosity| is the log verbosity used by the filesystem.
   // |debug| puts the filesystem into debug mode if set to true. This also
   // causes the process to run in the foreground, so that logs are piped through
@@ -53,9 +53,9 @@ class CdcFuseManager {
   // |access_idle_timeout_sec| defines the number of seconds after which data
   // provider is considered to be access-idling.
   absl::Status Start(const std::string& mount_dir, uint16_t local_port,
-                     uint16_t remote_port, int verbosity, bool debug,
+                     int verbosity, bool debug, bool singlethreaded,
-                     bool singlethreaded, bool enable_stats, bool check,
+                     bool enable_stats, bool check, uint64_t cache_capacity,
-                     uint64_t cache_capacity, uint32_t cleanup_timeout_sec,
+                     uint32_t cleanup_timeout_sec,
                      uint32_t access_idle_timeout_sec);
 
   // Stops the CDC FUSE.
@@ -65,33 +65,49 @@ class CdcFuseManager {
   bool IsHealthy() const;
 
  private:
-  // Runs the FUSE process on the gamelet from the given |remote_command| and
+  // Runs the remote FUSE process from the given |remote_command|. Returns the
-  // establishes a reverse SSH tunnel from the gamelet's |remote_port| to the
+  // remote port that the FUSE will connect to, once the port forwarding process
-  // workstation's |local_port|.
+  // is up.
   //
   // If the FUSE is not up-to-date or does not exist, sets |needs_deploy| to
   // true and returns OK. In that case, Deploy() needs to be called and the FUSE
   // process should be run again.
-  absl::Status RunFuseProcess(uint16_t local_port, uint16_t remote_port,
+  absl::StatusOr<int> RunFuseProcess(const std::string& remote_command,
-                              const std::string& remote_command,
                                      bool* needs_deploy);
 
-  // Deploys the gamelet components.
+  // Establishes a reverse SSH tunnel from |remote_port| to |local_port|.
+  absl::Status RunPortForwardingProcess(int local_port, int remote_port);
+
+  // Waits until FUSE can connect to its port. This essentially means that
+  // port forwarding is up and running.
+  absl::Status WaitForFuseConnected();
+
+  // Deploys the remote components.
   absl::Status Deploy();
 
-  // Output handler for FUSE's stdout. Sets |needs_deploy| to true if the output
+  // Output handler for FUSE's stdout.
-  // contains a magic marker to indicate that the binary has to be redeployed.
+  // Sets |fuse_not_up_to_date_| to true if the output contains a magic marker
-  // Called in a background thread.
+  // to indicate that the binary has to be redeployed.
-  absl::Status HandleFuseStdout(const char* data, size_t size,
+  // Sets |fuse_port_| to the remote gRPC port if the output contains a magic
-                                bool* needs_deploy);
+  // marker that has the port and indicates that the binary is up-to-date.
+  // Sets |fuse_update_check_finished_| to true if any of the above two markers
+  // was set. Called in a background thread.
+  // Sets |fuse_startup_finished_| to true if FUSE is connected to its port.
+  absl::Status HandleFuseStdout(const char* data, size_t size);
 
   std::string instance_;
   ProcessFactory* const process_factory_;
   RemoteUtil* const remote_util_;
 
   std::unique_ptr<Process> fuse_process_;
+  std::unique_ptr<Process> forwarding_process_;
   std::string fuse_stdout_;
-  std::atomic<bool> fuse_startup_finished_{false};
+
+  // Set by HandleFuseStdout
+  int fuse_port_ = 0;
+  bool fuse_not_up_to_date_ = false;
+  std::atomic_bool fuse_update_check_finished_{false};
+  std::atomic_bool fuse_connected_{false};
 };
 
 }  // namespace cdc_ft

cdc_stream/local_assets_stream_manager_client.cc

@@ -38,13 +38,13 @@ LocalAssetsStreamManagerClient::~LocalAssetsStreamManagerClient() = default;
 absl::Status LocalAssetsStreamManagerClient::StartSession(
     const std::string& src_dir, const std::string& user_host,
     const std::string& mount_dir, const std::string& ssh_command,
-    const std::string& scp_command) {
+    const std::string& sftp_command) {
   StartSessionRequest request;
   request.set_workstation_directory(src_dir);
   request.set_user_host(user_host);
   request.set_mount_dir(mount_dir);
   request.set_ssh_command(ssh_command);
-  request.set_scp_command(scp_command);
+  request.set_sftp_command(sftp_command);
 
   grpc::ClientContext context;
   StartSessionResponse response;

cdc_stream/local_assets_stream_manager_client.h

@@ -43,12 +43,12 @@ class LocalAssetsStreamManagerClient {
   // |user_host| is the Linux host, formatted as [user@:host].
   // |mount_dir| is the Linux target directory to stream to.
   // |ssh_command| is the ssh command and extra arguments to use.
-  // |scp_command| is the scp command and extra arguments to use.
+  // |sftp_command| is the sftp command and extra arguments to use.
   absl::Status StartSession(const std::string& src_dir,
                             const std::string& user_host,
                             const std::string& mount_dir,
                             const std::string& ssh_command,
-                            const std::string& scp_command);
+                            const std::string& sftp_command);
 
   // Stops the streaming session to the Linux target |user_host|:|mount_dir|.
   // |user_host| is the Linux host, formatted as [user@:host].

cdc_stream/local_assets_stream_manager_service_impl.cc

@@ -208,7 +208,7 @@ LocalAssetsStreamManagerServiceImpl::GetTargetForStadia(
   SessionTarget target;
   target.mount_dir = request.mount_dir();
   target.ssh_command = request.ssh_command();
-  target.scp_command = request.scp_command();
+  target.sftp_command = request.sftp_command();
 
   // Parse instance/project/org id.
   if (!ParseInstanceName(request.gamelet_name(), instance_id, project_id,
@@ -223,7 +223,7 @@ LocalAssetsStreamManagerServiceImpl::GetTargetForStadia(
                    InitSsh(*instance_id, *project_id, *organization_id));
 
   target.user_host = "cloudcast@" + instance_ip;
-  // Note: Port must be set with ssh_command (-p) and scp_command (-P).
+  // Note: Port must be set with ssh_command (-p) and sftp_command (-P).
   return target;
 }
 
@@ -233,7 +233,7 @@ SessionTarget LocalAssetsStreamManagerServiceImpl::GetTarget(
   target.user_host = request.user_host();
   target.mount_dir = request.mount_dir();
   target.ssh_command = request.ssh_command();
-  target.scp_command = request.scp_command();
+  target.sftp_command = request.sftp_command();
 
   *instance_id = absl::StrCat(target.user_host, ":", target.mount_dir);
   return target;

cdc_stream/multi_session.cc

@@ -22,6 +22,7 @@
 #include "common/platform.h"
 #include "common/port_manager.h"
 #include "common/process.h"
+#include "common/server_socket.h"
 #include "common/util.h"
 #include "data_store/disk_data_store.h"
 #include "manifest/content_id.h"
@@ -436,19 +437,8 @@ absl::Status MultiSession::Initialize() {
   }
 
   // Find an available local port.
-  local_asset_stream_port_ = cfg_.forward_port_first;
+  ASSIGN_OR_RETURN(local_asset_stream_port_, ServerSocket::FindAvailablePort(),
-  if (cfg_.forward_port_first < cfg_.forward_port_last) {
+                   "Failed to find an available local port");
-    std::unordered_set<int> ports;
-    ASSIGN_OR_RETURN(
-        ports,
-        PortManager::FindAvailableLocalPorts(cfg_.forward_port_first,
-                                             cfg_.forward_port_last,
-                                             "127.0.0.1", process_factory_),
-        "Failed to find an available local port in the range [%d, %d]",
-        cfg_.forward_port_first, cfg_.forward_port_last);
-    assert(!ports.empty());
-    local_asset_stream_port_ = *ports.begin();
-  }
 
   assert(!runner_);
   runner_ = std::make_unique<MultiSessionRunner>(
@@ -523,9 +513,7 @@ absl::Status MultiSession::StartSession(const std::string& instance_id,
 
   auto session = std::make_unique<Session>(
       instance_id, target, cfg_, process_factory_, std::move(metrics_recorder));
-  RETURN_IF_ERROR(session->Start(local_asset_stream_port_,
+  RETURN_IF_ERROR(session->Start(local_asset_stream_port_));
-                                 cfg_.forward_port_first,
-                                 cfg_.forward_port_last));
 
   // Wait for the FUSE to receive the first intermediate manifest.
   RETURN_IF_ERROR(runner_->WaitForManifestAck(instance_id, absl::Seconds(5)));

cdc_stream/multi_session_test.cc

@@ -158,7 +158,7 @@ class MultiSessionTest : public ManifestTestBase {
     EXPECT_EQ(data->file_count, file_count);
     EXPECT_EQ(data->min_chunk_size, 128 << 10);
     EXPECT_EQ(data->avg_chunk_size, 256 << 10);
-    EXPECT_EQ(data->max_chunk_size, 1024 << 10);
+    EXPECT_EQ(data->max_chunk_size, 512 << 10);
   }
 
   metrics::ManifestUpdateData GetManifestUpdateData(

cdc_stream/session.cc

@@ -55,8 +55,8 @@ Session::Session(std::string instance_id, const SessionTarget& target,
   if (!target.ssh_command.empty()) {
     remote_util_.SetSshCommand(target.ssh_command);
   }
-  if (!target.scp_command.empty()) {
+  if (!target.sftp_command.empty()) {
-    remote_util_.SetScpCommand(target.scp_command);
+    remote_util_.SetSftpCommand(target.sftp_command);
   }
 }
 
@@ -68,33 +68,18 @@ Session::~Session() {
   }
 }
 
-absl::Status Session::Start(int local_port, int first_remote_port,
+absl::Status Session::Start(int local_port) {
-                            int last_remote_port) {
-  // Find an available remote port.
-  int remote_port = first_remote_port;
-  if (first_remote_port < last_remote_port) {
-    std::unordered_set<int> ports;
-    ASSIGN_OR_RETURN(
-        ports,
-        PortManager::FindAvailableRemotePorts(
-            first_remote_port, last_remote_port, "127.0.0.1", process_factory_,
-            &remote_util_, kInstanceConnectionTimeoutSec),
-        "Failed to find an available remote port in the range [%d, %d]",
-        first_remote_port, last_remote_port);
-    assert(!ports.empty());
-    remote_port = *ports.begin();
-  }
-
   assert(!fuse_);
   fuse_ = std::make_unique<CdcFuseManager>(instance_id_, process_factory_,
                                            &remote_util_);
+
   RETURN_IF_ERROR(
-      fuse_->Start(mount_dir_, local_port, remote_port, cfg_.verbosity,
+      fuse_->Start(mount_dir_, local_port, cfg_.verbosity, cfg_.fuse_debug,
-                   cfg_.fuse_debug, cfg_.fuse_singlethreaded, cfg_.stats,
+                   cfg_.fuse_singlethreaded, cfg_.stats, cfg_.fuse_check,
-                   cfg_.fuse_check, cfg_.fuse_cache_capacity,
+                   cfg_.fuse_cache_capacity, cfg_.fuse_cleanup_timeout_sec,
-                   cfg_.fuse_cleanup_timeout_sec,
                    cfg_.fuse_access_idle_timeout_sec),
       "Failed to start instance component");
+
   return absl::OkStatus();
 }
 

cdc_stream/session.h

@@ -38,8 +38,8 @@ struct SessionTarget {
   std::string user_host;
   // Ssh command to use to connect to the remote target.
   std::string ssh_command;
-  // Scp command to use to copy files to the remote target.
+  // Sftp command to use to copy files to the remote target.
-  std::string scp_command;
+  std::string sftp_command;
   // Directory on the remote target where to mount the streamed directory.
   std::string mount_dir;
 };
@@ -58,9 +58,7 @@ class Session {
 
   // Starts the CDC FUSE on the instance with established port forwarding.
   // |local_port| is the local reverse forwarding port to use.
-  // [|first_remote_port|, |last_remote_port|] are the allowed remote ports.
+  absl::Status Start(int local_port);
-  absl::Status Start(int local_port, int first_remote_port,
-                     int last_remote_port);
 
   // Shuts down the connection to the instance.
   absl::Status Stop() ABSL_LOCKS_EXCLUDED(transferred_data_mu_);

cdc_stream/session_config.h

@@ -57,9 +57,10 @@ struct SessionConfig {
   // Time to wait until running a manifest update after detecting a file change.
   uint32_t file_change_wait_duration_ms = 0;
 
-  // Ports used for local port forwarding.
+  // Ports used for local port forwarding. Deprecated as forward ports are
-  uint16_t forward_port_first = 0;
+  // determined automatically now using ephemeral ports.
-  uint16_t forward_port_last = 0;
+  uint16_t deprecated_forward_port_first = 0;
+  uint16_t deprecated_forward_port_last = 0;
 };
 
 }  // namespace cdc_ft

cdc_stream/start_command.cc

@@ -22,6 +22,7 @@
 #include "common/log.h"
 #include "common/path.h"
 #include "common/process.h"
+#include "common/remote_util.h"
 #include "common/status_macros.h"
 #include "common/stopwatch.h"
 #include "common/util.h"
@@ -73,17 +74,28 @@ void StartCommand::RegisterCommandLineFlags(lyra::command& cmd) {
 
   path::GetEnv("CDC_SSH_COMMAND", &ssh_command_).IgnoreError();
   cmd.add_argument(
-      lyra::opt(ssh_command_, "ssh_command")
+      lyra::opt(ssh_command_, "cmd")
           .name("--ssh-command")
           .help("Path and arguments of ssh command to use, e.g. "
                 "\"C:\\path\\to\\ssh.exe -F config_file -p 1234\". Can also be "
                 "specified by the CDC_SSH_COMMAND environment variable."));
 
-  path::GetEnv("CDC_SCP_COMMAND", &scp_command_).IgnoreError();
+  path::GetEnv("CDC_SFTP_COMMAND", &sftp_command_).IgnoreError();
   cmd.add_argument(
-      lyra::opt(scp_command_, "scp_command")
+      lyra::opt(sftp_command_, "cmd")
+          .name("--sftp-command")
+          .help(
+              "Path and arguments of sftp command to use, e.g. "
+              "\"C:\\path\\to\\sftp.exe -F config_file -P 1234\". Can also be "
+              "specified by the CDC_SFTP_COMMAND environment variable."));
+
+  path::GetEnv("CDC_SCP_COMMAND", &deprecated_scp_command_).IgnoreError();
+  cmd.add_argument(
+      lyra::opt(deprecated_scp_command_, "cmd")
           .name("--scp-command")
-          .help("Path and arguments of scp command to use, e.g. "
+          .help("[Deprecated, use --sftp-command] Path and arguments of scp "
+                "command to "
+                "use, e.g. "
                 "\"C:\\path\\to\\scp.exe -F config_file -P 1234\". Can also be "
                 "specified by the CDC_SCP_COMMAND environment variable."));
 
@@ -106,9 +118,26 @@ absl::Status StartCommand::Run() {
   RETURN_IF_ERROR(LocalAssetsStreamManagerClient::ParseUserHostDir(
       user_host_dir_, &user_host, &mount_dir));
 
+  // Backwards compatibility after switching from scp to sftp.
+  if (sftp_command_.empty() && !deprecated_scp_command_.empty()) {
+    LOG_WARNING(
+        "The CDC_SCP_COMMAND environment variable and the --scp-command flag "
+        "are deprecated. Please set CDC_SFTP_COMMAND or --sftp-command "
+        "instead.");
+
+    sftp_command_ = RemoteUtil::ScpToSftpCommand(deprecated_scp_command_);
+    if (!sftp_command_.empty()) {
+      LOG_WARNING("Converted scp command '%s' to sftp command '%s'.",
+                  deprecated_scp_command_, sftp_command_);
+    } else {
+      LOG_WARNING("Failed to convert scp command '%s' to sftp command.",
+                  deprecated_scp_command_);
+    }
+  }
+
   LocalAssetsStreamManagerClient client(CreateChannel(service_port_));
   absl::Status status = client.StartSession(full_src_dir, user_host, mount_dir,
-                                            ssh_command_, scp_command_);
+                                            ssh_command_, sftp_command_);
 
   if (absl::IsUnavailable(status)) {
     LOG_DEBUG("StartSession status: %s", status.ToString());
@@ -122,7 +151,7 @@ absl::Status StartCommand::Run() {
       // state.
       LocalAssetsStreamManagerClient new_client(CreateChannel(service_port_));
       status = new_client.StartSession(full_src_dir, user_host, mount_dir,
-                                       ssh_command_, scp_command_);
+                                       ssh_command_, sftp_command_);
     }
   }
 

cdc_stream/start_command.h

@@ -44,9 +44,11 @@ class StartCommand : public BaseCommand {
   int verbosity_ = 0;
   uint16_t service_port_ = 0;
   std::string ssh_command_;
-  std::string scp_command_;
+  std::string sftp_command_;
   std::string src_dir_;
   std::string user_host_dir_;
+
+  std::string deprecated_scp_command_;
 };
 
 }  // namespace cdc_ft

cdc_stream/start_service_command.cc

@@ -141,7 +141,7 @@ absl::Status StartServiceCommand::RunService() {
     request.set_user_host(cfg_.dev_target().user_host);
     request.set_mount_dir(cfg_.dev_target().mount_dir);
     request.set_ssh_command(cfg_.dev_target().ssh_command);
-    request.set_scp_command(cfg_.dev_target().scp_command);
+    request.set_sftp_command(cfg_.dev_target().sftp_command);
     localassetsstreammanager::StartSessionResponse response;
     RETURN_ABSL_IF_ERROR(
         session_service.StartSession(nullptr, &request, &response));

common/BUILD

@@ -2,6 +2,40 @@ load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test")
 
 package(default_visibility = ["//visibility:public"])
 
+cc_library(
+    name = "ansi_filter",
+    srcs = ["ansi_filter.cc"],
+    hdrs = ["ansi_filter.h"],
+)
+
+cc_test(
+    name = "ansi_filter_test",
+    srcs = ["ansi_filter_test.cc"],
+    deps = [
+        ":ansi_filter",
+        "@com_google_googletest//:gtest",
+        "@com_google_googletest//:gtest_main",
+    ],
+)
+
+cc_library(
+    name = "arch_type",
+    srcs = ["arch_type.cc"],
+    hdrs = ["arch_type.h"],
+    deps = [":platform"],
+)
+
+cc_test(
+    name = "arch_type_test",
+    srcs = ["arch_type_test.cc"],
+    deps = [
+        ":arch_type",
+        "@com_google_absl//absl/strings",
+        "@com_google_googletest//:gtest",
+        "@com_google_googletest//:gtest_main",
+    ],
+)
+
 cc_library(
     name = "buffer",
     srcs = ["buffer.cc"],
@@ -18,6 +52,35 @@ cc_test(
     ],
 )
 
+cc_library(
+    name = "build_version",
+    srcs = ["build_version.cc"],
+    hdrs = ["build_version.h"],
+    # This definition should be replaced by release flow.
+    copts = ["-DCDC_BUILD_VERSION=DEV"],
+)
+
+cc_library(
+    name = "client_socket",
+    srcs = [
+        "client_socket.cc",
+        "socket_internal.h",
+    ],
+    hdrs = ["client_socket.h"],
+    linkopts = select({
+        "//tools:windows": [
+            "/DEFAULTLIB:Ws2_32.lib",  # Sockets, e.g. recv, send, WSA*.
+        ],
+        "//conditions:default": [],
+    }),
+    deps = [
+        ":log",
+        ":socket",
+        ":status",
+        ":util",
+    ],
+)
+
 cc_library(
     name = "clock",
     srcs = ["clock.cc"],
@@ -85,6 +148,16 @@ cc_test(
     ],
 )
 
+cc_library(
+    name = "fake_socket",
+    srcs = ["fake_socket.cc"],
+    hdrs = ["fake_socket.h"],
+    deps = [
+        ":socket",
+        "@com_google_absl//absl/status",
+    ],
+)
+
 cc_library(
     name = "file_watcher",
     srcs = [
@@ -142,6 +215,7 @@ cc_library(
     deps = [
         ":clock",
         ":platform",
+        ":stopwatch",
         "@com_google_absl//absl/strings:str_format",
         "@com_google_absl//absl/synchronization",
     ],
@@ -232,6 +306,7 @@ cc_library(
     hdrs = ["port_manager.h"],
     target_compatible_with = ["@platforms//os:windows"],
     deps = [
+        ":arch_type",
         ":remote_util",
         ":status",
         ":stopwatch",
@@ -338,6 +413,7 @@ cc_library(
     srcs = ["remote_util.cc"],
     hdrs = ["remote_util.h"],
     deps = [
+        ":arch_type",
         ":platform",
         ":process",
         ":sdk_util",
@@ -379,6 +455,7 @@ cc_library(
         ":path",
         ":platform",
         ":status",
+        "//common:build_version",
         "@com_google_absl//absl/status",
         "@com_google_absl//absl/strings:str_format",
     ],
@@ -396,6 +473,42 @@ cc_test(
     ],
 )
 
+cc_library(
+    name = "server_socket",
+    srcs = [
+        "server_socket.cc",
+        "socket_internal.h",
+    ],
+    hdrs = ["server_socket.h"],
+    linkopts = select({
+        "//tools:windows": [
+            "/DEFAULTLIB:Ws2_32.lib",  # Sockets, e.g. recv, send, WSA*.
+        ],
+        "//conditions:default": [],
+    }),
+    deps = [
+        ":log",
+        ":socket",
+        ":status",
+        ":util",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/status:statusor",
+    ],
+)
+
+cc_library(
+    name = "socket",
+    srcs = ["socket.cc"],
+    hdrs = ["socket.h"],
+    deps = [
+        ":log",
+        ":platform",
+        ":status",
+        ":util",
+        "@com_google_absl//absl/status",
+    ],
+)
+
 cc_library(
     name = "stats_collector",
     srcs = ["stats_collector.cc"],

common/ansi_filter.cc

@@ -0,0 +1,103 @@
+// Copyright 2023 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "common/ansi_filter.h"
+
+namespace cdc_ft {
+namespace ansi_filter {
+namespace {
+enum class State {
+  kNotInSequence,
+  kDCS,  // Starting with kESC + P or kDCSI, Device Control String.
+  kCS,   // Starting with kESC + [ or kCSI,  Control Sequence.
+  kOSC,  // Starting with kESC + ] or kOSCI, Operating System Command.
+};
+
+constexpr uint8_t kBEL = 0x07;   // Terminal bell.
+constexpr uint8_t kESC = 0x1B;   // ANSI escape character.
+constexpr uint8_t kST = 0x9C;    // String Terminator.
+constexpr uint8_t kDCSI = 0x90;  // Device Control String Introducer.
+constexpr uint8_t kCSI = 0x9B;   // Control Sequence Introducer.
+constexpr uint8_t kOSCI = 0x9D;  // Operating System Command Introducer
+
+}  // namespace
+
+std::string RemoveEscapeSequences(const std::string& input) {
+  State state = State::kNotInSequence;
+  std::string result;
+
+  for (size_t n = 0; n < input.size(); ++n) {
+    uint8_t ch = static_cast<uint8_t>(input[n]);
+    uint8_t next_ch =
+        static_cast<uint8_t>(n + 1 < input.size() ? input[n + 1] : 0);
+
+    switch (state) {
+      case State::kNotInSequence:
+        // Device Control String.
+        if ((ch == kESC && next_ch == 'P') || ch == kDCSI) {
+          n += ch == kESC ? 1 : 0;
+          state = State::kDCS;
+          break;
+        }
+
+        // Control Sequence.
+        if ((ch == kESC && next_ch == '[') || ch == kCSI) {
+          n += ch == kESC ? 1 : 0;
+          state = State::kCS;
+          break;
+        }
+
+        // Operating System Command.
+        if ((ch == kESC && next_ch == ']') || ch == kOSCI) {
+          n += ch == kESC ? 1 : 0;
+          state = State::kOSC;
+          break;
+        }
+
+        // Char does not belong to control sequence.
+        result.push_back(ch);
+        break;
+
+      case State::kDCS:
+        // Device control strings are ended by kST or ESC + \.
+        if (ch == kST || (ch == kESC && next_ch == '\\')) {
+          n += ch == kESC ? 1 : 0;
+          state = State::kNotInSequence;
+        }
+        break;
+
+      case State::kCS:
+        // Control sequence initializer are ended by a byte in 0x40�0x7E.
+        // https://en.wikipedia.org/wiki/ANSI_escape_code#CSIsection
+        if (ch >= 0x40 && ch <= 0x7E) {
+          state = State::kNotInSequence;
+        }
+        break;
+
+      case State::kOSC:
+        // Operating system commands are ended by kBEL, kST or ESC + \.
+        // https://invisible-island.net/xterm/ctlseqs/ctlseqs.html#h3-Operating-System-Commands
+        if (ch == kBEL || ch == kST || (ch == kESC && next_ch == '\\')) {
+          n += ch == kESC ? 1 : 0;
+          state = State::kNotInSequence;
+        }
+        break;
+    }
+  }
+
+  return result;
+}
+
+}  // namespace ansi_filter
+}  // namespace cdc_ft

common/ansi_filter.h

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef COMMON_ANSI_FILTER_H_
+#define COMMON_ANSI_FILTER_H_
+
+#include <string>
+
+namespace cdc_ft {
+namespace ansi_filter {
+
+// Removes ANSI escape sequences from a string.
+// |input| is a string that can contain ANSI escape sequences.
+// Returns the filtered string with ANSI escape sequences removed.
+// Example: The most common escape sequence sets a color, e.g.
+//            "This \x1b[1;32merror\x1b[0m is red."
+//          The filtered output is
+//            "This error is red."
+std::string RemoveEscapeSequences(const std::string& input);
+
+}  // namespace ansi_filter
+}  // namespace cdc_ft
+
+#endif  // COMMON_ANSI_FILTER_H_

common/ansi_filter_test.cc

@@ -0,0 +1,98 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "common/ansi_filter.h"
+
+#include "absl/strings/ascii.h"
+#include "gtest/gtest.h"
+
+namespace cdc_ft {
+namespace {
+
+// Actual sample output from running SSH with -tt on Windows.
+// Note the \0 after cmd.exe.
+constexpr char kSshOutput[] =
+    "\x1b[2J\x1b[?25l\x1b[m\x1b["
+    "H\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n"
+    "\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\x1b[H\x1b]0;c:"
+    "\\windows\\system32\\cmd.exe\0\a\x1b[?25h\x1b[?25'l\x1b[120X\x1b["
+    "120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b["
+    "120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b["
+    "120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b["
+    "120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b["
+    "120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b["
+    "120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b["
+    "120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b["
+    "120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b["
+    "120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b["
+    "120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b[120C\x1b[H\x1b[?25h "
+    "\x1b[H\x1b[?25l\r\nfoo";
+
+TEST(AnsiFilterTest, DoesNotExplodeOnEmptyString) {
+  EXPECT_EQ(ansi_filter::RemoveEscapeSequences(""), "");
+}
+
+TEST(AnsiFilterTest, KeepsUnescapedString) {
+  constexpr char kStr[] = "Lorem ipsum";
+  EXPECT_EQ(ansi_filter::RemoveEscapeSequences(kStr), kStr);
+}
+
+TEST(AnsiFilterTest, RemovesDeviceControlString) {
+  // Special commands for the device.
+  EXPECT_EQ(ansi_filter::RemoveEscapeSequences("foo\x1bPparams\x1b\\bar"),
+            "foobar");
+  EXPECT_EQ(ansi_filter::RemoveEscapeSequences("foo\x90params\x9c"
+                                               "bar"),
+            "foobar");
+}
+
+TEST(AnsiFilterTest, RemovesControlSequenceIntroducer) {
+  // E.g. the well-known regular ANSI color codes.
+  EXPECT_EQ(ansi_filter::RemoveEscapeSequences("foo\x1b[01;32mbar"), "foobar");
+  EXPECT_EQ(ansi_filter::RemoveEscapeSequences("foo\x9b"
+                                               "01;32mbar"),
+            "foobar");
+}
+
+TEST(AnsiFilterTest, RemovesOperatingSystemCommand) {
+  // E.g. setting the Window title.
+  // Not cool: OS commands can contain null-terminated string.
+  std::string str = "foo\x1b]0;c:\\path\\to\\foo.exe";
+  str.append(1, '\0');
+  str.append("\abar");
+  EXPECT_EQ(ansi_filter::RemoveEscapeSequences(str), "foobar");
+  EXPECT_EQ(ansi_filter::RemoveEscapeSequences("foo\x9dstring\x1b\\bar"),
+            "foobar");
+}
+
+TEST(AnsiFilterTest, RemovesRestIfNotTerminated) {
+  EXPECT_EQ(ansi_filter::RemoveEscapeSequences("foo\x1b[01;32"), "foo");
+}
+
+TEST(AnsiFilterTest, RemovesSequencesFromActualSshOutput) {
+  // Note: Can't just say str = kSshOutput because of the \0 in the string.
+  std::string str = std::string(kSshOutput, sizeof(kSshOutput) - 1);
+  std::string res = std::string(
+      absl::StripAsciiWhitespace(ansi_filter::RemoveEscapeSequences(str)));
+  EXPECT_EQ(res, "foo");
+}
+
+TEST(AnsiFilterTest, WorksForExampleFromDocumentation) {
+  std::string str = "This \x1b[1;32merror\x1b[0m is red.";
+  std::string res = std::string(ansi_filter::RemoveEscapeSequences(str));
+  EXPECT_EQ(res, "This error is red.");
+}
+
+}  // namespace
+}  // namespace cdc_ft

common/arch_type.cc

@@ -0,0 +1,70 @@
+// Copyright 2023 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "common/arch_type.h"
+
+#include <cassert>
+
+#include "common/platform.h"
+
+namespace cdc_ft {
+
+static constexpr char kUnhandledArchType[] = "Unhandled arch type";
+
+ArchType GetLocalArchType() {
+  // TODO(ljusten): Take CPU architecture into account.
+#if PLATFORM_WINDOWS
+  return ArchType::kWindows_x86_64;
+#elif PLATFORM_LINUX
+  return ArchType::kLinux_x86_64;
+#endif
+}
+
+bool IsWindowsArchType(ArchType arch_type) {
+  switch (arch_type) {
+    case ArchType::kWindows_x86_64:
+      return true;
+    case ArchType::kLinux_x86_64:
+      return false;
+    default:
+      assert(!kUnhandledArchType);
+      return false;
+  }
+}
+
+bool IsLinuxArchType(ArchType arch_type) {
+  switch (arch_type) {
+    case ArchType::kWindows_x86_64:
+      return false;
+    case ArchType::kLinux_x86_64:
+      return true;
+    default:
+      assert(!kUnhandledArchType);
+      return false;
+  }
+}
+
+const char* GetArchTypeStr(ArchType arch_type) {
+  switch (arch_type) {
+    case ArchType::kWindows_x86_64:
+      return "Windows_x86_64";
+    case ArchType::kLinux_x86_64:
+      return "Linux_x86_64";
+    default:
+      assert(!kUnhandledArchType);
+      return "Unknown";
+  }
+}
+
+}  // namespace cdc_ft

common/arch_type.h

@@ -0,0 +1,41 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef COMMON_ARCH_TYPE_H_
+#define COMMON_ARCH_TYPE_H_
+
+namespace cdc_ft {
+
+enum class ArchType {
+  kWindows_x86_64 = 0,
+  kLinux_x86_64 = 1,
+};
+
+// Returns the arch type of the current process.
+ArchType GetLocalArchType();
+
+// Returns true if |arch_type| is a Windows operating system.
+bool IsWindowsArchType(ArchType arch_type);
+
+// Returns true if |arch_type| is a Linux operating system.
+bool IsLinuxArchType(ArchType arch_type);
+
+// Returns a human readable string for |arch_type|.
+const char* GetArchTypeStr(ArchType arch_type);
+
+}  // namespace cdc_ft
+
+#endif  // COMMON_ARCH_TYPE_H_

common/arch_type_test.cc

@@ -0,0 +1,49 @@
+// Copyright 2023 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "common/arch_type.h"
+
+#include "absl/strings/match.h"
+#include "gtest/gtest.h"
+
+namespace cdc_ft {
+namespace {
+
+TEST(ArchTypeTest, GetLocalArchType) {
+#if PLATFORM_WINDOWS
+  EXPECT_TRUE(IsPlatformWindows(GetLocalArchType()));
+#elif PLATFORM_LINUX
+  EXPECT_TRUE(IsPlatformLinux(GetLocalArchType()));
+#endif
+}
+
+TEST(ArchTypeTest, IsWindowsArchType) {
+  EXPECT_TRUE(IsWindowsArchType(ArchType::kWindows_x86_64));
+  EXPECT_FALSE(IsWindowsArchType(ArchType::kLinux_x86_64));
+}
+
+TEST(ArchTypeTest, IsLinuxArchType) {
+  EXPECT_FALSE(IsLinuxArchType(ArchType::kWindows_x86_64));
+  EXPECT_TRUE(IsLinuxArchType(ArchType::kLinux_x86_64));
+}
+
+TEST(ArchTypeTest, GetArchTypeStr) {
+  EXPECT_TRUE(
+      absl::StrContains(GetArchTypeStr(ArchType::kWindows_x86_64), "Windows"));
+  EXPECT_TRUE(
+      absl::StrContains(GetArchTypeStr(ArchType::kLinux_x86_64), "Linux"));
+}
+
+}  // namespace
+}  // namespace cdc_ft

common/build_version.cc

@@ -0,0 +1,9 @@
+#include "build_version.h"
+
+#ifdef CDC_BUILD_VERSION
+#define TO_STR(arg) #arg
+#define TO_STR_VALUE(arg) TO_STR(arg)
+const char* BUILD_VERSION = TO_STR_VALUE(CDC_BUILD_VERSION);
+#else
+const char* BUILD_VERSION = DEV_BUILD_VERSION;
+#endif

common/build_version.h

@@ -0,0 +1,7 @@
+#ifndef COMMON_BUILD_VERSION_H_
+#define COMMON_BUILD_VERSION_H_
+
+#define DEV_BUILD_VERSION "DEV"
+extern const char* BUILD_VERSION;
+
+#endif

common/client_socket.cc

@@ -12,15 +12,14 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "cdc_rsync/client_socket.h"
+#include "common/client_socket.h"
-
-#include <winsock2.h>
-#include <ws2tcpip.h>
 
 #include <cassert>
 
 #include "common/log.h"
+#include "common/socket_internal.h"
 #include "common/status.h"
+#include "common/stopwatch.h"
 #include "common/util.h"
 
 namespace cdc_ft {
@@ -29,9 +28,9 @@ namespace {
 // Creates a status with the given |message| and the last WSA error.
 // Assigns Tag::kSocketEof for WSAECONNRESET errors.
 absl::Status MakeSocketStatus(const char* message) {
-  const int err = WSAGetLastError();
+  const int err = GetLastError();
-  absl::Status status = MakeStatus("%s: %s", message, Util::GetWin32Error(err));
+  absl::Status status = MakeStatus("%s: %s", message, GetErrorStr(err));
-  if (err == WSAECONNRESET) {
+  if (err == kErrConnReset) {
     status = SetTag(status, Tag::kSocketEof);
   }
   return status;
@@ -40,18 +39,39 @@ absl::Status MakeSocketStatus(const char* message) {
 }  // namespace
 
 struct ClientSocketInfo {
-  SOCKET socket;
+  SocketType socket;
 
-  ClientSocketInfo() : socket(INVALID_SOCKET) {}
+  ClientSocketInfo() : socket(kInvalidSocket) {}
 };
 
 ClientSocket::ClientSocket() = default;
 
 ClientSocket::~ClientSocket() { Disconnect(); }
 
+// static
+absl::Status ClientSocket::WaitForConnection(int port, absl::Duration timeout) {
+  assert(port != 0);
+  Stopwatch timeout_timer;
+  ClientSocket socket;
+  for (;;) {
+    absl::Status status = socket.Connect(port);
+    if (status.ok()) {
+      return absl::OkStatus();
+    }
+
+    if (timeout_timer.Elapsed() > timeout) {
+      return SetTag(
+          absl::DeadlineExceededError("Timeout while connecting to server"),
+          Tag::kConnectionTimeout);
+    }
+
+    Util::Sleep(50);
+  }
+}
+
 absl::Status ClientSocket::Connect(int port) {
   addrinfo hints;
-  ZeroMemory(&hints, sizeof(hints));
+  memset(&hints, 0, sizeof(hints));
   hints.ai_family = AF_INET;
   hints.ai_socktype = SOCK_STREAM;
   hints.ai_protocol = IPPROTO_TCP;
@@ -63,26 +83,26 @@ absl::Status ClientSocket::Connect(int port) {
   if (result != 0) {
     return MakeStatus("getaddrinfo() failed: %i", result);
   }
+  AddrInfoReleaser releaser(addr_infos);
 
   socket_info_ = std::make_unique<ClientSocketInfo>();
   int count = 0;
   for (addrinfo* curr = addr_infos; curr; curr = curr->ai_next, count++) {
     socket_info_->socket =
-        socket(addr_infos->ai_family, addr_infos->ai_socktype,
+        socket(curr->ai_family, curr->ai_socktype, curr->ai_protocol);
-               addr_infos->ai_protocol);
+    if (socket_info_->socket == kInvalidSocket) {
-    if (socket_info_->socket == INVALID_SOCKET) {
       LOG_DEBUG("socket() failed for addr_info %i: %s", count,
-                Util::GetWin32Error(WSAGetLastError()).c_str());
+                GetLastErrorStr());
       continue;
     }
 
     // Connect to server.
     result = connect(socket_info_->socket, curr->ai_addr,
                      static_cast<int>(curr->ai_addrlen));
-    if (result == SOCKET_ERROR) {
+    if (result == kSocketError) {
-      LOG_DEBUG("connect() failed for addr_info %i: %i", count, result);
+      LOG_DEBUG("connect() failed for addr_info %i: %s", count,
-      closesocket(socket_info_->socket);
+                GetLastErrorStr());
-      socket_info_->socket = INVALID_SOCKET;
+      Close(&socket_info_->socket);
       continue;
     }
 
@@ -90,9 +110,7 @@ absl::Status ClientSocket::Connect(int port) {
     break;
   }
 
-  freeaddrinfo(addr_infos);
+  if (socket_info_->socket == kInvalidSocket) {
-
-  if (socket_info_->socket == INVALID_SOCKET) {
     socket_info_.reset();
     return MakeStatus("Unable to connect to port %i", port);
   }
@@ -106,18 +124,15 @@ void ClientSocket::Disconnect() {
     return;
   }
 
-  if (socket_info_->socket != INVALID_SOCKET) {
+  Close(&socket_info_->socket);
-    closesocket(socket_info_->socket);
-    socket_info_->socket = INVALID_SOCKET;
-  }
-
   socket_info_.reset();
 }
 
 absl::Status ClientSocket::Send(const void* buffer, size_t size) {
-  int result = send(socket_info_->socket, static_cast<const char*>(buffer),
+  int result =
-                    static_cast<int>(size), /*flags */ 0);
+      HANDLE_EINTR(send(socket_info_->socket, static_cast<const char*>(buffer),
-  if (result == SOCKET_ERROR) {
+                        static_cast<int>(size), /*flags */ 0));
+  if (result == kSocketError) {
     return MakeSocketStatus("send() failed");
   }
 
@@ -133,9 +148,10 @@ absl::Status ClientSocket::Receive(void* buffer, size_t size,
   }
 
   int flags = allow_partial_read ? 0 : MSG_WAITALL;
-  int bytes_read = recv(socket_info_->socket, static_cast<char*>(buffer),
+  int bytes_read =
-                        static_cast<int>(size), flags);
+      HANDLE_EINTR(recv(socket_info_->socket, static_cast<char*>(buffer),
-  if (bytes_read == SOCKET_ERROR) {
+                        static_cast<int>(size), flags));
+  if (bytes_read == kSocketError) {
     return MakeSocketStatus("recv() failed");
   }
 
@@ -154,9 +170,9 @@ absl::Status ClientSocket::Receive(void* buffer, size_t size,
 }
 
 absl::Status ClientSocket::ShutdownSendingEnd() {
-  int result = shutdown(socket_info_->socket, SD_SEND);
+  int result = shutdown(socket_info_->socket, kSendingEnd);
-  if (result == SOCKET_ERROR) {
+  if (result == kSocketError) {
-    return MakeSocketStatus("shutdown() failed");
+    return MakeStatus("Socket shutdown failed: %s", GetLastErrorStr());
   }
 
   return absl::OkStatus();

common/client_socket.h

@@ -14,13 +14,13 @@
  * limitations under the License.
  */
 
-#ifndef CDC_RSYNC_CLIENT_SOCKET_H_
+#ifndef COMMON_CLIENT_SOCKET_H_
-#define CDC_RSYNC_CLIENT_SOCKET_H_
+#define COMMON_CLIENT_SOCKET_H_
 
 #include <memory>
 
 #include "absl/status/status.h"
-#include "cdc_rsync/base/socket.h"
+#include "common/socket.h"
 
 namespace cdc_ft {
 
@@ -29,6 +29,10 @@ class ClientSocket : public Socket {
   ClientSocket();
   ~ClientSocket();
 
+  // Polls until a connection to |port| succeeds.
+  // Returns TimeoutError
+  static absl::Status WaitForConnection(int port, absl::Duration timeout);
+
   // Connects to localhost on |port|.
   absl::Status Connect(int port);
 
@@ -50,4 +54,4 @@ class ClientSocket : public Socket {
 
 }  // namespace cdc_ft
 
-#endif  // CDC_RSYNC_CLIENT_SOCKET_H_
+#endif  // COMMON_CLIENT_SOCKET_H_

common/fake_socket.cc

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "cdc_rsync/base/fake_socket.h"
+#include "common/fake_socket.h"
 
 namespace cdc_ft {
 
@@ -39,7 +39,8 @@ absl::Status FakeSocket::Receive(void* buffer, size_t size,
   *bytes_received = 0;
   std::unique_lock<std::mutex> lock(data_mutex_);
   data_cv_.wait(lock, [this, size, allow_partial_read]() {
-    return allow_partial_read || data_.size() >= size || shutdown_;
+    size_t min_size = allow_partial_read ? 1 : size;
+    return data_.size() >= min_size || shutdown_;
   });
   if (shutdown_) {
     return absl::UnavailableError("Pipe is shut down");

common/fake_socket.h

@@ -14,14 +14,14 @@
  * limitations under the License.
  */
 
-#ifndef CDC_RSYNC_BASE_FAKE_SOCKET_H_
+#ifndef COMMON_FAKE_SOCKET_H_
-#define CDC_RSYNC_BASE_FAKE_SOCKET_H_
+#define COMMON_FAKE_SOCKET_H_
 
 #include <condition_variable>
 #include <mutex>
 
 #include "absl/status/status.h"
-#include "cdc_rsync/base/socket.h"
+#include "common/socket.h"
 
 namespace cdc_ft {
 

common/file_watcher_win.cc

@@ -65,7 +65,13 @@ int64_t ToUnixTime(LARGE_INTEGER windows_time) {
 // Background thread to read directory changes.
 class AsyncFileWatcher {
  public:
-  enum class FileWatcherState { kDefault, kFailed, kRunning, kShuttingDown };
+  enum class FileWatcherState {
+    kDefault,      // Not started.
+    kFailed,       // Some error during watching, e.g. directory got deleted.
+                   // Will attempt to recover automatically.
+    kWatching,     // Actively watching directory.
+    kShuttingDown  // Shutdown() was called, winding watcher down.
+  };
 
   using FileAction = FileWatcherWin::FileAction;
   using FileInfo = FileWatcherWin::FileInfo;
@@ -158,12 +164,17 @@ class AsyncFileWatcher {
     return dir_recreate_count_;
   }
 
-  bool IsWatching() const ABSL_LOCKS_EXCLUDED(state_mutex_) {
+  bool IsStarted() const ABSL_LOCKS_EXCLUDED(state_mutex_) {
     absl::MutexLock mutex(&state_mutex_);
     return state_ != FileWatcherState::kDefault &&
            state_ != FileWatcherState::kShuttingDown;
   }
 
+  bool IsWatching() const ABSL_LOCKS_EXCLUDED(state_mutex_) {
+    absl::MutexLock mutex(&state_mutex_);
+    return state_ == FileWatcherState::kWatching;
+  }
+
   bool IsShuttingDown() const ABSL_LOCKS_EXCLUDED(state_mutex_) {
     absl::MutexLock mutex(&state_mutex_);
     return state_ == FileWatcherState::kShuttingDown;
@@ -322,7 +333,7 @@ class AsyncFileWatcher {
       return;
     }
 
-    MaybeSetState(FileWatcherState::kRunning);
+    MaybeSetState(FileWatcherState::kWatching);
     // Initialize handles to watch: changes in |dir_path_| and shutdown
     // events.
     HANDLE watch_handles[] = {overlapped.hEvent, shutdown_event_.Get()};
@@ -585,7 +596,7 @@ absl::Status FileWatcherWin::StartWatching(FilesChangedCb files_changed_cb,
   async_watcher_ = std::make_unique<AsyncFileWatcher>(
       dir_path_, std::move(files_changed_cb), std::move(dir_recreated_cb),
       timeout_ms, enforceLegacyReadDirectoryChangesForTesting_);
-  while (GetStatus().ok() && !IsWatching()) {
+  while (GetStatus().ok() && !IsStarted()) {
     std::this_thread::sleep_for(std::chrono::milliseconds(1));
   }
   return GetStatus();
@@ -610,6 +621,10 @@ absl::Status FileWatcherWin::StopWatching() {
   return async_status;
 }
 
+bool FileWatcherWin::IsStarted() const {
+  return async_watcher_ ? async_watcher_->IsStarted() : false;
+}
+
 bool FileWatcherWin::IsWatching() const {
   return async_watcher_ ? async_watcher_->IsWatching() : false;
 }
@@ -627,7 +642,7 @@ uint32_t FileWatcherWin::GetDirRecreateEventCountForTesting() const {
 }
 
 void FileWatcherWin::EnforceLegacyReadDirectoryChangesForTesting() {
-  assert(!IsWatching());
+  assert(!IsStarted());
   enforceLegacyReadDirectoryChangesForTesting_ = true;
 }
 

common/file_watcher_win.h

@@ -77,7 +77,12 @@ class FileWatcherWin {
   // Stops watching directory changes.
   absl::Status StopWatching() ABSL_LOCKS_EXCLUDED(modified_files_mutex_);
 
-  // Indicates whether a directory is currently watched.
+  // Indicates whether StartWatching() was called, but StopWatching() was not
+  // called yet.
+  bool IsStarted() const;
+
+  // Indicates whether a directory is actively watched for changes. In contrast
+  // to IsStarted(), returns false while the directory does not exist.
   bool IsWatching() const;
 
   // Returns the watching status.

common/file_watcher_win_test.cc

@@ -135,8 +135,8 @@ class FileWatcherParameterizedTest : public ::testing::TestWithParam<bool> {
     return changed;
   }
 
-  // Polls for a second until the watcher is watching again.
+  // Polls for a second until the watcher is running again.
-  bool WaitForWatching() const {
+  bool WaitForRunning() const {
     for (int n = 0; n < 1000; ++n) {
       if (watcher_.IsWatching()) return true;
       Util::Sleep(1);
@@ -210,7 +210,7 @@ TEST_P(FileWatcherParameterizedTest, DirDoesNotExist) {
   if (legacyReadDirectoryChanges_)
     watcher_.EnforceLegacyReadDirectoryChangesForTesting();
   EXPECT_NOT_OK(watcher.StartWatching([this]() { OnFilesChanged(); }));
-  EXPECT_FALSE(watcher.IsWatching());
+  EXPECT_FALSE(watcher.IsStarted());
   absl::Status status = watcher.GetStatus();
   EXPECT_NOT_OK(status);
   EXPECT_TRUE(absl::IsFailedPrecondition(status));
@@ -549,8 +549,8 @@ TEST_P(FileWatcherParameterizedTest, RecreateWatchedDir) {
   EXPECT_TRUE(watcher_.GetModifiedFiles().empty());
   EXPECT_OK(watcher_.GetStatus());
 
-  // Wait until the watcher is watching again, or else we might miss the file.
+  // Wait until the watcher is running again, or else we might miss the file.
-  EXPECT_TRUE(WaitForWatching());
+  EXPECT_TRUE(WaitForRunning());
 
   // Creation of a new file should be detected.
   EXPECT_OK(path::WriteFile(first_file_path_, kFirstData, kFirstDataSize));
@@ -584,8 +584,8 @@ TEST_P(FileWatcherParameterizedTest, RecreateUpperDir) {
   EXPECT_TRUE(watcher_.GetModifiedFiles().empty());
   EXPECT_OK(watcher_.GetStatus());
 
-  // Wait until the watcher is watching again, or else we might miss the file.
+  // Wait until the watcher is running again, or else we might miss the file.
-  EXPECT_TRUE(WaitForWatching());
+  EXPECT_TRUE(WaitForRunning());
 
   // Creation of a new file should be detected.
   EXPECT_OK(path::WriteFile(first_file_path_, kFirstData, kFirstDataSize));

common/gamelet_component.cc

@@ -18,20 +18,36 @@
 
 #include "absl/strings/str_format.h"
 #include "absl/strings/str_split.h"
+#include "common/build_version.h"
 #include "common/path.h"
 #include "common/status.h"
 
 namespace cdc_ft {
 
-GameletComponent::GameletComponent(std::string filename, uint64_t size,
+GameletComponent::GameletComponent(std::string build_version,
+                                   std::string filename, uint64_t size,
                                    time_t modified_time)
-    : filename(filename), size(size), modified_time(modified_time) {}
+    : build_version(build_version),
+      filename(filename),
+      size(size),
+      modified_time(modified_time) {}
 
 GameletComponent::~GameletComponent() = default;
 
 bool GameletComponent::operator==(const GameletComponent& other) const {
-  return filename == other.filename && size == other.size &&
+  if (filename != other.filename) {
-         modified_time == other.modified_time;
+    return false;
+  }
+
+  // If either build version is the dev version, it means that the component was
+  // built locally, so that we can't compare build versions. Fall back to
+  // comparing file_size and  modified_time.
+  if (build_version != DEV_BUILD_VERSION &&
+      build_version != DEV_BUILD_VERSION) {
+    return build_version == other.build_version;
+  }
+
+  return size == other.size && modified_time == other.modified_time;
 }
 
 bool GameletComponent::operator!=(const GameletComponent& other) const {
@@ -49,7 +65,7 @@ absl::Status GameletComponent::Get(
     absl::Status status = path::GetStats(path, &stats);
     if (!status.ok())
       return WrapStatus(status, "GetStats() failed for '%s'", path);
-    components->emplace_back(path::BaseName(path), stats.size,
+    components->emplace_back(BUILD_VERSION, path::BaseName(path), stats.size,
                              stats.modified_time);
   }
 
@@ -61,9 +77,9 @@ std::string GameletComponent::ToCommandLineArgs(
     const std::vector<GameletComponent>& components) {
   std::string args;
   for (const GameletComponent& comp : components) {
-    args +=
+    args += absl::StrFormat("%s%s %s %u %d", args.empty() ? "" : " ",
-        absl::StrFormat("%s%s %u %d", args.empty() ? "" : " ",
+                            comp.build_version.c_str(), comp.filename.c_str(),
-                        comp.filename.c_str(), comp.size, comp.modified_time);
+                            comp.size, comp.modified_time);
   }
   return args;
 }
@@ -72,9 +88,9 @@ std::string GameletComponent::ToCommandLineArgs(
 std::vector<GameletComponent> GameletComponent::FromCommandLineArgs(
     int argc, const char** argv) {
   std::vector<GameletComponent> components;
-  for (int n = 0; n + 2 < argc; n += 3) {
+  for (int n = 0; n + 3 < argc; n += 4) {
-    components.emplace_back(argv[n], std::stol(argv[n + 1]),
+    components.emplace_back(argv[n], argv[n + 1], std::stol(argv[n + 2]),
-                            std::stol(argv[n + 2]));
+                            std::stol(argv[n + 3]));
   }
   return components;
 }

common/gamelet_component.h

@@ -28,11 +28,13 @@ namespace cdc_ft {
 // The components are considered fresh if both the timestamp and the file size
 // match.
 struct GameletComponent {
+  std::string build_version;
   std::string filename;
   uint64_t size;
   int64_t modified_time;
 
-  GameletComponent(std::string filename, uint64_t size, time_t modified_time);
+  GameletComponent(std::string build_version, std::string filename,
+                   uint64_t size, time_t modified_time);
   ~GameletComponent();
 
   bool operator==(const GameletComponent& other) const;

common/gamelet_component_test.cc

@@ -15,6 +15,7 @@
 #include "common/gamelet_component.h"
 
 #include "absl/strings/str_split.h"
+#include "common/build_version.h"
 #include "common/log.h"
 #include "common/path.h"
 #include "common/status_test_macros.h"
@@ -43,14 +44,14 @@ class GameletComponentTest : public ::testing::Test {
       path::Join(base_dir_, "other", "cdc_rsync_server");
 };
 
-TEST_F(GameletComponentTest, EqualityOperators) {
+TEST_F(GameletComponentTest, EqualityOperators_DevelopmentVersion) {
   constexpr uint64_t size1 = 1001;
   constexpr uint64_t size2 = 1002;
 
   constexpr int64_t modified_time1 = 5001;
   constexpr int64_t modified_time2 = 5002;
 
-  GameletComponent a("file1", size1, modified_time1);
+  GameletComponent a(DEV_BUILD_VERSION, "file1", size1, modified_time1);
 
   GameletComponent b = a;
   EXPECT_TRUE(a == b && !(a != b));
@@ -65,6 +66,38 @@ TEST_F(GameletComponentTest, EqualityOperators) {
   b = a;
   b.modified_time = modified_time2;
   EXPECT_TRUE(!(a == b) && a != b);
+
+  b = a;
+  b.size = size2;
+  b.build_version = "Specified";
+  EXPECT_TRUE(!(a == b) && a != b);
+
+  a.build_version = "Specified";
+  EXPECT_TRUE(a == b && !(a != b));
+}
+
+TEST_F(GameletComponentTest, EqualityOperators_SpecifiedVersion) {
+  constexpr uint64_t size1 = 1001;
+  constexpr uint64_t size2 = 1002;
+
+  constexpr int64_t modified_time1 = 5001;
+  constexpr int64_t modified_time2 = 5002;
+
+  GameletComponent a("Specified", "file1", size1, modified_time1);
+
+  GameletComponent b = a;
+  EXPECT_TRUE(a == b && !(a != b));
+
+  b.filename = "file2";
+  EXPECT_TRUE(!(a == b) && a != b);
+
+  b = a;
+  b.size = size2;
+  EXPECT_TRUE(a == b && !(a != b));
+
+  b = a;
+  b.modified_time = modified_time2;
+  EXPECT_TRUE(a == b && !(a != b));
 }
 
 TEST_F(GameletComponentTest, GetValidComponents) {
@@ -91,9 +124,30 @@ TEST_F(GameletComponentTest, GetChangedComponents) {
 
   // Force equal timestamps, so that we don't depend on when the files were
   // actually written to everyone's drives.
+  // Also force set build_version to developer since otherwise we would skip
+  // component size check.
   ASSERT_EQ(components.size(), other_components.size());
   for (size_t n = 0; n < components.size(); ++n) {
     other_components[n].modified_time = components[n].modified_time;
+    other_components[n].build_version = DEV_BUILD_VERSION;
+
+    EXPECT_NE(components, other_components);
+  }
+}
+
+TEST_F(GameletComponentTest, GetChangedComponents_BuildVersionChanged) {
+  std::vector<GameletComponent> components;
+  EXPECT_OK(GameletComponent::Get({valid_component_path_}, &components));
+
+  std::vector<GameletComponent> other_components;
+  EXPECT_OK(GameletComponent::Get({other_component_path_}, &other_components));
+
+  ASSERT_EQ(components.size(), other_components.size());
+  for (size_t n = 0; n < components.size(); ++n) {
+    other_components[n].modified_time = components[n].modified_time;
+    other_components[n].size = components[n].size;
+    components[n].build_version = "build_version";
+    other_components[n].build_version = "other_build_version";
 
     EXPECT_NE(components, other_components);
   }

common/log.cc

@@ -126,21 +126,22 @@ void ConsoleLog::WriteLogMessage(LogLevel level, const char* file, int line,
   absl::MutexLock lock(&mutex_);
 
   // Show leaner log messages in non-verbose mode.
-  bool show_file_func = GetLogLevel() <= LogLevel::kDebug;
+  bool show_time_file_func = GetLogLevel() <= LogLevel::kDebug;
   FILE* stdfile = level >= LogLevel::kError ? stderr : stdout;
 #if PLATFORM_WINDOWS
   HANDLE hConsole = GetStdHandle(STD_OUTPUT_HANDLE);
   SetConsoleTextAttribute(hConsole, GetConsoleColor(level));
-  if (show_file_func) {
+  if (show_time_file_func) {
-    fprintf(stdfile, "%s(%i): %s(): %s\n", file, line, func, message);
+    fprintf(stdfile, "%0.3f %s(%i): %s(): %s\n", stopwatch_.ElapsedSeconds(),
+            file, line, func, message);
   } else {
     fprintf(stdfile, "%s\n", message);
   }
   SetConsoleTextAttribute(hConsole, kLightGray);
 #else
-  if (show_file_func) {
+  if (show_time_file_func) {
-    fprintf(stdfile, "%-7s %s(%i): %s(): %s\n", GetLogLevelString(level), file,
+    fprintf(stdfile, "%-7s %0.3f %s(%i): %s(): %s\n", GetLogLevelString(level),
-            line, func, message);
+            stopwatch_.ElapsedSeconds(), file, line, func, message);
   } else {
     fprintf(stdfile, "%-7s %s\n", GetLogLevelString(level), message);
   }

common/log.h

@@ -22,6 +22,7 @@
 #include "absl/strings/str_format.h"
 #include "absl/synchronization/mutex.h"
 #include "common/clock.h"
+#include "common/stopwatch.h"
 
 namespace cdc_ft {
 
@@ -120,6 +121,7 @@ class ConsoleLog : public Log {
       ABSL_LOCKS_EXCLUDED(mutex_);
 
  private:
+  Stopwatch stopwatch_;
   absl::Mutex mutex_;
 };
 

common/path.cc

@@ -219,9 +219,12 @@ absl::Status ExpandPathVariables(std::string* path) {
   *path = Util::WideToUtf8Str(wchar_expanded);
   return absl::OkStatus();
 #else
+  // Exclude command substitution. It.s not what users of this method would
+  // expect and could lead to security issues.
   wordexp_t res;
-  wordexp(path->c_str(), &res, 0);
+  wordexp(path->c_str(), &res, WRDE_NOCMD);
   if (res.we_wordc > 1) {
+    wordfree(&res);
     return absl::InvalidArgumentError(
         "Path expands to multiple results (did you use * etc. ?");
   }
@@ -291,8 +294,8 @@ std::string GetDrivePrefix(const std::string& path) {
 
   if (path[0] != '\\') {
     size_t pos = path.find(":");
-    if (pos == std::string::npos) {
+    if (pos != 1) {
-      // E.g. "\path\to\file" or "path\to\file".
+      // E.g. "\path\to\file", "path\to\file" or "user@host:file".
       return std::string();
     }
 

common/path.h

@@ -104,8 +104,8 @@ absl::Status GetKnownFolderPath(FolderId folder_id, std::string* path);
 // Expands environment path variables like %APPDATA% on Windows or ~ on Linux.
 // On Windows, variables are matched case invariantly. Unknown environment
 // variables are not changed.
-// On Linux, performs a shell-like expansion. Returns an error if multiple
+// On Linux, performs a shell-like expansion, but without command substitution.
-// results would be returned, e.g. from *.txt.
+// Returns an error if multiple results would be returned, e.g. from *.txt.
 absl::Status ExpandPathVariables(std::string* path);
 
 // Returns the environment variable with given |name| in |value|.

common/path_test.cc

@@ -302,6 +302,7 @@ TEST_F(PathTest, GetDrivePrefix) {
   EXPECT_EQ(path::GetDrivePrefix("C:\\"), "C:");
   EXPECT_EQ(path::GetDrivePrefix("C:\\dir"), "C:");
   EXPECT_EQ(path::GetDrivePrefix("C:\\dir\\file"), "C:");
+  EXPECT_EQ(path::GetDrivePrefix("host:C:\\dir\\file"), "");
 }
 #endif
 

common/port_manager.h

@@ -17,12 +17,12 @@
 #ifndef COMMON_PORT_MANAGER_H_
 #define COMMON_PORT_MANAGER_H_
 
-#include <absl/status/statusor.h>
-
 #include <memory>
 #include <string>
 #include <unordered_set>
 
+#include "absl/status/statusor.h"
+#include "common/arch_type.h"
 #include "common/clock.h"
 
 namespace cdc_ft {
@@ -40,8 +40,8 @@ class PortManager {
   // synchronize port reservation. The range of possible ports managed by this
   // instance is [|first_port|, |last_port|]. |process_factory| is a valid
   // pointer to a ProcessFactory instance to run processes locally.
-  // |remote_util| is a valid pointer to a RemoteUtil instance to run processes
+  // |remote_util| is the RemoteUtil instance to run processes remotely. If it
-  // remotely.
+  //   is nullptr, no remote ports are reserved.
   PortManager(std::string unique_name, int first_port, int last_port,
               ProcessFactory* process_factory, RemoteUtil* remote_util,
               SystemClock* system_clock = DefaultSystemClock::GetInstance(),
@@ -53,9 +53,12 @@ class PortManager {
   // explicitly.
   // |remote_timeout_sec| is the timeout for finding available ports on the
   // remote instance.
-  // Returns a DeadlineExceeded error if the timeout is exceeded.
+  // |remote_arch_type| is the architecture of the remote device.
-  // Returns a ResourceExhausted error if no ports are available.
+  // Both |remote_timeout_sec| and |remote_arch_type| are ignored if
-  absl::StatusOr<int> ReservePort(int remote_timeout_sec);
+  // |remote_util| is nullptr. Returns a DeadlineExceeded error if the timeout
+  // is exceeded. Returns a ResourceExhausted error if no ports are available.
+  absl::StatusOr<int> ReservePort(int remote_timeout_sec,
+                                  ArchType remote_arch_type);
 
   // Releases a reserved port.
   absl::Status ReleasePort(int port);
@@ -66,34 +69,34 @@ class PortManager {
 
   // Finds available ports in the range [first_port, last_port] for port
   // forwarding on the local workstation.
-  // |ip| is the IP address to filter by.
+  // |arch_type| is the architecture of the local device.
   // |process_factory| is used to create a netstat process.
   // Returns ResourceExhaustedError if no port is available.
   static absl::StatusOr<std::unordered_set<int>> FindAvailableLocalPorts(
-      int first_port, int last_port, const char* ip,
+      int first_port, int last_port, ArchType arch_type,
       ProcessFactory* process_factory);
 
   // Finds available ports in the range [first_port, last_port] for port
   // forwarding on the instance.
-  // |ip| is the IP address to filter by.
+  // |arch_type| is the architecture of the remote device.
   // |process_factory| is used to create a netstat process.
   // |remote_util| is used to connect to the instance.
   // |timeout_sec| is the connection timeout in seconds.
   // Returns a DeadlineExceeded error if the timeout is exceeded.
   // Returns ResourceExhaustedError if no port is available.
   static absl::StatusOr<std::unordered_set<int>> FindAvailableRemotePorts(
-      int first_port, int last_port, const char* ip,
+      int first_port, int last_port, ArchType arch_type,
       ProcessFactory* process_factory, RemoteUtil* remote_util, int timeout_sec,
       SteadyClock* steady_clock = DefaultSteadyClock::GetInstance());
 
  private:
   // Returns a list of available ports in the range [|first_port|, |last_port|]
-  // from the given |netstat_output|. |ip| is the IP address to look for, e.g.
+  // from the given |netstat_output|.
-  // "127.0.0.1".
+  // |arch_type| is the architecture of the device where netstat was called.
   // Returns ResourceExhaustedError if no port is available.
   static absl::StatusOr<std::unordered_set<int>> FindAvailablePorts(
       int first_port, int last_port, const std::string& netstat_output,
-      const char* ip);
+      ArchType arch_type);
 
   int first_port_;
   int last_port_;

common/port_manager_test.cc

@@ -25,7 +25,6 @@
 namespace cdc_ft {
 namespace {
 
-constexpr int kSshPort = 12345;
 constexpr char kUserHost[] = "user@1.2.3.4";
 
 constexpr char kGuid[] = "f77bcdfe-368c-4c45-9f01-230c5e7e2132";
@@ -35,13 +34,18 @@ constexpr int kNumPorts = kLastPort - kFirstPort + 1;
 
 constexpr int kTimeoutSec = 1;
 
-constexpr char kLocalNetstat[] = "netstat -a -n -p tcp";
+constexpr char kWindowsNetstat[] = "netstat -a -n -p tcp";
-constexpr char kRemoteNetstat[] = "netstat --numeric --listening --tcp";
+constexpr char kLinuxNetstatOrSS[] =
+    "which ss2 && ss --numeric --listening --tcp || netstat --numeric "
+    "--listening --tcp";
 
-constexpr char kLocalNetstatOutFmt[] =
+constexpr char kWindowsNetstatOutFmt[] =
     "TCP    127.0.0.1:50000        127.0.0.1:%i        ESTABLISHED";
-constexpr char kRemoteNetstatOutFmt[] =
+constexpr char kLinuxNetstatOutFmt[] =
     "tcp        0      0 0.0.0.0:%i           0.0.0.0:*               LISTEN";
+constexpr char kLinuxSSOutFmt[] =
+    "LISTEN              0                    128                              "
+    "             0.0.0.0:%i                                     0.0.0.0:*  ";
 
 class PortManagerTest : public ::testing::Test {
  public:
@@ -67,10 +71,11 @@ class PortManagerTest : public ::testing::Test {
 };
 
 TEST_F(PortManagerTest, ReservePortSuccess) {
-  process_factory_.SetProcessOutput(kLocalNetstat, "", "", 0);
+  process_factory_.SetProcessOutput(kWindowsNetstat, "", "", 0);
-  process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 0);
+  process_factory_.SetProcessOutput(kLinuxNetstatOrSS, "", "", 0);
 
-  absl::StatusOr<int> port = port_manager_.ReservePort(kTimeoutSec);
+  absl::StatusOr<int> port =
+      port_manager_.ReservePort(kTimeoutSec, ArchType::kLinux_x86_64);
   ASSERT_OK(port);
   EXPECT_EQ(*port, kFirstPort);
 }
@@ -78,12 +83,13 @@ TEST_F(PortManagerTest, ReservePortSuccess) {
 TEST_F(PortManagerTest, ReservePortAllLocalPortsTaken) {
   std::string local_netstat_out = "";
   for (int port = kFirstPort; port <= kLastPort; ++port) {
-    local_netstat_out += absl::StrFormat(kLocalNetstatOutFmt, port);
+    local_netstat_out += absl::StrFormat(kWindowsNetstatOutFmt, port);
   }
-  process_factory_.SetProcessOutput(kLocalNetstat, local_netstat_out, "", 0);
+  process_factory_.SetProcessOutput(kWindowsNetstat, local_netstat_out, "", 0);
-  process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 0);
+  process_factory_.SetProcessOutput(kLinuxNetstatOrSS, "", "", 0);
 
-  absl::StatusOr<int> port = port_manager_.ReservePort(kTimeoutSec);
+  absl::StatusOr<int> port =
+      port_manager_.ReservePort(kTimeoutSec, ArchType::kLinux_x86_64);
   EXPECT_TRUE(absl::IsResourceExhausted(port.status()));
   EXPECT_TRUE(
       absl::StrContains(port.status().message(), "No port available in range"));
@@ -92,44 +98,48 @@ TEST_F(PortManagerTest, ReservePortAllLocalPortsTaken) {
 TEST_F(PortManagerTest, ReservePortAllRemotePortsTaken) {
   std::string remote_netstat_out = "";
   for (int port = kFirstPort; port <= kLastPort; ++port) {
-    remote_netstat_out += absl::StrFormat(kRemoteNetstatOutFmt, port);
+    remote_netstat_out += absl::StrFormat(kLinuxNetstatOutFmt, port);
   }
-  process_factory_.SetProcessOutput(kLocalNetstat, "", "", 0);
+  process_factory_.SetProcessOutput(kWindowsNetstat, "", "", 0);
-  process_factory_.SetProcessOutput(kRemoteNetstat, remote_netstat_out, "", 0);
+  process_factory_.SetProcessOutput(kLinuxNetstatOrSS, remote_netstat_out, "",
+                                    0);
 
-  absl::StatusOr<int> port = port_manager_.ReservePort(kTimeoutSec);
+  absl::StatusOr<int> port =
+      port_manager_.ReservePort(kTimeoutSec, ArchType::kLinux_x86_64);
   EXPECT_TRUE(absl::IsResourceExhausted(port.status()));
   EXPECT_TRUE(
       absl::StrContains(port.status().message(), "No port available in range"));
 }
 
 TEST_F(PortManagerTest, ReservePortLocalNetstatFails) {
-  process_factory_.SetProcessOutput(kLocalNetstat, "", "", 1);
+  process_factory_.SetProcessOutput(kWindowsNetstat, "", "", 1);
-  process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 0);
+  process_factory_.SetProcessOutput(kLinuxNetstatOrSS, "", "", 0);
 
-  absl::StatusOr<int> port = port_manager_.ReservePort(kTimeoutSec);
+  absl::StatusOr<int> port =
+      port_manager_.ReservePort(kTimeoutSec, ArchType::kLinux_x86_64);
   EXPECT_NOT_OK(port);
-  EXPECT_TRUE(
+  EXPECT_TRUE(absl::StrContains(port.status().message(),
-      absl::StrContains(port.status().message(),
+                                "Failed to find available local ports"));
-                        "Failed to find available ports on workstation"));
 }
 
 TEST_F(PortManagerTest, ReservePortRemoteNetstatFails) {
-  process_factory_.SetProcessOutput(kLocalNetstat, "", "", 0);
+  process_factory_.SetProcessOutput(kWindowsNetstat, "", "", 0);
-  process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 1);
+  process_factory_.SetProcessOutput(kLinuxNetstatOrSS, "", "", 1);
 
-  absl::StatusOr<int> port = port_manager_.ReservePort(kTimeoutSec);
+  absl::StatusOr<int> port =
+      port_manager_.ReservePort(kTimeoutSec, ArchType::kLinux_x86_64);
   EXPECT_NOT_OK(port);
   EXPECT_TRUE(absl::StrContains(port.status().message(),
-                                "Failed to find available ports on instance"));
+                                "Failed to find available remote ports"));
 }
 
 TEST_F(PortManagerTest, ReservePortRemoteNetstatTimesOut) {
-  process_factory_.SetProcessOutput(kLocalNetstat, "", "", 0);
+  process_factory_.SetProcessOutput(kWindowsNetstat, "", "", 0);
-  process_factory_.SetProcessNeverExits(kRemoteNetstat);
+  process_factory_.SetProcessNeverExits(kLinuxNetstatOrSS);
   steady_clock_.AutoAdvance(kTimeoutSec * 2 * 1000);
 
-  absl::StatusOr<int> port = port_manager_.ReservePort(kTimeoutSec);
+  absl::StatusOr<int> port =
+      port_manager_.ReservePort(kTimeoutSec, ArchType::kLinux_x86_64);
   EXPECT_NOT_OK(port);
   EXPECT_TRUE(absl::IsDeadlineExceeded(port.status()));
   EXPECT_TRUE(absl::StrContains(port.status().message(),
@@ -137,8 +147,8 @@ TEST_F(PortManagerTest, ReservePortRemoteNetstatTimesOut) {
 }
 
 TEST_F(PortManagerTest, ReservePortMultipleInstances) {
-  process_factory_.SetProcessOutput(kLocalNetstat, "", "", 0);
+  process_factory_.SetProcessOutput(kWindowsNetstat, "", "", 0);
-  process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 0);
+  process_factory_.SetProcessOutput(kLinuxNetstatOrSS, "", "", 0);
 
   PortManager port_manager2(kGuid, kFirstPort, kLastPort, &process_factory_,
                             &remote_util_);
@@ -146,55 +156,80 @@ TEST_F(PortManagerTest, ReservePortMultipleInstances) {
   // Port managers use shared memory, so different instances know about each
   // other. This would even work if |port_manager_| and |port_manager2| belonged
   // to different processes, but we don't test that here.
-  EXPECT_EQ(*port_manager_.ReservePort(kTimeoutSec), kFirstPort + 0);
+  EXPECT_EQ(*port_manager_.ReservePort(kTimeoutSec, ArchType::kLinux_x86_64),
-  EXPECT_EQ(*port_manager2.ReservePort(kTimeoutSec), kFirstPort + 1);
+            kFirstPort + 0);
-  EXPECT_EQ(*port_manager_.ReservePort(kTimeoutSec), kFirstPort + 2);
+  EXPECT_EQ(*port_manager2.ReservePort(kTimeoutSec, ArchType::kLinux_x86_64),
-  EXPECT_EQ(*port_manager2.ReservePort(kTimeoutSec), kFirstPort + 3);
+            kFirstPort + 1);
+  EXPECT_EQ(*port_manager_.ReservePort(kTimeoutSec, ArchType::kLinux_x86_64),
+            kFirstPort + 2);
+  EXPECT_EQ(*port_manager2.ReservePort(kTimeoutSec, ArchType::kLinux_x86_64),
+            kFirstPort + 3);
 }
 
 TEST_F(PortManagerTest, ReservePortReusesPortsInLRUOrder) {
-  process_factory_.SetProcessOutput(kLocalNetstat, "", "", 0);
+  process_factory_.SetProcessOutput(kWindowsNetstat, "", "", 0);
-  process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 0);
+  process_factory_.SetProcessOutput(kLinuxNetstatOrSS, "", "", 0);
 
   for (int n = 0; n < kNumPorts * 2; ++n) {
-    EXPECT_EQ(*port_manager_.ReservePort(kTimeoutSec),
+    EXPECT_EQ(*port_manager_.ReservePort(kTimeoutSec, ArchType::kLinux_x86_64),
               kFirstPort + n % kNumPorts);
     system_clock_.Advance(1000);
   }
 }
 
 TEST_F(PortManagerTest, ReleasePort) {
-  process_factory_.SetProcessOutput(kLocalNetstat, "", "", 0);
+  process_factory_.SetProcessOutput(kWindowsNetstat, "", "", 0);
-  process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 0);
+  process_factory_.SetProcessOutput(kLinuxNetstatOrSS, "", "", 0);
 
-  absl::StatusOr<int> port = port_manager_.ReservePort(kTimeoutSec);
+  absl::StatusOr<int> port =
+      port_manager_.ReservePort(kTimeoutSec, ArchType::kLinux_x86_64);
   EXPECT_EQ(*port, kFirstPort);
   EXPECT_OK(port_manager_.ReleasePort(*port));
-  port = port_manager_.ReservePort(kTimeoutSec);
+  port = port_manager_.ReservePort(kTimeoutSec, ArchType::kLinux_x86_64);
   EXPECT_EQ(*port, kFirstPort);
 }
 
 TEST_F(PortManagerTest, ReleasePortOnDestruction) {
-  process_factory_.SetProcessOutput(kLocalNetstat, "", "", 0);
+  process_factory_.SetProcessOutput(kWindowsNetstat, "", "", 0);
-  process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 0);
+  process_factory_.SetProcessOutput(kLinuxNetstatOrSS, "", "", 0);
 
   auto port_manager2 = std::make_unique<PortManager>(
       kGuid, kFirstPort, kLastPort, &process_factory_, &remote_util_);
-  EXPECT_EQ(*port_manager2->ReservePort(kTimeoutSec), kFirstPort + 0);
+  EXPECT_EQ(*port_manager2->ReservePort(kTimeoutSec, ArchType::kLinux_x86_64),
-  EXPECT_EQ(*port_manager_.ReservePort(kTimeoutSec), kFirstPort + 1);
+            kFirstPort + 0);
+  EXPECT_EQ(*port_manager_.ReservePort(kTimeoutSec, ArchType::kLinux_x86_64),
+            kFirstPort + 1);
   port_manager2.reset();
-  EXPECT_EQ(*port_manager_.ReservePort(kTimeoutSec), kFirstPort + 0);
+  EXPECT_EQ(*port_manager_.ReservePort(kTimeoutSec, ArchType::kLinux_x86_64),
+            kFirstPort + 0);
 }
 
-TEST_F(PortManagerTest, FindAvailableLocalPortsSuccess) {
+TEST_F(PortManagerTest, FindAvailableLocalPortsSuccessWindows) {
-  // First port is taken
+  // First port is in use.
   std::string local_netstat_out =
-      absl::StrFormat(kLocalNetstatOutFmt, kFirstPort);
+      absl::StrFormat(kWindowsNetstatOutFmt, kFirstPort);
-  process_factory_.SetProcessOutput(kLocalNetstat, local_netstat_out, "", 0);
+  process_factory_.SetProcessOutput(kWindowsNetstat, local_netstat_out, "", 0);
 
   absl::StatusOr<std::unordered_set<int>> ports =
-      PortManager::FindAvailableLocalPorts(kFirstPort, kLastPort, "127.0.0.1",
+      PortManager::FindAvailableLocalPorts(
-                                           &process_factory_);
+          kFirstPort, kLastPort, ArchType::kWindows_x86_64, &process_factory_);
+  ASSERT_OK(ports);
+  EXPECT_EQ(ports->size(), kNumPorts - 1);
+  for (int port = kFirstPort + 1; port <= kLastPort; ++port) {
+    EXPECT_TRUE(ports->find(port) != ports->end());
+  }
+}
+
+TEST_F(PortManagerTest, FindAvailableLocalPortsSuccessLinux) {
+  // First port is in use.
+  std::string local_netstat_out =
+      absl::StrFormat(kLinuxNetstatOutFmt, kFirstPort);
+  process_factory_.SetProcessOutput(kLinuxNetstatOrSS, local_netstat_out, "",
+                                    0);
+
+  absl::StatusOr<std::unordered_set<int>> ports =
+      PortManager::FindAvailableLocalPorts(
+          kFirstPort, kLastPort, ArchType::kLinux_x86_64, &process_factory_);
   ASSERT_OK(ports);
   EXPECT_EQ(ports->size(), kNumPorts - 1);
   for (int port = kFirstPort + 1; port <= kLastPort; ++port) {
@@ -203,31 +238,66 @@ TEST_F(PortManagerTest, FindAvailableLocalPortsSuccess) {
 }
 
 TEST_F(PortManagerTest, FindAvailableLocalPortsFailsNoPorts) {
-  // All ports taken
+  // All ports are in use.
   std::string local_netstat_out = "";
   for (int port = kFirstPort; port <= kLastPort; ++port) {
-    local_netstat_out += absl::StrFormat(kLocalNetstatOutFmt, port);
+    local_netstat_out += absl::StrFormat(kWindowsNetstatOutFmt, port);
   }
-  process_factory_.SetProcessOutput(kLocalNetstat, local_netstat_out, "", 0);
+  process_factory_.SetProcessOutput(kWindowsNetstat, local_netstat_out, "", 0);
 
   absl::StatusOr<std::unordered_set<int>> ports =
-      PortManager::FindAvailableLocalPorts(kFirstPort, kLastPort, "127.0.0.1",
+      PortManager::FindAvailableLocalPorts(
-                                           &process_factory_);
+          kFirstPort, kLastPort, ArchType::kWindows_x86_64, &process_factory_);
   EXPECT_TRUE(absl::IsResourceExhausted(ports.status()));
   EXPECT_TRUE(absl::StrContains(ports.status().message(),
                                 "No port available in range"));
 }
 
-TEST_F(PortManagerTest, FindAvailableRemotePortsSuccess) {
+TEST_F(PortManagerTest, FindAvailableRemotePortsSuccessLinux) {
-  // First port is taken
+  // First port is in use.
   std::string remote_netstat_out =
-      absl::StrFormat(kRemoteNetstatOutFmt, kFirstPort);
+      absl::StrFormat(kLinuxNetstatOutFmt, kFirstPort);
-  process_factory_.SetProcessOutput(kRemoteNetstat, remote_netstat_out, "", 0);
+  process_factory_.SetProcessOutput(kLinuxNetstatOrSS, remote_netstat_out, "",
+                                    0);
 
   absl::StatusOr<std::unordered_set<int>> ports =
-      PortManager::FindAvailableRemotePorts(kFirstPort, kLastPort, "0.0.0.0",
+      PortManager::FindAvailableRemotePorts(
-                                            &process_factory_, &remote_util_,
+          kFirstPort, kLastPort, ArchType::kLinux_x86_64, &process_factory_,
-                                            kTimeoutSec);
+          &remote_util_, kTimeoutSec);
+  ASSERT_OK(ports);
+  EXPECT_EQ(ports->size(), kNumPorts - 1);
+  for (int port = kFirstPort + 1; port <= kLastPort; ++port) {
+    EXPECT_TRUE(ports->find(port) != ports->end());
+  }
+}
+
+TEST_F(PortManagerTest, FindAvailableRemotePortsSuccessLinuxSS) {
+  // First port is in use, but reporting is done by SS, not netsta.
+  std::string remote_netstat_out = absl::StrFormat(kLinuxSSOutFmt, kFirstPort);
+  process_factory_.SetProcessOutput(kLinuxNetstatOrSS, remote_netstat_out, "",
+                                    0);
+
+  absl::StatusOr<std::unordered_set<int>> ports =
+      PortManager::FindAvailableRemotePorts(
+          kFirstPort, kLastPort, ArchType::kLinux_x86_64, &process_factory_,
+          &remote_util_, kTimeoutSec);
+  ASSERT_OK(ports);
+  EXPECT_EQ(ports->size(), kNumPorts - 1);
+  for (int port = kFirstPort + 1; port <= kLastPort; ++port) {
+    EXPECT_TRUE(ports->find(port) != ports->end());
+  }
+}
+
+TEST_F(PortManagerTest, FindAvailableRemotePortsSuccessWindows) {
+  // First port is in use.
+  std::string remote_netstat_out =
+      absl::StrFormat(kWindowsNetstatOutFmt, kFirstPort);
+  process_factory_.SetProcessOutput(kWindowsNetstat, remote_netstat_out, "", 0);
+
+  absl::StatusOr<std::unordered_set<int>> ports =
+      PortManager::FindAvailableRemotePorts(
+          kFirstPort, kLastPort, ArchType::kWindows_x86_64, &process_factory_,
+          &remote_util_, kTimeoutSec);
   ASSERT_OK(ports);
   EXPECT_EQ(ports->size(), kNumPorts - 1);
   for (int port = kFirstPort + 1; port <= kLastPort; ++port) {
@@ -236,17 +306,18 @@ TEST_F(PortManagerTest, FindAvailableRemotePortsSuccess) {
 }
 
 TEST_F(PortManagerTest, FindAvailableRemotePortsFailsNoPorts) {
-  // All ports taken
+  // All ports are in use.
   std::string remote_netstat_out = "";
   for (int port = kFirstPort; port <= kLastPort; ++port) {
-    remote_netstat_out += absl::StrFormat(kRemoteNetstatOutFmt, port);
+    remote_netstat_out += absl::StrFormat(kLinuxNetstatOutFmt, port);
   }
-  process_factory_.SetProcessOutput(kRemoteNetstat, remote_netstat_out, "", 0);
+  process_factory_.SetProcessOutput(kLinuxNetstatOrSS, remote_netstat_out, "",
+                                    0);
 
   absl::StatusOr<std::unordered_set<int>> ports =
-      PortManager::FindAvailableRemotePorts(kFirstPort, kLastPort, "0.0.0.0",
+      PortManager::FindAvailableRemotePorts(
-                                            &process_factory_, &remote_util_,
+          kFirstPort, kLastPort, ArchType::kLinux_x86_64, &process_factory_,
-                                            kTimeoutSec);
+          &remote_util_, kTimeoutSec);
   EXPECT_TRUE(absl::IsResourceExhausted(ports.status()));
   EXPECT_TRUE(absl::StrContains(ports.status().message(),
                                 "No port available in range"));

common/port_manager_win.cc

@@ -20,6 +20,7 @@
 #include <map>
 
 #include "absl/strings/str_split.h"
+#include "common/arch_type.h"
 #include "common/log.h"
 #include "common/process.h"
 #include "common/remote_util.h"
@@ -30,6 +31,44 @@
 
 namespace cdc_ft {
 
+constexpr char kErrorArchTypeUnhandled[] = "arch_type_unhandled";
+
+// Returns the arch-specific netstat command.
+const char* GetNetstatCommand(ArchType arch_type) {
+  if (IsWindowsArchType(arch_type)) {
+    // -a to get the connection and ports the computer is listening on.
+    // -n to get numerical addresses to avoid the overhead of getting names.
+    // -p tcp to limit the output to TCPv4 connections.
+    return "netstat -a -n -p tcp";
+  }
+
+  if (IsLinuxArchType(arch_type)) {
+    // Prefer ss over netstat. The flags out output are compatible.
+    // --numeric to get numerical addresses.
+    // --listening to get only listening sockets.
+    // --tcp to get only TCP connections.
+    return "which ss2 && ss --numeric --listening --tcp || netstat "
+           "--numeric --listening --tcp";
+  }
+
+  assert(!kErrorArchTypeUnhandled);
+  return "";
+}
+
+// Returns the arch-specific IP address to filter netstat results by.
+const char* GetNetstatFilterIp(ArchType arch_type) {
+  if (IsWindowsArchType(arch_type)) {
+    return "127.0.0.1";
+  }
+
+  if (IsLinuxArchType(arch_type)) {
+    return "0.0.0.0";
+  }
+
+  assert(!kErrorArchTypeUnhandled);
+  return "";
+}
+
 class SharedMemory {
  public:
   // Creates a new shared memory instance with given |name| and |size| in bytes.
@@ -121,21 +160,26 @@ PortManager::~PortManager() {
   }
 }
 
-absl::StatusOr<int> PortManager::ReservePort(int remote_timeout_sec) {
+absl::StatusOr<int> PortManager::ReservePort(int remote_timeout_sec,
+                                             ArchType remote_arch_type) {
   // Find available port on workstation.
   std::unordered_set<int> local_ports;
-  ASSIGN_OR_RETURN(local_ports,
+  ASSIGN_OR_RETURN(
-                   FindAvailableLocalPorts(first_port_, last_port_, "127.0.0.1",
+      local_ports,
-                                           process_factory_),
+      FindAvailableLocalPorts(first_port_, last_port_,
-                   "Failed to find available ports on workstation");
+                              ArchType::kWindows_x86_64, process_factory_),
+      "Failed to find available local ports");
 
   // Find available port on remote instance.
   std::unordered_set<int> remote_ports = local_ports;
-  ASSIGN_OR_RETURN(remote_ports,
+  if (remote_util_ != nullptr) {
-                   FindAvailableRemotePorts(first_port_, last_port_, "0.0.0.0",
+    ASSIGN_OR_RETURN(
+        remote_ports,
+        FindAvailableRemotePorts(first_port_, last_port_, remote_arch_type,
                                  process_factory_, remote_util_,
                                  remote_timeout_sec, steady_clock_),
-                   "Failed to find available ports on instance");
+        "Failed to find available remote ports");
+  }
 
   // Fetch shared memory.
   void* mem;
@@ -165,9 +209,9 @@ absl::StatusOr<int> PortManager::ReservePort(int remote_timeout_sec) {
       LOG_DEBUG("Trying to reserve port %i", port);
 
       // We have reserved this port. Double-check that it's actually not in use
-      // on both the workstation and the server.
+      // on both the local and the remote device
       if (local_ports.find(port) == local_ports.end()) {
-        LOG_DEBUG("Port %i not available on workstation", port);
+        LOG_DEBUG("Local port %i not available", port);
         InterlockedCompareExchange64(ts_ptr, now, port_timestamp);
         continue;
       }
@@ -177,7 +221,7 @@ absl::StatusOr<int> PortManager::ReservePort(int remote_timeout_sec) {
         continue;
       }
 
-      LOG_DEBUG("Port %i is available on workstation and instance", port);
+      LOG_DEBUG("Port %i is available", port);
       reserved_ports_.insert(port);
       return port;
     }
@@ -201,14 +245,11 @@ absl::Status PortManager::ReleasePort(int port) {
 
 // static
 absl::StatusOr<std::unordered_set<int>> PortManager::FindAvailableLocalPorts(
-    int first_port, int last_port, const char* ip,
+    int first_port, int last_port, ArchType arch_type,
     ProcessFactory* process_factory) {
-  // -a to get the connection and ports the computer is listening on.
+  // TODO: Use local APIs instead of netstat.
-  // -n to get numerical addresses to avoid the overhead of determining names.
-  // -p tcp to limit the output to TCPv4 connections.
-  // TODO: Use Windows API instead of netstat.
   ProcessStartInfo start_info;
-  start_info.command = "netstat -a -n -p tcp";
+  start_info.command = GetNetstatCommand(arch_type);
   start_info.name = "netstat";
   start_info.flags = ProcessFlags::kNoWindow;
 
@@ -228,21 +269,18 @@ absl::StatusOr<std::unordered_set<int>> PortManager::FindAvailableLocalPorts(
     return WrapStatus(status, "Failed to run netstat:\n%s", errors);
   }
 
-  LOG_DEBUG("netstat (workstation) output:\n%s", output);
+  LOG_DEBUG("netstat (local) output:\n%s", output);
-  return FindAvailablePorts(first_port, last_port, output, ip);
+  return FindAvailablePorts(first_port, last_port, output, arch_type);
 }
 
 // static
 absl::StatusOr<std::unordered_set<int>> PortManager::FindAvailableRemotePorts(
-    int first_port, int last_port, const char* ip,
+    int first_port, int last_port, ArchType arch_type,
     ProcessFactory* process_factory, RemoteUtil* remote_util, int timeout_sec,
     SteadyClock* steady_clock) {
-  // --numeric to get numerical addresses.
+  std::string remote_command = GetNetstatCommand(arch_type);
-  // --listening to get only listening sockets.
-  // --tcp to get only TCP connections.
-  std::string remote_command = "netstat --numeric --listening --tcp";
   ProcessStartInfo start_info =
-      remote_util->BuildProcessStartInfoForSsh(remote_command);
+      remote_util->BuildProcessStartInfoForSsh(remote_command, arch_type);
   start_info.name = "netstat";
   start_info.flags = ProcessFlags::kNoWindow;
 
@@ -278,25 +316,26 @@ absl::StatusOr<std::unordered_set<int>> PortManager::FindAvailableRemotePorts(
                       errors);
   }
 
-  LOG_DEBUG("netstat (instance) output:\n%s", output);
+  LOG_DEBUG("netstat (remote) output:\n%s", output);
-  return FindAvailablePorts(first_port, last_port, output, ip);
+  return FindAvailablePorts(first_port, last_port, output, arch_type);
 }
 
 // static
 absl::StatusOr<std::unordered_set<int>> PortManager::FindAvailablePorts(
     int first_port, int last_port, const std::string& netstat_output,
-    const char* ip) {
+    ArchType arch_type) {
   std::unordered_set<int> available_ports;
   std::vector<std::string> lines;
+  const char* filter_ip = GetNetstatFilterIp(arch_type);
   for (const auto& line : absl::StrSplit(netstat_output, '\n')) {
-    if (absl::StrContains(line, ip)) {
+    if (absl::StrContains(line, filter_ip)) {
       lines.push_back(std::string(line));
     }
   }
 
   for (int port = first_port; port <= last_port; ++port) {
     bool port_occupied = false;
-    std::string portToken = absl::StrFormat("%s:%i", ip, port);
+    std::string portToken = absl::StrFormat("%s:%i", filter_ip, port);
     for (const std::string& line : lines) {
       // Ports in the TIME_WAIT state can be reused. It is common that ports
       // stay in this state for O(minutes).

common/process.h

@@ -53,6 +53,10 @@ struct ProcessStartInfo {
   // Command line, UTF-8 encoded.
   std::string command;
 
+  // Full path to the process startup working directory.
+  // If empty, uses parent process working dir.
+  std::string startup_dir;
+
   // If set, the process stdin is redirected to a pipe.
   // It not set, the input is connected to the stdin of the calling process.
   bool redirect_stdin = false;

common/process_test.cc

@@ -342,5 +342,20 @@ TEST_F(ProcessTest, TerminateAlreadyExited) {
   EXPECT_OK(process->Terminate());
 }
 
+TEST_F(ProcessTest, StartupDir) {
+  ProcessStartInfo start_info;
+  start_info.command = "cmd /C cd";
+  start_info.startup_dir = "C:\\";
+
+  std::string std_out;
+  start_info.stdout_handler = [&std_out](const char* data, size_t) {
+    std_out += data;
+    return absl::OkStatus();
+  };
+
+  EXPECT_OK(process_factory_.Run(start_info));
+  EXPECT_EQ(std_out, "C:\\\r\n");
+}
+
 }  // namespace
 }  // namespace cdc_ft

common/process_win.cc

@@ -719,6 +719,10 @@ absl::Status WinProcess::Start() {
                       Util::GetLastWin32Error());
   }
 
+  std::wstring startup_dir = Util::Utf8ToWideStr(start_info_.startup_dir);
+  const wchar_t* startup_dir_cstr =
+      !startup_dir.empty() ? startup_dir.c_str() : nullptr;
+
   // Start the child process.
   success = CreateProcess(NULL,  // No module name (use command line)
                           command_cstr,
@@ -727,8 +731,7 @@ absl::Status WinProcess::Start() {
                           TRUE,  // Inherit handles
                           ToCreationFlags(start_info_.flags),
                           NULL,  // Use parent's environment block
-                          NULL,  // Use parent's starting directory
+                          startup_dir_cstr, &si, &process_info_->pi);
-                          &si, &process_info_->pi);
 
   if (!success) {
     return MakeStatus("CreateProcess() failed: %s", Util::GetLastWin32Error());

common/remote_util.cc

@@ -20,6 +20,8 @@
 #include "absl/strings/str_cat.h"
 #include "absl/strings/str_format.h"
 #include "common/path.h"
+#include "common/status_macros.h"
+#include "common/util.h"
 
 namespace cdc_ft {
 namespace {
@@ -32,6 +34,22 @@ std::string GetPortForwardingArg(int local_port, int remote_port,
   return absl::StrFormat("-L%i:localhost:%i ", local_port, remote_port);
 }
 
+const char* GetFlagsForArch(ArchType remote_arch_type) {
+  if (IsWindowsArchType(remote_arch_type)) {
+    // Disable pseudo-TTY. Otherwise, the output is riddled by Ansi control
+    // sequences and almost impossible to parse without handling them.
+    return "-T";
+  }
+
+  if (IsLinuxArchType(remote_arch_type)) {
+    // Force pseudo-TTY.
+    return "-tt";
+  }
+
+  assert(!"Unhandled arch type");
+  return "";
+}
+
 }  // namespace
 
 RemoteUtil::RemoteUtil(std::string user_host, int verbosity, bool quiet,
@@ -47,10 +65,34 @@ void RemoteUtil::SetScpCommand(std::string scp_command) {
   scp_command_ = std::move(scp_command);
 }
 
+void RemoteUtil::SetSftpCommand(std::string sftp_command) {
+  sftp_command_ = std::move(sftp_command);
+}
+
 void RemoteUtil::SetSshCommand(std::string ssh_command) {
   ssh_command_ = std::move(ssh_command);
 }
 
+// static
+std::string RemoteUtil::ScpToSftpCommand(std::string scp_command) {
+  // "scp", "SCP", "winscp.exe", "C:\path\to\scp", "/scppath/scp --foo" etc.
+  std::string lower_scp_command = scp_command;
+  std::transform(lower_scp_command.begin(), lower_scp_command.end(),
+                 lower_scp_command.begin(), ::tolower);
+  size_t pos = 0;
+  while ((pos = lower_scp_command.find("scp", pos)) != std::string::npos) {
+    // This may access the string at scp_command.size(), but that's well defined
+    // in C++11 and returns 0.
+    const char next_ch = lower_scp_command[pos + 3];
+    if ((next_ch == 0 || next_ch == '.' || next_ch == ' ')) {
+      return scp_command.replace(pos, 3, "sftp");
+    }
+    ++pos;
+  }
+
+  return std::string();
+}
+
 absl::Status RemoteUtil::Scp(std::vector<std::string> source_filepaths,
                              const std::string& dest, bool compress) {
   std::string source_args;
@@ -77,27 +119,82 @@ absl::Status RemoteUtil::Scp(std::vector<std::string> source_filepaths,
   return process_factory_->Run(start_info);
 }
 
+absl::Status RemoteUtil::Sftp(const std::string& commands,
+                              const std::string& initial_local_dir,
+                              bool compress) {
+  // sftp doesn't take |commands| as argument, so write it to a temp file.
+  std::string cmd_path =
+      path::Join(path::GetTempDir(), "__sftp_cmd__" + Util::GenerateUniqueId());
+  RETURN_IF_ERROR(path::WriteFile(cmd_path, commands),
+                  "Failed to write sftp commands to '%s'", cmd_path);
+
+  // -p preserves timestamps. This enables timestamp-based up-to-date checks.
+  ProcessStartInfo start_info;
+  start_info.flags = ProcessFlags::kNoWindow;
+  start_info.command = absl::StrFormat(
+      "%s %s %s -p -b %s %s", sftp_command_,
+      quiet_ || verbosity_ < 2 ? "-q" : "", compress ? "-C" : "",
+      QuoteForWindows(cmd_path), QuoteForWindows(user_host_));
+  start_info.name = "sftp";
+  start_info.startup_dir = initial_local_dir;
+  start_info.forward_output_to_log = forward_output_to_log_;
+
+  RETURN_IF_ERROR(process_factory_->Run(start_info));
+
+  // Note: Keep |cmd_path| in case of an error for debugging purposes.
+  path::RemoveFile(cmd_path).IgnoreError();
+  return absl::OkStatus();
+}
+
 absl::Status RemoteUtil::Chmod(const std::string& mode,
                                const std::string& remote_path, bool quiet) {
   std::string remote_command =
       absl::StrFormat("chmod %s %s %s", QuoteForSsh(mode),
                       QuoteForSsh(remote_path), quiet ? "-f" : "");
 
-  return Run(remote_command, "chmod");
+  return Run(remote_command, "chmod", ArchType::kLinux_x86_64);
 }
 
-absl::Status RemoteUtil::Run(std::string remote_command, std::string name) {
+absl::Status RemoteUtil::Run(std::string remote_command, std::string name,
+                             ArchType remote_arch_type) {
   ProcessStartInfo start_info =
-      BuildProcessStartInfoForSsh(std::move(remote_command));
+      BuildProcessStartInfoForSsh(std::move(remote_command), remote_arch_type);
   start_info.name = std::move(name);
   start_info.forward_output_to_log = forward_output_to_log_;
 
   return process_factory_->Run(start_info);
 }
 
+absl::Status RemoteUtil::RunWithCapture(std::string remote_command,
+                                        std::string name, std::string* std_out,
+                                        std::string* std_err,
+                                        ArchType remote_arch_type) {
+  ProcessStartInfo start_info =
+      BuildProcessStartInfoForSsh(std::move(remote_command), remote_arch_type);
+  start_info.name = std::move(name);
+  start_info.forward_output_to_log = forward_output_to_log_;
+
+  if (std_out) {
+    start_info.stdout_handler = [std_out](const char* data, size_t size) {
+      std_out->append(data, size);
+      return absl::OkStatus();
+    };
+  }
+
+  if (std_err) {
+    start_info.stderr_handler = [std_err](const char* data, size_t size) {
+      std_err->append(data, size);
+      return absl::OkStatus();
+    };
+  }
+
+  return process_factory_->Run(start_info);
+}
+
 ProcessStartInfo RemoteUtil::BuildProcessStartInfoForSsh(
-    std::string remote_command) {
+    std::string remote_command, ArchType remote_arch_type) {
-  return BuildProcessStartInfoForSshInternal("", "-- " + remote_command);
+  return BuildProcessStartInfoForSshInternal("", "-- " + remote_command,
+                                             remote_arch_type);
 }
 
 ProcessStartInfo RemoteUtil::BuildProcessStartInfoForSshPortForward(
@@ -105,28 +202,34 @@ ProcessStartInfo RemoteUtil::BuildProcessStartInfoForSshPortForward(
   // Usually, one would pass in -N here, but this makes the connection terribly
   // slow! As a workaround, don't use -N (will open a shell), but simply eat the
   // output.
+  // Note: Don't use the Windows args now. It implies -T instead of -tt, which
+  // will make the process exit immediately.
   ProcessStartInfo si = BuildProcessStartInfoForSshInternal(
-      GetPortForwardingArg(local_port, remote_port, reverse) + "-n ", "");
+      GetPortForwardingArg(local_port, remote_port, reverse) + "-n ", "",
+      ArchType::kLinux_x86_64);
   si.stdout_handler = [](const void*, size_t) { return absl::OkStatus(); };
   return si;
 }
 
 ProcessStartInfo RemoteUtil::BuildProcessStartInfoForSshPortForwardAndCommand(
-    int local_port, int remote_port, bool reverse, std::string remote_command) {
+    int local_port, int remote_port, bool reverse, std::string remote_command,
+    ArchType remote_arch_type) {
   return BuildProcessStartInfoForSshInternal(
       GetPortForwardingArg(local_port, remote_port, reverse),
-      "-- " + remote_command);
+      "-- " + remote_command, remote_arch_type);
 }
 
 ProcessStartInfo RemoteUtil::BuildProcessStartInfoForSshInternal(
-    std::string forward_arg, std::string remote_command_arg) {
+    std::string forward_arg, std::string remote_command_arg,
+    ArchType remote_arch_type) {
   ProcessStartInfo start_info;
   start_info.command = absl::StrFormat(
-      "%s %s -tt %s "
+      "%s %s %s %s "
       "-oServerAliveCountMax=6 "  // Number of lost msgs before ssh terminates
       "-oServerAliveInterval=5 "  // Time interval between alive msgs
       "%s %s",
-      ssh_command_, quiet_ || verbosity_ < 2 ? "-q" : "", forward_arg,
+      ssh_command_, GetFlagsForArch(remote_arch_type),
+      quiet_ || verbosity_ < 2 ? "-q" : "", forward_arg,
       QuoteForWindows(user_host_), remote_command_arg);
   start_info.forward_output_to_log = forward_output_to_log_;
   start_info.flags = ProcessFlags::kNoWindow;

common/remote_util.h

@@ -21,11 +21,12 @@
 #include <vector>
 
 #include "absl/status/status.h"
+#include "common/arch_type.h"
 #include "common/process.h"
 
 namespace cdc_ft {
 
-// Utilities for executing remote commands on a gamelet through SSH.
+// Utilities for executing remote commands on a remote device through SSH.
 // Windows-only.
 class RemoteUtil {
  public:
@@ -39,44 +40,92 @@ class RemoteUtil {
              ProcessFactory* process_factory, bool forward_output_to_log);
 
   // Sets the SCP command binary path and additional arguments, e.g.
-  //   C:\path\to\scp.exe -p 1234 -i <key_file> -oUserKnownHostsFile=known_hosts
+  //   C:\path\to\scp.exe -P 1234 -i <key_file> -oUserKnownHostsFile=known_hosts
-  // By default, searches scp.exe on the path environment variables.
+  // By default, searches scp on the path environment variables.
   void SetScpCommand(std::string scp_command);
 
+  // Sets the SFTP command binary path and additional arguments, e.g.
+  //   C:\path\to\sftp.exe -P 1234 -i <key_file>
+  //   -oUserKnownHostsFile=known_hosts
+  // By default, searches sftp on the path environment variables.
+  void SetSftpCommand(std::string sftp_command);
+
   // Sets the SSH command binary path and additional arguments, e.g.
-  //   C:\path\to\ssh.exe -P 1234 -i <key_file> -oUserKnownHostsFile=known_hosts
+  //   C:\path\to\ssh.exe -p 1234 -i <key_file> -oUserKnownHostsFile=known_hosts
-  // By default, searches ssh.exe on the path environment variables.
+  // By default, searches ssh on the path environment variables.
   void SetSshCommand(std::string ssh_command);
 
-  // Copies |source_filepaths| to the remote folder |dest| on the gamelet using
+  // Converts an scp command into an sftp command by simply replacing the first
-  // scp. If |compress| is true, compressed upload is used.
+  // occurrance of "scp.", "scp " or "scp\0" by sftp (case insensitive). This
+  // adds backwards compatibility after a switch from scp to sftp in case users
+  // still set CDC_SCP_COMMAND or --scp-command. Luckily, all relevant
+  // parameters of sftp and scp match.
+  // Returns an empty string if |scp_command| does not contain "scp".
+  // Returns bad results for tricky strings like "C:\scp.path\scp.exe".
+  static std::string ScpToSftpCommand(std::string scp_command);
+
+  // Copies |source_filepaths| to the remote folder |dest| on the remove device
+  // using scp. If |compress| is true, compressed upload is used.
   absl::Status Scp(std::vector<std::string> source_filepaths,
                    const std::string& dest, bool compress);
 
-  // Calls 'chmod |mode| |remote_path|' on the gamelet.
+  // Creates an sftp connection to the remote instance and executes the
+  // newline-separated SFTP |commands|. See
+  //   https://man7.org/linux/man-pages/man1/sftp.1.html
+  // for a list of available commands.
+  // |initial_local_dir| sets the initial local directory in sftp. This is
+  // useful since some sftp clients don't work with standard Windows paths and
+  // require for instance /cygdrive paths.
+  // If |compress| is true, compressed upload is used.
+  // Example: Create nested directories and copying an executable file.
+  //   -mkdir a
+  //   cd a
+  //   -mkdir b
+  //   cd b
+  //   put foo_executable
+  //   chmod 755 foo_executable
+  absl::Status Sftp(const std::string& commands,
+                    const std::string& initial_local_dir, bool compress);
+
+  // Calls 'chmod |mode| |remote_path|' on the remote device.
   absl::Status Chmod(const std::string& mode, const std::string& remote_path,
                      bool quiet = false);
 
-  // Runs |remote_command| on the gamelet. The command must be properly escaped.
+  // Runs |remote_command| on the remote device. The command must be properly
-  // |name| is the name of the command displayed in the logs.
+  // escaped. |name| is the name of the command displayed in the logs.
-  absl::Status Run(std::string remote_command, std::string name);
+  // |remote_arch_type| is the arch type of the remote device. It determines
+  // which type of pseudo console is used (-T on Windows, -tt on Linux). If the
+  // wrong arch type is passed, output might be corrupted, but otherwise the
+  // command will work.
+  absl::Status Run(std::string remote_command, std::string name,
+                   ArchType remote_arch_type);
 
-  // Builds an SSH command that executes |remote_command| on the gamelet.
+  // Same as Run(), but captures both stdout and stderr.
-  ProcessStartInfo BuildProcessStartInfoForSsh(std::string remote_command);
+  // If |std_out| or |std_err| are nullptr, the output is not captured.
+  // |remote_arch_type| is the arch type of the remote device, see Run().
+  absl::Status RunWithCapture(std::string remote_command, std::string name,
+                              std::string* std_out, std::string* std_err,
+                              ArchType remote_arch_type);
 
-  // Builds an SSH command that runs SSH port forwarding to the gamelet, using
+  // Builds an SSH command that executes |remote_command| on the remote device.
-  // the given |local_port| and |remote_port|.
+  // |remote_arch_type| is the arch type of the remote device, see Run().
-  // If |reverse| is true, sets up reverse port forwarding.
+  ProcessStartInfo BuildProcessStartInfoForSsh(std::string remote_command,
+                                               ArchType remote_arch_type);
+
+  // Builds an SSH command that runs SSH port forwarding to the remote device,
+  // using the given |local_port| and |remote_port|. If |reverse| is true, sets
+  // up reverse port forwarding.
   ProcessStartInfo BuildProcessStartInfoForSshPortForward(int local_port,
                                                           int remote_port,
                                                           bool reverse);
 
-  // Builds an SSH command that executes |remote_command| on the gamelet, using
+  // Builds an SSH command that executes |remote_command| on the remote device,
-  // port forwarding with given |local_port| and |remote_port|.
+  // using port forwarding with given |local_port| and |remote_port|. If
-  // If |reverse| is true, sets up reverse port forwarding.
+  // |reverse| is true, sets up reverse port forwarding.
+  // |remote_arch_type| is the arch type of the remote device, see Run().
   ProcessStartInfo BuildProcessStartInfoForSshPortForwardAndCommand(
-      int local_port, int remote_port, bool reverse,
+      int local_port, int remote_port, bool reverse, std::string remote_command,
-      std::string remote_command);
+      ArchType remote_arch_type);
 
   // Returns whether output is suppressed.
   bool Quiet() const { return quiet_; }
@@ -107,7 +156,8 @@ class RemoteUtil {
  private:
   // Common code for BuildProcessStartInfoForSsh*.
   ProcessStartInfo BuildProcessStartInfoForSshInternal(
-      std::string forward_arg, std::string remote_command);
+      std::string forward_arg, std::string remote_command,
+      ArchType remote_arch_type);
 
   const int verbosity_;
   const bool quiet_;
@@ -115,6 +165,7 @@ class RemoteUtil {
   const bool forward_output_to_log_;
 
   std::string scp_command_ = "scp";
+  std::string sftp_command_ = "sftp";
   std::string ssh_command_ = "ssh";
   std::string user_host_;
 };

common/remote_util_test.cc

@@ -59,8 +59,12 @@ class RemoteUtilTest : public ::testing::Test {
 };
 
 TEST_F(RemoteUtilTest, BuildProcessStartInfoForSsh) {
-  ProcessStartInfo si = util_.BuildProcessStartInfoForSsh(kCommand);
+  ProcessStartInfo si =
-  ExpectContains(si.command, {"ssh", kUserHostArg, kCommand});
+      util_.BuildProcessStartInfoForSsh(kCommand, ArchType::kLinux_x86_64);
+  ExpectContains(si.command, {"ssh", "-tt", kUserHostArg, kCommand});
+
+  si = util_.BuildProcessStartInfoForSsh(kCommand, ArchType::kWindows_x86_64);
+  ExpectContains(si.command, {"ssh", "-T", kUserHostArg, kCommand});
 }
 
 TEST_F(RemoteUtilTest, BuildProcessStartInfoForSshPortForward) {
@@ -75,19 +79,20 @@ TEST_F(RemoteUtilTest, BuildProcessStartInfoForSshPortForward) {
 
 TEST_F(RemoteUtilTest, BuildProcessStartInfoForSshPortForwardAndCommand) {
   ProcessStartInfo si = util_.BuildProcessStartInfoForSshPortForwardAndCommand(
-      kLocalPort, kRemotePort, kRegular, kCommand);
+      kLocalPort, kRemotePort, kRegular, kCommand, ArchType::kLinux_x86_64);
   ExpectContains(si.command,
                  {"ssh", kUserHostArg, kPortForwardingArg, kCommand});
 
   si = util_.BuildProcessStartInfoForSshPortForwardAndCommand(
-      kLocalPort, kRemotePort, kReverse, kCommand);
+      kLocalPort, kRemotePort, kReverse, kCommand, ArchType::kLinux_x86_64);
   ExpectContains(si.command,
                  {"ssh", kUserHostArg, kReversePortForwardingArg, kCommand});
 }
 TEST_F(RemoteUtilTest, BuildProcessStartInfoForSshWithCustomCommand) {
   constexpr char kCustomSshCmd[] = "C:\\path\\to\\ssh.exe --fooarg --bararg=42";
   util_.SetSshCommand(kCustomSshCmd);
-  ProcessStartInfo si = util_.BuildProcessStartInfoForSsh(kCommand);
+  ProcessStartInfo si =
+      util_.BuildProcessStartInfoForSsh(kCommand, ArchType::kLinux_x86_64);
   ExpectContains(si.command, {kCustomSshCmd});
 }
 
@@ -127,5 +132,27 @@ TEST_F(RemoteUtilTest, QuoteForSsh) {
             "\"~user-name69/\\\"foo\\\"\"");  // Nice!
 }
 
+TEST_F(RemoteUtilTest, ScpToSftpCommand) {
+  EXPECT_EQ(RemoteUtil::ScpToSftpCommand(""), "");
+  EXPECT_EQ(RemoteUtil::ScpToSftpCommand("scp"), "sftp");
+  EXPECT_EQ(RemoteUtil::ScpToSftpCommand("scp.exe"), "sftp.exe");
+  EXPECT_EQ(RemoteUtil::ScpToSftpCommand("scp --arg"), "sftp --arg");
+  EXPECT_EQ(RemoteUtil::ScpToSftpCommand("ScP --aRg"), "sftp --aRg");
+  EXPECT_EQ(RemoteUtil::ScpToSftpCommand("winscp"), "winsftp");
+  EXPECT_EQ(RemoteUtil::ScpToSftpCommand("winscp.exe"), "winsftp.exe");
+  EXPECT_EQ(RemoteUtil::ScpToSftpCommand("winscp --arg"), "winsftp --arg");
+  EXPECT_EQ(RemoteUtil::ScpToSftpCommand("C:\\path\\to\\scp"),
+            "C:\\path\\to\\sftp");
+  EXPECT_EQ(RemoteUtil::ScpToSftpCommand("C:\\path\\to\\scp.exe"),
+            "C:\\path\\to\\sftp.exe");
+  EXPECT_EQ(RemoteUtil::ScpToSftpCommand("C:\\path\\to\\scp.exe --arg"),
+            "C:\\path\\to\\sftp.exe --arg");
+  EXPECT_EQ(RemoteUtil::ScpToSftpCommand("C:\\scp.exe --argwithscp"),
+            "C:\\sftp.exe --argwithscp");
+  EXPECT_EQ(RemoteUtil::ScpToSftpCommand("C:\\path_with_scp\\scp"),
+            "C:\\path_with_scp\\sftp");
+  EXPECT_EQ(RemoteUtil::ScpToSftpCommand("C:\\path\\to\\somethingelse"), "");
+}
+
 }  // namespace
 }  // namespace cdc_ft

common/server_socket.cc

@@ -12,90 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "cdc_rsync_server/server_socket.h"
+#include "common/server_socket.h"
 
 #include "common/log.h"
-#include "common/platform.h"
+#include "common/socket_internal.h"
 #include "common/status.h"
-#include "common/util.h"
-
-#if PLATFORM_WINDOWS
-
-#include <winsock2.h>
-
-#elif PLATFORM_LINUX
-
-#include <netinet/in.h>
-#include <sys/socket.h>
-#include <unistd.h>
-
-#include <cerrno>
-
-#endif
 
 namespace cdc_ft {
-namespace {
-
-#if PLATFORM_WINDOWS
-
-using SocketType = SOCKET;
-using SockAddrType = SOCKADDR;
-constexpr SocketType kInvalidSocket = INVALID_SOCKET;
-constexpr int kSocketError = SOCKET_ERROR;
-constexpr int kSendingEnd = SD_SEND;
-
-constexpr int kErrAgain = WSAEWOULDBLOCK;  // There's no EAGAIN on Windows.
-constexpr int kErrWouldBlock = WSAEWOULDBLOCK;
-constexpr int kErrAddrInUse = WSAEADDRINUSE;
-
-int GetLastError() { return WSAGetLastError(); }
-std::string GetErrorStr(int err) { return Util::GetWin32Error(err); }
-void Close(SocketType* socket) {
-  if (*socket != kInvalidSocket) {
-    closesocket(*socket);
-    *socket = kInvalidSocket;
-  }
-}
-
-// Not necessary on Windows.
-#define HANDLE_EINTR(x) (x)
-
-#elif PLATFORM_LINUX
-
-using SocketType = int;
-using SockAddrType = sockaddr;
-constexpr SocketType kInvalidSocket = -1;
-constexpr int kSocketError = -1;
-constexpr int kSendingEnd = SHUT_WR;
-
-constexpr int kErrAgain = EAGAIN;
-constexpr int kErrWouldBlock = EWOULDBLOCK;
-constexpr int kErrAddrInUse = EADDRINUSE;
-
-int GetLastError() { return errno; }
-std::string GetErrorStr(int err) { return strerror(err); }
-void Close(SocketType* socket) {
-  if (*socket != kInvalidSocket) {
-    close(*socket);
-    *socket = kInvalidSocket;
-  }
-}
-
-// Keep re-evaluating the expression |x| while it returns EINTR.
-#define HANDLE_EINTR(x)                                     \
-  ({                                                        \
-    decltype(x) eintr_wrapper_result;                       \
-    do {                                                    \
-      eintr_wrapper_result = (x);                           \
-    } while (eintr_wrapper_result == -1 && errno == EINTR); \
-    eintr_wrapper_result;                                   \
-  })
-
-#endif
-
-std::string GetLastErrorStr() { return GetErrorStr(GetLastError()); }
-
-}  // namespace
 
 struct ServerSocketInfo {
   // Listening socket file descriptor (where new connections are accepted).
@@ -113,15 +36,63 @@ ServerSocket::~ServerSocket() {
   StopListening();
 }
 
-absl::Status ServerSocket::StartListening(int port) {
+// static
+absl::StatusOr<int> ServerSocket::FindAvailablePort() {
+  ServerSocket socket;
+  return socket.StartListening(0);
+}
+
+absl::StatusOr<int> ServerSocket::StartListening(int port) {
   if (socket_info_->listen_sock != kInvalidSocket) {
     return MakeStatus("Already listening");
   }
 
-  LOG_DEBUG("Open socket");
+  // Find addrinfos suitable for listening via IPV4 and IPV6.
-  socket_info_->listen_sock = socket(AF_INET, SOCK_STREAM, 0);
+  addrinfo hints;
+  addrinfo* addr_infos = nullptr;
+  memset(&hints, 0, sizeof(hints));
+  hints.ai_family = PF_UNSPEC;
+  hints.ai_socktype = SOCK_STREAM;
+  hints.ai_protocol = IPPROTO_TCP;
+  // AI_PASSIVE indicates that the addresses are used with bind(). The returned
+  // addresses will be the unspecified addresses for each family.
+  hints.ai_flags = AI_NUMERICHOST | AI_PASSIVE;
+  int result = getaddrinfo(/*address=*/nullptr, std::to_string(port).c_str(),
+                           &hints, &addr_infos);
+  if (result != 0) {
+    return MakeStatus("Getting address infos failed: %s", GetLastErrorStr());
+  }
+  AddrInfoReleaser releaser(addr_infos);
+
+  // Prefer IPV6 sockets. They can also accept IPV4 connections.
+  for (addrinfo* curr = addr_infos; curr; curr = curr->ai_next) {
+    if (curr->ai_family == PF_INET6) {
+      return StartListeningInternal(port, curr);
+    }
+  }
+
+  // Fall back to IPV4 sockets.
+  for (addrinfo* curr = addr_infos; curr; curr = curr->ai_next) {
+    if (curr->ai_family == PF_INET) {
+      return StartListeningInternal(port, curr);
+    }
+  }
+
+  return MakeStatus("No IPV4 and IPV6 network addresses available");
+}
+
+absl::StatusOr<int> ServerSocket::StartListeningInternal(int port,
+                                                         addrinfo* addr) {
+  assert(addr->ai_family == PF_INET || addr->ai_family == PF_INET6);
+  const char* family = addr->ai_family == PF_INET ? "IPV4" : "IPV6";
+
+  // Open a socket with the correct address family for this address.
+  LOG_DEBUG("Open %s listen socket", family);
+  socket_info_->listen_sock =
+      socket(addr->ai_family, addr->ai_socktype, addr->ai_protocol);
   if (socket_info_->listen_sock == kInvalidSocket) {
-    return MakeStatus("Creating listen socket failed: %s", GetLastErrorStr());
+    return MakeStatus("Creating %s listen socket failed: %s", family,
+                      GetLastErrorStr());
   }
 
   // If the program terminates abnormally, the socket might remain in a
@@ -136,16 +107,21 @@ absl::Status ServerSocket::StartListening(int port) {
     LOG_DEBUG("Enabling address reusal failed");
   }
 
-  LOG_DEBUG("Bind socket");
+  // Allow ipv4 connections on the ipv6 socket. By default, ipv6 sockets only
-  sockaddr_in serv_addr;
+  // allow ipv4 connections on Windows.
-  memset(&serv_addr, 0, sizeof(serv_addr));
+  if (addr->ai_family == PF_INET6) {
-  serv_addr.sin_family = AF_INET;
+    const int disable = 0;
-  serv_addr.sin_addr.s_addr = INADDR_ANY;
+    result =
-  serv_addr.sin_port = htons(port);
+        setsockopt(socket_info_->listen_sock, IPPROTO_IPV6, IPV6_V6ONLY,
+                   reinterpret_cast<const char*>(&disable), sizeof(disable));
+    if (result == kSocketError) {
+      LOG_DEBUG("Disabling IPV6-only failed");
+    }
+  }
 
-  result = bind(socket_info_->listen_sock,
+  LOG_DEBUG("Bind socket");
-                reinterpret_cast<const SockAddrType*>(&serv_addr),
+  result = bind(socket_info_->listen_sock, addr->ai_addr,
-                sizeof(serv_addr));
+                static_cast<int>(addr->ai_addrlen));
   if (result == kSocketError) {
     int err = GetLastError();
     absl::Status status =
@@ -159,6 +135,21 @@ absl::Status ServerSocket::StartListening(int port) {
     return status;
   }
 
+  if (port == 0) {
+    // Find out which port was auto-selected.
+    socklen_t len = addr->ai_addrlen;
+    result = getsockname(socket_info_->listen_sock, addr->ai_addr, &len);
+    if (result == kSocketError) {
+      Close(&socket_info_->listen_sock);
+      return MakeStatus("Getting port failed: %s", GetLastErrorStr());
+    }
+    if (addr->ai_family == PF_INET) {
+      port = ntohs(reinterpret_cast<sockaddr_in*>(addr->ai_addr)->sin_port);
+    } else if (addr->ai_family == PF_INET6) {
+      port = ntohs(reinterpret_cast<sockaddr_in6*>(addr->ai_addr)->sin6_port);
+    }
+  }
+
   LOG_DEBUG("Listen");
   result = listen(socket_info_->listen_sock, 1);
   if (result == kSocketError) {
@@ -167,18 +158,21 @@ absl::Status ServerSocket::StartListening(int port) {
     return MakeStatus("Listening to socket failed: %s", GetErrorStr(err));
   }
 
-  return absl::OkStatus();
+  return port;
 }
 
 void ServerSocket::StopListening() {
   Close(&socket_info_->listen_sock);
-  LOG_INFO("Stopped listening.");
+  LOG_DEBUG("Stopped listening.");
 }
 
 absl::Status ServerSocket::WaitForConnection() {
   if (socket_info_->conn_sock != kInvalidSocket) {
     return MakeStatus("Already connected");
   }
+  if (socket_info_->listen_sock == kInvalidSocket) {
+    return MakeStatus("Not listening");
+  }
 
   socket_info_->conn_sock = accept(socket_info_->listen_sock, nullptr, nullptr);
   if (socket_info_->conn_sock == kInvalidSocket) {
@@ -191,7 +185,7 @@ absl::Status ServerSocket::WaitForConnection() {
 
 void ServerSocket::Disconnect() {
   Close(&socket_info_->conn_sock);
-  LOG_INFO("Disconnected");
+  LOG_DEBUG("Disconnected");
 }
 
 absl::Status ServerSocket::ShutdownSendingEnd() {

common/server_socket.h

@@ -14,11 +14,14 @@
  * limitations under the License.
  */
 
-#ifndef CDC_RSYNC_SERVER_SERVER_SOCKET_H_
+#ifndef COMMON_SERVER_SOCKET_H_
-#define CDC_RSYNC_SERVER_SERVER_SOCKET_H_
+#define COMMON_SERVER_SOCKET_H_
 
 #include "absl/status/status.h"
-#include "cdc_rsync/base/socket.h"
+#include "absl/status/statusor.h"
+#include "common/socket.h"
+
+struct addrinfo;
 
 namespace cdc_ft {
 
@@ -27,8 +30,19 @@ class ServerSocket : public Socket {
   ServerSocket();
   ~ServerSocket();
 
+  // Returns an available ephemeral port that can be used as a listening port.
+  // Note that calling this function, followed by StartListening() or similar,
+  // is slightly racy as another process might use the port in the meantime.
+  // However, the OS usually returns ephemeral ports in a round-robin manner,
+  // and ports remain in TIME_WAIT state for a while, which may block other apps
+  // from reusing the port. Hence, the chances of races are small. Nevertheless,
+  // consider calling StartListening() with zero |port| if possible.
+  static absl::StatusOr<int> FindAvailablePort();
+
   // Starts listening for connections on |port|.
-  absl::Status StartListening(int port);
+  // Passing 0 as port will bind to any available port.
+  // Returns the port that was bound to.
+  absl::StatusOr<int> StartListening(int port);
 
   // Stops listening for connections. No-op if already stopped/never started.
   void StopListening();
@@ -50,6 +64,11 @@ class ServerSocket : public Socket {
                        size_t* bytes_received) override;
 
  private:
+  // Called by StartListening() for a specific IPV4 or IPV6 |addr_info|.
+  // Passing 0 as port will bind to any available port.
+  // Returns the port that was bound to.
+  absl::StatusOr<int> StartListeningInternal(int port, addrinfo* addr);
+
   std::unique_ptr<struct ServerSocketInfo> socket_info_;
 };
 

common/socket.cc

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-#include "cdc_rsync/base/socket.h"
+#include "common/socket.h"
 
 #include "common/log.h"
 #include "common/platform.h"

common/socket.h

@@ -14,8 +14,8 @@
  * limitations under the License.
  */
 
-#ifndef CDC_RSYNC_BASE_SOCKET_H_
+#ifndef COMMON_SOCKET_H_
-#define CDC_RSYNC_BASE_SOCKET_H_
+#define COMMON_SOCKET_H_
 
 #include "absl/status/status.h"
 
@@ -56,4 +56,4 @@ class SocketFinalizer {
 
 }  // namespace cdc_ft
 
-#endif  // CDC_RSYNC_BASE_SOCKET_H_
+#endif  // COMMON_SOCKET_H_

common/socket_internal.h

@@ -0,0 +1,121 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef COMMON_SOCKET_INTERNAL_H_
+
+#include "common/platform.h"
+#include "common/util.h"
+
+#if PLATFORM_WINDOWS
+
+#include <winsock2.h>
+#include <ws2tcpip.h>
+
+#elif PLATFORM_LINUX
+
+#include <netdb.h>
+#include <netinet/in.h>
+#include <sys/socket.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include <cerrno>
+
+#endif
+
+namespace cdc_ft {
+namespace {
+
+// Platform-specific abstractions for socket classes.
+
+#if PLATFORM_WINDOWS
+
+using SocketType = SOCKET;
+using SockAddrType = SOCKADDR;
+constexpr SocketType kInvalidSocket = INVALID_SOCKET;
+constexpr int kSocketError = SOCKET_ERROR;
+constexpr int kSendingEnd = SD_SEND;
+
+constexpr int kErrAgain = WSAEWOULDBLOCK;  // There's no EAGAIN on Windows.
+constexpr int kErrWouldBlock = WSAEWOULDBLOCK;
+constexpr int kErrAddrInUse = WSAEADDRINUSE;
+constexpr int kErrConnReset = WSAECONNRESET;
+
+int GetLastError() { return WSAGetLastError(); }
+
+std::string GetErrorStr(int err) { return Util::GetWin32Error(err); }
+
+void Close(SocketType* socket) {
+  if (*socket != kInvalidSocket) {
+    closesocket(*socket);
+    *socket = kInvalidSocket;
+  }
+}
+
+// Not necessary on Windows.
+#define HANDLE_EINTR(x) (x)
+
+#elif PLATFORM_LINUX
+
+using SocketType = int;
+using SockAddrType = sockaddr;
+constexpr SocketType kInvalidSocket = -1;
+constexpr int kSocketError = -1;
+constexpr int kSendingEnd = SHUT_WR;
+
+constexpr int kErrAgain = EAGAIN;
+constexpr int kErrWouldBlock = EWOULDBLOCK;
+constexpr int kErrAddrInUse = EADDRINUSE;
+constexpr int kErrConnReset = ECONNRESET;
+
+int GetLastError() { return errno; }
+
+std::string GetErrorStr(int err) { return strerror(err); }
+
+void Close(SocketType* socket) {
+  if (*socket != kInvalidSocket) {
+    close(*socket);
+    *socket = kInvalidSocket;
+  }
+}
+
+// Keep re-evaluating the expression |x| while it returns EINTR.
+#define HANDLE_EINTR(x)                                     \
+  ({                                                        \
+    decltype(x) eintr_wrapper_result;                       \
+    do {                                                    \
+      eintr_wrapper_result = (x);                           \
+    } while (eintr_wrapper_result == -1 && errno == EINTR); \
+    eintr_wrapper_result;                                   \
+  })
+
+#endif
+
+std::string GetLastErrorStr() { return GetErrorStr(GetLastError()); }
+
+class AddrInfoReleaser {
+ public:
+  AddrInfoReleaser(addrinfo* addr_infos) : addr_infos_(addr_infos) {}
+  ~AddrInfoReleaser() { freeaddrinfo(addr_infos_); }
+
+ private:
+  addrinfo* addr_infos_;
+};
+
+}  // namespace
+}  // namespace cdc_ft
+
+#endif  // COMMON_SOCKET_INTERNAL_H_

common/threadpool.cc

@@ -45,6 +45,11 @@ void Threadpool::Shutdown() {
   for (auto& worker : workers_) {
     if (worker.joinable()) worker.join();
   }
+
+  // Discard all completed tasks.
+  absl::MutexLock lock(&completed_tasks_mutex_);
+  std::queue<std::unique_ptr<Task>> empty;
+  std::swap(completed_tasks_, empty);
 }
 
 void Threadpool::QueueTask(std::unique_ptr<Task> task) {
@@ -77,6 +82,21 @@ std::unique_ptr<Task> Threadpool::GetCompletedTask() {
   return task;
 }
 
+void Threadpool::SetTaskCompletedCallback(TaskCompletedCallback cb) {
+  absl::MutexLock lock(&completed_tasks_mutex_);
+  on_task_completed_ = std::move(cb);
+}
+
+bool Threadpool::WaitForQueuedTasksAtMost(size_t count,
+                                          absl::Duration timeout) const {
+  absl::MutexLock lock(&task_queue_mutex_);
+  auto cond = [this, count]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(task_queue_mutex_) {
+    return shutdown_ || outstanding_task_count_ <= count;
+  };
+  return task_queue_mutex_.AwaitWithTimeout(absl::Condition(&cond), timeout) &&
+         outstanding_task_count_ <= count;
+}
+
 void Threadpool::ThreadWorkerMain() {
   bool task_finished = false;
   for (;;) {
@@ -85,7 +105,8 @@ void Threadpool::ThreadWorkerMain() {
       absl::MutexLock lock(&task_queue_mutex_);
 
       // Decrease task count here, so we don't have to lock again at the end of
-      // the loop.
+      // the loop. It is important to first push the task, then decrease this
+      // count. Otherwise, there's a race between Wait() and GetCompletedTask().
       if (task_finished) {
         assert(outstanding_task_count_ > 0);
         --outstanding_task_count_;
@@ -104,17 +125,18 @@ void Threadpool::ThreadWorkerMain() {
     }
 
     // Run task, but make it cancellable.
-    task->ThreadRun([this]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(
+    task->ThreadRun([this]() ABSL_LOCKS_EXCLUDED(task_queue_mutex_) -> bool {
-                        task_queue_mutex_) -> bool { return shutdown_; });
-
-    {
       absl::MutexLock lock(&task_queue_mutex_);
-      if (shutdown_) break;
+      return shutdown_;
-    }
+    });
 
     // Push task to completed queue.
     absl::MutexLock lock(&completed_tasks_mutex_);
+    if (on_task_completed_) {
+      on_task_completed_(std::move(task));
+    } else {
       completed_tasks_.push(std::move(task));
+    }
     task_finished = true;
   }
 }

common/threadpool.h

@@ -18,7 +18,6 @@
 #define COMMON_THREADPOOL_H_
 
 #include <atomic>
-#include <condition_variable>
 #include <functional>
 #include <memory>
 #include <queue>
@@ -57,7 +56,8 @@ class Threadpool {
   void QueueTask(std::unique_ptr<Task> task)
       ABSL_LOCKS_EXCLUDED(task_queue_mutex_);
 
-  // If available, returns the next completed task.
+  // Returns the next completed task if available or nullptr all are either
+  // queued or in progress.
   // For a single worker thread (|num_threads| == 1), tasks are completed in
   // FIFO order. This is no longer the case for multiple threads
   // (|num_threads| > 1). Tasks that got queued later might complete first.
@@ -71,6 +71,14 @@ class Threadpool {
   std::unique_ptr<Task> GetCompletedTask()
       ABSL_LOCKS_EXCLUDED(completed_tasks_mutex_);
 
+  using TaskCompletedCallback = std::function<void(std::unique_ptr<Task>)>;
+
+  // Set a callback that is called immediately in a background thread when a
+  // task is completed. The task will not be put onto the completed queue, so
+  // if this callback is set, do not call (Try)GetCompletedTask.
+  void SetTaskCompletedCallback(TaskCompletedCallback cb)
+      ABSL_LOCKS_EXCLUDED(completed_tasks_mutex_);
+
   // Returns the total number of worker threads in the pool.
   size_t NumThreads() const { return workers_.size(); }
 
@@ -80,6 +88,14 @@ class Threadpool {
     return outstanding_task_count_;
   }
 
+  // Block until the number of queued tasks drops below or equal to |count|, or
+  // until the timeout is exceeded, or until Shutdown() is called, whatever
+  // comes sooner. Returns true if less than or equal to |count| tasks are
+  // queued.
+  bool WaitForQueuedTasksAtMost(
+      size_t count, absl::Duration timeout = absl::InfiniteDuration()) const
+      ABSL_LOCKS_EXCLUDED(mutex_);
+
  private:
   // Background thread worker method. Picks tasks and runs them.
   void ThreadWorkerMain()
@@ -94,6 +110,8 @@ class Threadpool {
   absl::Mutex completed_tasks_mutex_;
   std::queue<std::unique_ptr<Task>> completed_tasks_
       ABSL_GUARDED_BY(completed_tasks_mutex_);
+  TaskCompletedCallback on_task_completed_
+      ABSL_GUARDED_BY(completed_tasks_mutex_);
 
   std::vector<std::thread> workers_;
 };

common/threadpool_test.cc

@@ -151,5 +151,37 @@ TEST_F(ThreadpoolTest, GetCompletedTask) {
   EXPECT_EQ(completed_task.get(), task);
 }
 
+TEST_F(ThreadpoolTest, SetTaskCompletedCallback) {
+  auto task_func = [](Task::IsCancelledPredicate) { /* empty */ };
+
+  Semaphore task_finished(0);
+  Threadpool pool(1);
+  std::atomic_bool finished = false;
+  pool.SetTaskCompletedCallback(
+      [&task_finished, &finished](std::unique_ptr<Task> task) {
+        finished = true;
+        task_finished.Signal();
+      });
+  pool.QueueTask(std::make_unique<TestTask>(task_func));
+  task_finished.Wait();
+  EXPECT_TRUE(finished);
+  EXPECT_FALSE(pool.TryGetCompletedTask());
+}
+
+TEST_F(ThreadpoolTest, WaitForQueuedTasksAtMost) {
+  Semaphore task_signal(0);
+  auto task_func = [&task_signal](Task::IsCancelledPredicate) {
+    task_signal.Wait();
+  };
+  Threadpool pool(1);
+  pool.QueueTask(std::make_unique<TestTask>(task_func));
+  pool.QueueTask(std::make_unique<TestTask>(task_func));
+  EXPECT_FALSE(pool.WaitForQueuedTasksAtMost(1, absl::Milliseconds(10)));
+  task_signal.Signal();
+  EXPECT_TRUE(pool.WaitForQueuedTasksAtMost(1, absl::Milliseconds(5000)));
+  EXPECT_EQ(pool.NumQueuedTasks(), 1);
+  task_signal.Signal();
+}
+
 }  // namespace
 }  // namespace cdc_ft