Add Google specific instructions to run without security key touches

.github/workflows/build_and_test.yml

@@ -23,26 +23,17 @@ jobs:
       - name: Initialize submodules
         run: git submodule update --init --recursive
 
-      - name: Create timestamp
-        run: |
-          printf -v date '%(%Y-%m)T' -1
-          echo "date=$date" >> $GITHUB_ENV
-
-      - name: Restore build cache
-        uses: actions/cache@v3
-        with:
-          path: bazel-cache
-          key: ${{ runner.os }}-bazel-cache-fastbuild-${{ env.date }}
-
       - name: Build (fastbuild)
-        run: bazel build --config=linux --disk_cache=bazel-cache -- //... -//third_party/...
+        run: |
+          bazel build --config=linux -- //... -//third_party/...
 
       # Skip file_finder_test: The test works when file_finder_test is run
       # directly, but not through bazel test. The reason is, bazel test
       # creates symlinks of test files, but the finder ignores symlinks.
       # Also run tests sequentially since some tests write to a common tmp dir.
       - name: Test (fastbuild)
-        run: bazel test --config=linux --disk_cache=bazel-cache --test_output=errors --local_test_jobs=1 -- //... -//third_party/... -//cdc_rsync_server:file_finder_test
+        run: |
+          bazel test --config=linux --test_output=errors --local_test_jobs=1 -- //... -//third_party/... -//cdc_rsync_server:file_finder_test
 
   Build-And-Test-Windows:
     runs-on: windows-2019
@@ -52,26 +43,16 @@ jobs:
       - name: Initialize submodules
         run: git submodule update --init --recursive
 
-      - name: Create timestamp
+      - name: Build
         run: |
-          $date = Get-Date -Format "yyyy-MM"
+          bazel build --config=windows //cdc_rsync //cdc_stream //tests_common //tests_cdc_stream //tests_cdc_rsync
-          echo "date=$date" >> $env:GITHUB_ENV
 
-      - name: Restore build cache
+      - name: Test
-        uses: actions/cache@v3
-        with:
-          path: bazel-cache
-          key: ${{ runner.os }}-bazel-cache-fastbuild-${{ env.date }}
-
-      - name: Build (fastbuild)
-        run: bazel build --config=windows --disk_cache=bazel-cache //cdc_rsync //cdc_stream //tests_common //tests_cdc_stream //tests_cdc_rsync
-
-      - name: Test (fastbuild)
         run: |
           bazel-bin\tests_common\tests_common.exe
           bazel-bin\tests_cdc_stream\tests_cdc_stream.exe
           bazel-bin\tests_cdc_rsync\tests_cdc_rsync.exe
-          bazel test --config=windows --disk_cache=bazel-cache --test_output=errors --local_test_jobs=1 `
+          bazel test --config=windows --test_output=errors --local_test_jobs=1 `
             //cdc_fuse_fs/... `
             //cdc_rsync/... `
             //cdc_rsync/base/... `

.github/workflows/create_release.yml

@@ -14,55 +14,21 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Use last commit hash as build version for the developer build.
-        if: "startsWith(github.ref, 'refs/heads/')"
-        run: echo "build_version=${GITHUB_SHA}" >> $GITHUB_ENV
-
-      - name: Use tag name as build version for the release build.
-        if: startsWith(github.ref, 'refs/tags/v')
-        run: echo "build_version=${GITHUB_REF#refs/tags/v}" >> $GITHUB_ENV
-
-      # This flow should not be used for pull requests. However the section
-      # below might be useful for testing purposes.
-      - name: Use last commit hash as build version for the pull request
-        if: startsWith(github.ref, 'refs/pull')
-        run: echo "build_version=${GITHUB_SHA}" >> $GITHUB_ENV
-
-      - name: Replace CDC_BUILD_VERSION
-        run: |
-          if grep -q "DCDC_BUILD_VERSION=DEV" "common/BUILD"; then
-              sed -i 's/DCDC_BUILD_VERSION=DEV/DCDC_BUILD_VERSION=${{ env.build_version }}/g' common/BUILD
-          else
-              echo "CDC_BUILD_VERSION was moved out from common/BUILD file."
-              echo "Please edit create_release.yaml workflow."
-              exit 1
-          fi
-
       - name: Initialize submodules
         run: git submodule update --init --recursive
 
-      - name: Create timestamp
+      - name: Build
         run: |
-          printf -v date '%(%Y-%m)T' -1
+          bazel build --config=linux --compilation_mode=opt --linkopt=-Wl,--strip-all --copt=-fdata-sections --copt=-ffunction-sections --linkopt=-Wl,--gc-sections \
-          echo "date=$date" >> $GITHUB_ENV
-
-      - name: Restore build cache
-        uses: actions/cache@v3
-        with:
-          path: bazel-cache
-          key: ${{ runner.os }}-bazel-cache-opt-${{ env.date }}
-
-      - name: Build (opt)
-        run: |
-          bazel build --config=linux --disk_cache=bazel-cache --compilation_mode=opt --linkopt=-Wl,--strip-all --copt=-fdata-sections --copt=-ffunction-sections --linkopt=-Wl,--gc-sections \
             //cdc_fuse_fs //cdc_rsync_server
 
-      - name: Test (opt)
+      - name: Test
         run: |
-          bazel test --config=linux --disk_cache=bazel-cache --compilation_mode=opt --linkopt=-Wl,--strip-all --copt=-fdata-sections --copt=-ffunction-sections --linkopt=-Wl,--gc-sections \
+          bazel test --config=linux --compilation_mode=opt --linkopt=-Wl,--strip-all --copt=-fdata-sections --copt=-ffunction-sections --linkopt=-Wl,--gc-sections \
                      --test_output=errors --local_test_jobs=1  \
                      -- //... -//third_party/... -//cdc_rsync_server:file_finder_test
 
+      # The artifact collector doesn't like the fact that bazel-bin is a symlink.
       - name: Copy artifacts
         run: |
           mkdir artifacts
@@ -81,65 +47,20 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Use last commit hash as build version for the developer build.
-        if: "startsWith(github.ref, 'refs/heads/')"
-        run: |
-          $build_version="${{ github.sha }}"
-          echo "build_version=$build_version" >> $env:GITHUB_ENV
-
-      - name: Use tag name as build version for the release build.
-        if: "startsWith(github.ref, 'refs/tags/v')"
-        run: |
-          $build_version="${{ github.ref }}".replace("refs/tags/v", "")
-          echo "build_version=$build_version" >> $env:GITHUB_ENV
-
-      # This flow should not be used for pull requests. However the section
-      # below might be useful for testing purposes.
-      - name: Use last commit hash as build version for the pull request
-        if: startsWith(github.ref, 'refs/pull')
-        run: |
-          $build_version="${{ github.sha }}"
-          echo "build_version=$build_version" >> $env:GITHUB_ENV
-
-      - name: Replace CDC_BUILD_VERSION
-        run: |
-          $cdc_version = Select-String -Path common/BUILD -Pattern "DCDC_BUILD_VERSION=DEV"
-          if ($cdc_version -ne $null) {
-              $build_file = Get-Content -path common/BUILD -Raw
-              $build_file = $build_file -replace 'DCDC_BUILD_VERSION=DEV','DCDC_BUILD_VERSION=${{ env.build_version }}'
-              $build_file | Set-Content -Path common/BUILD
-          }
-          else {
-              Write-Host "CDC_BUILD_VERSION was moved out from common/BUILD file."
-              Write-Host "Please edit create_release.yaml workflow."
-              exit 1
-          }
-
       - name: Initialize submodules
         run: git submodule update --init --recursive
 
-      - name: Create timestamp
+      - name: Build
         run: |
-          $date = Get-Date -Format "yyyy-MM"
+          bazel build --config=windows --compilation_mode=opt --copt=/GL `
-          echo "date=$date" >> $env:GITHUB_ENV
-
-      - name: Restore build cache
-        uses: actions/cache@v3
-        with:
-          path: bazel-cache
-          key: ${{ runner.os }}-bazel-cache-opt-${{ env.date }}
-
-      - name: Build (opt)
-        run: |
-          bazel build --config=windows --disk_cache=bazel-cache --compilation_mode=opt --copt=/GL `
                        //cdc_rsync //cdc_stream //tests_common //tests_cdc_stream //tests_cdc_rsync
 
-      - name: Test (opt)
+      - name: Test
         run: |
           bazel-bin\tests_common\tests_common.exe
           bazel-bin\tests_cdc_stream\tests_cdc_stream.exe
           bazel-bin\tests_cdc_rsync\tests_cdc_rsync.exe
-          bazel test --config=windows --disk_cache=bazel-cache --compilation_mode=opt --copt=/GL --test_output=errors --local_test_jobs=1 `
+          bazel test --config=windows --compilation_mode=opt --copt=/GL --test_output=errors --local_test_jobs=1 `
             //cdc_fuse_fs/... `
             //cdc_rsync/... `
             //cdc_rsync/base/... `
@@ -151,16 +72,14 @@ jobs:
             //manifest/... `
             //metrics/...
 
+      # The artifact collector doesn't like the fact that bazel-bin is a symlink.
       - name: Copy artifacts
         run: |
           mkdir artifacts
-          mkdir artifacts\docs
           cp bazel-bin/cdc_rsync/cdc_rsync.exe artifacts
-          cp bazel-bin/cdc_rsync_server/cdc_rsync_server.exe artifacts
           cp bazel-bin/cdc_stream/cdc_stream.exe artifacts
           cp LICENSE artifacts
           cp README.md artifacts
-          cp docs\* artifacts\docs
 
       - name: Upload artifacts
         uses: actions/upload-artifact@v3
@@ -178,10 +97,7 @@ jobs:
       - name: Zip binaries
         run: |
           # The ref resolves to "main" for latest and e.g. "v0.1.0" for tagged.
-          REF=${GITHUB_REF#refs/*/}
+          BINARIES_ZIP_NAME=cdc-file-transfer-binaries-${GITHUB_REF#refs/*/}-x64.zip
-          # For pull requests, this is e.g. '70/merge', so replace / with -.
-          REF=${REF/\//-}
-          BINARIES_ZIP_NAME=cdc-file-transfer-binaries-$REF-x64.zip
           echo "BINARIES_ZIP_NAME=$BINARIES_ZIP_NAME" >> $GITHUB_ENV
           zip -j $BINARIES_ZIP_NAME Windows-Artifacts/* Linux-Artifacts/*
 

.github/workflows/lint.yml

@@ -3,7 +3,7 @@ name: Lint
 on:
   push:
     branches:
-      - main
+      - master
   pull_request:
 
 jobs:

.gitignore

@@ -12,4 +12,3 @@ dependencies
 .qtc_clangd
 bazel-*
 user.bazelrc
-*.pyc

README.md

@@ -1,8 +1,8 @@
 # CDC File Transfer
 
-Born from the ashes of Stadia, this repository contains tools for syncing and
+Born from the ashes of Stadia, this repository contains tools for synching and
-streaming files from Windows to Windows or Linux. The tools are based on Content
+streaming files from Windows to Linux. They are based on Content Defined
-Defined Chunking (CDC), in particular
+Chunking (CDC), in particular
 [FastCDC](https://www.usenix.org/conference/atc16/technical-sessions/presentation/xia),
 to split up files into chunks.
 
@@ -38,103 +38,27 @@ version of the files available in the target directory.
 </p>
 
 The remote diffing algorithm is based on CDC. In our tests, it is up to 30x
-faster than the one used in `rsync` (1500 MB/s vs 50 MB/s).
+faster than the one used in rsync (1500 MB/s vs 50 MB/s).
 
-The following chart shows a comparison of `cdc_rsync` and Linux `rsync` running
+The following chart shows a comparison of `cdc_rsync` and Linux rsync running
 under Cygwin on Windows. The test data consists of 58 development builds
 of some game provided to us for evaluation purposes. The builds are 40-45 GB
 large. For this experiment, we uploaded the first build, then synced the second
 build with each of the two tools and measured the time. For example, syncing
-from build 1 to build 2 took 210 seconds with the Cygwin `rsync`, but only 75
+from build 1 to build 2 took 210 seconds with the Linux rsync, but only 75
 seconds with `cdc_rsync`. The three outliers are probably feature drops from
 another development branch, where the delta was much higher. Overall,
-`cdc_rsync` syncs files about **3 times faster** than Cygwin `rsync`.
+`cdc_rsync` syncs files about **3 times faster** than Linux rsync.
 
 <p align="center">
   <img src="docs/cdc_rsync_vs_cygwin_rsync.png" alt="Comparison of cdc_rsync and Linux rsync running in Cygwin" width="753" />
 </p>
 
-We also ran the experiment with the native Linux `rsync`, i.e syncing Linux to
-Linux, to rule out issues with Cygwin. Linux `rsync` performed on average 35%
-worse than Cygwin `rsync`, which can be attributed to CPU differences. We did
-not include it in the figure because of this, but you can find it
-[here](docs/cdc_rsync_vs_cygwin_rsync_vs_linux_rsync.png).
-
-### How does it work and why is it faster?
-
-The standard Linux `rsync` splits a file into fixed-size chunks of typically
-several KB.
-
-<p align="center">
-  <img src="docs/fixed_size_chunks.png" alt="Linux rsync uses fixed size chunks" width="258" />
-</p>
-
-If the file is modified in the middle, e.g. by inserting `xxxx` after `567`,
-this usually means that <span style="color: red">the modified chunks as well as
-all subsequent chunks</span> change.
-
-<p align="center">
-  <img src="docs/fixed_size_chunks_inserted.png" alt="Fixed size chunks after inserting data" width="301" />
-</p>
-
-The standard `rsync` algorithm hashes the chunks of the remote "old" file
-and sends the hashes to the local device. The local device then figures out 
-which parts of the "new" file matches known chunks.
-
-<p align="center">
-  <img src="docs/linux_rsync_animation.gif" alt="Syncing a file with the standard Linux rsync" width="855" />
-  <br>
-  Standard rsync algorithm
-</p>
-
-This is a simplification. The actual algorithm is more complicated and uses
-two hashes, a weak rolling hash and a strong hash, see
-[here](https://rsync.samba.org/tech_report/) for a great overview. What makes
-`rsync` relatively slow is the "no match" situation where the rolling hash does
-not match any remote hash, and the algorithm has to roll the hash forward and
-perform a hash map lookup for each byte. `rsync` goes to
-[great lengths](https://github.com/librsync/librsync/blob/master/src/hashtable.h)
-optimizing lookups.
-
-`cdc_rsync` does not use fixed-size chunks, but instead variable-size,
-content-defined chunks. That means, chunk boundaries are determined by the
-*local content* of the file, in practice a 64 byte sliding window. For more
-details, see
-[the FastCDC paper](https://www.usenix.org/conference/atc16/technical-sessions/presentation/xia)
-or take a look at [our implementation](fastcdc/fastcdc.h).
-
-<p align="center">
-  <img src="docs/variable_size_chunks.png" alt="cdc_rsync uses variable, content-defined size chunks" width="260" />
-</p>
-
-If the file is modified in the middle, only <span style="color: red">the modified
-chunks</span>, but not <span style="color: #38761d">subsequent chunks</span>
-change (unless they are less than 64 bytes away from the modifications).
-
-<p align="center">
-  <img src="docs/variable_size_chunks_inserted.png" alt="Content-defined chunks after inserting data" width="314" />
-</p>
-
-Computing the chunk boundaries is cheap and involves only a left-shift, a memory
-lookup, an `add` and an `and` operation for each input byte. This is cheaper
-than the hash map lookup for the standard `rsync` algorithm.
-
-Because of this, the `cdc_rsync` algorithm is faster than the standard
-`rsync`. It is also simpler. Since chunk boundaries move along with insertions
-or deletions, the task to match local and remote hashes is a trivial set
-difference operation. It does not involve a per-byte hash map lookup.
-
-<p align="center">
-  <img src="docs/cdc_rsync_animation.gif" alt="Syncing a file with cdc_rsync" width="857" />
-  <br>
-  cdc_rsync algorithm
-</p>
-
 ## CDC Stream
 
-`cdc_stream` is a tool to stream files and directories from a Windows machine to
+`cdc_stream` is a tool to stream files and directories from a Windows machine to a
-a Linux device. Conceptually, it is similar to
+Linux device. Conceptually, it is similar to [sshfs](https://github.com/libfuse/sshfs),
-[sshfs](https://github.com/libfuse/sshfs), but it is optimized for read speed.
+but it is optimized for read speed.
 * It caches streamed data on the Linux device.
 * If a file is re-read on Linux after it changed on Windows, only the
   differences are streamed again. The rest is read from the cache.
@@ -161,48 +85,18 @@ In one case, the game is streamed via `sshfs`, in the other case we use
   <img src="docs/cdc_stream_vs_sshfs.png" alt="Comparison of cdc_stream and sshfs" width="752" />
 </p>
 
-# Supported Platforms
-
-| `cdc_rsync`                  | From                 | To                   |
-|:-----------------------------|:--------------------:|:--------------------:|
-| Windows x86_64               | &check;              | &check; <sup>1</sup> |
-| Ubuntu 22.04 x86_64          | &cross; <sup>2</sup> | &check;              |
-| Ubuntu 22.04 aarch64         | &cross;              | &cross;              |
-| macOS 13 x86_64 <sup>3</sup> | &cross;              | &cross;              |
-| macOS 13 aarch64 <sup>3</sup>| &cross;              | &cross;              |
-
-| `cdc_stream`                 | From                 | To                   |
-|:-----------------------------|:--------------------:|:--------------------:|
-| Windows x86_64               | &check;              | &cross;              |
-| Ubuntu 22.04 x86_64          | &cross;              | &check;              |
-| Ubuntu 22.04 aarch64         | &cross;              | &cross;              |
-| macOS 13 x86_64 <sup>3</sup> | &cross;              | &cross;              |
-| macOS 13 aarch64 <sup>3</sup>| &cross;              | &cross;              |
-
-<span style="font-size: 0.8rem">
-
-<sup>1</sup> Only local syncs, e.g. `cdc_rsync C:\src\* C:\dst`. Support for
-remote syncs is being added, see
-[#61](https://github.com/google/cdc-file-transfer/issues/61).  
-<sup>2</sup> See [#56](https://github.com/google/cdc-file-transfer/issues/56).  
-<sup>3</sup> See [#62](https://github.com/google/cdc-file-transfer/issues/62).
-
-</span>
-
 # Getting Started
 
 Download the precompiled binaries from the
-[latest release](https://github.com/google/cdc-file-transfer/releases) to a
+[latest release](https://github.com/google/cdc-file-transfer/releases).
-Windows device and unzip them. The Linux binaries are automatically deployed
+We currently provide Linux binaries compiled on
-to `~/.cache/cdc-file-transfer` by the Windows tools. There is no need to manually
-deploy them. We currently provide Linux binaries compiled on
 [Github's latest Ubuntu](https://github.com/actions/runner-images) version.
 If the binaries work for you, you can skip the following two sections.
 
 Alternatively, the project can be built from source. Some binaries have to be
 built on Windows, some on Linux.
 
-## Prerequisites for Building
+## Prerequisites
 
 To build the tools from source, the following steps have to be executed on
 **both Windows and Linux**.
@@ -220,96 +114,148 @@ To build the tools from source, the following steps have to be executed on
   git submodule update --init --recursive
   ```
 
-Finally, install an SSH client on the Windows machine if not present.
+Finally, install an SSH client on the Windows device if not present.
-The file transfer tools require `ssh.exe` and `sftp.exe`.
+The file transfer tools require `ssh.exe` and `scp.exe`.
 
 ## Building
 
-The two tools CDC RSync and CDC Stream can be built and used independently.
+The two tools can be built and used independently.
 
 ### CDC RSync
 
-* On a Linux device, build the Linux components
+* Build Linux components
   ```
   bazel build --config linux --compilation_mode=opt --linkopt=-Wl,--strip-all --copt=-fdata-sections --copt=-ffunction-sections --linkopt=-Wl,--gc-sections //cdc_rsync_server
   ```
-* On a Windows device, build the Windows components
+* Build Windows components
   ```
   bazel build --config windows --compilation_mode=opt --copt=/GL //cdc_rsync
   ```
 * Copy the Linux build output file `cdc_rsync_server` from 
-  `bazel-bin/cdc_rsync_server` to `bazel-bin\cdc_rsync` on the Windows machine.
+  `bazel-bin/cdc_rsync_server` on the Linux system to `bazel-bin\cdc_rsync`
+  on the Windows machine.
 
 ### CDC Stream
 
-* On a Linux device, build the Linux components
+* Build Linux components
   ```
   bazel build --config linux --compilation_mode=opt --linkopt=-Wl,--strip-all --copt=-fdata-sections --copt=-ffunction-sections --linkopt=-Wl,--gc-sections //cdc_fuse_fs
   ```
-* On a Windows device, build the Windows components
+* Build Windows components
   ```
   bazel build --config windows --compilation_mode=opt --copt=/GL //cdc_stream
   ```
 * Copy the Linux build output files `cdc_fuse_fs` and `libfuse.so` from 
-  `bazel-bin/cdc_fuse_fs` to `bazel-bin\cdc_stream` on the Windows machine.
+  `bazel-bin/cdc_fuse_fs` on the Linux system to `bazel-bin\cdc_stream`
+  on the Windows machine.
 
 ## Usage
 
-The tools require a setup where you can use SSH and SFTP from the Windows
+The tools require a setup where you can use SSH and SCP from the Windows machine
-machine to the Linux device without entering a password, e.g. by using key-based
+to the Linux device without entering a password, e.g. by using key-based
 authentication.
 
-### Configuring SSH and SFTP
+### Configuring SSH and SCP
 
-By default, the tools search `ssh.exe` and `sftp.exe` from the path environment
+By default, the tools search `ssh.exe` and `scp.exe` from the path environment
 variable. If you can run the following commands in a Windows cmd without
 entering your password, you are all set:
 ```
 ssh user@linux.device.com
-sftp user@linux.device.com
+scp somefile.txt user@linux.device.com:
 ```
 Here, `user` is the Linux user and `linux.device.com` is the Linux host to
 SSH into or copy the file to.
 
-If additional arguments are required, it is recommended to provide an SSH config
+If `ssh.exe` or `scp.exe` cannot be found, or if additional arguments are
-file. By default, both `ssh.exe` and `sftp.exe` use the file at
+required, it is recommended to set the environment variables `CDC_SSH_COMMAND`
-`%USERPROFILE%\.ssh\config` on Windows, if it exists. A possible config file
+and `CDC_SCP_COMMAND`. The following example specifies a custom path to the SSH
-that sets a username, a port, an identity file and a known host file could look
+and SCP binaries, a custom SSH config file, a key file and a known hosts file:
-as follows:
 ```
-Host linux_device
+set CDC_SSH_COMMAND="C:\path with space\to\ssh.exe" -F C:\path\to\ssh_config -i C:\path\to\id_rsa -oStrictHostKeyChecking=yes -oUserKnownHostsFile="""C:\path\to\known_hosts"""
-	HostName linux.device.com
+set CDC_SCP_COMMAND="C:\path with space\to\scp.exe" -F C:\path\to\ssh_config -i C:\path\to\id_rsa -oStrictHostKeyChecking=yes -oUserKnownHostsFile="""C:\path\to\known_hosts"""
-	User user
-	Port 12345
-	IdentityFile C:\path\to\id_rsa
-	UserKnownHostsFile C:\path\to\known_hosts
 ```
-If `ssh.exe` or `sftp.exe` cannot be found, you can specify the full paths via
-the command line arguments `--ssh-command` and `--sftp-command` for `cdc_rsync`
-and `cdc_stream start` (see below), or set the environment variables
-`CDC_SSH_COMMAND` and `CDC_SFTP_COMMAND`, e.g.
-```
-set CDC_SSH_COMMAND="C:\path with space\to\ssh.exe"
-set CDC_SFTP_COMMAND="C:\path with space\to\sftp.exe"
-```
-Note that you can also specify SSH configuration via the environment variables
-instead of using a config file:
-```
-set CDC_SSH_COMMAND=C:\path\to\ssh.exe -p 12345 -i C:\path\to\id_rsa -oUserKnownHostsFile=C:\path\to\known_hosts
-set CDC_SFTP_COMMAND=C:\path\to\sftp.exe -P 12345 -i C:\path\to\id_rsa -oUserKnownHostsFile=C:\path\to\known_hosts
-```
-Note the small `-p` for `ssh.exe` and the capital `-P` for `sftp.exe`.
 
 #### Google Specific
 
-For Google internal usage, set the following environment variables to enable SSH
+For Google internal usage, there are two setups, a simple one that requires
-authentication using a Google security key:
+touching the security key for every remote access, and a slightly more complex
+one that tunnels all remote access through an SSH tunnel and only requires one
+security key touch to set up the tunnel.
+
+##### Setup Requiring Security Key Touches
+
+Set the following environment variables to enable SSH authentication using a
+Google security key:
 ```
 set CDC_SSH_COMMAND=C:\gnubby\bin\ssh.exe
-set CDC_SFTP_COMMAND=C:\gnubby\bin\sftp.exe
+set CDC_SCP_COMMAND=C:\gnubby\bin\scp.exe
 ```
 Note that you will have to touch the security key multiple times during the
-first run. Subsequent runs only require a single touch.
+first run of each tool as the Linux components have to be deployed first.
+Subsequent runs only require a single touch.
+
+##### Setup Not Requiring Security Key Touches
+
+This section explains how to run the tools without security key touch for every
+action.
+
+On Linux, generate a host key pair with
+```
+mkdir ~/sshd_runner
+ssh-keygen -f ~/sshd_runner/ssh_host_ed25519_key -N '' -t ed25519
+```
+
+On Windows, create `%USERPROFILE%\sshd_runner\known_hosts` and copy the public
+key from `~/sshd_runner/ssh_host_ed25519_key.pub` to it, using `localhost` and
+port `50000` (if the port is in use, use another port). The file should look
+similar to
+```
+[localhost]:50000 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOR1EInlod0Pm853Ew0p4eXaD3V8y9njf6EoyaPfc8ln	
+```
+
+Next, generate an auth key for the Windows machine with
+```
+ssh-keygen -f %USERPROFILE%\sshd_runner\id_ed25519 -N "" -t ed25519
+```
+Note the subtle difference `-N ''` vs `-N ""`.
+
+On Linux, create `~/sshd_runner/authorized_keys` and copy the public key from
+`%USERPROFILE%\sshd_runner\id_ed25519.pub` to it. The file should look similar
+to
+```
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKleiT/yRhZ9aMutOJ8XaAYX0SPUZHuS50HP4QK5kBft google\user@user-w
+```
+On Windows, open an SSH tunnel to the Linux machine through port 50000.
+This should require one gnubby touch.
+```
+C:\gnubby\bin\ssh.exe -N -L50000:localhost:50000 <user>.<location>.corp.google.com
+```
+On Linux, run the SSH server with
+```
+/usr/sbin/sshd -D -p 50000 -oHostKey=~/sshd_runner/ssh_host_ed25519_key -oAuthorizedKeysFile=~/sshd_runner/authorized_keys -oListenAddress=localhost:50000
+```
+It runs as user, not as root, and only accepts connections from localhost
+through port 50000, so it's secure.
+
+Now you should be able to run SSH on the Windows machine into the Linux device
+without security key touch
+```
+C:\Windows\System32\OpenSSH\ssh.exe -p 50000 -i %USERPROFILE%\sshd_runner\id_ed25519 -oUserKnownHostsFile=%USERPROFILE%\sshd_runner\known_hosts user@localhost
+```
+Next, set `CDC_SSH_COMMAND` and `CDC_SCP_COMMAND` to use the proper flags
+```
+set CDC_SSH_COMMAND=C:\Windows\System32\OpenSSH\ssh.exe -i %USERPROFILE%\sshd_runner\id_ed25519 -oUserKnownHostsFile=%USERPROFILE%\sshd_runner\known_hosts -p 50000
+set CDC_SCP_COMMAND=C:\Windows\System32\OpenSSH\scp.exe -i %USERPROFILE%\sshd_runner\id_ed25519 -oUserKnownHostsFile=%USERPROFILE%\sshd_runner\known_hosts -P 50000
+```
+Note the small `-p` for SSH and the capital `-P` for SCP.
+
+Now the file transfer tools should work without security key touch, but be sure
+to use `localhost` as target host as we're piping the data through the tunnel.
+```
+cdc_rsync C:\assets\* user@localhost:~/assets -vr
+cdc_stream start C:\assets user@localhost:~/assets
+```
 
 ### CDC RSync
 
@@ -332,19 +278,23 @@ To get per file progress, add `-v`:
 ```
 cdc_rsync C:\path\to\assets\* user@linux.device.com:~/assets -vr
 ```
-The tool also supports local syncs:
-```
-cdc_rsync C:\path\to\assets\* C:\path\to\destination -vr
-```
 
 ### CDC Stream
 
+`cdc_stream` consists of a background service, which has to be started in
+advance with
+```
+cdc_stream start-service
+```
+The service logs to `%APPDATA%\cdc-file-transfer\logs` by default. Try
+`cdc_stream --help` to get a list of available flags.
+
 To stream the Windows directory `C:\path\to\assets` to `~/assets` on the Linux
 device, run
 ```
 cdc_stream start C:\path\to\assets user@linux.device.com:~/assets
 ```
-This makes all files and directories in `C:\path\to\assets` available on
+This makes all files and directories of `C:\path\to\assets` available on
 `~/assets` immediately, as if it were a local copy. However, data is streamed
 from Windows to Linux as files are accessed.
 
@@ -352,39 +302,17 @@ To stop the streaming session, enter
 ```
 cdc_stream stop user@linux.device.com:~/assets
 ```
-The command also accepts wildcards. For instance, 
-```
-cdc_stream stop user@*:*
-```
-stops all existing streaming sessions for the given user.
 
 ## Troubleshooting
 
-On first run, `cdc_stream` starts a background service, which does all the work.
+`cdc_rsync` always logs to the console. By default, the `cdc_stream` service
-The `cdc_stream start` and `cdc_stream stop` commands are just RPC clients that
+logs to a timestamped file in `%APPDATA%\cdc-file-transfer\logs`. It can be
-talk to the service.
+switched to log to console by starting it with `--log-to-stdout`:
+```
+cdc_stream start-service --log_to_stdout
+```
 
-The service logs to `%APPDATA%\cdc-file-transfer\logs` by default. The logs are
+Both `cdc_rsync` and `cdc_stream` support command line flags to control log
-useful to investigate issues with asset streaming. To pass custom arguments, or
+verbosity. Passing `-vvv` prints debug logs, `-vvvv` prints verbose logs. The
-to debug the service, create a JSON config file at
+debug logs contain all SSH and SCP commands that are attempted to run, which is
-`%APPDATA%\cdc-file-transfer\cdc_stream.json` with command line flags.
+very useful for troubleshooting.
-For instance,
-```
-{ "verbosity":3 }
-```
-instructs the service to log debug messages. Try `cdc_stream start-service -h`
-for a list of available flags. Alternatively, run the service manually with
-```
-cdc_stream start-service
-```
-and pass the flags as command line arguments. When you run the service manually,
-the flag `--log-to-stdout` is particularly useful as it logs to the console
-instead of to the file.
-
-`cdc_rsync` always logs to the console. To increase log verbosity, pass `-vvv`
-for debug logs or `-vvvv` for verbose logs.
-
-For both sync and stream, the debug logs contain all SSH and SFTP commands that
-are attempted to run, which is very useful for troubleshooting. If a command
-fails unexpectedly, copy it and run it in isolation. Pass `-vv` or `-vvv` for
-additional debug output.

absl.natvis

@@ -1,31 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-
-<AutoVisualizer xmlns="http://schemas.microsoft.com/vstudio/debugger/natvis/2010">
-  <Type Name="absl::flat_hash_map&lt;*&gt;">
-    <DisplayString Condition="size_ == 0">empty</DisplayString>
-    <DisplayString>{{ size={size_} }}</DisplayString>
-    <Expand>
-      <Item Name="[size]" ExcludeView="simple">size_</Item>
-      <Item Name="[capacity]" ExcludeView="simple">capacity_</Item>
-      <CustomListItems MaxItemsPerView="5000">
-        <Variable Name="iSlot" InitialValue="0" />
-        <Size>size_</Size>
-        <Loop>
-          <!-- bool IsFull(ctrl_t c) { return c >= 0; } -->
-          <If Condition="ctrl_[iSlot] &gt;= 0">
-            <Item>slots_[iSlot]</Item>
-          </If>
-          <Exec>iSlot++</Exec>
-          <Break Condition="iSlot == capacity_" />
-        </Loop>
-      </CustomListItems>
-    </Expand>
-  </Type>
-  <Type Name="absl::container_internal::map_slot_type&lt;*&gt;">
-    <DisplayString>{value.first}:{value.second}</DisplayString>
-    <Expand>
-      <Item Name="[key]" ExcludeView="simple">value.first</Item>
-      <Item Name="[value]" ExcludeView="simple">value.second</Item>
-    </Expand>
-  </Type>
-</AutoVisualizer>

all_files.vcxitems

@@ -15,12 +15,8 @@
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="$(MSBuildThisFileDirectory)absl_helper\jedec_size_flag.cc" />
-    <ClCompile Include="$(MSBuildThisFileDirectory)cdc_fuse_fs\mock_config_stream_client.cc" />
-    <ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync\server_arch.cc" />
-    <ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync\server_arch_test.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_stream\asset_stream_config.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_stream\asset_stream_server.cc" />
-    <ClCompile Include="$(MSBuildThisFileDirectory)cdc_stream\background_service_client.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_stream\background_service_impl.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_stream\base_command.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_stream\cdc_fuse_manager.cc" />
@@ -37,12 +33,6 @@
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_stream\session_manager.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_stream\start_command.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_stream\start_service_command.cc" />
-    <ClCompile Include="$(MSBuildThisFileDirectory)cdc_stream\stop_service_command.cc" />
-    <ClCompile Include="$(MSBuildThisFileDirectory)fastcdc\fastcdc_test.cc" />
-    <ClCompile Include="$(MSBuildThisFileDirectory)manifest\pending_assets_queue.cc" />
-    <ClInclude Include="$(MSBuildThisFileDirectory)cdc_fuse_fs\mock_config_stream_client.h" />
-    <ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync\server_arch.h" />
-    <ClInclude Include="$(MSBuildThisFileDirectory)cdc_stream\background_service_client.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)cdc_stream\stop_command.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_stream\testing_asset_stream_server.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_fuse_fs\asset.cc" />
@@ -55,19 +45,13 @@
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_fuse_fs\mock_libfuse.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_indexer\indexer.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_indexer\main.cc" />
-    <ClCompile Include="$(MSBuildThisFileDirectory)common\ansi_filter.cc" />
-    <ClCompile Include="$(MSBuildThisFileDirectory)common\ansi_filter_test.cc" />
-    <ClCompile Include="$(MSBuildThisFileDirectory)common\arch_type.cc" />
-    <ClCompile Include="$(MSBuildThisFileDirectory)common\arch_type_test.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)common\buffer.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)common\buffer_test.cc" />
-    <ClCompile Include="$(MSBuildThisFileDirectory)common\client_socket.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)common\clock.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)common\dir_iter.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)common\dir_iter_test.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)common\errno_mapping.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)common\errno_mapping_test.cc" />
-    <ClCompile Include="$(MSBuildThisFileDirectory)common\fake_socket.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)common\file_watcher_win.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)common\file_watcher_win_test.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)common\gamelet_component.cc" />
@@ -78,6 +62,8 @@
     <ClCompile Include="$(MSBuildThisFileDirectory)common\path_filter.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)common\path_filter_test.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)common\path_test.cc" />
+    <ClCompile Include="$(MSBuildThisFileDirectory)common\port_manager_test.cc" />
+    <ClCompile Include="$(MSBuildThisFileDirectory)common\port_manager_win.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)common\process_test.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)common\process_win.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)common\remote_util.cc" />
@@ -87,8 +73,6 @@
     <ClCompile Include="$(MSBuildThisFileDirectory)common\sdk_util_test.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)common\semaphore.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)common\semaphore_test.cc" />
-    <ClCompile Include="$(MSBuildThisFileDirectory)common\server_socket.cc" />
-    <ClCompile Include="$(MSBuildThisFileDirectory)common\socket.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)common\stats_collector.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)common\status.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)common\stopwatch.cc" />
@@ -114,8 +98,10 @@
     <ClCompile Include="$(MSBuildThisFileDirectory)data_store\mem_data_store_test.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync\base\cdc_interface.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync\base\cdc_interface_test.cc" />
+    <ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync\base\fake_socket.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync\base\message_pump.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync\base\message_pump_test.cc" />
+    <ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync\client_socket.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync\file_finder_and_sender.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync\file_finder_and_sender_test.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync\cdc_rsync_client.cc" />
@@ -136,6 +122,7 @@
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync_server\file_finder_test.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync_server\cdc_rsync_server.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync_server\main.cc" />
+    <ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync_server\server_socket.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync_server\unzstd_stream.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)manifest\asset_builder.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)manifest\content_id.cc" />
@@ -155,12 +142,6 @@
     <ClCompile Include="$(MSBuildThisFileDirectory)metrics\messages.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)metrics\messages_test.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)metrics\metrics.cc" />
-    <ClInclude Include="$(MSBuildThisFileDirectory)cdc_stream\stop_service_command.h" />
-    <ClInclude Include="$(MSBuildThisFileDirectory)common\ansi_filter.h" />
-    <ClInclude Include="$(MSBuildThisFileDirectory)common\arch_type.h" />
-    <ClInclude Include="$(MSBuildThisFileDirectory)common\build_version.h" />
-    <ClInclude Include="$(MSBuildThisFileDirectory)fastcdc\fastcdc.h" />
-    <ClInclude Include="$(MSBuildThisFileDirectory)manifest\pending_assets_queue.h" />
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="$(MSBuildThisFileDirectory)absl_helper\jedec_size_flag.h" />
@@ -190,11 +171,9 @@
     <ClInclude Include="$(MSBuildThisFileDirectory)cdc_fuse_fs\mock_libfuse.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)cdc_indexer\indexer.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)common\buffer.h" />
-    <ClInclude Include="$(MSBuildThisFileDirectory)common\client_socket.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)common\clock.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)common\dir_iter.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)common\errno_mapping.h" />
-    <ClInclude Include="$(MSBuildThisFileDirectory)common\fake_socket.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)common\file_watcher_win.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)common\gamelet_component.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)common\grpc_status.h" />
@@ -202,13 +181,12 @@
     <ClInclude Include="$(MSBuildThisFileDirectory)common\path.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)common\path_filter.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)common\platform.h" />
+    <ClInclude Include="$(MSBuildThisFileDirectory)common\port_manager.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)common\process.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)common\remote_util.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)common\scoped_handle_win.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)common\sdk_util.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)common\semaphore.h" />
-    <ClInclude Include="$(MSBuildThisFileDirectory)common\server_socket.h" />
-    <ClInclude Include="$(MSBuildThisFileDirectory)common\socket.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)common\stats_collector.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)common\status.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)common\status_macros.h" />
@@ -228,9 +206,12 @@
     <ClInclude Include="$(MSBuildThisFileDirectory)data_store\grpc_reader.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)data_store\mem_data_store.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync\base\cdc_interface.h" />
+    <ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync\base\fake_socket.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync\base\message_pump.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync\base\server_exit_code.h" />
+    <ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync\base\socket.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync\client_file_info.h" />
+    <ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync\client_socket.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync\file_finder_and_sender.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync\cdc_rsync_client.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync\parallel_file_opener.h" />
@@ -242,6 +223,7 @@
     <ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync_server\file_finder.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync_server\file_info.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync_server\cdc_rsync_server.h" />
+    <ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync_server\server_socket.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync_server\unzstd_stream.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)manifest\asset_builder.h" />
     <ClInclude Include="$(MSBuildThisFileDirectory)manifest\content_id.h" />
@@ -275,7 +257,6 @@
     <None Include="$(MSBuildThisFileDirectory)cdc_rsync\README.md" />
     <None Include="$(MSBuildThisFileDirectory)cdc_rsync_server\BUILD" />
     <None Include="$(MSBuildThisFileDirectory)cdc_rsync_tests\BUILD" />
-    <None Include="$(MSBuildThisFileDirectory)fastcdc\BUILD" />
     <None Include="$(MSBuildThisFileDirectory)manifest\BUILD" />
     <None Include="$(MSBuildThisFileDirectory)manifest_cli\BUILD" />
     <None Include="$(MSBuildThisFileDirectory)metrics\BUILD" />
@@ -295,7 +276,6 @@
     <None Include="$(MSBuildThisFileDirectory)WORKSPACE" />
   </ItemGroup>
   <ItemGroup>
-    <Natvis Include="$(MSBuildThisFileDirectory)absl.natvis" />
     <Natvis Include="$(MSBuildThisFileDirectory)manifest.natvis" />
     <Natvis Include="$(MSBuildThisFileDirectory)protobuf.natvis" />
   </ItemGroup>

cdc_fuse_fs/BUILD

@@ -9,10 +9,8 @@ cc_binary(
         ":cdc_fuse_fs_lib",
         ":constants",
         "//absl_helper:jedec_size_flag",
-        "//common:client_socket",
         "//common:gamelet_component",
         "//common:log",
-        "//common:server_socket",
         "//data_store:data_provider",
         "//data_store:disk_data_store",
         "//data_store:grpc_reader",

cdc_fuse_fs/constants.h

@@ -19,22 +19,15 @@
 
 namespace cdc_ft {
 
-// FUSE prints
+// FUSE prints this to stdout when the binary timestamp and file size match the
-//   Port 12345 cdc_fuse_fs is up-to-date
+// file on the workstation.
-// to stdout when its version matches the version (=build version or
+static constexpr char kFuseUpToDate[] = "cdc_fuse_fs is up-to-date";
-// size/timestamp for DEV builds) on the local device. The port is the gRPC port
-// that FUSE will try to connect to.
-static constexpr char kFusePortPrefix[] = "Port ";
-static constexpr char kFuseUpToDate[] = " cdc_fuse_fs is up-to-date";
 
-// FUSE prints this to stdout when its version does not match the version on the
+// FUSE prints this to stdout when the binary timestamp or file size does not
-// local device. It indicates that the binary has to be redeployed.
+// match the file on the workstation. It indicates that the binary has to be
+// redeployed.
 static constexpr char kFuseNotUpToDate[] = "cdc_fuse_fs is not up-to-date";
 
-// FUSE prints this to stdout when it can connect to its port. This means that
-// port forwarding has finished setting up, and startup is finished.
-static constexpr char kFuseConnected[] = "cdc_fuse_fs is connected";
-
 }  // namespace cdc_ft
 
 #endif  // CDC_FUSE_FS_CONSTANTS_H_

cdc_fuse_fs/main.cc

@@ -21,11 +21,9 @@
 #include "cdc_fuse_fs/cdc_fuse_fs.h"
 #include "cdc_fuse_fs/config_stream_client.h"
 #include "cdc_fuse_fs/constants.h"
-#include "common/client_socket.h"
 #include "common/gamelet_component.h"
 #include "common/log.h"
 #include "common/path.h"
-#include "common/server_socket.h"
 #include "data_store/data_provider.h"
 #include "data_store/disk_data_store.h"
 #include "data_store/grpc_reader.h"
@@ -39,8 +37,6 @@ namespace {
 constexpr char kFuseFilename[] = "cdc_fuse_fs";
 constexpr char kLibFuseFilename[] = "libfuse.so";
 
-constexpr absl::Duration kConnectionTimeout = absl::Seconds(60);
-
 bool IsUpToDate(const std::string& components_arg) {
   // Components are expected to reside in the same dir as the executable.
   std::string component_dir;
@@ -111,6 +107,7 @@ ABSL_FLAG(
     "Whitespace-separated triples filename, size and timestamp of the "
     "workstation version of this binary and dependencies. Used for a fast "
     "up-to-date check.");
+ABSL_FLAG(uint16_t, port, 0, "Port to connect to on localhost");
 ABSL_FLAG(cdc_ft::JedecSize, prefetch_size, cdc_ft::JedecSize(512 << 10),
           "Additional data to request from the server when a FUSE read of "
           "maximum size is detected. This amount is added to the original "
@@ -141,6 +138,7 @@ int main(int argc, char* argv[]) {
   std::vector<char*> mount_args = absl::ParseCommandLine(argc, argv);
   std::string instance = absl::GetFlag(FLAGS_instance);
   std::string components = absl::GetFlag(FLAGS_components);
+  uint16_t port = absl::GetFlag(FLAGS_port);
   std::string cache_dir = absl::GetFlag(FLAGS_cache_dir);
   int cache_dir_levels = absl::GetFlag(FLAGS_cache_dir_levels);
   int verbosity = absl::GetFlag(FLAGS_verbosity);
@@ -161,18 +159,7 @@ int main(int argc, char* argv[]) {
     printf("%s\n", cdc_ft::kFuseNotUpToDate);
     return 0;
   }
-
+  printf("%s\n", cdc_ft::kFuseUpToDate);
-  // Find an available port.
-  absl::StatusOr<int> port_or = cdc_ft::ServerSocket::FindAvailablePort();
-  if (!port_or.ok()) {
-    LOG_ERROR("Failed to find available port: %s\n",
-              port_or.status().ToString());
-    return 1;
-  }
-  int port = *port_or;
-
-  // Write marker for the server.
-  printf("%s%i%s\n", cdc_ft::kFusePortPrefix, port, cdc_ft::kFuseUpToDate);
   fflush(stdout);
 
   // Create mount dir if it doesn't exist yet.
@@ -202,18 +189,6 @@ int main(int argc, char* argv[]) {
   store.value()->SetCapacity(cache_capacity);
   LOG_INFO("Caching chunks in '%s'", store.value()->RootDir());
 
-  // Wait for port forwarding to listen to |port|.
-  status =
-      cdc_ft::ClientSocket::WaitForConnection(port, cdc_ft::kConnectionTimeout);
-  if (!status.ok()) {
-    LOG_ERROR("Failed to connect to port %i: %s", port, status.ToString());
-    return static_cast<int>(status.code());
-  }
-
-  // Write another marker for the server.
-  printf("%s\n", cdc_ft::kFuseConnected);
-  fflush(stdout);
-
   // Start a gRpc client.
   std::string client_address = absl::StrFormat("localhost:%u", port);
   grpc::ChannelArguments channel_args;

cdc_indexer/README.md

@@ -14,7 +14,7 @@ experimentation. See the file `indexer.h` for preprocessor macros that can be
 enabled, for example:
 
 ```
-bazel build -c opt --copt=-DCDC_GEAR_BITS=32 //cdc_indexer
+bazel build -c opt --copt=-DCDC_GEAR_TABLE=1 //cdc_indexer
 ```
 
 At the end of the operation, the indexer outputs a summary of the results such
@@ -25,7 +25,7 @@ as the following:
 Operation succeeded.
 
 Chunk size (min/avg/max): 128 KB / 256 KB / 1024 KB  |  Threads: 12
-gear_table: 64 bit  |  threshold: 0x7fffc0001fff
+gear_table: 64 bit  |  mask_s: 0x49249249249249  |  mask_l: 0x1249249249
            Duration:           00:03
         Total files:               2
        Total chunks:           39203

cdc_indexer/indexer.cc

@@ -140,7 +140,8 @@ Indexer::Impl::Impl(const IndexerConfig& cfg,
   fastcdc::Config ccfg(cfg_.min_chunk_size, cfg_.avg_chunk_size,
                        cfg_.max_chunk_size);
   Indexer::Chunker chunker(ccfg, nullptr);
-  cfg_.threshold = chunker.Threshold();
+  cfg_.mask_s = chunker.Stage(0).mask;
+  cfg_.mask_l = chunker.Stage(chunker.StagesCount() - 1).mask;
   // Collect inputs.
   for (auto it = inputs.begin(); it != inputs.end(); ++it) {
     inputs_.push(*it);
@@ -367,7 +368,8 @@ IndexerConfig::IndexerConfig()
       max_chunk_size(0),
       max_chunk_size_step(0),
       num_threads(0),
-      threshold(0) {}
+      mask_s(0),
+      mask_l(0) {}
 
 Indexer::Indexer() : impl_(nullptr) {}
 

cdc_indexer/indexer.h

@@ -27,10 +27,16 @@
 #include "fastcdc/fastcdc.h"
 
 // Compile-time parameters for the FastCDC algorithm.
-#define CDC_GEAR_32BIT 32
+#define CDC_GEAR_32BIT 1
-#define CDC_GEAR_64BIT 64
+#define CDC_GEAR_64BIT 2
-#ifndef CDC_GEAR_BITS
+#ifndef CDC_GEAR_TABLE
-#define CDC_GEAR_BITS CDC_GEAR_64BIT
+#define CDC_GEAR_TABLE CDC_GEAR_64BIT
+#endif
+#ifndef CDC_MASK_STAGES
+#define CDC_MASK_STAGES 7
+#endif
+#ifndef CDC_MASK_BIT_LSHIFT_AMOUNT
+#define CDC_MASK_BIT_LSHIFT_AMOUNT 3
 #endif
 
 namespace cdc_ft {
@@ -60,20 +66,23 @@ struct IndexerConfig {
   uint32_t num_threads;
   // Which hash function to use.
   HashType hash_type;
-  // The threshold will be populated by the indexer, setting it here has no
+  // The masks will be populated by the indexer, setting them here has no
-  // effect. It is in this struct so that it can be conveniently accessed
+  // effect. They are in this struct so that they can be conveniently accessed
-  // when printing the operation summary (and since it is derived from the
+  // when printing the operation summary (and since they are derived from the
-  // configuration, it is technically part of it).
+  // configuration, they are technically part of it).
-  uint64_t threshold;
+  uint64_t mask_s;
+  uint64_t mask_l;
 };
 
 class Indexer {
  public:
   using hash_t = std::string;
-#if CDC_GEAR_BITS == CDC_GEAR_32BIT
+#if CDC_GEAR_TABLE == CDC_GEAR_32BIT
-  typedef fastcdc::Chunker32<> Chunker;
+  typedef fastcdc::Chunker32<CDC_MASK_STAGES, CDC_MASK_BIT_LSHIFT_AMOUNT>
-#elif CDC_GEAR_BITS == CDC_GEAR_64BIT
+      Chunker;
-  typedef fastcdc::Chunker64<> Chunker;
+#elif CDC_GEAR_TABLE == CDC_GEAR_64BIT
+  typedef fastcdc::Chunker64<CDC_MASK_STAGES, CDC_MASK_BIT_LSHIFT_AMOUNT>
+      Chunker;
 #else
 #error "Unknown gear table"
 #endif

cdc_indexer/main.cc

@@ -64,9 +64,9 @@ namespace {
 
 const char* GearTable() {
   // The following macros are defined in indexer.h.
-#if CDC_GEAR_BITS == CDC_GEAR_32BIT
+#if CDC_GEAR_TABLE == CDC_GEAR_32BIT
   return "32 bit";
-#elif CDC_GEAR_BITS == CDC_GEAR_64BIT
+#elif CDC_GEAR_TABLE == CDC_GEAR_64BIT
   return "64 bit";
 #else
 #error "Unknown gear table"
@@ -165,8 +165,9 @@ void ShowSummary(const IndexerConfig& cfg, const Indexer::OpStats& stats,
             << HumanBytes(cfg.max_chunk_size)
             << "  |  Hash: " << HashTypeToString(cfg.hash_type)
             << "  |  Threads: " << cfg.num_threads << std::endl;
-  std::cout << "gear_table: " << GearTable() << "  |  threshold: 0x" << std::hex
+  std::cout << "gear_table: " << GearTable() << "  |  mask_s: 0x" << std::hex
-            << cfg.threshold << std::dec << std::endl;
+            << cfg.mask_s << "  |  mask_l: 0x" << cfg.mask_l << std::dec
+            << std::endl;
   std::cout << std::setw(title_w) << "Duration:" << std::setw(num_w)
             << HumanDuration(elapsed) << std::endl;
   std::cout << std::setw(title_w) << "Total files:" << std::setw(num_w)
@@ -278,10 +279,11 @@ absl::Status WriteResultsFile(const std::string& filepath,
 
   path::FileCloser closer(fout);
 
-  static constexpr int num_columns = 14;
+  static constexpr int num_columns = 15;
   static const char* columns[num_columns] = {
       "gear_table",
-      "threshold",
+      "mask_s",
+      "mask_l",
       "Min chunk size [KiB]",
       "Avg chunk size [KiB]",
       "Max chunk size [KiB]",
@@ -330,7 +332,7 @@ absl::Status WriteResultsFile(const std::string& filepath,
   // Write user-supplied description
   if (!description.empty()) std::fprintf(fout, "%s,", description.c_str());
   // Write chunking params.
-  std::fprintf(fout, "%s,0x%zx,", GearTable(), cfg.threshold);
+  std::fprintf(fout, "%s,0x%zx,0x%zx,", GearTable(), cfg.mask_s, cfg.mask_l);
   std::fprintf(fout, "%zu,%zu,%zu,", cfg.min_chunk_size >> 10,
                cfg.avg_chunk_size >> 10, cfg.max_chunk_size >> 10);
   // Write speed, files, chunks.

cdc_rsync/BUILD

@@ -18,6 +18,19 @@ cc_library(
     hdrs = ["client_file_info.h"],
 )
 
+cc_library(
+    name = "client_socket",
+    srcs = ["client_socket.cc"],
+    hdrs = ["client_socket.h"],
+    target_compatible_with = ["@platforms//os:windows"],
+    deps = [
+        "//cdc_rsync/base:socket",
+        "//common:log",
+        "//common:status",
+        "//common:util",
+    ],
+)
+
 cc_library(
     name = "file_finder_and_sender",
     srcs = ["file_finder_and_sender.cc"],
@@ -41,8 +54,8 @@ cc_test(
     data = ["testdata/root.txt"] + glob(["testdata/file_finder_and_sender/**"]),
     deps = [
         ":file_finder_and_sender",
+        "//cdc_rsync/base:fake_socket",
         "//cdc_rsync/protos:messages_cc_proto",
-        "//common:fake_socket",
         "//common:status_test_macros",
         "//common:test_main",
         "@com_google_googletest//:gtest",
@@ -54,27 +67,32 @@ cc_library(
     name = "cdc_rsync_client",
     srcs = ["cdc_rsync_client.cc"],
     hdrs = ["cdc_rsync_client.h"],
+    linkopts = select({
+        "//tools:windows": [
+            "/DEFAULTLIB:Ws2_32.lib",  # Sockets, e.g. recv, send, WSA*.
+        ],
+        "//conditions:default": [],
+    }),
     target_compatible_with = ["@platforms//os:windows"],
     deps = [
+        ":client_socket",
         ":file_finder_and_sender",
         ":parallel_file_opener",
         ":progress_tracker",
-        ":server_arch",
         ":zstd_stream",
         "//cdc_rsync/base:cdc_interface",
         "//cdc_rsync/base:message_pump",
         "//cdc_rsync/base:server_exit_code",
+        "//cdc_rsync/base:socket",
         "//cdc_rsync/protos:messages_cc_proto",
-        "//common:client_socket",
         "//common:gamelet_component",
         "//common:log",
         "//common:path",
         "//common:path_filter",
         "//common:platform",
+        "//common:port_manager",
         "//common:process",
         "//common:remote_util",
-        "//common:server_socket",
-        "//common:socket",
         "//common:status",
         "//common:status_macros",
         "//common:threadpool",
@@ -112,7 +130,6 @@ cc_library(
     hdrs = ["params.h"],
     deps = [
         ":cdc_rsync_client",
-        "//common:build_version",
         "@com_github_zstd//:zstd",
         "@com_google_absl//absl/status",
     ],
@@ -154,37 +171,13 @@ cc_test(
     ],
 )
 
-cc_library(
-    name = "server_arch",
-    srcs = ["server_arch.cc"],
-    hdrs = ["server_arch.h"],
-    deps = [
-        "//common:ansi_filter",
-        "//common:arch_type",
-        "//common:path",
-        "//common:remote_util",
-        "@com_google_absl//absl/strings",
-    ],
-)
-
-cc_test(
-    name = "server_arch_test",
-    srcs = ["server_arch_test.cc"],
-    deps = [
-        ":server_arch",
-        "//common:test_main",
-        "@com_google_absl//absl/strings",
-        "@com_google_googletest//:gtest",
-    ],
-)
-
 cc_library(
     name = "zstd_stream",
     srcs = ["zstd_stream.cc"],
     hdrs = ["zstd_stream.h"],
     deps = [
+        ":client_socket",
         "//common:buffer",
-        "//common:client_socket",
         "//common:status",
         "//common:status_macros",
         "//common:stopwatch",
@@ -197,8 +190,8 @@ cc_test(
     srcs = ["zstd_stream_test.cc"],
     deps = [
         ":zstd_stream",
+        "//cdc_rsync/base:fake_socket",
         "//cdc_rsync_server:unzstd_stream",
-        "//common:fake_socket",
         "//common:status_test_macros",
         "//common:test_main",
         "@com_github_zstd//:zstd",

cdc_rsync/base/BUILD

@@ -28,21 +28,31 @@ cc_test(
     data = ["testdata/root.txt"] + glob(["testdata/cdc_interface/**"]),
     deps = [
         ":cdc_interface",
-        "//common:fake_socket",
+        ":fake_socket",
         "//common:status_test_macros",
         "//common:test_main",
         "@com_google_googletest//:gtest",
     ],
 )
 
+cc_library(
+    name = "fake_socket",
+    srcs = ["fake_socket.cc"],
+    hdrs = ["fake_socket.h"],
+    deps = [
+        "//cdc_rsync/base:socket",
+        "@com_google_absl//absl/status",
+    ],
+)
+
 cc_library(
     name = "message_pump",
     srcs = ["message_pump.cc"],
     hdrs = ["message_pump.h"],
     deps = [
+        ":socket",
         "//common:buffer",
         "//common:log",
-        "//common:socket",
         "//common:status",
         "@com_google_absl//absl/status",
         "@com_google_absl//absl/strings:str_format",
@@ -54,9 +64,9 @@ cc_test(
     name = "message_pump_test",
     srcs = ["message_pump_test.cc"],
     deps = [
+        ":fake_socket",
         ":message_pump",
         "//cdc_rsync/protos:messages_cc_proto",
-        "//common:fake_socket",
         "//common:status_test_macros",
         "//common:test_main",
         "@com_google_googletest//:gtest",
@@ -68,6 +78,11 @@ cc_library(
     hdrs = ["server_exit_code.h"],
 )
 
+cc_library(
+    name = "socket",
+    hdrs = ["socket.h"],
+)
+
 filegroup(
     name = "all_test_sources",
     srcs = glob(["*_test.cc"]),

cdc_rsync/base/cdc_interface_test.cc

@@ -17,8 +17,8 @@
 #include <cstdio>
 #include <fstream>
 
+#include "cdc_rsync/base/fake_socket.h"
 #include "cdc_rsync/base/message_pump.h"
-#include "common/fake_socket.h"
 #include "common/log.h"
 #include "common/path.h"
 #include "common/status_test_macros.h"

cdc_rsync/base/fake_socket.cc

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "common/fake_socket.h"
+#include "cdc_rsync/base/fake_socket.h"
 
 namespace cdc_ft {
 
@@ -39,8 +39,7 @@ absl::Status FakeSocket::Receive(void* buffer, size_t size,
   *bytes_received = 0;
   std::unique_lock<std::mutex> lock(data_mutex_);
   data_cv_.wait(lock, [this, size, allow_partial_read]() {
-    size_t min_size = allow_partial_read ? 1 : size;
+    return allow_partial_read || data_.size() >= size || shutdown_;
-    return data_.size() >= min_size || shutdown_;
   });
   if (shutdown_) {
     return absl::UnavailableError("Pipe is shut down");

cdc_rsync/base/fake_socket.h

@@ -14,14 +14,14 @@
  * limitations under the License.
  */
 
-#ifndef COMMON_FAKE_SOCKET_H_
+#ifndef CDC_RSYNC_BASE_FAKE_SOCKET_H_
-#define COMMON_FAKE_SOCKET_H_
+#define CDC_RSYNC_BASE_FAKE_SOCKET_H_
 
 #include <condition_variable>
 #include <mutex>
 
 #include "absl/status/status.h"
-#include "common/socket.h"
+#include "cdc_rsync/base/socket.h"
 
 namespace cdc_ft {
 

cdc_rsync/base/message_pump.cc

@@ -16,9 +16,9 @@
 
 #include "absl/status/status.h"
 #include "absl/strings/str_format.h"
+#include "cdc_rsync/base/socket.h"
 #include "common/buffer.h"
 #include "common/log.h"
-#include "common/socket.h"
 #include "common/status.h"
 #include "google/protobuf/message_lite.h"
 

cdc_rsync/base/message_pump_test.cc

@@ -14,8 +14,8 @@
 
 #include "cdc_rsync/base/message_pump.h"
 
+#include "cdc_rsync/base/fake_socket.h"
 #include "cdc_rsync/protos/messages.pb.h"
-#include "common/fake_socket.h"
 #include "common/log.h"
 #include "common/status.h"
 #include "common/status_test_macros.h"

cdc_rsync/base/socket.h

@@ -14,8 +14,8 @@
  * limitations under the License.
  */
 
-#ifndef COMMON_SOCKET_H_
+#ifndef CDC_RSYNC_BASE_SOCKET_H_
-#define COMMON_SOCKET_H_
+#define CDC_RSYNC_BASE_SOCKET_H_
 
 #include "absl/status/status.h"
 
@@ -26,14 +26,6 @@ class Socket {
   Socket() = default;
   virtual ~Socket() = default;
 
-  // Calls WSAStartup() on Windows, no-op on Linux.
-  // Must be called before using sockets.
-  static absl::Status Initialize();
-
-  // Calls WSACleanup() on Windows, no-op on Linux.
-  // Must be called after using sockets.
-  static absl::Status Shutdown();
-
   // Send data to the socket.
   virtual absl::Status Send(const void* buffer, size_t size) = 0;
 
@@ -48,12 +40,6 @@ class Socket {
                                size_t* bytes_received) = 0;
 };
 
-// Convenience class that calls Shutdown() on destruction. Logs on errors.
-class SocketFinalizer {
- public:
-  ~SocketFinalizer();
-};
-
 }  // namespace cdc_ft
 
-#endif  // COMMON_SOCKET_H_
+#endif  // CDC_RSYNC_BASE_SOCKET_H_

cdc_rsync/cdc_rsync.vcxproj

@@ -66,21 +66,20 @@
   </ItemDefinitionGroup>
   <!-- Bazel setup -->
   <PropertyGroup>
-    <BazelTargets>//cdc_rsync //cdc_rsync_server</BazelTargets>
+    <BazelTargets>//cdc_rsync</BazelTargets>
     <BazelOutputFile>cdc_rsync.exe</BazelOutputFile>
-    <BazelIncludePaths>..\;..\third_party\absl;..\bazel-cdc-file-transfer\external\com_github_blake3\c;..\bazel-cdc-file-transfer\external\com_github_zstd;..\third_party\googletest\googletest\include;..\bazel-cdc-file-transfer\external\com_google_protobuf\src;..\bazel-cdc-file-transfer\external\com_github_grpc_grpc\include;..\bazel-bin;$(VC_IncludePath);$(WindowsSDK_IncludePath)</BazelIncludePaths>
+    <BazelIncludePaths>..\;..\third_party\absl;..\bazel-cdc-file-transfer\external\com_github_blake3\c;..\bazel-stadia-file-transfer\external\com_github_zstd\lib;..\third_party\googletest\googletest\include;..\bazel-cdc-file-transfer\external\com_google_protobuf\src;$(VC_IncludePath);$(WindowsSDK_IncludePath)</BazelIncludePaths>
   </PropertyGroup>
   <Import Project="..\NMakeBazelProject.targets" />
   <!-- For some reason, msbuild doesn't include this file, so copy it explicitly. -->
-  <!-- TODO: Reenable copying the Linux file once we can cross-compile these. -->
+  <!-- TODO: Reenable once we can cross-compile these.
   <PropertyGroup>
-    <!-- <CdcRsyncLinuxServerFile>$(SolutionDir)bazel-out\k8-$(BazelCompilationMode)\bin\cdc_rsync_server\cdc_rsync_server</CdcRsyncLinuxServerFile> -->
+    <CdcRsyncServerFile>$(SolutionDir)bazel-out\k8-$(BazelCompilationMode)\bin\cdc_rsync_server\cdc_rsync_server</CdcRsyncServerFile>
-    <CdcRsyncWindowsServerFile>$(SolutionDir)bazel-out\x64_windows-$(BazelCompilationMode)\bin\cdc_rsync_server\cdc_rsync_server.exe</CdcRsyncWindowsServerFile>
   </PropertyGroup>
-  <Target Name="CopyServer" Inputs="$(CdcRsyncLinuxServerFile);$(CdcRsyncWindowsServerFile)" Outputs="$(OutDir)cdc_rsync_server;$(OutDir)cdc_rsync_server.exe" AfterTargets="Build">
+  <Target Name="CopyServer" Inputs="$(CdcRsyncServerFile)" Outputs="$(OutDir)cdc_rsync_server" AfterTargets="Build">
-    <!-- <Copy SourceFiles="$(CdcRsyncLinuxServerFile)" DestinationFiles="$(OutDir)cdc_rsync_server" SkipUnchangedFiles="true" /> -->
+    <Copy SourceFiles="$(CdcRsyncServerFile)" DestinationFiles="$(OutDir)cdc_rsync_server" />
-    <Copy SourceFiles="$(CdcRsyncWindowsServerFile)" DestinationFiles="$(OutDir)cdc_rsync_server.exe" SkipUnchangedFiles="true" />
   </Target>
+  -->
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>

cdc_rsync/cdc_rsync_client.cc

@@ -20,19 +20,16 @@
 #include "cdc_rsync/base/message_pump.h"
 #include "cdc_rsync/base/server_exit_code.h"
 #include "cdc_rsync/client_file_info.h"
+#include "cdc_rsync/client_socket.h"
 #include "cdc_rsync/file_finder_and_sender.h"
 #include "cdc_rsync/parallel_file_opener.h"
 #include "cdc_rsync/progress_tracker.h"
 #include "cdc_rsync/protos/messages.pb.h"
-#include "cdc_rsync/server_arch.h"
 #include "cdc_rsync/zstd_stream.h"
-#include "common/client_socket.h"
 #include "common/gamelet_component.h"
 #include "common/log.h"
 #include "common/path.h"
 #include "common/process.h"
-#include "common/remote_util.h"
-#include "common/server_socket.h"
 #include "common/status.h"
 #include "common/status_macros.h"
 #include "common/stopwatch.h"
@@ -47,6 +44,11 @@ constexpr int kExitCodeCouldNotExecute = 126;
 // Bash exit code if binary was not found.
 constexpr int kExitCodeNotFound = 127;
 
+constexpr int kForwardPortFirst = 44450;
+constexpr int kForwardPortLast = 44459;
+constexpr char kCdcServerFilename[] = "cdc_rsync_server";
+constexpr char kRemoteToolsBinDir[] = "~/.cache/cdc-file-transfer/bin/";
+
 SetOptionsRequest::FilterRule::Type ToProtoType(PathFilter::Rule::Type type) {
   switch (type) {
     case PathFilter::Rule::Type::kInclude:
@@ -97,21 +99,20 @@ CdcRsyncClient::CdcRsyncClient(const Options& options,
                                std::string user_host, std::string destination)
     : options_(options),
       sources_(std::move(sources)),
+      user_host_(std::move(user_host)),
       destination_(std::move(destination)),
+      remote_util_(options.verbosity, options.quiet, &process_factory_,
+                   /*forward_output_to_log=*/false),
+      port_manager_("cdc_rsync_ports_f77bcdfe-368c-4c45-9f01-230c5e7e2132",
+                    kForwardPortFirst, kForwardPortLast, &process_factory_,
+                    &remote_util_),
       printer_(options.quiet, Util::IsTTY() && !options.json),
       progress_(&printer_, options.verbosity, options.json) {
-  // If there is no |user_host|, we sync files locally!
-  if (!user_host.empty()) {
-    remote_util_ =
-        std::make_unique<RemoteUtil>(std::move(user_host), options.verbosity,
-                                     options.quiet, &process_factory_,
-                                     /*forward_output_to_log=*/false);
   if (!options_.ssh_command.empty()) {
-      remote_util_->SetSshCommand(options_.ssh_command);
+    remote_util_.SetSshCommand(options_.ssh_command);
-    }
-    if (!options_.sftp_command.empty()) {
-      remote_util_->SetSftpCommand(options_.sftp_command);
   }
+  if (!options_.scp_command.empty()) {
+    remote_util_.SetScpCommand(options_.scp_command);
   }
 }
 
@@ -121,42 +122,19 @@ CdcRsyncClient::~CdcRsyncClient() {
 }
 
 absl::Status CdcRsyncClient::Run() {
-  // For local syncs, cdc_rsync_server runs on this machine. For remote syncs,
+  // Initialize |remote_util_|.
-  // guess the architecture of the device that runs cdc_rsync_server from the
+  remote_util_.SetUserHostAndPort(user_host_, options_.port);
-  // destination path, e.g. "C:\path\to\dest" strongly indicates Windows.
-  ServerArch server_arch = IsRemoteConnection()
-                               ? ServerArch::GuessFromDestination(destination_)
-                               : ServerArch::DetectFromLocalDevice();
 
   // Start the server process.
-  absl::Status status = StartServer(server_arch);
+  absl::Status status = StartServer();
-  if (HasTag(status, Tag::kDeployServer) && server_arch.IsGuess() &&
-      server_exit_code_ != kServerExitCodeOutOfDate) {
-    // Server couldn't be run, e.g. not found or failed to start.
-    // Check whether we guessed the arch type wrong and try again.
-    // Note that in case of a local sync, or if the server actively reported
-    // that it's out-dated, there's no need to detect the arch.
-    LOG_DEBUG(
-        "Failed to start server, retrying after detecting remote arch: %s",
-        status.ToString());
-    const ArchType old_type = server_arch.GetType();
-    ASSIGN_OR_RETURN(server_arch,
-                     ServerArch::DetectFromRemoteDevice(remote_util_.get()));
-    if (server_arch.GetType() != old_type) {
-      LOG_DEBUG("Guessed server arch type wrong, guessed %s, actual %s.",
-                GetArchTypeStr(old_type), server_arch.GetTypeStr());
-      status = StartServer(server_arch);
-    }
-  }
-
   if (HasTag(status, Tag::kDeployServer)) {
     // Gamelet components are not deployed or out-dated. Deploy and retry.
-    status = DeployServer(server_arch);
+    status = DeployServer();
     if (!status.ok()) {
       return WrapStatus(status, "Failed to deploy server");
     }
 
-    status = StartServer(server_arch);
+    status = StartServer();
   }
   if (!status.ok()) {
     return WrapStatus(status, "Failed to start server");
@@ -188,7 +166,7 @@ absl::Status CdcRsyncClient::Run() {
   return status;
 }
 
-absl::Status CdcRsyncClient::StartServer(const ServerArch& arch) {
+absl::Status CdcRsyncClient::StartServer() {
   assert(!server_process_);
 
   // Components are expected to reside in the same dir as the executable.
@@ -200,41 +178,50 @@ absl::Status CdcRsyncClient::StartServer(const ServerArch& arch) {
 
   std::vector<GameletComponent> components;
   status = GameletComponent::Get(
-      {path::Join(component_dir, arch.CdcServerFilename())}, &components);
+      {path::Join(component_dir, kCdcServerFilename)}, &components);
   if (!status.ok()) {
     return MakeStatus(
         "Required instance component not found. Make sure the file "
-        "%s resides in the same folder as %s.",
+        "cdc_rsync_server resides in the same folder as cdc_rsync.exe.");
-        arch.CdcServerFilename(), ServerArch::CdcRsyncFilename());
   }
   std::string component_args = GameletComponent::ToCommandLineArgs(components);
 
-  ProcessStartInfo start_info;
+  // Find available local and remote ports for port forwarding.
-  start_info.name = "cdc_rsync_server";
+  absl::StatusOr<int> port_res = port_manager_.ReservePort(
-
+      /*check_remote=*/false, /*remote_timeout_sec unused*/ 0);
-  if (IsRemoteConnection()) {
+  constexpr char kErrorMsg[] = "Failed to find available port";
-    // Run cdc_rsync_server on the remote instance.
+  if (absl::IsDeadlineExceeded(port_res.status())) {
-    std::string remote_command =
+    // Server didn't respond in time.
-        arch.GetStartServerCommand(kExitCodeNotFound, component_args);
+    return SetTag(WrapStatus(port_res.status(), kErrorMsg),
-    assert(remote_util_);
+                  Tag::kConnectionTimeout);
-    start_info = remote_util_->BuildProcessStartInfoForSsh(remote_command,
-                                                           arch.GetType());
-  } else {
-    // Run cdc_rsync_server locally.
-    std::string exe_dir;
-    RETURN_IF_ERROR(path::GetExeDir(&exe_dir), "Failed to get exe directory");
-
-    std::string server_path = path::Join(exe_dir, arch.CdcServerFilename());
-    start_info.command = absl::StrFormat("%s %s", server_path, component_args);
   }
+  if (absl::IsResourceExhausted(port_res.status()))
+    return SetTag(WrapStatus(port_res.status(), kErrorMsg), Tag::kAddressInUse);
+  if (!port_res.ok())
+    return WrapStatus(port_res.status(), "Failed to find available port");
+  int port = *port_res;
+
+  std::string remote_server_path =
+      std::string(kRemoteToolsBinDir) + kCdcServerFilename;
+  // Test existence manually to prevent misleading bash output message
+  // "bash: .../cdc_rsync_server: No such file or directory".
+  // Also create the bin dir because otherwise scp below might fail.
+  std::string remote_command =
+      absl::StrFormat("mkdir -p %s; if [ ! -f %s ]; then exit %i; fi; %s %i %s",
+                      kRemoteToolsBinDir, remote_server_path, kExitCodeNotFound,
+                      remote_server_path, port, component_args);
+  ProcessStartInfo start_info =
+      remote_util_.BuildProcessStartInfoForSshPortForwardAndCommand(
+          port, port, false, remote_command);
+  start_info.name = "cdc_rsync_server";
 
   // Capture stdout, but forward to stdout for debugging purposes.
   start_info.stdout_handler = [this](const char* data, size_t /*data_size*/) {
     return HandleServerOutput(data);
   };
 
-  std::unique_ptr<Process> srv_process = process_factory_.Create(start_info);
+  std::unique_ptr<Process> process = process_factory_.Create(start_info);
-  status = srv_process->Start();
+  status = process->Start();
   if (!status.ok()) {
     return WrapStatus(status, "Failed to start cdc_rsync_server process");
   }
@@ -242,17 +229,17 @@ absl::Status CdcRsyncClient::StartServer(const ServerArch& arch) {
   // Wait until the server process is listening.
   Stopwatch timeout_timer;
   bool is_timeout = false;
-  auto detect_listening_or_timeout = [port = &server_listen_port_,
+  auto detect_listening_or_timeout = [is_listening = &is_server_listening_,
                                       timeout = options_.connection_timeout_sec,
                                       &timeout_timer, &is_timeout]() -> bool {
     is_timeout = timeout_timer.ElapsedSeconds() > timeout;
-    return *port != 0 || is_timeout;
+    return *is_listening || is_timeout;
   };
-  status = srv_process->RunUntil(detect_listening_or_timeout);
+  status = process->RunUntil(detect_listening_or_timeout);
   if (!status.ok()) {
     // Some internal process error. Note that this does NOT mean that
     // cdc_rsync_server does not exist. In that case, the ssh process exits with
-    // code kExitCodeNotFound.
+    // code 127.
     return status;
   }
   if (is_timeout) {
@@ -260,21 +247,15 @@ absl::Status CdcRsyncClient::StartServer(const ServerArch& arch) {
                   Tag::kConnectionTimeout);
   }
 
-  if (srv_process->HasExited()) {
+  if (process->HasExited()) {
     // Don't re-deploy for code > kServerExitCodeOutOfDate, which means that the
     // out-of-date check already passed on the server.
-    server_exit_code_ = srv_process->ExitCode();
+    server_exit_code_ = process->ExitCode();
     if (server_exit_code_ > kServerExitCodeOutOfDate &&
         server_exit_code_ <= kServerExitCodeMax) {
       return GetServerExitStatus(server_exit_code_, server_error_);
     }
 
-    // Don't re-deploy if we're not copying to a remote device. We can start
-    // cdc_rsync_server from the original location directly.
-    if (!IsRemoteConnection()) {
-      return GetServerExitStatus(server_exit_code_, server_error_);
-    }
-
     // Server exited before it started listening, most likely because of
     // outdated components (code kServerExitCodeOutOfDate) or because the server
     // wasn't deployed at all yet (code kExitCodeNotFound). Instruct caller
@@ -282,30 +263,13 @@ absl::Status CdcRsyncClient::StartServer(const ServerArch& arch) {
     return SetTag(MakeStatus("Redeploy server"), Tag::kDeployServer);
   }
 
-  // Start up sockets.
+  assert(is_server_listening_);
-  RETURN_IF_ERROR(Socket::Initialize(), "Failed to initialize sockets");
+  status = socket_.Connect(port);
-  socket_finalizer_ = std::make_unique<SocketFinalizer>();
+  if (!status.ok()) {
-
+    return WrapStatus(status, "Failed to initialize connection");
-  // Now that we know which port the server is using, set up port forwarding.
-  std::unique_ptr<Process> fwd_process;
-  int local_port = server_listen_port_;
-  if (IsRemoteConnection()) {
-    ASSIGN_OR_RETURN(local_port, ServerSocket::FindAvailablePort());
-    ProcessStartInfo start_info =
-        remote_util_->BuildProcessStartInfoForSshPortForward(
-            local_port, server_listen_port_, /*reverse=*/false);
-    start_info.forward_output_to_log = true;
-    fwd_process = process_factory_.Create(start_info);
-    RETURN_IF_ERROR(fwd_process->Start(),
-                    "Failed to start cdc_rsync_server process");
   }
 
-  // Wait for port forwarding to be up.
+  server_process_ = std::move(process);
-  RETURN_IF_ERROR(ClientSocket::WaitForConnection(
-      local_port, absl::Seconds(options_.connection_timeout_sec)));
-
-  server_process_ = std::move(srv_process);
-  port_forwarding_process_ = std::move(fwd_process);
   message_pump_.StartMessagePump();
   return absl::OkStatus();
 }
@@ -326,7 +290,6 @@ absl::Status CdcRsyncClient::StopServer() {
 
   server_exit_code_ = server_process_->ExitCode();
   server_process_.reset();
-  port_forwarding_process_.reset();
   return absl::OkStatus();
 }
 
@@ -358,29 +321,10 @@ absl::Status CdcRsyncClient::HandleServerOutput(const char* data) {
   }
 
   printer_.Print(stdout_data, false, Util::GetConsoleWidth());
-  if (server_listen_port_ == 0) {
+  if (!is_server_listening_) {
     server_output_.append(stdout_data);
-
+    is_server_listening_ =
-    // Parse port from "Port <n>: Server is listening".
+        server_output_.find("Server is listening") != std::string::npos;
-    size_t listening_pos = server_output_.find("Server is listening");
-    if (listening_pos != std::string::npos) {
-      // Search backwards until we find "Port ".
-      constexpr char port_key[] = "Port ";
-      size_t port_pos = server_output_.rfind(port_key, listening_pos);
-      if (port_pos == std::string::npos) {
-        return MakeStatus("Failed to find 'Port' marker in server output '%s'",
-                          server_output_);
-      }
-      assert(listening_pos > port_pos);
-      server_listen_port_ = atoi(
-          server_output_
-              .substr(port_pos + strlen(port_key), listening_pos - port_pos)
-              .c_str());
-      if (server_listen_port_ == 0) {
-        return MakeStatus("Failed to parse port from server output '%s'",
-                          server_output_);
-      }
-    }
   }
 
   return absl::OkStatus();
@@ -444,10 +388,8 @@ absl::Status CdcRsyncClient::Sync() {
   return status;
 }
 
-absl::Status CdcRsyncClient::DeployServer(const ServerArch& arch) {
+absl::Status CdcRsyncClient::DeployServer() {
   assert(!server_process_);
-  assert(remote_util_);
-  assert(IsRemoteConnection());
 
   std::string exe_dir;
   absl::Status status = path::GetExeDir(&exe_dir);
@@ -467,10 +409,32 @@ absl::Status CdcRsyncClient::DeployServer(const ServerArch& arch) {
   }
   printer_.Print(deploy_msg, true, Util::GetConsoleWidth());
 
-  // sftp cdc_rsync_server to the target.
+  // scp cdc_rsync_server to a temp location on the gamelet.
-  std::string commands = arch.GetDeploySftpCommands();
+  std::string remoteServerTmpPath =
-  RETURN_IF_ERROR(remote_util_->Sftp(commands, exe_dir, /*compress=*/false),
+      absl::StrFormat("%s%s.%s", kRemoteToolsBinDir, kCdcServerFilename,
-                  "Failed to deploy cdc_rsync_server");
+                      Util::GenerateUniqueId());
+  std::string localServerPath = path::Join(exe_dir, kCdcServerFilename);
+  status = remote_util_.Scp({localServerPath}, remoteServerTmpPath,
+                            /*compress=*/true);
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to copy cdc_rsync_server to instance");
+  }
+
+  // Do 3 things in one SSH command, to save time:
+  // - Make the old cdc_rsync_server writable (if it exists).
+  // - Make the new cdc_rsync_server executable.
+  // - Replace the old cdc_rsync_server by the new one.
+  std::string old_path = RemoteUtil::QuoteForWindows(
+      std::string(kRemoteToolsBinDir) + kCdcServerFilename);
+  std::string new_path = RemoteUtil::QuoteForWindows(remoteServerTmpPath);
+  std::string replace_cmd = absl::StrFormat(
+      " ([ ! -f %s ] || chmod u+w %s) && chmod a+x %s && mv %s %s", old_path,
+      old_path, new_path, new_path, old_path);
+  status = remote_util_.Run(replace_cmd, "chmod && chmod && mv");
+  if (!status.ok()) {
+    return WrapStatus(status,
+                      "Failed to replace old cdc_rsync_server by new one");
+  }
 
   return absl::OkStatus();
 }
@@ -647,7 +611,7 @@ absl::Status CdcRsyncClient::SendMissingFiles() {
 
   ParallelFileOpener file_opener(&files_, missing_file_indices_);
 
-  constexpr size_t kBufferSize = 128 * 1024;
+  constexpr size_t kBufferSize = 16000;
   for (uint32_t server_index = 0; server_index < missing_file_indices_.size();
        ++server_index) {
     uint32_t client_index = missing_file_indices_[server_index];
@@ -809,9 +773,9 @@ absl::Status CdcRsyncClient::StopCompressionStream() {
   message_pump_.FlushOutgoingQueue();
   message_pump_.RedirectOutput(nullptr);
 
-  // Finish compression stream and reset.
+  // Flush compression stream and reset.
-  RETURN_IF_ERROR(compression_stream_->Finish(),
+  RETURN_IF_ERROR(compression_stream_->Flush(),
-                  "Failed to finish compression stream");
+                  "Failed to flush compression stream");
   compression_stream_.reset();
 
   // Wait for the server ack. This must be done before sending more data.

cdc_rsync/cdc_rsync_client.h

@@ -21,23 +21,22 @@
 #include <vector>
 
 #include "absl/status/status.h"
-#include "absl/status/statusor.h"
 #include "cdc_rsync/base/message_pump.h"
+#include "cdc_rsync/client_socket.h"
 #include "cdc_rsync/progress_tracker.h"
-#include "common/client_socket.h"
 #include "common/path_filter.h"
-#include "common/process.h"
+#include "common/port_manager.h"
+#include "common/remote_util.h"
 
 namespace cdc_ft {
 
 class Process;
-class RemoteUtil;
-class ServerArch;
 class ZstdStream;
 
 class CdcRsyncClient {
  public:
   struct Options {
+    int port = RemoteUtil::kDefaultSshPort;
     bool delete_ = false;
     bool recursive = false;
     int verbosity = 0;
@@ -53,14 +52,10 @@ class CdcRsyncClient {
     int compress_level = 6;
     int connection_timeout_sec = 10;
     std::string ssh_command;
-    std::string sftp_command;
+    std::string scp_command;
     std::string sources_dir;  // Base dir for files loaded for --files-from.
     PathFilter filter;
 
-    // Backwards compatibility for switching from scp to sftp.
-    // Used internally, do not use.
-    std::string deprecated_scp_command;
-
     // Compression level 0 is invalid.
     static constexpr int kMinCompressLevel = -5;
     static constexpr int kMaxCompressLevel = 22;
@@ -77,7 +72,7 @@ class CdcRsyncClient {
  private:
   // Starts the server process. If the method returns a status with tag
   // |kTagDeployServer|, Run() calls DeployServer() and tries again.
-  absl::Status StartServer(const ServerArch& arch);
+  absl::Status StartServer();
 
   // Stops the server process.
   absl::Status StopServer();
@@ -89,7 +84,7 @@ class CdcRsyncClient {
   absl::Status Sync();
 
   // Copies all gamelet components to the gamelet.
-  absl::Status DeployServer(const ServerArch& arch);
+  absl::Status DeployServer();
 
   // Sends relevant options to the server.
   absl::Status SendOptions();
@@ -121,15 +116,13 @@ class CdcRsyncClient {
   // Stops the zstd compression stream.
   absl::Status StopCompressionStream();
 
-  // Returns true if the target is a remote target.
-  bool IsRemoteConnection() const { return remote_util_ != nullptr; }
-
   Options options_;
   std::vector<std::string> sources_;
+  const std::string user_host_;
   const std::string destination_;
   WinProcessFactory process_factory_;
-  std::unique_ptr<RemoteUtil> remote_util_;
+  RemoteUtil remote_util_;
-  std::unique_ptr<SocketFinalizer> socket_finalizer_;
+  PortManager port_manager_;
   ClientSocket socket_;
   MessagePump message_pump_{&socket_, MessagePump::PacketReceivedDelegate()};
   ConsoleProgressPrinter printer_;
@@ -137,11 +130,10 @@ class CdcRsyncClient {
   std::unique_ptr<ZstdStream> compression_stream_;
 
   std::unique_ptr<Process> server_process_;
-  std::unique_ptr<Process> port_forwarding_process_;
   std::string server_output_;  // Written in a background thread. Do not access
   std::string server_error_;   // while the server process is active.
   int server_exit_code_ = 0;
-  std::atomic_int server_listen_port_{0};
+  std::atomic_bool is_server_listening_{false};
   bool is_server_error_ = false;
 
   // All source files found on the client.

cdc_rsync/client_socket.cc

@@ -12,14 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "common/client_socket.h"
+#include "cdc_rsync/client_socket.h"
+
+#include <winsock2.h>
+#include <ws2tcpip.h>
 
 #include <cassert>
 
 #include "common/log.h"
-#include "common/socket_internal.h"
 #include "common/status.h"
-#include "common/stopwatch.h"
 #include "common/util.h"
 
 namespace cdc_ft {
@@ -28,9 +29,9 @@ namespace {
 // Creates a status with the given |message| and the last WSA error.
 // Assigns Tag::kSocketEof for WSAECONNRESET errors.
 absl::Status MakeSocketStatus(const char* message) {
-  const int err = GetLastError();
+  const int err = WSAGetLastError();
-  absl::Status status = MakeStatus("%s: %s", message, GetErrorStr(err));
+  absl::Status status = MakeStatus("%s: %s", message, Util::GetWin32Error(err));
-  if (err == kErrConnReset) {
+  if (err == WSAECONNRESET) {
     status = SetTag(status, Tag::kSocketEof);
   }
   return status;
@@ -38,71 +39,57 @@ absl::Status MakeSocketStatus(const char* message) {
 
 }  // namespace
 
-struct ClientSocketInfo {
+struct SocketInfo {
-  SocketType socket;
+  SOCKET socket;
 
-  ClientSocketInfo() : socket(kInvalidSocket) {}
+  SocketInfo() : socket(INVALID_SOCKET) {}
 };
 
 ClientSocket::ClientSocket() = default;
 
 ClientSocket::~ClientSocket() { Disconnect(); }
 
-// static
-absl::Status ClientSocket::WaitForConnection(int port, absl::Duration timeout) {
-  assert(port != 0);
-  Stopwatch timeout_timer;
-  ClientSocket socket;
-  for (;;) {
-    absl::Status status = socket.Connect(port);
-    if (status.ok()) {
-      return absl::OkStatus();
-    }
-
-    if (timeout_timer.Elapsed() > timeout) {
-      return SetTag(
-          absl::DeadlineExceededError("Timeout while connecting to server"),
-          Tag::kConnectionTimeout);
-    }
-
-    Util::Sleep(50);
-  }
-}
-
 absl::Status ClientSocket::Connect(int port) {
+  WSADATA wsaData;
+  int result = WSAStartup(MAKEWORD(2, 2), &wsaData);
+  if (result != 0) {
+    return MakeStatus("WSAStartup() failed: %i", result);
+  }
+
   addrinfo hints;
-  memset(&hints, 0, sizeof(hints));
+  ZeroMemory(&hints, sizeof(hints));
   hints.ai_family = AF_INET;
   hints.ai_socktype = SOCK_STREAM;
   hints.ai_protocol = IPPROTO_TCP;
 
   // Resolve the server address and port.
   addrinfo* addr_infos = nullptr;
-  int result = getaddrinfo("localhost", std::to_string(port).c_str(), &hints,
+  result = getaddrinfo("localhost", std::to_string(port).c_str(), &hints,
                        &addr_infos);
   if (result != 0) {
+    WSACleanup();
     return MakeStatus("getaddrinfo() failed: %i", result);
   }
-  AddrInfoReleaser releaser(addr_infos);
 
-  socket_info_ = std::make_unique<ClientSocketInfo>();
+  socket_info_ = std::make_unique<SocketInfo>();
   int count = 0;
   for (addrinfo* curr = addr_infos; curr; curr = curr->ai_next, count++) {
     socket_info_->socket =
-        socket(curr->ai_family, curr->ai_socktype, curr->ai_protocol);
+        socket(addr_infos->ai_family, addr_infos->ai_socktype,
-    if (socket_info_->socket == kInvalidSocket) {
+               addr_infos->ai_protocol);
+    if (socket_info_->socket == INVALID_SOCKET) {
       LOG_DEBUG("socket() failed for addr_info %i: %s", count,
-                GetLastErrorStr());
+                Util::GetWin32Error(WSAGetLastError()).c_str());
       continue;
     }
 
     // Connect to server.
     result = connect(socket_info_->socket, curr->ai_addr,
                      static_cast<int>(curr->ai_addrlen));
-    if (result == kSocketError) {
+    if (result == SOCKET_ERROR) {
-      LOG_DEBUG("connect() failed for addr_info %i: %s", count,
+      LOG_DEBUG("connect() failed for addr_info %i: %i", count, result);
-                GetLastErrorStr());
+      closesocket(socket_info_->socket);
-      Close(&socket_info_->socket);
+      socket_info_->socket = INVALID_SOCKET;
       continue;
     }
 
@@ -110,8 +97,11 @@ absl::Status ClientSocket::Connect(int port) {
     break;
   }
 
-  if (socket_info_->socket == kInvalidSocket) {
+  freeaddrinfo(addr_infos);
+
+  if (socket_info_->socket == INVALID_SOCKET) {
     socket_info_.reset();
+    WSACleanup();
     return MakeStatus("Unable to connect to port %i", port);
   }
 
@@ -124,15 +114,19 @@ void ClientSocket::Disconnect() {
     return;
   }
 
-  Close(&socket_info_->socket);
+  if (socket_info_->socket != INVALID_SOCKET) {
+    closesocket(socket_info_->socket);
+    socket_info_->socket = INVALID_SOCKET;
+  }
+
   socket_info_.reset();
+  WSACleanup();
 }
 
 absl::Status ClientSocket::Send(const void* buffer, size_t size) {
-  int result =
+  int result = send(socket_info_->socket, static_cast<const char*>(buffer),
-      HANDLE_EINTR(send(socket_info_->socket, static_cast<const char*>(buffer),
+                    static_cast<int>(size), /*flags */ 0);
-                        static_cast<int>(size), /*flags */ 0));
+  if (result == SOCKET_ERROR) {
-  if (result == kSocketError) {
     return MakeSocketStatus("send() failed");
   }
 
@@ -148,10 +142,9 @@ absl::Status ClientSocket::Receive(void* buffer, size_t size,
   }
 
   int flags = allow_partial_read ? 0 : MSG_WAITALL;
-  int bytes_read =
+  int bytes_read = recv(socket_info_->socket, static_cast<char*>(buffer),
-      HANDLE_EINTR(recv(socket_info_->socket, static_cast<char*>(buffer),
+                        static_cast<int>(size), flags);
-                        static_cast<int>(size), flags));
+  if (bytes_read == SOCKET_ERROR) {
-  if (bytes_read == kSocketError) {
     return MakeSocketStatus("recv() failed");
   }
 
@@ -170,9 +163,9 @@ absl::Status ClientSocket::Receive(void* buffer, size_t size,
 }
 
 absl::Status ClientSocket::ShutdownSendingEnd() {
-  int result = shutdown(socket_info_->socket, kSendingEnd);
+  int result = shutdown(socket_info_->socket, SD_SEND);
-  if (result == kSocketError) {
+  if (result == SOCKET_ERROR) {
-    return MakeStatus("Socket shutdown failed: %s", GetLastErrorStr());
+    return MakeSocketStatus("shutdown() failed");
   }
 
   return absl::OkStatus();

cdc_rsync/client_socket.h

@@ -14,13 +14,13 @@
  * limitations under the License.
  */
 
-#ifndef COMMON_CLIENT_SOCKET_H_
+#ifndef CDC_RSYNC_CLIENT_SOCKET_H_
-#define COMMON_CLIENT_SOCKET_H_
+#define CDC_RSYNC_CLIENT_SOCKET_H_
 
 #include <memory>
 
 #include "absl/status/status.h"
-#include "common/socket.h"
+#include "cdc_rsync/base/socket.h"
 
 namespace cdc_ft {
 
@@ -29,10 +29,6 @@ class ClientSocket : public Socket {
   ClientSocket();
   ~ClientSocket();
 
-  // Polls until a connection to |port| succeeds.
-  // Returns TimeoutError
-  static absl::Status WaitForConnection(int port, absl::Duration timeout);
-
   // Connects to localhost on |port|.
   absl::Status Connect(int port);
 
@@ -49,9 +45,9 @@ class ClientSocket : public Socket {
                        size_t* bytes_received) override;
 
  private:
-  std::unique_ptr<struct ClientSocketInfo> socket_info_;
+  std::unique_ptr<struct SocketInfo> socket_info_;
 };
 
 }  // namespace cdc_ft
 
-#endif  // COMMON_CLIENT_SOCKET_H_
+#endif  // CDC_RSYNC_CLIENT_SOCKET_H_

cdc_rsync/file_finder_and_sender_test.cc

@@ -18,8 +18,8 @@
 
 #include "absl/strings/str_format.h"
 #include "absl/strings/str_join.h"
+#include "cdc_rsync/base/fake_socket.h"
 #include "cdc_rsync/base/message_pump.h"
-#include "common/fake_socket.h"
 #include "common/log.h"
 #include "common/path.h"
 #include "common/path_filter.h"

cdc_rsync/main.cc

@@ -68,16 +68,16 @@ ReturnCode TagToMessage(cdc_ft::Tag tag,
 
     case cdc_ft::Tag::kDeployServer:
       *msg =
-          "Failed to deploy or run the instance components for unknown "
+          "Failed to deploy the instance components for unknown reasons. "
-          "reasons. Please report this issue.";
+          "Please report this issue.";
       return ReturnCode::kDeployFailed;
 
     case cdc_ft::Tag::kConnectionTimeout:
       // Server connection timed out. SSH probably stale.
       *msg = absl::StrFormat(
-          "Server connection timed out. Verify that the host '%s' "
+          "Server connection timed out. Verify that host '%s' and port '%i' "
-          "is correct, or specify a larger timeout with --contimeout.",
+          "are correct, or specify a larger timeout with --contimeout.",
-          params.user_host);
+          params.user_host, params.options.port);
       return ReturnCode::kConnectionTimeout;
 
     case cdc_ft::Tag::kCount:

cdc_rsync/params.cc

@@ -19,9 +19,7 @@
 #include "absl/status/status.h"
 #include "absl/strings/str_format.h"
 #include "absl/strings/str_split.h"
-#include "common/build_version.h"
 #include "common/path.h"
-#include "common/remote_util.h"
 #include "lib/zstd.h"
 
 namespace cdc_ft {
@@ -38,22 +36,24 @@ void PrintError(const absl::FormatSpec<Args...>& format, Args... args) {
 enum class OptionResult { kConsumedKey, kConsumedKeyValue, kError };
 
 const char kHelpText[] =
-    R"(
+    R"(Copy local files to a gamelet
-Matching files are skipped based on file size and modified time. For partially
+
-matching files only the differences are transferred. The destination directory
+Synchronizes local files and files on a gamelet. Matching files are skipped.
-can be the same Windows machine or a remote Windows or Linux device.
+For partially matching files only the deltas are transferred.
 
 Usage:
-  cdc_rsync [options] source [source]... [[user@]host:]destination
+  cdc_rsync [options] source [source]... [user@]host:destination
 
 Parameters:
-  source                    Local file or directory to be copied or synced
+  source                  Local file or directory to be copied
   user                    Remote SSH user name
   host                    Remote host or IP address
-  destination               Local or remote destination directory
+  destination             Remote destination directory
 
 Options:
-    --contimeout sec        Remote connection timeout in seconds (default: 10)
+    --ip string           Gamelet IP. Required.
+    --port number         SSH port to use. Required.
+    --contimeout sec      Gamelet connection timeout in seconds (default: 10)
 -q, --quiet               Quiet mode, only print errors
 -v, --verbose             Increase output verbosity
     --json                Print JSON progress
@@ -61,55 +61,47 @@ Options:
 -r, --recursive           Recurse into directories
     --delete              Delete extraneous files from destination directory
 -z, --compress            Compress file data during the transfer
-    --compress-level <num>  Explicitly set compression level (default: 6)
+    --compress-level num  Explicitly set compression level (default: 6)
 -c, --checksum            Skip files based on checksum, not mod-time & size
 -W, --whole-file          Always copy files whole,
                           do not apply delta-transfer algorithm
     --exclude pattern     Exclude files matching pattern
-    --exclude-from <file>   Read exclude patterns from file
+    --exclude-from file   Read exclude patterns from file
     --include pattern     Don't exclude files matching pattern
-    --include-from <file>   Read include patterns from file
+    --include-from file   Read include patterns from file
-    --files-from <file>     Read list of source files from file
+    --files-from file     Read list of source files from file
 -R, --relative            Use relative path names
     --existing            Skip creating new files on instance
-    --copy-dest <dir>       Use files from dir as sync base if files are missing
+    --copy-dest dir       Use files from dir as sync base if files are missing
-    --ssh-command <cmd>     Path and arguments of ssh command to use, e.g.
+    --ssh-command         Path and arguments of ssh command to use, e.g.
-                            "C:\path\to\ssh.exe -p 12345 -i id_rsa -oUserKnownHostsFile=known_hosts"
+                          C:\path\to\ssh.exe -F config -i id_rsa -oStrictHostKeyChecking=yes -oUserKnownHostsFile="""known_hosts"""
                           Can also be specified by the CDC_SSH_COMMAND environment variable.
-    --sftp-command <cmd>    Path and arguments of sftp command to use, e.g.
+    --scp-command         Path and arguments of scp command to use, e.g.
-                            "C:\path\to\sftp.exe -P 12345 -i id_rsa -oUserKnownHostsFile=known_hosts"
+                          C:\path\to\scp.exe -F config -i id_rsa -oStrictHostKeyChecking=yes -oUserKnownHostsFile="""known_hosts"""
-                            Can also be specified by the CDC_SFTP_COMMAND environment variable.
+                          Can also be specified by the CDC_SCP_COMMAND environment variable.
--h, --help                  Help for cdc_rsync
+-h  --help                Help for cdc_rsync
 )";
 
 constexpr char kSshCommandEnvVar[] = "CDC_SSH_COMMAND";
 constexpr char kScpCommandEnvVar[] = "CDC_SCP_COMMAND";
-constexpr char kSftpCommandEnvVar[] = "CDC_SFTP_COMMAND";
 
 // Populates some parameters from environment variables.
 void PopulateFromEnvVars(Parameters* parameters) {
   path::GetEnv(kSshCommandEnvVar, &parameters->options.ssh_command)
       .IgnoreError();
-  path::GetEnv(kScpCommandEnvVar, &parameters->options.deprecated_scp_command)
+  path::GetEnv(kScpCommandEnvVar, &parameters->options.scp_command)
       .IgnoreError();
-  path::GetEnv(kSftpCommandEnvVar, &parameters->options.sftp_command)
-      .IgnoreError();
-}
-
-// Returns false and prints an error if |value| is null or empty.
-bool ValidateValue(const std::string& option_name, const char* value) {
-  if (!value) {
-    PrintError("Option '%s' needs a value", option_name);
-    return false;
-  }
-  return true;
 }
 
 // Handles the --exclude-from and --include-from options.
 OptionResult HandleFilterRuleFile(const std::string& option_name,
                                   const char* path, PathFilter::Rule::Type type,
                                   Parameters* params) {
-  assert(path);
+  if (!path) {
+    PrintError("Option '%s' needs a value", option_name);
+    return OptionResult::kError;
+  }
+
   std::vector<std::string> patterns;
   absl::Status status = path::ReadAllLines(
       path, &patterns,
@@ -172,6 +164,13 @@ bool LoadFilesFrom(const std::string& files_from,
 
 OptionResult HandleParameter(const std::string& key, const char* value,
                              Parameters* params, bool* help) {
+  if (key == "port") {
+    if (value) {
+      params->options.port = atoi(value);
+    }
+    return OptionResult::kConsumedKeyValue;
+  }
+
   if (key == "delete") {
     params->options.delete_ = true;
     return OptionResult::kConsumedKey;
@@ -198,34 +197,29 @@ OptionResult HandleParameter(const std::string& key, const char* value,
   }
 
   if (key == "include") {
-    if (!ValidateValue(key, value)) return OptionResult::kError;
     params->options.filter.AddRule(PathFilter::Rule::Type::kInclude, value);
     return OptionResult::kConsumedKeyValue;
   }
 
   if (key == "include-from") {
-    if (!ValidateValue(key, value)) return OptionResult::kError;
     return HandleFilterRuleFile(key, value, PathFilter::Rule::Type::kInclude,
                                 params);
   }
 
   if (key == "exclude") {
-    if (!ValidateValue(key, value)) return OptionResult::kError;
     params->options.filter.AddRule(PathFilter::Rule::Type::kExclude, value);
     return OptionResult::kConsumedKeyValue;
   }
 
   if (key == "exclude-from") {
-    if (!ValidateValue(key, value)) return OptionResult::kError;
     return HandleFilterRuleFile(key, value, PathFilter::Rule::Type::kExclude,
                                 params);
   }
 
   if (key == "files-from") {
     // Implies -R.
-    if (!ValidateValue(key, value)) return OptionResult::kError;
     params->options.relative = true;
-    params->files_from = value;
+    params->files_from = value ? value : std::string();
     return OptionResult::kConsumedKeyValue;
   }
 
@@ -240,14 +234,16 @@ OptionResult HandleParameter(const std::string& key, const char* value,
   }
 
   if (key == "compress-level") {
-    if (!ValidateValue(key, value)) return OptionResult::kError;
+    if (value) {
       params->options.compress_level = atoi(value);
+    }
     return OptionResult::kConsumedKeyValue;
   }
 
   if (key == "contimeout") {
-    if (!ValidateValue(key, value)) return OptionResult::kError;
+    if (value) {
       params->options.connection_timeout_sec = atoi(value);
+    }
     return OptionResult::kConsumedKeyValue;
   }
 
@@ -272,8 +268,7 @@ OptionResult HandleParameter(const std::string& key, const char* value,
   }
 
   if (key == "copy-dest") {
-    if (!ValidateValue(key, value)) return OptionResult::kError;
+    params->options.copy_dest = value ? value : std::string();
-    params->options.copy_dest = value;
     return OptionResult::kConsumedKeyValue;
   }
 
@@ -283,28 +278,12 @@ OptionResult HandleParameter(const std::string& key, const char* value,
   }
 
   if (key == "ssh-command") {
-    if (!ValidateValue(key, value)) return OptionResult::kError;
+    params->options.ssh_command = value ? value : std::string();
-    params->options.ssh_command = value;
     return OptionResult::kConsumedKeyValue;
   }
 
   if (key == "scp-command") {
-    // Backwards compatibility. Note that this flag is hidden from the help.
+    params->options.scp_command = value ? value : std::string();
-    if (!ValidateValue(key, value)) return OptionResult::kError;
-    params->options.deprecated_scp_command = value;
-    return OptionResult::kConsumedKeyValue;
-  }
-
-  if (key == "sftp-command") {
-    if (!ValidateValue(key, value)) return OptionResult::kError;
-    params->options.sftp_command = value;
-    return OptionResult::kConsumedKeyValue;
-  }
-
-  if (key == "forward-port") {
-    // This param is no longer needed. Just print a warning for backwards
-    // compatibility.
-    std::cout << "--forward-port argument no longer needed" << std::endl;
     return OptionResult::kConsumedKeyValue;
   }
 
@@ -314,8 +293,6 @@ OptionResult HandleParameter(const std::string& key, const char* value,
 
 bool ValidateParameters(const Parameters& params, bool help) {
   if (help) {
-    std::cout << "cdc_rsync - Synchronize files and directories. Version: "
-              << BUILD_VERSION << "\n";
     std::cout << kHelpText;
     return false;
   }
@@ -325,6 +302,11 @@ bool ValidateParameters(const Parameters& params, bool help) {
     return false;
   }
 
+  if (params.options.port <= 0 || params.options.port > UINT16_MAX) {
+    PrintError("--port must specify a valid port");
+    return false;
+  }
+
   // Note: ZSTD_minCLevel() is ridiculously small (-131072), so use a
   // reasonable value.
   assert(ZSTD_minCLevel() <= Options::kMinCompressLevel);
@@ -369,6 +351,14 @@ bool ValidateParameters(const Parameters& params, bool help) {
     return false;
   }
 
+  if (params.user_host.empty()) {
+    PrintError(
+        "No remote host specified in destination '%s'. "
+        "Expected [user@]host:destination.",
+        params.destination);
+    return false;
+  }
+
   return true;
 }
 
@@ -379,7 +369,11 @@ bool CheckOptionResult(OptionResult result, const std::string& name,
       return true;
 
     case OptionResult::kConsumedKeyValue:
-      return ValidateValue(name, value);
+      if (!value) {
+        PrintError("Option '%s' needs a value", name);
+        return false;
+      }
+      return true;
 
     case OptionResult::kError:
       // Error message was already printed.
@@ -394,15 +388,16 @@ bool CheckOptionResult(OptionResult result, const std::string& name,
 // afterward and |user_host| is |user@foo.com|. Does not touch Windows drives,
 // e.g. C:\foo.
 void PopUserHost(std::string* destination, std::string* user_host) {
-  user_host->clear();
-
-  // Don't mistake the C part of C:\foo or \\share\C:\foo as user/host.
-  if (!path::GetDrivePrefix(*destination).empty()) return;
-
   std::vector<std::string> parts =
       absl::StrSplit(*destination, absl::MaxSplits(':', 1));
   if (parts.size() < 2) return;
 
+  // Don't mistake the C part of C:\foo as user/host.
+  if (parts[0].size() == 1 && toupper(parts[0][0]) >= 'A' &&
+      toupper(parts[0][0]) <= 'Z') {
+    return;
+  }
+
   *user_host = parts[0];
   *destination = parts[1];
 }
@@ -487,27 +482,6 @@ bool Parse(int argc, const char* const* argv, Parameters* parameters) {
 
   PopUserHost(&parameters->destination, &parameters->user_host);
 
-  // Backwards compabitility after switching to sftp. Convert scp to sftp
-  // command Note that this flag is hidden from the help.
-  if (parameters->options.sftp_command.empty() &&
-      !parameters->options.deprecated_scp_command.empty()) {
-    LOG_WARNING(
-        "The CDC_SCP_COMMAND environment variable and the --scp-command flag "
-        "are deprecated. Please set CDC_SFTP_COMMAND or --sftp-command "
-        "instead.");
-
-    parameters->options.sftp_command = RemoteUtil::ScpToSftpCommand(
-        parameters->options.deprecated_scp_command);
-    if (!parameters->options.sftp_command.empty()) {
-      LOG_WARNING("Converted scp command '%s' to sftp command '%s'.",
-                  parameters->options.deprecated_scp_command,
-                  parameters->options.sftp_command);
-    } else {
-      LOG_WARNING("Failed to convert scp command '%s' to sftp command.",
-                  parameters->options.deprecated_scp_command);
-    }
-  }
-
   if (!ValidateParameters(*parameters, help)) {
     return false;
   }

cdc_rsync/params_test.cc

@@ -32,10 +32,6 @@ constexpr char kUserHostDst[] = "user@host:destination";
 constexpr char kUserHost[] = "user@host";
 constexpr char kDst[] = "destination";
 
-constexpr char kSshCommandEnvVar[] = "CDC_SSH_COMMAND";
-constexpr char kScpCommandEnvVar[] = "CDC_SCP_COMMAND";
-constexpr char kSftpCommandEnvVar[] = "CDC_SFTP_COMMAND";
-
 class TestLog : public Log {
  public:
   explicit TestLog() : Log(LogLevel::kInfo) {}
@@ -64,11 +60,6 @@ class ParamsTest : public ::testing::Test {
   void TearDown() override {
     std::cout.rdbuf(prev_stdout_);
     std::cerr.rdbuf(prev_stderr_);
-
-    // Clear env. They seem to be sticky sometimes and leak into other tests.
-    path::SetEnv(kSshCommandEnvVar, "");
-    path::SetEnv(kScpCommandEnvVar, "");
-    path::SetEnv(kSftpCommandEnvVar, "");
   }
 
  protected:
@@ -106,6 +97,7 @@ class ParamsTest : public ::testing::Test {
 TEST_F(ParamsTest, ParseSucceedsDefaults) {
   const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst, NULL};
   EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
+  EXPECT_EQ(RemoteUtil::kDefaultSshPort, parameters_.options.port);
   EXPECT_FALSE(parameters_.options.delete_);
   EXPECT_FALSE(parameters_.options.recursive);
   EXPECT_EQ(0, parameters_.options.verbosity);
@@ -153,6 +145,13 @@ TEST_F(ParamsTest, ParseFailsOnCompressLevelEqualsNoValue) {
   ExpectError(NeedsValueError("compress-level"));
 }
 
+TEST_F(ParamsTest, ParseFailsOnPortEqualsNoValue) {
+  const char* argv[] = {"cdc_rsync.exe", "--port=", kSrc, kUserHostDst, NULL};
+  EXPECT_FALSE(
+      Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
+  ExpectError(NeedsValueError("port"));
+}
+
 TEST_F(ParamsTest, ParseFailsOnContimeoutEqualsNoValue) {
   const char* argv[] = {"cdc_rsync.exe", "--contimeout=", kSrc, kUserHostDst,
                         NULL};
@@ -161,57 +160,24 @@ TEST_F(ParamsTest, ParseFailsOnContimeoutEqualsNoValue) {
   ExpectError(NeedsValueError("contimeout"));
 }
 
-TEST_F(ParamsTest, ParseSucceedsWithSshSftpCommands) {
+TEST_F(ParamsTest, ParseSucceedsWithSshScpCommands) {
-  const char* argv[] = {
+  const char* argv[] = {"cdc_rsync.exe",        kSrc,
-      "cdc_rsync.exe",          kSrc, kUserHostDst, "--ssh-command=sshcmd",
+                        kUserHostDst,           "--ssh-command=sshcmd",
-      "--sftp-command=sftpcmd", NULL};
+                        "--scp-command=scpcmd", NULL};
   EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
-  EXPECT_EQ(parameters_.options.sftp_command, "sftpcmd");
+  EXPECT_EQ(parameters_.options.scp_command, "scpcmd");
   EXPECT_EQ(parameters_.options.ssh_command, "sshcmd");
 }
 
-TEST_F(ParamsTest, ParseSucceedsWithSshSftpCommandsByEnvVars) {
+TEST_F(ParamsTest, ParseSucceedsWithSshScpCommandsByEnvVars) {
-  EXPECT_OK(path::SetEnv(kSshCommandEnvVar, "sshcmd"));
+  EXPECT_OK(path::SetEnv("CDC_SSH_COMMAND", "sshcmd"));
-  EXPECT_OK(path::SetEnv(kSftpCommandEnvVar, "sftpcmd"));
+  EXPECT_OK(path::SetEnv("CDC_SCP_COMMAND", "scpcmd"));
   const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst, NULL};
   EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
-  EXPECT_EQ(parameters_.options.sftp_command, "sftpcmd");
+  EXPECT_EQ(parameters_.options.scp_command, "scpcmd");
   EXPECT_EQ(parameters_.options.ssh_command, "sshcmd");
 }
 
-TEST_F(ParamsTest, ParseSucceedsWithScpCommandFallback) {
-  const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst,
-                        "--scp-command=C:\\scp.exe foo", NULL};
-  EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
-  EXPECT_EQ(parameters_.options.sftp_command, "C:\\sftp.exe foo");
-}
-
-TEST_F(ParamsTest, ParseSucceedsWithScpCommandFallbackByEnvVar) {
-  EXPECT_OK(path::SetEnv(kScpCommandEnvVar, "C:\\scp.exe foo"));
-  const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst, NULL};
-  EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
-  EXPECT_EQ(parameters_.options.sftp_command, "C:\\sftp.exe foo");
-}
-
-TEST_F(ParamsTest, ParseSucceedsWithSftpOverwritingScp) {
-  const char* argv[] = {"cdc_rsync.exe",
-                        kSrc,
-                        kUserHostDst,
-                        "--scp-command=C:\\scp.exe foo",
-                        "--sftp-command=sftpcmd",
-                        NULL};
-  EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
-  EXPECT_EQ(parameters_.options.sftp_command, "sftpcmd");
-}
-
-TEST_F(ParamsTest, ParseSucceedsWithSftpEnvVarOverwritingScp) {
-  EXPECT_OK(path::SetEnv(kSftpCommandEnvVar, "sftpcmd"));
-  const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst,
-                        "--scp-command=C:\\scp.exe foo", NULL};
-  EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
-  EXPECT_EQ(parameters_.options.sftp_command, "sftpcmd");
-}
-
 TEST_F(ParamsTest, ParseSucceedsWithNoSshCommand) {
   const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst,
                         "--ssh-command=", NULL};
@@ -220,33 +186,26 @@ TEST_F(ParamsTest, ParseSucceedsWithNoSshCommand) {
   ExpectError(NeedsValueError("ssh-command"));
 }
 
-TEST_F(ParamsTest, ParseSucceedsWithNoSftpCommand) {
+TEST_F(ParamsTest, ParseSucceedsWithNoScpCommand) {
-  const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst, "--sftp-command",
+  const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst, "--scp-command",
                         NULL};
   EXPECT_FALSE(
       Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
-  ExpectError(NeedsValueError("sftp-command"));
+  ExpectError(NeedsValueError("scp-command"));
 }
 
-TEST_F(ParamsTest, ParseSucceedsOnNoUserHost) {
+TEST_F(ParamsTest, ParseFailsOnNoUserHost) {
   const char* argv[] = {"cdc_rsync.exe", kSrc, kDst, NULL};
-  EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
+  EXPECT_FALSE(
+      Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
+  ExpectError("No remote host specified");
 }
 
-TEST_F(ParamsTest, ParseDoesNotThinkDriveIsAHost) {
+TEST_F(ParamsTest, ParseDoesNotThinkCIsAHost) {
   const char* argv[] = {"cdc_rsync.exe", kSrc, "C:\\foo", NULL};
-  EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
+  EXPECT_FALSE(
-  EXPECT_TRUE(parameters_.user_host.empty());
+      Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
-
+  ExpectError("No remote host specified");
-  const char* argv2[] = {"cdc_rsync.exe", kSrc, "\\\\.\\C:\\foo", NULL};
-  EXPECT_TRUE(
-      Parse(static_cast<int>(std::size(argv2)) - 1, argv, &parameters_));
-  EXPECT_TRUE(parameters_.user_host.empty());
-
-  const char* argv3[] = {"cdc_rsync.exe", kSrc, "\\\\?\\C:\\foo", NULL};
-  EXPECT_TRUE(
-      Parse(static_cast<int>(std::size(argv3)) - 1, argv, &parameters_));
-  EXPECT_TRUE(parameters_.user_host.empty());
 }
 
 TEST_F(ParamsTest, ParseWithoutParametersFailsOnMissingSourceAndDestination) {
@@ -326,18 +285,13 @@ TEST_F(ParamsTest, ParseFailsOnUnknownKey) {
 }
 
 TEST_F(ParamsTest, ParseSucceedsWithSupportedKeyValue) {
-  const char* argv[] = {"cdc_rsync.exe",
+  const char* argv[] = {
-                        "--compress-level",
+      "cdc_rsync.exe", "--compress-level", "11", "--contimeout", "99", "--port",
-                        "11",
+      "4086",          "--copy-dest=dest", kSrc, kUserHostDst,   NULL};
-                        "--contimeout",
-                        "99",
-                        "--copy-dest=dest",
-                        kSrc,
-                        kUserHostDst,
-                        NULL};
   EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
   EXPECT_EQ(parameters_.options.compress_level, 11);
   EXPECT_EQ(parameters_.options.connection_timeout_sec, 99);
+  EXPECT_EQ(parameters_.options.port, 4086);
   EXPECT_EQ(parameters_.options.copy_dest, "dest");
   ExpectNoError();
 }
@@ -350,6 +304,13 @@ TEST_F(ParamsTest, ParseSucceedsWithSupportedKeyValueWithoutEqualityForChars) {
   ExpectNoError();
 }
 
+TEST_F(ParamsTest, ParseFailsOnInvalidPort) {
+  const char* argv[] = {"cdc_rsync.exe", "--port=0", kSrc, kUserHostDst, NULL};
+  EXPECT_FALSE(
+      Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
+  ExpectError("--port must specify a valid port");
+}
+
 TEST_F(ParamsTest, ParseFailsOnDeleteNeedsRecursive) {
   const char* argv[] = {"cdc_rsync.exe", "--delete", kSrc, kUserHostDst, NULL};
   EXPECT_FALSE(

cdc_rsync/server_arch.cc

@@ -1,271 +0,0 @@
-// Copyright 2022 Google LLC
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "cdc_rsync/server_arch.h"
-
-#include <filesystem>
-
-#include "absl/strings/match.h"
-#include "absl/strings/str_format.h"
-#include "absl/strings/str_split.h"
-#include "common/ansi_filter.h"
-#include "common/path.h"
-#include "common/remote_util.h"
-#include "common/status_macros.h"
-#include "common/util.h"
-
-namespace cdc_ft {
-namespace {
-
-constexpr char kErrorArchTypeUnhandled[] = "arch_type_unhandled";
-constexpr char kUnsupportedArchErrorFmt[] =
-    "Unsupported remote device architecture '%s'. If you think this is a "
-    "bug, or if this combination should be supported, please file a bug at "
-    "https://github.com/google/cdc-file-transfer.";
-
-absl::StatusOr<ArchType> GetArchTypeFromUname(const std::string& uname_out) {
-  // uname_out is "KERNEL MACHINE"
-  // Possible values for KERNEL: Linux (not sure what else).
-  // Possible values for MACHINE:
-  // https://stackoverflow.com/questions/45125516/possible-values-for-uname-m
-  // Relevant for us: x86_64, aarch64.
-  if (absl::StartsWith(uname_out, "Linux ")) {
-    // Linux kernel. Check CPU type.
-    if (absl::StrContains(uname_out, "x86_64")) {
-      return ArchType::kLinux_x86_64;
-    }
-  }
-
-  if (absl::StartsWith(uname_out, "MSYS_")) {
-    // Windows machine that happens to have Cygwin/MSYS on it. Check CPU type.
-    if (absl::StrContains(uname_out, "x86_64")) {
-      return ArchType::kWindows_x86_64;
-    }
-  }
-
-  return absl::UnimplementedError(
-      absl::StrFormat(kUnsupportedArchErrorFmt, uname_out));
-}
-
-absl::StatusOr<ArchType> GetArchTypeFromWinProcArch(
-    const std::string& arch_out) {
-  // Possible values: AMD64, IA64, ARM64, x86
-  if (absl::StrContains(arch_out, "AMD64")) {
-    return ArchType::kWindows_x86_64;
-  }
-
-  return absl::UnimplementedError(
-      absl::StrFormat(kUnsupportedArchErrorFmt, arch_out));
-}
-
-}  // namespace
-
-// static
-ServerArch ServerArch::GuessFromDestination(const std::string& destination) {
-  // Path starting with ~ or / -> Linux.
-  if (absl::StartsWith(destination, "~") ||
-      absl::StartsWith(destination, "/")) {
-    LOG_DEBUG("Guessed server arch type Linux based on ~ or /");
-    return ServerArch(ArchType::kLinux_x86_64, /*is_guess=*/true);
-  }
-
-  // Path starting with C: etc. -> Windows.
-  if (!path::GetDrivePrefix(destination).empty()) {
-    LOG_DEBUG("Guessed server arch type Windows based on drive prefix");
-    return ServerArch(ArchType::kWindows_x86_64, /*is_guess=*/true);
-  }
-
-  // Path with only / -> Linux.
-  if (absl::StrContains(destination, "/") &&
-      !absl::StrContains(destination, "\\")) {
-    LOG_DEBUG("Guessed server arch type Linux based on forward slashes");
-    return ServerArch(ArchType::kLinux_x86_64, /*is_guess=*/true);
-  }
-
-  // Path with only \\ -> Windows.
-  if (absl::StrContains(destination, "\\") &&
-      !absl::StrContains(destination, "/")) {
-    LOG_DEBUG("Guessed server arch type Windows based on backslashes");
-    return ServerArch(ArchType::kWindows_x86_64, /*is_guess=*/true);
-  }
-
-  // Default to Linux.
-  LOG_DEBUG("Guessed server arch type Linux as default");
-  return ServerArch(ArchType::kLinux_x86_64, /*is_guess=*/true);
-}
-
-// static
-ServerArch ServerArch::DetectFromLocalDevice() {
-  LOG_DEBUG("Detected local device type %s",
-            GetArchTypeStr(GetLocalArchType()));
-  return ServerArch(GetLocalArchType(), /*is_guess=*/false);
-}
-
-// static
-absl::StatusOr<ServerArch> ServerArch::DetectFromRemoteDevice(
-    RemoteUtil* remote_util) {
-  assert(remote_util);
-
-  // Run uname, assuming it's a Linux machine.
-  std::string uname_out;
-  std::string linux_cmd = "uname -sm";
-  absl::Status status = remote_util->RunWithCapture(
-      linux_cmd, "uname", &uname_out, nullptr, ArchType::kLinux_x86_64);
-  if (status.ok()) {
-    // Running uname on Windows, assuming it's Linux, leads to tons of ANSI
-    // escape sequences in the output. Remove them to at least get some readable
-    // output.
-    uname_out = absl::StripAsciiWhitespace(
-        ansi_filter::RemoveEscapeSequences(uname_out));
-
-    LOG_DEBUG("Uname returned '%s'", uname_out);
-    absl::StatusOr<ArchType> type = GetArchTypeFromUname(uname_out);
-    if (type.ok()) {
-      LOG_DEBUG("Detected server arch type '%s' from uname",
-                GetArchTypeStr(*type));
-      return ServerArch(*type, /*is_guess=*/false);
-    }
-    status = type.status();
-  }
-  LOG_DEBUG(
-      "Failed to detect arch type from uname; this is expected if the remote "
-      "machine is not Linux; will try Windows next: %s",
-      status.ToString());
-
-  // Check %PROCESSOR_ARCHITECTURE%, assuming it's a Windows machine.
-  // Note: That space after PROCESSOR_ARCHITECTURE is important or else Windows
-  //       command magic interprets quotes as part of the string.
-  std::string arch_out;
-  std::string windows_cmd = "\"cmd /C set PROCESSOR_ARCHITECTURE \"";
-  status = remote_util->RunWithCapture(windows_cmd,
-                                       "set PROCESSOR_ARCHITECTURE", &arch_out,
-                                       nullptr, ArchType::kWindows_x86_64);
-  if (status.ok()) {
-    LOG_DEBUG("PROCESSOR_ARCHITECTURE is '%s'", arch_out);
-    absl::StatusOr<ArchType> type = GetArchTypeFromWinProcArch(arch_out);
-    if (type.ok()) {
-      LOG_DEBUG("Detected server arch type '%s' from PROCESSOR_ARCHITECTURE",
-                GetArchTypeStr(*type));
-      return ServerArch(*type, /*is_guess=*/false);
-    }
-    status = type.status();
-  }
-  LOG_DEBUG("Failed to detect arch type from PROCESSOR_ARCHITECTURE: %s",
-            status.ToString());
-
-  return absl::InternalError("Failed to detect remote architecture");
-}
-
-// static
-std::string ServerArch::CdcRsyncFilename() {
-  switch (GetLocalArchType()) {
-    case ArchType::kWindows_x86_64:
-      return "cdc_rsync.exe";
-    case ArchType::kLinux_x86_64:
-      return "cdc_rsync";
-    default:
-      assert(!kErrorArchTypeUnhandled);
-      return std::string();
-  }
-}
-
-ServerArch::ServerArch(ArchType type, bool is_guess)
-    : type_(type), is_guess_(is_guess) {}
-
-ServerArch::~ServerArch() {}
-
-const char* ServerArch::GetTypeStr() const { return GetArchTypeStr(type_); }
-
-std::string ServerArch::CdcServerFilename() const {
-  switch (type_) {
-    case ArchType::kWindows_x86_64:
-      return "cdc_rsync_server.exe";
-    case ArchType::kLinux_x86_64:
-      return "cdc_rsync_server";
-    default:
-      assert(!kErrorArchTypeUnhandled);
-      return std::string();
-  }
-}
-
-std::string ServerArch::RemoteToolsBinDir() const {
-  if (IsWindowsArchType(type_)) {
-    return "AppData\\Roaming\\cdc-file-transfer\\bin\\";
-  }
-
-  if (IsLinuxArchType(type_)) {
-    return ".cache/cdc-file-transfer/bin/";
-  }
-
-  assert(!kErrorArchTypeUnhandled);
-  return std::string();
-}
-
-std::string ServerArch::GetStartServerCommand(int exit_code_not_found,
-                                              const std::string& args) const {
-  std::string server_path = RemoteToolsBinDir() + CdcServerFilename();
-
-  if (IsWindowsArchType(type_)) {
-    // TODO(ljusten): On Windows, ssh does not seem to forward the Powershell
-    // exit code (exit_code_not_found) to the process. However, that's really
-    // a minor issue and means we display "Deploying server..." instead of
-    // "Server not deployed. Deploying...";
-    return RemoteUtil::QuoteForWindows(
-        absl::StrFormat("powershell -Command "
-                        "Set-StrictMode -Version 2; "
-                        "$ErrorActionPreference = 'Stop'; "
-                        "if (-not (Test-Path -Path '%s')) { "
-                        "  exit %i; "
-                        "} "
-                        "%s %s",
-                        server_path, exit_code_not_found, server_path, args));
-  }
-
-  if (IsLinuxArchType(type_)) {
-    return absl::StrFormat("if [ ! -f %s ]; then exit %i; fi; %s %s",
-                           server_path, exit_code_not_found, server_path, args);
-  }
-
-  assert(!kErrorArchTypeUnhandled);
-  return std::string();
-}
-
-std::string ServerArch::GetDeploySftpCommands() const {
-  std::string commands;
-
-  // Create the remote tools bin dir if it doesn't exist yet.
-  // This assumes that sftp's remote startup directory is the home directory.
-  const std::string server_dir = path::ToUnix(RemoteToolsBinDir());
-  std::vector<std::string> dir_parts =
-      absl::StrSplit(server_dir, '/', absl::SkipEmpty());
-  for (const std::string& dir : dir_parts) {
-    // Use -mkdir to ignore errors if the directory already exists.
-    commands += absl::StrFormat("-mkdir %s\ncd %s\n", dir, dir);
-  }
-
-  // Copy the server binary to a temp location. This assumes that sftp's local
-  // startup directory is cdc_rsync's exe dir.
-  const std::string server_file = CdcServerFilename();
-  const std::string server_temp_file = server_file + Util::GenerateUniqueId();
-  commands += absl::StrFormat("put %s %s\n", server_file, server_temp_file);
-
-  // Restore permissions in case they changed and propagate temp file.
-  commands += absl::StrFormat("-chmod 755 %s\n", server_file);
-  commands += absl::StrFormat("chmod 755 %s\n", server_temp_file);
-  commands += absl::StrFormat("rename %s %s\n", server_temp_file, server_file);
-
-  return commands;
-}
-
-}  // namespace cdc_ft

cdc_rsync/server_arch.h

@@ -1,97 +0,0 @@
-/*
- * Copyright 2022 Google LLC
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef CDC_RSYNC_SERVER_ARCH_H_
-#define CDC_RSYNC_SERVER_ARCH_H_
-
-#include <string>
-
-#include "absl/status/statusor.h"
-#include "common/arch_type.h"
-
-namespace cdc_ft {
-
-class RemoteUtil;
-
-// Abstracts all architecture specifics of cdc_rsync_server deployment.
-// Comes in two flavors, "guessed" and "detected". Guesses are used as an
-// optimization. For instance, if one syncs to C:\some\path, it's clearly a
-// Windows machine and we can skip detection.
-class ServerArch {
- public:
-  // Guesses the arch type based on the destination path, e.g. path starting
-  // with C: indicate Windows. This is a guessed type. It may be wrong. For
-  // instance, if destination is just a single folder like "foo", the method
-  // defaults to Type::kLinux.
-  static ServerArch GuessFromDestination(const std::string& destination);
-
-  // Returns the arch type that matches the current process's type.
-  // This is not a guessed type, it is reliable.
-  static ServerArch DetectFromLocalDevice();
-
-  // Creates an  by properly detecting it on the remote device.
-  // This is more costly than guessing, but it is reliable.
-  static absl::StatusOr<ServerArch> DetectFromRemoteDevice(
-      RemoteUtil* remote_util);
-
-  // Returns the (local!) arch specific filename of cdc_rsync[.exe].
-  static std::string CdcRsyncFilename();
-
-  ServerArch(ArchType type, bool is_guess);
-  ~ServerArch();
-
-  // Accessor for the arch type.
-  ArchType GetType() const { return type_; }
-
-  // Returns the type as a human readable string.
-  const char* GetTypeStr() const;
-
-  // Returns true if the type was guessed and not detected.
-  bool IsGuess() const { return is_guess_; }
-
-  // Returns the arch-specific filename of cdc_rsync_server[.exe].
-  std::string CdcServerFilename() const;
-
-  // Returns the arch-specific directory where cdc_rsync_server is deployed.
-  std::string RemoteToolsBinDir() const;
-
-  // Returns an arch-specific SSH shell command that gets invoked in order to
-  // start cdc_rsync_server. The command
-  // - returns |exit_code_not_found| if cdc_rsync_server does not exist (to
-  //   prevent the confusing bash output message
-  //     "bash: .../cdc_rsync_server: No such file or directory"), and
-  // - runs the server with the provided |args|.
-  std::string GetStartServerCommand(int exit_code_not_found,
-                                    const std::string& args) const;
-
-  // Returns an arch-specific SFTP command sequence that deploys the server
-  // component on the target gets invoked after
-  // cdc_rsync_server has been copied to a temp location. The commands
-  // - create the cdc-file-transfer/bin folder if it doesn't exist yet,
-  // - make the old cdc_rsync_server writable if it exists,
-  // - copy cdc_rsync_server to a temp location,
-  // - make the new cdc_rsync_server executable (Linux only) and
-  // - replaces the existing cdc_rsync_server by the temp one.
-  std::string GetDeploySftpCommands() const;
-
- private:
-  ArchType type_;
-  bool is_guess_ = false;
-};
-
-}  // namespace cdc_ft
-
-#endif  // CDC_RSYNC_SERVER_ARCH_H_

cdc_rsync/server_arch_test.cc

@@ -1,104 +0,0 @@
-// Copyright 2022 Google LLC
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "cdc_rsync/server_arch.h"
-
-#include "absl/strings/match.h"
-#include "gtest/gtest.h"
-
-namespace cdc_ft {
-namespace {
-
-constexpr auto kLinux = ArchType::kLinux_x86_64;
-constexpr auto kWindows = ArchType::kWindows_x86_64;
-
-constexpr bool kNoGuess = false;
-
-TEST(ServerArchTest, GuessesLinuxIfPathStartsWithSlashOrTilde) {
-  EXPECT_EQ(ServerArch::GuessFromDestination("/linux/path").GetType(), kLinux);
-  EXPECT_EQ(ServerArch::GuessFromDestination("/linux\\path").GetType(), kLinux);
-  EXPECT_EQ(ServerArch::GuessFromDestination("~/linux/path").GetType(), kLinux);
-  EXPECT_EQ(ServerArch::GuessFromDestination("~/linux\\path").GetType(),
-            kLinux);
-  EXPECT_EQ(ServerArch::GuessFromDestination("~\\linux\\path").GetType(),
-            kLinux);
-}
-
-TEST(ServerArchTest, GuessesWindowsIfPathStartsWithDrive) {
-  EXPECT_EQ(ServerArch::GuessFromDestination("C:\\win\\path").GetType(),
-            kWindows);
-  EXPECT_EQ(ServerArch::GuessFromDestination("D:win").GetType(), kWindows);
-  EXPECT_EQ(ServerArch::GuessFromDestination("Z:\\win/path").GetType(),
-            kWindows);
-}
-
-TEST(ServerArchTest, GuessesLinuxIfPathOnlyHasForwardSlashes) {
-  EXPECT_EQ(ServerArch::GuessFromDestination("linux/path").GetType(), kLinux);
-}
-
-TEST(ServerArchTest, GuessesWindowsIfPathOnlyHasBackSlashes) {
-  EXPECT_EQ(ServerArch::GuessFromDestination("\\win\\path").GetType(),
-            kWindows);
-}
-
-TEST(ServerArchTest, GuessesLinuxByDefault) {
-  EXPECT_EQ(ServerArch::GuessFromDestination("/mixed\\path").GetType(), kLinux);
-  EXPECT_EQ(ServerArch::GuessFromDestination("/mixed\\path").GetType(), kLinux);
-  EXPECT_EQ(ServerArch::GuessFromDestination("C\\linux/path").GetType(),
-            kLinux);
-  EXPECT_EQ(ServerArch::GuessFromDestination("").GetType(), kLinux);
-}
-
-TEST(ServerArchTest, IsGuess) {
-  EXPECT_TRUE(ServerArch::GuessFromDestination("foo").IsGuess());
-  EXPECT_FALSE(ServerArch::DetectFromLocalDevice().IsGuess());
-}
-
-TEST(ServerArchTest, CdcServerFilename) {
-  EXPECT_FALSE(absl::StrContains(
-      ServerArch(kLinux, kNoGuess).CdcServerFilename(), "exe"));
-  EXPECT_TRUE(absl::StrContains(
-      ServerArch(kWindows, kNoGuess).CdcServerFilename(), "exe"));
-}
-
-TEST(ServerArchTest, RemoteToolsBinDir) {
-  const std::string linux_dir =
-      ServerArch(kLinux, kNoGuess).RemoteToolsBinDir();
-  EXPECT_TRUE(absl::StrContains(linux_dir, ".cache/"));
-
-  std::string win_dir = ServerArch(kWindows, kNoGuess).RemoteToolsBinDir();
-  EXPECT_TRUE(absl::StrContains(win_dir, "AppData\\Roaming\\"));
-}
-
-TEST(ServerArchTest, GetStartServerCommand) {
-  std::string cmd =
-      ServerArch(kWindows, kNoGuess).GetStartServerCommand(123, "foo bar");
-  EXPECT_TRUE(absl::StrContains(cmd, "123"));
-  EXPECT_TRUE(absl::StrContains(cmd, "cdc_rsync_server.exe foo bar"));
-
-  cmd = ServerArch(kLinux, kNoGuess).GetStartServerCommand(123, "foo bar");
-  EXPECT_TRUE(absl::StrContains(cmd, "123"));
-  EXPECT_TRUE(absl::StrContains(cmd, "cdc_rsync_server foo bar"));
-}
-
-TEST(ServerArchTest, GetDeployReplaceCommand) {
-  std::string cmd = ServerArch(kWindows, kNoGuess).GetDeploySftpCommands();
-  EXPECT_TRUE(absl::StrContains(cmd, "cdc_rsync_server.exe"));
-
-  cmd = ServerArch(kLinux, kNoGuess).GetDeploySftpCommands();
-  EXPECT_TRUE(absl::StrContains(cmd, "cdc_rsync_server"));
-}
-
-}  // namespace
-}  // namespace cdc_ft

cdc_rsync/zstd_stream.cc

@@ -27,19 +27,22 @@ namespace {
 // trigger a flush. This happens when files with no changes are diff'ed (this
 // produces very low volume data). Flushing prevents that the server gets stale
 // and becomes overwhelmed later.
-constexpr absl::Duration kDefaultAutoFlushPeriod = absl::Milliseconds(500);
+constexpr absl::Duration kMinCompressPeriod = absl::Milliseconds(500);
 
 }  // namespace
 
 ZstdStream::ZstdStream(Socket* socket, int level, uint32_t num_threads)
-    : socket_(socket),
+    : socket_(socket), cctx_(nullptr) {
-      cctx_(nullptr),
-      auto_flush_period_(kDefaultAutoFlushPeriod) {
   status_ = WrapStatus(Initialize(level, num_threads),
                        "Failed to initialize stream compressor");
 }
 
 ZstdStream::~ZstdStream() {
+  if (cctx_) {
+    ZSTD_freeCCtx(cctx_);
+    cctx_ = nullptr;
+  }
+
   {
     absl::MutexLock lock(&mutex_);
     shutdown_ = true;
@@ -47,11 +50,6 @@ ZstdStream::~ZstdStream() {
   if (compressor_thread_.joinable()) {
     compressor_thread_.join();
   }
-
-  if (cctx_) {
-    ZSTD_freeCCtx(cctx_);
-    cctx_ = nullptr;
-  }
 }
 
 absl::Status ZstdStream::Write(const void* data, size_t size) {
@@ -81,7 +79,7 @@ absl::Status ZstdStream::Write(const void* data, size_t size) {
   return absl::OkStatus();
 }
 
-absl::Status ZstdStream::Finish() {
+absl::Status ZstdStream::Flush() {
   absl::MutexLock lock(&mutex_);
   if (!status_.ok()) return status_;
 
@@ -136,7 +134,7 @@ void ZstdStream::ThreadCompressorMain() {
              in_buffer_.size() == in_buffer_.capacity();
     };
     bool flush =
-        !mutex_.AwaitWithTimeout(absl::Condition(&cond), auto_flush_period_);
+        !mutex_.AwaitWithTimeout(absl::Condition(&cond), kMinCompressPeriod);
     if (shutdown_) {
       return;
     }
@@ -146,7 +144,7 @@ void ZstdStream::ThreadCompressorMain() {
     const ZSTD_EndDirective mode = last_chunk_ ? ZSTD_e_end
                                    : flush     ? ZSTD_e_flush
                                                : ZSTD_e_continue;
-    LOG_VERBOSE("Compressing %u bytes (mode=%s)", in_buffer_.size(),
+    LOG_DEBUG("Compressing %u bytes (mode=%s)", in_buffer_.size(),
               mode == ZSTD_e_end     ? "end"
               : mode == ZSTD_e_flush ? "flush"
                                      : "continue");

cdc_rsync/zstd_stream.h

@@ -21,8 +21,8 @@
 
 #include "absl/status/status.h"
 #include "absl/synchronization/mutex.h"
+#include "cdc_rsync/base/socket.h"
 #include "common/buffer.h"
-#include "common/socket.h"
 #include "lib/zstd.h"
 
 namespace cdc_ft {
@@ -36,13 +36,8 @@ class ZstdStream {
   // Sends the given |data| to the compressor.
   absl::Status Write(const void* data, size_t size) ABSL_LOCKS_EXCLUDED(mutex_);
 
-  // Finishes the stream and flushes all remaining data.
+  // Flushes all remaining data and sends the compressed data to the socket.
-  absl::Status Finish() ABSL_LOCKS_EXCLUDED(mutex_);
+  absl::Status Flush() ABSL_LOCKS_EXCLUDED(mutex_);
-
-  // Flushes internal buffers if no new data is written for longer than this
-  // time. This makes sure that no data is stuck in the pipeline if no new input
-  // is available. Default is 500 ms.
-  void AutoFlushAfter(absl::Duration dur) { auto_flush_period_ = dur; }
 
  private:
   // Initializes the compressor and related data.
@@ -63,8 +58,6 @@ class ZstdStream {
   bool last_chunk_sent_ ABSL_GUARDED_BY(mutex_) = false;
   absl::Status status_ ABSL_GUARDED_BY(mutex_);
   std::thread compressor_thread_;
-
-  absl::Duration auto_flush_period_;
 };
 
 }  // namespace cdc_ft

cdc_rsync/zstd_stream_test.cc

@@ -14,8 +14,8 @@
 
 #include "cdc_rsync/zstd_stream.h"
 
+#include "cdc_rsync/base/fake_socket.h"
 #include "cdc_rsync_server/unzstd_stream.h"
-#include "common/fake_socket.h"
 #include "common/status_test_macros.h"
 #include "gtest/gtest.h"
 
@@ -32,7 +32,7 @@ class ZstdStreamTest : public ::testing::Test {
 TEST_F(ZstdStreamTest, Small) {
   const std::string want = "Lorem ipsum gibberisulum foobarberis";
   EXPECT_OK(cstream_.Write(want.data(), want.size()));
-  EXPECT_OK(cstream_.Finish());
+  EXPECT_OK(cstream_.Flush());
 
   Buffer buff(1024);
   size_t bytes_read;
@@ -43,52 +43,6 @@ TEST_F(ZstdStreamTest, Small) {
   EXPECT_EQ(got, want);
 }
 
-TEST_F(ZstdStreamTest, AutoFlushesAfterTimeout) {
-  const std::string want = "Lorem ipsum gibberisulum foobarberis";
-  cstream_.AutoFlushAfter(absl::Milliseconds(10));
-  EXPECT_OK(cstream_.Write(want.data(), want.size()));
-  // Note: No flush! cstream_ will compress and send the data after 10 ms.
-
-  // Only read as much data as we have written, or else dstream_.Read() will
-  // expect more data.
-  Buffer buff(want.size());
-  size_t bytes_read;
-  bool eof = false;
-  EXPECT_OK(dstream_.Read(buff.data(), buff.size(), &bytes_read, &eof));
-  EXPECT_FALSE(eof);
-  std::string got(buff.data(), bytes_read);
-  EXPECT_EQ(got, want);
-}
-
-// Regression test for an issue in UnzstdStream, where the reader tried to read
-// from the socket even though the output data was still available in internal
-// buffers.
-TEST_F(ZstdStreamTest, DeliversOutputBeforeReadingNewData) {
-  const std::string want1 = "I want";
-  const std::string want2 = "to eat cookies";
-  cstream_.AutoFlushAfter(absl::Milliseconds(10));
-  EXPECT_OK(cstream_.Write(want1.data(), want1.size()));
-  EXPECT_OK(cstream_.Write(want2.data(), want2.size()));
-  // Note: No flush! cstream_ will compress and send the data after 10 ms.
-
-  Buffer buff1(want1.size());
-  Buffer buff2(want2.size());
-  size_t bytes_read1, bytes_read2;
-  bool eof1 = false, eof2 = false;
-  EXPECT_OK(dstream_.Read(buff1.data(), buff1.size(), &bytes_read1, &eof1));
-
-  // There was a bug in dstream_.Read(), where the method would first try to
-  // read new input before uncompressing data, even though the data was already
-  // present in internal buffers.
-  EXPECT_OK(dstream_.Read(buff2.data(), buff2.size(), &bytes_read2, &eof2));
-  EXPECT_FALSE(eof1);
-  EXPECT_FALSE(eof2);
-  std::string got1(buff1.data(), bytes_read1);
-  std::string got2(buff2.data(), bytes_read2);
-  EXPECT_EQ(got1, want1);
-  EXPECT_EQ(got2, want2);
-}
-
 TEST_F(ZstdStreamTest, Large) {
   Buffer want(1024 * 1024 * 10 + 12345);
   constexpr uint64_t prime = 919393;
@@ -101,7 +55,7 @@ TEST_F(ZstdStreamTest, Large) {
     size_t size = std::min<size_t>(kChunkSize, want.size() - pos);
     EXPECT_OK(cstream_.Write(want.data() + pos, size));
   }
-  EXPECT_OK(cstream_.Finish());
+  EXPECT_OK(cstream_.Flush());
 
   bool eof = false;
   Buffer buff(128 * 1024);

cdc_rsync_server/BUILD

@@ -22,7 +22,7 @@ cc_test(
     srcs = ["file_deleter_and_sender_test.cc"],
     deps = [
         ":file_deleter_and_sender",
-        "//common:fake_socket",
+        "//cdc_rsync/base:fake_socket",
         "//common:status_test_macros",
         "//common:test_main",
         "@com_google_googletest//:gtest",
@@ -95,16 +95,15 @@ cc_binary(
         ":file_diff_generator",
         ":file_finder",
         ":file_info",
+        ":server_socket",
         ":unzstd_stream",
         "//cdc_rsync/base:cdc_interface",
         "//cdc_rsync/base:message_pump",
         "//cdc_rsync/base:server_exit_code",
-        "//common:build_version",
         "//common:clock",
         "//common:gamelet_component",
         "//common:log",
         "//common:path_filter",
-        "//common:server_socket",
         "//common:status",
         "//common:stopwatch",
         "//common:threadpool",
@@ -124,13 +123,26 @@ cc_library(
     hdrs = ["file_info.h"],
 )
 
+cc_library(
+    name = "server_socket",
+    srcs = ["server_socket.cc"],
+    hdrs = ["server_socket.h"],
+    target_compatible_with = ["@platforms//os:linux"],
+    deps = [
+        "//cdc_rsync/base:socket",
+        "//common:log",
+        "//common:status",
+        "@com_google_absl//absl/status",
+    ],
+)
+
 cc_library(
     name = "unzstd_stream",
     srcs = ["unzstd_stream.cc"],
     hdrs = ["unzstd_stream.h"],
     deps = [
         "//cdc_rsync/base:message_pump",
-        "//common:socket",
+        "//cdc_rsync/base:socket",
         "//common:status",
         "@com_github_zstd//:zstd",
         "@com_google_absl//absl/status",

cdc_rsync_server/cdc_rsync_server.cc

@@ -19,12 +19,11 @@
 #include "cdc_rsync/protos/messages.pb.h"
 #include "cdc_rsync_server/file_deleter_and_sender.h"
 #include "cdc_rsync_server/file_finder.h"
+#include "cdc_rsync_server/server_socket.h"
 #include "cdc_rsync_server/unzstd_stream.h"
 #include "common/log.h"
 #include "common/path.h"
-#include "common/server_socket.h"
 #include "common/status.h"
-#include "common/status_macros.h"
 #include "common/stopwatch.h"
 #include "common/threadpool.h"
 #include "common/util.h"
@@ -33,22 +32,9 @@ namespace cdc_ft {
 
 namespace {
 
-// Number of files for which to call fclose() and finalize files in parallel.
-constexpr size_t kNumFinalizerThreads = 8;
-
-// Max 16 files in the patcher and finalizer queues to prevent that too many
-// files are open concurrently.
-constexpr size_t kMaxQueueSize = 16;
-
 // Suffix for the patched file created from the basis file and the diff.
 constexpr char kIntermediatePathSuffix[] = ".__cdc_rsync_temp__";
 
-#if PLATFORM_WINDOWS
-constexpr char kServerFilename[] = "cdc_rsync_server.exe";
-#elif PLATFORM_LINUX
-constexpr char kServerFilename[] = "cdc_rsync_server";
-#endif
-
 uint16_t kExecutableBits =
     path::MODE_IXUSR | path::MODE_IXGRP | path::MODE_IXOTH;
 
@@ -57,9 +43,6 @@ uint16_t kExecutableBits =
 // |target_filepath| match, writes an intermediate file and replaces
 // the file at |target_filepath| with the intermediate file when all data has
 // been received.
-// Each PatchTask is queued twice, once to create the patched file, and once in
-// a different thread pool to close and finalize the patched file. This is
-// because fclose() can take a long time to finish, so it could block patching.
 class PatchTask : public Task {
  public:
   PatchTask(const std::string& base_filepath,
@@ -68,69 +51,38 @@ class PatchTask : public Task {
       : base_filepath_(base_filepath),
         target_filepath_(target_filepath),
         file_(file),
-        cdc_(cdc),
+        cdc_(cdc) {}
-        need_intermediate_file_(target_filepath_ == base_filepath_),
-        patched_filepath_(target_filepath_ == base_filepath_
-                              ? base_filepath_ + kIntermediatePathSuffix
-                              : target_filepath_) {}
 
   virtual ~PatchTask() = default;
 
-  PatchTask(const PatchTask& other) = delete;
-  PatchTask& operator=(const PatchTask& other) = delete;
-
   const ChangedFileInfo& File() const { return file_; }
 
   const absl::Status& Status() const { return status_; }
 
   // Task:
   void ThreadRun(IsCancelledPredicate is_cancelled) override {
-    // Each PatchTask is queued twice, once to apply the patch and once to
+    bool need_intermediate_file = target_filepath_ == base_filepath_;
-    // close and finalize the patched file.
+    std::string patched_filepath =
-    switch (state_) {
+        need_intermediate_file ? base_filepath_ + kIntermediatePathSuffix
-      case State::kPatching:
+                               : target_filepath_;
-        Patch();
-        state_ = State::kFinalizing;
-        break;
-      case State::kFinalizing:
-        Finalize();
-        state_ = State::kDone;
-        break;
-      default:
-        assert(!"Invalid state");
-    }
-  }
 
- private:
+    absl::StatusOr<FILE*> patched_file = path::OpenFile(patched_filepath, "wb");
-  void Patch() {
+    if (!patched_file.ok()) {
-    absl::StatusOr<FILE*> patched_fp = path::OpenFile(patched_filepath_, "wb");
+      status_ = patched_file.status();
-    if (!patched_fp.ok()) {
-      status_ = patched_fp.status();
       return;
     }
-    patched_fp_ = *patched_fp;
 
     // Receive diff stream from server and apply.
-    status_ =
+    bool is_executable = false;
-        cdc_->ReceiveDiffAndPatch(base_filepath_, patched_fp_, &is_executable_);
+    status_ = cdc_->ReceiveDiffAndPatch(base_filepath_, *patched_file,
-
+                                        &is_executable);
-    // The file is closed by Finalize() in a separate thread pool since fclose()
+    fclose(*patched_file);
-    // takes a while on some systems.
-  }
-
-  void Finalize() {
-    if (patched_fp_) {
-      fclose(patched_fp_);
-      patched_fp_ = nullptr;
-    }
-
     if (!status_.ok()) {
-      // Some error occurred during Patch().
       return;
     }
 
     // These bits are OR'ed on top of the mode bits.
-    uint16_t mode_or_bits = is_executable_ ? kExecutableBits : 0;
+    uint16_t mode_or_bits = is_executable ? kExecutableBits : 0;
 
     // Store mode from the original base path.
     path::Stats stats;
@@ -141,13 +93,13 @@ class PatchTask : public Task {
       return;
     }
 
-    if (need_intermediate_file_) {
+    if (need_intermediate_file) {
       // Replace |base_filepath_| (==|target_filepath_|) by the intermediate
       // file |patched_filepath|.
-      status_ = path::ReplaceFile(target_filepath_, patched_filepath_);
+      status_ = path::ReplaceFile(target_filepath_, patched_filepath);
       if (!status_.ok()) {
         status_ = WrapStatus(status_, "ReplaceFile() for '%s' by '%s' failed",
-                             base_filepath_, patched_filepath_);
+                             base_filepath_, patched_filepath);
         return;
       }
     } else {
@@ -168,82 +120,11 @@ class PatchTask : public Task {
     status_ = path::SetFileTime(target_filepath_, file_.client_modified_time);
   }
 
-  const std::string base_filepath_;
-  const std::string target_filepath_;
-  const ChangedFileInfo file_;
-  CdcInterface* const cdc_;
-  const bool need_intermediate_file_ = false;
-  const std::string patched_filepath_;
-
-  FILE* patched_fp_ = nullptr;
-  bool is_executable_ = false;
-  absl::Status status_;
-
-  // This task is queued twice, once to patch and once to close and finalize the
-  // patched file.
-  enum class State { kPatching, kFinalizing, kDone };
-  State state_ = State::kPatching;
-};
-
-// Background task that closes a file and sets the mtime and perms. This is done
-// in the background since closing a file might block for a long time.
-class FinalizeCopiedFileTask : public Task {
- public:
-  // Finalize |file| with given path |filepath|. |status| is the status from
-  // writing the file. On error, the file is only closed.
-  FinalizeCopiedFileTask(FILE* fp, FileInfo file, std::string filepath,
-                         bool is_executable, absl::Status status)
-      : fp_(fp),
-        file_(std::move(file)),
-        filepath_(std::move(filepath)),
-        is_executable_(is_executable),
-        status_(status) {}
-  virtual ~FinalizeCopiedFileTask() = default;
-
-  FinalizeCopiedFileTask(const FinalizeCopiedFileTask& other) = delete;
-  FinalizeCopiedFileTask& operator=(const FinalizeCopiedFileTask& other) =
-      delete;
-
-  const absl::Status& Status() const { return status_; }
-
-  // Task:
-  void ThreadRun(IsCancelledPredicate is_cancelled) override {
-    assert(fp_);
-    fclose(fp_);
-
-    if (!status_.ok()) {
-      // Writing the file failed, nothing to finalize.
-      status_ = WrapStatus(status_, "Failed to write file %s", filepath_);
-      return;
-    }
-
-    // Set file write time.
-    status_ = path::SetFileTime(filepath_, file_.modified_time);
-    if (!status_.ok()) {
-      status_ =
-          WrapStatus(status_, "Failed to set file mod time for %s", filepath_);
-      return;
-    }
-
-    // Set executable bit, but just print warnings as it's not critical.
-    if (is_executable_) {
-      path::Stats stats;
-      status_ = path::GetStats(filepath_, &stats);
-      if (status_.ok()) {
-        status_ = path::ChangeMode(filepath_, stats.mode | kExecutableBits);
-      }
-      if (!status_.ok()) {
-        LOG_WARNING("Failed to set executable bit on '%s': %s", filepath_,
-                    status_.ToString());
-      }
-    }
-  }
-
  private:
-  FILE* const fp_ = nullptr;
+  std::string base_filepath_;
-  const FileInfo file_;
+  std::string target_filepath_;
-  const std::string filepath_;
+  ChangedFileInfo file_;
-  const bool is_executable_;
+  CdcInterface* cdc_;
   absl::Status status_;
 };
 
@@ -267,7 +148,10 @@ PathFilter::Rule::Type ToInternalType(
 
 CdcRsyncServer::CdcRsyncServer() = default;
 
-CdcRsyncServer::~CdcRsyncServer() = default;
+CdcRsyncServer::~CdcRsyncServer() {
+  message_pump_.reset();
+  socket_.reset();
+}
 
 bool CdcRsyncServer::CheckComponents(
     const std::vector<GameletComponent>& components) {
@@ -279,8 +163,8 @@ bool CdcRsyncServer::CheckComponents(
   }
 
   std::vector<GameletComponent> our_components;
-  status = GameletComponent::Get({path::Join(component_dir, kServerFilename)},
+  status = GameletComponent::Get(
-                                 &our_components);
+      {path::Join(component_dir, "cdc_rsync_server")}, &our_components);
   if (!status.ok() || components != our_components) {
     return false;
   }
@@ -288,24 +172,22 @@ bool CdcRsyncServer::CheckComponents(
   return true;
 }
 
-absl::Status CdcRsyncServer::Run() {
+absl::Status CdcRsyncServer::Run(int port) {
-  RETURN_IF_ERROR(Socket::Initialize(), "Failed to initialize sockets");
-  socket_finalizer_ = std::make_unique<SocketFinalizer>();
-
   socket_ = std::make_unique<ServerSocket>();
-  int port;
+  absl::Status status = socket_->StartListening(port);
-  ASSIGN_OR_RETURN(port, socket_->StartListening(0),
+  if (!status.ok()) {
-                   "Failed to start listening for connections");
+    return WrapStatus(status, "Failed to start listening on port %i", port);
+  }
   LOG_INFO("cdc_rsync_server listening on port %i", port);
 
   // This is the marker for the client, so it knows it can connect.
-  // Print port first so the client can easily parse it when it sees "Server is
+  printf("Server is listening\n");
-  // listening" without dealing with half-transmitted data.
-  printf("Port %i: Server is listening\n", port);
   fflush(stdout);
 
-  RETURN_IF_ERROR(socket_->WaitForConnection(),
+  status = socket_->WaitForConnection();
-                  "Failed to establish a connection");
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to establish a connection");
+  }
 
   message_pump_ = std::make_unique<MessagePump>(
       socket_.get(),
@@ -313,7 +195,7 @@ absl::Status CdcRsyncServer::Run() {
   message_pump_->StartMessagePump();
 
   LOG_INFO("Client connected. Starting to sync.");
-  absl::Status status = Sync();
+  status = Sync();
   if (!status.ok()) {
     socket_->ShutdownSendingEnd().IgnoreError();
     return status;
@@ -602,7 +484,7 @@ absl::Status CdcRsyncServer::CreateMissingDirs() {
 template <typename T>
 absl::Status CdcRsyncServer::SendFileIndices(const char* file_type,
                                              const std::vector<T>& files) {
-  LOG_INFO("Sending indices of %s files to client", file_type);
+  LOG_INFO("Sending indices of missing files to client");
   constexpr char error_fmt[] = "Failed to send indices of %s files.";
 
   AddFileIndicesResponse response;
@@ -659,8 +541,6 @@ absl::Status CdcRsyncServer::HandleSendMissingFileData() {
     }
   }
 
-  Threadpool finalize_pool(kNumFinalizerThreads);
-
   for (uint32_t server_index = 0; server_index < diff_.missing_files.size();
        server_index++) {
     const FileInfo& file = diff_.missing_files[server_index];
@@ -680,10 +560,10 @@ absl::Status CdcRsyncServer::HandleSendMissingFileData() {
                         request.server_index(), server_index);
     }
 
-    // Remove |filepath| if it is a directory.
+    // Verify that there is no directory existing with the same name.
-    if (path::DirExists(filepath)) {
+    if (path::Exists(filepath) && path::DirExists(filepath)) {
       assert(!diff_.extraneous_dirs.empty());
-      status = path::RemoveFile(filepath);
+      absl::Status status = path::RemoveFile(filepath);
       if (!status.ok()) {
         return WrapStatus(
             status, "Failed to remove folder '%s' before creating file '%s'",
@@ -730,25 +610,27 @@ absl::Status CdcRsyncServer::HandleSendMissingFileData() {
     }
 
     status = path::StreamWriteFileContents(*fp, handler);
-    finalize_pool.QueueTask(std::make_unique<FinalizeCopiedFileTask>(
+    fclose(*fp);
-        *fp, file, filepath, is_executable, status));
+    if (!status.ok()) {
-    finalize_pool.WaitForQueuedTasksAtMost(kMaxQueueSize);
+      return WrapStatus(status, "Failed to write file %s", filepath);
-
-    // Drain finalize pool for the last file.
-    if (server_index + 1 == diff_.missing_files.size()) {
-      finalize_pool.Wait();
     }
 
-    // Check the results of completed tasks.
+    // Set file write time.
-    for (std::unique_ptr<Task> task = finalize_pool.TryGetCompletedTask();
+    status = path::SetFileTime(filepath, file.modified_time);
-         task != nullptr; task = finalize_pool.TryGetCompletedTask()) {
+    if (!status.ok()) {
-      const FinalizeCopiedFileTask* finalize_task =
+      return WrapStatus(status, "Failed to set file mod time for %s", filepath);
-          static_cast<FinalizeCopiedFileTask*>(task.get());
+    }
-      if (!finalize_task->Status().ok()) {
+
-        // Close and finish files that have already been copied, so we don't
+    // Set executable bit, but just print warnings as it's not critical.
-        // discard several already copied files because one failed.
+    if (is_executable) {
-        finalize_pool.Wait();
+      path::Stats stats;
-        return finalize_task->Status();
+      status = path::GetStats(filepath, &stats);
+      if (status.ok()) {
+        status = path::ChangeMode(filepath, stats.mode | kExecutableBits);
+      }
+      if (!status.ok()) {
+        LOG_WARNING("Failed to set executable bit on '%s': %s", filepath,
+                    status.ToString());
       }
     }
   }
@@ -789,20 +671,9 @@ absl::Status CdcRsyncServer::SyncChangedFiles() {
   // Pipeline sending signatures and patching files:
   // MAIN THREAD:   Send signatures to client.
   //                Only sends to the socket.
-  // PATCHER THREAD:    Receive diffs from client and create patch file.
+  // WORKER THREAD: Receive diffs from client and patch file.
   //                Only reads from the socket.
-  // FINALIZER THREADS: Close patched files and finalize them.
+  Threadpool pool(1);
-  Threadpool patch_pool(1);
-  Threadpool finalize_pool(kNumFinalizerThreads);
-
-  // Forward finished patch task immediately to finalize pool.
-  patch_pool.SetTaskCompletedCallback(
-      [&finalize_pool](std::unique_ptr<Task> task) {
-        // Spin if there are too many outstanding tasks, in order to limit the
-        // max number of outstanding tasks.
-        finalize_pool.QueueTask(std::move(task));
-        finalize_pool.WaitForQueuedTasksAtMost(kMaxQueueSize);
-      });
 
   for (uint32_t server_index = 0; server_index < diff_.changed_files.size();
        server_index++) {
@@ -828,28 +699,25 @@ absl::Status CdcRsyncServer::SyncChangedFiles() {
     }
 
     // Queue patching task.
-    patch_pool.QueueTask(std::make_unique<PatchTask>(
+    pool.QueueTask(std::make_unique<PatchTask>(base_filepath, target_filepath,
-        base_filepath, target_filepath, file, &cdc));
+                                               file, &cdc));
 
-    // Drain pools for the last file.
+    // Wait for the last file to finish.
     if (server_index + 1 == diff_.changed_files.size()) {
-      patch_pool.Wait();
+      pool.Wait();
-      finalize_pool.Wait();
     }
 
     // Check the results of completed tasks.
-    for (std::unique_ptr<Task> task = finalize_pool.TryGetCompletedTask();
+    std::unique_ptr<Task> task = pool.TryGetCompletedTask();
-         task != nullptr; task = finalize_pool.TryGetCompletedTask()) {
+    while (task) {
-      const PatchTask* patch_task = static_cast<PatchTask*>(task.get());
+      PatchTask* patch_task = static_cast<PatchTask*>(task.get());
       const std::string& task_path = patch_task->File().filepath;
       if (!patch_task->Status().ok()) {
-        // Close and finish files that have already been synced, so we don't
-        // discard several already synced files because one failed.
-        finalize_pool.Wait();
         return WrapStatus(patch_task->Status(), "Failed to patch file '%s'",
                           task_path);
       }
       LOG_INFO("Finished patching file %s", task_path.c_str());
+      task = pool.TryGetCompletedTask();
     }
   }
 

cdc_rsync_server/cdc_rsync_server.h

@@ -32,7 +32,6 @@ namespace cdc_ft {
 
 class MessagePump;
 class ServerSocket;
-class SocketFinalizer;
 
 class CdcRsyncServer {
  public:
@@ -43,10 +42,9 @@ class CdcRsyncServer {
   // up-to-date by checking their sizes and timestamps.
   bool CheckComponents(const std::vector<GameletComponent>& components);
 
-  // Listens to any available port, accepts a connection from the client and
+  // Listens to |port|, accepts a connection from the client and runs the rsync
-  // runs the rsync procedure. Prints "Port <n>: Server is listening" to stdout,
+  // procedure.
-  // so the client can retrieve the selected port.
+  absl::Status Run(int port);
-  absl::Status Run();
 
   // Returns the verbosity sent from the client. 0 by default.
   int GetVerbosity() const { return verbosity_; }
@@ -92,8 +90,6 @@ class CdcRsyncServer {
   // Used to toggle decompression.
   void Thread_OnPackageReceived(PacketType type);
 
-  // The order determines the correct destruction order, so keep it!
-  std::unique_ptr<SocketFinalizer> socket_finalizer_;
   std::unique_ptr<ServerSocket> socket_;
   std::unique_ptr<MessagePump> message_pump_;
 

cdc_rsync_server/cdc_rsync_server.vcxproj

@@ -49,7 +49,7 @@
   <PropertyGroup>
     <BazelTargets>//cdc_rsync_server:cdc_rsync_server</BazelTargets>
     <BazelOutputFile>cdc_rsync_server</BazelOutputFile>
-    <BazelIncludePaths>..\;..\third_party\absl;..\bazel-cdc-file-transfer\external\com_github_blake3\c;..\bazel-cdc-file-transfer\external\com_github_zstd;..\third_party\googletest\googletest\include;..\bazel-cdc-file-transfer\external\com_google_protobuf\src;..\bazel-bin</BazelIncludePaths>
+    <BazelIncludePaths>..\;..\third_party\absl;..\bazel-cdc-file-transfer\external\com_github_blake3\c;..\bazel-stadia-file-transfer\external\com_github_zstd\lib;..\third_party\googletest\googletest\include;..\bazel-cdc-file-transfer\external\com_google_protobuf\src</BazelIncludePaths>
   </PropertyGroup>
   <Import Project="..\NMakeBazelProject.targets" />
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />

cdc_rsync_server/file_deleter_and_sender_test.cc

@@ -14,8 +14,8 @@
 
 #include "cdc_rsync_server/file_deleter_and_sender.h"
 
+#include "cdc_rsync/base/fake_socket.h"
 #include "cdc_rsync/base/message_pump.h"
-#include "common/fake_socket.h"
 #include "common/log.h"
 #include "common/path.h"
 #include "common/status_test_macros.h"

cdc_rsync_server/main.cc

@@ -14,7 +14,6 @@
 
 #include "cdc_rsync/base/server_exit_code.h"
 #include "cdc_rsync_server/cdc_rsync_server.h"
-#include "common/build_version.h"
 #include "common/gamelet_component.h"
 #include "common/log.h"
 #include "common/status.h"
@@ -61,33 +60,33 @@ ServerExitCode GetExitCode(const absl::Status& status) {
 }  // namespace cdc_ft
 
 int main(int argc, const char** argv) {
-  if (argc < 5) {
+  if (argc < 2) {
-    printf(R"("cdc_rsync_server - Remote component of cdc_rsync. Version: %s
+    printf("Usage: cdc_rsync_server <port> cdc_rsync_server <size> <time> \n");
-
+    printf("       where <size> and <time> are the file size and modified\n");
-Usage: cdc_rsync_server <build_version> cdc_rsync_server <size> <modified_time>
+    printf("       timestamp (Unix epoch) of the corresponding component.\n");
-       <build_version>            build version embedded in the component
+    return cdc_ft::kServerExitCodeGenericStartup;
-       <size>                     file size of the component
+  }
-       <modified_time>            timestamp (Unix epoch) of the component)",
-           BUILD_VERSION);
 
+  int port = atoi(argv[1]);
+  if (port == 0) {
+    SendErrorMessage(absl::StrFormat("Invalid port '%s'", argv[1]).c_str());
     return cdc_ft::kServerExitCodeGenericStartup;
   }
 
   // The rest is expected to be sets of gamelet component info consisting of
-  // (version, filename, size, modified_time). This is used check whether the
+  // (filename, filesize, modified_time). This is used check whether the
   // components are up-to-date.
   std::vector<cdc_ft::GameletComponent> components =
-      cdc_ft::GameletComponent::FromCommandLineArgs(argc - 1, argv + 1);
+      cdc_ft::GameletComponent::FromCommandLineArgs(argc - 2, argv + 2);
 
   cdc_ft::Log::Initialize(
       std::make_unique<cdc_ft::ConsoleLog>(cdc_ft::LogLevel::kWarning));
   cdc_ft::CdcRsyncServer server;
-
   if (!server.CheckComponents(components)) {
     return cdc_ft::kServerExitCodeOutOfDate;
   }
 
-  absl::Status status = server.Run();
+  absl::Status status = server.Run(port);
   if (status.ok()) {
     return 0;
   }

cdc_rsync_server/server_socket.cc

@@ -0,0 +1,214 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "cdc_rsync_server/server_socket.h"
+
+#include <netinet/in.h>
+#include <sys/socket.h>
+#include <unistd.h>
+
+#include <cerrno>
+
+#include "common/log.h"
+#include "common/status.h"
+
+namespace cdc_ft {
+
+namespace {
+
+int kInvalidFd = -1;
+
+// Keep re-evaluating the expression |x| while it returns EINTR.
+#define HANDLE_EINTR(x)                                     \
+  ({                                                        \
+    decltype(x) eintr_wrapper_result;                       \
+    do {                                                    \
+      eintr_wrapper_result = (x);                           \
+    } while (eintr_wrapper_result == -1 && errno == EINTR); \
+    eintr_wrapper_result;                                   \
+  })
+
+}  // namespace
+
+ServerSocket::ServerSocket()
+    : Socket(), listen_sockfd_(kInvalidFd), conn_sockfd_(kInvalidFd) {}
+
+ServerSocket::~ServerSocket() {
+  Disconnect();
+  StopListening();
+}
+
+absl::Status ServerSocket::StartListening(int port) {
+  if (listen_sockfd_ != kInvalidFd) {
+    return MakeStatus("Already listening");
+  }
+
+  LOG_DEBUG("Open socket");
+  listen_sockfd_ = socket(AF_INET, SOCK_STREAM, 0);
+  if (listen_sockfd_ < 0) {
+    listen_sockfd_ = kInvalidFd;
+    return MakeStatus("socket() failed: %s", strerror(errno));
+  }
+
+  // If the program terminates abnormally, the socket might remain in a
+  // TIME_WAIT state and report "address already in use" on bind(). Setting
+  // SO_REUSEADDR works around that. See
+  // https://hea-www.harvard.edu/~fine/Tech/addrinuse.html
+  int enable = 1;
+  if (setsockopt(listen_sockfd_, SOL_SOCKET, SO_REUSEADDR, &enable,
+                 sizeof(enable)) < 0) {
+    LOG_DEBUG("setsockopt() failed");
+  }
+
+  LOG_DEBUG("Bind socket");
+  sockaddr_in serv_addr;
+  memset(&serv_addr, 0, sizeof(serv_addr));
+  serv_addr.sin_family = AF_INET;
+  serv_addr.sin_addr.s_addr = INADDR_ANY;
+  serv_addr.sin_port = htons(port);
+  if (bind(listen_sockfd_, (struct sockaddr*)&serv_addr, sizeof(serv_addr)) <
+      0) {
+    absl::Status status =
+        MakeStatus("bind() to port %i failed: %s", port, strerror(errno));
+    if (errno == EADDRINUSE) {
+      // Happens when two instances are run at the same time. Help callers to
+      // print reasonable errors.
+      status = SetTag(status, Tag::kAddressInUse);
+    }
+    close(listen_sockfd_);
+    listen_sockfd_ = kInvalidFd;
+
+    return status;
+  }
+
+  LOG_DEBUG("Listen");
+  listen(listen_sockfd_, 1);
+  return absl::OkStatus();
+}
+
+void ServerSocket::StopListening() {
+  if (listen_sockfd_ != kInvalidFd) {
+    close(listen_sockfd_);
+    listen_sockfd_ = kInvalidFd;
+  }
+
+  LOG_INFO("Stopped listening.");
+}
+
+absl::Status ServerSocket::WaitForConnection() {
+  if (conn_sockfd_ != kInvalidFd) {
+    return MakeStatus("Already connected");
+  }
+
+  sockaddr_in cli_addr;
+  socklen_t cli_len = sizeof(cli_addr);
+  conn_sockfd_ = accept(listen_sockfd_, (struct sockaddr*)&cli_addr, &cli_len);
+  if (conn_sockfd_ < 0) {
+    conn_sockfd_ = kInvalidFd;
+    return MakeStatus("accept() failed: %s", strerror(errno));
+  }
+
+  LOG_DEBUG("Client connected");
+  return absl::OkStatus();
+}
+
+void ServerSocket::Disconnect() {
+  if (conn_sockfd_ != kInvalidFd) {
+    close(conn_sockfd_);
+    conn_sockfd_ = kInvalidFd;
+  }
+
+  LOG_INFO("Disconnected");
+}
+
+absl::Status ServerSocket::ShutdownSendingEnd() {
+  int result = shutdown(conn_sockfd_, SHUT_WR);
+  if (result != 0) {
+    return MakeStatus("shutdown() failed: %s", strerror(errno));
+  }
+
+  return absl::OkStatus();
+}
+
+absl::Status ServerSocket::Send(const void* buffer, size_t size) {
+  const uint8_t* curr_ptr = reinterpret_cast<const uint8_t*>(buffer);
+  ssize_t bytes_left = size;
+  while (bytes_left > 0) {
+    ssize_t bytes_written =
+        HANDLE_EINTR(send(conn_sockfd_, curr_ptr, bytes_left, /*flags*/ 0));
+
+    if (bytes_written < 0) {
+      if (errno == EAGAIN || errno == EWOULDBLOCK) {
+        // Shouldn't happen as the socket should be blocking.
+        LOG_DEBUG("Socket would block");
+        continue;
+      }
+
+      return MakeStatus("write() to fd %i failed: %s", conn_sockfd_,
+                        strerror(errno));
+    }
+
+    bytes_left -= bytes_written;
+    curr_ptr += bytes_written;
+  }
+  return absl::OkStatus();
+}
+
+absl::Status ServerSocket::Receive(void* buffer, size_t size,
+                                   bool allow_partial_read,
+                                   size_t* bytes_received) {
+  *bytes_received = 0;
+  if (size == 0) {
+    return absl::OkStatus();
+  }
+
+  uint8_t* curr_ptr = reinterpret_cast<uint8_t*>(buffer);
+  ssize_t bytes_left = size;
+  while (bytes_left > 0) {
+    ssize_t bytes_read =
+        HANDLE_EINTR(recv(conn_sockfd_, curr_ptr, bytes_left, /*flags*/ 0));
+
+    if (bytes_read < 0) {
+      if (errno == EAGAIN || errno == EWOULDBLOCK) {
+        // Shouldn't happen as the socket should be blocking.
+        LOG_DEBUG("Socket would block");
+        continue;
+      }
+
+      return MakeStatus("recv() from fd %i failed: %s", conn_sockfd_,
+                        strerror(errno));
+    }
+
+    bytes_left -= bytes_read;
+    *bytes_received += bytes_read;
+    curr_ptr += bytes_read;
+
+    if (bytes_read == 0) {
+      // EOF. Make sure we're not in the middle of a message.
+      if (bytes_left < static_cast<ssize_t>(size)) {
+        return MakeStatus("EOF after partial read");
+      }
+
+      LOG_DEBUG("EOF() detected");
+      return SetTag(MakeStatus("EOF detected"), Tag::kSocketEof);
+    }
+
+    if (allow_partial_read) {
+      break;
+    }
+  }
+  return absl::OkStatus();
+}
+
+}  // namespace cdc_ft

cdc_rsync_server/server_socket.h

@@ -14,14 +14,11 @@
  * limitations under the License.
  */
 
-#ifndef COMMON_SERVER_SOCKET_H_
+#ifndef CDC_RSYNC_SERVER_SERVER_SOCKET_H_
-#define COMMON_SERVER_SOCKET_H_
+#define CDC_RSYNC_SERVER_SERVER_SOCKET_H_
 
 #include "absl/status/status.h"
-#include "absl/status/statusor.h"
+#include "cdc_rsync/base/socket.h"
-#include "common/socket.h"
-
-struct addrinfo;
 
 namespace cdc_ft {
 
@@ -30,19 +27,8 @@ class ServerSocket : public Socket {
   ServerSocket();
   ~ServerSocket();
 
-  // Returns an available ephemeral port that can be used as a listening port.
-  // Note that calling this function, followed by StartListening() or similar,
-  // is slightly racy as another process might use the port in the meantime.
-  // However, the OS usually returns ephemeral ports in a round-robin manner,
-  // and ports remain in TIME_WAIT state for a while, which may block other apps
-  // from reusing the port. Hence, the chances of races are small. Nevertheless,
-  // consider calling StartListening() with zero |port| if possible.
-  static absl::StatusOr<int> FindAvailablePort();
-
   // Starts listening for connections on |port|.
-  // Passing 0 as port will bind to any available port.
+  absl::Status StartListening(int port);
-  // Returns the port that was bound to.
-  absl::StatusOr<int> StartListening(int port);
 
   // Stops listening for connections. No-op if already stopped/never started.
   void StopListening();
@@ -64,12 +50,11 @@ class ServerSocket : public Socket {
                        size_t* bytes_received) override;
 
  private:
-  // Called by StartListening() for a specific IPV4 or IPV6 |addr_info|.
+  // Listening socket file descriptor (where new connections are accepted).
-  // Passing 0 as port will bind to any available port.
+  int listen_sockfd_;
-  // Returns the port that was bound to.
-  absl::StatusOr<int> StartListeningInternal(int port, addrinfo* addr);
 
-  std::unique_ptr<struct ServerSocketInfo> socket_info_;
+  // Connection socket file descriptor (where data is sent to/received from).
+  int conn_sockfd_;
 };
 
 }  // namespace cdc_ft

cdc_rsync_server/unzstd_stream.cc

@@ -14,7 +14,7 @@
 
 #include "cdc_rsync_server/unzstd_stream.h"
 
-#include "common/socket.h"
+#include "cdc_rsync/base/socket.h"
 #include "common/status.h"
 
 namespace cdc_ft {
@@ -41,20 +41,7 @@ absl::Status UnzstdStream::Read(void* out_buffer, size_t out_size,
 
   ZSTD_outBuffer output = {out_buffer, out_size, 0};
   while (output.pos < output.size && !*eof) {
-    // Decompress.
+    if (input_.pos == input_.size) {
-    size_t ret = ZSTD_decompressStream(dctx_, &output, &input_);
-    if (ZSTD_isError(ret)) {
-      return MakeStatus("Failed to decompress data: %s",
-                        ZSTD_getErrorName(ret));
-    }
-
-    *eof = (ret == 0);
-    if (*eof && input_.pos < input_.size) {
-      return MakeStatus("EOF with %u bytes input data available",
-                        input_.size - input_.pos);
-    }
-
-    if (input_.pos == input_.size && output.pos < output.size && !*eof) {
       // Read more compressed input data.
       // Allow partial reads since the stream could end any time.
       size_t in_size;
@@ -67,6 +54,19 @@ absl::Status UnzstdStream::Read(void* out_buffer, size_t out_size,
       input_.pos = 0;
       input_.size = in_size;
     }
+
+    // Decompress.
+    size_t ret = ZSTD_decompressStream(dctx_, &output, &input_);
+    if (ZSTD_isError(ret)) {
+      return MakeStatus("Failed to decompress data: %s",
+                        ZSTD_getErrorName(ret));
+    }
+
+    *eof = (ret == 0);
+    if (*eof && input_.pos < input_.size) {
+      return MakeStatus("EOF with %u bytes input data available",
+                        input_.size - input_.pos);
+    }
   }
 
   // Output buffer is full or eof.

cdc_stream/BUILD

@@ -12,7 +12,6 @@ cc_binary(
         ":start_command",
         ":start_service_command",
         ":stop_command",
-        ":stop_service_command",
         "//common:log",
         "//common:path",
     ],
@@ -41,24 +40,11 @@ cc_library(
     ],
 )
 
-cc_library(
-    name = "stop_service_command",
-    srcs = ["stop_service_command.cc"],
-    hdrs = ["stop_service_command.h"],
-    deps = [
-        ":asset_stream_config",
-        ":background_service_client",
-        ":base_command",
-        ":session_management_server",
-    ],
-)
-
 cc_library(
     name = "start_command",
     srcs = ["start_command.cc"],
     hdrs = ["start_command.h"],
     deps = [
-        ":background_service_client",
         ":base_command",
         ":local_assets_stream_manager_client",
         ":session_management_server",
@@ -92,18 +78,6 @@ cc_library(
     ],
 )
 
-cc_library(
-    name = "background_service_client",
-    srcs = ["background_service_client.cc"],
-    hdrs = ["background_service_client.h"],
-    deps = [
-        "//common:grpc_status",
-        "//common:status_macros",
-        "//proto:background_service_grpc_proto",
-        "@com_google_absl//absl/status",
-    ],
-)
-
 cc_library(
     name = "asset_stream_server",
     srcs = [
@@ -138,8 +112,6 @@ cc_library(
     deps = [
         ":base_command",
         ":multi_session",
-        ":session_management_server",
-        "//absl_helper:jedec_size_flag",
         "//common:log",
         "//common:path",
         "//common:status_macros",
@@ -208,11 +180,10 @@ cc_library(
         "//common:file_watcher",
         "//common:log",
         "//common:path",
-        "//common:path_filter",
+        "//common:port_manager",
         "//common:process",
         "//common:remote_util",
         "//common:sdk_util",
-        "//common:server_socket",
         "//common:status_macros",
         "//common:stopwatch",
         "//data_store:disk_data_store",

cdc_stream/asset_stream_config.cc

@@ -20,8 +20,6 @@
 #include "absl/strings/str_join.h"
 #include "absl_helper/jedec_size_flag.h"
 #include "cdc_stream/base_command.h"
-#include "cdc_stream/multi_session.h"
-#include "cdc_stream/session_management_server.h"
 #include "common/buffer.h"
 #include "common/path.h"
 #include "common/status_macros.h"
@@ -43,18 +41,6 @@ AssetStreamConfig::~AssetStreamConfig() = default;
 
 void AssetStreamConfig::RegisterCommandLineFlags(lyra::command& cmd,
                                                  BaseCommand& base_command) {
-  service_port_ = SessionManagementServer::kDefaultServicePort;
-  cmd.add_argument(lyra::opt(service_port_, "port")
-                       .name("--service-port")
-                       .help("Local port to use while connecting to the local "
-                             "asset stream service, default: " +
-                             std::to_string(service_port_)));
-
-  cmd.add_argument(lyra::opt(session_cfg_.deprecated_forward_port_range, "port")
-                       .name("--forward-port")
-                       .help("[Deprecated, ignored] TCP port or range used for "
-                             "SSH port forwarding"));
-
   session_cfg_.verbosity = kDefaultVerbosity;
   cmd.add_argument(lyra::opt(session_cfg_.verbosity, "num")
                        .name("--verbosity")
@@ -141,6 +127,14 @@ void AssetStreamConfig::RegisterCommandLineFlags(lyra::command& cmd,
           .name("--dev-user-host")
           .help("Username and host to stream to. See also --dev-src-dir."));
 
+  dev_target_.ssh_port = RemoteUtil::kDefaultSshPort;
+  cmd.add_argument(
+      lyra::opt(dev_target_.ssh_port, "port")
+          .name("--dev-ssh-port")
+          .help("SSH port to use for the connection to the host, default: " +
+                std::to_string(RemoteUtil::kDefaultSshPort) +
+                ". See also --dev-src-dir."));
+
   cmd.add_argument(
       lyra::opt(dev_target_.ssh_command, "cmd")
           .name("--dev-ssh-command")
@@ -148,9 +142,9 @@ void AssetStreamConfig::RegisterCommandLineFlags(lyra::command& cmd,
                 "connection to the host. See also --dev-src-dir."));
 
   cmd.add_argument(
-      lyra::opt(dev_target_.sftp_command, "cmd")
+      lyra::opt(dev_target_.scp_command, "cmd")
-          .name("--dev-sftp-command")
+          .name("--dev-scp-command")
-          .help("Sftp command and extra flags to use for the "
+          .help("Scp command and extra flags to use for the "
                 "connection to the host. See also --dev-src-dir."));
 
   cmd.add_argument(
@@ -180,7 +174,6 @@ absl::Status AssetStreamConfig::LoadFromFile(const std::string& path) {
     }                                     \
   } while (0)
 
-  ASSIGN_VAR(service_port_, "service-port", Int);
   ASSIGN_VAR(session_cfg_.verbosity, "verbosity", Int);
   ASSIGN_VAR(session_cfg_.fuse_debug, "debug", Bool);
   ASSIGN_VAR(session_cfg_.fuse_singlethreaded, "singlethreaded", Bool);
@@ -219,7 +212,6 @@ absl::Status AssetStreamConfig::LoadFromFile(const std::string& path) {
 
 std::string AssetStreamConfig::ToString() {
   std::ostringstream ss;
-  ss << "service-port                 = " << service_port_ << std::endl;
   ss << "verbosity                    = " << session_cfg_.verbosity
      << std::endl;
   ss << "debug                        = " << session_cfg_.fuse_debug
@@ -243,9 +235,10 @@ std::string AssetStreamConfig::ToString() {
      << session_cfg_.file_change_wait_duration_ms << std::endl;
   ss << "dev-src-dir                  = " << dev_src_dir_ << std::endl;
   ss << "dev-user-host                = " << dev_target_.user_host << std::endl;
+  ss << "dev-ssh-port                 = " << dev_target_.ssh_port << std::endl;
   ss << "dev-ssh-command              = " << dev_target_.ssh_command
      << std::endl;
-  ss << "dev-sftp-command             = " << dev_target_.sftp_command
+  ss << "dev-scp-command              = " << dev_target_.scp_command
      << std::endl;
   ss << "dev-mount-dir                = " << dev_target_.mount_dir << std::endl;
   return ss.str();

cdc_stream/asset_stream_config.h

@@ -48,21 +48,18 @@ class AssetStreamConfig {
   // Loads a configuration from the JSON file at |path| and overrides any config
   // values that are set in this file. Sample json file:
   // {
-  //   "service-port":44432
-  //   "forward-port-first":"44433"
-  //   "forward-port-last":"44442"
   //   "verbosity":3,
   //   "debug":0,
   //   "singlethreaded":0,
   //   "stats":0,
   //   "quiet":0,
   //   "check":0,
-  //   "log-to-stdout":0,
+  //   "log_to_stdout":0,
-  //   "cache-capacity":"150G",
+  //   "cache_capacity":"150G",
-  //   "cleanup-timeout":300,
+  //   "cleanup_timeout":300,
-  //   "access-idle-timeout":5,
+  //   "access_idle_timeout":5,
-  //   "manifest-updater-threads":4,
+  //   "manifest_updater_threads":4,
-  //   "file-change-wait-duration-ms":500
+  //   "file_change_wait_duration_ms":500
   // }
   // Returns NotFoundError if the file does not exist.
   // Returns InvalidArgumentError if the file is not valid JSON.
@@ -79,9 +76,6 @@ class AssetStreamConfig {
   // read from the JSON file.
   std::string GetFlagReadErrors();
 
-  // Gets the port to use for the asset streaming service.
-  uint16_t service_port() const { return service_port_; }
-
   // Session configuration.
   const SessionConfig& session_cfg() const { return session_cfg_; }
 
@@ -97,13 +91,6 @@ class AssetStreamConfig {
   bool log_to_stdout() const { return log_to_stdout_; }
 
  private:
-  // Jedec parser for Lyra options. Usage:
-  //   lyra::opt(JedecParser("size-flag", &size_bytes), "bytes"))
-  // Sets jedec_parse_error_ on error, Lyra doesn't support errors from lambdas.
-  std::function<void(const std::string&)> JedecParser(const char* flag_name,
-                                                      uint64_t* bytes);
-
-  uint16_t service_port_ = 0;
   SessionConfig session_cfg_;
   bool log_to_stdout_ = false;
 

cdc_stream/background_service_client.cc

@@ -1,56 +0,0 @@
-// Copyright 2022 Google LLC
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "cdc_stream/background_service_client.h"
-
-#include "absl/status/status.h"
-#include "common/grpc_status.h"
-#include "common/status_macros.h"
-#include "grpcpp/channel.h"
-
-namespace cdc_ft {
-
-using GetPidResponse = backgroundservice::GetPidResponse;
-using EmptyProto = google::protobuf::Empty;
-
-BackgroundServiceClient::BackgroundServiceClient(
-    std::shared_ptr<grpc::Channel> channel) {
-  stub_ = BackgroundService::NewStub(std::move(channel));
-}
-
-BackgroundServiceClient::~BackgroundServiceClient() = default;
-
-absl::Status BackgroundServiceClient::Exit() {
-  EmptyProto request;
-  EmptyProto response;
-  grpc::ClientContext context;
-  return ToAbslStatus(stub_->Exit(&context, request, &response));
-}
-
-absl::StatusOr<int> BackgroundServiceClient::GetPid() {
-  EmptyProto request;
-  GetPidResponse response;
-  grpc::ClientContext context;
-  RETURN_IF_ERROR(ToAbslStatus(stub_->GetPid(&context, request, &response)));
-  return response.pid();
-}
-
-absl::Status BackgroundServiceClient::IsHealthy() {
-  EmptyProto request;
-  EmptyProto response;
-  grpc::ClientContext context;
-  return ToAbslStatus(stub_->HealthCheck(&context, request, &response));
-}
-
-}  // namespace cdc_ft

cdc_stream/background_service_client.h

@@ -1,56 +0,0 @@
-/*
- * Copyright 2022 Google LLC
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef CDC_STREAM_BACKGROUND_SERVICE_CLIENT_H_
-#define CDC_STREAM_BACKGROUND_SERVICE_CLIENT_H_
-
-#include <memory>
-
-#include "absl/status/status.h"
-#include "absl/status/statusor.h"
-#include "proto/background_service.grpc.pb.h"
-
-namespace grpc_impl {
-class Channel;
-}
-
-namespace cdc_ft {
-
-// gRpc client for managing the asset streaming service.
-class BackgroundServiceClient {
- public:
-  // |channel| is a grpc channel to use.
-  explicit BackgroundServiceClient(std::shared_ptr<grpc::Channel> channel);
-
-  ~BackgroundServiceClient();
-
-  // Initialize service shutdown.
-  absl::Status Exit();
-
-  // Returns the PID of the service process.
-  absl::StatusOr<int> GetPid();
-
-  // Verifies that the service is running and able to take requests.
-  absl::Status IsHealthy();
-
- private:
-  using BackgroundService = backgroundservice::BackgroundService;
-  std::unique_ptr<BackgroundService::Stub> stub_;
-};
-
-}  // namespace cdc_ft
-
-#endif  // CDC_STREAM_BACKGROUND_SERVICE_CLIENT_H_

cdc_stream/background_service_impl.cc

@@ -23,32 +23,24 @@ namespace cdc_ft {
 
 BackgroundServiceImpl::BackgroundServiceImpl() {}
 
-BackgroundServiceImpl::~BackgroundServiceImpl() {
+BackgroundServiceImpl::~BackgroundServiceImpl() = default;
-  if (exit_thread_) {
-    exit_thread_->join();
-    exit_thread_.reset();
-  }
-}
 
 void BackgroundServiceImpl::SetExitCallback(ExitCallback exit_callback) {
   exit_callback_ = std::move(exit_callback);
 }
 
 grpc::Status BackgroundServiceImpl::Exit(grpc::ServerContext* context,
-                                         const EmptyProto* request,
+                                         const ExitRequest* request,
-                                         EmptyProto* response) {
+                                         ExitResponse* response) {
   LOG_INFO("RPC:Exit");
-  if (exit_callback_ && !exit_thread_) {
+  if (exit_callback_) {
-    // Fire up a thread so call the callback, since shutting down a server
+    return ToGrpcStatus(exit_callback_());
-    // won't finish until all RPCs are done.
-    exit_thread_ =
-        std::make_unique<std::thread>([cb = &exit_callback_]() { (*cb)(); });
   }
   return grpc::Status::OK;
 }
 
 grpc::Status BackgroundServiceImpl::GetPid(grpc::ServerContext* context,
-                                           const EmptyProto* request,
+                                           const GetPidRequest* request,
                                            GetPidResponse* response) {
   LOG_INFO("RPC:GetPid");
   response->set_pid(static_cast<int32_t>(Util::GetPid()));

cdc_stream/background_service_impl.h

@@ -17,9 +17,6 @@
 #ifndef CDC_STREAM_BACKGROUND_SERVICE_IMPL_H_
 #define CDC_STREAM_BACKGROUND_SERVICE_IMPL_H_
 
-#include <memory>
-#include <thread>
-
 #include "absl/status/status.h"
 #include "cdc_stream/background_service_impl.h"
 #include "cdc_stream/session_management_server.h"
@@ -33,6 +30,9 @@ namespace cdc_ft {
 class BackgroundServiceImpl final
     : public backgroundservice::BackgroundService::Service {
  public:
+  using ExitRequest = backgroundservice::ExitRequest;
+  using ExitResponse = backgroundservice::ExitResponse;
+  using GetPidRequest = backgroundservice::GetPidRequest;
   using GetPidResponse = backgroundservice::GetPidResponse;
   using EmptyProto = google::protobuf::Empty;
 
@@ -43,10 +43,11 @@ class BackgroundServiceImpl final
   using ExitCallback = std::function<absl::Status()>;
   void SetExitCallback(ExitCallback exit_callback);
 
-  grpc::Status Exit(grpc::ServerContext* context, const EmptyProto* request,
+  grpc::Status Exit(grpc::ServerContext* context, const ExitRequest* request,
-                    EmptyProto* response) override;
+                    ExitResponse* response) override;
 
-  grpc::Status GetPid(grpc::ServerContext* context, const EmptyProto* request,
+  grpc::Status GetPid(grpc::ServerContext* context,
+                      const GetPidRequest* request,
                       GetPidResponse* response) override;
 
   grpc::Status HealthCheck(grpc::ServerContext* context,
@@ -55,7 +56,6 @@ class BackgroundServiceImpl final
 
  private:
   ExitCallback exit_callback_;
-  std::unique_ptr<std::thread> exit_thread_;
 };
 
 }  // namespace cdc_ft

cdc_stream/base_command.cc

@@ -15,7 +15,6 @@
 #include "cdc_stream/base_command.h"
 
 #include "absl/strings/str_format.h"
-#include "absl/strings/str_split.h"
 #include "absl_helper/jedec_size_flag.h"
 #include "lyra/lyra.hpp"
 
@@ -45,7 +44,8 @@ void BaseCommand::Register(lyra::cli& cli) {
 
 std::function<void(const std::string&)> BaseCommand::JedecParser(
     const char* flag_name, uint64_t* bytes) {
-  return [flag_name, bytes, error = &parse_error_](const std::string& value) {
+  return [flag_name, bytes,
+          error = &jedec_parse_error_](const std::string& value) {
     JedecSize size;
     if (AbslParseFlag(value, &size, error)) {
       *bytes = size.Size();
@@ -83,8 +83,8 @@ void BaseCommand::CommandHandler(const lyra::group& g) {
     return;
   }
 
-  if (!parse_error_.empty()) {
+  if (!jedec_parse_error_.empty()) {
-    std::cerr << "Error: " << parse_error_ << std::endl;
+    std::cerr << "Error: " << jedec_parse_error_ << std::endl;
     *exit_code_ = 1;
     return;
   }

cdc_stream/base_command.h

@@ -82,9 +82,9 @@ class BaseCommand {
   // Extraneous positional args. Gets reported as error if present.
   std::string extra_positional_arg_;
 
-  // Errors from custom flag parsers, e.g. JEDEC sizes or port ranges.
+  // Errors from parsing JEDEC sizes.
   // Works around Lyra not accepting errors from parsers.
-  std::string parse_error_;
+  std::string jedec_parse_error_;
 };
 
 }  // namespace cdc_ft

cdc_stream/cdc_fuse_manager.cc

@@ -16,7 +16,6 @@
 
 #include "absl/strings/match.h"
 #include "absl/strings/str_format.h"
-#include "absl/strings/str_split.h"
 #include "cdc_fuse_fs/constants.h"
 #include "common/gamelet_component.h"
 #include "common/log.h"
@@ -27,33 +26,14 @@
 namespace cdc_ft {
 namespace {
 
-constexpr char kExeFilename[] = "cdc_stream.exe";
 constexpr char kFuseFilename[] = "cdc_fuse_fs";
 constexpr char kLibFuseFilename[] = "libfuse.so";
 constexpr char kFuseStdoutPrefix[] = "cdc_fuse_fs_stdout";
-constexpr char kRemoteToolsBinDir[] = ".cache/cdc-file-transfer/bin/";
+constexpr char kRemoteToolsBinDir[] = "~/.cache/cdc-file-transfer/bin/";
 
 // Cache directory on the gamelet to store data chunks.
 constexpr char kCacheDir[] = "~/.cache/cdc-file-transfer/chunks";
 
-// Parses the port from the FUSE stdout when FUSE is up-to-date. In that case,
-// the expected stdout is similar to "Port 12345 cdc_fuse_fs is up-to-date".
-absl::StatusOr<int> ParsePort(const std::string& fuse_stdout) {
-  // Search backwards until we find "Port ".
-  size_t port_pos = fuse_stdout.find(kFusePortPrefix);
-  if (port_pos == std::string::npos) {
-    return MakeStatus("Failed to find '%s' marker in server output '%s'",
-                      kFusePortPrefix, fuse_stdout);
-  }
-  int port =
-      atoi(fuse_stdout.substr(port_pos + strlen(kFusePortPrefix)).c_str());
-  if (port == 0) {
-    return MakeStatus("Failed to parse port from server output '%s'",
-                      fuse_stdout);
-  }
-  return port;
-}
-
 }  // namespace
 
 CdcFuseManager::CdcFuseManager(std::string instance,
@@ -73,34 +53,32 @@ absl::Status CdcFuseManager::Deploy() {
   std::string exe_dir;
   RETURN_IF_ERROR(path::GetExeDir(&exe_dir), "Failed to get exe directory");
 
-  // Create the remote tools bin dir if it doesn't exist yet.
+  // Set the cwd to the exe dir and pass the filenames to scp. Otherwise, some
-  // This assumes that sftp's remote startup directory is the home directory.
+  // scp implementations can get confused and create the wrong remote filenames.
-  std::vector<std::string> dir_parts =
+  path::SetCwd(exe_dir);
-      absl::StrSplit(kRemoteToolsBinDir, '/', absl::SkipEmpty());
-  std::string sftp_commands;
-  for (const std::string& dir : dir_parts) {
-    // Use -mkdir to ignore errors if the directory already exists.
-    sftp_commands += absl::StrFormat("-mkdir %s\ncd %s\n", dir, dir);
-  }
 
-  sftp_commands += absl::StrFormat("put %s\n", kFuseFilename);
+  // Copy FUSE to the gamelet.
-  sftp_commands += absl::StrFormat("put %s\n", kLibFuseFilename);
+  LOG_DEBUG("Copying FUSE");
-  sftp_commands += absl::StrFormat("chmod 755 %s\n", kFuseFilename);
+  RETURN_IF_ERROR(remote_util_->Scp({kFuseFilename, kLibFuseFilename},
+                                    kRemoteToolsBinDir, /*compress=*/false),
+                  "Failed to copy FUSE to gamelet");
+  LOG_DEBUG("Copying FUSE succeeded");
 
-  LOG_DEBUG("Deploying FUSE");
+  // Make FUSE executable. Note that sync does it automatically.
-  RETURN_IF_ERROR(
+  LOG_DEBUG("Making FUSE executable");
-      remote_util_->Sftp(sftp_commands, exe_dir, /*compress=*/false),
+  std::string remotePath = path::JoinUnix(kRemoteToolsBinDir, kFuseFilename);
-      "Failed to deploy FUSE");
+  RETURN_IF_ERROR(remote_util_->Chmod("a+x", remotePath),
-  LOG_DEBUG("Deploying FUSE succeeded");
+                  "Failed to set executable flag on FUSE");
+  LOG_DEBUG("Making FUSE succeeded");
 
   return absl::OkStatus();
 }
 
 absl::Status CdcFuseManager::Start(const std::string& mount_dir,
-                                   uint16_t local_port, int verbosity,
+                                   uint16_t local_port, uint16_t remote_port,
-                                   bool debug, bool singlethreaded,
+                                   int verbosity, bool debug,
-                                   bool enable_stats, bool check,
+                                   bool singlethreaded, bool enable_stats,
-                                   uint64_t cache_capacity,
+                                   bool check, uint64_t cache_capacity,
                                    uint32_t cleanup_timeout_sec,
                                    uint32_t access_idle_timeout_sec) {
   assert(!fuse_process_);
@@ -117,119 +95,80 @@ absl::Status CdcFuseManager::Start(const std::string& mount_dir,
   if (!status.ok()) {
     return absl::NotFoundError(absl::StrFormat(
         "Required gamelet component not found. Make sure the files %s and %s "
-        "reside in the same folder as %s.",
+        "reside in the same folder as stadia_assets_stream_manager_v3.exe.",
-        kFuseFilename, kLibFuseFilename, kExeFilename));
+        kFuseFilename, kLibFuseFilename));
   }
   std::string component_args = GameletComponent::ToCommandLineArgs(components);
 
   // Build the remote command.
   std::string remotePath = path::JoinUnix(kRemoteToolsBinDir, kFuseFilename);
   std::string remote_command = absl::StrFormat(
-      "LD_LIBRARY_PATH=%s %s "
+      "mkdir -p %s; LD_LIBRARY_PATH=%s %s "
       "--instance=%s "
-      "--components=%s --cache_dir=%s "
+      "--components=%s --port=%i --cache_dir=%s "
       "--verbosity=%i --cleanup_timeout=%i --access_idle_timeout=%i --stats=%i "
       "--check=%i --cache_capacity=%u -- -o allow_root -o ro -o nonempty -o "
       "auto_unmount %s%s%s",
-      kRemoteToolsBinDir, remotePath, RemoteUtil::QuoteForSsh(instance_),
+      kRemoteToolsBinDir, kRemoteToolsBinDir, remotePath,
-      RemoteUtil::QuoteForSsh(component_args), kCacheDir, verbosity,
+      RemoteUtil::QuoteForSsh(instance_),
-      cleanup_timeout_sec, access_idle_timeout_sec, enable_stats, check,
+      RemoteUtil::QuoteForSsh(component_args), remote_port, kCacheDir,
-      cache_capacity, debug ? "-d " : "", singlethreaded ? "-s " : "",
+      verbosity, cleanup_timeout_sec, access_idle_timeout_sec, enable_stats,
-      RemoteUtil::QuoteForSsh(mount_dir));
+      check, cache_capacity, RemoteUtil::QuoteForSsh(mount_dir),
+      debug ? " -d" : "", singlethreaded ? " -s" : "");
 
   bool needs_deploy = false;
-  int remote_port;
+  RETURN_IF_ERROR(
-  ASSIGN_OR_RETURN(remote_port, RunFuseProcess(remote_command, &needs_deploy));
+      RunFuseProcess(local_port, remote_port, remote_command, &needs_deploy));
   if (needs_deploy) {
     // Deploy and try again.
     RETURN_IF_ERROR(Deploy());
-    ASSIGN_OR_RETURN(remote_port,
+    RETURN_IF_ERROR(
-                     RunFuseProcess(remote_command, &needs_deploy));
+        RunFuseProcess(local_port, remote_port, remote_command, &needs_deploy));
   }
 
-  // Start port forwarding.
-  RETURN_IF_ERROR(RunPortForwardingProcess(local_port, remote_port));
-
-  // Wait until port forwarding is up and FUSE can connect to |remote_port|.
-  RETURN_IF_ERROR(WaitForFuseConnected());
-
   return absl::OkStatus();
 }
 
-absl::StatusOr<int> CdcFuseManager::RunFuseProcess(
+absl::Status CdcFuseManager::RunFuseProcess(uint16_t local_port,
-    const std::string& remote_command, bool* needs_deploy) {
+                                            uint16_t remote_port,
+                                            const std::string& remote_command,
+                                            bool* needs_deploy) {
   assert(!fuse_process_);
   assert(needs_deploy);
   *needs_deploy = false;
 
   LOG_DEBUG("Running FUSE process");
-  ProcessStartInfo start_info = remote_util_->BuildProcessStartInfoForSsh(
+  ProcessStartInfo start_info =
-      remote_command, ArchType::kLinux_x86_64);
+      remote_util_->BuildProcessStartInfoForSshPortForwardAndCommand(
+          local_port, remote_port, true, remote_command);
   start_info.name = kFuseFilename;
 
   // Capture stdout to determine whether a deploy is required.
   fuse_stdout_.clear();
-  fuse_port_ = 0;
+  fuse_startup_finished_ = false;
-  fuse_not_up_to_date_ = false;
+  start_info.stdout_handler = [this, needs_deploy](const char* data,
-  fuse_update_check_finished_ = false;
+                                                   size_t size) {
-  fuse_connected_ = false;
+    return HandleFuseStdout(data, size, needs_deploy);
-  start_info.stdout_handler = [this](const char* data, size_t size) {
-    return HandleFuseStdout(data, size);
   };
   fuse_process_ = process_factory_->Create(start_info);
   RETURN_IF_ERROR(fuse_process_->Start(), "Failed to start FUSE process");
   LOG_DEBUG("FUSE process started. Waiting for startup to finish.");
 
   // Run until process exits or startup finishes.
-  RETURN_IF_ERROR(fuse_process_->RunUntil(
+  auto startup_finished = [this]() { return fuse_startup_finished_.load(); };
-                      [this]() { return fuse_update_check_finished_.load(); }),
+  RETURN_IF_ERROR(fuse_process_->RunUntil(startup_finished),
                   "Failed to run FUSE process");
-  LOG_DEBUG("FUSE process update check complete.");
+  LOG_DEBUG("FUSE process startup complete.");
 
   // If the FUSE process exited before it could perform its up-to-date check, it
   // most likely happens because the binary does not exist and needs to be
   // deployed.
-  *needs_deploy = fuse_not_up_to_date_ ||
+  *needs_deploy |= !fuse_startup_finished_ && fuse_process_->HasExited() &&
-                  (!fuse_update_check_finished_ && fuse_process_->HasExited() &&
+                   fuse_process_->ExitCode() != 0;
-                   fuse_process_->ExitCode() != 0);
   if (*needs_deploy) {
     LOG_DEBUG("FUSE needs to be (re-)deployed.");
     fuse_process_.reset();
-  }
-
-  return fuse_port_;
-}
-
-absl::Status CdcFuseManager::RunPortForwardingProcess(int local_port,
-                                                      int remote_port) {
-  assert(fuse_process_);
-  assert(!forwarding_process_);
-
-  LOG_DEBUG(
-      "Running reverse port forwarding process, local port %i, remote port %i",
-      local_port, remote_port);
-  ProcessStartInfo start_info =
-      remote_util_->BuildProcessStartInfoForSshPortForward(
-          local_port, remote_port, /*reverse=*/true);
-  forwarding_process_ = process_factory_->Create(start_info);
-  RETURN_IF_ERROR(forwarding_process_->Start(),
-                  "Failed to start port forwarding process");
-
     return absl::OkStatus();
-}
-
-absl::Status CdcFuseManager::WaitForFuseConnected() {
-  assert(fuse_process_);
-  assert(forwarding_process_);
-
-  RETURN_IF_ERROR(
-      fuse_process_->RunUntil([this]() { return fuse_connected_.load(); }),
-      "Failed to run FUSE process");
-  LOG_DEBUG("FUSE process connected.");
-
-  if (!fuse_connected_ && fuse_process_->HasExited()) {
-    return MakeStatus("FUSE exited during startup with code %u",
-                      fuse_process_->ExitCode());
   }
 
   return absl::OkStatus();
@@ -240,37 +179,30 @@ absl::Status CdcFuseManager::Stop() {
     return absl::OkStatus();
   }
 
-  LOG_DEBUG("Terminating FUSE and port forwarding processes");
+  LOG_DEBUG("Terminating FUSE process");
   absl::Status status = fuse_process_->Terminate();
-  status.Update(forwarding_process_->Terminate());
   fuse_process_.reset();
-  forwarding_process_.reset();
   return status;
 }
 
 bool CdcFuseManager::IsHealthy() const {
-  return fuse_process_ && !fuse_process_->HasExited() && forwarding_process_ &&
+  return fuse_process_ && !fuse_process_->HasExited();
-         !forwarding_process_->HasExited();
 }
 
-absl::Status CdcFuseManager::HandleFuseStdout(const char* data, size_t size) {
+absl::Status CdcFuseManager::HandleFuseStdout(const char* data, size_t size,
-  // Don't capture stdout beyond startup.
+                                              bool* needs_deploy) {
-  if (!fuse_connected_) {
+  assert(needs_deploy);
-    fuse_stdout_.append(data, size);
 
-    // The remote component prints some magic strings to stdout to indicate
+  // Don't capture stdout beyond startup.
+  if (!fuse_startup_finished_) {
+    fuse_stdout_.append(data, size);
+    // The gamelet component prints some magic strings to stdout to indicate
     // whether it's up-to-date.
     if (absl::StrContains(fuse_stdout_, kFuseUpToDate)) {
-      ASSIGN_OR_RETURN(fuse_port_, ParsePort(fuse_stdout_));
+      fuse_startup_finished_ = true;
-      fuse_update_check_finished_ = true;
     } else if (absl::StrContains(fuse_stdout_, kFuseNotUpToDate)) {
-      fuse_not_up_to_date_ = true;
+      fuse_startup_finished_ = true;
-      fuse_update_check_finished_ = true;
+      *needs_deploy = true;
-    }
-
-    // It also prints stuff when it can connect to its port.
-    if (absl::StrContains(fuse_stdout_, kFuseConnected)) {
-      fuse_connected_ = true;
     }
   }
 

cdc_stream/cdc_fuse_manager.h

@@ -18,7 +18,6 @@
 #define CDC_STREAM_CDC_FUSE_MANAGER_H_
 
 #include "absl/status/status.h"
-#include "absl/status/statusor.h"
 #include "common/remote_util.h"
 
 namespace cdc_ft {
@@ -27,7 +26,7 @@ class Process;
 class ProcessFactory;
 class RemoteUtil;
 
-// Manages the remote CDC FUSE filesystem process.
+// Manages the gamelet-side CDC FUSE filesystem process.
 class CdcFuseManager {
  public:
   CdcFuseManager(std::string instance, ProcessFactory* process_factory,
@@ -37,10 +36,11 @@ class CdcFuseManager {
   CdcFuseManager(CdcFuseManager&) = delete;
   CdcFuseManager& operator=(CdcFuseManager&) = delete;
 
-  // Starts the remote CDC FUSE process. Deploys the binary if necessary.
+  // Starts the CDC FUSE and establishes a reverse SSH tunnel from the gamelet's
+  // |remote_port| to the workstation's |local_port|. Deploys the binary if
+  // necessary.
   //
   // |mount_dir| is the remote directory where to mount the FUSE.
-  // |local_port| is the local port used for gRPC connections to the FUSE.
   // |verbosity| is the log verbosity used by the filesystem.
   // |debug| puts the filesystem into debug mode if set to true. This also
   // causes the process to run in the foreground, so that logs are piped through
@@ -53,9 +53,9 @@ class CdcFuseManager {
   // |access_idle_timeout_sec| defines the number of seconds after which data
   // provider is considered to be access-idling.
   absl::Status Start(const std::string& mount_dir, uint16_t local_port,
-                     int verbosity, bool debug, bool singlethreaded,
+                     uint16_t remote_port, int verbosity, bool debug,
-                     bool enable_stats, bool check, uint64_t cache_capacity,
+                     bool singlethreaded, bool enable_stats, bool check,
-                     uint32_t cleanup_timeout_sec,
+                     uint64_t cache_capacity, uint32_t cleanup_timeout_sec,
                      uint32_t access_idle_timeout_sec);
 
   // Stops the CDC FUSE.
@@ -65,49 +65,33 @@ class CdcFuseManager {
   bool IsHealthy() const;
 
  private:
-  // Runs the remote FUSE process from the given |remote_command|. Returns the
+  // Runs the FUSE process on the gamelet from the given |remote_command| and
-  // remote port that the FUSE will connect to, once the port forwarding process
+  // establishes a reverse SSH tunnel from the gamelet's |remote_port| to the
-  // is up.
+  // workstation's |local_port|.
   //
   // If the FUSE is not up-to-date or does not exist, sets |needs_deploy| to
   // true and returns OK. In that case, Deploy() needs to be called and the FUSE
   // process should be run again.
-  absl::StatusOr<int> RunFuseProcess(const std::string& remote_command,
+  absl::Status RunFuseProcess(uint16_t local_port, uint16_t remote_port,
+                              const std::string& remote_command,
                               bool* needs_deploy);
 
-  // Establishes a reverse SSH tunnel from |remote_port| to |local_port|.
+  // Deploys the gamelet components.
-  absl::Status RunPortForwardingProcess(int local_port, int remote_port);
-
-  // Waits until FUSE can connect to its port. This essentially means that
-  // port forwarding is up and running.
-  absl::Status WaitForFuseConnected();
-
-  // Deploys the remote components.
   absl::Status Deploy();
 
-  // Output handler for FUSE's stdout.
+  // Output handler for FUSE's stdout. Sets |needs_deploy| to true if the output
-  // Sets |fuse_not_up_to_date_| to true if the output contains a magic marker
+  // contains a magic marker to indicate that the binary has to be redeployed.
-  // to indicate that the binary has to be redeployed.
+  // Called in a background thread.
-  // Sets |fuse_port_| to the remote gRPC port if the output contains a magic
+  absl::Status HandleFuseStdout(const char* data, size_t size,
-  // marker that has the port and indicates that the binary is up-to-date.
+                                bool* needs_deploy);
-  // Sets |fuse_update_check_finished_| to true if any of the above two markers
-  // was set. Called in a background thread.
-  // Sets |fuse_startup_finished_| to true if FUSE is connected to its port.
-  absl::Status HandleFuseStdout(const char* data, size_t size);
 
   std::string instance_;
   ProcessFactory* const process_factory_;
   RemoteUtil* const remote_util_;
 
   std::unique_ptr<Process> fuse_process_;
-  std::unique_ptr<Process> forwarding_process_;
   std::string fuse_stdout_;
-
+  std::atomic<bool> fuse_startup_finished_{false};
-  // Set by HandleFuseStdout
-  int fuse_port_ = 0;
-  bool fuse_not_up_to_date_ = false;
-  std::atomic_bool fuse_update_check_finished_{false};
-  std::atomic_bool fuse_connected_{false};
 };
 
 }  // namespace cdc_ft

cdc_stream/cdc_stream.vcxproj

@@ -47,7 +47,7 @@
     <AdditionalOptions>/std:c++17</AdditionalOptions>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
-    <OutDir>$(SolutionDir)bazel-out\x64_windows-opt\bin\cdc_stream\</OutDir>
+    <OutDir>$(SolutionDir)bazel-out\x64_windows-opt\bin\asset_stcdc_streamream_manager\</OutDir>
     <NMakePreprocessorDefinitions>UNICODE</NMakePreprocessorDefinitions>
     <AdditionalOptions>/std:c++17</AdditionalOptions>
   </PropertyGroup>

cdc_stream/local_assets_stream_manager_client.cc

@@ -28,6 +28,15 @@ using StartSessionResponse = localassetsstreammanager::StartSessionResponse;
 using StopSessionRequest = localassetsstreammanager::StopSessionRequest;
 using StopSessionResponse = localassetsstreammanager::StopSessionResponse;
 
+LocalAssetsStreamManagerClient::LocalAssetsStreamManagerClient(
+    uint16_t service_port) {
+  std::string client_address = absl::StrFormat("localhost:%u", service_port);
+  std::shared_ptr<grpc::Channel> channel = grpc::CreateCustomChannel(
+      client_address, grpc::InsecureChannelCredentials(),
+      grpc::ChannelArguments());
+  stub_ = LocalAssetsStreamManager::NewStub(std::move(channel));
+}
+
 LocalAssetsStreamManagerClient::LocalAssetsStreamManagerClient(
     std::shared_ptr<grpc::Channel> channel) {
   stub_ = LocalAssetsStreamManager::NewStub(std::move(channel));
@@ -36,15 +45,16 @@ LocalAssetsStreamManagerClient::LocalAssetsStreamManagerClient(
 LocalAssetsStreamManagerClient::~LocalAssetsStreamManagerClient() = default;
 
 absl::Status LocalAssetsStreamManagerClient::StartSession(
-    const std::string& src_dir, const std::string& user_host,
+    const std::string& src_dir, const std::string& user_host, uint16_t ssh_port,
     const std::string& mount_dir, const std::string& ssh_command,
-    const std::string& sftp_command) {
+    const std::string& scp_command) {
   StartSessionRequest request;
   request.set_workstation_directory(src_dir);
   request.set_user_host(user_host);
+  request.set_port(ssh_port);
   request.set_mount_dir(mount_dir);
   request.set_ssh_command(ssh_command);
-  request.set_sftp_command(sftp_command);
+  request.set_scp_command(scp_command);
 
   grpc::ClientContext context;
   StartSessionResponse response;

cdc_stream/local_assets_stream_manager_client.h

@@ -20,6 +20,7 @@
 #include <memory>
 
 #include "absl/status/status.h"
+#include "grpcpp/channel.h"
 #include "proto/local_assets_stream_manager.grpc.pb.h"
 
 namespace grpc_impl {
@@ -31,6 +32,8 @@ namespace cdc_ft {
 // gRpc client for starting/stopping asset streaming sessions.
 class LocalAssetsStreamManagerClient {
  public:
+  explicit LocalAssetsStreamManagerClient(uint16_t service_port);
+
   // |channel| is a grpc channel to use.
   explicit LocalAssetsStreamManagerClient(
       std::shared_ptr<grpc::Channel> channel);
@@ -41,14 +44,15 @@ class LocalAssetsStreamManagerClient {
   // Starting a second session to the same target will stop the first one.
   // |src_dir| is the Windows source directory to stream.
   // |user_host| is the Linux host, formatted as [user@:host].
+  // |ssh_port| is the SSH port to use while connecting to the host.
   // |mount_dir| is the Linux target directory to stream to.
   // |ssh_command| is the ssh command and extra arguments to use.
-  // |sftp_command| is the sftp command and extra arguments to use.
+  // |scp_command| is the scp command and extra arguments to use.
   absl::Status StartSession(const std::string& src_dir,
-                            const std::string& user_host,
+                            const std::string& user_host, uint16_t ssh_port,
                             const std::string& mount_dir,
                             const std::string& ssh_command,
-                            const std::string& sftp_command);
+                            const std::string& scp_command);
 
   // Stops the streaming session to the Linux target |user_host|:|mount_dir|.
   // |user_host| is the Linux host, formatted as [user@:host].

cdc_stream/local_assets_stream_manager_service_impl.cc

@@ -208,7 +208,7 @@ LocalAssetsStreamManagerServiceImpl::GetTargetForStadia(
   SessionTarget target;
   target.mount_dir = request.mount_dir();
   target.ssh_command = request.ssh_command();
-  target.sftp_command = request.sftp_command();
+  target.scp_command = request.scp_command();
 
   // Parse instance/project/org id.
   if (!ParseInstanceName(request.gamelet_name(), instance_id, project_id,
@@ -219,11 +219,12 @@ LocalAssetsStreamManagerServiceImpl::GetTargetForStadia(
 
   // Run 'ggp ssh init' to determine IP (host) and port.
   std::string instance_ip;
-  ASSIGN_OR_RETURN(instance_ip,
+  uint16_t instance_port = 0;
-                   InitSsh(*instance_id, *project_id, *organization_id));
+  RETURN_IF_ERROR(InitSsh(*instance_id, *project_id, *organization_id,
+                          &instance_ip, &instance_port));
 
   target.user_host = "cloudcast@" + instance_ip;
-  // Note: Port must be set with ssh_command (-p) and sftp_command (-P).
+  target.ssh_port = instance_port;
   return target;
 }
 
@@ -233,7 +234,10 @@ SessionTarget LocalAssetsStreamManagerServiceImpl::GetTarget(
   target.user_host = request.user_host();
   target.mount_dir = request.mount_dir();
   target.ssh_command = request.ssh_command();
-  target.sftp_command = request.sftp_command();
+  target.scp_command = request.scp_command();
+  target.ssh_port = request.port() > 0 && request.port() <= UINT16_MAX
+                        ? static_cast<uint16_t>(request.port())
+                        : RemoteUtil::kDefaultSshPort;
 
   *instance_id = absl::StrCat(target.user_host, ":", target.mount_dir);
   return target;
@@ -253,10 +257,13 @@ metrics::RequestOrigin LocalAssetsStreamManagerServiceImpl::ConvertOrigin(
   }
 }
 
-absl::StatusOr<std::string> LocalAssetsStreamManagerServiceImpl::InitSsh(
+absl::Status LocalAssetsStreamManagerServiceImpl::InitSsh(
     const std::string& instance_id, const std::string& project_id,
-    const std::string& organization_id) {
+    const std::string& organization_id, std::string* instance_ip,
+    uint16_t* instance_port) {
   SdkUtil sdk_util;
+  instance_ip->clear();
+  *instance_port = 0;
 
   ProcessStartInfo start_info;
   start_info.command = absl::StrFormat(
@@ -270,7 +277,6 @@ absl::StatusOr<std::string> LocalAssetsStreamManagerServiceImpl::InitSsh(
         absl::StrFormat(" --organization %s", Quoted(organization_id));
   }
   start_info.name = "ggp ssh init";
-  start_info.flags = ProcessFlags::kNoWindow;
 
   std::string output;
   start_info.stdout_handler = [&output, this](const char* data,
@@ -298,13 +304,22 @@ absl::StatusOr<std::string> LocalAssetsStreamManagerServiceImpl::InitSsh(
   }
 
   // Parse gamelet IP. Should be "Host: <instance_ip ip>".
-  std::string instance_ip;
+  if (!ParseValue(output, "Host", instance_ip)) {
-  if (!ParseValue(output, "Host", &instance_ip)) {
     return MakeStatus("Failed to parse host from ggp ssh init response\n%s",
                       output);
   }
 
-  return instance_ip;
+  // Parse ssh port. Should be "Port: <port>".
+  std::string port_string;
+  const bool result = ParseValue(output, "Port", &port_string);
+  int int_port = atoi(port_string.c_str());
+  if (!result || int_port == 0 || int_port <= 0 || int_port > UINT_MAX) {
+    return MakeStatus("Failed to parse ssh port from ggp ssh init response\n%s",
+                      output);
+  }
+
+  *instance_port = static_cast<uint16_t>(int_port);
+  return absl::OkStatus();
 }
 
 }  // namespace cdc_ft

cdc_stream/local_assets_stream_manager_service_impl.h

@@ -93,10 +93,11 @@ class LocalAssetsStreamManagerServiceImpl final
 
   // Initializes an ssh connection to a gamelet by calling 'ggp ssh init'.
   // |instance_id| must be set, |project_id|, |organization_id| are optional.
-  // Returns the instance's IP address.
+  // Returns |instance_ip| and |instance_port| (SSH port).
-  absl::StatusOr<std::string> InitSsh(const std::string& instance_id,
+  absl::Status InitSsh(const std::string& instance_id,
                        const std::string& project_id,
-                                      const std::string& organization_id);
+                       const std::string& organization_id,
+                       std::string* instance_ip, uint16_t* instance_port);
 
   const SessionConfig cfg_;
   SessionManager* const session_manager_;

cdc_stream/main.cc

@@ -15,31 +15,9 @@
 #include "cdc_stream/start_command.h"
 #include "cdc_stream/start_service_command.h"
 #include "cdc_stream/stop_command.h"
-#include "cdc_stream/stop_service_command.h"
-#include "common/platform.h"
 #include "lyra/lyra.hpp"
 
-#if PLATFORM_WINDOWS
+int main(int argc, char* argv[]) {
-int wmain(int argc, wchar_t* wargv[]) {
-  // Convert args from wide to UTF8 strings.
-  std::vector<std::string> utf8_str_args;
-  utf8_str_args.reserve(argc);
-  for (int i = 0; i < argc; i++) {
-    utf8_str_args.push_back(cdc_ft::Util::WideToUtf8Str(wargv[i]));
-  }
-
-  // Convert args from UTF8 strings to UTF8 c-strings.
-  std::vector<const char*> utf8_args;
-  utf8_args.reserve(argc);
-  for (const auto& utf8_str_arg : utf8_str_args) {
-    utf8_args.push_back(utf8_str_arg.c_str());
-  }
-
-  const char** argv = utf8_args.data();
-#else
-int main(int argc, char** argv) {
-#endif
-
   // Set up commands.
   auto cli = lyra::cli();
   bool show_help = false;
@@ -55,9 +33,6 @@ int main(int argc, char** argv) {
   cdc_ft::StartServiceCommand start_service_cmd(&exit_code);
   start_service_cmd.Register(cli);
 
-  cdc_ft::StopServiceCommand stop_service_cmd(&exit_code);
-  stop_service_cmd.Register(cli);
-
   // Parse args and run. Note that parse actually runs the commands.
   // exit_code is -1 if no command was run.
   auto result = cli.parse({argc, argv});

cdc_stream/multi_session.cc

@@ -18,10 +18,9 @@
 #include "common/file_watcher_win.h"
 #include "common/log.h"
 #include "common/path.h"
-#include "common/path_filter.h"
 #include "common/platform.h"
+#include "common/port_manager.h"
 #include "common/process.h"
-#include "common/server_socket.h"
 #include "common/util.h"
 #include "data_store/disk_data_store.h"
 #include "manifest/content_id.h"
@@ -34,6 +33,11 @@
 namespace cdc_ft {
 namespace {
 
+// Ports used by the asset streaming service for local port forwarding on
+// workstation and gamelet.
+constexpr int kAssetStreamPortFirst = 44433;
+constexpr int kAssetStreamPortLast = 44442;
+
 // Stats output period (if enabled).
 constexpr double kStatsPrintDelaySec = 0.1f;
 
@@ -436,8 +440,16 @@ absl::Status MultiSession::Initialize() {
   }
 
   // Find an available local port.
-  ASSIGN_OR_RETURN(local_asset_stream_port_, ServerSocket::FindAvailablePort(),
+  std::unordered_set<int> ports;
-                   "Failed to find an available local port");
+  ASSIGN_OR_RETURN(
+      ports,
+      PortManager::FindAvailableLocalPorts(kAssetStreamPortFirst,
+                                           kAssetStreamPortLast, "127.0.0.1",
+                                           process_factory_),
+      "Failed to find an available local port in the range [%d, %d]",
+      kAssetStreamPortFirst, kAssetStreamPortLast);
+  assert(!ports.empty());
+  local_asset_stream_port_ = *ports.begin();
 
   assert(!runner_);
   runner_ = std::make_unique<MultiSessionRunner>(
@@ -512,7 +524,8 @@ absl::Status MultiSession::StartSession(const std::string& instance_id,
 
   auto session = std::make_unique<Session>(
       instance_id, target, cfg_, process_factory_, std::move(metrics_recorder));
-  RETURN_IF_ERROR(session->Start(local_asset_stream_port_));
+  RETURN_IF_ERROR(session->Start(local_asset_stream_port_,
+                                 kAssetStreamPortFirst, kAssetStreamPortLast));
 
   // Wait for the FUSE to receive the first intermediate manifest.
   RETURN_IF_ERROR(runner_->WaitForManifestAck(instance_id, absl::Seconds(5)));
@@ -542,21 +555,6 @@ bool MultiSession::HasSession(const std::string& instance_id) {
   return sessions_.find(instance_id) != sessions_.end();
 }
 
-std::vector<std::string> MultiSession::MatchSessions(
-    const std::string& instance_id_filter) {
-  PathFilter filter;
-  filter.AddRule(PathFilter::Rule::Type::kInclude, instance_id_filter);
-  filter.AddRule(PathFilter::Rule::Type::kExclude, "*");
-
-  std::vector<std::string> matches;
-  for (const auto& [instance_id, session] : sessions_) {
-    if (filter.IsMatch(instance_id)) {
-      matches.push_back(instance_id);
-    }
-  }
-  return matches;
-}
-
 bool MultiSession::IsSessionHealthy(const std::string& instance_id) {
   absl::ReaderMutexLock lock(&sessions_mutex_);
   auto iter = sessions_.find(instance_id);
@@ -611,8 +609,7 @@ absl::StatusOr<std::string> MultiSession::GetCachePath(
   path::Append(&appdata_path, ".cache");
 #endif
 
-  std::string base_dir =
+  std::string base_dir = path::Join(appdata_path, "GGP", "asset_streaming");
-      path::Join(appdata_path, "cdc-file-transfer", "chunks");
   std::string cache_dir = GetCacheDir(src_dir);
 
   size_t total_size = base_dir.size() + 1 + cache_dir.size();

cdc_stream/multi_session.h

@@ -134,11 +134,6 @@ class MultiSessionRunner {
 // to an arbitrary number of gamelets.
 class MultiSession {
  public:
-  // Ports used by the asset streaming service for local port forwarding on
-  // workstation and gamelet.
-  static constexpr int kDefaultForwardPortFirst = 44433;
-  static constexpr int kDefaultForwardPortLast = 44442;
-
   // Maximum length of cache path. We must be able to write content hashes into
   // this path:
   // <cache path>\01234567890123456789<null terminator> = 260 characters.
@@ -153,7 +148,7 @@ class MultiSession {
   // |process_factory| abstracts process creation.
   // |data_store| can be passed for tests to override the default store used.
   // By default, the class uses a DiskDataStore that writes to
-  // %APPDATA%\cdc-file-transfer\chunks\<dir_derived_from_src_dir> on Windows.
+  // %APPDATA%\GGP\asset_streaming|<dir_derived_from_src_dir> on Windows.
   MultiSession(std::string src_dir, SessionConfig cfg,
                ProcessFactory* process_factory,
                MultiSessionMetricsRecorder const* metrics_recorder,
@@ -199,11 +194,6 @@ class MultiSession {
   absl::Status StopSession(const std::string& instance_id)
       ABSL_LOCKS_EXCLUDED(sessions_mutex_);
 
-  // Returns all instance ids that match the given filter. The filter may
-  // contain Windows-style wildcards, e.g. *, foo* or f?o.
-  // Matches are case sensitive.
-  std::vector<std::string> MatchSessions(const std::string& instance_id_filter);
-
   // Returns true if there is an existing session for |instance_id|.
   bool HasSession(const std::string& instance_id)
       ABSL_LOCKS_EXCLUDED(sessions_mutex_);
@@ -225,7 +215,7 @@ class MultiSession {
   static std::string GetCacheDir(std::string dir);
 
   // Returns the directory where manifest chunks are cached, e.g.
-  // "%APPDATA%\cdc-file-transfer\chunks\c__path_to_game_abcdef01" for
+  // "%APPDATA%\GGP\asset_streaming\c__path_to_game_abcdef01" for
   // "C:\path\to\game".
   // The returned path is shortened to |max_len| by removing UTF8 code points
   // from the beginning of the actual cache directory (c__path...) if necessary.

cdc_stream/multi_session_test.cc

@@ -158,7 +158,7 @@ class MultiSessionTest : public ManifestTestBase {
     EXPECT_EQ(data->file_count, file_count);
     EXPECT_EQ(data->min_chunk_size, 128 << 10);
     EXPECT_EQ(data->avg_chunk_size, 256 << 10);
-    EXPECT_EQ(data->max_chunk_size, 512 << 10);
+    EXPECT_EQ(data->max_chunk_size, 1024 << 10);
   }
 
   metrics::ManifestUpdateData GetManifestUpdateData(
@@ -241,7 +241,7 @@ TEST_F(MultiSessionTest, GetCachePath_ContainsExpectedParts) {
   ASSERT_OK(cache_path);
   EXPECT_TRUE(absl::EndsWith(*cache_path, kCacheDir)) << *cache_path;
   EXPECT_TRUE(
-      absl::StrContains(*cache_path, path::Join("cdc-file-transfer", "chunks")))
+      absl::StrContains(*cache_path, path::Join("GGP", "asset_streaming")))
       << *cache_path;
 }
 
@@ -253,7 +253,7 @@ TEST_F(MultiSessionTest, GetCachePath_ShortensLongPaths) {
   ASSERT_OK(cache_path);
   EXPECT_EQ(cache_path->size(), MultiSession::kDefaultMaxCachePathLen);
   EXPECT_TRUE(
-      absl::StrContains(*cache_path, path::Join("cdc-file-transfer", "chunks")))
+      absl::StrContains(*cache_path, path::Join("GGP", "asset_streaming")))
       << *cache_path;
   // The hash in the end of the path is kept and not shortened.
   EXPECT_EQ(cache_dir.substr(cache_dir.size() - MultiSession::kDirHashLen),
@@ -261,8 +261,7 @@ TEST_F(MultiSessionTest, GetCachePath_ShortensLongPaths) {
 }
 
 TEST_F(MultiSessionTest, GetCachePath_DoesNotSplitUtfCodePoints) {
-  // Find out the length of the %APPDATA%\cdc-file-transfer\chunks\" + hash
+  // Find out the length of the %APPDATA%\GGP\asset_streaming\" + hash part.
-  // part.
   absl::StatusOr<std::string> cache_path = MultiSession::GetCachePath("");
   ASSERT_OK(cache_path);
   size_t base_len = cache_path->size();
@@ -272,17 +271,17 @@ TEST_F(MultiSessionTest, GetCachePath_DoesNotSplitUtfCodePoints) {
   ASSERT_OK(cache_path);
   EXPECT_EQ(cache_path->size(), base_len);
 
-  // %APPDATA%\cdc-file-transfer\chunks\abcdefg
+  // %APPDATA%\GGP\asset_streaming\abcdefg
   cache_path = MultiSession::GetCachePath(u8"\u0200\u0200", base_len + 1);
   ASSERT_OK(cache_path);
   EXPECT_EQ(cache_path->size(), base_len);
 
-  // %APPDATA%\cdc-file-transfer\chunks\\u0200abcdefg
+  // %APPDATA%\GGP\asset_streaming\\u0200abcdefg
   cache_path = MultiSession::GetCachePath(u8"\u0200\u0200", base_len + 2);
   ASSERT_OK(cache_path);
   EXPECT_EQ(cache_path->size(), base_len + 2);
 
-  // %APPDATA%\cdc-file-transfer\chunks\\u0200abcdefg
+  // %APPDATA%\GGP\asset_streaming\\u0200abcdefg
   cache_path = MultiSession::GetCachePath(u8"\u0200\u0200", base_len + 3);
   ASSERT_OK(cache_path);
   EXPECT_EQ(cache_path->size(), base_len + 2);

cdc_stream/session.cc

@@ -16,6 +16,7 @@
 
 #include "cdc_stream/cdc_fuse_manager.h"
 #include "common/log.h"
+#include "common/port_manager.h"
 #include "common/status.h"
 #include "common/status_macros.h"
 #include "metrics/enums.h"
@@ -46,16 +47,16 @@ Session::Session(std::string instance_id, const SessionTarget& target,
       mount_dir_(target.mount_dir),
       cfg_(std::move(cfg)),
       process_factory_(process_factory),
-      remote_util_(target.user_host, cfg_.verbosity, cfg_.quiet,
+      remote_util_(cfg_.verbosity, cfg_.quiet, process_factory,
-                   process_factory,
                    /*forward_output_to_logging=*/true),
       metrics_recorder_(std::move(metrics_recorder)) {
   assert(metrics_recorder_);
+  remote_util_.SetUserHostAndPort(target.user_host, target.ssh_port);
   if (!target.ssh_command.empty()) {
     remote_util_.SetSshCommand(target.ssh_command);
   }
-  if (!target.sftp_command.empty()) {
+  if (!target.scp_command.empty()) {
-    remote_util_.SetSftpCommand(target.sftp_command);
+    remote_util_.SetScpCommand(target.scp_command);
   }
 }
 
@@ -67,18 +68,30 @@ Session::~Session() {
   }
 }
 
-absl::Status Session::Start(int local_port) {
+absl::Status Session::Start(int local_port, int first_remote_port,
+                            int last_remote_port) {
+  // Find an available remote port.
+  std::unordered_set<int> ports;
+  ASSIGN_OR_RETURN(
+      ports,
+      PortManager::FindAvailableRemotePorts(
+          first_remote_port, last_remote_port, "127.0.0.1", process_factory_,
+          &remote_util_, kInstanceConnectionTimeoutSec),
+      "Failed to find an available remote port in the range [%d, %d]",
+      first_remote_port, last_remote_port);
+  assert(!ports.empty());
+  int remote_port = *ports.begin();
+
   assert(!fuse_);
   fuse_ = std::make_unique<CdcFuseManager>(instance_id_, process_factory_,
                                            &remote_util_);
-
   RETURN_IF_ERROR(
-      fuse_->Start(mount_dir_, local_port, cfg_.verbosity, cfg_.fuse_debug,
+      fuse_->Start(mount_dir_, local_port, remote_port, cfg_.verbosity,
-                   cfg_.fuse_singlethreaded, cfg_.stats, cfg_.fuse_check,
+                   cfg_.fuse_debug, cfg_.fuse_singlethreaded, cfg_.stats,
-                   cfg_.fuse_cache_capacity, cfg_.fuse_cleanup_timeout_sec,
+                   cfg_.fuse_check, cfg_.fuse_cache_capacity,
+                   cfg_.fuse_cleanup_timeout_sec,
                    cfg_.fuse_access_idle_timeout_sec),
       "Failed to start instance component");
-
   return absl::OkStatus();
 }
 

cdc_stream/session.h

@@ -36,10 +36,12 @@ class Process;
 struct SessionTarget {
   // SSH username and hostname of the remote target, formed as [user@]host.
   std::string user_host;
+  // Port to use for SSH connections to the remote target.
+  uint16_t ssh_port;
   // Ssh command to use to connect to the remote target.
   std::string ssh_command;
-  // Sftp command to use to copy files to the remote target.
+  // Scp command to use to copy files to the remote target.
-  std::string sftp_command;
+  std::string scp_command;
   // Directory on the remote target where to mount the streamed directory.
   std::string mount_dir;
 };
@@ -58,7 +60,9 @@ class Session {
 
   // Starts the CDC FUSE on the instance with established port forwarding.
   // |local_port| is the local reverse forwarding port to use.
-  absl::Status Start(int local_port);
+  // [|first_remote_port|, |last_remote_port|] are the allowed remote ports.
+  absl::Status Start(int local_port, int first_remote_port,
+                     int last_remote_port);
 
   // Shuts down the connection to the instance.
   absl::Status Stop() ABSL_LOCKS_EXCLUDED(transferred_data_mu_);

cdc_stream/session_config.h

@@ -56,10 +56,6 @@ struct SessionConfig {
 
   // Time to wait until running a manifest update after detecting a file change.
   uint32_t file_change_wait_duration_ms = 0;
-
-  // Ports used for local port forwarding. Deprecated as forward ports are
-  // determined automatically now using ephemeral ports.
-  std::string deprecated_forward_port_range;
 };
 
 }  // namespace cdc_ft

cdc_stream/session_management_server.h

@@ -36,7 +36,7 @@ class ProcessFactory;
 // - Background
 class SessionManagementServer {
  public:
-  static constexpr uint16_t kDefaultServicePort = 44432;
+  static constexpr int kDefaultServicePort = 44432;
 
   SessionManagementServer(grpc::Service* session_service,
                           grpc::Service* background_service,

cdc_stream/session_manager.cc

@@ -136,24 +136,9 @@ absl::Status SessionManager::StartSession(
   return status;
 }
 
-absl::Status SessionManager::StopSession(
+absl::Status SessionManager::StopSession(const std::string& instance_id) {
-    const std::string& instance_id_filter) {
   absl::MutexLock lock(&sessions_mutex_);
-
+  return StopSessionInternal(instance_id);
-  std::vector<std::string> instance_ids;
-  for (const auto& [key, ms] : sessions_) {
-    auto ids = ms->MatchSessions(instance_id_filter);
-    instance_ids.insert(instance_ids.end(), ids.begin(), ids.end());
-  }
-  if (instance_ids.empty()) {
-    return absl::NotFoundError(
-        absl::StrFormat("No session found matching '%s'", instance_id_filter));
-  }
-
-  for (const std::string& instance_id : instance_ids) {
-    RETURN_IF_ERROR(StopSessionInternal(instance_id));
-  }
-  return absl::OkStatus();
 }
 
 MultiSession* SessionManager::GetMultiSession(const std::string& src_dir) {

cdc_stream/session_manager.h

@@ -58,11 +58,9 @@ class SessionManager {
                             metrics::SessionStartStatus* metrics_status)
       ABSL_LOCKS_EXCLUDED(sessions_mutex_);
 
-  // Stops all sessions that match the given |instance_id_filter|.
+  // Stops the session for the given |instance_id|.
-  // The filter may contain Windows-style wildcards like * and ?.
-  // Matching is case-sensitive.
   // Returns a NotFound error if no session exists.
-  absl::Status StopSession(const std::string& instance_id_filter)
+  absl::Status StopSession(const std::string& instance_id)
       ABSL_LOCKS_EXCLUDED(sessions_mutex_);
 
   // Shuts down all existing MultiSessions.

cdc_stream/start_command.cc

@@ -16,19 +16,12 @@
 
 #include <memory>
 
-#include "cdc_stream/background_service_client.h"
 #include "cdc_stream/local_assets_stream_manager_client.h"
 #include "cdc_stream/session_management_server.h"
 #include "common/log.h"
 #include "common/path.h"
-#include "common/process.h"
 #include "common/remote_util.h"
 #include "common/status_macros.h"
-#include "common/stopwatch.h"
-#include "common/util.h"
-#include "grpcpp/channel.h"
-#include "grpcpp/create_channel.h"
-#include "grpcpp/support/channel_arguments.h"
 #include "lyra/lyra.hpp"
 
 namespace cdc_ft {
@@ -36,19 +29,6 @@ namespace {
 constexpr int kDefaultVerbosity = 2;
 }  // namespace
 
-namespace {
-// Time to poll until the streaming service becomes healthy.
-constexpr double kServiceStartupTimeoutSec = 20.0;
-
-std::shared_ptr<grpc::Channel> CreateChannel(uint16_t service_port) {
-  std::string client_address = absl::StrFormat("localhost:%u", service_port);
-  return grpc::CreateCustomChannel(client_address,
-                                   grpc::InsecureChannelCredentials(),
-                                   grpc::ChannelArguments());
-}
-
-}  // namespace
-
 StartCommand::StartCommand(int* exit_code)
     : BaseCommand("start",
                   "Start streaming files from a Windows to a Linux device",
@@ -72,31 +52,28 @@ void StartCommand::RegisterCommandLineFlags(lyra::command& cmd) {
                 "asset stream service, default: " +
                 std::to_string(SessionManagementServer::kDefaultServicePort)));
 
+  ssh_port_ = RemoteUtil::kDefaultSshPort;
+  cmd.add_argument(
+      lyra::opt(ssh_port_, "port")
+          .name("--ssh-port")
+          .help("Port to use while connecting to the remote instance being "
+                "streamed to, default: " +
+                std::to_string(RemoteUtil::kDefaultSshPort)));
+
   path::GetEnv("CDC_SSH_COMMAND", &ssh_command_).IgnoreError();
   cmd.add_argument(
-      lyra::opt(ssh_command_, "cmd")
+      lyra::opt(ssh_command_, "ssh_command")
           .name("--ssh-command")
           .help("Path and arguments of ssh command to use, e.g. "
-                "\"C:\\path\\to\\ssh.exe -F config_file -p 1234\". Can also be "
+                "\"C:\\path\\to\\ssh.exe -F config_file\". Can also be "
                 "specified by the CDC_SSH_COMMAND environment variable."));
 
-  path::GetEnv("CDC_SFTP_COMMAND", &sftp_command_).IgnoreError();
+  path::GetEnv("CDC_SCP_COMMAND", &scp_command_).IgnoreError();
   cmd.add_argument(
-      lyra::opt(sftp_command_, "cmd")
+      lyra::opt(scp_command_, "scp_command")
-          .name("--sftp-command")
-          .help(
-              "Path and arguments of sftp command to use, e.g. "
-              "\"C:\\path\\to\\sftp.exe -F config_file -P 1234\". Can also be "
-              "specified by the CDC_SFTP_COMMAND environment variable."));
-
-  path::GetEnv("CDC_SCP_COMMAND", &deprecated_scp_command_).IgnoreError();
-  cmd.add_argument(
-      lyra::opt(deprecated_scp_command_, "cmd")
           .name("--scp-command")
-          .help("[Deprecated, use --sftp-command] Path and arguments of scp "
+          .help("Path and arguments of scp command to use, e.g. "
-                "command to "
+                "\"C:\\path\\to\\scp.exe -F config_file\". Can also be "
-                "use, e.g. "
-                "\"C:\\path\\to\\scp.exe -F config_file -P 1234\". Can also be "
                 "specified by the CDC_SCP_COMMAND environment variable."));
 
   cmd.add_argument(lyra::arg(PosArgValidator(&src_dir_), "dir")
@@ -104,7 +81,7 @@ void StartCommand::RegisterCommandLineFlags(lyra::command& cmd) {
                        .help("Windows directory to stream"));
 
   cmd.add_argument(
-      lyra::arg(PosArgValidator(&user_host_dir_), "[user@]host:dir")
+      lyra::arg(PosArgValidator(&user_host_dir_), "[user@]host:src-dir")
           .required()
           .help("Linux host and directory to stream to"));
 }
@@ -112,49 +89,16 @@ void StartCommand::RegisterCommandLineFlags(lyra::command& cmd) {
 absl::Status StartCommand::Run() {
   LogLevel level = Log::VerbosityToLogLevel(verbosity_);
   ScopedLog scoped_log(std::make_unique<ConsoleLog>(level));
+  LocalAssetsStreamManagerClient client(service_port_);
 
   std::string full_src_dir = path::GetFullPath(src_dir_);
   std::string user_host, mount_dir;
   RETURN_IF_ERROR(LocalAssetsStreamManagerClient::ParseUserHostDir(
       user_host_dir_, &user_host, &mount_dir));
 
-  // Backwards compatibility after switching from scp to sftp.
+  absl::Status status =
-  if (sftp_command_.empty() && !deprecated_scp_command_.empty()) {
+      client.StartSession(full_src_dir, user_host, ssh_port_, mount_dir,
-    LOG_WARNING(
+                          ssh_command_, scp_command_);
-        "The CDC_SCP_COMMAND environment variable and the --scp-command flag "
-        "are deprecated. Please set CDC_SFTP_COMMAND or --sftp-command "
-        "instead.");
-
-    sftp_command_ = RemoteUtil::ScpToSftpCommand(deprecated_scp_command_);
-    if (!sftp_command_.empty()) {
-      LOG_WARNING("Converted scp command '%s' to sftp command '%s'.",
-                  deprecated_scp_command_, sftp_command_);
-    } else {
-      LOG_WARNING("Failed to convert scp command '%s' to sftp command.",
-                  deprecated_scp_command_);
-    }
-  }
-
-  LocalAssetsStreamManagerClient client(CreateChannel(service_port_));
-  absl::Status status = client.StartSession(full_src_dir, user_host, mount_dir,
-                                            ssh_command_, sftp_command_);
-
-  if (absl::IsUnavailable(status)) {
-    LOG_DEBUG("StartSession status: %s", status.ToString());
-    LOG_INFO("Streaming service is unavailable. Starting it...");
-    status = StartStreamingService();
-
-    if (status.ok()) {
-      LOG_INFO("Streaming service successfully started");
-
-      // Recreate client. The old channel might still be in a transient failure
-      // state.
-      LocalAssetsStreamManagerClient new_client(CreateChannel(service_port_));
-      status = new_client.StartSession(full_src_dir, user_host, mount_dir,
-                                       ssh_command_, sftp_command_);
-    }
-  }
-
   if (status.ok()) {
     LOG_INFO("Started streaming directory '%s' to '%s:%s'", src_dir_, user_host,
              mount_dir);
@@ -163,44 +107,4 @@ absl::Status StartCommand::Run() {
   return status;
 }
 
-absl::Status StartCommand::StartStreamingService() {
-  std::string exe_dir;
-  RETURN_IF_ERROR(path::GetExeDir(&exe_dir),
-                  "Failed to get executable directory");
-  std::string exe_path = path::Join(exe_dir, "cdc_stream");
-
-  // Try starting the service first.
-  WinProcessFactory process_factory;
-  ProcessStartInfo start_info;
-  start_info.command =
-      absl::StrFormat("%s start-service --verbosity=%i --service-port=%i",
-                      exe_path, verbosity_, service_port_);
-  start_info.flags = ProcessFlags::kDetached;
-  std::unique_ptr<Process> service_process = process_factory.Create(start_info);
-  RETURN_IF_ERROR(service_process->Start(),
-                  "Failed to start asset streaming service");
-
-  // Poll until the service becomes healthy.
-  LOG_INFO("Streaming service initializing...");
-  Stopwatch sw;
-  while (sw.ElapsedSeconds() < kServiceStartupTimeoutSec) {
-    // The channel is in some transient failure state, and it's faster to
-    // reconnect instead of waiting for it to return.
-    BackgroundServiceClient bg_client(CreateChannel(service_port_));
-    absl::Status status = bg_client.IsHealthy();
-    if (status.ok()) {
-      return absl::OkStatus();
-    }
-    LOG_DEBUG("Health check result: %s", status.ToString());
-    Util::Sleep(100);
-  }
-
-  // Kill the process.
-  service_process->Terminate();
-  return absl::DeadlineExceededError(
-      absl::StrFormat("Timed out after %0.0f seconds waiting for the asset "
-                      "streaming service to become healthy",
-                      kServiceStartupTimeoutSec));
-}
-
 }  // namespace cdc_ft

cdc_stream/start_command.h

@@ -20,10 +20,6 @@
 #include "absl/status/status.h"
 #include "cdc_stream/base_command.h"
 
-namespace grpc {
-class Channel;
-}
-
 namespace cdc_ft {
 
 // Handler for the start command. Sends an RPC call to the service to starts a
@@ -38,17 +34,13 @@ class StartCommand : public BaseCommand {
   absl::Status Run() override;
 
  private:
-  // Starts the asset streaming service.
-  absl::Status StartStreamingService();
-
   int verbosity_ = 0;
   uint16_t service_port_ = 0;
+  uint16_t ssh_port_ = 0;
   std::string ssh_command_;
-  std::string sftp_command_;
+  std::string scp_command_;
   std::string src_dir_;
   std::string user_host_dir_;
-
-  std::string deprecated_scp_command_;
 };
 
 }  // namespace cdc_ft

cdc_stream/start_service_command.cc

@@ -39,11 +39,11 @@ std::string GetLogPath(const char* log_dir, const char* log_base_name) {
 }  // namespace
 
 StartServiceCommand::StartServiceCommand(int* exit_code)
-    : BaseCommand("start-service", "Start the streaming service", exit_code) {}
+    : BaseCommand("start-service", "Start streaming service", exit_code) {}
 StartServiceCommand::~StartServiceCommand() = default;
 
 void StartServiceCommand::RegisterCommandLineFlags(lyra::command& cmd) {
-  config_file_ = "%APPDATA%\\cdc-file-transfer\\cdc_stream.json";
+  config_file_ = "%APPDATA%\\cdc-file-transfer\\assets_stream_manager.json";
   cmd.add_argument(
       lyra::opt(config_file_, "path")
           .name("--config-file")
@@ -116,7 +116,7 @@ absl::StatusOr<std::unique_ptr<Log>> StartServiceCommand::GetLogger() {
   }
 
   return std::make_unique<FileLog>(
-      level, GetLogPath(log_dir_.c_str(), "cdc_stream").c_str());
+      level, GetLogPath(log_dir_.c_str(), "assets_stream_manager").c_str());
 }
 
 // Runs the session management service and returns when it finishes.
@@ -140,13 +140,15 @@ absl::Status StartServiceCommand::RunService() {
     request.set_workstation_directory(cfg_.dev_src_dir());
     request.set_user_host(cfg_.dev_target().user_host);
     request.set_mount_dir(cfg_.dev_target().mount_dir);
+    request.set_port(cfg_.dev_target().ssh_port);
     request.set_ssh_command(cfg_.dev_target().ssh_command);
-    request.set_sftp_command(cfg_.dev_target().sftp_command);
+    request.set_scp_command(cfg_.dev_target().scp_command);
     localassetsstreammanager::StartSessionResponse response;
     RETURN_ABSL_IF_ERROR(
         session_service.StartSession(nullptr, &request, &response));
   }
-  RETURN_IF_ERROR(sm_server.Start(cfg_.service_port()));
+  RETURN_IF_ERROR(
+      sm_server.Start(SessionManagementServer::kDefaultServicePort));
   sm_server.RunUntilShutdown();
   return absl::OkStatus();
 }

cdc_stream/stop_command.cc

@@ -21,9 +21,6 @@
 #include "common/log.h"
 #include "common/path.h"
 #include "common/status_macros.h"
-#include "grpcpp/channel.h"
-#include "grpcpp/create_channel.h"
-#include "grpcpp/support/channel_arguments.h"
 #include "lyra/lyra.hpp"
 
 namespace cdc_ft {
@@ -53,7 +50,7 @@ void StopCommand::RegisterCommandLineFlags(lyra::command& cmd) {
                 std::to_string(SessionManagementServer::kDefaultServicePort)));
 
   cmd.add_argument(
-      lyra::arg(PosArgValidator(&user_host_dir_), "[user@]host:dir")
+      lyra::arg(PosArgValidator(&user_host_dir_), "[user@]host:src-dir")
           .required()
           .help("Linux host and directory to stream to"));
 }
@@ -61,23 +58,11 @@ void StopCommand::RegisterCommandLineFlags(lyra::command& cmd) {
 absl::Status StopCommand::Run() {
   LogLevel level = Log::VerbosityToLogLevel(verbosity_);
   ScopedLog scoped_log(std::make_unique<ConsoleLog>(level));
-
+  LocalAssetsStreamManagerClient client(service_port_);
-  std::string client_address = absl::StrFormat("localhost:%u", service_port_);
-  std::shared_ptr<grpc::Channel> channel = grpc::CreateCustomChannel(
-      client_address, grpc::InsecureChannelCredentials(),
-      grpc::ChannelArguments());
-
-  LocalAssetsStreamManagerClient client(channel);
 
   std::string user_host, mount_dir;
-  if (user_host_dir_ == "*") {
-    // Convenience shortcut "*" for "*:*".
-    user_host = "*";
-    mount_dir = "*";
-  } else {
   RETURN_IF_ERROR(LocalAssetsStreamManagerClient::ParseUserHostDir(
       user_host_dir_, &user_host, &mount_dir));
-  }
 
   absl::Status status = client.StopSession(user_host, mount_dir);
   if (status.ok()) {

cdc_stream/stop_service_command.cc

@@ -1,70 +0,0 @@
-// Copyright 2022 Google LLC
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "cdc_stream/stop_service_command.h"
-
-#include "absl/strings/str_format.h"
-#include "cdc_stream/background_service_client.h"
-#include "cdc_stream/session_management_server.h"
-#include "common/log.h"
-#include "grpcpp/channel.h"
-#include "grpcpp/create_channel.h"
-#include "grpcpp/support/channel_arguments.h"
-#include "lyra/lyra.hpp"
-
-namespace cdc_ft {
-
-StopServiceCommand::StopServiceCommand(int* exit_code)
-    : BaseCommand("stop-service", "Stops the streaming service", exit_code) {}
-StopServiceCommand::~StopServiceCommand() = default;
-
-void StopServiceCommand::RegisterCommandLineFlags(lyra::command& cmd) {
-  verbosity_ = 2;
-  cmd.add_argument(lyra::opt(verbosity_, "num")
-                       .name("--verbosity")
-                       .help("Verbosity of the log output, default: " +
-                             std::to_string(verbosity_) +
-                             ".Increase to make logs more verbose."));
-
-  service_port_ = SessionManagementServer::kDefaultServicePort;
-  cmd.add_argument(lyra::opt(service_port_, "port")
-                       .name("--service-port")
-                       .help("Local port to use while connecting to the local "
-                             "asset stream service, default: " +
-                             std::to_string(service_port_)));
-}
-
-absl::Status StopServiceCommand::Run() {
-  LogLevel level = Log::VerbosityToLogLevel(verbosity_);
-  ScopedLog scoped_log(std::make_unique<ConsoleLog>(level));
-
-  std::string client_address = absl::StrFormat("localhost:%u", service_port_);
-  std::shared_ptr<grpc::Channel> channel = grpc::CreateCustomChannel(
-      client_address, grpc::InsecureChannelCredentials(),
-      grpc::ChannelArguments());
-
-  BackgroundServiceClient bg_client(channel);
-  absl::Status status = bg_client.Exit();
-  if (status.ok()) {
-    LOG_INFO("Stopped streaming service");
-  } else if (absl::IsUnavailable(status)) {
-    // Server wasn't running. This doesn't count as an error.
-    LOG_INFO("Streaming service already stopped");
-    return absl::OkStatus();
-  }
-
-  return status;
-}
-
-}  // namespace cdc_ft

cdc_stream/stop_service_command.h

@@ -1,44 +0,0 @@
-/*
- * Copyright 2022 Google LLC
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef CDC_STREAM_STOP_SERVICE_COMMAND_H_
-#define CDC_STREAM_STOP_SERVICE_COMMAND_H_
-
-#include <memory>
-
-#include "absl/status/status.h"
-#include "cdc_stream/base_command.h"
-
-namespace cdc_ft {
-
-// Handler for the stop-service command. Stops the asset streaming service.
-class StopServiceCommand : public BaseCommand {
- public:
-  explicit StopServiceCommand(int* exit_code);
-  ~StopServiceCommand();
-
-  // BaseCommand:
-  void RegisterCommandLineFlags(lyra::command& cmd) override;
-  absl::Status Run() override;
-
- private:
-  int verbosity_ = 0;
-  uint16_t service_port_ = 0;
-};
-
-}  // namespace cdc_ft
-
-#endif  // CDC_STREAM_STOP_SERVICE_COMMAND_H_

common/BUILD

@@ -2,40 +2,6 @@ load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test")
 
 package(default_visibility = ["//visibility:public"])
 
-cc_library(
-    name = "ansi_filter",
-    srcs = ["ansi_filter.cc"],
-    hdrs = ["ansi_filter.h"],
-)
-
-cc_test(
-    name = "ansi_filter_test",
-    srcs = ["ansi_filter_test.cc"],
-    deps = [
-        ":ansi_filter",
-        "@com_google_googletest//:gtest",
-        "@com_google_googletest//:gtest_main",
-    ],
-)
-
-cc_library(
-    name = "arch_type",
-    srcs = ["arch_type.cc"],
-    hdrs = ["arch_type.h"],
-    deps = [":platform"],
-)
-
-cc_test(
-    name = "arch_type_test",
-    srcs = ["arch_type_test.cc"],
-    deps = [
-        ":arch_type",
-        "@com_google_absl//absl/strings",
-        "@com_google_googletest//:gtest",
-        "@com_google_googletest//:gtest_main",
-    ],
-)
-
 cc_library(
     name = "buffer",
     srcs = ["buffer.cc"],
@@ -52,35 +18,6 @@ cc_test(
     ],
 )
 
-cc_library(
-    name = "build_version",
-    srcs = ["build_version.cc"],
-    hdrs = ["build_version.h"],
-    # This definition should be replaced by release flow.
-    copts = ["-DCDC_BUILD_VERSION=DEV"],
-)
-
-cc_library(
-    name = "client_socket",
-    srcs = [
-        "client_socket.cc",
-        "socket_internal.h",
-    ],
-    hdrs = ["client_socket.h"],
-    linkopts = select({
-        "//tools:windows": [
-            "/DEFAULTLIB:Ws2_32.lib",  # Sockets, e.g. recv, send, WSA*.
-        ],
-        "//conditions:default": [],
-    }),
-    deps = [
-        ":log",
-        ":socket",
-        ":status",
-        ":util",
-    ],
-)
-
 cc_library(
     name = "clock",
     srcs = ["clock.cc"],
@@ -148,16 +85,6 @@ cc_test(
     ],
 )
 
-cc_library(
-    name = "fake_socket",
-    srcs = ["fake_socket.cc"],
-    hdrs = ["fake_socket.h"],
-    deps = [
-        ":socket",
-        "@com_google_absl//absl/status",
-    ],
-)
-
 cc_library(
     name = "file_watcher",
     srcs = [
@@ -215,7 +142,6 @@ cc_library(
     deps = [
         ":clock",
         ":platform",
-        ":stopwatch",
         "@com_google_absl//absl/strings:str_format",
         "@com_google_absl//absl/synchronization",
     ],
@@ -300,6 +226,34 @@ cc_library(
     hdrs = ["platform.h"],
 )
 
+cc_library(
+    name = "port_manager",
+    srcs = ["port_manager_win.cc"],
+    hdrs = ["port_manager.h"],
+    target_compatible_with = ["@platforms//os:windows"],
+    deps = [
+        ":remote_util",
+        ":status",
+        ":stopwatch",
+        ":util",
+        "@com_google_absl//absl/status:statusor",
+    ],
+)
+
+cc_test(
+    name = "port_manager_test",
+    srcs = ["port_manager_test.cc"],
+    target_compatible_with = ["@platforms//os:windows"],
+    deps = [
+        ":port_manager",
+        ":status_test_macros",
+        ":stub_process",
+        ":test_main",
+        ":testing_clock",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
 cc_library(
     name = "process",
     srcs = ["process_win.cc"],
@@ -365,7 +319,6 @@ cc_library(
     srcs = ["remote_util.cc"],
     hdrs = ["remote_util.h"],
     deps = [
-        ":arch_type",
         ":platform",
         ":process",
         ":sdk_util",
@@ -407,7 +360,6 @@ cc_library(
         ":path",
         ":platform",
         ":status",
-        "//common:build_version",
         "@com_google_absl//absl/status",
         "@com_google_absl//absl/strings:str_format",
     ],
@@ -425,42 +377,6 @@ cc_test(
     ],
 )
 
-cc_library(
-    name = "server_socket",
-    srcs = [
-        "server_socket.cc",
-        "socket_internal.h",
-    ],
-    hdrs = ["server_socket.h"],
-    linkopts = select({
-        "//tools:windows": [
-            "/DEFAULTLIB:Ws2_32.lib",  # Sockets, e.g. recv, send, WSA*.
-        ],
-        "//conditions:default": [],
-    }),
-    deps = [
-        ":log",
-        ":socket",
-        ":status",
-        ":util",
-        "@com_google_absl//absl/status",
-        "@com_google_absl//absl/status:statusor",
-    ],
-)
-
-cc_library(
-    name = "socket",
-    srcs = ["socket.cc"],
-    hdrs = ["socket.h"],
-    deps = [
-        ":log",
-        ":platform",
-        ":status",
-        ":util",
-        "@com_google_absl//absl/status",
-    ],
-)
-
 cc_library(
     name = "stats_collector",
     srcs = ["stats_collector.cc"],

common/ansi_filter.cc

@@ -1,103 +0,0 @@
-// Copyright 2023 Google LLC
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "common/ansi_filter.h"
-
-namespace cdc_ft {
-namespace ansi_filter {
-namespace {
-enum class State {
-  kNotInSequence,
-  kDCS,  // Starting with kESC + P or kDCSI, Device Control String.
-  kCS,   // Starting with kESC + [ or kCSI,  Control Sequence.
-  kOSC,  // Starting with kESC + ] or kOSCI, Operating System Command.
-};
-
-constexpr uint8_t kBEL = 0x07;   // Terminal bell.
-constexpr uint8_t kESC = 0x1B;   // ANSI escape character.
-constexpr uint8_t kST = 0x9C;    // String Terminator.
-constexpr uint8_t kDCSI = 0x90;  // Device Control String Introducer.
-constexpr uint8_t kCSI = 0x9B;   // Control Sequence Introducer.
-constexpr uint8_t kOSCI = 0x9D;  // Operating System Command Introducer
-
-}  // namespace
-
-std::string RemoveEscapeSequences(const std::string& input) {
-  State state = State::kNotInSequence;
-  std::string result;
-
-  for (size_t n = 0; n < input.size(); ++n) {
-    uint8_t ch = static_cast<uint8_t>(input[n]);
-    uint8_t next_ch =
-        static_cast<uint8_t>(n + 1 < input.size() ? input[n + 1] : 0);
-
-    switch (state) {
-      case State::kNotInSequence:
-        // Device Control String.
-        if ((ch == kESC && next_ch == 'P') || ch == kDCSI) {
-          n += ch == kESC ? 1 : 0;
-          state = State::kDCS;
-          break;
-        }
-
-        // Control Sequence.
-        if ((ch == kESC && next_ch == '[') || ch == kCSI) {
-          n += ch == kESC ? 1 : 0;
-          state = State::kCS;
-          break;
-        }
-
-        // Operating System Command.
-        if ((ch == kESC && next_ch == ']') || ch == kOSCI) {
-          n += ch == kESC ? 1 : 0;
-          state = State::kOSC;
-          break;
-        }
-
-        // Char does not belong to control sequence.
-        result.push_back(ch);
-        break;
-
-      case State::kDCS:
-        // Device control strings are ended by kST or ESC + \.
-        if (ch == kST || (ch == kESC && next_ch == '\\')) {
-          n += ch == kESC ? 1 : 0;
-          state = State::kNotInSequence;
-        }
-        break;
-
-      case State::kCS:
-        // Control sequence initializer are ended by a byte in 0x40�0x7E.
-        // https://en.wikipedia.org/wiki/ANSI_escape_code#CSIsection
-        if (ch >= 0x40 && ch <= 0x7E) {
-          state = State::kNotInSequence;
-        }
-        break;
-
-      case State::kOSC:
-        // Operating system commands are ended by kBEL, kST or ESC + \.
-        // https://invisible-island.net/xterm/ctlseqs/ctlseqs.html#h3-Operating-System-Commands
-        if (ch == kBEL || ch == kST || (ch == kESC && next_ch == '\\')) {
-          n += ch == kESC ? 1 : 0;
-          state = State::kNotInSequence;
-        }
-        break;
-    }
-  }
-
-  return result;
-}
-
-}  // namespace ansi_filter
-}  // namespace cdc_ft

common/ansi_filter.h

@@ -1,37 +0,0 @@
-/*
- * Copyright 2023 Google LLC
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef COMMON_ANSI_FILTER_H_
-#define COMMON_ANSI_FILTER_H_
-
-#include <string>
-
-namespace cdc_ft {
-namespace ansi_filter {
-
-// Removes ANSI escape sequences from a string.
-// |input| is a string that can contain ANSI escape sequences.
-// Returns the filtered string with ANSI escape sequences removed.
-// Example: The most common escape sequence sets a color, e.g.
-//            "This \x1b[1;32merror\x1b[0m is red."
-//          The filtered output is
-//            "This error is red."
-std::string RemoveEscapeSequences(const std::string& input);
-
-}  // namespace ansi_filter
-}  // namespace cdc_ft
-
-#endif  // COMMON_ANSI_FILTER_H_

common/ansi_filter_test.cc

@@ -1,98 +0,0 @@
-// Copyright 2022 Google LLC
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "common/ansi_filter.h"
-
-#include "absl/strings/ascii.h"
-#include "gtest/gtest.h"
-
-namespace cdc_ft {
-namespace {
-
-// Actual sample output from running SSH with -tt on Windows.
-// Note the \0 after cmd.exe.
-constexpr char kSshOutput[] =
-    "\x1b[2J\x1b[?25l\x1b[m\x1b["
-    "H\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n"
-    "\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\x1b[H\x1b]0;c:"
-    "\\windows\\system32\\cmd.exe\0\a\x1b[?25h\x1b[?25'l\x1b[120X\x1b["
-    "120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b["
-    "120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b["
-    "120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b["
-    "120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b["
-    "120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b["
-    "120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b["
-    "120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b["
-    "120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b["
-    "120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b["
-    "120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b[120C\x1b[H\x1b[?25h "
-    "\x1b[H\x1b[?25l\r\nfoo";
-
-TEST(AnsiFilterTest, DoesNotExplodeOnEmptyString) {
-  EXPECT_EQ(ansi_filter::RemoveEscapeSequences(""), "");
-}
-
-TEST(AnsiFilterTest, KeepsUnescapedString) {
-  constexpr char kStr[] = "Lorem ipsum";
-  EXPECT_EQ(ansi_filter::RemoveEscapeSequences(kStr), kStr);
-}
-
-TEST(AnsiFilterTest, RemovesDeviceControlString) {
-  // Special commands for the device.
-  EXPECT_EQ(ansi_filter::RemoveEscapeSequences("foo\x1bPparams\x1b\\bar"),
-            "foobar");
-  EXPECT_EQ(ansi_filter::RemoveEscapeSequences("foo\x90params\x9c"
-                                               "bar"),
-            "foobar");
-}
-
-TEST(AnsiFilterTest, RemovesControlSequenceIntroducer) {
-  // E.g. the well-known regular ANSI color codes.
-  EXPECT_EQ(ansi_filter::RemoveEscapeSequences("foo\x1b[01;32mbar"), "foobar");
-  EXPECT_EQ(ansi_filter::RemoveEscapeSequences("foo\x9b"
-                                               "01;32mbar"),
-            "foobar");
-}
-
-TEST(AnsiFilterTest, RemovesOperatingSystemCommand) {
-  // E.g. setting the Window title.
-  // Not cool: OS commands can contain null-terminated string.
-  std::string str = "foo\x1b]0;c:\\path\\to\\foo.exe";
-  str.append(1, '\0');
-  str.append("\abar");
-  EXPECT_EQ(ansi_filter::RemoveEscapeSequences(str), "foobar");
-  EXPECT_EQ(ansi_filter::RemoveEscapeSequences("foo\x9dstring\x1b\\bar"),
-            "foobar");
-}
-
-TEST(AnsiFilterTest, RemovesRestIfNotTerminated) {
-  EXPECT_EQ(ansi_filter::RemoveEscapeSequences("foo\x1b[01;32"), "foo");
-}
-
-TEST(AnsiFilterTest, RemovesSequencesFromActualSshOutput) {
-  // Note: Can't just say str = kSshOutput because of the \0 in the string.
-  std::string str = std::string(kSshOutput, sizeof(kSshOutput) - 1);
-  std::string res = std::string(
-      absl::StripAsciiWhitespace(ansi_filter::RemoveEscapeSequences(str)));
-  EXPECT_EQ(res, "foo");
-}
-
-TEST(AnsiFilterTest, WorksForExampleFromDocumentation) {
-  std::string str = "This \x1b[1;32merror\x1b[0m is red.";
-  std::string res = std::string(ansi_filter::RemoveEscapeSequences(str));
-  EXPECT_EQ(res, "This error is red.");
-}
-
-}  // namespace
-}  // namespace cdc_ft

common/arch_type.cc

@@ -1,70 +0,0 @@
-// Copyright 2023 Google LLC
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "common/arch_type.h"
-
-#include <cassert>
-
-#include "common/platform.h"
-
-namespace cdc_ft {
-
-static constexpr char kUnhandledArchType[] = "Unhandled arch type";
-
-ArchType GetLocalArchType() {
-  // TODO(ljusten): Take CPU architecture into account.
-#if PLATFORM_WINDOWS
-  return ArchType::kWindows_x86_64;
-#elif PLATFORM_LINUX
-  return ArchType::kLinux_x86_64;
-#endif
-}
-
-bool IsWindowsArchType(ArchType arch_type) {
-  switch (arch_type) {
-    case ArchType::kWindows_x86_64:
-      return true;
-    case ArchType::kLinux_x86_64:
-      return false;
-    default:
-      assert(!kUnhandledArchType);
-      return false;
-  }
-}
-
-bool IsLinuxArchType(ArchType arch_type) {
-  switch (arch_type) {
-    case ArchType::kWindows_x86_64:
-      return false;
-    case ArchType::kLinux_x86_64:
-      return true;
-    default:
-      assert(!kUnhandledArchType);
-      return false;
-  }
-}
-
-const char* GetArchTypeStr(ArchType arch_type) {
-  switch (arch_type) {
-    case ArchType::kWindows_x86_64:
-      return "Windows_x86_64";
-    case ArchType::kLinux_x86_64:
-      return "Linux_x86_64";
-    default:
-      assert(!kUnhandledArchType);
-      return "Unknown";
-  }
-}
-
-}  // namespace cdc_ft

common/arch_type.h

@@ -1,41 +0,0 @@
-/*
- * Copyright 2023 Google LLC
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef COMMON_ARCH_TYPE_H_
-#define COMMON_ARCH_TYPE_H_
-
-namespace cdc_ft {
-
-enum class ArchType {
-  kWindows_x86_64 = 0,
-  kLinux_x86_64 = 1,
-};
-
-// Returns the arch type of the current process.
-ArchType GetLocalArchType();
-
-// Returns true if |arch_type| is a Windows operating system.
-bool IsWindowsArchType(ArchType arch_type);
-
-// Returns true if |arch_type| is a Linux operating system.
-bool IsLinuxArchType(ArchType arch_type);
-
-// Returns a human readable string for |arch_type|.
-const char* GetArchTypeStr(ArchType arch_type);
-
-}  // namespace cdc_ft
-
-#endif  // COMMON_ARCH_TYPE_H_

common/arch_type_test.cc

@@ -1,49 +0,0 @@
-// Copyright 2023 Google LLC
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "common/arch_type.h"
-
-#include "absl/strings/match.h"
-#include "gtest/gtest.h"
-
-namespace cdc_ft {
-namespace {
-
-TEST(ArchTypeTest, GetLocalArchType) {
-#if PLATFORM_WINDOWS
-  EXPECT_TRUE(IsPlatformWindows(GetLocalArchType()));
-#elif PLATFORM_LINUX
-  EXPECT_TRUE(IsPlatformLinux(GetLocalArchType()));
-#endif
-}
-
-TEST(ArchTypeTest, IsWindowsArchType) {
-  EXPECT_TRUE(IsWindowsArchType(ArchType::kWindows_x86_64));
-  EXPECT_FALSE(IsWindowsArchType(ArchType::kLinux_x86_64));
-}
-
-TEST(ArchTypeTest, IsLinuxArchType) {
-  EXPECT_FALSE(IsLinuxArchType(ArchType::kWindows_x86_64));
-  EXPECT_TRUE(IsLinuxArchType(ArchType::kLinux_x86_64));
-}
-
-TEST(ArchTypeTest, GetArchTypeStr) {
-  EXPECT_TRUE(
-      absl::StrContains(GetArchTypeStr(ArchType::kWindows_x86_64), "Windows"));
-  EXPECT_TRUE(
-      absl::StrContains(GetArchTypeStr(ArchType::kLinux_x86_64), "Linux"));
-}
-
-}  // namespace
-}  // namespace cdc_ft

common/build_version.cc

@@ -1,9 +0,0 @@
-#include "build_version.h"
-
-#ifdef CDC_BUILD_VERSION
-#define TO_STR(arg) #arg
-#define TO_STR_VALUE(arg) TO_STR(arg)
-const char* BUILD_VERSION = TO_STR_VALUE(CDC_BUILD_VERSION);
-#else
-const char* BUILD_VERSION = DEV_BUILD_VERSION;
-#endif

common/build_version.h

@@ -1,7 +0,0 @@
-#ifndef COMMON_BUILD_VERSION_H_
-#define COMMON_BUILD_VERSION_H_
-
-#define DEV_BUILD_VERSION "DEV"
-extern const char* BUILD_VERSION;
-
-#endif

common/file_watcher_win.cc

@@ -65,13 +65,7 @@ int64_t ToUnixTime(LARGE_INTEGER windows_time) {
 // Background thread to read directory changes.
 class AsyncFileWatcher {
  public:
-  enum class FileWatcherState {
+  enum class FileWatcherState { kDefault, kFailed, kRunning, kShuttingDown };
-    kDefault,      // Not started.
-    kFailed,       // Some error during watching, e.g. directory got deleted.
-                   // Will attempt to recover automatically.
-    kWatching,     // Actively watching directory.
-    kShuttingDown  // Shutdown() was called, winding watcher down.
-  };
 
   using FileAction = FileWatcherWin::FileAction;
   using FileInfo = FileWatcherWin::FileInfo;
@@ -164,17 +158,12 @@ class AsyncFileWatcher {
     return dir_recreate_count_;
   }
 
-  bool IsStarted() const ABSL_LOCKS_EXCLUDED(state_mutex_) {
+  bool IsWatching() const ABSL_LOCKS_EXCLUDED(state_mutex_) {
     absl::MutexLock mutex(&state_mutex_);
     return state_ != FileWatcherState::kDefault &&
            state_ != FileWatcherState::kShuttingDown;
   }
 
-  bool IsWatching() const ABSL_LOCKS_EXCLUDED(state_mutex_) {
-    absl::MutexLock mutex(&state_mutex_);
-    return state_ == FileWatcherState::kWatching;
-  }
-
   bool IsShuttingDown() const ABSL_LOCKS_EXCLUDED(state_mutex_) {
     absl::MutexLock mutex(&state_mutex_);
     return state_ == FileWatcherState::kShuttingDown;
@@ -333,7 +322,7 @@ class AsyncFileWatcher {
       return;
     }
 
-    MaybeSetState(FileWatcherState::kWatching);
+    MaybeSetState(FileWatcherState::kRunning);
     // Initialize handles to watch: changes in |dir_path_| and shutdown
     // events.
     HANDLE watch_handles[] = {overlapped.hEvent, shutdown_event_.Get()};
@@ -596,7 +585,7 @@ absl::Status FileWatcherWin::StartWatching(FilesChangedCb files_changed_cb,
   async_watcher_ = std::make_unique<AsyncFileWatcher>(
       dir_path_, std::move(files_changed_cb), std::move(dir_recreated_cb),
       timeout_ms, enforceLegacyReadDirectoryChangesForTesting_);
-  while (GetStatus().ok() && !IsStarted()) {
+  while (GetStatus().ok() && !IsWatching()) {
     std::this_thread::sleep_for(std::chrono::milliseconds(1));
   }
   return GetStatus();
@@ -621,10 +610,6 @@ absl::Status FileWatcherWin::StopWatching() {
   return async_status;
 }
 
-bool FileWatcherWin::IsStarted() const {
-  return async_watcher_ ? async_watcher_->IsStarted() : false;
-}
-
 bool FileWatcherWin::IsWatching() const {
   return async_watcher_ ? async_watcher_->IsWatching() : false;
 }
@@ -642,7 +627,7 @@ uint32_t FileWatcherWin::GetDirRecreateEventCountForTesting() const {
 }
 
 void FileWatcherWin::EnforceLegacyReadDirectoryChangesForTesting() {
-  assert(!IsStarted());
+  assert(!IsWatching());
   enforceLegacyReadDirectoryChangesForTesting_ = true;
 }
 

common/file_watcher_win.h

@@ -77,12 +77,7 @@ class FileWatcherWin {
   // Stops watching directory changes.
   absl::Status StopWatching() ABSL_LOCKS_EXCLUDED(modified_files_mutex_);
 
-  // Indicates whether StartWatching() was called, but StopWatching() was not
+  // Indicates whether a directory is currently watched.
-  // called yet.
-  bool IsStarted() const;
-
-  // Indicates whether a directory is actively watched for changes. In contrast
-  // to IsStarted(), returns false while the directory does not exist.
   bool IsWatching() const;
 
   // Returns the watching status.

common/file_watcher_win_test.cc

@@ -135,15 +135,6 @@ class FileWatcherParameterizedTest : public ::testing::TestWithParam<bool> {
     return changed;
   }
 
-  // Polls for a second until the watcher is running again.
-  bool WaitForRunning() const {
-    for (int n = 0; n < 1000; ++n) {
-      if (watcher_.IsWatching()) return true;
-      Util::Sleep(1);
-    }
-    return false;
-  }
-
   FileMap GetChangedFiles(size_t number_of_files) {
     FileMap modified_files;
 
@@ -210,7 +201,7 @@ TEST_P(FileWatcherParameterizedTest, DirDoesNotExist) {
   if (legacyReadDirectoryChanges_)
     watcher_.EnforceLegacyReadDirectoryChangesForTesting();
   EXPECT_NOT_OK(watcher.StartWatching([this]() { OnFilesChanged(); }));
-  EXPECT_FALSE(watcher.IsStarted());
+  EXPECT_FALSE(watcher.IsWatching());
   absl::Status status = watcher.GetStatus();
   EXPECT_NOT_OK(status);
   EXPECT_TRUE(absl::IsFailedPrecondition(status));
@@ -549,9 +540,6 @@ TEST_P(FileWatcherParameterizedTest, RecreateWatchedDir) {
   EXPECT_TRUE(watcher_.GetModifiedFiles().empty());
   EXPECT_OK(watcher_.GetStatus());
 
-  // Wait until the watcher is running again, or else we might miss the file.
-  EXPECT_TRUE(WaitForRunning());
-
   // Creation of a new file should be detected.
   EXPECT_OK(path::WriteFile(first_file_path_, kFirstData, kFirstDataSize));
 
@@ -584,9 +572,6 @@ TEST_P(FileWatcherParameterizedTest, RecreateUpperDir) {
   EXPECT_TRUE(watcher_.GetModifiedFiles().empty());
   EXPECT_OK(watcher_.GetStatus());
 
-  // Wait until the watcher is running again, or else we might miss the file.
-  EXPECT_TRUE(WaitForRunning());
-
   // Creation of a new file should be detected.
   EXPECT_OK(path::WriteFile(first_file_path_, kFirstData, kFirstDataSize));
 

common/gamelet_component.cc

@@ -18,36 +18,20 @@
 
 #include "absl/strings/str_format.h"
 #include "absl/strings/str_split.h"
-#include "common/build_version.h"
 #include "common/path.h"
 #include "common/status.h"
 
 namespace cdc_ft {
 
-GameletComponent::GameletComponent(std::string build_version,
+GameletComponent::GameletComponent(std::string filename, uint64_t size,
-                                   std::string filename, uint64_t size,
                                    time_t modified_time)
-    : build_version(build_version),
+    : filename(filename), size(size), modified_time(modified_time) {}
-      filename(filename),
-      size(size),
-      modified_time(modified_time) {}
 
 GameletComponent::~GameletComponent() = default;
 
 bool GameletComponent::operator==(const GameletComponent& other) const {
-  if (filename != other.filename) {
+  return filename == other.filename && size == other.size &&
-    return false;
+         modified_time == other.modified_time;
-  }
-
-  // If either build version is the dev version, it means that the component was
-  // built locally, so that we can't compare build versions. Fall back to
-  // comparing file_size and  modified_time.
-  if (build_version != DEV_BUILD_VERSION &&
-      build_version != DEV_BUILD_VERSION) {
-    return build_version == other.build_version;
-  }
-
-  return size == other.size && modified_time == other.modified_time;
 }
 
 bool GameletComponent::operator!=(const GameletComponent& other) const {
@@ -65,7 +49,7 @@ absl::Status GameletComponent::Get(
     absl::Status status = path::GetStats(path, &stats);
     if (!status.ok())
       return WrapStatus(status, "GetStats() failed for '%s'", path);
-    components->emplace_back(BUILD_VERSION, path::BaseName(path), stats.size,
+    components->emplace_back(path::BaseName(path), stats.size,
                              stats.modified_time);
   }
 
@@ -77,9 +61,9 @@ std::string GameletComponent::ToCommandLineArgs(
     const std::vector<GameletComponent>& components) {
   std::string args;
   for (const GameletComponent& comp : components) {
-    args += absl::StrFormat("%s%s %s %u %d", args.empty() ? "" : " ",
+    args +=
-                            comp.build_version.c_str(), comp.filename.c_str(),
+        absl::StrFormat("%s%s %u %d", args.empty() ? "" : " ",
-                            comp.size, comp.modified_time);
+                        comp.filename.c_str(), comp.size, comp.modified_time);
   }
   return args;
 }
@@ -88,9 +72,9 @@ std::string GameletComponent::ToCommandLineArgs(
 std::vector<GameletComponent> GameletComponent::FromCommandLineArgs(
     int argc, const char** argv) {
   std::vector<GameletComponent> components;
-  for (int n = 0; n + 3 < argc; n += 4) {
+  for (int n = 0; n + 2 < argc; n += 3) {
-    components.emplace_back(argv[n], argv[n + 1], std::stol(argv[n + 2]),
+    components.emplace_back(argv[n], std::stol(argv[n + 1]),
-                            std::stol(argv[n + 3]));
+                            std::stol(argv[n + 2]));
   }
   return components;
 }

common/gamelet_component.h

@@ -28,13 +28,11 @@ namespace cdc_ft {
 // The components are considered fresh if both the timestamp and the file size
 // match.
 struct GameletComponent {
-  std::string build_version;
   std::string filename;
   uint64_t size;
   int64_t modified_time;
 
-  GameletComponent(std::string build_version, std::string filename,
+  GameletComponent(std::string filename, uint64_t size, time_t modified_time);
-                   uint64_t size, time_t modified_time);
   ~GameletComponent();
 
   bool operator==(const GameletComponent& other) const;

common/gamelet_component_test.cc

@@ -15,7 +15,6 @@
 #include "common/gamelet_component.h"
 
 #include "absl/strings/str_split.h"
-#include "common/build_version.h"
 #include "common/log.h"
 #include "common/path.h"
 #include "common/status_test_macros.h"
@@ -44,14 +43,14 @@ class GameletComponentTest : public ::testing::Test {
       path::Join(base_dir_, "other", "cdc_rsync_server");
 };
 
-TEST_F(GameletComponentTest, EqualityOperators_DevelopmentVersion) {
+TEST_F(GameletComponentTest, EqualityOperators) {
   constexpr uint64_t size1 = 1001;
   constexpr uint64_t size2 = 1002;
 
   constexpr int64_t modified_time1 = 5001;
   constexpr int64_t modified_time2 = 5002;
 
-  GameletComponent a(DEV_BUILD_VERSION, "file1", size1, modified_time1);
+  GameletComponent a("file1", size1, modified_time1);
 
   GameletComponent b = a;
   EXPECT_TRUE(a == b && !(a != b));
@@ -66,38 +65,6 @@ TEST_F(GameletComponentTest, EqualityOperators_DevelopmentVersion) {
   b = a;
   b.modified_time = modified_time2;
   EXPECT_TRUE(!(a == b) && a != b);
-
-  b = a;
-  b.size = size2;
-  b.build_version = "Specified";
-  EXPECT_TRUE(!(a == b) && a != b);
-
-  a.build_version = "Specified";
-  EXPECT_TRUE(a == b && !(a != b));
-}
-
-TEST_F(GameletComponentTest, EqualityOperators_SpecifiedVersion) {
-  constexpr uint64_t size1 = 1001;
-  constexpr uint64_t size2 = 1002;
-
-  constexpr int64_t modified_time1 = 5001;
-  constexpr int64_t modified_time2 = 5002;
-
-  GameletComponent a("Specified", "file1", size1, modified_time1);
-
-  GameletComponent b = a;
-  EXPECT_TRUE(a == b && !(a != b));
-
-  b.filename = "file2";
-  EXPECT_TRUE(!(a == b) && a != b);
-
-  b = a;
-  b.size = size2;
-  EXPECT_TRUE(a == b && !(a != b));
-
-  b = a;
-  b.modified_time = modified_time2;
-  EXPECT_TRUE(a == b && !(a != b));
 }
 
 TEST_F(GameletComponentTest, GetValidComponents) {
@@ -124,30 +91,9 @@ TEST_F(GameletComponentTest, GetChangedComponents) {
 
   // Force equal timestamps, so that we don't depend on when the files were
   // actually written to everyone's drives.
-  // Also force set build_version to developer since otherwise we would skip
-  // component size check.
   ASSERT_EQ(components.size(), other_components.size());
   for (size_t n = 0; n < components.size(); ++n) {
     other_components[n].modified_time = components[n].modified_time;
-    other_components[n].build_version = DEV_BUILD_VERSION;
-
-    EXPECT_NE(components, other_components);
-  }
-}
-
-TEST_F(GameletComponentTest, GetChangedComponents_BuildVersionChanged) {
-  std::vector<GameletComponent> components;
-  EXPECT_OK(GameletComponent::Get({valid_component_path_}, &components));
-
-  std::vector<GameletComponent> other_components;
-  EXPECT_OK(GameletComponent::Get({other_component_path_}, &other_components));
-
-  ASSERT_EQ(components.size(), other_components.size());
-  for (size_t n = 0; n < components.size(); ++n) {
-    other_components[n].modified_time = components[n].modified_time;
-    other_components[n].size = components[n].size;
-    components[n].build_version = "build_version";
-    other_components[n].build_version = "other_build_version";
 
     EXPECT_NE(components, other_components);
   }

common/log.cc

@@ -126,22 +126,21 @@ void ConsoleLog::WriteLogMessage(LogLevel level, const char* file, int line,
   absl::MutexLock lock(&mutex_);
 
   // Show leaner log messages in non-verbose mode.
-  bool show_time_file_func = GetLogLevel() <= LogLevel::kDebug;
+  bool show_file_func = GetLogLevel() <= LogLevel::kDebug;
   FILE* stdfile = level >= LogLevel::kError ? stderr : stdout;
 #if PLATFORM_WINDOWS
   HANDLE hConsole = GetStdHandle(STD_OUTPUT_HANDLE);
   SetConsoleTextAttribute(hConsole, GetConsoleColor(level));
-  if (show_time_file_func) {
+  if (show_file_func) {
-    fprintf(stdfile, "%0.3f %s(%i): %s(): %s\n", stopwatch_.ElapsedSeconds(),
+    fprintf(stdfile, "%s(%i): %s(): %s\n", file, line, func, message);
-            file, line, func, message);
   } else {
     fprintf(stdfile, "%s\n", message);
   }
   SetConsoleTextAttribute(hConsole, kLightGray);
 #else
-  if (show_time_file_func) {
+  if (show_file_func) {
-    fprintf(stdfile, "%-7s %0.3f %s(%i): %s(): %s\n", GetLogLevelString(level),
+    fprintf(stdfile, "%-7s %s(%i): %s(): %s\n", GetLogLevelString(level), file,
-            stopwatch_.ElapsedSeconds(), file, line, func, message);
+            line, func, message);
   } else {
     fprintf(stdfile, "%-7s %s\n", GetLogLevelString(level), message);
   }

common/log.h

@@ -22,7 +22,6 @@
 #include "absl/strings/str_format.h"
 #include "absl/synchronization/mutex.h"
 #include "common/clock.h"
-#include "common/stopwatch.h"
 
 namespace cdc_ft {
 
@@ -121,7 +120,6 @@ class ConsoleLog : public Log {
       ABSL_LOCKS_EXCLUDED(mutex_);
 
  private:
-  Stopwatch stopwatch_;
   absl::Mutex mutex_;
 };
 

common/path.cc

@@ -219,12 +219,9 @@ absl::Status ExpandPathVariables(std::string* path) {
   *path = Util::WideToUtf8Str(wchar_expanded);
   return absl::OkStatus();
 #else
-  // Exclude command substitution. It.s not what users of this method would
-  // expect and could lead to security issues.
   wordexp_t res;
-  wordexp(path->c_str(), &res, WRDE_NOCMD);
+  wordexp(path->c_str(), &res, 0);
   if (res.we_wordc > 1) {
-    wordfree(&res);
     return absl::InvalidArgumentError(
         "Path expands to multiple results (did you use * etc. ?");
   }
@@ -294,8 +291,8 @@ std::string GetDrivePrefix(const std::string& path) {
 
   if (path[0] != '\\') {
     size_t pos = path.find(":");
-    if (pos != 1) {
+    if (pos == std::string::npos) {
-      // E.g. "\path\to\file", "path\to\file" or "user@host:file".
+      // E.g. "\path\to\file" or "path\to\file".
       return std::string();
     }
 

common/path.h

@@ -104,8 +104,8 @@ absl::Status GetKnownFolderPath(FolderId folder_id, std::string* path);
 // Expands environment path variables like %APPDATA% on Windows or ~ on Linux.
 // On Windows, variables are matched case invariantly. Unknown environment
 // variables are not changed.
-// On Linux, performs a shell-like expansion, but without command substitution.
+// On Linux, performs a shell-like expansion. Returns an error if multiple
-// Returns an error if multiple results would be returned, e.g. from *.txt.
+// results would be returned, e.g. from *.txt.
 absl::Status ExpandPathVariables(std::string* path);
 
 // Returns the environment variable with given |name| in |value|.

common/path_test.cc

@@ -302,7 +302,6 @@ TEST_F(PathTest, GetDrivePrefix) {
   EXPECT_EQ(path::GetDrivePrefix("C:\\"), "C:");
   EXPECT_EQ(path::GetDrivePrefix("C:\\dir"), "C:");
   EXPECT_EQ(path::GetDrivePrefix("C:\\dir\\file"), "C:");
-  EXPECT_EQ(path::GetDrivePrefix("host:C:\\dir\\file"), "");
 }
 #endif
 

common/port_manager.h

@@ -0,0 +1,113 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef COMMON_PORT_MANAGER_H_
+#define COMMON_PORT_MANAGER_H_
+
+#include <absl/status/statusor.h>
+
+#include <memory>
+#include <string>
+#include <unordered_set>
+
+#include "common/clock.h"
+
+namespace cdc_ft {
+
+class ProcessFactory;
+class RemoteUtil;
+class SharedMemory;
+
+// Class for reserving ports globally. Use if there can be multiple processes
+// of the same type that might request ports at the same time, e.g. multiple
+// cdc_rsync.exe processes running concurrently.
+class PortManager {
+ public:
+  // |unique_name| is a globally unique name used for shared memory to
+  // synchronize port reservation. The range of possible ports managed by this
+  // instance is [|first_port|, |last_port|]. |process_factory| is a valid
+  // pointer to a ProcessFactory instance to run processes locally.
+  // |remote_util| is a valid pointer to a RemoteUtil instance to run processes
+  // remotely.
+  PortManager(std::string unique_name, int first_port, int last_port,
+              ProcessFactory* process_factory, RemoteUtil* remote_util,
+              SystemClock* system_clock = DefaultSystemClock::GetInstance(),
+              SteadyClock* steady_clock = DefaultSteadyClock::GetInstance());
+  ~PortManager();
+
+  // Reserves a port in the range passed to the constructor. The port is
+  // released automatically upon destruction if ReleasePort() is not called
+  // explicitly.
+  // |check_remote| determines whether the remote port should be checked as
+  // well. If false, the check is skipped and a port might be returned that is
+  // still in use remotely.
+  // |remote_timeout_sec| is the timeout for finding available ports on the
+  // remote instance. Not used if |check_remote| is false.
+  // Returns a DeadlineExceeded error if the timeout is exceeded.
+  // Returns a ResourceExhausted error if no ports are available.
+  absl::StatusOr<int> ReservePort(bool check_remote, int remote_timeout_sec);
+
+  // Releases a reserved port.
+  absl::Status ReleasePort(int port);
+
+  //
+  // Lower-level interface for finding available ports directly.
+  //
+
+  // Finds available ports in the range [first_port, last_port] for port
+  // forwarding on the local workstation.
+  // |ip| is the IP address to filter by.
+  // |process_factory| is used to create a netstat process.
+  // Returns ResourceExhaustedError if no port is available.
+  static absl::StatusOr<std::unordered_set<int>> FindAvailableLocalPorts(
+      int first_port, int last_port, const char* ip,
+      ProcessFactory* process_factory);
+
+  // Finds available ports in the range [first_port, last_port] for port
+  // forwarding on the instance.
+  // |ip| is the IP address to filter by.
+  // |process_factory| is used to create a netstat process.
+  // |remote_util| is used to connect to the instance.
+  // |timeout_sec| is the connection timeout in seconds.
+  // Returns a DeadlineExceeded error if the timeout is exceeded.
+  // Returns ResourceExhaustedError if no port is available.
+  static absl::StatusOr<std::unordered_set<int>> FindAvailableRemotePorts(
+      int first_port, int last_port, const char* ip,
+      ProcessFactory* process_factory, RemoteUtil* remote_util, int timeout_sec,
+      SteadyClock* steady_clock = DefaultSteadyClock::GetInstance());
+
+ private:
+  // Returns a list of available ports in the range [|first_port|, |last_port|]
+  // from the given |netstat_output|. |ip| is the IP address to look for, e.g.
+  // "127.0.0.1".
+  // Returns ResourceExhaustedError if no port is available.
+  static absl::StatusOr<std::unordered_set<int>> FindAvailablePorts(
+      int first_port, int last_port, const std::string& netstat_output,
+      const char* ip);
+
+  int first_port_;
+  int last_port_;
+  ProcessFactory* process_factory_;
+  RemoteUtil* remote_util_;
+  SystemClock* system_clock_;
+  SteadyClock* steady_clock_;
+  std::unique_ptr<SharedMemory> shared_mem_;
+  std::unordered_set<int> reserved_ports_;
+};
+
+}  // namespace cdc_ft
+
+#endif  // COMMON_PORT_MANAGER_H_