[cdc_rsync] Add support for ServerSocket on Windows (#48)

Makes ServerSocket multi-platform, mainly by working around some small
API differences. The code is largely the same, there should be no
differences on Linux.

Also moves WSAStartup() and WSACleanup() up to the Socket level as
static methods because it's used by both ClientSocket and ServerSocket,
and because it doesn't make sense to do that in the socket class as
that would prevent one from using several sockets.

.github/workflows/build_and_test.yml

@@ -23,17 +23,26 @@ jobs:
       - name: Initialize submodules
         run: git submodule update --init --recursive
 
-      - name: Build (fastbuild)
+      - name: Create timestamp
         run: |
-          bazel build --config=linux -- //... -//third_party/...
+          printf -v date '%(%Y-%m)T' -1
+          echo "date=$date" >> $GITHUB_ENV
+
+      - name: Restore build cache
+        uses: actions/cache@v3
+        with:
+          path: bazel-cache
+          key: ${{ runner.os }}-bazel-cache-fastbuild-${{ env.date }}
+
+      - name: Build (fastbuild)
+        run: bazel build --config=linux --disk_cache=bazel-cache -- //... -//third_party/...
 
       # Skip file_finder_test: The test works when file_finder_test is run
       # directly, but not through bazel test. The reason is, bazel test
       # creates symlinks of test files, but the finder ignores symlinks.
       # Also run tests sequentially since some tests write to a common tmp dir.
       - name: Test (fastbuild)
-        run: |
-          bazel test --config=linux --test_output=errors --local_test_jobs=1 -- //... -//third_party/... -//cdc_rsync_server:file_finder_test
+        run: bazel test --config=linux --disk_cache=bazel-cache --test_output=errors --local_test_jobs=1 -- //... -//third_party/... -//cdc_rsync_server:file_finder_test
 
   Build-And-Test-Windows:
     runs-on: windows-2019
@@ -43,16 +52,26 @@ jobs:
       - name: Initialize submodules
         run: git submodule update --init --recursive
 
-      - name: Build
+      - name: Create timestamp
         run: |
-          bazel build --config=windows //cdc_rsync //cdc_stream //tests_common //tests_cdc_stream //tests_cdc_rsync
+          $date = Get-Date -Format "yyyy-MM"
+          echo "date=$date" >> $env:GITHUB_ENV
 
-      - name: Test
+      - name: Restore build cache
+        uses: actions/cache@v3
+        with:
+          path: bazel-cache
+          key: ${{ runner.os }}-bazel-cache-fastbuild-${{ env.date }}
+
+      - name: Build (fastbuild)
+        run: bazel build --config=windows --disk_cache=bazel-cache //cdc_rsync //cdc_stream //tests_common //tests_cdc_stream //tests_cdc_rsync
+
+      - name: Test (fastbuild)
         run: |
           bazel-bin\tests_common\tests_common.exe
           bazel-bin\tests_cdc_stream\tests_cdc_stream.exe
           bazel-bin\tests_cdc_rsync\tests_cdc_rsync.exe
-          bazel test --config=windows --test_output=errors --local_test_jobs=1 `
+          bazel test --config=windows --disk_cache=bazel-cache --test_output=errors --local_test_jobs=1 `
             //cdc_fuse_fs/... `
             //cdc_rsync/... `
             //cdc_rsync/base/... `

.github/workflows/create_release.yml

@@ -17,14 +17,25 @@ jobs:
       - name: Initialize submodules
         run: git submodule update --init --recursive
 
-      - name: Build
+      - name: Create timestamp
         run: |
-          bazel build --config=linux --compilation_mode=opt --linkopt=-Wl,--strip-all --copt=-fdata-sections --copt=-ffunction-sections --linkopt=-Wl,--gc-sections \
+          printf -v date '%(%Y-%m)T' -1
+          echo "date=$date" >> $GITHUB_ENV
+
+      - name: Restore build cache
+        uses: actions/cache@v3
+        with:
+          path: bazel-cache
+          key: ${{ runner.os }}-bazel-cache-opt-${{ env.date }}
+
+      - name: Build (opt)
+        run: |
+          bazel build --config=linux --disk_cache=bazel-cache --compilation_mode=opt --linkopt=-Wl,--strip-all --copt=-fdata-sections --copt=-ffunction-sections --linkopt=-Wl,--gc-sections \
             //cdc_fuse_fs //cdc_rsync_server
 
-      - name: Test
+      - name: Test (opt)
         run: |
-          bazel test --config=linux --compilation_mode=opt --linkopt=-Wl,--strip-all --copt=-fdata-sections --copt=-ffunction-sections --linkopt=-Wl,--gc-sections \
+          bazel test --config=linux --disk_cache=bazel-cache --compilation_mode=opt --linkopt=-Wl,--strip-all --copt=-fdata-sections --copt=-ffunction-sections --linkopt=-Wl,--gc-sections \
                      --test_output=errors --local_test_jobs=1  \
                      -- //... -//third_party/... -//cdc_rsync_server:file_finder_test
 
@@ -50,17 +61,28 @@ jobs:
       - name: Initialize submodules
         run: git submodule update --init --recursive
 
-      - name: Build
+      - name: Create timestamp
         run: |
-          bazel build --config=windows --compilation_mode=opt --copt=/GL `
+          $date = Get-Date -Format "yyyy-MM"
+          echo "date=$date" >> $env:GITHUB_ENV
+
+      - name: Restore build cache
+        uses: actions/cache@v3
+        with:
+          path: bazel-cache
+          key: ${{ runner.os }}-bazel-cache-opt-${{ env.date }}
+
+      - name: Build (opt)
+        run: |
+          bazel build --config=windows --disk_cache=bazel-cache --compilation_mode=opt --copt=/GL `
                        //cdc_rsync //cdc_stream //tests_common //tests_cdc_stream //tests_cdc_rsync
 
-      - name: Test
+      - name: Test (opt)
         run: |
           bazel-bin\tests_common\tests_common.exe
           bazel-bin\tests_cdc_stream\tests_cdc_stream.exe
           bazel-bin\tests_cdc_rsync\tests_cdc_rsync.exe
-          bazel test --config=windows --compilation_mode=opt --copt=/GL --test_output=errors --local_test_jobs=1 `
+          bazel test --config=windows --disk_cache=bazel-cache --compilation_mode=opt --copt=/GL --test_output=errors --local_test_jobs=1 `
             //cdc_fuse_fs/... `
             //cdc_rsync/... `
             //cdc_rsync/base/... `

.gitignore

@@ -12,3 +12,4 @@ dependencies
 .qtc_clangd
 bazel-*
 user.bazelrc
+*.pyc

README.md

@@ -167,95 +167,45 @@ scp somefile.txt user@linux.device.com:
 Here, `user` is the Linux user and `linux.device.com` is the Linux host to
 SSH into or copy the file to.
 
-If `ssh.exe` or `scp.exe` cannot be found, or if additional arguments are
-required, it is recommended to set the environment variables `CDC_SSH_COMMAND`
-and `CDC_SCP_COMMAND`. The following example specifies a custom path to the SSH
-and SCP binaries, a custom SSH config file, a key file and a known hosts file:
+If additional arguments are required, it is recommended to provide an SSH config
+file. By default, both `ssh.exe` and `scp.exe` use the file at
+`%USERPROFILE%\.ssh\config` on Windows, if it exists. A possible config file
+that sets a username, a port, an identity file and a known host file could look
+as follows:
 ```
-set CDC_SSH_COMMAND="C:\path with space\to\ssh.exe" -F C:\path\to\ssh_config -i C:\path\to\id_rsa -oStrictHostKeyChecking=yes -oUserKnownHostsFile="""C:\path\to\known_hosts"""
-set CDC_SCP_COMMAND="C:\path with space\to\scp.exe" -F C:\path\to\ssh_config -i C:\path\to\id_rsa -oStrictHostKeyChecking=yes -oUserKnownHostsFile="""C:\path\to\known_hosts"""
+Host linux_device
+	HostName linux.device.com
+	User user
+	Port 12345
+	IdentityFile C:\path\to\id_rsa
+	UserKnownHostsFile C:\path\to\known_hosts
 ```
+If `ssh.exe` or `scp.exe` cannot be found, you can specify the full paths via
+the command line arguments `--ssh-command` and `--scp-command` for `cdc_rsync`
+and `cdc_stream start` (see below), or set the environment variables
+`CDC_SSH_COMMAND` and `CDC_SCP_COMMAND`, e.g.
+```
+set CDC_SSH_COMMAND="C:\path with space\to\ssh.exe"
+set CDC_SCP_COMMAND="C:\path with space\to\scp.exe"
+```
+Note that you can also specify SSH configuration via the environment variables
+instead of using a config file:
+```
+set CDC_SSH_COMMAND=C:\path\to\ssh.exe -p 12345 -i C:\path\to\id_rsa -oUserKnownHostsFile=C:\path\to\known_hosts
+set CDC_SCP_COMMAND=C:\path\to\scp.exe -P 12345 -i C:\path\to\id_rsa -oUserKnownHostsFile=C:\path\to\known_hosts
+```
+Note the small `-p` for `ssh.exe` and the capital `-P` for `scp.exe`.
 
 #### Google Specific
 
-For Google internal usage, there are two setups, a simple one that requires
-touching the security key for every remote access, and a slightly more complex
-one that tunnels all remote access through an SSH tunnel and only requires one
-security key touch to set up the tunnel.
-
-##### Setup Requiring Security Key Touches
-
-Set the following environment variables to enable SSH authentication using a
-Google security key:
+For Google internal usage, set the following environment variables to enable SSH
+authentication using a Google security key:
 ```
 set CDC_SSH_COMMAND=C:\gnubby\bin\ssh.exe
 set CDC_SCP_COMMAND=C:\gnubby\bin\scp.exe
 ```
 Note that you will have to touch the security key multiple times during the
-first run of each tool as the Linux components have to be deployed first.
-Subsequent runs only require a single touch.
-
-##### Setup Not Requiring Security Key Touches
-
-This section explains how to run the tools without security key touch for every
-action.
-
-On Linux, generate a host key pair with
-```
-mkdir ~/sshd_runner
-ssh-keygen -f ~/sshd_runner/ssh_host_ed25519_key -N '' -t ed25519
-```
-
-On Windows, create `%USERPROFILE%\sshd_runner\known_hosts` and copy the public
-key from `~/sshd_runner/ssh_host_ed25519_key.pub` to it, using `localhost` and
-port `50000` (if the port is in use, use another port). The file should look
-similar to
-```
-[localhost]:50000 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOR1EInlod0Pm853Ew0p4eXaD3V8y9njf6EoyaPfc8ln	
-```
-
-Next, generate an auth key for the Windows machine with
-```
-ssh-keygen -f %USERPROFILE%\sshd_runner\id_ed25519 -N "" -t ed25519
-```
-Note the subtle difference `-N ''` vs `-N ""`.
-
-On Linux, create `~/sshd_runner/authorized_keys` and copy the public key from
-`%USERPROFILE%\sshd_runner\id_ed25519.pub` to it. The file should look similar
-to
-```
-ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKleiT/yRhZ9aMutOJ8XaAYX0SPUZHuS50HP4QK5kBft google\user@user-w
-```
-On Windows, open an SSH tunnel to the Linux machine through port 50000.
-This should require one gnubby touch.
-```
-C:\gnubby\bin\ssh.exe -N -L50000:localhost:50000 <user>.<location>.corp.google.com
-```
-On Linux, run the SSH server with
-```
-/usr/sbin/sshd -D -p 50000 -oHostKey=~/sshd_runner/ssh_host_ed25519_key -oAuthorizedKeysFile=~/sshd_runner/authorized_keys -oListenAddress=localhost:50000
-```
-It runs as user, not as root, and only accepts connections from localhost
-through port 50000, so it's secure.
-
-Now you should be able to run SSH on the Windows machine into the Linux device
-without security key touch
-```
-C:\Windows\System32\OpenSSH\ssh.exe -p 50000 -i %USERPROFILE%\sshd_runner\id_ed25519 -oUserKnownHostsFile=%USERPROFILE%\sshd_runner\known_hosts user@localhost
-```
-Next, set `CDC_SSH_COMMAND` and `CDC_SCP_COMMAND` to use the proper flags
-```
-set CDC_SSH_COMMAND=C:\Windows\System32\OpenSSH\ssh.exe -i %USERPROFILE%\sshd_runner\id_ed25519 -oUserKnownHostsFile=%USERPROFILE%\sshd_runner\known_hosts -p 50000
-set CDC_SCP_COMMAND=C:\Windows\System32\OpenSSH\scp.exe -i %USERPROFILE%\sshd_runner\id_ed25519 -oUserKnownHostsFile=%USERPROFILE%\sshd_runner\known_hosts -P 50000
-```
-Note the small `-p` for SSH and the capital `-P` for SCP.
-
-Now the file transfer tools should work without security key touch, but be sure
-to use `localhost` as target host as we're piping the data through the tunnel.
-```
-cdc_rsync C:\assets\* user@localhost:~/assets -vr
-cdc_stream start C:\assets user@localhost:~/assets
-```
+first run. Subsequent runs only require a single touch.
 
 ### CDC RSync
 
@@ -281,20 +231,12 @@ cdc_rsync C:\path\to\assets\* user@linux.device.com:~/assets -vr
 
 ### CDC Stream
 
-`cdc_stream` consists of a background service, which has to be started in
-advance with
-```
-cdc_stream start-service
-```
-The service logs to `%APPDATA%\cdc-file-transfer\logs` by default. Try
-`cdc_stream --help` to get a list of available flags.
-
 To stream the Windows directory `C:\path\to\assets` to `~/assets` on the Linux
 device, run
 ```
 cdc_stream start C:\path\to\assets user@linux.device.com:~/assets
 ```
-This makes all files and directories of `C:\path\to\assets` available on
+This makes all files and directories in `C:\path\to\assets` available on
 `~/assets` immediately, as if it were a local copy. However, data is streamed
 from Windows to Linux as files are accessed.
 
@@ -302,17 +244,39 @@ To stop the streaming session, enter
 ```
 cdc_stream stop user@linux.device.com:~/assets
 ```
+The command also accepts wildcards. For instance, 
+```
+cdc_stream stop user@*:*
+```
+stops all existing streaming sessions for the given user.
 
 ## Troubleshooting
 
-`cdc_rsync` always logs to the console. By default, the `cdc_stream` service
-logs to a timestamped file in `%APPDATA%\cdc-file-transfer\logs`. It can be
-switched to log to console by starting it with `--log-to-stdout`:
-```
-cdc_stream start-service --log_to_stdout
-```
+On first run, `cdc_stream` starts a background service, which does all the work.
+The `cdc_stream start` and `cdc_stream stop` commands are just RPC clients that
+talk to the service.
 
-Both `cdc_rsync` and `cdc_stream` support command line flags to control log
-verbosity. Passing `-vvv` prints debug logs, `-vvvv` prints verbose logs. The
-debug logs contain all SSH and SCP commands that are attempted to run, which is
-very useful for troubleshooting.
+The service logs to `%APPDATA%\cdc-file-transfer\logs` by default. The logs are
+useful to investigate issues with asset streaming. To pass custom arguments, or
+to debug the service, create a JSON config file at
+`%APPDATA%\cdc-file-transfer\cdc_stream.json` with command line flags.
+For instance,
+```
+{ "verbosity":3 }
+```
+instructs the service to log debug messages. Try `cdc_stream start-service -h`
+for a list of available flags. Alternatively, run the service manually with
+```
+cdc_stream start-service
+```
+and pass the flags as command line arguments. When you run the service manually,
+the flag `--log-to-stdout` is particularly useful as it logs to the console
+instead of to the file.
+
+`cdc_rsync` always logs to the console. To increase log verbosity, pass `-vvv`
+for debug logs or `-vvvv` for verbose logs.
+
+For both sync and stream, the debug logs contain all SSH and SCP commands that
+are attempted to run, which is very useful for troubleshooting. If a command
+fails unexpectedly, copy it and run it in isolation. Pass `-vv` or `-vvv` for
+additional debug output.

all_files.vcxitems

@@ -15,6 +15,7 @@
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="$(MSBuildThisFileDirectory)absl_helper\jedec_size_flag.cc" />
+    <ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync\base\socket.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_stream\asset_stream_config.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_stream\asset_stream_server.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_stream\background_service_impl.cc" />
@@ -33,6 +34,9 @@
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_stream\session_manager.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_stream\start_command.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_stream\start_service_command.cc" />
+    <ClCompile Include="$(MSBuildThisFileDirectory)cdc_stream\stop_service_command.cc" />
+    <ClCompile Include="$(MSBuildThisFileDirectory)common\port_range_parser.cc" />
+    <ClCompile Include="$(MSBuildThisFileDirectory)common\port_range_parser_test.cc" />
     <ClInclude Include="$(MSBuildThisFileDirectory)cdc_stream\stop_command.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_stream\testing_asset_stream_server.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)cdc_fuse_fs\asset.cc" />
@@ -142,6 +146,8 @@
     <ClCompile Include="$(MSBuildThisFileDirectory)metrics\messages.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)metrics\messages_test.cc" />
     <ClCompile Include="$(MSBuildThisFileDirectory)metrics\metrics.cc" />
+    <ClInclude Include="$(MSBuildThisFileDirectory)cdc_stream\stop_service_command.h" />
+    <ClInclude Include="$(MSBuildThisFileDirectory)common\port_range_parser.h" />
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="$(MSBuildThisFileDirectory)absl_helper\jedec_size_flag.h" />

cdc_rsync/BUILD

@@ -130,6 +130,7 @@ cc_library(
     hdrs = ["params.h"],
     deps = [
         ":cdc_rsync_client",
+        "//common:port_range_parser",
         "@com_github_zstd//:zstd",
         "@com_google_absl//absl/status",
     ],

cdc_rsync/base/BUILD

@@ -80,7 +80,15 @@ cc_library(
 
 cc_library(
     name = "socket",
+    srcs = ["socket.cc"],
     hdrs = ["socket.h"],
+    deps = [
+        "//common:log",
+        "//common:platform",
+        "//common:status",
+        "//common:util",
+        "@com_google_absl//absl/status",
+    ],
 )
 
 filegroup(

cdc_rsync/base/socket.cc

@@ -0,0 +1,65 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "cdc_rsync/base/socket.h"
+
+#include "common/log.h"
+#include "common/platform.h"
+#include "common/status.h"
+#include "common/util.h"
+
+#if PLATFORM_WINDOWS
+#include <winsock2.h>
+#endif
+
+namespace cdc_ft {
+
+// static
+absl::Status Socket::Initialize() {
+#if PLATFORM_WINDOWS
+  WSADATA wsaData;
+  const int result = WSAStartup(MAKEWORD(2, 2), &wsaData);
+  if (result != 0) {
+    return MakeStatus("WSAStartup() failed: %s", Util::GetWin32Error(result));
+  }
+  return absl::OkStatus();
+#elif PLATFORM_LINUX
+  return absl::OkStatus();
+#endif
+}
+
+// static
+absl::Status Socket::Shutdown() {
+#if PLATFORM_WINDOWS
+  const int result = WSACleanup();
+  if (result == SOCKET_ERROR) {
+    return MakeStatus("WSACleanup() failed: %s",
+                      Util::GetWin32Error(WSAGetLastError()));
+  }
+  return absl::OkStatus();
+#elif PLATFORM_LINUX
+  return absl::OkStatus();
+#endif
+}
+
+SocketFinalizer::~SocketFinalizer() {
+  absl::Status status = Socket::Shutdown();
+  if (!status.ok()) {
+    LOG_ERROR("Socket shutdown failed: %s", status.message())
+  }
+};
+
+}  // namespace cdc_ft

cdc_rsync/base/socket.h

@@ -26,6 +26,14 @@ class Socket {
   Socket() = default;
   virtual ~Socket() = default;
 
+  // Calls WSAStartup() on Windows, no-op on Linux.
+  // Must be called before using sockets.
+  static absl::Status Initialize();
+
+  // Calls WSACleanup() on Windows, no-op on Linux.
+  // Must be called after using sockets.
+  static absl::Status Shutdown();
+
   // Send data to the socket.
   virtual absl::Status Send(const void* buffer, size_t size) = 0;
 
@@ -40,6 +48,12 @@ class Socket {
                                size_t* bytes_received) = 0;
 };
 
+// Convenience class that calls Shutdown() on destruction. Logs on errors.
+class SocketFinalizer {
+ public:
+  ~SocketFinalizer();
+};
+
 }  // namespace cdc_ft
 
 #endif  // CDC_RSYNC_BASE_SOCKET_H_

cdc_rsync/cdc_rsync_client.cc

@@ -44,8 +44,6 @@ constexpr int kExitCodeCouldNotExecute = 126;
 // Bash exit code if binary was not found.
 constexpr int kExitCodeNotFound = 127;
 
-constexpr int kForwardPortFirst = 44450;
-constexpr int kForwardPortLast = 44459;
 constexpr char kCdcServerFilename[] = "cdc_rsync_server";
 constexpr char kRemoteToolsBinDir[] = "~/.cache/cdc-file-transfer/bin/";
 
@@ -99,13 +97,13 @@ CdcRsyncClient::CdcRsyncClient(const Options& options,
                                std::string user_host, std::string destination)
     : options_(options),
       sources_(std::move(sources)),
-      user_host_(std::move(user_host)),
       destination_(std::move(destination)),
-      remote_util_(options.verbosity, options.quiet, &process_factory_,
+      remote_util_(std::move(user_host), options.verbosity, options.quiet,
+                   &process_factory_,
                    /*forward_output_to_log=*/false),
       port_manager_("cdc_rsync_ports_f77bcdfe-368c-4c45-9f01-230c5e7e2132",
-                    kForwardPortFirst, kForwardPortLast, &process_factory_,
-                    &remote_util_),
+                    options.forward_port_first, options.forward_port_last,
+                    &process_factory_, &remote_util_),
       printer_(options.quiet, Util::IsTTY() && !options.json),
       progress_(&printer_, options.verbosity, options.json) {
   if (!options_.ssh_command.empty()) {
@@ -122,9 +120,6 @@ CdcRsyncClient::~CdcRsyncClient() {
 }
 
 absl::Status CdcRsyncClient::Run() {
-  // Initialize |remote_util_|.
-  remote_util_.SetUserHostAndPort(user_host_, options_.port);
-
   // Start the server process.
   absl::Status status = StartServer();
   if (HasTag(status, Tag::kDeployServer)) {
@@ -187,8 +182,11 @@ absl::Status CdcRsyncClient::StartServer() {
   std::string component_args = GameletComponent::ToCommandLineArgs(components);
 
   // Find available local and remote ports for port forwarding.
-  absl::StatusOr<int> port_res = port_manager_.ReservePort(
-      /*check_remote=*/false, /*remote_timeout_sec unused*/ 0);
+  // If only one port is in the given range, try that without checking.
+  int port = options_.forward_port_first;
+  if (options_.forward_port_first < options_.forward_port_last) {
+    absl::StatusOr<int> port_res =
+        port_manager_.ReservePort(options_.connection_timeout_sec);
     constexpr char kErrorMsg[] = "Failed to find available port";
     if (absl::IsDeadlineExceeded(port_res.status())) {
       // Server didn't respond in time.
@@ -196,10 +194,12 @@ absl::Status CdcRsyncClient::StartServer() {
                     Tag::kConnectionTimeout);
     }
     if (absl::IsResourceExhausted(port_res.status()))
-    return SetTag(WrapStatus(port_res.status(), kErrorMsg), Tag::kAddressInUse);
+      return SetTag(WrapStatus(port_res.status(), kErrorMsg),
+                    Tag::kAddressInUse);
     if (!port_res.ok())
       return WrapStatus(port_res.status(), "Failed to find available port");
-  int port = *port_res;
+    port = *port_res;
+  }
 
   std::string remote_server_path =
       std::string(kRemoteToolsBinDir) + kCdcServerFilename;
@@ -263,6 +263,12 @@ absl::Status CdcRsyncClient::StartServer() {
     return SetTag(MakeStatus("Redeploy server"), Tag::kDeployServer);
   }
 
+  status = Socket::Initialize();
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to initialize sockets");
+  }
+  socket_finalizer_ = std::make_unique<SocketFinalizer>();
+
   assert(is_server_listening_);
   status = socket_.Connect(port);
   if (!status.ok()) {

cdc_rsync/cdc_rsync_client.h

@@ -36,7 +36,6 @@ class ZstdStream;
 class CdcRsyncClient {
  public:
   struct Options {
-    int port = RemoteUtil::kDefaultSshPort;
     bool delete_ = false;
     bool recursive = false;
     int verbosity = 0;
@@ -51,6 +50,8 @@ class CdcRsyncClient {
     std::string copy_dest;
     int compress_level = 6;
     int connection_timeout_sec = 10;
+    int forward_port_first = 44450;
+    int forward_port_last = 44459;
     std::string ssh_command;
     std::string scp_command;
     std::string sources_dir;  // Base dir for files loaded for --files-from.
@@ -118,11 +119,11 @@ class CdcRsyncClient {
 
   Options options_;
   std::vector<std::string> sources_;
-  const std::string user_host_;
   const std::string destination_;
   WinProcessFactory process_factory_;
   RemoteUtil remote_util_;
   PortManager port_manager_;
+  std::unique_ptr<SocketFinalizer> socket_finalizer_;
   ClientSocket socket_;
   MessagePump message_pump_{&socket_, MessagePump::PacketReceivedDelegate()};
   ConsoleProgressPrinter printer_;

cdc_rsync/client_socket.cc

@@ -39,10 +39,10 @@ absl::Status MakeSocketStatus(const char* message) {
 
 }  // namespace
 
-struct SocketInfo {
+struct ClientSocketInfo {
   SOCKET socket;
 
-  SocketInfo() : socket(INVALID_SOCKET) {}
+  ClientSocketInfo() : socket(INVALID_SOCKET) {}
 };
 
 ClientSocket::ClientSocket() = default;
@@ -50,12 +50,6 @@ ClientSocket::ClientSocket() = default;
 ClientSocket::~ClientSocket() { Disconnect(); }
 
 absl::Status ClientSocket::Connect(int port) {
-  WSADATA wsaData;
-  int result = WSAStartup(MAKEWORD(2, 2), &wsaData);
-  if (result != 0) {
-    return MakeStatus("WSAStartup() failed: %i", result);
-  }
-
   addrinfo hints;
   ZeroMemory(&hints, sizeof(hints));
   hints.ai_family = AF_INET;
@@ -64,14 +58,13 @@ absl::Status ClientSocket::Connect(int port) {
 
   // Resolve the server address and port.
   addrinfo* addr_infos = nullptr;
-  result = getaddrinfo("localhost", std::to_string(port).c_str(), &hints,
+  int result = getaddrinfo("localhost", std::to_string(port).c_str(), &hints,
                            &addr_infos);
   if (result != 0) {
-    WSACleanup();
     return MakeStatus("getaddrinfo() failed: %i", result);
   }
 
-  socket_info_ = std::make_unique<SocketInfo>();
+  socket_info_ = std::make_unique<ClientSocketInfo>();
   int count = 0;
   for (addrinfo* curr = addr_infos; curr; curr = curr->ai_next, count++) {
     socket_info_->socket =
@@ -101,7 +94,6 @@ absl::Status ClientSocket::Connect(int port) {
 
   if (socket_info_->socket == INVALID_SOCKET) {
     socket_info_.reset();
-    WSACleanup();
     return MakeStatus("Unable to connect to port %i", port);
   }
 
@@ -120,7 +112,6 @@ void ClientSocket::Disconnect() {
   }
 
   socket_info_.reset();
-  WSACleanup();
 }
 
 absl::Status ClientSocket::Send(const void* buffer, size_t size) {

cdc_rsync/client_socket.h

@@ -45,7 +45,7 @@ class ClientSocket : public Socket {
                        size_t* bytes_received) override;
 
  private:
-  std::unique_ptr<struct SocketInfo> socket_info_;
+  std::unique_ptr<struct ClientSocketInfo> socket_info_;
 };
 
 }  // namespace cdc_ft

cdc_rsync/main.cc

@@ -75,9 +75,9 @@ ReturnCode TagToMessage(cdc_ft::Tag tag,
     case cdc_ft::Tag::kConnectionTimeout:
       // Server connection timed out. SSH probably stale.
       *msg = absl::StrFormat(
-          "Server connection timed out. Verify that host '%s' and port '%i' "
-          "are correct, or specify a larger timeout with --contimeout.",
-          params.user_host, params.options.port);
+          "Server connection timed out. Verify that the host '%s' "
+          "is correct, or specify a larger timeout with --contimeout.",
+          params.user_host);
       return ReturnCode::kConnectionTimeout;
 
     case cdc_ft::Tag::kCount:

cdc_rsync/params.cc

@@ -20,6 +20,7 @@
 #include "absl/strings/str_format.h"
 #include "absl/strings/str_split.h"
 #include "common/path.h"
+#include "common/port_range_parser.h"
 #include "lib/zstd.h"
 
 namespace cdc_ft {
@@ -51,8 +52,6 @@ Parameters:
   destination               Remote destination directory
 
 Options:
-    --ip string           Gamelet IP. Required.
-    --port number         SSH port to use. Required.
     --contimeout sec        Gamelet connection timeout in seconds (default: 10)
 -q, --quiet                 Quiet mode, only print errors
 -v, --verbose               Increase output verbosity
@@ -61,24 +60,26 @@ Options:
 -r, --recursive             Recurse into directories
     --delete                Delete extraneous files from destination directory
 -z, --compress              Compress file data during the transfer
-    --compress-level num  Explicitly set compression level (default: 6)
+    --compress-level <num>  Explicitly set compression level (default: 6)
 -c, --checksum              Skip files based on checksum, not mod-time & size
 -W, --whole-file            Always copy files whole,
                             do not apply delta-transfer algorithm
     --exclude pattern       Exclude files matching pattern
-    --exclude-from file   Read exclude patterns from file
+    --exclude-from <file>   Read exclude patterns from file
     --include pattern       Don't exclude files matching pattern
-    --include-from file   Read include patterns from file
-    --files-from file     Read list of source files from file
+    --include-from <file>   Read include patterns from file
+    --files-from <file>     Read list of source files from file
 -R, --relative              Use relative path names
     --existing              Skip creating new files on instance
-    --copy-dest dir       Use files from dir as sync base if files are missing
-    --ssh-command         Path and arguments of ssh command to use, e.g.
-                          C:\path\to\ssh.exe -F config -i id_rsa -oStrictHostKeyChecking=yes -oUserKnownHostsFile="""known_hosts"""
+    --copy-dest <dir>       Use files from dir as sync base if files are missing
+    --ssh-command <cmd>     Path and arguments of ssh command to use, e.g.
+                            "C:\path\to\ssh.exe -p 12345 -i id_rsa -oUserKnownHostsFile=known_hosts"
                             Can also be specified by the CDC_SSH_COMMAND environment variable.
-    --scp-command         Path and arguments of scp command to use, e.g.
-                          C:\path\to\scp.exe -F config -i id_rsa -oStrictHostKeyChecking=yes -oUserKnownHostsFile="""known_hosts"""
+    --scp-command <cmd>     Path and arguments of scp command to use, e.g.
+                            "C:\path\to\scp.exe -P 12345 -i id_rsa -oUserKnownHostsFile=known_hosts"
                             Can also be specified by the CDC_SCP_COMMAND environment variable.
+    --forward-port <port>   TCP port or range used for SSH port forwarding (default: 44450-44459).
+                            If a range is specified, searches for available ports (slower).
 -h  --help                  Help for cdc_rsync
 )";
 
@@ -93,15 +94,20 @@ void PopulateFromEnvVars(Parameters* parameters) {
       .IgnoreError();
 }
 
+// Returns false and prints an error if |value| is null or empty.
+bool ValidateValue(const std::string& option_name, const char* value) {
+  if (!value) {
+    PrintError("Option '%s' needs a value", option_name);
+    return false;
+  }
+  return true;
+}
+
 // Handles the --exclude-from and --include-from options.
 OptionResult HandleFilterRuleFile(const std::string& option_name,
                                   const char* path, PathFilter::Rule::Type type,
                                   Parameters* params) {
-  if (!path) {
-    PrintError("Option '%s' needs a value", option_name);
-    return OptionResult::kError;
-  }
-
+  assert(path);
   std::vector<std::string> patterns;
   absl::Status status = path::ReadAllLines(
       path, &patterns,
@@ -164,13 +170,6 @@ bool LoadFilesFrom(const std::string& files_from,
 
 OptionResult HandleParameter(const std::string& key, const char* value,
                              Parameters* params, bool* help) {
-  if (key == "port") {
-    if (value) {
-      params->options.port = atoi(value);
-    }
-    return OptionResult::kConsumedKeyValue;
-  }
-
   if (key == "delete") {
     params->options.delete_ = true;
     return OptionResult::kConsumedKey;
@@ -197,29 +196,34 @@ OptionResult HandleParameter(const std::string& key, const char* value,
   }
 
   if (key == "include") {
+    if (!ValidateValue(key, value)) return OptionResult::kError;
     params->options.filter.AddRule(PathFilter::Rule::Type::kInclude, value);
     return OptionResult::kConsumedKeyValue;
   }
 
   if (key == "include-from") {
+    if (!ValidateValue(key, value)) return OptionResult::kError;
     return HandleFilterRuleFile(key, value, PathFilter::Rule::Type::kInclude,
                                 params);
   }
 
   if (key == "exclude") {
+    if (!ValidateValue(key, value)) return OptionResult::kError;
     params->options.filter.AddRule(PathFilter::Rule::Type::kExclude, value);
     return OptionResult::kConsumedKeyValue;
   }
 
   if (key == "exclude-from") {
+    if (!ValidateValue(key, value)) return OptionResult::kError;
     return HandleFilterRuleFile(key, value, PathFilter::Rule::Type::kExclude,
                                 params);
   }
 
   if (key == "files-from") {
     // Implies -R.
+    if (!ValidateValue(key, value)) return OptionResult::kError;
     params->options.relative = true;
-    params->files_from = value ? value : std::string();
+    params->files_from = value;
     return OptionResult::kConsumedKeyValue;
   }
 
@@ -234,16 +238,14 @@ OptionResult HandleParameter(const std::string& key, const char* value,
   }
 
   if (key == "compress-level") {
-    if (value) {
+    if (!ValidateValue(key, value)) return OptionResult::kError;
     params->options.compress_level = atoi(value);
-    }
     return OptionResult::kConsumedKeyValue;
   }
 
   if (key == "contimeout") {
-    if (value) {
+    if (!ValidateValue(key, value)) return OptionResult::kError;
     params->options.connection_timeout_sec = atoi(value);
-    }
     return OptionResult::kConsumedKeyValue;
   }
 
@@ -268,7 +270,8 @@ OptionResult HandleParameter(const std::string& key, const char* value,
   }
 
   if (key == "copy-dest") {
-    params->options.copy_dest = value ? value : std::string();
+    if (!ValidateValue(key, value)) return OptionResult::kError;
+    params->options.copy_dest = value;
     return OptionResult::kConsumedKeyValue;
   }
 
@@ -278,12 +281,27 @@ OptionResult HandleParameter(const std::string& key, const char* value,
   }
 
   if (key == "ssh-command") {
-    params->options.ssh_command = value ? value : std::string();
+    if (!ValidateValue(key, value)) return OptionResult::kError;
+    params->options.ssh_command = value;
     return OptionResult::kConsumedKeyValue;
   }
 
   if (key == "scp-command") {
-    params->options.scp_command = value ? value : std::string();
+    if (!ValidateValue(key, value)) return OptionResult::kError;
+    params->options.scp_command = value;
+    return OptionResult::kConsumedKeyValue;
+  }
+
+  if (key == "forward-port") {
+    if (!ValidateValue(key, value)) return OptionResult::kError;
+    uint16_t first, last;
+    if (!port_range::Parse(value, &first, &last)) {
+      PrintError("Failed to parse %s=%s, expected <port> or <port1>-<port2>",
+                 key, value);
+      return OptionResult::kError;
+    }
+    params->options.forward_port_first = first;
+    params->options.forward_port_last = last;
     return OptionResult::kConsumedKeyValue;
   }
 
@@ -302,11 +320,6 @@ bool ValidateParameters(const Parameters& params, bool help) {
     return false;
   }
 
-  if (params.options.port <= 0 || params.options.port > UINT16_MAX) {
-    PrintError("--port must specify a valid port");
-    return false;
-  }
-
   // Note: ZSTD_minCLevel() is ridiculously small (-131072), so use a
   // reasonable value.
   assert(ZSTD_minCLevel() <= Options::kMinCompressLevel);
@@ -369,11 +382,7 @@ bool CheckOptionResult(OptionResult result, const std::string& name,
       return true;
 
     case OptionResult::kConsumedKeyValue:
-      if (!value) {
-        PrintError("Option '%s' needs a value", name);
-        return false;
-      }
-      return true;
+      return ValidateValue(name, value);
 
     case OptionResult::kError:
       // Error message was already printed.

cdc_rsync/params_test.cc

@@ -97,7 +97,6 @@ class ParamsTest : public ::testing::Test {
 TEST_F(ParamsTest, ParseSucceedsDefaults) {
   const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst, NULL};
   EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
-  EXPECT_EQ(RemoteUtil::kDefaultSshPort, parameters_.options.port);
   EXPECT_FALSE(parameters_.options.delete_);
   EXPECT_FALSE(parameters_.options.recursive);
   EXPECT_EQ(0, parameters_.options.verbosity);
@@ -145,13 +144,6 @@ TEST_F(ParamsTest, ParseFailsOnCompressLevelEqualsNoValue) {
   ExpectError(NeedsValueError("compress-level"));
 }
 
-TEST_F(ParamsTest, ParseFailsOnPortEqualsNoValue) {
-  const char* argv[] = {"cdc_rsync.exe", "--port=", kSrc, kUserHostDst, NULL};
-  EXPECT_FALSE(
-      Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
-  ExpectError(NeedsValueError("port"));
-}
-
 TEST_F(ParamsTest, ParseFailsOnContimeoutEqualsNoValue) {
   const char* argv[] = {"cdc_rsync.exe", "--contimeout=", kSrc, kUserHostDst,
                         NULL};
@@ -285,13 +277,18 @@ TEST_F(ParamsTest, ParseFailsOnUnknownKey) {
 }
 
 TEST_F(ParamsTest, ParseSucceedsWithSupportedKeyValue) {
-  const char* argv[] = {
-      "cdc_rsync.exe", "--compress-level", "11", "--contimeout", "99", "--port",
-      "4086",          "--copy-dest=dest", kSrc, kUserHostDst,   NULL};
+  const char* argv[] = {"cdc_rsync.exe",
+                        "--compress-level",
+                        "11",
+                        "--contimeout",
+                        "99",
+                        "--copy-dest=dest",
+                        kSrc,
+                        kUserHostDst,
+                        NULL};
   EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
   EXPECT_EQ(parameters_.options.compress_level, 11);
   EXPECT_EQ(parameters_.options.connection_timeout_sec, 99);
-  EXPECT_EQ(parameters_.options.port, 4086);
   EXPECT_EQ(parameters_.options.copy_dest, "dest");
   ExpectNoError();
 }
@@ -304,13 +301,6 @@ TEST_F(ParamsTest, ParseSucceedsWithSupportedKeyValueWithoutEqualityForChars) {
   ExpectNoError();
 }
 
-TEST_F(ParamsTest, ParseFailsOnInvalidPort) {
-  const char* argv[] = {"cdc_rsync.exe", "--port=0", kSrc, kUserHostDst, NULL};
-  EXPECT_FALSE(
-      Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
-  ExpectError("--port must specify a valid port");
-}
-
 TEST_F(ParamsTest, ParseFailsOnDeleteNeedsRecursive) {
   const char* argv[] = {"cdc_rsync.exe", "--delete", kSrc, kUserHostDst, NULL};
   EXPECT_FALSE(
@@ -546,6 +536,40 @@ TEST_F(ParamsTest, IncludeExcludeMixed_ProperOrder) {
   ExpectNoError();
 }
 
+TEST_F(ParamsTest, ForwardPort_Single) {
+  const char* argv[] = {"cdc_rsync.exe", "--forward-port=65535", kSrc,
+                        kUserHostDst, NULL};
+  EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
+  EXPECT_EQ(parameters_.options.forward_port_first, 65535);
+  EXPECT_EQ(parameters_.options.forward_port_last, 65535);
+  ExpectNoError();
+}
+
+TEST_F(ParamsTest, ForwardPort_Range) {
+  const char* argv[] = {
+      "cdc_rsync.exe", "--forward-port", "1-2", kSrc, kUserHostDst, NULL};
+  EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
+  EXPECT_EQ(parameters_.options.forward_port_first, 1);
+  EXPECT_EQ(parameters_.options.forward_port_last, 2);
+  ExpectNoError();
+}
+
+TEST_F(ParamsTest, ForwardPort_NoValue) {
+  const char* argv[] = {"cdc_rsync.exe", "--forward-port=", kSrc, kUserHostDst,
+                        NULL};
+  EXPECT_FALSE(
+      Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
+  ExpectError(NeedsValueError("forward-port"));
+}
+
+TEST_F(ParamsTest, ForwardPort_BadValueTooSmall) {
+  const char* argv[] = {"cdc_rsync.exe", "--forward-port=0", kSrc, kUserHostDst,
+                        NULL};
+  EXPECT_FALSE(
+      Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
+  ExpectError("Failed to parse");
+}
+
 }  // namespace
 }  // namespace params
 }  // namespace cdc_ft

cdc_rsync_server/BUILD

@@ -127,11 +127,17 @@ cc_library(
     name = "server_socket",
     srcs = ["server_socket.cc"],
     hdrs = ["server_socket.h"],
-    target_compatible_with = ["@platforms//os:linux"],
+    linkopts = select({
+        "//tools:windows": [
+            "/DEFAULTLIB:Ws2_32.lib",  # Sockets, e.g. recv, send, WSA*.
+        ],
+        "//conditions:default": [],
+    }),
     deps = [
         "//cdc_rsync/base:socket",
         "//common:log",
         "//common:status",
+        "//common:util",
         "@com_google_absl//absl/status",
     ],
 )

cdc_rsync_server/cdc_rsync_server.cc

@@ -148,10 +148,7 @@ PathFilter::Rule::Type ToInternalType(
 
 CdcRsyncServer::CdcRsyncServer() = default;
 
-CdcRsyncServer::~CdcRsyncServer() {
-  message_pump_.reset();
-  socket_.reset();
-}
+CdcRsyncServer::~CdcRsyncServer() = default;
 
 bool CdcRsyncServer::CheckComponents(
     const std::vector<GameletComponent>& components) {
@@ -173,8 +170,14 @@ bool CdcRsyncServer::CheckComponents(
 }
 
 absl::Status CdcRsyncServer::Run(int port) {
+  absl::Status status = Socket::Initialize();
+  if (!status.ok()) {
+    return WrapStatus(status, "Failed to initialize sockets");
+  }
+  socket_finalizer_ = std::make_unique<SocketFinalizer>();
+
   socket_ = std::make_unique<ServerSocket>();
-  absl::Status status = socket_->StartListening(port);
+  status = socket_->StartListening(port);
   if (!status.ok()) {
     return WrapStatus(status, "Failed to start listening on port %i", port);
   }
@@ -563,7 +566,7 @@ absl::Status CdcRsyncServer::HandleSendMissingFileData() {
     // Verify that there is no directory existing with the same name.
     if (path::Exists(filepath) && path::DirExists(filepath)) {
       assert(!diff_.extraneous_dirs.empty());
-      absl::Status status = path::RemoveFile(filepath);
+      status = path::RemoveFile(filepath);
       if (!status.ok()) {
         return WrapStatus(
             status, "Failed to remove folder '%s' before creating file '%s'",

cdc_rsync_server/cdc_rsync_server.h

@@ -32,6 +32,7 @@ namespace cdc_ft {
 
 class MessagePump;
 class ServerSocket;
+class SocketFinalizer;
 
 class CdcRsyncServer {
  public:
@@ -90,6 +91,8 @@ class CdcRsyncServer {
   // Used to toggle decompression.
   void Thread_OnPackageReceived(PacketType type);
 
+  // The order determines the correct destruction order, so keep it!
+  std::unique_ptr<SocketFinalizer> socket_finalizer_;
   std::unique_ptr<ServerSocket> socket_;
   std::unique_ptr<MessagePump> message_pump_;
 

cdc_rsync_server/server_socket.cc

@@ -14,20 +14,72 @@
 
 #include "cdc_rsync_server/server_socket.h"
 
+#include "common/log.h"
+#include "common/platform.h"
+#include "common/status.h"
+#include "common/util.h"
+
+#if PLATFORM_WINDOWS
+
+#include <winsock2.h>
+
+#elif PLATFORM_LINUX
+
 #include <netinet/in.h>
 #include <sys/socket.h>
 #include <unistd.h>
 
 #include <cerrno>
 
-#include "common/log.h"
-#include "common/status.h"
+#endif
 
 namespace cdc_ft {
-
 namespace {
 
-int kInvalidFd = -1;
+#if PLATFORM_WINDOWS
+
+using SocketType = SOCKET;
+using SockAddrType = SOCKADDR;
+constexpr SocketType kInvalidSocket = INVALID_SOCKET;
+constexpr int kSocketError = SOCKET_ERROR;
+constexpr int kSendingEnd = SD_SEND;
+
+constexpr int kErrAgain = WSAEWOULDBLOCK;  // There's no EAGAIN on Windows.
+constexpr int kErrWouldBlock = WSAEWOULDBLOCK;
+constexpr int kErrAddrInUse = WSAEADDRINUSE;
+
+int GetLastError() { return WSAGetLastError(); }
+std::string GetErrorStr(int err) { return Util::GetWin32Error(err); }
+void Close(SocketType* socket) {
+  if (*socket != kInvalidSocket) {
+    closesocket(*socket);
+    *socket = kInvalidSocket;
+  }
+}
+
+// Not necessary on Windows.
+#define HANDLE_EINTR(x) (x)
+
+#elif PLATFORM_LINUX
+
+using SocketType = int;
+using SockAddrType = sockaddr;
+constexpr SocketType kInvalidSocket = -1;
+constexpr int kSocketError = -1;
+constexpr int kSendingEnd = SHUT_WR;
+
+constexpr int kErrAgain = EAGAIN;
+constexpr int kErrWouldBlock = EWOULDBLOCK;
+constexpr int kErrAddrInUse = EADDRINUSE;
+
+int GetLastError() { return errno; }
+std::string GetErrorStr(int err) { return strerror(err); }
+void Close(SocketType* socket) {
+  if (*socket != kInvalidSocket) {
+    close(*socket);
+    *socket = kInvalidSocket;
+  }
+}
 
 // Keep re-evaluating the expression |x| while it returns EINTR.
 #define HANDLE_EINTR(x)                                     \
@@ -39,10 +91,22 @@ int kInvalidFd = -1;
     eintr_wrapper_result;                                   \
   })
 
+#endif
+
+std::string GetLastErrorStr() { return GetErrorStr(GetLastError()); }
+
 }  // namespace
 
+struct ServerSocketInfo {
+  // Listening socket file descriptor (where new connections are accepted).
+  SocketType listen_sock = kInvalidSocket;
+
+  // Connection socket file descriptor (where data is sent to/received from).
+  SocketType conn_sock = kInvalidSocket;
+};
+
 ServerSocket::ServerSocket()
-    : Socket(), listen_sockfd_(kInvalidFd), conn_sockfd_(kInvalidFd) {}
+    : Socket(), socket_info_(std::make_unique<ServerSocketInfo>()) {}
 
 ServerSocket::~ServerSocket() {
   Disconnect();
@@ -50,25 +114,26 @@ ServerSocket::~ServerSocket() {
 }
 
 absl::Status ServerSocket::StartListening(int port) {
-  if (listen_sockfd_ != kInvalidFd) {
+  if (socket_info_->listen_sock != kInvalidSocket) {
     return MakeStatus("Already listening");
   }
 
   LOG_DEBUG("Open socket");
-  listen_sockfd_ = socket(AF_INET, SOCK_STREAM, 0);
-  if (listen_sockfd_ < 0) {
-    listen_sockfd_ = kInvalidFd;
-    return MakeStatus("socket() failed: %s", strerror(errno));
+  socket_info_->listen_sock = socket(AF_INET, SOCK_STREAM, 0);
+  if (socket_info_->listen_sock == kInvalidSocket) {
+    return MakeStatus("Creating listen socket failed: %s", GetLastErrorStr());
   }
 
   // If the program terminates abnormally, the socket might remain in a
   // TIME_WAIT state and report "address already in use" on bind(). Setting
   // SO_REUSEADDR works around that. See
   // https://hea-www.harvard.edu/~fine/Tech/addrinuse.html
-  int enable = 1;
-  if (setsockopt(listen_sockfd_, SOL_SOCKET, SO_REUSEADDR, &enable,
-                 sizeof(enable)) < 0) {
-    LOG_DEBUG("setsockopt() failed");
+  const int enable = 1;
+  int result =
+      setsockopt(socket_info_->listen_sock, SOL_SOCKET, SO_REUSEADDR,
+                 reinterpret_cast<const char*>(&enable), sizeof(enable));
+  if (result == kSocketError) {
+    LOG_DEBUG("Enabling address reusal failed");
   }
 
   LOG_DEBUG("Bind socket");
@@ -77,46 +142,47 @@ absl::Status ServerSocket::StartListening(int port) {
   serv_addr.sin_family = AF_INET;
   serv_addr.sin_addr.s_addr = INADDR_ANY;
   serv_addr.sin_port = htons(port);
-  if (bind(listen_sockfd_, (struct sockaddr*)&serv_addr, sizeof(serv_addr)) <
-      0) {
+
+  result = bind(socket_info_->listen_sock,
+                reinterpret_cast<const SockAddrType*>(&serv_addr),
+                sizeof(serv_addr));
+  if (result == kSocketError) {
+    int err = GetLastError();
     absl::Status status =
-        MakeStatus("bind() to port %i failed: %s", port, strerror(errno));
-    if (errno == EADDRINUSE) {
+        MakeStatus("Binding to port %i failed: %s", port, GetErrorStr(err));
+    if (err == kErrAddrInUse) {
       // Happens when two instances are run at the same time. Help callers to
       // print reasonable errors.
       status = SetTag(status, Tag::kAddressInUse);
     }
-    close(listen_sockfd_);
-    listen_sockfd_ = kInvalidFd;
-
+    Close(&socket_info_->listen_sock);
     return status;
   }
 
   LOG_DEBUG("Listen");
-  listen(listen_sockfd_, 1);
+  result = listen(socket_info_->listen_sock, 1);
+  if (result == kSocketError) {
+    int err = GetLastError();
+    Close(&socket_info_->listen_sock);
+    return MakeStatus("Listening to socket failed: %s", GetErrorStr(err));
+  }
+
   return absl::OkStatus();
 }
 
 void ServerSocket::StopListening() {
-  if (listen_sockfd_ != kInvalidFd) {
-    close(listen_sockfd_);
-    listen_sockfd_ = kInvalidFd;
-  }
-
+  Close(&socket_info_->listen_sock);
   LOG_INFO("Stopped listening.");
 }
 
 absl::Status ServerSocket::WaitForConnection() {
-  if (conn_sockfd_ != kInvalidFd) {
+  if (socket_info_->conn_sock != kInvalidSocket) {
     return MakeStatus("Already connected");
   }
 
-  sockaddr_in cli_addr;
-  socklen_t cli_len = sizeof(cli_addr);
-  conn_sockfd_ = accept(listen_sockfd_, (struct sockaddr*)&cli_addr, &cli_len);
-  if (conn_sockfd_ < 0) {
-    conn_sockfd_ = kInvalidFd;
-    return MakeStatus("accept() failed: %s", strerror(errno));
+  socket_info_->conn_sock = accept(socket_info_->listen_sock, nullptr, nullptr);
+  if (socket_info_->conn_sock == kInvalidSocket) {
+    return MakeStatus("Accepting connection failed: %s", GetLastErrorStr());
   }
 
   LOG_DEBUG("Client connected");
@@ -124,39 +190,36 @@ absl::Status ServerSocket::WaitForConnection() {
 }
 
 void ServerSocket::Disconnect() {
-  if (conn_sockfd_ != kInvalidFd) {
-    close(conn_sockfd_);
-    conn_sockfd_ = kInvalidFd;
-  }
-
+  Close(&socket_info_->conn_sock);
   LOG_INFO("Disconnected");
 }
 
 absl::Status ServerSocket::ShutdownSendingEnd() {
-  int result = shutdown(conn_sockfd_, SHUT_WR);
-  if (result != 0) {
-    return MakeStatus("shutdown() failed: %s", strerror(errno));
+  int result = shutdown(socket_info_->conn_sock, kSendingEnd);
+  if (result == kSocketError) {
+    return MakeStatus("Socket shutdown failed: %s", GetLastErrorStr());
   }
 
   return absl::OkStatus();
 }
 
 absl::Status ServerSocket::Send(const void* buffer, size_t size) {
-  const uint8_t* curr_ptr = reinterpret_cast<const uint8_t*>(buffer);
-  ssize_t bytes_left = size;
+  const char* curr_ptr = reinterpret_cast<const char*>(buffer);
+  assert(size <= INT_MAX);
+  int bytes_left = static_cast<int>(size);
   while (bytes_left > 0) {
-    ssize_t bytes_written =
-        HANDLE_EINTR(send(conn_sockfd_, curr_ptr, bytes_left, /*flags*/ 0));
+    int bytes_written = HANDLE_EINTR(
+        send(socket_info_->conn_sock, curr_ptr, bytes_left, /*flags*/ 0));
 
     if (bytes_written < 0) {
-      if (errno == EAGAIN || errno == EWOULDBLOCK) {
+      const int err = GetLastError();
+      if (err == kErrAgain || err == kErrWouldBlock) {
         // Shouldn't happen as the socket should be blocking.
         LOG_DEBUG("Socket would block");
         continue;
       }
 
-      return MakeStatus("write() to fd %i failed: %s", conn_sockfd_,
-                        strerror(errno));
+      return MakeStatus("Sending to socket failed: %s", GetErrorStr(err));
     }
 
     bytes_left -= bytes_written;
@@ -173,21 +236,22 @@ absl::Status ServerSocket::Receive(void* buffer, size_t size,
     return absl::OkStatus();
   }
 
-  uint8_t* curr_ptr = reinterpret_cast<uint8_t*>(buffer);
-  ssize_t bytes_left = size;
+  char* curr_ptr = static_cast<char*>(buffer);
+  assert(size <= INT_MAX);
+  int bytes_left = size;
   while (bytes_left > 0) {
-    ssize_t bytes_read =
-        HANDLE_EINTR(recv(conn_sockfd_, curr_ptr, bytes_left, /*flags*/ 0));
+    int bytes_read = HANDLE_EINTR(
+        recv(socket_info_->conn_sock, curr_ptr, bytes_left, /*flags*/ 0));
 
     if (bytes_read < 0) {
-      if (errno == EAGAIN || errno == EWOULDBLOCK) {
+      const int err = GetLastError();
+      if (err == kErrAgain || err == kErrWouldBlock) {
         // Shouldn't happen as the socket should be blocking.
         LOG_DEBUG("Socket would block");
         continue;
       }
 
-      return MakeStatus("recv() from fd %i failed: %s", conn_sockfd_,
-                        strerror(errno));
+      return MakeStatus("Receiving from socket failed: %s", GetErrorStr(err));
     }
 
     bytes_left -= bytes_read;
@@ -196,7 +260,7 @@ absl::Status ServerSocket::Receive(void* buffer, size_t size,
 
     if (bytes_read == 0) {
       // EOF. Make sure we're not in the middle of a message.
-      if (bytes_left < static_cast<ssize_t>(size)) {
+      if (bytes_left < static_cast<int>(size)) {
         return MakeStatus("EOF after partial read");
       }
 

cdc_rsync_server/server_socket.h

@@ -50,11 +50,7 @@ class ServerSocket : public Socket {
                        size_t* bytes_received) override;
 
  private:
-  // Listening socket file descriptor (where new connections are accepted).
-  int listen_sockfd_;
-
-  // Connection socket file descriptor (where data is sent to/received from).
-  int conn_sockfd_;
+  std::unique_ptr<struct ServerSocketInfo> socket_info_;
 };
 
 }  // namespace cdc_ft

cdc_stream/BUILD

@@ -12,6 +12,7 @@ cc_binary(
         ":start_command",
         ":start_service_command",
         ":stop_command",
+        ":stop_service_command",
         "//common:log",
         "//common:path",
     ],
@@ -23,6 +24,7 @@ cc_library(
     hdrs = ["base_command.h"],
     deps = [
         "//absl_helper:jedec_size_flag",
+        "//common:port_range_parser",
         "@com_github_lyra//:lyra",
         "@com_google_absl//absl/status",
         "@com_google_absl//absl/strings:str_format",
@@ -40,11 +42,24 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "stop_service_command",
+    srcs = ["stop_service_command.cc"],
+    hdrs = ["stop_service_command.h"],
+    deps = [
+        ":asset_stream_config",
+        ":background_service_client",
+        ":base_command",
+        ":session_management_server",
+    ],
+)
+
 cc_library(
     name = "start_command",
     srcs = ["start_command.cc"],
     hdrs = ["start_command.h"],
     deps = [
+        ":background_service_client",
         ":base_command",
         ":local_assets_stream_manager_client",
         ":session_management_server",
@@ -78,6 +93,18 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "background_service_client",
+    srcs = ["background_service_client.cc"],
+    hdrs = ["background_service_client.h"],
+    deps = [
+        "//common:grpc_status",
+        "//common:status_macros",
+        "//proto:background_service_grpc_proto",
+        "@com_google_absl//absl/status",
+    ],
+)
+
 cc_library(
     name = "asset_stream_server",
     srcs = [
@@ -112,6 +139,8 @@ cc_library(
     deps = [
         ":base_command",
         ":multi_session",
+        ":session_management_server",
+        "//absl_helper:jedec_size_flag",
         "//common:log",
         "//common:path",
         "//common:status_macros",
@@ -180,6 +209,7 @@ cc_library(
         "//common:file_watcher",
         "//common:log",
         "//common:path",
+        "//common:path_filter",
         "//common:port_manager",
         "//common:process",
         "//common:remote_util",

cdc_stream/asset_stream_config.cc

@@ -20,6 +20,8 @@
 #include "absl/strings/str_join.h"
 #include "absl_helper/jedec_size_flag.h"
 #include "cdc_stream/base_command.h"
+#include "cdc_stream/multi_session.h"
+#include "cdc_stream/session_management_server.h"
 #include "common/buffer.h"
 #include "common/path.h"
 #include "common/status_macros.h"
@@ -41,6 +43,27 @@ AssetStreamConfig::~AssetStreamConfig() = default;
 
 void AssetStreamConfig::RegisterCommandLineFlags(lyra::command& cmd,
                                                  BaseCommand& base_command) {
+  service_port_ = SessionManagementServer::kDefaultServicePort;
+  cmd.add_argument(lyra::opt(service_port_, "port")
+                       .name("--service-port")
+                       .help("Local port to use while connecting to the local "
+                             "asset stream service, default: " +
+                             std::to_string(service_port_)));
+
+  session_cfg_.forward_port_first = MultiSession::kDefaultForwardPortFirst;
+  session_cfg_.forward_port_last = MultiSession::kDefaultForwardPortLast;
+  cmd.add_argument(
+      lyra::opt(base_command.PortRangeParser("--forward-port",
+                                             &session_cfg_.forward_port_first,
+                                             &session_cfg_.forward_port_last),
+                "port")
+          .name("--forward-port")
+          .help("TCP port or range used for SSH port forwarding, default: " +
+                std::to_string(MultiSession::kDefaultForwardPortFirst) + "-" +
+                std::to_string(MultiSession::kDefaultForwardPortLast) +
+                ". If a range is specified, searches for available ports "
+                "(slower)."));
+
   session_cfg_.verbosity = kDefaultVerbosity;
   cmd.add_argument(lyra::opt(session_cfg_.verbosity, "num")
                        .name("--verbosity")
@@ -127,14 +150,6 @@ void AssetStreamConfig::RegisterCommandLineFlags(lyra::command& cmd,
           .name("--dev-user-host")
           .help("Username and host to stream to. See also --dev-src-dir."));
 
-  dev_target_.ssh_port = RemoteUtil::kDefaultSshPort;
-  cmd.add_argument(
-      lyra::opt(dev_target_.ssh_port, "port")
-          .name("--dev-ssh-port")
-          .help("SSH port to use for the connection to the host, default: " +
-                std::to_string(RemoteUtil::kDefaultSshPort) +
-                ". See also --dev-src-dir."));
-
   cmd.add_argument(
       lyra::opt(dev_target_.ssh_command, "cmd")
           .name("--dev-ssh-command")
@@ -174,6 +189,9 @@ absl::Status AssetStreamConfig::LoadFromFile(const std::string& path) {
     }                                     \
   } while (0)
 
+  ASSIGN_VAR(service_port_, "service-port", Int);
+  ASSIGN_VAR(session_cfg_.forward_port_first, "forward-port-first", Int);
+  ASSIGN_VAR(session_cfg_.forward_port_last, "forward-port-last", Int);
   ASSIGN_VAR(session_cfg_.verbosity, "verbosity", Int);
   ASSIGN_VAR(session_cfg_.fuse_debug, "debug", Bool);
   ASSIGN_VAR(session_cfg_.fuse_singlethreaded, "singlethreaded", Bool);
@@ -212,6 +230,9 @@ absl::Status AssetStreamConfig::LoadFromFile(const std::string& path) {
 
 std::string AssetStreamConfig::ToString() {
   std::ostringstream ss;
+  ss << "service-port                 = " << service_port_ << std::endl;
+  ss << "forward-port                 = " << session_cfg_.forward_port_first
+     << "-" << session_cfg_.forward_port_last << std::endl;
   ss << "verbosity                    = " << session_cfg_.verbosity
      << std::endl;
   ss << "debug                        = " << session_cfg_.fuse_debug
@@ -235,7 +256,6 @@ std::string AssetStreamConfig::ToString() {
      << session_cfg_.file_change_wait_duration_ms << std::endl;
   ss << "dev-src-dir                  = " << dev_src_dir_ << std::endl;
   ss << "dev-user-host                = " << dev_target_.user_host << std::endl;
-  ss << "dev-ssh-port                 = " << dev_target_.ssh_port << std::endl;
   ss << "dev-ssh-command              = " << dev_target_.ssh_command
      << std::endl;
   ss << "dev-scp-command              = " << dev_target_.scp_command

cdc_stream/asset_stream_config.h

@@ -48,18 +48,21 @@ class AssetStreamConfig {
   // Loads a configuration from the JSON file at |path| and overrides any config
   // values that are set in this file. Sample json file:
   // {
+  //   "service-port":44432
+  //   "forward-port-first":"44433"
+  //   "forward-port-last":"44442"
   //   "verbosity":3,
   //   "debug":0,
   //   "singlethreaded":0,
   //   "stats":0,
   //   "quiet":0,
   //   "check":0,
-  //   "log_to_stdout":0,
-  //   "cache_capacity":"150G",
-  //   "cleanup_timeout":300,
-  //   "access_idle_timeout":5,
-  //   "manifest_updater_threads":4,
-  //   "file_change_wait_duration_ms":500
+  //   "log-to-stdout":0,
+  //   "cache-capacity":"150G",
+  //   "cleanup-timeout":300,
+  //   "access-idle-timeout":5,
+  //   "manifest-updater-threads":4,
+  //   "file-change-wait-duration-ms":500
   // }
   // Returns NotFoundError if the file does not exist.
   // Returns InvalidArgumentError if the file is not valid JSON.
@@ -76,6 +79,9 @@ class AssetStreamConfig {
   // read from the JSON file.
   std::string GetFlagReadErrors();
 
+  // Gets the port to use for the asset streaming service.
+  uint16_t service_port() const { return service_port_; }
+
   // Session configuration.
   const SessionConfig& session_cfg() const { return session_cfg_; }
 
@@ -91,6 +97,13 @@ class AssetStreamConfig {
   bool log_to_stdout() const { return log_to_stdout_; }
 
  private:
+  // Jedec parser for Lyra options. Usage:
+  //   lyra::opt(JedecParser("size-flag", &size_bytes), "bytes"))
+  // Sets jedec_parse_error_ on error, Lyra doesn't support errors from lambdas.
+  std::function<void(const std::string&)> JedecParser(const char* flag_name,
+                                                      uint64_t* bytes);
+
+  uint16_t service_port_ = 0;
   SessionConfig session_cfg_;
   bool log_to_stdout_ = false;
 

cdc_stream/background_service_client.cc

@@ -0,0 +1,56 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "cdc_stream/background_service_client.h"
+
+#include "absl/status/status.h"
+#include "common/grpc_status.h"
+#include "common/status_macros.h"
+#include "grpcpp/channel.h"
+
+namespace cdc_ft {
+
+using GetPidResponse = backgroundservice::GetPidResponse;
+using EmptyProto = google::protobuf::Empty;
+
+BackgroundServiceClient::BackgroundServiceClient(
+    std::shared_ptr<grpc::Channel> channel) {
+  stub_ = BackgroundService::NewStub(std::move(channel));
+}
+
+BackgroundServiceClient::~BackgroundServiceClient() = default;
+
+absl::Status BackgroundServiceClient::Exit() {
+  EmptyProto request;
+  EmptyProto response;
+  grpc::ClientContext context;
+  return ToAbslStatus(stub_->Exit(&context, request, &response));
+}
+
+absl::StatusOr<int> BackgroundServiceClient::GetPid() {
+  EmptyProto request;
+  GetPidResponse response;
+  grpc::ClientContext context;
+  RETURN_IF_ERROR(ToAbslStatus(stub_->GetPid(&context, request, &response)));
+  return response.pid();
+}
+
+absl::Status BackgroundServiceClient::IsHealthy() {
+  EmptyProto request;
+  EmptyProto response;
+  grpc::ClientContext context;
+  return ToAbslStatus(stub_->HealthCheck(&context, request, &response));
+}
+
+}  // namespace cdc_ft

cdc_stream/background_service_client.h

@@ -0,0 +1,56 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef CDC_STREAM_BACKGROUND_SERVICE_CLIENT_H_
+#define CDC_STREAM_BACKGROUND_SERVICE_CLIENT_H_
+
+#include <memory>
+
+#include "absl/status/status.h"
+#include "absl/status/statusor.h"
+#include "proto/background_service.grpc.pb.h"
+
+namespace grpc_impl {
+class Channel;
+}
+
+namespace cdc_ft {
+
+// gRpc client for managing the asset streaming service.
+class BackgroundServiceClient {
+ public:
+  // |channel| is a grpc channel to use.
+  explicit BackgroundServiceClient(std::shared_ptr<grpc::Channel> channel);
+
+  ~BackgroundServiceClient();
+
+  // Initialize service shutdown.
+  absl::Status Exit();
+
+  // Returns the PID of the service process.
+  absl::StatusOr<int> GetPid();
+
+  // Verifies that the service is running and able to take requests.
+  absl::Status IsHealthy();
+
+ private:
+  using BackgroundService = backgroundservice::BackgroundService;
+  std::unique_ptr<BackgroundService::Stub> stub_;
+};
+
+}  // namespace cdc_ft
+
+#endif  // CDC_STREAM_BACKGROUND_SERVICE_CLIENT_H_

cdc_stream/background_service_impl.cc

@@ -23,24 +23,32 @@ namespace cdc_ft {
 
 BackgroundServiceImpl::BackgroundServiceImpl() {}
 
-BackgroundServiceImpl::~BackgroundServiceImpl() = default;
+BackgroundServiceImpl::~BackgroundServiceImpl() {
+  if (exit_thread_) {
+    exit_thread_->join();
+    exit_thread_.reset();
+  }
+}
 
 void BackgroundServiceImpl::SetExitCallback(ExitCallback exit_callback) {
   exit_callback_ = std::move(exit_callback);
 }
 
 grpc::Status BackgroundServiceImpl::Exit(grpc::ServerContext* context,
-                                         const ExitRequest* request,
-                                         ExitResponse* response) {
+                                         const EmptyProto* request,
+                                         EmptyProto* response) {
   LOG_INFO("RPC:Exit");
-  if (exit_callback_) {
-    return ToGrpcStatus(exit_callback_());
+  if (exit_callback_ && !exit_thread_) {
+    // Fire up a thread so call the callback, since shutting down a server
+    // won't finish until all RPCs are done.
+    exit_thread_ =
+        std::make_unique<std::thread>([cb = &exit_callback_]() { (*cb)(); });
   }
   return grpc::Status::OK;
 }
 
 grpc::Status BackgroundServiceImpl::GetPid(grpc::ServerContext* context,
-                                           const GetPidRequest* request,
+                                           const EmptyProto* request,
                                            GetPidResponse* response) {
   LOG_INFO("RPC:GetPid");
   response->set_pid(static_cast<int32_t>(Util::GetPid()));

cdc_stream/background_service_impl.h

@@ -17,6 +17,9 @@
 #ifndef CDC_STREAM_BACKGROUND_SERVICE_IMPL_H_
 #define CDC_STREAM_BACKGROUND_SERVICE_IMPL_H_
 
+#include <memory>
+#include <thread>
+
 #include "absl/status/status.h"
 #include "cdc_stream/background_service_impl.h"
 #include "cdc_stream/session_management_server.h"
@@ -30,9 +33,6 @@ namespace cdc_ft {
 class BackgroundServiceImpl final
     : public backgroundservice::BackgroundService::Service {
  public:
-  using ExitRequest = backgroundservice::ExitRequest;
-  using ExitResponse = backgroundservice::ExitResponse;
-  using GetPidRequest = backgroundservice::GetPidRequest;
   using GetPidResponse = backgroundservice::GetPidResponse;
   using EmptyProto = google::protobuf::Empty;
 
@@ -43,11 +43,10 @@ class BackgroundServiceImpl final
   using ExitCallback = std::function<absl::Status()>;
   void SetExitCallback(ExitCallback exit_callback);
 
-  grpc::Status Exit(grpc::ServerContext* context, const ExitRequest* request,
-                    ExitResponse* response) override;
+  grpc::Status Exit(grpc::ServerContext* context, const EmptyProto* request,
+                    EmptyProto* response) override;
 
-  grpc::Status GetPid(grpc::ServerContext* context,
-                      const GetPidRequest* request,
+  grpc::Status GetPid(grpc::ServerContext* context, const EmptyProto* request,
                       GetPidResponse* response) override;
 
   grpc::Status HealthCheck(grpc::ServerContext* context,
@@ -56,6 +55,7 @@ class BackgroundServiceImpl final
 
  private:
   ExitCallback exit_callback_;
+  std::unique_ptr<std::thread> exit_thread_;
 };
 
 }  // namespace cdc_ft

cdc_stream/base_command.cc

@@ -15,7 +15,9 @@
 #include "cdc_stream/base_command.h"
 
 #include "absl/strings/str_format.h"
+#include "absl/strings/str_split.h"
 #include "absl_helper/jedec_size_flag.h"
+#include "common/port_range_parser.h"
 #include "lyra/lyra.hpp"
 
 namespace cdc_ft {
@@ -44,8 +46,7 @@ void BaseCommand::Register(lyra::cli& cli) {
 
 std::function<void(const std::string&)> BaseCommand::JedecParser(
     const char* flag_name, uint64_t* bytes) {
-  return [flag_name, bytes,
-          error = &jedec_parse_error_](const std::string& value) {
+  return [flag_name, bytes, error = &parse_error_](const std::string& value) {
     JedecSize size;
     if (AbslParseFlag(value, &size, error)) {
       *bytes = size.Size();
@@ -56,6 +57,18 @@ std::function<void(const std::string&)> BaseCommand::JedecParser(
   };
 }
 
+std::function<void(const std::string&)> BaseCommand::PortRangeParser(
+    const char* flag_name, uint16_t* first, uint16_t* last) {
+  return [flag_name, first, last,
+          error = &parse_error_](const std::string& value) {
+    if (!port_range::Parse(value.c_str(), first, last)) {
+      *error = absl::StrFormat(
+          "Failed to parse %s=%s, expected <port> or <port1>-<port2>",
+          flag_name, value);
+    }
+  };
+}
+
 std::function<void(const std::string&)> BaseCommand::PosArgValidator(
     std::string* str) {
   return [str, invalid_arg = &invalid_arg_](const std::string& value) {
@@ -83,8 +96,8 @@ void BaseCommand::CommandHandler(const lyra::group& g) {
     return;
   }
 
-  if (!jedec_parse_error_.empty()) {
-    std::cerr << "Error: " << jedec_parse_error_ << std::endl;
+  if (!parse_error_.empty()) {
+    std::cerr << "Error: " << parse_error_ << std::endl;
     *exit_code_ = 1;
     return;
   }

cdc_stream/base_command.h

@@ -48,6 +48,13 @@ class BaseCommand {
   std::function<void(const std::string&)> JedecParser(const char* flag_name,
                                                       uint64_t* bytes);
 
+  // Parser for single ports "123" or port ranges "123-234". Usage:
+  //   lyra::opt(PortRangeParser("port-flag", &first, &last), "port"))
+  // Automatically reports a parse failure on error.
+  std::function<void(const std::string&)> PortRangeParser(const char* flag_name,
+                                                          uint16_t* first,
+                                                          uint16_t* last);
+
   // Validator that should be used for all positional arguments. Lyra interprets
   // -u, --unknown_flag as positional argument. This validator makes sure that
   // a positional argument starting with - is reported as an error. Otherwise,
@@ -82,9 +89,9 @@ class BaseCommand {
   // Extraneous positional args. Gets reported as error if present.
   std::string extra_positional_arg_;
 
-  // Errors from parsing JEDEC sizes.
+  // Errors from custom flag parsers, e.g. JEDEC sizes or port ranges.
   // Works around Lyra not accepting errors from parsers.
-  std::string jedec_parse_error_;
+  std::string parse_error_;
 };
 
 }  // namespace cdc_ft

cdc_stream/cdc_fuse_manager.cc

@@ -26,6 +26,7 @@
 namespace cdc_ft {
 namespace {
 
+constexpr char kExeFilename[] = "cdc_stream.exe";
 constexpr char kFuseFilename[] = "cdc_fuse_fs";
 constexpr char kLibFuseFilename[] = "libfuse.so";
 constexpr char kFuseStdoutPrefix[] = "cdc_fuse_fs_stdout";
@@ -95,8 +96,8 @@ absl::Status CdcFuseManager::Start(const std::string& mount_dir,
   if (!status.ok()) {
     return absl::NotFoundError(absl::StrFormat(
         "Required gamelet component not found. Make sure the files %s and %s "
-        "reside in the same folder as stadia_assets_stream_manager_v3.exe.",
-        kFuseFilename, kLibFuseFilename));
+        "reside in the same folder as %s.",
+        kFuseFilename, kLibFuseFilename, kExeFilename));
   }
   std::string component_args = GameletComponent::ToCommandLineArgs(components);
 
@@ -113,8 +114,8 @@ absl::Status CdcFuseManager::Start(const std::string& mount_dir,
       RemoteUtil::QuoteForSsh(instance_),
       RemoteUtil::QuoteForSsh(component_args), remote_port, kCacheDir,
       verbosity, cleanup_timeout_sec, access_idle_timeout_sec, enable_stats,
-      check, cache_capacity, RemoteUtil::QuoteForSsh(mount_dir),
-      debug ? " -d" : "", singlethreaded ? " -s" : "");
+      check, cache_capacity, debug ? "-d " : "", singlethreaded ? "-s " : "",
+      RemoteUtil::QuoteForSsh(mount_dir));
 
   bool needs_deploy = false;
   RETURN_IF_ERROR(

cdc_stream/cdc_stream.vcxproj

@@ -47,7 +47,7 @@
     <AdditionalOptions>/std:c++17</AdditionalOptions>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
-    <OutDir>$(SolutionDir)bazel-out\x64_windows-opt\bin\asset_stcdc_streamream_manager\</OutDir>
+    <OutDir>$(SolutionDir)bazel-out\x64_windows-opt\bin\cdc_stream\</OutDir>
     <NMakePreprocessorDefinitions>UNICODE</NMakePreprocessorDefinitions>
     <AdditionalOptions>/std:c++17</AdditionalOptions>
   </PropertyGroup>

cdc_stream/local_assets_stream_manager_client.cc

@@ -28,15 +28,6 @@ using StartSessionResponse = localassetsstreammanager::StartSessionResponse;
 using StopSessionRequest = localassetsstreammanager::StopSessionRequest;
 using StopSessionResponse = localassetsstreammanager::StopSessionResponse;
 
-LocalAssetsStreamManagerClient::LocalAssetsStreamManagerClient(
-    uint16_t service_port) {
-  std::string client_address = absl::StrFormat("localhost:%u", service_port);
-  std::shared_ptr<grpc::Channel> channel = grpc::CreateCustomChannel(
-      client_address, grpc::InsecureChannelCredentials(),
-      grpc::ChannelArguments());
-  stub_ = LocalAssetsStreamManager::NewStub(std::move(channel));
-}
-
 LocalAssetsStreamManagerClient::LocalAssetsStreamManagerClient(
     std::shared_ptr<grpc::Channel> channel) {
   stub_ = LocalAssetsStreamManager::NewStub(std::move(channel));
@@ -45,13 +36,12 @@ LocalAssetsStreamManagerClient::LocalAssetsStreamManagerClient(
 LocalAssetsStreamManagerClient::~LocalAssetsStreamManagerClient() = default;
 
 absl::Status LocalAssetsStreamManagerClient::StartSession(
-    const std::string& src_dir, const std::string& user_host, uint16_t ssh_port,
+    const std::string& src_dir, const std::string& user_host,
     const std::string& mount_dir, const std::string& ssh_command,
     const std::string& scp_command) {
   StartSessionRequest request;
   request.set_workstation_directory(src_dir);
   request.set_user_host(user_host);
-  request.set_port(ssh_port);
   request.set_mount_dir(mount_dir);
   request.set_ssh_command(ssh_command);
   request.set_scp_command(scp_command);

cdc_stream/local_assets_stream_manager_client.h

@@ -20,7 +20,6 @@
 #include <memory>
 
 #include "absl/status/status.h"
-#include "grpcpp/channel.h"
 #include "proto/local_assets_stream_manager.grpc.pb.h"
 
 namespace grpc_impl {
@@ -32,8 +31,6 @@ namespace cdc_ft {
 // gRpc client for starting/stopping asset streaming sessions.
 class LocalAssetsStreamManagerClient {
  public:
-  explicit LocalAssetsStreamManagerClient(uint16_t service_port);
-
   // |channel| is a grpc channel to use.
   explicit LocalAssetsStreamManagerClient(
       std::shared_ptr<grpc::Channel> channel);
@@ -44,12 +41,11 @@ class LocalAssetsStreamManagerClient {
   // Starting a second session to the same target will stop the first one.
   // |src_dir| is the Windows source directory to stream.
   // |user_host| is the Linux host, formatted as [user@:host].
-  // |ssh_port| is the SSH port to use while connecting to the host.
   // |mount_dir| is the Linux target directory to stream to.
   // |ssh_command| is the ssh command and extra arguments to use.
   // |scp_command| is the scp command and extra arguments to use.
   absl::Status StartSession(const std::string& src_dir,
-                            const std::string& user_host, uint16_t ssh_port,
+                            const std::string& user_host,
                             const std::string& mount_dir,
                             const std::string& ssh_command,
                             const std::string& scp_command);

cdc_stream/local_assets_stream_manager_service_impl.cc

@@ -219,12 +219,11 @@ LocalAssetsStreamManagerServiceImpl::GetTargetForStadia(
 
   // Run 'ggp ssh init' to determine IP (host) and port.
   std::string instance_ip;
-  uint16_t instance_port = 0;
-  RETURN_IF_ERROR(InitSsh(*instance_id, *project_id, *organization_id,
-                          &instance_ip, &instance_port));
+  ASSIGN_OR_RETURN(instance_ip,
+                   InitSsh(*instance_id, *project_id, *organization_id));
 
   target.user_host = "cloudcast@" + instance_ip;
-  target.ssh_port = instance_port;
+  // Note: Port must be set with ssh_command (-p) and scp_command (-P).
   return target;
 }
 
@@ -235,9 +234,6 @@ SessionTarget LocalAssetsStreamManagerServiceImpl::GetTarget(
   target.mount_dir = request.mount_dir();
   target.ssh_command = request.ssh_command();
   target.scp_command = request.scp_command();
-  target.ssh_port = request.port() > 0 && request.port() <= UINT16_MAX
-                        ? static_cast<uint16_t>(request.port())
-                        : RemoteUtil::kDefaultSshPort;
 
   *instance_id = absl::StrCat(target.user_host, ":", target.mount_dir);
   return target;
@@ -257,13 +253,10 @@ metrics::RequestOrigin LocalAssetsStreamManagerServiceImpl::ConvertOrigin(
   }
 }
 
-absl::Status LocalAssetsStreamManagerServiceImpl::InitSsh(
+absl::StatusOr<std::string> LocalAssetsStreamManagerServiceImpl::InitSsh(
     const std::string& instance_id, const std::string& project_id,
-    const std::string& organization_id, std::string* instance_ip,
-    uint16_t* instance_port) {
+    const std::string& organization_id) {
   SdkUtil sdk_util;
-  instance_ip->clear();
-  *instance_port = 0;
 
   ProcessStartInfo start_info;
   start_info.command = absl::StrFormat(
@@ -277,6 +270,7 @@ absl::Status LocalAssetsStreamManagerServiceImpl::InitSsh(
         absl::StrFormat(" --organization %s", Quoted(organization_id));
   }
   start_info.name = "ggp ssh init";
+  start_info.flags = ProcessFlags::kNoWindow;
 
   std::string output;
   start_info.stdout_handler = [&output, this](const char* data,
@@ -304,22 +298,13 @@ absl::Status LocalAssetsStreamManagerServiceImpl::InitSsh(
   }
 
   // Parse gamelet IP. Should be "Host: <instance_ip ip>".
-  if (!ParseValue(output, "Host", instance_ip)) {
+  std::string instance_ip;
+  if (!ParseValue(output, "Host", &instance_ip)) {
     return MakeStatus("Failed to parse host from ggp ssh init response\n%s",
                       output);
   }
 
-  // Parse ssh port. Should be "Port: <port>".
-  std::string port_string;
-  const bool result = ParseValue(output, "Port", &port_string);
-  int int_port = atoi(port_string.c_str());
-  if (!result || int_port == 0 || int_port <= 0 || int_port > UINT_MAX) {
-    return MakeStatus("Failed to parse ssh port from ggp ssh init response\n%s",
-                      output);
-  }
-
-  *instance_port = static_cast<uint16_t>(int_port);
-  return absl::OkStatus();
+  return instance_ip;
 }
 
 }  // namespace cdc_ft

cdc_stream/local_assets_stream_manager_service_impl.h

@@ -93,11 +93,10 @@ class LocalAssetsStreamManagerServiceImpl final
 
   // Initializes an ssh connection to a gamelet by calling 'ggp ssh init'.
   // |instance_id| must be set, |project_id|, |organization_id| are optional.
-  // Returns |instance_ip| and |instance_port| (SSH port).
-  absl::Status InitSsh(const std::string& instance_id,
+  // Returns the instance's IP address.
+  absl::StatusOr<std::string> InitSsh(const std::string& instance_id,
                                       const std::string& project_id,
-                       const std::string& organization_id,
-                       std::string* instance_ip, uint16_t* instance_port);
+                                      const std::string& organization_id);
 
   const SessionConfig cfg_;
   SessionManager* const session_manager_;

cdc_stream/main.cc

@@ -15,9 +15,31 @@
 #include "cdc_stream/start_command.h"
 #include "cdc_stream/start_service_command.h"
 #include "cdc_stream/stop_command.h"
+#include "cdc_stream/stop_service_command.h"
+#include "common/platform.h"
 #include "lyra/lyra.hpp"
 
-int main(int argc, char* argv[]) {
+#if PLATFORM_WINDOWS
+int wmain(int argc, wchar_t* wargv[]) {
+  // Convert args from wide to UTF8 strings.
+  std::vector<std::string> utf8_str_args;
+  utf8_str_args.reserve(argc);
+  for (int i = 0; i < argc; i++) {
+    utf8_str_args.push_back(cdc_ft::Util::WideToUtf8Str(wargv[i]));
+  }
+
+  // Convert args from UTF8 strings to UTF8 c-strings.
+  std::vector<const char*> utf8_args;
+  utf8_args.reserve(argc);
+  for (const auto& utf8_str_arg : utf8_str_args) {
+    utf8_args.push_back(utf8_str_arg.c_str());
+  }
+
+  const char** argv = utf8_args.data();
+#else
+int main(int argc, char** argv) {
+#endif
+
   // Set up commands.
   auto cli = lyra::cli();
   bool show_help = false;
@@ -33,6 +55,9 @@ int main(int argc, char* argv[]) {
   cdc_ft::StartServiceCommand start_service_cmd(&exit_code);
   start_service_cmd.Register(cli);
 
+  cdc_ft::StopServiceCommand stop_service_cmd(&exit_code);
+  stop_service_cmd.Register(cli);
+
   // Parse args and run. Note that parse actually runs the commands.
   // exit_code is -1 if no command was run.
   auto result = cli.parse({argc, argv});

cdc_stream/multi_session.cc

@@ -18,6 +18,7 @@
 #include "common/file_watcher_win.h"
 #include "common/log.h"
 #include "common/path.h"
+#include "common/path_filter.h"
 #include "common/platform.h"
 #include "common/port_manager.h"
 #include "common/process.h"
@@ -33,11 +34,6 @@
 namespace cdc_ft {
 namespace {
 
-// Ports used by the asset streaming service for local port forwarding on
-// workstation and gamelet.
-constexpr int kAssetStreamPortFirst = 44433;
-constexpr int kAssetStreamPortLast = 44442;
-
 // Stats output period (if enabled).
 constexpr double kStatsPrintDelaySec = 0.1f;
 
@@ -440,16 +436,19 @@ absl::Status MultiSession::Initialize() {
   }
 
   // Find an available local port.
+  local_asset_stream_port_ = cfg_.forward_port_first;
+  if (cfg_.forward_port_first < cfg_.forward_port_last) {
     std::unordered_set<int> ports;
     ASSIGN_OR_RETURN(
         ports,
-      PortManager::FindAvailableLocalPorts(kAssetStreamPortFirst,
-                                           kAssetStreamPortLast, "127.0.0.1",
-                                           process_factory_),
+        PortManager::FindAvailableLocalPorts(cfg_.forward_port_first,
+                                             cfg_.forward_port_last,
+                                             "127.0.0.1", process_factory_),
         "Failed to find an available local port in the range [%d, %d]",
-      kAssetStreamPortFirst, kAssetStreamPortLast);
+        cfg_.forward_port_first, cfg_.forward_port_last);
     assert(!ports.empty());
     local_asset_stream_port_ = *ports.begin();
+  }
 
   assert(!runner_);
   runner_ = std::make_unique<MultiSessionRunner>(
@@ -525,7 +524,8 @@ absl::Status MultiSession::StartSession(const std::string& instance_id,
   auto session = std::make_unique<Session>(
       instance_id, target, cfg_, process_factory_, std::move(metrics_recorder));
   RETURN_IF_ERROR(session->Start(local_asset_stream_port_,
-                                 kAssetStreamPortFirst, kAssetStreamPortLast));
+                                 cfg_.forward_port_first,
+                                 cfg_.forward_port_last));
 
   // Wait for the FUSE to receive the first intermediate manifest.
   RETURN_IF_ERROR(runner_->WaitForManifestAck(instance_id, absl::Seconds(5)));
@@ -555,6 +555,21 @@ bool MultiSession::HasSession(const std::string& instance_id) {
   return sessions_.find(instance_id) != sessions_.end();
 }
 
+std::vector<std::string> MultiSession::MatchSessions(
+    const std::string& instance_id_filter) {
+  PathFilter filter;
+  filter.AddRule(PathFilter::Rule::Type::kInclude, instance_id_filter);
+  filter.AddRule(PathFilter::Rule::Type::kExclude, "*");
+
+  std::vector<std::string> matches;
+  for (const auto& [instance_id, session] : sessions_) {
+    if (filter.IsMatch(instance_id)) {
+      matches.push_back(instance_id);
+    }
+  }
+  return matches;
+}
+
 bool MultiSession::IsSessionHealthy(const std::string& instance_id) {
   absl::ReaderMutexLock lock(&sessions_mutex_);
   auto iter = sessions_.find(instance_id);
@@ -609,7 +624,8 @@ absl::StatusOr<std::string> MultiSession::GetCachePath(
   path::Append(&appdata_path, ".cache");
 #endif
 
-  std::string base_dir = path::Join(appdata_path, "GGP", "asset_streaming");
+  std::string base_dir =
+      path::Join(appdata_path, "cdc-file-transfer", "chunks");
   std::string cache_dir = GetCacheDir(src_dir);
 
   size_t total_size = base_dir.size() + 1 + cache_dir.size();

cdc_stream/multi_session.h

@@ -134,6 +134,11 @@ class MultiSessionRunner {
 // to an arbitrary number of gamelets.
 class MultiSession {
  public:
+  // Ports used by the asset streaming service for local port forwarding on
+  // workstation and gamelet.
+  static constexpr int kDefaultForwardPortFirst = 44433;
+  static constexpr int kDefaultForwardPortLast = 44442;
+
   // Maximum length of cache path. We must be able to write content hashes into
   // this path:
   // <cache path>\01234567890123456789<null terminator> = 260 characters.
@@ -148,7 +153,7 @@ class MultiSession {
   // |process_factory| abstracts process creation.
   // |data_store| can be passed for tests to override the default store used.
   // By default, the class uses a DiskDataStore that writes to
-  // %APPDATA%\GGP\asset_streaming|<dir_derived_from_src_dir> on Windows.
+  // %APPDATA%\cdc-file-transfer\chunks\<dir_derived_from_src_dir> on Windows.
   MultiSession(std::string src_dir, SessionConfig cfg,
                ProcessFactory* process_factory,
                MultiSessionMetricsRecorder const* metrics_recorder,
@@ -194,6 +199,11 @@ class MultiSession {
   absl::Status StopSession(const std::string& instance_id)
       ABSL_LOCKS_EXCLUDED(sessions_mutex_);
 
+  // Returns all instance ids that match the given filter. The filter may
+  // contain Windows-style wildcards, e.g. *, foo* or f?o.
+  // Matches are case sensitive.
+  std::vector<std::string> MatchSessions(const std::string& instance_id_filter);
+
   // Returns true if there is an existing session for |instance_id|.
   bool HasSession(const std::string& instance_id)
       ABSL_LOCKS_EXCLUDED(sessions_mutex_);
@@ -215,7 +225,7 @@ class MultiSession {
   static std::string GetCacheDir(std::string dir);
 
   // Returns the directory where manifest chunks are cached, e.g.
-  // "%APPDATA%\GGP\asset_streaming\c__path_to_game_abcdef01" for
+  // "%APPDATA%\cdc-file-transfer\chunks\c__path_to_game_abcdef01" for
   // "C:\path\to\game".
   // The returned path is shortened to |max_len| by removing UTF8 code points
   // from the beginning of the actual cache directory (c__path...) if necessary.

cdc_stream/multi_session_test.cc

@@ -241,7 +241,7 @@ TEST_F(MultiSessionTest, GetCachePath_ContainsExpectedParts) {
   ASSERT_OK(cache_path);
   EXPECT_TRUE(absl::EndsWith(*cache_path, kCacheDir)) << *cache_path;
   EXPECT_TRUE(
-      absl::StrContains(*cache_path, path::Join("GGP", "asset_streaming")))
+      absl::StrContains(*cache_path, path::Join("cdc-file-transfer", "chunks")))
       << *cache_path;
 }
 
@@ -253,7 +253,7 @@ TEST_F(MultiSessionTest, GetCachePath_ShortensLongPaths) {
   ASSERT_OK(cache_path);
   EXPECT_EQ(cache_path->size(), MultiSession::kDefaultMaxCachePathLen);
   EXPECT_TRUE(
-      absl::StrContains(*cache_path, path::Join("GGP", "asset_streaming")))
+      absl::StrContains(*cache_path, path::Join("cdc-file-transfer", "chunks")))
       << *cache_path;
   // The hash in the end of the path is kept and not shortened.
   EXPECT_EQ(cache_dir.substr(cache_dir.size() - MultiSession::kDirHashLen),
@@ -261,7 +261,8 @@ TEST_F(MultiSessionTest, GetCachePath_ShortensLongPaths) {
 }
 
 TEST_F(MultiSessionTest, GetCachePath_DoesNotSplitUtfCodePoints) {
-  // Find out the length of the %APPDATA%\GGP\asset_streaming\" + hash part.
+  // Find out the length of the %APPDATA%\cdc-file-transfer\chunks\" + hash
+  // part.
   absl::StatusOr<std::string> cache_path = MultiSession::GetCachePath("");
   ASSERT_OK(cache_path);
   size_t base_len = cache_path->size();
@@ -271,17 +272,17 @@ TEST_F(MultiSessionTest, GetCachePath_DoesNotSplitUtfCodePoints) {
   ASSERT_OK(cache_path);
   EXPECT_EQ(cache_path->size(), base_len);
 
-  // %APPDATA%\GGP\asset_streaming\abcdefg
+  // %APPDATA%\cdc-file-transfer\chunks\abcdefg
   cache_path = MultiSession::GetCachePath(u8"\u0200\u0200", base_len + 1);
   ASSERT_OK(cache_path);
   EXPECT_EQ(cache_path->size(), base_len);
 
-  // %APPDATA%\GGP\asset_streaming\\u0200abcdefg
+  // %APPDATA%\cdc-file-transfer\chunks\\u0200abcdefg
   cache_path = MultiSession::GetCachePath(u8"\u0200\u0200", base_len + 2);
   ASSERT_OK(cache_path);
   EXPECT_EQ(cache_path->size(), base_len + 2);
 
-  // %APPDATA%\GGP\asset_streaming\\u0200abcdefg
+  // %APPDATA%\cdc-file-transfer\chunks\\u0200abcdefg
   cache_path = MultiSession::GetCachePath(u8"\u0200\u0200", base_len + 3);
   ASSERT_OK(cache_path);
   EXPECT_EQ(cache_path->size(), base_len + 2);

cdc_stream/session.cc

@@ -47,11 +47,11 @@ Session::Session(std::string instance_id, const SessionTarget& target,
       mount_dir_(target.mount_dir),
       cfg_(std::move(cfg)),
       process_factory_(process_factory),
-      remote_util_(cfg_.verbosity, cfg_.quiet, process_factory,
+      remote_util_(target.user_host, cfg_.verbosity, cfg_.quiet,
+                   process_factory,
                    /*forward_output_to_logging=*/true),
       metrics_recorder_(std::move(metrics_recorder)) {
   assert(metrics_recorder_);
-  remote_util_.SetUserHostAndPort(target.user_host, target.ssh_port);
   if (!target.ssh_command.empty()) {
     remote_util_.SetSshCommand(target.ssh_command);
   }
@@ -71,6 +71,8 @@ Session::~Session() {
 absl::Status Session::Start(int local_port, int first_remote_port,
                             int last_remote_port) {
   // Find an available remote port.
+  int remote_port = first_remote_port;
+  if (first_remote_port < last_remote_port) {
     std::unordered_set<int> ports;
     ASSIGN_OR_RETURN(
         ports,
@@ -80,7 +82,8 @@ absl::Status Session::Start(int local_port, int first_remote_port,
         "Failed to find an available remote port in the range [%d, %d]",
         first_remote_port, last_remote_port);
     assert(!ports.empty());
-  int remote_port = *ports.begin();
+    remote_port = *ports.begin();
+  }
 
   assert(!fuse_);
   fuse_ = std::make_unique<CdcFuseManager>(instance_id_, process_factory_,

cdc_stream/session.h

@@ -36,8 +36,6 @@ class Process;
 struct SessionTarget {
   // SSH username and hostname of the remote target, formed as [user@]host.
   std::string user_host;
-  // Port to use for SSH connections to the remote target.
-  uint16_t ssh_port;
   // Ssh command to use to connect to the remote target.
   std::string ssh_command;
   // Scp command to use to copy files to the remote target.

cdc_stream/session_config.h

@@ -56,6 +56,10 @@ struct SessionConfig {
 
   // Time to wait until running a manifest update after detecting a file change.
   uint32_t file_change_wait_duration_ms = 0;
+
+  // Ports used for local port forwarding.
+  uint16_t forward_port_first = 0;
+  uint16_t forward_port_last = 0;
 };
 
 }  // namespace cdc_ft

cdc_stream/session_management_server.h

@@ -36,7 +36,7 @@ class ProcessFactory;
 // - Background
 class SessionManagementServer {
  public:
-  static constexpr int kDefaultServicePort = 44432;
+  static constexpr uint16_t kDefaultServicePort = 44432;
 
   SessionManagementServer(grpc::Service* session_service,
                           grpc::Service* background_service,

cdc_stream/session_manager.cc

@@ -136,9 +136,24 @@ absl::Status SessionManager::StartSession(
   return status;
 }
 
-absl::Status SessionManager::StopSession(const std::string& instance_id) {
+absl::Status SessionManager::StopSession(
+    const std::string& instance_id_filter) {
   absl::MutexLock lock(&sessions_mutex_);
-  return StopSessionInternal(instance_id);
+
+  std::vector<std::string> instance_ids;
+  for (const auto& [key, ms] : sessions_) {
+    auto ids = ms->MatchSessions(instance_id_filter);
+    instance_ids.insert(instance_ids.end(), ids.begin(), ids.end());
+  }
+  if (instance_ids.empty()) {
+    return absl::NotFoundError(
+        absl::StrFormat("No session found matching '%s'", instance_id_filter));
+  }
+
+  for (const std::string& instance_id : instance_ids) {
+    RETURN_IF_ERROR(StopSessionInternal(instance_id));
+  }
+  return absl::OkStatus();
 }
 
 MultiSession* SessionManager::GetMultiSession(const std::string& src_dir) {

cdc_stream/session_manager.h

@@ -58,9 +58,11 @@ class SessionManager {
                             metrics::SessionStartStatus* metrics_status)
       ABSL_LOCKS_EXCLUDED(sessions_mutex_);
 
-  // Stops the session for the given |instance_id|.
+  // Stops all sessions that match the given |instance_id_filter|.
+  // The filter may contain Windows-style wildcards like * and ?.
+  // Matching is case-sensitive.
   // Returns a NotFound error if no session exists.
-  absl::Status StopSession(const std::string& instance_id)
+  absl::Status StopSession(const std::string& instance_id_filter)
       ABSL_LOCKS_EXCLUDED(sessions_mutex_);
 
   // Shuts down all existing MultiSessions.

cdc_stream/start_command.cc

@@ -16,12 +16,18 @@
 
 #include <memory>
 
+#include "cdc_stream/background_service_client.h"
 #include "cdc_stream/local_assets_stream_manager_client.h"
 #include "cdc_stream/session_management_server.h"
 #include "common/log.h"
 #include "common/path.h"
-#include "common/remote_util.h"
+#include "common/process.h"
 #include "common/status_macros.h"
+#include "common/stopwatch.h"
+#include "common/util.h"
+#include "grpcpp/channel.h"
+#include "grpcpp/create_channel.h"
+#include "grpcpp/support/channel_arguments.h"
 #include "lyra/lyra.hpp"
 
 namespace cdc_ft {
@@ -29,6 +35,19 @@ namespace {
 constexpr int kDefaultVerbosity = 2;
 }  // namespace
 
+namespace {
+// Time to poll until the streaming service becomes healthy.
+constexpr double kServiceStartupTimeoutSec = 20.0;
+
+std::shared_ptr<grpc::Channel> CreateChannel(uint16_t service_port) {
+  std::string client_address = absl::StrFormat("localhost:%u", service_port);
+  return grpc::CreateCustomChannel(client_address,
+                                   grpc::InsecureChannelCredentials(),
+                                   grpc::ChannelArguments());
+}
+
+}  // namespace
+
 StartCommand::StartCommand(int* exit_code)
     : BaseCommand("start",
                   "Start streaming files from a Windows to a Linux device",
@@ -52,20 +71,12 @@ void StartCommand::RegisterCommandLineFlags(lyra::command& cmd) {
                 "asset stream service, default: " +
                 std::to_string(SessionManagementServer::kDefaultServicePort)));
 
-  ssh_port_ = RemoteUtil::kDefaultSshPort;
-  cmd.add_argument(
-      lyra::opt(ssh_port_, "port")
-          .name("--ssh-port")
-          .help("Port to use while connecting to the remote instance being "
-                "streamed to, default: " +
-                std::to_string(RemoteUtil::kDefaultSshPort)));
-
   path::GetEnv("CDC_SSH_COMMAND", &ssh_command_).IgnoreError();
   cmd.add_argument(
       lyra::opt(ssh_command_, "ssh_command")
           .name("--ssh-command")
           .help("Path and arguments of ssh command to use, e.g. "
-                "\"C:\\path\\to\\ssh.exe -F config_file\". Can also be "
+                "\"C:\\path\\to\\ssh.exe -F config_file -p 1234\". Can also be "
                 "specified by the CDC_SSH_COMMAND environment variable."));
 
   path::GetEnv("CDC_SCP_COMMAND", &scp_command_).IgnoreError();
@@ -73,7 +84,7 @@ void StartCommand::RegisterCommandLineFlags(lyra::command& cmd) {
       lyra::opt(scp_command_, "scp_command")
           .name("--scp-command")
           .help("Path and arguments of scp command to use, e.g. "
-                "\"C:\\path\\to\\scp.exe -F config_file\". Can also be "
+                "\"C:\\path\\to\\scp.exe -F config_file -P 1234\". Can also be "
                 "specified by the CDC_SCP_COMMAND environment variable."));
 
   cmd.add_argument(lyra::arg(PosArgValidator(&src_dir_), "dir")
@@ -81,7 +92,7 @@ void StartCommand::RegisterCommandLineFlags(lyra::command& cmd) {
                        .help("Windows directory to stream"));
 
   cmd.add_argument(
-      lyra::arg(PosArgValidator(&user_host_dir_), "[user@]host:src-dir")
+      lyra::arg(PosArgValidator(&user_host_dir_), "[user@]host:dir")
           .required()
           .help("Linux host and directory to stream to"));
 }
@@ -89,16 +100,32 @@ void StartCommand::RegisterCommandLineFlags(lyra::command& cmd) {
 absl::Status StartCommand::Run() {
   LogLevel level = Log::VerbosityToLogLevel(verbosity_);
   ScopedLog scoped_log(std::make_unique<ConsoleLog>(level));
-  LocalAssetsStreamManagerClient client(service_port_);
 
   std::string full_src_dir = path::GetFullPath(src_dir_);
   std::string user_host, mount_dir;
   RETURN_IF_ERROR(LocalAssetsStreamManagerClient::ParseUserHostDir(
       user_host_dir_, &user_host, &mount_dir));
 
-  absl::Status status =
-      client.StartSession(full_src_dir, user_host, ssh_port_, mount_dir,
+  LocalAssetsStreamManagerClient client(CreateChannel(service_port_));
+  absl::Status status = client.StartSession(full_src_dir, user_host, mount_dir,
                                             ssh_command_, scp_command_);
+
+  if (absl::IsUnavailable(status)) {
+    LOG_DEBUG("StartSession status: %s", status.ToString());
+    LOG_INFO("Streaming service is unavailable. Starting it...");
+    status = StartStreamingService();
+
+    if (status.ok()) {
+      LOG_INFO("Streaming service successfully started");
+
+      // Recreate client. The old channel might still be in a transient failure
+      // state.
+      LocalAssetsStreamManagerClient new_client(CreateChannel(service_port_));
+      status = new_client.StartSession(full_src_dir, user_host, mount_dir,
+                                       ssh_command_, scp_command_);
+    }
+  }
+
   if (status.ok()) {
     LOG_INFO("Started streaming directory '%s' to '%s:%s'", src_dir_, user_host,
              mount_dir);
@@ -107,4 +134,44 @@ absl::Status StartCommand::Run() {
   return status;
 }
 
+absl::Status StartCommand::StartStreamingService() {
+  std::string exe_dir;
+  RETURN_IF_ERROR(path::GetExeDir(&exe_dir),
+                  "Failed to get executable directory");
+  std::string exe_path = path::Join(exe_dir, "cdc_stream");
+
+  // Try starting the service first.
+  WinProcessFactory process_factory;
+  ProcessStartInfo start_info;
+  start_info.command =
+      absl::StrFormat("%s start-service --verbosity=%i --service-port=%i",
+                      exe_path, verbosity_, service_port_);
+  start_info.flags = ProcessFlags::kDetached;
+  std::unique_ptr<Process> service_process = process_factory.Create(start_info);
+  RETURN_IF_ERROR(service_process->Start(),
+                  "Failed to start asset streaming service");
+
+  // Poll until the service becomes healthy.
+  LOG_INFO("Streaming service initializing...");
+  Stopwatch sw;
+  while (sw.ElapsedSeconds() < kServiceStartupTimeoutSec) {
+    // The channel is in some transient failure state, and it's faster to
+    // reconnect instead of waiting for it to return.
+    BackgroundServiceClient bg_client(CreateChannel(service_port_));
+    absl::Status status = bg_client.IsHealthy();
+    if (status.ok()) {
+      return absl::OkStatus();
+    }
+    LOG_DEBUG("Health check result: %s", status.ToString());
+    Util::Sleep(100);
+  }
+
+  // Kill the process.
+  service_process->Terminate();
+  return absl::DeadlineExceededError(
+      absl::StrFormat("Timed out after %0.0f seconds waiting for the asset "
+                      "streaming service to become healthy",
+                      kServiceStartupTimeoutSec));
+}
+
 }  // namespace cdc_ft

cdc_stream/start_command.h

@@ -20,6 +20,10 @@
 #include "absl/status/status.h"
 #include "cdc_stream/base_command.h"
 
+namespace grpc {
+class Channel;
+}
+
 namespace cdc_ft {
 
 // Handler for the start command. Sends an RPC call to the service to starts a
@@ -34,9 +38,11 @@ class StartCommand : public BaseCommand {
   absl::Status Run() override;
 
  private:
+  // Starts the asset streaming service.
+  absl::Status StartStreamingService();
+
   int verbosity_ = 0;
   uint16_t service_port_ = 0;
-  uint16_t ssh_port_ = 0;
   std::string ssh_command_;
   std::string scp_command_;
   std::string src_dir_;

cdc_stream/start_service_command.cc

@@ -39,11 +39,11 @@ std::string GetLogPath(const char* log_dir, const char* log_base_name) {
 }  // namespace
 
 StartServiceCommand::StartServiceCommand(int* exit_code)
-    : BaseCommand("start-service", "Start streaming service", exit_code) {}
+    : BaseCommand("start-service", "Start the streaming service", exit_code) {}
 StartServiceCommand::~StartServiceCommand() = default;
 
 void StartServiceCommand::RegisterCommandLineFlags(lyra::command& cmd) {
-  config_file_ = "%APPDATA%\\cdc-file-transfer\\assets_stream_manager.json";
+  config_file_ = "%APPDATA%\\cdc-file-transfer\\cdc_stream.json";
   cmd.add_argument(
       lyra::opt(config_file_, "path")
           .name("--config-file")
@@ -116,7 +116,7 @@ absl::StatusOr<std::unique_ptr<Log>> StartServiceCommand::GetLogger() {
   }
 
   return std::make_unique<FileLog>(
-      level, GetLogPath(log_dir_.c_str(), "assets_stream_manager").c_str());
+      level, GetLogPath(log_dir_.c_str(), "cdc_stream").c_str());
 }
 
 // Runs the session management service and returns when it finishes.
@@ -140,15 +140,13 @@ absl::Status StartServiceCommand::RunService() {
     request.set_workstation_directory(cfg_.dev_src_dir());
     request.set_user_host(cfg_.dev_target().user_host);
     request.set_mount_dir(cfg_.dev_target().mount_dir);
-    request.set_port(cfg_.dev_target().ssh_port);
     request.set_ssh_command(cfg_.dev_target().ssh_command);
     request.set_scp_command(cfg_.dev_target().scp_command);
     localassetsstreammanager::StartSessionResponse response;
     RETURN_ABSL_IF_ERROR(
         session_service.StartSession(nullptr, &request, &response));
   }
-  RETURN_IF_ERROR(
-      sm_server.Start(SessionManagementServer::kDefaultServicePort));
+  RETURN_IF_ERROR(sm_server.Start(cfg_.service_port()));
   sm_server.RunUntilShutdown();
   return absl::OkStatus();
 }

cdc_stream/stop_command.cc

@@ -21,6 +21,9 @@
 #include "common/log.h"
 #include "common/path.h"
 #include "common/status_macros.h"
+#include "grpcpp/channel.h"
+#include "grpcpp/create_channel.h"
+#include "grpcpp/support/channel_arguments.h"
 #include "lyra/lyra.hpp"
 
 namespace cdc_ft {
@@ -50,7 +53,7 @@ void StopCommand::RegisterCommandLineFlags(lyra::command& cmd) {
                 std::to_string(SessionManagementServer::kDefaultServicePort)));
 
   cmd.add_argument(
-      lyra::arg(PosArgValidator(&user_host_dir_), "[user@]host:src-dir")
+      lyra::arg(PosArgValidator(&user_host_dir_), "[user@]host:dir")
           .required()
           .help("Linux host and directory to stream to"));
 }
@@ -58,11 +61,23 @@ void StopCommand::RegisterCommandLineFlags(lyra::command& cmd) {
 absl::Status StopCommand::Run() {
   LogLevel level = Log::VerbosityToLogLevel(verbosity_);
   ScopedLog scoped_log(std::make_unique<ConsoleLog>(level));
-  LocalAssetsStreamManagerClient client(service_port_);
+
+  std::string client_address = absl::StrFormat("localhost:%u", service_port_);
+  std::shared_ptr<grpc::Channel> channel = grpc::CreateCustomChannel(
+      client_address, grpc::InsecureChannelCredentials(),
+      grpc::ChannelArguments());
+
+  LocalAssetsStreamManagerClient client(channel);
 
   std::string user_host, mount_dir;
+  if (user_host_dir_ == "*") {
+    // Convenience shortcut "*" for "*:*".
+    user_host = "*";
+    mount_dir = "*";
+  } else {
     RETURN_IF_ERROR(LocalAssetsStreamManagerClient::ParseUserHostDir(
         user_host_dir_, &user_host, &mount_dir));
+  }
 
   absl::Status status = client.StopSession(user_host, mount_dir);
   if (status.ok()) {

cdc_stream/stop_service_command.cc

@@ -0,0 +1,70 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "cdc_stream/stop_service_command.h"
+
+#include "absl/strings/str_format.h"
+#include "cdc_stream/background_service_client.h"
+#include "cdc_stream/session_management_server.h"
+#include "common/log.h"
+#include "grpcpp/channel.h"
+#include "grpcpp/create_channel.h"
+#include "grpcpp/support/channel_arguments.h"
+#include "lyra/lyra.hpp"
+
+namespace cdc_ft {
+
+StopServiceCommand::StopServiceCommand(int* exit_code)
+    : BaseCommand("stop-service", "Stops the streaming service", exit_code) {}
+StopServiceCommand::~StopServiceCommand() = default;
+
+void StopServiceCommand::RegisterCommandLineFlags(lyra::command& cmd) {
+  verbosity_ = 2;
+  cmd.add_argument(lyra::opt(verbosity_, "num")
+                       .name("--verbosity")
+                       .help("Verbosity of the log output, default: " +
+                             std::to_string(verbosity_) +
+                             ".Increase to make logs more verbose."));
+
+  service_port_ = SessionManagementServer::kDefaultServicePort;
+  cmd.add_argument(lyra::opt(service_port_, "port")
+                       .name("--service-port")
+                       .help("Local port to use while connecting to the local "
+                             "asset stream service, default: " +
+                             std::to_string(service_port_)));
+}
+
+absl::Status StopServiceCommand::Run() {
+  LogLevel level = Log::VerbosityToLogLevel(verbosity_);
+  ScopedLog scoped_log(std::make_unique<ConsoleLog>(level));
+
+  std::string client_address = absl::StrFormat("localhost:%u", service_port_);
+  std::shared_ptr<grpc::Channel> channel = grpc::CreateCustomChannel(
+      client_address, grpc::InsecureChannelCredentials(),
+      grpc::ChannelArguments());
+
+  BackgroundServiceClient bg_client(channel);
+  absl::Status status = bg_client.Exit();
+  if (status.ok()) {
+    LOG_INFO("Stopped streaming service");
+  } else if (absl::IsUnavailable(status)) {
+    // Server wasn't running. This doesn't count as an error.
+    LOG_INFO("Streaming service already stopped");
+    return absl::OkStatus();
+  }
+
+  return status;
+}
+
+}  // namespace cdc_ft

cdc_stream/stop_service_command.h

@@ -0,0 +1,44 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef CDC_STREAM_STOP_SERVICE_COMMAND_H_
+#define CDC_STREAM_STOP_SERVICE_COMMAND_H_
+
+#include <memory>
+
+#include "absl/status/status.h"
+#include "cdc_stream/base_command.h"
+
+namespace cdc_ft {
+
+// Handler for the stop-service command. Stops the asset streaming service.
+class StopServiceCommand : public BaseCommand {
+ public:
+  explicit StopServiceCommand(int* exit_code);
+  ~StopServiceCommand();
+
+  // BaseCommand:
+  void RegisterCommandLineFlags(lyra::command& cmd) override;
+  absl::Status Run() override;
+
+ private:
+  int verbosity_ = 0;
+  uint16_t service_port_ = 0;
+};
+
+}  // namespace cdc_ft
+
+#endif  // CDC_STREAM_STOP_SERVICE_COMMAND_H_

common/BUILD

@@ -254,6 +254,25 @@ cc_test(
     ],
 )
 
+cc_library(
+    name = "port_range_parser",
+    srcs = ["port_range_parser.cc"],
+    hdrs = ["port_range_parser.h"],
+    deps = [
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_test(
+    name = "port_range_parser_test",
+    srcs = ["port_range_parser_test.cc"],
+    deps = [
+        ":port_range_parser",
+        ":test_main",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
 cc_library(
     name = "process",
     srcs = ["process_win.cc"],

common/file_watcher_win_test.cc

@@ -135,6 +135,15 @@ class FileWatcherParameterizedTest : public ::testing::TestWithParam<bool> {
     return changed;
   }
 
+  // Polls for a second until the watcher is watching again.
+  bool WaitForWatching() const {
+    for (int n = 0; n < 1000; ++n) {
+      if (watcher_.IsWatching()) return true;
+      Util::Sleep(1);
+    }
+    return false;
+  }
+
   FileMap GetChangedFiles(size_t number_of_files) {
     FileMap modified_files;
 
@@ -540,6 +549,9 @@ TEST_P(FileWatcherParameterizedTest, RecreateWatchedDir) {
   EXPECT_TRUE(watcher_.GetModifiedFiles().empty());
   EXPECT_OK(watcher_.GetStatus());
 
+  // Wait until the watcher is watching again, or else we might miss the file.
+  EXPECT_TRUE(WaitForWatching());
+
   // Creation of a new file should be detected.
   EXPECT_OK(path::WriteFile(first_file_path_, kFirstData, kFirstDataSize));
 
@@ -572,6 +584,9 @@ TEST_P(FileWatcherParameterizedTest, RecreateUpperDir) {
   EXPECT_TRUE(watcher_.GetModifiedFiles().empty());
   EXPECT_OK(watcher_.GetStatus());
 
+  // Wait until the watcher is watching again, or else we might miss the file.
+  EXPECT_TRUE(WaitForWatching());
+
   // Creation of a new file should be detected.
   EXPECT_OK(path::WriteFile(first_file_path_, kFirstData, kFirstDataSize));
 

common/port_manager.h

@@ -51,14 +51,11 @@ class PortManager {
   // Reserves a port in the range passed to the constructor. The port is
   // released automatically upon destruction if ReleasePort() is not called
   // explicitly.
-  // |check_remote| determines whether the remote port should be checked as
-  // well. If false, the check is skipped and a port might be returned that is
-  // still in use remotely.
   // |remote_timeout_sec| is the timeout for finding available ports on the
-  // remote instance. Not used if |check_remote| is false.
+  // remote instance.
   // Returns a DeadlineExceeded error if the timeout is exceeded.
   // Returns a ResourceExhausted error if no ports are available.
-  absl::StatusOr<int> ReservePort(bool check_remote, int remote_timeout_sec);
+  absl::StatusOr<int> ReservePort(int remote_timeout_sec);
 
   // Releases a reserved port.
   absl::Status ReleasePort(int port);

common/port_manager_test.cc

@@ -38,9 +38,6 @@ constexpr int kTimeoutSec = 1;
 constexpr char kLocalNetstat[] = "netstat -a -n -p tcp";
 constexpr char kRemoteNetstat[] = "netstat --numeric --listening --tcp";
 
-constexpr bool kCheckRemote = true;
-constexpr bool kNoCheckRemote = false;
-
 constexpr char kLocalNetstatOutFmt[] =
     "TCP    127.0.0.1:50000        127.0.0.1:%i        ESTABLISHED";
 constexpr char kRemoteNetstatOutFmt[] =
@@ -49,14 +46,14 @@ constexpr char kRemoteNetstatOutFmt[] =
 class PortManagerTest : public ::testing::Test {
  public:
   PortManagerTest()
-      : remote_util_(/*verbosity=*/0, /*quiet=*/false, &process_factory_,
+      : remote_util_(kUserHost, /*verbosity=*/0, /*quiet=*/false,
+                     &process_factory_,
                      /*forward_output_to_log=*/true),
         port_manager_(kGuid, kFirstPort, kLastPort, &process_factory_,
                       &remote_util_, &system_clock_, &steady_clock_) {}
 
   void SetUp() override {
     Log::Initialize(std::make_unique<ConsoleLog>(LogLevel::kInfo));
-    remote_util_.SetUserHostAndPort(kUserHost, kSshPort);
   }
 
   void TearDown() override { Log::Shutdown(); }
@@ -73,16 +70,7 @@ TEST_F(PortManagerTest, ReservePortSuccess) {
   process_factory_.SetProcessOutput(kLocalNetstat, "", "", 0);
   process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 0);
 
-  absl::StatusOr<int> port =
-      port_manager_.ReservePort(kCheckRemote, kTimeoutSec);
-  ASSERT_OK(port);
-  EXPECT_EQ(*port, kFirstPort);
-}
-
-TEST_F(PortManagerTest, ReservePortNoRemoteSuccess) {
-  process_factory_.SetProcessOutput(kLocalNetstat, "", "", 0);
-
-  absl::StatusOr<int> port = port_manager_.ReservePort(kNoCheckRemote, 0);
+  absl::StatusOr<int> port = port_manager_.ReservePort(kTimeoutSec);
   ASSERT_OK(port);
   EXPECT_EQ(*port, kFirstPort);
 }
@@ -95,8 +83,7 @@ TEST_F(PortManagerTest, ReservePortAllLocalPortsTaken) {
   process_factory_.SetProcessOutput(kLocalNetstat, local_netstat_out, "", 0);
   process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 0);
 
-  absl::StatusOr<int> port =
-      port_manager_.ReservePort(kCheckRemote, kTimeoutSec);
+  absl::StatusOr<int> port = port_manager_.ReservePort(kTimeoutSec);
   EXPECT_TRUE(absl::IsResourceExhausted(port.status()));
   EXPECT_TRUE(
       absl::StrContains(port.status().message(), "No port available in range"));
@@ -110,8 +97,7 @@ TEST_F(PortManagerTest, ReservePortAllRemotePortsTaken) {
   process_factory_.SetProcessOutput(kLocalNetstat, "", "", 0);
   process_factory_.SetProcessOutput(kRemoteNetstat, remote_netstat_out, "", 0);
 
-  absl::StatusOr<int> port =
-      port_manager_.ReservePort(kCheckRemote, kTimeoutSec);
+  absl::StatusOr<int> port = port_manager_.ReservePort(kTimeoutSec);
   EXPECT_TRUE(absl::IsResourceExhausted(port.status()));
   EXPECT_TRUE(
       absl::StrContains(port.status().message(), "No port available in range"));
@@ -121,8 +107,7 @@ TEST_F(PortManagerTest, ReservePortLocalNetstatFails) {
   process_factory_.SetProcessOutput(kLocalNetstat, "", "", 1);
   process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 0);
 
-  absl::StatusOr<int> port =
-      port_manager_.ReservePort(kCheckRemote, kTimeoutSec);
+  absl::StatusOr<int> port = port_manager_.ReservePort(kTimeoutSec);
   EXPECT_NOT_OK(port);
   EXPECT_TRUE(
       absl::StrContains(port.status().message(),
@@ -133,8 +118,7 @@ TEST_F(PortManagerTest, ReservePortRemoteNetstatFails) {
   process_factory_.SetProcessOutput(kLocalNetstat, "", "", 0);
   process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 1);
 
-  absl::StatusOr<int> port =
-      port_manager_.ReservePort(kCheckRemote, kTimeoutSec);
+  absl::StatusOr<int> port = port_manager_.ReservePort(kTimeoutSec);
   EXPECT_NOT_OK(port);
   EXPECT_TRUE(absl::StrContains(port.status().message(),
                                 "Failed to find available ports on instance"));
@@ -145,8 +129,7 @@ TEST_F(PortManagerTest, ReservePortRemoteNetstatTimesOut) {
   process_factory_.SetProcessNeverExits(kRemoteNetstat);
   steady_clock_.AutoAdvance(kTimeoutSec * 2 * 1000);
 
-  absl::StatusOr<int> port =
-      port_manager_.ReservePort(kCheckRemote, kTimeoutSec);
+  absl::StatusOr<int> port = port_manager_.ReservePort(kTimeoutSec);
   EXPECT_NOT_OK(port);
   EXPECT_TRUE(absl::IsDeadlineExceeded(port.status()));
   EXPECT_TRUE(absl::StrContains(port.status().message(),
@@ -163,14 +146,10 @@ TEST_F(PortManagerTest, ReservePortMultipleInstances) {
   // Port managers use shared memory, so different instances know about each
   // other. This would even work if |port_manager_| and |port_manager2| belonged
   // to different processes, but we don't test that here.
-  EXPECT_EQ(*port_manager_.ReservePort(kCheckRemote, kTimeoutSec),
-            kFirstPort + 0);
-  EXPECT_EQ(*port_manager2.ReservePort(kCheckRemote, kTimeoutSec),
-            kFirstPort + 1);
-  EXPECT_EQ(*port_manager_.ReservePort(kCheckRemote, kTimeoutSec),
-            kFirstPort + 2);
-  EXPECT_EQ(*port_manager2.ReservePort(kCheckRemote, kTimeoutSec),
-            kFirstPort + 3);
+  EXPECT_EQ(*port_manager_.ReservePort(kTimeoutSec), kFirstPort + 0);
+  EXPECT_EQ(*port_manager2.ReservePort(kTimeoutSec), kFirstPort + 1);
+  EXPECT_EQ(*port_manager_.ReservePort(kTimeoutSec), kFirstPort + 2);
+  EXPECT_EQ(*port_manager2.ReservePort(kTimeoutSec), kFirstPort + 3);
 }
 
 TEST_F(PortManagerTest, ReservePortReusesPortsInLRUOrder) {
@@ -178,7 +157,7 @@ TEST_F(PortManagerTest, ReservePortReusesPortsInLRUOrder) {
   process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 0);
 
   for (int n = 0; n < kNumPorts * 2; ++n) {
-    EXPECT_EQ(*port_manager_.ReservePort(kCheckRemote, kTimeoutSec),
+    EXPECT_EQ(*port_manager_.ReservePort(kTimeoutSec),
               kFirstPort + n % kNumPorts);
     system_clock_.Advance(1000);
   }
@@ -188,11 +167,10 @@ TEST_F(PortManagerTest, ReleasePort) {
   process_factory_.SetProcessOutput(kLocalNetstat, "", "", 0);
   process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 0);
 
-  absl::StatusOr<int> port =
-      port_manager_.ReservePort(kCheckRemote, kTimeoutSec);
+  absl::StatusOr<int> port = port_manager_.ReservePort(kTimeoutSec);
   EXPECT_EQ(*port, kFirstPort);
   EXPECT_OK(port_manager_.ReleasePort(*port));
-  port = port_manager_.ReservePort(kCheckRemote, kTimeoutSec);
+  port = port_manager_.ReservePort(kTimeoutSec);
   EXPECT_EQ(*port, kFirstPort);
 }
 
@@ -202,13 +180,10 @@ TEST_F(PortManagerTest, ReleasePortOnDestruction) {
 
   auto port_manager2 = std::make_unique<PortManager>(
       kGuid, kFirstPort, kLastPort, &process_factory_, &remote_util_);
-  EXPECT_EQ(*port_manager2->ReservePort(kCheckRemote, kTimeoutSec),
-            kFirstPort + 0);
-  EXPECT_EQ(*port_manager_.ReservePort(kCheckRemote, kTimeoutSec),
-            kFirstPort + 1);
+  EXPECT_EQ(*port_manager2->ReservePort(kTimeoutSec), kFirstPort + 0);
+  EXPECT_EQ(*port_manager_.ReservePort(kTimeoutSec), kFirstPort + 1);
   port_manager2.reset();
-  EXPECT_EQ(*port_manager_.ReservePort(kCheckRemote, kTimeoutSec),
-            kFirstPort + 0);
+  EXPECT_EQ(*port_manager_.ReservePort(kTimeoutSec), kFirstPort + 0);
 }
 
 TEST_F(PortManagerTest, FindAvailableLocalPortsSuccess) {

common/port_manager_win.cc

@@ -121,8 +121,7 @@ PortManager::~PortManager() {
   }
 }
 
-absl::StatusOr<int> PortManager::ReservePort(bool check_remote,
-                                             int remote_timeout_sec) {
+absl::StatusOr<int> PortManager::ReservePort(int remote_timeout_sec) {
   // Find available port on workstation.
   std::unordered_set<int> local_ports;
   ASSIGN_OR_RETURN(local_ports,
@@ -132,13 +131,11 @@ absl::StatusOr<int> PortManager::ReservePort(bool check_remote,
 
   // Find available port on remote instance.
   std::unordered_set<int> remote_ports = local_ports;
-  if (check_remote) {
   ASSIGN_OR_RETURN(remote_ports,
-                     FindAvailableRemotePorts(
-                         first_port_, last_port_, "0.0.0.0", process_factory_,
-                         remote_util_, remote_timeout_sec, steady_clock_),
+                   FindAvailableRemotePorts(first_port_, last_port_, "0.0.0.0",
+                                            process_factory_, remote_util_,
+                                            remote_timeout_sec, steady_clock_),
                    "Failed to find available ports on instance");
-  }
 
   // Fetch shared memory.
   void* mem;
@@ -213,6 +210,7 @@ absl::StatusOr<std::unordered_set<int>> PortManager::FindAvailableLocalPorts(
   ProcessStartInfo start_info;
   start_info.command = "netstat -a -n -p tcp";
   start_info.name = "netstat";
+  start_info.flags = ProcessFlags::kNoWindow;
 
   std::string output;
   start_info.stdout_handler = [&output](const char* data, size_t data_size) {
@@ -246,6 +244,7 @@ absl::StatusOr<std::unordered_set<int>> PortManager::FindAvailableRemotePorts(
   ProcessStartInfo start_info =
       remote_util->BuildProcessStartInfoForSsh(remote_command);
   start_info.name = "netstat";
+  start_info.flags = ProcessFlags::kNoWindow;
 
   std::string output;
   start_info.stdout_handler = [&output](const char* data, size_t data_size) {
@@ -288,9 +287,14 @@ absl::StatusOr<std::unordered_set<int>> PortManager::FindAvailablePorts(
     int first_port, int last_port, const std::string& netstat_output,
     const char* ip) {
   std::unordered_set<int> available_ports;
-  for (int port = first_port; port <= last_port; ++port) {
-    std::vector<std::string> lines = absl::StrSplit(netstat_output, '\n');
+  std::vector<std::string> lines;
+  for (const auto& line : absl::StrSplit(netstat_output, '\n')) {
+    if (absl::StrContains(line, ip)) {
+      lines.push_back(std::string(line));
+    }
+  }
 
+  for (int port = first_port; port <= last_port; ++port) {
     bool port_occupied = false;
     std::string portToken = absl::StrFormat("%s:%i", ip, port);
     for (const std::string& line : lines) {

common/port_range_parser.cc

@@ -0,0 +1,40 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "common/port_range_parser.h"
+
+#include <cassert>
+
+#include "absl/strings/str_split.h"
+
+namespace cdc_ft {
+namespace port_range {
+
+bool Parse(const char* value, uint16_t* first, uint16_t* last) {
+  assert(value);
+  *first = 0;
+  *last = 0;
+  std::vector<std::string> parts = absl::StrSplit(value, '-');
+  if (parts.empty() || parts.size() > 2) return false;
+  const int ifirst = atoi(parts[0].c_str());
+  const int ilast = parts.size() > 1 ? atoi(parts[1].c_str()) : ifirst;
+  if (ifirst <= 0 || ifirst > UINT16_MAX) return false;
+  if (ilast <= 0 || ilast > UINT16_MAX || ifirst > ilast) return false;
+  *first = static_cast<uint16_t>(ifirst);
+  *last = static_cast<uint16_t>(ilast);
+  return true;
+}
+
+}  // namespace port_range
+}  // namespace cdc_ft

common/port_range_parser.h

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef COMMON_PORT_RANGE_PARSER_H_
+#define COMMON_PORT_RANGE_PARSER_H_
+
+#include <cstdint>
+
+namespace cdc_ft {
+namespace port_range {
+
+// Parses |value| into a port range |first|-|last|.
+// If |value| is a single number a, assigns |first|=|last|=a.
+// If |value| is a range a-b, assigns |first|=a, |last|=b.
+bool Parse(const char* value, uint16_t* first, uint16_t* last);
+
+}  // namespace port_range
+}  // namespace cdc_ft
+
+#endif  // COMMON_PORT_RANGE_PARSER_H_

common/port_range_parser_test.cc

@@ -0,0 +1,69 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "common/port_range_parser.h"
+
+#include "gtest/gtest.h"
+
+namespace cdc_ft {
+namespace {
+
+TEST(PortRangeParserTest, SingleSuccess) {
+  uint16_t first, last;
+  EXPECT_TRUE(port_range::Parse("65535", &first, &last));
+  EXPECT_EQ(first, 65535);
+  EXPECT_EQ(last, 65535);
+}
+
+TEST(PortRangeParserTest, RangeSuccess) {
+  uint16_t first, last;
+  EXPECT_TRUE(port_range::Parse("1-2", &first, &last));
+  EXPECT_EQ(first, 1);
+  EXPECT_EQ(last, 2);
+}
+
+TEST(ParamsTest, NoValueFail) {
+  uint16_t first = 1, last = 1;
+  EXPECT_FALSE(port_range::Parse("", &first, &last));
+  EXPECT_EQ(first, 0);
+  EXPECT_EQ(last, 0);
+}
+
+TEST(ParamsTest, BadValueTooSmallFail) {
+  uint16_t first, last;
+  EXPECT_FALSE(port_range::Parse("0", &first, &last));
+}
+
+TEST(ParamsTest, BadValueNotIntegerFail) {
+  uint16_t first, last;
+  EXPECT_FALSE(port_range::Parse("port", &first, &last));
+}
+
+TEST(ParamsTest, ForwardPort_BadRangeTooBig) {
+  uint16_t first, last;
+  EXPECT_FALSE(port_range::Parse("50000-65536", &first, &last));
+}
+
+TEST(ParamsTest, ForwardPort_BadRangeFirstGtLast) {
+  uint16_t first, last;
+  EXPECT_FALSE(port_range::Parse("50001-50000", &first, &last));
+}
+
+TEST(ParamsTest, ForwardPort_BadRangeTwoMinus) {
+  uint16_t first, last;
+  EXPECT_FALSE(port_range::Parse("1-2-3", &first, &last));
+}
+
+}  // namespace
+}  // namespace cdc_ft

common/process.h

@@ -33,6 +33,12 @@ namespace cdc_ft {
 absl::Status LogOutput(const char* name, const char* data, size_t data_size,
                        absl::optional<LogLevel> log_level = {});
 
+enum class ProcessFlags {
+  kNone = 0,
+  kDetached = 1 << 0,
+  kNoWindow = 1 << 1,
+};
+
 struct ProcessStartInfo {
   // Handler for stdout/stderr. |data| is guaranteed to be NULL terminated, so
   // it may be used like a C-string if it's known to be text, e.g. for printf().
@@ -63,8 +69,14 @@ struct ProcessStartInfo {
   OutputHandler stdout_handler;
   OutputHandler stderr_handler;
 
+  // Flags that define additional properties of the process.
+  ProcessFlags flags = ProcessFlags::kNone;
+
   // Returns |name| if set, otherwise |command|.
   const std::string& Name() const;
+
+  // Tests ALL flags (flags & flag) == flag.
+  bool HasFlag(ProcessFlags flag) const;
 };
 
 // Runs a background process and pipes stdin/stdout/stderr.
@@ -75,6 +87,8 @@ class Process {
   static constexpr uint32_t kExitCodeFailedToGetExitCode = 4000000002;
 
   explicit Process(const ProcessStartInfo& start_info);
+
+  // Terminates the process unless it's running with ProcessFlags::kDetached.
   virtual ~Process();
 
   // Start the background process.
@@ -140,6 +154,16 @@ class WinProcessFactory : public ProcessFactory {
   std::unique_ptr<Process> Create(const ProcessStartInfo& start_info) override;
 };
 
+inline ProcessFlags operator|(ProcessFlags a, ProcessFlags b) {
+  using T = std::underlying_type_t<ProcessFlags>;
+  return static_cast<ProcessFlags>(static_cast<T>(a) | static_cast<T>(b));
+}
+
+inline ProcessFlags operator&(ProcessFlags a, ProcessFlags b) {
+  using T = std::underlying_type_t<ProcessFlags>;
+  return static_cast<ProcessFlags>(static_cast<T>(a) & static_cast<T>(b));
+}
+
 }  // namespace cdc_ft
 
 #endif  // COMMON_PROCESS_H_

common/process_win.cc

@@ -49,6 +49,24 @@ void SetThreadName(const std::string& name) {
   }
 }
 
+int ToCreationFlags(ProcessFlags pflags) {
+#define HANDLE_FLAG(pflag, cflag)  \
+  if ((pflags & pflag) == pflag) { \
+    cflags |= cflag;               \
+    pdone = pdone | pflag;         \
+  }
+
+  int cflags = 0;
+  ProcessFlags pdone = ProcessFlags::kNone;
+  HANDLE_FLAG(ProcessFlags::kDetached, DETACHED_PROCESS);
+  HANDLE_FLAG(ProcessFlags::kNoWindow, CREATE_NO_WINDOW);
+  assert(pflags == pdone);
+
+#undef HANDLE_FLAG
+
+  return cflags;
+}
+
 std::atomic_int g_pipe_serial_number{0};
 
 // Creates a pipe suitable for overlapped IO. Regular anonymous pipes in Windows
@@ -567,6 +585,10 @@ const std::string& ProcessStartInfo::Name() const {
   return !name.empty() ? name : command;
 }
 
+bool ProcessStartInfo::HasFlag(ProcessFlags flag) const {
+  return (flags & flag) == flag;
+}
+
 Process::Process(const ProcessStartInfo& start_info)
     : start_info_(start_info) {}
 
@@ -593,6 +615,8 @@ class WinProcess : public Process {
   absl::Status GetStatus() const override;
 
  private:
+  void Reset();
+
   std::unique_ptr<ProcessInfo> process_info_;
   std::unique_ptr<MessagePumpThread> message_pump_;
 };
@@ -600,7 +624,14 @@ class WinProcess : public Process {
 WinProcess::WinProcess(const ProcessStartInfo& start_info)
     : Process(start_info) {}
 
-WinProcess::~WinProcess() { Terminate().IgnoreError(); }
+WinProcess::~WinProcess() {
+  if (start_info_.HasFlag(ProcessFlags::kDetached)) {
+    // If the process runs detached, just reset handles, don't terminate it.
+    Reset();
+  } else {
+    Terminate().IgnoreError();
+  }
+}
 
 absl::Status WinProcess::Start() {
   LOG_INFO("Starting process %s", start_info_.command.c_str());
@@ -676,10 +707,13 @@ absl::Status WinProcess::Start() {
   }
 
   JOBOBJECT_EXTENDED_LIMIT_INFORMATION jeli = {0};
+  if (!start_info_.HasFlag(ProcessFlags::kDetached)) {
     jeli.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE;
+  }
   bool success = SetInformationJobObject(process_info_->job.Get(),
                                          JobObjectExtendedLimitInformation,
                                          &jeli, sizeof(jeli));
+
   if (!success) {
     return MakeStatus("SetInformationJobObject() failed: %s",
                       Util::GetLastWin32Error());
@@ -691,7 +725,7 @@ absl::Status WinProcess::Start() {
                           NULL,  // Process handle not inheritable
                           NULL,  // Thread handle not inheritable
                           TRUE,  // Inherit handles
-                          0,     // No creation flags
+                          ToCreationFlags(start_info_.flags),
                           NULL,  // Use parent's environment block
                           NULL,  // Use parent's starting directory
                           &si, &process_info_->pi);
@@ -785,34 +819,41 @@ absl::Status WinProcess::Terminate() {
     message_pump_.reset();
   }
 
-  if (process_info_) {
-    bool result = true;
-    if (should_terminate) {
-      result = TerminateProcess(process_info_->pi.hProcess, 0);
-      if (!result && GetLastError() == ERROR_ACCESS_DENIED) {
+  std::string error_msg;
+  if (process_info_ && should_terminate &&
+      !TerminateProcess(process_info_->pi.hProcess, 0)) {
+    if (GetLastError() == ERROR_ACCESS_DENIED) {
       // This means that the process has already exited, but in a way that
       // the exit wasn't properly reported to this code (e.g. the process got
       // killed somewhere). Just handle this silently.
       LOG_DEBUG("Process '%s' already exited", start_info_.Name());
-        result = true;
+    } else {
+      error_msg = Util::GetLastWin32Error();
     }
   }
 
+  // Reset handles.
+  Reset();
+
+  if (!error_msg.empty()) {
+    return MakeStatus("TerminateProcess() failed: %s", error_msg);
+  }
+  return absl::OkStatus();
+}
+
+void WinProcess::Reset() {
+  // Shut down message pump.
+  message_pump_.reset();
+
+  if (process_info_) {
     // Close the handles that are not scoped handles.
     ScopedHandle(process_info_->pi.hProcess).Close();
     ScopedHandle(process_info_->pi.hThread).Close();
 
     process_info_.reset();
-
-    if (!result) {
-      return MakeStatus("TerminateProcess() failed: %s",
-                        Util::GetLastWin32Error());
   }
 }
 
-  return absl::OkStatus();
-}
-
 ProcessFactory::~ProcessFactory() = default;
 
 absl::Status ProcessFactory::Run(const ProcessStartInfo& start_info) {

common/remote_util.cc

@@ -20,7 +20,6 @@
 #include "absl/strings/str_cat.h"
 #include "absl/strings/str_format.h"
 #include "common/path.h"
-#include "common/status.h"
 
 namespace cdc_ft {
 namespace {
@@ -35,19 +34,15 @@ std::string GetPortForwardingArg(int local_port, int remote_port,
 
 }  // namespace
 
-RemoteUtil::RemoteUtil(int verbosity, bool quiet,
+RemoteUtil::RemoteUtil(std::string user_host, int verbosity, bool quiet,
                        ProcessFactory* process_factory,
                        bool forward_output_to_log)
-    : verbosity_(verbosity),
+    : user_host_(std::move(user_host)),
+      verbosity_(verbosity),
       quiet_(quiet),
       process_factory_(process_factory),
       forward_output_to_log_(forward_output_to_log) {}
 
-void RemoteUtil::SetUserHostAndPort(std::string user_host, int port) {
-  user_host_ = std::move(user_host);
-  ssh_port_ = port;
-}
-
 void RemoteUtil::SetScpCommand(std::string scp_command) {
   scp_command_ = std::move(scp_command);
 }
@@ -58,11 +53,6 @@ void RemoteUtil::SetSshCommand(std::string ssh_command) {
 
 absl::Status RemoteUtil::Scp(std::vector<std::string> source_filepaths,
                              const std::string& dest, bool compress) {
-  absl::Status status = CheckUserHostPort();
-  if (!status.ok()) {
-    return status;
-  }
-
   std::string source_args;
   for (const std::string& sourceFilePath : source_filepaths) {
     // Workaround for scp thinking that C is a host in C:\path\to\foo.
@@ -75,14 +65,12 @@ absl::Status RemoteUtil::Scp(std::vector<std::string> source_filepaths,
 
   // -p preserves timestamps. This enables timestamp-based up-to-date checks.
   ProcessStartInfo start_info;
+  start_info.flags = ProcessFlags::kNoWindow;
   start_info.command = absl::StrFormat(
-      "%s "
-      "%s %s -p -T "
-      "-P %i %s "
-      "%s:%s",
-      QuoteForWindows(scp_command_), quiet_ || verbosity_ < 2 ? "-q" : "",
-      compress ? "-C" : "", ssh_port_, source_args, QuoteForWindows(user_host_),
-      QuoteForWindows(dest));
+      "%s %s %s -p -T "
+      "%s %s:%s",
+      scp_command_, quiet_ || verbosity_ < 2 ? "-q" : "", compress ? "-C" : "",
+      source_args, QuoteForWindows(user_host_), QuoteForWindows(dest));
   start_info.name = "scp";
   start_info.forward_output_to_log = forward_output_to_log_;
 
@@ -99,11 +87,6 @@ absl::Status RemoteUtil::Chmod(const std::string& mode,
 }
 
 absl::Status RemoteUtil::Run(std::string remote_command, std::string name) {
-  absl::Status status = CheckUserHostPort();
-  if (!status.ok()) {
-    return status;
-  }
-
   ProcessStartInfo start_info =
       BuildProcessStartInfoForSsh(std::move(remote_command));
   start_info.name = std::move(name);
@@ -139,14 +122,14 @@ ProcessStartInfo RemoteUtil::BuildProcessStartInfoForSshInternal(
     std::string forward_arg, std::string remote_command_arg) {
   ProcessStartInfo start_info;
   start_info.command = absl::StrFormat(
-      "%s "
-      "%s -tt "
+      "%s %s -tt %s "
       "-oServerAliveCountMax=6 "  // Number of lost msgs before ssh terminates
       "-oServerAliveInterval=5 "  // Time interval between alive msgs
-      "%s %s -p %i %s",
-      QuoteForWindows(ssh_command_), quiet_ || verbosity_ < 2 ? "-q" : "",
-      forward_arg, QuoteForWindows(user_host_), ssh_port_, remote_command_arg);
+      "%s %s",
+      ssh_command_, quiet_ || verbosity_ < 2 ? "-q" : "", forward_arg,
+      QuoteForWindows(user_host_), remote_command_arg);
   start_info.forward_output_to_log = forward_output_to_log_;
+  start_info.flags = ProcessFlags::kNoWindow;
   return start_info;
 }
 
@@ -198,12 +181,4 @@ std::string RemoteUtil::QuoteForSsh(const std::string& argument) {
                                       escaped.substr(slash_pos + 1), "\""));
 }
 
-absl::Status RemoteUtil::CheckUserHostPort() {
-  if (user_host_.empty() || ssh_port_ == 0) {
-    return MakeStatus("IP or port not set");
-  }
-
-  return absl::OkStatus();
-}
-
 }  // namespace cdc_ft

common/remote_util.h

@@ -29,46 +29,36 @@ namespace cdc_ft {
 // Windows-only.
 class RemoteUtil {
  public:
-  static constexpr int kDefaultSshPort = 22;
-
+  // |user_host| is the SSH [user@]host of the remote instance.
   // If |verbosity| is > 0 and |quiet| is false, output from scp, ssh etc.
   // commands is shown.
   // If |quiet| is true, scp, ssh etc. commands use quiet mode.
   // If |forward_output_to_log| is true, process output is forwarded to logging
   // instead of this process's stdout/stderr.
-  RemoteUtil(int verbosity, bool quiet, ProcessFactory* process_factory,
-             bool forward_output_to_log);
-
-  // Sets the SSH username and hostname of the remote instance, as well as the
-  // SSH tunnel port. |user_host| must be of the form [user@]host.
-  void SetUserHostAndPort(std::string user_host, int port);
+  RemoteUtil(std::string user_host, int verbosity, bool quiet,
+             ProcessFactory* process_factory, bool forward_output_to_log);
 
   // Sets the SCP command binary path and additional arguments, e.g.
-  //   C:\path\to\scp.exe -F <ssh_config> -i <key_file>
-  //     -oStrictHostKeyChecking=yes -oUserKnownHostsFile="""file"""
+  //   C:\path\to\scp.exe -p 1234 -i <key_file> -oUserKnownHostsFile=known_hosts
   // By default, searches scp.exe on the path environment variables.
   void SetScpCommand(std::string scp_command);
 
   // Sets the SSH command binary path and additional arguments, e.g.
-  //   C:\path\to\ssh.exe -F <ssh_config> -i <key_file>
-  //     -oStrictHostKeyChecking=yes -oUserKnownHostsFile="""file"""
+  //   C:\path\to\ssh.exe -P 1234 -i <key_file> -oUserKnownHostsFile=known_hosts
   // By default, searches ssh.exe on the path environment variables.
   void SetSshCommand(std::string ssh_command);
 
   // Copies |source_filepaths| to the remote folder |dest| on the gamelet using
   // scp. If |compress| is true, compressed upload is used.
-  // Must call SetUserHostAndPort before calling this method.
   absl::Status Scp(std::vector<std::string> source_filepaths,
                    const std::string& dest, bool compress);
 
   // Calls 'chmod |mode| |remote_path|' on the gamelet.
-  // Must call SetUserHostAndPort before calling this method.
   absl::Status Chmod(const std::string& mode, const std::string& remote_path,
                      bool quiet = false);
 
   // Runs |remote_command| on the gamelet. The command must be properly escaped.
   // |name| is the name of the command displayed in the logs.
-  // Must call SetUserHostAndPort before calling this method.
   absl::Status Run(std::string remote_command, std::string name);
 
   // Builds an SSH command that executes |remote_command| on the gamelet.
@@ -77,7 +67,6 @@ class RemoteUtil {
   // Builds an SSH command that runs SSH port forwarding to the gamelet, using
   // the given |local_port| and |remote_port|.
   // If |reverse| is true, sets up reverse port forwarding.
-  // Must call SetUserHostAndPort before calling this method.
   ProcessStartInfo BuildProcessStartInfoForSshPortForward(int local_port,
                                                           int remote_port,
                                                           bool reverse);
@@ -85,7 +74,6 @@ class RemoteUtil {
   // Builds an SSH command that executes |remote_command| on the gamelet, using
   // port forwarding with given |local_port| and |remote_port|.
   // If |reverse| is true, sets up reverse port forwarding.
-  // Must call SetUserHostAndPort before calling this method.
   ProcessStartInfo BuildProcessStartInfoForSshPortForwardAndCommand(
       int local_port, int remote_port, bool reverse,
       std::string remote_command);
@@ -117,9 +105,6 @@ class RemoteUtil {
   static std::string QuoteForSsh(const std::string& argument);
 
  private:
-  // Verifies that both |user_host_| and |ssh_port_| are set.
-  absl::Status CheckUserHostPort();
-
   // Common code for BuildProcessStartInfoForSsh*.
   ProcessStartInfo BuildProcessStartInfoForSshInternal(
       std::string forward_arg, std::string remote_command);
@@ -132,7 +117,6 @@ class RemoteUtil {
   std::string scp_command_ = "scp";
   std::string ssh_command_ = "ssh";
   std::string user_host_;
-  int ssh_port_ = kDefaultSshPort;
 };
 
 }  // namespace cdc_ft

common/remote_util_test.cc

@@ -21,9 +21,6 @@
 namespace cdc_ft {
 namespace {
 
-constexpr int kSshPort = 12345;
-constexpr char kSshPortArg[] = "-p 12345";
-
 constexpr char kUserHost[] = "user@example.com";
 constexpr char kUserHostArg[] = "\"user@example.com\"";
 
@@ -39,12 +36,11 @@ constexpr char kCommand[] = "my_command";
 class RemoteUtilTest : public ::testing::Test {
  public:
   RemoteUtilTest()
-      : util_(/*verbosity=*/0, /*quiet=*/false, &process_factory_,
+      : util_(kUserHost, /*verbosity=*/0, /*quiet=*/false, &process_factory_,
               /*forward_output_to_log=*/true) {}
 
   void SetUp() override {
     Log::Initialize(std::make_unique<ConsoleLog>(LogLevel::kInfo));
-    util_.SetUserHostAndPort(kUserHost, kSshPort);
   }
 
   void TearDown() override { Log::Shutdown(); }
@@ -64,31 +60,29 @@ class RemoteUtilTest : public ::testing::Test {
 
 TEST_F(RemoteUtilTest, BuildProcessStartInfoForSsh) {
   ProcessStartInfo si = util_.BuildProcessStartInfoForSsh(kCommand);
-  ExpectContains(si.command, {"ssh", kSshPortArg, kUserHostArg, kCommand});
+  ExpectContains(si.command, {"ssh", kUserHostArg, kCommand});
 }
 
 TEST_F(RemoteUtilTest, BuildProcessStartInfoForSshPortForward) {
   ProcessStartInfo si = util_.BuildProcessStartInfoForSshPortForward(
       kLocalPort, kRemotePort, kRegular);
-  ExpectContains(si.command,
-                 {"ssh", kSshPortArg, kUserHostArg, kPortForwardingArg});
+  ExpectContains(si.command, {"ssh", kUserHostArg, kPortForwardingArg});
 
   si = util_.BuildProcessStartInfoForSshPortForward(kLocalPort, kRemotePort,
                                                     kReverse);
-  ExpectContains(si.command,
-                 {"ssh", kSshPortArg, kUserHostArg, kReversePortForwardingArg});
+  ExpectContains(si.command, {"ssh", kUserHostArg, kReversePortForwardingArg});
 }
 
 TEST_F(RemoteUtilTest, BuildProcessStartInfoForSshPortForwardAndCommand) {
   ProcessStartInfo si = util_.BuildProcessStartInfoForSshPortForwardAndCommand(
       kLocalPort, kRemotePort, kRegular, kCommand);
-  ExpectContains(si.command, {"ssh", kSshPortArg, kUserHostArg,
-                              kPortForwardingArg, kCommand});
+  ExpectContains(si.command,
+                 {"ssh", kUserHostArg, kPortForwardingArg, kCommand});
 
   si = util_.BuildProcessStartInfoForSshPortForwardAndCommand(
       kLocalPort, kRemotePort, kReverse, kCommand);
-  ExpectContains(si.command, {"ssh", kSshPortArg, kUserHostArg,
-                              kReversePortForwardingArg, kCommand});
+  ExpectContains(si.command,
+                 {"ssh", kUserHostArg, kReversePortForwardingArg, kCommand});
 }
 TEST_F(RemoteUtilTest, BuildProcessStartInfoForSshWithCustomCommand) {
   constexpr char kCustomSshCmd[] = "C:\\path\\to\\ssh.exe --fooarg --bararg=42";

data_store/data_provider.cc

@@ -330,10 +330,10 @@ void DataProvider::CleanupThreadMain() {
       WriterMutexLockList locks;
       LockAllMutexes(&locks);
       chunks_updated_ = false;
-      LOG_DEBUG("Starting cache cleanup");
+      LOG_INFO("Starting cache cleanup");
       Stopwatch sw;
       absl::Status status = writer_->Cleanup();
-      LOG_DEBUG("Finished cache cleanup in %0.3f seconds", sw.ElapsedSeconds());
+      LOG_INFO("Finished cache cleanup in %0.3f seconds", sw.ElapsedSeconds());
       next_cleanup_time =
           steady_clock_->Now() + std::chrono::seconds(cleanup_timeout_sec_);
       absl::MutexLock cleaned_lock(&cleaned_mutex_);

integration_tests/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

integration_tests/cdc_rsync/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

integration_tests/cdc_rsync/all_tests.py

@@ -0,0 +1,44 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+import unittest
+
+from integration_tests.cdc_rsync import connection_test
+from integration_tests.cdc_rsync import deployment_test
+from integration_tests.cdc_rsync import dry_run_test
+from integration_tests.cdc_rsync import output_test
+from integration_tests.cdc_rsync import upload_test
+from integration_tests.framework import test_base
+
+
+# pylint: disable=g-doc-args,g-doc-return-or-yield
+def load_tests(loader, unused_tests, unused_pattern):
+  """Customizes the list of test cases to run.
+
+  See the Python documentation for details:
+  https://docs.python.org/3/library/unittest.html#load-tests-protocol
+  """
+  suite = unittest.TestSuite()
+  suite.addTests(loader.loadTestsFromModule(connection_test))
+  suite.addTests(loader.loadTestsFromModule(deployment_test))
+  suite.addTests(loader.loadTestsFromModule(dry_run_test))
+  suite.addTests(loader.loadTestsFromModule(output_test))
+  suite.addTests(loader.loadTestsFromModule(upload_test))
+
+  return suite
+
+
+if __name__ == '__main__':
+  test_base.main()

integration_tests/cdc_rsync/connection_test.py

@@ -0,0 +1,131 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""cdc_rsync connection test."""
+
+from concurrent import futures
+import socket
+import time
+
+from integration_tests.framework import utils
+from integration_tests.cdc_rsync import test_base
+
+RETURN_CODE_SUCCESS = 0
+RETURN_CODE_GENERIC_ERROR = 1
+RETURN_CODE_CONNECTION_TIMEOUT = 2
+RETURN_CODE_ADDRESS_IN_USE = 4
+
+FIRST_PORT = 44450
+LAST_PORT = 44459
+
+
+class ConnectionTest(test_base.CdcRsyncTest):
+  """cdc_rsync connection test class."""
+
+  def test_valid_instance(self):
+    """Runs rsync with --instance option for a valid id.
+
+       1) Uploads a file with --instance option instead of --ip --port.
+       2) Checks the file exists on the used instance.
+    """
+    utils.create_test_file(self.local_data_path, 1024)
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir)
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.does_file_exist_remotely(self.remote_data_path))
+
+  def test_invalid_instance(self):
+    """Runs rsync with --instance option for an invalid id.
+
+       1) Uploads a file with --instance option for a non-existing id.
+       2) Checks the error message.
+    """
+    bad_host = 'bad_host'
+
+    utils.create_test_file(self.local_data_path, 1024)
+    res = utils.run_rsync(self.local_data_path,
+                          bad_host + ":" + self.remote_base_dir)
+    self.assertEqual(res.returncode, RETURN_CODE_GENERIC_ERROR)
+    self.assertIn('lost connection', str(res.stderr))
+
+  def test_contimeout(self):
+    """Runs rsync with --contimeout option for an invalid ip.
+
+       1) Uploads a file with bad IP address.
+       2) Checks the error message and that it timed out after ~5 seconds.
+       3) Uploads a file with bad IP address and --contimeout 1.
+       4) Checks the error message and that it timed out after ~1 second.
+
+    """
+    utils.create_test_file(self.local_data_path, 1024)
+    bad_host = '192.0.2.1'
+    start = time.time()
+    res = utils.run_rsync(self.local_data_path,
+                          bad_host + ":" + self.remote_base_dir)
+    elapsed_time = time.time() - start
+    self.assertGreater(elapsed_time, 4.5)
+    self.assertEqual(res.returncode, RETURN_CODE_CONNECTION_TIMEOUT)
+    self.assertIn('Error: Server connection timed out', str(res.stderr))
+
+    start = time.time()
+    res = utils.run_rsync(self.local_data_path,
+                          bad_host + ":" + self.remote_base_dir,
+                          '--contimeout=1')
+    elapsed_time = time.time() - start
+    self.assertLess(elapsed_time, 3)
+    self.assertEqual(res.returncode, RETURN_CODE_CONNECTION_TIMEOUT)
+    self.assertIn('Error: Server connection timed out', str(res.stderr))
+
+  def test_multiple_instances(self):
+    """Runs multiple instances of rsync at the same time."""
+    num_instances = LAST_PORT - FIRST_PORT + 1
+
+    local_data_paths = []
+    for n in range(num_instances):
+      path = self.local_base_dir + ('testdata_%i.dat' % n)
+      utils.create_test_file(path, 1024)
+      local_data_paths.append(path)
+
+    with futures.ThreadPoolExecutor(max_workers=num_instances) as executor:
+      res = []
+      for n in range(num_instances):
+        res.append(
+            executor.submit(utils.run_rsync, local_data_paths[n],
+                            self.remote_base_dir))
+      for r in res:
+        self._assert_rsync_success(r.result())
+
+  def test_address_in_use(self):
+    """Blocks all ports and checks that rsync fails with the expected error."""
+    sockets = []
+    try:
+      # Occupy all ports.
+      for port in range(FIRST_PORT, LAST_PORT + 1):
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        sockets.append(s)
+        s.bind(('127.0.0.1', port))
+        s.listen()
+
+      # rsync shouldn't be able to find an available port now.
+      utils.create_test_file(self.local_data_path, 1024)
+      res = utils.run_rsync(self.local_data_path, self.remote_base_dir)
+      self.assertIn('All ports are already in use', str(res.stderr))
+
+    finally:
+      for s in sockets:
+        s.close()
+
+
+if __name__ == '__main__':
+  test_base.test_base.main()

integration_tests/cdc_rsync/deployment_test.py

@@ -0,0 +1,110 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""cdc_rsync deployment test."""
+
+from integration_tests.framework import utils
+from integration_tests.cdc_rsync import test_base
+
+REMOTE_FOLDER = '~/.cache/cdc-file-transfer/bin/'
+
+
+class DeploymentTest(test_base.CdcRsyncTest):
+  """cdc_rsync deployment test class."""
+
+  def _assert_deployment(self, initial_ts, file, msg):
+    """Checks rsync and library are uploaded and the given file's timestamp matches initial_ts."""
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir)
+    self._assert_rsync_success(res)
+    self.assertIn(msg, str(res.stdout))
+    changed_ts = utils.get_ssh_command_output('stat --format=%%y %s' %
+                                              REMOTE_FOLDER + file)
+    self.assertEqual(initial_ts, changed_ts)
+
+  def _change_file_preserve_timestamp(self, file):
+    """Changes a file preserving it timestamp."""
+    utils.get_ssh_command_output(
+        'touch -r %s %s' %
+        (REMOTE_FOLDER + file, REMOTE_FOLDER + file + '.tmp'))
+    utils.get_ssh_command_output('truncate -s +100 %s' % REMOTE_FOLDER + file)
+    utils.get_ssh_command_output(
+        'touch -r %s %s' %
+        (REMOTE_FOLDER + file + '.tmp', REMOTE_FOLDER + file))
+    utils.get_ssh_command_output('rm  %s' % (REMOTE_FOLDER + file + '.tmp'))
+
+  def test_no_server(self):
+    """Checks that cdc_rsync_server is uploaded if not present on the gamelet.
+
+      1) Wipes ‘/opt/developer/tools/bin/’ on the gamelet.
+      2) Uploads a file.
+      3) Verifies that cdc_rsync_server exists in that folder.
+    """
+    utils.get_ssh_command_output('rm -rf %s*' % REMOTE_FOLDER)
+    utils.create_test_file(self.local_data_path, 1024)
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir)
+    self._assert_rsync_success(res)
+    self.assertIn('Server not deployed. Deploying...', str(res.stdout))
+    self._assert_remote_dir_contains(['cdc_rsync_server'],
+                                     remote_dir=REMOTE_FOLDER,
+                                     pattern='"*"')
+
+  def test_modified_server(self):
+    """Checks that cdc_rsync_server is re-uploaded.
+
+      1) Touches cdc_rsync_server in ‘REMOTE_FOLDER’.
+      2) Uploads a file.
+      3) Verifies that cdc_rsync_server is re-uploaded.
+      4) Appends a few bytes to cdc_rsync_server while keeping its timestamp.
+      6) Uploads a file.
+      7) Verifies that cdc_rsync_server is re-uploaded.
+    """
+    # To be sure that cdc_rsync_server exist on the remote system
+    # do an "empty" copy.
+    utils.run_rsync(self.local_base_dir, self.remote_base_dir)
+
+    remote_server_path = REMOTE_FOLDER + 'cdc_rsync_server'
+    initial_ts = utils.get_ssh_command_output('stat --format=%%y %s' %
+                                              remote_server_path)
+    utils.get_ssh_command_output('touch -d \'1 November 2020 00:00\' %s' %
+                                 remote_server_path)
+    changed_ts = utils.get_ssh_command_output('stat --format=%%y %s' %
+                                              remote_server_path)
+    self.assertNotEqual(initial_ts, changed_ts)
+    utils.create_test_file(self.local_data_path, 1024)
+    self._assert_deployment(initial_ts, 'cdc_rsync_server',
+                            'Server outdated. Redeploying...')
+
+    self._change_file_preserve_timestamp('cdc_rsync_server')
+    self._assert_deployment(initial_ts, 'cdc_rsync_server',
+                            'Server outdated. Redeploying...')
+
+  def test_read_only_server(self):
+    """Checks that cdc_rsync_server is overwritten if it is read-only."""
+    utils.create_test_file(self.local_data_path, 1024)
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir)
+    self._assert_rsync_success(res)
+
+    # Modify cdc_rsync_server and wipe permissions.
+    remote_server_path = REMOTE_FOLDER + 'cdc_rsync_server'
+    utils.get_ssh_command_output('echo "xxx" > %s && chmod 0 %s' %
+                                 (remote_server_path, remote_server_path))
+
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir)
+    self._assert_rsync_success(res)
+    self.assertIn('Server failed to start. Redeploying...', str(res.stdout))
+
+
+if __name__ == '__main__':
+  test_base.test_base.main()

integration_tests/cdc_rsync/dry_run_test.py

@@ -0,0 +1,116 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""cdc_rsync dry-run test."""
+
+from integration_tests.framework import utils
+from integration_tests.cdc_rsync import test_base
+
+
+class DryRunTest(test_base.CdcRsyncTest):
+  """cdc_rsync dry-run test class."""
+
+  def test_dry_run(self):
+    """Verifies --dry-run option.
+
+      1) Uploads file1.txt and file2.txt.
+      2) Modifies file2.txt.
+      3) Dry-runs file2.txt and file3.txt with --dry-run -r --delete.
+      Result: a missing (file3.txt), a changed (file2.txt) and an extraneous
+      (file1.txt) file. No files should be changed on the server.
+
+    """
+
+    files = ['file1.txt', 'file2.txt', 'file3.txt']
+
+    for file in files:
+      utils.create_test_file(self.local_base_dir + file, 987)
+
+    res = utils.run_rsync(self.local_base_dir + 'file1.txt',
+                          self.local_base_dir + 'file2.txt',
+                          self.remote_base_dir, '-v')
+    self._assert_rsync_success(res)
+    self._assert_remote_dir_contains(['file1.txt', 'file2.txt'])
+
+    # Dry-run of uploading changed/new/to delete files.
+    utils.create_test_file(self.local_base_dir + 'file2.txt', 2534)
+    res = utils.run_rsync(self.local_base_dir + 'file2.txt',
+                          self.local_base_dir + 'file3.txt',
+                          self.remote_base_dir, '-v', '--dry-run', '--delete',
+                          '-r')
+    self._assert_rsync_success(res)
+    self.assertTrue(
+        utils.files_count_is(res, missing=1, changed=1, extraneous=1))
+    self._assert_remote_dir_does_not_contain(['file3.txt'])
+    self._assert_remote_dir_contains(['file1.txt', 'file2.txt'])
+    self.assertIn('file1.txt', str(res.stdout))
+    self.assertIn('deleted 1 / 1', str(res.stdout))
+    self.assertIn('file2.txt', str(res.stdout))
+    self.assertIn('D100%', str(res.stdout))
+    self.assertIn('file3.txt', str(res.stdout))
+    self.assertIn('C100%', str(res.stdout))
+    self.assertFalse(
+        utils.sha1_matches(self.local_base_dir + 'file2.txt',
+                           self.remote_base_dir + 'file2.txt'))
+
+  def test_dry_run_sync_folder_when_remote_file_recursive_with_delete(self):
+    """Dry-runs a recursive upload of a folder while removing a remote file with the same name with --delete."""
+
+    local_folder = self.local_base_dir + 'foldertocopy\\'
+    utils.create_test_directory(local_folder)
+    utils.get_ssh_command_output(
+        'mkdir -p %s && touch %s' %
+        (self.remote_base_dir, self.remote_base_dir + 'foldertocopy'))
+
+    res = utils.run_rsync(self.local_base_dir + 'foldertocopy',
+                          self.remote_base_dir, '-r', '--dry-run', '--delete')
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, extraneous=1, missing_dir=1))
+    self.assertFalse(
+        utils.does_directory_exist_remotely(self.remote_base_dir +
+                                            'foldertocopy'))
+    self.assertTrue(
+        utils.does_file_exist_remotely(self.remote_base_dir + 'foldertocopy'))
+    self.assertIn('1/1 file(s) and 0/0 folder(s) deleted', str(res.stdout))
+
+  def test_dry_run_sync_file_when_remote_folder_recursive_with_delete(self):
+    """Dry-runs a recursive upload of a file while removing an empty remote folder with the same name with --delete."""
+    utils.create_test_file(self.local_data_path, 1024)
+    utils.get_ssh_command_output('mkdir -p %s' % self.remote_data_path)
+
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir,
+                          '--dry-run', '-r', '--delete')
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, missing=1, extraneous_dir=1))
+    self.assertFalse(utils.does_file_exist_remotely(self.remote_data_path))
+    self.assertTrue(utils.does_directory_exist_remotely(self.remote_data_path))
+    self.assertIn('0/0 file(s) and 1/1 folder(s) deleted', str(res.stdout))
+
+  def test_dry_run_sync_file_when_remote_folder_empty(self):
+    """Dry-runs a non-recursive upload of a file while there is an empty remote folder with the same name."""
+    utils.create_test_file(self.local_data_path, 1024)
+    utils.get_ssh_command_output('mkdir -p %s' % self.remote_data_path)
+
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir,
+                          '--dry-run')
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, missing=1, extraneous_dir=1))
+    self.assertFalse(utils.does_file_exist_remotely(self.remote_data_path))
+    self.assertTrue(utils.does_directory_exist_remotely(self.remote_data_path))
+    self.assertNotIn('0/0 file(s) and 1/1 folder(s) deleted', str(res.stdout))
+
+
+if __name__ == '__main__':
+  test_base.test_base.main()

integration_tests/cdc_rsync/output_test.py

@@ -0,0 +1,243 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""cdc_rsync output test."""
+
+import json
+
+from integration_tests.framework import utils
+from integration_tests.cdc_rsync import test_base
+
+
+class OutputTest(test_base.CdcRsyncTest):
+  """cdc_rsync output test class."""
+
+  def test_plain(self):
+    """Runs rsync and verifies the total progress.
+
+      1) Uploads a file, verifies that the total progress is shown.
+      2) Uploads an empty folder with -r --delete options.
+      Verifies that the total delete messages are shown.
+    """
+    utils.create_test_file(self.local_data_path, 1024)
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir)
+    self._assert_rsync_success(res)
+    self.assertIn('100% TOT', str(res.stdout))
+
+    utils.remove_test_file(self.local_data_path)
+    res = utils.run_rsync(self.local_base_dir, self.remote_base_dir, '-r',
+                          '--delete')
+    self._assert_rsync_success(res)
+    self.assertIn('1/1 file(s) and 0/0 folder(s) deleted', str(res.stdout))
+
+  def test_verbose_1(self):
+    """Runs rsync with -v option for multiple files.
+
+      1) Uploads 3 files with ‘-v’.
+      Verifies that each file is listed in the output as ‘C100%’.
+      2) Modifies 3 files, uploads them again with ‘--v’.
+      Verifies that each file is listed in the output as ‘D100%’.
+      3) Uploads an empty folder with -r --delete options.
+      Verifies that the delete messages are shown.
+    """
+    files = ['file1. txt', 'file2.txt', 'file3.txt']
+    for file in files:
+      utils.create_test_file(self.local_base_dir + file, 1024)
+    res = utils.run_rsync(self.local_base_dir, self.remote_base_dir, '-v', '-r')
+    self._assert_rsync_success(res)
+    self.assertEqual(3, str(res.stdout).count('C100%'))
+
+    for file in files:
+      utils.create_test_file(self.local_base_dir + file, 2048)
+    res = utils.run_rsync(self.local_base_dir, self.remote_base_dir, '-v', '-r')
+    self._assert_rsync_success(res)
+    self.assertEqual(3, str(res.stdout).count('D100%'))
+
+    for file in files:
+      utils.remove_test_file(self.local_base_dir + file)
+    res = utils.run_rsync(self.local_base_dir, self.remote_base_dir, '-r',
+                          '--delete')
+    self._assert_rsync_success(res)
+    self.assertIn('will be deleted due to --delete', str(res.stdout))
+    self.assertIn('3/3 file(s) and 0/0 folder(s) deleted', str(res.stdout))
+
+  def test_verbose_2(self):
+    """Runs rsync with -vv option.
+
+      1) Uploads a file with ‘-vv’.
+      2) Verifies that additional logs show up.
+    """
+    utils.create_test_file(self.local_data_path, 1024)
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir, '-vv')
+    self._assert_rsync_success(res)
+    output = str(res.stdout)
+
+    # client-side output
+    self._assert_regex('Starting process', output)
+    self._assert_not_regex(
+        r'process\.cc\([0-9]+\): Start\(\): Starting process', output)
+
+    # server-side output
+    self._assert_regex(
+        'INFO    Finding all files in destination folder '
+        f"'{self.remote_base_dir}'", output)
+    self.assertNotIn('DEBUG', output)
+
+  def test_verbose_3(self):
+    """Runs rsync with -vvv option.
+
+      1) Uploads a file with ‘-vvv’.
+      Verifies that additional logs show up (LOG_DEBUG logs).
+      2) Uploads a file to  ‘/invalid’ with ‘-vvv’.
+      Verifies that error messages including filenames are shown.
+    """
+    utils.create_test_file(self.local_data_path, 1024)
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir, '-vvv')
+    self._assert_rsync_success(res)
+    output = str(res.stdout)
+
+    # client-side output
+    self._assert_regex(
+        r'cdc_rsync_client\.cc\([0-9]+\): SendOptions\(\): Sending options',
+        output)
+
+    # server-side output
+    self._assert_regex(
+        r'DEBUG   server_socket\.cc\([0-9]+\): Receive\(\): EOF\(\) detected',
+        output)
+
+    # TODO: Add a check here, as currently the output is misleading
+    # res = utils.run_rsync(self.local_data_path, '/invalid', '-vvv')
+
+  def test_verbose_4(self):
+    """Runs rsync with -vvv option.
+
+      1) Uploads a file with ‘-vvvv’.
+      2) Verifies that additional logs show up (LOG_VERBOSE logs).
+    """
+    utils.create_test_file(self.local_data_path, 1024)
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir, '-vvvv')
+    self._assert_rsync_success(res)
+    output = str(res.stdout)
+
+    # client-side output
+    self._assert_regex(
+        r'message_pump\.cc\([0-9]+\): ThreadDoSendPacket\(\): Sent packet of size',
+        output)
+
+    # server-side output
+    self._assert_regex(
+        r'VERBOSE message_pump\.cc\([0-9]+\): ThreadDoReceivePacket\(\): Received packet of size',
+        output)
+
+  def test_quiet(self):
+    """Runs rsync with -q option.
+
+      1) Uploads a file with ‘-q’.
+      2) Verifies that no output is shown.
+    """
+    utils.create_test_file(self.local_data_path, 1024)
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir, '-q')
+    self._assert_rsync_success(res)
+    self.assertEqual('\r\n', res.stdout)
+
+  def test_quiet_error(self):
+    """Runs rsync with -q option still showing errors.
+
+      1) Uploads a file with ‘-q’ and bad options.
+      2) Verifies that an error message is shown.
+    """
+    utils.create_test_file(self.local_data_path, 1024)
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir, '-q',
+                          '-t')
+    self.assertEqual(res.returncode, 1)
+    self.assertEqual('\r\n', str(res.stdout))
+    self.assertIn('Unknown option: \'t\'', str(res.stderr))
+    # TODO: Add a test case for the non-existing destination.
+
+  def test_existing_verbose_1(self):
+    """Runs rsync with -v --existing."""
+
+    files = ['file1.txt', 'file2.txt']
+    for file in files:
+      utils.create_test_file(self.local_base_dir + file, 1024)
+    res = utils.run_rsync(self.local_base_dir, self.remote_base_dir, '-r')
+    self._assert_rsync_success(res)
+
+    files.append('file3.txt')
+    for file in files:
+      utils.create_test_file(self.local_base_dir + file, 2048)
+    res = utils.run_rsync(self.local_base_dir, self.remote_base_dir, '-v', '-r',
+                          '--existing')
+    self._assert_rsync_success(res)
+    output = str(res.stdout)
+    self.assertEqual(2, output.count('D100%'))
+    self.assertNotIn('file3.txt', output)
+
+  def test_json_per_file(self):
+    """Runs rsync with -v --json."""
+
+    local_path = self.local_base_dir + 'test.txt'
+    utils.create_test_file(local_path, 1024)
+    res = utils.run_rsync(local_path, self.remote_base_dir, '-v', '--json')
+    self._assert_rsync_success(res)
+    output = str(res.stdout)
+
+    for val in self.parse_json(output):
+      self.assertEqual(val['file'], 'test.txt')
+      self.assertEqual(val['operation'], 'Copy')
+      self.assertEqual(val['size'], 1024)
+
+      # Those are actually all floats, but sometimes they get rounded to ints.
+      self.assertTrue(self.is_float_or_int(val['bytes_per_second']))
+      self.assertTrue(self.is_float_or_int(val['duration']))
+      self.assertTrue(self.is_float_or_int(val['eta']))
+      self.assertTrue(self.is_float_or_int(val['total_duration']))
+      self.assertTrue(self.is_float_or_int(val['total_eta']))
+      self.assertTrue(self.is_float_or_int(val['total_progress']))
+
+  def test_json_total(self):
+    """Runs rsync with --json."""
+
+    local_path = self.local_base_dir + 'test.txt'
+    utils.create_test_file(local_path, 1024)
+    res = utils.run_rsync(local_path, self.remote_base_dir, '--json')
+    self._assert_rsync_success(res)
+    output = str(res.stdout)
+
+    for val in self.parse_json(output):
+      self.assertNotIn('file', val)
+
+      # Those are actually all floats, but sometimes they get rounded to ints.
+      self.assertTrue(self.is_float_or_int(val['total_duration']))
+      self.assertTrue(self.is_float_or_int(val['total_eta']))
+      self.assertTrue(self.is_float_or_int(val['total_progress']))
+
+  def parse_json(self, output):
+    """Parses the JSON lines of output."""
+    lines = output.split('\r\n')
+    json_values = []
+    for line in lines:
+      if str.startswith(line, '{'):
+        json_values.append(json.loads(line.strip()))
+    return json_values
+
+  def is_float_or_int(self, val):
+    """Returns true if val is a float or an int."""
+    return isinstance(val, float) or isinstance(val, int)
+
+
+if __name__ == '__main__':
+  test_base.test_base.main()

integration_tests/cdc_rsync/test_base.py

@@ -0,0 +1,114 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""cdc_rsync base test class."""
+
+import datetime
+import logging
+import tempfile
+import re
+import unittest
+
+from integration_tests.framework import utils
+from integration_tests.framework import test_base
+
+
+class CdcRsyncTest(unittest.TestCase):
+  """cdc_rsync base test class."""
+
+  tmp_dir = None
+  local_base_dir = None
+  remote_base_dir = None
+  local_data_path = None
+  remote_data_path = None
+
+  def setUp(self):
+    """Cleans up the remote test data folder, logs a marker, and initializes random."""
+    super(CdcRsyncTest, self).setUp()
+    logging.debug('CdcRsyncTest -> setUp')
+
+    utils.initialize(test_base.Flags.binary_path, None,
+                     test_base.Flags.user_host)
+
+    now_str = datetime.datetime.now().strftime('%Y%m%d-%H%M%S')
+    self.tmp_dir = tempfile.TemporaryDirectory(
+        prefix=f'_cdc_rsync_test_{now_str}')
+    self.local_base_dir = self.tmp_dir.name + '\\'
+    self.remote_base_dir = f'/tmp/_cdc_rsync_test_{now_str}/'
+    self.local_data_path = self.local_base_dir + 'testdata.dat'
+    self.remote_data_path = self.remote_base_dir + 'testdata.dat'
+
+    logging.info('Local base dir: "%s"', self.local_base_dir)
+    logging.info('Remote base dir: "%s"', self.remote_base_dir)
+    utils.initialize_random()
+
+  def tearDown(self):
+    """Cleans up the local and remote temp directories."""
+    super(CdcRsyncTest, self).tearDown()
+    logging.debug('CdcRsyncTest -> tearDown')
+    self.tmp_dir.cleanup()
+    utils.get_ssh_command_output(f'rm -rf {self.remote_base_dir}')
+
+  def _assert_rsync_success(self, res):
+    """Asserts if the return code is 0 and outputs return message with args."""
+    self.assertEqual(res.returncode, 0, 'Return value is ' + str(res))
+
+  def _assert_regex(self, regex, value):
+    """Asserts that the regex string matches the given value."""
+    self.assertIsNotNone(
+        re.search(regex, value), f'"Regex {regex}" does not match "{value}"')
+
+  def _assert_not_regex(self, regex, value):
+    """Asserts that the regex string does not match the given value."""
+    self.assertIsNone(
+        re.search(regex, value),
+        f'"Regex {regex}" unexpectedly matches "{value}"')
+
+  def _assert_remote_dir_contains(self,
+                                  file_list,
+                                  remote_dir=None,
+                                  pattern='"*.[t|d]*"'):
+    """Asserts that the remote base dir contains exactly the list of files.
+
+    Args:
+        file_list (list of strings): List of relative file paths to check
+        remote_dir (string, optional): Remote directory. Defaults to
+            remote_base_dir
+        pattern (string, optional): Pattern for matching file names.
+    """
+    find_res = utils.get_ssh_command_output(
+        'cd %s && find -name %s -print' %
+        (remote_dir or self.remote_base_dir, pattern))
+
+    # Note that assertCountEqual compares items independently of order
+    # (not just the size of the list).
+    found = sorted(
+        filter(lambda item: item and item != '.', find_res.split('\r\n')))
+    expected = sorted(['./' + f for f in file_list])
+    self.assertListEqual(found, expected)
+
+  def _assert_remote_dir_does_not_contain(self, file_list):
+    """Asserts that the remote base dir contains none of the listed files.
+
+    Args:
+        file_list (list of strings): List of relative file paths to check
+    """
+    find_res = utils.get_ssh_command_output(
+        'cd %s && find -name "*.[t|d]*" -print' % self.remote_base_dir)
+
+    found = set(file_name for file_name in filter(None, find_res.split('\n')))
+
+    for file in file_list:
+      self.assertNotIn('./' + file, found)

integration_tests/cdc_rsync/upload_test.py

@@ -0,0 +1,894 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""cdc_rsync upload test."""
+
+import json
+import logging
+import os
+import subprocess
+import time
+
+from integration_tests.framework import utils
+from integration_tests.cdc_rsync import test_base
+
+
+class UploadTest(test_base.CdcRsyncTest):
+  """cdc_rsync upload test class."""
+
+  def test_single_uncompressed(self):
+    """Uploads and syncs a file uncompressed."""
+
+    self._do_test_single(compressed=False)
+
+  def test_upload_compressed(self):
+    """Uploads and syncs a file compressed."""
+
+    self._do_test_single(compressed=True)
+
+  def _do_test_single(self, compressed):
+    """Runs rsync 3 times and validates results.
+
+       1) Uploads a file, checks sha1 hashes.
+       2) Uploads the same file again, checks nothing changed.
+       3) Modifies the file and uploads again. Checks sha1 hashes.
+
+    Args:
+        compressed (bool): Whether to append '--compress' or not.
+    """
+    compressed_arg = '--compress' if compressed else None
+
+    utils.create_test_file(self.local_data_path, 1024)
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir,
+                          compressed_arg)
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, missing=1))
+    self.assertTrue(
+        utils.sha1_matches(self.local_data_path, self.remote_data_path))
+
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir,
+                          compressed_arg)
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, matching=1))
+
+    utils.create_test_file(self.local_data_path, 2534)
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir,
+                          compressed_arg)
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, changed=1))
+    self.assertTrue(
+        utils.sha1_matches(self.local_data_path, self.remote_data_path))
+
+  def test_backslash_in_dest_folder(self):
+    r"""Verifies uploading to \mnt\developer."""
+
+    filepath = os.path.join(self.local_base_dir, 'file1.txt')
+    utils.create_test_file(filepath, 1)
+    res = utils.run_rsync(filepath, self.remote_base_dir.replace('/', '\\'))
+    self.assertTrue(utils.files_count_is(res, missing=1))
+    self._assert_remote_dir_contains(['file1.txt'])
+
+  def test_backslash_in_source_folder(self):
+    r"""Verifies uploading from /source/folder."""
+
+    filepath = os.path.join(self.local_base_dir, 'file1.txt')
+    utils.create_test_file(filepath, 1)
+    filepath = filepath.replace('\\', '/')
+    res = utils.run_rsync(filepath, self.remote_base_dir)
+    self.assertTrue(utils.files_count_is(res, missing=1))
+    self._assert_remote_dir_contains(['file1.txt'])
+
+  def test_single_unicode(self):
+    """Uploads a file with a non-ascii unicode path and checks sha1 signatures."""
+
+    nonascii_local_data_path = self.local_base_dir + '⛽⛽⛽⛽⛽⛽⛽⛽.dat'
+    nonascii_remote_data_path = self.remote_base_dir + '⛽⛽⛽⛽⛽⛽⛽⛽.dat'
+    utils.create_test_file(nonascii_local_data_path, 1024)
+    # In order to check that non-ascii characters are not considered as
+    # wildcard
+    # ?  characters, create a second file.  Only 1 file should be uploaded.
+    utils.create_test_file(self.local_data_path, 1024)
+    res = utils.run_rsync(nonascii_local_data_path, self.remote_base_dir, None)
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, missing=1))
+    self.assertTrue(
+        utils.sha1_matches(nonascii_local_data_path, nonascii_remote_data_path))
+
+  def test_uncompressed_no_empty_folders(self):
+    """Uploads and syncs multiple files uncompressed in different folders."""
+
+    self._do_test_no_empty_folders(compressed=False)
+
+  def test_compressed_no_empty_folders(self):
+    """Uploads and syncs multiple files compressed in different folders."""
+
+    self._do_test_no_empty_folders(compressed=True)
+
+  def _do_test_no_empty_folders(self, compressed):
+    """Runs rsync with(out) -r for a non-trivial directory and validates results.
+
+       1) Uploads a source directory with -r, checks sha1 hashes.
+       |-- rootdir
+       |   |-- dir1
+       |      |-- file1_1.txt
+       |      |-- file1_2.txt
+       |   |-- dir2
+       |      |-- file2_1.txt
+       |   |-- file0.txt
+       2) Uploads the same source directory again without -r,
+       checks nothing has changed. The directory should be just skipped.
+       3) Uploads the same source directory with --delete option and with -r.
+       Nothing should change.
+       4) Removes dir1 and dir2 locally.
+       Uploads the same source directory with --delete option and with -r.
+       dir1 and dir2 should be removed from the remote instance.
+
+    Args:
+        compressed (bool): Whether to append '--compress' or not.
+    """
+    compressed_arg = '--compress' if compressed else None
+    local_root_path = self.local_base_dir + 'rootdir'
+    remote_root_path = self.remote_base_dir + 'rootdir/'
+    utils.create_test_file(local_root_path + '\\dir1\\file1_1.txt', 1024)
+    utils.create_test_file(local_root_path + '\\dir1\\file1_2.txt', 1024)
+    utils.create_test_file(local_root_path + '\\dir2\\file2_1.txt', 1024)
+    utils.create_test_file(local_root_path + '\\file0.txt', 1024)
+    res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg,
+                          '-r')
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, missing=4, missing_dir=3))
+    self.assertTrue(
+        utils.sha1_matches(local_root_path + '\\dir1\\file1_1.txt',
+                           remote_root_path + 'dir1/file1_1.txt'))
+    self.assertTrue(
+        utils.sha1_matches(local_root_path + '\\dir1\\file1_2.txt',
+                           remote_root_path + 'dir1/file1_2.txt'))
+    self.assertTrue(
+        utils.sha1_matches(local_root_path + '\\dir2\\file2_1.txt',
+                           remote_root_path + 'dir2/file2_1.txt'))
+    self.assertTrue(
+        utils.sha1_matches(local_root_path + '\\file0.txt',
+                           remote_root_path + 'file0.txt'))
+
+    res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg)
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, extraneous_dir=1))
+
+    res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg,
+                          '-r', '--delete')
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, matching=4, matching_dir=3))
+
+    utils.remove_test_directory(local_root_path + '\\dir1\\')
+    utils.remove_test_directory(local_root_path + '\\dir2\\')
+    res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg,
+                          '-r', '--delete')
+    self._assert_rsync_success(res)
+    self.assertTrue(
+        utils.files_count_is(
+            res, matching=1, extraneous=3, matching_dir=1, extraneous_dir=2))
+    self.assertFalse(
+        utils.does_directory_exist_remotely(remote_root_path + 'dir1'))
+    self.assertFalse(
+        utils.does_directory_exist_remotely(remote_root_path + 'dir2'))
+
+  def _do_test_no_empty_folders_with_backslash(self, compressed):
+    """Runs rsync with(out) -r for a non-trivial directory with a trailing backslash.
+
+       1) Uploads a source directory with -r, checks sha1 hashes.
+       Everything from rootdir should be copied except rootdir itself.
+       |-- rootdir
+       |   |-- dir1
+       |      |-- file1_1.txt
+       |      |-- file1_2.txt
+       |   |-- dir2
+       |      |-- file2_1.txt
+       |   |-- file0.txt
+       2) Uploads the same source directory again without -r,
+       checks nothing has changed. The directory should be just skipped.
+       3) Uploads the same source directory with --delete option and with -r.
+       Nothing should change.
+       4) Removes dir1 and dir2 locally.
+       Uploads the same source directory with --delete option and with -r.
+       dir1 and dir2 should be removed from the remote instance.
+
+    Args:
+        compressed (bool): Whether to append '--compress' or not.
+    """
+    compressed_arg = '--compress' if compressed else None
+    local_root_path = self.local_base_dir + 'rootdir\\'
+    utils.create_test_file(local_root_path + 'dir1\\file1_1.txt', 1024)
+    utils.create_test_file(local_root_path + 'dir1\\file1_2.txt', 1024)
+    utils.create_test_file(local_root_path + 'dir2\\file2_1.txt', 1024)
+    utils.create_test_file(local_root_path + 'file0.txt', 1024)
+    res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg,
+                          '-r')
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, missing=4, missing_dir=2))
+    self.assertTrue(
+        utils.sha1_matches(local_root_path + 'dir1\\file1_1.txt',
+                           self.remote_base_dir + 'dir1/file1_1.txt'))
+    self.assertTrue(
+        utils.sha1_matches(local_root_path + 'dir1\\file1_2.txt',
+                           self.remote_base_dir + 'dir1/file1_2.txt'))
+    self.assertTrue(
+        utils.sha1_matches(local_root_path + 'dir2\\file2_1.txt',
+                           self.remote_base_dir + 'dir2/file2_1.txt'))
+    self.assertTrue(
+        utils.sha1_matches(local_root_path + 'file0.txt',
+                           self.remote_base_dir + 'file0.txt'))
+
+    res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg)
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(
+        res, extraneous=1, extraneous_dir=2))  # file0.txt, dir1, dir2
+
+    res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg,
+                          '-r', '--delete')
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, matching=4, matching_dir=2))
+
+    utils.remove_test_directory(local_root_path + '\\dir1\\')
+    utils.remove_test_directory(local_root_path + '\\dir2\\')
+    res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg,
+                          '-r', '--delete')
+    self._assert_rsync_success(res)
+    self.assertTrue(
+        utils.files_count_is(res, matching=1, extraneous=3, extraneous_dir=2))
+    self.assertFalse(
+        utils.does_directory_exist_remotely(self.remote_base_dir + 'dir1'))
+    self.assertFalse(
+        utils.does_directory_exist_remotely(self.remote_base_dir + 'dir2'))
+
+  def test_uncompressed_no_empty_folders_with_backslash(self):
+    """Uploads multiple files uncompressed from a folder with a trailing backslash."""
+
+    self._do_test_no_empty_folders_with_backslash(compressed=False)
+
+  def test_compressed_no_empty_folders_with_backslash(self):
+    """Uploads multiple files compressed from a folder with a trailing backslash."""
+
+    self._do_test_no_empty_folders_with_backslash(compressed=True)
+
+  def test_uncompressed_with_empty_folders(self):
+    """Uploads and syncs multiple files uncompressed and empty folders."""
+
+    self._do_test_with_empty_folders(compressed=False)
+
+  def test_compressed_with_empty_folders(self):
+    """Uploads and syncs multiple files compress and empty folders."""
+
+    self._do_test_with_empty_folders(compressed=True)
+
+  def _do_test_with_empty_folders(self, compressed):
+    """Runs rsync with(out) -r for a non-trivial directory with empty folders.
+
+       1) Uploads a source directory with -r, checks sha1 hashes.
+       |-- rootdir
+       |   |-- dir1
+       |      |-- emptydir2
+       |      |-- file1_1.txt
+       |      |-- file1_2.txt
+       |   |-- dir2
+       |      |-- file2_1.txt
+       |   |-- emptydir1
+       |   |-- file0.txt
+       2) Uploads the same source directory again without -r,
+       checks nothing has changed. The directory should be just skipped.
+       3) Uploads the same source directory with --delete option and with -r.
+       Nothing should change.
+       4) Removes dir1 and dir2 locally.
+       Uploads the same source directory with --delete option and with -r.
+       dir1 and dir2 should be removed from the remote instance.
+
+    Args:
+        compressed (bool): Whether to append '--compress' or not.
+    """
+    compressed_arg = '--compress' if compressed else None
+    local_root_path = self.local_base_dir + 'rootdir'
+    remote_root_path = self.remote_base_dir + 'rootdir/'
+    utils.create_test_file(local_root_path + '\\dir1\\file1_1.txt', 1024)
+    utils.create_test_file(local_root_path + '\\dir1\\file1_2.txt', 1024)
+    utils.create_test_directory(local_root_path + '\\dir1\\emptydir2\\')
+    utils.create_test_file(local_root_path + '\\dir2\\file2_1.txt', 1024)
+    utils.create_test_file(local_root_path + '\\file0.txt', 1024)
+    utils.create_test_directory(local_root_path + '\\emptydir1\\')
+    res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg,
+                          '-r')
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, missing=4, missing_dir=5))
+    self.assertTrue(
+        utils.sha1_matches(local_root_path + '\\dir1\\file1_1.txt',
+                           remote_root_path + 'dir1/file1_1.txt'))
+    self.assertTrue(
+        utils.sha1_matches(local_root_path + '\\dir1\\file1_2.txt',
+                           remote_root_path + 'dir1/file1_2.txt'))
+    self.assertTrue(
+        utils.sha1_matches(local_root_path + '\\dir2\\file2_1.txt',
+                           remote_root_path + 'dir2/file2_1.txt'))
+    self.assertTrue(
+        utils.sha1_matches(local_root_path + '\\file0.txt',
+                           remote_root_path + 'file0.txt'))
+
+    res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg)
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, extraneous_dir=1))
+
+    res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg,
+                          '-r', '--delete')
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, matching=4, matching_dir=5))
+
+    utils.remove_test_directory(local_root_path + '\\dir1\\')
+    utils.remove_test_directory(local_root_path + '\\dir2\\')
+    res = utils.run_rsync(local_root_path, self.remote_base_dir, compressed_arg,
+                          '-r', '--delete')
+    self._assert_rsync_success(res)
+    self.assertTrue(
+        utils.files_count_is(
+            res, matching=1, extraneous=3, matching_dir=2, extraneous_dir=3))
+    self.assertIn('3/3 file(s) and 3/3 folder(s) deleted', res.stdout)
+    self.assertFalse(
+        utils.does_directory_exist_remotely(remote_root_path + 'dir1'))
+    self.assertFalse(
+        utils.does_directory_exist_remotely(remote_root_path + 'dir2'))
+
+  def test_upload_empty_file(self):
+    """Uploads an empty file and checks sha1 signatures."""
+
+    empty_local_data_path = self.local_base_dir + 'emptyfile.dat'
+    empty_remote_data_path = self.remote_base_dir + 'emptyfile.dat'
+    utils.create_test_file(empty_local_data_path, 0)
+    res = utils.run_rsync(empty_local_data_path, self.remote_base_dir, None)
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, missing=1))
+    self.assertTrue(
+        utils.sha1_matches(empty_local_data_path, empty_remote_data_path))
+
+  def test_upload_empty_folder_with_backslash(self):
+    """Uploads an empty folder with a trailing backslash."""
+
+    self._do_test_upload_empty_folder(with_backslash=True)
+
+  def test_upload_empty_folder_no_backslash(self):
+    """Uploads an empty folder without a trailing backslash."""
+
+    self._do_test_upload_empty_folder(with_backslash=False)
+
+  def _do_test_upload_empty_folder(self, with_backslash=False):
+    """Uploads an empty folder."""
+
+    local_data_dir = (
+        self.local_base_dir +
+        'empty_folder\\' if with_backslash else self.local_base_dir +
+        'empty_folder')
+    res = utils.run_rsync(local_data_dir, self.remote_base_dir, None)
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, missing=0))
+
+  def test_whole_file_uncompressed(self):
+    """Uploads and syncs a file uncompressed with --whole-file."""
+
+    self._do_test_whole_file(compressed=False)
+
+  def test_whole_file_compressed(self):
+    """Uploads and syncs a file compressed with --whole-file."""
+
+    self._do_test_whole_file(compressed=True)
+
+  def _do_test_whole_file(self, compressed):
+    """Runs rsync 3 times with --whole-file -v options and validates results.
+
+       1) Uploads a file.
+       2) Modifies the file and uploads it with --whole-file and -v options.
+       Checks the output contains C100%, not D100%.
+       3) Modifies the file and uploads it with -W and -v options.
+       Checks the output contains C100%, not D100%.
+
+    Args:
+        compressed (bool): Whether to append '--compress' or not.
+    """
+    compressed_arg = '--compress' if compressed else None
+
+    utils.create_test_file(self.local_data_path, 1024)
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir,
+                          compressed_arg)
+
+    utils.create_test_file(self.local_data_path, 2534)
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir,
+                          compressed_arg, '--whole-file', '-v')
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, changed=1))
+    self.assertIn('will be copied due to -W/--whole-file', str(res.stdout))
+    self.assertIn('C100%', str(res.stdout))
+    self.assertTrue(
+        utils.sha1_matches(self.local_data_path, self.remote_data_path))
+
+    utils.create_test_file(self.local_data_path, 3456)
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir,
+                          compressed_arg, '-W', '-v')
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, changed=1))
+    self.assertIn('C100%', str(res.stdout))
+    self.assertTrue(
+        utils.sha1_matches(self.local_data_path, self.remote_data_path))
+
+  def test_keep_file_permissions(self):
+    """Verifies that file permissions are kept for changed files."""
+
+    # Upload a file and check permissions.
+    utils.create_test_file(self.local_data_path, 1024)
+    utils.run_rsync(self.local_data_path, self.remote_base_dir)
+    ls_res = utils.get_ssh_command_output('ls -al %s' % self.remote_data_path)
+    self.assertIn('-rw-r--r--', ls_res)
+
+    # Add executable bit.
+    utils.get_ssh_command_output('chmod a+x %s*' % self.remote_data_path)
+    ls_res = utils.get_ssh_command_output('ls -al %s' % self.remote_data_path)
+    self.assertIn('-rwxr-xr-x', ls_res)
+
+    # Sync file again and verify permissions don't change.
+    utils.create_test_file(self.local_data_path, 1337)
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir)
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, changed=1))
+    ls_res = utils.get_ssh_command_output('ls -al %s' % self.remote_data_path)
+    self.assertIn('-rwxr-xr-x', ls_res)
+
+  def test_include_exclude(self):
+    """Verifies the --include and --exclude options."""
+
+    files = [
+        'file1.txt', 'folder1\\file2.txt', 'folder1\\file3.dat',
+        'folder1\\folder2\\file4.txt', 'folder3\\file5.txt'
+    ]
+
+    for file in files:
+      utils.create_test_file(self.local_base_dir + file, 987)
+
+    # Upload file2.txt and file3.dat.
+    res = utils.run_rsync(self.local_base_dir + '*', self.remote_base_dir, '-r',
+                          '--include=*\\file2.txt', '--exclude=*.txt')
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, missing=2, missing_dir=3))
+    self._assert_remote_dir_contains(['folder1/file2.txt', 'folder1/file3.dat'])
+
+    # Upload all except *.dat with --delete, make sure file3.dat is kept.
+    utils.remove_test_file(self.local_base_dir + 'folder1\\file3.dat')
+    res = utils.run_rsync(self.local_base_dir + '*', self.remote_base_dir, '-r',
+                          '--delete', '--exclude=*.dat')
+    self._assert_rsync_success(res)
+    self.assertTrue(
+        utils.files_count_is(res, missing=3, matching=1, matching_dir=3))
+    self._assert_remote_dir_contains([
+        'file1.txt', 'folder1/file2.txt', 'folder1/file3.dat',
+        'folder1/folder2/file4.txt', 'folder3/file5.txt'
+    ])
+
+  def test_exclude_include_from(self):
+    """Verifies the --include-from and --exclude-from options."""
+
+    files = [
+        'file1.txt', 'folder1\\file2.txt', 'folder1\\file3.dat',
+        'folder1\\folder2\\file4.txt', 'folder3\\file5.txt'
+    ]
+
+    for file in files:
+      utils.create_test_file(self.local_base_dir + file, 987)
+
+    include_file = self.local_base_dir + 'include.txt'
+    with open(include_file, 'wt') as f:
+      f.writelines(['file1.txt\n', 'folder3\\file5.txt'])
+
+    exclude_file = self.local_base_dir + 'exclude.txt'
+    with open(exclude_file, 'wt') as f:
+      f.writelines(['*.txt'])
+
+    res = utils.run_rsync('-r', '--include-from', include_file,
+                          '--exclude-from', exclude_file,
+                          self.local_base_dir + '*', self.remote_base_dir)
+    self.assertTrue(utils.files_count_is(res, missing=3, missing_dir=3))
+    self._assert_remote_dir_contains(
+        ['file1.txt', 'folder1/file3.dat', 'folder3/file5.txt'])
+
+  def test_files_from(self):
+    """Verifies the --files-from option."""
+
+    files = [
+        'file1.txt', 'folder1\\file2.txt', 'folder1\\file3.dat',
+        'folder1\\folder2\\file4.txt', 'folder3\\file5.txt'
+    ]
+
+    for file in files:
+      utils.create_test_file(self.local_base_dir + file, 987)
+
+    sources_file = self.local_base_dir + 'sources.txt'
+    with open(sources_file, 'wt') as f:
+      f.writelines([
+          'file1.txt\n',
+          '\n',
+          '  folder1\\file3.dat  \n',
+          'folder1\\.\\folder2\\file4.txt\n',  # .\\ = rel path marker
+          '  folder3\\file5.txt\n',
+          '\n'
+      ])
+
+    res = utils.run_rsync('--files-from', sources_file, self.local_base_dir,
+                          self.remote_base_dir)
+    self.assertTrue(utils.files_count_is(res, missing=4))
+    self._assert_remote_dir_contains([
+        'file1.txt', 'folder1/file3.dat', 'folder2/file4.txt',
+        'folder3/file5.txt'
+    ])
+
+    # Upload again to check that nothing changes.
+    res = utils.run_rsync('--files-from', sources_file, self.local_base_dir,
+                          self.remote_base_dir)
+    self.assertTrue(utils.files_count_is(res, matching=4, extraneous_dir=3))
+
+  def test_checksum_file(self):
+    """Uploads and syncs a file with --checksum.
+
+       1) Uploads a file.
+       2) Uploads a file with --checksum option. As the file was not changed, it
+       is recognized as matched. The output should contain D100%.
+       3) Uploads the same file with --whole-file --checksum -v.
+       Checks the output contains C100%, not D100%.
+       4) Modifies the file without changing its content. The file is
+       synchronized, the output should contain D100%.
+
+    """
+
+    utils.create_test_file(self.local_data_path, 1024)
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir)
+
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir,
+                          '--checksum', '-v')
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, matching=1))
+    self.assertIn('D100%', str(res.stdout))
+    self.assertIn('will be synced due to -c/--checksum', str(res.stdout))
+
+    utils.create_test_file(self.local_data_path, 2534)
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir,
+                          '--checksum', '-v', '--whole-file')
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, changed=1))
+    self.assertIn('C100%', str(res.stdout))
+    self.assertIn('will be copied due to -c/--checksum and -W/--whole-file',
+                  str(res.stdout))
+    self.assertTrue(
+        utils.sha1_matches(self.local_data_path, self.remote_data_path))
+
+    utils.change_modified_time(self.local_data_path)
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir, '-c',
+                          '-v')
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, changed=1))
+    self.assertIn('D100%', str(res.stdout))
+
+  def test_sync_folder_when_remote_file_non_recursive(self):
+    """Non-recursively uploads a folder while there is a remote file with the same name."""
+
+    local_folder = self.local_base_dir + 'foldertocopy\\'
+    utils.create_test_directory(local_folder)
+    utils.get_ssh_command_output(
+        'mkdir -p %s && touch %s' %
+        (self.remote_base_dir, self.remote_base_dir + 'foldertocopy'))
+
+    res = utils.run_rsync(self.local_base_dir + 'foldertocopy',
+                          self.remote_base_dir)
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, extraneous=1))
+    self.assertFalse(
+        utils.does_directory_exist_remotely(self.remote_base_dir +
+                                            'foldertocopy'))
+    self.assertTrue(
+        utils.does_file_exist_remotely(self.remote_base_dir + 'foldertocopy'))
+
+  def test_sync_folder_when_remote_file_recursive_with_delete(self):
+    """Recursively uploads a folder while removing a remote file with the same name with --delete."""
+
+    local_folder = self.local_base_dir + 'foldertocopy\\'
+    utils.create_test_directory(local_folder)
+    utils.get_ssh_command_output(
+        'mkdir -p %s && touch %s' %
+        (self.remote_base_dir, self.remote_base_dir + 'foldertocopy'))
+
+    res = utils.run_rsync(self.local_base_dir + 'foldertocopy',
+                          self.remote_base_dir, '-r', '--delete')
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, extraneous=1, missing_dir=1))
+    self.assertTrue(
+        utils.does_directory_exist_remotely(self.remote_base_dir +
+                                            'foldertocopy'))
+    self.assertFalse(
+        utils.does_file_exist_remotely(self.remote_base_dir + 'foldertocopy'))
+    self.assertIn('1/1 file(s) and 0/0 folder(s) deleted', str(res.stdout))
+
+  def test_sync_file_when_remote_folder_recursive_with_delete(self):
+    """Recursively uploads a file while removing a remote folder with the same name with --delete."""
+    utils.create_test_file(self.local_data_path, 1024)
+    utils.get_ssh_command_output('mkdir -p %s' % self.remote_data_path)
+
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir,
+                          '--delete', '-r')
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, missing=1, extraneous_dir=1))
+    self.assertTrue(
+        utils.sha1_matches(self.local_data_path, self.remote_data_path))
+    self.assertFalse(utils.does_directory_exist_remotely(self.remote_data_path))
+    self.assertIn('0/0 file(s) and 1/1 folder(s) deleted', str(res.stdout))
+
+  def test_sync_file_when_remote_folder_empty_non_recursive(self):
+    """Non-recursively uploads a file while there is an empty remote folder with the same name."""
+    self._do_test_sync_file_when_remote_folder_empty(recursive=False)
+
+  def test_sync_file_when_remote_folder_empty_recursive(self):
+    """Recursively uploads a file while there is an empty remote folder with the same name."""
+    self._do_test_sync_file_when_remote_folder_empty(recursive=True)
+
+  def _do_test_sync_file_when_remote_folder_empty(self, recursive):
+    """Uploads a file while there is an empty remote folder with the same name.
+
+    Args:
+        recursive (bool): Whether to append '-r' or not.
+    """
+    flag = '-r' if recursive else None
+    utils.create_test_file(self.local_data_path, 1024)
+    utils.get_ssh_command_output('mkdir -p %s' % self.remote_data_path)
+
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir, flag)
+    self._assert_rsync_success(res)
+    self.assertTrue(utils.files_count_is(res, missing=1, extraneous_dir=1))
+    self.assertTrue(
+        utils.sha1_matches(self.local_data_path, self.remote_data_path))
+    self.assertFalse(utils.does_directory_exist_remotely(self.remote_data_path))
+    self.assertNotIn('0/0 file(s) and 1/1 folder(s) deleted', str(res.stdout))
+
+  def test_sync_file_when_remote_folder_non_empty_non_recursive(self):
+    """Non-recursively uploads a file while there is a non-empty remote folder with the same name."""
+    self._do_test_sync_file_when_remote_folder_non_empty(recursive=False)
+
+  def test_sync_file_when_remote_folder_non_empty_recursive(self):
+    """Recursively uploads a file while there is a non-empty remote folder with the same name."""
+    self._do_test_sync_file_when_remote_folder_non_empty(recursive=True)
+
+  def _do_test_sync_file_when_remote_folder_non_empty(self, recursive):
+    """Uploads a file while there is a non-empty remote folder with the same name.
+
+    Args:
+        recursive (bool): Whether to append '-r' or not.
+    """
+    flag = '-r' if recursive else None
+    utils.create_test_file(self.local_data_path, 1024)
+    utils.get_ssh_command_output('mkdir -p %s' % self.remote_data_path)
+    utils.get_ssh_command_output(
+        'mkdir -p %s && touch %s' %
+        (self.remote_base_dir, self.remote_data_path + '/file1.txt'))
+
+    res = utils.run_rsync(self.local_data_path, self.remote_base_dir, flag)
+    self.assertIn('remove() failed: Directory not empty.', str(res.stderr))
+    if recursive:
+      self.assertTrue(
+          utils.files_count_is(res, missing=1, extraneous=1, extraneous_dir=1))
+    else:
+      self.assertTrue(utils.files_count_is(res, missing=1, extraneous_dir=1))
+    self.assertTrue(utils.does_directory_exist_remotely(self.remote_data_path))
+    self.assertTrue(
+        utils.does_file_exist_remotely(self.remote_data_path + '/file1.txt'))
+    self.assertFalse(utils.does_file_exist_remotely(self.remote_data_path))
+
+  def test_upload_from_dot(self):
+    """Uploads files from the current directory ('.')."""
+    utils.create_test_file(self.local_base_dir + 'file1.txt', 1024)
+    utils.create_test_file(self.local_base_dir + 'dir\\file2.txt', 1024)
+
+    prev_cwd = os.getcwd()
+    os.chdir(self.local_base_dir)
+    try:
+      # Uploading recursivly should pick up all files and dirs.
+      res = utils.run_rsync('.', self.remote_base_dir, '-r')
+      self.assertTrue(utils.files_count_is(res, missing=2, missing_dir=1))
+      self._assert_remote_dir_contains(['file1.txt', 'dir/file2.txt'])
+
+      # Uploading again should not change anything.
+      res = utils.run_rsync('.', self.remote_base_dir, '-r')
+      self.assertTrue(utils.files_count_is(res, matching=2, matching_dir=1))
+
+      # Verify that non-recursive uploads do nothing.
+      res = utils.run_rsync('.', self.remote_base_dir)
+      self.assertTrue(utils.files_count_is(res, extraneous=1, extraneous_dir=1))
+    finally:
+      os.chdir(prev_cwd)
+
+  def test_upload_from_dotdot(self):
+    """Uploads files from the parent directory ('..')."""
+    utils.create_test_file(self.local_base_dir + 'file1.txt', 1024)
+    utils.create_test_file(self.local_base_dir + 'dir\\file2.txt', 1024)
+
+    prev_cwd = os.getcwd()
+    os.chdir(self.local_base_dir + 'dir')
+    try:
+      # Uploading recursivly should pick up all files and dirs.
+      res = utils.run_rsync('..', self.remote_base_dir, '-r')
+      self.assertTrue(utils.files_count_is(res, missing=2, missing_dir=1))
+      self._assert_remote_dir_contains(['file1.txt', 'dir/file2.txt'])
+
+      # Uploading again should not change anything.
+      res = utils.run_rsync('..', self.remote_base_dir, '-r')
+      self.assertTrue(utils.files_count_is(res, matching=2, matching_dir=1))
+
+      # Verify that non-recursive uploads do nothing.
+      res = utils.run_rsync('..', self.remote_base_dir)
+      self.assertTrue(utils.files_count_is(res, extraneous=1, extraneous_dir=1))
+    finally:
+      os.chdir(prev_cwd)
+
+  def test_existing(self):
+    """Runs rsync with --existing for a non-trivial directory.
+
+       1) Uploads a source directory with -r.
+       |-- rootdir
+       |   |-- dir1
+       |      |-- emptydir2
+       |      |-- file1_1.txt
+       |      |-- file1_2.txt -> rename to file1_3.txt (step 2)
+       |      |-- (step2) emptydir3
+       |   |-- dir2
+       |      |-- file2_1.txt
+       |   |-- emptydir1      -> rename emptydir4 (step 2)
+       |   |-- file0.txt      -> change (step 2)
+       2) Add new files/folders, remove and change some files/folders.
+       3) Uploads the same source directory with --existing option and with -r.
+       Only files existing on the server are changed, nothing is removed.
+       4) Uploads the same source directory with --existing --delete -r.
+       Files non-existing on the server are deleted.
+    """
+    local_root_path = self.local_base_dir + 'rootdir'
+    remote_root_path = self.remote_base_dir + 'rootdir/'
+
+    files = [
+        '\\dir1\\file1_1.txt', '\\dir1\\file1_2.txt', '\\dir2\\file2_1.txt',
+        '\\file0.txt'
+    ]
+    for file in files:
+      utils.create_test_file(local_root_path + file, 1024)
+    dirs = ['\\dir1\\emptydir2\\', '\\emptydir1\\']
+    for directory in dirs:
+      utils.create_test_directory(local_root_path + directory)
+    res = utils.run_rsync(local_root_path, self.remote_base_dir, '-r')
+    self._assert_rsync_success(res)
+
+    utils.remove_test_file(local_root_path + '\\dir1\\file1_2.txt')
+    utils.create_test_file(local_root_path + '\\dir1\\file1_3.txt', 1024)
+    utils.create_test_directory(local_root_path + '\\dir1\\emptydir3\\')
+    utils.remove_test_directory(local_root_path + '\\emptydir1\\')
+    utils.create_test_directory(local_root_path + '\\emptydir4\\')
+    utils.create_test_file(local_root_path + '\\file0.txt', 2034)
+
+    res = utils.run_rsync(local_root_path, self.remote_base_dir, '-r',
+                          '--existing')
+    self._assert_rsync_success(res)
+    self.assertTrue(
+        utils.files_count_is(
+            res,
+            missing=1,
+            missing_dir=2,
+            matching=2,
+            matching_dir=4,
+            changed=1,
+            extraneous=1,
+            extraneous_dir=1))
+    self.assertTrue(
+        utils.does_directory_exist_remotely(remote_root_path + 'emptydir1'))
+    self.assertFalse(
+        utils.does_directory_exist_remotely(remote_root_path + 'emptydir4'))
+    self.assertFalse(
+        utils.does_directory_exist_remotely(remote_root_path +
+                                            'dir1/emptydir3'))
+    self.assertTrue(
+        utils.does_file_exist_remotely(remote_root_path + 'dir1/file1_2.txt'))
+    self.assertFalse(
+        utils.does_file_exist_remotely(remote_root_path + 'dir1/file1_3.txt'))
+
+    res = utils.run_rsync(local_root_path, self.remote_base_dir, '-r',
+                          '--existing', '--delete')
+    self._assert_rsync_success(res)
+    self.assertTrue(
+        utils.files_count_is(
+            res,
+            missing=1,
+            missing_dir=2,
+            matching=3,
+            matching_dir=4,
+            extraneous=1,
+            extraneous_dir=1))
+    self.assertIn('1/1 file(s) and 1/1 folder(s) deleted', res.stdout)
+    self.assertFalse(
+        utils.does_directory_exist_remotely(remote_root_path + 'emptydir1'))
+    self.assertFalse(
+        utils.does_file_exist_remotely(remote_root_path + 'dir2/file1_2.txt'))
+
+  def test_copy_dest(self):
+    r"""Runs rsync with --copy-dest option.
+
+       Copies testdata.dat to 
+       Copies the "cdc_rsync_e2e_test" package locally and syncs it with
+       --copy-dest. Verifies that the files are actually sync'ed (D), not
+       copied (C).
+
+    Raises:
+        Exception: On timeout waiting for mount to appear (after 20 seconds)
+    """
+
+    copy_dest_dir = self.remote_base_dir + 'copy_dest_dir'
+
+    utils.create_test_file(self.local_data_path, 1024)
+    res = utils.run_rsync(self.local_data_path, copy_dest_dir)
+    self._assert_rsync_success(res)
+
+    # Upload package using --package.
+    res = utils.run_rsync('--copy-dest', copy_dest_dir, self.local_data_path,
+                          self.remote_base_dir, '-v')
+    self._assert_rsync_success(res)
+    self.assertIn('D100%', res.stdout)
+    self.assertNotIn('C100%', res.stdout)
+
+  def test_upload_executables(self):
+    """Uploads executable files and checks that they have the x bit set."""
+
+    # Use the cdc rsync binaries as test executables.
+    local_exe_path = utils.CDC_RSYNC_PATH
+    local_elf_path = os.path.join(
+        os.path.dirname(local_exe_path), 'cdc_rsync_server')
+
+    remote_exe_path = self.remote_base_dir + os.path.basename(local_exe_path)
+    remote_elf_path = self.remote_base_dir + os.path.basename(local_elf_path)
+
+    # Copy the files to the gamelet.
+    res = utils.run_rsync(local_exe_path, local_elf_path, self.remote_base_dir)
+    self._assert_rsync_success(res)
+
+    # Check that both files have the executable bit set.
+    stats = utils.get_ssh_command_output('stat -c "%%a" %s %s' %
+                                         (remote_exe_path, remote_elf_path))
+    self.assertEqual(stats.count('755'), 2, stats)
+
+    # Remove executable bits.
+    utils.get_ssh_command_output('chmod -x %s %s' %
+                                 (remote_exe_path, remote_elf_path))
+
+    # Sync again, using -c to force a sync.
+    res = utils.run_rsync('-c', local_exe_path, local_elf_path,
+                          self.remote_base_dir)
+    self._assert_rsync_success(res)
+
+    # Validate that the executable bits were restored.
+    stats = utils.get_ssh_command_output('stat -c "%%a" %s %s' %
+                                         (remote_exe_path, remote_elf_path))
+    self.assertEqual(stats.count('755'), 2, stats)
+
+  def _run(self, args):
+    logging.debug('Running %s', ' '.join(args))
+    res = subprocess.run(args, capture_output=True)
+    self.assertEqual(res.returncode, 0, 'Command failed: ' + str(res))
+    res.stdout = res.stdout.decode('ascii')
+    logging.debug('\r\n%s', res.stdout)
+    return res
+
+
+if __name__ == '__main__':
+  test_base.test_base.main()

integration_tests/cdc_stream/__init__.py

@@ -0,0 +1 @@
+

integration_tests/cdc_stream/all_tests.py

@@ -0,0 +1,42 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+import unittest
+
+from integration_tests.cdc_stream import cache_test
+from integration_tests.cdc_stream import consistency_test
+from integration_tests.cdc_stream import directory_test
+from integration_tests.cdc_stream import general_test
+from integration_tests.framework import test_base
+
+
+# pylint: disable=g-doc-args,g-doc-return-or-yield
+def load_tests(loader, unused_tests, unused_pattern):
+  """Customizes the list of test cases to run.
+
+  See the Python documentation for details:
+  https://docs.python.org/3/library/unittest.html#load-tests-protocol
+  """
+  suite = unittest.TestSuite()
+  suite.addTests(loader.loadTestsFromModule(cache_test))
+  suite.addTests(loader.loadTestsFromModule(consistency_test))
+  suite.addTests(loader.loadTestsFromModule(directory_test))
+  suite.addTests(loader.loadTestsFromModule(general_test))
+
+  return suite
+
+
+if __name__ == '__main__':
+  test_base.main()

integration_tests/cdc_stream/cache_test.py

@@ -0,0 +1,116 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""cdc_stream cache test."""
+
+import logging
+import os
+import posixpath
+import time
+
+from integration_tests.framework import utils
+from integration_tests.cdc_stream import test_base
+
+
+class CacheTest(test_base.CdcStreamTest):
+  """cdc_stream test class for cache."""
+
+  cache_capacity = 10 * 1024 * 1024  # 10MB
+  cleanup_timeout_sec = 2
+  access_idle_timeout_sec = 2
+  cleanup_time = 5  # estimated cleanup time
+  # Returns a list of files and directories with mtimes in the cache.
+  # 2021-10-19 01:09:30.070055513 -0700 /var/cache/asset_streaming
+  cache_cmd = ('find %s -exec stat --format \"%%y %%n\" '
+               '\"{}\" \\;') % (
+                   test_base.CdcStreamTest.cache_dir)
+
+  @classmethod
+  def setUpClass(cls):
+    super().setUpClass()
+    logging.debug('CacheTest -> setUpClass')
+
+    config_json = ('{\"cache-capacity\":\"%s\",\"cleanup-timeout\":%i,'
+                   '\"access-idle-timeout\":%i}') % (
+                       cls.cache_capacity, cls.cleanup_timeout_sec,
+                       cls.access_idle_timeout_sec)
+    cls._start_service(config_json)
+
+  def test_cache_reused(self):
+    """Cache survives remount and is reused."""
+    filename = '1.txt'
+    utils.create_test_file(
+        os.path.join(self.local_base_dir, filename), 7 * 1024 * 1024)
+    self._start()
+    self._test_dir_content(files=[filename], dirs=[])
+    # Read the file => fill the cache.file_transfer
+    utils.get_ssh_command_output('cat %s > /dev/null' %
+                                 posixpath.join(self.remote_base_dir, filename))
+    cache_size = self._get_cache_size_in_bytes()
+    cache_files = utils.get_ssh_command_output(self.cache_cmd)
+
+    self._stop()
+    self._assert_cdc_fuse_mounted(success=False)
+    self._assert_cache()
+
+    self._start()
+    self._assert_cdc_fuse_mounted()
+    self._test_dir_content(files=[filename], dirs=[])
+    utils.get_ssh_command_output('cat %s > /dev/null' %
+                                 posixpath.join(self.remote_base_dir, filename))
+    # The same manifest should be re-used. No change in the cache is expected.
+    self.assertEqual(self._get_cache_size_in_bytes(), cache_size)
+    # The mtimes of the files should have changed after each Get() operation.
+    self.assertNotEqual(
+        utils.get_ssh_command_output(self.cache_cmd), cache_files)
+
+  def test_set_cache_capacity_old_chunks_removed(self):
+    # Command to return the oldest mtime in the cache directory.
+    ts_cmd = ('find %s -type f -printf \"%%T@\\n\" '
+              '| sort -n | head -n 1') % (
+                  self.cache_dir)
+    # Stream a file.
+    filename = '1.txt'
+    utils.create_test_file(
+        os.path.join(self.local_base_dir, filename), 11 * 1024 * 1024)
+    self._start()
+    self._test_dir_content(files=[filename], dirs=[])
+    utils.get_ssh_command_output('cat %s > /dev/null' %
+                                 posixpath.join(self.remote_base_dir, filename))
+    # Extract the oldest file.
+    oldest_ts = utils.get_ssh_command_output(ts_cmd)
+    original = utils.get_ssh_command_output(self.ls_cmd)
+
+    # Add and read one more file.
+    filename2 = '2.txt'
+    utils.create_test_file(
+        os.path.join(self.local_base_dir, filename2), 11 * 1024 * 1024)
+    self.assertTrue(self._wait_until_remote_dir_changed(original))
+    utils.get_ssh_command_output(
+        'cat %s > /dev/null' % posixpath.join(self.remote_base_dir, filename2))
+
+    # Wait some time till the cache is cleaned up.
+    wait_sec = self.cleanup_timeout_sec + self.access_idle_timeout_sec + self.cleanup_time
+    logging.info(f'Waiting {wait_sec} seconds until the cache is cleaned up')
+    time.sleep(wait_sec)
+    self.assertLessEqual(self._get_cache_size_in_bytes(), self.cache_capacity)
+    new_oldest_ts = utils.get_ssh_command_output(ts_cmd)
+
+    self.assertGreater(new_oldest_ts, oldest_ts)
+    self._test_dir_content(files=[filename, filename2], dirs=[])
+
+
+if __name__ == '__main__':
+  test_base.test_base.main()

integration_tests/cdc_stream/consistency_test.py

@@ -0,0 +1,496 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""cdc_stream consistency test."""
+
+import glob
+import logging
+import os
+import queue
+import re
+import string
+import time
+
+from integration_tests.framework import utils
+from integration_tests.cdc_stream import test_base
+
+
+class ConsistencyTest(test_base.CdcStreamTest):
+  """cdc_stream test class for CDC FUSE consistency."""
+
+  @classmethod
+  def setUpClass(cls):
+    super().setUpClass()
+    logging.debug('ConsistencyTest -> setUpClass')
+
+    config_json = '{\"debug\":1, \"check\":1, \"verbosity\":3}'
+    cls._start_service(config_json)
+
+  def _wait_until_remote_dir_matches(self, files, dirs, counter=20):
+    """Wait until the directory content has changed.
+
+    Args:
+        files (list of strings): List of relative file paths.
+        dirs (list of strings): List of relative directory paths.
+        counter (int): The number of retries.
+    Returns:
+        bool: Whether the content of the remote directory matches the local one.
+    """
+    dirs = [directory.replace('\\', '/').rstrip('/') for directory in dirs]
+    files = [file.replace('\\', '/') for file in files]
+    sha1_local = self.sha1sum_local_batch(files)
+    for _ in range(counter):
+      utils.get_ssh_command_output('ls -al %s' % self.remote_base_dir)
+      found = utils.get_sorted_files(self.remote_base_dir, '"*"')
+      expected = sorted(['./' + f for f in files + dirs])
+      if found == expected:
+        if not files:
+          return True
+        sha1_remote = self.sha1sum_remote_batch()
+        if sha1_local == sha1_remote:
+          return True
+      time.sleep(1)
+    return False
+
+  def _generate_random_name(self, depth):
+    """Generate a random name for a file/directory name.
+
+    Args:
+        depth (int): Depth of the directory structure.
+    Returns:
+        string: Random string.
+    """
+    max_path_len = 260  # Windows limitation for a path length.
+    # 4 symbols are reserved for file extension .txt.
+    max_path_len_no_root = max_path_len - len(self.local_base_dir) - 4
+    # As a Windows path is limited to 260 symbols it is necesary to consider the
+    # depth of the full path.
+    # +1 is for the last file, -2: for a path separator + down rounding.
+    max_file_name_len = int(max_path_len_no_root / (depth + 1) - 2)
+    length = utils.RANDOM.randint(1, max_file_name_len)
+
+    # Consider only upper case and digits, as 1.txt and 1.TXT result in 1 file
+    # on Windows.
+    name = ''.join(
+        utils.RANDOM.choice(string.ascii_uppercase + string.digits)
+        for i in range(length))
+    return name
+
+  def _generate_dir_list(self, depth, num_leaf_dirs):
+    """Generate a list of directories.
+
+    Args:
+        depth (int): Depth of the directory structure.
+        num_leaf_dirs (int): How many leaf directories should be generated.
+    Returns:
+        queue of list of strings: Relative paths of directories to be created.
+    """
+    dirs = queue.Queue(maxsize=0)
+    if depth == 0:
+      return dirs
+    top_num = utils.RANDOM.randint(1, 1 + num_leaf_dirs)
+    for _ in range(top_num):
+      directory = self._generate_random_name(depth)
+      dirs.put([directory])
+    new_dirs = queue.Queue(maxsize=0)
+    for _ in range(depth - 1):
+      while not dirs.empty():
+        curr_set = dirs.get()
+        missing_dirs = num_leaf_dirs - new_dirs.qsize() - dirs.qsize()
+        if missing_dirs > 0:
+          num_dir = utils.RANDOM.randint(1, missing_dirs)
+          for _ in range(num_dir):
+            name = self._generate_random_name(depth)
+            path = curr_set.copy()
+            path.append(name)
+            new_dirs.put(path)
+        else:
+          new_dirs.put(curr_set)
+      new_dirs, dirs = dirs, new_dirs
+    for _ in range(num_leaf_dirs - dirs.qsize()):
+      dirs.put([self._generate_random_name(depth)])
+    return dirs
+
+  def _generate_files(self, dirs, size, depth, min_file_num, max_file_num):
+    """Create files in given directories.
+
+    Args:
+        dirs (set of strings): Relative paths for directories.
+        size (int): Total size of files to be created.
+        depth (int): Depth of the directory hierarchy.
+        min_file_num (int): Minimal number of files, which can be created in a
+        directory.
+        max_file_num (int): Maximal number of files, which can be created in a
+        directory.
+    Returns:
+        list of strings: Set of relative paths of created files.
+    """
+    files = set()
+    for directory in dirs:
+      number_of_files = utils.RANDOM.randint(min_file_num, max_file_num)
+      for _ in range(number_of_files):
+        # Add a file extension not to compare if a similar directory exists.
+        file_name = self._generate_random_name(depth=depth) + '.txt'
+        if file_name not in files:
+          file_path = os.path.join(directory, file_name)
+          files.add(file_path)
+          # Do not create files larger than 1 GB.
+          file_size = utils.RANDOM.randint(0, min(1024 * 1024 * 1024, size))
+          size -= file_size
+          utils.create_test_file(
+              os.path.join(self.local_base_dir, file_path), file_size)
+        if size <= 0:
+          return files
+    # Create files for the remaining size.
+    if size > 0:
+      number_of_files = utils.RANDOM.randint(min_file_num, max_file_num)
+      for _ in range(number_of_files):
+        file_name = self._generate_random_name(depth)
+        files.add(file_name)
+        utils.create_test_file(
+            os.path.join(self.local_base_dir, file_name),
+            int(size / number_of_files))
+    return files
+
+  def _generate_dir_paths(self, dirs):
+    """Create directories.
+
+    Args:
+        dirs (queue of lists of strings): Relative paths for directories.
+    Returns:
+        set of strings: Relative paths for created directories.
+    """
+    paths = set()
+    for dir_set in dirs.queue:
+      curr_path = ''
+      for name in dir_set:
+        # It is necessary to add the last separator.
+        # Otherwise, the leaf directory will not be created.
+        curr_path = os.path.join(curr_path, name) + '\\'
+        paths.add(curr_path)
+      utils.create_test_directory(os.path.join(self.local_base_dir, curr_path))
+    return paths
+
+  def _generate_streamed_dir(self, size, depth, min_file_num=1, max_file_num=1):
+    """Generate a streamed directory.
+
+    Args:
+        size (int): Total size of files to create in the directory.
+        depth (int): Depth of the directory hierarchy.
+        min_file_num (int): Minimal number of files, which can be created in a
+        single directory.
+        max_file_num (int): Maximal number of files, which can be created in a
+        single directory.
+    Returns:
+        two sets of strings: Relative paths for created files and directories.
+    """
+    num_leaf_dirs = 0
+    if depth > 0:
+      num_leaf_dirs = utils.RANDOM.randint(0, 100)
+    logging.debug(('CdcStreamConsistencyTest -> _generate_streamed_dir'
+                   ' of depth %i and number of leaf directories %i'), depth,
+                  num_leaf_dirs)
+    dirs = self._generate_dir_paths(
+        self._generate_dir_list(depth, num_leaf_dirs))
+    files = self._generate_files(
+        dirs=dirs,
+        size=size,
+        depth=depth,
+        min_file_num=min_file_num,
+        max_file_num=max_file_num)
+    logging.debug(
+        ('CdcStreamConsistencyTest -> _generate_streamed_dir: generated'
+         ' %i files, %i directories, depth %i'), len(files), len(dirs), depth)
+    return files, dirs
+
+  def _recreate_data(self, files, dirs):
+    """Recreate test data and check that it can be read on a gamelet.
+
+    Args:
+        files (list of strings): List of relative file paths.
+        dirs (list of strings): List of relative directory paths.
+    """
+    logging.debug('CdcStreamConsistencyTest -> _recreate_data')
+    self._create_test_data(files=files, dirs=dirs)
+    self.assertTrue(self._wait_until_remote_dir_matches(files=files, dirs=dirs))
+    self._assert_cdc_fuse_mounted()
+
+  def _assert_inode_consistency_line(self, line, updated_proto=0, updated=0):
+    """Assert if the numbers of inodes specific states are correct.
+
+    Args:
+        line (string): Statement like Initialized=X, updated_proto=X, updated=X,
+        invalid=X.
+        updated_proto(int): Expected number of inodes whose protos were updated.
+        updated(int): Expected number of inodes whose contents were updated.
+    """
+    self.assertIn(('Initialized=0, updated_proto=%i,'
+                   ' updated=%i, invalid=0') % (updated_proto, updated), line)
+
+  def _assert_consistency_line(self, line):
+    """Assert if there are no invalid and initialized nodes.
+
+    Args:
+        line (string): Statement like Initialized=X, updated_proto=X, updated=X,
+        invalid=X.
+    """
+    self.assertIn(('Initialized=0,'), line)
+    self.assertIn(('invalid=0'), line)
+
+  def _assert_inode_consistency(self, update_map, log_file):
+    """Assert that the amount of updated inodes is correct.
+
+    Args:
+        update_map (dict): Mapping of inodes' types to their amount.
+        log_file (string): Absolute path to the log file.
+    """
+    with open(log_file) as file:
+      success_count = 0
+      for line in file:
+        if 'Initialized=' in line:
+          self._assert_inode_consistency_line(
+              line,
+              updated_proto=update_map[success_count][0],
+              updated=update_map[success_count][1])
+        if 'FUSE consistency check succeeded' in line:
+          success_count += 1
+        self.assertNotIn('FUSE consistency check:', line)
+
+  def _assert_consistency(self, log_file):
+    """Assert that there is no error consistency messages in the log.
+
+    Args:
+        log_file (string): Absolute path to the log file.
+    """
+
+    def assert_initialized_line(line):
+      self.assertNotIn('FUSE consistency check:', line)
+      if 'Initialized=' in line:
+        self._assert_consistency_line(line)
+
+    joined_line = ''
+    with open(log_file) as file:
+      for line in file:
+        # Matches log lines with a log level
+        # 2022-01-23 05:18:12.401 DEBUG   process_win.cc(546): LogOutput():
+        #            cdc_fuse_fs_stdout: DEBUG   cdc_fuse_fs.cc(1165):
+        #            CheckFUSEConsistency(): Initialized=
+        # Matches log lines without log level
+        # 2022-01-23 05:18:12.401 INFO    process_win.cc(536): LogOutput():
+        #            cdc_fuse_fs_stdout: 0, updated_proto=437, updated=563,
+        #            invalid
+        match = re.match(
+            r'[0-9]{4}-[0-9]{2}-[0-9]{2}\s+'
+            r'[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]+\s+'
+            r'[A-Z]+\s+'
+            r'(?:[._a-zA-Z0-9()]+:\s+){2}'
+            r'cdc_fuse_fs_stdout:\s+'
+            r'((?:DEBUG|INFO|WARNING|ERROR)\s+)?(.*)', line)
+        if match is None:
+          continue
+        log_level = match.group(1)
+        log_msg = match.group(2)
+        # A client side log level marks the beginning of a new log line
+        if log_level:
+          assert_initialized_line(joined_line)
+          joined_line = log_msg.rstrip('\r\n')
+        else:
+          joined_line += log_msg.rstrip('\r\n')
+      assert_initialized_line(joined_line)
+
+  def _get_log_file(self):
+    """Find the newest log file for asset streaming 3.0.
+
+    Returns:
+        string: Absolute file path for the log file.
+    """
+    log_dir = os.path.join(os.environ['APPDATA'], 'cdc-file-transfer', 'logs')
+    log_files = glob.glob(os.path.join(log_dir, 'cdc_stream*.log'))
+    latest_file = max(log_files, key=os.path.getctime)
+    logging.debug(('CdcStreamConsistencyTest -> _get_log_file:'
+                   ' the current log file is %s'), latest_file)
+    return latest_file
+
+  def _mount_with_data(self, files, dirs):
+    """Mount a directory, check the content.
+
+    Args:
+        files (list of strings): List of relative file paths.
+        dirs (list of strings): List of relative directory paths.
+    """
+    self._start()
+    self._test_random_dir_content(files=files, dirs=dirs)
+    self._assert_cache()
+    self._assert_cdc_fuse_mounted()
+
+  def test_consistency_fixed_data(self):
+    """Execute consistency check on a small directory.
+
+    Streamed directory layout:
+       |-- rootdir
+       |   |-- dir1
+       |      |-- emptydir2
+       |      |-- file1_1.txt
+       |      |-- file1_2.txt
+       |   |-- dir2
+       |      |-- file2_1.txt
+       |   |-- emptydir1
+       |   |-- file0.txt
+    """
+    files = [
+        'dir1\\file1_1.txt', 'dir1\\file1_2.txt', 'dir2\\file2_1.txt',
+        'file0.txt'
+    ]
+    dirs = ['dir1\\emptydir2\\', 'emptydir1\\', 'dir1\\', 'dir2\\']
+    self._create_test_data(files=files, dirs=dirs)
+    self._mount_with_data(files, dirs)
+
+    # Recreate test data.
+    log_file = self._get_log_file()
+    self._recreate_data(files=files, dirs=dirs)
+
+    # In total there should be 2 checks:
+    # - For initial manifest when no data was read,
+    # - Two additional caused by the directory change.
+    self._assert_inode_consistency([[0, 0], [2, 6], [2, 6]], log_file)
+
+  def _test_consistency_random(self, files, dirs):
+    """Mount and check consistency, recreate the data and re-check consistency.
+
+    Args:
+        files (list of strings): List of relative file paths.
+        dirs (list of strings): List of relative directory paths.
+    """
+    self._mount_with_data(files=files, dirs=dirs)
+
+    # Recreate test data.
+    log_file = self._get_log_file()
+    self._recreate_data(files=files, dirs=dirs)
+    self._assert_consistency(log_file)
+
+  def sha1sum_local_batch(self, files):
+    """Calculate sha1sum of files in the streamed directory on the workstation.
+
+    Args:
+        files (list of strings): List of relative file paths to check.
+    Returns:
+        string: Concatenated sha1 hashes with relative posix file names.
+    """
+    files.sort()
+    sha1sum_local = ''
+    for file in files:
+      full_path = os.path.join(self.local_base_dir, file.replace('/', '\\'))
+      sha1sum_local += utils.sha1sum_local(full_path) + file
+    return sha1sum_local
+
+  def sha1sum_remote_batch(self):
+    """Calculate sha1sum of files in the streamed directory on the gamelet.
+
+    Returns:
+        string: Concatenated sha1 hashes with relative posix file names.
+    """
+    sha1sum_remote = utils.get_ssh_command_output(
+        'find %s -type f -exec sha1sum \'{}\' + | sort -k 2' %
+        self.remote_base_dir)
+    # Example:
+    # original: d664613df491478095fa201fac435112
+    # /tmp/_cdc_stream_test/E8KPXXS1MYKLIQGAI4I6/M0
+    # final: d664613df491478095fa201fac435112E8KPXXS1MYKLIQGAI4I6/M0
+    sha1sum_remote = sha1sum_remote.replace(self.remote_base_dir, '').replace(
+        ' ', '').replace('\r', '').replace('\n', '').replace('\t', '')
+    return sha1sum_remote
+
+  def _test_random_dir_content(self, files, dirs):
+    """Check the streamed randomly generated directory's content on gamelet.
+
+    Args:
+        files (list of strings): List of relative file paths to check.
+        dirs (list of strings): List of relative dir paths to check.
+    """
+    dirs = [directory.replace('\\', '/').rstrip('/') for directory in dirs]
+    files = [file.replace('\\', '/') for file in files]
+
+    utils.get_ssh_command_output('ls -al %s' % self.remote_base_dir)
+    self._assert_remote_dir_matches(files + dirs)
+    if not files:
+      return
+
+    sha1_local = self.sha1sum_local_batch(files)
+    sha1_remote = self.sha1sum_remote_batch()
+    self.assertEqual(sha1_local, sha1_remote)
+
+  def test_consistency_random_100MB_10files_per_dir(self):
+    """Consistency check: modification, 100MB, 10 files/directory."""
+    files, dirs = self._generate_streamed_dir(
+        size=100 * 1024 * 1024,
+        depth=utils.RANDOM.randint(0, 10),
+        max_file_num=10)
+    self._test_consistency_random(files=files, dirs=dirs)
+
+  def test_consistency_random_100MB_exact_1000files_no_dir(self):
+    """Consistency check: modification, 100MB, 1000 files/root."""
+    files, dirs = self._generate_streamed_dir(
+        size=100 * 1024 * 1024, depth=0, min_file_num=1000, max_file_num=1000)
+    self._test_consistency_random(files=files, dirs=dirs)
+
+  def test_consistency_random_100MB_1000files_per_dir_one_level(self):
+    """Consistency check: modification, 100MB, max. 1000 files/dir, depth 1."""
+    files, dirs = self._generate_streamed_dir(
+        size=100 * 1024 * 1024, depth=1, max_file_num=1000)
+    self._test_consistency_random(files=files, dirs=dirs)
+
+  def _test_consistency_random_delete(self, files, dirs):
+    """Remove and recreate a streamed directory.
+
+    Args:
+        files (list of strings): List of relative file paths.
+        dirs (list of strings): List of relative directory paths.
+    """
+    self._mount_with_data(files, dirs)
+
+    # Remove directory on workstation => empty directory on gamelet.
+    utils.get_ssh_command_output(self.ls_cmd)
+    utils.remove_test_directory(self.local_base_dir)
+
+    self.assertTrue(self._wait_until_remote_dir_matches(files=[], dirs=[]))
+    self._assert_cdc_fuse_mounted()
+
+    log_file = self._get_log_file()
+    self._recreate_data(files=files, dirs=dirs)
+    self._assert_consistency(log_file)
+
+  def test_consistency_random_delete_100MB_10files_per_dir(self):
+    """Consistency check: removal, 100MB, 10 files/directory."""
+    files, dirs = self._generate_streamed_dir(
+        size=100 * 1024 * 1024,
+        depth=utils.RANDOM.randint(0, 10),
+        max_file_num=10)
+    self._test_consistency_random_delete(files=files, dirs=dirs)
+
+  def test_consistency_random_delete_100MB_exact_1000files_no_dir(self):
+    """Consistency check: removal, 100MB, 1000 files/root."""
+    files, dirs = self._generate_streamed_dir(
+        size=100 * 1024 * 1024, depth=0, min_file_num=1000, max_file_num=1000)
+    self._test_consistency_random_delete(files=files, dirs=dirs)
+
+  def test_consistency_random_delete_100MB_1000files_per_dir_one_level(self):
+    """Consistency check: removal, 100MB, max. 1000 files/directory, depth 1."""
+    files, dirs = self._generate_streamed_dir(
+        size=100 * 1024 * 1024, depth=1, max_file_num=1000)
+    self._test_consistency_random_delete(files=files, dirs=dirs)
+
+
+if __name__ == '__main__':
+  test_base.test_base.main()

integration_tests/cdc_stream/directory_test.py

@@ -0,0 +1,124 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""cdc_stream directory Test."""
+
+import os
+
+from integration_tests.framework import utils
+from integration_tests.cdc_stream import test_base
+
+
+class DirectoryTest(test_base.CdcStreamTest):
+  """cdc_stream test class for modifications of streamed directory."""
+
+  def _assert_mount_fails(self, directory):
+    """Check that mounting a directory fails.
+
+    Args:
+        directory (string): name of a file/directory to be streamed.
+    """
+    with self.assertRaises(Exception):
+      self._start(directory)
+
+  def test_recreate_streamed_dir(self):
+    """Survive recreation of a streamed directory.
+
+    Streamed directory layout:
+       |-- rootdir
+       |   |-- dir1
+       |      |-- emptydir2
+       |      |-- file1_1.txt
+       |      |-- file1_2.txt
+       |   |-- dir2
+       |      |-- file2_1.txt
+       |   |-- emptydir1
+       |   |-- file0.txt
+    """
+    files = [
+        'dir1\\file1_1.txt', 'dir1\\file1_2.txt', 'dir2\\file2_1.txt',
+        'file0.txt'
+    ]
+    dirs = ['dir1\\emptydir2\\', 'emptydir1\\', 'dir1\\', 'dir2\\']
+    self._create_test_data(files, dirs)
+    self._start()
+    self._test_dir_content(files=files, dirs=dirs)
+    self._assert_cache()
+    self._assert_cdc_fuse_mounted()
+    original = utils.get_ssh_command_output(self.ls_cmd)
+
+    # Remove directory on workstation => empty directory on gamelet.
+    utils.remove_test_directory(self.local_base_dir)
+    self.assertTrue(self._wait_until_remote_dir_changed(original))
+    self._test_dir_content(files=[], dirs=[])
+    self._assert_cdc_fuse_mounted()
+
+    original = utils.get_ssh_command_output(self.ls_cmd)
+    # Recreate directory, add files => the content becomes visible again.
+    self._create_test_data(files, dirs)
+    self.assertTrue(self._wait_until_remote_dir_changed(original))
+    self._test_dir_content(files=files, dirs=dirs)
+    self._assert_cdc_fuse_mounted()
+
+  def test_non_existing_streamed_dir_fail(self):
+    """Fail if the streamed directory does not exist."""
+    streamed_dir = os.path.join(self.local_base_dir, 'non_existing')
+    self._assert_mount_fails(streamed_dir)
+    self._test_dir_content(files=[], dirs=[])
+    self._assert_cdc_fuse_mounted(success=False)
+
+  def test_streamed_dir_as_file_fail(self):
+    """Fail if the streamed path is a file."""
+    streamed_file = os.path.join(self.local_base_dir, 'file')
+    utils.create_test_file(streamed_file, 1024)
+    self._assert_mount_fails(streamed_file)
+    self._test_dir_content(files=[], dirs=[])
+    self._assert_cdc_fuse_mounted(success=False)
+
+  def test_remount_recreated_streamed_dir(self):
+    """Remounting a directory, which is currently removed, stops streaming session."""
+    files = [
+        'dir1\\file1_1.txt', 'dir1\\file1_2.txt', 'dir2\\file2_1.txt',
+        'file0.txt'
+    ]
+    dirs = ['dir1\\emptydir2\\', 'emptydir1\\', 'dir1\\', 'dir2\\']
+    self._create_test_data(files, dirs)
+    self._start()
+    self._test_dir_content(files=files, dirs=dirs)
+    self._assert_cache()
+    self._assert_cdc_fuse_mounted()
+    original = utils.get_ssh_command_output(self.ls_cmd)
+
+    # Remove directory on workstation => empty directory on gamelet.
+    utils.remove_test_directory(self.local_base_dir)
+    self.assertTrue(self._wait_until_remote_dir_changed(original))
+    self._test_dir_content(files=[], dirs=[])
+    self._assert_cdc_fuse_mounted()
+
+    # Remount for the same directory fails and stops an existing session.
+    self._assert_mount_fails(self.local_base_dir)
+    self._test_dir_content(files=[], dirs=[])
+
+    # Create a new folder and mount -> should succeed.
+    test_dir = 'Temp'
+    file_name = 'test_file.txt'
+    utils.create_test_file(
+        os.path.join(self.local_base_dir, test_dir, file_name), 100)
+    self._start(os.path.join(self.local_base_dir, test_dir))
+    self._assert_remote_dir_matches([file_name])
+
+
+if __name__ == '__main__':
+  test_base.test_base.main()

integration_tests/cdc_stream/general_test.py

@@ -0,0 +1,275 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""cdc_stream general test."""
+
+import os
+import posixpath
+import shutil
+
+from integration_tests.framework import utils
+from integration_tests.cdc_stream import test_base
+
+
+class GeneralTest(test_base.CdcStreamTest):
+  """cdc_stream general test class."""
+
+  def test_stream(self):
+    """Stream an existing directory."""
+    files = [
+        'dir1\\file1_1.txt', 'dir1\\file1_2.txt', 'dir2\\file2_1.txt',
+        'file0.txt'
+    ]
+    dirs = ['dir1\\emptydir2\\', 'emptydir1\\', 'dir1\\', 'dir2\\']
+    self._create_test_data(files, dirs)
+    self._start()
+    self._test_dir_content(files=files, dirs=dirs)
+    self._assert_cache()
+    self._assert_cdc_fuse_mounted()
+
+  def test_update_file(self):
+    """File updates are visible on gamelet."""
+    filename = 'file1.txt'
+    utils.create_test_file(os.path.join(self.local_base_dir, filename), 1024)
+    self._start()
+    self._test_dir_content(files=[filename], dirs=[])
+    cache_size = self._get_cache_size_in_bytes()
+    original = utils.get_ssh_command_output(self.ls_cmd)
+
+    # Modify the file, cache should become larger.
+    utils.create_test_file(os.path.join(self.local_base_dir, filename), 2048)
+
+    self.assertTrue(self._wait_until_remote_dir_changed(original))
+    self._test_dir_content(files=[filename], dirs=[])
+    self.assertGreater(self._get_cache_size_in_bytes(), cache_size)
+
+  def test_add_file(self):
+    """New file is visible on gamelet."""
+    self._start()
+    self._test_dir_content(files=[], dirs=[])
+    cache_size = self._get_cache_size_in_bytes()
+    # Create a file, cache should become larger.
+    filename = 'file1.txt'
+    original = utils.get_ssh_command_output(self.ls_cmd)
+
+    utils.create_test_file(os.path.join(self.local_base_dir, filename), 1024)
+
+    self.assertTrue(self._wait_until_remote_dir_changed(original))
+    self._test_dir_content(files=[filename], dirs=[])
+    self.assertGreater(self._get_cache_size_in_bytes(), cache_size)
+
+  def test_change_mtime(self):
+    """Change of mtime is visible on gamelet."""
+    filename = 'file1.txt'
+    file_local_path = os.path.join(self.local_base_dir, filename)
+    utils.create_test_file(file_local_path, 1024)
+    self._start()
+    mtime = os.path.getmtime(file_local_path)
+    self._test_dir_content(files=[filename], dirs=[])
+    cache_size = self._get_cache_size_in_bytes()
+    original = utils.get_ssh_command_output(self.ls_cmd)
+
+    # Change mtime of the file, a new manifest should be created.
+    utils.change_modified_time(file_local_path)
+
+    self.assertTrue(self._wait_until_remote_dir_changed(original))
+
+    # Cache should become larger.
+    self._test_dir_content(files=[filename], dirs=[])
+    self.assertNotEqual(os.path.getmtime(file_local_path), mtime)
+    self.assertGreater(self._get_cache_size_in_bytes(), cache_size)
+
+  def test_remove_file(self):
+    """File removal is visible on gamelet."""
+    filename = 'file1.txt'
+    file_local_path = os.path.join(self.local_base_dir, filename)
+    utils.create_test_file(file_local_path, 1024)
+    self._start()
+    self._test_dir_content(files=[filename], dirs=[])
+    cache_size = self._get_cache_size_in_bytes()
+    original = utils.get_ssh_command_output(self.ls_cmd)
+
+    # After removing a file, the manifest is updated.
+    utils.remove_test_file(file_local_path)
+
+    self.assertTrue(self._wait_until_remote_dir_changed(original))
+
+    self._test_dir_content(files=[], dirs=[])
+    self.assertGreater(self._get_cache_size_in_bytes(), cache_size)
+
+    filename = 'file1.txt'
+    file_local_path = os.path.join(self.local_base_dir, filename)
+    utils.create_test_file(file_local_path, 1024)
+    self._start()
+    self._test_dir_content(files=[filename], dirs=[])
+    cache_size = self._get_cache_size_in_bytes()
+
+    # After a file is renamed, the manifest is updated.
+    renamed_filename = 'file2.txt'
+    os.rename(file_local_path,
+              os.path.join(self.local_base_dir, renamed_filename))
+
+    self.assertTrue(self._wait_until_remote_dir_changed(original))
+
+    self._test_dir_content(files=[renamed_filename], dirs=[])
+    self.assertGreater(self._get_cache_size_in_bytes(), cache_size)
+
+  def test_add_directory(self):
+    """A new directory is visible on gamelet."""
+    self._start()
+    self._test_dir_content(files=[], dirs=[])
+    cache_size = self._get_cache_size_in_bytes()
+    original = utils.get_ssh_command_output(self.ls_cmd)
+
+    # Create a directory, cache becomes larger as a new manifest arrived.
+    directory = 'dir1\\'
+    dir_local_path = os.path.join(self.local_base_dir, directory)
+    utils.create_test_directory(dir_local_path)
+
+    self.assertTrue(self._wait_until_remote_dir_changed(original))
+    self._test_dir_content(files=[], dirs=[directory])
+    self.assertGreater(self._get_cache_size_in_bytes(), cache_size)
+
+  def test_remove_directory(self):
+    """A directory removal is visible on gamelet."""
+    directory = 'dir1\\'
+    dir_local_path = os.path.join(self.local_base_dir, directory)
+
+    utils.create_test_directory(dir_local_path)
+    self._start()
+    self._test_dir_content(files=[], dirs=[directory])
+    cache_size = self._get_cache_size_in_bytes()
+    original = utils.get_ssh_command_output(self.ls_cmd)
+
+    # After removing a file, the manifest is updated.
+    utils.remove_test_directory(dir_local_path)
+
+    self.assertTrue(self._wait_until_remote_dir_changed(original))
+
+    self._test_dir_content(files=[], dirs=[])
+    self.assertGreater(self._get_cache_size_in_bytes(), cache_size)
+
+  def test_rename_directory(self):
+    """A renamed directory is visible on gamelet."""
+    directory = 'dir1\\'
+    dir_local_path = os.path.join(self.local_base_dir, directory)
+
+    utils.create_test_directory(dir_local_path)
+    self._start()
+    self._test_dir_content(files=[], dirs=[directory])
+    cache_size = self._get_cache_size_in_bytes()
+    original = utils.get_ssh_command_output(self.ls_cmd)
+
+    # After removing a file, the manifest us updated.
+    renamed_directory = 'dir2\\'
+    os.rename(dir_local_path,
+              os.path.join(self.local_base_dir, renamed_directory))
+
+    self.assertTrue(self._wait_until_remote_dir_changed(original))
+
+    self._test_dir_content(files=[], dirs=[renamed_directory])
+    self.assertGreater(self._get_cache_size_in_bytes(), cache_size)
+
+  def test_detect_executables(self):
+    """Executable bits are propagated to gamelet."""
+    # Add an .exe, an ELF file and a .sh file to the streamed directory.
+    cdc_stream_dir = os.path.dirname(utils.CDC_STREAM_PATH)
+    exe_filename = os.path.basename(utils.CDC_STREAM_PATH)
+    elf_filename = 'cdc_fuse_fs'
+    sh_filename = 'script.sh'
+
+    shutil.copyfile(
+        os.path.join(cdc_stream_dir, exe_filename),
+        os.path.join(self.local_base_dir, exe_filename))
+    shutil.copyfile(
+        os.path.join(cdc_stream_dir, elf_filename),
+        os.path.join(self.local_base_dir, elf_filename))
+    with open(os.path.join(self.local_base_dir, sh_filename), 'w') as f:
+      f.write('#!/path/to/bash\n\nls -al')
+
+    files = [exe_filename, elf_filename, sh_filename]
+    self._start()
+    self._test_dir_content(files=files, dirs=[], is_exe=True)
+    self._assert_cache()
+
+  def test_resend_corrupted_chunks(self):
+    """Corrupted chunks are recovered."""
+    filename = 'file1.txt'
+    remote_file_path = posixpath.join(self.remote_base_dir, filename)
+    utils.create_test_file(os.path.join(self.local_base_dir, filename), 1024)
+    self._start()
+
+    manifest_chunk = utils.get_ssh_command_output('find %s -type f' %
+                                                  self.cache_dir).rstrip('\r\n')
+
+    # Read the file without caching.
+    utils.get_ssh_command_output('dd if=%s bs=1K of=/dev/null iflag=direct' %
+                                 remote_file_path)
+
+    # Find any data chunk.
+    data_chunks = utils.get_ssh_command_output('find %s -type f' %
+                                               self.cache_dir)
+    chunk_path = manifest_chunk
+    for chunk in data_chunks.splitlines():
+      if manifest_chunk not in chunk:
+        chunk_path = chunk.rstrip('\r\n')
+        break
+    chunk_data = utils.get_ssh_command_output('cat %s' % chunk_path)
+
+    # Modify the chosen data chunk.
+    utils.get_ssh_command_output('dd if=/dev/zero of=%s bs=1 count=3' %
+                                 chunk_path)
+    self.assertNotEqual(chunk_data,
+                        utils.get_ssh_command_output('cat %s' % chunk_path))
+
+    # Read the file again, the chunk should be recovered.
+    self._test_dir_content(files=[filename], dirs=[])
+    self.assertEqual(chunk_data,
+                     utils.get_ssh_command_output('cat %s' % chunk_path),
+                     'The corrupted chunk was not recreated')
+
+  def test_unicode(self):
+    """Stream a directory with non-ASCII Unicode paths."""
+    streamed_dir = '⛽⛽⛽'
+    filename = '⛽⛽⛽⛽⛽⛽⛽⛽.dat'
+    nonascii_local_data_path = os.path.join(self.local_base_dir, streamed_dir,
+                                            filename)
+    nonascii_remote_data_path = posixpath.join(self.remote_base_dir, filename)
+    utils.create_test_file(nonascii_local_data_path, 1024)
+    self._start(os.path.join(self.local_base_dir, streamed_dir))
+    self._assert_cache()
+    self.assertTrue(
+        utils.sha1_matches(nonascii_local_data_path, nonascii_remote_data_path))
+
+  def test_recovery(self):
+    """Remount succeeds also if FUSE was killed at the previous execution."""
+    files = [
+        'dir1\\file1_1.txt', 'dir1\\file1_2.txt', 'dir2\\file2_1.txt',
+        'file0.txt'
+    ]
+    dirs = ['dir1\\emptydir2\\', 'emptydir1\\', 'dir1\\', 'dir2\\']
+    self._create_test_data(files, dirs)
+    self._start()
+    self._test_dir_content(files=files, dirs=dirs)
+    self._assert_cache()
+    self._assert_cdc_fuse_mounted()
+    utils.get_ssh_command_output('killall cdc_fuse_fs')
+    self._test_dir_content(files=[], dirs=[])
+    self._start()
+    self._test_dir_content(files=files, dirs=dirs)
+
+
+if __name__ == '__main__':
+  test_base.test_base.main()

integration_tests/cdc_stream/test_base.py

@@ -0,0 +1,266 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""cdc_stream test."""
+
+import datetime
+import logging
+import os
+import posixpath
+import tempfile
+import time
+import subprocess
+import unittest
+
+from integration_tests.framework import utils
+from integration_tests.framework import test_base
+
+
+class CdcStreamTest(unittest.TestCase):
+  """cdc_stream test class."""
+
+  # Grpc status codes.
+  NOT_FOUND = 5
+  SERVICE_UNAVAILABLE = 14
+
+  tmp_dir = None
+  local_base_dir = None
+  remote_base_dir = '/tmp/_cdc_stream_test/'
+  cache_dir = '~/.cache/cdc-file-transfer/chunks/'
+  service_port_arg = None
+  service_running = False
+
+  # Returns a list of files and directories with mtimes in the remote directory.
+  # For example, 2021-10-13 07:49:25.512766391 -0700 /tmp/_cdc_stream_test/2.txt
+  ls_cmd = ('find %s -exec stat --format \"%%y %%n\" \"{}\" \\;') % (
+      remote_base_dir)
+
+  @classmethod
+  def setUpClass(cls) -> None:
+    super().setUpClass()
+    logging.debug('CdcStreamTest -> setUpClass')
+
+    utils.initialize(None, test_base.Flags.binary_path,
+                     test_base.Flags.user_host)
+    cls.service_port_arg = f'--service-port={test_base.Flags.service_port}'
+    cls._stop_service()
+    with tempfile.NamedTemporaryFile() as tf:
+      cls.config_path = tf.name
+
+  @classmethod
+  def tearDownClass(cls):
+    logging.debug('CdcStreamTest -> tearDownClass')
+    cls._stop_service()
+    if os.path.exists(cls.config_path):
+      os.remove(cls.config_path)
+
+  def setUp(self):
+    """Stops the service, cleans up cache and streamed directory and initializes random."""
+    super(CdcStreamTest, self).setUp()
+    logging.debug('CdcStreamTest -> setUp')
+
+    now_str = datetime.datetime.now().strftime('%Y%m%d-%H%M%S')
+    self.tmp_dir = tempfile.TemporaryDirectory(
+        prefix=f'_cdc_stream_test_{now_str}')
+    self.local_base_dir = self.tmp_dir.name + '\\base\\'
+    utils.create_test_directory(self.local_base_dir)
+
+    logging.info('Local base dir: "%s"', self.local_base_dir)
+    logging.info('Remote base dir: "%s"', self.remote_base_dir)
+    utils.initialize_random()
+    self._stop(ignore_not_found=True)
+    self._clean_cache()
+
+  def tearDown(self):
+    super(CdcStreamTest, self).tearDown()
+    logging.debug('CdcStreamTest -> tearDown')
+    self.tmp_dir.cleanup()
+
+  @classmethod
+  def _start_service(cls, config_json=None):
+    """Starts the asset streaming service.
+
+    Args:
+        config_json (string, optional): Config JSON string. Defaults to None.
+    """
+    config_arg = None
+    if config_json:
+      with open(cls.config_path, 'wt') as file:
+        file.write(config_json)
+      config_arg = f'--config-file={cls.config_path}'
+
+    # Note: Service must be spawned in a background process.
+    args = ['start-service', config_arg, cls.service_port_arg]
+    command = [utils.CDC_STREAM_PATH, *filter(None, args)]
+
+    # Workaround issue with unicode logging.
+    logging.debug(
+        'Executing %s ',
+        ' '.join(command).encode('utf-8').decode('ascii', 'backslashreplace'))
+    subprocess.Popen(command)
+    cls.service_running = True
+
+  @classmethod
+  def _stop_service(cls):
+    res = utils.run_stream('stop-service', cls.service_port_arg)
+    if res.returncode != 0:
+      logging.warn(f'Stopping service failed: {res}')
+    cls.service_running = False
+
+  def _start(self, local_dir=None):
+    """Starts streaming the given directory
+
+    Args:
+        local_dir (string): Directory to stream. Defaults to local_base_dir.
+    """
+    res = utils.run_stream('start', local_dir or self.local_base_dir,
+                           utils.target(self.remote_base_dir),
+                           self.service_port_arg)
+    self._assert_stream_success(res)
+
+  def _stop(self, ignore_not_found=False):
+    """Stops streaming to the target
+
+    Args:
+        local_dir (string): Directory to stream. Defaults to local_base_dir.
+    """
+    if not self.service_running:
+      return
+    res = utils.run_stream('stop', utils.target(self.remote_base_dir),
+                           self.service_port_arg)
+    if ignore_not_found and res.returncode == self.NOT_FOUND:
+      return
+    self._assert_stream_success(res)
+
+  def _assert_stream_success(self, res):
+    """Asserts if the return code is 0 and outputs return message with args."""
+    self.assertEqual(res.returncode, 0, 'Return value is ' + str(res))
+
+  def _assert_remote_dir_matches(self, file_list):
+    """Asserts that the remote directory matches the list of files and directories.
+
+    Args:
+        file_list (list of strings): List of relative paths to check.
+    """
+    found = utils.get_sorted_files(self.remote_base_dir, '"*"')
+    expected = sorted(['./' + f for f in file_list])
+    self.assertListEqual(found, expected)
+
+  def _get_cache_size_in_bytes(self):
+    """Returns the asset streaming cache size in bytes.
+
+    Returns:
+        bool: Cache size in bytes.
+    """
+    result = utils.get_ssh_command_output('du -sb %s | awk \'{print $1}\'' %
+                                          self.cache_dir)
+    logging.info(f'Cache capacity is {int(result)}')
+    return int(result)
+
+  def _assert_cache(self):
+    """Asserts that the asset streaming cache contains some data."""
+    cache_size = self._get_cache_size_in_bytes()
+    # On Linux, an empty directory occupies 4KB.
+    self.assertTrue(int(cache_size) >= 4096)
+    self.assertGreater(
+        int(utils.get_ssh_command_output('ls %s | wc -l' % self.cache_dir)), 0)
+
+  def _assert_cdc_fuse_mounted(self, success=True):
+    """Asserts that CDC FUSE is appropriately mounted."""
+    logging.info(f'Asserting that FUSE is {"" if success else "not "}mounted')
+    result = utils.get_ssh_command_output('cat /etc/mtab | grep fuse')
+    if success:
+      self.assertIn(f'{self.remote_base_dir[:-1]} fuse.', result)
+    else:
+      self.assertNotIn(f'{self.remote_base_dir[:-1]} fuse.', result)
+
+  def _clean_cache(self):
+    """Removes all data from the asset streaming caches."""
+    logging.info(f'Clearing cache')
+    utils.get_ssh_command_output('rm -rf %s' %
+                                 posixpath.join(self.cache_dir, '*'))
+    cache_dir = os.path.join(os.environ['APPDATA'], 'cdc-file-transfer',
+                             'chunks')
+    utils.remove_test_directory(cache_dir)
+
+  def _create_test_data(self, files, dirs):
+    """Create test data locally.
+
+    Args:
+        files (list of strings): List of relative file paths to create.
+        dirs (list of strings): List of relative dir paths to create.
+    """
+    logging.info(
+        f'Creating test data with {len(files)} files and {len(dirs)} dirs')
+    for directory in dirs:
+      utils.create_test_directory(os.path.join(self.local_base_dir, directory))
+    for file in files:
+      utils.create_test_file(os.path.join(self.local_base_dir, file), 1024)
+
+  def _wait_until_remote_dir_changed(self, original, counter=20):
+    """Wait until the directory content has changed.
+
+    Args:
+        original (string): The original file list of the remote directory.
+        counter (int): The number of retries.
+    Returns:
+        bool: Whether the content of the remote directory has changed.
+    """
+    logging.info(f'Waiting until remote dir changes')
+    for _ in range(counter):
+      if utils.get_ssh_command_output(self.ls_cmd) != original:
+        return True
+      time.sleep(0.1)
+      logging.info(f'Still waiting...')
+    return False
+
+  def _test_dir_content(self, files, dirs, is_exe=False):
+    """Check the streamed directory's content on gamelet.
+
+    Args:
+        files (list of strings): List of relative file paths to check.
+        dirs (list of strings): List of relative dir paths to check.
+        is_exe (bool): Flag which identifies whether files are executables.
+    """
+    logging.info(
+        f'Testing dir content with {len(files)} files and {len(dirs)} dirs')
+    dirs = [directory.replace('\\', '/').rstrip('/') for directory in dirs]
+    files = [file.replace('\\', '/') for file in files]
+
+    # Read the content of the directory once to load some data.
+    utils.get_ssh_command_output('ls -al %s' % self.remote_base_dir)
+    self._assert_remote_dir_matches(files + dirs)
+    if not dirs and not files:
+      return
+    file_list = list()
+    mapping = dict()
+    for file in files:
+      full_name = posixpath.join(self.remote_base_dir, file)
+      self.assertTrue(
+          utils.sha1_matches(
+              os.path.join(self.local_base_dir, file), full_name))
+      file_list.append(full_name)
+      if is_exe:
+        mapping[full_name] = '-rwxr-xr-x'
+      else:
+        mapping[full_name] = '-rw-r--r--'
+    for directory in dirs:
+      full_name = posixpath.join(self.remote_base_dir, directory)
+      file_list.append(full_name)
+      mapping[full_name] = 'drwxr-xr-x'
+
+    ls_res = utils.get_ssh_command_output('ls -ld %s' % ' '.join(file_list))
+    for line in ls_res.splitlines():
+      self.assertIn(mapping[list(filter(None, line.split(' ')))[8]], line)

integration_tests/framework/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

integration_tests/framework/test_base.py

@@ -0,0 +1,63 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""Test main and flags."""
+
+import argparse
+import contextlib
+import logging
+import sys
+import unittest
+
+from integration_tests.framework import test_runner
+
+
+class Flags(object):
+  binary_path = None
+  user_host = None
+  service_port = 0
+
+
+def main():
+  parser = argparse.ArgumentParser(description='End-to-end integration test.')
+  parser.add_argument('--binary_path', help='Target [user@]host', required=True)
+  parser.add_argument('--user_host', help='Target [user@]host', required=True)
+  parser.add_argument(
+      '--service_port',
+      type=int,
+      help='Asset streaming service port',
+      default=44432)
+  parser.add_argument('--log_file', help='Log file path')
+
+  # Capture all remaining arguments to pass to unittest.main().
+  args, unittest_args = parser.parse_known_args()
+  Flags.binary_path = args.binary_path
+  Flags.user_host = args.user_host
+  Flags.service_port = args.service_port
+
+  # Log to STDERR
+  log_format = ('%(levelname)-8s%(asctime)s '
+                '%(filename)s:%(lineno)-3d %(message)s')
+  log_stream = sys.stderr
+
+  if args.log_file:
+    log_stream = open(args.log_file, 'w')
+
+  with log_stream:
+    logging.basicConfig(
+        format=log_format, level=logging.DEBUG, stream=log_stream)
+
+    unittest.main(
+        argv=sys.argv[:1] + unittest_args, testRunner=test_runner.TestRunner())

integration_tests/framework/test_runner.py

@@ -0,0 +1,66 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""Test runner, adds some sugar around logs to make them easier to read."""
+
+import logging
+import traceback
+import unittest
+
+
+class TestRunner(object):
+  """Runner producing test xml output."""
+
+  def run(self, test):  # pylint: disable=invalid-name
+    result = TestResult()
+    logging.info('Running tests...')
+    test(result)
+    logging.info('\n\n******************* TESTS FINISHED *******************\n')
+    logging.info('Ran %d tests with %d errors and %d failures', result.testsRun,
+                 len(result.errors), len(result.failures))
+    for test_and_stack in result.failures:
+      logging.info('\n\n[  TEST FAILED  ] %s\n', test_and_stack[0])
+      logging.info(
+          '%s', test_and_stack[1].replace('\\\\r',
+                                          '\r').replace('\\\\n', '\n').replace(
+                                              '\\r', '\r').replace('\\n', '\n'))
+
+    return result
+
+
+class TestResult(unittest.TestResult):
+
+  def startTest(self, test):
+    """Called when the given test is about to be run."""
+    logging.info('\n\n===== BEGIN TEST CASE: %s =====\n', test)
+    unittest.TestResult.startTest(self, test)
+
+  def stopTest(self, test):
+    """Called when the given test has been run."""
+    unittest.TestResult.stopTest(self, test)
+    logging.info('\n\n===== END TEST CASE: %s =====\n', test)
+
+  def addError(self, test, err):
+    unittest.TestResult.addError(self, test, err)
+    self._LogFailureInfo(err)
+
+  def addFailure(self, test, err):
+    unittest.TestResult.addFailure(self, test, err)
+    self._LogFailureInfo(err)
+
+  def _LogFailureInfo(self, err):
+    exctype, exc, tb = err
+    detail = ''.join(traceback.format_exception(exctype, exc, tb))
+    logging.error('FAILURE: %s', detail)

integration_tests/framework/utils.py

@@ -0,0 +1,346 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""Utils for file transfer tests."""
+
+import hashlib
+import logging
+import os
+import pathlib
+import random
+import shutil
+import string
+import subprocess
+import time
+import sys
+
+CDC_RSYNC_PATH = None
+CDC_STREAM_PATH = None
+USER_HOST = None
+
+SHA1_LEN = 40
+SHA1_BUF_SIZE = 65536
+RANDOM = random.Random()
+
+
+def initialize(cdc_rsync_path, cdc_stream_path, user_host):
+  """Sets global variables."""
+  global CDC_RSYNC_PATH, CDC_STREAM_PATH, USER_HOST
+
+  CDC_RSYNC_PATH = cdc_rsync_path
+  CDC_STREAM_PATH = cdc_stream_path
+  USER_HOST = user_host
+
+
+def initialize_random():
+  """Sets random seed."""
+  global RANDOM
+  seed = int(time.time())
+  logging.debug('Use random seed %i', seed)
+  RANDOM.seed(seed)
+
+
+def _remove_carriage_return_lines(text):
+  r"""Removes *\r, keeps only *\r\n lines.
+
+  Args:
+      text (string): Text to remove lines from (usually cdc_rsync output).
+
+  Returns:
+      string: Text with lines removed.
+  """
+
+  # Some lines have \r\r\n, treat them properly.
+  ret = ''
+  for line in text.replace('\r\r', '\r').split('\r\n'):
+    ret += line.split('\r')[-1] + '\r\n'
+  return ret
+
+
+def target(dir):
+  """Prepends user@host: to dir."""
+  return USER_HOST + ":" + dir
+
+
+def run_rsync(*args):
+  """Runs cdc_rsync with given args.
+
+  The last positional argument is assumed to be the destination. The user/host
+  prefix [user@]host: is optional. If it does not have one, then it is prefixed
+  by |USER_HOST|:.
+
+  Args:
+      *args (string): cdc_rsync arguments.
+
+  Returns:
+      CompletedProcess: cdc_rsync process info with exit code and stdout/stderr.
+  """
+
+  # Prefix last positional argument with [user@]host: if it doesn't have such
+  # a prefix yet. Note that this won't work in all cases, e.g. if
+  # '--exclude', 'file' is passed. Use '--exclude=file' instead.
+  args_list = list(filter(None, args))
+  for n in range(len(args_list) - 1, 0, -1):
+    if args_list[n][0] != '-' and not ':' in args_list[n]:
+      args_list[n] = target(args_list[n])
+      break
+
+  command = [CDC_RSYNC_PATH, *args_list]
+
+  # Workaround issue with unicode logging.
+  logging.debug(
+      'Executing %s ',
+      ' '.join(command).encode('utf-8').decode('ascii', 'backslashreplace'))
+  res = subprocess.run(command, capture_output=True)
+  # Remove lines ending with \r since those are temp display lines.
+  res.stdout = _remove_carriage_return_lines(res.stdout.decode('ascii'))
+  if res.stdout.strip():
+    logging.debug('\r\n%s', res.stdout)
+  return res
+
+
+def run_stream(*args):
+  """Runs cdc_stream with given args.
+
+  Args:
+      *args (string): cdc_stream arguments.
+
+  Returns:
+      CompletedProcess: cdc_stream process info with exit code and stdout/stderr.
+  """
+
+  command = [CDC_STREAM_PATH, *filter(None, args)]
+
+  # Workaround issue with unicode logging.
+  logging.debug(
+      'Executing %s ',
+      ' '.join(command).encode('utf-8').decode('ascii', 'backslashreplace'))
+  return subprocess.run(command)
+
+
+def files_count_is(cdc_rsync_res,
+                   missing=0,
+                   missing_dir=0,
+                   changed=0,
+                   matching=0,
+                   matching_dir=0,
+                   extraneous=0,
+                   extraneous_dir=0):
+  r"""Verifies that the output of cdc_rsync indicates the given file counts.
+
+  Args:
+      cdc_rsync_res (CompletedProcess): Completed cdc_rsync process
+      missing (int, optional): Number of missing files. Defaults to 0.
+      missing_dir (int, optional): Number of missing folders. Defaults to 0.
+      changed (int, optional): Number of changed files. Defaults to 0.
+      matching (int, optional): Number of matching files. Defaults to 0.
+      matching_dir (int, optional): Number of matching folders. Defaults to 0.
+      extraneous (int, optional): Number of extraneous files. Defaults to 0.
+      extraneous_dir (int, optional): Number of extraneous folders. \ Defaults
+        to 0.
+
+  Returns:
+      bool: True if all file counts match.
+  """
+  missing_ok = '%i file(s) and %i folder(s) are not present' % (
+      missing, missing_dir) in cdc_rsync_res.stdout
+  changed_ok = '%i file(s) changed' % (changed) in cdc_rsync_res.stdout
+  matching_ok = '%i file(s) and %i folder(s) match' % (
+      matching, matching_dir) in cdc_rsync_res.stdout or """%i file(s) and %i \
+folder(s) have matching modified time and size""" % (
+          matching, matching_dir) in cdc_rsync_res.stdout
+  extraneous_ok = """%i file(s) and %i folder(s) on the instance do not exist \
+on this machine""" % (extraneous, extraneous_dir) in cdc_rsync_res.stdout
+  return missing_ok and changed_ok and matching_ok and extraneous_ok
+
+
+def sha1sum_local(filepath):
+  """Computes the sha1 hash of a local file.
+
+  Args:
+      filepath (string): Path of the local (Windows) file
+
+  Returns:
+      string: sha1 hash
+  """
+  sha1 = hashlib.sha1()
+  with open(filepath, 'rb') as f:
+    while True:
+      data = f.read(SHA1_BUF_SIZE)
+      if not data:
+        break
+      sha1.update(data)
+  return sha1.hexdigest()
+
+
+def sha1sum_remote(filepath):
+  """Computes the sha1 hash of a remote file.
+
+  Args:
+      filepath (string): Path of the remote (Linux) file
+
+  Returns:
+      string: sha1 hash
+  """
+  return get_ssh_command_output('sha1sum %s' % filepath)[0:SHA1_LEN]
+
+
+def sha1_matches(local_path, remote_path):
+  """Compares the sha1 hashes of a local and a remote file.
+
+  Args:
+      local_path (string): Path of the local (Windows) file
+      remote_path (string): Path of the remote (Linux) file
+
+  Returns:
+      bool: True if the sha1 hashes match
+  """
+
+  sha1_local = sha1sum_local(local_path)
+  sha1_remote = sha1sum_remote(remote_path)
+  return sha1_local == sha1_remote
+
+
+def create_test_file(local_path, size, printable_data=True, append=False):
+  """Creates a test file with random text of given size.
+
+  Args:
+      local_path (string): Local path of the file to create.
+      size (integer): Size of the file to create (bytes).
+      printable_data (bool, optional): If the data should be printable. Writing
+          a file with printable data is slower, for 1GB of data this takes ~5
+          minutes, in comparison to ~2 seconds for non printable data. Defaults
+          to True.
+      append (bool, optional): If append mode should be used. Defaults to False.
+  """
+  pathlib.Path(os.path.dirname(local_path)).mkdir(parents=True, exist_ok=True)
+
+  mode = None
+  random_bytes = None
+  if printable_data:
+    mode = 'at' if append else 'wt'
+    random_bytes = ''.join(
+        RANDOM.choices(string.ascii_uppercase + string.digits, k=size))
+  else:
+    mode = 'ab' if append else 'wb'
+    random_bytes = os.urandom(size)
+
+  with open(local_path, mode) as f:
+    if size > 0:
+      f.write(random_bytes)
+
+
+def remove_test_file(local_path):
+  """Deletes a test file.
+
+  Args:
+      local_path (string): Local path of the file to delete.
+  """
+  os.remove(local_path)
+
+
+def create_test_directory(local_path):
+  """Creates a directory.
+
+  Args:
+      local_path (string): Local path of the directory to create.
+  """
+  pathlib.Path(os.path.dirname(local_path)).mkdir(parents=True, exist_ok=True)
+
+
+def remove_test_directory(local_path):
+  """Removes a directory with its content.
+
+  Args:
+      local_path (string): Local path of the directory to remove.
+  """
+  shutil.rmtree(pathlib.Path(os.path.dirname(local_path)), ignore_errors=True)
+
+
+def does_directory_exist_remotely(path):
+  """Checks if a directory exists on the remote instance.
+
+  Args:
+      path (string): Path of the remote (Linux) directory
+
+  Returns:
+      bool: True if a directory exists.
+  """
+  return 'yes' in get_ssh_command_output('test -d %s && echo "yes"' % path)
+
+
+def does_file_exist_remotely(path):
+  """Checks if a file exists on the remote instance.
+
+  Args:
+      path (string): Path of the remote (Linux) file
+
+  Returns:
+      bool: True if a file exists.
+  """
+  return 'yes' in get_ssh_command_output('test -f %s && echo "yes"' % path)
+
+
+def change_modified_time(path):
+  """Changes the modified time of the given file.
+
+  Args:
+      path (string): Path of the local file
+  """
+  stats = os.stat(path)
+  os.utime(path, (stats.st_atime, stats.st_mtime + 1))
+
+
+def get_ssh_command_output(cmd):
+  """Runs an SSH command using the command from the CDC_SSH_COMMAND env var.
+
+  Args:
+      cmd (string): Command that is being run remotely
+
+  Returns:
+      string: The output of the ssh command.
+  """
+  ssh_command = os.environ.get('CDC_SSH_COMMAND') or "ssh"
+  full_ssh_cmd = '%s -tt "%s" -- %s' % (ssh_command, USER_HOST,
+                                        quote_argument(cmd))
+  res = subprocess.run(full_ssh_cmd, capture_output=True)
+  if res.returncode != 0:
+    logging.warning('SSH command %s failed with code %i, stderr: %s', cmd,
+                    res.returncode, res.stderr)
+  return res.stdout.decode('ascii', errors='replace')
+
+
+def quote_argument(argument):
+  # This isn't fully generic, but does the job... It doesn't handle when the
+  # argument already escapes quotes, for instance.
+  return '"' + argument.replace('"', '\\"') + '"'
+
+
+def get_sorted_files(remote_dir, pattern='"*.[t|d]*"'):
+  """Returns a sorted list of files in the remote_dir.
+
+  Args:
+      remote_dir (string): Remote directory.
+      pattern (string, optional): Pattern for matching file names.
+
+  Returns:
+      string: Sorted list of files found in the remote directory.
+  """
+  find_res = get_ssh_command_output('cd %s && find -name %s -print' %
+                                    (remote_dir, pattern))
+
+  found = sorted(
+      filter(lambda item: item and item != '.', find_res.split('\r\n')))
+  return found

proto/background_service.proto

@@ -23,22 +23,16 @@ import "google/protobuf/empty.proto";
 service BackgroundService {
   // Exit is used to ask the service to exit. In the case of the process
   // manager, this cascades to all background processes.
-  rpc Exit(ExitRequest) returns (ExitResponse) {}
+  rpc Exit(google.protobuf.Empty) returns (google.protobuf.Empty) {}
 
   // GetPid is used to get the PID of the service process.
-  rpc GetPid(GetPidRequest) returns (GetPidResponse) {}
+  rpc GetPid(google.protobuf.Empty) returns (GetPidResponse) {}
 
   // HealthCheck is used to verify that the service is running. It returns an
   // empty protobuf if the service is ready to serve requests.
   rpc HealthCheck(google.protobuf.Empty) returns (google.protobuf.Empty) {}
 }
 
-message ExitRequest {}
-
-message ExitResponse {}
-
-message GetPidRequest {}
-
 message GetPidResponse {
   int32 pid = 1;
 }

proto/local_assets_stream_manager.proto

@@ -45,9 +45,6 @@ message StartSessionRequest {
   string user_host = 7;
   // Remote directory where to mount the streamed directory.
   string mount_dir = 8;
-  // SSH port to use while connecting to the remote instance.
-  // Optional, falls back to port 22 (default SSH port).
-  int32 port = 9;
   // SSH command to connect to the remote instance.
   // Optional, falls back to searching ssh.
   string ssh_command = 10;
@@ -55,7 +52,7 @@ message StartSessionRequest {
   // Optional, falls back to searching scp.
   string scp_command = 11;
 
-  reserved 1, 3, 4;
+  reserved 1, 3, 4, 9;
 }
 
 message StartSessionResponse {}
@@ -65,9 +62,10 @@ message StopSessionRequest {
   // ID of assets streaming target gamelet.
   // Only used by Stadia. Should set either this or user_host_dir.
   string gamelet_id = 1;
-  // Username and host, in the form [user@]host.
+  // Username and host, in the form [user@]host. Accepts wildcards * and ?.
   string user_host = 2;
   // Remote directory where the streamed directory is mounted.
+  // Accepts wildcards * and ?.
   string mount_dir = 3;
 }
 

tests_common/BUILD

@@ -30,6 +30,7 @@ cc_binary(
         "//common:path_filter",
         "//common:platform",
         "//common:port_manager",
+        "//common:port_range_parser",
         "//common:process",
         "//common:remote_util",
         "//common:sdk_util",