catarrh/cdc-file-transfer
1
0
Fork 0
mirror of https://github.com/google/cdc-file-transfer.git synced 2026-09-13 01:10:44 +03:00
Code Issues Packages Projects Releases Wiki Activity

Compare commits

base: catarrh/cdc-file-transfer:main
Branches Tags
catarrh/cdc-file-transfer:main catarrh/cdc-file-transfer:dependabot/github_actions/dot-github/workflows/github_actions-14598a9f5f catarrh/cdc-file-transfer:sync_windows catarrh/cdc-file-transfer:readme
catarrh/cdc-file-transfer:latest catarrh/cdc-file-transfer:v1.0.0
..
compare: catarrh/cdc-file-transfer:readme
Branches Tags
catarrh/cdc-file-transfer:dependabot/github_actions/dot-github/workflows/github_actions-14598a9f5f catarrh/cdc-file-transfer:sync_windows catarrh/cdc-file-transfer:main catarrh/cdc-file-transfer:readme
catarrh/cdc-file-transfer:latest catarrh/cdc-file-transfer:v1.0.0

1 Commits
main .. readme

Author SHA1 Message Date
Lutz Justen f09310bc91 Add Google specific instructions to run without security key touches 2022-12-02 10:02:44 +01:00
157 changed files with 1816 additions and 8065 deletions
Expand all files Collapse all files
.github/workflows/build_and_test.yml
+8 -27
View File
@@ -23,26 +23,17 @@ jobs:
- name: Initialize submodules
run: git submodule update --init --recursive
- name: Create timestamp
run: |
printf -v date '%(%Y-%m)T' -1
echo "date=$date" >> $GITHUB_ENV
- name: Restore build cache
uses: actions/cache@v3
with:
path: bazel-cache
key: ${{ runner.os }}-bazel-cache-fastbuild-${{ env.date }}
- name: Build (fastbuild)
run: bazel build --config=linux --disk_cache=bazel-cache -- //... -//third_party/...
run: |
bazel build --config=linux -- //... -//third_party/...
# Skip file_finder_test: The test works when file_finder_test is run
# directly, but not through bazel test. The reason is, bazel test
# creates symlinks of test files, but the finder ignores symlinks.
# Also run tests sequentially since some tests write to a common tmp dir.
- name: Test (fastbuild)
run: bazel test --config=linux --disk_cache=bazel-cache --test_output=errors --local_test_jobs=1 -- //... -//third_party/... -//cdc_rsync_server:file_finder_test
run: |
bazel test --config=linux --test_output=errors --local_test_jobs=1 -- //... -//third_party/... -//cdc_rsync_server:file_finder_test
Build-And-Test-Windows:
runs-on: windows-2019
@@ -52,26 +43,16 @@ jobs:
- name: Initialize submodules
run: git submodule update --init --recursive
- name: Create timestamp
- name: Build
run: |
$date = Get-Date -Format "yyyy-MM"
echo "date=$date" >> $env:GITHUB_ENV
bazel build --config=windows //cdc_rsync //cdc_stream //tests_common //tests_cdc_stream //tests_cdc_rsync
- name: Restore build cache
uses: actions/cache@v3
with:
path: bazel-cache
key: ${{ runner.os }}-bazel-cache-fastbuild-${{ env.date }}
- name: Build (fastbuild)
run: bazel build --config=windows --disk_cache=bazel-cache //cdc_rsync //cdc_stream //tests_common //tests_cdc_stream //tests_cdc_rsync
- name: Test (fastbuild)
- name: Test
run: |
bazel-bin\tests_common\tests_common.exe
bazel-bin\tests_cdc_stream\tests_cdc_stream.exe
bazel-bin\tests_cdc_rsync\tests_cdc_rsync.exe
bazel test --config=windows --disk_cache=bazel-cache --test_output=errors --local_test_jobs=1 `
bazel test --config=windows --test_output=errors --local_test_jobs=1 `
//cdc_fuse_fs/... `
//cdc_rsync/... `
//cdc_rsync/base/... `
.github/workflows/create_release.yml
+11 -95
View File
@@ -14,55 +14,21 @@ jobs:
steps:
- uses: actions/checkout@v3
- name: Use last commit hash as build version for the developer build.
if: "startsWith(github.ref, 'refs/heads/')"
run: echo "build_version=${GITHUB_SHA}" >> $GITHUB_ENV
- name: Use tag name as build version for the release build.
if: startsWith(github.ref, 'refs/tags/v')
run: echo "build_version=${GITHUB_REF#refs/tags/v}" >> $GITHUB_ENV
# This flow should not be used for pull requests. However the section
# below might be useful for testing purposes.
- name: Use last commit hash as build version for the pull request
if: startsWith(github.ref, 'refs/pull')
run: echo "build_version=${GITHUB_SHA}" >> $GITHUB_ENV
- name: Replace CDC_BUILD_VERSION
run: |
if grep -q "DCDC_BUILD_VERSION=DEV" "common/BUILD"; then
sed -i 's/DCDC_BUILD_VERSION=DEV/DCDC_BUILD_VERSION=${{ env.build_version }}/g' common/BUILD
else
echo "CDC_BUILD_VERSION was moved out from common/BUILD file."
echo "Please edit create_release.yaml workflow."
exit 1
fi
- name: Initialize submodules
run: git submodule update --init --recursive
- name: Create timestamp
- name: Build
run: |
printf -v date '%(%Y-%m)T' -1
echo "date=$date" >> $GITHUB_ENV
- name: Restore build cache
uses: actions/cache@v3
with:
path: bazel-cache
key: ${{ runner.os }}-bazel-cache-opt-${{ env.date }}
- name: Build (opt)
run: |
bazel build --config=linux --disk_cache=bazel-cache --compilation_mode=opt --linkopt=-Wl,--strip-all --copt=-fdata-sections --copt=-ffunction-sections --linkopt=-Wl,--gc-sections \
bazel build --config=linux --compilation_mode=opt --linkopt=-Wl,--strip-all --copt=-fdata-sections --copt=-ffunction-sections --linkopt=-Wl,--gc-sections \
//cdc_fuse_fs //cdc_rsync_server
- name: Test (opt)
- name: Test
run: |
bazel test --config=linux --disk_cache=bazel-cache --compilation_mode=opt --linkopt=-Wl,--strip-all --copt=-fdata-sections --copt=-ffunction-sections --linkopt=-Wl,--gc-sections \
bazel test --config=linux --compilation_mode=opt --linkopt=-Wl,--strip-all --copt=-fdata-sections --copt=-ffunction-sections --linkopt=-Wl,--gc-sections \
--test_output=errors --local_test_jobs=1 \
-- //... -//third_party/... -//cdc_rsync_server:file_finder_test
# The artifact collector doesn't like the fact that bazel-bin is a symlink.
- name: Copy artifacts
run: |
mkdir artifacts
@@ -81,65 +47,20 @@ jobs:
steps:
- uses: actions/checkout@v3
- name: Use last commit hash as build version for the developer build.
if: "startsWith(github.ref, 'refs/heads/')"
run: |
$build_version="${{ github.sha }}"
echo "build_version=$build_version" >> $env:GITHUB_ENV
- name: Use tag name as build version for the release build.
if: "startsWith(github.ref, 'refs/tags/v')"
run: |
$build_version="${{ github.ref }}".replace("refs/tags/v", "")
echo "build_version=$build_version" >> $env:GITHUB_ENV
# This flow should not be used for pull requests. However the section
# below might be useful for testing purposes.
- name: Use last commit hash as build version for the pull request
if: startsWith(github.ref, 'refs/pull')
run: |
$build_version="${{ github.sha }}"
echo "build_version=$build_version" >> $env:GITHUB_ENV
- name: Replace CDC_BUILD_VERSION
run: |
$cdc_version = Select-String -Path common/BUILD -Pattern "DCDC_BUILD_VERSION=DEV"
if ($cdc_version -ne $null) {
$build_file = Get-Content -path common/BUILD -Raw
$build_file = $build_file -replace 'DCDC_BUILD_VERSION=DEV','DCDC_BUILD_VERSION=${{ env.build_version }}'
$build_file | Set-Content -Path common/BUILD
}
else {
Write-Host "CDC_BUILD_VERSION was moved out from common/BUILD file."
Write-Host "Please edit create_release.yaml workflow."
exit 1
}
- name: Initialize submodules
run: git submodule update --init --recursive
- name: Create timestamp
- name: Build
run: |
$date = Get-Date -Format "yyyy-MM"
echo "date=$date" >> $env:GITHUB_ENV
- name: Restore build cache
uses: actions/cache@v3
with:
path: bazel-cache
key: ${{ runner.os }}-bazel-cache-opt-${{ env.date }}
- name: Build (opt)
run: |
bazel build --config=windows --disk_cache=bazel-cache --compilation_mode=opt --copt=/GL `
bazel build --config=windows --compilation_mode=opt --copt=/GL `
//cdc_rsync //cdc_stream //tests_common //tests_cdc_stream //tests_cdc_rsync
- name: Test (opt)
- name: Test
run: |
bazel-bin\tests_common\tests_common.exe
bazel-bin\tests_cdc_stream\tests_cdc_stream.exe
bazel-bin\tests_cdc_rsync\tests_cdc_rsync.exe
bazel test --config=windows --disk_cache=bazel-cache --compilation_mode=opt --copt=/GL --test_output=errors --local_test_jobs=1 `
bazel test --config=windows --compilation_mode=opt --copt=/GL --test_output=errors --local_test_jobs=1 `
//cdc_fuse_fs/... `
//cdc_rsync/... `
//cdc_rsync/base/... `
@@ -151,16 +72,14 @@ jobs:
//manifest/... `
//metrics/...
# The artifact collector doesn't like the fact that bazel-bin is a symlink.
- name: Copy artifacts
run: |
mkdir artifacts
mkdir artifacts\docs
cp bazel-bin/cdc_rsync/cdc_rsync.exe artifacts
cp bazel-bin/cdc_rsync_server/cdc_rsync_server.exe artifacts
cp bazel-bin/cdc_stream/cdc_stream.exe artifacts
cp LICENSE artifacts
cp README.md artifacts
cp docs\* artifacts\docs
- name: Upload artifacts
uses: actions/upload-artifact@v3
@@ -178,10 +97,7 @@ jobs:
- name: Zip binaries
run: |
# The ref resolves to "main" for latest and e.g. "v0.1.0" for tagged.
REF=${GITHUB_REF#refs/*/}
# For pull requests, this is e.g. '70/merge', so replace / with -.
REF=${REF/\//-}
BINARIES_ZIP_NAME=cdc-file-transfer-binaries-$REF-x64.zip
BINARIES_ZIP_NAME=cdc-file-transfer-binaries-${GITHUB_REF#refs/*/}-x64.zip
echo "BINARIES_ZIP_NAME=$BINARIES_ZIP_NAME" >> $GITHUB_ENV
zip -j $BINARIES_ZIP_NAME Windows-Artifacts/* Linux-Artifacts/*
.github/workflows/lint.yml
+1 -1
View File
@@ -3,7 +3,7 @@ name: Lint
on:
push:
branches:
- main
- master
pull_request:
jobs:
.gitignore
-1
View File
@@ -12,4 +12,3 @@ dependencies
.qtc_clangd
bazel-*
user.bazelrc
*.pyc
README.md
+128 -200
View File
@@ -1,8 +1,8 @@
# CDC File Transfer
Born from the ashes of Stadia, this repository contains tools for syncing and
streaming files from Windows to Windows or Linux. The tools are based on Content
Defined Chunking (CDC), in particular
Born from the ashes of Stadia, this repository contains tools for synching and
streaming files from Windows to Linux. They are based on Content Defined
Chunking (CDC), in particular
[FastCDC](https://www.usenix.org/conference/atc16/technical-sessions/presentation/xia),
to split up files into chunks.
@@ -38,103 +38,27 @@ version of the files available in the target directory.
</p>
The remote diffing algorithm is based on CDC. In our tests, it is up to 30x
faster than the one used in `rsync` (1500 MB/s vs 50 MB/s).
faster than the one used in rsync (1500 MB/s vs 50 MB/s).
The following chart shows a comparison of `cdc_rsync` and Linux `rsync` running
The following chart shows a comparison of `cdc_rsync` and Linux rsync running
under Cygwin on Windows. The test data consists of 58 development builds
of some game provided to us for evaluation purposes. The builds are 40-45 GB
large. For this experiment, we uploaded the first build, then synced the second
build with each of the two tools and measured the time. For example, syncing
from build 1 to build 2 took 210 seconds with the Cygwin `rsync`, but only 75
from build 1 to build 2 took 210 seconds with the Linux rsync, but only 75
seconds with `cdc_rsync`. The three outliers are probably feature drops from
another development branch, where the delta was much higher. Overall,
`cdc_rsync` syncs files about **3 times faster** than Cygwin `rsync`.
`cdc_rsync` syncs files about **3 times faster** than Linux rsync.
<p align="center">
<img src="docs/cdc_rsync_vs_cygwin_rsync.png" alt="Comparison of cdc_rsync and Linux rsync running in Cygwin" width="753" />
</p>
We also ran the experiment with the native Linux `rsync`, i.e syncing Linux to
Linux, to rule out issues with Cygwin. Linux `rsync` performed on average 35%
worse than Cygwin `rsync`, which can be attributed to CPU differences. We did
not include it in the figure because of this, but you can find it
[here](docs/cdc_rsync_vs_cygwin_rsync_vs_linux_rsync.png).
### How does it work and why is it faster?
The standard Linux `rsync` splits a file into fixed-size chunks of typically
several KB.
<p align="center">
<img src="docs/fixed_size_chunks.png" alt="Linux rsync uses fixed size chunks" width="258" />
</p>
If the file is modified in the middle, e.g. by inserting `xxxx` after `567`,
this usually means that <span style="color: red">the modified chunks as well as
all subsequent chunks</span> change.
<p align="center">
<img src="docs/fixed_size_chunks_inserted.png" alt="Fixed size chunks after inserting data" width="301" />
</p>
The standard `rsync` algorithm hashes the chunks of the remote "old" file
and sends the hashes to the local device. The local device then figures out
which parts of the "new" file matches known chunks.
<p align="center">
<img src="docs/linux_rsync_animation.gif" alt="Syncing a file with the standard Linux rsync" width="855" />
<br>
Standard rsync algorithm
</p>
This is a simplification. The actual algorithm is more complicated and uses
two hashes, a weak rolling hash and a strong hash, see
[here](https://rsync.samba.org/tech_report/) for a great overview. What makes
`rsync` relatively slow is the "no match" situation where the rolling hash does
not match any remote hash, and the algorithm has to roll the hash forward and
perform a hash map lookup for each byte. `rsync` goes to
[great lengths](https://github.com/librsync/librsync/blob/master/src/hashtable.h)
optimizing lookups.
`cdc_rsync` does not use fixed-size chunks, but instead variable-size,
content-defined chunks. That means, chunk boundaries are determined by the
*local content* of the file, in practice a 64 byte sliding window. For more
details, see
[the FastCDC paper](https://www.usenix.org/conference/atc16/technical-sessions/presentation/xia)
or take a look at [our implementation](fastcdc/fastcdc.h).
<p align="center">
<img src="docs/variable_size_chunks.png" alt="cdc_rsync uses variable, content-defined size chunks" width="260" />
</p>
If the file is modified in the middle, only <span style="color: red">the modified
chunks</span>, but not <span style="color: #38761d">subsequent chunks</span>
change (unless they are less than 64 bytes away from the modifications).
<p align="center">
<img src="docs/variable_size_chunks_inserted.png" alt="Content-defined chunks after inserting data" width="314" />
</p>
Computing the chunk boundaries is cheap and involves only a left-shift, a memory
lookup, an `add` and an `and` operation for each input byte. This is cheaper
than the hash map lookup for the standard `rsync` algorithm.
Because of this, the `cdc_rsync` algorithm is faster than the standard
`rsync`. It is also simpler. Since chunk boundaries move along with insertions
or deletions, the task to match local and remote hashes is a trivial set
difference operation. It does not involve a per-byte hash map lookup.
<p align="center">
<img src="docs/cdc_rsync_animation.gif" alt="Syncing a file with cdc_rsync" width="857" />
<br>
cdc_rsync algorithm
</p>
## CDC Stream
`cdc_stream` is a tool to stream files and directories from a Windows machine to
a Linux device. Conceptually, it is similar to
[sshfs](https://github.com/libfuse/sshfs), but it is optimized for read speed.
`cdc_stream` is a tool to stream files and directories from a Windows machine to a
Linux device. Conceptually, it is similar to [sshfs](https://github.com/libfuse/sshfs),
but it is optimized for read speed.
* It caches streamed data on the Linux device.
* If a file is re-read on Linux after it changed on Windows, only the
differences are streamed again. The rest is read from the cache.
@@ -161,48 +85,18 @@ In one case, the game is streamed via `sshfs`, in the other case we use
<img src="docs/cdc_stream_vs_sshfs.png" alt="Comparison of cdc_stream and sshfs" width="752" />
</p>
# Supported Platforms
| `cdc_rsync` | From | To |
|:-----------------------------|:--------------------:|:--------------------:|
| Windows x86_64 | &check; | &check; <sup>1</sup> |
| Ubuntu 22.04 x86_64 | &cross; <sup>2</sup> | &check; |
| Ubuntu 22.04 aarch64 | &cross; | &cross; |
| macOS 13 x86_64 <sup>3</sup> | &cross; | &cross; |
| macOS 13 aarch64 <sup>3</sup>| &cross; | &cross; |
| `cdc_stream` | From | To |
|:-----------------------------|:--------------------:|:--------------------:|
| Windows x86_64 | &check; | &cross; |
| Ubuntu 22.04 x86_64 | &cross; | &check; |
| Ubuntu 22.04 aarch64 | &cross; | &cross; |
| macOS 13 x86_64 <sup>3</sup> | &cross; | &cross; |
| macOS 13 aarch64 <sup>3</sup>| &cross; | &cross; |
<span style="font-size: 0.8rem">
<sup>1</sup> Only local syncs, e.g. `cdc_rsync C:\src\* C:\dst`. Support for
remote syncs is being added, see
[#61](https://github.com/google/cdc-file-transfer/issues/61).
<sup>2</sup> See [#56](https://github.com/google/cdc-file-transfer/issues/56).
<sup>3</sup> See [#62](https://github.com/google/cdc-file-transfer/issues/62).
</span>
# Getting Started
Download the precompiled binaries from the
[latest release](https://github.com/google/cdc-file-transfer/releases) to a
Windows device and unzip them. The Linux binaries are automatically deployed
to `~/.cache/cdc-file-transfer` by the Windows tools. There is no need to manually
deploy them. We currently provide Linux binaries compiled on
[latest release](https://github.com/google/cdc-file-transfer/releases).
We currently provide Linux binaries compiled on
[Github's latest Ubuntu](https://github.com/actions/runner-images) version.
If the binaries work for you, you can skip the following two sections.
Alternatively, the project can be built from source. Some binaries have to be
built on Windows, some on Linux.
## Prerequisites for Building
## Prerequisites
To build the tools from source, the following steps have to be executed on
**both Windows and Linux**.
@@ -220,96 +114,148 @@ To build the tools from source, the following steps have to be executed on
git submodule update --init --recursive
```
Finally, install an SSH client on the Windows machine if not present.
The file transfer tools require `ssh.exe` and `sftp.exe`.
Finally, install an SSH client on the Windows device if not present.
The file transfer tools require `ssh.exe` and `scp.exe`.
## Building
The two tools CDC RSync and CDC Stream can be built and used independently.
The two tools can be built and used independently.
### CDC RSync
* On a Linux device, build the Linux components
* Build Linux components
```
bazel build --config linux --compilation_mode=opt --linkopt=-Wl,--strip-all --copt=-fdata-sections --copt=-ffunction-sections --linkopt=-Wl,--gc-sections //cdc_rsync_server
```
* On a Windows device, build the Windows components
* Build Windows components
```
bazel build --config windows --compilation_mode=opt --copt=/GL //cdc_rsync
```
* Copy the Linux build output file `cdc_rsync_server` from
`bazel-bin/cdc_rsync_server` to `bazel-bin\cdc_rsync` on the Windows machine.
`bazel-bin/cdc_rsync_server` on the Linux system to `bazel-bin\cdc_rsync`
on the Windows machine.
### CDC Stream
* On a Linux device, build the Linux components
* Build Linux components
```
bazel build --config linux --compilation_mode=opt --linkopt=-Wl,--strip-all --copt=-fdata-sections --copt=-ffunction-sections --linkopt=-Wl,--gc-sections //cdc_fuse_fs
```
* On a Windows device, build the Windows components
* Build Windows components
```
bazel build --config windows --compilation_mode=opt --copt=/GL //cdc_stream
```
* Copy the Linux build output files `cdc_fuse_fs` and `libfuse.so` from
`bazel-bin/cdc_fuse_fs` to `bazel-bin\cdc_stream` on the Windows machine.
`bazel-bin/cdc_fuse_fs` on the Linux system to `bazel-bin\cdc_stream`
on the Windows machine.
## Usage
The tools require a setup where you can use SSH and SFTP from the Windows
machine to the Linux device without entering a password, e.g. by using key-based
The tools require a setup where you can use SSH and SCP from the Windows machine
to the Linux device without entering a password, e.g. by using key-based
authentication.
### Configuring SSH and SFTP
### Configuring SSH and SCP
By default, the tools search `ssh.exe` and `sftp.exe` from the path environment
By default, the tools search `ssh.exe` and `scp.exe` from the path environment
variable. If you can run the following commands in a Windows cmd without
entering your password, you are all set:
```
ssh user@linux.device.com
sftp user@linux.device.com
scp somefile.txt user@linux.device.com:
```
Here, `user` is the Linux user and `linux.device.com` is the Linux host to
SSH into or copy the file to.
If additional arguments are required, it is recommended to provide an SSH config
file. By default, both `ssh.exe` and `sftp.exe` use the file at
`%USERPROFILE%\.ssh\config` on Windows, if it exists. A possible config file
that sets a username, a port, an identity file and a known host file could look
as follows:
If `ssh.exe` or `scp.exe` cannot be found, or if additional arguments are
required, it is recommended to set the environment variables `CDC_SSH_COMMAND`
and `CDC_SCP_COMMAND`. The following example specifies a custom path to the SSH
and SCP binaries, a custom SSH config file, a key file and a known hosts file:
```
Host linux_device
HostName linux.device.com
User user
Port 12345
IdentityFile C:\path\to\id_rsa
UserKnownHostsFile C:\path\to\known_hosts
set CDC_SSH_COMMAND="C:\path with space\to\ssh.exe" -F C:\path\to\ssh_config -i C:\path\to\id_rsa -oStrictHostKeyChecking=yes -oUserKnownHostsFile="""C:\path\to\known_hosts"""
set CDC_SCP_COMMAND="C:\path with space\to\scp.exe" -F C:\path\to\ssh_config -i C:\path\to\id_rsa -oStrictHostKeyChecking=yes -oUserKnownHostsFile="""C:\path\to\known_hosts"""
```
If `ssh.exe` or `sftp.exe` cannot be found, you can specify the full paths via
the command line arguments `--ssh-command` and `--sftp-command` for `cdc_rsync`
and `cdc_stream start` (see below), or set the environment variables
`CDC_SSH_COMMAND` and `CDC_SFTP_COMMAND`, e.g.
```
set CDC_SSH_COMMAND="C:\path with space\to\ssh.exe"
set CDC_SFTP_COMMAND="C:\path with space\to\sftp.exe"
```
Note that you can also specify SSH configuration via the environment variables
instead of using a config file:
```
set CDC_SSH_COMMAND=C:\path\to\ssh.exe -p 12345 -i C:\path\to\id_rsa -oUserKnownHostsFile=C:\path\to\known_hosts
set CDC_SFTP_COMMAND=C:\path\to\sftp.exe -P 12345 -i C:\path\to\id_rsa -oUserKnownHostsFile=C:\path\to\known_hosts
```
Note the small `-p` for `ssh.exe` and the capital `-P` for `sftp.exe`.
#### Google Specific
For Google internal usage, set the following environment variables to enable SSH
authentication using a Google security key:
For Google internal usage, there are two setups, a simple one that requires
touching the security key for every remote access, and a slightly more complex
one that tunnels all remote access through an SSH tunnel and only requires one
security key touch to set up the tunnel.
##### Setup Requiring Security Key Touches
Set the following environment variables to enable SSH authentication using a
Google security key:
```
set CDC_SSH_COMMAND=C:\gnubby\bin\ssh.exe
set CDC_SFTP_COMMAND=C:\gnubby\bin\sftp.exe
set CDC_SCP_COMMAND=C:\gnubby\bin\scp.exe
```
Note that you will have to touch the security key multiple times during the
first run. Subsequent runs only require a single touch.
first run of each tool as the Linux components have to be deployed first.
Subsequent runs only require a single touch.
##### Setup Not Requiring Security Key Touches
This section explains how to run the tools without security key touch for every
action.
On Linux, generate a host key pair with
```
mkdir ~/sshd_runner
ssh-keygen -f ~/sshd_runner/ssh_host_ed25519_key -N '' -t ed25519
```
On Windows, create `%USERPROFILE%\sshd_runner\known_hosts` and copy the public
key from `~/sshd_runner/ssh_host_ed25519_key.pub` to it, using `localhost` and
port `50000` (if the port is in use, use another port). The file should look
similar to
```
[localhost]:50000 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOR1EInlod0Pm853Ew0p4eXaD3V8y9njf6EoyaPfc8ln
```
Next, generate an auth key for the Windows machine with
```
ssh-keygen -f %USERPROFILE%\sshd_runner\id_ed25519 -N "" -t ed25519
```
Note the subtle difference `-N ''` vs `-N ""`.
On Linux, create `~/sshd_runner/authorized_keys` and copy the public key from
`%USERPROFILE%\sshd_runner\id_ed25519.pub` to it. The file should look similar
to
```
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKleiT/yRhZ9aMutOJ8XaAYX0SPUZHuS50HP4QK5kBft google\user@user-w
```
On Windows, open an SSH tunnel to the Linux machine through port 50000.
This should require one gnubby touch.
```
C:\gnubby\bin\ssh.exe -N -L50000:localhost:50000 <user>.<location>.corp.google.com
```
On Linux, run the SSH server with
```
/usr/sbin/sshd -D -p 50000 -oHostKey=~/sshd_runner/ssh_host_ed25519_key -oAuthorizedKeysFile=~/sshd_runner/authorized_keys -oListenAddress=localhost:50000
```
It runs as user, not as root, and only accepts connections from localhost
through port 50000, so it's secure.
Now you should be able to run SSH on the Windows machine into the Linux device
without security key touch
```
C:\Windows\System32\OpenSSH\ssh.exe -p 50000 -i %USERPROFILE%\sshd_runner\id_ed25519 -oUserKnownHostsFile=%USERPROFILE%\sshd_runner\known_hosts user@localhost
```
Next, set `CDC_SSH_COMMAND` and `CDC_SCP_COMMAND` to use the proper flags
```
set CDC_SSH_COMMAND=C:\Windows\System32\OpenSSH\ssh.exe -i %USERPROFILE%\sshd_runner\id_ed25519 -oUserKnownHostsFile=%USERPROFILE%\sshd_runner\known_hosts -p 50000
set CDC_SCP_COMMAND=C:\Windows\System32\OpenSSH\scp.exe -i %USERPROFILE%\sshd_runner\id_ed25519 -oUserKnownHostsFile=%USERPROFILE%\sshd_runner\known_hosts -P 50000
```
Note the small `-p` for SSH and the capital `-P` for SCP.
Now the file transfer tools should work without security key touch, but be sure
to use `localhost` as target host as we're piping the data through the tunnel.
```
cdc_rsync C:\assets\* user@localhost:~/assets -vr
cdc_stream start C:\assets user@localhost:~/assets
```
### CDC RSync
@@ -332,19 +278,23 @@ To get per file progress, add `-v`:
```
cdc_rsync C:\path\to\assets\* user@linux.device.com:~/assets -vr
```
The tool also supports local syncs:
```
cdc_rsync C:\path\to\assets\* C:\path\to\destination -vr
```
### CDC Stream
`cdc_stream` consists of a background service, which has to be started in
advance with
```
cdc_stream start-service
```
The service logs to `%APPDATA%\cdc-file-transfer\logs` by default. Try
`cdc_stream --help` to get a list of available flags.
To stream the Windows directory `C:\path\to\assets` to `~/assets` on the Linux
device, run
```
cdc_stream start C:\path\to\assets user@linux.device.com:~/assets
```
This makes all files and directories in `C:\path\to\assets` available on
This makes all files and directories of `C:\path\to\assets` available on
`~/assets` immediately, as if it were a local copy. However, data is streamed
from Windows to Linux as files are accessed.
@@ -352,39 +302,17 @@ To stop the streaming session, enter
```
cdc_stream stop user@linux.device.com:~/assets
```
The command also accepts wildcards. For instance,
```
cdc_stream stop user@*:*
```
stops all existing streaming sessions for the given user.
## Troubleshooting
On first run, `cdc_stream` starts a background service, which does all the work.
The `cdc_stream start` and `cdc_stream stop` commands are just RPC clients that
talk to the service.
`cdc_rsync` always logs to the console. By default, the `cdc_stream` service
logs to a timestamped file in `%APPDATA%\cdc-file-transfer\logs`. It can be
switched to log to console by starting it with `--log-to-stdout`:
```
cdc_stream start-service --log_to_stdout
```
The service logs to `%APPDATA%\cdc-file-transfer\logs` by default. The logs are
useful to investigate issues with asset streaming. To pass custom arguments, or
to debug the service, create a JSON config file at
`%APPDATA%\cdc-file-transfer\cdc_stream.json` with command line flags.
For instance,
```
{ "verbosity":3 }
```
instructs the service to log debug messages. Try `cdc_stream start-service -h`
for a list of available flags. Alternatively, run the service manually with
```
cdc_stream start-service
```
and pass the flags as command line arguments. When you run the service manually,
the flag `--log-to-stdout` is particularly useful as it logs to the console
instead of to the file.
`cdc_rsync` always logs to the console. To increase log verbosity, pass `-vvv`
for debug logs or `-vvvv` for verbose logs.
For both sync and stream, the debug logs contain all SSH and SFTP commands that
are attempted to run, which is very useful for troubleshooting. If a command
fails unexpectedly, copy it and run it in isolation. Pass `-vv` or `-vvv` for
additional debug output.
Both `cdc_rsync` and `cdc_stream` support command line flags to control log
verbosity. Passing `-vvv` prints debug logs, `-vvvv` prints verbose logs. The
debug logs contain all SSH and SCP commands that are attempted to run, which is
very useful for troubleshooting.
__init__.py
View File
absl.natvis
-31
View File
@@ -1,31 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<AutoVisualizer xmlns="http://schemas.microsoft.com/vstudio/debugger/natvis/2010">
<Type Name="absl::flat_hash_map&lt;*&gt;">
<DisplayString Condition="size_ == 0">empty</DisplayString>
<DisplayString>{{ size={size_} }}</DisplayString>
<Expand>
<Item Name="[size]" ExcludeView="simple">size_</Item>
<Item Name="[capacity]" ExcludeView="simple">capacity_</Item>
<CustomListItems MaxItemsPerView="5000">
<Variable Name="iSlot" InitialValue="0" />
<Size>size_</Size>
<Loop>
<!-- bool IsFull(ctrl_t c) { return c >= 0; } -->
<If Condition="ctrl_[iSlot] &gt;= 0">
<Item>slots_[iSlot]</Item>
</If>
<Exec>iSlot++</Exec>
<Break Condition="iSlot == capacity_" />
</Loop>
</CustomListItems>
</Expand>
</Type>
<Type Name="absl::container_internal::map_slot_type&lt;*&gt;">
<DisplayString>{value.first}:{value.second}</DisplayString>
<Expand>
<Item Name="[key]" ExcludeView="simple">value.first</Item>
<Item Name="[value]" ExcludeView="simple">value.second</Item>
</Expand>
</Type>
</AutoVisualizer>
all_files.vcxitems
+7 -33
View File
@@ -15,12 +15,8 @@
</ItemGroup>
<ItemGroup>
<ClCompile Include="$(MSBuildThisFileDirectory)absl_helper\jedec_size_flag.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)cdc_fuse_fs\mock_config_stream_client.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync\server_arch.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync\server_arch_test.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)cdc_stream\asset_stream_config.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)cdc_stream\asset_stream_server.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)cdc_stream\background_service_client.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)cdc_stream\background_service_impl.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)cdc_stream\base_command.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)cdc_stream\cdc_fuse_manager.cc" />
@@ -37,12 +33,6 @@
<ClCompile Include="$(MSBuildThisFileDirectory)cdc_stream\session_manager.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)cdc_stream\start_command.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)cdc_stream\start_service_command.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)cdc_stream\stop_service_command.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)fastcdc\fastcdc_test.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)manifest\pending_assets_queue.cc" />
<ClInclude Include="$(MSBuildThisFileDirectory)cdc_fuse_fs\mock_config_stream_client.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync\server_arch.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)cdc_stream\background_service_client.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)cdc_stream\stop_command.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)cdc_stream\testing_asset_stream_server.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)cdc_fuse_fs\asset.cc" />
@@ -55,19 +45,13 @@
<ClCompile Include="$(MSBuildThisFileDirectory)cdc_fuse_fs\mock_libfuse.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)cdc_indexer\indexer.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)cdc_indexer\main.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)common\ansi_filter.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)common\ansi_filter_test.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)common\arch_type.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)common\arch_type_test.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)common\buffer.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)common\buffer_test.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)common\client_socket.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)common\clock.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)common\dir_iter.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)common\dir_iter_test.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)common\errno_mapping.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)common\errno_mapping_test.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)common\fake_socket.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)common\file_watcher_win.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)common\file_watcher_win_test.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)common\gamelet_component.cc" />
@@ -80,8 +64,6 @@
<ClCompile Include="$(MSBuildThisFileDirectory)common\path_test.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)common\port_manager_test.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)common\port_manager_win.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)common\port_range_parser.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)common\port_range_parser_test.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)common\process_test.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)common\process_win.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)common\remote_util.cc" />
@@ -91,8 +73,6 @@
<ClCompile Include="$(MSBuildThisFileDirectory)common\sdk_util_test.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)common\semaphore.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)common\semaphore_test.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)common\server_socket.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)common\socket.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)common\stats_collector.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)common\status.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)common\stopwatch.cc" />
@@ -118,8 +98,10 @@
<ClCompile Include="$(MSBuildThisFileDirectory)data_store\mem_data_store_test.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync\base\cdc_interface.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync\base\cdc_interface_test.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync\base\fake_socket.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync\base\message_pump.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync\base\message_pump_test.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync\client_socket.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync\file_finder_and_sender.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync\file_finder_and_sender_test.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync\cdc_rsync_client.cc" />
@@ -140,6 +122,7 @@
<ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync_server\file_finder_test.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync_server\cdc_rsync_server.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync_server\main.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync_server\server_socket.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)cdc_rsync_server\unzstd_stream.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)manifest\asset_builder.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)manifest\content_id.cc" />
@@ -159,13 +142,6 @@
<ClCompile Include="$(MSBuildThisFileDirectory)metrics\messages.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)metrics\messages_test.cc" />
<ClCompile Include="$(MSBuildThisFileDirectory)metrics\metrics.cc" />
<ClInclude Include="$(MSBuildThisFileDirectory)cdc_stream\stop_service_command.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)common\ansi_filter.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)common\arch_type.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)common\build_version.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)common\port_range_parser.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)fastcdc\fastcdc.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)manifest\pending_assets_queue.h" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="$(MSBuildThisFileDirectory)absl_helper\jedec_size_flag.h" />
@@ -195,11 +171,9 @@
<ClInclude Include="$(MSBuildThisFileDirectory)cdc_fuse_fs\mock_libfuse.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)cdc_indexer\indexer.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)common\buffer.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)common\client_socket.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)common\clock.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)common\dir_iter.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)common\errno_mapping.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)common\fake_socket.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)common\file_watcher_win.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)common\gamelet_component.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)common\grpc_status.h" />
@@ -213,8 +187,6 @@
<ClInclude Include="$(MSBuildThisFileDirectory)common\scoped_handle_win.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)common\sdk_util.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)common\semaphore.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)common\server_socket.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)common\socket.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)common\stats_collector.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)common\status.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)common\status_macros.h" />
@@ -234,9 +206,12 @@
<ClInclude Include="$(MSBuildThisFileDirectory)data_store\grpc_reader.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)data_store\mem_data_store.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync\base\cdc_interface.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync\base\fake_socket.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync\base\message_pump.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync\base\server_exit_code.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync\base\socket.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync\client_file_info.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync\client_socket.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync\file_finder_and_sender.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync\cdc_rsync_client.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync\parallel_file_opener.h" />
@@ -248,6 +223,7 @@
<ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync_server\file_finder.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync_server\file_info.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync_server\cdc_rsync_server.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync_server\server_socket.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)cdc_rsync_server\unzstd_stream.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)manifest\asset_builder.h" />
<ClInclude Include="$(MSBuildThisFileDirectory)manifest\content_id.h" />
@@ -281,7 +257,6 @@
<None Include="$(MSBuildThisFileDirectory)cdc_rsync\README.md" />
<None Include="$(MSBuildThisFileDirectory)cdc_rsync_server\BUILD" />
<None Include="$(MSBuildThisFileDirectory)cdc_rsync_tests\BUILD" />
<None Include="$(MSBuildThisFileDirectory)fastcdc\BUILD" />
<None Include="$(MSBuildThisFileDirectory)manifest\BUILD" />
<None Include="$(MSBuildThisFileDirectory)manifest_cli\BUILD" />
<None Include="$(MSBuildThisFileDirectory)metrics\BUILD" />
@@ -301,7 +276,6 @@
<None Include="$(MSBuildThisFileDirectory)WORKSPACE" />
</ItemGroup>
<ItemGroup>
<Natvis Include="$(MSBuildThisFileDirectory)absl.natvis" />
<Natvis Include="$(MSBuildThisFileDirectory)manifest.natvis" />
<Natvis Include="$(MSBuildThisFileDirectory)protobuf.natvis" />
</ItemGroup>
cdc_fuse_fs/BUILD
-2
View File
@@ -9,10 +9,8 @@ cc_binary(
":cdc_fuse_fs_lib",
":constants",
"//absl_helper:jedec_size_flag",
"//common:client_socket",
"//common:gamelet_component",
"//common:log",
"//common:server_socket",
"//data_store:data_provider",
"//data_store:disk_data_store",
"//data_store:grpc_reader",
cdc_fuse_fs/constants.h
+5 -12
View File
@@ -19,22 +19,15 @@
namespace cdc_ft {
// FUSE prints
// Port 12345 cdc_fuse_fs is up-to-date
// to stdout when its version matches the version (=build version or
// size/timestamp for DEV builds) on the local device. The port is the gRPC port
// that FUSE will try to connect to.
static constexpr char kFusePortPrefix[] = "Port ";
// FUSE prints this to stdout when the binary timestamp and file size match the
// file on the workstation.
static constexpr char kFuseUpToDate[] = "cdc_fuse_fs is up-to-date";
// FUSE prints this to stdout when its version does not match the version on the
// local device. It indicates that the binary has to be redeployed.
// FUSE prints this to stdout when the binary timestamp or file size does not
// match the file on the workstation. It indicates that the binary has to be
// redeployed.
static constexpr char kFuseNotUpToDate[] = "cdc_fuse_fs is not up-to-date";
// FUSE prints this to stdout when it can connect to its port. This means that
// port forwarding has finished setting up, and startup is finished.
static constexpr char kFuseConnected[] = "cdc_fuse_fs is connected";
} // namespace cdc_ft
#endif // CDC_FUSE_FS_CONSTANTS_H_
cdc_fuse_fs/main.cc
+3 -28
View File
@@ -21,11 +21,9 @@
#include "cdc_fuse_fs/cdc_fuse_fs.h"
#include "cdc_fuse_fs/config_stream_client.h"
#include "cdc_fuse_fs/constants.h"
#include "common/client_socket.h"
#include "common/gamelet_component.h"
#include "common/log.h"
#include "common/path.h"
#include "common/server_socket.h"
#include "data_store/data_provider.h"
#include "data_store/disk_data_store.h"
#include "data_store/grpc_reader.h"
@@ -39,8 +37,6 @@ namespace {
constexpr char kFuseFilename[] = "cdc_fuse_fs";
constexpr char kLibFuseFilename[] = "libfuse.so";
constexpr absl::Duration kConnectionTimeout = absl::Seconds(60);
bool IsUpToDate(const std::string& components_arg) {
// Components are expected to reside in the same dir as the executable.
std::string component_dir;
@@ -111,6 +107,7 @@ ABSL_FLAG(
"Whitespace-separated triples filename, size and timestamp of the "
"workstation version of this binary and dependencies. Used for a fast "
"up-to-date check.");
ABSL_FLAG(uint16_t, port, 0, "Port to connect to on localhost");
ABSL_FLAG(cdc_ft::JedecSize, prefetch_size, cdc_ft::JedecSize(512 << 10),
"Additional data to request from the server when a FUSE read of "
"maximum size is detected. This amount is added to the original "
@@ -141,6 +138,7 @@ int main(int argc, char* argv[]) {
std::vector<char*> mount_args = absl::ParseCommandLine(argc, argv);
std::string instance = absl::GetFlag(FLAGS_instance);
std::string components = absl::GetFlag(FLAGS_components);
uint16_t port = absl::GetFlag(FLAGS_port);
std::string cache_dir = absl::GetFlag(FLAGS_cache_dir);
int cache_dir_levels = absl::GetFlag(FLAGS_cache_dir_levels);
int verbosity = absl::GetFlag(FLAGS_verbosity);
@@ -161,18 +159,7 @@ int main(int argc, char* argv[]) {
printf("%s\n", cdc_ft::kFuseNotUpToDate);
return 0;
}
// Find an available port.
absl::StatusOr<int> port_or = cdc_ft::ServerSocket::FindAvailablePort();
if (!port_or.ok()) {
LOG_ERROR("Failed to find available port: %s\n",
port_or.status().ToString());
return 1;
}
int port = *port_or;
// Write marker for the server.
printf("%s%i%s\n", cdc_ft::kFusePortPrefix, port, cdc_ft::kFuseUpToDate);
printf("%s\n", cdc_ft::kFuseUpToDate);
fflush(stdout);
// Create mount dir if it doesn't exist yet.
@@ -202,18 +189,6 @@ int main(int argc, char* argv[]) {
store.value()->SetCapacity(cache_capacity);
LOG_INFO("Caching chunks in '%s'", store.value()->RootDir());
// Wait for port forwarding to listen to |port|.
status =
cdc_ft::ClientSocket::WaitForConnection(port, cdc_ft::kConnectionTimeout);
if (!status.ok()) {
LOG_ERROR("Failed to connect to port %i: %s", port, status.ToString());
return static_cast<int>(status.code());
}
// Write another marker for the server.
printf("%s\n", cdc_ft::kFuseConnected);
fflush(stdout);
// Start a gRpc client.
std::string client_address = absl::StrFormat("localhost:%u", port);
grpc::ChannelArguments channel_args;
cdc_indexer/README.md
+2 -2
View File
@@ -14,7 +14,7 @@ experimentation. See the file `indexer.h` for preprocessor macros that can be
enabled, for example:
```
bazel build -c opt --copt=-DCDC_GEAR_BITS=32 //cdc_indexer
bazel build -c opt --copt=-DCDC_GEAR_TABLE=1 //cdc_indexer
```
At the end of the operation, the indexer outputs a summary of the results such
@@ -25,7 +25,7 @@ as the following:
Operation succeeded.
Chunk size (min/avg/max): 128 KB / 256 KB / 1024 KB | Threads: 12
gear_table: 64 bit | threshold: 0x7fffc0001fff
gear_table: 64 bit | mask_s: 0x49249249249249 | mask_l: 0x1249249249
Duration: 00:03
Total files: 2
Total chunks: 39203
cdc_indexer/indexer.cc
+4 -2
View File
@@ -140,7 +140,8 @@ Indexer::Impl::Impl(const IndexerConfig& cfg,
fastcdc::Config ccfg(cfg_.min_chunk_size, cfg_.avg_chunk_size,
cfg_.max_chunk_size);
Indexer::Chunker chunker(ccfg, nullptr);
cfg_.threshold = chunker.Threshold();
cfg_.mask_s = chunker.Stage(0).mask;
cfg_.mask_l = chunker.Stage(chunker.StagesCount() - 1).mask;
// Collect inputs.
for (auto it = inputs.begin(); it != inputs.end(); ++it) {
inputs_.push(*it);
@@ -367,7 +368,8 @@ IndexerConfig::IndexerConfig()
max_chunk_size(0),
max_chunk_size_step(0),
num_threads(0),
threshold(0) {}
mask_s(0),
mask_l(0) {}
Indexer::Indexer() : impl_(nullptr) {}
cdc_indexer/indexer.h
+22 -13
View File
@@ -27,10 +27,16 @@
#include "fastcdc/fastcdc.h"
// Compile-time parameters for the FastCDC algorithm.
#define CDC_GEAR_32BIT 32
#define CDC_GEAR_64BIT 64
#ifndef CDC_GEAR_BITS
#define CDC_GEAR_BITS CDC_GEAR_64BIT
#define CDC_GEAR_32BIT 1
#define CDC_GEAR_64BIT 2
#ifndef CDC_GEAR_TABLE
#define CDC_GEAR_TABLE CDC_GEAR_64BIT
#endif
#ifndef CDC_MASK_STAGES
#define CDC_MASK_STAGES 7
#endif
#ifndef CDC_MASK_BIT_LSHIFT_AMOUNT
#define CDC_MASK_BIT_LSHIFT_AMOUNT 3
#endif
namespace cdc_ft {
@@ -60,20 +66,23 @@ struct IndexerConfig {
uint32_t num_threads;
// Which hash function to use.
HashType hash_type;
// The threshold will be populated by the indexer, setting it here has no
// effect. It is in this struct so that it can be conveniently accessed
// when printing the operation summary (and since it is derived from the
// configuration, it is technically part of it).
uint64_t threshold;
// The masks will be populated by the indexer, setting them here has no
// effect. They are in this struct so that they can be conveniently accessed
// when printing the operation summary (and since they are derived from the
// configuration, they are technically part of it).
uint64_t mask_s;
uint64_t mask_l;
};
class Indexer {
public:
using hash_t = std::string;
#if CDC_GEAR_BITS == CDC_GEAR_32BIT
typedef fastcdc::Chunker32<> Chunker;
#elif CDC_GEAR_BITS == CDC_GEAR_64BIT
typedef fastcdc::Chunker64<> Chunker;
#if CDC_GEAR_TABLE == CDC_GEAR_32BIT
typedef fastcdc::Chunker32<CDC_MASK_STAGES, CDC_MASK_BIT_LSHIFT_AMOUNT>
Chunker;
#elif CDC_GEAR_TABLE == CDC_GEAR_64BIT
typedef fastcdc::Chunker64<CDC_MASK_STAGES, CDC_MASK_BIT_LSHIFT_AMOUNT>
Chunker;
#else
#error "Unknown gear table"
#endif
cdc_indexer/main.cc
+9 -7
View File
@@ -64,9 +64,9 @@ namespace {
const char* GearTable() {
// The following macros are defined in indexer.h.
#if CDC_GEAR_BITS == CDC_GEAR_32BIT
#if CDC_GEAR_TABLE == CDC_GEAR_32BIT
return "32 bit";
#elif CDC_GEAR_BITS == CDC_GEAR_64BIT
#elif CDC_GEAR_TABLE == CDC_GEAR_64BIT
return "64 bit";
#else
#error "Unknown gear table"
@@ -165,8 +165,9 @@ void ShowSummary(const IndexerConfig& cfg, const Indexer::OpStats& stats,
<< HumanBytes(cfg.max_chunk_size)
<< " | Hash: " << HashTypeToString(cfg.hash_type)
<< " | Threads: " << cfg.num_threads << std::endl;
std::cout << "gear_table: " << GearTable() << " | threshold: 0x" << std::hex
<< cfg.threshold << std::dec << std::endl;
std::cout << "gear_table: " << GearTable() << " | mask_s: 0x" << std::hex
<< cfg.mask_s << " | mask_l: 0x" << cfg.mask_l << std::dec
<< std::endl;
std::cout << std::setw(title_w) << "Duration:" << std::setw(num_w)
<< HumanDuration(elapsed) << std::endl;
std::cout << std::setw(title_w) << "Total files:" << std::setw(num_w)
@@ -278,10 +279,11 @@ absl::Status WriteResultsFile(const std::string& filepath,
path::FileCloser closer(fout);
static constexpr int num_columns = 14;
static constexpr int num_columns = 15;
static const char* columns[num_columns] = {
"gear_table",
"threshold",
"mask_s",
"mask_l",
"Min chunk size [KiB]",
"Avg chunk size [KiB]",
"Max chunk size [KiB]",
@@ -330,7 +332,7 @@ absl::Status WriteResultsFile(const std::string& filepath,
// Write user-supplied description
if (!description.empty()) std::fprintf(fout, "%s,", description.c_str());
// Write chunking params.
std::fprintf(fout, "%s,0x%zx,", GearTable(), cfg.threshold);
std::fprintf(fout, "%s,0x%zx,0x%zx,", GearTable(), cfg.mask_s, cfg.mask_l);
std::fprintf(fout, "%zu,%zu,%zu,", cfg.min_chunk_size >> 10,
cfg.avg_chunk_size >> 10, cfg.max_chunk_size >> 10);
// Write speed, files, chunks.
cdc_rsync/BUILD
+25 -33
View File
@@ -18,6 +18,19 @@ cc_library(
hdrs = ["client_file_info.h"],
)
cc_library(
name = "client_socket",
srcs = ["client_socket.cc"],
hdrs = ["client_socket.h"],
target_compatible_with = ["@platforms//os:windows"],
deps = [
"//cdc_rsync/base:socket",
"//common:log",
"//common:status",
"//common:util",
],
)
cc_library(
name = "file_finder_and_sender",
srcs = ["file_finder_and_sender.cc"],
@@ -41,8 +54,8 @@ cc_test(
data = ["testdata/root.txt"] + glob(["testdata/file_finder_and_sender/**"]),
deps = [
":file_finder_and_sender",
"//cdc_rsync/base:fake_socket",
"//cdc_rsync/protos:messages_cc_proto",
"//common:fake_socket",
"//common:status_test_macros",
"//common:test_main",
"@com_google_googletest//:gtest",
@@ -54,27 +67,32 @@ cc_library(
name = "cdc_rsync_client",
srcs = ["cdc_rsync_client.cc"],
hdrs = ["cdc_rsync_client.h"],
linkopts = select({
"//tools:windows": [
"/DEFAULTLIB:Ws2_32.lib", # Sockets, e.g. recv, send, WSA*.
],
"//conditions:default": [],
}),
target_compatible_with = ["@platforms//os:windows"],
deps = [
":client_socket",
":file_finder_and_sender",
":parallel_file_opener",
":progress_tracker",
":server_arch",
":zstd_stream",
"//cdc_rsync/base:cdc_interface",
"//cdc_rsync/base:message_pump",
"//cdc_rsync/base:server_exit_code",
"//cdc_rsync/base:socket",
"//cdc_rsync/protos:messages_cc_proto",
"//common:client_socket",
"//common:gamelet_component",
"//common:log",
"//common:path",
"//common:path_filter",
"//common:platform",
"//common:port_manager",
"//common:process",
"//common:remote_util",
"//common:server_socket",
"//common:socket",
"//common:status",
"//common:status_macros",
"//common:threadpool",
@@ -112,8 +130,6 @@ cc_library(
hdrs = ["params.h"],
deps = [
":cdc_rsync_client",
"//common:build_version",
"//common:port_range_parser",
"@com_github_zstd//:zstd",
"@com_google_absl//absl/status",
],
@@ -155,37 +171,13 @@ cc_test(
],
)
cc_library(
name = "server_arch",
srcs = ["server_arch.cc"],
hdrs = ["server_arch.h"],
deps = [
"//common:ansi_filter",
"//common:arch_type",
"//common:path",
"//common:remote_util",
"@com_google_absl//absl/strings",
],
)
cc_test(
name = "server_arch_test",
srcs = ["server_arch_test.cc"],
deps = [
":server_arch",
"//common:test_main",
"@com_google_absl//absl/strings",
"@com_google_googletest//:gtest",
],
)
cc_library(
name = "zstd_stream",
srcs = ["zstd_stream.cc"],
hdrs = ["zstd_stream.h"],
deps = [
":client_socket",
"//common:buffer",
"//common:client_socket",
"//common:status",
"//common:status_macros",
"//common:stopwatch",
@@ -198,8 +190,8 @@ cc_test(
srcs = ["zstd_stream_test.cc"],
deps = [
":zstd_stream",
"//cdc_rsync/base:fake_socket",
"//cdc_rsync_server:unzstd_stream",
"//common:fake_socket",
"//common:status_test_macros",
"//common:test_main",
"@com_github_zstd//:zstd",
cdc_rsync/base/BUILD
+18 -3
View File
@@ -28,21 +28,31 @@ cc_test(
data = ["testdata/root.txt"] + glob(["testdata/cdc_interface/**"]),
deps = [
":cdc_interface",
"//common:fake_socket",
":fake_socket",
"//common:status_test_macros",
"//common:test_main",
"@com_google_googletest//:gtest",
],
)
cc_library(
name = "fake_socket",
srcs = ["fake_socket.cc"],
hdrs = ["fake_socket.h"],
deps = [
"//cdc_rsync/base:socket",
"@com_google_absl//absl/status",
],
)
cc_library(
name = "message_pump",
srcs = ["message_pump.cc"],
hdrs = ["message_pump.h"],
deps = [
":socket",
"//common:buffer",
"//common:log",
"//common:socket",
"//common:status",
"@com_google_absl//absl/status",
"@com_google_absl//absl/strings:str_format",
@@ -54,9 +64,9 @@ cc_test(
name = "message_pump_test",
srcs = ["message_pump_test.cc"],
deps = [
":fake_socket",
":message_pump",
"//cdc_rsync/protos:messages_cc_proto",
"//common:fake_socket",
"//common:status_test_macros",
"//common:test_main",
"@com_google_googletest//:gtest",
@@ -68,6 +78,11 @@ cc_library(
hdrs = ["server_exit_code.h"],
)
cc_library(
name = "socket",
hdrs = ["socket.h"],
)
filegroup(
name = "all_test_sources",
srcs = glob(["*_test.cc"]),
cdc_rsync/base/cdc_interface_test.cc
+1 -1
View File
@@ -17,8 +17,8 @@
#include <cstdio>
#include <fstream>
#include "cdc_rsync/base/fake_socket.h"
#include "cdc_rsync/base/message_pump.h"
#include "common/fake_socket.h"
#include "common/log.h"
#include "common/path.h"
#include "common/status_test_macros.h"
common/fake_socket.cc → cdc_rsync/base/fake_socket.cc
+2 -3
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/fake_socket.h"
#include "cdc_rsync/base/fake_socket.h"
namespace cdc_ft {
@@ -39,8 +39,7 @@ absl::Status FakeSocket::Receive(void* buffer, size_t size,
*bytes_received = 0;
std::unique_lock<std::mutex> lock(data_mutex_);
data_cv_.wait(lock, [this, size, allow_partial_read]() {
size_t min_size = allow_partial_read ? 1 : size;
return data_.size() >= min_size || shutdown_;
return allow_partial_read || data_.size() >= size || shutdown_;
});
if (shutdown_) {
return absl::UnavailableError("Pipe is shut down");
common/fake_socket.h → cdc_rsync/base/fake_socket.h
+3 -3
View File
@@ -14,14 +14,14 @@
* limitations under the License.
*/
#ifndef COMMON_FAKE_SOCKET_H_
#define COMMON_FAKE_SOCKET_H_
#ifndef CDC_RSYNC_BASE_FAKE_SOCKET_H_
#define CDC_RSYNC_BASE_FAKE_SOCKET_H_
#include <condition_variable>
#include <mutex>
#include "absl/status/status.h"
#include "common/socket.h"
#include "cdc_rsync/base/socket.h"
namespace cdc_ft {
cdc_rsync/base/message_pump.cc
+1 -1
View File
@@ -16,9 +16,9 @@
#include "absl/status/status.h"
#include "absl/strings/str_format.h"
#include "cdc_rsync/base/socket.h"
#include "common/buffer.h"
#include "common/log.h"
#include "common/socket.h"
#include "common/status.h"
#include "google/protobuf/message_lite.h"
cdc_rsync/base/message_pump_test.cc
+1 -1
View File
@@ -14,8 +14,8 @@
#include "cdc_rsync/base/message_pump.h"
#include "cdc_rsync/base/fake_socket.h"
#include "cdc_rsync/protos/messages.pb.h"
#include "common/fake_socket.h"
#include "common/log.h"
#include "common/status.h"
#include "common/status_test_macros.h"
common/socket.h → cdc_rsync/base/socket.h
+3 -17
View File
@@ -14,8 +14,8 @@
* limitations under the License.
*/
#ifndef COMMON_SOCKET_H_
#define COMMON_SOCKET_H_
#ifndef CDC_RSYNC_BASE_SOCKET_H_
#define CDC_RSYNC_BASE_SOCKET_H_
#include "absl/status/status.h"
@@ -26,14 +26,6 @@ class Socket {
Socket() = default;
virtual ~Socket() = default;
// Calls WSAStartup() on Windows, no-op on Linux.
// Must be called before using sockets.
static absl::Status Initialize();
// Calls WSACleanup() on Windows, no-op on Linux.
// Must be called after using sockets.
static absl::Status Shutdown();
// Send data to the socket.
virtual absl::Status Send(const void* buffer, size_t size) = 0;
@@ -48,12 +40,6 @@ class Socket {
size_t* bytes_received) = 0;
};
// Convenience class that calls Shutdown() on destruction. Logs on errors.
class SocketFinalizer {
public:
~SocketFinalizer();
};
} // namespace cdc_ft
#endif // COMMON_SOCKET_H_
#endif // CDC_RSYNC_BASE_SOCKET_H_
cdc_rsync/cdc_rsync.vcxproj
+7 -8
View File
@@ -66,21 +66,20 @@
</ItemDefinitionGroup>
<!-- Bazel setup -->
<PropertyGroup>
<BazelTargets>//cdc_rsync //cdc_rsync_server</BazelTargets>
<BazelTargets>//cdc_rsync</BazelTargets>
<BazelOutputFile>cdc_rsync.exe</BazelOutputFile>
<BazelIncludePaths>..\;..\third_party\absl;..\bazel-cdc-file-transfer\external\com_github_blake3\c;..\bazel-cdc-file-transfer\external\com_github_zstd;..\third_party\googletest\googletest\include;..\bazel-cdc-file-transfer\external\com_google_protobuf\src;..\bazel-cdc-file-transfer\external\com_github_grpc_grpc\include;..\bazel-bin;$(VC_IncludePath);$(WindowsSDK_IncludePath)</BazelIncludePaths>
<BazelIncludePaths>..\;..\third_party\absl;..\bazel-cdc-file-transfer\external\com_github_blake3\c;..\bazel-stadia-file-transfer\external\com_github_zstd\lib;..\third_party\googletest\googletest\include;..\bazel-cdc-file-transfer\external\com_google_protobuf\src;$(VC_IncludePath);$(WindowsSDK_IncludePath)</BazelIncludePaths>
</PropertyGroup>
<Import Project="..\NMakeBazelProject.targets" />
<!-- For some reason, msbuild doesn't include this file, so copy it explicitly. -->
<!-- TODO: Reenable copying the Linux file once we can cross-compile these. -->
<!-- TODO: Reenable once we can cross-compile these.
<PropertyGroup>
<!-- <CdcRsyncLinuxServerFile>$(SolutionDir)bazel-out\k8-$(BazelCompilationMode)\bin\cdc_rsync_server\cdc_rsync_server</CdcRsyncLinuxServerFile> -->
<CdcRsyncWindowsServerFile>$(SolutionDir)bazel-out\x64_windows-$(BazelCompilationMode)\bin\cdc_rsync_server\cdc_rsync_server.exe</CdcRsyncWindowsServerFile>
<CdcRsyncServerFile>$(SolutionDir)bazel-out\k8-$(BazelCompilationMode)\bin\cdc_rsync_server\cdc_rsync_server</CdcRsyncServerFile>
</PropertyGroup>
<Target Name="CopyServer" Inputs="$(CdcRsyncLinuxServerFile);$(CdcRsyncWindowsServerFile)" Outputs="$(OutDir)cdc_rsync_server;$(OutDir)cdc_rsync_server.exe" AfterTargets="Build">
<!-- <Copy SourceFiles="$(CdcRsyncLinuxServerFile)" DestinationFiles="$(OutDir)cdc_rsync_server" SkipUnchangedFiles="true" /> -->
<Copy SourceFiles="$(CdcRsyncWindowsServerFile)" DestinationFiles="$(OutDir)cdc_rsync_server.exe" SkipUnchangedFiles="true" />
<Target Name="CopyServer" Inputs="$(CdcRsyncServerFile)" Outputs="$(OutDir)cdc_rsync_server" AfterTargets="Build">
<Copy SourceFiles="$(CdcRsyncServerFile)" DestinationFiles="$(OutDir)cdc_rsync_server" />
</Target>
-->
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
cdc_rsync/cdc_rsync_client.cc
+97 -133
View File
@@ -20,19 +20,16 @@
#include "cdc_rsync/base/message_pump.h"
#include "cdc_rsync/base/server_exit_code.h"
#include "cdc_rsync/client_file_info.h"
#include "cdc_rsync/client_socket.h"
#include "cdc_rsync/file_finder_and_sender.h"
#include "cdc_rsync/parallel_file_opener.h"
#include "cdc_rsync/progress_tracker.h"
#include "cdc_rsync/protos/messages.pb.h"
#include "cdc_rsync/server_arch.h"
#include "cdc_rsync/zstd_stream.h"
#include "common/client_socket.h"
#include "common/gamelet_component.h"
#include "common/log.h"
#include "common/path.h"
#include "common/process.h"
#include "common/remote_util.h"
#include "common/server_socket.h"
#include "common/status.h"
#include "common/status_macros.h"
#include "common/stopwatch.h"
@@ -47,6 +44,11 @@ constexpr int kExitCodeCouldNotExecute = 126;
// Bash exit code if binary was not found.
constexpr int kExitCodeNotFound = 127;
constexpr int kForwardPortFirst = 44450;
constexpr int kForwardPortLast = 44459;
constexpr char kCdcServerFilename[] = "cdc_rsync_server";
constexpr char kRemoteToolsBinDir[] = "~/.cache/cdc-file-transfer/bin/";
SetOptionsRequest::FilterRule::Type ToProtoType(PathFilter::Rule::Type type) {
switch (type) {
case PathFilter::Rule::Type::kInclude:
@@ -97,21 +99,20 @@ CdcRsyncClient::CdcRsyncClient(const Options& options,
std::string user_host, std::string destination)
: options_(options),
sources_(std::move(sources)),
user_host_(std::move(user_host)),
destination_(std::move(destination)),
remote_util_(options.verbosity, options.quiet, &process_factory_,
/*forward_output_to_log=*/false),
port_manager_("cdc_rsync_ports_f77bcdfe-368c-4c45-9f01-230c5e7e2132",
kForwardPortFirst, kForwardPortLast, &process_factory_,
&remote_util_),
printer_(options.quiet, Util::IsTTY() && !options.json),
progress_(&printer_, options.verbosity, options.json) {
// If there is no |user_host|, we sync files locally!
if (!user_host.empty()) {
remote_util_ =
std::make_unique<RemoteUtil>(std::move(user_host), options.verbosity,
options.quiet, &process_factory_,
/*forward_output_to_log=*/false);
if (!options_.ssh_command.empty()) {
remote_util_->SetSshCommand(options_.ssh_command);
}
if (!options_.sftp_command.empty()) {
remote_util_->SetSftpCommand(options_.sftp_command);
remote_util_.SetSshCommand(options_.ssh_command);
}
if (!options_.scp_command.empty()) {
remote_util_.SetScpCommand(options_.scp_command);
}
}
@@ -121,42 +122,19 @@ CdcRsyncClient::~CdcRsyncClient() {
}
absl::Status CdcRsyncClient::Run() {
// For local syncs, cdc_rsync_server runs on this machine. For remote syncs,
// guess the architecture of the device that runs cdc_rsync_server from the
// destination path, e.g. "C:\path\to\dest" strongly indicates Windows.
ServerArch server_arch = IsRemoteConnection()
? ServerArch::GuessFromDestination(destination_)
: ServerArch::DetectFromLocalDevice();
// Initialize |remote_util_|.
remote_util_.SetUserHostAndPort(user_host_, options_.port);
// Start the server process.
absl::Status status = StartServer(server_arch);
if (HasTag(status, Tag::kDeployServer) && server_arch.IsGuess() &&
server_exit_code_ != kServerExitCodeOutOfDate) {
// Server couldn't be run, e.g. not found or failed to start.
// Check whether we guessed the arch type wrong and try again.
// Note that in case of a local sync, or if the server actively reported
// that it's out-dated, there's no need to detect the arch.
LOG_DEBUG(
"Failed to start server, retrying after detecting remote arch: %s",
status.ToString());
const ArchType old_type = server_arch.GetType();
ASSIGN_OR_RETURN(server_arch,
ServerArch::DetectFromRemoteDevice(remote_util_.get()));
if (server_arch.GetType() != old_type) {
LOG_DEBUG("Guessed server arch type wrong, guessed %s, actual %s.",
GetArchTypeStr(old_type), server_arch.GetTypeStr());
status = StartServer(server_arch);
}
}
absl::Status status = StartServer();
if (HasTag(status, Tag::kDeployServer)) {
// Gamelet components are not deployed or out-dated. Deploy and retry.
status = DeployServer(server_arch);
status = DeployServer();
if (!status.ok()) {
return WrapStatus(status, "Failed to deploy server");
}
status = StartServer(server_arch);
status = StartServer();
}
if (!status.ok()) {
return WrapStatus(status, "Failed to start server");
@@ -188,7 +166,7 @@ absl::Status CdcRsyncClient::Run() {
return status;
}
absl::Status CdcRsyncClient::StartServer(const ServerArch& arch) {
absl::Status CdcRsyncClient::StartServer() {
assert(!server_process_);
// Components are expected to reside in the same dir as the executable.
@@ -200,41 +178,50 @@ absl::Status CdcRsyncClient::StartServer(const ServerArch& arch) {
std::vector<GameletComponent> components;
status = GameletComponent::Get(
{path::Join(component_dir, arch.CdcServerFilename())}, &components);
{path::Join(component_dir, kCdcServerFilename)}, &components);
if (!status.ok()) {
return MakeStatus(
"Required instance component not found. Make sure the file "
"%s resides in the same folder as %s.",
arch.CdcServerFilename(), ServerArch::CdcRsyncFilename());
"cdc_rsync_server resides in the same folder as cdc_rsync.exe.");
}
std::string component_args = GameletComponent::ToCommandLineArgs(components);
ProcessStartInfo start_info;
start_info.name = "cdc_rsync_server";
if (IsRemoteConnection()) {
// Run cdc_rsync_server on the remote instance.
std::string remote_command =
arch.GetStartServerCommand(kExitCodeNotFound, component_args);
assert(remote_util_);
start_info = remote_util_->BuildProcessStartInfoForSsh(remote_command,
arch.GetType());
} else {
// Run cdc_rsync_server locally.
std::string exe_dir;
RETURN_IF_ERROR(path::GetExeDir(&exe_dir), "Failed to get exe directory");
std::string server_path = path::Join(exe_dir, arch.CdcServerFilename());
start_info.command = absl::StrFormat("%s %s", server_path, component_args);
// Find available local and remote ports for port forwarding.
absl::StatusOr<int> port_res = port_manager_.ReservePort(
/*check_remote=*/false, /*remote_timeout_sec unused*/ 0);
constexpr char kErrorMsg[] = "Failed to find available port";
if (absl::IsDeadlineExceeded(port_res.status())) {
// Server didn't respond in time.
return SetTag(WrapStatus(port_res.status(), kErrorMsg),
Tag::kConnectionTimeout);
}
if (absl::IsResourceExhausted(port_res.status()))
return SetTag(WrapStatus(port_res.status(), kErrorMsg), Tag::kAddressInUse);
if (!port_res.ok())
return WrapStatus(port_res.status(), "Failed to find available port");
int port = *port_res;
std::string remote_server_path =
std::string(kRemoteToolsBinDir) + kCdcServerFilename;
// Test existence manually to prevent misleading bash output message
// "bash: .../cdc_rsync_server: No such file or directory".
// Also create the bin dir because otherwise scp below might fail.
std::string remote_command =
absl::StrFormat("mkdir -p %s; if [ ! -f %s ]; then exit %i; fi; %s %i %s",
kRemoteToolsBinDir, remote_server_path, kExitCodeNotFound,
remote_server_path, port, component_args);
ProcessStartInfo start_info =
remote_util_.BuildProcessStartInfoForSshPortForwardAndCommand(
port, port, false, remote_command);
start_info.name = "cdc_rsync_server";
// Capture stdout, but forward to stdout for debugging purposes.
start_info.stdout_handler = [this](const char* data, size_t /*data_size*/) {
return HandleServerOutput(data);
};
std::unique_ptr<Process> srv_process = process_factory_.Create(start_info);
status = srv_process->Start();
std::unique_ptr<Process> process = process_factory_.Create(start_info);
status = process->Start();
if (!status.ok()) {
return WrapStatus(status, "Failed to start cdc_rsync_server process");
}
@@ -242,17 +229,17 @@ absl::Status CdcRsyncClient::StartServer(const ServerArch& arch) {
// Wait until the server process is listening.
Stopwatch timeout_timer;
bool is_timeout = false;
auto detect_listening_or_timeout = [port = &server_listen_port_,
auto detect_listening_or_timeout = [is_listening = &is_server_listening_,
timeout = options_.connection_timeout_sec,
&timeout_timer, &is_timeout]() -> bool {
is_timeout = timeout_timer.ElapsedSeconds() > timeout;
return *port != 0 || is_timeout;
return *is_listening || is_timeout;
};
status = srv_process->RunUntil(detect_listening_or_timeout);
status = process->RunUntil(detect_listening_or_timeout);
if (!status.ok()) {
// Some internal process error. Note that this does NOT mean that
// cdc_rsync_server does not exist. In that case, the ssh process exits with
// code kExitCodeNotFound.
// code 127.
return status;
}
if (is_timeout) {
@@ -260,21 +247,15 @@ absl::Status CdcRsyncClient::StartServer(const ServerArch& arch) {
Tag::kConnectionTimeout);
}
if (srv_process->HasExited()) {
if (process->HasExited()) {
// Don't re-deploy for code > kServerExitCodeOutOfDate, which means that the
// out-of-date check already passed on the server.
server_exit_code_ = srv_process->ExitCode();
server_exit_code_ = process->ExitCode();
if (server_exit_code_ > kServerExitCodeOutOfDate &&
server_exit_code_ <= kServerExitCodeMax) {
return GetServerExitStatus(server_exit_code_, server_error_);
}
// Don't re-deploy if we're not copying to a remote device. We can start
// cdc_rsync_server from the original location directly.
if (!IsRemoteConnection()) {
return GetServerExitStatus(server_exit_code_, server_error_);
}
// Server exited before it started listening, most likely because of
// outdated components (code kServerExitCodeOutOfDate) or because the server
// wasn't deployed at all yet (code kExitCodeNotFound). Instruct caller
@@ -282,30 +263,13 @@ absl::Status CdcRsyncClient::StartServer(const ServerArch& arch) {
return SetTag(MakeStatus("Redeploy server"), Tag::kDeployServer);
}
// Start up sockets.
RETURN_IF_ERROR(Socket::Initialize(), "Failed to initialize sockets");
socket_finalizer_ = std::make_unique<SocketFinalizer>();
// Now that we know which port the server is using, set up port forwarding.
std::unique_ptr<Process> fwd_process;
int local_port = server_listen_port_;
if (IsRemoteConnection()) {
ASSIGN_OR_RETURN(local_port, ServerSocket::FindAvailablePort());
ProcessStartInfo start_info =
remote_util_->BuildProcessStartInfoForSshPortForward(
local_port, server_listen_port_, /*reverse=*/false);
start_info.forward_output_to_log = true;
fwd_process = process_factory_.Create(start_info);
RETURN_IF_ERROR(fwd_process->Start(),
"Failed to start cdc_rsync_server process");
assert(is_server_listening_);
status = socket_.Connect(port);
if (!status.ok()) {
return WrapStatus(status, "Failed to initialize connection");
}
// Wait for port forwarding to be up.
RETURN_IF_ERROR(ClientSocket::WaitForConnection(
local_port, absl::Seconds(options_.connection_timeout_sec)));
server_process_ = std::move(srv_process);
port_forwarding_process_ = std::move(fwd_process);
server_process_ = std::move(process);
message_pump_.StartMessagePump();
return absl::OkStatus();
}
@@ -326,7 +290,6 @@ absl::Status CdcRsyncClient::StopServer() {
server_exit_code_ = server_process_->ExitCode();
server_process_.reset();
port_forwarding_process_.reset();
return absl::OkStatus();
}
@@ -358,29 +321,10 @@ absl::Status CdcRsyncClient::HandleServerOutput(const char* data) {
}
printer_.Print(stdout_data, false, Util::GetConsoleWidth());
if (server_listen_port_ == 0) {
if (!is_server_listening_) {
server_output_.append(stdout_data);
// Parse port from "Port <n>: Server is listening".
size_t listening_pos = server_output_.find("Server is listening");
if (listening_pos != std::string::npos) {
// Search backwards until we find "Port ".
constexpr char port_key[] = "Port ";
size_t port_pos = server_output_.rfind(port_key, listening_pos);
if (port_pos == std::string::npos) {
return MakeStatus("Failed to find 'Port' marker in server output '%s'",
server_output_);
}
assert(listening_pos > port_pos);
server_listen_port_ = atoi(
server_output_
.substr(port_pos + strlen(port_key), listening_pos - port_pos)
.c_str());
if (server_listen_port_ == 0) {
return MakeStatus("Failed to parse port from server output '%s'",
server_output_);
}
}
is_server_listening_ =
server_output_.find("Server is listening") != std::string::npos;
}
return absl::OkStatus();
@@ -444,10 +388,8 @@ absl::Status CdcRsyncClient::Sync() {
return status;
}
absl::Status CdcRsyncClient::DeployServer(const ServerArch& arch) {
absl::Status CdcRsyncClient::DeployServer() {
assert(!server_process_);
assert(remote_util_);
assert(IsRemoteConnection());
std::string exe_dir;
absl::Status status = path::GetExeDir(&exe_dir);
@@ -467,10 +409,32 @@ absl::Status CdcRsyncClient::DeployServer(const ServerArch& arch) {
}
printer_.Print(deploy_msg, true, Util::GetConsoleWidth());
// sftp cdc_rsync_server to the target.
std::string commands = arch.GetDeploySftpCommands();
RETURN_IF_ERROR(remote_util_->Sftp(commands, exe_dir, /*compress=*/false),
"Failed to deploy cdc_rsync_server");
// scp cdc_rsync_server to a temp location on the gamelet.
std::string remoteServerTmpPath =
absl::StrFormat("%s%s.%s", kRemoteToolsBinDir, kCdcServerFilename,
Util::GenerateUniqueId());
std::string localServerPath = path::Join(exe_dir, kCdcServerFilename);
status = remote_util_.Scp({localServerPath}, remoteServerTmpPath,
/*compress=*/true);
if (!status.ok()) {
return WrapStatus(status, "Failed to copy cdc_rsync_server to instance");
}
// Do 3 things in one SSH command, to save time:
// - Make the old cdc_rsync_server writable (if it exists).
// - Make the new cdc_rsync_server executable.
// - Replace the old cdc_rsync_server by the new one.
std::string old_path = RemoteUtil::QuoteForWindows(
std::string(kRemoteToolsBinDir) + kCdcServerFilename);
std::string new_path = RemoteUtil::QuoteForWindows(remoteServerTmpPath);
std::string replace_cmd = absl::StrFormat(
" ([ ! -f %s ] || chmod u+w %s) && chmod a+x %s && mv %s %s", old_path,
old_path, new_path, new_path, old_path);
status = remote_util_.Run(replace_cmd, "chmod && chmod && mv");
if (!status.ok()) {
return WrapStatus(status,
"Failed to replace old cdc_rsync_server by new one");
}
return absl::OkStatus();
}
@@ -647,7 +611,7 @@ absl::Status CdcRsyncClient::SendMissingFiles() {
ParallelFileOpener file_opener(&files_, missing_file_indices_);
constexpr size_t kBufferSize = 128 * 1024;
constexpr size_t kBufferSize = 16000;
for (uint32_t server_index = 0; server_index < missing_file_indices_.size();
++server_index) {
uint32_t client_index = missing_file_indices_[server_index];
@@ -809,9 +773,9 @@ absl::Status CdcRsyncClient::StopCompressionStream() {
message_pump_.FlushOutgoingQueue();
message_pump_.RedirectOutput(nullptr);
// Finish compression stream and reset.
RETURN_IF_ERROR(compression_stream_->Finish(),
"Failed to finish compression stream");
// Flush compression stream and reset.
RETURN_IF_ERROR(compression_stream_->Flush(),
"Failed to flush compression stream");
compression_stream_.reset();
// Wait for the server ack. This must be done before sending more data.
cdc_rsync/cdc_rsync_client.h
+11 -19
View File
@@ -21,23 +21,22 @@
#include <vector>
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "cdc_rsync/base/message_pump.h"
#include "cdc_rsync/client_socket.h"
#include "cdc_rsync/progress_tracker.h"
#include "common/client_socket.h"
#include "common/path_filter.h"
#include "common/process.h"
#include "common/port_manager.h"
#include "common/remote_util.h"
namespace cdc_ft {
class Process;
class RemoteUtil;
class ServerArch;
class ZstdStream;
class CdcRsyncClient {
public:
struct Options {
int port = RemoteUtil::kDefaultSshPort;
bool delete_ = false;
bool recursive = false;
int verbosity = 0;
@@ -53,14 +52,10 @@ class CdcRsyncClient {
int compress_level = 6;
int connection_timeout_sec = 10;
std::string ssh_command;
std::string sftp_command;
std::string scp_command;
std::string sources_dir; // Base dir for files loaded for --files-from.
PathFilter filter;
// Backwards compatibility for switching from scp to sftp.
// Used internally, do not use.
std::string deprecated_scp_command;
// Compression level 0 is invalid.
static constexpr int kMinCompressLevel = -5;
static constexpr int kMaxCompressLevel = 22;
@@ -77,7 +72,7 @@ class CdcRsyncClient {
private:
// Starts the server process. If the method returns a status with tag
// |kTagDeployServer|, Run() calls DeployServer() and tries again.
absl::Status StartServer(const ServerArch& arch);
absl::Status StartServer();
// Stops the server process.
absl::Status StopServer();
@@ -89,7 +84,7 @@ class CdcRsyncClient {
absl::Status Sync();
// Copies all gamelet components to the gamelet.
absl::Status DeployServer(const ServerArch& arch);
absl::Status DeployServer();
// Sends relevant options to the server.
absl::Status SendOptions();
@@ -121,15 +116,13 @@ class CdcRsyncClient {
// Stops the zstd compression stream.
absl::Status StopCompressionStream();
// Returns true if the target is a remote target.
bool IsRemoteConnection() const { return remote_util_ != nullptr; }
Options options_;
std::vector<std::string> sources_;
const std::string user_host_;
const std::string destination_;
WinProcessFactory process_factory_;
std::unique_ptr<RemoteUtil> remote_util_;
std::unique_ptr<SocketFinalizer> socket_finalizer_;
RemoteUtil remote_util_;
PortManager port_manager_;
ClientSocket socket_;
MessagePump message_pump_{&socket_, MessagePump::PacketReceivedDelegate()};
ConsoleProgressPrinter printer_;
@@ -137,11 +130,10 @@ class CdcRsyncClient {
std::unique_ptr<ZstdStream> compression_stream_;
std::unique_ptr<Process> server_process_;
std::unique_ptr<Process> port_forwarding_process_;
std::string server_output_; // Written in a background thread. Do not access
std::string server_error_; // while the server process is active.
int server_exit_code_ = 0;
std::atomic_int server_listen_port_{0};
std::atomic_bool is_server_listening_{false};
bool is_server_error_ = false;
// All source files found on the client.
common/client_socket.cc → cdc_rsync/client_socket.cc
+47 -54
View File
@@ -12,14 +12,15 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/client_socket.h"
#include "cdc_rsync/client_socket.h"
#include <winsock2.h>
#include <ws2tcpip.h>
#include <cassert>
#include "common/log.h"
#include "common/socket_internal.h"
#include "common/status.h"
#include "common/stopwatch.h"
#include "common/util.h"
namespace cdc_ft {
@@ -28,9 +29,9 @@ namespace {
// Creates a status with the given |message| and the last WSA error.
// Assigns Tag::kSocketEof for WSAECONNRESET errors.
absl::Status MakeSocketStatus(const char* message) {
const int err = GetLastError();
absl::Status status = MakeStatus("%s: %s", message, GetErrorStr(err));
if (err == kErrConnReset) {
const int err = WSAGetLastError();
absl::Status status = MakeStatus("%s: %s", message, Util::GetWin32Error(err));
if (err == WSAECONNRESET) {
status = SetTag(status, Tag::kSocketEof);
}
return status;
@@ -38,71 +39,57 @@ absl::Status MakeSocketStatus(const char* message) {
} // namespace
struct ClientSocketInfo {
SocketType socket;
struct SocketInfo {
SOCKET socket;
ClientSocketInfo() : socket(kInvalidSocket) {}
SocketInfo() : socket(INVALID_SOCKET) {}
};
ClientSocket::ClientSocket() = default;
ClientSocket::~ClientSocket() { Disconnect(); }
// static
absl::Status ClientSocket::WaitForConnection(int port, absl::Duration timeout) {
assert(port != 0);
Stopwatch timeout_timer;
ClientSocket socket;
for (;;) {
absl::Status status = socket.Connect(port);
if (status.ok()) {
return absl::OkStatus();
}
if (timeout_timer.Elapsed() > timeout) {
return SetTag(
absl::DeadlineExceededError("Timeout while connecting to server"),
Tag::kConnectionTimeout);
}
Util::Sleep(50);
}
}
absl::Status ClientSocket::Connect(int port) {
WSADATA wsaData;
int result = WSAStartup(MAKEWORD(2, 2), &wsaData);
if (result != 0) {
return MakeStatus("WSAStartup() failed: %i", result);
}
addrinfo hints;
memset(&hints, 0, sizeof(hints));
ZeroMemory(&hints, sizeof(hints));
hints.ai_family = AF_INET;
hints.ai_socktype = SOCK_STREAM;
hints.ai_protocol = IPPROTO_TCP;
// Resolve the server address and port.
addrinfo* addr_infos = nullptr;
int result = getaddrinfo("localhost", std::to_string(port).c_str(), &hints,
result = getaddrinfo("localhost", std::to_string(port).c_str(), &hints,
&addr_infos);
if (result != 0) {
WSACleanup();
return MakeStatus("getaddrinfo() failed: %i", result);
}
AddrInfoReleaser releaser(addr_infos);
socket_info_ = std::make_unique<ClientSocketInfo>();
socket_info_ = std::make_unique<SocketInfo>();
int count = 0;
for (addrinfo* curr = addr_infos; curr; curr = curr->ai_next, count++) {
socket_info_->socket =
socket(curr->ai_family, curr->ai_socktype, curr->ai_protocol);
if (socket_info_->socket == kInvalidSocket) {
socket(addr_infos->ai_family, addr_infos->ai_socktype,
addr_infos->ai_protocol);
if (socket_info_->socket == INVALID_SOCKET) {
LOG_DEBUG("socket() failed for addr_info %i: %s", count,
GetLastErrorStr());
Util::GetWin32Error(WSAGetLastError()).c_str());
continue;
}
// Connect to server.
result = connect(socket_info_->socket, curr->ai_addr,
static_cast<int>(curr->ai_addrlen));
if (result == kSocketError) {
LOG_DEBUG("connect() failed for addr_info %i: %s", count,
GetLastErrorStr());
Close(&socket_info_->socket);
if (result == SOCKET_ERROR) {
LOG_DEBUG("connect() failed for addr_info %i: %i", count, result);
closesocket(socket_info_->socket);
socket_info_->socket = INVALID_SOCKET;
continue;
}
@@ -110,8 +97,11 @@ absl::Status ClientSocket::Connect(int port) {
break;
}
if (socket_info_->socket == kInvalidSocket) {
freeaddrinfo(addr_infos);
if (socket_info_->socket == INVALID_SOCKET) {
socket_info_.reset();
WSACleanup();
return MakeStatus("Unable to connect to port %i", port);
}
@@ -124,15 +114,19 @@ void ClientSocket::Disconnect() {
return;
}
Close(&socket_info_->socket);
if (socket_info_->socket != INVALID_SOCKET) {
closesocket(socket_info_->socket);
socket_info_->socket = INVALID_SOCKET;
}
socket_info_.reset();
WSACleanup();
}
absl::Status ClientSocket::Send(const void* buffer, size_t size) {
int result =
HANDLE_EINTR(send(socket_info_->socket, static_cast<const char*>(buffer),
static_cast<int>(size), /*flags */ 0));
if (result == kSocketError) {
int result = send(socket_info_->socket, static_cast<const char*>(buffer),
static_cast<int>(size), /*flags */ 0);
if (result == SOCKET_ERROR) {
return MakeSocketStatus("send() failed");
}
@@ -148,10 +142,9 @@ absl::Status ClientSocket::Receive(void* buffer, size_t size,
}
int flags = allow_partial_read ? 0 : MSG_WAITALL;
int bytes_read =
HANDLE_EINTR(recv(socket_info_->socket, static_cast<char*>(buffer),
static_cast<int>(size), flags));
if (bytes_read == kSocketError) {
int bytes_read = recv(socket_info_->socket, static_cast<char*>(buffer),
static_cast<int>(size), flags);
if (bytes_read == SOCKET_ERROR) {
return MakeSocketStatus("recv() failed");
}
@@ -170,9 +163,9 @@ absl::Status ClientSocket::Receive(void* buffer, size_t size,
}
absl::Status ClientSocket::ShutdownSendingEnd() {
int result = shutdown(socket_info_->socket, kSendingEnd);
if (result == kSocketError) {
return MakeStatus("Socket shutdown failed: %s", GetLastErrorStr());
int result = shutdown(socket_info_->socket, SD_SEND);
if (result == SOCKET_ERROR) {
return MakeSocketStatus("shutdown() failed");
}
return absl::OkStatus();
common/client_socket.h → cdc_rsync/client_socket.h
+5 -9
View File
@@ -14,13 +14,13 @@
* limitations under the License.
*/
#ifndef COMMON_CLIENT_SOCKET_H_
#define COMMON_CLIENT_SOCKET_H_
#ifndef CDC_RSYNC_CLIENT_SOCKET_H_
#define CDC_RSYNC_CLIENT_SOCKET_H_
#include <memory>
#include "absl/status/status.h"
#include "common/socket.h"
#include "cdc_rsync/base/socket.h"
namespace cdc_ft {
@@ -29,10 +29,6 @@ class ClientSocket : public Socket {
ClientSocket();
~ClientSocket();
// Polls until a connection to |port| succeeds.
// Returns TimeoutError
static absl::Status WaitForConnection(int port, absl::Duration timeout);
// Connects to localhost on |port|.
absl::Status Connect(int port);
@@ -49,9 +45,9 @@ class ClientSocket : public Socket {
size_t* bytes_received) override;
private:
std::unique_ptr<struct ClientSocketInfo> socket_info_;
std::unique_ptr<struct SocketInfo> socket_info_;
};
} // namespace cdc_ft
#endif // COMMON_CLIENT_SOCKET_H_
#endif // CDC_RSYNC_CLIENT_SOCKET_H_
cdc_rsync/file_finder_and_sender_test.cc
+1 -1
View File
@@ -18,8 +18,8 @@
#include "absl/strings/str_format.h"
#include "absl/strings/str_join.h"
#include "cdc_rsync/base/fake_socket.h"
#include "cdc_rsync/base/message_pump.h"
#include "common/fake_socket.h"
#include "common/log.h"
#include "common/path.h"
#include "common/path_filter.h"
cdc_rsync/main.cc
+5 -5
View File
@@ -68,16 +68,16 @@ ReturnCode TagToMessage(cdc_ft::Tag tag,
case cdc_ft::Tag::kDeployServer:
*msg =
"Failed to deploy or run the instance components for unknown "
"reasons. Please report this issue.";
"Failed to deploy the instance components for unknown reasons. "
"Please report this issue.";
return ReturnCode::kDeployFailed;
case cdc_ft::Tag::kConnectionTimeout:
// Server connection timed out. SSH probably stale.
*msg = absl::StrFormat(
"Server connection timed out. Verify that the host '%s' "
"is correct, or specify a larger timeout with --contimeout.",
params.user_host);
"Server connection timed out. Verify that host '%s' and port '%i' "
"are correct, or specify a larger timeout with --contimeout.",
params.user_host, params.options.port);
return ReturnCode::kConnectionTimeout;
case cdc_ft::Tag::kCount:
cdc_rsync/params.cc
+66 -93
View File
@@ -19,10 +19,7 @@
#include "absl/status/status.h"
#include "absl/strings/str_format.h"
#include "absl/strings/str_split.h"
#include "common/build_version.h"
#include "common/path.h"
#include "common/port_range_parser.h"
#include "common/remote_util.h"
#include "lib/zstd.h"
namespace cdc_ft {
@@ -39,22 +36,24 @@ void PrintError(const absl::FormatSpec<Args...>& format, Args... args) {
enum class OptionResult { kConsumedKey, kConsumedKeyValue, kError };
const char kHelpText[] =
R"(
Matching files are skipped based on file size and modified time. For partially
matching files only the differences are transferred. The destination directory
can be the same Windows machine or a remote Windows or Linux device.
R"(Copy local files to a gamelet
Synchronizes local files and files on a gamelet. Matching files are skipped.
For partially matching files only the deltas are transferred.
Usage:
cdc_rsync [options] source [source]... [[user@]host:]destination
cdc_rsync [options] source [source]... [user@]host:destination
Parameters:
source Local file or directory to be copied or synced
source Local file or directory to be copied
user Remote SSH user name
host Remote host or IP address
destination Local or remote destination directory
destination Remote destination directory
Options:
--contimeout sec Remote connection timeout in seconds (default: 10)
--ip string Gamelet IP. Required.
--port number SSH port to use. Required.
--contimeout sec Gamelet connection timeout in seconds (default: 10)
-q, --quiet Quiet mode, only print errors
-v, --verbose Increase output verbosity
--json Print JSON progress
@@ -62,55 +61,47 @@ Options:
-r, --recursive Recurse into directories
--delete Delete extraneous files from destination directory
-z, --compress Compress file data during the transfer
--compress-level <num> Explicitly set compression level (default: 6)
--compress-level num Explicitly set compression level (default: 6)
-c, --checksum Skip files based on checksum, not mod-time & size
-W, --whole-file Always copy files whole,
do not apply delta-transfer algorithm
--exclude pattern Exclude files matching pattern
--exclude-from <file> Read exclude patterns from file
--exclude-from file Read exclude patterns from file
--include pattern Don't exclude files matching pattern
--include-from <file> Read include patterns from file
--files-from <file> Read list of source files from file
--include-from file Read include patterns from file
--files-from file Read list of source files from file
-R, --relative Use relative path names
--existing Skip creating new files on instance
--copy-dest <dir> Use files from dir as sync base if files are missing
--ssh-command <cmd> Path and arguments of ssh command to use, e.g.
"C:\path\to\ssh.exe -p 12345 -i id_rsa -oUserKnownHostsFile=known_hosts"
--copy-dest dir Use files from dir as sync base if files are missing
--ssh-command Path and arguments of ssh command to use, e.g.
C:\path\to\ssh.exe -F config -i id_rsa -oStrictHostKeyChecking=yes -oUserKnownHostsFile="""known_hosts"""
Can also be specified by the CDC_SSH_COMMAND environment variable.
--sftp-command <cmd> Path and arguments of sftp command to use, e.g.
"C:\path\to\sftp.exe -P 12345 -i id_rsa -oUserKnownHostsFile=known_hosts"
Can also be specified by the CDC_SFTP_COMMAND environment variable.
-h, --help Help for cdc_rsync
--scp-command Path and arguments of scp command to use, e.g.
C:\path\to\scp.exe -F config -i id_rsa -oStrictHostKeyChecking=yes -oUserKnownHostsFile="""known_hosts"""
Can also be specified by the CDC_SCP_COMMAND environment variable.
-h --help Help for cdc_rsync
)";
constexpr char kSshCommandEnvVar[] = "CDC_SSH_COMMAND";
constexpr char kScpCommandEnvVar[] = "CDC_SCP_COMMAND";
constexpr char kSftpCommandEnvVar[] = "CDC_SFTP_COMMAND";
// Populates some parameters from environment variables.
void PopulateFromEnvVars(Parameters* parameters) {
path::GetEnv(kSshCommandEnvVar, &parameters->options.ssh_command)
.IgnoreError();
path::GetEnv(kScpCommandEnvVar, &parameters->options.deprecated_scp_command)
path::GetEnv(kScpCommandEnvVar, &parameters->options.scp_command)
.IgnoreError();
path::GetEnv(kSftpCommandEnvVar, &parameters->options.sftp_command)
.IgnoreError();
}
// Returns false and prints an error if |value| is null or empty.
bool ValidateValue(const std::string& option_name, const char* value) {
if (!value) {
PrintError("Option '%s' needs a value", option_name);
return false;
}
return true;
}
// Handles the --exclude-from and --include-from options.
OptionResult HandleFilterRuleFile(const std::string& option_name,
const char* path, PathFilter::Rule::Type type,
Parameters* params) {
assert(path);
if (!path) {
PrintError("Option '%s' needs a value", option_name);
return OptionResult::kError;
}
std::vector<std::string> patterns;
absl::Status status = path::ReadAllLines(
path, &patterns,
@@ -173,6 +164,13 @@ bool LoadFilesFrom(const std::string& files_from,
OptionResult HandleParameter(const std::string& key, const char* value,
Parameters* params, bool* help) {
if (key == "port") {
if (value) {
params->options.port = atoi(value);
}
return OptionResult::kConsumedKeyValue;
}
if (key == "delete") {
params->options.delete_ = true;
return OptionResult::kConsumedKey;
@@ -199,34 +197,29 @@ OptionResult HandleParameter(const std::string& key, const char* value,
}
if (key == "include") {
if (!ValidateValue(key, value)) return OptionResult::kError;
params->options.filter.AddRule(PathFilter::Rule::Type::kInclude, value);
return OptionResult::kConsumedKeyValue;
}
if (key == "include-from") {
if (!ValidateValue(key, value)) return OptionResult::kError;
return HandleFilterRuleFile(key, value, PathFilter::Rule::Type::kInclude,
params);
}
if (key == "exclude") {
if (!ValidateValue(key, value)) return OptionResult::kError;
params->options.filter.AddRule(PathFilter::Rule::Type::kExclude, value);
return OptionResult::kConsumedKeyValue;
}
if (key == "exclude-from") {
if (!ValidateValue(key, value)) return OptionResult::kError;
return HandleFilterRuleFile(key, value, PathFilter::Rule::Type::kExclude,
params);
}
if (key == "files-from") {
// Implies -R.
if (!ValidateValue(key, value)) return OptionResult::kError;
params->options.relative = true;
params->files_from = value;
params->files_from = value ? value : std::string();
return OptionResult::kConsumedKeyValue;
}
@@ -241,14 +234,16 @@ OptionResult HandleParameter(const std::string& key, const char* value,
}
if (key == "compress-level") {
if (!ValidateValue(key, value)) return OptionResult::kError;
if (value) {
params->options.compress_level = atoi(value);
}
return OptionResult::kConsumedKeyValue;
}
if (key == "contimeout") {
if (!ValidateValue(key, value)) return OptionResult::kError;
if (value) {
params->options.connection_timeout_sec = atoi(value);
}
return OptionResult::kConsumedKeyValue;
}
@@ -273,8 +268,7 @@ OptionResult HandleParameter(const std::string& key, const char* value,
}
if (key == "copy-dest") {
if (!ValidateValue(key, value)) return OptionResult::kError;
params->options.copy_dest = value;
params->options.copy_dest = value ? value : std::string();
return OptionResult::kConsumedKeyValue;
}
@@ -284,28 +278,12 @@ OptionResult HandleParameter(const std::string& key, const char* value,
}
if (key == "ssh-command") {
if (!ValidateValue(key, value)) return OptionResult::kError;
params->options.ssh_command = value;
params->options.ssh_command = value ? value : std::string();
return OptionResult::kConsumedKeyValue;
}
if (key == "scp-command") {
// Backwards compatibility. Note that this flag is hidden from the help.
if (!ValidateValue(key, value)) return OptionResult::kError;
params->options.deprecated_scp_command = value;
return OptionResult::kConsumedKeyValue;
}
if (key == "sftp-command") {
if (!ValidateValue(key, value)) return OptionResult::kError;
params->options.sftp_command = value;
return OptionResult::kConsumedKeyValue;
}
if (key == "forward-port") {
// This param is no longer needed. Just print a warning for backwards
// compatibility.
std::cout << "--forward-port argument no longer needed" << std::endl;
params->options.scp_command = value ? value : std::string();
return OptionResult::kConsumedKeyValue;
}
@@ -315,8 +293,6 @@ OptionResult HandleParameter(const std::string& key, const char* value,
bool ValidateParameters(const Parameters& params, bool help) {
if (help) {
std::cout << "cdc_rsync - Synchronize files and directories. Version: "
<< BUILD_VERSION << "\n";
std::cout << kHelpText;
return false;
}
@@ -326,6 +302,11 @@ bool ValidateParameters(const Parameters& params, bool help) {
return false;
}
if (params.options.port <= 0 || params.options.port > UINT16_MAX) {
PrintError("--port must specify a valid port");
return false;
}
// Note: ZSTD_minCLevel() is ridiculously small (-131072), so use a
// reasonable value.
assert(ZSTD_minCLevel() <= Options::kMinCompressLevel);
@@ -370,6 +351,14 @@ bool ValidateParameters(const Parameters& params, bool help) {
return false;
}
if (params.user_host.empty()) {
PrintError(
"No remote host specified in destination '%s'. "
"Expected [user@]host:destination.",
params.destination);
return false;
}
return true;
}
@@ -380,7 +369,11 @@ bool CheckOptionResult(OptionResult result, const std::string& name,
return true;
case OptionResult::kConsumedKeyValue:
return ValidateValue(name, value);
if (!value) {
PrintError("Option '%s' needs a value", name);
return false;
}
return true;
case OptionResult::kError:
// Error message was already printed.
@@ -395,15 +388,16 @@ bool CheckOptionResult(OptionResult result, const std::string& name,
// afterward and |user_host| is |user@foo.com|. Does not touch Windows drives,
// e.g. C:\foo.
void PopUserHost(std::string* destination, std::string* user_host) {
user_host->clear();
// Don't mistake the C part of C:\foo or \\share\C:\foo as user/host.
if (!path::GetDrivePrefix(*destination).empty()) return;
std::vector<std::string> parts =
absl::StrSplit(*destination, absl::MaxSplits(':', 1));
if (parts.size() < 2) return;
// Don't mistake the C part of C:\foo as user/host.
if (parts[0].size() == 1 && toupper(parts[0][0]) >= 'A' &&
toupper(parts[0][0]) <= 'Z') {
return;
}
*user_host = parts[0];
*destination = parts[1];
}
@@ -488,27 +482,6 @@ bool Parse(int argc, const char* const* argv, Parameters* parameters) {
PopUserHost(&parameters->destination, &parameters->user_host);
// Backwards compabitility after switching to sftp. Convert scp to sftp
// command Note that this flag is hidden from the help.
if (parameters->options.sftp_command.empty() &&
!parameters->options.deprecated_scp_command.empty()) {
LOG_WARNING(
"The CDC_SCP_COMMAND environment variable and the --scp-command flag "
"are deprecated. Please set CDC_SFTP_COMMAND or --sftp-command "
"instead.");
parameters->options.sftp_command = RemoteUtil::ScpToSftpCommand(
parameters->options.deprecated_scp_command);
if (!parameters->options.sftp_command.empty()) {
LOG_WARNING("Converted scp command '%s' to sftp command '%s'.",
parameters->options.deprecated_scp_command,
parameters->options.sftp_command);
} else {
LOG_WARNING("Failed to convert scp command '%s' to sftp command.",
parameters->options.deprecated_scp_command);
}
}
if (!ValidateParameters(*parameters, help)) {
return false;
}
cdc_rsync/params_test.cc
+39 -78
View File
@@ -32,10 +32,6 @@ constexpr char kUserHostDst[] = "user@host:destination";
constexpr char kUserHost[] = "user@host";
constexpr char kDst[] = "destination";
constexpr char kSshCommandEnvVar[] = "CDC_SSH_COMMAND";
constexpr char kScpCommandEnvVar[] = "CDC_SCP_COMMAND";
constexpr char kSftpCommandEnvVar[] = "CDC_SFTP_COMMAND";
class TestLog : public Log {
public:
explicit TestLog() : Log(LogLevel::kInfo) {}
@@ -64,11 +60,6 @@ class ParamsTest : public ::testing::Test {
void TearDown() override {
std::cout.rdbuf(prev_stdout_);
std::cerr.rdbuf(prev_stderr_);
// Clear env. They seem to be sticky sometimes and leak into other tests.
path::SetEnv(kSshCommandEnvVar, "");
path::SetEnv(kScpCommandEnvVar, "");
path::SetEnv(kSftpCommandEnvVar, "");
}
protected:
@@ -106,6 +97,7 @@ class ParamsTest : public ::testing::Test {
TEST_F(ParamsTest, ParseSucceedsDefaults) {
const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst, NULL};
EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
EXPECT_EQ(RemoteUtil::kDefaultSshPort, parameters_.options.port);
EXPECT_FALSE(parameters_.options.delete_);
EXPECT_FALSE(parameters_.options.recursive);
EXPECT_EQ(0, parameters_.options.verbosity);
@@ -153,6 +145,13 @@ TEST_F(ParamsTest, ParseFailsOnCompressLevelEqualsNoValue) {
ExpectError(NeedsValueError("compress-level"));
}
TEST_F(ParamsTest, ParseFailsOnPortEqualsNoValue) {
const char* argv[] = {"cdc_rsync.exe", "--port=", kSrc, kUserHostDst, NULL};
EXPECT_FALSE(
Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
ExpectError(NeedsValueError("port"));
}
TEST_F(ParamsTest, ParseFailsOnContimeoutEqualsNoValue) {
const char* argv[] = {"cdc_rsync.exe", "--contimeout=", kSrc, kUserHostDst,
NULL};
@@ -161,57 +160,24 @@ TEST_F(ParamsTest, ParseFailsOnContimeoutEqualsNoValue) {
ExpectError(NeedsValueError("contimeout"));
}
TEST_F(ParamsTest, ParseSucceedsWithSshSftpCommands) {
const char* argv[] = {
"cdc_rsync.exe", kSrc, kUserHostDst, "--ssh-command=sshcmd",
"--sftp-command=sftpcmd", NULL};
TEST_F(ParamsTest, ParseSucceedsWithSshScpCommands) {
const char* argv[] = {"cdc_rsync.exe", kSrc,
kUserHostDst, "--ssh-command=sshcmd",
"--scp-command=scpcmd", NULL};
EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
EXPECT_EQ(parameters_.options.sftp_command, "sftpcmd");
EXPECT_EQ(parameters_.options.scp_command, "scpcmd");
EXPECT_EQ(parameters_.options.ssh_command, "sshcmd");
}
TEST_F(ParamsTest, ParseSucceedsWithSshSftpCommandsByEnvVars) {
EXPECT_OK(path::SetEnv(kSshCommandEnvVar, "sshcmd"));
EXPECT_OK(path::SetEnv(kSftpCommandEnvVar, "sftpcmd"));
TEST_F(ParamsTest, ParseSucceedsWithSshScpCommandsByEnvVars) {
EXPECT_OK(path::SetEnv("CDC_SSH_COMMAND", "sshcmd"));
EXPECT_OK(path::SetEnv("CDC_SCP_COMMAND", "scpcmd"));
const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst, NULL};
EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
EXPECT_EQ(parameters_.options.sftp_command, "sftpcmd");
EXPECT_EQ(parameters_.options.scp_command, "scpcmd");
EXPECT_EQ(parameters_.options.ssh_command, "sshcmd");
}
TEST_F(ParamsTest, ParseSucceedsWithScpCommandFallback) {
const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst,
"--scp-command=C:\\scp.exe foo", NULL};
EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
EXPECT_EQ(parameters_.options.sftp_command, "C:\\sftp.exe foo");
}
TEST_F(ParamsTest, ParseSucceedsWithScpCommandFallbackByEnvVar) {
EXPECT_OK(path::SetEnv(kScpCommandEnvVar, "C:\\scp.exe foo"));
const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst, NULL};
EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
EXPECT_EQ(parameters_.options.sftp_command, "C:\\sftp.exe foo");
}
TEST_F(ParamsTest, ParseSucceedsWithSftpOverwritingScp) {
const char* argv[] = {"cdc_rsync.exe",
kSrc,
kUserHostDst,
"--scp-command=C:\\scp.exe foo",
"--sftp-command=sftpcmd",
NULL};
EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
EXPECT_EQ(parameters_.options.sftp_command, "sftpcmd");
}
TEST_F(ParamsTest, ParseSucceedsWithSftpEnvVarOverwritingScp) {
EXPECT_OK(path::SetEnv(kSftpCommandEnvVar, "sftpcmd"));
const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst,
"--scp-command=C:\\scp.exe foo", NULL};
EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
EXPECT_EQ(parameters_.options.sftp_command, "sftpcmd");
}
TEST_F(ParamsTest, ParseSucceedsWithNoSshCommand) {
const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst,
"--ssh-command=", NULL};
@@ -220,33 +186,26 @@ TEST_F(ParamsTest, ParseSucceedsWithNoSshCommand) {
ExpectError(NeedsValueError("ssh-command"));
}
TEST_F(ParamsTest, ParseSucceedsWithNoSftpCommand) {
const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst, "--sftp-command",
TEST_F(ParamsTest, ParseSucceedsWithNoScpCommand) {
const char* argv[] = {"cdc_rsync.exe", kSrc, kUserHostDst, "--scp-command",
NULL};
EXPECT_FALSE(
Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
ExpectError(NeedsValueError("sftp-command"));
ExpectError(NeedsValueError("scp-command"));
}
TEST_F(ParamsTest, ParseSucceedsOnNoUserHost) {
TEST_F(ParamsTest, ParseFailsOnNoUserHost) {
const char* argv[] = {"cdc_rsync.exe", kSrc, kDst, NULL};
EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
EXPECT_FALSE(
Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
ExpectError("No remote host specified");
}
TEST_F(ParamsTest, ParseDoesNotThinkDriveIsAHost) {
TEST_F(ParamsTest, ParseDoesNotThinkCIsAHost) {
const char* argv[] = {"cdc_rsync.exe", kSrc, "C:\\foo", NULL};
EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
EXPECT_TRUE(parameters_.user_host.empty());
const char* argv2[] = {"cdc_rsync.exe", kSrc, "\\\\.\\C:\\foo", NULL};
EXPECT_TRUE(
Parse(static_cast<int>(std::size(argv2)) - 1, argv, &parameters_));
EXPECT_TRUE(parameters_.user_host.empty());
const char* argv3[] = {"cdc_rsync.exe", kSrc, "\\\\?\\C:\\foo", NULL};
EXPECT_TRUE(
Parse(static_cast<int>(std::size(argv3)) - 1, argv, &parameters_));
EXPECT_TRUE(parameters_.user_host.empty());
EXPECT_FALSE(
Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
ExpectError("No remote host specified");
}
TEST_F(ParamsTest, ParseWithoutParametersFailsOnMissingSourceAndDestination) {
@@ -326,18 +285,13 @@ TEST_F(ParamsTest, ParseFailsOnUnknownKey) {
}
TEST_F(ParamsTest, ParseSucceedsWithSupportedKeyValue) {
const char* argv[] = {"cdc_rsync.exe",
"--compress-level",
"11",
"--contimeout",
"99",
"--copy-dest=dest",
kSrc,
kUserHostDst,
NULL};
const char* argv[] = {
"cdc_rsync.exe", "--compress-level", "11", "--contimeout", "99", "--port",
"4086", "--copy-dest=dest", kSrc, kUserHostDst, NULL};
EXPECT_TRUE(Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
EXPECT_EQ(parameters_.options.compress_level, 11);
EXPECT_EQ(parameters_.options.connection_timeout_sec, 99);
EXPECT_EQ(parameters_.options.port, 4086);
EXPECT_EQ(parameters_.options.copy_dest, "dest");
ExpectNoError();
}
@@ -350,6 +304,13 @@ TEST_F(ParamsTest, ParseSucceedsWithSupportedKeyValueWithoutEqualityForChars) {
ExpectNoError();
}
TEST_F(ParamsTest, ParseFailsOnInvalidPort) {
const char* argv[] = {"cdc_rsync.exe", "--port=0", kSrc, kUserHostDst, NULL};
EXPECT_FALSE(
Parse(static_cast<int>(std::size(argv)) - 1, argv, &parameters_));
ExpectError("--port must specify a valid port");
}
TEST_F(ParamsTest, ParseFailsOnDeleteNeedsRecursive) {
const char* argv[] = {"cdc_rsync.exe", "--delete", kSrc, kUserHostDst, NULL};
EXPECT_FALSE(
cdc_rsync/server_arch.cc
-271
View File
@@ -1,271 +0,0 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "cdc_rsync/server_arch.h"
#include <filesystem>
#include "absl/strings/match.h"
#include "absl/strings/str_format.h"
#include "absl/strings/str_split.h"
#include "common/ansi_filter.h"
#include "common/path.h"
#include "common/remote_util.h"
#include "common/status_macros.h"
#include "common/util.h"
namespace cdc_ft {
namespace {
constexpr char kErrorArchTypeUnhandled[] = "arch_type_unhandled";
constexpr char kUnsupportedArchErrorFmt[] =
"Unsupported remote device architecture '%s'. If you think this is a "
"bug, or if this combination should be supported, please file a bug at "
"https://github.com/google/cdc-file-transfer.";
absl::StatusOr<ArchType> GetArchTypeFromUname(const std::string& uname_out) {
// uname_out is "KERNEL MACHINE"
// Possible values for KERNEL: Linux (not sure what else).
// Possible values for MACHINE:
// https://stackoverflow.com/questions/45125516/possible-values-for-uname-m
// Relevant for us: x86_64, aarch64.
if (absl::StartsWith(uname_out, "Linux ")) {
// Linux kernel. Check CPU type.
if (absl::StrContains(uname_out, "x86_64")) {
return ArchType::kLinux_x86_64;
}
}
if (absl::StartsWith(uname_out, "MSYS_")) {
// Windows machine that happens to have Cygwin/MSYS on it. Check CPU type.
if (absl::StrContains(uname_out, "x86_64")) {
return ArchType::kWindows_x86_64;
}
}
return absl::UnimplementedError(
absl::StrFormat(kUnsupportedArchErrorFmt, uname_out));
}
absl::StatusOr<ArchType> GetArchTypeFromWinProcArch(
const std::string& arch_out) {
// Possible values: AMD64, IA64, ARM64, x86
if (absl::StrContains(arch_out, "AMD64")) {
return ArchType::kWindows_x86_64;
}
return absl::UnimplementedError(
absl::StrFormat(kUnsupportedArchErrorFmt, arch_out));
}
} // namespace
// static
ServerArch ServerArch::GuessFromDestination(const std::string& destination) {
// Path starting with ~ or / -> Linux.
if (absl::StartsWith(destination, "~") ||
absl::StartsWith(destination, "/")) {
LOG_DEBUG("Guessed server arch type Linux based on ~ or /");
return ServerArch(ArchType::kLinux_x86_64, /*is_guess=*/true);
}
// Path starting with C: etc. -> Windows.
if (!path::GetDrivePrefix(destination).empty()) {
LOG_DEBUG("Guessed server arch type Windows based on drive prefix");
return ServerArch(ArchType::kWindows_x86_64, /*is_guess=*/true);
}
// Path with only / -> Linux.
if (absl::StrContains(destination, "/") &&
!absl::StrContains(destination, "\\")) {
LOG_DEBUG("Guessed server arch type Linux based on forward slashes");
return ServerArch(ArchType::kLinux_x86_64, /*is_guess=*/true);
}
// Path with only \\ -> Windows.
if (absl::StrContains(destination, "\\") &&
!absl::StrContains(destination, "/")) {
LOG_DEBUG("Guessed server arch type Windows based on backslashes");
return ServerArch(ArchType::kWindows_x86_64, /*is_guess=*/true);
}
// Default to Linux.
LOG_DEBUG("Guessed server arch type Linux as default");
return ServerArch(ArchType::kLinux_x86_64, /*is_guess=*/true);
}
// static
ServerArch ServerArch::DetectFromLocalDevice() {
LOG_DEBUG("Detected local device type %s",
GetArchTypeStr(GetLocalArchType()));
return ServerArch(GetLocalArchType(), /*is_guess=*/false);
}
// static
absl::StatusOr<ServerArch> ServerArch::DetectFromRemoteDevice(
RemoteUtil* remote_util) {
assert(remote_util);
// Run uname, assuming it's a Linux machine.
std::string uname_out;
std::string linux_cmd = "uname -sm";
absl::Status status = remote_util->RunWithCapture(
linux_cmd, "uname", &uname_out, nullptr, ArchType::kLinux_x86_64);
if (status.ok()) {
// Running uname on Windows, assuming it's Linux, leads to tons of ANSI
// escape sequences in the output. Remove them to at least get some readable
// output.
uname_out = absl::StripAsciiWhitespace(
ansi_filter::RemoveEscapeSequences(uname_out));
LOG_DEBUG("Uname returned '%s'", uname_out);
absl::StatusOr<ArchType> type = GetArchTypeFromUname(uname_out);
if (type.ok()) {
LOG_DEBUG("Detected server arch type '%s' from uname",
GetArchTypeStr(*type));
return ServerArch(*type, /*is_guess=*/false);
}
status = type.status();
}
LOG_DEBUG(
"Failed to detect arch type from uname; this is expected if the remote "
"machine is not Linux; will try Windows next: %s",
status.ToString());
// Check %PROCESSOR_ARCHITECTURE%, assuming it's a Windows machine.
// Note: That space after PROCESSOR_ARCHITECTURE is important or else Windows
// command magic interprets quotes as part of the string.
std::string arch_out;
std::string windows_cmd = "\"cmd /C set PROCESSOR_ARCHITECTURE \"";
status = remote_util->RunWithCapture(windows_cmd,
"set PROCESSOR_ARCHITECTURE", &arch_out,
nullptr, ArchType::kWindows_x86_64);
if (status.ok()) {
LOG_DEBUG("PROCESSOR_ARCHITECTURE is '%s'", arch_out);
absl::StatusOr<ArchType> type = GetArchTypeFromWinProcArch(arch_out);
if (type.ok()) {
LOG_DEBUG("Detected server arch type '%s' from PROCESSOR_ARCHITECTURE",
GetArchTypeStr(*type));
return ServerArch(*type, /*is_guess=*/false);
}
status = type.status();
}
LOG_DEBUG("Failed to detect arch type from PROCESSOR_ARCHITECTURE: %s",
status.ToString());
return absl::InternalError("Failed to detect remote architecture");
}
// static
std::string ServerArch::CdcRsyncFilename() {
switch (GetLocalArchType()) {
case ArchType::kWindows_x86_64:
return "cdc_rsync.exe";
case ArchType::kLinux_x86_64:
return "cdc_rsync";
default:
assert(!kErrorArchTypeUnhandled);
return std::string();
}
}
ServerArch::ServerArch(ArchType type, bool is_guess)
: type_(type), is_guess_(is_guess) {}
ServerArch::~ServerArch() {}
const char* ServerArch::GetTypeStr() const { return GetArchTypeStr(type_); }
std::string ServerArch::CdcServerFilename() const {
switch (type_) {
case ArchType::kWindows_x86_64:
return "cdc_rsync_server.exe";
case ArchType::kLinux_x86_64:
return "cdc_rsync_server";
default:
assert(!kErrorArchTypeUnhandled);
return std::string();
}
}
std::string ServerArch::RemoteToolsBinDir() const {
if (IsWindowsArchType(type_)) {
return "AppData\\Roaming\\cdc-file-transfer\\bin\\";
}
if (IsLinuxArchType(type_)) {
return ".cache/cdc-file-transfer/bin/";
}
assert(!kErrorArchTypeUnhandled);
return std::string();
}
std::string ServerArch::GetStartServerCommand(int exit_code_not_found,
const std::string& args) const {
std::string server_path = RemoteToolsBinDir() + CdcServerFilename();
if (IsWindowsArchType(type_)) {
// TODO(ljusten): On Windows, ssh does not seem to forward the Powershell
// exit code (exit_code_not_found) to the process. However, that's really
// a minor issue and means we display "Deploying server..." instead of
// "Server not deployed. Deploying...";
return RemoteUtil::QuoteForWindows(
absl::StrFormat("powershell -Command "
"Set-StrictMode -Version 2; "
"$ErrorActionPreference = 'Stop'; "
"if (-not (Test-Path -Path '%s')) { "
" exit %i; "
"} "
"%s %s",
server_path, exit_code_not_found, server_path, args));
}
if (IsLinuxArchType(type_)) {
return absl::StrFormat("if [ ! -f %s ]; then exit %i; fi; %s %s",
server_path, exit_code_not_found, server_path, args);
}
assert(!kErrorArchTypeUnhandled);
return std::string();
}
std::string ServerArch::GetDeploySftpCommands() const {
std::string commands;
// Create the remote tools bin dir if it doesn't exist yet.
// This assumes that sftp's remote startup directory is the home directory.
const std::string server_dir = path::ToUnix(RemoteToolsBinDir());
std::vector<std::string> dir_parts =
absl::StrSplit(server_dir, '/', absl::SkipEmpty());
for (const std::string& dir : dir_parts) {
// Use -mkdir to ignore errors if the directory already exists.
commands += absl::StrFormat("-mkdir %s\ncd %s\n", dir, dir);
}
// Copy the server binary to a temp location. This assumes that sftp's local
// startup directory is cdc_rsync's exe dir.
const std::string server_file = CdcServerFilename();
const std::string server_temp_file = server_file + Util::GenerateUniqueId();
commands += absl::StrFormat("put %s %s\n", server_file, server_temp_file);
// Restore permissions in case they changed and propagate temp file.
commands += absl::StrFormat("-chmod 755 %s\n", server_file);
commands += absl::StrFormat("chmod 755 %s\n", server_temp_file);
commands += absl::StrFormat("rename %s %s\n", server_temp_file, server_file);
return commands;
}
} // namespace cdc_ft
cdc_rsync/server_arch.h
-97
View File
@@ -1,97 +0,0 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef CDC_RSYNC_SERVER_ARCH_H_
#define CDC_RSYNC_SERVER_ARCH_H_
#include <string>
#include "absl/status/statusor.h"
#include "common/arch_type.h"
namespace cdc_ft {
class RemoteUtil;
// Abstracts all architecture specifics of cdc_rsync_server deployment.
// Comes in two flavors, "guessed" and "detected". Guesses are used as an
// optimization. For instance, if one syncs to C:\some\path, it's clearly a
// Windows machine and we can skip detection.
class ServerArch {
public:
// Guesses the arch type based on the destination path, e.g. path starting
// with C: indicate Windows. This is a guessed type. It may be wrong. For
// instance, if destination is just a single folder like "foo", the method
// defaults to Type::kLinux.
static ServerArch GuessFromDestination(const std::string& destination);
// Returns the arch type that matches the current process's type.
// This is not a guessed type, it is reliable.
static ServerArch DetectFromLocalDevice();
// Creates an by properly detecting it on the remote device.
// This is more costly than guessing, but it is reliable.
static absl::StatusOr<ServerArch> DetectFromRemoteDevice(
RemoteUtil* remote_util);
// Returns the (local!) arch specific filename of cdc_rsync[.exe].
static std::string CdcRsyncFilename();
ServerArch(ArchType type, bool is_guess);
~ServerArch();
// Accessor for the arch type.
ArchType GetType() const { return type_; }
// Returns the type as a human readable string.
const char* GetTypeStr() const;
// Returns true if the type was guessed and not detected.
bool IsGuess() const { return is_guess_; }
// Returns the arch-specific filename of cdc_rsync_server[.exe].
std::string CdcServerFilename() const;
// Returns the arch-specific directory where cdc_rsync_server is deployed.
std::string RemoteToolsBinDir() const;
// Returns an arch-specific SSH shell command that gets invoked in order to
// start cdc_rsync_server. The command
// - returns |exit_code_not_found| if cdc_rsync_server does not exist (to
// prevent the confusing bash output message
// "bash: .../cdc_rsync_server: No such file or directory"), and
// - runs the server with the provided |args|.
std::string GetStartServerCommand(int exit_code_not_found,
const std::string& args) const;
// Returns an arch-specific SFTP command sequence that deploys the server
// component on the target gets invoked after
// cdc_rsync_server has been copied to a temp location. The commands
// - create the cdc-file-transfer/bin folder if it doesn't exist yet,
// - make the old cdc_rsync_server writable if it exists,
// - copy cdc_rsync_server to a temp location,
// - make the new cdc_rsync_server executable (Linux only) and
// - replaces the existing cdc_rsync_server by the temp one.
std::string GetDeploySftpCommands() const;
private:
ArchType type_;
bool is_guess_ = false;
};
} // namespace cdc_ft
#endif // CDC_RSYNC_SERVER_ARCH_H_
cdc_rsync/server_arch_test.cc
-104
View File
@@ -1,104 +0,0 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "cdc_rsync/server_arch.h"
#include "absl/strings/match.h"
#include "gtest/gtest.h"
namespace cdc_ft {
namespace {
constexpr auto kLinux = ArchType::kLinux_x86_64;
constexpr auto kWindows = ArchType::kWindows_x86_64;
constexpr bool kNoGuess = false;
TEST(ServerArchTest, GuessesLinuxIfPathStartsWithSlashOrTilde) {
EXPECT_EQ(ServerArch::GuessFromDestination("/linux/path").GetType(), kLinux);
EXPECT_EQ(ServerArch::GuessFromDestination("/linux\\path").GetType(), kLinux);
EXPECT_EQ(ServerArch::GuessFromDestination("~/linux/path").GetType(), kLinux);
EXPECT_EQ(ServerArch::GuessFromDestination("~/linux\\path").GetType(),
kLinux);
EXPECT_EQ(ServerArch::GuessFromDestination("~\\linux\\path").GetType(),
kLinux);
}
TEST(ServerArchTest, GuessesWindowsIfPathStartsWithDrive) {
EXPECT_EQ(ServerArch::GuessFromDestination("C:\\win\\path").GetType(),
kWindows);
EXPECT_EQ(ServerArch::GuessFromDestination("D:win").GetType(), kWindows);
EXPECT_EQ(ServerArch::GuessFromDestination("Z:\\win/path").GetType(),
kWindows);
}
TEST(ServerArchTest, GuessesLinuxIfPathOnlyHasForwardSlashes) {
EXPECT_EQ(ServerArch::GuessFromDestination("linux/path").GetType(), kLinux);
}
TEST(ServerArchTest, GuessesWindowsIfPathOnlyHasBackSlashes) {
EXPECT_EQ(ServerArch::GuessFromDestination("\\win\\path").GetType(),
kWindows);
}
TEST(ServerArchTest, GuessesLinuxByDefault) {
EXPECT_EQ(ServerArch::GuessFromDestination("/mixed\\path").GetType(), kLinux);
EXPECT_EQ(ServerArch::GuessFromDestination("/mixed\\path").GetType(), kLinux);
EXPECT_EQ(ServerArch::GuessFromDestination("C\\linux/path").GetType(),
kLinux);
EXPECT_EQ(ServerArch::GuessFromDestination("").GetType(), kLinux);
}
TEST(ServerArchTest, IsGuess) {
EXPECT_TRUE(ServerArch::GuessFromDestination("foo").IsGuess());
EXPECT_FALSE(ServerArch::DetectFromLocalDevice().IsGuess());
}
TEST(ServerArchTest, CdcServerFilename) {
EXPECT_FALSE(absl::StrContains(
ServerArch(kLinux, kNoGuess).CdcServerFilename(), "exe"));
EXPECT_TRUE(absl::StrContains(
ServerArch(kWindows, kNoGuess).CdcServerFilename(), "exe"));
}
TEST(ServerArchTest, RemoteToolsBinDir) {
const std::string linux_dir =
ServerArch(kLinux, kNoGuess).RemoteToolsBinDir();
EXPECT_TRUE(absl::StrContains(linux_dir, ".cache/"));
std::string win_dir = ServerArch(kWindows, kNoGuess).RemoteToolsBinDir();
EXPECT_TRUE(absl::StrContains(win_dir, "AppData\\Roaming\\"));
}
TEST(ServerArchTest, GetStartServerCommand) {
std::string cmd =
ServerArch(kWindows, kNoGuess).GetStartServerCommand(123, "foo bar");
EXPECT_TRUE(absl::StrContains(cmd, "123"));
EXPECT_TRUE(absl::StrContains(cmd, "cdc_rsync_server.exe foo bar"));
cmd = ServerArch(kLinux, kNoGuess).GetStartServerCommand(123, "foo bar");
EXPECT_TRUE(absl::StrContains(cmd, "123"));
EXPECT_TRUE(absl::StrContains(cmd, "cdc_rsync_server foo bar"));
}
TEST(ServerArchTest, GetDeployReplaceCommand) {
std::string cmd = ServerArch(kWindows, kNoGuess).GetDeploySftpCommands();
EXPECT_TRUE(absl::StrContains(cmd, "cdc_rsync_server.exe"));
cmd = ServerArch(kLinux, kNoGuess).GetDeploySftpCommands();
EXPECT_TRUE(absl::StrContains(cmd, "cdc_rsync_server"));
}
} // namespace
} // namespace cdc_ft
cdc_rsync/zstd_stream.cc
+10 -12
View File
@@ -27,19 +27,22 @@ namespace {
// trigger a flush. This happens when files with no changes are diff'ed (this
// produces very low volume data). Flushing prevents that the server gets stale
// and becomes overwhelmed later.
constexpr absl::Duration kDefaultAutoFlushPeriod = absl::Milliseconds(500);
constexpr absl::Duration kMinCompressPeriod = absl::Milliseconds(500);
} // namespace
ZstdStream::ZstdStream(Socket* socket, int level, uint32_t num_threads)
: socket_(socket),
cctx_(nullptr),
auto_flush_period_(kDefaultAutoFlushPeriod) {
: socket_(socket), cctx_(nullptr) {
status_ = WrapStatus(Initialize(level, num_threads),
"Failed to initialize stream compressor");
}
ZstdStream::~ZstdStream() {
if (cctx_) {
ZSTD_freeCCtx(cctx_);
cctx_ = nullptr;
}
{
absl::MutexLock lock(&mutex_);
shutdown_ = true;
@@ -47,11 +50,6 @@ ZstdStream::~ZstdStream() {
if (compressor_thread_.joinable()) {
compressor_thread_.join();
}
if (cctx_) {
ZSTD_freeCCtx(cctx_);
cctx_ = nullptr;
}
}
absl::Status ZstdStream::Write(const void* data, size_t size) {
@@ -81,7 +79,7 @@ absl::Status ZstdStream::Write(const void* data, size_t size) {
return absl::OkStatus();
}
absl::Status ZstdStream::Finish() {
absl::Status ZstdStream::Flush() {
absl::MutexLock lock(&mutex_);
if (!status_.ok()) return status_;
@@ -136,7 +134,7 @@ void ZstdStream::ThreadCompressorMain() {
in_buffer_.size() == in_buffer_.capacity();
};
bool flush =
!mutex_.AwaitWithTimeout(absl::Condition(&cond), auto_flush_period_);
!mutex_.AwaitWithTimeout(absl::Condition(&cond), kMinCompressPeriod);
if (shutdown_) {
return;
}
@@ -146,7 +144,7 @@ void ZstdStream::ThreadCompressorMain() {
const ZSTD_EndDirective mode = last_chunk_ ? ZSTD_e_end
: flush ? ZSTD_e_flush
: ZSTD_e_continue;
LOG_VERBOSE("Compressing %u bytes (mode=%s)", in_buffer_.size(),
LOG_DEBUG("Compressing %u bytes (mode=%s)", in_buffer_.size(),
mode == ZSTD_e_end ? "end"
: mode == ZSTD_e_flush ? "flush"
: "continue");
cdc_rsync/zstd_stream.h
+3 -10
View File
@@ -21,8 +21,8 @@
#include "absl/status/status.h"
#include "absl/synchronization/mutex.h"
#include "cdc_rsync/base/socket.h"
#include "common/buffer.h"
#include "common/socket.h"
#include "lib/zstd.h"
namespace cdc_ft {
@@ -36,13 +36,8 @@ class ZstdStream {
// Sends the given |data| to the compressor.
absl::Status Write(const void* data, size_t size) ABSL_LOCKS_EXCLUDED(mutex_);
// Finishes the stream and flushes all remaining data.
absl::Status Finish() ABSL_LOCKS_EXCLUDED(mutex_);
// Flushes internal buffers if no new data is written for longer than this
// time. This makes sure that no data is stuck in the pipeline if no new input
// is available. Default is 500 ms.
void AutoFlushAfter(absl::Duration dur) { auto_flush_period_ = dur; }
// Flushes all remaining data and sends the compressed data to the socket.
absl::Status Flush() ABSL_LOCKS_EXCLUDED(mutex_);
private:
// Initializes the compressor and related data.
@@ -63,8 +58,6 @@ class ZstdStream {
bool last_chunk_sent_ ABSL_GUARDED_BY(mutex_) = false;
absl::Status status_ ABSL_GUARDED_BY(mutex_);
std::thread compressor_thread_;
absl::Duration auto_flush_period_;
};
} // namespace cdc_ft
cdc_rsync/zstd_stream_test.cc
+3 -49
View File
@@ -14,8 +14,8 @@
#include "cdc_rsync/zstd_stream.h"
#include "cdc_rsync/base/fake_socket.h"
#include "cdc_rsync_server/unzstd_stream.h"
#include "common/fake_socket.h"
#include "common/status_test_macros.h"
#include "gtest/gtest.h"
@@ -32,7 +32,7 @@ class ZstdStreamTest : public ::testing::Test {
TEST_F(ZstdStreamTest, Small) {
const std::string want = "Lorem ipsum gibberisulum foobarberis";
EXPECT_OK(cstream_.Write(want.data(), want.size()));
EXPECT_OK(cstream_.Finish());
EXPECT_OK(cstream_.Flush());
Buffer buff(1024);
size_t bytes_read;
@@ -43,52 +43,6 @@ TEST_F(ZstdStreamTest, Small) {
EXPECT_EQ(got, want);
}
TEST_F(ZstdStreamTest, AutoFlushesAfterTimeout) {
const std::string want = "Lorem ipsum gibberisulum foobarberis";
cstream_.AutoFlushAfter(absl::Milliseconds(10));
EXPECT_OK(cstream_.Write(want.data(), want.size()));
// Note: No flush! cstream_ will compress and send the data after 10 ms.
// Only read as much data as we have written, or else dstream_.Read() will
// expect more data.
Buffer buff(want.size());
size_t bytes_read;
bool eof = false;
EXPECT_OK(dstream_.Read(buff.data(), buff.size(), &bytes_read, &eof));
EXPECT_FALSE(eof);
std::string got(buff.data(), bytes_read);
EXPECT_EQ(got, want);
}
// Regression test for an issue in UnzstdStream, where the reader tried to read
// from the socket even though the output data was still available in internal
// buffers.
TEST_F(ZstdStreamTest, DeliversOutputBeforeReadingNewData) {
const std::string want1 = "I want";
const std::string want2 = "to eat cookies";
cstream_.AutoFlushAfter(absl::Milliseconds(10));
EXPECT_OK(cstream_.Write(want1.data(), want1.size()));
EXPECT_OK(cstream_.Write(want2.data(), want2.size()));
// Note: No flush! cstream_ will compress and send the data after 10 ms.
Buffer buff1(want1.size());
Buffer buff2(want2.size());
size_t bytes_read1, bytes_read2;
bool eof1 = false, eof2 = false;
EXPECT_OK(dstream_.Read(buff1.data(), buff1.size(), &bytes_read1, &eof1));
// There was a bug in dstream_.Read(), where the method would first try to
// read new input before uncompressing data, even though the data was already
// present in internal buffers.
EXPECT_OK(dstream_.Read(buff2.data(), buff2.size(), &bytes_read2, &eof2));
EXPECT_FALSE(eof1);
EXPECT_FALSE(eof2);
std::string got1(buff1.data(), bytes_read1);
std::string got2(buff2.data(), bytes_read2);
EXPECT_EQ(got1, want1);
EXPECT_EQ(got2, want2);
}
TEST_F(ZstdStreamTest, Large) {
Buffer want(1024 * 1024 * 10 + 12345);
constexpr uint64_t prime = 919393;
@@ -101,7 +55,7 @@ TEST_F(ZstdStreamTest, Large) {
size_t size = std::min<size_t>(kChunkSize, want.size() - pos);
EXPECT_OK(cstream_.Write(want.data() + pos, size));
}
EXPECT_OK(cstream_.Finish());
EXPECT_OK(cstream_.Flush());
bool eof = false;
Buffer buff(128 * 1024);
cdc_rsync_server/BUILD
+16 -4
View File
@@ -22,7 +22,7 @@ cc_test(
srcs = ["file_deleter_and_sender_test.cc"],
deps = [
":file_deleter_and_sender",
"//common:fake_socket",
"//cdc_rsync/base:fake_socket",
"//common:status_test_macros",
"//common:test_main",
"@com_google_googletest//:gtest",
@@ -95,16 +95,15 @@ cc_binary(
":file_diff_generator",
":file_finder",
":file_info",
":server_socket",
":unzstd_stream",
"//cdc_rsync/base:cdc_interface",
"//cdc_rsync/base:message_pump",
"//cdc_rsync/base:server_exit_code",
"//common:build_version",
"//common:clock",
"//common:gamelet_component",
"//common:log",
"//common:path_filter",
"//common:server_socket",
"//common:status",
"//common:stopwatch",
"//common:threadpool",
@@ -124,13 +123,26 @@ cc_library(
hdrs = ["file_info.h"],
)
cc_library(
name = "server_socket",
srcs = ["server_socket.cc"],
hdrs = ["server_socket.h"],
target_compatible_with = ["@platforms//os:linux"],
deps = [
"//cdc_rsync/base:socket",
"//common:log",
"//common:status",
"@com_google_absl//absl/status",
],
)
cc_library(
name = "unzstd_stream",
srcs = ["unzstd_stream.cc"],
hdrs = ["unzstd_stream.h"],
deps = [
"//cdc_rsync/base:message_pump",
"//common:socket",
"//cdc_rsync/base:socket",
"//common:status",
"@com_github_zstd//:zstd",
"@com_google_absl//absl/status",
cdc_rsync_server/cdc_rsync_server.cc
+71 -203
View File
@@ -19,12 +19,11 @@
#include "cdc_rsync/protos/messages.pb.h"
#include "cdc_rsync_server/file_deleter_and_sender.h"
#include "cdc_rsync_server/file_finder.h"
#include "cdc_rsync_server/server_socket.h"
#include "cdc_rsync_server/unzstd_stream.h"
#include "common/log.h"
#include "common/path.h"
#include "common/server_socket.h"
#include "common/status.h"
#include "common/status_macros.h"
#include "common/stopwatch.h"
#include "common/threadpool.h"
#include "common/util.h"
@@ -33,22 +32,9 @@ namespace cdc_ft {
namespace {
// Number of files for which to call fclose() and finalize files in parallel.
constexpr size_t kNumFinalizerThreads = 8;
// Max 16 files in the patcher and finalizer queues to prevent that too many
// files are open concurrently.
constexpr size_t kMaxQueueSize = 16;
// Suffix for the patched file created from the basis file and the diff.
constexpr char kIntermediatePathSuffix[] = ".__cdc_rsync_temp__";
#if PLATFORM_WINDOWS
constexpr char kServerFilename[] = "cdc_rsync_server.exe";
#elif PLATFORM_LINUX
constexpr char kServerFilename[] = "cdc_rsync_server";
#endif
uint16_t kExecutableBits =
path::MODE_IXUSR | path::MODE_IXGRP | path::MODE_IXOTH;
@@ -57,9 +43,6 @@ uint16_t kExecutableBits =
// |target_filepath| match, writes an intermediate file and replaces
// the file at |target_filepath| with the intermediate file when all data has
// been received.
// Each PatchTask is queued twice, once to create the patched file, and once in
// a different thread pool to close and finalize the patched file. This is
// because fclose() can take a long time to finish, so it could block patching.
class PatchTask : public Task {
public:
PatchTask(const std::string& base_filepath,
@@ -68,69 +51,38 @@ class PatchTask : public Task {
: base_filepath_(base_filepath),
target_filepath_(target_filepath),
file_(file),
cdc_(cdc),
need_intermediate_file_(target_filepath_ == base_filepath_),
patched_filepath_(target_filepath_ == base_filepath_
? base_filepath_ + kIntermediatePathSuffix
: target_filepath_) {}
cdc_(cdc) {}
virtual ~PatchTask() = default;
PatchTask(const PatchTask& other) = delete;
PatchTask& operator=(const PatchTask& other) = delete;
const ChangedFileInfo& File() const { return file_; }
const absl::Status& Status() const { return status_; }
// Task:
void ThreadRun(IsCancelledPredicate is_cancelled) override {
// Each PatchTask is queued twice, once to apply the patch and once to
// close and finalize the patched file.
switch (state_) {
case State::kPatching:
Patch();
state_ = State::kFinalizing;
break;
case State::kFinalizing:
Finalize();
state_ = State::kDone;
break;
default:
assert(!"Invalid state");
}
}
bool need_intermediate_file = target_filepath_ == base_filepath_;
std::string patched_filepath =
need_intermediate_file ? base_filepath_ + kIntermediatePathSuffix
: target_filepath_;
private:
void Patch() {
absl::StatusOr<FILE*> patched_fp = path::OpenFile(patched_filepath_, "wb");
if (!patched_fp.ok()) {
status_ = patched_fp.status();
absl::StatusOr<FILE*> patched_file = path::OpenFile(patched_filepath, "wb");
if (!patched_file.ok()) {
status_ = patched_file.status();
return;
}
patched_fp_ = *patched_fp;
// Receive diff stream from server and apply.
status_ =
cdc_->ReceiveDiffAndPatch(base_filepath_, patched_fp_, &is_executable_);
// The file is closed by Finalize() in a separate thread pool since fclose()
// takes a while on some systems.
}
void Finalize() {
if (patched_fp_) {
fclose(patched_fp_);
patched_fp_ = nullptr;
}
bool is_executable = false;
status_ = cdc_->ReceiveDiffAndPatch(base_filepath_, *patched_file,
&is_executable);
fclose(*patched_file);
if (!status_.ok()) {
// Some error occurred during Patch().
return;
}
// These bits are OR'ed on top of the mode bits.
uint16_t mode_or_bits = is_executable_ ? kExecutableBits : 0;
uint16_t mode_or_bits = is_executable ? kExecutableBits : 0;
// Store mode from the original base path.
path::Stats stats;
@@ -141,13 +93,13 @@ class PatchTask : public Task {
return;
}
if (need_intermediate_file_) {
if (need_intermediate_file) {
// Replace |base_filepath_| (==|target_filepath_|) by the intermediate
// file |patched_filepath|.
status_ = path::ReplaceFile(target_filepath_, patched_filepath_);
status_ = path::ReplaceFile(target_filepath_, patched_filepath);
if (!status_.ok()) {
status_ = WrapStatus(status_, "ReplaceFile() for '%s' by '%s' failed",
base_filepath_, patched_filepath_);
base_filepath_, patched_filepath);
return;
}
} else {
@@ -168,82 +120,11 @@ class PatchTask : public Task {
status_ = path::SetFileTime(target_filepath_, file_.client_modified_time);
}
const std::string base_filepath_;
const std::string target_filepath_;
const ChangedFileInfo file_;
CdcInterface* const cdc_;
const bool need_intermediate_file_ = false;
const std::string patched_filepath_;
FILE* patched_fp_ = nullptr;
bool is_executable_ = false;
absl::Status status_;
// This task is queued twice, once to patch and once to close and finalize the
// patched file.
enum class State { kPatching, kFinalizing, kDone };
State state_ = State::kPatching;
};
// Background task that closes a file and sets the mtime and perms. This is done
// in the background since closing a file might block for a long time.
class FinalizeCopiedFileTask : public Task {
public:
// Finalize |file| with given path |filepath|. |status| is the status from
// writing the file. On error, the file is only closed.
FinalizeCopiedFileTask(FILE* fp, FileInfo file, std::string filepath,
bool is_executable, absl::Status status)
: fp_(fp),
file_(std::move(file)),
filepath_(std::move(filepath)),
is_executable_(is_executable),
status_(status) {}
virtual ~FinalizeCopiedFileTask() = default;
FinalizeCopiedFileTask(const FinalizeCopiedFileTask& other) = delete;
FinalizeCopiedFileTask& operator=(const FinalizeCopiedFileTask& other) =
delete;
const absl::Status& Status() const { return status_; }
// Task:
void ThreadRun(IsCancelledPredicate is_cancelled) override {
assert(fp_);
fclose(fp_);
if (!status_.ok()) {
// Writing the file failed, nothing to finalize.
status_ = WrapStatus(status_, "Failed to write file %s", filepath_);
return;
}
// Set file write time.
status_ = path::SetFileTime(filepath_, file_.modified_time);
if (!status_.ok()) {
status_ =
WrapStatus(status_, "Failed to set file mod time for %s", filepath_);
return;
}
// Set executable bit, but just print warnings as it's not critical.
if (is_executable_) {
path::Stats stats;
status_ = path::GetStats(filepath_, &stats);
if (status_.ok()) {
status_ = path::ChangeMode(filepath_, stats.mode | kExecutableBits);
}
if (!status_.ok()) {
LOG_WARNING("Failed to set executable bit on '%s': %s", filepath_,
status_.ToString());
}
}
}
private:
FILE* const fp_ = nullptr;
const FileInfo file_;
const std::string filepath_;
const bool is_executable_;
std::string base_filepath_;
std::string target_filepath_;
ChangedFileInfo file_;
CdcInterface* cdc_;
absl::Status status_;
};
@@ -267,7 +148,10 @@ PathFilter::Rule::Type ToInternalType(
CdcRsyncServer::CdcRsyncServer() = default;
CdcRsyncServer::~CdcRsyncServer() = default;
CdcRsyncServer::~CdcRsyncServer() {
message_pump_.reset();
socket_.reset();
}
bool CdcRsyncServer::CheckComponents(
const std::vector<GameletComponent>& components) {
@@ -279,8 +163,8 @@ bool CdcRsyncServer::CheckComponents(
}
std::vector<GameletComponent> our_components;
status = GameletComponent::Get({path::Join(component_dir, kServerFilename)},
&our_components);
status = GameletComponent::Get(
{path::Join(component_dir, "cdc_rsync_server")}, &our_components);
if (!status.ok() || components != our_components) {
return false;
}
@@ -288,24 +172,22 @@ bool CdcRsyncServer::CheckComponents(
return true;
}
absl::Status CdcRsyncServer::Run() {
RETURN_IF_ERROR(Socket::Initialize(), "Failed to initialize sockets");
socket_finalizer_ = std::make_unique<SocketFinalizer>();
absl::Status CdcRsyncServer::Run(int port) {
socket_ = std::make_unique<ServerSocket>();
int port;
ASSIGN_OR_RETURN(port, socket_->StartListening(0),
"Failed to start listening for connections");
absl::Status status = socket_->StartListening(port);
if (!status.ok()) {
return WrapStatus(status, "Failed to start listening on port %i", port);
}
LOG_INFO("cdc_rsync_server listening on port %i", port);
// This is the marker for the client, so it knows it can connect.
// Print port first so the client can easily parse it when it sees "Server is
// listening" without dealing with half-transmitted data.
printf("Port %i: Server is listening\n", port);
printf("Server is listening\n");
fflush(stdout);
RETURN_IF_ERROR(socket_->WaitForConnection(),
"Failed to establish a connection");
status = socket_->WaitForConnection();
if (!status.ok()) {
return WrapStatus(status, "Failed to establish a connection");
}
message_pump_ = std::make_unique<MessagePump>(
socket_.get(),
@@ -313,7 +195,7 @@ absl::Status CdcRsyncServer::Run() {
message_pump_->StartMessagePump();
LOG_INFO("Client connected. Starting to sync.");
absl::Status status = Sync();
status = Sync();
if (!status.ok()) {
socket_->ShutdownSendingEnd().IgnoreError();
return status;
@@ -602,7 +484,7 @@ absl::Status CdcRsyncServer::CreateMissingDirs() {
template <typename T>
absl::Status CdcRsyncServer::SendFileIndices(const char* file_type,
const std::vector<T>& files) {
LOG_INFO("Sending indices of %s files to client", file_type);
LOG_INFO("Sending indices of missing files to client");
constexpr char error_fmt[] = "Failed to send indices of %s files.";
AddFileIndicesResponse response;
@@ -659,8 +541,6 @@ absl::Status CdcRsyncServer::HandleSendMissingFileData() {
}
}
Threadpool finalize_pool(kNumFinalizerThreads);
for (uint32_t server_index = 0; server_index < diff_.missing_files.size();
server_index++) {
const FileInfo& file = diff_.missing_files[server_index];
@@ -680,10 +560,10 @@ absl::Status CdcRsyncServer::HandleSendMissingFileData() {
request.server_index(), server_index);
}
// Remove |filepath| if it is a directory.
if (path::DirExists(filepath)) {
// Verify that there is no directory existing with the same name.
if (path::Exists(filepath) && path::DirExists(filepath)) {
assert(!diff_.extraneous_dirs.empty());
status = path::RemoveFile(filepath);
absl::Status status = path::RemoveFile(filepath);
if (!status.ok()) {
return WrapStatus(
status, "Failed to remove folder '%s' before creating file '%s'",
@@ -730,25 +610,27 @@ absl::Status CdcRsyncServer::HandleSendMissingFileData() {
}
status = path::StreamWriteFileContents(*fp, handler);
finalize_pool.QueueTask(std::make_unique<FinalizeCopiedFileTask>(
*fp, file, filepath, is_executable, status));
finalize_pool.WaitForQueuedTasksAtMost(kMaxQueueSize);
// Drain finalize pool for the last file.
if (server_index + 1 == diff_.missing_files.size()) {
finalize_pool.Wait();
fclose(*fp);
if (!status.ok()) {
return WrapStatus(status, "Failed to write file %s", filepath);
}
// Check the results of completed tasks.
for (std::unique_ptr<Task> task = finalize_pool.TryGetCompletedTask();
task != nullptr; task = finalize_pool.TryGetCompletedTask()) {
const FinalizeCopiedFileTask* finalize_task =
static_cast<FinalizeCopiedFileTask*>(task.get());
if (!finalize_task->Status().ok()) {
// Close and finish files that have already been copied, so we don't
// discard several already copied files because one failed.
finalize_pool.Wait();
return finalize_task->Status();
// Set file write time.
status = path::SetFileTime(filepath, file.modified_time);
if (!status.ok()) {
return WrapStatus(status, "Failed to set file mod time for %s", filepath);
}
// Set executable bit, but just print warnings as it's not critical.
if (is_executable) {
path::Stats stats;
status = path::GetStats(filepath, &stats);
if (status.ok()) {
status = path::ChangeMode(filepath, stats.mode | kExecutableBits);
}
if (!status.ok()) {
LOG_WARNING("Failed to set executable bit on '%s': %s", filepath,
status.ToString());
}
}
}
@@ -789,20 +671,9 @@ absl::Status CdcRsyncServer::SyncChangedFiles() {
// Pipeline sending signatures and patching files:
// MAIN THREAD: Send signatures to client.
// Only sends to the socket.
// PATCHER THREAD: Receive diffs from client and create patch file.
// WORKER THREAD: Receive diffs from client and patch file.
// Only reads from the socket.
// FINALIZER THREADS: Close patched files and finalize them.
Threadpool patch_pool(1);
Threadpool finalize_pool(kNumFinalizerThreads);
// Forward finished patch task immediately to finalize pool.
patch_pool.SetTaskCompletedCallback(
[&finalize_pool](std::unique_ptr<Task> task) {
// Spin if there are too many outstanding tasks, in order to limit the
// max number of outstanding tasks.
finalize_pool.QueueTask(std::move(task));
finalize_pool.WaitForQueuedTasksAtMost(kMaxQueueSize);
});
Threadpool pool(1);
for (uint32_t server_index = 0; server_index < diff_.changed_files.size();
server_index++) {
@@ -828,28 +699,25 @@ absl::Status CdcRsyncServer::SyncChangedFiles() {
}
// Queue patching task.
patch_pool.QueueTask(std::make_unique<PatchTask>(
base_filepath, target_filepath, file, &cdc));
pool.QueueTask(std::make_unique<PatchTask>(base_filepath, target_filepath,
file, &cdc));
// Drain pools for the last file.
// Wait for the last file to finish.
if (server_index + 1 == diff_.changed_files.size()) {
patch_pool.Wait();
finalize_pool.Wait();
pool.Wait();
}
// Check the results of completed tasks.
for (std::unique_ptr<Task> task = finalize_pool.TryGetCompletedTask();
task != nullptr; task = finalize_pool.TryGetCompletedTask()) {
const PatchTask* patch_task = static_cast<PatchTask*>(task.get());
std::unique_ptr<Task> task = pool.TryGetCompletedTask();
while (task) {
PatchTask* patch_task = static_cast<PatchTask*>(task.get());
const std::string& task_path = patch_task->File().filepath;
if (!patch_task->Status().ok()) {
// Close and finish files that have already been synced, so we don't
// discard several already synced files because one failed.
finalize_pool.Wait();
return WrapStatus(patch_task->Status(), "Failed to patch file '%s'",
task_path);
}
LOG_INFO("Finished patching file %s", task_path.c_str());
task = pool.TryGetCompletedTask();
}
}
cdc_rsync_server/cdc_rsync_server.h
+3 -7
View File
@@ -32,7 +32,6 @@ namespace cdc_ft {
class MessagePump;
class ServerSocket;
class SocketFinalizer;
class CdcRsyncServer {
public:
@@ -43,10 +42,9 @@ class CdcRsyncServer {
// up-to-date by checking their sizes and timestamps.
bool CheckComponents(const std::vector<GameletComponent>& components);
// Listens to any available port, accepts a connection from the client and
// runs the rsync procedure. Prints "Port <n>: Server is listening" to stdout,
// so the client can retrieve the selected port.
absl::Status Run();
// Listens to |port|, accepts a connection from the client and runs the rsync
// procedure.
absl::Status Run(int port);
// Returns the verbosity sent from the client. 0 by default.
int GetVerbosity() const { return verbosity_; }
@@ -92,8 +90,6 @@ class CdcRsyncServer {
// Used to toggle decompression.
void Thread_OnPackageReceived(PacketType type);
// The order determines the correct destruction order, so keep it!
std::unique_ptr<SocketFinalizer> socket_finalizer_;
std::unique_ptr<ServerSocket> socket_;
std::unique_ptr<MessagePump> message_pump_;
cdc_rsync_server/cdc_rsync_server.vcxproj
+1 -1
View File
@@ -49,7 +49,7 @@
<PropertyGroup>
<BazelTargets>//cdc_rsync_server:cdc_rsync_server</BazelTargets>
<BazelOutputFile>cdc_rsync_server</BazelOutputFile>
<BazelIncludePaths>..\;..\third_party\absl;..\bazel-cdc-file-transfer\external\com_github_blake3\c;..\bazel-cdc-file-transfer\external\com_github_zstd;..\third_party\googletest\googletest\include;..\bazel-cdc-file-transfer\external\com_google_protobuf\src;..\bazel-bin</BazelIncludePaths>
<BazelIncludePaths>..\;..\third_party\absl;..\bazel-cdc-file-transfer\external\com_github_blake3\c;..\bazel-stadia-file-transfer\external\com_github_zstd\lib;..\third_party\googletest\googletest\include;..\bazel-cdc-file-transfer\external\com_google_protobuf\src</BazelIncludePaths>
</PropertyGroup>
<Import Project="..\NMakeBazelProject.targets" />
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
cdc_rsync_server/file_deleter_and_sender_test.cc
+1 -1
View File
@@ -14,8 +14,8 @@
#include "cdc_rsync_server/file_deleter_and_sender.h"
#include "cdc_rsync/base/fake_socket.h"
#include "cdc_rsync/base/message_pump.h"
#include "common/fake_socket.h"
#include "common/log.h"
#include "common/path.h"
#include "common/status_test_macros.h"
cdc_rsync_server/main.cc
+12 -13
View File
@@ -14,7 +14,6 @@
#include "cdc_rsync/base/server_exit_code.h"
#include "cdc_rsync_server/cdc_rsync_server.h"
#include "common/build_version.h"
#include "common/gamelet_component.h"
#include "common/log.h"
#include "common/status.h"
@@ -61,33 +60,33 @@ ServerExitCode GetExitCode(const absl::Status& status) {
} // namespace cdc_ft
int main(int argc, const char** argv) {
if (argc < 5) {
printf(R"("cdc_rsync_server - Remote component of cdc_rsync. Version: %s
Usage: cdc_rsync_server <build_version> cdc_rsync_server <size> <modified_time>
<build_version> build version embedded in the component
<size> file size of the component
<modified_time> timestamp (Unix epoch) of the component)",
BUILD_VERSION);
if (argc < 2) {
printf("Usage: cdc_rsync_server <port> cdc_rsync_server <size> <time> \n");
printf(" where <size> and <time> are the file size and modified\n");
printf(" timestamp (Unix epoch) of the corresponding component.\n");
return cdc_ft::kServerExitCodeGenericStartup;
}
int port = atoi(argv[1]);
if (port == 0) {
SendErrorMessage(absl::StrFormat("Invalid port '%s'", argv[1]).c_str());
return cdc_ft::kServerExitCodeGenericStartup;
}
// The rest is expected to be sets of gamelet component info consisting of
// (version, filename, size, modified_time). This is used check whether the
// (filename, filesize, modified_time). This is used check whether the
// components are up-to-date.
std::vector<cdc_ft::GameletComponent> components =
cdc_ft::GameletComponent::FromCommandLineArgs(argc - 1, argv + 1);
cdc_ft::GameletComponent::FromCommandLineArgs(argc - 2, argv + 2);
cdc_ft::Log::Initialize(
std::make_unique<cdc_ft::ConsoleLog>(cdc_ft::LogLevel::kWarning));
cdc_ft::CdcRsyncServer server;
if (!server.CheckComponents(components)) {
return cdc_ft::kServerExitCodeOutOfDate;
}
absl::Status status = server.Run();
absl::Status status = server.Run(port);
if (status.ok()) {
return 0;
}
cdc_rsync_server/server_socket.cc
+214
View File
@@ -0,0 +1,214 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "cdc_rsync_server/server_socket.h"
#include <netinet/in.h>
#include <sys/socket.h>
#include <unistd.h>
#include <cerrno>
#include "common/log.h"
#include "common/status.h"
namespace cdc_ft {
namespace {
int kInvalidFd = -1;
// Keep re-evaluating the expression |x| while it returns EINTR.
#define HANDLE_EINTR(x) \
({ \
decltype(x) eintr_wrapper_result; \
do { \
eintr_wrapper_result = (x); \
} while (eintr_wrapper_result == -1 && errno == EINTR); \
eintr_wrapper_result; \
})
} // namespace
ServerSocket::ServerSocket()
: Socket(), listen_sockfd_(kInvalidFd), conn_sockfd_(kInvalidFd) {}
ServerSocket::~ServerSocket() {
Disconnect();
StopListening();
}
absl::Status ServerSocket::StartListening(int port) {
if (listen_sockfd_ != kInvalidFd) {
return MakeStatus("Already listening");
}
LOG_DEBUG("Open socket");
listen_sockfd_ = socket(AF_INET, SOCK_STREAM, 0);
if (listen_sockfd_ < 0) {
listen_sockfd_ = kInvalidFd;
return MakeStatus("socket() failed: %s", strerror(errno));
}
// If the program terminates abnormally, the socket might remain in a
// TIME_WAIT state and report "address already in use" on bind(). Setting
// SO_REUSEADDR works around that. See
// https://hea-www.harvard.edu/~fine/Tech/addrinuse.html
int enable = 1;
if (setsockopt(listen_sockfd_, SOL_SOCKET, SO_REUSEADDR, &enable,
sizeof(enable)) < 0) {
LOG_DEBUG("setsockopt() failed");
}
LOG_DEBUG("Bind socket");
sockaddr_in serv_addr;
memset(&serv_addr, 0, sizeof(serv_addr));
serv_addr.sin_family = AF_INET;
serv_addr.sin_addr.s_addr = INADDR_ANY;
serv_addr.sin_port = htons(port);
if (bind(listen_sockfd_, (struct sockaddr*)&serv_addr, sizeof(serv_addr)) <
0) {
absl::Status status =
MakeStatus("bind() to port %i failed: %s", port, strerror(errno));
if (errno == EADDRINUSE) {
// Happens when two instances are run at the same time. Help callers to
// print reasonable errors.
status = SetTag(status, Tag::kAddressInUse);
}
close(listen_sockfd_);
listen_sockfd_ = kInvalidFd;
return status;
}
LOG_DEBUG("Listen");
listen(listen_sockfd_, 1);
return absl::OkStatus();
}
void ServerSocket::StopListening() {
if (listen_sockfd_ != kInvalidFd) {
close(listen_sockfd_);
listen_sockfd_ = kInvalidFd;
}
LOG_INFO("Stopped listening.");
}
absl::Status ServerSocket::WaitForConnection() {
if (conn_sockfd_ != kInvalidFd) {
return MakeStatus("Already connected");
}
sockaddr_in cli_addr;
socklen_t cli_len = sizeof(cli_addr);
conn_sockfd_ = accept(listen_sockfd_, (struct sockaddr*)&cli_addr, &cli_len);
if (conn_sockfd_ < 0) {
conn_sockfd_ = kInvalidFd;
return MakeStatus("accept() failed: %s", strerror(errno));
}
LOG_DEBUG("Client connected");
return absl::OkStatus();
}
void ServerSocket::Disconnect() {
if (conn_sockfd_ != kInvalidFd) {
close(conn_sockfd_);
conn_sockfd_ = kInvalidFd;
}
LOG_INFO("Disconnected");
}
absl::Status ServerSocket::ShutdownSendingEnd() {
int result = shutdown(conn_sockfd_, SHUT_WR);
if (result != 0) {
return MakeStatus("shutdown() failed: %s", strerror(errno));
}
return absl::OkStatus();
}
absl::Status ServerSocket::Send(const void* buffer, size_t size) {
const uint8_t* curr_ptr = reinterpret_cast<const uint8_t*>(buffer);
ssize_t bytes_left = size;
while (bytes_left > 0) {
ssize_t bytes_written =
HANDLE_EINTR(send(conn_sockfd_, curr_ptr, bytes_left, /*flags*/ 0));
if (bytes_written < 0) {
if (errno == EAGAIN || errno == EWOULDBLOCK) {
// Shouldn't happen as the socket should be blocking.
LOG_DEBUG("Socket would block");
continue;
}
return MakeStatus("write() to fd %i failed: %s", conn_sockfd_,
strerror(errno));
}
bytes_left -= bytes_written;
curr_ptr += bytes_written;
}
return absl::OkStatus();
}
absl::Status ServerSocket::Receive(void* buffer, size_t size,
bool allow_partial_read,
size_t* bytes_received) {
*bytes_received = 0;
if (size == 0) {
return absl::OkStatus();
}
uint8_t* curr_ptr = reinterpret_cast<uint8_t*>(buffer);
ssize_t bytes_left = size;
while (bytes_left > 0) {
ssize_t bytes_read =
HANDLE_EINTR(recv(conn_sockfd_, curr_ptr, bytes_left, /*flags*/ 0));
if (bytes_read < 0) {
if (errno == EAGAIN || errno == EWOULDBLOCK) {
// Shouldn't happen as the socket should be blocking.
LOG_DEBUG("Socket would block");
continue;
}
return MakeStatus("recv() from fd %i failed: %s", conn_sockfd_,
strerror(errno));
}
bytes_left -= bytes_read;
*bytes_received += bytes_read;
curr_ptr += bytes_read;
if (bytes_read == 0) {
// EOF. Make sure we're not in the middle of a message.
if (bytes_left < static_cast<ssize_t>(size)) {
return MakeStatus("EOF after partial read");
}
LOG_DEBUG("EOF() detected");
return SetTag(MakeStatus("EOF detected"), Tag::kSocketEof);
}
if (allow_partial_read) {
break;
}
}
return absl::OkStatus();
}
} // namespace cdc_ft
common/server_socket.h → cdc_rsync_server/server_socket.h
+8 -23
View File
@@ -14,14 +14,11 @@
* limitations under the License.
*/
#ifndef COMMON_SERVER_SOCKET_H_
#define COMMON_SERVER_SOCKET_H_
#ifndef CDC_RSYNC_SERVER_SERVER_SOCKET_H_
#define CDC_RSYNC_SERVER_SERVER_SOCKET_H_
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "common/socket.h"
struct addrinfo;
#include "cdc_rsync/base/socket.h"
namespace cdc_ft {
@@ -30,19 +27,8 @@ class ServerSocket : public Socket {
ServerSocket();
~ServerSocket();
// Returns an available ephemeral port that can be used as a listening port.
// Note that calling this function, followed by StartListening() or similar,
// is slightly racy as another process might use the port in the meantime.
// However, the OS usually returns ephemeral ports in a round-robin manner,
// and ports remain in TIME_WAIT state for a while, which may block other apps
// from reusing the port. Hence, the chances of races are small. Nevertheless,
// consider calling StartListening() with zero |port| if possible.
static absl::StatusOr<int> FindAvailablePort();
// Starts listening for connections on |port|.
// Passing 0 as port will bind to any available port.
// Returns the port that was bound to.
absl::StatusOr<int> StartListening(int port);
absl::Status StartListening(int port);
// Stops listening for connections. No-op if already stopped/never started.
void StopListening();
@@ -64,12 +50,11 @@ class ServerSocket : public Socket {
size_t* bytes_received) override;
private:
// Called by StartListening() for a specific IPV4 or IPV6 |addr_info|.
// Passing 0 as port will bind to any available port.
// Returns the port that was bound to.
absl::StatusOr<int> StartListeningInternal(int port, addrinfo* addr);
// Listening socket file descriptor (where new connections are accepted).
int listen_sockfd_;
std::unique_ptr<struct ServerSocketInfo> socket_info_;
// Connection socket file descriptor (where data is sent to/received from).
int conn_sockfd_;
};
} // namespace cdc_ft
cdc_rsync_server/unzstd_stream.cc
+15 -15
View File
@@ -14,7 +14,7 @@
#include "cdc_rsync_server/unzstd_stream.h"
#include "common/socket.h"
#include "cdc_rsync/base/socket.h"
#include "common/status.h"
namespace cdc_ft {
@@ -41,20 +41,7 @@ absl::Status UnzstdStream::Read(void* out_buffer, size_t out_size,
ZSTD_outBuffer output = {out_buffer, out_size, 0};
while (output.pos < output.size && !*eof) {
// Decompress.
size_t ret = ZSTD_decompressStream(dctx_, &output, &input_);
if (ZSTD_isError(ret)) {
return MakeStatus("Failed to decompress data: %s",
ZSTD_getErrorName(ret));
}
*eof = (ret == 0);
if (*eof && input_.pos < input_.size) {
return MakeStatus("EOF with %u bytes input data available",
input_.size - input_.pos);
}
if (input_.pos == input_.size && output.pos < output.size && !*eof) {
if (input_.pos == input_.size) {
// Read more compressed input data.
// Allow partial reads since the stream could end any time.
size_t in_size;
@@ -67,6 +54,19 @@ absl::Status UnzstdStream::Read(void* out_buffer, size_t out_size,
input_.pos = 0;
input_.size = in_size;
}
// Decompress.
size_t ret = ZSTD_decompressStream(dctx_, &output, &input_);
if (ZSTD_isError(ret)) {
return MakeStatus("Failed to decompress data: %s",
ZSTD_getErrorName(ret));
}
*eof = (ret == 0);
if (*eof && input_.pos < input_.size) {
return MakeStatus("EOF with %u bytes input data available",
input_.size - input_.pos);
}
}
// Output buffer is full or eof.
cdc_stream/BUILD
-31
View File
@@ -12,7 +12,6 @@ cc_binary(
":start_command",
":start_service_command",
":stop_command",
":stop_service_command",
"//common:log",
"//common:path",
],
@@ -24,7 +23,6 @@ cc_library(
hdrs = ["base_command.h"],
deps = [
"//absl_helper:jedec_size_flag",
"//common:port_range_parser",
"@com_github_lyra//:lyra",
"@com_google_absl//absl/status",
"@com_google_absl//absl/strings:str_format",
@@ -42,24 +40,11 @@ cc_library(
],
)
cc_library(
name = "stop_service_command",
srcs = ["stop_service_command.cc"],
hdrs = ["stop_service_command.h"],
deps = [
":asset_stream_config",
":background_service_client",
":base_command",
":session_management_server",
],
)
cc_library(
name = "start_command",
srcs = ["start_command.cc"],
hdrs = ["start_command.h"],
deps = [
":background_service_client",
":base_command",
":local_assets_stream_manager_client",
":session_management_server",
@@ -93,18 +78,6 @@ cc_library(
],
)
cc_library(
name = "background_service_client",
srcs = ["background_service_client.cc"],
hdrs = ["background_service_client.h"],
deps = [
"//common:grpc_status",
"//common:status_macros",
"//proto:background_service_grpc_proto",
"@com_google_absl//absl/status",
],
)
cc_library(
name = "asset_stream_server",
srcs = [
@@ -139,8 +112,6 @@ cc_library(
deps = [
":base_command",
":multi_session",
":session_management_server",
"//absl_helper:jedec_size_flag",
"//common:log",
"//common:path",
"//common:status_macros",
@@ -209,12 +180,10 @@ cc_library(
"//common:file_watcher",
"//common:log",
"//common:path",
"//common:path_filter",
"//common:port_manager",
"//common:process",
"//common:remote_util",
"//common:sdk_util",
"//common:server_socket",
"//common:status_macros",
"//common:stopwatch",
"//data_store:disk_data_store",
cdc_stream/asset_stream_config.cc
+13 -24
View File
@@ -20,8 +20,6 @@
#include "absl/strings/str_join.h"
#include "absl_helper/jedec_size_flag.h"
#include "cdc_stream/base_command.h"
#include "cdc_stream/multi_session.h"
#include "cdc_stream/session_management_server.h"
#include "common/buffer.h"
#include "common/path.h"
#include "common/status_macros.h"
@@ -43,22 +41,6 @@ AssetStreamConfig::~AssetStreamConfig() = default;
void AssetStreamConfig::RegisterCommandLineFlags(lyra::command& cmd,
BaseCommand& base_command) {
service_port_ = SessionManagementServer::kDefaultServicePort;
cmd.add_argument(lyra::opt(service_port_, "port")
.name("--service-port")
.help("Local port to use while connecting to the local "
"asset stream service, default: " +
std::to_string(service_port_)));
cmd.add_argument(lyra::opt(base_command.PortRangeParser(
"--forward-port",
&session_cfg_.deprecated_forward_port_first,
&session_cfg_.deprecated_forward_port_last),
"port")
.name("--forward-port")
.help("[Deprecated, ignored] TCP port or range used for "
"SSH port forwarding"));
session_cfg_.verbosity = kDefaultVerbosity;
cmd.add_argument(lyra::opt(session_cfg_.verbosity, "num")
.name("--verbosity")
@@ -145,6 +127,14 @@ void AssetStreamConfig::RegisterCommandLineFlags(lyra::command& cmd,
.name("--dev-user-host")
.help("Username and host to stream to. See also --dev-src-dir."));
dev_target_.ssh_port = RemoteUtil::kDefaultSshPort;
cmd.add_argument(
lyra::opt(dev_target_.ssh_port, "port")
.name("--dev-ssh-port")
.help("SSH port to use for the connection to the host, default: " +
std::to_string(RemoteUtil::kDefaultSshPort) +
". See also --dev-src-dir."));
cmd.add_argument(
lyra::opt(dev_target_.ssh_command, "cmd")
.name("--dev-ssh-command")
@@ -152,9 +142,9 @@ void AssetStreamConfig::RegisterCommandLineFlags(lyra::command& cmd,
"connection to the host. See also --dev-src-dir."));
cmd.add_argument(
lyra::opt(dev_target_.sftp_command, "cmd")
.name("--dev-sftp-command")
.help("Sftp command and extra flags to use for the "
lyra::opt(dev_target_.scp_command, "cmd")
.name("--dev-scp-command")
.help("Scp command and extra flags to use for the "
"connection to the host. See also --dev-src-dir."));
cmd.add_argument(
@@ -184,7 +174,6 @@ absl::Status AssetStreamConfig::LoadFromFile(const std::string& path) {
} \
} while (0)
ASSIGN_VAR(service_port_, "service-port", Int);
ASSIGN_VAR(session_cfg_.verbosity, "verbosity", Int);
ASSIGN_VAR(session_cfg_.fuse_debug, "debug", Bool);
ASSIGN_VAR(session_cfg_.fuse_singlethreaded, "singlethreaded", Bool);
@@ -223,7 +212,6 @@ absl::Status AssetStreamConfig::LoadFromFile(const std::string& path) {
std::string AssetStreamConfig::ToString() {
std::ostringstream ss;
ss << "service-port = " << service_port_ << std::endl;
ss << "verbosity = " << session_cfg_.verbosity
<< std::endl;
ss << "debug = " << session_cfg_.fuse_debug
@@ -247,9 +235,10 @@ std::string AssetStreamConfig::ToString() {
<< session_cfg_.file_change_wait_duration_ms << std::endl;
ss << "dev-src-dir = " << dev_src_dir_ << std::endl;
ss << "dev-user-host = " << dev_target_.user_host << std::endl;
ss << "dev-ssh-port = " << dev_target_.ssh_port << std::endl;
ss << "dev-ssh-command = " << dev_target_.ssh_command
<< std::endl;
ss << "dev-sftp-command = " << dev_target_.sftp_command
ss << "dev-scp-command = " << dev_target_.scp_command
<< std::endl;
ss << "dev-mount-dir = " << dev_target_.mount_dir << std::endl;
return ss.str();
cdc_stream/asset_stream_config.h
+6 -19
View File
@@ -48,21 +48,18 @@ class AssetStreamConfig {
// Loads a configuration from the JSON file at |path| and overrides any config
// values that are set in this file. Sample json file:
// {
// "service-port":44432
// "forward-port-first":"44433"
// "forward-port-last":"44442"
// "verbosity":3,
// "debug":0,
// "singlethreaded":0,
// "stats":0,
// "quiet":0,
// "check":0,
// "log-to-stdout":0,
// "cache-capacity":"150G",
// "cleanup-timeout":300,
// "access-idle-timeout":5,
// "manifest-updater-threads":4,
// "file-change-wait-duration-ms":500
// "log_to_stdout":0,
// "cache_capacity":"150G",
// "cleanup_timeout":300,
// "access_idle_timeout":5,
// "manifest_updater_threads":4,
// "file_change_wait_duration_ms":500
// }
// Returns NotFoundError if the file does not exist.
// Returns InvalidArgumentError if the file is not valid JSON.
@@ -79,9 +76,6 @@ class AssetStreamConfig {
// read from the JSON file.
std::string GetFlagReadErrors();
// Gets the port to use for the asset streaming service.
uint16_t service_port() const { return service_port_; }
// Session configuration.
const SessionConfig& session_cfg() const { return session_cfg_; }
@@ -97,13 +91,6 @@ class AssetStreamConfig {
bool log_to_stdout() const { return log_to_stdout_; }
private:
// Jedec parser for Lyra options. Usage:
// lyra::opt(JedecParser("size-flag", &size_bytes), "bytes"))
// Sets jedec_parse_error_ on error, Lyra doesn't support errors from lambdas.
std::function<void(const std::string&)> JedecParser(const char* flag_name,
uint64_t* bytes);
uint16_t service_port_ = 0;
SessionConfig session_cfg_;
bool log_to_stdout_ = false;
cdc_stream/background_service_client.cc
-56
View File
@@ -1,56 +0,0 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "cdc_stream/background_service_client.h"
#include "absl/status/status.h"
#include "common/grpc_status.h"
#include "common/status_macros.h"
#include "grpcpp/channel.h"
namespace cdc_ft {
using GetPidResponse = backgroundservice::GetPidResponse;
using EmptyProto = google::protobuf::Empty;
BackgroundServiceClient::BackgroundServiceClient(
std::shared_ptr<grpc::Channel> channel) {
stub_ = BackgroundService::NewStub(std::move(channel));
}
BackgroundServiceClient::~BackgroundServiceClient() = default;
absl::Status BackgroundServiceClient::Exit() {
EmptyProto request;
EmptyProto response;
grpc::ClientContext context;
return ToAbslStatus(stub_->Exit(&context, request, &response));
}
absl::StatusOr<int> BackgroundServiceClient::GetPid() {
EmptyProto request;
GetPidResponse response;
grpc::ClientContext context;
RETURN_IF_ERROR(ToAbslStatus(stub_->GetPid(&context, request, &response)));
return response.pid();
}
absl::Status BackgroundServiceClient::IsHealthy() {
EmptyProto request;
EmptyProto response;
grpc::ClientContext context;
return ToAbslStatus(stub_->HealthCheck(&context, request, &response));
}
} // namespace cdc_ft
cdc_stream/background_service_client.h
-56
View File
@@ -1,56 +0,0 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef CDC_STREAM_BACKGROUND_SERVICE_CLIENT_H_
#define CDC_STREAM_BACKGROUND_SERVICE_CLIENT_H_
#include <memory>
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "proto/background_service.grpc.pb.h"
namespace grpc_impl {
class Channel;
}
namespace cdc_ft {
// gRpc client for managing the asset streaming service.
class BackgroundServiceClient {
public:
// |channel| is a grpc channel to use.
explicit BackgroundServiceClient(std::shared_ptr<grpc::Channel> channel);
~BackgroundServiceClient();
// Initialize service shutdown.
absl::Status Exit();
// Returns the PID of the service process.
absl::StatusOr<int> GetPid();
// Verifies that the service is running and able to take requests.
absl::Status IsHealthy();
private:
using BackgroundService = backgroundservice::BackgroundService;
std::unique_ptr<BackgroundService::Stub> stub_;
};
} // namespace cdc_ft
#endif // CDC_STREAM_BACKGROUND_SERVICE_CLIENT_H_
cdc_stream/background_service_impl.cc
+6 -14
View File
@@ -23,32 +23,24 @@ namespace cdc_ft {
BackgroundServiceImpl::BackgroundServiceImpl() {}
BackgroundServiceImpl::~BackgroundServiceImpl() {
if (exit_thread_) {
exit_thread_->join();
exit_thread_.reset();
}
}
BackgroundServiceImpl::~BackgroundServiceImpl() = default;
void BackgroundServiceImpl::SetExitCallback(ExitCallback exit_callback) {
exit_callback_ = std::move(exit_callback);
}
grpc::Status BackgroundServiceImpl::Exit(grpc::ServerContext* context,
const EmptyProto* request,
EmptyProto* response) {
const ExitRequest* request,
ExitResponse* response) {
LOG_INFO("RPC:Exit");
if (exit_callback_ && !exit_thread_) {
// Fire up a thread so call the callback, since shutting down a server
// won't finish until all RPCs are done.
exit_thread_ =
std::make_unique<std::thread>([cb = &exit_callback_]() { (*cb)(); });
if (exit_callback_) {
return ToGrpcStatus(exit_callback_());
}
return grpc::Status::OK;
}
grpc::Status BackgroundServiceImpl::GetPid(grpc::ServerContext* context,
const EmptyProto* request,
const GetPidRequest* request,
GetPidResponse* response) {
LOG_INFO("RPC:GetPid");
response->set_pid(static_cast<int32_t>(Util::GetPid()));
cdc_stream/background_service_impl.h
+7 -7
View File
@@ -17,9 +17,6 @@
#ifndef CDC_STREAM_BACKGROUND_SERVICE_IMPL_H_
#define CDC_STREAM_BACKGROUND_SERVICE_IMPL_H_
#include <memory>
#include <thread>
#include "absl/status/status.h"
#include "cdc_stream/background_service_impl.h"
#include "cdc_stream/session_management_server.h"
@@ -33,6 +30,9 @@ namespace cdc_ft {
class BackgroundServiceImpl final
: public backgroundservice::BackgroundService::Service {
public:
using ExitRequest = backgroundservice::ExitRequest;
using ExitResponse = backgroundservice::ExitResponse;
using GetPidRequest = backgroundservice::GetPidRequest;
using GetPidResponse = backgroundservice::GetPidResponse;
using EmptyProto = google::protobuf::Empty;
@@ -43,10 +43,11 @@ class BackgroundServiceImpl final
using ExitCallback = std::function<absl::Status()>;
void SetExitCallback(ExitCallback exit_callback);
grpc::Status Exit(grpc::ServerContext* context, const EmptyProto* request,
EmptyProto* response) override;
grpc::Status Exit(grpc::ServerContext* context, const ExitRequest* request,
ExitResponse* response) override;
grpc::Status GetPid(grpc::ServerContext* context, const EmptyProto* request,
grpc::Status GetPid(grpc::ServerContext* context,
const GetPidRequest* request,
GetPidResponse* response) override;
grpc::Status HealthCheck(grpc::ServerContext* context,
@@ -55,7 +56,6 @@ class BackgroundServiceImpl final
private:
ExitCallback exit_callback_;
std::unique_ptr<std::thread> exit_thread_;
};
} // namespace cdc_ft
cdc_stream/base_command.cc
+4 -17
View File
@@ -15,9 +15,7 @@
#include "cdc_stream/base_command.h"
#include "absl/strings/str_format.h"
#include "absl/strings/str_split.h"
#include "absl_helper/jedec_size_flag.h"
#include "common/port_range_parser.h"
#include "lyra/lyra.hpp"
namespace cdc_ft {
@@ -46,7 +44,8 @@ void BaseCommand::Register(lyra::cli& cli) {
std::function<void(const std::string&)> BaseCommand::JedecParser(
const char* flag_name, uint64_t* bytes) {
return [flag_name, bytes, error = &parse_error_](const std::string& value) {
return [flag_name, bytes,
error = &jedec_parse_error_](const std::string& value) {
JedecSize size;
if (AbslParseFlag(value, &size, error)) {
*bytes = size.Size();
@@ -57,18 +56,6 @@ std::function<void(const std::string&)> BaseCommand::JedecParser(
};
}
std::function<void(const std::string&)> BaseCommand::PortRangeParser(
const char* flag_name, uint16_t* first, uint16_t* last) {
return [flag_name, first, last,
error = &parse_error_](const std::string& value) {
if (!port_range::Parse(value.c_str(), first, last)) {
*error = absl::StrFormat(
"Failed to parse %s=%s, expected <port> or <port1>-<port2>",
flag_name, value);
}
};
}
std::function<void(const std::string&)> BaseCommand::PosArgValidator(
std::string* str) {
return [str, invalid_arg = &invalid_arg_](const std::string& value) {
@@ -96,8 +83,8 @@ void BaseCommand::CommandHandler(const lyra::group& g) {
return;
}
if (!parse_error_.empty()) {
std::cerr << "Error: " << parse_error_ << std::endl;
if (!jedec_parse_error_.empty()) {
std::cerr << "Error: " << jedec_parse_error_ << std::endl;
*exit_code_ = 1;
return;
}
cdc_stream/base_command.h
+2 -9
View File
@@ -48,13 +48,6 @@ class BaseCommand {
std::function<void(const std::string&)> JedecParser(const char* flag_name,
uint64_t* bytes);
// Parser for single ports "123" or port ranges "123-234". Usage:
// lyra::opt(PortRangeParser("port-flag", &first, &last), "port"))
// Automatically reports a parse failure on error.
std::function<void(const std::string&)> PortRangeParser(const char* flag_name,
uint16_t* first,
uint16_t* last);
// Validator that should be used for all positional arguments. Lyra interprets
// -u, --unknown_flag as positional argument. This validator makes sure that
// a positional argument starting with - is reported as an error. Otherwise,
@@ -89,9 +82,9 @@ class BaseCommand {
// Extraneous positional args. Gets reported as error if present.
std::string extra_positional_arg_;
// Errors from custom flag parsers, e.g. JEDEC sizes or port ranges.
// Errors from parsing JEDEC sizes.
// Works around Lyra not accepting errors from parsers.
std::string parse_error_;
std::string jedec_parse_error_;
};
} // namespace cdc_ft
cdc_stream/cdc_fuse_manager.cc
+62 -130
View File
@@ -16,7 +16,6 @@
#include "absl/strings/match.h"
#include "absl/strings/str_format.h"
#include "absl/strings/str_split.h"
#include "cdc_fuse_fs/constants.h"
#include "common/gamelet_component.h"
#include "common/log.h"
@@ -27,33 +26,14 @@
namespace cdc_ft {
namespace {
constexpr char kExeFilename[] = "cdc_stream.exe";
constexpr char kFuseFilename[] = "cdc_fuse_fs";
constexpr char kLibFuseFilename[] = "libfuse.so";
constexpr char kFuseStdoutPrefix[] = "cdc_fuse_fs_stdout";
constexpr char kRemoteToolsBinDir[] = ".cache/cdc-file-transfer/bin/";
constexpr char kRemoteToolsBinDir[] = "~/.cache/cdc-file-transfer/bin/";
// Cache directory on the gamelet to store data chunks.
constexpr char kCacheDir[] = "~/.cache/cdc-file-transfer/chunks";
// Parses the port from the FUSE stdout when FUSE is up-to-date. In that case,
// the expected stdout is similar to "Port 12345 cdc_fuse_fs is up-to-date".
absl::StatusOr<int> ParsePort(const std::string& fuse_stdout) {
// Search backwards until we find "Port ".
size_t port_pos = fuse_stdout.find(kFusePortPrefix);
if (port_pos == std::string::npos) {
return MakeStatus("Failed to find '%s' marker in server output '%s'",
kFusePortPrefix, fuse_stdout);
}
int port =
atoi(fuse_stdout.substr(port_pos + strlen(kFusePortPrefix)).c_str());
if (port == 0) {
return MakeStatus("Failed to parse port from server output '%s'",
fuse_stdout);
}
return port;
}
} // namespace
CdcFuseManager::CdcFuseManager(std::string instance,
@@ -73,34 +53,32 @@ absl::Status CdcFuseManager::Deploy() {
std::string exe_dir;
RETURN_IF_ERROR(path::GetExeDir(&exe_dir), "Failed to get exe directory");
// Create the remote tools bin dir if it doesn't exist yet.
// This assumes that sftp's remote startup directory is the home directory.
std::vector<std::string> dir_parts =
absl::StrSplit(kRemoteToolsBinDir, '/', absl::SkipEmpty());
std::string sftp_commands;
for (const std::string& dir : dir_parts) {
// Use -mkdir to ignore errors if the directory already exists.
sftp_commands += absl::StrFormat("-mkdir %s\ncd %s\n", dir, dir);
}
// Set the cwd to the exe dir and pass the filenames to scp. Otherwise, some
// scp implementations can get confused and create the wrong remote filenames.
path::SetCwd(exe_dir);
sftp_commands += absl::StrFormat("put %s\n", kFuseFilename);
sftp_commands += absl::StrFormat("put %s\n", kLibFuseFilename);
sftp_commands += absl::StrFormat("chmod 755 %s\n", kFuseFilename);
// Copy FUSE to the gamelet.
LOG_DEBUG("Copying FUSE");
RETURN_IF_ERROR(remote_util_->Scp({kFuseFilename, kLibFuseFilename},
kRemoteToolsBinDir, /*compress=*/false),
"Failed to copy FUSE to gamelet");
LOG_DEBUG("Copying FUSE succeeded");
LOG_DEBUG("Deploying FUSE");
RETURN_IF_ERROR(
remote_util_->Sftp(sftp_commands, exe_dir, /*compress=*/false),
"Failed to deploy FUSE");
LOG_DEBUG("Deploying FUSE succeeded");
// Make FUSE executable. Note that sync does it automatically.
LOG_DEBUG("Making FUSE executable");
std::string remotePath = path::JoinUnix(kRemoteToolsBinDir, kFuseFilename);
RETURN_IF_ERROR(remote_util_->Chmod("a+x", remotePath),
"Failed to set executable flag on FUSE");
LOG_DEBUG("Making FUSE succeeded");
return absl::OkStatus();
}
absl::Status CdcFuseManager::Start(const std::string& mount_dir,
uint16_t local_port, int verbosity,
bool debug, bool singlethreaded,
bool enable_stats, bool check,
uint64_t cache_capacity,
uint16_t local_port, uint16_t remote_port,
int verbosity, bool debug,
bool singlethreaded, bool enable_stats,
bool check, uint64_t cache_capacity,
uint32_t cleanup_timeout_sec,
uint32_t access_idle_timeout_sec) {
assert(!fuse_process_);
@@ -117,121 +95,82 @@ absl::Status CdcFuseManager::Start(const std::string& mount_dir,
if (!status.ok()) {
return absl::NotFoundError(absl::StrFormat(
"Required gamelet component not found. Make sure the files %s and %s "
"reside in the same folder as %s.",
kFuseFilename, kLibFuseFilename, kExeFilename));
"reside in the same folder as stadia_assets_stream_manager_v3.exe.",
kFuseFilename, kLibFuseFilename));
}
std::string component_args = GameletComponent::ToCommandLineArgs(components);
// Build the remote command.
std::string remotePath = path::JoinUnix(kRemoteToolsBinDir, kFuseFilename);
std::string remote_command = absl::StrFormat(
"LD_LIBRARY_PATH=%s %s "
"mkdir -p %s; LD_LIBRARY_PATH=%s %s "
"--instance=%s "
"--components=%s --cache_dir=%s "
"--components=%s --port=%i --cache_dir=%s "
"--verbosity=%i --cleanup_timeout=%i --access_idle_timeout=%i --stats=%i "
"--check=%i --cache_capacity=%u -- -o allow_root -o ro -o nonempty -o "
"auto_unmount %s%s%s",
kRemoteToolsBinDir, remotePath, RemoteUtil::QuoteForSsh(instance_),
RemoteUtil::QuoteForSsh(component_args), kCacheDir, verbosity,
cleanup_timeout_sec, access_idle_timeout_sec, enable_stats, check,
cache_capacity, debug ? "-d " : "", singlethreaded ? "-s " : "",
RemoteUtil::QuoteForSsh(mount_dir));
kRemoteToolsBinDir, kRemoteToolsBinDir, remotePath,
RemoteUtil::QuoteForSsh(instance_),
RemoteUtil::QuoteForSsh(component_args), remote_port, kCacheDir,
verbosity, cleanup_timeout_sec, access_idle_timeout_sec, enable_stats,
check, cache_capacity, RemoteUtil::QuoteForSsh(mount_dir),
debug ? " -d" : "", singlethreaded ? " -s" : "");
bool needs_deploy = false;
int remote_port;
ASSIGN_OR_RETURN(remote_port, RunFuseProcess(remote_command, &needs_deploy));
RETURN_IF_ERROR(
RunFuseProcess(local_port, remote_port, remote_command, &needs_deploy));
if (needs_deploy) {
// Deploy and try again.
RETURN_IF_ERROR(Deploy());
ASSIGN_OR_RETURN(remote_port,
RunFuseProcess(remote_command, &needs_deploy));
RETURN_IF_ERROR(
RunFuseProcess(local_port, remote_port, remote_command, &needs_deploy));
}
// Start port forwarding.
RETURN_IF_ERROR(RunPortForwardingProcess(local_port, remote_port));
// Wait until port forwarding is up and FUSE can connect to |remote_port|.
RETURN_IF_ERROR(WaitForFuseConnected());
return absl::OkStatus();
}
absl::StatusOr<int> CdcFuseManager::RunFuseProcess(
const std::string& remote_command, bool* needs_deploy) {
absl::Status CdcFuseManager::RunFuseProcess(uint16_t local_port,
uint16_t remote_port,
const std::string& remote_command,
bool* needs_deploy) {
assert(!fuse_process_);
assert(needs_deploy);
*needs_deploy = false;
LOG_DEBUG("Running FUSE process");
ProcessStartInfo start_info = remote_util_->BuildProcessStartInfoForSsh(
remote_command, ArchType::kLinux_x86_64);
ProcessStartInfo start_info =
remote_util_->BuildProcessStartInfoForSshPortForwardAndCommand(
local_port, remote_port, true, remote_command);
start_info.name = kFuseFilename;
// Capture stdout to determine whether a deploy is required.
fuse_stdout_.clear();
fuse_port_ = 0;
fuse_not_up_to_date_ = false;
fuse_update_check_finished_ = false;
fuse_connected_ = false;
start_info.stdout_handler = [this](const char* data, size_t size) {
return HandleFuseStdout(data, size);
fuse_startup_finished_ = false;
start_info.stdout_handler = [this, needs_deploy](const char* data,
size_t size) {
return HandleFuseStdout(data, size, needs_deploy);
};
fuse_process_ = process_factory_->Create(start_info);
RETURN_IF_ERROR(fuse_process_->Start(), "Failed to start FUSE process");
LOG_DEBUG("FUSE process started. Waiting for startup to finish.");
// Run until process exits or startup finishes.
RETURN_IF_ERROR(fuse_process_->RunUntil(
[this]() { return fuse_update_check_finished_.load(); }),
auto startup_finished = [this]() { return fuse_startup_finished_.load(); };
RETURN_IF_ERROR(fuse_process_->RunUntil(startup_finished),
"Failed to run FUSE process");
LOG_DEBUG("FUSE process update check complete.");
LOG_DEBUG("FUSE process startup complete.");
// If the FUSE process exited before it could perform its up-to-date check, it
// most likely happens because the binary does not exist and needs to be
// deployed.
*needs_deploy = fuse_not_up_to_date_ ||
(!fuse_update_check_finished_ && fuse_process_->HasExited() &&
fuse_process_->ExitCode() != 0);
*needs_deploy |= !fuse_startup_finished_ && fuse_process_->HasExited() &&
fuse_process_->ExitCode() != 0;
if (*needs_deploy) {
LOG_DEBUG("FUSE needs to be (re-)deployed.");
fuse_process_.reset();
}
return fuse_port_;
}
absl::Status CdcFuseManager::RunPortForwardingProcess(int local_port,
int remote_port) {
assert(fuse_process_);
assert(!forwarding_process_);
LOG_DEBUG(
"Running reverse port forwarding process, local port %i, remote port %i",
local_port, remote_port);
ProcessStartInfo start_info =
remote_util_->BuildProcessStartInfoForSshPortForward(
local_port, remote_port, /*reverse=*/true);
forwarding_process_ = process_factory_->Create(start_info);
RETURN_IF_ERROR(forwarding_process_->Start(),
"Failed to start port forwarding process");
return absl::OkStatus();
}
absl::Status CdcFuseManager::WaitForFuseConnected() {
assert(fuse_process_);
assert(forwarding_process_);
RETURN_IF_ERROR(
fuse_process_->RunUntil([this]() { return fuse_connected_.load(); }),
"Failed to run FUSE process");
LOG_DEBUG("FUSE process connected.");
if (!fuse_connected_ && fuse_process_->HasExited()) {
return MakeStatus("FUSE exited during startup with code %u",
fuse_process_->ExitCode());
}
return absl::OkStatus();
}
@@ -240,37 +179,30 @@ absl::Status CdcFuseManager::Stop() {
return absl::OkStatus();
}
LOG_DEBUG("Terminating FUSE and port forwarding processes");
LOG_DEBUG("Terminating FUSE process");
absl::Status status = fuse_process_->Terminate();
status.Update(forwarding_process_->Terminate());
fuse_process_.reset();
forwarding_process_.reset();
return status;
}
bool CdcFuseManager::IsHealthy() const {
return fuse_process_ && !fuse_process_->HasExited() && forwarding_process_ &&
!forwarding_process_->HasExited();
return fuse_process_ && !fuse_process_->HasExited();
}
absl::Status CdcFuseManager::HandleFuseStdout(const char* data, size_t size) {
// Don't capture stdout beyond startup.
if (!fuse_connected_) {
fuse_stdout_.append(data, size);
absl::Status CdcFuseManager::HandleFuseStdout(const char* data, size_t size,
bool* needs_deploy) {
assert(needs_deploy);
// The remote component prints some magic strings to stdout to indicate
// Don't capture stdout beyond startup.
if (!fuse_startup_finished_) {
fuse_stdout_.append(data, size);
// The gamelet component prints some magic strings to stdout to indicate
// whether it's up-to-date.
if (absl::StrContains(fuse_stdout_, kFuseUpToDate)) {
ASSIGN_OR_RETURN(fuse_port_, ParsePort(fuse_stdout_));
fuse_update_check_finished_ = true;
fuse_startup_finished_ = true;
} else if (absl::StrContains(fuse_stdout_, kFuseNotUpToDate)) {
fuse_not_up_to_date_ = true;
fuse_update_check_finished_ = true;
}
// It also prints stuff when it can connect to its port.
if (absl::StrContains(fuse_stdout_, kFuseConnected)) {
fuse_connected_ = true;
fuse_startup_finished_ = true;
*needs_deploy = true;
}
}
cdc_stream/cdc_fuse_manager.h
+19 -35
View File
@@ -18,7 +18,6 @@
#define CDC_STREAM_CDC_FUSE_MANAGER_H_
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "common/remote_util.h"
namespace cdc_ft {
@@ -27,7 +26,7 @@ class Process;
class ProcessFactory;
class RemoteUtil;
// Manages the remote CDC FUSE filesystem process.
// Manages the gamelet-side CDC FUSE filesystem process.
class CdcFuseManager {
public:
CdcFuseManager(std::string instance, ProcessFactory* process_factory,
@@ -37,10 +36,11 @@ class CdcFuseManager {
CdcFuseManager(CdcFuseManager&) = delete;
CdcFuseManager& operator=(CdcFuseManager&) = delete;
// Starts the remote CDC FUSE process. Deploys the binary if necessary.
// Starts the CDC FUSE and establishes a reverse SSH tunnel from the gamelet's
// |remote_port| to the workstation's |local_port|. Deploys the binary if
// necessary.
//
// |mount_dir| is the remote directory where to mount the FUSE.
// |local_port| is the local port used for gRPC connections to the FUSE.
// |verbosity| is the log verbosity used by the filesystem.
// |debug| puts the filesystem into debug mode if set to true. This also
// causes the process to run in the foreground, so that logs are piped through
@@ -53,9 +53,9 @@ class CdcFuseManager {
// |access_idle_timeout_sec| defines the number of seconds after which data
// provider is considered to be access-idling.
absl::Status Start(const std::string& mount_dir, uint16_t local_port,
int verbosity, bool debug, bool singlethreaded,
bool enable_stats, bool check, uint64_t cache_capacity,
uint32_t cleanup_timeout_sec,
uint16_t remote_port, int verbosity, bool debug,
bool singlethreaded, bool enable_stats, bool check,
uint64_t cache_capacity, uint32_t cleanup_timeout_sec,
uint32_t access_idle_timeout_sec);
// Stops the CDC FUSE.
@@ -65,49 +65,33 @@ class CdcFuseManager {
bool IsHealthy() const;
private:
// Runs the remote FUSE process from the given |remote_command|. Returns the
// remote port that the FUSE will connect to, once the port forwarding process
// is up.
// Runs the FUSE process on the gamelet from the given |remote_command| and
// establishes a reverse SSH tunnel from the gamelet's |remote_port| to the
// workstation's |local_port|.
//
// If the FUSE is not up-to-date or does not exist, sets |needs_deploy| to
// true and returns OK. In that case, Deploy() needs to be called and the FUSE
// process should be run again.
absl::StatusOr<int> RunFuseProcess(const std::string& remote_command,
absl::Status RunFuseProcess(uint16_t local_port, uint16_t remote_port,
const std::string& remote_command,
bool* needs_deploy);
// Establishes a reverse SSH tunnel from |remote_port| to |local_port|.
absl::Status RunPortForwardingProcess(int local_port, int remote_port);
// Waits until FUSE can connect to its port. This essentially means that
// port forwarding is up and running.
absl::Status WaitForFuseConnected();
// Deploys the remote components.
// Deploys the gamelet components.
absl::Status Deploy();
// Output handler for FUSE's stdout.
// Sets |fuse_not_up_to_date_| to true if the output contains a magic marker
// to indicate that the binary has to be redeployed.
// Sets |fuse_port_| to the remote gRPC port if the output contains a magic
// marker that has the port and indicates that the binary is up-to-date.
// Sets |fuse_update_check_finished_| to true if any of the above two markers
// was set. Called in a background thread.
// Sets |fuse_startup_finished_| to true if FUSE is connected to its port.
absl::Status HandleFuseStdout(const char* data, size_t size);
// Output handler for FUSE's stdout. Sets |needs_deploy| to true if the output
// contains a magic marker to indicate that the binary has to be redeployed.
// Called in a background thread.
absl::Status HandleFuseStdout(const char* data, size_t size,
bool* needs_deploy);
std::string instance_;
ProcessFactory* const process_factory_;
RemoteUtil* const remote_util_;
std::unique_ptr<Process> fuse_process_;
std::unique_ptr<Process> forwarding_process_;
std::string fuse_stdout_;
// Set by HandleFuseStdout
int fuse_port_ = 0;
bool fuse_not_up_to_date_ = false;
std::atomic_bool fuse_update_check_finished_{false};
std::atomic_bool fuse_connected_{false};
std::atomic<bool> fuse_startup_finished_{false};
};
} // namespace cdc_ft
cdc_stream/cdc_stream.vcxproj
+1 -1
View File
@@ -47,7 +47,7 @@
<AdditionalOptions>/std:c++17</AdditionalOptions>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<OutDir>$(SolutionDir)bazel-out\x64_windows-opt\bin\cdc_stream\</OutDir>
<OutDir>$(SolutionDir)bazel-out\x64_windows-opt\bin\asset_stcdc_streamream_manager\</OutDir>
<NMakePreprocessorDefinitions>UNICODE</NMakePreprocessorDefinitions>
<AdditionalOptions>/std:c++17</AdditionalOptions>
</PropertyGroup>
cdc_stream/local_assets_stream_manager_client.cc
+13 -3
View File
@@ -28,6 +28,15 @@ using StartSessionResponse = localassetsstreammanager::StartSessionResponse;
using StopSessionRequest = localassetsstreammanager::StopSessionRequest;
using StopSessionResponse = localassetsstreammanager::StopSessionResponse;
LocalAssetsStreamManagerClient::LocalAssetsStreamManagerClient(
uint16_t service_port) {
std::string client_address = absl::StrFormat("localhost:%u", service_port);
std::shared_ptr<grpc::Channel> channel = grpc::CreateCustomChannel(
client_address, grpc::InsecureChannelCredentials(),
grpc::ChannelArguments());
stub_ = LocalAssetsStreamManager::NewStub(std::move(channel));
}
LocalAssetsStreamManagerClient::LocalAssetsStreamManagerClient(
std::shared_ptr<grpc::Channel> channel) {
stub_ = LocalAssetsStreamManager::NewStub(std::move(channel));
@@ -36,15 +45,16 @@ LocalAssetsStreamManagerClient::LocalAssetsStreamManagerClient(
LocalAssetsStreamManagerClient::~LocalAssetsStreamManagerClient() = default;
absl::Status LocalAssetsStreamManagerClient::StartSession(
const std::string& src_dir, const std::string& user_host,
const std::string& src_dir, const std::string& user_host, uint16_t ssh_port,
const std::string& mount_dir, const std::string& ssh_command,
const std::string& sftp_command) {
const std::string& scp_command) {
StartSessionRequest request;
request.set_workstation_directory(src_dir);
request.set_user_host(user_host);
request.set_port(ssh_port);
request.set_mount_dir(mount_dir);
request.set_ssh_command(ssh_command);
request.set_sftp_command(sftp_command);
request.set_scp_command(scp_command);
grpc::ClientContext context;
StartSessionResponse response;
cdc_stream/local_assets_stream_manager_client.h
+7 -3
View File
@@ -20,6 +20,7 @@
#include <memory>
#include "absl/status/status.h"
#include "grpcpp/channel.h"
#include "proto/local_assets_stream_manager.grpc.pb.h"
namespace grpc_impl {
@@ -31,6 +32,8 @@ namespace cdc_ft {
// gRpc client for starting/stopping asset streaming sessions.
class LocalAssetsStreamManagerClient {
public:
explicit LocalAssetsStreamManagerClient(uint16_t service_port);
// |channel| is a grpc channel to use.
explicit LocalAssetsStreamManagerClient(
std::shared_ptr<grpc::Channel> channel);
@@ -41,14 +44,15 @@ class LocalAssetsStreamManagerClient {
// Starting a second session to the same target will stop the first one.
// |src_dir| is the Windows source directory to stream.
// |user_host| is the Linux host, formatted as [user@:host].
// |ssh_port| is the SSH port to use while connecting to the host.
// |mount_dir| is the Linux target directory to stream to.
// |ssh_command| is the ssh command and extra arguments to use.
// |sftp_command| is the sftp command and extra arguments to use.
// |scp_command| is the scp command and extra arguments to use.
absl::Status StartSession(const std::string& src_dir,
const std::string& user_host,
const std::string& user_host, uint16_t ssh_port,
const std::string& mount_dir,
const std::string& ssh_command,
const std::string& sftp_command);
const std::string& scp_command);
// Stops the streaming session to the Linux target |user_host|:|mount_dir|.
// |user_host| is the Linux host, formatted as [user@:host].
cdc_stream/local_assets_stream_manager_service_impl.cc
+26 -11
View File
@@ -208,7 +208,7 @@ LocalAssetsStreamManagerServiceImpl::GetTargetForStadia(
SessionTarget target;
target.mount_dir = request.mount_dir();
target.ssh_command = request.ssh_command();
target.sftp_command = request.sftp_command();
target.scp_command = request.scp_command();
// Parse instance/project/org id.
if (!ParseInstanceName(request.gamelet_name(), instance_id, project_id,
@@ -219,11 +219,12 @@ LocalAssetsStreamManagerServiceImpl::GetTargetForStadia(
// Run 'ggp ssh init' to determine IP (host) and port.
std::string instance_ip;
ASSIGN_OR_RETURN(instance_ip,
InitSsh(*instance_id, *project_id, *organization_id));
uint16_t instance_port = 0;
RETURN_IF_ERROR(InitSsh(*instance_id, *project_id, *organization_id,
&instance_ip, &instance_port));
target.user_host = "cloudcast@" + instance_ip;
// Note: Port must be set with ssh_command (-p) and sftp_command (-P).
target.ssh_port = instance_port;
return target;
}
@@ -233,7 +234,10 @@ SessionTarget LocalAssetsStreamManagerServiceImpl::GetTarget(
target.user_host = request.user_host();
target.mount_dir = request.mount_dir();
target.ssh_command = request.ssh_command();
target.sftp_command = request.sftp_command();
target.scp_command = request.scp_command();
target.ssh_port = request.port() > 0 && request.port() <= UINT16_MAX
? static_cast<uint16_t>(request.port())
: RemoteUtil::kDefaultSshPort;
*instance_id = absl::StrCat(target.user_host, ":", target.mount_dir);
return target;
@@ -253,10 +257,13 @@ metrics::RequestOrigin LocalAssetsStreamManagerServiceImpl::ConvertOrigin(
}
}
absl::StatusOr<std::string> LocalAssetsStreamManagerServiceImpl::InitSsh(
absl::Status LocalAssetsStreamManagerServiceImpl::InitSsh(
const std::string& instance_id, const std::string& project_id,
const std::string& organization_id) {
const std::string& organization_id, std::string* instance_ip,
uint16_t* instance_port) {
SdkUtil sdk_util;
instance_ip->clear();
*instance_port = 0;
ProcessStartInfo start_info;
start_info.command = absl::StrFormat(
@@ -270,7 +277,6 @@ absl::StatusOr<std::string> LocalAssetsStreamManagerServiceImpl::InitSsh(
absl::StrFormat(" --organization %s", Quoted(organization_id));
}
start_info.name = "ggp ssh init";
start_info.flags = ProcessFlags::kNoWindow;
std::string output;
start_info.stdout_handler = [&output, this](const char* data,
@@ -298,13 +304,22 @@ absl::StatusOr<std::string> LocalAssetsStreamManagerServiceImpl::InitSsh(
}
// Parse gamelet IP. Should be "Host: <instance_ip ip>".
std::string instance_ip;
if (!ParseValue(output, "Host", &instance_ip)) {
if (!ParseValue(output, "Host", instance_ip)) {
return MakeStatus("Failed to parse host from ggp ssh init response\n%s",
output);
}
return instance_ip;
// Parse ssh port. Should be "Port: <port>".
std::string port_string;
const bool result = ParseValue(output, "Port", &port_string);
int int_port = atoi(port_string.c_str());
if (!result || int_port == 0 || int_port <= 0 || int_port > UINT_MAX) {
return MakeStatus("Failed to parse ssh port from ggp ssh init response\n%s",
output);
}
*instance_port = static_cast<uint16_t>(int_port);
return absl::OkStatus();
}
} // namespace cdc_ft
cdc_stream/local_assets_stream_manager_service_impl.h
+4 -3
View File
@@ -93,10 +93,11 @@ class LocalAssetsStreamManagerServiceImpl final
// Initializes an ssh connection to a gamelet by calling 'ggp ssh init'.
// |instance_id| must be set, |project_id|, |organization_id| are optional.
// Returns the instance's IP address.
absl::StatusOr<std::string> InitSsh(const std::string& instance_id,
// Returns |instance_ip| and |instance_port| (SSH port).
absl::Status InitSsh(const std::string& instance_id,
const std::string& project_id,
const std::string& organization_id);
const std::string& organization_id,
std::string* instance_ip, uint16_t* instance_port);
const SessionConfig cfg_;
SessionManager* const session_manager_;
cdc_stream/main.cc
+1 -26
View File
@@ -15,31 +15,9 @@
#include "cdc_stream/start_command.h"
#include "cdc_stream/start_service_command.h"
#include "cdc_stream/stop_command.h"
#include "cdc_stream/stop_service_command.h"
#include "common/platform.h"
#include "lyra/lyra.hpp"
#if PLATFORM_WINDOWS
int wmain(int argc, wchar_t* wargv[]) {
// Convert args from wide to UTF8 strings.
std::vector<std::string> utf8_str_args;
utf8_str_args.reserve(argc);
for (int i = 0; i < argc; i++) {
utf8_str_args.push_back(cdc_ft::Util::WideToUtf8Str(wargv[i]));
}
// Convert args from UTF8 strings to UTF8 c-strings.
std::vector<const char*> utf8_args;
utf8_args.reserve(argc);
for (const auto& utf8_str_arg : utf8_str_args) {
utf8_args.push_back(utf8_str_arg.c_str());
}
const char** argv = utf8_args.data();
#else
int main(int argc, char** argv) {
#endif
int main(int argc, char* argv[]) {
// Set up commands.
auto cli = lyra::cli();
bool show_help = false;
@@ -55,9 +33,6 @@ int main(int argc, char** argv) {
cdc_ft::StartServiceCommand start_service_cmd(&exit_code);
start_service_cmd.Register(cli);
cdc_ft::StopServiceCommand stop_service_cmd(&exit_code);
stop_service_cmd.Register(cli);
// Parse args and run. Note that parse actually runs the commands.
// exit_code is -1 if no command was run.
auto result = cli.parse({argc, argv});
cdc_stream/multi_session.cc
+18 -22
View File
@@ -18,11 +18,9 @@
#include "common/file_watcher_win.h"
#include "common/log.h"
#include "common/path.h"
#include "common/path_filter.h"
#include "common/platform.h"
#include "common/port_manager.h"
#include "common/process.h"
#include "common/server_socket.h"
#include "common/util.h"
#include "data_store/disk_data_store.h"
#include "manifest/content_id.h"
@@ -35,6 +33,11 @@
namespace cdc_ft {
namespace {
// Ports used by the asset streaming service for local port forwarding on
// workstation and gamelet.
constexpr int kAssetStreamPortFirst = 44433;
constexpr int kAssetStreamPortLast = 44442;
// Stats output period (if enabled).
constexpr double kStatsPrintDelaySec = 0.1f;
@@ -437,8 +440,16 @@ absl::Status MultiSession::Initialize() {
}
// Find an available local port.
ASSIGN_OR_RETURN(local_asset_stream_port_, ServerSocket::FindAvailablePort(),
"Failed to find an available local port");
std::unordered_set<int> ports;
ASSIGN_OR_RETURN(
ports,
PortManager::FindAvailableLocalPorts(kAssetStreamPortFirst,
kAssetStreamPortLast, "127.0.0.1",
process_factory_),
"Failed to find an available local port in the range [%d, %d]",
kAssetStreamPortFirst, kAssetStreamPortLast);
assert(!ports.empty());
local_asset_stream_port_ = *ports.begin();
assert(!runner_);
runner_ = std::make_unique<MultiSessionRunner>(
@@ -513,7 +524,8 @@ absl::Status MultiSession::StartSession(const std::string& instance_id,
auto session = std::make_unique<Session>(
instance_id, target, cfg_, process_factory_, std::move(metrics_recorder));
RETURN_IF_ERROR(session->Start(local_asset_stream_port_));
RETURN_IF_ERROR(session->Start(local_asset_stream_port_,
kAssetStreamPortFirst, kAssetStreamPortLast));
// Wait for the FUSE to receive the first intermediate manifest.
RETURN_IF_ERROR(runner_->WaitForManifestAck(instance_id, absl::Seconds(5)));
@@ -543,21 +555,6 @@ bool MultiSession::HasSession(const std::string& instance_id) {
return sessions_.find(instance_id) != sessions_.end();
}
std::vector<std::string> MultiSession::MatchSessions(
const std::string& instance_id_filter) {
PathFilter filter;
filter.AddRule(PathFilter::Rule::Type::kInclude, instance_id_filter);
filter.AddRule(PathFilter::Rule::Type::kExclude, "*");
std::vector<std::string> matches;
for (const auto& [instance_id, session] : sessions_) {
if (filter.IsMatch(instance_id)) {
matches.push_back(instance_id);
}
}
return matches;
}
bool MultiSession::IsSessionHealthy(const std::string& instance_id) {
absl::ReaderMutexLock lock(&sessions_mutex_);
auto iter = sessions_.find(instance_id);
@@ -612,8 +609,7 @@ absl::StatusOr<std::string> MultiSession::GetCachePath(
path::Append(&appdata_path, ".cache");
#endif
std::string base_dir =
path::Join(appdata_path, "cdc-file-transfer", "chunks");
std::string base_dir = path::Join(appdata_path, "GGP", "asset_streaming");
std::string cache_dir = GetCacheDir(src_dir);
size_t total_size = base_dir.size() + 1 + cache_dir.size();
cdc_stream/multi_session.h
+2 -12
View File
@@ -134,11 +134,6 @@ class MultiSessionRunner {
// to an arbitrary number of gamelets.
class MultiSession {
public:
// Ports used by the asset streaming service for local port forwarding on
// workstation and gamelet.
static constexpr int kDefaultForwardPortFirst = 44433;
static constexpr int kDefaultForwardPortLast = 44442;
// Maximum length of cache path. We must be able to write content hashes into
// this path:
// <cache path>\01234567890123456789<null terminator> = 260 characters.
@@ -153,7 +148,7 @@ class MultiSession {
// |process_factory| abstracts process creation.
// |data_store| can be passed for tests to override the default store used.
// By default, the class uses a DiskDataStore that writes to
// %APPDATA%\cdc-file-transfer\chunks\<dir_derived_from_src_dir> on Windows.
// %APPDATA%\GGP\asset_streaming|<dir_derived_from_src_dir> on Windows.
MultiSession(std::string src_dir, SessionConfig cfg,
ProcessFactory* process_factory,
MultiSessionMetricsRecorder const* metrics_recorder,
@@ -199,11 +194,6 @@ class MultiSession {
absl::Status StopSession(const std::string& instance_id)
ABSL_LOCKS_EXCLUDED(sessions_mutex_);
// Returns all instance ids that match the given filter. The filter may
// contain Windows-style wildcards, e.g. *, foo* or f?o.
// Matches are case sensitive.
std::vector<std::string> MatchSessions(const std::string& instance_id_filter);
// Returns true if there is an existing session for |instance_id|.
bool HasSession(const std::string& instance_id)
ABSL_LOCKS_EXCLUDED(sessions_mutex_);
@@ -225,7 +215,7 @@ class MultiSession {
static std::string GetCacheDir(std::string dir);
// Returns the directory where manifest chunks are cached, e.g.
// "%APPDATA%\cdc-file-transfer\chunks\c__path_to_game_abcdef01" for
// "%APPDATA%\GGP\asset_streaming\c__path_to_game_abcdef01" for
// "C:\path\to\game".
// The returned path is shortened to |max_len| by removing UTF8 code points
// from the beginning of the actual cache directory (c__path...) if necessary.
cdc_stream/multi_session_test.cc
+7 -8
View File
@@ -158,7 +158,7 @@ class MultiSessionTest : public ManifestTestBase {
EXPECT_EQ(data->file_count, file_count);
EXPECT_EQ(data->min_chunk_size, 128 << 10);
EXPECT_EQ(data->avg_chunk_size, 256 << 10);
EXPECT_EQ(data->max_chunk_size, 512 << 10);
EXPECT_EQ(data->max_chunk_size, 1024 << 10);
}
metrics::ManifestUpdateData GetManifestUpdateData(
@@ -241,7 +241,7 @@ TEST_F(MultiSessionTest, GetCachePath_ContainsExpectedParts) {
ASSERT_OK(cache_path);
EXPECT_TRUE(absl::EndsWith(*cache_path, kCacheDir)) << *cache_path;
EXPECT_TRUE(
absl::StrContains(*cache_path, path::Join("cdc-file-transfer", "chunks")))
absl::StrContains(*cache_path, path::Join("GGP", "asset_streaming")))
<< *cache_path;
}
@@ -253,7 +253,7 @@ TEST_F(MultiSessionTest, GetCachePath_ShortensLongPaths) {
ASSERT_OK(cache_path);
EXPECT_EQ(cache_path->size(), MultiSession::kDefaultMaxCachePathLen);
EXPECT_TRUE(
absl::StrContains(*cache_path, path::Join("cdc-file-transfer", "chunks")))
absl::StrContains(*cache_path, path::Join("GGP", "asset_streaming")))
<< *cache_path;
// The hash in the end of the path is kept and not shortened.
EXPECT_EQ(cache_dir.substr(cache_dir.size() - MultiSession::kDirHashLen),
@@ -261,8 +261,7 @@ TEST_F(MultiSessionTest, GetCachePath_ShortensLongPaths) {
}
TEST_F(MultiSessionTest, GetCachePath_DoesNotSplitUtfCodePoints) {
// Find out the length of the %APPDATA%\cdc-file-transfer\chunks\" + hash
// part.
// Find out the length of the %APPDATA%\GGP\asset_streaming\" + hash part.
absl::StatusOr<std::string> cache_path = MultiSession::GetCachePath("");
ASSERT_OK(cache_path);
size_t base_len = cache_path->size();
@@ -272,17 +271,17 @@ TEST_F(MultiSessionTest, GetCachePath_DoesNotSplitUtfCodePoints) {
ASSERT_OK(cache_path);
EXPECT_EQ(cache_path->size(), base_len);
// %APPDATA%\cdc-file-transfer\chunks\abcdefg
// %APPDATA%\GGP\asset_streaming\abcdefg
cache_path = MultiSession::GetCachePath(u8"\u0200\u0200", base_len + 1);
ASSERT_OK(cache_path);
EXPECT_EQ(cache_path->size(), base_len);
// %APPDATA%\cdc-file-transfer\chunks\\u0200abcdefg
// %APPDATA%\GGP\asset_streaming\\u0200abcdefg
cache_path = MultiSession::GetCachePath(u8"\u0200\u0200", base_len + 2);
ASSERT_OK(cache_path);
EXPECT_EQ(cache_path->size(), base_len + 2);
// %APPDATA%\cdc-file-transfer\chunks\\u0200abcdefg
// %APPDATA%\GGP\asset_streaming\\u0200abcdefg
cache_path = MultiSession::GetCachePath(u8"\u0200\u0200", base_len + 3);
ASSERT_OK(cache_path);
EXPECT_EQ(cache_path->size(), base_len + 2);
cdc_stream/session.cc
+22 -10
View File
@@ -47,16 +47,16 @@ Session::Session(std::string instance_id, const SessionTarget& target,
mount_dir_(target.mount_dir),
cfg_(std::move(cfg)),
process_factory_(process_factory),
remote_util_(target.user_host, cfg_.verbosity, cfg_.quiet,
process_factory,
remote_util_(cfg_.verbosity, cfg_.quiet, process_factory,
/*forward_output_to_logging=*/true),
metrics_recorder_(std::move(metrics_recorder)) {
assert(metrics_recorder_);
remote_util_.SetUserHostAndPort(target.user_host, target.ssh_port);
if (!target.ssh_command.empty()) {
remote_util_.SetSshCommand(target.ssh_command);
}
if (!target.sftp_command.empty()) {
remote_util_.SetSftpCommand(target.sftp_command);
if (!target.scp_command.empty()) {
remote_util_.SetScpCommand(target.scp_command);
}
}
@@ -68,18 +68,30 @@ Session::~Session() {
}
}
absl::Status Session::Start(int local_port) {
absl::Status Session::Start(int local_port, int first_remote_port,
int last_remote_port) {
// Find an available remote port.
std::unordered_set<int> ports;
ASSIGN_OR_RETURN(
ports,
PortManager::FindAvailableRemotePorts(
first_remote_port, last_remote_port, "127.0.0.1", process_factory_,
&remote_util_, kInstanceConnectionTimeoutSec),
"Failed to find an available remote port in the range [%d, %d]",
first_remote_port, last_remote_port);
assert(!ports.empty());
int remote_port = *ports.begin();
assert(!fuse_);
fuse_ = std::make_unique<CdcFuseManager>(instance_id_, process_factory_,
&remote_util_);
RETURN_IF_ERROR(
fuse_->Start(mount_dir_, local_port, cfg_.verbosity, cfg_.fuse_debug,
cfg_.fuse_singlethreaded, cfg_.stats, cfg_.fuse_check,
cfg_.fuse_cache_capacity, cfg_.fuse_cleanup_timeout_sec,
fuse_->Start(mount_dir_, local_port, remote_port, cfg_.verbosity,
cfg_.fuse_debug, cfg_.fuse_singlethreaded, cfg_.stats,
cfg_.fuse_check, cfg_.fuse_cache_capacity,
cfg_.fuse_cleanup_timeout_sec,
cfg_.fuse_access_idle_timeout_sec),
"Failed to start instance component");
return absl::OkStatus();
}
cdc_stream/session.h
+7 -3
View File
@@ -36,10 +36,12 @@ class Process;
struct SessionTarget {
// SSH username and hostname of the remote target, formed as [user@]host.
std::string user_host;
// Port to use for SSH connections to the remote target.
uint16_t ssh_port;
// Ssh command to use to connect to the remote target.
std::string ssh_command;
// Sftp command to use to copy files to the remote target.
std::string sftp_command;
// Scp command to use to copy files to the remote target.
std::string scp_command;
// Directory on the remote target where to mount the streamed directory.
std::string mount_dir;
};
@@ -58,7 +60,9 @@ class Session {
// Starts the CDC FUSE on the instance with established port forwarding.
// |local_port| is the local reverse forwarding port to use.
absl::Status Start(int local_port);
// [|first_remote_port|, |last_remote_port|] are the allowed remote ports.
absl::Status Start(int local_port, int first_remote_port,
int last_remote_port);
// Shuts down the connection to the instance.
absl::Status Stop() ABSL_LOCKS_EXCLUDED(transferred_data_mu_);
cdc_stream/session_config.h
-5
View File
@@ -56,11 +56,6 @@ struct SessionConfig {
// Time to wait until running a manifest update after detecting a file change.
uint32_t file_change_wait_duration_ms = 0;
// Ports used for local port forwarding. Deprecated as forward ports are
// determined automatically now using ephemeral ports.
uint16_t deprecated_forward_port_first = 0;
uint16_t deprecated_forward_port_last = 0;
};
} // namespace cdc_ft
cdc_stream/session_management_server.h
+1 -1
View File
@@ -36,7 +36,7 @@ class ProcessFactory;
// - Background
class SessionManagementServer {
public:
static constexpr uint16_t kDefaultServicePort = 44432;
static constexpr int kDefaultServicePort = 44432;
SessionManagementServer(grpc::Service* session_service,
grpc::Service* background_service,
cdc_stream/session_manager.cc
+2 -17
View File
@@ -136,24 +136,9 @@ absl::Status SessionManager::StartSession(
return status;
}
absl::Status SessionManager::StopSession(
const std::string& instance_id_filter) {
absl::Status SessionManager::StopSession(const std::string& instance_id) {
absl::MutexLock lock(&sessions_mutex_);
std::vector<std::string> instance_ids;
for (const auto& [key, ms] : sessions_) {
auto ids = ms->MatchSessions(instance_id_filter);
instance_ids.insert(instance_ids.end(), ids.begin(), ids.end());
}
if (instance_ids.empty()) {
return absl::NotFoundError(
absl::StrFormat("No session found matching '%s'", instance_id_filter));
}
for (const std::string& instance_id : instance_ids) {
RETURN_IF_ERROR(StopSessionInternal(instance_id));
}
return absl::OkStatus();
return StopSessionInternal(instance_id);
}
MultiSession* SessionManager::GetMultiSession(const std::string& src_dir) {
cdc_stream/session_manager.h
+2 -4
View File
@@ -58,11 +58,9 @@ class SessionManager {
metrics::SessionStartStatus* metrics_status)
ABSL_LOCKS_EXCLUDED(sessions_mutex_);
// Stops all sessions that match the given |instance_id_filter|.
// The filter may contain Windows-style wildcards like * and ?.
// Matching is case-sensitive.
// Stops the session for the given |instance_id|.
// Returns a NotFound error if no session exists.
absl::Status StopSession(const std::string& instance_id_filter)
absl::Status StopSession(const std::string& instance_id)
ABSL_LOCKS_EXCLUDED(sessions_mutex_);
// Shuts down all existing MultiSessions.
cdc_stream/start_command.cc
+19 -115
View File
@@ -16,19 +16,12 @@
#include <memory>
#include "cdc_stream/background_service_client.h"
#include "cdc_stream/local_assets_stream_manager_client.h"
#include "cdc_stream/session_management_server.h"
#include "common/log.h"
#include "common/path.h"
#include "common/process.h"
#include "common/remote_util.h"
#include "common/status_macros.h"
#include "common/stopwatch.h"
#include "common/util.h"
#include "grpcpp/channel.h"
#include "grpcpp/create_channel.h"
#include "grpcpp/support/channel_arguments.h"
#include "lyra/lyra.hpp"
namespace cdc_ft {
@@ -36,19 +29,6 @@ namespace {
constexpr int kDefaultVerbosity = 2;
} // namespace
namespace {
// Time to poll until the streaming service becomes healthy.
constexpr double kServiceStartupTimeoutSec = 20.0;
std::shared_ptr<grpc::Channel> CreateChannel(uint16_t service_port) {
std::string client_address = absl::StrFormat("localhost:%u", service_port);
return grpc::CreateCustomChannel(client_address,
grpc::InsecureChannelCredentials(),
grpc::ChannelArguments());
}
} // namespace
StartCommand::StartCommand(int* exit_code)
: BaseCommand("start",
"Start streaming files from a Windows to a Linux device",
@@ -72,31 +52,28 @@ void StartCommand::RegisterCommandLineFlags(lyra::command& cmd) {
"asset stream service, default: " +
std::to_string(SessionManagementServer::kDefaultServicePort)));
ssh_port_ = RemoteUtil::kDefaultSshPort;
cmd.add_argument(
lyra::opt(ssh_port_, "port")
.name("--ssh-port")
.help("Port to use while connecting to the remote instance being "
"streamed to, default: " +
std::to_string(RemoteUtil::kDefaultSshPort)));
path::GetEnv("CDC_SSH_COMMAND", &ssh_command_).IgnoreError();
cmd.add_argument(
lyra::opt(ssh_command_, "cmd")
lyra::opt(ssh_command_, "ssh_command")
.name("--ssh-command")
.help("Path and arguments of ssh command to use, e.g. "
"\"C:\\path\\to\\ssh.exe -F config_file -p 1234\". Can also be "
"\"C:\\path\\to\\ssh.exe -F config_file\". Can also be "
"specified by the CDC_SSH_COMMAND environment variable."));
path::GetEnv("CDC_SFTP_COMMAND", &sftp_command_).IgnoreError();
path::GetEnv("CDC_SCP_COMMAND", &scp_command_).IgnoreError();
cmd.add_argument(
lyra::opt(sftp_command_, "cmd")
.name("--sftp-command")
.help(
"Path and arguments of sftp command to use, e.g. "
"\"C:\\path\\to\\sftp.exe -F config_file -P 1234\". Can also be "
"specified by the CDC_SFTP_COMMAND environment variable."));
path::GetEnv("CDC_SCP_COMMAND", &deprecated_scp_command_).IgnoreError();
cmd.add_argument(
lyra::opt(deprecated_scp_command_, "cmd")
lyra::opt(scp_command_, "scp_command")
.name("--scp-command")
.help("[Deprecated, use --sftp-command] Path and arguments of scp "
"command to "
"use, e.g. "
"\"C:\\path\\to\\scp.exe -F config_file -P 1234\". Can also be "
.help("Path and arguments of scp command to use, e.g. "
"\"C:\\path\\to\\scp.exe -F config_file\". Can also be "
"specified by the CDC_SCP_COMMAND environment variable."));
cmd.add_argument(lyra::arg(PosArgValidator(&src_dir_), "dir")
@@ -104,7 +81,7 @@ void StartCommand::RegisterCommandLineFlags(lyra::command& cmd) {
.help("Windows directory to stream"));
cmd.add_argument(
lyra::arg(PosArgValidator(&user_host_dir_), "[user@]host:dir")
lyra::arg(PosArgValidator(&user_host_dir_), "[user@]host:src-dir")
.required()
.help("Linux host and directory to stream to"));
}
@@ -112,49 +89,16 @@ void StartCommand::RegisterCommandLineFlags(lyra::command& cmd) {
absl::Status StartCommand::Run() {
LogLevel level = Log::VerbosityToLogLevel(verbosity_);
ScopedLog scoped_log(std::make_unique<ConsoleLog>(level));
LocalAssetsStreamManagerClient client(service_port_);
std::string full_src_dir = path::GetFullPath(src_dir_);
std::string user_host, mount_dir;
RETURN_IF_ERROR(LocalAssetsStreamManagerClient::ParseUserHostDir(
user_host_dir_, &user_host, &mount_dir));
// Backwards compatibility after switching from scp to sftp.
if (sftp_command_.empty() && !deprecated_scp_command_.empty()) {
LOG_WARNING(
"The CDC_SCP_COMMAND environment variable and the --scp-command flag "
"are deprecated. Please set CDC_SFTP_COMMAND or --sftp-command "
"instead.");
sftp_command_ = RemoteUtil::ScpToSftpCommand(deprecated_scp_command_);
if (!sftp_command_.empty()) {
LOG_WARNING("Converted scp command '%s' to sftp command '%s'.",
deprecated_scp_command_, sftp_command_);
} else {
LOG_WARNING("Failed to convert scp command '%s' to sftp command.",
deprecated_scp_command_);
}
}
LocalAssetsStreamManagerClient client(CreateChannel(service_port_));
absl::Status status = client.StartSession(full_src_dir, user_host, mount_dir,
ssh_command_, sftp_command_);
if (absl::IsUnavailable(status)) {
LOG_DEBUG("StartSession status: %s", status.ToString());
LOG_INFO("Streaming service is unavailable. Starting it...");
status = StartStreamingService();
if (status.ok()) {
LOG_INFO("Streaming service successfully started");
// Recreate client. The old channel might still be in a transient failure
// state.
LocalAssetsStreamManagerClient new_client(CreateChannel(service_port_));
status = new_client.StartSession(full_src_dir, user_host, mount_dir,
ssh_command_, sftp_command_);
}
}
absl::Status status =
client.StartSession(full_src_dir, user_host, ssh_port_, mount_dir,
ssh_command_, scp_command_);
if (status.ok()) {
LOG_INFO("Started streaming directory '%s' to '%s:%s'", src_dir_, user_host,
mount_dir);
@@ -163,44 +107,4 @@ absl::Status StartCommand::Run() {
return status;
}
absl::Status StartCommand::StartStreamingService() {
std::string exe_dir;
RETURN_IF_ERROR(path::GetExeDir(&exe_dir),
"Failed to get executable directory");
std::string exe_path = path::Join(exe_dir, "cdc_stream");
// Try starting the service first.
WinProcessFactory process_factory;
ProcessStartInfo start_info;
start_info.command =
absl::StrFormat("%s start-service --verbosity=%i --service-port=%i",
exe_path, verbosity_, service_port_);
start_info.flags = ProcessFlags::kDetached;
std::unique_ptr<Process> service_process = process_factory.Create(start_info);
RETURN_IF_ERROR(service_process->Start(),
"Failed to start asset streaming service");
// Poll until the service becomes healthy.
LOG_INFO("Streaming service initializing...");
Stopwatch sw;
while (sw.ElapsedSeconds() < kServiceStartupTimeoutSec) {
// The channel is in some transient failure state, and it's faster to
// reconnect instead of waiting for it to return.
BackgroundServiceClient bg_client(CreateChannel(service_port_));
absl::Status status = bg_client.IsHealthy();
if (status.ok()) {
return absl::OkStatus();
}
LOG_DEBUG("Health check result: %s", status.ToString());
Util::Sleep(100);
}
// Kill the process.
service_process->Terminate();
return absl::DeadlineExceededError(
absl::StrFormat("Timed out after %0.0f seconds waiting for the asset "
"streaming service to become healthy",
kServiceStartupTimeoutSec));
}
} // namespace cdc_ft
cdc_stream/start_command.h
+2 -10
View File
@@ -20,10 +20,6 @@
#include "absl/status/status.h"
#include "cdc_stream/base_command.h"
namespace grpc {
class Channel;
}
namespace cdc_ft {
// Handler for the start command. Sends an RPC call to the service to starts a
@@ -38,17 +34,13 @@ class StartCommand : public BaseCommand {
absl::Status Run() override;
private:
// Starts the asset streaming service.
absl::Status StartStreamingService();
int verbosity_ = 0;
uint16_t service_port_ = 0;
uint16_t ssh_port_ = 0;
std::string ssh_command_;
std::string sftp_command_;
std::string scp_command_;
std::string src_dir_;
std::string user_host_dir_;
std::string deprecated_scp_command_;
};
} // namespace cdc_ft
cdc_stream/start_service_command.cc
+7 -5
View File
@@ -39,11 +39,11 @@ std::string GetLogPath(const char* log_dir, const char* log_base_name) {
} // namespace
StartServiceCommand::StartServiceCommand(int* exit_code)
: BaseCommand("start-service", "Start the streaming service", exit_code) {}
: BaseCommand("start-service", "Start streaming service", exit_code) {}
StartServiceCommand::~StartServiceCommand() = default;
void StartServiceCommand::RegisterCommandLineFlags(lyra::command& cmd) {
config_file_ = "%APPDATA%\\cdc-file-transfer\\cdc_stream.json";
config_file_ = "%APPDATA%\\cdc-file-transfer\\assets_stream_manager.json";
cmd.add_argument(
lyra::opt(config_file_, "path")
.name("--config-file")
@@ -116,7 +116,7 @@ absl::StatusOr<std::unique_ptr<Log>> StartServiceCommand::GetLogger() {
}
return std::make_unique<FileLog>(
level, GetLogPath(log_dir_.c_str(), "cdc_stream").c_str());
level, GetLogPath(log_dir_.c_str(), "assets_stream_manager").c_str());
}
// Runs the session management service and returns when it finishes.
@@ -140,13 +140,15 @@ absl::Status StartServiceCommand::RunService() {
request.set_workstation_directory(cfg_.dev_src_dir());
request.set_user_host(cfg_.dev_target().user_host);
request.set_mount_dir(cfg_.dev_target().mount_dir);
request.set_port(cfg_.dev_target().ssh_port);
request.set_ssh_command(cfg_.dev_target().ssh_command);
request.set_sftp_command(cfg_.dev_target().sftp_command);
request.set_scp_command(cfg_.dev_target().scp_command);
localassetsstreammanager::StartSessionResponse response;
RETURN_ABSL_IF_ERROR(
session_service.StartSession(nullptr, &request, &response));
}
RETURN_IF_ERROR(sm_server.Start(cfg_.service_port()));
RETURN_IF_ERROR(
sm_server.Start(SessionManagementServer::kDefaultServicePort));
sm_server.RunUntilShutdown();
return absl::OkStatus();
}
cdc_stream/stop_command.cc
+2 -17
View File
@@ -21,9 +21,6 @@
#include "common/log.h"
#include "common/path.h"
#include "common/status_macros.h"
#include "grpcpp/channel.h"
#include "grpcpp/create_channel.h"
#include "grpcpp/support/channel_arguments.h"
#include "lyra/lyra.hpp"
namespace cdc_ft {
@@ -53,7 +50,7 @@ void StopCommand::RegisterCommandLineFlags(lyra::command& cmd) {
std::to_string(SessionManagementServer::kDefaultServicePort)));
cmd.add_argument(
lyra::arg(PosArgValidator(&user_host_dir_), "[user@]host:dir")
lyra::arg(PosArgValidator(&user_host_dir_), "[user@]host:src-dir")
.required()
.help("Linux host and directory to stream to"));
}
@@ -61,23 +58,11 @@ void StopCommand::RegisterCommandLineFlags(lyra::command& cmd) {
absl::Status StopCommand::Run() {
LogLevel level = Log::VerbosityToLogLevel(verbosity_);
ScopedLog scoped_log(std::make_unique<ConsoleLog>(level));
std::string client_address = absl::StrFormat("localhost:%u", service_port_);
std::shared_ptr<grpc::Channel> channel = grpc::CreateCustomChannel(
client_address, grpc::InsecureChannelCredentials(),
grpc::ChannelArguments());
LocalAssetsStreamManagerClient client(channel);
LocalAssetsStreamManagerClient client(service_port_);
std::string user_host, mount_dir;
if (user_host_dir_ == "*") {
// Convenience shortcut "*" for "*:*".
user_host = "*";
mount_dir = "*";
} else {
RETURN_IF_ERROR(LocalAssetsStreamManagerClient::ParseUserHostDir(
user_host_dir_, &user_host, &mount_dir));
}
absl::Status status = client.StopSession(user_host, mount_dir);
if (status.ok()) {
cdc_stream/stop_service_command.cc
-70
View File
@@ -1,70 +0,0 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "cdc_stream/stop_service_command.h"
#include "absl/strings/str_format.h"
#include "cdc_stream/background_service_client.h"
#include "cdc_stream/session_management_server.h"
#include "common/log.h"
#include "grpcpp/channel.h"
#include "grpcpp/create_channel.h"
#include "grpcpp/support/channel_arguments.h"
#include "lyra/lyra.hpp"
namespace cdc_ft {
StopServiceCommand::StopServiceCommand(int* exit_code)
: BaseCommand("stop-service", "Stops the streaming service", exit_code) {}
StopServiceCommand::~StopServiceCommand() = default;
void StopServiceCommand::RegisterCommandLineFlags(lyra::command& cmd) {
verbosity_ = 2;
cmd.add_argument(lyra::opt(verbosity_, "num")
.name("--verbosity")
.help("Verbosity of the log output, default: " +
std::to_string(verbosity_) +
".Increase to make logs more verbose."));
service_port_ = SessionManagementServer::kDefaultServicePort;
cmd.add_argument(lyra::opt(service_port_, "port")
.name("--service-port")
.help("Local port to use while connecting to the local "
"asset stream service, default: " +
std::to_string(service_port_)));
}
absl::Status StopServiceCommand::Run() {
LogLevel level = Log::VerbosityToLogLevel(verbosity_);
ScopedLog scoped_log(std::make_unique<ConsoleLog>(level));
std::string client_address = absl::StrFormat("localhost:%u", service_port_);
std::shared_ptr<grpc::Channel> channel = grpc::CreateCustomChannel(
client_address, grpc::InsecureChannelCredentials(),
grpc::ChannelArguments());
BackgroundServiceClient bg_client(channel);
absl::Status status = bg_client.Exit();
if (status.ok()) {
LOG_INFO("Stopped streaming service");
} else if (absl::IsUnavailable(status)) {
// Server wasn't running. This doesn't count as an error.
LOG_INFO("Streaming service already stopped");
return absl::OkStatus();
}
return status;
}
} // namespace cdc_ft
cdc_stream/stop_service_command.h
-44
View File
@@ -1,44 +0,0 @@
/*
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef CDC_STREAM_STOP_SERVICE_COMMAND_H_
#define CDC_STREAM_STOP_SERVICE_COMMAND_H_
#include <memory>
#include "absl/status/status.h"
#include "cdc_stream/base_command.h"
namespace cdc_ft {
// Handler for the stop-service command. Stops the asset streaming service.
class StopServiceCommand : public BaseCommand {
public:
explicit StopServiceCommand(int* exit_code);
~StopServiceCommand();
// BaseCommand:
void RegisterCommandLineFlags(lyra::command& cmd) override;
absl::Status Run() override;
private:
int verbosity_ = 0;
uint16_t service_port_ = 0;
};
} // namespace cdc_ft
#endif // CDC_STREAM_STOP_SERVICE_COMMAND_H_
common/BUILD
-132
View File
@@ -2,40 +2,6 @@ load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test")
package(default_visibility = ["//visibility:public"])
cc_library(
name = "ansi_filter",
srcs = ["ansi_filter.cc"],
hdrs = ["ansi_filter.h"],
)
cc_test(
name = "ansi_filter_test",
srcs = ["ansi_filter_test.cc"],
deps = [
":ansi_filter",
"@com_google_googletest//:gtest",
"@com_google_googletest//:gtest_main",
],
)
cc_library(
name = "arch_type",
srcs = ["arch_type.cc"],
hdrs = ["arch_type.h"],
deps = [":platform"],
)
cc_test(
name = "arch_type_test",
srcs = ["arch_type_test.cc"],
deps = [
":arch_type",
"@com_google_absl//absl/strings",
"@com_google_googletest//:gtest",
"@com_google_googletest//:gtest_main",
],
)
cc_library(
name = "buffer",
srcs = ["buffer.cc"],
@@ -52,35 +18,6 @@ cc_test(
],
)
cc_library(
name = "build_version",
srcs = ["build_version.cc"],
hdrs = ["build_version.h"],
# This definition should be replaced by release flow.
copts = ["-DCDC_BUILD_VERSION=DEV"],
)
cc_library(
name = "client_socket",
srcs = [
"client_socket.cc",
"socket_internal.h",
],
hdrs = ["client_socket.h"],
linkopts = select({
"//tools:windows": [
"/DEFAULTLIB:Ws2_32.lib", # Sockets, e.g. recv, send, WSA*.
],
"//conditions:default": [],
}),
deps = [
":log",
":socket",
":status",
":util",
],
)
cc_library(
name = "clock",
srcs = ["clock.cc"],
@@ -148,16 +85,6 @@ cc_test(
],
)
cc_library(
name = "fake_socket",
srcs = ["fake_socket.cc"],
hdrs = ["fake_socket.h"],
deps = [
":socket",
"@com_google_absl//absl/status",
],
)
cc_library(
name = "file_watcher",
srcs = [
@@ -215,7 +142,6 @@ cc_library(
deps = [
":clock",
":platform",
":stopwatch",
"@com_google_absl//absl/strings:str_format",
"@com_google_absl//absl/synchronization",
],
@@ -306,7 +232,6 @@ cc_library(
hdrs = ["port_manager.h"],
target_compatible_with = ["@platforms//os:windows"],
deps = [
":arch_type",
":remote_util",
":status",
":stopwatch",
@@ -329,25 +254,6 @@ cc_test(
],
)
cc_library(
name = "port_range_parser",
srcs = ["port_range_parser.cc"],
hdrs = ["port_range_parser.h"],
deps = [
"@com_google_absl//absl/strings",
],
)
cc_test(
name = "port_range_parser_test",
srcs = ["port_range_parser_test.cc"],
deps = [
":port_range_parser",
":test_main",
"@com_google_googletest//:gtest",
],
)
cc_library(
name = "process",
srcs = ["process_win.cc"],
@@ -413,7 +319,6 @@ cc_library(
srcs = ["remote_util.cc"],
hdrs = ["remote_util.h"],
deps = [
":arch_type",
":platform",
":process",
":sdk_util",
@@ -455,7 +360,6 @@ cc_library(
":path",
":platform",
":status",
"//common:build_version",
"@com_google_absl//absl/status",
"@com_google_absl//absl/strings:str_format",
],
@@ -473,42 +377,6 @@ cc_test(
],
)
cc_library(
name = "server_socket",
srcs = [
"server_socket.cc",
"socket_internal.h",
],
hdrs = ["server_socket.h"],
linkopts = select({
"//tools:windows": [
"/DEFAULTLIB:Ws2_32.lib", # Sockets, e.g. recv, send, WSA*.
],
"//conditions:default": [],
}),
deps = [
":log",
":socket",
":status",
":util",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
],
)
cc_library(
name = "socket",
srcs = ["socket.cc"],
hdrs = ["socket.h"],
deps = [
":log",
":platform",
":status",
":util",
"@com_google_absl//absl/status",
],
)
cc_library(
name = "stats_collector",
srcs = ["stats_collector.cc"],
common/ansi_filter.cc
-103
View File
@@ -1,103 +0,0 @@
// Copyright 2023 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/ansi_filter.h"
namespace cdc_ft {
namespace ansi_filter {
namespace {
enum class State {
kNotInSequence,
kDCS, // Starting with kESC + P or kDCSI, Device Control String.
kCS, // Starting with kESC + [ or kCSI, Control Sequence.
kOSC, // Starting with kESC + ] or kOSCI, Operating System Command.
};
constexpr uint8_t kBEL = 0x07; // Terminal bell.
constexpr uint8_t kESC = 0x1B; // ANSI escape character.
constexpr uint8_t kST = 0x9C; // String Terminator.
constexpr uint8_t kDCSI = 0x90; // Device Control String Introducer.
constexpr uint8_t kCSI = 0x9B; // Control Sequence Introducer.
constexpr uint8_t kOSCI = 0x9D; // Operating System Command Introducer
} // namespace
std::string RemoveEscapeSequences(const std::string& input) {
State state = State::kNotInSequence;
std::string result;
for (size_t n = 0; n < input.size(); ++n) {
uint8_t ch = static_cast<uint8_t>(input[n]);
uint8_t next_ch =
static_cast<uint8_t>(n + 1 < input.size() ? input[n + 1] : 0);
switch (state) {
case State::kNotInSequence:
// Device Control String.
if ((ch == kESC && next_ch == 'P') || ch == kDCSI) {
n += ch == kESC ? 1 : 0;
state = State::kDCS;
break;
}
// Control Sequence.
if ((ch == kESC && next_ch == '[') || ch == kCSI) {
n += ch == kESC ? 1 : 0;
state = State::kCS;
break;
}
// Operating System Command.
if ((ch == kESC && next_ch == ']') || ch == kOSCI) {
n += ch == kESC ? 1 : 0;
state = State::kOSC;
break;
}
// Char does not belong to control sequence.
result.push_back(ch);
break;
case State::kDCS:
// Device control strings are ended by kST or ESC + \.
if (ch == kST || (ch == kESC && next_ch == '\\')) {
n += ch == kESC ? 1 : 0;
state = State::kNotInSequence;
}
break;
case State::kCS:
// Control sequence initializer are ended by a byte in 0x40�0x7E.
// https://en.wikipedia.org/wiki/ANSI_escape_code#CSIsection
if (ch >= 0x40 && ch <= 0x7E) {
state = State::kNotInSequence;
}
break;
case State::kOSC:
// Operating system commands are ended by kBEL, kST or ESC + \.
// https://invisible-island.net/xterm/ctlseqs/ctlseqs.html#h3-Operating-System-Commands
if (ch == kBEL || ch == kST || (ch == kESC && next_ch == '\\')) {
n += ch == kESC ? 1 : 0;
state = State::kNotInSequence;
}
break;
}
}
return result;
}
} // namespace ansi_filter
} // namespace cdc_ft
common/ansi_filter.h
-37
View File
@@ -1,37 +0,0 @@
/*
* Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef COMMON_ANSI_FILTER_H_
#define COMMON_ANSI_FILTER_H_
#include <string>
namespace cdc_ft {
namespace ansi_filter {
// Removes ANSI escape sequences from a string.
// |input| is a string that can contain ANSI escape sequences.
// Returns the filtered string with ANSI escape sequences removed.
// Example: The most common escape sequence sets a color, e.g.
// "This \x1b[1;32merror\x1b[0m is red."
// The filtered output is
// "This error is red."
std::string RemoveEscapeSequences(const std::string& input);
} // namespace ansi_filter
} // namespace cdc_ft
#endif // COMMON_ANSI_FILTER_H_
common/ansi_filter_test.cc
-98
View File
@@ -1,98 +0,0 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/ansi_filter.h"
#include "absl/strings/ascii.h"
#include "gtest/gtest.h"
namespace cdc_ft {
namespace {
// Actual sample output from running SSH with -tt on Windows.
// Note the \0 after cmd.exe.
constexpr char kSshOutput[] =
"\x1b[2J\x1b[?25l\x1b[m\x1b["
"H\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n"
"\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\x1b[H\x1b]0;c:"
"\\windows\\system32\\cmd.exe\0\a\x1b[?25h\x1b[?25'l\x1b[120X\x1b["
"120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b["
"120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b["
"120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b["
"120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b["
"120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b["
"120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b["
"120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b["
"120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b["
"120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b["
"120C\r\n\x1b[120X\x1b[120C\r\n\x1b[120X\x1b[120C\x1b[H\x1b[?25h "
"\x1b[H\x1b[?25l\r\nfoo";
TEST(AnsiFilterTest, DoesNotExplodeOnEmptyString) {
EXPECT_EQ(ansi_filter::RemoveEscapeSequences(""), "");
}
TEST(AnsiFilterTest, KeepsUnescapedString) {
constexpr char kStr[] = "Lorem ipsum";
EXPECT_EQ(ansi_filter::RemoveEscapeSequences(kStr), kStr);
}
TEST(AnsiFilterTest, RemovesDeviceControlString) {
// Special commands for the device.
EXPECT_EQ(ansi_filter::RemoveEscapeSequences("foo\x1bPparams\x1b\\bar"),
"foobar");
EXPECT_EQ(ansi_filter::RemoveEscapeSequences("foo\x90params\x9c"
"bar"),
"foobar");
}
TEST(AnsiFilterTest, RemovesControlSequenceIntroducer) {
// E.g. the well-known regular ANSI color codes.
EXPECT_EQ(ansi_filter::RemoveEscapeSequences("foo\x1b[01;32mbar"), "foobar");
EXPECT_EQ(ansi_filter::RemoveEscapeSequences("foo\x9b"
"01;32mbar"),
"foobar");
}
TEST(AnsiFilterTest, RemovesOperatingSystemCommand) {
// E.g. setting the Window title.
// Not cool: OS commands can contain null-terminated string.
std::string str = "foo\x1b]0;c:\\path\\to\\foo.exe";
str.append(1, '\0');
str.append("\abar");
EXPECT_EQ(ansi_filter::RemoveEscapeSequences(str), "foobar");
EXPECT_EQ(ansi_filter::RemoveEscapeSequences("foo\x9dstring\x1b\\bar"),
"foobar");
}
TEST(AnsiFilterTest, RemovesRestIfNotTerminated) {
EXPECT_EQ(ansi_filter::RemoveEscapeSequences("foo\x1b[01;32"), "foo");
}
TEST(AnsiFilterTest, RemovesSequencesFromActualSshOutput) {
// Note: Can't just say str = kSshOutput because of the \0 in the string.
std::string str = std::string(kSshOutput, sizeof(kSshOutput) - 1);
std::string res = std::string(
absl::StripAsciiWhitespace(ansi_filter::RemoveEscapeSequences(str)));
EXPECT_EQ(res, "foo");
}
TEST(AnsiFilterTest, WorksForExampleFromDocumentation) {
std::string str = "This \x1b[1;32merror\x1b[0m is red.";
std::string res = std::string(ansi_filter::RemoveEscapeSequences(str));
EXPECT_EQ(res, "This error is red.");
}
} // namespace
} // namespace cdc_ft
common/arch_type.cc
-70
View File
@@ -1,70 +0,0 @@
// Copyright 2023 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/arch_type.h"
#include <cassert>
#include "common/platform.h"
namespace cdc_ft {
static constexpr char kUnhandledArchType[] = "Unhandled arch type";
ArchType GetLocalArchType() {
// TODO(ljusten): Take CPU architecture into account.
#if PLATFORM_WINDOWS
return ArchType::kWindows_x86_64;
#elif PLATFORM_LINUX
return ArchType::kLinux_x86_64;
#endif
}
bool IsWindowsArchType(ArchType arch_type) {
switch (arch_type) {
case ArchType::kWindows_x86_64:
return true;
case ArchType::kLinux_x86_64:
return false;
default:
assert(!kUnhandledArchType);
return false;
}
}
bool IsLinuxArchType(ArchType arch_type) {
switch (arch_type) {
case ArchType::kWindows_x86_64:
return false;
case ArchType::kLinux_x86_64:
return true;
default:
assert(!kUnhandledArchType);
return false;
}
}
const char* GetArchTypeStr(ArchType arch_type) {
switch (arch_type) {
case ArchType::kWindows_x86_64:
return "Windows_x86_64";
case ArchType::kLinux_x86_64:
return "Linux_x86_64";
default:
assert(!kUnhandledArchType);
return "Unknown";
}
}
} // namespace cdc_ft
common/arch_type.h
-41
View File
@@ -1,41 +0,0 @@
/*
* Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef COMMON_ARCH_TYPE_H_
#define COMMON_ARCH_TYPE_H_
namespace cdc_ft {
enum class ArchType {
kWindows_x86_64 = 0,
kLinux_x86_64 = 1,
};
// Returns the arch type of the current process.
ArchType GetLocalArchType();
// Returns true if |arch_type| is a Windows operating system.
bool IsWindowsArchType(ArchType arch_type);
// Returns true if |arch_type| is a Linux operating system.
bool IsLinuxArchType(ArchType arch_type);
// Returns a human readable string for |arch_type|.
const char* GetArchTypeStr(ArchType arch_type);
} // namespace cdc_ft
#endif // COMMON_ARCH_TYPE_H_
common/arch_type_test.cc
-49
View File
@@ -1,49 +0,0 @@
// Copyright 2023 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "common/arch_type.h"
#include "absl/strings/match.h"
#include "gtest/gtest.h"
namespace cdc_ft {
namespace {
TEST(ArchTypeTest, GetLocalArchType) {
#if PLATFORM_WINDOWS
EXPECT_TRUE(IsPlatformWindows(GetLocalArchType()));
#elif PLATFORM_LINUX
EXPECT_TRUE(IsPlatformLinux(GetLocalArchType()));
#endif
}
TEST(ArchTypeTest, IsWindowsArchType) {
EXPECT_TRUE(IsWindowsArchType(ArchType::kWindows_x86_64));
EXPECT_FALSE(IsWindowsArchType(ArchType::kLinux_x86_64));
}
TEST(ArchTypeTest, IsLinuxArchType) {
EXPECT_FALSE(IsLinuxArchType(ArchType::kWindows_x86_64));
EXPECT_TRUE(IsLinuxArchType(ArchType::kLinux_x86_64));
}
TEST(ArchTypeTest, GetArchTypeStr) {
EXPECT_TRUE(
absl::StrContains(GetArchTypeStr(ArchType::kWindows_x86_64), "Windows"));
EXPECT_TRUE(
absl::StrContains(GetArchTypeStr(ArchType::kLinux_x86_64), "Linux"));
}
} // namespace
} // namespace cdc_ft
common/build_version.cc
-9
View File
@@ -1,9 +0,0 @@
#include "build_version.h"
#ifdef CDC_BUILD_VERSION
#define TO_STR(arg) #arg
#define TO_STR_VALUE(arg) TO_STR(arg)
const char* BUILD_VERSION = TO_STR_VALUE(CDC_BUILD_VERSION);
#else
const char* BUILD_VERSION = DEV_BUILD_VERSION;
#endif
common/build_version.h
-7
View File
@@ -1,7 +0,0 @@
#ifndef COMMON_BUILD_VERSION_H_
#define COMMON_BUILD_VERSION_H_
#define DEV_BUILD_VERSION "DEV"
extern const char* BUILD_VERSION;
#endif
common/file_watcher_win.cc
+5 -20
View File
@@ -65,13 +65,7 @@ int64_t ToUnixTime(LARGE_INTEGER windows_time) {
// Background thread to read directory changes.
class AsyncFileWatcher {
public:
enum class FileWatcherState {
kDefault, // Not started.
kFailed, // Some error during watching, e.g. directory got deleted.
// Will attempt to recover automatically.
kWatching, // Actively watching directory.
kShuttingDown // Shutdown() was called, winding watcher down.
};
enum class FileWatcherState { kDefault, kFailed, kRunning, kShuttingDown };
using FileAction = FileWatcherWin::FileAction;
using FileInfo = FileWatcherWin::FileInfo;
@@ -164,17 +158,12 @@ class AsyncFileWatcher {
return dir_recreate_count_;
}
bool IsStarted() const ABSL_LOCKS_EXCLUDED(state_mutex_) {
bool IsWatching() const ABSL_LOCKS_EXCLUDED(state_mutex_) {
absl::MutexLock mutex(&state_mutex_);
return state_ != FileWatcherState::kDefault &&
state_ != FileWatcherState::kShuttingDown;
}
bool IsWatching() const ABSL_LOCKS_EXCLUDED(state_mutex_) {
absl::MutexLock mutex(&state_mutex_);
return state_ == FileWatcherState::kWatching;
}
bool IsShuttingDown() const ABSL_LOCKS_EXCLUDED(state_mutex_) {
absl::MutexLock mutex(&state_mutex_);
return state_ == FileWatcherState::kShuttingDown;
@@ -333,7 +322,7 @@ class AsyncFileWatcher {
return;
}
MaybeSetState(FileWatcherState::kWatching);
MaybeSetState(FileWatcherState::kRunning);
// Initialize handles to watch: changes in |dir_path_| and shutdown
// events.
HANDLE watch_handles[] = {overlapped.hEvent, shutdown_event_.Get()};
@@ -596,7 +585,7 @@ absl::Status FileWatcherWin::StartWatching(FilesChangedCb files_changed_cb,
async_watcher_ = std::make_unique<AsyncFileWatcher>(
dir_path_, std::move(files_changed_cb), std::move(dir_recreated_cb),
timeout_ms, enforceLegacyReadDirectoryChangesForTesting_);
while (GetStatus().ok() && !IsStarted()) {
while (GetStatus().ok() && !IsWatching()) {
std::this_thread::sleep_for(std::chrono::milliseconds(1));
}
return GetStatus();
@@ -621,10 +610,6 @@ absl::Status FileWatcherWin::StopWatching() {
return async_status;
}
bool FileWatcherWin::IsStarted() const {
return async_watcher_ ? async_watcher_->IsStarted() : false;
}
bool FileWatcherWin::IsWatching() const {
return async_watcher_ ? async_watcher_->IsWatching() : false;
}
@@ -642,7 +627,7 @@ uint32_t FileWatcherWin::GetDirRecreateEventCountForTesting() const {
}
void FileWatcherWin::EnforceLegacyReadDirectoryChangesForTesting() {
assert(!IsStarted());
assert(!IsWatching());
enforceLegacyReadDirectoryChangesForTesting_ = true;
}
common/file_watcher_win.h
+1 -6
View File
@@ -77,12 +77,7 @@ class FileWatcherWin {
// Stops watching directory changes.
absl::Status StopWatching() ABSL_LOCKS_EXCLUDED(modified_files_mutex_);
// Indicates whether StartWatching() was called, but StopWatching() was not
// called yet.
bool IsStarted() const;
// Indicates whether a directory is actively watched for changes. In contrast
// to IsStarted(), returns false while the directory does not exist.
// Indicates whether a directory is currently watched.
bool IsWatching() const;
// Returns the watching status.
common/file_watcher_win_test.cc
+1 -16
View File
@@ -135,15 +135,6 @@ class FileWatcherParameterizedTest : public ::testing::TestWithParam<bool> {
return changed;
}
// Polls for a second until the watcher is running again.
bool WaitForRunning() const {
for (int n = 0; n < 1000; ++n) {
if (watcher_.IsWatching()) return true;
Util::Sleep(1);
}
return false;
}
FileMap GetChangedFiles(size_t number_of_files) {
FileMap modified_files;
@@ -210,7 +201,7 @@ TEST_P(FileWatcherParameterizedTest, DirDoesNotExist) {
if (legacyReadDirectoryChanges_)
watcher_.EnforceLegacyReadDirectoryChangesForTesting();
EXPECT_NOT_OK(watcher.StartWatching([this]() { OnFilesChanged(); }));
EXPECT_FALSE(watcher.IsStarted());
EXPECT_FALSE(watcher.IsWatching());
absl::Status status = watcher.GetStatus();
EXPECT_NOT_OK(status);
EXPECT_TRUE(absl::IsFailedPrecondition(status));
@@ -549,9 +540,6 @@ TEST_P(FileWatcherParameterizedTest, RecreateWatchedDir) {
EXPECT_TRUE(watcher_.GetModifiedFiles().empty());
EXPECT_OK(watcher_.GetStatus());
// Wait until the watcher is running again, or else we might miss the file.
EXPECT_TRUE(WaitForRunning());
// Creation of a new file should be detected.
EXPECT_OK(path::WriteFile(first_file_path_, kFirstData, kFirstDataSize));
@@ -584,9 +572,6 @@ TEST_P(FileWatcherParameterizedTest, RecreateUpperDir) {
EXPECT_TRUE(watcher_.GetModifiedFiles().empty());
EXPECT_OK(watcher_.GetStatus());
// Wait until the watcher is running again, or else we might miss the file.
EXPECT_TRUE(WaitForRunning());
// Creation of a new file should be detected.
EXPECT_OK(path::WriteFile(first_file_path_, kFirstData, kFirstDataSize));
common/gamelet_component.cc
+11 -27
View File
@@ -18,36 +18,20 @@
#include "absl/strings/str_format.h"
#include "absl/strings/str_split.h"
#include "common/build_version.h"
#include "common/path.h"
#include "common/status.h"
namespace cdc_ft {
GameletComponent::GameletComponent(std::string build_version,
std::string filename, uint64_t size,
GameletComponent::GameletComponent(std::string filename, uint64_t size,
time_t modified_time)
: build_version(build_version),
filename(filename),
size(size),
modified_time(modified_time) {}
: filename(filename), size(size), modified_time(modified_time) {}
GameletComponent::~GameletComponent() = default;
bool GameletComponent::operator==(const GameletComponent& other) const {
if (filename != other.filename) {
return false;
}
// If either build version is the dev version, it means that the component was
// built locally, so that we can't compare build versions. Fall back to
// comparing file_size and modified_time.
if (build_version != DEV_BUILD_VERSION &&
build_version != DEV_BUILD_VERSION) {
return build_version == other.build_version;
}
return size == other.size && modified_time == other.modified_time;
return filename == other.filename && size == other.size &&
modified_time == other.modified_time;
}
bool GameletComponent::operator!=(const GameletComponent& other) const {
@@ -65,7 +49,7 @@ absl::Status GameletComponent::Get(
absl::Status status = path::GetStats(path, &stats);
if (!status.ok())
return WrapStatus(status, "GetStats() failed for '%s'", path);
components->emplace_back(BUILD_VERSION, path::BaseName(path), stats.size,
components->emplace_back(path::BaseName(path), stats.size,
stats.modified_time);
}
@@ -77,9 +61,9 @@ std::string GameletComponent::ToCommandLineArgs(
const std::vector<GameletComponent>& components) {
std::string args;
for (const GameletComponent& comp : components) {
args += absl::StrFormat("%s%s %s %u %d", args.empty() ? "" : " ",
comp.build_version.c_str(), comp.filename.c_str(),
comp.size, comp.modified_time);
args +=
absl::StrFormat("%s%s %u %d", args.empty() ? "" : " ",
comp.filename.c_str(), comp.size, comp.modified_time);
}
return args;
}
@@ -88,9 +72,9 @@ std::string GameletComponent::ToCommandLineArgs(
std::vector<GameletComponent> GameletComponent::FromCommandLineArgs(
int argc, const char** argv) {
std::vector<GameletComponent> components;
for (int n = 0; n + 3 < argc; n += 4) {
components.emplace_back(argv[n], argv[n + 1], std::stol(argv[n + 2]),
std::stol(argv[n + 3]));
for (int n = 0; n + 2 < argc; n += 3) {
components.emplace_back(argv[n], std::stol(argv[n + 1]),
std::stol(argv[n + 2]));
}
return components;
}
common/gamelet_component.h
+1 -3
View File
@@ -28,13 +28,11 @@ namespace cdc_ft {
// The components are considered fresh if both the timestamp and the file size
// match.
struct GameletComponent {
std::string build_version;
std::string filename;
uint64_t size;
int64_t modified_time;
GameletComponent(std::string build_version, std::string filename,
uint64_t size, time_t modified_time);
GameletComponent(std::string filename, uint64_t size, time_t modified_time);
~GameletComponent();
bool operator==(const GameletComponent& other) const;
common/gamelet_component_test.cc
+2 -56
View File
@@ -15,7 +15,6 @@
#include "common/gamelet_component.h"
#include "absl/strings/str_split.h"
#include "common/build_version.h"
#include "common/log.h"
#include "common/path.h"
#include "common/status_test_macros.h"
@@ -44,14 +43,14 @@ class GameletComponentTest : public ::testing::Test {
path::Join(base_dir_, "other", "cdc_rsync_server");
};
TEST_F(GameletComponentTest, EqualityOperators_DevelopmentVersion) {
TEST_F(GameletComponentTest, EqualityOperators) {
constexpr uint64_t size1 = 1001;
constexpr uint64_t size2 = 1002;
constexpr int64_t modified_time1 = 5001;
constexpr int64_t modified_time2 = 5002;
GameletComponent a(DEV_BUILD_VERSION, "file1", size1, modified_time1);
GameletComponent a("file1", size1, modified_time1);
GameletComponent b = a;
EXPECT_TRUE(a == b && !(a != b));
@@ -66,38 +65,6 @@ TEST_F(GameletComponentTest, EqualityOperators_DevelopmentVersion) {
b = a;
b.modified_time = modified_time2;
EXPECT_TRUE(!(a == b) && a != b);
b = a;
b.size = size2;
b.build_version = "Specified";
EXPECT_TRUE(!(a == b) && a != b);
a.build_version = "Specified";
EXPECT_TRUE(a == b && !(a != b));
}
TEST_F(GameletComponentTest, EqualityOperators_SpecifiedVersion) {
constexpr uint64_t size1 = 1001;
constexpr uint64_t size2 = 1002;
constexpr int64_t modified_time1 = 5001;
constexpr int64_t modified_time2 = 5002;
GameletComponent a("Specified", "file1", size1, modified_time1);
GameletComponent b = a;
EXPECT_TRUE(a == b && !(a != b));
b.filename = "file2";
EXPECT_TRUE(!(a == b) && a != b);
b = a;
b.size = size2;
EXPECT_TRUE(a == b && !(a != b));
b = a;
b.modified_time = modified_time2;
EXPECT_TRUE(a == b && !(a != b));
}
TEST_F(GameletComponentTest, GetValidComponents) {
@@ -124,30 +91,9 @@ TEST_F(GameletComponentTest, GetChangedComponents) {
// Force equal timestamps, so that we don't depend on when the files were
// actually written to everyone's drives.
// Also force set build_version to developer since otherwise we would skip
// component size check.
ASSERT_EQ(components.size(), other_components.size());
for (size_t n = 0; n < components.size(); ++n) {
other_components[n].modified_time = components[n].modified_time;
other_components[n].build_version = DEV_BUILD_VERSION;
EXPECT_NE(components, other_components);
}
}
TEST_F(GameletComponentTest, GetChangedComponents_BuildVersionChanged) {
std::vector<GameletComponent> components;
EXPECT_OK(GameletComponent::Get({valid_component_path_}, &components));
std::vector<GameletComponent> other_components;
EXPECT_OK(GameletComponent::Get({other_component_path_}, &other_components));
ASSERT_EQ(components.size(), other_components.size());
for (size_t n = 0; n < components.size(); ++n) {
other_components[n].modified_time = components[n].modified_time;
other_components[n].size = components[n].size;
components[n].build_version = "build_version";
other_components[n].build_version = "other_build_version";
EXPECT_NE(components, other_components);
}
common/log.cc
+6 -7
View File
@@ -126,22 +126,21 @@ void ConsoleLog::WriteLogMessage(LogLevel level, const char* file, int line,
absl::MutexLock lock(&mutex_);
// Show leaner log messages in non-verbose mode.
bool show_time_file_func = GetLogLevel() <= LogLevel::kDebug;
bool show_file_func = GetLogLevel() <= LogLevel::kDebug;
FILE* stdfile = level >= LogLevel::kError ? stderr : stdout;
#if PLATFORM_WINDOWS
HANDLE hConsole = GetStdHandle(STD_OUTPUT_HANDLE);
SetConsoleTextAttribute(hConsole, GetConsoleColor(level));
if (show_time_file_func) {
fprintf(stdfile, "%0.3f %s(%i): %s(): %s\n", stopwatch_.ElapsedSeconds(),
file, line, func, message);
if (show_file_func) {
fprintf(stdfile, "%s(%i): %s(): %s\n", file, line, func, message);
} else {
fprintf(stdfile, "%s\n", message);
}
SetConsoleTextAttribute(hConsole, kLightGray);
#else
if (show_time_file_func) {
fprintf(stdfile, "%-7s %0.3f %s(%i): %s(): %s\n", GetLogLevelString(level),
stopwatch_.ElapsedSeconds(), file, line, func, message);
if (show_file_func) {
fprintf(stdfile, "%-7s %s(%i): %s(): %s\n", GetLogLevelString(level), file,
line, func, message);
} else {
fprintf(stdfile, "%-7s %s\n", GetLogLevelString(level), message);
}
common/log.h
-2
View File
@@ -22,7 +22,6 @@
#include "absl/strings/str_format.h"
#include "absl/synchronization/mutex.h"
#include "common/clock.h"
#include "common/stopwatch.h"
namespace cdc_ft {
@@ -121,7 +120,6 @@ class ConsoleLog : public Log {
ABSL_LOCKS_EXCLUDED(mutex_);
private:
Stopwatch stopwatch_;
absl::Mutex mutex_;
};
common/path.cc
+3 -6
View File
@@ -219,12 +219,9 @@ absl::Status ExpandPathVariables(std::string* path) {
*path = Util::WideToUtf8Str(wchar_expanded);
return absl::OkStatus();
#else
// Exclude command substitution. It.s not what users of this method would
// expect and could lead to security issues.
wordexp_t res;
wordexp(path->c_str(), &res, WRDE_NOCMD);
wordexp(path->c_str(), &res, 0);
if (res.we_wordc > 1) {
wordfree(&res);
return absl::InvalidArgumentError(
"Path expands to multiple results (did you use * etc. ?");
}
@@ -294,8 +291,8 @@ std::string GetDrivePrefix(const std::string& path) {
if (path[0] != '\\') {
size_t pos = path.find(":");
if (pos != 1) {
// E.g. "\path\to\file", "path\to\file" or "user@host:file".
if (pos == std::string::npos) {
// E.g. "\path\to\file" or "path\to\file".
return std::string();
}
common/path.h
+2 -2
View File
@@ -104,8 +104,8 @@ absl::Status GetKnownFolderPath(FolderId folder_id, std::string* path);
// Expands environment path variables like %APPDATA% on Windows or ~ on Linux.
// On Windows, variables are matched case invariantly. Unknown environment
// variables are not changed.
// On Linux, performs a shell-like expansion, but without command substitution.
// Returns an error if multiple results would be returned, e.g. from *.txt.
// On Linux, performs a shell-like expansion. Returns an error if multiple
// results would be returned, e.g. from *.txt.
absl::Status ExpandPathVariables(std::string* path);
// Returns the environment variable with given |name| in |value|.
common/path_test.cc
-1
View File
@@ -302,7 +302,6 @@ TEST_F(PathTest, GetDrivePrefix) {
EXPECT_EQ(path::GetDrivePrefix("C:\\"), "C:");
EXPECT_EQ(path::GetDrivePrefix("C:\\dir"), "C:");
EXPECT_EQ(path::GetDrivePrefix("C:\\dir\\file"), "C:");
EXPECT_EQ(path::GetDrivePrefix("host:C:\\dir\\file"), "");
}
#endif
common/port_manager.h
+18 -18
View File
@@ -17,12 +17,12 @@
#ifndef COMMON_PORT_MANAGER_H_
#define COMMON_PORT_MANAGER_H_
#include <absl/status/statusor.h>
#include <memory>
#include <string>
#include <unordered_set>
#include "absl/status/statusor.h"
#include "common/arch_type.h"
#include "common/clock.h"
namespace cdc_ft {
@@ -40,8 +40,8 @@ class PortManager {
// synchronize port reservation. The range of possible ports managed by this
// instance is [|first_port|, |last_port|]. |process_factory| is a valid
// pointer to a ProcessFactory instance to run processes locally.
// |remote_util| is the RemoteUtil instance to run processes remotely. If it
// is nullptr, no remote ports are reserved.
// |remote_util| is a valid pointer to a RemoteUtil instance to run processes
// remotely.
PortManager(std::string unique_name, int first_port, int last_port,
ProcessFactory* process_factory, RemoteUtil* remote_util,
SystemClock* system_clock = DefaultSystemClock::GetInstance(),
@@ -51,14 +51,14 @@ class PortManager {
// Reserves a port in the range passed to the constructor. The port is
// released automatically upon destruction if ReleasePort() is not called
// explicitly.
// |check_remote| determines whether the remote port should be checked as
// well. If false, the check is skipped and a port might be returned that is
// still in use remotely.
// |remote_timeout_sec| is the timeout for finding available ports on the
// remote instance.
// |remote_arch_type| is the architecture of the remote device.
// Both |remote_timeout_sec| and |remote_arch_type| are ignored if
// |remote_util| is nullptr. Returns a DeadlineExceeded error if the timeout
// is exceeded. Returns a ResourceExhausted error if no ports are available.
absl::StatusOr<int> ReservePort(int remote_timeout_sec,
ArchType remote_arch_type);
// remote instance. Not used if |check_remote| is false.
// Returns a DeadlineExceeded error if the timeout is exceeded.
// Returns a ResourceExhausted error if no ports are available.
absl::StatusOr<int> ReservePort(bool check_remote, int remote_timeout_sec);
// Releases a reserved port.
absl::Status ReleasePort(int port);
@@ -69,34 +69,34 @@ class PortManager {
// Finds available ports in the range [first_port, last_port] for port
// forwarding on the local workstation.
// |arch_type| is the architecture of the local device.
// |ip| is the IP address to filter by.
// |process_factory| is used to create a netstat process.
// Returns ResourceExhaustedError if no port is available.
static absl::StatusOr<std::unordered_set<int>> FindAvailableLocalPorts(
int first_port, int last_port, ArchType arch_type,
int first_port, int last_port, const char* ip,
ProcessFactory* process_factory);
// Finds available ports in the range [first_port, last_port] for port
// forwarding on the instance.
// |arch_type| is the architecture of the remote device.
// |ip| is the IP address to filter by.
// |process_factory| is used to create a netstat process.
// |remote_util| is used to connect to the instance.
// |timeout_sec| is the connection timeout in seconds.
// Returns a DeadlineExceeded error if the timeout is exceeded.
// Returns ResourceExhaustedError if no port is available.
static absl::StatusOr<std::unordered_set<int>> FindAvailableRemotePorts(
int first_port, int last_port, ArchType arch_type,
int first_port, int last_port, const char* ip,
ProcessFactory* process_factory, RemoteUtil* remote_util, int timeout_sec,
SteadyClock* steady_clock = DefaultSteadyClock::GetInstance());
private:
// Returns a list of available ports in the range [|first_port|, |last_port|]
// from the given |netstat_output|.
// |arch_type| is the architecture of the device where netstat was called.
// from the given |netstat_output|. |ip| is the IP address to look for, e.g.
// "127.0.0.1".
// Returns ResourceExhaustedError if no port is available.
static absl::StatusOr<std::unordered_set<int>> FindAvailablePorts(
int first_port, int last_port, const std::string& netstat_output,
ArchType arch_type);
const char* ip);
int first_port_;
int last_port_;

Some files were not shown because too many files have changed in this diff Show More