catarrh/cdc-file-transfer
1
0
Fork 0
mirror of https://github.com/google/cdc-file-transfer.git synced 2026-09-13 01:10:44 +03:00
Code Issues Packages Projects Releases Wiki Activity

[cdc_rsync] Detect remote architecture (#86)

Browse Source 
Improves ServerArch so that it can detect the remote architecture by
running uname and checking %PROCESSOR_ARCHITECTURE%. So far, only
x64 Linux and x64 Windows are supported, but in the future it is easy
to add support for others, e.g. aarch64, as well.

Before the detection is run, the remote architecture is guessed first
based on the destination. For instance, if the destination directory
starts with "C:\", it pretty much means Windows. If cdc_rsync_server
exists and runs fine, there's no need for detection.

Since also PortManager depends on the remote architecture, it has to
be adjusted as well. So far, PortManager assumeed that "local" means
Windows and "remote" means Linux. This is no longer the case for
syncing to Windows devices, so this CL adds the necessary abstractions
to PortManager.

Also refactors ArchType into a separate class in common, since it is
used now from several places. It is also expanded to handle future
changes that add support for different processor architectures, e.g.
aarch64.
This commit is contained in:
Lutz Justen
2023-02-01 11:51:20 +01:00
committed by GitHub
parent 3194678007
commit ee4118c6bf
19 changed files with 696 additions and 228 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/port_manager_test.cc
+113 -67
View File
@@ -25,7 +25,6 @@
namespace cdc_ft {
namespace {
constexpr int kSshPort = 12345;
constexpr char kUserHost[] = "user@1.2.3.4";
constexpr char kGuid[] = "f77bcdfe-368c-4c45-9f01-230c5e7e2132";
@@ -35,12 +34,12 @@ constexpr int kNumPorts = kLastPort - kFirstPort + 1;
constexpr int kTimeoutSec = 1;
constexpr char kLocalNetstat[] = "netstat -a -n -p tcp";
constexpr char kRemoteNetstat[] = "netstat --numeric --listening --tcp";
constexpr char kWindowsNetstat[] = "netstat -a -n -p tcp";
constexpr char kLinuxNetstat[] = "netstat --numeric --listening --tcp";
constexpr char kLocalNetstatOutFmt[] =
constexpr char kWindowsNetstatOutFmt[] =
"TCP 127.0.0.1:50000 127.0.0.1:%i ESTABLISHED";
constexpr char kRemoteNetstatOutFmt[] =
constexpr char kLinuxNetstatOutFmt[] =
"tcp 0 0 0.0.0.0:%i 0.0.0.0:* LISTEN";
class PortManagerTest : public ::testing::Test {
@@ -67,10 +66,11 @@ class PortManagerTest : public ::testing::Test {
};
TEST_F(PortManagerTest, ReservePortSuccess) {
process_factory_.SetProcessOutput(kLocalNetstat, "", "", 0);
process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 0);
process_factory_.SetProcessOutput(kWindowsNetstat, "", "", 0);
process_factory_.SetProcessOutput(kLinuxNetstat, "", "", 0);
absl::StatusOr<int> port = port_manager_.ReservePort(kTimeoutSec);
absl::StatusOr<int> port =
port_manager_.ReservePort(kTimeoutSec, ArchType::kLinux_x86_64);
ASSERT_OK(port);
EXPECT_EQ(*port, kFirstPort);
}
@@ -78,12 +78,13 @@ TEST_F(PortManagerTest, ReservePortSuccess) {
TEST_F(PortManagerTest, ReservePortAllLocalPortsTaken) {
std::string local_netstat_out = "";
for (int port = kFirstPort; port <= kLastPort; ++port) {
local_netstat_out += absl::StrFormat(kLocalNetstatOutFmt, port);
local_netstat_out += absl::StrFormat(kWindowsNetstatOutFmt, port);
}
process_factory_.SetProcessOutput(kLocalNetstat, local_netstat_out, "", 0);
process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 0);
process_factory_.SetProcessOutput(kWindowsNetstat, local_netstat_out, "", 0);
process_factory_.SetProcessOutput(kLinuxNetstat, "", "", 0);
absl::StatusOr<int> port = port_manager_.ReservePort(kTimeoutSec);
absl::StatusOr<int> port =
port_manager_.ReservePort(kTimeoutSec, ArchType::kLinux_x86_64);
EXPECT_TRUE(absl::IsResourceExhausted(port.status()));
EXPECT_TRUE(
absl::StrContains(port.status().message(), "No port available in range"));
@@ -92,22 +93,24 @@ TEST_F(PortManagerTest, ReservePortAllLocalPortsTaken) {
TEST_F(PortManagerTest, ReservePortAllRemotePortsTaken) {
std::string remote_netstat_out = "";
for (int port = kFirstPort; port <= kLastPort; ++port) {
remote_netstat_out += absl::StrFormat(kRemoteNetstatOutFmt, port);
remote_netstat_out += absl::StrFormat(kLinuxNetstatOutFmt, port);
}
process_factory_.SetProcessOutput(kLocalNetstat, "", "", 0);
process_factory_.SetProcessOutput(kRemoteNetstat, remote_netstat_out, "", 0);
process_factory_.SetProcessOutput(kWindowsNetstat, "", "", 0);
process_factory_.SetProcessOutput(kLinuxNetstat, remote_netstat_out, "", 0);
absl::StatusOr<int> port = port_manager_.ReservePort(kTimeoutSec);
absl::StatusOr<int> port =
port_manager_.ReservePort(kTimeoutSec, ArchType::kLinux_x86_64);
EXPECT_TRUE(absl::IsResourceExhausted(port.status()));
EXPECT_TRUE(
absl::StrContains(port.status().message(), "No port available in range"));
}
TEST_F(PortManagerTest, ReservePortLocalNetstatFails) {
process_factory_.SetProcessOutput(kLocalNetstat, "", "", 1);
process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 0);
process_factory_.SetProcessOutput(kWindowsNetstat, "", "", 1);
process_factory_.SetProcessOutput(kLinuxNetstat, "", "", 0);
absl::StatusOr<int> port = port_manager_.ReservePort(kTimeoutSec);
absl::StatusOr<int> port =
port_manager_.ReservePort(kTimeoutSec, ArchType::kLinux_x86_64);
EXPECT_NOT_OK(port);
EXPECT_TRUE(
absl::StrContains(port.status().message(),
@@ -115,21 +118,23 @@ TEST_F(PortManagerTest, ReservePortLocalNetstatFails) {
}
TEST_F(PortManagerTest, ReservePortRemoteNetstatFails) {
process_factory_.SetProcessOutput(kLocalNetstat, "", "", 0);
process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 1);
process_factory_.SetProcessOutput(kWindowsNetstat, "", "", 0);
process_factory_.SetProcessOutput(kLinuxNetstat, "", "", 1);
absl::StatusOr<int> port = port_manager_.ReservePort(kTimeoutSec);
absl::StatusOr<int> port =
port_manager_.ReservePort(kTimeoutSec, ArchType::kLinux_x86_64);
EXPECT_NOT_OK(port);
EXPECT_TRUE(absl::StrContains(port.status().message(),
"Failed to find available ports on instance"));
}
TEST_F(PortManagerTest, ReservePortRemoteNetstatTimesOut) {
process_factory_.SetProcessOutput(kLocalNetstat, "", "", 0);
process_factory_.SetProcessNeverExits(kRemoteNetstat);
process_factory_.SetProcessOutput(kWindowsNetstat, "", "", 0);
process_factory_.SetProcessNeverExits(kLinuxNetstat);
steady_clock_.AutoAdvance(kTimeoutSec * 2 * 1000);
absl::StatusOr<int> port = port_manager_.ReservePort(kTimeoutSec);
absl::StatusOr<int> port =
port_manager_.ReservePort(kTimeoutSec, ArchType::kLinux_x86_64);
EXPECT_NOT_OK(port);
EXPECT_TRUE(absl::IsDeadlineExceeded(port.status()));
EXPECT_TRUE(absl::StrContains(port.status().message(),
@@ -137,8 +142,8 @@ TEST_F(PortManagerTest, ReservePortRemoteNetstatTimesOut) {
}
TEST_F(PortManagerTest, ReservePortMultipleInstances) {
process_factory_.SetProcessOutput(kLocalNetstat, "", "", 0);
process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 0);
process_factory_.SetProcessOutput(kWindowsNetstat, "", "", 0);
process_factory_.SetProcessOutput(kLinuxNetstat, "", "", 0);
PortManager port_manager2(kGuid, kFirstPort, kLastPort, &process_factory_,
&remote_util_);
@@ -146,55 +151,79 @@ TEST_F(PortManagerTest, ReservePortMultipleInstances) {
// Port managers use shared memory, so different instances know about each
// other. This would even work if |port_manager_| and |port_manager2| belonged
// to different processes, but we don't test that here.
EXPECT_EQ(*port_manager_.ReservePort(kTimeoutSec), kFirstPort + 0);
EXPECT_EQ(*port_manager2.ReservePort(kTimeoutSec), kFirstPort + 1);
EXPECT_EQ(*port_manager_.ReservePort(kTimeoutSec), kFirstPort + 2);
EXPECT_EQ(*port_manager2.ReservePort(kTimeoutSec), kFirstPort + 3);
EXPECT_EQ(*port_manager_.ReservePort(kTimeoutSec, ArchType::kLinux_x86_64),
kFirstPort + 0);
EXPECT_EQ(*port_manager2.ReservePort(kTimeoutSec, ArchType::kLinux_x86_64),
kFirstPort + 1);
EXPECT_EQ(*port_manager_.ReservePort(kTimeoutSec, ArchType::kLinux_x86_64),
kFirstPort + 2);
EXPECT_EQ(*port_manager2.ReservePort(kTimeoutSec, ArchType::kLinux_x86_64),
kFirstPort + 3);
}
TEST_F(PortManagerTest, ReservePortReusesPortsInLRUOrder) {
process_factory_.SetProcessOutput(kLocalNetstat, "", "", 0);
process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 0);
process_factory_.SetProcessOutput(kWindowsNetstat, "", "", 0);
process_factory_.SetProcessOutput(kLinuxNetstat, "", "", 0);
for (int n = 0; n < kNumPorts * 2; ++n) {
EXPECT_EQ(*port_manager_.ReservePort(kTimeoutSec),
EXPECT_EQ(*port_manager_.ReservePort(kTimeoutSec, ArchType::kLinux_x86_64),
kFirstPort + n % kNumPorts);
system_clock_.Advance(1000);
}
}
TEST_F(PortManagerTest, ReleasePort) {
process_factory_.SetProcessOutput(kLocalNetstat, "", "", 0);
process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 0);
process_factory_.SetProcessOutput(kWindowsNetstat, "", "", 0);
process_factory_.SetProcessOutput(kLinuxNetstat, "", "", 0);
absl::StatusOr<int> port = port_manager_.ReservePort(kTimeoutSec);
absl::StatusOr<int> port =
port_manager_.ReservePort(kTimeoutSec, ArchType::kLinux_x86_64);
EXPECT_EQ(*port, kFirstPort);
EXPECT_OK(port_manager_.ReleasePort(*port));
port = port_manager_.ReservePort(kTimeoutSec);
port = port_manager_.ReservePort(kTimeoutSec, ArchType::kLinux_x86_64);
EXPECT_EQ(*port, kFirstPort);
}
TEST_F(PortManagerTest, ReleasePortOnDestruction) {
process_factory_.SetProcessOutput(kLocalNetstat, "", "", 0);
process_factory_.SetProcessOutput(kRemoteNetstat, "", "", 0);
process_factory_.SetProcessOutput(kWindowsNetstat, "", "", 0);
process_factory_.SetProcessOutput(kLinuxNetstat, "", "", 0);
auto port_manager2 = std::make_unique<PortManager>(
kGuid, kFirstPort, kLastPort, &process_factory_, &remote_util_);
EXPECT_EQ(*port_manager2->ReservePort(kTimeoutSec), kFirstPort + 0);
EXPECT_EQ(*port_manager_.ReservePort(kTimeoutSec), kFirstPort + 1);
EXPECT_EQ(*port_manager2->ReservePort(kTimeoutSec, ArchType::kLinux_x86_64),
kFirstPort + 0);
EXPECT_EQ(*port_manager_.ReservePort(kTimeoutSec, ArchType::kLinux_x86_64),
kFirstPort + 1);
port_manager2.reset();
EXPECT_EQ(*port_manager_.ReservePort(kTimeoutSec), kFirstPort + 0);
EXPECT_EQ(*port_manager_.ReservePort(kTimeoutSec, ArchType::kLinux_x86_64),
kFirstPort + 0);
}
TEST_F(PortManagerTest, FindAvailableLocalPortsSuccess) {
// First port is taken
TEST_F(PortManagerTest, FindAvailableLocalPortsSuccessWindows) {
// First port is in use.
std::string local_netstat_out =
absl::StrFormat(kLocalNetstatOutFmt, kFirstPort);
process_factory_.SetProcessOutput(kLocalNetstat, local_netstat_out, "", 0);
absl::StrFormat(kWindowsNetstatOutFmt, kFirstPort);
process_factory_.SetProcessOutput(kWindowsNetstat, local_netstat_out, "", 0);
absl::StatusOr<std::unordered_set<int>> ports =
PortManager::FindAvailableLocalPorts(kFirstPort, kLastPort, "127.0.0.1",
&process_factory_);
PortManager::FindAvailableLocalPorts(
kFirstPort, kLastPort, ArchType::kWindows_x86_64, &process_factory_);
ASSERT_OK(ports);
EXPECT_EQ(ports->size(), kNumPorts - 1);
for (int port = kFirstPort + 1; port <= kLastPort; ++port) {
EXPECT_TRUE(ports->find(port) != ports->end());
}
}
TEST_F(PortManagerTest, FindAvailableLocalPortsSuccessLinux) {
// First port is in use.
std::string local_netstat_out =
absl::StrFormat(kLinuxNetstatOutFmt, kFirstPort);
process_factory_.SetProcessOutput(kLinuxNetstat, local_netstat_out, "", 0);
absl::StatusOr<std::unordered_set<int>> ports =
PortManager::FindAvailableLocalPorts(
kFirstPort, kLastPort, ArchType::kLinux_x86_64, &process_factory_);
ASSERT_OK(ports);
EXPECT_EQ(ports->size(), kNumPorts - 1);
for (int port = kFirstPort + 1; port <= kLastPort; ++port) {
@@ -203,31 +232,48 @@ TEST_F(PortManagerTest, FindAvailableLocalPortsSuccess) {
}
TEST_F(PortManagerTest, FindAvailableLocalPortsFailsNoPorts) {
// All ports taken
// All ports are in use.
std::string local_netstat_out = "";
for (int port = kFirstPort; port <= kLastPort; ++port) {
local_netstat_out += absl::StrFormat(kLocalNetstatOutFmt, port);
local_netstat_out += absl::StrFormat(kWindowsNetstatOutFmt, port);
}
process_factory_.SetProcessOutput(kLocalNetstat, local_netstat_out, "", 0);
process_factory_.SetProcessOutput(kWindowsNetstat, local_netstat_out, "", 0);
absl::StatusOr<std::unordered_set<int>> ports =
PortManager::FindAvailableLocalPorts(kFirstPort, kLastPort, "127.0.0.1",
&process_factory_);
PortManager::FindAvailableLocalPorts(
kFirstPort, kLastPort, ArchType::kWindows_x86_64, &process_factory_);
EXPECT_TRUE(absl::IsResourceExhausted(ports.status()));
EXPECT_TRUE(absl::StrContains(ports.status().message(),
"No port available in range"));
}
TEST_F(PortManagerTest, FindAvailableRemotePortsSuccess) {
// First port is taken
TEST_F(PortManagerTest, FindAvailableRemotePortsSuccessLinux) {
// First port is in use.
std::string remote_netstat_out =
absl::StrFormat(kRemoteNetstatOutFmt, kFirstPort);
process_factory_.SetProcessOutput(kRemoteNetstat, remote_netstat_out, "", 0);
absl::StrFormat(kLinuxNetstatOutFmt, kFirstPort);
process_factory_.SetProcessOutput(kLinuxNetstat, remote_netstat_out, "", 0);
absl::StatusOr<std::unordered_set<int>> ports =
PortManager::FindAvailableRemotePorts(kFirstPort, kLastPort, "0.0.0.0",
&process_factory_, &remote_util_,
kTimeoutSec);
PortManager::FindAvailableRemotePorts(
kFirstPort, kLastPort, ArchType::kLinux_x86_64, &process_factory_,
&remote_util_, kTimeoutSec);
ASSERT_OK(ports);
EXPECT_EQ(ports->size(), kNumPorts - 1);
for (int port = kFirstPort + 1; port <= kLastPort; ++port) {
EXPECT_TRUE(ports->find(port) != ports->end());
}
}
TEST_F(PortManagerTest, FindAvailableRemotePortsSuccessWindows) {
// First port is in use.
std::string remote_netstat_out =
absl::StrFormat(kWindowsNetstatOutFmt, kFirstPort);
process_factory_.SetProcessOutput(kWindowsNetstat, remote_netstat_out, "", 0);
absl::StatusOr<std::unordered_set<int>> ports =
PortManager::FindAvailableRemotePorts(
kFirstPort, kLastPort, ArchType::kWindows_x86_64, &process_factory_,
&remote_util_, kTimeoutSec);
ASSERT_OK(ports);
EXPECT_EQ(ports->size(), kNumPorts - 1);
for (int port = kFirstPort + 1; port <= kLastPort; ++port) {
@@ -236,17 +282,17 @@ TEST_F(PortManagerTest, FindAvailableRemotePortsSuccess) {
}
TEST_F(PortManagerTest, FindAvailableRemotePortsFailsNoPorts) {
// All ports taken
// All ports are in use.
std::string remote_netstat_out = "";
for (int port = kFirstPort; port <= kLastPort; ++port) {
remote_netstat_out += absl::StrFormat(kRemoteNetstatOutFmt, port);
remote_netstat_out += absl::StrFormat(kLinuxNetstatOutFmt, port);
}
process_factory_.SetProcessOutput(kRemoteNetstat, remote_netstat_out, "", 0);
process_factory_.SetProcessOutput(kLinuxNetstat, remote_netstat_out, "", 0);
absl::StatusOr<std::unordered_set<int>> ports =
PortManager::FindAvailableRemotePorts(kFirstPort, kLastPort, "0.0.0.0",
&process_factory_, &remote_util_,
kTimeoutSec);
PortManager::FindAvailableRemotePorts(
kFirstPort, kLastPort, ArchType::kLinux_x86_64, &process_factory_,
&remote_util_, kTimeoutSec);
EXPECT_TRUE(absl::IsResourceExhausted(ports.status()));
EXPECT_TRUE(absl::StrContains(ports.status().message(),
"No port available in range"));