diff --git a/src/roma/byob/config/config.h b/src/roma/byob/config/config.h index 5d25f754..3b6a07f3 100644 --- a/src/roma/byob/config/config.h +++ b/src/roma/byob/config/config.h @@ -24,13 +24,51 @@ #ifndef LIB_MOUNTS #if defined(__aarch64__) -#define LIB_MOUNTS "/lib,/usr/lib" +#define LIB_MOUNTS "/lib" #else -#define LIB_MOUNTS "/lib,/lib64,/usr/lib" +#define LIB_MOUNTS "/lib,/lib64" #endif #endif /* LIB_MOUNTS */ namespace privacy_sandbox::server_common::byob { + +enum class Mode { + kModeSandbox, + kModeNoSandbox, + kModeSandboxDebug, +}; + +inline bool AbslParseFlag(absl::string_view text, Mode* mode, + std::string* error) { + if (text == "on") { + *mode = Mode::kModeSandbox; + return true; + } + if (text == "debug") { + *mode = Mode::kModeSandboxDebug; + return true; + } + if (text == "off") { + *mode = Mode::kModeNoSandbox; + return true; + } + *error = "Supported values: on, off, debug."; + return false; +} + +inline std::string AbslUnparseFlag(Mode mode) { + switch (mode) { + case Mode::kModeSandbox: + return "on"; + case Mode::kModeSandboxDebug: + return "debug"; + case Mode::kModeNoSandbox: + return "off"; + default: + return absl::StrCat(mode); + } +} + template struct Config { std::uint64_t memory_limit_soft = 0; diff --git a/src/roma/byob/container/BUILD.bazel b/src/roma/byob/container/BUILD.bazel index c3ddb421..e3e1f5ce 100644 --- a/src/roma/byob/container/BUILD.bazel +++ b/src/roma/byob/container/BUILD.bazel @@ -96,6 +96,7 @@ cc_binary( "//src/roma/byob/dispatcher", "//src/roma/byob/dispatcher:dispatcher_cc_proto", "//src/roma/byob/dispatcher:dispatcher_grpc", + "//src/roma/byob/utility:utils", "//src/util/status_macro:status_macros", "//src/util/status_macro:status_util", "@com_github_grpc_grpc//:grpc++", @@ -204,9 +205,11 @@ docker rm $$CID outs = ["byob_runtime_container_with_dir_{}.tar".format(user.flavor)], cmd_bash = """ TMPDIR="$$(mktemp --directory roma_container_tmp.XXXXXXXXXX)" +MACHINE="$$(uname --machine)" mkdir --parents --mode=700 "$$TMPDIR/{roma_container_dir}/{roma_container_root_dir}" tar --extract --file="$(location :byob_runtime_container_{flavor}.tar)" --directory="$$TMPDIR/{roma_container_dir}/{roma_container_root_dir}" tar --extract --file="$(location :container_config_{flavor}.tar)" --directory="$$TMPDIR/{roma_container_dir}" +cp "$$TMPDIR/{roma_container_dir}/{roma_container_root_dir}/usr/lib/$$MACHINE-linux-gnu"/libstdc++.so* "$$TMPDIR/{roma_container_dir}/{roma_container_root_dir}"/lib/$$MACHINE-linux-gnu/ tar --create --gzip --owner={user}:{uid} --group={group}:{gid} --file=$@ --directory="$$TMPDIR" "{roma_container_dir}/" rm -rf "$$TMPDIR" """.format( diff --git a/src/roma/byob/container/run_workers.cc b/src/roma/byob/container/run_workers.cc index 9a7d7b49..05570a19 100644 --- a/src/roma/byob/container/run_workers.cc +++ b/src/roma/byob/container/run_workers.cc @@ -59,6 +59,7 @@ #include "src/roma/byob/dispatcher/dispatcher.grpc.pb.h" #include "src/roma/byob/dispatcher/dispatcher.h" #include "src/roma/byob/dispatcher/dispatcher.pb.h" +#include "src/roma/byob/utility/utils.h" #include "src/util/status_macro/status_macros.h" #include "src/util/status_macro/status_util.h" @@ -141,32 +142,6 @@ absl::StatusOr CreatePivotRootDir() { return pivot_root_dir; } -absl::Status CreateDirectories(const std::filesystem::path& path) { - if (std::error_code ec; !std::filesystem::create_directories(path, ec)) { - return absl::InternalError( - absl::StrCat("Failed to create '", path.native(), "': ", ec.message())); - } - return absl::OkStatus(); -} - -absl::Status RemoveDirectories(const std::filesystem::path& path) { - if (std::error_code ec; std::filesystem::remove_all(path, ec) == - static_cast(-1)) { - return absl::InternalError(absl::StrCat( - "Failed to remove_all '", path.native(), "': ", ec.message())); - } - return absl::OkStatus(); -} - -absl::Status Mount(const char* source, const char* target, - const char* filesystemtype, int mountflags) { - if (::mount(source, target, filesystemtype, mountflags, nullptr) == -1) { - return absl::ErrnoToStatus(errno, absl::StrCat("Failed to mount '", source, - "' to '", target, "'")); - } - return absl::OkStatus(); -} - absl::Status Dup2(int oldfd, int newfd) { if (::dup2(oldfd, newfd) == -1) { return absl::ErrnoToStatus(errno, @@ -175,53 +150,6 @@ absl::Status Dup2(int oldfd, int newfd) { return absl::OkStatus(); } -absl::Status SetupPivotRoot( - const std::filesystem::path& pivot_root_dir, - absl::Span> - sources_and_targets) { - PS_RETURN_IF_ERROR(RemoveDirectories(pivot_root_dir)); - - // Set up restricted filesystem for worker using pivot_root - // pivot_root doesn't work under an MS_SHARED mount point. - // https://man7.org/linux/man-pages/man2/pivot_root.2.html. - PS_RETURN_IF_ERROR(Mount(nullptr, "/", nullptr, MS_REC | MS_PRIVATE)); - for (const auto& [source, target] : sources_and_targets) { - PS_RETURN_IF_ERROR(CreateDirectories(target)); - PS_RETURN_IF_ERROR(Mount(source.c_str(), target.c_str(), nullptr, MS_BIND)); - } - - // MS_REC needed here to get other mounts (/lib, /lib64 etc) - PS_RETURN_IF_ERROR(Mount(pivot_root_dir.c_str(), pivot_root_dir.c_str(), - "bind", MS_REC | MS_BIND)); - PS_RETURN_IF_ERROR(Mount(pivot_root_dir.c_str(), pivot_root_dir.c_str(), - "bind", MS_REC | MS_SLAVE)); - { - const std::filesystem::path pivot_dir = pivot_root_dir / "pivot"; - PS_RETURN_IF_ERROR(CreateDirectories(pivot_dir)); - if (::syscall(SYS_pivot_root, pivot_root_dir.c_str(), pivot_dir.c_str()) == - -1) { - return absl::ErrnoToStatus( - errno, - absl::StrCat("syscall(SYS_pivot_root, '", pivot_root_dir.c_str(), - "', '", pivot_dir.c_str(), "')")); - } - } - if (::chdir("/") == -1) { - return absl::ErrnoToStatus(errno, "chdir('/')"); - } - if (::umount2("/pivot", MNT_DETACH) == -1) { - return absl::ErrnoToStatus(errno, "mount2('/pivot', MNT_DETACH)"); - } - if (::rmdir("/pivot") == -1) { - return absl::ErrnoToStatus(errno, "rmdir('/pivot')"); - } - for (const auto& [source, _] : sources_and_targets) { - PS_RETURN_IF_ERROR( - Mount(source.c_str(), source.c_str(), nullptr, MS_REMOUNT | MS_BIND)); - } - return absl::OkStatus(); -} - absl::Status SetupSandbox(const WorkerImplArg& worker_impl_arg) { int log_fd = -1; if (worker_impl_arg.enable_log_egress) { @@ -243,8 +171,8 @@ absl::Status SetupSandbox(const WorkerImplArg& worker_impl_arg) { PS_RETURN_IF_ERROR(Dup2(log_fd, STDOUT_FILENO)); PS_RETURN_IF_ERROR(Dup2(log_fd, STDERR_FILENO)); } - return SetupPivotRoot(worker_impl_arg.pivot_root_dir, - worker_impl_arg.sources_and_targets); + return ::privacy_sandbox::server_common::byob::SetupPivotRoot( + worker_impl_arg.pivot_root_dir, worker_impl_arg.sources_and_targets); } constexpr uint32_t MaxIntDecimalLength() { @@ -335,7 +263,8 @@ int ReloaderImpl(void* arg) { // created by the worker. reloader_sources_and_targets.push_back( {socket_dir, *pivot_root_dir / socket_dir.relative_path()}); - CHECK_OK(SetupPivotRoot(*pivot_root_dir, reloader_sources_and_targets)); + CHECK_OK(::privacy_sandbox::server_common::byob::SetupPivotRoot( + *pivot_root_dir, reloader_sources_and_targets)); } while (true) { // Start a new worker. @@ -623,7 +552,9 @@ int main(int argc, char** argv) { return -1; } absl::Cleanup progdir_cleanup = [&prog_dir] { - if (absl::Status status = RemoveDirectories(prog_dir); !status.ok()) { + if (absl::Status status = + ::privacy_sandbox::server_common::byob::RemoveDirectories(prog_dir); + !status.ok()) { LOG(ERROR) << status; } }; diff --git a/src/roma/byob/interface/BUILD.bazel b/src/roma/byob/interface/BUILD.bazel index 9807537b..c837aeb9 100644 --- a/src/roma/byob/interface/BUILD.bazel +++ b/src/roma/byob/interface/BUILD.bazel @@ -27,6 +27,7 @@ cc_library( "//src/core/common/uuid", "//src/roma/byob/config", "//src/roma/byob/dispatcher", + "//src/roma/byob/utility:utils", "//src/util:execution_token", "//src/util/status_macro:status_macros", "@com_google_absl//absl/log", diff --git a/src/roma/byob/interface/roma_service.cc b/src/roma/byob/interface/roma_service.cc index e5849b85..7ed583d5 100644 --- a/src/roma/byob/interface/roma_service.cc +++ b/src/roma/byob/interface/roma_service.cc @@ -29,21 +29,30 @@ #include "absl/log/check.h" #include "absl/log/log.h" #include "absl/strings/str_cat.h" +#include "src/roma/byob/utility/utils.h" namespace privacy_sandbox::server_common::byob::internal::roma_service { LocalHandle::LocalHandle(int pid, std::string_view mounts, std::string_view control_socket_path, std::string_view udf_socket_path, - std::string_view log_dir) + std::string_view socket_dir, std::string_view log_dir) : pid_(pid) { // The following block does not run in the parent process. if (pid_ == 0) { // Set the process group id to the process id. PCHECK(::setpgid(/*pid=*/0, /*pgid=*/0) == 0); - const std::string run_workers_path = std::filesystem::path(CONTAINER_PATH) / - CONTAINER_ROOT_RELPATH / "server" / - "bin" / "run_workers"; + const std::string root_dir = + std::filesystem::path(CONTAINER_PATH) / CONTAINER_ROOT_RELPATH; + std::vector> + sources_and_targets = { + {log_dir, + root_dir / std::filesystem::path(log_dir).relative_path()}, + {socket_dir, + root_dir / std::filesystem::path(socket_dir).relative_path()}, + {"/dev", root_dir / std::filesystem::path("/dev").relative_path()}}; + CHECK_OK(::privacy_sandbox::server_common::byob::SetupPivotRoot( + root_dir, sources_and_targets, /*cleanup_pivot_root_dir=*/false)); const std::string mounts_flag = absl::StrCat("--mounts=", mounts); const std::string control_socket_name_flag = absl::StrCat("--control_socket_name=", control_socket_path); @@ -51,7 +60,7 @@ LocalHandle::LocalHandle(int pid, std::string_view mounts, absl::StrCat("--udf_socket_name=", udf_socket_path); const std::string log_dir_flag = absl::StrCat("--log_dir=", log_dir); const char* argv[] = { - run_workers_path.c_str(), + "/server/bin/run_workers", mounts_flag.c_str(), control_socket_name_flag.c_str(), udf_socket_name_flag.c_str(), @@ -98,14 +107,13 @@ ByobHandle::ByobHandle(int pid, std::string_view mounts, nlohmann::json::parse(std::string(std::istreambuf_iterator(ifs), std::istreambuf_iterator())); } - constexpr std::string_view log_dir_mount_point = "/tmp/udf_logs"; config["root"] = {{"path", CONTAINER_ROOT_RELPATH}}; config["process"]["args"] = { "/server/bin/run_workers", absl::StrCat("--mounts=", mounts), absl::StrCat("--control_socket_name=", control_socket_path), absl::StrCat("--udf_socket_name=", udf_socket_path), - absl::StrCat("--log_dir=", log_dir_mount_point), + absl::StrCat("--log_dir=", log_dir), }; config["process"]["rlimits"] = {}; // If a memory limit has been configured, apply it. @@ -132,7 +140,7 @@ ByobHandle::ByobHandle(int pid, std::string_view mounts, }, { {"source", log_dir}, - {"destination", log_dir_mount_point}, + {"destination", log_dir}, {"type", "bind"}, {"options", {"rbind", "rprivate"}}, }, diff --git a/src/roma/byob/interface/roma_service.h b/src/roma/byob/interface/roma_service.h index 813ffadc..97f0e980 100644 --- a/src/roma/byob/interface/roma_service.h +++ b/src/roma/byob/interface/roma_service.h @@ -55,8 +55,9 @@ namespace internal::roma_service { class LocalHandle final { public: LocalHandle(int pid, std::string_view mounts, - std::string_view control_socket_path, - std::string_view udf_socket_path, std::string_view log_dir); + std::string_view udf_socket_path, + std::string_view control_socket_path, std::string_view sockdir, + std::string_view log_dir); ~LocalHandle(); private: @@ -79,43 +80,6 @@ class ByobHandle final { } // namespace internal::roma_service -enum class Mode { - kModeSandbox, - kModeNoSandbox, - kModeSandboxDebug, -}; - -inline bool AbslParseFlag(absl::string_view text, Mode* mode, - std::string* error) { - if (text == "on") { - *mode = Mode::kModeSandbox; - return true; - } - if (text == "debug") { - *mode = Mode::kModeSandboxDebug; - return true; - } - if (text == "off") { - *mode = Mode::kModeNoSandbox; - return true; - } - *error = "Supported values: on, off, debug."; - return false; -} - -inline std::string AbslUnparseFlag(Mode mode) { - switch (mode) { - case Mode::kModeSandbox: - return "on"; - case Mode::kModeSandboxDebug: - return "debug"; - case Mode::kModeNoSandbox: - return "off"; - default: - return absl::StrCat(mode); - } -} - template class RomaService final { public: @@ -142,6 +106,7 @@ class RomaService final { std::filesystem::path udf_socket_path = socket_dir_ / "byob_rpc.sock"; const int pid = ::fork(); + ::unshare(CLONE_NEWNS); if (pid == -1) { return absl::ErrnoToStatus(errno, "fork()"); } @@ -158,7 +123,7 @@ class RomaService final { case Mode::kModeNoSandbox: handle_.emplace( pid, config.lib_mounts, control_socket_path.c_str(), - udf_socket_path.c_str(), log_dir_.c_str()); + udf_socket_path.c_str(), socket_dir_.c_str(), log_dir_.c_str()); break; default: return absl::InternalError("Unsupported mode in switch"); diff --git a/src/roma/byob/test/roma_byob_test.cc b/src/roma/byob/test/roma_byob_test.cc index 9bc00d96..0566b977 100644 --- a/src/roma/byob/test/roma_byob_test.cc +++ b/src/roma/byob/test/roma_byob_test.cc @@ -204,7 +204,7 @@ TEST(RomaByobTest, LoadBinaryInNonSandboxMode) { roma_service.Register(kUdfPath / kCPlusPlusBinaryFilename, notif, notif_status, /*num_workers=*/1); - EXPECT_TRUE(code_id.status().ok()); + EXPECT_TRUE(code_id.status().ok()) << code_id.status(); EXPECT_TRUE(notif.WaitForNotificationWithTimeout(absl::Minutes(1))); EXPECT_TRUE(notif_status.ok()); } diff --git a/src/roma/byob/utility/BUILD.bazel b/src/roma/byob/utility/BUILD.bazel index 844b30f3..96f4718c 100644 --- a/src/roma/byob/utility/BUILD.bazel +++ b/src/roma/byob/utility/BUILD.bazel @@ -19,12 +19,13 @@ cc_library( srcs = ["utils.cc"], hdrs = ["utils.h"], visibility = [ - "//src/roma/byob/benchmark:__subpackages__", - "//src/roma/byob/test:__subpackages__", + "//src/roma/byob:__subpackages__", ], deps = [ - "//src/roma/byob/interface:roma_service", + "//src/roma/byob/config", + "//src/util/status_macro:status_macros", "@com_google_absl//absl/log", + "@com_google_absl//absl/status", "@libcap", ], ) diff --git a/src/roma/byob/utility/utils.cc b/src/roma/byob/utility/utils.cc index e1c3e090..3ddd4ad0 100644 --- a/src/roma/byob/utility/utils.cc +++ b/src/roma/byob/utility/utils.cc @@ -15,11 +15,44 @@ #include "src/roma/byob/utility/utils.h" #include +#include +#include + +#include +#include #include "absl/log/log.h" -#include "src/roma/byob/interface/roma_service.h" +#include "absl/status/status.h" +#include "src/roma/byob/config/config.h" +#include "src/util/status_macro/status_macros.h" namespace privacy_sandbox::server_common::byob { +absl::Status CreateDirectories(const std::filesystem::path& path) { + std::error_code ec; + if (std::filesystem::create_directories(path, ec); ec) { + return absl::InternalError( + absl::StrCat("Failed to create '", path.native(), "': ", ec.message())); + } + return absl::OkStatus(); +} + +absl::Status RemoveDirectories(const std::filesystem::path& path) { + if (std::error_code ec; std::filesystem::remove_all(path, ec) == + static_cast(-1)) { + return absl::InternalError(absl::StrCat( + "Failed to remove_all '", path.native(), "': ", ec.message())); + } + return absl::OkStatus(); +} + +absl::Status Mount(const char* source, const char* target, + const char* filesystemtype, int mountflags) { + if (::mount(source, target, filesystemtype, mountflags, nullptr) == -1) { + return absl::ErrnoToStatus(errno, absl::StrCat("Failed to mount '", source, + "' to '", target, "'")); + } + return absl::OkStatus(); +} using ::privacy_sandbox::server_common::byob::Mode; @@ -48,4 +81,53 @@ bool HasClonePermissionsByobWorker(Mode mode) { } } +absl::Status SetupPivotRoot( + const std::filesystem::path& pivot_root_dir, + absl::Span> + sources_and_targets, + const bool cleanup_pivot_root_dir) { + if (cleanup_pivot_root_dir) { + PS_RETURN_IF_ERROR(RemoveDirectories(pivot_root_dir)); + } + // Set up restricted filesystem for worker using pivot_root + // pivot_root doesn't work under an MS_SHARED mount point. + // https://man7.org/linux/man-pages/man2/pivot_root.2.html. + PS_RETURN_IF_ERROR(Mount(nullptr, "/", nullptr, MS_REC | MS_PRIVATE)); + for (const auto& [source, target] : sources_and_targets) { + PS_RETURN_IF_ERROR(CreateDirectories(target)); + PS_RETURN_IF_ERROR(Mount(source.c_str(), target.c_str(), nullptr, MS_BIND)); + } + + // MS_REC needed here to get other mounts (/lib, /lib64 etc) + PS_RETURN_IF_ERROR(Mount(pivot_root_dir.c_str(), pivot_root_dir.c_str(), + "bind", MS_REC | MS_BIND)); + PS_RETURN_IF_ERROR(Mount(pivot_root_dir.c_str(), pivot_root_dir.c_str(), + "bind", MS_REC | MS_SLAVE)); + { + const std::filesystem::path pivot_dir = pivot_root_dir / "pivot"; + PS_RETURN_IF_ERROR(CreateDirectories(pivot_dir)); + if (::syscall(SYS_pivot_root, pivot_root_dir.c_str(), pivot_dir.c_str()) == + -1) { + return absl::ErrnoToStatus( + errno, + absl::StrCat("syscall(SYS_pivot_root, '", pivot_root_dir.c_str(), + "', '", pivot_dir.c_str(), "')")); + } + } + if (::chdir("/") == -1) { + return absl::ErrnoToStatus(errno, "chdir('/')"); + } + if (::umount2("/pivot", MNT_DETACH) == -1) { + return absl::ErrnoToStatus(errno, "mount2('/pivot', MNT_DETACH)"); + } + if (::rmdir("/pivot") == -1) { + return absl::ErrnoToStatus(errno, "rmdir('/pivot')"); + } + for (const auto& [source, _] : sources_and_targets) { + PS_RETURN_IF_ERROR( + Mount(source.c_str(), source.c_str(), nullptr, MS_REMOUNT | MS_BIND)); + } + return absl::OkStatus(); +} + } // namespace privacy_sandbox::server_common::byob diff --git a/src/roma/byob/utility/utils.h b/src/roma/byob/utility/utils.h index 336380d0..8eabec73 100644 --- a/src/roma/byob/utility/utils.h +++ b/src/roma/byob/utility/utils.h @@ -13,14 +13,29 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#include +#include -#include "src/roma/byob/interface/roma_service.h" +#include "absl/status/status.h" +#include "absl/types/span.h" +#include "src/roma/byob/config/config.h" namespace privacy_sandbox::server_common::byob { +absl::Status CreateDirectories(const std::filesystem::path& path); +absl::Status RemoveDirectories(const std::filesystem::path& path); + +absl::Status Mount(const char* source, const char* target, + const char* filesystemtype, int mountflags); + // Returns false if the calling process does not have CAP_SYS_ADMIN privileges // to create non-Sandbox mode worker. True otherwise. bool HasClonePermissionsByobWorker( ::privacy_sandbox::server_common::byob::Mode mode); +absl::Status SetupPivotRoot( + const std::filesystem::path& pivot_root_dir, + absl::Span> + sources_and_targets, + bool cleanup_pivot_root_dir = true); } // namespace privacy_sandbox::server_common::byob