feat: Remove /usr/lib dependency

Bug: N/A Change-Id: I753f9078b58d1390574529b7db6e53bbcd1c910b GitOrigin-RevId: cacf42d55175749ceb3d563a5f7bea9ba6ba2aeb
privacysandbox · Dec 17, 2024 · 5fbc6c4 · 5fbc6c4
1 parent 607faa1
commit 5fbc6c4
Show file tree

Hide file tree

Showing 10 changed files with 177 additions and 133 deletions.
diff --git a/src/roma/byob/config/config.h b/src/roma/byob/config/config.h
@@ -24,13 +24,51 @@
 
 #ifndef LIB_MOUNTS
 #if defined(__aarch64__)
-#define LIB_MOUNTS "/lib,/usr/lib"
+#define LIB_MOUNTS "/lib"
 #else
-#define LIB_MOUNTS "/lib,/lib64,/usr/lib"
+#define LIB_MOUNTS "/lib,/lib64"
 #endif
 #endif /* LIB_MOUNTS */
 
 namespace privacy_sandbox::server_common::byob {
+
+enum class Mode {
+  kModeSandbox,
+  kModeNoSandbox,
+  kModeSandboxDebug,
+};
+
+inline bool AbslParseFlag(absl::string_view text, Mode* mode,
+                          std::string* error) {
+  if (text == "on") {
+    *mode = Mode::kModeSandbox;
+    return true;
+  }
+  if (text == "debug") {
+    *mode = Mode::kModeSandboxDebug;
+    return true;
+  }
+  if (text == "off") {
+    *mode = Mode::kModeNoSandbox;
+    return true;
+  }
+  *error = "Supported values: on, off, debug.";
+  return false;
+}
+
+inline std::string AbslUnparseFlag(Mode mode) {
+  switch (mode) {
+    case Mode::kModeSandbox:
+      return "on";
+    case Mode::kModeSandboxDebug:
+      return "debug";
+    case Mode::kModeNoSandbox:
+      return "off";
+    default:
+      return absl::StrCat(mode);
+  }
+}
+
 template <typename TMetadata = google::scp::roma::DefaultMetadata>
 struct Config {
   std::uint64_t memory_limit_soft = 0;

diff --git a/src/roma/byob/container/BUILD.bazel b/src/roma/byob/container/BUILD.bazel
@@ -96,6 +96,7 @@ cc_binary(
         "//src/roma/byob/dispatcher",
         "//src/roma/byob/dispatcher:dispatcher_cc_proto",
         "//src/roma/byob/dispatcher:dispatcher_grpc",
+        "//src/roma/byob/utility:utils",
         "//src/util/status_macro:status_macros",
         "//src/util/status_macro:status_util",
         "@com_github_grpc_grpc//:grpc++",
@@ -204,9 +205,11 @@ docker rm $$CID
         outs = ["byob_runtime_container_with_dir_{}.tar".format(user.flavor)],
         cmd_bash = """
 TMPDIR="$$(mktemp --directory roma_container_tmp.XXXXXXXXXX)"
+MACHINE="$$(uname --machine)"
 mkdir --parents --mode=700 "$$TMPDIR/{roma_container_dir}/{roma_container_root_dir}"
 tar --extract --file="$(location :byob_runtime_container_{flavor}.tar)" --directory="$$TMPDIR/{roma_container_dir}/{roma_container_root_dir}"
 tar --extract --file="$(location :container_config_{flavor}.tar)" --directory="$$TMPDIR/{roma_container_dir}"
+cp "$$TMPDIR/{roma_container_dir}/{roma_container_root_dir}/usr/lib/$$MACHINE-linux-gnu"/libstdc++.so* "$$TMPDIR/{roma_container_dir}/{roma_container_root_dir}"/lib/$$MACHINE-linux-gnu/
 tar --create --gzip --owner={user}:{uid} --group={group}:{gid} --file=$@ --directory="$$TMPDIR" "{roma_container_dir}/"
 rm -rf "$$TMPDIR"
 """.format(

diff --git a/src/roma/byob/container/run_workers.cc b/src/roma/byob/container/run_workers.cc
@@ -59,6 +59,7 @@
 #include "src/roma/byob/dispatcher/dispatcher.grpc.pb.h"
 #include "src/roma/byob/dispatcher/dispatcher.h"
 #include "src/roma/byob/dispatcher/dispatcher.pb.h"
+#include "src/roma/byob/utility/utils.h"
 #include "src/util/status_macro/status_macros.h"
 #include "src/util/status_macro/status_util.h"
 
@@ -141,32 +142,6 @@ absl::StatusOr<std::filesystem::path> CreatePivotRootDir() {
   return pivot_root_dir;
 }
 
-absl::Status CreateDirectories(const std::filesystem::path& path) {
-  if (std::error_code ec; !std::filesystem::create_directories(path, ec)) {
-    return absl::InternalError(
-        absl::StrCat("Failed to create '", path.native(), "': ", ec.message()));
-  }
-  return absl::OkStatus();
-}
-
-absl::Status RemoveDirectories(const std::filesystem::path& path) {
-  if (std::error_code ec; std::filesystem::remove_all(path, ec) ==
-                          static_cast<std::uintmax_t>(-1)) {
-    return absl::InternalError(absl::StrCat(
-        "Failed to remove_all '", path.native(), "': ", ec.message()));
-  }
-  return absl::OkStatus();
-}
-
-absl::Status Mount(const char* source, const char* target,
-                   const char* filesystemtype, int mountflags) {
-  if (::mount(source, target, filesystemtype, mountflags, nullptr) == -1) {
-    return absl::ErrnoToStatus(errno, absl::StrCat("Failed to mount '", source,
-                                                   "' to '", target, "'"));
-  }
-  return absl::OkStatus();
-}
-
 absl::Status Dup2(int oldfd, int newfd) {
   if (::dup2(oldfd, newfd) == -1) {
     return absl::ErrnoToStatus(errno,
@@ -175,53 +150,6 @@ absl::Status Dup2(int oldfd, int newfd) {
   return absl::OkStatus();
 }
 
-absl::Status SetupPivotRoot(
-    const std::filesystem::path& pivot_root_dir,
-    absl::Span<const std::pair<std::filesystem::path, std::filesystem::path>>
-        sources_and_targets) {
-  PS_RETURN_IF_ERROR(RemoveDirectories(pivot_root_dir));
-
-  // Set up restricted filesystem for worker using pivot_root
-  // pivot_root doesn't work under an MS_SHARED mount point.
-  // https://man7.org/linux/man-pages/man2/pivot_root.2.html.
-  PS_RETURN_IF_ERROR(Mount(nullptr, "/", nullptr, MS_REC | MS_PRIVATE));
-  for (const auto& [source, target] : sources_and_targets) {
-    PS_RETURN_IF_ERROR(CreateDirectories(target));
-    PS_RETURN_IF_ERROR(Mount(source.c_str(), target.c_str(), nullptr, MS_BIND));
-  }
-
-  // MS_REC needed here to get other mounts (/lib, /lib64 etc)
-  PS_RETURN_IF_ERROR(Mount(pivot_root_dir.c_str(), pivot_root_dir.c_str(),
-                           "bind", MS_REC | MS_BIND));
-  PS_RETURN_IF_ERROR(Mount(pivot_root_dir.c_str(), pivot_root_dir.c_str(),
-                           "bind", MS_REC | MS_SLAVE));
-  {
-    const std::filesystem::path pivot_dir = pivot_root_dir / "pivot";
-    PS_RETURN_IF_ERROR(CreateDirectories(pivot_dir));
-    if (::syscall(SYS_pivot_root, pivot_root_dir.c_str(), pivot_dir.c_str()) ==
-        -1) {
-      return absl::ErrnoToStatus(
-          errno,
-          absl::StrCat("syscall(SYS_pivot_root, '", pivot_root_dir.c_str(),
-                       "', '", pivot_dir.c_str(), "')"));
-    }
-  }
-  if (::chdir("/") == -1) {
-    return absl::ErrnoToStatus(errno, "chdir('/')");
-  }
-  if (::umount2("/pivot", MNT_DETACH) == -1) {
-    return absl::ErrnoToStatus(errno, "mount2('/pivot', MNT_DETACH)");
-  }
-  if (::rmdir("/pivot") == -1) {
-    return absl::ErrnoToStatus(errno, "rmdir('/pivot')");
-  }
-  for (const auto& [source, _] : sources_and_targets) {
-    PS_RETURN_IF_ERROR(
-        Mount(source.c_str(), source.c_str(), nullptr, MS_REMOUNT | MS_BIND));
-  }
-  return absl::OkStatus();
-}
-
 absl::Status SetupSandbox(const WorkerImplArg& worker_impl_arg) {
   int log_fd = -1;
   if (worker_impl_arg.enable_log_egress) {
@@ -243,8 +171,8 @@ absl::Status SetupSandbox(const WorkerImplArg& worker_impl_arg) {
     PS_RETURN_IF_ERROR(Dup2(log_fd, STDOUT_FILENO));
     PS_RETURN_IF_ERROR(Dup2(log_fd, STDERR_FILENO));
   }
-  return SetupPivotRoot(worker_impl_arg.pivot_root_dir,
-                        worker_impl_arg.sources_and_targets);
+  return ::privacy_sandbox::server_common::byob::SetupPivotRoot(
+      worker_impl_arg.pivot_root_dir, worker_impl_arg.sources_and_targets);
 }
 
 constexpr uint32_t MaxIntDecimalLength() {
@@ -335,7 +263,8 @@ int ReloaderImpl(void* arg) {
     // created by the worker.
     reloader_sources_and_targets.push_back(
         {socket_dir, *pivot_root_dir / socket_dir.relative_path()});
-    CHECK_OK(SetupPivotRoot(*pivot_root_dir, reloader_sources_and_targets));
+    CHECK_OK(::privacy_sandbox::server_common::byob::SetupPivotRoot(
+        *pivot_root_dir, reloader_sources_and_targets));
   }
   while (true) {
     // Start a new worker.
@@ -623,7 +552,9 @@ int main(int argc, char** argv) {
     return -1;
   }
   absl::Cleanup progdir_cleanup = [&prog_dir] {
-    if (absl::Status status = RemoveDirectories(prog_dir); !status.ok()) {
+    if (absl::Status status =
+            ::privacy_sandbox::server_common::byob::RemoveDirectories(prog_dir);
+        !status.ok()) {
       LOG(ERROR) << status;
     }
   };

diff --git a/src/roma/byob/interface/BUILD.bazel b/src/roma/byob/interface/BUILD.bazel
@@ -27,6 +27,7 @@ cc_library(
         "//src/core/common/uuid",
         "//src/roma/byob/config",
         "//src/roma/byob/dispatcher",
+        "//src/roma/byob/utility:utils",
         "//src/util:execution_token",
         "//src/util/status_macro:status_macros",
         "@com_google_absl//absl/log",

diff --git a/src/roma/byob/interface/roma_service.cc b/src/roma/byob/interface/roma_service.cc
@@ -29,29 +29,38 @@
 #include "absl/log/check.h"
 #include "absl/log/log.h"
 #include "absl/strings/str_cat.h"
+#include "src/roma/byob/utility/utils.h"
 
 namespace privacy_sandbox::server_common::byob::internal::roma_service {
 
 LocalHandle::LocalHandle(int pid, std::string_view mounts,
                          std::string_view control_socket_path,
                          std::string_view udf_socket_path,
-                         std::string_view log_dir)
+                         std::string_view socket_dir, std::string_view log_dir)
     : pid_(pid) {
   // The following block does not run in the parent process.
   if (pid_ == 0) {
     // Set the process group id to the process id.
     PCHECK(::setpgid(/*pid=*/0, /*pgid=*/0) == 0);
-    const std::string run_workers_path = std::filesystem::path(CONTAINER_PATH) /
-                                         CONTAINER_ROOT_RELPATH / "server" /
-                                         "bin" / "run_workers";
+    const std::string root_dir =
+        std::filesystem::path(CONTAINER_PATH) / CONTAINER_ROOT_RELPATH;
+    std::vector<std::pair<std::filesystem::path, std::filesystem::path>>
+        sources_and_targets = {
+            {log_dir,
+             root_dir / std::filesystem::path(log_dir).relative_path()},
+            {socket_dir,
+             root_dir / std::filesystem::path(socket_dir).relative_path()},
+            {"/dev", root_dir / std::filesystem::path("/dev").relative_path()}};
+    CHECK_OK(::privacy_sandbox::server_common::byob::SetupPivotRoot(
+        root_dir, sources_and_targets, /*cleanup_pivot_root_dir=*/false));
     const std::string mounts_flag = absl::StrCat("--mounts=", mounts);
     const std::string control_socket_name_flag =
         absl::StrCat("--control_socket_name=", control_socket_path);
     const std::string udf_socket_name_flag =
         absl::StrCat("--udf_socket_name=", udf_socket_path);
     const std::string log_dir_flag = absl::StrCat("--log_dir=", log_dir);
     const char* argv[] = {
-        run_workers_path.c_str(),
+        "/server/bin/run_workers",
         mounts_flag.c_str(),
         control_socket_name_flag.c_str(),
         udf_socket_name_flag.c_str(),
@@ -98,14 +107,13 @@ ByobHandle::ByobHandle(int pid, std::string_view mounts,
           nlohmann::json::parse(std::string(std::istreambuf_iterator<char>(ifs),
                                             std::istreambuf_iterator<char>()));
     }
-    constexpr std::string_view log_dir_mount_point = "/tmp/udf_logs";
     config["root"] = {{"path", CONTAINER_ROOT_RELPATH}};
     config["process"]["args"] = {
         "/server/bin/run_workers",
         absl::StrCat("--mounts=", mounts),
         absl::StrCat("--control_socket_name=", control_socket_path),
         absl::StrCat("--udf_socket_name=", udf_socket_path),
-        absl::StrCat("--log_dir=", log_dir_mount_point),
+        absl::StrCat("--log_dir=", log_dir),
     };
     config["process"]["rlimits"] = {};
     // If a memory limit has been configured, apply it.
@@ -132,7 +140,7 @@ ByobHandle::ByobHandle(int pid, std::string_view mounts,
         },
         {
             {"source", log_dir},
-            {"destination", log_dir_mount_point},
+            {"destination", log_dir},
             {"type", "bind"},
             {"options", {"rbind", "rprivate"}},
         },

diff --git a/src/roma/byob/interface/roma_service.h b/src/roma/byob/interface/roma_service.h
@@ -55,8 +55,9 @@ namespace internal::roma_service {
 class LocalHandle final {
  public:
   LocalHandle(int pid, std::string_view mounts,
-              std::string_view control_socket_path,
-              std::string_view udf_socket_path, std::string_view log_dir);
+              std::string_view udf_socket_path,
+              std::string_view control_socket_path, std::string_view sockdir,
+              std::string_view log_dir);
   ~LocalHandle();
 
  private:
@@ -79,43 +80,6 @@ class ByobHandle final {
 
 }  // namespace internal::roma_service
 
-enum class Mode {
-  kModeSandbox,
-  kModeNoSandbox,
-  kModeSandboxDebug,
-};
-
-inline bool AbslParseFlag(absl::string_view text, Mode* mode,
-                          std::string* error) {
-  if (text == "on") {
-    *mode = Mode::kModeSandbox;
-    return true;
-  }
-  if (text == "debug") {
-    *mode = Mode::kModeSandboxDebug;
-    return true;
-  }
-  if (text == "off") {
-    *mode = Mode::kModeNoSandbox;
-    return true;
-  }
-  *error = "Supported values: on, off, debug.";
-  return false;
-}
-
-inline std::string AbslUnparseFlag(Mode mode) {
-  switch (mode) {
-    case Mode::kModeSandbox:
-      return "on";
-    case Mode::kModeSandboxDebug:
-      return "debug";
-    case Mode::kModeNoSandbox:
-      return "off";
-    default:
-      return absl::StrCat(mode);
-  }
-}
-
 template <typename TMetadata = google::scp::roma::DefaultMetadata>
 class RomaService final {
  public:
@@ -142,6 +106,7 @@ class RomaService final {
     std::filesystem::path udf_socket_path = socket_dir_ / "byob_rpc.sock";
 
     const int pid = ::fork();
+    ::unshare(CLONE_NEWNS);
     if (pid == -1) {
       return absl::ErrnoToStatus(errno, "fork()");
     }
@@ -158,7 +123,7 @@ class RomaService final {
       case Mode::kModeNoSandbox:
         handle_.emplace<internal::roma_service::LocalHandle>(
             pid, config.lib_mounts, control_socket_path.c_str(),
-            udf_socket_path.c_str(), log_dir_.c_str());
+            udf_socket_path.c_str(), socket_dir_.c_str(), log_dir_.c_str());
         break;
       default:
         return absl::InternalError("Unsupported mode in switch");

diff --git a/src/roma/byob/test/roma_byob_test.cc b/src/roma/byob/test/roma_byob_test.cc
@@ -204,7 +204,7 @@ TEST(RomaByobTest, LoadBinaryInNonSandboxMode) {
       roma_service.Register(kUdfPath / kCPlusPlusBinaryFilename, notif,
                             notif_status, /*num_workers=*/1);
 
-  EXPECT_TRUE(code_id.status().ok());
+  EXPECT_TRUE(code_id.status().ok()) << code_id.status();
   EXPECT_TRUE(notif.WaitForNotificationWithTimeout(absl::Minutes(1)));
   EXPECT_TRUE(notif_status.ok());
 }

diff --git a/src/roma/byob/utility/BUILD.bazel b/src/roma/byob/utility/BUILD.bazel
@@ -19,12 +19,13 @@ cc_library(
     srcs = ["utils.cc"],
     hdrs = ["utils.h"],
     visibility = [
-        "//src/roma/byob/benchmark:__subpackages__",
-        "//src/roma/byob/test:__subpackages__",
+        "//src/roma/byob:__subpackages__",
     ],
     deps = [
-        "//src/roma/byob/interface:roma_service",
+        "//src/roma/byob/config",
+        "//src/util/status_macro:status_macros",
         "@com_google_absl//absl/log",
+        "@com_google_absl//absl/status",
         "@libcap",
     ],
 )