From d15286ce461ccf2f51eab96c73fdfdfe1e461320 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?dj8yf0=CE=BCl?= <26653921+dj8yfo@users.noreply.github.com> Date: Wed, 18 Dec 2024 13:08:13 +0200 Subject: [PATCH] fix: running `near_workspaces::compile_project` concurrently in tests (#266) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sometimes the final destination file is written with non-wasm-opt-ed content, which might be the reason of failures in concurrent context when many tests run near_workspaces::compile_project --------- Co-authored-by: dj8yf0μl Co-authored-by: Vadim Volodin --- cargo-near-build/Cargo.toml | 3 +- cargo-near-build/src/fs.rs | 41 ++++++++++++++++++- cargo-near-build/src/near/build/mod.rs | 56 ++++++++++++++++++++------ 3 files changed, 83 insertions(+), 17 deletions(-) diff --git a/cargo-near-build/Cargo.toml b/cargo-near-build/Cargo.toml index ccea2296..dc6e4bb5 100644 --- a/cargo-near-build/Cargo.toml +++ b/cargo-near-build/Cargo.toml @@ -30,7 +30,7 @@ git2 = { version = "0.19", optional = true } home = { version = "0.5.9", optional = true } pathdiff = { version = "0.2.1", features = ["camino"], optional = true } unix_path = { version = "1.0.1", optional = true } -tempfile = { version = "3.10.1", optional = true } +tempfile = { version = "3.10.1" } shell-words = { version = "1.0.0", optional = true } wasm-opt = "=0.116.1" humantime = "2.1.0" @@ -57,7 +57,6 @@ docker = [ "dep:home", "dep:pathdiff", "dep:unix_path", - "dep:tempfile", "dep:nix", "dep:shell-words", ] diff --git a/cargo-near-build/src/fs.rs b/cargo-near-build/src/fs.rs index 87a111f6..b5dc7701 100644 --- a/cargo-near-build/src/fs.rs +++ b/cargo-near-build/src/fs.rs @@ -4,8 +4,12 @@ use eyre::WrapErr; /// Copy a file to a destination. /// /// Does nothing if the destination is the same as the source to avoid truncating the file. -pub fn copy(from: &Utf8Path, to: &Utf8Path) -> eyre::Result { - let out_path = to.join(from.file_name().unwrap()); +pub fn copy(from: &Utf8Path, out_dir: &Utf8Path) -> eyre::Result { + if !out_dir.is_dir() { + return Err(eyre::eyre!("`{}` should be a directory", out_dir)); + } + let out_path = out_dir.join(from.file_name().unwrap()); + tracing::debug!("Copying file `{}` -> `{}`", from, out_path,); if from != out_path { std::fs::copy(from, &out_path) .wrap_err_with(|| format!("failed to copy `{}` to `{}`", from, out_path))?; @@ -13,6 +17,39 @@ pub fn copy(from: &Utf8Path, to: &Utf8Path) -> eyre::Result { Ok(out_path) } +/// Copy a file to a file destination. +/// +/// Does nothing if the destination is the same as the source to avoid truncating the file. +pub fn copy_to_file(from: &Utf8Path, to: &Utf8Path) -> eyre::Result { + tracing::debug!("Copying file `{}` -> `{}`", from, to,); + + if !from.is_file() { + return Err(eyre::eyre!( + "`{}` is expected to exist and be a file \ + or point to a file in case of a symlink", + from + )); + } + if from != to && to.is_file() { + let from_content = std::fs::read(from)?; + let to_content = std::fs::read(to)?; + + if from_content == to_content { + tracing::debug!( + "skipped copying file `{}` -> `{}` on identical contents", + from, + to, + ); + return Ok(to.to_path_buf()); + } + } + if from != to { + std::fs::copy(from, to) + .wrap_err_with(|| format!("failed to copy `{}` to `{}`", from, to))?; + } + Ok(to.to_path_buf()) +} + /// Create the directory if it doesn't exist, and return the absolute path to it. pub fn force_canonicalize_dir(dir: &Utf8Path) -> eyre::Result { std::fs::create_dir_all(dir) diff --git a/cargo-near-build/src/near/build/mod.rs b/cargo-near-build/src/near/build/mod.rs index 585671d4..60deb98a 100644 --- a/cargo-near-build/src/near/build/mod.rs +++ b/cargo-near-build/src/near/build/mod.rs @@ -4,6 +4,7 @@ use crate::types::near::build::buildtime_env; use camino::Utf8PathBuf; use colored::Colorize; use near_abi::BuildInfo; +use tempfile::NamedTempFile; use crate::types::near::build::output::CompilationArtifact; use crate::types::near::build::side_effects::ArtifactMessages; @@ -162,19 +163,14 @@ pub fn run(args: Opts) -> eyre::Result { color, )?; - wasm_artifact.path = crate::fs::copy(&wasm_artifact.path, &out_dir)?; - - if !args.no_wasmopt { - println!(); - pretty_print::handle_step( - "Running an optimize for size post-step with wasm-opt...", - || { - wasm_opt::OptimizationOptions::new_optimize_for_size() - .run(&wasm_artifact.path, &wasm_artifact.path)?; - Ok(()) - }, - )?; - } + wasm_artifact.path = { + let (from_path, _maybe_tmpfile) = + maybe_wasm_opt_step(&wasm_artifact.path, args.no_wasmopt)?; + crate::fs::copy_to_file( + &from_path, + &out_dir.join(wasm_artifact.path.file_name().expect("has filename")), + )? + }; wasm_artifact.builder_version_info = Some(builder_version_info); @@ -206,3 +202,37 @@ pub fn run(args: Opts) -> eyre::Result { pretty_print::duration(start, "cargo near build"); Ok(wasm_artifact) } + +fn maybe_wasm_opt_step( + input_path: &Utf8PathBuf, + no_wasmopt: bool, +) -> eyre::Result<(Utf8PathBuf, Option)> { + let result = if !no_wasmopt { + let opt_destination = tempfile::Builder::new() + .prefix("optimized-") + .suffix(".wasm") + .tempfile()?; + println!(); + pretty_print::handle_step( + "Running an optimize for size post-step with wasm-opt...", + || { + println!( + "{} -> {}", + format!("{}", input_path).cyan(), + format!("{}", opt_destination.path().to_string_lossy()).cyan() + ); + wasm_opt::OptimizationOptions::new_optimize_for_size() + .run(input_path, opt_destination.path())?; + Ok(()) + }, + )?; + + ( + Utf8PathBuf::try_from(opt_destination.path().to_path_buf())?, + Some(opt_destination), + ) + } else { + (input_path.clone(), None) + }; + Ok(result) +}