diff --git a/eden/mononoke/commit_rewriting/megarepo/tool/cli.rs b/eden/mononoke/commit_rewriting/megarepo/tool/cli.rs index da39451afcba6..d904b3ba78f13 100644 --- a/eden/mononoke/commit_rewriting/megarepo/tool/cli.rs +++ b/eden/mononoke/commit_rewriting/megarepo/tool/cli.rs @@ -54,9 +54,6 @@ pub const MANUAL_COMMIT_SYNC: &str = "manual-commit-sync"; pub const MAPPING_VERSION_NAME: &str = "mapping-version-name"; pub const MARK_NOT_SYNCED_COMMAND: &str = "mark-not-synced"; pub const MARK_PUBLIC: &str = "mark-public"; -pub const MAX_NUM_OF_MOVES_IN_COMMIT: &str = "max-num-of-moves-in-commit"; -pub const MOVE: &str = "move"; -pub const ORIGIN_REPO: &str = "origin-repo"; pub const OVERWRITE: &str = "overwrite"; pub const PARENTS: &str = "parents"; pub const PATH_REGEX: &str = "path-regex"; @@ -160,39 +157,6 @@ fn get_commit_factory<'a>( })) } -fn add_resulting_commit_args<'a, 'b>(subcommand: App<'a, 'b>) -> App<'a, 'b> { - subcommand - .arg( - Arg::with_name(COMMIT_AUTHOR) - .help("commit author to use") - .takes_value(true) - .required(true), - ) - .arg( - Arg::with_name(COMMIT_MESSAGE) - .help("commit message to use") - .takes_value(true) - .required(true), - ) - .arg( - Arg::with_name(MARK_PUBLIC) - .help("add the resulting commit to the public phase") - .long(MARK_PUBLIC), - ) - .arg( - Arg::with_name(COMMIT_DATE_RFC3339) - .help("commit date to use (default is now)") - .long(COMMIT_DATE_RFC3339) - .takes_value(true), - ) - .arg( - Arg::with_name(COMMIT_BOOKMARK) - .help("bookmark to point to resulting commits (no sanity checks, will move existing bookmark, be careful)") - .long(COMMIT_BOOKMARK) - .takes_value(true) - ) -} - fn add_light_resulting_commit_args<'a, 'b>(subcommand: App<'a, 'b>) -> App<'a, 'b> { subcommand .arg( @@ -216,35 +180,6 @@ fn add_light_resulting_commit_args<'a, 'b>(subcommand: App<'a, 'b>) -> App<'a, ' } pub fn setup_app<'a, 'b>() -> MononokeClapApp<'a, 'b> { - let move_subcommand = SubCommand::with_name(MOVE) - .about("create a move commit, using a provided spec") - .arg( - Arg::with_name(MAX_NUM_OF_MOVES_IN_COMMIT) - .long(MAX_NUM_OF_MOVES_IN_COMMIT) - .help("how many files a single commit moves (note - that might create a stack of move commits instead of just one)") - .takes_value(true) - .required(false), - ) - .arg( - Arg::with_name(MAPPING_VERSION_NAME) - .long(MAPPING_VERSION_NAME) - .help("which mapping version to use when remapping from small to large repo") - .takes_value(true) - .required(true), - ) - .arg( - Arg::with_name(ORIGIN_REPO) - .help("use predefined mover for part of megarepo, coming from this repo") - .takes_value(true) - .required(true), - ) - .arg( - Arg::with_name(CHANGESET) - .help("a changeset hash or bookmark of move commit's parent") - .takes_value(true) - .required(true), - ); - let sync_diamond_subcommand = SubCommand::with_name(SYNC_DIAMOND_MERGE) .about("sync a diamond merge commit from a small repo into large repo") .arg( @@ -697,7 +632,6 @@ pub fn setup_app<'a, 'b>() -> MononokeClapApp<'a, 'b> { .with_advanced_args_hidden() .with_source_and_target_repos() .build() - .subcommand(add_resulting_commit_args(move_subcommand)) .subcommand(sync_diamond_subcommand) .subcommand(add_light_resulting_commit_args(pre_merge_delete_subcommand)) .subcommand(history_fixup_delete_subcommand) diff --git a/eden/mononoke/commit_rewriting/megarepo/tool/main.rs b/eden/mononoke/commit_rewriting/megarepo/tool/main.rs index 3c0f344eb08b7..009fea3c2e5cc 100644 --- a/eden/mononoke/commit_rewriting/megarepo/tool/main.rs +++ b/eden/mononoke/commit_rewriting/megarepo/tool/main.rs @@ -5,7 +5,6 @@ * GNU General Public License version 2. */ -use std::num::NonZeroU64; use std::sync::Arc; use anyhow::bail; @@ -64,11 +63,8 @@ use megarepolib::chunking::Chunker; use megarepolib::commit_sync_config_utils::diff_small_repo_commit_sync_configs; use megarepolib::common::create_and_save_bonsai; use megarepolib::common::delete_files_in_chunks; -use megarepolib::common::StackPosition; use megarepolib::history_fixup_delete::create_history_fixup_deletes; use megarepolib::history_fixup_delete::HistoryFixupDeletes; -use megarepolib::perform_move; -use megarepolib::perform_stack_move; use megarepolib::pre_merge_delete::create_pre_merge_delete; use megarepolib::pre_merge_delete::PreMergeDelete; use megarepolib::working_copy::get_working_copy_paths_by_prefixes; @@ -80,7 +76,6 @@ use mononoke_types::ChangesetId; use mononoke_types::FileChange; use mononoke_types::NonRootMPath; use mononoke_types::RepositoryId; -use movers::get_small_to_large_mover; use movers::Mover; use mutable_counters::MutableCounters; use phases::Phases; @@ -146,9 +141,6 @@ use crate::cli::LIMIT; use crate::cli::MANUAL_COMMIT_SYNC; use crate::cli::MAPPING_VERSION_NAME; use crate::cli::MARK_NOT_SYNCED_COMMAND; -use crate::cli::MAX_NUM_OF_MOVES_IN_COMMIT; -use crate::cli::MOVE; -use crate::cli::ORIGIN_REPO; use crate::cli::OVERWRITE; use crate::cli::PARENTS; use crate::cli::PATH; @@ -197,77 +189,6 @@ pub struct Repo( SqlQueryConfig, ); -async fn run_move<'a>( - ctx: &CoreContext, - matches: &MononokeMatches<'a>, - sub_m: &ArgMatches<'a>, -) -> Result<(), Error> { - let origin_repo = - RepositoryId::new(args::get_i32_opt(sub_m, ORIGIN_REPO).expect("Origin repo is missing")); - let resulting_changeset_args = cs_args_from_matches(sub_m)?; - let move_parent = sub_m.value_of(CHANGESET).unwrap().to_owned(); - - let mapping_version_name = sub_m - .value_of(MAPPING_VERSION_NAME) - .ok_or_else(|| format_err!("mapping-version-name is not specified"))?; - let mapping_version = CommitSyncConfigVersion(mapping_version_name.to_string()); - - let live_commit_sync_config = - get_live_commit_sync_config(ctx, ctx.fb, matches, matches.config_store(), origin_repo) - .await - .context("building live_commit_sync_config")?; - - let commit_sync_config = live_commit_sync_config - .get_commit_sync_config_by_version(origin_repo, &mapping_version) - .await?; - let mover = get_small_to_large_mover(&commit_sync_config, origin_repo).unwrap(); - - let max_num_of_moves_in_commit: Option = - args::get_and_parse_opt(sub_m, MAX_NUM_OF_MOVES_IN_COMMIT); - - let repo = args::not_shardmanager_compatible::open_repo::( - ctx.fb, - &ctx.logger().clone(), - matches, - ) - .await?; - - let parent_bcs_id = helpers::csid_resolve(ctx, &repo, move_parent).await?; - - if let Some(max_num_of_moves_in_commit) = max_num_of_moves_in_commit { - let changesets = perform_stack_move( - ctx, - &repo, - parent_bcs_id, - mover.as_ref(), - max_num_of_moves_in_commit, - |num: StackPosition| { - let mut args = resulting_changeset_args.clone(); - let message = args.message + &format!(" #{}", num.0); - args.message = message; - args - }, - ) - .await?; - info!( - ctx.logger(), - "created {} commits, with the last commit {:?}", - changesets.len(), - changesets.last() - ); - } else { - perform_move( - ctx, - &repo, - parent_bcs_id, - mover.as_ref(), - resulting_changeset_args, - ) - .await?; - } - Ok(()) -} - async fn run_sync_diamond_merge<'a>( ctx: &CoreContext, matches: &MononokeMatches<'a>, @@ -1402,7 +1323,6 @@ fn main(fb: FacebookInit) -> Result<()> { (MARK_NOT_SYNCED_COMMAND, Some(sub_m)) => { run_mark_not_synced(ctx, &matches, sub_m).await } - (MOVE, Some(sub_m)) => run_move(ctx, &matches, sub_m).await, (RUN_MOVER, Some(sub_m)) => run_mover(ctx, &matches, sub_m).await, (SYNC_COMMIT_AND_ANCESTORS, Some(sub_m)) => { run_sync_commit_and_ancestors(ctx, &matches, sub_m).await diff --git a/eden/mononoke/tests/integration/megarepo/test-megarepo-tool-sync-diamond-merge.t b/eden/mononoke/tests/integration/megarepo/test-megarepo-tool-sync-diamond-merge.t index dc77eb9eedf9b..f4c51b526f283 100644 --- a/eden/mononoke/tests/integration/megarepo/test-megarepo-tool-sync-diamond-merge.t +++ b/eden/mononoke/tests/integration/megarepo/test-megarepo-tool-sync-diamond-merge.t @@ -59,14 +59,16 @@ blobimport hg servers repos into Mononoke repos $ export COMMIT_DATE="1985-09-04T00:00:00.00Z" move things in small repo with merge - $ megarepo_tool move 1 with_merge_master user "with merge move" --mark-public \ - > --commit-date-rfc3339 "$COMMIT_DATE" --bookmark with_merge_move \ - > --mapping-version-name TEST_VERSION_NAME &> /dev/null + $ quiet mononoke_admin megarepo move-commit --repo-id 0 --source-repo-id 1 \ + > -B with_merge_master -a user -m "with merge move" --mark-public \ + > --commit-date-rfc3339 "$COMMIT_DATE" --set-bookmark with_merge_move \ + > --mapping-version-name TEST_VERSION_NAME move things in another small repo - $ megarepo_tool move 2 another_master user "another move" --mark-public \ - > --commit-date-rfc3339 "$COMMIT_DATE" --bookmark another_move \ - > --mapping-version-name TEST_VERSION_NAME &> /dev/null + $ quiet mononoke_admin megarepo move-commit --repo-id 0 --source-repo-id 2 \ + > -B another_master -a user -m "another move" --mark-public \ + > --commit-date-rfc3339 "$COMMIT_DATE" --set-bookmark another_move \ + > --mapping-version-name TEST_VERSION_NAME merge things in both repos $ mononoke_admin megarepo merge --repo-id 0 -B with_merge_move -B another_move -a user \ diff --git a/eden/mononoke/tests/integration/megarepo/test-megarepo-tool.t b/eden/mononoke/tests/integration/megarepo/test-megarepo-tool.t index ce1a3e676ecdd..416868ad6e69e 100644 --- a/eden/mononoke/tests/integration/megarepo/test-megarepo-tool.t +++ b/eden/mononoke/tests/integration/megarepo/test-megarepo-tool.t @@ -59,9 +59,10 @@ blobimport $ export COMMIT_DATE="1985-09-04T00:00:00.00Z" move things in fbsource - $ RUST_BACKTRACE=1 megarepo_tool move 1 fbsource_master user "fbsource move" --mark-public --commit-date-rfc3339 "$COMMIT_DATE" --bookmark fbsource_move --mapping-version-name TEST_VERSION_NAME - * using repo "repo" repoid RepositoryId(0) (glob) - * changeset resolved as: * (glob) + $ RUST_BACKTRACE=1 mononoke_admin megarepo move-commit --repo-id 0 \ + > --source-repo-id 1 -B fbsource_master -a user -m "fbsource move" \ + > --mark-public --commit-date-rfc3339 "$COMMIT_DATE" \ + > --set-bookmark fbsource_move --mapping-version-name TEST_VERSION_NAME * Marked as public * (glob) * Setting bookmark * "fbsource_move" * to point to * (glob) * Setting bookmark * "fbsource_move" * finished (glob) @@ -69,9 +70,10 @@ move things in fbsource * Hg equivalent of *: HgChangesetId(HgNodeHash(Sha1(*))) (glob) move things in ovrsource in a stack - $ megarepo_tool move 2 ovrsource_master user "ovrsource stack move" --mark-public --commit-date-rfc3339 "$COMMIT_DATE" --max-num-of-moves-in-commit 1 --bookmark ovrsource_move --mapping-version-name TEST_VERSION_NAME - * using repo "repo" repoid RepositoryId(0) (glob) - * changeset resolved as: * (glob) + $ mononoke_admin megarepo move-commit --repo-id 0 --source-repo-id 2 \ + > -B ovrsource_master -a user -m "ovrsource stack move" --mark-public \ + > --commit-date-rfc3339 "$COMMIT_DATE" --max-num-of-moves-in-commit 1 \ + > --set-bookmark ovrsource_move --mapping-version-name TEST_VERSION_NAME * Marked as public * (glob) * Setting bookmark * "ovrsource_move" * to point to * (glob) * Setting bookmark * "ovrsource_move" * finished (glob) diff --git a/eden/mononoke/tools/admin/Cargo.toml b/eden/mononoke/tools/admin/Cargo.toml index 2549a16d558e2..c1c4d85d0125a 100644 --- a/eden/mononoke/tools/admin/Cargo.toml +++ b/eden/mononoke/tools/admin/Cargo.toml @@ -70,6 +70,7 @@ gix-hash = "0.15.1" gix-object = "0.46.0" itertools = "0.14.0" justknobs = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "main" } +live_commit_sync_config = { version = "0.1.0", path = "../../commit_rewriting/live_commit_sync_config" } manifest = { version = "0.1.0", path = "../../manifest" } maplit = "1.0" megarepo_error = { version = "0.1.0", path = "../../megarepo_api/megarepo_error" } @@ -80,6 +81,7 @@ metaconfig_types = { version = "0.1.0", path = "../../metaconfig/types" } mononoke_api = { version = "0.1.0", path = "../../mononoke_api" } mononoke_app = { version = "0.1.0", path = "../../cmdlib/mononoke_app" } mononoke_types = { version = "0.1.0", path = "../../mononoke_types" } +movers = { version = "0.1.0", path = "../../commit_rewriting/movers" } mutable_counters = { version = "0.1.0", path = "../../mutable_counters" } mutable_renames = { version = "0.1.0", path = "../../mutable_renames" } packfile = { version = "0.1.0", path = "../../git/packfile" } diff --git a/eden/mononoke/tools/admin/src/commands/megarepo.rs b/eden/mononoke/tools/admin/src/commands/megarepo.rs index 01d44bc6360cd..898b0363353b5 100644 --- a/eden/mononoke/tools/admin/src/commands/megarepo.rs +++ b/eden/mononoke/tools/admin/src/commands/megarepo.rs @@ -7,6 +7,7 @@ pub(crate) mod common; mod merge; +mod move_commit; mod pushredirection; use anyhow::Result; @@ -15,6 +16,7 @@ use clap::Subcommand; use mononoke_app::MononokeApp; use self::merge::MergeArgs; +use self::move_commit::MoveArgs; use self::pushredirection::PushRedirectionArgs; /// Manage megarepo @@ -29,6 +31,7 @@ enum MegarepoSubcommand { /// Manage which repos are pushredirected to the large repo PushRedirection(PushRedirectionArgs), Merge(MergeArgs), + MoveCommit(MoveArgs), } pub async fn run(app: MononokeApp, args: CommandArgs) -> Result<()> { @@ -37,6 +40,7 @@ pub async fn run(app: MononokeApp, args: CommandArgs) -> Result<()> { match args.subcommand { MegarepoSubcommand::PushRedirection(args) => pushredirection::run(&ctx, app, args).await?, MegarepoSubcommand::Merge(args) => merge::run(&ctx, app, args).await?, + MegarepoSubcommand::MoveCommit(args) => move_commit::run(&ctx, app, args).await?, } Ok(()) diff --git a/eden/mononoke/tools/admin/src/commands/megarepo/common.rs b/eden/mononoke/tools/admin/src/commands/megarepo/common.rs index bd41d6c1313bd..0771178e3467d 100644 --- a/eden/mononoke/tools/admin/src/commands/megarepo/common.rs +++ b/eden/mononoke/tools/admin/src/commands/megarepo/common.rs @@ -5,11 +5,22 @@ * GNU General Public License version 2. */ +use std::sync::Arc; + use anyhow::Error; use anyhow::Result; +use blobstore_factory::MetadataSqlFactory; use bookmarks::BookmarkKey; +use context::CoreContext; +use live_commit_sync_config::CfgrLiveCommitSyncConfig; use megarepolib::common::ChangesetArgs as MegarepoNewChangesetArgs; +use mononoke_api::Repo; +use mononoke_app::args::AsRepoArg; +use mononoke_app::args::RepoArgs; +use mononoke_app::MononokeApp; use mononoke_types::DateTime; +use pushredirect::SqlPushRedirectionConfigBuilder; +use sql_query_config::SqlQueryConfigArc; #[derive(Debug, clap::Args, Clone)] pub(crate) struct ResultingChangesetArgs { @@ -51,3 +62,30 @@ impl TryInto for ResultingChangesetArgs { Ok(res) } } + +pub(crate) async fn get_live_commit_sync_config( + _ctx: &CoreContext, + app: MononokeApp, + repo_args: RepoArgs, +) -> Result> { + let config_store = app.environment().config_store.clone(); + let repo: Arc = app.open_repo_unredacted(&repo_args).await?; + let (_, repo_config) = app.repo_config(repo_args.as_repo_arg())?; + let sql_factory: MetadataSqlFactory = MetadataSqlFactory::new( + app.fb, + repo_config.storage_config.metadata.clone(), + app.mysql_options().clone(), + *app.readonly_storage(), + ) + .await?; + let builder = sql_factory + .open::() + .await?; + let push_redirection_config = builder.build(repo.sql_query_config_arc()); + let live_commit_sync_config = Arc::new(CfgrLiveCommitSyncConfig::new( + &config_store, + Arc::new(push_redirection_config), + )?); + + Ok(live_commit_sync_config) +} diff --git a/eden/mononoke/tools/admin/src/commands/megarepo/move_commit.rs b/eden/mononoke/tools/admin/src/commands/megarepo/move_commit.rs new file mode 100644 index 0000000000000..1610fa5983625 --- /dev/null +++ b/eden/mononoke/tools/admin/src/commands/megarepo/move_commit.rs @@ -0,0 +1,107 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This software may be used and distributed according to the terms of the + * GNU General Public License version 2. + */ + +use std::num::NonZero; + +use anyhow::Context; +use anyhow::Result; +use context::CoreContext; +use live_commit_sync_config::LiveCommitSyncConfig; +use megarepolib::common::ChangesetArgs as MegarepoNewChangesetArgs; +use megarepolib::common::StackPosition; +use megarepolib::perform_move; +use megarepolib::perform_stack_move; +use metaconfig_types::CommitSyncConfigVersion; +use mononoke_api::Repo; +use mononoke_app::args::ChangesetArgs; +use mononoke_app::args::RepoArgs; +use mononoke_app::args::SourceRepoArgs; +use mononoke_app::MononokeApp; +use movers::get_small_to_large_mover; +use repo_identity::RepoIdentityRef; +use slog::info; + +use super::common::get_live_commit_sync_config; +use super::common::ResultingChangesetArgs; + +/// Create a move commit, using a provided spec +#[derive(Debug, clap::Args)] +pub struct MoveArgs { + #[clap(flatten, help = "Repo containing the commit to be moved")] + repo_args: RepoArgs, + + #[clap( + flatten, + help = "Use predefined mover for part of megarepo, coming from this repo" + )] + source_repo_args: SourceRepoArgs, + + #[clap(flatten)] + move_parent_cs: ChangesetArgs, + + #[clap( + long, + help = "how many files a single commit moves (note - that might create a stack of move commits instead of just one)" + )] + max_num_of_moves_in_commit: Option>, + + #[clap( + long, + help = "which mapping version to use when remapping from small to large repo" + )] + mapping_version_name: String, + + #[command(flatten)] + pub res_cs_args: ResultingChangesetArgs, +} + +pub async fn run(ctx: &CoreContext, app: MononokeApp, args: MoveArgs) -> Result<()> { + let repo: Repo = app.open_repo(&args.repo_args).await?; + let source_repo: Repo = app.open_repo(&args.source_repo_args).await?; + + let move_parent_bcs_id = args.move_parent_cs.resolve_changeset(ctx, &repo).await?; + + let mapping_version = CommitSyncConfigVersion(args.mapping_version_name); + + let live_commit_sync_config = get_live_commit_sync_config(ctx, app, args.repo_args) + .await + .context("building live_commit_sync_config")?; + + let source_repo_id = source_repo.repo_identity().id(); + let commit_sync_config = live_commit_sync_config + .get_commit_sync_config_by_version(source_repo_id, &mapping_version) + .await?; + let mover = get_small_to_large_mover(&commit_sync_config, source_repo_id).unwrap(); + + let new_cs_args: MegarepoNewChangesetArgs = args.res_cs_args.try_into()?; + + if let Some(max_num_of_moves_in_commit) = args.max_num_of_moves_in_commit { + let changesets = perform_stack_move( + ctx, + &repo, + move_parent_bcs_id, + mover.as_ref(), + max_num_of_moves_in_commit, + |num: StackPosition| { + let mut args = new_cs_args.clone(); + let message = args.message + &format!(" #{}", num.0); + args.message = message; + args + }, + ) + .await?; + info!( + ctx.logger(), + "created {} commits, with the last commit {:?}", + changesets.len(), + changesets.last() + ); + } else { + perform_move(ctx, &repo, move_parent_bcs_id, mover.as_ref(), new_cs_args).await?; + } + Ok(()) +}