From 5133af59f4b2e435d2301d289f245a75b783d1e7 Mon Sep 17 00:00:00 2001 From: Fabricio Aguiar Date: Mon, 25 Jul 2022 19:11:06 +0100 Subject: [PATCH] Update CI files + black [noissue] --- .ci/scripts/changelog.py | 10 +- .ci/scripts/redmine.py | 4 +- .ci/scripts/update_ci_branches.py | 10 +- .ci/scripts/validate_commit_message.py | 10 +- flake8.cfg => .flake8 | 10 +- .github/template_gitref | 2 +- .github/workflows/ci.yml | 6 +- .github/workflows/scripts/docs-publisher.py | 8 +- .github/workflows/scripts/release.py | 4 +- .../stage-changelog-for-default-branch.py | 4 +- docs/conf.py | 72 +- func_test_settings.py | 8 +- lint_requirements.txt | 5 + pulp_2to3_migration/__init__.py | 2 +- pulp_2to3_migration/app/__init__.py | 8 +- pulp_2to3_migration/app/constants.py | 10 +- pulp_2to3_migration/app/json_schema.py | 4 +- pulp_2to3_migration/app/migration.py | 153 +++-- .../app/migrations/0001_initial.py | 477 +++++++++---- ...ove_pulp2distributor_pulp2_auto_publish.py | 6 +- .../0003_pulp2lazycatalog_indices.py | 14 +- .../0004_modularity_metafile_erratum_rpm.py | 278 +++++--- .../app/migrations/0005_pulp2distribution.py | 50 +- .../app/migrations/0006_pulp2_comps.py | 198 ++++-- .../0007_pulp2repository_pulp3_repository.py | 14 +- .../app/migrations/0008_pulp2srpm.py | 67 +- .../0009_pulp2erratum_allow_null.py | 54 +- .../0010_pulp2lazycatalog_is_migrated.py | 6 +- .../app/migrations/0011_lce_revision.py | 6 +- .../app/migrations/0012_size_bigint.py | 10 +- .../0013_fix_file_remote_migration.py | 34 +- .../app/migrations/0014_gzip_repodata.py | 62 +- .../migrations/0015_add_created_updated.py | 10 +- .../app/migrations/0016_type_index.py | 32 +- .../app/migrations/0017_pulp2debpackage.py | 102 +-- .../0018_pulp2distributor_pulp2_repos.py | 12 +- .../0019_remove_pulp2_repository.py | 10 +- .../app/migrations/0020_fix_defaults.py | 26 +- .../migrations/0021_pulp2content_add_subid.py | 12 +- .../0022_add_structured_deb_types.py | 163 +++-- .../0023_remove_pulp2importer_repo_field.py | 6 +- .../app/migrations/0024_reposetup.py | 54 +- .../migrations/0025_remove_p2content_dups.py | 24 +- .../app/migrations/0026_p2content_UQ.py | 24 +- .../0027_pulp2repocontent_created_index.py | 8 +- .../migrations/0028_create_missing_indices.py | 52 +- .../migrations/0029_distribution_change.py | 16 +- .../app/migrations/0030_update_json_field.py | 80 +-- .../0031_add_repoid_to_deb_types.py | 56 +- pulp_2to3_migration/app/models/base.py | 152 +++-- pulp_2to3_migration/app/models/content.py | 40 +- pulp_2to3_migration/app/models/repository.py | 62 +- pulp_2to3_migration/app/plugin/__init__.py | 13 +- pulp_2to3_migration/app/plugin/api.py | 2 +- pulp_2to3_migration/app/plugin/content.py | 209 ++++-- .../app/plugin/deb/migrator.py | 50 +- .../app/plugin/deb/pulp2_models.py | 48 +- .../app/plugin/deb/pulp_2to3_models.py | 286 ++++---- .../app/plugin/deb/repository.py | 32 +- .../app/plugin/docker/migrator.py | 110 +-- .../app/plugin/docker/pulp2_models.py | 62 +- .../app/plugin/docker/pulp_2to3_models.py | 171 +++-- .../app/plugin/docker/repository.py | 29 +- .../app/plugin/docker/utils.py | 14 +- .../app/plugin/iso/migrator.py | 17 +- .../app/plugin/iso/pulp2_models.py | 15 +- .../app/plugin/iso/pulp_2to3_models.py | 35 +- .../app/plugin/iso/repository.py | 27 +- pulp_2to3_migration/app/plugin/migrator.py | 1 + pulp_2to3_migration/app/plugin/repository.py | 69 +- .../app/plugin/rpm/comps_utils.py | 17 +- pulp_2to3_migration/app/plugin/rpm/erratum.py | 69 +- .../app/plugin/rpm/migrator.py | 117 ++-- .../app/plugin/rpm/package_utils.py | 8 +- .../app/plugin/rpm/pulp2_models.py | 271 +++++--- .../app/plugin/rpm/pulp_2to3_models.py | 642 +++++++++++------- .../app/plugin/rpm/repository.py | 69 +- pulp_2to3_migration/app/plugin/rpm/utils.py | 4 +- .../app/plugin/rpm/xml_utils.py | 47 +- pulp_2to3_migration/app/pre_migration.py | 444 +++++++----- pulp_2to3_migration/app/serializers.py | 91 +-- pulp_2to3_migration/app/settings.py | 22 +- pulp_2to3_migration/app/tasks/__init__.py | 2 +- pulp_2to3_migration/app/tasks/migrate.py | 10 +- pulp_2to3_migration/app/tasks/reset.py | 54 +- pulp_2to3_migration/app/viewsets.py | 104 +-- pulp_2to3_migration/exceptions.py | 2 + pulp_2to3_migration/pulp2/base.py | 41 +- pulp_2to3_migration/pulp2/connection.py | 181 +++-- .../tests/functional/common_plans.py | 303 +++++---- .../tests/functional/constants.py | 52 +- .../tests/functional/dynaconf_config.py | 6 +- .../tests/functional/file_base.py | 3 +- .../tests/functional/rpm_base.py | 51 +- .../tests/functional/test_file_migration.py | 132 ++-- .../tests/functional/test_file_reset.py | 130 ++-- .../functional/test_migration_behaviour.py | 330 +++++---- .../tests/functional/test_migration_plan.py | 67 +- .../functional/test_migration_plan_changes.py | 214 +++--- .../tests/functional/test_multiple_plugins.py | 31 +- .../tests/functional/test_rpm_repo.py | 143 ++-- .../tests/functional/test_rpm_rerun.py | 442 ++++++------ pulp_2to3_migration/tests/functional/util.py | 24 +- pyproject.toml | 3 +- setup.py | 6 +- 105 files changed, 4792 insertions(+), 3077 deletions(-) rename flake8.cfg => .flake8 (60%) create mode 100644 lint_requirements.txt diff --git a/.ci/scripts/changelog.py b/.ci/scripts/changelog.py index 179a0f38..1a5ed89f 100755 --- a/.ci/scripts/changelog.py +++ b/.ci/scripts/changelog.py @@ -5,8 +5,14 @@ from git import Repo repo = Repo(os.getcwd()) -heads = repo.git.ls_remote("--heads", "https://github.com/pulp/pulp-2to3-migration.git").split("\n") -branches = [h.split("/")[-1] for h in heads if re.search(r"^([0-9]+)\.([0-9]+)$", h.split("/")[-1])] +heads = repo.git.ls_remote( + "--heads", "https://github.com/pulp/pulp-2to3-migration.git" +).split("\n") +branches = [ + h.split("/")[-1] + for h in heads + if re.search(r"^([0-9]+)\.([0-9]+)$", h.split("/")[-1]) +] branches.sort(key=lambda ver: Version(ver), reverse=True) diff --git a/.ci/scripts/redmine.py b/.ci/scripts/redmine.py index 82c9b36c..9b8a2199 100755 --- a/.ci/scripts/redmine.py +++ b/.ci/scripts/redmine.py @@ -20,7 +20,9 @@ query_issues = REDMINE_QUERY_URL.split("=")[-1].split(",") milestone_name = redmine.version.get(MILESTONE_URL.split("/")[-1].split(".")[0]).name if milestone_name != RELEASE: - raise RuntimeError(f"Milestone name, '{milestone_name}', does not match version, '{RELEASE}'.") + raise RuntimeError( + f"Milestone name, '{milestone_name}', does not match version, '{RELEASE}'." + ) to_update = [] for issue in query_issues: diff --git a/.ci/scripts/update_ci_branches.py b/.ci/scripts/update_ci_branches.py index c1a1d6fe..2af7ff0c 100755 --- a/.ci/scripts/update_ci_branches.py +++ b/.ci/scripts/update_ci_branches.py @@ -11,8 +11,14 @@ initial_branch = None repo = Repo(os.getcwd()) -heads = repo.git.ls_remote("--heads", "https://github.com/pulp/pulp-2to3-migration.git").split("\n") -branches = [h.split("/")[-1] for h in heads if re.search(r"^([0-9]+)\.([0-9]+)$", h.split("/")[-1])] +heads = repo.git.ls_remote( + "--heads", "https://github.com/pulp/pulp-2to3-migration.git" +).split("\n") +branches = [ + h.split("/")[-1] + for h in heads + if re.search(r"^([0-9]+)\.([0-9]+)$", h.split("/")[-1]) +] branches.sort(key=lambda ver: Version(ver)) headers = { diff --git a/.ci/scripts/validate_commit_message.py b/.ci/scripts/validate_commit_message.py index da5b2c06..b08539fa 100755 --- a/.ci/scripts/validate_commit_message.py +++ b/.ci/scripts/validate_commit_message.py @@ -23,7 +23,9 @@ KEYWORDS = ["fixes", "closes"] sha = sys.argv[1] -message = subprocess.check_output(["git", "log", "--format=%B", "-n 1", sha]).decode("utf-8") +message = subprocess.check_output(["git", "log", "--format=%B", "-n 1", sha]).decode( + "utf-8" +) g = Github(os.environ.get("GITHUB_TOKEN")) repo = g.get_repo("pulp/pulp-2to3-migration") @@ -66,7 +68,11 @@ def __check_changelog(issue): __check_changelog(issue) else: if NO_ISSUE in message: - print("Commit {sha} has no issues but is tagged {tag}.".format(sha=sha[0:7], tag=NO_ISSUE)) + print( + "Commit {sha} has no issues but is tagged {tag}.".format( + sha=sha[0:7], tag=NO_ISSUE + ) + ) elif "Merge" in message and "cherry picked from commit" in message: pass else: diff --git a/flake8.cfg b/.flake8 similarity index 60% rename from flake8.cfg rename to .flake8 index 920a8b16..80dee302 100644 --- a/flake8.cfg +++ b/.flake8 @@ -1,11 +1,17 @@ [flake8] exclude = ./docs/*,*/migrations/*, ./pulp_2to3_migration/app/json_schema.py -ignore = W503,Q000,Q003,D100,D104,D105,D106,D200,D202,D204,D205,D400,D401,D402 +ignore = E203,W503,Q000,Q003,D100,D104,D106,D200,D205,D400,D401,D402 max-line-length = 100 +# Flake8 builtin codes +# -------------------- +# E203: no whitespace around ':'. disabled until https://github.com/PyCQA/pycodestyle/issues/373 is fixed +# W503: This enforces operators before line breaks which is not pep8 or black compatible. + # Flake8-quotes extension codes # ----------------------------- # Q000: double or single quotes only, default is double (don't want to enforce this) +# Q003: Change outer quotes to avoid escaping inner quotes # Flake8-docstring extension codes # -------------------------------- @@ -13,5 +19,7 @@ max-line-length = 100 # D104: missing docstring in public package # D106: missing docstring in public nested class (complains about "class Meta:" and documenting those is silly) # D200: one-line docstring should fit on one line with quotes +# D205: 1 blank line required between summary line and description +# D400: First line should end with a period # D401: first line should be imperative (nitpicky) # D402: first line should not be the function’s “signature” (false positives) diff --git a/.github/template_gitref b/.github/template_gitref index 1b851a07..1c38d930 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-150-gc95d3f2 +2021.08.26-152-g5ab12e7 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bb3f8594..6db563ca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,9 +42,9 @@ jobs: with: python-version: "3.8" - # dev_requirements contains tools needed for flake8, etc. + # lint_requirements contains tools needed for flake8, etc. - name: Install requirements - run: pip3 install -r dev_requirements.txt + run: pip3 install -r lint_requirements.txt - name: Check commit message if: github.event_name == 'pull_request' @@ -63,7 +63,7 @@ jobs: # Lint code. - name: Run flake8 - run: flake8 --config flake8.cfg + run: flake8 - name: Run extra lint checks run: "[ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh" diff --git a/.github/workflows/scripts/docs-publisher.py b/.github/workflows/scripts/docs-publisher.py index 47abfb2e..277a4872 100755 --- a/.github/workflows/scripts/docs-publisher.py +++ b/.github/workflows/scripts/docs-publisher.py @@ -112,7 +112,9 @@ def main(): if exit_code != 0: raise RuntimeError("An error occurred while pushing docs.") elif build_type == "tag": - if (not re.search("[a-zA-Z]", branch) or "post" in branch) and len(branch.split(".")) > 2: + if (not re.search("[a-zA-Z]", branch) or "post" in branch) and len( + branch.split(".") + ) > 2: # Only publish docs at the root if this is the latest version r = requests.get("https://pypi.org/pypi/pulp-2to3-migration/json") latest_version = version.parse(json.loads(r.text)["info"]["version"]) @@ -131,7 +133,9 @@ def main(): # publish to the root of docs.pulpproject.org if publish_at_root: version_components = branch.split(".") - x_y_version = "{}.{}".format(version_components[0], version_components[1]) + x_y_version = "{}.{}".format( + version_components[0], version_components[1] + ) remote_path_arg = "%s@%s:%s" % (USERNAME, HOSTNAME, SITE_ROOT) rsync_command = [ "rsync", diff --git a/.github/workflows/scripts/release.py b/.github/workflows/scripts/release.py index bfd1ee8e..409b99ee 100755 --- a/.github/workflows/scripts/release.py +++ b/.github/workflows/scripts/release.py @@ -161,7 +161,9 @@ def create_tag_and_build_package(repo, desired_tag, commit_sha, plugin_path): """ ) -parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter, description=helper) +parser = argparse.ArgumentParser( + formatter_class=argparse.RawTextHelpFormatter, description=helper +) parser.add_argument( "release_version", diff --git a/.github/workflows/scripts/stage-changelog-for-default-branch.py b/.github/workflows/scripts/stage-changelog-for-default-branch.py index 433c6598..c3dafbae 100755 --- a/.github/workflows/scripts/stage-changelog-for-default-branch.py +++ b/.github/workflows/scripts/stage-changelog-for-default-branch.py @@ -23,7 +23,9 @@ """ ) -parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter, description=helper) +parser = argparse.ArgumentParser( + formatter_class=argparse.RawTextHelpFormatter, description=helper +) parser.add_argument( "release_version", diff --git a/docs/conf.py b/docs/conf.py index ec345474..52ae817b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -14,7 +14,8 @@ # import os import sys -sys.path.insert(0, os.path.abspath('..')) # noqa + +sys.path.insert(0, os.path.abspath("..")) # noqa import pulp_2to3_migration @@ -26,9 +27,9 @@ # -- Project information ----------------------------------------------------- -project = 'pulp-2to3-migration' -copyright = '2021, Pulp Team' -author = 'Pulp Team' +project = "pulp-2to3-migration" +copyright = "2021, Pulp Team" +author = "Pulp Team" # The short X.Y version @@ -47,23 +48,23 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.doctest', - 'sphinx.ext.intersphinx', - 'sphinx.ext.todo', + "sphinx.ext.autodoc", + "sphinx.ext.doctest", + "sphinx.ext.intersphinx", + "sphinx.ext.todo", ] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] -source_suffix = '.rst' +source_suffix = ".rst" # The master toctree document. -master_doc = 'index' +master_doc = "index" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -75,10 +76,10 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path . -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # -- Options for HTML output ------------------------------------------------- @@ -86,7 +87,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'alabaster' +html_theme = "alabaster" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -95,15 +96,15 @@ # html_theme_options = {} # Add in the restapi.html page for the API docs -html_additional_pages = {'restapi': 'restapi.html'} +html_additional_pages = {"restapi": "restapi.html"} # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # Add in a banner -html_js_files = ['eol_banner.js'] +html_js_files = ["eol_banner.js"] # Custom sidebar templates, must be a dictionary that maps document names # to template names. @@ -119,7 +120,7 @@ # -- Options for HTMLHelp output --------------------------------------------- # Output file base name for HTML help builder. -htmlhelp_basename = 'pulp-2to3-migrationdoc' +htmlhelp_basename = "pulp-2to3-migrationdoc" # -- Options for LaTeX output ------------------------------------------------ @@ -128,15 +129,12 @@ # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). # # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. # # 'preamble': '', - # Latex figure (float) alignment # # 'figure_align': 'htbp', @@ -146,8 +144,13 @@ # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'pulp-2to3-migration.tex', 'pulp-2to3-migration Documentation', - 'Pulp Team', 'manual'), + ( + master_doc, + "pulp-2to3-migration.tex", + "pulp-2to3-migration Documentation", + "Pulp Team", + "manual", + ), ] @@ -156,8 +159,13 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - (master_doc, 'pulp-2to3-migration', 'pulp-2to3-migration Documentation', - [author], 1) + ( + master_doc, + "pulp-2to3-migration", + "pulp-2to3-migration Documentation", + [author], + 1, + ) ] @@ -167,9 +175,15 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'pulp-2to3-migration', 'pulp-2to3-migration Documentation', - author, 'pulp-2to3-migration', 'One line description of project.', - 'Miscellaneous'), + ( + master_doc, + "pulp-2to3-migration", + "pulp-2to3-migration Documentation", + author, + "pulp-2to3-migration", + "One line description of project.", + "Miscellaneous", + ), ] @@ -178,4 +192,4 @@ # -- Options for intersphinx extension --------------------------------------- # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'https://docs.python.org/': None} +intersphinx_mapping = {"https://docs.python.org/": None} diff --git a/func_test_settings.py b/func_test_settings.py index 1555380f..9c9f4eb9 100644 --- a/func_test_settings.py +++ b/func_test_settings.py @@ -1,7 +1 @@ -DATABASES = { - "default": { - "NAME": "pulp", - "USER": "pulp", - "PASSWORD": "pulp" - } -} +DATABASES = {"default": {"NAME": "pulp", "USER": "pulp", "PASSWORD": "pulp"}} diff --git a/lint_requirements.txt b/lint_requirements.txt new file mode 100644 index 00000000..1e7231a5 --- /dev/null +++ b/lint_requirements.txt @@ -0,0 +1,5 @@ +# python packages handy for developers, but not required by pulp +black +check-manifest +flake8 +flake8-black diff --git a/pulp_2to3_migration/__init__.py b/pulp_2to3_migration/__init__.py index 5d4c1569..06815cd2 100644 --- a/pulp_2to3_migration/__init__.py +++ b/pulp_2to3_migration/__init__.py @@ -1 +1 @@ -default_app_config = 'pulp_2to3_migration.app.Pulp2To3MigrationPluginAppConfig' +default_app_config = "pulp_2to3_migration.app.Pulp2To3MigrationPluginAppConfig" diff --git a/pulp_2to3_migration/app/__init__.py b/pulp_2to3_migration/app/__init__.py index d0844b4f..d17986e0 100644 --- a/pulp_2to3_migration/app/__init__.py +++ b/pulp_2to3_migration/app/__init__.py @@ -4,7 +4,7 @@ class Pulp2To3MigrationPluginAppConfig(PulpPluginAppConfig): """Entry point for the pulp_2to3_migration plugin.""" - name = 'pulp_2to3_migration.app' - label = 'pulp_2to3_migration' - version = '0.17.0.dev' - python_package_name = 'pulp-2to3-migration' + name = "pulp_2to3_migration.app" + label = "pulp_2to3_migration" + version = "0.17.0.dev" + python_package_name = "pulp-2to3-migration" diff --git a/pulp_2to3_migration/app/constants.py b/pulp_2to3_migration/app/constants.py index 48aed7ad..51750073 100644 --- a/pulp_2to3_migration/app/constants.py +++ b/pulp_2to3_migration/app/constants.py @@ -1,12 +1,12 @@ # for tasking system to ensure only one migration is run at a time -PULP_2TO3_MIGRATION_RESOURCE = 'pulp_2to3_migration' +PULP_2TO3_MIGRATION_RESOURCE = "pulp_2to3_migration" PULP_2TO3_POLICIES = { - 'immediate': 'immediate', - 'on_demand': 'on_demand', - 'background': 'on_demand', + "immediate": "immediate", + "on_demand": "on_demand", + "background": "on_demand", } -NOT_USED = 'Not Used' +NOT_USED = "Not Used" DEFAULT_BATCH_SIZE = 1000 diff --git a/pulp_2to3_migration/app/json_schema.py b/pulp_2to3_migration/app/json_schema.py index 4587ea25..ee675129 100644 --- a/pulp_2to3_migration/app/json_schema.py +++ b/pulp_2to3_migration/app/json_schema.py @@ -1,4 +1,4 @@ -SCHEMA = '''{ +SCHEMA = """{ "$schema": "http://json-schema.org/draft-07/schema#", "title": "MigrationPlan", "type": "object", @@ -57,4 +57,4 @@ }, "required": ["plugins"], "additionalProperties": false -}''' +}""" diff --git a/pulp_2to3_migration/app/migration.py b/pulp_2to3_migration/app/migration.py index 0c5384d3..0dac20f7 100644 --- a/pulp_2to3_migration/app/migration.py +++ b/pulp_2to3_migration/app/migration.py @@ -38,15 +38,16 @@ def migrate_content(plan, skip_corrupted=False): no task failure. """ - progress_data = dict(message='Migrating content to Pulp 3', code='migrating.content', total=0) + progress_data = dict( + message="Migrating content to Pulp 3", code="migrating.content", total=0 + ) with ProgressReport(**progress_data) as pb: # schedule content migration into Pulp 3 using pre-migrated Pulp 2 content for plugin in plan.get_plugin_plans(): # only used for progress bar counters content_types = plugin.migrator.content_models.keys() num_to_migrate = Pulp2Content.objects.filter( - pulp2_content_type_id__in=content_types, - pulp3_content=None + pulp2_content_type_id__in=content_types, pulp3_content=None ).count() pb.total += num_to_migrate @@ -65,15 +66,13 @@ def migrate_repositories(plan): """ progress_data = dict( - message='Creating repositories in Pulp 3', code='creating.repositories', total=0 + message="Creating repositories in Pulp 3", code="creating.repositories", total=0 ) with ProgressReport(**progress_data) as pb: for plugin in plan.get_plugin_plans(): # all pulp2 repos in current plan were already migrated, no need to proceed not_migrated_repos = Pulp2Repository.objects.filter( - is_migrated=False, - not_in_plan=False, - pulp2_repo_type=plugin.type + is_migrated=False, not_in_plan=False, pulp2_repo_type=plugin.type ) if not not_migrated_repos.exists(): continue @@ -96,13 +95,15 @@ def migrate_repositories(plan): description = pulp2repo.pulp2_description repository_class = plugin.migrator.pulp3_repository repo, created = repository_class.objects.get_or_create( - name=pulp3_repo_name, - defaults={'description': description}) + name=pulp3_repo_name, defaults={"description": description} + ) pulp2_repo_ids = [] - repo_version_setup = repos_to_create[pulp3_repo_name].get('repository_versions') + repo_version_setup = repos_to_create[pulp3_repo_name].get( + "repository_versions" + ) for repo_version in repo_version_setup: - pulp2_repo_ids.append(repo_version['repo_id']) + pulp2_repo_ids.append(repo_version["repo_id"]) pulp2repos_qs = Pulp2Repository.objects.filter( pulp2_repo_id__in=pulp2_repo_ids, pulp3_repository__isnull=True ) @@ -129,11 +130,12 @@ def migrate_importers(plan): importer_migrators.update(**plugin.migrator.importer_migrators) progress_data = dict( - message='Migrating importers to Pulp 3', code='migrating.importers', total=0 + message="Migrating importers to Pulp 3", code="migrating.importers", total=0 ) with ProgressReport(**progress_data) as pb: pulp2importers_qs = Pulp2Importer.objects.filter( - is_migrated=False, not_in_plan=False) + is_migrated=False, not_in_plan=False + ) pb.total += pulp2importers_qs.count() pb.save() @@ -161,11 +163,12 @@ def complex_repo_migration(plugin_type, pulp3_repo_setup, repo_name): repo_name: Name of the repo to be migrated """ from pulp_2to3_migration.app.plugin import PLUGIN_MIGRATORS + migrator = PLUGIN_MIGRATORS.get(plugin_type) distributor_migrators = migrator.distributor_migrators distributor_types = list(distributor_migrators.keys()) - repo_versions_setup = pulp3_repo_setup[repo_name]['repository_versions'] + repo_versions_setup = pulp3_repo_setup[repo_name]["repository_versions"] signing_service = None signing_service_name = pulp3_repo_setup[repo_name].get("signing_service") @@ -183,12 +186,13 @@ def complex_repo_migration(plugin_type, pulp3_repo_setup, repo_name): # importer might not be migrated, e.g. config is empty or it's not specified in a MP pulp3_remote = None - pulp2_importer_repo_id = pulp3_repo_setup[repo_name].get('pulp2_importer_repository_id') + pulp2_importer_repo_id = pulp3_repo_setup[repo_name].get( + "pulp2_importer_repository_id" + ) if pulp2_importer_repo_id: try: pulp2_importer = Pulp2Importer.objects.get( - pulp2_repo_id=pulp2_importer_repo_id, - not_in_plan=False + pulp2_repo_id=pulp2_importer_repo_id, not_in_plan=False ) pulp3_remote = pulp2_importer.pulp3_remote except Pulp2Importer.DoesNotExist: @@ -196,18 +200,13 @@ def complex_repo_migration(plugin_type, pulp3_repo_setup, repo_name): task_group = TaskGroup.current() # find appropriate group_progress_reports that later will be updated - progress_dist = task_group.group_progress_reports.filter( - code='create.distribution' - ) - progress_rv = task_group.group_progress_reports.filter( - code='create.repo_version' - ) + progress_dist = task_group.group_progress_reports.filter(code="create.distribution") + progress_rv = task_group.group_progress_reports.filter(code="create.repo_version") for pulp2_repo_info in repo_versions_setup: try: pulp2_repo = Pulp2Repository.objects.get( - pulp2_repo_id=pulp2_repo_info['repo_id'], - not_in_plan=False + pulp2_repo_id=pulp2_repo_info["repo_id"], not_in_plan=False ) except Pulp2Repository.DoesNotExist: # not in Pulp 2 anymore @@ -220,13 +219,13 @@ def complex_repo_migration(plugin_type, pulp3_repo_setup, repo_name): for pulp2_repo_info in repo_versions_setup: # find pulp2repo by id - repo_id = pulp2_repo_info['repo_id'] - dist_repositories = pulp2_repo_info['dist_repo_ids'] + repo_id = pulp2_repo_info["repo_id"] + dist_repositories = pulp2_repo_info["dist_repo_ids"] try: - migrated_repo = Pulp2Repository.objects.get(pulp2_repo_id=repo_id, - not_in_plan=False, - is_migrated=True) + migrated_repo = Pulp2Repository.objects.get( + pulp2_repo_id=repo_id, not_in_plan=False, is_migrated=True + ) except Pulp2Repository.DoesNotExist: # not in Pulp 2 anymore continue @@ -240,13 +239,16 @@ def complex_repo_migration(plugin_type, pulp3_repo_setup, repo_name): # decrease the number of total because some dists have already been migrated decrease_total = len(dist_repositories) - len(pulp2dist) if decrease_total: - progress_dist.update(total=F('total') - decrease_total) + progress_dist.update(total=F("total") - decrease_total) for dist in pulp2dist: dist_migrator = distributor_migrators.get(dist.pulp2_type_id) migrate_repo_distributor( - dist_migrator, progress_dist, dist, - migrated_repo.pulp3_repository_version, signing_service + dist_migrator, + progress_dist, + dist, + migrated_repo.pulp3_repository_version, + signing_service, ) # add distirbutors specified in the complex plan # these can be native and not native distributors @@ -269,19 +271,25 @@ def create_repoversions_publications_distributions(plan, parallel=True): for plugin in plan.get_plugin_plans(): # verify whether all pulp2 repos and distributors have been migrated not_migrated_repos = Pulp2Repository.objects.filter( - is_migrated=False, - not_in_plan=False, - pulp2_repo_type=plugin.type) + is_migrated=False, not_in_plan=False, pulp2_repo_type=plugin.type + ) not_migrated_dists = Pulp2Distributor.objects.filter( is_migrated=False, not_in_plan=False, - pulp2_type_id__in=plugin.migrator.distributor_migrators.keys()) + pulp2_type_id__in=plugin.migrator.distributor_migrators.keys(), + ) # no need to proceed - everything is migrated if not not_migrated_repos and not not_migrated_dists: continue - not_migrated_repo_ids = not_migrated_repos.values_list('pulp2_repo_id', flat=True) - not_migrated_repo_ids_dists = not_migrated_dists.values_list('pulp2_repo_id', flat=True) - repos_ids_to_check = set(not_migrated_repo_ids).union(not_migrated_repo_ids_dists) + not_migrated_repo_ids = not_migrated_repos.values_list( + "pulp2_repo_id", flat=True + ) + not_migrated_repo_ids_dists = not_migrated_dists.values_list( + "pulp2_repo_id", flat=True + ) + repos_ids_to_check = set(not_migrated_repo_ids).union( + not_migrated_repo_ids_dists + ) pulp3_repo_setup = plugin.get_repo_creation_setup() @@ -290,14 +298,14 @@ def create_repoversions_publications_distributions(plan, parallel=True): if parallel: for repo_name in pulp3_repo_setup: - repo_versions = pulp3_repo_setup[repo_name]['repository_versions'] + repo_versions = pulp3_repo_setup[repo_name]["repository_versions"] needs_a_task = False for repo_ver in repo_versions: - repos = set(repo_ver['dist_repo_ids'] + [repo_ver['repo_id']]) + repos = set(repo_ver["dist_repo_ids"] + [repo_ver["repo_id"]]) # check whether any resources are not migrated and need a task if repos.intersection(repos_ids_to_check): needs_a_task = True - dist_to_create += len(repo_ver['dist_repo_ids']) + dist_to_create += len(repo_ver["dist_repo_ids"]) if needs_a_task: repo_ver_to_create += len(repo_versions) repo = Repository.objects.get(name=repo_name).cast() @@ -306,15 +314,15 @@ def create_repoversions_publications_distributions(plan, parallel=True): complex_repo_migration, exclusive_resources=[repo], args=task_args, - task_group=TaskGroup.current() + task_group=TaskGroup.current(), ) else: # Serial (non-parallel) for repo_name in pulp3_repo_setup: - repo_versions = pulp3_repo_setup[repo_name]['repository_versions'] + repo_versions = pulp3_repo_setup[repo_name]["repository_versions"] needs_a_task = False for repo_ver in repo_versions: - repos = set(repo_ver['dist_repo_ids'] + [repo_ver['repo_id']]) + repos = set(repo_ver["dist_repo_ids"] + [repo_ver["repo_id"]]) # check whether any resources are not migrated and need a task if repos.intersection(repos_ids_to_check): needs_a_task = True @@ -323,12 +331,14 @@ def create_repoversions_publications_distributions(plan, parallel=True): complex_repo_migration(*task_args) task_group = TaskGroup.current() - progress_rv = task_group.group_progress_reports.filter(code='create.repo_version') - progress_rv.update(total=F('total') + repo_ver_to_create) + progress_rv = task_group.group_progress_reports.filter( + code="create.repo_version" + ) + progress_rv.update(total=F("total") + repo_ver_to_create) progress_dist = task_group.group_progress_reports.filter( - code='create.distribution' + code="create.distribution" ) - progress_dist.update(total=F('total') + dist_to_create) + progress_dist.update(total=F("total") + dist_to_create) def create_repo_version(progress_rv, pulp2_repo, pulp3_remote=None): @@ -343,6 +353,7 @@ def create_repo_version(progress_rv, pulp2_repo, pulp3_remote=None): pulp2_repo(Pulp2Repository): a pre-migrated repository to create a repo version for pulp3_remote(remote): a pulp3 remote """ + def detect_path_overlap(paths): """ Check for valid POSIX paths (ie ones that aren't duplicated and don't overlap). @@ -395,9 +406,9 @@ def resolve_path_overlap(version): overlap resolution """ - paths = ContentArtifact.objects.filter(content__pk__in=version.content).values_list( - "relative_path", flat=True - ) + paths = ContentArtifact.objects.filter( + content__pk__in=version.content + ).values_list("relative_path", flat=True) paths = list(paths) max_conflicts = version.content.count() - 1 @@ -412,7 +423,7 @@ def resolve_path_overlap(version): # Content Artifacts with conflicting relative paths ordered by pulp2 creation time cas_with_conflicts = ContentArtifact.objects.filter( content__pk__in=version.content, relative_path=bad_path - ).order_by('-content__pulp2content__pulp2_last_updated') + ).order_by("-content__pulp2content__pulp2_last_updated") conflict_count = cas_with_conflicts.count() if conflict_count > 1: @@ -424,9 +435,11 @@ def resolve_path_overlap(version): removed_count = conflict_count - 1 _logger.info( _( - 'Duplicated paths have been found in Pulp 3 repo `{repo}`: {path}. ' - 'Removed: {num}.' - ).format(repo=version.repository.name, path=bad_path, num=removed_count) + "Duplicated paths have been found in Pulp 3 repo `{repo}`: {path}. " + "Removed: {num}." + ).format( + repo=version.repository.name, path=bad_path, num=removed_count + ) ) for j in range(removed_count): paths.remove(bad_path) @@ -438,8 +451,8 @@ def resolve_path_overlap(version): ) _logger.info( _( - 'Overlapping paths have been found in Pulp 3 repo `{repo}`: Removed ' - 'content with {path} path.' + "Overlapping paths have been found in Pulp 3 repo `{repo}`: Removed " + "content with {path} path." ).format(repo=version.repository.name, path=bad_path) ) # exclude the resolved path from further search @@ -454,19 +467,22 @@ def resolve_path_overlap(version): pulp3_repo.save() if pulp2_repo.is_migrated: - progress_rv.update(total=F('total') - 1) + progress_rv.update(total=F("total") - 1) return unit_ids = Pulp2RepoContent.objects.filter(pulp2_repository=pulp2_repo).values_list( - 'pulp2_unit_id', flat=True) + "pulp2_unit_id", flat=True + ) incoming_content = set( Pulp2Content.objects.filter( Q(pulp2_id__in=unit_ids) & (Q(pulp2_repo=None) | Q(pulp2_repo=pulp2_repo)), - ).only('pulp3_content').values_list('pulp3_content__pk', flat=True) + ) + .only("pulp3_content") + .values_list("pulp3_content__pk", flat=True) ) with pulp3_repo.new_version() as new_version: - repo_content = set(new_version.content.values_list('pk', flat=True)) + repo_content = set(new_version.content.values_list("pk", flat=True)) to_add = incoming_content - repo_content to_delete = repo_content - incoming_content new_version.add_content(Content.objects.filter(pk__in=to_add)) @@ -476,12 +492,12 @@ def resolve_path_overlap(version): is_empty_repo = not pulp2_repo.pulp3_repository_version if new_version.complete: pulp2_repo.pulp3_repository_version = new_version - progress_rv.update(done=F('done') + 1) + progress_rv.update(done=F("done") + 1) elif is_empty_repo: pulp2_repo.pulp3_repository_version = pulp3_repo.latest_version() - progress_rv.update(done=F('done') + 1) + progress_rv.update(done=F("done") + 1) else: - progress_rv.update(total=F('total') - 1) + progress_rv.update(total=F("total") - 1) pulp2_repo.is_migrated = True pulp2_repo.save() @@ -500,13 +516,14 @@ def migrate_repo_distributor( """ publication, distribution, created = dist_migrator.migrate_to_pulp3( - pulp2dist, repo_version, signing_service) + pulp2dist, repo_version, signing_service + ) if publication: pulp2dist.pulp3_publication = publication pulp2dist.pulp3_distribution = distribution pulp2dist.is_migrated = True pulp2dist.save() - progress_dist.update(done=F('done') + 1) + progress_dist.update(done=F("done") + 1) # CreatedResource were added here because publications and repo versions # were listed among created resources and distributions were not. it could # create some confusion remotes are not listed still diff --git a/pulp_2to3_migration/app/migrations/0001_initial.py b/pulp_2to3_migration/app/migrations/0001_initial.py index 4af672a5..8de3f8a0 100644 --- a/pulp_2to3_migration/app/migrations/0001_initial.py +++ b/pulp_2to3_migration/app/migrations/0001_initial.py @@ -12,214 +12,421 @@ class Migration(migrations.Migration): initial = True dependencies = [ - ('core', '0018_auto_20191127_2350'), + ("core", "0018_auto_20191127_2350"), ] operations = [ migrations.CreateModel( - name='MigrationPlan', + name="MigrationPlan", fields=[ - ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('pulp_created', models.DateTimeField(auto_now_add=True)), - ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)), - ('plan', django.contrib.postgres.fields.jsonb.JSONField()), + ( + "pulp_id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("pulp_created", models.DateTimeField(auto_now_add=True)), + ("pulp_last_updated", models.DateTimeField(auto_now=True, null=True)), + ("plan", django.contrib.postgres.fields.jsonb.JSONField()), ], options={ - 'abstract': False, + "abstract": False, }, ), migrations.CreateModel( - name='Pulp2Content', + name="Pulp2Content", fields=[ - ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('pulp_created', models.DateTimeField(auto_now_add=True)), - ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)), - ('pulp2_id', models.CharField(max_length=255)), - ('pulp2_content_type_id', models.CharField(max_length=255)), - ('pulp2_last_updated', models.PositiveIntegerField()), - ('pulp2_storage_path', models.TextField(null=True)), - ('downloaded', models.BooleanField(default=False)), - ('pulp3_content', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='core.Content')), + ( + "pulp_id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("pulp_created", models.DateTimeField(auto_now_add=True)), + ("pulp_last_updated", models.DateTimeField(auto_now=True, null=True)), + ("pulp2_id", models.CharField(max_length=255)), + ("pulp2_content_type_id", models.CharField(max_length=255)), + ("pulp2_last_updated", models.PositiveIntegerField()), + ("pulp2_storage_path", models.TextField(null=True)), + ("downloaded", models.BooleanField(default=False)), + ( + "pulp3_content", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to="core.Content", + ), + ), ], options={ - 'unique_together': {('pulp2_id', 'pulp2_content_type_id')}, + "unique_together": {("pulp2_id", "pulp2_content_type_id")}, }, ), migrations.CreateModel( - name='Pulp2Repository', + name="Pulp2Repository", fields=[ - ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('pulp_created', models.DateTimeField(auto_now_add=True)), - ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)), - ('pulp2_object_id', models.CharField(max_length=255, unique=True)), - ('pulp2_repo_id', models.TextField()), - ('pulp2_description', models.TextField(null=True)), - ('pulp2_last_unit_added', models.DateTimeField(null=True)), - ('pulp2_last_unit_removed', models.DateTimeField(null=True)), - ('is_migrated', models.BooleanField(default=False)), - ('not_in_plan', models.BooleanField(default=False)), - ('pulp2_repo_type', models.CharField(max_length=25)), - ('pulp3_repository_remote', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='core.Remote')), - ('pulp3_repository_version', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='core.RepositoryVersion')), + ( + "pulp_id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("pulp_created", models.DateTimeField(auto_now_add=True)), + ("pulp_last_updated", models.DateTimeField(auto_now=True, null=True)), + ("pulp2_object_id", models.CharField(max_length=255, unique=True)), + ("pulp2_repo_id", models.TextField()), + ("pulp2_description", models.TextField(null=True)), + ("pulp2_last_unit_added", models.DateTimeField(null=True)), + ("pulp2_last_unit_removed", models.DateTimeField(null=True)), + ("is_migrated", models.BooleanField(default=False)), + ("not_in_plan", models.BooleanField(default=False)), + ("pulp2_repo_type", models.CharField(max_length=25)), + ( + "pulp3_repository_remote", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to="core.Remote", + ), + ), + ( + "pulp3_repository_version", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to="core.RepositoryVersion", + ), + ), ], options={ - 'abstract': False, + "abstract": False, }, ), migrations.CreateModel( - name='Pulp2LazyCatalog', + name="Pulp2LazyCatalog", fields=[ - ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('pulp_created', models.DateTimeField(auto_now_add=True)), - ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)), - ('pulp2_importer_id', models.CharField(max_length=255)), - ('pulp2_unit_id', models.CharField(max_length=255)), - ('pulp2_content_type_id', models.CharField(max_length=255)), - ('pulp2_storage_path', models.TextField()), - ('pulp2_url', models.TextField()), - ('pulp2_revision', models.IntegerField(default=0)), + ( + "pulp_id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("pulp_created", models.DateTimeField(auto_now_add=True)), + ("pulp_last_updated", models.DateTimeField(auto_now=True, null=True)), + ("pulp2_importer_id", models.CharField(max_length=255)), + ("pulp2_unit_id", models.CharField(max_length=255)), + ("pulp2_content_type_id", models.CharField(max_length=255)), + ("pulp2_storage_path", models.TextField()), + ("pulp2_url", models.TextField()), + ("pulp2_revision", models.IntegerField(default=0)), ], options={ - 'unique_together': {('pulp2_storage_path', 'pulp2_importer_id', 'pulp2_revision')}, + "unique_together": { + ("pulp2_storage_path", "pulp2_importer_id", "pulp2_revision") + }, }, ), migrations.CreateModel( - name='Pulp2Importer', + name="Pulp2Importer", fields=[ - ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('pulp_created', models.DateTimeField(auto_now_add=True)), - ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)), - ('pulp2_object_id', models.CharField(max_length=255, unique=True)), - ('pulp2_type_id', models.CharField(max_length=255)), - ('pulp2_config', django.contrib.postgres.fields.jsonb.JSONField()), - ('pulp2_last_updated', models.DateTimeField()), - ('pulp2_repo_id', models.TextField()), - ('is_migrated', models.BooleanField(default=False)), - ('not_in_plan', models.BooleanField(default=False)), - ('pulp2_repository', models.OneToOneField(null=True, on_delete=django.db.models.deletion.CASCADE, to='pulp_2to3_migration.Pulp2Repository')), - ('pulp3_remote', models.OneToOneField(null=True, on_delete=django.db.models.deletion.SET_NULL, to='core.Remote')), + ( + "pulp_id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("pulp_created", models.DateTimeField(auto_now_add=True)), + ("pulp_last_updated", models.DateTimeField(auto_now=True, null=True)), + ("pulp2_object_id", models.CharField(max_length=255, unique=True)), + ("pulp2_type_id", models.CharField(max_length=255)), + ("pulp2_config", django.contrib.postgres.fields.jsonb.JSONField()), + ("pulp2_last_updated", models.DateTimeField()), + ("pulp2_repo_id", models.TextField()), + ("is_migrated", models.BooleanField(default=False)), + ("not_in_plan", models.BooleanField(default=False)), + ( + "pulp2_repository", + models.OneToOneField( + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="pulp_2to3_migration.Pulp2Repository", + ), + ), + ( + "pulp3_remote", + models.OneToOneField( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to="core.Remote", + ), + ), ], options={ - 'abstract': False, + "abstract": False, }, ), migrations.CreateModel( - name='Pulp2Tag', + name="Pulp2Tag", fields=[ - ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('pulp_created', models.DateTimeField(auto_now_add=True)), - ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)), - ('name', models.TextField()), - ('tagged_manifest', models.CharField(max_length=255)), - ('repo_id', models.TextField()), - ('pulp2content', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='docker_tag_detail_model', to='pulp_2to3_migration.Pulp2Content')), + ( + "pulp_id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("pulp_created", models.DateTimeField(auto_now_add=True)), + ("pulp_last_updated", models.DateTimeField(auto_now=True, null=True)), + ("name", models.TextField()), + ("tagged_manifest", models.CharField(max_length=255)), + ("repo_id", models.TextField()), + ( + "pulp2content", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="docker_tag_detail_model", + to="pulp_2to3_migration.Pulp2Content", + ), + ), ], options={ - 'default_related_name': 'docker_tag_detail_model', - 'unique_together': {('name', 'tagged_manifest', 'repo_id', 'pulp2content')}, + "default_related_name": "docker_tag_detail_model", + "unique_together": { + ("name", "tagged_manifest", "repo_id", "pulp2content") + }, }, ), migrations.CreateModel( - name='Pulp2RepoContent', + name="Pulp2RepoContent", fields=[ - ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('pulp_created', models.DateTimeField(auto_now_add=True)), - ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)), - ('pulp2_unit_id', models.CharField(max_length=255)), - ('pulp2_content_type_id', models.CharField(max_length=255)), - ('pulp2_repository', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='pulp_2to3_migration.Pulp2Repository')), + ( + "pulp_id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("pulp_created", models.DateTimeField(auto_now_add=True)), + ("pulp_last_updated", models.DateTimeField(auto_now=True, null=True)), + ("pulp2_unit_id", models.CharField(max_length=255)), + ("pulp2_content_type_id", models.CharField(max_length=255)), + ( + "pulp2_repository", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="pulp_2to3_migration.Pulp2Repository", + ), + ), ], options={ - 'unique_together': {('pulp2_repository', 'pulp2_unit_id')}, + "unique_together": {("pulp2_repository", "pulp2_unit_id")}, }, ), migrations.CreateModel( - name='Pulp2ManifestList', + name="Pulp2ManifestList", fields=[ - ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('pulp_created', models.DateTimeField(auto_now_add=True)), - ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)), - ('digest', models.CharField(max_length=255)), - ('schema_version', models.IntegerField()), - ('media_type', models.CharField(max_length=80)), - ('listed_manifests', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=255), size=None)), - ('pulp2content', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='docker_manifest_list_detail_model', to='pulp_2to3_migration.Pulp2Content')), + ( + "pulp_id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("pulp_created", models.DateTimeField(auto_now_add=True)), + ("pulp_last_updated", models.DateTimeField(auto_now=True, null=True)), + ("digest", models.CharField(max_length=255)), + ("schema_version", models.IntegerField()), + ("media_type", models.CharField(max_length=80)), + ( + "listed_manifests", + django.contrib.postgres.fields.ArrayField( + base_field=models.CharField(max_length=255), size=None + ), + ), + ( + "pulp2content", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="docker_manifest_list_detail_model", + to="pulp_2to3_migration.Pulp2Content", + ), + ), ], options={ - 'default_related_name': 'docker_manifest_list_detail_model', - 'unique_together': {('digest', 'pulp2content')}, + "default_related_name": "docker_manifest_list_detail_model", + "unique_together": {("digest", "pulp2content")}, }, ), migrations.CreateModel( - name='Pulp2Manifest', + name="Pulp2Manifest", fields=[ - ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('pulp_created', models.DateTimeField(auto_now_add=True)), - ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)), - ('digest', models.CharField(max_length=255)), - ('schema_version', models.IntegerField()), - ('media_type', models.CharField(max_length=80)), - ('blobs', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=255), size=None)), - ('config_blob', models.CharField(max_length=255, null=True)), - ('pulp2content', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='docker_manifest_detail_model', to='pulp_2to3_migration.Pulp2Content')), + ( + "pulp_id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("pulp_created", models.DateTimeField(auto_now_add=True)), + ("pulp_last_updated", models.DateTimeField(auto_now=True, null=True)), + ("digest", models.CharField(max_length=255)), + ("schema_version", models.IntegerField()), + ("media_type", models.CharField(max_length=80)), + ( + "blobs", + django.contrib.postgres.fields.ArrayField( + base_field=models.CharField(max_length=255), size=None + ), + ), + ("config_blob", models.CharField(max_length=255, null=True)), + ( + "pulp2content", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="docker_manifest_detail_model", + to="pulp_2to3_migration.Pulp2Content", + ), + ), ], options={ - 'default_related_name': 'docker_manifest_detail_model', - 'unique_together': {('digest', 'pulp2content')}, + "default_related_name": "docker_manifest_detail_model", + "unique_together": {("digest", "pulp2content")}, }, ), migrations.CreateModel( - name='Pulp2ISO', + name="Pulp2ISO", fields=[ - ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('pulp_created', models.DateTimeField(auto_now_add=True)), - ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)), - ('name', models.TextField()), - ('checksum', models.CharField(max_length=64)), - ('size', models.BigIntegerField()), - ('pulp2content', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='iso_detail_model', to='pulp_2to3_migration.Pulp2Content')), + ( + "pulp_id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("pulp_created", models.DateTimeField(auto_now_add=True)), + ("pulp_last_updated", models.DateTimeField(auto_now=True, null=True)), + ("name", models.TextField()), + ("checksum", models.CharField(max_length=64)), + ("size", models.BigIntegerField()), + ( + "pulp2content", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="iso_detail_model", + to="pulp_2to3_migration.Pulp2Content", + ), + ), ], options={ - 'default_related_name': 'iso_detail_model', - 'unique_together': {('name', 'checksum', 'size', 'pulp2content')}, + "default_related_name": "iso_detail_model", + "unique_together": {("name", "checksum", "size", "pulp2content")}, }, ), migrations.CreateModel( - name='Pulp2Distributor', + name="Pulp2Distributor", fields=[ - ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('pulp_created', models.DateTimeField(auto_now_add=True)), - ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)), - ('pulp2_object_id', models.CharField(max_length=255, unique=True)), - ('pulp2_id', models.TextField()), - ('pulp2_type_id', models.CharField(max_length=255)), - ('pulp2_config', django.contrib.postgres.fields.jsonb.JSONField()), - ('pulp2_auto_publish', models.BooleanField()), - ('pulp2_last_updated', models.DateTimeField()), - ('pulp2_repo_id', models.TextField()), - ('is_migrated', models.BooleanField(default=False)), - ('not_in_plan', models.BooleanField(default=False)), - ('pulp2_repository', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='pulp_2to3_migration.Pulp2Repository')), - ('pulp3_distribution', models.OneToOneField(null=True, on_delete=django.db.models.deletion.SET_NULL, to='core.BaseDistribution')), - ('pulp3_publication', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='core.Publication')), + ( + "pulp_id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("pulp_created", models.DateTimeField(auto_now_add=True)), + ("pulp_last_updated", models.DateTimeField(auto_now=True, null=True)), + ("pulp2_object_id", models.CharField(max_length=255, unique=True)), + ("pulp2_id", models.TextField()), + ("pulp2_type_id", models.CharField(max_length=255)), + ("pulp2_config", django.contrib.postgres.fields.jsonb.JSONField()), + ("pulp2_auto_publish", models.BooleanField()), + ("pulp2_last_updated", models.DateTimeField()), + ("pulp2_repo_id", models.TextField()), + ("is_migrated", models.BooleanField(default=False)), + ("not_in_plan", models.BooleanField(default=False)), + ( + "pulp2_repository", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="pulp_2to3_migration.Pulp2Repository", + ), + ), + ( + "pulp3_distribution", + models.OneToOneField( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to="core.BaseDistribution", + ), + ), + ( + "pulp3_publication", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to="core.Publication", + ), + ), ], options={ - 'unique_together': {('pulp2_repository', 'pulp2_id')}, + "unique_together": {("pulp2_repository", "pulp2_id")}, }, ), migrations.CreateModel( - name='Pulp2Blob', + name="Pulp2Blob", fields=[ - ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('pulp_created', models.DateTimeField(auto_now_add=True)), - ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)), - ('digest', models.CharField(max_length=255)), - ('media_type', models.CharField(max_length=80)), - ('pulp2content', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='docker_blob_detail_model', to='pulp_2to3_migration.Pulp2Content')), + ( + "pulp_id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("pulp_created", models.DateTimeField(auto_now_add=True)), + ("pulp_last_updated", models.DateTimeField(auto_now=True, null=True)), + ("digest", models.CharField(max_length=255)), + ("media_type", models.CharField(max_length=80)), + ( + "pulp2content", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="docker_blob_detail_model", + to="pulp_2to3_migration.Pulp2Content", + ), + ), ], options={ - 'default_related_name': 'docker_blob_detail_model', - 'unique_together': {('digest', 'pulp2content')}, + "default_related_name": "docker_blob_detail_model", + "unique_together": {("digest", "pulp2content")}, }, ), ] diff --git a/pulp_2to3_migration/app/migrations/0002_remove_pulp2distributor_pulp2_auto_publish.py b/pulp_2to3_migration/app/migrations/0002_remove_pulp2distributor_pulp2_auto_publish.py index fea9330d..2e7f3a03 100644 --- a/pulp_2to3_migration/app/migrations/0002_remove_pulp2distributor_pulp2_auto_publish.py +++ b/pulp_2to3_migration/app/migrations/0002_remove_pulp2distributor_pulp2_auto_publish.py @@ -6,12 +6,12 @@ class Migration(migrations.Migration): dependencies = [ - ('pulp_2to3_migration', '0001_initial'), + ("pulp_2to3_migration", "0001_initial"), ] operations = [ migrations.RemoveField( - model_name='pulp2distributor', - name='pulp2_auto_publish', + model_name="pulp2distributor", + name="pulp2_auto_publish", ), ] diff --git a/pulp_2to3_migration/app/migrations/0003_pulp2lazycatalog_indices.py b/pulp_2to3_migration/app/migrations/0003_pulp2lazycatalog_indices.py index 14fc0ab4..3324de2c 100644 --- a/pulp_2to3_migration/app/migrations/0003_pulp2lazycatalog_indices.py +++ b/pulp_2to3_migration/app/migrations/0003_pulp2lazycatalog_indices.py @@ -6,16 +6,20 @@ class Migration(migrations.Migration): dependencies = [ - ('pulp_2to3_migration', '0002_remove_pulp2distributor_pulp2_auto_publish'), + ("pulp_2to3_migration", "0002_remove_pulp2distributor_pulp2_auto_publish"), ] operations = [ migrations.AddIndex( - model_name='pulp2lazycatalog', - index=models.Index(fields=['pulp2_unit_id'], name='pulp_2to3_m_pulp2_u_c60485_idx'), + model_name="pulp2lazycatalog", + index=models.Index( + fields=["pulp2_unit_id"], name="pulp_2to3_m_pulp2_u_c60485_idx" + ), ), migrations.AddIndex( - model_name='pulp2lazycatalog', - index=models.Index(fields=['pulp2_content_type_id'], name='pulp_2to3_m_pulp2_c_766098_idx'), + model_name="pulp2lazycatalog", + index=models.Index( + fields=["pulp2_content_type_id"], name="pulp_2to3_m_pulp2_c_766098_idx" + ), ), ] diff --git a/pulp_2to3_migration/app/migrations/0004_modularity_metafile_erratum_rpm.py b/pulp_2to3_migration/app/migrations/0004_modularity_metafile_erratum_rpm.py index 3f4afa17..7677827f 100644 --- a/pulp_2to3_migration/app/migrations/0004_modularity_metafile_erratum_rpm.py +++ b/pulp_2to3_migration/app/migrations/0004_modularity_metafile_erratum_rpm.py @@ -9,131 +9,229 @@ class Migration(migrations.Migration): dependencies = [ - ('pulp_2to3_migration', '0003_pulp2lazycatalog_indices'), + ("pulp_2to3_migration", "0003_pulp2lazycatalog_indices"), ] operations = [ migrations.AddField( - model_name='pulp2content', - name='pulp2_repo', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='pulp_2to3_migration.Pulp2Repository'), + model_name="pulp2content", + name="pulp2_repo", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to="pulp_2to3_migration.Pulp2Repository", + ), ), migrations.AlterUniqueTogether( - name='pulp2content', - unique_together={('pulp2_id', 'pulp2_content_type_id', 'pulp2_repo')}, + name="pulp2content", + unique_together={("pulp2_id", "pulp2_content_type_id", "pulp2_repo")}, ), migrations.CreateModel( - name='Pulp2YumRepoMetadataFile', + name="Pulp2YumRepoMetadataFile", fields=[ - ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('pulp_created', models.DateTimeField(auto_now_add=True)), - ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)), - ('data_type', models.CharField(max_length=20)), - ('checksum', models.CharField(max_length=128)), - ('checksum_type', models.CharField(max_length=6)), - ('repo_id', models.TextField()), - ('pulp2content', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='yum_repo_metadata_file_detail_model', to='pulp_2to3_migration.Pulp2Content')), + ( + "pulp_id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("pulp_created", models.DateTimeField(auto_now_add=True)), + ("pulp_last_updated", models.DateTimeField(auto_now=True, null=True)), + ("data_type", models.CharField(max_length=20)), + ("checksum", models.CharField(max_length=128)), + ("checksum_type", models.CharField(max_length=6)), + ("repo_id", models.TextField()), + ( + "pulp2content", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="yum_repo_metadata_file_detail_model", + to="pulp_2to3_migration.Pulp2Content", + ), + ), ], options={ - 'default_related_name': 'yum_repo_metadata_file_detail_model', - 'unique_together': {('data_type', 'repo_id', 'pulp2content')}, + "default_related_name": "yum_repo_metadata_file_detail_model", + "unique_together": {("data_type", "repo_id", "pulp2content")}, }, ), migrations.CreateModel( - name='Pulp2Rpm', + name="Pulp2Rpm", fields=[ - ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('pulp_created', models.DateTimeField(auto_now_add=True)), - ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)), - ('name', models.TextField()), - ('epoch', models.TextField()), - ('version', models.TextField()), - ('release', models.TextField()), - ('arch', models.TextField()), - ('checksum', models.TextField()), - ('checksumtype', models.TextField()), - ('repodata', django.contrib.postgres.fields.jsonb.JSONField(verbose_name=dict)), - ('is_modular', models.BooleanField(default=False)), - ('size', models.PositiveIntegerField()), - ('filename', models.TextField()), - ('pulp2content', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='rpm_detail_model', to='pulp_2to3_migration.Pulp2Content')), + ( + "pulp_id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("pulp_created", models.DateTimeField(auto_now_add=True)), + ("pulp_last_updated", models.DateTimeField(auto_now=True, null=True)), + ("name", models.TextField()), + ("epoch", models.TextField()), + ("version", models.TextField()), + ("release", models.TextField()), + ("arch", models.TextField()), + ("checksum", models.TextField()), + ("checksumtype", models.TextField()), + ( + "repodata", + django.contrib.postgres.fields.jsonb.JSONField(verbose_name=dict), + ), + ("is_modular", models.BooleanField(default=False)), + ("size", models.PositiveIntegerField()), + ("filename", models.TextField()), + ( + "pulp2content", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="rpm_detail_model", + to="pulp_2to3_migration.Pulp2Content", + ), + ), ], options={ - 'default_related_name': 'rpm_detail_model', - 'unique_together': {('name', 'epoch', 'version', 'release', 'arch', 'checksumtype', 'checksum', 'pulp2content')}, + "default_related_name": "rpm_detail_model", + "unique_together": { + ( + "name", + "epoch", + "version", + "release", + "arch", + "checksumtype", + "checksum", + "pulp2content", + ) + }, }, ), migrations.CreateModel( - name='Pulp2ModulemdDefaults', + name="Pulp2ModulemdDefaults", fields=[ - ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('pulp_created', models.DateTimeField(auto_now_add=True)), - ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)), - ('module', models.TextField()), - ('stream', models.TextField()), - ('profiles', django.contrib.postgres.fields.jsonb.JSONField(verbose_name=dict)), - ('digest', models.TextField()), - ('repo_id', models.TextField()), - ('pulp2content', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='modulemd_defaults_detail_model', to='pulp_2to3_migration.Pulp2Content')), + ( + "pulp_id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("pulp_created", models.DateTimeField(auto_now_add=True)), + ("pulp_last_updated", models.DateTimeField(auto_now=True, null=True)), + ("module", models.TextField()), + ("stream", models.TextField()), + ( + "profiles", + django.contrib.postgres.fields.jsonb.JSONField(verbose_name=dict), + ), + ("digest", models.TextField()), + ("repo_id", models.TextField()), + ( + "pulp2content", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="modulemd_defaults_detail_model", + to="pulp_2to3_migration.Pulp2Content", + ), + ), ], options={ - 'default_related_name': 'modulemd_defaults_detail_model', - 'unique_together': {('digest', 'repo_id', 'pulp2content')}, + "default_related_name": "modulemd_defaults_detail_model", + "unique_together": {("digest", "repo_id", "pulp2content")}, }, ), migrations.CreateModel( - name='Pulp2Modulemd', + name="Pulp2Modulemd", fields=[ - ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('pulp_created', models.DateTimeField(auto_now_add=True)), - ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)), - ('name', models.TextField()), - ('stream', models.TextField()), - ('version', models.BigIntegerField()), - ('context', models.TextField()), - ('arch', models.TextField()), - ('artifacts', django.contrib.postgres.fields.jsonb.JSONField()), - ('checksum', models.TextField()), - ('dependencies', django.contrib.postgres.fields.jsonb.JSONField()), - ('pulp2content', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='modulemd_detail_model', to='pulp_2to3_migration.Pulp2Content')), + ( + "pulp_id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("pulp_created", models.DateTimeField(auto_now_add=True)), + ("pulp_last_updated", models.DateTimeField(auto_now=True, null=True)), + ("name", models.TextField()), + ("stream", models.TextField()), + ("version", models.BigIntegerField()), + ("context", models.TextField()), + ("arch", models.TextField()), + ("artifacts", django.contrib.postgres.fields.jsonb.JSONField()), + ("checksum", models.TextField()), + ("dependencies", django.contrib.postgres.fields.jsonb.JSONField()), + ( + "pulp2content", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="modulemd_detail_model", + to="pulp_2to3_migration.Pulp2Content", + ), + ), ], options={ - 'default_related_name': 'modulemd_detail_model', - 'unique_together': {('name', 'stream', 'version', 'context', 'arch', 'pulp2content')}, + "default_related_name": "modulemd_detail_model", + "unique_together": { + ("name", "stream", "version", "context", "arch", "pulp2content") + }, }, ), migrations.CreateModel( - name='Pulp2Erratum', + name="Pulp2Erratum", fields=[ - ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('pulp_created', models.DateTimeField(auto_now_add=True)), - ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)), - ('errata_id', models.TextField()), - ('updated', models.TextField()), - ('repo_id', models.TextField()), - ('issued', models.TextField()), - ('status', models.TextField()), - ('description', models.TextField()), - ('pushcount', models.TextField()), - ('references', django.contrib.postgres.fields.jsonb.JSONField()), - ('reboot_suggested', models.BooleanField()), - ('relogin_suggested', models.BooleanField()), - ('restart_suggested', models.BooleanField()), - ('errata_from', models.TextField()), - ('severity', models.TextField()), - ('rights', models.TextField()), - ('version', models.TextField()), - ('release', models.TextField()), - ('errata_type', models.TextField()), - ('pkglist', django.contrib.postgres.fields.jsonb.JSONField()), - ('title', models.TextField()), - ('solution', models.TextField()), - ('summary', models.TextField()), - ('pulp2content', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='erratum_detail_model', to='pulp_2to3_migration.Pulp2Content')), + ( + "pulp_id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("pulp_created", models.DateTimeField(auto_now_add=True)), + ("pulp_last_updated", models.DateTimeField(auto_now=True, null=True)), + ("errata_id", models.TextField()), + ("updated", models.TextField()), + ("repo_id", models.TextField()), + ("issued", models.TextField()), + ("status", models.TextField()), + ("description", models.TextField()), + ("pushcount", models.TextField()), + ("references", django.contrib.postgres.fields.jsonb.JSONField()), + ("reboot_suggested", models.BooleanField()), + ("relogin_suggested", models.BooleanField()), + ("restart_suggested", models.BooleanField()), + ("errata_from", models.TextField()), + ("severity", models.TextField()), + ("rights", models.TextField()), + ("version", models.TextField()), + ("release", models.TextField()), + ("errata_type", models.TextField()), + ("pkglist", django.contrib.postgres.fields.jsonb.JSONField()), + ("title", models.TextField()), + ("solution", models.TextField()), + ("summary", models.TextField()), + ( + "pulp2content", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="erratum_detail_model", + to="pulp_2to3_migration.Pulp2Content", + ), + ), ], options={ - 'default_related_name': 'erratum_detail_model', - 'unique_together': {('errata_id', 'repo_id')}, + "default_related_name": "erratum_detail_model", + "unique_together": {("errata_id", "repo_id")}, }, ), ] diff --git a/pulp_2to3_migration/app/migrations/0005_pulp2distribution.py b/pulp_2to3_migration/app/migrations/0005_pulp2distribution.py index dc81a03d..bd726ff5 100644 --- a/pulp_2to3_migration/app/migrations/0005_pulp2distribution.py +++ b/pulp_2to3_migration/app/migrations/0005_pulp2distribution.py @@ -8,26 +8,50 @@ class Migration(migrations.Migration): dependencies = [ - ('pulp_2to3_migration', '0004_modularity_metafile_erratum_rpm'), + ("pulp_2to3_migration", "0004_modularity_metafile_erratum_rpm"), ] operations = [ migrations.CreateModel( - name='Pulp2Distribution', + name="Pulp2Distribution", fields=[ - ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('pulp_created', models.DateTimeField(auto_now_add=True)), - ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)), - ('distribution_id', models.TextField()), - ('family', models.TextField()), - ('variant', models.TextField()), - ('version', models.TextField()), - ('arch', models.TextField()), - ('pulp2content', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='distribution_detail_model', to='pulp_2to3_migration.Pulp2Content')), + ( + "pulp_id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("pulp_created", models.DateTimeField(auto_now_add=True)), + ("pulp_last_updated", models.DateTimeField(auto_now=True, null=True)), + ("distribution_id", models.TextField()), + ("family", models.TextField()), + ("variant", models.TextField()), + ("version", models.TextField()), + ("arch", models.TextField()), + ( + "pulp2content", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="distribution_detail_model", + to="pulp_2to3_migration.Pulp2Content", + ), + ), ], options={ - 'default_related_name': 'distribution_detail_model', - 'unique_together': {('distribution_id', 'family', 'variant', 'version', 'arch', 'pulp2content')}, + "default_related_name": "distribution_detail_model", + "unique_together": { + ( + "distribution_id", + "family", + "variant", + "version", + "arch", + "pulp2content", + ) + }, }, ), ] diff --git a/pulp_2to3_migration/app/migrations/0006_pulp2_comps.py b/pulp_2to3_migration/app/migrations/0006_pulp2_comps.py index e88fef32..517ba5f6 100644 --- a/pulp_2to3_migration/app/migrations/0006_pulp2_comps.py +++ b/pulp_2to3_migration/app/migrations/0006_pulp2_comps.py @@ -9,90 +9,170 @@ class Migration(migrations.Migration): dependencies = [ - ('pulp_2to3_migration', '0005_pulp2distribution'), + ("pulp_2to3_migration", "0005_pulp2distribution"), ] operations = [ migrations.CreateModel( - name='Pulp2PackageLangpacks', + name="Pulp2PackageLangpacks", fields=[ - ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('pulp_created', models.DateTimeField(auto_now_add=True)), - ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)), - ('matches', django.contrib.postgres.fields.jsonb.JSONField()), - ('repo_id', models.TextField()), - ('pulp2content', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='package_langpacks_detail_model', to='pulp_2to3_migration.Pulp2Content')), + ( + "pulp_id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("pulp_created", models.DateTimeField(auto_now_add=True)), + ("pulp_last_updated", models.DateTimeField(auto_now=True, null=True)), + ("matches", django.contrib.postgres.fields.jsonb.JSONField()), + ("repo_id", models.TextField()), + ( + "pulp2content", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="package_langpacks_detail_model", + to="pulp_2to3_migration.Pulp2Content", + ), + ), ], options={ - 'default_related_name': 'package_langpacks_detail_model', - 'unique_together': {('repo_id', 'pulp2content')}, + "default_related_name": "package_langpacks_detail_model", + "unique_together": {("repo_id", "pulp2content")}, }, ), migrations.CreateModel( - name='Pulp2PackageGroup', + name="Pulp2PackageGroup", fields=[ - ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('pulp_created', models.DateTimeField(auto_now_add=True)), - ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)), - ('package_group_id', models.TextField()), - ('repo_id', models.TextField()), - ('default', models.BooleanField()), - ('user_visible', models.BooleanField()), - ('display_order', models.IntegerField(null=True)), - ('name', models.TextField()), - ('description', models.TextField()), - ('packages', django.contrib.postgres.fields.jsonb.JSONField()), - ('desc_by_lang', django.contrib.postgres.fields.jsonb.JSONField(verbose_name=dict)), - ('name_by_lang', django.contrib.postgres.fields.jsonb.JSONField(verbose_name=dict)), - ('biarch_only', models.BooleanField(default=False)), - ('pulp2content', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='package_group_detail_model', to='pulp_2to3_migration.Pulp2Content')), + ( + "pulp_id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("pulp_created", models.DateTimeField(auto_now_add=True)), + ("pulp_last_updated", models.DateTimeField(auto_now=True, null=True)), + ("package_group_id", models.TextField()), + ("repo_id", models.TextField()), + ("default", models.BooleanField()), + ("user_visible", models.BooleanField()), + ("display_order", models.IntegerField(null=True)), + ("name", models.TextField()), + ("description", models.TextField()), + ("packages", django.contrib.postgres.fields.jsonb.JSONField()), + ( + "desc_by_lang", + django.contrib.postgres.fields.jsonb.JSONField(verbose_name=dict), + ), + ( + "name_by_lang", + django.contrib.postgres.fields.jsonb.JSONField(verbose_name=dict), + ), + ("biarch_only", models.BooleanField(default=False)), + ( + "pulp2content", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="package_group_detail_model", + to="pulp_2to3_migration.Pulp2Content", + ), + ), ], options={ - 'default_related_name': 'package_group_detail_model', - 'unique_together': {('repo_id', 'package_group_id', 'pulp2content')}, + "default_related_name": "package_group_detail_model", + "unique_together": {("repo_id", "package_group_id", "pulp2content")}, }, ), migrations.CreateModel( - name='Pulp2PackageEnvironment', + name="Pulp2PackageEnvironment", fields=[ - ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('pulp_created', models.DateTimeField(auto_now_add=True)), - ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)), - ('package_environment_id', models.TextField()), - ('repo_id', models.TextField()), - ('display_order', models.IntegerField(null=True)), - ('name', models.TextField()), - ('description', models.TextField()), - ('group_ids', django.contrib.postgres.fields.jsonb.JSONField()), - ('option_ids', django.contrib.postgres.fields.jsonb.JSONField()), - ('desc_by_lang', django.contrib.postgres.fields.jsonb.JSONField(default=dict)), - ('name_by_lang', django.contrib.postgres.fields.jsonb.JSONField(default=dict)), - ('pulp2content', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='package_environment_detail_model', to='pulp_2to3_migration.Pulp2Content')), + ( + "pulp_id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("pulp_created", models.DateTimeField(auto_now_add=True)), + ("pulp_last_updated", models.DateTimeField(auto_now=True, null=True)), + ("package_environment_id", models.TextField()), + ("repo_id", models.TextField()), + ("display_order", models.IntegerField(null=True)), + ("name", models.TextField()), + ("description", models.TextField()), + ("group_ids", django.contrib.postgres.fields.jsonb.JSONField()), + ("option_ids", django.contrib.postgres.fields.jsonb.JSONField()), + ( + "desc_by_lang", + django.contrib.postgres.fields.jsonb.JSONField(default=dict), + ), + ( + "name_by_lang", + django.contrib.postgres.fields.jsonb.JSONField(default=dict), + ), + ( + "pulp2content", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="package_environment_detail_model", + to="pulp_2to3_migration.Pulp2Content", + ), + ), ], options={ - 'default_related_name': 'package_environment_detail_model', - 'unique_together': {('repo_id', 'package_environment_id', 'pulp2content')}, + "default_related_name": "package_environment_detail_model", + "unique_together": { + ("repo_id", "package_environment_id", "pulp2content") + }, }, ), migrations.CreateModel( - name='Pulp2PackageCategory', + name="Pulp2PackageCategory", fields=[ - ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('pulp_created', models.DateTimeField(auto_now_add=True)), - ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)), - ('package_category_id', models.TextField()), - ('repo_id', models.TextField()), - ('display_order', models.IntegerField(null=True)), - ('name', models.TextField()), - ('description', models.TextField()), - ('packagegroupids', django.contrib.postgres.fields.jsonb.JSONField()), - ('desc_by_lang', django.contrib.postgres.fields.jsonb.JSONField(verbose_name=dict)), - ('name_by_lang', django.contrib.postgres.fields.jsonb.JSONField(verbose_name=dict)), - ('pulp2content', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='package_category_detail_model', to='pulp_2to3_migration.Pulp2Content')), + ( + "pulp_id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("pulp_created", models.DateTimeField(auto_now_add=True)), + ("pulp_last_updated", models.DateTimeField(auto_now=True, null=True)), + ("package_category_id", models.TextField()), + ("repo_id", models.TextField()), + ("display_order", models.IntegerField(null=True)), + ("name", models.TextField()), + ("description", models.TextField()), + ("packagegroupids", django.contrib.postgres.fields.jsonb.JSONField()), + ( + "desc_by_lang", + django.contrib.postgres.fields.jsonb.JSONField(verbose_name=dict), + ), + ( + "name_by_lang", + django.contrib.postgres.fields.jsonb.JSONField(verbose_name=dict), + ), + ( + "pulp2content", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="package_category_detail_model", + to="pulp_2to3_migration.Pulp2Content", + ), + ), ], options={ - 'default_related_name': 'package_category_detail_model', - 'unique_together': {('repo_id', 'package_category_id', 'pulp2content')}, + "default_related_name": "package_category_detail_model", + "unique_together": {("repo_id", "package_category_id", "pulp2content")}, }, ), ] diff --git a/pulp_2to3_migration/app/migrations/0007_pulp2repository_pulp3_repository.py b/pulp_2to3_migration/app/migrations/0007_pulp2repository_pulp3_repository.py index 40967354..70cc8994 100644 --- a/pulp_2to3_migration/app/migrations/0007_pulp2repository_pulp3_repository.py +++ b/pulp_2to3_migration/app/migrations/0007_pulp2repository_pulp3_repository.py @@ -7,14 +7,18 @@ class Migration(migrations.Migration): dependencies = [ - ('core', '0026_task_group'), - ('pulp_2to3_migration', '0006_pulp2_comps'), + ("core", "0026_task_group"), + ("pulp_2to3_migration", "0006_pulp2_comps"), ] operations = [ migrations.AddField( - model_name='pulp2repository', - name='pulp3_repository', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='core.Repository'), + model_name="pulp2repository", + name="pulp3_repository", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to="core.Repository", + ), ), ] diff --git a/pulp_2to3_migration/app/migrations/0008_pulp2srpm.py b/pulp_2to3_migration/app/migrations/0008_pulp2srpm.py index 4dd13bde..e3557af6 100644 --- a/pulp_2to3_migration/app/migrations/0008_pulp2srpm.py +++ b/pulp_2to3_migration/app/migrations/0008_pulp2srpm.py @@ -9,32 +9,61 @@ class Migration(migrations.Migration): dependencies = [ - ('pulp_2to3_migration', '0007_pulp2repository_pulp3_repository'), + ("pulp_2to3_migration", "0007_pulp2repository_pulp3_repository"), ] operations = [ migrations.CreateModel( - name='Pulp2Srpm', + name="Pulp2Srpm", fields=[ - ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('pulp_created', models.DateTimeField(auto_now_add=True)), - ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)), - ('name', models.TextField()), - ('epoch', models.TextField()), - ('version', models.TextField()), - ('release', models.TextField()), - ('arch', models.TextField()), - ('checksum', models.TextField()), - ('checksumtype', models.TextField()), - ('repodata', django.contrib.postgres.fields.jsonb.JSONField(verbose_name=dict)), - ('is_modular', models.BooleanField(default=False)), - ('size', models.PositiveIntegerField()), - ('filename', models.TextField()), - ('pulp2content', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='srpm_detail_model', to='pulp_2to3_migration.Pulp2Content')), + ( + "pulp_id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("pulp_created", models.DateTimeField(auto_now_add=True)), + ("pulp_last_updated", models.DateTimeField(auto_now=True, null=True)), + ("name", models.TextField()), + ("epoch", models.TextField()), + ("version", models.TextField()), + ("release", models.TextField()), + ("arch", models.TextField()), + ("checksum", models.TextField()), + ("checksumtype", models.TextField()), + ( + "repodata", + django.contrib.postgres.fields.jsonb.JSONField(verbose_name=dict), + ), + ("is_modular", models.BooleanField(default=False)), + ("size", models.PositiveIntegerField()), + ("filename", models.TextField()), + ( + "pulp2content", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="srpm_detail_model", + to="pulp_2to3_migration.Pulp2Content", + ), + ), ], options={ - 'default_related_name': 'srpm_detail_model', - 'unique_together': {('name', 'epoch', 'version', 'release', 'arch', 'checksumtype', 'checksum', 'pulp2content')}, + "default_related_name": "srpm_detail_model", + "unique_together": { + ( + "name", + "epoch", + "version", + "release", + "arch", + "checksumtype", + "checksum", + "pulp2content", + ) + }, }, ), ] diff --git a/pulp_2to3_migration/app/migrations/0009_pulp2erratum_allow_null.py b/pulp_2to3_migration/app/migrations/0009_pulp2erratum_allow_null.py index 44ec2d79..07aac2b2 100644 --- a/pulp_2to3_migration/app/migrations/0009_pulp2erratum_allow_null.py +++ b/pulp_2to3_migration/app/migrations/0009_pulp2erratum_allow_null.py @@ -6,73 +6,73 @@ class Migration(migrations.Migration): dependencies = [ - ('pulp_2to3_migration', '0008_pulp2srpm'), + ("pulp_2to3_migration", "0008_pulp2srpm"), ] operations = [ migrations.AlterField( - model_name='pulp2erratum', - name='description', + model_name="pulp2erratum", + name="description", field=models.TextField(null=True), ), migrations.AlterField( - model_name='pulp2erratum', - name='errata_from', + model_name="pulp2erratum", + name="errata_from", field=models.TextField(null=True), ), migrations.AlterField( - model_name='pulp2erratum', - name='errata_type', + model_name="pulp2erratum", + name="errata_type", field=models.TextField(null=True), ), migrations.AlterField( - model_name='pulp2erratum', - name='issued', + model_name="pulp2erratum", + name="issued", field=models.TextField(null=True), ), migrations.AlterField( - model_name='pulp2erratum', - name='pushcount', + model_name="pulp2erratum", + name="pushcount", field=models.TextField(null=True), ), migrations.AlterField( - model_name='pulp2erratum', - name='release', + model_name="pulp2erratum", + name="release", field=models.TextField(null=True), ), migrations.AlterField( - model_name='pulp2erratum', - name='rights', + model_name="pulp2erratum", + name="rights", field=models.TextField(null=True), ), migrations.AlterField( - model_name='pulp2erratum', - name='severity', + model_name="pulp2erratum", + name="severity", field=models.TextField(null=True), ), migrations.AlterField( - model_name='pulp2erratum', - name='solution', + model_name="pulp2erratum", + name="solution", field=models.TextField(null=True), ), migrations.AlterField( - model_name='pulp2erratum', - name='status', + model_name="pulp2erratum", + name="status", field=models.TextField(null=True), ), migrations.AlterField( - model_name='pulp2erratum', - name='summary', + model_name="pulp2erratum", + name="summary", field=models.TextField(null=True), ), migrations.AlterField( - model_name='pulp2erratum', - name='title', + model_name="pulp2erratum", + name="title", field=models.TextField(null=True), ), migrations.AlterField( - model_name='pulp2erratum', - name='version', + model_name="pulp2erratum", + name="version", field=models.TextField(null=True), ), ] diff --git a/pulp_2to3_migration/app/migrations/0010_pulp2lazycatalog_is_migrated.py b/pulp_2to3_migration/app/migrations/0010_pulp2lazycatalog_is_migrated.py index 10131180..62b981e8 100644 --- a/pulp_2to3_migration/app/migrations/0010_pulp2lazycatalog_is_migrated.py +++ b/pulp_2to3_migration/app/migrations/0010_pulp2lazycatalog_is_migrated.py @@ -6,13 +6,13 @@ class Migration(migrations.Migration): dependencies = [ - ('pulp_2to3_migration', '0009_pulp2erratum_allow_null'), + ("pulp_2to3_migration", "0009_pulp2erratum_allow_null"), ] operations = [ migrations.AddField( - model_name='pulp2lazycatalog', - name='is_migrated', + model_name="pulp2lazycatalog", + name="is_migrated", field=models.BooleanField(default=False), ), ] diff --git a/pulp_2to3_migration/app/migrations/0011_lce_revision.py b/pulp_2to3_migration/app/migrations/0011_lce_revision.py index 8ae9fb85..c524691d 100644 --- a/pulp_2to3_migration/app/migrations/0011_lce_revision.py +++ b/pulp_2to3_migration/app/migrations/0011_lce_revision.py @@ -6,13 +6,13 @@ class Migration(migrations.Migration): dependencies = [ - ('pulp_2to3_migration', '0010_pulp2lazycatalog_is_migrated'), + ("pulp_2to3_migration", "0010_pulp2lazycatalog_is_migrated"), ] operations = [ migrations.AlterField( - model_name='pulp2lazycatalog', - name='pulp2_revision', + model_name="pulp2lazycatalog", + name="pulp2_revision", field=models.IntegerField(default=1), ), ] diff --git a/pulp_2to3_migration/app/migrations/0012_size_bigint.py b/pulp_2to3_migration/app/migrations/0012_size_bigint.py index 708cf2ae..8d04d117 100644 --- a/pulp_2to3_migration/app/migrations/0012_size_bigint.py +++ b/pulp_2to3_migration/app/migrations/0012_size_bigint.py @@ -6,18 +6,18 @@ class Migration(migrations.Migration): dependencies = [ - ('pulp_2to3_migration', '0011_lce_revision'), + ("pulp_2to3_migration", "0011_lce_revision"), ] operations = [ migrations.AlterField( - model_name='pulp2rpm', - name='size', + model_name="pulp2rpm", + name="size", field=models.BigIntegerField(), ), migrations.AlterField( - model_name='pulp2srpm', - name='size', + model_name="pulp2srpm", + name="size", field=models.BigIntegerField(), ), ] diff --git a/pulp_2to3_migration/app/migrations/0013_fix_file_remote_migration.py b/pulp_2to3_migration/app/migrations/0013_fix_file_remote_migration.py index e7bce70b..8d3cdba6 100644 --- a/pulp_2to3_migration/app/migrations/0013_fix_file_remote_migration.py +++ b/pulp_2to3_migration/app/migrations/0013_fix_file_remote_migration.py @@ -7,21 +7,23 @@ def fix_file_remote(apps, schema_editor): with transaction.atomic(): - Pulp2Importer = apps.get_model('pulp_2to3_migration', 'Pulp2Importer') - Remote = apps.get_model('core', 'Remote') - Pulp2LazyCatalog = apps.get_model('pulp_2to3_migration', 'Pulp2LazyCatalog') - pulp2isoimporter_qs = Pulp2Importer.objects.filter(pulp2_type_id='iso_importer', - is_migrated=True) + Pulp2Importer = apps.get_model("pulp_2to3_migration", "Pulp2Importer") + Remote = apps.get_model("core", "Remote") + Pulp2LazyCatalog = apps.get_model("pulp_2to3_migration", "Pulp2LazyCatalog") + pulp2isoimporter_qs = Pulp2Importer.objects.filter( + pulp2_type_id="iso_importer", is_migrated=True + ) pulp2lazycatalog_to_update = [] remotes_pk_to_delete = [] for importer in pulp2isoimporter_qs: pulp2_config = importer.pulp2_config - if pulp2_config.get('feed'): - pulp2_config['feed'] = urljoin(pulp2_config['feed'], 'PULP_MANIFEST') + if pulp2_config.get("feed"): + pulp2_config["feed"] = urljoin(pulp2_config["feed"], "PULP_MANIFEST") remotes_pk_to_delete.append(importer.pulp3_remote.pk) # find LCEs pulp2lazycatalog = Pulp2LazyCatalog.objects.filter( - pulp2_importer_id=importer.pulp2_object_id) + pulp2_importer_id=importer.pulp2_object_id + ) for lce in pulp2lazycatalog: lce.is_migrated = False pulp2lazycatalog_to_update.extend(pulp2lazycatalog) @@ -30,18 +32,20 @@ def fix_file_remote(apps, schema_editor): importer.pulp3_remote = None Remote.objects.filter(pk__in=remotes_pk_to_delete).delete() - Pulp2Importer.objects.bulk_update(objs=pulp2isoimporter_qs, - fields=['is_migrated', 'pulp2_config', 'pulp3_remote'], - batch_size=1000) - Pulp2LazyCatalog.objects.bulk_update(objs=pulp2lazycatalog_to_update, - fields=['is_migrated'], - batch_size=1000) + Pulp2Importer.objects.bulk_update( + objs=pulp2isoimporter_qs, + fields=["is_migrated", "pulp2_config", "pulp3_remote"], + batch_size=1000, + ) + Pulp2LazyCatalog.objects.bulk_update( + objs=pulp2lazycatalog_to_update, fields=["is_migrated"], batch_size=1000 + ) class Migration(migrations.Migration): dependencies = [ - ('pulp_2to3_migration', '0012_size_bigint'), + ("pulp_2to3_migration", "0012_size_bigint"), ] operations = [ diff --git a/pulp_2to3_migration/app/migrations/0014_gzip_repodata.py b/pulp_2to3_migration/app/migrations/0014_gzip_repodata.py index 9a339712..9d9bbf71 100644 --- a/pulp_2to3_migration/app/migrations/0014_gzip_repodata.py +++ b/pulp_2to3_migration/app/migrations/0014_gzip_repodata.py @@ -13,18 +13,26 @@ def gzip_repodata(apps, schema_editor): """ def gzip_pkg_repodata(pkg): - pkg.primary_template_gz = gzip.zlib.compress(pkg.repodata.get('primary').encode()) - pkg.other_template_gz = gzip.zlib.compress(pkg.repodata.get('other').encode()) - pkg.filelists_template_gz = gzip.zlib.compress(pkg.repodata.get('filelists').encode()) - pkg.repodata = '' + pkg.primary_template_gz = gzip.zlib.compress( + pkg.repodata.get("primary").encode() + ) + pkg.other_template_gz = gzip.zlib.compress(pkg.repodata.get("other").encode()) + pkg.filelists_template_gz = gzip.zlib.compress( + pkg.repodata.get("filelists").encode() + ) + pkg.repodata = "" return pkg - - Pulp2Rpm = apps.get_model('pulp_2to3_migration', 'Pulp2Rpm') - Pulp2Srpm = apps.get_model('pulp_2to3_migration', 'Pulp2Srpm') + + Pulp2Rpm = apps.get_model("pulp_2to3_migration", "Pulp2Rpm") + Pulp2Srpm = apps.get_model("pulp_2to3_migration", "Pulp2Srpm") batch_size = 500 - fields_to_update = ('repodata', 'primary_template_gz', 'other_template_gz', - 'filelists_template_gz') + fields_to_update = ( + "repodata", + "primary_template_gz", + "other_template_gz", + "filelists_template_gz", + ) for model in (Pulp2Rpm, Pulp2Srpm): batch_to_save = [] @@ -41,50 +49,48 @@ def gzip_pkg_repodata(pkg): class Migration(migrations.Migration): dependencies = [ - ('pulp_2to3_migration', '0013_fix_file_remote_migration'), + ("pulp_2to3_migration", "0013_fix_file_remote_migration"), ] operations = [ migrations.AddField( - model_name='pulp2rpm', - name='filelists_template_gz', + model_name="pulp2rpm", + name="filelists_template_gz", field=models.BinaryField(null=True), ), migrations.AddField( - model_name='pulp2rpm', - name='other_template_gz', + model_name="pulp2rpm", + name="other_template_gz", field=models.BinaryField(null=True), ), migrations.AddField( - model_name='pulp2rpm', - name='primary_template_gz', + model_name="pulp2rpm", + name="primary_template_gz", field=models.BinaryField(null=True), ), migrations.AddField( - model_name='pulp2srpm', - name='filelists_template_gz', + model_name="pulp2srpm", + name="filelists_template_gz", field=models.BinaryField(null=True), ), migrations.AddField( - model_name='pulp2srpm', - name='other_template_gz', + model_name="pulp2srpm", + name="other_template_gz", field=models.BinaryField(null=True), ), migrations.AddField( - model_name='pulp2srpm', - name='primary_template_gz', + model_name="pulp2srpm", + name="primary_template_gz", field=models.BinaryField(null=True), ), - migrations.RunPython(gzip_repodata), - # Those fields can be removed only after data migration happened migrations.RemoveField( - model_name='pulp2rpm', - name='repodata', + model_name="pulp2rpm", + name="repodata", ), migrations.RemoveField( - model_name='pulp2srpm', - name='repodata', + model_name="pulp2srpm", + name="repodata", ), ] diff --git a/pulp_2to3_migration/app/migrations/0015_add_created_updated.py b/pulp_2to3_migration/app/migrations/0015_add_created_updated.py index 80313377..d9586073 100644 --- a/pulp_2to3_migration/app/migrations/0015_add_created_updated.py +++ b/pulp_2to3_migration/app/migrations/0015_add_created_updated.py @@ -6,18 +6,18 @@ class Migration(migrations.Migration): dependencies = [ - ('pulp_2to3_migration', '0014_gzip_repodata'), + ("pulp_2to3_migration", "0014_gzip_repodata"), ] operations = [ migrations.AddField( - model_name='pulp2repocontent', - name='pulp2_created', + model_name="pulp2repocontent", + name="pulp2_created", field=models.DateTimeField(null=True), ), migrations.AddField( - model_name='pulp2repocontent', - name='pulp2_updated', + model_name="pulp2repocontent", + name="pulp2_updated", field=models.DateTimeField(null=True), ), ] diff --git a/pulp_2to3_migration/app/migrations/0016_type_index.py b/pulp_2to3_migration/app/migrations/0016_type_index.py index 21157548..a397f075 100644 --- a/pulp_2to3_migration/app/migrations/0016_type_index.py +++ b/pulp_2to3_migration/app/migrations/0016_type_index.py @@ -6,28 +6,38 @@ class Migration(migrations.Migration): dependencies = [ - ('pulp_2to3_migration', '0015_add_created_updated'), + ("pulp_2to3_migration", "0015_add_created_updated"), ] operations = [ migrations.AddIndex( - model_name='pulp2content', - index=models.Index(fields=['pulp2_content_type_id'], name='pulp_2to3_m_pulp2_c_7621eb_idx'), + model_name="pulp2content", + index=models.Index( + fields=["pulp2_content_type_id"], name="pulp_2to3_m_pulp2_c_7621eb_idx" + ), ), migrations.AddIndex( - model_name='pulp2distributor', - index=models.Index(fields=['pulp2_type_id'], name='pulp_2to3_m_pulp2_t_f161fa_idx'), + model_name="pulp2distributor", + index=models.Index( + fields=["pulp2_type_id"], name="pulp_2to3_m_pulp2_t_f161fa_idx" + ), ), migrations.AddIndex( - model_name='pulp2importer', - index=models.Index(fields=['pulp2_type_id'], name='pulp_2to3_m_pulp2_t_e87f6f_idx'), + model_name="pulp2importer", + index=models.Index( + fields=["pulp2_type_id"], name="pulp_2to3_m_pulp2_t_e87f6f_idx" + ), ), migrations.AddIndex( - model_name='pulp2repocontent', - index=models.Index(fields=['pulp2_content_type_id'], name='pulp_2to3_m_pulp2_c_6007dc_idx'), + model_name="pulp2repocontent", + index=models.Index( + fields=["pulp2_content_type_id"], name="pulp_2to3_m_pulp2_c_6007dc_idx" + ), ), migrations.AddIndex( - model_name='pulp2repository', - index=models.Index(fields=['pulp2_repo_type'], name='pulp_2to3_m_pulp2_r_536467_idx'), + model_name="pulp2repository", + index=models.Index( + fields=["pulp2_repo_type"], name="pulp_2to3_m_pulp2_r_536467_idx" + ), ), ] diff --git a/pulp_2to3_migration/app/migrations/0017_pulp2debpackage.py b/pulp_2to3_migration/app/migrations/0017_pulp2debpackage.py index 411f78f8..25e07015 100644 --- a/pulp_2to3_migration/app/migrations/0017_pulp2debpackage.py +++ b/pulp_2to3_migration/app/migrations/0017_pulp2debpackage.py @@ -9,52 +9,76 @@ class Migration(migrations.Migration): dependencies = [ - ('pulp_2to3_migration', '0016_type_index'), + ("pulp_2to3_migration", "0016_type_index"), ] operations = [ migrations.CreateModel( - name='Pulp2DebPackage', + name="Pulp2DebPackage", fields=[ - ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('pulp_created', models.DateTimeField(auto_now_add=True)), - ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)), - ('checksumtype', models.TextField()), - ('checksum', models.TextField()), - ('filename', models.TextField()), - ('size', models.BigIntegerField()), - ('extra_control_fields', models.TextField(null=True)), - ('package', models.TextField()), - ('source', models.TextField(null=True)), - ('version', models.TextField()), - ('essential', models.TextField(null=True)), - ('installed_size', models.TextField(null=True)), - ('maintainer', models.TextField(null=True)), - ('original_maintainer', models.TextField(null=True)), - ('architecture', models.TextField()), - ('replaces', models.TextField(null=True)), - ('provides', models.TextField(null=True)), - ('depends', models.TextField(null=True)), - ('pre_depends', models.TextField(null=True)), - ('recommends', models.TextField(null=True)), - ('suggests', models.TextField(null=True)), - ('enhances', models.TextField(null=True)), - ('conflicts', models.TextField(null=True)), - ('breaks', models.TextField(null=True)), - ('description', models.TextField(null=True)), - ('multi_arch', models.TextField(null=True)), - ('homepage', models.TextField(null=True)), - ('built_using', models.TextField(null=True)), - ('description_md5', models.TextField(null=True)), - ('build_essential', models.TextField(null=True)), - ('tag', models.TextField(null=True)), - ('section', models.TextField(null=True)), - ('priority', models.TextField(null=True)), - ('pulp2content', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='deb_detail_model', to='pulp_2to3_migration.Pulp2Content')), + ( + "pulp_id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("pulp_created", models.DateTimeField(auto_now_add=True)), + ("pulp_last_updated", models.DateTimeField(auto_now=True, null=True)), + ("checksumtype", models.TextField()), + ("checksum", models.TextField()), + ("filename", models.TextField()), + ("size", models.BigIntegerField()), + ("extra_control_fields", models.TextField(null=True)), + ("package", models.TextField()), + ("source", models.TextField(null=True)), + ("version", models.TextField()), + ("essential", models.TextField(null=True)), + ("installed_size", models.TextField(null=True)), + ("maintainer", models.TextField(null=True)), + ("original_maintainer", models.TextField(null=True)), + ("architecture", models.TextField()), + ("replaces", models.TextField(null=True)), + ("provides", models.TextField(null=True)), + ("depends", models.TextField(null=True)), + ("pre_depends", models.TextField(null=True)), + ("recommends", models.TextField(null=True)), + ("suggests", models.TextField(null=True)), + ("enhances", models.TextField(null=True)), + ("conflicts", models.TextField(null=True)), + ("breaks", models.TextField(null=True)), + ("description", models.TextField(null=True)), + ("multi_arch", models.TextField(null=True)), + ("homepage", models.TextField(null=True)), + ("built_using", models.TextField(null=True)), + ("description_md5", models.TextField(null=True)), + ("build_essential", models.TextField(null=True)), + ("tag", models.TextField(null=True)), + ("section", models.TextField(null=True)), + ("priority", models.TextField(null=True)), + ( + "pulp2content", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="deb_detail_model", + to="pulp_2to3_migration.Pulp2Content", + ), + ), ], options={ - 'default_related_name': 'deb_detail_model', - 'unique_together': {('package', 'version', 'architecture', 'checksumtype', 'checksum', 'pulp2content')}, + "default_related_name": "deb_detail_model", + "unique_together": { + ( + "package", + "version", + "architecture", + "checksumtype", + "checksum", + "pulp2content", + ) + }, }, bases=(django_lifecycle.mixins.LifecycleModelMixin, models.Model), ), diff --git a/pulp_2to3_migration/app/migrations/0018_pulp2distributor_pulp2_repos.py b/pulp_2to3_migration/app/migrations/0018_pulp2distributor_pulp2_repos.py index 9d225b4a..3224f1a1 100644 --- a/pulp_2to3_migration/app/migrations/0018_pulp2distributor_pulp2_repos.py +++ b/pulp_2to3_migration/app/migrations/0018_pulp2distributor_pulp2_repos.py @@ -7,7 +7,7 @@ def make_pulp2repos_m2m(apps, schema_editor): """ Add many2many relation between pulp2repo and pulp2dist. """ - Pulp2Distributor = apps.get_model('pulp_2to3_migration', 'Pulp2Distributor') + Pulp2Distributor = apps.get_model("pulp_2to3_migration", "Pulp2Distributor") pulp2_dists_qs = Pulp2Distributor.objects.all() for dist in pulp2_dists_qs.iterator(): @@ -18,14 +18,16 @@ def make_pulp2repos_m2m(apps, schema_editor): class Migration(migrations.Migration): dependencies = [ - ('pulp_2to3_migration', '0017_pulp2debpackage'), + ("pulp_2to3_migration", "0017_pulp2debpackage"), ] operations = [ migrations.AddField( - model_name='pulp2distributor', - name='pulp2_repos', - field=models.ManyToManyField(related_name='pulp2_dists', to='pulp_2to3_migration.Pulp2Repository'), + model_name="pulp2distributor", + name="pulp2_repos", + field=models.ManyToManyField( + related_name="pulp2_dists", to="pulp_2to3_migration.Pulp2Repository" + ), ), migrations.RunPython(make_pulp2repos_m2m), ] diff --git a/pulp_2to3_migration/app/migrations/0019_remove_pulp2_repository.py b/pulp_2to3_migration/app/migrations/0019_remove_pulp2_repository.py index 70968b55..706bc9b4 100644 --- a/pulp_2to3_migration/app/migrations/0019_remove_pulp2_repository.py +++ b/pulp_2to3_migration/app/migrations/0019_remove_pulp2_repository.py @@ -6,16 +6,16 @@ class Migration(migrations.Migration): dependencies = [ - ('pulp_2to3_migration', '0018_pulp2distributor_pulp2_repos'), + ("pulp_2to3_migration", "0018_pulp2distributor_pulp2_repos"), ] operations = [ migrations.AlterUniqueTogether( - name='pulp2distributor', - unique_together={('pulp2_object_id',)}, + name="pulp2distributor", + unique_together={("pulp2_object_id",)}, ), migrations.RemoveField( - model_name='pulp2distributor', - name='pulp2_repository', + model_name="pulp2distributor", + name="pulp2_repository", ), ] diff --git a/pulp_2to3_migration/app/migrations/0020_fix_defaults.py b/pulp_2to3_migration/app/migrations/0020_fix_defaults.py index a93725be..0710cc97 100644 --- a/pulp_2to3_migration/app/migrations/0020_fix_defaults.py +++ b/pulp_2to3_migration/app/migrations/0020_fix_defaults.py @@ -7,38 +7,38 @@ class Migration(migrations.Migration): dependencies = [ - ('pulp_2to3_migration', '0019_remove_pulp2_repository'), + ("pulp_2to3_migration", "0019_remove_pulp2_repository"), ] operations = [ migrations.AlterField( - model_name='pulp2modulemddefaults', - name='profiles', + model_name="pulp2modulemddefaults", + name="profiles", field=django.contrib.postgres.fields.jsonb.JSONField(default=dict), ), migrations.AlterField( - model_name='pulp2modulemd', - name='dependencies', + model_name="pulp2modulemd", + name="dependencies", field=django.contrib.postgres.fields.jsonb.JSONField(default=list), ), migrations.AlterField( - model_name='pulp2packagecategory', - name='desc_by_lang', + model_name="pulp2packagecategory", + name="desc_by_lang", field=django.contrib.postgres.fields.jsonb.JSONField(default=dict), ), migrations.AlterField( - model_name='pulp2packagecategory', - name='name_by_lang', + model_name="pulp2packagecategory", + name="name_by_lang", field=django.contrib.postgres.fields.jsonb.JSONField(default=dict), ), migrations.AlterField( - model_name='pulp2packagegroup', - name='desc_by_lang', + model_name="pulp2packagegroup", + name="desc_by_lang", field=django.contrib.postgres.fields.jsonb.JSONField(default=dict), ), migrations.AlterField( - model_name='pulp2packagegroup', - name='name_by_lang', + model_name="pulp2packagegroup", + name="name_by_lang", field=django.contrib.postgres.fields.jsonb.JSONField(default=dict), ), ] diff --git a/pulp_2to3_migration/app/migrations/0021_pulp2content_add_subid.py b/pulp_2to3_migration/app/migrations/0021_pulp2content_add_subid.py index b86ffb30..e07010bb 100644 --- a/pulp_2to3_migration/app/migrations/0021_pulp2content_add_subid.py +++ b/pulp_2to3_migration/app/migrations/0021_pulp2content_add_subid.py @@ -6,17 +6,19 @@ class Migration(migrations.Migration): dependencies = [ - ('pulp_2to3_migration', '0020_fix_defaults'), + ("pulp_2to3_migration", "0020_fix_defaults"), ] operations = [ migrations.AddField( - model_name='pulp2content', - name='pulp2_subid', + model_name="pulp2content", + name="pulp2_subid", field=models.CharField(blank=True, max_length=255), ), migrations.AlterUniqueTogether( - name='pulp2content', - unique_together={('pulp2_id', 'pulp2_content_type_id', 'pulp2_repo', 'pulp2_subid')}, + name="pulp2content", + unique_together={ + ("pulp2_id", "pulp2_content_type_id", "pulp2_repo", "pulp2_subid") + }, ), ] diff --git a/pulp_2to3_migration/app/migrations/0022_add_structured_deb_types.py b/pulp_2to3_migration/app/migrations/0022_add_structured_deb_types.py index b7a0303f..5e31f8ea 100644 --- a/pulp_2to3_migration/app/migrations/0022_add_structured_deb_types.py +++ b/pulp_2to3_migration/app/migrations/0022_add_structured_deb_types.py @@ -9,80 +9,151 @@ class Migration(migrations.Migration): dependencies = [ - ('pulp_2to3_migration', '0021_pulp2content_add_subid'), + ("pulp_2to3_migration", "0021_pulp2content_add_subid"), ] operations = [ migrations.CreateModel( - name='Pulp2DebReleaseArchitecture', + name="Pulp2DebReleaseArchitecture", fields=[ - ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('pulp_created', models.DateTimeField(auto_now_add=True)), - ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)), - ('architecture', models.TextField()), - ('distribution', models.TextField()), - ('codename', models.TextField()), - ('suite', models.TextField()), - ('pulp2content', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='deb_release_architecture_detail_model', to='pulp_2to3_migration.Pulp2Content')), + ( + "pulp_id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("pulp_created", models.DateTimeField(auto_now_add=True)), + ("pulp_last_updated", models.DateTimeField(auto_now=True, null=True)), + ("architecture", models.TextField()), + ("distribution", models.TextField()), + ("codename", models.TextField()), + ("suite", models.TextField()), + ( + "pulp2content", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="deb_release_architecture_detail_model", + to="pulp_2to3_migration.Pulp2Content", + ), + ), ], options={ - 'default_related_name': 'deb_release_architecture_detail_model', - 'unique_together': {('architecture', 'distribution', 'codename', 'suite')}, + "default_related_name": "deb_release_architecture_detail_model", + "unique_together": { + ("architecture", "distribution", "codename", "suite") + }, }, bases=(django_lifecycle.mixins.LifecycleModelMixin, models.Model), ), migrations.CreateModel( - name='Pulp2DebRelease', + name="Pulp2DebRelease", fields=[ - ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('pulp_created', models.DateTimeField(auto_now_add=True)), - ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)), - ('distribution', models.TextField()), - ('codename', models.TextField()), - ('suite', models.TextField(null=True)), - ('pulp2content', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='deb_release_detail_model', to='pulp_2to3_migration.Pulp2Content')), + ( + "pulp_id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("pulp_created", models.DateTimeField(auto_now_add=True)), + ("pulp_last_updated", models.DateTimeField(auto_now=True, null=True)), + ("distribution", models.TextField()), + ("codename", models.TextField()), + ("suite", models.TextField(null=True)), + ( + "pulp2content", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="deb_release_detail_model", + to="pulp_2to3_migration.Pulp2Content", + ), + ), ], options={ - 'default_related_name': 'deb_release_detail_model', - 'unique_together': {('codename', 'suite', 'distribution')}, + "default_related_name": "deb_release_detail_model", + "unique_together": {("codename", "suite", "distribution")}, }, bases=(django_lifecycle.mixins.LifecycleModelMixin, models.Model), ), migrations.CreateModel( - name='Pulp2DebComponentPackage', + name="Pulp2DebComponentPackage", fields=[ - ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('pulp_created', models.DateTimeField(auto_now_add=True)), - ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)), - ('package_relative_path', models.TextField()), - ('package_sha256', models.TextField()), - ('component', models.TextField()), - ('distribution', models.TextField()), - ('codename', models.TextField()), - ('suite', models.TextField()), - ('pulp2content', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='deb_component_package_detail_model', to='pulp_2to3_migration.Pulp2Content')), + ( + "pulp_id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("pulp_created", models.DateTimeField(auto_now_add=True)), + ("pulp_last_updated", models.DateTimeField(auto_now=True, null=True)), + ("package_relative_path", models.TextField()), + ("package_sha256", models.TextField()), + ("component", models.TextField()), + ("distribution", models.TextField()), + ("codename", models.TextField()), + ("suite", models.TextField()), + ( + "pulp2content", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="deb_component_package_detail_model", + to="pulp_2to3_migration.Pulp2Content", + ), + ), ], options={ - 'default_related_name': 'deb_component_package_detail_model', - 'unique_together': {('package_relative_path', 'package_sha256', 'component', 'distribution', 'codename', 'suite')}, + "default_related_name": "deb_component_package_detail_model", + "unique_together": { + ( + "package_relative_path", + "package_sha256", + "component", + "distribution", + "codename", + "suite", + ) + }, }, bases=(django_lifecycle.mixins.LifecycleModelMixin, models.Model), ), migrations.CreateModel( - name='Pulp2DebComponent', + name="Pulp2DebComponent", fields=[ - ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('pulp_created', models.DateTimeField(auto_now_add=True)), - ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)), - ('distribution', models.TextField()), - ('codename', models.TextField()), - ('component', models.TextField()), - ('suite', models.TextField()), - ('pulp2content', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='deb_component_detail_model', to='pulp_2to3_migration.Pulp2Content')), + ( + "pulp_id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("pulp_created", models.DateTimeField(auto_now_add=True)), + ("pulp_last_updated", models.DateTimeField(auto_now=True, null=True)), + ("distribution", models.TextField()), + ("codename", models.TextField()), + ("component", models.TextField()), + ("suite", models.TextField()), + ( + "pulp2content", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="deb_component_detail_model", + to="pulp_2to3_migration.Pulp2Content", + ), + ), ], options={ - 'default_related_name': 'deb_component_detail_model', - 'unique_together': {('component', 'codename', 'suite', 'distribution')}, + "default_related_name": "deb_component_detail_model", + "unique_together": {("component", "codename", "suite", "distribution")}, }, bases=(django_lifecycle.mixins.LifecycleModelMixin, models.Model), ), diff --git a/pulp_2to3_migration/app/migrations/0023_remove_pulp2importer_repo_field.py b/pulp_2to3_migration/app/migrations/0023_remove_pulp2importer_repo_field.py index 36c99476..20b8af2c 100644 --- a/pulp_2to3_migration/app/migrations/0023_remove_pulp2importer_repo_field.py +++ b/pulp_2to3_migration/app/migrations/0023_remove_pulp2importer_repo_field.py @@ -6,12 +6,12 @@ class Migration(migrations.Migration): dependencies = [ - ('pulp_2to3_migration', '0022_add_structured_deb_types'), + ("pulp_2to3_migration", "0022_add_structured_deb_types"), ] operations = [ migrations.RemoveField( - model_name='pulp2importer', - name='pulp2_repository', + model_name="pulp2importer", + name="pulp2_repository", ), ] diff --git a/pulp_2to3_migration/app/migrations/0024_reposetup.py b/pulp_2to3_migration/app/migrations/0024_reposetup.py index d07a34b3..96608168 100644 --- a/pulp_2to3_migration/app/migrations/0024_reposetup.py +++ b/pulp_2to3_migration/app/migrations/0024_reposetup.py @@ -8,34 +8,56 @@ class Migration(migrations.Migration): dependencies = [ - ('pulp_2to3_migration', '0023_remove_pulp2importer_repo_field'), + ("pulp_2to3_migration", "0023_remove_pulp2importer_repo_field"), ] operations = [ migrations.CreateModel( - name='RepoSetup', + name="RepoSetup", fields=[ - ('pulp_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('pulp_created', models.DateTimeField(auto_now_add=True)), - ('pulp_last_updated', models.DateTimeField(auto_now=True, null=True)), - ('pulp2_repo_id', models.TextField()), - ('pulp2_repo_type', models.CharField(max_length=25)), - ('pulp2_resource_repo_id', models.TextField(blank=True)), - ('pulp2_resource_type', models.SmallIntegerField(choices=[(0, 'importer'), (1, 'distributor')])), - ('status', models.SmallIntegerField(choices=[(0, 'old'), (1, 'up to date'), (2, 'new')])), + ( + "pulp_id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("pulp_created", models.DateTimeField(auto_now_add=True)), + ("pulp_last_updated", models.DateTimeField(auto_now=True, null=True)), + ("pulp2_repo_id", models.TextField()), + ("pulp2_repo_type", models.CharField(max_length=25)), + ("pulp2_resource_repo_id", models.TextField(blank=True)), + ( + "pulp2_resource_type", + models.SmallIntegerField( + choices=[(0, "importer"), (1, "distributor")] + ), + ), + ( + "status", + models.SmallIntegerField( + choices=[(0, "old"), (1, "up to date"), (2, "new")] + ), + ), ], bases=(django_lifecycle.mixins.LifecycleModelMixin, models.Model), ), migrations.AddIndex( - model_name='reposetup', - index=models.Index(fields=['pulp2_resource_type'], name='pulp_2to3_m_pulp2_r_d9a309_idx'), + model_name="reposetup", + index=models.Index( + fields=["pulp2_resource_type"], name="pulp_2to3_m_pulp2_r_d9a309_idx" + ), ), migrations.AddIndex( - model_name='reposetup', - index=models.Index(fields=['status'], name='pulp_2to3_m_status_47e94a_idx'), + model_name="reposetup", + index=models.Index(fields=["status"], name="pulp_2to3_m_status_47e94a_idx"), ), migrations.AlterUniqueTogether( - name='reposetup', - unique_together={('pulp2_repo_id', 'pulp2_resource_repo_id', 'pulp2_resource_type')}, + name="reposetup", + unique_together={ + ("pulp2_repo_id", "pulp2_resource_repo_id", "pulp2_resource_type") + }, ), ] diff --git a/pulp_2to3_migration/app/migrations/0025_remove_p2content_dups.py b/pulp_2to3_migration/app/migrations/0025_remove_p2content_dups.py index 6422d56d..f21bc319 100644 --- a/pulp_2to3_migration/app/migrations/0025_remove_p2content_dups.py +++ b/pulp_2to3_migration/app/migrations/0025_remove_p2content_dups.py @@ -6,15 +6,29 @@ def fix_dups(apps, schema_editor): pulp2content = apps.get_model("pulp_2to3_migration", "Pulp2Content") # find sets of p2-id/p2-type/-2-repo/p2-sub dups - p2content = pulp2content.objects.values('pulp2_id', 'pulp2_content_type_id', 'pulp2_repo_id', 'pulp2_subid').order_by('-pulp2_id') - p2content = p2content.annotate(count_id=models.Count("pulp2_id")).filter(count_id__gt=1) + p2content = pulp2content.objects.values( + "pulp2_id", "pulp2_content_type_id", "pulp2_repo_id", "pulp2_subid" + ).order_by("-pulp2_id") + p2content = p2content.annotate(count_id=models.Count("pulp2_id")).filter( + count_id__gt=1 + ) # for each set of dups... for dup in p2content: # ...find the pulp-ids of entries in that set (ordering by pulp2-last-update such that # newest(largest-timestamp) is first entry). - unique_keys = {x: dup[x] for x in ['pulp2_id', 'pulp2_content_type_id', 'pulp2_repo_id', 'pulp2_subid']} - to_delete = pulp2content.objects.filter(**unique_keys).order_by('-pulp2_last_updated') + unique_keys = { + x: dup[x] + for x in [ + "pulp2_id", + "pulp2_content_type_id", + "pulp2_repo_id", + "pulp2_subid", + ] + } + to_delete = pulp2content.objects.filter(**unique_keys).order_by( + "-pulp2_last_updated" + ) # exclude the first/newest one... to_delete = to_delete.exclude(pk=to_delete[0].pk) # ...and delete the rest @@ -24,7 +38,7 @@ def fix_dups(apps, schema_editor): class Migration(migrations.Migration): dependencies = [ - ('pulp_2to3_migration', '0024_reposetup'), + ("pulp_2to3_migration", "0024_reposetup"), ] operations = [ diff --git a/pulp_2to3_migration/app/migrations/0026_p2content_UQ.py b/pulp_2to3_migration/app/migrations/0026_p2content_UQ.py index 3afa33ca..b02a0a18 100644 --- a/pulp_2to3_migration/app/migrations/0026_p2content_UQ.py +++ b/pulp_2to3_migration/app/migrations/0026_p2content_UQ.py @@ -7,20 +7,32 @@ class Migration(migrations.Migration): dependencies = [ - ('pulp_2to3_migration', '0025_remove_p2content_dups'), + ("pulp_2to3_migration", "0025_remove_p2content_dups"), ] operations = [ migrations.AlterUniqueTogether( - name='pulp2content', + name="pulp2content", unique_together=set(), ), migrations.AddConstraint( - model_name='pulp2content', - constraint=models.UniqueConstraint(fields=('pulp2_id', 'pulp2_content_type_id', 'pulp2_repo', 'pulp2_subid'), name='unique_with_optional'), + model_name="pulp2content", + constraint=models.UniqueConstraint( + fields=( + "pulp2_id", + "pulp2_content_type_id", + "pulp2_repo", + "pulp2_subid", + ), + name="unique_with_optional", + ), ), migrations.AddConstraint( - model_name='pulp2content', - constraint=models.UniqueConstraint(condition=models.Q(pulp2_repo=None), fields=('pulp2_id', 'pulp2_content_type_id', 'pulp2_subid'), name='unique_without_optional'), + model_name="pulp2content", + constraint=models.UniqueConstraint( + condition=models.Q(pulp2_repo=None), + fields=("pulp2_id", "pulp2_content_type_id", "pulp2_subid"), + name="unique_without_optional", + ), ), ] diff --git a/pulp_2to3_migration/app/migrations/0027_pulp2repocontent_created_index.py b/pulp_2to3_migration/app/migrations/0027_pulp2repocontent_created_index.py index 03a11e0e..18cb7bcd 100644 --- a/pulp_2to3_migration/app/migrations/0027_pulp2repocontent_created_index.py +++ b/pulp_2to3_migration/app/migrations/0027_pulp2repocontent_created_index.py @@ -6,12 +6,14 @@ class Migration(migrations.Migration): dependencies = [ - ('pulp_2to3_migration', '0026_p2content_UQ'), + ("pulp_2to3_migration", "0026_p2content_UQ"), ] operations = [ migrations.AddIndex( - model_name='pulp2repocontent', - index=models.Index(fields=['pulp2_created'], name='pulp_2to3_m_pulp2_c_267ffa_idx'), + model_name="pulp2repocontent", + index=models.Index( + fields=["pulp2_created"], name="pulp_2to3_m_pulp2_c_267ffa_idx" + ), ), ] diff --git a/pulp_2to3_migration/app/migrations/0028_create_missing_indices.py b/pulp_2to3_migration/app/migrations/0028_create_missing_indices.py index d3bd13f0..b5165fd7 100644 --- a/pulp_2to3_migration/app/migrations/0028_create_missing_indices.py +++ b/pulp_2to3_migration/app/migrations/0028_create_missing_indices.py @@ -6,41 +6,57 @@ class Migration(migrations.Migration): dependencies = [ - ('pulp_2to3_migration', '0027_pulp2repocontent_created_index'), + ("pulp_2to3_migration", "0027_pulp2repocontent_created_index"), ] operations = [ migrations.AddIndex( - model_name='pulp2content', - index=models.Index(fields=['pulp2_last_updated'], name='pulp_2to3_m_pulp2_l_c8b0d6_idx'), + model_name="pulp2content", + index=models.Index( + fields=["pulp2_last_updated"], name="pulp_2to3_m_pulp2_l_c8b0d6_idx" + ), ), migrations.AddIndex( - model_name='pulp2distributor', - index=models.Index(fields=['pulp2_last_updated'], name='pulp_2to3_m_pulp2_l_2c5996_idx'), + model_name="pulp2distributor", + index=models.Index( + fields=["pulp2_last_updated"], name="pulp_2to3_m_pulp2_l_2c5996_idx" + ), ), migrations.AddIndex( - model_name='pulp2distributor', - index=models.Index(fields=['pulp2_repo_id'], name='pulp_2to3_m_pulp2_r_77bd32_idx'), + model_name="pulp2distributor", + index=models.Index( + fields=["pulp2_repo_id"], name="pulp_2to3_m_pulp2_r_77bd32_idx" + ), ), migrations.AddIndex( - model_name='pulp2importer', - index=models.Index(fields=['pulp2_last_updated'], name='pulp_2to3_m_pulp2_l_2bbfde_idx'), + model_name="pulp2importer", + index=models.Index( + fields=["pulp2_last_updated"], name="pulp_2to3_m_pulp2_l_2bbfde_idx" + ), ), migrations.AddIndex( - model_name='pulp2repository', - index=models.Index(fields=['pulp2_last_unit_added'], name='pulp_2to3_m_pulp2_l_270629_idx'), + model_name="pulp2repository", + index=models.Index( + fields=["pulp2_last_unit_added"], name="pulp_2to3_m_pulp2_l_270629_idx" + ), ), migrations.AddIndex( - model_name='pulp2repository', - index=models.Index(fields=['pulp2_last_unit_removed'], name='pulp_2to3_m_pulp2_l_2de33d_idx'), + model_name="pulp2repository", + index=models.Index( + fields=["pulp2_last_unit_removed"], + name="pulp_2to3_m_pulp2_l_2de33d_idx", + ), ), migrations.AddIndex( - model_name='pulp2repository', - index=models.Index(fields=['pulp2_repo_id'], name='pulp_2to3_m_pulp2_r_68b419_idx'), + model_name="pulp2repository", + index=models.Index( + fields=["pulp2_repo_id"], name="pulp_2to3_m_pulp2_r_68b419_idx" + ), ), migrations.AddIndex( - model_name='pulp2importer', - index=models.Index(fields=['pulp2_repo_id'], name='pulp_2to3_m_pulp2_r_46adaf_idx'), + model_name="pulp2importer", + index=models.Index( + fields=["pulp2_repo_id"], name="pulp_2to3_m_pulp2_r_46adaf_idx" + ), ), - ] diff --git a/pulp_2to3_migration/app/migrations/0029_distribution_change.py b/pulp_2to3_migration/app/migrations/0029_distribution_change.py index d4ce4054..16fb2f7e 100644 --- a/pulp_2to3_migration/app/migrations/0029_distribution_change.py +++ b/pulp_2to3_migration/app/migrations/0029_distribution_change.py @@ -6,15 +6,17 @@ _logger = logging.getLogger(__name__) + def unset_distribution_field(apps, schema_editor): - Pulp2Distributor = apps.get_model('pulp_2to3_migration', 'pulp2distributor') + Pulp2Distributor = apps.get_model("pulp_2to3_migration", "pulp2distributor") nrows = Pulp2Distributor.objects.update(pulp3_distribution=None, is_migrated=False) _logger.debug("Updated {} Pulp2Distributor rows.".format(nrows)) + class Migration(migrations.Migration): dependencies = [ - ('pulp_2to3_migration', '0028_create_missing_indices'), + ("pulp_2to3_migration", "0028_create_missing_indices"), ] operations = [ @@ -22,8 +24,12 @@ class Migration(migrations.Migration): code=unset_distribution_field, ), migrations.AlterField( - model_name='pulp2distributor', - name='pulp3_distribution', - field=models.OneToOneField(null=True, on_delete=django.db.models.deletion.SET_NULL, to='core.Distribution'), + model_name="pulp2distributor", + name="pulp3_distribution", + field=models.OneToOneField( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to="core.Distribution", + ), ), ] diff --git a/pulp_2to3_migration/app/migrations/0030_update_json_field.py b/pulp_2to3_migration/app/migrations/0030_update_json_field.py index aad4e5b1..48acdc9f 100644 --- a/pulp_2to3_migration/app/migrations/0030_update_json_field.py +++ b/pulp_2to3_migration/app/migrations/0030_update_json_field.py @@ -6,103 +6,103 @@ class Migration(migrations.Migration): dependencies = [ - ('pulp_2to3_migration', '0029_distribution_change'), + ("pulp_2to3_migration", "0029_distribution_change"), ] - operations = [ + operations = [ migrations.AlterField( - model_name='migrationplan', - name='plan', + model_name="migrationplan", + name="plan", field=models.JSONField(), ), migrations.AlterField( - model_name='pulp2distributor', - name='pulp2_config', + model_name="pulp2distributor", + name="pulp2_config", field=models.JSONField(), ), migrations.AlterField( - model_name='pulp2erratum', - name='pkglist', + model_name="pulp2erratum", + name="pkglist", field=models.JSONField(), ), migrations.AlterField( - model_name='pulp2erratum', - name='references', + model_name="pulp2erratum", + name="references", field=models.JSONField(), ), migrations.AlterField( - model_name='pulp2importer', - name='pulp2_config', + model_name="pulp2importer", + name="pulp2_config", field=models.JSONField(), ), migrations.AlterField( - model_name='pulp2modulemd', - name='artifacts', + model_name="pulp2modulemd", + name="artifacts", field=models.JSONField(), ), migrations.AlterField( - model_name='pulp2modulemd', - name='dependencies', + model_name="pulp2modulemd", + name="dependencies", field=models.JSONField(default=list), ), migrations.AlterField( - model_name='pulp2modulemddefaults', - name='profiles', + model_name="pulp2modulemddefaults", + name="profiles", field=models.JSONField(default=dict), ), migrations.AlterField( - model_name='pulp2packagecategory', - name='desc_by_lang', + model_name="pulp2packagecategory", + name="desc_by_lang", field=models.JSONField(default=dict), ), migrations.AlterField( - model_name='pulp2packagecategory', - name='name_by_lang', + model_name="pulp2packagecategory", + name="name_by_lang", field=models.JSONField(default=dict), ), migrations.AlterField( - model_name='pulp2packagecategory', - name='packagegroupids', + model_name="pulp2packagecategory", + name="packagegroupids", field=models.JSONField(), ), migrations.AlterField( - model_name='pulp2packageenvironment', - name='desc_by_lang', + model_name="pulp2packageenvironment", + name="desc_by_lang", field=models.JSONField(default=dict), ), migrations.AlterField( - model_name='pulp2packageenvironment', - name='group_ids', + model_name="pulp2packageenvironment", + name="group_ids", field=models.JSONField(), ), migrations.AlterField( - model_name='pulp2packageenvironment', - name='name_by_lang', + model_name="pulp2packageenvironment", + name="name_by_lang", field=models.JSONField(default=dict), ), migrations.AlterField( - model_name='pulp2packageenvironment', - name='option_ids', + model_name="pulp2packageenvironment", + name="option_ids", field=models.JSONField(), ), migrations.AlterField( - model_name='pulp2packagegroup', - name='desc_by_lang', + model_name="pulp2packagegroup", + name="desc_by_lang", field=models.JSONField(default=dict), ), migrations.AlterField( - model_name='pulp2packagegroup', - name='name_by_lang', + model_name="pulp2packagegroup", + name="name_by_lang", field=models.JSONField(default=dict), ), migrations.AlterField( - model_name='pulp2packagegroup', - name='packages', + model_name="pulp2packagegroup", + name="packages", field=models.JSONField(), ), migrations.AlterField( - model_name='pulp2packagelangpacks', - name='matches', + model_name="pulp2packagelangpacks", + name="matches", field=models.JSONField(), ), ] diff --git a/pulp_2to3_migration/app/migrations/0031_add_repoid_to_deb_types.py b/pulp_2to3_migration/app/migrations/0031_add_repoid_to_deb_types.py index 73b795ce..ea9bce66 100644 --- a/pulp_2to3_migration/app/migrations/0031_add_repoid_to_deb_types.py +++ b/pulp_2to3_migration/app/migrations/0031_add_repoid_to_deb_types.py @@ -6,48 +6,62 @@ class Migration(migrations.Migration): dependencies = [ - ('pulp_2to3_migration', '0030_update_json_field'), + ("pulp_2to3_migration", "0030_update_json_field"), ] operations = [ migrations.AddField( - model_name='pulp2debcomponent', - name='repoid', - field=models.TextField(default='blub'), + model_name="pulp2debcomponent", + name="repoid", + field=models.TextField(default="blub"), preserve_default=False, ), migrations.AddField( - model_name='pulp2debrelease', - name='repoid', - field=models.TextField(default='blub'), + model_name="pulp2debrelease", + name="repoid", + field=models.TextField(default="blub"), preserve_default=False, ), migrations.AddField( - model_name='pulp2debcomponentpackage', - name='repoid', - field=models.TextField(default='blub'), + model_name="pulp2debcomponentpackage", + name="repoid", + field=models.TextField(default="blub"), preserve_default=False, ), migrations.AddField( - model_name='pulp2debreleasearchitecture', - name='repoid', - field=models.TextField(default='blub'), + model_name="pulp2debreleasearchitecture", + name="repoid", + field=models.TextField(default="blub"), preserve_default=False, ), migrations.AlterUniqueTogether( - name='pulp2debcomponent', - unique_together={('component', 'codename', 'suite', 'distribution', 'repoid')}, + name="pulp2debcomponent", + unique_together={ + ("component", "codename", "suite", "distribution", "repoid") + }, ), migrations.AlterUniqueTogether( - name='pulp2debrelease', - unique_together={('codename', 'suite', 'distribution', 'repoid')}, + name="pulp2debrelease", + unique_together={("codename", "suite", "distribution", "repoid")}, ), migrations.AlterUniqueTogether( - name='pulp2debcomponentpackage', - unique_together={('package_relative_path', 'package_sha256', 'component', 'distribution', 'codename', 'suite', 'repoid')}, + name="pulp2debcomponentpackage", + unique_together={ + ( + "package_relative_path", + "package_sha256", + "component", + "distribution", + "codename", + "suite", + "repoid", + ) + }, ), migrations.AlterUniqueTogether( - name='pulp2debreleasearchitecture', - unique_together={('architecture', 'distribution', 'codename', 'suite', 'repoid')}, + name="pulp2debreleasearchitecture", + unique_together={ + ("architecture", "distribution", "codename", "suite", "repoid") + }, ), ] diff --git a/pulp_2to3_migration/app/models/base.py b/pulp_2to3_migration/app/models/base.py index 1e81b5ac..26892def 100644 --- a/pulp_2to3_migration/app/models/base.py +++ b/pulp_2to3_migration/app/models/base.py @@ -66,10 +66,14 @@ def get_repo_types(plan): # - This is a big query, paginate? # - Filter by repos from the plan # - Query any but one record for a repo - for rec in RepositoryContentUnit.objects().\ - only('repo_id', 'unit_type_id').as_pymongo().no_cache(): - repo_id = rec['repo_id'] - unit_type_id = rec['unit_type_id'] + for rec in ( + RepositoryContentUnit.objects() + .only("repo_id", "unit_type_id") + .as_pymongo() + .no_cache() + ): + repo_id = rec["repo_id"] + unit_type_id = rec["unit_type_id"] # a type for a repo is already known or this content/repo type is not supported if repo_id in repo_id_to_type or unit_type_id not in content_type_to_plugin: @@ -88,6 +92,7 @@ class MigrationPlan(BaseModel): Fields: plan (models.JSONField): The migration plan in the JSON format """ + plan = models.JSONField() _real_plan = None @@ -114,12 +119,14 @@ def plan_view(self): # exists in pulp 2. This is really tricky and a little messy, because it needs # to happen after the migration plan has been parsed. repository_ids = self.type_to_repo_ids[plugin_plan.type] - repositories = Repository.objects().filter( - repo_id__in=repository_ids - ).only("repo_id") + repositories = ( + Repository.objects() + .filter(repo_id__in=repository_ids) + .only("repo_id") + ) for repository in repositories.as_pymongo().no_cache(): - repo_id = repository['repo_id'] + repo_id = repository["repo_id"] plugin_plan.repositories_to_create[repo_id] = { "pulp2_importer_repository_id": repo_id, "repository_versions": [ @@ -127,13 +134,15 @@ def plan_view(self): "repo_id": repo_id, "dist_repo_ids": [repo_id], } - ] + ], } plugin_plan.repositories_importers_to_migrate.append(repo_id) plugin_plan.repositories_to_migrate.append(repo_id) plugin_plan.repositories_distributors_to_migrate.append(repo_id) - RepoSetup.set_importer(repo_id, plugin_plan.type, importer_repo_id=repo_id) + RepoSetup.set_importer( + repo_id, plugin_plan.type, importer_repo_id=repo_id + ) RepoSetup.set_distributors( repo_id, plugin_plan.type, distributor_repo_ids=[repo_id] ) @@ -162,13 +171,15 @@ def get_missing_resources(self): """ ret = {} if self.plan_view.missing_repositories: - ret['repositories'] = self.plan_view.missing_repositories + ret["repositories"] = self.plan_view.missing_repositories if self.plan_view.repositories_missing_importers: - ret['repositories_missing_importers'] = \ - self.plan_view.repositories_missing_importers + ret[ + "repositories_missing_importers" + ] = self.plan_view.repositories_missing_importers if self.plan_view.repositories_missing_distributors: - ret['repositories_missing_distributors'] = \ - self.plan_view.repositories_missing_distributors + ret[ + "repositories_missing_distributors" + ] = self.plan_view.repositories_missing_distributors return ret @@ -176,7 +187,7 @@ class _InternalMigrationPlan: def __init__(self, migration_plan): self._plugin_plans = [] - for plugin_data in migration_plan.plan['plugins']: + for plugin_data in migration_plan.plan["plugins"]: self._plugin_plans.append(PluginMigrationPlan(plugin_data)) self.repositories_missing_importers = [] @@ -190,41 +201,56 @@ def __init__(self, migration_plan): @property def all_repositories_importers_to_migrate(self): # flat list of all importers to migrate - return list(itertools.chain.from_iterable( - [plugin.repositories_importers_to_migrate for plugin in self._plugin_plans] - )) + return list( + itertools.chain.from_iterable( + [ + plugin.repositories_importers_to_migrate + for plugin in self._plugin_plans + ] + ) + ) @property def all_repositories_to_migrate(self): # flat list of all repositories to migrate - return list(itertools.chain.from_iterable( - [plugin.repositories_to_migrate for plugin in self._plugin_plans] - )) + return list( + itertools.chain.from_iterable( + [plugin.repositories_to_migrate for plugin in self._plugin_plans] + ) + ) @property def all_repositories_distributors_to_migrate(self): # flat list of all distributors to migrate - return list(itertools.chain.from_iterable( - [plugin.repositories_distributors_to_migrate for plugin in self._plugin_plans] - )) + return list( + itertools.chain.from_iterable( + [ + plugin.repositories_distributors_to_migrate + for plugin in self._plugin_plans + ] + ) + ) def _check_missing(self): importers = Importer.objects( - repo_id__in=self.all_repositories_importers_to_migrate).only('repo_id') + repo_id__in=self.all_repositories_importers_to_migrate + ).only("repo_id") present = set(importer.repo_id for importer in importers) expected = set(self.all_repositories_importers_to_migrate) self.repositories_missing_importers = list(expected - present) repositories = Repository.objects( - repo_id__in=self.all_repositories_to_migrate).only('repo_id') + repo_id__in=self.all_repositories_to_migrate + ).only("repo_id") present = set(repository.repo_id for repository in repositories) expected = set(self.all_repositories_to_migrate) self.missing_repositories = list(expected - present) distributors = Distributor.objects( - repo_id__in=self.all_repositories_distributors_to_migrate).only('repo_id') + repo_id__in=self.all_repositories_distributors_to_migrate + ).only("repo_id") present = set(distributor.repo_id for distributor in distributors) expected = set(self.all_repositories_distributors_to_migrate) @@ -309,37 +335,46 @@ def _parse_plugin_plan(self, repository_data): # Circular import avoidance from pulp_2to3_migration.app.plugin import PLUGIN_MIGRATORS - self.type = repository_data['type'] + self.type = repository_data["type"] self.migrator = PLUGIN_MIGRATORS.get(self.type) - repositories = repository_data.get('repositories') + repositories = repository_data.get("repositories") if repositories: self.empty = False for repository in repositories: - name = repository['name'] + name = repository["name"] - importer_repo_id = repository.get('pulp2_importer_repository_id') + importer_repo_id = repository.get("pulp2_importer_repository_id") if importer_repo_id: self.repositories_importers_to_migrate.append(importer_repo_id) repository_versions = [] - for repository_version in repository.get('repository_versions', []): - pulp2_repository_id = repository_version['pulp2_repository_id'] + for repository_version in repository.get("repository_versions", []): + pulp2_repository_id = repository_version["pulp2_repository_id"] self.repositories_to_migrate.append(pulp2_repository_id) distributor_repo_ids = repository_version.get( - 'pulp2_distributor_repository_ids', [] + "pulp2_distributor_repository_ids", [] + ) + self.repositories_distributors_to_migrate.extend( + distributor_repo_ids ) - self.repositories_distributors_to_migrate.extend(distributor_repo_ids) repository_versions.append( - {'repo_id': pulp2_repository_id, 'dist_repo_ids': distributor_repo_ids} + { + "repo_id": pulp2_repository_id, + "dist_repo_ids": distributor_repo_ids, + } ) signing_service = repository.get("signing_service") - RepoSetup.set_importer(pulp2_repository_id, self.type, importer_repo_id) - RepoSetup.set_distributors(pulp2_repository_id, self.type, distributor_repo_ids) + RepoSetup.set_importer( + pulp2_repository_id, self.type, importer_repo_id + ) + RepoSetup.set_distributors( + pulp2_repository_id, self.type, distributor_repo_ids + ) self.repositories_to_create[name] = { "pulp2_importer_repository_id": importer_repo_id, @@ -361,21 +396,19 @@ class RepoSetup(BaseModel): status (models.SmallIntegerField): status of the record """ + OLD = 0 UP_TO_DATE = 1 NEW = 2 STATUS_CHOICES = ( - (OLD, 'old'), - (UP_TO_DATE, 'up to date'), - (NEW, 'new'), + (OLD, "old"), + (UP_TO_DATE, "up to date"), + (NEW, "new"), ) IMPORTER = 0 DISTRIBUTOR = 1 - RESOURCE_TYPE_CHOICES = ( - (IMPORTER, 'importer'), - (DISTRIBUTOR, 'distributor') - ) + RESOURCE_TYPE_CHOICES = ((IMPORTER, "importer"), (DISTRIBUTOR, "distributor")) pulp2_repo_id = models.TextField() pulp2_repo_type = models.CharField(max_length=25) @@ -384,10 +417,14 @@ class RepoSetup(BaseModel): status = models.SmallIntegerField(choices=STATUS_CHOICES) class Meta: - unique_together = ('pulp2_repo_id', 'pulp2_resource_repo_id', 'pulp2_resource_type') + unique_together = ( + "pulp2_repo_id", + "pulp2_resource_repo_id", + "pulp2_resource_type", + ) indexes = [ - models.Index(fields=['pulp2_resource_type']), - models.Index(fields=['status']) + models.Index(fields=["pulp2_resource_type"]), + models.Index(fields=["status"]), ] @classmethod @@ -440,8 +477,8 @@ def set_importer(cls, repo_id, repo_type, importer_repo_id): pulp2_resource_type=cls.IMPORTER, pulp2_repo_type=repo_type, pulp2_repo_id=repo_id, - pulp2_resource_repo_id=importer_repo_id or '', - defaults={'status': cls.NEW} + pulp2_resource_repo_id=importer_repo_id or "", + defaults={"status": cls.NEW}, ) is_unchanged_relation = not created and relation.status == cls.OLD @@ -470,7 +507,7 @@ def set_distributors(cls, repo_id, repo_type, distributor_repo_ids): pulp2_repo_id=repo_id, pulp2_resource_type=cls.DISTRIBUTOR, pulp2_resource_repo_id__in=distributor_repo_ids, - status=cls.OLD + status=cls.OLD, ).update(status=cls.UP_TO_DATE) no_new_relations = up_to_date_count == len(distributor_repo_ids) @@ -484,7 +521,7 @@ def set_distributors(cls, repo_id, repo_type, distributor_repo_ids): pulp2_resource_type=cls.DISTRIBUTOR, pulp2_repo_type=repo_type, pulp2_repo_id=repo_id, - pulp2_resource_repo_id=distributor_repo_id or '', + pulp2_resource_repo_id=distributor_repo_id or "", status=cls.NEW, ) except IntegrityError: @@ -499,9 +536,12 @@ def mark_changed_relations(cls, plugins): Args: plugins(list): List of plugin names specified in the Migration Plan """ - changed_relations_repo_ids = RepoSetup.objects.filter(pulp2_repo_type__in=plugins).exclude( - status=cls.UP_TO_DATE - ).only('pulp2_repo_id').values_list('pulp2_repo_id', flat=True) + changed_relations_repo_ids = ( + RepoSetup.objects.filter(pulp2_repo_type__in=plugins) + .exclude(status=cls.UP_TO_DATE) + .only("pulp2_repo_id") + .values_list("pulp2_repo_id", flat=True) + ) Pulp2Repository.objects.filter( pulp2_repo_id__in=changed_relations_repo_ids ).update(is_migrated=False) diff --git a/pulp_2to3_migration/app/models/content.py b/pulp_2to3_migration/app/models/content.py index c73d7687..d13f1daf 100644 --- a/pulp_2to3_migration/app/models/content.py +++ b/pulp_2to3_migration/app/models/content.py @@ -2,7 +2,9 @@ from django.db.models import Q from django.db.models.constraints import UniqueConstraint -from pulpcore.app.models import Content # it has to be imported directly from pulpcore see #5353 +from pulpcore.app.models import ( + Content, +) # it has to be imported directly from pulpcore see #5353 from pulpcore.plugin.models import BaseModel from .repository import Pulp2Repository @@ -28,28 +30,38 @@ class Pulp2Content(BaseModel): corresponds to N content units n Pulp 3. """ + pulp2_id = models.CharField(max_length=255) pulp2_content_type_id = models.CharField(max_length=255) pulp2_last_updated = models.PositiveIntegerField() pulp2_storage_path = models.TextField(null=True) downloaded = models.BooleanField(default=False) pulp3_content = models.ForeignKey(Content, on_delete=models.SET_NULL, null=True) - pulp2_repo = models.ForeignKey(Pulp2Repository, on_delete=models.SET_NULL, null=True) + pulp2_repo = models.ForeignKey( + Pulp2Repository, on_delete=models.SET_NULL, null=True + ) pulp2_subid = models.CharField(max_length=255, blank=True) class Meta: constraints = [ UniqueConstraint( - fields=['pulp2_id', 'pulp2_content_type_id', 'pulp2_repo', 'pulp2_subid'], - name='unique_with_optional' + fields=[ + "pulp2_id", + "pulp2_content_type_id", + "pulp2_repo", + "pulp2_subid", + ], + name="unique_with_optional", + ), + UniqueConstraint( + fields=["pulp2_id", "pulp2_content_type_id", "pulp2_subid"], + condition=Q(pulp2_repo=None), + name="unique_without_optional", ), - UniqueConstraint(fields=['pulp2_id', 'pulp2_content_type_id', 'pulp2_subid'], - condition=Q(pulp2_repo=None), - name='unique_without_optional'), ] indexes = [ - models.Index(fields=['pulp2_content_type_id']), - models.Index(fields=['pulp2_last_updated']), + models.Index(fields=["pulp2_content_type_id"]), + models.Index(fields=["pulp2_last_updated"]), ] @@ -66,9 +78,10 @@ class Pulp2to3Content(BaseModel): pulp2content (models.ForeignKey): pulp 2 content this pre-migrated content corresponds to """ + pulp2content = models.ForeignKey(Pulp2Content, on_delete=models.CASCADE) - pulp2_type = '' + pulp2_type = "" set_pulp2_repo = False class Meta: @@ -118,6 +131,7 @@ class Pulp2LazyCatalog(BaseModel): pulp2_revision (models.IntegerField): A revision of the entry for the specific pulp2_storage_path and pulp2_importer_id """ + pulp2_importer_id = models.CharField(max_length=255) pulp2_unit_id = models.CharField(max_length=255) pulp2_content_type_id = models.CharField(max_length=255) @@ -127,8 +141,8 @@ class Pulp2LazyCatalog(BaseModel): is_migrated = models.BooleanField(default=False) class Meta: - unique_together = ('pulp2_storage_path', 'pulp2_importer_id', 'pulp2_revision') + unique_together = ("pulp2_storage_path", "pulp2_importer_id", "pulp2_revision") indexes = [ - models.Index(fields=['pulp2_unit_id']), - models.Index(fields=['pulp2_content_type_id']) + models.Index(fields=["pulp2_unit_id"]), + models.Index(fields=["pulp2_content_type_id"]), ] diff --git a/pulp_2to3_migration/app/models/repository.py b/pulp_2to3_migration/app/models/repository.py index 24a8a806..6ae4f9b5 100644 --- a/pulp_2to3_migration/app/models/repository.py +++ b/pulp_2to3_migration/app/models/repository.py @@ -34,6 +34,7 @@ class Pulp2Repository(BaseModel): repository was migrated to pulp3_repository_remote (models.ForeignKey): Pulp 3 remote to use with the migrated Pulp2 """ + pulp2_object_id = models.CharField(max_length=255, unique=True) pulp2_repo_id = models.TextField() pulp2_description = models.TextField(null=True) @@ -47,26 +48,26 @@ class Pulp2Repository(BaseModel): # a migration run and then recreated with exactly the same name and content in pulp 2, # and then migration was run again. In Pulp 3 it's the same repo and repo version which # should point to a new pulp2repository object. - pulp3_repository_version = models.ForeignKey(RepositoryVersion, - on_delete=models.SET_NULL, - null=True) + pulp3_repository_version = models.ForeignKey( + RepositoryVersion, on_delete=models.SET_NULL, null=True + ) # The same importer/remote can be used for multiple repositories, thus it's a foreign key. - pulp3_repository_remote = models.ForeignKey(Remote, - on_delete=models.SET_NULL, - null=True) + pulp3_repository_remote = models.ForeignKey( + Remote, on_delete=models.SET_NULL, null=True + ) # This is needed for migrating Variants of the DistributionTree - pulp3_repository = models.ForeignKey(Repository, - on_delete=models.SET_NULL, - null=True) + pulp3_repository = models.ForeignKey( + Repository, on_delete=models.SET_NULL, null=True + ) class Meta: indexes = [ - models.Index(fields=['pulp2_repo_type']), - models.Index(fields=['pulp2_last_unit_added']), - models.Index(fields=['pulp2_last_unit_removed']), - models.Index(fields=['pulp2_repo_id']), + models.Index(fields=["pulp2_repo_type"]), + models.Index(fields=["pulp2_last_unit_added"]), + models.Index(fields=["pulp2_last_unit_removed"]), + models.Index(fields=["pulp2_repo_id"]), ] @@ -85,6 +86,7 @@ class Pulp2RepoContent(BaseModel): Relations: pulp2_repository (models.ForeignKey): Pulp 2 repository this content belongs to """ + pulp2_unit_id = models.CharField(max_length=255) pulp2_content_type_id = models.CharField(max_length=255) pulp2_created = models.DateTimeField(null=True) @@ -93,10 +95,10 @@ class Pulp2RepoContent(BaseModel): pulp2_repository = models.ForeignKey(Pulp2Repository, on_delete=models.CASCADE) class Meta: - unique_together = ('pulp2_repository', 'pulp2_unit_id') + unique_together = ("pulp2_repository", "pulp2_unit_id") indexes = [ - models.Index(fields=['pulp2_content_type_id']), - models.Index(fields=['pulp2_created']) + models.Index(fields=["pulp2_content_type_id"]), + models.Index(fields=["pulp2_created"]), ] @@ -117,6 +119,7 @@ class Pulp2Importer(BaseModel): Relations: pulp3_remote (models.OneToOneField): Pulp 3 remote which this importer was migrated to """ + pulp2_object_id = models.CharField(max_length=255, unique=True) pulp2_type_id = models.CharField(max_length=255) pulp2_config = models.JSONField() @@ -129,9 +132,9 @@ class Pulp2Importer(BaseModel): class Meta: indexes = [ - models.Index(fields=['pulp2_type_id']), - models.Index(fields=['pulp2_last_updated']), - models.Index(fields=['pulp2_repo_id']), + models.Index(fields=["pulp2_type_id"]), + models.Index(fields=["pulp2_last_updated"]), + models.Index(fields=["pulp2_repo_id"]), ] @@ -157,6 +160,7 @@ class Pulp2Distributor(BaseModel): pulp3_distribution (models.OneToOneField): Pulp 3 distribution this distributor was migrated to """ + pulp2_object_id = models.CharField(max_length=255, unique=True) pulp2_id = models.TextField() pulp2_type_id = models.CharField(max_length=255) @@ -167,21 +171,23 @@ class Pulp2Distributor(BaseModel): not_in_plan = models.BooleanField(default=False) # each pulp2 repository can have multiple distributors - pulp2_repos = models.ManyToManyField(Pulp2Repository, related_name='pulp2_dists') + pulp2_repos = models.ManyToManyField(Pulp2Repository, related_name="pulp2_dists") # the same publication/repo version can be published by multiple distributors - pulp3_publication = models.ForeignKey(Publication, on_delete=models.SET_NULL, null=True) + pulp3_publication = models.ForeignKey( + Publication, on_delete=models.SET_NULL, null=True + ) # due to base_path overlap restriction, a distribution can't correspond to multiple pulp 2 # distributors, thus one-to-one relationship. - pulp3_distribution = models.OneToOneField(Distribution, - on_delete=models.SET_NULL, - null=True) + pulp3_distribution = models.OneToOneField( + Distribution, on_delete=models.SET_NULL, null=True + ) class Meta: - unique_together = ('pulp2_object_id',) + unique_together = ("pulp2_object_id",) indexes = [ - models.Index(fields=['pulp2_type_id']), - models.Index(fields=['pulp2_last_updated']), - models.Index(fields=['pulp2_repo_id']), + models.Index(fields=["pulp2_type_id"]), + models.Index(fields=["pulp2_last_updated"]), + models.Index(fields=["pulp2_repo_id"]), ] diff --git a/pulp_2to3_migration/app/plugin/__init__.py b/pulp_2to3_migration/app/plugin/__init__.py index ba0a2087..0150d077 100644 --- a/pulp_2to3_migration/app/plugin/__init__.py +++ b/pulp_2to3_migration/app/plugin/__init__.py @@ -8,16 +8,19 @@ # { plugin_name: PluginMigratorClass } PLUGIN_MIGRATORS = {} -MissingMigrator = namedtuple('MissingMigrator', 'pulp3_plugin') +MissingMigrator = namedtuple("MissingMigrator", "pulp3_plugin") if not PLUGIN_MIGRATORS: - for entry_point in pkg_resources.iter_entry_points(group='migrators'): + for entry_point in pkg_resources.iter_entry_points(group="migrators"): try: PLUGIN_MIGRATORS[entry_point.name] = entry_point.load() except ModuleNotFoundError as exc: - _logger.info(_( - 'Plugin %s is not installed in pulp3 ' - 'therefore it will not be migrated from pulp2') % exc.name + _logger.info( + _( + "Plugin %s is not installed in pulp3 " + "therefore it will not be migrated from pulp2" + ) + % exc.name ) missing = MissingMigrator(exc.msg) PLUGIN_MIGRATORS[entry_point.name] = missing diff --git a/pulp_2to3_migration/app/plugin/api.py b/pulp_2to3_migration/app/plugin/api.py index 9e4caa7d..aeb8801e 100644 --- a/pulp_2to3_migration/app/plugin/api.py +++ b/pulp_2to3_migration/app/plugin/api.py @@ -9,5 +9,5 @@ from .repository import ( # noqa is_different_relative_url, Pulp2to3Importer, - Pulp2to3Distributor + Pulp2to3Distributor, ) diff --git a/pulp_2to3_migration/app/plugin/content.py b/pulp_2to3_migration/app/plugin/content.py index a525ac18..ad572313 100644 --- a/pulp_2to3_migration/app/plugin/content.py +++ b/pulp_2to3_migration/app/plugin/content.py @@ -78,7 +78,7 @@ def pipeline_stages(self): ContentSaver(), RemoteArtifactSaver(), UpdateLCEs(), - RelatePulp2to3Content() + RelatePulp2to3Content(), ] return pipeline @@ -110,8 +110,13 @@ def __init__(self, migrator, skip_corrupted=False): self.migrator = migrator self.skip_corrupted = skip_corrupted - async def create_artifact(self, pulp2_storage_path, expected_digests={}, expected_size=None, - downloaded=True): + async def create_artifact( + self, + pulp2_storage_path, + expected_digests={}, + expected_size=None, + downloaded=True, + ): """ Create a hard link if possible and then create an Artifact. @@ -120,28 +125,40 @@ async def create_artifact(self, pulp2_storage_path, expected_digests={}, expecte """ if not downloaded: if not expected_digests: - raise ValueError(_('No digest is provided for on_demand content creation. Pulp 2 ' - 'storage path: {}'.format(pulp2_storage_path))) + raise ValueError( + _( + "No digest is provided for on_demand content creation. Pulp 2 " + "storage path: {}".format(pulp2_storage_path) + ) + ) artifact = Artifact(**expected_digests) artifact.size = expected_size return artifact try: - artifact = Artifact.init_and_validate(pulp2_storage_path, - expected_digests=expected_digests, - expected_size=expected_size) + artifact = Artifact.init_and_validate( + pulp2_storage_path, + expected_digests=expected_digests, + expected_size=expected_size, + ) except (DigestValidationError, FileNotFoundError, SizeValidationError): if self.skip_corrupted: - _logger.warn(f'The content located in {pulp2_storage_path} is missing or ' - f'corrupted. It was skipped during Pulp 2to3 migration.') + _logger.warn( + f"The content located in {pulp2_storage_path} is missing or " + f"corrupted. It was skipped during Pulp 2to3 migration." + ) return - raise ArtifactValidationError(f'The content located in {pulp2_storage_path} is ' - f'missing or corrupted. Repair it in pulp2 and re-run ' - f'the migration. Alternatively, run migration with ' - f'skip_corrupted=True.') + raise ArtifactValidationError( + f"The content located in {pulp2_storage_path} is " + f"missing or corrupted. Repair it in pulp2 and re-run " + f"the migration. Alternatively, run migration with " + f"skip_corrupted=True." + ) pulp3_storage_relative_path = storage.get_artifact_path(artifact.sha256) - pulp3_storage_path = os.path.join(settings.MEDIA_ROOT, pulp3_storage_relative_path) + pulp3_storage_path = os.path.join( + settings.MEDIA_ROOT, pulp3_storage_relative_path + ) os.makedirs(os.path.dirname(pulp3_storage_path), exist_ok=True) is_copied = False @@ -150,18 +167,20 @@ async def create_artifact(self, pulp2_storage_path, expected_digests={}, expecte except FileExistsError: pass except OSError: - _logger.debug(_('Hard link cannot be created, file will be copied.')) + _logger.debug(_("Hard link cannot be created, file will be copied.")) shutil.copy2(pulp2_storage_path, pulp3_storage_path) is_copied = True if not expected_digests: - expected_digests = {'sha256': artifact.sha256} + expected_digests = {"sha256": artifact.sha256} if is_copied: # recalculate checksums to ensure that after being copied a file is still fine - artifact = Artifact.init_and_validate(file=pulp3_storage_path, - expected_digests=expected_digests, - expected_size=expected_size) + artifact = Artifact.init_and_validate( + file=pulp3_storage_path, + expected_digests=expected_digests, + expected_size=expected_size, + ) else: # a hard link has been created or a file has already been in the pulp 3 storage, so # artifact's path can be just updated and no checksum recalculation is needed. @@ -181,9 +200,7 @@ async def run(self): # We are waiting on the coroutine to finish, because the order of the processed # content for plugins like Container and RPM is important because of the relations # between the content types. - await asyncio.gather( - self.migrate_to_pulp3(cmodel, ctype) - ) + await asyncio.gather(self.migrate_to_pulp3(cmodel, ctype)) async def migrate_to_pulp3(self, content_model, content_type): """ @@ -198,6 +215,7 @@ async def migrate_to_pulp3(self, content_model, content_type): migrator: A plugin migrator to be used content_type: type of pulp2 content that is being mirated """ + @functools.lru_cache(maxsize=20) def get_remote_by_importer_id(importer_id): """ @@ -223,35 +241,42 @@ def get_remote_by_importer_id(importer_id): if is_lazy_type: # go through all of the content that haven't been migrated OR have been migrated # but have new lazy catalog entries. - units_with_new_lces = Pulp2LazyCatalog.objects.filter( - is_migrated=False).values('pulp2_unit_id').distinct() + units_with_new_lces = ( + Pulp2LazyCatalog.objects.filter(is_migrated=False) + .values("pulp2_unit_id") + .distinct() + ) already_migrated = ~Q(pulp2content__pulp3_content=None) no_new_lces = ~Q(pulp2content__pulp2_id__in=units_with_new_lces) - pulp_2to3_detail_qs = content_model.objects.exclude(already_migrated & no_new_lces) + pulp_2to3_detail_qs = content_model.objects.exclude( + already_migrated & no_new_lces + ) else: # go through all of the content that haven't been migrated - pulp_2to3_detail_qs = content_model.objects.filter(pulp2content__pulp3_content=None) + pulp_2to3_detail_qs = content_model.objects.filter( + pulp2content__pulp3_content=None + ) # order by pulp2_repo if it's set if content_model.set_pulp2_repo: - pulp_2to3_detail_qs = pulp_2to3_detail_qs.order_by('repo_id') + pulp_2to3_detail_qs = pulp_2to3_detail_qs.order_by("repo_id") async with ProgressReport( - message='Migrating {} content to Pulp 3'.format(content_type), - code='migrating.{}.content'.format(self.migrator.pulp2_plugin), - total=await sync_to_async(pulp_2to3_detail_qs.count)() + message="Migrating {} content to Pulp 3".format(content_type), + code="migrating.{}.content".format(self.migrator.pulp2_plugin), + total=await sync_to_async(pulp_2to3_detail_qs.count)(), ) as pb: select_extra = [ - 'pulp2content', - 'pulp2content__pulp3_content', + "pulp2content", + "pulp2content__pulp3_content", ] if content_model.set_pulp2_repo: - select_extra.append('pulp2content__pulp2_repo') + select_extra.append("pulp2content__pulp2_repo") pulp_2to3_detail_qs = pulp_2to3_detail_qs.select_related(*select_extra) async for pulp_2to3_detail_content in sync_to_async_iterable( - pulp_2to3_detail_qs.iterator(chunk_size=800) + pulp_2to3_detail_qs.iterator(chunk_size=800) ): dc = None pulp2content = await sync_to_async(Pulp2Content.objects.get)( @@ -265,19 +290,29 @@ def get_remote_by_importer_id(importer_id): pulp2_unit_id=pulp2content.pulp2_id, is_migrated=False, ) - await sync_to_async(bool)(pulp2lazycatalog) # force queryset to evaluate + await sync_to_async(bool)( + pulp2lazycatalog + ) # force queryset to evaluate if not pulp2content.downloaded and not pulp2lazycatalog: # A distribution tree can be from an on_demand repo but without any images, # e.g. CentOS 8 High Availability. Do not skip in that case. if not is_multi_artifact: - _logger.warn(_( - 'On_demand content cannot be migrated without an entry in the ' - 'lazy catalog, pulp2 unit_id: {}'.format(pulp2content.pulp2_id)) + _logger.warn( + _( + "On_demand content cannot be migrated without an entry in the " + "lazy catalog, pulp2 unit_id: {}".format( + pulp2content.pulp2_id + ) + ) ) continue - if pulp2content.pulp3_content is not None and is_lazy_type and pulp2lazycatalog: + if ( + pulp2content.pulp3_content is not None + and is_lazy_type + and pulp2lazycatalog + ): # find already created pulp3 content pulp3content = pulp2content.pulp3_content extra_info = None @@ -288,7 +323,9 @@ def get_remote_by_importer_id(importer_id): _logger.warning( _( "Failed to find or instantiate extra_info for multi-artifact " - "pulp2 unit_id: {} ; skipping".format(pulp2content.pulp2_id) + "pulp2 unit_id: {} ; skipping".format( + pulp2content.pulp2_id + ) ) ) continue @@ -308,7 +345,7 @@ def get_remote_by_importer_id(importer_id): ) continue - future_relations = {'pulp2content': pulp2content} + future_relations = {"pulp2content": pulp2content} if extra_info: future_relations.update(extra_info) @@ -319,15 +356,14 @@ def get_remote_by_importer_id(importer_id): missing_artifact = False remote_declarative_artifacts = [] - for image_relative_path in extra_info['download']['images']: + for image_relative_path in extra_info["download"]["images"]: remote_url_tuples = [] image_path = os.path.join(base_path, image_relative_path) downloaded = os.path.exists(image_path) if downloaded: - artifact = await self.create_artifact(image_path, - None, - None, - downloaded=downloaded) + artifact = await self.create_artifact( + image_path, None, None, downloaded=downloaded + ) if artifact is None: continue else: @@ -355,7 +391,8 @@ def get_remote_by_importer_id(importer_id): url=lce.pulp2_url, relative_path=image_relative_path, remote=remote, - deferred_download=not downloaded) + deferred_download=not downloaded, + ) remote_declarative_artifacts.append(da) if not remote_url_tuples: @@ -371,7 +408,8 @@ def get_remote_by_importer_id(importer_id): url=NOT_USED, relative_path=image_relative_path, remote=None, - deferred_download=False) + deferred_download=False, + ) d_artifacts.append(da) d_artifacts.extend(remote_declarative_artifacts) @@ -379,23 +417,30 @@ def get_remote_by_importer_id(importer_id): # Only skip the rest of the steps if there are any images that are expected # to be downloaded. There are distribution trees without images in the wild, # e.g. CentOS 8 High Availability. - if missing_artifact and extra_info['download']['images']: - _logger.warn(_( - 'On_demand content cannot be migrated without a remote ' - 'pulp2 unit_id: {}'.format(pulp2content.pulp2_id)) + if missing_artifact and extra_info["download"]["images"]: + _logger.warn( + _( + "On_demand content cannot be migrated without a remote " + "pulp2 unit_id: {}".format(pulp2content.pulp2_id) + ) ) continue for lce in pulp2lazycatalog: lce.is_migrated = True - future_relations.update({'lces': list(pulp2lazycatalog)}) + future_relations.update({"lces": list(pulp2lazycatalog)}) # We do this last because we need the remote url which is only found in the LCE # of the image files. There is no LCE for the .treeinfo file itself. - relative_path = pulp_2to3_detail_content.relative_path_for_content_artifact - treeinfo_path = os.path.join(pulp2content.pulp2_storage_path, relative_path) + relative_path = ( + pulp_2to3_detail_content.relative_path_for_content_artifact + ) + treeinfo_path = os.path.join( + pulp2content.pulp2_storage_path, relative_path + ) artifact = await self.create_artifact( - treeinfo_path, None, None, downloaded=True) + treeinfo_path, None, None, downloaded=True + ) if artifact is None: continue if remotes: @@ -417,7 +462,9 @@ def get_remote_by_importer_id(importer_id): deferred_download=False, ) d_artifacts.append(da) - dc = DeclarativeContent(content=pulp3content, d_artifacts=d_artifacts) + dc = DeclarativeContent( + content=pulp3content, d_artifacts=d_artifacts + ) dc.extra_data = future_relations await self.put(dc) # not all content units have files, create DC without artifact @@ -433,7 +480,7 @@ def get_remote_by_importer_id(importer_id): pulp2content.pulp2_storage_path, pulp_2to3_detail_content.expected_digests, pulp_2to3_detail_content.expected_size, - downloaded=pulp2content.downloaded + downloaded=pulp2content.downloaded, ) if artifact is None: if pb: @@ -468,16 +515,19 @@ def get_remote_by_importer_id(importer_id): url=lce.pulp2_url, relative_path=relative_path, remote=remote, - deferred_download=deferred_download) + deferred_download=deferred_download, + ) lce.is_migrated = True - dc = DeclarativeContent(content=pulp3content, d_artifacts=[da]) + dc = DeclarativeContent( + content=pulp3content, d_artifacts=[da] + ) # yes, all LCEs are assigned for each dc to be resolved at a later # stage. Some LCEs might be "bad" and not have a migrated importer # but we still need to resolved such. It creates some duplicated LCEs # to process later but ensures that all are resolved if at least one # valid one is migrated. - future_relations.update({'lces': list(pulp2lazycatalog)}) + future_relations.update({"lces": list(pulp2lazycatalog)}) dc.extra_data = future_relations await self.put(dc) @@ -485,9 +535,10 @@ def get_remote_by_importer_id(importer_id): # No migratable LCE available if deferred_download: _logger.warn( - _('On_demand content cannot be migrated without a remote ' - 'pulp2 unit_id: {}'.format(pulp2content.pulp2_id) - ) + _( + "On_demand content cannot be migrated without a remote " + "pulp2 unit_id: {}".format(pulp2content.pulp2_id) + ) ) continue @@ -496,7 +547,8 @@ def get_remote_by_importer_id(importer_id): url=NOT_USED, relative_path=relative_path, remote=None, - deferred_download=False) + deferred_download=False, + ) dc = DeclarativeContent(content=pulp3content, d_artifacts=[da]) dc.extra_data = future_relations await self.put(dc) @@ -523,22 +575,26 @@ class UpdateLCEs(Stage): Update migrated pulp2lazy_catalog entries with the is_migrated set to True only after RemoteArtifact has been saved. """ + async def run(self): """ Find LCEs in the extra_data and flip the is_migrated flag to True """ async for batch in self.batches(): + def process_batch(): pulp2lces_batch = [] with transaction.atomic(): for d_content in batch: - lces = d_content.extra_data.get('lces') + lces = d_content.extra_data.get("lces") if lces: pulp2lces_batch.extend(lces) - Pulp2LazyCatalog.objects.bulk_update(objs=pulp2lces_batch, - fields=['is_migrated'], - batch_size=DEFAULT_BATCH_SIZE) + Pulp2LazyCatalog.objects.bulk_update( + objs=pulp2lces_batch, + fields=["is_migrated"], + batch_size=DEFAULT_BATCH_SIZE, + ) await sync_to_async(process_batch)() for d_content in batch: @@ -552,6 +608,7 @@ class RelatePulp2to3Content(Stage): This relation signifies that the migration of this piece of content is done. Without this stage *all* the content will be migrated on every migration plan run. """ + async def run(self): """ Saves the relation between Pulp2Content and migrated Pulp 3 content. @@ -560,17 +617,21 @@ async def run(self): of a declarative Pulp 3 content. """ async for batch in self.batches(): + def process_batch(): pulp2content_batch = [] with transaction.atomic(): for d_content in batch: - pulp2content = d_content.extra_data.get('pulp2content') + pulp2content = d_content.extra_data.get("pulp2content") pulp2content.pulp3_content = d_content.content.master pulp2content_batch.append(pulp2content) - pulp2content.__class__.objects.bulk_update(objs=pulp2content_batch, - fields=['pulp3_content'], - batch_size=DEFAULT_BATCH_SIZE) + pulp2content.__class__.objects.bulk_update( + objs=pulp2content_batch, + fields=["pulp3_content"], + batch_size=DEFAULT_BATCH_SIZE, + ) + await sync_to_async(process_batch)() for d_content in batch: await self.put(d_content) diff --git a/pulp_2to3_migration/app/plugin/deb/migrator.py b/pulp_2to3_migration/app/plugin/deb/migrator.py index a9479504..278f9b71 100644 --- a/pulp_2to3_migration/app/plugin/deb/migrator.py +++ b/pulp_2to3_migration/app/plugin/deb/migrator.py @@ -34,38 +34,44 @@ class DebMigrator(Pulp2to3PluginMigrator): Check parent class. """ - pulp2_plugin = 'deb' + + pulp2_plugin = "deb" pulp2_content_models = { - 'deb': pulp2_models.DebPackage, - 'deb_release': pulp2_models.DebRelease, - 'deb_component': pulp2_models.DebComponent, + "deb": pulp2_models.DebPackage, + "deb_release": pulp2_models.DebRelease, + "deb_component": pulp2_models.DebComponent, } - pulp2_collection = 'units_deb' - pulp3_plugin = 'pulp_deb' + pulp2_collection = "units_deb" + pulp3_plugin = "pulp_deb" pulp3_repository = pulp3_models.AptRepository - content_models = OrderedDict([ - ('deb_release', pulp_2to3_models.Pulp2DebRelease), - ('deb', pulp_2to3_models.Pulp2DebPackage), - ('deb_component', pulp_2to3_models.Pulp2DebComponent), - ('deb_component_package', pulp_2to3_models.Pulp2DebComponentPackage), - ('deb_component_architecture', pulp_2to3_models.Pulp2DebReleaseArchitecture), - ]) + content_models = OrderedDict( + [ + ("deb_release", pulp_2to3_models.Pulp2DebRelease), + ("deb", pulp_2to3_models.Pulp2DebPackage), + ("deb_component", pulp_2to3_models.Pulp2DebComponent), + ("deb_component_package", pulp_2to3_models.Pulp2DebComponentPackage), + ( + "deb_component_architecture", + pulp_2to3_models.Pulp2DebReleaseArchitecture, + ), + ] + ) importer_migrators = { - 'deb_importer': repository.DebImporter, + "deb_importer": repository.DebImporter, } distributor_migrators = { - 'deb_distributor': repository.DebDistributor, + "deb_distributor": repository.DebDistributor, } future_types = { - 'deb': pulp_2to3_models.Pulp2DebPackage, - 'deb_release': pulp_2to3_models.Pulp2DebRelease, - 'deb_component': pulp_2to3_models.Pulp2DebComponent, + "deb": pulp_2to3_models.Pulp2DebPackage, + "deb_release": pulp_2to3_models.Pulp2DebRelease, + "deb_component": pulp_2to3_models.Pulp2DebComponent, } artifactless_types = { - 'deb_release': pulp_2to3_models.Pulp2DebRelease, - 'deb_component': pulp_2to3_models.Pulp2DebComponent, - 'deb_component_package': pulp_2to3_models.Pulp2DebComponentPackage, - 'deb_component_architecture': pulp_2to3_models.Pulp2DebReleaseArchitecture, + "deb_release": pulp_2to3_models.Pulp2DebRelease, + "deb_component": pulp_2to3_models.Pulp2DebComponent, + "deb_component_package": pulp_2to3_models.Pulp2DebComponentPackage, + "deb_component_architecture": pulp_2to3_models.Pulp2DebReleaseArchitecture, } @classmethod diff --git a/pulp_2to3_migration/app/plugin/deb/pulp2_models.py b/pulp_2to3_migration/app/plugin/deb/pulp2_models.py index 66ce9361..9344d92b 100644 --- a/pulp_2to3_migration/app/plugin/deb/pulp2_models.py +++ b/pulp_2to3_migration/app/plugin/deb/pulp2_models.py @@ -12,12 +12,13 @@ class DebPackage(FileContentUnit): It will become a pulp_deb Package type in Pulp 3. """ - TYPE_ID = 'deb' + + TYPE_ID = "deb" UNIT_KEY_DEB = ("name", "version", "architecture", "checksumtype", "checksum") meta = { - 'collection': 'units_deb', - 'indexes': list(UNIT_KEY_DEB), + "collection": "units_deb", + "indexes": list(UNIT_KEY_DEB), } unit_key_fields = UNIT_KEY_DEB @@ -36,7 +37,7 @@ class DebPackage(FileContentUnit): # Other required fields: filename = mongoengine.StringField(required=True) - REQUIRED_FIELDS = list(UNIT_KEY_DEB).append('filename') + REQUIRED_FIELDS = list(UNIT_KEY_DEB).append("filename") # Named checksum fields: md5sum = mongoengine.StringField() @@ -63,8 +64,17 @@ class DebPackage(FileContentUnit): suggests = mongoengine.DynamicField() # List of relational fields: - REL_FIELDS = ['breaks', 'conflicts', 'depends', 'enhances', 'pre_depends', - 'provides', 'recommends', 'replaces', 'suggests'] + REL_FIELDS = [ + "breaks", + "conflicts", + "depends", + "enhances", + "pre_depends", + "provides", + "recommends", + "replaces", + "suggests", + ] # The control file fields dict: # Note: This stores a dict of strings as used within the python-debian @@ -86,7 +96,7 @@ class DebPackage(FileContentUnit): original_maintainer = mongoengine.StringField() # Fields retained for backwards compatibility: - _ns = mongoengine.StringField(required=True, default=meta['collection']) + _ns = mongoengine.StringField(required=True, default=meta["collection"]) _content_type_id = mongoengine.StringField(required=True, default=TYPE_ID) # A dict translating all control file field names from this class into their @@ -121,11 +131,12 @@ class DebComponent(ContentUnit): """ This unittype represents a deb release/distribution component. """ - TYPE_ID = 'deb_component' - UNIT_KEY_DEB_COMPONENT = ('name', 'distribution', 'repoid') + + TYPE_ID = "deb_component" + UNIT_KEY_DEB_COMPONENT = ("name", "distribution", "repoid") meta = { - 'collection': "units_deb_component", - 'indexes': list(UNIT_KEY_DEB_COMPONENT), + "collection": "units_deb_component", + "indexes": list(UNIT_KEY_DEB_COMPONENT), } unit_key_fields = UNIT_KEY_DEB_COMPONENT @@ -136,7 +147,7 @@ class DebComponent(ContentUnit): packages = mongoengine.ListField() # For backward compatibility - _ns = mongoengine.StringField(required=True, default=meta['collection']) + _ns = mongoengine.StringField(required=True, default=meta["collection"]) _content_type_id = mongoengine.StringField(required=True, default=TYPE_ID) @property @@ -144,23 +155,24 @@ def plain_component(self): """ Returns the plain component without any directory prefixes. """ - return self.name.strip('/').split('/')[-1] + return self.name.strip("/").split("/")[-1] @property def prefixed_component(self): """ Returns the component with additional directory prefixes for complex distributions. """ - prefix = '/'.join(self.distribution.split('/')[1:]).strip('/') - return (prefix + '/' + self.plain_component).strip('/') + prefix = "/".join(self.distribution.split("/")[1:]).strip("/") + return (prefix + "/" + self.plain_component).strip("/") class DebRelease(ContentUnit): """ This unittype represents a deb release (also referred to as a "distribution"). """ - TYPE_ID = 'deb_release' - UNIT_KEY_DEB_RELEASE = ('distribution', 'repoid') + + TYPE_ID = "deb_release" + UNIT_KEY_DEB_RELEASE = ("distribution", "repoid") meta = dict(collection="units_deb_release", indexes=list(UNIT_KEY_DEB_RELEASE)) unit_key_fields = UNIT_KEY_DEB_RELEASE @@ -170,5 +182,5 @@ class DebRelease(ContentUnit): suite = mongoengine.StringField() # For backward compatibility - _ns = mongoengine.StringField(required=True, default=meta['collection']) + _ns = mongoengine.StringField(required=True, default=meta["collection"]) _content_type_id = mongoengine.StringField(required=True, default=TYPE_ID) diff --git a/pulp_2to3_migration/app/plugin/deb/pulp_2to3_models.py b/pulp_2to3_migration/app/plugin/deb/pulp_2to3_models.py index a6de82de..d362ce9a 100644 --- a/pulp_2to3_migration/app/plugin/deb/pulp_2to3_models.py +++ b/pulp_2to3_migration/app/plugin/deb/pulp_2to3_models.py @@ -25,7 +25,8 @@ class Pulp2DebPackage(Pulp2to3Content): Pulp 2to3 detail content model to store pulp 2 DebPackage content details for Pulp 3 content creation. """ - pulp2_type = 'deb' + + pulp2_type = "deb" checksumtype = django_models.TextField() checksum = django_models.TextField() filename = django_models.TextField() @@ -61,43 +62,43 @@ class Pulp2DebPackage(Pulp2to3Content): # Ordered list of control file fields explicitly known to pulp_deb: control_field_map = { - 'package': 'Package', - 'source': 'Source', - 'version': 'Version', - 'essential': 'Essential', - 'installed_size': 'Installed-Size', - 'maintainer': 'Maintainer', - 'original_maintainer': 'Original-Maintainer', - 'architecture': 'Architecture', - 'replaces': 'Replaces', - 'provides': 'Provides', - 'depends': 'Depends', - 'pre_depends': 'Pre-Depends', - 'recommends': 'Recommends', - 'suggests': 'Suggests', - 'enhances': 'Enhances', - 'conflicts': 'Conflicts', - 'breaks': 'Breaks', - 'description': 'Description', - 'multi_arch': 'Multi-Arch', - 'homepage': 'Homepage', - 'built_using': 'Built-Using', - 'description_md5': 'Description-md5', - 'build_essential': 'Build-Essential', - 'tag': 'Tag', - 'section': 'Section', - 'priority': 'Priority', + "package": "Package", + "source": "Source", + "version": "Version", + "essential": "Essential", + "installed_size": "Installed-Size", + "maintainer": "Maintainer", + "original_maintainer": "Original-Maintainer", + "architecture": "Architecture", + "replaces": "Replaces", + "provides": "Provides", + "depends": "Depends", + "pre_depends": "Pre-Depends", + "recommends": "Recommends", + "suggests": "Suggests", + "enhances": "Enhances", + "conflicts": "Conflicts", + "breaks": "Breaks", + "description": "Description", + "multi_arch": "Multi-Arch", + "homepage": "Homepage", + "built_using": "Built-Using", + "description_md5": "Description-md5", + "build_essential": "Build-Essential", + "tag": "Tag", + "section": "Section", + "priority": "Priority", } class Meta: - default_related_name = 'deb_detail_model' + default_related_name = "deb_detail_model" unique_together = ( - 'package', - 'version', - 'architecture', - 'checksumtype', - 'checksum', - 'pulp2content', + "package", + "version", + "architecture", + "checksumtype", + "checksum", + "pulp2content", ) @property @@ -123,34 +124,44 @@ def pre_migrate_content_detail(cls, content_batch): Args: content_batch(list of Pulp2Content): pre-migrated generic data for Pulp 2 content. """ - pulp2_unit_fields = set([ - 'checksumtype', - 'checksum', - 'filename', - 'size', - 'control_fields', - ]) - pulp2_id_obj_map = {pulp2content.pulp2_id: pulp2content for pulp2content in content_batch} + pulp2_unit_fields = set( + [ + "checksumtype", + "checksum", + "filename", + "size", + "control_fields", + ] + ) + pulp2_id_obj_map = { + pulp2content.pulp2_id: pulp2content for pulp2content in content_batch + } pulp2_ids = pulp2_id_obj_map.keys() - pulp2_content_batch = pulp2_models.DebPackage.objects.filter( - id__in=pulp2_ids, - ).as_pymongo().only(*pulp2_unit_fields) + pulp2_content_batch = ( + pulp2_models.DebPackage.objects.filter( + id__in=pulp2_ids, + ) + .as_pymongo() + .only(*pulp2_unit_fields) + ) pulp2deb_to_save = [] for deb in pulp2_content_batch: - pre_migrate_fields = {k: deb['control_fields'][v] - for k, v in cls.control_field_map.items() - if v in deb['control_fields'].keys()} - keys = deb['control_fields'].keys() - cls.control_field_map.values() - extra_control_fields = {k: deb['control_fields'][k] for k in keys} + pre_migrate_fields = { + k: deb["control_fields"][v] + for k, v in cls.control_field_map.items() + if v in deb["control_fields"].keys() + } + keys = deb["control_fields"].keys() - cls.control_field_map.values() + extra_control_fields = {k: deb["control_fields"][k] for k in keys} pulp2deb_to_save.append( Pulp2DebPackage( - checksumtype=deb['checksumtype'], - checksum=deb['checksum'], - filename=deb['filename'], + checksumtype=deb["checksumtype"], + checksum=deb["checksum"], + filename=deb["filename"], extra_control_fields=json.dumps(extra_control_fields), - pulp2content=pulp2_id_obj_map[deb['_id']], - size=deb['size'], + pulp2content=pulp2_id_obj_map[deb["_id"]], + size=deb["size"], **pre_migrate_fields, ) ) @@ -198,15 +209,16 @@ class Pulp2DebRelease(Pulp2to3Content): Pulp 2to3 detail content model to store pulp 2 DebRelease content details for Pulp 3 content creation. """ - pulp2_type = 'deb_release' + + pulp2_type = "deb_release" distribution = django_models.TextField() codename = django_models.TextField() suite = django_models.TextField(null=True) repoid = django_models.TextField() class Meta: - default_related_name = 'deb_release_detail_model' - unique_together = ('codename', 'suite', 'distribution', 'repoid') + default_related_name = "deb_release_detail_model" + unique_together = ("codename", "suite", "distribution", "repoid") @classmethod def pre_migrate_content_detail(cls, content_batch): @@ -219,17 +231,19 @@ def pre_migrate_content_detail(cls, content_batch): pulp2_unit_map = {pulp2unit.pulp2_id: pulp2unit for pulp2unit in content_batch} pulp2_ids = pulp2_unit_map.keys() pulp2_unit_batch = pulp2_models.DebRelease.objects.filter(id__in=pulp2_ids) - units_to_save = [Pulp2DebRelease(distribution=release.distribution, - codename=release.codename, - suite=release.suite, - repoid=release.repoid, - pulp2content=pulp2_unit_map[release.id],) - for release in pulp2_unit_batch] + units_to_save = [ + Pulp2DebRelease( + distribution=release.distribution, + codename=release.codename, + suite=release.suite, + repoid=release.repoid, + pulp2content=pulp2_unit_map[release.id], + ) + for release in pulp2_unit_batch + ] cls.objects.bulk_create( - units_to_save, - ignore_conflicts=True, - batch_size=DEFAULT_BATCH_SIZE + units_to_save, ignore_conflicts=True, batch_size=DEFAULT_BATCH_SIZE ) def create_pulp3_content(self): @@ -252,7 +266,8 @@ class Pulp2DebComponent(Pulp2to3Content): Pulp 2to3 detail content model to store pulp 2 DebComponent content details for Pulp 3 content creation. """ - pulp2_type = 'deb_component' + + pulp2_type = "deb_component" distribution = django_models.TextField() codename = django_models.TextField() component = django_models.TextField() @@ -260,8 +275,8 @@ class Pulp2DebComponent(Pulp2to3Content): repoid = django_models.TextField() class Meta: - default_related_name = 'deb_component_detail_model' - unique_together = ('component', 'codename', 'suite', 'distribution', 'repoid') + default_related_name = "deb_component_detail_model" + unique_together = ("component", "codename", "suite", "distribution", "repoid") @classmethod def pre_migrate_content_detail(cls, content_batch): @@ -300,24 +315,35 @@ def pre_migrate_content_detail(cls, content_batch): distribution=distribution, ).first() suite = release.suite - component_units_to_save.append(Pulp2DebComponent( - distribution=distribution, - codename=codename, - component=component, - suite=suite, - repoid=repoid, - pulp2content=pulp2_base_record, - )) + component_units_to_save.append( + Pulp2DebComponent( + distribution=distribution, + codename=codename, + component=component, + suite=suite, + repoid=repoid, + pulp2content=pulp2_base_record, + ) + ) architectures = set() for package_id in component_unit.packages: - package_unit = pulp2_models.DebPackage.objects.filter(id__in=[package_id]).first() + package_unit = pulp2_models.DebPackage.objects.filter( + id__in=[package_id] + ).first() package_relative_path = package_unit.filename package_sha256 = package_unit.checksum # We are using the sha256 of the concatenated unique_together fields for the subid: - pulp2_subid_string = (component + codename + suite + distribution + repoid - + package_relative_path + package_sha256) - pulp2_subid = sha256(pulp2_subid_string.encode('utf-8')).hexdigest() + pulp2_subid_string = ( + component + + codename + + suite + + distribution + + repoid + + package_relative_path + + package_sha256 + ) + pulp2_subid = sha256(pulp2_subid_string.encode("utf-8")).hexdigest() pulp2_sub_record = Pulp2Content( pulp2_subid=pulp2_subid, @@ -327,26 +353,32 @@ def pre_migrate_content_detail(cls, content_batch): pulp2_storage_path=pulp2_base_record.pulp2_storage_path, downloaded=pulp2_base_record.downloaded, ) - _logger.debug('Adding Pulp2Content subrecord {}'.format(pulp2_sub_record)) + _logger.debug( + "Adding Pulp2Content subrecord {}".format(pulp2_sub_record) + ) pulp2_sub_records_to_save.append(pulp2_sub_record) - component_package_units_to_save.append(Pulp2DebComponentPackage( - package_relative_path=package_relative_path, - package_sha256=package_sha256, - component=component, - distribution=distribution, - codename=codename, - suite=suite, - repoid=repoid, - pulp2content=pulp2_sub_record, - )) + component_package_units_to_save.append( + Pulp2DebComponentPackage( + package_relative_path=package_relative_path, + package_sha256=package_sha256, + component=component, + distribution=distribution, + codename=codename, + suite=suite, + repoid=repoid, + pulp2content=pulp2_sub_record, + ) + ) architectures.add(package_unit.architecture) - architectures.discard('all') + architectures.discard("all") for architecture in architectures: # We are using the sha256 of the concatenated unique_together fields for the subid: - pulp2_subid_string = architecture + distribution + codename + suite + repoid - pulp2_subid = sha256(pulp2_subid_string.encode('utf-8')).hexdigest() + pulp2_subid_string = ( + architecture + distribution + codename + suite + repoid + ) + pulp2_subid = sha256(pulp2_subid_string.encode("utf-8")).hexdigest() pulp2_sub_record = Pulp2Content( pulp2_subid=pulp2_subid, @@ -356,37 +388,41 @@ def pre_migrate_content_detail(cls, content_batch): pulp2_storage_path=pulp2_base_record.pulp2_storage_path, downloaded=pulp2_base_record.downloaded, ) - _logger.debug('Adding Pulp2Content subrecord {}'.format(pulp2_sub_record)) + _logger.debug( + "Adding Pulp2Content subrecord {}".format(pulp2_sub_record) + ) pulp2_sub_records_to_save.append(pulp2_sub_record) - release_architecture_units_to_save.append(Pulp2DebReleaseArchitecture( - architecture=architecture, - distribution=distribution, - codename=codename, - suite=suite, - repoid=repoid, - pulp2content=pulp2_sub_record, - )) + release_architecture_units_to_save.append( + Pulp2DebReleaseArchitecture( + architecture=architecture, + distribution=distribution, + codename=codename, + suite=suite, + repoid=repoid, + pulp2content=pulp2_sub_record, + ) + ) Pulp2Content.objects.bulk_create( pulp2_sub_records_to_save, ignore_conflicts=True, - batch_size=DEFAULT_BATCH_SIZE + batch_size=DEFAULT_BATCH_SIZE, ) cls.objects.bulk_create( component_units_to_save, ignore_conflicts=True, - batch_size=DEFAULT_BATCH_SIZE + batch_size=DEFAULT_BATCH_SIZE, ) Pulp2DebComponentPackage.objects.bulk_create( component_package_units_to_save, ignore_conflicts=True, - batch_size=DEFAULT_BATCH_SIZE + batch_size=DEFAULT_BATCH_SIZE, ) Pulp2DebReleaseArchitecture.objects.bulk_create( release_architecture_units_to_save, ignore_conflicts=True, - batch_size=DEFAULT_BATCH_SIZE + batch_size=DEFAULT_BATCH_SIZE, ) def create_pulp3_content(self): @@ -412,7 +448,8 @@ class Pulp2DebComponentPackage(Pulp2to3Content): Pulp 2to3 detail content model to store pulp 2 DebComponent content details for Pulp 3 content creation. """ - pulp2_type = 'deb_component' + + pulp2_type = "deb_component" package_relative_path = django_models.TextField() package_sha256 = django_models.TextField() component = django_models.TextField() @@ -422,15 +459,15 @@ class Pulp2DebComponentPackage(Pulp2to3Content): repoid = django_models.TextField() class Meta: - default_related_name = 'deb_component_package_detail_model' + default_related_name = "deb_component_package_detail_model" unique_together = ( - 'package_relative_path', - 'package_sha256', - 'component', - 'distribution', - 'codename', - 'suite', - 'repoid', + "package_relative_path", + "package_sha256", + "component", + "distribution", + "codename", + "suite", + "repoid", ) @classmethod @@ -475,7 +512,8 @@ class Pulp2DebReleaseArchitecture(Pulp2to3Content): Pulp 2to3 detail content model to store pulp 2 DebComponent content details for Pulp 3 content creation. """ - pulp2_type = 'deb_component' + + pulp2_type = "deb_component" architecture = django_models.TextField() distribution = django_models.TextField() codename = django_models.TextField() @@ -483,8 +521,14 @@ class Pulp2DebReleaseArchitecture(Pulp2to3Content): repoid = django_models.TextField() class Meta: - default_related_name = 'deb_release_architecture_detail_model' - unique_together = ('architecture', 'distribution', 'codename', 'suite', 'repoid') + default_related_name = "deb_release_architecture_detail_model" + unique_together = ( + "architecture", + "distribution", + "codename", + "suite", + "repoid", + ) @classmethod def pre_migrate_content_detail(cls, content_batch): diff --git a/pulp_2to3_migration/app/plugin/deb/repository.py b/pulp_2to3_migration/app/plugin/deb/repository.py index 7793b933..dfdef06b 100644 --- a/pulp_2to3_migration/app/plugin/deb/repository.py +++ b/pulp_2to3_migration/app/plugin/deb/repository.py @@ -17,6 +17,7 @@ class DebImporter(Pulp2to3Importer): """ Interface to migrate Pulp 2 Deb importer. """ + pulp3_remote_models = [AptRemote] @classmethod @@ -33,14 +34,16 @@ def migrate_to_pulp3(cls, pulp2importer): """ pulp2_config = pulp2importer.pulp2_config base_config, name = cls.parse_base_config(pulp2importer, pulp2_config) - base_config['distributions'] = pulp2_config.get('releases', 'stable').replace(',', ' ') - components = pulp2_config.get('components') + base_config["distributions"] = pulp2_config.get("releases", "stable").replace( + ",", " " + ) + components = pulp2_config.get("components") if components: - base_config['components'] = components.replace(',', ' ') - architectures = pulp2_config.get('architectures') + base_config["components"] = components.replace(",", " ") + architectures = pulp2_config.get("architectures") if architectures: - base_config['architectures'] = architectures.replace(',', ' ') - base_config['gpgkey'] = pulp2_config.get('gpg_keys') + base_config["architectures"] = architectures.replace(",", " ") + base_config["gpgkey"] = pulp2_config.get("gpg_keys") return AptRemote.objects.update_or_create(name=name, defaults=base_config) @@ -48,6 +51,7 @@ class DebDistributor(Pulp2to3Distributor): """ Interface to migrate Pulp 2 Deb distributor. """ + pulp3_publication_models = [AptPublication] pulp3_distribution_models = [AptDistribution] @@ -68,7 +72,9 @@ def migrate_to_pulp3(cls, pulp2distributor, repo_version, signing_service): signing_service_pk = signing_service.pk if signing_service else None # this will go away with the simple-complex plan conversion work if not repo_version: - repo = pulp2distributor.pulp2_repos.filter(not_in_plan=False, is_migrated=True) + repo = pulp2distributor.pulp2_repos.filter( + not_in_plan=False, is_migrated=True + ) repo_version = repo[0].pulp3_repository_version publication = repo_version.publication_set.filter(complete=True).first() if not publication: @@ -77,19 +83,19 @@ def migrate_to_pulp3(cls, pulp2distributor, repo_version, signing_service): repo_version.pk, simple=True, structured=True, - signing_service_pk=signing_service_pk + signing_service_pk=signing_service_pk, ) publication = repo_version.publication_set.filter(complete=True).first() # create distribution pulp2_config = pulp2distributor.pulp2_config distribution_data = cls.parse_base_config(pulp2distributor, pulp2_config) - base_path = pulp2_config.get('relative_url', pulp2distributor.pulp2_repo_id) - distribution_data['base_path'] = base_path.strip('/') - distribution_data['publication'] = publication + base_path = pulp2_config.get("relative_url", pulp2distributor.pulp2_repo_id) + distribution_data["base_path"] = base_path.strip("/") + distribution_data["publication"] = publication distribution, created = AptDistribution.objects.update_or_create( - name=distribution_data['name'], - base_path=distribution_data['base_path'], + name=distribution_data["name"], + base_path=distribution_data["base_path"], defaults=distribution_data, ) diff --git a/pulp_2to3_migration/app/plugin/docker/migrator.py b/pulp_2to3_migration/app/plugin/docker/migrator.py index da76aa01..62e6b84e 100644 --- a/pulp_2to3_migration/app/plugin/docker/migrator.py +++ b/pulp_2to3_migration/app/plugin/docker/migrator.py @@ -1,4 +1,3 @@ - import asyncio import json from collections import OrderedDict @@ -48,47 +47,48 @@ class DockerMigrator(Pulp2to3PluginMigrator): """ An entry point for migration the Pulp 2 Docker plugin to Pulp 3. """ - pulp2_plugin = 'docker' + + pulp2_plugin = "docker" pulp2_content_models = { - 'docker_blob': pulp2_models.Blob, - 'docker_manifest': pulp2_models.Manifest, - 'docker_manifest_list': pulp2_models.ManifestList, - 'docker_tag': pulp2_models.Tag, + "docker_blob": pulp2_models.Blob, + "docker_manifest": pulp2_models.Manifest, + "docker_manifest_list": pulp2_models.ManifestList, + "docker_tag": pulp2_models.Tag, } - pulp2_collection = 'units_docker_manifest' + pulp2_collection = "units_docker_manifest" # will be renamed to pulp_container - pulp3_plugin = 'pulp_container' + pulp3_plugin = "pulp_container" pulp3_repository = ContainerRepository - content_models = OrderedDict([ - ('docker_blob', Pulp2Blob), - ('docker_manifest', Pulp2Manifest), - ('docker_manifest_list', Pulp2ManifestList), - ('docker_tag', Pulp2Tag), - ]) + content_models = OrderedDict( + [ + ("docker_blob", Pulp2Blob), + ("docker_manifest", Pulp2Manifest), + ("docker_manifest_list", Pulp2ManifestList), + ("docker_tag", Pulp2Tag), + ] + ) mutable_content_models = { - 'docker_tag': Pulp2Tag, + "docker_tag": Pulp2Tag, } importer_migrators = { - 'docker_importer': DockerImporter, + "docker_importer": DockerImporter, } distributor_migrators = { - 'docker_distributor_web': DockerDistributor, + "docker_distributor_web": DockerDistributor, } - premigrate_hook = { - 'docker_tag': utils.find_tags - } + premigrate_hook = {"docker_tag": utils.find_tags} artifactless_types = { - 'docker_tag': Pulp2Tag, + "docker_tag": Pulp2Tag, } future_types = { - 'docker_manifest': Pulp2Manifest, - 'docker_manifest_list': Pulp2ManifestList + "docker_manifest": Pulp2Manifest, + "docker_manifest_list": Pulp2ManifestList, } @classmethod @@ -147,33 +147,41 @@ async def run(self): Relate each item in the input queue to objects specified on the DeclarativeContent. """ async for batch in self.batches(): + def process_batch(): manifestlist_manifest_batch = [] blob_manifest_batch = [] manifest_batch = [] with transaction.atomic(): for dc in batch: - if dc.extra_data.get('man_rel'): + if dc.extra_data.get("man_rel"): thru = self.relate_manifest_to_list(dc) manifestlist_manifest_batch.extend(thru) - elif dc.extra_data.get('blob_rel'): + elif dc.extra_data.get("blob_rel"): thru = self.relate_blob(dc) blob_manifest_batch.extend(thru) - if dc.extra_data.get('config_blob_rel'): + if dc.extra_data.get("config_blob_rel"): manifest_to_update = self.relate_config_blob(dc) manifest_batch.append(manifest_to_update) - ManifestListManifest.objects.bulk_create(objs=manifestlist_manifest_batch, - ignore_conflicts=True, - batch_size=DEFAULT_BATCH_SIZE) - BlobManifest.objects.bulk_create(objs=blob_manifest_batch, - ignore_conflicts=True, - batch_size=DEFAULT_BATCH_SIZE) + ManifestListManifest.objects.bulk_create( + objs=manifestlist_manifest_batch, + ignore_conflicts=True, + batch_size=DEFAULT_BATCH_SIZE, + ) + BlobManifest.objects.bulk_create( + objs=blob_manifest_batch, + ignore_conflicts=True, + batch_size=DEFAULT_BATCH_SIZE, + ) + + Manifest.objects.bulk_update( + objs=manifest_batch, + fields=["config_blob"], + batch_size=DEFAULT_BATCH_SIZE, + ) - Manifest.objects.bulk_update(objs=manifest_batch, - fields=['config_blob'], - batch_size=DEFAULT_BATCH_SIZE) await sync_to_async(process_batch)() for dc in batch: await self.put(dc) @@ -185,7 +193,7 @@ def relate_config_blob(self, dc): Args: dc (pulpcore.plugin.stages.DeclarativeContent): dc for a Manifest """ - configured_dc_id = dc.extra_data.get('config_blob_rel') + configured_dc_id = dc.extra_data.get("config_blob_rel") # find blob by id # We are relying on the order of the processed DC # Blobs should have passed through ContentSaver stage already @@ -200,7 +208,7 @@ def relate_blob(self, dc): Args: dc (pulpcore.plugin.stages.DeclarativeContent): dc for a Manifest """ - related_dc_id_list = dc.extra_data.get('blob_rel') + related_dc_id_list = dc.extra_data.get("blob_rel") # find blob by id # We are relying on the order of the processed DC # Blobs should have passed through ContentSaver stage already @@ -217,7 +225,7 @@ def relate_manifest_to_list(self, dc): Args: dc (pulpcore.plugin.stages.DeclarativeContent): dc for a Manifest list """ - related_dc_id_list = dc.extra_data.get('man_rel') + related_dc_id_list = dc.extra_data.get("man_rel") # find manifests by id # We are relying on the order of the processed DC # Manifests should have passed through ContentSaver stage already @@ -226,22 +234,24 @@ def relate_manifest_to_list(self, dc): with dc.content._artifacts.get().file.open() as content_file: raw = content_file.read() content_data = json.loads(raw) - manifests_from_json = content_data['manifests'] + manifests_from_json = content_data["manifests"] mlm = [] for manifest in manifests_from_json: - digest = manifest['digest'] + digest = manifest["digest"] for item in man_list: if item.digest == digest: - platform = manifest['platform'] - thru = ManifestListManifest(manifest_list=item, image_manifest=dc.content, - architecture=platform['architecture'], - os=platform['os'], - features=platform.get('features', ''), - variant=platform.get('variant', ''), - os_version=platform.get('os.version', ''), - os_features=platform.get('os.features', '') - ) + platform = manifest["platform"] + thru = ManifestListManifest( + manifest_list=item, + image_manifest=dc.content, + architecture=platform["architecture"], + os=platform["os"], + features=platform.get("features", ""), + variant=platform.get("variant", ""), + os_version=platform.get("os.version", ""), + os_features=platform.get("os.features", ""), + ) mlm.append(thru) break return mlm @@ -264,7 +274,7 @@ def _pre_save(self, batch): """ for dc in batch: if isinstance(dc.content, Tag): - related_man_id = dc.extra_data.get('tag_rel') + related_man_id = dc.extra_data.get("tag_rel") # find manifest by id # We are relying on the order of the processed DC # Manifests should have passed through ContentSaver stage already diff --git a/pulp_2to3_migration/app/plugin/docker/pulp2_models.py b/pulp_2to3_migration/app/plugin/docker/pulp2_models.py index 9bb805ec..8a5c6d50 100644 --- a/pulp_2to3_migration/app/plugin/docker/pulp2_models.py +++ b/pulp_2to3_migration/app/plugin/docker/pulp2_models.py @@ -19,19 +19,20 @@ class Blob(FileContentUnit): It will become a Blob content type in Pulp 3 world. """ + digest = StringField(required=True) - _ns = StringField(default='units_docker_blob') - _content_type_id = StringField(required=True, default='docker_blob') + _ns = StringField(default="units_docker_blob") + _content_type_id = StringField(required=True, default="docker_blob") - unit_key_fields = ('digest') - unit_display_name = 'docker blob' - unit_description = 'docker blob' + unit_key_fields = "digest" + unit_display_name = "docker blob" + unit_description = "docker blob" - TYPE_ID = 'docker_blob' + TYPE_ID = "docker_blob" meta = { - 'collection': 'units_docker_blob', + "collection": "units_docker_blob", } @@ -39,6 +40,7 @@ class FSLayer(EmbeddedDocument): """ This EmbeddedDocument is used in the Manifest.fs_layers field. It references a Blob Document. """ + # This will be the digest of a Blob document. blob_sum = StringField(required=True) size = IntField() @@ -49,23 +51,24 @@ class Manifest(FileContentUnit): """ This model represents a Docker v2, Schema 1 Image Manifest and Schema 2 Image Manifest. """ + digest = StringField(required=True) schema_version = IntField(required=True) fs_layers = ListField(field=EmbeddedDocumentField(FSLayer), required=True) config_layer = StringField() # For backward compatibility - _ns = StringField(default='units_docker_manfest') - _content_type_id = StringField(required=True, default='docker_manifest') + _ns = StringField(default="units_docker_manfest") + _content_type_id = StringField(required=True, default="docker_manifest") - unit_key_fields = ('digest',) - unit_display_name = 'docker manifest' - unit_description = 'docker manifest' + unit_key_fields = ("digest",) + unit_display_name = "docker manifest" + unit_description = "docker manifest" - TYPE_ID = 'docker_manifest' + TYPE_ID = "docker_manifest" meta = { - 'collection': 'units_docker_manifest', + "collection": "units_docker_manifest", } @@ -74,6 +77,7 @@ class EmbeddedManifest(EmbeddedDocument): This EmbeddedDocument is used in the ManifestList.manifests field. It references a ManifestList. """ + digest = StringField(required=True) os = StringField() arch = StringField() @@ -83,6 +87,7 @@ class ManifestList(FileContentUnit): """ This model represents a Docker v2, Schema 2 Manifest list """ + digest = StringField(required=True) schema_version = IntField(required=True) manifests = ListField(EmbeddedDocumentField(EmbeddedManifest)) @@ -90,17 +95,17 @@ class ManifestList(FileContentUnit): amd64_schema_version = IntField() # For backward compatibility - _ns = StringField(default='units_docker_manifest_list') - _content_type_id = StringField(required=True, default='docker_manifest_list') + _ns = StringField(default="units_docker_manifest_list") + _content_type_id = StringField(required=True, default="docker_manifest_list") - unit_key_fields = ('digest',) - unit_display_name = 'docker manifest list' - unit_description = 'docker manifest list' + unit_key_fields = ("digest",) + unit_display_name = "docker manifest list" + unit_description = "docker manifest list" - TYPE_ID = 'docker_manifest_list' + TYPE_ID = "docker_manifest_list" meta = { - 'collection': 'units_docker_manifest_list', + "collection": "units_docker_manifest_list", } @@ -108,21 +113,22 @@ class Tag(ContentUnit): """ This class is used to represent Docker v2 tags. """ + name = StringField(required=True) manifest_digest = StringField(required=True) repo_id = StringField(required=True) schema_version = IntField(required=True) manifest_type = StringField(required=True) - _ns = StringField(default='units_docker_tag') - _content_type_id = StringField(required=True, default='docker_tag') + _ns = StringField(default="units_docker_tag") + _content_type_id = StringField(required=True, default="docker_tag") - unit_key_fields = ('name', 'repo_id', 'schema_version', 'manifest_type') - unit_display_name = 'docker tag' - unit_description = 'docker tag' + unit_key_fields = ("name", "repo_id", "schema_version", "manifest_type") + unit_display_name = "docker tag" + unit_description = "docker tag" - TYPE_ID = 'docker_tag' + TYPE_ID = "docker_tag" meta = { - 'collection': 'units_docker_tag', + "collection": "units_docker_tag", } diff --git a/pulp_2to3_migration/app/plugin/docker/pulp_2to3_models.py b/pulp_2to3_migration/app/plugin/docker/pulp_2to3_models.py index ea19cccc..d5ea189e 100644 --- a/pulp_2to3_migration/app/plugin/docker/pulp_2to3_models.py +++ b/pulp_2to3_migration/app/plugin/docker/pulp_2to3_models.py @@ -15,20 +15,21 @@ class Pulp2Blob(Pulp2to3Content): Pulp 2to3 detail content model to store Pulp 2 Blob content details for Pulp 3 content creation. """ + digest = models.CharField(max_length=255) media_type = models.CharField(max_length=80) - pulp2_type = 'docker_blob' - checksum_type = 'sha256' + pulp2_type = "docker_blob" + checksum_type = "sha256" class Meta: - unique_together = ('digest', 'pulp2content') - default_related_name = 'docker_blob_detail_model' + unique_together = ("digest", "pulp2content") + default_related_name = "docker_blob_detail_model" @property def expected_digests(self): """Return expected digests.""" - return {self.checksum_type: self.digest.split(':')[1]} + return {self.checksum_type: self.digest.split(":")[1]} @property def expected_size(self): @@ -49,15 +50,22 @@ def pre_migrate_content_detail(cls, content_batch): content_batch(list of Pulp2Content): pre-migrated generic data for Pulp 2 content. """ - pulp2_id_obj_map = {pulp2content.pulp2_id: pulp2content for pulp2content in content_batch} + pulp2_id_obj_map = { + pulp2content.pulp2_id: pulp2content for pulp2content in content_batch + } pulp2_ids = pulp2_id_obj_map.keys() pulp2_blob_content_batch = pulp2_models.Blob.objects.filter(id__in=pulp2_ids) - pulp2blob_to_save = [Pulp2Blob(digest=blob.digest, - media_type=MEDIA_TYPE.REGULAR_BLOB, - pulp2content=pulp2_id_obj_map[blob.id]) - for blob in pulp2_blob_content_batch] - cls.objects.bulk_create(pulp2blob_to_save, ignore_conflicts=True, - batch_size=DEFAULT_BATCH_SIZE) + pulp2blob_to_save = [ + Pulp2Blob( + digest=blob.digest, + media_type=MEDIA_TYPE.REGULAR_BLOB, + pulp2content=pulp2_id_obj_map[blob.id], + ) + for blob in pulp2_blob_content_batch + ] + cls.objects.bulk_create( + pulp2blob_to_save, ignore_conflicts=True, batch_size=DEFAULT_BATCH_SIZE + ) def create_pulp3_content(self): """ @@ -71,18 +79,19 @@ class Pulp2Manifest(Pulp2to3Content): Pulp 2to3 detail content model to store pulp 2 Manifest content details for Pulp 3 content creation. """ + digest = models.CharField(max_length=255) schema_version = models.IntegerField() media_type = models.CharField(max_length=80) blobs = ArrayField(models.CharField(max_length=255)) config_blob = models.CharField(max_length=255, null=True) - pulp2_type = 'docker_manifest' - checksum_type = 'sha256' + pulp2_type = "docker_manifest" + checksum_type = "sha256" class Meta: - unique_together = ('digest', 'pulp2content') - default_related_name = 'docker_manifest_detail_model' + unique_together = ("digest", "pulp2content") + default_related_name = "docker_manifest_detail_model" @property def expected_digests(self): @@ -108,11 +117,16 @@ def pre_migrate_content_detail(cls, content_batch): content_batch(list of Pulp2Content): pre-migrated generic data for Pulp 2 content. """ + def _get_media_type(schema_version): """ Return media_type of the manifest. """ - return MEDIA_TYPE.MANIFEST_V2 if schema_version == 2 else MEDIA_TYPE.MANIFEST_V1 + return ( + MEDIA_TYPE.MANIFEST_V2 + if schema_version == 2 + else MEDIA_TYPE.MANIFEST_V1 + ) def _get_blobs(layers): """ @@ -124,30 +138,40 @@ def _get_blobs(layers): blobs.append(layer.blob_sum) return blobs - pulp2_id_obj_map = {pulp2content.pulp2_id: pulp2content for pulp2content in content_batch} + pulp2_id_obj_map = { + pulp2content.pulp2_id: pulp2content for pulp2content in content_batch + } pulp2_ids = pulp2_id_obj_map.keys() pulp2_m_content_batch = pulp2_models.Manifest.objects.filter(id__in=pulp2_ids) pulp2m_to_save = [] for m in pulp2_m_content_batch: pulp2m_to_save.append( - Pulp2Manifest(digest=m.digest, - media_type=_get_media_type(m.schema_version), - schema_version=m.schema_version, - config_blob=m.config_layer, - blobs=_get_blobs(m.fs_layers), - pulp2content=pulp2_id_obj_map[m.id]) + Pulp2Manifest( + digest=m.digest, + media_type=_get_media_type(m.schema_version), + schema_version=m.schema_version, + config_blob=m.config_layer, + blobs=_get_blobs(m.fs_layers), + pulp2content=pulp2_id_obj_map[m.id], + ) ) - cls.objects.bulk_create(pulp2m_to_save, ignore_conflicts=True, - batch_size=DEFAULT_BATCH_SIZE) + cls.objects.bulk_create( + pulp2m_to_save, ignore_conflicts=True, batch_size=DEFAULT_BATCH_SIZE + ) def create_pulp3_content(self): """ Create a Pulp 3 Manifest unit for saving it later in a bulk operation. """ - future_relations = {'blob_rel': self.blobs, 'config_blob_rel': self.config_blob} - return (Manifest(digest=self.digest, - media_type=self.media_type, - schema_version=self.schema_version), future_relations) + future_relations = {"blob_rel": self.blobs, "config_blob_rel": self.config_blob} + return ( + Manifest( + digest=self.digest, + media_type=self.media_type, + schema_version=self.schema_version, + ), + future_relations, + ) class Pulp2ManifestList(Pulp2to3Content): @@ -155,23 +179,24 @@ class Pulp2ManifestList(Pulp2to3Content): Pulp 2to3 detail content model to store pulp 2 ManifestList content details for Pulp 3 content creation. """ + digest = models.CharField(max_length=255) media_type = models.CharField(max_length=80) schema_version = models.IntegerField() media_type = models.CharField(max_length=80) listed_manifests = ArrayField(models.CharField(max_length=255)) - pulp2_type = 'docker_manifest_list' - checksum_type = 'sha256' + pulp2_type = "docker_manifest_list" + checksum_type = "sha256" class Meta: - unique_together = ('digest', 'pulp2content') - default_related_name = 'docker_manifest_list_detail_model' + unique_together = ("digest", "pulp2content") + default_related_name = "docker_manifest_list_detail_model" @property def expected_digests(self): """Return expected digests.""" - return {self.checksum_type: self.digest.split(':')[1]} + return {self.checksum_type: self.digest.split(":")[1]} @property def expected_size(self): @@ -193,26 +218,40 @@ def pre_migrate_content_detail(cls, content_batch): content_batch(list of Pulp2Content): pre-migrated generic data for Pulp 2 content. """ - pulp2_id_obj_map = {pulp2content.pulp2_id: pulp2content for pulp2content in content_batch} + pulp2_id_obj_map = { + pulp2content.pulp2_id: pulp2content for pulp2content in content_batch + } pulp2_ids = pulp2_id_obj_map.keys() - pulp2_m_content_batch = pulp2_models.ManifestList.objects.filter(id__in=pulp2_ids) - pulp2m_to_save = [Pulp2ManifestList(digest=m.digest, - media_type=MEDIA_TYPE.MANIFEST_LIST, - schema_version=m.schema_version, - listed_manifests=[man.digest for man in m.manifests], - pulp2content=pulp2_id_obj_map[m.id]) - for m in pulp2_m_content_batch] - cls.objects.bulk_create(pulp2m_to_save, ignore_conflicts=True, - batch_size=DEFAULT_BATCH_SIZE) + pulp2_m_content_batch = pulp2_models.ManifestList.objects.filter( + id__in=pulp2_ids + ) + pulp2m_to_save = [ + Pulp2ManifestList( + digest=m.digest, + media_type=MEDIA_TYPE.MANIFEST_LIST, + schema_version=m.schema_version, + listed_manifests=[man.digest for man in m.manifests], + pulp2content=pulp2_id_obj_map[m.id], + ) + for m in pulp2_m_content_batch + ] + cls.objects.bulk_create( + pulp2m_to_save, ignore_conflicts=True, batch_size=DEFAULT_BATCH_SIZE + ) def create_pulp3_content(self): """ Create a Pulp 3 Manifest unit for saving it later in a bulk operation. """ - future_relations = {'man_rel': self.listed_manifests} - return (Manifest(digest=self.digest, - media_type=self.media_type, - schema_version=self.schema_version), future_relations) + future_relations = {"man_rel": self.listed_manifests} + return ( + Manifest( + digest=self.digest, + media_type=self.media_type, + schema_version=self.schema_version, + ), + future_relations, + ) class Pulp2Tag(Pulp2to3Content): @@ -220,16 +259,17 @@ class Pulp2Tag(Pulp2to3Content): Pulp 2to3 detail content model to store pulp 2 Tag content details for Pulp 3 content creation. """ + name = models.TextField() tagged_manifest = models.CharField(max_length=255) repo_id = models.TextField() - pulp2_type = 'docker_tag' - checksum_type = 'sha256' + pulp2_type = "docker_tag" + checksum_type = "sha256" class Meta: - unique_together = ('name', 'tagged_manifest', 'repo_id', 'pulp2content') - default_related_name = 'docker_tag_detail_model' + unique_together = ("name", "tagged_manifest", "repo_id", "pulp2content") + default_related_name = "docker_tag_detail_model" @property def relative_path_for_content_artifact(self): @@ -245,20 +285,27 @@ def pre_migrate_content_detail(cls, content_batch): content_batch(list of Pulp2Content): pre-migrated generic data for Pulp 2 content. """ - pulp2_id_obj_map = {pulp2content.pulp2_id: pulp2content for pulp2content in content_batch} + pulp2_id_obj_map = { + pulp2content.pulp2_id: pulp2content for pulp2content in content_batch + } pulp2_ids = pulp2_id_obj_map.keys() pulp2_tag_content_batch = pulp2_models.Tag.objects.filter(id__in=pulp2_ids) - pulp2tag_to_save = [Pulp2Tag(name=tag.name, - tagged_manifest=tag.manifest_digest, - repo_id=tag.repo_id, - pulp2content=pulp2_id_obj_map[tag.id]) - for tag in pulp2_tag_content_batch] - cls.objects.bulk_create(pulp2tag_to_save, ignore_conflicts=True, - batch_size=DEFAULT_BATCH_SIZE) + pulp2tag_to_save = [ + Pulp2Tag( + name=tag.name, + tagged_manifest=tag.manifest_digest, + repo_id=tag.repo_id, + pulp2content=pulp2_id_obj_map[tag.id], + ) + for tag in pulp2_tag_content_batch + ] + cls.objects.bulk_create( + pulp2tag_to_save, ignore_conflicts=True, batch_size=DEFAULT_BATCH_SIZE + ) def create_pulp3_content(self): """ Create a Pulp 3 Tag unit for saving it later in a bulk operation. """ - future_relations = {'tag_rel': self.tagged_manifest} + future_relations = {"tag_rel": self.tagged_manifest} return (Tag(name=self.name), future_relations) diff --git a/pulp_2to3_migration/app/plugin/docker/repository.py b/pulp_2to3_migration/app/plugin/docker/repository.py index 4ac6c225..5d2ddca5 100644 --- a/pulp_2to3_migration/app/plugin/docker/repository.py +++ b/pulp_2to3_migration/app/plugin/docker/repository.py @@ -7,6 +7,7 @@ class DockerImporter(Pulp2to3Importer): """ Interface to migrate Pulp 2 Docker importer """ + pulp3_remote_models = [ContainerRemote] @classmethod @@ -24,8 +25,8 @@ def migrate_to_pulp3(cls, pulp2importer): pulp2_config = pulp2importer.pulp2_config base_config, name = cls.parse_base_config(pulp2importer, pulp2_config) # what to do if there is no upstream name? - base_config['upstream_name'] = pulp2_config.get('upstream_name', '') - base_config['include_tags'] = pulp2_config.get('tags') + base_config["upstream_name"] = pulp2_config.get("upstream_name", "") + base_config["include_tags"] = pulp2_config.get("tags") return ContainerRemote.objects.update_or_create(name=name, defaults=base_config) @@ -33,6 +34,7 @@ class DockerDistributor(Pulp2to3Distributor): """ Interface to migrate Pulp 2 Docker distributor """ + pulp3_distribution_models = [ContainerDistribution] @classmethod @@ -50,17 +52,21 @@ def migrate_to_pulp3(cls, pulp2distributor, repo_version, signing_service): """ # this will go away with the simple-complex plan conversion work if not repo_version: - repo = pulp2distributor.pulp2_repos.filter(not_in_plan=False, is_migrated=True) + repo = pulp2distributor.pulp2_repos.filter( + not_in_plan=False, is_migrated=True + ) repo_version = repo[0].pulp3_repository_version pulp2_config = pulp2distributor.pulp2_config base_config = cls.parse_base_config(pulp2distributor, pulp2_config) - base_config['base_path'] = pulp2_config.get( - 'repo-registry-id', pulp2distributor.pulp2_repo_id) - base_config['repository_version'] = repo_version + base_config["base_path"] = pulp2_config.get( + "repo-registry-id", pulp2distributor.pulp2_repo_id + ) + base_config["repository_version"] = repo_version distribution, created = ContainerDistribution.objects.update_or_create( - name=base_config['name'], - base_path=base_config['base_path'], - defaults=base_config) + name=base_config["name"], + base_path=base_config["base_path"], + defaults=base_config, + ) return None, distribution, created @classmethod @@ -93,8 +99,9 @@ def needs_new_distribution(cls, pulp2distributor): if not pulp2distributor.pulp3_distribution: return True - new_base_path = pulp2distributor.pulp2_config.get('repo-registry-id', - pulp2distributor.pulp2_repo_id) + new_base_path = pulp2distributor.pulp2_config.get( + "repo-registry-id", pulp2distributor.pulp2_repo_id + ) current_base_path = pulp2distributor.pulp3_distribution.base_path if new_base_path != current_base_path: return True diff --git a/pulp_2to3_migration/app/plugin/docker/utils.py b/pulp_2to3_migration/app/plugin/docker/utils.py index e7ec15b5..696dc212 100644 --- a/pulp_2to3_migration/app/plugin/docker/utils.py +++ b/pulp_2to3_migration/app/plugin/docker/utils.py @@ -9,14 +9,18 @@ def find_tags(): """ # sort the schema version in desc mode. - sort_stage = {'$sort': {'schema_version': -1}} + sort_stage = {"$sort": {"schema_version": -1}} # group tags by name and repo_id; take just first result out of the 2 tags with the same name - group_stage1 = {'$group': {'_id': {'name': '$name', 'repo_id': '$repo_id'}, - 'tags_id': {'$first': '$_id'}}} - group_stage2 = {'$group': {'_id': None, 'tags_ids': {'$addToSet': '$tags_id'}}} + group_stage1 = { + "$group": { + "_id": {"name": "$name", "repo_id": "$repo_id"}, + "tags_id": {"$first": "$_id"}, + } + } + group_stage2 = {"$group": {"_id": None, "tags_ids": {"$addToSet": "$tags_id"}}} result = pulp2_models.Tag.objects.aggregate( [sort_stage, group_stage1, group_stage2], allowDiskUse=True ) if result._has_next(): - return result.next()['tags_ids'] + return result.next()["tags_ids"] return [] diff --git a/pulp_2to3_migration/app/plugin/iso/migrator.py b/pulp_2to3_migration/app/plugin/iso/migrator.py index aa38e119..baef2a4e 100644 --- a/pulp_2to3_migration/app/plugin/iso/migrator.py +++ b/pulp_2to3_migration/app/plugin/iso/migrator.py @@ -27,24 +27,25 @@ class IsoMigrator(Pulp2to3PluginMigrator): importer_migrators(dict): {'importer_type_id': 'pulp_2to3 importer interface/migrator'} """ - pulp2_plugin = 'iso' + + pulp2_plugin = "iso" pulp2_content_models = { - 'iso': ISO, + "iso": ISO, } - pulp2_collection = 'units_iso' - pulp3_plugin = 'pulp_file' + pulp2_collection = "units_iso" + pulp3_plugin = "pulp_file" pulp3_repository = FileRepository content_models = { - 'iso': Pulp2ISO, + "iso": Pulp2ISO, } importer_migrators = { - 'iso_importer': IsoImporter, + "iso_importer": IsoImporter, } distributor_migrators = { - 'iso_distributor': IsoDistributor, + "iso_distributor": IsoDistributor, } lazy_types = { - 'iso': Pulp2ISO, + "iso": Pulp2ISO, } @classmethod diff --git a/pulp_2to3_migration/app/plugin/iso/pulp2_models.py b/pulp_2to3_migration/app/plugin/iso/pulp2_models.py index c1a7de23..7452bdd2 100644 --- a/pulp_2to3_migration/app/plugin/iso/pulp2_models.py +++ b/pulp_2to3_migration/app/plugin/iso/pulp2_models.py @@ -16,19 +16,20 @@ class ISO(FileContentUnit): It will become a File content type in Pulp 3 world. """ + name = StringField(required=True) checksum = StringField(required=True) size = IntField(required=True) - _ns = StringField(default='units_iso') - _content_type_id = StringField(required=True, default='iso') + _ns = StringField(default="units_iso") + _content_type_id = StringField(required=True, default="iso") - unit_key_fields = ('name', 'checksum', 'size') - unit_display_name = 'ISO' - unit_description = 'ISO' + unit_key_fields = ("name", "checksum", "size") + unit_display_name = "ISO" + unit_description = "ISO" - TYPE_ID = 'iso' + TYPE_ID = "iso" meta = { - 'collection': 'units_iso', + "collection": "units_iso", } diff --git a/pulp_2to3_migration/app/plugin/iso/pulp_2to3_models.py b/pulp_2to3_migration/app/plugin/iso/pulp_2to3_models.py index 7495da25..2eaa40eb 100644 --- a/pulp_2to3_migration/app/plugin/iso/pulp_2to3_models.py +++ b/pulp_2to3_migration/app/plugin/iso/pulp_2to3_models.py @@ -12,16 +12,17 @@ class Pulp2ISO(Pulp2to3Content): """ Pulp 2to3 detail content model to store pulp 2 ISO content details for Pulp 3 content creation. """ + name = models.TextField() checksum = models.CharField(max_length=64) size = models.BigIntegerField() - pulp2_type = 'iso' - checksum_type = 'sha256' + pulp2_type = "iso" + checksum_type = "sha256" class Meta: - unique_together = ('name', 'checksum', 'size', 'pulp2content') - default_related_name = 'iso_detail_model' + unique_together = ("name", "checksum", "size", "pulp2content") + default_related_name = "iso_detail_model" @property def expected_digests(self): @@ -48,20 +49,26 @@ def pre_migrate_content_detail(cls, content_batch): """ # TODO: all pulp2content objects from the batch are in memory. Concerns? - pulp2_id_obj_map = {pulp2content.pulp2_id: pulp2content for pulp2content in content_batch} + pulp2_id_obj_map = { + pulp2content.pulp2_id: pulp2content for pulp2content in content_batch + } pulp2_ids = pulp2_id_obj_map.keys() pulp2_iso_content_batch = ISO.objects.filter(id__in=pulp2_ids) - pulp2iso_to_save = [Pulp2ISO(name=iso.name, - checksum=iso.checksum, - size=iso.size, - pulp2content=pulp2_id_obj_map[iso.id]) - for iso in pulp2_iso_content_batch] - cls.objects.bulk_create(pulp2iso_to_save, ignore_conflicts=True, - batch_size=DEFAULT_BATCH_SIZE) + pulp2iso_to_save = [ + Pulp2ISO( + name=iso.name, + checksum=iso.checksum, + size=iso.size, + pulp2content=pulp2_id_obj_map[iso.id], + ) + for iso in pulp2_iso_content_batch + ] + cls.objects.bulk_create( + pulp2iso_to_save, ignore_conflicts=True, batch_size=DEFAULT_BATCH_SIZE + ) def create_pulp3_content(self): """ Create a Pulp 3 FileContent unit for saving it later in a bulk operation. """ - return (FileContent(relative_path=self.name, - digest=self.checksum), None) + return (FileContent(relative_path=self.name, digest=self.checksum), None) diff --git a/pulp_2to3_migration/app/plugin/iso/repository.py b/pulp_2to3_migration/app/plugin/iso/repository.py index f3487aa7..656a254a 100644 --- a/pulp_2to3_migration/app/plugin/iso/repository.py +++ b/pulp_2to3_migration/app/plugin/iso/repository.py @@ -3,7 +3,7 @@ from pulp_2to3_migration.app.plugin.api import ( is_different_relative_url, Pulp2to3Importer, - Pulp2to3Distributor + Pulp2to3Distributor, ) from pulp_file.app.models import FileRemote, FilePublication, FileDistribution @@ -14,6 +14,7 @@ class IsoImporter(Pulp2to3Importer): """ Interface to migrate Pulp 2 ISO importer """ + pulp3_remote_models = [FileRemote] @classmethod @@ -32,9 +33,9 @@ def migrate_to_pulp3(cls, pulp2importer): base_config, name = cls.parse_base_config(pulp2importer, pulp2_config) # pulp3 remote requires url set to the manifest # pulp2 iso importer is compatible only with repos that contain namely PULP_MANIFEST - url = base_config.get('url') + url = base_config.get("url") if url: - base_config['url'] = os.path.join(url, 'PULP_MANIFEST') + base_config["url"] = os.path.join(url, "PULP_MANIFEST") return FileRemote.objects.update_or_create(name=name, defaults=base_config) @@ -42,6 +43,7 @@ class IsoDistributor(Pulp2to3Distributor): """ Interface to migrate Pulp 2 ISO distributor """ + pulp3_publication_models = [FilePublication] pulp3_distribution_models = [FileDistribution] @@ -61,22 +63,27 @@ def migrate_to_pulp3(cls, pulp2distributor, repo_version, signing_service): # this will go away with the simple-complex plan conversion work if not repo_version: - repo = pulp2distributor.pulp2_repos.filter(not_in_plan=False, is_migrated=True) + repo = pulp2distributor.pulp2_repos.filter( + not_in_plan=False, is_migrated=True + ) repo_version = repo[0].pulp3_repository_version publication = repo_version.publication_set.filter(complete=True).first() if not publication: # create publication - publish('PULP_MANIFEST', repo_version.pk) + publish("PULP_MANIFEST", repo_version.pk) publication = repo_version.publication_set.filter(complete=True).first() # create distribution pulp2_config = pulp2distributor.pulp2_config base_config = cls.parse_base_config(pulp2distributor, pulp2_config) - base_config['base_path'] = pulp2_config.get('relative_url', pulp2distributor.pulp2_repo_id) - base_config['publication'] = publication + base_config["base_path"] = pulp2_config.get( + "relative_url", pulp2distributor.pulp2_repo_id + ) + base_config["publication"] = publication distribution, created = FileDistribution.objects.update_or_create( - name=base_config['name'], - base_path=base_config['base_path'], - defaults=base_config) + name=base_config["name"], + base_path=base_config["base_path"], + defaults=base_config, + ) return publication, distribution, created diff --git a/pulp_2to3_migration/app/plugin/migrator.py b/pulp_2to3_migration/app/plugin/migrator.py index 327ed4a0..ac73f9f7 100644 --- a/pulp_2to3_migration/app/plugin/migrator.py +++ b/pulp_2to3_migration/app/plugin/migrator.py @@ -28,6 +28,7 @@ class Pulp2to3PluginMigrator: Optional. """ + mutable_content_models = {} premigrate_hook = {} artifactless_types = {} diff --git a/pulp_2to3_migration/app/plugin/repository.py b/pulp_2to3_migration/app/plugin/repository.py index 82d11622..c2c28281 100644 --- a/pulp_2to3_migration/app/plugin/repository.py +++ b/pulp_2to3_migration/app/plugin/repository.py @@ -13,6 +13,7 @@ class Pulp2to3Importer: pulp3_remote_models(list): a list of models for Remotes which plugin supports """ + pulp3_remote_models = [] class Meta: @@ -25,49 +26,52 @@ def parse_base_config(pulp2importer, pulp2_config): """ base_config = {} proxy_url, credentials, host = None, None, None - pulp2_proxy_host = pulp2_config.get('proxy_host') - pulp2_proxy_port = pulp2_config.get('proxy_port') - pulp2_proxy_username = pulp2_config.get('proxy_username') - pulp2_proxy_password = pulp2_config.get('proxy_password') + pulp2_proxy_host = pulp2_config.get("proxy_host") + pulp2_proxy_port = pulp2_config.get("proxy_port") + pulp2_proxy_username = pulp2_config.get("proxy_username") + pulp2_proxy_password = pulp2_config.get("proxy_password") if pulp2_proxy_username: - credentials = '{}:{}'.format(pulp2_proxy_username, pulp2_proxy_password) + credentials = "{}:{}".format(pulp2_proxy_username, pulp2_proxy_password) if pulp2_proxy_host: parsed_url = urlparse(pulp2_proxy_host) scheme = parsed_url.scheme host = parsed_url.hostname if pulp2_proxy_port: - host += ':{}'.format(pulp2_proxy_port) + host += ":{}".format(pulp2_proxy_port) if credentials: - proxy_url = '{}://{}@{}'.format(scheme, credentials, host) + proxy_url = "{}://{}@{}".format(scheme, credentials, host) else: - proxy_url = '{}://{}'.format(scheme, host) - remote_name = '{}-{}'.format(pulp2importer.pulp2_object_id, - pulp2importer.pulp2_repo_id) - base_config['proxy_url'] = proxy_url - username = pulp2_config.get('basic_auth_username') - password = pulp2_config.get('basic_auth_password') - feed = pulp2_config.get('feed', '') # what to do if there is no feed? - if feed and '@' in feed: + proxy_url = "{}://{}".format(scheme, host) + remote_name = "{}-{}".format( + pulp2importer.pulp2_object_id, pulp2importer.pulp2_repo_id + ) + base_config["proxy_url"] = proxy_url + username = pulp2_config.get("basic_auth_username") + password = pulp2_config.get("basic_auth_password") + feed = pulp2_config.get("feed", "") # what to do if there is no feed? + if feed and "@" in feed: # move out the credentials from the feed parsed_feed = urlparse(feed) if not username: username = parsed_feed.username if not password: password = parsed_feed.password - _, netloc_split = parsed_feed.netloc.rsplit('@', maxsplit=1) + _, netloc_split = parsed_feed.netloc.rsplit("@", maxsplit=1) feed = urlunparse(parsed_feed._replace(netloc=netloc_split)) - base_config['url'] = feed - base_config['username'] = username - base_config['password'] = password + base_config["url"] = feed + base_config["username"] = username + base_config["password"] = password - base_config['ca_cert'] = pulp2_config.get('ssl_ca_cert') - base_config['client_cert'] = pulp2_config.get('ssl_client_cert') - base_config['client_key'] = pulp2_config.get('ssl_client_key') + base_config["ca_cert"] = pulp2_config.get("ssl_ca_cert") + base_config["client_cert"] = pulp2_config.get("ssl_client_cert") + base_config["client_key"] = pulp2_config.get("ssl_client_key") # True by default? - base_config['tls_validation'] = pulp2_config.get('ssl_validation', True) - base_config['download_concurrency'] = pulp2_config.get('max_downloads') or 20 - policy = PULP_2TO3_POLICIES.get(pulp2_config.get('download_policy', 'immediate')) - base_config['policy'] = policy + base_config["tls_validation"] = pulp2_config.get("ssl_validation", True) + base_config["download_concurrency"] = pulp2_config.get("max_downloads") or 20 + policy = PULP_2TO3_POLICIES.get( + pulp2_config.get("download_policy", "immediate") + ) + base_config["policy"] = policy return base_config, remote_name @classmethod @@ -97,6 +101,7 @@ class Pulp2to3Distributor: pulp3_distribution_models(list): a list of models for Distributions which plugin supports """ + pulp3_publication_models = [] pulp3_distribution_models = [] @@ -109,9 +114,10 @@ def parse_base_config(pulp2distributor, pulp2_config): Parse and return basic config. """ base_config = {} - name = '{}-{}'.format(pulp2distributor.pulp2_object_id, - pulp2distributor.pulp2_repo_id) - base_config['name'] = name + name = "{}-{}".format( + pulp2distributor.pulp2_object_id, pulp2distributor.pulp2_repo_id + ) + base_config["name"] = name return base_config @classmethod @@ -181,8 +187,9 @@ def is_different_relative_url(pulp2distributor): if not pulp2distributor.pulp3_distribution: return True - new_base_path = pulp2distributor.pulp2_config.get('relative_url', - pulp2distributor.pulp2_repo_id) + new_base_path = pulp2distributor.pulp2_config.get( + "relative_url", pulp2distributor.pulp2_repo_id + ) current_base_path = pulp2distributor.pulp3_distribution.base_path if new_base_path != current_base_path: return True diff --git a/pulp_2to3_migration/app/plugin/rpm/comps_utils.py b/pulp_2to3_migration/app/plugin/rpm/comps_utils.py index 8b085b60..07d416f5 100644 --- a/pulp_2to3_migration/app/plugin/rpm/comps_utils.py +++ b/pulp_2to3_migration/app/plugin/rpm/comps_utils.py @@ -21,7 +21,7 @@ def langpacks_to_libcomps(obj): """ strdict = libcomps.StrDict() for dct in obj.matches: - strdict[dct['name']] = dct['install'] + strdict[dct["name"]] = dct["install"] return strdict @@ -113,7 +113,7 @@ def _packages_to_grplist(packages): A list """ - return [{'name': pkg, 'default': False} for pkg in packages] + return [{"name": pkg, "default": False} for pkg in packages] def _list_to_pkglist(packages): @@ -156,11 +156,10 @@ def _packages_to_optionlist(packages): """ option_list = [] for pkg in packages: - if not isinstance(pkg['default'], bool): - if pkg['default'].lower() == 'false': - pkg['default'] = False - elif pkg['default'].lower() == 'true': - pkg['default'] = True - option_list.append({'name': pkg['group'], - 'default': pkg['default']}) + if not isinstance(pkg["default"], bool): + if pkg["default"].lower() == "false": + pkg["default"] = False + elif pkg["default"].lower() == "true": + pkg["default"] = True + option_list.append({"name": pkg["group"], "default": pkg["default"]}) return option_list diff --git a/pulp_2to3_migration/app/plugin/rpm/erratum.py b/pulp_2to3_migration/app/plugin/rpm/erratum.py index eb3fdb1d..7bb602d1 100644 --- a/pulp_2to3_migration/app/plugin/rpm/erratum.py +++ b/pulp_2to3_migration/app/plugin/rpm/erratum.py @@ -29,26 +29,27 @@ def get_pulp2_filtered_collections(pulp2erratum, repo_pkg_nevra, repo_module_nsv A pkglist to migrate """ + def get_module_nsvca(module): - return (module['name'], - module['stream'], - int(module['version']), - module['context'], - module['arch']) + return ( + module["name"], + module["stream"], + int(module["version"]), + module["context"], + module["arch"], + ) def get_pkg_nevra(pkg): - return (pkg['name'], - pkg['epoch'] or '0', - pkg['version'], - pkg['release'], - pkg['arch']) + return ( + pkg["name"], + pkg["epoch"] or "0", + pkg["version"], + pkg["release"], + pkg["arch"], + ) filtered_pkglist = [] - default_collection = { - 'name': 'default', - 'short': 'def', - 'packages': [] - } + default_collection = {"name": "default", "short": "def", "packages": []} filtered_pkglist.append(default_collection) if not repo_pkg_nevra: # If there are no packages in a repo, no pkglist will ever contain packages. @@ -60,7 +61,7 @@ def get_pkg_nevra(pkg): seen_non_modular_packages = set() seen_modules = set() for collection in pulp2erratum.pkglist: - module = collection.get('module') + module = collection.get("module") if module: # inside a modular collection nsvca = get_module_nsvca(module) @@ -69,29 +70,29 @@ def get_pkg_nevra(pkg): continue seen_modules.add(nsvca) current_collection = { - 'name': collection.get('name'), - 'short': collection.get('short'), - 'module': module, - 'packages': [] + "name": collection.get("name"), + "short": collection.get("short"), + "module": module, + "packages": [], } filtered_pkglist.append(current_collection) else: # the first and default collection collects the non-modular packages current_collection = filtered_pkglist[0] - current_collection['name'] = collection.get('name') - current_collection['short'] = collection.get('short') + current_collection["name"] = collection.get("name") + current_collection["short"] = collection.get("short") - for package in collection.get('packages', []): + for package in collection.get("packages", []): if not module: # only non-modular packages are tracked; # modular packages are not tracked because the same package can be present # in different modules and duplicated modules are already filtered out. - if package['filename'] in seen_non_modular_packages: + if package["filename"] in seen_non_modular_packages: continue - seen_non_modular_packages.add(package['filename']) + seen_non_modular_packages.add(package["filename"]) nevra = get_pkg_nevra(package) if nevra in repo_pkg_nevra: - current_collection['packages'].append(package) + current_collection["packages"].append(package) return filtered_pkglist @@ -114,9 +115,9 @@ def get_package_checksum(errata_pkg): If found, a tuple with a checksum type id in createrepo_c and a checksum itself """ - checksums = errata_pkg.get('sum', []) - checksum_type_v2 = errata_pkg.get('type') - checksum_v2 = errata_pkg.get('sums') + checksums = errata_pkg.get("sum", []) + checksum_type_v2 = errata_pkg.get("type") + checksum_v2 = errata_pkg.get("sums") if checksum_type_v2 and checksum_v2: checksums.extend([checksum_type_v2, checksum_v2]) @@ -154,7 +155,7 @@ def get_bool(value): """ if value: - if isinstance(value, str) and value.lower() == 'true': + if isinstance(value, str) and value.lower() == "true": return True if isinstance(value, int): return bool(value) @@ -183,12 +184,12 @@ def get_datetime(datetime_str): """ # remove UTC part - if datetime_str.endswith(' UTC'): + if datetime_str.endswith(" UTC"): datetime_str = datetime_str[:-4] # if it's as short as '%Y-%m-%d', try adding time so `parse_datetime` could parse it if len(datetime_str.strip()) == 10: - datetime_str = f'{datetime_str.strip()} 00:00' + datetime_str = f"{datetime_str.strip()} 00:00" datetime_obj = parse_datetime(datetime_str) if not datetime_obj: @@ -199,7 +200,9 @@ def get_datetime(datetime_str): pass # We don't know what else to do with it - _logger.warn(f'Unsupported datetime format {datetime_str}, resetting to 1970-01-01 00:00') + _logger.warn( + f"Unsupported datetime format {datetime_str}, resetting to 1970-01-01 00:00" + ) datetime_obj = datetime.datetime(1970, 1, 1, 0, 0) return datetime_obj diff --git a/pulp_2to3_migration/app/plugin/rpm/migrator.py b/pulp_2to3_migration/app/plugin/rpm/migrator.py index 9378ec87..10a9657e 100644 --- a/pulp_2to3_migration/app/plugin/rpm/migrator.py +++ b/pulp_2to3_migration/app/plugin/rpm/migrator.py @@ -72,67 +72,68 @@ class RpmMigrator(Pulp2to3PluginMigrator): importer_migrators(dict): {'importer_type_id': 'pulp_2to3 importer interface/migrator'} """ - pulp2_plugin = 'rpm' + + pulp2_plugin = "rpm" pulp2_content_models = { - 'rpm': pulp2_models.RPM, - 'srpm': pulp2_models.SRPM, - 'distribution': pulp2_models.Distribution, - 'erratum': pulp2_models.Errata, - 'modulemd': pulp2_models.Modulemd, - 'modulemd_defaults': pulp2_models.ModulemdDefaults, - 'yum_repo_metadata_file': pulp2_models.YumMetadataFile, - 'package_langpacks': pulp2_models.PackageLangpacks, - 'package_group': pulp2_models.PackageGroup, - 'package_category': pulp2_models.PackageCategory, - 'package_environment': pulp2_models.PackageEnvironment, + "rpm": pulp2_models.RPM, + "srpm": pulp2_models.SRPM, + "distribution": pulp2_models.Distribution, + "erratum": pulp2_models.Errata, + "modulemd": pulp2_models.Modulemd, + "modulemd_defaults": pulp2_models.ModulemdDefaults, + "yum_repo_metadata_file": pulp2_models.YumMetadataFile, + "package_langpacks": pulp2_models.PackageLangpacks, + "package_group": pulp2_models.PackageGroup, + "package_category": pulp2_models.PackageCategory, + "package_environment": pulp2_models.PackageEnvironment, } - pulp2_collection = 'units_rpm' - pulp3_plugin = 'pulp_rpm' + pulp2_collection = "units_rpm" + pulp3_plugin = "pulp_rpm" pulp3_repository = RpmRepository - content_models = OrderedDict([ - ('rpm', Pulp2Rpm), - ('srpm', Pulp2Srpm), - ('distribution', Pulp2Distribution), - ('erratum', Pulp2Erratum), - ('modulemd', Pulp2Modulemd), - ('modulemd_defaults', Pulp2ModulemdDefaults), - ('yum_repo_metadata_file', Pulp2YumRepoMetadataFile), - ('package_langpacks', Pulp2PackageLangpacks), - ('package_group', Pulp2PackageGroup), - ('package_category', Pulp2PackageCategory), - ('package_environment', Pulp2PackageEnvironment), - ]) + content_models = OrderedDict( + [ + ("rpm", Pulp2Rpm), + ("srpm", Pulp2Srpm), + ("distribution", Pulp2Distribution), + ("erratum", Pulp2Erratum), + ("modulemd", Pulp2Modulemd), + ("modulemd_defaults", Pulp2ModulemdDefaults), + ("yum_repo_metadata_file", Pulp2YumRepoMetadataFile), + ("package_langpacks", Pulp2PackageLangpacks), + ("package_group", Pulp2PackageGroup), + ("package_category", Pulp2PackageCategory), + ("package_environment", Pulp2PackageEnvironment), + ] + ) mutable_content_models = { - 'erratum': Pulp2Erratum, - 'modulemd': Pulp2Modulemd, - 'modulemd_defaults': Pulp2ModulemdDefaults, + "erratum": Pulp2Erratum, + "modulemd": Pulp2Modulemd, + "modulemd_defaults": Pulp2ModulemdDefaults, } importer_migrators = { - 'yum_importer': RpmImporter, + "yum_importer": RpmImporter, } distributor_migrators = { - 'yum_distributor': RpmDistributor, + "yum_distributor": RpmDistributor, } lazy_types = { - 'distribution': Pulp2Distribution, - 'rpm': Pulp2Rpm, - 'srpm': Pulp2Srpm, + "distribution": Pulp2Distribution, + "rpm": Pulp2Rpm, + "srpm": Pulp2Srpm, } future_types = { - 'rpm': Pulp2Rpm, + "rpm": Pulp2Rpm, } artifactless_types = { - 'erratum': Pulp2Erratum, - 'package_langpacks': Pulp2PackageLangpacks, - 'package_group': Pulp2PackageGroup, - 'package_category': Pulp2PackageCategory, - 'package_environment': Pulp2PackageEnvironment, - } - multi_artifact_types = { - 'distribution': Pulp2Distribution + "erratum": Pulp2Erratum, + "package_langpacks": Pulp2PackageLangpacks, + "package_group": Pulp2PackageGroup, + "package_category": Pulp2PackageCategory, + "package_environment": Pulp2PackageEnvironment, } + multi_artifact_types = {"distribution": Pulp2Distribution} premigrate_hook = { - 'yum_repo_metadata_file': exclude_unsupported_metadata, + "yum_repo_metadata_file": exclude_unsupported_metadata, } @classmethod @@ -193,6 +194,7 @@ async def run(self): Relate each item in the input queue to objects specified on the DeclarativeContent. """ async for batch in self.batches(): + def process_batch(): modulemd_packages_batch = [] with transaction.atomic(): @@ -202,9 +204,11 @@ def process_batch(): modulemd_packages_batch.extend(thru) ModulemdPackages = Modulemd.packages.through - ModulemdPackages.objects.bulk_create(objs=modulemd_packages_batch, - ignore_conflicts=True, - batch_size=DEFAULT_BATCH_SIZE) + ModulemdPackages.objects.bulk_create( + objs=modulemd_packages_batch, + ignore_conflicts=True, + batch_size=DEFAULT_BATCH_SIZE, + ) await sync_to_async(process_batch)() for dc in batch: @@ -231,12 +235,15 @@ def relate_packages_to_module(self, module_dc): version=nevra[2], release=nevra[3], arch=nevra[4], - is_modular=True) + is_modular=True, + ) packages_list = [] if pq: - packages_list = Package.objects.filter(pq).only( - 'pk', 'name', 'epoch', 'version', 'release', 'arch' - ).iterator() + packages_list = ( + Package.objects.filter(pq) + .only("pk", "name", "epoch", "version", "release", "arch") + .iterator() + ) thru = [] # keep track of rpm nevra for which we already created a relation with module. @@ -246,6 +253,10 @@ def relate_packages_to_module(self, module_dc): for pkg in packages_list: nevra = pkg.nevra if nevra not in already_related: - thru.append(ModulemdPackages(package_id=pkg.pk, modulemd_id=module_dc.content.pk)) + thru.append( + ModulemdPackages( + package_id=pkg.pk, modulemd_id=module_dc.content.pk + ) + ) already_related.append(nevra) return thru diff --git a/pulp_2to3_migration/app/plugin/rpm/package_utils.py b/pulp_2to3_migration/app/plugin/rpm/package_utils.py index 6eae7419..ca155674 100644 --- a/pulp_2to3_migration/app/plugin/rpm/package_utils.py +++ b/pulp_2to3_migration/app/plugin/rpm/package_utils.py @@ -26,9 +26,9 @@ def nevra(name): raise ValueError(msg) arch_dot_pos = name.rfind(".") - arch = name[arch_dot_pos + 1:] + arch = name[arch_dot_pos + 1 :] - return nevr(name[:arch_dot_pos]) + (arch, ) + return nevr(name[:arch_dot_pos]) + (arch,) def nevr(name): @@ -50,12 +50,12 @@ def nevr(name): raise ValueError(msg) release_dash_pos = name.rfind("-") - release = name[release_dash_pos + 1:] + release = name[release_dash_pos + 1 :] name_epoch_version = name[:release_dash_pos] name_dash_pos = name_epoch_version.rfind("-") package_name = name_epoch_version[:name_dash_pos] - epoch_version = name_epoch_version[name_dash_pos + 1:].split(":") + epoch_version = name_epoch_version[name_dash_pos + 1 :].split(":") if len(epoch_version) == 1: epoch = 0 version = epoch_version[0] diff --git a/pulp_2to3_migration/app/plugin/rpm/pulp2_models.py b/pulp_2to3_migration/app/plugin/rpm/pulp2_models.py index c9f0be93..8d567fcd 100644 --- a/pulp_2to3_migration/app/plugin/rpm/pulp2_models.py +++ b/pulp_2to3_migration/app/plugin/rpm/pulp2_models.py @@ -34,7 +34,7 @@ class NonMetadataPackage(FileContentUnit): release_sort_index = StringField() # not used in the migration plugin meta = { - 'abstract': True, + "abstract": True, } @@ -78,14 +78,30 @@ class RpmBase(NonMetadataPackage): requires = ListField() # not used in the migration plugin recommends = ListField() # not used in the migration plugin - unit_key_fields = ('name', 'epoch', 'version', 'release', 'arch', 'checksumtype', 'checksum') + unit_key_fields = ( + "name", + "epoch", + "version", + "release", + "arch", + "checksumtype", + "checksum", + ) meta = { - 'indexes': [ - "name", "epoch", "version", "release", "arch", "filename", "checksum", "checksumtype", - "version_sort_index", ("version_sort_index", "release_sort_index") + "indexes": [ + "name", + "epoch", + "version", + "release", + "arch", + "filename", + "checksum", + "checksumtype", + "version_sort_index", + ("version_sort_index", "release_sort_index"), ], - 'abstract': True, + "abstract": True, } @@ -95,21 +111,22 @@ class RPM(RpmBase): It will become a Package content type in Pulp 3 world. """ - TYPE_ID = 'rpm' + + TYPE_ID = "rpm" # For backward compatibility - _ns = StringField(default='units_rpm') + _ns = StringField(default="units_rpm") _content_type_id = StringField(required=True, default=TYPE_ID) - unit_display_name = 'RPM' - unit_description = 'RPM' - unit_referenced_types = ['erratum'] + unit_display_name = "RPM" + unit_description = "RPM" + unit_referenced_types = ["erratum"] is_modular = BooleanField(default=False) meta = { - 'collection': 'units_rpm', - 'allow_inheritance': False, + "collection": "units_rpm", + "allow_inheritance": False, } @@ -119,18 +136,17 @@ class SRPM(RpmBase): It will become a Package content type in Pulp 3 world. """ - TYPE_ID = 'srpm' + + TYPE_ID = "srpm" # For backward compatibility - _ns = StringField(default='units_srpm') - _content_type_id = StringField(required=True, default='srpm') + _ns = StringField(default="units_srpm") + _content_type_id = StringField(required=True, default="srpm") - unit_display_name = 'SRPM' - unit_description = 'SRPM' + unit_display_name = "SRPM" + unit_description = "SRPM" - meta = { - 'collection': 'units_srpm', - 'allow_inheritance': False} + meta = {"collection": "units_srpm", "allow_inheritance": False} class Errata(ContentUnit): @@ -139,11 +155,12 @@ class Errata(ContentUnit): It will become an Advisory content type in Pulp 3 world. """ - TYPE_ID = 'erratum' + + TYPE_ID = "erratum" errata_id = StringField(required=True) status = StringField() - updated = StringField(required=True, default='') + updated = StringField(required=True, default="") description = StringField() issued = StringField() pushcount = StringField() @@ -151,7 +168,7 @@ class Errata(ContentUnit): reboot_suggested = BooleanField() relogin_suggested = BooleanField() restart_suggested = BooleanField() - errata_from = StringField(db_field='from') + errata_from = StringField(db_field="from") severity = StringField() rights = StringField() version = StringField() @@ -163,20 +180,27 @@ class Errata(ContentUnit): summary = StringField() # For backward compatibility - _ns = StringField(default='units_erratum') - _content_type_id = StringField(required=True, default='erratum') + _ns = StringField(default="units_erratum") + _content_type_id = StringField(required=True, default="erratum") - unit_key_fields = ('errata_id',) - unit_display_name = 'Erratum' - unit_description = 'Erratum advisory information' - unit_referenced_types = ['rpm'] + unit_key_fields = ("errata_id",) + unit_display_name = "Erratum" + unit_description = "Erratum advisory information" + unit_referenced_types = ["rpm"] meta = { - 'indexes': [ - "version", "release", "type", "status", "updated", "issued", "severity", "references" + "indexes": [ + "version", + "release", + "type", + "status", + "updated", + "issued", + "severity", + "references", ], - 'collection': 'units_erratum', - 'allow_inheritance': False, + "collection": "units_erratum", + "allow_inheritance": False, } @@ -190,17 +214,19 @@ class ErratumPkglist(Document): via sync or upload), no new pkglist is created i pulp 2. During migration all the pkglists related to an erratum are used and filtered out accordingly. """ + errata_id = StringField(required=True) repo_id = StringField(required=True) collections = ListField() - _ns = StringField(default='erratum_pkglists') + _ns = StringField(default="erratum_pkglists") - model_key_fields = ('errata_id', 'repo_id') - meta = {'collection': 'erratum_pkglists', - 'allow_inheritance': False, - 'indexes': ['errata_id', - {'fields': model_key_fields, 'unique': True}]} + model_key_fields = ("errata_id", "repo_id") + meta = { + "collection": "erratum_pkglists", + "allow_inheritance": False, + "indexes": ["errata_id", {"fields": model_key_fields, "unique": True}], + } class YumMetadataFile(FileContentUnit): @@ -217,26 +243,28 @@ class YumMetadataFile(FileContentUnit): checksum_type = StringField() # For backward compatibility - _ns = StringField(default='units_yum_repo_metadata_file') - _content_type_id = StringField(required=True, default='yum_repo_metadata_file') + _ns = StringField(default="units_yum_repo_metadata_file") + _content_type_id = StringField(required=True, default="yum_repo_metadata_file") - unit_key_fields = ('data_type', 'repo_id') - unit_display_name = 'YUM Repository Metadata File' - unit_description = 'YUM Repository Metadata File' + unit_key_fields = ("data_type", "repo_id") + unit_display_name = "YUM Repository Metadata File" + unit_description = "YUM Repository Metadata File" - TYPE_ID = 'yum_repo_metadata_file' + TYPE_ID = "yum_repo_metadata_file" meta = { - 'indexes': ['data_type'], - 'collection': 'units_yum_repo_metadata_file', - 'allow_inheritance': False} + "indexes": ["data_type"], + "collection": "units_yum_repo_metadata_file", + "allow_inheritance": False, + } class Modulemd(FileContentUnit): """ A model for Pulp 2 Modulemd content type. """ - TYPE_ID = 'modulemd' + + TYPE_ID = "modulemd" # Unit key fields NSVCA name = StringField(required=True) @@ -253,23 +281,32 @@ class Modulemd(FileContentUnit): dependencies = ListField() # For backward compatibility - _ns = StringField(default='units_modulemd') + _ns = StringField(default="units_modulemd") _content_type_id = StringField(required=True, default=TYPE_ID) - unit_key_fields = ('name', 'stream', 'version', 'context', 'arch', ) - unit_display_name = 'Modulemd' - unit_description = 'Modulemd' + unit_key_fields = ( + "name", + "stream", + "version", + "context", + "arch", + ) + unit_display_name = "Modulemd" + unit_description = "Modulemd" - meta = {'collection': 'units_modulemd', - 'indexes': ['artifacts'], - 'allow_inheritance': False} + meta = { + "collection": "units_modulemd", + "indexes": ["artifacts"], + "allow_inheritance": False, + } class ModulemdDefaults(FileContentUnit): """ A model for Pulp 2 Modulemd content type. """ - TYPE_ID = 'modulemd_defaults' + + TYPE_ID = "modulemd_defaults" # Unit key fields name = StringField(required=True) @@ -281,16 +318,21 @@ class ModulemdDefaults(FileContentUnit): checksum = StringField() # For backward compatibility - _ns = StringField(default='units_modulemd_defaults') + _ns = StringField(default="units_modulemd_defaults") _content_type_id = StringField(required=True, default=TYPE_ID) - unit_key_fields = ('name', 'repo_id',) - unit_display_name = 'ModulemdDefaults' - unit_description = 'ModulemdDefaults' + unit_key_fields = ( + "name", + "repo_id", + ) + unit_display_name = "ModulemdDefaults" + unit_description = "ModulemdDefaults" - meta = {'collection': 'units_modulemd_defaults', - 'indexes': ['repo_id'], - 'allow_inheritance': False} + meta = { + "collection": "units_modulemd_defaults", + "indexes": ["repo_id"], + "allow_inheritance": False, + } class Distribution(FileContentUnit): @@ -299,11 +341,12 @@ class Distribution(FileContentUnit): A distribution tree is described by a file in root of an RPM repository named either "treeinfo" or ".treeinfo". """ - TYPE_ID = 'distribution' + + TYPE_ID = "distribution" distribution_id = StringField(required=True) family = StringField(required=True) - variant = StringField(default='') + variant = StringField(default="") version = StringField(required=True) arch = StringField(required=True) @@ -315,23 +358,26 @@ class Distribution(FileContentUnit): version_sort_index = StringField() # For backward compatibility - _ns = StringField(default='units_distribution') - _content_type_id = StringField(required=True, default='distribution') + _ns = StringField(default="units_distribution") + _content_type_id = StringField(required=True, default="distribution") - unit_key_fields = ('distribution_id', 'family', 'variant', 'version', 'arch') - unit_display_name = 'Distribution' - unit_description = 'Kickstart trees and all accompanying files' + unit_key_fields = ("distribution_id", "family", "variant", "version", "arch") + unit_display_name = "Distribution" + unit_description = "Kickstart trees and all accompanying files" - meta = {'collection': 'units_distribution', - 'indexes': ['distribution_id', 'family', 'variant', 'version', 'arch'], - 'allow_inheritance': False} + meta = { + "collection": "units_distribution", + "indexes": ["distribution_id", "family", "variant", "version", "arch"], + "allow_inheritance": False, + } class PackageGroup(ContentUnit): """ A model for Pulp 2 PackageGroup content type. """ - TYPE_ID = 'package_group' + + TYPE_ID = "package_group" package_group_id = StringField(required=True) repo_id = StringField(required=True) @@ -350,27 +396,34 @@ class PackageGroup(ContentUnit): conditional_package_names = ListField() # For backward compatibility - _ns = StringField(default='units_package_group') + _ns = StringField(default="units_package_group") _content_type_id = StringField(required=True, default=TYPE_ID) - unit_key_fields = ('package_group_id', 'repo_id') - unit_display_name = 'Package Group' - unit_description = 'Yum Package group information' + unit_key_fields = ("package_group_id", "repo_id") + unit_display_name = "Package Group" + unit_description = "Yum Package group information" meta = { - 'indexes': [ - 'package_group_id', 'repo_id', 'name', 'mandatory_package_names', - 'conditional_package_names', 'optional_package_names', 'default_package_names' + "indexes": [ + "package_group_id", + "repo_id", + "name", + "mandatory_package_names", + "conditional_package_names", + "optional_package_names", + "default_package_names", ], - 'collection': 'units_package_group', - 'allow_inheritance': False} + "collection": "units_package_group", + "allow_inheritance": False, + } class PackageCategory(ContentUnit): """ A model for Pulp 2 PackageCategory content type. """ - TYPE_ID = 'package_category' + + TYPE_ID = "package_category" package_category_id = StringField(required=True) repo_id = StringField(required=True) @@ -383,26 +436,26 @@ class PackageCategory(ContentUnit): name = StringField() # For backward compatibility - _ns = StringField(default='units_package_category') + _ns = StringField(default="units_package_category") _content_type_id = StringField(required=True, default=TYPE_ID) - unit_key_fields = ('package_category_id', 'repo_id') - unit_display_name = 'Package Category' - unit_description = 'Yum Package category information' + unit_key_fields = ("package_category_id", "repo_id") + unit_display_name = "Package Category" + unit_description = "Yum Package category information" meta = { - 'indexes': [ - 'package_category_id', 'repo_id', 'name', 'packagegroupids' - ], - 'collection': 'units_package_category', - 'allow_inheritance': False} + "indexes": ["package_category_id", "repo_id", "name", "packagegroupids"], + "collection": "units_package_category", + "allow_inheritance": False, + } class PackageEnvironment(ContentUnit): """ A model for Pulp 2 PackageEnvironment content type. """ - TYPE_ID = 'package_environment' + + TYPE_ID = "package_environment" package_environment_id = StringField(required=True) repo_id = StringField(required=True) @@ -416,34 +469,34 @@ class PackageEnvironment(ContentUnit): name = StringField() # For backward compatibility - _ns = StringField(default='units_package_environment') + _ns = StringField(default="units_package_environment") _content_type_id = StringField(required=True, default=TYPE_ID) - unit_key_fields = ('package_environment_id', 'repo_id') - unit_display_name = 'Package Environment' - unit_description = 'Yum Package environment information' + unit_key_fields = ("package_environment_id", "repo_id") + unit_display_name = "Package Environment" + unit_description = "Yum Package environment information" meta = { - 'indexes': ['package_environment_id', 'repo_id', 'name', 'group_ids'], - 'collection': 'units_package_environment', - 'allow_inheritance': False} + "indexes": ["package_environment_id", "repo_id", "name", "group_ids"], + "collection": "units_package_environment", + "allow_inheritance": False, + } class PackageLangpacks(ContentUnit): """ A model for Pulp 2 PackageLangpacks content type. """ - TYPE_ID = 'package_langpacks' + + TYPE_ID = "package_langpacks" repo_id = StringField(required=True) matches = ListField() # For backward compatibility - _ns = StringField(default='units_package_langpacks') + _ns = StringField(default="units_package_langpacks") _content_type_id = StringField(required=True, default=TYPE_ID) - unit_key_fields = ('repo_id',) + unit_key_fields = ("repo_id",) - meta = { - 'collection': 'units_package_langpacks', - 'allow_inheritance': False} + meta = {"collection": "units_package_langpacks", "allow_inheritance": False} diff --git a/pulp_2to3_migration/app/plugin/rpm/pulp_2to3_models.py b/pulp_2to3_migration/app/plugin/rpm/pulp_2to3_models.py index 472b462a..d60a100f 100644 --- a/pulp_2to3_migration/app/plugin/rpm/pulp_2to3_models.py +++ b/pulp_2to3_migration/app/plugin/rpm/pulp_2to3_models.py @@ -9,10 +9,7 @@ from django.db import models from pulp_2to3_migration.app.constants import DEFAULT_BATCH_SIZE -from pulp_2to3_migration.app.models import ( - Pulp2to3Content, - Pulp2RepoContent -) +from pulp_2to3_migration.app.models import Pulp2to3Content, Pulp2RepoContent from pulp_rpm.app.comps import dict_digest from pulp_rpm.app.advisory import hash_update_record @@ -32,10 +29,7 @@ UpdateReference, ) -from pulp_rpm.app.kickstart.treeinfo import ( - PulpTreeInfo, - TreeinfoData -) +from pulp_rpm.app.kickstart.treeinfo import PulpTreeInfo, TreeinfoData from . import pulp2_models @@ -43,32 +37,34 @@ langpacks_to_libcomps, pkg_cat_to_libcomps, pkg_env_to_libcomps, - pkg_grp_to_libcomps + pkg_grp_to_libcomps, ) from .erratum import ( get_bool, get_datetime, get_package_checksum, - get_pulp2_filtered_collections + get_pulp2_filtered_collections, ) from .xml_utils import get_cr_obj -SRPM_UNIT_FIELDS = set([ - 'name', - 'epoch', - 'version', - 'release', - 'arch', - 'checksum', - 'checksumtype', - 'repodata', - 'size', - 'filename', - 'pk', -]) +SRPM_UNIT_FIELDS = set( + [ + "name", + "epoch", + "version", + "release", + "arch", + "checksum", + "checksumtype", + "repodata", + "size", + "filename", + "pk", + ] +) -RPM_UNIT_FIELDS = SRPM_UNIT_FIELDS | set(['is_modular']) +RPM_UNIT_FIELDS = SRPM_UNIT_FIELDS | set(["is_modular"]) class Pulp2RpmBase(Pulp2to3Content): @@ -118,30 +114,38 @@ def pre_migrate_content_detail(cls, content_batch): content_batch(list of Pulp2Content): pre-migrated generic data for Pulp 2 content. """ - pulp2_id_obj_map = {pulp2content.pulp2_id: pulp2content for pulp2content in content_batch} + pulp2_id_obj_map = { + pulp2content.pulp2_id: pulp2content for pulp2content in content_batch + } pulp2_ids = pulp2_id_obj_map.keys() - pulp2_content_batch = cls.pulp2_model.objects.filter(id__in=pulp2_ids).as_pymongo().only( - *cls.unit_fields) + pulp2_content_batch = ( + cls.pulp2_model.objects.filter(id__in=pulp2_ids) + .as_pymongo() + .only(*cls.unit_fields) + ) pulp2rpm_to_save = [] for rpm in pulp2_content_batch: pulp2rpm_to_save.append( - cls(name=rpm['name'], - epoch=rpm['epoch'], - version=rpm['version'], - release=rpm['release'], - arch=rpm['arch'], - checksum=rpm['checksum'], - checksumtype=rpm['checksumtype'], - primary_template_gz=rpm['repodata']['primary'], - filelists_template_gz=rpm['repodata']['filelists'], - other_template_gz=rpm['repodata']['other'], - is_modular=rpm.get('is_modular', False), - size=rpm['size'], - filename=rpm['filename'], - pulp2content=pulp2_id_obj_map[rpm['_id']]) + cls( + name=rpm["name"], + epoch=rpm["epoch"], + version=rpm["version"], + release=rpm["release"], + arch=rpm["arch"], + checksum=rpm["checksum"], + checksumtype=rpm["checksumtype"], + primary_template_gz=rpm["repodata"]["primary"], + filelists_template_gz=rpm["repodata"]["filelists"], + other_template_gz=rpm["repodata"]["other"], + is_modular=rpm.get("is_modular", False), + size=rpm["size"], + filename=rpm["filename"], + pulp2content=pulp2_id_obj_map[rpm["_id"]], + ) ) - cls.objects.bulk_create(pulp2rpm_to_save, ignore_conflicts=True, - batch_size=DEFAULT_BATCH_SIZE) + cls.objects.bulk_create( + pulp2rpm_to_save, ignore_conflicts=True, batch_size=DEFAULT_BATCH_SIZE + ) def create_pulp3_content(self): """ @@ -149,7 +153,7 @@ def create_pulp3_content(self): """ cr_package = get_cr_obj(self) pkg_dict = Package.createrepo_to_dict(cr_package) - pkg_dict['is_modular'] = self.is_modular + pkg_dict["is_modular"] = self.is_modular return (Package(**pkg_dict), None) @@ -161,13 +165,20 @@ class Pulp2Rpm(Pulp2RpmBase): unit_fields = RPM_UNIT_FIELDS pulp2_model = pulp2_models.RPM - pulp2_type = 'rpm' + pulp2_type = "rpm" class Meta: unique_together = ( - 'name', 'epoch', 'version', 'release', 'arch', 'checksumtype', 'checksum', - 'pulp2content') - default_related_name = 'rpm_detail_model' + "name", + "epoch", + "version", + "release", + "arch", + "checksumtype", + "checksum", + "pulp2content", + ) + default_related_name = "rpm_detail_model" class Pulp2Srpm(Pulp2RpmBase): @@ -178,13 +189,20 @@ class Pulp2Srpm(Pulp2RpmBase): unit_fields = SRPM_UNIT_FIELDS pulp2_model = pulp2_models.SRPM - pulp2_type = 'srpm' + pulp2_type = "srpm" class Meta: unique_together = ( - 'name', 'epoch', 'version', 'release', 'arch', 'checksumtype', 'checksum', - 'pulp2content') - default_related_name = 'srpm_detail_model' + "name", + "epoch", + "version", + "release", + "arch", + "checksumtype", + "checksum", + "pulp2content", + ) + default_related_name = "srpm_detail_model" class Pulp2Erratum(Pulp2to3Content): @@ -192,6 +210,7 @@ class Pulp2Erratum(Pulp2to3Content): Pulp 2to3 detail content model to store Pulp2 Errata content details. """ + # Required fields errata_id = models.TextField() updated = models.TextField() @@ -216,13 +235,13 @@ class Pulp2Erratum(Pulp2to3Content): solution = models.TextField(null=True) summary = models.TextField(null=True) - pulp2_type = 'erratum' + pulp2_type = "erratum" set_pulp2_repo = True cached_repo_data = {} class Meta: - unique_together = ('errata_id', 'repo_id') - default_related_name = 'erratum_detail_model' + unique_together = ("errata_id", "repo_id") + default_related_name = "erratum_detail_model" @classmethod def get_repo_data(cls, pulp2_repo): @@ -244,37 +263,48 @@ def get_repo_data(cls, pulp2_repo): repo_module_nsvca = [] # gather info about available packages - package_pulp2_ids = Pulp2RepoContent.objects.filter( - pulp2_repository=pulp2_repo, - pulp2_content_type_id='rpm' - ).only('pulp2_unit_id').values_list('pulp2_unit_id', flat=True) + package_pulp2_ids = ( + Pulp2RepoContent.objects.filter( + pulp2_repository=pulp2_repo, pulp2_content_type_id="rpm" + ) + .only("pulp2_unit_id") + .values_list("pulp2_unit_id", flat=True) + ) pulp2rpms = Pulp2Rpm.objects.filter( pulp2content__pulp2_id__in=package_pulp2_ids - ).values( - 'name', 'epoch', 'version', 'release', 'arch' - ) + ).values("name", "epoch", "version", "release", "arch") for pkg in pulp2rpms.iterator(): repo_pkg_nevra.append( - (pkg['name'], pkg.get('epoch', '0'), pkg['version'], pkg['release'], pkg['arch']) + ( + pkg["name"], + pkg.get("epoch", "0"), + pkg["version"], + pkg["release"], + pkg["arch"], + ) ) # gather info about available modules - modulemd_pulp2_ids = Pulp2RepoContent.objects.filter( - pulp2_repository=pulp2_repo, - pulp2_content_type_id='modulemd' - ).only('pulp2_unit_id').values_list('pulp2_unit_id', flat=True) + modulemd_pulp2_ids = ( + Pulp2RepoContent.objects.filter( + pulp2_repository=pulp2_repo, pulp2_content_type_id="modulemd" + ) + .only("pulp2_unit_id") + .values_list("pulp2_unit_id", flat=True) + ) pulp2modulemds = Pulp2Modulemd.objects.filter( pulp2content__pulp2_id__in=modulemd_pulp2_ids - ).values( - 'name', 'stream', 'version', 'context', 'arch' - ) + ).values("name", "stream", "version", "context", "arch") for module in pulp2modulemds.iterator(): repo_module_nsvca.append( ( - module['name'], module['stream'], module['version'], - module['context'], module['arch'] + module["name"], + module["stream"], + module["version"], + module["context"], + module["arch"], ) ) @@ -282,8 +312,8 @@ def get_repo_data(cls, pulp2_repo): # a repo it belongs to) cls.cached_repo_data.clear() cls.cached_repo_data[pulp2_repo.pk] = { - 'packages': repo_pkg_nevra, - 'modules': repo_module_nsvca + "packages": repo_pkg_nevra, + "modules": repo_module_nsvca, } return cls.cached_repo_data[pulp2_repo.pk] @@ -296,6 +326,7 @@ def pre_migrate_content_detail(cls, content_batch): content_batch(list of Pulp2Content): pre-migrated generic data for Pulp 2 content. """ + def get_pkglist(errata_id): """ Get a pkglist for a specified erratum. @@ -309,28 +340,38 @@ def get_pkglist(errata_id): Args: errata_id(str): Id of an erratum to get a pkglist for """ - match_stage = {'$match': {'errata_id': errata_id}} - unwind_collections_stage = {'$unwind': '$collections'} - unwind_packages_stage = {'$unwind': '$collections.packages'} + match_stage = {"$match": {"errata_id": errata_id}} + unwind_collections_stage = {"$unwind": "$collections"} + unwind_packages_stage = {"$unwind": "$collections.packages"} # Group all packages by their relation to a module specified in each collection. # All non-modular RPMs will be in a single collection. - group_stage = {'$group': {'_id': '$collections.module', - 'packages': {'$addToSet': '$collections.packages'}}} + group_stage = { + "$group": { + "_id": "$collections.module", + "packages": {"$addToSet": "$collections.packages"}, + } + } collections = pulp2_models.ErratumPkglist.objects.aggregate( - match_stage, unwind_collections_stage, unwind_packages_stage, group_stage, - allowDiskUse=True) + match_stage, + unwind_collections_stage, + unwind_packages_stage, + group_stage, + allowDiskUse=True, + ) pkglist = [] for collection_idx, collection in enumerate(collections): # To preserve the original format of a pkglist the 'short' and 'name' # keys are added. 'short' can be an empty string, collection 'name' # should be unique within an erratum. - item = {'packages': collection['packages'], - 'short': '', - 'name': 'collection-%s' % collection_idx} - if collection['_id']: - item['module'] = collection['_id'] + item = { + "packages": collection["packages"], + "short": "", + "name": "collection-%s" % collection_idx, + } + if collection["_id"]: + item["module"] = collection["_id"] pkglist.append(item) return pkglist @@ -340,11 +381,14 @@ def get_pkglist(errata_id): repo_id = pulp2content.pulp2_repo.pk pulp2_id_obj_map[pulp2content.pulp2_id][repo_id] = pulp2content pulp2_ids = pulp2_id_obj_map.keys() - pulp2_erratum_content_batch = pulp2_models.Errata.objects.filter(id__in=pulp2_ids) + pulp2_erratum_content_batch = pulp2_models.Errata.objects.filter( + id__in=pulp2_ids + ) for erratum in pulp2_erratum_content_batch: for repo_id, pulp2content in pulp2_id_obj_map[erratum.id].items(): pulp2erratum_to_save.append( - cls(errata_id=erratum.errata_id, + cls( + errata_id=erratum.errata_id, updated=erratum.updated, repo_id=repo_id, issued=erratum.issued, @@ -365,9 +409,12 @@ def get_pkglist(errata_id): title=erratum.title, solution=erratum.solution, summary=erratum.summary, - pulp2content=pulp2content)) - cls.objects.bulk_create(pulp2erratum_to_save, ignore_conflicts=True, - batch_size=DEFAULT_BATCH_SIZE) + pulp2content=pulp2content, + ) + ) + cls.objects.bulk_create( + pulp2erratum_to_save, ignore_conflicts=True, batch_size=DEFAULT_BATCH_SIZE + ) def get_collections(self): """ @@ -376,9 +423,7 @@ def get_collections(self): """ repo_data = Pulp2Erratum.get_repo_data(self.pulp2content.pulp2_repo) return get_pulp2_filtered_collections( - self, - repo_data['packages'], - repo_data['modules'] + self, repo_data["packages"], repo_data["modules"] ) def create_pulp3_content(self): @@ -406,30 +451,30 @@ def create_pulp3_content(self): collections = self.get_collections() for collection in collections: col = cr.UpdateCollection() - col.shortname = collection.get('short') - col.name = collection.get('name') - module = collection.get('module') + col.shortname = collection.get("short") + col.name = collection.get("name") + module = collection.get("module") if module: cr_module = cr.UpdateCollectionModule() - cr_module.name = module['name'] - cr_module.stream = module['stream'] - cr_module.version = int(module['version']) - cr_module.context = module['context'] - cr_module.arch = module['arch'] + cr_module.name = module["name"] + cr_module.stream = module["stream"] + cr_module.version = int(module["version"]) + cr_module.context = module["context"] + cr_module.arch = module["arch"] col.module = cr_module - for package in collection.get('packages', []): + for package in collection.get("packages", []): pkg = cr.UpdateCollectionPackage() - pkg.name = package['name'] - pkg.version = package['version'] - pkg.release = package['release'] - pkg.epoch = package['epoch'] - pkg.arch = package['arch'] - pkg.src = package.get('src') - pkg.filename = package['filename'] - pkg.reboot_suggested = get_bool(package.get('reboot_suggested')) - pkg.restart_suggested = get_bool(package.get('restart_suggested')) - pkg.relogin_suggested = get_bool(package.get('relogin_suggested')) + pkg.name = package["name"] + pkg.version = package["version"] + pkg.release = package["release"] + pkg.epoch = package["epoch"] + pkg.arch = package["arch"] + pkg.src = package.get("src") + pkg.filename = package["filename"] + pkg.reboot_suggested = get_bool(package.get("reboot_suggested")) + pkg.restart_suggested = get_bool(package.get("restart_suggested")) + pkg.relogin_suggested = get_bool(package.get("relogin_suggested")) checksum_tuple = get_package_checksum(package) if checksum_tuple: pkg.sum_type, pkg.sum = checksum_tuple @@ -439,15 +484,15 @@ def create_pulp3_content(self): for reference in self.references: ref = cr.UpdateReference() - ref.href = reference.get('href') - ref.id = reference.get('id') - ref.type = reference.get('type') - ref.title = reference.get('title') + ref.href = reference.get("href") + ref.id = reference.get("id") + ref.type = reference.get("type") + ref.title = reference.get("title") rec.append_reference(ref) update_record = UpdateRecord(**UpdateRecord.createrepo_to_dict(rec)) update_record.digest = hash_update_record(rec) - relations = {'collections': defaultdict(list), 'references': []} + relations = {"collections": defaultdict(list), "references": []} for collection in rec.collections: coll_dict = UpdateCollection.createrepo_to_dict(collection) @@ -456,12 +501,12 @@ def create_pulp3_content(self): for package in collection.packages: pkg_dict = UpdateCollectionPackage.createrepo_to_dict(package) pkg = UpdateCollectionPackage(**pkg_dict) - relations['collections'][coll].append(pkg) + relations["collections"][coll].append(pkg) for reference in rec.references: reference_dict = UpdateReference.createrepo_to_dict(reference) ref = UpdateReference(**reference_dict) - relations['references'].append(ref) + relations["references"].append(ref) return (update_record, relations) @@ -471,16 +516,17 @@ class Pulp2YumRepoMetadataFile(Pulp2to3Content): Pulp 2to3 detail content model to store pulp 2 yum_repo_metadata_file content details for Pulp 3 content creation. """ + data_type = models.CharField(max_length=20) checksum = models.CharField(max_length=128) checksum_type = models.CharField(max_length=6) repo_id = models.TextField() - pulp2_type = 'yum_repo_metadata_file' + pulp2_type = "yum_repo_metadata_file" class Meta: - unique_together = ('data_type', 'repo_id', 'pulp2content') - default_related_name = 'yum_repo_metadata_file_detail_model' + unique_together = ("data_type", "repo_id", "pulp2content") + default_related_name = "yum_repo_metadata_file_detail_model" @property def expected_digests(self): @@ -497,7 +543,7 @@ def relative_path_for_content_artifact(self): """Return relative path.""" path = self.pulp2content.pulp2_storage_path metadata_file_name = os.path.basename(path) - return os.path.join('repodata', metadata_file_name) + return os.path.join("repodata", metadata_file_name) @classmethod def pre_migrate_content_detail(cls, content_batch): @@ -508,29 +554,40 @@ def pre_migrate_content_detail(cls, content_batch): content_batch(list of Pulp2Content): pre-migrated generic data for Pulp 2 content. """ - pulp2_id_obj_map = {pulp2content.pulp2_id: pulp2content for pulp2content in content_batch} + pulp2_id_obj_map = { + pulp2content.pulp2_id: pulp2content for pulp2content in content_batch + } pulp2_ids = pulp2_id_obj_map.keys() - pulp2_metadata_content_batch = pulp2_models.YumMetadataFile.objects.filter(id__in=pulp2_ids) + pulp2_metadata_content_batch = pulp2_models.YumMetadataFile.objects.filter( + id__in=pulp2_ids + ) pulp2metadata_to_save = [ cls( data_type=meta.data_type, checksum=meta.checksum, checksum_type=meta.checksum_type, repo_id=meta.repo_id, - pulp2content=pulp2_id_obj_map[meta.id] - ) for meta in pulp2_metadata_content_batch.no_cache() + pulp2content=pulp2_id_obj_map[meta.id], + ) + for meta in pulp2_metadata_content_batch.no_cache() ] - cls.objects.bulk_create(pulp2metadata_to_save, ignore_conflicts=True, - batch_size=DEFAULT_BATCH_SIZE) + cls.objects.bulk_create( + pulp2metadata_to_save, ignore_conflicts=True, batch_size=DEFAULT_BATCH_SIZE + ) def create_pulp3_content(self): """ Create a Pulp 3 RepoMetadataFile unit for saving it later in a bulk operation. """ - return (RepoMetadataFile(data_type=self.data_type, - checksum=self.checksum, - checksum_type=self.checksum_type, - relative_path=self.relative_path_for_content_artifact), None,) + return ( + RepoMetadataFile( + data_type=self.data_type, + checksum=self.checksum, + checksum_type=self.checksum_type, + relative_path=self.relative_path_for_content_artifact, + ), + None, + ) class Pulp2Modulemd(Pulp2to3Content): @@ -549,17 +606,23 @@ class Pulp2Modulemd(Pulp2to3Content): checksum = models.TextField() dependencies = models.JSONField(default=list) - pulp2_type = 'modulemd' + pulp2_type = "modulemd" class Meta: unique_together = ( - 'name', 'stream', 'version', 'context', 'arch', 'pulp2content') - default_related_name = 'modulemd_detail_model' + "name", + "stream", + "version", + "context", + "arch", + "pulp2content", + ) + default_related_name = "modulemd_detail_model" @property def expected_digests(self): """Return expected digests.""" - return {'sha256': self.checksum} + return {"sha256": self.checksum} @property def expected_size(self): @@ -569,9 +632,9 @@ def expected_size(self): @property def relative_path_for_content_artifact(self): """Return relative path.""" - relative_path = '{}{}{}{}{}snippet'.format( - self.name, self.stream, self.version, - self.context, self.arch) + relative_path = "{}{}{}{}{}snippet".format( + self.name, self.stream, self.version, self.context, self.arch + ) return relative_path @@ -584,11 +647,14 @@ def pre_migrate_content_detail(cls, content_batch): content_batch(list of Pulp2Content): pre-migrated generic data for Pulp 2 content. """ - pulp2_id_obj_map = {pulp2content.pulp2_id: pulp2content for pulp2content in content_batch} + pulp2_id_obj_map = { + pulp2content.pulp2_id: pulp2content for pulp2content in content_batch + } pulp2_ids = pulp2_id_obj_map.keys() pulp2_content_batch = pulp2_models.Modulemd.objects.filter(id__in=pulp2_ids) pulp2modules_to_save = [ - cls(name=md.name, + cls( + name=md.name, stream=md.stream, version=md.version, context=md.context, @@ -596,18 +662,30 @@ def pre_migrate_content_detail(cls, content_batch): dependencies=md.dependencies or [], artifacts=md.artifacts, checksum=md.checksum, - pulp2content=pulp2_id_obj_map[md.id]) - for md in pulp2_content_batch] - cls.objects.bulk_create(pulp2modules_to_save, ignore_conflicts=True, - batch_size=DEFAULT_BATCH_SIZE) + pulp2content=pulp2_id_obj_map[md.id], + ) + for md in pulp2_content_batch + ] + cls.objects.bulk_create( + pulp2modules_to_save, ignore_conflicts=True, batch_size=DEFAULT_BATCH_SIZE + ) def create_pulp3_content(self): """ Create a Pulp 3 Module content for saving it later in a bulk operation. """ - return (Modulemd(name=self.name, stream=self.stream, version=self.version, - context=self.context, arch=self.arch, artifacts=self.artifacts, - dependencies=self.dependencies), None) + return ( + Modulemd( + name=self.name, + stream=self.stream, + version=self.version, + context=self.context, + arch=self.arch, + artifacts=self.artifacts, + dependencies=self.dependencies, + ), + None, + ) class Pulp2ModulemdDefaults(Pulp2to3Content): @@ -622,16 +700,16 @@ class Pulp2ModulemdDefaults(Pulp2to3Content): digest = models.TextField() repo_id = models.TextField() - pulp2_type = 'modulemd_defaults' + pulp2_type = "modulemd_defaults" class Meta: - unique_together = ('digest', 'repo_id', 'pulp2content') - default_related_name = 'modulemd_defaults_detail_model' + unique_together = ("digest", "repo_id", "pulp2content") + default_related_name = "modulemd_defaults_detail_model" @property def expected_digests(self): """Return expected digests.""" - return {'sha256': self.digest} + return {"sha256": self.digest} @property def expected_size(self): @@ -641,7 +719,7 @@ def expected_size(self): @property def relative_path_for_content_artifact(self): """Return relative path.""" - relative_path = '{}{}snippet'.format(self.module, self.stream) + relative_path = "{}{}snippet".format(self.module, self.stream) return relative_path @@ -655,39 +733,53 @@ def pre_migrate_content_detail(cls, content_batch): """ - pulp2_id_obj_map = {pulp2content.pulp2_id: pulp2content for pulp2content in content_batch} + pulp2_id_obj_map = { + pulp2content.pulp2_id: pulp2content for pulp2content in content_batch + } pulp2_ids = pulp2_id_obj_map.keys() - pulp2_content_batch = pulp2_models.ModulemdDefaults.objects.filter(id__in=pulp2_ids) + pulp2_content_batch = pulp2_models.ModulemdDefaults.objects.filter( + id__in=pulp2_ids + ) pulp2defaults_to_save = [] for defaults in pulp2_content_batch: kwargs = { - 'module': defaults.name, - 'digest': defaults.checksum, - 'repo_id': defaults.repo_id, - 'pulp2content': pulp2_id_obj_map[defaults.id] + "module": defaults.name, + "digest": defaults.checksum, + "repo_id": defaults.repo_id, + "pulp2content": pulp2_id_obj_map[defaults.id], } if defaults.stream: - kwargs['stream'] = defaults.stream + kwargs["stream"] = defaults.stream if defaults.profiles: if isinstance(defaults.profiles, str): # "normal path" - the data was BSON-encoded and then saved into MongoDB # as a utf-8 encoded string - kwargs['profiles'] = bson.decode(bytes(defaults.profiles, encoding='utf-8')) + kwargs["profiles"] = bson.decode( + bytes(defaults.profiles, encoding="utf-8") + ) elif isinstance(defaults.profiles, bytes): # "hotfix path" (#8893) - the data was BSON-encoded and then saved into MongoDB # as raw binary data - kwargs['profiles'] = bson.decode(defaults.profiles) + kwargs["profiles"] = bson.decode(defaults.profiles) pulp2defaults_to_save.append(cls(**kwargs)) - cls.objects.bulk_create(pulp2defaults_to_save, ignore_conflicts=True, - batch_size=DEFAULT_BATCH_SIZE) + cls.objects.bulk_create( + pulp2defaults_to_save, ignore_conflicts=True, batch_size=DEFAULT_BATCH_SIZE + ) def create_pulp3_content(self): """ Create a Pulp 3 Module content for saving it later in a bulk operation. """ - return (ModulemdDefaults(module=self.module, stream=self.stream, - profiles=self.profiles, digest=self.digest), None) + return ( + ModulemdDefaults( + module=self.module, + stream=self.stream, + profiles=self.profiles, + digest=self.digest, + ), + None, + ) class Pulp2Distribution(Pulp2to3Content): @@ -708,8 +800,14 @@ class Pulp2Distribution(Pulp2to3Content): class Meta: unique_together = ( - 'distribution_id', 'family', 'variant', 'version', 'arch', 'pulp2content') - default_related_name = 'distribution_detail_model' + "distribution_id", + "family", + "variant", + "version", + "arch", + "pulp2content", + ) + default_related_name = "distribution_detail_model" @property def relative_path_for_content_artifact(self): @@ -726,20 +824,29 @@ def pre_migrate_content_detail(cls, content_batch): content_batch(list of Pulp2Content): pre-migrated generic data for Pulp 2 content. """ - pulp2_id_obj_map = {pulp2content.pulp2_id: pulp2content for pulp2content in content_batch} + pulp2_id_obj_map = { + pulp2content.pulp2_id: pulp2content for pulp2content in content_batch + } pulp2_ids = pulp2_id_obj_map.keys() pulp2_distribution_content_batch = pulp2_models.Distribution.objects.filter( - id__in=pulp2_ids) + id__in=pulp2_ids + ) pulp2distribution_to_save = [ - cls(distribution_id=distribution.distribution_id, + cls( + distribution_id=distribution.distribution_id, family=distribution.family, variant=distribution.variant, version=distribution.version, arch=distribution.arch, - pulp2content=pulp2_id_obj_map[distribution.id]) - for distribution in pulp2_distribution_content_batch] - cls.objects.bulk_create(pulp2distribution_to_save, ignore_conflicts=True, - batch_size=DEFAULT_BATCH_SIZE) + pulp2content=pulp2_id_obj_map[distribution.id], + ) + for distribution in pulp2_distribution_content_batch + ] + cls.objects.bulk_create( + pulp2distribution_to_save, + ignore_conflicts=True, + batch_size=DEFAULT_BATCH_SIZE, + ) def get_treeinfo_serialized(self): """ @@ -752,36 +859,42 @@ def get_treeinfo_serialized(self): namespaces = [".treeinfo", "treeinfo"] for namespace in namespaces: treeinfo = PulpTreeInfo() - treeinfo_path = os.path.join(self.pulp2content.pulp2_storage_path, namespace) + treeinfo_path = os.path.join( + self.pulp2content.pulp2_storage_path, namespace + ) try: treeinfo.load(f=treeinfo_path) except FileNotFoundError: continue - with open(treeinfo_path, 'rb') as treeinfo_fd: + with open(treeinfo_path, "rb") as treeinfo_fd: treeinfo_digest = hashlib.sha256(treeinfo_fd.read()).hexdigest() self.filename = namespace treeinfo_parsed = treeinfo.parsed_sections() - treeinfo_serialized = TreeinfoData(treeinfo_parsed).to_dict(filename=namespace) + treeinfo_serialized = TreeinfoData(treeinfo_parsed).to_dict( + filename=namespace + ) # Pulp 2 only knows about the top level kickstart repository - treeinfo_serialized["repositories"] = {'.': None} + treeinfo_serialized["repositories"] = {".": None} # Pulp 2 did not support addon repositories, so we should not list them here either - treeinfo_serialized['addons'] = {} + treeinfo_serialized["addons"] = {} # Pulp 2 only supported variants that are in the root of the repository variants = {} - for name, variant in treeinfo_serialized['variants'].items(): - if variant['repository'] == '.': + for name, variant in treeinfo_serialized["variants"].items(): + if variant["repository"] == ".": variants[name] = variant - treeinfo_serialized['variants'] = variants + treeinfo_serialized["variants"] = variants # Set older timestamp so Pulp will fetch all the addons during the next sync # We can't reset it to 0, some creative repository providers use the same # name/release in many distribution trees and the only way to distinguish tem is by # the build timestamp. e.g. CentOS 8 BaseOs, Appstream, PowerTools, HighAvailability. orig_build_timestamp = int( - float(treeinfo_serialized['distribution_tree']['build_timestamp']) + float(treeinfo_serialized["distribution_tree"]["build_timestamp"]) ) - treeinfo_serialized['distribution_tree']['build_timestamp'] = orig_build_timestamp - 1 - treeinfo_serialized['distribution_tree']['digest'] = treeinfo_digest + treeinfo_serialized["distribution_tree"]["build_timestamp"] = ( + orig_build_timestamp - 1 + ) + treeinfo_serialized["distribution_tree"]["digest"] = treeinfo_digest return treeinfo_serialized return None @@ -800,8 +913,10 @@ def create_pulp3_content(self): if treeinfo_serialized is None: return None, None else: - return (DistributionTree(**treeinfo_serialized["distribution_tree"]), - treeinfo_serialized) + return ( + DistributionTree(**treeinfo_serialized["distribution_tree"]), + treeinfo_serialized, + ) class Pulp2PackageLangpacks(Pulp2to3Content): @@ -809,14 +924,15 @@ class Pulp2PackageLangpacks(Pulp2to3Content): Pulp 2to3 detail content model to store pulp 2 package_langpacks content details for Pulp 3 content creation. """ + matches = models.JSONField() repo_id = models.TextField() - pulp2_type = 'package_langpacks' + pulp2_type = "package_langpacks" class Meta: - unique_together = ('repo_id', 'pulp2content') - default_related_name = 'package_langpacks_detail_model' + unique_together = ("repo_id", "pulp2content") + default_related_name = "package_langpacks_detail_model" @classmethod def pre_migrate_content_detail(cls, content_batch): @@ -827,16 +943,24 @@ def pre_migrate_content_detail(cls, content_batch): content_batch(list of Pulp2Content): pre-migrated generic data for Pulp 2 content. """ - pulp2_id_obj_map = {pulp2content.pulp2_id: pulp2content for pulp2content in content_batch} + pulp2_id_obj_map = { + pulp2content.pulp2_id: pulp2content for pulp2content in content_batch + } pulp2_ids = pulp2_id_obj_map.keys() - pulp2_content_batch = pulp2_models.PackageLangpacks.objects.filter(id__in=pulp2_ids) + pulp2_content_batch = pulp2_models.PackageLangpacks.objects.filter( + id__in=pulp2_ids + ) pulp2langpacks_to_save = [ - cls(repo_id=p.repo_id, + cls( + repo_id=p.repo_id, matches=p.matches, - pulp2content=pulp2_id_obj_map[p.id]) - for p in pulp2_content_batch] - cls.objects.bulk_create(pulp2langpacks_to_save, ignore_conflicts=True, - batch_size=DEFAULT_BATCH_SIZE) + pulp2content=pulp2_id_obj_map[p.id], + ) + for p in pulp2_content_batch + ] + cls.objects.bulk_create( + pulp2langpacks_to_save, ignore_conflicts=True, batch_size=DEFAULT_BATCH_SIZE + ) def create_pulp3_content(self): """ @@ -844,8 +968,12 @@ def create_pulp3_content(self): """ langpacks = langpacks_to_libcomps(self) langpacks_dict = PackageLangpacks.libcomps_to_dict(langpacks) - return (PackageLangpacks(matches=langpacks_dict['matches'], - digest=dict_digest(langpacks_dict)), None) + return ( + PackageLangpacks( + matches=langpacks_dict["matches"], digest=dict_digest(langpacks_dict) + ), + None, + ) class Pulp2PackageGroup(Pulp2to3Content): @@ -853,6 +981,7 @@ class Pulp2PackageGroup(Pulp2to3Content): Pulp 2to3 detail content model to store pulp 2 package_group content details for Pulp 3 content creation. """ + package_group_id = models.TextField() repo_id = models.TextField() @@ -867,11 +996,11 @@ class Pulp2PackageGroup(Pulp2to3Content): name_by_lang = models.JSONField(default=dict) biarch_only = models.BooleanField(default=False) - pulp2_type = 'package_group' + pulp2_type = "package_group" class Meta: - unique_together = ('repo_id', 'package_group_id', 'pulp2content') - default_related_name = 'package_group_detail_model' + unique_together = ("repo_id", "package_group_id", "pulp2content") + default_related_name = "package_group_detail_model" @classmethod def pre_migrate_content_detail(cls, content_batch): @@ -882,30 +1011,41 @@ def pre_migrate_content_detail(cls, content_batch): content_batch(list of Pulp2Content): pre-migrated generic data for Pulp 2 content. """ + def _get_packages(group): """Merge in a list all package types""" # order of type of packages is important and should be preserved - return [(3, group.mandatory_package_names), (0, group.default_package_names), - (1, group.optional_package_names), (2, group.conditional_package_names)] - - pulp2_id_obj_map = {pulp2content.pulp2_id: pulp2content for pulp2content in content_batch} + return [ + (3, group.mandatory_package_names), + (0, group.default_package_names), + (1, group.optional_package_names), + (2, group.conditional_package_names), + ] + + pulp2_id_obj_map = { + pulp2content.pulp2_id: pulp2content for pulp2content in content_batch + } pulp2_ids = pulp2_id_obj_map.keys() pulp2_content_batch = pulp2_models.PackageGroup.objects.filter(id__in=pulp2_ids) pulp2groups_to_save = [ - cls(repo_id=p.repo_id, + cls( + repo_id=p.repo_id, package_group_id=p.package_group_id, default=p.default, user_visible=p.user_visible, display_order=p.display_order, name=p.name, - description=p.description if p.description else '', + description=p.description if p.description else "", packages=_get_packages(p), desc_by_lang=p.translated_description, name_by_lang=p.translated_name, - pulp2content=pulp2_id_obj_map[p.id]) - for p in pulp2_content_batch] - cls.objects.bulk_create(pulp2groups_to_save, ignore_conflicts=True, - batch_size=DEFAULT_BATCH_SIZE) + pulp2content=pulp2_id_obj_map[p.id], + ) + for p in pulp2_content_batch + ] + cls.objects.bulk_create( + pulp2groups_to_save, ignore_conflicts=True, batch_size=DEFAULT_BATCH_SIZE + ) def create_pulp3_content(self): """ @@ -913,11 +1053,11 @@ def create_pulp3_content(self): """ group = pkg_grp_to_libcomps(self) group_dict = PackageGroup.libcomps_to_dict(group) - packages = group_dict['packages'] + packages = group_dict["packages"] # ugly stuff for pkg in packages: - pkg['requires'] = pkg['requires'] or None - group_dict['digest'] = dict_digest(group_dict) + pkg["requires"] = pkg["requires"] or None + group_dict["digest"] = dict_digest(group_dict) return (PackageGroup(**group_dict), None) @@ -926,6 +1066,7 @@ class Pulp2PackageCategory(Pulp2to3Content): Pulp 2to3 detail content model to store pulp 2 package_category content details for Pulp 3 content creation. """ + package_category_id = models.TextField() repo_id = models.TextField() @@ -936,11 +1077,11 @@ class Pulp2PackageCategory(Pulp2to3Content): desc_by_lang = models.JSONField(default=dict) name_by_lang = models.JSONField(default=dict) - pulp2_type = 'package_category' + pulp2_type = "package_category" class Meta: - unique_together = ('repo_id', 'package_category_id', 'pulp2content') - default_related_name = 'package_category_detail_model' + unique_together = ("repo_id", "package_category_id", "pulp2content") + default_related_name = "package_category_detail_model" @classmethod def pre_migrate_content_detail(cls, content_batch): @@ -951,22 +1092,30 @@ def pre_migrate_content_detail(cls, content_batch): content_batch(list of Pulp2Content): pre-migrated generic data for Pulp 2 content. """ - pulp2_id_obj_map = {pulp2content.pulp2_id: pulp2content for pulp2content in content_batch} + pulp2_id_obj_map = { + pulp2content.pulp2_id: pulp2content for pulp2content in content_batch + } pulp2_ids = pulp2_id_obj_map.keys() - pulp2_content_batch = pulp2_models.PackageCategory.objects.filter(id__in=pulp2_ids) + pulp2_content_batch = pulp2_models.PackageCategory.objects.filter( + id__in=pulp2_ids + ) pulp2groups_to_save = [ - cls(repo_id=p.repo_id, + cls( + repo_id=p.repo_id, package_category_id=p.package_category_id, display_order=p.display_order, name=p.name, - description=p.description if p.description else '', + description=p.description if p.description else "", packagegroupids=p.packagegroupids, desc_by_lang=p.translated_description, name_by_lang=p.translated_name, - pulp2content=pulp2_id_obj_map[p.id]) - for p in pulp2_content_batch] - cls.objects.bulk_create(pulp2groups_to_save, ignore_conflicts=True, - batch_size=DEFAULT_BATCH_SIZE) + pulp2content=pulp2_id_obj_map[p.id], + ) + for p in pulp2_content_batch + ] + cls.objects.bulk_create( + pulp2groups_to_save, ignore_conflicts=True, batch_size=DEFAULT_BATCH_SIZE + ) def create_pulp3_content(self): """ @@ -974,7 +1123,7 @@ def create_pulp3_content(self): """ cat = pkg_cat_to_libcomps(self) category_dict = PackageCategory.libcomps_to_dict(cat) - category_dict['digest'] = dict_digest(category_dict) + category_dict["digest"] = dict_digest(category_dict) return (PackageCategory(**category_dict), None) @@ -983,6 +1132,7 @@ class Pulp2PackageEnvironment(Pulp2to3Content): Pulp 2to3 detail content model to store pulp 2 package_environment content details for Pulp 3 content creation. """ + package_environment_id = models.TextField() repo_id = models.TextField() @@ -994,11 +1144,11 @@ class Pulp2PackageEnvironment(Pulp2to3Content): desc_by_lang = models.JSONField(default=dict) name_by_lang = models.JSONField(default=dict) - pulp2_type = 'package_environment' + pulp2_type = "package_environment" class Meta: - unique_together = ('repo_id', 'package_environment_id', 'pulp2content') - default_related_name = 'package_environment_detail_model' + unique_together = ("repo_id", "package_environment_id", "pulp2content") + default_related_name = "package_environment_detail_model" @classmethod def pre_migrate_content_detail(cls, content_batch): @@ -1009,23 +1159,31 @@ def pre_migrate_content_detail(cls, content_batch): content_batch(list of Pulp2Content): pre-migrated generic data for Pulp 2 content. """ - pulp2_id_obj_map = {pulp2content.pulp2_id: pulp2content for pulp2content in content_batch} + pulp2_id_obj_map = { + pulp2content.pulp2_id: pulp2content for pulp2content in content_batch + } pulp2_ids = pulp2_id_obj_map.keys() - pulp2_content_batch = pulp2_models.PackageEnvironment.objects.filter(id__in=pulp2_ids) + pulp2_content_batch = pulp2_models.PackageEnvironment.objects.filter( + id__in=pulp2_ids + ) pulp2envs_to_save = [ - cls(repo_id=p.repo_id, + cls( + repo_id=p.repo_id, package_environment_id=p.package_environment_id, display_order=p.display_order, name=p.name, - description=p.description if p.description else '', + description=p.description if p.description else "", group_ids=p.group_ids, option_ids=p.options, desc_by_lang=p.translated_description, name_by_lang=p.translated_name, - pulp2content=pulp2_id_obj_map[p.id]) - for p in pulp2_content_batch] - cls.objects.bulk_create(pulp2envs_to_save, ignore_conflicts=True, - batch_size=DEFAULT_BATCH_SIZE) + pulp2content=pulp2_id_obj_map[p.id], + ) + for p in pulp2_content_batch + ] + cls.objects.bulk_create( + pulp2envs_to_save, ignore_conflicts=True, batch_size=DEFAULT_BATCH_SIZE + ) def create_pulp3_content(self): """ @@ -1033,5 +1191,5 @@ def create_pulp3_content(self): """ env = pkg_env_to_libcomps(self) environment_dict = PackageEnvironment.libcomps_to_dict(env) - environment_dict['digest'] = dict_digest(environment_dict) + environment_dict["digest"] = dict_digest(environment_dict) return (PackageEnvironment(**environment_dict), None) diff --git a/pulp_2to3_migration/app/plugin/rpm/repository.py b/pulp_2to3_migration/app/plugin/rpm/repository.py index 5c0778a2..6c42fd2b 100644 --- a/pulp_2to3_migration/app/plugin/rpm/repository.py +++ b/pulp_2to3_migration/app/plugin/rpm/repository.py @@ -1,14 +1,10 @@ from pulp_2to3_migration.app.plugin.api import ( is_different_relative_url, Pulp2to3Importer, - Pulp2to3Distributor + Pulp2to3Distributor, ) -from pulp_rpm.app.models import ( - RpmRemote, - RpmPublication, - RpmDistribution -) +from pulp_rpm.app.models import RpmRemote, RpmPublication, RpmDistribution from pulp_rpm.app.tasks.publishing import publish from urllib.parse import urlparse, urlunparse @@ -18,6 +14,7 @@ class RpmImporter(Pulp2to3Importer): """ Interface to migrate Pulp 2 RPM importer """ + pulp3_remote_models = [RpmRemote] @classmethod @@ -34,14 +31,14 @@ def migrate_to_pulp3(cls, pulp2importer): """ pulp2_config = pulp2importer.pulp2_config base_config, name = cls.parse_base_config(pulp2importer, pulp2_config) - sles_auth_token = pulp2_config.get('query_auth_token') + sles_auth_token = pulp2_config.get("query_auth_token") if sles_auth_token: - base_config['sles_auth_token'] = sles_auth_token + base_config["sles_auth_token"] = sles_auth_token else: - url = urlparse(pulp2_config.get('feed', '')) - if url.query and '=' not in url.query and '&' not in url.query: - base_config['sles_auth_token'] = url.query - base_config['url'] = urlunparse(url._replace(query='')) + url = urlparse(pulp2_config.get("feed", "")) + if url.query and "=" not in url.query and "&" not in url.query: + base_config["sles_auth_token"] = url.query + base_config["url"] = urlunparse(url._replace(query="")) return RpmRemote.objects.update_or_create(name=name, defaults=base_config) @@ -49,6 +46,7 @@ class RpmDistributor(Pulp2to3Distributor): """ Interface to migrate Pulp 2 RPM distributor """ + pulp3_publication_models = [RpmPublication] pulp3_distribution_models = [RpmDistribution] @@ -71,31 +69,37 @@ def migrate_to_pulp3(cls, pulp2distributor, repo_version, signing_service): # this will go away with the simple-complex plan conversion work if not repo_version: - repo = pulp2distributor.pulp2_repos.filter(not_in_plan=False, is_migrated=True) + repo = pulp2distributor.pulp2_repos.filter( + not_in_plan=False, is_migrated=True + ) repo_version = repo[0].pulp3_repository_version publication = repo_version.publication_set.filter(complete=True).first() if not publication: - pulp2_checksum_type = pulp2_config.get('checksum_type') + pulp2_checksum_type = pulp2_config.get("checksum_type") checksum_types = None if pulp2_checksum_type: checksum_types = { - 'metadata': pulp2_checksum_type, - 'package': pulp2_checksum_type + "metadata": pulp2_checksum_type, + "package": pulp2_checksum_type, } else: # Set the checksum type based on content in a repo, pulp 2 supports only one # checksum type for packages in a repo. It is important to set checksum type for # Pulp 3 to Pulp 2 sync use case. - package_qs = repo_version.content.filter(pulp_type='rpm.package') + package_qs = repo_version.content.filter(pulp_type="rpm.package") if package_qs.count(): pkg_checksum_type = package_qs.first().cast().checksum_type checksum_types = { - 'metadata': pkg_checksum_type, - 'package': pkg_checksum_type + "metadata": pkg_checksum_type, + "package": pkg_checksum_type, } - sqlite = pulp2_config.get('generate_sqlite', False) + sqlite = pulp2_config.get("generate_sqlite", False) try: - publish(repo_version.pk, checksum_types=checksum_types, sqlite_metadata=sqlite) + publish( + repo_version.pk, + checksum_types=checksum_types, + sqlite_metadata=sqlite, + ) except TypeError: # hack, pulp_rpm <3.9 doesn't support sqlite_metadata kwarg publish(repo_version.pk, checksum_types=checksum_types) @@ -105,13 +109,14 @@ def migrate_to_pulp3(cls, pulp2distributor, repo_version, signing_service): distribution_data = cls.parse_base_config(pulp2distributor, pulp2_config) # ensure that the base_path does not end with / in Pulp 3, it's often present in Pulp 2. - base_path = pulp2_config.get('relative_url', pulp2distributor.pulp2_repo_id) - distribution_data['base_path'] = base_path.rstrip('/') - distribution_data['publication'] = publication + base_path = pulp2_config.get("relative_url", pulp2distributor.pulp2_repo_id) + distribution_data["base_path"] = base_path.rstrip("/") + distribution_data["publication"] = publication distribution, created = RpmDistribution.objects.update_or_create( - name=distribution_data['name'], - base_path=distribution_data['base_path'], - defaults=distribution_data) + name=distribution_data["name"], + base_path=distribution_data["base_path"], + defaults=distribution_data, + ) return publication, distribution, created @classmethod @@ -129,10 +134,14 @@ def needs_new_publication(cls, pulp2distributor): if not pulp2distributor.pulp3_publication: return True - new_checksum_type = pulp2distributor.pulp2_config.get('checksum_type') - current_checksum_type = pulp2distributor.pulp3_publication.cast().metadata_checksum_type + new_checksum_type = pulp2distributor.pulp2_config.get("checksum_type") + current_checksum_type = ( + pulp2distributor.pulp3_publication.cast().metadata_checksum_type + ) - is_default_checksum_type = new_checksum_type is None and current_checksum_type == 'sha256' + is_default_checksum_type = ( + new_checksum_type is None and current_checksum_type == "sha256" + ) if new_checksum_type != current_checksum_type and not is_default_checksum_type: return True diff --git a/pulp_2to3_migration/app/plugin/rpm/utils.py b/pulp_2to3_migration/app/plugin/rpm/utils.py index a34e534d..0cfe3ef2 100644 --- a/pulp_2to3_migration/app/plugin/rpm/utils.py +++ b/pulp_2to3_migration/app/plugin/rpm/utils.py @@ -7,7 +7,7 @@ def exclude_unsupported_metadata(): """ Exclude .zck and .xz metadata from the list of content to premigrate. """ - exclude_zck = mongo_Q(data_type__not__endswith='_zck') - exclude_xz = mongo_Q(data_type__not__endswith='_xz') + exclude_zck = mongo_Q(data_type__not__endswith="_zck") + exclude_xz = mongo_Q(data_type__not__endswith="_xz") supported_content = YumMetadataFile.objects.filter(exclude_zck & exclude_xz) return [c.id for c in supported_content] diff --git a/pulp_2to3_migration/app/plugin/rpm/xml_utils.py b/pulp_2to3_migration/app/plugin/rpm/xml_utils.py index 73418f03..75e548b6 100644 --- a/pulp_2to3_migration/app/plugin/rpm/xml_utils.py +++ b/pulp_2to3_migration/app/plugin/rpm/xml_utils.py @@ -18,15 +18,15 @@ ESCAPE_TEMPLATE_VARS_TAGS = { - 'primary': ('description', 'summary'), - 'other': ('changelog',), - 'filelists': ('file',) + "primary": ("description", "summary"), + "other": ("changelog",), + "filelists": ("file",), } -METADATA_TYPES = ('primary', 'other', 'filelists') +METADATA_TYPES = ("primary", "other", "filelists") -XmlElement = namedtuple('xml_element', ['start', 'end']) +XmlElement = namedtuple("xml_element", ["start", "end"]) TEMPLATETAG_MAP = dict((sym, name) for name, sym in TemplateTagNode.mapping.items()) -SYMBOLS_PATTERN = '(%s)' % '|'.join(TEMPLATETAG_MAP.keys()) +SYMBOLS_PATTERN = "(%s)" % "|".join(TEMPLATETAG_MAP.keys()) def _substitute_special_chars(template): @@ -44,8 +44,8 @@ def _substitute_special_chars(template): """ return re.sub( SYMBOLS_PATTERN, - lambda mobj: '{%% templatetag %s %%}' % TEMPLATETAG_MAP[mobj.group(1)], - template + lambda mobj: "{%% templatetag %s %%}" % TEMPLATETAG_MAP[mobj.group(1)], + template, ) @@ -84,9 +84,9 @@ def _escape_django_syntax_chars(template, tag_name): str: a Django template with the escaped syntax characters in the specified element """ - start_tag_pattern = r'<%s.*?(?' % tag_name - end_tag_pattern = r'' % tag_name - complete_tag_pattern = r'(%s)(.*)(%s)' % (start_tag_pattern, end_tag_pattern) + start_tag_pattern = r"<%s.*?(?" % tag_name + end_tag_pattern = r"" % tag_name + complete_tag_pattern = r"(%s)(.*)(%s)" % (start_tag_pattern, end_tag_pattern) tag_re = re.compile(complete_tag_pattern, flags=re.DOTALL) template = tag_re.sub(_generate_tag_replacement_str, template) return template @@ -105,9 +105,9 @@ def render_primary(template, checksum, checksumtype): str: a rendered primary.xml snippet """ - for tag in ESCAPE_TEMPLATE_VARS_TAGS['primary']: + for tag in ESCAPE_TEMPLATE_VARS_TAGS["primary"]: template = _escape_django_syntax_chars(template, tag) - context = Context({'checksum': checksum, 'checksumtype': checksumtype}) + context = Context({"checksum": checksum, "checksumtype": checksumtype}) return Template(template).render(context) @@ -124,9 +124,9 @@ def render_other(template, checksum): """ - for tag in ESCAPE_TEMPLATE_VARS_TAGS['other']: + for tag in ESCAPE_TEMPLATE_VARS_TAGS["other"]: template = _escape_django_syntax_chars(template, tag) - context = Context({'pkgid': checksum}) + context = Context({"pkgid": checksum}) return Template(template).render(context) @@ -142,14 +142,15 @@ def render_filelists(template, checksum): str: a rendered filelists.xml snippet """ - for tag in ESCAPE_TEMPLATE_VARS_TAGS['filelists']: + for tag in ESCAPE_TEMPLATE_VARS_TAGS["filelists"]: template = _escape_django_syntax_chars(template, tag) - context = Context({'pkgid': checksum}) + context = Context({"pkgid": checksum}) return Template(template).render(context) # Additional utils (not from pulp2) # + def render_metadata(pkg, md_type): """ Render a requested metadata type for a package. @@ -163,13 +164,13 @@ def render_metadata(pkg, md_type): if md_type not in METADATA_TYPES: return - if md_type == 'primary': + if md_type == "primary": xml_template = decompress_repodata(pkg.primary_template_gz) return render_primary(xml_template, pkg.checksum, pkg.checksumtype) - elif md_type == 'other': + elif md_type == "other": xml_template = decompress_repodata(pkg.other_template_gz) return render_other(xml_template, pkg.checksum) - elif md_type == 'filelists': + elif md_type == "filelists": xml_template = decompress_repodata(pkg.filelists_template_gz) return render_filelists(xml_template, pkg.checksum) @@ -185,9 +186,9 @@ def get_cr_obj(pkg): createrepo_c.Package: createrepo_c Package object for the requested package """ - primary_xml = render_metadata(pkg, 'primary') - other_xml = render_metadata(pkg, 'other') - filelists_xml = render_metadata(pkg, 'filelists') + primary_xml = render_metadata(pkg, "primary") + other_xml = render_metadata(pkg, "other") + filelists_xml = render_metadata(pkg, "filelists") return parse_repodata(primary_xml, filelists_xml, other_xml) diff --git a/pulp_2to3_migration/app/pre_migration.py b/pulp_2to3_migration/app/pre_migration.py index 6a996c0f..b688ac6b 100644 --- a/pulp_2to3_migration/app/pre_migration.py +++ b/pulp_2to3_migration/app/pre_migration.py @@ -36,7 +36,7 @@ _logger = logging.getLogger(__name__) -ContentModel = namedtuple('ContentModel', ['pulp2', 'pulp_2to3_detail']) +ContentModel = namedtuple("ContentModel", ["pulp2", "pulp_2to3_detail"]) def pre_migrate_all_content(plan): @@ -46,7 +46,7 @@ def pre_migrate_all_content(plan): Args: plan (MigrationPlan): Migration Plan to use for migration. """ - _logger.debug('Pre-migrating Pulp 2 content') + _logger.debug("Pre-migrating Pulp 2 content") # get all the content models for the migrating plugins for plugin in plan.get_plugin_plans(): @@ -57,17 +57,24 @@ def pre_migrate_all_content(plan): # postgresql model pulp_2to3_detail_model = plugin.migrator.content_models[content_type] - content_model = ContentModel(pulp2=pulp2_content_model, - pulp_2to3_detail=pulp_2to3_detail_model) + content_model = ContentModel( + pulp2=pulp2_content_model, pulp_2to3_detail=pulp_2to3_detail_model + ) # identify wether the content is mutable - mutable_type = content_model.pulp2.TYPE_ID in plugin.migrator.mutable_content_models + mutable_type = ( + content_model.pulp2.TYPE_ID in plugin.migrator.mutable_content_models + ) # identify wether the content is lazy lazy_type = content_model.pulp2.TYPE_ID in plugin.migrator.lazy_types # check if the content type has a premigration hook premigrate_hook = None if content_model.pulp2.TYPE_ID in plugin.migrator.premigrate_hook: - premigrate_hook = plugin.migrator.premigrate_hook[content_model.pulp2.TYPE_ID] - pre_migrate_content_type(content_model, mutable_type, lazy_type, premigrate_hook) + premigrate_hook = plugin.migrator.premigrate_hook[ + content_model.pulp2.TYPE_ID + ] + pre_migrate_content_type( + content_model, mutable_type, lazy_type, premigrate_hook + ) def pre_migrate_content_type(content_model, mutable_type, lazy_type, premigrate_hook): @@ -78,6 +85,7 @@ def pre_migrate_content_type(content_model, mutable_type, lazy_type, premigrate_ content_model: Models for content which is being migrated. mutable_type: Boolean that indicates whether the content type is mutable. """ + def delete_removed_pulp2_content(content_model): """ Delete Pulp2Content records for content which is no longer present in Pulp2. @@ -90,18 +98,17 @@ def delete_removed_pulp2_content(content_model): """ content_type = content_model.pulp2.TYPE_ID - mongo_content_qs = content_model.pulp2.objects().only('id') - mongo_content_ids = {c['_id'] for c in mongo_content_qs.as_pymongo().no_cache()} + mongo_content_qs = content_model.pulp2.objects().only("id") + mongo_content_ids = {c["_id"] for c in mongo_content_qs.as_pymongo().no_cache()} premigrated_content_ids = set( - Pulp2Content.objects.filter( - pulp2_content_type_id=content_type - ).only('pulp2_id').values_list('pulp2_id', flat=True) + Pulp2Content.objects.filter(pulp2_content_type_id=content_type) + .only("pulp2_id") + .values_list("pulp2_id", flat=True) ) content_ids_to_delete = premigrated_content_ids - mongo_content_ids if content_ids_to_delete: Pulp2Content.objects.filter( - pulp2_content_type_id=content_type, - pulp2_id__in=content_ids_to_delete + pulp2_content_type_id=content_type, pulp2_id__in=content_ids_to_delete ).delete() batch_size = ( @@ -121,10 +128,14 @@ def delete_removed_pulp2_content(content_model): # the latest timestamp we have in the migration tool Pulp2Content table for this content type content_qs = Pulp2Content.objects.filter(pulp2_content_type_id=content_type) - last_updated = content_qs.aggregate(Max('pulp2_last_updated'))['pulp2_last_updated__max'] or 0 - _logger.debug('The latest migrated {type} content has {timestamp} timestamp.'.format( - type=content_type, - timestamp=last_updated)) + last_updated = ( + content_qs.aggregate(Max("pulp2_last_updated"))["pulp2_last_updated__max"] or 0 + ) + _logger.debug( + "The latest migrated {type} content has {timestamp} timestamp.".format( + type=content_type, timestamp=last_updated + ) + ) query_args = {} if premigrate_hook: @@ -136,34 +147,39 @@ def delete_removed_pulp2_content(content_model): ).order_by("_last_updated") total_content = mongo_content_qs.count() - _logger.debug('Total count for {type} content to migrate: {total}'.format( - type=content_type, - total=total_content)) + _logger.debug( + "Total count for {type} content to migrate: {total}".format( + type=content_type, total=total_content + ) + ) pulp2content_pb = ProgressReport( - message='Pre-migrating Pulp 2 {} content (general info)'.format(content_type), - code='premigrating.content.general', + message="Pre-migrating Pulp 2 {} content (general info)".format(content_type), + code="premigrating.content.general", total=total_content, - state=TASK_STATES.RUNNING) + state=TASK_STATES.RUNNING, + ) pulp2content_pb.save() pulp2detail_pb = ProgressReport( - message='Pre-migrating Pulp 2 {} content (detail info)'.format(content_type), - code='premigrating.content.detail', + message="Pre-migrating Pulp 2 {} content (detail info)".format(content_type), + code="premigrating.content.detail", total=total_content, - state=TASK_STATES.RUNNING) + state=TASK_STATES.RUNNING, + ) pulp2detail_pb.save() existing_count = 0 if mutable_type: pulp2_content_ids = [] - for c in mongo_content_qs.only('id', '_last_updated').no_cache().as_pymongo(): - if c['_last_updated'] == last_updated: + for c in mongo_content_qs.only("id", "_last_updated").no_cache().as_pymongo(): + if c["_last_updated"] == last_updated: if Pulp2Content.objects.filter( - pulp2_last_updated=last_updated, pulp2_id=c['_id']).exists(): + pulp2_last_updated=last_updated, pulp2_id=c["_id"] + ).exists(): continue - pulp2_content_ids.append(c['_id']) + pulp2_content_ids.append(c["_id"]) # This is a mutable content type. Query for the existing pulp2content. # If any was found, it means that the migrated content is older than the incoming. @@ -173,17 +189,20 @@ def delete_removed_pulp2_content(content_model): pulp2mutatedcontent.extend(pulp2_content_ids) outdated.delete() - mongo_fields = set(['id', '_storage_path', '_last_updated', '_content_type_id']) - if hasattr(content_model.pulp2, 'downloaded'): - mongo_fields.add('downloaded') + mongo_fields = set(["id", "_storage_path", "_last_updated", "_content_type_id"]) + if hasattr(content_model.pulp2, "downloaded"): + mongo_fields.add("downloaded") - batched_mongo_content_qs = mongo_content_qs.only(*mongo_fields).batch_size(batch_size) + batched_mongo_content_qs = mongo_content_qs.only(*mongo_fields).batch_size( + batch_size + ) for i, record in enumerate(batched_mongo_content_qs.no_cache()): if record._last_updated == last_updated: # corner case - content with the last``last_updated`` date might be pre-migrated; # check if this content is already pre-migrated - migrated = Pulp2Content.objects.filter(pulp2_last_updated=last_updated, - pulp2_id=record.id) + migrated = Pulp2Content.objects.filter( + pulp2_last_updated=last_updated, pulp2_id=record.id + ) if migrated.exists(): existing_count += 1 @@ -194,7 +213,7 @@ def delete_removed_pulp2_content(content_model): continue # very old pulp2 content will not have downloaded field set (prior to lazy sync) - downloaded = hasattr(record, 'downloaded') and ( + downloaded = hasattr(record, "downloaded") and ( record.downloaded or record.downloaded is None ) @@ -202,14 +221,14 @@ def delete_removed_pulp2_content(content_model): # This content requires to set pulp 2 repo. E.g. for errata, because 1 pulp2 # content unit is converted into N pulp3 content units and repo_id is the only # way to have unique records for those. - content_relations = Pulp2RepoContent.objects.filter( - pulp2_unit_id=record.id, - pulp2_content_type_id=record._content_type_id, - pulp2_repository__not_in_plan=False, - ).select_related( - 'pulp2_repository' - ).only( - 'pulp2_repository' + content_relations = ( + Pulp2RepoContent.objects.filter( + pulp2_unit_id=record.id, + pulp2_content_type_id=record._content_type_id, + pulp2_repository__not_in_plan=False, + ) + .select_related("pulp2_repository") + .only("pulp2_repository") ) for relation in content_relations.iterator(): item = Pulp2Content( @@ -221,7 +240,8 @@ def delete_removed_pulp2_content(content_model): pulp2_repo=relation.pulp2_repository, ) _logger.debug( - 'Add content item to the list to migrate: {item}'.format(item=item)) + "Add content item to the list to migrate: {item}".format(item=item) + ) pulp2content.append(item) pulp2content_pb.total += 1 pulp2detail_pb.total += 1 @@ -236,20 +256,27 @@ def delete_removed_pulp2_content(content_model): pulp2_content_type_id=record._content_type_id, pulp2_last_updated=record._last_updated, pulp2_storage_path=record._storage_path, - downloaded=downloaded + downloaded=downloaded, + ) + _logger.debug( + "Add content item to the list to migrate: {item}".format(item=item) ) - _logger.debug('Add content item to the list to migrate: {item}'.format(item=item)) pulp2content.append(item) # determine if the batch needs to be saved, also take into account whether there is # anything in the pulp2content to be saved - save_batch = pulp2content and (len(pulp2content) >= batch_size or i == total_content - 1) + save_batch = pulp2content and ( + len(pulp2content) >= batch_size or i == total_content - 1 + ) if save_batch: _logger.debug( - 'Bulk save for generic content info, saved so far: {index}'.format(index=i + 1) + "Bulk save for generic content info, saved so far: {index}".format( + index=i + 1 + ) + ) + pulp2content_batch = Pulp2Content.objects.bulk_create( + pulp2content, ignore_conflicts=True ) - pulp2content_batch = Pulp2Content.objects.bulk_create(pulp2content, - ignore_conflicts=True) # bulk_create(ignore_conflicts=True) hands back the same item-set we passed in, # *even if* it decided to update an existing db-record rather than creating a new @@ -273,7 +300,9 @@ def delete_removed_pulp2_content(content_model): pulp2content_pb.done += content_saved pulp2content_pb.save() - content_model.pulp_2to3_detail.pre_migrate_content_detail(pulp2content_batch) + content_model.pulp_2to3_detail.pre_migrate_content_detail( + pulp2content_batch + ) pulp2detail_pb.done += content_saved pulp2detail_pb.save() @@ -291,19 +320,26 @@ def delete_removed_pulp2_content(content_model): last_updated = datetime.utcfromtimestamp(last_updated) # Query all new relations for that content since the last run - content_relations = Pulp2RepoContent.objects.filter( - pulp2_content_type_id=content_type, - pulp2_repository__not_in_plan=False, - pulp2_created__gte=last_updated - ).select_related( - 'pulp2_repository' - ).only( - 'pulp2_repository', 'pulp2_created', - ).order_by('pulp2_created') + content_relations = ( + Pulp2RepoContent.objects.filter( + pulp2_content_type_id=content_type, + pulp2_repository__not_in_plan=False, + pulp2_created__gte=last_updated, + ) + .select_related("pulp2_repository") + .only( + "pulp2_repository", + "pulp2_created", + ) + .order_by("pulp2_created") + ) mongo_content_qs = content_model.pulp2.objects( - id__in=content_relations.values_list('pulp2_unit_id', flat=True)) - batched_mongo_content_qs = mongo_content_qs.only(*mongo_fields).batch_size(batch_size) + id__in=content_relations.values_list("pulp2_unit_id", flat=True) + ) + batched_mongo_content_qs = mongo_content_qs.only(*mongo_fields).batch_size( + batch_size + ) pulp2_content_by_id = { record.id: record for record in batched_mongo_content_qs.no_cache() } @@ -311,14 +347,14 @@ def delete_removed_pulp2_content(content_model): for relation in content_relations: record = pulp2_content_by_id[relation.pulp2_unit_id] # very old pulp2 content will not have downloaded field set (prior to lazy sync) - downloaded = hasattr(record, 'downloaded') and ( + downloaded = hasattr(record, "downloaded") and ( record.downloaded or record.downloaded is None ) specific_content_q = Q( pulp2_content_type_id=record._content_type_id, pulp2_id=record.id, pulp2_repo=relation.pulp2_repository, - pulp2_subid='', + pulp2_subid="", ) # Ensure that no existing pulp2content slipped into bulk_create. @@ -338,10 +374,11 @@ def delete_removed_pulp2_content(content_model): pulp2_last_updated=int(relation.pulp2_created.timestamp()), pulp2_storage_path=record._storage_path, downloaded=downloaded, - pulp2_repo=relation.pulp2_repository + pulp2_repo=relation.pulp2_repository, ) _logger.debug( - 'Add content item to the list to migrate: {item}'.format(item=item)) + "Add content item to the list to migrate: {item}".format(item=item) + ) pulp2content.append(item) pulp2content_pb.total += 1 pulp2detail_pb.total += 1 @@ -364,10 +401,15 @@ def delete_removed_pulp2_content(content_model): # in this case, we still need to update the is_migrated flag manually because of errata. # in pulp2 sync and copy cases of updated errata are not covered # only when uploading errata last_unit_added is updated on all the repos that contain it - mutated_content = Pulp2RepoContent.objects.filter(pulp2_unit_id__in=pulp2mutatedcontent) + mutated_content = Pulp2RepoContent.objects.filter( + pulp2_unit_id__in=pulp2mutatedcontent + ) repo_to_update_ids = mutated_content.values_list( - 'pulp2_repository_id', flat=True).distinct() - Pulp2Repository.objects.filter(pk__in=repo_to_update_ids).update(is_migrated=False) + "pulp2_repository_id", flat=True + ).distinct() + Pulp2Repository.objects.filter(pk__in=repo_to_update_ids).update( + is_migrated=False + ) if lazy_type: pre_migrate_lazycatalog(content_type) @@ -393,17 +435,21 @@ def pre_migrate_lazycatalog(content_type): mongo_lce_qs = LazyCatalogEntry.objects(unit_type_id=content_type) for lce in mongo_lce_qs.batch_size(batch_size).as_pymongo().no_cache(): - item = Pulp2LazyCatalog(pulp2_importer_id=lce['importer_id'], - pulp2_unit_id=lce['unit_id'], - pulp2_content_type_id=lce['unit_type_id'], - pulp2_storage_path=lce['path'], - pulp2_url=lce['url'], - pulp2_revision=lce['revision'], - is_migrated=False) + item = Pulp2LazyCatalog( + pulp2_importer_id=lce["importer_id"], + pulp2_unit_id=lce["unit_id"], + pulp2_content_type_id=lce["unit_type_id"], + pulp2_storage_path=lce["path"], + pulp2_url=lce["url"], + pulp2_revision=lce["revision"], + is_migrated=False, + ) pulp2lazycatalog.append(item) if len(pulp2lazycatalog) >= batch_size: - Pulp2LazyCatalog.objects.bulk_create(pulp2lazycatalog, ignore_conflicts=True) + Pulp2LazyCatalog.objects.bulk_create( + pulp2lazycatalog, ignore_conflicts=True + ) pulp2lazycatalog.clear() else: Pulp2LazyCatalog.objects.bulk_create(pulp2lazycatalog, ignore_conflicts=True) @@ -430,10 +476,13 @@ def pre_migrate_all_without_content(plan): plan(MigrationPlan): A Migration Plan """ - _logger.debug('Pre-migrating Pulp 2 repositories') + _logger.debug("Pre-migrating Pulp 2 repositories") - with ProgressReport(message='Processing Pulp 2 repositories, importers, distributors', - code='processing.repositories', total=0) as pb: + with ProgressReport( + message="Processing Pulp 2 repositories, importers, distributors", + code="processing.repositories", + total=0, + ) as pb: for plugin_plan in plan.get_plugin_plans(): repos = plugin_plan.get_repositories() @@ -451,21 +500,37 @@ def pre_migrate_all_without_content(plan): dist_type_q = Q(pulp2_type_id__in=distributor_types) plugin_pulp2repos = Pulp2Repository.objects.filter(repo_type_q) - repo_premigrated_last_by_added = plugin_pulp2repos.aggregate( - Max('pulp2_last_unit_added') - )['pulp2_last_unit_added__max'] or epoch - repo_premigrated_last_by_removed = plugin_pulp2repos.aggregate( - Max('pulp2_last_unit_removed') - )['pulp2_last_unit_removed__max'] or epoch - imp_premigrated_last = Pulp2Importer.objects.filter(imp_type_q).aggregate( - Max('pulp2_last_updated') - )['pulp2_last_updated__max'] or epoch - dist_premigrated_last = Pulp2Distributor.objects.filter(dist_type_q).aggregate( - Max('pulp2_last_updated') - )['pulp2_last_updated__max'] or epoch - - is_content_added_q = mongo_Q(last_unit_added__gte=repo_premigrated_last_by_added) - is_content_removed_q = mongo_Q(last_unit_removed__gte=repo_premigrated_last_by_removed) + repo_premigrated_last_by_added = ( + plugin_pulp2repos.aggregate(Max("pulp2_last_unit_added"))[ + "pulp2_last_unit_added__max" + ] + or epoch + ) + repo_premigrated_last_by_removed = ( + plugin_pulp2repos.aggregate(Max("pulp2_last_unit_removed"))[ + "pulp2_last_unit_removed__max" + ] + or epoch + ) + imp_premigrated_last = ( + Pulp2Importer.objects.filter(imp_type_q).aggregate( + Max("pulp2_last_updated") + )["pulp2_last_updated__max"] + or epoch + ) + dist_premigrated_last = ( + Pulp2Distributor.objects.filter(dist_type_q).aggregate( + Max("pulp2_last_updated") + )["pulp2_last_updated__max"] + or epoch + ) + + is_content_added_q = mongo_Q( + last_unit_added__gte=repo_premigrated_last_by_added + ) + is_content_removed_q = mongo_Q( + last_unit_removed__gte=repo_premigrated_last_by_removed + ) is_new_enough_repo_q = is_content_added_q | is_content_removed_q is_empty_repo_q = mongo_Q(last_unit_added__exists=False) is_new_enough_imp_q = mongo_Q(last_updated__gte=imp_premigrated_last) @@ -476,29 +541,29 @@ def pre_migrate_all_without_content(plan): updated_importers = Importer.objects( imp_repo_id_q & is_new_enough_imp_q - ).only('repo_id') + ).only("repo_id") updated_imp_repos = set(imp.repo_id for imp in updated_importers) updated_distributors = Distributor.objects( dist_repo_id_q & is_new_enough_dist_q - ).only('repo_id') + ).only("repo_id") updated_dist_repos = set(dist.repo_id for dist in updated_distributors) updated_impdist_repos = updated_imp_repos | updated_dist_repos - mongo_updated_repo_q = repo_repo_id_q & (is_new_enough_repo_q | is_empty_repo_q) + mongo_updated_repo_q = repo_repo_id_q & ( + is_new_enough_repo_q | is_empty_repo_q + ) mongo_updated_imp_dist_repo_q = mongo_Q(repo_id__in=updated_impdist_repos) mongo_repo_qs = Repository.objects( mongo_updated_repo_q | mongo_updated_imp_dist_repo_q - ).order_by('last_unit_added') + ).order_by("last_unit_added") pb.total += mongo_repo_qs.count() pb.save() - for repo_data in mongo_repo_qs.only('id', - 'repo_id', - 'last_unit_added', - 'last_unit_removed', - 'description'): + for repo_data in mongo_repo_qs.only( + "id", "repo_id", "last_unit_added", "last_unit_removed", "description" + ): repo_id = repo_data.repo_id with transaction.atomic(): if repo_id in repos: @@ -528,26 +593,33 @@ def pre_migrate_repo(record, repo_id_to_type): repo(Pulp2Repository): A pre-migrated repository """ - last_unit_added = (record.last_unit_added - and timezone.make_aware(record.last_unit_added, timezone.utc)) - last_unit_removed = (record.last_unit_removed - and timezone.make_aware(record.last_unit_removed, timezone.utc)) + last_unit_added = record.last_unit_added and timezone.make_aware( + record.last_unit_added, timezone.utc + ) + last_unit_removed = record.last_unit_removed and timezone.make_aware( + record.last_unit_removed, timezone.utc + ) repo, created = Pulp2Repository.objects.get_or_create( pulp2_object_id=record.id, - defaults={'pulp2_repo_id': record.repo_id, - 'pulp2_last_unit_added': last_unit_added, - 'pulp2_last_unit_removed': last_unit_removed, - 'pulp2_description': record.description, - 'pulp2_repo_type': repo_id_to_type[record.repo_id], - 'is_migrated': False}) + defaults={ + "pulp2_repo_id": record.repo_id, + "pulp2_last_unit_added": last_unit_added, + "pulp2_last_unit_removed": last_unit_removed, + "pulp2_description": record.description, + "pulp2_repo_type": repo_id_to_type[record.repo_id], + "is_migrated": False, + }, + ) if not created: # if it was marked as such because it was not present in the migration plan repo.not_in_plan = False # check if there were any changes since last time - is_changed = (last_unit_added != repo.pulp2_last_unit_added - or last_unit_removed != repo.pulp2_last_unit_removed) + is_changed = ( + last_unit_added != repo.pulp2_last_unit_added + or last_unit_removed != repo.pulp2_last_unit_removed + ) if is_changed: repo.pulp2_last_unit_added = last_unit_added repo.last_unit_removed = last_unit_removed @@ -581,29 +653,32 @@ def pre_migrate_importer(repo_id, importer_types): # or it has an empty config return - importer_data = mongo_importer_qs.only('id', - 'repo_id', - 'importer_type_id', - 'last_updated', - 'config').first() + importer_data = mongo_importer_qs.only( + "id", "repo_id", "importer_type_id", "last_updated", "config" + ).first() - if not importer_data.config.get('feed'): + if not importer_data.config.get("feed"): # Pulp 3 remotes require URL - msg = 'Importer from {repo} cannot be migrated because it does not have a feed'.format( - repo=repo_id) + msg = "Importer from {repo} cannot be migrated because it does not have a feed".format( + repo=repo_id + ) _logger.warn(msg) return - last_updated = (importer_data.last_updated - and timezone.make_aware(importer_data.last_updated, timezone.utc)) + last_updated = importer_data.last_updated and timezone.make_aware( + importer_data.last_updated, timezone.utc + ) importer, created = Pulp2Importer.objects.get_or_create( pulp2_object_id=importer_data.id, - defaults={'pulp2_type_id': importer_data.importer_type_id, - 'pulp2_last_updated': last_updated, - 'pulp2_config': importer_data.config, - 'pulp2_repo_id': repo_id, - 'is_migrated': False}) + defaults={ + "pulp2_type_id": importer_data.importer_type_id, + "pulp2_last_updated": last_updated, + "pulp2_config": importer_data.config, + "pulp2_repo_id": repo_id, + "is_migrated": False, + }, + ) if not created: # if it was marked as such because it was not present in the migration plan @@ -611,7 +686,7 @@ def pre_migrate_importer(repo_id, importer_types): # check if there were any changes since last time if last_updated != importer.pulp2_last_updated: # remove Remote in case of feed change - if importer.pulp2_config.get('feed') != importer_data.config.get('feed'): + if importer.pulp2_config.get("feed") != importer_data.config.get("feed"): if importer.pulp3_remote: importer.pulp3_remote.delete() importer.pulp3_remote = None @@ -632,8 +707,9 @@ def pre_migrate_distributor(repo_id, distributor_migrators): distributor_migrators(dict): supported distributor types and their models for migration """ distributor_types = list(distributor_migrators.keys()) - mongo_distributor_q = mongo_Q(repo_id=repo_id, - distributor_type_id__in=distributor_types) + mongo_distributor_q = mongo_Q( + repo_id=repo_id, distributor_type_id__in=distributor_types + ) mongo_distributor_qs = Distributor.objects(mongo_distributor_q) if not mongo_distributor_qs: @@ -643,17 +719,21 @@ def pre_migrate_distributor(repo_id, distributor_migrators): return for dist_data in mongo_distributor_qs: - last_updated = (dist_data.last_updated - and timezone.make_aware(dist_data.last_updated, timezone.utc)) + last_updated = dist_data.last_updated and timezone.make_aware( + dist_data.last_updated, timezone.utc + ) distributor, created = Pulp2Distributor.objects.get_or_create( pulp2_object_id=dist_data.id, - defaults={'pulp2_id': dist_data.distributor_id, - 'pulp2_type_id': dist_data.distributor_type_id, - 'pulp2_last_updated': last_updated, - 'pulp2_config': dist_data.config, - 'pulp2_repo_id': repo_id, - 'is_migrated': False}) + defaults={ + "pulp2_id": dist_data.distributor_id, + "pulp2_type_id": dist_data.distributor_type_id, + "pulp2_last_updated": last_updated, + "pulp2_config": dist_data.config, + "pulp2_repo_id": repo_id, + "is_migrated": False, + }, + ) if not created: # if it was marked as such because it was not present in the migration plan @@ -665,15 +745,23 @@ def pre_migrate_distributor(repo_id, distributor_migrators): distributor.is_migrated = False dist_migrator = distributor_migrators.get(distributor.pulp2_type_id) needs_new_publication = dist_migrator.needs_new_publication(distributor) - needs_new_distribution = dist_migrator.needs_new_distribution(distributor) - remove_publication = needs_new_publication and distributor.pulp3_publication - remove_distribution = needs_new_distribution and distributor.pulp3_distribution + needs_new_distribution = dist_migrator.needs_new_distribution( + distributor + ) + remove_publication = ( + needs_new_publication and distributor.pulp3_publication + ) + remove_distribution = ( + needs_new_distribution and distributor.pulp3_distribution + ) if remove_publication: # check if publication is shared by multiple distributions # on the corresponding distributor flip the flag to false so the affected # distribution will be updated with the new publication - pulp2dists = distributor.pulp3_publication.pulp2distributor_set.all() + pulp2dists = ( + distributor.pulp3_publication.pulp2distributor_set.all() + ) for dist in pulp2dists: if dist.is_migrated: dist.is_migrated = False @@ -705,13 +793,15 @@ def pre_migrate_repocontent(repo): mongo_repocontent_qs = RepositoryContentUnit.objects(mongo_repocontent_q) repocontent = [] - for repocontent_data in mongo_repocontent_qs.exclude('repo_id').as_pymongo().no_cache(): + for repocontent_data in ( + mongo_repocontent_qs.exclude("repo_id").as_pymongo().no_cache() + ): item = Pulp2RepoContent( - pulp2_unit_id=repocontent_data['unit_id'], - pulp2_content_type_id=repocontent_data['unit_type_id'], + pulp2_unit_id=repocontent_data["unit_id"], + pulp2_content_type_id=repocontent_data["unit_type_id"], pulp2_repository=repo, - pulp2_created=repocontent_data['created'], - pulp2_updated=repocontent_data['updated'] + pulp2_created=repocontent_data["created"], + pulp2_updated=repocontent_data["updated"], ) repocontent.append(item) @@ -743,31 +833,43 @@ def handle_outdated_resources(plan): repos_to_consider = set(inplan_repos).intersection(repos_to_consider) mongo_repo_q = mongo_Q(repo_id__in=repos_to_consider) - mongo_repo_obj_ids = set(str(i.id) for i in Repository.objects(mongo_repo_q).only('id')) + mongo_repo_obj_ids = set( + str(i.id) for i in Repository.objects(mongo_repo_q).only("id") + ) repo_type_q = Q(pulp2_repo_type=plugin_plan.type) inplan_repo_q = Q(pulp2_object_id__in=mongo_repo_obj_ids) - Pulp2Repository.objects.filter(repo_type_q).exclude(inplan_repo_q).update(not_in_plan=True) + Pulp2Repository.objects.filter(repo_type_q).exclude(inplan_repo_q).update( + not_in_plan=True + ) # Mark removed or excluded importers inplan_imp_repos = plugin_plan.get_importers_repos() mongo_imp_q = mongo_Q(repo_id__in=inplan_imp_repos) - mongo_imp_obj_ids = set(str(i.id) for i in Importer.objects(mongo_imp_q).only('id')) + mongo_imp_obj_ids = set( + str(i.id) for i in Importer.objects(mongo_imp_q).only("id") + ) imp_types = plugin_plan.migrator.importer_migrators.keys() imp_type_q = Q(pulp2_type_id__in=imp_types) inplan_imp_q = Q(pulp2_object_id__in=mongo_imp_obj_ids) - Pulp2Importer.objects.filter(imp_type_q).exclude(inplan_imp_q).update(not_in_plan=True) + Pulp2Importer.objects.filter(imp_type_q).exclude(inplan_imp_q).update( + not_in_plan=True + ) # Mark removed or excluded distributors inplan_dist_repos = plugin_plan.get_distributors_repos() mongo_dist_q = mongo_Q(repo_id__in=inplan_dist_repos) - mongo_dist_obj_ids = set(str(i.id) for i in Distributor.objects(mongo_dist_q).only('id')) + mongo_dist_obj_ids = set( + str(i.id) for i in Distributor.objects(mongo_dist_q).only("id") + ) dist_types = plugin_plan.migrator.distributor_migrators.keys() dist_type_q = Q(pulp2_type_id__in=dist_types) inplan_dist_q = Q(pulp2_object_id__in=mongo_dist_obj_ids) - Pulp2Distributor.objects.filter(dist_type_q).exclude(inplan_dist_q).update(not_in_plan=True) + Pulp2Distributor.objects.filter(dist_type_q).exclude(inplan_dist_q).update( + not_in_plan=True + ) # Delete old Publications/Distributions which are no longer present in Pulp2. @@ -782,17 +884,19 @@ def handle_outdated_resources(plan): Q(is_migrated=False) | Q(not_in_plan=True) ) - old_dist_query = Q(pulp3_distribution__isnull=False) | Q(pulp3_publication__isnull=False) - old_dist_query &= Q(pulp2_repos__in=repos_with_old_distributions_qs) | Q(not_in_plan=True) + old_dist_query = Q(pulp3_distribution__isnull=False) | Q( + pulp3_publication__isnull=False + ) + old_dist_query &= Q(pulp2_repos__in=repos_with_old_distributions_qs) | Q( + not_in_plan=True + ) with transaction.atomic(): pulp2distributors_with_old_distributions_qs = Pulp2Distributor.objects.filter( old_dist_query ) - pulp2distributors_with_old_distributions_qs.update( - is_migrated=False - ) + pulp2distributors_with_old_distributions_qs.update(is_migrated=False) # If publication is shared by multiple distributions, on the corresponding distributors # flip the flag to false so the affected distributions will be updated with the new @@ -805,17 +909,21 @@ def handle_outdated_resources(plan): # Delete outdated publications Publication.objects.filter( - pulp2distributor__in=pulp2distributors_with_old_distributions_qs).delete() + pulp2distributor__in=pulp2distributors_with_old_distributions_qs + ).delete() # Delete outdated distributions Distribution.objects.filter( - pulp2distributor__in=pulp2distributors_with_old_distributions_qs).delete() + pulp2distributor__in=pulp2distributors_with_old_distributions_qs + ).delete() # Remove relations to the pulp2repository in case the relation changed. # Pulp2Distributors with is_migrated=false is handled and re-added properly at # migration stage. # NOTE: this needs to be removed last, the queries above use this relation. - not_migrated_dists = Pulp2Distributor.objects.filter(is_migrated=False).only('pulp_id') + not_migrated_dists = Pulp2Distributor.objects.filter(is_migrated=False).only( + "pulp_id" + ) Pulp2Distributor.pulp2_repos.through.objects.filter( pulp2distributor__in=not_migrated_dists ).delete() diff --git a/pulp_2to3_migration/app/serializers.py b/pulp_2to3_migration/app/serializers.py index 815a7b88..07561e0b 100644 --- a/pulp_2to3_migration/app/serializers.py +++ b/pulp_2to3_migration/app/serializers.py @@ -37,17 +37,16 @@ def get_pulp_href(obj): class MigrationPlanSerializer(ModelSerializer): """Serializer for migration plan model.""" - pulp_href = IdentityField( - view_name='migration-plans-detail' - ) + + pulp_href = IdentityField(view_name="migration-plans-detail") plan = serializers.JSONField( - help_text=_('Migration Plan in JSON format'), + help_text=_("Migration Plan in JSON format"), required=True, ) class Meta: - fields = ModelSerializer.Meta.fields + ('plan', ) + fields = ModelSerializer.Meta.fields + ("plan",) model = MigrationPlan def validate(self, data): @@ -59,10 +58,10 @@ def validate(self, data): """ schema = json.loads(SCHEMA) validator = Draft7Validator(schema) - if isinstance(data['plan'], str): - loaded_plan = json.loads(data['plan']) - elif isinstance(data['plan'], dict): - loaded_plan = data['plan'] + if isinstance(data["plan"], str): + loaded_plan = json.loads(data["plan"]) + elif isinstance(data["plan"], dict): + loaded_plan = data["plan"] else: raise serializers.ValidationError( _("Must provide a (JSON-encoded) string or dict for 'plan', not list") @@ -75,11 +74,13 @@ def validate(self, data): _("Provided Migration Plan format is invalid:'{}'".format(err)) ) plugins_to_migrate = set() - for plugin_type in loaded_plan['plugins']: - plugins_to_migrate.add(plugin_type['type']) - if len(loaded_plan['plugins']) != len(plugins_to_migrate): + for plugin_type in loaded_plan["plugins"]: + plugins_to_migrate.add(plugin_type["type"]) + if len(loaded_plan["plugins"]) != len(plugins_to_migrate): raise serializers.ValidationError( - _("Provided Migration Plan contains same plugin type specified more that once.") + _( + "Provided Migration Plan contains same plugin type specified more that once." + ) ) # MongoDB connection initialization connection.initialize() @@ -100,7 +101,7 @@ def validate(self, data): raise serializers.ValidationError( _("Plugin {} is not installed in pulp2.".format(plugin)) ) - data['plan'] = loaded_plan + data["plan"] = loaded_plan return data @@ -108,28 +109,35 @@ class MigrationPlanRunSerializer(serializers.Serializer): """ A serializer for running a migration plan. """ + validate = serializers.BooleanField( - help_text=_('If ``True``, migration cannot happen without successful validation ' - 'of the Migration Plan.'), + help_text=_( + "If ``True``, migration cannot happen without successful validation " + "of the Migration Plan." + ), required=False, default=False, - write_only=True + write_only=True, ) dry_run = serializers.BooleanField( - help_text=_('If ``True``, performs validation of a Migration Plan only, no migration is ' - 'run.'), + help_text=_( + "If ``True``, performs validation of a Migration Plan only, no migration is " + "run." + ), required=False, default=False, - write_only=True + write_only=True, ) skip_corrupted = serializers.BooleanField( - help_text=_('If ``True``, skips corrupted or missing Pulp 2 content without causing a task ' - 'failure. If you need this content, run a sync task in Pulp 3 for a repo of ' - 'interest to bring any missing content back. Alternatively, repair it in Pulp 2' - 'and re-run the migration task.'), + help_text=_( + "If ``True``, skips corrupted or missing Pulp 2 content without causing a task " + "failure. If you need this content, run a sync task in Pulp 3 for a repo of " + "interest to bring any missing content back. Alternatively, repair it in Pulp 2" + "and re-run the migration task." + ), required=False, default=False, - write_only=True + write_only=True, ) @@ -137,16 +145,17 @@ class Pulp2ContentSerializer(ModelSerializer): """ A serializer for the Pulp2Content model """ - pulp_href = IdentityField( - view_name='pulp2content-detail' - ) + + pulp_href = IdentityField(view_name="pulp2content-detail") pulp2_id = serializers.CharField(max_length=255) pulp2_content_type_id = serializers.CharField(max_length=255) pulp2_last_updated = serializers.IntegerField() pulp2_storage_path = serializers.CharField(allow_null=True) downloaded = serializers.BooleanField(default=False) pulp3_content = DetailRelatedField( - required=False, allow_null=True, queryset=Pulp2Content.objects.all(), + required=False, + allow_null=True, + queryset=Pulp2Content.objects.all(), view_name_pattern=r"content(-.*/.*)?-detail", ) @@ -172,15 +181,20 @@ def to_representation(self, instance): content doesn't belong to any pulp3_repository_version. """ result = super(Pulp2ContentSerializer, self).to_representation(instance) - if not result.get('pulp3_repository_version'): - result.pop('pulp3_repository_version', None) + if not result.get("pulp3_repository_version"): + result.pop("pulp3_repository_version", None) return result class Meta: - fields = ModelSerializer.Meta.fields + ('pulp2_id', 'pulp2_content_type_id', - 'pulp2_last_updated', 'pulp2_storage_path', - 'downloaded', 'pulp3_content', - 'pulp3_repository_version') + fields = ModelSerializer.Meta.fields + ( + "pulp2_id", + "pulp2_content_type_id", + "pulp2_last_updated", + "pulp2_storage_path", + "downloaded", + "pulp3_content", + "pulp3_repository_version", + ) model = Pulp2Content @@ -188,9 +202,8 @@ class Pulp2RepositoriesSerializer(ModelSerializer): """ A serializer for the Pulp2Repositories """ - pulp_href = IdentityField( - view_name='pulp2repositories-detail' - ) + + pulp_href = IdentityField(view_name="pulp2repositories-detail") pulp2_object_id = serializers.CharField(max_length=255) pulp2_repo_id = serializers.CharField() pulp2_repo_type = serializers.CharField() @@ -199,7 +212,7 @@ class Pulp2RepositoriesSerializer(ModelSerializer): pulp3_repository_version = RepositoryVersionRelatedField( required=False, - help_text=_('RepositoryVersion to be served'), + help_text=_("RepositoryVersion to be served"), allow_null=True, ) diff --git a/pulp_2to3_migration/app/settings.py b/pulp_2to3_migration/app/settings.py index 696470bc..cba489bb 100644 --- a/pulp_2to3_migration/app/settings.py +++ b/pulp_2to3_migration/app/settings.py @@ -1,17 +1,17 @@ PULP2_MONGODB = { - 'name': 'pulp_database', - 'seeds': 'localhost:27017', - 'username': '', - 'password': '', - 'replica_set': '', - 'ssl': False, - 'ssl_keyfile': '', - 'ssl_certfile': '', - 'verify_ssl': True, - 'ca_path': '/etc/pki/tls/certs/ca-bundle.crt', + "name": "pulp_database", + "seeds": "localhost:27017", + "username": "", + "password": "", + "replica_set": "", + "ssl": False, + "ssl_keyfile": "", + "ssl_certfile": "", + "verify_ssl": True, + "ca_path": "/etc/pki/tls/certs/ca-bundle.crt", } -ALLOWED_CONTENT_CHECKSUMS = ['md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512'] +ALLOWED_CONTENT_CHECKSUMS = ["md5", "sha1", "sha224", "sha256", "sha384", "sha512"] CONTENT_PREMIGRATION_BATCH_SIZE = 1000 diff --git a/pulp_2to3_migration/app/tasks/__init__.py b/pulp_2to3_migration/app/tasks/__init__.py index 88d425cc..13cd0948 100644 --- a/pulp_2to3_migration/app/tasks/__init__.py +++ b/pulp_2to3_migration/app/tasks/__init__.py @@ -1,2 +1,2 @@ from .migrate import migrate_from_pulp2 # noqa -from .reset import reset_pulp3_data # noqa \ No newline at end of file +from .reset import reset_pulp3_data # noqa diff --git a/pulp_2to3_migration/app/tasks/migrate.py b/pulp_2to3_migration/app/tasks/migrate.py index bd034f35..a837c587 100644 --- a/pulp_2to3_migration/app/tasks/migrate.py +++ b/pulp_2to3_migration/app/tasks/migrate.py @@ -27,7 +27,9 @@ _logger = logging.getLogger(__name__) -def migrate_from_pulp2(migration_plan_pk, validate=False, dry_run=False, skip_corrupted=False): +def migrate_from_pulp2( + migration_plan_pk, validate=False, dry_run=False, skip_corrupted=False +): """ Main task to migrate from Pulp 2 to Pulp 3. @@ -60,11 +62,13 @@ def migrate_from_pulp2(migration_plan_pk, validate=False, dry_run=False, skip_co GroupProgressReport( message="Repo version creation", code="create.repo_version", - task_group=task_group).save() + task_group=task_group, + ).save() GroupProgressReport( message="Distribution creation", code="create.distribution", - task_group=task_group).save() + task_group=task_group, + ).save() current_task = Task.current() current_task.task_group = task_group current_task.save() diff --git a/pulp_2to3_migration/app/tasks/reset.py b/pulp_2to3_migration/app/tasks/reset.py index 5a4ec243..9dc8d0b6 100644 --- a/pulp_2to3_migration/app/tasks/reset.py +++ b/pulp_2to3_migration/app/tasks/reset.py @@ -30,54 +30,68 @@ def reset_pulp3_data(migration_plan_pk): """ plan = MigrationPlan.objects.get(pk=migration_plan_pk) pulp2_plugins = plan.get_plugins() - pb_data = dict(message="Resetting data for Pulp 3 plugins", code="reset.pulp3.data", - total=len(pulp2_plugins)) + pb_data = dict( + message="Resetting data for Pulp 3 plugins", + code="reset.pulp3.data", + total=len(pulp2_plugins), + ) with ProgressReport(**pb_data) as pb: for plugin in plan.get_plugin_plans(): RepoSetup.reset_plugin(plugin.type) for dist_migrator in plugin.migrator.distributor_migrators.values(): for dist_model in dist_migrator.pulp3_distribution_models: - dist_model.objects.all().only('pk').delete() + dist_model.objects.all().only("pk").delete() for pub_model in dist_migrator.pulp3_publication_models: - pub_model.objects.all().only('pk').delete() + pub_model.objects.all().only("pk").delete() for imp_migrator in plugin.migrator.importer_migrators.values(): for remote_model in imp_migrator.pulp3_remote_models: - remote_model.objects.all().only('pk').delete() + remote_model.objects.all().only("pk").delete() repo_model = plugin.migrator.pulp3_repository - if hasattr(repo_model, 'sub_repo'): + if hasattr(repo_model, "sub_repo"): # sub_repos can't be deleted until the content referring to them is not removed - repo_model.objects.filter(sub_repo=False).only('pk').delete() + repo_model.objects.filter(sub_repo=False).only("pk").delete() else: - repo_model.objects.all().only('pk').delete() + repo_model.objects.all().only("pk").delete() for content_model in repo_model.CONTENT_TYPES: - content_model.objects.all().only('pk').delete() + content_model.objects.all().only("pk").delete() - if hasattr(repo_model, 'sub_repo'): + if hasattr(repo_model, "sub_repo"): # after content is removed we can delete the remaining repositories - repo_model.objects.all().only('pk').delete() + repo_model.objects.all().only("pk").delete() pb.increment() - pb_data = dict(message="Removing pre-migrated data", code="reset.premigrated.data", total=1) + pb_data = dict( + message="Removing pre-migrated data", code="reset.premigrated.data", total=1 + ) with ProgressReport(**pb_data) as pb: for plugin in plan.get_plugin_plans(): - for content_type, pulp2to3_content_model in plugin.migrator.content_models.items(): - pulp2to3_content_model.objects.all().only('pk').delete() - Pulp2Content.objects.filter(pulp2_content_type_id=content_type).only('pk').delete() + for ( + content_type, + pulp2to3_content_model, + ) in plugin.migrator.content_models.items(): + pulp2to3_content_model.objects.all().only("pk").delete() + Pulp2Content.objects.filter(pulp2_content_type_id=content_type).only( + "pk" + ).delete() Pulp2LazyCatalog.objects.filter( pulp2_content_type_id=content_type - ).only('pk').delete() + ).only("pk").delete() Pulp2RepoContent.objects.filter( pulp2_content_type_id=content_type - ).only('pk').delete() + ).only("pk").delete() for importer_type in plugin.migrator.importer_migrators: - Pulp2Importer.objects.filter(pulp2_type_id=importer_type).only('pk').delete() + Pulp2Importer.objects.filter(pulp2_type_id=importer_type).only( + "pk" + ).delete() for dist_type in plugin.migrator.distributor_migrators: - Pulp2Distributor.objects.filter(pulp2_type_id=dist_type).only('pk').delete() + Pulp2Distributor.objects.filter(pulp2_type_id=dist_type).only( + "pk" + ).delete() Pulp2Repository.objects.filter( pulp2_repo_type=plugin.migrator.pulp2_plugin - ).only('pk').delete() + ).only("pk").delete() pb.increment() diff --git a/pulp_2to3_migration/app/viewsets.py b/pulp_2to3_migration/app/viewsets.py index daf19ff4..2151bbea 100644 --- a/pulp_2to3_migration/app/viewsets.py +++ b/pulp_2to3_migration/app/viewsets.py @@ -9,7 +9,7 @@ from pulpcore.app.viewsets.base import DATETIME_FILTER_OPTIONS from pulpcore.app.viewsets.custom_filters import ( HyperlinkRelatedFilter, - IsoDateTimeFilter + IsoDateTimeFilter, ) from pulpcore.plugin.models import Task @@ -44,22 +44,20 @@ def is_migration_plan_running(): """ qs = Task.objects.filter( - state__in=['waiting', 'running'], - reserved_resources_record=['pulp_2to3_migration'] + state__in=["waiting", "running"], + reserved_resources_record=["pulp_2to3_migration"], ) if qs: return True groups_with_running_tasks = Task.objects.filter( - state__in=['waiting', 'running'], - task_group__isnull=False - ).values_list('task_group_id', flat=True) + state__in=["waiting", "running"], task_group__isnull=False + ).values_list("task_group_id", flat=True) groups_with_migration_tasks = Task.objects.filter( - task_group__isnull=False, - reserved_resources_record=['pulp_2to3_migration'] - ).values_list('task_group_id', flat=True) + task_group__isnull=False, reserved_resources_record=["pulp_2to3_migration"] + ).values_list("task_group_id", flat=True) if groups_with_running_tasks.intersection(groups_with_migration_tasks): return True @@ -67,71 +65,77 @@ def is_migration_plan_running(): return False -class MigrationPlanViewSet(NamedModelViewSet, - mixins.CreateModelMixin, - mixins.RetrieveModelMixin, - mixins.DestroyModelMixin, - mixins.ListModelMixin): +class MigrationPlanViewSet( + NamedModelViewSet, + mixins.CreateModelMixin, + mixins.RetrieveModelMixin, + mixins.DestroyModelMixin, + mixins.ListModelMixin, +): """ MigrationPlan ViewSet. """ - endpoint_name = 'migration-plans' + + endpoint_name = "migration-plans" queryset = MigrationPlan.objects.all() serializer_class = MigrationPlanSerializer @extend_schema( summary="Run migration plan", description="Trigger an asynchronous task to run a migration from Pulp 2.", - responses={202: AsyncOperationResponseSerializer} + responses={202: AsyncOperationResponseSerializer}, ) - @action(detail=True, methods=('post',), serializer_class=MigrationPlanRunSerializer) + @action(detail=True, methods=("post",), serializer_class=MigrationPlanRunSerializer) def run(self, request, pk): """Run the migration plan.""" migration_plan = self.get_object() serializer = MigrationPlanRunSerializer( - data=request.data, - context={'request': request} + data=request.data, context={"request": request} ) serializer.is_valid(raise_exception=True) - validate = serializer.validated_data.get('validate', False) - dry_run = serializer.validated_data.get('dry_run', False) - skip_corrupted = serializer.validated_data.get('skip_corrupted', False) + validate = serializer.validated_data.get("validate", False) + dry_run = serializer.validated_data.get("dry_run", False) + skip_corrupted = serializer.validated_data.get("skip_corrupted", False) if is_migration_plan_running(): - raise ValidationError(_("Only one migration plan can run or be reset at a time")) + raise ValidationError( + _("Only one migration plan can run or be reset at a time") + ) result = dispatch( migrate_from_pulp2, exclusive_resources=[PULP_2TO3_MIGRATION_RESOURCE], kwargs={ - 'migration_plan_pk': str(migration_plan.pk), - 'validate': validate, - 'dry_run': dry_run, - 'skip_corrupted': skip_corrupted - } + "migration_plan_pk": str(migration_plan.pk), + "validate": validate, + "dry_run": dry_run, + "skip_corrupted": skip_corrupted, + }, ) return OperationPostponedResponse(result, request) @extend_schema( summary="Reset Pulp 3 data for plugins specified in the migration plan", description="Trigger an asynchronous task to remove data from Pulp 3 related to the " - "plugins specified in the migration plan.", - responses={202: AsyncOperationResponseSerializer} + "plugins specified in the migration plan.", + responses={202: AsyncOperationResponseSerializer}, ) - @action(detail=True, methods=('post',), serializer_class=None) + @action(detail=True, methods=("post",), serializer_class=None) def reset(self, request, pk): """Reset Pulp 3 data for plugins specified in the migration plan.""" migration_plan = self.get_object() if is_migration_plan_running(): - raise ValidationError(_("Only one migration plan can run or be reset at a time")) + raise ValidationError( + _("Only one migration plan can run or be reset at a time") + ) result = dispatch( reset_pulp3_data, exclusive_resources=[PULP_2TO3_MIGRATION_RESOURCE], kwargs={ - 'migration_plan_pk': str(migration_plan.pk), - } + "migration_plan_pk": str(migration_plan.pk), + }, ) return OperationPostponedResponse(result, request) @@ -140,26 +144,30 @@ class Pulp2ContentFilter(BaseFilterSet): """ Filter for Pulp2Content ViewSet. """ + pulp2_id = filters.CharFilter() pulp2_content_type_id = filters.CharFilter() - pulp2_last_updated = IsoDateTimeFilter(field_name='pulp2_last_updated') + pulp2_last_updated = IsoDateTimeFilter(field_name="pulp2_last_updated") pulp3_content = HyperlinkRelatedFilter() class Meta: model = Pulp2Content fields = { - 'pulp2_id': ['exact', 'in'], - 'pulp2_content_type_id': ['exact', 'in'], - 'pulp2_last_updated': DATETIME_FILTER_OPTIONS, - 'pulp3_content': ['exact'] + "pulp2_id": ["exact", "in"], + "pulp2_content_type_id": ["exact", "in"], + "pulp2_last_updated": DATETIME_FILTER_OPTIONS, + "pulp3_content": ["exact"], } -class Pulp2ContentViewSet(NamedModelViewSet, mixins.RetrieveModelMixin, mixins.ListModelMixin): +class Pulp2ContentViewSet( + NamedModelViewSet, mixins.RetrieveModelMixin, mixins.ListModelMixin +): """ ViewSet for Pulp2Content model. """ - endpoint_name = 'pulp2content' + + endpoint_name = "pulp2content" queryset = Pulp2Content.objects.all() serializer_class = Pulp2ContentSerializer filterset_class = Pulp2ContentFilter @@ -169,6 +177,7 @@ class Pulp2RepositoriesFilter(BaseFilterSet): """ Filter for Pulp2Repositories ViewSet. """ + pulp2_repo_id = filters.CharFilter() is_migrated = filters.BooleanFilter() not_in_plan = filters.BooleanFilter() @@ -176,17 +185,20 @@ class Pulp2RepositoriesFilter(BaseFilterSet): class Meta: model = Pulp2Repository fields = { - 'pulp2_repo_id': ['exact', 'in'], - 'is_migrated': ['exact'], - 'not_in_plan': ['exact'] + "pulp2_repo_id": ["exact", "in"], + "is_migrated": ["exact"], + "not_in_plan": ["exact"], } -class Pulp2RepositoriesViewSet(NamedModelViewSet, mixins.RetrieveModelMixin, mixins.ListModelMixin): +class Pulp2RepositoriesViewSet( + NamedModelViewSet, mixins.RetrieveModelMixin, mixins.ListModelMixin +): """ ViewSet for Pulp2Repositories model. """ - endpoint_name = 'pulp2repositories' + + endpoint_name = "pulp2repositories" queryset = Pulp2Repository.objects.all() serializer_class = Pulp2RepositoriesSerializer filterset_class = Pulp2RepositoriesFilter diff --git a/pulp_2to3_migration/exceptions.py b/pulp_2to3_migration/exceptions.py index 5d17f974..02911852 100644 --- a/pulp_2to3_migration/exceptions.py +++ b/pulp_2to3_migration/exceptions.py @@ -27,6 +27,7 @@ class PlanValidationError(Exception): e.g. Repository specified does not exist. """ + pass @@ -35,6 +36,7 @@ class ArtifactValidationError(PulpException): Exception for the issues with artifact creation during migration. """ + def __init__(self, msg): """ :param msg: error message specifying what exactly is out of place diff --git a/pulp_2to3_migration/pulp2/base.py b/pulp_2to3_migration/pulp2/base.py index 0b871f15..4985c15a 100644 --- a/pulp_2to3_migration/pulp2/base.py +++ b/pulp_2to3_migration/pulp2/base.py @@ -19,13 +19,14 @@ class ContentUnit(Document): unit_key_fields must be a tuple of strings, each of which is a valid field name of the subcalss. """ + id = StringField(primary_key=True) pulp_user_metadata = DictField() _last_updated = IntField(required=True) _storage_path = StringField() meta = { - 'abstract': True, + "abstract": True, } @@ -33,10 +34,11 @@ class FileContentUnit(ContentUnit): """ A Pulp 2 content unit representing content that is of type *file*. """ + downloaded = BooleanField() meta = { - 'abstract': True, + "abstract": True, } @@ -44,7 +46,8 @@ class Repository(Document): """ Defines schema for a pulp 2 repository in the `repos` collection. """ - repo_id = StringField(required=True, regex=r'^[.\-_A-Za-z0-9]+$') + + repo_id = StringField(required=True, regex=r"^[.\-_A-Za-z0-9]+$") display_name = StringField() # not used in the migration plugin description = StringField() notes = DictField() # not used in the migration plugin @@ -54,10 +57,9 @@ class Repository(Document): last_unit_removed = DateTimeField() # For backward compatibility - _ns = StringField(default='repos') + _ns = StringField(default="repos") - meta = {'collection': 'repos', - 'allow_inheritance': False} + meta = {"collection": "repos", "allow_inheritance": False} class RepositoryContentUnit(Document): @@ -65,6 +67,7 @@ class RepositoryContentUnit(Document): Represents the link between a pulp2 repository and the units associated with it. Defines the schema for the documents in repo_content_units collection. """ + repo_id = StringField(required=True) unit_id = StringField(required=True) unit_type_id = StringField(required=True) @@ -72,16 +75,16 @@ class RepositoryContentUnit(Document): updated = StringField(required=True) # For backward compatibility - _ns = StringField(default='repo_content_units') + _ns = StringField(default="repo_content_units") - meta = {'collection': 'repo_content_units', - 'allow_inheritance': False} + meta = {"collection": "repo_content_units", "allow_inheritance": False} class Importer(Document): """ Defines schema for a pulp 2 importer in the `repo_importers` collection. """ + repo_id = StringField(required=True) importer_type_id = StringField(required=True) config = DictField() @@ -91,18 +94,18 @@ class Importer(Document): last_override_config = DictField() # not used in the migration plugin # For backward compatibility - _ns = StringField(default='repo_importers') + _ns = StringField(default="repo_importers") - meta = {'collection': 'repo_importers', - 'allow_inheritance': False} + meta = {"collection": "repo_importers", "allow_inheritance": False} class Distributor(Document): """ Defines schema for a pulp 2 distributor in the 'repo_distributors' collection. """ + repo_id = StringField(required=True) - distributor_id = StringField(required=True, regex=r'^[\-_A-Za-z0-9]+$') + distributor_id = StringField(required=True, regex=r"^[\-_A-Za-z0-9]+$") distributor_type_id = StringField(required=True) config = DictField() auto_publish = BooleanField(default=False) @@ -111,16 +114,16 @@ class Distributor(Document): last_override_config = DictField() # not used in the migration plugin scratchpad = DictField() # not used in the migration plugin - _ns = StringField(default='repo_distributors') + _ns = StringField(default="repo_distributors") - meta = {'collection': 'repo_distributors', - 'allow_inheritance': False} + meta = {"collection": "repo_distributors", "allow_inheritance": False} class LazyCatalogEntry(Document): """ A Pulp 2 catalog of content that can be downloaded from a specific URL. """ + path = StringField(required=True) importer_id = StringField(required=True) unit_id = StringField(required=True) @@ -132,9 +135,9 @@ class LazyCatalogEntry(Document): data = DictField() # not used in the migration plugin # For backward compatibility - _ns = StringField(default='lazy_content_catalog') + _ns = StringField(default="lazy_content_catalog") meta = { - 'collection': 'lazy_content_catalog', - 'allow_inheritance': False, + "collection": "lazy_content_catalog", + "allow_inheritance": False, } diff --git a/pulp_2to3_migration/pulp2/connection.py b/pulp_2to3_migration/pulp2/connection.py index 79121b3a..4b17b53b 100644 --- a/pulp_2to3_migration/pulp2/connection.py +++ b/pulp_2to3_migration/pulp2/connection.py @@ -40,7 +40,9 @@ raise ConfigurationError(_("PULP2_MONGODB is not configured in your settings.")) -def initialize(name=None, seeds=None, max_pool_size=None, replica_set=None, max_timeout=32): +def initialize( + name=None, seeds=None, max_pool_size=None, replica_set=None, max_timeout=32 +): """ Initialize the connection pool and top-level database for pulp. Calling this more than once will raise a RuntimeError. @@ -60,91 +62,119 @@ def initialize(name=None, seeds=None, max_pool_size=None, replica_set=None, max_ connection_kwargs = {} if name is None: - name = pulp2_mongodb.get('name') + name = pulp2_mongodb.get("name") if seeds is None: - seeds = pulp2_mongodb.get('seeds') - seeds_list = seeds.split(',') + seeds = pulp2_mongodb.get("seeds") + seeds_list = seeds.split(",") if max_pool_size is None: # we may want to make this configurable, but then again, we may not max_pool_size = _DEFAULT_MAX_POOL_SIZE - connection_kwargs['maxPoolSize'] = max_pool_size + connection_kwargs["maxPoolSize"] = max_pool_size if replica_set is None: - if pulp2_mongodb.get('replica_set'): - replica_set = pulp2_mongodb.get('replica_set') + if pulp2_mongodb.get("replica_set"): + replica_set = pulp2_mongodb.get("replica_set") if replica_set is not None: - connection_kwargs['replicaSet'] = replica_set + connection_kwargs["replicaSet"] = replica_set # Process SSL settings - if pulp2_mongodb.get('ssl'): - connection_kwargs['ssl'] = True - ssl_keyfile = pulp2_mongodb.get('ssl_keyfile') - ssl_certfile = pulp2_mongodb.get('ssl_certfile') + if pulp2_mongodb.get("ssl"): + connection_kwargs["ssl"] = True + ssl_keyfile = pulp2_mongodb.get("ssl_keyfile") + ssl_certfile = pulp2_mongodb.get("ssl_certfile") if ssl_keyfile: - connection_kwargs['ssl_keyfile'] = ssl_keyfile + connection_kwargs["ssl_keyfile"] = ssl_keyfile if ssl_certfile: - connection_kwargs['ssl_certfile'] = ssl_certfile - verify_ssl = pulp2_mongodb.get('verify_ssl') - connection_kwargs['ssl_cert_reqs'] = ssl.CERT_REQUIRED if verify_ssl else ssl.CERT_NONE - connection_kwargs['ssl_ca_certs'] = pulp2_mongodb.get('ca_path') + connection_kwargs["ssl_certfile"] = ssl_certfile + verify_ssl = pulp2_mongodb.get("verify_ssl") + connection_kwargs["ssl_cert_reqs"] = ( + ssl.CERT_REQUIRED if verify_ssl else ssl.CERT_NONE + ) + connection_kwargs["ssl_ca_certs"] = pulp2_mongodb.get("ca_path") # If username & password have been specified in the database config, # attempt to authenticate to the database - username = pulp2_mongodb.get('username') - password = pulp2_mongodb.get('password') + username = pulp2_mongodb.get("username") + password = pulp2_mongodb.get("password") if username: - _logger.debug(_('Attempting username and password authentication.')) - connection_kwargs['username'] = username - connection_kwargs['password'] = password + _logger.debug(_("Attempting username and password authentication.")) + connection_kwargs["username"] = username + connection_kwargs["password"] = password elif password and not username: - raise ConfigurationError(_("The config specified a database password, but is " - "missing a database username.")) + raise ConfigurationError( + _( + "The config specified a database password, but is " + "missing a database username." + ) + ) # Wait until the Mongo database is available - mongo_retry_timeout_seconds_generator = itertools.chain([1, 2, 4, 8, 16], - itertools.repeat(32)) + mongo_retry_timeout_seconds_generator = itertools.chain( + [1, 2, 4, 8, 16], itertools.repeat(32) + ) - if seeds != '': + if seeds != "": if len(seeds_list) > 1 and not replica_set: - raise ConfigurationError(_("Database 'replica_set' config must be specified " - "when more than one seed is provided.")) + raise ConfigurationError( + _( + "Database 'replica_set' config must be specified " + "when more than one seed is provided." + ) + ) while True: - _CONNECTION = _connect_to_one_of_seeds(connection_kwargs, seeds_list, name) + _CONNECTION = _connect_to_one_of_seeds( + connection_kwargs, seeds_list, name + ) if _CONNECTION: - db_version = semantic_version.Version(_CONNECTION.server_info()['version']) + db_version = semantic_version.Version( + _CONNECTION.server_info()["version"] + ) if db_version < MONGO_MINIMUM_VERSION: - raise RuntimeError(_("Pulp requires Mongo version %s, but DB is reporting" - "version %s") % (MONGO_MINIMUM_VERSION, - db_version)) + raise RuntimeError( + _( + "Pulp requires Mongo version %s, but DB is reporting" + "version %s" + ) + % (MONGO_MINIMUM_VERSION, db_version) + ) break else: - next_delay = min(next(mongo_retry_timeout_seconds_generator), max_timeout) - msg = _("Could not connect to any of MongoDB seeds at %(url)s:\n... Waiting " - "%(retry_timeout)s seconds and trying again.") - _logger.error(msg % {'retry_timeout': next_delay, 'url': seeds}) + next_delay = min( + next(mongo_retry_timeout_seconds_generator), max_timeout + ) + msg = _( + "Could not connect to any of MongoDB seeds at %(url)s:\n... Waiting " + "%(retry_timeout)s seconds and trying again." + ) + _logger.error(msg % {"retry_timeout": next_delay, "url": seeds}) time.sleep(next_delay) else: - raise ConfigurationError(_("Database 'seeds' config must include at least one " - "hostname:port value.")) + raise ConfigurationError( + _( + "Database 'seeds' config must include at least one " + "hostname:port value." + ) + ) try: _DATABASE = mongoengine.connection.get_db() except OperationFailure as error: if error.code == 18: - msg = _('Authentication to MongoDB ' - 'with username and password failed.') + msg = _( + "Authentication to MongoDB " "with username and password failed." + ) raise RuntimeError(msg) _DATABASE.add_son_manipulator(NamespaceInjector()) # Query the collection names to ensure that we are authenticated properly - _logger.debug(_('Querying the database to validate the connection.')) + _logger.debug(_("Querying the database to validate the connection.")) _DATABASE.collection_names() except Exception as e: - _logger.critical(_('Database initialization failed: %s') % str(e)) + _logger.critical(_("Database initialization failed: %s") % str(e)) _CONNECTION = None _DATABASE = None raise @@ -162,18 +192,18 @@ def _connect_to_one_of_seeds(connection_kwargs, seeds_list, db_name): """ connection_kwargs = copy.deepcopy(connection_kwargs) for seed in seeds_list: - connection_kwargs.update({'host': seed.strip()}) + connection_kwargs.update({"host": seed.strip()}) try: _logger.info("Attempting to connect to %(host)s" % connection_kwargs) shadow_connection_kwargs = copy.deepcopy(connection_kwargs) - if connection_kwargs.get('password'): - shadow_connection_kwargs['password'] = '*****' - _logger.debug(_('Connection Arguments: %s') % shadow_connection_kwargs) + if connection_kwargs.get("password"): + shadow_connection_kwargs["password"] = "*****" + _logger.debug(_("Connection Arguments: %s") % shadow_connection_kwargs) connection = mongoengine.connect(db_name, **connection_kwargs) return connection except mongoengine.MongoEngineConnectionError as e: msg = _("Could not connect to MongoDB at %(url)s:\n%(e)s\n") - _logger.info(msg % {'url': seed, 'e': str(e)}) + _logger.info(msg % {"url": seed, "e": str(e)}) class UnsafeRetry(object): @@ -181,12 +211,32 @@ class UnsafeRetry(object): Class that decorates PyMongo to retry in the event of AutoReconnect exceptions. """ - _decorated_methods = ('get_lasterror_options', 'set_lasterror_options', - 'unset_lasterror_options', 'insert', 'save', 'update', 'remove', 'drop', - 'find', 'find_one', 'count', 'create_index', 'ensure_index', - 'drop_index', 'drop_indexes', 'reindex', 'index_information', 'options', - 'group', 'rename', 'distinct', 'map_reduce', 'inline_map_reduce', - 'find_and_modify') + _decorated_methods = ( + "get_lasterror_options", + "set_lasterror_options", + "unset_lasterror_options", + "insert", + "save", + "update", + "remove", + "drop", + "find", + "find_one", + "count", + "create_index", + "ensure_index", + "drop_index", + "drop_indexes", + "reindex", + "index_information", + "options", + "group", + "rename", + "distinct", + "map_reduce", + "inline_map_reduce", + "find_and_modify", + ) @classmethod def decorate_instance(cls, instance, full_name): @@ -199,11 +249,15 @@ def decorate_instance(cls, instance, full_name): :type full_name: str """ - unsafe_autoretry = pulp2_mongodb.get('unsafe_autoretry') + unsafe_autoretry = pulp2_mongodb.get("unsafe_autoretry") if unsafe_autoretry: for m in cls._decorated_methods: try: - setattr(instance, m, cls.retry_decorator(full_name)(getattr(instance, m))) + setattr( + instance, + m, + cls.retry_decorator(full_name)(getattr(instance, m)), + ) except AttributeError: pass @@ -217,7 +271,6 @@ def retry_decorator(full_name=None): """ def _decorator(method): - @wraps(method) def retry(*args, **kwargs): while True: @@ -225,8 +278,10 @@ def retry(*args, **kwargs): return method(*args, **kwargs) except AutoReconnect: - msg = _('%(method)s operation failed on %(name)s') % { - 'method': method.__name__, 'name': full_name} + msg = _("%(method)s operation failed on %(name)s") % { + "method": method.__name__, + "name": full_name, + } _logger.error(msg) time.sleep(0.3) @@ -250,10 +305,10 @@ def __init__(self, database, name, create=False, **kwargs): UnsafeRetry.decorate_instance(instance=self, full_name=self.full_name) def __getstate__(self): - return {'name': self.name} + return {"name": self.name} def __setstate__(self, state): - return get_collection(state['name']) + return get_collection(state["name"]) def query(self, criteria): """ @@ -286,7 +341,7 @@ def get_collection(name, create=False): global _DATABASE if _DATABASE is None: - raise RuntimeError(_('Cannot get collection from uninitialized database')) + raise RuntimeError(_("Cannot get collection from uninitialized database")) return PulpCollection(_DATABASE, name, create=create) diff --git a/pulp_2to3_migration/tests/functional/common_plans.py b/pulp_2to3_migration/tests/functional/common_plans.py index a13198c5..c247b399 100644 --- a/pulp_2to3_migration/tests/functional/common_plans.py +++ b/pulp_2to3_migration/tests/functional/common_plans.py @@ -5,102 +5,111 @@ import json -DEB_COMPLEX_PLAN = json.dumps({ - "plugins": [{ - "type": "deb", - "repositories": [ +DEB_COMPLEX_PLAN = json.dumps( + { + "plugins": [ { - "name": "debian-empty", - "pulp2_importer_repository_id": "debian-empty", - "repository_versions": [ + "type": "deb", + "repositories": [ { - "pulp2_repository_id": "debian-empty", # content count: 0 - "pulp2_distributor_repository_ids": ["debian-empty"] - } - ] - }, - { - "name": "debian", - "pulp2_importer_repository_id": "debian", - "repository_versions": [ + "name": "debian-empty", + "pulp2_importer_repository_id": "debian-empty", + "repository_versions": [ + { + "pulp2_repository_id": "debian-empty", # content count: 0 + "pulp2_distributor_repository_ids": ["debian-empty"], + } + ], + }, { - "pulp2_repository_id": "debian", - "pulp2_distributor_repository_ids": ["debian"] - } - ] - }, - { - "name": "debian-complex-dists", - "pulp2_importer_repository_id": "debian-complex-dists", - "repository_versions": [ + "name": "debian", + "pulp2_importer_repository_id": "debian", + "repository_versions": [ + { + "pulp2_repository_id": "debian", + "pulp2_distributor_repository_ids": ["debian"], + } + ], + }, { - "pulp2_repository_id": "debian-complex-dists", - "pulp2_distributor_repository_ids": ["debian-complex-dists"] - } - ] - }, - { - "name": "debian_update", - "pulp2_importer_repository_id": "debian_update", - "repository_versions": [ + "name": "debian-complex-dists", + "pulp2_importer_repository_id": "debian-complex-dists", + "repository_versions": [ + { + "pulp2_repository_id": "debian-complex-dists", + "pulp2_distributor_repository_ids": [ + "debian-complex-dists" + ], + } + ], + }, { - "pulp2_repository_id": "debian_update", - "pulp2_distributor_repository_ids": ["debian_update"] - } - ] - }, - ], - - }] -}) + "name": "debian_update", + "pulp2_importer_repository_id": "debian_update", + "repository_versions": [ + { + "pulp2_repository_id": "debian_update", + "pulp2_distributor_repository_ids": ["debian_update"], + } + ], + }, + ], + } + ] + } +) -FILE_COMPLEX_PLAN = json.dumps({ - "plugins": [{ - "type": "iso", - "repositories": [ +FILE_COMPLEX_PLAN = json.dumps( + { + "plugins": [ { - "name": "file", - "pulp2_importer_repository_id": "file", # policy: immediate - "repository_versions": [ + "type": "iso", + "repositories": [ { - "pulp2_repository_id": "file", # content count: iso - 3 - "pulp2_distributor_repository_ids": ["file"] - } - ] - }, - { - "name": "file2", - "pulp2_importer_repository_id": "file2", # policy: on_demand - "repository_versions": [ + "name": "file", + "pulp2_importer_repository_id": "file", # policy: immediate + "repository_versions": [ + { + "pulp2_repository_id": "file", # content count: iso - 3 + "pulp2_distributor_repository_ids": ["file"], + } + ], + }, { - "pulp2_repository_id": "file2", # content count: iso - 3 - "pulp2_distributor_repository_ids": ["file2"] - } - ] - }, - { - "name": "file-large", - "pulp2_importer_repository_id": "file-large", # policy: immediate - "repository_versions": [ + "name": "file2", + "pulp2_importer_repository_id": "file2", # policy: on_demand + "repository_versions": [ + { + "pulp2_repository_id": "file2", # content count: iso - 3 + "pulp2_distributor_repository_ids": ["file2"], + } + ], + }, { - "pulp2_repository_id": "file-large", # content count: iso - 10 - "pulp2_distributor_repository_ids": ["file-large"] - } - ] - }, - { - "name": "file-many", - "pulp2_importer_repository_id": "file-many", # policy: on_demand - "repository_versions": [ + "name": "file-large", + "pulp2_importer_repository_id": "file-large", # policy: immediate + "repository_versions": [ + { + "pulp2_repository_id": "file-large", # content count: iso - 10 + "pulp2_distributor_repository_ids": ["file-large"], + } + ], + }, { - "pulp2_repository_id": "file-many", # content count: iso - 250 - "pulp2_distributor_repository_ids": ["file-many"] - } - ] - }, + "name": "file-many", + "pulp2_importer_repository_id": "file-many", # policy: on_demand + "repository_versions": [ + { + "pulp2_repository_id": "file-many", # content count: iso - 250 + "pulp2_distributor_repository_ids": ["file-many"], + } + ], + }, + ], + } ] - }] -}) + } +) INITIAL_REPOSITORIES = [ { @@ -109,9 +118,9 @@ "repository_versions": [ { "pulp2_repository_id": "rpm-empty", # content count: 0 - "pulp2_distributor_repository_ids": ["rpm-empty"] + "pulp2_distributor_repository_ids": ["rpm-empty"], } - ] + ], }, { "name": "rpm-empty-for-copy", @@ -119,9 +128,9 @@ "repository_versions": [ { "pulp2_repository_id": "rpm-empty-for-copy", # content count: 0 - "pulp2_distributor_repository_ids": ["rpm-empty-for-copy"] + "pulp2_distributor_repository_ids": ["rpm-empty-for-copy"], } - ] + ], }, { "name": "rpm-with-modules", @@ -129,9 +138,9 @@ "repository_versions": [ { "pulp2_repository_id": "rpm-with-modules", - "pulp2_distributor_repository_ids": ["rpm-with-modules"] + "pulp2_distributor_repository_ids": ["rpm-with-modules"], } - ] + ], }, { "name": "rpm-distribution-tree", @@ -139,9 +148,9 @@ "repository_versions": [ { "pulp2_repository_id": "rpm-distribution-tree", - "pulp2_distributor_repository_ids": ["rpm-distribution-tree"] + "pulp2_distributor_repository_ids": ["rpm-distribution-tree"], } - ] + ], }, { "name": "srpm-unsigned", @@ -149,74 +158,84 @@ "repository_versions": [ { "pulp2_repository_id": "srpm-unsigned", - "pulp2_distributor_repository_ids": ["srpm-unsigned"] + "pulp2_distributor_repository_ids": ["srpm-unsigned"], } - ] + ], }, ] -RPM_COMPLEX_PLAN = json.dumps({ - "plugins": [{ - "type": "rpm", - "repositories": INITIAL_REPOSITORIES, - }] -}) +RPM_COMPLEX_PLAN = json.dumps( + { + "plugins": [ + { + "type": "rpm", + "repositories": INITIAL_REPOSITORIES, + } + ] + } +) FILE_SIMPLE_PLAN = json.dumps({"plugins": [{"type": "iso"}]}) RPM_SIMPLE_PLAN = json.dumps({"plugins": [{"type": "rpm"}]}) DEB_SIMPLE_PLAN = json.dumps({"plugins": [{"type": "deb"}]}) -FILE_DISTRIBUTOR_DIFF_PLAN = json.dumps({ - "plugins": [{ - "type": "iso", - "repositories": [ +FILE_DISTRIBUTOR_DIFF_PLAN = json.dumps( + { + "plugins": [ { - "name": "file", - "pulp2_importer_repository_id": "file", # policy: immediate - "repository_versions": [ + "type": "iso", + "repositories": [ { - "pulp2_repository_id": "file", # content count: iso - 3 - "pulp2_distributor_repository_ids": ["file-many"] - } - ] - }, - { - "name": "file-many", - "pulp2_importer_repository_id": "file-many", # policy: on_demand - "repository_versions": [ + "name": "file", + "pulp2_importer_repository_id": "file", # policy: immediate + "repository_versions": [ + { + "pulp2_repository_id": "file", # content count: iso - 3 + "pulp2_distributor_repository_ids": ["file-many"], + } + ], + }, { - "pulp2_repository_id": "file-many", # content count: iso - 250 - "pulp2_distributor_repository_ids": ["file"] - } - ] + "name": "file-many", + "pulp2_importer_repository_id": "file-many", # policy: on_demand + "repository_versions": [ + { + "pulp2_repository_id": "file-many", # content count: iso - 250 + "pulp2_distributor_repository_ids": ["file"], + } + ], + }, + ], } ] - }] -}) + } +) -FILE_IMPORTER_DIFF_PLAN = json.dumps({ - "plugins": [{ - "type": "iso", - "repositories": [ +FILE_IMPORTER_DIFF_PLAN = json.dumps( + { + "plugins": [ { - "name": "file", - "pulp2_importer_repository_id": "file-many", # policy: on_demand - "repository_versions": [ + "type": "iso", + "repositories": [ { - "pulp2_repository_id": "file" # content count: iso - 3 - } - ] - }, - { - "name": "file-many", - "pulp2_importer_repository_id": "file", # policy: immediate - "repository_versions": [ + "name": "file", + "pulp2_importer_repository_id": "file-many", # policy: on_demand + "repository_versions": [ + {"pulp2_repository_id": "file"} # content count: iso - 3 + ], + }, { - "pulp2_repository_id": "file-many" # content count: iso - 250 - } - ] + "name": "file-many", + "pulp2_importer_repository_id": "file", # policy: immediate + "repository_versions": [ + { + "pulp2_repository_id": "file-many" # content count: iso - 250 + } + ], + }, + ], } ] - }] -}) + } +) diff --git a/pulp_2to3_migration/tests/functional/constants.py b/pulp_2to3_migration/tests/functional/constants.py index b3a18be1..0114280a 100644 --- a/pulp_2to3_migration/tests/functional/constants.py +++ b/pulp_2to3_migration/tests/functional/constants.py @@ -1,36 +1,32 @@ BINDINGS_CONFIGURATION = { - 'username': 'admin', - 'password': 'password', - 'host': 'https://pulp', + "username": "admin", + "password": "password", + "host": "https://pulp", } TABLES_TO_KEEP = ( # django's sqlclear or sqlflush excludes this table when cleaning up the db - 'django_migrations', - + "django_migrations", # not to create an admin user every time - 'auth_user', - + "auth_user", # not to be doomed by the lack of permissions - 'auth_group', - 'auth_permission', - 'core_accesspolicy', - 'core_role', - 'core_role_permissions', - 'core_userrole', - 'core_grouprole', - + "auth_group", + "auth_permission", + "core_accesspolicy", + "core_role", + "core_role_permissions", + "core_userrole", + "core_grouprole", # 'auth_permission' references it, so it should not be truncated - 'django_content_type', - + "django_content_type", # not to freak out the tasking system - 'core_reservedresource', - 'core_reservedresourcerecord', - 'core_task', - 'core_taskgroup', - 'core_taskreservedresource', - 'core_taskreservedresourcerecord', - 'core_worker', + "core_reservedresource", + "core_reservedresourcerecord", + "core_task", + "core_taskgroup", + "core_taskreservedresource", + "core_taskreservedresourcerecord", + "core_worker", ) TRUNCATE_TABLES_QUERY_BASH = f""" @@ -46,6 +42,8 @@ $$; """ # noqa -FILE_URL = 'https://repos.fedorapeople.org/pulp/pulp/fixtures/file/PULP_MANIFEST' -FILE_MANY_URL = 'https://repos.fedorapeople.org/pulp/pulp/fixtures/file-many/PULP_MANIFEST' -FIXTURES_BASE_URL = 'https://fixtures.pulpproject.org/' +FILE_URL = "https://repos.fedorapeople.org/pulp/pulp/fixtures/file/PULP_MANIFEST" +FILE_MANY_URL = ( + "https://repos.fedorapeople.org/pulp/pulp/fixtures/file-many/PULP_MANIFEST" +) +FIXTURES_BASE_URL = "https://fixtures.pulpproject.org/" diff --git a/pulp_2to3_migration/tests/functional/dynaconf_config.py b/pulp_2to3_migration/tests/functional/dynaconf_config.py index 4e1817db..dcbdf73b 100644 --- a/pulp_2to3_migration/tests/functional/dynaconf_config.py +++ b/pulp_2to3_migration/tests/functional/dynaconf_config.py @@ -2,8 +2,8 @@ settings = Dynaconf( settings_files=[ - 'func_test_settings.py', - '/pulp-2to3-migration/func_test_settings.py', - '/etc/pulp/settings.py' + "func_test_settings.py", + "/pulp-2to3-migration/func_test_settings.py", + "/etc/pulp/settings.py", ] ) diff --git a/pulp_2to3_migration/tests/functional/file_base.py b/pulp_2to3_migration/tests/functional/file_base.py index 62ea74f9..105a3ffd 100644 --- a/pulp_2to3_migration/tests/functional/file_base.py +++ b/pulp_2to3_migration/tests/functional/file_base.py @@ -30,6 +30,7 @@ class BaseTestFile: """ Test ISO migration. """ + smash_cfg = smash_config.get_config() smash_cli_client = cli.Client(smash_cfg) @@ -67,7 +68,7 @@ def run_migration(cls, plan, run_params={}): task(pulpcore.app.models.Task): a migration task created for this plan """ - mp = cls.migration_plans_api.create({'plan': plan}) + mp = cls.migration_plans_api.create({"plan": plan}) mp_run_response = cls.migration_plans_api.run(mp.pulp_href, run_params) task = monitor_task(mp_run_response.task) monitor_task_group(task.task_group) diff --git a/pulp_2to3_migration/tests/functional/rpm_base.py b/pulp_2to3_migration/tests/functional/rpm_base.py index 1436164d..c5bf5ada 100644 --- a/pulp_2to3_migration/tests/functional/rpm_base.py +++ b/pulp_2to3_migration/tests/functional/rpm_base.py @@ -38,6 +38,7 @@ class BaseTestRpm: """ Test RPM migration re-runs. """ + smash_cfg = smash_config.get_config() smash_cli_client = cli.Client(smash_cfg) plan_initial = None @@ -60,15 +61,15 @@ def setUpClass(cls): cls.rpm_distribution_api = DistributionsRpmApi(rpm_client) cls.rpm_publication_api = PublicationsRpmApi(rpm_client) cls.rpm_content_apis = { - 'advisory': ContentAdvisoriesApi(rpm_client), - 'disttree': ContentDistributionTreesApi(rpm_client), - 'modulemd': ContentModulemdsApi(rpm_client), - 'modulemd-defaults': ContentModulemdDefaultsApi(rpm_client), - 'category': ContentPackagecategoriesApi(rpm_client), - 'environment': ContentPackageenvironmentsApi(rpm_client), - 'group': ContentPackagegroupsApi(rpm_client), - 'langpack': ContentPackagelangpacksApi(rpm_client), - 'package': ContentPackagesApi(rpm_client), + "advisory": ContentAdvisoriesApi(rpm_client), + "disttree": ContentDistributionTreesApi(rpm_client), + "modulemd": ContentModulemdsApi(rpm_client), + "modulemd-defaults": ContentModulemdDefaultsApi(rpm_client), + "category": ContentPackagecategoriesApi(rpm_client), + "environment": ContentPackageenvironmentsApi(rpm_client), + "group": ContentPackagegroupsApi(rpm_client), + "langpack": ContentPackagelangpacksApi(rpm_client), + "package": ContentPackagesApi(rpm_client), } cls.migration_plans_api = MigrationPlansApi(migration_client) cls.pulp2content_api = Pulp2ContentApi(migration_client) @@ -87,7 +88,7 @@ def run_migration(cls, plan, run_params={}): task(pulpcore.app.models.Task): a migration task created for this plan """ - mp = cls.migration_plans_api.create({'plan': plan}) + mp = cls.migration_plans_api.create({"plan": plan}) mp_run_response = cls.migration_plans_api.run(mp.pulp_href, run_params) task = monitor_task(mp_run_response.task) monitor_task_group(task.task_group) @@ -121,9 +122,8 @@ def __init__(self, *args, is_simple=False, **kwargs): @property def repositories(self): - """Return dictionaries of content counts. - """ - repositories = self.get('content_rerun', None) or self['content_initial'] + """Return dictionaries of content counts.""" + repositories = self.get("content_rerun", None) or self["content_initial"] if self.is_simple: return {k: v for k, v in repositories.items() if v} else: @@ -131,43 +131,38 @@ def repositories(self): @property def new_repositories(self): - """Return the dictionaries for new repositories only. - """ + """Return the dictionaries for new repositories only.""" repos = [] - for repo_name, new_content in self['content_rerun'].items(): + for repo_name, new_content in self["content_rerun"].items(): if self.is_simple: - initial_content = self['content_initial'].get(repo_name, None) + initial_content = self["content_initial"].get(repo_name, None) if not initial_content and new_content: repos.append(repo_name) else: - if repo_name not in self['content_initial']: + if repo_name not in self["content_initial"]: repos.append(repo_name) return repos @property def content_total(self): - """Return content count dictionary. - """ + """Return content count dictionary.""" # we can't just sum up the content counts because the same content could be in # multiple repos - return self['content_total'] + return self["content_total"] @property def remotes(self): - """Return the count of remotes. - """ + """Return the count of remotes.""" # for complex plans, you can use one remote for many repos, so we can't assume # the number of repos == the number of repositories - return self['remotes'] + return self["remotes"] @property def publications(self): - """Return the count of publications. - """ + """Return the count of publications.""" return len(self.repositories) @property def distributions(self): - """Return the count of distributions. - """ + """Return the count of distributions.""" return len(self.repositories) diff --git a/pulp_2to3_migration/tests/functional/test_file_migration.py b/pulp_2to3_migration/tests/functional/test_file_migration.py index 8c37bd67..c87921fb 100644 --- a/pulp_2to3_migration/tests/functional/test_file_migration.py +++ b/pulp_2to3_migration/tests/functional/test_file_migration.py @@ -4,81 +4,85 @@ from pulp_2to3_migration.tests.functional.util import ( get_psql_smash_cmd, - set_pulp2_snapshot + set_pulp2_snapshot, ) from .constants import TRUNCATE_TABLES_QUERY_BASH from .file_base import BaseTestFile -PULP_2_ISO_FIXTURE_DATA = { - 'file': 3, - 'file2': 3, - 'file-many': 250, - 'file-large': 10 -} +PULP_2_ISO_FIXTURE_DATA = {"file": 3, "file2": 3, "file-many": 250, "file-large": 10} EMPTY_ISO_MIGRATION_PLAN = json.dumps({"plugins": [{"type": "iso"}]}) -SPECIFIC_REPOS_MIGRATION_PLAN = json.dumps({ - "plugins": [{ - "type": "iso", - "repositories": [ +SPECIFIC_REPOS_MIGRATION_PLAN = json.dumps( + { + "plugins": [ { - "name": "file", - "pulp2_importer_repository_id": "file", - "repository_versions": [ + "type": "iso", + "repositories": [ { - "pulp2_repository_id": "file", - "pulp2_distributor_repository_ids": ["file"] - } - ] - }, - { - "name": "file2", - "pulp2_importer_repository_id": "file2", - "repository_versions": [ + "name": "file", + "pulp2_importer_repository_id": "file", + "repository_versions": [ + { + "pulp2_repository_id": "file", + "pulp2_distributor_repository_ids": ["file"], + } + ], + }, { - "pulp2_repository_id": "file2", - "pulp2_distributor_repository_ids": ["file2"] - } - ] - }, + "name": "file2", + "pulp2_importer_repository_id": "file2", + "repository_versions": [ + { + "pulp2_repository_id": "file2", + "pulp2_distributor_repository_ids": ["file2"], + } + ], + }, + ], + } ] - }] -}) -DIFFERENT_IMPORTER_MIGRATION_PLAN = json.dumps({ - "plugins": [{ - "type": "iso", - "repositories": [ + } +) +DIFFERENT_IMPORTER_MIGRATION_PLAN = json.dumps( + { + "plugins": [ { - "name": "file", - "pulp2_importer_repository_id": "file2", - "repository_versions": [ + "type": "iso", + "repositories": [ { - "pulp2_repository_id": "file", - "pulp2_distributor_repository_ids": ["file"] - } - ] - }, - { - "name": "file2", - "pulp2_importer_repository_id": "file2", - "repository_versions": [ + "name": "file", + "pulp2_importer_repository_id": "file2", + "repository_versions": [ + { + "pulp2_repository_id": "file", + "pulp2_distributor_repository_ids": ["file"], + } + ], + }, { - "pulp2_repository_id": "file2", - "pulp2_distributor_repository_ids": ["file2"] - } - ] - }, + "name": "file2", + "pulp2_importer_repository_id": "file2", + "repository_versions": [ + { + "pulp2_repository_id": "file2", + "pulp2_distributor_repository_ids": ["file2"], + } + ], + }, + ], + } ] - }] -}) + } +) # TODO: # - Check that distributions are created properly # - Check that remotes are created properly + class TestMigrationPlan(BaseTestFile, unittest.TestCase): """Test the APIs for creating a Migration Plan.""" @@ -88,7 +92,7 @@ def setUpClass(cls): Populate needed pulp2 snapshot. """ super().setUpClass() - set_pulp2_snapshot(name='file_base_4repos') + set_pulp2_snapshot(name="file_base_4repos") def tearDown(self): """ @@ -104,17 +108,25 @@ def _do_test(self, repos, migration_plan): for repo_id in repos: pulp3repos = self.file_repo_api.list(name=repo_id) # Assert that there is a result - self.failIf(not pulp3repos.results, - "Missing a Pulp 3 repository for Pulp 2 " - "repository id '{}'".format(repo_id)) + self.failIf( + not pulp3repos.results, + "Missing a Pulp 3 repository for Pulp 2 " + "repository id '{}'".format(repo_id), + ) repo_href = pulp3repos.results[0].pulp_href # Assert that the name in pulp 3 matches the repo_id in pulp 2 self.assertEqual(repo_id, pulp3repos.results[0].name) # Assert that there is a Repository Version with the same number of content units as # associated with the repository in Pulp 2. - repo_version_href = self.file_repo_versions_api.list(repo_href).results[0].pulp_href - repo_version_content = self.file_content_api.list(repository_version=repo_version_href) - self.assertEqual(PULP_2_ISO_FIXTURE_DATA[repo_id], repo_version_content.count) + repo_version_href = ( + self.file_repo_versions_api.list(repo_href).results[0].pulp_href + ) + repo_version_content = self.file_content_api.list( + repository_version=repo_version_href + ) + self.assertEqual( + PULP_2_ISO_FIXTURE_DATA[repo_id], repo_version_content.count + ) # TODO: count only not_in_plan=False repositories from ../pulp2repositories/ endpoint self.assertEqual(len(repos), self.file_repo_api.list().count) diff --git a/pulp_2to3_migration/tests/functional/test_file_reset.py b/pulp_2to3_migration/tests/functional/test_file_reset.py index da9341d0..6db6f193 100644 --- a/pulp_2to3_migration/tests/functional/test_file_reset.py +++ b/pulp_2to3_migration/tests/functional/test_file_reset.py @@ -1,7 +1,10 @@ import time import unittest -from pulp_2to3_migration.tests.functional.util import get_psql_smash_cmd, set_pulp2_snapshot +from pulp_2to3_migration.tests.functional.util import ( + get_psql_smash_cmd, + set_pulp2_snapshot, +) from pulpcore.client.pulp_2to3_migration.exceptions import ApiException from pulp_smash.pulp3.bindings import monitor_task, monitor_task_group @@ -12,8 +15,8 @@ PULP_2_ISO_FIXTURE_DATA = { - 'repositories': 4, - 'content': 266, + "repositories": 4, + "content": 266, } @@ -26,7 +29,7 @@ def setUpClass(cls): Populate needed pulp2 snapshot. """ super().setUpClass() - set_pulp2_snapshot(name='file_base_4repos') + set_pulp2_snapshot(name="file_base_4repos") def tearDown(self): """ @@ -46,17 +49,22 @@ def test_reset_file_plugin(self): self.run_migration(FILE_SIMPLE_PLAN) # Assert that pre-migrated data is there - self.assertEqual(self.pulp2repositories_api.list().count, - PULP_2_ISO_FIXTURE_DATA['repositories']) - self.assertEqual(self.pulp2content_api.list().count, - PULP_2_ISO_FIXTURE_DATA['content']) + self.assertEqual( + self.pulp2repositories_api.list().count, + PULP_2_ISO_FIXTURE_DATA["repositories"], + ) + self.assertEqual( + self.pulp2content_api.list().count, PULP_2_ISO_FIXTURE_DATA["content"] + ) # Assert that Pulp 3 data is there - self.assertEqual(self.file_repo_api.list().count, - PULP_2_ISO_FIXTURE_DATA['repositories']) - self.assertEqual(self.file_content_api.list().count, - PULP_2_ISO_FIXTURE_DATA['content']) - - mp = self.migration_plans_api.create({'plan': FILE_SIMPLE_PLAN}) + self.assertEqual( + self.file_repo_api.list().count, PULP_2_ISO_FIXTURE_DATA["repositories"] + ) + self.assertEqual( + self.file_content_api.list().count, PULP_2_ISO_FIXTURE_DATA["content"] + ) + + mp = self.migration_plans_api.create({"plan": FILE_SIMPLE_PLAN}) self._reset_pulp3_data(mp) # Assert that pre-migrated data is no longer there self.assertEqual(self.pulp2repositories_api.list().count, 0) @@ -74,44 +82,59 @@ def test_no_reset_plugin_data(self): """ self.run_migration(FILE_SIMPLE_PLAN) # Assert that pre-migrated data is there - self.assertEqual(self.pulp2repositories_api.list().count, - PULP_2_ISO_FIXTURE_DATA['repositories']) - self.assertEqual(self.pulp2content_api.list().count, - PULP_2_ISO_FIXTURE_DATA['content']) + self.assertEqual( + self.pulp2repositories_api.list().count, + PULP_2_ISO_FIXTURE_DATA["repositories"], + ) + self.assertEqual( + self.pulp2content_api.list().count, PULP_2_ISO_FIXTURE_DATA["content"] + ) # Assert that Pulp 3 data is there - self.assertEqual(self.file_repo_api.list().count, - PULP_2_ISO_FIXTURE_DATA['repositories']) - self.assertEqual(self.file_content_api.list().count, - PULP_2_ISO_FIXTURE_DATA['content']) - - mp_rpm = self.migration_plans_api.create({'plan': RPM_SIMPLE_PLAN}) + self.assertEqual( + self.file_repo_api.list().count, PULP_2_ISO_FIXTURE_DATA["repositories"] + ) + self.assertEqual( + self.file_content_api.list().count, PULP_2_ISO_FIXTURE_DATA["content"] + ) + + mp_rpm = self.migration_plans_api.create({"plan": RPM_SIMPLE_PLAN}) self._reset_pulp3_data(mp_rpm) # Assert that pre-migrated data is still there - self.assertEqual(self.pulp2repositories_api.list().count, - PULP_2_ISO_FIXTURE_DATA['repositories']) - self.assertEqual(self.pulp2content_api.list().count, - PULP_2_ISO_FIXTURE_DATA['content']) + self.assertEqual( + self.pulp2repositories_api.list().count, + PULP_2_ISO_FIXTURE_DATA["repositories"], + ) + self.assertEqual( + self.pulp2content_api.list().count, PULP_2_ISO_FIXTURE_DATA["content"] + ) # Assert that Pulp 3 data is still there - self.assertEqual(self.file_repo_api.list().count, - PULP_2_ISO_FIXTURE_DATA['repositories']) - self.assertEqual(self.file_content_api.list().count, - PULP_2_ISO_FIXTURE_DATA['content']) + self.assertEqual( + self.file_repo_api.list().count, PULP_2_ISO_FIXTURE_DATA["repositories"] + ) + self.assertEqual( + self.file_content_api.list().count, PULP_2_ISO_FIXTURE_DATA["content"] + ) def test_migrate_after_reset(self): """Test that migration runs successfully after the reset of Pulp 3 and pre-migrated data.""" self.run_migration(FILE_SIMPLE_PLAN) # Assert that pre-migrated data is there - self.assertEqual(self.pulp2repositories_api.list().count, - PULP_2_ISO_FIXTURE_DATA['repositories']) - self.assertEqual(self.pulp2content_api.list().count, - PULP_2_ISO_FIXTURE_DATA['content']) + self.assertEqual( + self.pulp2repositories_api.list().count, + PULP_2_ISO_FIXTURE_DATA["repositories"], + ) + self.assertEqual( + self.pulp2content_api.list().count, PULP_2_ISO_FIXTURE_DATA["content"] + ) # Assert that Pulp 3 data is there - self.assertEqual(self.file_repo_api.list().count, - PULP_2_ISO_FIXTURE_DATA['repositories']) - self.assertEqual(self.file_content_api.list().count, - PULP_2_ISO_FIXTURE_DATA['content']) - - mp = self.migration_plans_api.create({'plan': FILE_SIMPLE_PLAN}) + self.assertEqual( + self.file_repo_api.list().count, PULP_2_ISO_FIXTURE_DATA["repositories"] + ) + self.assertEqual( + self.file_content_api.list().count, PULP_2_ISO_FIXTURE_DATA["content"] + ) + + mp = self.migration_plans_api.create({"plan": FILE_SIMPLE_PLAN}) self._reset_pulp3_data(mp) # Assert that pre-migrated data is no longer there self.assertEqual(self.pulp2repositories_api.list().count, 0) @@ -122,19 +145,24 @@ def test_migrate_after_reset(self): self.run_migration(FILE_SIMPLE_PLAN) # Assert that pre-migrated data is back there - self.assertEqual(self.pulp2repositories_api.list().count, - PULP_2_ISO_FIXTURE_DATA['repositories']) - self.assertEqual(self.pulp2content_api.list().count, - PULP_2_ISO_FIXTURE_DATA['content']) + self.assertEqual( + self.pulp2repositories_api.list().count, + PULP_2_ISO_FIXTURE_DATA["repositories"], + ) + self.assertEqual( + self.pulp2content_api.list().count, PULP_2_ISO_FIXTURE_DATA["content"] + ) # Assert that Pulp 3 data is back there - self.assertEqual(self.file_repo_api.list().count, - PULP_2_ISO_FIXTURE_DATA['repositories']) - self.assertEqual(self.file_content_api.list().count, - PULP_2_ISO_FIXTURE_DATA['content']) + self.assertEqual( + self.file_repo_api.list().count, PULP_2_ISO_FIXTURE_DATA["repositories"] + ) + self.assertEqual( + self.file_content_api.list().count, PULP_2_ISO_FIXTURE_DATA["content"] + ) def test_no_reset_when_migration(self): """Test that reset is not run when migration is.""" - mp = self.migration_plans_api.create({'plan': FILE_SIMPLE_PLAN}) + mp = self.migration_plans_api.create({"plan": FILE_SIMPLE_PLAN}) # run the migration plan and then immediately run reset without waiting mp_run_response = self.migration_plans_api.run(mp.pulp_href, {}) diff --git a/pulp_2to3_migration/tests/functional/test_migration_behaviour.py b/pulp_2to3_migration/tests/functional/test_migration_behaviour.py index ea1b603b..fcb88e98 100644 --- a/pulp_2to3_migration/tests/functional/test_migration_behaviour.py +++ b/pulp_2to3_migration/tests/functional/test_migration_behaviour.py @@ -2,7 +2,10 @@ import time import unittest -from pulp_2to3_migration.tests.functional.util import get_psql_smash_cmd, set_pulp2_snapshot +from pulp_2to3_migration.tests.functional.util import ( + get_psql_smash_cmd, + set_pulp2_snapshot, +) from .common_plans import ( FILE_COMPLEX_PLAN, @@ -13,123 +16,138 @@ from .constants import FILE_MANY_URL, FILE_URL, TRUNCATE_TABLES_QUERY_BASH from .file_base import BaseTestFile -MANY_INTO_ONE_PLAN = json.dumps({ - "plugins": [{ - "type": "iso", - "repositories": [ +MANY_INTO_ONE_PLAN = json.dumps( + { + "plugins": [ { - "name": "file", - "pulp2_importer_repository_id": "file-many", # policy: on_demand - "repository_versions": [ - { - "pulp2_repository_id": "file" # content count: iso - 3 - }, + "type": "iso", + "repositories": [ { - "pulp2_repository_id": "file-large" # content count: iso - 10 - }, - { - "pulp2_repository_id": "file-many" # content count: iso - 250 + "name": "file", + "pulp2_importer_repository_id": "file-many", # policy: on_demand + "repository_versions": [ + {"pulp2_repository_id": "file"}, # content count: iso - 3 + { + "pulp2_repository_id": "file-large" # content count: iso - 10 + }, + { + "pulp2_repository_id": "file-many" # content count: iso - 250 + }, + ], } - ] + ], } ] - }] -}) + } +) -IMPORTER_NO_REPO_PLAN = json.dumps({ - "plugins": [{ - "type": "iso", - "repositories": [ +IMPORTER_NO_REPO_PLAN = json.dumps( + { + "plugins": [ { - "name": "file", - "pulp2_importer_repository_id": "file-many", # policy: on_demand - "repository_versions": [ + "type": "iso", + "repositories": [ { - "pulp2_repository_id": "file" # content count: iso - 3 + "name": "file", + "pulp2_importer_repository_id": "file-many", # policy: on_demand + "repository_versions": [ + {"pulp2_repository_id": "file"} # content count: iso - 3 + ], } - ] + ], } ] - }] -}) + } +) -DISTRIBUTOR_NO_REPO_PLAN = json.dumps({ - "plugins": [{ - "type": "iso", - "repositories": [ +DISTRIBUTOR_NO_REPO_PLAN = json.dumps( + { + "plugins": [ { - "name": "file", - "pulp2_importer_repository_id": "file", # policy: immediate - "repository_versions": [ + "type": "iso", + "repositories": [ { - "pulp2_repository_id": "file", # content count: iso - 3 - "pulp2_distributor_repository_ids": ["file-many"] + "name": "file", + "pulp2_importer_repository_id": "file", # policy: immediate + "repository_versions": [ + { + "pulp2_repository_id": "file", # content count: iso - 3 + "pulp2_distributor_repository_ids": ["file-many"], + } + ], } - ] + ], } ] - }] -}) + } +) -NO_ON_DEMAND_IMPORTER_PLAN = json.dumps({ - "plugins": [{ - "type": "iso", - "repositories": [ +NO_ON_DEMAND_IMPORTER_PLAN = json.dumps( + { + "plugins": [ { - "name": "file-many", - "pulp2_importer_repository_id": "file", # policy: immediate - "repository_versions": [ + "type": "iso", + "repositories": [ { - "pulp2_repository_id": "file-many", # content count: iso - 250 - "pulp2_distributor_repository_ids": ["file-many"] + "name": "file-many", + "pulp2_importer_repository_id": "file", # policy: immediate + "repository_versions": [ + { + "pulp2_repository_id": "file-many", # content count: iso - 250 + "pulp2_distributor_repository_ids": ["file-many"], + } + ], } - ] + ], } ] - }] -}) + } +) -NO_IMMEDIATE_IMPORTER_PLAN = json.dumps({ - "plugins": [{ - "type": "iso", - "repositories": [ +NO_IMMEDIATE_IMPORTER_PLAN = json.dumps( + { + "plugins": [ { - "name": "file", - "pulp2_importer_repository_id": "file-many", # policy: on_demand - "repository_versions": [ + "type": "iso", + "repositories": [ { - "pulp2_repository_id": "file", # content count: iso - 3 - "pulp2_distributor_repository_ids": ["file"] + "name": "file", + "pulp2_importer_repository_id": "file-many", # policy: on_demand + "repository_versions": [ + { + "pulp2_repository_id": "file", # content count: iso - 3 + "pulp2_distributor_repository_ids": ["file"], + } + ], } - ] + ], } ] - }] -}) + } +) -NO_IMPORTER_PLAN = json.dumps({ - "plugins": [{ - "type": "iso", - "repositories": [ +NO_IMPORTER_PLAN = json.dumps( + { + "plugins": [ { - "name": "file", - "repository_versions": [ + "type": "iso", + "repositories": [ { - "pulp2_repository_id": "file", # content count: iso - 3 - "pulp2_distributor_repository_ids": ["file"] + "name": "file", + "repository_versions": [ + { + "pulp2_repository_id": "file", # content count: iso - 3 + "pulp2_distributor_repository_ids": ["file"], + } + ], } - ] + ], } ] - }] -}) + } +) -CONTENT_COUNT = { - 'file': 3, - 'file2': 3, - 'file-large': 10, - 'file-many': 250 -} +CONTENT_COUNT = {"file": 3, "file2": 3, "file-large": 10, "file-many": 250} class TestMigrationBehaviour(BaseTestFile, unittest.TestCase): @@ -141,7 +159,7 @@ def setUpClass(cls): Populate needed pulp2 snapshot. """ super().setUpClass() - set_pulp2_snapshot(name='file_base_4repos') + set_pulp2_snapshot(name="file_base_4repos") def tearDown(self): """ @@ -158,10 +176,12 @@ def _test_pulp2content(self, plan): Check the first content in the list. """ self.run_migration(plan) - pulp2content = self.pulp2content_api.list(ordering='pulp2_id', limit=1).results[0] + pulp2content = self.pulp2content_api.list(ordering="pulp2_id", limit=1).results[ + 0 + ] content_href = pulp2content.pulp3_content file_content = self.file_content_api.read(content_href) - self.assertEqual(file_content.relative_path, '137.iso') + self.assertEqual(file_content.relative_path, "137.iso") def _test_pulp2repositories(self, plan): """ @@ -171,25 +191,33 @@ def _test_pulp2repositories(self, plan): """ self.run_migration(plan) pulp2repository = self.pulp2repositories_api.list( - ordering='pulp2_repo_id', limit=1 + ordering="pulp2_repo_id", limit=1 ).results[0] pulp3_repo = self.file_repo_api.read(pulp2repository.pulp3_repository_href) pulp3_remote = self.file_remote_api.read(pulp2repository.pulp3_remote_href) - pulp3_pub = self.file_publication_api.read(pulp2repository.pulp3_publication_href) - pulp3_dist = self.file_distribution_api.read(pulp2repository.pulp3_distribution_hrefs[0]) + pulp3_pub = self.file_publication_api.read( + pulp2repository.pulp3_publication_href + ) + pulp3_dist = self.file_distribution_api.read( + pulp2repository.pulp3_distribution_hrefs[0] + ) self.assertEqual(self.pulp2repositories_api.list().count, 4) self.assertTrue(pulp2repository.is_migrated) - self.assertEqual(pulp2repository.pulp2_repo_id, 'file') - self.assertEqual(pulp3_repo.name, 'file') - self.assertEqual(pulp3_repo.latest_version_href, pulp2repository.pulp3_repository_version) + self.assertEqual(pulp2repository.pulp2_repo_id, "file") + self.assertEqual(pulp3_repo.name, "file") + self.assertEqual( + pulp3_repo.latest_version_href, pulp2repository.pulp3_repository_version + ) self.assertEqual(pulp3_remote.url, FILE_URL) - self.assertEqual(pulp3_remote.policy, 'immediate') + self.assertEqual(pulp3_remote.policy, "immediate") self.assertEqual(pulp3_remote.pulp_href, pulp2repository.pulp3_remote_href) - self.assertEqual(pulp3_pub.manifest, 'PULP_MANIFEST') - self.assertEqual(pulp3_pub.repository_version, pulp2repository.pulp3_repository_version) + self.assertEqual(pulp3_pub.manifest, "PULP_MANIFEST") + self.assertEqual( + pulp3_pub.repository_version, pulp2repository.pulp3_repository_version + ) self.assertEqual(pulp3_pub.distributions[0], pulp3_dist.pulp_href) - self.assertEqual(pulp3_dist.base_path, 'file') + self.assertEqual(pulp3_dist.base_path, "file") def test_pulp2content_simple_plan(self): """Test pulp2content endpoint for a simple plan.""" @@ -219,7 +247,9 @@ def test_many_into_one_repo(self): pulp3_repo = self.file_repo_api.list().results[0] self.assertEqual(self.file_repo_api.list().count, 1) - self.assertEqual(self.file_repo_versions_api.list(pulp3_repo.pulp_href).count, 4) + self.assertEqual( + self.file_repo_versions_api.list(pulp3_repo.pulp_href).count, 4 + ) self.assertEqual(self.file_content_api.list().count, 263) for pulp2repo in pulp2repositories: with self.subTest(pulp2repo=pulp2repo): @@ -228,7 +258,9 @@ def test_many_into_one_repo(self): repo_content = self.file_content_api.list( repository_version=pulp2repo.pulp3_repository_version ) - self.assertEqual(repo_content.count, CONTENT_COUNT[pulp2repo.pulp2_repo_id]) + self.assertEqual( + repo_content.count, CONTENT_COUNT[pulp2repo.pulp2_repo_id] + ) def test_importer_different_repo(self): """ @@ -237,21 +269,23 @@ def test_importer_different_repo(self): Importers are swapped in the plan. """ self.run_migration(FILE_IMPORTER_DIFF_PLAN) - pulp2repositories = self.pulp2repositories_api.list(ordering='pulp2_repo_id').results + pulp2repositories = self.pulp2repositories_api.list( + ordering="pulp2_repo_id" + ).results pulp2repo1, pulp2repo2 = pulp2repositories pulp3_remote1 = self.file_remote_api.read(pulp2repo1.pulp3_remote_href) pulp3_remote2 = self.file_remote_api.read(pulp2repo2.pulp3_remote_href) pulp3_repo1 = self.file_repo_api.read(pulp2repo1.pulp3_repository_href) pulp3_repo2 = self.file_repo_api.read(pulp2repo2.pulp3_repository_href) - self.assertEqual(pulp2repo1.pulp2_repo_id, 'file') - self.assertEqual(pulp2repo2.pulp2_repo_id, 'file-many') + self.assertEqual(pulp2repo1.pulp2_repo_id, "file") + self.assertEqual(pulp2repo2.pulp2_repo_id, "file-many") self.assertTrue(pulp2repo1.is_migrated) self.assertTrue(pulp2repo2.is_migrated) self.assertEqual(pulp3_remote1.url, FILE_MANY_URL) self.assertEqual(pulp3_remote2.url, FILE_URL) - self.assertEqual(pulp3_remote1.policy, 'on_demand') - self.assertEqual(pulp3_remote2.policy, 'immediate') + self.assertEqual(pulp3_remote1.policy, "on_demand") + self.assertEqual(pulp3_remote2.policy, "immediate") self.assertEqual(pulp3_repo1.remote, pulp2repo1.pulp3_remote_href) self.assertEqual(pulp3_repo2.remote, pulp2repo2.pulp3_remote_href) @@ -262,23 +296,33 @@ def test_distributor_different_repo(self): Distributors are swapped in the plan. """ self.run_migration(FILE_DISTRIBUTOR_DIFF_PLAN) - pulp2repositories = self.pulp2repositories_api.list(ordering='pulp2_repo_id').results + pulp2repositories = self.pulp2repositories_api.list( + ordering="pulp2_repo_id" + ).results pulp2repo1, pulp2repo2 = pulp2repositories pulp3_pub1 = self.file_publication_api.read(pulp2repo1.pulp3_publication_href) pulp3_pub2 = self.file_publication_api.read(pulp2repo2.pulp3_publication_href) - pulp3_dist1 = self.file_distribution_api.read(pulp2repo1.pulp3_distribution_hrefs[0]) - pulp3_dist2 = self.file_distribution_api.read(pulp2repo2.pulp3_distribution_hrefs[0]) - - self.assertEqual(pulp2repo1.pulp2_repo_id, 'file') - self.assertEqual(pulp2repo2.pulp2_repo_id, 'file-many') + pulp3_dist1 = self.file_distribution_api.read( + pulp2repo1.pulp3_distribution_hrefs[0] + ) + pulp3_dist2 = self.file_distribution_api.read( + pulp2repo2.pulp3_distribution_hrefs[0] + ) + + self.assertEqual(pulp2repo1.pulp2_repo_id, "file") + self.assertEqual(pulp2repo2.pulp2_repo_id, "file-many") self.assertTrue(pulp2repo1.is_migrated) self.assertTrue(pulp2repo2.is_migrated) - self.assertEqual(pulp3_pub1.repository_version, pulp2repo1.pulp3_repository_version) - self.assertEqual(pulp3_pub2.repository_version, pulp2repo2.pulp3_repository_version) + self.assertEqual( + pulp3_pub1.repository_version, pulp2repo1.pulp3_repository_version + ) + self.assertEqual( + pulp3_pub2.repository_version, pulp2repo2.pulp3_repository_version + ) self.assertEqual(pulp3_pub1.distributions[0], pulp3_dist1.pulp_href) self.assertEqual(pulp3_pub2.distributions[0], pulp3_dist2.pulp_href) - self.assertEqual(pulp3_dist1.base_path, 'file-many') - self.assertEqual(pulp3_dist2.base_path, 'file') + self.assertEqual(pulp3_dist1.base_path, "file-many") + self.assertEqual(pulp3_dist2.base_path, "file") def test_importer_no_repo(self): """Test that an importer can be migrated without its native Pulp 2 repo.""" @@ -290,57 +334,81 @@ def test_importer_no_repo(self): self.assertEqual(self.pulp2repositories_api.list().count, 1) self.assertEqual(self.file_remote_api.list().count, 1) self.assertTrue(pulp2repository.is_migrated) - self.assertEqual(pulp2repository.pulp2_repo_id, 'file') + self.assertEqual(pulp2repository.pulp2_repo_id, "file") self.assertEqual(pulp3_remote.url, FILE_MANY_URL) - self.assertEqual(pulp3_remote.policy, 'on_demand') + self.assertEqual(pulp3_remote.policy, "on_demand") self.assertEqual(pulp3_repo.remote, pulp2repository.pulp3_remote_href) def test_distributor_no_repo(self): """Test that a distributor can be migrated without its native Pulp 2 repo.""" self.run_migration(DISTRIBUTOR_NO_REPO_PLAN) pulp2repository = self.pulp2repositories_api.list().results[0] - pulp3_pub = self.file_publication_api.read(pulp2repository.pulp3_publication_href) - pulp3_dist = self.file_distribution_api.read(pulp2repository.pulp3_distribution_hrefs[0]) + pulp3_pub = self.file_publication_api.read( + pulp2repository.pulp3_publication_href + ) + pulp3_dist = self.file_distribution_api.read( + pulp2repository.pulp3_distribution_hrefs[0] + ) self.assertEqual(self.pulp2repositories_api.list().count, 1) self.assertEqual(self.file_distribution_api.list().count, 1) self.assertTrue(pulp2repository.is_migrated) - self.assertEqual(pulp2repository.pulp2_repo_id, 'file') - self.assertEqual(pulp3_pub.repository_version, pulp2repository.pulp3_repository_version) + self.assertEqual(pulp2repository.pulp2_repo_id, "file") + self.assertEqual( + pulp3_pub.repository_version, pulp2repository.pulp3_repository_version + ) self.assertEqual(pulp3_pub.distributions[0], pulp3_dist.pulp_href) - self.assertEqual(pulp3_dist.base_path, 'file-many') + self.assertEqual(pulp3_dist.base_path, "file-many") def test_no_on_demand_importer(self): """Test that if there is no importer for on_demand content, such content is not migrated.""" self.run_migration(NO_ON_DEMAND_IMPORTER_PLAN) pulp3_repo = self.file_repo_api.list().results[0] - repo_content = self.file_content_api.list(repository_version=pulp3_repo.latest_version_href) + repo_content = self.file_content_api.list( + repository_version=pulp3_repo.latest_version_href + ) - self.assertEqual(self.file_repo_versions_api.list(pulp3_repo.pulp_href).count, 1) + self.assertEqual( + self.file_repo_versions_api.list(pulp3_repo.pulp_href).count, 1 + ) self.assertEqual(repo_content.count, 0) def test_no_immediate_importer(self): """Test that if there is no importer for downloaded content, such content is migrated.""" self.run_migration(NO_IMMEDIATE_IMPORTER_PLAN) pulp3_repo = self.file_repo_api.list().results[0] - repo_content = self.file_content_api.list(repository_version=pulp3_repo.latest_version_href) + repo_content = self.file_content_api.list( + repository_version=pulp3_repo.latest_version_href + ) - self.assertEqual(self.file_repo_versions_api.list(pulp3_repo.pulp_href).count, 2) + self.assertEqual( + self.file_repo_versions_api.list(pulp3_repo.pulp_href).count, 2 + ) self.assertEqual(repo_content.count, 3) def test_no_importer(self): """Test that if there is no importer specified at all, migration is still working fine.""" self.run_migration(NO_IMPORTER_PLAN) pulp3_repo = self.file_repo_api.list().results[0] - repo_content = self.file_content_api.list(repository_version=pulp3_repo.latest_version_href) + repo_content = self.file_content_api.list( + repository_version=pulp3_repo.latest_version_href + ) pulp2repository = self.pulp2repositories_api.list().results[0] - pulp3_pub = self.file_publication_api.read(pulp2repository.pulp3_publication_href) - pulp3_dist = self.file_distribution_api.read(pulp2repository.pulp3_distribution_hrefs[0]) - - self.assertEqual(self.file_repo_versions_api.list(pulp3_repo.pulp_href).count, 2) + pulp3_pub = self.file_publication_api.read( + pulp2repository.pulp3_publication_href + ) + pulp3_dist = self.file_distribution_api.read( + pulp2repository.pulp3_distribution_hrefs[0] + ) + + self.assertEqual( + self.file_repo_versions_api.list(pulp3_repo.pulp_href).count, 2 + ) self.assertEqual(repo_content.count, 3) - self.assertEqual(pulp2repository.pulp2_repo_id, 'file') + self.assertEqual(pulp2repository.pulp2_repo_id, "file") self.assertEqual(pulp3_repo.remote, pulp2repository.pulp3_remote_href) - self.assertEqual(pulp3_pub.repository_version, pulp2repository.pulp3_repository_version) + self.assertEqual( + pulp3_pub.repository_version, pulp2repository.pulp3_repository_version + ) self.assertEqual(pulp3_pub.distributions[0], pulp3_dist.pulp_href) - self.assertEqual(pulp3_dist.base_path, 'file') + self.assertEqual(pulp3_dist.base_path, "file") diff --git a/pulp_2to3_migration/tests/functional/test_migration_plan.py b/pulp_2to3_migration/tests/functional/test_migration_plan.py index 248660a4..a201a09b 100644 --- a/pulp_2to3_migration/tests/functional/test_migration_plan.py +++ b/pulp_2to3_migration/tests/functional/test_migration_plan.py @@ -3,7 +3,10 @@ import unittest from pulpcore.client.pulp_2to3_migration.exceptions import ApiException -from pulp_2to3_migration.tests.functional.util import get_psql_smash_cmd, set_pulp2_snapshot +from pulp_2to3_migration.tests.functional.util import ( + get_psql_smash_cmd, + set_pulp2_snapshot, +) from pulp_smash.pulp3.bindings import monitor_task, monitor_task_group, PulpTaskError from .common_plans import FILE_SIMPLE_PLAN, FILE_COMPLEX_PLAN @@ -11,27 +14,35 @@ from .file_base import BaseTestFile EXTRA_COMMA_PLAN = '{"plugins": [{"type": "iso"},]}' -MISSING_RESOURCE_PLAN = json.dumps({ - "plugins": [{ - "type": "iso", - "repositories": [ +MISSING_RESOURCE_PLAN = json.dumps( + { + "plugins": [ { - "name": "file", - "pulp2_importer_repository_id": "non-existing importer", # policy: immediate - "repository_versions": [ + "type": "iso", + "repositories": [ { - "pulp2_repository_id": "non-existing repo", # content count: iso - 3 - "pulp2_distributor_repository_ids": ["non-existing distributor"] + "name": "file", + "pulp2_importer_repository_id": "non-existing importer", # policy:immediate + "repository_versions": [ + { + "pulp2_repository_id": "non-existing repo", # content count: iso-3 + "pulp2_distributor_repository_ids": [ + "non-existing distributor" + ], + } + ], } - ] + ], } ] - }] -}) -MISSING_RESOURCE_ERROR = 'Validation failed: resources missing '\ - '{\'repositories\': [\'non-existing repo\'], '\ - '\'repositories_missing_importers\': [\'non-existing importer\'], '\ - '\'repositories_missing_distributors\': [\'non-existing distributor\']}' + } +) +MISSING_RESOURCE_ERROR = ( + "Validation failed: resources missing " + "{'repositories': ['non-existing repo'], " + "'repositories_missing_importers': ['non-existing importer'], " + "'repositories_missing_distributors': ['non-existing distributor']}" +) UNKNOWN_KEY_PLAN = json.dumps({"plugins": [{"type": "iso", "unknown key": "value"}]}) @@ -44,7 +55,7 @@ def setUpClass(cls): Populate needed pulp2 snapshot. """ super().setUpClass() - set_pulp2_snapshot(name='file_base_4repos') + set_pulp2_snapshot(name="file_base_4repos") def tearDown(self): """ @@ -56,7 +67,7 @@ def tearDown(self): def _do_test_parallel(self, plan, outcome): """Test that there were multiple tasks running in parallel as a part of a task group.""" - mp = self.migration_plans_api.create({'plan': plan}) + mp = self.migration_plans_api.create({"plan": plan}) mp_run_response = self.migration_plans_api.run(mp.pulp_href, {}) task = monitor_task(mp_run_response.task) group = monitor_task_group(task.task_group) @@ -70,34 +81,36 @@ def test_load_complex_plan(self): """Test that a complex Migration Plan can be loaded and run.""" self.run_migration(FILE_COMPLEX_PLAN) - @unittest.skip('not fixed yet, https://pulp.plan.io/issues/7948') + @unittest.skip("not fixed yet, https://pulp.plan.io/issues/7948") def test_load_extra_comma_plan(self): """Test that a proper exception is risen when there is a syntax error in a plan.""" with self.assertRaises(ApiException) as exc: - self.migration_plans_api.create({'plan': EXTRA_COMMA_PLAN}) + self.migration_plans_api.create({"plan": EXTRA_COMMA_PLAN}) self.assertEqual(exc.code, 400) def test_unknown_key_plan(self): """Test that Migration Plan creation fails if some unknown keys are mentioned in it.""" with self.assertRaises(ApiException) as exc: - self.migration_plans_api.create({'plan': UNKNOWN_KEY_PLAN}) + self.migration_plans_api.create({"plan": UNKNOWN_KEY_PLAN}) self.assertEqual(exc.code, 400) def test_validate_plan(self): """Test that pulp 2 resources are validated.""" - self.run_migration(FILE_COMPLEX_PLAN, {'validate': True}) + self.run_migration(FILE_COMPLEX_PLAN, {"validate": True}) def test_validate_missing_resource(self): """Test that pulp 2 missing resource is noticed and reported.""" - mp = self.migration_plans_api.create({'plan': MISSING_RESOURCE_PLAN}) - mp_run_response = self.migration_plans_api.run(mp.pulp_href, {'validate': True}) + mp = self.migration_plans_api.create({"plan": MISSING_RESOURCE_PLAN}) + mp_run_response = self.migration_plans_api.run(mp.pulp_href, {"validate": True}) with self.assertRaises(PulpTaskError) as exc: monitor_task(mp_run_response.task) - self.assertEqual(exc.exception.task.error['description'], MISSING_RESOURCE_ERROR) + self.assertEqual( + exc.exception.task.error["description"], MISSING_RESOURCE_ERROR + ) def test_run_only_one_plan(self): """Test that only one plan can be run at a time""" - mp = self.migration_plans_api.create({'plan': FILE_SIMPLE_PLAN}) + mp = self.migration_plans_api.create({"plan": FILE_SIMPLE_PLAN}) # run twice mp_run_response = self.migration_plans_api.run(mp.pulp_href, {}) diff --git a/pulp_2to3_migration/tests/functional/test_migration_plan_changes.py b/pulp_2to3_migration/tests/functional/test_migration_plan_changes.py index 0bb3047b..132b068d 100644 --- a/pulp_2to3_migration/tests/functional/test_migration_plan_changes.py +++ b/pulp_2to3_migration/tests/functional/test_migration_plan_changes.py @@ -2,67 +2,85 @@ import time import unittest -from .common_plans import FILE_COMPLEX_PLAN, FILE_DISTRIBUTOR_DIFF_PLAN, FILE_IMPORTER_DIFF_PLAN +from .common_plans import ( + FILE_COMPLEX_PLAN, + FILE_DISTRIBUTOR_DIFF_PLAN, + FILE_IMPORTER_DIFF_PLAN, +) from .constants import FILE_MANY_URL, FILE_URL, TRUNCATE_TABLES_QUERY_BASH from .file_base import BaseTestFile from .util import get_psql_smash_cmd, set_pulp2_snapshot -FILE_2DISTRIBUTORS_PLAN = json.dumps({ # 2 distributors for the file repo, and one - for file2. - "plugins": [{ - "type": "iso", - "repositories": [ +FILE_2DISTRIBUTORS_PLAN = json.dumps( + { # 2 distributors for the file repo, and one - for file2. + "plugins": [ { - "name": "file", - "pulp2_importer_repository_id": "file", # policy: immediate - "repository_versions": [ + "type": "iso", + "repositories": [ { - "pulp2_repository_id": "file", # content count: iso - 3 - "pulp2_distributor_repository_ids": ["file", "file-many"] - } - ] - }, - { - "name": "file2", - "pulp2_importer_repository_id": "file2", # policy: on_demand - "repository_versions": [ + "name": "file", + "pulp2_importer_repository_id": "file", # policy: immediate + "repository_versions": [ + { + "pulp2_repository_id": "file", # content count: iso - 3 + "pulp2_distributor_repository_ids": [ + "file", + "file-many", + ], + } + ], + }, { - "pulp2_repository_id": "file2", # content count: iso - 3 - "pulp2_distributor_repository_ids": ["file2"] - } - ] - }, + "name": "file2", + "pulp2_importer_repository_id": "file2", # policy: on_demand + "repository_versions": [ + { + "pulp2_repository_id": "file2", # content count: iso - 3 + "pulp2_distributor_repository_ids": ["file2"], + } + ], + }, + ], + } ] - }] -}) + } +) -FILE_2DISTRIBUTORS_MOVED_PLAN = json.dumps({ # now a distributor moved from file repo to file2 one. - "plugins": [{ - "type": "iso", - "repositories": [ +FILE_2DISTRIBUTORS_MOVED_PLAN = json.dumps( + { # now a distributor moved from file repo to file2 one. + "plugins": [ { - "name": "file", - "pulp2_importer_repository_id": "file", # policy: immediate - "repository_versions": [ + "type": "iso", + "repositories": [ { - "pulp2_repository_id": "file", # content count: iso - 3 - "pulp2_distributor_repository_ids": ["file"] - } - ] - }, - { - "name": "file2", - "pulp2_importer_repository_id": "file2", # policy: on_demand - "repository_versions": [ + "name": "file", + "pulp2_importer_repository_id": "file", # policy: immediate + "repository_versions": [ + { + "pulp2_repository_id": "file", # content count: iso - 3 + "pulp2_distributor_repository_ids": ["file"], + } + ], + }, { - "pulp2_repository_id": "file2", # content count: iso - 3 - "pulp2_distributor_repository_ids": ["file2", "file-many"] - } - ] - }, + "name": "file2", + "pulp2_importer_repository_id": "file2", # policy: on_demand + "repository_versions": [ + { + "pulp2_repository_id": "file2", # content count: iso - 3 + "pulp2_distributor_repository_ids": [ + "file2", + "file-many", + ], + } + ], + }, + ], + } ] - }] -}) + } +) class TestMigrationPlanChanges(BaseTestFile, unittest.TestCase): @@ -74,7 +92,7 @@ def setUpClass(cls): Populate needed pulp2 snapshot. """ super().setUpClass() - set_pulp2_snapshot(name='file_base_4repos') + set_pulp2_snapshot(name="file_base_4repos") def tearDown(self): """ @@ -93,33 +111,51 @@ def test_importer_swap(self): """ # run for the first time with the standard plan self.run_migration(FILE_COMPLEX_PLAN) - pulp2repo_file_1run = self.pulp2repositories_api.list(pulp2_repo_id='file').results[0] + pulp2repo_file_1run = self.pulp2repositories_api.list( + pulp2_repo_id="file" + ).results[0] pulp2repo_filemany_1run = self.pulp2repositories_api.list( - pulp2_repo_id='file-many').results[0] - pulp3_repo_file_1run = self.file_repo_api.read(pulp2repo_file_1run.pulp3_repository_href) + pulp2_repo_id="file-many" + ).results[0] + pulp3_repo_file_1run = self.file_repo_api.read( + pulp2repo_file_1run.pulp3_repository_href + ) pulp3_repo_filemany_1run = self.file_repo_api.read( pulp2repo_filemany_1run.pulp3_repository_href ) - pulp3_remote_file_1run = self.file_remote_api.read(pulp2repo_file_1run.pulp3_remote_href) + pulp3_remote_file_1run = self.file_remote_api.read( + pulp2repo_file_1run.pulp3_remote_href + ) pulp3_remote_filemany_1run = self.file_remote_api.read( pulp2repo_filemany_1run.pulp3_remote_href ) self.assertEqual(pulp3_remote_file_1run.url, FILE_URL) self.assertEqual(pulp3_remote_filemany_1run.url, FILE_MANY_URL) - self.assertEqual(pulp3_repo_file_1run.remote, pulp2repo_file_1run.pulp3_remote_href) - self.assertEqual(pulp3_repo_filemany_1run.remote, pulp2repo_filemany_1run.pulp3_remote_href) + self.assertEqual( + pulp3_repo_file_1run.remote, pulp2repo_file_1run.pulp3_remote_href + ) + self.assertEqual( + pulp3_repo_filemany_1run.remote, pulp2repo_filemany_1run.pulp3_remote_href + ) # run a plan with swapped importers self.run_migration(FILE_IMPORTER_DIFF_PLAN) - pulp2repo_file_2run = self.pulp2repositories_api.list(pulp2_repo_id='file').results[0] + pulp2repo_file_2run = self.pulp2repositories_api.list( + pulp2_repo_id="file" + ).results[0] pulp2repo_filemany_2run = self.pulp2repositories_api.list( - pulp2_repo_id='file-many').results[0] - pulp3_repo_file_2run = self.file_repo_api.read(pulp2repo_file_2run.pulp3_repository_href) + pulp2_repo_id="file-many" + ).results[0] + pulp3_repo_file_2run = self.file_repo_api.read( + pulp2repo_file_2run.pulp3_repository_href + ) pulp3_repo_filemany_2run = self.file_repo_api.read( pulp2repo_filemany_2run.pulp3_repository_href ) - pulp3_remote_file_2run = self.file_remote_api.read(pulp2repo_file_2run.pulp3_remote_href) + pulp3_remote_file_2run = self.file_remote_api.read( + pulp2repo_file_2run.pulp3_remote_href + ) pulp3_remote_filemany_2run = self.file_remote_api.read( pulp2repo_filemany_2run.pulp3_remote_href ) @@ -128,8 +164,12 @@ def test_importer_swap(self): self.assertEqual(pulp3_remote_file_1run, pulp3_remote_filemany_2run) self.assertEqual(pulp3_remote_filemany_1run, pulp3_remote_file_2run) # both Pulp2Repository and Pulp 3 "Repository" should still match - self.assertEqual(pulp3_repo_file_2run.remote, pulp2repo_file_2run.pulp3_remote_href) - self.assertEqual(pulp3_repo_filemany_2run.remote, pulp2repo_filemany_2run.pulp3_remote_href) + self.assertEqual( + pulp3_repo_file_2run.remote, pulp2repo_file_2run.pulp3_remote_href + ) + self.assertEqual( + pulp3_repo_filemany_2run.remote, pulp2repo_filemany_2run.pulp3_remote_href + ) def test_distributor_swap(self): """ @@ -140,9 +180,12 @@ def test_distributor_swap(self): """ # run for the first time with the standard plan self.run_migration(FILE_COMPLEX_PLAN) - pulp2repo_file_1run = self.pulp2repositories_api.list(pulp2_repo_id='file').results[0] + pulp2repo_file_1run = self.pulp2repositories_api.list( + pulp2_repo_id="file" + ).results[0] pulp2repo_filemany_1run = self.pulp2repositories_api.list( - pulp2_repo_id='file-many').results[0] + pulp2_repo_id="file-many" + ).results[0] pulp3_pub_file_1run = self.file_publication_api.read( pulp2repo_file_1run.pulp3_publication_href ) @@ -158,9 +201,12 @@ def test_distributor_swap(self): # run a plan with swapped distributors self.run_migration(FILE_DISTRIBUTOR_DIFF_PLAN) - pulp2repo_file_2run = self.pulp2repositories_api.list(pulp2_repo_id='file').results[0] + pulp2repo_file_2run = self.pulp2repositories_api.list( + pulp2_repo_id="file" + ).results[0] pulp2repo_filemany_2run = self.pulp2repositories_api.list( - pulp2_repo_id='file-many').results[0] + pulp2_repo_id="file-many" + ).results[0] pulp3_pub_file_2run = self.file_publication_api.read( pulp2repo_file_2run.pulp3_publication_href ) @@ -174,10 +220,14 @@ def test_distributor_swap(self): pulp2repo_filemany_2run.pulp3_distribution_hrefs[0] ) - self.assertEqual(pulp3_dist_file_1run.base_path, 'file') - self.assertEqual(pulp3_dist_filemany_1run.base_path, 'file-many') - self.assertEqual(pulp3_dist_file_1run.base_path, pulp3_dist_filemany_2run.base_path) - self.assertEqual(pulp3_dist_filemany_1run.base_path, pulp3_dist_file_2run.base_path) + self.assertEqual(pulp3_dist_file_1run.base_path, "file") + self.assertEqual(pulp3_dist_filemany_1run.base_path, "file-many") + self.assertEqual( + pulp3_dist_file_1run.base_path, pulp3_dist_filemany_2run.base_path + ) + self.assertEqual( + pulp3_dist_filemany_1run.base_path, pulp3_dist_file_2run.base_path + ) # No publications should be re-used self.assertNotEqual(pulp3_pub_file_1run, pulp3_pub_file_2run) @@ -195,8 +245,12 @@ def test_distributor_move(self): """Test when a distributor moved from one repo to another in the migration plan.""" # run for the first time with the plan for 2 distributors in one repo self.run_migration(FILE_2DISTRIBUTORS_PLAN) - pulp2repo_file = self.pulp2repositories_api.list(pulp2_repo_id='file').results[0] - pulp2repo_file2 = self.pulp2repositories_api.list(pulp2_repo_id='file2').results[0] + pulp2repo_file = self.pulp2repositories_api.list(pulp2_repo_id="file").results[ + 0 + ] + pulp2repo_file2 = self.pulp2repositories_api.list( + pulp2_repo_id="file2" + ).results[0] pulp3_file_dist_base_paths_1run = [] pulp3_file2_dist_base_paths_1run = [] @@ -209,8 +263,12 @@ def test_distributor_move(self): # run a plan with one distributor moved to another repo in the plan self.run_migration(FILE_2DISTRIBUTORS_MOVED_PLAN) - pulp2repo_file = self.pulp2repositories_api.list(pulp2_repo_id='file').results[0] - pulp2repo_file2 = self.pulp2repositories_api.list(pulp2_repo_id='file2').results[0] + pulp2repo_file = self.pulp2repositories_api.list(pulp2_repo_id="file").results[ + 0 + ] + pulp2repo_file2 = self.pulp2repositories_api.list( + pulp2_repo_id="file2" + ).results[0] pulp3_file_dist_base_paths_2run = [] pulp3_file2_dist_base_paths_2run = [] @@ -226,9 +284,9 @@ def test_distributor_move(self): self.assertEqual(len(pulp3_file_dist_base_paths_2run), 1) self.assertEqual(len(pulp3_file2_dist_base_paths_2run), 2) - self.assertIn('file', pulp3_file_dist_base_paths_1run) - self.assertIn('file-many', pulp3_file_dist_base_paths_1run) - self.assertIn('file2', pulp3_file2_dist_base_paths_1run) - self.assertIn('file', pulp3_file_dist_base_paths_2run) - self.assertIn('file2', pulp3_file2_dist_base_paths_2run) - self.assertIn('file-many', pulp3_file2_dist_base_paths_2run) + self.assertIn("file", pulp3_file_dist_base_paths_1run) + self.assertIn("file-many", pulp3_file_dist_base_paths_1run) + self.assertIn("file2", pulp3_file2_dist_base_paths_1run) + self.assertIn("file", pulp3_file_dist_base_paths_2run) + self.assertIn("file2", pulp3_file2_dist_base_paths_2run) + self.assertIn("file-many", pulp3_file2_dist_base_paths_2run) diff --git a/pulp_2to3_migration/tests/functional/test_multiple_plugins.py b/pulp_2to3_migration/tests/functional/test_multiple_plugins.py index d503eb04..15cf11a1 100644 --- a/pulp_2to3_migration/tests/functional/test_multiple_plugins.py +++ b/pulp_2to3_migration/tests/functional/test_multiple_plugins.py @@ -35,7 +35,10 @@ Pulp2RepositoriesApi, ) -from pulp_2to3_migration.tests.functional.util import get_psql_smash_cmd, set_pulp2_snapshot +from pulp_2to3_migration.tests.functional.util import ( + get_psql_smash_cmd, + set_pulp2_snapshot, +) from pulp_smash import cli from pulp_smash import config as smash_config @@ -49,6 +52,7 @@ class BaseTestMultiplePlugins: """ Test migration of multiple plugins. """ + smash_cfg = smash_config.get_config() smash_cli_client = cli.Client(smash_cfg) @@ -78,15 +82,15 @@ def setUpClass(cls): cls.rpm_distribution_api = DistributionsRpmApi(rpm_client) cls.rpm_publication_api = PublicationsRpmApi(rpm_client) cls.rpm_content_apis = { - 'advisory': ContentAdvisoriesApi(rpm_client), - 'disttree': ContentDistributionTreesApi(rpm_client), - 'modulemd': ContentModulemdsApi(rpm_client), - 'modulemd-defaults': ContentModulemdDefaultsApi(rpm_client), - 'category': ContentPackagecategoriesApi(rpm_client), - 'environment': ContentPackageenvironmentsApi(rpm_client), - 'group': ContentPackagegroupsApi(rpm_client), - 'langpack': ContentPackagelangpacksApi(rpm_client), - 'package': ContentPackagesApi(rpm_client), + "advisory": ContentAdvisoriesApi(rpm_client), + "disttree": ContentDistributionTreesApi(rpm_client), + "modulemd": ContentModulemdsApi(rpm_client), + "modulemd-defaults": ContentModulemdDefaultsApi(rpm_client), + "category": ContentPackagecategoriesApi(rpm_client), + "environment": ContentPackageenvironmentsApi(rpm_client), + "group": ContentPackagegroupsApi(rpm_client), + "langpack": ContentPackagelangpacksApi(rpm_client), + "package": ContentPackagesApi(rpm_client), } # Create api clients for Migration @@ -107,7 +111,7 @@ def run_migration(cls, plan, run_params={}): task(pulpcore.app.models.Task): a migration task created for this plan """ - mp = cls.migration_plans_api.create({'plan': plan}) + mp = cls.migration_plans_api.create({"plan": plan}) mp_run_response = cls.migration_plans_api.run(mp.pulp_href, run_params) task = monitor_task(mp_run_response.task) monitor_task_group(task.task_group) @@ -127,12 +131,13 @@ class TestRpmIsoMigration(BaseTestMultiplePlugins, unittest.TestCase): """ Test RPM and ISO migration """ + def test_rpm_iso_migration_sequential(self): """Test migrating RPM plugin and ISO plugin in two separate runs""" - set_pulp2_snapshot(name='rpm_base_4repos') + set_pulp2_snapshot(name="rpm_base_4repos") self.run_migration(RPM_SIMPLE_PLAN) - set_pulp2_snapshot(name='file_base_4repos') + set_pulp2_snapshot(name="file_base_4repos") self.run_migration(FILE_SIMPLE_PLAN) rpm_repo_count = 3 diff --git a/pulp_2to3_migration/tests/functional/test_rpm_repo.py b/pulp_2to3_migration/tests/functional/test_rpm_repo.py index d9266c1a..1fc51fc6 100644 --- a/pulp_2to3_migration/tests/functional/test_rpm_repo.py +++ b/pulp_2to3_migration/tests/functional/test_rpm_repo.py @@ -8,65 +8,68 @@ from .constants import FIXTURES_BASE_URL from .rpm_base import BaseTestRpm, RepoInfo -RPM_KICKSTART_NO_IMPORTER_PLAN = json.dumps({ - "plugins": [{ - "type": "rpm", - "repositories": [ +RPM_KICKSTART_NO_IMPORTER_PLAN = json.dumps( + { + "plugins": [ { - "name": "c8base", - "repository_versions": [ + "type": "rpm", + "repositories": [ { - "pulp2_repository_id": "c8base", - "pulp2_distributor_repository_ids": ["c8base"] - }, - ] + "name": "c8base", + "repository_versions": [ + { + "pulp2_repository_id": "c8base", + "pulp2_distributor_repository_ids": ["c8base"], + }, + ], + } + ], } ] - }] -}) + } +) PULP_2_RPM_DATA = { - 'remotes': 3, - 'content_initial': { - 'rpm-empty': {}, - 'rpm-empty-for-copy': {}, - 'rpm-with-modules': { - 'advisory': 6, - 'modulemd': 10, - 'modulemd-defaults': 3, - 'category': 1, - 'group': 2, - 'langpack': 1, - 'package': 35, + "remotes": 3, + "content_initial": { + "rpm-empty": {}, + "rpm-empty-for-copy": {}, + "rpm-with-modules": { + "advisory": 6, + "modulemd": 10, + "modulemd-defaults": 3, + "category": 1, + "group": 2, + "langpack": 1, + "package": 35, }, - 'rpm-distribution-tree': { - 'disttree': 1, - 'environment': 1, - 'category': 1, - 'group': 1, - 'langpack': 1, - 'package': 1, + "rpm-distribution-tree": { + "disttree": 1, + "environment": 1, + "category": 1, + "group": 1, + "langpack": 1, + "package": 1, }, - 'srpm-unsigned': { - 'advisory': 2, - 'category': 1, - 'group': 2, - 'langpack': 1, - 'package': 3, + "srpm-unsigned": { + "advisory": 2, + "category": 1, + "group": 2, + "langpack": 1, + "package": 3, }, }, - 'content_total': { - 'package': 38, - 'advisory': 8, - 'modulemd': 10, - 'modulemd-defaults': 3, - 'disttree': 1, - 'environment': 1, - 'category': 3, - 'group': 5, - 'langpack': 3, + "content_total": { + "package": 38, + "advisory": 8, + "modulemd": 10, + "modulemd-defaults": 3, + "disttree": 1, + "environment": 1, + "category": 3, + "group": 5, + "langpack": 3, }, - } @@ -74,6 +77,7 @@ class BaseTestRpmRepo(BaseTestRpm): """ Test RPM repo, importer and distributor migration. """ + @classmethod def setUpClass(cls): """ @@ -81,7 +85,7 @@ def setUpClass(cls): """ super().setUpClass() - set_pulp2_snapshot(name='rpm_base_4repos') + set_pulp2_snapshot(name="rpm_base_4repos") cls.run_migration(cls.plan_initial) def test_rpm_repo_migration(self): @@ -91,12 +95,16 @@ def test_rpm_repo_migration(self): Check that names are migrated correctly and that the number of versions and content count is correct. """ - self.assertEqual(self.rpm_repo_api.list().count, len(self.repo_info.repositories)) + self.assertEqual( + self.rpm_repo_api.list().count, len(self.repo_info.repositories) + ) # content count in total for content_type, api in self.rpm_content_apis.items(): with self.subTest(content_type=content_type): - self.assertEqual(api.list().count, self.repo_info.content_total[content_type]) + self.assertEqual( + api.list().count, self.repo_info.content_total[content_type] + ) for repo in self.rpm_repo_api.list().results: with self.subTest(repo=repo): @@ -107,10 +115,12 @@ def test_rpm_repo_migration(self): # content count per repo for content_type, api in self.rpm_content_apis.items(): with self.subTest(content_type=content_type): - repo_content = api.list(repository_version=repo.latest_version_href) + repo_content = api.list( + repository_version=repo.latest_version_href + ) self.assertEqual( repo_content.count, - self.repo_info.repositories[repo.name].get(content_type, 0) + self.repo_info.repositories[repo.name].get(content_type, 0), ) def test_rpm_importer_migration(self): @@ -120,20 +130,24 @@ def test_rpm_importer_migration(self): self.assertEqual(self.rpm_remote_api.list().count, self.repo_info.remotes) for remote in self.rpm_remote_api.list().results: with self.subTest(remote=remote): - repo_name = '-'.join(remote.name.split('-')[1:]) - repo_url = os.path.join(FIXTURES_BASE_URL, repo_name) + '/' + repo_name = "-".join(remote.name.split("-")[1:]) + repo_url = os.path.join(FIXTURES_BASE_URL, repo_name) + "/" self.assertEqual(remote.url, repo_url) - self.assertEqual(remote.policy, 'on_demand') + self.assertEqual(remote.policy, "on_demand") def test_rpm_distributor_migration(self): """ Test that RPM distributors are correctly migrated. """ - self.assertEqual(self.rpm_publication_api.list().count, self.repo_info.publications) - self.assertEqual(self.rpm_distribution_api.list().count, self.repo_info.distributions) + self.assertEqual( + self.rpm_publication_api.list().count, self.repo_info.publications + ) + self.assertEqual( + self.rpm_distribution_api.list().count, self.repo_info.distributions + ) for dist in self.rpm_distribution_api.list().results: with self.subTest(dist=dist): - base_path = '-'.join(dist.name.split('-')[1:]) + base_path = "-".join(dist.name.split("-")[1:]) self.assertEqual(dist.base_path, base_path) @@ -141,6 +155,7 @@ class TestRpmRepoMigrationSimplePlan(BaseTestRpmRepo, unittest.TestCase): """ Test RPM repo migration using simple migration plan. """ + plan_initial = RPM_SIMPLE_PLAN repo_info = RepoInfo(PULP_2_RPM_DATA, is_simple=True) @@ -149,16 +164,20 @@ class TestRpmRepoMigrationComplexPlan(BaseTestRpmRepo, unittest.TestCase): """ Test RPM repo migration using complex migration plan. """ + plan_initial = RPM_COMPLEX_PLAN repo_info = RepoInfo(PULP_2_RPM_DATA) -@unittest.skip('The image files of a kickstart repo are too large for github, need to find ' - 'smaller ones.') +@unittest.skip( + "The image files of a kickstart repo are too large for github, need to find " + "smaller ones." +) class TestRpmKickstartImmediateNoImporterPlan(unittest.TestCase): """ Test RPM repo migration when a kickstart tree is downloaded but its importer is not migrated. """ + @classmethod def setUpClass(cls): """ @@ -166,7 +185,7 @@ def setUpClass(cls): """ super().setUpClass() - set_pulp2_snapshot(name='rpm_kickstart_immediate_no_rpm') + set_pulp2_snapshot(name="rpm_kickstart_immediate_no_rpm") def test_rpm_repo_migration(self): """ diff --git a/pulp_2to3_migration/tests/functional/test_rpm_rerun.py b/pulp_2to3_migration/tests/functional/test_rpm_rerun.py index f615ab38..ad03dcc5 100644 --- a/pulp_2to3_migration/tests/functional/test_rpm_rerun.py +++ b/pulp_2to3_migration/tests/functional/test_rpm_rerun.py @@ -4,7 +4,10 @@ from collections import defaultdict -from pulp_2to3_migration.tests.functional.util import get_psql_smash_cmd, set_pulp2_snapshot +from pulp_2to3_migration.tests.functional.util import ( + get_psql_smash_cmd, + set_pulp2_snapshot, +) from .common_plans import INITIAL_REPOSITORIES, RPM_SIMPLE_PLAN, RPM_COMPLEX_PLAN from .constants import TRUNCATE_TABLES_QUERY_BASH @@ -18,156 +21,166 @@ "repository_versions": [ { "pulp2_repository_id": "rpm-richnweak-deps", - "pulp2_distributor_repository_ids": ["rpm-richnweak-deps"] + "pulp2_distributor_repository_ids": ["rpm-richnweak-deps"], } - ] + ], } ] -RPM_RERUN_PLAN = json.dumps({ - "plugins": [{ - "type": "rpm", - "repositories": RERUN_REPOSITORIES - }] -}) - -COPY_INTO_SAME_REPO_PLAN = json.dumps({ - "plugins": [{ - "type": "rpm", - "repositories": [ +RPM_RERUN_PLAN = json.dumps( + {"plugins": [{"type": "rpm", "repositories": RERUN_REPOSITORIES}]} +) + +COPY_INTO_SAME_REPO_PLAN = json.dumps( + { + "plugins": [ { - "name": "rpm-with-modules", - "pulp2_importer_repository_id": "rpm-with-modules", - "repository_versions": [ - { - "pulp2_repository_id": "rpm-with-modules", - "pulp2_distributor_repository_ids": ["rpm-with-modules"] - }, + "type": "rpm", + "repositories": [ { - "pulp2_repository_id": "rpm-with-modules-copy", - "pulp2_distributor_repository_ids": ["rpm-with-modules-copy"] - }, - ] + "name": "rpm-with-modules", + "pulp2_importer_repository_id": "rpm-with-modules", + "repository_versions": [ + { + "pulp2_repository_id": "rpm-with-modules", + "pulp2_distributor_repository_ids": [ + "rpm-with-modules" + ], + }, + { + "pulp2_repository_id": "rpm-with-modules-copy", + "pulp2_distributor_repository_ids": [ + "rpm-with-modules-copy" + ], + }, + ], + } + ], } ] - }] -}) + } +) -COPY_INTO_NEW_REPO_PLAN = json.dumps({ - "plugins": [{ - "type": "rpm", - "repositories": [ +COPY_INTO_NEW_REPO_PLAN = json.dumps( + { + "plugins": [ { - "name": "rpm-with-modules", - "pulp2_importer_repository_id": "rpm-with-modules", - "repository_versions": [ + "type": "rpm", + "repositories": [ { - "pulp2_repository_id": "rpm-with-modules", - "pulp2_distributor_repository_ids": ["rpm-with-modules"] - } - ] - }, - { - "name": "rpm-with-modules-copy", - "pulp2_importer_repository_id": "rpm-with-modules-copy", - "repository_versions": [ + "name": "rpm-with-modules", + "pulp2_importer_repository_id": "rpm-with-modules", + "repository_versions": [ + { + "pulp2_repository_id": "rpm-with-modules", + "pulp2_distributor_repository_ids": [ + "rpm-with-modules" + ], + } + ], + }, { - "pulp2_repository_id": "rpm-with-modules-copy", - "pulp2_distributor_repository_ids": ["rpm-with-modules-copy"] - } - ] + "name": "rpm-with-modules-copy", + "pulp2_importer_repository_id": "rpm-with-modules-copy", + "repository_versions": [ + { + "pulp2_repository_id": "rpm-with-modules-copy", + "pulp2_distributor_repository_ids": [ + "rpm-with-modules-copy" + ], + } + ], + }, + ], } ] - }] -}) + } +) PULP_2_RPM_DATA = { - 'remotes': 4, - 'content_initial': { - 'rpm-empty': {}, - 'rpm-empty-for-copy': {}, - 'rpm-with-modules': { - 'advisory': 6, - 'modulemd': 10, - 'modulemd-defaults': 3, - 'category': 1, - 'group': 2, - 'langpack': 1, - 'package': 34, + "remotes": 4, + "content_initial": { + "rpm-empty": {}, + "rpm-empty-for-copy": {}, + "rpm-with-modules": { + "advisory": 6, + "modulemd": 10, + "modulemd-defaults": 3, + "category": 1, + "group": 2, + "langpack": 1, + "package": 34, }, - 'rpm-distribution-tree': { - 'disttree': 1, - 'environment': 1, - 'category': 1, - 'group': 1, - 'langpack': 1, - 'package': 1, + "rpm-distribution-tree": { + "disttree": 1, + "environment": 1, + "category": 1, + "group": 1, + "langpack": 1, + "package": 1, }, - 'srpm-unsigned': { - 'advisory': 2, - 'category': 1, - 'group': 2, - 'langpack': 1, - 'package': 3, + "srpm-unsigned": { + "advisory": 2, + "category": 1, + "group": 2, + "langpack": 1, + "package": 3, }, }, - 'content_rerun': { - 'rpm-empty': {}, - 'rpm-empty-for-copy': { - 'package': 1 - }, - 'rpm-with-modules': { - 'advisory': 6, - 'modulemd': 10, - 'modulemd-defaults': 3, - 'category': 1, - 'group': 2, - 'langpack': 1, - 'package': 34, + "content_rerun": { + "rpm-empty": {}, + "rpm-empty-for-copy": {"package": 1}, + "rpm-with-modules": { + "advisory": 6, + "modulemd": 10, + "modulemd-defaults": 3, + "category": 1, + "group": 2, + "langpack": 1, + "package": 34, }, - 'rpm-distribution-tree': { - 'disttree': 1, - 'environment': 1, - 'category': 1, - 'group': 1, - 'langpack': 1, - 'package': 1, + "rpm-distribution-tree": { + "disttree": 1, + "environment": 1, + "category": 1, + "group": 1, + "langpack": 1, + "package": 1, }, - 'srpm-unsigned': { - 'advisory': 2, - 'category': 1, - 'group': 2, - 'langpack': 1, - 'package': 3, + "srpm-unsigned": { + "advisory": 2, + "category": 1, + "group": 2, + "langpack": 1, + "package": 3, }, - 'rpm-richnweak-deps': { - 'package': 14, + "rpm-richnweak-deps": { + "package": 14, }, }, - 'content_total': { - 'package': 53, - 'advisory': 8, - 'modulemd': 10, - 'modulemd-defaults': 3, - 'disttree': 1, - 'environment': 1, - 'category': 3, - 'group': 5, - 'langpack': 3, + "content_total": { + "package": 53, + "advisory": 8, + "modulemd": 10, + "modulemd-defaults": 3, + "disttree": 1, + "environment": 1, + "category": 3, + "group": 5, + "langpack": 3, }, - 'content_added': 15, - 'versions': { - 'rpm-empty': 1, - 'rpm-empty-for-copy': 2, - 'rpm-with-modules': 3, - 'rpm-distribution-tree': 2, - 'srpm-unsigned': 2, - 'rpm-richnweak-deps': 2 - + "content_added": 15, + "versions": { + "rpm-empty": 1, + "rpm-empty-for-copy": 2, + "rpm-with-modules": 3, + "rpm-distribution-tree": 2, + "srpm-unsigned": 2, + "rpm-richnweak-deps": 2, }, - 'new_remotes': 2, - 'new_publications': 4, - 'new_distributions': 5, + "new_remotes": 2, + "new_publications": 4, + "new_distributions": 5, } @@ -175,6 +188,7 @@ class BaseTestRpmRerun(BaseTestRpm): """ Test RPM migration re-runs. """ + @classmethod def setUpClass(cls): """ @@ -185,43 +199,39 @@ def setUpClass(cls): def collect_migration_data(): """Collect data from a migration run to use later for comparison.""" data = { - 'repos': defaultdict(dict), - 'remotes': defaultdict(dict), - 'publications': defaultdict(dict), - 'distributions': defaultdict(dict), - 'pulp2content': defaultdict(dict), + "repos": defaultdict(dict), + "remotes": defaultdict(dict), + "publications": defaultdict(dict), + "distributions": defaultdict(dict), + "pulp2content": defaultdict(dict), } for repo in cls.rpm_repo_api.list().results: - latest_version = cls.rpm_repo_versions_api.read(repo.latest_version_href) - data['repos'][repo.name] = { - 'created': repo.pulp_created, - 'latest_version_number': latest_version.number, - 'latest_version_created': latest_version.pulp_created, + latest_version = cls.rpm_repo_versions_api.read( + repo.latest_version_href + ) + data["repos"][repo.name] = { + "created": repo.pulp_created, + "latest_version_number": latest_version.number, + "latest_version_created": latest_version.pulp_created, } for remote in cls.rpm_remote_api.list().results: - data['remotes'][remote.name] = { - 'created': remote.pulp_created - } + data["remotes"][remote.name] = {"created": remote.pulp_created} for pub in cls.rpm_publication_api.list().results: - data['publications'][pub.repository] = { - 'created': pub.pulp_created - } + data["publications"][pub.repository] = {"created": pub.pulp_created} for dist in cls.rpm_distribution_api.list().results: - data['distributions'][dist.name] = { - 'created': dist.pulp_created - } + data["distributions"][dist.name] = {"created": dist.pulp_created} for pulp2content in cls.pulp2content_api.list().results: - data['pulp2content'][pulp2content.pulp2_id] = { - 'pulp3content': pulp2content.pulp3_content + data["pulp2content"][pulp2content.pulp2_id] = { + "pulp3content": pulp2content.pulp3_content } return data - set_pulp2_snapshot(name='rpm_base_4repos') + set_pulp2_snapshot(name="rpm_base_4repos") cls.task_initial = cls.run_migration(cls.plan_initial) cls.first_migration_data = collect_migration_data() - set_pulp2_snapshot(name='rpm_base_4repos_rerun_changes') + set_pulp2_snapshot(name="rpm_base_4repos_rerun_changes") cls.task_rerun = cls.run_migration(cls.plan_rerun) def test_rpm_only_added_content(self): @@ -232,15 +242,17 @@ def test_rpm_only_added_content(self): """ content_added = 0 for pr in self.task_rerun.progress_reports: - if pr.code == 'migrating.content': + if pr.code == "migrating.content": content_added = pr.done break - self.assertEqual(content_added, self.repo_info['content_added']) + self.assertEqual(content_added, self.repo_info["content_added"]) # content count in total for content_type, api in self.rpm_content_apis.items(): with self.subTest(content_type=content_type): - self.assertEqual(api.list().count, self.repo_info.content_total[content_type]) + self.assertEqual( + api.list().count, self.repo_info.content_total[content_type] + ) def test_rpm_only_added_or_changed_repos(self): """ @@ -248,30 +260,38 @@ def test_rpm_only_added_or_changed_repos(self): Compare timestamps from initial run. And make sure repos are migrated correctly. """ - self.assertEqual(self.rpm_repo_api.list().count, len(self.repo_info.repositories)) + self.assertEqual( + self.rpm_repo_api.list().count, len(self.repo_info.repositories) + ) new_repo_count = 0 for repo in self.rpm_repo_api.list().results: with self.subTest(repo=repo): - if repo.name in self.first_migration_data['repos']: - repo_data = self.first_migration_data['repos'][repo.name] + if repo.name in self.first_migration_data["repos"]: + repo_data = self.first_migration_data["repos"][repo.name] repo_version = self.rpm_repo_versions_api.list( - repo.pulp_href, number=repo_data['latest_version_number'] + repo.pulp_href, number=repo_data["latest_version_number"] ).results[0] - self.assertEqual(repo.pulp_created, repo_data['created']) - self.assertEqual(repo_version.pulp_created, repo_data['latest_version_created']) + self.assertEqual(repo.pulp_created, repo_data["created"]) + self.assertEqual( + repo_version.pulp_created, repo_data["latest_version_created"] + ) else: new_repo_count += 1 repo_versions = self.rpm_repo_versions_api.list(repo.pulp_href) - self.assertEqual(repo_versions.count, self.repo_info['versions'][repo.name]) + self.assertEqual( + repo_versions.count, self.repo_info["versions"][repo.name] + ) # content count per repo for content_type, api in self.rpm_content_apis.items(): with self.subTest(content_type=content_type): - repo_content = api.list(repository_version=repo.latest_version_href) + repo_content = api.list( + repository_version=repo.latest_version_href + ) self.assertEqual( repo_content.count, - self.repo_info.repositories[repo.name].get(content_type, 0) + self.repo_info.repositories[repo.name].get(content_type, 0), ) self.assertEqual(new_repo_count, len(self.repo_info.new_repositories)) @@ -281,23 +301,23 @@ def test_rpm_importers_only_added_or_with_changed_config(self): The only changed feed is for the 'rpm-with-modules' repo. """ - REPO_FEED_CHANGE = 'rpm-with-modules' - self.assertEqual(self.rpm_remote_api.list().count, self.repo_info['remotes']) + REPO_FEED_CHANGE = "rpm-with-modules" + self.assertEqual(self.rpm_remote_api.list().count, self.repo_info["remotes"]) new_remote_count = 0 for remote in self.rpm_remote_api.list().results: with self.subTest(remote=remote): - if remote.name in self.first_migration_data['remotes']: - remote_data = self.first_migration_data['remotes'][remote.name] - repo_name = '-'.join(remote.name.split('-')[1:]) + if remote.name in self.first_migration_data["remotes"]: + remote_data = self.first_migration_data["remotes"][remote.name] + repo_name = "-".join(remote.name.split("-")[1:]) if repo_name == REPO_FEED_CHANGE: - self.assertNotEqual(remote.pulp_created, remote_data['created']) + self.assertNotEqual(remote.pulp_created, remote_data["created"]) new_remote_count += 1 else: - self.assertEqual(remote.pulp_created, remote_data['created']) + self.assertEqual(remote.pulp_created, remote_data["created"]) else: new_remote_count += 1 - self.assertEqual(new_remote_count, self.repo_info['new_remotes']) + self.assertEqual(new_remote_count, self.repo_info["new_remotes"]) def test_rpm_distributors_only_added_or_with_changed_config(self): """ @@ -307,47 +327,58 @@ def test_rpm_distributors_only_added_or_with_changed_config(self): A change in checksum type forces to recreate publications and distributors. A change in relative_url/base_path forces to recreate distributors. """ - REPO_NEW_CONTENT = 'rpm-empty-for-copy' - REPO_REMOVED_CONTENT = 'rpm-with-modules' - REPO_CHECKSUMTYPE_CHANGE = 'rpm-distribution-tree' - REPO_BASE_PATH_CHANGE = 'srpm-unsigned' - CHANGED_PUB_REPOS = (REPO_NEW_CONTENT, REPO_REMOVED_CONTENT, REPO_CHECKSUMTYPE_CHANGE) + REPO_NEW_CONTENT = "rpm-empty-for-copy" + REPO_REMOVED_CONTENT = "rpm-with-modules" + REPO_CHECKSUMTYPE_CHANGE = "rpm-distribution-tree" + REPO_BASE_PATH_CHANGE = "srpm-unsigned" + CHANGED_PUB_REPOS = ( + REPO_NEW_CONTENT, + REPO_REMOVED_CONTENT, + REPO_CHECKSUMTYPE_CHANGE, + ) CHANGED_DIST_REPOS = ( - REPO_NEW_CONTENT, REPO_REMOVED_CONTENT, REPO_CHECKSUMTYPE_CHANGE, REPO_BASE_PATH_CHANGE + REPO_NEW_CONTENT, + REPO_REMOVED_CONTENT, + REPO_CHECKSUMTYPE_CHANGE, + REPO_BASE_PATH_CHANGE, + ) + self.assertEqual( + self.rpm_publication_api.list().count, self.repo_info.publications + ) + self.assertEqual( + self.rpm_distribution_api.list().count, self.repo_info.distributions ) - self.assertEqual(self.rpm_publication_api.list().count, self.repo_info.publications) - self.assertEqual(self.rpm_distribution_api.list().count, self.repo_info.distributions) new_publication_count = 0 for pub in self.rpm_publication_api.list().results: with self.subTest(pub=pub): - if pub.repository in self.first_migration_data['publications']: - pub_data = self.first_migration_data['publications'][pub.repository] + if pub.repository in self.first_migration_data["publications"]: + pub_data = self.first_migration_data["publications"][pub.repository] repo_name = self.rpm_repo_api.read(pub.repository).name if repo_name in CHANGED_PUB_REPOS: - self.assertNotEqual(pub.pulp_created, pub_data['created']) + self.assertNotEqual(pub.pulp_created, pub_data["created"]) new_publication_count += 1 else: - self.assertEqual(pub.pulp_created, pub_data['created']) + self.assertEqual(pub.pulp_created, pub_data["created"]) else: new_publication_count += 1 - self.assertEqual(new_publication_count, self.repo_info['new_publications']) + self.assertEqual(new_publication_count, self.repo_info["new_publications"]) new_distribution_count = 0 for dist in self.rpm_distribution_api.list().results: with self.subTest(dist=dist): - if dist.name in self.first_migration_data['distributions']: - dist_data = self.first_migration_data['distributions'][dist.name] - repo_name = '-'.join(dist.name.split('-')[1:]) + if dist.name in self.first_migration_data["distributions"]: + dist_data = self.first_migration_data["distributions"][dist.name] + repo_name = "-".join(dist.name.split("-")[1:]) if repo_name in CHANGED_DIST_REPOS: - self.assertNotEqual(dist.pulp_created, dist_data['created']) + self.assertNotEqual(dist.pulp_created, dist_data["created"]) new_distribution_count += 1 else: - self.assertEqual(dist.pulp_created, dist_data['created']) + self.assertEqual(dist.pulp_created, dist_data["created"]) else: new_distribution_count += 1 - self.assertEqual(new_distribution_count, self.repo_info['new_distributions']) + self.assertEqual(new_distribution_count, self.repo_info["new_distributions"]) def test_old_premigrated_content_removed(self): """ @@ -357,9 +388,9 @@ def test_old_premigrated_content_removed(self): cleanup is run. The Pulp2Content record for this package should be removed, package itself in Pulp 3 should stay. """ - removed_package = self.rpm_content_apis['package'].list(name='bear').results[0] - for pulp2_id, pulp2content in self.first_migration_data['pulp2content'].items(): - if pulp2content['pulp3content'] == removed_package.pulp_href: + removed_package = self.rpm_content_apis["package"].list(name="bear").results[0] + for pulp2_id, pulp2content in self.first_migration_data["pulp2content"].items(): + if pulp2content["pulp3content"] == removed_package.pulp_href: break self.assertEqual(self.pulp2content_api.list(pulp2_id=pulp2_id).count, 0) @@ -369,6 +400,7 @@ class TestRpmRerunSimplePlan(BaseTestRpmRerun, unittest.TestCase): """ Test RPM repo migration using simple migration plan. """ + plan_initial = RPM_SIMPLE_PLAN plan_rerun = RPM_SIMPLE_PLAN repo_info = RepoInfo(PULP_2_RPM_DATA, is_simple=True) @@ -378,6 +410,7 @@ class TestRpmRerunComplexPlan(BaseTestRpmRerun, unittest.TestCase): """ Test RPM repo migration using complex migration plan. """ + plan_initial = RPM_COMPLEX_PLAN plan_rerun = RPM_RERUN_PLAN repo_info = RepoInfo(PULP_2_RPM_DATA) @@ -387,6 +420,7 @@ class TestRpmRerunCopy(BaseTestRpm, unittest.TestCase): """ Test RPM repo migration when content is copied in Pulp 2 between the runs. """ + plan_initial = RPM_COMPLEX_PLAN def tearDown(self): @@ -403,17 +437,23 @@ def test_migrate_copy_into_same_repo(self): All advisories should be in place. """ - set_pulp2_snapshot(name='rpm_base_4repos') + set_pulp2_snapshot(name="rpm_base_4repos") self.task_initial = self.run_migration(self.plan_initial) - set_pulp2_snapshot(name='rpm_base_4repos_rerun_copy') + set_pulp2_snapshot(name="rpm_base_4repos_rerun_copy") self.task_rerun = self.run_migration(COPY_INTO_SAME_REPO_PLAN) - pulp2repo = self.pulp2repositories_api.list(pulp2_repo_id='rpm-with-modules').results[0] + pulp2repo = self.pulp2repositories_api.list( + pulp2_repo_id="rpm-with-modules" + ).results[0] version_href = pulp2repo.pulp3_repository_version - repo_advisories = self.rpm_content_apis['advisory'].list(repository_version=version_href) + repo_advisories = self.rpm_content_apis["advisory"].list( + repository_version=version_href + ) self.assertEqual(repo_advisories.count, 6) - self.assertEqual(self.rpm_distribution_api.list(base_path='rpm-with-modules-copy').count, 1) + self.assertEqual( + self.rpm_distribution_api.list(base_path="rpm-with-modules-copy").count, 1 + ) def test_migrate_copy_into_new_repo(self): """ @@ -421,17 +461,21 @@ def test_migrate_copy_into_new_repo(self): All advisories should be in place. """ - set_pulp2_snapshot(name='rpm_base_4repos') + set_pulp2_snapshot(name="rpm_base_4repos") self.task_initial = self.run_migration(self.plan_initial) - set_pulp2_snapshot(name='rpm_base_4repos_rerun_copy') + set_pulp2_snapshot(name="rpm_base_4repos_rerun_copy") self.task_rerun = self.run_migration(COPY_INTO_NEW_REPO_PLAN) pulp2repo = self.pulp2repositories_api.list( - pulp2_repo_id='rpm-with-modules-copy' + pulp2_repo_id="rpm-with-modules-copy" ).results[0] version_href = pulp2repo.pulp3_repository_version - repo_advisories = self.rpm_content_apis['advisory'].list(repository_version=version_href) + repo_advisories = self.rpm_content_apis["advisory"].list( + repository_version=version_href + ) self.assertEqual(self.rpm_repo_versions_api.read(version_href).number, 1) self.assertEqual(repo_advisories.count, 6) - self.assertEqual(self.rpm_distribution_api.list(base_path='rpm-with-modules-copy').count, 1) + self.assertEqual( + self.rpm_distribution_api.list(base_path="rpm-with-modules-copy").count, 1 + ) diff --git a/pulp_2to3_migration/tests/functional/util.py b/pulp_2to3_migration/tests/functional/util.py index 9c623fec..fc67a848 100644 --- a/pulp_2to3_migration/tests/functional/util.py +++ b/pulp_2to3_migration/tests/functional/util.py @@ -23,13 +23,15 @@ def get_psql_smash_cmd(sql_statement): tuple: a command in the format for pulp smash cli client """ - host = 'localhost' - user = settings.DATABASES['default']['USER'] - password = settings.DATABASES['default']['PASSWORD'] - dbname = settings.DATABASES['default']['NAME'] + host = "localhost" + user = settings.DATABASES["default"]["USER"] + password = settings.DATABASES["default"]["PASSWORD"] + dbname = settings.DATABASES["default"]["NAME"] return ( - 'psql', '-c', sql_statement, - f'postgresql://{user}:{password}@{host}/{dbname}' + "psql", + "-c", + sql_statement, + f"postgresql://{user}:{password}@{host}/{dbname}", ) @@ -44,16 +46,16 @@ def set_pulp2_snapshot(name): smash_cfg = smash_config.get_config() smash_cli_client = cli.Client(smash_cfg) pulp2_mongodb_conf = utils.get_pulp_setting(smash_cli_client, "PULP2_MONGODB") - mongodb_host = pulp2_mongodb_conf['seeds'].split(':')[0] + mongodb_host = pulp2_mongodb_conf["seeds"].split(":")[0] - pulp2_fs_setup_script_path = '/tmp/set_pulp2.sh' + pulp2_fs_setup_script_path = "/tmp/set_pulp2.sh" # for running tests locally - if smash_cfg.hosts[0].hostname == 'localhost': + if smash_cfg.hosts[0].hostname == "localhost": basepath = os.path.dirname(os.path.realpath(__file__)) - pulp2_fs_setup_script_path = os.path.join(basepath, 'scripts/set_pulp2.sh') + pulp2_fs_setup_script_path = os.path.join(basepath, "scripts/set_pulp2.sh") - cmd = ('bash', pulp2_fs_setup_script_path, mongodb_host, name) + cmd = ("bash", pulp2_fs_setup_script_path, mongodb_host, name) smash_cli_client.run(cmd, sudo=True) # needs to be done locally otherwise auth is required because password is provided in the diff --git a/pyproject.toml b/pyproject.toml index cbc4d46e..89ce1d93 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,11 +22,12 @@ ignore = [ "dev_requirements.txt", "doc_requirements.txt", "docs/**", - "flake8.cfg", "template_config.yml", ".travis/**", ".travis.yml", ".pep8speaks.yml", ".ci/**", ".github/**", + "lint_requirements.txt", + ".flake8", ] diff --git a/setup.py b/setup.py index 32d9d234..bc48753b 100755 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ "iso = pulp_2to3_migration.app.plugin.iso.migrator:IsoMigrator", "docker = pulp_2to3_migration.app.plugin.docker.migrator:DockerMigrator", "deb = pulp_2to3_migration.app.plugin.deb.migrator:DebMigrator", - "rpm = pulp_2to3_migration.app.plugin.rpm.migrator:RpmMigrator" - ] - } + "rpm = pulp_2to3_migration.app.plugin.rpm.migrator:RpmMigrator", + ], + }, )