From 92e81404cf23df7957088033e1a73a708dc5b783 Mon Sep 17 00:00:00 2001 From: Fabricio Aguiar Date: Mon, 5 Sep 2022 11:43:55 +0100 Subject: [PATCH] Update CI files [noissue] --- .ci/scripts/changelog.py | 10 +- .ci/scripts/redmine.py | 4 +- .ci/scripts/update_ci_branches.py | 10 +- .ci/scripts/validate_commit_message.py | 10 +- .github/workflows/scripts/docs-publisher.py | 8 +- .../workflows/scripts/post_before_script.sh | 8 +- .github/workflows/scripts/release.py | 4 +- .../stage-changelog-for-default-branch.py | 4 +- pulp_2to3_migration/app/migration.py | 54 ++---- pulp_2to3_migration/app/models/base.py | 57 ++----- pulp_2to3_migration/app/models/content.py | 4 +- pulp_2to3_migration/app/models/repository.py | 16 +- pulp_2to3_migration/app/plugin/content.py | 46 ++--- .../app/plugin/deb/pulp_2to3_models.py | 24 +-- .../app/plugin/deb/repository.py | 8 +- .../app/plugin/docker/pulp_2to3_models.py | 26 +-- .../app/plugin/docker/repository.py | 4 +- .../app/plugin/iso/pulp_2to3_models.py | 4 +- .../app/plugin/iso/repository.py | 8 +- pulp_2to3_migration/app/plugin/repository.py | 12 +- pulp_2to3_migration/app/plugin/rpm/erratum.py | 4 +- .../app/plugin/rpm/migrator.py | 6 +- .../app/plugin/rpm/pulp_2to3_models.py | 94 +++------- .../app/plugin/rpm/repository.py | 12 +- pulp_2to3_migration/app/pre_migration.py | 160 +++++------------- pulp_2to3_migration/app/serializers.py | 7 +- pulp_2to3_migration/app/tasks/migrate.py | 4 +- pulp_2to3_migration/app/tasks/reset.py | 28 ++- pulp_2to3_migration/app/viewsets.py | 20 +-- pulp_2to3_migration/pulp2/connection.py | 34 +--- .../tests/functional/common_plans.py | 8 +- .../tests/functional/constants.py | 4 +- .../tests/functional/test_deb_repo.py | 16 +- .../tests/functional/test_file_migration.py | 15 +- .../tests/functional/test_file_reset.py | 60 ++----- .../functional/test_migration_behaviour.py | 110 +++--------- .../tests/functional/test_migration_plan.py | 8 +- .../functional/test_migration_plan_changes.py | 72 ++------ .../tests/functional/test_rpm_repo.py | 23 +-- .../tests/functional/test_rpm_rerun.py | 78 +++------ pyproject.toml | 16 ++ 41 files changed, 304 insertions(+), 796 deletions(-) diff --git a/.ci/scripts/changelog.py b/.ci/scripts/changelog.py index 1a5ed89f..179a0f38 100755 --- a/.ci/scripts/changelog.py +++ b/.ci/scripts/changelog.py @@ -5,14 +5,8 @@ from git import Repo repo = Repo(os.getcwd()) -heads = repo.git.ls_remote( - "--heads", "https://github.com/pulp/pulp-2to3-migration.git" -).split("\n") -branches = [ - h.split("/")[-1] - for h in heads - if re.search(r"^([0-9]+)\.([0-9]+)$", h.split("/")[-1]) -] +heads = repo.git.ls_remote("--heads", "https://github.com/pulp/pulp-2to3-migration.git").split("\n") +branches = [h.split("/")[-1] for h in heads if re.search(r"^([0-9]+)\.([0-9]+)$", h.split("/")[-1])] branches.sort(key=lambda ver: Version(ver), reverse=True) diff --git a/.ci/scripts/redmine.py b/.ci/scripts/redmine.py index 9b8a2199..82c9b36c 100755 --- a/.ci/scripts/redmine.py +++ b/.ci/scripts/redmine.py @@ -20,9 +20,7 @@ query_issues = REDMINE_QUERY_URL.split("=")[-1].split(",") milestone_name = redmine.version.get(MILESTONE_URL.split("/")[-1].split(".")[0]).name if milestone_name != RELEASE: - raise RuntimeError( - f"Milestone name, '{milestone_name}', does not match version, '{RELEASE}'." - ) + raise RuntimeError(f"Milestone name, '{milestone_name}', does not match version, '{RELEASE}'.") to_update = [] for issue in query_issues: diff --git a/.ci/scripts/update_ci_branches.py b/.ci/scripts/update_ci_branches.py index 2af7ff0c..c1a1d6fe 100755 --- a/.ci/scripts/update_ci_branches.py +++ b/.ci/scripts/update_ci_branches.py @@ -11,14 +11,8 @@ initial_branch = None repo = Repo(os.getcwd()) -heads = repo.git.ls_remote( - "--heads", "https://github.com/pulp/pulp-2to3-migration.git" -).split("\n") -branches = [ - h.split("/")[-1] - for h in heads - if re.search(r"^([0-9]+)\.([0-9]+)$", h.split("/")[-1]) -] +heads = repo.git.ls_remote("--heads", "https://github.com/pulp/pulp-2to3-migration.git").split("\n") +branches = [h.split("/")[-1] for h in heads if re.search(r"^([0-9]+)\.([0-9]+)$", h.split("/")[-1])] branches.sort(key=lambda ver: Version(ver)) headers = { diff --git a/.ci/scripts/validate_commit_message.py b/.ci/scripts/validate_commit_message.py index c270cff7..543c865f 100755 --- a/.ci/scripts/validate_commit_message.py +++ b/.ci/scripts/validate_commit_message.py @@ -19,9 +19,7 @@ NO_ISSUE = "[noissue]" CHANGELOG_EXTS = [".feature", ".bugfix", ".doc", ".removal", ".misc", ".deprecation"] sha = sys.argv[1] -message = subprocess.check_output(["git", "log", "--format=%B", "-n 1", sha]).decode( - "utf-8" -) +message = subprocess.check_output(["git", "log", "--format=%B", "-n 1", sha]).decode("utf-8") KEYWORDS = ["fixes", "closes"] @@ -68,11 +66,7 @@ def __check_changelog(issue): __check_changelog(issue) else: if NO_ISSUE in message: - print( - "Commit {sha} has no issues but is tagged {tag}.".format( - sha=sha[0:7], tag=NO_ISSUE - ) - ) + print("Commit {sha} has no issues but is tagged {tag}.".format(sha=sha[0:7], tag=NO_ISSUE)) elif "Merge" in message and "cherry picked from commit" in message: pass else: diff --git a/.github/workflows/scripts/docs-publisher.py b/.github/workflows/scripts/docs-publisher.py index 277a4872..47abfb2e 100755 --- a/.github/workflows/scripts/docs-publisher.py +++ b/.github/workflows/scripts/docs-publisher.py @@ -112,9 +112,7 @@ def main(): if exit_code != 0: raise RuntimeError("An error occurred while pushing docs.") elif build_type == "tag": - if (not re.search("[a-zA-Z]", branch) or "post" in branch) and len( - branch.split(".") - ) > 2: + if (not re.search("[a-zA-Z]", branch) or "post" in branch) and len(branch.split(".")) > 2: # Only publish docs at the root if this is the latest version r = requests.get("https://pypi.org/pypi/pulp-2to3-migration/json") latest_version = version.parse(json.loads(r.text)["info"]["version"]) @@ -133,9 +131,7 @@ def main(): # publish to the root of docs.pulpproject.org if publish_at_root: version_components = branch.split(".") - x_y_version = "{}.{}".format( - version_components[0], version_components[1] - ) + x_y_version = "{}.{}".format(version_components[0], version_components[1]) remote_path_arg = "%s@%s:%s" % (USERNAME, HOSTNAME, SITE_ROOT) rsync_command = [ "rsync", diff --git a/.github/workflows/scripts/post_before_script.sh b/.github/workflows/scripts/post_before_script.sh index ac8eb754..7776a642 100755 --- a/.github/workflows/scripts/post_before_script.sh +++ b/.github/workflows/scripts/post_before_script.sh @@ -24,10 +24,10 @@ PULP2_MONGODB = { EOF" # Restarting single container services -cmd_prefix bash -c "s6-svc -r /var/run/s6/services/pulpcore-api" -cmd_prefix bash -c "s6-svc -r /var/run/s6/services/pulpcore-content" -cmd_prefix bash -c "s6-svc -r /var/run/s6/services/pulpcore-worker@1" -cmd_prefix bash -c "s6-svc -r /var/run/s6/services/pulpcore-worker@2" +cmd_prefix bash -c "s6-svc -r /var/run/s6-rc/servicedirs/pulpcore-api" +cmd_prefix bash -c "s6-svc -r /var/run/s6-rc/servicedirs/pulpcore-content" +cmd_prefix bash -c "s6-svc -r /var/run/s6-rc/servicedirs/pulpcore-worker@1" +cmd_prefix bash -c "s6-svc -r /var/run/s6-rc/servicedirs/pulpcore-worker@2" # install mongo and copy a script which we need to use for func tests to roll out a pulp 2 snapshot cmd_prefix bash -c "cat > /etc/yum.repos.d/mongodb-org-3.6.repo <= batch_size or i == total_content - 1 - ) + save_batch = pulp2content and (len(pulp2content) >= batch_size or i == total_content - 1) if save_batch: _logger.debug( - "Bulk save for generic content info, saved so far: {index}".format( - index=i + 1 - ) + "Bulk save for generic content info, saved so far: {index}".format(index=i + 1) ) pulp2content_batch = Pulp2Content.objects.bulk_create( pulp2content, ignore_conflicts=True @@ -301,9 +283,7 @@ def delete_removed_pulp2_content(content_model): pulp2content_pb.done += content_saved pulp2content_pb.save() - content_model.pulp_2to3_detail.pre_migrate_content_detail( - pulp2content_batch - ) + content_model.pulp_2to3_detail.pre_migrate_content_detail(pulp2content_batch) pulp2detail_pb.done += content_saved pulp2detail_pb.save() @@ -343,12 +323,8 @@ def delete_removed_pulp2_content(content_model): pulp2_content_by_id = {} paginator = Paginator(pulp2_unit_id_qs, batch_size) for page in range(1, paginator.num_pages + 1): - mongo_content_qs = content_model.pulp2.objects( - id__in=paginator.page(page).object_list - ) - batched_mongo_content_qs = mongo_content_qs.only(*mongo_fields).batch_size( - batch_size - ) + mongo_content_qs = content_model.pulp2.objects(id__in=paginator.page(page).object_list) + batched_mongo_content_qs = mongo_content_qs.only(*mongo_fields).batch_size(batch_size) for record in batched_mongo_content_qs.no_cache(): pulp2_content_by_id[record.id] = record @@ -384,9 +360,7 @@ def delete_removed_pulp2_content(content_model): downloaded=downloaded, pulp2_repo=relation.pulp2_repository, ) - _logger.debug( - "Add content item to the list to migrate: {item}".format(item=item) - ) + _logger.debug("Add content item to the list to migrate: {item}".format(item=item)) pulp2content.append(item) pulp2content_pb.total += 1 pulp2detail_pb.total += 1 @@ -409,15 +383,11 @@ def delete_removed_pulp2_content(content_model): # in this case, we still need to update the is_migrated flag manually because of errata. # in pulp2 sync and copy cases of updated errata are not covered # only when uploading errata last_unit_added is updated on all the repos that contain it - mutated_content = Pulp2RepoContent.objects.filter( - pulp2_unit_id__in=pulp2mutatedcontent - ) + mutated_content = Pulp2RepoContent.objects.filter(pulp2_unit_id__in=pulp2mutatedcontent) repo_to_update_ids = mutated_content.values_list( "pulp2_repository_id", flat=True ).distinct() - Pulp2Repository.objects.filter(pk__in=repo_to_update_ids).update( - is_migrated=False - ) + Pulp2Repository.objects.filter(pk__in=repo_to_update_ids).update(is_migrated=False) if lazy_type: pre_migrate_lazycatalog(content_type) @@ -455,9 +425,7 @@ def pre_migrate_lazycatalog(content_type): pulp2lazycatalog.append(item) if len(pulp2lazycatalog) >= batch_size: - Pulp2LazyCatalog.objects.bulk_create( - pulp2lazycatalog, ignore_conflicts=True - ) + Pulp2LazyCatalog.objects.bulk_create(pulp2lazycatalog, ignore_conflicts=True) pulp2lazycatalog.clear() else: Pulp2LazyCatalog.objects.bulk_create(pulp2lazycatalog, ignore_conflicts=True) @@ -521,24 +489,20 @@ def pre_migrate_all_without_content(plan): or epoch ) imp_premigrated_last = ( - Pulp2Importer.objects.filter(imp_type_q).aggregate( - Max("pulp2_last_updated") - )["pulp2_last_updated__max"] + Pulp2Importer.objects.filter(imp_type_q).aggregate(Max("pulp2_last_updated"))[ + "pulp2_last_updated__max" + ] or epoch ) dist_premigrated_last = ( - Pulp2Distributor.objects.filter(dist_type_q).aggregate( - Max("pulp2_last_updated") - )["pulp2_last_updated__max"] + Pulp2Distributor.objects.filter(dist_type_q).aggregate(Max("pulp2_last_updated"))[ + "pulp2_last_updated__max" + ] or epoch ) - is_content_added_q = mongo_Q( - last_unit_added__gte=repo_premigrated_last_by_added - ) - is_content_removed_q = mongo_Q( - last_unit_removed__gte=repo_premigrated_last_by_removed - ) + is_content_added_q = mongo_Q(last_unit_added__gte=repo_premigrated_last_by_added) + is_content_removed_q = mongo_Q(last_unit_removed__gte=repo_premigrated_last_by_removed) is_new_enough_repo_q = is_content_added_q | is_content_removed_q is_empty_repo_q = mongo_Q(last_unit_added__exists=False) is_new_enough_imp_q = mongo_Q(last_updated__gte=imp_premigrated_last) @@ -547,19 +511,17 @@ def pre_migrate_all_without_content(plan): imp_repo_id_q = mongo_Q(repo_id__in=importers_repos) dist_repo_id_q = mongo_Q(repo_id__in=distributors_repos) - updated_importers = Importer.objects( - imp_repo_id_q & is_new_enough_imp_q - ).only("repo_id") + updated_importers = Importer.objects(imp_repo_id_q & is_new_enough_imp_q).only( + "repo_id" + ) updated_imp_repos = set(imp.repo_id for imp in updated_importers) - updated_distributors = Distributor.objects( - dist_repo_id_q & is_new_enough_dist_q - ).only("repo_id") + updated_distributors = Distributor.objects(dist_repo_id_q & is_new_enough_dist_q).only( + "repo_id" + ) updated_dist_repos = set(dist.repo_id for dist in updated_distributors) updated_impdist_repos = updated_imp_repos | updated_dist_repos - mongo_updated_repo_q = repo_repo_id_q & ( - is_new_enough_repo_q | is_empty_repo_q - ) + mongo_updated_repo_q = repo_repo_id_q & (is_new_enough_repo_q | is_empty_repo_q) mongo_updated_imp_dist_repo_q = mongo_Q(repo_id__in=updated_impdist_repos) mongo_repo_qs = Repository.objects( @@ -715,9 +677,7 @@ def pre_migrate_distributor(repo_id, distributor_migrators): distributor_migrators(dict): supported distributor types and their models for migration """ distributor_types = list(distributor_migrators.keys()) - mongo_distributor_q = mongo_Q( - repo_id=repo_id, distributor_type_id__in=distributor_types - ) + mongo_distributor_q = mongo_Q(repo_id=repo_id, distributor_type_id__in=distributor_types) mongo_distributor_qs = Distributor.objects(mongo_distributor_q) if not mongo_distributor_qs: @@ -753,23 +713,15 @@ def pre_migrate_distributor(repo_id, distributor_migrators): distributor.is_migrated = False dist_migrator = distributor_migrators.get(distributor.pulp2_type_id) needs_new_publication = dist_migrator.needs_new_publication(distributor) - needs_new_distribution = dist_migrator.needs_new_distribution( - distributor - ) - remove_publication = ( - needs_new_publication and distributor.pulp3_publication - ) - remove_distribution = ( - needs_new_distribution and distributor.pulp3_distribution - ) + needs_new_distribution = dist_migrator.needs_new_distribution(distributor) + remove_publication = needs_new_publication and distributor.pulp3_publication + remove_distribution = needs_new_distribution and distributor.pulp3_distribution if remove_publication: # check if publication is shared by multiple distributions # on the corresponding distributor flip the flag to false so the affected # distribution will be updated with the new publication - pulp2dists = ( - distributor.pulp3_publication.pulp2distributor_set.all() - ) + pulp2dists = distributor.pulp3_publication.pulp2distributor_set.all() for dist in pulp2dists: if dist.is_migrated: dist.is_migrated = False @@ -801,9 +753,7 @@ def pre_migrate_repocontent(repo): mongo_repocontent_qs = RepositoryContentUnit.objects(mongo_repocontent_q) repocontent = [] - for repocontent_data in ( - mongo_repocontent_qs.exclude("repo_id").as_pymongo().no_cache() - ): + for repocontent_data in mongo_repocontent_qs.exclude("repo_id").as_pymongo().no_cache(): item = Pulp2RepoContent( pulp2_unit_id=repocontent_data["unit_id"], pulp2_content_type_id=repocontent_data["unit_type_id"], @@ -841,43 +791,31 @@ def handle_outdated_resources(plan): repos_to_consider = set(inplan_repos).intersection(repos_to_consider) mongo_repo_q = mongo_Q(repo_id__in=repos_to_consider) - mongo_repo_obj_ids = set( - str(i.id) for i in Repository.objects(mongo_repo_q).only("id") - ) + mongo_repo_obj_ids = set(str(i.id) for i in Repository.objects(mongo_repo_q).only("id")) repo_type_q = Q(pulp2_repo_type=plugin_plan.type) inplan_repo_q = Q(pulp2_object_id__in=mongo_repo_obj_ids) - Pulp2Repository.objects.filter(repo_type_q).exclude(inplan_repo_q).update( - not_in_plan=True - ) + Pulp2Repository.objects.filter(repo_type_q).exclude(inplan_repo_q).update(not_in_plan=True) # Mark removed or excluded importers inplan_imp_repos = plugin_plan.get_importers_repos() mongo_imp_q = mongo_Q(repo_id__in=inplan_imp_repos) - mongo_imp_obj_ids = set( - str(i.id) for i in Importer.objects(mongo_imp_q).only("id") - ) + mongo_imp_obj_ids = set(str(i.id) for i in Importer.objects(mongo_imp_q).only("id")) imp_types = plugin_plan.migrator.importer_migrators.keys() imp_type_q = Q(pulp2_type_id__in=imp_types) inplan_imp_q = Q(pulp2_object_id__in=mongo_imp_obj_ids) - Pulp2Importer.objects.filter(imp_type_q).exclude(inplan_imp_q).update( - not_in_plan=True - ) + Pulp2Importer.objects.filter(imp_type_q).exclude(inplan_imp_q).update(not_in_plan=True) # Mark removed or excluded distributors inplan_dist_repos = plugin_plan.get_distributors_repos() mongo_dist_q = mongo_Q(repo_id__in=inplan_dist_repos) - mongo_dist_obj_ids = set( - str(i.id) for i in Distributor.objects(mongo_dist_q).only("id") - ) + mongo_dist_obj_ids = set(str(i.id) for i in Distributor.objects(mongo_dist_q).only("id")) dist_types = plugin_plan.migrator.distributor_migrators.keys() dist_type_q = Q(pulp2_type_id__in=dist_types) inplan_dist_q = Q(pulp2_object_id__in=mongo_dist_obj_ids) - Pulp2Distributor.objects.filter(dist_type_q).exclude(inplan_dist_q).update( - not_in_plan=True - ) + Pulp2Distributor.objects.filter(dist_type_q).exclude(inplan_dist_q).update(not_in_plan=True) # Delete old Publications/Distributions which are no longer present in Pulp2. @@ -892,12 +830,8 @@ def handle_outdated_resources(plan): Q(is_migrated=False) | Q(not_in_plan=True) ) - old_dist_query = Q(pulp3_distribution__isnull=False) | Q( - pulp3_publication__isnull=False - ) - old_dist_query &= Q(pulp2_repos__in=repos_with_old_distributions_qs) | Q( - not_in_plan=True - ) + old_dist_query = Q(pulp3_distribution__isnull=False) | Q(pulp3_publication__isnull=False) + old_dist_query &= Q(pulp2_repos__in=repos_with_old_distributions_qs) | Q(not_in_plan=True) with transaction.atomic(): pulp2distributors_with_old_distributions_qs = Pulp2Distributor.objects.filter( @@ -929,9 +863,7 @@ def handle_outdated_resources(plan): # Pulp2Distributors with is_migrated=false is handled and re-added properly at # migration stage. # NOTE: this needs to be removed last, the queries above use this relation. - not_migrated_dists = Pulp2Distributor.objects.filter(is_migrated=False).only( - "pulp_id" - ) + not_migrated_dists = Pulp2Distributor.objects.filter(is_migrated=False).only("pulp_id") Pulp2Distributor.pulp2_repos.through.objects.filter( pulp2distributor__in=not_migrated_dists ).delete() diff --git a/pulp_2to3_migration/app/serializers.py b/pulp_2to3_migration/app/serializers.py index 07561e0b..8c8b8af3 100644 --- a/pulp_2to3_migration/app/serializers.py +++ b/pulp_2to3_migration/app/serializers.py @@ -78,9 +78,7 @@ def validate(self, data): plugins_to_migrate.add(plugin_type["type"]) if len(loaded_plan["plugins"]) != len(plugins_to_migrate): raise serializers.ValidationError( - _( - "Provided Migration Plan contains same plugin type specified more that once." - ) + _("Provided Migration Plan contains same plugin type specified more that once.") ) # MongoDB connection initialization connection.initialize() @@ -121,8 +119,7 @@ class MigrationPlanRunSerializer(serializers.Serializer): ) dry_run = serializers.BooleanField( help_text=_( - "If ``True``, performs validation of a Migration Plan only, no migration is " - "run." + "If ``True``, performs validation of a Migration Plan only, no migration is " "run." ), required=False, default=False, diff --git a/pulp_2to3_migration/app/tasks/migrate.py b/pulp_2to3_migration/app/tasks/migrate.py index a837c587..51197602 100644 --- a/pulp_2to3_migration/app/tasks/migrate.py +++ b/pulp_2to3_migration/app/tasks/migrate.py @@ -27,9 +27,7 @@ _logger = logging.getLogger(__name__) -def migrate_from_pulp2( - migration_plan_pk, validate=False, dry_run=False, skip_corrupted=False -): +def migrate_from_pulp2(migration_plan_pk, validate=False, dry_run=False, skip_corrupted=False): """ Main task to migrate from Pulp 2 to Pulp 3. diff --git a/pulp_2to3_migration/app/tasks/reset.py b/pulp_2to3_migration/app/tasks/reset.py index 9dc8d0b6..4dade52f 100644 --- a/pulp_2to3_migration/app/tasks/reset.py +++ b/pulp_2to3_migration/app/tasks/reset.py @@ -64,9 +64,7 @@ def reset_pulp3_data(migration_plan_pk): pb.increment() - pb_data = dict( - message="Removing pre-migrated data", code="reset.premigrated.data", total=1 - ) + pb_data = dict(message="Removing pre-migrated data", code="reset.premigrated.data", total=1) with ProgressReport(**pb_data) as pb: for plugin in plan.get_plugin_plans(): for ( @@ -74,24 +72,18 @@ def reset_pulp3_data(migration_plan_pk): pulp2to3_content_model, ) in plugin.migrator.content_models.items(): pulp2to3_content_model.objects.all().only("pk").delete() - Pulp2Content.objects.filter(pulp2_content_type_id=content_type).only( + Pulp2Content.objects.filter(pulp2_content_type_id=content_type).only("pk").delete() + Pulp2LazyCatalog.objects.filter(pulp2_content_type_id=content_type).only( "pk" ).delete() - Pulp2LazyCatalog.objects.filter( - pulp2_content_type_id=content_type - ).only("pk").delete() - Pulp2RepoContent.objects.filter( - pulp2_content_type_id=content_type - ).only("pk").delete() - for importer_type in plugin.migrator.importer_migrators: - Pulp2Importer.objects.filter(pulp2_type_id=importer_type).only( + Pulp2RepoContent.objects.filter(pulp2_content_type_id=content_type).only( "pk" ).delete() + for importer_type in plugin.migrator.importer_migrators: + Pulp2Importer.objects.filter(pulp2_type_id=importer_type).only("pk").delete() for dist_type in plugin.migrator.distributor_migrators: - Pulp2Distributor.objects.filter(pulp2_type_id=dist_type).only( - "pk" - ).delete() - Pulp2Repository.objects.filter( - pulp2_repo_type=plugin.migrator.pulp2_plugin - ).only("pk").delete() + Pulp2Distributor.objects.filter(pulp2_type_id=dist_type).only("pk").delete() + Pulp2Repository.objects.filter(pulp2_repo_type=plugin.migrator.pulp2_plugin).only( + "pk" + ).delete() pb.increment() diff --git a/pulp_2to3_migration/app/viewsets.py b/pulp_2to3_migration/app/viewsets.py index 2151bbea..d33947aa 100644 --- a/pulp_2to3_migration/app/viewsets.py +++ b/pulp_2to3_migration/app/viewsets.py @@ -89,18 +89,14 @@ class MigrationPlanViewSet( def run(self, request, pk): """Run the migration plan.""" migration_plan = self.get_object() - serializer = MigrationPlanRunSerializer( - data=request.data, context={"request": request} - ) + serializer = MigrationPlanRunSerializer(data=request.data, context={"request": request}) serializer.is_valid(raise_exception=True) validate = serializer.validated_data.get("validate", False) dry_run = serializer.validated_data.get("dry_run", False) skip_corrupted = serializer.validated_data.get("skip_corrupted", False) if is_migration_plan_running(): - raise ValidationError( - _("Only one migration plan can run or be reset at a time") - ) + raise ValidationError(_("Only one migration plan can run or be reset at a time")) result = dispatch( migrate_from_pulp2, @@ -126,9 +122,7 @@ def reset(self, request, pk): migration_plan = self.get_object() if is_migration_plan_running(): - raise ValidationError( - _("Only one migration plan can run or be reset at a time") - ) + raise ValidationError(_("Only one migration plan can run or be reset at a time")) result = dispatch( reset_pulp3_data, @@ -160,9 +154,7 @@ class Meta: } -class Pulp2ContentViewSet( - NamedModelViewSet, mixins.RetrieveModelMixin, mixins.ListModelMixin -): +class Pulp2ContentViewSet(NamedModelViewSet, mixins.RetrieveModelMixin, mixins.ListModelMixin): """ ViewSet for Pulp2Content model. """ @@ -191,9 +183,7 @@ class Meta: } -class Pulp2RepositoriesViewSet( - NamedModelViewSet, mixins.RetrieveModelMixin, mixins.ListModelMixin -): +class Pulp2RepositoriesViewSet(NamedModelViewSet, mixins.RetrieveModelMixin, mixins.ListModelMixin): """ ViewSet for Pulp2Repositories model. """ diff --git a/pulp_2to3_migration/pulp2/connection.py b/pulp_2to3_migration/pulp2/connection.py index 4b17b53b..fe1a7419 100644 --- a/pulp_2to3_migration/pulp2/connection.py +++ b/pulp_2to3_migration/pulp2/connection.py @@ -40,9 +40,7 @@ raise ConfigurationError(_("PULP2_MONGODB is not configured in your settings.")) -def initialize( - name=None, seeds=None, max_pool_size=None, replica_set=None, max_timeout=32 -): +def initialize(name=None, seeds=None, max_pool_size=None, replica_set=None, max_timeout=32): """ Initialize the connection pool and top-level database for pulp. Calling this more than once will raise a RuntimeError. @@ -90,9 +88,7 @@ def initialize( if ssl_certfile: connection_kwargs["ssl_certfile"] = ssl_certfile verify_ssl = pulp2_mongodb.get("verify_ssl") - connection_kwargs["ssl_cert_reqs"] = ( - ssl.CERT_REQUIRED if verify_ssl else ssl.CERT_NONE - ) + connection_kwargs["ssl_cert_reqs"] = ssl.CERT_REQUIRED if verify_ssl else ssl.CERT_NONE connection_kwargs["ssl_ca_certs"] = pulp2_mongodb.get("ca_path") # If username & password have been specified in the database config, @@ -125,26 +121,17 @@ def initialize( ) ) while True: - _CONNECTION = _connect_to_one_of_seeds( - connection_kwargs, seeds_list, name - ) + _CONNECTION = _connect_to_one_of_seeds(connection_kwargs, seeds_list, name) if _CONNECTION: - db_version = semantic_version.Version( - _CONNECTION.server_info()["version"] - ) + db_version = semantic_version.Version(_CONNECTION.server_info()["version"]) if db_version < MONGO_MINIMUM_VERSION: raise RuntimeError( - _( - "Pulp requires Mongo version %s, but DB is reporting" - "version %s" - ) + _("Pulp requires Mongo version %s, but DB is reporting" "version %s") % (MONGO_MINIMUM_VERSION, db_version) ) break else: - next_delay = min( - next(mongo_retry_timeout_seconds_generator), max_timeout - ) + next_delay = min(next(mongo_retry_timeout_seconds_generator), max_timeout) msg = _( "Could not connect to any of MongoDB seeds at %(url)s:\n... Waiting " "%(retry_timeout)s seconds and trying again." @@ -153,19 +140,14 @@ def initialize( time.sleep(next_delay) else: raise ConfigurationError( - _( - "Database 'seeds' config must include at least one " - "hostname:port value." - ) + _("Database 'seeds' config must include at least one " "hostname:port value.") ) try: _DATABASE = mongoengine.connection.get_db() except OperationFailure as error: if error.code == 18: - msg = _( - "Authentication to MongoDB " "with username and password failed." - ) + msg = _("Authentication to MongoDB " "with username and password failed.") raise RuntimeError(msg) _DATABASE.add_son_manipulator(NamespaceInjector()) diff --git a/pulp_2to3_migration/tests/functional/common_plans.py b/pulp_2to3_migration/tests/functional/common_plans.py index c247b399..3641df0d 100644 --- a/pulp_2to3_migration/tests/functional/common_plans.py +++ b/pulp_2to3_migration/tests/functional/common_plans.py @@ -37,9 +37,7 @@ "repository_versions": [ { "pulp2_repository_id": "debian-complex-dists", - "pulp2_distributor_repository_ids": [ - "debian-complex-dists" - ], + "pulp2_distributor_repository_ids": ["debian-complex-dists"], } ], }, @@ -229,9 +227,7 @@ "name": "file-many", "pulp2_importer_repository_id": "file", # policy: immediate "repository_versions": [ - { - "pulp2_repository_id": "file-many" # content count: iso - 250 - } + {"pulp2_repository_id": "file-many"} # content count: iso - 250 ], }, ], diff --git a/pulp_2to3_migration/tests/functional/constants.py b/pulp_2to3_migration/tests/functional/constants.py index 0114280a..4c260a04 100644 --- a/pulp_2to3_migration/tests/functional/constants.py +++ b/pulp_2to3_migration/tests/functional/constants.py @@ -43,7 +43,5 @@ """ # noqa FILE_URL = "https://repos.fedorapeople.org/pulp/pulp/fixtures/file/PULP_MANIFEST" -FILE_MANY_URL = ( - "https://repos.fedorapeople.org/pulp/pulp/fixtures/file-many/PULP_MANIFEST" -) +FILE_MANY_URL = "https://repos.fedorapeople.org/pulp/pulp/fixtures/file-many/PULP_MANIFEST" FIXTURES_BASE_URL = "https://fixtures.pulpproject.org/" diff --git a/pulp_2to3_migration/tests/functional/test_deb_repo.py b/pulp_2to3_migration/tests/functional/test_deb_repo.py index 31a57743..b1263e6f 100644 --- a/pulp_2to3_migration/tests/functional/test_deb_repo.py +++ b/pulp_2to3_migration/tests/functional/test_deb_repo.py @@ -64,9 +64,7 @@ def test_deb_repo_migration(self): Check that names are migrated correctly and that the number of versions and content count is correct. """ - self.assertEqual( - self.deb_repo_api.list().count, len(self.repo_info.repositories) - ) + self.assertEqual(self.deb_repo_api.list().count, len(self.repo_info.repositories)) # content count in total for content_type, api in self.deb_content_apis.items(): @@ -86,9 +84,7 @@ def test_deb_repo_migration(self): # content count per repo for content_type, api in self.deb_content_apis.items(): with self.subTest(content_type=content_type): - repo_content = api.list( - repository_version=repo.latest_version_href - ) + repo_content = api.list(repository_version=repo.latest_version_href) self.assertEqual( repo_content.count, self.repo_info.repositories[repo.name].get(content_type, 0), @@ -111,12 +107,8 @@ def test_deb_distributor_migration(self): """ Test that DEB distributors are correctly migrated. """ - self.assertEqual( - self.deb_publication_api.list().count, self.repo_info.publications - ) - self.assertEqual( - self.deb_distribution_api.list().count, self.repo_info.distributions - ) + self.assertEqual(self.deb_publication_api.list().count, self.repo_info.publications) + self.assertEqual(self.deb_distribution_api.list().count, self.repo_info.distributions) for dist in self.deb_distribution_api.list().results: with self.subTest(dist=dist): base_path = "-".join(dist.name.split("-")[1:]) diff --git a/pulp_2to3_migration/tests/functional/test_file_migration.py b/pulp_2to3_migration/tests/functional/test_file_migration.py index c87921fb..8eee2381 100644 --- a/pulp_2to3_migration/tests/functional/test_file_migration.py +++ b/pulp_2to3_migration/tests/functional/test_file_migration.py @@ -110,23 +110,16 @@ def _do_test(self, repos, migration_plan): # Assert that there is a result self.failIf( not pulp3repos.results, - "Missing a Pulp 3 repository for Pulp 2 " - "repository id '{}'".format(repo_id), + "Missing a Pulp 3 repository for Pulp 2 " "repository id '{}'".format(repo_id), ) repo_href = pulp3repos.results[0].pulp_href # Assert that the name in pulp 3 matches the repo_id in pulp 2 self.assertEqual(repo_id, pulp3repos.results[0].name) # Assert that there is a Repository Version with the same number of content units as # associated with the repository in Pulp 2. - repo_version_href = ( - self.file_repo_versions_api.list(repo_href).results[0].pulp_href - ) - repo_version_content = self.file_content_api.list( - repository_version=repo_version_href - ) - self.assertEqual( - PULP_2_ISO_FIXTURE_DATA[repo_id], repo_version_content.count - ) + repo_version_href = self.file_repo_versions_api.list(repo_href).results[0].pulp_href + repo_version_content = self.file_content_api.list(repository_version=repo_version_href) + self.assertEqual(PULP_2_ISO_FIXTURE_DATA[repo_id], repo_version_content.count) # TODO: count only not_in_plan=False repositories from ../pulp2repositories/ endpoint self.assertEqual(len(repos), self.file_repo_api.list().count) diff --git a/pulp_2to3_migration/tests/functional/test_file_reset.py b/pulp_2to3_migration/tests/functional/test_file_reset.py index 6db6f193..435f2f33 100644 --- a/pulp_2to3_migration/tests/functional/test_file_reset.py +++ b/pulp_2to3_migration/tests/functional/test_file_reset.py @@ -53,16 +53,10 @@ def test_reset_file_plugin(self): self.pulp2repositories_api.list().count, PULP_2_ISO_FIXTURE_DATA["repositories"], ) - self.assertEqual( - self.pulp2content_api.list().count, PULP_2_ISO_FIXTURE_DATA["content"] - ) + self.assertEqual(self.pulp2content_api.list().count, PULP_2_ISO_FIXTURE_DATA["content"]) # Assert that Pulp 3 data is there - self.assertEqual( - self.file_repo_api.list().count, PULP_2_ISO_FIXTURE_DATA["repositories"] - ) - self.assertEqual( - self.file_content_api.list().count, PULP_2_ISO_FIXTURE_DATA["content"] - ) + self.assertEqual(self.file_repo_api.list().count, PULP_2_ISO_FIXTURE_DATA["repositories"]) + self.assertEqual(self.file_content_api.list().count, PULP_2_ISO_FIXTURE_DATA["content"]) mp = self.migration_plans_api.create({"plan": FILE_SIMPLE_PLAN}) self._reset_pulp3_data(mp) @@ -86,16 +80,10 @@ def test_no_reset_plugin_data(self): self.pulp2repositories_api.list().count, PULP_2_ISO_FIXTURE_DATA["repositories"], ) - self.assertEqual( - self.pulp2content_api.list().count, PULP_2_ISO_FIXTURE_DATA["content"] - ) + self.assertEqual(self.pulp2content_api.list().count, PULP_2_ISO_FIXTURE_DATA["content"]) # Assert that Pulp 3 data is there - self.assertEqual( - self.file_repo_api.list().count, PULP_2_ISO_FIXTURE_DATA["repositories"] - ) - self.assertEqual( - self.file_content_api.list().count, PULP_2_ISO_FIXTURE_DATA["content"] - ) + self.assertEqual(self.file_repo_api.list().count, PULP_2_ISO_FIXTURE_DATA["repositories"]) + self.assertEqual(self.file_content_api.list().count, PULP_2_ISO_FIXTURE_DATA["content"]) mp_rpm = self.migration_plans_api.create({"plan": RPM_SIMPLE_PLAN}) self._reset_pulp3_data(mp_rpm) @@ -104,16 +92,10 @@ def test_no_reset_plugin_data(self): self.pulp2repositories_api.list().count, PULP_2_ISO_FIXTURE_DATA["repositories"], ) - self.assertEqual( - self.pulp2content_api.list().count, PULP_2_ISO_FIXTURE_DATA["content"] - ) + self.assertEqual(self.pulp2content_api.list().count, PULP_2_ISO_FIXTURE_DATA["content"]) # Assert that Pulp 3 data is still there - self.assertEqual( - self.file_repo_api.list().count, PULP_2_ISO_FIXTURE_DATA["repositories"] - ) - self.assertEqual( - self.file_content_api.list().count, PULP_2_ISO_FIXTURE_DATA["content"] - ) + self.assertEqual(self.file_repo_api.list().count, PULP_2_ISO_FIXTURE_DATA["repositories"]) + self.assertEqual(self.file_content_api.list().count, PULP_2_ISO_FIXTURE_DATA["content"]) def test_migrate_after_reset(self): """Test that migration runs successfully after the reset of Pulp 3 and pre-migrated data.""" @@ -123,16 +105,10 @@ def test_migrate_after_reset(self): self.pulp2repositories_api.list().count, PULP_2_ISO_FIXTURE_DATA["repositories"], ) - self.assertEqual( - self.pulp2content_api.list().count, PULP_2_ISO_FIXTURE_DATA["content"] - ) + self.assertEqual(self.pulp2content_api.list().count, PULP_2_ISO_FIXTURE_DATA["content"]) # Assert that Pulp 3 data is there - self.assertEqual( - self.file_repo_api.list().count, PULP_2_ISO_FIXTURE_DATA["repositories"] - ) - self.assertEqual( - self.file_content_api.list().count, PULP_2_ISO_FIXTURE_DATA["content"] - ) + self.assertEqual(self.file_repo_api.list().count, PULP_2_ISO_FIXTURE_DATA["repositories"]) + self.assertEqual(self.file_content_api.list().count, PULP_2_ISO_FIXTURE_DATA["content"]) mp = self.migration_plans_api.create({"plan": FILE_SIMPLE_PLAN}) self._reset_pulp3_data(mp) @@ -149,16 +125,10 @@ def test_migrate_after_reset(self): self.pulp2repositories_api.list().count, PULP_2_ISO_FIXTURE_DATA["repositories"], ) - self.assertEqual( - self.pulp2content_api.list().count, PULP_2_ISO_FIXTURE_DATA["content"] - ) + self.assertEqual(self.pulp2content_api.list().count, PULP_2_ISO_FIXTURE_DATA["content"]) # Assert that Pulp 3 data is back there - self.assertEqual( - self.file_repo_api.list().count, PULP_2_ISO_FIXTURE_DATA["repositories"] - ) - self.assertEqual( - self.file_content_api.list().count, PULP_2_ISO_FIXTURE_DATA["content"] - ) + self.assertEqual(self.file_repo_api.list().count, PULP_2_ISO_FIXTURE_DATA["repositories"]) + self.assertEqual(self.file_content_api.list().count, PULP_2_ISO_FIXTURE_DATA["content"]) def test_no_reset_when_migration(self): """Test that reset is not run when migration is.""" diff --git a/pulp_2to3_migration/tests/functional/test_migration_behaviour.py b/pulp_2to3_migration/tests/functional/test_migration_behaviour.py index fcb88e98..10950142 100644 --- a/pulp_2to3_migration/tests/functional/test_migration_behaviour.py +++ b/pulp_2to3_migration/tests/functional/test_migration_behaviour.py @@ -27,12 +27,8 @@ "pulp2_importer_repository_id": "file-many", # policy: on_demand "repository_versions": [ {"pulp2_repository_id": "file"}, # content count: iso - 3 - { - "pulp2_repository_id": "file-large" # content count: iso - 10 - }, - { - "pulp2_repository_id": "file-many" # content count: iso - 250 - }, + {"pulp2_repository_id": "file-large"}, # content count: iso - 10 + {"pulp2_repository_id": "file-many"}, # content count: iso - 250 ], } ], @@ -176,9 +172,7 @@ def _test_pulp2content(self, plan): Check the first content in the list. """ self.run_migration(plan) - pulp2content = self.pulp2content_api.list(ordering="pulp2_id", limit=1).results[ - 0 - ] + pulp2content = self.pulp2content_api.list(ordering="pulp2_id", limit=1).results[0] content_href = pulp2content.pulp3_content file_content = self.file_content_api.read(content_href) self.assertEqual(file_content.relative_path, "137.iso") @@ -195,27 +189,19 @@ def _test_pulp2repositories(self, plan): ).results[0] pulp3_repo = self.file_repo_api.read(pulp2repository.pulp3_repository_href) pulp3_remote = self.file_remote_api.read(pulp2repository.pulp3_remote_href) - pulp3_pub = self.file_publication_api.read( - pulp2repository.pulp3_publication_href - ) - pulp3_dist = self.file_distribution_api.read( - pulp2repository.pulp3_distribution_hrefs[0] - ) + pulp3_pub = self.file_publication_api.read(pulp2repository.pulp3_publication_href) + pulp3_dist = self.file_distribution_api.read(pulp2repository.pulp3_distribution_hrefs[0]) self.assertEqual(self.pulp2repositories_api.list().count, 4) self.assertTrue(pulp2repository.is_migrated) self.assertEqual(pulp2repository.pulp2_repo_id, "file") self.assertEqual(pulp3_repo.name, "file") - self.assertEqual( - pulp3_repo.latest_version_href, pulp2repository.pulp3_repository_version - ) + self.assertEqual(pulp3_repo.latest_version_href, pulp2repository.pulp3_repository_version) self.assertEqual(pulp3_remote.url, FILE_URL) self.assertEqual(pulp3_remote.policy, "immediate") self.assertEqual(pulp3_remote.pulp_href, pulp2repository.pulp3_remote_href) self.assertEqual(pulp3_pub.manifest, "PULP_MANIFEST") - self.assertEqual( - pulp3_pub.repository_version, pulp2repository.pulp3_repository_version - ) + self.assertEqual(pulp3_pub.repository_version, pulp2repository.pulp3_repository_version) self.assertEqual(pulp3_pub.distributions[0], pulp3_dist.pulp_href) self.assertEqual(pulp3_dist.base_path, "file") @@ -247,9 +233,7 @@ def test_many_into_one_repo(self): pulp3_repo = self.file_repo_api.list().results[0] self.assertEqual(self.file_repo_api.list().count, 1) - self.assertEqual( - self.file_repo_versions_api.list(pulp3_repo.pulp_href).count, 4 - ) + self.assertEqual(self.file_repo_versions_api.list(pulp3_repo.pulp_href).count, 4) self.assertEqual(self.file_content_api.list().count, 263) for pulp2repo in pulp2repositories: with self.subTest(pulp2repo=pulp2repo): @@ -258,9 +242,7 @@ def test_many_into_one_repo(self): repo_content = self.file_content_api.list( repository_version=pulp2repo.pulp3_repository_version ) - self.assertEqual( - repo_content.count, CONTENT_COUNT[pulp2repo.pulp2_repo_id] - ) + self.assertEqual(repo_content.count, CONTENT_COUNT[pulp2repo.pulp2_repo_id]) def test_importer_different_repo(self): """ @@ -269,9 +251,7 @@ def test_importer_different_repo(self): Importers are swapped in the plan. """ self.run_migration(FILE_IMPORTER_DIFF_PLAN) - pulp2repositories = self.pulp2repositories_api.list( - ordering="pulp2_repo_id" - ).results + pulp2repositories = self.pulp2repositories_api.list(ordering="pulp2_repo_id").results pulp2repo1, pulp2repo2 = pulp2repositories pulp3_remote1 = self.file_remote_api.read(pulp2repo1.pulp3_remote_href) pulp3_remote2 = self.file_remote_api.read(pulp2repo2.pulp3_remote_href) @@ -296,29 +276,19 @@ def test_distributor_different_repo(self): Distributors are swapped in the plan. """ self.run_migration(FILE_DISTRIBUTOR_DIFF_PLAN) - pulp2repositories = self.pulp2repositories_api.list( - ordering="pulp2_repo_id" - ).results + pulp2repositories = self.pulp2repositories_api.list(ordering="pulp2_repo_id").results pulp2repo1, pulp2repo2 = pulp2repositories pulp3_pub1 = self.file_publication_api.read(pulp2repo1.pulp3_publication_href) pulp3_pub2 = self.file_publication_api.read(pulp2repo2.pulp3_publication_href) - pulp3_dist1 = self.file_distribution_api.read( - pulp2repo1.pulp3_distribution_hrefs[0] - ) - pulp3_dist2 = self.file_distribution_api.read( - pulp2repo2.pulp3_distribution_hrefs[0] - ) + pulp3_dist1 = self.file_distribution_api.read(pulp2repo1.pulp3_distribution_hrefs[0]) + pulp3_dist2 = self.file_distribution_api.read(pulp2repo2.pulp3_distribution_hrefs[0]) self.assertEqual(pulp2repo1.pulp2_repo_id, "file") self.assertEqual(pulp2repo2.pulp2_repo_id, "file-many") self.assertTrue(pulp2repo1.is_migrated) self.assertTrue(pulp2repo2.is_migrated) - self.assertEqual( - pulp3_pub1.repository_version, pulp2repo1.pulp3_repository_version - ) - self.assertEqual( - pulp3_pub2.repository_version, pulp2repo2.pulp3_repository_version - ) + self.assertEqual(pulp3_pub1.repository_version, pulp2repo1.pulp3_repository_version) + self.assertEqual(pulp3_pub2.repository_version, pulp2repo2.pulp3_repository_version) self.assertEqual(pulp3_pub1.distributions[0], pulp3_dist1.pulp_href) self.assertEqual(pulp3_pub2.distributions[0], pulp3_dist2.pulp_href) self.assertEqual(pulp3_dist1.base_path, "file-many") @@ -343,20 +313,14 @@ def test_distributor_no_repo(self): """Test that a distributor can be migrated without its native Pulp 2 repo.""" self.run_migration(DISTRIBUTOR_NO_REPO_PLAN) pulp2repository = self.pulp2repositories_api.list().results[0] - pulp3_pub = self.file_publication_api.read( - pulp2repository.pulp3_publication_href - ) - pulp3_dist = self.file_distribution_api.read( - pulp2repository.pulp3_distribution_hrefs[0] - ) + pulp3_pub = self.file_publication_api.read(pulp2repository.pulp3_publication_href) + pulp3_dist = self.file_distribution_api.read(pulp2repository.pulp3_distribution_hrefs[0]) self.assertEqual(self.pulp2repositories_api.list().count, 1) self.assertEqual(self.file_distribution_api.list().count, 1) self.assertTrue(pulp2repository.is_migrated) self.assertEqual(pulp2repository.pulp2_repo_id, "file") - self.assertEqual( - pulp3_pub.repository_version, pulp2repository.pulp3_repository_version - ) + self.assertEqual(pulp3_pub.repository_version, pulp2repository.pulp3_repository_version) self.assertEqual(pulp3_pub.distributions[0], pulp3_dist.pulp_href) self.assertEqual(pulp3_dist.base_path, "file-many") @@ -364,51 +328,33 @@ def test_no_on_demand_importer(self): """Test that if there is no importer for on_demand content, such content is not migrated.""" self.run_migration(NO_ON_DEMAND_IMPORTER_PLAN) pulp3_repo = self.file_repo_api.list().results[0] - repo_content = self.file_content_api.list( - repository_version=pulp3_repo.latest_version_href - ) + repo_content = self.file_content_api.list(repository_version=pulp3_repo.latest_version_href) - self.assertEqual( - self.file_repo_versions_api.list(pulp3_repo.pulp_href).count, 1 - ) + self.assertEqual(self.file_repo_versions_api.list(pulp3_repo.pulp_href).count, 1) self.assertEqual(repo_content.count, 0) def test_no_immediate_importer(self): """Test that if there is no importer for downloaded content, such content is migrated.""" self.run_migration(NO_IMMEDIATE_IMPORTER_PLAN) pulp3_repo = self.file_repo_api.list().results[0] - repo_content = self.file_content_api.list( - repository_version=pulp3_repo.latest_version_href - ) + repo_content = self.file_content_api.list(repository_version=pulp3_repo.latest_version_href) - self.assertEqual( - self.file_repo_versions_api.list(pulp3_repo.pulp_href).count, 2 - ) + self.assertEqual(self.file_repo_versions_api.list(pulp3_repo.pulp_href).count, 2) self.assertEqual(repo_content.count, 3) def test_no_importer(self): """Test that if there is no importer specified at all, migration is still working fine.""" self.run_migration(NO_IMPORTER_PLAN) pulp3_repo = self.file_repo_api.list().results[0] - repo_content = self.file_content_api.list( - repository_version=pulp3_repo.latest_version_href - ) + repo_content = self.file_content_api.list(repository_version=pulp3_repo.latest_version_href) pulp2repository = self.pulp2repositories_api.list().results[0] - pulp3_pub = self.file_publication_api.read( - pulp2repository.pulp3_publication_href - ) - pulp3_dist = self.file_distribution_api.read( - pulp2repository.pulp3_distribution_hrefs[0] - ) - - self.assertEqual( - self.file_repo_versions_api.list(pulp3_repo.pulp_href).count, 2 - ) + pulp3_pub = self.file_publication_api.read(pulp2repository.pulp3_publication_href) + pulp3_dist = self.file_distribution_api.read(pulp2repository.pulp3_distribution_hrefs[0]) + + self.assertEqual(self.file_repo_versions_api.list(pulp3_repo.pulp_href).count, 2) self.assertEqual(repo_content.count, 3) self.assertEqual(pulp2repository.pulp2_repo_id, "file") self.assertEqual(pulp3_repo.remote, pulp2repository.pulp3_remote_href) - self.assertEqual( - pulp3_pub.repository_version, pulp2repository.pulp3_repository_version - ) + self.assertEqual(pulp3_pub.repository_version, pulp2repository.pulp3_repository_version) self.assertEqual(pulp3_pub.distributions[0], pulp3_dist.pulp_href) self.assertEqual(pulp3_dist.base_path, "file") diff --git a/pulp_2to3_migration/tests/functional/test_migration_plan.py b/pulp_2to3_migration/tests/functional/test_migration_plan.py index a201a09b..2d03c4e1 100644 --- a/pulp_2to3_migration/tests/functional/test_migration_plan.py +++ b/pulp_2to3_migration/tests/functional/test_migration_plan.py @@ -26,9 +26,7 @@ "repository_versions": [ { "pulp2_repository_id": "non-existing repo", # content count: iso-3 - "pulp2_distributor_repository_ids": [ - "non-existing distributor" - ], + "pulp2_distributor_repository_ids": ["non-existing distributor"], } ], } @@ -104,9 +102,7 @@ def test_validate_missing_resource(self): mp_run_response = self.migration_plans_api.run(mp.pulp_href, {"validate": True}) with self.assertRaises(PulpTaskError) as exc: monitor_task(mp_run_response.task) - self.assertEqual( - exc.exception.task.error["description"], MISSING_RESOURCE_ERROR - ) + self.assertEqual(exc.exception.task.error["description"], MISSING_RESOURCE_ERROR) def test_run_only_one_plan(self): """Test that only one plan can be run at a time""" diff --git a/pulp_2to3_migration/tests/functional/test_migration_plan_changes.py b/pulp_2to3_migration/tests/functional/test_migration_plan_changes.py index 132b068d..8a577dacc 100644 --- a/pulp_2to3_migration/tests/functional/test_migration_plan_changes.py +++ b/pulp_2to3_migration/tests/functional/test_migration_plan_changes.py @@ -111,51 +111,35 @@ def test_importer_swap(self): """ # run for the first time with the standard plan self.run_migration(FILE_COMPLEX_PLAN) - pulp2repo_file_1run = self.pulp2repositories_api.list( - pulp2_repo_id="file" - ).results[0] + pulp2repo_file_1run = self.pulp2repositories_api.list(pulp2_repo_id="file").results[0] pulp2repo_filemany_1run = self.pulp2repositories_api.list( pulp2_repo_id="file-many" ).results[0] - pulp3_repo_file_1run = self.file_repo_api.read( - pulp2repo_file_1run.pulp3_repository_href - ) + pulp3_repo_file_1run = self.file_repo_api.read(pulp2repo_file_1run.pulp3_repository_href) pulp3_repo_filemany_1run = self.file_repo_api.read( pulp2repo_filemany_1run.pulp3_repository_href ) - pulp3_remote_file_1run = self.file_remote_api.read( - pulp2repo_file_1run.pulp3_remote_href - ) + pulp3_remote_file_1run = self.file_remote_api.read(pulp2repo_file_1run.pulp3_remote_href) pulp3_remote_filemany_1run = self.file_remote_api.read( pulp2repo_filemany_1run.pulp3_remote_href ) self.assertEqual(pulp3_remote_file_1run.url, FILE_URL) self.assertEqual(pulp3_remote_filemany_1run.url, FILE_MANY_URL) - self.assertEqual( - pulp3_repo_file_1run.remote, pulp2repo_file_1run.pulp3_remote_href - ) - self.assertEqual( - pulp3_repo_filemany_1run.remote, pulp2repo_filemany_1run.pulp3_remote_href - ) + self.assertEqual(pulp3_repo_file_1run.remote, pulp2repo_file_1run.pulp3_remote_href) + self.assertEqual(pulp3_repo_filemany_1run.remote, pulp2repo_filemany_1run.pulp3_remote_href) # run a plan with swapped importers self.run_migration(FILE_IMPORTER_DIFF_PLAN) - pulp2repo_file_2run = self.pulp2repositories_api.list( - pulp2_repo_id="file" - ).results[0] + pulp2repo_file_2run = self.pulp2repositories_api.list(pulp2_repo_id="file").results[0] pulp2repo_filemany_2run = self.pulp2repositories_api.list( pulp2_repo_id="file-many" ).results[0] - pulp3_repo_file_2run = self.file_repo_api.read( - pulp2repo_file_2run.pulp3_repository_href - ) + pulp3_repo_file_2run = self.file_repo_api.read(pulp2repo_file_2run.pulp3_repository_href) pulp3_repo_filemany_2run = self.file_repo_api.read( pulp2repo_filemany_2run.pulp3_repository_href ) - pulp3_remote_file_2run = self.file_remote_api.read( - pulp2repo_file_2run.pulp3_remote_href - ) + pulp3_remote_file_2run = self.file_remote_api.read(pulp2repo_file_2run.pulp3_remote_href) pulp3_remote_filemany_2run = self.file_remote_api.read( pulp2repo_filemany_2run.pulp3_remote_href ) @@ -164,12 +148,8 @@ def test_importer_swap(self): self.assertEqual(pulp3_remote_file_1run, pulp3_remote_filemany_2run) self.assertEqual(pulp3_remote_filemany_1run, pulp3_remote_file_2run) # both Pulp2Repository and Pulp 3 "Repository" should still match - self.assertEqual( - pulp3_repo_file_2run.remote, pulp2repo_file_2run.pulp3_remote_href - ) - self.assertEqual( - pulp3_repo_filemany_2run.remote, pulp2repo_filemany_2run.pulp3_remote_href - ) + self.assertEqual(pulp3_repo_file_2run.remote, pulp2repo_file_2run.pulp3_remote_href) + self.assertEqual(pulp3_repo_filemany_2run.remote, pulp2repo_filemany_2run.pulp3_remote_href) def test_distributor_swap(self): """ @@ -180,9 +160,7 @@ def test_distributor_swap(self): """ # run for the first time with the standard plan self.run_migration(FILE_COMPLEX_PLAN) - pulp2repo_file_1run = self.pulp2repositories_api.list( - pulp2_repo_id="file" - ).results[0] + pulp2repo_file_1run = self.pulp2repositories_api.list(pulp2_repo_id="file").results[0] pulp2repo_filemany_1run = self.pulp2repositories_api.list( pulp2_repo_id="file-many" ).results[0] @@ -201,9 +179,7 @@ def test_distributor_swap(self): # run a plan with swapped distributors self.run_migration(FILE_DISTRIBUTOR_DIFF_PLAN) - pulp2repo_file_2run = self.pulp2repositories_api.list( - pulp2_repo_id="file" - ).results[0] + pulp2repo_file_2run = self.pulp2repositories_api.list(pulp2_repo_id="file").results[0] pulp2repo_filemany_2run = self.pulp2repositories_api.list( pulp2_repo_id="file-many" ).results[0] @@ -222,12 +198,8 @@ def test_distributor_swap(self): self.assertEqual(pulp3_dist_file_1run.base_path, "file") self.assertEqual(pulp3_dist_filemany_1run.base_path, "file-many") - self.assertEqual( - pulp3_dist_file_1run.base_path, pulp3_dist_filemany_2run.base_path - ) - self.assertEqual( - pulp3_dist_filemany_1run.base_path, pulp3_dist_file_2run.base_path - ) + self.assertEqual(pulp3_dist_file_1run.base_path, pulp3_dist_filemany_2run.base_path) + self.assertEqual(pulp3_dist_filemany_1run.base_path, pulp3_dist_file_2run.base_path) # No publications should be re-used self.assertNotEqual(pulp3_pub_file_1run, pulp3_pub_file_2run) @@ -245,12 +217,8 @@ def test_distributor_move(self): """Test when a distributor moved from one repo to another in the migration plan.""" # run for the first time with the plan for 2 distributors in one repo self.run_migration(FILE_2DISTRIBUTORS_PLAN) - pulp2repo_file = self.pulp2repositories_api.list(pulp2_repo_id="file").results[ - 0 - ] - pulp2repo_file2 = self.pulp2repositories_api.list( - pulp2_repo_id="file2" - ).results[0] + pulp2repo_file = self.pulp2repositories_api.list(pulp2_repo_id="file").results[0] + pulp2repo_file2 = self.pulp2repositories_api.list(pulp2_repo_id="file2").results[0] pulp3_file_dist_base_paths_1run = [] pulp3_file2_dist_base_paths_1run = [] @@ -263,12 +231,8 @@ def test_distributor_move(self): # run a plan with one distributor moved to another repo in the plan self.run_migration(FILE_2DISTRIBUTORS_MOVED_PLAN) - pulp2repo_file = self.pulp2repositories_api.list(pulp2_repo_id="file").results[ - 0 - ] - pulp2repo_file2 = self.pulp2repositories_api.list( - pulp2_repo_id="file2" - ).results[0] + pulp2repo_file = self.pulp2repositories_api.list(pulp2_repo_id="file").results[0] + pulp2repo_file2 = self.pulp2repositories_api.list(pulp2_repo_id="file2").results[0] pulp3_file_dist_base_paths_2run = [] pulp3_file2_dist_base_paths_2run = [] diff --git a/pulp_2to3_migration/tests/functional/test_rpm_repo.py b/pulp_2to3_migration/tests/functional/test_rpm_repo.py index 1fc51fc6..813f3462 100644 --- a/pulp_2to3_migration/tests/functional/test_rpm_repo.py +++ b/pulp_2to3_migration/tests/functional/test_rpm_repo.py @@ -95,16 +95,12 @@ def test_rpm_repo_migration(self): Check that names are migrated correctly and that the number of versions and content count is correct. """ - self.assertEqual( - self.rpm_repo_api.list().count, len(self.repo_info.repositories) - ) + self.assertEqual(self.rpm_repo_api.list().count, len(self.repo_info.repositories)) # content count in total for content_type, api in self.rpm_content_apis.items(): with self.subTest(content_type=content_type): - self.assertEqual( - api.list().count, self.repo_info.content_total[content_type] - ) + self.assertEqual(api.list().count, self.repo_info.content_total[content_type]) for repo in self.rpm_repo_api.list().results: with self.subTest(repo=repo): @@ -115,9 +111,7 @@ def test_rpm_repo_migration(self): # content count per repo for content_type, api in self.rpm_content_apis.items(): with self.subTest(content_type=content_type): - repo_content = api.list( - repository_version=repo.latest_version_href - ) + repo_content = api.list(repository_version=repo.latest_version_href) self.assertEqual( repo_content.count, self.repo_info.repositories[repo.name].get(content_type, 0), @@ -139,12 +133,8 @@ def test_rpm_distributor_migration(self): """ Test that RPM distributors are correctly migrated. """ - self.assertEqual( - self.rpm_publication_api.list().count, self.repo_info.publications - ) - self.assertEqual( - self.rpm_distribution_api.list().count, self.repo_info.distributions - ) + self.assertEqual(self.rpm_publication_api.list().count, self.repo_info.publications) + self.assertEqual(self.rpm_distribution_api.list().count, self.repo_info.distributions) for dist in self.rpm_distribution_api.list().results: with self.subTest(dist=dist): base_path = "-".join(dist.name.split("-")[1:]) @@ -170,8 +160,7 @@ class TestRpmRepoMigrationComplexPlan(BaseTestRpmRepo, unittest.TestCase): @unittest.skip( - "The image files of a kickstart repo are too large for github, need to find " - "smaller ones." + "The image files of a kickstart repo are too large for github, need to find " "smaller ones." ) class TestRpmKickstartImmediateNoImporterPlan(unittest.TestCase): """ diff --git a/pulp_2to3_migration/tests/functional/test_rpm_rerun.py b/pulp_2to3_migration/tests/functional/test_rpm_rerun.py index ad03dcc5..5824a6c2 100644 --- a/pulp_2to3_migration/tests/functional/test_rpm_rerun.py +++ b/pulp_2to3_migration/tests/functional/test_rpm_rerun.py @@ -27,9 +27,7 @@ } ] -RPM_RERUN_PLAN = json.dumps( - {"plugins": [{"type": "rpm", "repositories": RERUN_REPOSITORIES}]} -) +RPM_RERUN_PLAN = json.dumps({"plugins": [{"type": "rpm", "repositories": RERUN_REPOSITORIES}]}) COPY_INTO_SAME_REPO_PLAN = json.dumps( { @@ -43,15 +41,11 @@ "repository_versions": [ { "pulp2_repository_id": "rpm-with-modules", - "pulp2_distributor_repository_ids": [ - "rpm-with-modules" - ], + "pulp2_distributor_repository_ids": ["rpm-with-modules"], }, { "pulp2_repository_id": "rpm-with-modules-copy", - "pulp2_distributor_repository_ids": [ - "rpm-with-modules-copy" - ], + "pulp2_distributor_repository_ids": ["rpm-with-modules-copy"], }, ], } @@ -73,9 +67,7 @@ "repository_versions": [ { "pulp2_repository_id": "rpm-with-modules", - "pulp2_distributor_repository_ids": [ - "rpm-with-modules" - ], + "pulp2_distributor_repository_ids": ["rpm-with-modules"], } ], }, @@ -85,9 +77,7 @@ "repository_versions": [ { "pulp2_repository_id": "rpm-with-modules-copy", - "pulp2_distributor_repository_ids": [ - "rpm-with-modules-copy" - ], + "pulp2_distributor_repository_ids": ["rpm-with-modules-copy"], } ], }, @@ -206,9 +196,7 @@ def collect_migration_data(): "pulp2content": defaultdict(dict), } for repo in cls.rpm_repo_api.list().results: - latest_version = cls.rpm_repo_versions_api.read( - repo.latest_version_href - ) + latest_version = cls.rpm_repo_versions_api.read(repo.latest_version_href) data["repos"][repo.name] = { "created": repo.pulp_created, "latest_version_number": latest_version.number, @@ -250,9 +238,7 @@ def test_rpm_only_added_content(self): # content count in total for content_type, api in self.rpm_content_apis.items(): with self.subTest(content_type=content_type): - self.assertEqual( - api.list().count, self.repo_info.content_total[content_type] - ) + self.assertEqual(api.list().count, self.repo_info.content_total[content_type]) def test_rpm_only_added_or_changed_repos(self): """ @@ -260,9 +246,7 @@ def test_rpm_only_added_or_changed_repos(self): Compare timestamps from initial run. And make sure repos are migrated correctly. """ - self.assertEqual( - self.rpm_repo_api.list().count, len(self.repo_info.repositories) - ) + self.assertEqual(self.rpm_repo_api.list().count, len(self.repo_info.repositories)) new_repo_count = 0 for repo in self.rpm_repo_api.list().results: with self.subTest(repo=repo): @@ -272,23 +256,17 @@ def test_rpm_only_added_or_changed_repos(self): repo.pulp_href, number=repo_data["latest_version_number"] ).results[0] self.assertEqual(repo.pulp_created, repo_data["created"]) - self.assertEqual( - repo_version.pulp_created, repo_data["latest_version_created"] - ) + self.assertEqual(repo_version.pulp_created, repo_data["latest_version_created"]) else: new_repo_count += 1 repo_versions = self.rpm_repo_versions_api.list(repo.pulp_href) - self.assertEqual( - repo_versions.count, self.repo_info["versions"][repo.name] - ) + self.assertEqual(repo_versions.count, self.repo_info["versions"][repo.name]) # content count per repo for content_type, api in self.rpm_content_apis.items(): with self.subTest(content_type=content_type): - repo_content = api.list( - repository_version=repo.latest_version_href - ) + repo_content = api.list(repository_version=repo.latest_version_href) self.assertEqual( repo_content.count, self.repo_info.repositories[repo.name].get(content_type, 0), @@ -342,12 +320,8 @@ def test_rpm_distributors_only_added_or_with_changed_config(self): REPO_CHECKSUMTYPE_CHANGE, REPO_BASE_PATH_CHANGE, ) - self.assertEqual( - self.rpm_publication_api.list().count, self.repo_info.publications - ) - self.assertEqual( - self.rpm_distribution_api.list().count, self.repo_info.distributions - ) + self.assertEqual(self.rpm_publication_api.list().count, self.repo_info.publications) + self.assertEqual(self.rpm_distribution_api.list().count, self.repo_info.distributions) new_publication_count = 0 for pub in self.rpm_publication_api.list().results: with self.subTest(pub=pub): @@ -442,18 +416,12 @@ def test_migrate_copy_into_same_repo(self): set_pulp2_snapshot(name="rpm_base_4repos_rerun_copy") self.task_rerun = self.run_migration(COPY_INTO_SAME_REPO_PLAN) - pulp2repo = self.pulp2repositories_api.list( - pulp2_repo_id="rpm-with-modules" - ).results[0] + pulp2repo = self.pulp2repositories_api.list(pulp2_repo_id="rpm-with-modules").results[0] version_href = pulp2repo.pulp3_repository_version - repo_advisories = self.rpm_content_apis["advisory"].list( - repository_version=version_href - ) + repo_advisories = self.rpm_content_apis["advisory"].list(repository_version=version_href) self.assertEqual(repo_advisories.count, 6) - self.assertEqual( - self.rpm_distribution_api.list(base_path="rpm-with-modules-copy").count, 1 - ) + self.assertEqual(self.rpm_distribution_api.list(base_path="rpm-with-modules-copy").count, 1) def test_migrate_copy_into_new_repo(self): """ @@ -466,16 +434,12 @@ def test_migrate_copy_into_new_repo(self): set_pulp2_snapshot(name="rpm_base_4repos_rerun_copy") self.task_rerun = self.run_migration(COPY_INTO_NEW_REPO_PLAN) - pulp2repo = self.pulp2repositories_api.list( - pulp2_repo_id="rpm-with-modules-copy" - ).results[0] + pulp2repo = self.pulp2repositories_api.list(pulp2_repo_id="rpm-with-modules-copy").results[ + 0 + ] version_href = pulp2repo.pulp3_repository_version - repo_advisories = self.rpm_content_apis["advisory"].list( - repository_version=version_href - ) + repo_advisories = self.rpm_content_apis["advisory"].list(repository_version=version_href) self.assertEqual(self.rpm_repo_versions_api.read(version_href).number, 1) self.assertEqual(repo_advisories.count, 6) - self.assertEqual( - self.rpm_distribution_api.list(base_path="rpm-with-modules-copy").count, 1 - ) + self.assertEqual(self.rpm_distribution_api.list(base_path="rpm-with-modules-copy").count, 1) diff --git a/pyproject.toml b/pyproject.toml index 89ce1d93..7afc6a31 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,22 @@ title_format = "{version} ({project_date})" template = "CHANGES/.TEMPLATE.rst" issue_format = "`#{issue} `__" +[tool.black] +line-length = 100 +target-version = ["py36", "py37"] +exclude = ''' +/( + \.eggs + | \.git + | \.venv + | _build + | build + | dist + | migrations + | docs +)/ +''' + [tool.check-manifest] ignore = [ ".bumpversion.cfg",