From 5c4c88c99c8cb3d8970ef0c687c582ce5816994e Mon Sep 17 00:00:00 2001 From: Fabricio Aguiar Date: Mon, 27 Jun 2022 21:04:09 +0100 Subject: [PATCH] Update CI [noissue] --- .ci/ansible/smash-config.json | 4 +- .ci/ansible/start_container.yaml | 2 +- .ci/assets/bindings/.gitkeep | 0 .github/template_gitref | 2 +- .github/workflows/ci.yml | 5 - .github/workflows/release.yml | 1 + .github/workflows/scripts/install.sh | 9 +- .../scripts/install_python_client.sh | 6 +- .github/workflows/scripts/script.sh | 104 +++++++----------- .../tests/functional/dynaconf_config.py | 8 +- pulp_2to3_migration/tests/functional/util.py | 10 +- 11 files changed, 66 insertions(+), 85 deletions(-) delete mode 100644 .ci/assets/bindings/.gitkeep diff --git a/.ci/ansible/smash-config.json b/.ci/ansible/smash-config.json index 0ea30eca..941122de 100644 --- a/.ci/ansible/smash-config.json +++ b/.ci/ansible/smash-config.json @@ -6,7 +6,7 @@ ], "selinux enabled": false, "version": "3", - "aiohttp_fixtures_origin": "172.18.0.1" + "aiohttp_fixtures_origin": "127.0.0.1" }, "hosts": [ { @@ -26,7 +26,7 @@ "pulp workers": {}, "redis": {}, "shell": { - "transport": "docker" + "transport": "local" } } } diff --git a/.ci/ansible/start_container.yaml b/.ci/ansible/start_container.yaml index 09da1355..e23a1880 100644 --- a/.ci/ansible/start_container.yaml +++ b/.ci/ansible/start_container.yaml @@ -41,7 +41,7 @@ image: "{{ item.image }}" auto_remove: true recreate: true - privileged: false + privileged: true networks: - name: pulp_ci_bridge aliases: "{{ item.name }}" diff --git a/.ci/assets/bindings/.gitkeep b/.ci/assets/bindings/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/.github/template_gitref b/.github/template_gitref index 6a9abd61..71be7abb 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-140-g5de156f +2021.08.26-146-gae643c8 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 42b98329..22a57157 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -145,11 +145,6 @@ jobs: run: .github/workflows/scripts/install_python_client.sh shell: bash - - name: Install Ruby client - if: ${{ env.TEST == 'bindings' }} - run: .github/workflows/scripts/install_ruby_client.sh - shell: bash - - name: Before Script run: .github/workflows/scripts/before_script.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bfe222b9..5104e828 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -340,6 +340,7 @@ jobs: set -euv export COMMIT_MSG=$(git log --format=%B --no-merges -1) export GH_ISSUES=$(echo $COMMIT_MSG | grep -o "GH Issues: .*" | awk '{print $3}') + pip install pygithub echo "GH Issues $GH_ISSUES" python .ci/scripts/update_github.py diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index fbb6bd88..4f07cba8 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -22,9 +22,6 @@ if [[ "$TEST" = "docs" || "$TEST" = "publish" ]]; then pip install -r doc_requirements.txt fi -pip install -e ../pulpcore -e ../pulp_file -e ../pulp_container -e ../pulp_rpm -e ../pulp_deb -pip install -r functest_requirements.txt - cd .ci/ansible/ TAG=ci_build @@ -76,6 +73,8 @@ plugins: source: pulp_rpm~=3.17.0 - name: pulp_deb source: pulp_deb~=2.18.0 + - name: pulp-smash + source: ./pulp-smash VARSYAML else cat >> vars/main.yaml << VARSYAML @@ -95,6 +94,8 @@ plugins: source: $PULP_DEB - name: pulpcore source: "${PULPCORE}" + - name: pulp-smash + source: ./pulp-smash VARSYAML fi @@ -105,6 +106,8 @@ services: volumes: - ./settings:/etc/pulp - ./ssh:/keys/ + - ~/.config:/root/.config + - ../../../pulp-openapi-generator:/root/pulp-openapi-generator VARSYAML cat >> vars/main.yaml << VARSYAML diff --git a/.github/workflows/scripts/install_python_client.sh b/.github/workflows/scripts/install_python_client.sh index 3ebaae0f..c2ced4a6 100755 --- a/.github/workflows/scripts/install_python_client.sh +++ b/.github/workflows/scripts/install_python_client.sh @@ -7,7 +7,7 @@ # # For more info visit https://github.com/pulp/plugin_template -set -euv +set -mveuo pipefail export PULP_URL="${PULP_URL:-https://pulp}" @@ -30,7 +30,7 @@ export response=$(curl --write-out %{http_code} --silent --output /dev/null http if [ "$response" == "200" ]; then echo "pulp-2to3-migration client $VERSION has already been released. Installing from PyPI." - pip install pulp-2to3-migration-client==$VERSION + docker exec pulp pip3 install pulp-2to3-migration-client==$VERSION mkdir -p dist tar cvf python-client.tar ./dist exit @@ -41,7 +41,7 @@ rm -rf pulp_2to3_migration-client ./generate.sh pulp_2to3_migration python $VERSION cd pulp_2to3_migration-client python setup.py sdist bdist_wheel --python-tag py3 -find . -name "*.whl" -exec pip install {} \; +find . -name "*.whl" -exec docker exec pulp pip3 install /root/pulp-openapi-generator/pulp_2to3_migration-client/{} \; tar cvf ../../pulp-2to3-migration/python-client.tar ./dist find ./docs/* -exec sed -i 's/Back to README/Back to HOME/g' {} \; diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index abd01452..513c2dbe 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -61,60 +61,40 @@ if [[ "$TEST" == "plugin-from-pypi" ]]; then git checkout ${COMPONENT_VERSION} -- pulp-2to3-migration/tests/ fi +echo "machine pulp +login admin +password password +" | cmd_stdin_prefix bash -c "cat > /root/.netrc" + +cat unittest_requirements.txt | cmd_stdin_prefix bash -c "cat > /tmp/unittest_requirements.txt" +cat functest_requirements.txt | cmd_stdin_prefix bash -c "cat > /tmp/functest_requirements.txt" +cmd_prefix pip3 install -r /tmp/unittest_requirements.txt +cmd_prefix pip3 install -r /tmp/functest_requirements.txt +cmd_prefix pip3 install --upgrade ../pulp-smash + cd ../pulp-openapi-generator +./generate.sh pulp_2to3_migration python +cmd_prefix pip3 install /root/pulp-openapi-generator/pulp_2to3_migration-client +sudo rm -rf ./pulp_2to3_migration-client ./generate.sh pulpcore python -pip install ./pulpcore-client -rm -rf ./pulpcore-client -if [[ "$TEST" = 'bindings' ]]; then - ./generate.sh pulpcore ruby 0 - cd pulpcore-client - gem build pulpcore_client.gemspec - gem install --both ./pulpcore_client-0.gem -fi +cmd_prefix pip3 install /root/pulp-openapi-generator/pulpcore-client +sudo rm -rf ./pulpcore-client ./generate.sh pulp_file python -pip install ./pulp_file-client -rm -rf ./pulp_file-client -if [[ "$TEST" = 'bindings' ]]; then - ./generate.sh pulp_file ruby 0 - cd pulp_file-client - gem build pulp_file_client.gemspec - gem install --both ./pulp_file_client-0.gem - cd .. -fi +cmd_prefix pip3 install /root/pulp-openapi-generator/pulp_file-client +sudo rm -rf ./pulp_file-client ./generate.sh pulp_container python -pip install ./pulp_container-client -rm -rf ./pulp_container-client -if [[ "$TEST" = 'bindings' ]]; then - ./generate.sh pulp_container ruby 0 - cd pulp_container-client - gem build pulp_container_client.gemspec - gem install --both ./pulp_container_client-0.gem - cd .. -fi +cmd_prefix pip3 install /root/pulp-openapi-generator/pulp_container-client +sudo rm -rf ./pulp_container-client ./generate.sh pulp_rpm python -pip install ./pulp_rpm-client -rm -rf ./pulp_rpm-client -if [[ "$TEST" = 'bindings' ]]; then - ./generate.sh pulp_rpm ruby 0 - cd pulp_rpm-client - gem build pulp_rpm_client.gemspec - gem install --both ./pulp_rpm_client-0.gem - cd .. -fi +cmd_prefix pip3 install /root/pulp-openapi-generator/pulp_rpm-client +sudo rm -rf ./pulp_rpm-client ./generate.sh pulp_deb python -pip install ./pulp_deb-client -rm -rf ./pulp_deb-client -if [[ "$TEST" = 'bindings' ]]; then - ./generate.sh pulp_deb ruby 0 - cd pulp_deb-client - gem build pulp_deb_client.gemspec - gem install --both ./pulp_deb_client-0.gem - cd .. -fi +cmd_prefix pip3 install /root/pulp-openapi-generator/pulp_deb-client +sudo rm -rf ./pulp_deb-client cd $REPO_ROOT -cat unittest_requirements.txt | cmd_stdin_prefix bash -c "cat > /tmp/unittest_requirements.txt" -cmd_prefix pip3 install -r /tmp/unittest_requirements.txt +CERTIFI=$(cmd_prefix python3 -c 'import certifi; print(certifi.where())') +cmd_prefix bash -c "cat /etc/pulp/certs/pulp_webserver.crt | tee -a "$CERTIFI" > /dev/null" # check for any uncommitted migrations echo "Checking for uncommitted migrations..." @@ -126,17 +106,11 @@ if [[ "$TEST" != "upgrade" ]]; then fi # Run functional tests -export PYTHONPATH=$REPO_ROOT/../pulp_file${PYTHONPATH:+:${PYTHONPATH}} -export PYTHONPATH=$REPO_ROOT/../pulp_container${PYTHONPATH:+:${PYTHONPATH}} -export PYTHONPATH=$REPO_ROOT/../pulp_rpm${PYTHONPATH:+:${PYTHONPATH}} -export PYTHONPATH=$REPO_ROOT/../pulp_deb${PYTHONPATH:+:${PYTHONPATH}} -export PYTHONPATH=$REPO_ROOT${PYTHONPATH:+:${PYTHONPATH}} - if [[ "$TEST" == "performance" ]]; then if [[ -z ${PERFORMANCE_TEST+x} ]]; then - pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulp_2to3_migration.tests.performance + cmd_prefix bash -c "pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulp_2to3_migration.tests.performance" else - pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulp_2to3_migration.tests.performance.test_$PERFORMANCE_TEST + cmd_prefix bash -c "pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulp_2to3_migration.tests.performance.test_$PERFORMANCE_TEST" fi exit fi @@ -145,22 +119,18 @@ if [ -f $FUNC_TEST_SCRIPT ]; then source $FUNC_TEST_SCRIPT else - if [[ "$GITHUB_WORKFLOW" == "2To3-Migration Nightly CI/CD" ]]; then - pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_2to3_migration.tests.functional -m parallel -n 8 --nightly - pytest -v -r sx --color=yes --pyargs pulp_2to3_migration.tests.functional -m "not parallel" --nightly + if [[ "$GITHUB_WORKFLOW" == "2To3-Migration Nightly CI/CD" ]] || [[ "${RELEASE_WORKFLOW:-false}" == "true" ]]; then + cmd_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_2to3_migration.tests.functional -m parallel -n 8 --nightly" + cmd_prefix bash -c "pytest -v -r sx --color=yes --pyargs pulp_2to3_migration.tests.functional -m 'not parallel' --nightly" + + + cmd_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulpcore.tests.functional -m 'from_pulpcore_for_all_plugins and parallel' -n 8 --nightly" + cmd_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulpcore.tests.functional -m 'from_pulpcore_for_all_plugins and not parallel' --nightly" - - pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulpcore.tests.functional -m "from_pulpcore_for_all_plugins and parallel" -n 8 --nightly - pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulpcore.tests.functional -m "from_pulpcore_for_all_plugins and not parallel" --nightly - else - pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_2to3_migration.tests.functional -m parallel -n 8 - pytest -v -r sx --color=yes --pyargs pulp_2to3_migration.tests.functional -m "not parallel" + cmd_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_2to3_migration.tests.functional -m parallel -n 8" + cmd_prefix bash -c "pytest -v -r sx --color=yes --pyargs pulp_2to3_migration.tests.functional -m 'not parallel'" - - pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulpcore.tests.functional -m "from_pulpcore_for_all_plugins and parallel" -n 8 - pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulpcore.tests.functional -m "from_pulpcore_for_all_plugins and not parallel" - fi fi diff --git a/pulp_2to3_migration/tests/functional/dynaconf_config.py b/pulp_2to3_migration/tests/functional/dynaconf_config.py index f5563297..4e1817db 100644 --- a/pulp_2to3_migration/tests/functional/dynaconf_config.py +++ b/pulp_2to3_migration/tests/functional/dynaconf_config.py @@ -1,3 +1,9 @@ from dynaconf import Dynaconf -settings = Dynaconf(settings_files=['func_test_settings.py', '/etc/pulp/settings.py']) +settings = Dynaconf( + settings_files=[ + 'func_test_settings.py', + '/pulp-2to3-migration/func_test_settings.py', + '/etc/pulp/settings.py' + ] +) diff --git a/pulp_2to3_migration/tests/functional/util.py b/pulp_2to3_migration/tests/functional/util.py index 6339808e..9c623fec 100644 --- a/pulp_2to3_migration/tests/functional/util.py +++ b/pulp_2to3_migration/tests/functional/util.py @@ -58,6 +58,12 @@ def set_pulp2_snapshot(name): # needs to be done locally otherwise auth is required because password is provided in the # cleartext. - cmd = ('mongo', 'pulp_database', '--eval', 'db.createUser({user:"ci_cd",pwd:"ci_cd", ' - 'roles:["readWrite"]});') + cmd = ( + "mongo", + "pulp_database", + "--eval", + 'db.createUser({user:"ci_cd",pwd:"ci_cd",roles:["readWrite"],passwordDigestor:"server"});', + "--host", + mongodb_host, + ) subprocess.run(cmd)