diff --git a/.ansible-lint b/.ansible-lint index 75ae44c14e4..68a403363e3 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -37,3 +37,5 @@ exclude_paths: - tests/files/custom_cni/cilium.yaml - venv - .github +mock_modules: + - gluster.gluster.gluster_volume diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000000..8e95b181971 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +docs/_sidebar.md linguist-generated=true diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index aa9c1813005..2ef2e3760da 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,4 +1,5 @@ --- +blank_issues_enabled: false contact_links: - name: Support Request url: https://kubernetes.slack.com/channels/kubespray diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 3707b690a6f..0458aa60bd0 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,4 +4,6 @@ updates: directory: "/" schedule: interval: "weekly" - labels: [ "dependencies" ] + labels: + - dependencies + - release-note-none diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 61cde2c9e41..0449c18c0b6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,23 +1,17 @@ --- stages: - build - - unit-tests + - test - deploy-part1 - - moderator - - deploy-part2 - - deploy-part3 - - deploy-special + - deploy-extended variables: - KUBESPRAY_VERSION: v2.24.1 + KUBESPRAY_VERSION: v2.26.0 FAILFASTCI_NAMESPACE: 'kargo-ci' GITLAB_REPOSITORY: 'kargo-ci/kubernetes-sigs-kubespray' ANSIBLE_FORCE_COLOR: "true" + ANSIBLE_STDOUT_CALLBACK: "debug" MAGIC: "ci check this" - TEST_ID: "$CI_PIPELINE_ID-$CI_JOB_ID" - CI_TEST_VARS: "./tests/files/${CI_JOB_NAME}.yml" - CI_TEST_REGISTRY_MIRROR: "./tests/common/_docker_hub_registry_mirror.yml" - CI_TEST_SETTING: "./tests/common/_kubespray_test_settings.yml" GS_ACCESS_KEY_ID: $GS_KEY GS_SECRET_ACCESS_KEY: $GS_SECRET CONTAINER_ENGINE: docker @@ -25,13 +19,11 @@ variables: GCE_PREEMPTIBLE: "false" ANSIBLE_KEEP_REMOTE_FILES: "1" ANSIBLE_CONFIG: ./tests/ansible.cfg - ANSIBLE_INVENTORY: ./inventory/sample/${CI_JOB_NAME}-${BUILD_NUMBER}.ini - IDEMPOT_CHECK: "false" RESET_CHECK: "false" REMOVE_NODE_CHECK: "false" UPGRADE_TEST: "false" MITOGEN_ENABLE: "false" - ANSIBLE_LOG_LEVEL: "-vv" + ANSIBLE_VERBOSITY: 2 RECOVER_CONTROL_PLANE_TEST: "false" RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[2:]:kube_control_plane[1:]" TERRAFORM_VERSION: 1.3.7 @@ -43,15 +35,26 @@ before_script: .job: &job tags: - - packet + - ffci image: $PIPELINE_IMAGE artifacts: when: always paths: - cluster-dump/ + needs: + - pipeline-image + +.job-moderated: + extends: .job + needs: + - pipeline-image + - ci-not-authorized + - check-galaxy-version # lint + - pre-commit # lint + - vagrant-validate # lint .testcases: &testcases - <<: *job + extends: .job-moderated retry: 1 interruptible: true before_script: @@ -61,23 +64,38 @@ before_script: script: - ./tests/scripts/testcases_run.sh after_script: - - chronic ./tests/scripts/testcases_cleanup.sh + - ./tests/scripts/testcases_cleanup.sh # For failfast, at least 1 job must be defined in .gitlab-ci.yml # Premoderated with manual actions -ci-authorized: - extends: .job - stage: moderator +ci-not-authorized: + stage: build + before_script: [] + after_script: [] + rules: + # LGTM or ok-to-test labels + - if: $PR_LABELS =~ /.*,(lgtm|approved|ok-to-test).*|^(lgtm|approved|ok-to-test).*/i + variables: + CI_OK_TO_TEST: '0' + when: always + - if: $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "trigger" + variables: + CI_OK_TO_TEST: '0' + - if: $CI_COMMIT_BRANCH == "master" + variables: + CI_OK_TO_TEST: '0' + - when: always + variables: + CI_OK_TO_TEST: '1' script: - - /bin/sh scripts/premoderator.sh - except: ['triggers', 'master'] - # Disable ci moderator - only: [] + - exit $CI_OK_TO_TEST + tags: + - ffci + needs: [] include: - .gitlab-ci/build.yml - .gitlab-ci/lint.yml - - .gitlab-ci/shellcheck.yml - .gitlab-ci/terraform.yml - .gitlab-ci/packet.yml - .gitlab-ci/vagrant.yml diff --git a/.gitlab-ci/build.yml b/.gitlab-ci/build.yml index 0f1824b5674..89cbd196c7b 100644 --- a/.gitlab-ci/build.yml +++ b/.gitlab-ci/build.yml @@ -1,40 +1,32 @@ --- -.build: +.build-container: + cache: + key: $CI_COMMIT_REF_SLUG + paths: + - image-cache + tags: + - ffci stage: build image: - name: moby/buildkit:rootless - entrypoint: [""] + name: gcr.io/kaniko-project/executor:debug + entrypoint: [''] variables: - BUILDKITD_FLAGS: --oci-worker-no-process-sandbox + TAG: $CI_COMMIT_SHORT_SHA + PROJECT_DIR: $CI_PROJECT_DIR + DOCKERFILE: Dockerfile + GODEBUG: "http2client=0" before_script: - - mkdir ~/.docker - - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > ~/.docker/config.json - -pipeline image: - extends: .build + - echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$(echo -n ${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD} | base64)\"}}}" > /kaniko/.docker/config.json script: - - | - buildctl-daemonless.sh build \ - --frontend=dockerfile.v0 \ - --local context=. \ - --local dockerfile=. \ - --opt filename=./pipeline.Dockerfile \ - --output type=image,name=$PIPELINE_IMAGE,push=true \ - --import-cache type=registry,ref=$CI_REGISTRY_IMAGE/pipeline:cache - rules: - - if: '$CI_COMMIT_REF_NAME != $CI_DEFAULT_BRANCH' + - /kaniko/executor --cache=true + --cache-dir=image-cache + --context $PROJECT_DIR + --dockerfile $PROJECT_DIR/$DOCKERFILE + --label 'git-branch'=$CI_COMMIT_REF_SLUG + --label 'git-tag=$CI_COMMIT_TAG' + --destination $PIPELINE_IMAGE -pipeline image and build cache: - extends: .build - script: - - | - buildctl-daemonless.sh build \ - --frontend=dockerfile.v0 \ - --local context=. \ - --local dockerfile=. \ - --opt filename=./pipeline.Dockerfile \ - --output type=image,name=$PIPELINE_IMAGE,push=true \ - --import-cache type=registry,ref=$CI_REGISTRY_IMAGE/pipeline:cache \ - --export-cache type=registry,ref=$CI_REGISTRY_IMAGE/pipeline:cache,mode=max - rules: - - if: '$CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH' +pipeline-image: + extends: .build-container + variables: + DOCKERFILE: pipeline.Dockerfile diff --git a/.gitlab-ci/lint.yml b/.gitlab-ci/lint.yml index 5e4b81abb0b..cf81c551752 100644 --- a/.gitlab-ci/lint.yml +++ b/.gitlab-ci/lint.yml @@ -1,118 +1,35 @@ --- -yamllint: - extends: .job - stage: unit-tests - tags: [light] +pre-commit: + stage: test + tags: + - ffci + image: 'ghcr.io/pre-commit-ci/runner-image@sha256:aaf2c7b38b22286f2d381c11673bec571c28f61dd086d11b43a1c9444a813cef' variables: - LANG: C.UTF-8 + PRE_COMMIT_HOME: /pre-commit-cache script: - - yamllint --strict . - except: ['triggers', 'master'] + - pre-commit run --all-files + cache: + key: pre-commit-all + paths: + - /pre-commit-cache + needs: [] vagrant-validate: extends: .job - stage: unit-tests - tags: [light] + stage: test + tags: [ffci] variables: VAGRANT_VERSION: 2.3.7 script: - - ./tests/scripts/vagrant-validate.sh - except: ['triggers', 'master'] - -ansible-lint: - extends: .job - stage: unit-tests - tags: [light] - script: - - ansible-lint -v - except: ['triggers', 'master'] - -jinja-syntax-check: - extends: .job - stage: unit-tests - tags: [light] - script: - - "find -name '*.j2' -exec tests/scripts/check-templates.py {} +" - except: ['triggers', 'master'] - -syntax-check: - extends: .job - stage: unit-tests - tags: [light] - variables: - ANSIBLE_INVENTORY: inventory/local-tests.cfg - ANSIBLE_REMOTE_USER: root - ANSIBLE_BECOME: "true" - ANSIBLE_BECOME_USER: root - ANSIBLE_VERBOSITY: "3" - script: - - ansible-playbook --syntax-check cluster.yml - - ansible-playbook --syntax-check playbooks/cluster.yml - - ansible-playbook --syntax-check upgrade-cluster.yml - - ansible-playbook --syntax-check playbooks/upgrade_cluster.yml - - ansible-playbook --syntax-check reset.yml - - ansible-playbook --syntax-check playbooks/reset.yml - - ansible-playbook --syntax-check extra_playbooks/upgrade-only-k8s.yml - except: ['triggers', 'master'] - -collection-build-install-sanity-check: - extends: .job - stage: unit-tests - tags: [light] - variables: - ANSIBLE_COLLECTIONS_PATH: "./ansible_collections" - script: - - ansible-galaxy collection build - - ansible-galaxy collection install kubernetes_sigs-kubespray-$(grep "^version:" galaxy.yml | awk '{print $2}').tar.gz - - ansible-galaxy collection list $(egrep -i '(name:\s+|namespace:\s+)' galaxy.yml | awk '{print $2}' | tr '\n' '.' | sed 's|\.$||g') | grep "^kubernetes_sigs.kubespray" - - test -f ansible_collections/kubernetes_sigs/kubespray/playbooks/cluster.yml - - test -f ansible_collections/kubernetes_sigs/kubespray/playbooks/reset.yml + - ./tests/scripts/vagrant-validate.sh except: ['triggers', 'master'] -tox-inventory-builder: - stage: unit-tests - tags: [light] - extends: .job - before_script: - - ./tests/scripts/rebase.sh - script: - - pip3 install tox - - cd contrib/inventory_builder && tox - except: ['triggers', 'master'] - -markdownlint: - stage: unit-tests - tags: [light] - image: node - before_script: - - npm install -g markdownlint-cli@0.22.0 - script: - - markdownlint $(find . -name '*.md' | grep -vF './.git') --ignore docs/_sidebar.md --ignore contrib/dind/README.md - -check-readme-versions: - stage: unit-tests - tags: [light] - image: python:3 - script: - - tests/scripts/check_readme_versions.sh +# TODO: convert to pre-commit hook check-galaxy-version: - stage: unit-tests - tags: [light] - image: python:3 - script: - - tests/scripts/check_galaxy_version.sh - -check-typo: - stage: unit-tests - tags: [light] - image: python:3 - script: - - tests/scripts/check_typo.sh - -ci-matrix: - stage: unit-tests - tags: [light] + needs: [] + stage: test + tags: [ffci] image: python:3 script: - - tests/scripts/md-table/test.sh + - tests/scripts/check_galaxy_version.sh diff --git a/.gitlab-ci/molecule.yml b/.gitlab-ci/molecule.yml index 94e5a5606fb..b37cd03ba3a 100644 --- a/.gitlab-ci/molecule.yml +++ b/.gitlab-ci/molecule.yml @@ -1,30 +1,50 @@ --- - .molecule: - tags: [c3.small.x86] + tags: [ffci-vm-med] only: [/^pr-.*$/] except: ['triggers'] - image: $PIPELINE_IMAGE + image: quay.io/kubespray/vm-kubespray-ci:v13 services: [] stage: deploy-part1 + needs: [] + # - ci-not-authorized + variables: + VAGRANT_DEFAULT_PROVIDER: "libvirt" + VAGRANT_HOME: "$CI_PROJECT_DIR/.vagrant.d" + PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" before_script: - - tests/scripts/rebase.sh - - ./tests/scripts/vagrant_clean.sh + - mkdir -p $VAGRANT_HOME + - groups + - python3 -m venv citest + - source citest/bin/activate + - vagrant plugin expunge --reinstall --force --no-tty + - vagrant plugin install vagrant-libvirt + - pip install --no-compile --no-cache-dir pip -U + - pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/requirements.txt + - pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/tests/requirements.txt + - ./tests/scripts/rebase.sh + - ./tests/scripts/vagrant_clean.sh script: - - ./tests/scripts/molecule_run.sh + - ./tests/scripts/molecule_run.sh after_script: - - chronic ./tests/scripts/molecule_logs.sh + - ./tests/scripts/molecule_logs.sh artifacts: when: always paths: - - molecule_logs/ + - molecule_logs/ + cache: + key: $CI_JOB_NAME_SLUG + paths: + - .vagrant.d/boxes + - .cache/pip + policy: pull-push # TODO: change to "pull" when not on main # CI template for periodic CI jobs # Enabled when PERIODIC_CI_ENABLED var is set .molecule_periodic: only: variables: - - $PERIODIC_CI_ENABLED + - $PERIODIC_CI_ENABLED allow_failure: true extends: .molecule @@ -34,50 +54,50 @@ molecule_full: molecule_no_container_engines: extends: .molecule script: - - ./tests/scripts/molecule_run.sh -e container-engine + - ./tests/scripts/molecule_run.sh -e container-engine when: on_success molecule_docker: extends: .molecule script: - - ./tests/scripts/molecule_run.sh -i container-engine/cri-dockerd + - ./tests/scripts/molecule_run.sh -i container-engine/cri-dockerd when: on_success molecule_containerd: extends: .molecule script: - - ./tests/scripts/molecule_run.sh -i container-engine/containerd + - ./tests/scripts/molecule_run.sh -i container-engine/containerd when: on_success molecule_cri-o: extends: .molecule - stage: deploy-part2 + stage: deploy-part1 script: - - ./tests/scripts/molecule_run.sh -i container-engine/cri-o + - ./tests/scripts/molecule_run.sh -i container-engine/cri-o allow_failure: true when: on_success -# Stage 3 container engines don't get as much attention so allow them to fail -molecule_kata: - extends: .molecule - stage: deploy-part3 - script: - - ./tests/scripts/molecule_run.sh -i container-engine/kata-containers - when: manual -# FIXME: this test is broken (perma-failing) +# # Stage 3 container engines don't get as much attention so allow them to fail +# molecule_kata: +# extends: .molecule +# stage: deploy-extended +# script: +# - ./tests/scripts/molecule_run.sh -i container-engine/kata-containers +# when: manual +# # FIXME: this test is broken (perma-failing) molecule_gvisor: extends: .molecule - stage: deploy-part3 + stage: deploy-extended script: - - ./tests/scripts/molecule_run.sh -i container-engine/gvisor + - ./tests/scripts/molecule_run.sh -i container-engine/gvisor when: manual # FIXME: this test is broken (perma-failing) molecule_youki: extends: .molecule - stage: deploy-part3 + stage: deploy-extended script: - - ./tests/scripts/molecule_run.sh -i container-engine/youki + - ./tests/scripts/molecule_run.sh -i container-engine/youki when: manual # FIXME: this test is broken (perma-failing) diff --git a/.gitlab-ci/packet.yml b/.gitlab-ci/packet.yml index 3023f721274..33abc301648 100644 --- a/.gitlab-ci/packet.yml +++ b/.gitlab-ci/packet.yml @@ -6,13 +6,55 @@ CI_PLATFORM: packet SSH_USER: kubespray tags: - - packet - except: [triggers] + - ffci + needs: + - pipeline-image + - ci-not-authorized # CI template for PRs .packet_pr: - only: [/^pr-.*$/] + stage: deploy-part1 + rules: + - if: $PR_LABELS =~ /.*ci-short.*/ + when: manual + allow_failure: true + - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/ + when: on_success + - when: manual + allow_failure: true + extends: .packet + + ## Uncomment this to have multiple stages + # needs: + # - packet_ubuntu20-calico-all-in-one + +.packet_pr_short: + stage: deploy-part1 extends: .packet + rules: + - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/ + when: on_success + - when: manual + allow_failure: true + +.packet_pr_manual: + extends: .packet_pr + stage: deploy-extended + rules: + - if: $PR_LABELS =~ /.*ci-full.*/ + when: on_success + # Else run as manual + - when: manual + allow_failure: true + +.packet_pr_extended: + extends: .packet_pr + stage: deploy-extended + rules: + - if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/ + when: on_success + - when: manual + allow_failure: true # CI template for periodic CI jobs # Enabled when PERIODIC_CI_ENABLED var is set @@ -23,315 +65,182 @@ allow_failure: true extends: .packet -packet_cleanup_old: - stage: deploy-part1 - extends: .packet_periodic - script: - - cd tests - - make cleanup-packet - after_script: [] - # The ubuntu20-calico-all-in-one jobs are meant as early stages to prevent running the full CI if something is horribly broken packet_ubuntu20-calico-all-in-one: stage: deploy-part1 - extends: .packet_pr - when: on_success + extends: .packet_pr_short variables: RESET_CHECK: "true" # ### PR JOBS PART2 -packet_ubuntu20-all-in-one-docker: - stage: deploy-part2 - extends: .packet_pr - when: on_success - -packet_ubuntu20-calico-all-in-one-hardening: - stage: deploy-part2 - extends: .packet_pr - when: on_success - -packet_ubuntu22-all-in-one-docker: - stage: deploy-part2 - extends: .packet_pr - when: on_success +packet_ubuntu20-crio: + extends: .packet_pr_manual packet_ubuntu22-calico-all-in-one: - stage: deploy-part2 extends: .packet_pr - when: on_success -packet_ubuntu22-calico-etcd-datastore: - stage: deploy-part2 +packet_ubuntu22-calico-all-in-one-upgrade: extends: .packet_pr - when: on_success + variables: + UPGRADE_TEST: graceful -packet_centos7-flannel-addons-ha: +packet_ubuntu24-calico-etcd-datastore: extends: .packet_pr - stage: deploy-part2 - when: on_success packet_almalinux8-crio: extends: .packet_pr - stage: deploy-part2 - when: on_success - allow_failure: true -packet_ubuntu20-crio: +packet_almalinux8-kube-ovn: extends: .packet_pr - stage: deploy-part2 - when: manual -packet_fedora37-crio: +packet_debian11-calico-collection: extends: .packet_pr - stage: deploy-part2 - when: manual -packet_ubuntu20-flannel-ha: - stage: deploy-part2 +packet_debian11-macvlan: extends: .packet_pr - when: manual -packet_debian10-cilium-svc-proxy: - stage: deploy-part2 - extends: .packet_periodic - when: on_success - -packet_debian10-calico: - stage: deploy-part2 +packet_debian12-cilium: extends: .packet_pr - when: on_success -packet_debian10-docker: - stage: deploy-part2 +packet_rockylinux8-calico: extends: .packet_pr - when: on_success -packet_debian11-calico: - stage: deploy-part2 +packet_rockylinux9-cilium: extends: .packet_pr - when: on_success + variables: + RESET_CHECK: "true" -packet_debian11-docker: - stage: deploy-part2 +packet_amazon-linux-2-all-in-one: extends: .packet_pr - when: on_success -packet_debian12-calico: - stage: deploy-part2 +packet_opensuse-docker-cilium: extends: .packet_pr - when: on_success -packet_debian12-docker: - stage: deploy-part2 +packet_ubuntu20-cilium-sep: extends: .packet_pr - when: on_success - -packet_debian12-cilium: - stage: deploy-part2 - extends: .packet_periodic - when: on_success -packet_centos7-calico-ha-once-localhost: - stage: deploy-part2 +packet_openeuler24-calico: extends: .packet_pr - when: on_success - variables: - # This will instruct Docker not to start over TLS. - DOCKER_TLS_CERTDIR: "" - services: - - docker:19.03.9-dind -packet_almalinux8-kube-ovn: - stage: deploy-part2 +packet_ubuntu20-calico-all-in-one-hardening: extends: .packet_pr - when: on_success -packet_almalinux8-calico: - stage: deploy-part2 - extends: .packet_pr - when: on_success +## Extended +packet_debian11-docker: + extends: .packet_pr_extended -packet_rockylinux8-calico: - stage: deploy-part2 - extends: .packet_pr - when: on_success +packet_debian12-docker: + extends: .packet_pr_extended -packet_rockylinux9-calico: - stage: deploy-part2 - extends: .packet_pr - when: on_success +packet_debian12-calico: + extends: .packet_pr_extended -packet_rockylinux9-cilium: - stage: deploy-part2 - extends: .packet_pr - when: on_success +packet_almalinux8-calico-remove-node: + extends: .packet_pr_extended variables: - RESET_CHECK: "true" - -packet_almalinux8-docker: - stage: deploy-part2 - extends: .packet_pr - when: on_success - -packet_amazon-linux-2-all-in-one: - stage: deploy-part2 - extends: .packet_pr - when: on_success + REMOVE_NODE_CHECK: "true" + REMOVE_NODE_NAME: "instance-3" -packet_fedora38-docker-weave: - stage: deploy-part2 - extends: .packet_pr - when: on_success - allow_failure: true +packet_rockylinux9-calico: + extends: .packet_pr_extended -packet_opensuse-docker-cilium: - stage: deploy-part2 - extends: .packet_pr - when: on_success +packet_almalinux8-calico: + extends: .packet_pr_extended -# ### MANUAL JOBS +packet_almalinux8-docker: + extends: .packet_pr_extended -packet_ubuntu20-docker-weave-sep: - stage: deploy-part2 - extends: .packet_pr - when: manual +packet_ubuntu24-calico-all-in-one: + extends: .packet_pr_extended -packet_ubuntu20-cilium-sep: - stage: deploy-special - extends: .packet_pr - when: manual +packet_ubuntu20-calico-etcd-kubeadm: + extends: .packet_pr_extended -packet_ubuntu20-flannel-ha-once: - stage: deploy-part2 - extends: .packet_pr - when: manual +packet_ubuntu24-all-in-one-docker: + extends: .packet_pr_extended -# Calico HA eBPF -packet_almalinux8-calico-ha-ebpf: - stage: deploy-part2 - extends: .packet_pr - when: manual +packet_ubuntu22-all-in-one-docker: + extends: .packet_pr_extended -packet_debian10-macvlan: - stage: deploy-part2 - extends: .packet_pr - when: manual +# ### MANUAL JOBS +packet_fedora39-crio: + extends: .packet_pr_manual -packet_centos7-calico-ha: - stage: deploy-part2 - extends: .packet_pr - when: manual +packet_ubuntu20-flannel-ha: + extends: .packet_pr_manual -packet_centos7-multus-calico: - stage: deploy-part2 - extends: .packet_pr - when: manual +packet_ubuntu20-all-in-one-docker: + extends: .packet_pr_manual -packet_fedora38-docker-calico: - stage: deploy-part2 - extends: .packet_periodic - when: on_success - variables: - RESET_CHECK: "true" +packet_ubuntu20-flannel-ha-once: + extends: .packet_pr_manual -packet_fedora37-calico-selinux: - stage: deploy-part2 - extends: .packet_periodic - when: on_success +packet_fedora39-calico-swap-selinux: + extends: .packet_pr_manual -packet_fedora37-calico-swap-selinux: - stage: deploy-part2 - extends: .packet_pr - when: manual +packet_almalinux8-calico-ha-ebpf: + extends: .packet_pr_manual packet_almalinux8-calico-nodelocaldns-secondary: - stage: deploy-part2 - extends: .packet_pr - when: manual - -packet_fedora38-kube-ovn: - stage: deploy-part2 - extends: .packet_periodic - when: on_success + extends: .packet_pr_manual packet_debian11-custom-cni: - stage: deploy-part2 - extends: .packet_pr - when: manual + extends: .packet_pr_manual packet_debian11-kubelet-csr-approver: - stage: deploy-part2 - extends: .packet_pr - when: manual + extends: .packet_pr_manual packet_debian12-custom-cni-helm: - stage: deploy-part2 - extends: .packet_pr - when: manual + extends: .packet_pr_manual -# ### PR JOBS PART3 -# Long jobs (45min+) +packet_ubuntu20-calico-ha-wireguard: + extends: .packet_pr_manual -packet_centos7-weave-upgrade-ha: - stage: deploy-part3 +# PERIODIC +packet_fedora40-docker-calico: + stage: deploy-extended extends: .packet_periodic - when: on_success variables: - UPGRADE_TEST: basic + RESET_CHECK: "true" + +packet_fedora39-calico-selinux: + stage: deploy-extended + extends: .packet_periodic packet_ubuntu20-calico-etcd-kubeadm-upgrade-ha: - stage: deploy-part3 + stage: deploy-extended extends: .packet_periodic - when: on_success variables: UPGRADE_TEST: basic -# Calico HA Wireguard -packet_ubuntu20-calico-ha-wireguard: - stage: deploy-part2 - extends: .packet_pr - when: manual - -packet_debian11-calico-upgrade: - stage: deploy-part3 - extends: .packet_pr - when: on_success - variables: - UPGRADE_TEST: graceful - -packet_almalinux8-calico-remove-node: - stage: deploy-part3 - extends: .packet_pr - when: on_success - variables: - REMOVE_NODE_CHECK: "true" - REMOVE_NODE_NAME: "instance-3" - -packet_ubuntu20-calico-etcd-kubeadm: - stage: deploy-part3 - extends: .packet_pr - when: on_success packet_debian11-calico-upgrade-once: - stage: deploy-part3 + stage: deploy-extended extends: .packet_periodic - when: on_success variables: UPGRADE_TEST: graceful packet_ubuntu20-calico-ha-recover: - stage: deploy-part3 + stage: deploy-extended extends: .packet_periodic - when: on_success variables: RECOVER_CONTROL_PLANE_TEST: "true" RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[2:]:kube_control_plane[1:]" packet_ubuntu20-calico-ha-recover-noquorum: - stage: deploy-part3 + stage: deploy-extended extends: .packet_periodic - when: on_success variables: RECOVER_CONTROL_PLANE_TEST: "true" RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[1:]:kube_control_plane[1:]" + +packet_debian11-calico-upgrade: + stage: deploy-extended + extends: .packet_periodic + variables: + UPGRADE_TEST: graceful + +packet_debian12-cilium-svc-proxy: + stage: deploy-extended + extends: .packet_periodic diff --git a/.gitlab-ci/pre-commit-dynamic-stub.yml b/.gitlab-ci/pre-commit-dynamic-stub.yml new file mode 100644 index 00000000000..f91b5e8103c --- /dev/null +++ b/.gitlab-ci/pre-commit-dynamic-stub.yml @@ -0,0 +1,17 @@ +--- +# stub pipeline for dynamic generation +pre-commit: + tags: + - light + image: 'ghcr.io/pre-commit-ci/runner-image@sha256:aaf2c7b38b22286f2d381c11673bec571c28f61dd086d11b43a1c9444a813cef' + variables: + PRE_COMMIT_HOME: /pre-commit-cache + script: + - pre-commit run --all-files + cache: + key: pre-commit-$HOOK_ID + paths: + - /pre-commit-cache + parallel: + matrix: + - HOOK_ID: diff --git a/.gitlab-ci/shellcheck.yml b/.gitlab-ci/shellcheck.yml deleted file mode 100644 index 307e121c5c4..00000000000 --- a/.gitlab-ci/shellcheck.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -shellcheck: - extends: .job - stage: unit-tests - tags: [light] - variables: - SHELLCHECK_VERSION: v0.7.1 - before_script: - - ./tests/scripts/rebase.sh - - curl --silent --location "https://github.com/koalaman/shellcheck/releases/download/"${SHELLCHECK_VERSION}"/shellcheck-"${SHELLCHECK_VERSION}".linux.x86_64.tar.xz" | tar -xJv - - cp shellcheck-"${SHELLCHECK_VERSION}"/shellcheck /usr/bin/ - - shellcheck --version - script: - # Run shellcheck for all *.sh - - find . -name '*.sh' -not -path './.git/*' | xargs shellcheck --severity error - except: ['triggers', 'master'] diff --git a/.gitlab-ci/terraform.yml b/.gitlab-ci/terraform.yml index 32991776c98..22352281511 100644 --- a/.gitlab-ci/terraform.yml +++ b/.gitlab-ci/terraform.yml @@ -2,6 +2,10 @@ # Tests for contrib/terraform/ .terraform_install: extends: .job + needs: + - ci-not-authorized + - pipeline-image + stage: deploy-part1 before_script: - update-alternatives --install /usr/bin/python python /usr/bin/python3 1 - ./tests/scripts/rebase.sh @@ -24,17 +28,19 @@ .terraform_validate: extends: .terraform_install - stage: unit-tests - tags: [light] + tags: [ffci] only: ['master', /^pr-.*$/] script: - terraform -chdir="contrib/terraform/$PROVIDER" validate - terraform -chdir="contrib/terraform/$PROVIDER" fmt -check -diff + stage: test + needs: + - pipeline-image .terraform_apply: extends: .terraform_install - tags: [light] - stage: deploy-part3 + tags: [ffci] + stage: deploy-extended when: manual only: [/^pr-.*$/] artifacts: @@ -51,7 +57,7 @@ - tests/scripts/testcases_run.sh after_script: # Cleanup regardless of exit code - - chronic ./tests/scripts/testcases_cleanup.sh + - ./tests/scripts/testcases_cleanup.sh tf-validate-openstack: extends: .terraform_validate @@ -146,8 +152,7 @@ tf-validate-nifcloud: TF_VAR_router_id: "ab95917c-41fb-4881-b507-3a6dfe9403df" tf-elastx_cleanup: - stage: unit-tests - tags: [light] + tags: [ffci] image: python variables: <<: *elastx_variables @@ -155,10 +160,11 @@ tf-elastx_cleanup: - pip install -r scripts/openstack-cleanup/requirements.txt script: - ./scripts/openstack-cleanup/main.py + allow_failure: true tf-elastx_ubuntu20-calico: extends: .terraform_apply - stage: deploy-part3 + stage: deploy-part1 when: on_success allow_failure: true variables: diff --git a/.gitlab-ci/vagrant.yml b/.gitlab-ci/vagrant.yml index a02e3983566..891011020ce 100644 --- a/.gitlab-ci/vagrant.yml +++ b/.gitlab-ci/vagrant.yml @@ -1,64 +1,71 @@ --- - .vagrant: extends: .testcases + needs: + - ci-not-authorized variables: CI_PLATFORM: "vagrant" SSH_USER: "vagrant" VAGRANT_DEFAULT_PROVIDER: "libvirt" KUBESPRAY_VAGRANT_CONFIG: tests/files/${CI_JOB_NAME}.rb - tags: [c3.small.x86] - only: [/^pr-.*$/] - except: ['triggers'] - image: $PIPELINE_IMAGE + DOCKER_NAME: vagrant + VAGRANT_ANSIBLE_TAGS: facts + VAGRANT_HOME: "$CI_PROJECT_DIR/.vagrant.d" + PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" + tags: [ffci-vm-large] + # only: [/^pr-.*$/] + # except: ['triggers'] + image: quay.io/kubespray/vm-kubespray-ci:v13 services: [] before_script: + - echo $USER + - python3 -m venv citest + - source citest/bin/activate + - vagrant plugin expunge --reinstall --force --no-tty + - vagrant plugin install vagrant-libvirt + - pip install --no-compile --no-cache-dir pip -U + - pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/requirements.txt + - pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/tests/requirements.txt - ./tests/scripts/vagrant_clean.sh script: - ./tests/scripts/testcases_run.sh - after_script: - - chronic ./tests/scripts/testcases_cleanup.sh + cache: + key: $CI_JOB_NAME_SLUG + paths: + - .vagrant.d/boxes + - .cache/pip + policy: pull-push # TODO: change to "pull" when not on main vagrant_ubuntu20-calico-dual-stack: - stage: deploy-part2 + stage: deploy-extended extends: .vagrant when: manual # FIXME: this test if broken (perma-failing) -vagrant_ubuntu20-weave-medium: - stage: deploy-part2 - extends: .vagrant - when: manual - vagrant_ubuntu20-flannel: - stage: deploy-part2 + stage: deploy-part1 extends: .vagrant when: on_success allow_failure: false vagrant_ubuntu20-flannel-collection: - stage: deploy-part2 + stage: deploy-extended extends: .vagrant - when: on_success + when: manual vagrant_ubuntu20-kube-router-sep: - stage: deploy-part2 + stage: deploy-extended extends: .vagrant when: manual # Service proxy test fails connectivity testing vagrant_ubuntu20-kube-router-svc-proxy: - stage: deploy-part2 + stage: deploy-extended extends: .vagrant when: manual -vagrant_fedora37-kube-router: - stage: deploy-part2 +vagrant_fedora39-kube-router: + stage: deploy-extended extends: .vagrant when: manual # FIXME: this test if broken (perma-failing) - -vagrant_centos7-kube-router: - stage: deploy-part2 - extends: .vagrant - when: manual diff --git a/.markdownlint.yaml b/.markdownlint.yaml deleted file mode 100644 index 8ece4c7613e..00000000000 --- a/.markdownlint.yaml +++ /dev/null @@ -1,3 +0,0 @@ ---- -MD013: false -MD029: false diff --git a/.md_style.rb b/.md_style.rb new file mode 100644 index 00000000000..73adf8ae47e --- /dev/null +++ b/.md_style.rb @@ -0,0 +1,4 @@ +all +exclude_rule 'MD013' +exclude_rule 'MD029' +rule 'MD007', :indent => 2 diff --git a/.mdlrc b/.mdlrc new file mode 100644 index 00000000000..8ca55a8cee5 --- /dev/null +++ b/.mdlrc @@ -0,0 +1 @@ +style "#{File.dirname(__FILE__)}/.md_style.rb" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c2380522a33..d730535038c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,8 +1,7 @@ --- repos: - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.4.0 + rev: v5.0.0 hooks: - id: check-added-large-files - id: check-case-conflict @@ -16,59 +15,72 @@ repos: - id: trailing-whitespace - repo: https://github.com/adrienverge/yamllint.git - rev: v1.27.1 + rev: v1.35.1 hooks: - id: yamllint args: [--strict] - repo: https://github.com/markdownlint/markdownlint - rev: v0.11.0 + rev: v0.12.0 hooks: - id: markdownlint - args: [ -r, "~MD013,~MD029" ] - exclude: "^.git" + exclude: "^.github|(^docs/_sidebar\\.md$)" - - repo: https://github.com/jumanjihouse/pre-commit-hooks - rev: 3.0.0 + - repo: https://github.com/shellcheck-py/shellcheck-py + rev: v0.10.0.1 hooks: - id: shellcheck - args: [ --severity, "error" ] + args: ["--severity=error"] exclude: "^.git" files: "\\.sh$" - - repo: local + - repo: https://github.com/ansible/ansible-lint + rev: v24.12.2 hooks: - id: ansible-lint - name: ansible-lint - entry: ansible-lint -v - language: python - pass_filenames: false additional_dependencies: - - .[community] + - jmespath==1.0.1 + - netaddr==1.3.0 + - distlib - - id: ansible-syntax-check - name: ansible-syntax-check - entry: env ANSIBLE_INVENTORY=inventory/local-tests.cfg ANSIBLE_REMOTE_USER=root ANSIBLE_BECOME="true" ANSIBLE_BECOME_USER=root ANSIBLE_VERBOSITY="3" ansible-playbook --syntax-check - language: python - files: "^cluster.yml|^upgrade-cluster.yml|^reset.yml|^extra_playbooks/upgrade-only-k8s.yml" - - - id: tox-inventory-builder - name: tox-inventory-builder - entry: bash -c "cd contrib/inventory_builder && tox" - language: python - pass_filenames: false + - repo: https://github.com/golangci/misspell + rev: v0.6.0 + hooks: + - id: misspell + exclude: "OWNERS_ALIASES$" + - repo: local + hooks: - id: check-readme-versions name: check-readme-versions entry: tests/scripts/check_readme_versions.sh language: script pass_filenames: false + - id: collection-build-install + name: Build and install kubernetes-sigs.kubespray Ansible collection + language: python + additional_dependencies: + - ansible-core>=2.16.4 + - distlib + entry: tests/scripts/collection-build-install.sh + pass_filenames: false + + - id: generate-docs-sidebar + name: generate-docs-sidebar + entry: scripts/gen_docs_sidebar.sh + language: script + pass_filenames: false + - id: ci-matrix name: ci-matrix - entry: tests/scripts/md-table/test.sh - language: script + entry: tests/scripts/md-table/main.py + language: python pass_filenames: false + additional_dependencies: + - jinja2 + - pathlib + - pyaml - id: jinja-syntax-check name: jinja-syntax-check @@ -77,4 +89,4 @@ repos: types: - jinja additional_dependencies: - - Jinja2 + - jinja2 diff --git a/.yamllint b/.yamllint index aa14324a9f8..56786e0a988 100644 --- a/.yamllint +++ b/.yamllint @@ -6,7 +6,7 @@ ignore: | .github/ # Generated file tests/files/custom_cni/cilium.yaml - +# https://ansible.readthedocs.io/projects/lint/rules/yaml/ rules: braces: min-spaces-inside: 0 @@ -14,9 +14,15 @@ rules: brackets: min-spaces-inside: 0 max-spaces-inside: 1 + comments: + min-spaces-from-content: 1 + # https://github.com/adrienverge/yamllint/issues/384 + comments-indentation: false indentation: spaces: 2 indent-sequences: consistent line-length: disable new-line-at-end-of-file: disable - truthy: disable + octal-values: + forbid-implicit-octal: true # yamllint defaults to false + forbid-explicit-octal: true # yamllint defaults to false diff --git a/Makefile b/Makefile deleted file mode 100644 index 793e763dc12..00000000000 --- a/Makefile +++ /dev/null @@ -1,7 +0,0 @@ -mitogen: - @echo Mitogen support is deprecated. - @echo Please run the following command manually: - @echo ansible-playbook -c local mitogen.yml -vv -clean: - rm -rf dist/ - rm *.retry diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES index 19bf120f704..5dd20051527 100644 --- a/OWNERS_ALIASES +++ b/OWNERS_ALIASES @@ -6,15 +6,17 @@ aliases: - mzaian - oomichi - yankay + - ant31 + - vannten kubespray-reviewers: - cyclinder - erikjiang - mrfreezeex - mzaian + - tico88612 - vannten - yankay kubespray-emeritus_approvers: - - ant31 - atoms - chadswen - luckysb diff --git a/README.md b/README.md index a502e4c26ba..a3124384123 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ If you have questions, check the documentation at [kubespray.io](https://kubespray.io) and join us on the [kubernetes slack](https://kubernetes.slack.com), channel **\#kubespray**. You can get your invite [here](http://slack.k8s.io/) -- Can be deployed on **[AWS](docs/aws.md), GCE, [Azure](docs/azure.md), [OpenStack](docs/openstack.md), [vSphere](docs/vsphere.md), [Equinix Metal](docs/equinix-metal.md) (bare metal), Oracle Cloud Infrastructure (Experimental), or Baremetal** +- Can be deployed on **[AWS](docs/cloud_providers/aws.md), GCE, [Azure](docs/cloud_providers/azure.md), [OpenStack](docs/cloud_controllers/openstack.md), [vSphere](docs/cloud_controllers/vsphere.md), [Equinix Metal](docs/cloud_providers/equinix-metal.md) (bare metal), Oracle Cloud Infrastructure (Experimental), or Baremetal** - **Highly available** cluster - **Composable** (Choice of the network plugin for instance) - Supports most popular **Linux distributions** @@ -19,74 +19,11 @@ Below are several ways to use Kubespray to deploy a Kubernetes cluster. #### Usage -Install Ansible according to [Ansible installation guide](/docs/ansible.md#installing-ansible) -then run the following steps: - -```ShellSession -# Copy ``inventory/sample`` as ``inventory/mycluster`` -cp -rfp inventory/sample inventory/mycluster - -# Update Ansible inventory file with inventory builder -declare -a IPS=(10.10.1.3 10.10.1.4 10.10.1.5) -CONFIG_FILE=inventory/mycluster/hosts.yaml python3 contrib/inventory_builder/inventory.py ${IPS[@]} - -# Review and change parameters under ``inventory/mycluster/group_vars`` -cat inventory/mycluster/group_vars/all/all.yml -cat inventory/mycluster/group_vars/k8s_cluster/k8s-cluster.yml - -# Clean up old Kubernetes cluster with Ansible Playbook - run the playbook as root -# The option `--become` is required, as for example cleaning up SSL keys in /etc/, -# uninstalling old packages and interacting with various systemd daemons. -# Without --become the playbook will fail to run! -# And be mind it will remove the current kubernetes cluster (if it's running)! -ansible-playbook -i inventory/mycluster/hosts.yaml --become --become-user=root reset.yml - -# Deploy Kubespray with Ansible Playbook - run the playbook as root -# The option `--become` is required, as for example writing SSL keys in /etc/, -# installing packages and interacting with various systemd daemons. -# Without --become the playbook will fail to run! -ansible-playbook -i inventory/mycluster/hosts.yaml --become --become-user=root cluster.yml -``` - -Note: When Ansible is already installed via system packages on the control node, -Python packages installed via `sudo pip install -r requirements.txt` will go to -a different directory tree (e.g. `/usr/local/lib/python2.7/dist-packages` on -Ubuntu) from Ansible's (e.g. `/usr/lib/python2.7/dist-packages/ansible` still on -Ubuntu). As a consequence, the `ansible-playbook` command will fail with: - -```raw -ERROR! no action detected in task. This often indicates a misspelled module name, or incorrect module path. -``` - -This likely indicates that a task depends on a module present in ``requirements.txt``. - -One way of addressing this is to uninstall the system Ansible package then -reinstall Ansible via ``pip``, but this not always possible and one must -take care regarding package versions. -A workaround consists of setting the `ANSIBLE_LIBRARY` -and `ANSIBLE_MODULE_UTILS` environment variables respectively to -the `ansible/modules` and `ansible/module_utils` subdirectories of the ``pip`` -installation location, which is the ``Location`` shown by running -`pip show [package]` before executing `ansible-playbook`. - -A simple way to ensure you get all the correct version of Ansible is to use -the [pre-built docker image from Quay](https://quay.io/repository/kubespray/kubespray?tab=tags). -You will then need to use [bind mounts](https://docs.docker.com/storage/bind-mounts/) -to access the inventory and SSH key in the container, like this: - -```ShellSession -git checkout v2.24.1 -docker pull quay.io/kubespray/kubespray:v2.24.1 -docker run --rm -it --mount type=bind,source="$(pwd)"/inventory/sample,dst=/inventory \ - --mount type=bind,source="${HOME}"/.ssh/id_rsa,dst=/root/.ssh/id_rsa \ - quay.io/kubespray/kubespray:v2.24.1 bash -# Inside the container you may now run the kubespray playbooks: -ansible-playbook -i /inventory/inventory.ini --private-key /root/.ssh/id_rsa cluster.yml -``` +See [Getting started](/docs/getting_started/getting-started.md) #### Collection -See [here](docs/ansible_collection.md) if you wish to use this repository as an Ansible collection +See [here](docs/ansible/ansible_collection.md) if you wish to use this repository as an Ansible collection ### Vagrant @@ -99,7 +36,7 @@ python -V && pip -V If this returns the version of the software, you're good to go. If not, download and install Python from here -Install Ansible according to [Ansible installation guide](/docs/ansible.md#installing-ansible) +Install Ansible according to [Ansible installation guide](/docs/ansible/ansible.md#installing-ansible) then run the following step: ```ShellSession @@ -109,79 +46,76 @@ vagrant up ## Documents - [Requirements](#requirements) -- [Kubespray vs ...](docs/comparisons.md) -- [Getting started](docs/getting-started.md) -- [Setting up your first cluster](docs/setting-up-your-first-cluster.md) -- [Ansible inventory and tags](docs/ansible.md) -- [Integration with existing ansible repo](docs/integration.md) -- [Deployment data variables](docs/vars.md) -- [DNS stack](docs/dns-stack.md) -- [HA mode](docs/ha-mode.md) +- [Kubespray vs ...](docs/getting_started/comparisons.md) +- [Getting started](docs/getting_started/getting-started.md) +- [Setting up your first cluster](docs/getting_started/setting-up-your-first-cluster.md) +- [Ansible inventory and tags](docs/ansible/ansible.md) +- [Integration with existing ansible repo](docs/operations/integration.md) +- [Deployment data variables](docs/ansible/vars.md) +- [DNS stack](docs/advanced/dns-stack.md) +- [HA mode](docs/operations/ha-mode.md) - [Network plugins](#network-plugins) -- [Vagrant install](docs/vagrant.md) -- [Flatcar Container Linux bootstrap](docs/flatcar.md) -- [Fedora CoreOS bootstrap](docs/fcos.md) -- [openSUSE setup](docs/opensuse.md) -- [Downloaded artifacts](docs/downloads.md) -- [Cloud providers](docs/cloud.md) -- [OpenStack](docs/openstack.md) -- [AWS](docs/aws.md) -- [Azure](docs/azure.md) -- [vSphere](docs/vsphere.md) -- [Equinix Metal](docs/equinix-metal.md) -- [Large deployments](docs/large-deployments.md) -- [Adding/replacing a node](docs/nodes.md) -- [Upgrades basics](docs/upgrades.md) -- [Air-Gap installation](docs/offline-environment.md) -- [NTP](docs/ntp.md) -- [Hardening](docs/hardening.md) -- [Mirror](docs/mirror.md) -- [Roadmap](docs/roadmap.md) +- [Vagrant install](docs/developers/vagrant.md) +- [Flatcar Container Linux bootstrap](docs/operating_systems/flatcar.md) +- [Fedora CoreOS bootstrap](docs/operating_systems/fcos.md) +- [openSUSE setup](docs/operating_systems/opensuse.md) +- [Downloaded artifacts](docs/advanced/downloads.md) +- [Equinix Metal](docs/cloud_providers/equinix-metal.md) +- [OpenStack](docs/cloud_controllers/openstack.md) +- [vSphere](docs/cloud_controllers/vsphere.md) +- [Large deployments](docs/operations/large-deployments.md) +- [Adding/replacing a node](docs/operations/nodes.md) +- [Upgrades basics](docs/operations/upgrades.md) +- [Air-Gap installation](docs/operations/offline-environment.md) +- [NTP](docs/advanced/ntp.md) +- [Hardening](docs/operations/hardening.md) +- [Mirror](docs/operations/mirror.md) +- [Roadmap](docs/roadmap/roadmap.md) ## Supported Linux Distributions - **Flatcar Container Linux by Kinvolk** -- **Debian** Bookworm, Bullseye, Buster -- **Ubuntu** 20.04, 22.04 -- **CentOS/RHEL** 7, [8, 9](docs/centos.md#centos-8) -- **Fedora** 37, 38 -- **Fedora CoreOS** (see [fcos Note](docs/fcos.md)) +- **Debian** Bookworm, Bullseye +- **Ubuntu** 20.04, 22.04, 24.04 +- **CentOS/RHEL** [8, 9](docs/operating_systems/centos.md#centos-8) +- **Fedora** 39, 40 +- **Fedora CoreOS** (see [fcos Note](docs/operating_systems/fcos.md)) - **openSUSE** Leap 15.x/Tumbleweed -- **Oracle Linux** 7, [8, 9](docs/centos.md#centos-8) -- **Alma Linux** [8, 9](docs/centos.md#centos-8) -- **Rocky Linux** [8, 9](docs/centos.md#centos-8) -- **Kylin Linux Advanced Server V10** (experimental: see [kylin linux notes](docs/kylinlinux.md)) -- **Amazon Linux 2** (experimental: see [amazon linux notes](docs/amazonlinux.md)) -- **UOS Linux** (experimental: see [uos linux notes](docs/uoslinux.md)) -- **openEuler** (experimental: see [openEuler notes](docs/openeuler.md)) +- **Oracle Linux** [8, 9](docs/operating_systems/centos.md#centos-8) +- **Alma Linux** [8, 9](docs/operating_systems/centos.md#centos-8) +- **Rocky Linux** [8, 9](docs/operating_systems/centos.md#centos-8) +- **Kylin Linux Advanced Server V10** (experimental: see [kylin linux notes](docs/operating_systems/kylinlinux.md)) +- **Amazon Linux 2** (experimental: see [amazon linux notes](docs/operating_systems/amazonlinux.md)) +- **UOS Linux** (experimental: see [uos linux notes](docs/operating_systems/uoslinux.md)) +- **openEuler** (experimental: see [openEuler notes](docs/operating_systems/openeuler.md)) Note: Upstart/SysV init based OS types are not supported. ## Supported Components - Core - - [kubernetes](https://github.com/kubernetes/kubernetes) v1.29.5 - - [etcd](https://github.com/etcd-io/etcd) v3.5.12 - - [docker](https://www.docker.com/) v24.0 (see [Note](#container-runtime-notes)) - - [containerd](https://containerd.io/) v1.7.16 - - [cri-o](http://cri-o.io/) v1.29.1 (experimental: see [CRI-O Note](docs/cri-o.md). Only on fedora, ubuntu and centos based OS) + - [kubernetes](https://github.com/kubernetes/kubernetes) v1.31.4 + - [etcd](https://github.com/etcd-io/etcd) v3.5.16 + - [docker](https://www.docker.com/) v26.1 + - [containerd](https://containerd.io/) v1.7.24 + - [cri-o](http://cri-o.io/) v1.31.0 (experimental: see [CRI-O Note](docs/CRI/cri-o.md). Only on fedora, ubuntu and centos based OS) - Network Plugin - [cni-plugins](https://github.com/containernetworking/plugins) v1.2.0 - - [calico](https://github.com/projectcalico/calico) v3.27.3 - - [cilium](https://github.com/cilium/cilium) v1.15.4 + - [calico](https://github.com/projectcalico/calico) v3.29.1 + - [cilium](https://github.com/cilium/cilium) v1.15.9 - [flannel](https://github.com/flannel-io/flannel) v0.22.0 - - [kube-ovn](https://github.com/alauda/kube-ovn) v1.11.5 + - [kube-ovn](https://github.com/alauda/kube-ovn) v1.12.21 - [kube-router](https://github.com/cloudnativelabs/kube-router) v2.0.0 - [multus](https://github.com/k8snetworkplumbingwg/multus-cni) v3.8 - - [weave](https://github.com/weaveworks/weave) v2.8.1 - - [kube-vip](https://github.com/kube-vip/kube-vip) v0.5.12 + - [weave](https://github.com/rajch/weave) v2.8.7 + - [kube-vip](https://github.com/kube-vip/kube-vip) v0.8.0 - Application - - [cert-manager](https://github.com/jetstack/cert-manager) v1.13.2 - - [coredns](https://github.com/coredns/coredns) v1.11.1 - - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) v1.10.1 + - [cert-manager](https://github.com/jetstack/cert-manager) v1.15.3 + - [coredns](https://github.com/coredns/coredns) v1.11.3 + - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) v1.11.2 - [krew](https://github.com/kubernetes-sigs/krew) v0.4.4 - [argocd](https://argoproj.github.io/) v2.11.0 - - [helm](https://helm.sh/) v3.14.2 + - [helm](https://helm.sh/) v3.16.4 - [metallb](https://metallb.universe.tf/) v0.13.9 - [registry](https://github.com/distribution/distribution) v2.8.1 - Storage Plugin @@ -189,22 +123,21 @@ Note: Upstart/SysV init based OS types are not supported. - [rbd-provisioner](https://github.com/kubernetes-incubator/external-storage) v2.1.1-k8s1.11 - [aws-ebs-csi-plugin](https://github.com/kubernetes-sigs/aws-ebs-csi-driver) v0.5.0 - [azure-csi-plugin](https://github.com/kubernetes-sigs/azuredisk-csi-driver) v1.10.0 - - [cinder-csi-plugin](https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/cinder-csi-plugin/using-cinder-csi-plugin.md) v1.29.0 + - [cinder-csi-plugin](https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/cinder-csi-plugin/using-cinder-csi-plugin.md) v1.30.0 - [gcp-pd-csi-plugin](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver) v1.9.2 - [local-path-provisioner](https://github.com/rancher/local-path-provisioner) v0.0.24 - [local-volume-provisioner](https://github.com/kubernetes-sigs/sig-storage-local-static-provisioner) v2.5.0 - - [node-feature-discovery](https://github.com/kubernetes-sigs/node-feature-discovery) v0.14.2 + - [node-feature-discovery](https://github.com/kubernetes-sigs/node-feature-discovery) v0.16.4 ## Container Runtime Notes -- Supported Docker versions are 18.09, 19.03, 20.10, 23.0 and 24.0. The *recommended* Docker version is 24.0. `Kubelet` might break on docker's non-standard version numbering (it no longer uses semantic versioning). To ensure auto-updates don't break your cluster look into e.g. the YUM ``versionlock`` plugin or ``apt pin``). - The cri-o version should be aligned with the respective kubernetes version (i.e. kube_version=1.20.x, crio_version=1.20) ## Requirements -- **Minimum required version of Kubernetes is v1.27** +- **Minimum required version of Kubernetes is v1.29** - **Ansible v2.14+, Jinja 2.11+ and python-netaddr is installed on the machine that will run Ansible commands** -- The target servers must have **access to the Internet** in order to pull docker images. Otherwise, additional configuration is required (See [Offline Environment](docs/offline-environment.md)) +- The target servers must have **access to the Internet** in order to pull docker images. Otherwise, additional configuration is required (See [Offline Environment](docs/operations/offline-environment.md)) - The target servers are configured to allow **IPv4 forwarding**. - If using IPv6 for pods and services, the target servers are configured to allow **IPv6 forwarding**. - The **firewalls are not managed**, you'll need to implement your own rules the way you used to. @@ -225,7 +158,7 @@ These limits are safeguarded by Kubespray. Actual requirements for your workload You can choose among ten network plugins. (default: `calico`, except Vagrant uses `flannel`) -- [flannel](docs/flannel.md): gre/vxlan (layer 2) networking. +- [flannel](docs/CNI/flannel.md): gre/vxlan (layer 2) networking. - [Calico](https://docs.tigera.io/calico/latest/about/) is a networking and network policy provider. Calico supports a flexible set of networking options designed to give you the most efficient networking across a range of situations, including non-overlay @@ -234,32 +167,32 @@ You can choose among ten network plugins. (default: `calico`, except Vagrant use - [cilium](http://docs.cilium.io/en/latest/): layer 3/4 networking (as well as layer 7 to protect and secure application protocols), supports dynamic insertion of BPF bytecode into the Linux kernel to implement security services, networking and visibility logic. -- [weave](docs/weave.md): Weave is a lightweight container overlay network that doesn't require an external K/V database cluster. +- [weave](docs/CNI/weave.md): Weave is a lightweight container overlay network that doesn't require an external K/V database cluster. (Please refer to `weave` [troubleshooting documentation](https://www.weave.works/docs/net/latest/troubleshooting/)). -- [kube-ovn](docs/kube-ovn.md): Kube-OVN integrates the OVN-based Network Virtualization with Kubernetes. It offers an advanced Container Network Fabric for Enterprises. +- [kube-ovn](docs/CNI/kube-ovn.md): Kube-OVN integrates the OVN-based Network Virtualization with Kubernetes. It offers an advanced Container Network Fabric for Enterprises. -- [kube-router](docs/kube-router.md): Kube-router is a L3 CNI for Kubernetes networking aiming to provide operational +- [kube-router](docs/CNI/kube-router.md): Kube-router is a L3 CNI for Kubernetes networking aiming to provide operational simplicity and high performance: it uses IPVS to provide Kube Services Proxy (if setup to replace kube-proxy), iptables for network policies, and BGP for ods L3 networking (with optionally BGP peering with out-of-cluster BGP peers). It can also optionally advertise routes to Kubernetes cluster Pods CIDRs, ClusterIPs, ExternalIPs and LoadBalancerIPs. -- [macvlan](docs/macvlan.md): Macvlan is a Linux network driver. Pods have their own unique Mac and Ip address, connected directly the physical (layer 2) network. +- [macvlan](docs/CNI/macvlan.md): Macvlan is a Linux network driver. Pods have their own unique Mac and Ip address, connected directly the physical (layer 2) network. -- [multus](docs/multus.md): Multus is a meta CNI plugin that provides multiple network interface support to pods. For each interface Multus delegates CNI calls to secondary CNI plugins such as Calico, macvlan, etc. +- [multus](docs/CNI/multus.md): Multus is a meta CNI plugin that provides multiple network interface support to pods. For each interface Multus delegates CNI calls to secondary CNI plugins such as Calico, macvlan, etc. - [custom_cni](roles/network-plugin/custom_cni/) : You can specify some manifests that will be applied to the clusters to bring you own CNI and use non-supported ones by Kubespray. See `tests/files/custom_cni/README.md` and `tests/files/custom_cni/values.yaml`for an example with a CNI provided by a Helm Chart. The network plugin to use is defined by the variable `kube_network_plugin`. There is also an option to leverage built-in cloud provider networking instead. -See also [Network checker](docs/netcheck.md). +See also [Network checker](docs/advanced/netcheck.md). ## Ingress Plugins - [nginx](https://kubernetes.github.io/ingress-nginx): the NGINX Ingress Controller. -- [metallb](docs/metallb.md): the MetalLB bare-metal service LoadBalancer provider. +- [metallb](docs/ingress/metallb.md): the MetalLB bare-metal service LoadBalancer provider. ## Community docs and resources @@ -280,4 +213,4 @@ See also [Network checker](docs/netcheck.md). CI/end-to-end tests sponsored by: [CNCF](https://cncf.io), [Equinix Metal](https://metal.equinix.com/), [OVHcloud](https://www.ovhcloud.com/), [ELASTX](https://elastx.se/). -See the [test matrix](docs/test_cases.md) for details. +See the [test matrix](docs/developers/test_cases.md) for details. diff --git a/RELEASE.md b/RELEASE.md index 8248a097bc9..27495311ae0 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -16,6 +16,7 @@ The Kubespray Project is released on an as-needed basis. The process is as follo 1. The release issue is closed 1. An announcement email is sent to `dev@kubernetes.io` with the subject `[ANNOUNCE] Kubespray $VERSION is released` 1. The topic of the #kubespray channel is updated with `vX.Y.Z is released! | ...` +1. Create/Update Issue for upgradeing kubernetes and [k8s-conformance](https://github.com/cncf/k8s-conformance) ## Major/minor releases and milestones diff --git a/SECURITY_CONTACTS b/SECURITY_CONTACTS index 21703b32858..5b743285438 100644 --- a/SECURITY_CONTACTS +++ b/SECURITY_CONTACTS @@ -9,7 +9,7 @@ # # DO NOT REPORT SECURITY VULNERABILITIES DIRECTLY TO THESE NAMES, FOLLOW THE # INSTRUCTIONS AT https://kubernetes.io/security/ -mattymo floryut -oomichi -cristicalin +ant31 +VannTen +yankay diff --git a/Vagrantfile b/Vagrantfile index df48261f7b9..643aee42998 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -1,7 +1,7 @@ # -*- mode: ruby -*- # # vi: set ft=ruby : -# For help on using kubespray with vagrant, check out docs/vagrant.md +# For help on using kubespray with vagrant, check out docs/developers/vagrant.md require 'fileutils' @@ -21,21 +21,21 @@ SUPPORTED_OS = { "flatcar-edge" => {box: "flatcar-edge", user: "core", box_url: FLATCAR_URL_TEMPLATE % ["edge"]}, "ubuntu2004" => {box: "generic/ubuntu2004", user: "vagrant"}, "ubuntu2204" => {box: "generic/ubuntu2204", user: "vagrant"}, - "centos" => {box: "centos/7", user: "vagrant"}, - "centos-bento" => {box: "bento/centos-7.6", user: "vagrant"}, + "ubuntu2404" => {box: "bento/ubuntu-24.04", user: "vagrant"}, "centos8" => {box: "centos/8", user: "vagrant"}, "centos8-bento" => {box: "bento/centos-8", user: "vagrant"}, "almalinux8" => {box: "almalinux/8", user: "vagrant"}, "almalinux8-bento" => {box: "bento/almalinux-8", user: "vagrant"}, "rockylinux8" => {box: "rockylinux/8", user: "vagrant"}, "rockylinux9" => {box: "rockylinux/9", user: "vagrant"}, - "fedora37" => {box: "fedora/37-cloud-base", user: "vagrant"}, - "fedora38" => {box: "fedora/38-cloud-base", user: "vagrant"}, + "fedora39" => {box: "fedora/39-cloud-base", user: "vagrant"}, + "fedora40" => {box: "fedora/40-cloud-base", user: "vagrant"}, + "fedora39-arm64" => {box: "bento/fedora-39-arm64", user: "vagrant"}, + "fedora40-arm64" => {box: "bento/fedora-40", user: "vagrant"}, "opensuse" => {box: "opensuse/Leap-15.4.x86_64", user: "vagrant"}, "opensuse-tumbleweed" => {box: "opensuse/Tumbleweed.x86_64", user: "vagrant"}, "oraclelinux" => {box: "generic/oracle7", user: "vagrant"}, "oraclelinux8" => {box: "generic/oracle8", user: "vagrant"}, - "rhel7" => {box: "generic/rhel7", user: "vagrant"}, "rhel8" => {box: "generic/rhel8", user: "vagrant"}, "debian11" => {box: "debian/bullseye64", user: "vagrant"}, "debian12" => {box: "debian/bookworm64", user: "vagrant"}, @@ -57,6 +57,8 @@ $subnet ||= "172.18.8" $subnet_ipv6 ||= "fd3c:b398:0698:0756" $os ||= "ubuntu2004" $network_plugin ||= "flannel" +$inventory ||= "inventory/sample" +$inventories ||= [$inventory] # Setting multi_networking to true will install Multus: https://github.com/k8snetworkplumbingwg/multus-cni $multi_networking ||= "False" $download_run_once ||= "True" @@ -95,19 +97,6 @@ if ! SUPPORTED_OS.key?($os) end $box = SUPPORTED_OS[$os][:box] -# if $inventory is not set, try to use example -$inventory = "inventory/sample" if ! $inventory -$inventory = File.absolute_path($inventory, File.dirname(__FILE__)) - -# if $inventory has a hosts.ini file use it, otherwise copy over -# vars etc to where vagrant expects dynamic inventory to be -if ! File.exist?(File.join(File.dirname($inventory), "hosts.ini")) - $vagrant_ansible = File.join(File.absolute_path($vagrant_dir), "provisioners", "ansible") - FileUtils.mkdir_p($vagrant_ansible) if ! File.exist?($vagrant_ansible) - $vagrant_inventory = File.join($vagrant_ansible,"inventory") - FileUtils.rm_f($vagrant_inventory) - FileUtils.ln_s($inventory, $vagrant_inventory) -end if Vagrant.has_plugin?("vagrant-proxyconf") $no_proxy = ENV['NO_PROXY'] || ENV['no_proxy'] || "127.0.0.1,localhost" @@ -206,7 +195,7 @@ Vagrant.configure("2") do |config| node.vm.network "forwarded_port", guest: guest, host: host, auto_correct: true end - if ["rhel7","rhel8"].include? $os + if ["rhel8"].include? $os # Vagrant synced_folder rsync options cannot be used for RHEL boxes as Rsync package cannot # be installed until the host is registered with a valid Red Hat support subscription node.vm.synced_folder ".", "/vagrant", disabled: false @@ -237,15 +226,16 @@ Vagrant.configure("2") do |config| node.vm.provision "shell", inline: "rm -f /etc/modprobe.d/local.conf" node.vm.provision "shell", inline: "sed -i '/net.ipv6.conf.all.disable_ipv6/d' /etc/sysctl.d/99-sysctl.conf /etc/sysctl.conf" end - # Hack for fedora37/38 to get the IP address of the second interface - if ["fedora37", "fedora38"].include? $os + # Hack for fedora39/40 to get the IP address of the second interface + if ["fedora39", "fedora40", "fedora39-arm64", "fedora40-arm64"].include? $os config.vm.provision "shell", inline: <<-SHELL - nmcli conn modify 'Wired connection 2' ipv4.addresses $(cat /etc/sysconfig/network-scripts/ifcfg-eth1 | grep IPADDR | cut -d "=" -f2) + nmcli conn modify 'Wired connection 2' ipv4.addresses $(cat /etc/sysconfig/network-scripts/ifcfg-eth1 | grep IPADDR | cut -d "=" -f2)/24 nmcli conn modify 'Wired connection 2' ipv4.method manual service NetworkManager restart SHELL end + # Rockylinux boxes needs UEFI if ["rockylinux8", "rockylinux9"].include? $os config.vm.provider "libvirt" do |domain| @@ -254,7 +244,7 @@ Vagrant.configure("2") do |config| end # Disable firewalld on oraclelinux/redhat vms - if ["oraclelinux","oraclelinux8","rhel7","rhel8","rockylinux8"].include? $os + if ["oraclelinux","oraclelinux8", "rhel8","rockylinux8"].include? $os node.vm.provision "shell", inline: "systemctl stop firewalld; systemctl disable firewalld" end @@ -277,6 +267,7 @@ Vagrant.configure("2") do |config| "local_path_provisioner_enabled": "#{$local_path_provisioner_enabled}", "local_path_provisioner_claim_root": "#{$local_path_provisioner_claim_root}", "ansible_ssh_user": SUPPORTED_OS[$os][:user], + "ansible_ssh_private_key_file": File.join(Dir.home, ".vagrant.d", "insecure_private_key"), "unsafe_show_logs": "True" } @@ -287,14 +278,13 @@ Vagrant.configure("2") do |config| ansible.playbook = $playbook ansible.compatibility_mode = "2.0" ansible.verbose = $ansible_verbosity - $ansible_inventory_path = File.join( $inventory, "hosts.ini") - if File.exist?($ansible_inventory_path) - ansible.inventory_path = $ansible_inventory_path - end ansible.become = true ansible.limit = "all,localhost" ansible.host_key_checking = false - ansible.raw_arguments = ["--forks=#{$num_instances}", "--flush-cache", "-e ansible_become_pass=vagrant"] + ansible.raw_arguments = ["--forks=#{$num_instances}", + "--flush-cache", + "-e ansible_become_pass=vagrant"] + + $inventories.map {|inv| ["-i", inv]}.flatten ansible.host_vars = host_vars ansible.extra_vars = $extra_vars if $ansible_tags != "" diff --git a/ansible.cfg b/ansible.cfg index 48a3ff01bfa..e7952378336 100644 --- a/ansible.cfg +++ b/ansible.cfg @@ -11,6 +11,7 @@ gathering = smart fact_caching = jsonfile fact_caching_connection = /tmp fact_caching_timeout = 86400 +timeout = 300 stdout_callback = default display_skipped_hosts = no library = ./library diff --git a/contrib/azurerm/README.md b/contrib/azurerm/README.md index f24a5ec2321..8869ec09114 100644 --- a/contrib/azurerm/README.md +++ b/contrib/azurerm/README.md @@ -49,7 +49,7 @@ If you need to delete all resources from a resource group, simply call: ## Installing Ansible and the dependencies -Install Ansible according to [Ansible installation guide](/docs/ansible.md#installing-ansible) +Install Ansible according to [Ansible installation guide](/docs/ansible/ansible.md#installing-ansible) ## Generating an inventory for kubespray diff --git a/contrib/azurerm/generate-inventory.yml b/contrib/azurerm/generate-inventory.yml index 01ee386626e..59e1e90b6a4 100644 --- a/contrib/azurerm/generate-inventory.yml +++ b/contrib/azurerm/generate-inventory.yml @@ -1,6 +1,6 @@ --- - name: Generate Azure inventory hosts: localhost - gather_facts: False + gather_facts: false roles: - generate-inventory diff --git a/contrib/azurerm/generate-inventory_2.yml b/contrib/azurerm/generate-inventory_2.yml index 9173e1d8204..8c2cbff86b5 100644 --- a/contrib/azurerm/generate-inventory_2.yml +++ b/contrib/azurerm/generate-inventory_2.yml @@ -1,6 +1,6 @@ --- - name: Generate Azure inventory hosts: localhost - gather_facts: False + gather_facts: false roles: - generate-inventory_2 diff --git a/contrib/azurerm/generate-templates.yml b/contrib/azurerm/generate-templates.yml index f1fcb626f8b..f2cf231bc4d 100644 --- a/contrib/azurerm/generate-templates.yml +++ b/contrib/azurerm/generate-templates.yml @@ -1,6 +1,6 @@ --- - name: Generate Azure templates hosts: localhost - gather_facts: False + gather_facts: false roles: - generate-templates diff --git a/contrib/azurerm/roles/generate-inventory/tasks/main.yml b/contrib/azurerm/roles/generate-inventory/tasks/main.yml index 3eb121aa0a7..f93f1b6b281 100644 --- a/contrib/azurerm/roles/generate-inventory/tasks/main.yml +++ b/contrib/azurerm/roles/generate-inventory/tasks/main.yml @@ -12,4 +12,4 @@ template: src: inventory.j2 dest: "{{ playbook_dir }}/inventory" - mode: 0644 + mode: "0644" diff --git a/contrib/azurerm/roles/generate-inventory_2/tasks/main.yml b/contrib/azurerm/roles/generate-inventory_2/tasks/main.yml index c628154a038..267755b1285 100644 --- a/contrib/azurerm/roles/generate-inventory_2/tasks/main.yml +++ b/contrib/azurerm/roles/generate-inventory_2/tasks/main.yml @@ -22,10 +22,10 @@ template: src: inventory.j2 dest: "{{ playbook_dir }}/inventory" - mode: 0644 + mode: "0644" - name: Generate Load Balancer variables template: src: loadbalancer_vars.j2 dest: "{{ playbook_dir }}/loadbalancer_vars.yml" - mode: 0644 + mode: "0644" diff --git a/contrib/azurerm/roles/generate-templates/tasks/main.yml b/contrib/azurerm/roles/generate-templates/tasks/main.yml index 294ee96fc86..057d4d00547 100644 --- a/contrib/azurerm/roles/generate-templates/tasks/main.yml +++ b/contrib/azurerm/roles/generate-templates/tasks/main.yml @@ -8,13 +8,13 @@ path: "{{ base_dir }}" state: directory recurse: true - mode: 0755 + mode: "0755" - name: Store json files in base_dir template: src: "{{ item }}" dest: "{{ base_dir }}/{{ item }}" - mode: 0644 + mode: "0644" with_items: - network.json - storage.json diff --git a/contrib/dind/README.md b/contrib/dind/README.md deleted file mode 100644 index 5e72cfc2cf5..00000000000 --- a/contrib/dind/README.md +++ /dev/null @@ -1,177 +0,0 @@ -# Kubespray DIND experimental setup - -This ansible playbook creates local docker containers -to serve as Kubernetes "nodes", which in turn will run -"normal" Kubernetes docker containers, a mode usually -called DIND (Docker-IN-Docker). - -The playbook has two roles: - -- dind-host: creates the "nodes" as containers in localhost, with - appropriate settings for DIND (privileged, volume mapping for dind - storage, etc). -- dind-cluster: customizes each node container to have required - system packages installed, and some utils (swapoff, lsattr) - symlinked to /bin/true to ease mimicking a real node. - -This playbook has been test with Ubuntu 16.04 as host and ubuntu:16.04 -as docker images (note that dind-cluster has specific customization -for these images). - -The playbook also creates a `/tmp/kubespray.dind.inventory_builder.sh` -helper (wraps up running `contrib/inventory_builder/inventory.py` with -node containers IPs and prefix). - -## Deploying - -See below for a complete successful run: - -1. Create the node containers - -```shell -# From the kubespray root dir -cd contrib/dind -pip install -r requirements.txt - -ansible-playbook -i hosts dind-cluster.yaml - -# Back to kubespray root -cd ../.. -``` - -NOTE: if the playbook run fails with something like below error -message, you may need to specifically set `ansible_python_interpreter`, -see `./hosts` file for an example expanded localhost entry. - -```shell -failed: [localhost] (item=kube-node1) => {"changed": false, "item": "kube-node1", "msg": "Failed to import docker or docker-py - No module named requests.exceptions. Try `pip install docker` or `pip install docker-py` (Python 2.6)"} -``` - -2. Customize kubespray-dind.yaml - -Note that there's coupling between above created node containers -and `kubespray-dind.yaml` settings, in particular regarding selected `node_distro` -(as set in `group_vars/all/all.yaml`), and docker settings. - -```shell -$EDITOR contrib/dind/kubespray-dind.yaml -``` - -3. Prepare the inventory and run the playbook - -```shell -INVENTORY_DIR=inventory/local-dind -mkdir -p ${INVENTORY_DIR} -rm -f ${INVENTORY_DIR}/hosts.ini -CONFIG_FILE=${INVENTORY_DIR}/hosts.ini /tmp/kubespray.dind.inventory_builder.sh - -ansible-playbook --become -e ansible_ssh_user=debian -i ${INVENTORY_DIR}/hosts.ini cluster.yml --extra-vars @contrib/dind/kubespray-dind.yaml -``` - -NOTE: You could also test other distros without editing files by -passing `--extra-vars` as per below commandline, -replacing `DISTRO` by either `debian`, `ubuntu`, `centos`, `fedora`: - -```shell -cd contrib/dind -ansible-playbook -i hosts dind-cluster.yaml --extra-vars node_distro=DISTRO - -cd ../.. -CONFIG_FILE=inventory/local-dind/hosts.ini /tmp/kubespray.dind.inventory_builder.sh -ansible-playbook --become -e ansible_ssh_user=DISTRO -i inventory/local-dind/hosts.ini cluster.yml --extra-vars @contrib/dind/kubespray-dind.yaml --extra-vars bootstrap_os=DISTRO -``` - -## Resulting deployment - -See below to get an idea on how a completed deployment looks like, -from the host where you ran kubespray playbooks. - -### node_distro: debian - -Running from an Ubuntu Xenial host: - -```shell -$ uname -a -Linux ip-xx-xx-xx-xx 4.4.0-1069-aws #79-Ubuntu SMP Mon Sep 24 -15:01:41 UTC 2018 x86_64 x86_64 x86_64 GNU/Linux - -$ docker ps -CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES -1835dd183b75 debian:9.5 "sh -c 'apt-get -qy …" 43 minutes ago Up 43 minutes kube-node5 -30b0af8d2924 debian:9.5 "sh -c 'apt-get -qy …" 43 minutes ago Up 43 minutes kube-node4 -3e0d1510c62f debian:9.5 "sh -c 'apt-get -qy …" 43 minutes ago Up 43 minutes kube-node3 -738993566f94 debian:9.5 "sh -c 'apt-get -qy …" 44 minutes ago Up 44 minutes kube-node2 -c581ef662ed2 debian:9.5 "sh -c 'apt-get -qy …" 44 minutes ago Up 44 minutes kube-node1 - -$ docker exec kube-node1 kubectl get node -NAME STATUS ROLES AGE VERSION -kube-node1 Ready master,node 18m v1.12.1 -kube-node2 Ready master,node 17m v1.12.1 -kube-node3 Ready node 17m v1.12.1 -kube-node4 Ready node 17m v1.12.1 -kube-node5 Ready node 17m v1.12.1 - -$ docker exec kube-node1 kubectl get pod --all-namespaces -NAMESPACE NAME READY STATUS RESTARTS AGE -default netchecker-agent-67489 1/1 Running 0 2m51s -default netchecker-agent-6qq6s 1/1 Running 0 2m51s -default netchecker-agent-fsw92 1/1 Running 0 2m51s -default netchecker-agent-fw6tl 1/1 Running 0 2m51s -default netchecker-agent-hostnet-8f2zb 1/1 Running 0 3m -default netchecker-agent-hostnet-gq7ml 1/1 Running 0 3m -default netchecker-agent-hostnet-jfkgv 1/1 Running 0 3m -default netchecker-agent-hostnet-kwfwx 1/1 Running 0 3m -default netchecker-agent-hostnet-r46nm 1/1 Running 0 3m -default netchecker-agent-lxdrn 1/1 Running 0 2m51s -default netchecker-server-864bd4c897-9vstl 1/1 Running 0 2m40s -default sh-68fcc6db45-qf55h 1/1 Running 1 12m -kube-system coredns-7598f59475-6vknq 1/1 Running 0 14m -kube-system coredns-7598f59475-l5q5x 1/1 Running 0 14m -kube-system kube-apiserver-kube-node1 1/1 Running 0 17m -kube-system kube-apiserver-kube-node2 1/1 Running 0 18m -kube-system kube-controller-manager-kube-node1 1/1 Running 0 18m -kube-system kube-controller-manager-kube-node2 1/1 Running 0 18m -kube-system kube-proxy-5xx9d 1/1 Running 0 17m -kube-system kube-proxy-cdqq4 1/1 Running 0 17m -kube-system kube-proxy-n64ls 1/1 Running 0 17m -kube-system kube-proxy-pswmj 1/1 Running 0 18m -kube-system kube-proxy-x89qw 1/1 Running 0 18m -kube-system kube-scheduler-kube-node1 1/1 Running 4 17m -kube-system kube-scheduler-kube-node2 1/1 Running 4 18m -kube-system kubernetes-dashboard-5db4d9f45f-548rl 1/1 Running 0 14m -kube-system nginx-proxy-kube-node3 1/1 Running 4 17m -kube-system nginx-proxy-kube-node4 1/1 Running 4 17m -kube-system nginx-proxy-kube-node5 1/1 Running 4 17m -kube-system weave-net-42bfr 2/2 Running 0 16m -kube-system weave-net-6gt8m 2/2 Running 0 16m -kube-system weave-net-88nnc 2/2 Running 0 16m -kube-system weave-net-shckr 2/2 Running 0 16m -kube-system weave-net-xr46t 2/2 Running 0 16m - -$ docker exec kube-node1 curl -s http://localhost:31081/api/v1/connectivity_check -{"Message":"All 10 pods successfully reported back to the server","Absent":null,"Outdated":null} -``` - -## Using ./run-test-distros.sh - -You can use `./run-test-distros.sh` to run a set of tests via DIND, -and excerpt from this script, to get an idea: - -```shell -# The SPEC file(s) must have two arrays as e.g. -# DISTROS=(debian centos) -# EXTRAS=( -# 'kube_network_plugin=calico' -# 'kube_network_plugin=flannel' -# 'kube_network_plugin=weave' -# ) -# that will be tested in a "combinatory" way (e.g. from above there'll be -# be 6 test runs), creating a sequenced -nn.out with each output. -# -# Each $EXTRAS element will be whitespace split, and passed as --extra-vars -# to main kubespray ansible-playbook run. -``` - -See e.g. `test-some_distros-most_CNIs.env` and -`test-some_distros-kube_router_combo.env` in particular for a richer -set of CNI specific `--extra-vars` combo. diff --git a/contrib/dind/dind-cluster.yaml b/contrib/dind/dind-cluster.yaml deleted file mode 100644 index 258837d083c..00000000000 --- a/contrib/dind/dind-cluster.yaml +++ /dev/null @@ -1,11 +0,0 @@ ---- -- name: Create nodes as docker containers - hosts: localhost - gather_facts: False - roles: - - { role: dind-host } - -- name: Customize each node containers - hosts: containers - roles: - - { role: dind-cluster } diff --git a/contrib/dind/group_vars/all/all.yaml b/contrib/dind/group_vars/all/all.yaml deleted file mode 100644 index fd619a05e9f..00000000000 --- a/contrib/dind/group_vars/all/all.yaml +++ /dev/null @@ -1,3 +0,0 @@ ---- -# See distro.yaml for supported node_distro images -node_distro: debian diff --git a/contrib/dind/group_vars/all/distro.yaml b/contrib/dind/group_vars/all/distro.yaml deleted file mode 100644 index b9c2670e9c5..00000000000 --- a/contrib/dind/group_vars/all/distro.yaml +++ /dev/null @@ -1,41 +0,0 @@ ---- -distro_settings: - debian: &DEBIAN - image: "debian:9.5" - user: "debian" - pid1_exe: /lib/systemd/systemd - init: | - sh -c "apt-get -qy update && apt-get -qy install systemd-sysv dbus && exec /sbin/init" - raw_setup: apt-get -qy update && apt-get -qy install dbus python sudo iproute2 - raw_setup_done: test -x /usr/bin/sudo - agetty_svc: getty@* - ssh_service: ssh - extra_packages: [] - ubuntu: - <<: *DEBIAN - image: "ubuntu:16.04" - user: "ubuntu" - init: | - /sbin/init - centos: &CENTOS - image: "centos:7" - user: "centos" - pid1_exe: /usr/lib/systemd/systemd - init: | - /sbin/init - raw_setup: yum -qy install policycoreutils dbus python sudo iproute iptables - raw_setup_done: test -x /usr/bin/sudo - agetty_svc: getty@* serial-getty@* - ssh_service: sshd - extra_packages: [] - fedora: - <<: *CENTOS - image: "fedora:latest" - user: "fedora" - raw_setup: yum -qy install policycoreutils dbus python sudo iproute iptables; mkdir -p /etc/modules-load.d - extra_packages: - - hostname - - procps - - findutils - - kmod - - iputils diff --git a/contrib/dind/hosts b/contrib/dind/hosts deleted file mode 100644 index 356aa2675bf..00000000000 --- a/contrib/dind/hosts +++ /dev/null @@ -1,15 +0,0 @@ -[local] -# If you created a virtualenv for ansible, you may need to specify running the -# python binary from there instead: -#localhost ansible_connection=local ansible_python_interpreter=/home/user/kubespray/.venv/bin/python -localhost ansible_connection=local - -[containers] -kube-node1 -kube-node2 -kube-node3 -kube-node4 -kube-node5 - -[containers:vars] -ansible_connection=docker diff --git a/contrib/dind/kubespray-dind.yaml b/contrib/dind/kubespray-dind.yaml deleted file mode 100644 index ecfb5573a70..00000000000 --- a/contrib/dind/kubespray-dind.yaml +++ /dev/null @@ -1,22 +0,0 @@ ---- -# kubespray-dind.yaml: minimal kubespray ansible playbook usable for DIND -# See contrib/dind/README.md -kube_api_anonymous_auth: true - -kubelet_fail_swap_on: false - -# Docker nodes need to have been created with same "node_distro: debian" -# at contrib/dind/group_vars/all/all.yaml -bootstrap_os: debian - -docker_version: latest - -docker_storage_options: -s overlay2 --storage-opt overlay2.override_kernel_check=true -g /dind/docker - -dns_mode: coredns - -deploy_netchecker: True -netcheck_agent_image_repo: quay.io/l23network/k8s-netchecker-agent -netcheck_server_image_repo: quay.io/l23network/k8s-netchecker-server -netcheck_agent_image_tag: v1.0 -netcheck_server_image_tag: v1.0 diff --git a/contrib/dind/requirements.txt b/contrib/dind/requirements.txt deleted file mode 100644 index bdb9670965e..00000000000 --- a/contrib/dind/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -docker diff --git a/contrib/dind/roles/dind-cluster/tasks/main.yaml b/contrib/dind/roles/dind-cluster/tasks/main.yaml deleted file mode 100644 index 1cf819f68c5..00000000000 --- a/contrib/dind/roles/dind-cluster/tasks/main.yaml +++ /dev/null @@ -1,73 +0,0 @@ ---- -- name: Set_fact distro_setup - set_fact: - distro_setup: "{{ distro_settings[node_distro] }}" - -- name: Set_fact other distro settings - set_fact: - distro_user: "{{ distro_setup['user'] }}" - distro_ssh_service: "{{ distro_setup['ssh_service'] }}" - distro_extra_packages: "{{ distro_setup['extra_packages'] }}" - -- name: Null-ify some linux tools to ease DIND - file: - src: "/bin/true" - dest: "{{ item }}" - state: link - force: yes - with_items: - # DIND box may have swap enable, don't bother - - /sbin/swapoff - # /etc/hosts handling would fail on trying to copy file attributes on edit, - # void it by successfully returning nil output - - /usr/bin/lsattr - # disable selinux-isms, sp needed if running on non-Selinux host - - /usr/sbin/semodule - -- name: Void installing dpkg docs and man pages on Debian based distros - copy: - content: | - # Delete locales - path-exclude=/usr/share/locale/* - # Delete man pages - path-exclude=/usr/share/man/* - # Delete docs - path-exclude=/usr/share/doc/* - path-include=/usr/share/doc/*/copyright - dest: /etc/dpkg/dpkg.cfg.d/01_nodoc - mode: 0644 - when: - - ansible_os_family == 'Debian' - -- name: Install system packages to better match a full-fledge node - package: - name: "{{ item }}" - state: present - with_items: "{{ distro_extra_packages + ['rsyslog', 'openssh-server'] }}" - -- name: Start needed services - service: - name: "{{ item }}" - state: started - with_items: - - rsyslog - - "{{ distro_ssh_service }}" - -- name: Create distro user "{{ distro_user }}" - user: - name: "{{ distro_user }}" - uid: 1000 - # groups: sudo - append: yes - -- name: Allow password-less sudo to "{{ distro_user }}" - copy: - content: "{{ distro_user }} ALL=(ALL) NOPASSWD:ALL" - dest: "/etc/sudoers.d/{{ distro_user }}" - mode: 0640 - -- name: "Add my pubkey to {{ distro_user }} user authorized keys" - ansible.posix.authorized_key: - user: "{{ distro_user }}" - state: present - key: "{{ lookup('file', lookup('env', 'HOME') + '/.ssh/id_rsa.pub') }}" diff --git a/contrib/dind/roles/dind-host/tasks/main.yaml b/contrib/dind/roles/dind-host/tasks/main.yaml deleted file mode 100644 index e44047f4ded..00000000000 --- a/contrib/dind/roles/dind-host/tasks/main.yaml +++ /dev/null @@ -1,87 +0,0 @@ ---- -- name: Set_fact distro_setup - set_fact: - distro_setup: "{{ distro_settings[node_distro] }}" - -- name: Set_fact other distro settings - set_fact: - distro_image: "{{ distro_setup['image'] }}" - distro_init: "{{ distro_setup['init'] }}" - distro_pid1_exe: "{{ distro_setup['pid1_exe'] }}" - distro_raw_setup: "{{ distro_setup['raw_setup'] }}" - distro_raw_setup_done: "{{ distro_setup['raw_setup_done'] }}" - distro_agetty_svc: "{{ distro_setup['agetty_svc'] }}" - -- name: Create dind node containers from "containers" inventory section - community.docker.docker_container: - image: "{{ distro_image }}" - name: "{{ item }}" - state: started - hostname: "{{ item }}" - command: "{{ distro_init }}" - # recreate: yes - privileged: true - tmpfs: - - /sys/module/nf_conntrack/parameters - volumes: - - /boot:/boot - - /lib/modules:/lib/modules - - "{{ item }}:/dind/docker" - register: containers - with_items: "{{ groups.containers }}" - tags: - - addresses - -- name: Gather list of containers IPs - set_fact: - addresses: "{{ containers.results | map(attribute='ansible_facts') | map(attribute='docker_container') | map(attribute='NetworkSettings') | map(attribute='IPAddress') | list }}" - tags: - - addresses - -- name: Create inventory_builder helper already set with the list of node containers' IPs - template: - src: inventory_builder.sh.j2 - dest: /tmp/kubespray.dind.inventory_builder.sh - mode: 0755 - tags: - - addresses - -- name: Install needed packages into node containers via raw, need to wait for possible systemd packages to finish installing - raw: | - # agetty processes churn a lot of cpu time failing on inexistent ttys, early STOP them, to rip them in below task - pkill -STOP agetty || true - {{ distro_raw_setup_done }} && echo SKIPPED && exit 0 - until [ "$(readlink /proc/1/exe)" = "{{ distro_pid1_exe }}" ] ; do sleep 1; done - {{ distro_raw_setup }} - delegate_to: "{{ item._ansible_item_label | default(item.item) }}" - with_items: "{{ containers.results }}" - register: result - changed_when: result.stdout.find("SKIPPED") < 0 - -- name: Remove gettys from node containers - raw: | - until test -S /var/run/dbus/system_bus_socket; do sleep 1; done - systemctl disable {{ distro_agetty_svc }} - systemctl stop {{ distro_agetty_svc }} - delegate_to: "{{ item._ansible_item_label | default(item.item) }}" - with_items: "{{ containers.results }}" - changed_when: false - -# Running systemd-machine-id-setup doesn't create a unique id for each node container on Debian, -# handle manually -- name: Re-create unique machine-id (as we may just get what comes in the docker image), needed by some CNIs for mac address seeding (notably weave) - raw: | - echo {{ item | hash('sha1') }} > /etc/machine-id.new - mv -b /etc/machine-id.new /etc/machine-id - cmp /etc/machine-id /etc/machine-id~ || true - systemctl daemon-reload - delegate_to: "{{ item._ansible_item_label | default(item.item) }}" - with_items: "{{ containers.results }}" - -- name: Early hack image install to adapt for DIND - raw: | - rm -fv /usr/bin/udevadm /usr/sbin/udevadm - delegate_to: "{{ item._ansible_item_label | default(item.item) }}" - with_items: "{{ containers.results }}" - register: result - changed_when: result.stdout.find("removed") >= 0 diff --git a/contrib/dind/roles/dind-host/templates/inventory_builder.sh.j2 b/contrib/dind/roles/dind-host/templates/inventory_builder.sh.j2 deleted file mode 100644 index 48e17583e37..00000000000 --- a/contrib/dind/roles/dind-host/templates/inventory_builder.sh.j2 +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -# NOTE: if you change HOST_PREFIX, you also need to edit ./hosts [containers] section -HOST_PREFIX=kube-node python3 contrib/inventory_builder/inventory.py {% for ip in addresses %} {{ ip }} {% endfor %} diff --git a/contrib/dind/run-test-distros.sh b/contrib/dind/run-test-distros.sh deleted file mode 100755 index 3695276aaaa..00000000000 --- a/contrib/dind/run-test-distros.sh +++ /dev/null @@ -1,93 +0,0 @@ -#!/bin/bash -# Q&D test'em all: creates full DIND kubespray deploys -# for each distro, verifying it via netchecker. - -info() { - local msg="$*" - local date="$(date -Isec)" - echo "INFO: [$date] $msg" -} -pass_or_fail() { - local rc="$?" - local msg="$*" - local date="$(date -Isec)" - [ $rc -eq 0 ] && echo "PASS: [$date] $msg" || echo "FAIL: [$date] $msg" - return $rc -} -test_distro() { - local distro=${1:?};shift - local extra="${*:-}" - local prefix="${distro[${extra}]}" - ansible-playbook -i hosts dind-cluster.yaml -e node_distro=$distro - pass_or_fail "$prefix: dind-nodes" || return 1 - (cd ../.. - INVENTORY_DIR=inventory/local-dind - mkdir -p ${INVENTORY_DIR} - rm -f ${INVENTORY_DIR}/hosts.ini - CONFIG_FILE=${INVENTORY_DIR}/hosts.ini /tmp/kubespray.dind.inventory_builder.sh - # expand $extra with -e in front of each word - extra_args=""; for extra_arg in $extra; do extra_args="$extra_args -e $extra_arg"; done - ansible-playbook --become -e ansible_ssh_user=$distro -i \ - ${INVENTORY_DIR}/hosts.ini cluster.yml \ - -e @contrib/dind/kubespray-dind.yaml -e bootstrap_os=$distro ${extra_args} - pass_or_fail "$prefix: kubespray" - ) || return 1 - local node0=${NODES[0]} - docker exec ${node0} kubectl get pod --all-namespaces - pass_or_fail "$prefix: kube-api" || return 1 - let retries=60 - while ((retries--)); do - # Some CNI may set NodePort on "main" node interface address (thus no localhost NodePort) - # e.g. kube-router: https://github.com/cloudnativelabs/kube-router/pull/217 - docker exec ${node0} curl -m2 -s http://${NETCHECKER_HOST:?}:31081/api/v1/connectivity_check | grep successfully && break - sleep 2 - done - [ $retries -ge 0 ] - pass_or_fail "$prefix: netcheck" || return 1 -} - -NODES=($(egrep ^kube_node hosts)) -NETCHECKER_HOST=localhost - -: ${OUTPUT_DIR:=./out} -mkdir -p ${OUTPUT_DIR} - -# The SPEC file(s) must have two arrays as e.g. -# DISTROS=(debian centos) -# EXTRAS=( -# 'kube_network_plugin=calico' -# 'kube_network_plugin=flannel' -# 'kube_network_plugin=weave' -# ) -# that will be tested in a "combinatory" way (e.g. from above there'll be -# be 6 test runs), creating a sequenced -nn.out with each output. -# -# Each $EXTRAS element will be whitespace split, and passed as --extra-vars -# to main kubespray ansible-playbook run. - -SPECS=${*:?Missing SPEC files, e.g. test-most_distros-some_CNIs.env} -for spec in ${SPECS}; do - unset DISTROS EXTRAS - echo "Loading file=${spec} ..." - . ${spec} || continue - : ${DISTROS:?} || continue - echo "DISTROS:" "${DISTROS[@]}" - echo "EXTRAS->" - printf " %s\n" "${EXTRAS[@]}" - let n=1 - for distro in "${DISTROS[@]}"; do - for extra in "${EXTRAS[@]:-NULL}"; do - # Magic value to let this for run once: - [[ ${extra} == NULL ]] && unset extra - docker rm -f "${NODES[@]}" - printf -v file_out "%s/%s-%02d.out" ${OUTPUT_DIR} ${spec} $((n++)) - { - info "${distro}[${extra}] START: file_out=${file_out}" - time test_distro ${distro} ${extra} - } |& tee ${file_out} - # sleeping for the sake of the human to verify if they want - sleep 2m - done - done -done -egrep -H '^(....:|real)' $(ls -tr ${OUTPUT_DIR}/*.out) diff --git a/contrib/dind/test-most_distros-some_CNIs.env b/contrib/dind/test-most_distros-some_CNIs.env deleted file mode 100644 index f6e4e1a8ca3..00000000000 --- a/contrib/dind/test-most_distros-some_CNIs.env +++ /dev/null @@ -1,11 +0,0 @@ -# Test spec file: used from ./run-test-distros.sh, will run -# each distro in $DISTROS overloading main kubespray ansible-playbook run -# Get all DISTROS from distro.yaml (shame no yaml parsing, but nuff anyway) -# DISTROS="${*:-$(egrep -o '^ \w+' group_vars/all/distro.yaml|paste -s)}" -DISTROS=(debian ubuntu centos fedora) - -# Each line below will be added as --extra-vars to main playbook run -EXTRAS=( - 'kube_network_plugin=calico' - 'kube_network_plugin=weave' -) diff --git a/contrib/dind/test-some_distros-kube_router_combo.env b/contrib/dind/test-some_distros-kube_router_combo.env deleted file mode 100644 index f2677129be6..00000000000 --- a/contrib/dind/test-some_distros-kube_router_combo.env +++ /dev/null @@ -1,6 +0,0 @@ -DISTROS=(debian centos) -NETCHECKER_HOST=${NODES[0]} -EXTRAS=( - 'kube_network_plugin=kube-router {"kube_router_run_service_proxy":false}' - 'kube_network_plugin=kube-router {"kube_router_run_service_proxy":true}' -) diff --git a/contrib/dind/test-some_distros-most_CNIs.env b/contrib/dind/test-some_distros-most_CNIs.env deleted file mode 100644 index 2fb185c156a..00000000000 --- a/contrib/dind/test-some_distros-most_CNIs.env +++ /dev/null @@ -1,8 +0,0 @@ -DISTROS=(debian centos) -EXTRAS=( - 'kube_network_plugin=calico {}' - 'kube_network_plugin=canal {}' - 'kube_network_plugin=cilium {}' - 'kube_network_plugin=flannel {}' - 'kube_network_plugin=weave {}' -) diff --git a/contrib/inventory_builder/inventory.py b/contrib/inventory_builder/inventory.py deleted file mode 100644 index 76e7c0c4664..00000000000 --- a/contrib/inventory_builder/inventory.py +++ /dev/null @@ -1,480 +0,0 @@ -#!/usr/bin/env python3 -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Usage: inventory.py ip1 [ip2 ...] -# Examples: inventory.py 10.10.1.3 10.10.1.4 10.10.1.5 -# -# Advanced usage: -# Add another host after initial creation: inventory.py 10.10.1.5 -# Add range of hosts: inventory.py 10.10.1.3-10.10.1.5 -# Add hosts with different ip and access ip: -# inventory.py 10.0.0.1,192.168.10.1 10.0.0.2,192.168.10.2 10.0.0.3,192.168.1.3 -# Add hosts with a specific hostname, ip, and optional access ip: -# inventory.py first,10.0.0.1,192.168.10.1 second,10.0.0.2 last,10.0.0.3 -# Delete a host: inventory.py -10.10.1.3 -# Delete a host by id: inventory.py -node1 -# -# Load a YAML or JSON file with inventory data: inventory.py load hosts.yaml -# YAML file should be in the following format: -# group1: -# host1: -# ip: X.X.X.X -# var: val -# group2: -# host2: -# ip: X.X.X.X - -from collections import OrderedDict -from ipaddress import ip_address -from ruamel.yaml import YAML - -import os -import re -import subprocess -import sys - -ROLES = ['all', 'kube_control_plane', 'kube_node', 'etcd', 'k8s_cluster', - 'calico_rr'] -PROTECTED_NAMES = ROLES -AVAILABLE_COMMANDS = ['help', 'print_cfg', 'print_ips', 'print_hostnames', - 'load', 'add'] -_boolean_states = {'1': True, 'yes': True, 'true': True, 'on': True, - '0': False, 'no': False, 'false': False, 'off': False} -yaml = YAML() -yaml.Representer.add_representer(OrderedDict, yaml.Representer.represent_dict) - - -def get_var_as_bool(name, default): - value = os.environ.get(name, '') - return _boolean_states.get(value.lower(), default) - -# Configurable as shell vars start - - -CONFIG_FILE = os.environ.get("CONFIG_FILE", "./inventory/sample/hosts.yaml") -# Remove the reference of KUBE_MASTERS after some deprecation cycles. -KUBE_CONTROL_HOSTS = int(os.environ.get("KUBE_CONTROL_HOSTS", - os.environ.get("KUBE_MASTERS", 2))) -# Reconfigures cluster distribution at scale -SCALE_THRESHOLD = int(os.environ.get("SCALE_THRESHOLD", 50)) -MASSIVE_SCALE_THRESHOLD = int(os.environ.get("MASSIVE_SCALE_THRESHOLD", 200)) - -DEBUG = get_var_as_bool("DEBUG", True) -HOST_PREFIX = os.environ.get("HOST_PREFIX", "node") -USE_REAL_HOSTNAME = get_var_as_bool("USE_REAL_HOSTNAME", False) - -# Configurable as shell vars end - - -class KubesprayInventory(object): - - def __init__(self, changed_hosts=None, config_file=None): - self.config_file = config_file - self.yaml_config = {} - loadPreviousConfig = False - printHostnames = False - # See whether there are any commands to process - if changed_hosts and changed_hosts[0] in AVAILABLE_COMMANDS: - if changed_hosts[0] == "add": - loadPreviousConfig = True - changed_hosts = changed_hosts[1:] - elif changed_hosts[0] == "print_hostnames": - loadPreviousConfig = True - printHostnames = True - else: - self.parse_command(changed_hosts[0], changed_hosts[1:]) - sys.exit(0) - - # If the user wants to remove a node, we need to load the config anyway - if changed_hosts and changed_hosts[0][0] == "-": - loadPreviousConfig = True - - if self.config_file and loadPreviousConfig: # Load previous YAML file - try: - self.hosts_file = open(config_file, 'r') - self.yaml_config = yaml.load(self.hosts_file) - except OSError as e: - # I am assuming we are catching "cannot open file" exceptions - print(e) - sys.exit(1) - - if printHostnames: - self.print_hostnames() - sys.exit(0) - - self.ensure_required_groups(ROLES) - - if changed_hosts: - changed_hosts = self.range2ips(changed_hosts) - self.hosts = self.build_hostnames(changed_hosts, - loadPreviousConfig) - self.purge_invalid_hosts(self.hosts.keys(), PROTECTED_NAMES) - self.set_all(self.hosts) - self.set_k8s_cluster() - etcd_hosts_count = 3 if len(self.hosts.keys()) >= 3 else 1 - self.set_etcd(list(self.hosts.keys())[:etcd_hosts_count]) - if len(self.hosts) >= SCALE_THRESHOLD: - self.set_kube_control_plane(list(self.hosts.keys())[ - etcd_hosts_count:(etcd_hosts_count + KUBE_CONTROL_HOSTS)]) - else: - self.set_kube_control_plane( - list(self.hosts.keys())[:KUBE_CONTROL_HOSTS]) - self.set_kube_node(self.hosts.keys()) - if len(self.hosts) >= SCALE_THRESHOLD: - self.set_calico_rr(list(self.hosts.keys())[:etcd_hosts_count]) - else: # Show help if no options - self.show_help() - sys.exit(0) - - self.write_config(self.config_file) - - def write_config(self, config_file): - if config_file: - with open(self.config_file, 'w') as f: - yaml.dump(self.yaml_config, f) - - else: - print("WARNING: Unable to save config. Make sure you set " - "CONFIG_FILE env var.") - - def debug(self, msg): - if DEBUG: - print("DEBUG: {0}".format(msg)) - - def get_ip_from_opts(self, optstring): - if 'ip' in optstring: - return optstring['ip'] - else: - raise ValueError("IP parameter not found in options") - - def ensure_required_groups(self, groups): - for group in groups: - if group == 'all': - self.debug("Adding group {0}".format(group)) - if group not in self.yaml_config: - all_dict = OrderedDict([('hosts', OrderedDict({})), - ('children', OrderedDict({}))]) - self.yaml_config = {'all': all_dict} - else: - self.debug("Adding group {0}".format(group)) - if group not in self.yaml_config['all']['children']: - self.yaml_config['all']['children'][group] = {'hosts': {}} - - def get_host_id(self, host): - '''Returns integer host ID (without padding) from a given hostname.''' - try: - short_hostname = host.split('.')[0] - return int(re.findall("\\d+$", short_hostname)[-1]) - except IndexError: - raise ValueError("Host name must end in an integer") - - # Keeps already specified hosts, - # and adds or removes the hosts provided as an argument - def build_hostnames(self, changed_hosts, loadPreviousConfig=False): - existing_hosts = OrderedDict() - highest_host_id = 0 - # Load already existing hosts from the YAML - if loadPreviousConfig: - try: - for host in self.yaml_config['all']['hosts']: - # Read configuration of an existing host - hostConfig = self.yaml_config['all']['hosts'][host] - existing_hosts[host] = hostConfig - # If the existing host seems - # to have been created automatically, detect its ID - if host.startswith(HOST_PREFIX): - host_id = self.get_host_id(host) - if host_id > highest_host_id: - highest_host_id = host_id - except Exception as e: - # I am assuming we are catching automatically - # created hosts without IDs - print(e) - sys.exit(1) - - # FIXME(mattymo): Fix condition where delete then add reuses highest id - next_host_id = highest_host_id + 1 - next_host = "" - - all_hosts = existing_hosts.copy() - for host in changed_hosts: - # Delete the host from config the hostname/IP has a "-" prefix - if host[0] == "-": - realhost = host[1:] - if self.exists_hostname(all_hosts, realhost): - self.debug("Marked {0} for deletion.".format(realhost)) - all_hosts.pop(realhost) - elif self.exists_ip(all_hosts, realhost): - self.debug("Marked {0} for deletion.".format(realhost)) - self.delete_host_by_ip(all_hosts, realhost) - # Host/Argument starts with a digit, - # then we assume its an IP address - elif host[0].isdigit(): - if ',' in host: - ip, access_ip = host.split(',') - else: - ip = host - access_ip = host - if self.exists_hostname(all_hosts, host): - self.debug("Skipping existing host {0}.".format(host)) - continue - elif self.exists_ip(all_hosts, ip): - self.debug("Skipping existing host {0}.".format(ip)) - continue - - if USE_REAL_HOSTNAME: - cmd = ("ssh -oStrictHostKeyChecking=no " - + access_ip + " 'hostname -s'") - next_host = subprocess.check_output(cmd, shell=True) - next_host = next_host.strip().decode('ascii') - else: - # Generates a hostname because we have only an IP address - next_host = "{0}{1}".format(HOST_PREFIX, next_host_id) - next_host_id += 1 - # Uses automatically generated node name - # in case we dont provide it. - all_hosts[next_host] = {'ansible_host': access_ip, - 'ip': ip, - 'access_ip': access_ip} - # Host/Argument starts with a letter, then we assume its a hostname - elif host[0].isalpha(): - if ',' in host: - try: - hostname, ip, access_ip = host.split(',') - except Exception: - hostname, ip = host.split(',') - access_ip = ip - if self.exists_hostname(all_hosts, host): - self.debug("Skipping existing host {0}.".format(host)) - continue - elif self.exists_ip(all_hosts, ip): - self.debug("Skipping existing host {0}.".format(ip)) - continue - all_hosts[hostname] = {'ansible_host': access_ip, - 'ip': ip, - 'access_ip': access_ip} - return all_hosts - - # Expand IP ranges into individual addresses - def range2ips(self, hosts): - reworked_hosts = [] - - def ips(start_address, end_address): - try: - # Python 3.x - start = int(ip_address(start_address)) - end = int(ip_address(end_address)) - except Exception: - # Python 2.7 - start = int(ip_address(str(start_address))) - end = int(ip_address(str(end_address))) - return [ip_address(ip).exploded for ip in range(start, end + 1)] - - for host in hosts: - if '-' in host and not (host.startswith('-') or host[0].isalpha()): - start, end = host.strip().split('-') - try: - reworked_hosts.extend(ips(start, end)) - except ValueError: - raise Exception("Range of ip_addresses isn't valid") - else: - reworked_hosts.append(host) - return reworked_hosts - - def exists_hostname(self, existing_hosts, hostname): - return hostname in existing_hosts.keys() - - def exists_ip(self, existing_hosts, ip): - for host_opts in existing_hosts.values(): - if ip == self.get_ip_from_opts(host_opts): - return True - return False - - def delete_host_by_ip(self, existing_hosts, ip): - for hostname, host_opts in existing_hosts.items(): - if ip == self.get_ip_from_opts(host_opts): - del existing_hosts[hostname] - return - raise ValueError("Unable to find host by IP: {0}".format(ip)) - - def purge_invalid_hosts(self, hostnames, protected_names=[]): - for role in self.yaml_config['all']['children']: - if role != 'k8s_cluster' and self.yaml_config['all']['children'][role]['hosts']: # noqa - all_hosts = self.yaml_config['all']['children'][role]['hosts'].copy() # noqa - for host in all_hosts.keys(): - if host not in hostnames and host not in protected_names: - self.debug( - "Host {0} removed from role {1}".format(host, role)) # noqa - del self.yaml_config['all']['children'][role]['hosts'][host] # noqa - # purge from all - if self.yaml_config['all']['hosts']: - all_hosts = self.yaml_config['all']['hosts'].copy() - for host in all_hosts.keys(): - if host not in hostnames and host not in protected_names: - self.debug("Host {0} removed from role all".format(host)) - del self.yaml_config['all']['hosts'][host] - - def add_host_to_group(self, group, host, opts=""): - self.debug("adding host {0} to group {1}".format(host, group)) - if group == 'all': - if self.yaml_config['all']['hosts'] is None: - self.yaml_config['all']['hosts'] = {host: None} - self.yaml_config['all']['hosts'][host] = opts - elif group != 'k8s_cluster:children': - if self.yaml_config['all']['children'][group]['hosts'] is None: - self.yaml_config['all']['children'][group]['hosts'] = { - host: None} - else: - self.yaml_config['all']['children'][group]['hosts'][host] = None # noqa - - def set_kube_control_plane(self, hosts): - for host in hosts: - self.add_host_to_group('kube_control_plane', host) - - def set_all(self, hosts): - for host, opts in hosts.items(): - self.add_host_to_group('all', host, opts) - - def set_k8s_cluster(self): - k8s_cluster = {'children': {'kube_control_plane': None, - 'kube_node': None}} - self.yaml_config['all']['children']['k8s_cluster'] = k8s_cluster - - def set_calico_rr(self, hosts): - for host in hosts: - if host in self.yaml_config['all']['children']['kube_control_plane']: # noqa - self.debug("Not adding {0} to calico_rr group because it " - "conflicts with kube_control_plane " - "group".format(host)) - continue - if host in self.yaml_config['all']['children']['kube_node']: - self.debug("Not adding {0} to calico_rr group because it " - "conflicts with kube_node group".format(host)) - continue - self.add_host_to_group('calico_rr', host) - - def set_kube_node(self, hosts): - for host in hosts: - if len(self.yaml_config['all']['hosts']) >= SCALE_THRESHOLD: - if host in self.yaml_config['all']['children']['etcd']['hosts']: # noqa - self.debug("Not adding {0} to kube_node group because of " - "scale deployment and host is in etcd " - "group.".format(host)) - continue - if len(self.yaml_config['all']['hosts']) >= MASSIVE_SCALE_THRESHOLD: # noqa - if host in self.yaml_config['all']['children']['kube_control_plane']['hosts']: # noqa - self.debug("Not adding {0} to kube_node group because of " - "scale deployment and host is in " - "kube_control_plane group.".format(host)) - continue - self.add_host_to_group('kube_node', host) - - def set_etcd(self, hosts): - for host in hosts: - self.add_host_to_group('etcd', host) - - def load_file(self, files=None): - '''Directly loads JSON to inventory.''' - - if not files: - raise Exception("No input file specified.") - - import json - - for filename in list(files): - # Try JSON - try: - with open(filename, 'r') as f: - data = json.load(f) - except ValueError: - raise Exception("Cannot read %s as JSON, or CSV", filename) - - self.ensure_required_groups(ROLES) - self.set_k8s_cluster() - for group, hosts in data.items(): - self.ensure_required_groups([group]) - for host, opts in hosts.items(): - optstring = {'ansible_host': opts['ip'], - 'ip': opts['ip'], - 'access_ip': opts['ip']} - self.add_host_to_group('all', host, optstring) - self.add_host_to_group(group, host) - self.write_config(self.config_file) - - def parse_command(self, command, args=None): - if command == 'help': - self.show_help() - elif command == 'print_cfg': - self.print_config() - elif command == 'print_ips': - self.print_ips() - elif command == 'print_hostnames': - self.print_hostnames() - elif command == 'load': - self.load_file(args) - else: - raise Exception("Invalid command specified.") - - def show_help(self): - help_text = '''Usage: inventory.py ip1 [ip2 ...] -Examples: inventory.py 10.10.1.3 10.10.1.4 10.10.1.5 - -Available commands: -help - Display this message -print_cfg - Write inventory file to stdout -print_ips - Write a space-delimited list of IPs from "all" group -print_hostnames - Write a space-delimited list of Hostnames from "all" group -add - Adds specified hosts into an already existing inventory - -Advanced usage: -Create new or overwrite old inventory file: inventory.py 10.10.1.5 -Add another host after initial creation: inventory.py add 10.10.1.6 -Add range of hosts: inventory.py 10.10.1.3-10.10.1.5 -Add hosts with different ip and access ip: inventory.py 10.0.0.1,192.168.10.1 10.0.0.2,192.168.10.2 10.0.0.3,192.168.10.3 -Add hosts with a specific hostname, ip, and optional access ip: first,10.0.0.1,192.168.10.1 second,10.0.0.2 last,10.0.0.3 -Delete a host: inventory.py -10.10.1.3 -Delete a host by id: inventory.py -node1 - -Configurable env vars: -DEBUG Enable debug printing. Default: True -CONFIG_FILE File to write config to Default: ./inventory/sample/hosts.yaml -HOST_PREFIX Host prefix for generated hosts. Default: node -KUBE_CONTROL_HOSTS Set the number of kube-control-planes. Default: 2 -SCALE_THRESHOLD Separate ETCD role if # of nodes >= 50 -MASSIVE_SCALE_THRESHOLD Separate K8s control-plane and ETCD if # of nodes >= 200 -''' # noqa - print(help_text) - - def print_config(self): - yaml.dump(self.yaml_config, sys.stdout) - - def print_hostnames(self): - print(' '.join(self.yaml_config['all']['hosts'].keys())) - - def print_ips(self): - ips = [] - for host, opts in self.yaml_config['all']['hosts'].items(): - ips.append(self.get_ip_from_opts(opts)) - print(' '.join(ips)) - - -def main(argv=None): - if not argv: - argv = sys.argv[1:] - KubesprayInventory(argv, CONFIG_FILE) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/contrib/inventory_builder/requirements.txt b/contrib/inventory_builder/requirements.txt deleted file mode 100644 index c54501a4bd3..00000000000 --- a/contrib/inventory_builder/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -configparser>=3.3.0 -ipaddress -ruamel.yaml>=0.15.88 diff --git a/contrib/inventory_builder/setup.cfg b/contrib/inventory_builder/setup.cfg deleted file mode 100644 index a775367e200..00000000000 --- a/contrib/inventory_builder/setup.cfg +++ /dev/null @@ -1,3 +0,0 @@ -[metadata] -name = kubespray-inventory-builder -version = 0.1 diff --git a/contrib/inventory_builder/setup.py b/contrib/inventory_builder/setup.py deleted file mode 100644 index 43c5ca1b496..00000000000 --- a/contrib/inventory_builder/setup.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT -import setuptools - -# In python < 2.7.4, a lazy loading of package `pbr` will break -# setuptools if some other modules registered functions in `atexit`. -# solution from: http://bugs.python.org/issue15881#msg170215 -try: - import multiprocessing # noqa -except ImportError: - pass - -setuptools.setup( - setup_requires=[], - pbr=False) diff --git a/contrib/inventory_builder/test-requirements.txt b/contrib/inventory_builder/test-requirements.txt deleted file mode 100644 index 98a662a4dde..00000000000 --- a/contrib/inventory_builder/test-requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -hacking>=0.10.2 -mock>=1.3.0 -pytest>=2.8.0 diff --git a/contrib/inventory_builder/tests/test_inventory.py b/contrib/inventory_builder/tests/test_inventory.py deleted file mode 100644 index 5d6649d6830..00000000000 --- a/contrib/inventory_builder/tests/test_inventory.py +++ /dev/null @@ -1,595 +0,0 @@ -# Copyright 2016 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import inventory -from io import StringIO -import unittest -from unittest import mock - -from collections import OrderedDict -import sys - -path = "./contrib/inventory_builder/" -if path not in sys.path: - sys.path.append(path) - -import inventory # noqa - - -class TestInventoryPrintHostnames(unittest.TestCase): - - @mock.patch('ruamel.yaml.YAML.load') - def test_print_hostnames(self, load_mock): - mock_io = mock.mock_open(read_data='') - load_mock.return_value = OrderedDict({'all': {'hosts': { - 'node1': {'ansible_host': '10.90.0.2', - 'ip': '10.90.0.2', - 'access_ip': '10.90.0.2'}, - 'node2': {'ansible_host': '10.90.0.3', - 'ip': '10.90.0.3', - 'access_ip': '10.90.0.3'}}}}) - with mock.patch('builtins.open', mock_io): - with self.assertRaises(SystemExit) as cm: - with mock.patch('sys.stdout', new_callable=StringIO) as stdout: - inventory.KubesprayInventory( - changed_hosts=["print_hostnames"], - config_file="file") - self.assertEqual("node1 node2\n", stdout.getvalue()) - self.assertEqual(cm.exception.code, 0) - - -class TestInventory(unittest.TestCase): - @mock.patch('inventory.sys') - def setUp(self, sys_mock): - sys_mock.exit = mock.Mock() - super(TestInventory, self).setUp() - self.data = ['10.90.3.2', '10.90.3.3', '10.90.3.4'] - self.inv = inventory.KubesprayInventory() - - def test_get_ip_from_opts(self): - optstring = {'ansible_host': '10.90.3.2', - 'ip': '10.90.3.2', - 'access_ip': '10.90.3.2'} - expected = "10.90.3.2" - result = self.inv.get_ip_from_opts(optstring) - self.assertEqual(expected, result) - - def test_get_ip_from_opts_invalid(self): - optstring = "notanaddr=value something random!chars:D" - self.assertRaisesRegex(ValueError, "IP parameter not found", - self.inv.get_ip_from_opts, optstring) - - def test_ensure_required_groups(self): - groups = ['group1', 'group2'] - self.inv.ensure_required_groups(groups) - for group in groups: - self.assertIn(group, self.inv.yaml_config['all']['children']) - - def test_get_host_id(self): - hostnames = ['node99', 'no99de01', '01node01', 'node1.domain', - 'node3.xyz123.aaa'] - expected = [99, 1, 1, 1, 3] - for hostname, expected in zip(hostnames, expected): - result = self.inv.get_host_id(hostname) - self.assertEqual(expected, result) - - def test_get_host_id_invalid(self): - bad_hostnames = ['node', 'no99de', '01node', 'node.111111'] - for hostname in bad_hostnames: - self.assertRaisesRegex(ValueError, "Host name must end in an", - self.inv.get_host_id, hostname) - - def test_build_hostnames_add_duplicate(self): - changed_hosts = ['10.90.0.2'] - expected = OrderedDict([('node3', - {'ansible_host': '10.90.0.2', - 'ip': '10.90.0.2', - 'access_ip': '10.90.0.2'})]) - self.inv.yaml_config['all']['hosts'] = expected - result = self.inv.build_hostnames(changed_hosts, True) - self.assertEqual(expected, result) - - def test_build_hostnames_add_two(self): - changed_hosts = ['10.90.0.2', '10.90.0.3'] - expected = OrderedDict([ - ('node1', {'ansible_host': '10.90.0.2', - 'ip': '10.90.0.2', - 'access_ip': '10.90.0.2'}), - ('node2', {'ansible_host': '10.90.0.3', - 'ip': '10.90.0.3', - 'access_ip': '10.90.0.3'})]) - self.inv.yaml_config['all']['hosts'] = OrderedDict() - result = self.inv.build_hostnames(changed_hosts) - self.assertEqual(expected, result) - - def test_build_hostnames_add_three(self): - changed_hosts = ['10.90.0.2', '10.90.0.3', '10.90.0.4'] - expected = OrderedDict([ - ('node1', {'ansible_host': '10.90.0.2', - 'ip': '10.90.0.2', - 'access_ip': '10.90.0.2'}), - ('node2', {'ansible_host': '10.90.0.3', - 'ip': '10.90.0.3', - 'access_ip': '10.90.0.3'}), - ('node3', {'ansible_host': '10.90.0.4', - 'ip': '10.90.0.4', - 'access_ip': '10.90.0.4'})]) - result = self.inv.build_hostnames(changed_hosts) - self.assertEqual(expected, result) - - def test_build_hostnames_add_one(self): - changed_hosts = ['10.90.0.2'] - expected = OrderedDict([('node1', - {'ansible_host': '10.90.0.2', - 'ip': '10.90.0.2', - 'access_ip': '10.90.0.2'})]) - result = self.inv.build_hostnames(changed_hosts) - self.assertEqual(expected, result) - - def test_build_hostnames_delete_first(self): - changed_hosts = ['-10.90.0.2'] - existing_hosts = OrderedDict([ - ('node1', {'ansible_host': '10.90.0.2', - 'ip': '10.90.0.2', - 'access_ip': '10.90.0.2'}), - ('node2', {'ansible_host': '10.90.0.3', - 'ip': '10.90.0.3', - 'access_ip': '10.90.0.3'})]) - self.inv.yaml_config['all']['hosts'] = existing_hosts - expected = OrderedDict([ - ('node2', {'ansible_host': '10.90.0.3', - 'ip': '10.90.0.3', - 'access_ip': '10.90.0.3'})]) - result = self.inv.build_hostnames(changed_hosts, True) - self.assertEqual(expected, result) - - def test_build_hostnames_delete_by_hostname(self): - changed_hosts = ['-node1'] - existing_hosts = OrderedDict([ - ('node1', {'ansible_host': '10.90.0.2', - 'ip': '10.90.0.2', - 'access_ip': '10.90.0.2'}), - ('node2', {'ansible_host': '10.90.0.3', - 'ip': '10.90.0.3', - 'access_ip': '10.90.0.3'})]) - self.inv.yaml_config['all']['hosts'] = existing_hosts - expected = OrderedDict([ - ('node2', {'ansible_host': '10.90.0.3', - 'ip': '10.90.0.3', - 'access_ip': '10.90.0.3'})]) - result = self.inv.build_hostnames(changed_hosts, True) - self.assertEqual(expected, result) - - def test_exists_hostname_positive(self): - hostname = 'node1' - expected = True - existing_hosts = OrderedDict([ - ('node1', {'ansible_host': '10.90.0.2', - 'ip': '10.90.0.2', - 'access_ip': '10.90.0.2'}), - ('node2', {'ansible_host': '10.90.0.3', - 'ip': '10.90.0.3', - 'access_ip': '10.90.0.3'})]) - result = self.inv.exists_hostname(existing_hosts, hostname) - self.assertEqual(expected, result) - - def test_exists_hostname_negative(self): - hostname = 'node99' - expected = False - existing_hosts = OrderedDict([ - ('node1', {'ansible_host': '10.90.0.2', - 'ip': '10.90.0.2', - 'access_ip': '10.90.0.2'}), - ('node2', {'ansible_host': '10.90.0.3', - 'ip': '10.90.0.3', - 'access_ip': '10.90.0.3'})]) - result = self.inv.exists_hostname(existing_hosts, hostname) - self.assertEqual(expected, result) - - def test_exists_ip_positive(self): - ip = '10.90.0.2' - expected = True - existing_hosts = OrderedDict([ - ('node1', {'ansible_host': '10.90.0.2', - 'ip': '10.90.0.2', - 'access_ip': '10.90.0.2'}), - ('node2', {'ansible_host': '10.90.0.3', - 'ip': '10.90.0.3', - 'access_ip': '10.90.0.3'})]) - result = self.inv.exists_ip(existing_hosts, ip) - self.assertEqual(expected, result) - - def test_exists_ip_negative(self): - ip = '10.90.0.200' - expected = False - existing_hosts = OrderedDict([ - ('node1', {'ansible_host': '10.90.0.2', - 'ip': '10.90.0.2', - 'access_ip': '10.90.0.2'}), - ('node2', {'ansible_host': '10.90.0.3', - 'ip': '10.90.0.3', - 'access_ip': '10.90.0.3'})]) - result = self.inv.exists_ip(existing_hosts, ip) - self.assertEqual(expected, result) - - def test_delete_host_by_ip_positive(self): - ip = '10.90.0.2' - expected = OrderedDict([ - ('node2', {'ansible_host': '10.90.0.3', - 'ip': '10.90.0.3', - 'access_ip': '10.90.0.3'})]) - existing_hosts = OrderedDict([ - ('node1', {'ansible_host': '10.90.0.2', - 'ip': '10.90.0.2', - 'access_ip': '10.90.0.2'}), - ('node2', {'ansible_host': '10.90.0.3', - 'ip': '10.90.0.3', - 'access_ip': '10.90.0.3'})]) - self.inv.delete_host_by_ip(existing_hosts, ip) - self.assertEqual(expected, existing_hosts) - - def test_delete_host_by_ip_negative(self): - ip = '10.90.0.200' - existing_hosts = OrderedDict([ - ('node1', {'ansible_host': '10.90.0.2', - 'ip': '10.90.0.2', - 'access_ip': '10.90.0.2'}), - ('node2', {'ansible_host': '10.90.0.3', - 'ip': '10.90.0.3', - 'access_ip': '10.90.0.3'})]) - self.assertRaisesRegex(ValueError, "Unable to find host", - self.inv.delete_host_by_ip, existing_hosts, ip) - - def test_purge_invalid_hosts(self): - proper_hostnames = ['node1', 'node2'] - bad_host = 'doesnotbelong2' - existing_hosts = OrderedDict([ - ('node1', {'ansible_host': '10.90.0.2', - 'ip': '10.90.0.2', - 'access_ip': '10.90.0.2'}), - ('node2', {'ansible_host': '10.90.0.3', - 'ip': '10.90.0.3', - 'access_ip': '10.90.0.3'}), - ('doesnotbelong2', {'whateveropts=ilike'})]) - self.inv.yaml_config['all']['hosts'] = existing_hosts - self.inv.purge_invalid_hosts(proper_hostnames) - self.assertNotIn( - bad_host, self.inv.yaml_config['all']['hosts'].keys()) - - def test_add_host_to_group(self): - group = 'etcd' - host = 'node1' - opts = {'ip': '10.90.0.2'} - - self.inv.add_host_to_group(group, host, opts) - self.assertEqual( - self.inv.yaml_config['all']['children'][group]['hosts'].get(host), - None) - - def test_set_kube_control_plane(self): - group = 'kube_control_plane' - host = 'node1' - - self.inv.set_kube_control_plane([host]) - self.assertIn( - host, self.inv.yaml_config['all']['children'][group]['hosts']) - - def test_set_all(self): - hosts = OrderedDict([ - ('node1', 'opt1'), - ('node2', 'opt2')]) - - self.inv.set_all(hosts) - for host, opt in hosts.items(): - self.assertEqual( - self.inv.yaml_config['all']['hosts'].get(host), opt) - - def test_set_k8s_cluster(self): - group = 'k8s_cluster' - expected_hosts = ['kube_node', 'kube_control_plane'] - - self.inv.set_k8s_cluster() - for host in expected_hosts: - self.assertIn( - host, - self.inv.yaml_config['all']['children'][group]['children']) - - def test_set_kube_node(self): - group = 'kube_node' - host = 'node1' - - self.inv.set_kube_node([host]) - self.assertIn( - host, self.inv.yaml_config['all']['children'][group]['hosts']) - - def test_set_etcd(self): - group = 'etcd' - host = 'node1' - - self.inv.set_etcd([host]) - self.assertIn( - host, self.inv.yaml_config['all']['children'][group]['hosts']) - - def test_scale_scenario_one(self): - num_nodes = 50 - hosts = OrderedDict() - - for hostid in range(1, num_nodes+1): - hosts["node" + str(hostid)] = "" - - self.inv.set_all(hosts) - self.inv.set_etcd(list(hosts.keys())[0:3]) - self.inv.set_kube_control_plane(list(hosts.keys())[0:2]) - self.inv.set_kube_node(hosts.keys()) - for h in range(3): - self.assertFalse( - list(hosts.keys())[h] in - self.inv.yaml_config['all']['children']['kube_node']['hosts']) - - def test_scale_scenario_two(self): - num_nodes = 500 - hosts = OrderedDict() - - for hostid in range(1, num_nodes+1): - hosts["node" + str(hostid)] = "" - - self.inv.set_all(hosts) - self.inv.set_etcd(list(hosts.keys())[0:3]) - self.inv.set_kube_control_plane(list(hosts.keys())[3:5]) - self.inv.set_kube_node(hosts.keys()) - for h in range(5): - self.assertFalse( - list(hosts.keys())[h] in - self.inv.yaml_config['all']['children']['kube_node']['hosts']) - - def test_range2ips_range(self): - changed_hosts = ['10.90.0.2', '10.90.0.4-10.90.0.6', '10.90.0.8'] - expected = ['10.90.0.2', - '10.90.0.4', - '10.90.0.5', - '10.90.0.6', - '10.90.0.8'] - result = self.inv.range2ips(changed_hosts) - self.assertEqual(expected, result) - - def test_range2ips_incorrect_range(self): - host_range = ['10.90.0.4-a.9b.c.e'] - self.assertRaisesRegex(Exception, "Range of ip_addresses isn't valid", - self.inv.range2ips, host_range) - - def test_build_hostnames_create_with_one_different_ips(self): - changed_hosts = ['10.90.0.2,192.168.0.2'] - expected = OrderedDict([('node1', - {'ansible_host': '192.168.0.2', - 'ip': '10.90.0.2', - 'access_ip': '192.168.0.2'})]) - result = self.inv.build_hostnames(changed_hosts) - self.assertEqual(expected, result) - - def test_build_hostnames_create_with_two_different_ips(self): - changed_hosts = ['10.90.0.2,192.168.0.2', '10.90.0.3,192.168.0.3'] - expected = OrderedDict([ - ('node1', {'ansible_host': '192.168.0.2', - 'ip': '10.90.0.2', - 'access_ip': '192.168.0.2'}), - ('node2', {'ansible_host': '192.168.0.3', - 'ip': '10.90.0.3', - 'access_ip': '192.168.0.3'})]) - result = self.inv.build_hostnames(changed_hosts) - self.assertEqual(expected, result) - - def test_build_hostnames_create_with_three_different_ips(self): - changed_hosts = ['10.90.0.2,192.168.0.2', - '10.90.0.3,192.168.0.3', - '10.90.0.4,192.168.0.4'] - expected = OrderedDict([ - ('node1', {'ansible_host': '192.168.0.2', - 'ip': '10.90.0.2', - 'access_ip': '192.168.0.2'}), - ('node2', {'ansible_host': '192.168.0.3', - 'ip': '10.90.0.3', - 'access_ip': '192.168.0.3'}), - ('node3', {'ansible_host': '192.168.0.4', - 'ip': '10.90.0.4', - 'access_ip': '192.168.0.4'})]) - result = self.inv.build_hostnames(changed_hosts) - self.assertEqual(expected, result) - - def test_build_hostnames_overwrite_one_with_different_ips(self): - changed_hosts = ['10.90.0.2,192.168.0.2'] - expected = OrderedDict([('node1', - {'ansible_host': '192.168.0.2', - 'ip': '10.90.0.2', - 'access_ip': '192.168.0.2'})]) - existing = OrderedDict([('node5', - {'ansible_host': '192.168.0.5', - 'ip': '10.90.0.5', - 'access_ip': '192.168.0.5'})]) - self.inv.yaml_config['all']['hosts'] = existing - result = self.inv.build_hostnames(changed_hosts) - self.assertEqual(expected, result) - - def test_build_hostnames_overwrite_three_with_different_ips(self): - changed_hosts = ['10.90.0.2,192.168.0.2'] - expected = OrderedDict([('node1', - {'ansible_host': '192.168.0.2', - 'ip': '10.90.0.2', - 'access_ip': '192.168.0.2'})]) - existing = OrderedDict([ - ('node3', {'ansible_host': '192.168.0.3', - 'ip': '10.90.0.3', - 'access_ip': '192.168.0.3'}), - ('node4', {'ansible_host': '192.168.0.4', - 'ip': '10.90.0.4', - 'access_ip': '192.168.0.4'}), - ('node5', {'ansible_host': '192.168.0.5', - 'ip': '10.90.0.5', - 'access_ip': '192.168.0.5'})]) - self.inv.yaml_config['all']['hosts'] = existing - result = self.inv.build_hostnames(changed_hosts) - self.assertEqual(expected, result) - - def test_build_hostnames_different_ips_add_duplicate(self): - changed_hosts = ['10.90.0.2,192.168.0.2'] - expected = OrderedDict([('node3', - {'ansible_host': '192.168.0.2', - 'ip': '10.90.0.2', - 'access_ip': '192.168.0.2'})]) - existing = expected - self.inv.yaml_config['all']['hosts'] = existing - result = self.inv.build_hostnames(changed_hosts, True) - self.assertEqual(expected, result) - - def test_build_hostnames_add_two_different_ips_into_one_existing(self): - changed_hosts = ['10.90.0.3,192.168.0.3', '10.90.0.4,192.168.0.4'] - expected = OrderedDict([ - ('node2', {'ansible_host': '192.168.0.2', - 'ip': '10.90.0.2', - 'access_ip': '192.168.0.2'}), - ('node3', {'ansible_host': '192.168.0.3', - 'ip': '10.90.0.3', - 'access_ip': '192.168.0.3'}), - ('node4', {'ansible_host': '192.168.0.4', - 'ip': '10.90.0.4', - 'access_ip': '192.168.0.4'})]) - - existing = OrderedDict([ - ('node2', {'ansible_host': '192.168.0.2', - 'ip': '10.90.0.2', - 'access_ip': '192.168.0.2'})]) - self.inv.yaml_config['all']['hosts'] = existing - result = self.inv.build_hostnames(changed_hosts, True) - self.assertEqual(expected, result) - - def test_build_hostnames_add_two_different_ips_into_two_existing(self): - changed_hosts = ['10.90.0.4,192.168.0.4', '10.90.0.5,192.168.0.5'] - expected = OrderedDict([ - ('node2', {'ansible_host': '192.168.0.2', - 'ip': '10.90.0.2', - 'access_ip': '192.168.0.2'}), - ('node3', {'ansible_host': '192.168.0.3', - 'ip': '10.90.0.3', - 'access_ip': '192.168.0.3'}), - ('node4', {'ansible_host': '192.168.0.4', - 'ip': '10.90.0.4', - 'access_ip': '192.168.0.4'}), - ('node5', {'ansible_host': '192.168.0.5', - 'ip': '10.90.0.5', - 'access_ip': '192.168.0.5'})]) - - existing = OrderedDict([ - ('node2', {'ansible_host': '192.168.0.2', - 'ip': '10.90.0.2', - 'access_ip': '192.168.0.2'}), - ('node3', {'ansible_host': '192.168.0.3', - 'ip': '10.90.0.3', - 'access_ip': '192.168.0.3'})]) - self.inv.yaml_config['all']['hosts'] = existing - result = self.inv.build_hostnames(changed_hosts, True) - self.assertEqual(expected, result) - - def test_build_hostnames_add_two_different_ips_into_three_existing(self): - changed_hosts = ['10.90.0.5,192.168.0.5', '10.90.0.6,192.168.0.6'] - expected = OrderedDict([ - ('node2', {'ansible_host': '192.168.0.2', - 'ip': '10.90.0.2', - 'access_ip': '192.168.0.2'}), - ('node3', {'ansible_host': '192.168.0.3', - 'ip': '10.90.0.3', - 'access_ip': '192.168.0.3'}), - ('node4', {'ansible_host': '192.168.0.4', - 'ip': '10.90.0.4', - 'access_ip': '192.168.0.4'}), - ('node5', {'ansible_host': '192.168.0.5', - 'ip': '10.90.0.5', - 'access_ip': '192.168.0.5'}), - ('node6', {'ansible_host': '192.168.0.6', - 'ip': '10.90.0.6', - 'access_ip': '192.168.0.6'})]) - - existing = OrderedDict([ - ('node2', {'ansible_host': '192.168.0.2', - 'ip': '10.90.0.2', - 'access_ip': '192.168.0.2'}), - ('node3', {'ansible_host': '192.168.0.3', - 'ip': '10.90.0.3', - 'access_ip': '192.168.0.3'}), - ('node4', {'ansible_host': '192.168.0.4', - 'ip': '10.90.0.4', - 'access_ip': '192.168.0.4'})]) - self.inv.yaml_config['all']['hosts'] = existing - result = self.inv.build_hostnames(changed_hosts, True) - self.assertEqual(expected, result) - - # Add two IP addresses into a config that has - # three already defined IP addresses. One of the IP addresses - # is a duplicate. - def test_build_hostnames_add_two_duplicate_one_overlap(self): - changed_hosts = ['10.90.0.4,192.168.0.4', '10.90.0.5,192.168.0.5'] - expected = OrderedDict([ - ('node2', {'ansible_host': '192.168.0.2', - 'ip': '10.90.0.2', - 'access_ip': '192.168.0.2'}), - ('node3', {'ansible_host': '192.168.0.3', - 'ip': '10.90.0.3', - 'access_ip': '192.168.0.3'}), - ('node4', {'ansible_host': '192.168.0.4', - 'ip': '10.90.0.4', - 'access_ip': '192.168.0.4'}), - ('node5', {'ansible_host': '192.168.0.5', - 'ip': '10.90.0.5', - 'access_ip': '192.168.0.5'})]) - - existing = OrderedDict([ - ('node2', {'ansible_host': '192.168.0.2', - 'ip': '10.90.0.2', - 'access_ip': '192.168.0.2'}), - ('node3', {'ansible_host': '192.168.0.3', - 'ip': '10.90.0.3', - 'access_ip': '192.168.0.3'}), - ('node4', {'ansible_host': '192.168.0.4', - 'ip': '10.90.0.4', - 'access_ip': '192.168.0.4'})]) - self.inv.yaml_config['all']['hosts'] = existing - result = self.inv.build_hostnames(changed_hosts, True) - self.assertEqual(expected, result) - - # Add two duplicate IP addresses into a config that has - # three already defined IP addresses - def test_build_hostnames_add_two_duplicate_two_overlap(self): - changed_hosts = ['10.90.0.3,192.168.0.3', '10.90.0.4,192.168.0.4'] - expected = OrderedDict([ - ('node2', {'ansible_host': '192.168.0.2', - 'ip': '10.90.0.2', - 'access_ip': '192.168.0.2'}), - ('node3', {'ansible_host': '192.168.0.3', - 'ip': '10.90.0.3', - 'access_ip': '192.168.0.3'}), - ('node4', {'ansible_host': '192.168.0.4', - 'ip': '10.90.0.4', - 'access_ip': '192.168.0.4'})]) - - existing = OrderedDict([ - ('node2', {'ansible_host': '192.168.0.2', - 'ip': '10.90.0.2', - 'access_ip': '192.168.0.2'}), - ('node3', {'ansible_host': '192.168.0.3', - 'ip': '10.90.0.3', - 'access_ip': '192.168.0.3'}), - ('node4', {'ansible_host': '192.168.0.4', - 'ip': '10.90.0.4', - 'access_ip': '192.168.0.4'})]) - self.inv.yaml_config['all']['hosts'] = existing - result = self.inv.build_hostnames(changed_hosts, True) - self.assertEqual(expected, result) diff --git a/contrib/inventory_builder/tox.ini b/contrib/inventory_builder/tox.ini deleted file mode 100644 index c9c70428cc4..00000000000 --- a/contrib/inventory_builder/tox.ini +++ /dev/null @@ -1,34 +0,0 @@ -[tox] -minversion = 1.6 -skipsdist = True -envlist = pep8 - -[testenv] -allowlist_externals = py.test -usedevelop = True -deps = - -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt -setenv = VIRTUAL_ENV={envdir} -passenv = - http_proxy - HTTP_PROXY - https_proxy - HTTPS_PROXY - no_proxy - NO_PROXY -commands = pytest -vv #{posargs:./tests} - -[testenv:pep8] -usedevelop = False -allowlist_externals = bash -commands = - bash -c "find {toxinidir}/* -type f -name '*.py' -print0 | xargs -0 flake8" - -[testenv:venv] -commands = {posargs} - -[flake8] -show-source = true -builtins = _ -exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg diff --git a/contrib/kvm-setup/kvm-setup.yml b/contrib/kvm-setup/kvm-setup.yml index b8d44058767..73b81978969 100644 --- a/contrib/kvm-setup/kvm-setup.yml +++ b/contrib/kvm-setup/kvm-setup.yml @@ -1,8 +1,8 @@ --- - name: Prepare Hypervisor to later install kubespray VMs hosts: localhost - gather_facts: False - become: yes + gather_facts: false + become: true vars: bootstrap_os: none roles: diff --git a/contrib/kvm-setup/roles/kvm-setup/tasks/main.yml b/contrib/kvm-setup/roles/kvm-setup/tasks/main.yml index 3e8ade645bb..dfcd3be7367 100644 --- a/contrib/kvm-setup/roles/kvm-setup/tasks/main.yml +++ b/contrib/kvm-setup/roles/kvm-setup/tasks/main.yml @@ -11,12 +11,12 @@ - name: Install required packages apt: - upgrade: yes - update_cache: yes + upgrade: true + update_cache: true cache_valid_time: 3600 name: "{{ item }}" state: present - install_recommends: no + install_recommends: false with_items: - dnsutils - ntp diff --git a/contrib/kvm-setup/roles/kvm-setup/tasks/sysctl.yml b/contrib/kvm-setup/roles/kvm-setup/tasks/sysctl.yml index 52bc83f0998..75b7ff8fd22 100644 --- a/contrib/kvm-setup/roles/kvm-setup/tasks/sysctl.yml +++ b/contrib/kvm-setup/roles/kvm-setup/tasks/sysctl.yml @@ -20,7 +20,7 @@ br-netfilter owner: root group: root - mode: 0644 + mode: "0644" when: br_netfilter is defined @@ -30,7 +30,7 @@ value: 1 sysctl_file: "{{ sysctl_file_path }}" state: present - reload: yes + reload: true - name: Set bridge-nf-call-{arptables,iptables} to 0 ansible.posix.sysctl: @@ -38,7 +38,7 @@ state: present value: 0 sysctl_file: "{{ sysctl_file_path }}" - reload: yes + reload: true with_items: - net.bridge.bridge-nf-call-arptables - net.bridge.bridge-nf-call-ip6tables diff --git a/contrib/kvm-setup/roles/kvm-setup/tasks/user.yml b/contrib/kvm-setup/roles/kvm-setup/tasks/user.yml index c2d31230263..e8ab34afde2 100644 --- a/contrib/kvm-setup/roles/kvm-setup/tasks/user.yml +++ b/contrib/kvm-setup/roles/kvm-setup/tasks/user.yml @@ -11,7 +11,7 @@ state: directory owner: "{{ k8s_deployment_user }}" group: "{{ k8s_deployment_user }}" - mode: 0700 + mode: "0700" - name: Configure sudo for deployment user copy: @@ -20,13 +20,13 @@ dest: "/etc/sudoers.d/55-k8s-deployment" owner: root group: root - mode: 0644 + mode: "0644" - name: Write private SSH key copy: src: "{{ k8s_deployment_user_pkey_path }}" dest: "/home/{{ k8s_deployment_user }}/.ssh/id_rsa" - mode: 0400 + mode: "0400" owner: "{{ k8s_deployment_user }}" group: "{{ k8s_deployment_user }}" when: k8s_deployment_user_pkey_path is defined @@ -41,7 +41,7 @@ - name: Fix ssh-pub-key permissions file: path: "/home/{{ k8s_deployment_user }}/.ssh/authorized_keys" - mode: 0600 + mode: "0600" owner: "{{ k8s_deployment_user }}" group: "{{ k8s_deployment_user }}" when: k8s_deployment_user_pkey_path is defined diff --git a/contrib/mitogen/mitogen.yml b/contrib/mitogen/mitogen.yml index 1ccc9a99c4c..77018d69310 100644 --- a/contrib/mitogen/mitogen.yml +++ b/contrib/mitogen/mitogen.yml @@ -14,7 +14,7 @@ file: path: "{{ item }}" state: directory - mode: 0755 + mode: "0755" become: false loop: - "{{ playbook_dir }}/plugins/mitogen" @@ -25,7 +25,7 @@ url: "{{ mitogen_url }}" dest: "{{ playbook_dir }}/dist/mitogen_{{ mitogen_version }}.tar.gz" validate_certs: true - mode: 0644 + mode: "0644" - name: Extract archive unarchive: @@ -40,7 +40,7 @@ - name: Add strategy to ansible.cfg community.general.ini_file: path: ansible.cfg - mode: 0644 + mode: "0644" section: "{{ item.section | d('defaults') }}" option: "{{ item.option }}" value: "{{ item.value }}" diff --git a/contrib/network-storage/glusterfs/roles/glusterfs/README.md b/contrib/network-storage/glusterfs/roles/glusterfs/README.md index dda243df04f..9e5bf5dcfbc 100644 --- a/contrib/network-storage/glusterfs/roles/glusterfs/README.md +++ b/contrib/network-storage/glusterfs/roles/glusterfs/README.md @@ -21,7 +21,7 @@ glusterfs_default_release: "" You can specify a `default_release` for apt on Debian/Ubuntu by overriding this variable. This is helpful if you need a different package or version for the main GlusterFS packages (e.g. GlusterFS 3.5.x instead of 3.2.x with the `wheezy-backports` default release on Debian Wheezy). ```yaml -glusterfs_ppa_use: yes +glusterfs_ppa_use: true glusterfs_ppa_version: "3.5" ``` diff --git a/contrib/network-storage/glusterfs/roles/glusterfs/client/defaults/main.yml b/contrib/network-storage/glusterfs/roles/glusterfs/client/defaults/main.yml index b9f0d2d1d3b..c3fff2e6324 100644 --- a/contrib/network-storage/glusterfs/roles/glusterfs/client/defaults/main.yml +++ b/contrib/network-storage/glusterfs/roles/glusterfs/client/defaults/main.yml @@ -1,7 +1,7 @@ --- # For Ubuntu. glusterfs_default_release: "" -glusterfs_ppa_use: yes +glusterfs_ppa_use: true glusterfs_ppa_version: "4.1" # Gluster configuration. diff --git a/contrib/network-storage/glusterfs/roles/glusterfs/client/tasks/main.yml b/contrib/network-storage/glusterfs/roles/glusterfs/client/tasks/main.yml index 248f21efa5b..947cf8aa231 100644 --- a/contrib/network-storage/glusterfs/roles/glusterfs/client/tasks/main.yml +++ b/contrib/network-storage/glusterfs/roles/glusterfs/client/tasks/main.yml @@ -15,7 +15,7 @@ file: path: "{{ item }}" state: directory - mode: 0775 + mode: "0775" with_items: - "{{ gluster_mount_dir }}" when: ansible_os_family in ["Debian","RedHat"] and groups['gfs-cluster'] is defined diff --git a/contrib/network-storage/glusterfs/roles/glusterfs/client/tasks/setup-Debian.yml b/contrib/network-storage/glusterfs/roles/glusterfs/client/tasks/setup-Debian.yml index da7a4d8decc..0d7cc18747a 100644 --- a/contrib/network-storage/glusterfs/roles/glusterfs/client/tasks/setup-Debian.yml +++ b/contrib/network-storage/glusterfs/roles/glusterfs/client/tasks/setup-Debian.yml @@ -3,7 +3,7 @@ apt_repository: repo: 'ppa:gluster/glusterfs-{{ glusterfs_ppa_version }}' state: present - update_cache: yes + update_cache: true register: glusterfs_ppa_added when: glusterfs_ppa_use diff --git a/contrib/network-storage/glusterfs/roles/glusterfs/server/defaults/main.yml b/contrib/network-storage/glusterfs/roles/glusterfs/server/defaults/main.yml index ef9a71eba40..7d6e1025b1f 100644 --- a/contrib/network-storage/glusterfs/roles/glusterfs/server/defaults/main.yml +++ b/contrib/network-storage/glusterfs/roles/glusterfs/server/defaults/main.yml @@ -1,7 +1,7 @@ --- # For Ubuntu. glusterfs_default_release: "" -glusterfs_ppa_use: yes +glusterfs_ppa_use: true glusterfs_ppa_version: "3.12" # Gluster configuration. diff --git a/contrib/network-storage/glusterfs/roles/glusterfs/server/tasks/main.yml b/contrib/network-storage/glusterfs/roles/glusterfs/server/tasks/main.yml index 50f849c01d3..a9f7698a37e 100644 --- a/contrib/network-storage/glusterfs/roles/glusterfs/server/tasks/main.yml +++ b/contrib/network-storage/glusterfs/roles/glusterfs/server/tasks/main.yml @@ -43,13 +43,13 @@ service: name: "{{ glusterfs_daemon }}" state: started - enabled: yes + enabled: true - name: Ensure Gluster brick and mount directories exist. file: path: "{{ item }}" state: directory - mode: 0775 + mode: "0775" with_items: - "{{ gluster_brick_dir }}" - "{{ gluster_mount_dir }}" @@ -62,7 +62,7 @@ replicas: "{{ groups['gfs-cluster'] | length }}" cluster: "{% for item in groups['gfs-cluster'] -%}{{ hostvars[item]['ip'] | default(hostvars[item].ansible_default_ipv4['address']) }}{% if not loop.last %},{% endif %}{%- endfor %}" host: "{{ inventory_hostname }}" - force: yes + force: true run_once: true when: groups['gfs-cluster'] | length > 1 @@ -73,7 +73,7 @@ brick: "{{ gluster_brick_dir }}" cluster: "{% for item in groups['gfs-cluster'] -%}{{ hostvars[item]['ip'] | default(hostvars[item].ansible_default_ipv4['address']) }}{% if not loop.last %},{% endif %}{%- endfor %}" host: "{{ inventory_hostname }}" - force: yes + force: true run_once: true when: groups['gfs-cluster'] | length <= 1 @@ -101,7 +101,7 @@ template: dest: "{{ gluster_mount_dir }}/.test-file.txt" src: test-file.txt - mode: 0644 + mode: "0644" when: groups['gfs-cluster'] is defined and inventory_hostname == groups['gfs-cluster'][0] - name: Unmount glusterfs diff --git a/contrib/network-storage/glusterfs/roles/glusterfs/server/tasks/setup-Debian.yml b/contrib/network-storage/glusterfs/roles/glusterfs/server/tasks/setup-Debian.yml index 104735903ee..4d4b1b4b80d 100644 --- a/contrib/network-storage/glusterfs/roles/glusterfs/server/tasks/setup-Debian.yml +++ b/contrib/network-storage/glusterfs/roles/glusterfs/server/tasks/setup-Debian.yml @@ -3,7 +3,7 @@ apt_repository: repo: 'ppa:gluster/glusterfs-{{ glusterfs_ppa_version }}' state: present - update_cache: yes + update_cache: true register: glusterfs_ppa_added when: glusterfs_ppa_use diff --git a/contrib/network-storage/glusterfs/roles/kubernetes-pv/ansible/tasks/main.yaml b/contrib/network-storage/glusterfs/roles/kubernetes-pv/ansible/tasks/main.yaml index ed62e282e38..cf2bd0ee5cb 100644 --- a/contrib/network-storage/glusterfs/roles/kubernetes-pv/ansible/tasks/main.yaml +++ b/contrib/network-storage/glusterfs/roles/kubernetes-pv/ansible/tasks/main.yaml @@ -3,7 +3,7 @@ template: src: "{{ item.file }}" dest: "{{ kube_config_dir }}/{{ item.dest }}" - mode: 0644 + mode: "0644" with_items: - { file: glusterfs-kubernetes-endpoint.json.j2, type: ep, dest: glusterfs-kubernetes-endpoint.json} - { file: glusterfs-kubernetes-pv.yml.j2, type: pv, dest: glusterfs-kubernetes-pv.yml} diff --git a/contrib/network-storage/heketi/heketi-tear-down.yml b/contrib/network-storage/heketi/heketi-tear-down.yml index e64f085cb67..8c9d1c3a000 100644 --- a/contrib/network-storage/heketi/heketi-tear-down.yml +++ b/contrib/network-storage/heketi/heketi-tear-down.yml @@ -6,6 +6,6 @@ - name: Teardown disks in heketi hosts: heketi-node - become: yes + become: true roles: - { role: tear-down-disks } diff --git a/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/deploy.yml b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/deploy.yml index 866fe30bf6a..94d44015071 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/deploy.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/deploy.yml @@ -4,7 +4,7 @@ template: src: "heketi-bootstrap.json.j2" dest: "{{ kube_config_dir }}/heketi-bootstrap.json" - mode: 0640 + mode: "0640" register: "rendering" - name: "Kubernetes Apps | Install and configure Heketi Bootstrap" kube: diff --git a/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/topology.yml b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/topology.yml index 2f3efd4dd1f..b011c024b68 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/topology.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/topology.yml @@ -10,7 +10,7 @@ template: src: "topology.json.j2" dest: "{{ kube_config_dir }}/topology.json" - mode: 0644 + mode: "0644" - name: "Copy topology configuration into container." changed_when: false command: "{{ bin_dir }}/kubectl cp {{ kube_config_dir }}/topology.json {{ initial_heketi_pod_name }}:/tmp/topology.json" diff --git a/contrib/network-storage/heketi/roles/provision/tasks/glusterfs.yml b/contrib/network-storage/heketi/roles/provision/tasks/glusterfs.yml index 973c6685141..239e780d88a 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/glusterfs.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/glusterfs.yml @@ -3,7 +3,7 @@ template: src: "glusterfs-daemonset.json.j2" dest: "{{ kube_config_dir }}/glusterfs-daemonset.json" - mode: 0644 + mode: "0644" become: true register: "rendering" - name: "Kubernetes Apps | Install and configure GlusterFS daemonset" @@ -33,7 +33,7 @@ template: src: "heketi-service-account.json.j2" dest: "{{ kube_config_dir }}/heketi-service-account.json" - mode: 0644 + mode: "0644" become: true register: "rendering" - name: "Kubernetes Apps | Install and configure Heketi Service Account" diff --git a/contrib/network-storage/heketi/roles/provision/tasks/heketi.yml b/contrib/network-storage/heketi/roles/provision/tasks/heketi.yml index a8549df4581..30c68c2bc53 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/heketi.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/heketi.yml @@ -4,7 +4,7 @@ template: src: "heketi-deployment.json.j2" dest: "{{ kube_config_dir }}/heketi-deployment.json" - mode: 0644 + mode: "0644" register: "rendering" - name: "Kubernetes Apps | Install and configure Heketi" diff --git a/contrib/network-storage/heketi/roles/provision/tasks/secret.yml b/contrib/network-storage/heketi/roles/provision/tasks/secret.yml index c455b6f6ddc..816bb156c27 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/secret.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/secret.yml @@ -28,7 +28,7 @@ template: src: "heketi.json.j2" dest: "{{ kube_config_dir }}/heketi.json" - mode: 0644 + mode: "0644" - name: "Deploy Heketi config secret" when: "secret_state.stdout | length == 0" diff --git a/contrib/network-storage/heketi/roles/provision/tasks/storage.yml b/contrib/network-storage/heketi/roles/provision/tasks/storage.yml index 055e179a34b..c3f8ebf2e58 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/storage.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/storage.yml @@ -5,7 +5,7 @@ template: src: "heketi-storage.json.j2" dest: "{{ kube_config_dir }}/heketi-storage.json" - mode: 0644 + mode: "0644" register: "rendering" - name: "Kubernetes Apps | Install and configure Heketi Storage" kube: diff --git a/contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml b/contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml index bd4f6666be6..fc57302bcbd 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml @@ -16,7 +16,7 @@ template: src: "storageclass.yml.j2" dest: "{{ kube_config_dir }}/storageclass.yml" - mode: 0644 + mode: "0644" register: "rendering" - name: "Kubernetes Apps | Install and configure Storace Class" kube: diff --git a/contrib/network-storage/heketi/roles/provision/tasks/topology.yml b/contrib/network-storage/heketi/roles/provision/tasks/topology.yml index aa662083ea5..edd5bd9e88f 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/topology.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/topology.yml @@ -10,7 +10,7 @@ template: src: "topology.json.j2" dest: "{{ kube_config_dir }}/topology.json" - mode: 0644 + mode: "0644" - name: "Copy topology configuration into container." # noqa no-handler when: "rendering.changed" command: "{{ bin_dir }}/kubectl cp {{ kube_config_dir }}/topology.json {{ heketi_pod_name }}:/tmp/topology.json" diff --git a/contrib/offline/generate_list.yml b/contrib/offline/generate_list.yml index bebf3496899..f103d031f72 100644 --- a/contrib/offline/generate_list.yml +++ b/contrib/offline/generate_list.yml @@ -1,7 +1,7 @@ --- - name: Collect container images for offline deployment hosts: localhost - become: no + become: false roles: # Just load default variables from roles. @@ -16,7 +16,7 @@ template: src: ./contrib/offline/temp/{{ item }}.list.template dest: ./contrib/offline/temp/{{ item }}.list - mode: 0644 + mode: "0644" with_items: - files - images diff --git a/contrib/os-services/roles/prepare/tasks/main.yml b/contrib/os-services/roles/prepare/tasks/main.yml index e95dcef1510..487b3b6f1ce 100644 --- a/contrib/os-services/roles/prepare/tasks/main.yml +++ b/contrib/os-services/roles/prepare/tasks/main.yml @@ -7,17 +7,17 @@ service_facts: - name: Disable service firewalld - systemd: + systemd_service: name: firewalld state: stopped - enabled: no + enabled: false when: "'firewalld.service' in services and services['firewalld.service'].status != 'not-found'" - name: Disable service ufw - systemd: + systemd_service: name: ufw state: stopped - enabled: no + enabled: false when: "'ufw.service' in services and services['ufw.service'].status != 'not-found'" diff --git a/contrib/packaging/rpm/kubespray.spec b/contrib/packaging/rpm/kubespray.spec deleted file mode 100644 index cc8a7527910..00000000000 --- a/contrib/packaging/rpm/kubespray.spec +++ /dev/null @@ -1,62 +0,0 @@ -%global srcname kubespray - -%{!?upstream_version: %global upstream_version %{version}%{?milestone}} - -Name: kubespray -Version: master -Release: %(git describe | sed -r 's/v(\S+-?)-(\S+)-(\S+)/\1.dev\2+\3/') -Summary: Ansible modules for installing Kubernetes - -Group: System Environment/Libraries -License: ASL 2.0 -Url: https://github.com/kubernetes-sigs/kubespray -Source0: https://github.com/kubernetes-sigs/kubespray/archive/%{upstream_version}.tar.gz#/%{name}-%{release}.tar.gz - -BuildArch: noarch -BuildRequires: git -BuildRequires: python2 -BuildRequires: python2-devel -BuildRequires: python2-setuptools -BuildRequires: python-d2to1 -BuildRequires: python2-pbr - -Requires: ansible >= 2.5.0 -Requires: python-jinja2 >= 2.10 -Requires: python-netaddr -Requires: python-pbr - -%description - -Ansible-kubespray is a set of Ansible modules and playbooks for -installing a Kubernetes cluster. If you have questions, join us -on the https://slack.k8s.io, channel '#kubespray'. - -%prep -%autosetup -n %{name}-%{upstream_version} -S git - - -%build -export PBR_VERSION=%{release} -%{__python2} setup.py build bdist_rpm - - -%install -export PBR_VERSION=%{release} -export SKIP_PIP_INSTALL=1 -%{__python2} setup.py install --skip-build --root %{buildroot} bdist_rpm - - -%files -%doc %{_docdir}/%{name}/README.md -%doc %{_docdir}/%{name}/inventory/sample/hosts.ini -%config %{_sysconfdir}/%{name}/ansible.cfg -%config %{_sysconfdir}/%{name}/inventory/sample/group_vars/all.yml -%config %{_sysconfdir}/%{name}/inventory/sample/group_vars/k8s_cluster.yml -%license %{_docdir}/%{name}/LICENSE -%{python2_sitelib}/%{srcname}-%{release}-py%{python2_version}.egg-info -%{_datarootdir}/%{name}/roles/ -%{_datarootdir}/%{name}/playbooks/ -%defattr(-,root,root) - - -%changelog diff --git a/contrib/terraform/equinix/README.md b/contrib/terraform/equinix/README.md index d1eb71f3f92..f81e066a068 100644 --- a/contrib/terraform/equinix/README.md +++ b/contrib/terraform/equinix/README.md @@ -35,7 +35,7 @@ now six total etcd replicas. ## Requirements - [Install Terraform](https://www.terraform.io/intro/getting-started/install.html) -- [Install Ansible dependencies](/docs/ansible.md#installing-ansible) +- [Install Ansible dependencies](/docs/ansible/ansible.md#installing-ansible) - Account with Equinix Metal - An SSH key pair diff --git a/contrib/terraform/hetzner/templates/inventory.tpl b/contrib/terraform/hetzner/templates/inventory.tpl index 56666e162e3..08d63693f97 100644 --- a/contrib/terraform/hetzner/templates/inventory.tpl +++ b/contrib/terraform/hetzner/templates/inventory.tpl @@ -12,8 +12,8 @@ ${list_master} ${list_worker} [k8s_cluster:children] -kube-master -kube-node +kube_control_plane +kube_node [k8s_cluster:vars] network_id=${network_id} diff --git a/contrib/terraform/nifcloud/README.md b/contrib/terraform/nifcloud/README.md index 8c46df402f5..a6dcf014855 100644 --- a/contrib/terraform/nifcloud/README.md +++ b/contrib/terraform/nifcloud/README.md @@ -72,6 +72,7 @@ The setup looks like following ```bash ./generate-inventory.sh > sample-inventory/inventory.ini + ``` * Export Variables: diff --git a/contrib/terraform/openstack/README.md b/contrib/terraform/openstack/README.md index 923aa314074..8162a9f949d 100644 --- a/contrib/terraform/openstack/README.md +++ b/contrib/terraform/openstack/README.md @@ -60,17 +60,17 @@ You can create many different kubernetes topologies by setting the number of different classes of hosts. For each class there are options for allocating floating IP addresses or not. -- Master nodes with etcd -- Master nodes without etcd +- Control plane nodes with etcd +- Control plane nodes without etcd - Standalone etcd hosts - Kubernetes worker nodes Note that the Ansible script will report an invalid configuration if you wind up with an even number of etcd instances since that is not a valid configuration. This restriction includes standalone etcd nodes that are deployed in a cluster along with -master nodes with etcd replicas. As an example, if you have three master nodes with -etcd replicas and three standalone etcd nodes, the script will fail since there are -now six total etcd replicas. +control plane nodes with etcd replicas. As an example, if you have three control plane +nodes with etcd replicas and three standalone etcd nodes, the script will fail since +there are now six total etcd replicas. ### GlusterFS shared file system @@ -258,7 +258,8 @@ For your cluster, edit `inventory/$CLUSTER/cluster.tfvars`. |`bastion_fips` | A list of floating IPs that you have already pre-allocated; they will be attached to bastion node instead of creating new random floating IPs. | |`external_net` | UUID of the external network that will be routed to | |`flavor_k8s_master`,`flavor_k8s_node`,`flavor_etcd`, `flavor_bastion`,`flavor_gfs_node` | Flavor depends on your openstack installation, you can get available flavor IDs through `openstack flavor list` | -|`image`,`image_gfs` | Name of the image to use in provisioning the compute resources. Should already be loaded into glance. | +|`image`,`image_gfs`, `image_master` | Name of the image to use in provisioning the compute resources. Should already be loaded into glance. | +|`image_uuid`,`image_gfs_uuid`, `image_master_uuid` | UUID of the image to use in provisioning the compute resources. Should already be loaded into glance. | |`ssh_user`,`ssh_user_gfs` | The username to ssh into the image with. This usually depends on the image you have selected | |`public_key_path` | Path on your local workstation to the public key file you wish to use in creating the key pairs | |`number_of_k8s_masters`, `number_of_k8s_masters_no_floating_ip` | Number of nodes that serve as both master and etcd. These can be provisioned with or without floating IP addresses| @@ -299,10 +300,10 @@ For your cluster, edit `inventory/$CLUSTER/cluster.tfvars`. |`force_null_port_security` | Set `null` instead of `true` or `false` for `port_security`. `false` by default | |`k8s_nodes` | Map containing worker node definition, see explanation below | |`k8s_masters` | Map containing master node definition, see explanation for k8s_nodes and `sample-inventory/cluster.tfvars` | -| `k8s_master_loadbalancer_enabled`| Enable and use an Octavia load balancer for the K8s master nodes | -| `k8s_master_loadbalancer_listener_port` | Define via which port the K8s Api should be exposed. `6443` by default | -| `k8s_master_loadbalancer_server_port` | Define via which port the K8S api is available on the mas. `6443` by default | -| `k8s_master_loadbalancer_public_ip` | Specify if an existing floating IP should be used for the load balancer. A new floating IP is assigned by default | +|`k8s_master_loadbalancer_enabled` | Enable and use an Octavia load balancer for the K8s master nodes | +|`k8s_master_loadbalancer_listener_port` | Define via which port the K8s Api should be exposed. `6443` by default | +|`k8s_master_loadbalancer_server_port` | Define via which port the K8S api is available on the master nodes. `6443` by default | +|`k8s_master_loadbalancer_public_ip` | Specify if an existing floating IP should be used for the load balancer. A new floating IP is assigned by default | ##### k8s_nodes @@ -317,7 +318,8 @@ k8s_nodes: node-name: az: string # Name of the AZ flavor: string # Flavor ID to use - floating_ip: bool # If floating IPs should be created or not + floating_ip: bool # If floating IPs should be used or not + reserved_floating_ip: string # If floating_ip is true use existing floating IP, if reserved_floating_ip is an empty string and floating_ip is true, a new floating IP will be created extra_groups: string # (optional) Additional groups to add for kubespray, defaults to no groups image_id: string # (optional) Image ID to use, defaults to var.image_id or var.image root_volume_size_in_gb: number # (optional) Size of the block storage to use as root disk, defaults to var.node_root_volume_size_in_gb or to use volume from flavor otherwise @@ -619,7 +621,7 @@ Edit `inventory/$CLUSTER/group_vars/k8s_cluster/k8s_cluster.yml`: - Set variable **kube_network_plugin** to your desired networking plugin. - **flannel** works out-of-the-box - - **calico** requires [configuring OpenStack Neutron ports](/docs/openstack.md) to allow service and pod subnets + - **calico** requires [configuring OpenStack Neutron ports](/docs/cloud_controllers/openstack.md) to allow service and pod subnets ```yml # Choose network plugin (calico, weave or flannel) diff --git a/contrib/terraform/openstack/modules/compute/variables.tf b/contrib/terraform/openstack/modules/compute/variables.tf index 006cce9efe8..ed478de3c2f 100644 --- a/contrib/terraform/openstack/modules/compute/variables.tf +++ b/contrib/terraform/openstack/modules/compute/variables.tf @@ -89,11 +89,15 @@ variable "k8s_node_fips" { } variable "k8s_masters_fips" { - type = map + type = map(object({ + address = string + })) } variable "k8s_nodes_fips" { - type = map + type = map(object({ + address = string + })) } variable "bastion_fips" { @@ -136,8 +140,9 @@ variable "k8s_masters" { type = map(object({ az = string flavor = string - floating_ip = bool etcd = bool + floating_ip = bool + reserved_floating_ip = optional(string) image_id = optional(string) root_volume_size_in_gb = optional(number) volume_type = optional(string) @@ -150,6 +155,7 @@ variable "k8s_nodes" { az = string flavor = string floating_ip = bool + reserved_floating_ip = optional(string) extra_groups = optional(string) image_id = optional(string) root_volume_size_in_gb = optional(number) diff --git a/contrib/terraform/openstack/modules/ips/main.tf b/contrib/terraform/openstack/modules/ips/main.tf index b0989dcc224..55ef82673df 100644 --- a/contrib/terraform/openstack/modules/ips/main.tf +++ b/contrib/terraform/openstack/modules/ips/main.tf @@ -15,7 +15,7 @@ resource "openstack_networking_floatingip_v2" "k8s_master" { } resource "openstack_networking_floatingip_v2" "k8s_masters" { - for_each = var.number_of_k8s_masters == 0 && var.number_of_k8s_masters_no_etcd == 0 ? { for key, value in var.k8s_masters : key => value if value.floating_ip } : {} + for_each = var.number_of_k8s_masters == 0 && var.number_of_k8s_masters_no_etcd == 0 ? { for key, value in var.k8s_masters : key => value if value.floating_ip && (lookup(value, "reserved_floating_ip", "") == "") } : {} pool = var.floatingip_pool depends_on = [null_resource.dummy_dependency] } @@ -40,7 +40,7 @@ resource "openstack_networking_floatingip_v2" "bastion" { } resource "openstack_networking_floatingip_v2" "k8s_nodes" { - for_each = var.number_of_k8s_nodes == 0 ? { for key, value in var.k8s_nodes : key => value if value.floating_ip } : {} + for_each = var.number_of_k8s_nodes == 0 ? { for key, value in var.k8s_nodes : key => value if value.floating_ip && (lookup(value, "reserved_floating_ip", "") == "") } : {} pool = var.floatingip_pool depends_on = [null_resource.dummy_dependency] } diff --git a/contrib/terraform/openstack/modules/ips/outputs.tf b/contrib/terraform/openstack/modules/ips/outputs.tf index 3ff4622abf6..670481109af 100644 --- a/contrib/terraform/openstack/modules/ips/outputs.tf +++ b/contrib/terraform/openstack/modules/ips/outputs.tf @@ -1,10 +1,33 @@ +locals { + k8s_masters_reserved_fips = { + for key, value in var.k8s_masters : key => { + address = value.reserved_floating_ip + } if value.floating_ip && (lookup(value, "reserved_floating_ip", "") != "") + } + k8s_masters_create_fips = { + for key, value in openstack_networking_floatingip_v2.k8s_masters : key => { + address = value.address + } + } + k8s_nodes_reserved_fips = { + for key, value in var.k8s_nodes : key => { + address = value.reserved_floating_ip + } if value.floating_ip && (lookup(value, "reserved_floating_ip", "") != "") + } + k8s_nodes_create_fips = { + for key, value in openstack_networking_floatingip_v2.k8s_nodes : key => { + address = value.address + } + } +} + # If k8s_master_fips is already defined as input, keep the same value since new FIPs have not been created. output "k8s_master_fips" { value = length(var.k8s_master_fips) > 0 ? var.k8s_master_fips : openstack_networking_floatingip_v2.k8s_master[*].address } output "k8s_masters_fips" { - value = openstack_networking_floatingip_v2.k8s_masters + value = merge(local.k8s_masters_create_fips, local.k8s_masters_reserved_fips) } # If k8s_master_fips is already defined as input, keep the same value since new FIPs have not been created. @@ -17,7 +40,7 @@ output "k8s_node_fips" { } output "k8s_nodes_fips" { - value = openstack_networking_floatingip_v2.k8s_nodes + value = merge(local.k8s_nodes_create_fips, local.k8s_nodes_reserved_fips) } output "bastion_fips" { diff --git a/contrib/terraform/terraform.py b/contrib/terraform/terraform.py index f67b9d82dba..c22eb9f41c8 100755 --- a/contrib/terraform/terraform.py +++ b/contrib/terraform/terraform.py @@ -98,7 +98,7 @@ def iterresources(filenames): def _clean_dc(dcname): # Consul DCs are strictly alphanumeric with underscores and hyphens - # ensure that the consul_dc attribute meets these requirements. - return re.sub('[^\w_\-]', '-', dcname) + return re.sub(r'[^\w_\-]', '-', dcname) def iterhosts(resources): @@ -304,7 +304,7 @@ def openstack_host(resource, module_name): try: if 'metadata.prefer_ipv6' in raw_attrs and raw_attrs['metadata.prefer_ipv6'] == "1": attrs.update({ - 'ansible_host': re.sub("[\[\]]", "", raw_attrs['access_ip_v6']), + 'ansible_host': re.sub(r"[\[\]]", "", raw_attrs['access_ip_v6']), 'publicly_routable': True, }) else: @@ -368,7 +368,7 @@ def iter_host_ips(hosts, ips): 'ansible_host': ip, }) - if 'use_access_ip' in host[1]['metadata'] and host[1]['metadata']['use_access_ip'] == "0": + if 'use_access_ip' in host[1]['metadata'] and host[1]['metadata']['use_access_ip'] == "0" and 'access_ip' in host[1]: host[1].pop('access_ip') yield host diff --git a/contrib/terraform/upcloud/cluster-settings.tfvars b/contrib/terraform/upcloud/cluster-settings.tfvars index 45a374900f7..7e1296fd3a9 100644 --- a/contrib/terraform/upcloud/cluster-settings.tfvars +++ b/contrib/terraform/upcloud/cluster-settings.tfvars @@ -1,5 +1,11 @@ # See: https://developers.upcloud.com/1.3/5-zones/ -zone = "fi-hel1" +zone = "fi-hel1" +private_cloud = false + +# Only used if private_cloud = true, public zone equivalent +# For example use finnish public zone for finnish private zone +public_zone = "fi-hel2" + username = "ubuntu" # Prefix to use for all resources to separate them from other resources @@ -146,4 +152,4 @@ server_groups = { # ] # anti_affinity_policy = "yes" # } -} \ No newline at end of file +} diff --git a/contrib/terraform/upcloud/main.tf b/contrib/terraform/upcloud/main.tf index 30986667dd5..78fc0786fb6 100644 --- a/contrib/terraform/upcloud/main.tf +++ b/contrib/terraform/upcloud/main.tf @@ -11,8 +11,10 @@ provider "upcloud" { module "kubernetes" { source = "./modules/kubernetes-cluster" - prefix = var.prefix - zone = var.zone + prefix = var.prefix + zone = var.zone + private_cloud = var.private_cloud + public_zone = var.public_zone template_name = var.template_name username = var.username diff --git a/contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf b/contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf index 2adeb44678f..91c8b9ec7b3 100644 --- a/contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf +++ b/contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf @@ -54,11 +54,12 @@ resource "upcloud_server" "master" { if machine.node_type == "master" } - hostname = "${local.resource-prefix}${each.key}" - plan = each.value.plan - cpu = each.value.plan == null ? each.value.cpu : null - mem = each.value.plan == null ? each.value.mem : null - zone = var.zone + hostname = "${local.resource-prefix}${each.key}" + plan = each.value.plan + cpu = each.value.plan == null ? null : each.value.cpu + mem = each.value.plan == null ? null : each.value.mem + zone = var.zone + server_group = each.value.server_group == null ? null : upcloud_server_group.server_groups[each.value.server_group].id template { storage = var.template_name @@ -111,11 +112,13 @@ resource "upcloud_server" "worker" { if machine.node_type == "worker" } - hostname = "${local.resource-prefix}${each.key}" - plan = each.value.plan - cpu = each.value.plan == null ? each.value.cpu : null - mem = each.value.plan == null ? each.value.mem : null - zone = var.zone + hostname = "${local.resource-prefix}${each.key}" + plan = each.value.plan + cpu = each.value.plan == null ? null : each.value.cpu + mem = each.value.plan == null ? null : each.value.mem + zone = var.zone + server_group = each.value.server_group == null ? null : upcloud_server_group.server_groups[each.value.server_group].id + template { storage = var.template_name @@ -512,8 +515,18 @@ resource "upcloud_loadbalancer" "lb" { configured_status = "started" name = "${local.resource-prefix}lb" plan = var.loadbalancer_plan - zone = var.zone - network = upcloud_network.private.id + zone = var.private_cloud ? var.public_zone : var.zone + networks { + name = "Private-Net" + type = "private" + family = "IPv4" + network = upcloud_network.private.id + } + networks { + name = "Public-Net" + type = "public" + family = "IPv4" + } } resource "upcloud_loadbalancer_backend" "lb_backend" { @@ -534,6 +547,9 @@ resource "upcloud_loadbalancer_frontend" "lb_frontend" { mode = "tcp" port = each.value.port default_backend_name = upcloud_loadbalancer_backend.lb_backend[each.key].name + networks { + name = "Public-Net" + } } resource "upcloud_loadbalancer_static_backend_member" "lb_backend_member" { @@ -557,5 +573,9 @@ resource "upcloud_server_group" "server_groups" { title = each.key anti_affinity_policy = each.value.anti_affinity_policy labels = {} - members = [for server in each.value.servers : merge(upcloud_server.master, upcloud_server.worker)[server].id] -} \ No newline at end of file + # Managed upstream via upcloud_server resource + members = [] + lifecycle { + ignore_changes = [members] + } +} diff --git a/contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf b/contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf index ad2cc70f020..4c41cc08f1a 100644 --- a/contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf +++ b/contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf @@ -6,6 +6,14 @@ variable "zone" { type = string } +variable "private_cloud" { + type = bool +} + +variable "public_zone" { + type = string +} + variable "template_name" {} variable "username" {} @@ -20,6 +28,7 @@ variable "machines" { cpu = string mem = string disk_size = number + server_group : string additional_disks = map(object({ size = number tier = string @@ -104,6 +113,5 @@ variable "server_groups" { type = map(object({ anti_affinity_policy = string - servers = list(string) })) -} \ No newline at end of file +} diff --git a/contrib/terraform/upcloud/modules/kubernetes-cluster/versions.tf b/contrib/terraform/upcloud/modules/kubernetes-cluster/versions.tf index 3138453bef5..81a2d1b57ac 100644 --- a/contrib/terraform/upcloud/modules/kubernetes-cluster/versions.tf +++ b/contrib/terraform/upcloud/modules/kubernetes-cluster/versions.tf @@ -3,7 +3,7 @@ terraform { required_providers { upcloud = { source = "UpCloudLtd/upcloud" - version = "~>2.12.0" + version = "~>5.6.0" } } required_version = ">= 0.13" diff --git a/contrib/terraform/upcloud/sample-inventory/cluster.tfvars b/contrib/terraform/upcloud/sample-inventory/cluster.tfvars index 58536674f68..d1546004bcc 100644 --- a/contrib/terraform/upcloud/sample-inventory/cluster.tfvars +++ b/contrib/terraform/upcloud/sample-inventory/cluster.tfvars @@ -146,4 +146,4 @@ server_groups = { # ] # anti_affinity_policy = "yes" # } -} \ No newline at end of file +} diff --git a/contrib/terraform/upcloud/variables.tf b/contrib/terraform/upcloud/variables.tf index 9d8c90c1769..0ea81b99a4e 100644 --- a/contrib/terraform/upcloud/variables.tf +++ b/contrib/terraform/upcloud/variables.tf @@ -9,6 +9,15 @@ variable "zone" { description = "The zone where to run the cluster" } +variable "private_cloud" { + description = "Whether the environment is in the private cloud region" + default = false +} + +variable "public_zone" { + description = "The public zone equivalent if the cluster is running in a private cloud zone" +} + variable "template_name" { description = "Block describing the preconfigured operating system" } @@ -32,6 +41,7 @@ variable "machines" { cpu = string mem = string disk_size = number + server_group : string additional_disks = map(object({ size = number tier = string @@ -142,7 +152,6 @@ variable "server_groups" { type = map(object({ anti_affinity_policy = string - servers = list(string) })) default = {} diff --git a/contrib/terraform/upcloud/versions.tf b/contrib/terraform/upcloud/versions.tf index 3138453bef5..81a2d1b57ac 100644 --- a/contrib/terraform/upcloud/versions.tf +++ b/contrib/terraform/upcloud/versions.tf @@ -3,7 +3,7 @@ terraform { required_providers { upcloud = { source = "UpCloudLtd/upcloud" - version = "~>2.12.0" + version = "~>5.6.0" } } required_version = ">= 0.13" diff --git a/docs/calico.md b/docs/CNI/calico.md similarity index 97% rename from docs/calico.md rename to docs/CNI/calico.md index fb9ab06f8ad..2b3890914b4 100644 --- a/docs/calico.md +++ b/docs/CNI/calico.md @@ -89,9 +89,12 @@ node1 ansible_ssh_host=95.54.0.12 local_as=xxxxxx Peers can be defined using the `peers` variable (see docs/calico_peer_example examples). In order to define global peers, the `peers` variable can be defined in group_vars with the "scope" attribute of each global peer set to "global". -In order to define peers on a per node basis, the `peers` variable must be defined in hostvars. +In order to define peers on a per node basis, the `peers` variable must be defined in hostvars or group_vars with the "scope" attribute unset or set to "node". + NB: Ansible's `hash_behaviour` is by default set to "replace", thus defining both global and per node peers would end up with having only per node peers. If having both global and per node peers defined was meant to happen, global peers would have to be defined in hostvars for each host (as well as per node peers) +NB²: Peers definition at node scope can be customized with additional fields `filters`, `sourceAddress` and `numAllowedLocalASNumbers` (see for details) + Since calico 3.4, Calico supports advertising Kubernetes service cluster IPs over BGP, just as it advertises pod IPs. This can be enabled by setting the following variable as follow in group_vars (k8s_cluster/k8s-net-calico.yml) @@ -127,8 +130,7 @@ recommended here: You need to edit your inventory and add: * `calico_rr` group with nodes in it. `calico_rr` can be combined with - `kube_node` and/or `kube_control_plane`. `calico_rr` group also must be a child - group of `k8s_cluster` group. + `kube_node` and/or `kube_control_plane`. * `cluster_id` by route reflector node/group (see details [here](https://hub.docker.com/r/calico/routereflector/)) Here's an example of Kubespray inventory with standalone route reflectors: @@ -157,11 +159,6 @@ node3 node4 node5 -[k8s_cluster:children] -kube_node -kube_control_plane -calico_rr - [calico_rr] rr0 rr1 @@ -382,7 +379,7 @@ To clean up any ipvs leftovers: Calico node, typha and kube-controllers need to be able to talk to the kubernetes API. Please reference the [Enabling eBPF Calico Docs](https://docs.projectcalico.org/maintenance/ebpf/enabling-bpf) for guidelines on how to do this. -Kubespray sets up the `kubernetes-services-endpoint` configmap based on the contents of the `loadbalancer_apiserver` inventory variable documented in [HA Mode](/docs/ha-mode.md). +Kubespray sets up the `kubernetes-services-endpoint` configmap based on the contents of the `loadbalancer_apiserver` inventory variable documented in [HA Mode](/docs/operations/ha-mode.md). If no external loadbalancer is used, Calico eBPF can also use the localhost loadbalancer option. We are able to do so only if you use the same port for the localhost apiserver loadbalancer and the kube-apiserver. In this case Calico Automatic Host Endpoints need to be enabled to allow services like `coredns` and `metrics-server` to communicate with the kubernetes host endpoint. See [this blog post](https://www.projectcalico.org/securing-kubernetes-nodes-with-calico-automatic-host-endpoints/) on enabling automatic host endpoints. @@ -424,7 +421,7 @@ calico_wireguard_enabled: true The following OSes will require enabling the EPEL repo in order to bring in wireguard tools: -* CentOS 7 & 8 +* CentOS 8 * AlmaLinux 8 * Rocky Linux 8 * Amazon Linux 2 diff --git a/docs/cilium.md b/docs/CNI/cilium.md similarity index 61% rename from docs/cilium.md rename to docs/CNI/cilium.md index afc808c643e..fed25ddb5f9 100644 --- a/docs/cilium.md +++ b/docs/CNI/cilium.md @@ -45,10 +45,144 @@ cilium_pool_mask_size Specifies the size allocated to node.ipam.podCIDRs from cl cilium_pool_mask_size_ipv6: "120" ``` +### IP Load Balancer Pools + +Cilium's IP Load Balancer Pools can be configured with the `cilium_loadbalancer_ip_pools` variable: + +```yml +cilium_loadbalancer_ip_pools: + - name: "blue-pool" + cidrs: + - "10.0.10.0/24" +``` + +For further information, check [LB IPAM documentation](https://docs.cilium.io/en/stable/network/lb-ipam/) + +### BGP Control Plane + +Cilium's BGP Control Plane can be enabled by setting `cilium_enable_bgp_control_plane` to `true`.: + +```yml +cilium_enable_bgp_control_plane: true +``` + +For further information, check [BGP Peering Policy documentation](https://docs.cilium.io/en/latest/network/bgp-control-plane/bgp-control-plane-v1/) + +### BGP Control Plane Resources (New bgpv2 API v1.16+) + +Cilium BGP control plane is managed by a set of custom resources which provide a flexible way to configure BGP peers, policies, and advertisements. + +Cilium's BGP Instances can be configured with the `cilium_bgp_cluster_configs` variable: + +```yml +cilium_bgp_cluster_configs: + - name: "cilium-bgp" + spec: + bgpInstances: + - name: "instance-64512" + localASN: 64512 + peers: + - name: "peer-64512-tor1" + peerASN: 64512 + peerAddress: '10.47.1.1' + peerConfigRef: + name: "cilium-peer" + nodeSelector: + matchExpressions: + - {key: somekey, operator: NotIn, values: ['never-used-value']} +``` + +Cillium's BGP Peers can be configured with the `cilium_bgp_peer_configs` variable: + +```yml +cilium_bgp_peer_configs: + - name: cilium-peer + spec: + # authSecretRef: bgp-auth-secret + gracefulRestart: + enabled: true + restartTimeSeconds: 15 + families: + - afi: ipv4 + safi: unicast + advertisements: + matchLabels: + advertise: "bgp" + - afi: ipv6 + safi: unicast + advertisements: + matchLabels: + advertise: "bgp" +``` + +Cillium's BGP Advertisements can be configured with the `cilium_bgp_advertisements` variable: + +```yml +cilium_bgp_advertisements: + - name: bgp-advertisements + labels: + advertise: bgp + spec: + advertisements: + - advertisementType: "PodCIDR" + attributes: + communities: + standard: [ "64512:99" ] + - advertisementType: "Service" + service: + addresses: + - ClusterIP + - ExternalIP + - LoadBalancerIP + selector: + matchExpressions: + - {key: somekey, operator: NotIn, values: ['never-used-value']} +``` + +Cillium's BGP Node Config Overrides can be configured with the `cilium_bgp_node_config_overrides` variable: + +```yml +cilium_bgp_node_config_overrides: + - name: bgpv2-cplane-dev-multi-homing-worker + spec: + bgpInstances: + - name: "instance-65000" + routerID: "192.168.10.1" + localPort: 1790 + peers: + - name: "peer-65000-tor1" + localAddress: fd00:10:0:2::2 + - name: "peer-65000-tor2" + localAddress: fd00:11:0:2::2 +``` + +For further information, check [BGP Control Plane Resources documentation](https://docs.cilium.io/en/latest/network/bgp-control-plane/bgp-control-plane-v2/) + +### BGP Peering Policies (Legacy < v1.16) + +Cilium's BGP Peering Policies can be configured with the `cilium_bgp_peering_policies` variable: + +```yml +cilium_bgp_peering_policies: + - name: "01-bgp-peering-policy" + spec: + virtualRouters: + - localASN: 64512 + exportPodCIDR: false + neighbors: + - peerAddress: '10.47.1.1/24' + peerASN: 64512 + serviceSelector: + matchExpressions: + - {key: somekey, operator: NotIn, values: ['never-used-value']} +``` + +For further information, check [BGP Peering Policy documentation](https://docs.cilium.io/en/latest/network/bgp-control-plane/bgp-control-plane-v1/#bgp-peering-policy-legacy) + ## Kube-proxy replacement with Cilium Cilium can run without kube-proxy by setting `cilium_kube_proxy_replacement` -to `strict`. +to `strict` (< v1.16) or `true` (Cilium v1.16+ no longer accepts `strict`, however this is converted to `true` by kubespray when running v1.16+). Without kube-proxy, cilium needs to know the address of the kube-apiserver and this must be set globally for all Cilium components (agents and operators). @@ -99,7 +233,7 @@ cilium_operator_extra_volume_mounts: ## Choose Cilium version ```yml -cilium_version: v1.15.4 +cilium_version: v1.12.1 ``` ## Add variable to config @@ -132,7 +266,7 @@ Wireguard option is only available in Cilium 1.10.0 and newer. ### IPsec Encryption -For further information, make sure to check the official [Cilium documentation.](https://docs.cilium.io/en/stable/gettingstarted/encryption-ipsec/) +For further information, make sure to check the official [Cilium documentation.](https://docs.cilium.io/en/stable/security/network/encryption-ipsec/) To enable IPsec encryption, you just need to set three variables. @@ -157,7 +291,7 @@ echo "cilium_ipsec_key: "$(echo -n "3 rfc4106(gcm(aes)) $(echo $(dd if=/dev/uran ### Wireguard Encryption -For further information, make sure to check the official [Cilium documentation.](https://docs.cilium.io/en/stable/gettingstarted/encryption-wireguard/) +For further information, make sure to check the official [Cilium documentation.](https://docs.cilium.io/en/stable/security/network/encryption-wireguard/) To enable Wireguard encryption, you just need to set two variables. @@ -170,14 +304,14 @@ Kubespray currently supports Linux distributions with Wireguard Kernel mode on L ## Bandwidth Manager -Cilium’s bandwidth manager supports the kubernetes.io/egress-bandwidth Pod annotation. +Cilium's bandwidth manager supports the kubernetes.io/egress-bandwidth Pod annotation. Bandwidth enforcement currently does not work in combination with L7 Cilium Network Policies. In case they select the Pod at egress, then the bandwidth enforcement will be disabled for those Pods. Bandwidth Manager requires a v5.1.x or more recent Linux kernel. -For further information, make sure to check the official [Cilium documentation.](https://docs.cilium.io/en/v1.12/gettingstarted/bandwidth-manager/) +For further information, make sure to check the official [Cilium documentation](https://docs.cilium.io/en/latest/network/kubernetes/bandwidth-manager/) To use this function, set the following parameters @@ -185,6 +319,26 @@ To use this function, set the following parameters cilium_enable_bandwidth_manager: true ``` +## Host Firewall + +Host Firewall enforces security policies for Kubernetes nodes. It is disable by default, since it can break the cluster connectivity. + +```yaml +cilium_enable_host_firewall: true +``` + +For further information, check [host firewall documentation](https://docs.cilium.io/en/latest/security/host-firewall/) + +## Policy Audit Mode + +When _Policy Audit Mode_ is enabled, no network policy is enforced. This feature helps to validate the impact of host policies before enforcing them. + +```yaml +cilium_policy_audit_mode: true +``` + +It is disable by default, and should not be enabled in production. + ## Install Cilium Hubble k8s-net-cilium.yml: diff --git a/docs/cni.md b/docs/CNI/cni.md similarity index 100% rename from docs/cni.md rename to docs/CNI/cni.md diff --git a/docs/flannel.md b/docs/CNI/flannel.md similarity index 100% rename from docs/flannel.md rename to docs/CNI/flannel.md diff --git a/docs/kube-ovn.md b/docs/CNI/kube-ovn.md similarity index 100% rename from docs/kube-ovn.md rename to docs/CNI/kube-ovn.md diff --git a/docs/kube-router.md b/docs/CNI/kube-router.md similarity index 100% rename from docs/kube-router.md rename to docs/CNI/kube-router.md diff --git a/docs/macvlan.md b/docs/CNI/macvlan.md similarity index 100% rename from docs/macvlan.md rename to docs/CNI/macvlan.md diff --git a/docs/multus.md b/docs/CNI/multus.md similarity index 81% rename from docs/multus.md rename to docs/CNI/multus.md index 1f724848db3..c08ad6808e6 100644 --- a/docs/multus.md +++ b/docs/CNI/multus.md @@ -17,6 +17,22 @@ kube_network_plugin_multus: true will install Multus and Calico and configure Multus to use Calico as the primary network plugin. +Namespace isolation enables a mode where Multus only allows pods to access custom resources (the `NetworkAttachmentDefinitions`) within the namespace where that pod resides. To enable namespace isolation: + +```yml +multus_namespace_isolation: true +``` + +### Cilium compatibility + +If you are using `cilium` as the primary CNI you'll have to set `cilium_cni_exclusive` to `false` to avoid cillium reverting multus config. + +```yml +kube_network_plugin: cilium +kube_network_plugin_multus: true +cilium_cni_exclusive: false +``` + ## Using Multus Once Multus is installed, you can create CNI configurations (as a CRD objects) for additional networks, in this case a macvlan CNI configuration is defined. You may replace the config field with any valid CNI configuration where the CNI binary is available on the nodes. diff --git a/docs/weave.md b/docs/CNI/weave.md similarity index 100% rename from docs/weave.md rename to docs/CNI/weave.md diff --git a/docs/containerd.md b/docs/CRI/containerd.md similarity index 100% rename from docs/containerd.md rename to docs/CRI/containerd.md diff --git a/docs/cri-o.md b/docs/CRI/cri-o.md similarity index 100% rename from docs/cri-o.md rename to docs/CRI/cri-o.md diff --git a/docs/docker.md b/docs/CRI/docker.md similarity index 88% rename from docs/docker.md rename to docs/CRI/docker.md index b5de70274c4..aba0051d83c 100644 --- a/docs/docker.md +++ b/docs/CRI/docker.md @@ -16,14 +16,6 @@ Enabling the `overlay2` graph driver: docker_storage_options: -s overlay2 ``` -Enabling `docker_container_storage_setup`, it will configure devicemapper driver on Centos7 or RedHat7. -Deployers must be define a disk path for `docker_container_storage_setup_devs`, otherwise docker-storage-setup will be executed incorrectly. - -```yaml -docker_container_storage_setup: true -docker_container_storage_setup_devs: /dev/vdb -``` - Changing the Docker cgroup driver (native.cgroupdriver); valid options are `systemd` or `cgroupfs`, default is `systemd`: ```yaml diff --git a/docs/gvisor.md b/docs/CRI/gvisor.md similarity index 100% rename from docs/gvisor.md rename to docs/CRI/gvisor.md diff --git a/docs/kata-containers.md b/docs/CRI/kata-containers.md similarity index 100% rename from docs/kata-containers.md rename to docs/CRI/kata-containers.md diff --git a/docs/aws-ebs-csi.md b/docs/CSI/aws-ebs-csi.md similarity index 100% rename from docs/aws-ebs-csi.md rename to docs/CSI/aws-ebs-csi.md diff --git a/docs/azure-csi.md b/docs/CSI/azure-csi.md similarity index 100% rename from docs/azure-csi.md rename to docs/CSI/azure-csi.md diff --git a/docs/cinder-csi.md b/docs/CSI/cinder-csi.md similarity index 100% rename from docs/cinder-csi.md rename to docs/CSI/cinder-csi.md diff --git a/docs/gcp-pd-csi.md b/docs/CSI/gcp-pd-csi.md similarity index 100% rename from docs/gcp-pd-csi.md rename to docs/CSI/gcp-pd-csi.md diff --git a/docs/vsphere-csi.md b/docs/CSI/vsphere-csi.md similarity index 91% rename from docs/vsphere-csi.md rename to docs/CSI/vsphere-csi.md index af58440cae4..01fa9b2dc46 100644 --- a/docs/vsphere-csi.md +++ b/docs/CSI/vsphere-csi.md @@ -23,20 +23,20 @@ You need to source the vSphere credentials you use to deploy your machines that | external_vsphere_password | TRUE | string | | | Password for vCenter (Can also be specified with the `VSPHERE_PASSWORD` environment variable) | | external_vsphere_datacenter | TRUE | string | | | Datacenter name to use | | external_vsphere_kubernetes_cluster_id | TRUE | string | | "kubernetes-cluster-id" | Kubernetes cluster ID to use | -| external_vsphere_version | TRUE | string | | "6.7u3" | Vmware Vsphere version where located all VMs | -| external_vsphere_cloud_controller_image_tag | TRUE | string | | "latest" | Kubernetes cluster ID to use | -| vsphere_syncer_image_tag | TRUE | string | | "v2.2.1" | Syncer image tag to use | -| vsphere_csi_attacher_image_tag | TRUE | string | | "v3.1.0" | CSI attacher image tag to use | -| vsphere_csi_controller | TRUE | string | | "v2.2.1" | CSI controller image tag to use | +| external_vsphere_version | TRUE | string | | "7.0u1" | Vmware Vsphere version where located all VMs | +| external_vsphere_cloud_controller_image_tag | TRUE | string | | "v1.31.0" | CPI manager image tag to use | +| vsphere_syncer_image_tag | TRUE | string | | "v3.3.1" | Syncer image tag to use | +| vsphere_csi_attacher_image_tag | TRUE | string | | "v4.3.0" | CSI attacher image tag to use | +| vsphere_csi_controller | TRUE | string | | "v3.3.1" | CSI controller image tag to use | | vsphere_csi_controller_replicas | TRUE | integer | | 1 | Number of pods Kubernetes should deploy for the CSI controller | -| vsphere_csi_liveness_probe_image_tag | TRUE | string | | "v2.2.0" | CSI liveness probe image tag to use | +| vsphere_csi_liveness_probe_image_tag | TRUE | string | | "v2.10.0" | CSI liveness probe image tag to use | | vsphere_csi_provisioner_image_tag | TRUE | string | | "v2.1.0" | CSI provisioner image tag to use | -| vsphere_csi_node_driver_registrar_image_tag | TRUE | string | | "v1.1.0" | CSI node driver registrar image tag to use | -| vsphere_csi_driver_image_tag | TRUE | string | | "v1.0.2" | CSI driver image tag to use | -| vsphere_csi_resizer_tag | TRUE | string | | "v1.1.0" | CSI resizer image tag to use | +| vsphere_csi_node_driver_registrar_image_tag | TRUE | string | | "v3.5.0" | CSI node driver registrar image tag to use | +| vsphere_csi_driver_image_tag | TRUE | string | | "v3.3.1" | CSI driver image tag to use | +| vsphere_csi_resizer_tag | TRUE | string | | "v1.8.0" | CSI resizer image tag to use | | vsphere_csi_aggressive_node_drain | FALSE | boolean | | false | Enable aggressive node drain strategy | -| vsphere_csi_aggressive_node_unreachable_timeout | FALSE | int | 300 | | Timeout till node will be drained when it in an unreachable state | -| vsphere_csi_aggressive_node_not_ready_timeout | FALSE | int | 300 | | Timeout till node will be drained when it in not-ready state | +| vsphere_csi_aggressive_node_unreachable_timeout | FALSE | int | | 300 | Timeout till node will be drained when it in an unreachable state | +| vsphere_csi_aggressive_node_not_ready_timeout | FALSE | int | | 300 | Timeout till node will be drained when it in not-ready state | | vsphere_csi_namespace | TRUE | string | | "kube-system" | vSphere CSI namespace to use; kube-system for backward compatibility, should be change to vmware-system-csi on the long run | ## Usage example diff --git a/docs/_sidebar.md b/docs/_sidebar.md index 8901e69840e..c11a646df83 100644 --- a/docs/_sidebar.md +++ b/docs/_sidebar.md @@ -1,66 +1,96 @@ * [Readme](/) -* [Comparisons](/docs/comparisons.md) -* [Getting started](/docs/getting-started.md) -* [Ansible](docs/ansible.md) -* [Variables](/docs/vars.md) -* Operations - * [Integration](docs/integration.md) - * [Upgrades](/docs/upgrades.md) - * [HA Mode](docs/ha-mode.md) - * [Adding/replacing a node](docs/nodes.md) - * [Large deployments](docs/large-deployments.md) - * [Air-Gap Installation](docs/offline-environment.md) +* Advanced + * [Arch](/docs/advanced/arch.md) + * [Cert Manager](/docs/advanced/cert_manager.md) + * [Dns-stack](/docs/advanced/dns-stack.md) + * [Downloads](/docs/advanced/downloads.md) + * [Gcp-lb](/docs/advanced/gcp-lb.md) + * [Kubernetes-reliability](/docs/advanced/kubernetes-reliability.md) + * [Mitogen](/docs/advanced/mitogen.md) + * [Netcheck](/docs/advanced/netcheck.md) + * [Ntp](/docs/advanced/ntp.md) + * [Proxy](/docs/advanced/proxy.md) + * [Registry](/docs/advanced/registry.md) +* Ansible + * [Ansible](/docs/ansible/ansible.md) + * [Ansible Collection](/docs/ansible/ansible_collection.md) + * [Inventory](/docs/ansible/inventory.md) + * [Vars](/docs/ansible/vars.md) +* Cloud Controllers + * [Openstack](/docs/cloud_controllers/openstack.md) + * [Vsphere](/docs/cloud_controllers/vsphere.md) +* Cloud Providers + * [Aws](/docs/cloud_providers/aws.md) + * [Azure](/docs/cloud_providers/azure.md) + * [Cloud](/docs/cloud_providers/cloud.md) + * [Equinix-metal](/docs/cloud_providers/equinix-metal.md) * CNI - * [Calico](docs/calico.md) - * [Flannel](docs/flannel.md) - * [Cilium](docs/cilium.md) - * [Kube Router](docs/kube-router.md) - * [Kube OVN](docs/kube-ovn.md) - * [Weave](docs/weave.md) - * [Multus](docs/multus.md) -* Ingress - * [kube-vip](docs/kube-vip.md) - * [ALB Ingress](docs/ingress_controller/alb_ingress_controller.md) - * [MetalLB](docs/metallb.md) - * [Nginx Ingress](docs/ingress_controller/ingress_nginx.md) -* [Cloud providers](docs/cloud.md) - * [AWS](docs/aws.md) - * [Azure](docs/azure.md) - * [OpenStack](/docs/openstack.md) - * [Equinix Metal](/docs/equinix-metal.md) - * [vSphere](/docs/vsphere.md) -* [Operating Systems](docs/bootstrap-os.md) - * [Flatcar Container Linux](docs/flatcar.md) - * [Fedora CoreOS](docs/fcos.md) - * [OpenSUSE](docs/opensuse.md) - * [RedHat Enterprise Linux](docs/rhel.md) - * [CentOS/OracleLinux/AlmaLinux/Rocky Linux](docs/centos.md) - * [Kylin Linux Advanced Server V10](docs/kylinlinux.md) - * [Amazon Linux 2](docs/amazonlinux.md) - * [UOS Linux](docs/uoslinux.md) - * [openEuler notes](docs/openeuler.md) + * [Calico](/docs/CNI/calico.md) + * [Cilium](/docs/CNI/cilium.md) + * [Cni](/docs/CNI/cni.md) + * [Flannel](/docs/CNI/flannel.md) + * [Kube-ovn](/docs/CNI/kube-ovn.md) + * [Kube-router](/docs/CNI/kube-router.md) + * [Macvlan](/docs/CNI/macvlan.md) + * [Multus](/docs/CNI/multus.md) + * [Weave](/docs/CNI/weave.md) * CRI - * [Containerd](docs/containerd.md) - * [Docker](docs/docker.md) - * [CRI-O](docs/cri-o.md) - * [Kata Containers](docs/kata-containers.md) - * [gVisor](docs/gvisor.md) -* Advanced - * [Proxy](/docs/proxy.md) - * [Downloads](docs/downloads.md) - * [Netcheck](docs/netcheck.md) - * [Cert Manager](docs/cert_manager.md) - * [DNS Stack](docs/dns-stack.md) - * [Kubernetes reliability](docs/kubernetes-reliability.md) - * [Local Registry](docs/kubernetes-apps/registry.md) - * [NTP](docs/ntp.md) -* External Storage Provisioners - * [RBD Provisioner](docs/kubernetes-apps/rbd_provisioner.md) - * [CEPHFS Provisioner](docs/kubernetes-apps/cephfs_provisioner.md) - * [Local Volume Provisioner](docs/kubernetes-apps/local_volume_provisioner.md) + * [Containerd](/docs/CRI/containerd.md) + * [Cri-o](/docs/CRI/cri-o.md) + * [Docker](/docs/CRI/docker.md) + * [Gvisor](/docs/CRI/gvisor.md) + * [Kata-containers](/docs/CRI/kata-containers.md) +* CSI + * [Aws-ebs-csi](/docs/CSI/aws-ebs-csi.md) + * [Azure-csi](/docs/CSI/azure-csi.md) + * [Cinder-csi](/docs/CSI/cinder-csi.md) + * [Gcp-pd-csi](/docs/CSI/gcp-pd-csi.md) + * [Vsphere-csi](/docs/CSI/vsphere-csi.md) * Developers - * [Test cases](docs/test_cases.md) - * [Vagrant](docs/vagrant.md) - * [CI Matrix](docs/ci.md) - * [CI Setup](docs/ci-setup.md) -* [Roadmap](docs/roadmap.md) + * [Ci-setup](/docs/developers/ci-setup.md) + * [Ci](/docs/developers/ci.md) + * [Test Cases](/docs/developers/test_cases.md) + * [Vagrant](/docs/developers/vagrant.md) +* External Storage Provisioners + * [Cephfs Provisioner](/docs/external_storage_provisioners/cephfs_provisioner.md) + * [Local Volume Provisioner](/docs/external_storage_provisioners/local_volume_provisioner.md) + * [Rbd Provisioner](/docs/external_storage_provisioners/rbd_provisioner.md) + * [Scheduler Plugins](/docs/external_storage_provisioners/scheduler_plugins.md) +* Getting Started + * [Comparisons](/docs/getting_started/comparisons.md) + * [Getting-started](/docs/getting_started/getting-started.md) + * [Setting-up-your-first-cluster](/docs/getting_started/setting-up-your-first-cluster.md) +* Ingress + * [Alb Ingress Controller](/docs/ingress/alb_ingress_controller.md) + * [Ingress Nginx](/docs/ingress/ingress_nginx.md) + * [Kube-vip](/docs/ingress/kube-vip.md) + * [Metallb](/docs/ingress/metallb.md) +* Operating Systems + * [Amazonlinux](/docs/operating_systems/amazonlinux.md) + * [Bootstrap-os](/docs/operating_systems/bootstrap-os.md) + * [Centos](/docs/operating_systems/centos.md) + * [Fcos](/docs/operating_systems/fcos.md) + * [Flatcar](/docs/operating_systems/flatcar.md) + * [Kylinlinux](/docs/operating_systems/kylinlinux.md) + * [Openeuler](/docs/operating_systems/openeuler.md) + * [Opensuse](/docs/operating_systems/opensuse.md) + * [Rhel](/docs/operating_systems/rhel.md) + * [Uoslinux](/docs/operating_systems/uoslinux.md) +* Operations + * [Cgroups](/docs/operations/cgroups.md) + * [Encrypting-secret-data-at-rest](/docs/operations/encrypting-secret-data-at-rest.md) + * [Etcd](/docs/operations/etcd.md) + * [Ha-mode](/docs/operations/ha-mode.md) + * [Hardening](/docs/operations/hardening.md) + * [Integration](/docs/operations/integration.md) + * [Large-deployments](/docs/operations/large-deployments.md) + * [Mirror](/docs/operations/mirror.md) + * [Nodes](/docs/operations/nodes.md) + * [Offline-environment](/docs/operations/offline-environment.md) + * [Port-requirements](/docs/operations/port-requirements.md) + * [Recover-control-plane](/docs/operations/recover-control-plane.md) + * [Upgrades](/docs/operations/upgrades.md) +* Roadmap + * [Roadmap](/docs/roadmap/roadmap.md) +* Upgrades + * [Migrate Docker2containerd](/docs/upgrades/migrate_docker2containerd.md) diff --git a/docs/arch.md b/docs/advanced/arch.md similarity index 100% rename from docs/arch.md rename to docs/advanced/arch.md diff --git a/docs/cert_manager.md b/docs/advanced/cert_manager.md similarity index 100% rename from docs/cert_manager.md rename to docs/advanced/cert_manager.md diff --git a/docs/dns-stack.md b/docs/advanced/dns-stack.md similarity index 98% rename from docs/dns-stack.md rename to docs/advanced/dns-stack.md index ae560d4a8c1..9524d2f85d2 100644 --- a/docs/dns-stack.md +++ b/docs/advanced/dns-stack.md @@ -64,6 +64,10 @@ Custom options to be added to the kubernetes coredns plugin. Extra domains to be forwarded to the kubernetes coredns plugin. +### coredns_additional_configs + +Extra configuration to be added to CoreDNS configuration + ### coredns_rewrite_block [Rewrite](https://coredns.io/plugins/rewrite/) plugin block to perform internal message rewriting. @@ -290,6 +294,10 @@ nodelocaldns_external_zones: See [dns_etchosts](#dns_etchosts-coredns) above. +### nodelocaldns_additional_configs + +Extra configuration to be added to CoreDNS configuration + ### Nodelocal DNS HA Under some circumstances the single POD nodelocaldns implementation may not be able to be replaced soon enough and a cluster upgrade or a nodelocaldns upgrade can cause DNS requests to time out for short intervals. If for any reason your applications cannot tolerate this behavior you can enable a redundant nodelocal DNS pod on each node: diff --git a/docs/downloads.md b/docs/advanced/downloads.md similarity index 100% rename from docs/downloads.md rename to docs/advanced/downloads.md diff --git a/docs/gcp-lb.md b/docs/advanced/gcp-lb.md similarity index 85% rename from docs/gcp-lb.md rename to docs/advanced/gcp-lb.md index 8e8f8c427b0..80369186a34 100644 --- a/docs/gcp-lb.md +++ b/docs/advanced/gcp-lb.md @@ -1,5 +1,7 @@ # GCP Load Balancers for type=LoadBalacer of Kubernetes Services +> **Removed**: Since v1.31 (the Kubespray counterpart is v2.27), Kubernetes no longer supports `cloud_provider`. (except external cloud provider) + Google Cloud Platform can be used for creation of Kubernetes Service Load Balancer. This feature is able to deliver by adding parameters to `kube-controller-manager` and `kubelet`. You need specify: diff --git a/docs/kubernetes-reliability.md b/docs/advanced/kubernetes-reliability.md similarity index 100% rename from docs/kubernetes-reliability.md rename to docs/advanced/kubernetes-reliability.md diff --git a/docs/mitogen.md b/docs/advanced/mitogen.md similarity index 100% rename from docs/mitogen.md rename to docs/advanced/mitogen.md diff --git a/docs/netcheck.md b/docs/advanced/netcheck.md similarity index 100% rename from docs/netcheck.md rename to docs/advanced/netcheck.md diff --git a/docs/ntp.md b/docs/advanced/ntp.md similarity index 89% rename from docs/ntp.md rename to docs/advanced/ntp.md index a91e09efc2b..a928d771aee 100644 --- a/docs/ntp.md +++ b/docs/advanced/ntp.md @@ -48,3 +48,9 @@ Force sync time immediately by NTP after the ntp installed, which is useful in n ```ShellSession ntp_force_sync_immediately: true ``` + +When using Ubuntu 24.04 or a distribution that already has `systemd-timesyncd` installed, use the `ntpsec` package. + +```ShellSession +ntp_package: ntpsec +``` diff --git a/docs/proxy.md b/docs/advanced/proxy.md similarity index 100% rename from docs/proxy.md rename to docs/advanced/proxy.md diff --git a/docs/kubernetes-apps/registry.md b/docs/advanced/registry.md similarity index 100% rename from docs/kubernetes-apps/registry.md rename to docs/advanced/registry.md diff --git a/docs/ansible.md b/docs/ansible/ansible.md similarity index 66% rename from docs/ansible.md rename to docs/ansible/ansible.md index 3fbf57ea501..dd428880251 100644 --- a/docs/ansible.md +++ b/docs/ansible/ansible.md @@ -34,100 +34,22 @@ Based on the table below and the available python version for your ansible host |-----------------|----------------| | >= 2.16.4 | 3.10-3.12 | -## Inventory +## Customize Ansible vars -The inventory is composed of 3 groups: - -* **kube_node** : list of kubernetes nodes where the pods will run. -* **kube_control_plane** : list of servers where kubernetes control plane components (apiserver, scheduler, controller) will run. -* **etcd**: list of servers to compose the etcd server. You should have at least 3 servers for failover purpose. - -Note: do not modify the children of _k8s_cluster_, like putting -the _etcd_ group into the _k8s_cluster_, unless you are certain -to do that and you have it fully contained in the latter: - -```ShellSession -etcd ⊂ k8s_cluster => kube_node ∩ etcd = etcd -``` - -When _kube_node_ contains _etcd_, you define your etcd cluster to be as well schedulable for Kubernetes workloads. -If you want it a standalone, make sure those groups do not intersect. -If you want the server to act both as control-plane and node, the server must be defined -on both groups _kube_control_plane_ and _kube_node_. If you want a standalone and -unschedulable control plane, the server must be defined only in the _kube_control_plane_ and -not _kube_node_. - -There are also two special groups: - -* **calico_rr** : explained for [advanced Calico networking cases](/docs/calico.md) -* **bastion** : configure a bastion host if your nodes are not directly reachable - -Below is a complete inventory example: - -```ini -## Configure 'ip' variable to bind kubernetes services on a -## different ip than the default iface -node1 ansible_host=95.54.0.12 ip=10.3.0.1 -node2 ansible_host=95.54.0.13 ip=10.3.0.2 -node3 ansible_host=95.54.0.14 ip=10.3.0.3 -node4 ansible_host=95.54.0.15 ip=10.3.0.4 -node5 ansible_host=95.54.0.16 ip=10.3.0.5 -node6 ansible_host=95.54.0.17 ip=10.3.0.6 - -[kube_control_plane] -node1 -node2 - -[etcd] -node1 -node2 -node3 - -[kube_node] -node2 -node3 -node4 -node5 -node6 - -[k8s_cluster:children] -kube_node -kube_control_plane -``` - -## Group vars and overriding variables precedence - -The group variables to control main deployment options are located in the directory ``inventory/sample/group_vars``. -Optional variables are located in the `inventory/sample/group_vars/all.yml`. -Mandatory variables that are common for at least one role (or a node group) can be found in the -`inventory/sample/group_vars/k8s_cluster.yml`. -There are also role vars for docker, kubernetes preinstall and control plane roles. -According to the [ansible docs](https://docs.ansible.com/ansible/latest/playbooks_variables.html#variable-precedence-where-should-i-put-a-variable), -those cannot be overridden from the group vars. In order to override, one should use -the `-e` runtime flags (most simple way) or other layers described in the docs. - -Kubespray uses only a few layers to override things (or expect them to -be overridden for roles): +Kubespray expects users to use one of the following variables sources for settings and customization: | Layer | Comment | |----------------------------------------|------------------------------------------------------------------------------| -| **role defaults** | provides best UX to override things for Kubespray deployments | -| inventory vars | Unused | -| **inventory group_vars** | Expects users to use ``all.yml``,``k8s_cluster.yml`` etc. to override things | -| inventory host_vars | Unused | -| playbook group_vars | Unused | -| playbook host_vars | Unused | -| **host facts** | Kubespray overrides for internal roles' logic, like state flags | -| play vars | Unused | -| play vars_prompt | Unused | -| play vars_files | Unused | -| registered vars | Unused | -| set_facts | Kubespray overrides those, for some places | -| **role and include vars** | Provides bad UX to override things! Use extra vars to enforce | -| block vars (only for tasks in block) | Kubespray overrides for internal roles' logic | -| task vars (only for the task) | Unused for roles, but only for helper scripts | +| inventory vars | | +| - **inventory group_vars** | most used | +| - inventory host_vars | host specifc vars overrides, group_vars is usually more practical | | **extra vars** (always win precedence) | override with ``ansible-playbook -e @foo.yml`` | +[!IMPORTANT] +Extra vars are best used to override kubespray internal variables, for instances, roles/vars/. +Those vars are usually **not expected** (by Kubespray developers) to be modified by end users, and not part of Kubespray +interface. Thus they can change, disappear, or break stuff unexpectedly. + ## Ansible tags The following tags are defined in playbooks: @@ -155,6 +77,7 @@ The following tags are defined in playbooks: | container_engine_accelerator | Enable nvidia accelerator for runtimes | | container-engine | Configuring container engines | | container-runtimes | Configuring container runtimes | +| control-plane | Configuring K8s control plane node role | | coredns | Configuring coredns deployment | | crio | Configuring crio container engine for hosts | | crun | Configuring crun runtime | @@ -182,8 +105,6 @@ The following tags are defined in playbooks: | init | Windows kubernetes init nodes | | iptables | Flush and clear iptable when resetting | | k8s-pre-upgrade | Upgrading K8s cluster | -| k8s-secrets | Configuring K8s certs/keys | -| k8s-gen-tokens | Configuring K8s tokens | | kata-containers | Configuring kata-containers runtime | | krew | Install and manage krew | | kubeadm | Roles linked to kubeadm tasks | @@ -199,7 +120,7 @@ The following tags are defined in playbooks: | local-path-provisioner | Configure External provisioner: local-path | | local-volume-provisioner | Configure External provisioner: local-volume | | macvlan | Network plugin macvlan | -| master | Configuring K8s master node role | +| master (DEPRECATED) | Deprecated - see `control-plane` | | metallb | Installing and configuring metallb | | metrics_server | Configuring metrics_server | | netchecker | Installing netchecker K8s app | @@ -210,7 +131,7 @@ The following tags are defined in playbooks: | node | Configuring K8s minion (compute) node role | | nodelocaldns | Configuring nodelocaldns daemonset | | node-label | Tasks linked to labeling of nodes | -| node-webhook | Tasks linked to webhook (grating access to resources) | +| node-webhook | Tasks linked to webhook (granting access to resources)| | nvidia_gpu | Enable nvidia accelerator for runtimes | | oci | Cloud provider: oci | | persistent_volumes | Configure csi volumes | @@ -231,6 +152,7 @@ The following tags are defined in playbooks: | services | Remove services (etcd, kubelet etc...) when resetting | | snapshot | Enabling csi snapshot | | snapshot-controller | Configuring csi snapshot controller | +| system-packages | Install packages using OS package manager | | upgrade | Upgrading, f.e. container images/binaries | | upload | Distributing images/binaries across hosts | | vsphere-csi-driver | Configuring csi driver: vsphere | @@ -266,42 +188,32 @@ ansible-playbook -i inventory/sample/hosts.ini cluster.yml \ --tags download --skip-tags upload,upgrade ``` -Note: use `--tags` and `--skip-tags` wise and only if you're 100% sure what you're doing. - -## Bastion host - -If you prefer to not make your nodes publicly accessible (nodes with private IPs only), -you can use a so-called _bastion_ host to connect to your nodes. To specify and use a bastion, -simply add a line to your inventory, where you have to replace x.x.x.x with the public IP of the -bastion host. - -```ShellSession -[bastion] -bastion ansible_host=x.x.x.x -``` - -For more information about Ansible and bastion hosts, read -[Running Ansible Through an SSH Bastion Host](https://blog.scottlowe.org/2015/12/24/running-ansible-through-ssh-bastion-host/) +Note: use `--tags` and `--skip-tags` wisely and only if you're 100% sure what you're doing. ## Mitogen -Mitogen support is deprecated, please see [mitogen related docs](/docs/mitogen.md) for usage and reasons for deprecation. +Mitogen support is deprecated, please see [mitogen related docs](/docs/advanced/mitogen.md) for usage and reasons for deprecation. -## Beyond ansible 2.9 +## Troubleshooting Ansible issues -Ansible project has decided, in order to ease their maintenance burden, to split between -two projects which are now joined under the Ansible umbrella. +Having the wrong version of ansible, ansible collections or python dependencies can cause issue. +In particular, Kubespray ship custom modules which Ansible needs to find, for which you should specify [ANSIBLE_LIBRAY](https://docs.ansible.com/ansible/latest/dev_guide/developing_locally.html#adding-a-module-or-plugin-outside-of-a-collection) -Ansible-base (2.10.x branch) will contain just the ansible language implementation while -ansible modules that were previously bundled into a single repository will be part of the -ansible 3.x package. Please see [this blog post](https://blog.while-true-do.io/ansible-release-3-0-0/) -that explains in detail the need and the evolution plan. +```ShellSession +export ANSIBLE_LIBRAY=/library` +``` -**Note:** this change means that ansible virtual envs cannot be upgraded with `pip install -U`. -You first need to uninstall your old ansible (pre 2.10) version and install the new one. +A simple way to ensure you get all the correct version of Ansible is to use +the [pre-built docker image from Quay](https://quay.io/repository/kubespray/kubespray?tab=tags). +You will then need to use [bind mounts](https://docs.docker.com/storage/bind-mounts/) +to access the inventory and SSH key in the container, like this: ```ShellSession -pip uninstall ansible ansible-base ansible-core -cd kubespray/ -pip install -U . +git checkout v2.26.0 +docker pull quay.io/kubespray/kubespray:v2.26.0 +docker run --rm -it --mount type=bind,source="$(pwd)"/inventory/sample,dst=/inventory \ + --mount type=bind,source="${HOME}"/.ssh/id_rsa,dst=/root/.ssh/id_rsa \ + quay.io/kubespray/kubespray:v2.26.0 bash +# Inside the container you may now run the kubespray playbooks: +ansible-playbook -i /inventory/inventory.ini --private-key /root/.ssh/id_rsa cluster.yml ``` diff --git a/docs/ansible_collection.md b/docs/ansible/ansible_collection.md similarity index 100% rename from docs/ansible_collection.md rename to docs/ansible/ansible_collection.md diff --git a/docs/ansible/inventory.md b/docs/ansible/inventory.md new file mode 100644 index 00000000000..58ea690b215 --- /dev/null +++ b/docs/ansible/inventory.md @@ -0,0 +1,71 @@ +# Inventory + +The inventory is composed of 3 groups: + +* **kube_node** : list of kubernetes nodes where the pods will run. +* **kube_control_plane** : list of servers where kubernetes control plane components (apiserver, scheduler, controller) will run. +* **etcd**: list of servers to compose the etcd server. You should have at least 3 servers for failover purpose. + +When _kube_node_ contains _etcd_, you define your etcd cluster to be as well schedulable for Kubernetes workloads. +If you want it a standalone, make sure those groups do not intersect. +If you want the server to act both as control-plane and node, the server must be defined +on both groups _kube_control_plane_ and _kube_node_. If you want a standalone and +unschedulable control plane, the server must be defined only in the _kube_control_plane_ and +not _kube_node_. + +There are also two special groups: + +* **calico_rr** : explained for [advanced Calico networking cases](/docs/CNI/calico.md) +* **bastion** : configure a bastion host if your nodes are not directly reachable + +Lastly, the **k8s_cluster** is dynamically defined as the union of **kube_node**, **kube_control_plane** and **calico_rr**. +This is used internally and for the purpose of defining whole cluster variables (`/group_vars/k8s_cluster/*.yml`) + +Below is a complete inventory example: + +```ini +## Configure 'ip' variable to bind kubernetes services on a +## different ip than the default iface +node1 ansible_host=95.54.0.12 ip=10.3.0.1 +node2 ansible_host=95.54.0.13 ip=10.3.0.2 +node3 ansible_host=95.54.0.14 ip=10.3.0.3 +node4 ansible_host=95.54.0.15 ip=10.3.0.4 +node5 ansible_host=95.54.0.16 ip=10.3.0.5 +node6 ansible_host=95.54.0.17 ip=10.3.0.6 + +[kube_control_plane] +node1 +node2 + +[etcd] +node1 +node2 +node3 + +[kube_node] +node2 +node3 +node4 +node5 +node6 +``` + +## Inventory customization + +See [Customize Ansible vars](/docs/ansible/ansible.md#customize-ansible-vars) +and [Ansible documentation on group_vars](https://docs.ansible.com/ansible/latest/inventory_guide/intro_inventory.html#assigning-a-variable-to-many-machines-group-variables) + +## Bastion host + +If you prefer to not make your nodes publicly accessible (nodes with private IPs only), +you can use a so-called _bastion_ host to connect to your nodes. To specify and use a bastion, +simply add a line to your inventory, where you have to replace x.x.x.x with the public IP of the +bastion host. + +```ShellSession +[bastion] +bastion ansible_host=x.x.x.x +``` + +For more information about Ansible and bastion hosts, read +[Running Ansible Through an SSH Bastion Host](https://blog.scottlowe.org/2015/12/24/running-ansible-through-ssh-bastion-host/) diff --git a/docs/vars.md b/docs/ansible/vars.md similarity index 91% rename from docs/vars.md rename to docs/ansible/vars.md index 0e83b18311a..0bda826f57b 100644 --- a/docs/vars.md +++ b/docs/ansible/vars.md @@ -35,9 +35,9 @@ Some variables of note include: ## Addressing variables * *ip* - IP to use for binding services (host var). This would **usually** be the public ip. -* *access_ip* - IP for other hosts to use to connect to. Often required when - deploying from a cloud, such as OpenStack or GCE and you have separate - public/floating and private IPs. This would **usually** be the private ip. +* *access_ip* - IP to use from other hosts to connect to this host. Often required when deploying + from a cloud, such as OpenStack or GCE and you have separate public/floating and private IPs. + This would **usually** be the private ip. * *ansible_default_ipv4.address* - Not Kubespray-specific, but it is used if ip and access_ip are undefined * *ip6* - IPv6 address to use for binding services. (host var) @@ -46,11 +46,11 @@ Some variables of note include: * *loadbalancer_apiserver* - If defined, all hosts will connect to this address instead of localhost for kube_control_planes and kube_control_plane[0] for kube_nodes. See more details in the - [HA guide](/docs/ha-mode.md). + [HA guide](/docs/operations/ha-mode.md). * *loadbalancer_apiserver_localhost* - makes all hosts to connect to the apiserver internally load balanced endpoint. Mutual exclusive to the `loadbalancer_apiserver`. See more details in the - [HA guide](/docs/ha-mode.md). + [HA guide](/docs/operations/ha-mode.md). ## Cluster variables @@ -104,8 +104,7 @@ following default cluster parameters: * *enable_coredns_k8s_endpoint_pod_names* - If enabled, it configures endpoint_pod_names option for kubernetes plugin. on the CoreDNS service. -* *cloud_provider* - Enable extra Kubelet option if operating inside GCE or - OpenStack (default is unset) +* *cloud_provider* - The provider for cloud services. (default is unset, Set to `external` for running with an external cloud provider) * *kube_feature_gates* - A list of key=value pairs that describe feature gates for alpha/experimental Kubernetes features. (defaults is `[]`). @@ -172,7 +171,7 @@ variables to match your requirements. * *dns_upstream_forward_extra_opts* - Options to add in the forward section of coredns/nodelocaldns related to upstream DNS servers For more information, see [DNS -Stack](https://github.com/kubernetes-sigs/kubespray/blob/master/docs/dns-stack.md). +Stack](https://github.com/kubernetes-sigs/kubespray/blob/master/docs/advanced/dns-stack.md). ## Other service variables @@ -216,6 +215,8 @@ Stack](https://github.com/kubernetes-sigs/kubespray/blob/master/docs/dns-stack.m The percent is calculated by dividing this field value by 100, so the field value must be between 0 and 100, inclusive. When specified, the value must be less than imageGCHighThresholdPercent. Default: 80 +* *kubelet_max_parallel_image_pulls* - Sets the maximum number of image pulls in parallel. The value is `1` by default which means the default is serial image pulling, set it to a integer great than `1` to enable image pulling in parallel. + * *kubelet_make_iptables_util_chains* - If `true`, causes the kubelet ensures a set of `iptables` rules are present on host. * *kubelet_cpu_manager_policy* - If set to `static`, allows pods with certain resource characteristics to be granted increased CPU affinity and exclusivity on the node. And it should be set with `kube_reserved` or `system-reserved`, enable this with the following guide:[Control CPU Management Policies on the Node](https://kubernetes.io/docs/tasks/administer-cluster/cpu-management-policies/) @@ -243,6 +244,10 @@ kubelet_cpu_manager_policy_options: By default the `kubelet_secure_addresses` is set with the `10.0.0.110` the ansible control host uses `eth0` to connect to the machine. In case you want to use `eth1` as the outgoing interface on which `kube-apiserver` connects to the `kubelet`s, you should override the variable in this way: `kubelet_secure_addresses: "192.168.1.110"`. +* *kubelet_systemd_wants_dependencies* - List of kubelet service dependencies, other than container runtime. + + If you use nfs dynamically mounted volumes, sometimes rpc-statd does not start within the kubelet. You can fix it with this parameter : `kubelet_systemd_wants_dependencies: ["rpc-statd.service"]` This will add `Wants=rpc-statd.service` in `[Unit]` section of /etc/systemd/system/kubelet.service + * *node_labels* - Labels applied to nodes via `kubectl label node`. For example, labels can be set in the inventory as variables or more widely in group_vars. *node_labels* can only be defined as a dict: @@ -290,8 +295,8 @@ node_taints: For all kube components, custom flags can be passed in. This allows for edge cases where users need changes to the default deployment that may not be applicable to all deployments. -Extra flags for the kubelet can be specified using these variables, -in the form of dicts of key-value pairs of configuration parameters that will be inserted into the kubelet YAML config file. The `kubelet_node_config_extra_args` apply kubelet settings only to nodes and not control planes. Example: +Extra flags for the kubelet can be specified using these variables, in the form of dicts of key-value pairs of +configuration parameters that will be inserted into the kubelet YAML config file. Example: ```yml kubelet_config_extra_args: @@ -306,14 +311,10 @@ kubelet_config_extra_args: The possible vars are: * *kubelet_config_extra_args* -* *kubelet_node_config_extra_args* Previously, the same parameters could be passed as flags to kubelet binary with the following vars: * *kubelet_custom_flags* -* *kubelet_node_custom_flags* - -The `kubelet_node_custom_flags` apply kubelet settings only to nodes and not control planes. Example: ```yml kubelet_custom_flags: @@ -331,6 +332,13 @@ in the form of dicts of key-value pairs of configuration parameters that will be * *kube_kubeadm_controller_extra_args* * *kube_kubeadm_scheduler_extra_args* +### Kubeadm patches + +When extra flags are not sufficient and there is a need to further customize kubernetes components, +[kubeadm patches](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/control-plane-flags/#patches) +can be used. +You should use the [`kubeadm_patches` variable](../../roles/kubernetes/kubeadm_common/defaults/main.yml) for that purpose. + ## App variables * *helm_version* - Only supports v3.x. Existing v2 installs (with Tiller) will not be modified and need to be removed manually. diff --git a/docs/calico_peer_example/new-york.yml b/docs/calico_peer_example/new-york.yml index af497a9a910..3f44e5593f6 100644 --- a/docs/calico_peer_example/new-york.yml +++ b/docs/calico_peer_example/new-york.yml @@ -2,9 +2,13 @@ # peers: # - router_id: "10.99.0.34" # as: "65xxx" +# filters: [] +# numallowedlocalasnumbers: 0 # sourceaddress: "None" # - router_id: "10.99.0.35" # as: "65xxx" +# filters: [] +# numallowedlocalasnumbers: 0 # sourceaddress: "None" # loadbalancer_apiserver: diff --git a/docs/calico_peer_example/paris.yml b/docs/calico_peer_example/paris.yml index 1768e030589..746a7ddc7b8 100644 --- a/docs/calico_peer_example/paris.yml +++ b/docs/calico_peer_example/paris.yml @@ -2,9 +2,13 @@ # peers: # - router_id: "10.99.0.2" # as: "65xxx" +# filters: [] +# numallowedlocalasnumbers: 0 # sourceaddress: "None" # - router_id: "10.99.0.3" # as: "65xxx" +# filters: [] +# numallowedlocalasnumbers: 0 # sourceaddress: "None" # loadbalancer_apiserver: diff --git a/docs/centos.md b/docs/centos.md deleted file mode 100644 index 4b6b7331e39..00000000000 --- a/docs/centos.md +++ /dev/null @@ -1,12 +0,0 @@ -# CentOS and derivatives - -## CentOS 7 - -The maximum python version officially supported in CentOS is 3.6. Ansible as of version 5 (ansible core 2.12.x) increased their python requirement to python 3.8 and above. -Kubespray supports multiple ansible versions but only the default (5.x) gets wide testing coverage. If your deployment host is CentOS 7 it is recommended to use one of the earlier versions still supported. - -## CentOS 8 - -If you have containers that are using iptables in the host network namespace (`hostNetwork=true`), -you need to ensure they are using iptables-nft. -An example how k8s do the autodetection can be found [in this PR](https://github.com/kubernetes/kubernetes/pull/82966) diff --git a/docs/ci.md b/docs/ci.md deleted file mode 100644 index 0fed5e1062e..00000000000 --- a/docs/ci.md +++ /dev/null @@ -1,57 +0,0 @@ -# CI test coverage - -To generate this Matrix run `./tests/scripts/md-table/main.py` - -## containerd - -| OS / CNI | calico | cilium | custom_cni | flannel | kube-ovn | kube-router | macvlan | weave | -|---| --- | --- | --- | --- | --- | --- | --- | --- | -almalinux8 | :white_check_mark: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | -amazon | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | -centos7 | :white_check_mark: | :x: | :x: | :white_check_mark: | :x: | :white_check_mark: | :x: | :white_check_mark: | -debian10 | :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | -debian11 | :white_check_mark: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | -debian12 | :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | -fedora37 | :white_check_mark: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | -fedora38 | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | -opensuse | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | -rockylinux8 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | -rockylinux9 | :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | -ubuntu20 | :white_check_mark: | :white_check_mark: | :x: | :white_check_mark: | :x: | :white_check_mark: | :x: | :white_check_mark: | -ubuntu22 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | - -## crio - -| OS / CNI | calico | cilium | custom_cni | flannel | kube-ovn | kube-router | macvlan | weave | -|---| --- | --- | --- | --- | --- | --- | --- | --- | -almalinux8 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | -amazon | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | -centos7 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | -debian10 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | -debian11 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | -debian12 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | -fedora37 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | -fedora38 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | -opensuse | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | -rockylinux8 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | -rockylinux9 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | -ubuntu20 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | -ubuntu22 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | - -## docker - -| OS / CNI | calico | cilium | custom_cni | flannel | kube-ovn | kube-router | macvlan | weave | -|---| --- | --- | --- | --- | --- | --- | --- | --- | -almalinux8 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | -amazon | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | -centos7 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | -debian10 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | -debian11 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | -debian12 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | -fedora37 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | -fedora38 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: | -opensuse | :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | -rockylinux8 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | -rockylinux9 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | -ubuntu20 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: | -ubuntu22 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | diff --git a/docs/openstack.md b/docs/cloud_controllers/openstack.md similarity index 98% rename from docs/openstack.md rename to docs/cloud_controllers/openstack.md index 6f53da53444..2f07b64e84c 100644 --- a/docs/openstack.md +++ b/docs/cloud_controllers/openstack.md @@ -1,4 +1,3 @@ - # OpenStack ## Known compatible public clouds @@ -77,9 +76,9 @@ The cloud provider is configured to have Octavia by default in Kubespray. external_openstack_lbaas_network_id: "Neutron network ID to create LBaaS VIP" external_openstack_lbaas_manage_security_groups: false external_openstack_lbaas_create_monitor: false - external_openstack_lbaas_monitor_delay: 5 + external_openstack_lbaas_monitor_delay: 5s external_openstack_lbaas_monitor_max_retries: 1 - external_openstack_lbaas_monitor_timeout: 3 + external_openstack_lbaas_monitor_timeout: 3s external_openstack_lbaas_internal_lb: false ``` diff --git a/docs/vsphere.md b/docs/cloud_controllers/vsphere.md similarity index 99% rename from docs/vsphere.md rename to docs/cloud_controllers/vsphere.md index a75a25d9a7c..72a2c1dbad0 100644 --- a/docs/vsphere.md +++ b/docs/cloud_controllers/vsphere.md @@ -54,7 +54,7 @@ external_vsphere_kubernetes_cluster_id: "kubernetes-cluster-id" vsphere_csi_enabled: true ``` -For a more fine-grained CSI setup, refer to the [vsphere-csi](/docs/vsphere-csi.md) documentation. +For a more fine-grained CSI setup, refer to the [vsphere-csi](/docs/CSI/vsphere-csi.md) documentation. ### Deployment diff --git a/docs/aws.md b/docs/cloud_providers/aws.md similarity index 98% rename from docs/aws.md rename to docs/cloud_providers/aws.md index 45938980c3e..41706fdd568 100644 --- a/docs/aws.md +++ b/docs/cloud_providers/aws.md @@ -1,5 +1,7 @@ # AWS +> **Removed**: Since v1.31 (the Kubespray counterpart is v2.27), Kubernetes no longer supports `cloud_provider`. (except external cloud provider) + To deploy kubespray on [AWS](https://aws.amazon.com/) uncomment the `cloud_provider` option in `group_vars/all.yml` and set it to `'aws'`. Refer to the [Kubespray Configuration](#kubespray-configuration) for customizing the provider. Prior to creating your instances, you **must** ensure that you have created IAM roles and policies for both "kubernetes-master" and "kubernetes-node". You can find the IAM policies [here](https://github.com/kubernetes-sigs/kubespray/tree/master/contrib/aws_iam/). See the [IAM Documentation](https://aws.amazon.com/documentation/iam/) if guidance is needed on how to set these up. When you bring your instances online, associate them with the respective IAM role. Nodes that are only to be used for Etcd do not need a role. diff --git a/docs/azure.md b/docs/cloud_providers/azure.md similarity index 97% rename from docs/azure.md rename to docs/cloud_providers/azure.md index a164ea75701..50d2f1d1c2b 100644 --- a/docs/azure.md +++ b/docs/cloud_providers/azure.md @@ -1,5 +1,7 @@ # Azure +> **Removed**: Since v1.31 (the Kubespray counterpart is v2.27), Kubernetes no longer supports `cloud_provider`. (except external cloud provider) + To deploy Kubernetes on [Azure](https://azure.microsoft.com) uncomment the `cloud_provider` option in `group_vars/all/all.yml` and set it to `'azure'`. All your instances are required to run in a resource group and a routing table has to be attached to the subnet your instances are in. diff --git a/docs/cloud.md b/docs/cloud_providers/cloud.md similarity index 72% rename from docs/cloud.md rename to docs/cloud_providers/cloud.md index d7fcfef7fdc..d88a3aeccc6 100644 --- a/docs/cloud.md +++ b/docs/cloud_providers/cloud.md @@ -1,5 +1,7 @@ # Cloud providers +> **Removed**: Since v1.31 (the Kubespray counterpart is v2.27), Kubernetes no longer supports `cloud_provider`. (except external cloud provider) + ## Provisioning You can deploy instances in your cloud environment in several ways. Examples include Terraform, Ansible (ec2 and gce modules), and manual creation. diff --git a/docs/equinix-metal.md b/docs/cloud_providers/equinix-metal.md similarity index 98% rename from docs/equinix-metal.md rename to docs/cloud_providers/equinix-metal.md index ccdabaed29e..3616dc9ef4d 100644 --- a/docs/equinix-metal.md +++ b/docs/cloud_providers/equinix-metal.md @@ -54,7 +54,7 @@ cd kubespray ## Install Ansible -Install Ansible according to [Ansible installation guide](/docs/ansible.md#installing-ansible) +Install Ansible according to [Ansible installation guide](/docs/ansible/ansible.md#installing-ansible) ## Cluster Definition diff --git a/docs/ci-setup.md b/docs/developers/ci-setup.md similarity index 97% rename from docs/ci-setup.md rename to docs/developers/ci-setup.md index a9d380eab75..e397c579b76 100644 --- a/docs/ci-setup.md +++ b/docs/developers/ci-setup.md @@ -5,8 +5,8 @@ 1. build: build a docker image to be used in the pipeline 2. unit-tests: fast jobs for fast feedback (linting, etc...) 3. deploy-part1: small number of jobs to test if the PR works with default settings -4. deploy-part2: slow jobs testing different platforms, OS, settings, CNI, etc... -5. deploy-part3: very slow jobs (upgrades, etc...) +4. deploy-extended: slow jobs testing different platforms, OS, settings, CNI, etc... +5. deploy-extended: very slow jobs (upgrades, etc...) ## Runners @@ -90,7 +90,7 @@ containerd_registries_mirrors: capabilities: ["pull", "resolve"] skip_verify: false -containerd_max_container_log_line_size: -1 +containerd_max_container_log_line_size: 16384 crio_registries_mirrors: - prefix: docker.io diff --git a/docs/developers/ci.md b/docs/developers/ci.md new file mode 100644 index 00000000000..13ee3ac6994 --- /dev/null +++ b/docs/developers/ci.md @@ -0,0 +1,57 @@ +# CI test coverage + +To generate this Matrix run `./tests/scripts/md-table/main.py` + +## containerd + +| OS / CNI | calico | cilium | custom_cni | flannel | kube-ovn | kube-router | macvlan | +|---| --- | --- | --- | --- | --- | --- | --- | +almalinux8 | :white_check_mark: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | +amazon | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | +debian11 | :white_check_mark: | :x: | :white_check_mark: | :x: | :x: | :x: | :white_check_mark: | +debian12 | :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: | +fedora39 | :white_check_mark: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | +fedora40 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | +openeuler24 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | +opensuse | :x: | :x: | :x: | :x: | :x: | :x: | :x: | +rockylinux8 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | +rockylinux9 | :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | +ubuntu20 | :white_check_mark: | :white_check_mark: | :x: | :white_check_mark: | :x: | :white_check_mark: | :x: | +ubuntu22 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | +ubuntu24 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | + +## crio + +| OS / CNI | calico | cilium | custom_cni | flannel | kube-ovn | kube-router | macvlan | +|---| --- | --- | --- | --- | --- | --- | --- | +almalinux8 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | +amazon | :x: | :x: | :x: | :x: | :x: | :x: | :x: | +debian11 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | +debian12 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | +fedora39 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | +fedora40 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | +openeuler24 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | +opensuse | :x: | :x: | :x: | :x: | :x: | :x: | :x: | +rockylinux8 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | +rockylinux9 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | +ubuntu20 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | +ubuntu22 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | +ubuntu24 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | + +## docker + +| OS / CNI | calico | cilium | custom_cni | flannel | kube-ovn | kube-router | macvlan | +|---| --- | --- | --- | --- | --- | --- | --- | +almalinux8 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | +amazon | :x: | :x: | :x: | :x: | :x: | :x: | :x: | +debian11 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | +debian12 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | +fedora39 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | +fedora40 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | +openeuler24 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | +opensuse | :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | +rockylinux8 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | +rockylinux9 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | +ubuntu20 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | +ubuntu22 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | +ubuntu24 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | diff --git a/docs/test_cases.md b/docs/developers/test_cases.md similarity index 71% rename from docs/test_cases.md rename to docs/developers/test_cases.md index b26d3aa50f4..96d72ea3ba0 100644 --- a/docs/test_cases.md +++ b/docs/developers/test_cases.md @@ -1,6 +1,6 @@ # Node Layouts -There are six node layout types: `default`, `separate`, `ha`, `scale`, `all-in-one`, and `node-etcd-client`. +There are five node layout types: `default`, `separate`, `ha`, `all-in-one`, and `node-etcd-client`. `default` is a non-HA two nodes setup with one separate `kube_node` and the `etcd` group merged with the `kube_control_plane`. @@ -11,11 +11,6 @@ and the `etcd` group merged with the `kube_control_plane`. `ha` layout consists of two etcd nodes, two control planes and a single worker node, with role intersection. -`scale` layout can be combined with above layouts (`ha-scale`, `separate-scale`). It includes 200 fake hosts -in the Ansible inventory. This helps test TLS certificate generation at scale -to prevent regressions and profile certain long-running tasks. These nodes are -never actually deployed, but certificates are generated for them. - `all-in-one` layout use a single node for with `kube_control_plane`, `etcd` and `kube_node` merged. `node-etcd-client` layout consists of a 4 nodes cluster, all of them in `kube_node`, first 3 in `etcd` and only one `kube_control_plane`. @@ -25,7 +20,7 @@ Note, the canal network plugin deploys flannel as well plus calico policy contro ## Test cases -The [CI Matrix](/docs/ci.md) displays OS, Network Plugin and Container Manager tested. +The [CI Matrix](/docs/developers/ci.md) displays OS, Network Plugin and Container Manager tested. All tests are breakdown into 3 "stages" ("Stage" means a build step of the build pipeline) as follows: diff --git a/docs/vagrant.md b/docs/developers/vagrant.md similarity index 80% rename from docs/vagrant.md rename to docs/developers/vagrant.md index 9e12351af1c..3fac5851658 100644 --- a/docs/vagrant.md +++ b/docs/developers/vagrant.md @@ -29,7 +29,26 @@ You can override the default settings in the `Vagrantfile` either by directly modifying the `Vagrantfile` or through an override file. In the same directory as the `Vagrantfile`, create a folder called `vagrant` and create `config.rb` file in it. -An example of how to configure this file is given below. + +Example: + +```ruby +# vagrant/config.rb +$instance_name_prefix = "kub" +$vm_cpus = 1 +$num_instances = 3 +$os = "centos8-bento" +$subnet = "10.0.20" +$network_plugin = "flannel" + +$extra_vars = { + dns_domain: my.custom.domain +} +# or +$extra_vars = "path/to/extra/vars/file.yml" +``` + +For all available options look at the Vagrantfile (search for "CONFIG") ## Use alternative OS for Vagrant @@ -52,78 +71,38 @@ speed, the variable 'download_run_once' is set. This will make kubespray download all files and containers just once and then redistributes them to the other nodes and as a bonus, also cache all downloads locally and re-use them on the next provisioning run. For more information on download settings -see [download documentation](/docs/downloads.md). +see [download documentation](/docs/advanced/downloads.md). ## Example use of Vagrant The following is an example of setting up and running kubespray using `vagrant`. -For repeated runs, you could save the script to a file in the root of the -kubespray and run it by executing `source `. +Customize your settings as shown, above, then run the commands: ```ShellSession # use virtualenv to install all python requirements VENVDIR=venv -virtualenv --python=/usr/bin/python3.7 $VENVDIR -source $VENVDIR/bin/activate -pip install -r requirements.txt - -# prepare an inventory to test with -INV=inventory/my_lab -rm -rf ${INV}.bak &> /dev/null -mv ${INV} ${INV}.bak &> /dev/null -cp -a inventory/sample ${INV} -rm -f ${INV}/hosts.ini - -# customize the vagrant environment -mkdir vagrant -cat << EOF > vagrant/config.rb -\$instance_name_prefix = "kub" -\$vm_cpus = 1 -\$num_instances = 3 -\$os = "centos-bento" -\$subnet = "10.0.20" -\$network_plugin = "flannel" -\$inventory = "$INV" -\$shared_folders = { 'temp/docker_rpms' => "/var/cache/yum/x86_64/7/docker-ce/packages" } -\$extra_vars = { - dns_domain: my.custom.domain -} -# or -\$extra_vars = "path/to/extra/vars/file.yml" -EOF +$ virtualenv --python=/usr/bin/python3.7 $VENVDIR +$ source $VENVDIR/bin/activate +$ pip install -r requirements.txt -# make the rpm cache -mkdir -p temp/docker_rpms +$ vagrant up -vagrant up - -# make a copy of the downloaded docker rpm, to speed up the next provisioning run -scp kub-1:/var/cache/yum/x86_64/7/docker-ce/packages/* temp/docker_rpms/ - -# copy kubectl access configuration in place -mkdir $HOME/.kube/ &> /dev/null -ln -s $PWD/$INV/artifacts/admin.conf $HOME/.kube/config +# Access the cluster +$ export INV=.vagrant/provisionners/ansible/inventory +$ export KUBECONFIG=${INV}/artifacts/admin.conf # make the kubectl binary available -sudo ln -s $PWD/$INV/artifacts/kubectl /usr/local/bin/kubectl -#or -export PATH=$PATH:$PWD/$INV/artifacts +$ export PATH=$PATH:$PWD/$INV/artifacts ``` If a vagrant run failed and you've made some changes to fix the issue causing the fail, here is how you would re-run ansible: ```ShellSession -ansible-playbook -vvv -i .vagrant/provisioners/ansible/inventory/vagrant_ansible_inventory cluster.yml +vagrant provision ``` If all went well, you check if it's all working as expected: -```ShellSession -kubectl get nodes -``` - -The output should look like this: - ```ShellSession $ kubectl get nodes NAME STATUS ROLES AGE VERSION @@ -134,12 +113,6 @@ kub-3 Ready 3m7s v1.22.5 Another nice test is the following: -```ShellSession -kubectl get pods --all-namespaces -o wide -``` - -Which should yield something like the following: - ```ShellSession $ kubectl get pods --all-namespaces -o wide NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES diff --git a/docs/kubernetes-apps/cephfs_provisioner.md b/docs/external_storage_provisioners/cephfs_provisioner.md similarity index 100% rename from docs/kubernetes-apps/cephfs_provisioner.md rename to docs/external_storage_provisioners/cephfs_provisioner.md diff --git a/docs/kubernetes-apps/local_volume_provisioner.md b/docs/external_storage_provisioners/local_volume_provisioner.md similarity index 100% rename from docs/kubernetes-apps/local_volume_provisioner.md rename to docs/external_storage_provisioners/local_volume_provisioner.md diff --git a/docs/kubernetes-apps/rbd_provisioner.md b/docs/external_storage_provisioners/rbd_provisioner.md similarity index 100% rename from docs/kubernetes-apps/rbd_provisioner.md rename to docs/external_storage_provisioners/rbd_provisioner.md diff --git a/docs/kubernetes-apps/scheduler_plugins.md b/docs/external_storage_provisioners/scheduler_plugins.md similarity index 100% rename from docs/kubernetes-apps/scheduler_plugins.md rename to docs/external_storage_provisioners/scheduler_plugins.md diff --git a/docs/comparisons.md b/docs/getting_started/comparisons.md similarity index 100% rename from docs/comparisons.md rename to docs/getting_started/comparisons.md diff --git a/docs/getting-started.md b/docs/getting_started/getting-started.md similarity index 78% rename from docs/getting-started.md rename to docs/getting_started/getting-started.md index 32660d1d39c..77fdf244f14 100644 --- a/docs/getting-started.md +++ b/docs/getting_started/getting-started.md @@ -1,43 +1,33 @@ # Getting started -## Building your own inventory +## Install ansible -Ansible inventory can be stored in 3 formats: YAML, JSON, or INI-like. There is -an example inventory located -[here](https://github.com/kubernetes-sigs/kubespray/blob/master/inventory/sample/inventory.ini). +Install Ansible according to [Ansible installation guide](/docs/ansible/ansible.md#installing-ansible). -You can use an -[inventory generator](https://github.com/kubernetes-sigs/kubespray/blob/master/contrib/inventory_builder/inventory.py) -to create or modify an Ansible inventory. Currently, it is limited in -functionality and is only used for configuring a basic Kubespray cluster inventory, but it does -support creating inventory file for large clusters as well. It now supports -separated ETCD and Kubernetes control plane roles from node role if the size exceeds a -certain threshold. Run `python3 contrib/inventory_builder/inventory.py help` for more information. +## Building your own inventory -Example inventory generator usage: +Ansible inventory can be stored in 3 formats: YAML, JSON, or INI-like. See the +[example inventory](/inventory/sample/inventory.ini) +and [Ansible documentation on building your inventory](https://docs.ansible.com/ansible/latest/inventory_guide/intro_inventory.html), +and [details on the inventory structure expected by Kubespray](/docs/ansible/inventory.md). ```ShellSession -cp -r inventory/sample inventory/mycluster -declare -a IPS=(10.10.1.3 10.10.1.4 10.10.1.5) -CONFIG_FILE=inventory/mycluster/hosts.yml python3 contrib/inventory_builder/inventory.py ${IPS[@]} -``` - -Then use `inventory/mycluster/hosts.yml` as inventory file. + inventory/mycluster/inventory.ini -## Starting custom deployment - -Once you have an inventory, you may want to customize deployment data vars -and start the deployment: +# Review and change parameters under ``inventory/mycluster/group_vars`` + inventory/mycluster/group_vars/all.yml # for every node, including etcd + inventory/mycluster/group_vars/k8s_cluster.yml # for every node in the cluster (not etcd when it's separate) + inventory/mycluster/group_vars/kube_control_plane.yml # for the control plane + inventory/myclsuter/group_vars/kube_node.yml # for worker nodes +``` -**IMPORTANT**: Edit my\_inventory/groups\_vars/\*.yaml to override data vars: +## Installing the cluster ```ShellSession -ansible-playbook -i inventory/mycluster/hosts.yml cluster.yml -b -v \ +ansible-playbook -i inventory/mycluster/ cluster.yml -b -v \ --private-key=~/.ssh/private_key ``` -See more details in the [ansible guide](/docs/ansible.md). - ### Adding nodes You may want to add worker, control plane or etcd nodes to your existing cluster. This can be done by re-running the `cluster.yml` playbook, or you can target the bare minimum needed to get kubelet installed on the worker and talking to your control planes. This is especially helpful when doing something like autoscaling your clusters. @@ -81,7 +71,7 @@ kube-apiserver via port 8080. A kubeconfig file is not necessary in this case, because kubectl will use to connect. The kubeconfig files generated will point to localhost (on kube_control_planes) and kube_node hosts will connect either to a localhost nginx proxy or to a loadbalancer if configured. -More details on this process are in the [HA guide](/docs/ha-mode.md). +More details on this process are in the [HA guide](/docs/operations/ha-mode.md). Kubespray permits connecting to the cluster remotely on any IP of any kube_control_plane host on port 6443 by default. However, this requires @@ -140,5 +130,5 @@ If desired, copy admin.conf to ~/.kube/config. ## Setting up your first cluster -[Setting up your first cluster](/docs/setting-up-your-first-cluster.md) is an +[Setting up your first cluster](/docs/getting_started/setting-up-your-first-cluster.md) is an applied step-by-step guide for setting up your first cluster with Kubespray. diff --git a/docs/setting-up-your-first-cluster.md b/docs/getting_started/setting-up-your-first-cluster.md similarity index 96% rename from docs/setting-up-your-first-cluster.md rename to docs/getting_started/setting-up-your-first-cluster.md index a8200a3ca7a..94eefcb523f 100644 --- a/docs/setting-up-your-first-cluster.md +++ b/docs/getting_started/setting-up-your-first-cluster.md @@ -212,17 +212,15 @@ Copy ``inventory/sample`` as ``inventory/mycluster``: cp -rfp inventory/sample inventory/mycluster ``` -Update Ansible inventory file with inventory builder: +Update the sample Ansible inventory file with ip given by gcloud: ```ShellSession -declare -a IPS=($(gcloud compute instances list --filter="tags.items=kubernetes-the-kubespray-way" --format="value(EXTERNAL_IP)" | tr '\n' ' ')) -CONFIG_FILE=inventory/mycluster/hosts.yaml python3 contrib/inventory_builder/inventory.py ${IPS[@]} +gcloud compute instances list --filter="tags.items=kubernetes-the-kubespray-way" ``` -Open the generated `inventory/mycluster/hosts.yaml` file and adjust it so -that controller-0, controller-1 and controller-2 are control plane nodes and -worker-0, worker-1 and worker-2 are worker nodes. Also update the `ip` to the respective local VPC IP and -remove the `access_ip`. +Open `inventory/mycluster/inventory.ini` file and add it so +that controller-0, controller-1 and controller-2 in the `kube_control_plane` group and +worker-0, worker-1 and worker-2 in the `kube_node` group. Add respective `ip` to the respective local VPC IP for each node. The main configuration for the cluster is stored in `inventory/mycluster/group_vars/k8s_cluster/k8s_cluster.yml`. In this file we @@ -242,7 +240,7 @@ the kubernetes cluster, just change the 'false' to 'true' for Now we will deploy the configuration: ```ShellSession -ansible-playbook -i inventory/mycluster/hosts.yaml -u $USERNAME -b -v --private-key=~/.ssh/id_rsa cluster.yml +ansible-playbook -i inventory/mycluster/ -u $USERNAME -b -v --private-key=~/.ssh/id_rsa cluster.yml ``` Ansible will now execute the playbook, this can take up to 20 minutes. @@ -596,7 +594,7 @@ If you want to keep the VMs and just remove the cluster state, you can simply run another Ansible playbook: ```ShellSession -ansible-playbook -i inventory/mycluster/hosts.yaml -u $USERNAME -b -v --private-key=~/.ssh/id_rsa reset.yml +ansible-playbook -i inventory/mycluster/ -u $USERNAME -b -v --private-key=~/.ssh/id_rsa reset.yml ``` Resetting the cluster to the VMs original state usually takes about a couple diff --git a/docs/ingress_controller/alb_ingress_controller.md b/docs/ingress/alb_ingress_controller.md similarity index 100% rename from docs/ingress_controller/alb_ingress_controller.md rename to docs/ingress/alb_ingress_controller.md diff --git a/docs/ingress_controller/ingress_nginx.md b/docs/ingress/ingress_nginx.md similarity index 99% rename from docs/ingress_controller/ingress_nginx.md rename to docs/ingress/ingress_nginx.md index f465dc60f59..3aa184b797d 100644 --- a/docs/ingress_controller/ingress_nginx.md +++ b/docs/ingress/ingress_nginx.md @@ -35,7 +35,7 @@ kubectl create clusterrolebinding cluster-admin-binding \ The following **Mandatory Command** is required for all deployments except for AWS. See below for the AWS version. ```console -kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.10.1/deploy/static/provider/cloud/deploy.yaml +kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.11.2/deploy/static/provider/cloud/deploy.yaml ``` ### Provider Specific Steps diff --git a/docs/kube-vip.md b/docs/ingress/kube-vip.md similarity index 100% rename from docs/kube-vip.md rename to docs/ingress/kube-vip.md diff --git a/docs/metallb.md b/docs/ingress/metallb.md similarity index 100% rename from docs/metallb.md rename to docs/ingress/metallb.md diff --git a/docs/mirror.md b/docs/mirror.md deleted file mode 100644 index 3138d20340a..00000000000 --- a/docs/mirror.md +++ /dev/null @@ -1,66 +0,0 @@ -# Public Download Mirror - -The public mirror is useful to make the public resources download quickly in some areas of the world. (such as China). - -## Configuring Kubespray to use a mirror site - -You can follow the [offline](offline-environment.md) to config the image/file download configuration to the public mirror site. If you want to download quickly in China, the configuration can be like: - -```shell -gcr_image_repo: "gcr.m.daocloud.io" -kube_image_repo: "k8s.m.daocloud.io" -docker_image_repo: "docker.m.daocloud.io" -quay_image_repo: "quay.m.daocloud.io" -github_image_repo: "ghcr.m.daocloud.io" - -files_repo: "https://files.m.daocloud.io" -``` - -Use mirror sites only if you trust the provider. The Kubespray team cannot verify their reliability or security. -You can replace the `m.daocloud.io` with any site you want. - -## Example Usage Full Steps - -You can follow the full steps to use the kubesray with mirror. for example: - -Install Ansible according to Ansible installation guide then run the following steps: - -```shell -# Copy ``inventory/sample`` as ``inventory/mycluster`` -cp -rfp inventory/sample inventory/mycluster - -# Update Ansible inventory file with inventory builder -declare -a IPS=(10.10.1.3 10.10.1.4 10.10.1.5) -CONFIG_FILE=inventory/mycluster/hosts.yaml python3 contrib/inventory_builder/inventory.py ${IPS[@]} - -# Use the download mirror -cp inventory/mycluster/group_vars/all/offline.yml inventory/mycluster/group_vars/all/mirror.yml -sed -i -E '/# .*\{\{ files_repo/s/^# //g' inventory/mycluster/group_vars/all/mirror.yml -tee -a inventory/mycluster/group_vars/all/mirror.yml </group_vars/k8s_cluster.yml +gcr_image_repo: "gcr.m.daocloud.io" +kube_image_repo: "k8s.m.daocloud.io" +docker_image_repo: "docker.m.daocloud.io" +quay_image_repo: "quay.m.daocloud.io" +github_image_repo: "ghcr.m.daocloud.io" + +files_repo: "https://files.m.daocloud.io" +``` + +Use mirror sites only if you trust the provider. The Kubespray team cannot verify their reliability or security. +You can replace the `m.daocloud.io` with any site you want. + +## Community-run mirror sites + +DaoCloud(China) + +* [image-mirror](https://github.com/DaoCloud/public-image-mirror) +* [files-mirror](https://github.com/DaoCloud/public-binary-files-mirror) diff --git a/docs/nodes.md b/docs/operations/nodes.md similarity index 100% rename from docs/nodes.md rename to docs/operations/nodes.md diff --git a/docs/offline-environment.md b/docs/operations/offline-environment.md similarity index 96% rename from docs/offline-environment.md rename to docs/operations/offline-environment.md index bbd412c5359..0c5e1bf9e3f 100644 --- a/docs/offline-environment.md +++ b/docs/operations/offline-environment.md @@ -103,7 +103,9 @@ If you use the settings like the one above, you'll need to define in your invent can store them anywhere as long as it's accessible by kubespray. It's recommended to use `*_version` in the path so that you don't need to modify this setting everytime kubespray upgrades one of these components. * `yum_repo`/`debian_repo`/`ubuntu_repo`: OS package repository depending on your OS, should point to your internal - repository. Adjust the path accordingly. + repository. Adjust the path accordingly. Used only for Docker/Containerd packages (if needed); other packages might + be installed from other repositories. You might disable installing packages from other repositories by skipping + the `system-packages` tag ## Install Kubespray Python Packages diff --git a/docs/port-requirements.md b/docs/operations/port-requirements.md similarity index 100% rename from docs/port-requirements.md rename to docs/operations/port-requirements.md diff --git a/docs/recover-control-plane.md b/docs/operations/recover-control-plane.md similarity index 99% rename from docs/recover-control-plane.md rename to docs/operations/recover-control-plane.md index 7cda08afb2b..d54aa13f58a 100644 --- a/docs/recover-control-plane.md +++ b/docs/operations/recover-control-plane.md @@ -1,4 +1,3 @@ - # Recovering the control plane To recover from broken nodes in the control plane use the "recover\-control\-plane.yml" playbook. @@ -8,7 +7,6 @@ Examples of what broken means in this context: * One or more bare metal node(s) suffer from unrecoverable hardware failure * One or more node(s) fail during patching or upgrading * Etcd database corruption - * Other node related failures leaving your control plane degraded or nonfunctional __Note that you need at least one functional node to be able to recover using this method.__ diff --git a/docs/upgrades.md b/docs/operations/upgrades.md similarity index 99% rename from docs/upgrades.md rename to docs/operations/upgrades.md index 6c915c76585..1be7608976d 100644 --- a/docs/upgrades.md +++ b/docs/operations/upgrades.md @@ -392,7 +392,7 @@ ansible-playbook -b -i inventory/sample/hosts.ini cluster.yml --tags=etcd --limi Upgrade kubelet: ```ShellSession -ansible-playbook -b -i inventory/sample/hosts.ini cluster.yml --tags=node --skip-tags=k8s-gen-certs,k8s-gen-tokens +ansible-playbook -b -i inventory/sample/hosts.ini cluster.yml --tags=node --skip-tags=k8s-gen-certs ``` Upgrade Kubernetes master components: @@ -425,7 +425,7 @@ Please note that **migrating container engines is not officially supported by Ku As of Kubespray 2.18.0, containerd is already the default container engine. If you have the chance, it is advisable and safer to reset and redeploy the entire cluster with a new container engine. -* [Migrating from Docker to Containerd](upgrades/migrate_docker2containerd.md) +* [Migrating from Docker to Containerd](/docs/upgrades/migrate_docker2containerd.md) ## System upgrade diff --git a/docs/roadmap.md b/docs/roadmap/roadmap.md similarity index 100% rename from docs/roadmap.md rename to docs/roadmap/roadmap.md diff --git a/extra_playbooks/migrate_openstack_provider.yml b/extra_playbooks/migrate_openstack_provider.yml index bba3d918c1f..002fa715597 100644 --- a/extra_playbooks/migrate_openstack_provider.yml +++ b/extra_playbooks/migrate_openstack_provider.yml @@ -13,7 +13,7 @@ tasks: - name: Include kubespray-default variables include_vars: ../roles/kubespray-defaults/defaults/main/main.yml - - name: Copy get_cinder_pvs.sh to master + - name: Copy get_cinder_pvs.sh to first control plane node copy: src: get_cinder_pvs.sh dest: /tmp diff --git a/extra_playbooks/upgrade-only-k8s.yml b/extra_playbooks/upgrade-only-k8s.yml index 4207f8d28cb..5556f267483 100644 --- a/extra_playbooks/upgrade-only-k8s.yml +++ b/extra_playbooks/upgrade-only-k8s.yml @@ -12,7 +12,7 @@ - name: Setup ssh config to use the bastion hosts: localhost - gather_facts: False + gather_facts: false roles: - { role: kubespray-defaults} - { role: bastion-ssh-config, tags: ["localhost", "bastion"]} @@ -36,7 +36,7 @@ - { role: kubespray-defaults} - { role: kubernetes/preinstall, tags: preinstall } -- name: Handle upgrades to master components first to maintain backwards compat. +- name: Handle upgrades to control plane components first to maintain backwards compat. hosts: kube_control_plane any_errors_fatal: "{{ any_errors_fatal | default(true) }}" serial: 1 diff --git a/galaxy.yml b/galaxy.yml index da5a89a44fb..029b94afe22 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -2,49 +2,25 @@ namespace: kubernetes_sigs description: Deploy a production ready Kubernetes cluster name: kubespray -version: 2.25.0 +version: 2.27.0 readme: README.md authors: - The Kubespray maintainers (https://kubernetes.slack.com/channels/kubespray) tags: - infrastructure repository: https://github.com/kubernetes-sigs/kubespray +issues: https://github.com/kubernetes-sigs/kubespray/issues +documentation: https://kubespray.io license_file: LICENSE dependencies: ansible.utils: '>=2.5.0' - community.general: '>=3.0.0' -build_ignore: - - .github - - '*.tar.gz' - - extra_playbooks - - inventory - - scripts - - test-infra - - .ansible-lint - - .editorconfig - - .gitignore - - .gitlab-ci - - .gitlab-ci.yml - - .gitmodules - - .markdownlint.yaml - - .nojekyll - - .pre-commit-config.yaml - - .yamllint - - Dockerfile - - FILES.json - - MANIFEST.json - - Makefile - - Vagrantfile - - _config.yml - - ansible.cfg - - requirements*txt - - setup.cfg - - setup.py - - index.html - - reset.yml - - cluster.yml - - scale.yml - - recover-control-plane.yml - - remove-node.yml - - upgrade-cluster.yml - - library + community.crypto: '>=2.22.3' + community.general: '>=7.0.0' + ansible.netcommon: '>=5.3.0' + ansible.posix: '>=1.5.4' + community.docker: '>=3.11.0' + kubernetes.core: '>=2.4.2' +manifest: + directives: + - recursive-exclude tests ** + - recursive-include roles **/files/* diff --git a/index.html b/index.html index 0a3d17d125e..31d21576803 100644 --- a/index.html +++ b/index.html @@ -6,6 +6,7 @@ +