diff --git a/.github/workflows/testing-apply-aws-infra.yaml b/.github/workflows/testing-apply-aws-infra.yaml new file mode 100644 index 00000000..fe1af340 --- /dev/null +++ b/.github/workflows/testing-apply-aws-infra.yaml @@ -0,0 +1,181 @@ +name: TESTING Apply AWS infrastructure + +on: + pull_request_review: + types: [submitted] + workflow_dispatch: + inputs: + pr_number: + description: "Pull request number" + required: true + +defaults: + run: + working-directory: ./scripts + +env: + PR_NUMBER: ${{github.event.pull_request.number }} + PLAN_WORKFLOW_NAME: testing-plan-aws-infra.yaml + # TF_CACHE_DIR: ${{ github.workspace }}/tofu/.terraform + +jobs: + apply-test-aws-k8s-infra: + if: github.event.review.state == 'approved' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + environment: testing + steps: + - uses: actions/checkout@v4 + + - name: Set PR_NUMBER env variable + if: github.event_name == 'workflow_dispatch' + run: | + # If event is workflow_dispatch, use the event.workflow_dispatch.inputs.pr_number + echo "PR_NUMBER=${{ github.event.inputs.pr_number }}" >> $GITHUB_ENV + + - name: Init AWS credentials + uses: aws-actions/configure-aws-credentials@v4.0.1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY }} + aws-region: ${{ vars.TF_VAR_REGION }} + + - name: Set up Tofu + uses: opentofu/setup-opentofu@v1.0.1 + with: + tofu_wrapper: false + + # - name: Create Terraform Plugin Cache Dir + # run: mkdir --parents ${{ env.TF_CACHE_DIR }} + + # - name: Cache OpenTofu plugins + # uses: actions/cache@v2 + # id: cache_opentofu_plugins + # with: + # path: ${{ env.TF_CACHE_DIR }} + # key: tofu-${{hashFiles('./tofu/.terraform.lock.hcl')}} + + - name: Get artifacts + uses: dawidd6/action-download-artifact@v3 + with: + workflow: ${{ env.PLAN_WORKFLOW_NAME }} + pr: ${{ env.PR_NUMBER }} + workflow_conclusion: success + name: artifacts-${{ env.PR_NUMBER }} + path: artifacts + + - name: Copy artifacts + working-directory: ./artifacts + run: | + cp plan-aws.out ../tofu/aws/plan-aws + cp terraform.tfvars ../tofu/terraform.tfvars + + - name: Apply AWS infrastructure + id: apply_infrastructure + timeout-minutes: 20 + continue-on-error: true + working-directory: ./tofu/aws + run: | + tofu init + tofu apply -auto-approve -state-out ../../state/state-aws plan-aws + + - name: Set AWS infra output variables + continue-on-error: true + working-directory: ./tofu/aws + run: | + echo "TF_VAR_falkordb_eks_cluster_name=$(tofu output -state=../../state/state-aws -raw falkordb_eks_cluster_name)" >> $GITHUB_ENV + echo "TF_VAR_falkordb_s3_backup_name=$(tofu output -state=../../state/state-aws -raw falkordb_s3_backup_name)" >> $GITHUB_ENV + echo "TF_VAR_falkordb_eks_cluster_oidc_issuer_url=$(tofu output -state=../../state/state-aws -raw falkordb_eks_cluster_oidc_issuer_url)" >> $GITHUB_ENV + echo "TF_VAR_falkordb_eks_cluster_oidc_issuer_arn=$(tofu output -state=../../state/state-aws -raw falkordb_eks_cluster_oidc_issuer_arn)" >> $GITHUB_ENV + echo "TF_VAR_falkordb_eks_cluster_role_arn=$(tofu output -state=../../state/state-aws -raw falkordb_eks_cluster_role_arn)" >> $GITHUB_ENV + echo "TF_VAR_falkordb_eks_cluster_endpoint=$(tofu output -state=../../state/state-aws -raw falkordb_eks_cluster_endpoint)" >> $GITHUB_ENV + echo "TF_VAR_falkordb_eks_cluster_certificate_autority=$(tofu output -state=../../state/state-aws -raw falkordb_eks_cluster_certificate_autority)" >> $GITHUB_ENV + + - name: Archive AWS target state + uses: actions/upload-artifact@v4 + with: + name: state-aws-${{ env.PR_NUMBER }} + path: state/state-aws + if-no-files-found: error + + - name: Get EKS cluster credentials + id: get_eks_credentials + if: steps.apply_infrastructure.outcome == 'success' + continue-on-error: true + run: | + ./aws_update_kubeconfig.sh testing-cluster-${{ env.PR_NUMBER }} ${{ secrets.TF_VAR_ASSUME_ROLE_ARN }} + + - name: Check connection to EKS cluster + if: steps.get_eks_credentials.outcome == 'success' + continue-on-error: true + run: | + kubectl get nodes + + - name: Apply k8s module + if: steps.get_eks_credentials.outcome == 'success' + id: apply_k8s_module + timeout-minutes: 20 + continue-on-error: true + working-directory: ./tofu/k8s + run: | + tofu init + tofu plan -out=local-k8s -var-file=../terraform.tfvars -var "assume_role_arn=${{ secrets.TF_VAR_ASSUME_ROLE_ARN }}" -var "falkordb_password=${{ secrets.TF_VAR_FALKORDB_PASSWORD }}" + tofu apply -auto-approve -state-out ../../state/state-k8s local-k8s + + - name: Archive K8S target state + uses: actions/upload-artifact@v4 + with: + name: state-k8s-${{ env.PR_NUMBER }} + path: state/state-k8s + if-no-files-found: error + + - name: Get FalkorDB endpoint + if: steps.apply_k8s_module.outcome == 'success' + continue-on-error: true + id: lb + run: | + FALKORDB_HOST=$(kubectl get svc -n falkordb falkordb-redis -o jsonpath='{.status.loadBalancer.ingress[0].hostname}') + # If FALKORDB_HOST is empty, wait 30 seconds and try again + if [ -z "$FALKORDB_HOST" ]; then + sleep 30 + FALKORDB_HOST=$(kubectl get svc -n falkordb falkordb-redis -o jsonpath='{.status.loadBalancer.ingress[0].hostname}') + fi + + # If FALKORDB_HOST is still empty, exit with error + if [ -z "$FALKORDB_HOST" ]; then + echo "FALKORDB_HOST is empty" + echo "$(kubectl get svc -n falkordb falkordb-redis -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')" + exit 1 + fi + + echo "FALKORDB_HOST=$FALKORDB_HOST" >> $GITHUB_ENV + echo "FALKORDB_PORT=6379" >> $GITHUB_ENV + echo "FALKORDB_PASSWORD=${{ secrets.TF_VAR_FALKORDB_PASSWORD }}" >> $GITHUB_ENV + + - name: Run python tests + id: python_tests + if: steps.lb.outcome == 'success' + working-directory: . + continue-on-error: true + run: | + python3 -m venv .venv + source .venv/bin/activate + pip install -r requirements.txt + pytest + + - name: Destroy K8S infrastructure + if: always() + working-directory: ./tofu/k8s + run: tofu destroy -auto-approve -state ../../state/state-k8s -var-file=../terraform.tfvars -var "assume_role_arn=${{ secrets.TF_VAR_ASSUME_ROLE_ARN }}" -var "falkordb_password=${{ secrets.TF_VAR_FALKORDB_PASSWORD }}" + + - name: Destroy AWS infrastructure + if: always() + working-directory: ./tofu/aws + run: tofu destroy -auto-approve -state ../../state/state-aws -var-file=../terraform.tfvars -var "assume_role_arn=${{ secrets.TF_VAR_ASSUME_ROLE_ARN }}" -var "eks_auth_role=${{ secrets.TF_VAR_EKS_AUTH_ROLE }}" + + - name: Fail if tests did not pass + if: always() + run: | + # Check if python tests conclusion is success + if [ ${{ steps.python_tests.outcome }} != 'success' ]; then + exit 1 + fi diff --git a/.github/workflows/testing-plan-aws-infra.yaml b/.github/workflows/testing-plan-aws-infra.yaml new file mode 100644 index 00000000..4165273e --- /dev/null +++ b/.github/workflows/testing-plan-aws-infra.yaml @@ -0,0 +1,108 @@ +name: TESTING Plan AWS infrastructure + +on: + pull_request: + branches: [main] + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +defaults: + run: + working-directory: ./scripts + +env: + PR_NUMBER: ${{ github.event.pull_request.number }} + # TF_CACHE_DIR: ${{ github.workspace }}/tofu/.terraform + +jobs: + plan-infra: + runs-on: ubuntu-latest + environment: testing + steps: + - uses: actions/checkout@v4 + + - name: Init AWS credentials + uses: aws-actions/configure-aws-credentials@v4.0.1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY }} + aws-region: ${{ vars.TF_VAR_REGION }} + + - name: Set up testing environment variables + env: + name: testing-cluster-${{ env.PR_NUMBER }} + tenant_name: testing-tenant-${{ env.PR_NUMBER }} + region: ${{ vars.TF_VAR_REGION }} + k8s_version: ${{ vars.TF_VAR_K8S_VERSION }} + k8s_instance_type: ${{ vars.TF_VAR_K8S_INSTANCE_TYPE }} + k8s_node_count: ${{ vars.TF_VAR_K8S_NODE_COUNT }} + k8s_node_min_count: ${{ vars.TF_VAR_K8S_NODE_MIN_COUNT }} + k8s_node_max_count: ${{ vars.TF_VAR_K8S_NODE_MAX_COUNT }} + backup_retention_period: ${{ vars.TF_VAR_BACKUP_RETENTION_PERIOD }} + falkordb_version: v4.0.3 + falkordb_cpu: ${{ vars.TF_VAR_FALKORDB_CPU }} + falkordb_memory: ${{ vars.TF_VAR_FALKORDB_MEMORY }} + persistance_size: ${{ vars.TF_VAR_PERSISTANCE_SIZE }} + falkordb_replicas: ${{ vars.TF_VAR_FALKORDB_REPLICAS }} + grafana_admin_password: ${{ vars.TF_VAR_GRAFANA_ADMIN_PASSWORD }} + backup_schedule: ${{ vars.TF_VAR_BACKUP_SCHEDULE }} + # falkordb_domain: ${{ vars.TF_VAR_FALKORDB_DOMAIN }} + + # If we add these here, they will be exported in the tfvars file + # assume_role_arn: ${{ secrets.TF_VAR_ASSUME_ROLE_ARN }} + # eks_auth_role: ${{ secrets.TF_VAR_EKS_AUTH_ROLE }} + # falkordb_hosted_zone_id: ${{ secrets.TF_VAR_FALKORDB_HOSTED_ZONE_ID }} + # falkordb_password: ${{ secrets.TF_VAR_FALKORDB_PASSWORD }} + run: | + ./create_tfvars_from_env.sh + mkdir -p ${GITHUB_WORKSPACE}/artifacts + cp ../tofu/terraform.tfvars ${GITHUB_WORKSPACE}/artifacts/terraform.tfvars + + - name: Set up Tofu + uses: opentofu/setup-opentofu@v1.0.1 + + # - name: Create Tofu Plugin Cache Dir + # run: | + # echo 'plugin_cache_dir="$HOME/.terraform.d/plugin-cache"' >~/.terraformrc + # mkdir --parents ~/.terraform.d/plugin-cache + + # - name: Cache OpenTofu modules + # uses: actions/cache@v2 + # id: cache_opentofu_plugins + # with: + # path: ${{ env.TF_CACHE_DIR }} + # key: tofu-${{hashFiles('./tofu/.terraform.lock.hcl')}} + + # - name: Init infrastructure + # # if: steps.cache_opentofu_plugins.outputs.cache-hit != 'true' + # run: ./tofu_init.sh + + # - name: Run tests + # run: ./tofu_test.sh + + - name: Plan AWS module + id: plan_aws + working-directory: ./tofu/aws + run: | + tofu init + tofu test + tofu plan -out ../../local-aws -var "assume_role_arn=${{ secrets.TF_VAR_ASSUME_ROLE_ARN }}" -var "eks_auth_role=${{ secrets.TF_VAR_EKS_AUTH_ROLE }}" + cp ../../local-aws ${GITHUB_WORKSPACE}/artifacts/plan-aws.out + + - name: Plan K8S module + id: plan_k8s + working-directory: ./tofu/k8s + run: | + tofu init + tofu test + tofu plan -out ../../local-k8s -var "falkordb_s3_backup_name='test-backup-bucket-s3'" -var "assume_role_arn=${{ secrets.TF_VAR_ASSUME_ROLE_ARN }}" -var "falkordb_password=${{ secrets.TF_VAR_FALKORDB_PASSWORD }}" -var "falkordb_eks_cluster_oidc_issuer_url=''" -var "falkordb_eks_cluster_oidc_issuer_arn=''" -var "falkordb_eks_cluster_endpoint=''" -var "falkordb_eks_cluster_certificate_autority=dGVzdA==" + cp ../../local-k8s ${GITHUB_WORKSPACE}/artifacts/plan-k8s.out + + - name: Archive artifacts + uses: actions/upload-artifact@v4 + with: + name: artifacts-${{ env.PR_NUMBER }} + path: artifacts diff --git a/.gitignore b/.gitignore index f8f6ce01..dce0aa42 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,6 @@ terraform.tfvars terraform.tfstate terraform.tfstate.backup -local \ No newline at end of file +local +.venv +tofu/__tests__/__pycache__ \ No newline at end of file diff --git a/README.md b/README.md index 36e150a4..757d5a12 100644 --- a/README.md +++ b/README.md @@ -20,14 +20,20 @@ Scripts to help work with this repository 5. tofu_delete_workspace.sh: Delete workspace. 6. tofu_list_workspace.sh: List available workspaces. 7. tofu_plan.sh: Generate execution plan to be deployed. -8. tofu_apply.sh: Deploy the plan to the cloud provider. -9. tofu_destroy.sh: Delete the deployment from the cloud provider. -10. tofu_output.sh: Show deployment output. -11. aws_update_kubeconfig.sh: Update kubectl config. -12. kubectl_connect_falkordb_master.sh: Port forward into the FalkorDB master node. -13. kubectl_connect_grafana.sh: Port forward into the grafana gui. -14. kubectl_connect_prometheus.sh: Port forward into the prometheus gui. -15. kubectl_connect_alertmanager.sh: Port forward into the alert manager gui. +8. tofu_plan_aws.sh: Generate execution plan to be deployed for the AWS target. +9. tofu_plan_k8s.sh: Generate execution plan to be deployed for the K8S target. +10. tofu_apply.sh: Deploy the plan to the cloud provider. +11. tofu_apply_aws.sh: Deploy the AWS target to the cloud provider. +12. tofu_apply_k8s.sh: Deploy the K8S target to the cloud provider. +13. tofu_destroy.sh: Delete the deployment from the cloud provider. +14. tofu_output.sh: Show deployment output. +15. tofu_show.sh: Show the state configuration. +16. tofu_test.sh: Run Tofu tests. +17. aws_update_kubeconfig.sh: Update kubectl config. +18. kubectl_connect_falkordb_master.sh: Port forward into the FalkorDB master node. +19. kubectl_connect_grafana.sh: Port forward into the grafana gui. +20. kubectl_connect_prometheus.sh: Port forward into the prometheus gui. +21. kubectl_connect_alertmanager.sh: Port forward into the alert manager gui. # Tofu diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..dc0f5a9d --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +FalkorDB==1.0.1 +pytest==6.2.4 \ No newline at end of file diff --git a/scripts/aws_update_kubeconfig.sh b/scripts/aws_update_kubeconfig.sh index aad6e3f0..c4d47a0c 100755 --- a/scripts/aws_update_kubeconfig.sh +++ b/scripts/aws_update_kubeconfig.sh @@ -1 +1 @@ -aws eks update-kubeconfig --name $1 \ No newline at end of file +aws eks update-kubeconfig --name $1 --role-arn $2 \ No newline at end of file diff --git a/scripts/create_tfvars_from_env.sh b/scripts/create_tfvars_from_env.sh new file mode 100755 index 00000000..8311284b --- /dev/null +++ b/scripts/create_tfvars_from_env.sh @@ -0,0 +1,20 @@ + echo "name=\"$name\" + tenant_name=\"$name\" + region=\"$region\" + k8s_version=\"$k8s_version\" + k8s_instance_type=\"$k8s_instance_type\" + k8s_node_count=\"$k8s_node_count\" + k8s_node_min_count=\"$k8s_node_min_count\" + k8s_node_max_count=\"$k8s_node_max_count\" + backup_retention_period=\"$backup_retention_period\" + falkordb_version=\"$falkordb_version\" + falkordb_cpu=\"$falkordb_cpu\" + falkordb_memory=\"$falkordb_memory\" + persistance_size=\"$persistance_size\" + falkordb_replicas=\"$falkordb_replicas\" + grafana_admin_password=\"$grafana_admin_password\" + backup_schedule=\"$backup_schedule\" + # falkordb_domain=\"$falkordb_domain\" + " > ../tofu/terraform.tfvars +cp ../tofu/terraform.tfvars ../tofu/aws/terraform.tfvars +cp ../tofu/terraform.tfvars ../tofu/k8s/terraform.tfvars \ No newline at end of file diff --git a/scripts/tofu_apply_aws.sh b/scripts/tofu_apply_aws.sh new file mode 100755 index 00000000..15951670 --- /dev/null +++ b/scripts/tofu_apply_aws.sh @@ -0,0 +1,5 @@ +PLAN_PATH=local-aws +if [ -n "$1" ]; then + PLAN_PATH=$1 +fi +cd ../tofu && tofu apply -state-out=../state/state-aws -auto-approve $PLAN_PATH \ No newline at end of file diff --git a/scripts/tofu_apply_k8s.sh b/scripts/tofu_apply_k8s.sh new file mode 100755 index 00000000..950298c8 --- /dev/null +++ b/scripts/tofu_apply_k8s.sh @@ -0,0 +1,5 @@ +PLAN_PATH=local-k8s +if [ -n "$1" ]; then + PLAN_PATH=$1 +fi +cd ../tofu && tofu apply -state-out=../state/state-k8s $PLAN_PATH \ No newline at end of file diff --git a/scripts/tofu_plan_aws.sh b/scripts/tofu_plan_aws.sh new file mode 100755 index 00000000..ed7eff76 --- /dev/null +++ b/scripts/tofu_plan_aws.sh @@ -0,0 +1 @@ +cd ../tofu && tofu plan -target=module.aws -out=local-aws \ No newline at end of file diff --git a/scripts/tofu_plan_k8s.sh b/scripts/tofu_plan_k8s.sh new file mode 100755 index 00000000..81dd5770 --- /dev/null +++ b/scripts/tofu_plan_k8s.sh @@ -0,0 +1 @@ +cd ../tofu && tofu plan -target=module.k8s -out=local-k8s \ No newline at end of file diff --git a/scripts/tofu_show.sh b/scripts/tofu_show.sh new file mode 100755 index 00000000..91690c6a --- /dev/null +++ b/scripts/tofu_show.sh @@ -0,0 +1,5 @@ +STATE_PATH=local +if [ -n "$1" ]; then + STATE_PATH=$1 +fi +cd ../tofu && tofu show -no-color $STATE_PATH \ No newline at end of file diff --git a/scripts/tofu_test.sh b/scripts/tofu_test.sh new file mode 100755 index 00000000..b81e2047 --- /dev/null +++ b/scripts/tofu_test.sh @@ -0,0 +1 @@ +cd ../tofu && tofu test \ No newline at end of file diff --git a/tofu/.terraform.lock.hcl b/tofu/.terraform.lock.hcl deleted file mode 100644 index d914b676..00000000 --- a/tofu/.terraform.lock.hcl +++ /dev/null @@ -1,176 +0,0 @@ -# This file is maintained automatically by "tofu init". -# Manual edits may be lost in future updates. - -provider "registry.opentofu.org/cloudposse/utils" { - version = "1.14.0" - constraints = ">= 0.17.0" - hashes = [ - "h1:QdhjirI2+PduIqMpbUimUy1bO66uy8e0lljtppXoINk=", - "h1:x0t9pfIH0MmCqC50RvsNzR0LiJMamtxGU6Zydw1YBwo=", - "zh:2e058d206b66224bd5f8f78a03d52f19064eeed179115847fbb8f0e3da34d04c", - "zh:3035a0db2ef0950ff6a41764b6b250cdf74084b549c8443da883fdb723367e29", - "zh:3750baccaa7a174e7eb97aca6d2442fbe09a508d99edac81e4948083b4df01d7", - "zh:4ab57c32778160d15169bcd343537bbca48f3a14ccca410c31a415b591ec628a", - "zh:6ad18d78fe54ad3d0a5a4f43488fe9fecf81031106a3d955a102a4791d760751", - "zh:6bf256f03fbdfb0aff9751bc2bc4f9a81d8d967ae935cec1855d998fe6e80f03", - "zh:8aad026d7c561bf0ada4e0ec8f861fc8339957c3f973661424790c3869abc003", - "zh:98becc6c0e4766512f8157bee6fa66c475afd2155a0f9b3d90917c3510ac9010", - "zh:a5b85c73b0701c9b4f0c0dc4cc2ca50f07d93de85ecac9737abe5d2f1aae0ec3", - "zh:aa817735af915dae7dd2020d3b6eb8fabb2e6618e6c2736c0950d83c2f64a519", - "zh:cce2c5fe013a5996a648cdea91a7d934b6faf5d99d6affc2a5a83e058448758e", - "zh:e174e3dbe46a2848720d26c0672964c95c9219cb8dc6f8eeee4d30757f2ced1c", - "zh:e6c84d2ccac3063aab46706ceeb8f1439d03d4ed824daac9ab0253f0902a4e14", - "zh:eada9372e27880b62558e396d2ee1e96f876c2da915fb0c665317f004d619115", - ] -} - -provider "registry.opentofu.org/gavinbunney/kubectl" { - version = "1.14.0" - constraints = ">= 1.9.4" - hashes = [ - "h1:ItrWfCZMzM2JmvDncihBMalNLutsAk7kyyxVRaipftY=", - "h1:gLFn+RvP37sVzp9qnFCwngRjjFV649r6apjxvJ1E/SE=", - "zh:0350f3122ff711984bbc36f6093c1fe19043173fad5a904bce27f86afe3cc858", - "zh:07ca36c7aa7533e8325b38232c77c04d6ef1081cb0bac9d56e8ccd51f12f2030", - "zh:0c351afd91d9e994a71fe64bbd1662d0024006b3493bb61d46c23ea3e42a7cf5", - "zh:39f1a0aa1d589a7e815b62b5aa11041040903b061672c4cfc7de38622866cbc4", - "zh:428d3a321043b78e23c91a8d641f2d08d6b97f74c195c654f04d2c455e017de5", - "zh:4baf5b1de2dfe9968cc0f57fd4be5a741deb5b34ee0989519267697af5f3eee5", - "zh:6131a927f9dffa014ab5ca5364ac965fe9b19830d2bbf916a5b2865b956fdfcf", - "zh:c62e0c9fd052cbf68c5c2612af4f6408c61c7e37b615dc347918d2442dd05e93", - "zh:f0beffd7ce78f49ead612e4b1aefb7cb6a461d040428f514f4f9cc4e5698ac65", - ] -} - -provider "registry.opentofu.org/hashicorp/aws" { - version = "5.32.1" - constraints = ">= 3.35.0, >= 3.72.0, >= 4.0.0, >= 4.9.0, >= 4.19.0, >= 4.33.0, >= 4.36.0, >= 4.47.0, >= 4.57.0, >= 5.0.0, >= 5.20.0" - hashes = [ - "h1:Bvwj0aWe6M5WoA25UsNX2mCetQ63jS4ztey85L1ZDr4=", - "h1:QC1gLT7uj0FBJWebysvd23YqNTJy+X5FX9rPpYskt8k=", - "zh:071e9a25b06245397ac2cba16c6e63e50d97a0e20f4739fef277aa86502a2b7b", - "zh:17862eff5baa48a42ad1b7bcea473bf7b6506d972f0fb60db95dcea643146231", - "zh:3e74b8e9d7a1e56a30a78f289445486e6e2219bdd62eaceb1327d1391b0efb77", - "zh:625041e1810ecffade955bab6e0f79245267bcf8718676790df598bf0a98a3b4", - "zh:89b2fd164545d78fef5f6d78b1e3ec112799ce8324f1670f2be2d282cded4274", - "zh:9035d114ec4ccef482e123498f4808c08ec45c34d71175fba2f163291a0acd17", - "zh:931fa1782ec4c38e5c9886eb863d89eb8ae1033fbebba678ce86d777f15f5db1", - "zh:d636fc72da55fb6f2b514c5870f6c8bdfbfa3684a7fd864a8f4fb415a3a9ed8c", - "zh:e13a1496f93683940ddafda2a10277112f04ecc63de49a45a6c910018a1f5b10", - "zh:fae2aadb4c6ec7fa78a5f0bdfa2080c28a6128e35517339df33b91193ca7299c", - ] -} - -provider "registry.opentofu.org/hashicorp/cloudinit" { - version = "2.3.3" - constraints = ">= 2.0.0" - hashes = [ - "h1:2DqQ1QlY+CiievWJmyLah8DcFv2FnFRtvclfJh/6NNI=", - "h1:f6GM+JAyvGdU2FHFOncYo+dP/Ve/6OiHs29czbTFtzY=", - "zh:132f1782bb198a635892ea4b116fd69ffabcf4b6b11f86c57faf53b19575c23d", - "zh:21e7ab6820990f314de03be87af71cb4bae2409fa18007d11cfa60066a7f924c", - "zh:2a6a71194f3923ba6136c8a17765f505fa3e20624f4cd1078f36bdb92cafbe00", - "zh:2cf69cac676eb20e5f82b1dbb739c30b963fd6010e430e1b0bf3dfedc6554000", - "zh:3c508f6ef48fc8073d2e4ebd1ea1532b52e4d7ac679908d73891e8f4b451a71d", - "zh:5ab08771183c7dd6070ae95be84154540f15c41b34606e55fe87639e0bfddc0c", - "zh:af20410183201bbbe4e13f7fa69f0a57eea5b925e3092036f1aaa2767f1a7516", - "zh:bbb60400a2c17aa31728b348d4f7ba2de8a20b014b0c0658c7ff1f54a4e1f776", - "zh:e8cd5c617707b5e5f78a2dba45e864b7690930f39aa6c84e9455e9f3943cb83c", - "zh:f43b99f6b6d581d2745e4f0cfdeb0425f381c113bebf2cc95c08c8f8c2d6506b", - ] -} - -provider "registry.opentofu.org/hashicorp/helm" { - version = "2.12.1" - constraints = ">= 1.0.0, >= 2.6.0, >= 2.9.0, < 3.0.0" - hashes = [ - "h1:S0+5VN/viVA4YYpm9q45bZ903EqP3bwjv5abps+a3lE=", - "h1:ajWSFsohX3kQNLs8DbQd93UJlKTUy4HnccLZ2xWCfFM=", - "zh:0349149992646530c33314cb973eba68757606a037017ba47e56db695d4b3afe", - "zh:3138ffe23c481b01419a4a21adf83538efe6e698b421c4a8f7d142b198518709", - "zh:44658e3070405b88fbd76161ecddde62f478dc31aaebee3b93c2f2783a6d45f9", - "zh:5600a3407dfb8b77da7561490157afa8ad505c864a5dd35ed8d678e9ad8378ca", - "zh:6445e359c813ecbb7c2edf722ed0d1f33dfb171b6a7b470f40cf1e24045b7441", - "zh:7973054604c7f5a51600f6e63fa0327d05b29fac2bffd222c21660cbdd2939f9", - "zh:7c59e2d4602ab5d9de0ba8e442ec1fc425c8f143581018d1e7f645298a124f01", - "zh:8c0fb411dd5de664ac5e801d70507781790c4fc196518a56966d66d0963c240c", - "zh:a6a988c91bbf1828a8fc55001f10c7d06c5c53dc718ee7cd6814bdfa2e6652e0", - "zh:b7935d7dacd7e5a91ff9d17cfb04ce88c9100e563fd88487d14519e8d8d8b2e1", - ] -} - -provider "registry.opentofu.org/hashicorp/kubernetes" { - version = "2.25.2" - constraints = ">= 1.10.0, >= 2.10.0, >= 2.16.0, >= 2.20.0, < 3.0.0" - hashes = [ - "h1:9ovdI6lk1kqU2uMp6pyZ8fPn2tEP0C1U4DTUSS1JlH8=", - "h1:ZWY+veN6C6BNcbcncNZ1JRUTawTiusdUGtJ2X931xlQ=", - "zh:38d35c069a7f5a7c360ce2ddf6da7f22fd25290f76697d5ab0adbc9e7cae4db8", - "zh:4b1967b873c6262d8c8f57e8cea2cd029c43912555f688cdff4f824193be4e50", - "zh:57e6cfda60c084bb141ea1d8f0ad5881b1bbda92dfcb090e752b09d2cc153b84", - "zh:5af23ccba04c2fef7cff9cb17ed5c10ac9d89098706244d1be4d2acbf44f2ae2", - "zh:67cd0b02deba8361b2689afe4b2f67e38ca68539a89eb88ceaef7fae6f7e6f0b", - "zh:7b4d1fa9d9612919e29a9424a316af6b4c5de766debd8a823b8ad0f28ac2d9c6", - "zh:b7a9ca5d1a734675de32c9bbdc78e8569a10aef140fe6d176efed393e64e1d71", - "zh:c97777211b3f46ede86fce794ab515185e1145af6a4cd57effdfcddc3cccdcf7", - "zh:e7921548f82f68694ad51974bad7cc6c853134a9851912a04da4cdba34eb6214", - "zh:fa7b37554ed32422a239b707380ae778a8918e4917a76aadd92fe559a353a10c", - ] -} - -provider "registry.opentofu.org/hashicorp/random" { - version = "3.6.0" - constraints = ">= 3.0.0" - hashes = [ - "h1:/xwPFz7kMERBIEk8i6UJt2fTvgzMFbwKlcyCvRJO8Ok=", - "h1:6QMZ6JACl+V2t8daN5GTlw22EtG7nhc3BbkbJDw2a5M=", - "zh:486a1c921eab5c51a480f2eb0ad85173f207c9b7bb215f3893e58bc38d3b7c75", - "zh:6901b3afa4607d1e31934ba91ed2625215ada42b3518c3a9adeeac7a5f656dc3", - "zh:7e93752c9de710e417191353ad1a41b5a60432ab7ef4f8b556bf248297ec5e23", - "zh:c795d3d319e8ee7be972746b935963b7e772a6a14080261a35c03915c1f9ccb2", - "zh:cd4f8bcaf332828d1736c73874549c25e427737f136173c7b61e2df3db50e5d9", - "zh:e0103eb2e280989c3d9ffda5d6b413e8f583be21bc1d5754c6e9ca87ecc1c44a", - "zh:f4fbec2510322d5b7ad584a92436b5dbd0f2e897a3ec538932af59e245a4c8e4", - "zh:f5418842afd4aa7676e2456e425e8f573cb2b9bffd29bd7de09d91845644ab24", - "zh:f572a26f93d00ec42461ce478678366e570fa4497e2273f9d47f24cdfc4b42b4", - "zh:ff1f07c561a3f7f219b6fee1647a559933b5dd6181753e164c3978fd47a11685", - ] -} - -provider "registry.opentofu.org/hashicorp/time" { - version = "0.10.0" - constraints = ">= 0.9.0" - hashes = [ - "h1:LZb3ZijtbUDBYaKoEim7oX2OkP9GqOBfp2fWwgRZhUs=", - "h1:jSQ7QTA2lhnzZHX2nL4+1sbBFhAB8HYrQCuF4YRIgKM=", - "zh:0997e9e82dbf3b01b783d75e4ad14dfa135d7dea9ef2e6ccd48348ed9dd31c27", - "zh:2794dc8a5b79f331ad780b757dd7a04e539551cc8b8c50d25ebeb43994c7fe1d", - "zh:2f5e81ebc5c4d9329c392de67483fb2144d2c0cae4001e6ec2c83e1ab71d62ca", - "zh:5cda8141aeeb594ca7a59e3ada5d9c8d8901a3467c07e263f2c0a4a08170ea53", - "zh:816aaa6eee3a053f29db48bed081e5f3be218336e41d1da385c29592916fb7c7", - "zh:8606f3ce3784927e732516368e69712cf127c5c71334384d6be84da4c555558c", - "zh:8b87a1472271a1dc359c4c1069ba62248c56240009c37598af75e4b888172cd7", - "zh:dc1617df212201ded78038634d5f68b7551b5763361d3a20ab8b5a55640557eb", - "zh:f9b10a794c6d3760cfeb21d3c4db551385a528837dcb523ba46b59f43f1935ac", - "zh:fa51a2259a691ad24ff1426a9b460dfa6d293835544e53915731c858608db024", - ] -} - -provider "registry.opentofu.org/hashicorp/tls" { - version = "4.0.5" - constraints = ">= 3.0.0" - hashes = [ - "h1:LWGTWAUrC+/iTsNq0vxANvGOp+7Jnl4wAnSOW2Shqjc=", - "h1:zEH0OgSkeXDqNWzmOUWDczrUwyyujAHvnbW79qdxVMI=", - "zh:05a7dc3ac92005485714f87541ad6d0d478988b478c5774227a7d39b01660050", - "zh:547e0def44080456169bf77c21037aa6dc9e7f3e644a8f6a2c5fc3e6c15cf560", - "zh:6842b03d050ae1a4f1aaed2a2b1ca707eae84ae45ae492e4bb57c3d48c26e1f1", - "zh:6ced0a9eaaba12377f3a9b08df2fd9b83ae3cb357f859eb6aecf24852f718d9a", - "zh:766bcdf71a7501da73d4805d05764dcb7c848619fa7c04b3b9bd514e5ce9e4aa", - "zh:84cc8617ce0b9a3071472863f43152812e5e8544802653f636c866ef96f1ed34", - "zh:b1939e0d44c89315173b78228c1cf8660a6924604e75ced7b89e45196ce4f45e", - "zh:ced317916e13326766427790b1d8946c4151c4f3b0efd8f720a3bc24abe065fa", - "zh:ec9ff3412cf84ba81ca88328b62c17842b803ef406ae19152c13860b356b259c", - "zh:ff064f0071e98702e542e1ce00c0465b7cd186782fe9ccab8b8830cac0f10dd4", - ] -} diff --git a/tofu/__tests__/test_falkordb_connection.py b/tofu/__tests__/test_falkordb_connection.py new file mode 100644 index 00000000..09923caa --- /dev/null +++ b/tofu/__tests__/test_falkordb_connection.py @@ -0,0 +1,28 @@ +import os +from falkordb import FalkorDB + + +def test_falkordb_connection(): + """Test the connection to FalkorDB""" + + host = os.environ.get("FALKORDB_HOST") + port = os.environ.get("FALKORDB_PORT") + password = os.environ.get("FALKORDB_PASSWORD") + + print(f"Connecting to FalkorDB at {host}:{port}") + + assert host is not None + assert port is not None and port.isdigit() + + try: + db = FalkorDB(host, port, password) + + graph = db.select_graph("test") + + res = graph.query("RETURN 1") + + assert res.result_set[0][0] == 1 + + except Exception as e: + print(e) + assert False diff --git a/tofu/aws/.terraform.lock.hcl b/tofu/aws/.terraform.lock.hcl new file mode 100644 index 00000000..7695fb2a --- /dev/null +++ b/tofu/aws/.terraform.lock.hcl @@ -0,0 +1,110 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "registry.opentofu.org/hashicorp/aws" { + version = "5.34.0" + constraints = ">= 4.0.0, >= 4.9.0, >= 4.33.0, >= 4.36.0, >= 4.47.0, >= 4.57.0, >= 5.0.0, >= 5.20.0" + hashes = [ + "h1:Ofv10Aw1nrPZ4amBcpge3lodQ7acaY7H/DeuUA4EsOU=", + "zh:01ab6b6ae075e2d09d67578af05c325117c40d407f1f3602caa95d31b52218bd", + "zh:18b938e0272e52e18f79eb8e355bb81397d859e7ac98d66fba1c142d142314b6", + "zh:225cad47a8a224bf5fc7ac47bc46746f9143f8ad39a2b2f0b5306bd580a5994a", + "zh:35fcfacd4f8ed71a6f9854eeb862431ca832cb732e97fb02e3ea3f764610db3f", + "zh:4018cd69689857968afe31cf67ef4796b45f08c27012daff56201618621a690b", + "zh:442ec76a21e9a55c3d3e38c5d57087f5c5127326237af10463ec26c2355f3102", + "zh:8417f0a78697223f2a38dd3d1df88d69891a6b2168aabcb4916afd6008cf1609", + "zh:c48b6103112efc02711f046625a60f76912be5f3f590c8bf68d94895c0d45f1c", + "zh:d11f4cec70f382b716241104dabfd1c4cc6b95c836e7a8c469ab64f62a8824a6", + "zh:d52b6d37066685eb3e3a4bd32fcc7bc68aad1e81cbd3240feaa138313c740e81", + ] +} + +provider "registry.opentofu.org/hashicorp/cloudinit" { + version = "2.3.3" + constraints = ">= 2.0.0" + hashes = [ + "h1:M19EHqgAqMHiVYlS3qEQOy+MZWOurTXyUq3kjnMhh+M=", + "zh:132f1782bb198a635892ea4b116fd69ffabcf4b6b11f86c57faf53b19575c23d", + "zh:21e7ab6820990f314de03be87af71cb4bae2409fa18007d11cfa60066a7f924c", + "zh:2a6a71194f3923ba6136c8a17765f505fa3e20624f4cd1078f36bdb92cafbe00", + "zh:2cf69cac676eb20e5f82b1dbb739c30b963fd6010e430e1b0bf3dfedc6554000", + "zh:3c508f6ef48fc8073d2e4ebd1ea1532b52e4d7ac679908d73891e8f4b451a71d", + "zh:5ab08771183c7dd6070ae95be84154540f15c41b34606e55fe87639e0bfddc0c", + "zh:af20410183201bbbe4e13f7fa69f0a57eea5b925e3092036f1aaa2767f1a7516", + "zh:bbb60400a2c17aa31728b348d4f7ba2de8a20b014b0c0658c7ff1f54a4e1f776", + "zh:e8cd5c617707b5e5f78a2dba45e864b7690930f39aa6c84e9455e9f3943cb83c", + "zh:f43b99f6b6d581d2745e4f0cfdeb0425f381c113bebf2cc95c08c8f8c2d6506b", + ] +} + +provider "registry.opentofu.org/hashicorp/helm" { + version = "2.12.1" + constraints = ">= 2.9.0" + hashes = [ + "h1:mRK57Pn5YGikn9jT4GyZtB1zf5gvu9ynNbwWq6YuPyA=", + "zh:0349149992646530c33314cb973eba68757606a037017ba47e56db695d4b3afe", + "zh:3138ffe23c481b01419a4a21adf83538efe6e698b421c4a8f7d142b198518709", + "zh:44658e3070405b88fbd76161ecddde62f478dc31aaebee3b93c2f2783a6d45f9", + "zh:5600a3407dfb8b77da7561490157afa8ad505c864a5dd35ed8d678e9ad8378ca", + "zh:6445e359c813ecbb7c2edf722ed0d1f33dfb171b6a7b470f40cf1e24045b7441", + "zh:7973054604c7f5a51600f6e63fa0327d05b29fac2bffd222c21660cbdd2939f9", + "zh:7c59e2d4602ab5d9de0ba8e442ec1fc425c8f143581018d1e7f645298a124f01", + "zh:8c0fb411dd5de664ac5e801d70507781790c4fc196518a56966d66d0963c240c", + "zh:a6a988c91bbf1828a8fc55001f10c7d06c5c53dc718ee7cd6814bdfa2e6652e0", + "zh:b7935d7dacd7e5a91ff9d17cfb04ce88c9100e563fd88487d14519e8d8d8b2e1", + ] +} + +provider "registry.opentofu.org/hashicorp/kubernetes" { + version = "2.25.2" + constraints = ">= 2.10.0, >= 2.20.0" + hashes = [ + "h1:0rrxDjtJb63VSS9npSlikqJGH2L726byPwnP7fd3B/4=", + "zh:38d35c069a7f5a7c360ce2ddf6da7f22fd25290f76697d5ab0adbc9e7cae4db8", + "zh:4b1967b873c6262d8c8f57e8cea2cd029c43912555f688cdff4f824193be4e50", + "zh:57e6cfda60c084bb141ea1d8f0ad5881b1bbda92dfcb090e752b09d2cc153b84", + "zh:5af23ccba04c2fef7cff9cb17ed5c10ac9d89098706244d1be4d2acbf44f2ae2", + "zh:67cd0b02deba8361b2689afe4b2f67e38ca68539a89eb88ceaef7fae6f7e6f0b", + "zh:7b4d1fa9d9612919e29a9424a316af6b4c5de766debd8a823b8ad0f28ac2d9c6", + "zh:b7a9ca5d1a734675de32c9bbdc78e8569a10aef140fe6d176efed393e64e1d71", + "zh:c97777211b3f46ede86fce794ab515185e1145af6a4cd57effdfcddc3cccdcf7", + "zh:e7921548f82f68694ad51974bad7cc6c853134a9851912a04da4cdba34eb6214", + "zh:fa7b37554ed32422a239b707380ae778a8918e4917a76aadd92fe559a353a10c", + ] +} + +provider "registry.opentofu.org/hashicorp/time" { + version = "0.10.0" + constraints = ">= 0.9.0" + hashes = [ + "h1:58xhugSr4hDVmzUqkAuVQIJjJ0E7cyOmnqTM0TQboEs=", + "zh:0997e9e82dbf3b01b783d75e4ad14dfa135d7dea9ef2e6ccd48348ed9dd31c27", + "zh:2794dc8a5b79f331ad780b757dd7a04e539551cc8b8c50d25ebeb43994c7fe1d", + "zh:2f5e81ebc5c4d9329c392de67483fb2144d2c0cae4001e6ec2c83e1ab71d62ca", + "zh:5cda8141aeeb594ca7a59e3ada5d9c8d8901a3467c07e263f2c0a4a08170ea53", + "zh:816aaa6eee3a053f29db48bed081e5f3be218336e41d1da385c29592916fb7c7", + "zh:8606f3ce3784927e732516368e69712cf127c5c71334384d6be84da4c555558c", + "zh:8b87a1472271a1dc359c4c1069ba62248c56240009c37598af75e4b888172cd7", + "zh:dc1617df212201ded78038634d5f68b7551b5763361d3a20ab8b5a55640557eb", + "zh:f9b10a794c6d3760cfeb21d3c4db551385a528837dcb523ba46b59f43f1935ac", + "zh:fa51a2259a691ad24ff1426a9b460dfa6d293835544e53915731c858608db024", + ] +} + +provider "registry.opentofu.org/hashicorp/tls" { + version = "4.0.5" + constraints = ">= 3.0.0" + hashes = [ + "h1:ILGm1+RP2+eIDc+YQ+xWgNX7Dcb9cD9OuvJHqUxtjmE=", + "zh:05a7dc3ac92005485714f87541ad6d0d478988b478c5774227a7d39b01660050", + "zh:547e0def44080456169bf77c21037aa6dc9e7f3e644a8f6a2c5fc3e6c15cf560", + "zh:6842b03d050ae1a4f1aaed2a2b1ca707eae84ae45ae492e4bb57c3d48c26e1f1", + "zh:6ced0a9eaaba12377f3a9b08df2fd9b83ae3cb357f859eb6aecf24852f718d9a", + "zh:766bcdf71a7501da73d4805d05764dcb7c848619fa7c04b3b9bd514e5ce9e4aa", + "zh:84cc8617ce0b9a3071472863f43152812e5e8544802653f636c866ef96f1ed34", + "zh:b1939e0d44c89315173b78228c1cf8660a6924604e75ced7b89e45196ce4f45e", + "zh:ced317916e13326766427790b1d8946c4151c4f3b0efd8f720a3bc24abe065fa", + "zh:ec9ff3412cf84ba81ca88328b62c17842b803ef406ae19152c13860b356b259c", + "zh:ff064f0071e98702e542e1ce00c0465b7cd186782fe9ccab8b8830cac0f10dd4", + ] +} diff --git a/tofu/aws/main.tf b/tofu/aws/main.tf index e1b42aeb..f9cc6640 100644 --- a/tofu/aws/main.tf +++ b/tofu/aws/main.tf @@ -1,9 +1,15 @@ provider "aws" { region = var.region + assume_role { + role_arn = var.assume_role_arn + } +} +data "aws_caller_identity" "current" { +} + +data "aws_availability_zones" "available" { } -data "aws_caller_identity" "current" {} -data "aws_availability_zones" "available" {} locals { vpc_cidr = "10.0.0.0/16" @@ -12,8 +18,6 @@ locals { tags = { customer = var.name } - - falkordb_s3_backup_location = "${module.falkordb_backup_s3_bucket.s3_bucket_id}/backups" } ################################################################################ @@ -28,6 +32,14 @@ module "eks" { cluster_version = var.k8s_version cluster_endpoint_public_access = true + aws_auth_accounts = [ + data.aws_caller_identity.current.account_id + ] + + aws_auth_roles = [ + var.eks_auth_role + ] + vpc_id = module.vpc.vpc_id subnet_ids = module.vpc.private_subnets @@ -38,13 +50,11 @@ module "eks" { min_size = var.k8s_node_min_count max_size = var.k8s_node_max_count desired_size = var.k8s_node_count - network_interfaces = [{ - associate_public_ip_address = true - }] } } tags = local.tags + } ################################################################################ @@ -72,6 +82,7 @@ module "eks_blueprints_addons" { } tags = local.tags + } data "aws_iam_policy_document" "assume_role" { @@ -91,7 +102,7 @@ data "aws_iam_policy_document" "assume_role" { } resource "aws_iam_role" "falkordb_backup_role" { - name = "eks-pod-identity-falkordb_backup_role" + name = "${var.name}-falkordb_backup_role" assume_role_policy = data.aws_iam_policy_document.assume_role.json } @@ -176,56 +187,17 @@ module "falkordb_backup_s3_bucket" { } } - tags = local.tags -} - -resource "aws_s3_bucket_lifecycle_configuration" "falkordb_backup_s3_bucket_lifecycle_configuration" { - bucket = module.falkordb_backup_s3_bucket.s3_bucket_id - - rule { - id = "falkordb-rule" - - filter { - prefix = "backups/" - } - - expiration { - days = var.backup_retention_period - } - abort_incomplete_multipart_upload { - days_after_initiation = 3 - } - - status = "Enabled" - } -} - -module "ebs_kms_key" { - source = "terraform-aws-modules/kms/aws" - version = "~> 1.5" - - description = "Customer managed key to encrypt EKS managed node group volumes" - - # Policy - key_administrators = [data.aws_caller_identity.current.arn] - key_service_roles_for_autoscaling = [ - # required for the ASG to manage encrypted volumes for nodes - "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling", - # required for the cluster / persistentvolume-controller to create encrypted PVCs - module.eks.cluster_iam_role_arn, - ] - - # Aliases - aliases = ["eks/${var.name}/ebs"] + force_destroy = true tags = local.tags + } module "ebs_csi_driver_irsa" { source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" version = "~> 5.20" - role_name_prefix = "${module.eks.cluster_name}-ebs-csi-driver-" + role_name_prefix = module.eks.cluster_name attach_ebs_csi_policy = true @@ -237,4 +209,4 @@ module "ebs_csi_driver_irsa" { } tags = local.tags -} \ No newline at end of file +} diff --git a/tofu/aws/outputs.tf b/tofu/aws/outputs.tf index 8be4c740..5faeddb6 100644 --- a/tofu/aws/outputs.tf +++ b/tofu/aws/outputs.tf @@ -6,23 +6,32 @@ output "falkordb_eks_cluster_name" { description = "EKS cluster name" value = module.eks.cluster_name } -output "falkordb_eks_endpoint" { - description = "EKS endpoint" +output "falkordb_s3_backup_name" { + description = "Backup bucket name" + value = module.falkordb_backup_s3_bucket.s3_bucket_id +} + +output "falkordb_eks_cluster_endpoint" { + description = "EKS cluster endpoint" value = module.eks.cluster_endpoint } -output "falkordb_cluster_certificate_authority_data" { - description = "EKS cluster certificate authority data" + +output "falkordb_eks_cluster_certificate_autority" { + description = "EKS cluster certificate autority" value = module.eks.cluster_certificate_authority_data } -output "falkordb_s3_backup_location" { - description = "S3 backup location" - value = local.falkordb_s3_backup_location + +output "falkordb_eks_cluster_role_arn" { + description = "EKS cluster role ARN" + value = module.eks.cluster_iam_role_arn } -output "falkordb_eks_oidc_issuer" { - description = "EKS OIDC issuer" + +output "falkordb_eks_cluster_oidc_issuer_url" { + description = "EKS cluster OIDC issuer URL" value = module.eks.cluster_oidc_issuer_url } -output "falkordb_eks_oidc_provider_arn" { - description = "EKS OIDC provider ARN" + +output "falkordb_eks_cluster_oidc_issuer_arn" { + description = "EKS cluster OIDC issuer ARN" value = module.eks.oidc_provider_arn } \ No newline at end of file diff --git a/tofu/aws/variables.tf b/tofu/aws/variables.tf index 9255ad06..be374076 100644 --- a/tofu/aws/variables.tf +++ b/tofu/aws/variables.tf @@ -1,3 +1,4 @@ + variable "name" { type = string } @@ -6,6 +7,14 @@ variable "region" { type = string } +variable "assume_role_arn" { + type = string +} + +variable "eks_auth_role" { + type = string +} + variable "k8s_version" { type = string } @@ -25,7 +34,3 @@ variable "k8s_node_min_count" { variable "k8s_node_max_count" { type = number } - -variable "backup_retention_period" { - type = number -} \ No newline at end of file diff --git a/tofu/aws/versions.tf b/tofu/aws/versions.tf deleted file mode 100644 index 2f671c08..00000000 --- a/tofu/aws/versions.tf +++ /dev/null @@ -1,10 +0,0 @@ -terraform { - required_version = ">= 1.0" - - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 4.47" - } - } -} \ No newline at end of file diff --git a/tofu/k8s/.terraform.lock.hcl b/tofu/k8s/.terraform.lock.hcl new file mode 100644 index 00000000..e3a20ed2 --- /dev/null +++ b/tofu/k8s/.terraform.lock.hcl @@ -0,0 +1,90 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "registry.opentofu.org/gavinbunney/kubectl" { + version = "1.14.0" + constraints = ">= 1.9.4" + hashes = [ + "h1:mX2AOFIMIxJmW5kM8DT51gloIOKCr9iT6W8yodnUyfs=", + "zh:0350f3122ff711984bbc36f6093c1fe19043173fad5a904bce27f86afe3cc858", + "zh:07ca36c7aa7533e8325b38232c77c04d6ef1081cb0bac9d56e8ccd51f12f2030", + "zh:0c351afd91d9e994a71fe64bbd1662d0024006b3493bb61d46c23ea3e42a7cf5", + "zh:39f1a0aa1d589a7e815b62b5aa11041040903b061672c4cfc7de38622866cbc4", + "zh:428d3a321043b78e23c91a8d641f2d08d6b97f74c195c654f04d2c455e017de5", + "zh:4baf5b1de2dfe9968cc0f57fd4be5a741deb5b34ee0989519267697af5f3eee5", + "zh:6131a927f9dffa014ab5ca5364ac965fe9b19830d2bbf916a5b2865b956fdfcf", + "zh:c62e0c9fd052cbf68c5c2612af4f6408c61c7e37b615dc347918d2442dd05e93", + "zh:f0beffd7ce78f49ead612e4b1aefb7cb6a461d040428f514f4f9cc4e5698ac65", + ] +} + +provider "registry.opentofu.org/hashicorp/aws" { + version = "5.34.0" + constraints = ">= 3.35.0, >= 4.33.0" + hashes = [ + "h1:Ofv10Aw1nrPZ4amBcpge3lodQ7acaY7H/DeuUA4EsOU=", + "zh:01ab6b6ae075e2d09d67578af05c325117c40d407f1f3602caa95d31b52218bd", + "zh:18b938e0272e52e18f79eb8e355bb81397d859e7ac98d66fba1c142d142314b6", + "zh:225cad47a8a224bf5fc7ac47bc46746f9143f8ad39a2b2f0b5306bd580a5994a", + "zh:35fcfacd4f8ed71a6f9854eeb862431ca832cb732e97fb02e3ea3f764610db3f", + "zh:4018cd69689857968afe31cf67ef4796b45f08c27012daff56201618621a690b", + "zh:442ec76a21e9a55c3d3e38c5d57087f5c5127326237af10463ec26c2355f3102", + "zh:8417f0a78697223f2a38dd3d1df88d69891a6b2168aabcb4916afd6008cf1609", + "zh:c48b6103112efc02711f046625a60f76912be5f3f590c8bf68d94895c0d45f1c", + "zh:d11f4cec70f382b716241104dabfd1c4cc6b95c836e7a8c469ab64f62a8824a6", + "zh:d52b6d37066685eb3e3a4bd32fcc7bc68aad1e81cbd3240feaa138313c740e81", + ] +} + +provider "registry.opentofu.org/hashicorp/helm" { + version = "2.12.1" + constraints = ">= 1.0.0, < 3.0.0" + hashes = [ + "h1:mRK57Pn5YGikn9jT4GyZtB1zf5gvu9ynNbwWq6YuPyA=", + "zh:0349149992646530c33314cb973eba68757606a037017ba47e56db695d4b3afe", + "zh:3138ffe23c481b01419a4a21adf83538efe6e698b421c4a8f7d142b198518709", + "zh:44658e3070405b88fbd76161ecddde62f478dc31aaebee3b93c2f2783a6d45f9", + "zh:5600a3407dfb8b77da7561490157afa8ad505c864a5dd35ed8d678e9ad8378ca", + "zh:6445e359c813ecbb7c2edf722ed0d1f33dfb171b6a7b470f40cf1e24045b7441", + "zh:7973054604c7f5a51600f6e63fa0327d05b29fac2bffd222c21660cbdd2939f9", + "zh:7c59e2d4602ab5d9de0ba8e442ec1fc425c8f143581018d1e7f645298a124f01", + "zh:8c0fb411dd5de664ac5e801d70507781790c4fc196518a56966d66d0963c240c", + "zh:a6a988c91bbf1828a8fc55001f10c7d06c5c53dc718ee7cd6814bdfa2e6652e0", + "zh:b7935d7dacd7e5a91ff9d17cfb04ce88c9100e563fd88487d14519e8d8d8b2e1", + ] +} + +provider "registry.opentofu.org/hashicorp/kubernetes" { + version = "2.25.2" + constraints = ">= 1.10.0, < 3.0.0" + hashes = [ + "h1:0rrxDjtJb63VSS9npSlikqJGH2L726byPwnP7fd3B/4=", + "zh:38d35c069a7f5a7c360ce2ddf6da7f22fd25290f76697d5ab0adbc9e7cae4db8", + "zh:4b1967b873c6262d8c8f57e8cea2cd029c43912555f688cdff4f824193be4e50", + "zh:57e6cfda60c084bb141ea1d8f0ad5881b1bbda92dfcb090e752b09d2cc153b84", + "zh:5af23ccba04c2fef7cff9cb17ed5c10ac9d89098706244d1be4d2acbf44f2ae2", + "zh:67cd0b02deba8361b2689afe4b2f67e38ca68539a89eb88ceaef7fae6f7e6f0b", + "zh:7b4d1fa9d9612919e29a9424a316af6b4c5de766debd8a823b8ad0f28ac2d9c6", + "zh:b7a9ca5d1a734675de32c9bbdc78e8569a10aef140fe6d176efed393e64e1d71", + "zh:c97777211b3f46ede86fce794ab515185e1145af6a4cd57effdfcddc3cccdcf7", + "zh:e7921548f82f68694ad51974bad7cc6c853134a9851912a04da4cdba34eb6214", + "zh:fa7b37554ed32422a239b707380ae778a8918e4917a76aadd92fe559a353a10c", + ] +} + +provider "registry.opentofu.org/hashicorp/random" { + version = "3.6.0" + hashes = [ + "h1:dMz6mC37wIEfyrB0VA8gzPx/tsCQJuQmjpoxUgwwEqU=", + "zh:486a1c921eab5c51a480f2eb0ad85173f207c9b7bb215f3893e58bc38d3b7c75", + "zh:6901b3afa4607d1e31934ba91ed2625215ada42b3518c3a9adeeac7a5f656dc3", + "zh:7e93752c9de710e417191353ad1a41b5a60432ab7ef4f8b556bf248297ec5e23", + "zh:c795d3d319e8ee7be972746b935963b7e772a6a14080261a35c03915c1f9ccb2", + "zh:cd4f8bcaf332828d1736c73874549c25e427737f136173c7b61e2df3db50e5d9", + "zh:e0103eb2e280989c3d9ffda5d6b413e8f583be21bc1d5754c6e9ca87ecc1c44a", + "zh:f4fbec2510322d5b7ad584a92436b5dbd0f2e897a3ec538932af59e245a4c8e4", + "zh:f5418842afd4aa7676e2456e425e8f573cb2b9bffd29bd7de09d91845644ab24", + "zh:f572a26f93d00ec42461ce478678366e570fa4497e2273f9d47f24cdfc4b42b4", + "zh:ff1f07c561a3f7f219b6fee1647a559933b5dd6181753e164c3978fd47a11685", + ] +} diff --git a/tofu/k8s/main.tf b/tofu/k8s/main.tf index 237bc1a5..8c138709 100644 --- a/tofu/k8s/main.tf +++ b/tofu/k8s/main.tf @@ -1,14 +1,33 @@ + provider "kubernetes" { - host = var.falkordb_eks_endpoint - cluster_ca_certificate = base64decode(var.falkordb_cluster_certificate_authority_data) + host = var.falkordb_eks_cluster_endpoint + cluster_ca_certificate = base64decode(var.falkordb_eks_cluster_certificate_autority) exec { api_version = "client.authentication.k8s.io/v1beta1" command = "aws" - args = ["eks", "get-token", "--cluster-name", var.falkordb_eks_cluster_name] + args = ["eks", "get-token", "--cluster-name", var.falkordb_eks_cluster_name, "--role-arn", var.assume_role_arn] + } +} + + +provider "helm" { + kubernetes { + host = var.falkordb_eks_cluster_endpoint + cluster_ca_certificate = base64decode(var.falkordb_eks_cluster_certificate_autority) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + args = ["eks", "get-token", "--cluster-name", var.falkordb_eks_cluster_name, "--role-arn", var.assume_role_arn] + } } } +data "aws_caller_identity" "current" { +} + + resource "kubernetes_namespace" "backup_namespace" { metadata { name = "falkordb-backup" @@ -60,6 +79,27 @@ resource "kubernetes_cluster_role_binding" "falkordb_role_binding" { } } +resource "aws_s3_bucket_lifecycle_configuration" "falkordb_backup_s3_bucket_lifecycle_configuration" { + bucket = var.falkordb_s3_backup_name + + rule { + id = "falkordb-rule-${var.tenant_name}" + + filter { + prefix = "${var.tenant_name}/" + } + + expiration { + days = var.backup_retention_period + } + abort_incomplete_multipart_upload { + days_after_initiation = 3 + } + + status = "Enabled" + } +} + resource "kubernetes_cron_job_v1" "falkorbd_backup" { metadata { name = "falkordb-backup" @@ -80,13 +120,17 @@ resource "kubernetes_cron_job_v1" "falkorbd_backup" { name = "backup" image = "amazon/aws-cli" # https://docs.aws.amazon.com/eks/latest/userguide/install-kubectl.html - command = ["/bin/sh", "-c", "curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.28.3/2023-11-14/bin/linux/amd64/kubectl; chmod +x kubectl; ./kubectl exec falkordb-redis-node-0 --namespace falkordb -- redis-cli -a '${random_password.password.result}' save; ./kubectl cp falkordb-redis-node-0:/data/dump.rdb dump.rdb -c redis --namespace falkordb; aws s3 cp dump.rdb s3://${var.falkordb_s3_backup_location}/dump$(date +'%Y-%m-%d_%H-%M-%S').rdb"] + command = ["/bin/sh", "-c", "curl -O https://s3.us-west-2.amazonaws.com/amazon-eks/1.28.3/2023-11-14/bin/linux/amd64/kubectl; chmod +x kubectl; ./kubectl exec falkordb-redis-node-0 --namespace falkordb -- redis-cli -a '${local.falkordb_password}' save; ./kubectl cp falkordb-redis-node-0:/data/dump.rdb dump.rdb -c redis --namespace falkordb; aws s3 cp dump.rdb s3://${var.falkordb_s3_backup_name}/${var.tenant_name}/dump$(date +'%Y-%m-%d_%H-%M-%S').rdb"] } } } } } } + + depends_on = [ + kubernetes_namespace.backup_namespace + ] } # https://docs.syseleven.de/metakube-accelerator/building-blocks/observability-monitoring/kube-prometheus-stack#adding-grafana-dashboards @@ -102,37 +146,70 @@ resource "kubernetes_config_map" "falkordb_grafana_dashboard" { } } -provider "helm" { - kubernetes { - host = var.falkordb_eks_endpoint - cluster_ca_certificate = base64decode(var.falkordb_cluster_certificate_authority_data) - - exec { - api_version = "client.authentication.k8s.io/v1beta1" - command = "aws" - args = ["eks", "get-token", "--cluster-name", var.falkordb_eks_cluster_name] - } - } -} - resource "random_password" "password" { length = 16 special = true override_special = "" } +locals { + falkordb_password = var.falkordb_password != null ? var.falkordb_password : random_password.password.result + + tags = { + customer = var.tenant_name + } +} + +module "ebs_kms_key" { + source = "terraform-aws-modules/kms/aws" + version = "~> 2.1.0" + + description = "Customer managed key to encrypt EKS managed node group volumes" + + # Policy + key_administrators = var.key_administrators + key_service_roles_for_autoscaling = var.key_service_roles_for_autoscaling + + tags = local.tags +} + +# Create a storage class for EBS volumes +# resource "kubernetes_storage_class" "falkordb" { +# metadata { +# name = "falkordb-storage-class" +# } + +# storage_provisioner = "ebs.csi.aws.com" + +# reclaim_policy = "Delete" +# allow_volume_expansion = true + +# volume_binding_mode = "WaitForFirstConsumer" +# parameters = { +# type = "gp2" +# encrypted = "true" +# kmsKeyId = module.ebs_kms_key.key_id +# } +# } + # https://github.com/bitnami/charts/tree/main/bitnami/redis resource "helm_release" "falkordb" { name = "falkordb" - namespace = "falkordb" + namespace = kubernetes_namespace.falkordb.metadata[0].name version = "18.6.3" + # Necessary so there's enough time to finish installing + timeout = 600 + + # Must be cluster name so we can destroy the load balancer + description = var.falkordb_eks_cluster_name + repository = "https://charts.bitnami.com/bitnami" chart = "redis" set { name = "global.redis.password" - value = random_password.password.result + value = local.falkordb_password } set { name = "image.repository" @@ -238,11 +315,22 @@ resource "helm_release" "falkordb" { name = "useExternalDNS.suffix" value = "falkordb.io" } + + # set { + # name = "global.storageClass" + # value = "falkordb-storage-class" + # } + + + depends_on = [ + module.load_balancer_controller + ] } + resource "helm_release" "falkordb-monitoring" { name = "falkordb-monitoring" - namespace = "falkordb-monitoring" + namespace = kubernetes_namespace.falkordb_monitoring.metadata[0].name repository = "https://prometheus-community.github.io/helm-charts" chart = "kube-prometheus-stack" @@ -270,7 +358,7 @@ resource "helm_release" "falkordb-monitoring" { } set { name = "grafana.additionalDataSources[0].secureJsonData.password" - value = random_password.password.result + value = local.falkordb_password } set { name = "grafana.additionalDataSources[0].editable" @@ -279,25 +367,26 @@ resource "helm_release" "falkordb-monitoring" { } # https://github.com/kubernetes-sigs/external-dns -module "eks-external-dns" { - source = "lablabs/eks-external-dns/aws" - version = "1.2.0" - cluster_identity_oidc_issuer = var.falkordb_eks_oidc_issuer - cluster_identity_oidc_issuer_arn = var.falkordb_eks_oidc_provider_arn - - settings = { - "policy" = "upsert-only" - "aws.zoneType" = "public" - "domainFilters[0]" = var.falkordb_domain - "txtOwnerId" = var.falkordb_hosted_zone_id - } -} +# module "eks-external-dns" { +# source = "lablabs/eks-external-dns/aws" +# version = "1.2.0" +# cluster_identity_oidc_issuer = data.aws_iam_openid_connect_provider.cluster.url +# cluster_identity_oidc_issuer_arn = data.aws_iam_openid_connect_provider.cluster.arn + +# settings = { +# "policy" = "upsert-only" +# "aws.zoneType" = "public" +# "domainFilters[0]" = var.falkordb_domain +# "txtOwnerId" = var.falkordb_hosted_zone_id +# } +# } module "load_balancer_controller" { source = "git::https://github.com/DNXLabs/terraform-aws-eks-lb-controller.git" - cluster_identity_oidc_issuer = var.falkordb_eks_oidc_issuer - cluster_identity_oidc_issuer_arn = var.falkordb_eks_oidc_provider_arn + cluster_identity_oidc_issuer = var.falkordb_eks_cluster_oidc_issuer_url + cluster_identity_oidc_issuer_arn = var.falkordb_eks_cluster_oidc_issuer_arn cluster_name = var.falkordb_eks_cluster_name - helm_chart_version = "1.6.2" -} \ No newline at end of file + helm_chart_version = "1.6.2" + +} diff --git a/tofu/k8s/variables.tf b/tofu/k8s/variables.tf index 214a340b..abb65f5c 100644 --- a/tofu/k8s/variables.tf +++ b/tofu/k8s/variables.tf @@ -1,7 +1,25 @@ +variable "region" { + type = string +} + +variable "assume_role_arn" { + type = string +} + +variable "tenant_name" { + type = string +} + variable "falkordb_version" { type = string } +variable "falkordb_password" { + type = string + sensitive = true + nullable = true +} + variable "falkordb_cpu" { type = string } @@ -27,33 +45,48 @@ variable "backup_schedule" { } variable "falkordb_eks_cluster_name" { - type = string + type = string + default = "cluster_name" } -variable "falkordb_eks_endpoint" { +variable "falkordb_s3_backup_name" { type = string } -variable "falkordb_cluster_certificate_authority_data" { - type = string +# variable "falkordb_domain" { +# type = string +# } + +# variable "falkordb_hosted_zone_id" { +# type = string +# } + +variable "backup_retention_period" { + type = number } -variable "falkordb_s3_backup_location" { - type = string +variable "key_administrators" { + type = list(string) + default = [] } -variable "falkordb_eks_oidc_provider_arn" { +variable "key_service_roles_for_autoscaling" { + type = list(string) + default = [] +} + +variable "falkordb_eks_cluster_oidc_issuer_url" { type = string } -variable "falkordb_eks_oidc_issuer" { +variable "falkordb_eks_cluster_oidc_issuer_arn" { type = string } -variable "falkordb_domain" { +variable "falkordb_eks_cluster_endpoint" { type = string } -variable "falkordb_hosted_zone_id" { +variable "falkordb_eks_cluster_certificate_autority" { type = string -} \ No newline at end of file +} diff --git a/tofu/k8s/versions.tf b/tofu/k8s/versions.tf deleted file mode 100644 index ee0fa0c3..00000000 --- a/tofu/k8s/versions.tf +++ /dev/null @@ -1,18 +0,0 @@ -terraform { - required_version = ">= 1.0" - - required_providers { - helm = { - source = "hashicorp/helm" - version = ">= 2.9" - } - kubernetes = { - source = "hashicorp/kubernetes" - version = ">= 2.20" - } - random = { - source = "hashicorp/random" - version = ">= 3.0" - } - } -} \ No newline at end of file diff --git a/tofu/main.tf b/tofu/main.tf deleted file mode 100644 index 343f6b28..00000000 --- a/tofu/main.tf +++ /dev/null @@ -1,29 +0,0 @@ -module "aws" { - source = "./aws" - name = var.name - region = var.region - k8s_version = var.k8s_version - k8s_instance_type = var.k8s_instance_type - k8s_node_count = var.k8s_node_count - k8s_node_min_count = var.k8s_node_min_count - k8s_node_max_count = var.k8s_node_max_count - backup_retention_period = var.backup_retention_period -} -module "k8s" { - source = "./k8s" - falkordb_version = var.falkordb_version - falkordb_cpu = var.falkordb_cpu - falkordb_memory = var.falkordb_memory - persistance_size = var.persistance_size - falkordb_replicas = var.falkordb_replicas - grafana_admin_password = var.grafana_admin_password - backup_schedule = var.backup_schedule - falkordb_domain = var.falkordb_domain - falkordb_hosted_zone_id = var.falkordb_hosted_zone_id - falkordb_eks_cluster_name = module.aws.falkordb_eks_cluster_name - falkordb_eks_endpoint = module.aws.falkordb_eks_endpoint - falkordb_cluster_certificate_authority_data = module.aws.falkordb_cluster_certificate_authority_data - falkordb_s3_backup_location = module.aws.falkordb_s3_backup_location - falkordb_eks_oidc_issuer = module.aws.falkordb_eks_oidc_issuer - falkordb_eks_oidc_provider_arn = module.aws.falkordb_eks_oidc_provider_arn -} \ No newline at end of file diff --git a/tofu/template.tfvars b/tofu/template.tfvars deleted file mode 100644 index 6b9b18da..00000000 --- a/tofu/template.tfvars +++ /dev/null @@ -1,17 +0,0 @@ -name = "" -region = "" -k8s_version = "1.28" -k8s_instance_type = "t3.medium" -k8s_node_count = 2 -k8s_node_min_count = 2 -k8s_node_max_count = 3 -backup_retention_period = 7 -falkordb_version = "v4.0.3" -falkordb_cpu = "500m" -falkordb_memory = "1Gi" -persistance_size = "8Gi" -falkordb_replicas = 2 -grafana_admin_password = "admin" -backup_schedule = "0 * * * *" -falkordb_domain = "" -falkordb_hosted_zone_id = "" \ No newline at end of file diff --git a/tofu/variables.tf b/tofu/variables.tf deleted file mode 100644 index 972fb483..00000000 --- a/tofu/variables.tf +++ /dev/null @@ -1,67 +0,0 @@ -variable "name" { - type = string -} - -variable "region" { - type = string -} - -variable "k8s_version" { - type = string -} - -variable "k8s_instance_type" { - type = string -} - -variable "k8s_node_count" { - type = number -} - -variable "k8s_node_min_count" { - type = number -} - -variable "k8s_node_max_count" { - type = number -} - -variable "backup_retention_period" { - type = number -} - -variable "falkordb_version" { - type = string -} - -variable "falkordb_cpu" { - type = string -} - -variable "falkordb_memory" { - type = string -} - -variable "persistance_size" { - type = string -} - -variable "falkordb_replicas" { - type = number -} - -variable "grafana_admin_password" { - type = string -} - -variable "backup_schedule" { - type = string -} - -variable "falkordb_domain" { - type = string -} - -variable "falkordb_hosted_zone_id" { - type = string -} \ No newline at end of file