From 65e9723d2d74226b3cd3da1a223dbbf6a1b11cac Mon Sep 17 00:00:00 2001 From: Jeff DeWan Date: Tue, 15 Oct 2024 14:10:45 -0700 Subject: [PATCH 01/26] OP-517: Add deploy pipelines for livepoll and pandoc --- .github/actions/private-ecr-login/action.yml | 39 ++++ .github/workflows/build-livepoll.yml | 230 +++++++++++++++++++ .github/workflows/manual-deploy.yml | 30 +++ .github/workflows/shared-deploy.yml | 143 ++++++++++++ .github/workflows/tag-deploy-livepoll.yml | 16 ++ .github/workflows/tag-deploy-pandoc.yml | 16 ++ .github/workflows/terraform-deploy.yml | 88 +++++++ .github/workflows/terraform.yml | 31 +++ deploy/ecs_deploy.sh | 60 +++++ deploy/livepoll-environments.json | 43 ++++ deploy/pandoc-environments.json | 55 +++++ deploy/terraform/livepoll/_locals.tf | 5 + deploy/terraform/livepoll/_provider.tf | 25 ++ deploy/terraform/livepoll/parameter_store.tf | 23 ++ deploy/terraform/livepoll/variables.tf | 11 + deploy/terraform/pandoc/_locals.tf | 6 + deploy/terraform/pandoc/_provider.tf | 25 ++ deploy/terraform/pandoc/variables.tf | 11 + docker/Dockerfile-livepoll | 1 - docker/Dockerfile-pandoc | 16 +- docker/Dockerfile-pandoc-cron | 13 -- 21 files changed, 872 insertions(+), 15 deletions(-) create mode 100644 .github/actions/private-ecr-login/action.yml create mode 100644 .github/workflows/build-livepoll.yml create mode 100644 .github/workflows/manual-deploy.yml create mode 100644 .github/workflows/shared-deploy.yml create mode 100644 .github/workflows/tag-deploy-livepoll.yml create mode 100644 .github/workflows/tag-deploy-pandoc.yml create mode 100644 .github/workflows/terraform-deploy.yml create mode 100644 .github/workflows/terraform.yml create mode 100755 deploy/ecs_deploy.sh create mode 100644 deploy/livepoll-environments.json create mode 100644 deploy/pandoc-environments.json create mode 100644 deploy/terraform/livepoll/_locals.tf create mode 100644 deploy/terraform/livepoll/_provider.tf create mode 100644 deploy/terraform/livepoll/parameter_store.tf create mode 100644 deploy/terraform/livepoll/variables.tf create mode 100644 deploy/terraform/pandoc/_locals.tf create mode 100644 deploy/terraform/pandoc/_provider.tf create mode 100644 deploy/terraform/pandoc/variables.tf delete mode 100644 docker/Dockerfile-pandoc-cron diff --git a/.github/actions/private-ecr-login/action.yml b/.github/actions/private-ecr-login/action.yml new file mode 100644 index 0000000..774a645 --- /dev/null +++ b/.github/actions/private-ecr-login/action.yml @@ -0,0 +1,39 @@ +name: 'Log into Private ECR' +description: 'Github OIDC auth and assume role into account, then use AWS ECR Login action' +inputs: + aws_ci_account: + description: 'AWS Account ID for CI' + required: false + default: 824635284302 + aws_user_account: + description: 'AWS Account ID for users' + required: false + default: 265299512749 +outputs: + registry: + description: "ECR Registry" + value: ${{ steps.login-ecr.outputs.registry }} +runs: + using: "composite" + steps: + - name: assume oidc role + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-west-2 + role-to-assume: arn:aws:iam::${{ inputs.aws_user_account }}:role/ci-oidc-role + role-session-name: github-actions-oidc + role-duration-seconds: 900 + - name: assume target role + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }} + aws-session-token: ${{ env.AWS_SESSION_TOKEN }} + aws-region: us-west-2 + role-to-assume: arn:aws:iam::${{ inputs.aws_ci_account }}:role/ci-role + role-session-name: github-actions-private-ecr + role-duration-seconds: 900 + + - name: Login to Private ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 \ No newline at end of file diff --git a/.github/workflows/build-livepoll.yml b/.github/workflows/build-livepoll.yml new file mode 100644 index 0000000..8858f07 --- /dev/null +++ b/.github/workflows/build-livepoll.yml @@ -0,0 +1,230 @@ +name: build-and-push +env: + LIVEPOLL_ECR_REPOSITORY: livepoll + PANDOC_ECR_REPOSITORY: pandoc +on: + pull_request: + types: [ opened, synchronize, reopened ] + push: + branches: + - main + - rc/** + +jobs: + # this job gets a password for public ECR to use in future steps + ecr-public-auth: + runs-on: ubuntu-latest + name: Login to Public ECR + timeout-minutes: 5 + permissions: + id-token: write # Used for AWS OIDC auth + outputs: + ECR_PASSWORD: ${{ steps.ecr-auth.outputs.ECR_PASSWORD }} + steps: + # This is required to role chain from the OIDC role to other cross-account roles + # https://github.com/aws-actions/configure-aws-credentials/issues/279 + - name: OIDC Auth + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-west-2 + role-to-assume: arn:aws:iam::265299512749:role/ci-oidc-role + role-session-name: github-actions + role-duration-seconds: 900 + + - name: Assume role in CI Account + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }} + aws-session-token: ${{ env.AWS_SESSION_TOKEN }} + aws-region: us-west-2 + role-to-assume: arn:aws:iam::824635284302:role/ci-role + role-session-name: github-actions + role-duration-seconds: 1200 + + - name: Set Public ECR Auth + id: ecr-auth + run: echo "ECR_PASSWORD=$(aws ecr-public --region us-east-1 get-login-password)" >> $GITHUB_OUTPUT + + validate-terraform: + runs-on: ubuntu-latest + name: Validate Terraform + timeout-minutes: 5 + permissions: + id-token: write # Used for AWS OIDC auth + contents: read + actions: read + steps: + - uses: actions/checkout@v4 + name: Checkout project + + - name: OIDC Auth to AWS + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-west-2 + role-to-assume: arn:aws:iam::${{ secrets.aws_user_account }}:role/ci-oidc-role + role-session-name: github-actions + role-duration-seconds: 900 + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + + - name: Terraform Init + id: init + run: terraform init -backend=false + working-directory: deploy/terraform + + - name: Terraform Format + id: fmt + run: terraform fmt -check + + - name: Terraform Validate + id: validate + run: terraform validate -no-color + working-directory: deploy/terraform + + build-and-push-livepoll: + runs-on: ubuntu-latest + name: Build and Push Livepoll Docker Image + timeout-minutes: 30 + needs: [ecr-public-auth, validate-terraform] + permissions: + id-token: write # Used for AWS OIDC auth + contents: read + actions: read + steps: + - uses: actions/checkout@v4 + name: Checkout project + + - id: read_tree_hash + name: Read git tree hash + run: | + tree_hash=$(git rev-parse HEAD:) + echo "tree_hash=$tree_hash" >> $GITHUB_OUTPUT + + - id: set_branch_name + name: Read git branch name + run: | + branch_name=${GITHUB_REF##*/} + echo "branch_name=$branch_name" >> $GITHUB_OUTPUT + + - uses: ./.github/actions/private-ecr-login + name: Login to Private ECR + id: login-ecr + + - name: Login to Public ECR + uses: docker/login-action@v3 + with: + registry: public.ecr.aws/j7v6u1e0 + username: AWS + password: ${{ needs.ecr-public-auth.outputs.ECR_PASSWORD }} + + - name: Check for prebuilt image + id: prebuilt_check + run: | + docker manifest inspect ${{ steps.login-ecr.outputs.registry }}/${{ env.LIVEPOLL_ECR_REPOSITORY }}:gha-tree-${{ steps.read_tree_hash.outputs.tree_hash }} || echo "image_exists=$?" >> $GITHUB_OUTPUT + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + if: steps.prebuilt_check.outputs.image_exists != 0 + + - name: Docker build and push + uses: docker/build-push-action@v5 + if: steps.prebuilt_check.outputs.image_exists != 0 + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + with: + context: . + file: docker/Dockerfile-livepoll + platforms: linux/amd64 + push: true + tags: | + ${{ steps.login-ecr.outputs.registry }}/${{ env.LIVEPOLL_ECR_REPOSITORY }}:gha-commit-${{ github.sha }} + ${{ steps.login-ecr.outputs.registry }}/${{ env.LIVEPOLL_ECR_REPOSITORY }}:gha-tree-${{ steps.read_tree_hash.outputs.tree_hash }} + ${{ steps.login-ecr.outputs.registry }}/${{ env.LIVEPOLL_ECR_REPOSITORY }}:gha-build-${{ steps.set_branch_name.outputs.branch_name }}-${{ github.run_number }}-${{ github.run_attempt }} + ${{ steps.login-ecr.outputs.registry }}/${{ env.LIVEPOLL_ECR_REPOSITORY }}:gha-build-${{ steps.set_branch_name.outputs.branch_name }}-latest + cache-from: type=gha + cache-to: type=gha,mode=max + + build-and-push-pandoc: + runs-on: ubuntu-latest + name: Build and Push Pandoc Docker Image + timeout-minutes: 30 + needs: [ecr-public-auth, validate-terraform] + permissions: + id-token: write # Used for AWS OIDC auth + contents: read + actions: read + steps: + - uses: actions/checkout@v4 + name: Checkout project + + - id: read_tree_hash + name: Read git tree hash + run: | + tree_hash=$(git rev-parse HEAD:) + echo "tree_hash=$tree_hash" >> $GITHUB_OUTPUT + + - id: set_branch_name + name: Read git branch name + run: | + branch_name=${GITHUB_REF##*/} + echo "branch_name=$branch_name" >> $GITHUB_OUTPUT + + - uses: ./.github/actions/private-ecr-login + name: Login to Private ECR + id: login-ecr + + - name: Login to Public ECR + uses: docker/login-action@v3 + with: + registry: public.ecr.aws/j7v6u1e0 + username: AWS + password: ${{ needs.ecr-public-auth.outputs.ECR_PASSWORD }} + + - name: Check for prebuilt image + id: prebuilt_check + run: | + docker manifest inspect ${{ steps.login-ecr.outputs.registry }}/${{ env.PANDOC_ECR_REPOSITORY }}:gha-tree-${{ steps.read_tree_hash.outputs.tree_hash }} || echo "image_exists=$?" >> $GITHUB_OUTPUT + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + if: steps.prebuilt_check.outputs.image_exists != 0 + + - name: Docker build and push + uses: docker/build-push-action@v5 + if: steps.prebuilt_check.outputs.image_exists != 0 + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + with: + context: . + file: docker/Dockerfile-pandoc + target: pandoc + platforms: linux/amd64 + push: true + tags: | + ${{ steps.login-ecr.outputs.registry }}/${{ env.PANDOC_ECR_REPOSITORY }}:gha-commit-${{ github.sha }} + ${{ steps.login-ecr.outputs.registry }}/${{ env.PANDOC_ECR_REPOSITORY }}:gha-tree-${{ steps.read_tree_hash.outputs.tree_hash }} + ${{ steps.login-ecr.outputs.registry }}/${{ env.PANDOC_ECR_REPOSITORY }}:gha-build-${{ steps.set_branch_name.outputs.branch_name }}-${{ github.run_number }}-${{ github.run_attempt }} + ${{ steps.login-ecr.outputs.registry }}/${{ env.PANDOC_ECR_REPOSITORY }}:gha-build-${{ steps.set_branch_name.outputs.branch_name }}-latest + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Docker build and push + uses: docker/build-push-action@v5 + if: steps.prebuilt_check.outputs.image_exists != 0 + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + with: + context: . + file: docker/Dockerfile-pandoc + target: cron + platforms: linux/amd64 + push: true + tags: | + ${{ steps.login-ecr.outputs.registry }}/${{ env.PANDOC_ECR_REPOSITORY }}:gha-commit-${{ github.sha }}-cron + ${{ steps.login-ecr.outputs.registry }}/${{ env.PANDOC_ECR_REPOSITORY }}:gha-tree-${{ steps.read_tree_hash.outputs.tree_hash }}-cron + ${{ steps.login-ecr.outputs.registry }}/${{ env.PANDOC_ECR_REPOSITORY }}:gha-build-${{ steps.set_branch_name.outputs.branch_name }}-${{ github.run_number }}-${{ github.run_attempt }}-cron + ${{ steps.login-ecr.outputs.registry }}/${{ env.PANDOC_ECR_REPOSITORY }}:gha-build-${{ steps.set_branch_name.outputs.branch_name }}-latest-cron + cache-from: type=gha + cache-to: type=gha,mode=max \ No newline at end of file diff --git a/.github/workflows/manual-deploy.yml b/.github/workflows/manual-deploy.yml new file mode 100644 index 0000000..d0d5985 --- /dev/null +++ b/.github/workflows/manual-deploy.yml @@ -0,0 +1,30 @@ +name: manual-deploy +on: + workflow_dispatch: + inputs: + environment: + description: 'Environment' + required: true + default: 'dev' + type: choice + options: + - dev + - staging + project: + description: 'Project' + required: true + default: 'livepoll' + type: choice + options: + - livepoll + - pandoc +jobs: + ecs-deploy: + name: ECS Deployment + uses: ./.github/workflows/shared-deploy.yml + with: + environment: ${{ github.event.inputs.environment }} + project: ${{ github.event.inputs.project }} + secrets: + aws_user_account: ${{ secrets.AWS_USER_ACCOUNT }} + aws_ci_account: ${{ secrets.AWS_CI_ACCOUNT }} \ No newline at end of file diff --git a/.github/workflows/shared-deploy.yml b/.github/workflows/shared-deploy.yml new file mode 100644 index 0000000..ae877c6 --- /dev/null +++ b/.github/workflows/shared-deploy.yml @@ -0,0 +1,143 @@ +name: shared-deployment-workflow +permissions: + id-token: write # Used for AWS OIDC auth + contents: read # This is required for actions/checkout +on: + workflow_call: + inputs: + environment: + description: 'Environment name passed from the caller workflow' + required: true + type: string + project: + description: 'Project name passed from the caller workflow' + required: true + type: string + secrets: + aws_user_account: + description: 'AWS Account ID for IAM users' + required: true + aws_ci_account: + description: 'AWS Account ID for IAM users' + required: true + +jobs: + # this job gets a password for public ECR to use in future steps + ecr-public-auth: + runs-on: ubuntu-latest + name: Login to Public ECR + timeout-minutes: 5 + permissions: + id-token: write # Used for AWS OIDC auth + outputs: + ECR_PASSWORD: ${{ steps.ecr-auth.outputs.ECR_PASSWORD }} + steps: + # This is required to role chain from the OIDC role to other cross-account roles + # https://github.com/aws-actions/configure-aws-credentials/issues/279 + - name: OIDC Auth + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-west-2 + role-to-assume: arn:aws:iam::${{ secrets.aws_user_account }}:role/ci-oidc-role + role-session-name: github-actions + role-duration-seconds: 900 + + - name: Assume role in CI Account + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }} + aws-session-token: ${{ env.AWS_SESSION_TOKEN }} + aws-region: us-west-2 + role-to-assume: arn:aws:iam::${{ secrets.aws_ci_account }}:role/ci-role + role-session-name: github-actions + role-duration-seconds: 1200 + + - name: Set Public ECR Auth + id: ecr-auth + run: echo "ECR_PASSWORD=$(aws ecr-public --region us-east-1 get-login-password)" >> $GITHUB_OUTPUT + + terraform-deploy: + name: Terraform Deployment + uses: ./.github/workflows/terraform-deploy.yml + with: + environment: ${{ inputs.environment }} + project: ${{ inputs.project }} + secrets: + aws_user_account: ${{ secrets.AWS_USER_ACCOUNT }} + aws_ci_account: ${{ secrets.AWS_CI_ACCOUNT }} + + ecs-deploy: + name: ECS Deployment + timeout-minutes: 30 + runs-on: ubuntu-latest + needs: [ecr-public-auth,terraform-deploy] + steps: + - uses: actions/checkout@v4 + name: Checkout project + + - uses: ./.github/actions/private-ecr-login + name: Login to Private ECR + id: login-ecr + + - name: Login to Public ECR + uses: docker/login-action@v3 + with: + registry: public.ecr.aws/o9c0t3w1 + username: AWS + password: ${{ needs.ecr-public-auth.outputs.ECR_PASSWORD }} + + + - id: read_env_json + name: Read Environment JSON + run: | + env_json=$(jq -c '.environments[] | select(.environment_label=="${{ inputs.environment }}")' ./deploy/${{ inputs.project }}-environments.json) + echo "env_json=$env_json" >> $GITHUB_OUTPUT + - id: set_env_metadata + name: Set Environment Metadata + run: | + echo "account=${{ fromJSON(steps.read_env_json.outputs.env_json).account }}" >> $GITHUB_OUTPUT + echo "ssm_prefix=${{ fromJSON(steps.read_env_json.outputs.env_json).ssm_prefix }}" >> $GITHUB_OUTPUT + echo "webapp_ssm_prefix=${{ fromJSON(steps.read_env_json.outputs.env_json).webapp.ssm_prefix }}" >> $GITHUB_OUTPUT + echo "shoryuken_ssm_prefix=${{ fromJSON(steps.read_env_json.outputs.env_json).shoryuken.ssm_prefix }}" >> $GITHUB_OUTPUT + echo "cluster_name=${{ fromJSON(steps.read_env_json.outputs.env_json).fargate.cluster_name }}" >> $GITHUB_OUTPUT + + - name: OIDC Auth to AWS + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-west-2 + role-to-assume: arn:aws:iam::${{ secrets.aws_user_account }}:role/ci-oidc-role + role-session-name: github-actions + role-duration-seconds: 900 + + - name: Assume role in target account + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }} + aws-session-token: ${{ env.AWS_SESSION_TOKEN }} + aws-region: us-west-2 + role-to-assume: arn:aws:iam::${{ steps.set_env_metadata.outputs.account }}:role/ci-role + role-session-name: github-actions + role-duration-seconds: 1200 + + - id: read_tree_hash + name: Read git tree hash + run: | + tree_hash=$(git rev-parse HEAD:) + echo "tree_hash=$tree_hash" >> $GITHUB_OUTPUT + + - name: Verify image + run: | + if ! docker manifest inspect ${{ steps.login-ecr.outputs.registry }}/${{ inputs.project }}:gha-tree-${{ steps.read_tree_hash.outputs.tree_hash }}; then + echo "If this is a PR build, you may need to pull in changes from the target branch into your PR branch." + exit 1 + fi + + - name: Run deploy script + run: | + deploy/ecs_deploy.sh \ + --cluster-name ${{ steps.set_env_metadata.outputs.cluster_name }} \ + --ssm-prefix ${{ steps.set_env_metadata.outputs.ssm_prefix }} \ + --account-number ${{ secrets.aws_ci_account }} \ + --project-name ${{ inputs.project }} \ No newline at end of file diff --git a/.github/workflows/tag-deploy-livepoll.yml b/.github/workflows/tag-deploy-livepoll.yml new file mode 100644 index 0000000..12f8351 --- /dev/null +++ b/.github/workflows/tag-deploy-livepoll.yml @@ -0,0 +1,16 @@ +name: tag-deploy +on: + push: + tags: + - release/livepoll/* + +jobs: + ecs-deploy: + name: ECS Deployment + uses: ./.github/workflows/shared-deploy.yml + with: + environment: prod + project: livepoll + secrets: + aws_user_account: ${{ secrets.AWS_USER_ACCOUNT }} + aws_ci_account: ${{ secrets.AWS_CI_ACCOUNT }} \ No newline at end of file diff --git a/.github/workflows/tag-deploy-pandoc.yml b/.github/workflows/tag-deploy-pandoc.yml new file mode 100644 index 0000000..cd0ba9d --- /dev/null +++ b/.github/workflows/tag-deploy-pandoc.yml @@ -0,0 +1,16 @@ +name: tag-deploy +on: + push: + tags: + - release/pandoc/* + +jobs: + ecs-deploy: + name: ECS Deployment + uses: ./.github/workflows/shared-deploy.yml + with: + environment: prod + project: pandoc + secrets: + aws_user_account: ${{ secrets.AWS_USER_ACCOUNT }} + aws_ci_account: ${{ secrets.AWS_CI_ACCOUNT }} \ No newline at end of file diff --git a/.github/workflows/terraform-deploy.yml b/.github/workflows/terraform-deploy.yml new file mode 100644 index 0000000..ad2c09b --- /dev/null +++ b/.github/workflows/terraform-deploy.yml @@ -0,0 +1,88 @@ +name: terraform-deployment-workflow +permissions: + id-token: write # Used for AWS OIDC auth + contents: read # This is required for actions/checkout +on: + workflow_call: + inputs: + environment: + description: 'Environment name passed from the caller workflow' + required: true + type: string + project: + description: 'Project name passed from the caller workflow' + required: true + type: string + secrets: + aws_user_account: + description: 'AWS Account ID for IAM users' + required: true + aws_ci_account: + description: 'AWS Account ID for IAM users' + required: true + +jobs: + # this job gets a password for public ECR to use in future steps + terraform-deploy: + name: Terraform Deployment + timeout-minutes: 30 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + name: Checkout project + + - id: read_env_json + name: Read Environment JSON + run: | + env_json=$(jq -c '.environments[] | select(.environment_label=="${{ inputs.environment }}")' ./deploy/${{ inputs.project }}-environments.json) + echo "env_json=$env_json" >> $GITHUB_OUTPUT + + - id: set_env_metadata + name: Set Environment Metadata + run: | + echo "account=${{ fromJSON(steps.read_env_json.outputs.env_json).account }}" >> $GITHUB_OUTPUT + echo "region=${{ fromJSON(steps.read_env_json.outputs.env_json).region }}" >> $GITHUB_OUTPUT + echo "terraform_state_key=${{ fromJSON(steps.read_env_json.outputs.env_json).terraform_state_key }}" >> $GITHUB_OUTPUT + echo "ssm_prefix=${{ fromJSON(steps.read_env_json.outputs.env_json).ssm_prefix }}" >> $GITHUB_OUTPUT + echo "cluster_name=${{ fromJSON(steps.read_env_json.outputs.env_json).fargate.cluster_name }}" >> $GITHUB_OUTPUT + + + - name: OIDC Auth to AWS + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-west-2 + role-to-assume: arn:aws:iam::${{ secrets.aws_user_account }}:role/ci-oidc-role + role-session-name: github-actions + role-duration-seconds: 900 + + # Based on https://developer.hashicorp.com/terraform/tutorials/automation/github-actions + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + + - name: Terraform Init + id: init + run: terraform init -backend-config=key=${{ steps.set_env_metadata.outputs.terraform_state_key }} + working-directory: deploy/${{ inputs.project }}/terraform + + - name: Terraform Format + id: fmt + run: terraform fmt -check + + - name: Terraform Validate + id: validate + run: terraform validate -no-color + working-directory: deploy/${{ inputs.project }}/terraform + + - name: Terraform Plan + id: plan + run: | + terraform plan -no-color -input=false -out terraform.tfplan \ + -var environment=${{ inputs.environment }} \ + -var account=${{ steps.set_env_metadata.outputs.account }} \ + -var region=${{ steps.set_env_metadata.outputs.region }} + working-directory: deploy/${{ inputs.project }}/terraform + + - name: Terraform Apply + id: apply + run: terraform apply -auto-approve -input=false terraform.tfplan + working-directory: deploy/${{ inputs.project }}/terraform \ No newline at end of file diff --git a/.github/workflows/terraform.yml b/.github/workflows/terraform.yml new file mode 100644 index 0000000..8780537 --- /dev/null +++ b/.github/workflows/terraform.yml @@ -0,0 +1,31 @@ +name: terraform-deploy +on: + workflow_dispatch: + inputs: + environment: + description: 'Environment' + required: true + default: 'dev' + type: choice + options: + - dev + - staging + - prod + project: + description: 'Project' + required: true + default: 'livepoll' + type: choice + options: + - livepoll + - pandoc +jobs: + terraform-deploy: + name: Terraform Deployment + uses: ./.github/workflows/terraform-deploy.yml + with: + environment: ${{ github.event.inputs.environment }} + project: ${{ github.event.inputs.project }} + secrets: + aws_user_account: ${{ secrets.AWS_USER_ACCOUNT }} + aws_ci_account: ${{ secrets.AWS_CI_ACCOUNT }} diff --git a/deploy/ecs_deploy.sh b/deploy/ecs_deploy.sh new file mode 100755 index 0000000..9c0cc92 --- /dev/null +++ b/deploy/ecs_deploy.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# +# This script: +# - deploys a docker image to an ECS environment +# +# Required environment variables: +# - GITHUB_WORKFLOW - provided by GHA +# - GITHUB_RUN_NUMBER - provided by GHA +# - GITHUB_RUN_ATTEMPT - provided by GHA +# - GITHUB_SHA - provided by GHA + +set -euox pipefail + +export AWS_ECR_IMAGE_TAG=gha-tree-$(git rev-parse HEAD:) +export VERSION_LABEL=gha-build-$GITHUB_WORKFLOW-$GITHUB_RUN_NUMBER-$GITHUB_RUN_ATTEMPT + +while [[ $# -gt 0 ]]; do + case $1 in + -c|--cluster-name) + ECS_CLUSTER_NAME="$2" + shift # past argument + shift # past value + ;; + -s|--ssm-prefix) + SSM_PREFIX="$2" + shift # past argument + shift # past value + ;; + -n|--account-number) + AWS_CI_ACCOUNT="$2" + shift # past argument + shift # past value + ;; + -p|--project-name) + PROJECT_NAME="$2" + shift + shift + ;; + -*|--*) + echo "Unknown option $1" + exit 1 + ;; + esac +done + +ECS_SERVICE_NAME="${ECS_CLUSTER_NAME}" +ECR_REPOSITORY="${PROJECT_NAME}" +CONTAINER_NAME="${PROJECT_NAME}" + +echo -e "\n============================================================" +echo " Beginning deployment to AWS Fargate environment: $ECS_CLUSTER_NAME $ECS_SERVICE_NAME" +echo -e "============================================================\n" + +# Deploy the latest image to Fargate for the webapp +docker run -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_SESSION_TOKEN \ + fabfuel/ecs-deploy:1.15.0 ecs deploy --region us-west-2 --no-deregister --timeout 600 --rollback \ + -i ${CONTAINER_NAME} $AWS_CI_ACCOUNT.dkr.ecr.us-west-2.amazonaws.com/$ECR_REPOSITORY:$AWS_ECR_IMAGE_TAG \ + -e ${CONTAINER_NAME} PARAMETER_STORE_PREFIX_LIST ${SSM_PREFIX} \ + -e ${CONTAINER_NAME} AWS_REGION us-west-2 \ + $ECS_CLUSTER_NAME $ECS_SERVICE_NAME diff --git a/deploy/livepoll-environments.json b/deploy/livepoll-environments.json new file mode 100644 index 0000000..ffb7ef0 --- /dev/null +++ b/deploy/livepoll-environments.json @@ -0,0 +1,43 @@ +{ + "environments": [ + { + "environment_label": "dev", + "account": 725843923591, + "region": "us-west-2", + "terraform_state_key": "accounts/lumen-stage/us-west-2/dev/livepoll-app/config/terraform.tfstate", + "ssm_prefix": "/lumen-services/us-west-2/dev/livepoll", + "fargate": { + "cluster_name": "dev-livepoll" + }, + "webapp": { + "ssm_prefix": "/lumen-services/us-west-2/dev/livepoll/webapp" + } + }, + { + "environment_label": "staging", + "account": 725843923591, + "region": "us-west-2", + "terraform_state_key": "accounts/lumen-stage/us-west-2/staging/livepoll-app/config/terraform.tfstate", + "ssm_prefix": "/lumen-services/us-west-2/staging/livepoll", + "fargate": { + "cluster_name": "staging-livepoll" + }, + "webapp": { + "ssm_prefix": "/lumen-services/us-west-2/staging/livepoll/webapp" + } + }, + { + "environment_label": "prod", + "account": 523740042085, + "region": "us-west-2", + "terraform_state_key": "accounts/lumen-prod/us-west-2/prod/livepoll-app/config/terraform.tfstate", + "ssm_prefix": "/lumen-services/us-west-2/prod/livepoll", + "fargate": { + "cluster_name": "prod-livepoll" + }, + "webapp": { + "ssm_prefix": "/lumen-services/us-west-2/prod/livepoll/webapp" + } + } + ] +} \ No newline at end of file diff --git a/deploy/pandoc-environments.json b/deploy/pandoc-environments.json new file mode 100644 index 0000000..5ee173b --- /dev/null +++ b/deploy/pandoc-environments.json @@ -0,0 +1,55 @@ +{ + "environments": [ + { + "environment_label": "dev", + "account": 725843923591, + "region": "us-west-2", + "terraform_state_key": "accounts/lumen-stage/us-west-2/dev/pandoc-app/config/terraform.tfstate", + "ssm_prefix": "/lumen-services/us-west-2/dev/pandoc", + "fargate": { + "cluster_name": "dev-pandoc" + }, + "webapp": { + "ssm_prefix": "/lumen-services/us-west-2/dev/pandoc/webapp" + }, + "cron": { + "cluster_name": "dev-pandoc", + "ssm_prefix": "/lumen-services/us-west-2/dev/pandoc/cron" + } + }, + { + "environment_label": "staging", + "account": 725843923591, + "region": "us-west-2", + "terraform_state_key": "accounts/lumen-stage/us-west-2/staging/pandoc-app/config/terraform.tfstate", + "ssm_prefix": "/lumen-services/us-west-2/staging/pandoc", + "fargate": { + "cluster_name": "staging-pandoc" + }, + "webapp": { + "ssm_prefix": "/lumen-services/us-west-2/staging/pandoc/webapp" + }, + "cron": { + "cluster_name": "staging-pandoc", + "ssm_prefix": "/lumen-services/us-west-2/staging/pandoc/cron" + } + }, + { + "environment_label": "prod", + "account": 523740042085, + "region": "us-west-2", + "terraform_state_key": "accounts/lumen-prod/us-west-2/prod/pandoc-app/config/terraform.tfstate", + "ssm_prefix": "/lumen-services/us-west-2/prod/pandoc", + "fargate": { + "cluster_name": "prod-pandoc" + }, + "webapp": { + "ssm_prefix": "/lumen-services/us-west-2/prod/pandoc/webapp" + }, + "cron": { + "cluster_name": "prod-pandoc", + "ssm_prefix": "/lumen-services/us-west-2/prod/pandoc/cron" + } + } + ] +} \ No newline at end of file diff --git a/deploy/terraform/livepoll/_locals.tf b/deploy/terraform/livepoll/_locals.tf new file mode 100644 index 0000000..42293c2 --- /dev/null +++ b/deploy/terraform/livepoll/_locals.tf @@ -0,0 +1,5 @@ +locals { + service_name = "livepoll" + parameter_store_prefix = "/lumen-services/${var.region}/${var.environment}/${local.service_name}" + parameter_store_prefix_webapp = "${local.parameter_store_prefix}/webapp" +} \ No newline at end of file diff --git a/deploy/terraform/livepoll/_provider.tf b/deploy/terraform/livepoll/_provider.tf new file mode 100644 index 0000000..037d21c --- /dev/null +++ b/deploy/terraform/livepoll/_provider.tf @@ -0,0 +1,25 @@ +provider "aws" { + region = var.region + + assume_role { + role_arn = "arn:aws:iam::${var.account}:role/ci-role" + } +} + +terraform { + required_version = ">= 1.0.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 4.0" + } + } + backend "s3" { + encrypt = true + bucket = "lumen-terraform-state" + dynamodb_table = "lumen-terraform-state-lock" + region = "us-west-2" + role_arn = "arn:aws:iam::824635284302:role/terraform" + } +} diff --git a/deploy/terraform/livepoll/parameter_store.tf b/deploy/terraform/livepoll/parameter_store.tf new file mode 100644 index 0000000..48a8820 --- /dev/null +++ b/deploy/terraform/livepoll/parameter_store.tf @@ -0,0 +1,23 @@ +resource "aws_ssm_parameter" "LIVEPOLL_USE_INSECURE_HTTP" { + name = "${local.parameter_store_prefix}/LIVEPOLL_USE_INSECURE_HTTP" + type = "SecureString" + value = "true" # initial value only, updates not handled in terraform + + tags = local.common_tags + + lifecycle { + ignore_changes = [value] + } +} + +resource "aws_ssm_parameter" "LIVEPOLL_PASSWORD" { + name = "${local.parameter_store_prefix}/LIVEPOLL_PASSWORD" + type = "SecureString" + value = "replace_me" # initial value only, updates not handled in terraform + + tags = local.common_tags + + lifecycle { + ignore_changes = [value] + } +} \ No newline at end of file diff --git a/deploy/terraform/livepoll/variables.tf b/deploy/terraform/livepoll/variables.tf new file mode 100644 index 0000000..e960b8d --- /dev/null +++ b/deploy/terraform/livepoll/variables.tf @@ -0,0 +1,11 @@ +variable "environment" { + type = string +} + +variable "region" { + type = string +} + +variable "account" { + type = string +} diff --git a/deploy/terraform/pandoc/_locals.tf b/deploy/terraform/pandoc/_locals.tf new file mode 100644 index 0000000..f57e1d2 --- /dev/null +++ b/deploy/terraform/pandoc/_locals.tf @@ -0,0 +1,6 @@ +locals { + service_name = "pandoc" + parameter_store_prefix = "/lumen-services/${var.region}/${var.environment}/${local.service_name}" + parameter_store_prefix_webapp = "${local.parameter_store_prefix}/webapp" + parameter_store_prefix_cron = "${local.parameter_store_prefix}/cron" +} \ No newline at end of file diff --git a/deploy/terraform/pandoc/_provider.tf b/deploy/terraform/pandoc/_provider.tf new file mode 100644 index 0000000..037d21c --- /dev/null +++ b/deploy/terraform/pandoc/_provider.tf @@ -0,0 +1,25 @@ +provider "aws" { + region = var.region + + assume_role { + role_arn = "arn:aws:iam::${var.account}:role/ci-role" + } +} + +terraform { + required_version = ">= 1.0.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 4.0" + } + } + backend "s3" { + encrypt = true + bucket = "lumen-terraform-state" + dynamodb_table = "lumen-terraform-state-lock" + region = "us-west-2" + role_arn = "arn:aws:iam::824635284302:role/terraform" + } +} diff --git a/deploy/terraform/pandoc/variables.tf b/deploy/terraform/pandoc/variables.tf new file mode 100644 index 0000000..e960b8d --- /dev/null +++ b/deploy/terraform/pandoc/variables.tf @@ -0,0 +1,11 @@ +variable "environment" { + type = string +} + +variable "region" { + type = string +} + +variable "account" { + type = string +} diff --git a/docker/Dockerfile-livepoll b/docker/Dockerfile-livepoll index 3788f3c..0e5b251 100644 --- a/docker/Dockerfile-livepoll +++ b/docker/Dockerfile-livepoll @@ -7,4 +7,3 @@ COPY livepoll/* /app/ RUN npm install CMD ["node", "/app/index.js"] - diff --git a/docker/Dockerfile-pandoc b/docker/Dockerfile-pandoc index 01d97a5..07b0065 100644 --- a/docker/Dockerfile-pandoc +++ b/docker/Dockerfile-pandoc @@ -1,4 +1,4 @@ -FROM php:8.2-apache +FROM php:8.2-apache AS pandoc RUN apt-get update \ && apt-get install -y --no-install-recommends --no-install-suggests \ @@ -14,3 +14,17 @@ RUN mkdir \ /var/www/datatmp \ /var/www/html/imgs +FROM debian:12.5 AS cron + +RUN apt-get update \ + && apt-get install -y --no-install-suggests --no-install-recommends \ + cron \ + && apt-get clean + +COPY pandoc/cleanup-old-files /etc/cron.daily + +RUN chmod 755 /etc/cron.daily/cleanup-old-files + +CMD ["cron", "-f"] + + \ No newline at end of file diff --git a/docker/Dockerfile-pandoc-cron b/docker/Dockerfile-pandoc-cron deleted file mode 100644 index 270b0ae..0000000 --- a/docker/Dockerfile-pandoc-cron +++ /dev/null @@ -1,13 +0,0 @@ -FROM debian:12.5 - -RUN apt-get update \ - && apt-get install -y --no-install-suggests --no-install-recommends \ - cron \ - && apt-get clean - -COPY pandoc/cleanup-old-files /etc/cron.daily - -RUN chmod 755 /etc/cron.daily/cleanup-old-files - -CMD ["cron", "-f"] - From d29e2e7f634cbbc2188487d3ab15478fabd865ed Mon Sep 17 00:00:00 2001 From: Jeff DeWan Date: Tue, 15 Oct 2024 14:15:28 -0700 Subject: [PATCH 02/26] OP-517: Need to add github action files to repository main branch --- .github/actions/private-ecr-login/action.yml | 39 ++++ .github/workflows/build-livepoll.yml | 230 +++++++++++++++++++ .github/workflows/manual-deploy.yml | 30 +++ .github/workflows/shared-deploy.yml | 143 ++++++++++++ .github/workflows/tag-deploy-livepoll.yml | 16 ++ .github/workflows/tag-deploy-pandoc.yml | 16 ++ .github/workflows/terraform-deploy.yml | 88 +++++++ .github/workflows/terraform.yml | 31 +++ 8 files changed, 593 insertions(+) create mode 100644 .github/actions/private-ecr-login/action.yml create mode 100644 .github/workflows/build-livepoll.yml create mode 100644 .github/workflows/manual-deploy.yml create mode 100644 .github/workflows/shared-deploy.yml create mode 100644 .github/workflows/tag-deploy-livepoll.yml create mode 100644 .github/workflows/tag-deploy-pandoc.yml create mode 100644 .github/workflows/terraform-deploy.yml create mode 100644 .github/workflows/terraform.yml diff --git a/.github/actions/private-ecr-login/action.yml b/.github/actions/private-ecr-login/action.yml new file mode 100644 index 0000000..774a645 --- /dev/null +++ b/.github/actions/private-ecr-login/action.yml @@ -0,0 +1,39 @@ +name: 'Log into Private ECR' +description: 'Github OIDC auth and assume role into account, then use AWS ECR Login action' +inputs: + aws_ci_account: + description: 'AWS Account ID for CI' + required: false + default: 824635284302 + aws_user_account: + description: 'AWS Account ID for users' + required: false + default: 265299512749 +outputs: + registry: + description: "ECR Registry" + value: ${{ steps.login-ecr.outputs.registry }} +runs: + using: "composite" + steps: + - name: assume oidc role + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-west-2 + role-to-assume: arn:aws:iam::${{ inputs.aws_user_account }}:role/ci-oidc-role + role-session-name: github-actions-oidc + role-duration-seconds: 900 + - name: assume target role + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }} + aws-session-token: ${{ env.AWS_SESSION_TOKEN }} + aws-region: us-west-2 + role-to-assume: arn:aws:iam::${{ inputs.aws_ci_account }}:role/ci-role + role-session-name: github-actions-private-ecr + role-duration-seconds: 900 + + - name: Login to Private ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 \ No newline at end of file diff --git a/.github/workflows/build-livepoll.yml b/.github/workflows/build-livepoll.yml new file mode 100644 index 0000000..8858f07 --- /dev/null +++ b/.github/workflows/build-livepoll.yml @@ -0,0 +1,230 @@ +name: build-and-push +env: + LIVEPOLL_ECR_REPOSITORY: livepoll + PANDOC_ECR_REPOSITORY: pandoc +on: + pull_request: + types: [ opened, synchronize, reopened ] + push: + branches: + - main + - rc/** + +jobs: + # this job gets a password for public ECR to use in future steps + ecr-public-auth: + runs-on: ubuntu-latest + name: Login to Public ECR + timeout-minutes: 5 + permissions: + id-token: write # Used for AWS OIDC auth + outputs: + ECR_PASSWORD: ${{ steps.ecr-auth.outputs.ECR_PASSWORD }} + steps: + # This is required to role chain from the OIDC role to other cross-account roles + # https://github.com/aws-actions/configure-aws-credentials/issues/279 + - name: OIDC Auth + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-west-2 + role-to-assume: arn:aws:iam::265299512749:role/ci-oidc-role + role-session-name: github-actions + role-duration-seconds: 900 + + - name: Assume role in CI Account + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }} + aws-session-token: ${{ env.AWS_SESSION_TOKEN }} + aws-region: us-west-2 + role-to-assume: arn:aws:iam::824635284302:role/ci-role + role-session-name: github-actions + role-duration-seconds: 1200 + + - name: Set Public ECR Auth + id: ecr-auth + run: echo "ECR_PASSWORD=$(aws ecr-public --region us-east-1 get-login-password)" >> $GITHUB_OUTPUT + + validate-terraform: + runs-on: ubuntu-latest + name: Validate Terraform + timeout-minutes: 5 + permissions: + id-token: write # Used for AWS OIDC auth + contents: read + actions: read + steps: + - uses: actions/checkout@v4 + name: Checkout project + + - name: OIDC Auth to AWS + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-west-2 + role-to-assume: arn:aws:iam::${{ secrets.aws_user_account }}:role/ci-oidc-role + role-session-name: github-actions + role-duration-seconds: 900 + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + + - name: Terraform Init + id: init + run: terraform init -backend=false + working-directory: deploy/terraform + + - name: Terraform Format + id: fmt + run: terraform fmt -check + + - name: Terraform Validate + id: validate + run: terraform validate -no-color + working-directory: deploy/terraform + + build-and-push-livepoll: + runs-on: ubuntu-latest + name: Build and Push Livepoll Docker Image + timeout-minutes: 30 + needs: [ecr-public-auth, validate-terraform] + permissions: + id-token: write # Used for AWS OIDC auth + contents: read + actions: read + steps: + - uses: actions/checkout@v4 + name: Checkout project + + - id: read_tree_hash + name: Read git tree hash + run: | + tree_hash=$(git rev-parse HEAD:) + echo "tree_hash=$tree_hash" >> $GITHUB_OUTPUT + + - id: set_branch_name + name: Read git branch name + run: | + branch_name=${GITHUB_REF##*/} + echo "branch_name=$branch_name" >> $GITHUB_OUTPUT + + - uses: ./.github/actions/private-ecr-login + name: Login to Private ECR + id: login-ecr + + - name: Login to Public ECR + uses: docker/login-action@v3 + with: + registry: public.ecr.aws/j7v6u1e0 + username: AWS + password: ${{ needs.ecr-public-auth.outputs.ECR_PASSWORD }} + + - name: Check for prebuilt image + id: prebuilt_check + run: | + docker manifest inspect ${{ steps.login-ecr.outputs.registry }}/${{ env.LIVEPOLL_ECR_REPOSITORY }}:gha-tree-${{ steps.read_tree_hash.outputs.tree_hash }} || echo "image_exists=$?" >> $GITHUB_OUTPUT + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + if: steps.prebuilt_check.outputs.image_exists != 0 + + - name: Docker build and push + uses: docker/build-push-action@v5 + if: steps.prebuilt_check.outputs.image_exists != 0 + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + with: + context: . + file: docker/Dockerfile-livepoll + platforms: linux/amd64 + push: true + tags: | + ${{ steps.login-ecr.outputs.registry }}/${{ env.LIVEPOLL_ECR_REPOSITORY }}:gha-commit-${{ github.sha }} + ${{ steps.login-ecr.outputs.registry }}/${{ env.LIVEPOLL_ECR_REPOSITORY }}:gha-tree-${{ steps.read_tree_hash.outputs.tree_hash }} + ${{ steps.login-ecr.outputs.registry }}/${{ env.LIVEPOLL_ECR_REPOSITORY }}:gha-build-${{ steps.set_branch_name.outputs.branch_name }}-${{ github.run_number }}-${{ github.run_attempt }} + ${{ steps.login-ecr.outputs.registry }}/${{ env.LIVEPOLL_ECR_REPOSITORY }}:gha-build-${{ steps.set_branch_name.outputs.branch_name }}-latest + cache-from: type=gha + cache-to: type=gha,mode=max + + build-and-push-pandoc: + runs-on: ubuntu-latest + name: Build and Push Pandoc Docker Image + timeout-minutes: 30 + needs: [ecr-public-auth, validate-terraform] + permissions: + id-token: write # Used for AWS OIDC auth + contents: read + actions: read + steps: + - uses: actions/checkout@v4 + name: Checkout project + + - id: read_tree_hash + name: Read git tree hash + run: | + tree_hash=$(git rev-parse HEAD:) + echo "tree_hash=$tree_hash" >> $GITHUB_OUTPUT + + - id: set_branch_name + name: Read git branch name + run: | + branch_name=${GITHUB_REF##*/} + echo "branch_name=$branch_name" >> $GITHUB_OUTPUT + + - uses: ./.github/actions/private-ecr-login + name: Login to Private ECR + id: login-ecr + + - name: Login to Public ECR + uses: docker/login-action@v3 + with: + registry: public.ecr.aws/j7v6u1e0 + username: AWS + password: ${{ needs.ecr-public-auth.outputs.ECR_PASSWORD }} + + - name: Check for prebuilt image + id: prebuilt_check + run: | + docker manifest inspect ${{ steps.login-ecr.outputs.registry }}/${{ env.PANDOC_ECR_REPOSITORY }}:gha-tree-${{ steps.read_tree_hash.outputs.tree_hash }} || echo "image_exists=$?" >> $GITHUB_OUTPUT + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + if: steps.prebuilt_check.outputs.image_exists != 0 + + - name: Docker build and push + uses: docker/build-push-action@v5 + if: steps.prebuilt_check.outputs.image_exists != 0 + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + with: + context: . + file: docker/Dockerfile-pandoc + target: pandoc + platforms: linux/amd64 + push: true + tags: | + ${{ steps.login-ecr.outputs.registry }}/${{ env.PANDOC_ECR_REPOSITORY }}:gha-commit-${{ github.sha }} + ${{ steps.login-ecr.outputs.registry }}/${{ env.PANDOC_ECR_REPOSITORY }}:gha-tree-${{ steps.read_tree_hash.outputs.tree_hash }} + ${{ steps.login-ecr.outputs.registry }}/${{ env.PANDOC_ECR_REPOSITORY }}:gha-build-${{ steps.set_branch_name.outputs.branch_name }}-${{ github.run_number }}-${{ github.run_attempt }} + ${{ steps.login-ecr.outputs.registry }}/${{ env.PANDOC_ECR_REPOSITORY }}:gha-build-${{ steps.set_branch_name.outputs.branch_name }}-latest + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Docker build and push + uses: docker/build-push-action@v5 + if: steps.prebuilt_check.outputs.image_exists != 0 + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + with: + context: . + file: docker/Dockerfile-pandoc + target: cron + platforms: linux/amd64 + push: true + tags: | + ${{ steps.login-ecr.outputs.registry }}/${{ env.PANDOC_ECR_REPOSITORY }}:gha-commit-${{ github.sha }}-cron + ${{ steps.login-ecr.outputs.registry }}/${{ env.PANDOC_ECR_REPOSITORY }}:gha-tree-${{ steps.read_tree_hash.outputs.tree_hash }}-cron + ${{ steps.login-ecr.outputs.registry }}/${{ env.PANDOC_ECR_REPOSITORY }}:gha-build-${{ steps.set_branch_name.outputs.branch_name }}-${{ github.run_number }}-${{ github.run_attempt }}-cron + ${{ steps.login-ecr.outputs.registry }}/${{ env.PANDOC_ECR_REPOSITORY }}:gha-build-${{ steps.set_branch_name.outputs.branch_name }}-latest-cron + cache-from: type=gha + cache-to: type=gha,mode=max \ No newline at end of file diff --git a/.github/workflows/manual-deploy.yml b/.github/workflows/manual-deploy.yml new file mode 100644 index 0000000..d0d5985 --- /dev/null +++ b/.github/workflows/manual-deploy.yml @@ -0,0 +1,30 @@ +name: manual-deploy +on: + workflow_dispatch: + inputs: + environment: + description: 'Environment' + required: true + default: 'dev' + type: choice + options: + - dev + - staging + project: + description: 'Project' + required: true + default: 'livepoll' + type: choice + options: + - livepoll + - pandoc +jobs: + ecs-deploy: + name: ECS Deployment + uses: ./.github/workflows/shared-deploy.yml + with: + environment: ${{ github.event.inputs.environment }} + project: ${{ github.event.inputs.project }} + secrets: + aws_user_account: ${{ secrets.AWS_USER_ACCOUNT }} + aws_ci_account: ${{ secrets.AWS_CI_ACCOUNT }} \ No newline at end of file diff --git a/.github/workflows/shared-deploy.yml b/.github/workflows/shared-deploy.yml new file mode 100644 index 0000000..ae877c6 --- /dev/null +++ b/.github/workflows/shared-deploy.yml @@ -0,0 +1,143 @@ +name: shared-deployment-workflow +permissions: + id-token: write # Used for AWS OIDC auth + contents: read # This is required for actions/checkout +on: + workflow_call: + inputs: + environment: + description: 'Environment name passed from the caller workflow' + required: true + type: string + project: + description: 'Project name passed from the caller workflow' + required: true + type: string + secrets: + aws_user_account: + description: 'AWS Account ID for IAM users' + required: true + aws_ci_account: + description: 'AWS Account ID for IAM users' + required: true + +jobs: + # this job gets a password for public ECR to use in future steps + ecr-public-auth: + runs-on: ubuntu-latest + name: Login to Public ECR + timeout-minutes: 5 + permissions: + id-token: write # Used for AWS OIDC auth + outputs: + ECR_PASSWORD: ${{ steps.ecr-auth.outputs.ECR_PASSWORD }} + steps: + # This is required to role chain from the OIDC role to other cross-account roles + # https://github.com/aws-actions/configure-aws-credentials/issues/279 + - name: OIDC Auth + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-west-2 + role-to-assume: arn:aws:iam::${{ secrets.aws_user_account }}:role/ci-oidc-role + role-session-name: github-actions + role-duration-seconds: 900 + + - name: Assume role in CI Account + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }} + aws-session-token: ${{ env.AWS_SESSION_TOKEN }} + aws-region: us-west-2 + role-to-assume: arn:aws:iam::${{ secrets.aws_ci_account }}:role/ci-role + role-session-name: github-actions + role-duration-seconds: 1200 + + - name: Set Public ECR Auth + id: ecr-auth + run: echo "ECR_PASSWORD=$(aws ecr-public --region us-east-1 get-login-password)" >> $GITHUB_OUTPUT + + terraform-deploy: + name: Terraform Deployment + uses: ./.github/workflows/terraform-deploy.yml + with: + environment: ${{ inputs.environment }} + project: ${{ inputs.project }} + secrets: + aws_user_account: ${{ secrets.AWS_USER_ACCOUNT }} + aws_ci_account: ${{ secrets.AWS_CI_ACCOUNT }} + + ecs-deploy: + name: ECS Deployment + timeout-minutes: 30 + runs-on: ubuntu-latest + needs: [ecr-public-auth,terraform-deploy] + steps: + - uses: actions/checkout@v4 + name: Checkout project + + - uses: ./.github/actions/private-ecr-login + name: Login to Private ECR + id: login-ecr + + - name: Login to Public ECR + uses: docker/login-action@v3 + with: + registry: public.ecr.aws/o9c0t3w1 + username: AWS + password: ${{ needs.ecr-public-auth.outputs.ECR_PASSWORD }} + + + - id: read_env_json + name: Read Environment JSON + run: | + env_json=$(jq -c '.environments[] | select(.environment_label=="${{ inputs.environment }}")' ./deploy/${{ inputs.project }}-environments.json) + echo "env_json=$env_json" >> $GITHUB_OUTPUT + - id: set_env_metadata + name: Set Environment Metadata + run: | + echo "account=${{ fromJSON(steps.read_env_json.outputs.env_json).account }}" >> $GITHUB_OUTPUT + echo "ssm_prefix=${{ fromJSON(steps.read_env_json.outputs.env_json).ssm_prefix }}" >> $GITHUB_OUTPUT + echo "webapp_ssm_prefix=${{ fromJSON(steps.read_env_json.outputs.env_json).webapp.ssm_prefix }}" >> $GITHUB_OUTPUT + echo "shoryuken_ssm_prefix=${{ fromJSON(steps.read_env_json.outputs.env_json).shoryuken.ssm_prefix }}" >> $GITHUB_OUTPUT + echo "cluster_name=${{ fromJSON(steps.read_env_json.outputs.env_json).fargate.cluster_name }}" >> $GITHUB_OUTPUT + + - name: OIDC Auth to AWS + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-west-2 + role-to-assume: arn:aws:iam::${{ secrets.aws_user_account }}:role/ci-oidc-role + role-session-name: github-actions + role-duration-seconds: 900 + + - name: Assume role in target account + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }} + aws-session-token: ${{ env.AWS_SESSION_TOKEN }} + aws-region: us-west-2 + role-to-assume: arn:aws:iam::${{ steps.set_env_metadata.outputs.account }}:role/ci-role + role-session-name: github-actions + role-duration-seconds: 1200 + + - id: read_tree_hash + name: Read git tree hash + run: | + tree_hash=$(git rev-parse HEAD:) + echo "tree_hash=$tree_hash" >> $GITHUB_OUTPUT + + - name: Verify image + run: | + if ! docker manifest inspect ${{ steps.login-ecr.outputs.registry }}/${{ inputs.project }}:gha-tree-${{ steps.read_tree_hash.outputs.tree_hash }}; then + echo "If this is a PR build, you may need to pull in changes from the target branch into your PR branch." + exit 1 + fi + + - name: Run deploy script + run: | + deploy/ecs_deploy.sh \ + --cluster-name ${{ steps.set_env_metadata.outputs.cluster_name }} \ + --ssm-prefix ${{ steps.set_env_metadata.outputs.ssm_prefix }} \ + --account-number ${{ secrets.aws_ci_account }} \ + --project-name ${{ inputs.project }} \ No newline at end of file diff --git a/.github/workflows/tag-deploy-livepoll.yml b/.github/workflows/tag-deploy-livepoll.yml new file mode 100644 index 0000000..12f8351 --- /dev/null +++ b/.github/workflows/tag-deploy-livepoll.yml @@ -0,0 +1,16 @@ +name: tag-deploy +on: + push: + tags: + - release/livepoll/* + +jobs: + ecs-deploy: + name: ECS Deployment + uses: ./.github/workflows/shared-deploy.yml + with: + environment: prod + project: livepoll + secrets: + aws_user_account: ${{ secrets.AWS_USER_ACCOUNT }} + aws_ci_account: ${{ secrets.AWS_CI_ACCOUNT }} \ No newline at end of file diff --git a/.github/workflows/tag-deploy-pandoc.yml b/.github/workflows/tag-deploy-pandoc.yml new file mode 100644 index 0000000..cd0ba9d --- /dev/null +++ b/.github/workflows/tag-deploy-pandoc.yml @@ -0,0 +1,16 @@ +name: tag-deploy +on: + push: + tags: + - release/pandoc/* + +jobs: + ecs-deploy: + name: ECS Deployment + uses: ./.github/workflows/shared-deploy.yml + with: + environment: prod + project: pandoc + secrets: + aws_user_account: ${{ secrets.AWS_USER_ACCOUNT }} + aws_ci_account: ${{ secrets.AWS_CI_ACCOUNT }} \ No newline at end of file diff --git a/.github/workflows/terraform-deploy.yml b/.github/workflows/terraform-deploy.yml new file mode 100644 index 0000000..ad2c09b --- /dev/null +++ b/.github/workflows/terraform-deploy.yml @@ -0,0 +1,88 @@ +name: terraform-deployment-workflow +permissions: + id-token: write # Used for AWS OIDC auth + contents: read # This is required for actions/checkout +on: + workflow_call: + inputs: + environment: + description: 'Environment name passed from the caller workflow' + required: true + type: string + project: + description: 'Project name passed from the caller workflow' + required: true + type: string + secrets: + aws_user_account: + description: 'AWS Account ID for IAM users' + required: true + aws_ci_account: + description: 'AWS Account ID for IAM users' + required: true + +jobs: + # this job gets a password for public ECR to use in future steps + terraform-deploy: + name: Terraform Deployment + timeout-minutes: 30 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + name: Checkout project + + - id: read_env_json + name: Read Environment JSON + run: | + env_json=$(jq -c '.environments[] | select(.environment_label=="${{ inputs.environment }}")' ./deploy/${{ inputs.project }}-environments.json) + echo "env_json=$env_json" >> $GITHUB_OUTPUT + + - id: set_env_metadata + name: Set Environment Metadata + run: | + echo "account=${{ fromJSON(steps.read_env_json.outputs.env_json).account }}" >> $GITHUB_OUTPUT + echo "region=${{ fromJSON(steps.read_env_json.outputs.env_json).region }}" >> $GITHUB_OUTPUT + echo "terraform_state_key=${{ fromJSON(steps.read_env_json.outputs.env_json).terraform_state_key }}" >> $GITHUB_OUTPUT + echo "ssm_prefix=${{ fromJSON(steps.read_env_json.outputs.env_json).ssm_prefix }}" >> $GITHUB_OUTPUT + echo "cluster_name=${{ fromJSON(steps.read_env_json.outputs.env_json).fargate.cluster_name }}" >> $GITHUB_OUTPUT + + + - name: OIDC Auth to AWS + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-west-2 + role-to-assume: arn:aws:iam::${{ secrets.aws_user_account }}:role/ci-oidc-role + role-session-name: github-actions + role-duration-seconds: 900 + + # Based on https://developer.hashicorp.com/terraform/tutorials/automation/github-actions + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + + - name: Terraform Init + id: init + run: terraform init -backend-config=key=${{ steps.set_env_metadata.outputs.terraform_state_key }} + working-directory: deploy/${{ inputs.project }}/terraform + + - name: Terraform Format + id: fmt + run: terraform fmt -check + + - name: Terraform Validate + id: validate + run: terraform validate -no-color + working-directory: deploy/${{ inputs.project }}/terraform + + - name: Terraform Plan + id: plan + run: | + terraform plan -no-color -input=false -out terraform.tfplan \ + -var environment=${{ inputs.environment }} \ + -var account=${{ steps.set_env_metadata.outputs.account }} \ + -var region=${{ steps.set_env_metadata.outputs.region }} + working-directory: deploy/${{ inputs.project }}/terraform + + - name: Terraform Apply + id: apply + run: terraform apply -auto-approve -input=false terraform.tfplan + working-directory: deploy/${{ inputs.project }}/terraform \ No newline at end of file diff --git a/.github/workflows/terraform.yml b/.github/workflows/terraform.yml new file mode 100644 index 0000000..8780537 --- /dev/null +++ b/.github/workflows/terraform.yml @@ -0,0 +1,31 @@ +name: terraform-deploy +on: + workflow_dispatch: + inputs: + environment: + description: 'Environment' + required: true + default: 'dev' + type: choice + options: + - dev + - staging + - prod + project: + description: 'Project' + required: true + default: 'livepoll' + type: choice + options: + - livepoll + - pandoc +jobs: + terraform-deploy: + name: Terraform Deployment + uses: ./.github/workflows/terraform-deploy.yml + with: + environment: ${{ github.event.inputs.environment }} + project: ${{ github.event.inputs.project }} + secrets: + aws_user_account: ${{ secrets.AWS_USER_ACCOUNT }} + aws_ci_account: ${{ secrets.AWS_CI_ACCOUNT }} From 2f70d36069fd41a2cc0fbe9c9fd50638cf377d4a Mon Sep 17 00:00:00 2001 From: Jeff DeWan Date: Tue, 15 Oct 2024 14:25:27 -0700 Subject: [PATCH 03/26] Rename --- ...{build-livepoll.yml => build-and-push.yml} | 35 ------------------- 1 file changed, 35 deletions(-) rename .github/workflows/{build-livepoll.yml => build-and-push.yml} (83%) diff --git a/.github/workflows/build-livepoll.yml b/.github/workflows/build-and-push.yml similarity index 83% rename from .github/workflows/build-livepoll.yml rename to .github/workflows/build-and-push.yml index 8858f07..d6dec4d 100644 --- a/.github/workflows/build-livepoll.yml +++ b/.github/workflows/build-and-push.yml @@ -11,41 +11,6 @@ on: - rc/** jobs: - # this job gets a password for public ECR to use in future steps - ecr-public-auth: - runs-on: ubuntu-latest - name: Login to Public ECR - timeout-minutes: 5 - permissions: - id-token: write # Used for AWS OIDC auth - outputs: - ECR_PASSWORD: ${{ steps.ecr-auth.outputs.ECR_PASSWORD }} - steps: - # This is required to role chain from the OIDC role to other cross-account roles - # https://github.com/aws-actions/configure-aws-credentials/issues/279 - - name: OIDC Auth - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-region: us-west-2 - role-to-assume: arn:aws:iam::265299512749:role/ci-oidc-role - role-session-name: github-actions - role-duration-seconds: 900 - - - name: Assume role in CI Account - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }} - aws-session-token: ${{ env.AWS_SESSION_TOKEN }} - aws-region: us-west-2 - role-to-assume: arn:aws:iam::824635284302:role/ci-role - role-session-name: github-actions - role-duration-seconds: 1200 - - - name: Set Public ECR Auth - id: ecr-auth - run: echo "ECR_PASSWORD=$(aws ecr-public --region us-east-1 get-login-password)" >> $GITHUB_OUTPUT - validate-terraform: runs-on: ubuntu-latest name: Validate Terraform From 95c1d188447c68327a70a6918e7af4289ba6acb9 Mon Sep 17 00:00:00 2001 From: Jeff DeWan Date: Tue, 15 Oct 2024 14:26:06 -0700 Subject: [PATCH 04/26] Remove ECR public --- .github/workflows/build-and-push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-push.yml b/.github/workflows/build-and-push.yml index d6dec4d..b990b1a 100644 --- a/.github/workflows/build-and-push.yml +++ b/.github/workflows/build-and-push.yml @@ -52,7 +52,7 @@ jobs: runs-on: ubuntu-latest name: Build and Push Livepoll Docker Image timeout-minutes: 30 - needs: [ecr-public-auth, validate-terraform] + needs: [validate-terraform] permissions: id-token: write # Used for AWS OIDC auth contents: read From 6a6826d6fdfb5eb371f652b2377cce55950f3b95 Mon Sep 17 00:00:00 2001 From: Jeff DeWan Date: Tue, 15 Oct 2024 14:26:30 -0700 Subject: [PATCH 05/26] Remove ECR public --- .github/workflows/build-and-push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-push.yml b/.github/workflows/build-and-push.yml index b990b1a..5898c98 100644 --- a/.github/workflows/build-and-push.yml +++ b/.github/workflows/build-and-push.yml @@ -115,7 +115,7 @@ jobs: runs-on: ubuntu-latest name: Build and Push Pandoc Docker Image timeout-minutes: 30 - needs: [ecr-public-auth, validate-terraform] + needs: [validate-terraform] permissions: id-token: write # Used for AWS OIDC auth contents: read From e183b92656a9312ccbceabfc3312b7d8367d43b8 Mon Sep 17 00:00:00 2001 From: Jeff DeWan Date: Tue, 15 Oct 2024 14:30:10 -0700 Subject: [PATCH 06/26] Remove ECR public --- .github/workflows/build-and-push.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/.github/workflows/build-and-push.yml b/.github/workflows/build-and-push.yml index 5898c98..4d694c9 100644 --- a/.github/workflows/build-and-push.yml +++ b/.github/workflows/build-and-push.yml @@ -77,13 +77,6 @@ jobs: name: Login to Private ECR id: login-ecr - - name: Login to Public ECR - uses: docker/login-action@v3 - with: - registry: public.ecr.aws/j7v6u1e0 - username: AWS - password: ${{ needs.ecr-public-auth.outputs.ECR_PASSWORD }} - - name: Check for prebuilt image id: prebuilt_check run: | @@ -140,13 +133,6 @@ jobs: name: Login to Private ECR id: login-ecr - - name: Login to Public ECR - uses: docker/login-action@v3 - with: - registry: public.ecr.aws/j7v6u1e0 - username: AWS - password: ${{ needs.ecr-public-auth.outputs.ECR_PASSWORD }} - - name: Check for prebuilt image id: prebuilt_check run: | From 576a43d31ee0baf3c47e4c09ef39974dbf096215 Mon Sep 17 00:00:00 2001 From: Jeff DeWan Date: Tue, 15 Oct 2024 14:33:46 -0700 Subject: [PATCH 07/26] Fix project path --- .github/workflows/terraform-deploy.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/terraform-deploy.yml b/.github/workflows/terraform-deploy.yml index ad2c09b..8747e14 100644 --- a/.github/workflows/terraform-deploy.yml +++ b/.github/workflows/terraform-deploy.yml @@ -62,16 +62,17 @@ jobs: - name: Terraform Init id: init run: terraform init -backend-config=key=${{ steps.set_env_metadata.outputs.terraform_state_key }} - working-directory: deploy/${{ inputs.project }}/terraform + working-directory: deploy/terraform/${{ inputs.project }} - name: Terraform Format id: fmt run: terraform fmt -check + working-directory: deploy/terraform/${{ inputs.project }} - name: Terraform Validate id: validate run: terraform validate -no-color - working-directory: deploy/${{ inputs.project }}/terraform + working-directory: deploy/terraform/${{ inputs.project }} - name: Terraform Plan id: plan @@ -80,9 +81,9 @@ jobs: -var environment=${{ inputs.environment }} \ -var account=${{ steps.set_env_metadata.outputs.account }} \ -var region=${{ steps.set_env_metadata.outputs.region }} - working-directory: deploy/${{ inputs.project }}/terraform + working-directory: deploy/terraform/${{ inputs.project }} - name: Terraform Apply id: apply run: terraform apply -auto-approve -input=false terraform.tfplan - working-directory: deploy/${{ inputs.project }}/terraform \ No newline at end of file + working-directory: deploy/terraform/${{ inputs.project }} \ No newline at end of file From 8765826968e31f3e8983c7622d204d69cf1478e9 Mon Sep 17 00:00:00 2001 From: Jeff DeWan Date: Tue, 15 Oct 2024 14:38:52 -0700 Subject: [PATCH 08/26] fmt --- deploy/terraform/livepoll/_locals.tf | 4 ++-- deploy/terraform/pandoc/_locals.tf | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/deploy/terraform/livepoll/_locals.tf b/deploy/terraform/livepoll/_locals.tf index 42293c2..d80b489 100644 --- a/deploy/terraform/livepoll/_locals.tf +++ b/deploy/terraform/livepoll/_locals.tf @@ -1,5 +1,5 @@ locals { - service_name = "livepoll" - parameter_store_prefix = "/lumen-services/${var.region}/${var.environment}/${local.service_name}" + service_name = "livepoll" + parameter_store_prefix = "/lumen-services/${var.region}/${var.environment}/${local.service_name}" parameter_store_prefix_webapp = "${local.parameter_store_prefix}/webapp" } \ No newline at end of file diff --git a/deploy/terraform/pandoc/_locals.tf b/deploy/terraform/pandoc/_locals.tf index f57e1d2..eb86aee 100644 --- a/deploy/terraform/pandoc/_locals.tf +++ b/deploy/terraform/pandoc/_locals.tf @@ -1,6 +1,6 @@ locals { - service_name = "pandoc" - parameter_store_prefix = "/lumen-services/${var.region}/${var.environment}/${local.service_name}" + service_name = "pandoc" + parameter_store_prefix = "/lumen-services/${var.region}/${var.environment}/${local.service_name}" parameter_store_prefix_webapp = "${local.parameter_store_prefix}/webapp" - parameter_store_prefix_cron = "${local.parameter_store_prefix}/cron" + parameter_store_prefix_cron = "${local.parameter_store_prefix}/cron" } \ No newline at end of file From 038a133d621162075a35c5ee49cb0b4b8d82b718 Mon Sep 17 00:00:00 2001 From: Jeff DeWan Date: Tue, 15 Oct 2024 14:41:18 -0700 Subject: [PATCH 09/26] fix terraform --- .github/workflows/build-and-push.yml | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-and-push.yml b/.github/workflows/build-and-push.yml index 4d694c9..53c1e75 100644 --- a/.github/workflows/build-and-push.yml +++ b/.github/workflows/build-and-push.yml @@ -34,19 +34,30 @@ jobs: - name: Setup Terraform uses: hashicorp/setup-terraform@v3 - - name: Terraform Init - id: init + - name: Livepoll Terraform Init run: terraform init -backend=false - working-directory: deploy/terraform + working-directory: deploy/terraform/livepoll - - name: Terraform Format - id: fmt + - name: Livepoll Terraform Format run: terraform fmt -check + working-directory: deploy/terraform/livepoll - - name: Terraform Validate + - name: Livepoll Terraform Validate id: validate run: terraform validate -no-color - working-directory: deploy/terraform + working-directory: deploy/terraform/livepoll + + - name: Pandoc Terraform Init + run: terraform init -backend=false + working-directory: deploy/terraform/pandoc + + - name: Pandoc Terraform Format + run: terraform fmt -check + working-directory: deploy/terraform/pandoc + + - name: Pandoc Terraform Validate + run: terraform validate -no-color + working-directory: deploy/terraform/pandoc build-and-push-livepoll: runs-on: ubuntu-latest From b824090bd4864e49b3e2612d59615af17f1b3606 Mon Sep 17 00:00:00 2001 From: Jeff DeWan Date: Tue, 15 Oct 2024 14:43:15 -0700 Subject: [PATCH 10/26] fix terraform --- deploy/terraform/livepoll/_provider.tf | 9 +++++++++ deploy/terraform/livepoll/parameter_store.tf | 4 ---- deploy/terraform/pandoc/_provider.tf | 9 +++++++++ 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/deploy/terraform/livepoll/_provider.tf b/deploy/terraform/livepoll/_provider.tf index 037d21c..63e3723 100644 --- a/deploy/terraform/livepoll/_provider.tf +++ b/deploy/terraform/livepoll/_provider.tf @@ -4,6 +4,15 @@ provider "aws" { assume_role { role_arn = "arn:aws:iam::${var.account}:role/ci-role" } + + default_tags { + tags = { + Environment = var.environment + Terraform = "true" + Terraform_Project = local.service_name + Service = local.service_name + } + } } terraform { diff --git a/deploy/terraform/livepoll/parameter_store.tf b/deploy/terraform/livepoll/parameter_store.tf index 48a8820..8f8456c 100644 --- a/deploy/terraform/livepoll/parameter_store.tf +++ b/deploy/terraform/livepoll/parameter_store.tf @@ -3,8 +3,6 @@ resource "aws_ssm_parameter" "LIVEPOLL_USE_INSECURE_HTTP" { type = "SecureString" value = "true" # initial value only, updates not handled in terraform - tags = local.common_tags - lifecycle { ignore_changes = [value] } @@ -15,8 +13,6 @@ resource "aws_ssm_parameter" "LIVEPOLL_PASSWORD" { type = "SecureString" value = "replace_me" # initial value only, updates not handled in terraform - tags = local.common_tags - lifecycle { ignore_changes = [value] } diff --git a/deploy/terraform/pandoc/_provider.tf b/deploy/terraform/pandoc/_provider.tf index 037d21c..63e3723 100644 --- a/deploy/terraform/pandoc/_provider.tf +++ b/deploy/terraform/pandoc/_provider.tf @@ -4,6 +4,15 @@ provider "aws" { assume_role { role_arn = "arn:aws:iam::${var.account}:role/ci-role" } + + default_tags { + tags = { + Environment = var.environment + Terraform = "true" + Terraform_Project = local.service_name + Service = local.service_name + } + } } terraform { From 5f566c5135fc6e79d17d3c7355abdfdbfab4c835 Mon Sep 17 00:00:00 2001 From: Jeff DeWan Date: Tue, 15 Oct 2024 14:46:30 -0700 Subject: [PATCH 11/26] Remove public ecr --- .github/workflows/shared-deploy.yml | 45 +---------------------------- 1 file changed, 1 insertion(+), 44 deletions(-) diff --git a/.github/workflows/shared-deploy.yml b/.github/workflows/shared-deploy.yml index ae877c6..770bf3b 100644 --- a/.github/workflows/shared-deploy.yml +++ b/.github/workflows/shared-deploy.yml @@ -22,41 +22,6 @@ on: required: true jobs: - # this job gets a password for public ECR to use in future steps - ecr-public-auth: - runs-on: ubuntu-latest - name: Login to Public ECR - timeout-minutes: 5 - permissions: - id-token: write # Used for AWS OIDC auth - outputs: - ECR_PASSWORD: ${{ steps.ecr-auth.outputs.ECR_PASSWORD }} - steps: - # This is required to role chain from the OIDC role to other cross-account roles - # https://github.com/aws-actions/configure-aws-credentials/issues/279 - - name: OIDC Auth - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-region: us-west-2 - role-to-assume: arn:aws:iam::${{ secrets.aws_user_account }}:role/ci-oidc-role - role-session-name: github-actions - role-duration-seconds: 900 - - - name: Assume role in CI Account - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }} - aws-session-token: ${{ env.AWS_SESSION_TOKEN }} - aws-region: us-west-2 - role-to-assume: arn:aws:iam::${{ secrets.aws_ci_account }}:role/ci-role - role-session-name: github-actions - role-duration-seconds: 1200 - - - name: Set Public ECR Auth - id: ecr-auth - run: echo "ECR_PASSWORD=$(aws ecr-public --region us-east-1 get-login-password)" >> $GITHUB_OUTPUT - terraform-deploy: name: Terraform Deployment uses: ./.github/workflows/terraform-deploy.yml @@ -71,7 +36,7 @@ jobs: name: ECS Deployment timeout-minutes: 30 runs-on: ubuntu-latest - needs: [ecr-public-auth,terraform-deploy] + needs: [terraform-deploy] steps: - uses: actions/checkout@v4 name: Checkout project @@ -79,14 +44,6 @@ jobs: - uses: ./.github/actions/private-ecr-login name: Login to Private ECR id: login-ecr - - - name: Login to Public ECR - uses: docker/login-action@v3 - with: - registry: public.ecr.aws/o9c0t3w1 - username: AWS - password: ${{ needs.ecr-public-auth.outputs.ECR_PASSWORD }} - - id: read_env_json name: Read Environment JSON From e429f8b09e580b1154e9fabff2546b4e2de8dacb Mon Sep 17 00:00:00 2001 From: Jeff DeWan Date: Tue, 15 Oct 2024 14:48:33 -0700 Subject: [PATCH 12/26] Add docker entrypoint --- docker/Dockerfile-livepoll | 4 +++ docker/Dockerfile-pandoc | 4 +++ docker/docker-entrypoint.sh | 54 +++++++++++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+) create mode 100755 docker/docker-entrypoint.sh diff --git a/docker/Dockerfile-livepoll b/docker/Dockerfile-livepoll index 0e5b251..17b8c36 100644 --- a/docker/Dockerfile-livepoll +++ b/docker/Dockerfile-livepoll @@ -6,4 +6,8 @@ COPY livepoll/* /app/ RUN npm install +COPY docker/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh + +ENTRYPOINT [ "/usr/bin/docker-entrypoint.sh" ] + CMD ["node", "/app/index.js"] diff --git a/docker/Dockerfile-pandoc b/docker/Dockerfile-pandoc index 07b0065..5657f4a 100644 --- a/docker/Dockerfile-pandoc +++ b/docker/Dockerfile-pandoc @@ -14,6 +14,10 @@ RUN mkdir \ /var/www/datatmp \ /var/www/html/imgs +COPY docker/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh + +ENTRYPOINT [ "/usr/bin/docker-entrypoint.sh" ] + FROM debian:12.5 AS cron RUN apt-get update \ diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh new file mode 100755 index 0000000..b943458 --- /dev/null +++ b/docker/docker-entrypoint.sh @@ -0,0 +1,54 @@ +#!/bin/bash +set -eo pipefail + +# Required environment variables: +# - PARAMETER_STORE_PREFIX_LIST: Comma-separated list of paths in SSM Parameter Store from which environment variables are pulled. +# If items from multiple paths conflict, the later one is used. + +# https://www.geekcafe.com/blog/how-to-load-parameter-store-values-into-an-ec2s-environment-variables +setup_parameter_store() { + PREFIX=$1 + + parameters=$(aws ssm get-parameters-by-path \ + --path "${PREFIX}" \ + --region ${AWS_REGION} \ + --with-decrypt \ + ) + success=$? + + if [ $success -eq 0 ]; then + # bail if no parameters found + param_count=$(echo "$parameters" | jq '.Parameters | length') + if [ $param_count -eq 0 ]; then + echo "No parameters found in '${PREFIX}!'" + exit 1 + fi + + # split("/")[-1] will grab the last token as the "name" for the env variable + export_statement=$(echo "$parameters" \ + | jq -r '.Parameters[] | "echo \"Pulling from " + .Name + "\"; export " + (.Name | split("/")[-1]) + "=\"" + .Value + "\""' \ + ) + sorted=$(echo "$export_statement" | sort) + + # eval the export strings + eval "$sorted" + else + echo "Failed to retrieve AWS SSM Parameters from '${PREFIX}!'" + exit 1 + fi +} + +if [ ! -z ${PARAMETER_STORE_PREFIX_LIST} ]; then + prefixes=$(echo $PARAMETER_STORE_PREFIX_LIST | tr "," "\n") + + for prefix in $prefixes + do + setup_parameter_store $prefix + done + +else + echo "PARAMETER_STORE_PREFIX_LIST is not set, skipping parameter store setup." +fi + +# Then exec the container's main process (what's set as CMD in the Dockerfile). +exec "$@" \ No newline at end of file From b629b3a3f018bfd384ca5fb6af3344e5c5c74085 Mon Sep 17 00:00:00 2001 From: Jeff DeWan Date: Tue, 15 Oct 2024 14:54:26 -0700 Subject: [PATCH 13/26] Install awscli and jq --- docker/Dockerfile-livepoll | 6 ++++++ docker/Dockerfile-pandoc | 2 ++ 2 files changed, 8 insertions(+) diff --git a/docker/Dockerfile-livepoll b/docker/Dockerfile-livepoll index 17b8c36..de61c4c 100644 --- a/docker/Dockerfile-livepoll +++ b/docker/Dockerfile-livepoll @@ -1,5 +1,11 @@ FROM node:22 +RUN apt-get update \ + && apt-get install -y --no-install-recommends --no-install-suggests \ + jq \ + awscli \ + && apt-get clean + WORKDIR /app COPY livepoll/* /app/ diff --git a/docker/Dockerfile-pandoc b/docker/Dockerfile-pandoc index 5657f4a..c1bd304 100644 --- a/docker/Dockerfile-pandoc +++ b/docker/Dockerfile-pandoc @@ -3,6 +3,8 @@ FROM php:8.2-apache AS pandoc RUN apt-get update \ && apt-get install -y --no-install-recommends --no-install-suggests \ pandoc \ + jq \ + awscli \ && apt-get clean COPY pandoc/* /var/www/html/ From a2ff1f5d2082ea98889c35ea4e251d9167cdf5ea Mon Sep 17 00:00:00 2001 From: Jeff DeWan Date: Wed, 16 Oct 2024 17:04:37 -0700 Subject: [PATCH 14/26] Split entrypoint and fix pandoc --- docker/Dockerfile-livepoll | 2 +- docker/Dockerfile-pandoc | 4 +- ...r-entrypoint.sh => livepoll-entrypoint.sh} | 0 docker/pandoc-entrypoint.sh | 55 +++++++++++++++++++ 4 files changed, 59 insertions(+), 2 deletions(-) rename docker/{docker-entrypoint.sh => livepoll-entrypoint.sh} (100%) create mode 100755 docker/pandoc-entrypoint.sh diff --git a/docker/Dockerfile-livepoll b/docker/Dockerfile-livepoll index de61c4c..deeb460 100644 --- a/docker/Dockerfile-livepoll +++ b/docker/Dockerfile-livepoll @@ -12,7 +12,7 @@ COPY livepoll/* /app/ RUN npm install -COPY docker/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh +COPY docker/livepoll-entrypoint.sh /usr/bin/docker-entrypoint.sh ENTRYPOINT [ "/usr/bin/docker-entrypoint.sh" ] diff --git a/docker/Dockerfile-pandoc b/docker/Dockerfile-pandoc index c1bd304..ecd84a2 100644 --- a/docker/Dockerfile-pandoc +++ b/docker/Dockerfile-pandoc @@ -16,10 +16,12 @@ RUN mkdir \ /var/www/datatmp \ /var/www/html/imgs -COPY docker/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh +COPY docker/pandoc-entrypoint.sh /usr/bin/docker-entrypoint.sh ENTRYPOINT [ "/usr/bin/docker-entrypoint.sh" ] +CMD [ "apache2-foreground" ] + FROM debian:12.5 AS cron RUN apt-get update \ diff --git a/docker/docker-entrypoint.sh b/docker/livepoll-entrypoint.sh similarity index 100% rename from docker/docker-entrypoint.sh rename to docker/livepoll-entrypoint.sh diff --git a/docker/pandoc-entrypoint.sh b/docker/pandoc-entrypoint.sh new file mode 100755 index 0000000..3fea76d --- /dev/null +++ b/docker/pandoc-entrypoint.sh @@ -0,0 +1,55 @@ +#!/bin/bash +set -eo pipefail + +# Required environment variables: +# - PARAMETER_STORE_PREFIX_LIST: Comma-separated list of paths in SSM Parameter Store from which environment variables are pulled. +# If items from multiple paths conflict, the later one is used. + +# https://www.geekcafe.com/blog/how-to-load-parameter-store-values-into-an-ec2s-environment-variables +setup_parameter_store() { + PREFIX=$1 + + parameters=$(aws ssm get-parameters-by-path \ + --path "${PREFIX}" \ + --region ${AWS_REGION} \ + --with-decrypt \ + ) + success=$? + + if [ $success -eq 0 ]; then + # bail if no parameters found + param_count=$(echo "$parameters" | jq '.Parameters | length') + if [ $param_count -eq 0 ]; then + echo "No parameters found in '${PREFIX}!'" + exit 1 + fi + + # split("/")[-1] will grab the last token as the "name" for the env variable + export_statement=$(echo "$parameters" \ + | jq -r '.Parameters[] | "echo \"Pulling from " + .Name + "\"; export " + (.Name | split("/")[-1]) + "=\"" + .Value + "\""' \ + ) + sorted=$(echo "$export_statement" | sort) + + # eval the export strings + eval "$sorted" + else + echo "Failed to retrieve AWS SSM Parameters from '${PREFIX}!'" + exit 1 + fi +} + +if [ ! -z ${PARAMETER_STORE_PREFIX_LIST} ]; then + prefixes=$(echo $PARAMETER_STORE_PREFIX_LIST | tr "," "\n") + + for prefix in $prefixes + do + setup_parameter_store $prefix + done + +else + echo "PARAMETER_STORE_PREFIX_LIST is not set, skipping parameter store setup." +fi + +# Then exec the upstream entrypoint with what's set as CMD in the Dockerfile +# https://github.com/docker-library/php/blob/9c50f6a789b0337345934eee0d46e74d1e6dc8e2/8.2/bookworm/apache/Dockerfile +exec /usr/local/bin/docker-entrypoint.sh "$@" \ No newline at end of file From 7cb28ba22f711b8a87f47263e267b202d6d1b61c Mon Sep 17 00:00:00 2001 From: Jeff DeWan Date: Wed, 16 Oct 2024 17:16:01 -0700 Subject: [PATCH 15/26] Fix entrypoint --- docker/pandoc-entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/pandoc-entrypoint.sh b/docker/pandoc-entrypoint.sh index 3fea76d..e78e9d7 100755 --- a/docker/pandoc-entrypoint.sh +++ b/docker/pandoc-entrypoint.sh @@ -52,4 +52,4 @@ fi # Then exec the upstream entrypoint with what's set as CMD in the Dockerfile # https://github.com/docker-library/php/blob/9c50f6a789b0337345934eee0d46e74d1e6dc8e2/8.2/bookworm/apache/Dockerfile -exec /usr/local/bin/docker-entrypoint.sh "$@" \ No newline at end of file +exec /usr/local/bin/docker-php-entrypoint "$@" \ No newline at end of file From 78bc3b4202bf56ae153da163e37bb6911e17c16b Mon Sep 17 00:00:00 2001 From: Jeff DeWan Date: Wed, 16 Oct 2024 17:52:11 -0700 Subject: [PATCH 16/26] Add health.html, update docker-compose --- docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6e36b23..f2208db 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,11 +1,10 @@ -version: '3' - services: pandoc: build: context: ./ dockerfile: docker/Dockerfile-pandoc + target: pandoc expose: - 80 ports: @@ -28,7 +27,8 @@ services: pandoc-cron: build: context: ./ - dockerfile: docker/Dockerfile-pandoc-cron + dockerfile: docker/Dockerfile-pandoc + target: cron volumes: - pandoc-temp-datatmp:/var/www/datatmp From bf1665a4c817fce936fc0eae89997d38a3b82a47 Mon Sep 17 00:00:00 2001 From: Jeff DeWan Date: Wed, 16 Oct 2024 18:04:02 -0700 Subject: [PATCH 17/26] Add health.html, update docker-compose --- pandoc/health.html | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 pandoc/health.html diff --git a/pandoc/health.html b/pandoc/health.html new file mode 100644 index 0000000..e69de29 From 29dcdadb8bd3db5405453e0931a50dfff035fa05 Mon Sep 17 00:00:00 2001 From: Jeff DeWan Date: Thu, 17 Oct 2024 12:19:41 -0700 Subject: [PATCH 18/26] Deploy cron and remove terraform and entrypoint for pandoc --- .github/workflows/shared-deploy.yml | 1 + deploy/ecs_deploy.sh | 16 ++++++++ deploy/terraform/pandoc/_locals.tf | 6 --- deploy/terraform/pandoc/_provider.tf | 34 ----------------- deploy/terraform/pandoc/variables.tf | 11 ------ docker/Dockerfile-pandoc | 6 --- docker/pandoc-entrypoint.sh | 55 ---------------------------- 7 files changed, 17 insertions(+), 112 deletions(-) delete mode 100644 deploy/terraform/pandoc/_locals.tf delete mode 100644 deploy/terraform/pandoc/_provider.tf delete mode 100644 deploy/terraform/pandoc/variables.tf delete mode 100755 docker/pandoc-entrypoint.sh diff --git a/.github/workflows/shared-deploy.yml b/.github/workflows/shared-deploy.yml index 770bf3b..ae84a00 100644 --- a/.github/workflows/shared-deploy.yml +++ b/.github/workflows/shared-deploy.yml @@ -25,6 +25,7 @@ jobs: terraform-deploy: name: Terraform Deployment uses: ./.github/workflows/terraform-deploy.yml + if: ${{ inputs.project }} == "livepoll" with: environment: ${{ inputs.environment }} project: ${{ inputs.project }} diff --git a/deploy/ecs_deploy.sh b/deploy/ecs_deploy.sh index 9c0cc92..5e70d8d 100755 --- a/deploy/ecs_deploy.sh +++ b/deploy/ecs_deploy.sh @@ -58,3 +58,19 @@ docker run -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_SESSION_TOKEN \ -e ${CONTAINER_NAME} PARAMETER_STORE_PREFIX_LIST ${SSM_PREFIX} \ -e ${CONTAINER_NAME} AWS_REGION us-west-2 \ $ECS_CLUSTER_NAME $ECS_SERVICE_NAME + +if [[ "${PROJECT_NAME}" == "pandoc" ]]; then + + ECS_SERVICE_NAME_CRON="${ECS_SERVICE_NAME}-cron" + echo -e "\n============================================================" + echo " Beginning deployment to AWS Fargate environment: $ECS_CLUSTER_NAME $ECS_SERVICE_NAME_CRON" + echo -e "============================================================\n" + + # Deploy the latest image to Fargate for the cronjob container + docker run -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_SESSION_TOKEN \ + fabfuel/ecs-deploy:1.15.0 ecs deploy --region us-west-2 --no-deregister --timeout 600 --rollback \ + -i ${CONTAINER_NAME}-cron $AWS_CI_ACCOUNT.dkr.ecr.us-west-2.amazonaws.com/$ECR_REPOSITORY:$AWS_ECR_IMAGE_TAG-cron \ + -e ${CONTAINER_NAME}-cron PARAMETER_STORE_PREFIX_LIST ${SSM_PREFIX} \ + -e ${CONTAINER_NAME}-cron AWS_REGION us-west-2 \ + $ECS_CLUSTER_NAME $ECS_SERVICE_NAME_CRON +fi \ No newline at end of file diff --git a/deploy/terraform/pandoc/_locals.tf b/deploy/terraform/pandoc/_locals.tf deleted file mode 100644 index eb86aee..0000000 --- a/deploy/terraform/pandoc/_locals.tf +++ /dev/null @@ -1,6 +0,0 @@ -locals { - service_name = "pandoc" - parameter_store_prefix = "/lumen-services/${var.region}/${var.environment}/${local.service_name}" - parameter_store_prefix_webapp = "${local.parameter_store_prefix}/webapp" - parameter_store_prefix_cron = "${local.parameter_store_prefix}/cron" -} \ No newline at end of file diff --git a/deploy/terraform/pandoc/_provider.tf b/deploy/terraform/pandoc/_provider.tf deleted file mode 100644 index 63e3723..0000000 --- a/deploy/terraform/pandoc/_provider.tf +++ /dev/null @@ -1,34 +0,0 @@ -provider "aws" { - region = var.region - - assume_role { - role_arn = "arn:aws:iam::${var.account}:role/ci-role" - } - - default_tags { - tags = { - Environment = var.environment - Terraform = "true" - Terraform_Project = local.service_name - Service = local.service_name - } - } -} - -terraform { - required_version = ">= 1.0.0" - - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 4.0" - } - } - backend "s3" { - encrypt = true - bucket = "lumen-terraform-state" - dynamodb_table = "lumen-terraform-state-lock" - region = "us-west-2" - role_arn = "arn:aws:iam::824635284302:role/terraform" - } -} diff --git a/deploy/terraform/pandoc/variables.tf b/deploy/terraform/pandoc/variables.tf deleted file mode 100644 index e960b8d..0000000 --- a/deploy/terraform/pandoc/variables.tf +++ /dev/null @@ -1,11 +0,0 @@ -variable "environment" { - type = string -} - -variable "region" { - type = string -} - -variable "account" { - type = string -} diff --git a/docker/Dockerfile-pandoc b/docker/Dockerfile-pandoc index ecd84a2..015e712 100644 --- a/docker/Dockerfile-pandoc +++ b/docker/Dockerfile-pandoc @@ -16,12 +16,6 @@ RUN mkdir \ /var/www/datatmp \ /var/www/html/imgs -COPY docker/pandoc-entrypoint.sh /usr/bin/docker-entrypoint.sh - -ENTRYPOINT [ "/usr/bin/docker-entrypoint.sh" ] - -CMD [ "apache2-foreground" ] - FROM debian:12.5 AS cron RUN apt-get update \ diff --git a/docker/pandoc-entrypoint.sh b/docker/pandoc-entrypoint.sh deleted file mode 100755 index e78e9d7..0000000 --- a/docker/pandoc-entrypoint.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash -set -eo pipefail - -# Required environment variables: -# - PARAMETER_STORE_PREFIX_LIST: Comma-separated list of paths in SSM Parameter Store from which environment variables are pulled. -# If items from multiple paths conflict, the later one is used. - -# https://www.geekcafe.com/blog/how-to-load-parameter-store-values-into-an-ec2s-environment-variables -setup_parameter_store() { - PREFIX=$1 - - parameters=$(aws ssm get-parameters-by-path \ - --path "${PREFIX}" \ - --region ${AWS_REGION} \ - --with-decrypt \ - ) - success=$? - - if [ $success -eq 0 ]; then - # bail if no parameters found - param_count=$(echo "$parameters" | jq '.Parameters | length') - if [ $param_count -eq 0 ]; then - echo "No parameters found in '${PREFIX}!'" - exit 1 - fi - - # split("/")[-1] will grab the last token as the "name" for the env variable - export_statement=$(echo "$parameters" \ - | jq -r '.Parameters[] | "echo \"Pulling from " + .Name + "\"; export " + (.Name | split("/")[-1]) + "=\"" + .Value + "\""' \ - ) - sorted=$(echo "$export_statement" | sort) - - # eval the export strings - eval "$sorted" - else - echo "Failed to retrieve AWS SSM Parameters from '${PREFIX}!'" - exit 1 - fi -} - -if [ ! -z ${PARAMETER_STORE_PREFIX_LIST} ]; then - prefixes=$(echo $PARAMETER_STORE_PREFIX_LIST | tr "," "\n") - - for prefix in $prefixes - do - setup_parameter_store $prefix - done - -else - echo "PARAMETER_STORE_PREFIX_LIST is not set, skipping parameter store setup." -fi - -# Then exec the upstream entrypoint with what's set as CMD in the Dockerfile -# https://github.com/docker-library/php/blob/9c50f6a789b0337345934eee0d46e74d1e6dc8e2/8.2/bookworm/apache/Dockerfile -exec /usr/local/bin/docker-php-entrypoint "$@" \ No newline at end of file From 9ba0277d634f4da6a9c10cd45a75551edb2193a8 Mon Sep 17 00:00:00 2001 From: Jeff DeWan Date: Thu, 17 Oct 2024 12:22:11 -0700 Subject: [PATCH 19/26] Remove pandoc terraform --- .github/workflows/build-and-push.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.github/workflows/build-and-push.yml b/.github/workflows/build-and-push.yml index 53c1e75..36b5492 100644 --- a/.github/workflows/build-and-push.yml +++ b/.github/workflows/build-and-push.yml @@ -46,18 +46,6 @@ jobs: id: validate run: terraform validate -no-color working-directory: deploy/terraform/livepoll - - - name: Pandoc Terraform Init - run: terraform init -backend=false - working-directory: deploy/terraform/pandoc - - - name: Pandoc Terraform Format - run: terraform fmt -check - working-directory: deploy/terraform/pandoc - - - name: Pandoc Terraform Validate - run: terraform validate -no-color - working-directory: deploy/terraform/pandoc build-and-push-livepoll: runs-on: ubuntu-latest From cd53ac02d0d36afedcd165ef3ab9556821483541 Mon Sep 17 00:00:00 2001 From: Jeff DeWan Date: Thu, 17 Oct 2024 12:27:07 -0700 Subject: [PATCH 20/26] Remove pandoc terraform --- .github/workflows/build-and-push.yml | 1 - .github/workflows/shared-deploy.yml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build-and-push.yml b/.github/workflows/build-and-push.yml index 36b5492..e72909f 100644 --- a/.github/workflows/build-and-push.yml +++ b/.github/workflows/build-and-push.yml @@ -107,7 +107,6 @@ jobs: runs-on: ubuntu-latest name: Build and Push Pandoc Docker Image timeout-minutes: 30 - needs: [validate-terraform] permissions: id-token: write # Used for AWS OIDC auth contents: read diff --git a/.github/workflows/shared-deploy.yml b/.github/workflows/shared-deploy.yml index ae84a00..b8a3590 100644 --- a/.github/workflows/shared-deploy.yml +++ b/.github/workflows/shared-deploy.yml @@ -25,7 +25,7 @@ jobs: terraform-deploy: name: Terraform Deployment uses: ./.github/workflows/terraform-deploy.yml - if: ${{ inputs.project }} == "livepoll" + if: inputs.project == 'livepoll' with: environment: ${{ inputs.environment }} project: ${{ inputs.project }} From 1d7ebbe2bb76687eb47f3f3b26322b57db3a27f3 Mon Sep 17 00:00:00 2001 From: Jeff DeWan Date: Thu, 17 Oct 2024 12:29:47 -0700 Subject: [PATCH 21/26] Run empty terraform --- .github/workflows/build-and-push.yml | 13 +++++++++++ .github/workflows/shared-deploy.yml | 1 - deploy/terraform/pandoc/_locals.tf | 5 ++++ deploy/terraform/pandoc/_provider.tf | 34 ++++++++++++++++++++++++++++ deploy/terraform/pandoc/variables.tf | 11 +++++++++ 5 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 deploy/terraform/pandoc/_locals.tf create mode 100644 deploy/terraform/pandoc/_provider.tf create mode 100644 deploy/terraform/pandoc/variables.tf diff --git a/.github/workflows/build-and-push.yml b/.github/workflows/build-and-push.yml index e72909f..53c1e75 100644 --- a/.github/workflows/build-and-push.yml +++ b/.github/workflows/build-and-push.yml @@ -46,6 +46,18 @@ jobs: id: validate run: terraform validate -no-color working-directory: deploy/terraform/livepoll + + - name: Pandoc Terraform Init + run: terraform init -backend=false + working-directory: deploy/terraform/pandoc + + - name: Pandoc Terraform Format + run: terraform fmt -check + working-directory: deploy/terraform/pandoc + + - name: Pandoc Terraform Validate + run: terraform validate -no-color + working-directory: deploy/terraform/pandoc build-and-push-livepoll: runs-on: ubuntu-latest @@ -107,6 +119,7 @@ jobs: runs-on: ubuntu-latest name: Build and Push Pandoc Docker Image timeout-minutes: 30 + needs: [validate-terraform] permissions: id-token: write # Used for AWS OIDC auth contents: read diff --git a/.github/workflows/shared-deploy.yml b/.github/workflows/shared-deploy.yml index b8a3590..770bf3b 100644 --- a/.github/workflows/shared-deploy.yml +++ b/.github/workflows/shared-deploy.yml @@ -25,7 +25,6 @@ jobs: terraform-deploy: name: Terraform Deployment uses: ./.github/workflows/terraform-deploy.yml - if: inputs.project == 'livepoll' with: environment: ${{ inputs.environment }} project: ${{ inputs.project }} diff --git a/deploy/terraform/pandoc/_locals.tf b/deploy/terraform/pandoc/_locals.tf new file mode 100644 index 0000000..e228494 --- /dev/null +++ b/deploy/terraform/pandoc/_locals.tf @@ -0,0 +1,5 @@ +locals { + service_name = "pandoc" + parameter_store_prefix = "/lumen-services/${var.region}/${var.environment}/${local.service_name}" + parameter_store_prefix_webapp = "${local.parameter_store_prefix}/webapp" +} \ No newline at end of file diff --git a/deploy/terraform/pandoc/_provider.tf b/deploy/terraform/pandoc/_provider.tf new file mode 100644 index 0000000..63e3723 --- /dev/null +++ b/deploy/terraform/pandoc/_provider.tf @@ -0,0 +1,34 @@ +provider "aws" { + region = var.region + + assume_role { + role_arn = "arn:aws:iam::${var.account}:role/ci-role" + } + + default_tags { + tags = { + Environment = var.environment + Terraform = "true" + Terraform_Project = local.service_name + Service = local.service_name + } + } +} + +terraform { + required_version = ">= 1.0.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 4.0" + } + } + backend "s3" { + encrypt = true + bucket = "lumen-terraform-state" + dynamodb_table = "lumen-terraform-state-lock" + region = "us-west-2" + role_arn = "arn:aws:iam::824635284302:role/terraform" + } +} diff --git a/deploy/terraform/pandoc/variables.tf b/deploy/terraform/pandoc/variables.tf new file mode 100644 index 0000000..e960b8d --- /dev/null +++ b/deploy/terraform/pandoc/variables.tf @@ -0,0 +1,11 @@ +variable "environment" { + type = string +} + +variable "region" { + type = string +} + +variable "account" { + type = string +} From 861f7536ab64e62fa696a44036d277302f7c1afd Mon Sep 17 00:00:00 2001 From: Jeff DeWan Date: Thu, 17 Oct 2024 13:09:22 -0700 Subject: [PATCH 22/26] Remove cron deploy, chown files --- deploy/ecs_deploy.sh | 16 ---------------- docker/Dockerfile-pandoc | 2 +- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/deploy/ecs_deploy.sh b/deploy/ecs_deploy.sh index 5e70d8d..9c0cc92 100755 --- a/deploy/ecs_deploy.sh +++ b/deploy/ecs_deploy.sh @@ -58,19 +58,3 @@ docker run -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_SESSION_TOKEN \ -e ${CONTAINER_NAME} PARAMETER_STORE_PREFIX_LIST ${SSM_PREFIX} \ -e ${CONTAINER_NAME} AWS_REGION us-west-2 \ $ECS_CLUSTER_NAME $ECS_SERVICE_NAME - -if [[ "${PROJECT_NAME}" == "pandoc" ]]; then - - ECS_SERVICE_NAME_CRON="${ECS_SERVICE_NAME}-cron" - echo -e "\n============================================================" - echo " Beginning deployment to AWS Fargate environment: $ECS_CLUSTER_NAME $ECS_SERVICE_NAME_CRON" - echo -e "============================================================\n" - - # Deploy the latest image to Fargate for the cronjob container - docker run -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_SESSION_TOKEN \ - fabfuel/ecs-deploy:1.15.0 ecs deploy --region us-west-2 --no-deregister --timeout 600 --rollback \ - -i ${CONTAINER_NAME}-cron $AWS_CI_ACCOUNT.dkr.ecr.us-west-2.amazonaws.com/$ECR_REPOSITORY:$AWS_ECR_IMAGE_TAG-cron \ - -e ${CONTAINER_NAME}-cron PARAMETER_STORE_PREFIX_LIST ${SSM_PREFIX} \ - -e ${CONTAINER_NAME}-cron AWS_REGION us-west-2 \ - $ECS_CLUSTER_NAME $ECS_SERVICE_NAME_CRON -fi \ No newline at end of file diff --git a/docker/Dockerfile-pandoc b/docker/Dockerfile-pandoc index 015e712..726b961 100644 --- a/docker/Dockerfile-pandoc +++ b/docker/Dockerfile-pandoc @@ -7,7 +7,7 @@ RUN apt-get update \ awscli \ && apt-get clean -COPY pandoc/* /var/www/html/ +COPY --chown=www-data:www-data pandoc/* /var/www/html/ RUN mkdir \ /var/www/datatmp \ From 0efdeca7f116aaa5e0d8058d5b7849b3afc9d714 Mon Sep 17 00:00:00 2001 From: Jeff DeWan Date: Thu, 17 Oct 2024 16:24:20 -0700 Subject: [PATCH 23/26] Cleanup --- docker/Dockerfile-pandoc | 1 + pandoc/health.html | 0 2 files changed, 1 insertion(+) delete mode 100644 pandoc/health.html diff --git a/docker/Dockerfile-pandoc b/docker/Dockerfile-pandoc index 726b961..4bff90f 100644 --- a/docker/Dockerfile-pandoc +++ b/docker/Dockerfile-pandoc @@ -8,6 +8,7 @@ RUN apt-get update \ && apt-get clean COPY --chown=www-data:www-data pandoc/* /var/www/html/ +RUN rm /var/www/html/cleanup-old-files RUN mkdir \ /var/www/datatmp \ diff --git a/pandoc/health.html b/pandoc/health.html deleted file mode 100644 index e69de29..0000000 From cf5f6182b8c453aeedc984c7b39c17c5c7e524c8 Mon Sep 17 00:00:00 2001 From: Jeff DeWan Date: Tue, 22 Oct 2024 13:24:59 -0700 Subject: [PATCH 24/26] Remove pandoc cron container and rely on EFS lifecycle rules to archive old files --- .github/workflows/build-and-push.yml | 19 ------------------- deploy/pandoc-environments.json | 12 ------------ docker-compose.yml | 21 --------------------- docker/Dockerfile-pandoc | 18 +----------------- pandoc/cleanup-old-files | 5 ----- 5 files changed, 1 insertion(+), 74 deletions(-) delete mode 100644 pandoc/cleanup-old-files diff --git a/.github/workflows/build-and-push.yml b/.github/workflows/build-and-push.yml index 53c1e75..0cebf5b 100644 --- a/.github/workflows/build-and-push.yml +++ b/.github/workflows/build-and-push.yml @@ -171,22 +171,3 @@ jobs: ${{ steps.login-ecr.outputs.registry }}/${{ env.PANDOC_ECR_REPOSITORY }}:gha-build-${{ steps.set_branch_name.outputs.branch_name }}-latest cache-from: type=gha cache-to: type=gha,mode=max - - - name: Docker build and push - uses: docker/build-push-action@v5 - if: steps.prebuilt_check.outputs.image_exists != 0 - env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - with: - context: . - file: docker/Dockerfile-pandoc - target: cron - platforms: linux/amd64 - push: true - tags: | - ${{ steps.login-ecr.outputs.registry }}/${{ env.PANDOC_ECR_REPOSITORY }}:gha-commit-${{ github.sha }}-cron - ${{ steps.login-ecr.outputs.registry }}/${{ env.PANDOC_ECR_REPOSITORY }}:gha-tree-${{ steps.read_tree_hash.outputs.tree_hash }}-cron - ${{ steps.login-ecr.outputs.registry }}/${{ env.PANDOC_ECR_REPOSITORY }}:gha-build-${{ steps.set_branch_name.outputs.branch_name }}-${{ github.run_number }}-${{ github.run_attempt }}-cron - ${{ steps.login-ecr.outputs.registry }}/${{ env.PANDOC_ECR_REPOSITORY }}:gha-build-${{ steps.set_branch_name.outputs.branch_name }}-latest-cron - cache-from: type=gha - cache-to: type=gha,mode=max \ No newline at end of file diff --git a/deploy/pandoc-environments.json b/deploy/pandoc-environments.json index 5ee173b..f742786 100644 --- a/deploy/pandoc-environments.json +++ b/deploy/pandoc-environments.json @@ -11,10 +11,6 @@ }, "webapp": { "ssm_prefix": "/lumen-services/us-west-2/dev/pandoc/webapp" - }, - "cron": { - "cluster_name": "dev-pandoc", - "ssm_prefix": "/lumen-services/us-west-2/dev/pandoc/cron" } }, { @@ -28,10 +24,6 @@ }, "webapp": { "ssm_prefix": "/lumen-services/us-west-2/staging/pandoc/webapp" - }, - "cron": { - "cluster_name": "staging-pandoc", - "ssm_prefix": "/lumen-services/us-west-2/staging/pandoc/cron" } }, { @@ -45,10 +37,6 @@ }, "webapp": { "ssm_prefix": "/lumen-services/us-west-2/prod/pandoc/webapp" - }, - "cron": { - "cluster_name": "prod-pandoc", - "ssm_prefix": "/lumen-services/us-west-2/prod/pandoc/cron" } } ] diff --git a/docker-compose.yml b/docker-compose.yml index f2208db..a3c0997 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,7 +4,6 @@ services: build: context: ./ dockerfile: docker/Dockerfile-pandoc - target: pandoc expose: - 80 ports: @@ -22,26 +21,6 @@ services: stdin_open: false tty: false - # The pandoc container does not run cron, so run cron - # in its own container with a shared volume. - pandoc-cron: - build: - context: ./ - dockerfile: docker/Dockerfile-pandoc - target: cron - - volumes: - - pandoc-temp-datatmp:/var/www/datatmp - - pandoc-temp-imgs:/var/www/html/imgs - - hostname: pandoc-cron - domainname: example.com - - restart: unless-stopped - privileged: true - stdin_open: false - tty: false - livepoll: build: context: ./ diff --git a/docker/Dockerfile-pandoc b/docker/Dockerfile-pandoc index 4bff90f..7597add 100644 --- a/docker/Dockerfile-pandoc +++ b/docker/Dockerfile-pandoc @@ -1,4 +1,4 @@ -FROM php:8.2-apache AS pandoc +FROM php:8.2-apache RUN apt-get update \ && apt-get install -y --no-install-recommends --no-install-suggests \ @@ -8,7 +8,6 @@ RUN apt-get update \ && apt-get clean COPY --chown=www-data:www-data pandoc/* /var/www/html/ -RUN rm /var/www/html/cleanup-old-files RUN mkdir \ /var/www/datatmp \ @@ -16,18 +15,3 @@ RUN mkdir \ && chown www-data:www-data \ /var/www/datatmp \ /var/www/html/imgs - -FROM debian:12.5 AS cron - -RUN apt-get update \ - && apt-get install -y --no-install-suggests --no-install-recommends \ - cron \ - && apt-get clean - -COPY pandoc/cleanup-old-files /etc/cron.daily - -RUN chmod 755 /etc/cron.daily/cleanup-old-files - -CMD ["cron", "-f"] - - \ No newline at end of file diff --git a/pandoc/cleanup-old-files b/pandoc/cleanup-old-files deleted file mode 100644 index 3a06f20..0000000 --- a/pandoc/cleanup-old-files +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh - -find /var/www/datatmp/* -mtime +1 -exec rm {} \; - -find /var/www/html/imgs/* -mtime +1 -exec rm {} \; From 6c1c73fd489662c6c9d93a870a01ee189f6b4899 Mon Sep 17 00:00:00 2001 From: Jeff DeWan Date: Tue, 22 Oct 2024 13:30:13 -0700 Subject: [PATCH 25/26] Fix build --- .github/workflows/build-and-push.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build-and-push.yml b/.github/workflows/build-and-push.yml index 0cebf5b..180eaac 100644 --- a/.github/workflows/build-and-push.yml +++ b/.github/workflows/build-and-push.yml @@ -161,7 +161,6 @@ jobs: with: context: . file: docker/Dockerfile-pandoc - target: pandoc platforms: linux/amd64 push: true tags: | From 8e9cc1f67f58d950713763e7f42015ddb7eecc51 Mon Sep 17 00:00:00 2001 From: Jeff DeWan Date: Wed, 23 Oct 2024 11:15:34 -0700 Subject: [PATCH 26/26] Bump PHP to latest and use production PHP defaults for pandoc --- docker/Dockerfile-pandoc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile-pandoc b/docker/Dockerfile-pandoc index 7597add..9921665 100644 --- a/docker/Dockerfile-pandoc +++ b/docker/Dockerfile-pandoc @@ -1,4 +1,4 @@ -FROM php:8.2-apache +FROM php:8.3-apache RUN apt-get update \ && apt-get install -y --no-install-recommends --no-install-suggests \ @@ -15,3 +15,6 @@ RUN mkdir \ && chown www-data:www-data \ /var/www/datatmp \ /var/www/html/imgs + +# enable php production defaults +RUN cp $PHP_INI_DIR/php.ini-production $PHP_INI_DIR/php.ini \ No newline at end of file