From d16abb294a304cb6866eb2b83eeab1318933ba0e Mon Sep 17 00:00:00 2001 From: Austin Macdonald Date: Wed, 25 Sep 2024 15:05:15 -0500 Subject: [PATCH 01/10] Inital commit to add GH action to generate report --- .github/workflows/report.yaml | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .github/workflows/report.yaml diff --git a/.github/workflows/report.yaml b/.github/workflows/report.yaml new file mode 100644 index 0000000..96758af --- /dev/null +++ b/.github/workflows/report.yaml @@ -0,0 +1,31 @@ +name: Generate Data Usage Report + +on: + pull_request: + branches: + - main + +jobs: + deploy: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v3 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # TODO param region + aws-region: us-east-2 + + - name: Configure kubectl with AWS EKS + # TODO param name, region + run: | + aws eks update-kubeconfig --name eks-dandihub --region us-east-2 + + - name: Sanity check + run: | + kubectl get pods -n jupyterhub From 713d64cbd12e5aa08b6e4c448f1e04ee15adc6b7 Mon Sep 17 00:00:00 2001 From: Austin Macdonald Date: Wed, 25 Sep 2024 15:13:59 -0500 Subject: [PATCH 02/10] Assume Jupyterhub Provisioning Role --- .github/workflows/report.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/report.yaml b/.github/workflows/report.yaml index 96758af..829dc77 100644 --- a/.github/workflows/report.yaml +++ b/.github/workflows/report.yaml @@ -21,6 +21,16 @@ jobs: # TODO param region aws-region: us-east-2 + - name: Assume JupyterhubProvisioningRole + # TODO param ProvisioningRoleARN and name ^ + run: | + ROLE_ARN="arn:aws:iam::278212569472:role/JupyterhubProvisioningRole" + CREDS=$(aws sts assume-role --role-arn $ROLE_ARN --role-session-name "GitHubActionsSession") + export AWS_ACCESS_KEY_ID=$(echo $CREDS | jq -r '.Credentials.AccessKeyId') + export AWS_SECRET_ACCESS_KEY=$(echo $CREDS | jq -r '.Credentials.SecretAccessKey') + export AWS_SESSION_TOKEN=$(echo $CREDS | jq -r '.Credentials.SessionToken') + + - name: Configure kubectl with AWS EKS # TODO param name, region run: | From 519360c7b004d9807fde9e524cca853b115facce Mon Sep 17 00:00:00 2001 From: Austin Macdonald Date: Wed, 25 Sep 2024 15:16:02 -0500 Subject: [PATCH 03/10] Fixup: indent --- .github/workflows/report.yaml | 58 +++++++++++++++++------------------ 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/.github/workflows/report.yaml b/.github/workflows/report.yaml index 829dc77..162b09f 100644 --- a/.github/workflows/report.yaml +++ b/.github/workflows/report.yaml @@ -10,32 +10,32 @@ jobs: runs-on: ubuntu-latest steps: - - name: Checkout code - uses: actions/checkout@v3 - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v3 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # TODO param region - aws-region: us-east-2 - - - name: Assume JupyterhubProvisioningRole - # TODO param ProvisioningRoleARN and name ^ - run: | - ROLE_ARN="arn:aws:iam::278212569472:role/JupyterhubProvisioningRole" - CREDS=$(aws sts assume-role --role-arn $ROLE_ARN --role-session-name "GitHubActionsSession") - export AWS_ACCESS_KEY_ID=$(echo $CREDS | jq -r '.Credentials.AccessKeyId') - export AWS_SECRET_ACCESS_KEY=$(echo $CREDS | jq -r '.Credentials.SecretAccessKey') - export AWS_SESSION_TOKEN=$(echo $CREDS | jq -r '.Credentials.SessionToken') - - - - name: Configure kubectl with AWS EKS - # TODO param name, region - run: | - aws eks update-kubeconfig --name eks-dandihub --region us-east-2 - - - name: Sanity check - run: | - kubectl get pods -n jupyterhub + - name: Checkout code + uses: actions/checkout@v3 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v3 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # TODO param region + aws-region: us-east-2 + + - name: Assume JupyterhubProvisioningRole + # TODO param ProvisioningRoleARN and name ^ + run: | + ROLE_ARN="arn:aws:iam::278212569472:role/JupyterhubProvisioningRole" + CREDS=$(aws sts assume-role --role-arn $ROLE_ARN --role-session-name "GitHubActionsSession") + export AWS_ACCESS_KEY_ID=$(echo $CREDS | jq -r '.Credentials.AccessKeyId') + export AWS_SECRET_ACCESS_KEY=$(echo $CREDS | jq -r '.Credentials.SecretAccessKey') + export AWS_SESSION_TOKEN=$(echo $CREDS | jq -r '.Credentials.SessionToken') + + + - name: Configure kubectl with AWS EKS + # TODO param name, region + run: | + aws eks update-kubeconfig --name eks-dandihub --region us-east-2 + + - name: Sanity check + run: | + kubectl get pods -n jupyterhub From e6f481441b6512f51f17c22e83a608e66e9551ba Mon Sep 17 00:00:00 2001 From: Austin Macdonald Date: Wed, 25 Sep 2024 15:17:03 -0500 Subject: [PATCH 04/10] Rename job --- .github/workflows/report.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/report.yaml b/.github/workflows/report.yaml index 162b09f..5b1f053 100644 --- a/.github/workflows/report.yaml +++ b/.github/workflows/report.yaml @@ -6,7 +6,7 @@ on: - main jobs: - deploy: + generate_data_usage_report: runs-on: ubuntu-latest steps: From 72496f4a7846ef70856408c154f89651ee8a5d55 Mon Sep 17 00:00:00 2001 From: Austin Macdonald Date: Wed, 25 Sep 2024 15:23:29 -0500 Subject: [PATCH 05/10] Add assumed role to update-kubeconfig --- .github/workflows/report.yaml | 4 ++-- README.md | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/report.yaml b/.github/workflows/report.yaml index 5b1f053..8229875 100644 --- a/.github/workflows/report.yaml +++ b/.github/workflows/report.yaml @@ -32,9 +32,9 @@ jobs: - name: Configure kubectl with AWS EKS - # TODO param name, region + # TODO param name, region role-arn run: | - aws eks update-kubeconfig --name eks-dandihub --region us-east-2 + aws eks update-kubeconfig --name eks-dandihub --region us-east-2 --role-arn arn:aws:iam::278212569472:role/JupyterhubProvisioningRole - name: Sanity check run: | diff --git a/README.md b/README.md index 5af6edf..4337f54 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,9 @@ # Dandihub +TODO + - add provisioning role to cluser :q + - + This Terraform blueprint creates a Kubernetes environment (EKS) and installs JupyterHub. Based on [AWS Data on EKS JupyterHub](https://github.com/awslabs/data-on-eks/tree/main/ai-ml/jupyterhub). ## Table of Contents From 8428d3a2025421775320f39542f4cb9057559499 Mon Sep 17 00:00:00 2001 From: Austin Macdonald Date: Wed, 25 Sep 2024 15:26:10 -0500 Subject: [PATCH 06/10] No need to add ProvisioningRole to masters --- README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.md b/README.md index 4337f54..5af6edf 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,5 @@ # Dandihub -TODO - - add provisioning role to cluser :q - - - This Terraform blueprint creates a Kubernetes environment (EKS) and installs JupyterHub. Based on [AWS Data on EKS JupyterHub](https://github.com/awslabs/data-on-eks/tree/main/ai-ml/jupyterhub). ## Table of Contents From e170b593494033ec743255a6e823dea156d85e60 Mon Sep 17 00:00:00 2001 From: Austin Macdonald Date: Wed, 25 Sep 2024 15:47:01 -0500 Subject: [PATCH 07/10] Deploy a pod to the cluster, and schedule with Karpenter --- .github/manifests/hello-world.yaml | 20 ++++++++++++++++++++ .github/workflows/report.yaml | 20 ++++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 .github/manifests/hello-world.yaml diff --git a/.github/manifests/hello-world.yaml b/.github/manifests/hello-world.yaml new file mode 100644 index 0000000..1977f33 --- /dev/null +++ b/.github/manifests/hello-world.yaml @@ -0,0 +1,20 @@ +# manifests/hello-world-pod.yaml +apiVersion: v1 +kind: Pod +metadata: + name: hello-world-pod +spec: + containers: + - name: hello + image: busybox + command: ['sh', '-c', 'echo Hello, World! && sleep 30'] + nodeSelector: + NodeGroupType: default + NodePool: default + hub.jupyter.org/node-purpose: user + tolerations: + - key: "hub.jupyter.org/dedicated" + operator: "Equal" + value: "user" + effect: "NoSchedule" + diff --git a/.github/workflows/report.yaml b/.github/workflows/report.yaml index 8229875..d8dff65 100644 --- a/.github/workflows/report.yaml +++ b/.github/workflows/report.yaml @@ -39,3 +39,23 @@ jobs: - name: Sanity check run: | kubectl get pods -n jupyterhub + + # Step 4: Deploy Hello World Pod from manifest + - name: Deploy Hello World Pod + run: | + kubectl apply -f manifests/hello-world-pod.yaml + + # Step 5: Wait for Pod to Complete + - name: Wait for Hello World Pod to complete + run: | + kubectl wait --for=condition=Ready pod/hello-world-pod --timeout=60s + + # Step 6: Get Pod Logs to verify it ran successfully + - name: Get Hello World Pod logs + run: | + kubectl logs hello-world-pod + + # Step 7: Cleanup - Delete the Pod + - name: Delete Hello World Pod + run: | + kubectl delete pod hello-world-pod From bfce04694ef3b4be7fd00a54ae1738f2a02f169e Mon Sep 17 00:00:00 2001 From: Austin Macdonald Date: Wed, 25 Sep 2024 15:48:20 -0500 Subject: [PATCH 08/10] Fixup: correct path to pod manifest --- .github/workflows/report.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/report.yaml b/.github/workflows/report.yaml index d8dff65..a498d99 100644 --- a/.github/workflows/report.yaml +++ b/.github/workflows/report.yaml @@ -39,11 +39,11 @@ jobs: - name: Sanity check run: | kubectl get pods -n jupyterhub - + # Step 4: Deploy Hello World Pod from manifest - name: Deploy Hello World Pod run: | - kubectl apply -f manifests/hello-world-pod.yaml + kubectl apply -f .github/manifests/hello-world-pod.yaml # Step 5: Wait for Pod to Complete - name: Wait for Hello World Pod to complete From 0993129e3452e35980279e0fa14a99d1b76203fa Mon Sep 17 00:00:00 2001 From: Austin Macdonald Date: Wed, 25 Sep 2024 15:49:42 -0500 Subject: [PATCH 09/10] Fixup again ugh, rename file --- .github/manifests/{hello-world.yaml => hello-world-pod.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/manifests/{hello-world.yaml => hello-world-pod.yaml} (100%) diff --git a/.github/manifests/hello-world.yaml b/.github/manifests/hello-world-pod.yaml similarity index 100% rename from .github/manifests/hello-world.yaml rename to .github/manifests/hello-world-pod.yaml From 87027d220f91ce52f33fc59617ab3d1a96595935 Mon Sep 17 00:00:00 2001 From: Austin Macdonald Date: Wed, 25 Sep 2024 15:59:09 -0500 Subject: [PATCH 10/10] Delete Pod even if previous step times out (Also increase timeout) --- .github/workflows/report.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/report.yaml b/.github/workflows/report.yaml index a498d99..bc1d088 100644 --- a/.github/workflows/report.yaml +++ b/.github/workflows/report.yaml @@ -48,14 +48,17 @@ jobs: # Step 5: Wait for Pod to Complete - name: Wait for Hello World Pod to complete run: | - kubectl wait --for=condition=Ready pod/hello-world-pod --timeout=60s + kubectl wait --for=condition=Ready pod/hello-world-pod --timeout=300s # 5 minutes + continue-on-error: true # Allow the workflow to continue even if this step fails - # Step 6: Get Pod Logs to verify it ran successfully + # Step 6: Get Pod Logs to verify it ran successfully, only if Step 5 succeeds - name: Get Hello World Pod logs run: | kubectl logs hello-world-pod + if: ${{ success() }} # Only run this step if the previous step was successful - # Step 7: Cleanup - Delete the Pod + # Step 7: Cleanup - Always run this step, even if previous steps fail - name: Delete Hello World Pod run: | kubectl delete pod hello-world-pod + if: ${{ always() }} # Always run this step, even if other steps fail