From eedbd9fef7dd9b21a13f4af3a52b2964d1baf31c Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Thu, 2 May 2024 10:51:03 -0500 Subject: [PATCH 01/14] Support builds-on/runs-on arm --- charms/worker/charmcraft.yaml | 15 +++++++++++++++ charms/worker/k8s/charmcraft.yaml | 14 ++++++++++++++ charms/worker/requirements.txt | 2 -- 3 files changed, 29 insertions(+), 2 deletions(-) delete mode 100644 charms/worker/requirements.txt diff --git a/charms/worker/charmcraft.yaml b/charms/worker/charmcraft.yaml index c544bc74..b567c149 100644 --- a/charms/worker/charmcraft.yaml +++ b/charms/worker/charmcraft.yaml @@ -43,6 +43,20 @@ bases: - name: ubuntu channel: "24.04" architectures: [amd64] + - build-on: + - name: ubuntu + channel: "20.04" + architectures: [arm64] + run-on: + - name: ubuntu + channel: "20.04" + architectures: [arm64] + - name: ubuntu + channel: "22.04" + architectures: [arm64] + - name: ubuntu + channel: "24.04" + architectures: [arm64] config: options: labels: @@ -59,6 +73,7 @@ parts: plugin: charm build-packages: [git] charm-entrypoint: k8s/src/charm.py + charm-requirements: [k8s/requirements.txt] promote: # move paths out of ./k8s to ./ since # charmcraft assumes ./lib to be there diff --git a/charms/worker/k8s/charmcraft.yaml b/charms/worker/k8s/charmcraft.yaml index 3f078164..b68cb8a3 100644 --- a/charms/worker/k8s/charmcraft.yaml +++ b/charms/worker/k8s/charmcraft.yaml @@ -52,6 +52,20 @@ bases: - name: ubuntu channel: "24.04" architectures: [amd64] + - build-on: + - name: ubuntu + channel: "20.04" + architectures: [arm64] + run-on: + - name: ubuntu + channel: "20.04" + architectures: [arm64] + - name: ubuntu + channel: "22.04" + architectures: [arm64] + - name: ubuntu + channel: "24.04" + architectures: [arm64] config: options: annotations: diff --git a/charms/worker/requirements.txt b/charms/worker/requirements.txt deleted file mode 100644 index fa105ff3..00000000 --- a/charms/worker/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ --r k8s/requirements.txt - From f7570bb1f6bfa3a787a8bee37c0e31cacaa4d66c Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Thu, 2 May 2024 11:52:35 -0500 Subject: [PATCH 02/14] Build ARM64 and AMD64 charms --- .github/workflows/build-charm.yaml | 10 ++++++---- tests/integration/conftest.py | 24 ++++++++++++++++++++++-- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-charm.yaml b/.github/workflows/build-charm.yaml index 61c3c91f..0a2e17d3 100644 --- a/.github/workflows/build-charm.yaml +++ b/.github/workflows/build-charm.yaml @@ -6,10 +6,12 @@ name: Build Charms on: workflow_call: inputs: - runs-on: + arch: + type: string + description: Architecture of the built charm file + runs-on-label: type: string description: Image runner for building the images - default: ubuntu-22.04 working-directory: type: string description: The working directory for jobs @@ -18,7 +20,7 @@ on: jobs: build-charms: name: Build and push charms - runs-on: ${{ inputs.runs-on }} + runs-on: ${{ inputs.runs-on-label }} steps: - uses: actions/checkout@v4 - uses: canonical/setup-lxd@v0.1.1 @@ -61,6 +63,6 @@ jobs: if: ${{ env.CHARM_FILE != '' && !cancelled() }} uses: actions/upload-artifact@v4 with: - name: ${{ env.CHARM_NAME }}-charm + name: ${{ env.CHARM_NAME }}-${{ inputs.arch }}-charm path: ${{ inputs.working-directory }}/${{ env.CHARM_FILE }} if-no-files-found: error diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 1191a338..444c1cdb 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -82,12 +82,14 @@ class Charm: Attrs: ops_test: Instance of the pytest-operator plugin + arch: Cloud Architecture path: Path to the charm file metadata: Charm's metadata app_name: Preferred name of the juju application """ ops_test: OpsTest + arch: str path: Path _charmfile: Optional[Path] = None @@ -122,7 +124,8 @@ async def resolve(self, charm_files: List[str]) -> Path: Path().glob(charm_name), # Look in top-level path self.path.glob(charm_name), # Look in charm-level path ) - self._charmfile, *_ = filter(lambda s: s.name.startswith(header), potentials) + arch_choices = filter(lambda s: self.arch in str(s), potentials) + self._charmfile, *_ = filter(lambda s: s.name.startswith(header), arch_choices) log.info("For %s found charmfile %s", self.app_name, self._charmfile) except ValueError: log.warning("No pre-built charm is available, let's build it") @@ -202,6 +205,22 @@ def add_constraints(self, constraints: Dict[str, str]): app["constraints"] = " ".join(f"{k}={v}" for k, v in existing.items()) +async def cloud_arch(ops_test: OpsTest) -> str: + """Return current architecture of the selected controller + + Args: + ops_test (OpsTest): ops_test plugin + + Returns: + string describing current architecture of the underlying cloud + """ + assert ops_test.model, "Model must be present" + controller = await ops_test.model.get_controller() + controller_model = await controller.get_model("controller") + arch = set(machine.safe_data["hardware-characteristics"]["arch"] for machine in controller_model.machines.values()) + return arch.pop() + + async def cloud_type(ops_test: OpsTest) -> Tuple[str, bool]: """Return current cloud type of the selected controller @@ -321,9 +340,10 @@ async def kubernetes_cluster(request: pytest.FixtureRequest, ops_test: OpsTest): return log.info("Deploying cluster using %s bundle.", bundle_file) + arch = await cloud_arch(ops_test) charm_path = ("worker/k8s", "worker") - charms = [Charm(ops_test, Path("charms") / p) for p in charm_path] + charms = [Charm(ops_test, arch, Path("charms") / p) for p in charm_path] charm_files = await asyncio.gather( *[charm.resolve(request.config.option.charm_files) for charm in charms] ) From bb17644f0cfd9125ae78fe0f4da49300fc526c80 Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Thu, 2 May 2024 11:56:55 -0500 Subject: [PATCH 03/14] Make use of self-hosted runners --- tests/integration/conftest.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 444c1cdb..8e451f21 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -217,7 +217,10 @@ async def cloud_arch(ops_test: OpsTest) -> str: assert ops_test.model, "Model must be present" controller = await ops_test.model.get_controller() controller_model = await controller.get_model("controller") - arch = set(machine.safe_data["hardware-characteristics"]["arch"] for machine in controller_model.machines.values()) + arch = set( + machine.safe_data["hardware-characteristics"]["arch"] + for machine in controller_model.machines.values() + ) return arch.pop() From 57c6cea06a575a22ec79f40facbc0fb9a0b93fe6 Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Thu, 2 May 2024 12:02:30 -0500 Subject: [PATCH 04/14] Use runner arch in build cache --- .github/workflows/build-charm.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-charm.yaml b/.github/workflows/build-charm.yaml index 0a2e17d3..bc3837fb 100644 --- a/.github/workflows/build-charm.yaml +++ b/.github/workflows/build-charm.yaml @@ -41,7 +41,7 @@ jobs: uses: actions/cache@v4 with: path: ${{ inputs.working-directory }}packed/ - key: ${{ runner.os}}-${{ env.CHARM_NAME }}-charm-${{ env.SRC_HASH }} + key: ${{ runner.os }}-${{ runner.arch }}-${{ env.CHARM_NAME }}-charm-${{ env.SRC_HASH }} - name: Use cached charm if: ${{ env.CHARM_NAME != 'UNKNOWN' && steps.cache-charm.outputs.cache-hit == 'true' }} From 319849550eb18f16cc9f2a3aea86398b804f8fb9 Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Thu, 2 May 2024 12:11:06 -0500 Subject: [PATCH 05/14] Run charmcraft with lxd permissions --- .github/workflows/build-charm.yaml | 68 ------------------------------ 1 file changed, 68 deletions(-) delete mode 100644 .github/workflows/build-charm.yaml diff --git a/.github/workflows/build-charm.yaml b/.github/workflows/build-charm.yaml deleted file mode 100644 index bc3837fb..00000000 --- a/.github/workflows/build-charm.yaml +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright 2023 Canonical Ltd. -# See LICENSE file for licensing details. - -name: Build Charms - -on: - workflow_call: - inputs: - arch: - type: string - description: Architecture of the built charm file - runs-on-label: - type: string - description: Image runner for building the images - working-directory: - type: string - description: The working directory for jobs - default: "./" - -jobs: - build-charms: - name: Build and push charms - runs-on: ${{ inputs.runs-on-label }} - steps: - - uses: actions/checkout@v4 - - uses: canonical/setup-lxd@v0.1.1 - - - name: Get hash of the source - run: echo "SRC_HASH=${{ hashFiles('**/charms/**/*.yaml', '**/charms/**/*.py', '**/charms/**/requirements.txt') }}" >> $GITHUB_ENV - - - name: Extract charm name - working-directory: ${{ inputs.working-directory }} - run: echo "CHARM_NAME=$([ -f charmcraft.yaml ] && yq '.name' charmcraft.yaml || echo UNKNOWN)" >> $GITHUB_ENV - - - name: Read charmcraft version file - id: charmcraft - run: echo "channel=$(cat .charmcraft-channel)" >> $GITHUB_OUTPUT - - - name: Cache charm output - id: cache-charm - uses: actions/cache@v4 - with: - path: ${{ inputs.working-directory }}packed/ - key: ${{ runner.os }}-${{ runner.arch }}-${{ env.CHARM_NAME }}-charm-${{ env.SRC_HASH }} - - - name: Use cached charm - if: ${{ env.CHARM_NAME != 'UNKNOWN' && steps.cache-charm.outputs.cache-hit == 'true' }} - working-directory: ${{ inputs.working-directory }}/${{ matrix.path }} - run: | - echo "CHARM_FILE=$(ls packed/${{env.CHARM_NAME}}_*.charm)" >> $GITHUB_ENV - - - name: Pack charm - if: ${{ env.CHARM_NAME != 'UNKNOWN' && !cancelled() && steps.cache-charm.outputs.cache-hit != 'true' }} - working-directory: ${{ inputs.working-directory }}/${{ matrix.path }} - run: | - sudo snap install charmcraft --classic --channel ${{ steps.charmcraft.outputs.channel }} - charmcraft pack -v - mkdir -p packed - mv *.charm packed/ - echo "CHARM_FILE=$(ls packed/${{env.CHARM_NAME}}_*.charm)" >> $GITHUB_ENV - - - name: Upload charm artifact - if: ${{ env.CHARM_FILE != '' && !cancelled() }} - uses: actions/upload-artifact@v4 - with: - name: ${{ env.CHARM_NAME }}-${{ inputs.arch }}-charm - path: ${{ inputs.working-directory }}/${{ env.CHARM_FILE }} - if-no-files-found: error From 03bb3bc0ae447176e73f7935f40367ca6d74e3ab Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Thu, 9 May 2024 12:41:39 -0500 Subject: [PATCH 06/14] attempt to use better-build workflows --- .../integration_test_betterbuild.yaml | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 .github/workflows/integration_test_betterbuild.yaml diff --git a/.github/workflows/integration_test_betterbuild.yaml b/.github/workflows/integration_test_betterbuild.yaml new file mode 100644 index 00000000..c5888efa --- /dev/null +++ b/.github/workflows/integration_test_betterbuild.yaml @@ -0,0 +1,58 @@ + +name: Integration tests + +on: + pull_request: + +jobs: + extra-args: + runs-on: ubuntu-latest + outputs: + args: ${{ steps.flags.outputs.args }} + steps: + - name: Determine extra args + id: flags + env: + TITLE: ${{ github.event.pull_request.title }} + JOB: ${{ github.job }} + WORKFLOW: ${{ github.workflow }} + run: | + EXTRA_ARGS="--crash-dump=on-failure" + if [[ "$TITLE" == *"[COS]"* ]]; then + EXTRA_ARGS="$EXTRA_ARGS --cos" + fi + echo "args=$EXTRA_ARGS" >> "$GITHUB_OUTPUT" + + # build-all-charms: + # needs: [extra-args] + # strategy: + # matrix: + # arch: + # - arch: amd64 + # runs-on-label: X64 + # - arch: arm64 + # runs-on-label: ARM64 + # uses: ./.github/workflows/build-charm.yaml + # with: + # working-directory: "./charms/worker/" + # arch: ${{ matrix.arch.arch }} + # runs-on-label: ${{ matrix.arch.runs-on-label }} + + integration-tests: + uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main + needs: [extra-args] + secrets: inherit + with: + provider: lxd + juju-channel: 3.3/stable + extra-arguments: ${{needs.extra-args.outputs.args}} + load-test-enabled: false + modules: '["test_k8s", "test_etcd"]' + zap-enabled: false + self-hosted-runner: true + self-hosted-runner-label: "large" + trivy-fs-enabled: true + trivy-image-config: "trivy.yaml" + tmate-debug: true + test-timeout: 120 + test-tox-env: integration From 14992e04e81b181799b8e260e69be2097906d6ff Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Thu, 9 May 2024 13:16:13 -0500 Subject: [PATCH 07/14] Use arch specific better builder --- .../workflows/integration_test_betterbuild.yaml | 14 +++++++++++--- .github/workflows/publish-charms.yaml | 3 ++- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration_test_betterbuild.yaml b/.github/workflows/integration_test_betterbuild.yaml index c5888efa..e69c4417 100644 --- a/.github/workflows/integration_test_betterbuild.yaml +++ b/.github/workflows/integration_test_betterbuild.yaml @@ -39,20 +39,28 @@ jobs: # runs-on-label: ${{ matrix.arch.runs-on-label }} integration-tests: - uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main + uses: addyess/operator-workflows/.github/workflows/integration_test.yaml@multiarch-building needs: [extra-args] secrets: inherit + strategy: + matrix: + arch: + - {id: amd64, builder-label: ubuntu-22.04, tester-label: large} +# - {id: arm64, builder-label: ARM64, tester-label: ARM64} with: + identifier: ${{matrix.arch.id}} provider: lxd juju-channel: 3.3/stable extra-arguments: ${{needs.extra-args.outputs.args}} load-test-enabled: false + builder-runner-label: ${{matrix.arch.builder-label}} modules: '["test_k8s", "test_etcd"]' zap-enabled: false - self-hosted-runner: true - self-hosted-runner-label: "large" + # self-hosted-runner: true + # self-hosted-runner-label: ${{matrix.arch.tester-label}} trivy-fs-enabled: true trivy-image-config: "trivy.yaml" tmate-debug: true test-timeout: 120 test-tox-env: integration + working-directory: ./charms/ diff --git a/.github/workflows/publish-charms.yaml b/.github/workflows/publish-charms.yaml index ad0962c4..8c197afc 100644 --- a/.github/workflows/publish-charms.yaml +++ b/.github/workflows/publish-charms.yaml @@ -49,10 +49,11 @@ jobs: - { path: ./charms/worker/, tagPrefix: k8s-worker } arch: - amd64 + - arm64 secrets: inherit with: channel: ${{needs.configure-channel.outputs.track}}/${{needs.configure-channel.outputs.risk}} working-directory: ${{ matrix.charm.path }} charmcraft-channel: ${{ needs.charmcraft-channel.outputs.channel }} tag-prefix: ${{ matrix.charm.tagPrefix }} - identifier: ${{ matrix.arch}}-k8s + identifier: ${{matrix.arch}}-k8s From a23d0df4062aec7a1c4438f259d4adfac32512dc Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Thu, 9 May 2024 14:46:31 -0500 Subject: [PATCH 08/14] Switch back to using main branch of operator-workflows --- .github/workflows/integration_test.yaml | 2 +- .../integration_test_betterbuild.yaml | 66 ------------------- .github/workflows/promote-charms.yaml | 1 + 3 files changed, 2 insertions(+), 67 deletions(-) delete mode 100644 .github/workflows/integration_test_betterbuild.yaml diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 1a9d15e8..c9487d2a 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -5,7 +5,6 @@ on: pull_request: jobs: - extra-args: runs-on: ubuntu-latest outputs: @@ -41,6 +40,7 @@ jobs: matrix: arch: - {id: amd64, builder-label: ubuntu-22.04, tester-arch: x64} + #- {id: arm64, builder-label: ARM64, tester-label: ARM64} suite: ["k8s", "etcd", "ceph"] with: identifier: ${{ matrix.arch.id }}-${{ matrix.suite }} diff --git a/.github/workflows/integration_test_betterbuild.yaml b/.github/workflows/integration_test_betterbuild.yaml deleted file mode 100644 index e69c4417..00000000 --- a/.github/workflows/integration_test_betterbuild.yaml +++ /dev/null @@ -1,66 +0,0 @@ - -name: Integration tests - -on: - pull_request: - -jobs: - extra-args: - runs-on: ubuntu-latest - outputs: - args: ${{ steps.flags.outputs.args }} - steps: - - name: Determine extra args - id: flags - env: - TITLE: ${{ github.event.pull_request.title }} - JOB: ${{ github.job }} - WORKFLOW: ${{ github.workflow }} - run: | - EXTRA_ARGS="--crash-dump=on-failure" - if [[ "$TITLE" == *"[COS]"* ]]; then - EXTRA_ARGS="$EXTRA_ARGS --cos" - fi - echo "args=$EXTRA_ARGS" >> "$GITHUB_OUTPUT" - - # build-all-charms: - # needs: [extra-args] - # strategy: - # matrix: - # arch: - # - arch: amd64 - # runs-on-label: X64 - # - arch: arm64 - # runs-on-label: ARM64 - # uses: ./.github/workflows/build-charm.yaml - # with: - # working-directory: "./charms/worker/" - # arch: ${{ matrix.arch.arch }} - # runs-on-label: ${{ matrix.arch.runs-on-label }} - - integration-tests: - uses: addyess/operator-workflows/.github/workflows/integration_test.yaml@multiarch-building - needs: [extra-args] - secrets: inherit - strategy: - matrix: - arch: - - {id: amd64, builder-label: ubuntu-22.04, tester-label: large} -# - {id: arm64, builder-label: ARM64, tester-label: ARM64} - with: - identifier: ${{matrix.arch.id}} - provider: lxd - juju-channel: 3.3/stable - extra-arguments: ${{needs.extra-args.outputs.args}} - load-test-enabled: false - builder-runner-label: ${{matrix.arch.builder-label}} - modules: '["test_k8s", "test_etcd"]' - zap-enabled: false - # self-hosted-runner: true - # self-hosted-runner-label: ${{matrix.arch.tester-label}} - trivy-fs-enabled: true - trivy-image-config: "trivy.yaml" - tmate-debug: true - test-timeout: 120 - test-tox-env: integration - working-directory: ./charms/ diff --git a/.github/workflows/promote-charms.yaml b/.github/workflows/promote-charms.yaml index 99ea66cb..d53978a4 100644 --- a/.github/workflows/promote-charms.yaml +++ b/.github/workflows/promote-charms.yaml @@ -69,6 +69,7 @@ jobs: charm-directory: ${{ fromJson(needs.select-charms.outputs.charms) }} arch: - amd64 + - arm64 uses: canonical/operator-workflows/.github/workflows/promote_charm.yaml@main with: base-architecture: ${{ matrix.arch }} From e96e92236702ca8cfb7b8b8f91a1e605e7b6219e Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Tue, 14 May 2024 16:35:05 +0200 Subject: [PATCH 09/14] re-enable arm64 builds --- .github/workflows/integration_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index c9487d2a..2b90dd92 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -40,7 +40,7 @@ jobs: matrix: arch: - {id: amd64, builder-label: ubuntu-22.04, tester-arch: x64} - #- {id: arm64, builder-label: ARM64, tester-label: ARM64} + - {id: arm64, builder-label: ARM64, tester-label: ARM64} suite: ["k8s", "etcd", "ceph"] with: identifier: ${{ matrix.arch.id }}-${{ matrix.suite }} From 4641fe708e739a478c2ec6d9df1f7624ff544545 Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Thu, 16 May 2024 08:59:19 +0200 Subject: [PATCH 10/14] Set arch constraint on every application in a bundle --- tests/integration/conftest.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 8e451f21..c39cdb96 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -184,6 +184,19 @@ def switch(self, name: str, path: Path): app["charm"] = str(path.resolve()) app["channel"] = None + def set_arch(self, arch: str): + """Set the architecture of each application in the bundle. + + Args: + arch (str): Which architecture + """ + for name in self.applications: + app = self.applications[name] + constraint_list = app.get("constraints") or "" + constraints = dict(constraint.split("=") for constraint in constraint_list) + constraints["arch"] = arch + app["constraints"] = " ".join(f"{k}={v}" for k, v in constraints.items()) + def drop_constraints(self): """Remove constraints on applications. Useful for testing on lxd.""" for app in self.applications.values(): @@ -360,7 +373,7 @@ async def kubernetes_cluster(request: pytest.FixtureRequest, ops_test: OpsTest): bundle.add_constraints({"virt-type": "virtual-machine"}) if request.config.option.apply_proxy: await cloud_proxied(ops_test) - + bundle.set_arch(arch) for path, charm in zip(charm_files, charms): bundle.switch(charm.app_name, path) async with deploy_model(request, ops_test, model, bundle) as the_model: From 4c42ca41ede8d59d8529a0127b6386004b2505e7 Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Thu, 23 May 2024 07:58:03 -0500 Subject: [PATCH 11/14] Update the charm contribution guide --- charms/CONTRIBUTION.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/charms/CONTRIBUTION.md b/charms/CONTRIBUTION.md index 8b571d08..81ad55ed 100644 --- a/charms/CONTRIBUTION.md +++ b/charms/CONTRIBUTION.md @@ -7,7 +7,6 @@ The `k8s` and `k8s-worker` charms are noticeably tucked into one-another. ``` └── worker ├── charmcraft.yaml - ├── requirements.txt └── k8s ├── charmcraft.yaml ├── lib @@ -25,15 +24,14 @@ The unique parts of the charm are what are in each charm's top-level directory: ``` charmcraft.yaml -config.yaml -actions.yaml -metadata.yaml -requirements.yaml +.jujuignore +icon.svg +README.md ``` In order to exclude the `k8s` exclusive components from the `k8s-worker` charm, charmcraft will read the `worker/.jujuignore` file to determine what to leave out of the final charm. -### What's not +### What's shared The shared portions of each charm are within `worker/k8s` (except for the above mentioned exclusions). This includes shared libraries from `worker/k8s/lib`, shared source from `worker/k8s/src`, shared python dependencies from `worker/k8s/requirements.txt` From aa3d6f8181cb4c4df2be412458aca14fa55d49f7 Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Thu, 23 May 2024 12:52:23 -0500 Subject: [PATCH 12/14] Appropriately add arch to constraints --- tests/integration/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index c39cdb96..f43344c0 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -193,9 +193,9 @@ def set_arch(self, arch: str): for name in self.applications: app = self.applications[name] constraint_list = app.get("constraints") or "" - constraints = dict(constraint.split("=") for constraint in constraint_list) + constraints = dict(constraint.split("=") for constraint in constraint_list.split()) constraints["arch"] = arch - app["constraints"] = " ".join(f"{k}={v}" for k, v in constraints.items()) + app["constraints"] = " ".join(f"{k}={v}" for k, v in sorted(constraints.items())) def drop_constraints(self): """Remove constraints on applications. Useful for testing on lxd.""" From 7ffcab035f46e66ea01b9d7e11c4d22265849e52 Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Fri, 25 Oct 2024 22:19:08 -0500 Subject: [PATCH 13/14] Adapt to latest workflow merge --- .github/workflows/integration_test.yaml | 2 +- tests/integration/conftest.py | 27 ++++++++++++------------- tox.ini | 2 +- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 2b90dd92..8f9a8ba6 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -40,7 +40,7 @@ jobs: matrix: arch: - {id: amd64, builder-label: ubuntu-22.04, tester-arch: x64} - - {id: arm64, builder-label: ARM64, tester-label: ARM64} + - {id: arm64, builder-label: ARM64, tester-arch: ARM64} suite: ["k8s", "etcd", "ceph"] with: identifier: ${{ matrix.arch.id }}-${{ matrix.suite }} diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index f43344c0..90c8fa0e 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -184,19 +184,6 @@ def switch(self, name: str, path: Path): app["charm"] = str(path.resolve()) app["channel"] = None - def set_arch(self, arch: str): - """Set the architecture of each application in the bundle. - - Args: - arch (str): Which architecture - """ - for name in self.applications: - app = self.applications[name] - constraint_list = app.get("constraints") or "" - constraints = dict(constraint.split("=") for constraint in constraint_list.split()) - constraints["arch"] = arch - app["constraints"] = " ".join(f"{k}={v}" for k, v in sorted(constraints.items())) - def drop_constraints(self): """Remove constraints on applications. Useful for testing on lxd.""" for app in self.applications.values(): @@ -217,6 +204,18 @@ def add_constraints(self, constraints: Dict[str, str]): existing.update(constraints) app["constraints"] = " ".join(f"{k}={v}" for k, v in existing.items()) + def add_constraints(self, constraints: Dict[str, str]): + """Add constraints to applications. + + Args: + constraints: Mapping of constraints to add to applications. + """ + for app in self.applications.values(): + val: str = app["constraints"] + existing = dict(kv.split("=", 1) for kv in val.split()) + existing.update(constraints) + app["constraints"] = " ".join(f"{k}={v}" for k, v in existing.items()) + async def cloud_arch(ops_test: OpsTest) -> str: """Return current architecture of the selected controller @@ -373,7 +372,7 @@ async def kubernetes_cluster(request: pytest.FixtureRequest, ops_test: OpsTest): bundle.add_constraints({"virt-type": "virtual-machine"}) if request.config.option.apply_proxy: await cloud_proxied(ops_test) - bundle.set_arch(arch) + bundle.add_constraints({"arch": arch}) for path, charm in zip(charm_files, charms): bundle.switch(charm.app_name, path) async with deploy_model(request, ops_test, model, bundle) as the_model: diff --git a/tox.ini b/tox.ini index 2ba264f9..9241a42b 100644 --- a/tox.ini +++ b/tox.ini @@ -53,7 +53,7 @@ deps = types-PyYAML types-requests -r{toxinidir}/test_requirements.txt - -r{toxinidir}/charms/worker/requirements.txt + -r{toxinidir}/charms/worker/k8s/requirements.txt commands = pydocstyle {[vars]src_path} codespell {toxinidir} --skip {toxinidir}/.git --skip {toxinidir}/.tox \ From c71c624571241d9862297dadb70c72ce2672bbc3 Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Sat, 26 Oct 2024 07:05:28 -0500 Subject: [PATCH 14/14] add arch constraint to each application --- .github/workflows/integration_test.yaml | 8 ++- .../k8s/templates/snap_installation.yaml | 2 + tests/integration/conftest.py | 66 +++++++++++++------ 3 files changed, 53 insertions(+), 23 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 8f9a8ba6..bec1cbac 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -41,12 +41,16 @@ jobs: arch: - {id: amd64, builder-label: ubuntu-22.04, tester-arch: x64} - {id: arm64, builder-label: ARM64, tester-arch: ARM64} - suite: ["k8s", "etcd", "ceph"] + suite: [k8s, etcd, ceph] + exclude: + - {arch: {id: arm64}, suite: ceph} with: identifier: ${{ matrix.arch.id }}-${{ matrix.suite }} builder-runner-label: ${{ matrix.arch.builder-label }} charmcraft-channel: ${{ needs.charmcraft-channel.outputs.channel }} - extra-arguments: ${{needs.extra-args.outputs.args}} -k test_${{ matrix.suite }} + extra-arguments: >- + ${{needs.extra-args.outputs.args}} -k test_${{ matrix.suite }} + ${{ matrix.arch.id == 'arm64' && ' --lxd-containers' || '' }} juju-channel: 3/stable load-test-enabled: false provider: lxd diff --git a/charms/worker/k8s/templates/snap_installation.yaml b/charms/worker/k8s/templates/snap_installation.yaml index 31244c6d..f8528062 100644 --- a/charms/worker/k8s/templates/snap_installation.yaml +++ b/charms/worker/k8s/templates/snap_installation.yaml @@ -5,7 +5,9 @@ amd64: - name: k8s install-type: store channel: edge + classic: true arm64: - name: k8s install-type: store channel: edge + classic: true \ No newline at end of file diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 90c8fa0e..ffd7857a 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -6,6 +6,7 @@ import contextlib import json import logging +import re import shlex from dataclasses import dataclass, field from itertools import chain @@ -92,6 +93,23 @@ class Charm: arch: str path: Path _charmfile: Optional[Path] = None + _URL_RE = re.compile(r"ch:(?P\w+)/(?P\w+)/(?P.+)") + + @staticmethod + def craft_url(charm: str, series: str, arch: str) -> str: + """Craft a charm URL. + + Args: + charm: Charm name + series: Cloud series + arch: Cloud architecture + + Returns: + string: URL to the charm + """ + if m := Charm._URL_RE.match(charm): + charm = m.group("charm") + return f"ch:{arch}/{series}/{charm}" @property def metadata(self) -> dict: @@ -145,19 +163,25 @@ class Bundle: ops_test: Instance of the pytest-operator plugin path: Path to the bundle file content: Loaded content from the path + arch: Cloud Architecture render: Path to a rendered bundle applications: Mapping of applications in the bundle. """ ops_test: OpsTest path: Path + arch: str _content: Mapping = field(default_factory=dict) @property def content(self) -> Mapping: """Yaml content of the bundle loaded into a dict""" if not self._content: - self._content = yaml.safe_load(self.path.read_bytes()) + loaded = yaml.safe_load(self.path.read_bytes()) + series = loaded.get("series", "focal") + for app in loaded["applications"].values(): + app["charm"] = Charm.craft_url(app["charm"], series=series, arch=self.arch) + self._content = loaded return self._content @property @@ -168,6 +192,7 @@ def applications(self) -> Mapping[str, dict]: @property def render(self) -> Path: """Path to written bundle config to be deployed.""" + self.add_constraints({"arch": self.arch}) target = self.ops_test.tmp_path / "bundles" / self.path.name target.parent.mkdir(exist_ok=True, parents=True) yaml.safe_dump(self.content, target.open("w")) @@ -204,18 +229,6 @@ def add_constraints(self, constraints: Dict[str, str]): existing.update(constraints) app["constraints"] = " ".join(f"{k}={v}" for k, v in existing.items()) - def add_constraints(self, constraints: Dict[str, str]): - """Add constraints to applications. - - Args: - constraints: Mapping of constraints to add to applications. - """ - for app in self.applications.values(): - val: str = app["constraints"] - existing = dict(kv.split("=", 1) for kv in val.split()) - existing.update(constraints) - app["constraints"] = " ".join(f"{k}={v}" for k, v in existing.items()) - async def cloud_arch(ops_test: OpsTest) -> str: """Return current architecture of the selected controller @@ -338,14 +351,26 @@ async def deploy_model( log.fatal("Failed to determine model: model_name=%s", model_name) +def bundle_file(request) -> Path: + """Fixture to get bundle file. + + Args: + request: pytest request object + + Returns: + path to test's bundle file + """ + _file = "test-bundle.yaml" + bundle_marker = request.node.get_closest_marker("bundle_file") + if bundle_marker: + _file = bundle_marker.args[0] + return Path(__file__).parent / "data" / _file + + @pytest_asyncio.fixture(scope="module") async def kubernetes_cluster(request: pytest.FixtureRequest, ops_test: OpsTest): """Deploy local kubernetes charms.""" - bundle_file = "test-bundle.yaml" - bundle_marker = request.node.get_closest_marker("bundle_file") - if bundle_marker: - bundle_file = bundle_marker.args[0] - bundle_path = Path(__file__).parent / "data" / bundle_file + bundle_path = bundle_file(request) model = "main" with ops_test.model_context(model) as the_model: @@ -362,7 +387,7 @@ async def kubernetes_cluster(request: pytest.FixtureRequest, ops_test: OpsTest): charm_files = await asyncio.gather( *[charm.resolve(request.config.option.charm_files) for charm in charms] ) - bundle = Bundle(ops_test, bundle_path) + bundle = Bundle(ops_test, bundle_path, arch) _type, _vms = await cloud_type(ops_test) if _type == "lxd" and not _vms: log.info("Drop lxd machine constraints") @@ -372,7 +397,6 @@ async def kubernetes_cluster(request: pytest.FixtureRequest, ops_test: OpsTest): bundle.add_constraints({"virt-type": "virtual-machine"}) if request.config.option.apply_proxy: await cloud_proxied(ops_test) - bundle.add_constraints({"arch": arch}) for path, charm in zip(charm_files, charms): bundle.switch(charm.app_name, path) async with deploy_model(request, ops_test, model, bundle) as the_model: @@ -388,7 +412,7 @@ async def grafana_agent(kubernetes_cluster: Model): machine_series = juju.utils.get_version_series(data["base"].split("@")[1]) await kubernetes_cluster.deploy( - f"ch:{machine_arch}/{machine_series}/grafana-agent", + Charm.craft_url("grafana-agent", machine_series, machine_arch), channel="stable", series=machine_series, )