Rust Hydration (#201)

* Update Docker with Rust tooling, split into database and pypgstac as separate images * update pypgstac to add rust hydration module * cleanup * add private json object to items and collections. add geometry, datetime, and end_datetime generated fields to collections * update changelog * update ci * update workflow for release * fix test on incremental migrations * switch from serde to pyo3 * fixes from PR review * fix: remove cargo registry This fixes a permissions error when running the test scripts as non-root. * deps: track Cargo.lock * fix: small rust cleanups --------- Co-authored-by: Pete Gadomski <[email protected]>
stac-utils · Sep 19, 2023 · 90d4364 · 90d4364
1 parent 45b1f96
commit 90d4364
Show file tree

Hide file tree

Showing 58 changed files with 6,296 additions and 514 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -6,6 +6,7 @@
 venv/*
 */.direnv/*
 */.ruff_cache/*
+*/.pytest_cache/*
 */.vscode/*
 */.mypy_cache/*
 */.pgadmin/*
@@ -14,5 +15,5 @@ venv/*
 */.github/*
 */env/*
 Dockerfile
-docker-compose.yml
+docker compose.yml
 */.devcontainer/*
diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml
@@ -12,21 +12,224 @@ env:
   DOCKER_BUILDKIT: 1
 
 jobs:
-  test:
-    name: test
+  changes:
     runs-on: ubuntu-latest
+    permissions:
+      pull-requests: read
+    outputs:
+      pgdocker: ${{ steps.check.outputs.pgtag }}
+      buildpgdocker: ${{ steps.check.outputs.buildpg }}
+      pyrustdocker: ${{ steps.check.outputs.pytag }}
+      buildpyrustdocker: ${{ steps.check.outputs.buildpy }}
+
+    steps:
+    - uses: actions/checkout@v3
+    - uses: dorny/paths-filter@v2
+      id: filter
+      with:
+        filters: |
+          pgstac:
+            - 'docker/pgstac/**'
+          pypgstac:
+            - 'docker/pypgstac/**'
+    - id: check
+      run: |
+        buildpg=false;
+        ref=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}};
+        [[ "${{ steps.filter.outputs.pgstac }}" == "true" ]] && buildpg=true || ref=main;
+        echo "pgtag=${{ env.REGISTRY }}/stac-utils/pgstac-postgres:$ref" >>$GITHUB_OUTPUT;
+        echo "buildpg=$buildpg" >>$GITHUB_OUTPUT;
+        buildy=false;
+        ref=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}};
+        [[ "${{ steps.filter.outputs.pypgstac }}" == "true" ]] && buildpy=true || ref=main;
+        echo "pytag=${{ env.REGISTRY }}/stac-utils/pgstac-pyrust:$ref" >>$GITHUB_OUTPUT;
+        echo "buildpy=$buildpg" >>$GITHUB_OUTPUT;
+
+  buildpg:
+    name: Build and push base postgres
+    if: ${{ needs.changes.outputs.buildpgdocker == 'true' }}
+    runs-on: ubuntu-latest
+    needs: [changes]
     steps:
       - uses: actions/checkout@v3
-      - uses: docker/setup-buildx-action@v1
-      - name: builder
-        id: builder
-        uses: docker/build-push-action@v2
+      - uses: docker/setup-buildx-action@v2
+      - name: Log in to the Container registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build and Push Base Postgres
+        uses: docker/build-push-action@v4
         with:
           context: .
-          load: true
-          push: false
+          target: pgstacbase-plrust
+          file: docker/pgstac/Dockerfile
+          tags: ${{ needs.changes.outputs.pgdocker }}
+          push: true
           cache-from: type=gha
           cache-to: type=gha, mode=max
 
+  buildpyrust:
+    name: Build and push base pyrust
+    if: ${{ needs.changes.outputs.buildpyrustdocker == 'true' }}
+    runs-on: ubuntu-latest
+    needs: [changes]
+    steps:
+      - uses: actions/checkout@v3
+      - uses: docker/setup-buildx-action@v2
+      - name: Log in to the Container registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build and Push Base pyrust
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          target: pyrustbase
+          file: docker/pypgstac/Dockerfile
+          tags: ${{ needs.changes.outputs.pyrustdocker }}
+          push: true
+          cache-from: type=gha
+          cache-to: type=gha, mode=max
+
+  test:
+    name: test
+    needs: [changes, buildpg, buildpyrust]
+    runs-on: ubuntu-latest
+    container:
+      image:  ${{ needs.changes.outputs.pyrustdocker }}
+      env:
+        PGPASSWORD: postgres
+        PGHOST: postgres
+        PGDATABASE: postgres
+        PGUSER: postgres
+
+    services:
+      postgres:
+        env:
+          POSTGRES_PASSWORD: postgres
+        image: ${{ needs.changes.outputs.pgdocker }}
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install pypgstac
+        working-directory: /__w/pgstac/pgstac/src/pypgstac
+        run:  |
+          pip install .[dev,test,psycopg];
+
       - name: Run tests
-        run: docker run --rm ${{ steps.builder.outputs.imageid }}  test
+        working-directory: /__w/pgstac/pgstac/docker/pypgstac/bin
+        run:  |
+          ./test
+
+  linux:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        target: [x86_64, x86, aarch64, armv7, s390x, ppc64le]
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          working-directory: src/pypgstac
+          target: ${{ matrix.target }}
+          args: --release --out /home/runner/work/pgstac/pgstac/dist
+          sccache: 'true'
+          manylinux: auto
+      - name: Upload wheels
+        uses: actions/upload-artifact@v3
+        with:
+          name: wheels
+          path: /home/runner/work/pgstac/pgstac/dist/*
+
+  windows:
+    runs-on: windows-latest
+    strategy:
+      matrix:
+        target: [x64, x86]
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+          architecture: ${{ matrix.target }}
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          working-directory: src/pypgstac
+          target: ${{ matrix.target }}
+          args: --release --out /home/runner/work/pgstac/pgstac/dist
+          sccache: 'true'
+      - name: Upload wheels
+        uses: actions/upload-artifact@v3
+        with:
+          name: wheels
+          path: /home/runner/work/pgstac/pgstac/dist/*
+
+  macos:
+    runs-on: macos-latest
+    strategy:
+      matrix:
+        target: [x86_64, aarch64]
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          working-directory: src/pypgstac
+          target: ${{ matrix.target }}
+          args: --release --out /tmp/dist
+          sccache: 'true'
+      - name: Upload wheels
+        uses: actions/upload-artifact@v3
+        with:
+          name: wheels
+          path: /tmp/dist/*
+
+  sdist:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Build sdist
+        uses: PyO3/maturin-action@v1
+        with:
+          working-directory: src/pypgstac
+          command: sdist
+          args: --out /home/runner/work/pgstac/pgstac/dist
+      - name: Upload sdist
+        uses: actions/upload-artifact@v3
+        with:
+          name: wheels
+          path: /home/runner/work/pgstac/pgstac/dist/*
+
+  # release:
+  #   name: Release
+  #   runs-on: ubuntu-latest
+  #   if: "startsWith(github.ref, 'refs/tags/')"
+  #   needs: [linux, windows, macos, sdist]
+  #   steps:
+  #     - uses: actions/download-artifact@v3
+  #       with:
+  #         name: wheels
+  #     - name: Publish to PyPI
+  #       uses: PyO3/maturin-action@v1
+  #       env:
+  #         MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
+  #       with:
+  #         command: upload
+  #         args: --skip-existing *