Merge pull request #2868 from devitocodes/JDBetteridge/unique_ci_docker_tag

JDBetteridge/Composite actions

Author: mloubout

.github/actions/docker-build/action.yaml

@@ -0,0 +1,50 @@
+name: Docker build action
+description: Composite action for building Devito Docker containers
+author: "Devito"
+
+inputs:
+# The only supported GHA input type is string
+  file:
+    description: "Dockerfile containing build instructions"
+    required: true
+    default: Dockerfile
+  tag:
+    description: "Tag to add to the built image"
+    required: true
+  base:
+    description: "Base docker image to build on top of"
+    default: ""
+  args:
+    description: "Arguments to pass to `docker build`"
+    required: true
+    default: ""
+
+outputs:
+  unique:
+    description: "Unique identifier for the CI run"
+    value: ${{ steps.uniquetag.outputs.unique }}
+
+runs:
+  using: "composite"
+  steps:
+    - id: uniquetag
+      name: "Generate unique CI tag"
+      shell: bash
+      run: |
+        UNIQUE=$(echo "${GITHUB_RUN_ID}_${GITHUB_RUN_ATTEMPT}" | cksum | cut -f 1 -d " ")
+        echo "Unique ID: ${UNIQUE}"
+        echo "unique=${UNIQUE}" >> "$GITHUB_OUTPUT"
+
+    - id: dockerbuild
+      name: "Build docker container"
+      shell: bash
+      env:
+        BASE: ${{ inputs.base }}
+      run: |
+        docker build \
+          --pull \
+          --file ${{ inputs.file }} \
+          --tag ${{ inputs.tag }}_${{ steps.uniquetag.outputs.unique }} \
+          ${BASE:+--build-arg base=$BASE} \
+          ${{ inputs.args }} \
+          .

.github/actions/docker-clean/action.yaml

@@ -0,0 +1,21 @@
+name: Docker cleanup action
+description: Composite action for removing Docker images
+author: "Devito"
+
+inputs:
+# The only supported GHA input type is string
+  uid:
+    description: "Unique identifier output from docker-build action"
+    required: true
+  tag:
+    description: "Tag of the built image to use"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - id: dockerclean
+      name: "Cleanup docker image"
+      shell: bash
+      run: |
+          docker image rm -f "${{ inputs.tag }}_${{ inputs.uid }}"

.github/actions/docker-run/action.yaml

@@ -0,0 +1,61 @@
+name: Docker run action
+description: Composite action for running commands in Docker containers
+author: "Devito"
+
+inputs:
+# The only supported GHA input type is string
+  uid:
+    description: "Unique identifier output from docker-build action"
+    required: true
+  tag:
+    description: "Tag of the built image to use"
+    required: true
+  name:
+    description: "Name substring for docker to use when running the command"
+    default: ""
+  args:
+    description: "Arguments to pass to `docker run`, `--init -t --rm` are always added"
+    required: true
+    default: ""
+  env:
+    description: "Environment variables to set inside the docker container, one environment variable per line"
+    required: true
+    default: ""
+  command:
+    description: "Command to execute inside of the docker container"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - id: processenv
+      name: Process environment variable list
+      shell: bash
+      env:
+        ENV_INPUT: ${{ inputs.env }}
+      run: |
+        ENV_STRING=""
+        # Read line by line with here string, safely handling spaces within the values
+        while IFS= read -r LINE; do
+          if [[ -n "$LINE" ]]; then
+            ENV_STRING="$ENV_STRING --env $LINE"
+          fi
+        done <<< "$ENV_INPUT"
+        # Remove the leading space from the first concatenation
+        ENV_STRING="${ENV_STRING# }"
+        echo "docker_environment_args=$ENV_STRING" >> "$GITHUB_OUTPUT"
+
+    - id: dockerrun
+      name: "Run command ${{ inputs.command }} in ${{ inputs.tag }} docker container"
+      shell: bash
+      env:
+        NAME: ${{ input.name }}
+      run: |
+        docker run \
+          --init -t --rm \
+          ${{ inputs.args }} \
+          --name "ci-${NAME:-${{ inputs.tag }}}-${{ inputs.uid }}" \
+          --env-file=docker/coverage.env \
+          ${{ steps.processenv.outputs.docker_environment_args }} \
+          "${{ inputs.tag }}_${{ inputs.uid }}" \
+          ${{ inputs.command }}

.github/actions/readme.md

@@ -0,0 +1,109 @@
+# Devito Actions
+
+The Devito actions can either be used from the Github interface:
+```yaml
+uses: devitocodes/devito/.github/actions/docker-build@main
+```
+
+Or internally from within the repository:
+```yaml
+uses: ./.github/actions/docker-run
+```
+
+## `docker-build`
+
+Inputs:
+
+- `file`: Dockerfile containing build instructions (default: `Dockerfile`)
+- `tag`: Tag to add to the built image
+- `base`: Base docker image to build on top of
+- `args`: Arguments to pass to `docker build`
+
+Outputs:
+
+- `unique`: Unique identifier for the CI run (ie: `${{ steps.build.outputs.unique }}`)
+
+Example:
+
+```yaml
+jobs:
+  test:
+    steps:
+    - id: build
+      name: Build docker image for Devito
+      uses: ./.github/actions/docker-build
+      with:
+        file: docker/Dockerfile.devito
+        tag: tag-related-to-test-config
+        base: base-image-to-build-from
+        args: "--more-docker-build-args"
+```
+
+## `docker-run`
+
+Inputs:
+
+- `uid`: Unique identifier output from docker-build action
+- `tag`: Tag of the built image to use
+- `name`: Name substring for docker to use when running the command (optional)
+- `args`: Arguments to pass to `docker run`, `--init -t --rm` are always added
+- `env`: Environment variables to set inside the docker container, one environment variable per line
+-command: Command to execute inside of the docker container
+
+### Notes
+
+- The UID must be unique, easily obtained from build action
+- The tag must match built image, easily obtained from build action
+- If you provide a custom name `foo` the container name will be `ci-foo-UUUUUUUUUU` where UUUUUUUUUU is the UID
+- The default args `--init -t --rm` are _always_ added
+- Environment variables must be passed a single environment variable per line, best achieved with the (`|`) syntax in yaml
+- Only a single command is executed, not a list of commands. Using `;` or `&&` will result in subsequent commands being executed outside of the docker environment
+
+Example:
+
+```yaml
+jobs:
+  test:
+    steps:
+    - name: Run a command in a previously built Docker container
+      uses: ./.github/actions/docker-run
+      with:
+        uid: ${{ steps.build.outputs.unique }}
+        tag: tag-related-to-test-config
+        name: completely-optional-name
+        args: "--more-docker-run-args"
+        env: |
+          FOO=value1
+          BAR=value2
+        command: |
+          mpiexec -n 4 \
+            python \
+              my_complicated_script.py --arg1 -v
+```
+
+## `docker-clean`
+
+Inputs:
+
+- `uid`: Unique identifier output from docker-build action
+- `tag`: Tag of the built image to use
+
+### Notes
+
+- UID must be unique, easily obtained from build action
+- Tag must match built image, easily obtained from build action
+- Use `if: always()` to always clean up the image, even if the workflow fails
+
+Example:
+
+```yaml
+jobs:
+  test:
+    steps:
+    - name: Cleanup Docker image
+      if: always()
+      uses: ./.github/actions/docker-clean
+      with:
+        uid: ${{ steps.build.outputs.unique }}
+        tag: tag-related-to-test-config
+```

.github/workflows/docker-devito.yaml

@@ -178,7 +178,7 @@ jobs:
         run: |
          docker run ${{ matrix.flag }} --rm -t --name "${CONTAINER_NAME}" \
            devitocodes/devito:${{ matrix.tag }}-dev \
-           pytest ${{ matrix.test }}
+           pytest -v ${{ matrix.test }}
 
   deploy-devito-manifest:
     needs: deploy-devito
@@ -292,4 +292,4 @@ jobs:
         run: |
          docker run ${{ matrix.flag }} --rm -t --name "${CONTAINER_NAME}" \
            devitocodes/devito:${{ matrix.tag }}-dev \
-           pytest ${{ matrix.test }}
+           pytest -v ${{ matrix.test }}

.github/workflows/examples-mpi.yaml

@@ -14,8 +14,7 @@ concurrency:
   cancel-in-progress: true
 
 on:
-  # Trigger the workflow on push or pull request,
-  # but only for the main branch
+  # Trigger the workflow on push or pull request, but only for the main branch
   push:
     branches:
       - main
@@ -72,15 +71,17 @@ jobs:
         ipcluster start --profile=mpi --engines=mpi -n 4 --daemonize
         # A few seconds to ensure workers are ready
         sleep 10
-        py.test --nbval examples/mpi
+        pytest -v --nbval examples/mpi
         ipcluster stop --profile=mpi
 
     - name: Test seismic examples
       run: |
-        mpirun ${{ matrix.mpiarg }} pytest examples/seismic/tti/tti_example.py
-        mpirun ${{ matrix.mpiarg }} pytest examples/seismic/elastic/elastic_example.py
-        mpirun ${{ matrix.mpiarg }} pytest examples/seismic/viscoacoustic/viscoacoustic_example.py
-        mpirun ${{ matrix.mpiarg }} pytest examples/seismic/viscoelastic/viscoelastic_example.py
+        mpirun ${{ matrix.mpiarg }} \
+          pytest -v \
+            examples/seismic/tti/tti_example.py \
+            examples/seismic/elastic/elastic_example.py \
+            examples/seismic/viscoacoustic/viscoacoustic_example.py \
+            examples/seismic/viscoelastic/viscoelastic_example.py
 
     - name: Test fwi examples with mpi
       run: |

.github/workflows/examples.yaml

@@ -8,8 +8,7 @@ concurrency:
   cancel-in-progress: true
 
 on:
-  # Trigger the workflow on push or pull request,
-  # but only for the main branch
+  # Trigger the workflow on push or pull request, but only for the main branch
   push:
     branches:
       - main
@@ -32,7 +31,7 @@ jobs:
       DEVITO_LANGUAGE: "openmp"
 
     strategy:
-      # Prevent all build to stop if a single one fails
+      # Prevent cancellation if a single workflow fails
       fail-fast: false
 
     steps:

.github/workflows/lint.yaml

@@ -7,8 +7,7 @@ concurrency:
   cancel-in-progress: true
 
 on:
-   # Trigger the workflow on push or pull request,
-   # but only for the main branch
+   # Trigger the workflow on push or pull request, but only for the main branch
    push:
      branches:
        - main