From 3530a97c4794f3ee11ddba5676eb4b5ce8c6d5cf Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Thu, 29 Jul 2021 16:57:22 +0200 Subject: [PATCH] Cache backend API example Signed-off-by: CrazyMax --- .github/workflows/ci.yml | 88 ++++++++++++++++++++----------- README.md | 7 --- docs/advanced/cache.md | 111 +++++++++++++++++++++++++++++++++++---- 3 files changed, 159 insertions(+), 47 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bd30bed..063a36d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,9 +70,6 @@ jobs: name: Inspect run: | docker buildx imagetools inspect localhost:5000/name/app:1.0.0 - - - name: Image digest - run: echo ${{ steps.docker_build.outputs.digest }} - name: Check digest run: | @@ -133,9 +130,6 @@ jobs: name: Inspect run: | docker buildx imagetools inspect localhost:5000/name/app:1.0.0 - - - name: Image digest - run: echo ${{ steps.docker_build.outputs.digest }} - name: Check digest run: | @@ -191,9 +185,6 @@ jobs: name: Inspect run: | docker buildx imagetools inspect localhost:5000/name/app:1.0.0 - - - name: Image digest - run: echo ${{ steps.docker_build.outputs.digest }} - name: Check digest run: | @@ -392,9 +383,6 @@ jobs: name: Inspect run: | docker buildx imagetools inspect localhost:5000/name/app:1.0.0 - - - name: Image digest - run: echo ${{ steps.docker_build.outputs.digest }} - name: Check digest run: | @@ -447,9 +435,6 @@ jobs: name: Inspect (1) run: | docker buildx imagetools inspect localhost:5000/name/app:latest - - - name: Image digest (1) - run: echo ${{ steps.docker_build.outputs.digest }} - name: Check digest (1) run: | @@ -480,9 +465,6 @@ jobs: name: Inspect (2) run: | docker buildx imagetools inspect localhost:5000/name/app:latest - - - name: Image digest (2) - run: echo ${{ steps.docker_build2.outputs.digest }} - name: Check digest (2) run: | @@ -503,7 +485,7 @@ jobs: if: always() uses: crazy-max/ghaction-dump-context@v1 - github-cache-first: + local-cache-first: runs-on: ubuntu-latest outputs: digest: ${{ steps.docker_build.outputs.digest }} @@ -531,7 +513,7 @@ jobs: uses: actions/cache@v2 with: path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-ghcache-${{ github.sha }} + key: ${{ runner.os }}-buildx-local-${{ github.sha }} restore-keys: | ${{ runner.os }}-buildx-ghcache- - @@ -557,9 +539,6 @@ jobs: name: Inspect run: | docker buildx imagetools inspect localhost:5000/name/app:1.0.0 - - - name: Image digest - run: echo ${{ steps.docker_build.outputs.digest }} - name: Check digest run: | @@ -572,9 +551,9 @@ jobs: if: always() uses: crazy-max/ghaction-dump-context@v1 - github-cache-hit: + local-cache-hit: runs-on: ubuntu-latest - needs: github-cache-first + needs: local-cache-first services: registry: image: registry:2 @@ -600,7 +579,7 @@ jobs: id: cache with: path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-ghcache-${{ github.sha }} + key: ${{ runner.os }}-buildx-local-${{ github.sha }} restore-keys: | ${{ runner.os }}-buildx-ghcache- - @@ -622,9 +601,6 @@ jobs: name: Inspect run: | docker buildx imagetools inspect localhost:5000/name/app:1.0.0 - - - name: Image digest - run: echo ${{ steps.docker_build.outputs.digest }} - name: Check digest run: | @@ -635,8 +611,8 @@ jobs: - name: Compare digests run: | - echo Compare "${{ needs.github-cache-first.outputs.digest }}" with "${{ steps.docker_build.outputs.digest }}" - if [ "${{ needs.github-cache-first.outputs.digest }}" != "${{ steps.docker_build.outputs.digest }}" ]; then + echo Compare "${{ needs.local-cache-first.outputs.digest }}" with "${{ steps.docker_build.outputs.digest }}" + if [ "${{ needs.local-cache-first.outputs.digest }}" != "${{ steps.docker_build.outputs.digest }}" ]; then echo "::error::Digests should be identical" exit 1 fi @@ -647,3 +623,53 @@ jobs: name: Dump context if: always() uses: crazy-max/ghaction-dump-context@v1 + + github-cache: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + buildx_version: + - "" + - latest + services: + registry: + image: registry:2 + ports: + - 5000:5000 + steps: + - + name: Checkout + uses: actions/checkout@v2 + - + name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + with: + version: ${{ matrix.buildx_version }} + driver-opts: | + network=host + buildkitd-flags: --debug + - + name: Build and push + uses: ./ + with: + context: ./test + file: ./test/multi.Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + tags: | + localhost:5000/name/app:latest + localhost:5000/name/app:1.0.0 + cache-from: type=gha,scope=ci-${{ matrix.buildx_version }} + cache-to: type=gha,scope=ci-${{ matrix.buildx_version }} + - + name: Inspect + run: | + docker buildx imagetools inspect localhost:5000/name/app:1.0.0 + - + name: Dump context + if: always() + uses: crazy-max/ghaction-dump-context@v1 diff --git a/README.md b/README.md index 83b0bd1..8a17995 100644 --- a/README.md +++ b/README.md @@ -33,8 +33,6 @@ ___ * [Push to multi-registries](docs/advanced/push-multi-registries.md) * [Copy between registries](docs/advanced/copy-between-registries.md) * [Cache](docs/advanced/cache.md) - * [Registry cache](docs/advanced/cache.md#registry-cache) - * [GitHub cache](docs/advanced/cache.md#github-cache) * [Local registry](docs/advanced/local-registry.md) * [Export image to Docker](docs/advanced/export-docker.md) * [Share built image between jobs](docs/advanced/share-image-jobs.md) @@ -100,9 +98,6 @@ jobs: with: push: true tags: user/app:latest - - - name: Image digest - run: echo ${{ steps.docker_build.outputs.digest }} ``` Building from the current repository automatically uses the [GitHub Token](https://help.github.com/en/actions/configuring-and-managing-workflows/authenticating-with-the-github_token) @@ -170,8 +165,6 @@ jobs: * [Push to multi-registries](docs/advanced/push-multi-registries.md) * [Copy between registries](docs/advanced/copy-between-registries.md) * [Cache](docs/advanced/cache.md) - * [Registry cache](docs/advanced/cache.md#registry-cache) - * [GitHub cache](docs/advanced/cache.md#github-cache) * [Local registry](docs/advanced/local-registry.md) * [Export image to Docker](docs/advanced/export-docker.md) * [Share built image between jobs](docs/advanced/share-image-jobs.md) diff --git a/docs/advanced/cache.md b/docs/advanced/cache.md index 0c0592d..a02b618 100644 --- a/docs/advanced/cache.md +++ b/docs/advanced/cache.md @@ -1,13 +1,18 @@ # Cache +* [Inline cache](#inline-cache) * [Registry cache](#registry-cache) * [GitHub cache](#github-cache) + * [Cache backend API](#cache-backend-api) + * [Local cache](#local-cache) -> More info about buildx cache: https://github.com/docker/buildx/blob/master/docs/reference/buildx_build.md#cache-from +> More info about cache on [BuildKit](https://github.com/moby/buildkit#export-cache) and [Buildx](https://github.com/docker/buildx/blob/master/docs/reference/buildx_build.md#cache-from) repositories. -## Registry cache +## Inline cache -You can import/export cache from a cache manifest or (special) image configuration on the registry. +In most case you want to use the [`type=inline` cache exporter](https://github.com/moby/buildkit#inline-push-image-and-cache-together). +However, note that the `inline` cache exporter only supports `min` cache mode. To enable `max` cache mode, push the +image and the cache separately by using the `registry` cache exporter as shown in the [next example](#registry-cache). ```yaml name: ci @@ -44,16 +49,104 @@ jobs: cache-to: type=inline ``` +## Registry cache + +You can import/export cache from a cache manifest or (special) image configuration on the registry with the +[`type=registry` cache exporter](https://github.com/crazy-max/buildkit/tree/cache-docs#registry-push-image-and-cache-separately). + +```yaml +name: ci + +on: + push: + branches: + - 'master' + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v2 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - + name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - + name: Build and push + uses: docker/build-push-action@v2 + with: + context: . + push: true + tags: user/app:latest + cache-from: type=registry,ref=user/app:buildcache + cache-to: type=registry,ref=user/app:buildcache,mode=max +``` + ## GitHub cache +### Cache backend API + +> :test_tube: This cache exporter is considered EXPERIMENTAL until further notice. Please provide feedback on +> [BuildKit repository](https://github.com/moby/buildkit) if you encounter any issues. + +Since [buildx 0.6.0](https://github.com/docker/buildx/releases/tag/v0.6.0) and [BuildKit 0.9.0](https://github.com/moby/buildkit/releases/tag/v0.9.0), +you can use the [`type=gha` cache exporter](https://github.com/moby/buildkit/tree/master#github-actions-cache-experimental). + +GitHub Actions cache exporter backend uses the [GitHub Cache API](https://github.com/tonistiigi/go-actions-cache/blob/master/api.md) +to fetch and upload cache blobs. That's why this type of cache should be exclusively used in a GitHub Action workflow +as the `url` (`$ACTIONS_CACHE_URL`) and `token` (`$ACTIONS_RUNTIME_TOKEN`) attributes are populated when a workflow +is started. + +```yaml +name: ci + +on: + push: + branches: + - 'master' + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v2 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - + name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - + name: Build and push + uses: docker/build-push-action@v2 + with: + context: . + push: true + tags: user/app:latest + cache-from: type=gha + cache-to: type=gha,mode=max +``` + +### Local cache + > :warning: At the moment caches are copied over the existing cache so it [keeps growing](https://github.com/docker/build-push-action/issues/252). > The `Move cache` step is used as a temporary fix (see https://github.com/moby/buildkit/issues/1896). -> :rocket: There is a new cache backend using GitHub cache being developed that will lighten your workflow. -> More info: https://github.com/docker/buildx/pull/535 - -You can leverage [GitHub cache](https://docs.github.com/en/actions/configuring-and-managing-workflows/caching-dependencies-to-speed-up-workflows) -using [actions/cache](https://github.com/actions/cache) with this action: +You can also leverage [GitHub cache](https://docs.github.com/en/actions/configuring-and-managing-workflows/caching-dependencies-to-speed-up-workflows) +using [actions/cache](https://github.com/actions/cache) and [`type=local` cache exporter](https://github.com/moby/buildkit#local-directory-1) +with this action: ```yaml name: ci @@ -95,7 +188,7 @@ jobs: push: true tags: user/app:latest cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache-new + cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max - # Temp fix # https://github.com/docker/build-push-action/issues/252