From 05289d3eb643c0ae89742e56685b525de5fc6ce9 Mon Sep 17 00:00:00 2001 From: Joe Previte Date: Tue, 27 Sep 2022 11:46:37 -0700 Subject: [PATCH] chore(ci): move platform steps to release (#5587) * refactor: move platform steps build -> release * fixup! refactor: move platform steps build -> release * refactor: download npm package * refactor: upload release-packages to draft * refactor: remove draft step * refactor: e2e rely on build now * refactor: use npm package in e2e * fix: update release workflow * fixup: update cache key * fixup: checkout submodules in e2e steps * fixup: try install in release dir * fix: copy node to release package * docs: add notes about test for terminal * fixup * try cp instead * try this * fixup: clean it all up * fixup: update names * fixup: add proxy back * fixup: add comment --- .github/workflows/build.yaml | 293 ++++----------------------------- .github/workflows/release.yaml | 222 ++++++++++++++++++++++++- test/e2e/terminal.test.ts | 1 + 3 files changed, 254 insertions(+), 262 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 8d8afb5c8..1d72596e1 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -127,7 +127,7 @@ jobs: run: yarn lint:ts build: - name: Build + name: Build code-server runs-on: ubuntu-latest timeout-minutes: 30 env: @@ -223,6 +223,7 @@ jobs: path: ./package.tar.gz npm: + name: Publish npm package # the npm-package gets uploaded as an artifact in Build # so we need that to complete before this runs needs: build @@ -274,239 +275,11 @@ jobs: npm install -g @coder/code-server-pr@${{ github.event.number }} ``` - # TODO: cache building yarn --production - # possibly 2m30s of savings(?) - # this requires refactoring our release scripts - package-linux-amd64: - name: x86-64 Linux build - needs: build - runs-on: ubuntu-latest - timeout-minutes: 15 - container: "centos:7" - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - - steps: - - name: Checkout repo - uses: actions/checkout@v3 - - - name: Install Node.js v16 - uses: actions/setup-node@v3 - with: - node-version: "16" - - - name: Install development tools - run: | - yum install -y epel-release centos-release-scl make - yum install -y devtoolset-9-{make,gcc,gcc-c++} jq rsync python3 - - - name: Install nfpm and envsubst - run: | - mkdir -p ~/.local/bin - curl -sSfL https://github.com/goreleaser/nfpm/releases/download/v2.3.1/nfpm_2.3.1_`uname -s`_`uname -m`.tar.gz | tar -C ~/.local/bin -zxv nfpm - curl -sSfL https://github.com/a8m/envsubst/releases/download/v1.1.0/envsubst-`uname -s`-`uname -m` -o envsubst - chmod +x envsubst - mv envsubst ~/.local/bin - echo "$HOME/.local/bin" >> $GITHUB_PATH - - - name: Install yarn - run: npm install -g yarn - - - name: Download npm package - uses: actions/download-artifact@v3 - with: - name: npm-package - - - name: Decompress npm package - run: tar -xzf package.tar.gz - - # NOTE: && here is deliberate - GitHub puts each line in its own `.sh` - # file when running inside a docker container. - - name: Build standalone release - run: source scl_source enable devtoolset-9 && yarn release:standalone - - - name: Fetch dependencies from cache - id: cache-node-modules - uses: actions/cache@v3 - with: - path: "**/node_modules" - key: yarn-build-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - yarn-build- - - - name: Install test dependencies - if: steps.cache-node-modules.outputs.cache-hit != 'true' - run: SKIP_SUBMODULE_DEPS=1 yarn --frozen-lockfile - - - name: Run integration tests on standalone release - run: yarn test:integration - - - name: Upload coverage report to Codecov - uses: codecov/codecov-action@v3 - with: - token: ${{ secrets.CODECOV_TOKEN }} - if: success() - - - name: Build packages with nfpm - run: yarn package - - - name: Upload release artifacts - uses: actions/upload-artifact@v3 - with: - name: release-packages - path: ./release-packages - - # NOTE@oxy: - # We use Ubuntu 16.04 here, so that our build is more compatible - # with older libc versions. We used to (Q1'20) use CentOS 7 here, - # but it has a full update EOL of Q4'20 and a 'critical security' - # update EOL of 2024. We're dropping full support a few years before - # the final EOL, but I don't believe CentOS 7 has a large arm64 userbase. - # It is not feasible to cross-compile with CentOS. - - # Cross-compile notes: To compile native dependencies for arm64, - # we install the aarch64/armv7l cross toolchain and then set it as the default - # compiler/linker/etc. with the AR/CC/CXX/LINK environment variables. - # qemu-user-static on ubuntu-16.04 currently doesn't run Node correctly, - # so we just build with "native"/x86_64 node, then download arm64/armv7l node - # and then put it in our release. We can't smoke test the cross build this way, - # but this means we don't need to maintain a self-hosted runner! - - # NOTE@jsjoeio: - # We used to use 16.04 until GitHub deprecated it on September 20, 2021 - # See here: https://github.com/actions/virtual-environments/pull/3862/files - package-linux-cross: - name: Linux cross-compile builds - needs: build - runs-on: ubuntu-18.04 - timeout-minutes: 15 - strategy: - matrix: - include: - - prefix: aarch64-linux-gnu - arch: arm64 - - prefix: arm-linux-gnueabihf - arch: armv7l - - env: - AR: ${{ format('{0}-ar', matrix.prefix) }} - CC: ${{ format('{0}-gcc', matrix.prefix) }} - CXX: ${{ format('{0}-g++', matrix.prefix) }} - LINK: ${{ format('{0}-g++', matrix.prefix) }} - NPM_CONFIG_ARCH: ${{ matrix.arch }} - NODE_VERSION: v16.13.0 - - steps: - - name: Checkout repo - uses: actions/checkout@v3 - - - name: Install Node.js v16 - uses: actions/setup-node@v3 - with: - node-version: "16" - - - name: Install nfpm - run: | - mkdir -p ~/.local/bin - curl -sSfL https://github.com/goreleaser/nfpm/releases/download/v2.3.1/nfpm_2.3.1_`uname -s`_`uname -m`.tar.gz | tar -C ~/.local/bin -zxv nfpm - echo "$HOME/.local/bin" >> $GITHUB_PATH - - - name: Install cross-compiler - run: sudo apt update && sudo apt install $PACKAGE - env: - PACKAGE: ${{ format('g++-{0}', matrix.prefix) }} - - - name: Download npm package - uses: actions/download-artifact@v3 - with: - name: npm-package - - - name: Decompress npm package - run: tar -xzf package.tar.gz - - - name: Build standalone release - run: yarn release:standalone - - - name: Replace node with cross-compile equivalent - run: | - wget https://nodejs.org/dist/${NODE_VERSION}/node-${NODE_VERSION}-linux-${NPM_CONFIG_ARCH}.tar.xz - tar -xf node-${NODE_VERSION}-linux-${NPM_CONFIG_ARCH}.tar.xz node-${NODE_VERSION}-linux-${NPM_CONFIG_ARCH}/bin/node --strip-components=2 - mv ./node ./release-standalone/lib/node - - - name: Build packages with nfpm - run: yarn package ${NPM_CONFIG_ARCH} - - - name: Upload release artifacts - uses: actions/upload-artifact@v3 - with: - name: release-packages - path: ./release-packages - - package-macos-amd64: - name: x86-64 macOS build - needs: build - runs-on: macos-latest - timeout-minutes: 15 - steps: - - name: Checkout repo - uses: actions/checkout@v3 - - - name: Install Node.js v16 - uses: actions/setup-node@v3 - with: - node-version: "16" - - - name: Install nfpm - run: | - mkdir -p ~/.local/bin - curl -sSfL https://github.com/goreleaser/nfpm/releases/download/v2.3.1/nfpm_2.3.1_`uname -s`_`uname -m`.tar.gz | tar -C ~/.local/bin -zxv nfpm - echo "$HOME/.local/bin" >> $GITHUB_PATH - - - name: Download npm package - uses: actions/download-artifact@v3 - with: - name: npm-package - - - name: Decompress npm package - run: tar -xzf package.tar.gz - - - name: Build standalone release - run: yarn release:standalone - - - name: Fetch dependencies from cache - id: cache-node-modules - uses: actions/cache@v3 - with: - path: "**/node_modules" - key: yarn-build-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - yarn-build- - - - name: Install test dependencies - if: steps.cache-node-modules.outputs.cache-hit != 'true' - run: SKIP_SUBMODULE_DEPS=1 yarn install - - - name: Run native module tests on standalone release - run: yarn test:native - - - name: Build packages with nfpm - run: yarn package - - - name: Upload release artifacts - uses: actions/upload-artifact@v3 - with: - name: release-packages - path: ./release-packages - test-e2e: - name: End-to-end tests - needs: package-linux-amd64 + name: Run e2e tests + needs: build runs-on: ubuntu-latest timeout-minutes: 15 - env: - # Since we build code-server we might as well run tests from the release - # since VS Code will load faster due to the bundling. - CODE_SERVER_TEST_ENTRY: "./release-packages/code-server-linux-amd64" steps: - name: Checkout repo uses: actions/checkout@v3 @@ -525,17 +298,16 @@ jobs: restore-keys: | yarn-build- - - name: Download release packages + - name: Download npm package uses: actions/download-artifact@v3 with: - name: release-packages - path: ./release-packages + name: npm-package - - name: Untar code-server release - run: | - cd release-packages - tar -xzf code-server*-linux-amd64.tar.gz - mv code-server*-linux-amd64 code-server-linux-amd64 + - name: Decompress npm package + run: tar -xzf package.tar.gz + + - name: Install release package dependencies + run: cd release && yarn install - name: Install dependencies if: steps.cache-node-modules.outputs.cache-hit != 'true' @@ -546,8 +318,14 @@ jobs: ./test/node_modules/.bin/playwright install-deps ./test/node_modules/.bin/playwright install + # TODO@jsjoeio - this is temp. We need to do an actual fix + - name: Copy node binary to release + run: | + ln -s $(which node) ./release/lib/node + ln -s $(which node) ./release/node + - name: Run end-to-end tests - run: yarn test:e2e --global-timeout 840000 + run: CODE_SERVER_TEST_ENTRY=./release yarn test:e2e --global-timeout 840000 - name: Upload test artifacts if: always() @@ -557,17 +335,13 @@ jobs: path: ./test/test-results - name: Remove release packages and test artifacts - run: rm -rf ./release-packages ./test/test-results + run: rm -rf ./release ./test/test-results test-e2e-proxy: - name: End-to-end tests behind proxy - needs: package-linux-amd64 + name: Run e2e tests behind proxy + needs: build runs-on: ubuntu-latest timeout-minutes: 25 - env: - # Since we build code-server we might as well run tests from the release - # since VS Code will load faster due to the bundling. - CODE_SERVER_TEST_ENTRY: "./release-packages/code-server-linux-amd64" steps: - name: Checkout repo uses: actions/checkout@v3 @@ -586,17 +360,16 @@ jobs: restore-keys: | yarn-build- - - name: Download release packages + - name: Download npm package uses: actions/download-artifact@v3 with: - name: release-packages - path: ./release-packages + name: npm-package - - name: Untar code-server release - run: | - cd release-packages - tar -xzf code-server*-linux-amd64.tar.gz - mv code-server*-linux-amd64 code-server-linux-amd64 + - name: Decompress npm package + run: tar -xzf package.tar.gz + + - name: Install release package dependencies + run: cd release && yarn install - name: Install dependencies if: steps.cache-node-modules.outputs.cache-hit != 'true' @@ -627,8 +400,14 @@ jobs: - name: Start Caddy run: sudo ~/.cache/caddy/caddy start --config ./ci/Caddyfile + # TODO@jsjoeio - this is temp. We need to do an actual fix + - name: Copy node binary to release + run: | + ln -s $(which node) ./release/lib/node + ln -s $(which node) ./release/node + - name: Run end-to-end tests - run: yarn test:e2e:proxy + run: CODE_SERVER_TEST_ENTRY=./release yarn test:e2e:proxy --global-timeout 840000 - name: Stop Caddy if: always() @@ -642,4 +421,4 @@ jobs: path: ./test/test-results - name: Remove release packages and test artifacts - run: rm -rf ./release-packages ./test/test-results + run: rm -rf ./release ./test/test-results diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 181b98580..61ec8a5ba 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -14,12 +14,42 @@ concurrency: cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: - draft: + # TODO: cache building yarn --production + # possibly 2m30s of savings(?) + # this requires refactoring our release scripts + package-linux-amd64: + name: x86-64 Linux build runs-on: ubuntu-latest + timeout-minutes: 15 + container: "centos:7" + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + steps: - - uses: actions/checkout@v3 + - name: Checkout repo + uses: actions/checkout@v3 + + - name: Install Node.js v16 + uses: actions/setup-node@v3 with: - fetch-depth: 0 + node-version: "16" + + - name: Install development tools + run: | + yum install -y epel-release centos-release-scl make + yum install -y devtoolset-9-{make,gcc,gcc-c++} jq rsync python3 + + - name: Install nfpm and envsubst + run: | + mkdir -p ~/.local/bin + curl -sSfL https://github.com/goreleaser/nfpm/releases/download/v2.3.1/nfpm_2.3.1_`uname -s`_`uname -m`.tar.gz | tar -C ~/.local/bin -zxv nfpm + curl -sSfL https://github.com/a8m/envsubst/releases/download/v1.1.0/envsubst-`uname -s`-`uname -m` -o envsubst + chmod +x envsubst + mv envsubst ~/.local/bin + echo "$HOME/.local/bin" >> $GITHUB_PATH + + - name: Install yarn + run: npm install -g yarn - name: Download artifacts uses: dawidd6/action-download-artifact@v2 @@ -29,8 +59,190 @@ jobs: workflow: build.yaml workflow_conclusion: completed check_artifacts: true - name: release-packages - path: ./release-packages + name: npm-package + + - name: Decompress npm package + run: tar -xzf package.tar.gz + + # NOTE: && here is deliberate - GitHub puts each line in its own `.sh` + # file when running inside a docker container. + - name: Build standalone release + run: source scl_source enable devtoolset-9 && yarn release:standalone + + - name: Fetch dependencies from cache + id: cache-node-modules + uses: actions/cache@v3 + with: + path: "**/node_modules" + key: yarn-build-${{ hashFiles('**/yarn.lock') }} + restore-keys: | + yarn-build- + + - name: Install test dependencies + if: steps.cache-node-modules.outputs.cache-hit != 'true' + run: SKIP_SUBMODULE_DEPS=1 yarn --frozen-lockfile + + - name: Run integration tests on standalone release + run: yarn test:integration + + - name: Upload coverage report to Codecov + uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} + if: success() + + - name: Build packages with nfpm + run: yarn package + + - uses: softprops/action-gh-release@v1 + with: + draft: true + discussion_category_name: "📣 Announcements" + files: ./release-packages/* + + # NOTE@oxy: + # We use Ubuntu 16.04 here, so that our build is more compatible + # with older libc versions. We used to (Q1'20) use CentOS 7 here, + # but it has a full update EOL of Q4'20 and a 'critical security' + # update EOL of 2024. We're dropping full support a few years before + # the final EOL, but I don't believe CentOS 7 has a large arm64 userbase. + # It is not feasible to cross-compile with CentOS. + + # Cross-compile notes: To compile native dependencies for arm64, + # we install the aarch64/armv7l cross toolchain and then set it as the default + # compiler/linker/etc. with the AR/CC/CXX/LINK environment variables. + # qemu-user-static on ubuntu-16.04 currently doesn't run Node correctly, + # so we just build with "native"/x86_64 node, then download arm64/armv7l node + # and then put it in our release. We can't smoke test the cross build this way, + # but this means we don't need to maintain a self-hosted runner! + + # NOTE@jsjoeio: + # We used to use 16.04 until GitHub deprecated it on September 20, 2021 + # See here: https://github.com/actions/virtual-environments/pull/3862/files + package-linux-cross: + name: Linux cross-compile builds + runs-on: ubuntu-18.04 + timeout-minutes: 15 + strategy: + matrix: + include: + - prefix: aarch64-linux-gnu + arch: arm64 + - prefix: arm-linux-gnueabihf + arch: armv7l + + env: + AR: ${{ format('{0}-ar', matrix.prefix) }} + CC: ${{ format('{0}-gcc', matrix.prefix) }} + CXX: ${{ format('{0}-g++', matrix.prefix) }} + LINK: ${{ format('{0}-g++', matrix.prefix) }} + NPM_CONFIG_ARCH: ${{ matrix.arch }} + NODE_VERSION: v16.13.0 + + steps: + - name: Checkout repo + uses: actions/checkout@v3 + + - name: Install Node.js v16 + uses: actions/setup-node@v3 + with: + node-version: "16" + + - name: Install nfpm + run: | + mkdir -p ~/.local/bin + curl -sSfL https://github.com/goreleaser/nfpm/releases/download/v2.3.1/nfpm_2.3.1_`uname -s`_`uname -m`.tar.gz | tar -C ~/.local/bin -zxv nfpm + echo "$HOME/.local/bin" >> $GITHUB_PATH + + - name: Install cross-compiler + run: sudo apt update && sudo apt install $PACKAGE + env: + PACKAGE: ${{ format('g++-{0}', matrix.prefix) }} + + - name: Download artifacts + uses: dawidd6/action-download-artifact@v2 + id: download + with: + branch: ${{ github.ref }} + workflow: build.yaml + workflow_conclusion: completed + check_artifacts: true + name: npm-package + + - name: Decompress npm package + run: tar -xzf package.tar.gz + + - name: Build standalone release + run: yarn release:standalone + + - name: Replace node with cross-compile equivalent + run: | + wget https://nodejs.org/dist/${NODE_VERSION}/node-${NODE_VERSION}-linux-${NPM_CONFIG_ARCH}.tar.xz + tar -xf node-${NODE_VERSION}-linux-${NPM_CONFIG_ARCH}.tar.xz node-${NODE_VERSION}-linux-${NPM_CONFIG_ARCH}/bin/node --strip-components=2 + mv ./node ./release-standalone/lib/node + + - name: Build packages with nfpm + run: yarn package ${NPM_CONFIG_ARCH} + + - uses: softprops/action-gh-release@v1 + with: + draft: true + discussion_category_name: "📣 Announcements" + files: ./release-packages/* + + package-macos-amd64: + name: x86-64 macOS build + runs-on: macos-latest + timeout-minutes: 15 + steps: + - name: Checkout repo + uses: actions/checkout@v3 + + - name: Install Node.js v16 + uses: actions/setup-node@v3 + with: + node-version: "16" + + - name: Install nfpm + run: | + mkdir -p ~/.local/bin + curl -sSfL https://github.com/goreleaser/nfpm/releases/download/v2.3.1/nfpm_2.3.1_`uname -s`_`uname -m`.tar.gz | tar -C ~/.local/bin -zxv nfpm + echo "$HOME/.local/bin" >> $GITHUB_PATH + + - name: Download artifacts + uses: dawidd6/action-download-artifact@v2 + id: download + with: + branch: ${{ github.ref }} + workflow: build.yaml + workflow_conclusion: completed + check_artifacts: true + name: npm-package + + - name: Decompress npm package + run: tar -xzf package.tar.gz + + - name: Build standalone release + run: yarn release:standalone + + - name: Fetch dependencies from cache + id: cache-node-modules + uses: actions/cache@v3 + with: + path: "**/node_modules" + key: yarn-build-${{ hashFiles('**/yarn.lock') }} + restore-keys: | + yarn-build- + + - name: Install test dependencies + if: steps.cache-node-modules.outputs.cache-hit != 'true' + run: SKIP_SUBMODULE_DEPS=1 yarn install + + - name: Run native module tests on standalone release + run: yarn test:native + + - name: Build packages with nfpm + run: yarn package - uses: softprops/action-gh-release@v1 with: diff --git a/test/e2e/terminal.test.ts b/test/e2e/terminal.test.ts index df7ceeebf..a8180a654 100644 --- a/test/e2e/terminal.test.ts +++ b/test/e2e/terminal.test.ts @@ -30,6 +30,7 @@ describe("Integrated Terminal", [], {}, () => { expect(stdout).toMatch(address) }) + // TODO@jsjoeio - add test to make sure full code-server path works test("should be able to invoke `code-server` to open a file", async ({ codeServerPage }) => { const tmpFolderPath = await tmpdir(testName) const tmpFile = path.join(tmpFolderPath, "test-file")