From 92da02ef3e5ff49314f9be56cd351b1b97a576b2 Mon Sep 17 00:00:00 2001 From: Asher Date: Tue, 14 Jan 2020 12:29:57 -0600 Subject: [PATCH] Add Drone CI caching --- .drone.yml | 436 ++++++++++++++++++++++++++++++++++------------ .gitignore | 8 +- .travis.yml | 2 +- Dockerfile | 5 - README.md | 10 +- scripts/cacher.sh | 53 ++++++ 6 files changed, 390 insertions(+), 124 deletions(-) create mode 100644 scripts/cacher.sh diff --git a/.drone.yml b/.drone.yml index a0b1a9774..ea52e5117 100644 --- a/.drone.yml +++ b/.drone.yml @@ -1,135 +1,353 @@ kind: pipeline type: docker -name: build-amd64 +name: amd64:linux platform: arch: amd64 steps: +- name: cache:restore + image: node:12 + commands: + - ./scripts/cacher.sh - - name: build - image: node:12 - commands: - - apt update && apt install -y build-essential git libsecret-1-dev libx11-dev libxkbfile-dev - - if [ -z $DRONE_TAG ]; then yarn build 1.41.1 daily ; else yarn build 1.41.1 $DRONE_TAG; fi - - if [ -z $DRONE_TAG ]; then yarn binary 1.41.1 daily ; else yarn binary 1.41.1 $DRONE_TAG; fi - - if [ -z $DRONE_TAG ]; then yarn package 1.41.1 daily ; else yarn package 1.41.1 $DRONE_TAG; fi +- name: build + image: node:12 + commands: + - apt update && apt install -y build-essential git libsecret-1-dev libx11-dev libxkbfile-dev + - timeout 50m ./scripts/ci.bash || echo 'Timed out or failed; continuing so we can preserve cache for the next run' - - name: build-alpine - image: node:12-alpine - commands: - - apk add libxkbfile-dev libsecret-dev build-base git - - if [ -z $DRONE_TAG ]; then yarn build 1.41.1 daily ; else yarn build 1.41.1 $DRONE_TAG; fi - - if [ -z $DRONE_TAG ]; then yarn binary 1.41.1 daily ; else yarn binary 1.41.1 $DRONE_TAG; fi - - if [ -z $DRONE_TAG ]; then yarn package 1.41.1 daily ; else yarn package 1.41.1 $DRONE_TAG; fi +- name: cache:package + image: node:12 + commands: + - ./scripts/cacher.sh + when: + event: push - - name: publish - image: plugins/github-release - settings: - api_key: - from_secret: github_token - files: release/*.tar.gz - when: +- name: cache:push + image: plugins/gcs + settings: + source: cache-upload/ + target: codesrv-ci.cdr.sh + token: + from_secret: gcs-token + when: + event: push + +- name: test + image: node:12 + commands: + - yarn test + +- name: publish:github + image: plugins/github-release + settings: + api_key: + from_secret: github_token + files: release/*.tar.gz + when: + event: tag + +- name: publish:docker + image: banzaicloud/drone-kaniko + settings: + username: + from_secret: docker_user + password: + from_secret: docker_password + tags: + - latest + - v2 + - ${DRONE_TAG} + build_args: + - codeServerVersion=${DRONE_TAG} + - vscodeVersion=1.41.1 + when: event: tag - - name: docker-publish - image: banzaicloud/drone-kaniko - settings: - username: - from_secret: docker_user - password: - from_secret: docker_password - tags: - - latest - - ${DRONE_TAG} - build_args: - - codeServerVersion=${DRONE_TAG} - - vscodeVersion=1.41.1 - when: - event: tag - --- kind: pipeline type: docker -name: build-arm64 +name: amd64:alpine + +platform: + arch: amd64 + +steps: +- name: cache:restore + image: node:12-alpine + commands: + - ./scripts/cacher.sh + +- name: build + image: node:12-alpine + commands: + - apk add libxkbfile-dev libsecret-dev build-base git bash python + - timeout 50m ./scripts/ci.bash || echo 'Timed out or failed; continuing so we can preserve cache for the next run' + +- name: cache:package + image: node:12-alpine + commands: + - ./scripts/cacher.sh + when: + event: push + +- name: cache:push + image: plugins/gcs + settings: + source: cache-upload/ + target: codesrv-ci.cdr.sh + token: + from_secret: gcs-token + when: + event: push + +- name: test + image: node:12-alpine + commands: + - yarn test + +- name: publish:github + image: plugins/github-release + settings: + api_key: + from_secret: github_token + files: release/*.tar.gz + when: + event: tag + +--- +kind: pipeline +type: docker +name: arm64:linux platform: arch: arm64 steps: +- name: cache:restore + image: node:12 + commands: + - ./scripts/cacher.sh - - name: build - image: node:12 - commands: - - apt update && apt install -y build-essential git libsecret-1-dev libx11-dev libxkbfile-dev - - if [ -z $DRONE_TAG ]; then yarn build 1.41.1 daily ; else yarn build 1.41.1 $DRONE_TAG; fi - - if [ -z $DRONE_TAG ]; then yarn binary 1.41.1 daily ; else yarn binary 1.41.1 $DRONE_TAG; fi - - if [ -z $DRONE_TAG ]; then yarn package 1.41.1 daily ; else yarn package 1.41.1 $DRONE_TAG; fi +- name: build + image: node:12 + commands: + - apt update && apt install -y build-essential git libsecret-1-dev libx11-dev libxkbfile-dev + - timeout 50m ./scripts/ci.bash || echo 'Timed out or failed; continuing so we can preserve cache for the next run' - - name: build-alpine - image: node:12-alpine - commands: - - apk add libxkbfile-dev libsecret-dev build-base git - - if [ -z $DRONE_TAG ]; then yarn build 1.41.1 daily ; else yarn build 1.41.1 $DRONE_TAG; fi - - if [ -z $DRONE_TAG ]; then yarn binary 1.41.1 daily ; else yarn binary 1.41.1 $DRONE_TAG; fi - - if [ -z $DRONE_TAG ]; then yarn package 1.41.1 daily ; else yarn package 1.41.1 $DRONE_TAG; fi +- name: cache:package + image: node:12 + commands: + - ./scripts/cacher.sh + when: + event: push - - name: publish - image: plugins/github-release - settings: - api_key: - from_secret: github_token - files: release/*.tar.gz - when: +- name: cache:push + image: plugins/gcs + settings: + source: cache-upload/ + target: codesrv-ci.cdr.sh + token: + from_secret: gcs-token + when: + event: push + +- name: test + image: node:12 + commands: + - yarn test + +- name: publish:github + image: plugins/github-release + settings: + api_key: + from_secret: github_token + files: release/*.tar.gz + when: + event: tag + +- name: publish:docker + image: banzaicloud/drone-kaniko + settings: + username: + from_secret: docker_user + password: + from_secret: docker_password + tags: + - arm64 + - ${DRONE_TAG}-arm64 + build_args: + - codeServerVersion=$DRONE_TAG + - vscodeVersion=1.41.1 + when: event: tag - - name: docker-publish - image: banzaicloud/drone-kaniko - settings: - username: - from_secret: docker_user - password: - from_secret: docker_password - tags: - - latest-arm64 - - ${DRONE_TAG}-arm64 - build_args: - - codeServerVersion=$DRONE_TAG - - vscodeVersion=1.41.1 - when: - event: tag +--- +kind: pipeline +type: docker +name: arm64:alpine -# --- -# kind: pipeline -# type: docker -# name: build-arm -# -# platform: -# arch: arm -# -# steps: -# -# - name: build -# image: node:12 -# commands: -# - apt update && apt install -y build-essential git libsecret-1-dev libx11-dev libxkbfile-dev -# - yarn -# - if [ -z $DRONE_TAG ]; then yarn build 1.41.1 daily ; else yarn build 1.41.1 $DRONE_TAG; fi -# - if [ -z $DRONE_TAG ]; then yarn binary 1.41.1 daily ; else yarn binary 1.41.1 $DRONE_TAG; fi -# -# - name: build-alpine -# image: node:12-alpine -# commands: -# - apk add libxkbfile-dev libsecret-dev build-base git -# - if [ -z $DRONE_TAG ]; then yarn build 1.41.1 daily ; else yarn build 1.41.1 $DRONE_TAG; fi -# - if [ -z $DRONE_TAG ]; then yarn binary 1.41.1 daily ; else yarn binary 1.41.1 $DRONE_TAG; fi -# - if [ -z $DRONE_TAG ]; then yarn package 1.41.1 daily ; else yarn package 1.41.1 $DRONE_TAG; fi -# -# - name: publish -# image: plugins/github-release -# settings: -# api_key: -# from_secret: github_token -# files: release/*.tar.gz -# when: -# event: tag +platform: + arch: arm64 + +steps: +- name: cache:restore + image: node:12-alpine + commands: + - ./scripts/cacher.sh + +- name: build + image: node:12-alpine + commands: + - apk add libxkbfile-dev libsecret-dev build-base git bash python + - timeout 50m ./scripts/ci.bash || echo 'Timed out or failed; continuing so we can preserve cache for the next run' + +- name: cache:package + image: node:12-alpine + commands: + - ./scripts/cacher.sh + when: + event: push + +- name: cache:push + image: plugins/gcs + settings: + source: cache-upload/ + target: codesrv-ci.cdr.sh + token: + from_secret: gcs-token + when: + event: push + +- name: test + image: node:12-alpine + commands: + - yarn test + +- name: publish:github + image: plugins/github-release + settings: + api_key: + from_secret: github_token + files: release/*.tar.gz + when: + event: tag + +--- +kind: pipeline +type: docker +name: arm:linux + +platform: + arch: arm + +steps: +- name: cache:restore + image: node:12 + commands: + - ./scripts/cacher.sh + +- name: build + image: node:12 + commands: + - apt update && apt install -y build-essential git libsecret-1-dev libx11-dev libxkbfile-dev + - timeout 50m ./scripts/ci.bash || echo 'Timed out or failed; continuing so we can preserve cache for the next run' + +- name: cache:package + image: node:12 + commands: + - ./scripts/cacher.sh + when: + event: push + +- name: cache:push + image: plugins/gcs + settings: + source: cache-upload/ + target: codesrv-ci.cdr.sh + token: + from_secret: gcs-token + when: + event: push + +- name: test + image: node:12 + commands: + - yarn test + +- name: publish:github + image: plugins/github-release + settings: + api_key: + from_secret: github_token + files: release/*.tar.gz + when: + event: tag + +- name: publish:docker + image: banzaicloud/drone-kaniko + settings: + username: + from_secret: docker_user + password: + from_secret: docker_password + tags: + - arm + - ${DRONE_TAG}-arm + build_args: + - codeServerVersion=$DRONE_TAG + - vscodeVersion=1.41.1 + when: + event: tag + +--- +kind: pipeline +type: docker +name: arm:alpine + +platform: + arch: arm + +steps: +- name: cache:restore + image: node:12-alpine + commands: + - ./scripts/cacher.sh + +- name: build + image: node:12-alpine + commands: + - apk add libxkbfile-dev libsecret-dev build-base git bash python + - timeout 50m ./scripts/ci.bash || echo 'Timed out or failed; continuing so we can preserve cache for the next run' + +- name: cache:package + image: node:12-alpine + commands: + - ./scripts/cacher.sh + when: + event: push + +- name: cache:push + image: plugins/gcs + settings: + source: cache-upload/ + target: codesrv-ci.cdr.sh + token: + from_secret: gcs-token + when: + event: push + +- name: test + image: node:12-alpine + commands: + - yarn test + +- name: publish:github + image: plugins/github-release + settings: + api_key: + from_secret: github_token + files: release/*.tar.gz + when: + event: tag diff --git a/.gitignore b/.gitignore index 58f35a34b..0a7a0e8f5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ node_modules -build -release -binaries -source +/build +/release +/binaries +/lib diff --git a/.travis.yml b/.travis.yml index c7b5a301c..db0e0f615 100644 --- a/.travis.yml +++ b/.travis.yml @@ -59,4 +59,4 @@ cache: timeout: 1000 yarn: true directories: - - source + - source/vscode-1.41.1-source diff --git a/Dockerfile b/Dockerfile index bf8964ad1..6eb3ff73e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,11 +49,6 @@ RUN mkdir -p /home/coder/project # To avoid EACCES issues on f.ex Crostini (ChromeOS) RUN mkdir -p /home/coder/.local/share/code-server -# FIXME: permission fix required for bind-mounts -# See GH-1270 -RUN sudo chmod -R g+rw /home/coder/project && sudo chmod -R g+rw /home/coder/.local/share/code-server && \ - sudo chown -R coder:coder /home/coder - WORKDIR /home/coder/project # This ensures we have a volume mounted even if the user forgot to do bind diff --git a/README.md b/README.md index d5779c6ab..38a25a352 100644 --- a/README.md +++ b/README.md @@ -64,11 +64,11 @@ See before building. ```shell -export OUT=/path/to/output/build # Optional if only building. Required if also developing. -yarn build ${vscodeVersion} ${codeServerVersion} # See travis.yml for the VS Code version to use. - # The code-server version can be anything you want. -node /path/to/output/build/out/vs/server/main.js # You can run the built JavaScript with Node. -yarn binary ${vscodeVersion} ${codeServerVersion} # Or you can package it into a binary. +export OUT=/path/to/output/build # Optional if only building. Required if also developing. +yarn build $vscodeVersion $codeServerVersion # See travis.yml for the VS Code version to use. + # The code-server version can be anything you want. +node /path/to/output/build/out/vs/server/main.js # You can run the built JavaScript with Node. +yarn binary $vscodeVersion $codeServerVersion # Or you can package it into a binary. ``` ## Security diff --git a/scripts/cacher.sh b/scripts/cacher.sh new file mode 100644 index 000000000..c623e5d6c --- /dev/null +++ b/scripts/cacher.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env sh +# cacher.sh -- Restore and rebuild cache. +# Cache paths are designed to work with multi-arch builds and are organized +# based on the branch or tag. The master branch cache is used as a fallback. +# This will download and package the cache but it will not upload it. + +set -eu + +# Try restoring from each argument in turn until we get something. +restore() { + for branch in "$@" ; do + if [ -n "$branch" ] ; then + cache_path="https://codesrv-ci.cdr.sh/cache/$branch/$tar.tar.gz" + if wget "$cache_path" ; then + tar xzvf "$tar.tar.gz" + break + fi + fi + done +} + +# We need to cache the built-in extensions and Node modules. Everything inside +# the cache-upload directory will be uploaded as-is to the code-server bucket. +package() { + mkdir -p "cache-upload/cache/$1" + tar czfv "cache-upload/cache/$1/$tar.tar.gz" node_modules source/vscode-1.41.1-source +} + +main() { + cd "$(dirname "$0")/.." + + # Get the branch for this build. + branch=${DRONE_BRANCH:-${DRONE_SOURCE_BRANCH:-${DRONE_TAG:-}}} + + # The cache will be named based on the arch, platform, and libc. + arch=$DRONE_STAGE_ARCH + platform=${PLATFORM:-linux} + case $DRONE_STAGE_NAME in + *alpine*) libc=musl ;; + * ) libc=glibc ;; + esac + + tar="$platform-$arch-$libc" + + # The action is determined by the name of the step. + case $DRONE_STEP_NAME in + *restore*) restore "$branch" "$DRONE_REPO_BRANCH" ;; + *rebuild*|*package*) package "$branch" ;; + *) exit 1 ;; + esac +} + +main "$@"