From b54982550b0efbb4423401dee9a981a11ee96786 Mon Sep 17 00:00:00 2001 From: "G.J.R. Timmer" Date: Mon, 5 Jul 2021 00:36:51 +0200 Subject: [PATCH 1/5] fix apply chown permissions in parallel for large workspace --- readme-vars.yml | 1 + root/etc/cont-init.d/30-config | 19 +++++++++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/readme-vars.yml b/readme-vars.yml index 08dbd39..01068cf 100644 --- a/readme-vars.yml +++ b/readme-vars.yml @@ -76,6 +76,7 @@ app_setup_block: | # changelog changelogs: - { date: "11.07.21:", desc: "Bump node to 14 to fix builds" } + - { date: "05.07.21:", desc: "Fix slow `chown` on large workspace" } - { date: "08.05.21:", desc: "Fix doc link" } - { date: "04.02.20:", desc: "Allow setting gui password via hash using env var `HASHED_PASSWORD`." } - { date: "23.12.20:", desc: "Allow setting sudo password via hash using env var `SUDO_PASSWORD_HASH`." } diff --git a/root/etc/cont-init.d/30-config b/root/etc/cont-init.d/30-config index b3f6cd4..32fdf2a 100644 --- a/root/etc/cont-init.d/30-config +++ b/root/etc/cont-init.d/30-config @@ -18,5 +18,20 @@ if [ -n "${SUDO_PASSWORD}" ] || [ -n "${SUDO_PASSWORD_HASH}" ]; then fi # permissions -chown -R abc:abc \ - /config +if [ -f "/usr/bin/find" ] && [ -f "/usr/bin/xargs" ]; then + # Split workload between config and workspace + echo "setting permissions::configuration" + CORES=$(nproc --all) + find /config -maxdepth 4 -mindepth 1 -path /config/workspace -prune -false -o -type d | \ + xargs --max-args=1 --max-procs=$(($CORES*2*8)) \ + chown -R abc:abc + + echo "setting permissions::workspace" + chown abc:abc /config/workspace + find /config/workspace -maxdepth 4 -mindepth 1 -type d | \ + xargs --max-args=1 --max-procs=$(($CORES*2*16)) \ + chown -R abc:abc +else + chown -R abc:abc \ + /config +fi From a1608a1d3c676076a82e50ba0b0c5f9c4699a5b2 Mon Sep 17 00:00:00 2001 From: "G.J.R. Timmer" Date: Mon, 5 Jul 2021 01:15:55 +0200 Subject: [PATCH 2/5] Fix shellcheck warnings --- root/etc/cont-init.d/30-config | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/root/etc/cont-init.d/30-config b/root/etc/cont-init.d/30-config index 32fdf2a..877ac95 100644 --- a/root/etc/cont-init.d/30-config +++ b/root/etc/cont-init.d/30-config @@ -22,14 +22,14 @@ if [ -f "/usr/bin/find" ] && [ -f "/usr/bin/xargs" ]; then # Split workload between config and workspace echo "setting permissions::configuration" CORES=$(nproc --all) - find /config -maxdepth 4 -mindepth 1 -path /config/workspace -prune -false -o -type d | \ - xargs --max-args=1 --max-procs=$(($CORES*2*8)) \ + find /config -maxdepth 4 -mindepth 1 -path /config/workspace -prune -false -o -type d -print0 | \ + xargs --max-args=1 --max-procs=$((CORES*2*8)) \ chown -R abc:abc echo "setting permissions::workspace" chown abc:abc /config/workspace - find /config/workspace -maxdepth 4 -mindepth 1 -type d | \ - xargs --max-args=1 --max-procs=$(($CORES*2*16)) \ + find /config/workspace -maxdepth 4 -mindepth 1 -type d -print0 | \ + xargs --max-args=1 --max-procs=$((CORES*2*16)) \ chown -R abc:abc else chown -R abc:abc \ From d76a6d56cd00822b6a986632be15e97973db229b Mon Sep 17 00:00:00 2001 From: "G.J.R. Timmer" Date: Mon, 5 Jul 2021 01:22:21 +0200 Subject: [PATCH 3/5] Fix xargs do not run if empty --- root/etc/cont-init.d/30-config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/root/etc/cont-init.d/30-config b/root/etc/cont-init.d/30-config index 877ac95..cab6a93 100644 --- a/root/etc/cont-init.d/30-config +++ b/root/etc/cont-init.d/30-config @@ -23,13 +23,13 @@ if [ -f "/usr/bin/find" ] && [ -f "/usr/bin/xargs" ]; then echo "setting permissions::configuration" CORES=$(nproc --all) find /config -maxdepth 4 -mindepth 1 -path /config/workspace -prune -false -o -type d -print0 | \ - xargs --max-args=1 --max-procs=$((CORES*2*8)) \ + xargs -r --max-args=1 --max-procs=$((CORES*2*8)) \ chown -R abc:abc echo "setting permissions::workspace" chown abc:abc /config/workspace find /config/workspace -maxdepth 4 -mindepth 1 -type d -print0 | \ - xargs --max-args=1 --max-procs=$((CORES*2*16)) \ + xargs -r --max-args=1 --max-procs=$((CORES*2*16)) \ chown -R abc:abc else chown -R abc:abc \ From 2ef2329327db5ba8a775c9fc77a465af9573d34e Mon Sep 17 00:00:00 2001 From: "G.J.R. Timmer" Date: Thu, 16 Sep 2021 14:51:12 +0200 Subject: [PATCH 4/5] fix chown --- root/etc/cont-init.d/30-config | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/root/etc/cont-init.d/30-config b/root/etc/cont-init.d/30-config index cab6a93..1c09036 100644 --- a/root/etc/cont-init.d/30-config +++ b/root/etc/cont-init.d/30-config @@ -19,19 +19,19 @@ fi # permissions if [ -f "/usr/bin/find" ] && [ -f "/usr/bin/xargs" ]; then + CORES=$(nproc --all) + # Split workload between config and workspace echo "setting permissions::configuration" - CORES=$(nproc --all) - find /config -maxdepth 4 -mindepth 1 -path /config/workspace -prune -false -o -type d -print0 | \ - xargs -r --max-args=1 --max-procs=$((CORES*2*8)) \ + find /config -path /config/workspace -prune -false -o -type d -print0 | \ + xargs --null -r --max-args=1 --max-procs=$((CORES*2*8)) \ chown -R abc:abc echo "setting permissions::workspace" chown abc:abc /config/workspace - find /config/workspace -maxdepth 4 -mindepth 1 -type d -print0 | \ - xargs -r --max-args=1 --max-procs=$((CORES*2*16)) \ - chown -R abc:abc else - chown -R abc:abc \ - /config + # Set permissions on data mount + # do not decend into the workspace + chown -R abc:abc "$(ls /config -I workspace)" + chown abc:abc /config/workspace fi From 442dac7b54ebe38aa6068db4a7394e6dae214e66 Mon Sep 17 00:00:00 2001 From: aptalca Date: Thu, 16 Sep 2021 13:08:53 -0400 Subject: [PATCH 5/5] update changelog --- Jenkinsfile | 15 ++++++++++----- README.md | 1 + readme-vars.yml | 2 +- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3f2abd2..1e361b9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -375,7 +375,9 @@ pipeline { // Build Docker container for push to LS Repo stage('Build-Single') { when { - environment name: 'MULTIARCH', value: 'false' + expression { + env.MULTIARCH == 'false' || params.PACKAGE_CHECK == 'true' + } environment name: 'EXIT_STATUS', value: '' } steps { @@ -400,7 +402,10 @@ pipeline { // Build MultiArch Docker containers for push to LS Repo stage('Build-Multi') { when { - environment name: 'MULTIARCH', value: 'true' + allOf { + environment name: 'MULTIARCH', value: 'true' + expression { params.PACKAGE_CHECK == 'false' } + } environment name: 'EXIT_STATUS', value: '' } parallel { @@ -505,7 +510,7 @@ pipeline { sh '''#! /bin/bash set -e TEMPDIR=$(mktemp -d) - if [ "${MULTIARCH}" == "true" ]; then + if [ "${MULTIARCH}" == "true" ] && [ "${PACKAGE_CHECK}" == "false" ]; then LOCAL_CONTAINER=${IMAGE}:amd64-${META_TAG} else LOCAL_CONTAINER=${IMAGE}:${META_TAG} @@ -566,7 +571,7 @@ pipeline { steps { sh '''#! /bin/bash echo "Packages were updated. Cleaning up the image and exiting." - if [ "${MULTIARCH}" == "true" ]; then + if [ "${MULTIARCH}" == "true" ] && [ "${PACKAGE_CHECK}" == "false" ]; then docker rmi ${IMAGE}:amd64-${META_TAG} else docker rmi ${IMAGE}:${META_TAG} @@ -590,7 +595,7 @@ pipeline { steps { sh '''#! /bin/bash echo "There are no package updates. Cleaning up the image and exiting." - if [ "${MULTIARCH}" == "true" ]; then + if [ "${MULTIARCH}" == "true" ] && [ "${PACKAGE_CHECK}" == "false" ]; then docker rmi ${IMAGE}:amd64-${META_TAG} else docker rmi ${IMAGE}:${META_TAG} diff --git a/README.md b/README.md index c20df5d..2fd914c 100644 --- a/README.md +++ b/README.md @@ -260,6 +260,7 @@ Once registered you can define the dockerfile to use with `-f Dockerfile.aarch64 ## Versions +* **16.09.21:** - Fix slow `chown` on large workspace (contents of workspace folder no longer chowned). * **11.07.21:** - Bump node to 14 to fix builds * **08.05.21:** - Fix doc link * **04.02.20:** - Allow setting gui password via hash using env var `HASHED_PASSWORD`. diff --git a/readme-vars.yml b/readme-vars.yml index 01068cf..2fea39f 100644 --- a/readme-vars.yml +++ b/readme-vars.yml @@ -75,8 +75,8 @@ app_setup_block: | # changelog changelogs: + - { date: "16.09.21:", desc: "Fix slow `chown` on large workspace (contents of workspace folder no longer chowned)." } - { date: "11.07.21:", desc: "Bump node to 14 to fix builds" } - - { date: "05.07.21:", desc: "Fix slow `chown` on large workspace" } - { date: "08.05.21:", desc: "Fix doc link" } - { date: "04.02.20:", desc: "Allow setting gui password via hash using env var `HASHED_PASSWORD`." } - { date: "23.12.20:", desc: "Allow setting sudo password via hash using env var `SUDO_PASSWORD_HASH`." }