From 76bdb7e65c5a9b35e822d3fd22ce5fdf6669e2ff Mon Sep 17 00:00:00 2001 From: Benoit Pierre Date: Mon, 13 May 2024 21:33:53 +0200 Subject: [PATCH] ci/circle: improve caching Don't cache the whole base directory; instead independently cache: - the build output directory - the ccache directory This allow completely avoiding the need for building base if a cache (keyed on the relevant parts of the git tree for base) is successfully restored, as well as reducing the amount of cached data. Additionally, ensure caches are isolated by branch name (but allow falling back to a "master" cache for restore): we don't want PRs using each others' caches. --- .ci/after_success.sh | 2 +- .ci/build.sh | 9 ++- .ci/test.sh | 4 +- .circleci/config.yml | 180 ++++++++++++++++++++++++++++++------------- 4 files changed, 138 insertions(+), 57 deletions(-) diff --git a/.ci/after_success.sh b/.ci/after_success.sh index 7d16a78b6..049eb8290 100755 --- a/.ci/after_success.sh +++ b/.ci/after_success.sh @@ -15,7 +15,7 @@ if [ -z "${CIRCLE_PULL_REQUEST}" ] && [ "${CIRCLE_BRANCH}" = 'master' ]; then if [ "${CIRCLE_NODE_INDEX}" = 0 ]; then travis_retry make coverage - pushd koreader-*/koreader && { + pushd install/koreader && { # see https://github.com/codecov/example-lua bash <(curl -s https://codecov.io/bash) } && popd || exit diff --git a/.ci/build.sh b/.ci/build.sh index 04e486302..4448cf5b7 100755 --- a/.ci/build.sh +++ b/.ci/build.sh @@ -4,4 +4,11 @@ CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=/dev/null source "${CI_DIR}/common.sh" -make all +# Build. +cmd=(make all) +if [[ -d base/build ]]; then + cmd+=(--assume-old=base) +fi +"${cmd[@]}" + +# vim: sw=4 diff --git a/.ci/test.sh b/.ci/test.sh index 47fda1d70..5bb7fa444 100755 --- a/.ci/test.sh +++ b/.ci/test.sh @@ -4,9 +4,11 @@ CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=/dev/null source "${CI_DIR}/common.sh" -pushd koreader-emulator-x86_64-linux-gnu/koreader && { +pushd install/koreader && { # the circleci command spits out newlines; we want spaces instead BUSTED_SPEC_FILE="$(circleci tests glob "spec/front/unit/*_spec.lua" | circleci tests split --split-by=timings --timings-type=filename | tr '\n' ' ')" } && popd || exit make testfront BUSTED_SPEC_FILE="${BUSTED_SPEC_FILE}" + +# vim: sw=4 diff --git a/.circleci/config.yml b/.circleci/config.yml index 0e5d4f928..85f4cc3c6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,80 +1,125 @@ -version: 2 +version: "2.1" -workflows: - version: 2 - build: - jobs: - - build - - docs: - context: koreader-vars - filters: - branches: - only: master - requires: - - build +# Parameters. {{{ + +parameters: + + # Bump this to reset all caches. + cache_epoch: + type: integer + default: 0 + +# }}} + +# Executors. {{{ + +executors: + + base: + docker: + - image: koreader/kobase:0.3.2-20.04 + auth: + username: $DOCKER_USERNAME + password: $DOCKER_PASSWORD + +# }}} + +# Jobs. {{{ jobs: + + # Build. {{{ + build: - docker: - - image: koreader/kobase:0.3.2-20.04 - auth: - username: $DOCKER_USERNAME - password: $DOCKER_PASSWORD - environment: - EMULATE_READER: 1 + executor: base environment: BASH_ENV: "~/.bashrc" + CCACHE_MAXSIZE: "256M" + EMULATE_READER: "1" + MAKEFLAGS: "OUTPUT_DIR=build INSTALL_DIR=install" parallelism: 2 steps: + # Checkout / fetch. {{{ - checkout - # need to init some stuff first or git will complain when sticking in base cache - - run: git submodule update --init base && git -C base submodule update --depth 1 --init --jobs 3 --recursive - # we can't use command output directly for cache check so we write it to git-rev-base - - run: pushd base && git_rev_base=$(git describe HEAD) && popd && echo $git_rev_base && echo $git_rev_base >git-rev-base - - restore_cache: - keys: - - build-{{ arch }}-{{ checksum "git-rev-base" }} - - # installs everything and caches base - run: - name: fetch + name: Fetch command: .ci/fetch.sh + # }}} + # Check. - run: - name: check + name: Check command: .ci/check.sh + # Restore / setup caches. {{{ - run: - name: build + name: Generate cache key + command: make -C base TARGET= cache-key + - restore_cache: + name: Restore build directory + keys: + - &CACHE_KEY_BUILD_DIR '<< pipeline.parameters.cache_epoch >>-{{ .Environment.CIRCLE_JOB }}-build-{{ arch }}-{{ checksum "base/cache-key" }}' + - restore_cache: + name: Restore build cache + keys: + - &CACHE_KEY_BUILD_CACHE '<< pipeline.parameters.cache_epoch >>-{{ .Environment.CIRCLE_JOB }}-ccache-{{ arch }}-{{ checksum "base/cache-key" }}' + - '<< pipeline.parameters.cache_epoch >>-{{ .Environment.CIRCLE_JOB }}-ccache-{{ arch }}-' + - run: + name: Setup build cache + command: | + set -x + which ccache + ccache --version + ccache --zero-stats + ccache --show-config + # }}} + # Build. + - run: + name: Build command: .ci/build.sh - # we want to save cache prior to testing so we don't have to clean it up + # Clean / save caches. {{{ + # We want to save cache prior to testing so we don't have to clean it up. + - run: + name: Clean caches + when: always + command: | + set -x + # Trim the build directory. + rm -rf base/build/thirdparty + ccache --cleanup >/dev/null + ccache --show-stats - save_cache: - key: build-{{ arch }}-{{ checksum "git-rev-base" }} + name: Save build cache + key: *CACHE_KEY_BUILD_CACHE paths: - - "/home/ko/.ccache" - - "base" - - # our lovely unit tests + - /home/ko/.ccache + - save_cache: + name: Save build directory + key: *CACHE_KEY_BUILD_DIR + paths: + - base/build + # }}} + # Tests / coverage. {{{ + # Our lovely unit tests. - run: - name: test + name: Test command: .ci/test.sh - - # docs, coverage, and test timing (can we use two outputs at once?); master branch only + # Docs, coverage, and test timing (can we use two outputs at once?); master branch only. - run: - name: coverage + name: Coverage command: .ci/after_success.sh - # by storing the test results CircleCI automatically distributes tests based on execution time + # By storing the test results CircleCI automatically distributes tests based on execution time. - store_test_results: - path: koreader-emulator-x86_64-linux-gnu/koreader - # CircleCI doesn't make the test results available as artifacts (October 2017) + path: &TESTS_XML install/koreader/junit-test-results.xml + # CircleCI doesn't make the test results available as artifacts (October 2017). - store_artifacts: - path: koreader-emulator-x86_64-linux-gnu/koreader/junit-test-results.xml + path: *TESTS_XML + # }}} + + # }}} + + # Docs. {{{ + docs: - docker: - - image: koreader/kobase:0.3.2-20.04 - auth: - username: $DOCKER_USERNAME - password: $DOCKER_PASSWORD - environment: - EMULATE_READER: 1 + executor: base environment: BASH_ENV: "~/.bashrc" parallelism: 1 @@ -83,8 +128,35 @@ jobs: - run: name: fetch command: .ci/fetch.sh - # docs, coverage, and test timing (can we use two outputs at once?); master branch only - run: name: docs-and-translation command: .ci/after_success_docs_translation.sh + + # }}} + +# }}} + +# Workflows. {{{ + +workflows: + + version: 2 + + build: + + jobs: + + - build + + - docs: + context: koreader-vars + filters: + branches: + only: master + requires: + - build + +# }}} + +# vim: foldmethod=marker foldlevel=0