Newer
Older
########################################################################################################################
# EXECUTORS #
########################################################################################################################
# CircleCI base Node + Headless browsers + Clojure CLI - big one
builder:
working_directory: /home/circleci/metabase/metabase/
docker:
- image: metabase/ci:java-11-clj-1.11.0.1100.04-2022-build
# Java 11 tests also test Metabase with the at-rest encryption enabled. See
# https://metabase.com/docs/latest/operations-guide/encrypting-database-details-at-rest.html for an explanation of
# what this means.
java-11:
working_directory: /home/circleci/metabase/metabase/
docker:
- image: metabase/ci:java-11-clj-1.11.0.1100.04-2022-build
environment:
MB_ENCRYPTION_SECRET_KEY: Orw0AAyzkO/kPTLJRxiyKoBHXa/d6ZcO+p+gpZO/wSQ=
working_directory: /home/circleci/metabase/metabase/
docker:
- image: metabase/ci:java-17-clj-1.11.0.1100.04-2022-build
mongo-4-0-ssl:
working_directory: /home/circleci/metabase/metabase/
docker:
- image: metabase/ci:java-11-clj-1.11.0.1100.04-2022-build
environment:
MB_TEST_MONGO_REQUIRES_SSL: true
- image: metabase/qa-databases:mongo-sample-4.0
command: mongod --dbpath /data/db2/ --sslMode requireSSL --sslPEMKeyFile /etc/mongo/metamongo.pem --sslCAFile /etc/mongo/metaca.crt
mongo-5-0-ssl:
working_directory: /home/circleci/metabase/metabase/
docker:
- image: metabase/ci:java-11-clj-1.11.0.1100.04-2022-build
environment:
MB_TEST_MONGO_REQUIRES_SSL: true
- image: metabase/qa-databases:mongo-sample-5.0
command: mongod --dbpath /data/db2/ --tlsMode requireTLS --tlsCertificateKeyFile /etc/mongo/metamongo.pem --tlsCAFile /etc/mongo/metaca.crt
########################################################################################################################
########################################################################################################################
# `default_parameters` isn't a key that CircleCI uses, but this form lets us reuse parameter definitions
default_parameters: &Params
edition:
default: "oss"
# .BACKEND-CHECKSUMS and .MODULE-CHECKSUMS are created during the checkout step; see that step
# for exact details as to what they contain.
#
# To support cache busting, we create a file named .CACHE-PREFIX in the checkout step and use its checksum as the
# prefix for every cache key. If the commit message DOES NOT include [ci nocache], we create an empty file; the
# checksum will always be the same for this file. If the commit message DOES include [ci nocache], we'll write the
# unique ID of the current pipeline to .CACHE-PREFIX which will effectively bust our caches whenever it's used.
### Deps Keys ###
# Why don't we use fallback keys for backend deps? We used to, but it allowed the cache to grow
# uncontrollably since old deps would continue to accumulate. Restoring big caches is really slow in Circle. It's
# actually faster to recreate the deps cache from scratch whenever we need to which keeps the size down.
cache-key-backend-deps: &CacheKeyBackendDeps
# TODO -- this should actually include the Java source files and the Spark SQL AOT source files as well since we now
# compile those as part of this step. FIXME
key: v5-{{ checksum ".CACHE-PREFIX" }}-be-deps-{{ checksum "deps.edn" }}-{{ checksum ".SCRIPTS-DEPS-CHECKSUMS" }}
# Key used for implementation of run-on-change -- this is the cache key that contains the .SUCCESS dummy file
Cam Saul
committed
# By default the key ALWAYS includes the name of the test job itself ($CIRCLE_JOB) so you don't need to add that yourself.
Cam Saul
committed
key: v5-{{ checksum ".CACHE-PREFIX" }}-run-on-change-{{ .Environment.CIRCLE_JOB }}-<< parameters.checksum >>
########################################################################################################################
# COMMANDS #
########################################################################################################################
commands:
attach-workspace:
steps:
- attach_workspace:
at: /home/circleci/
# For the restore-deps-cache commands below, only restore the cache if there's an exact match. This means whatever
# is in the cache will be exactly what's used and the cache won't keep growing uncontrollably going forward.
restore-be-deps-cache:
steps:
- restore_cache:
name: Restore cached backend dependencies
<<: *CacheKeyBackendDeps
# run-on-change lets you only run steps if changes have happened to relevant files since the last time it was run
# successfully. Uses a cache key to record successful runs -- cache key should be unique for job and relevant source
# files -- use a checksum! It works like this:
#
# 1. Calculate a cache key using a checksum of relevant files for the step in question, e.g. a backend linter step
#
# 2. When the step completes successfully, create a dummy file .SUCCESS and cache it with that cache key.
#
# 3. On subsequent runs:
#
# a. Attempt to restore the cache using an exact match for this cache key
#
# b. If we have a cache entry for that key, .SUCCESS will get restored
#
# c. If this command has the skip-job-if-commit-message-includes-ci-quick option enabled, and commit message includes
# [ci quick], create a dummy file .SUCCESS if not already present. Ignored for master/release branches.
#
# d. If commit message includes [ci noskip], delete .SUCCESS so the job will be forced to run.
#
# e. If .SUCCESS is present, we can skip the rest of the job, including potentially slow steps like restoring
# dependency caches or the like. This logs a link to the last successful (not skipped) run of the job
#
# Important! If this step is skipped because no changes have happened, the entire JOB will halt with a success
# status -- no steps that happen AFTER run-on-change will be ran. Keep this in mind!
#
# f. If .SUCCESS is not present, proceed as normal, and create and cache .SUCCESS if the job succeeds
run-on-change:
parameters:
steps:
type: steps
# Whether to skip the rest of the job if commit message includes [ci quick]
skip-job-if-commit-message-includes-ci-quick:
type: boolean
default: false
steps:
- restore_cache:
name: Restore dummy file .SUCCESS if it exists for cache key << parameters.checksum >>
<<: *CacheKeyRunOnChange
- when:
condition: << parameters.skip-job-if-commit-message-includes-ci-quick >>
steps:
- run:
name: "Skip tests (create dummy file .SUCCESS) if commit message contains [ci quick] and branch isn't a master/release branch"
command: |
if [[ "$CIRCLE_BRANCH" =~ ^master|release-.+$ ]]; then
echo "branch '$CIRCLE_BRANCH' is a master or release branch: ignoring [ci quick]"
elif [[ `cat .COMMIT` == *"[ci quick]"* ]]; then
echo 'Commit message includes [ci quick]. Creating dummy file .SUCCESS'
touch .SUCCESS
else
echo 'Commit message does not include [ci quick]'
fi
- run:
name: "Force test run (delete dummy file .SUCCESS) if commit message includes [ci noskip]"
command: |
if [[ `cat .COMMIT` == *"[ci noskip]"* ]]; then
echo 'Commit message includes [ci noskip] -- forcing test run (delete .SUCCESS)'
rm -f .SUCCESS
else
echo 'Commit message does not include [ci noskip]'
fi
command: |
if [ -f .SUCCESS ]; then
echo '.SUCCESS is present: skipping rest of job.'
echo "Link to last successful run (if available): $(cat .SUCCESS)"
circleci-agent step halt
fi
- steps: << parameters.steps >>
- run:
command: |
echo "$CIRCLE_BUILD_URL" > .SUCCESS
- save_cache:
name: Persist dummy file .SUCCESS to cache with key << parameters.checksum >>
paths:
- /home/circleci/metabase/metabase/.SUCCESS
- run:
name: Delete dummy file .SUCCESS so subsequent steps don't see it
command: rm /home/circleci/metabase/metabase/.SUCCESS
# Creates a file that contains checksums for all the files found using the find command with supplied arguments.
# You can use a checksum of the checksum file for cache keys including run-on-change cache keys.
create-checksum-file:
parameters:
filename:
type: string
find-args:
type: string
steps:
- run:
name: Create << parameters.filename >> checksum file
command: |
for file in `find << parameters.find-args >> | sort`; do
echo `md5sum "$file"` >> "<< parameters.filename >>"
done
if [ ! -f "<< parameters.filename >>" ]; then
echo 'Error: no matching files. Did you remember to attach the workspace?'
exit 1
fi
echo "Created checksums for $(cat << parameters.filename >> | wc -l) files"
parameters:
before-steps:
type: steps
default: []
type: string
after-steps:
type: steps
default: []
<<: *Params
steps:
- restore-be-deps-cache
- steps: << parameters.before-steps >>
- run:
name: clojure << parameters.clojure-args >>:<< parameters.edition >>:<< parameters.edition >>-dev
clojure << parameters.clojure-args >>:<< parameters.edition >>:<< parameters.edition >>-dev
no_output_timeout: 15m
- steps: << parameters.after-steps >>
- store_test_results:
path: /home/circleci/metabase/metabase/target/junit
wait-for-port:
parameters:
port:
type: integer
steps:
- run:
name: Wait for port << parameters.port >> to be ready
while ! nc -z localhost << parameters.port >>; do sleep 0.1; done
no_output_timeout: 15m
fetch-jdbc-driver:
parameters:
source:
type: string
dest:
type: string
steps:
- run:
name: Make plugins dir
command: mkdir /home/circleci/metabase/metabase/plugins
- run:
name: Download JDBC driver JAR << parameters.dest >>
wget --output-document=plugins/<< parameters.dest >> ${<< parameters.source >>}
no_output_timeout: 15m
run-command:
parameters:
command:
type: string
steps:
- run:
name: Run command
command: << parameters.command >>
########################################################################################################################
########################################################################################################################
# .BACKEND-CHECKSUMS is every Clojure source file as well as dependency files like deps.edn and plugin manifests
- create-checksum-file:
filename: .BACKEND-CHECKSUMS
find-args: ". -type f -name '*.clj' -or -name '*.cljc' -or -name '*.java' -or -name '*.edn' -or -name '*.yaml' -or -name sample-dataset.db.mv.db"
# .SCRIPTS-DEPS-CHECKSUMS is all the deps.edn files inside ./bin
- create-checksum-file:
filename: .SCRIPTS-DEPS-CHECKSUMS
find-args: "bin -type f -name 'deps.edn'"
# .MODULES-CHECKSUMS is every Clojure source file in the modules/ directory as well as plugin manifests
- create-checksum-file:
filename: .MODULES-CHECKSUMS
find-args: "./modules -type f -name '*.clj' -or -name metabase-plugin.yaml"
name: Save last git commit message to .COMMIT
command: git log -1 > .COMMIT
name: Determine what to do to .git directory
command: |
if [[ $CIRCLE_BRANCH == release* ]]; then
echo 'This is a release branch; preserving .git directory to determine version'
else
echo 'This is not a release branch; removing .git directory (not needed for tests)'
rm -rf /home/circleci/metabase/metabase/.git
fi
- run:
name: Remove ./OSX directory (not needed for tests)
command: rm -rf /home/circleci/metabase/metabase/OSX
# .CACHE-PREFIX is described above in the Cache Keys section of this file
- run:
name: 'Create cache key prefix .CACHE-PREFIX to bust caches if commit message includes [ci nocache]'
command: |
if [[ `cat .COMMIT` == *"[ci nocache]"* ]]; then
echo 'Commit message includes [ci nocache]; using cache-busting prefix'
echo '<< pipeline.id >>' > .CACHE-PREFIX
else
echo '' > .CACHE-PREFIX
fi
Cam Saul
committed
- run:
name: 'If $CIRCLE_JOB is unset then bust the cache (see #24128 for details)'
command: |
if [ -z "$CIRCLE_JOB" ]; then
echo '$CIRCLE_JOB is unset. Using cache-busting prefix.'
echo '<< pipeline.id >>' > .CACHE-PREFIX
fi
- run:
name: Create static visualization js bundle
command: yarn build-static-viz
- run:
name: 'Check for branch name to bust caches if it is a release branch'
command: |
if [[ $CIRCLE_BRANCH == release* || $CIRCLE_BRANCH == master ]]; then
echo 'This is a release or master branch; using cache-busting prefix'
Cam Saul
committed
echo '<< pipeline.id >>' > .CACHE-PREFIX
- run:
name: Make SSL certificates for Mongo available
command: >-
curl https://raw.githubusercontent.com/metabase/metabase-qa/master/dbs/mongo/certificates/metabase.crt
-o /home/circleci/metabase/metabase/test_resources/ssl/mongo/metabase.crt
https://raw.githubusercontent.com/metabase/metabase-qa/master/dbs/mongo/certificates/metabase.key
-o /home/circleci/metabase/metabase/test_resources/ssl/mongo/metabase.key
https://raw.githubusercontent.com/metabase/metabase-qa/master/dbs/mongo/certificates/metaca.crt
-o /home/circleci/metabase/metabase/test_resources/ssl/mongo/metaca.crt
- persist_to_workspace:
root: /home/circleci/
paths:
- metabase/metabase
########################################################################################################################
# BACKEND #
########################################################################################################################
parameters:
<<: *Params
# This step is pretty slow, even with the cache, so only run it if deps.edn has changed
checksum: 'v5-{{ checksum "deps.edn" }}-{{ checksum ".SCRIPTS-DEPS-CHECKSUMS" }}'
steps:
- restore-be-deps-cache
- run:
name: Compile Java source file(s)
command: clojure -X:deps prep
- run:
name: Fetch dependencies
command: clojure -P -X:dev:ci:ee:ee-dev:drivers:drivers-dev
- run:
name: Fetch dependencies (./bin/build/build-mb)
command: cd /home/circleci/metabase/metabase/bin/build-mb && clojure -P -M:test
# Not sure why this is needed since you would think build-mb would fetch this stuff as well. It doesn't
# seem to fetch everything tho. :shrug:
- run:
name: Fetch dependencies (./bin/build/build-drivers)
command: cd /home/circleci/metabase/metabase/bin/build-drivers && clojure -P -M:test
name: Cache backend dependencies
<<: *CacheKeyBackendDeps
paths:
- /home/circleci/.m2
- /home/circleci/.gitlibs
- /home/circleci/metabase/metabase/java/target/classes
- /home/circleci/metabase/metabase/modules/drivers/sparksql/target/classes
parameters:
e:
type: executor
before-steps:
type: steps
default: []
after-steps:
type: steps
default: []
skip-when-no-change:
type: boolean
default: false
java-version:
type: string
default: ""
version:
type: string
default: ""
condition: << parameters.skip-when-no-change >>
steps:
- run-on-change:
checksum: '{{ checksum ".BACKEND-CHECKSUMS" }}'
before-steps: << parameters.before-steps >>
after-steps: << parameters.after-steps >>
edition: << parameters.edition >>
- unless:
condition: << parameters.skip-when-no-change >>
before-steps: << parameters.before-steps >>
after-steps: << parameters.after-steps >>
edition: << parameters.edition >>
be-linter-reflection-warnings:
checksum: '{{ checksum ".BACKEND-CHECKSUMS" }}-{{ checksum "bin/reflection-linter" }}'
steps:
- restore-be-deps-cache
- run:
name: Run reflection warnings checker
command: ./bin/reflection-linter
no_output_timeout: 15m
test-driver:
parameters:
e:
type: executor
driver:
type: string
timeout:
type: string
before-steps:
type: steps
default: []
after-steps:
type: steps
default: []
description:
type: string
default: ""
extra-env:
type: string
default: ""
test-args:
type: string
default: ""
version:
type: string
default: ""
checksum: '{{ checksum ".BACKEND-CHECKSUMS" }}'
skip-job-if-commit-message-includes-ci-quick: true
steps:
- restore-be-deps-cache
- steps: << parameters.before-steps >>
- run:
name: Test << parameters.driver >> driver << parameters.description >>
environment:
Cam Saul
committed
DRIVERS: << parameters.driver >>
command: >
<< parameters.extra-env >> clojure -X:dev:ci:ee:ee-dev:drivers:drivers-dev:test
<< parameters.test-args >>
no_output_timeout: << parameters.timeout >>
- store_test_results:
path: /home/circleci/metabase/metabase/target/junit
- steps: << parameters.after-steps >>
########################################################################################################################
# WORKFLOWS #
########################################################################################################################
# `default_matrix` isn't a key that CircleCI uses, but this form lets us reuse the matrix: block
default_matrix: &Matrix
matrix:
parameters:
edition: ["ee", "oss"]
workflows:
version: 2
build:
jobs:
- checkout
- be-deps:
requires:
- checkout
name: be-linter-cloverage
requires:
- be-deps
# TODO FIXME
clojure-args: -X:dev:ee:ee-dev:test:cloverage
after-steps:
- run:
name: Upload code coverage to codecov.io
command: bash <(curl -s https://codecov.io/bash) -F back-end
skip-when-no-change: true
version: ["mongo-4-0-ssl", "mongo-5-0-ssl"]
name: be-tests-<< matrix.version >>-ee
description: "(<< matrix.version >>)"
extra-env: >-
MB_MONGO_TEST_USER=metabase
MB_MONGO_TEST_PASSWORD=metasample123
name: be-tests-oracle-ee
before-steps:
- fetch-jdbc-driver:
source: ORACLE_JDBC_JAR
dest: ojdbc8.jar
- run:
name: Ensure truststore file
command: ls /home/circleci/metabase/metabase/resources/certificates/rds_root_ca_truststore.jks
extra-env: >-
MB_ORACLE_SSL_TEST_SSL=true
MB_ORACLE_SSL_TEST_PORT=2484
MB_ORACLE_SSL_TEST_SSL_USE_TRUSTSTORE=true
MB_ORACLE_SSL_TEST_SSL_TRUSTSTORE_PATH=/home/circleci/metabase/metabase/resources/certificates/rds_root_ca_truststore.jks
MB_ORACLE_SSL_TEST_SSL_TRUSTSTORE_OPTIONS=local
MB_ORACLE_SSL_TEST_SSL_TRUSTSTORE_PASSWORD_VALUE=metabase