Skip to content
Snippets Groups Projects
Unverified Commit e80b3f0f authored by Cam Saul's avatar Cam Saul Committed by GitHub
Browse files

Merge pull request #9840 from metabase/circle-ci-buildfile-cleanup 

CircleCI config simplifications step 1
parents 1b3c60cf 7a663b1e
Branches
Tags
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 294 additions and 417 deletions
.circleci/config.yml
+ 294
− 417
View file @ e80b3f0f
defaults: &defaults
working_directory: /home/circleci/metabase/metabase/
docker:
- image: circleci/clojure:lein-2.8.1-node-browsers
version: 2.1
restore-be-deps-cache: &restore-be-deps-cache
keys:
- be-deps-{{ checksum "project.clj" }}
- be-deps-
########################################################################################################################
# EXECUTORS #
########################################################################################################################
executors:
default:
working_directory: /home/circleci/metabase/metabase/
docker:
- image: circleci/clojure:lein-2.8.1-node-browsers
postgres-9-6:
working_directory: /home/circleci/metabase/metabase/
docker:
- image: circleci/clojure:lein-2.8.1-node-browsers
- image: circleci/postgres:9.6-alpine
environment:
POSTGRES_USER: circle_test
POSTGRES_DB: circle_test
environment:
MB_DB_TYPE: postgres
MB_DB_PORT: 5432
MB_DB_HOST: localhost
MB_DB_DBNAME: circle_test
MB_DB_USER: circle_test
MB_POSTGRESQL_TEST_USER: circle_test
mysql-5-7:
working_directory: /home/circleci/metabase/metabase/
docker:
- image: circleci/clojure:lein-2.8.1-node-browsers
- image: circleci/mysql:5.7.23
environment:
MB_DB_TYPE: mysql
MB_DB_HOST: localhost
MB_DB_PORT: 3306
MB_DB_DBNAME: circle_test
MB_DB_USER: root
MB_MYSQL_TEST_USER: root
########################################################################################################################
# COMMANDS #
########################################################################################################################
commands:
attach-workspace:
steps:
- attach_workspace:
at: /home/circleci/
restore-be-deps-cache:
steps:
- restore_cache:
keys:
- be-deps-{{ checksum "project.clj" }}
- be-deps-
restore-fe-deps-cache:
steps:
- restore_cache:
keys:
- fe-deps-{{ checksum "yarn.lock" }}
- fe-deps-
fetch-jdbc-driver-jar:
parameters:
driver:
type: string
source:
type: string
dest:
type: string
steps:
- run:
name: Make plugins dir
command: mkdir /home/circleci/metabase/metabase/plugins
- run:
name: Download << parameters.driver >> JDBC driver JAR
command: >
/home/circleci/metabase/metabase/.circleci/skip-driver-tests.sh << parameters.driver >> ||
wget --output-document=plugins/<< parameters.dest >> ${<< parameters.source >>}
no_output_timeout: 5m
wait-for-driver:
parameters:
driver:
type: string
port:
type: integer
steps:
- run:
name: Wait for << parameters.driver >> to be ready
command: >
/home/circleci/metabase/metabase/.circleci/skip-driver-tests.sh << parameters.driver >> ||
while ! nc -z localhost << parameters.port >>; do sleep 0.1; done
no_output_timeout: 5m
run-lein-command:
parameters:
command:
type: string
command-name:
type: string
steps:
- attach-workspace
- restore-be-deps-cache
- run:
name: << parameters.command-name >>
command: lein with-profile +ci << parameters.command >>
no_output_timeout: 5m
test-driver:
parameters:
driver:
type: string
before-steps:
type: steps
default: []
timeout:
type: string
default: 5m
steps:
- attach-workspace
- restore-be-deps-cache
- steps: << parameters.before-steps >>
- run:
name: Test << parameters.driver >> driver
environment:
DRIVERS: h2,<< parameters.driver >>
command: >
/home/circleci/metabase/metabase/.circleci/skip-driver-tests.sh << parameters.driver >> ||
lein with-profile +ci test
no_output_timeout: << parameters.timeout >>
test-migrate-from-h2:
parameters:
db-type:
type: string
steps:
- attach-workspace
- restore-be-deps-cache
- run:
name: Test migrating from H2 -> << parameters.db-type >>
environment:
MB_DB_TYPE: << parameters.db-type >>
MB_DB_HOST: localhost
command: >
lein run load-from-h2 ./frontend/test/__runner__/test_db_fixture.db
no_output_timeout: 5m
restore-fe-deps-cache: &restore-fe-deps-cache
keys:
- fe-deps-{{ checksum "yarn.lock" }}
- fe-deps-
version: 2.1
jobs:
########################################################################################################################
......@@ -21,7 +157,7 @@ jobs:
########################################################################################################################
checkout:
<<: *defaults
executor: default
steps:
- restore_cache:
keys:
......@@ -53,12 +189,10 @@ jobs:
########################################################################################################################
be-deps:
<<: *defaults
executor: default
steps:
- attach_workspace:
at: /home/circleci/
- restore_cache:
<<: *restore-be-deps-cache
- attach-workspace
- restore-be-deps-cache
- run: lein with-profile +include-all-drivers deps
- save_cache:
key: be-deps-{{ checksum "project.clj" }}
......@@ -66,447 +200,209 @@ jobs:
- /home/circleci/.m2
be-tests:
<<: *defaults
executor: default
steps:
- attach_workspace:
at: /home/circleci/
- restore_cache:
<<: *restore-be-deps-cache
- run:
name: Run backend unit tests with Java 8
command: lein with-profile +ci test
no_output_timeout: 5m
- run-lein-command:
command-name: Run backend unit tests with Java 8
command: test
be-tests-java-11:
<<: *defaults
executor: default
docker:
- image: circleci/clojure:openjdk-11-lein-2.8.1
steps:
- attach_workspace:
at: /home/circleci/
- restore_cache:
<<: *restore-be-deps-cache
- run:
name: Run backend unit tests with Java 11
command: lein with-profile +ci test
no_output_timeout: 5m
- run-lein-command:
command-name: Run backend unit tests with Java 11
command: test
be-linter-eastwood:
<<: *defaults
executor: default
steps:
- attach_workspace:
at: /home/circleci/
- restore_cache:
<<: *restore-be-deps-cache
- run:
name: Run Eastwood linter
command: lein with-profile +ci eastwood
no_output_timeout: 5m
- run-lein-command:
command-name: Run eastwood linter
command: eastwood
be-linter-docstring-checker:
<<: *defaults
executor: default
steps:
- attach_workspace:
at: /home/circleci/
- restore_cache:
<<: *restore-be-deps-cache
- run:
name: Run dockstring-checker
command: lein with-profile +ci docstring-checker
no_output_timeout: 5m
- run-lein-command:
command-name: Check docstrings
command: docstring-checker
be-linter-namespace-decls:
<<: *defaults
executor: default
steps:
- attach_workspace:
at: /home/circleci/
- restore_cache:
<<: *restore-be-deps-cache
- run:
name: Run namespace decl checker
command: lein with-profile +ci check-namespace-decls
no_output_timeout: 5m
- run-lein-command:
command-name: Check namespace declarations
command: check-namespace-decls
be-linter-bikeshed:
<<: *defaults
executor: default
steps:
- attach_workspace:
at: /home/circleci/
- restore_cache:
<<: *restore-be-deps-cache
- run:
name: Run Bikeshed linter
command: lein with-profile +ci bikeshed
no_output_timeout: 5m
- run-lein-command:
command-name: Run bikeshed linter
command: bikeshed
be-linter-reflection-warnings:
<<: *defaults
executor: default
steps:
- attach_workspace:
at: /home/circleci/
- restore_cache:
<<: *restore-be-deps-cache
- attach-workspace
- restore-be-deps-cache
- run:
name: Run reflection warnings checker
command: ./bin/reflection-linter
no_output_timeout: 5m
be-tests-mysql:
working_directory: /home/circleci/metabase/metabase/
docker:
- image: circleci/clojure:lein-2.8.1-node-browsers
- image: circleci/mysql:5.7.23
executor: mysql-5-7
environment:
MB_ENCRYPTION_SECRET_KEY: Orw0AAyzkO/kPTLJRxiyKoBHXa/d6ZcO+p+gpZO/wSQ=
steps:
- attach_workspace:
at: /home/circleci/
- restore_cache:
<<: *restore-be-deps-cache
- run:
name: Run backend unit tests (MySQL)
environment:
DRIVERS: h2,mysql
MB_ENCRYPTION_SECRET_KEY: Orw0AAyzkO/kPTLJRxiyKoBHXa/d6ZcO+p+gpZO/wSQ=
MB_DB_TYPE: mysql
MB_DB_HOST: localhost
MB_DB_PORT: 3306
MB_DB_DBNAME: circle_test
MB_DB_USER: root
MB_MYSQL_TEST_USER: root
command: >
/home/circleci/metabase/metabase/.circleci/skip-driver-tests.sh mysql ||
lein with-profile +ci test
no_output_timeout: 5m
- test-driver:
driver: mysql
be-tests-postgres:
working_directory: /home/circleci/metabase/metabase/
docker:
- image: circleci/clojure:lein-2.8.1-node-browsers
- image: circleci/postgres:9.6-alpine
environment:
POSTGRES_USER: circle_test
POSTGRES_DB: circle_test
executor: postgres-9-6
steps:
- attach_workspace:
at: /home/circleci/
- restore_cache:
<<: *restore-be-deps-cache
- run:
name: Run backend unit tests (Postgres)
environment:
DRIVERS: h2,postgres
MB_DB_TYPE: postgres
MB_DB_PORT: 5432
MB_DB_HOST: localhost
MB_DB_DBNAME: circle_test
MB_DB_USER: circle_test
MB_POSTGRESQL_TEST_USER: circle_test
command: >
/home/circleci/metabase/metabase/.circleci/skip-driver-tests.sh postgres ||
lein with-profile +ci test
no_output_timeout: 5m
- test-driver:
driver: postgres
be-tests-sparksql:
working_directory: /home/circleci/metabase/metabase/
executor: default
docker:
- image: circleci/clojure:lein-2.8.1-node-browsers
- image: metabase/spark:2.1.1
steps:
- attach_workspace:
at: /home/circleci/
- restore_cache:
<<: *restore-be-deps-cache
- run:
name: Wait for SparkSQL to be ready
command: >
/home/circleci/metabase/metabase/.circleci/skip-driver-tests.sh sparksql ||
while ! nc -z localhost 10000; do sleep 0.1; done
no_output_timeout: 5m
- run:
name: Run backend unit tests (SparkSQL)
environment:
DRIVERS: h2,sparksql
command: >
/home/circleci/metabase/metabase/.circleci/skip-driver-tests.sh sparksql ||
lein with-profile +ci test
no_output_timeout: 5m
- test-driver:
driver: sparksql
before-steps:
- wait-for-driver:
driver: sparksql
port: 10000
be-tests-mongo:
working_directory: /home/circleci/metabase/metabase/
executor: default
docker:
- image: circleci/clojure:lein-2.8.1-node-browsers
- image: circleci/mongo:3.4
steps:
- attach_workspace:
at: /home/circleci/
- restore_cache:
<<: *restore-be-deps-cache
- run:
name: Run backend unit tests (MongoDB 3.4)
environment:
DRIVERS: h2,mongo
command: >
/home/circleci/metabase/metabase/.circleci/skip-driver-tests.sh mongo ||
lein with-profile +ci test
no_output_timeout: 5m
- test-driver:
driver: mongo
be-tests-vertica:
working_directory: /home/circleci/metabase/metabase/
executor: default
docker:
- image: circleci/clojure:lein-2.8.1-node-browsers
- image: sumitchawla/vertica
steps:
- attach_workspace:
at: /home/circleci/
- restore_cache:
<<: *restore-be-deps-cache
- run:
name: Make plugins dir
command: mkdir /home/circleci/metabase/metabase/plugins
- run:
name: Download Vertica JAR
command: >
/home/circleci/metabase/metabase/.circleci/skip-driver-tests.sh vertica ||
wget --output-document=plugins/vertica-jdbc-7.1.2-0.jar $VERTICA_JDBC_JAR
no_output_timeout: 5m
- run:
name: Run backend unit tests (Vertica)
environment:
DRIVERS: h2,vertica
command: >
/home/circleci/metabase/metabase/.circleci/skip-driver-tests.sh vertica ||
lein with-profile +ci test
no_output_timeout: 5m
- test-driver:
driver: vertica
before-steps:
- fetch-jdbc-driver-jar:
driver: vertica
source: VERTICA_JDBC_JAR
dest: vertica-jdbc-7.1.2-0.jar
be-tests-snowflake:
working_directory: /home/circleci/metabase/metabase/
executor: default
docker:
- image: circleci/clojure:lein-2.8.1-node-browsers
steps:
- attach_workspace:
at: /home/circleci/
- restore_cache:
<<: *restore-be-deps-cache
- run:
name: Run backend unit tests (Snowflake)
environment:
DRIVERS: h2,snowflake
command: >
/home/circleci/metabase/metabase/.circleci/skip-driver-tests.sh snowflake ||
lein with-profile +ci test
no_output_timeout: 10m
yaml-linter:
<<: *defaults
steps:
- attach_workspace:
at: /home/circleci/
- run:
name: Install yamllint
command: npm install yaml-lint
no_output_timeout: 2m
- run:
name: Lint YAML files
command: ./node_modules/.bin/yamllint `find resources -name '*.yaml'`
no_output_timeout: 2m
- test-driver:
driver: snowflake
timeout: 10m
be-tests-sqlserver:
<<: *defaults
executor: default
steps:
- attach_workspace:
at: /home/circleci/
- restore_cache:
<<: *restore-be-deps-cache
- run:
name: Run backend unit tests (SQL Server)
environment:
DRIVERS: h2,sqlserver
command: >
/home/circleci/metabase/metabase/.circleci/skip-driver-tests.sh sqlserver ||
lein with-profile +ci test
no_output_timeout: 5m
- test-driver:
driver: sqlserver
be-tests-bigquery:
<<: *defaults
executor: default
steps:
- attach_workspace:
at: /home/circleci/
- restore_cache:
<<: *restore-be-deps-cache
- run:
name: Run backend unit tests (BigQuery)
environment:
DRIVERS: h2,bigquery
command: >
/home/circleci/metabase/metabase/.circleci/skip-driver-tests.sh bigquery ||
lein with-profile +ci test
no_output_timeout: 5m
- test-driver:
driver: bigquery
be-tests-googleanalytics:
<<: *defaults
executor: default
steps:
- attach_workspace:
at: /home/circleci/
- restore_cache:
<<: *restore-be-deps-cache
- run:
name: Run backend unit tests (Google Analytics)
environment:
DRIVERS: h2,googleanalytics
command: >
/home/circleci/metabase/metabase/.circleci/skip-driver-tests.sh googleanalytics ||
lein with-profile +ci test
no_output_timeout: 5m
- test-driver:
driver: googleanalytics
be-tests-sqlite:
<<: *defaults
executor: default
steps:
- attach_workspace:
at: /home/circleci/
- restore_cache:
<<: *restore-be-deps-cache
- run:
name: Run backend unit tests (SQLite)
environment:
DRIVERS: h2,sqlite
command: >
/home/circleci/metabase/metabase/.circleci/skip-driver-tests.sh sqlite ||
lein with-profile +ci test
- test-driver:
driver: sqlite
be-tests-druid:
<<: *defaults
executor: default
steps:
- attach_workspace:
at: /home/circleci/
- restore_cache:
<<: *restore-be-deps-cache
- run:
name: Run backend unit tests (Druid)
environment:
DRIVERS: h2,druid
command: >
/home/circleci/metabase/metabase/.circleci/skip-driver-tests.sh druid ||
lein with-profile +ci test
no_output_timeout: 5m
- test-driver:
driver: druid
be-tests-redshift:
<<: *defaults
executor: default
steps:
- attach_workspace:
at: /home/circleci/
- restore_cache:
<<: *restore-be-deps-cache
- run:
name: Run backend unit tests (Redshift)
environment:
DRIVERS: h2,redshift
command: >
/home/circleci/metabase/metabase/.circleci/skip-driver-tests.sh redshift ||
lein with-profile +ci test
no_output_timeout: 10m
- test-driver:
driver: redshift
timeout: 10m
be-tests-presto:
working_directory: /home/circleci/metabase/metabase/
executor: default
docker:
- image: circleci/clojure:lein-2.8.1-node-browsers
- image: metabase/presto-mb-ci
environment:
JAVA_TOOL_OPTIONS: "-Xmx2g"
environment:
MB_PRESTO_TEST_HOST: localhost
MB_PRESTO_TEST_PORT: 8080
steps:
- attach_workspace:
at: /home/circleci/
- restore_cache:
<<: *restore-be-deps-cache
- run:
name: Wait for Presto to be ready
command: >
/home/circleci/metabase/metabase/.circleci/skip-driver-tests.sh presto ||
while ! nc -z localhost 8080; do sleep 0.1; done
no_output_timeout: 5m
- run:
name: Run backend unit tests (Presto)
environment:
DRIVERS: h2,presto
MB_PRESTO_TEST_HOST: localhost
MB_PRESTO_TEST_PORT: 8080
command: >
/home/circleci/metabase/metabase/.circleci/skip-driver-tests.sh presto ||
lein with-profile +ci test
no_output_timeout: 5m
- test-driver:
driver: presto
before-steps:
- wait-for-driver:
driver: presto
port: 8080
be-tests-oracle:
<<: *defaults
executor: default
steps:
- attach_workspace:
at: /home/circleci/
- restore_cache:
<<: *restore-be-deps-cache
- run:
name: Make plugins dir
command: mkdir /home/circleci/metabase/metabase/plugins
- test-driver:
driver: oracle
before-steps:
- fetch-jdbc-driver-jar:
driver: oracle
source: ORACLE_JDBC_JAR
dest: ojdbc7.jar
yaml-linter:
executor: default
steps:
- attach-workspace
- run:
name: Download Oracle JAR
command: >
/home/circleci/metabase/metabase/.circleci/skip-driver-tests.sh oracle ||
wget --output-document=/home/circleci/metabase/metabase/plugins/ojdbc7.jar $ORACLE_JDBC_JAR
name: Install yamllint
command: npm install yaml-lint
no_output_timeout: 2m
- run:
name: Run backend unit tests (Oracle)
environment:
DRIVERS: h2,oracle
command: >
/home/circleci/metabase/metabase/.circleci/skip-driver-tests.sh oracle ||
lein with-profile +ci test
no_output_timeout: 5m
name: Lint YAML files
command: ./node_modules/.bin/yamllint `find resources -name '*.yaml'`
no_output_timeout: 2m
be-tests-migrate-to-postgres:
working_directory: /home/circleci/metabase/metabase/
docker:
- image: circleci/clojure:lein-2.8.1-node-browsers
- image: circleci/postgres:9.6-alpine
environment:
POSTGRES_USER: circle_test
POSTGRES_DB: circle_test
executor: postgres-9-6
steps:
- attach_workspace:
at: /home/circleci/
- restore_cache:
<<: *restore-be-deps-cache
- run:
name: Test migrating from H2 -> Postgres
environment:
MB_DB_TYPE: postgres
MB_DB_PORT: 5432
MB_DB_HOST: localhost
MB_DB_DBNAME: circle_test
MB_DB_USER: circle_test
# Test with relative path / no file: prefix
command: >
lein run load-from-h2 ./frontend/test/__runner__/test_db_fixture.db
no_output_timeout: 5m
- test-migrate-from-h2:
db-type: postgres
be-tests-migrate-to-mysql:
working_directory: /home/circleci/metabase/metabase/
docker:
- image: circleci/clojure:lein-2.8.1-node-browsers
- image: circleci/mysql:5.7.23
executor: mysql-5-7
steps:
- attach_workspace:
at: /home/circleci/
- restore_cache:
<<: *restore-be-deps-cache
- run:
name: Test migrating from H2 -> MySQL
environment:
MB_DB_TYPE: mysql
MB_DB_HOST: localhost
MB_DB_PORT: 3306
MB_DB_DBNAME: circle_test
MB_DB_USER: root
# Test including file: prefix & absolute path
command: >
lein run load-from-h2 file:/home/circleci/metabase/metabase/frontend/test/__runner__/test_db_fixture.db
no_output_timeout: 5m
- test-migrate-from-h2:
db-type: mysql
########################################################################################################################
......@@ -514,12 +410,10 @@ jobs:
########################################################################################################################
fe-deps:
<<: *defaults
executor: default
steps:
- attach_workspace:
at: /home/circleci/
- restore_cache:
<<: *restore-fe-deps-cache
- attach-workspace
- restore-fe-deps-cache
- run:
name: Run yarn if yarn.lock checksum has changed
command: >
......@@ -539,84 +433,70 @@ jobs:
- /home/circleci/yarn.lock.checksum
fe-linter-eslint:
<<: *defaults
executor: default
steps:
- attach_workspace:
at: /home/circleci/
- restore_cache:
<<: *restore-fe-deps-cache
- attach-workspace
- restore-fe-deps-cache
- run:
name: Run ESLint linter
command: yarn lint-eslint
no_output_timeout: 5m
fe-linter-prettier:
<<: *defaults
executor: default
steps:
- attach_workspace:
at: /home/circleci/
- restore_cache:
<<: *restore-fe-deps-cache
- attach-workspace
- restore-fe-deps-cache
- run:
name: Run Prettier formatting linter
command: yarn lint-prettier
no_output_timeout: 5m
fe-linter-flow:
<<: *defaults
executor: default
steps:
- attach_workspace:
at: /home/circleci/
- restore_cache:
<<: *restore-fe-deps-cache
- attach-workspace
- restore-fe-deps-cache
- run:
name: Run Flow type checker
command: yarn flow
no_output_timeout: 5m
fe-tests-karma:
<<: *defaults
executor: default
steps:
- attach_workspace:
at: /home/circleci/
- restore_cache:
<<: *restore-fe-deps-cache
- attach-workspace
- restore-fe-deps-cache
- run:
name: Run frontend tests (karma)
command: yarn run test-karma
no_output_timeout: 5m
fe-tests-unit:
<<: *defaults
executor: default
steps:
- attach_workspace:
at: /home/circleci/
- restore_cache:
<<: *restore-fe-deps-cache
- attach-workspace
- restore-fe-deps-cache
- run:
name: Run frontend unit tests
command: yarn run test-unit
no_output_timeout: 5m
fe-tests-integration:
<<: *defaults
executor: default
steps:
- attach_workspace:
at: /home/circleci/
- restore_cache:
<<: *restore-fe-deps-cache
- attach-workspace
- restore-fe-deps-cache
- run:
name: Run frontend integration tests
command: yarn run test-integration
no_output_timeout: 5m
build-uberjar:
<<: *defaults
executor: default
steps:
- attach_workspace:
at: /home/circleci/
- restore_cache:
<<: *restore-be-deps-cache
- attach-workspace
- restore-be-deps-cache
- restore_cache:
keys:
- uberjar-{{ checksum "./backend-checksums.txt" }}
......@@ -633,12 +513,10 @@ jobs:
- /home/circleci/metabase/metabase/target/uberjar/metabase.jar
fe-tests-e2e:
<<: *defaults
executor: default
steps:
- attach_workspace:
at: /home/circleci/
- restore_cache:
<<: *restore-fe-deps-cache
- attach-workspace
- restore-fe-deps-cache
- restore_cache:
keys:
- uberjar-{{ checksum "./backend-checksums.txt" }}
......@@ -656,10 +534,9 @@ jobs:
########################################################################################################################
deploy-master:
<<: *defaults
executor: default
steps:
- attach_workspace:
at: /home/circleci/
- attach-workspace
- run: ./bin/deploy-webhook $DEPLOY_WEBHOOK
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment