config.yml

version: 2.1

########################################################################################################################
#                                                      EXECUTORS                                                       #
########################################################################################################################

executors:
  # CircleCI base Node + Headless browsers + Clojure CLI - big one
  # Maildev runs by default with all Cypress tests
  clojure-and-node-and-browsers:
    working_directory: /home/circleci/metabase/metabase/
    docker:
      - image: metabase/ci:circleci-java-11-clj-1.10.3.929-07-27-2021-node-browsers
      - image: maildev/maildev
      - image: metabase/qa-databases:postgres-sample-12
      - image: metabase/qa-databases:mongo-sample-4.0
      - image: metabase/qa-databases:mysql-sample-8

  java-8:
    working_directory: /home/circleci/metabase/metabase/
    docker:
      - image: metabase/ci:circleci-java-8-clj-1.10.3.929-07-27-2021-node-browsers

  # Java 11 tests also test Metabase with the at-rest encryption enabled. See
  # https://metabase.com/docs/latest/operations-guide/encrypting-database-details-at-rest.html for an explanation of
  # what this means.
  java-11:
    working_directory: /home/circleci/metabase/metabase/
    docker:
      - image: metabase/ci:circleci-java-11-clj-1.10.3.929-07-27-2021-node-browsers
        environment:
          MB_ENCRYPTION_SECRET_KEY: Orw0AAyzkO/kPTLJRxiyKoBHXa/d6ZcO+p+gpZO/wSQ=

  java-16:
    working_directory: /home/circleci/metabase/metabase/
    docker:
      - image: metabase/ci:circleci-java-16-clj-1.10.3.929-07-27-2021-node-browsers

  postgres-9-6:
    working_directory: /home/circleci/metabase/metabase/
    docker:
      - image: metabase/ci:circleci-java-11-clj-1.10.3.929-07-27-2021-node-browsers
        environment:
          MB_DB_TYPE: postgres
          MB_DB_PORT: 5432
          MB_DB_HOST: localhost
          MB_DB_DBNAME: circle_test
          MB_DB_USER: circle_test
          MB_POSTGRESQL_TEST_USER: circle_test
      - image: circleci/postgres:9.6-alpine
        environment:
          POSTGRES_USER: circle_test
          POSTGRES_DB: circle_test

  postgres-latest:
    working_directory: /home/circleci/metabase/metabase/
    docker:
      - image: metabase/ci:circleci-java-11-clj-1.10.3.929-07-27-2021-node-browsers
        environment:
          MB_DB_TYPE: postgres
          MB_DB_PORT: 5432
          MB_DB_HOST: localhost
          MB_DB_DBNAME: metabase_test
          MB_DB_USER: metabase_test
          MB_POSTGRESQL_TEST_USER: metabase_test
      - image: circleci/postgres:latest
        environment:
          POSTGRES_USER: metabase_test
          POSTGRES_DB: metabase_test
          POSTGRES_HOST_AUTH_METHOD: trust

  mysql-5-7:
    working_directory: /home/circleci/metabase/metabase/
    docker:
      - image: metabase/ci:circleci-java-11-clj-1.10.3.929-07-27-2021-node-browsers
        environment:
          MB_DB_TYPE: mysql
          MB_DB_HOST: localhost
          MB_DB_PORT: 3306
          MB_DB_DBNAME: circle_test
          MB_DB_USER: root
          MB_MYSQL_TEST_USER: root
      - image: circleci/mysql:5.7.23

  mysql-latest:
    working_directory: /home/circleci/metabase/metabase/
    docker:
      - image: metabase/ci:circleci-java-11-clj-1.10.3.929-07-27-2021-node-browsers
        environment:
          MB_DB_TYPE: mysql
          MB_DB_HOST: localhost
          MB_DB_PORT: 3306
          MB_DB_DBNAME: circle_test
          MB_DB_USER: root
          MB_MYSQL_TEST_USER: root
      - image: circleci/mysql:latest

  mariadb-10-2:
    working_directory: /home/circleci/metabase/metabase/
    docker:
      - image: metabase/ci:circleci-java-11-clj-1.10.3.929-07-27-2021-node-browsers
        environment:
          MB_DB_TYPE: mysql
          MB_DB_HOST: localhost
          MB_DB_PORT: 3306
          MB_DB_DBNAME: circle_test
          MB_DB_USER: root
          MB_MYSQL_TEST_USER: root
      - image: circleci/mariadb:10.2.23

  mariadb-latest:
    working_directory: /home/circleci/metabase/metabase/
    docker:
      - image: metabase/ci:circleci-java-11-clj-1.10.3.929-07-27-2021-node-browsers
        environment:
          MB_DB_TYPE: mysql
          MB_DB_HOST: localhost
          MB_DB_PORT: 3306
          MB_DB_DBNAME: circle_test
          MB_DB_USER: root
          MB_MYSQL_TEST_USER: root
      - image: circleci/mariadb:latest
        environment:
          # MYSQL_DATABASE: metabase_test
          # MYSQL_USER: root
          # MYSQL_ALLOW_EMPTY_PASSWORD: yes

  mongo-4-0:
     working_directory: /home/circleci/metabase/metabase/
     docker:
       - image: metabase/ci:circleci-java-11-clj-1.10.3.929-07-27-2021-node-browsers
       - image: circleci/mongo:4.0

  mongo-latest:
     working_directory: /home/circleci/metabase/metabase/
     docker:
       - image: metabase/ci:circleci-java-11-clj-1.10.3.929-07-27-2021-node-browsers
       - image: circleci/mongo:latest

  presto-186:
    working_directory: /home/circleci/metabase/metabase/
    docker:
      - image: metabase/ci:circleci-java-11-clj-1.10.3.929-07-27-2021-node-browsers
      - image: metabase/presto-mb-ci:0.186
        environment:
          JAVA_TOOL_OPTIONS: "-Xmx2g"
    # Run instance with 8GB or RAM instead of the default 4GB for medium instances. The Presto Docker image runs
    # OOM sometimes with the default medium size.
    resource_class: large

  presto-jdbc-env:
    working_directory: /home/circleci/metabase/metabase/
    docker:
      - image: metabase/ci:circleci-java-11-clj-1.10.3.929-07-27-2021-node-browsers
      - image: metabase/presto-mb-ci:latest # version 0.254
        environment:
          JAVA_TOOL_OPTIONS: "-Xmx2g"
          MB_PRESTO_JDBC_TEST_CATALOG: test_data
          MB_PRESTO_JDBC_TEST_HOST: localhost
          MB_PRESTO_JDBC_TEST_PORT: 8443
          MB_PRESTO_JDBC_TEST_SSL: true
          MB_PRESTO_JDBC_TEST_USER: metabase
          MB_PRESTO_JDBC_TEST_PASSWORD: metabase
          MB_ENABLE_PRESTO_JDBC_DRIVER: true
          MB_PRESTO_JDBC_TEST_ADDITIONAL_OPTIONS: >
            SSLTrustStorePath=/tmp/cacerts-with-presto-ssl.jks&SSLTrustStorePassword=changeit
    # (see above)
    resource_class: large

  sparksql:
    working_directory: /home/circleci/metabase/metabase/
    docker:
      - image: metabase/ci:circleci-java-11-clj-1.10.3.929-07-27-2021-node-browsers
      - image: metabase/spark:2.1.1

  vertica:
    working_directory: /home/circleci/metabase/metabase/
    docker:
      - image: metabase/ci:circleci-java-11-clj-1.10.3.929-07-27-2021-node-browsers
      - image: sumitchawla/vertica

  sqlserver:
    working_directory: /home/circleci/metabase/metabase/
    docker:
      - image: metabase/ci:circleci-java-11-clj-1.10.3.929-07-27-2021-node-browsers
        environment:
          MB_SQLSERVER_TEST_HOST: localhost
          MB_SQLSERVER_TEST_PASSWORD: 'P@ssw0rd'
          MB_SQLSERVER_TEST_USER: SA
      - image: mcr.microsoft.com/mssql/server:2017-latest
        environment:
          ACCEPT_EULA: Y
          SA_PASSWORD: 'P@ssw0rd'
          MSSQL_MEMORY_LIMIT_MB: 1024

  druid:
    working_directory: /home/circleci/metabase/metabase/
    docker:
      - image: metabase/ci:circleci-java-11-clj-1.10.3.929-07-27-2021-node-browsers
      - image: metabase/druid:0.20.2
        environment:
          CLUSTER_SIZE: nano-quickstart
    # Run Docker images with 8GB or RAM instead of the default 4GB for medium instances. The Druid Docker image runs
    # OOM all the time with the default medium size.
    resource_class: large

########################################################################################################################
#                                             MAP FRAGMENTS AND CACHE KEYS                                             #
########################################################################################################################

# `default_parameters` isn't a key that CircleCI uses, but this form lets us reuse parameter definitions
default_parameters: &Params
  edition:
    type: enum
    enum: ["oss", "ee"]
    default: "oss"

# .BACKEND-CHECKSUMS, .FRONTEND-CHECKSUMS, and .MODULE-CHECKSUMS are created during the checkout step; see that step
# for exact details as to what they contain.
#
# To support cache busting, we create a file named .CACHE-PREFIX in the checkout step and use its checksum as the
# prefix for every cache key. If the commit message DOES NOT include [ci nocache], we create an empty file; the
# checksum will always be the same for this file. If the commit message DOES include [ci nocache], we'll write the
# unique ID of the current pipeline to .CACHE-PREFIX which will effectively bust our caches whenever it's used.

### Deps Keys ###

# Why don't we use fallback keys for backend/frontend deps? We used to, but it allowed the cache to grow
# uncontrollably since old deps would continue to accumulate. Restoring big caches is really slow in Circle. It's
# actually faster to recreate the deps cache from scratch whenever we need to which keeps the size down.
cache-key-backend-deps: &CacheKeyBackendDeps
  # TODO -- this should actually include the Java source files and the Spark SQL AOT source files as well since we now
  # compile those as part of this step. FIXME
  key: v5-{{ checksum ".CACHE-PREFIX" }}-be-deps-{{ checksum "deps.edn" }}-{{ checksum ".SCRIPTS-DEPS-CHECKSUMS" }}

cache-key-frontend-deps: &CacheKeyFrontendDeps
  key: v5-{{ checksum ".CACHE-PREFIX" }}-fe-deps-{{ checksum "yarn.lock" }}

# Key used for implementation of run-on-change -- this is the cache key that contains the .SUCCESS dummy file
# By default the key ALWAYS includes the name of the test job itself ($CIRCLE_STAGE) so you don't need to add that yourself.
cache-key-run-on-change: &CacheKeyRunOnChange
  key: v5-{{ checksum ".CACHE-PREFIX" }}-run-on-change-{{ .Environment.CIRCLE_STAGE }}-<< parameters.checksum >>

# Key for the local maven installation of metabase-core (used by build-uberjar-drivers)
cache-key-metabase-core: &CacheKeyMetabaseCore
  key: v5-{{ checksum ".CACHE-PREFIX" }}-metabase-core-{{ checksum ".BACKEND-CHECKSUMS" }}

# Key for the drivers built by build-uberjar-drivers
cache-key-drivers: &CacheKeyDrivers
  key: v5-{{ checksum ".CACHE-PREFIX" }}-drivers-<< parameters.edition >>-{{ checksum ".MODULES-CHECKSUMS" }}-{{ checksum ".BACKEND-CHECKSUMS" }}-<< parameters.edition >>

# This is also used by the uberjar-build-drivers step; this is a unique situation because the build-drivers script has
# logic to determine whether to rebuild drivers or not that is quite a bit more sophisticated that the run-on-change
# stuff in this file. e.g. if I only change the bigquery driver, the script is smart enough to not rebuild the
# redshift driver.
cache-keys-drivers-with-fallback-keys: &CacheKeyDrivers_WithFallbackKeys
  keys:
    - v5-{{ checksum ".CACHE-PREFIX" }}-drivers-<< parameters.edition >>-{{ checksum ".MODULES-CHECKSUMS" }}-{{ checksum ".BACKEND-CHECKSUMS" }}
    - v5-{{ checksum ".CACHE-PREFIX" }}-drivers-<< parameters.edition >>-{{ checksum ".MODULES-CHECKSUMS" }}
    - v5-{{ checksum ".CACHE-PREFIX" }}-drivers-<< parameters.edition >>-

# Key for frontend client built by uberjar-build-frontend step
cache-key-frontend: &CacheKeyFrontend
  key: v5-{{ checksum ".CACHE-PREFIX" }}-frontend-<< parameters.edition >>-{{ checksum ".FRONTEND-CHECKSUMS" }}

# Key for uberjar built by build-uberjar
cache-key-uberjar: &CacheKeyUberjar
  key: v5-{{ checksum ".CACHE-PREFIX" }}-uberjar-<< parameters.edition >>-{{ checksum ".BACKEND-CHECKSUMS" }}-{{ checksum ".FRONTEND-CHECKSUMS" }}

cache-key-snowplow-deps: &CacheKeySnowplowDeps
  key: v5-{{ checksum ".CACHE-PREFIX" }}-snowplow-deps


########################################################################################################################
#                                                       COMMANDS                                                       #
########################################################################################################################

commands:
  attach-workspace:
    steps:
      - attach_workspace:
          at: /home/circleci/

  # For the restore-deps-cache commands below, only restore the cache if there's an exact match. This means whatever
  # is in the cache will be exactly what's used and the cache won't keep growing uncontrollably going forward.

  restore-be-deps-cache:
    steps:
      - restore_cache:
          name: Restore cached backend dependencies
          <<: *CacheKeyBackendDeps

  restore-fe-deps-cache:
    steps:
      - restore_cache:
          name: Restore cached frontend dependencies
          <<: *CacheKeyFrontendDeps

  restore-snowplow-deps-cache:
    steps:
      - restore_cache:
          name: Restore Snowplow Micro JAR
          <<: *CacheKeySnowplowDeps

  # run-on-change lets you only run steps if changes have happened to relevant files since the last time it was run
  # successfully. Uses a cache key to record successful runs -- cache key should be unique for job and relevant source
  # files -- use a checksum! It works like this:
  #
  # 1. Calculate a cache key using a checksum of relevant files for the step in question, e.g. a backend linter step
  #    might use a checksum of all .clj files.
  #
  # 2. When the step completes successfully, create a dummy file .SUCCESS and cache it with that cache key.
  #
  # 3. On subsequent runs:
  #
  #    a. Attempt to restore the cache using an exact match for this cache key
  #
  #    b. If we have a cache entry for that key, .SUCCESS will get restored
  #
  #    c. If this command has the skip-job-if-commit-message-includes-ci-quick option enabled, and commit message includes
  #       [ci quick], create a dummy file .SUCCESS if not already present. Ignored for master/release branches.
  #
  #    d. If commit message includes [ci noskip], delete .SUCCESS so the job will be forced to run.
  #
  #    e. If .SUCCESS is present, we can skip the rest of the job, including potentially slow steps like restoring
  #       dependency caches or the like. This logs a link to the last successful (not skipped) run of the job
  #
  #       Important! If this step is skipped because no changes have happened, the entire JOB will halt with a success
  #       status -- no steps that happen AFTER run-on-change will be ran. Keep this in mind!
  #
  #   f. If .SUCCESS is not present, proceed as normal, and create and cache .SUCCESS if the job succeeds
  run-on-change:
    parameters:
      checksum:
        type: string
        default: ""
      steps:
        type: steps
      # Whether to skip the rest of the job if commit message includes [ci quick]
      skip-job-if-commit-message-includes-ci-quick:
        type: boolean
        default: false
    steps:
      - restore_cache:
          name: Restore dummy file .SUCCESS if it exists for cache key << parameters.checksum >>
          <<: *CacheKeyRunOnChange
      - when:
          condition: << parameters.skip-job-if-commit-message-includes-ci-quick >>
          steps:
            - run:
                name: "Skip tests (create dummy file .SUCCESS) if commit message contains [ci quick] and branch isn't a master/release branch"
                command: |
                  if [[ "$CIRCLE_BRANCH" =~ ^master|release-.+$ ]]; then
                      echo "branch '$CIRCLE_BRANCH' is a master or release branch: ignoring [ci quick]"
                  elif [[ `cat .COMMIT` == *"[ci quick]"* ]]; then
                      echo 'Commit message includes [ci quick]. Creating dummy file .SUCCESS'
                      touch .SUCCESS
                  else
                      echo 'Commit message does not include [ci quick]'
                  fi
      - run:
          name: "Force test run (delete dummy file .SUCCESS) if commit message includes [ci noskip]"
          command: |
            if [[ `cat .COMMIT` == *"[ci noskip]"* ]]; then
                echo 'Commit message includes [ci noskip] -- forcing test run (delete .SUCCESS)'
                rm -f .SUCCESS
            else
                echo 'Commit message does not include [ci noskip]'
            fi
      - run:
          name: Skip rest of job if .SUCCESS exists
          command: |
            if [ -f .SUCCESS ]; then
                echo '.SUCCESS is present: skipping rest of job.'
                echo "Link to last successful run (if available): $(cat .SUCCESS)"
                circleci-agent step halt
            fi
      - steps: << parameters.steps >>
      - run:
          name: Create dummy file .SUCCESS
          command: |
            echo "$CIRCLE_BUILD_URL" > .SUCCESS
      - save_cache:
          name: Persist dummy file .SUCCESS to cache with key << parameters.checksum >>
          <<: *CacheKeyRunOnChange
          paths:
            - /home/circleci/metabase/metabase/.SUCCESS
      - run:
          name: Delete dummy file .SUCCESS so subsequent steps don't see it
          command: rm /home/circleci/metabase/metabase/.SUCCESS

  # Creates a file that contains checksums for all the files found using the find command with supplied arguments.
  # You can use a checksum of the checksum file for cache keys including run-on-change cache keys.
  create-checksum-file:
    parameters:
      filename:
        type: string
      find-args:
        type: string
    steps:
      - run:
          name: Create << parameters.filename >> checksum file
          command: |
            for file in `find << parameters.find-args >> | sort`; do
                echo `md5sum "$file"` >> "<< parameters.filename >>"
            done
            if [ ! -f "<< parameters.filename >>" ]; then
                echo 'Error: no matching files. Did you remember to attach the workspace?'
                exit 1
            fi
            echo "Created checksums for $(cat << parameters.filename >> | wc -l) files"

  run-clojure-command:
    parameters:
      before-steps:
        type: steps
        default: []
      clojure-args:
        type: string
      after-steps:
        type: steps
        default: []
      <<: *Params
    steps:
      - restore-be-deps-cache
      - steps: << parameters.before-steps >>
      - run:
          name: clojure << parameters.clojure-args >>:<< parameters.edition >>:<< parameters.edition >>-dev
          command: |
            clojure << parameters.clojure-args >>:<< parameters.edition >>:<< parameters.edition >>-dev
          no_output_timeout: 15m
      - steps: << parameters.after-steps >>
      - store_test_results:
          path: /home/circleci/metabase/metabase/target/junit

  run-yarn-command:
    parameters:
      command-name:
        type: string
      command:
        type: string
      before-steps:
        type: steps
        default: []
      after-steps:
        type: steps
        default: []
      skip-when-no-change:
        type: boolean
        default: false
    steps:
      - attach-workspace
      - when:
          condition: << parameters.skip-when-no-change >>
          steps:
            - run-on-change:
                checksum: '{{ checksum ".FRONTEND-CHECKSUMS" }}'
                steps:
                  - restore-fe-deps-cache
                  - steps: << parameters.before-steps >>
                  - run:
                      name: << parameters.command-name >>
                      command: yarn << parameters.command >>
                      no_output_timeout: 15m
                  - steps: << parameters.after-steps >>
      - unless:
          condition: << parameters.skip-when-no-change >>
          steps:
            - restore-fe-deps-cache
            - steps: << parameters.before-steps >>
            - run:
                name: << parameters.command-name >>
                command: yarn << parameters.command >>
                no_output_timeout: 15m
            - steps: << parameters.after-steps >>

  wait-for-port:
    parameters:
      port:
        type: integer
    steps:
      - run:
          name: Wait for port << parameters.port >> to be ready
          command: |
            while ! nc -z localhost << parameters.port >>; do sleep 0.1; done
          no_output_timeout: 15m

  wait-for-databases:
    steps:
      - wait-for-port:
          port: 3306 # mysql
      - wait-for-port:
          port: 5432 # postgres
      - wait-for-port:
          port: 27017 # mongo

  fetch-jdbc-driver:
    parameters:
      source:
        type: string
      dest:
        type: string
    steps:
      - run:
          name: Make plugins dir
          command: mkdir /home/circleci/metabase/metabase/plugins
      - run:
          name: Download JDBC driver JAR << parameters.dest >>
          command: |
            wget --output-document=plugins/<< parameters.dest >> ${<< parameters.source >>}
          no_output_timeout: 15m

  run-command:
    parameters:
      command:
        type: string
    steps:
      - run:
          name: Run command
          command: << parameters.command >>

  run-snowplow-micro:
    steps:
      - restore-snowplow-deps-cache
      - run:
          name: Run Snowplow Micro
          command: |
            java -cp snowplow-micro.jar:snowplow com.snowplowanalytics.snowplow.micro.Main --collector-config snowplow/micro.conf --iglu snowplow/iglu.json
          background: true
      - wait-for-port:
          port: 9090

jobs:

########################################################################################################################
#                                                    CHECKOUT ETC.                                                     #
########################################################################################################################

  checkout:
    executor: clojure-and-node-and-browsers
    steps:
      - checkout
      # .BACKEND-CHECKSUMS is every Clojure source file as well as dependency files like deps.edn and plugin manifests
      - create-checksum-file:
          filename: .BACKEND-CHECKSUMS
          find-args: ". -type f -name '*.clj' -or -name '*.cljc' -or -name '*.java' -or -name '*.edn' -or -name '*.yaml' -or -name sample-dataset.db.mv.db"
      # .SCRIPTS-DEPS-CHECKSUMS is all the deps.edn files inside ./bin
      - create-checksum-file:
          filename: .SCRIPTS-DEPS-CHECKSUMS
          find-args: "bin -type f -name 'deps.edn'"
      # .FRONTEND-CHECKSUMS is every JavaScript source file as well as dependency files like yarn.lock (sans all frontend test files)
      - create-checksum-file:
          filename: .FRONTEND-CHECKSUMS
          find-args: ". -type f '(' -name '*.js' -or -name '*.jsx' -or -name '*.cljc' -or -name '*.cljs' -or -name '*.json' -or -name yarn.lock -or -name sample-dataset.db.mv.db ')' ! -path '*/frontend/test/*'"
      # .E2E-TESTS-CHECKSUMS is every `*.cy.spes.js` file as well as e2e support JavaScript files
      - create-checksum-file:
          filename: .E2E-TESTS-CHECKSUMS
          find-args: "./frontend/test/ -name '*.cy.*' -or -type f -path '*/__support__/e2e/*'"
      # .MODULES-CHECKSUMS is every Clojure source file in the modules/ directory as well as plugin manifests
      - create-checksum-file:
          filename: .MODULES-CHECKSUMS
          find-args: "./modules -type f -name '*.clj' -or -name metabase-plugin.yaml"
      - run:
          name: Save last git commit message to .COMMIT
          command: git log -1 > .COMMIT
      - run:
          name: Determine what to do to .git directory
          command: |
            if [[ $CIRCLE_BRANCH == release* ]]; then
              echo 'This is a release branch; preserving .git directory to determine version'
            else
              echo 'This is not a release branch; removing .git directory (not needed for tests)'
              rm -rf /home/circleci/metabase/metabase/.git
            fi

      - run:
          name: Remove ./OSX directory (not needed for tests)
          command: rm -rf /home/circleci/metabase/metabase/OSX
      # .CACHE-PREFIX is described above in the Cache Keys section of this file
      - run:
          name: 'Create cache key prefix .CACHE-PREFIX to bust caches if commit message includes [ci nocache]'
          command: |
            if [[ `cat .COMMIT` == *"[ci nocache]"* ]]; then
                echo 'Commit message includes [ci nocache]; using cache-busting prefix'
                echo '<< pipeline.id >>' > .CACHE-PREFIX
            else
               echo '' > .CACHE-PREFIX
            fi
      - run:
          name: 'Check for branch name to bust caches if it is a release branch'
          command: |
            if [[ $CIRCLE_BRANCH == release* || $CIRCLE_BRANCH == master  ]]; then
              echo 'This is a release or master branch; using cache-busting prefix'
                echo '<< pipeline.id >>' > .CACHE-PREFIX
            fi
      - run-yarn-command:
          command-name: Create static visualization js bundle
          command: build-static-viz
      - persist_to_workspace:
          root: /home/circleci/
          paths:
            - metabase/metabase

  check-migrations:
    executor:
      clojure-and-node-and-browsers
    steps:
      - attach-workspace
      - create-checksum-file:
          filename: .MIGRATIONS-CHECKSUM
          find-args: resources/migrations -type f -name '*.yaml'
      - create-checksum-file:
          filename: .MIGRATIONS-LINTER-CHECKSUMS
          find-args: bin/lint-migrations-file -type f -name '*.clj' -or -name 'deps.edn'
      - run-on-change:
          checksum: '{{ checksum ".MIGRATIONS-CHECKSUM" }}-{{ checksum ".MIGRATIONS-LINTER-CHECKSUMS" }}'
          steps:
            - run:
                name: Verify Liquibase Migrations
                command: ./bin/lint-migrations-file.sh
                no_output_timeout: 15m

########################################################################################################################
#                                                       BACKEND                                                        #
########################################################################################################################

  be-deps:
    executor: clojure-and-node-and-browsers
    parameters:
      <<: *Params
    steps:
      - attach-workspace
      # This step is pretty slow, even with the cache, so only run it if deps.edn has changed
      - run-on-change:
          checksum: 'v5-{{ checksum "deps.edn" }}-{{ checksum ".SCRIPTS-DEPS-CHECKSUMS" }}'
          steps:
            - restore-be-deps-cache
            - run:
                name: Compile Java source file(s)
                command: clojure -X:deps prep
            - run:
                name: Compile driver AOT namespaces
                command: cd modules/drivers && clojure -X:deps prep
            - run:
                name: Fetch dependencies
                command: clojure -P -X:dev:ci:ee:ee-dev:drivers:drivers-dev
            - run:
                name: Fetch dependencies (./bin/build/build-mb)
                command: cd /home/circleci/metabase/metabase/bin/build-mb && clojure -P -M:test
            # Not sure why this is needed since you would think build-mb would fetch this stuff as well. It doesn't
            # seem to fetch everything tho. :shrug:
            - run:
                name: Fetch dependencies (./bin/build/build-drivers)
                command: cd /home/circleci/metabase/metabase/bin/build-drivers && clojure -P -M:test
            - save_cache:
                name: Cache backend dependencies
                <<: *CacheKeyBackendDeps
                paths:
                  - /home/circleci/.m2
                  - /home/circleci/.gitlibs
                  - /home/circleci/metabase/metabase/java/target/classes
                  - /home/circleci/metabase/metabase/modules/drivers/sparksql/target/classes

  clojure:
    parameters:
      e:
        type: executor
        default: clojure-and-node-and-browsers
      before-steps:
        type: steps
        default: []
      clojure-args:
        type: string
      after-steps:
        type: steps
        default: []
      skip-when-no-change:
        type: boolean
        default: false
      <<: *Params
    executor: << parameters.e >>
    steps:
      - attach-workspace
      - when:
          condition: << parameters.skip-when-no-change >>
          steps:
            - run-on-change:
                checksum: '{{ checksum ".BACKEND-CHECKSUMS" }}'
                steps:
                  - run-clojure-command:
                      before-steps: << parameters.before-steps >>
                      clojure-args: << parameters.clojure-args >>
                      after-steps: << parameters.after-steps >>
                      edition: << parameters.edition >>
      - unless:
          condition: << parameters.skip-when-no-change >>
          steps:
            - run-clojure-command:
                before-steps: << parameters.before-steps >>
                clojure-args: << parameters.clojure-args >>
                after-steps: << parameters.after-steps >>
                edition: << parameters.edition >>

  be-linter-reflection-warnings:
    executor: clojure-and-node-and-browsers
    steps:
      - attach-workspace
      - run-on-change:
          checksum: '{{ checksum ".BACKEND-CHECKSUMS" }}-{{ checksum "bin/reflection-linter" }}'
          steps:
            - restore-be-deps-cache
            - run:
                name: Run reflection warnings checker
                command: ./bin/reflection-linter
                no_output_timeout: 15m

  test-driver:
    parameters:
      e:
        type: executor
        default: clojure-and-node-and-browsers
      driver:
        type: string
      timeout:
        type: string
        default: 15m
      before-steps:
        type: steps
        default: []
      after-steps:
        type: steps
        default: []
      description:
        type: string
        default: ""
      extra-env:
        type: string
        default: ""
      test-args:
        type: string
        default: ""
    executor: << parameters.e >>
    steps:
      - attach-workspace
      - run-on-change:
          checksum: '{{ checksum ".BACKEND-CHECKSUMS" }}'
          skip-job-if-commit-message-includes-ci-quick: true
          steps:
            - restore-be-deps-cache
            - steps: << parameters.before-steps >>
            - run:
                name: Test << parameters.driver >> driver << parameters.description >>
                environment:
                  DRIVERS: << parameters.driver >>
                command: >
                  << parameters.extra-env >> clojure -X:dev:ci:ee:ee-dev:drivers:drivers-dev:test
                  << parameters.test-args >>
                no_output_timeout: << parameters.timeout >>
            - store_test_results:
                path: /home/circleci/metabase/metabase/target/junit
            - steps: << parameters.after-steps >>

  test-build-scripts:
    executor: clojure-and-node-and-browsers
    steps:
      - attach-workspace
      - run-on-change:
          checksum: '{{ checksum ".BACKEND-CHECKSUMS" }}'
          steps:
            - restore-be-deps-cache
            - run:
                name: Run metabuild-common build script tests
                command: |
                  cd /home/circleci/metabase/metabase/bin/common && clojure -M:test
                no_output_timeout: 15m
            - run:
                name: Run build-drivers build script tests
                command: |
                  cd /home/circleci/metabase/metabase/bin/build-drivers && clojure -M:test
                no_output_timeout: 15m
            - run:
                name: Run i18n script tests
                command: |
                  cd /home/circleci/metabase/metabase/bin/i18n && clojure -M:test
                no_output_timeout: 15m
            - run:
                name: Run build-mb build script tests
                command: |
                  cd /home/circleci/metabase/metabase/bin/build-mb && clojure -M:test
                no_output_timeout: 15m
            - run:
                name: Run release script tests
                command: |
                  cd /home/circleci/metabase/metabase/bin/release && clojure -M:test
                no_output_timeout: 15m
            - run:
                name: Run Liquibase migrations linter tests
                command: |
                  cd /home/circleci/metabase/metabase/bin/lint-migrations-file && clojure -M:test
                no_output_timeout: 15m


########################################################################################################################
#                                                       FRONTEND                                                       #
########################################################################################################################

  fe-deps:
    executor: clojure-and-node-and-browsers
    steps:
      - attach-workspace
      # This step is *really* slow, so we can skip it if yarn.lock hasn't changed since last time we ran it
      - run-on-change:
          checksum: '{{ checksum "yarn.lock" }}'
          steps:
            - restore-fe-deps-cache
            - run:
                name: Run yarn to install deps
                command: rm -rf node_modules/ && yarn --frozen-lockfile;
                no_output_timeout: 15m
            - save_cache:
                name: Cache frontend dependencies
                <<: *CacheKeyFrontendDeps
                paths:
                  - /home/circleci/.yarn
                  - /home/circleci/.yarn-cache
                  - /home/circleci/metabase/metabase/node_modules
                  - /home/circleci/.cache/Cypress

  shared-tests-cljs:
    executor: clojure-and-node-and-browsers
    steps:
      - run-yarn-command:
          command-name: Run Cljs tests for shared/ code
          command: run test-cljs
          skip-when-no-change: true

  # Unlike the other build-uberjar steps, this step should be run once overall and the results can be shared between
  # OSS and EE uberjars.
  build-uberjar-drivers:
    executor: clojure-and-node-and-browsers
    parameters:
      <<: *Params
    steps:
      - attach-workspace
      - run-on-change:
          # .MODULES-CHECKSUMS is a subset of .BACKEND-CHECKSUMS.
          #
          # We have both versions so we can try to load cached drivers that match MODULES-CHECKSUMS but not
          # BACKEND-CHECKSUMS as a whole.
          #
          # The build-drivers script is smart enough to only rebuild drivers if needed -- there's a chance we won't
          # need to rebuild them.
          checksum: '{{ checksum ".MODULES-CHECKSUMS" }}-{{ checksum ".BACKEND-CHECKSUMS" }}'
          steps:
            - restore-be-deps-cache            #
            - restore_cache:
                <<: *CacheKeyMetabaseCore
            - restore_cache:
                name: Restore cached drivers uberjars from previous runs
                <<: *CacheKeyDrivers_WithFallbackKeys
            - run:
                name: Build << parameters.edition >> drivers if needed
                command: ./bin/build-drivers.sh << parameters.edition >>
                no_output_timeout: 15m
            - save_cache:
                name: Cache local Maven installation of metabase-core
                <<: *CacheKeyMetabaseCore
                paths:
                  - /home/circleci/.m2/repository/metabase-core
            - save_cache:
                name: Cache the built drivers
                <<: *CacheKeyDrivers
                paths:
                  - /home/circleci/metabase/metabase/resources/modules

  # Build the frontend client. parameters.edition determines whether we build the OSS or EE version.
  build-uberjar-frontend:
    parameters:
      <<: *Params
    executor: clojure-and-node-and-browsers
    resource_class: large
    steps:
      - attach-workspace
      - run-on-change:
          checksum: '{{ checksum ".FRONTEND-CHECKSUMS" }}'
          steps:
            - restore-fe-deps-cache
            - restore-be-deps-cache
            - run:
                name: Build frontend
                environment:
                  MB_EDITION: << parameters.edition >>
                command: ./bin/build version frontend
                no_output_timeout: 15m
            - save_cache:
                name: Cache the built frontend
                <<: *CacheKeyFrontend
                paths:
                  - /home/circleci/metabase/metabase/resources/frontend_client

  # Build the uberjar. parmeters.edition determines whether we build the OSS or EE version.
  build-uberjar:
    parameters:
      <<: *Params
    executor: clojure-and-node-and-browsers
    resource_class: large
    steps:
      - attach-workspace
      - run-on-change:
          checksum: '{{ checksum ".BACKEND-CHECKSUMS" }}-{{ checksum ".FRONTEND-CHECKSUMS" }}'
          steps:
            - restore_cache:
                name: Restore cached uberjar from previous runs
                <<: *CacheKeyUberjar
            - run:
                name: Skip rest of job if uberjar already exists
                command: |
                   if [ -f './target/uberjar/metabase.jar' ]; then
                       circleci-agent step halt
                   fi
            - restore-be-deps-cache
            - restore_cache:
                name: Restore cached drivers built by previous step
                <<: *CacheKeyDrivers
            - restore_cache:
                name: Restore cached FE built by previous step
                <<: *CacheKeyFrontend
            - run:
                name: Build uberjar
                environment:
                  # INTERACTIVE=false will tell the clojure build scripts not to do interactive retries etc.
                  INTERACTIVE: "false"
                  MB_EDITION: << parameters.edition >>
                command: |
                  if [[ $CIRCLE_BRANCH == release* || $CIRCLE_BRANCH == master  ]]; then
                    echo 'This is a release or master branch; building a complete Uberjar'
                    ./bin/build
                  else
                    ./bin/build version uberjar
                  fi
                no_output_timeout: 15m
            - store_artifacts:
                path: /home/circleci/metabase/metabase/target/uberjar/metabase.jar
            - store_artifacts:
                path: /home/circleci/metabase/metabase/resources/version.properties
            - save_cache:
                name: Cache the built uberjar & version.properties
                <<: *CacheKeyUberjar
                paths:
                  - /home/circleci/metabase/metabase/target/uberjar/metabase.jar
                  - /home/circleci/metabase/metabase/resources/version.properties

  fe-tests-cypress:
    parameters:
      e:
        type: executor
        default: clojure-and-node-and-browsers
      cypress-group:
        type: string
      source-folder:
        type: string
        default: ""
      folder:
        type: string
        default: ""
      test-files:
        type: string
        default: ""
      qa-db:
        type: boolean
        default: false
      snowplow:
        type: boolean
        default: false
      before-steps:
        type: steps
        default: []
      <<: *Params
    executor: << parameters.e >>
    environment:
      MB_EDITION: << parameters.edition >>
      CYPRESS_GROUP:  << parameters.cypress-group >>
      QA_DB_ENABLED: << parameters.qa-db >>
      DISPLAY: ""
      MB_SNOWPLOW_AVAILABLE: << parameters.snowplow >>
    steps:
      - attach-workspace
      - run-on-change:
          checksum: '{{ checksum ".BACKEND-CHECKSUMS" }}-{{ checksum ".FRONTEND-CHECKSUMS" }}-{{ checksum ".E2E-TESTS-CHECKSUMS" }}'
          steps:
            - run-yarn-command:
                command-name: Run Cypress tests
                before-steps:
                  - restore_cache:
                      name: Restore cached uberjar built in previous step
                      <<: *CacheKeyUberjar
                  - steps: << parameters.before-steps >>
                # Make both `test-files` and `source-folder` parameters optional. Translates to: if `parameter` => run associated flag (`--spec` and `--folder`, respectively)
                command: |
                  run test-cypress-no-build <<# parameters.test-files >> --spec << parameters.test-files >> <</ parameters.test-files >> <<# parameters.source-folder >> --folder << parameters.source-folder >> <</ parameters.source-folder >>