Migrate some driver tests to GitHub Actions (#25769)

* Migrate some driver tests to GitHub Actions

* Druid
* MariaDB 10.2
* MariaDB latest
* MongoDB 4.0
* MongoDB 5.0
* MongoDB latest
* MySQL 5.7
* Postgres 9.6
* Postgres latest
* Presto
* Spark
* SQLite
* SQL Server 2017

* Don't use Buildjet runner

* Don't run the driver tests for draft PRs

* Use Buildjet for Spark driver tests

* Use Buildjet for Druid driver tests

.circleci/config.yml

@@ -27,52 +27,6 @@ executors:
     docker:
       - image: metabase/ci:java-17-clj-1.11.0.1100.04-2022-build
 
-  postgres-9-6:
-    working_directory: /home/circleci/metabase/metabase/
-    docker:
-      - image: metabase/ci:java-11-clj-1.11.0.1100.04-2022-build
-        environment:
-          MB_DB_TYPE: postgres
-          MB_DB_PORT: 5432
-          MB_DB_HOST: localhost
-          MB_DB_DBNAME: circle_test
-          MB_DB_USER: circle_test
-          MB_POSTGRESQL_TEST_USER: circle_test
-      - image: circleci/postgres:9.6-alpine
-        environment:
-          POSTGRES_USER: circle_test
-          POSTGRES_DB: circle_test
-
-  postgres-latest:
-    working_directory: /home/circleci/metabase/metabase/
-    docker:
-      - image: metabase/ci:java-11-clj-1.11.0.1100.04-2022-build
-        environment:
-          MB_DB_TYPE: postgres
-          MB_DB_PORT: 5432
-          MB_DB_HOST: localhost
-          MB_DB_DBNAME: metabase_test
-          MB_DB_USER: metabase_test
-          MB_POSTGRESQL_TEST_USER: metabase_test
-      - image: circleci/postgres:latest
-        environment:
-          POSTGRES_USER: metabase_test
-          POSTGRES_DB: metabase_test
-          POSTGRES_HOST_AUTH_METHOD: trust
-
-  mysql-5-7:
-    working_directory: /home/circleci/metabase/metabase/
-    docker:
-      - image: metabase/ci:java-11-clj-1.11.0.1100.04-2022-build
-        environment:
-          MB_DB_TYPE: mysql
-          MB_DB_HOST: localhost
-          MB_DB_PORT: 3306
-          MB_DB_DBNAME: circle_test
-          MB_DB_USER: root
-          MB_MYSQL_TEST_USER: root
-      - image: circleci/mysql:5.7.23
-
   mysql-latest:
     working_directory: /home/circleci/metabase/metabase/
     docker:
@@ -86,42 +40,6 @@ executors:
           MB_MYSQL_TEST_USER: root
       - image: circleci/mysql:latest
 
-  mariadb-10-2:
-    working_directory: /home/circleci/metabase/metabase/
-    docker:
-      - image: metabase/ci:java-11-clj-1.11.0.1100.04-2022-build
-        environment:
-          MB_DB_TYPE: mysql
-          MB_DB_HOST: localhost
-          MB_DB_PORT: 3306
-          MB_DB_DBNAME: circle_test
-          MB_DB_USER: root
-          MB_MYSQL_TEST_USER: root
-      - image: circleci/mariadb:10.2.23
-
-  mariadb-latest:
-    working_directory: /home/circleci/metabase/metabase/
-    docker:
-      - image: metabase/ci:java-11-clj-1.11.0.1100.04-2022-build
-        environment:
-          MB_DB_TYPE: mysql
-          MB_DB_HOST: localhost
-          MB_DB_PORT: 3306
-          MB_DB_DBNAME: circle_test
-          MB_DB_USER: root
-          MB_MYSQL_TEST_USER: root
-      - image: circleci/mariadb:latest
-        environment:
-          # MYSQL_DATABASE: metabase_test
-          # MYSQL_USER: root
-          # MYSQL_ALLOW_EMPTY_PASSWORD: yes
-
-  mongo-4-0:
-     working_directory: /home/circleci/metabase/metabase/
-     docker:
-       - image: metabase/ci:java-11-clj-1.11.0.1100.04-2022-build
-       - image: metabase/qa-databases:mongo-sample-4.0
-
   mongo-4-0-ssl:
      working_directory: /home/circleci/metabase/metabase/
      docker:
@@ -131,12 +49,6 @@ executors:
        - image: metabase/qa-databases:mongo-sample-4.0
          command: mongod --dbpath /data/db2/ --sslMode requireSSL --sslPEMKeyFile /etc/mongo/metamongo.pem --sslCAFile /etc/mongo/metaca.crt
 
-  mongo-5-0:
-     working_directory: /home/circleci/metabase/metabase/
-     docker:
-       - image: metabase/ci:java-11-clj-1.11.0.1100.04-2022-build
-       - image: metabase/qa-databases:mongo-sample-5.0
-
   mongo-5-0-ssl:
      working_directory: /home/circleci/metabase/metabase/
      docker:
@@ -146,16 +58,6 @@ executors:
        - image: metabase/qa-databases:mongo-sample-5.0
          command: mongod --dbpath /data/db2/ --tlsMode requireTLS --tlsCertificateKeyFile /etc/mongo/metamongo.pem --tlsCAFile /etc/mongo/metaca.crt
 
-  mongo-latest:
-     working_directory: /home/circleci/metabase/metabase/
-     docker:
-       - image: metabase/ci:java-11-clj-1.11.0.1100.04-2022-build
-       - image: circleci/mongo:latest
-         environment:
-           MONGO_INITDB_ROOT_USERNAME: metabase
-           MONGO_INITDB_ROOT_PASSWORD: metasample123
-
-
   presto-186:
     working_directory: /home/circleci/metabase/metabase/
     docker:
@@ -186,44 +88,12 @@ executors:
     # (see above)
     resource_class: large
 
-  sparksql:
-    working_directory: /home/circleci/metabase/metabase/
-    docker:
-      - image: metabase/ci:java-11-clj-1.11.0.1100.04-2022-build
-      - image: metabase/spark:3.2.1
-    resource_class: large
-
   vertica:
     working_directory: /home/circleci/metabase/metabase/
     docker:
       - image: metabase/ci:java-11-clj-1.11.0.1100.04-2022-build
       - image: sumitchawla/vertica
 
-  sqlserver:
-    working_directory: /home/circleci/metabase/metabase/
-    docker:
-      - image: metabase/ci:java-11-clj-1.11.0.1100.04-2022-build
-        environment:
-          MB_SQLSERVER_TEST_HOST: localhost
-          MB_SQLSERVER_TEST_PASSWORD: 'P@ssw0rd'
-          MB_SQLSERVER_TEST_USER: SA
-      - image: mcr.microsoft.com/mssql/server:2017-latest
-        environment:
-          ACCEPT_EULA: Y
-          SA_PASSWORD: 'P@ssw0rd'
-          MSSQL_MEMORY_LIMIT_MB: 1024
-
-  druid:
-    working_directory: /home/circleci/metabase/metabase/
-    docker:
-      - image: metabase/ci:java-11-clj-1.11.0.1100.04-2022-build
-      - image: metabase/druid:0.20.2
-        environment:
-          CLUSTER_SIZE: nano-quickstart
-    # Run Docker images with 8GB or RAM instead of the default 4GB for medium instances. The Druid Docker image runs
-    # OOM all the time with the default medium size.
-    resource_class: large
-
 ########################################################################################################################
 #                                             MAP FRAGMENTS AND CACHE KEYS                                             #
 ########################################################################################################################
@@ -719,22 +589,12 @@ workflows:
       - test-driver:
           matrix:
             parameters:
-              driver: ["bigquery-cloud-sdk", "googleanalytics", "sqlite"]
+              driver: ["bigquery-cloud-sdk", "googleanalytics"]
           name: be-tests-<< matrix.driver >>-ee
           requires:
             - be-tests-java-11-ee
           driver: << matrix.driver >>
 
-      - test-driver:
-          matrix:
-            parameters:
-              driver: ["sqlserver", "druid"]
-          name: be-tests-<< matrix.driver >>-ee
-          requires:
-            - be-tests-java-11-ee
-          e: << matrix.driver >>
-          driver: << matrix.driver >>
-
       - test-driver:
           name: be-google-related-drivers-classpath-test
           requires:
@@ -747,7 +607,7 @@ workflows:
       - test-driver:
           matrix:
             parameters:
-              version: ["mongo-4-0", "mongo-5-0", "mongo-latest", "mongo-4-0-ssl", "mongo-5-0-ssl"]
+              version: ["mongo-4-0-ssl", "mongo-5-0-ssl"]
           name: be-tests-<< matrix.version >>-ee
           description: "(<< matrix.version >>)"
           requires:
@@ -755,18 +615,6 @@ workflows:
           e: << matrix.version >>
           driver: mongo
 
-      - test-driver:
-          matrix:
-            parameters:
-              version: ["mysql-5-7", "mariadb-10-2", "mariadb-latest"]
-          name: be-tests-<< matrix.version >>-ee
-          description: "(<< matrix.version >>)"
-          requires:
-            - be-tests-java-11-ee
-          e:
-            name: << matrix.version >>
-          driver: mysql
-
       - test-driver:
           name: be-tests-mysql-latest-ee
           description: "(MySQL latest)"
@@ -809,36 +657,6 @@ workflows:
             MB_ORACLE_SSL_TEST_SSL_TRUSTSTORE_OPTIONS=local
             MB_ORACLE_SSL_TEST_SSL_TRUSTSTORE_PASSWORD_VALUE=metabase
 
-      - test-driver:
-          name: be-tests-postgres-ee
-          description: "(9.6)"
-          requires:
-            - be-tests-java-11-ee
-          e: postgres-9-6
-          driver: postgres
-
-      - test-driver:
-          name: be-tests-postgres-latest-ee
-          description: "(Latest)"
-          requires:
-            - be-tests-java-11-ee
-          e: postgres-latest
-          driver: postgres
-          extra-env: >-
-            MB_POSTGRES_SSL_TEST_SSL=true
-            MB_POSTGRES_SSL_TEST_SSL_MODE=verify-full
-            MB_POSTGRES_SSL_TEST_SSL_ROOT_CERT_PATH=/home/circleci/metabase/metabase/test-resources/certificates/us-east-2-bundle.pem
-
-      - test-driver:
-          name: be-tests-presto-ee
-          requires:
-            - be-tests-java-11-ee
-          e: presto-186
-          before-steps:
-            - wait-for-port:
-                port: 8080
-          driver: presto
-
       - test-driver:
           name: be-tests-presto-jdbc-ee
           requires:
@@ -889,16 +707,6 @@ workflows:
           driver: snowflake
           timeout: 115m
 
-      - test-driver:
-          name: be-tests-sparksql-ee
-          requires:
-            - be-tests-java-11-ee
-          e: sparksql
-          before-steps:
-            - wait-for-port:
-                port: 10000
-          driver: sparksql
-
       - test-driver:
           name: be-tests-vertica-ee
           requires:

.github/actions/test-driver/action.yml

+name: Test database driver
+inputs:
+  junit-name:
+    required: true
+    default: 'driver'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Prepare front-end environment
+      uses: ./.github/actions/prepare-frontend
+    - name: Prepare back-end environment
+      uses: ./.github/actions/prepare-backend
+    - name: Build static viz frontend
+      run: yarn build-static-viz
+      shell: bash
+    - name: Test database driver
+      run: clojure -X:dev:ci:ee:ee-dev:drivers:drivers-dev:test
+      shell: bash
+    - name: Publish Test Report (JUnit)
+      uses: mikepenz/action-junit-report@v2
+      if: always()
+      with:
+        report_paths: 'target/junit/**/*_test.xml'
+        check_name: JUnit Test Report ${{ inputs.junit-name }}

.github/workflows/drivers-skipped-checks.yml

+# Required checks with path filtering rules will block pull requests from merging if they change only the excluded files.
+# This is a workaround to allow the PR to be merged.
+# https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/defining-the-mergeability-of-pull-requests/troubleshooting-required-status-checks#handling-skipped-but-required-checks
+name: Driver Tests
+
+on:
+  pull_request:
+    paths:
+      - "docs/**"
+      - "**.md"
+      - "frontend/test/**"
+      - "enterprise/frontend/test/**"
+
+jobs:
+
+  be-tests-druid-ee:
+    runs-on: ubuntu-20.04
+    steps:
+      - run: |
+          echo "Didn't run due to conditional filtering"
+
+  be-tests-mariadb-10-2-ee:
+    runs-on: ubuntu-20.04
+    steps:
+      - run: |
+          echo "Didn't run due to conditional filtering"
+
+  be-tests-mariadb-latest-ee:
+    runs-on: ubuntu-20.04
+    steps:
+      - run: |
+          echo "Didn't run due to conditional filtering"
+
+  be-tests-mongo-4-0-ee:
+    runs-on: ubuntu-20.04
+    steps:
+      - run: |
+          echo "Didn't run due to conditional filtering"
+
+  be-tests-mongo-5-0-ee:
+    runs-on: ubuntu-20.04
+    steps:
+      - run: |
+          echo "Didn't run due to conditional filtering"
+
+  be-tests-mongo-latest-ee:
+    runs-on: ubuntu-20.04
+    steps:
+      - run: |
+          echo "Didn't run due to conditional filtering"
+
+  be-tests-mysql-5-7-ee:
+    runs-on: ubuntu-20.04
+    steps:
+      - run: |
+          echo "Didn't run due to conditional filtering"
+
+  be-tests-postgres-ee:
+    runs-on: ubuntu-20.04
+    steps:
+      - run: |
+          echo "Didn't run due to conditional filtering"
+
+  be-tests-postgres-latest-ee:
+    runs-on: ubuntu-20.04
+    steps:
+      - run: |
+          echo "Didn't run due to conditional filtering"
+
+  be-tests-presto-186-ee:
+    runs-on: ubuntu-20.04
+    steps:
+      - run: |
+          echo "Didn't run due to conditional filtering"
+
+  be-tests-sparksql-ee:
+    runs-on: ubuntu-20.04
+    steps:
+      - run: |
+          echo "Didn't run due to conditional filtering"
+
+  be-tests-sqlite-ee:
+    runs-on: ubuntu-20.04
+    steps:
+      - run: |
+          echo "Didn't run due to conditional filtering"
+
+  be-tests-sqlserver-ee:
+    runs-on: ubuntu-20.04
+    steps:
+      - run: |
+          echo "Didn't run due to conditional filtering"

.github/workflows/drivers.yml

+name: Driver Tests
+
+on:
+  push:
+    branches:
+      - 'master'
+      - 'release-**'
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+    paths-ignore:
+      - "docs/**"
+      - "**.md"
+      - "frontend/test/**"
+      - "enterprise/frontend/test/**"
+
+jobs:
+
+  be-tests-druid-ee:
+    runs-on: buildjet-2vcpu-ubuntu-2004
+    timeout-minutes: 25
+    env:
+      CI: 'true'
+      DRIVERS: druid
+    services:
+      druid:
+        image: metabase/druid:0.20.2
+        ports:
+          - "8082:8082"
+        env:
+          CLUSTER_SIZE: nano-quickstart
+    steps:
+    - uses: actions/checkout@v3
+    - name: Test Druid driver
+      uses: ./.github/actions/test-driver
+      with:
+        junit-name: 'be-tests-druid-ee'
+
+  be-tests-mariadb-10-2-ee:
+    runs-on: ubuntu-20.04
+    timeout-minutes: 25
+    env:
+      CI: 'true'
+      DRIVERS: mysql
+      MB_DB_TYPE: mysql
+      MB_DB_HOST: localhost
+      MB_DB_PORT: 3306
+      MB_DB_DBNAME: circle_test
+      MB_DB_USER: root
+      MB_MYSQL_TEST_USER: root
+    services:
+      mariadb:
+        image: circleci/mariadb:10.2.23
+        ports:
+          - "3306:3306"
+    steps:
+    - uses: actions/checkout@v3
+    - name: Test MariaDB driver (10.2)
+      uses: ./.github/actions/test-driver
+      with:
+        junit-name: 'be-tests-mariadb-10-2-ee'
+
+  be-tests-mariadb-latest-ee:
+    runs-on: ubuntu-20.04
+    timeout-minutes: 25
+    env:
+      CI: 'true'
+      DRIVERS: mysql
+      MB_DB_TYPE: mysql
+      MB_DB_HOST: localhost
+      MB_DB_PORT: 3306
+      MB_DB_DBNAME: circle_test
+      MB_DB_USER: root
+      MB_MYSQL_TEST_USER: root
+    services:
+      mariadb:
+        image: circleci/mariadb:latest
+        ports:
+          - "3306:3306"
+    steps:
+    - uses: actions/checkout@v3
+    - name: Test MariaDB driver (latest)
+      uses: ./.github/actions/test-driver
+      with:
+        junit-name: 'be-tests-mariadb-latest-ee'
+
+  be-tests-mongo-4-0-ee:
+    runs-on: ubuntu-20.04
+    timeout-minutes: 25
+    env:
+      CI: 'true'
+      DRIVERS: mongo
+    services:
+      mongodb:
+        image: metabase/qa-databases:mongo-sample-4.0
+        ports:
+          - "27017:27017"
+    steps:
+    - uses: actions/checkout@v3
+    - name: Test MongoDB driver (4.0)
+      uses: ./.github/actions/test-driver
+      with:
+        junit-name: 'be-tests-mongo-4-0-ee'
+
+  be-tests-mongo-5-0-ee:
+    runs-on: ubuntu-20.04
+    timeout-minutes: 25
+    env:
+      CI: 'true'
+      DRIVERS: mongo
+    services:
+      mongodb:
+        image: metabase/qa-databases:mongo-sample-5.0
+        ports:
+          - "27017:27017"
+    steps:
+    - uses: actions/checkout@v3
+    - name: Test MongoDB driver (5.0)
+      uses: ./.github/actions/test-driver
+      with:
+        junit-name: 'be-tests-mongo-5-0-ee'
+
+  be-tests-mongo-latest-ee:
+    runs-on: ubuntu-20.04
+    timeout-minutes: 25
+    env:
+      CI: 'true'
+      DRIVERS: mongo
+    services:
+      mongodb:
+        image: circleci/mongo:latest
+        ports:
+          - "27017:27017"
+        env:
+          MONGO_INITDB_ROOT_USERNAME: metabase
+          MONGO_INITDB_ROOT_PASSWORD: metasample123
+    steps:
+    - uses: actions/checkout@v3
+    - name: Test MongoDB driver (latest)
+      uses: ./.github/actions/test-driver
+      with:
+        junit-name: 'be-tests-mongo-latest-ee'
+
+  be-tests-mysql-5-7-ee:
+    runs-on: ubuntu-20.04
+    timeout-minutes: 25
+    env:
+      CI: 'true'
+      DRIVERS: mysql
+      MB_DB_TYPE: mysql
+      MB_DB_HOST: localhost
+      MB_DB_PORT: 3306
+      MB_DB_DBNAME: circle_test
+      MB_DB_USER: root
+      MB_MYSQL_TEST_USER: root
+    services:
+      mysql:
+        image: circleci/mysql:5.7.23
+        ports:
+          - "3306:3306"
+    steps:
+    - uses: actions/checkout@v3
+    - name: Test MySQL driver (5.7)
+      uses: ./.github/actions/test-driver
+      with:
+        junit-name: 'be-tests-mysql-5-7-ee'
+
+  be-tests-postgres-ee:
+    runs-on: ubuntu-20.04
+    timeout-minutes: 25
+    env:
+      CI: 'true'
+      DRIVERS: postgres
+      MB_DB_TYPE: postgres
+      MB_DB_PORT: 5432
+      MB_DB_HOST: localhost
+      MB_DB_DBNAME: circle_test
+      MB_DB_USER: circle_test
+      MB_POSTGRESQL_TEST_USER: circle_test
+    services:
+      postgres:
+        image: circleci/postgres:9.6-alpine
+        ports:
+          - "5432:5432"
+        env:
+          POSTGRES_USER: circle_test
+          POSTGRES_DB: circle_test
+    steps:
+    - uses: actions/checkout@v3
+    - name: Test Postgres driver (9.6)
+      uses: ./.github/actions/test-driver
+      with:
+        junit-name: 'be-tests-postgres-ee'
+
+  be-tests-postgres-latest-ee:
+    runs-on: ubuntu-20.04
+    timeout-minutes: 25
+    env:
+      CI: 'true'
+      DRIVERS: postgres
+      MB_DB_TYPE: postgres
+      MB_DB_PORT: 5432
+      MB_DB_HOST: localhost
+      MB_DB_DBNAME: circle_test
+      MB_DB_USER: circle_test
+      MB_POSTGRESQL_TEST_USER: circle_test
+      MB_POSTGRES_SSL_TEST_SSL: true
+      MB_POSTGRES_SSL_TEST_SSL_MODE: verify-full
+      MB_POSTGRES_SSL_TEST_SSL_ROOT_CERT_PATH: 'test-resources/certificates/us-east-2-bundle.pem'
+    services:
+      postgres:
+        image: circleci/postgres:latest
+        ports:
+          - "5432:5432"
+        env:
+          POSTGRES_USER: circle_test
+          POSTGRES_DB: circle_test
+          POSTGRES_HOST_AUTH_METHOD: trust
+    steps:
+    - uses: actions/checkout@v3
+    - name: Test Postgres driver (latest)
+      uses: ./.github/actions/test-driver
+      with:
+        junit-name: 'be-tests-postgres-latest-ee'
+
+  be-tests-presto-186-ee:
+    runs-on: ubuntu-20.04
+    timeout-minutes: 25
+    env:
+      CI: 'true'
+      DRIVERS: presto
+    services:
+      presto:
+        image: metabase/presto-mb-ci:0.186
+        ports:
+          - "8080:8080"
+        env:
+          JAVA_TOOL_OPTIONS: "-Xmx2g"
+    steps:
+    - uses: actions/checkout@v3
+    - name: Test Presto 0.186 driver
+      uses: ./.github/actions/test-driver
+      with:
+        junit-name: 'be-tests-presto-186-ee'
+
+  be-tests-sparksql-ee:
+    runs-on: buildjet-2vcpu-ubuntu-2004
+    timeout-minutes: 25
+    env:
+      CI: 'true'
+      DRIVERS: sparksql
+    services:
+      sparksql:
+        image: metabase/spark:3.2.1
+        ports:
+          - "10000:10000"
+    steps:
+    - uses: actions/checkout@v3
+    - name: Test Spark driver
+      uses: ./.github/actions/test-driver
+      with:
+        junit-name: 'be-tests-sparksql-ee'
+
+  be-tests-sqlite-ee:
+    runs-on: ubuntu-20.04
+    timeout-minutes: 25
+    env:
+      CI: 'true'
+      DRIVERS: sqlite
+    steps:
+    - uses: actions/checkout@v3
+    - name: Test SQLite driver
+      uses: ./.github/actions/test-driver
+      with:
+        junit-name: 'be-tests-sqlite-ee'
+
+  be-tests-sqlserver-ee:
+    runs-on: ubuntu-20.04
+    timeout-minutes: 25
+    env:
+      CI: 'true'
+      DRIVERS: sqlserver
+      MB_SQLSERVER_TEST_HOST: localhost
+      MB_SQLSERVER_TEST_PASSWORD: 'P@ssw0rd'
+      MB_SQLSERVER_TEST_USER: SA
+    services:
+      sqlserver:
+        image: mcr.microsoft.com/mssql/server:2017-latest
+        ports:
+          - "1433:1433"
+        env:
+          ACCEPT_EULA: Y
+          SA_PASSWORD: 'P@ssw0rd'
+          MSSQL_MEMORY_LIMIT_MB: 1024
+    steps:
+    - uses: actions/checkout@v3
+    - name: Test MS SQL Server driver
+      uses: ./.github/actions/test-driver
+      with:
+        junit-name: 'be-tests-sqlserver-ee'