From 23c85cf032613ebdd734daea534ddf8275499a3d Mon Sep 17 00:00:00 2001
From: Oreon Lothamer <73498677+oreonl@users.noreply.github.com>
Date: Wed, 2 Oct 2024 07:55:40 -1000
Subject: [PATCH] build ee-extra docker image (#47960)
---
.github/actions/build-ee-extra/action.yml | 126 ++++++++++++++++++++++
.github/workflows/uberjar.yml | 21 +++-
2 files changed, 146 insertions(+), 1 deletion(-)
create mode 100644 .github/actions/build-ee-extra/action.yml
diff --git a/.github/actions/build-ee-extra/action.yml b/.github/actions/build-ee-extra/action.yml
new file mode 100644
index 00000000000..4d4423fd32c
--- /dev/null
+++ b/.github/actions/build-ee-extra/action.yml
@@ -0,0 +1,126 @@
+name: Build ee-extra
+description: Build Metabase Enterprise Edition with extra features
+
+inputs:
+ iam-role:
+ description: "The IAM role to assume"
+ required: true
+
+runs:
+ using: composite
+ steps:
+ - name: Prepare Java
+ uses: actions/setup-java@v3
+ with:
+ java-version: '17'
+ distribution: 'temurin'
+ - name: Reveal its original version.properties
+ run: jar xf ./bin/docker/metabase.jar version.properties && cat version.properties
+ shell: bash
+ - name: get major version
+ id: major_version
+ run: echo "major_version=$(cat version.properties | grep 'tag=' | cut -d'.' -f2)" >> $GITHUB_OUTPUT
+ shell: bash
+ - name: get drivers registry
+ run: |
+ curl -OL https://raw.githubusercontent.com/metabase/metabase-registry/main/registry.yaml
+ shell: bash
+ - uses: actions/setup-node@v4
+ with:
+ node-version: lts/Hydrogen # 18.x.x
+ - run: rm package.json && npm install yaml
+ shell: bash
+ - name: parse drivers registry
+ uses: actions/github-script@v7
+ id: drivers_registry
+ with:
+ script: | # js
+ const { parse } = require('yaml');
+ const fs = require('fs');
+ const registry = parse(fs.readFileSync('registry.yaml', 'utf8'));
+
+ const drivers = registry.drivers;
+ const majorVersion = '${{ steps.major_version.outputs.major_version }}';
+
+ const getDriverUrl = (driverName) => {
+ const driver = drivers.find(driver => driver.name === driverName);
+
+ if (!driver) {
+ throw new Error(`Driver ${driverName} not found in the registry`);
+ }
+
+ const driverUrl = driver.versions[majorVersion] ?? driver.versions.default;
+
+ if (!driverUrl) {
+ throw new Error(`Driver ${driverName} version ${majorVersion} not found in the registry`);
+ }
+
+ return driverUrl;
+ };
+
+ const extraDrivers = {
+ firebolt: getDriverUrl('firebolt'),
+ starburst: getDriverUrl('starburst'),
+ exasol: getDriverUrl('exasol'),
+ exasol_jdbc: drivers.find(d => d.name === 'exasol').deps[0],
+ clickhouse: getDriverUrl('clickhouse'),
+ ocient: getDriverUrl('ocient'),
+ materialize: getDriverUrl('materialize'),
+ };
+
+ console.log(extraDrivers);
+
+ return extraDrivers;
+ - name: Download Firebolt driver
+ run: |
+ curl -OL ${{ fromJson(steps.drivers_registry.outputs.result).firebolt }}
+ working-directory: modules
+ shell: bash
+ - name: Download Starburst driver
+ run: |
+ curl -OL ${{ fromJson(steps.drivers_registry.outputs.result).starburst }}
+ shell: bash
+ working-directory: modules
+ - name: Download Exasol driver
+ run: |
+ curl -OL ${{ fromJson(steps.drivers_registry.outputs.result).exasol }}
+ curl -OL ${{ fromJson(steps.drivers_registry.outputs.result).exasol_jdbc }}
+ shell: bash
+ working-directory: modules
+ - name: Download Clickhouse driver
+ run: |
+ curl -OL ${{ fromJson(steps.drivers_registry.outputs.result).clickhouse }}
+ shell: bash
+ working-directory: modules
+ - name: Download Ocient driver
+ run: |
+ curl -OL ${{ fromJson(steps.drivers_registry.outputs.result).ocient }}
+ shell: bash
+ working-directory: modules
+ - name: Download Materialize driver
+ run: |
+ curl -OL ${{ fromJson(steps.drivers_registry.outputs.result).materialize }}
+ shell: bash
+ working-directory: modules
+ - name: Bundle additional drivers into the Uberjar
+ run: jar uf bin/docker/metabase.jar modules/*.jar
+ shell: bash
+ - name: Launch uberjar
+ run: java -jar ./bin/docker/metabase.jar &
+ shell: bash
+ - name: Wait for Metabase to start
+ run: while ! curl 'http://localhost:3000/api/health' | grep '{"status":"ok"}'; do sleep 1; done
+ shell: bash
+ - name: Kill the java process
+ run: pkill -f metabase.jar
+ shell: bash
+ - name: configure aws credentials
+ if: ${{ github.ref_name == 'master' }}
+ uses: aws-actions/configure-aws-credentials@v4
+ with:
+ role-to-assume: ${{ inputs.iam-role }}
+ role-session-name: GitHub_to_AWS_via_FederatedOIDC
+ aws-region: us-east-1
+ - name: Login to Amazon ECR
+ if: ${{ github.ref_name == 'master' }}
+ uses: aws-actions/amazon-ecr-login@v2
diff --git a/.github/workflows/uberjar.yml b/.github/workflows/uberjar.yml
index da52399fc15..ea9e294f0fa 100644
--- a/.github/workflows/uberjar.yml
+++ b/.github/workflows/uberjar.yml
@@ -80,12 +80,15 @@ jobs:
needs: check_jar_health
strategy:
matrix:
- edition: [ee, oss]
+ edition: [ee, oss, ee-extra]
services:
registry:
image: registry:2
ports:
- 5000:5000
+ permissions:
+ id-token: write
+ contents: read
steps:
- name: Extract and clean branch name
shell: bash
@@ -98,12 +101,24 @@ jobs:
with:
ref: ${{ github.event.inputs.commit }}
- name: Download uploaded artifacts to insert into container
+ if: ${{ matrix.edition != 'ee-extra' }}
uses: actions/download-artifact@v4
with:
name: metabase-${{ matrix.edition }}-${{ github.sha }}-uberjar
path: bin/docker/
+ - name: Download uploaded artifacts to insert into container
+ if: ${{ matrix.edition == 'ee-extra' }}
+ uses: actions/download-artifact@v4
+ with:
+ name: metabase-ee-${{ github.sha }}-uberjar
+ path: bin/docker/
- name: Move the ${{ matrix.edition }} uberjar to the context dir
run: mv bin/docker/target/uberjar/metabase.jar bin/docker/.
+ - name: Add partner drivers to the container
+ if: ${{ matrix.edition == 'ee-extra' }}
+ uses: ./.github/actions/build-ee-extra
+ with:
+ iam-role: ${{ secrets.METABASE_EE_EXTRA_IAM_ROLE }}
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v2.5.0
@@ -141,6 +156,10 @@ jobs:
if: ${{ (github.ref_name == 'master') && matrix.edition == 'oss' }}
run: docker tag localhost:5000/metabase-dev:${{ steps.extract_branch.outputs.branch }}-oss ${{ github.repository_owner }}/metabase-head:latest && docker push ${{ github.repository_owner }}/metabase-head:latest
+ - name: Retag and push images if master (ee-extra)
+ if: ${{ (github.ref_name == 'master') && matrix.edition == 'ee-extra' }}
+ run: docker tag localhost:5000/metabase-dev:${{ steps.extract_branch.outputs.branch }}-ee-extra ${{ secrets.METABASE_EE_EXTRA_CONTAINER_REGISTRY }}/metabase-enterprise-head:latest && docker push ${{ secrets.METABASE_EE_EXTRA_CONTAINER_REGISTRY }}/metabase-enterprise-head:latest
+
- name: Retag and push images if dev branch
if: ${{ !(startsWith(github.ref_name,'master') || startsWith(github.ref_name,'backport')) && matrix.edition == 'ee' }}
run: docker tag localhost:5000/metabase-dev:${{ steps.extract_branch.outputs.branch }}-ee ${{ github.repository_owner }}/metabase-dev:${{ steps.extract_branch.outputs.branch }} && docker push ${{ github.repository_owner }}/metabase-dev:${{ steps.extract_branch.outputs.branch }}
--
GitLab