make our scripts for packaging docker images and aws-eb version files a bit...

make our scripts for packaging docker images and aws-eb version files a bit more generic so they are more reusable.

bin/aws-eb-docker/.ebextensions/01_metabase.config

+######
+# Metabase Report server Elastic Beanstalk configuration
+# Modify the environmental variables below to customize your installation
+# Comment out a variable to disable a feature
+#####
+container_commands:
+    #customize_env: 
+        #env:
+            #NGINX_SERVER_NAME: metabase.example.com
+            #NGINX_FORCE_SSL: 1
+            #PAPERTRAIL_HOSTNAME: $HOSTNAME
+            #PAPERTRAIL_HOST: foobar.papertrailapp.com
+            #PAPERTRAIL_PORT: 12345
+            #PAPERTRAIL_FILES: /var/log/nginx/access.log /var/log/nginx/error.log
+        #command: true
+        #ignoreErrors: false
+
+    01_server-name:
+        command: ".ebextensions/metabase_config/metabase-setup.sh server_name"
+        test: test $NGINX_SERVER_NAME
+        ignoreErrors: true
+
+    02_server_https:
+        command: ".ebextensions/metabase_config/metabase-setup.sh server_https"
+        test: test $NGINX_FORCE_SSL
+        ignoreErrors: true
+
+    03_log_x_real_ip:
+        command: ".ebextensions/metabase_config/metabase-setup.sh log_x_real_ip"
+        ignoreErrors: true
+
+    04_install_papertrail:
+        command: ".ebextensions/metabase_config/metabase-setup.sh install_papertrail"
+        test: test $PAPERTRAIL_HOST
+        ignoreErrors: true
+
+    05_try_papertrail:
+        command: "/sbin/service remote_syslog restart"
+        test: test -e /etc/log_files.yml
+        ignoreErrors: true
+
+    06_try_nginx:
+        command: "/sbin/service nginx restart"
+        test: nginx -t
+        ignoreErrors: false
+
+option_settings:
+  - namespace: aws:elasticbeanstalk:command
+    option_name: Timeout
+    value: 600

bin/aws-eb-docker/.ebextensions/metabase_config/metabase-setup.sh

+#!/bin/bash
+####
+# Metabase Report server Elastic Beanstalk metabase-setup.sh
+# Modify the environmental variables to customize your installation
+# Unset a variable to disable a feature
+####
+
+# add files to papertrail
+pt_files () {
+    sed -i '/  - .*/d' /etc/log_files.yml
+    set -f
+    for file in $PAPERTRAIL_FILES; do
+        sed -i 's|files:|files:\n  - '$file'|' /etc/log_files.yml
+    done
+    set +f
+}
+
+# papertail remote host
+pt_remote_host () {
+    sed -i "s/.*host:.*/  host: $PAPERTRAIL_HOST/" /etc/log_files.yml
+}
+
+# papertail remote port
+pt_port () {
+    sed -i "s/.*port:.*/  port: $PAPERTRAIL_PORT/" /etc/log_files.yml
+}
+
+# papertail local host
+pt_local_host () {
+    eval export PAPERTRAIL_HOSTNAME=$PAPERTRAIL_HOSTNAME # expand vars like $HOSTNAME
+    sed -i "s/.*hostname:.*/hostname: $PAPERTRAIL_HOSTNAME/" /etc/log_files.yml
+}
+
+# nginx server name
+server_name () {
+    [[ "$NGINX_SERVER_NAME" ]] && cp_default_server
+    cd /etc/nginx/sites-available/
+    if [[ "$NGINX_SERVER_NAME" ]] ; then
+        if ! grep -q server_name elasticbeanstalk-nginx-docker-proxy.conf ; then
+            sed -i "s|listen 80\;|listen 80\;\n        server_name $NGINX_SERVER_NAME \*\.$NGINX_SERVER_NAME\;\n|" elasticbeanstalk-nginx-docker-proxy.conf
+        fi
+    else
+        # no hostname passed, disable default_server
+        sed -i '/server_name/d' elasticbeanstalk-nginx-docker-proxy.conf
+        [[ -e /etc/nginx/sites-enabled/default_server ]] && rm /etc/nginx/sites-enabled/default_server
+    fi
+}
+
+# enable https redirect
+server_https () {
+    cd /etc/nginx/sites-available/
+    if [[ "$NGINX_FORCE_SSL" ]] && ! grep -q https elasticbeanstalk-nginx-docker-proxy.conf ; then
+        sed -i 's|location \/ {|location \/ {\n\n        if ($http_x_forwarded_proto != "https") {\n                rewrite ^ https:\/\/$host$request_uri? permanent;\n        }\n|' elasticbeanstalk-nginx-docker-proxy.conf
+    fi
+}
+
+# download, install and configure papertrail
+install_papertrail () {
+    cp .ebextensions/metabase_config/papertrail/log_files.yml /etc/log_files.yml && chmod 644 /etc/log_files.yml
+    cp .ebextensions/metabase_config/papertrail/remote_syslog /etc/init.d/remote_syslog && chmod 555 /etc/init.d/remote_syslog
+    cd /tmp/
+    wget -q "https://github.com/papertrail/remote_syslog2/releases/download/v0.14/remote_syslog_linux_amd64.tar.gz" &&
+        tar xzf remote_syslog_linux_amd64.tar.gz
+    /sbin/service remote_syslog stop
+    mv /tmp/remote_syslog/remote_syslog /usr/local/bin/
+    rm -rf remote_syslog_linux_amd64.tar.gz remote_syslog
+    # Setup Papertrail
+    [[ "$PAPERTRAIL_HOST" ]] && pt_remote_host
+    [[ "$PAPERTRAIL_PORT" ]] && pt_port
+    [[ "$PAPERTRAIL_FILES" ]] && pt_files
+    [[ "$PAPERTRAIL_HOSTNAME" ]] && pt_local_host
+}
+
+# enable default_server to drop DNS poisoning
+cp_default_server () {
+    cp .ebextensions/metabase_config/nginx/default_server /etc/nginx/sites-available/default_server
+    [[ ! -e /etc/nginx/sites-enabled/default_server ]] &&
+        ln -s /etc/nginx/sites-available/default_server /etc/nginx/sites-enabled/default_server
+}
+
+# update nginx logging to include x_real_ip
+log_x_real_ip () {
+    cp .ebextensions/metabase_config/nginx/log_x_real_ip.conf /etc/nginx/conf.d/log_x_real_ip.conf
+    cd  /etc/nginx/sites-available
+    if ! grep -q access_log *-proxy.conf ; then 
+        sed -i 's|location \/ {|location \/ {\n\n        access_log \/var\/log\/nginx\/access.log log_x_real_ip;\n|' *-proxy.conf
+    fi
+}
+
+case $1 in
+server_name)
+    server_name
+    ;;
+server_https)
+    server_https
+    ;;
+install_papertrail)
+    install_papertrail
+    ;;
+log_x_real_ip)
+    log_x_real_ip
+    ;;
+esac

bin/aws-eb-docker/.ebextensions/metabase_config/nginx/default_server

+# /etc/nginx/sites-available/default_server
+server {
+    listen 80 default_server;
+
+    location /api/health {
+        access_log off;
+        proxy_pass            http://docker;
+        proxy_http_version    1.1;
+        proxy_set_header    Connection            $connection_upgrade;
+        proxy_set_header    Upgrade               $http_upgrade;
+        proxy_set_header    Host                  $host;
+        proxy_set_header    X-Real-IP             $remote_addr;
+        proxy_set_header    X-Forwarded-For       $proxy_add_x_forwarded_for;
+    }
+
+    location / {
+        return 444;
+    }
+}

bin/aws-eb-docker/.ebextensions/metabase_config/nginx/log_x_real_ip.conf

+# /etc/nginx/conf.d/log_x_real_ip.conf
+log_format log_x_real_ip '$proxy_add_x_forwarded_for - [$time_local] '
+                      '$request $status $body_bytes_sent '
+                      '$http_referer $http_user_agent';

bin/aws-eb-docker/.ebextensions/metabase_config/papertrail/log_files.yml

+# /etc/log_files.yml - Papertrail
+files:
+  - /var/log/nginx/access.log
+hostname:
+destination:
+  host:
+  port:
+  protocol: tls
+new_file_check_interval: "10" # Check every 10 seconds

bin/aws-eb-docker/.ebextensions/metabase_config/papertrail/remote_syslog

+#!/bin/bash
+# /etc/init.d/remote_syslog
+#
+# remote_syslog This shell script takes care of starting and stopping
+#               remote_syslog daemon
+#
+# chkconfig: - 58 74
+# description: papertrail/remote_syslog \
+#              https://github.com/papertrail/remote_syslog2/blob/master/examples/remote_syslog.init.d
+
+### BEGIN INIT INFO
+# Provides: remote_syslog
+# Required-Start: $network $local_fs $remote_fs
+# Required-Stop: $network $local_fs $remote_fs
+# Should-Start: $syslog $named ntpdate
+# Should-Stop: $syslog $named
+# Short-Description: start and stop remote_errolog
+# Description: papertrail/remote_syslog2
+#              https://github.com/papertrail/remote_syslog2/blob/master/examples/remote_syslog.init.d
+### END INIT INFO
+
+# Source function library.
+. /etc/init.d/functions
+
+# Source networking configuration.
+. /etc/sysconfig/network
+ 
+prog="/usr/local/bin/remote_syslog"   
+config="/etc/log_files.yml"
+pid_dir="/var/run"
+
+EXTRAOPTIONS=""
+
+pid_file="$pid_dir/remote_syslog.pid"
+
+PATH=/sbin:/bin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin
+
+RETVAL=0
+
+is_running(){
+    # Do we have PID-file?
+    if [ -f "$pid_file" ]; then
+        # Check if proc is running
+        pid=`cat "$pid_file" 2> /dev/null`
+        if [[ $pid != "" ]]; then
+            exepath=`readlink /proc/"$pid"/exe 2> /dev/null`
+            exe=`basename "$exepath"`
+            if [[ $exe == "remote_syslog" ]]; then
+                # Process is running
+                return 0
+            fi
+        fi
+    fi
+    return 1
+}
+
+start(){
+    echo -n $"Starting $prog: "
+    unset HOME MAIL USER USERNAME
+    $prog -c $config --pid-file=$pid_file $EXTRAOPTIONS
+    RETVAL=$?
+    echo
+    return $RETVAL
+}
+
+stop(){
+    echo -n $"Stopping $prog: "
+    if (is_running); then
+      kill `cat $pid_file`
+      RETVAL=$?
+      echo
+      return $RETVAL
+    else
+      echo "$pid_file not found"
+    fi
+}
+
+status(){
+    echo -n $"Checking for $pid_file: "
+
+    if (is_running); then
+      echo "found"
+    else
+      echo "not found"
+    fi
+}
+
+reload(){
+    restart
+}
+
+restart(){
+    stop
+    start
+}
+
+condrestart(){
+    is_running && restart
+    return 0
+}
+
+# See how we were called.
+case "$1" in
+    start)
+  start
+  ;;
+    stop)
+  stop
+  ;;
+    status)
+  status
+  ;;
+    restart)
+  restart
+  ;;
+    reload)
+  reload
+  ;;
+    condrestart)
+  condrestart
+  ;;
+    *)
+  echo $"Usage: $0 {start|stop|status|restart|condrestart|reload}"
+  RETVAL=1
+esac
+
+exit $RETVAL

bin/aws-eb-docker/Dockerrun.aws.json.template

+{
+  "AWSEBDockerrunVersion": "1",
+  "Image": {
+    "Name": "metabase/@@MB_REPOSITORY@@:@@MB_TAG@@",
+    "Update": "true"
+  },
+  "Ports": [
+    {
+      "ContainerPort": "3000"
+    }
+  ],
+  "Logging": "/var/log/metabase"
+}

bin/aws-eb-docker/build-eb-version.sh

+#!/bin/bash
+
+BASEDIR=$(dirname $0)
+source "$BASEDIR/functions"
+
+MB_TAG=$1
+if [ -z $MB_TAG ]; then
+    echo "usage: $0 <release-name> <docker-repository>"
+    exit 1
+fi
+
+MB_DOCKER_REPOSITORY=$2
+if [ -z $MB_DOCKER_REPOSITORY ]; then
+    echo "usage: $0 <release-name> <docker-repository>"
+    exit 1
+fi
+
+
+make_eb_version ${MB_TAG} ${MB_DOCKER_REPOSITORY}

bin/aws-eb-docker/deploy-eb-version.sh

+#!/bin/bash
+
+BASEDIR=$(dirname $0)
+source "$BASEDIR/functions"
+
+MB_TAG=$1
+if [ -z $MB_TAG ]; then
+    echo "usage: $0 <release-name> <docker-repository> <eb-environment>"
+    exit 1
+fi
+
+MB_DOCKER_REPOSITORY=$2
+if [ -z $MB_DOCKER_REPOSITORY ]; then
+    echo "usage: $0 <release-name> <docker-repository> <eb-environment>"
+    exit 1
+fi
+
+EB_ENVIRONMENT=$3
+if [ -z $EB_ENVIRONMENT ]; then
+    echo "usage: $0 <release-name> <docker-repository> <eb-environment>"
+    exit 1
+fi
+
+if [ -z "$AWS_DEFAULT_PROFILE" ]; then
+    echo "You must set the AWS_DEFAULT_PROFILE environment variable in order to deploy to AWS!"
+    exit 1
+fi
+
+
+make_eb_version ${MB_TAG} ${MB_DOCKER_REPOSITORY}
+upload_eb_version ${MB_TAG}
+create_eb_version ${MB_TAG}
+deploy_version ${MB_TAG} ${EB_ENVIRONMENT}

bin/aws-eb-docker/functions

+#!/bin/bash
+set -eo pipefail
+
+BASEDIR=$(dirname $0)
+CURRENTDIR=$PWD
+PROJECT_ROOT=$(cd ${BASEDIR}/..; pwd)
+
+ARTIFACTS_S3BUCKET=${S3BUCKET:=metabase-artifacts}
+
+
+export LANG=en_US.UTF-8
+export LANGUAGE=$LANG
+export LC_ALL=$LANG
+
+
+make_eb_version() {
+    MB_TAG=$1
+    MB_DOCKER_REPOSITORY=$2
+
+    $(which locale) | $(which sort) || true
+
+    if [[ -z $MB_TAG ]]; then
+        echo "release name not provided!"
+        exit 1
+    fi
+
+    if [[ -z $MB_DOCKER_REPOSITORY ]]; then
+        echo "docker repository name not provided!"
+        exit 1
+    fi
+
+    RELEASE_FILE="/tmp/${MB_TAG}.zip"
+
+    echo "Building Elastic Beanstalk app version for Metabase using Docker Image: metabase/${MB_DOCKER_REPOSITORY}:${MB_TAG}"
+
+    # dynamically insert our MB version into the EB config file
+    sed "s/@@MB_REPOSITORY@@/${MB_DOCKER_REPOSITORY}/" < ${BASEDIR}/Dockerrun.aws.json.template > ${BASEDIR}/Dockerrun.aws.json.tmp
+    sed "s/@@MB_TAG@@/${MB_TAG}/" < ${BASEDIR}/Dockerrun.aws.json.tmp > ${BASEDIR}/Dockerrun.aws.json
+
+    # create our EB zip file
+    cd $BASEDIR; zip -r ${RELEASE_FILE} .ebextensions Dockerrun.aws.json; cd $CURRENTDIR
+
+    # clean up the temporary Dockerrun.aws.json file we created
+    rm ${BASEDIR}/Dockerrun.aws.json.tmp
+    rm ${BASEDIR}/Dockerrun.aws.json
+}
+
+upload_eb_version() {
+    MB_TAG=$1
+
+    $(which locale) | $(which sort) || true
+
+    if [[ -z $MB_TAG ]]; then
+        echo "release name not provided!"
+        exit 1
+    fi
+
+    echo "uploading /tmp/${MB_TAG}.zip -> $ARTIFACTS_S3BUCKET/eb/"
+    aws s3 cp /tmp/${MB_TAG}.zip s3://$ARTIFACTS_S3BUCKET/eb/${MB_TAG}.zip
+}
+
+create_eb_version() {
+    EB_APPLICATION=Metabase
+    MB_TAG=$1
+
+    $(which locale) | $(which sort) || true
+
+    if [[ -z $MB_TAG ]]; then
+        echo "release name not provided!"
+        exit 1
+    fi
+
+    echo "Creating app version in EB"
+    aws elasticbeanstalk create-application-version --no-auto-create-application --region us-east-1 --application-name ${EB_APPLICATION} --version-label ${MB_TAG} --source-bundle S3Bucket="${ARTIFACTS_S3BUCKET}",S3Key="eb/${MB_TAG}.zip"
+}
+
+deploy_version() {
+    MB_TAG=$1
+    EB_ENVIRONMENT=$2
+
+    $(which locale) | $(which sort) || true
+
+    if [[ -z $MB_TAG ]]; then
+        echo "release name not provided!"
+        exit 1
+    fi
+
+    if [[ -z $EB_ENVIRONMENT ]]; then
+        echo "beanstalk environment not provided!"
+        exit 1
+    fi
+
+    aws elasticbeanstalk update-environment --region us-east-1 --environment-name ${EB_ENVIRONMENT} --version-label ${MB_TAG}
+}

bin/docker/Dockerfile

+FROM java:openjdk-7-jre
+
+ENV LC_ALL C
+ENV LANG C.UTF-8
+ENV DEBIAN_FRONTEND noninteractive
+ENV DEBCONF_NONINTERACTIVE_SEEN true
+
+# add Metabase jar
+COPY ./metabase.jar /app/
+
+# add our run script to the image
+COPY ./run_metabase.sh /app/
+RUN chmod 755 /app/run_metabase.sh
+
+# expose our default runtime port
+EXPOSE 3000
+
+# run it
+ENTRYPOINT ["/app/run_metabase.sh"]

bin/docker/build_image.sh

+#!/bin/bash
+
+BASEDIR=$(dirname $0)
+PROJECT_ROOT="$BASEDIR/../.."
+
+DOCKERHUB_NAMESPACE=metabase
+
+
+BUILD_TYPE=$1
+if [ -z $BUILD_TYPE ]; then
+    echo "usage: $0 <source|release> <release-name> [--publish]"
+    exit 1
+fi
+
+MB_TAG=$2
+if [ -z $MB_TAG ]; then
+    echo "usage: $0 <source|release> <release-name> [--publish] [--latest]"
+    exit 1
+fi
+
+if [ "$3" == "--publish" ]; then
+    PUBLISH="YES"
+fi
+
+if [ "$4" == "--latest" ]; then
+    LATEST="YES"
+fi
+
+if [ "$PUBLISH" == "YES" ] && [ -z "$DOCKERHUB_EMAIL" -o -z "$DOCKERHUB_USERNAME" -o -z "$DOCKERHUB_PASSWORD" ]; then
+    echo "In order to publish an image to Dockerhub you must set \$DOCKERHUB_EMAIL, \$DOCKERHUB_USERNAME and \$DOCKERHUB_PASSWORD before running."
+    exit 1
+fi
+
+# TODO: verify we have access to docker cmd and minimum version?
+
+
+if [ "$BUILD_TYPE" == "release" ]; then
+    DOCKERHUB_REPOSITORY=metabase
+    DOCKER_IMAGE="${DOCKERHUB_NAMESPACE}/${DOCKERHUB_REPOSITORY}:${MB_TAG}"
+
+    echo "Building Docker image ${DOCKER_IMAGE} from official Metabase release ${MB_TAG}"
+
+    # download the official version of Metabase which matches our tag
+    curl -f -o ${BASEDIR}/metabase.jar http://downloads.metabase.com/${MB_TAG}/metabase.jar
+
+    if [[ $? -ne 0 ]]; then
+        echo "Download failed!"
+        exit 1
+    fi
+else
+    DOCKERHUB_REPOSITORY=metabase-head
+    DOCKER_IMAGE="${DOCKERHUB_NAMESPACE}/${DOCKERHUB_REPOSITORY}:${MB_TAG}"
+
+    echo "Building Docker image ${DOCKER_IMAGE} from local source"
+
+    # trigger a full build
+    ${PROJECT_ROOT}/bin/build
+
+    if [ $? -eq 1 ]; then
+        echo "Build failed!"
+        exit 1
+    fi
+
+    # copy our built uberjar so that we can add it to our image
+    cp ${PROJECT_ROOT}/target/uberjar/metabase.jar ${BASEDIR}/metabase.jar
+fi
+
+
+# now tell docker to build our image
+# TODO: —-no-cache=true
+docker build -t ${DOCKER_IMAGE} $BASEDIR
+
+# TODO: validate our built docker image
+
+
+if [ "$PUBLISH" == "YES" ]; then
+    echo "Publishing image ${DOCKER_IMAGE} to Dockerhub"
+
+    # make sure that we are logged into dockerhub
+    docker login --email="${DOCKERHUB_EMAIL}" --username="${DOCKERHUB_USERNAME}" --password="${DOCKERHUB_PASSWORD}"
+
+    # push the built image to dockerhub
+    docker push ${DOCKER_IMAGE}
+
+    # TODO: quick check against dockerhub to see that our new image made it
+
+    if [ "$LATEST" == "YES" ]; then
+        # tag our recent versioned image as "latest"
+        docker tag -f ${DOCKER_IMAGE} ${DOCKERHUB_NAMESPACE}/${DOCKERHUB_REPOSITORY}:latest
+
+        # then push it as well
+        docker push ${DOCKERHUB_NAMESPACE}/${DOCKERHUB_REPOSITORY}:latest
+
+        # TODO: validate push succeeded
+    fi
+fi
+
+# TODO: cleanup after ourselves and remove the Metabase binary we downloaded
+rm -f ${BASEDIR}/metabase.jar
+
+echo "Done"
+

bin/docker/run_metabase.sh

+#!/bin/bash
+
+# if nobody manually set a host to list on then go with $HOSTNAME
+if [ -z "$MB_JETTY_HOST" ]; then
+    export MB_JETTY_HOST=$HOSTNAME
+fi
+
+
+# Metabase Database Info - this is just about what db the Metabase application uses for internal storage
+
+# AWS Elastic Beanstalk w/ RDS
+if [ ! -z "$RDS_HOSTNAME" ]; then
+    # EEK: this is a bit fragile.  if user picks a non-standard port for their db we are screwed :(
+    if [ "$MB_DB_PORT" == "3306" ]; then
+        export MB_DB_TYPE=mysql
+    else
+        export MB_DB_TYPE=postgres
+    fi
+
+    export MB_DB_DBNAME=$RDS_DB_NAME
+    export MB_DB_USER=$RDS_USERNAME
+    export MB_DB_PASS=$RDS_PASSWORD
+    export MB_DB_HOST=$RDS_HOSTNAME
+    export MB_DB_PORT=$RDS_PORT
+fi
+
+
+# Launch the application
+java -Dlogfile.path=target/log -XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC -jar /app/metabase.jar