build: repo update testable and more robus.

- adds script for integration testing
- package path parameterized
- more robust updates

.circleci/config.yml

@@ -370,10 +370,10 @@ jobs:
           command: './scripts/build/load-signing-key.sh'
       - run:
           name: Update Debian repository
-          command: './scripts/build/update_repo/update-deb.sh "enterprise" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG"'
+          command: './scripts/build/update_repo/update-deb.sh "enterprise" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG" "enterprise-dist"'
       - run:
           name: Update RPM repository
-          command: './scripts/build/update_repo/update-rpm.sh "enterprise" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG"'
+          command: './scripts/build/update_repo/update-rpm.sh "enterprise" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG" "enterprise-dist"'
 
 
   deploy-master:
@@ -433,10 +433,10 @@ jobs:
           command: './scripts/build/load-signing-key.sh'
       - run:
           name: Update Debian repository
-          command: './scripts/build/update_repo/update-deb.sh "oss" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG"'
+          command: './scripts/build/update_repo/update-deb.sh "oss" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG" "dist"'
       - run:
           name: Update RPM repository
-          command: './scripts/build/update_repo/update-rpm.sh "oss" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG"'
+          command: './scripts/build/update_repo/update-rpm.sh "oss" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG" "dist"'
 
 workflows:
   version: 2

scripts/build/update_repo/init-deb-repo.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+# Run this if you need to recreate the debian repository for some reason
+
+# Setup environment
+cp scripts/build/update_repo/aptly.conf /etc/aptly.conf
+mkdir -p /deb-repo/db   \
+         /deb-repo/repo \
+         /deb-repo/tmp
+
+aptly repo create -distribution=stable -component=main grafana
+aptly repo create -distribution=beta -component=main beta

scripts/build/update_repo/test-update-deb-repo.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+GPG_PASS=${1:-}
+
+./scripts/build/update_repo/update-deb.sh "oss" "$GPG_PASS" "v5.4.3" "dist" "grafana-testing-aptly-db" "grafana-testing-repo"

scripts/build/update_repo/update-deb.sh

@@ -3,10 +3,14 @@
 RELEASE_TYPE="${1:-}"
 GPG_PASS="${2:-}"
 RELEASE_TAG="${3:-}"
+DIST_PATH="${4:-}"
+GCP_DB_BUCKET="${5:-grafana-aptly-db}"
+GCP_REPO_BUCKET="${6:-grafana-repo}"
+
 REPO="grafana"
 
-if [ -z "$RELEASE_TYPE" -o -z "$GPG_PASS" ]; then
-    echo "Both RELEASE_TYPE (arg 1) and GPG_PASS (arg 2) has to be set"
+if [ -z "$RELEASE_TYPE" -o -z "$GPG_PASS" -o -z "$DIST_PATH" ]; then
+    echo "Both RELEASE_TYPE (arg 1), GPG_PASS (arg 2) and DIST_PATH (arg 4) has to be set"
     exit 1
 fi
 
@@ -28,30 +32,32 @@ mkdir -p /deb-repo/db   \
          /deb-repo/tmp
 
 # Download the database
-gsutil -m rsync -r "gs://grafana-aptly-db/$RELEASE_TYPE" /deb-repo/db
+gsutil -m rsync -r -d "gs://$GCP_DB_BUCKET/$RELEASE_TYPE" /deb-repo/db
 
 # Add the new release to the repo
-aptly publish drop grafana filesystem:repo:grafana || true
-aptly publish drop beta filesystem:repo:grafana || true
-cp ./dist/*.deb /deb-repo/tmp
+cp $DIST_PATH/*.deb /deb-repo/tmp
 rm /deb-repo/tmp/grafana_latest*.deb || true
-aptly repo add "$REPO" ./dist
+aptly repo add "$REPO" /deb-repo/tmp #adds too many packages in enterprise
 
 # Setup signing and sign the repo
 
 echo "allow-loopback-pinentry" > ~/.gnupg/gpg-agent.conf
 echo "pinentry-mode loopback" > ~/.gnupg/gpg.conf
 
+pkill gpg-agent || true
 touch /tmp/sign-this
+rm /tmp/sign-this.asc || true
 ./scripts/build/update_repo/unlock-gpg-key.sh "$GPG_PASS"
 rm /tmp/sign-this /tmp/sign-this.asc
 
-aptly publish repo grafana filesystem:repo:grafana
-aptly publish repo beta filesystem:repo:grafana
+aptly publish update stable filesystem:repo:grafana
+aptly publish update beta filesystem:repo:grafana
 
 # Update the repo and db on gcp
-gsutil -m rsync -r -d /deb-repo/db "gs://grafana-aptly-db/$RELEASE_TYPE"
-gsutil -m rsync -r -d /deb-repo/repo/grafana "gs://grafana-repo/$RELEASE_TYPE/deb"
+## TODO: need to update this to push the binaries first and then the metadata so that we dont cache the binaries missing.
+
+gsutil -m rsync -r -d /deb-repo/db "gs://$GCP_DB_BUCKET/$RELEASE_TYPE"
+gsutil -m rsync -r -d /deb-repo/repo/grafana "gs://$GCP_REPO_BUCKET/$RELEASE_TYPE/deb"
 
 # usage:
 # 

scripts/build/update_repo/update-rpm.sh

@@ -2,12 +2,13 @@
 
 RELEASE_TYPE="${1:-}"
 GPG_PASS="${2:-}"
-
 RELEASE_TAG="${3:-}"
+DIST_PATH="${4:-}"
+
 REPO="rpm"
 
-if [ -z "$RELEASE_TYPE" -o -z "$GPG_PASS" ]; then
-    echo "Both RELEASE_TYPE (arg 1) and GPG_PASS (arg 2) has to be set"
+if [ -z "$RELEASE_TYPE" -o -z "$GPG_PASS" -o -z "$DIST_PATH" ]; then
+    echo "Both RELEASE_TYPE (arg 1), GPG_PASS (arg 2) and DIST_PATH (arg 4) has to be set"
     exit 1
 fi
 
@@ -30,10 +31,11 @@ mkdir -p /rpm-repo
 gsutil -m rsync -r "$BUCKET" /rpm-repo
 
 # Add the new release to the repo
-cp ./dist/*.rpm /rpm-repo
+cp $DIST_PATH/*.rpm /rpm-repo # adds to many files for enterprise
 rm /rpm-repo/grafana-latest-1*.rpm || true
 cd /rpm-repo
 createrepo .
+cd /go/src/github.com/grafana/grafana
 
 # Setup signing and sign the repo
 
@@ -56,4 +58,4 @@ gsutil -m rsync -r -d /rpm-repo "$BUCKET"
 # gpgcheck=1
 # gpgkey=https://packages.grafana.com/gpg.key
 # sslverify=1
-# sslcacert=/etc/pki/tls/certs/ca-bundle.crt
+# sslcacert=/etc/pki/tls/certs/ca-bundle.crt