Merge pull request #14924 from xlson/repo-update-improvements

Repo update stability improvements

.circleci/config.yml

@@ -323,7 +323,7 @@ jobs:
 
   deploy-enterprise-master:
     docker:
-      - image: grafana/grafana-ci-deploy:1.1.0
+      - image: grafana/grafana-ci-deploy:1.2.0
     steps:
       - attach_workspace:
           at: .
@@ -346,7 +346,7 @@ jobs:
 
   deploy-enterprise-release:
     docker:
-    - image: grafana/grafana-ci-deploy:1.1.0
+    - image: grafana/grafana-ci-deploy:1.2.0
     steps:
       - attach_workspace:
          at: .
@@ -370,15 +370,15 @@ jobs:
           command: './scripts/build/load-signing-key.sh'
       - run:
           name: Update Debian repository
-          command: './scripts/build/update_repo/update-deb.sh "enterprise" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG"'
+          command: './scripts/build/update_repo/update-deb.sh "enterprise" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG" "enterprise-dist"'
       - run:
           name: Update RPM repository
-          command: './scripts/build/update_repo/update-rpm.sh "enterprise" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG"'
+          command: './scripts/build/update_repo/update-rpm.sh "enterprise" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG" "enterprise-dist"'
 
 
   deploy-master:
     docker:
-      - image: grafana/grafana-ci-deploy:1.1.0
+      - image: grafana/grafana-ci-deploy:1.2.0
     steps:
       - attach_workspace:
           at: .
@@ -408,7 +408,7 @@ jobs:
 
   deploy-release:
     docker:
-      - image: grafana/grafana-ci-deploy:1.1.0
+      - image: grafana/grafana-ci-deploy:1.2.0
     steps:
       - checkout
       - attach_workspace:
@@ -433,10 +433,10 @@ jobs:
           command: './scripts/build/load-signing-key.sh'
       - run:
           name: Update Debian repository
-          command: './scripts/build/update_repo/update-deb.sh "oss" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG"'
+          command: './scripts/build/update_repo/update-deb.sh "oss" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG" "dist"'
       - run:
           name: Update RPM repository
-          command: './scripts/build/update_repo/update-rpm.sh "oss" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG"'
+          command: './scripts/build/update_repo/update-rpm.sh "oss" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG" "dist"'
 
 workflows:
   version: 2

scripts/build/build-all.sh

@@ -30,10 +30,12 @@ fi
 
 echo "Build arguments: $OPT"
 
-go run build.go -goarch armv7 -cc ${CCARMV7} ${OPT} build
-go run build.go -goarch arm64 -cc ${CCARM64} ${OPT} build
-go run build.go -goos darwin -cc ${CCOSX64} ${OPT} build
-
+# build only amd64 for enterprise
+if echo "$EXTRA_OPTS" | grep -vq enterprise ; then
+  go run build.go -goarch armv7 -cc ${CCARMV7} ${OPT} build
+  go run build.go -goarch arm64 -cc ${CCARM64} ${OPT} build
+  go run build.go -goos darwin -cc ${CCOSX64} ${OPT} build
+fi
 go run build.go -goos windows -cc ${CCWIN64} ${OPT} build
 
 # Do not remove CC from the linux build, its there for compatibility with Centos6
@@ -62,22 +64,26 @@ echo "Packaging"
 go run build.go -goos linux -pkg-arch amd64 ${OPT} package-only
 #removing amd64 phantomjs bin for armv7/arm64 packages
 rm tools/phantomjs/phantomjs
-go run build.go -goos linux -pkg-arch armv7 ${OPT} package-only
-go run build.go -goos linux -pkg-arch arm64 ${OPT} package-only
 
-if [ -d '/tmp/phantomjs/darwin' ]; then
-  cp /tmp/phantomjs/darwin/phantomjs tools/phantomjs/phantomjs
-else
-  echo 'PhantomJS binaries for darwin missing!'
+# build only amd64 for enterprise
+if echo "$EXTRA_OPTS" | grep -vq enterprise ; then
+  go run build.go -goos linux -pkg-arch armv7 ${OPT} package-only
+  go run build.go -goos linux -pkg-arch arm64 ${OPT} package-only
+
+  if [ -d '/tmp/phantomjs/darwin' ]; then
+    cp /tmp/phantomjs/darwin/phantomjs tools/phantomjs/phantomjs
+  else
+    echo 'PhantomJS binaries for darwin missing!'
+  fi
+  go run build.go -goos darwin -pkg-arch amd64 ${OPT} package-only
 fi
-go run build.go -goos darwin -pkg-arch amd64 ${OPT} package-only
 
 if [ -d '/tmp/phantomjs/windows' ]; then
   cp /tmp/phantomjs/windows/phantomjs.exe tools/phantomjs/phantomjs.exe
-  rm tools/phantomjs/phantomjs
+  rm tools/phantomjs/phantomjs || true
 else
     echo 'PhantomJS binaries for Windows missing!'
 fi
 go run build.go -goos windows -pkg-arch amd64 ${OPT} package-only
 
-go run build.go latest
+go run build.go latest

scripts/build/update_repo/README.md

@@ -0,0 +1,28 @@
+# Repository updates deb/rpm
+
+
+
+## Testing
+
+It's possible to test the repo updates for rpm and deb by running the test scripts within a docker container like this. Tests are being executed by using two buckets on gcp setup for testing.
+
+```bash
+docker run -ti --rm -u 0:0 grafana/grafana-ci-deploy:1.2.0 bash # 1.2.0 is the newest image at the time of writing
+# in the container:
+mkdir -p /go/src/github.com/grafana/dist
+cd /go/src/github.com/grafana
+
+#outside of container:
+cd <grafana project dir>/..
+docker cp grafana <container_name>:/go/src/github.com/grafana/.
+docker cp <gpg.key used for signing> <container_name>:/private.key
+
+#in container:
+gpg --batch --allow-secret-key-import --import /private.key
+cd dist && wget https://dl.grafana.com/oss/release/grafana_5.4.3_amd64.deb && wget https://dl.grafana.com/oss/release/grafana-5.4.3-1.x86_64.rpm && cd ..
+
+#run these scripts:
+./script/build/update_repo/test-update-deb-repo.sh <gpg key password>
+./script/build/update_repo/test-update-rpm-repo.sh <gpg key password>
+
+```

scripts/build/update_repo/init-deb-repo.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+# Run this if you need to recreate the debian repository for some reason
+
+# Setup environment
+cp scripts/build/update_repo/aptly.conf /etc/aptly.conf
+mkdir -p /deb-repo/db   \
+         /deb-repo/repo \
+         /deb-repo/tmp
+
+aptly repo create -distribution=stable -component=main grafana
+aptly repo create -distribution=beta -component=main beta

scripts/build/update_repo/test-update-deb-repo.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+GPG_PASS=${1:-}
+
+./scripts/build/update_repo/update-deb.sh "oss" "$GPG_PASS" "v5.4.3" "dist" "grafana-testing-aptly-db" "grafana-testing-repo"

scripts/build/update_repo/test-update-rpm-repo.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+GPG_PASS=${1:-}
+
+./scripts/build/update_repo/update-rpm.sh "oss" "$GPG_PASS" "v5.4.3" "dist" "grafana-testing-repo"

scripts/build/update_repo/update-deb.sh

@@ -3,10 +3,14 @@
 RELEASE_TYPE="${1:-}"
 GPG_PASS="${2:-}"
 RELEASE_TAG="${3:-}"
+DIST_PATH="${4:-}"
+GCP_DB_BUCKET="${5:-grafana-aptly-db}"
+GCP_REPO_BUCKET="${6:-grafana-repo}"
+
 REPO="grafana"
 
-if [ -z "$RELEASE_TYPE" -o -z "$GPG_PASS" ]; then
-    echo "Both RELEASE_TYPE (arg 1) and GPG_PASS (arg 2) has to be set"
+if [ -z "$RELEASE_TYPE" -o -z "$GPG_PASS" -o -z "$DIST_PATH" ]; then
+    echo "Both RELEASE_TYPE (arg 1), GPG_PASS (arg 2) and DIST_PATH (arg 4) has to be set"
     exit 1
 fi
 
@@ -22,36 +26,42 @@ fi
 set -e
 
 # Setup environment
-cp scripts/build/update_repo/aptly.conf /etc/aptly.conf
+cp scripts/build/update_repo/aptly.conf ~/.aptly.conf
+
 mkdir -p /deb-repo/db   \
          /deb-repo/repo \
          /deb-repo/tmp
 
 # Download the database
-gsutil -m rsync -r "gs://grafana-aptly-db/$RELEASE_TYPE" /deb-repo/db
+gsutil -m rsync -r -d "gs://$GCP_DB_BUCKET/$RELEASE_TYPE" /deb-repo/db
 
 # Add the new release to the repo
-aptly publish drop grafana filesystem:repo:grafana || true
-aptly publish drop beta filesystem:repo:grafana || true
-cp ./dist/*.deb /deb-repo/tmp
+cp $DIST_PATH/*.deb /deb-repo/tmp
 rm /deb-repo/tmp/grafana_latest*.deb || true
-aptly repo add "$REPO" ./dist
+aptly repo add "$REPO" /deb-repo/tmp #adds too many packages in enterprise
 
 # Setup signing and sign the repo
 
 echo "allow-loopback-pinentry" > ~/.gnupg/gpg-agent.conf
 echo "pinentry-mode loopback" > ~/.gnupg/gpg.conf
 
+pkill gpg-agent || true
 touch /tmp/sign-this
+rm /tmp/sign-this.asc || true
 ./scripts/build/update_repo/unlock-gpg-key.sh "$GPG_PASS"
 rm /tmp/sign-this /tmp/sign-this.asc
 
-aptly publish repo grafana filesystem:repo:grafana
-aptly publish repo beta filesystem:repo:grafana
+aptly publish update stable filesystem:repo:grafana
+aptly publish update beta filesystem:repo:grafana
 
 # Update the repo and db on gcp
-gsutil -m rsync -r -d /deb-repo/db "gs://grafana-aptly-db/$RELEASE_TYPE"
-gsutil -m rsync -r -d /deb-repo/repo/grafana "gs://grafana-repo/$RELEASE_TYPE/deb"
+
+gsutil -m rsync -r -d /deb-repo/db "gs://$GCP_DB_BUCKET/$RELEASE_TYPE"
+
+# Uploads the binaries before the metadata (to prevent 404's for debs)
+gsutil -m rsync -r /deb-repo/repo/grafana/pool "gs://$GCP_REPO_BUCKET/$RELEASE_TYPE/deb/pool"
+
+gsutil -m rsync -r -d /deb-repo/repo/grafana "gs://$GCP_REPO_BUCKET/$RELEASE_TYPE/deb"
 
 # usage:
 # 

scripts/build/update_repo/update-rpm.sh

@@ -2,12 +2,14 @@
 
 RELEASE_TYPE="${1:-}"
 GPG_PASS="${2:-}"
-
 RELEASE_TAG="${3:-}"
+DIST_PATH="${4:-}"
+GCP_REPO_BUCKET="${5:-grafana-repo}"
+
 REPO="rpm"
 
-if [ -z "$RELEASE_TYPE" -o -z "$GPG_PASS" ]; then
-    echo "Both RELEASE_TYPE (arg 1) and GPG_PASS (arg 2) has to be set"
+if [ -z "$RELEASE_TYPE" -o -z "$GPG_PASS" -o -z "$DIST_PATH" ]; then
+    echo "Both RELEASE_TYPE (arg 1), GPG_PASS (arg 2) and DIST_PATH (arg 4) has to be set"
     exit 1
 fi
 
@@ -23,17 +25,17 @@ fi
 set -e
 
 # Setup environment
-BUCKET="gs://grafana-repo/$RELEASE_TYPE/$REPO"
+BUCKET="gs://$GCP_REPO_BUCKET/$RELEASE_TYPE/$REPO"
+
 mkdir -p /rpm-repo
 
 # Download the database
 gsutil -m rsync -r "$BUCKET" /rpm-repo
 
 # Add the new release to the repo
-cp ./dist/*.rpm /rpm-repo
+cp $DIST_PATH/*.rpm /rpm-repo # adds to many files for enterprise
 rm /rpm-repo/grafana-latest-1*.rpm || true
-cd /rpm-repo
-createrepo .
+createrepo /rpm-repo
 
 # Setup signing and sign the repo
 
@@ -45,6 +47,7 @@ pkill gpg-agent || true
 ./scripts/build/update_repo/sign-rpm-repo.sh "$GPG_PASS"
 
 # Update the repo and db on gcp
+gsutil -m cp /rpm-repo/*.rpm "$BUCKET" # sync binaries first to avoid cache misses
 gsutil -m rsync -r -d /rpm-repo "$BUCKET"
 
 # usage:
@@ -56,4 +59,4 @@ gsutil -m rsync -r -d /rpm-repo "$BUCKET"
 # gpgcheck=1
 # gpgkey=https://packages.grafana.com/gpg.key
 # sslverify=1
-# sslcacert=/etc/pki/tls/certs/ca-bundle.crt
+# sslcacert=/etc/pki/tls/certs/ca-bundle.crt