area/circleci: Speed up circleci build process for branches and pr (#16778)

* speed up circleci build
* convert build.sh to functions and add args for different build types
* add yarn cache
* add build latest to packaging step
* fast build should not try to tag non-amd builds
* remove signing step for prs-and-branches

.circleci/config.yml

@@ -198,6 +198,117 @@ jobs:
           paths:
             - dist/*
 
+  build-fast-backend:
+    docker:
+     - image: grafana/build-container:1.2.6
+    working_directory: /go/src/github.com/grafana/grafana
+    steps:
+      - checkout
+      - run:
+          name: prepare build tools
+          command: '/tmp/bootstrap.sh'
+      - run:
+          name: build grafana backend
+          command: './scripts/build/build.sh --fast --backend-only'
+      - persist_to_workspace:
+          root: .
+          paths:
+            - bin/*
+
+  build-fast-frontend:
+    docker:
+     - image: grafana/build-container:1.2.6
+    working_directory: /go/src/github.com/grafana/grafana
+    steps:
+      - checkout
+      - run:
+          name: prepare build tools
+          command: '/tmp/bootstrap.sh'
+      - restore_cache:
+          key: frontend-dependency-cache-{{ checksum "yarn.lock" }}
+      - run:
+          name: build grafana frontend
+          command: './scripts/build/build.sh --fast --frontend-only'
+      - save_cache:
+          key: frontend-dependency-cache-{{ checksum "yarn.lock" }}
+          paths:
+            - node_modules
+      - persist_to_workspace:
+          root: .
+          paths:
+            - public/build/*
+            - tools/phantomjs/*
+
+  build-fast-package:
+    docker:
+     - image: grafana/build-container:1.2.6
+    working_directory: /go/src/github.com/grafana/grafana
+    steps:
+      - checkout
+      - attach_workspace:
+          at: .
+      - restore_cache:
+          key: frontend-dependency-cache-{{ checksum "yarn.lock" }}
+      - run:
+          name: prepare build tools
+          command: '/tmp/bootstrap.sh'
+      - run:
+          name: package grafana
+          command: './scripts/build/build.sh --fast --package-only'
+      - run:
+          name: sha-sum packages
+          command: 'go run build.go sha-dist'
+      - run:
+          name: Test Grafana.com release publisher
+          command: 'cd scripts/build/release_publisher && go test .'
+      - persist_to_workspace:
+          root: /go/src/github.com/grafana/grafana
+          paths:
+            - dist/*
+
+  build-fast-save:
+    docker:
+     - image: grafana/build-container:1.2.6
+    working_directory: /go/src/github.com/grafana/grafana
+    steps:
+      - checkout
+      - attach_workspace:
+          at: .
+      - restore_cache:
+          key: dependency-cache-{{ checksum "yarn.lock" }}
+      - run:
+          name: debug cache
+          command: 'ls -al /go/src/github.com/grafana/grafana/node_modules'
+      - run:
+          name: prepare build tools
+          command: '/tmp/bootstrap.sh'
+      - run:
+          name: build grafana backend
+          command: './scripts/build/build.sh --fast --backend-only'
+      - run:
+          name: build grafana frontend
+          command: './scripts/build/build.sh --fast --frontend-only'
+      - save_cache:
+          key: dependency-cache-{{ checksum "yarn.lock" }}
+          paths:
+            - /go/src/github.com/grafana/grafana/node_modules
+      - run:
+          name: package grafana
+          command: './scripts/build/build.sh --fast --package-only'
+      - run:
+          name: sign packages
+          command: './scripts/build/sign_packages.sh'
+      - run:
+          name: sha-sum packages
+          command: 'go run build.go sha-dist'
+      - run:
+          name: Test Grafana.com release publisher
+          command: 'cd scripts/build/release_publisher && go test .'
+      - persist_to_workspace:
+          root: .
+          paths:
+            - dist/*
+
   grafana-docker-master:
     machine:
       image: circleci/classic:201808-01
@@ -224,7 +335,7 @@ jobs:
       - run: docker info
       - run: docker run --privileged linuxkit/binfmt:v0.6
       - run: cp dist/grafana-latest.linux-*.tar.gz packaging/docker
-      - run: cd packaging/docker && ./build.sh "${CIRCLE_SHA1}"
+      - run: cd packaging/docker && ./build.sh --fast "${CIRCLE_SHA1}"
 
   grafana-docker-release:
     machine:
@@ -556,8 +667,15 @@ workflows:
 
   build-branches-and-prs:
       jobs:
-        - build:
+        - build-fast-backend:
+            filters: *filter-not-release-or-master
+        - build-fast-frontend:
             filters: *filter-not-release-or-master
+        - build-fast-package:
+            filters: *filter-not-release-or-master
+            requires:
+              - build-fast-backend
+              - build-fast-frontend
         - codespell:
             filters: *filter-not-release-or-master
         - backend-lint:
@@ -574,7 +692,7 @@ workflows:
             filters: *filter-not-release-or-master
         - grafana-docker-pr:
             requires:
-              - build
+              - build-fast-package
               - test-backend
               - test-frontend
               - codespell
@@ -585,7 +703,7 @@ workflows:
             filters: *filter-not-release-or-master
         - store-build-artifacts:
             requires:
-              - build
+              - build-fast-package
               - test-backend
               - test-frontend
               - codespell

packaging/docker/Dockerfile

@@ -17,6 +17,7 @@ FROM ${BASE_IMAGE}
 
 ARG GF_UID="472"
 ARG GF_GID="472"
+ARG DEBIAN_FRONTEND=noninteractive
 
 ENV PATH=/usr/share/grafana/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
     GF_PATHS_CONFIG="/etc/grafana/grafana.ini" \

packaging/docker/build.sh

@@ -1,4 +1,19 @@
 #!/bin/sh
+BUILD_FAST=0
+
+while [ "$1" != "" ]; do
+  case "$1" in
+    "--fast")
+      BUILD_FAST=1
+      echo "Fast build enabled"
+      shift
+      ;;
+    * )
+      # unknown param causes args to be passed through to $@
+      break
+      ;;
+  esac
+done
 
 _grafana_tag=${1:-}
 _docker_repo=${2:-grafana/grafana}
@@ -27,19 +42,28 @@ docker_build () {
 		--no-cache=true .
 }
 
+docker_tag_linux_amd64 () {
+	repo=$1
+	tag=$2
+	docker tag "${_docker_repo}:${_grafana_version}" "${repo}:${tag}"
+}
+
 # Tag docker images of all architectures
 docker_tag_all () {
 	repo=$1
 	tag=$2
-	docker tag "${_docker_repo}:${_grafana_version}" "${repo}:${tag}"
-	docker tag "${_docker_repo}-arm32v7-linux:${_grafana_version}" "${repo}-arm32v7-linux:${tag}"
-	docker tag "${_docker_repo}-arm64v8-linux:${_grafana_version}" "${repo}-arm64v8-linux:${tag}"
+	docker_tag_linux_amd64 $1 $2
+	if [ $BUILD_FAST = "0" ]; then
+		docker tag "${_docker_repo}-arm32v7-linux:${_grafana_version}" "${repo}-arm32v7-linux:${tag}"
+		docker tag "${_docker_repo}-arm64v8-linux:${_grafana_version}" "${repo}-arm64v8-linux:${tag}"
+	fi
 }
 
 docker_build "debian:stretch-slim" "grafana-latest.linux-x64.tar.gz" "${_docker_repo}:${_grafana_version}"
-docker_build "arm32v7/debian:stretch-slim" "grafana-latest.linux-armv7.tar.gz" "${_docker_repo}-arm32v7-linux:${_grafana_version}"
-docker_build "arm64v8/debian:stretch-slim" "grafana-latest.linux-arm64.tar.gz" "${_docker_repo}-arm64v8-linux:${_grafana_version}"
-
+if [ $BUILD_FAST = "0" ]; then
+	docker_build "arm32v7/debian:stretch-slim" "grafana-latest.linux-armv7.tar.gz" "${_docker_repo}-arm32v7-linux:${_grafana_version}"
+	docker_build "arm64v8/debian:stretch-slim" "grafana-latest.linux-arm64.tar.gz" "${_docker_repo}-arm64v8-linux:${_grafana_version}"
+fi
 # Tag as 'latest' for official release; otherwise tag as grafana/grafana:master
 if echo "$_grafana_tag" | grep -q "^v"; then
 	docker_tag_all "${_docker_repo}" "latest"

scripts/build/build.sh

@@ -1,20 +1,58 @@
 #!/bin/bash
-
 #
 #   This script is executed from within the container.
 #
-
 set -e
 
-EXTRA_OPTS="$@"
-
+##########
 CCARMV6=/opt/rpi-tools/arm-bcm2708/arm-linux-gnueabihf/bin/arm-linux-gnueabihf-gcc
 CCARMV7=arm-linux-gnueabihf-gcc
 CCARM64=aarch64-linux-gnu-gcc
 CCX64=/tmp/x86_64-centos6-linux-gnu/bin/x86_64-centos6-linux-gnu-gcc
-
+##########
 GOPATH=/go
 REPO_PATH=$GOPATH/src/github.com/grafana/grafana
+##########
+
+BUILD_FAST=0
+BUILD_BACKEND=1
+BUILD_FRONTEND=1
+BUILD_PACKAGE=1
+
+while [ "$1" != "" ]; do
+  case "$1" in
+    "--fast")
+      BUILD_FAST=1
+      echo "Fast build enabled"
+      shift
+      ;;
+    "--backend-only")
+      BUILD_FRONTEND=0
+      BUILD_PACKAGE=0
+      echo "Building only backend"
+      shift
+      ;;
+    "--frontend-only")
+      BUILD_BACKEND=0
+      BUILD_PACKAGE=0
+      echo "Building only frontend"
+      shift
+      ;;
+    "--package-only")
+      BUILD_BACKEND=0
+      BUILD_FRONTEND=0
+      echo "Building only packaging"
+      shift
+      ;;
+    * )
+      # unknown param causes args to be passed through to $@
+      break
+      ;;
+  esac
+done
+
+EXTRA_OPTS="$@"
+
 
 cd /go/src/github.com/grafana/grafana
 echo "current dir: $(pwd)"
@@ -31,31 +69,77 @@ fi
 echo "Build arguments: $OPT"
 echo "current dir: $(pwd)"
 
-go run build.go -goarch armv6 -cc ${CCARMV6} ${OPT} build
-go run build.go -goarch armv7 -cc ${CCARMV7} ${OPT} build
-go run build.go -goarch arm64 -cc ${CCARM64} ${OPT} build
-
-CC=${CCX64} go run build.go ${OPT} build
-
-yarn install --pure-lockfile --no-progress
-
-echo "Building frontend"
-go run build.go ${OPT} build-frontend
-
-if [ -d "dist" ]; then
-  rm -rf dist
+function build_backend_linux_amd64() {
+  if [ ! -d "dist" ]; then
+    mkdir dist
+  fi
+  CC=${CCX64} go run build.go ${OPT} build
+}
+
+function build_backend() {
+  if [ ! -d "dist" ]; then
+    mkdir dist
+  fi
+
+  go run build.go -goarch armv6 -cc ${CCARMV6} ${OPT} build
+  go run build.go -goarch armv7 -cc ${CCARMV7} ${OPT} build
+  go run build.go -goarch arm64 -cc ${CCARM64} ${OPT} build
+  build_backend_linux_amd64
+}
+
+function build_frontend() {
+  if [ ! -d "dist" ]; then
+    mkdir dist
+  fi
+  yarn install --pure-lockfile --no-progress
+  echo "Building frontend"
+  go run build.go ${OPT} build-frontend
+  echo "FRONTEND: finished"
+}
+
+function package_linux_amd64() {
+  echo "Packaging Linux AMD64"
+  go run build.go -goos linux -pkg-arch amd64 ${OPT} package-only
+  go run build.go latest
+  echo "PACKAGE LINUX AMD64: finished"
+}
+
+function package_all() {
+  echo "Packaging ALL"
+  go run build.go -goos linux -pkg-arch armv6 ${OPT} -skipRpm package-only
+  go run build.go -goos linux -pkg-arch armv7 ${OPT} package-only
+  go run build.go -goos linux -pkg-arch arm64 ${OPT} package-only
+  package_linux_amd64
+  echo "PACKAGE ALL: finished"
+}
+
+function package_setup() {
+  echo "Packaging: Setup environment"
+  if [ -d "dist" ]; then
+    rm -rf dist
+  fi
+  mkdir dist
+  go run build.go -gen-version ${OPT} > dist/grafana.version
+  # Load ruby, needed for packing with fpm
+  source /etc/profile.d/rvm.sh
+}
+
+if [ $BUILD_FAST = "0" ]; then
+  build_backend
+  build_frontend
+  package_setup
+  package_all
+else
+  if [ $BUILD_BACKEND = "1" ]; then
+    build_backend_linux_amd64
+  fi
+  if [ $BUILD_FRONTEND = "1" ]; then
+    build_frontend
+  fi
+  if [ $BUILD_PACKAGE = "1" ]; then
+    package_setup
+    package_linux_amd64
+    # last step
+    #go run build.go latest
+  fi
 fi
-
-mkdir dist
-go run build.go -gen-version ${OPT} > dist/grafana.version
-
-# Load ruby, needed for packing with fpm
-source /etc/profile.d/rvm.sh
-
-echo "Packaging"
-go run build.go -goos linux -pkg-arch amd64 ${OPT} package-only
-go run build.go -goos linux -pkg-arch armv6 ${OPT} -skipRpm package-only
-go run build.go -goos linux -pkg-arch armv7 ${OPT} package-only
-go run build.go -goos linux -pkg-arch arm64 ${OPT} package-only
-
-go run build.go latest