Build: Adds e2e tests back to master workflow with better error messages and with artifacts (#17374)

* Refactor: Adds better error message and removes chromium download progess

* Test: Adds e2e tests to pr builds for testing purpose

* Tests: Changes path to screenshots

* Tests: Adds failing test just to test message and artifacts

* Tests: Removes failing test

.circleci/config.yml

@@ -91,6 +91,12 @@ jobs:
               name: run end-to-end tests
               command: 'env BASE_URL=http://127.0.0.1:3000 yarn e2e-tests'
               no_output_timeout: 5m
+          - store_artifacts:
+                path: public/e2e-test/screenShots/theTruth
+                destination: expected-screenshots
+          - store_artifacts:
+                path: public/e2e-test/screenShots/theOutput
+                destination: output-screenshots               
 
   codespell:
     docker:
@@ -662,6 +668,10 @@ workflows:
             - mysql-integration-test
             - postgres-integration-test
           filters: *filter-only-master
+      - end-to-end-test:
+          requires:
+            - grafana-docker-master
+          filters: *filter-only-master
   release:
     jobs:
       - build-all:

public/e2e-test/core/images.ts

@@ -23,8 +23,21 @@ export const compareScreenShots = async (fileName: string) =>
         return;
       }
 
-      expect(screenShotFromTest.width).toEqual(screenShotFromTruth.width);
-      expect(screenShotFromTest.height).toEqual(screenShotFromTruth.height);
+      if (screenShotFromTest.width !== screenShotFromTruth.width) {
+        throw new Error(
+          `The screenshot:[${fileName}] taken during the test has a width:[${
+            screenShotFromTest.width
+          }] that differs from the expected: [${screenShotFromTruth.width}].`
+        );
+      }
+
+      if (screenShotFromTest.height !== screenShotFromTruth.height) {
+        throw new Error(
+          `The screenshot:[${fileName}] taken during the test has a width:[${
+            screenShotFromTest.height
+          }] that differs from the expected: [${screenShotFromTruth.height}].`
+        );
+      }
 
       const diff = new PNG({ width: screenShotFromTest.width, height: screenShotFromTruth.height });
       const numDiffPixels = pixelmatch(
@@ -36,7 +49,27 @@ export const compareScreenShots = async (fileName: string) =>
         { threshold: 0.1 }
       );
 
-      expect(numDiffPixels).toBe(0);
+      if (numDiffPixels !== 0) {
+        const localMessage = `\nCompare the output from expected:[${constants.screenShotsTruthDir}] with outcome:[${
+          constants.screenShotsOutputDir
+        }]`;
+        const circleCIMessage = '\nCheck the Artifacts tab in the CircleCi build output for the actual screenshots.';
+        const checkMessage = process.env.CIRCLE_SHA1 ? circleCIMessage : localMessage;
+        let msg = `\nThe screenshot:[${
+          constants.screenShotsOutputDir
+        }/${fileName}.png] taken during the test differs by:[${numDiffPixels}] pixels from the expected.`;
+        msg += '\n';
+        msg += checkMessage;
+        msg += '\n';
+        msg += '\n  If the difference between expected and outcome is NOT acceptable then do the following:';
+        msg += '\n    - Check the code for changes that causes this difference, fix that and retry.';
+        msg += '\n';
+        msg += '\n  If the difference between expected and outcome is acceptable then do the following:';
+        msg += '\n    - Replace the expected image with the outcome and retry.';
+        msg += '\n';
+        throw new Error(msg);
+      }
+
       resolve();
     };
 

public/e2e-test/install/install.ts

@@ -11,7 +11,9 @@ export const downloadBrowserIfNeeded = async (): Promise<void> => {
 
   console.log('Did not find any local revisions for browser, downloading latest this might take a while.');
   await browserFetcher.download(constants.chromiumRevision, (downloaded, total) => {
-    console.log(`Downloaded ${downloaded}bytes of ${total}bytes.`);
+    if (downloaded === total) {
+      console.log('Chromium successfully downloaded');
+    }
   });
 };