Add google cloud storage

Author: tinayuangao

package.json

@@ -31,7 +31,10 @@
     "@angular/forms": "^2.2.0",
     "@angular/http": "^2.2.0",
     "@angular/platform-browser": "^2.2.0",
+    "@types/es6-promise": "0.0.32",
     "core-js": "^2.4.1",
+    "google-cloud": "^0.45.1",
+    "image-diff": "^1.6.3",
     "rxjs": "5.0.0-beta.12",
     "systemjs": "0.19.38",
     "zone.js": "^0.6.23"

tools/gulp/gulpfile.ts

@@ -12,3 +12,4 @@ import './tasks/unit-test';
 import './tasks/docs';
 import './tasks/aot';
 import './tasks/payload';
+import './tasks/screenshots';

tools/gulp/task_helpers.ts

@@ -17,6 +17,7 @@ const gulpAutoprefixer = require('gulp-autoprefixer');
 const gulpConnect = require('gulp-connect');
 const resolveBin = require('resolve-bin');
 const firebaseAdmin = require('firebase-admin');
+const gcloud = require('google-cloud');
 
 
 /** If the string passed in is a glob, returns it, otherwise append '**\/*' to it. */
@@ -233,3 +234,36 @@ export function openFirebaseDatabase() {
 export function isTravisPushBuild() {
   return process.env['TRAVIS_PULL_REQUEST'] === 'false';
 }
+
+/** Open Google Cloud Storage for screenshots */
+export function openScreenshotsCloudStorage() {
+  // Enable Storage
+  let gcs = gcloud.storage({
+    projectId: 'material2-screenshots',
+    credentials: {
+      client_email: 'firebase-adminsdk-t4209@material2-screenshots.iam.gserviceaccount.com',
+      private_key: (process.env['MATERIAL2_SCREENSHOT_FIREBASE_KEY'] || '').replace(/\\n/g, '\n')
+    },
+  });
+
+  // Reference an existing bucket.
+  return gcs.bucket('material2-screenshots.appspot.com');
+}
+
+/** Opens a connection to the firebase realtime database for screenshots. */
+export function openScreenshotsFirebaseDatabase() {
+  // Initialize the Firebase application with admin credentials.
+  // Credentials need to be for a Service Account, which can be created in the Firebase console.
+  let screenshotApp = firebaseAdmin.initializeApp({
+    credential: firebaseAdmin.credential.cert({
+      project_id: 'material2-screenshots',
+      client_email: 'firebase-adminsdk-t4209@material2-screenshots.iam.gserviceaccount.com',
+      // In Travis CI the private key will be incorrect because the line-breaks are escaped.
+      // The line-breaks need to persist in the service account private key.
+      private_key: (process.env['MATERIAL2_SCREENSHOT_FIREBASE_KEY'] || '').replace(/\\n/g, '\n')
+    }),
+    databaseURL: 'https://material2-screenshots.firebaseio.com'
+  }, 'material2-screenshots');
+
+  return screenshotApp.database();
+}

tools/gulp/tasks/screenshots.ts

@@ -0,0 +1,135 @@
+import {task} from 'gulp';
+import {readdirSync, statSync, existsSync, mkdirSync} from 'fs';
+import {openScreenshotsCloudStorage, openScreenshotsFirebaseDatabase} from '../task_helpers';
+const imageDiff = require('image-diff');
+
+const SCREENSHOT_DIR = './screenshots';
+const FIREBASE_FILELIST = 'screenshot/filenames';
+const FIREBASE_REPORT = 'screenshot/reports';
+
+/** Task which upload screenshots generated from e2e test. */
+task('screenshots', () => {
+  let prNumber = process.env['TRAVIS_PULL_REQUEST'];
+  if (prNumber) {
+    let database = openScreenshotsFirebaseDatabase();
+    return getFilenameList(database)
+      .then((filenames: string[]) => {
+        return downloadReferenceScreenshots(filenames, database)
+          .then((results: any) => {
+            return compareScreenshots(filenames, database, prNumber);
+          });
+      })
+      .then((results: boolean) => {
+        return database.ref(FIREBASE_REPORT).child(`${prNumber}/result`).set(results);
+      })
+      .then(() => setFilenameList(database, prNumber))
+      .then(() => uploadScreenshots(prNumber, 'diff'))
+      .then(() => uploadScreenshots(prNumber, 'test'))
+      .then(() => database.goOffline(), () => database.goOffline());
+  }
+});
+
+/** Get a list of filenames from firebase database. */
+function getFilenameList(database: any) : Promise<string[]> {
+  return database.ref(FIREBASE_FILELIST).once('value').then(function(snapshots: any) {
+    return snapshots.val();
+  });
+}
+
+/** Upload a list of filenames to firebase database as reference. */
+function setFilenameList(database: any,
+                         reportKey?: string): Promise<any> {
+  let filenames: string[] = [];
+  readdirSync(SCREENSHOT_DIR).map(function(file) {
+    let fullName = SCREENSHOT_DIR + '/' + file;
+    let key = file.replace('.screenshot.png', '');
+    if (!statSync(fullName).isDirectory() && key) {
+      filenames.push(file);
+    }
+  });
+  let filelistDatabase = reportKey ?
+    database.ref(FIREBASE_REPORT).child(reportKey).child('filenames') :
+    database.ref(FIREBASE_FILELIST);
+  return filelistDatabase.set(filenames);
+}
+
+/** Upload screenshots to google cloud storage. */
+function uploadScreenshots(reportKey?: string, mode?: 'test' | 'diff') {
+  let bucket = openScreenshotsCloudStorage();
+
+  let promises: Promise<any>[] = [];
+  let localDir = mode == 'diff' ? `${SCREENSHOT_DIR}/diff` : SCREENSHOT_DIR;
+  readdirSync(localDir).map(function(file) {
+    let fileName = localDir + '/' + file;
+    let key = file.replace('.screenshot.png', '');
+    let destination = (mode == null || !reportKey) ?
+      `references/${file}` : `screenshots/${reportKey}/${mode}/${file}`;
+
+    if (!statSync(fileName).isDirectory() && key) {
+      promises.push(bucket.upload(fileName, { destination: destination }));
+    }
+  });
+  return Promise.all(promises);
+}
+
+/** Check whether the directory exists. If not then create one. */
+function _makeDir(dirName: string) {
+  if (!existsSync(dirName)) {
+    mkdirSync(dirName, '744');
+  }
+}
+
+/** Download references screenshots. */
+function downloadReferenceScreenshots(
+    filenames: string[], database: any): Promise<any> {
+  _makeDir(`${SCREENSHOT_DIR}/references`);
+
+  return Promise.all(filenames.map((filename: string) => {
+    return _downloadReferenceScreenshot(filename);
+  }));
+}
+
+/** Download one reference screenshot */
+function _downloadReferenceScreenshot(filename: string): Promise<any> {
+  let bucket = openScreenshotsCloudStorage();
+  return bucket.file(`references/${filename}`).download({
+    destination: `${SCREENSHOT_DIR}/references/${filename}`
+  });
+}
+
+/** Compare the test result and the reference. */
+function compareScreenshots(filenames: string[], database: any, reportKey: string): Promise<any> {
+  return Promise.all(filenames.map((filename) =>
+    _compareScreenshot(filename, database, reportKey)))
+      .then((results: any) => results.every((value: boolean) => value == true));
+}
+
+function _compareScreenshot(filename: string, database: any,
+                             reportKey: string): Promise<any> {
+  let expectedUrl = `${SCREENSHOT_DIR}/references/${filename}`;
+  let actualUrl = `${SCREENSHOT_DIR}/${filename}`;
+  let diffUrl = `${SCREENSHOT_DIR}/diff/${filename}`;
+  let filenameKey = filename.replace('.screenshot.png', '');
+
+  if (existsSync(expectedUrl) && existsSync(actualUrl)) {
+    return new Promise(function(resolve, reject) {
+      imageDiff({
+        actualImage: actualUrl,
+        expectedImage: expectedUrl,
+        diffImage: diffUrl,
+      }, function (err: any, imagesAreSame: boolean) {
+        if (err) {
+          console.log(err);
+          imagesAreSame = false;
+          reject(err);
+        }
+        resolve(imagesAreSame);
+        return database.ref(FIREBASE_REPORT).child(`${reportKey}/results/${filenameKey}`)
+          .set(imagesAreSame);
+      });
+    });
+  } else {
+    return database.ref(FIREBASE_REPORT).child(`${reportKey}/results/${filenameKey}`)
+      .set(false).then(() => false);
+  }
+}