Cross-browser testing using SauceLabs and TravisCI

.travis.yml

@@ -3,10 +3,18 @@ language: node_js
 node_js:
   - "8"
 dist: trusty
-addons:
-  chrome: stable
 cache:
   directories:
     - node_modules
+addons:
+  chrome: stable
+  sauce_connect: true
+before_install:
+  - npm i -g [email protected]
 script:
   - npm test
+  - npm test:ci
+env:
+  global:
+  - secure: iHuXK1dZnoxh7Jm15eRcAG7qxaMbkhxB7qJuvlYL6sucpYdDWYDmmQxWdbU7BxWFlqq05eyRjQqTK342Z2I7ScsZLGCjW7vUPsluYjx+LKuMN0kHY5ayqur4pX8pgOn+faTzSWfP44R1YO0DTM15idcgu7uMC1KvXdiC6ayrp+U=
+  - secure: DsLP1gfQVVs8ZGECQl8YcIqXU/ADRoQLEp3tVuKg3suOYcqsJSXmozQajO8ld/PXQZgWODNDfeK0AzaKNVWsVhdflaGbXpE6Gzdr6F+zTkU3Ma0HZ3HPTQP5o9qraGbG5JFKqtHPek/7BeZ4aC6ReY63d846+L2Ev7he5QvSCHE=

Gruntfile.js

@@ -197,6 +197,32 @@ module.exports = function(grunt) {
       }
     },
 
+    'saucelabs-mocha': {
+      all: {
+        options: {
+          urls: [
+            'http://127.0.0.1:9999/test/index.html',
+            'http://127.0.0.1:9999/test/integration/index.html'
+          ],
+          sauceConfig: {
+            'record-video': false,
+            'record-screenshots': false
+          },
+          build: process.env.TRAVIS_BUILD_NUMBER,
+          testname:
+            'Raven.js' +
+            (process.env.TRAVIS_JOB_NUMBER ? ' #' + process.env.TRAVIS_JOB_NUMBER : ''),
+          browsers: [
+            ['macOS 10.12', 'chrome', 'latest'],
+            ['macOS 10.12', 'firefox', 'latest'],
+            ['macOS 10.12', 'safari', '10.0']
+          ],
+          public: 'public',
+          tunnelArgs: ['--verbose']
+        }
+      }
+    },
+
     release: {
       options: {
         npm: false,
@@ -229,6 +255,12 @@ module.exports = function(grunt) {
     },
 
     connect: {
+      ci: {
+        options: {
+          port: 9999
+        }
+      },
+
       test: {
         options: {
           port: 8000,
@@ -301,6 +333,14 @@ module.exports = function(grunt) {
     grunt.config.set('pkg', pkg);
   });
 
+  grunt.registerTask('config:ci', 'Verify CI config', function() {
+    if (!process.env.SAUCE_USERNAME)
+      console.warn('No SAUCE_USERNAME env variable defined.');
+    if (!process.env.SAUCE_ACCESS_KEY)
+      console.warn('No SAUCE_ACCESS_KEY env variable defined.');
+    if (!process.env.SAUCE_USERNAME || !process.env.SAUCE_ACCESS_KEY) process.exit(1);
+  });
+
   // Grunt contrib tasks
   grunt.loadNpmTasks('grunt-contrib-uglify');
   grunt.loadNpmTasks('grunt-contrib-clean');
@@ -313,6 +353,7 @@ module.exports = function(grunt) {
   grunt.loadNpmTasks('grunt-s3');
   grunt.loadNpmTasks('grunt-gitinfo');
   grunt.loadNpmTasks('grunt-sri');
+  grunt.loadNpmTasks('grunt-saucelabs');
 
   // Build tasks
   grunt.registerTask('_prep', ['clean', 'gitinfo', 'version']);
@@ -335,6 +376,13 @@ module.exports = function(grunt) {
   grunt.registerTask('build', ['build.plugins-combined']);
   grunt.registerTask('dist', ['build.core', 'copy:dist']);
 
+  grunt.registerTask('test:ci', [
+    'config:ci',
+    'build.test',
+    'connect:ci',
+    'saucelabs-mocha'
+  ]);
+
   // Webserver tasks
   grunt.registerTask('run:test', ['connect:test']);
   grunt.registerTask('run:docs', ['connect:docs']);

README.md

@@ -6,7 +6,9 @@
     <h1>Raven.js - Sentry SDK for JavaScript</h1>
 </p>
 
-[![Build Status](https://travis-ci.org/getsentry/raven-js.svg?branch=master)](https://travis-ci.org/getsentry/raven-js) [![npm](https://img.shields.io/npm/v/raven-js.svg)](https://www.npmjs.com/package/raven-js) [![npm](https://img.shields.io/npm/dm/raven-js.svg)](https://www.npmjs.com/package/raven-js)
+[![Sauce Test Status](https://saucelabs.com/buildstatus/kamilsentry)](https://saucelabs.com/u/kamilsentry)[![Build Status](https://travis-ci.org/getsentry/raven-js.svg?branch=master)](https://travis-ci.org/getsentry/raven-js) [![npm](https://img.shields.io/npm/v/raven-js.svg)](https://www.npmjs.com/package/raven-js) [![npm](https://img.shields.io/npm/dm/raven-js.svg)](https://www.npmjs.com/package/raven-js)
+
+[![Sauce Test Status](https://saucelabs.com/browser-matrix/kamilsentry.svg)](https://saucelabs.com/u/kamilsentry)
 
 ## Requirements
 

package.json

@@ -21,13 +21,18 @@
     "test": "npm run lint && grunt build.test && npm run test:unit && npm run test:integration && npm run test:typescript",
     "test:unit": "mocha-chrome test/index.html",
     "test:integration": "mocha-chrome test/integration/index.html --chrome-flags '[\"--disable-web-security\"]' --ignore-resource-errors --ignore-exceptions",
-    "test:typescript": "tsc --noEmit --noImplicitAny typescript/raven-tests.ts"
+    "test:typescript": "tsc --noEmit --noImplicitAny typescript/raven-tests.ts",
+    "test:ci": "npm run lint && grunt test:ci"
+  },
+  "engines": {
+    "node": "~8.0.0",
+    "npm": "~5.3.0"
   },
   "devDependencies": {
     "bluebird": "^3.4.1",
     "browserify-versionify": "^1.0.6",
     "bundle-collapser": "^1.2.1",
-    "chai": "2.3.0",
+    "chai": "^4.1.1",
     "derequire": "2.0.3",
     "es6-promise": "^4.0.5",
     "eslint": "^4.6.1",
@@ -42,14 +47,15 @@
     "grunt-gitinfo": "^0.1.7",
     "grunt-release": "^0.13.0",
     "grunt-s3": "0.2.0-alpha.3",
+    "grunt-saucelabs": "^9.0.0",
     "grunt-sri": "mattrobenolt/grunt-sri#pretty",
     "husky": "^0.14.3",
     "lint-staged": "^4.0.4",
-    "mocha": "2.5.3",
+    "mocha": "^3.5.0",
     "mocha-chrome": "^0.2.1",
     "prettier": "^1.6.1",
     "proxyquireify": "^3.0.2",
-    "sinon": "1.7.3",
+    "sinon": "^3.2.1",
     "through2": "^2.0.0",
     "typescript": "^1.8.10",
     "whatwg-fetch": "^1.0.0"

test/index.html

@@ -4,56 +4,65 @@
     <title>Raven.js Test Suite</title>
     <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
     <link rel="stylesheet" href="../node_modules/mocha/mocha.css" />
-  </head>
-  <body>
-    <div id="mocha"></div>
+
     <!-- Mocha -->
     <script src="../node_modules/mocha/mocha.js"></script>
     <script src="../node_modules/chai/chai.js"></script>
     <script>
-        mocha.ui('bdd');
-        mocha.reporter('html');
-        var assert = chai.assert
+      mocha.ui('bdd');
+      mocha.reporter('html');
+      var assert = chai.assert
     </script>
 
     <!-- Mocking -->
     <script src="../node_modules/sinon/pkg/sinon.js"></script>
     <script>
-    window.__DEV__ = true;
-    beforeEach(function() {
+      window.__DEV__ = true;
+
+      beforeEach(function() {
         this.sinon = sinon.sandbox.create();
-    });
+      });
 
-    afterEach(function() {
+      afterEach(function() {
         this.sinon.restore();
-    });
+      });
     </script>
 
     <!-- Tests -->
     <script src="../build/raven.test.js"></script>
+  </head>
+  <body>
+    <div id="mocha"></div>
 
     <script>
-        (function(runner){
-          var failed = [];
-
-          runner.on('fail', function(test, err){
-            failed.push({
-              title: test.title,
-              fullTitle: test.fullTitle(),
-              error: {
-                message: err.message,
-                stack: err.stack
-              }
-            });
-          });
+      window.onload = function () {
+        var runner = mocha.run();
+        var failedTests = [];
 
-          runner.on('end', function(){
-            runner.stats.failed = failed;
-            if (typeof global !== "undefined") {
-              global.mochaResults = runner.stats;
-            }
+        var flattenTitles = function (test) {
+          var titles = [];
+          while (test.parent.title){
+            titles.push(test.parent.title);
+            test = test.parent;
+          }
+          return titles.reverse();
+        };
+
+        runner.on('fail', function (test, err) {
+          failedTests.push({
+            name: test.title,
+            result: false,
+            message: err.message,
+            stack: err.stack,
+            titles: flattenTitles(test)
           });
-        })(mocha.run());
+        });
+
+        runner.on('end', function () {
+          window.mochaResults = runner.stats;
+          window.mochaResults.reports = failedTests;
+        });
+      };
     </script>
   </body>
 </html>

test/integration/frame.html

@@ -1,4 +1,4 @@
-<!DOCTYPE html>
+<!doctype html>
 <html>
 <head>
   <meta charset="utf-8">

test/integration/index.html

@@ -4,61 +4,52 @@
     <title>Raven.js Test Suite</title>
     <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
     <link rel="stylesheet" href="../../node_modules/mocha/mocha.css" />
-  </head>
-  <body>
-    <div id="mocha"></div>
+
     <!-- Mocha -->
     <script src="../../node_modules/mocha/mocha.js"></script>
     <script src="../../node_modules/chai/chai.js"></script>
     <script>
-        mocha.ui('bdd');
-        mocha.reporter('html');
-        var assert = chai.assert
-    </script>
-
-    <!-- Mocking -->
-    <script src="../../node_modules/sinon/pkg/sinon.js"></script>
-    <script src="../../build/raven.js"></script>
-    <script>
-    beforeEach(function() {
-        this.sinon = sinon.sandbox.create();
-        var xhr = sinon.useFakeXMLHttpRequest();
-        window.requests = this.requests = [];
-
-        xhr.onCreate = function (xhr) {
-          requests.push(xhr);
-        };
-    });
-
-    afterEach(function() {
-        this.sinon.restore();
-    });
+      mocha.ui('bdd');
+      mocha.reporter('html');
+      var assert = chai.assert
     </script>
 
     <!-- Tests -->
+    <script src="../../build/raven.js"></script>
     <script src="test.js"></script>
+  </head>
+  <body>
+    <div id="mocha"></div>
 
     <script>
-      (function (runner) {
-        window.runner = runner;
-        var failed = [];
+      window.onload = function () {
+        var runner = mocha.run();
+        var failedTests = [];
+
+        var flattenTitles = function (test) {
+          var titles = [];
+          while (test.parent.title){
+            titles.push(test.parent.title);
+            test = test.parent;
+          }
+          return titles.reverse();
+        };
 
         runner.on('fail', function (test, err) {
-          failed.push({
-            title: test.title,
-            fullTitle: test.fullTitle(),
-            error: {
-              message: err.message,
-              stack: err.stack
-            }
+          failedTests.push({
+            name: test.title,
+            result: false,
+            message: err.message,
+            stack: err.stack,
+            titles: flattenTitles(test)
           });
         });
 
         runner.on('end', function () {
-          runner.stats.failed = failed;
           window.mochaResults = runner.stats;
+          window.mochaResults.reports = failedTests;
         });
-      })(mocha.run());
+      };
     </script>
   </body>
 </html>

test/integration/test.js

@@ -36,6 +36,8 @@ function parseUrl(url) {
 }
 
 describe('integration', function() {
+  this.timeout(10000);
+
   beforeEach(function(done) {
     this.iframe = createIframe(done);
   });

test/plugins/react-native.test.js

@@ -172,7 +172,10 @@ describe('React Native plugin', function() {
       );
       assert.equal(lastXhr.method, 'POST');
       assert.equal(lastXhr.requestBody, '{"foo":"bar"}');
-      assert.equal(lastXhr.requestHeaders['Content-type'], 'application/json');
+      assert.equal(
+        lastXhr.requestHeaders['Content-type'],
+        'application/json;charset=utf-8'
+      );
       assert.equal(lastXhr.requestHeaders['Origin'], 'react-native://');
     });
 
@@ -257,7 +260,7 @@ describe('React Native plugin', function() {
     it('clears persisted errors after they are reported', function() {
       var payload = {abc: 123};
       var callback;
-      self.sinon.stub(Raven, '_sendProcessedPayload', function(p, cb) {
+      self.sinon.stub(Raven, '_sendProcessedPayload').callsFake(function(p, cb) {
         callback = cb;
       });
       reactNativePlugin._restorePayload = self.sinon
@@ -276,7 +279,7 @@ describe('React Native plugin', function() {
     it('does not clear persisted errors if there is an error reporting', function() {
       var payload = {abc: 123};
       var callback;
-      self.sinon.stub(Raven, '_sendProcessedPayload', function(p, cb) {
+      self.sinon.stub(Raven, '_sendProcessedPayload').callsFake(function(p, cb) {
         callback = cb;
       });
       reactNativePlugin._restorePayload = self.sinon