Return and handle promises in tests

.circleci/test.sh

@@ -6,7 +6,7 @@ set +o pipefail
 
 ROOT=$(dirname $0)/..
 EXIT_STATE=0
-MAX_AUTO_RETRY=5
+MAX_AUTO_RETRY=0
 
 log () {
     echo -e "\n$1"
@@ -55,6 +55,7 @@ case $1 in
 
         SHARDS=($(node $ROOT/tasks/shard_jasmine_tests.js --limit=5 --tag=gl | circleci tests split))
         for s in ${SHARDS[@]}; do
+            MAX_AUTO_RETRY=1
             retry npm run test-jasmine -- "$s" --tags=gl --skip-tags=noCI --doNotFailOnEmptyTestSuite
         done
 
@@ -67,6 +68,7 @@ case $1 in
         SHARDS=($(node $ROOT/tasks/shard_jasmine_tests.js --limit=1 --tag=flaky | circleci tests split))
 
         for s in ${SHARDS[@]}; do
+            MAX_AUTO_RETRY=5
             retry npm run test-jasmine -- "$s" --tags=flaky --skip-tags=noCI
         done
 
@@ -80,6 +82,7 @@ case $1 in
         ;;
 
     image2)
+        MAX_AUTO_RETRY=5
         retry npm run test-image -- --just-flaky
         npm run test-export     || EXIT_STATE=$?
         exit $EXIT_STATE

test/jasmine/tests/animate_test.js

@@ -719,7 +719,7 @@ describe('Animating multiple axes', function() {
         destroyGraphDiv();
     });
 
-    it('@flaky updates ranges of secondary axes', function(done) {
+    it('updates ranges of secondary axes', function(done) {
         Plotly.plot(gd, [
             {y: [1, 2, 3]},
             {y: [1, 2, 3], yaxis: 'y2'}
@@ -750,7 +750,7 @@ describe('Animating multiple axes', function() {
         .then(done);
     });
 
-    it('@flaky updates ranges of secondary axes (date + category case)', function(done) {
+    it('updates ranges of secondary axes (date + category case)', function(done) {
         Plotly.plot(gd, [
             {x: ['2018-01-01', '2019-01-01', '2020-01-01'], y: [1, 2, 3]},
             {x: ['a', 'b', 'c'], y: [1, 2, 3], xaxis: 'x2', yaxis: 'y2'}

test/jasmine/tests/annotations_test.js

@@ -187,11 +187,13 @@ describe('annotations relayout', function() {
         expect(typeof MathJax).toBe('undefined');
         mockLayout.annotations[14].text = '$x+y+z$';
 
-        Plotly.plot(gd, mockData, mockLayout).then(done);
-
-        spyOn(Loggers, 'warn');
+        Plotly.plot(gd, mockData, mockLayout)
+        .then(function() {
+            spyOn(Loggers, 'warn');
 
-        Plotly.setPlotConfig({queueLength: 3});
+            return Plotly.setPlotConfig({queueLength: 3});
+        })
+        .then(done);
     });
 
     afterEach(function() {
@@ -327,7 +329,8 @@ describe('annotations relayout', function() {
 
         assertText(0, 'left top');
 
-        Plotly.relayout(gd, 'annotations[0].text', 'hello').then(function() {
+        Plotly.relayout(gd, 'annotations[0].text', 'hello')
+        .then(function() {
             assertText(0, 'hello');
 
             return Plotly.relayout(gd, 'annotations[0].text', null);
@@ -427,11 +430,15 @@ describe('annotations relayout', function() {
         {'annotations[0]': 'not an object'},
         {'annotations[100]': {text: 'bad index'}}
     ].forEach(function(update) {
-        it('warns on ambiguous combinations and invalid values: ' + JSON.stringify(update), function() {
-            Plotly.relayout(gd, update);
-            expect(Loggers.warn).toHaveBeenCalled();
-            // we could test the results here, but they're ambiguous and/or undefined so why bother?
-            // the important thing is the developer is warned that something went wrong.
+        it('warns on ambiguous combinations and invalid values: ' + JSON.stringify(update), function(done) {
+            Plotly.relayout(gd, update)
+            .then(function() {
+                expect(Loggers.warn).toHaveBeenCalled();
+                // we could test the results here, but they're ambiguous and/or undefined so why bother?
+                // the important thing is the developer is warned that something went wrong.
+            })
+            .catch(failTest)
+            .then(done);
         });
     });
 

test/jasmine/tests/carpet_test.js

@@ -724,6 +724,7 @@ describe('scattercarpet hover labels', function() {
             [200, 200], fig,
             [['a: 0.200', 'b: 3.500', 'y: 2.900'], 'a = 0.2']
         )
+        .catch(failTest)
         .then(done);
     });
 
@@ -736,6 +737,7 @@ describe('scattercarpet hover labels', function() {
             [200, 200], fig,
             [['a: 0.200', 'b: 3.500', 'y: 2.900', 'D'], 'a = 0.2']
         )
+        .catch(failTest)
         .then(done);
     });
 
@@ -747,6 +749,7 @@ describe('scattercarpet hover labels', function() {
             [200, 200], fig,
             [['a: 0.200', 'y: 2.900'], null]
         )
+        .catch(failTest)
         .then(done);
     });
 
@@ -758,6 +761,7 @@ describe('scattercarpet hover labels', function() {
             [200, 200], fig,
             [['b: 3.500', 'y: 2.900'], null]
         )
+        .catch(failTest)
         .then(done);
     });
 
@@ -769,6 +773,7 @@ describe('scattercarpet hover labels', function() {
             [200, 200], fig,
             [['f(0.2, 3.5) = 2.900'], 'scattercarpet #5']
         )
+        .catch(failTest)
         .then(done);
     });
 
@@ -780,6 +785,7 @@ describe('scattercarpet hover labels', function() {
             [200, 200], fig,
             [['f(0.2, 3.5) = 3.0'], 'pt #3']
         )
+        .catch(failTest)
         .then(done);
     });
 });

test/jasmine/tests/cartesian_interact_test.js

@@ -32,7 +32,6 @@ describe('zoom box element', function() {
         mockCopy.layout.dragmode = 'zoom';
 
         Plotly.plot(gd, mockCopy.data, mockCopy.layout)
-        .catch(failTest)
         .then(done);
     });
 
@@ -974,7 +973,7 @@ describe('axis zoom/pan and main plot zoom', function() {
             var yr0 = [-0.211, 3.211];
 
             var specs = [{
-                desc: 'zoombox on xy',
+                desc: '@flaky zoombox on xy',
                 drag: ['xy', 'nsew', 30, 30],
                 exp: [
                     [['xaxis', 'xaxis2', 'xaxis3'], [1.457, 2.328]],
@@ -2338,7 +2337,7 @@ describe('Event data:', function() {
             expect('marker.colorbar.tickvals' in pt).toBe(false, 'marker.colorbar.tickvals');
             expect('marker.colorbar.ticktext' in pt).toBe(false, 'marker.colorbar.ticktext');
         })
-        .catch(fail)
+        .catch(failTest)
         .then(done);
     });
 
@@ -2366,7 +2365,7 @@ describe('Event data:', function() {
             expect('marker.colorbar.tickvals' in pt).toBe(false, 'marker.colorbar.tickvals');
             expect('marker.colorbar.ticktext' in pt).toBe(false, 'marker.colorbar.ticktext');
         })
-        .catch(fail)
+        .catch(failTest)
         .then(done);
     });
 });

test/jasmine/tests/choropleth_test.js

@@ -205,6 +205,7 @@ describe('Test choropleth hover:', function() {
                 fig,
                 ['RUS\n10', 'trace 1']
             )
+            .catch(failTest)
             .then(done);
         });
     });
@@ -220,6 +221,7 @@ describe('Test choropleth hover:', function() {
                 fig,
                 ['tpl 10', 'x']
             )
+            .catch(failTest)
             .then(done);
         });
     });
@@ -236,6 +238,7 @@ describe('Test choropleth hover:', function() {
                 fig,
                 ['tExT', null]
             )
+            .catch(failTest)
             .then(done);
         });
     });
@@ -252,6 +255,7 @@ describe('Test choropleth hover:', function() {
                 fig,
                 ['-text-', null]
             )
+            .catch(failTest)
             .then(done);
         });
     });
@@ -269,6 +273,7 @@ describe('Test choropleth hover:', function() {
                 fig,
                 ['-text-', null]
             )
+            .catch(failTest)
             .then(done);
         });
     });
@@ -295,6 +300,7 @@ describe('Test choropleth hover:', function() {
                     fontFamily: 'Roboto'
                 }
             )
+            .catch(failTest)
             .then(done);
         });
     });
@@ -310,6 +316,7 @@ describe('Test choropleth hover:', function() {
                 fig,
                 ['RUS', 'trace 1']
             )
+            .catch(failTest)
             .then(done);
         });
     });
@@ -330,15 +337,19 @@ describe('Test choropleth hover:', function() {
 
         [false, true].forEach(function(hasCssTransform) {
             it('- base case (truncate z decimals), hasCssTransform: ' + hasCssTransform, function(done) {
-                run(hasCssTransform, pos, base(), exp).then(done);
+                run(hasCssTransform, pos, base(), exp)
+                .catch(failTest)
+                .then(done);
             });
         });
 
         [false, true].forEach(function(hasCssTransform) {
             it('- hovertemplate case (same z truncation), hasCssTransform: ' + hasCssTransform, function(done) {
                 var fig = base();
                 fig.hovertemplate = '%{z}<extra>%{location}</extra>';
-                run(hasCssTransform, pos, fig, exp).then(done);
+                run(hasCssTransform, pos, fig, exp)
+                .catch(failTest)
+                .then(done);
             });
         });
     });
@@ -350,7 +361,9 @@ describe('Test choropleth hover:', function() {
             fig.data[0].hovertemplate = '%{properties.name}<extra>%{ct[0]:.1f} | %{ct[1]:.1f}</extra>';
             fig.layout.geo.projection = {scale: 20};
 
-            run(hasCssTransform, [300, 200], fig, ['New York', '-75.1 | 42.6']).then(done);
+            run(hasCssTransform, [300, 200], fig, ['New York', '-75.1 | 42.6'])
+            .catch(failTest)
+            .then(done);
         });
     });
 });

test/jasmine/tests/choroplethmapbox_test.js

@@ -516,7 +516,7 @@ describe('Test choroplethmapbox convert:', function() {
     });
 });
 
-describe('@noCI Test choroplethmapbox hover:', function() {
+describe('Test choroplethmapbox hover:', function() {
     var gd;
 
     afterEach(function(done) {
@@ -653,7 +653,7 @@ describe('@noCI Test choroplethmapbox hover:', function() {
     });
 });
 
-describe('@noCI Test choroplethmapbox interactions:', function() {
+describe('Test choroplethmapbox interactions:', function() {
     var gd;
 
     var geojson = {