Update o11y stability improvements

bin/commands/runs.js

@@ -111,7 +111,10 @@ module.exports = function run(args, rawArgs) {
     /* 
       Send build start to Observability
     */
-    if(isTestObservabilitySession) await launchTestSession(bsConfig, bsConfigPath);
+    if(isTestObservabilitySession) {
+      await launchTestSession(bsConfig, bsConfigPath);
+      utils.setO11yProcessHooks(null, bsConfig, args, null, buildReportData);
+    }
 
     // accept the system env list from bsconf and set it
     utils.setSystemEnvs(bsConfig);
@@ -264,6 +267,9 @@ module.exports = function run(args, rawArgs) {
                 markBlockEnd('createBuild');
                 markBlockEnd('total');
                 utils.setProcessHooks(data.build_id, bsConfig, bs_local, args, buildReportData);
+                if(isTestObservabilitySession) {
+                  utils.setO11yProcessHooks(data.build_id, bsConfig, bs_local, args, buildReportData);
+                }
                 let message = `${data.message}! ${Constants.userMessages.BUILD_CREATED} with build id: ${data.build_id}`;
                 let dashboardLink = `${Constants.userMessages.VISIT_DASHBOARD} ${data.dashboard_url}`;
                 buildReportData = { 'build_id': data.build_id, 'parallels': userSpecifiedParallels, ...buildReportData }

bin/helpers/utils.js

@@ -22,7 +22,7 @@ const usageReporting = require("./usageReporting"),
   config = require("../helpers/config"),
   pkg = require('../../package.json'),
   transports = require('./logger').transports,
-  { findGitConfig, printBuildLink, isTestObservabilitySession, isBrowserstackInfra, shouldReRunObservabilityTests } = require('../testObservability/helper/helper'),
+  o11yHelpers = require('../testObservability/helper/helper'),
   { OBSERVABILITY_ENV_VARS, TEST_OBSERVABILITY_REPORTER } = require('../testObservability/helper/constants');
 
 const request = require('request');
@@ -480,7 +480,7 @@ exports.setNodeVersion = (bsConfig, args) => {
 // specs can be passed via command line args as a string
 // command line args takes precedence over config
 exports.setUserSpecs = (bsConfig, args) => {
-  if(isBrowserstackInfra() && isTestObservabilitySession() && shouldReRunObservabilityTests()) {
+  if(o11yHelpers.isBrowserstackInfra() && o11yHelpers.isTestObservabilitySession() && o11yHelpers.shouldReRunObservabilityTests()) {
     bsConfig.run_settings.specs = process.env.BROWSERSTACK_RERUN_TESTS;
     return;
   }
@@ -580,8 +580,8 @@ exports.setSystemEnvs = (bsConfig) => {
       envKeys[key] = process.env[key];
     });
 
-    let gitConfigPath = findGitConfig(process.cwd());
-    if(!isBrowserstackInfra()) process.env.OBSERVABILITY_GIT_CONFIG_PATH_LOCAL = gitConfigPath;
+    let gitConfigPath = o11yHelpers.findGitConfig(process.cwd());
+    if(!o11yHelpers.isBrowserstackInfra()) process.env.OBSERVABILITY_GIT_CONFIG_PATH_LOCAL = gitConfigPath;
     if(gitConfigPath) {
       const relativePathFromGitConfig = path.relative(gitConfigPath, process.cwd());
       envKeys["OBSERVABILITY_GIT_CONFIG_PATH"] = relativePathFromGitConfig ? relativePathFromGitConfig : 'DEFAULT';
@@ -1184,8 +1184,8 @@ exports.handleSyncExit = (exitCode, dashboard_url) => {
     syncCliLogger.info(Constants.userMessages.BUILD_REPORT_MESSAGE);
     syncCliLogger.info(dashboard_url);
   }
-  if(isTestObservabilitySession()) {
-    printBuildLink(true, exitCode);
+  if(o11yHelpers.isTestObservabilitySession()) {
+    o11yHelpers.printBuildLink(true, exitCode);
   } else {
     process.exit(exitCode);
   }
@@ -1288,7 +1288,7 @@ exports.setConfig = (bsConfig, args) => {
 
 // blindly send other passed configs with run_settings and handle at backend
 exports.setOtherConfigs = (bsConfig, args) => {
-  if(isTestObservabilitySession() && process.env.BS_TESTOPS_JWT) {
+  if(o11yHelpers.isTestObservabilitySession() && process.env.BS_TESTOPS_JWT) {
     bsConfig["run_settings"]["reporter"] = TEST_OBSERVABILITY_REPORTER;
     return;
   }
@@ -1453,14 +1453,37 @@ exports.setProcessHooks = (buildId, bsConfig, bsLocal, args, buildReportData) =>
   process.on('uncaughtException', processExitHandler.bind(this, bindData));
 }
 
+exports.setO11yProcessHooks = (() => {
+  let bindData = {};
+  let handlerAdded = false;
+  return (buildId, bsConfig, bsLocal, args, buildReportData) => {
+    bindData.buildId = buildId;
+    bindData.bsConfig = bsConfig;
+    bindData.bsLocal = bsLocal;
+    bindData.args = args;
+    bindData.buildReportData = buildReportData;
+    if (handlerAdded) return;
+    handlerAdded = true;
+    process.on('beforeExit', processO11yExitHandler.bind(this, bindData));
+  }
+})()
+
 async function processExitHandler(exitData){
   logger.warn(Constants.userMessages.PROCESS_KILL_MESSAGE);
   await this.stopBrowserStackBuild(exitData.bsConfig, exitData.args, exitData.buildId, null, exitData.buildReportData);
   await this.stopLocalBinary(exitData.bsConfig, exitData.bsLocalInstance, exitData.args, null, exitData.buildReportData);
-  await printBuildLink(true);
+  await o11yHelpers.printBuildLink(true);
   process.exit(0);
 }
 
+async function processO11yExitHandler(exitData){
+  if (exitData.buildId) {
+    await o11yHelpers.printBuildLink(false);
+  } else {
+    await o11yHelpers.printBuildLink(true);
+  }
+}
+
 exports.fetchZipSize = (fileName) => {
   try {
     let stats = fs.statSync(fileName)

bin/testObservability/helper/helper.js

@@ -64,8 +64,11 @@ const supportFileCleanup = () => {
   });
 }
 
+exports.buildStopped = false;
+
 exports.printBuildLink = async (shouldStopSession, exitCode = null) => {
-  if(!this.isTestObservabilitySession()) return;
+  if(!this.isTestObservabilitySession() || exports.buildStopped) return;
+  exports.buildStopped = true;
   try {
     if(shouldStopSession) {
       supportFileCleanup();

bin/testObservability/reporter/index.js

@@ -71,6 +71,7 @@ class MyReporter {
     this._paths = new PathHelper({ cwd: process.cwd() }, this._testEnv.location_prefix);
     this.currentTestSteps = [];
     this.currentTestCucumberSteps = [];
+    this.hooksStarted = {};
     this.beforeHooks = [];
     this.platformDetailsMap = {};
     this.runStatusMarkedHash = {};
@@ -110,6 +111,9 @@ class MyReporter {
             } else {
               this.runStatusMarkedHash[hook.hookAnalyticsId] = true;
             }
+
+            // Remove hooks added at hook start
+            delete this.hooksStarted[hook.hookAnalyticsId];
             await this.sendTestRunEvent(hook,undefined,false,"HookRunFinished");
           }
         }
@@ -152,12 +156,14 @@ class MyReporter {
       })
 
       .on(EVENT_TEST_BEGIN, async (test) => {
+        if (test.isSkipped) return;
         if(this.testObservability == true) {
           await this.testStarted(test);
         }
       })
 
       .on(EVENT_TEST_END, async (test) => {
+        if (test.isSkipped) return;
         if(this.testObservability == true) {
           if(!this.runStatusMarkedHash[test.testAnalyticsId]) {
             if(test.testAnalyticsId) this.runStatusMarkedHash[test.testAnalyticsId] = true;
@@ -435,6 +441,38 @@ class MyReporter {
         };
         await uploadEventData(buildUpdateData);
       }
+
+      // Add started hooks to the hash
+      if(eventType === 'HookRunStarted' && ['BEFORE_EACH', 'AFTER_EACH', 'BEFORE_ALL'].includes(testData['hook_type'])) {
+        this.hooksStarted[testData.uuid] = uploadData;
+      }
+
+      // Send pending hook finsihed events for hook starts
+      if (eventType === 'TestRunFinished') {
+        Object.values(this.hooksStarted).forEach(async hookData => {
+          hookData['event_type'] = 'HookRunFinished';
+          hookData['hook_run'] = {
+            ...hookData['hook_run'],
+            result: uploadData['test_run'].result,
+            failure: uploadData['test_run'].failure,
+            failure_type: uploadData['test_run'].failure_type,
+            failure_reason: uploadData['test_run'].failure_reason,
+            failure_reason_expanded: uploadData['test_run'].failure_reason_expanded,
+            failure_backtrace: uploadData['test_run'].failure_backtrace
+
+          }
+
+          if (hookData['hook_run']['hook_type'] === 'BEFORE_ALL') {
+            hookData['hook_run'].finished_at = uploadData['test_run'].finished_at;
+            hookData['hook_run'].duration_in_ms = new Date(hookData['hook_run'].finished_at).getTime() - new Date(hookData['hook_run'].started_at).getTime();
+          } else {
+            hookData['hook_run'].finished_at = hookData['hook_run'].started_at;
+            hookData['hook_run'].duration_in_ms = 0;
+          }
+          await uploadEventData(hookData);
+        })
+        this.hooksStarted = {};
+      }
     } catch(error) {
       debug(`Exception in populating test data for event ${eventType} with error : ${error}`, true, error);
     }

test/unit/bin/helpers/utils.js

@@ -21,7 +21,8 @@ const utils = require('../../../../bin/helpers/utils'),
   fileHelpers = require('../../../../bin/helpers/fileHelpers'),
   testObjects = require('../../support/fixtures/testObjects'),
   syncLogger = require('../../../../bin/helpers/logger').syncCliLogger,
-  Contants = require('../../../../bin/helpers/constants');
+  Contants = require('../../../../bin/helpers/constants'),
+  o11yHelpers = require('../../../../bin/testObservability/helper/helper');
 const browserstack = require('browserstack-local');
 const { CYPRESS_V10_AND_ABOVE_TYPE, CYPRESS_V9_AND_OLDER_TYPE } = require('../../../../bin/helpers/constants');
 const { winstonLogger, syncCliLogger } = require('../../../../bin/helpers/logger');
@@ -3396,6 +3397,72 @@ describe('utils', () => {
     });
   });
 
+  describe('setO11yProcessHooks', () => {
+    it('should handle multiple calls', (done) => {
+      let buildId = null;
+      let bsConfig = testObjects.sampleBsConfig;
+      let bsLocalStub = sinon.stub();
+      let args= {};
+
+      let printBuildLinkStub = sinon.stub(o11yHelpers, 'printBuildLink').returns(Promise.resolve(true));
+      let processOnSpy = sinon.spy(process, 'on');
+
+      utils.setO11yProcessHooks(buildId, bsConfig, bsLocalStub, args);
+      sinon.assert.calledOnce(processOnSpy);
+      processOnSpy.restore();
+      processOnSpy = sinon.spy(process, 'on');
+      utils.setO11yProcessHooks('build_id', bsConfig, bsLocalStub, args);
+      sinon.assert.notCalled(processOnSpy);
+      processOnSpy.restore();
+      process.on('beforeExit', () => {
+        sinon.assert.calledOnce(printBuildLinkStub);
+        sinon.assert.calledWith(printBuildLinkStub, false);
+        done();
+      });
+      process.emit('beforeExit');
+      printBuildLinkStub.restore();
+      sinon.stub.restore();
+    });
+
+    it('should handle "beforeExit" event, with build id', (done) => {
+      let buildId = 'build_id';
+      let bsConfig = testObjects.sampleBsConfig;
+      let bsLocalStub = sinon.stub();
+      let args= {};
+
+      let printBuildLinkStub = sinon.stub(o11yHelpers, 'printBuildLink').returns(Promise.resolve(true));
+
+      utils.setO11yProcessHooks(buildId, bsConfig, bsLocalStub, args);
+      process.on('beforeExit', () => {
+        sinon.assert.calledOnce(printBuildLinkStub);
+        sinon.assert.calledWith(printBuildLinkStub, false);
+        done();
+      });
+      process.emit('beforeExit');
+      printBuildLinkStub.restore();
+      sinon.stub.restore();
+    });
+
+    it('should handle "beforeExit" event, without build id', (done) => {
+      let buildId = null;
+      let bsConfig = testObjects.sampleBsConfig;
+      let bsLocalStub = sinon.stub();
+      let args= {};
+
+      let printBuildLinkStub = sinon.stub(o11yHelpers, 'printBuildLink').returns(Promise.resolve(true));
+
+      utils.setO11yProcessHooks(buildId, bsConfig, bsLocalStub, args);
+      process.on('beforeExit', () => {
+        sinon.assert.calledOnce(printBuildLinkStub);
+        sinon.assert.calledWith(printBuildLinkStub, true);
+        done();
+      });
+      process.emit('beforeExit');
+      printBuildLinkStub.restore();
+      sinon.stub.restore();
+    });
+  });
+
   describe('fetchZipSize', () => {
     it('should return size in bytes if file is present', () => {
       sinon.stub(fs, 'statSync').returns({size: 123});