fix(ci): stabilize BrowserStack Test workflow

.github/workflows/cross-browser-testing-beta.yml

@@ -5,6 +5,7 @@ jobs:
   browserstack-beta-test:
     name: 'BrowserStack Beta Browsers Test'
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     steps:
       - name: 'BrowserStack Env Setup'
         uses: browserstack/github-actions/setup-env@master

.github/workflows/cross-browser-testing.yml

@@ -4,10 +4,18 @@
 name: 'BrowserStack Test'
 on: [push, pull_request, workflow_dispatch]
 
+# Share a group across push and pull_request for the same branch so they
+# do not run two BrowserStack matrices at once.
+concurrency:
+  group: browserstack-test-${{ github.head_ref || github.ref_name }}
+  cancel-in-progress: true
+
 jobs:
   browserstack-test:
     name: 'BrowserStack Test'
     runs-on: ubuntu-latest
+    # Cap hangs from half-open BrowserStack tunnels (GitHub default is 6h).
+    timeout-minutes: 30
     steps:
 
       - name: 'BrowserStack Env Setup'  # Invokes the setup-env action

test/cross-browser-testing/browserstack.karma.beta.config.js

@@ -1,27 +1,13 @@
-const { DEBUG } = process.env;
+const {
+    getSharedKarmaSettings,
+} = require('./browserstack.karma.shared');
 
 const files = [
   '../lib/geomock.js',
   '../../dist/mparticle.js',
   '../test-bundle.js',
 ];
 
-let captureConsole = false;
-let browserConsoleLogOptions = {};
-
-if (DEBUG === 'true') {
-    browserConsoleLogOptions = {
-        level: 'log',
-        format: '%b %T: %m',
-        terminal: true,
-    };
-    captureConsole = true;
-} else {
-    browserConsoleLogOptions = {
-        terminal: false,
-    };
-}
-
 const customLaunchers = {
   bs_chrome_mac_tahoe_beta: {
     base: 'BrowserStack',
@@ -97,31 +83,12 @@ const customLaunchers = {
 
 module.exports = function(config) {
   config.set({
-    browserStack: {
-      username: process.env.BS_USERNAME,
-      accessKey: process.env.BS_ACCESS_KEY
-    },
-    autoWatch: false,
+    ...getSharedKarmaSettings({
+      files,
+      junitOutputFile: 'test-karma-beta.xml',
+    }),
     customLaunchers,
     browsers: Object.keys(customLaunchers),
-    frameworks: ['mocha', 'should'],
-    files,
-    reporters: ['progress', 'junit'],
-    colors: true,
-    singleRun: true,
-    debug: true,
     logLevel: config.LOG_INFO,
-    browserConsoleLogOptions,
-    client: {
-      captureConsole,
-    },
-    junitReporter: {
-      outputDir: 'reports/',
-      outputFile: 'test-karma-beta.xml',
-    },
-    browserDisconnectTimeout: 50000,
-    browserDisconnectTolerance: 5,
-    concurrency: 5,
   });
 };
-

test/cross-browser-testing/browserstack.karma.config.js

@@ -1,34 +1,16 @@
-const { DEBUG } = process.env;
+const {
+    getSharedKarmaSettings,
+} = require('./browserstack.karma.shared');
 
 const files = [
   '../lib/geomock.js',
   '../../dist/mparticle.js',
   './CBT-tests-es5.js',
 ];
 
-let captureConsole = false;
-let browserConsoleLogOptions = {};
-
-// Allows console logs to appear when doing npm run test:debug
-if (DEBUG === 'true') {
-    browserConsoleLogOptions = {
-        level: 'log',
-        format: '%b %T: %m',
-        terminal: true,
-    };
-    captureConsole = true;
-} else {
-    browserConsoleLogOptions = {
-        terminal: false,
-    };
-}
-
 const customLaunchers = {
-    // Full list of supported browsers - https://www.browserstack.com/list-of-browsers-and-platforms/live
-    // https://www.w3schools.com/js/js_versions.asp shows a list of browsers that support ES6.  
-    // The below list is primarily the version just before that, or if that version was not available on Browserstack to test, the next version was
-    // All versions below, including earlier versions of each browser, have a combined ~0.37% market share according to
-    // www.browserslist.dev.  Query for "opera < 38, safari < 12, chrome < 51, firefox <52, edge < 15"
+    // Last pre-ES6 versions on BrowserStack. Legacy EdgeHTML (15–18) is
+    // omitted (~0.02% share).
     bs_chrome_mac_50: {
       base: 'BrowserStack',
       browser: 'chrome',
@@ -43,13 +25,6 @@ const customLaunchers = {
       os: 'OS X',
       os_version: 'Mojave'
     },
-    bs_edge_windows_15: {
-      base: 'BrowserStack',
-      browser: 'edge',
-      browser_version: '15.0',
-      os: 'Windows',
-      os_version: '10'
-    },
     bs_safari_mac_11: {
       base: 'BrowserStack',
       browser: 'safari',
@@ -68,33 +43,18 @@ const customLaunchers = {
 
 module.exports = function(config) {
     config.set({
-      browserStack: {
-        username: process.env.BS_USERNAME,
-        accessKey: process.env.BS_ACCESS_KEY
-      },
-      autoWatch: false,
+      ...getSharedKarmaSettings({
+        files,
+        junitOutputFile: 'test-karma.xml',
+        extra: {
+          // Session queueing often exceeds karma's 60s capture default.
+          captureTimeout: 300000,
+          // Slow VMs can stall longer than the 30s no-activity default.
+          browserNoActivityTimeout: 120000,
+        },
+      }),
       customLaunchers,
       browsers: Object.keys(customLaunchers),
-      frameworks: ['mocha', 'should'],
-      files,
-      reporters: ['progress', 'junit'],
-      colors: true,
-      singleRun: true,
-      debug: true,
       logLevel: config.LOG_INFO,
-      browserConsoleLogOptions,
-      client: {
-        captureConsole,
-      },
-      junitReporter: {
-        outputDir: 'reports/',
-        outputFile: 'test-karma.xml',
-      },
-      // These settings are added because the connection to Browserstack
-      // can sometimes be unstable, requiring re-connections, or a longer than 
-      // 2000 ms (default) timeout 
-      browserDisconnectTimeout: 50000,
-      browserDisconnectTolerance: 5,
-      concurrency: 5,
     });
 };

test/cross-browser-testing/browserstack.karma.shared.js

@@ -0,0 +1,73 @@
+'use strict';
+
+function getBrowserStackOptions() {
+    return {
+        username: process.env.BS_USERNAME,
+        accessKey: process.env.BS_ACCESS_KEY,
+        // Pin CI sessions to the workflow tunnel; locally karma still
+        // starts its own when BROWSERSTACK_LOCAL_IDENTIFIER is unset.
+        ...(process.env.BROWSERSTACK_LOCAL_IDENTIFIER
+            ? {
+                  startTunnel: false,
+                  localIdentifier:
+                      process.env.BROWSERSTACK_LOCAL_IDENTIFIER,
+              }
+            : {}),
+    };
+}
+
+function getDebugConsoleOptions() {
+    if (process.env.DEBUG === 'true') {
+        return {
+            captureConsole: true,
+            browserConsoleLogOptions: {
+                level: 'log',
+                format: '%b %T: %m',
+                terminal: true,
+            },
+        };
+    }
+
+    return {
+        captureConsole: false,
+        browserConsoleLogOptions: {
+            terminal: false,
+        },
+    };
+}
+
+function getSharedKarmaSettings({ files, junitOutputFile, extra }) {
+    const { captureConsole, browserConsoleLogOptions } =
+        getDebugConsoleOptions();
+
+    return {
+        browserStack: getBrowserStackOptions(),
+        autoWatch: false,
+        frameworks: ['mocha', 'should'],
+        files,
+        reporters: ['progress', 'junit'],
+        colors: true,
+        singleRun: true,
+        debug: true,
+        browserConsoleLogOptions,
+        client: {
+            captureConsole,
+            // Slow BrowserStack VMs need more than mocha's 2s default.
+            mocha: {
+                timeout: 20000,
+            },
+        },
+        junitReporter: {
+            outputDir: 'reports/',
+            outputFile: junitOutputFile,
+        },
+        browserDisconnectTimeout: 50000,
+        browserDisconnectTolerance: 5,
+        concurrency: 5,
+        ...extra,
+    };
+}
+
+module.exports = {
+    getSharedKarmaSettings,
+};

test/src/config/utils.js

@@ -604,7 +604,8 @@ var pluses = /\+/g,
     },
     waitForCondition = function async(
         conditionFn,
-        timeout = 200,
+        // BrowserStack VMs need more than 200ms for mocked async work.
+        timeout = 3000,
         interval = 10
     ) {
         return new Promise((resolve, reject) => {

test/src/tests-core-sdk.js

@@ -975,6 +975,10 @@ describe('core SDK', function() {
         // fetching the config is async and we need to wait for it to finish
         mParticle.getInstance()._Store.isInitialized.should.equal(true);
 
+        // Init's identify (mocked 400) must finish or the next identify
+        // is rejected as already in flight on slow browsers.
+        await waitForCondition(hasIdentityCallInflightReturned);
+
         // have to manually call identify although it was called as part of init because we can only mock the server response once
         fetchMockSuccess(urls.identify, {
             mpid: 'MPID1',

test/src/tests-mparticle-instance-manager.ts

@@ -320,14 +320,18 @@ describe('mParticle instance manager', () => {
         });
 
         it('creates multiple instances with their own cookies', async () => {
-            await waitForCondition(hasConfigurationReturned);
-            const cookies1 = window.localStorage.getItem('mprtcl-v4_wtTest1');
-            const cookies2 = window.localStorage.getItem('mprtcl-v4_wtTest2');
-            const cookies3 = window.localStorage.getItem('mprtcl-v4_wtTest3');
-
-            cookies1.includes('apiKey1').should.equal(true);
-            cookies2.includes('apiKey2').should.equal(true);
-            cookies3.includes('apiKey3').should.equal(true);
+            // hasConfigurationReturned only covers the default instance;
+            // instance 2/3 cookies can still be unset on slow VMs.
+            await waitForCondition(() => {
+                const cookies1 = window.localStorage.getItem('mprtcl-v4_wtTest1');
+                const cookies2 = window.localStorage.getItem('mprtcl-v4_wtTest2');
+                const cookies3 = window.localStorage.getItem('mprtcl-v4_wtTest3');
+                return (
+                    cookies1?.includes('apiKey1') &&
+                    cookies2?.includes('apiKey2') &&
+                    cookies3?.includes('apiKey3')
+                );
+            });
         });
 
         it('logs events to their own instances', async () => {