diff --git a/.github/workflows/cross-browser-testing-beta.yml b/.github/workflows/cross-browser-testing-beta.yml index 073249d52..1063efe8e 100644 --- a/.github/workflows/cross-browser-testing-beta.yml +++ b/.github/workflows/cross-browser-testing-beta.yml @@ -5,6 +5,7 @@ jobs: browserstack-beta-test: name: 'BrowserStack Beta Browsers Test' runs-on: ubuntu-latest + timeout-minutes: 30 steps: - name: 'BrowserStack Env Setup' uses: browserstack/github-actions/setup-env@master diff --git a/.github/workflows/cross-browser-testing.yml b/.github/workflows/cross-browser-testing.yml index f899e7a6b..66df83a04 100644 --- a/.github/workflows/cross-browser-testing.yml +++ b/.github/workflows/cross-browser-testing.yml @@ -4,10 +4,18 @@ name: 'BrowserStack Test' on: [push, pull_request, workflow_dispatch] +# Share a group across push and pull_request for the same branch so they +# do not run two BrowserStack matrices at once. +concurrency: + group: browserstack-test-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true + jobs: browserstack-test: name: 'BrowserStack Test' runs-on: ubuntu-latest + # Cap hangs from half-open BrowserStack tunnels (GitHub default is 6h). + timeout-minutes: 30 steps: - name: 'BrowserStack Env Setup' # Invokes the setup-env action diff --git a/test/cross-browser-testing/browserstack.karma.beta.config.js b/test/cross-browser-testing/browserstack.karma.beta.config.js index 9887d171e..8fbef3930 100644 --- a/test/cross-browser-testing/browserstack.karma.beta.config.js +++ b/test/cross-browser-testing/browserstack.karma.beta.config.js @@ -1,4 +1,6 @@ -const { DEBUG } = process.env; +const { + getSharedKarmaSettings, +} = require('./browserstack.karma.shared'); const files = [ '../lib/geomock.js', @@ -6,22 +8,6 @@ const files = [ '../test-bundle.js', ]; -let captureConsole = false; -let browserConsoleLogOptions = {}; - -if (DEBUG === 'true') { - browserConsoleLogOptions = { - level: 'log', - format: '%b %T: %m', - terminal: true, - }; - captureConsole = true; -} else { - browserConsoleLogOptions = { - terminal: false, - }; -} - const customLaunchers = { bs_chrome_mac_tahoe_beta: { base: 'BrowserStack', @@ -97,31 +83,12 @@ const customLaunchers = { module.exports = function(config) { config.set({ - browserStack: { - username: process.env.BS_USERNAME, - accessKey: process.env.BS_ACCESS_KEY - }, - autoWatch: false, + ...getSharedKarmaSettings({ + files, + junitOutputFile: 'test-karma-beta.xml', + }), customLaunchers, browsers: Object.keys(customLaunchers), - frameworks: ['mocha', 'should'], - files, - reporters: ['progress', 'junit'], - colors: true, - singleRun: true, - debug: true, logLevel: config.LOG_INFO, - browserConsoleLogOptions, - client: { - captureConsole, - }, - junitReporter: { - outputDir: 'reports/', - outputFile: 'test-karma-beta.xml', - }, - browserDisconnectTimeout: 50000, - browserDisconnectTolerance: 5, - concurrency: 5, }); }; - diff --git a/test/cross-browser-testing/browserstack.karma.config.js b/test/cross-browser-testing/browserstack.karma.config.js index 556f09986..dcfc388c3 100644 --- a/test/cross-browser-testing/browserstack.karma.config.js +++ b/test/cross-browser-testing/browserstack.karma.config.js @@ -1,4 +1,6 @@ -const { DEBUG } = process.env; +const { + getSharedKarmaSettings, +} = require('./browserstack.karma.shared'); const files = [ '../lib/geomock.js', @@ -6,29 +8,9 @@ const files = [ './CBT-tests-es5.js', ]; -let captureConsole = false; -let browserConsoleLogOptions = {}; - -// Allows console logs to appear when doing npm run test:debug -if (DEBUG === 'true') { - browserConsoleLogOptions = { - level: 'log', - format: '%b %T: %m', - terminal: true, - }; - captureConsole = true; -} else { - browserConsoleLogOptions = { - terminal: false, - }; -} - const customLaunchers = { - // Full list of supported browsers - https://www.browserstack.com/list-of-browsers-and-platforms/live - // https://www.w3schools.com/js/js_versions.asp shows a list of browsers that support ES6. - // The below list is primarily the version just before that, or if that version was not available on Browserstack to test, the next version was - // All versions below, including earlier versions of each browser, have a combined ~0.37% market share according to - // www.browserslist.dev. Query for "opera < 38, safari < 12, chrome < 51, firefox <52, edge < 15" + // Last pre-ES6 versions on BrowserStack. Legacy EdgeHTML (15–18) is + // omitted (~0.02% share). bs_chrome_mac_50: { base: 'BrowserStack', browser: 'chrome', @@ -43,13 +25,6 @@ const customLaunchers = { os: 'OS X', os_version: 'Mojave' }, - bs_edge_windows_15: { - base: 'BrowserStack', - browser: 'edge', - browser_version: '15.0', - os: 'Windows', - os_version: '10' - }, bs_safari_mac_11: { base: 'BrowserStack', browser: 'safari', @@ -68,33 +43,18 @@ const customLaunchers = { module.exports = function(config) { config.set({ - browserStack: { - username: process.env.BS_USERNAME, - accessKey: process.env.BS_ACCESS_KEY - }, - autoWatch: false, + ...getSharedKarmaSettings({ + files, + junitOutputFile: 'test-karma.xml', + extra: { + // Session queueing often exceeds karma's 60s capture default. + captureTimeout: 300000, + // Slow VMs can stall longer than the 30s no-activity default. + browserNoActivityTimeout: 120000, + }, + }), customLaunchers, browsers: Object.keys(customLaunchers), - frameworks: ['mocha', 'should'], - files, - reporters: ['progress', 'junit'], - colors: true, - singleRun: true, - debug: true, logLevel: config.LOG_INFO, - browserConsoleLogOptions, - client: { - captureConsole, - }, - junitReporter: { - outputDir: 'reports/', - outputFile: 'test-karma.xml', - }, - // These settings are added because the connection to Browserstack - // can sometimes be unstable, requiring re-connections, or a longer than - // 2000 ms (default) timeout - browserDisconnectTimeout: 50000, - browserDisconnectTolerance: 5, - concurrency: 5, }); }; diff --git a/test/cross-browser-testing/browserstack.karma.shared.js b/test/cross-browser-testing/browserstack.karma.shared.js new file mode 100644 index 000000000..e5994150a --- /dev/null +++ b/test/cross-browser-testing/browserstack.karma.shared.js @@ -0,0 +1,73 @@ +'use strict'; + +function getBrowserStackOptions() { + return { + username: process.env.BS_USERNAME, + accessKey: process.env.BS_ACCESS_KEY, + // Pin CI sessions to the workflow tunnel; locally karma still + // starts its own when BROWSERSTACK_LOCAL_IDENTIFIER is unset. + ...(process.env.BROWSERSTACK_LOCAL_IDENTIFIER + ? { + startTunnel: false, + localIdentifier: + process.env.BROWSERSTACK_LOCAL_IDENTIFIER, + } + : {}), + }; +} + +function getDebugConsoleOptions() { + if (process.env.DEBUG === 'true') { + return { + captureConsole: true, + browserConsoleLogOptions: { + level: 'log', + format: '%b %T: %m', + terminal: true, + }, + }; + } + + return { + captureConsole: false, + browserConsoleLogOptions: { + terminal: false, + }, + }; +} + +function getSharedKarmaSettings({ files, junitOutputFile, extra }) { + const { captureConsole, browserConsoleLogOptions } = + getDebugConsoleOptions(); + + return { + browserStack: getBrowserStackOptions(), + autoWatch: false, + frameworks: ['mocha', 'should'], + files, + reporters: ['progress', 'junit'], + colors: true, + singleRun: true, + debug: true, + browserConsoleLogOptions, + client: { + captureConsole, + // Slow BrowserStack VMs need more than mocha's 2s default. + mocha: { + timeout: 20000, + }, + }, + junitReporter: { + outputDir: 'reports/', + outputFile: junitOutputFile, + }, + browserDisconnectTimeout: 50000, + browserDisconnectTolerance: 5, + concurrency: 5, + ...extra, + }; +} + +module.exports = { + getSharedKarmaSettings, +}; diff --git a/test/src/config/utils.js b/test/src/config/utils.js index 7e2ea6470..3387cf1de 100644 --- a/test/src/config/utils.js +++ b/test/src/config/utils.js @@ -604,7 +604,8 @@ var pluses = /\+/g, }, waitForCondition = function async( conditionFn, - timeout = 200, + // BrowserStack VMs need more than 200ms for mocked async work. + timeout = 3000, interval = 10 ) { return new Promise((resolve, reject) => { diff --git a/test/src/tests-core-sdk.js b/test/src/tests-core-sdk.js index 6eb263fb4..5c1c1e4db 100644 --- a/test/src/tests-core-sdk.js +++ b/test/src/tests-core-sdk.js @@ -975,6 +975,10 @@ describe('core SDK', function() { // fetching the config is async and we need to wait for it to finish mParticle.getInstance()._Store.isInitialized.should.equal(true); + // Init's identify (mocked 400) must finish or the next identify + // is rejected as already in flight on slow browsers. + await waitForCondition(hasIdentityCallInflightReturned); + // have to manually call identify although it was called as part of init because we can only mock the server response once fetchMockSuccess(urls.identify, { mpid: 'MPID1', diff --git a/test/src/tests-mparticle-instance-manager.ts b/test/src/tests-mparticle-instance-manager.ts index 980d6baae..c3f0160d7 100644 --- a/test/src/tests-mparticle-instance-manager.ts +++ b/test/src/tests-mparticle-instance-manager.ts @@ -320,14 +320,18 @@ describe('mParticle instance manager', () => { }); it('creates multiple instances with their own cookies', async () => { - await waitForCondition(hasConfigurationReturned); - const cookies1 = window.localStorage.getItem('mprtcl-v4_wtTest1'); - const cookies2 = window.localStorage.getItem('mprtcl-v4_wtTest2'); - const cookies3 = window.localStorage.getItem('mprtcl-v4_wtTest3'); - - cookies1.includes('apiKey1').should.equal(true); - cookies2.includes('apiKey2').should.equal(true); - cookies3.includes('apiKey3').should.equal(true); + // hasConfigurationReturned only covers the default instance; + // instance 2/3 cookies can still be unset on slow VMs. + await waitForCondition(() => { + const cookies1 = window.localStorage.getItem('mprtcl-v4_wtTest1'); + const cookies2 = window.localStorage.getItem('mprtcl-v4_wtTest2'); + const cookies3 = window.localStorage.getItem('mprtcl-v4_wtTest3'); + return ( + cookies1?.includes('apiKey1') && + cookies2?.includes('apiKey2') && + cookies3?.includes('apiKey3') + ); + }); }); it('logs events to their own instances', async () => {