Skip to content

Add optional multi-page HAR generation #160

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
tinyopsstudio wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Add optional multi-page HAR generation #160

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 62 additions & 15 deletions index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@ const log = debug('chrome-har');

const defaultOptions = {
includeResourcesFromDiskCache: false,
includeTextFromResponseBody: false
includeTextFromResponseBody: false,
allowMultiPage: false
};
const isEmpty = o => !o;

Expand All @@ -44,6 +45,10 @@ function addFromFirstRequest(page, params) {
}
}

function hasEntries(page, entries) {
return entries.some(entry => entry.pageref === page.id);
}

function populateRedirectResponse(page, params, entries, options) {
const previousEntry = entries.find(
entry => entry._requestId === params.requestId
Expand Down Expand Up @@ -79,6 +84,20 @@ export function harFromMessages(messages, options) {
responseReceivedExtraInfos = [],
currentPageId;

function addPage(frameId, title = '', extra = {}) {
currentPageId = randomUUID();
const page = {
id: currentPageId,
startedDateTime: '',
title,
pageTimings: {},
__frameId: frameId,
...extra
};
pages.push(page);
return page;
}

for (const message of messages) {
const params = message.params;

Expand All @@ -90,25 +109,36 @@ export function harFromMessages(messages, options) {

switch (method) {
case 'Page.frameStartedLoading':
case 'Page.frameScheduledNavigation':
case 'Page.frameRequestedNavigation':
case 'Page.navigatedWithinDocument': {
{
const frameId = params.frameId;
const rootFrame = rootFrameMappings.get(frameId) || frameId;
if (pages.some(page => page.__frameId === rootFrame)) {
const pageExists = pages.some(page => page.__frameId === rootFrame);
const lastPage = pages.at(-1);
if (!options.allowMultiPage && pageExists) {
continue;
}
if (
options.allowMultiPage &&
method === 'Page.frameStartedLoading' &&
lastPage?.__frameId === rootFrame &&
(lastPage.__pendingNavigation ||
lastPage.__createdFromDocumentRequest)
) {
continue;
}
currentPageId = randomUUID();
const title =
method === 'Page.navigatedWithinDocument' ? params.url : '';
const page = {
id: currentPageId,
startedDateTime: '',
title: title,
pageTimings: {},
__frameId: rootFrame
};
pages.push(page);
method === 'Page.navigatedWithinDocument' ||
method === 'Page.frameScheduledNavigation'
? params.url
: '';
const page = addPage(rootFrame, title, {
__pendingNavigation:
method === 'Page.frameScheduledNavigation' ||
method === 'Page.frameRequestedNavigation'
});
// do we have any unmmapped requests, add them
if (entriesWithoutPage.length > 0) {
// update page
Expand Down Expand Up @@ -153,7 +183,7 @@ export function harFromMessages(messages, options) {
// creating a new page, so one measurement = one HAR page.
case 'SoftNavigation.detected': {
{
const page = pages.at(-1);
let page = pages.at(-1);
if (page) {
page.title = params.url || '';
page._softNavigation = true;
Expand All @@ -178,7 +208,7 @@ export function harFromMessages(messages, options) {
ignoredRequests.add(params.requestId);
continue;
}
const page = pages.at(-1);
let page = pages.at(-1);
const cookieHeader = getHeaderValue(request.headers, 'Cookie');

//Before we used to remove the hash framgment because of Chrome do that but:
Expand Down Expand Up @@ -269,6 +299,20 @@ export function harFromMessages(messages, options) {
populateRedirectResponse(page, params, entries, options);
}

if (
options.allowMultiPage &&
params.type === 'Document' &&
page &&
page.__frameId ===
(rootFrameMappings.get(params.frameId) || params.frameId) &&
hasEntries(page, entries) &&
!params.redirectResponse
) {
page = addPage(params.frameId, request.url, {
__createdFromDocumentRequest: true
});
}

if (!page) {
log(
`Request will be sent with requestId ${params.requestId} that can't be mapped to any page at the moment.`
Expand All @@ -279,6 +323,7 @@ export function harFromMessages(messages, options) {
continue;
}

entry.pageref = page.id;
entries.push(entry);

// this is the first request for this page, so set timestamp of page.
Expand Down Expand Up @@ -431,7 +476,9 @@ export function harFromMessages(messages, options) {
const frameId =
rootFrameMappings.get(params.frameId) || params.frameId;
const page =
pages.find(page => page.__frameId === frameId) || pages.at(-1);
pages.find(page => page.id === entry.pageref) ||
pages.find(page => page.__frameId === frameId) ||
pages.at(-1);
if (!page) {
log(
`Received network response for requestId ${params.requestId} that can't be mapped to any page.`
Expand Down
123 changes: 123 additions & 0 deletions test/tests.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,70 @@ function sortedByRequestTime(entries) {
return entries.sort((e1, e2) => e1._requestTime - e2._requestTime);
}

function resourceMessages({
requestId,
frameId,
url,
timestamp,
wallTime,
type = 'Document'
}) {
return [
{
method: 'Network.requestWillBeSent',
params: {
requestId,
frameId,
loaderId: 'L1',
documentURL: url,
request: {
url,
method: 'GET',
headers: {},
initialPriority: type === 'Document' ? 'High' : 'Low'
},
timestamp,
wallTime,
initiator: { type: 'other' },
type
}
},
{
method: 'Network.responseReceived',
params: {
requestId,
frameId,
loaderId: 'L1',
timestamp: timestamp + 0.1,
type,
response: {
url,
status: 200,
statusText: 'OK',
headers: { 'content-type': 'text/html' },
mimeType: 'text/html',
fromDiskCache: false,
fromServiceWorker: false,
encodedDataLength: 100,
protocol: 'http/1.1',
connectionId: 1,
remoteIPAddress: '127.0.0.1',
timing: {
requestTime: timestamp,
sendStart: 0,
sendEnd: 1,
receiveHeadersEnd: 2
}
}
}
},
{
method: 'Network.loadingFinished',
params: { requestId, timestamp: timestamp + 0.2, encodedDataLength: 100 }
}
];
}

function testAllHARs(t, options) {
return perflogs().then(filenames => {
const promises = filenames.map(filename => {
Expand Down Expand Up @@ -338,6 +402,65 @@ test('Click on link in Chrome should create new page', t => {
});
});

test('Optionally creates pages for root frame navigations', t => {
const frameId = 'F1';
const messages = [
{ method: 'Page.frameStartedLoading', params: { frameId } },
...resourceMessages({
requestId: '1',
frameId,
url: 'https://example.com/page1.html',
timestamp: 1,
wallTime: 1_700_000_000
}),
...resourceMessages({
requestId: '2',
frameId,
url: 'https://example.com/page1.js',
timestamp: 1.3,
wallTime: 1_700_000_000.3,
type: 'Script'
}),
{
method: 'Page.frameScheduledNavigation',
params: { frameId, url: 'https://example.com/page2.html' }
},
...resourceMessages({
requestId: '3',
frameId,
url: 'https://example.com/page2.html',
timestamp: 2,
wallTime: 1_700_000_001
}),
{ method: 'Page.frameStartedLoading', params: { frameId } },
...resourceMessages({
requestId: '4',
frameId,
url: 'https://example.com/page2.js',
timestamp: 2.3,
wallTime: 1_700_000_001.3,
type: 'Script'
})
];

const defaultHar = harFromMessages(messages);
t.is(defaultHar.log.pages.length, 1);

const multiPageHar = harFromMessages(messages, { allowMultiPage: true });
t.deepEqual(
multiPageHar.log.pages.map(page => page.title),
['https://example.com/page1.html', 'https://example.com/page2.html']
);

const pagerefByUrl = Object.fromEntries(
multiPageHar.log.entries.map(entry => [entry.request.url, entry.pageref])
);
t.is(pagerefByUrl['https://example.com/page1.html'], 'page_1');
t.is(pagerefByUrl['https://example.com/page1.js'], 'page_1');
t.is(pagerefByUrl['https://example.com/page2.html'], 'page_2');
t.is(pagerefByUrl['https://example.com/page2.js'], 'page_2');
});

test('Includes pushed assets', t => {
const perflogPath = perflog('akamai-h2push.json');
return parsePerflog(perflogPath)
Expand Down