Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/field-name-transformation-performance.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"slonik-interceptor-field-name-transformation": minor
---

improve `transformRow` performance: rows are copied by a function that is compiled once per set of fields into an object literal, instead of being assembled one property at a time on a null-prototype object (which V8 keeps in slower dictionary mode). Transforming 200,000 rows of 12 fields is ~25x faster, and reading fields of the transformed rows ~12x faster. Rows are now plain objects, and rows whose field names do not need to be transformed are returned as they are. Also fixes reusing the transformation of a different set of fields when field names contain commas, or when `test` depends on the field type.
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ connection.any(sql`
`);
```

Field name transformation uses the `afterQuery` interceptor method to format field names.
Field name transformation uses the `transformRow` interceptor method to format field names. Rows are copied by a function that is compiled once per set of fields, and rows whose field names do not need to be transformed are returned as they are.

## Installation

Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { createFieldNameTransformationInterceptor } from "./createFieldNameTransformationInterceptor.js";
import test from "ava";
import type { QueryContext } from "slonik";
import type { Field, QueryContext, QueryResultRow } from "slonik";

const createQueryContext = (): QueryContext => {
return {
Expand All @@ -18,37 +18,169 @@ const createQueryContext = (): QueryContext => {
} as unknown as QueryContext;
};

test("transforms field names to camelcase", (t) => {
const interceptor = createFieldNameTransformationInterceptor({
test: (field) => {
return /^[\d_a-z]+$/u.test(field.name);
},
const createFields = (names: readonly string[], dataTypeId = 25): Field[] => {
return names.map((name) => {
return {
dataTypeId,
name,
};
});
};

const { transformRow } = interceptor;
const createTransformRow = (isTransformed: (field: Field) => boolean) => {
const { transformRow } = createFieldNameTransformationInterceptor({
test: isTransformed,
});

if (!transformRow) {
throw new Error("Unexpected state.");
}

const result = transformRow(
createQueryContext(),
{
sql: "SELECT 1",
values: [],
},
{
foo_bar: 1,
},
[
// Each call simulates a new query.
return (row: QueryResultRow, fields: readonly Field[]) => {
return transformRow(
createQueryContext(),
{
dataTypeId: 1,
name: "foo_bar",
sql: "SELECT 1",
values: [],
},
],
row,
fields,
);
};
};

const isSnakeCase = (field: Field) => {
return /^[\d_a-z]+$/u.test(field.name);
};

test("transforms field names to camelcase", (t) => {
const transformRow = createTransformRow(isSnakeCase);

t.deepEqual(transformRow({ foo_bar: 1 }, createFields(["foo_bar"])), {
fooBar: 1,
});
});

test("does not transform fields that do not pass the test", (t) => {
const transformRow = createTransformRow((field) => {
return field.name !== "__typename" && isSnakeCase(field);
});

t.deepEqual(
transformRow(
{ __typename: "person", full_name: "John Doe", id: 1 },
createFields(["id", "full_name", "__typename"]),
),
{
__typename: "person",
fullName: "John Doe",
id: 1,
},
);
});

test("produces plain objects", (t) => {
const transformRow = createTransformRow(isSnakeCase);

const result = transformRow({ foo_bar: 1 }, createFields(["foo_bar"]));

t.is(Object.getPrototypeOf(result), Object.prototype);
});

test("returns the row as it is when no field name needs to be transformed", (t) => {
const transformRow = createTransformRow(isSnakeCase);

const row = { foo: 1 };

t.is(transformRow(row, createFields(["foo"])), row);
});

test("defines a __proto__ field as a property instead of setting the prototype", (t) => {
const transformRow = createTransformRow((field) => {
return field.name !== "__proto__" && isSnakeCase(field);
});

const row = JSON.parse('{"__proto__": {"polluted": true}, "foo_bar": 1}');

const result = transformRow(row, createFields(["__proto__", "foo_bar"]));

t.is(Object.getPrototypeOf(result), Object.prototype);
t.deepEqual(Object.keys(result), ["__proto__", "fooBar"]);
t.deepEqual(Object.getOwnPropertyDescriptor(result, "__proto__")?.value, { polluted: true });
});

test("handles field names that contain special characters", (t) => {
const transformRow = createTransformRow(isSnakeCase);

const names = [
"foo_bar",
'a"b',
"c\\d",
"e\nf",
"g\u2028h",
'"]; throw new Error("injected"); //',
];

const result = transformRow(
Object.fromEntries(
names.map((name, index) => {
return [name, index];
}),
),
createFields(names),
);

t.deepEqual(result, {
'"]; throw new Error("injected"); //': 5,
'a"b': 1,
"c\\d": 2,
"e\nf": 3,
fooBar: 0,
"g\u2028h": 4,
});
});

test("does not reuse the mapping of fields whose names join to the same string", (t) => {
const transformRow = createTransformRow(isSnakeCase);

t.deepEqual(transformRow({ "foo_bar,baz_qux": 1 }, createFields(["foo_bar,baz_qux"])), {
"foo_bar,baz_qux": 1,
});

t.deepEqual(transformRow({ baz_qux: 2, foo_bar: 1 }, createFields(["foo_bar", "baz_qux"])), {
bazQux: 2,
fooBar: 1,
});
});

test("does not reuse the mapping of fields with the same names but different types", (t) => {
const transformRow = createTransformRow((field) => {
return field.dataTypeId === 25;
});

t.deepEqual(transformRow({ foo_bar: "a" }, createFields(["foo_bar"], 25)), {
fooBar: "a",
});

t.deepEqual(transformRow({ foo_bar: 1 }, createFields(["foo_bar"], 23)), {
foo_bar: 1,
});
});

test("tests the fields once per set of fields", (t) => {
let calls = 0;

const transformRow = createTransformRow((field) => {
calls++;

return isSnakeCase(field);
});

const fields = createFields(["foo_bar", "baz_qux"]);

transformRow({ baz_qux: 2, foo_bar: 1 }, fields);
transformRow({ baz_qux: 4, foo_bar: 3 }, fields);

t.is(calls, 2);
});
Original file line number Diff line number Diff line change
@@ -1,51 +1,127 @@
import camelcase from "camelcase";
import type { Field, Interceptor, QueryResultRow } from "slonik";

type RowMapper = (row: QueryResultRow) => QueryResultRow;

/**
* [Row field name, transformed field name] pairs, in the order of the row's fields.
*/
type FieldNames = ReadonlyArray<readonly [string, string]>;

// Used for sets of fields whose names do not need to be transformed.
const passthroughRowMapper: RowMapper = (row) => {
return row;
};

/**
* Compiles a function that copies a row into an object literal with the transformed field names.
* V8 allocates an object literal with all of its properties in a single step and in a fixed shape
* ("fast properties"), which is much cheaper to create and to read than an object that is
* assembled one property at a time.
*/
const compileRowMapper = (fieldNames: FieldNames): RowMapper => {
const properties = fieldNames.map(([name, transformedName]) => {
// The field names are embedded as JSON string literals, so they cannot inject code.
// A literal `"__proto__": value` property would set the prototype; a computed key does not.
const key =
transformedName === "__proto__"
? "[" + JSON.stringify(transformedName) + "]"
: JSON.stringify(transformedName);

return key + ":row[" + JSON.stringify(name) + "]";
});

// oxlint-disable-next-line no-new-func
return new Function("row", "return {" + properties.join(",") + "};") as RowMapper;
};

// Used where code generation from strings is not allowed.
const createRowMapper = (fieldNames: FieldNames): RowMapper => {
return (row) => {
const result: QueryResultRow = {};

for (const [name, transformedName] of fieldNames) {
if (transformedName === "__proto__") {
Object.defineProperty(result, transformedName, {
configurable: true,
enumerable: true,
value: row[name],
writable: true,
});
} else {
result[transformedName] = row[name];
}
}

return result;
};
};

const createRowMapperForFields = (
row: QueryResultRow,
fields: readonly Field[],
test: (field: Field) => boolean,
): RowMapper => {
const transformedNames = new Map<string, string>();

for (const field of fields) {
transformedNames.set(field.name, test(field) ? camelcase(field.name) : field.name);
}

const fieldNames: FieldNames = Object.keys(row).map((name): readonly [string, string] => {
return [name, transformedNames.get(name) ?? name];
});

if (
fieldNames.every(([name, transformedName]) => {
return name === transformedName;
})
) {
return passthroughRowMapper;
}

try {
return compileRowMapper(fieldNames);
} catch {
return createRowMapper(fieldNames);
}
};

export const createFieldNameTransformationInterceptor = ({
test,
}: {
test: (field: Field) => boolean;
}): Interceptor => {
const cachedMappers = new Map<string, (row: Record<string, unknown>) => QueryResultRow>();
// Row mappers are created once per set of fields and reused for every row of every query that
// produces the same set of fields.
const rowMappers = new Map<string, RowMapper>();

return {
name: "slonik-interceptor-field-name-transformation",
// eslint-disable-next-line @typescript-eslint/no-explicit-any
transformRow: (context: any, query, row, fields) => {
let mapper = context.sandbox.mapper;

if (!mapper) {
const mapperKey = fields.map((field) => field.name).join(",");

mapper = cachedMappers.get(mapperKey);

if (!mapper) {
// Create object with null prototype for slightly better performance
const fieldMapping = Object.create(null);

for (const field of fields) {
fieldMapping[field.name] = test(field) ? camelcase(field.name) : field.name;
}

const keys = Object.keys(row);
transformRow: (context, query, row, fields) => {
let rowMapper = context.sandbox.fieldNameTransformationRowMapper as RowMapper | undefined;

mapper = (currentRow: Record<string, unknown>) => {
const result = Object.create(null);
if (!rowMapper) {
// `test` receives the whole field, so the key includes the field types. Field names cannot
// contain the NUL character, which makes the key unambiguous.
const key = fields
.map((field) => {
return field.name + ":" + String(field.dataTypeId);
})
.join("\u0000");

for (const key of keys) {
result[fieldMapping[key]] = currentRow[key];
}
rowMapper = rowMappers.get(key);

return result;
};
if (!rowMapper) {
rowMapper = createRowMapperForFields(row, fields, test);

cachedMappers.set(mapperKey, mapper);
rowMappers.set(key, rowMapper);
}

context.sandbox.mapper = mapper;
context.sandbox.fieldNameTransformationRowMapper = rowMapper;
}

return mapper(row);
return rowMapper(row);
},
};
};
Loading