Skip to content

wip: lazy enrichments #2479

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Ellpeck wants to merge 1 commit into
base: main
Choose a base branch
from
+162 −130
Draft

wip: lazy enrichments #2479

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 36 additions & 33 deletions src/linter/rules/absolute-path.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,15 +136,15 @@ export const ABSOLUTE_PATH = {
}
return q.unique();
},
processSearchResult: (elements, config, data): { results: AbsoluteFilePathResult[], '.meta': AbsoluteFilePathMetadata } => {
processSearchResult: async(elements, config, data): Promise<{ results: AbsoluteFilePathResult[], '.meta': AbsoluteFilePathMetadata }> => {
const metadata: AbsoluteFilePathMetadata = {
totalConsidered: 0,
totalUnknown: 0
};
const queryResults = elements.enrichmentContent(Enrichment.QueryData)?.queries;
const queryResults = (await elements.enrichmentContent(Enrichment.QueryData))?.queries;
const regex = config.absolutePathRegex ? new RegExp(config.absolutePathRegex) : undefined;
return {
results: elements.getElements().flatMap(element => {
results: (await Promise.all(elements.getElements().map(async element => {
metadata.totalConsidered++;
const node = element.node;
const wd = inferWd(node.info.file, config.useAsWd);
Expand All @@ -159,43 +159,46 @@ export const ABSOLUTE_PATH = {
} else {
return [];
}
} else if(enrichmentContent(element, Enrichment.QueryData)) {
const result = queryResults[enrichmentContent(element, Enrichment.QueryData).query] as QueryResults<'dependencies'>['dependencies'];
const mappedStrings = result.read.filter(r => r.value !== undefined && r.value !== Unknown && isAbsolutePath(r.value, regex)).map(r => {
const elem = data.normalize.idMap.get(r.nodeId);
return {
certainty: LintingResultCertainty.Certain,
filePath: r.value,
loc: elem ? SourceLocation.fromNode(elem) ?? SourceLocation.invalid() : SourceLocation.invalid(),
quickFix: buildQuickFix(elem, r.value as string, wd)
};
});
if(mappedStrings.length > 0) {
return mappedStrings;
} else if(result.read.every(r => r.value !== Unknown)) {
// if we have no absolute paths, but all paths are known, we can return an empty array
return [];
}
} else {
const dfNode = data.dataflow.graph.getVertex(node.info.id);
if(isFunctionCallVertex(dfNode)) {
const handler = dfNode.name ? PathFunctions.get(dfNode.name) : undefined;
const strings = handler ? handler(data.dataflow.graph, dfNode, data.analyzer.inspectContext()) : [];
if(strings) {
return strings.filter(s => isAbsolutePath(s, regex)).map(str => ({
certainty: LintingResultCertainty.Uncertain,
filePath: str,
loc: SourceLocation.fromNode(element.node) ?? SourceLocation.invalid(),
quickFix: undefined
}));
const queryData = await enrichmentContent(element, Enrichment.QueryData);
if(queryData) {
const result = queryResults[queryData.query] as QueryResults<'dependencies'>['dependencies'];
const mappedStrings = result.read.filter(r => r.value !== undefined && r.value !== Unknown && isAbsolutePath(r.value, regex)).map(r => {
const elem = data.normalize.idMap.get(r.nodeId);
return {
certainty: LintingResultCertainty.Certain,
filePath: r.value,
loc: elem ? SourceLocation.fromNode(elem) ?? SourceLocation.invalid() : SourceLocation.invalid(),
quickFix: buildQuickFix(elem, r.value as string, wd)
};
});
if(mappedStrings.length > 0) {
return mappedStrings;
} else if(result.read.every(r => r.value !== Unknown)) {
// if we have no absolute paths, but all paths are known, we can return an empty array
return [];
}
} else {
const dfNode = data.dataflow.graph.getVertex(node.info.id);
if(isFunctionCallVertex(dfNode)) {
const handler = dfNode.name ? PathFunctions.get(dfNode.name) : undefined;
const strings = handler ? handler(data.dataflow.graph, dfNode, data.analyzer.inspectContext()) : [];
if(strings) {
return strings.filter(s => isAbsolutePath(s, regex)).map(str => ({
certainty: LintingResultCertainty.Uncertain,
filePath: str,
loc: SourceLocation.fromNode(element.node) ?? SourceLocation.invalid(),
quickFix: undefined
}));
}
}
// check whether the df node is a function call that returns a file path
}
// check whether the df node is a function call that returns a file path
}

metadata.totalUnknown++;
return undefined;
}).filter(isNotUndefined).map(r => compactRecord(r) as AbsoluteFilePathResult),
}))).flat().filter(isNotUndefined).map(r => compactRecord(r) as AbsoluteFilePathResult),
'.meta': metadata
};
},
Expand Down
25 changes: 14 additions & 11 deletions src/linter/rules/dead-code.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,23 +36,26 @@ export const DEAD_CODE = {
checkReachable: true,
simplificationPasses: config.simplificationPasses ?? [...DefaultCfgSimplificationOrder, 'analyze-dead-code']
}),
processSearchResult: (elements, _config, _data) => {
processSearchResult: async(elements, _config, _data) => {
const meta: DeadCodeMetadata = {
consideredNodes: 0
};
return {
results: combineResults(
elements.getElements()
.filter(element => {
(await Promise.all(elements.getElements()
.map(async element => {
meta.consideredNodes++;
const cfgInformation = enrichmentContent(element, Enrichment.CfgInformation);
return element.node.info.role !== RoleInParent.ExpressionListGrouping && !cfgInformation.isReachable;
})
.map(element => ({
certainty: LintingResultCertainty.Certain,
involvedId: element.node.info.id,
loc: SourceLocation.fromNode(element.node)
}))
const cfgInformation = await enrichmentContent(element, Enrichment.CfgInformation);
if(element.node.info.role !== RoleInParent.ExpressionListGrouping && !cfgInformation.isReachable){
return [({
certainty: LintingResultCertainty.Certain,
involvedId: element.node.info.id,
loc: SourceLocation.fromNode(element.node)
})];
} else {
return [];
}
}))).flat()
.filter(element => isNotUndefined(element.loc)) as Writable<DeadCodeResult>[]
),
'.meta': meta
Expand Down
6 changes: 3 additions & 3 deletions src/linter/rules/file-path-validity.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,15 +46,15 @@ export const FILE_PATH_VALIDITY = {
readFunctions: config.additionalReadFunctions,
writeFunctions: config.additionalWriteFunctions
}).with(Enrichment.CfgInformation),
processSearchResult: (elements, config, data): { results: FilePathValidityResult[], '.meta': FilePathValidityMetadata } => {
const cfg = elements.enrichmentContent(Enrichment.CfgInformation).cfg.graph;
processSearchResult: async(elements, config, data): Promise<{ results: FilePathValidityResult[], '.meta': FilePathValidityMetadata }> => {
const cfg = (await elements.enrichmentContent(Enrichment.CfgInformation)).cfg.graph;
const metadata: FilePathValidityMetadata = {
totalReads: 0,
totalUnknown: 0,
totalWritesBeforeAlways: 0,
totalValid: 0
};
const results = elements.enrichmentContent(Enrichment.QueryData).queries['dependencies'];
const results = (await elements.enrichmentContent(Enrichment.QueryData)).queries['dependencies'];
return {
results: elements.getElements().flatMap(element => {
const matchingRead = results.read.find(r => r.nodeId === element.node.info.id);
Expand Down
12 changes: 5 additions & 7 deletions src/linter/rules/function-finder-util.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ export const functionFinderUtil = {
})
);
},
processSearchResult: <T extends FlowrSearchElement<ParentInformation>[]>(
processSearchResult: async <T extends FlowrSearchElement<ParentInformation>[]>(
elements: FlowrSearchElements<ParentInformation, T>,
_config: unknown,
_data: unknown,
Expand All @@ -62,10 +62,10 @@ export const functionFinderUtil = {
totalFunctionDefinitions: 0
};

const results = refineSearch(elements.getElements())
.flatMap(element => {
const results = (await Promise.all(refineSearch(elements.getElements())
.map(async element => {
metadata.totalCalls++;
return enrichmentContent(element, Enrichment.CallTargets).targets.map(target => {
return (await enrichmentContent(element, Enrichment.CallTargets)).targets.map(target => {
metadata.totalFunctionDefinitions++;
return {
node: element.node,
Expand All @@ -74,7 +74,7 @@ export const functionFinderUtil = {
certainty: element.certainty
};
});
});
}))).flat();

return {
results:
Expand Down Expand Up @@ -127,5 +127,3 @@ export const functionFinderUtil = {
return Ternary.Never;
}
};


31 changes: 17 additions & 14 deletions src/linter/rules/seeded-randomness.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ export const SEEDED_RANDOMNESS = {
{ callName: config.randomnessProducers.filter(p => p.type === 'function').map(p => p.name) },
{ callName: getDefaultAssignments().flatMap(b => b.names).map(Identifier.getName), cascadeIf: () => CascadeAction.Continue }
]),
processSearchResult: (elements, config, { dataflow, analyzer }) => {
processSearchResult: async(elements, config, { dataflow, analyzer }) => {
const assignmentProducers = new Set<string>(config.randomnessProducers.filter(p => p.type === 'assignment').map(p => p.name));
const assignmentArgIndexes = new Map<string, number>(getDefaultAssignments().flatMap(a => a.names.map(n => ([Identifier.getName(n), a.config?.swapSourceAndTarget ? 1 : 0]))));
const metadata: SeededRandomnessMeta = {
Expand All @@ -81,22 +81,25 @@ export const SEEDED_RANDOMNESS = {
callsWithOtherBranchProducers: 0
};
return {
results: elements.getElements()
results: (await Promise.all((await Promise.all(elements.getElements()
// map and filter consumers
.flatMap(element => enrichmentContent(element, Enrichment.CallTargets).targets.map(target => {
metadata.consumerCalls++;
return {
involvedId: element.node.info.id,
loc: SourceLocation.fromNode(element.node) ?? SourceLocation.invalid(),
target: target as BrandedIdentifier,
searchElement: element
};
}))
.map(async element => {
const content = await enrichmentContent(element, Enrichment.CallTargets);
return content.targets.map(target => {
metadata.consumerCalls++;
return {
involvedId: element.node.info.id,
loc: SourceLocation.fromNode(element.node) ?? SourceLocation.invalid(),
target: target as BrandedIdentifier,
searchElement: element
};
});
}))).flat()
// filter by calls that aren't preceded by a randomness producer
.flatMap(element => {
.map(async element => {
const dfgElement = dataflow.graph.getVertex(element.searchElement.node.info.id);
const cds = dfgElement ? new Set(dfgElement.cds) : new Set();
const producers = enrichmentContent(element.searchElement, Enrichment.LastCall).linkedIds
const producers = (await enrichmentContent(element.searchElement, Enrichment.LastCall)).linkedIds
.map(e => dataflow.graph.getVertex(e.node.info.id) as DataflowGraphVertexFunctionCall);
const { assignment, func } = Object.groupBy(producers, f => assignmentArgIndexes.has(Identifier.getName(f.name)) ? 'assignment' : 'func');
let nonConstant = false;
Expand Down Expand Up @@ -155,7 +158,7 @@ export const SEEDED_RANDOMNESS = {
function: element.target,
loc: element.loc
}];
}),
}))).flat(),
'.meta': metadata
};
},
Expand Down
35 changes: 18 additions & 17 deletions src/search/flowr-search-filters.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import { Enrichment, enrichmentContent } from './search-executor/search-enricher
import type { DataflowInformation } from '../dataflow/info';
import { Identifier } from '../dataflow/environments/identifier';
import type { BuiltInProcName } from '../dataflow/environments/built-in-proc-name';
import type { AsyncOrSync } from 'ts-essentials';

export type FlowrFilterName = keyof typeof FlowrFilters;
interface FlowrFilterWithArgs<Filter extends FlowrFilterName, Args extends FlowrFilterArgs<Filter>> {
Expand All @@ -32,7 +33,7 @@ export enum FlowrFilter {
*/
OriginKind = 'origin-kind'
}
export type FlowrFilterFunction <T> = (e: FlowrSearchElement<ParentInformation>, args: T, data: { dataflow: DataflowInformation }) => boolean;
export type FlowrFilterFunction <T> = (e: FlowrSearchElement<ParentInformation>, args: T, data: { dataflow: DataflowInformation }) => AsyncOrSync<boolean>;

export const ValidFlowrFilters: Set<string> = new Set(Object.values(FlowrFilter));
export const ValidFlowrFiltersReverse = Object.fromEntries(Object.entries(FlowrFilter).map(([k, v]) => [v, k]));
Expand All @@ -41,9 +42,9 @@ export const FlowrFilters = {
[FlowrFilter.DropEmptyArguments]: ((e: FlowrSearchElement<ParentInformation>, _args: never) => {
return e.node.type !== RType.Argument || e.node.name !== undefined;
}) satisfies FlowrFilterFunction<never>,
[FlowrFilter.MatchesEnrichment]: ((e: FlowrSearchElement<ParentInformation>, args: MatchesEnrichmentArgs<Enrichment>) => {
[FlowrFilter.MatchesEnrichment]: (async(e: FlowrSearchElement<ParentInformation>, args: MatchesEnrichmentArgs<Enrichment>) => {
if(args.enrichment === Enrichment.CallTargets) {
const c: CallTargetsContent = enrichmentContent(e, Enrichment.CallTargets);
const c: CallTargetsContent = await enrichmentContent(e, Enrichment.CallTargets);
if(c === undefined || c.targets === undefined) {
return false;
}
Expand All @@ -57,7 +58,7 @@ export const FlowrFilters = {
}
return false;
} else {
const content = JSON.stringify(enrichmentContent(e, args.enrichment));
const content = JSON.stringify(await enrichmentContent(e, args.enrichment));
return content !== undefined && args.test.test(content);
}
}) satisfies FlowrFilterFunction<MatchesEnrichmentArgs<Enrichment>>,
Expand Down Expand Up @@ -241,19 +242,19 @@ interface FilterData {
}

const evalVisit = {
and: ({ left, right }: BooleanBinaryNode<BooleanNode>, data: FilterData) =>
evalTree(left, data) && evalTree(right, data),
or: ({ left, right }: BooleanBinaryNode<BooleanNode>, data: FilterData) =>
evalTree(left, data) || evalTree(right, data),
xor: ({ left, right }: BooleanBinaryNode<BooleanNode>, data: FilterData) =>
evalTree(left, data) !== evalTree(right, data),
not: ({ operand }: BooleanUnaryNode<BooleanNode>, data: FilterData) =>
!evalTree(operand, data),
and: async({ left, right }: BooleanBinaryNode<BooleanNode>, data: FilterData) =>
(await evalTree(left, data)) && (await evalTree(right, data)),
or: async({ left, right }: BooleanBinaryNode<BooleanNode>, data: FilterData) =>
(await evalTree(left, data)) || (await evalTree(right, data)),
xor: async({ left, right }: BooleanBinaryNode<BooleanNode>, data: FilterData) =>
(await evalTree(left, data)) !== (await evalTree(right, data)),
not: async({ operand }: BooleanUnaryNode<BooleanNode>, data: FilterData) =>
!(await evalTree(operand, data)),
'r-type': ({ value }: LeafRType, { element }: FilterData) =>
element.node.type === value,
'vertex-type': ({ value }: LeafVertexType, { data, element }: FilterData) =>
data.dataflow.graph.getVertex(element.node.info.id)?.tag === value,
'special': ({ value }: LeafSpecial, { data, element }: FilterData) => {
'special': async({ value }: LeafSpecial, { data, element }: FilterData) => {
const name = typeof value === 'string' ? value : value.name;
const args = typeof value === 'string' ? undefined as unknown as FlowrFilterArgs<FlowrFilter> : value.args;
const getHandler = FlowrFilters[name];
Expand All @@ -264,17 +265,17 @@ const evalVisit = {
}
};

function evalTree(tree: BooleanNode, data: FilterData): boolean {
async function evalTree(tree: BooleanNode, data: FilterData): Promise<boolean> {
/* we ensure that the types fit */
return evalVisit[tree.type](tree as never, data);
}

/**
* Evaluates the given filter expression against the provided data.
*/
export function evalFilter<Filter extends FlowrFilter>(filter: FlowrFilterExpression<Filter>, data: FilterData): boolean {
export async function evalFilter<Filter extends FlowrFilter>(filter: FlowrFilterExpression<Filter>, data: FilterData): Promise<boolean> {
if(filter instanceof FlowrFilterCombinator) {
return evalTree(filter.get(), data);
return await evalTree(filter.get(), data);
} else if(typeof filter === 'string' && ValidFlowrFilters.has(filter)) {
const handler = FlowrFilters[filter as FlowrFilter];
return handler(data.element, undefined as unknown as FlowrFilterArgs<FlowrFilter>, data.data);
Expand All @@ -284,6 +285,6 @@ export function evalFilter<Filter extends FlowrFilter>(filter: FlowrFilterExpres
return handler(data.element, args, data.data);
} else {
const tree = FlowrFilterCombinator.is(filter as FlowrFilterExpression);
return evalTree(tree.get(), data);
return await evalTree(tree.get(), data);
}
}
21 changes: 13 additions & 8 deletions src/search/flowr-search.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,9 @@ import {
type EnrichmentElementContent,
type EnrichmentSearchArguments,
type EnrichmentSearchContent
, Enrichments } from './search-executor/search-enrichers';
, Enrichments,
LazyEnrichmentContent
} from './search-executor/search-enrichers';
import type { ReadonlyFlowrAnalysisProvider } from '../project/flowr-analyzer';

/**
Expand All @@ -17,7 +19,7 @@ import type { ReadonlyFlowrAnalysisProvider } from '../project/flowr-analyzer';
*/
export interface FlowrSearchElement<Info> {
readonly node: RNode<Info>;
readonly enrichments?: { [E in Enrichment]?: EnrichmentElementContent<E> }
readonly enrichments?: { [E in Enrichment]?: LazyEnrichmentContent<EnrichmentElementContent<E>> }
}

export interface FlowrSearchNodeBase<Type extends string, Name extends string, Args extends Record<string, unknown> | undefined> {
Expand Down Expand Up @@ -70,7 +72,7 @@ export interface FlowrSearchGetFilter extends Record<string, unknown> {
/** Intentionally, we abstract away from an array to avoid the use of conventional typescript operations */
export class FlowrSearchElements<Info = NoInfo, Elements extends FlowrSearchElement<Info>[] = FlowrSearchElement<Info>[]> {
private elements: Elements = [] as unknown as Elements;
private enrichments: { [E in Enrichment]?: EnrichmentSearchContent<E> } = {};
private enrichments: { [E in Enrichment]?: LazyEnrichmentContent<EnrichmentSearchContent<E>> } = {};

public constructor(elements?: Elements) {
if(elements) {
Expand Down Expand Up @@ -112,18 +114,21 @@ export class FlowrSearchElements<Info = NoInfo, Elements extends FlowrSearchElem
*
* Please note that this function does not also enrich individual elements, which is done through {@link enrichElement}. Both functions are called in a concise manner in {@link FlowrSearchBuilder.with}, which is the preferred way to add enrichments to a search.
*/
public async enrich<E extends Enrichment>(data: ReadonlyFlowrAnalysisProvider, enrichment: E, args?: EnrichmentSearchArguments<E>): Promise<this> {
public enrich<E extends Enrichment>(data: ReadonlyFlowrAnalysisProvider, enrichment: E, args?: EnrichmentSearchArguments<E>): this {
const enrichmentData = Enrichments[enrichment] as unknown as EnrichmentData<EnrichmentElementContent<E>, EnrichmentElementArguments<E>, EnrichmentSearchContent<E>, EnrichmentSearchArguments<E>>;
if(enrichmentData.enrichSearch !== undefined) {
const enricher = enrichmentData.enrichSearch;
if(enricher !== undefined) {
const prevEnrichment = this.enrichments?.[enrichment];
this.enrichments = {
...this.enrichments ?? {},
[enrichment]: await enrichmentData.enrichSearch(this as FlowrSearchElements<ParentInformation>, data, args, this.enrichments?.[enrichment])
[enrichment]: new LazyEnrichmentContent<EnrichmentSearchContent<E>>(async() =>
await enricher(this as FlowrSearchElements<ParentInformation>, data, args, prevEnrichment && (() => prevEnrichment.get())))
};
}
return this;
}

public enrichmentContent<E extends Enrichment>(enrichment: E): EnrichmentSearchContent<E> {
return this.enrichments?.[enrichment] as EnrichmentSearchContent<E>;
public async enrichmentContent<E extends Enrichment>(enrichment: E): Promise<EnrichmentSearchContent<E>> {
return await (this.enrichments?.[enrichment] as LazyEnrichmentContent<EnrichmentSearchContent<E>>)?.get();
}
}
Loading
Loading