file level parallelization of the dataflow analysis

package.json

@@ -30,7 +30,8 @@
     "wiki:watch": "ts-node-dev src/cli/wiki.ts -- --keep-alive",
     "build": "tsc --project .",
     "build-dev": "npm run build && npm run build:copy-wasm",
-    "build:bundle-flowr": "npm run build && esbuild --bundle dist/src/cli/flowr.js --platform=node --tree-shaking=true --bundle --minify --external:clipboardy --target=node22 --outfile=dist/src/cli/flowr.min.js && npm run build:copy-wasm",
+    "build:bundle-flowr": "npm run build && esbuild --bundle dist/src/cli/flowr.js --platform=node --tree-shaking=true --bundle --minify --external:clipboardy --target=node22 --outfile=dist/src/cli/flowr.min.js && npm run build:bundle-worker && npm run build:copy-wasm",
+    "build:bundle-worker": "npm run build && esbuild --bundle dist/src/dataflow/parallel/worker.js --platform=node --tree-shaking=true --bundle --minify --external:clipboardy --target=node22 --outfile=dist/src/cli/worker.js",
     "build:copy-wasm": "mkdir -p dist/node_modules/@eagleoutice/tree-sitter-r/ && mkdir -p dist/node_modules/web-tree-sitter && cp node_modules/@eagleoutice/tree-sitter-r/tree-sitter-r.wasm dist/node_modules/@eagleoutice/tree-sitter-r/  && cp node_modules/web-tree-sitter/tree-sitter.wasm dist/node_modules/web-tree-sitter/",
     "lint-local": "npx eslint --version && npx eslint src/ test/ --rule \"no-warning-comments: off\"",
     "lint": "npm run license-compat -- --summary && npx eslint --version && npx eslint src/ test/",
@@ -177,8 +178,10 @@
     "@types/object-hash": "^3.0.6",
     "@types/seedrandom": "^3.0.8",
     "@types/semver": "^7.7.0",
+    "@types/serialize-javascript": "^5.0.4",
     "@types/tmp": "^0.2.6",
     "@types/ws": "^8.18.1",
+    "@types/wtfnode": "^0.10.0",
     "@typescript-eslint/eslint-plugin": "^8.40.0",
     "@vitest/coverage-v8": "^3.2.4",
     "esbuild": "^0.25.9",
@@ -210,15 +213,19 @@
     "n3": "^1.23.1",
     "object-hash": "^3.0.0",
     "object-sizeof": "^2.6.5",
+    "piscina": "^5.1.4",
     "rotating-file-stream": "^3.2.6",
     "seedrandom": "^3.0.5",
     "semver": "^7.7.1",
+    "serialize-javascript": "^7.0.2",
     "tar": "^7.4.3",
+    "tinypool": "^2.0.0",
     "tmp": "^0.2.3",
     "ts-essentials": "^10.1.1",
     "tslog": "^4.9.3",
     "web-tree-sitter": "^0.24.7",
     "ws": "^8.18.0",
+    "wtfnode": "^0.10.1",
     "xpath-ts2": "^1.4.2"
   }
 }

src/config.ts

@@ -7,6 +7,8 @@ import Joi from 'joi';
 import type { BuiltInDefinitions } from './dataflow/environments/built-in-config';
 import type { KnownParser } from './r-bridge/parser';
 import type { DeepWritable } from 'ts-essentials';
+import type { WorkerPoolSettings } from './dataflow/parallel/threadpool';
+import { WorkerpoolDefaultSettings } from './dataflow/parallel/threadpool';
 
 export enum VariableResolve {
 	/** Don't resolve constants at all */
@@ -188,6 +190,19 @@ export interface FlowrConfigOptions extends MergeableRecord {
 			}
 		}
 	}
+
+    readonly optimizations: {
+        readonly fileParallelization: boolean;
+
+        readonly dataflowOperationParallelization: boolean;
+
+        readonly deferredFunctionEvaluation: boolean;
+    }
+
+    readonly workerPool: {
+
+        readonly poolSettings: WorkerPoolSettings;
+    }
 }
 
 export interface TreeSitterEngineConfig extends MergeableRecord {
@@ -261,6 +276,14 @@ export const defaultConfigOptions: FlowrConfigOptions = {
 				maxReadLines:      1e6
 			}
 		}
+	},
+	optimizations: {
+		fileParallelization:              false,
+		dataflowOperationParallelization: false,
+		deferredFunctionEvaluation:       false,
+	},
+	workerPool: {
+		poolSettings: WorkerpoolDefaultSettings
 	}
 };
 
@@ -359,6 +382,7 @@ export function cloneConfig(config: FlowrConfigOptions): FlowrConfigOptions {
 	return JSON.parse(JSON.stringify(config)) as FlowrConfigOptions;
 }
 
+
 /**
  * Loads the flowr config from the given file or the default locations.
  */

src/core/steps/all/core/20-dataflow.ts

@@ -26,8 +26,15 @@ const staticDataflowCommon = {
 	dependencies: [ 'normalize' ],
 } as const;
 
-function processor(results: { normalize?: NormalizedAst }, input: { parser?: Parser<KnownParserType>, context?: FlowrAnalyzerContext }) {
-	return produceDataFlowGraph(input.parser as Parser<KnownParserType>, results.normalize as NormalizedAst, input.context as FlowrAnalyzerContext);
+function processor(
+	results: { normalize?: NormalizedAst },
+	input: { parser?: Parser<KnownParserType>,
+	context?:         FlowrAnalyzerContext,}) {
+	return produceDataFlowGraph(
+		input.parser as Parser<KnownParserType>,
+		results.normalize as NormalizedAst,
+		input.context as FlowrAnalyzerContext,
+	);
 }
 
 export const STATIC_DATAFLOW = {

src/dataflow/environments/built-in.ts

@@ -44,7 +44,7 @@ import type {
 	BuiltInFunctionDefinition,
 	BuiltInReplacementDefinition
 } from './built-in-config';
-import type { ReadOnlyFlowrAnalyzerContext } from '../../project/context/flowr-analyzer-context';
+import type { FlowrAnalyzerContext, ReadOnlyFlowrAnalyzerContext } from '../../project/context/flowr-analyzer-context';
 
 export type BuiltIn = `built-in:${string}`;
 
@@ -212,6 +212,88 @@ export type ConfigOfBuiltInMappingName<N extends BuiltInMappingName> = Parameter
 
 export type BuiltInMemory = Map<Identifier, IdentifierDefinition[]>
 
+type SerializedBuiltInEntry = {
+	__kind: 'builtin-entry';
+	name:   Identifier;
+	//definedAt: BuiltIn;
+};
+
+type SerializedIdentifierDefinition =
+    | IdentifierDefinition
+    | SerializedBuiltInEntry;
+
+export type SerializedBuiltInMemory =
+    Map<Identifier, SerializedIdentifierDefinition[]>;
+
+function isPureBuiltInEntry(
+	defs: readonly IdentifierDefinition[]
+): boolean {
+	return defs.every(
+		d =>
+			(d.type === ReferenceType.BuiltInFunction ||
+			 d.type === ReferenceType.BuiltInConstant) &&
+			isBuiltIn(d.definedAt)
+	);
+}
+
+function isSerializedBuiltinEntry(
+	def: SerializedIdentifierDefinition
+): def is SerializedBuiltInEntry {
+	return (
+		typeof def === 'object' &&
+		def !== null &&
+		(def as SerializedBuiltInEntry).__kind === 'builtin-entry'
+	);
+}
+
+
+/**
+ *
+ */
+export function serializeBuiltInMemory(mem: BuiltInMemory): SerializedBuiltInMemory{
+	const serMem = new Map<Identifier, SerializedIdentifierDefinition[]>();
+
+	for(const [id, defs] of mem.entries()){
+		if(isPureBuiltInEntry(defs)){
+			/** just save the name, to recover later */
+			serMem.set(id, [{
+				__kind: 'builtin-entry',
+				name:   id,
+			}]);
+		} else {
+			/** handle normally */
+			serMem.set(id, defs);
+		}
+	}
+	return serMem;
+}
+
+/**
+ *
+ */
+export function deserializeBuiltInMemory(
+	mem: SerializedBuiltInMemory,
+	ctx: FlowrAnalyzerContext
+): BuiltInMemory {
+	const deSerMem = new Map<Identifier, IdentifierDefinition[]>();
+	for( const [id,defs] of mem.entries()){
+		if(defs.length === 1 && isSerializedBuiltinEntry(defs[0])){
+			const builtInDefs = ctx.env.builtInEnvironment.memory.get(defs[0].name);
+
+			if(!builtInDefs){
+				console.warn('Could not recover builtin defs entry: ', defs[0]);
+				continue;
+			}
+			// convert to mutable data and clone to sever connection to readonly object
+			deSerMem.set(id, defs.map(def => ({ ...def })) as IdentifierDefinition[]);
+		} else {
+			deSerMem.set(id, defs as IdentifierDefinition[]);
+		}
+
+	}
+	return deSerMem;
+}
+
 export class BuiltIns {
 	/**
 	 * Register a built-in constant (like `NULL` or `TRUE`) to the given {@link builtIns}