diff --git a/src/cli/wiki.ts b/src/cli/wiki.ts index c932112d197..6fbdcebd3b4 100644 --- a/src/cli/wiki.ts +++ b/src/cli/wiki.ts @@ -1,16 +1,5 @@ -import { makeDocContextForTypes } from '../documentation/wiki-mk/doc-context'; -import { RShell } from '../r-bridge/shell'; -import { TreeSitterExecutor } from '../r-bridge/lang-4.x/tree-sitter/tree-sitter-executor'; -import type { DocMakerArgs, DocMakerLike, DocMakerOutputArgs } from '../documentation/wiki-mk/doc-maker'; -import fs from 'fs'; -import { setMinLevelOfAllLogs } from '../../test/functionality/_helper/log'; -import { LogLevel } from '../util/log'; -import type { OptionDefinition } from 'command-line-usage'; -import commandLineUsage from 'command-line-usage'; -import commandLineArgs from 'command-line-args'; import { flowrVersion } from '../util/version'; import { WikiFaq } from '../documentation/wiki-faq'; -import { ansiFormatter, ColorEffect, Colors, FontStyles } from '../util/text/ansi'; import { DocCapabilities, WikiCore, WikiDataflowGraph, @@ -26,10 +15,12 @@ import { WikiAnalyzer } from '../documentation/wiki-analyzer'; import { IssueLintingRule } from '../documentation/issue-linting-rule'; import { DocReadme } from '../documentation/doc-readme'; import { WikiLinter } from '../documentation/wiki-linter'; -import os from 'os'; import { WikiSetup } from '../documentation/wiki-setup'; import { WikiOverview } from '../documentation/wiki-overview'; import { WikiAbsint } from '../documentation/wiki-absint'; +import type { DocMakerLike } from '../documentation/wiki-mk/doc-maker'; +import { FlowrRefs } from '../documentation/doc-util/doc-files'; +import { wikiCli } from '../documentation/wiki-mk/cli'; export const AllWikiDocuments = [ new WikiFaq(), @@ -53,137 +44,13 @@ export const AllWikiDocuments = [ new DocCapabilities() ] as const satisfies DocMakerLike[]; -export type ValidWikiDocumentTargets = ReturnType; -export type ValidWikiDocumentTargetsNoSuffix = ValidWikiDocumentTargets extends `${infer Name}.${string}` ? Name : never; - -function sortByLeastRecentChanged(wikis: DocMakerLike[]): DocMakerLike[] { - return wikis.slice().sort((a, b) => { - const aStat = fs.existsSync(a.getProducer()) ? fs.statSync(a.getProducer()) : undefined; - const bStat = fs.existsSync(b.getProducer()) ? fs.statSync(b.getProducer()) : undefined; - const aMTime = aStat ? aStat.mtime.getTime() : 0; - const bMTime = bStat ? bStat.mtime.getTime() : 0; - return bMTime - aMTime; - }); -} - -/** - * Updates and optionally re-creates all flowR wikis. - */ -export async function makeAllWikis(force: boolean, filter: string[] | undefined) { - const setupStart = new Date(); - console.log('Setting up wiki generation...'); - const shell = new RShell(); - console.log(' * R shell initialized'); - await TreeSitterExecutor.initTreeSitter(); - const treeSitter = new TreeSitterExecutor(); - console.log(' * Tree-sitter parser initialized'); - const ctx = makeDocContextForTypes(shell); - console.log(' * Wiki context prepared'); - if(force) { - console.log(ansiFormatter.format('Forcing wiki regeneration (existing files will be overwritten)', { style: FontStyles.Bold, color: Colors.Yellow, effect: ColorEffect.Foreground })); - } - const info: DocMakerArgs & DocMakerOutputArgs = { - ctx, - shell, treeSitter, - force, - readFileSync(f: fs.PathLike) { - try { - return fs.readFileSync(f); - } catch{ - return undefined; - } - }, - writeFileSync: fs.writeFileSync - }; - - console.log(`Setup for wiki generation took ${(new Date().getTime() - setupStart.getTime())}ms`); - const changedWikis = new Set(); - try { - const sortedDocs = sortByLeastRecentChanged(AllWikiDocuments); - console.log(`Generating ${sortedDocs.length} wikis/docs, sorted by most recently updated...`); - for(const doc of sortedDocs) { - const type = doc.getTarget().toLowerCase().includes('wiki') ? 'Wiki' : 'Doc'; - if(filter && !filter.some(f => doc.getTarget().includes(f))) { - console.log(` * Skipping ${type} (filtered out): ${doc.getTarget()}`); - continue; - } - const now = new Date(); - console.log(ansiFormatter.format(` [${doc.getTarget()}] Updating ${type}...`, { style: FontStyles.Bold, color: Colors.Cyan, effect: ColorEffect.Foreground })); - const changed = await doc.make(info); - const text = changed ? `${type} updated` : `${type} identical, no changes made`; - if(changed) { - changedWikis.add(doc.getTarget()); - } - const color = changed ? Colors.Green : Colors.White; - console.log(ansiFormatter.format(` [${doc.getTarget()}] ${text}: ${doc.getTarget()} (took ${new Date().getTime() - now.getTime()}ms)`, { color, effect: ColorEffect.Foreground })); - for(const out of doc.getWrittenSubfiles()) { - changedWikis.add(out); - console.log(` - Also updated: ${out}`); - } - } - } catch(error) { - console.error('Error while generating documents:', error); - } finally { - shell.close(); - } - console.log('All wikis processed in ' + (new Date().getTime() - setupStart.getTime()) + 'ms'); - console.log(` * Changed ${changedWikis.size} wiki/doc files.`); - // write a temp file in the os temp dir with the changed wikis - const filename=`${os.tmpdir()}/flowr-wiki-changed-files.txt`; - fs.writeFileSync(`${filename}`, Array.from(changedWikis).join('\n')); - console.log(` * List of changed wikis/docs written to ${filename}`); -} +export type AllWikiDocuments = typeof AllWikiDocuments; if(require.main === module) { - const wikiOptions: OptionDefinition[] = [ - { name: 'force', alias: 'F', type: Boolean, description: 'Overwrite existing wiki files, even if nothing changes' }, - { name: 'filter', alias: 'f', type: String, multiple: true, description: 'Only generate wikis whose target path contains the given string' }, - { name: 'help', alias: 'h', type: Boolean, description: 'Print this usage guide for the wiki generator' }, - { name: 'keep-alive', type: Boolean, description: 'Keep-alive wiki generator (only sensible with a reloading script like node --watch)' }, - ]; - - interface WikiCliOptions { - force: boolean; - filter?: string[]; - help: boolean; - 'keep-alive': boolean; - } - - const optionHelp = [ - { - header: `flowR (version ${flowrVersion().toString()})`, - content: 'Documentation (wiki, issue, ...) generator for flowR' - }, - { - header: 'Synopsis', - content: [ - '$ wiki {bold --help}', - '$ wiki {bold --force}', - '$ wiki {bold --filter} {italic "dataflow"}' - ] - }, - { - header: 'Options', - optionList: wikiOptions - } - ]; - - setMinLevelOfAllLogs(LogLevel.Fatal); - // parse args - const options = commandLineArgs(wikiOptions) as WikiCliOptions; - if(options.help) { - console.log(commandLineUsage(optionHelp)); - process.exit(0); - } - void makeAllWikis(options.force, options.filter).catch(err => { - console.error('Error while generating wikis:', err); - process.exit(1); - }).then(() => { - if(options['keep-alive']) { - console.log('Wiki generator running in keep-alive mode...'); - setInterval(() => { - // do nothing, just keep alive - }, 100); - } + wikiCli({ + docs: AllWikiDocuments, + refs: FlowrRefs, + header: `flowR (version ${flowrVersion().toString()})`, + content: 'Documentation (wiki, issue, ...) generator for flowR' }); } diff --git a/src/documentation/data/interface/doc-writing-code.ts b/src/documentation/data/interface/doc-writing-code.ts index e5ea1f8ac13..eb455706c9a 100644 --- a/src/documentation/data/interface/doc-writing-code.ts +++ b/src/documentation/data/interface/doc-writing-code.ts @@ -7,6 +7,7 @@ import { requestFromInput } from '../../../r-bridge/retriever'; import { block, details } from '../../doc-util/doc-structure'; import { TreeSitterExecutor } from '../../../r-bridge/lang-4.x/tree-sitter/tree-sitter-executor'; import { FlowrAnalyzerBuilder } from '../../../project/flowr-analyzer-builder'; +import type { AllWikiDocuments } from '../../../cli/wiki'; async function staticSliceExample() { const analyzer = await new FlowrAnalyzerBuilder() @@ -28,7 +29,7 @@ async function staticSliceExample() { */ export function explainWritingCode(_shell: RShell, ctx: GeneralDocContext): string { return `_flowR_ can be used as a ${ctx.linkPage('flowr:npm', 'module')} and offers several main classes and interfaces that are interesting for extension writers -(see the ${ctx.linkPage('flowr:vscode', 'Visual Studio Code extension')} or the ${ctx.linkPage('wiki/Core')} wiki page for more information). +(see the ${ctx.linkPage('flowr:vscode', 'Visual Studio Code extension')} or the ${ctx.linkPage('wiki/Core')} wiki page for more information). ### Creating Analyses with _flowR_ @@ -39,13 +40,13 @@ ${ ctx.code(staticSliceExample, { dropLinesEnd: 2, dropLinesStart: 1, hideDefinedAt: true }) } -For more information, please have a look at the ${ctx.linkPage('wiki/Analyzer')} wiki page, which explains how to construct and use the ${ctx.link(FlowrAnalyzer)} in more detail. -To work with specific perspectives, you can also consult the respective pages like the ${ctx.linkPage('wiki/Dataflow Graph')} or the ${ctx.linkPage('wiki/Abstract Interpretation')} wiki pages. +For more information, please have a look at the ${ctx.linkPage('wiki/Analyzer')} wiki page, which explains how to construct and use the ${ctx.link(FlowrAnalyzer)} in more detail. +To work with specific perspectives, you can also consult the respective pages like the ${ctx.linkPage('wiki/Dataflow Graph')} or the ${ctx.linkPage('wiki/Abstract Interpretation')} wiki pages. ### The Pipeline Executor (Low-Level Interface) Once, in the beginning, _flowR_ was meant to produce a dataflow graph merely to provide *program slices*. -However, with continuous updates, the ${ctx.linkPage('wiki/Dataflow Graph')} repeatedly proves to be the more interesting part. +However, with continuous updates, the ${ctx.linkPage('wiki/Dataflow Graph')} repeatedly proves to be the more interesting part. With this, we restructured _flowR_'s originally *hardcoded* pipeline to be far more flexible. Now, it can be theoretically extended or replaced with arbitrary steps, optional steps, and what we call 'decorations' of these steps. In short, a slicing pipeline using the ${ctx.link(PipelineExecutor)} looks like this: @@ -82,7 +83,7 @@ See the in-code documentation for more information. ### Using the ${ctx.link(RShell)} to Interact with R The ${ctx.link(RShell)} class allows interfacing with the \`R\` ecosystem installed on the host system. -Please have a look at ${ctx.linkPage('wiki/Engines', 'flowR\'s Engines')} for more information on alternatives (for example, the ${ctx.link(TreeSitterExecutor)}). +Please have a look at ${ctx.linkPage('wiki/Engines', 'flowR\'s Engines')} for more information on alternatives (for example, the ${ctx.link(TreeSitterExecutor)}). ${ block({ diff --git a/src/documentation/doc-readme.ts b/src/documentation/doc-readme.ts index a466c6ff832..29ef8a4880e 100644 --- a/src/documentation/doc-readme.ts +++ b/src/documentation/doc-readme.ts @@ -21,6 +21,7 @@ import { NewIssueUrl } from './doc-util/doc-issue'; import { joinWithLast } from '../util/text/strings'; import type { DocMakerArgs } from './wiki-mk/doc-maker'; import { DocMaker } from './wiki-mk/doc-maker'; +import type { AllWikiDocuments } from '../cli/wiki'; const PublicationsMain: { header: string, description: string, doi: string, bibtex: string }[] = [ { @@ -194,7 +195,7 @@ ${await documentReplSession(treeSitter, [{ `), ' ')} * 📚 **dependency analysis**\\ - Given your analysis project, flowR offers a plethora of so-called ${ctx.linkPage('wiki/Query API', 'queries')} to get more information about your code. + Given your analysis project, flowR offers a plethora of so-called ${ctx.linkPage('wiki/Query API', 'queries')} to get more information about your code. An important query is the [dependencies query](${FlowrWikiBaseRef}/Query-API#dependencies-query), which shows you the library your project needs, the data files it reads, the scripts it sources, and the data it outputs. @@ -208,17 +209,17 @@ The following showcases the dependency view of the [Visual Studio Code extension * 🚀 **fast call-graph, data-, and control-flow graphs**\\ Within just [${'' + textWithTooltip(roundToDecimals(await getLatestDfAnalysisTime('"social-science" Benchmark Suite (tree-sitter)'), 1) + ' ms', 'This measurement is automatically fetched from the latest benchmark!') + ''} (as of ${new Date(await getLastBenchmarkUpdate()).toLocaleDateString('en-US', dateOptions)})](${FlowrSiteBaseRef}/wiki/stats/benchmark), _flowR_ can analyze the data- and control-flow of the average real-world R script. See the ${ctx.linkPage('flowr:benchmarks', 'benchmarks')} for more information, - and consult the ${ctx.linkPage('wiki/Dataflow Graph', 'wiki pages')} for more details on the ${ctx.linkPage('wiki/Dataflow Graph', 'dataflow graphs')} as well as ${ctx.linkPage('wiki/Dataflow Graph', 'call graphs', 'perspectives-cg')}. + and consult the ${ctx.linkPage('wiki/Dataflow Graph', 'wiki pages')} for more details on the ${ctx.linkPage('wiki/Dataflow Graph', 'dataflow graphs')} as well as ${ctx.linkPage('wiki/Dataflow Graph', 'call graphs', 'perspectives-cg')}. ${prefixLines(details('Example: Generating a dataflow graph with flowR', ` You can investigate flowR's analyses using the [REPL](${FlowrWikiBaseRef}/Interface#using-the-repl). -Commands like ${getReplCommand('dataflow*')} allow you to view a ${ctx.linkPage('wiki/Dataflow Graph', 'dataflow graph')} for a given R script. +Commands like ${getReplCommand('dataflow*')} allow you to view a ${ctx.linkPage('wiki/Dataflow Graph', 'dataflow graph')} for a given R script. Let's have a look at the following example: ${codeBlock('r', getFileContentFromRoot('test/testfiles/example.R'))} -To get the ${ctx.linkPage('wiki/Dataflow Graph', 'dataflow graph')} for this script, you can use the following command: +To get the ${ctx.linkPage('wiki/Dataflow Graph', 'dataflow graph')} for this script, you can use the following command: ${await documentReplSession(treeSitter, [{ command: ':dataflow* test/testfiles/example.R', @@ -243,8 +244,8 @@ If you are already using flowR and want to give feedback, please consider fillin ## ⭐ Getting Started -To get started with _flowR_ and its features, please check out the ${ctx.linkPage('wiki/Overview')} wiki page. -The ${ctx.linkPage('wiki/Setup')} wiki page explains how you can download and setup _flowR_ on your system. +To get started with _flowR_ and its features, please check out the ${ctx.linkPage('wiki/Overview')} wiki page. +The ${ctx.linkPage('wiki/Setup')} wiki page explains how you can download and setup _flowR_ on your system. With docker 🐳️, the following line should be enough (and drop you directly into the read-eval-print loop): ${codeBlock('shell', 'docker run -it --rm eagleoutice/flowr')} @@ -281,7 +282,7 @@ ${printPublications()} ## 🚀 Contributing -We welcome every contribution! Please check out the ${ctx.linkPage('wiki/Onboarding', 'developer onboarding')} section in the wiki for all the information you will need. +We welcome every contribution! Please check out the ${ctx.linkPage('wiki/Onboarding', 'developer onboarding')} section in the wiki for all the information you will need. ### Contributors diff --git a/src/documentation/doc-util/doc-files.ts b/src/documentation/doc-util/doc-files.ts index 7dbde5e1d27..7446307c977 100644 --- a/src/documentation/doc-util/doc-files.ts +++ b/src/documentation/doc-util/doc-files.ts @@ -1,4 +1,5 @@ -import fs from 'fs'; +import fs from 'node:fs'; +import type { DocRefs } from '../wiki-mk/doc-context'; export const FlowrGithubGroupName = 'flowr-analysis'; export const FlowrGithubBaseRef = `https://github.com/${FlowrGithubGroupName}`; @@ -13,6 +14,14 @@ export const FlowrVsCode = 'https://marketplace.visualstudio.com/items?itemName= export const FlowrPositron = 'https://open-vsx.org/extension/code-inspect/vscode-flowr'; export const FlowrRStudioAddin = `${FlowrGithubBaseRef}/rstudio-addin-flowr`; export const FlowrRAdapter = `${FlowrGithubBaseRef}/flowr-r-adapter`; + +export const FlowrRefs: DocRefs = { + GitHub: { + Ref: FlowrGithubRef, + FileBaseRef: RemoteFlowrFilePathBaseRef + } +}; + /** * Returns a markdown link to the given file path relative to the project root. */ diff --git a/src/documentation/doc-util/doc-types.ts b/src/documentation/doc-util/doc-types.ts index e32a14de388..ec6431a0887 100644 --- a/src/documentation/doc-util/doc-types.ts +++ b/src/documentation/doc-util/doc-types.ts @@ -353,7 +353,7 @@ function getTypePathForTypeScript({ filePath }: Pick, prefix = RemoteFlowrFilePathBaseRef): string { const fromSource = getTypePathForTypeScript(elem); @@ -683,13 +683,14 @@ function retrieveNode(name: string, hierarchy: readonly TypeElementInSource[], f * If you create a wiki, please refer to the functions provided by the {@link GeneralDocContext}. * @param name - The name of the type, e.g. `MyType`, may include a container, e.g.,`MyContainer::MyType` (this works with function nestings too) * Use `:::` if you want to access a scoped function, but the name should be displayed without the scope + * @param prefix - Prefix to put before the actual path to the code (e.g., link to the root of the code tree on GitHub) * @param hierarchy - The hierarchy of types to search in * @param codeStyle - Whether to use code style for the link * @param realNameWrapper - How to highlight the function in name in the `x::y` format? * @param fuzzy - Whether to use fuzzy matching when searching for the type * @param type - Optionally restrict to a certain type of element */ -export function shortLink(name: string, hierarchy: readonly TypeElementInSource[], codeStyle = true, realNameWrapper = 'b', fuzzy?: boolean, type?: TypeElementKind): string { +export function shortLink(name: string, prefix: string, hierarchy: readonly TypeElementInSource[], codeStyle = true, realNameWrapper = 'b', fuzzy?: boolean, type?: TypeElementKind): string { const res = retrieveNode(name, hierarchy, fuzzy, type); if(!res) { console.error(`Could not find node ${name} when resolving short link!`); @@ -701,7 +702,7 @@ export function shortLink(name: string, hierarchy: readonly TypeElementInSource[ pkg = undefined; } const comments = node.comments?.join('\n').replace(/\\?\n|```[a-zA-Z]*|\s\s*/g, ' ').replace(/<\/?code>|`/g, '').replace(/<\/?p\/?>/g, ' ').replace(/"/g, '\'') ?? ''; - return `${codeStyle ? '' : ''}${ + return `${codeStyle ? '' : ''}${ (node.comments?.length ?? 0) > 0 ? textWithTooltip(pkg ? `${pkg}::<${realNameWrapper}>${mainName}` : mainName, comments.length > 400 ? comments.slice(0, 400) + '...' : comments) : pkg ? `${pkg}::<${realNameWrapper}>${mainName}` : mainName @@ -711,19 +712,20 @@ export function shortLink(name: string, hierarchy: readonly TypeElementInSource[ /** * Create a short link to a type in the documentation. - * If you create a wiki, please refer to the functions provided by the {@link GeneralWikiContext}. + * If you create a wiki, please refer to the functions provided by the {@link GeneralDocContext}. * @param name - The name of the type, e.g. `MyType`, may include a container, e.g.,`MyContainer::MyType` (this works with function nestings too) * Use `:::` if you want to access a scoped function, but the name should be displayed without the scope * @param hierarchy - The hierarchy of types to search in + * @param prefix - Prefix to put before the actual path to the code (e.g., link to the root of the code tree on GitHub) */ -export function shortLinkFile(name: string, hierarchy: readonly TypeElementInSource[]): string { +export function shortLinkFile(name: string, prefix: string, hierarchy: readonly TypeElementInSource[]): string { const res = retrieveNode(name, hierarchy); if(!res) { console.error(`Could not find node ${name} when resolving short link!`); return ''; } const [,, node] = res; - return `${getTypePathForTypeScript(node)}`; + return `${getTypePathForTypeScript(node)}`; } export interface GetDocumentationForTypeFilters { @@ -734,7 +736,7 @@ export interface GetDocumentationForTypeFilters { /** * Retrieve documentation comments for a type. - * If you create a wiki, please refer to the functions provided by the {@link GeneralWikiContext}. + * If you create a wiki, please refer to the functions provided by the {@link GeneralDocContext}. * @param name - The name of the type, e.g. `MyType`, may include a container, e.g.,`MyContainer::MyType` (this works with function nestings too) * Use `:::` if you want to access a scoped function, but the name should be displayed without the scope * @param hierarchy - The hierarchy of types to search in diff --git a/src/documentation/wiki-absint.ts b/src/documentation/wiki-absint.ts index d4789b4ddb0..26b9ffc3d13 100644 --- a/src/documentation/wiki-absint.ts +++ b/src/documentation/wiki-absint.ts @@ -15,6 +15,7 @@ import { details, section } from './doc-util/doc-structure'; import type { DocMakerArgs } from './wiki-mk/doc-maker'; import { DocMaker } from './wiki-mk/doc-maker'; import { Identifier } from '../dataflow/environments/identifier'; +import type { AllWikiDocuments } from '../cli/wiki'; class IntervalInferenceVisitor extends AbstractInterpretationVisitor { protected override onNumberConstant({ vertex, node }: { vertex: DataflowGraphVertexValue, node: RNumber }): void { @@ -132,7 +133,7 @@ ${codeBlock('mermaid', ctx.mermaid(AbstractDomain, { simplify: true, reverse: tr ${section('Abstract Interpretation', 2, 'abstract-interpretation')} -We perform abstract interpretation by forward-traversing the ${ctx.linkPage('wiki/Control Flow Graph', 'control flow graph')} of _flowR_ using an ${ctx.link(AbstractInterpretationVisitor)}. For each visited control flow vertex, the visitor retrieves the current abstract state by joining the abstract states of the predecessors, applies the abstract semantics of the visited control flow vertex to the current state, and updates the abstract state of the currently visited vertex to the current state. The visitor already handles assignments and (delayed) widening at widening points. However, the visitor does not yet support interprocedural abstract interpretation. +We perform abstract interpretation by forward-traversing the ${ctx.linkPage('wiki/Control Flow Graph', 'control flow graph')} of _flowR_ using an ${ctx.link(AbstractInterpretationVisitor)}. For each visited control flow vertex, the visitor retrieves the current abstract state by joining the abstract states of the predecessors, applies the abstract semantics of the visited control flow vertex to the current state, and updates the abstract state of the currently visited vertex to the current state. The visitor already handles assignments and (delayed) widening at widening points. However, the visitor does not yet support interprocedural abstract interpretation. To implement a custom abstract interpretation analysis, we can just create a new class and extend the ${ctx.link(AbstractInterpretationVisitor)}. The abstract interpretation visitor uses a ${ctx.link(StateAbstractDomain)} to capture the current abstract state at each vertex in the control flow graph. Hence, the abstract interpretation visitor requires a value abstract domain \`Domain\` as type parameter to specify the state abstract domain. We can then extend the callback functions of the ${ctx.link(AbstractInterpretationVisitor)} to implement the abstract semantics of expressions, such as ${ctx.link(`${SemanticCfgGuidedVisitor.name}:::onNumberConstant`)}, ${ctx.link(`${AbstractInterpretationVisitor.name}:::onFunctionCall`)}, and ${ctx.link(`${SemanticCfgGuidedVisitor.name}:::onReplacementCall`)} (make sure to still call the respective super function). The abstract interpretation visitor provides the following functions to retrieve the currently inferred values: @@ -153,7 +154,7 @@ If we now want to run the interval inference, we can write the following code: ${ctx.code(inferIntervals, { dropLinesStart: 1, dropLinesEnd: 5 })} -We first need a ${ctx.linkPage('wiki/Analyzer', 'flowR analyzer')} (in this case, using the ${ctx.linkPage('wiki/Engines', 'tree-sitter engine')}). In this example, we want to analyze a small example code that assigns \`42\` to the variable \`x\`, randomly assigns \`6\` or \`12\` to the variable \`y\`, and assignes the sum of \`x\` and \`y\` to the variable \`z\`. For the abstract interpretation visitor, we need to retrieve the ${ctx.linkPage('wiki/Normalized AST', 'normalized AST')}, ${ctx.linkPage('wiki/Dataflow Graph', 'dataflow graph')}, ${ctx.linkPage('wiki/Control Flow Graph', 'control flow graph')}, and context of the flowR anaylzer. Additionally, we need to provide a state abstract domain for the visitor -- in this case, a state abstract domain for the interval domain. We then construct a new ${ctx.link(IntervalInferenceVisitor)} using the control flow graph, dataflow graph, normalized AST, analyzer context, and state abstract domain, and start the visitor using ${ctx.linkM(AbstractInterpretationVisitor, 'start', { hideClass: true })}. After the visitor is finished, we retrieve the inferred abstract state at the end of the program using ${ctx.linkM(AbstractInterpretationVisitor, 'getEndState', { hideClass: true })}. +We first need a ${ctx.linkPage('wiki/Analyzer', 'flowR analyzer')} (in this case, using the ${ctx.linkPage('wiki/Engines', 'tree-sitter engine')}). In this example, we want to analyze a small example code that assigns \`42\` to the variable \`x\`, randomly assigns \`6\` or \`12\` to the variable \`y\`, and assignes the sum of \`x\` and \`y\` to the variable \`z\`. For the abstract interpretation visitor, we need to retrieve the ${ctx.linkPage('wiki/Normalized AST', 'normalized AST')}, ${ctx.linkPage('wiki/Dataflow Graph', 'dataflow graph')}, ${ctx.linkPage('wiki/Control Flow Graph', 'control flow graph')}, and context of the flowR anaylzer. Additionally, we need to provide a state abstract domain for the visitor -- in this case, a state abstract domain for the interval domain. We then construct a new ${ctx.link(IntervalInferenceVisitor)} using the control flow graph, dataflow graph, normalized AST, analyzer context, and state abstract domain, and start the visitor using ${ctx.linkM(AbstractInterpretationVisitor, 'start', { hideClass: true })}. After the visitor is finished, we retrieve the inferred abstract state at the end of the program using ${ctx.linkM(AbstractInterpretationVisitor, 'getEndState', { hideClass: true })}. If we now print the inferred abstract state at the end of the program, we get the following output: diff --git a/src/documentation/wiki-analyzer.ts b/src/documentation/wiki-analyzer.ts index 7e4b4b098e7..5d15b21bd10 100644 --- a/src/documentation/wiki-analyzer.ts +++ b/src/documentation/wiki-analyzer.ts @@ -39,6 +39,7 @@ import { FlowrAnalyzerPlugin } from '../project/plugins/flowr-analyzer-plugin'; import { FlowrAnalyzerEnvironmentContext } from '../project/context/flowr-analyzer-environment-context'; import { FlowrAnalyzerFunctionsContext } from '../project/context/flowr-analyzer-functions-context'; import { FlowrAnalyzerMetaContext } from '../project/context/flowr-analyzer-meta-context'; +import type { AllWikiDocuments } from '../cli/wiki'; async function analyzerQuickExample() { const analyzer = await new FlowrAnalyzerBuilder() @@ -170,7 +171,7 @@ Likewise, ${ctx.linkM(FlowrAnalyzer, 'peekNormalize', { codeFont: true, realName Again, ${ctx.linkM(FlowrAnalyzer, 'peekDataflow', { codeFont: true, realNameWrapper: 'i' })} allows you to inspect the dataflow graph if it was already computed (but without triggering a computation). * ${ctx.linkM(FlowrAnalyzer, 'controlflow')} to compute the [Control Flow Graph](${FlowrWikiBaseRef}/Control%20Flow%20Graph)\\ Also, ${ctx.linkM(FlowrAnalyzer, 'peekControlflow', { codeFont: true, realNameWrapper: 'i' })} returns the control flow graph if it was already computed but without triggering a computation. -* ${ctx.linkM(FlowrAnalyzer, 'callGraph')} to compute the ${ctx.linkPage('wiki/Dataflow Graph', 'call graph', 'perspectives-cg')} of the analyzed code\\ +* ${ctx.linkM(FlowrAnalyzer, 'callGraph')} to compute the ${ctx.linkPage('wiki/Dataflow Graph', 'call graph', 'perspectives-cg')} of the analyzed code\\ Likewise, ${ctx.linkM(FlowrAnalyzer, 'peekCallGraph', { codeFont: true, realNameWrapper: 'i' })} allows you to inspect the call graph if it was already computed (but without triggering a computation). * ${ctx.linkM(FlowrAnalyzer, 'query')} to run [queries](${FlowrWikiBaseRef}/Query-API) on the analyzed code. * ${ctx.linkM(FlowrAnalyzer, 'runSearch')} to run a search query on the analyzed code using the [search API](${FlowrWikiBaseRef}/Search%20API) diff --git a/src/documentation/wiki-cfg.ts b/src/documentation/wiki-cfg.ts index 73538d0edbb..d9f00108e3f 100644 --- a/src/documentation/wiki-cfg.ts +++ b/src/documentation/wiki-cfg.ts @@ -36,6 +36,7 @@ import type { DocMakerArgs } from './wiki-mk/doc-maker'; import { DocMaker } from './wiki-mk/doc-maker'; import { prefixLines } from './doc-util/doc-general'; import { codeBlock } from './doc-util/doc-code'; +import type { AllWikiDocuments } from '../cli/wiki'; const CfgLongExample = `f <- function(a, b = 3) { if(a > b) { @@ -556,7 +557,7 @@ ${ ${section('Working With Exit Points', 3, 'cfg-exit-points')} -With the ${ctx.linkPage('wiki/Dataflow Graph')} you already get a \`${DfEdge.typeToName(EdgeType.Returns)}\` edge that tells you what a function call returns +With the ${ctx.linkPage('wiki/Dataflow Graph')} you already get a \`${DfEdge.typeToName(EdgeType.Returns)}\` edge that tells you what a function call returns (given that this function call does neither transform nor create a value). But the control flow perspective gives you more! Given a simple addition like \`x + 1\`, the CFG looks like this: diff --git a/src/documentation/wiki-dataflow-graph.ts b/src/documentation/wiki-dataflow-graph.ts index dc754ebcf01..2f4f77641b7 100644 --- a/src/documentation/wiki-dataflow-graph.ts +++ b/src/documentation/wiki-dataflow-graph.ts @@ -61,6 +61,7 @@ import type { KnownParser } from '../r-bridge/parser'; import type { MermaidMarkdownMark } from '../util/mermaid/info'; import { FlowrAnalyzer } from '../project/flowr-analyzer'; import { BuiltInProcName } from '../dataflow/environments/built-in'; +import type { AllWikiDocuments } from '../cli/wiki'; async function subExplanation(parser: KnownParser, { description, code, expectedSubgraph }: SubExplanationParameters): Promise { expectedSubgraph = await verifyExpectedSubgraph(parser, code, expectedSubgraph); @@ -707,8 +708,8 @@ However, nested definitions can carry it (in the nested case, \`x\` is defined b name: 'Calls Edge', type: EdgeType.Calls, description: `Link the [function call](#function-call-vertex) to the [function definition](#function-definition-vertex) that is called. To find all called definitions, - please use the ${ctx.link(getOriginInDfg.name)} function, as explained in ${ctx.linkPage('wiki/Dataflow Graph', 'working with the dataflow graph', 'Working-with-the-Dataflow-Graph')}. - If you are interested in the call graph, refer to ${ctx.linkM(FlowrAnalyzer, 'callGraph')} and consult the ${ctx.linkPage('wiki/Dataflow Graph', 'call graph wiki', 'perspectives-cg')} for more information. + please use the ${ctx.link(getOriginInDfg.name)} function, as explained in ${ctx.linkPage('wiki/Dataflow Graph', 'working with the dataflow graph', 'Working-with-the-Dataflow-Graph')}. + If you are interested in the call graph, refer to ${ctx.linkM(FlowrAnalyzer, 'callGraph')} and consult the ${ctx.linkPage('wiki/Dataflow Graph', 'call graph wiki', 'perspectives-cg')} for more information. `, code: 'foo <- function() {}\nfoo()', expectedSubgraph: emptyGraph().calls('2@foo', '1@function') @@ -884,7 +885,7 @@ export class WikiDataflowGraph extends DocMaker<'wiki/Dataflow Graph.md'> { return ` This page briefly summarizes flowR's dataflow graph, represented by the ${ctx.link(DataflowGraph)} class within the code. In case you want to manually build such a graph (e.g., for testing), you can use the ${ctx.link(DataflowGraphBuilder)}. -If you are interested in which features we support and which features are still to be worked on, please refer to our ${ctx.linkPage('wiki/Capabilities')} page. +If you are interested in which features we support and which features are still to be worked on, please refer to our ${ctx.linkPage('wiki/Capabilities')} page. In summary, we discuss the following topics: - [Vertices](#vertices) @@ -906,7 +907,7 @@ wiki page if you are unsure. > [!TIP] > If you want to investigate the dataflow graph, > you can either use the [Visual Studio Code extension](${FlowrVsCode}) or the ${ctx.replCmd('dataflow*')} -> command in the REPL (see the ${ctx.linkPage('wiki/Interface', 'Interface wiki page')} for more information). +> command in the REPL (see the ${ctx.linkPage('wiki/Interface', 'Interface wiki page')} for more information). > There is also a simplified perspective available with ${ctx.replCmd('dataflowsimple*')} that does not show everything but is easier to read. > For small graphs, you can also use ${ctx.replCmd('dataflowascii')} to print the graph as ASCII art. > @@ -922,7 +923,7 @@ ${await printDfGraphForCode(treeSitter, 'x <- 3\ny <- x + 1\ny')} The above dataflow graph showcases the general gist. We define a dataflow graph as a directed graph G = (V, E), differentiating between ${getAllVertices().length} types of vertices V and ${getAllEdges().length} types of edges E allowing each vertex to have a single, and each edge to have multiple distinct types. Additionally, every node may have links to its [control dependencies](#control-dependencies) (which you may view as a ${nth(getAllEdges().length + 1)} edge type, -although they are explicitly no data dependency and relate to the ${ctx.linkPage('wiki/Control Flow Graph')}. +although they are explicitly no data dependency and relate to the ${ctx.linkPage('wiki/Control Flow Graph')}.
@@ -1153,8 +1154,8 @@ ${section('Working with the Dataflow Graph', 2, 'dfg-working')} The ${ctx.link('DataflowInformation')} is the core result of _flowR_ and summarizes a lot of information. Depending on what you are interested in, there exists a plethora of functions and queries to help you out, answering the most important questions: -* The **${ctx.linkPage('wiki/Query API')}** provides many functions to query the dataflow graph for specific information (dependencies, calls, slices, clusters, ...) -* The **${ctx.linkPage('wiki/Search API')}** allows you to search for specific vertices or edges in the dataflow graph or the original program +* The **${ctx.linkPage('wiki/Query API')}** provides many functions to query the dataflow graph for specific information (dependencies, calls, slices, clusters, ...) +* The **${ctx.linkPage('wiki/Search API')}** allows you to search for specific vertices or edges in the dataflow graph or the original program * ${ctx.link(recoverName)} and ${ctx.link(recoverContent)} to get the name or content of a vertex in the dataflow graph * ${ctx.link(resolveIdToValue)} to resolve the value of a variable or id (if possible, see [below](#dfg-resolving-values)) * ${ctx.link(getAliases)} to get all (potentially currently) aliases of a given definition @@ -1167,8 +1168,8 @@ FlowR also provides various helper objects (with the same name as the correspond * ${ctx.link('Identifier', undefined, { type: 'variable' })} to get helpful functions wrt. identifiers * ${ctx.link('FunctionArgument', undefined, { type: 'variable' })} to get helpful functions wrt. function arguments -Some of these functions have been explained in their respective wiki pages. However, some are part of the ${ctx.linkPage('wiki/Dataflow Graph', 'Dataflow Graph API')} and so we explain them here. -If you are interested in which features we support and which features are still to be worked on, please refer to our ${ctx.linkPage('wiki/Capabilities', 'capabilities')} page. +Some of these functions have been explained in their respective wiki pages. However, some are part of the ${ctx.linkPage('wiki/Dataflow Graph', 'Dataflow Graph API')} and so we explain them here. +If you are interested in which features we support and which features are still to be worked on, please refer to our ${ctx.linkPage('wiki/Capabilities', 'capabilities')} page. ${section('Resolving Values', 3, 'dfg-resolving-values')} diff --git a/src/documentation/wiki-interface.ts b/src/documentation/wiki-interface.ts index c2a0c72217a..4e6ccaca75f 100644 --- a/src/documentation/wiki-interface.ts +++ b/src/documentation/wiki-interface.ts @@ -21,6 +21,7 @@ import type { KnownParser } from '../r-bridge/parser'; import type { GeneralDocContext } from './wiki-mk/doc-context'; import { BuiltInProcName } from '../dataflow/environments/built-in'; import { explainWritingCode } from './data/interface/doc-writing-code'; +import type { AllWikiDocuments } from '../cli/wiki'; async function explainServer(parser: KnownParser): Promise { documentAllServerMessages(); @@ -104,7 +105,7 @@ async function explainRepl(parser: KnownParser, ctx: GeneralDocContext): Promise return ` > [!NOTE] > To execute arbitrary R commands with a repl request, _flowR_ has to be started explicitly with ${ctx.cliOption('flowr', 'r-session-access')}. -> Please be aware that this introduces a security risk and note that this relies on the ${ctx.linkPage('wiki/Engines', '`r-shell` engine')} . +> Please be aware that this introduces a security risk and note that this relies on the ${ctx.linkPage('wiki/Engines', '`r-shell` engine')} . Although primarily meant for users to explore, there is nothing which forbids simply calling _flowR_ as a subprocess to use standard-in, -output, and -error @@ -148,8 +149,8 @@ the REPL will re-use previously obtained information and not re-parse the code a } Generally, many commands offer shortcut versions in the REPL. Many queries, for example, offer a shortened format (see the example below). -Of special note, the ${ctx.linkPage('wiki/Query API', 'Config Query', 'Config-Query')} -can be used to also modify the currently active configuration of _flowR_ within the REPL (see the ${ctx.linkPage('wiki/Query API', 'wiki page', 'Config-Query')} for more information). +Of special note, the ${ctx.linkPage('wiki/Query API', 'Config Query', 'Config-Query')} +can be used to also modify the currently active configuration of _flowR_ within the REPL (see the ${ctx.linkPage('wiki/Query API', 'wiki page', 'Config-Query')} for more information). ### Example: Retrieving the Dataflow Graph @@ -206,7 +207,7 @@ ${await documentReplSession(parser, [{ description: 'Run the linter on the given code, with only the `dead-code` rule enabled.' }], { openOutput: true })} -For more information on the available queries, please check out the ${ctx.linkPage('wiki/Query API', 'Query API')}. +For more information on the available queries, please check out the ${ctx.linkPage('wiki/Query API', 'Query API')}. `; } @@ -328,7 +329,7 @@ export class WikiInterface extends DocMaker<'wiki/Interface.md'> { protected async text({ shell, ctx, treeSitter }: DocMakerArgs): Promise { return ` -Although far from being as detailed as the in-depth explanation of ${ctx.linkPage('wiki/Core', '_flowR_')}, +Although far from being as detailed as the in-depth explanation of ${ctx.linkPage('wiki/Core', '_flowR_')}, this wiki page explains how to interface with _flowR_ in more detail. In general, command line arguments and other options provide short descriptions on hover over. diff --git a/src/documentation/wiki-linter.ts b/src/documentation/wiki-linter.ts index fb5d03a712e..00717c98b4c 100644 --- a/src/documentation/wiki-linter.ts +++ b/src/documentation/wiki-linter.ts @@ -1,10 +1,10 @@ import { autoGenHeader } from './doc-util/doc-auto-gen'; -import { FlowrWikiBaseRef, linkFlowRSourceFile } from './doc-util/doc-files'; +import { FlowrRefs, FlowrWikiBaseRef, linkFlowRSourceFile, RemoteFlowrFilePathBaseRef } from './doc-util/doc-files'; import { type LintingRuleNames, LintingRules } from '../linter/linter-rules'; import { codeBlock } from './doc-util/doc-code'; import { showQuery } from './doc-util/doc-query'; import { type TypeElementInSource, type TypeReport, getDocumentationForType, getTypePathLink, getTypesFromFolder, mermaidHide, shortLink, shortLinkFile } from './doc-util/doc-types'; -import path from 'path'; +import path from 'node:path'; import { documentReplSession } from './doc-util/doc-repl'; import { section } from './doc-util/doc-structure'; import { LintingRuleTag } from '../linter/linter-tags'; @@ -87,7 +87,7 @@ ${args.length >= 7 ? `\nAnd using the following [configuration](#configuration): We expect the linter to report the following: ${codeBlock('ts', prettyPrintExpectedOutput(args[4].getText(report.source)))} -See [here](${getTypePathLink({ filePath: report.source.fileName, lineNumber: report.lineNumber })}) for the test-case implementation. +See [here](${getTypePathLink({ filePath: report.source.fileName, lineNumber: report.lineNumber }, RemoteFlowrFilePathBaseRef)}) for the test-case implementation. `; } @@ -180,7 +180,7 @@ df[6, "value"] const certaintyText = `\`${textWithTooltip(rule.info.certainty, certaintyDoc)}\``; if(format === 'short') { ruleExplanations.set(name, () => Promise.resolve(` - **[${rule.info.name}](${FlowrWikiBaseRef}/${encodeURIComponent(getPageNameForLintingRule(name))}):** ${rule.info.description} [see ${shortLinkFile(ruleType, types)}]\\ + **[${rule.info.name}](${FlowrWikiBaseRef}/${encodeURIComponent(getPageNameForLintingRule(name))}):** ${rule.info.description} [see ${shortLinkFile(ruleType, RemoteFlowrFilePathBaseRef, types)}]\\ ${tags} `.trim())); @@ -196,7 +196,7 @@ ${tags} This rule is a ${certaintyText} rule. ${rule.info.description}\\ -_This linting rule is implemented in ${shortLinkFile(ruleType, types)}._ +_This linting rule is implemented in ${shortLinkFile(ruleType, RemoteFlowrFilePathBaseRef, types)}._ ### Configuration @@ -206,7 +206,7 @@ The \`${name}\` rule accepts the following configuration options: ${ Object.getOwnPropertyNames(LintingRules[name].info.defaultConfig).sort().map(key => - `- ${shortLink(`${configType}:::${key}`, types)}\\\n${getDocumentationForType(`${configType}::${key}`, types)}` + `- ${shortLink(`${configType}:::${key}`, FlowrRefs.GitHub.FileBaseRef, types)}\\\n${getDocumentationForType(`${configType}::${key}`, types)}` ).join('\n') } diff --git a/src/documentation/wiki-mk/cli.ts b/src/documentation/wiki-mk/cli.ts new file mode 100644 index 00000000000..a16a74dc27e --- /dev/null +++ b/src/documentation/wiki-mk/cli.ts @@ -0,0 +1,161 @@ +import type { DocMakerArgs, DocMakerLike, DocMakerOutputArgs } from './doc-maker'; +import { RShell } from '../../r-bridge/shell'; +import { TreeSitterExecutor } from '../../r-bridge/lang-4.x/tree-sitter/tree-sitter-executor'; +import type { DocRefs } from './doc-context'; +import { makeDocContextForTypes } from './doc-context'; +import { ansiFormatter, ColorEffect, Colors, FontStyles } from '../../util/text/ansi'; +import fs from 'fs'; +import os from 'os'; +import type { OptionDefinition } from 'command-line-usage'; +import commandLineUsage from 'command-line-usage'; +import commandLineArgs from 'command-line-args'; +import { setMinLevelOfAllLogs } from '../../../test/functionality/_helper/log'; +import { LogLevel } from '../../util/log'; + +function sortByLeastRecentChanged(wikis: DocMakerLike[]): DocMakerLike[] { + return wikis.slice().sort((a, b) => { + const aStat = fs.existsSync(a.getProducer()) ? fs.statSync(a.getProducer()) : undefined; + const bStat = fs.existsSync(b.getProducer()) ? fs.statSync(b.getProducer()) : undefined; + const aMTime = aStat ? aStat.mtime.getTime() : 0; + const bMTime = bStat ? bStat.mtime.getTime() : 0; + return bMTime - aMTime; + }); +} + +/** + * Updates and optionally re-creates all flowR wikis. + */ +export async function makeAllWikis(docs: DocMakerLike[], refs: DocRefs, force: boolean, filter: string[] | undefined, rootFolders?: string[]) { + const setupStart = new Date(); + console.log('Setting up wiki generation...'); + const shell = new RShell(); + console.log(' * R shell initialized'); + await TreeSitterExecutor.initTreeSitter(); + const treeSitter = new TreeSitterExecutor(); + console.log(' * Tree-sitter parser initialized'); + const ctx = makeDocContextForTypes(refs, shell, ...rootFolders ?? []); + console.log(' * Wiki context prepared'); + if(force) { + console.log(ansiFormatter.format('Forcing wiki regeneration (existing files will be overwritten)', { style: FontStyles.Bold, color: Colors.Yellow, effect: ColorEffect.Foreground })); + } + const info: DocMakerArgs & DocMakerOutputArgs = { + ctx, + shell, treeSitter, + force, + readFileSync(f: fs.PathLike) { + try { + return fs.readFileSync(f); + } catch{ + return undefined; + } + }, + writeFileSync: fs.writeFileSync + }; + + console.log(`Setup for wiki generation took ${(Date.now() - setupStart.getTime())}ms`); + const changedWikis = new Set(); + try { + const sortedDocs = sortByLeastRecentChanged(docs); + console.log(`Generating ${sortedDocs.length} wikis/docs, sorted by most recently updated...`); + for(const doc of sortedDocs) { + const type = doc.getTarget().toLowerCase().includes('wiki') ? 'Wiki' : 'Doc'; + if(filter && !filter.some(f => doc.getTarget().includes(f))) { + console.log(` * Skipping ${type} (filtered out): ${doc.getTarget()}`); + continue; + } + const now = new Date(); + console.log(ansiFormatter.format(` [${doc.getTarget()}] Updating ${type}...`, { style: FontStyles.Bold, color: Colors.Cyan, effect: ColorEffect.Foreground })); + const changed = await doc.make(info); + const text = changed ? `${type} updated` : `${type} identical, no changes made`; + if(changed) { + changedWikis.add(doc.getTarget()); + } + const color = changed ? Colors.Green : Colors.White; + console.log(ansiFormatter.format(` [${doc.getTarget()}] ${text}: ${doc.getTarget()} (took ${Date.now() - now.getTime()}ms)`, { color, effect: ColorEffect.Foreground })); + for(const out of doc.getWrittenSubfiles()) { + changedWikis.add(out); + console.log(` - Also updated: ${out}`); + } + } + } catch(error) { + console.error('Error while generating documents:', error); + } finally { + shell.close(); + } + console.log('All wikis processed in ' + (Date.now() - setupStart.getTime()) + 'ms'); + console.log(` * Changed ${changedWikis.size} wiki/doc files.`); + // write a temp file in the os temp dir with the changed wikis + const filename=`${os.tmpdir()}/flowr-wiki-changed-files.txt`; + fs.writeFileSync(`${filename}`, Array.from(changedWikis).join('\n')); + console.log(` * List of changed wikis/docs written to ${filename}`); +} + +interface WikiOptions { + /** The pages to create the wiki from */ + docs: DocMakerLike[], + /** Interface containing various references */ + refs: DocRefs, + /** Paths to folders containing the necessary type definitions */ + rootFolders?: string[], + /** To be printed by {@link commandLineUsage} */ + header: string, + /** Content to be printed by {@link commandLineUsage} */ + content: string +} + +/** + * Generic CLI function for creating wiki pages from {@link DocMakerLike} classes. + */ +export function wikiCli({ docs, refs, rootFolders, header, content }: WikiOptions) { + const wikiOptions: OptionDefinition[] = [ + { name: 'force', alias: 'F', type: Boolean, description: 'Overwrite existing wiki files, even if nothing changes' }, + { name: 'filter', alias: 'f', type: String, multiple: true, description: 'Only generate wikis whose target path contains the given string' }, + { name: 'help', alias: 'h', type: Boolean, description: 'Print this usage guide for the wiki generator' }, + { name: 'keep-alive', type: Boolean, description: 'Keep-alive wiki generator (only sensible with a reloading script like node --watch)' }, + ]; + + interface WikiCliOptions { + force: boolean; + filter?: string[]; + help: boolean; + 'keep-alive': boolean; + } + + const optionHelp = [ + { + header, + content + }, + { + header: 'Synopsis', + content: [ + '$ wiki {bold --help}', + '$ wiki {bold --force}', + '$ wiki {bold --filter} {italic "dataflow"}' + ] + }, + { + header: 'Options', + optionList: wikiOptions + } + ]; + + setMinLevelOfAllLogs(LogLevel.Fatal); + // parse args + const options = commandLineArgs(wikiOptions) as WikiCliOptions; + if(options.help) { + console.log(commandLineUsage(optionHelp)); + process.exit(0); + } + void makeAllWikis(docs, refs, options.force, options.filter, rootFolders).catch(err => { + console.error('Error while generating wikis:', err); + process.exit(1); + }).then(() => { + if(options['keep-alive']) { + console.log('Wiki generator running in keep-alive mode...'); + setInterval(() => { + // do nothing, just keep alive + }, 100); + } + }); +} diff --git a/src/documentation/wiki-mk/doc-context.ts b/src/documentation/wiki-mk/doc-context.ts index 5cafcc52cfa..593999500f9 100644 --- a/src/documentation/wiki-mk/doc-context.ts +++ b/src/documentation/wiki-mk/doc-context.ts @@ -9,7 +9,6 @@ import path from 'path'; import { guard } from '../../util/assert'; import { autoGenHeader } from '../doc-util/doc-auto-gen'; import type { RShell } from '../../r-bridge/shell'; -import type { ValidWikiDocumentTargetsNoSuffix } from '../../cli/wiki'; import type { PathLike } from 'fs'; import { FlowrDockerRef, @@ -23,6 +22,7 @@ import type { scripts } from '../../cli/common/scripts-info'; import type { ScriptOptions } from '../doc-util/doc-cli-option'; import { getReplCommand, getCliLongOptionOf } from '../doc-util/doc-cli-option'; import type { ReplCommandNames } from '../../cli/repl/commands/repl-commands'; +import type { DocMakerLike } from './doc-maker'; /** * Context available when generating documentation for a wiki in markdown format @@ -68,6 +68,9 @@ type NamedPrototype = { prototype: { constructor: { name: string } } }; type ProtoKeys = T extends { prototype: infer P } ? keyof P : never; type StaticKeys = T extends { prototype: infer P } ? Exclude : never; +export type ValidWikiDocumentTargets = ReturnType; +export type ValidWikiDocumentTargetsNoSuffix = ValidWikiDocumentTargets extends `${infer Name}.${string}` ? Name : never; + export const ConstantWikiLinkInfo = { 'flowr:npm': FlowrNpmRef, 'flowr:github': FlowrGithubRef, @@ -230,7 +233,7 @@ export interface GeneralDocContext { * @param linkText - Optional text to display for the link. If not provided, the page name will be used. * @param segment - An optional segment within the page to link to (e.g., a header anchor). */ - linkPage(pageName: ValidWikiDocumentTargetsNoSuffix | keyof typeof ConstantWikiLinkInfo, linkText?: string, segment?: string): string; + linkPage(pageName: ValidWikiDocumentTargetsNoSuffix | keyof typeof ConstantWikiLinkInfo, linkText?: string, segment?: string): string; /** * Generates a link to a code file in the code base. @@ -273,9 +276,20 @@ export interface GeneralDocContext { replCmd(commandName: ReplCommandNames, quote?: boolean, showStar?: boolean): string } +/** + * Various references used to link code elements, other Wiki pages, etc. + */ +export interface DocRefs { + GitHub: { + Ref: string; + FileBaseRef: string; + }; +} + /** * Creates a wiki context for generating documentation for code elements. * This context provides methods to create links, code snippets, and documentation for code elements. + * @param refs - Various references used to link code elements, other Wiki pages, etc. * @param shell - An optional RShell instance to retrieve the R version for the auto-generation header. * @param rootFolders - The root folder(s) of the code base to analyze. Defaults to flower's `src/` **and** `test/` folder. * @example @@ -286,12 +300,15 @@ export interface GeneralDocContext { * ``` */ export function makeDocContextForTypes( + refs: DocRefs, shell?: RShell, ...rootFolders: string[] ): GeneralDocContext { if(rootFolders.length === 0) { - rootFolders.push(path.resolve(__dirname, '../../../src')); - rootFolders.push(path.resolve(__dirname, '../../../test/functionality')); + rootFolders.push( + path.resolve(__dirname, '../../../src'), + path.resolve(__dirname, '../../../test/functionality') + ); } const { info, program } = getTypesFromFolder({ rootFolder: rootFolders, typeNameForMermaid: undefined }); return { @@ -300,7 +317,7 @@ export function makeDocContextForTypes( }, link(element: ElementIdOrRef, fmt?: LinkFormat, filter?: ElementFilter): string { guard(filter?.file === undefined, 'filtering for files is not yet supported for link'); - return shortLink(getNameFromElementIdOrRef(element), info, fmt?.codeFont, fmt?.realNameWrapper, filter?.fuzzy, filter?.type); + return shortLink(getNameFromElementIdOrRef(element), refs.GitHub.FileBaseRef, info, fmt?.codeFont, fmt?.realNameWrapper, filter?.fuzzy, filter?.type); }, linkM(cls: T, element: ProtoKeys | StaticKeys, fmt?: LinkFormat & { hideClass?: boolean }, filter?: ElementFilter): string { const className = cls.prototype.constructor.name; @@ -309,7 +326,7 @@ export function makeDocContextForTypes( return this.link(fullName, fmt, filter); }, linkFile(element: ElementIdOrRef): string { - return shortLinkFile(getNameFromElementIdOrRef(element), info); + return shortLinkFile(getNameFromElementIdOrRef(element), refs.GitHub.FileBaseRef, info); }, hierarchy(element: ElementIdOrRef, fmt?: Omit, filter?: ElementFilter): string { guard(filter === undefined, 'ElementFilter is not yet supported for hierarchy'); @@ -336,13 +353,13 @@ export function makeDocContextForTypes( ...options }) as string; }, - linkPage(pageName: ValidWikiDocumentTargetsNoSuffix | keyof typeof ConstantWikiLinkInfo, linkText?: string, segment?: string): string { + linkPage(pageName: ValidWikiDocumentTargetsNoSuffix | keyof typeof ConstantWikiLinkInfo, linkText?: string, segment?: string): string { const text = linkText ?? pageName.split('/').pop() ?? pageName; let link: string; if(pageName in ConstantWikiLinkInfo) { link = ConstantWikiLinkInfo[pageName as keyof typeof ConstantWikiLinkInfo]; } else { - link = `${FlowrWikiBaseRef}/${pageName.toLowerCase().replace(/ /g, '-')}`; + link = `${refs.GitHub.Ref}/${pageName.toLowerCase().replace(/ /g, '-')}`; } return `[${text}](${link}${segment ? `#${segment}` : ''})`; }, diff --git a/src/documentation/wiki-normalized-ast.ts b/src/documentation/wiki-normalized-ast.ts index 11c78312e1c..9c0c697a8bc 100644 --- a/src/documentation/wiki-normalized-ast.ts +++ b/src/documentation/wiki-normalized-ast.ts @@ -15,6 +15,7 @@ import { DocMaker } from './wiki-mk/doc-maker'; import { parseRoxygenCommentsOfNode } from '../r-bridge/roxygen2/roxygen-parse'; import type { RNumber } from '../r-bridge/lang-4.x/ast/model/nodes/r-number'; import type { RBinaryOp } from '../r-bridge/lang-4.x/ast/model/nodes/r-binary-op'; +import type { AllWikiDocuments } from '../cli/wiki'; async function quickNormalizedAstMultipleFiles() { const analyzer = await new FlowrAnalyzerBuilder() @@ -222,7 +223,7 @@ As a simple showcase, we want to use the fold to evaluate numeric expressions co ${ctx.code(MyMathFold, { dropLinesStart: 1 })} -Now, we can use the ${ctx.link(FlowrAnalyzer)} (see the ${ctx.linkPage('wiki/Analyzer')} wiki page) to get the ${ctx.linkPage('wiki/Normalized AST', 'normalized AST')} and apply the fold: +Now, we can use the ${ctx.link(FlowrAnalyzer)} (see the ${ctx.linkPage('wiki/Analyzer')} wiki page) to get the ${ctx.linkPage('wiki/Normalized AST', 'normalized AST')} and apply the fold: ${ctx.code(useMyMathFoldExample, { dropLinesStart: 1, dropLinesEnd: 2, hideDefinedAt: true })} diff --git a/src/documentation/wiki-overview.ts b/src/documentation/wiki-overview.ts index 945906baeec..44f9c7e1097 100644 --- a/src/documentation/wiki-overview.ts +++ b/src/documentation/wiki-overview.ts @@ -4,6 +4,7 @@ import { FlowrGithubBaseRef, FlowrPositron, FlowrRAdapter, FlowrRStudioAddin, Fl import { RShell } from '../r-bridge/shell'; import { DataflowGraph } from '../dataflow/graph/graph'; import { FlowrAnalyzer } from '../project/flowr-analyzer'; +import type { AllWikiDocuments } from '../cli/wiki'; /** * https://github.com/flowr-analysis/flowr/wiki/Overview @@ -16,7 +17,7 @@ export class WikiOverview extends DocMaker<'wiki/Overview.md'> { public text({ ctx }: DocMakerArgs): string { return ` First of all, if you have never used _flowR_ before, -please refer to the ${ctx.linkPage('wiki/Setup')} wiki page first, +please refer to the ${ctx.linkPage('wiki/Setup')} wiki page first, for instructions on how to install _flowR_. @@ -48,7 +49,7 @@ It is available with the [\`benchmark\`](#benchmark-the-slicer) script. The statistics module is mostly independent of the slicer and can be used to analyze R files regarding their use of function definitions, assignments, and more. It is used to identify common patterns in R code and is available with the [\`statistics\`](#generate-usage-statistics-of-r-code) script. The [core](https://github.com/flowr-analysis/flowr/tree/main/src/core) module contains _flowR_'s read-eval-print loop (REPL) and -_flowR_'s server. Furthermore, it contains the root definitions of how _flowR_ slices (see the ${ctx.linkPage('wiki/Interface')} wiki page for more information). +_flowR_'s server. Furthermore, it contains the root definitions of how _flowR_ slices (see the ${ctx.linkPage('wiki/Interface')} wiki page for more information). The [utility](https://github.com/flowr-analysis/flowr/tree/main/src/util) module is of no further interest for the usage of _flowR_ @@ -68,14 +69,14 @@ Similarly, we offer an [Addin for RStudio](${FlowrRStudioAddin}), as well as an ⚒️ If you compile the _flowR_ sources yourself, you can access _flowR_ by the main script \`npm run flowr\` or in the development mode \`npm run main-dev\`. Independent of your way of launching *flowr*, we will write simply \`flowr\` for either (🐳️) \`docker run -it --rm eagleoutice/flowr:latest\` or (⚒️) \`npm run flowr\`. -See the ${ctx.linkPage('wiki/Setup')} wiki page for more information on how to get _flowR_ running. +See the ${ctx.linkPage('wiki/Setup')} wiki page for more information on how to get _flowR_ running. ### The Read-Eval-Print Loop (REPL) Once you launched _flowR_, you should see a small \`R>\` prompt. Use \`:help\` to receive instructions on how to use the REPL and what features are available (most prominently, you can access all [scripts](#calling-the-scripts-directly) simply by adding a colon before them). In general, all commands start with a colon (\`:\`), everything else is interpreted as a R expression which is directly evaluated by the underlying R shell -(however, due to security concerns, you need to start _flowR_ with ${ctx.cliOption('flowr', 'r-session-access')} and use the \`r-shell\` ${ctx.linkPage('wiki/Engines', 'engine')} to allow this). -See the ${ctx.linkPage('wiki/Interface')} wiki page for more information on usage and the available commands. +(however, due to security concerns, you need to start _flowR_ with ${ctx.cliOption('flowr', 'r-session-access')} and use the \`r-shell\` ${ctx.linkPage('wiki/Engines', 'engine')} to allow this). +See the ${ctx.linkPage('wiki/Interface')} wiki page for more information on usage and the available commands. The following GIF showcases a simple example session: ![Example of a simple REPL session](gif/repl-demo-opt.gif) @@ -110,7 +111,7 @@ Sleep 20000ms ### The Server Instead of the REPL, you can start _flowR_ in "([TCP](https://de.wikipedia.org/wiki/Transmission_Control_Protocol)) server-mode" using \`flowr --server\` (write \`flowr --help\` to find out more). Together with the server option, you can configure the port with ${ctx.cliOption('flowr', 'port')}. -The supported requests are documented alongside the internal documentation, see the ${ctx.linkPage('wiki/Interface')} wiki page for more information. +The supported requests are documented alongside the internal documentation, see the ${ctx.linkPage('wiki/Interface')} wiki page for more information.
Small demonstration using netcat @@ -153,7 +154,7 @@ Sleep 200ms
The server allows accessing the REPL as well -(see the ${ctx.linkPage('wiki/Interface')} wiki page for more information). +(see the ${ctx.linkPage('wiki/Interface')} wiki page for more information). ## Calling the Scripts Directly diff --git a/src/documentation/wiki-query.ts b/src/documentation/wiki-query.ts index 07dbc444dc5..c6fb4a5e420 100644 --- a/src/documentation/wiki-query.ts +++ b/src/documentation/wiki-query.ts @@ -49,6 +49,7 @@ import { executeCallGraphQuery } from '../queries/catalog/call-graph-query/call- import { executeRecursionQuery } from '../queries/catalog/inspect-recursion-query/inspect-recursion-query-executor'; import { executeDoesCallQuery } from '../queries/catalog/does-call-query/does-call-query-executor'; import { executeExceptionQuery } from '../queries/catalog/inspect-exceptions-query/inspect-exception-query-executor'; +import type { AllWikiDocuments } from '../cli/wiki'; registerQueryDocumentation('call-context', { @@ -156,7 +157,7 @@ registerQueryDocumentation('call-graph', { buildExplanation: async(shell: RShell, ctx: GeneralDocContext) => { const exampleCode = 'x + 1'; return ` -This query calculates and returns the ${ctx.linkPage('wiki/Dataflow Graph', 'call graph', 'perspectives-cg')} of the given code. +This query calculates and returns the ${ctx.linkPage('wiki/Dataflow Graph', 'call graph', 'perspectives-cg')} of the given code. Using the example code \`${exampleCode}\`, the following query returns the dataflow graph of the code: ${ @@ -893,7 +894,7 @@ export class WikiQuery extends DocMaker<'wiki/Query API.md'> { protected async text({ ctx, shell }: DocMakerArgs): Promise { return ` This page briefly summarizes flowR's query API, represented by the ${executeQueries.name} function in ${getFilePathMd('../queries/query.ts')}. -Please see the ${ctx.linkPage('wiki/Interface')} wiki page for more information on how to access this API. +Please see the ${ctx.linkPage('wiki/Interface')} wiki page for more information on how to access this API. ${ block({ diff --git a/src/documentation/wiki-setup.ts b/src/documentation/wiki-setup.ts index b454d9b9bec..45297ccfca6 100644 --- a/src/documentation/wiki-setup.ts +++ b/src/documentation/wiki-setup.ts @@ -1,6 +1,7 @@ import type { DocMakerArgs } from './wiki-mk/doc-maker'; import { DocMaker } from './wiki-mk/doc-maker'; import { FlowrDockerRef, FlowrPositron, FlowrRStudioAddin, FlowrVsCode } from './doc-util/doc-files'; +import type { AllWikiDocuments } from '../cli/wiki'; /** * https://github.com/flowr-analysis/flowr/wiki/Setup @@ -61,7 +62,7 @@ To start flowr as a server, you can run: docker run -it --rm -p1042:1042 eagleoutice/flowr --server \`\`\` -For more information, see the ${ctx.linkPage('wiki/Interface')} wiki page. +For more information, see the ${ctx.linkPage('wiki/Interface')} wiki page. ## ⚒️ Building From Scratch