Skip to content
Open
Show file tree
Hide file tree
Changes from 47 commits
Commits
Show all changes
61 commits
Select commit Hold shift + click to select a range
54c3375
feat(grading): add interactive AI comment selection and word-level di…
tzj04 Feb 25, 2026
8ee56da
feature(ai_grading): Rendering generate comments button based on xml …
leongyiquan Feb 25, 2026
e244d31
Merge remote-tracking branch 'yiquan/feat/generate-comments-toggling'…
tzj04 Feb 25, 2026
186f47c
feat(grading): Add LLM feedback submission for grading comments
tzj04 Mar 3, 2026
645fe92
feat(groundControl): Add LLM usage statistics monitoring
tzj04 Mar 3, 2026
8f89539
feat(core): Add LLM types and request handlers
tzj04 Mar 3, 2026
d598501
feat: 3 tiered prompt generation and prompt button adjustments
leongyiquan Mar 8, 2026
895e3a2
feat(grading): streamline AI comment selection with inline editor syn…
tzj04 Mar 16, 2026
28e3c3d
refactor(api): simplify saveChosenComments to answer-based endpoint
tzj04 Mar 16, 2026
7f2523d
Merge branch 'master' into feature-ai-comments
tzj04 Mar 16, 2026
43cdbe3
merge: resolve conflicts and integrate 3-tier prompting with ai comments
leongyiquan Mar 20, 2026
eacfe94
feat: implement AI token cost tracking and stats UI
leongyiquan Mar 20, 2026
9cfa21a
Added a new tab for coursewide summary for LLM stats
leongyiquan Mar 28, 2026
df6a1e1
Fixed fail save bug
leongyiquan Mar 28, 2026
518102c
prevent unsafe changes bug
leongyiquan Mar 28, 2026
0d4eb8c
fix(grading): prevent false unsaved state and guard save-and-continue…
tzj04 Mar 29, 2026
ffda3b2
fix(grading): preserve empty AI comment slots to keep index mapping s…
tzj04 Mar 29, 2026
e0fa5f7
fix: add timeout fallback for save controls unlock on grading failure
tzj04 Mar 29, 2026
cb0bc0d
feat(grading): make LLM feedback star rating keyboard-accessible and …
tzj04 Mar 29, 2026
58831f5
fix(grading): use typed selector hook in GradingEditor
tzj04 Mar 29, 2026
23979a1
fix(grading): use shared typed dispatch hook to avoid restricted reac…
tzj04 Mar 29, 2026
d04b4ec
fix(grading): gate is_llm using grading.enable_llm_grading instead of…
tzj04 Mar 30, 2026
4636129
fix(grading): keep LLM comment UX visible without prompts and gate pr…
tzj04 Mar 30, 2026
d1802d8
merge: resolve conflicts with upstream master
tzj04 Mar 30, 2026
9a9c51b
fix: Replaced direct yield of postGrading with a redux-saga CALL effect
tzj04 Mar 30, 2026
903368f
fix(navbar): remove stale hasLlmContent prop from academy navbar righ…
tzj04 Mar 30, 2026
426cdaf
chore(deps): update caniuse-lite browsers data
tzj04 Mar 30, 2026
c67418d
test(backend-saga): fix submit grading-and-continue navigation assert…
tzj04 Mar 30, 2026
fbe8b57
fix: sort imports in academyRoutes.test.ts
tzj04 Mar 30, 2026
d53877f
fix(types): add local declaration shim for @blueprintjs/core
tzj04 Mar 31, 2026
382fb50
fix(groundControl): resolve React 19 runtime compatibility issues
tzj04 Mar 31, 2026
3aaa053
fix(llm-stats): label feedback by task display order
tzj04 Mar 31, 2026
8672ad9
fix(grading): persist and rehydrate AI comment selection state; keep …
tzj04 Mar 31, 2026
2cd0377
fixed hard coded bug for llm in grading page
leongyiquan Mar 31, 2026
4e9ce4e
fixed data handling mismatch
leongyiquan Mar 31, 2026
9ec0c41
fixed formatting issue with \n
leongyiquan Mar 31, 2026
f1cc545
Merge branch 'master' into feature-ai-comments
martin-henz Apr 1, 2026
90f52f2
fix(ground-control): ensure action buttons remain visible on small sc…
tzj04 Apr 3, 2026
1beb5b1
fix(grading): refactor AI comment selection with unified UI and persi…
tzj04 Apr 3, 2026
71e04d9
fix(grading): refactor AI comment selection
tzj04 Apr 3, 2026
d2580aa
Update LLMStatsTab and LLMStatsPage
tzj04 Apr 3, 2026
1617adf
Merge branch 'tzj04-feature-ai-comments'
tzj04 Apr 3, 2026
0bf8aa0
Save LLMStats updates before merging master
tzj04 Apr 3, 2026
5fb13e2
Merge branch 'master' into feature-ai-comments
tzj04 Apr 3, 2026
c938620
Merge branch 'master' of https://github.com/source-academy/frontend i…
tzj04 Apr 3, 2026
608f723
fix(grading): persist cleared AI comment selection after re-generate …
tzj04 Apr 3, 2026
b71637f
Merge branch 'master' into feature-ai-comments
martin-henz Apr 6, 2026
e14306a
Fixed rollback logic for AI comments
leongyiquan Apr 7, 2026
047c2d5
Merge branch 'master' into feature-ai-comments
tzj04 Apr 7, 2026
a4b15f5
Merge branch 'master' into feature-ai-comments
tzj04 Apr 9, 2026
a16bfaa
fix(grading): prevent AI comment rollback corruption on regenerate+sa…
tzj04 Apr 9, 2026
efc78d9
resolve merge conlficts with 'master'
tzj04 Apr 9, 2026
1512451
Fix: resolve conflicts between local master and upstream
tzj04 Apr 13, 2026
e91be09
Merge branch 'master' into feature-ai-comments
tzj04 Apr 13, 2026
4febf6a
Fix incorrect past merge resolution
RichDom2185 May 7, 2026
f717559
Merge branch 'master' of https://github.com/source-academy/frontend i…
RichDom2185 May 7, 2026
4f1b9a3
Undo more incorrect changes
RichDom2185 May 7, 2026
80975b5
Remove unnecessary Vitest config change
RichDom2185 May 7, 2026
f5c2920
Merge branch 'master' of https://github.com/source-academy/frontend i…
RichDom2185 May 7, 2026
09c4815
Fix errors post-merge
RichDom2185 May 7, 2026
4186f20
Migrate changes post-merge
RichDom2185 May 7, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
363 changes: 363 additions & 0 deletions src/commons/adminStats/LLMStatsTab.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,363 @@
import {
Button,
Card,
Divider,
H3,
H4,
Icon,
Intent,
NonIdealState,
Spinner,
SpinnerSize,
Tag
} from '@blueprintjs/core';
import { IconNames } from '@blueprintjs/icons';
import React, { useEffect, useMemo, useState } from 'react';

import { LLMAssessmentStat, LLMCourseStat, LLMQuestionStat } from '../assessment/AssessmentTypes';
import { getLLMCourseStats } from '../sagas/RequestsSaga';
import { useTokens } from '../utils/Hooks';

const formatNumber = (value: number | string | null | undefined): string => {
if (value === null || value === undefined || value === '') {
return '0';
}
if (typeof value === 'string') {
const parsed = Number(value);
return Number.isNaN(parsed) ? value : parsed.toLocaleString();
}
return value.toLocaleString();
};

const formatCost = (value: number | string | null | undefined): string => {
if (value === null || value === undefined || value === '') {
return '$0.00';
}
const parsed = Number(value);
if (Number.isNaN(parsed)) {
return `${value}`;
}
return `$${parsed.toFixed(2)}`;
};

const getTotalTokens = (assessment: LLMAssessmentStat): number => {
return (assessment.llm_total_input_tokens || 0) + (assessment.llm_total_output_tokens || 0);
};

const getQuestionTotalTokens = (question: LLMQuestionStat): number => {
return (question.llm_total_input_tokens || 0) + (question.llm_total_output_tokens || 0);
};

const LLMStatsTab: React.FC = () => {
const { accessToken, refreshToken } = useTokens();
const [loading, setLoading] = useState(false);
const [error, setError] = useState<string | null>(null);
const [data, setData] = useState<LLMCourseStat | null>(null);
const [sortKey, setSortKey] = useState<
'title' | 'category' | 'total_uses' | 'total_tokens' | 'llm_total_cost' | 'avg_rating'
>('llm_total_cost');
const [sortDirection, setSortDirection] = useState<'asc' | 'desc'>('desc');
const [expanded, setExpanded] = useState<Set<number>>(new Set());

useEffect(() => {
const fetch = async () => {
setLoading(true);
setError(null);
try {
const resp = await getLLMCourseStats({ accessToken, refreshToken });
if (!resp) {
setError('Failed to load LLM statistics.');
} else {
setData(resp);
}
} catch (err) {
setError('Failed to load LLM statistics.');
} finally {
setLoading(false);
}
};

fetch();
}, [accessToken, refreshToken]);

const sortedAssessments: LLMAssessmentStat[] = useMemo(() => {
if (!data?.assessments) {
return [];
}

const assessments = [...data.assessments];
const compare = (a: LLMAssessmentStat, b: LLMAssessmentStat): number => {
const direction = sortDirection === 'asc' ? 1 : -1;
switch (sortKey) {
case 'title':
return direction * a.title.localeCompare(b.title);
case 'category':
return direction * a.category.localeCompare(b.category);
case 'total_uses':
return direction * (a.total_uses - b.total_uses);
case 'avg_rating':
return direction * ((a.avg_rating || 0) - (b.avg_rating || 0));
case 'llm_total_cost':
return direction * (Number(a.llm_total_cost || 0) - Number(b.llm_total_cost || 0));
case 'total_tokens':
return direction * (getTotalTokens(a) - getTotalTokens(b));
default:
return 0;
}
};

return assessments.sort(compare);
}, [data, sortKey, sortDirection]);

const toggleSort = (key: typeof sortKey) => {
if (sortKey === key) {
setSortDirection(sortDirection === 'asc' ? 'desc' : 'asc');
} else {
setSortKey(key);
setSortDirection('desc');
}
};

const toggleRow = (assessmentId: number) => {
const next = new Set(expanded);
if (next.has(assessmentId)) {
next.delete(assessmentId);
} else {
next.add(assessmentId);
}
setExpanded(next);
};

const downloadReport = () => {
if (!data) {
return;
}

const headers = [
'Assessment Title',
'Category',
'Question Number',
'Uses',
'Input Tokens',
'Output Tokens',
'Total Tokens',
'Cost',
'Avg Rating'
];

const rows: string[] = [];
data.assessments.forEach(assessment => {
if (assessment.questions.length === 0) {
rows.push(
[
assessment.title,
assessment.category,
'',
assessment.total_uses.toString(),
assessment.llm_total_input_tokens.toString(),
assessment.llm_total_output_tokens.toString(),
getTotalTokens(assessment).toString(),
Number(assessment.llm_total_cost || 0).toFixed(6),
assessment.avg_rating?.toString() ?? ''
].join(',')
);
} else {
assessment.questions.forEach(question => {
rows.push(
[
assessment.title,
assessment.category,
question.display_order.toString(),
question.total_uses.toString(),
question.llm_total_input_tokens.toString(),
question.llm_total_output_tokens.toString(),
getQuestionTotalTokens(question).toString(),
Number(question.llm_total_cost || 0).toFixed(6),
question.avg_rating?.toString() ?? ''
].join(',')
);
});
}
});

const content = [headers.join(','), ...rows].join('\n');
const blob = new Blob([content], { type: 'text/csv;charset=utf-8;' });
const url = URL.createObjectURL(blob);
const link = document.createElement('a');

link.href = url;
link.setAttribute('download', 'llm_stats_report.csv');
document.body.appendChild(link);
link.click();
document.body.removeChild(link);
URL.revokeObjectURL(url);
};

if (loading) {
return (
<div style={{ padding: '20px', textAlign: 'center' }}>
<Spinner size={SpinnerSize.STANDARD} />
</div>
);
}

if (error) {
return (
<NonIdealState
icon={IconNames.ERROR}
title="Unable to load LLM statistics"
description={error}
/>
);
}

if (!data || data.assessments.length === 0) {
return (
<NonIdealState
icon={IconNames.CHART}
title="No LLM usage data yet"
description="Once students start generating comments, this panel will show LLM statistics."
/>
);
}

return (
<div style={{ padding: '14px' }}>
<div style={{ display: 'flex', gap: '12px', marginBottom: '12px', flexWrap: 'wrap' }}>
<Card style={{ flex: 1, textAlign: 'center' }}>
<H3 style={{ margin: 0 }}>{formatNumber(data.course_total_input_tokens)}</H3>
<div>Course-wide Input Tokens</div>
</Card>
<Card style={{ flex: 1, textAlign: 'center' }}>
<H3 style={{ margin: 0 }}>{formatNumber(data.course_total_output_tokens)}</H3>
<div>Course-wide Output Tokens</div>
</Card>
<Card style={{ flex: 1, textAlign: 'center' }}>
<H3 style={{ margin: 0 }}>{formatCost(data.course_total_cost)}</H3>
<div>Total LLM Expenditure</div>
</Card>
</div>

<div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center' }}>
<H4 style={{ margin: '8px 0' }}>Assessments</H4>
<Button icon={IconNames.DOWNLOAD} intent={Intent.PRIMARY} onClick={downloadReport}>
Download Report
</Button>
</div>

<table
className="bp3-html-table bp3-html-table-striped bp3-html-table-condensed"
style={{ width: '100%', marginTop: '8px' }}
>
<thead>
<tr>
<th>
<Button minimal={true} small={true} onClick={() => toggleSort('title')}>
Title
</Button>
</th>
<th>
<Button minimal={true} small={true} onClick={() => toggleSort('category')}>
Category
</Button>
</th>
<th>
<Button minimal={true} small={true} onClick={() => toggleSort('total_uses')}>
Uses
</Button>
</th>
<th>
<Button minimal={true} small={true} onClick={() => toggleSort('total_tokens')}>
Total Tokens
</Button>
</th>
<th>
<Button minimal={true} small={true} onClick={() => toggleSort('llm_total_cost')}>
Total Cost
</Button>
</th>
<th>
<Button minimal={true} small={true} onClick={() => toggleSort('avg_rating')}>
Avg Rating
</Button>
</th>
</tr>
</thead>
<tbody>
{sortedAssessments.map(assessment => {
const isExpanded = expanded.has(assessment.assessment_id);
return (
<React.Fragment key={assessment.assessment_id}>
<tr
style={{ cursor: 'pointer' }}
onClick={() => toggleRow(assessment.assessment_id)}
>
<td>
<span style={{ marginRight: '6px' }}>
<Icon icon={isExpanded ? IconNames.CARET_DOWN : IconNames.CARET_RIGHT} />
</span>
{assessment.title}
</td>
<td>{assessment.category}</td>
<td>{assessment.total_uses}</td>
<td>{formatNumber(getTotalTokens(assessment))}</td>
<td>{formatCost(assessment.llm_total_cost)}</td>
<td>
{assessment.avg_rating === null || assessment.avg_rating === undefined
? 'N/A'
: assessment.avg_rating.toFixed(2)}
</td>
</tr>
{isExpanded && assessment.questions.length > 0 && (
<tr>
<td colSpan={6} style={{ padding: 0, background: '#f3f3f3' }}>
<div style={{ padding: '8px' }}>
<table
className="bp3-html-table bp3-html-table-condensed"
style={{ width: '100%' }}
>
<thead>
<tr>
<th>Question</th>
<th>Uses</th>
<th>Tokens</th>
<th>Cost</th>
<th>Avg Rating</th>
</tr>
</thead>
<tbody>
{assessment.questions.map(question => (
<tr key={question.question_id}>
<td>{question.display_order}</td>
<td>{question.total_uses}</td>
<td>{formatNumber(getQuestionTotalTokens(question))}</td>
<td>{formatCost(question.llm_total_cost)}</td>
<td>
{question.avg_rating === null || question.avg_rating === undefined
? 'N/A'
: question.avg_rating.toFixed(2)}
</td>
</tr>
))}
</tbody>
</table>
</div>
</td>
</tr>
)}
</React.Fragment>
);
})}
</tbody>
</table>

<Divider style={{ marginTop: '16px' }} />

<Tag minimal intent={Intent.PRIMARY} style={{ marginTop: '8px' }}>
Data is shown for published assessments and questions with LLM prompts.
</Tag>
</div>
);
};

export default LLMStatsTab;
1 change: 1 addition & 0 deletions src/commons/application/ApplicationTypes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -541,6 +541,7 @@ export const defaultSession: SessionState = {
students: undefined,
teamFormationOverviews: undefined,
gradings: {},
gradingSaveResult: undefined,
notifications: []
};

Expand Down
7 changes: 7 additions & 0 deletions src/commons/application/actions/SessionActions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,13 @@ const SessionActions = createActions('session', {
* no id for Grading.
*/
updateGrading: (submissionId: number, grading: GradingQuery) => ({ submissionId, grading }),
updateGradingSaveResult: (
submissionId: number,
questionId: number,
success: boolean,
saveAndContinue: boolean,
requestId: number
) => ({ submissionId, questionId, success, saveAndContinue, requestId }),
unsubmitSubmission: (submissionId: number) => ({ submissionId }),
// Publishing / unpublishing actions
publishGrading: (submissionId: number) => ({ submissionId }),
Expand Down
Loading
Loading