experiments

download_model.log

@@ -0,0 +1,5 @@
+Downloading shards:   0%|          | 0/6 [00:00<?, ?it/s]Downloading shards:  17%|█▋        | 1/6 [00:37<03:06, 37.34s/it]Downloading shards:  33%|███▎      | 2/6 [03:08<06:32, 98.13s/it]Downloading shards:  50%|█████     | 3/6 [04:31<04:57, 99.08s/it]Downloading shards:  67%|██████▋   | 4/6 [07:02<03:41, 110.88s/itDownloading shards:  83%|████████▎ | 5/6 [08:22<01:49, 109.16s/it]Downloading shards: 100%|██████████| 6/6 [10:20<00:00, 102.41s/it]Downloading shards: 100%|██████████| 6/6 [10:20<00:00, 103.35s/it]LLoading checkpoint shards:   0%|          | 0/6 [00:00<?, ?it/Loading checkpoint shards:  33%|███▎      | 2/6 [00:00<00:00, 18.95it/Loading checkpoint shards:  67%|██████▋   | 4/6 [00:00<00:00, 18.05it/Loading checkpoint shards: 100%|██████████| 6/6 [00:00<00:00, 18.60it/s]Loading checkpoint shards: 100%|██████████| 6/6 [00:00<00:00, 18.53it/sDone
+nload complete
+��| 6/6 [10:32<00:00, 103.11s/it]Downloading shards: 100%|██████████| 6/6 [10:32<00:00, 105.46s/it]
+Loading checkpoint shards:   0%|          | 0/6 [00:00<?, ?it/s]Loading checkpoint shards:  33%|███▎      | 2/6 [00:00<00:00, 18.67it/s]Loading checkpoint shards:  67%|██████▋   | 4/6 [00:00<00:00, 17.83it/s]Loading checkpoint shards: 100%|██████████| 6/6 [00:00<00:00, 18.39it/s]Loading checkpoint shards: 100%|██████████| 6/6 [00:00<00:00, 18.32it/s]
+Download complete

experiments/analyze_results.py

@@ -0,0 +1,40 @@
+"""
+Analyze and visualize results from run_trajectory_uq.py
+Usage: python3 analyze_results.py results/triviaqa_traj_uq.json
+"""
+
+import json
+import sys
+import numpy as np
+
+def main(result_file):
+    with open(result_file) as f:
+        data = json.load(f)
+
+    print("=" * 60)
+    print("Trajectory-Based UQ for LLaDA — TriviaQA Results")
+    print("=" * 60)
+    print(f"Model:      {data['args']['model_name']}")
+    print(f"N examples: {data['n_total']}")
+    print(f"Steps:      {data['args']['steps']}")
+    print(f"Gen length: {data['args']['gen_length']}")
+    print(f"Accuracy:   {data['accuracy']:.3f} ({data['n_correct']}/{data['n_total']})")
+    print()
+
+    print("Uncertainty Metric Evaluation:")
+    print(f"  {'Metric':<22} {'AUROC':>8} {'ECE':>8}")
+    print("  " + "-" * 40)
+    for metric, vals in data['eval_results'].items():
+        print(f"  {metric:<22} {vals['auroc']:>8.4f} {vals['ece']:>8.4f}")
+
+    print()
+    print("Sample predictions (first 5):")
+    for s in data['samples'][:5]:
+        status = "✓" if s['correct'] else "✗"
+        print(f"  [{status}] Q: {s['question'][:60]}")
+        print(f"      Gen: {s['generated'][:60]}")
+        print(f"      mean_entropy={s['mean_entropy']:.3f}  traj_var={s['traj_variance']:.3f}")
+        print()
+
+if __name__ == '__main__':
+    main(sys.argv[1] if len(sys.argv) > 1 else 'results/triviaqa_traj_uq.json')