Update reporting / figures

Makefile

@@ -116,7 +116,7 @@ run_envs: clone_yamls
 
 
 knit_report: clone_reports
-	R -e 'rmarkdown::render("$(REPORTS_DIR)/07_metrics_across_backends.Rmd", params = list(performance_bn = "performance-results.rds", metrics_bn = "metrics-results.rds", clustering_dir =  "."))'
-	R -e 'rmarkdown::render("$(REPORTS_DIR)/08_performances_across_backends.Rmd", params = list(performance_bn = "performance-results.rds", metrics_bn = "metrics-results.rds", clustering_dir =  "."))'
-	python parse_results.py
-	R -e 'rmarkdown::render("analyze_results.Rmd")'
+	# R -e 'rmarkdown::render("$(REPORTS_DIR)/07_metrics_across_backends.Rmd", params = list(performance_bn = "performance-results.rds", metrics_bn = "metrics-results.rds", clustering_dir =  "."))'
+	# R -e 'rmarkdown::render("$(REPORTS_DIR)/08_performances_across_backends.Rmd", params = list(performance_bn = "performance-results.rds", metrics_bn = "metrics-results.rds", clustering_dir =  "."))'
+	python parse_results.py > aggregated_results.json
+	R -e 'rmarkdown::render("analyze_results_izaskun.Rmd")'

analyze_results_izaskun.Rmd

@@ -0,0 +1,224 @@
+---
+title: "Clustbench Performance Analysis"
+output:
+  html_document: default
+date: "`r Sys.Date()`"
+---
+
+```{r setup, include=TRUE}
+library(knitr)
+library(tidyverse)
+library(jsonlite)
+library(ggplot2)
+
+knitr::opts_chunk$set(
+  echo    = TRUE,
+  warning = TRUE,
+  message = TRUE,
+  fig.path = "plots/",
+  dev = c("png", "svg"),
+  fig.width  = 6,
+  fig.height = 6)
+
+
+```
+
+## Load Data
+
+```{r load-data}
+# Run the Python script and capture JSON output
+json_output <- system("python3 parse_results.py 2>/dev/null", intern = TRUE)
+json_text <- paste(json_output, collapse = "\n")
+```
+
+
+```{r}
+# flatten configurations into long rows of param_name/param_value
+flatten_configurations <- function(cfgs) {
+  if (is.null(cfgs) || length(cfgs) == 0) {
+    return(data.frame(parameter_dir = NA_character_,
+                      param_name    = NA_character_,
+                      param_value   = NA_character_,
+                      stringsAsFactors = FALSE))
+  }
+  out <- data.frame()
+  for (cfg in cfgs) {
+    parameter_dir <- cfg$parameter_dir
+    params <- cfg$parameters
+    if (is.data.frame(params)) params <- as.list(params)
+    flat <- unlist(params, use.names = TRUE)
+    for (p in names(flat)) {
+      out <- rbind(out, data.frame(
+        parameter_dir = parameter_dir,
+        param_name    = p,
+        param_value   = as.character(flat[[p]]),
+        stringsAsFactors = FALSE
+      ))
+    }
+  }
+  out
+}
+
+flatten_record <- function(rec) {
+  pm   <- rec$metrics$partition_metrics
+  perf <- rec$performance
+  cfg_df <- flatten_configurations(rec$configurations)
+
+  rows <- list()
+  idx  <- 0
+
+  for (metric_name in names(pm)) {
+    metric_vals <- pm[[metric_name]]
+    for (k in names(metric_vals)) {
+      for (i in seq_len(nrow(cfg_df))) {
+        idx <- idx + 1
+        rows[[idx]] <- data.frame(
+          backend      = rec$backend,
+          seed         = rec$seed,
+          run          = rec$run,
+          generator    = rec$generator,
+          dataset_name = rec$dataset_name,
+          method       = rec$method,
+          path         = rec$path,
+          method_params= rec$method_params,
+          method_full  = rec$method_full,
+          parameter_dir= cfg_df$parameter_dir[i],
+          param_name   = cfg_df$param_name[i],
+          param_value  = cfg_df$param_value[i],
+          k            = as.integer(k),
+          metric_name  = metric_name,
+          metric_value = metric_vals[[k]],
+          # performance metrics
+          s        = perf$s,
+          h_m_s    = perf[["h:m:s"]],
+          max_rss  = perf$max_rss,
+          max_vms  = perf$max_vms,
+          max_uss  = perf$max_uss,
+          max_pss  = perf$max_pss,
+          io_in    = perf$io_in,
+          io_out   = perf$io_out,
+          mean_load= perf$mean_load,
+          cpu_time = perf$cpu_time,
+          stringsAsFactors = FALSE
+        )
+      }
+    }
+  }
+  do.call(rbind, rows)
+}
+
+records <- jsonlite::fromJSON(json_output, simplifyVector = FALSE)
+
+fd <- do.call(rbind, lapply(records, flatten_record))
+
+head(fd)
+
+table(fd$param_name)
+write.csv(fd, file = 'aggregated_results.csv')
+
+```
+
+```{r}
+print(ggplot(fd[fd$backend %in% c("conda","oras","envmodules"), ],
+       aes(x = backend, y = cpu_time, fill = backend)) +
+  geom_boxplot(outlier.alpha = 0.3) +
+  theme_minimal(base_size = 14) +
+  labs(title = "CPU time by backend",
+       x = "Backend",
+       y = "CPU Time (s)") +
+  scale_fill_brewer(palette = "Set2"))
+```
+
+
+```{r}
+
+ggplot(fd[fd$backend %in% c("conda","oras","envmodules"), ],
+       aes(x = backend, y = metric_value, color = backend)) +
+  geom_boxplot(outlier.alpha = 0.3) +
+  facet_wrap(~metric_name, scales = "free_y") +
+  theme_minimal(base_size = 14) +
+  labs(title = "Metrics consistency across backends",
+       x = "Backend",
+       y = "Metric Value") +
+  scale_color_brewer(palette = "Set1")
+```
+
+```{r}
+df_run <- fd %>%
+  select(dataset_name, method, k, metric_name, run, seed, backend, metric_value)
+
+
+# a bit of collapsing here!
+wide_run <- fd %>%
+  filter(backend %in% c("conda","oras","envmodules")) %>%
+  group_by(dataset_name, method, k, metric_name, run, seed, backend) %>%
+  summarise(metric_value = mean(as.numeric(metric_value), na.rm = TRUE), .groups = "drop") %>%
+  pivot_wider(names_from = backend, values_from = metric_value)
+
+# Check structure
+str(wide_run)
+
+cors_run <- wide_run %>%
+  group_by(metric_name) %>%
+  summarise(
+    cor_conda_oras   = cor(conda, oras, use="complete.obs"),
+    cor_conda_envmodules  = cor(conda, envmodules, use="complete.obs"),
+    cor_oras_envmodules = cor(oras, envmodules, use="complete.obs")
+  )
+
+print(cors_run)
+
+```
+
+
+```{r, fig.width = 10, fig.height=10}
+
+perf_metrics <- c("cpu_time","max_rss","max_vms","max_uss",
+                  "max_pss","io_in","io_out","mean_load")
+
+fd_long <- fd %>%
+  filter(backend %in% c("conda","oras","envmodules")) %>%
+  pivot_longer(cols = all_of(perf_metrics),
+               names_to = "metric",
+               values_to = "value")
+
+#  boxplots + jittered scatter, faceted by metric
+ggplot(fd_long, aes(x = backend, y = value, fill = backend)) +
+  geom_boxplot(outlier.alpha = 0.3) +
+  geom_jitter(width = 0.2, alpha = 0.1, size = 1, color = "black") +
+  facet_wrap(~metric, scales = "free_y") +
+  theme_minimal(base_size = 14) +
+  labs(title = "Performance metrics by backend",
+       x = "Backend",
+       y = "Value") +
+  scale_fill_brewer(palette = "Set2")
+
+```
+
+Choice of `k` - we lack the annotation of the true k in the long CSV
+
+
+```{r, fig.height = 10, fig.width = 10}
+
+ggplot(fd, aes(x = factor(k), y = metric_value, fill = method)) +
+  geom_jitter(width = 0.2, alpha = 0.4, size = 1, fill = "black") +
+  facet_grid(method ~ metric_name + dataset_name, scales = "free_y") +
+  theme_minimal(base_size = 14) +
+  labs(title = "Clustering metric sensitivity to k",
+       x = "number of clusters k (true or not)",
+       y = "Metric value")
+```
+
+
+
+```{r, fig.height = 10, fig.width = 10}
+ggplot(fd, aes(x = factor(k), y = metric_value, fill = method)) +
+    geom_jitter(width = 0.2, alpha = 0.4, size = 1, color = "black") +
+    facet_grid(dataset_name + method ~ metric_name, scales = "free_y") +
+    theme_minimal(base_size = 14) +
+    labs(title = "Clustering metric sensitivity to k",
+         x = "Number of clusters (k), true or not",
+         y = "Metric value")
+
+
+```

parse_results.py

@@ -13,40 +13,83 @@
 from typing import Dict, List, Optional
 
 
+# def parse_result_path(path: Path) -> Dict[str, str]:
+#     """
+#     Parse a result path and extract components.
+
+#     """
+#     parts = path.parts
+
+#     # print(pars)
+
+#     result = {}
+
+#     # Parse out-{backend}-{rep}
+#     out_match = re.match( r"out_(?P<backend>[a-zA-Z0-9]+)_seed_(?P<seed>\d+)_run_(?P<run>\d+)", parts)
+#     if out_match:
+#         result['backend'] = out_match.group(1)
+#         result['seed'] = out_match.group(2)
+#         result['rep'] = out_match.group(3)
+
+#     # Find dataset_generator part
+#     for part in parts:
+#         if part.startswith('dataset_generator-'):
+#             # Parse dataset_generator-{generator}_dataset_name-{name}
+#             dataset_match = re.match(r'dataset_generator-([^_]+)_dataset_name-(.+)', part)
+#             if dataset_match:
+#                 result['generator'] = dataset_match.group(1)
+#                 result['dataset_name'] = dataset_match.group(2)
+#             break
+
+#     # The method is the last part (after clustering/)
+#     if 'clustering' in parts:
+#         clustering_idx = parts.index('clustering')
+#         if clustering_idx + 1 < len(parts):
+#             result['method'] = parts[clustering_idx + 1]
+
+#     result['path'] = str(path)
+
+#     return result
+
+
 def parse_result_path(path: Path) -> Dict[str, str]:
     """
-    Parse a result path and extract components.
-
-    Pattern: out-{backend}-{rep}/data/clustbench/dataset_generator-{generator}_dataset_name-{name}/clustering/{method}
+    Parse a result path and extract components:
+    - backend, seed, run (from out_* directories)
+    - dataset generator and dataset name
+    - method (after clustering/)
     """
     parts = path.parts
+    result: Dict[str, str] = {}
 
-    result = {}
+    # parse out_{backend}_seed_{seed}_run_{run}
+    out_match = re.match(
+        r"out_(?P<backend>[a-zA-Z0-9]+)_seed_(?P<seed>\d+)_run_(?P<run>\d+)",
+        parts[0])
 
-    # Parse out-{backend}-{rep}
-    out_match = re.match(r'out-([^-]+)-(\d+)', parts[0])
     if out_match:
-        result['backend'] = out_match.group(1)
-        result['rep'] = out_match.group(2)
+        result["backend"] = out_match.group("backend")
+        result["seed"] = out_match.group("seed")
+        result["run"] = out_match.group("run")
 
     # Find dataset_generator part
     for part in parts:
-        if part.startswith('dataset_generator-'):
-            # Parse dataset_generator-{generator}_dataset_name-{name}
-            dataset_match = re.match(r'dataset_generator-([^_]+)_dataset_name-(.+)', part)
+        if part.startswith("dataset_generator-"):
+            dataset_match = re.match(
+                r"dataset_generator-([^_]+)_dataset_name-(.+)", part
+            )
             if dataset_match:
-                result['generator'] = dataset_match.group(1)
-                result['dataset_name'] = dataset_match.group(2)
+                result["generator"] = dataset_match.group(1)
+                result["dataset_name"] = dataset_match.group(2)
             break
 
     # The method is the last part (after clustering/)
-    if 'clustering' in parts:
-        clustering_idx = parts.index('clustering')
+    if "clustering" in parts:
+        clustering_idx = parts.index("clustering")
         if clustering_idx + 1 < len(parts):
-            result['method'] = parts[clustering_idx + 1]
-
-    result['path'] = str(path)
+            result["method"] = parts[clustering_idx + 1]
 
+    result["path"] = str(path)
     return result
 
 
@@ -112,7 +155,7 @@ def parse_metric_scores(scores_file: Path) -> Optional[Dict[str, float]]:
         k_strings = [k.strip() for k in lines[0].strip().split(',')]
         k_values = []
         for k_str in k_strings:
-            match = re.match(r'k=(\d+)', k_str)
+            match = re.match(r'.*k=(\d+)*', k_str)
             if match:
                 k_values.append(int(match.group(1)))
             else:
@@ -185,7 +228,7 @@ def parse_metrics(param_dir: Path) -> Dict[str, Dict[str, Dict[str, float]]]:
     return metrics
 
 
-def find_results(base_dir: str = '.', pattern: str = 'out-*/data/clustbench/dataset_generator-*/clustering/*') -> List[Dict[str, str]]:
+def find_results(base_dir: str = '.', pattern: str = 'out_*/data/clustbench/dataset_generator-*/clustering/*') -> List[Dict[str, str]]:
     """
     Find all result directories matching the pattern.