Merge branch 'develop' of https://github.kcl.ac.uk/K23064919/smallGroupProject into develop

best_model.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:23a4c08eaad4b40290eca84e6a8fa3e1d69bdf4312d5db6db5de96d1d8753024
-size 130261986
+oid sha256:d3c19d6a5fea8043e6fda261763b7909aaed487b83991f29ca395b2ce7c8e591
+size 20532322

dataPrep/helpers/clearml_data.py

@@ -11,7 +11,7 @@ Takes latest Data Prep ClearML task from project and reconstruct:
 - data loaders for both full and subset datasets
 - Aug settings used
 '''
-def extract_latest_data_task(project_name: str = "Small Group Project", num_workers: int = 8):
+def extract_latest_data_task(project_name: str = "Small Group Project", num_workers: int = 0):
 
   # --------- Get latest Data Preparation task from ClearML ---------
 

dataPrep/helpers/transforms_loaders.py

@@ -103,13 +103,15 @@ def make_dataset_loaders(dataset, seed, batch_size, test_size, aug_config, worke
         pin_memory=True,
         num_workers=workers
     )
+    class_names = dataset.features['label'].names
 
     print(f"\nWorkers used in DataLoaders: {workers}\n")
 
     dataset_loaders = {
         "train": train_loader,
         "val": val_loader,
-        "test": test_loader
+        "test": test_loader,
+        "classNames": class_names
     }
 
     return dataset_loaders

testingModel/helpers/evaluation.py

@@ -1,43 +1,88 @@
-import torch
-from torch.nn import CrossEntropyLoss
-
-
-"""
-Evaluates a trained model on a dataloader that returns batches like:
-    batch["image"] -> Tensor [B, 3, 256, 256]
-    batch["label"] -> Tensor [B]
-
-Returns dict:
-    { "accuracy": float, "loss": float }
-"""
-def make_predictions(model, dataloader, device):
-
-    model.eval()
-    criterion = CrossEntropyLoss()
-
-    total_loss = 0
-    total_correct = 0
-    total_samples = 0
-
-    with torch.no_grad():
-        for batch in dataloader:
-
-            # Move tensors to device
-            images = batch["image"].to(device)
-            labels = batch["label"].to(device).long()
-
-            # Forward pass
-            outputs = model(images)
-            loss = criterion(outputs, labels)
-
-            total_loss += loss.item() * images.size(0)
-            total_correct += (outputs.argmax(dim=1) == labels).sum().item()
-            total_samples += labels.size(0)
-
-    accuracy = total_correct / total_samples
-    avg_loss = total_loss / total_samples
-
-    return {
-        "accuracy": accuracy,
-        "loss": avg_loss,
-    }
+import torch
+from torch.nn import CrossEntropyLoss
+import numpy as np
+import matplotlib.pyplot as plt
+
+
+"""
+Evaluates a trained model on a dataloader that returns batches like:
+    batch["image"] -> Tensor [B, 3, 256, 256]
+    batch["label"] -> Tensor [B]
+"""
+def make_predictions(model, dataloader, device):
+
+    model.eval()
+    criterion = CrossEntropyLoss()
+
+    total_loss = 0
+    total_correct = 0
+    total_samples = 0
+
+    all_preds = []
+    all_labels = []
+
+    with torch.no_grad():
+        for batch in dataloader:
+
+            # Move tensors to device
+            images = batch["image"].to(device)
+            labels = batch["label"].to(device).long()
+
+            # Forward pass
+            outputs = model(images)
+            loss = criterion(outputs, labels)
+            preds = outputs.argmax(dim=1)
+
+            total_loss += loss.item() * images.size(0)
+            total_correct += (preds == labels).sum().item()
+            total_samples += labels.size(0)
+
+            # Accumulate all predictions and labels
+            all_preds.extend(preds.tolist())
+            all_labels.extend(labels.tolist())
+
+    accuracy = total_correct / total_samples
+    avg_loss = total_loss / total_samples
+
+    return {
+        "accuracy": accuracy,
+        "loss": avg_loss,
+        "predictions": np.array(all_preds),
+        "labels": np.array(all_labels),
+    }
+
+
+# Computes per-class accuracies
+def class_accuracies(labels, preds, num_classes):
+    correct = np.zeros(num_classes, dtype=int)
+    counts = np.zeros(num_classes, dtype=int)
+    accuracies = np.zeros(num_classes, dtype=float)
+
+    for true, pred in zip(labels, preds):
+        counts[true] += 1
+        if true == pred:
+            correct[true] += 1
+
+    # Calculate accuracies
+    for i in range(num_classes):
+        if counts[i] > 0:
+            accuracies[i] = round(correct[i] / counts[i], 4)
+        else:
+            accuracies[i] = 0.0 
+
+    return accuracies
+
+
+def plot_class_accuracies(accuracies, class_names):
+    fig, ax = plt.subplots(figsize=(12, 6))
+    
+    ax.set_title("Per-Class Accuracy")
+    ax.set_xlabel("Class")
+    ax.set_ylabel("Accuracy")
+    ax.set_ylim(0, 1.0)
+    ax.bar(class_names, accuracies)
+
+    plt.xticks(rotation=90)
+    plt.tight_layout()
+
+    return fig

testingModel/run_testing.py

@@ -1,76 +1,98 @@
-from clearml import Task
-from dataPrep.helpers.clearml_data import extract_latest_data_task
-
-import torch
-from models.modelOne import modelOne
-from testingModel.helpers.evaluation import make_predictions
-
-
-# -------------- Load Data --------------
-project_name = "Small Group Project"
-subset_loaders, full_loaders, data_prep_metadata = extract_latest_data_task(project_name=project_name)
-
-
-# -------- ClearML Testing Task Setup --------
-testing_task = Task.init(
-    project_name=f"{project_name}/Model Testing",
-    task_name="Model Testing",
-    task_type=Task.TaskTypes.testing,
-    reuse_last_task_id=False,
-)
-
-# Reference the data prep task used
-testing_logger = testing_task.get_logger()
-testing_task.connect(data_prep_metadata, name="data_prep_metadata_READONLY")
-
-CLEARML_TRAINING_ID = "5bac154a885b4acbaa07d8588027bb27"
-
-# Testing parameters - Modify these when experimenting
-testing_config = {
-    "model_train_id": CLEARML_TRAINING_ID,
-    "num_classes": 39,
-    "model_path": "best_model.pt",
-}
-testing_task.connect(testing_config)
-
-# Load the model weights from ClearML training task
-training_task = Task.get_task(task_id=testing_config["model_train_id"])
-model_artifact = training_task.artifacts.get("best_model")
-model_path = model_artifact.get_local_copy()
-
-# Reference training metadata
-training_hyperparams = training_task.get_parameters_as_dict()
-testing_task.connect(training_hyperparams['General'], name="training_metadata_READONLY")
-
-
-# -------- Rebuild the ML model --------
-model = modelOne()
-state_dict = torch.load(model_path, map_location="cpu") # Load to CPU first
-model.load_state_dict(state_dict)
-model.eval() # set dropout & batch norm layers to eval mode
-
-# Move model to GPU if available
-device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-model.to(device)
-
-
-# -------------------- Test model on test set --------------------
-testing_logger.report_text("Starting evaluation on TEST SUBSET...\n")
-test_subset = subset_loaders['test']
-
-subset_results = make_predictions(model, test_subset, device)
-
-
-# Accuracy & Loss logging
-testing_logger.report_single_value(name="Test Subset Accuracy", value=subset_results["accuracy"])
-testing_logger.report_single_value(name="Test Subset Loss", value=subset_results["loss"])
-
-
-# --------- Complete -----------------
-print("\n------ Testing Complete ------")
-testing_logger.report_text(
-    f"TEST SUBSET RESULTS:\n"
-    f"Loss: {subset_results['loss']:.4f}\n"
-    f"Accuracy: {subset_results['accuracy']:.4f}\n"
-)
-testing_task.close()
+from clearml import Task
+from dataPrep.helpers.clearml_data import extract_latest_data_task
+
+import torch
+from models.modelOne import modelOne
+from models.modelTwo import BetterCNN
+from testingModel.helpers.evaluation import make_predictions, class_accuracies, plot_class_accuracies
+
+
+# -------------- Load Data --------------
+project_name = "Small Group Project"
+subset_loaders, full_loaders, data_prep_metadata = extract_latest_data_task(project_name=project_name)
+
+
+# -------- ClearML Testing Task Setup --------
+testing_task = Task.init(
+    project_name=f"{project_name}/Model Testing",
+    task_name="Model Testing",
+    task_type=Task.TaskTypes.testing,
+    reuse_last_task_id=False,
+)
+
+# Reference the data prep task used
+testing_logger = testing_task.get_logger()
+testing_task.connect(data_prep_metadata, name="data_prep_metadata_READONLY")
+
+CLEARML_TRAINING_ID = "dca82d7c2f404c249f2e5325aaf77207"
+
+# Testing parameters - Modify these when experimenting
+testing_config = {
+    "model_train_id": CLEARML_TRAINING_ID,
+    "num_classes": 39,
+    "model_path": "best_model.pt",
+}
+testing_task.connect(testing_config)
+
+# Load the model weights from ClearML training task
+training_task = Task.get_task(task_id=testing_config["model_train_id"])
+model_artifact = training_task.artifacts.get("best_model")
+model_path = model_artifact.get_local_copy()
+
+# Reference training metadata
+training_hyperparams = training_task.get_parameters_as_dict()
+testing_task.connect(training_hyperparams['General'], name="training_metadata_READONLY")
+
+
+# -------- Rebuild the ML model --------
+model = BetterCNN(noOfClasses=testing_config["num_classes"])
+state_dict = torch.load(model_path, map_location="cpu") # Load to CPU first
+model.load_state_dict(state_dict)
+model.eval() # set dropout & batch norm layers to eval mode
+
+# Move model to GPU if available
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+model.to(device)
+
+
+# -------------------- Test model on test set --------------------
+testing_logger.report_text("Starting evaluation on TEST SUBSET...\n")
+test_subset = subset_loaders['test']
+
+subset_results = make_predictions(model, test_subset, device)
+
+
+# Accuracy & Loss logging
+testing_logger.report_single_value(name="Test Subset Accuracy", value=subset_results["accuracy"])
+testing_logger.report_single_value(name="Test Subset Loss", value=subset_results["loss"])
+
+# Compute per-class accuracy
+preds = subset_results["predictions"]
+labels = subset_results["labels"]
+class_acc = class_accuracies(
+    labels,
+    preds,
+    num_classes=testing_config["num_classes"]
+)
+
+# Plot with formatted class names
+class_names = subset_loaders['classNames']
+formatted_class_names = [" ".join(name.replace('_', ' ').split()) for name in class_names]
+acc_fig = plot_class_accuracies(class_acc, formatted_class_names)
+
+# Log accuracies plot to ClearML
+testing_logger.report_matplotlib_figure(
+    title="Subset Per-Class Accuracy",
+    series="Class Accuracy",
+    figure=acc_fig
+)
+
+
+# --------- Complete -----------------
+print("\n------ Testing Complete ------")
+testing_logger.report_text(
+    f"TEST SUBSET RESULTS:\n"
+    f"Loss: {subset_results['loss']:.4f}\n"
+    f"Accuracy: {subset_results['accuracy']:.4f}\n"
+)
+testing_task.close()

trainingModel/run_training.py

@@ -1,8 +1,8 @@
-import os
 from clearml import Task
 from dataPrep.helpers.clearml_data import extract_latest_data_task
 
 import torch
+from models.modelOne import modelOne
 from models.modelTwo import BetterCNN
 from trainingModel.helpers.Training import train_model