evaluate.py

"""
evaluate.py
-----------
Loads the trained weights, evaluates the network on the held-out test
set, and produces:
    - overall test accuracy, precision/recall/F1 (macro-averaged)
    - a 35x35 confusion matrix (computed from scratch with NumPy, then
      plotted with matplotlib for visualization only)
    - a plot of training/validation loss & accuracy curves
    - a qualitative analysis of >= 5 misclassified test samples
"""

import os
import sys
import numpy as np
import matplotlib.pyplot as plt

sys.path.append(os.path.dirname(__file__))
from data.preprocess import load_and_prepare
from model.neural_network import NeuralNetwork

BASE = os.path.dirname(__file__)
OUT = os.path.join(BASE, "outputs")
ASSETS = os.path.join(BASE, "report_assets")
os.makedirs(ASSETS, exist_ok=True)


def load_trained_network():
    """Rebuild a NeuralNetwork instance and load saved weights into it."""
    w = np.load(os.path.join(OUT, "trained_weights.npz"))
    layer_sizes = list(w["layer_sizes"])
    net = NeuralNetwork(layer_sizes)
    num_layers = len(layer_sizes) - 1
    net.W = [w[f"W{i}"] for i in range(num_layers)]
    net.b = [w[f"b{i}"] for i in range(num_layers)]
    return net


def confusion_matrix(y_true, y_pred, num_classes):
    """
    Compute an num_classes x num_classes confusion matrix from scratch.
    Rows = true class, columns = predicted class.
    """
    cm = np.zeros((num_classes, num_classes), dtype=np.int64)
    for t, p in zip(y_true, y_pred):
        cm[t, p] += 1
    return cm


def precision_recall_f1(cm):
    """
    Compute per-class precision, recall, F1 from a confusion matrix,
    plus macro-averaged totals. All from scratch with NumPy.
    """
    num_classes = cm.shape[0]
    tp = np.diag(cm).astype(np.float64)
    fp = cm.sum(axis=0) - tp
    fn = cm.sum(axis=1) - tp

    precision = np.divide(tp, tp + fp, out=np.zeros_like(tp), where=(tp + fp) != 0)
    recall = np.divide(tp, tp + fn, out=np.zeros_like(tp), where=(tp + fn) != 0)
    f1 = np.divide(2 * precision * recall, precision + recall,
                    out=np.zeros_like(tp), where=(precision + recall) != 0)

    return precision, recall, f1, precision.mean(), recall.mean(), f1.mean()


def plot_confusion_matrix(cm, class_names, path):
    fig, ax = plt.subplots(figsize=(12, 11))
    im = ax.imshow(cm, cmap="Blues")
    ax.set_xticks(range(len(class_names)))
    ax.set_yticks(range(len(class_names)))
    ax.set_xticklabels(class_names, fontsize=7)
    ax.set_yticklabels(class_names, fontsize=7)
    ax.set_xlabel("Predicted label")
    ax.set_ylabel("True label")
    ax.set_title("Confusion Matrix - Test Set")
    plt.colorbar(im, fraction=0.046, pad=0.04)
    plt.tight_layout()
    plt.savefig(path, dpi=130)
    plt.close(fig)


def plot_training_curves(history, path):
    fig, axes = plt.subplots(1, 2, figsize=(12, 4.5))
    axes[0].plot(history["train_loss"], label="Train loss")
    axes[0].plot(history["val_loss"], label="Val loss")
    axes[0].set_xlabel("Epoch"); axes[0].set_ylabel("Cross-entropy loss")
    axes[0].set_title("Loss curves"); axes[0].legend()

    axes[1].plot(history["train_acc"], label="Train acc")
    axes[1].plot(history["val_acc"], label="Val acc")
    axes[1].set_xlabel("Epoch"); axes[1].set_ylabel("Accuracy")
    axes[1].set_title("Accuracy curves"); axes[1].legend()

    plt.tight_layout()
    plt.savefig(path, dpi=130)
    plt.close(fig)


def analyze_errors(X_test, y_test, y_pred, probs, class_names, images_shape, n=8):
    """
    Pick n misclassified test samples (favoring high-confidence
    mistakes, which are the most informative failure cases) and return
    a list of dicts describing each: true label, predicted label,
    model confidence, and the flattened image (for plotting).
    """
    wrong_idx = np.where(y_pred != y_test)[0]
    confidences = probs[wrong_idx, y_pred[wrong_idx]]
    order = wrong_idx[np.argsort(-confidences)]  # most confident mistakes first
    chosen = order[:n]

    results = []
    for idx in chosen:
        results.append({
            "index": int(idx),
            "true": class_names[y_test[idx]],
            "pred": class_names[y_pred[idx]],
            "confidence": float(probs[idx, y_pred[idx]]),
            "image": X_test[idx].reshape(images_shape),
        })
    return results


def plot_error_grid(errors, path):
    n = len(errors)
    cols = min(n, 5)
    rows = int(np.ceil(n / cols))
    fig, axes = plt.subplots(rows, cols, figsize=(3 * cols, 3.2 * rows))
    axes = np.array(axes).reshape(-1)
    for ax, e in zip(axes, errors):
        ax.imshow(e["image"], cmap="gray")
        ax.set_title(f"True: {e['true']}  Pred: {e['pred']}\nconf={e['confidence']:.2f}",
                     fontsize=9)
        ax.axis("off")
    for ax in axes[len(errors):]:
        ax.axis("off")
    plt.tight_layout()
    plt.subplots_adjust(hspace=0.5)
    plt.savefig(path, dpi=130)
    plt.close(fig)


if __name__ == "__main__":
    npz_path = os.path.join(BASE, "data", "char_dataset.npz")
    data = load_and_prepare(npz_path)
    class_names = list(data["class_names"])

    net = load_trained_network()
    history = dict(np.load(os.path.join(OUT, "history.npz")))

    X_test, y_test = data["X_test"], data["y_test"]
    probs = net.predict_proba(X_test)
    y_pred = np.argmax(probs, axis=1)

    test_acc = float(np.mean(y_pred == y_test))
    cm = confusion_matrix(y_test, y_pred, num_classes=len(class_names))
    precision, recall, f1, macro_p, macro_r, macro_f1 = precision_recall_f1(cm)

    print(f"Test accuracy: {test_acc:.4f}")
    print(f"Macro precision: {macro_p:.4f}  Macro recall: {macro_r:.4f}  Macro F1: {macro_f1:.4f}")

    plot_confusion_matrix(cm, class_names, os.path.join(ASSETS, "confusion_matrix.png"))
    plot_training_curves(history, os.path.join(ASSETS, "training_curves.png"))

    errors = analyze_errors(X_test, y_test, y_pred, probs, class_names, (28, 28), n=8)
    plot_error_grid(errors, os.path.join(ASSETS, "error_analysis.png"))

    print("\nMost confident misclassifications:")
    for e in errors:
        print(f"  true={e['true']:>2}  pred={e['pred']:>2}  confidence={e['confidence']:.3f}")

    # Per-class report
    print("\nPer-class precision/recall/F1:")
    for i, c in enumerate(class_names):
        print(f"  {c:>2}: precision={precision[i]:.3f} recall={recall[i]:.3f} f1={f1[i]:.3f}")

    np.savez_compressed(os.path.join(OUT, "eval_results.npz"),
                         cm=cm, precision=precision, recall=recall, f1=f1,
                         test_acc=test_acc, macro_p=macro_p, macro_r=macro_r, macro_f1=macro_f1)