{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "11a71a20",
   "metadata": {},
   "source": [
    "# LogMiniLM — BGL Detection, XAI, SHAP, Charts, and Learning Curves\n",
    "\n",
    "Full reproducible notebook for the final thesis experiment."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f3c44739",
   "metadata": {},
   "source": [
    "## Step 0 — Install and import libraries\n",
    "\n",
    "This step installs and imports the required libraries.\n",
    "\n",
    "### Input\n",
    "No dataset is processed yet.\n",
    "\n",
    "### Output\n",
    "Python libraries become available for parsing logs, generating MiniLM embeddings, training the Transformer model, and evaluating binary anomaly detection.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3be1bd91",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip -q install sentence-transformers\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0f5a72c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "# =========================\n",
    "# Reproducibility environment variables\n",
    "# =========================\n",
    "# These must be set before CUDA operations. In Colab, run cells from the top after a runtime restart.\n",
    "os.environ[\"PYTHONHASHSEED\"] = \"42\"\n",
    "os.environ[\"CUBLAS_WORKSPACE_CONFIG\"] = \":4096:8\"\n",
    "\n",
    "import re\n",
    "import gc\n",
    "import random\n",
    "from pathlib import Path\n",
    "from collections import Counter\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "from sentence_transformers import SentenceTransformer\n",
    "from sklearn.metrics import (\n",
    "    accuracy_score,\n",
    "    precision_score,\n",
    "    recall_score,\n",
    "    f1_score,\n",
    "    confusion_matrix,\n",
    "    classification_report,\n",
    ")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "80d8eb30",
   "metadata": {},
   "source": [
    "## Step 1 — Configuration\n",
    "\n",
    "All important experiment parameters are collected in one place.\n",
    "\n",
    "### Important code behavior\n",
    "\n",
    "- `WINDOW_SIZE = 10`: every 10 consecutive logs become one sequence.\n",
    "- `STEP_SIZE = 10`: the next sequence starts after 10 logs, so there is no overlap.\n",
    "- `TRAIN_RATIO = 0.80`: the first 80% of sequences are used for training and the final 20% for testing.\n",
    "- `PROJECTION_DIM = 128`: MiniLM embeddings are reduced from 384 dimensions to 128 before the Transformer.\n",
    "\n",
    "### Example\n",
    "\n",
    "If the first 30 logs are available:\n",
    "\n",
    "```text\n",
    "Sequence 1 = logs 0–9\n",
    "Sequence 2 = logs 10–19\n",
    "Sequence 3 = logs 20–29\n",
    "```\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "677f407f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# =========================\n",
    "# Reproducibility\n",
    "# =========================\n",
    "SEED = 42\n",
    "\n",
    "def reset_all_seeds(seed=SEED):\n",
    "    \"\"\"Reset Python, NumPy, and PyTorch seeds.\"\"\"\n",
    "    random.seed(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    if torch.cuda.is_available():\n",
    "        torch.cuda.manual_seed(seed)\n",
    "        torch.cuda.manual_seed_all(seed)\n",
    "\n",
    "reset_all_seeds(SEED)\n",
    "\n",
    "torch.backends.cudnn.deterministic = True\n",
    "torch.backends.cudnn.benchmark = False\n",
    "\n",
    "# Disable TF32 to reduce small GPU numerical differences.\n",
    "if torch.cuda.is_available():\n",
    "    torch.backends.cuda.matmul.allow_tf32 = False\n",
    "    torch.backends.cudnn.allow_tf32 = False\n",
    "\n",
    "# warn_only=True avoids crashing if a Colab/PyTorch operation has no fully deterministic implementation.\n",
    "torch.use_deterministic_algorithms(True, warn_only=True)\n",
    "\n",
    "# =========================\n",
    "# Dataset path\n",
    "# =========================\n",
    "# Update this path if your BGL.log file is inside a subfolder.\n",
    "BGL_PATH = \"/content/drive/MyDrive/BGL.log\"\n",
    "\n",
    "# Set to None for the full BGL dataset.\n",
    "# For quick testing only, use a number such as 100000.\n",
    "MAX_LINES = None\n",
    "\n",
    "# =========================\n",
    "# Sequence construction\n",
    "# =========================\n",
    "WINDOW_SIZE = 10\n",
    "STEP_SIZE = 10\n",
    "TRAIN_RATIO = 0.80\n",
    "VALIDATION_RATIO_WITHIN_TRAIN = 0.10  # used only for learning curves / overfitting check\n",
    "\n",
    "# =========================\n",
    "# MiniLM semantic encoder\n",
    "# =========================\n",
    "MINILM_MODEL_NAME = \"sentence-transformers/all-MiniLM-L6-v2\"\n",
    "ENCODE_BATCH_SIZE = 256\n",
    "\n",
    "# =========================\n",
    "# LogMiniLM model settings\n",
    "# =========================\n",
    "PROJECTION_DIM = 128\n",
    "NUM_TRANSFORMER_LAYERS = 1\n",
    "NUM_ATTENTION_HEADS = 2\n",
    "FEEDFORWARD_DIM = 256\n",
    "DROPOUT = 0.1\n",
    "\n",
    "# =========================\n",
    "# Training settings\n",
    "# =========================\n",
    "BATCH_SIZE = 256\n",
    "EPOCHS = 10\n",
    "LEARNING_RATE = 1e-3\n",
    "ANOMALY_WEIGHT_MULTIPLIER = 2.0\n",
    "\n",
    "# Keep AMP disabled for the final thesis run because mixed precision can create small GPU-to-GPU differences.\n",
    "USE_AMP = False\n",
    "\n",
    "# =========================\n",
    "# Cache/output paths\n",
    "# =========================\n",
    "# Correct lowercase spelling: logminilm = LogMiniLM.\n",
    "CACHE_DIR = Path(\"/content/drive/MyDrive/logminilm_cache\")\n",
    "CACHE_DIR.mkdir(parents=True, exist_ok=True)\n",
    "\n",
    "UNIQUE_TEXTS_PATH = CACHE_DIR / \"bgl_unique_normalized_messages.csv\"\n",
    "EMBEDDINGS_PATH = CACHE_DIR / \"bgl_minilm_unique_embeddings.npy\"\n",
    "EVENT_ID_PATH = CACHE_DIR / \"bgl_line_event_ids.npy\"\n",
    "LABELS_PATH = CACHE_DIR / \"bgl_line_labels.npy\"\n",
    "METADATA_PATH = CACHE_DIR / \"bgl_metadata_for_alerts_light.parquet\"\n",
    "\n",
    "RESULTS_PATH = CACHE_DIR / \"logminilm_results.csv\"\n",
    "PREDICTIONS_PATH = CACHE_DIR / \"logminilm_predictions_for_xai.csv\"\n",
    "TOP_XAI_CANDIDATES_PATH = CACHE_DIR / \"logminilm_top_xai_candidates.csv\"\n",
    "MODEL_PATH = CACHE_DIR / \"logminilm_model.pt\"\n",
    "\n",
    "# =========================\n",
    "# Device\n",
    "# =========================\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(\"Device:\", device)\n",
    "if torch.cuda.is_available():\n",
    "    print(\"GPU:\", torch.cuda.get_device_name(0))\n",
    "print(\"Seed:\", SEED)\n",
    "print(\"AMP enabled:\", USE_AMP)\n",
    "print(\"Cache directory:\", CACHE_DIR)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "965b191e",
   "metadata": {},
   "source": [
    "### Reproducibility note\n",
    "\n",
    "For the final thesis run, use **Runtime → Restart runtime**, then run the notebook from the first cell to the last cell. This ensures the deterministic environment variables are applied before CUDA operations.\n",
    "\n",
    "This version also uses the corrected output folder name:\n",
    "\n",
    "```text\n",
    "/content/drive/MyDrive/logminilm_cache\n",
    "```\n",
    "\n",
    "If you previously used `/content/drive/MyDrive/logminiml_cache`, this notebook will create a new corrected cache folder and may regenerate embeddings the first time.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "39fa64e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Safety check: this notebook is fixed to 10-line non-overlapping windows.\n",
    "assert WINDOW_SIZE == 10, f\"Expected WINDOW_SIZE=10, got {WINDOW_SIZE}\"\n",
    "assert STEP_SIZE == 10, f\"Expected STEP_SIZE=10, got {STEP_SIZE}\"\n",
    "print(f\"Confirmed sequence setting: window_size={WINDOW_SIZE}, step_size={STEP_SIZE}\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4feb2365",
   "metadata": {},
   "source": [
    "## Step 2 — Mount Google Drive and verify the dataset\n",
    "\n",
    "The notebook expects `BGL.log` in Google Drive. This cell safely mounts Drive only if needed.\n",
    "\n",
    "### Input\n",
    "Expected dataset path:\n",
    "\n",
    "```text\n",
    "/content/drive/MyDrive/BGL.log\n",
    "```\n",
    "\n",
    "### Output\n",
    "A confirmation that the file exists.\n",
    "\n",
    "### Common issue\n",
    "If the file is in a folder, update `BGL_PATH`, for example:\n",
    "\n",
    "```python\n",
    "BGL_PATH = \"/content/drive/MyDrive/bgl_data/BGL.log\"\n",
    "```\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d3e6243e",
   "metadata": {},
   "outputs": [],
   "source": [
    "from google.colab import drive\n",
    "\n",
    "# Safe Drive mount: avoids failing when Drive is already mounted.\n",
    "if not os.path.exists(\"/content/drive/MyDrive\"):\n",
    "    drive.mount(\"/content/drive\")\n",
    "else:\n",
    "    print(\"Google Drive is already mounted.\")\n",
    "\n",
    "print(\"BGL file exists:\", os.path.exists(BGL_PATH))\n",
    "if not os.path.exists(BGL_PATH):\n",
    "    raise FileNotFoundError(\n",
    "        f\"BGL file not found at: {BGL_PATH}\\n\"\n",
    "        \"Update BGL_PATH to the real location of BGL.log in your Drive.\"\n",
    "    )\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4d598c60",
   "metadata": {},
   "source": [
    "## Step 3 — Parse BGL raw logs\n",
    "\n",
    "Each BGL log line contains a label and several metadata fields.\n",
    "\n",
    "### BGL label rule\n",
    "\n",
    "```text\n",
    "- = Normal\n",
    "Any other label = Anomaly\n",
    "```\n",
    "\n",
    "### Example raw line\n",
    "\n",
    "```text\n",
    "- 1117838570 2005.06.03 R02-M1-N0-C:J12-U11 2005-06-03-15.42.50.675872 R02-M1-N0-C:J12-U11 RAS KERNEL INFO instruction cache parity error corrected\n",
    "```\n",
    "\n",
    "### Parsed output example\n",
    "\n",
    "| Field | Example |\n",
    "|---|---|\n",
    "| label | 0 |\n",
    "| source | RAS |\n",
    "| component | KERNEL |\n",
    "| severity | INFO |\n",
    "| event_message | instruction cache parity error corrected |\n",
    "\n",
    "The model will not use timestamp or node directly as input, but these fields are preserved for alert reporting and XAI.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "682151e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_bgl_line(line, line_index):\n",
    "    '''Parse one BGL raw log line into structured fields.\n",
    "\n",
    "    Expected BGL layout:\n",
    "    label unix_timestamp date node full_time repeated_node source component severity message...\n",
    "    '''\n",
    "    parts = line.strip().split()\n",
    "    if len(parts) < 10:\n",
    "        return None\n",
    "\n",
    "    raw_label = parts[0]\n",
    "    label = 0 if raw_label == \"-\" else 1\n",
    "\n",
    "    return {\n",
    "        \"line_index\": line_index,\n",
    "        \"raw_label\": raw_label,\n",
    "        \"label\": label,\n",
    "        \"unix_timestamp\": parts[1],\n",
    "        \"date\": parts[2],\n",
    "        \"node\": parts[3],\n",
    "        \"full_time\": parts[4],\n",
    "        \"repeated_node\": parts[5],\n",
    "        \"source\": parts[6],\n",
    "        \"component\": parts[7],\n",
    "        \"severity\": parts[8],\n",
    "        \"event_message\": \" \".join(parts[9:]),\n",
    "    }\n",
    "\n",
    "records = []\n",
    "with open(BGL_PATH, \"r\", errors=\"ignore\") as f:\n",
    "    for i, line in enumerate(f):\n",
    "        if MAX_LINES is not None and i >= MAX_LINES:\n",
    "            break\n",
    "        parsed = parse_bgl_line(line, i)\n",
    "        if parsed is not None:\n",
    "            records.append(parsed)\n",
    "\n",
    "df = pd.DataFrame(records)\n",
    "\n",
    "print(\"Parsed rows:\", len(df))\n",
    "print(\"Line label distribution:\", Counter(df[\"label\"]))\n",
    "display(df.head(3))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "38aaa3d8",
   "metadata": {},
   "source": [
    "## Step 4 — Build metadata-reduced model input and normalize it\n",
    "\n",
    "The model input is built from event-related fields only:\n",
    "\n",
    "```text\n",
    "source + component + severity + event_message\n",
    "```\n",
    "\n",
    "This excludes direct shortcut fields such as `date`, `full_time`, and `node`.\n",
    "\n",
    "### Normalization examples\n",
    "\n",
    "| Before | After |\n",
    "|---|---|\n",
    "| `RAS KERNEL INFO CE sym 2, at 0x0b85eee0, mask 0x05` | `ras kernel info ce sym <num> at <hex> mask <hex>` |\n",
    "| `socket to 172.16.96.116:33569` | `socket to <ip> <num>` |\n",
    "\n",
    "### Why normalize?\n",
    "\n",
    "Normalization reduces memorization of unique values such as addresses, IDs, timestamps, and random numbers.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "64537460",
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalize_log_text(text):\n",
    "    '''Normalize a log text string while preserving the semantic meaning.\n",
    "\n",
    "    The goal is to replace variable values with general placeholders so the\n",
    "    model focuses on event meaning rather than memorizing IDs or numbers.\n",
    "    '''\n",
    "    text = str(text).lower()\n",
    "\n",
    "    # Replace IP addresses before replacing generic numbers.\n",
    "    text = re.sub(r\"\\b\\d{1,3}(?:\\.\\d{1,3}){3}\\b\", \" <ip> \", text)\n",
    "\n",
    "    # Replace dates/times if they appear inside the message text.\n",
    "    text = re.sub(r\"\\b\\d{4}[.-]\\d{2}[.-]\\d{2}\\b\", \" <date> \", text)\n",
    "    text = re.sub(r\"\\b\\d{2}:\\d{2}:\\d{2}(?:\\.\\d+)?\\b\", \" <time> \", text)\n",
    "\n",
    "    # Replace hexadecimal values such as 0x00004ed8.\n",
    "    text = re.sub(r\"0x[0-9a-fA-F]+\", \" <hex> \", text)\n",
    "\n",
    "    # Replace BGL-like node IDs if they appear inside message text.\n",
    "    text = re.sub(r\"r\\d{2}-m\\d-n[a-f0-9]-[ci]:j\\d{2}-u\\d{2}\", \" <node> \", text)\n",
    "\n",
    "    # Replace standalone numbers.\n",
    "    text = re.sub(r\"\\b\\d+\\b\", \" <num> \", text)\n",
    "\n",
    "    # Clean punctuation spacing while keeping placeholder tokens such as <hex>.\n",
    "    text = re.sub(r\"[^a-zA-Z0-9_<>]+\", \" \", text)\n",
    "    text = re.sub(r\"\\s+\", \" \", text).strip()\n",
    "    return text\n",
    "\n",
    "# Build model input text without timestamp/node shortcut fields.\n",
    "df[\"model_text\"] = (\n",
    "    df[\"source\"].astype(str) + \" \" +\n",
    "    df[\"component\"].astype(str) + \" \" +\n",
    "    df[\"severity\"].astype(str) + \" \" +\n",
    "    df[\"event_message\"].astype(str)\n",
    ")\n",
    "\n",
    "df[\"model_text_norm\"] = df[\"model_text\"].apply(normalize_log_text)\n",
    "\n",
    "print(\"Example from the dataset\")\n",
    "print(\"Before:\", df.loc[0, \"model_text\"])\n",
    "print(\"After :\", df.loc[0, \"model_text_norm\"])\n",
    "\n",
    "print(\"\\nSynthetic normalization example\")\n",
    "sample_text = \"RAS KERNEL INFO CE sym 2, at 0x0b85eee0, mask 0x05 socket to 172.16.96.116:33569\"\n",
    "print(\"Before:\", sample_text)\n",
    "print(\"After :\", normalize_log_text(sample_text))\n",
    "\n",
    "display(df[[\"label\", \"source\", \"component\", \"severity\", \"event_message\", \"model_text_norm\"]].head(5))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2df7fb78",
   "metadata": {},
   "source": [
    "## Step 5 — Encode only unique normalized messages\n",
    "\n",
    "This is the main efficiency improvement in LogMiniLM.\n",
    "\n",
    "Instead of encoding every log line with MiniLM, the notebook encodes only the unique normalized messages.\n",
    "\n",
    "### Example\n",
    "\n",
    "If the same normalized message appears 10,000 times:\n",
    "\n",
    "```text\n",
    "ras kernel info instruction cache parity error corrected\n",
    "```\n",
    "\n",
    "MiniLM encodes it once, and all repeated lines reuse the same `event_id`.\n",
    "\n",
    "### Output\n",
    "\n",
    "- `unique_texts`: list of unique normalized messages.\n",
    "- `line_event_ids`: for each log line, an integer pointing to its unique message.\n",
    "- `line_labels`: normal/anomaly label for each line.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "70c1bb8e",
   "metadata": {},
   "outputs": [],
   "source": [
    "unique_texts = pd.Series(df[\"model_text_norm\"].unique()).sort_values().reset_index(drop=True)\n",
    "text_to_event_id = {text: idx for idx, text in enumerate(unique_texts)}\n",
    "\n",
    "line_event_ids = df[\"model_text_norm\"].map(text_to_event_id).astype(np.int32).values\n",
    "line_labels = df[\"label\"].astype(np.int8).values\n",
    "\n",
    "print(\"Total log lines:\", len(df))\n",
    "print(\"Unique normalized messages:\", len(unique_texts))\n",
    "print(\"Reduction ratio:\", round(len(df) / max(len(unique_texts), 1), 2), \"log lines per unique message\")\n",
    "\n",
    "example_idx = 0\n",
    "print(\"\\nExample mapping\")\n",
    "print(\"Normalized text:\", df.loc[example_idx, \"model_text_norm\"])\n",
    "print(\"Event ID:\", int(line_event_ids[example_idx]))\n",
    "\n",
    "# Save reusable mapping outputs.\n",
    "unique_texts.to_frame(name=\"model_text_norm\").to_csv(UNIQUE_TEXTS_PATH, index=False)\n",
    "np.save(EVENT_ID_PATH, line_event_ids)\n",
    "np.save(LABELS_PATH, line_labels)\n",
    "\n",
    "# Save lightweight metadata for later XAI and alert reporting.\n",
    "# raw_log is intentionally excluded to reduce memory/storage usage.\n",
    "metadata_cols = [\n",
    "    \"line_index\", \"raw_label\", \"label\", \"date\", \"full_time\", \"node\",\n",
    "    \"source\", \"component\", \"severity\", \"event_message\", \"model_text_norm\"\n",
    "]\n",
    "df[metadata_cols].to_parquet(METADATA_PATH, index=False)\n",
    "\n",
    "print(\"\\nSaved unique texts:\", UNIQUE_TEXTS_PATH)\n",
    "print(\"Saved line event IDs:\", EVENT_ID_PATH)\n",
    "print(\"Saved lightweight metadata:\", METADATA_PATH)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cd2f9bdb",
   "metadata": {},
   "source": [
    "## Step 6 — Generate or load cached MiniLM embeddings\n",
    "\n",
    "MiniLM converts each unique normalized message into a semantic vector.\n",
    "\n",
    "### Input\n",
    "\n",
    "```text\n",
    "unique_texts = [message_1, message_2, ..., message_N]\n",
    "```\n",
    "\n",
    "### Output\n",
    "\n",
    "```text\n",
    "unique_embeddings shape = [number_of_unique_messages, 384]\n",
    "```\n",
    "\n",
    "### Example\n",
    "\n",
    "If `event_id = 25`, then line 25 does not store text directly during training. Instead, it points to:\n",
    "\n",
    "```python\n",
    "unique_embeddings[25]\n",
    "```\n",
    "\n",
    "which is a MiniLM semantic vector.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d370c48c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_or_create_minilm_embeddings(unique_texts, embeddings_path):\n",
    "    '''Load cached MiniLM embeddings if valid; otherwise encode and save them.'''\n",
    "    if embeddings_path.exists():\n",
    "        cached = np.load(embeddings_path).astype(np.float32)\n",
    "        if cached.shape[0] == len(unique_texts):\n",
    "            print(\"Loading valid cached embeddings from:\", embeddings_path)\n",
    "            return cached\n",
    "        else:\n",
    "            print(\"Cached embeddings exist but do not match current unique_texts.\")\n",
    "            print(\"Cached rows:\", cached.shape[0], \"Current unique messages:\", len(unique_texts))\n",
    "            print(\"Recomputing embeddings...\")\n",
    "\n",
    "    print(\"Encoding unique normalized messages with MiniLM...\")\n",
    "    encoder = SentenceTransformer(MINILM_MODEL_NAME, device=str(device))\n",
    "    embeddings = encoder.encode(\n",
    "        unique_texts.tolist(),\n",
    "        batch_size=ENCODE_BATCH_SIZE,\n",
    "        show_progress_bar=True,\n",
    "        convert_to_numpy=True,\n",
    "        normalize_embeddings=False,\n",
    "    ).astype(np.float32)\n",
    "\n",
    "    np.save(embeddings_path, embeddings)\n",
    "    print(\"Saved embeddings to:\", embeddings_path)\n",
    "\n",
    "    # Free the encoder model after embedding generation to reduce GPU/RAM use.\n",
    "    del encoder\n",
    "    gc.collect()\n",
    "    if torch.cuda.is_available():\n",
    "        torch.cuda.empty_cache()\n",
    "\n",
    "    return embeddings\n",
    "\n",
    "unique_embeddings = load_or_create_minilm_embeddings(unique_texts, EMBEDDINGS_PATH)\n",
    "\n",
    "print(\"Unique embeddings shape:\", unique_embeddings.shape)\n",
    "print(\"Embedding dtype:\", unique_embeddings.dtype)\n",
    "print(\"First embedding first 5 values:\", unique_embeddings[0, :5])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "41034541",
   "metadata": {},
   "source": [
    "## Step 7 — Build fixed 10-line sequences\n",
    "\n",
    "Each sequence contains 10 consecutive log events.\n",
    "\n",
    "### Sequence label rule\n",
    "\n",
    "```text\n",
    "If any log line inside the sequence is anomalous → sequence label = 1\n",
    "Otherwise → sequence label = 0\n",
    "```\n",
    "\n",
    "### Example\n",
    "\n",
    "```text\n",
    "Line labels: [0, 0, 0, 1, 0, 0, 0, 0, 0, 0]\n",
    "Sequence label = 1\n",
    "```\n",
    "\n",
    "because at least one event is anomalous.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "92d69a49",
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_fixed_line_sequences(event_ids, labels, window_size=10, step_size=10):\n",
    "    '''Build non-overlapping fixed-line windows from event IDs and labels.'''\n",
    "    X, y, ranges = [], [], []\n",
    "\n",
    "    for start in range(0, len(event_ids) - window_size + 1, step_size):\n",
    "        end = start + window_size\n",
    "\n",
    "        # Model input: event IDs for the 10 log lines.\n",
    "        seq_ids = event_ids[start:end]\n",
    "\n",
    "        # Sequence label: anomalous if any log inside the window is anomalous.\n",
    "        seq_labels = labels[start:end]\n",
    "        seq_label = 1 if seq_labels.max() == 1 else 0\n",
    "\n",
    "        X.append(seq_ids)\n",
    "        y.append(seq_label)\n",
    "        ranges.append((start, end - 1))\n",
    "\n",
    "    return np.array(X, dtype=np.int32), np.array(y, dtype=np.int8), np.array(ranges, dtype=np.int32)\n",
    "\n",
    "X_ids, y_seq, seq_ranges = build_fixed_line_sequences(\n",
    "    line_event_ids,\n",
    "    line_labels,\n",
    "    window_size=WINDOW_SIZE,\n",
    "    step_size=STEP_SIZE,\n",
    ")\n",
    "\n",
    "if len(X_ids) == 0:\n",
    "    raise ValueError(\"No sequences were created. Check dataset size, WINDOW_SIZE, and STEP_SIZE.\")\n",
    "\n",
    "print(\"X_ids shape:\", X_ids.shape)\n",
    "print(\"y_seq shape:\", y_seq.shape)\n",
    "print(\"Sequence label distribution:\", Counter(y_seq))\n",
    "\n",
    "print(\"\\nExample sequence\")\n",
    "print(\"Event IDs:\", X_ids[0])\n",
    "print(\"Sequence label:\", int(y_seq[0]))\n",
    "print(\"Line range:\", seq_ranges[0])\n",
    "\n",
    "display(df.iloc[seq_ranges[0][0]:seq_ranges[0][1] + 1][[\n",
    "    \"line_index\", \"label\", \"full_time\", \"node\", \"severity\", \"event_message\", \"model_text_norm\"\n",
    "]])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "87c40b8a",
   "metadata": {},
   "source": [
    "## Step 8 — Temporal 80/20 split\n",
    "\n",
    "The split is chronological:\n",
    "\n",
    "```text\n",
    "Train = first 80% of sequences\n",
    "Test  = final 20% of sequences\n",
    "```\n",
    "\n",
    "This is more realistic than random splitting because production systems train on historical logs and detect future logs.\n",
    "\n",
    "### Output\n",
    "\n",
    "The cell prints the normal/anomaly distribution in train and test sets. The test set must contain both classes for valid metrics.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4d11794e",
   "metadata": {},
   "outputs": [],
   "source": [
    "split_idx = int(len(X_ids) * TRAIN_RATIO)\n",
    "\n",
    "X_train, X_test = X_ids[:split_idx], X_ids[split_idx:]\n",
    "y_train, y_test = y_seq[:split_idx], y_seq[split_idx:]\n",
    "ranges_train, ranges_test = seq_ranges[:split_idx], seq_ranges[split_idx:]\n",
    "\n",
    "print(\"Train shape:\", X_train.shape, Counter(y_train))\n",
    "print(\"Test shape:\", X_test.shape, Counter(y_test))\n",
    "\n",
    "# Safety check: binary metrics are meaningful only if test has normal and anomaly samples.\n",
    "test_counts = Counter(y_test)\n",
    "if test_counts[0] == 0 or test_counts[1] == 0:\n",
    "    raise ValueError(\n",
    "        f\"Invalid temporal test distribution: {test_counts}. \"\n",
    "        \"The test set must contain both normal and anomaly sequences.\"\n",
    "    )\n",
    "\n",
    "print(\"\\nTemporal split example\")\n",
    "print(\"Last training sequence line range:\", ranges_train[-1])\n",
    "print(\"First testing sequence line range:\", ranges_test[0])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "62e6dbee",
   "metadata": {},
   "source": [
    "## Step 9 — Create training, validation, and test dataloaders\n",
    "\n",
    "This step creates three dataloaders:\n",
    "\n",
    "- **Training loader**: used to update model weights.\n",
    "- **Validation loader**: used only to monitor learning curves and check overfitting/underfitting.\n",
    "- **Test loader**: untouched temporal holdout used for final reported metrics.\n",
    "\n",
    "The validation set is taken from the end of the temporal training partition. The test set remains unchanged."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "851cac7d",
   "metadata": {},
   "outputs": [],
   "source": [
    "class EventIdSequenceDataset(Dataset):\n",
    "    \"\"\"Dataset returning event-ID sequences and binary labels.\"\"\"\n",
    "\n",
    "    def __init__(self, X_ids, y):\n",
    "        self.X_ids = torch.tensor(X_ids, dtype=torch.long)\n",
    "        self.y = torch.tensor(y, dtype=torch.float32)\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.y)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        return self.X_ids[idx], self.y[idx], idx\n",
    "\n",
    "# ------------------------------------------------------------\n",
    "# Validation split for learning curves\n",
    "# ------------------------------------------------------------\n",
    "# The original temporal test set is not touched.\n",
    "# The validation set is created from the end of the training partition.\n",
    "val_size = int(len(X_train) * VALIDATION_RATIO_WITHIN_TRAIN)\n",
    "\n",
    "if val_size <= 0:\n",
    "    raise ValueError(\"Validation set is empty. Increase training data or reduce VALIDATION_RATIO_WITHIN_TRAIN.\")\n",
    "\n",
    "X_train_model = X_train[:-val_size]\n",
    "y_train_model = y_train[:-val_size]\n",
    "ranges_train_model = ranges_train[:-val_size]\n",
    "\n",
    "X_val = X_train[-val_size:]\n",
    "y_val = y_train[-val_size:]\n",
    "ranges_val = ranges_train[-val_size:]\n",
    "\n",
    "# Safety check: validation metrics are most useful when both classes exist.\n",
    "val_counts = Counter(y_val)\n",
    "print(\"Training-for-model shape:\", X_train_model.shape, Counter(y_train_model))\n",
    "print(\"Validation shape:\", X_val.shape, val_counts)\n",
    "print(\"Final test shape:\", X_test.shape, Counter(y_test))\n",
    "\n",
    "if val_counts[0] == 0 or val_counts[1] == 0:\n",
    "    print(\"Warning: validation split does not contain both classes. Learning curves may be less informative.\")\n",
    "\n",
    "train_dataset = EventIdSequenceDataset(X_train_model, y_train_model)\n",
    "val_dataset = EventIdSequenceDataset(X_val, y_val)\n",
    "test_dataset = EventIdSequenceDataset(X_test, y_test)\n",
    "\n",
    "# Reset seeds immediately before DataLoader creation.\n",
    "reset_all_seeds(SEED)\n",
    "loader_generator = torch.Generator()\n",
    "loader_generator.manual_seed(SEED)\n",
    "\n",
    "pin_memory = device.type == \"cuda\"\n",
    "train_loader = DataLoader(\n",
    "    train_dataset,\n",
    "    batch_size=BATCH_SIZE,\n",
    "    shuffle=True,\n",
    "    generator=loader_generator,\n",
    "    num_workers=0,          # deterministic in Colab; avoids worker seed differences\n",
    "    pin_memory=pin_memory,\n",
    ")\n",
    "val_loader = DataLoader(\n",
    "    val_dataset,\n",
    "    batch_size=BATCH_SIZE,\n",
    "    shuffle=False,\n",
    "    num_workers=0,\n",
    "    pin_memory=pin_memory,\n",
    ")\n",
    "test_loader = DataLoader(\n",
    "    test_dataset,\n",
    "    batch_size=BATCH_SIZE,\n",
    "    shuffle=False,\n",
    "    num_workers=0,\n",
    "    pin_memory=pin_memory,\n",
    ")\n",
    "\n",
    "print(\"Train batches:\", len(train_loader))\n",
    "print(\"Validation batches:\", len(val_loader))\n",
    "print(\"Test batches:\", len(test_loader))\n",
    "\n",
    "batch_event_ids, batch_labels, batch_indices = next(iter(train_loader))\n",
    "print(\"\\nExample batch shapes\")\n",
    "print(\"event_ids:\", batch_event_ids.shape)\n",
    "print(\"labels:\", batch_labels.shape)\n",
    "print(\"indices:\", batch_indices.shape)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "67a381e3",
   "metadata": {},
   "source": [
    "## Step 10 — Define the LogMiniLM model\n",
    "\n",
    "LogMiniLM uses cached MiniLM embeddings as a frozen semantic embedding layer.\n",
    "\n",
    "### Forward pass\n",
    "\n",
    "```text\n",
    "event IDs\n",
    "  ↓\n",
    "MiniLM embedding lookup [batch, 10, 384]\n",
    "  ↓\n",
    "Projection layer [batch, 10, 128]\n",
    "  ↓\n",
    "Position embedding\n",
    "  ↓\n",
    "Transformer encoder\n",
    "  ↓\n",
    "Mean pooling\n",
    "  ↓\n",
    "Binary classifier\n",
    "  ↓\n",
    "Anomaly logit\n",
    "```\n",
    "\n",
    "### Why freeze MiniLM embeddings?\n",
    "\n",
    "The embeddings are already generated and cached. Freezing them keeps training lightweight and reduces GPU memory use.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "97e5f701",
   "metadata": {},
   "outputs": [],
   "source": [
    "class LogMiniLM(nn.Module):\n",
    "    '''Lightweight semantic Transformer for BGL log anomaly detection.\n",
    "\n",
    "    The model name is LogMiniLM. It uses MiniLM semantic embeddings and a compact\n",
    "    Transformer encoder for sequence-level binary classification.\n",
    "    '''\n",
    "\n",
    "    def __init__(\n",
    "        self,\n",
    "        embedding_matrix,\n",
    "        projection_dim=128,\n",
    "        window_size=10,\n",
    "        num_heads=2,\n",
    "        num_layers=1,\n",
    "        feedforward_dim=256,\n",
    "        dropout=0.1,\n",
    "    ):\n",
    "        super().__init__()\n",
    "\n",
    "        # Frozen semantic lookup table: event_id -> MiniLM embedding.\n",
    "        embedding_tensor = torch.tensor(embedding_matrix, dtype=torch.float32)\n",
    "        self.semantic_embedding = nn.Embedding.from_pretrained(embedding_tensor, freeze=True)\n",
    "\n",
    "        # Reduce MiniLM dimension from 384 to 128 for lighter Transformer compute.\n",
    "        original_dim = embedding_tensor.shape[1]\n",
    "        self.projection = nn.Linear(original_dim, projection_dim)\n",
    "\n",
    "        # Learnable position embeddings so the model knows event order inside the window.\n",
    "        self.position_embedding = nn.Parameter(torch.zeros(1, window_size, projection_dim))\n",
    "\n",
    "        encoder_layer = nn.TransformerEncoderLayer(\n",
    "            d_model=projection_dim,\n",
    "            nhead=num_heads,\n",
    "            dim_feedforward=feedforward_dim,\n",
    "            dropout=dropout,\n",
    "            batch_first=True,\n",
    "            activation=\"gelu\",\n",
    "        )\n",
    "        self.transformer = nn.TransformerEncoder(encoder_layer, num_layers=num_layers)\n",
    "\n",
    "        self.norm = nn.LayerNorm(projection_dim)\n",
    "        self.classifier = nn.Sequential(\n",
    "            nn.Dropout(dropout),\n",
    "            nn.Linear(projection_dim, 1),\n",
    "        )\n",
    "\n",
    "    def forward(self, event_ids):\n",
    "        # event_ids shape: [batch, window_size]\n",
    "        x = self.semantic_embedding(event_ids)       # [batch, window, 384]\n",
    "        x = self.projection(x)                       # [batch, window, 128]\n",
    "        x = x + self.position_embedding[:, :x.size(1), :]\n",
    "        x = self.transformer(x)                      # [batch, window, 128]\n",
    "        x = self.norm(x)\n",
    "        pooled = x.mean(dim=1)                       # [batch, 128]\n",
    "        logits = self.classifier(pooled).squeeze(-1) # [batch]\n",
    "        return logits\n",
    "\n",
    "# Reset seeds immediately before model initialization.\n",
    "reset_all_seeds(SEED)\n",
    "\n",
    "model = LogMiniLM(\n",
    "    embedding_matrix=unique_embeddings,\n",
    "    projection_dim=PROJECTION_DIM,\n",
    "    window_size=WINDOW_SIZE,\n",
    "    num_heads=NUM_ATTENTION_HEADS,\n",
    "    num_layers=NUM_TRANSFORMER_LAYERS,\n",
    "    feedforward_dim=FEEDFORWARD_DIM,\n",
    "    dropout=DROPOUT,\n",
    ").to(device)\n",
    "\n",
    "num_params = sum(p.numel() for p in model.parameters())\n",
    "trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
    "\n",
    "print(model)\n",
    "print(\"Total parameters:\", f\"{num_params:,}\")\n",
    "print(\"Trainable parameters:\", f\"{trainable_params:,}\")\n",
    "\n",
    "# Example forward pass before training.\n",
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    example_logits = model(batch_event_ids[:2].to(device))\n",
    "    example_probs = torch.sigmoid(example_logits).detach().cpu().numpy()\n",
    "print(\"\\nExample model output before training\")\n",
    "print(\"Logits:\", example_logits.detach().cpu().numpy())\n",
    "print(\"Probabilities:\", example_probs)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e2efdae9",
   "metadata": {},
   "source": [
    "## Step 11 — Handle class imbalance\n",
    "\n",
    "BGL is imbalanced: normal sequences are much more common than anomalous sequences.\n",
    "\n",
    "LogMiniLM uses `BCEWithLogitsLoss` with `pos_weight` so anomaly sequences receive higher weight during training.\n",
    "\n",
    "### Example\n",
    "\n",
    "If the training set has:\n",
    "\n",
    "```text\n",
    "normal = 360,000\n",
    "anomaly = 17,000\n",
    "```\n",
    "\n",
    "then the anomaly class receives a higher weight to reduce the chance of predicting everything as normal.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d449ef67",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Class weights are computed from the actual training subset used for weight updates.\n",
    "num_pos = int((y_train_model == 1).sum())\n",
    "num_neg = int((y_train_model == 0).sum())\n",
    "\n",
    "base_pos_weight = num_neg / max(num_pos, 1)\n",
    "pos_weight_value = base_pos_weight * ANOMALY_WEIGHT_MULTIPLIER\n",
    "\n",
    "print(\"Model-training normal sequences:\", num_neg)\n",
    "print(\"Model-training anomaly sequences:\", num_pos)\n",
    "print(\"Validation normal sequences:\", int((y_val == 0).sum()))\n",
    "print(\"Validation anomaly sequences:\", int((y_val == 1).sum()))\n",
    "print(\"Base pos_weight:\", base_pos_weight)\n",
    "print(\"Final pos_weight:\", pos_weight_value)\n",
    "\n",
    "criterion = nn.BCEWithLogitsLoss(\n",
    "    pos_weight=torch.tensor([pos_weight_value], device=device, dtype=torch.float32)\n",
    ")\n",
    "optimizer = torch.optim.AdamW(model.parameters(), lr=LEARNING_RATE, weight_decay=1e-4)\n",
    "scaler = torch.cuda.amp.GradScaler(enabled=(device.type == \"cuda\" and USE_AMP))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4e47dc47",
   "metadata": {},
   "source": [
    "## Step 12 — Train LogMiniLM and track learning curves\n",
    "\n",
    "This step trains the model and records both training and validation behavior across epochs.\n",
    "\n",
    "The curves help check:\n",
    "\n",
    "- **Overfitting**: training loss decreases while validation loss increases.\n",
    "- **Underfitting**: both training and validation losses remain high.\n",
    "- **Good fit**: both curves improve and remain relatively close.\n",
    "\n",
    "The test set is not used during training or validation monitoring."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "22cead4b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_one_epoch(model, loader, optimizer, criterion):\n",
    "    \"\"\"Train the model for one epoch and return average loss.\"\"\"\n",
    "    model.train()\n",
    "    total_loss = 0.0\n",
    "    total_count = 0\n",
    "\n",
    "    for event_ids, labels, _ in loader:\n",
    "        event_ids = event_ids.to(device, non_blocking=True)\n",
    "        labels = labels.to(device, non_blocking=True)\n",
    "\n",
    "        optimizer.zero_grad(set_to_none=True)\n",
    "\n",
    "        with torch.cuda.amp.autocast(enabled=(device.type == \"cuda\" and USE_AMP)):\n",
    "            logits = model(event_ids)\n",
    "            loss = criterion(logits, labels)\n",
    "\n",
    "        scaler.scale(loss).backward()\n",
    "        scaler.step(optimizer)\n",
    "        scaler.update()\n",
    "\n",
    "        total_loss += loss.item() * labels.size(0)\n",
    "        total_count += labels.size(0)\n",
    "\n",
    "    return total_loss / max(total_count, 1)\n",
    "\n",
    "\n",
    "def evaluate_loader_for_learning_curve(model, loader, criterion, threshold=0.50):\n",
    "    \"\"\"Evaluate loss and metrics for a dataloader during training.\"\"\"\n",
    "    model.eval()\n",
    "    total_loss = 0.0\n",
    "    total_count = 0\n",
    "    all_probs = []\n",
    "    all_labels = []\n",
    "\n",
    "    with torch.no_grad():\n",
    "        for event_ids, labels, _ in loader:\n",
    "            event_ids = event_ids.to(device, non_blocking=True)\n",
    "            labels = labels.to(device, non_blocking=True)\n",
    "\n",
    "            logits = model(event_ids)\n",
    "            loss = criterion(logits, labels)\n",
    "            probs = torch.sigmoid(logits)\n",
    "\n",
    "            total_loss += loss.item() * labels.size(0)\n",
    "            total_count += labels.size(0)\n",
    "            all_probs.append(probs.detach().cpu().numpy())\n",
    "            all_labels.append(labels.detach().cpu().numpy())\n",
    "\n",
    "    avg_loss = total_loss / max(total_count, 1)\n",
    "    all_probs = np.concatenate(all_probs)\n",
    "    all_labels = np.concatenate(all_labels).astype(int)\n",
    "    preds = (all_probs >= threshold).astype(int)\n",
    "\n",
    "    return {\n",
    "        \"loss\": float(avg_loss),\n",
    "        \"accuracy\": accuracy_score(all_labels, preds),\n",
    "        \"precision\": precision_score(all_labels, preds, zero_division=0),\n",
    "        \"recall\": recall_score(all_labels, preds, zero_division=0),\n",
    "        \"f1\": f1_score(all_labels, preds, zero_division=0),\n",
    "    }\n",
    "\n",
    "\n",
    "LEARNING_HISTORY_PATH = CACHE_DIR / \"logminilm_learning_history.csv\"\n",
    "\n",
    "history = {\n",
    "    \"epoch\": [],\n",
    "    \"train_loss\": [],\n",
    "    \"val_loss\": [],\n",
    "    \"val_accuracy\": [],\n",
    "    \"val_precision\": [],\n",
    "    \"val_recall\": [],\n",
    "    \"val_f1\": [],\n",
    "}\n",
    "\n",
    "# Reset seeds immediately before training.\n",
    "reset_all_seeds(SEED)\n",
    "\n",
    "for epoch in range(1, EPOCHS + 1):\n",
    "    train_loss = train_one_epoch(model, train_loader, optimizer, criterion)\n",
    "    val_metrics = evaluate_loader_for_learning_curve(model, val_loader, criterion, threshold=0.50)\n",
    "\n",
    "    history[\"epoch\"].append(epoch)\n",
    "    history[\"train_loss\"].append(train_loss)\n",
    "    history[\"val_loss\"].append(val_metrics[\"loss\"])\n",
    "    history[\"val_accuracy\"].append(val_metrics[\"accuracy\"])\n",
    "    history[\"val_precision\"].append(val_metrics[\"precision\"])\n",
    "    history[\"val_recall\"].append(val_metrics[\"recall\"])\n",
    "    history[\"val_f1\"].append(val_metrics[\"f1\"])\n",
    "\n",
    "    print(\n",
    "        f\"Epoch {epoch:02d}/{EPOCHS} | \"\n",
    "        f\"Train loss: {train_loss:.6f} | \"\n",
    "        f\"Val loss: {val_metrics['loss']:.6f} | \"\n",
    "        f\"Val F1: {val_metrics['f1']:.6f} | \"\n",
    "        f\"Val Acc: {val_metrics['accuracy']:.6f}\"\n",
    "    )\n",
    "\n",
    "history_df = pd.DataFrame(history)\n",
    "history_df.to_csv(LEARNING_HISTORY_PATH, index=False)\n",
    "\n",
    "print(\"\\nSaved learning history to:\", LEARNING_HISTORY_PATH)\n",
    "display(history_df)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2e1680f2",
   "metadata": {},
   "source": [
    "## Step 13 — Evaluate with threshold sweep\n",
    "\n",
    "The model outputs anomaly probabilities.\n",
    "\n",
    "A threshold converts probability into a class label:\n",
    "\n",
    "```text\n",
    "probability >= threshold → anomaly\n",
    "probability < threshold  → normal\n",
    "```\n",
    "\n",
    "Because anomaly detection is imbalanced, the notebook tests several thresholds and selects the best F1-score.\n",
    "\n",
    "### Output\n",
    "A table containing accuracy, precision, recall, F1-score, and confusion matrix values for each threshold.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bb71bcdf",
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_probabilities(model, loader):\n",
    "    '''Return probabilities, true labels, and local test indices.'''\n",
    "    model.eval()\n",
    "    all_probs = []\n",
    "    all_labels = []\n",
    "    all_indices = []\n",
    "\n",
    "    with torch.no_grad():\n",
    "        for event_ids, labels, batch_indices in loader:\n",
    "            event_ids = event_ids.to(device, non_blocking=True)\n",
    "            logits = model(event_ids)\n",
    "            probs = torch.sigmoid(logits).detach().cpu().numpy()\n",
    "\n",
    "            all_probs.append(probs)\n",
    "            all_labels.append(labels.numpy())\n",
    "            all_indices.append(batch_indices.numpy())\n",
    "\n",
    "    return (\n",
    "        np.concatenate(all_probs),\n",
    "        np.concatenate(all_labels).astype(int),\n",
    "        np.concatenate(all_indices).astype(int),\n",
    "    )\n",
    "\n",
    "\n",
    "def evaluate_binary(y_true, y_pred):\n",
    "    '''Compute binary classification metrics with a stable confusion matrix.'''\n",
    "    cm = confusion_matrix(y_true, y_pred, labels=[0, 1])\n",
    "    tn, fp, fn, tp = cm.ravel()\n",
    "    return {\n",
    "        \"accuracy\": accuracy_score(y_true, y_pred),\n",
    "        \"precision\": precision_score(y_true, y_pred, zero_division=0),\n",
    "        \"recall\": recall_score(y_true, y_pred, zero_division=0),\n",
    "        \"f1\": f1_score(y_true, y_pred, zero_division=0),\n",
    "        \"tn\": int(tn),\n",
    "        \"fp\": int(fp),\n",
    "        \"fn\": int(fn),\n",
    "        \"tp\": int(tp),\n",
    "    }\n",
    "\n",
    "probs, true_labels, test_indices = predict_probabilities(model, test_loader)\n",
    "\n",
    "thresholds = [0.10, 0.15, 0.20, 0.25, 0.30, 0.35, 0.40, 0.45, 0.50, 0.60, 0.70, 0.80, 0.90]\n",
    "rows = []\n",
    "\n",
    "for th in thresholds:\n",
    "    preds = (probs >= th).astype(int)\n",
    "    metrics = evaluate_binary(true_labels, preds)\n",
    "    metrics.update({\n",
    "        \"model\": \"LogMiniLM\",\n",
    "        \"window_method\": \"fixed_10_line\",\n",
    "        \"threshold\": th,\n",
    "        \"window_size\": WINDOW_SIZE,\n",
    "        \"step_size\": STEP_SIZE,\n",
    "        \"projection_dim\": PROJECTION_DIM,\n",
    "        \"transformer_layers\": NUM_TRANSFORMER_LAYERS,\n",
    "        \"attention_heads\": NUM_ATTENTION_HEADS,\n",
    "        \"feedforward_dim\": FEEDFORWARD_DIM,\n",
    "        \"train_sequences\": len(y_train_model),\n",
    "        \"validation_sequences\": len(y_val),\n",
    "        \"test_sequences\": len(y_test),\n",
    "        \"unique_messages\": len(unique_texts),\n",
    "    })\n",
    "    rows.append(metrics)\n",
    "\n",
    "results_df = pd.DataFrame(rows).sort_values(\"f1\", ascending=False).reset_index(drop=True)\n",
    "results_df.to_csv(RESULTS_PATH, index=False)\n",
    "\n",
    "print(\"Saved results to:\", RESULTS_PATH)\n",
    "display(results_df)\n",
    "\n",
    "best_row = results_df.iloc[0]\n",
    "best_threshold = float(best_row[\"threshold\"])\n",
    "print(\"\\nBest threshold:\", best_threshold)\n",
    "print(\"Best F1-score:\", best_row[\"f1\"])\n",
    "print(\"Best accuracy:\", best_row[\"accuracy\"])\n",
    "\n",
    "best_preds = (probs >= best_threshold).astype(int)\n",
    "print(\"\\nClassification report at best threshold\")\n",
    "print(classification_report(true_labels, best_preds, target_names=[\"Normal\", \"Anomaly\"], zero_division=0))\n",
    "\n",
    "print(\"\\nPrediction probability statistics\")\n",
    "print(\"Min:\", probs.min())\n",
    "print(\"Max:\", probs.max())\n",
    "print(\"Mean:\", probs.mean())\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "99211d27",
   "metadata": {},
   "source": [
    "## Step 14 — Save prediction outputs for XAI\n",
    "\n",
    "For XAI, each sequence prediction must be linked back to the original log lines.\n",
    "\n",
    "### Saved fields\n",
    "\n",
    "- sequence local test index\n",
    "- start and end line numbers\n",
    "- start and end time\n",
    "- start and end node\n",
    "- true label\n",
    "- predicted label\n",
    "- anomaly probability\n",
    "- threshold\n",
    "\n",
    "### Memory-safe design\n",
    "\n",
    "This step does not reload the full raw log file. It uses the lightweight metadata already saved earlier.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e3cdf67b",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_best = (probs >= best_threshold).astype(int)\n",
    "\n",
    "# Use lightweight metadata. If df is still available, use it directly; otherwise load the lightweight parquet.\n",
    "if \"df\" in globals():\n",
    "    metadata_df = df[[\n",
    "        \"line_index\", \"raw_label\", \"label\", \"date\", \"full_time\", \"node\",\n",
    "        \"source\", \"component\", \"severity\", \"event_message\", \"model_text_norm\"\n",
    "    ]]\n",
    "else:\n",
    "    metadata_df = pd.read_parquet(METADATA_PATH)\n",
    "\n",
    "prediction_rows = []\n",
    "for local_idx, prob, y_true, y_pred in zip(test_indices, probs, true_labels, y_pred_best):\n",
    "    start_line, end_line = ranges_test[local_idx]\n",
    "    start_meta = metadata_df.iloc[start_line]\n",
    "    end_meta = metadata_df.iloc[end_line]\n",
    "\n",
    "    prediction_rows.append({\n",
    "        \"sequence_local_test_index\": int(local_idx),\n",
    "        \"start_line\": int(start_line),\n",
    "        \"end_line\": int(end_line),\n",
    "        \"start_time\": start_meta[\"full_time\"],\n",
    "        \"end_time\": end_meta[\"full_time\"],\n",
    "        \"start_node\": start_meta[\"node\"],\n",
    "        \"end_node\": end_meta[\"node\"],\n",
    "        \"true_label\": int(y_true),\n",
    "        \"predicted_label\": int(y_pred),\n",
    "        \"anomaly_probability\": float(prob),\n",
    "        \"threshold\": best_threshold,\n",
    "    })\n",
    "\n",
    "predictions_df = pd.DataFrame(prediction_rows)\n",
    "predictions_df.to_csv(PREDICTIONS_PATH, index=False)\n",
    "\n",
    "print(\"Saved predictions for XAI to:\", PREDICTIONS_PATH)\n",
    "display(predictions_df.head())\n",
    "\n",
    "# Save a small high-value subset for quick XAI inspection.\n",
    "# Priority: false positives, false negatives, and highest-probability anomaly predictions.\n",
    "fp_df = predictions_df[(predictions_df.true_label == 0) & (predictions_df.predicted_label == 1)].copy()\n",
    "fn_df = predictions_df[(predictions_df.true_label == 1) & (predictions_df.predicted_label == 0)].copy()\n",
    "tp_top_df = predictions_df[(predictions_df.true_label == 1) & (predictions_df.predicted_label == 1)].sort_values(\n",
    "    \"anomaly_probability\", ascending=False\n",
    ").head(100)\n",
    "\n",
    "top_xai_df = pd.concat([fp_df, fn_df, tp_top_df], ignore_index=True)\n",
    "top_xai_df.to_csv(TOP_XAI_CANDIDATES_PATH, index=False)\n",
    "print(\"Saved top XAI candidates to:\", TOP_XAI_CANDIDATES_PATH)\n",
    "print(\"FP count:\", len(fp_df), \"FN count:\", len(fn_df), \"Top TP saved:\", len(tp_top_df))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3c2e0b99",
   "metadata": {},
   "source": [
    "## Step 15 — Save the trained model checkpoint\n",
    "\n",
    "The checkpoint is used in the XAI stage.\n",
    "\n",
    "### Saved content\n",
    "\n",
    "- LogMiniLM model weights\n",
    "- model configuration\n",
    "- best threshold\n",
    "- best metrics\n",
    "- cache paths\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "08036205",
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoint = {\n",
    "    \"model_state_dict\": model.state_dict(),\n",
    "    \"config\": {\n",
    "        \"model_name\": \"LogMiniLM\",\n",
    "        \"minilm_model_name\": MINILM_MODEL_NAME,\n",
    "        \"window_size\": WINDOW_SIZE,\n",
    "        \"step_size\": STEP_SIZE,\n",
    "        \"split\": \"temporal_80_20_with_validation_monitoring\",\n",
    "        \"validation_ratio_within_train\": VALIDATION_RATIO_WITHIN_TRAIN,\n",
    "        \"projection_dim\": PROJECTION_DIM,\n",
    "        \"num_transformer_layers\": NUM_TRANSFORMER_LAYERS,\n",
    "        \"num_attention_heads\": NUM_ATTENTION_HEADS,\n",
    "        \"feedforward_dim\": FEEDFORWARD_DIM,\n",
    "        \"dropout\": DROPOUT,\n",
    "        \"best_threshold\": best_threshold,\n",
    "    },\n",
    "    \"best_metrics\": best_row.to_dict(),\n",
    "    \"paths\": {\n",
    "        \"unique_texts_path\": str(UNIQUE_TEXTS_PATH),\n",
    "        \"embeddings_path\": str(EMBEDDINGS_PATH),\n",
    "        \"event_id_path\": str(EVENT_ID_PATH),\n",
    "        \"labels_path\": str(LABELS_PATH),\n",
    "        \"metadata_path\": str(METADATA_PATH),\n",
    "        \"predictions_path\": str(PREDICTIONS_PATH),\n",
    "        \"top_xai_candidates_path\": str(TOP_XAI_CANDIDATES_PATH),\n",
    "        \"learning_history_path\": str(LEARNING_HISTORY_PATH),\n",
    "    },\n",
    "}\n",
    "\n",
    "torch.save(checkpoint, MODEL_PATH)\n",
    "print(\"Saved LogMiniLM checkpoint to:\", MODEL_PATH)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a11b352f",
   "metadata": {},
   "source": [
    "## Step 16 — Thesis-ready summary\n",
    "\n",
    "Use this wording in the thesis methodology/results section:\n",
    "\n",
    "> LogMiniLM was designed as a lightweight semantic Transformer model for log anomaly detection. The model first applies metadata-reduced preprocessing, where timestamp and node information are preserved for reporting but excluded from direct model input to reduce shortcut learning. Normalized log messages are then deduplicated so that MiniLM embeddings are generated only once for each unique normalized message. These cached embeddings are mapped back to log lines through event identifiers. A projection layer reduces the MiniLM embedding dimension from 384 to 128 before a compact Transformer encoder models the fixed 10-line log sequences. The final classifier predicts whether each sequence is normal or anomalous, and a threshold sweep is used to select the best F1-score.\n",
    "\n",
    "### Final selected pipeline\n",
    "\n",
    "```text\n",
    "Raw BGL logs\n",
    "  ↓\n",
    "Label extraction and metadata separation\n",
    "  ↓\n",
    "Normalize model input text\n",
    "  ↓\n",
    "Encode unique normalized messages with MiniLM\n",
    "  ↓\n",
    "Cache semantic embeddings\n",
    "  ↓\n",
    "Build fixed 10-line sequences\n",
    "  ↓\n",
    "Temporal 80/20 split\n",
    "  ↓\n",
    "384 → 128 projection\n",
    "  ↓\n",
    "Compact Transformer encoder\n",
    "  ↓\n",
    "Binary classification head\n",
    "  ↓\n",
    "Threshold sweep and anomaly prediction\n",
    "  ↓\n",
    "Prediction outputs for XAI\n",
    "```\n",
    "\n",
    "### Selected output files\n",
    "\n",
    "```text\n",
    "logminilm_results.csv\n",
    "logminilm_predictions_for_xai.csv\n",
    "logminilm_top_xai_candidates.csv\n",
    "logminilm_model.pt\n",
    "bgl_metadata_for_alerts_light.parquet\n",
    "logminilm_learning_history.csv\n",
    "```\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "86d1e6a7",
   "metadata": {},
   "source": [
    "## Step 17 — XAI setup: event-level occlusion explanation\n",
    "\n",
    "This section explains **why LogMiniLM predicts a 10-line sequence as anomalous**.\n",
    "\n",
    "The main method is **event-level occlusion**:\n",
    "\n",
    "1. Take a predicted anomalous sequence.\n",
    "2. Replace one log event at a time with a neutral baseline event.\n",
    "3. Recompute the anomaly probability.\n",
    "4. Measure the probability drop.\n",
    "5. Rank events by contribution.\n",
    "\n",
    "A larger probability drop means the removed event was more important for the anomaly decision."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e67d5930",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================================================\n",
    "# XAI output paths\n",
    "# ============================================================\n",
    "XAI_LOCAL_PATH = CACHE_DIR / \"logminilm_xai_local_event_importance.csv\"\n",
    "XAI_GLOBAL_PATH = CACHE_DIR / \"logminilm_xai_global_summary.csv\"\n",
    "XAI_FAITHFULNESS_PATH = CACHE_DIR / \"logminilm_xai_faithfulness.csv\"\n",
    "XAI_CASE_STUDY_PATH = CACHE_DIR / \"logminilm_xai_case_study.csv\"\n",
    "\n",
    "# Number of sequences to explain.\n",
    "# Increase this if you want more global XAI coverage, but it will take longer.\n",
    "MAX_XAI_SEQUENCES = 100\n",
    "\n",
    "# Select correctly detected anomaly sequences first because they are the main explanation target.\n",
    "tp_candidates = predictions_df[\n",
    "    (predictions_df[\"true_label\"] == 1) &\n",
    "    (predictions_df[\"predicted_label\"] == 1)\n",
    "].sort_values(\"anomaly_probability\", ascending=False)\n",
    "\n",
    "# Also keep some false positives and false negatives for error analysis if available.\n",
    "fp_candidates = predictions_df[\n",
    "    (predictions_df[\"true_label\"] == 0) &\n",
    "    (predictions_df[\"predicted_label\"] == 1)\n",
    "].sort_values(\"anomaly_probability\", ascending=False)\n",
    "\n",
    "fn_candidates = predictions_df[\n",
    "    (predictions_df[\"true_label\"] == 1) &\n",
    "    (predictions_df[\"predicted_label\"] == 0)\n",
    "].sort_values(\"anomaly_probability\", ascending=False)\n",
    "\n",
    "xai_candidates = pd.concat([\n",
    "    tp_candidates.head(MAX_XAI_SEQUENCES),\n",
    "    fp_candidates.head(20),\n",
    "    fn_candidates.head(20),\n",
    "], ignore_index=True)\n",
    "\n",
    "print(\"TP candidates:\", len(tp_candidates))\n",
    "print(\"FP candidates:\", len(fp_candidates))\n",
    "print(\"FN candidates:\", len(fn_candidates))\n",
    "print(\"Selected XAI candidates:\", len(xai_candidates))\n",
    "display(xai_candidates.head())\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1db93992",
   "metadata": {},
   "source": [
    "## Step 18 — Choose a neutral baseline event for masking\n",
    "\n",
    "For occlusion, we need to replace one event at a time with a neutral event.\n",
    "\n",
    "We use the **most frequent event ID appearing in normal training sequences** as the baseline. This is a simple and reproducible masking strategy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aff91312",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================================================\n",
    "# Find a neutral baseline event ID from normal training sequences\n",
    "# ============================================================\n",
    "normal_train_sequences = X_train[y_train == 0]\n",
    "if len(normal_train_sequences) == 0:\n",
    "    raise ValueError(\"No normal training sequences found. Cannot build neutral baseline event.\")\n",
    "\n",
    "flat_normal_event_ids = normal_train_sequences.reshape(-1)\n",
    "baseline_event_id = int(pd.Series(flat_normal_event_ids).value_counts().idxmax())\n",
    "baseline_text = unique_texts.iloc[baseline_event_id]\n",
    "\n",
    "print(\"Baseline event ID:\", baseline_event_id)\n",
    "print(\"Baseline normalized text:\", baseline_text)\n",
    "\n",
    "# Thesis note:\n",
    "# This baseline represents a common normal event and is used as the neutral replacement\n",
    "# during occlusion-based explanation.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e469fab6",
   "metadata": {},
   "source": [
    "## Step 19 — Event-level occlusion function\n",
    "\n",
    "For each event position in the 10-line sequence:\n",
    "\n",
    "```text\n",
    "importance = original anomaly probability - masked anomaly probability\n",
    "```\n",
    "\n",
    "If importance is high, that event strongly supports the anomaly prediction."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "35e344f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_sequence_probability(model, sequence_event_ids):\n",
    "    \"\"\"Predict anomaly probability for one sequence of event IDs.\"\"\"\n",
    "    model.eval()\n",
    "    seq_tensor = torch.tensor(sequence_event_ids, dtype=torch.long).unsqueeze(0).to(device)\n",
    "    with torch.no_grad():\n",
    "        logits = model(seq_tensor)\n",
    "        prob = torch.sigmoid(logits).item()\n",
    "    return float(prob)\n",
    "\n",
    "\n",
    "def explain_sequence_by_occlusion(model, sequence_event_ids, baseline_event_id):\n",
    "    \"\"\"Return event-level occlusion importance for a single sequence.\"\"\"\n",
    "    sequence_event_ids = np.array(sequence_event_ids, dtype=np.int64)\n",
    "    original_prob = predict_sequence_probability(model, sequence_event_ids)\n",
    "\n",
    "    explanation_rows = []\n",
    "    for position in range(len(sequence_event_ids)):\n",
    "        masked_sequence = sequence_event_ids.copy()\n",
    "        original_event_id = int(masked_sequence[position])\n",
    "        masked_sequence[position] = baseline_event_id\n",
    "\n",
    "        masked_prob = predict_sequence_probability(model, masked_sequence)\n",
    "        importance = original_prob - masked_prob\n",
    "\n",
    "        explanation_rows.append({\n",
    "            \"event_position\": int(position + 1),\n",
    "            \"original_event_id\": original_event_id,\n",
    "            \"masked_event_id\": int(baseline_event_id),\n",
    "            \"original_probability\": float(original_prob),\n",
    "            \"masked_probability\": float(masked_prob),\n",
    "            \"importance_drop\": float(importance),\n",
    "        })\n",
    "\n",
    "    return pd.DataFrame(explanation_rows).sort_values(\n",
    "        \"importance_drop\", ascending=False\n",
    "    ).reset_index(drop=True)\n",
    "\n",
    "# Quick test on the highest-probability anomaly candidate.\n",
    "if len(tp_candidates) > 0:\n",
    "    example_local_idx = int(tp_candidates.iloc[0][\"sequence_local_test_index\"])\n",
    "    example_seq = X_test[example_local_idx]\n",
    "    example_explanation = explain_sequence_by_occlusion(model, example_seq, baseline_event_id)\n",
    "    display(example_explanation)\n",
    "else:\n",
    "    print(\"No true-positive anomaly candidates found for quick XAI test.\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d2f68209",
   "metadata": {},
   "source": [
    "## Step 20 — Generate local XAI explanations with metadata\n",
    "\n",
    "This step attaches each event importance score to operational metadata:\n",
    "\n",
    "- timestamp\n",
    "- node\n",
    "- component\n",
    "- severity\n",
    "- original event message\n",
    "- normalized model input\n",
    "\n",
    "This makes the explanation useful for investigation and thesis case studies."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "20b083b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================================================\n",
    "# Generate local XAI explanations\n",
    "# ============================================================\n",
    "local_rows = []\n",
    "faithfulness_rows = []\n",
    "\n",
    "# Ensure metadata exists.\n",
    "if \"metadata_df\" not in globals():\n",
    "    if \"df\" in globals():\n",
    "        metadata_df = df[[\n",
    "            \"line_index\", \"raw_label\", \"label\", \"date\", \"full_time\", \"node\",\n",
    "            \"source\", \"component\", \"severity\", \"event_message\", \"model_text_norm\"\n",
    "        ]]\n",
    "    else:\n",
    "        metadata_df = pd.read_parquet(METADATA_PATH)\n",
    "\n",
    "for _, cand in xai_candidates.iterrows():\n",
    "    local_idx = int(cand[\"sequence_local_test_index\"])\n",
    "    start_line = int(cand[\"start_line\"])\n",
    "    end_line = int(cand[\"end_line\"])\n",
    "    true_label = int(cand[\"true_label\"])\n",
    "    predicted_label = int(cand[\"predicted_label\"])\n",
    "    original_prob_from_eval = float(cand[\"anomaly_probability\"])\n",
    "\n",
    "    seq_event_ids = X_test[local_idx]\n",
    "    exp_df = explain_sequence_by_occlusion(model, seq_event_ids, baseline_event_id)\n",
    "\n",
    "    # Top event faithfulness for this sequence.\n",
    "    top_row = exp_df.iloc[0]\n",
    "    faithfulness_rows.append({\n",
    "        \"sequence_local_test_index\": local_idx,\n",
    "        \"start_line\": start_line,\n",
    "        \"end_line\": end_line,\n",
    "        \"true_label\": true_label,\n",
    "        \"predicted_label\": predicted_label,\n",
    "        \"original_probability\": float(top_row[\"original_probability\"]),\n",
    "        \"top_event_position\": int(top_row[\"event_position\"]),\n",
    "        \"top_event_importance_drop\": float(top_row[\"importance_drop\"]),\n",
    "        \"top_event_masked_probability\": float(top_row[\"masked_probability\"]),\n",
    "    })\n",
    "\n",
    "    # Add metadata per event.\n",
    "    for _, row in exp_df.iterrows():\n",
    "        event_position = int(row[\"event_position\"])\n",
    "        line_number = start_line + event_position - 1\n",
    "\n",
    "        meta = metadata_df.iloc[line_number]\n",
    "        local_rows.append({\n",
    "            \"sequence_local_test_index\": local_idx,\n",
    "            \"start_line\": start_line,\n",
    "            \"end_line\": end_line,\n",
    "            \"event_position\": event_position,\n",
    "            \"line_index\": int(meta[\"line_index\"]),\n",
    "            \"true_label\": true_label,\n",
    "            \"predicted_label\": predicted_label,\n",
    "            \"original_probability\": float(row[\"original_probability\"]),\n",
    "            \"masked_probability\": float(row[\"masked_probability\"]),\n",
    "            \"importance_drop\": float(row[\"importance_drop\"]),\n",
    "            \"original_event_id\": int(row[\"original_event_id\"]),\n",
    "            \"raw_label\": meta[\"raw_label\"],\n",
    "            \"line_label\": int(meta[\"label\"]),\n",
    "            \"full_time\": meta[\"full_time\"],\n",
    "            \"node\": meta[\"node\"],\n",
    "            \"source\": meta[\"source\"],\n",
    "            \"component\": meta[\"component\"],\n",
    "            \"severity\": meta[\"severity\"],\n",
    "            \"event_message\": meta[\"event_message\"],\n",
    "            \"model_text_norm\": meta[\"model_text_norm\"],\n",
    "        })\n",
    "\n",
    "local_xai_df = pd.DataFrame(local_rows)\n",
    "faithfulness_df = pd.DataFrame(faithfulness_rows)\n",
    "\n",
    "local_xai_df.to_csv(XAI_LOCAL_PATH, index=False)\n",
    "faithfulness_df.to_csv(XAI_FAITHFULNESS_PATH, index=False)\n",
    "\n",
    "print(\"Saved local XAI explanations to:\", XAI_LOCAL_PATH)\n",
    "print(\"Saved faithfulness results to:\", XAI_FAITHFULNESS_PATH)\n",
    "print(\"Local XAI rows:\", len(local_xai_df))\n",
    "print(\"Faithfulness rows:\", len(faithfulness_df))\n",
    "\n",
    "display(local_xai_df.head(20))\n",
    "display(faithfulness_df.head())\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "43881c3f",
   "metadata": {},
   "source": [
    "## Step 21 — Global XAI summary\n",
    "\n",
    "The global summary aggregates the most influential events across many anomalous predictions.\n",
    "\n",
    "This helps answer:\n",
    "\n",
    "```text\n",
    "Which components, severities, and message patterns most often drive anomaly decisions?\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "36f0b01b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================================================\n",
    "# Global XAI summary\n",
    "# ============================================================\n",
    "# Focus on positive importance only. Negative values mean masking increased probability.\n",
    "positive_xai = local_xai_df[local_xai_df[\"importance_drop\"] > 0].copy()\n",
    "\n",
    "if len(positive_xai) == 0:\n",
    "    print(\"No positive importance scores found. Global summary will be empty.\")\n",
    "    global_summary_df = pd.DataFrame()\n",
    "else:\n",
    "    global_summary_df = (\n",
    "        positive_xai\n",
    "        .groupby([\"component\", \"severity\", \"event_message\"], dropna=False)\n",
    "        .agg(\n",
    "            frequency=(\"event_message\", \"count\"),\n",
    "            mean_importance=(\"importance_drop\", \"mean\"),\n",
    "            max_importance=(\"importance_drop\", \"max\"),\n",
    "            mean_original_probability=(\"original_probability\", \"mean\"),\n",
    "        )\n",
    "        .reset_index()\n",
    "        .sort_values([\"frequency\", \"mean_importance\"], ascending=[False, False])\n",
    "    )\n",
    "\n",
    "global_summary_df.to_csv(XAI_GLOBAL_PATH, index=False)\n",
    "\n",
    "print(\"Saved global XAI summary to:\", XAI_GLOBAL_PATH)\n",
    "display(global_summary_df.head(20))\n",
    "\n",
    "print(\"\\nFaithfulness summary\")\n",
    "if len(faithfulness_df) > 0:\n",
    "    display(faithfulness_df[[\"top_event_importance_drop\"]].describe())\n",
    "else:\n",
    "    print(\"No faithfulness rows available.\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a09ef43c",
   "metadata": {},
   "source": [
    "## Step 22 — XAI case study\n",
    "\n",
    "This step extracts one clear anomaly case and ranks the ten events by importance.\n",
    "\n",
    "Use this table in the thesis to show a local explanation example."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a7eaaee8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================================================\n",
    "# Save one thesis-friendly XAI case study\n",
    "# ============================================================\n",
    "if len(tp_candidates) > 0:\n",
    "    # Use highest-probability true-positive anomaly as the case study.\n",
    "    case_local_idx = int(tp_candidates.iloc[0][\"sequence_local_test_index\"])\n",
    "    case_df = local_xai_df[local_xai_df[\"sequence_local_test_index\"] == case_local_idx].copy()\n",
    "    case_df = case_df.sort_values(\"importance_drop\", ascending=False).reset_index(drop=True)\n",
    "    case_df.insert(0, \"importance_rank\", np.arange(1, len(case_df) + 1))\n",
    "\n",
    "    case_df.to_csv(XAI_CASE_STUDY_PATH, index=False)\n",
    "    print(\"Saved XAI case study to:\", XAI_CASE_STUDY_PATH)\n",
    "    print(\"Case study sequence local test index:\", case_local_idx)\n",
    "    display(case_df[[\n",
    "        \"importance_rank\", \"event_position\", \"importance_drop\",\n",
    "        \"original_probability\", \"masked_probability\", \"full_time\", \"node\",\n",
    "        \"component\", \"severity\", \"raw_label\", \"event_message\"\n",
    "    ]])\n",
    "else:\n",
    "    print(\"No true-positive anomaly candidates found. Cannot create a TP case study.\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "11775798",
   "metadata": {},
   "source": [
    "## Step 23 — Optional visualization for one local explanation\n",
    "\n",
    "The bar chart shows the importance of each event in the selected 10-line sequence.\n",
    "\n",
    "Higher bars mean the event contributed more strongly to the anomaly prediction."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ba441595",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Optional visualization: event importance for the case study.\n",
    "# This cell is safe to skip if you only need CSV outputs.\n",
    "\n",
    "if \"case_df\" in globals() and len(case_df) > 0:\n",
    "    import matplotlib.pyplot as plt\n",
    "\n",
    "    plot_df = case_df.sort_values(\"event_position\")\n",
    "\n",
    "    plt.figure(figsize=(10, 4))\n",
    "    plt.bar(plot_df[\"event_position\"].astype(str), plot_df[\"importance_drop\"])\n",
    "    plt.xlabel(\"Event position in 10-line sequence\")\n",
    "    plt.ylabel(\"Importance drop\")\n",
    "    plt.title(\"LogMiniLM Event-Level Occlusion Explanation\")\n",
    "    plt.tight_layout()\n",
    "    plt.show()\n",
    "else:\n",
    "    print(\"No case_df available for plotting.\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0d4c1102",
   "metadata": {},
   "source": [
    "## Step 24 — Thesis-ready XAI summary\n",
    "\n",
    "Run this cell to print a concise summary that can be copied into the thesis."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "20d4a60e",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"XAI method: Event-level occlusion-based explanation\")\n",
    "print(\"Explained sequences:\", len(faithfulness_df))\n",
    "print(\"Local XAI output:\", XAI_LOCAL_PATH)\n",
    "print(\"Global XAI output:\", XAI_GLOBAL_PATH)\n",
    "print(\"Faithfulness output:\", XAI_FAITHFULNESS_PATH)\n",
    "print(\"Case study output:\", XAI_CASE_STUDY_PATH)\n",
    "\n",
    "if len(faithfulness_df) > 0:\n",
    "    mean_drop = faithfulness_df[\"top_event_importance_drop\"].mean()\n",
    "    median_drop = faithfulness_df[\"top_event_importance_drop\"].median()\n",
    "    max_drop = faithfulness_df[\"top_event_importance_drop\"].max()\n",
    "    print(\"\\nFaithfulness scores based on top-event probability drop\")\n",
    "    print(\"Mean drop:\", mean_drop)\n",
    "    print(\"Median drop:\", median_drop)\n",
    "    print(\"Max drop:\", max_drop)\n",
    "\n",
    "print(\"\"\"\n",
    "Thesis-ready wording:\n",
    "Event-level occlusion-based explanation was applied to interpret LogMiniLM predictions.\n",
    "For each predicted anomalous sequence, one log event was masked at a time using a common normal baseline event, and the anomaly probability was recalculated.\n",
    "The importance of each event was measured as the decrease in anomaly probability after masking.\n",
    "Events producing the largest probability drops were considered the most influential contributors to the anomaly decision.\n",
    "The explanation was linked back to operational metadata, including timestamp, node, component, severity, and message content, to support human interpretation and incident investigation.\n",
    "\"\"\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e8c9e2af",
   "metadata": {},
   "source": [
    "## Step 25 — Top-k occlusion faithfulness\n",
    "\n",
    "This step checks whether masking multiple important events gives stronger evidence than masking only the top event. It saves `logminilm_xai_topk_faithfulness.csv`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "495b9ebe",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================================================\n",
    "# Step 25 — Top-k occlusion faithfulness\n",
    "# ============================================================\n",
    "\n",
    "TOP_K_VALUES = [1, 2, 3, 5]\n",
    "TOPK_FAITHFULNESS_PATH = CACHE_DIR / \"logminilm_xai_topk_faithfulness.csv\"\n",
    "\n",
    "topk_rows = []\n",
    "\n",
    "for _, cand in xai_candidates.iterrows():\n",
    "\n",
    "    local_idx = int(cand[\"sequence_local_test_index\"])\n",
    "    start_line = int(cand[\"start_line\"])\n",
    "    end_line = int(cand[\"end_line\"])\n",
    "    true_label = int(cand[\"true_label\"])\n",
    "    predicted_label = int(cand[\"predicted_label\"])\n",
    "\n",
    "    seq_event_ids = np.array(X_test[local_idx], dtype=np.int64)\n",
    "\n",
    "    exp_df = explain_sequence_by_occlusion(\n",
    "        model,\n",
    "        seq_event_ids,\n",
    "        baseline_event_id\n",
    "    )\n",
    "\n",
    "    original_prob = float(exp_df[\"original_probability\"].iloc[0])\n",
    "\n",
    "    ranked_positions = (\n",
    "        exp_df\n",
    "        .sort_values(\"importance_drop\", ascending=False)\n",
    "        [\"event_position\"]\n",
    "        .astype(int)\n",
    "        .tolist()\n",
    "    )\n",
    "\n",
    "    for k in TOP_K_VALUES:\n",
    "\n",
    "        masked_seq = seq_event_ids.copy()\n",
    "\n",
    "        for pos in ranked_positions[:k]:\n",
    "            masked_seq[pos - 1] = baseline_event_id\n",
    "\n",
    "        masked_prob = predict_sequence_probability(model, masked_seq)\n",
    "        probability_drop = original_prob - masked_prob\n",
    "\n",
    "        topk_rows.append({\n",
    "            \"sequence_local_test_index\": local_idx,\n",
    "            \"start_line\": start_line,\n",
    "            \"end_line\": end_line,\n",
    "            \"true_label\": true_label,\n",
    "            \"predicted_label\": predicted_label,\n",
    "            \"k\": k,\n",
    "            \"original_probability\": original_prob,\n",
    "            \"topk_masked_probability\": float(masked_prob),\n",
    "            \"topk_probability_drop\": float(probability_drop),\n",
    "        })\n",
    "\n",
    "topk_faithfulness_df = pd.DataFrame(topk_rows)\n",
    "topk_faithfulness_df.to_csv(TOPK_FAITHFULNESS_PATH, index=False)\n",
    "\n",
    "print(\"Saved top-k faithfulness to:\", TOPK_FAITHFULNESS_PATH)\n",
    "\n",
    "display(\n",
    "    topk_faithfulness_df\n",
    "    .groupby(\"k\")[\"topk_probability_drop\"]\n",
    "    .describe()\n",
    ")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ee0e88c2",
   "metadata": {},
   "source": [
    "## Step 26 — Select strongest XAI case study\n",
    "\n",
    "This step selects the sequence whose top event caused the largest anomaly probability drop."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c7036fd4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================================================\n",
    "# Step 26 — Select strongest XAI case study\n",
    "# Select a sequence where the explanation is highly faithful\n",
    "# ============================================================\n",
    "\n",
    "if len(faithfulness_df) == 0:\n",
    "    raise ValueError(\"faithfulness_df is empty. Run Step 20 before this step.\")\n",
    "\n",
    "strong_case = faithfulness_df.sort_values(\n",
    "    \"top_event_importance_drop\",\n",
    "    ascending=False\n",
    ").iloc[0]\n",
    "\n",
    "case_local_idx = int(strong_case[\"sequence_local_test_index\"])\n",
    "case_start_line = int(strong_case[\"start_line\"])\n",
    "case_end_line = int(strong_case[\"end_line\"])\n",
    "\n",
    "print(\"Selected strong XAI case study\")\n",
    "print(\"Sequence local test index:\", case_local_idx)\n",
    "print(\"Start line:\", case_start_line)\n",
    "print(\"End line:\", case_end_line)\n",
    "print(\"Original probability:\", strong_case[\"original_probability\"])\n",
    "print(\"Top event position:\", strong_case[\"top_event_position\"])\n",
    "print(\"Top event importance drop:\", strong_case[\"top_event_importance_drop\"])\n",
    "print(\"Top event masked probability:\", strong_case[\"top_event_masked_probability\"])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c1d70f67",
   "metadata": {},
   "source": [
    "## Step 27 — Build thesis-ready strong case study table\n",
    "\n",
    "This step creates the full 10-event explanation table for the selected strong case."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "75209c7b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================================================\n",
    "# Step 27 — Build thesis-ready strong case study table\n",
    "# ============================================================\n",
    "\n",
    "case_xai_df = local_xai_df[\n",
    "    local_xai_df[\"sequence_local_test_index\"] == case_local_idx\n",
    "].copy()\n",
    "\n",
    "if len(case_xai_df) == 0:\n",
    "    raise ValueError(\"No local XAI rows found for case_local_idx. Check Step 20 output.\")\n",
    "\n",
    "case_xai_df = case_xai_df.sort_values(\n",
    "    \"importance_drop\",\n",
    "    ascending=False\n",
    ").reset_index(drop=True)\n",
    "\n",
    "case_xai_df.insert(\n",
    "    0,\n",
    "    \"importance_rank\",\n",
    "    range(1, len(case_xai_df) + 1)\n",
    ")\n",
    "\n",
    "case_table = case_xai_df[[\n",
    "    \"importance_rank\",\n",
    "    \"event_position\",\n",
    "    \"importance_drop\",\n",
    "    \"original_probability\",\n",
    "    \"masked_probability\",\n",
    "    \"full_time\",\n",
    "    \"node\",\n",
    "    \"component\",\n",
    "    \"severity\",\n",
    "    \"raw_label\",\n",
    "    \"event_message\"\n",
    "]]\n",
    "\n",
    "CASE_STUDY_STRONG_PATH = CACHE_DIR / \"logminilm_xai_strong_case_study.csv\"\n",
    "case_table.to_csv(CASE_STUDY_STRONG_PATH, index=False)\n",
    "\n",
    "print(\"Saved strong XAI case study to:\", CASE_STUDY_STRONG_PATH)\n",
    "display(case_table)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "59fb76f6",
   "metadata": {},
   "source": [
    "## Step 28 — Show top-5 influential events\n",
    "\n",
    "This is the shorter table that is usually best for the thesis."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0d66ead4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================================================\n",
    "# Step 28 — Top 5 influential events for thesis\n",
    "# ============================================================\n",
    "\n",
    "top5_case = case_table.head(5).copy()\n",
    "\n",
    "TOP5_CASE_PATH = CACHE_DIR / \"logminilm_xai_strong_case_study_top5.csv\"\n",
    "top5_case.to_csv(TOP5_CASE_PATH, index=False)\n",
    "\n",
    "print(\"Saved top-5 case study to:\", TOP5_CASE_PATH)\n",
    "display(top5_case)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b8880811",
   "metadata": {},
   "source": [
    "## Step 29 — Generate written explanation automatically\n",
    "\n",
    "This cell prints a paragraph that can be adapted into Chapter 4."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c17427d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================================================\n",
    "# Step 29 — Generate thesis-ready textual explanation\n",
    "# ============================================================\n",
    "\n",
    "top_event = case_table.iloc[0]\n",
    "\n",
    "original_prob = float(top_event[\"original_probability\"])\n",
    "masked_prob = float(top_event[\"masked_probability\"])\n",
    "importance_drop = float(top_event[\"importance_drop\"])\n",
    "\n",
    "print(\"Thesis-ready case study explanation:\\n\")\n",
    "\n",
    "print(f\"\"\"\n",
    "A representative true-positive anomaly sequence was selected for local explanation using event-level occlusion.\n",
    "The original anomaly probability of the sequence was {original_prob:.6f}.\n",
    "The most influential event was located at position {int(top_event[\"event_position\"])} in the 10-line sequence.\n",
    "\n",
    "The event was:\n",
    "Component: {top_event[\"component\"]}\n",
    "Severity: {top_event[\"severity\"]}\n",
    "Raw label: {top_event[\"raw_label\"]}\n",
    "Message: {top_event[\"event_message\"]}\n",
    "\n",
    "After masking this event with a neutral normal baseline event, the anomaly probability decreased to {masked_prob:.6f}.\n",
    "This produced an importance drop of {importance_drop:.6f}, indicating that this event had a strong causal influence on the LogMiniLM anomaly prediction.\n",
    "The explanation was also mapped back to operational metadata, including timestamp and node, making the prediction interpretable for investigation.\n",
    "\"\"\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f99a345d",
   "metadata": {},
   "source": [
    "## Step 30 — Optional bar chart for strong XAI case study\n",
    "\n",
    "This creates a PNG chart that can be used in the thesis or slides."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "627b4701",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================================================\n",
    "# Step 30 — Bar chart for strong XAI case study\n",
    "# ============================================================\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plot_df = case_table.sort_values(\"event_position\")\n",
    "\n",
    "plt.figure(figsize=(10, 4))\n",
    "plt.bar(\n",
    "    plot_df[\"event_position\"].astype(str),\n",
    "    plot_df[\"importance_drop\"]\n",
    ")\n",
    "\n",
    "plt.xlabel(\"Event position in 10-line sequence\")\n",
    "plt.ylabel(\"Importance drop\")\n",
    "plt.title(\"Event-Level Occlusion Explanation for Selected Anomaly Sequence\")\n",
    "plt.tight_layout()\n",
    "\n",
    "CASE_STUDY_PLOT_PATH = CACHE_DIR / \"logminilm_xai_strong_case_study_plot.png\"\n",
    "plt.savefig(CASE_STUDY_PLOT_PATH, dpi=300, bbox_inches=\"tight\")\n",
    "plt.show()\n",
    "\n",
    "print(\"Saved case study plot to:\", CASE_STUDY_PLOT_PATH)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "07cfa4fc",
   "metadata": {},
   "source": [
    "## Step 31 — Install and import SHAP\n",
    "\n",
    "SHAP is added as a supporting event-level explanation method. Occlusion remains the primary XAI method."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "49bd2cd8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================================================\n",
    "# Step 31 — Install and import SHAP\n",
    "# ============================================================\n",
    "\n",
    "!pip install shap -q\n",
    "\n",
    "import shap\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "SHAP_LOCAL_PATH = CACHE_DIR / \"logminilm_shap_event_level_explanations.csv\"\n",
    "SHAP_CASE_PATH = CACHE_DIR / \"logminilm_shap_case_study.csv\"\n",
    "SHAP_COMPARE_PATH = CACHE_DIR / \"logminilm_shap_occlusion_comparison.csv\"\n",
    "\n",
    "print(\"SHAP imported successfully.\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6cdaadfd",
   "metadata": {},
   "source": [
    "## Step 32 — Select sequences for SHAP\n",
    "\n",
    "SHAP is slower than occlusion, so this step explains the strong case plus a small set of high-confidence true-positive anomaly sequences."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "71d5802a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================================================\n",
    "# Step 32 — Select sequences for SHAP explanation\n",
    "# ============================================================\n",
    "\n",
    "# Explain the strong case study first.\n",
    "strong_case_idx = case_local_idx\n",
    "\n",
    "# Also explain a few high-confidence true-positive anomalies.\n",
    "SHAP_NUM_TP = 10\n",
    "\n",
    "tp_for_shap = (\n",
    "    predictions_df[\n",
    "        (predictions_df[\"true_label\"] == 1) &\n",
    "        (predictions_df[\"predicted_label\"] == 1)\n",
    "    ]\n",
    "    .sort_values(\"anomaly_probability\", ascending=False)\n",
    "    .head(SHAP_NUM_TP)\n",
    ")\n",
    "\n",
    "shap_sequence_indices = [strong_case_idx] + [\n",
    "    int(x) for x in tp_for_shap[\"sequence_local_test_index\"].tolist()\n",
    "]\n",
    "\n",
    "# Remove duplicates while preserving order.\n",
    "shap_sequence_indices = list(dict.fromkeys(shap_sequence_indices))\n",
    "\n",
    "print(\"Number of sequences selected for SHAP:\", len(shap_sequence_indices))\n",
    "print(\"Selected SHAP sequence indices:\", shap_sequence_indices)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "12cc5ae5",
   "metadata": {},
   "source": [
    "## Step 33 — Define SHAP event-level prediction function\n",
    "\n",
    "For SHAP, each feature is an event position in the 10-line sequence. SHAP uses a binary mask: `1` means keep the original event, and `0` means replace it with the neutral baseline event."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "facce1e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================================================\n",
    "# Step 33 — SHAP prediction function for event-level masking\n",
    "# ============================================================\n",
    "\n",
    "def predict_probability_from_event_ids(sequence_event_ids):\n",
    "    \"\"\"Predict anomaly probability for one sequence of event IDs.\"\"\"\n",
    "    model.eval()\n",
    "\n",
    "    seq_tensor = torch.tensor(\n",
    "        sequence_event_ids,\n",
    "        dtype=torch.long\n",
    "    ).unsqueeze(0).to(device)\n",
    "\n",
    "    with torch.no_grad():\n",
    "        logits = model(seq_tensor)\n",
    "        prob = torch.sigmoid(logits).item()\n",
    "\n",
    "    return float(prob)\n",
    "\n",
    "\n",
    "def make_shap_mask_predict_fn(original_sequence_event_ids, baseline_event_id):\n",
    "    \"\"\"\n",
    "    Create a SHAP-compatible prediction function for one sequence.\n",
    "\n",
    "    SHAP input:\n",
    "        mask_matrix shape = number_of_samples × 10\n",
    "\n",
    "    Each mask value:\n",
    "        1 = keep original event\n",
    "        0 = replace event with neutral baseline event\n",
    "\n",
    "    Output:\n",
    "        anomaly probability for each masked version\n",
    "    \"\"\"\n",
    "\n",
    "    original_sequence_event_ids = np.array(\n",
    "        original_sequence_event_ids,\n",
    "        dtype=np.int64\n",
    "    )\n",
    "\n",
    "    def shap_predict_fn(mask_matrix):\n",
    "        mask_matrix = np.array(mask_matrix)\n",
    "\n",
    "        outputs = []\n",
    "\n",
    "        for mask_row in mask_matrix:\n",
    "            # Convert SHAP mask values to binary keep/remove decisions.\n",
    "            binary_mask = (mask_row >= 0.5).astype(int)\n",
    "\n",
    "            perturbed_sequence = original_sequence_event_ids.copy()\n",
    "\n",
    "            for pos in range(len(perturbed_sequence)):\n",
    "                if binary_mask[pos] == 0:\n",
    "                    perturbed_sequence[pos] = baseline_event_id\n",
    "\n",
    "            prob = predict_probability_from_event_ids(perturbed_sequence)\n",
    "            outputs.append(prob)\n",
    "\n",
    "        return np.array(outputs)\n",
    "\n",
    "    return shap_predict_fn\n",
    "\n",
    "\n",
    "print(\"SHAP event-level prediction function is ready.\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "91bb8008",
   "metadata": {},
   "source": [
    "## Step 34 — Run event-level SHAP\n",
    "\n",
    "This computes SHAP values for the selected sequences. Increase `SHAP_NSAMPLES` only if you need more stable SHAP estimates and have enough time."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "177459ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================================================\n",
    "# Step 34 — Run event-level SHAP\n",
    "# ============================================================\n",
    "\n",
    "shap_rows = []\n",
    "\n",
    "# SHAP background:\n",
    "# all-zero mask means all events are replaced by the neutral baseline event.\n",
    "background_mask = np.zeros((1, WINDOW_SIZE))\n",
    "\n",
    "# SHAP explains the full original sequence:\n",
    "# all-one mask means all original events are kept.\n",
    "full_mask = np.ones((1, WINDOW_SIZE))\n",
    "\n",
    "# Higher nsamples = more stable but slower.\n",
    "SHAP_NSAMPLES = 200\n",
    "\n",
    "for seq_num, local_idx in enumerate(shap_sequence_indices, start=1):\n",
    "\n",
    "    print(f\"\\nRunning SHAP for sequence {seq_num}/{len(shap_sequence_indices)}\")\n",
    "    print(\"Local test index:\", local_idx)\n",
    "\n",
    "    sequence_event_ids = np.array(X_test[local_idx], dtype=np.int64)\n",
    "\n",
    "    original_prob = predict_probability_from_event_ids(sequence_event_ids)\n",
    "\n",
    "    shap_predict_fn = make_shap_mask_predict_fn(\n",
    "        sequence_event_ids,\n",
    "        baseline_event_id\n",
    "    )\n",
    "\n",
    "    explainer = shap.KernelExplainer(\n",
    "        shap_predict_fn,\n",
    "        background_mask\n",
    "    )\n",
    "\n",
    "    shap_values = explainer.shap_values(\n",
    "        full_mask,\n",
    "        nsamples=SHAP_NSAMPLES\n",
    "    )\n",
    "\n",
    "    # shap_values may come as list or ndarray depending on SHAP version.\n",
    "    if isinstance(shap_values, list):\n",
    "        shap_values_array = np.array(shap_values[0]).reshape(-1)\n",
    "    else:\n",
    "        shap_values_array = np.array(shap_values).reshape(-1)\n",
    "\n",
    "    expected_value = explainer.expected_value\n",
    "\n",
    "    if isinstance(expected_value, (list, np.ndarray)):\n",
    "        expected_value = float(np.array(expected_value).reshape(-1)[0])\n",
    "    else:\n",
    "        expected_value = float(expected_value)\n",
    "\n",
    "    for pos in range(WINDOW_SIZE):\n",
    "        event_id = int(sequence_event_ids[pos])\n",
    "        shap_value = float(shap_values_array[pos])\n",
    "\n",
    "        shap_rows.append({\n",
    "            \"sequence_local_test_index\": int(local_idx),\n",
    "            \"event_position\": int(pos + 1),\n",
    "            \"event_id\": event_id,\n",
    "            \"original_probability\": float(original_prob),\n",
    "            \"shap_expected_value\": float(expected_value),\n",
    "            \"shap_value\": shap_value,\n",
    "            \"abs_shap_value\": abs(shap_value),\n",
    "        })\n",
    "\n",
    "shap_xai_df = pd.DataFrame(shap_rows)\n",
    "\n",
    "shap_xai_df = shap_xai_df.sort_values(\n",
    "    [\"sequence_local_test_index\", \"abs_shap_value\"],\n",
    "    ascending=[True, False]\n",
    ").reset_index(drop=True)\n",
    "\n",
    "shap_xai_df.to_csv(SHAP_LOCAL_PATH, index=False)\n",
    "\n",
    "print(\"\\nSaved SHAP explanations to:\", SHAP_LOCAL_PATH)\n",
    "display(shap_xai_df.head(30))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4fe63544",
   "metadata": {},
   "source": [
    "## Step 35 — Attach metadata to SHAP explanations\n",
    "\n",
    "This maps SHAP values back to timestamp, node, component, severity, label, and event message."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8494fd6c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================================================\n",
    "# Step 35 — Attach metadata to SHAP event explanations\n",
    "# ============================================================\n",
    "\n",
    "# Build a lookup from predictions_df for start/end lines.\n",
    "pred_lookup = predictions_df.set_index(\"sequence_local_test_index\")\n",
    "\n",
    "shap_meta_rows = []\n",
    "\n",
    "for _, row in shap_xai_df.iterrows():\n",
    "\n",
    "    local_idx = int(row[\"sequence_local_test_index\"])\n",
    "    event_position = int(row[\"event_position\"])\n",
    "\n",
    "    pred_row = pred_lookup.loc[local_idx]\n",
    "\n",
    "    start_line = int(pred_row[\"start_line\"])\n",
    "    end_line = int(pred_row[\"end_line\"])\n",
    "    true_label = int(pred_row[\"true_label\"])\n",
    "    predicted_label = int(pred_row[\"predicted_label\"])\n",
    "\n",
    "    line_number = start_line + event_position - 1\n",
    "\n",
    "    meta = metadata_df.iloc[line_number]\n",
    "\n",
    "    shap_meta_rows.append({\n",
    "        \"sequence_local_test_index\": local_idx,\n",
    "        \"start_line\": start_line,\n",
    "        \"end_line\": end_line,\n",
    "        \"event_position\": event_position,\n",
    "        \"line_index\": int(meta[\"line_index\"]),\n",
    "        \"true_label\": true_label,\n",
    "        \"predicted_label\": predicted_label,\n",
    "        \"original_probability\": float(row[\"original_probability\"]),\n",
    "        \"shap_expected_value\": float(row[\"shap_expected_value\"]),\n",
    "        \"shap_value\": float(row[\"shap_value\"]),\n",
    "        \"abs_shap_value\": float(row[\"abs_shap_value\"]),\n",
    "        \"event_id\": int(row[\"event_id\"]),\n",
    "        \"raw_label\": meta[\"raw_label\"],\n",
    "        \"line_label\": int(meta[\"label\"]),\n",
    "        \"full_time\": meta[\"full_time\"],\n",
    "        \"node\": meta[\"node\"],\n",
    "        \"source\": meta[\"source\"],\n",
    "        \"component\": meta[\"component\"],\n",
    "        \"severity\": meta[\"severity\"],\n",
    "        \"event_message\": meta[\"event_message\"],\n",
    "        \"model_text_norm\": meta[\"model_text_norm\"],\n",
    "    })\n",
    "\n",
    "shap_meta_df = pd.DataFrame(shap_meta_rows)\n",
    "\n",
    "shap_meta_df = shap_meta_df.sort_values(\n",
    "    [\"sequence_local_test_index\", \"abs_shap_value\"],\n",
    "    ascending=[True, False]\n",
    ").reset_index(drop=True)\n",
    "\n",
    "shap_meta_df.to_csv(SHAP_LOCAL_PATH, index=False)\n",
    "\n",
    "print(\"Saved SHAP metadata explanations to:\", SHAP_LOCAL_PATH)\n",
    "display(shap_meta_df.head(30))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3260b4ad",
   "metadata": {},
   "source": [
    "## Step 36 — SHAP case study for the strongest anomaly\n",
    "\n",
    "This creates a SHAP table for the same strong case selected in Step 26."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "725505bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================================================\n",
    "# Step 36 — SHAP case study for the strongest anomaly\n",
    "# ============================================================\n",
    "\n",
    "shap_case_df = shap_meta_df[\n",
    "    shap_meta_df[\"sequence_local_test_index\"] == case_local_idx\n",
    "].copy()\n",
    "\n",
    "shap_case_df = shap_case_df.sort_values(\n",
    "    \"abs_shap_value\",\n",
    "    ascending=False\n",
    ").reset_index(drop=True)\n",
    "\n",
    "shap_case_df.insert(\n",
    "    0,\n",
    "    \"shap_rank\",\n",
    "    range(1, len(shap_case_df) + 1)\n",
    ")\n",
    "\n",
    "shap_case_table = shap_case_df[[\n",
    "    \"shap_rank\",\n",
    "    \"event_position\",\n",
    "    \"shap_value\",\n",
    "    \"abs_shap_value\",\n",
    "    \"original_probability\",\n",
    "    \"shap_expected_value\",\n",
    "    \"full_time\",\n",
    "    \"node\",\n",
    "    \"component\",\n",
    "    \"severity\",\n",
    "    \"raw_label\",\n",
    "    \"event_message\"\n",
    "]]\n",
    "\n",
    "shap_case_table.to_csv(SHAP_CASE_PATH, index=False)\n",
    "\n",
    "print(\"Saved SHAP case study to:\", SHAP_CASE_PATH)\n",
    "display(shap_case_table)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d5ce415c",
   "metadata": {},
   "source": [
    "## Step 37 — Compare SHAP with occlusion\n",
    "\n",
    "This checks whether SHAP and occlusion identify the same top event."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "710a2d9e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================================================\n",
    "# Step 37 — Compare SHAP ranking with occlusion ranking\n",
    "# ============================================================\n",
    "\n",
    "comparison_rows = []\n",
    "\n",
    "for local_idx in shap_sequence_indices:\n",
    "\n",
    "    # Top SHAP event.\n",
    "    shap_seq = shap_meta_df[\n",
    "        shap_meta_df[\"sequence_local_test_index\"] == local_idx\n",
    "    ].copy()\n",
    "\n",
    "    shap_seq = shap_seq.sort_values(\n",
    "        \"abs_shap_value\",\n",
    "        ascending=False\n",
    "    )\n",
    "\n",
    "    if len(shap_seq) == 0:\n",
    "        continue\n",
    "\n",
    "    top_shap = shap_seq.iloc[0]\n",
    "\n",
    "    # Top occlusion event.\n",
    "    occ_seq = local_xai_df[\n",
    "        local_xai_df[\"sequence_local_test_index\"] == local_idx\n",
    "    ].copy()\n",
    "\n",
    "    occ_seq = occ_seq.sort_values(\n",
    "        \"importance_drop\",\n",
    "        ascending=False\n",
    "    )\n",
    "\n",
    "    if len(occ_seq) == 0:\n",
    "        continue\n",
    "\n",
    "    top_occ = occ_seq.iloc[0]\n",
    "\n",
    "    comparison_rows.append({\n",
    "        \"sequence_local_test_index\": int(local_idx),\n",
    "        \"top_shap_event_position\": int(top_shap[\"event_position\"]),\n",
    "        \"top_shap_value\": float(top_shap[\"shap_value\"]),\n",
    "        \"top_shap_abs_value\": float(top_shap[\"abs_shap_value\"]),\n",
    "        \"top_shap_component\": top_shap[\"component\"],\n",
    "        \"top_shap_severity\": top_shap[\"severity\"],\n",
    "        \"top_shap_message\": top_shap[\"event_message\"],\n",
    "\n",
    "        \"top_occlusion_event_position\": int(top_occ[\"event_position\"]),\n",
    "        \"top_occlusion_importance_drop\": float(top_occ[\"importance_drop\"]),\n",
    "        \"top_occlusion_component\": top_occ[\"component\"],\n",
    "        \"top_occlusion_severity\": top_occ[\"severity\"],\n",
    "        \"top_occlusion_message\": top_occ[\"event_message\"],\n",
    "\n",
    "        \"same_top_event\": int(\n",
    "            int(top_shap[\"event_position\"]) == int(top_occ[\"event_position\"])\n",
    "        )\n",
    "    })\n",
    "\n",
    "shap_occ_compare_df = pd.DataFrame(comparison_rows)\n",
    "\n",
    "shap_occ_compare_df.to_csv(SHAP_COMPARE_PATH, index=False)\n",
    "\n",
    "print(\"Saved SHAP vs Occlusion comparison to:\", SHAP_COMPARE_PATH)\n",
    "\n",
    "if len(shap_occ_compare_df) > 0:\n",
    "    agreement_rate = shap_occ_compare_df[\"same_top_event\"].mean()\n",
    "    print(\"Top-event agreement rate:\", agreement_rate)\n",
    "\n",
    "display(shap_occ_compare_df)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e4fef8da",
   "metadata": {},
   "source": [
    "## Step 38 — Optional SHAP bar chart for case study\n",
    "\n",
    "This chart shows whether each event pushes the prediction toward anomaly or normal."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f1c890c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================================================\n",
    "# Step 38 — Optional SHAP bar chart for case study\n",
    "# ============================================================\n",
    "\n",
    "if len(shap_case_table) > 0:\n",
    "\n",
    "    plot_df = shap_case_table.sort_values(\"event_position\")\n",
    "\n",
    "    plt.figure(figsize=(10, 4))\n",
    "    plt.bar(\n",
    "        plot_df[\"event_position\"].astype(str),\n",
    "        plot_df[\"shap_value\"]\n",
    "    )\n",
    "\n",
    "    plt.axhline(0, linewidth=1)\n",
    "    plt.xlabel(\"Event position in 10-line sequence\")\n",
    "    plt.ylabel(\"SHAP value\")\n",
    "    plt.title(\"Event-Level SHAP Explanation for Selected Anomaly Sequence\")\n",
    "    plt.tight_layout()\n",
    "\n",
    "    SHAP_CASE_PLOT_PATH = CACHE_DIR / \"logminilm_shap_case_study_plot.png\"\n",
    "    plt.savefig(SHAP_CASE_PLOT_PATH, dpi=300, bbox_inches=\"tight\")\n",
    "    plt.show()\n",
    "\n",
    "    print(\"Saved SHAP case study plot to:\", SHAP_CASE_PLOT_PATH)\n",
    "\n",
    "else:\n",
    "    print(\"No SHAP case study available for plotting.\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1b571dd9",
   "metadata": {},
   "source": [
    "## Step 39 — Thesis-ready SHAP summary\n",
    "\n",
    "This prints the supporting SHAP outputs and the agreement rate between SHAP and occlusion."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c7169377",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================================================\n",
    "# Step 39 — Thesis-ready SHAP summary\n",
    "# ============================================================\n",
    "\n",
    "print(\"SHAP event-level explanation completed.\")\n",
    "print(\"Saved SHAP local explanations:\", SHAP_LOCAL_PATH)\n",
    "print(\"Saved SHAP case study:\", SHAP_CASE_PATH)\n",
    "print(\"Saved SHAP vs Occlusion comparison:\", SHAP_COMPARE_PATH)\n",
    "\n",
    "if \"shap_occ_compare_df\" in globals() and len(shap_occ_compare_df) > 0:\n",
    "    agreement_rate = shap_occ_compare_df[\"same_top_event\"].mean()\n",
    "    print(\"SHAP-Occlusion top-event agreement rate:\", agreement_rate)\n",
    "\n",
    "print(\"\"\"\n",
    "Thesis-ready wording:\n",
    "\n",
    "SHAP was applied as a supporting XAI method at the event level.\n",
    "Each position in the 10-line LogMiniLM input sequence was treated as an interpretable feature.\n",
    "For each selected sequence, SHAP estimated the contribution of keeping each event relative to a neutral baseline sequence.\n",
    "Positive SHAP values indicate events that push the model toward an anomaly prediction, while lower or negative values indicate weaker or normalizing contributions.\n",
    "The SHAP explanations were mapped back to operational metadata including timestamp, node, component, severity, label, and event message.\n",
    "Finally, SHAP rankings were compared with occlusion-based rankings to check whether both explanation methods identified the same influential events.\n",
    "\"\"\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "99357d24",
   "metadata": {},
   "source": [
    "# Part 4 — Thesis Charts and Figure Export\n",
    "\n",
    "These steps generate presentation-ready charts for the earlier pipeline stages, detection results, and XAI validation. Run them after the detection evaluation and XAI/SHAP sections have completed.\n",
    "\n",
    "The figures are saved under `logminilm_cache/figures/` in Google Drive."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c4f87788",
   "metadata": {},
   "source": [
    "## Step 40 — Figure directory and chart helpers\n",
    "\n",
    "This step creates a dedicated folder for thesis figures and defines a small helper for saving charts consistently."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "377a1841",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================================================\n",
    "# Step 40 — Figure directory and chart helpers\n",
    "# ============================================================\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from pathlib import Path\n",
    "\n",
    "FIGURES_DIR = CACHE_DIR / \"figures\"\n",
    "FIGURES_DIR.mkdir(parents=True, exist_ok=True)\n",
    "\n",
    "print(\"Figures will be saved to:\", FIGURES_DIR)\n",
    "\n",
    "\n",
    "def save_current_figure(filename):\n",
    "    \"\"\"Save the current matplotlib figure as a high-resolution PNG.\"\"\"\n",
    "    path = FIGURES_DIR / filename\n",
    "    plt.savefig(path, dpi=300, bbox_inches=\"tight\")\n",
    "    print(\"Saved figure to:\", path)\n",
    "    return path\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "01f765d3",
   "metadata": {},
   "source": [
    "## Step 41 — BGL line-level class distribution\n",
    "\n",
    "This chart shows the original normal/anomaly imbalance before sequence construction."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d0741b4b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================================================\n",
    "# Step 41 — BGL line-level class distribution\n",
    "# ============================================================\n",
    "\n",
    "if \"metadata_df\" in globals() and \"label\" in metadata_df.columns:\n",
    "    line_labels_source = metadata_df\n",
    "elif \"df\" in globals() and \"label\" in df.columns:\n",
    "    line_labels_source = df\n",
    "else:\n",
    "    raise ValueError(\"Could not find metadata_df or df with a 'label' column.\")\n",
    "\n",
    "line_label_counts = line_labels_source[\"label\"].value_counts().sort_index()\n",
    "\n",
    "line_distribution_df = pd.DataFrame({\n",
    "    \"class\": [\"Normal\", \"Anomaly\"],\n",
    "    \"count\": [\n",
    "        int(line_label_counts.get(0, 0)),\n",
    "        int(line_label_counts.get(1, 0)),\n",
    "    ]\n",
    "})\n",
    "\n",
    "line_distribution_df[\"percentage\"] = (\n",
    "    line_distribution_df[\"count\"] / line_distribution_df[\"count\"].sum() * 100\n",
    ")\n",
    "\n",
    "print(\"Line-level class distribution\")\n",
    "display(line_distribution_df)\n",
    "\n",
    "plt.figure(figsize=(6, 4))\n",
    "plt.bar(line_distribution_df[\"class\"], line_distribution_df[\"count\"])\n",
    "plt.xlabel(\"Log label\")\n",
    "plt.ylabel(\"Number of log lines\")\n",
    "plt.title(\"BGL Line-Level Class Distribution\")\n",
    "plt.tight_layout()\n",
    "save_current_figure(\"01_bgl_line_level_class_distribution.png\")\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3315716e",
   "metadata": {},
   "source": [
    "## Step 42 — Sequence-level class distribution\n",
    "\n",
    "This chart shows the class distribution after converting raw logs into fixed 10-line sequences. It includes the model-training subset, validation subset, and untouched temporal test set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f43d0514",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================================================\n",
    "# Step 42 — Sequence-level class distribution\n",
    "# ============================================================\n",
    "\n",
    "sequence_distribution_df = pd.DataFrame({\n",
    "    \"split_class\": [\n",
    "        \"Train Normal\", \"Train Anomaly\",\n",
    "        \"Validation Normal\", \"Validation Anomaly\",\n",
    "        \"Test Normal\", \"Test Anomaly\"\n",
    "    ],\n",
    "    \"count\": [\n",
    "        int(np.sum(y_train_model == 0)),\n",
    "        int(np.sum(y_train_model == 1)),\n",
    "        int(np.sum(y_val == 0)),\n",
    "        int(np.sum(y_val == 1)),\n",
    "        int(np.sum(y_test == 0)),\n",
    "        int(np.sum(y_test == 1)),\n",
    "    ]\n",
    "})\n",
    "\n",
    "print(\"Sequence-level class distribution\")\n",
    "display(sequence_distribution_df)\n",
    "\n",
    "plt.figure(figsize=(10, 4))\n",
    "plt.bar(sequence_distribution_df[\"split_class\"], sequence_distribution_df[\"count\"])\n",
    "plt.xlabel(\"Dataset split and class\")\n",
    "plt.ylabel(\"Number of sequences\")\n",
    "plt.title(\"BGL Sequence-Level Class Distribution\")\n",
    "plt.xticks(rotation=25, ha=\"right\")\n",
    "plt.tight_layout()\n",
    "save_current_figure(\"02_bgl_sequence_level_class_distribution.png\")\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a7e26e9e",
   "metadata": {},
   "source": [
    "## Step 43 — Threshold versus F1-score\n",
    "\n",
    "This chart justifies the selected decision threshold by showing how F1-score changes across thresholds."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d90f8033",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================================================\n",
    "# Step 43 — Threshold vs F1-score\n",
    "# ============================================================\n",
    "\n",
    "plot_df = results_df.sort_values(\"threshold\").copy()\n",
    "best_row = results_df.sort_values(\"f1\", ascending=False).iloc[0]\n",
    "best_threshold = float(best_row[\"threshold\"])\n",
    "best_f1 = float(best_row[\"f1\"])\n",
    "\n",
    "plt.figure(figsize=(8, 4))\n",
    "plt.plot(plot_df[\"threshold\"], plot_df[\"f1\"], marker=\"o\")\n",
    "plt.axvline(best_threshold, linestyle=\"--\")\n",
    "plt.xlabel(\"Decision threshold\")\n",
    "plt.ylabel(\"F1-score\")\n",
    "plt.title(\"LogMiniLM F1-score Across Decision Thresholds\")\n",
    "plt.tight_layout()\n",
    "save_current_figure(\"03_logminilm_threshold_vs_f1.png\")\n",
    "plt.show()\n",
    "\n",
    "print(\"Best threshold:\", best_threshold)\n",
    "print(\"Best F1-score:\", best_f1)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "80a1f640",
   "metadata": {},
   "source": [
    "## Step 44 — Precision, recall, and F1-score across thresholds\n",
    "\n",
    "This chart shows the threshold sensitivity of the final model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4e5434d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================================================\n",
    "# Step 44 — Precision, Recall, and F1 across thresholds\n",
    "# ============================================================\n",
    "\n",
    "plot_df = results_df.sort_values(\"threshold\").copy()\n",
    "\n",
    "plt.figure(figsize=(9, 4))\n",
    "plt.plot(plot_df[\"threshold\"], plot_df[\"precision\"], marker=\"o\", label=\"Precision\")\n",
    "plt.plot(plot_df[\"threshold\"], plot_df[\"recall\"], marker=\"o\", label=\"Recall\")\n",
    "plt.plot(plot_df[\"threshold\"], plot_df[\"f1\"], marker=\"o\", label=\"F1-score\")\n",
    "plt.axvline(best_threshold, linestyle=\"--\", label=f\"Best threshold = {best_threshold:.2f}\")\n",
    "plt.xlabel(\"Decision threshold\")\n",
    "plt.ylabel(\"Metric value\")\n",
    "plt.title(\"LogMiniLM Threshold Sensitivity\")\n",
    "plt.legend()\n",
    "plt.tight_layout()\n",
    "save_current_figure(\"04_logminilm_threshold_sensitivity.png\")\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "04d20445",
   "metadata": {},
   "source": [
    "## Step 45 — Confusion matrix\n",
    "\n",
    "This chart summarizes correct and incorrect predictions at the best threshold."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "85ecda33",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================================================\n",
    "# Step 45 — Confusion matrix\n",
    "# ============================================================\n",
    "\n",
    "tn = int(best_row[\"tn\"])\n",
    "fp = int(best_row[\"fp\"])\n",
    "fn = int(best_row[\"fn\"])\n",
    "tp = int(best_row[\"tp\"])\n",
    "\n",
    "cm = np.array([\n",
    "    [tn, fp],\n",
    "    [fn, tp]\n",
    "])\n",
    "\n",
    "plt.figure(figsize=(5, 4))\n",
    "plt.imshow(cm)\n",
    "plt.title(\"LogMiniLM Confusion Matrix\")\n",
    "plt.xlabel(\"Predicted label\")\n",
    "plt.ylabel(\"True label\")\n",
    "plt.xticks([0, 1], [\"Normal\", \"Anomaly\"])\n",
    "plt.yticks([0, 1], [\"Normal\", \"Anomaly\"])\n",
    "\n",
    "for i in range(2):\n",
    "    for j in range(2):\n",
    "        plt.text(j, i, f\"{cm[i, j]:,}\", ha=\"center\", va=\"center\", fontsize=12)\n",
    "\n",
    "plt.colorbar()\n",
    "plt.tight_layout()\n",
    "save_current_figure(\"05_logminilm_confusion_matrix.png\")\n",
    "plt.show()\n",
    "\n",
    "print(\"TN, FP, FN, TP:\", tn, fp, fn, tp)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cb3f9e2f",
   "metadata": {},
   "source": [
    "## Step 46 — Prediction probability distribution\n",
    "\n",
    "This chart compares anomaly probabilities for true normal and true anomalous sequences."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b0c14620",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================================================\n",
    "# Step 46 — Prediction probability distribution\n",
    "# ============================================================\n",
    "\n",
    "normal_probs = predictions_df[predictions_df[\"true_label\"] == 0][\"anomaly_probability\"]\n",
    "anomaly_probs = predictions_df[predictions_df[\"true_label\"] == 1][\"anomaly_probability\"]\n",
    "\n",
    "plt.figure(figsize=(8, 4))\n",
    "plt.hist(normal_probs, bins=50, alpha=0.7, label=\"Normal\")\n",
    "plt.hist(anomaly_probs, bins=50, alpha=0.7, label=\"Anomaly\")\n",
    "plt.xlabel(\"Anomaly probability\")\n",
    "plt.ylabel(\"Number of sequences\")\n",
    "plt.title(\"Distribution of LogMiniLM Anomaly Probabilities\")\n",
    "plt.legend()\n",
    "plt.tight_layout()\n",
    "save_current_figure(\"06_logminilm_probability_distribution.png\")\n",
    "plt.show()\n",
    "\n",
    "print(\"Normal probability statistics\")\n",
    "display(normal_probs.describe())\n",
    "print(\"Anomaly probability statistics\")\n",
    "display(anomaly_probs.describe())\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "23f26e3e",
   "metadata": {},
   "source": [
    "## Step 47 — Global XAI mean importance by event position\n",
    "\n",
    "This chart aggregates occlusion importance across explained true-positive anomaly sequences."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a97243ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================================================\n",
    "# Step 47 — Global XAI mean importance by event position\n",
    "# ============================================================\n",
    "\n",
    "all_anomaly_xai = local_xai_df[\n",
    "    (local_xai_df[\"true_label\"] == 1) &\n",
    "    (local_xai_df[\"predicted_label\"] == 1)\n",
    "].copy()\n",
    "\n",
    "position_importance_df = (\n",
    "    all_anomaly_xai\n",
    "    .groupby(\"event_position\")\n",
    "    .agg(\n",
    "        mean_importance=(\"importance_drop\", \"mean\"),\n",
    "        median_importance=(\"importance_drop\", \"median\"),\n",
    "        max_importance=(\"importance_drop\", \"max\"),\n",
    "        frequency=(\"importance_drop\", \"count\")\n",
    "    )\n",
    "    .reset_index()\n",
    "    .sort_values(\"event_position\")\n",
    ")\n",
    "\n",
    "print(\"Explained anomaly sequences:\", all_anomaly_xai[\"sequence_local_test_index\"].nunique())\n",
    "display(position_importance_df)\n",
    "\n",
    "plt.figure(figsize=(10, 4))\n",
    "plt.bar(position_importance_df[\"event_position\"].astype(str), position_importance_df[\"mean_importance\"])\n",
    "plt.xlabel(\"Event position in 10-line sequence\")\n",
    "plt.ylabel(\"Mean importance drop\")\n",
    "plt.title(\"Average Event-Level Occlusion Importance Across Anomaly Sequences\")\n",
    "plt.tight_layout()\n",
    "save_current_figure(\"07_xai_mean_importance_by_event_position.png\")\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3e05fb79",
   "metadata": {},
   "source": [
    "## Step 48 — Strong local XAI case study chart\n",
    "\n",
    "This chart visualizes the strongest local case study selected by occlusion."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "83a88eda",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================================================\n",
    "# Step 48 — Strong local XAI case study chart\n",
    "# ============================================================\n",
    "\n",
    "if \"case_table\" in globals() and len(case_table) > 0:\n",
    "    strong_plot_df = case_table.sort_values(\"event_position\").copy()\n",
    "elif \"case_df\" in globals() and len(case_df) > 0:\n",
    "    strong_plot_df = case_df.sort_values(\"event_position\").copy()\n",
    "else:\n",
    "    raise ValueError(\"No case_table or case_df available. Run the strong XAI case study steps first.\")\n",
    "\n",
    "plt.figure(figsize=(10, 4))\n",
    "plt.bar(strong_plot_df[\"event_position\"].astype(str), strong_plot_df[\"importance_drop\"])\n",
    "plt.xlabel(\"Event position in 10-line sequence\")\n",
    "plt.ylabel(\"Importance drop\")\n",
    "plt.title(\"LogMiniLM Event-Level Occlusion Explanation\")\n",
    "plt.tight_layout()\n",
    "save_current_figure(\"08_xai_strong_case_study_occlusion.png\")\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f759395c",
   "metadata": {},
   "source": [
    "## Step 49 — Top-k faithfulness chart\n",
    "\n",
    "This chart shows whether masking more top-ranked events increases the probability drop."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f96743a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================================================\n",
    "# Step 49 — Top-k faithfulness chart\n",
    "# ============================================================\n",
    "\n",
    "if \"topk_faithfulness_df\" not in globals():\n",
    "    TOPK_FAITHFULNESS_PATH = CACHE_DIR / \"logminilm_xai_topk_faithfulness.csv\"\n",
    "    topk_faithfulness_df = pd.read_csv(TOPK_FAITHFULNESS_PATH)\n",
    "\n",
    "topk_summary_df = (\n",
    "    topk_faithfulness_df\n",
    "    .groupby(\"k\")\n",
    "    .agg(\n",
    "        mean_drop=(\"topk_probability_drop\", \"mean\"),\n",
    "        median_drop=(\"topk_probability_drop\", \"median\"),\n",
    "        max_drop=(\"topk_probability_drop\", \"max\")\n",
    "    )\n",
    "    .reset_index()\n",
    ")\n",
    "\n",
    "display(topk_summary_df)\n",
    "\n",
    "plt.figure(figsize=(7, 4))\n",
    "plt.plot(topk_summary_df[\"k\"], topk_summary_df[\"mean_drop\"], marker=\"o\", label=\"Mean drop\")\n",
    "plt.plot(topk_summary_df[\"k\"], topk_summary_df[\"median_drop\"], marker=\"o\", label=\"Median drop\")\n",
    "plt.xlabel(\"Number of top events masked\")\n",
    "plt.ylabel(\"Probability drop\")\n",
    "plt.title(\"Top-k Occlusion Faithfulness\")\n",
    "plt.legend()\n",
    "plt.tight_layout()\n",
    "save_current_figure(\"09_xai_topk_faithfulness.png\")\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e7d6274b",
   "metadata": {},
   "source": [
    "## Step 50 — SHAP versus occlusion agreement chart\n",
    "\n",
    "This chart summarizes how often SHAP and occlusion identify the same top event."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d6af317f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================================================\n",
    "# Step 50 — SHAP vs Occlusion agreement chart\n",
    "# ============================================================\n",
    "\n",
    "if \"shap_occ_compare_df\" not in globals():\n",
    "    SHAP_COMPARE_PATH = CACHE_DIR / \"logminilm_shap_occlusion_comparison.csv\"\n",
    "    shap_occ_compare_df = pd.read_csv(SHAP_COMPARE_PATH)\n",
    "\n",
    "agreement_counts = shap_occ_compare_df[\"same_top_event\"].value_counts().sort_index()\n",
    "\n",
    "agreement_df = pd.DataFrame({\n",
    "    \"agreement\": [\"Different top event\", \"Same top event\"],\n",
    "    \"count\": [\n",
    "        int(agreement_counts.get(0, 0)),\n",
    "        int(agreement_counts.get(1, 0))\n",
    "    ]\n",
    "})\n",
    "\n",
    "agreement_rate = shap_occ_compare_df[\"same_top_event\"].mean()\n",
    "\n",
    "display(agreement_df)\n",
    "print(\"Top-event agreement rate:\", agreement_rate)\n",
    "\n",
    "plt.figure(figsize=(6, 4))\n",
    "plt.bar(agreement_df[\"agreement\"], agreement_df[\"count\"])\n",
    "plt.xlabel(\"Agreement result\")\n",
    "plt.ylabel(\"Number of sequences\")\n",
    "plt.title(f\"SHAP vs Occlusion Top-Event Agreement ({agreement_rate:.2%})\")\n",
    "plt.tight_layout()\n",
    "save_current_figure(\"10_shap_occlusion_top_event_agreement.png\")\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c75b9bb3",
   "metadata": {},
   "source": [
    "## Step 51 — Learning curves for overfitting and underfitting check\n",
    "\n",
    "These charts answer the supervisor’s question about whether learning curves were used to check overfitting or underfitting.\n",
    "\n",
    "- If training loss decreases while validation loss increases, that suggests overfitting.\n",
    "- If both losses remain high, that suggests underfitting.\n",
    "- If validation F1 improves and stabilizes, that suggests useful generalization."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10aeed54",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================================================\n",
    "# Step 51 — Learning curves for overfitting and underfitting check\n",
    "# ============================================================\n",
    "\n",
    "if \"history_df\" not in globals():\n",
    "    LEARNING_HISTORY_PATH = CACHE_DIR / \"logminilm_learning_history.csv\"\n",
    "    if LEARNING_HISTORY_PATH.exists():\n",
    "        history_df = pd.read_csv(LEARNING_HISTORY_PATH)\n",
    "        print(\"Loaded learning history from:\", LEARNING_HISTORY_PATH)\n",
    "    else:\n",
    "        raise ValueError(\"history_df is not available and logminilm_learning_history.csv was not found.\")\n",
    "\n",
    "print(\"Learning history\")\n",
    "display(history_df)\n",
    "\n",
    "# Training vs validation loss curve\n",
    "plt.figure(figsize=(8, 4))\n",
    "plt.plot(history_df[\"epoch\"], history_df[\"train_loss\"], marker=\"o\", label=\"Training loss\")\n",
    "plt.plot(history_df[\"epoch\"], history_df[\"val_loss\"], marker=\"o\", label=\"Validation loss\")\n",
    "plt.xlabel(\"Epoch\")\n",
    "plt.ylabel(\"Loss\")\n",
    "plt.title(\"LogMiniLM Training and Validation Loss\")\n",
    "plt.legend()\n",
    "plt.tight_layout()\n",
    "save_current_figure(\"10_logminilm_training_validation_loss_curve.png\")\n",
    "plt.show()\n",
    "\n",
    "# Validation F1 curve\n",
    "plt.figure(figsize=(8, 4))\n",
    "plt.plot(history_df[\"epoch\"], history_df[\"val_f1\"], marker=\"o\", label=\"Validation F1-score\")\n",
    "plt.xlabel(\"Epoch\")\n",
    "plt.ylabel(\"F1-score\")\n",
    "plt.title(\"LogMiniLM Validation F1-score Across Epochs\")\n",
    "plt.legend()\n",
    "plt.tight_layout()\n",
    "save_current_figure(\"11_logminilm_validation_f1_curve.png\")\n",
    "plt.show()\n",
    "\n",
    "# Combined validation metrics curve\n",
    "plt.figure(figsize=(8, 4))\n",
    "plt.plot(history_df[\"epoch\"], history_df[\"val_accuracy\"], marker=\"o\", label=\"Validation accuracy\")\n",
    "plt.plot(history_df[\"epoch\"], history_df[\"val_precision\"], marker=\"o\", label=\"Validation precision\")\n",
    "plt.plot(history_df[\"epoch\"], history_df[\"val_recall\"], marker=\"o\", label=\"Validation recall\")\n",
    "plt.plot(history_df[\"epoch\"], history_df[\"val_f1\"], marker=\"o\", label=\"Validation F1\")\n",
    "plt.xlabel(\"Epoch\")\n",
    "plt.ylabel(\"Metric value\")\n",
    "plt.title(\"LogMiniLM Validation Metrics Across Epochs\")\n",
    "plt.legend()\n",
    "plt.tight_layout()\n",
    "save_current_figure(\"12_logminilm_validation_metrics_curve.png\")\n",
    "plt.show()\n",
    "\n",
    "print(\"Thesis note:\")\n",
    "print(\"Learning curves were used to check overfitting and underfitting by comparing training loss with validation loss and monitoring validation F1-score across epochs.\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6e666e76",
   "metadata": {},
   "source": [
    "## Step 52 — Figure manifest\n",
    "\n",
    "This step lists all saved chart files so they can be copied into Chapter 4."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4ffc2960",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================================================\n",
    "# Step 52 — Figure manifest\n",
    "# ============================================================\n",
    "\n",
    "figure_files = sorted(FIGURES_DIR.glob(\"*.png\"))\n",
    "\n",
    "figure_manifest_df = pd.DataFrame({\n",
    "    \"figure_file\": [str(p) for p in figure_files],\n",
    "    \"filename\": [p.name for p in figure_files]\n",
    "})\n",
    "\n",
    "FIGURE_MANIFEST_PATH = FIGURES_DIR / \"figure_manifest.csv\"\n",
    "figure_manifest_df.to_csv(FIGURE_MANIFEST_PATH, index=False)\n",
    "\n",
    "print(\"Saved figure manifest to:\", FIGURE_MANIFEST_PATH)\n",
    "display(figure_manifest_df)\n",
    "\n",
    "print(\"\\nRecommended Chapter 4 figure order:\")\n",
    "print(\"1. BGL line-level class distribution\")\n",
    "print(\"2. BGL sequence-level class distribution\")\n",
    "print(\"3. Threshold sensitivity\")\n",
    "print(\"4. Confusion matrix\")\n",
    "print(\"5. Prediction probability distribution\")\n",
    "print(\"6. Global XAI mean importance by event position\")\n",
    "print(\"7. Strong local XAI case study\")\n",
    "print(\"8. Top-k faithfulness\")\n",
    "print(\"9. SHAP vs occlusion agreement\")\n",
    "print(\"10. Training vs validation loss curve\")\n",
    "print(\"11. Validation F1 curve\")\n",
    "print(\"12. Validation metrics curve\")\n"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "gpuType": "T4",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  },
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}