{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "163c40e0-6c8b-440f-95cc-2e9caa1276b4", "metadata": {}, "outputs": [], "source": [ "# تم تنزيل البيانات من موقع كاجل :https://www.kaggle.com/datasets/losif01/mit-ai-news-dataset" ] }, { "cell_type": "code", "execution_count": 2, "id": "e102f609-7416-4113-8054-126f86d81c81", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Preview of the dataset:\n", " Unnamed: 0.1 Unnamed: 0 \\\n", "0 0 0 \n", "1 1 1 \n", "2 2 2 \n", "3 3 3 \n", "4 4 4 \n", "\n", " title author \\\n", "0 A new model predicts how molecules will dissol... Anne Trafton \n", "1 Researchers glimpse the inner workings of prot... Anne Trafton \n", "2 How AI could speed the development of RNA vacc... Anne Trafton \n", "3 Using generative AI, researchers design compou... Anne Trafton \n", "4 A new way to test how well AI systems classify... David Chandler \n", "\n", " publication_date datetime \\\n", "0 August 19, 2025 2025-08-19 09:00:00+00:00 \n", "1 August 18, 2025 2025-08-18 19:00:00+00:00 \n", "2 August 15, 2025 2025-08-15 09:00:00+00:00 \n", "3 August 14, 2025 2025-08-14 15:00:00+00:00 \n", "4 August 13, 2025 2025-08-13 19:00:00+00:00 \n", "\n", " summary \\\n", "0 Solubility predictions could make it easier to... \n", "1 A new approach can reveal the features AI mode... \n", "2 MIT engineers used a machine-learning model to... \n", "3 The team used two different AI approaches to d... \n", "4 As large language models increasingly dominate... \n", "\n", " body \\\n", "0 Using machine learning, MIT chemical engineers... \n", "1 Within the past few years, models that can pre... \n", "2 Using artificial intelligence, MIT researchers... \n", "3 With help from artificial intelligence, MIT re... \n", "4 Is this movie review a rave or a pan? Is this ... \n", "\n", " paper_link \\\n", "0 https://www.nature.com/articles/s41467-025-627... \n", "1 Not provided / no paper was made \n", "2 https://www.nature.com/articles/s41565-025-019... \n", "3 Not provided / no paper was made \n", "4 Not provided / no paper was made \n", "\n", " url \n", "0 https://news.mit.edu/2025/new-model-predicts-h... \n", "1 https://news.mit.edu/2025/researchers-glimpse-... \n", "2 https://news.mit.edu/2025/how-ai-could-speed-d... \n", "3 https://news.mit.edu/2025/using-generative-ai-... \n", "4 https://news.mit.edu/2025/new-way-test-how-wel... \n", "\n", "Shape of dataset: (314, 10)\n", "\n", "Column info:\n", "\n", "RangeIndex: 314 entries, 0 to 313\n", "Data columns (total 10 columns):\n", " # Column Non-Null Count Dtype \n", "--- ------ -------------- ----- \n", " 0 Unnamed: 0.1 314 non-null int64 \n", " 1 Unnamed: 0 314 non-null int64 \n", " 2 title 314 non-null object\n", " 3 author 314 non-null object\n", " 4 publication_date 314 non-null object\n", " 5 datetime 314 non-null object\n", " 6 summary 314 non-null object\n", " 7 body 314 non-null object\n", " 8 paper_link 314 non-null object\n", " 9 url 314 non-null object\n", "dtypes: int64(2), object(8)\n", "memory usage: 24.7+ KB\n", "None\n", "\n", "Missing values per column:\n", "Unnamed: 0.1 0\n", "Unnamed: 0 0\n", "title 0\n", "author 0\n", "publication_date 0\n", "datetime 0\n", "summary 0\n", "body 0\n", "paper_link 0\n", "url 0\n", "dtype: int64\n", "\n", "Text columns analysis:\n", "Column: title\n", " - Non-null: 314\n", " - Unique values: 314\n", " - Avg length: 61.32\n", " - Max length: 120\n", "----------------------------------------\n", "Column: author\n", " - Non-null: 314\n", " - Unique values: 74\n", " - Avg length: 11.65\n", " - Max length: 24\n", "----------------------------------------\n", "Column: publication_date\n", " - Non-null: 314\n", " - Unique values: 215\n", " - Avg length: 14.46\n", " - Max length: 18\n", "----------------------------------------\n", "Column: datetime\n", " - Non-null: 314\n", " - Unique values: 302\n", " - Avg length: 25.00\n", " - Max length: 25\n", "----------------------------------------\n", "Column: summary\n", " - Non-null: 314\n", " - Unique values: 314\n", " - Avg length: 140.50\n", " - Max length: 211\n", "----------------------------------------\n", "Column: body\n", " - Non-null: 314\n", " - Unique values: 314\n", " - Avg length: 6802.67\n", " - Max length: 15656\n", "----------------------------------------\n", "Column: paper_link\n", " - Non-null: 314\n", " - Unique values: 37\n", " - Avg length: 34.06\n", " - Max length: 50\n", "----------------------------------------\n", "Column: url\n", " - Non-null: 314\n", " - Unique values: 314\n", " - Avg length: 83.99\n", " - Max length: 123\n", "----------------------------------------\n" ] } ], "source": [ "import pandas as pd\n", "import numpy as np\n", "\n", "df = pd.read_csv(\"MIT_AI_ARTICLES.csv\") \n", "\n", "print(\"Preview of the dataset:\")\n", "print(df.head())\n", "\n", "print(\"\\nShape of dataset:\", df.shape)\n", "print(\"\\nColumn info:\")\n", "print(df.info())\n", "\n", "print(\"\\nMissing values per column:\")\n", "print(df.isnull().sum())\n", "\n", "text_cols = df.select_dtypes(include=[\"object\"]).columns.tolist()\n", "if text_cols:\n", " print(\"\\nText columns analysis:\")\n", " for col in text_cols:\n", " lengths = df[col].astype(str).map(len)\n", " print(f\"Column: {col}\")\n", " print(f\" - Non-null: {df[col].notnull().sum()}\")\n", " print(f\" - Unique values: {df[col].nunique(dropna=True)}\")\n", " print(f\" - Avg length: {lengths.mean():.2f}\")\n", " print(f\" - Max length: {lengths.max()}\")\n", " print(\"-\"*40)" ] }, { "cell_type": "code", "execution_count": 4, "id": "fe484739-accf-4c14-97e6-899fd915e02f", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " text \\\n", "0 A new model predicts how molecules will dissol... \n", "1 Researchers glimpse the inner workings of prot... \n", "2 How AI could speed the development of RNA vacc... \n", "3 Using generative AI, researchers design compou... \n", "4 A new way to test how well AI systems classify... \n", "\n", " clean_text \n", "0 new model predicts molecule dissolve different... \n", "1 researcher glimpse inner working protein langu... \n", "2 ai could speed development rna vaccine rna the... \n", "3 using generative ai researcher design compound... \n", "4 new way test well ai system classify text larg... \n" ] } ], "source": [ "import pandas as pd\n", "import re\n", "import nltk\n", "from nltk.corpus import stopwords\n", "from nltk.stem import WordNetLemmatizer\n", "\n", "\n", "def ensure_nltk_resource(path, pkg):\n", " try:\n", " nltk.data.find(path)\n", " except LookupError:\n", " nltk.download(pkg, quiet=True)\n", "\n", "ensure_nltk_resource('corpora/stopwords', 'stopwords')\n", "ensure_nltk_resource('corpora/wordnet', 'wordnet')\n", "ensure_nltk_resource('corpora/omw-1.4', 'omw-1.4')\n", "\n", "df = pd.read_csv(\"MIT_AI_ARTICLES.csv\")\n", "df['text'] = df['title'].astype(str) + \" \" + df['summary'].astype(str) + \" \" + df['body'].astype(str)\n", "\n", "stop_words = set(stopwords.words('english'))\n", "lemmatizer = WordNetLemmatizer()\n", "\n", "def clean_text(text):\n", " text = text.lower()\n", " text = re.sub(r'[^a-z\\s]', '', text)\n", " tokens = text.split()\n", " tokens = [lemmatizer.lemmatize(word) for word in tokens if word not in stop_words]\n", " return \" \".join(tokens)\n", "\n", "df['clean_text'] = df['text'].apply(clean_text)\n", "print(df[['text', 'clean_text']].head())" ] }, { "cell_type": "code", "execution_count": 5, "id": "65b5b284-27cf-454a-9b8c-d17c35b636c9", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "TF-IDF shape: (314, 11695)\n", "Silhouette per k: {3: 0.011638482752834642, 4: 0.01255932039741692, 5: 0.011645055168493108, 6: 0.011670092732404266, 8: 0.012032775142610766, 10: 0.014240090482628297}\n", "Best k by silhouette: 10\n", "\n", "=== Cluster 0 — Top 15 terms ===\n", "agent, algorithm, task, privacy, ai agent, noise, ai, training, attacker, wu, adversarial, human, maes, traffic, reinforcement learning\n", "\n", "=== Cluster 1 — Top 15 terms ===\n", "model, llm, language, language model, researcher, image, task, data, ai, token, like, could, human, user, system\n", "\n", "=== Cluster 2 — Top 15 terms ===\n", "college, computing, science, people, schwarzman, technology, work, health, story, housing, brain, award, college computing, economics, schwarzman college\n", "\n", "=== Cluster 3 — Top 15 terms ===\n", "ai, health, patient, technology, care, model, cancer, human, people, health care, medical, system, industry, cell, data\n", "\n", "=== Cluster 4 — Top 15 terms ===\n", "robot, task, model, environment, diffusion, action, robotics, robotic, data, csail, researcher, policy, object, warehouse, control\n", "\n", "=== Cluster 5 — Top 15 terms ===\n", "protein, material, model, cell, researcher, structure, image, could, sequence, dna, buehler, method, transistor, ai, graph\n", "\n", "=== Cluster 6 — Top 15 terms ===\n", "data, algorithm, model, symmetry, system, image, network, could, researcher, problem, chip, ai, learning, operation, drone\n", "\n", "=== Cluster 7 — Top 15 terms ===\n", "music, artist, art, music technology, college, program, musical, computing, ai, egozy, musician, college computing, medium, student, schwarzman college\n", "\n", "=== Cluster 8 — Top 15 terms ===\n", "molecule, drug, model, chemical, prediction, molecular, compound, antibiotic, reaction, structure, researcher, property, atom, bacteria, material\n", "\n", "=== Cluster 9 — Top 15 terms ===\n", "energy, design, ai, data, manufacturing, technology, learning, science, climate, material, student, engineering, data center, project, program\n" ] } ], "source": [ "import pandas as pd\n", "import numpy as np\n", "\n", "texts = df['clean_text'].astype(str).values\n", "\n", "from sklearn.feature_extraction.text import TfidfVectorizer\n", "\n", "tfidf = TfidfVectorizer(\n", " max_features=50000, \n", " ngram_range=(1,2), \n", " min_df=3, \n", " max_df=0.9 \n", ")\n", "\n", "X = tfidf.fit_transform(texts)\n", "print(\"TF-IDF shape:\", X.shape) \n", "\n", "\n", "from sklearn.cluster import KMeans\n", "from sklearn.metrics import silhouette_score\n", "\n", "candidate_k = [3, 4, 5, 6, 8, 10]\n", "sil_scores = {}\n", "\n", "for k in candidate_k:\n", " kmeans_tmp = KMeans(n_clusters=k, n_init=\"auto\", random_state=42)\n", " labels_tmp = kmeans_tmp.fit_predict(X)\n", " sil = silhouette_score(X, labels_tmp)\n", " sil_scores[k] = sil\n", "\n", "best_k = max(sil_scores, key=sil_scores.get)\n", "print(\"Silhouette per k:\", sil_scores)\n", "print(\"Best k by silhouette:\", best_k)\n", "\n", "kmeans = KMeans(n_clusters=best_k, n_init=\"auto\", random_state=42)\n", "cluster_labels = kmeans.fit_predict(X)\n", "\n", "def top_terms_per_cluster(model, vectorizer, top_n=15):\n", " terms = np.array(vectorizer.get_feature_names_out())\n", " centers = model.cluster_centers_\n", " for c in range(model.n_clusters):\n", " top_idx = np.argsort(centers[c])[::-1][:top_n]\n", " print(f\"\\n=== Cluster {c} — Top {top_n} terms ===\")\n", " print(\", \".join(terms[top_idx]))\n", "\n", "from scipy.sparse import csr_matrix\n", "\n", "def approx_cluster_centers(X, labels, n_clusters):\n", " centers = []\n", " for c in range(n_clusters):\n", " rows = X[labels == c]\n", " if hasattr(rows, \"toarray\"):\n", " mean_vec = np.asarray(rows.mean(axis=0)).ravel()\n", " else:\n", " mean_vec = rows.mean(axis=0)\n", " centers.append(mean_vec)\n", " return np.vstack(centers)\n", "\n", "approx_centers = approx_cluster_centers(X, cluster_labels, best_k)\n", "\n", "terms = np.array(tfidf.get_feature_names_out())\n", "for c in range(best_k):\n", " top_idx = np.argsort(approx_centers[c])[::-1][:15]\n", " print(f\"\\n=== Cluster {c} — Top 15 terms ===\")\n", " print(\", \".join(terms[top_idx]))\n", "\n", "df['cluster'] = cluster_labels\n", "\n", "TARGET_COL = None \n", "\n", "if TARGET_COL is not None and TARGET_COL in df.columns:\n", " y = df[TARGET_COL].astype(str).values\n", "\n", " from sklearn.model_selection import train_test_split\n", " X_train, X_test, y_train, y_test = train_test_split(\n", " X, y, test_size=0.2, random_state=42, stratify=y\n", " )\n", "\n", " from sklearn.linear_model import LogisticRegression\n", " from sklearn.naive_bayes import MultinomialNB\n", " from sklearn.metrics import classification_report, accuracy_score, f1_score\n", "\n", " models = {\n", " \"LogReg\": LogisticRegression(max_iter=200, n_jobs=None), \n", " \"MultinomialNB\": MultinomialNB()\n", " }\n", "\n", " for name, clf in models.items():\n", " clf.fit(X_train, y_train)\n", " preds = clf.predict(X_test)\n", " acc = accuracy_score(y_test, preds)\n", " f1 = f1_score(y_test, preds, average=\"weighted\")\n", " print(f\"\\n=== {name} ===\")\n", " print(f\"Accuracy: {acc:.4f} | F1 (weighted): {f1:.4f}\")\n", " print(classification_report(y_test, preds)[:1200]) \n" ] }, { "cell_type": "code", "execution_count": 6, "id": "5a1a752d-7085-4958-a2e4-b33834a16be3", "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np\n", "\n", "from sklearn.model_selection import train_test_split\n", "from sklearn.feature_extraction.text import TfidfVectorizer\n", "from sklearn.linear_model import LogisticRegression\n", "from sklearn.svm import LinearSVC\n", "from sklearn.naive_bayes import MultinomialNB\n", "from sklearn.metrics import (\n", " accuracy_score, precision_recall_fscore_support,\n", " classification_report, confusion_matrix\n", ")\n", "import joblib\n", "\n", "assert \"clean_text\" in df.columns, \"clean_text column is missing — run the preprocessing cell first.\"" ] }, { "cell_type": "code", "execution_count": 7, "id": "f1402f24-28e6-4061-9a42-befa0da33764", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Classes: target\n", "Other 109\n", "Adam Zewe 74\n", "Alex Shipps 24\n", "Zach Winn 23\n", "Rachel Gordon 20\n", "Unknown 18\n", "Anne Trafton 14\n", "Peter Dizikes 9\n", "Jennifer Chu 9\n", "Michaela Jarvis 7\n", "Lauren Hinkel 7\n", "Name: count, dtype: int64\n" ] } ], "source": [ "author_counts = df[\"author\"].value_counts()\n", "top_authors = author_counts.head(10).index.tolist()\n", "\n", "def map_author(a):\n", " return a if a in top_authors else \"Other\"\n", "\n", "df[\"target\"] = df[\"author\"].astype(str).map(map_author)\n", "\n", "print(\"Classes:\", df[\"target\"].value_counts())" ] }, { "cell_type": "code", "execution_count": 8, "id": "29a871f0-439f-4fe3-a514-c484a1cb0a6d", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Train size: 251 Test size: 63\n" ] } ], "source": [ "X_text = df[\"clean_text\"].astype(str).values\n", "y = df[\"target\"].astype(str).values\n", "\n", "X_train_text, X_test_text, y_train, y_test = train_test_split(\n", " X_text, y, test_size=0.2, random_state=42, stratify=y\n", ")\n", "\n", "print(\"Train size:\", len(X_train_text), \"Test size:\", len(X_test_text))" ] }, { "cell_type": "code", "execution_count": 9, "id": "3a284d59-adbf-4836-bbf1-c543eae90f0f", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "TF-IDF shape (train): (251, 8590)\n" ] } ], "source": [ "tfidf = TfidfVectorizer(\n", " max_features=50000, \n", " ngram_range=(1,2),\n", " min_df=3,\n", " max_df=0.9,\n", " stop_words=\"english\",\n", " sublinear_tf=True\n", ")\n", "\n", "X_train = tfidf.fit_transform(X_train_text)\n", "X_test = tfidf.transform(X_test_text)\n", "\n", "print(\"TF-IDF shape (train):\", X_train.shape)\n" ] }, { "cell_type": "code", "execution_count": 10, "id": "a622147d-5609-42f5-ba6c-bf082d450ccf", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "=== LinearSVC ===\n", "Accuracy : 0.7778\n", "F1 (macro) : 0.5173\n", "F1 (weighted) : 0.7302\n", "\n", "Classification Report (truncated):\n", " precision recall f1-score support\n", "\n", " Adam Zewe 0.79 1.00 0.88 15\n", " Alex Shipps 0.83 1.00 0.91 5\n", " Anne Trafton 1.00 0.67 0.80 3\n", " Jennifer Chu 0.00 0.00 0.00 2\n", " Lauren Hinkel 0.00 0.00 0.00 1\n", "Michaela Jarvis 0.00 0.00 0.00 1\n", " Other 0.70 0.86 0.78 22\n", " Peter Dizikes 1.00 0.50 0.67 2\n", " Rachel Gordon 1.00 0.75 0.86 4\n", " Unknown 0.00 0.00 0.00 3\n", " Zach Winn 0.80 0.80 0.80 5\n", "\n", " accuracy 0.78 63\n", " macro avg 0.56 0.51 0.52 63\n", " weighted avg 0.71 0.78 0.73 63\n", "\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "C:\\Users\\Youssef Saad\\.conda\\envs\\Project-NLP\\Lib\\site-packages\\sklearn\\metrics\\_classification.py:1731: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", " _warn_prf(average, modifier, f\"{metric.capitalize()} is\", result.shape[0])\n", "C:\\Users\\Youssef Saad\\.conda\\envs\\Project-NLP\\Lib\\site-packages\\sklearn\\metrics\\_classification.py:1731: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", " _warn_prf(average, modifier, f\"{metric.capitalize()} is\", result.shape[0])\n", "C:\\Users\\Youssef Saad\\.conda\\envs\\Project-NLP\\Lib\\site-packages\\sklearn\\metrics\\_classification.py:1731: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", " _warn_prf(average, modifier, f\"{metric.capitalize()} is\", result.shape[0])\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "=== LogReg ===\n", "Accuracy : 0.7619\n", "F1 (macro) : 0.6102\n", "F1 (weighted) : 0.7410\n", "\n", "Classification Report (truncated):\n", " precision recall f1-score support\n", "\n", " Adam Zewe 0.88 0.93 0.90 15\n", " Alex Shipps 0.83 1.00 0.91 5\n", " Anne Trafton 0.67 0.67 0.67 3\n", " Jennifer Chu 1.00 0.50 0.67 2\n", " Lauren Hinkel 0.50 1.00 0.67 1\n", "Michaela Jarvis 0.00 0.00 0.00 1\n", " Other 0.74 0.77 0.76 22\n", " Peter Dizikes 1.00 0.50 0.67 2\n", " Rachel Gordon 0.75 0.75 0.75 4\n", " Unknown 0.00 0.00 0.00 3\n", " Zach Winn 0.67 0.80 0.73 5\n", "\n", " accuracy 0.76 63\n", " macro avg 0.64 0.63 0.61 63\n", " weighted avg 0.74 0.76 0.74 63\n", "\n", "\n", "=== MultinomialNB ===\n", "Accuracy : 0.5714\n", "F1 (macro) : 0.1308\n", "F1 (weighted) : 0.4229\n", "\n", "Classification Report (truncated):\n", " precision recall f1-score support\n", "\n", " Adam Zewe 0.56 1.00 0.71 15\n", " Alex Shipps 0.00 0.00 0.00 5\n", " Anne Trafton 0.00 0.00 0.00 3\n", " Jennifer Chu 0.00 0.00 0.00 2\n", " Lauren Hinkel 0.00 0.00 0.00 1\n", "Michaela Jarvis 0.00 0.00 0.00 1\n", " Other 0.58 0.95 0.72 22\n", " Peter Dizikes 0.00 0.00 0.00 2\n", " Rachel Gordon 0.00 0.00 0.00 4\n", " Unknown 0.00 0.00 0.00 3\n", " Zach Winn 0.00 0.00 0.00 5\n", "\n", " accuracy 0.57 63\n", " macro avg 0.10 0.18 0.13 63\n", " weighted avg 0.34 0.57 0.42 63\n", "\n", "\n", "=== Summary ===\n", " model accuracy f1_macro f1_weighted precision_macro \\\n", "1 LogReg 0.761905 0.610165 0.741044 0.639163 \n", "0 LinearSVC 0.777778 0.517342 0.730220 0.556956 \n", "2 MultinomialNB 0.571429 0.130766 0.422942 0.103535 \n", "\n", " recall_macro precision_weighted recall_weighted \n", "1 0.629339 0.736284 0.761905 \n", "0 0.507300 0.706194 0.777778 \n", "2 0.177686 0.335979 0.571429 \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "C:\\Users\\Youssef Saad\\.conda\\envs\\Project-NLP\\Lib\\site-packages\\sklearn\\linear_model\\_sag.py:348: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n", " warnings.warn(\n", "C:\\Users\\Youssef Saad\\.conda\\envs\\Project-NLP\\Lib\\site-packages\\sklearn\\metrics\\_classification.py:1731: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", " _warn_prf(average, modifier, f\"{metric.capitalize()} is\", result.shape[0])\n", "C:\\Users\\Youssef Saad\\.conda\\envs\\Project-NLP\\Lib\\site-packages\\sklearn\\metrics\\_classification.py:1731: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", " _warn_prf(average, modifier, f\"{metric.capitalize()} is\", result.shape[0])\n", "C:\\Users\\Youssef Saad\\.conda\\envs\\Project-NLP\\Lib\\site-packages\\sklearn\\metrics\\_classification.py:1731: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", " _warn_prf(average, modifier, f\"{metric.capitalize()} is\", result.shape[0])\n", "C:\\Users\\Youssef Saad\\.conda\\envs\\Project-NLP\\Lib\\site-packages\\sklearn\\metrics\\_classification.py:1731: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", " _warn_prf(average, modifier, f\"{metric.capitalize()} is\", result.shape[0])\n", "C:\\Users\\Youssef Saad\\.conda\\envs\\Project-NLP\\Lib\\site-packages\\sklearn\\metrics\\_classification.py:1731: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", " _warn_prf(average, modifier, f\"{metric.capitalize()} is\", result.shape[0])\n", "C:\\Users\\Youssef Saad\\.conda\\envs\\Project-NLP\\Lib\\site-packages\\sklearn\\metrics\\_classification.py:1731: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", " _warn_prf(average, modifier, f\"{metric.capitalize()} is\", result.shape[0])\n" ] } ], "source": [ "models = {\n", " \"LinearSVC\": LinearSVC(class_weight=\"balanced\", random_state=42),\n", " \"LogReg\": LogisticRegression(solver=\"saga\", max_iter=2000, class_weight=\"balanced\", random_state=42),\n", " \"MultinomialNB\": MultinomialNB(alpha=0.5)\n", "}\n", "\n", "rows = []\n", "trained = {}\n", "\n", "for name, clf in models.items():\n", " clf.fit(X_train, y_train)\n", " y_pred = clf.predict(X_test)\n", "\n", " acc = accuracy_score(y_test, y_pred)\n", " p_mac, r_mac, f1_mac, _ = precision_recall_fscore_support(y_test, y_pred, average=\"macro\", zero_division=0)\n", " p_w, r_w, f1_w, _ = precision_recall_fscore_support(y_test, y_pred, average=\"weighted\", zero_division=0)\n", "\n", " print(f\"\\n=== {name} ===\")\n", " print(f\"Accuracy : {acc:.4f}\")\n", " print(f\"F1 (macro) : {f1_mac:.4f}\")\n", " print(f\"F1 (weighted) : {f1_w:.4f}\")\n", " print(\"\\nClassification Report (truncated):\")\n", " print(classification_report(y_test, y_pred)[:1500])\n", "\n", " rows.append({\n", " \"model\": name,\n", " \"accuracy\": acc,\n", " \"f1_macro\": f1_mac,\n", " \"f1_weighted\": f1_w,\n", " \"precision_macro\": p_mac,\n", " \"recall_macro\": r_mac,\n", " \"precision_weighted\": p_w,\n", " \"recall_weighted\": r_w\n", " })\n", " trained[name] = clf\n", "\n", "results_df = pd.DataFrame(rows).sort_values(\"f1_weighted\", ascending=False)\n", "print(\"\\n=== Summary ===\")\n", "print(results_df)" ] }, { "cell_type": "code", "execution_count": 11, "id": "c07af8b9-2a46-4bc6-b05e-9cd5d0dab5bf", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "Saved artifacts: tfidf_vectorizer.joblib, *.joblib, model_results.csv, test_predictions.csv\n", "Done.\n" ] } ], "source": [ "joblib.dump(tfidf, \"tfidf_vectorizer.joblib\")\n", "for name, model in trained.items():\n", " joblib.dump(model, f\"{name}.joblib\")\n", "\n", "results_df.to_csv(\"model_results.csv\", index=False)\n", "\n", "best_name = results_df.iloc[0][\"model\"]\n", "best_model = trained[best_name]\n", "y_pred_best = best_model.predict(X_test)\n", "\n", "pd.DataFrame({\n", " \"text\": X_test_text,\n", " \"true\": y_test,\n", " \"pred\": y_pred_best\n", "}).to_csv(\"test_predictions.csv\", index=False)\n", "\n", "print(\"\\nSaved artifacts: tfidf_vectorizer.joblib, *.joblib, model_results.csv, test_predictions.csv\")\n", "print(\"Done.\")" ] }, { "cell_type": "code", "execution_count": null, "id": "53f0f8bd-1d8f-41c9-84fb-5d6ced881bff", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.13.5" } }, "nbformat": 4, "nbformat_minor": 5 }