{ "cells": [ { "cell_type": "code", "execution_count": 2, "id": "14144f9b-1f1b-4165-98a3-2f4acef8b874", "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np\n", "from sklearn.model_selection import train_test_split, GridSearchCV\n", "from sklearn.feature_selection import SelectKBest, chi2, RFE\n", "from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier\n", "from sklearn.tree import DecisionTreeClassifier\n", "from sklearn.svm import SVC\n", "from sklearn.naive_bayes import GaussianNB\n", "from sklearn.neighbors import KNeighborsClassifier\n", "from sklearn.metrics import (\n", " accuracy_score, precision_score, recall_score, f1_score,\n", " mean_absolute_error, mean_squared_error, r2_score\n", ")" ] }, { "cell_type": "code", "execution_count": 4, "id": "7111b7a5-d03d-4a85-831d-45e2cebce4ba", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
PregnanciesGlucoseBloodPressureSkinThicknessInsulinBMIDiabetesPedigreeFunctionAgeOutcome
061487235033.60.627501
11856629026.60.351310
28183640023.30.672321
318966239428.10.167210
40137403516843.12.288331
\n", "
" ], "text/plain": [ " Pregnancies Glucose BloodPressure SkinThickness Insulin BMI \\\n", "0 6 148 72 35 0 33.6 \n", "1 1 85 66 29 0 26.6 \n", "2 8 183 64 0 0 23.3 \n", "3 1 89 66 23 94 28.1 \n", "4 0 137 40 35 168 43.1 \n", "\n", " DiabetesPedigreeFunction Age Outcome \n", "0 0.627 50 1 \n", "1 0.351 31 0 \n", "2 0.672 32 1 \n", "3 0.167 21 0 \n", "4 2.288 33 1 " ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# 1. Read the dataset\n", "data = pd.read_csv('diabetes.csv')\n", "data.head()" ] }, { "cell_type": "code", "execution_count": 6, "id": "7ecbdff9-f4fa-4f73-a52a-4987eb96a57e", "metadata": {}, "outputs": [], "source": [ "# 2. Handle missing values\n", "data.fillna(data.mean(), inplace=True)" ] }, { "cell_type": "code", "execution_count": 8, "id": "6907ed7e-97d0-4632-a001-62395d7d4de1", "metadata": {}, "outputs": [], "source": [ "# 3. Prepare X and y\n", "X = data.drop('Outcome', axis=1)\n", "y = data['Outcome']" ] }, { "cell_type": "code", "execution_count": 10, "id": "3e158332-2ef5-4508-a583-9dd452b338a1", "metadata": {}, "outputs": [], "source": [ "# 4. Split dataset into train and test sets\n", "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)" ] }, { "cell_type": "code", "execution_count": 12, "id": "15367f61-d376-4c9e-b7f6-631f0df54c0f", "metadata": {}, "outputs": [], "source": [ "# 5. Define evaluation function\n", "def evaluate_model(model_name, y_true, y_pred):\n", " accuracy = accuracy_score(y_true, y_pred)\n", " precision = precision_score(y_true, y_pred, average='weighted')\n", " recall = recall_score(y_true, y_pred, average='weighted')\n", " f1 = f1_score(y_true, y_pred, average='weighted')\n", " mae = mean_absolute_error(y_true, y_pred)\n", " mse = mean_squared_error(y_true, y_pred)\n", " rmse = np.sqrt(mse)\n", " r2 = r2_score(y_true, y_pred)\n", "\n", " metrics = {\n", " 'Model Name': model_name,\n", " 'Accuracy': accuracy,\n", " 'Precision': precision,\n", " 'Recall': recall,\n", " 'F1 Score': f1,\n", " 'MAE': mae,\n", " 'MSE': mse,\n", " 'RMSE': rmse,\n", " 'R2 Score': r2\n", " }\n", " return metrics" ] }, { "cell_type": "code", "execution_count": 14, "id": "86038913-298b-4770-a095-833806afb7a8", "metadata": {}, "outputs": [], "source": [ "# 6. Define all models and their hyperparameter grids\n", "models = {\n", " 'Random Forest': (RandomForestClassifier(random_state=42), {\n", " 'n_estimators': [100, 200],\n", " 'max_depth': [None, 10, 20],\n", " 'min_samples_split': [2, 5]\n", " }),\n", " 'Decision Tree': (DecisionTreeClassifier(random_state=42), {\n", " 'max_depth': [None, 10, 20],\n", " 'min_samples_split': [2, 5, 10]\n", " }),\n", " 'SVC': (SVC(probability=True, random_state=42), {\n", " 'C': [0.1, 1, 10],\n", " 'kernel': ['linear', 'rbf']\n", " }),\n", " 'Naive Bayes': (GaussianNB(), None),\n", " 'KNN': (KNeighborsClassifier(), {\n", " 'n_neighbors': [3, 5, 7],\n", " 'weights': ['uniform', 'distance']\n", " }),\n", " 'Gradient Boosting': (GradientBoostingClassifier(random_state=42), {\n", " 'n_estimators': [100, 200, 300],\n", " 'learning_rate': [0.01, 0.05, 0.1],\n", " 'max_depth': [3, 5, 7],\n", " 'min_samples_split': [2, 5, 10]\n", " })\n", "}\n" ] }, { "cell_type": "code", "execution_count": 16, "id": "93cba31a-d421-4f84-a2bd-9bb745aa28a0", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "=== Results using FULL Features ===\n", "Best parameters for Random Forest: {'max_depth': None, 'min_samples_split': 5, 'n_estimators': 100}\n", "Model Name: Random Forest (Full Features)\n", "Accuracy: 0.7338\n", "Precision: 0.7373\n", "Recall: 0.7338\n", "F1 Score: 0.7353\n", "MAE: 0.2662\n", "MSE: 0.2662\n", "RMSE: 0.5160\n", "R2 Score: -0.1596\n", "----------------------------------------\n", "Best parameters for Decision Tree: {'max_depth': None, 'min_samples_split': 5}\n", "Model Name: Decision Tree (Full Features)\n", "Accuracy: 0.7597\n", "Precision: 0.7685\n", "Recall: 0.7597\n", "F1 Score: 0.7625\n", "MAE: 0.2403\n", "MSE: 0.2403\n", "RMSE: 0.4902\n", "R2 Score: -0.0465\n", "----------------------------------------\n", "Best parameters for SVC: {'C': 0.1, 'kernel': 'linear'}\n", "Model Name: SVC (Full Features)\n", "Accuracy: 0.7532\n", "Precision: 0.7532\n", "Recall: 0.7532\n", "F1 Score: 0.7532\n", "MAE: 0.2468\n", "MSE: 0.2468\n", "RMSE: 0.4967\n", "R2 Score: -0.0747\n", "----------------------------------------\n", "Model Name: Naive Bayes (Full Features)\n", "Accuracy: 0.7662\n", "Precision: 0.7707\n", "Recall: 0.7662\n", "F1 Score: 0.7679\n", "MAE: 0.2338\n", "MSE: 0.2338\n", "RMSE: 0.4835\n", "R2 Score: -0.0182\n", "----------------------------------------\n", "Best parameters for KNN: {'n_neighbors': 7, 'weights': 'distance'}\n", "Model Name: KNN (Full Features)\n", "Accuracy: 0.6818\n", "Precision: 0.6859\n", "Recall: 0.6818\n", "F1 Score: 0.6836\n", "MAE: 0.3182\n", "MSE: 0.3182\n", "RMSE: 0.5641\n", "R2 Score: -0.3859\n", "----------------------------------------\n", "Best parameters for Gradient Boosting: {'learning_rate': 0.01, 'max_depth': 3, 'min_samples_split': 10, 'n_estimators': 300}\n", "Model Name: Gradient Boosting (Full Features)\n", "Accuracy: 0.7727\n", "Precision: 0.7704\n", "Recall: 0.7727\n", "F1 Score: 0.7712\n", "MAE: 0.2273\n", "MSE: 0.2273\n", "RMSE: 0.4767\n", "R2 Score: 0.0101\n", "----------------------------------------\n" ] } ], "source": [ "# 7. Train and evaluate models on FULL features\n", "print(\"\\n=== Results using FULL Features ===\")\n", "for model_name, (model, param_grid) in models.items():\n", " if param_grid:\n", " grid_search = GridSearchCV(model, param_grid, cv=5, scoring='accuracy', n_jobs=-1)\n", " grid_search.fit(X_train, y_train)\n", " model = grid_search.best_estimator_\n", " print(f\"Best parameters for {model_name}: {grid_search.best_params_}\")\n", " else:\n", " model.fit(X_train, y_train)\n", " y_pred = model.predict(X_test)\n", " results = evaluate_model(f\"{model_name} (Full Features)\", y_test, y_pred)\n", " for key, value in results.items():\n", " print(f\"{key}: {value:.4f}\" if isinstance(value, float) else f\"{key}: {value}\")\n", " print(\"-\"*40)" ] }, { "cell_type": "code", "execution_count": 18, "id": "5fe39d9c-6a4b-4089-b305-44ce23de4f45", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "=== Feature Selection using Filter Method (SelectKBest) ===\n", " Feature Score\n", "4 Insulin 1197.140821\n", "1 Glucose 1082.927430\n", "7 Age 194.164018\n", "5 BMI 107.766125\n", "0 Pregnancies 77.452968\n", "3 SkinThickness 24.007290\n", "2 BloodPressure 20.372904\n", "6 DiabetesPedigreeFunction 3.541524\n" ] } ], "source": [ "# 8. Feature Selection: Filter Method (SelectKBest)\n", "select_feature = SelectKBest(chi2, k=5).fit(X_train, y_train)\n", "X_train_selected = select_feature.transform(X_train)\n", "X_test_selected = select_feature.transform(X_test)\n", "\n", "print(\"\\n=== Feature Selection using Filter Method (SelectKBest) ===\")\n", "print(pd.DataFrame({'Feature': X.columns, 'Score': select_feature.scores_}).sort_values(by='Score', ascending=False))" ] }, { "cell_type": "code", "execution_count": 20, "id": "caf01c22-cdef-4a4c-b82d-d7f6a872b45b", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "=== Results using Filter Method (SelectKBest) ===\n", "Best parameters for Random Forest: {'max_depth': 10, 'min_samples_split': 5, 'n_estimators': 100}\n", "Model Name: Random Forest (Filter Features)\n", "Accuracy: 0.7597\n", "Precision: 0.7630\n", "Recall: 0.7597\n", "F1 Score: 0.7611\n", "MAE: 0.2403\n", "MSE: 0.2403\n", "RMSE: 0.4902\n", "R2 Score: -0.0465\n", "----------------------------------------\n", "Best parameters for Decision Tree: {'max_depth': None, 'min_samples_split': 10}\n", "Model Name: Decision Tree (Filter Features)\n", "Accuracy: 0.6753\n", "Precision: 0.6809\n", "Recall: 0.6753\n", "F1 Score: 0.6777\n", "MAE: 0.3247\n", "MSE: 0.3247\n", "RMSE: 0.5698\n", "R2 Score: -0.4141\n", "----------------------------------------\n", "Best parameters for SVC: {'C': 10, 'kernel': 'rbf'}\n", "Model Name: SVC (Filter Features)\n", "Accuracy: 0.7597\n", "Precision: 0.7559\n", "Recall: 0.7597\n", "F1 Score: 0.7570\n", "MAE: 0.2403\n", "MSE: 0.2403\n", "RMSE: 0.4902\n", "R2 Score: -0.0465\n", "----------------------------------------\n", "Model Name: Naive Bayes (Filter Features)\n", "Accuracy: 0.7597\n", "Precision: 0.7630\n", "Recall: 0.7597\n", "F1 Score: 0.7611\n", "MAE: 0.2403\n", "MSE: 0.2403\n", "RMSE: 0.4902\n", "R2 Score: -0.0465\n", "----------------------------------------\n", "Best parameters for KNN: {'n_neighbors': 7, 'weights': 'uniform'}\n", "Model Name: KNN (Filter Features)\n", "Accuracy: 0.7078\n", "Precision: 0.7116\n", "Recall: 0.7078\n", "F1 Score: 0.7094\n", "MAE: 0.2922\n", "MSE: 0.2922\n", "RMSE: 0.5406\n", "R2 Score: -0.2727\n", "----------------------------------------\n", "Best parameters for Gradient Boosting: {'learning_rate': 0.01, 'max_depth': 3, 'min_samples_split': 10, 'n_estimators': 200}\n", "Model Name: Gradient Boosting (Filter Features)\n", "Accuracy: 0.7662\n", "Precision: 0.7631\n", "Recall: 0.7662\n", "F1 Score: 0.7641\n", "MAE: 0.2338\n", "MSE: 0.2338\n", "RMSE: 0.4835\n", "R2 Score: -0.0182\n", "----------------------------------------\n" ] } ], "source": [ "# 9. Train and evaluate models on Filter Method features\n", "print(\"\\n=== Results using Filter Method (SelectKBest) ===\")\n", "for model_name, (model, param_grid) in models.items():\n", " if param_grid:\n", " grid_search = GridSearchCV(model, param_grid, cv=5, scoring='accuracy', n_jobs=-1)\n", " grid_search.fit(X_train_selected, y_train)\n", " model = grid_search.best_estimator_\n", " print(f\"Best parameters for {model_name}: {grid_search.best_params_}\")\n", " else:\n", " model.fit(X_train_selected, y_train)\n", " y_pred = model.predict(X_test_selected)\n", " results = evaluate_model(f\"{model_name} (Filter Features)\", y_test, y_pred)\n", " for key, value in results.items():\n", " print(f\"{key}: {value:.4f}\" if isinstance(value, float) else f\"{key}: {value}\")\n", " print(\"-\"*40)" ] }, { "cell_type": "code", "execution_count": 22, "id": "95970f75-2bf9-44f3-861f-e51504733fca", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "Selected Features using RFE: ['Glucose', 'BloodPressure', 'BMI', 'DiabetesPedigreeFunction', 'Age']\n" ] } ], "source": [ "# 10. Feature Selection: Wrapper Method (RFE)\n", "rfe_selector = RFE(RandomForestClassifier(random_state=43), n_features_to_select=5)\n", "rfe_selector.fit(X_train, y_train)\n", "X_train_selected_rfe = rfe_selector.transform(X_train)\n", "X_test_selected_rfe = rfe_selector.transform(X_test)\n", "\n", "selected_features_rfe = X.columns[rfe_selector.get_support()]\n", "print(\"\\nSelected Features using RFE:\", list(selected_features_rfe))" ] }, { "cell_type": "code", "execution_count": 24, "id": "343cf79e-445c-4ecd-93e6-b9800a48059a", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "=== Results using Wrapper Method (RFE) ===\n", "Best parameters for Random Forest: {'max_depth': None, 'min_samples_split': 2, 'n_estimators': 200}\n", "Model Name: Random Forest (Wrapper Features)\n", "Accuracy: 0.7727\n", "Precision: 0.7759\n", "Recall: 0.7727\n", "F1 Score: 0.7740\n", "MAE: 0.2273\n", "MSE: 0.2273\n", "RMSE: 0.4767\n", "R2 Score: 0.0101\n", "----------------------------------------\n", "Best parameters for Decision Tree: {'max_depth': 10, 'min_samples_split': 10}\n", "Model Name: Decision Tree (Wrapper Features)\n", "Accuracy: 0.7468\n", "Precision: 0.7458\n", "Recall: 0.7468\n", "F1 Score: 0.7462\n", "MAE: 0.2532\n", "MSE: 0.2532\n", "RMSE: 0.5032\n", "R2 Score: -0.1030\n", "----------------------------------------\n", "Best parameters for SVC: {'C': 10, 'kernel': 'linear'}\n", "Model Name: SVC (Wrapper Features)\n", "Accuracy: 0.7662\n", "Precision: 0.7645\n", "Recall: 0.7662\n", "F1 Score: 0.7652\n", "MAE: 0.2338\n", "MSE: 0.2338\n", "RMSE: 0.4835\n", "R2 Score: -0.0182\n", "----------------------------------------\n", "Model Name: Naive Bayes (Wrapper Features)\n", "Accuracy: 0.7662\n", "Precision: 0.7631\n", "Recall: 0.7662\n", "F1 Score: 0.7641\n", "MAE: 0.2338\n", "MSE: 0.2338\n", "RMSE: 0.4835\n", "R2 Score: -0.0182\n", "----------------------------------------\n", "Best parameters for KNN: {'n_neighbors': 7, 'weights': 'uniform'}\n", "Model Name: KNN (Wrapper Features)\n", "Accuracy: 0.7468\n", "Precision: 0.7426\n", "Recall: 0.7468\n", "F1 Score: 0.7438\n", "MAE: 0.2532\n", "MSE: 0.2532\n", "RMSE: 0.5032\n", "R2 Score: -0.1030\n", "----------------------------------------\n", "Best parameters for Gradient Boosting: {'learning_rate': 0.01, 'max_depth': 3, 'min_samples_split': 5, 'n_estimators': 300}\n", "Model Name: Gradient Boosting (Wrapper Features)\n", "Accuracy: 0.7662\n", "Precision: 0.7631\n", "Recall: 0.7662\n", "F1 Score: 0.7641\n", "MAE: 0.2338\n", "MSE: 0.2338\n", "RMSE: 0.4835\n", "R2 Score: -0.0182\n", "----------------------------------------\n" ] } ], "source": [ "# 11. Train and evaluate models on Wrapper Method features\n", "print(\"\\n=== Results using Wrapper Method (RFE) ===\")\n", "for model_name, (model, param_grid) in models.items():\n", " if param_grid:\n", " grid_search = GridSearchCV(model, param_grid, cv=5, scoring='accuracy', n_jobs=-1)\n", " grid_search.fit(X_train_selected_rfe, y_train)\n", " model = grid_search.best_estimator_\n", " print(f\"Best parameters for {model_name}: {grid_search.best_params_}\")\n", " else:\n", " model.fit(X_train_selected_rfe, y_train)\n", " y_pred = model.predict(X_test_selected_rfe)\n", " results = evaluate_model(f\"{model_name} (Wrapper Features)\", y_test, y_pred)\n", " for key, value in results.items():\n", " print(f\"{key}: {value:.4f}\" if isinstance(value, float) else f\"{key}: {value}\")\n", " print(\"-\"*40)" ] }, { "cell_type": "code", "execution_count": null, "id": "975447c5-77ba-429a-a1d7-9601292878d7", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python [conda env:.conda-hhh]", "language": "python", "name": "conda-env-.conda-hhh-py" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.12.9" } }, "nbformat": 4, "nbformat_minor": 5 }