{ "cells": [ { "cell_type": "code", "execution_count": 5, "id": "1aa8acff-b7d1-4597-9b98-655505d66e1c", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Training LogisticRegression...\n", "Best parameters for LogisticRegression: {'C': 10}\n", "Training DecisionTree...\n", "Best parameters for DecisionTree: {'max_depth': 3}\n", "Training RandomForest...\n", "Best parameters for RandomForest: {'max_depth': 10, 'n_estimators': 150}\n", "Training SVM...\n", "Best parameters for SVM: {'C': 1, 'kernel': 'rbf'}\n", "Training KNN...\n", "Best parameters for KNN: {'n_neighbors': 5}\n", "Training NaiveBayes...\n", "Best parameters for NaiveBayes: {}\n", "LogisticRegression Accuracy: 0.7532\n", " precision recall f1-score support\n", "\n", " 0 0.81 0.80 0.81 99\n", " 1 0.65 0.67 0.66 55\n", "\n", " accuracy 0.75 154\n", " macro avg 0.73 0.74 0.73 154\n", "weighted avg 0.76 0.75 0.75 154\n", "\n", "DecisionTree Accuracy: 0.7597\n", " precision recall f1-score support\n", "\n", " 0 0.80 0.84 0.82 99\n", " 1 0.68 0.62 0.65 55\n", "\n", " accuracy 0.76 154\n", " macro avg 0.74 0.73 0.73 154\n", "weighted avg 0.76 0.76 0.76 154\n", "\n", "RandomForest Accuracy: 0.7468\n", " precision recall f1-score support\n", "\n", " 0 0.81 0.80 0.80 99\n", " 1 0.64 0.65 0.65 55\n", "\n", " accuracy 0.75 154\n", " macro avg 0.72 0.73 0.73 154\n", "weighted avg 0.75 0.75 0.75 154\n", "\n", "SVM Accuracy: 0.7338\n", " precision recall f1-score support\n", "\n", " 0 0.77 0.83 0.80 99\n", " 1 0.65 0.56 0.60 55\n", "\n", " accuracy 0.73 154\n", " macro avg 0.71 0.70 0.70 154\n", "weighted avg 0.73 0.73 0.73 154\n", "\n", "KNN Accuracy: 0.6948\n", " precision recall f1-score support\n", "\n", " 0 0.75 0.80 0.77 99\n", " 1 0.58 0.51 0.54 55\n", "\n", " accuracy 0.69 154\n", " macro avg 0.66 0.65 0.66 154\n", "weighted avg 0.69 0.69 0.69 154\n", "\n", "NaiveBayes Accuracy: 0.7662\n", " precision recall f1-score support\n", "\n", " 0 0.83 0.80 0.81 99\n", " 1 0.66 0.71 0.68 55\n", "\n", " accuracy 0.77 154\n", " macro avg 0.75 0.75 0.75 154\n", "weighted avg 0.77 0.77 0.77 154\n", "\n", "\n", "Feature Selection\n", "Selected features using SelectKBest: [0 1 3 4 6]\n", "Selected features using RFE: [0 1 3 4 5]\n", "\n", "Evaluating LogisticRegression on SelectKBest features:\n", "Accuracy on selected features: 0.7013\n", "\n", "Evaluating LogisticRegression on RFE features:\n", "Accuracy on selected features: 0.7143\n", "\n", "Evaluating DecisionTree on SelectKBest features:\n", "Accuracy on selected features: 0.6688\n", "\n", "Evaluating DecisionTree on RFE features:\n", "Accuracy on selected features: 0.6558\n", "\n", "Evaluating RandomForest on SelectKBest features:\n", "Accuracy on selected features: 0.6753\n", "\n", "Evaluating RandomForest on RFE features:\n", "Accuracy on selected features: 0.6883\n", "\n", "Evaluating SVM on SelectKBest features:\n", "Accuracy on selected features: 0.7013\n", "\n", "Evaluating SVM on RFE features:\n", "Accuracy on selected features: 0.7208\n", "\n", "Evaluating KNN on SelectKBest features:\n", "Accuracy on selected features: 0.6039\n", "\n", "Evaluating KNN on RFE features:\n", "Accuracy on selected features: 0.6169\n", "\n", "Evaluating NaiveBayes on SelectKBest features:\n", "Accuracy on selected features: 0.7208\n", "\n", "Evaluating NaiveBayes on RFE features:\n", "Accuracy on selected features: 0.7078\n" ] } ], "source": [ "import pandas as pd\n", "import numpy as np\n", "from sklearn.model_selection import train_test_split, GridSearchCV\n", "from sklearn.impute import SimpleImputer\n", "from sklearn.preprocessing import StandardScaler\n", "from sklearn.linear_model import LogisticRegression\n", "from sklearn.tree import DecisionTreeClassifier\n", "from sklearn.ensemble import RandomForestClassifier\n", "from sklearn.svm import SVC\n", "from sklearn.neighbors import KNeighborsClassifier\n", "from sklearn.naive_bayes import GaussianNB\n", "from sklearn.metrics import accuracy_score, classification_report\n", "from sklearn.feature_selection import SelectKBest, chi2, RFE\n", "\n", "df = pd.read_csv(\"diabetes.csv\")\n", "\n", "# Handle missing values\n", "#Data is cleaned\n", "imputer = SimpleImputer(strategy='mean')\n", "X = df.drop('Outcome', axis=1)\n", "X = imputer.fit_transform(X)\n", "y = df['Outcome']\n", "\n", "# Split data into training and testing sets\n", "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n", "\n", "# Standardize the data\n", "scaler = StandardScaler()\n", "X_train = scaler.fit_transform(X_train)\n", "X_test = scaler.transform(X_test)\n", "\n", "# Define machine learning models and hyperparameters\n", "models = {\n", " 'LogisticRegression': LogisticRegression(),\n", " 'DecisionTree': DecisionTreeClassifier(),\n", " 'RandomForest': RandomForestClassifier(),\n", " 'SVM': SVC(),\n", " 'KNN': KNeighborsClassifier(),\n", " 'NaiveBayes': GaussianNB()\n", "}\n", "\n", "params = {\n", " 'LogisticRegression': {'C': [0.01, 0.1, 1, 10]},\n", " 'DecisionTree': {'max_depth': [3, 5, 7]},\n", " 'RandomForest': {'n_estimators': [50, 100, 150], 'max_depth': [5, 7, 10]},\n", " 'SVM': {'C': [0.01, 0.1, 1], 'kernel': ['linear', 'rbf']},\n", " 'KNN': {'n_neighbors': [3, 5, 7]},\n", " 'NaiveBayes': {}\n", "}\n", "\n", "# Apply models and GridSearchCV\n", "best_estimators = {}\n", "for model_name, model in models.items():\n", " print(f\"Training {model_name}...\")\n", " grid = GridSearchCV(model, params[model_name], cv=5, scoring='accuracy', n_jobs=-1)\n", " grid.fit(X_train, y_train)\n", " best_estimators[model_name] = grid.best_estimator_\n", " print(f\"Best parameters for {model_name}: {grid.best_params_}\")\n", " \n", "# Evaluate models\n", "for model_name, model in best_estimators.items():\n", " y_pred = model.predict(X_test)\n", " print(f\"{model_name} Accuracy: {accuracy_score(y_test, y_pred):.4f}\")\n", " print(classification_report(y_test, y_pred))\n", "\n", "# Feature Selection\n", "# Ensure all features are non-negative\n", "X_train_non_negative = np.abs(X_train) # Take absolute values to remove negatives\n", "X_test_non_negative = np.abs(X_test)\n", "\n", "# Feature Selection\n", "print(\"\\nFeature Selection\")\n", "# Method 1: SelectKBest (Chi-Square)\n", "k_best = SelectKBest(score_func=chi2, k=5)\n", "X_new_kbest = k_best.fit_transform(X_train_non_negative, y_train) # Use non-negative data\n", "selected_features_kbest = k_best.get_support(indices=True)\n", "print(\"Selected features using SelectKBest:\", selected_features_kbest)\n", "\n", "# Method 2: Recursive Feature Elimination (RFE)\n", "logreg = LogisticRegression()\n", "rfe = RFE(estimator=logreg, n_features_to_select=5)\n", "X_new_rfe = rfe.fit_transform(X_train_non_negative, y_train) # Use non-negative data\n", "selected_features_rfe = rfe.get_support(indices=True)\n", "print(\"Selected features using RFE:\", selected_features_rfe)\n", "\n", "# Reapply ML models on selected features\n", "def evaluate_on_selected_features(X_selected, X_test_selected, y_train, y_test, model):\n", " model.fit(X_selected, y_train)\n", " y_pred = model.predict(X_test_selected)\n", " print(f\"Accuracy on selected features: {accuracy_score(y_test, y_pred):.4f}\")\n", "\n", "X_test_kbest = k_best.transform(X_test_non_negative) # Use non-negative data\n", "X_test_rfe = rfe.transform(X_test_non_negative) # Use non-negative data\n", "\n", "for model_name, model in best_estimators.items():\n", " print(f\"\\nEvaluating {model_name} on SelectKBest features:\")\n", " evaluate_on_selected_features(X_new_kbest, X_test_kbest, y_train, y_test, model)\n", "\n", " print(f\"\\nEvaluating {model_name} on RFE features:\")\n", " evaluate_on_selected_features(X_new_rfe, X_test_rfe, y_train, y_test, model)\n" ] }, { "cell_type": "code", "execution_count": null, "id": "05075d07-5ae9-449b-aebb-ab3cfa737a95", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "anaconda-panel-2023.05-py310", "language": "python", "name": "conda-env-anaconda-panel-2023.05-py310-py" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.5" } }, "nbformat": 4, "nbformat_minor": 5 }