# -*- coding: utf-8 -*-
"""Copy of project 2026.ipynb

Automatically generated by Colab.

Original file is located at
    https://colab.research.google.com/drive/1P7lRVtVwKYZgl0VaWuH4UTFcPWW6wuHz
"""

from google.colab import files
files.upload()

!mkdir -p ~/.kaggle
!cp kaggle.json ~/.kaggle/kaggle.json
!chmod 600 ~/.kaggle/kaggle.json
!ls -la ~/.kaggle

!kaggle --version
!kaggle datasets list -s backblaze | head

!kaggle datasets download -d backblaze/hard-drive-test-data
!ls -lh

!unzip -o hard-drive-test-data.zip
!ls -lh

import os
print(os.listdir())

# إذا شفت harddrive.csv:
import pandas as pd
df = pd.read_csv("harddrive.csv")
print(df.shape)
df.head()

df = df.drop(['date', 'serial_number', 'model'], axis=1)
df.shape

threshold = 0.8
missing_ratio = df.isnull().mean()
cols_to_drop = missing_ratio[missing_ratio > threshold].index
df = df.drop(columns=cols_to_drop)

df = df.fillna(0)

df.shape

df['failure'].value_counts(normalize=True)

df_majority = df[df['failure'] == 0]
df_minority = df[df['failure'] == 1]

df_majority_sample = df_majority.sample(n=50000, random_state=42)

df_balanced = pd.concat([df_majority_sample, df_minority])
df_balanced = df_balanced.sample(frac=1, random_state=42)

df_balanced['failure'].value_counts()

from sklearn.model_selection import train_test_split

X = df_balanced.drop('failure', axis=1)
y = df_balanced['failure']

X_train, X_test, y_train, y_test = train_test_split(
    X, y,
    test_size=0.2,
    random_state=42,
    stratify=y
)

print(X_train.shape, X_test.shape)

from sklearn.ensemble import RandomForestClassifier

rf = RandomForestClassifier(
    n_estimators=100,
    random_state=42,
    class_weight='balanced'
)

rf.fit(X_train, y_train)

y_pred_rf = rf.predict(X_test)

from sklearn.metrics import classification_report, roc_auc_score

print(classification_report(y_test, y_pred_rf))

y_prob_rf = rf.predict_proba(X_test)[:,1]
print("RF AUC:", roc_auc_score(y_test, y_prob_rf))

threshold = 0.2
y_pred_new = (y_prob_rf > threshold).astype(int)

print(classification_report(y_test, y_pred_new))

from sklearn.metrics import roc_curve

fpr, tpr, thresholds = roc_curve(y_test, y_prob_rf)

optimal_idx = (tpr - fpr).argmax()
optimal_threshold = thresholds[optimal_idx]

print("Best Threshold:", optimal_threshold)

y_pred_opt = (y_prob_rf > optimal_threshold).astype(int)

print(classification_report(y_test, y_pred_opt))

import matplotlib.pyplot as plt
from sklearn.metrics import roc_curve, roc_auc_score

fpr, tpr, _ = roc_curve(y_test, y_prob_rf)

plt.figure(figsize=(6,6))
plt.plot(fpr, tpr, label=f"AUC = {roc_auc_score(y_test, y_prob_rf):.3f}")
plt.plot([0,1], [0,1], linestyle='--')
plt.xlabel("False Positive Rate")
plt.ylabel("True Positive Rate")
plt.title("ROC Curve - Random Forest")
plt.legend()
plt.show()

import matplotlib.pyplot as plt
from sklearn.metrics import roc_curve, roc_auc_score

fpr, tpr, _ = roc_curve(y_test, y_prob_rf)

plt.figure(figsize=(6,6))
plt.plot(fpr, tpr, label=f"AUC = {roc_auc_score(y_test, y_prob_rf):.3f}")
plt.plot([0,1], [0,1], linestyle='--')
plt.xlabel("False Positive Rate")
plt.ylabel("True Positive Rate")
plt.title("ROC Curve - Random Forest")
plt.legend()
plt.show()

from sklearn.metrics import recall_score, precision_score, roc_auc_score

# Random Forest - Optimized threshold
rf_recall = recall_score(y_test, y_pred_opt)
rf_precision = precision_score(y_test, y_pred_opt)
rf_auc = roc_auc_score(y_test, y_prob_rf)

print("RF Recall:", rf_recall)
print("RF Precision:", rf_precision)
print("RF AUC:", rf_auc)

import pandas as pd

comparison = pd.DataFrame({
    "Model": ["Random Forest (Optimized)"],
    "Recall": [rf_recall],
    "Precision": [rf_precision],
    "AUC": [rf_auc]
})

comparison

from xgboost import XGBClassifier
from sklearn.metrics import classification_report, roc_auc_score

# إنشاء النموذج
xgb = XGBClassifier(
    n_estimators=200,
    max_depth=6,
    learning_rate=0.1,
    scale_pos_weight=(len(y_train) - sum(y_train)) / sum(y_train),
    random_state=42
)

# تدريب
xgb.fit(X_train, y_train)

# التنبؤ
y_pred_xgb = xgb.predict(X_test)
y_prob_xgb = xgb.predict_proba(X_test)[:,1]

# طباعة النتائج
print(classification_report(y_test, y_pred_xgb))
print("XGB AUC:", roc_auc_score(y_test, y_prob_xgb))

from sklearn.metrics import recall_score, precision_score, roc_auc_score

xgb_recall = recall_score(y_test, y_pred_xgb)
xgb_precision = precision_score(y_test, y_pred_xgb)
xgb_auc = roc_auc_score(y_test, y_prob_xgb)

print("XGB Recall:", xgb_recall)
print("XGB Precision:", xgb_precision)
print("XGB AUC:", xgb_auc)

from xgboost import plot_importance
import matplotlib.pyplot as plt

plt.figure(figsize=(10,8))
plot_importance(xgb, max_num_features=15)
plt.show()

import seaborn as sns
import matplotlib.pyplot as plt

sns.boxplot(x=df_balanced['failure'], y=df_balanced['smart_197_raw'])
plt.title("SMART_197_raw Distribution by Failure Class")
plt.show()