{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "946e852d",
   "metadata": {},
   "source": [
    "### Toymodel as an initial test\n",
    "This is only a quick test to see if the model is applicable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0952f099",
   "metadata": {},
   "outputs": [
    {
     "ename": "",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31mRunning cells with '.venv (Python 3.13.7)' requires the ipykernel package.\n",
      "\u001b[1;31mInstall 'ipykernel' into the Python environment. \n",
      "\u001b[1;31mCommand: '/home/jaknyst/Documents/MLPproject/.venv/bin/python -m pip install ipykernel -U --force-reinstall'"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import train_test_split, RandomizedSearchCV, KFold, StratifiedKFold, GridSearchCV\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.tree import DecisionTreeClassifier, plot_tree\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.metrics import mean_squared_error, mean_absolute_error , r2_score\n",
    "from sklearn.metrics import classification_report, confusion_matrix, ConfusionMatrixDisplay\n",
    "from scipy.stats import randint\n",
    "\n",
    "# Load dataset\n",
    "df = pd.read_csv('./Datasets/adult.csv', comment = '#')\n",
    "\n",
    "# First, create a copy of the dataframe to avoid modifying the original\n",
    "df_encoded = df.copy()\n",
    "df_encoded.drop(['fnlwgt', 'education'], axis=1, inplace=True)\n",
    "\n",
    "# drop all rows that contain '?'\n",
    "for column in df_encoded.columns:\n",
    "    df_encoded = df_encoded[df_encoded[column] != '?']\n",
    "\n",
    "# Apply label encoding to categorical columns\n",
    "label_encoder = LabelEncoder()\n",
    "categorical_columns = ['workclass', 'marital.status', 'occupation', \n",
    "                      'relationship', 'race', 'sex', 'native.country', 'income']\n",
    "\n",
    "for column in categorical_columns:\n",
    "    df_encoded[column] = label_encoder.fit_transform(df_encoded[column])\n",
    "\n",
    "# Now properly separate features and target\n",
    "X = df_encoded.drop(columns=['income'])\n",
    "y = df_encoded['income']\n",
    "\n",
    "# Split the data\n",
    "X_train_val, X_test, y_train_val, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "X_train, X_val, y_train, y_val = train_test_split(X_train_val, y_train_val, test_size=0.25, random_state=42)\n",
    "\n",
    "\n",
    "# Build pipeline\n",
    "model = Pipeline([\n",
    "    ('full_dt_classifier', DecisionTreeClassifier(random_state=42))  # Train Decision Tree Regressor\n",
    "])\n",
    "\n",
    "# Train the model\n",
    "model.fit(X_train, y_train)\n",
    "y_pred = model.predict(X_val)\n",
    "\n",
    "# Visualize the decision tree\n",
    "plt.figure(figsize=(12, 12))\n",
    "plot_tree(\n",
    "    model.named_steps['full_dt_classifier'],\n",
    "    feature_names=X.columns,\n",
    "    class_names=[\"Poor\", \"Rich\"],\n",
    "    filled=True,\n",
    "    rounded=True,\n",
    "    max_depth=5,       # Keep tree shallow for readability\n",
    "    fontsize=3,\n",
    "    precision=2        # Limit decimals\n",
    ")\n",
    "\n",
    "#plt.savefig('decision_tree.pdf', format='pdf', dpi=300)\n",
    "plt.show()\n",
    "\n",
    "CM = confusion_matrix(y_val, y_pred)\n",
    "disp = ConfusionMatrixDisplay(confusion_matrix=CM, display_labels=[\"Poor\", \"Rich\"])\n",
    "disp.plot(cmap=\"Blues\")\n",
    "plt.title(\"Confusion Matrix\")\n",
    "plt.show()  \n",
    "\n",
    "print(\"Classification Report:\")\n",
    "print(classification_report(y_val, y_pred, target_names=[\"Poor\", \"Rich\"]))\n",
    "\n",
    "print(X_train_val.columns)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bfaae28c",
   "metadata": {},
   "source": [
    "### Hyperparameter tuning\n",
    "I think this section is pretty unecessary. It can probably be removed, but I'm going to keep it for now, just in case. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e567e4e9",
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
      "\u001b[31mKeyboardInterrupt\u001b[39m                         Traceback (most recent call last)",
      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[4]\u001b[39m\u001b[32m, line 8\u001b[39m\n\u001b[32m      6\u001b[39m skf = StratifiedKFold(n_splits=\u001b[32m10\u001b[39m, shuffle=\u001b[38;5;28;01mTrue\u001b[39;00m, random_state=\u001b[32m42\u001b[39m)\n\u001b[32m      7\u001b[39m grid_search = GridSearchCV(model, param_grid, scoring=\u001b[33m'\u001b[39m\u001b[33maccuracy\u001b[39m\u001b[33m'\u001b[39m, cv=skf, n_jobs=-\u001b[32m1\u001b[39m)\n\u001b[32m----> \u001b[39m\u001b[32m8\u001b[39m \u001b[43mgrid_search\u001b[49m\u001b[43m.\u001b[49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\u001b[43mX_train\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43my_train\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m     11\u001b[39m \u001b[38;5;66;03m# Best model training\u001b[39;00m\n\u001b[32m     12\u001b[39m best_model = grid_search.best_estimator_\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/MLP/Projects/MLPproject/.venv/lib/python3.12/site-packages/sklearn/base.py:1365\u001b[39m, in \u001b[36m_fit_context.<locals>.decorator.<locals>.wrapper\u001b[39m\u001b[34m(estimator, *args, **kwargs)\u001b[39m\n\u001b[32m   1358\u001b[39m     estimator._validate_params()\n\u001b[32m   1360\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m config_context(\n\u001b[32m   1361\u001b[39m     skip_parameter_validation=(\n\u001b[32m   1362\u001b[39m         prefer_skip_nested_validation \u001b[38;5;129;01mor\u001b[39;00m global_skip_validation\n\u001b[32m   1363\u001b[39m     )\n\u001b[32m   1364\u001b[39m ):\n\u001b[32m-> \u001b[39m\u001b[32m1365\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfit_method\u001b[49m\u001b[43m(\u001b[49m\u001b[43mestimator\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/MLP/Projects/MLPproject/.venv/lib/python3.12/site-packages/sklearn/model_selection/_search.py:1051\u001b[39m, in \u001b[36mBaseSearchCV.fit\u001b[39m\u001b[34m(self, X, y, **params)\u001b[39m\n\u001b[32m   1045\u001b[39m     results = \u001b[38;5;28mself\u001b[39m._format_results(\n\u001b[32m   1046\u001b[39m         all_candidate_params, n_splits, all_out, all_more_results\n\u001b[32m   1047\u001b[39m     )\n\u001b[32m   1049\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m results\n\u001b[32m-> \u001b[39m\u001b[32m1051\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_run_search\u001b[49m\u001b[43m(\u001b[49m\u001b[43mevaluate_candidates\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m   1053\u001b[39m \u001b[38;5;66;03m# multimetric is determined here because in the case of a callable\u001b[39;00m\n\u001b[32m   1054\u001b[39m \u001b[38;5;66;03m# self.scoring the return type is only known after calling\u001b[39;00m\n\u001b[32m   1055\u001b[39m first_test_score = all_out[\u001b[32m0\u001b[39m][\u001b[33m\"\u001b[39m\u001b[33mtest_scores\u001b[39m\u001b[33m\"\u001b[39m]\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/MLP/Projects/MLPproject/.venv/lib/python3.12/site-packages/sklearn/model_selection/_search.py:1605\u001b[39m, in \u001b[36mGridSearchCV._run_search\u001b[39m\u001b[34m(self, evaluate_candidates)\u001b[39m\n\u001b[32m   1603\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34m_run_search\u001b[39m(\u001b[38;5;28mself\u001b[39m, evaluate_candidates):\n\u001b[32m   1604\u001b[39m \u001b[38;5;250m    \u001b[39m\u001b[33;03m\"\"\"Search all candidates in param_grid\"\"\"\u001b[39;00m\n\u001b[32m-> \u001b[39m\u001b[32m1605\u001b[39m     \u001b[43mevaluate_candidates\u001b[49m\u001b[43m(\u001b[49m\u001b[43mParameterGrid\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mparam_grid\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/MLP/Projects/MLPproject/.venv/lib/python3.12/site-packages/sklearn/model_selection/_search.py:997\u001b[39m, in \u001b[36mBaseSearchCV.fit.<locals>.evaluate_candidates\u001b[39m\u001b[34m(candidate_params, cv, more_results)\u001b[39m\n\u001b[32m    989\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m.verbose > \u001b[32m0\u001b[39m:\n\u001b[32m    990\u001b[39m     \u001b[38;5;28mprint\u001b[39m(\n\u001b[32m    991\u001b[39m         \u001b[33m\"\u001b[39m\u001b[33mFitting \u001b[39m\u001b[38;5;132;01m{0}\u001b[39;00m\u001b[33m folds for each of \u001b[39m\u001b[38;5;132;01m{1}\u001b[39;00m\u001b[33m candidates,\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m    992\u001b[39m         \u001b[33m\"\u001b[39m\u001b[33m totalling \u001b[39m\u001b[38;5;132;01m{2}\u001b[39;00m\u001b[33m fits\u001b[39m\u001b[33m\"\u001b[39m.format(\n\u001b[32m    993\u001b[39m             n_splits, n_candidates, n_candidates * n_splits\n\u001b[32m    994\u001b[39m         )\n\u001b[32m    995\u001b[39m     )\n\u001b[32m--> \u001b[39m\u001b[32m997\u001b[39m out = \u001b[43mparallel\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m    998\u001b[39m \u001b[43m    \u001b[49m\u001b[43mdelayed\u001b[49m\u001b[43m(\u001b[49m\u001b[43m_fit_and_score\u001b[49m\u001b[43m)\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m    999\u001b[39m \u001b[43m        \u001b[49m\u001b[43mclone\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbase_estimator\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1000\u001b[39m \u001b[43m        \u001b[49m\u001b[43mX\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1001\u001b[39m \u001b[43m        \u001b[49m\u001b[43my\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1002\u001b[39m \u001b[43m        \u001b[49m\u001b[43mtrain\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtrain\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1003\u001b[39m \u001b[43m        \u001b[49m\u001b[43mtest\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtest\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1004\u001b[39m \u001b[43m        \u001b[49m\u001b[43mparameters\u001b[49m\u001b[43m=\u001b[49m\u001b[43mparameters\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1005\u001b[39m \u001b[43m        \u001b[49m\u001b[43msplit_progress\u001b[49m\u001b[43m=\u001b[49m\u001b[43m(\u001b[49m\u001b[43msplit_idx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mn_splits\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1006\u001b[39m \u001b[43m        \u001b[49m\u001b[43mcandidate_progress\u001b[49m\u001b[43m=\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcand_idx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mn_candidates\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1007\u001b[39m \u001b[43m        \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mfit_and_score_kwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1008\u001b[39m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m   1009\u001b[39m \u001b[43m    \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43m(\u001b[49m\u001b[43mcand_idx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mparameters\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m(\u001b[49m\u001b[43msplit_idx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m(\u001b[49m\u001b[43mtrain\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtest\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mproduct\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m   1010\u001b[39m \u001b[43m        \u001b[49m\u001b[38;5;28;43menumerate\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mcandidate_params\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1011\u001b[39m \u001b[43m        \u001b[49m\u001b[38;5;28;43menumerate\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mcv\u001b[49m\u001b[43m.\u001b[49m\u001b[43msplit\u001b[49m\u001b[43m(\u001b[49m\u001b[43mX\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43my\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mrouted_params\u001b[49m\u001b[43m.\u001b[49m\u001b[43msplitter\u001b[49m\u001b[43m.\u001b[49m\u001b[43msplit\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1012\u001b[39m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m   1013\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m   1015\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(out) < \u001b[32m1\u001b[39m:\n\u001b[32m   1016\u001b[39m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[32m   1017\u001b[39m         \u001b[33m\"\u001b[39m\u001b[33mNo fits were performed. \u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m   1018\u001b[39m         \u001b[33m\"\u001b[39m\u001b[33mWas the CV iterator empty? \u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m   1019\u001b[39m         \u001b[33m\"\u001b[39m\u001b[33mWere there no candidates?\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m   1020\u001b[39m     )\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/MLP/Projects/MLPproject/.venv/lib/python3.12/site-packages/sklearn/utils/parallel.py:82\u001b[39m, in \u001b[36mParallel.__call__\u001b[39m\u001b[34m(self, iterable)\u001b[39m\n\u001b[32m     73\u001b[39m warning_filters = warnings.filters\n\u001b[32m     74\u001b[39m iterable_with_config_and_warning_filters = (\n\u001b[32m     75\u001b[39m     (\n\u001b[32m     76\u001b[39m         _with_config_and_warning_filters(delayed_func, config, warning_filters),\n\u001b[32m   (...)\u001b[39m\u001b[32m     80\u001b[39m     \u001b[38;5;28;01mfor\u001b[39;00m delayed_func, args, kwargs \u001b[38;5;129;01min\u001b[39;00m iterable\n\u001b[32m     81\u001b[39m )\n\u001b[32m---> \u001b[39m\u001b[32m82\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m.\u001b[49m\u001b[34;43m__call__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43miterable_with_config_and_warning_filters\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/MLP/Projects/MLPproject/.venv/lib/python3.12/site-packages/joblib/parallel.py:2072\u001b[39m, in \u001b[36mParallel.__call__\u001b[39m\u001b[34m(self, iterable)\u001b[39m\n\u001b[32m   2066\u001b[39m \u001b[38;5;66;03m# The first item from the output is blank, but it makes the interpreter\u001b[39;00m\n\u001b[32m   2067\u001b[39m \u001b[38;5;66;03m# progress until it enters the Try/Except block of the generator and\u001b[39;00m\n\u001b[32m   2068\u001b[39m \u001b[38;5;66;03m# reaches the first `yield` statement. This starts the asynchronous\u001b[39;00m\n\u001b[32m   2069\u001b[39m \u001b[38;5;66;03m# dispatch of the tasks to the workers.\u001b[39;00m\n\u001b[32m   2070\u001b[39m \u001b[38;5;28mnext\u001b[39m(output)\n\u001b[32m-> \u001b[39m\u001b[32m2072\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m output \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m.return_generator \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;43mlist\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43moutput\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/MLP/Projects/MLPproject/.venv/lib/python3.12/site-packages/joblib/parallel.py:1682\u001b[39m, in \u001b[36mParallel._get_outputs\u001b[39m\u001b[34m(self, iterator, pre_dispatch)\u001b[39m\n\u001b[32m   1679\u001b[39m     \u001b[38;5;28;01myield\u001b[39;00m\n\u001b[32m   1681\u001b[39m     \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m._backend.retrieval_context():\n\u001b[32m-> \u001b[39m\u001b[32m1682\u001b[39m         \u001b[38;5;28;01myield from\u001b[39;00m \u001b[38;5;28mself\u001b[39m._retrieve()\n\u001b[32m   1684\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mGeneratorExit\u001b[39;00m:\n\u001b[32m   1685\u001b[39m     \u001b[38;5;66;03m# The generator has been garbage collected before being fully\u001b[39;00m\n\u001b[32m   1686\u001b[39m     \u001b[38;5;66;03m# consumed. This aborts the remaining tasks if possible and warn\u001b[39;00m\n\u001b[32m   1687\u001b[39m     \u001b[38;5;66;03m# the user if necessary.\u001b[39;00m\n\u001b[32m   1688\u001b[39m     \u001b[38;5;28mself\u001b[39m._exception = \u001b[38;5;28;01mTrue\u001b[39;00m\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/MLP/Projects/MLPproject/.venv/lib/python3.12/site-packages/joblib/parallel.py:1800\u001b[39m, in \u001b[36mParallel._retrieve\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m   1789\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m.return_ordered:\n\u001b[32m   1790\u001b[39m     \u001b[38;5;66;03m# Case ordered: wait for completion (or error) of the next job\u001b[39;00m\n\u001b[32m   1791\u001b[39m     \u001b[38;5;66;03m# that have been dispatched and not retrieved yet. If no job\u001b[39;00m\n\u001b[32m   (...)\u001b[39m\u001b[32m   1795\u001b[39m     \u001b[38;5;66;03m# control only have to be done on the amount of time the next\u001b[39;00m\n\u001b[32m   1796\u001b[39m     \u001b[38;5;66;03m# dispatched job is pending.\u001b[39;00m\n\u001b[32m   1797\u001b[39m     \u001b[38;5;28;01mif\u001b[39;00m (nb_jobs == \u001b[32m0\u001b[39m) \u001b[38;5;129;01mor\u001b[39;00m (\n\u001b[32m   1798\u001b[39m         \u001b[38;5;28mself\u001b[39m._jobs[\u001b[32m0\u001b[39m].get_status(timeout=\u001b[38;5;28mself\u001b[39m.timeout) == TASK_PENDING\n\u001b[32m   1799\u001b[39m     ):\n\u001b[32m-> \u001b[39m\u001b[32m1800\u001b[39m         \u001b[43mtime\u001b[49m\u001b[43m.\u001b[49m\u001b[43msleep\u001b[49m\u001b[43m(\u001b[49m\u001b[32;43m0.01\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[32m   1801\u001b[39m         \u001b[38;5;28;01mcontinue\u001b[39;00m\n\u001b[32m   1803\u001b[39m \u001b[38;5;28;01melif\u001b[39;00m nb_jobs == \u001b[32m0\u001b[39m:\n\u001b[32m   1804\u001b[39m     \u001b[38;5;66;03m# Case unordered: jobs are added to the list of jobs to\u001b[39;00m\n\u001b[32m   1805\u001b[39m     \u001b[38;5;66;03m# retrieve `self._jobs` only once completed or in error, which\u001b[39;00m\n\u001b[32m   (...)\u001b[39m\u001b[32m   1811\u001b[39m     \u001b[38;5;66;03m# timeouts before any other dispatched job has completed and\u001b[39;00m\n\u001b[32m   1812\u001b[39m     \u001b[38;5;66;03m# been added to `self._jobs` to be retrieved.\u001b[39;00m\n",
      "\u001b[31mKeyboardInterrupt\u001b[39m: "
     ]
    }
   ],
   "source": [
    "param_grid = {\n",
    "                'full_dt_classifier__max_depth': [None, 10, 11, 12, 13, 14, 15, 16],\n",
    "                'full_dt_classifier__min_samples_split': [3, 4, 5, 6, 7, 8, 9],\n",
    "                'full_dt_classifier__min_samples_leaf': [1, 2, 3, 4, 5]\n",
    "            }\n",
    "skf = StratifiedKFold(n_splits=10, shuffle=True, random_state=42)\n",
    "grid_search = GridSearchCV(model, param_grid, scoring='accuracy', cv=skf, n_jobs=-1)\n",
    "grid_search.fit(X_train, y_train)\n",
    "\n",
    "\n",
    "# Best model training\n",
    "best_model = grid_search.best_estimator_\n",
    "y_pred_best = best_model.predict(X_val)\n",
    "\n",
    "print(\"Classification Report:\")\n",
    "print(classification_report(y_val, y_pred_best, target_names=[\"Poor\", \"Rich\"]))\n",
    "\n",
    "best_max_depth = best_model.named_steps['full_dt_classifier'].max_depth\n",
    "best_min_samples_split = best_model.named_steps['full_dt_classifier'].min_samples_split\n",
    "best_min_samples_leaf = best_model.named_steps['full_dt_classifier'].min_samples_leaf\n",
    "\n",
    "print(f'Best max_depth: {best_max_depth}')\n",
    "print(f'Best min_samples_split: {best_min_samples_split}')\n",
    "print(f'Best min_samples_leaf: {best_min_samples_leaf}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a37f45b4",
   "metadata": {},
   "source": [
    "### Classifier comparison\n",
    "This is taken from the ensemble lab. A lot of models are excluded, but they are easy to implement into the class, for now It's kept short to minimize the execution time. We can also tune more parameters for each model, but of course that would also increase the execution time."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e68b1ea9",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV, StratifiedKFold\n",
    "import time\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.metrics import (accuracy_score, precision_score, \n",
    "                             recall_score, f1_score, \n",
    "                             confusion_matrix, ConfusionMatrixDisplay)\n",
    "from sklearn.ensemble import (RandomForestClassifier, BaggingClassifier, \n",
    "                              AdaBoostClassifier, GradientBoostingClassifier)\n",
    "from xgboost import XGBClassifier        # Requires installation of the package ; Not a native function in sklearn\n",
    "from lightgbm import LGBMClassifier      # Requires installation of the package; Not a native function in sklearn\n",
    "from catboost import CatBoostClassifier  # Requires installation of the package; Not a native function in sklearn\n",
    "\n",
    "class ClassifierComparisonOpt:\n",
    "    def __init__(self, X, y, test_size=0.25, use_bootstrap=True, random_state=42, cv_folds=10):\n",
    "        # Split data stratified by labels\n",
    "        X_train, X_test, y_train, y_test = train_test_split(\n",
    "            X, y, test_size=test_size, stratify=y, random_state=random_state)\n",
    "\n",
    "        # Scale features\n",
    "        #scaler = StandardScaler()\n",
    "        #self.X_train = scaler.fit_transform(X_train)\n",
    "        #self.X_test  = scaler.transform(X_test)\n",
    "\n",
    "        # Scaling not necessary for decision trees\n",
    "        self.X_train = X_train\n",
    "        self.X_test = X_test\n",
    "\n",
    "        self.y_train       = y_train\n",
    "        self.y_test        = y_test\n",
    "        self.use_bootstrap = use_bootstrap\n",
    "        self.cv_folds      = cv_folds\n",
    "        self.models        = {}\n",
    "        self.results       = {}\n",
    "        self.results_df    = None\n",
    "        self.feature_names = list(X.columns)\n",
    "\n",
    "    def get_models_with_params(self):\n",
    "        return {\n",
    "            'Decision Tree': (DecisionTreeClassifier(random_state=42), {\n",
    "                'max_depth': [None, 5, 10, 20],\n",
    "                'min_samples_split': [2, 5, 10],\n",
    "                #'min_samples_leaf': [2, 4, 6]\n",
    "            }),\n",
    "            'Random Forest': (RandomForestClassifier(bootstrap=self.use_bootstrap, random_state=42), {\n",
    "                'n_estimators': [50, 100, 200],\n",
    "                'max_depth': [None, 5, 10, 20],\n",
    "                'min_samples_split': [2, 5, 10],\n",
    "                #'min_samples_leaf': [2, 4, 6]\n",
    "            }),\n",
    "            'XGBoost': (XGBClassifier(eval_metric='mlogloss', random_state=42), {\n",
    "                'n_estimators': [50, 100, 200],\n",
    "                'learning_rate': [0.01, 0.1, 0.2, 0.5, 1.0],\n",
    "                'max_depth': [3, 5, 10]\n",
    "            })\n",
    "        }\n",
    "\n",
    "    def fit_models(self, models_with_params=None):\n",
    "        # Set self.get_models_with_params() as a default parameter distribution\n",
    "        if models_with_params == None:\n",
    "            models_with_params = self.get_models_with_params()\n",
    "\n",
    "        results_list = []\n",
    "        cv = StratifiedKFold(n_splits=self.cv_folds, shuffle=True, random_state=42)\n",
    "\n",
    "        for name, (model, param_grid) in models_with_params.items():\n",
    "            print(f\"Tuning {name} ...\")\n",
    "            \n",
    "            grid_search = GridSearchCV(model, param_grid, scoring='accuracy', cv=cv, n_jobs=-2)\n",
    "            \n",
    "            start_train = time.time()\n",
    "            grid_search.fit(self.X_train, self.y_train)\n",
    "            end_train = time.time()\n",
    "\n",
    "            best_model = grid_search.best_estimator_\n",
    "            y_pred = best_model.predict(self.X_test)\n",
    "            end_pred = time.time()\n",
    "\n",
    "            self.models[name] = {\n",
    "                'model': best_model,\n",
    "                'confusion_matrix': confusion_matrix(self.y_test, y_pred)\n",
    "            }\n",
    "\n",
    "            metrics = {\n",
    "                'Model': name,\n",
    "                'Accuracy': accuracy_score(self.y_test, y_pred),\n",
    "                'Precision': precision_score(self.y_test, y_pred, average='weighted', zero_division=0),\n",
    "                'Recall': recall_score(self.y_test, y_pred, average='weighted'),\n",
    "                'F1 Score': f1_score(self.y_test, y_pred, average='weighted'),\n",
    "                'Best Params': grid_search.best_params_,\n",
    "                'Training Time (s)': (end_train - start_train),\n",
    "                'Prediction Time (s)': (end_pred - end_train),\n",
    "                'Total Time (s)': (end_pred - start_train)\n",
    "            }\n",
    "\n",
    "            results_list.append(metrics)\n",
    "\n",
    "        self.results_df = pd.DataFrame(results_list)\n",
    "\n",
    "    def print_summary(self):\n",
    "        print(\"\\n------ Results Sorted by Accuracy ------\")\n",
    "        print(self.results_df.sort_values(by='Accuracy', ascending=False).to_string(index=False))\n",
    "\n",
    "        print(\"\\n------ Results Sorted by Total Time ------\")\n",
    "        print(self.results_df.sort_values(by='Total Time (s)', ascending=True).to_string(index=False))\n",
    "\n",
    "    # Show feature importance for models that support it          \n",
    "    def show_feature_importance(self):\n",
    "        importance = {}\n",
    "        features = self.feature_names\n",
    "\n",
    "        for name, result in self.models.items():\n",
    "            model = result['model']\n",
    "            if hasattr(model, 'feature_importances_'):\n",
    "                importance[name] = model.feature_importances_\n",
    "            elif hasattr(model, 'coef_'):\n",
    "                coef = model.coef_\n",
    "                if coef.ndim == 1:\n",
    "                    importance[name] = np.abs(coef)\n",
    "                else:\n",
    "                    importance[name] = np.mean(np.abs(coef), axis=0)\n",
    "            else:\n",
    "                print(f\"Feature importance not available for model {name}\")\n",
    "\n",
    "        for name, imp in importance.items():\n",
    "            sorted_idx = np.argsort(imp)[::-1]\n",
    "            sorted_features = [features[i] for i in sorted_idx]\n",
    "            plt.figure()\n",
    "            plt.bar(range(len(imp)), imp[sorted_idx], align='center')\n",
    "            plt.xticks(range(len(imp)), sorted_features, rotation=70)\n",
    "            plt.title(f\"Feature importance for {name}\")\n",
    "            plt.xlabel(\"Feature index\")\n",
    "            plt.ylabel(\"Importance score\")\n",
    "            plt.grid(True, linestyle='--', alpha=0.6)\n",
    "            plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "da377fc1",
   "metadata": {},
   "source": [
    "### Compare models with coarse hyperparameter tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d6fd1fee",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Decision Tree ...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Random Forest ...\n",
      "Tuning XGBoost ...\n",
      "\n",
      "------ Results Sorted by Accuracy ------\n",
      "        Model  Accuracy  Precision   Recall  F1 Score                                                     Best Params  Training Time (s)  Prediction Time (s)  Total Time (s)\n",
      "      XGBoost  0.868059   0.863590 0.868059  0.863957     {'learning_rate': 0.2, 'max_depth': 5, 'n_estimators': 100}          25.874328             0.005939       25.880267\n",
      "Random Forest  0.858942   0.853479 0.858942  0.853443 {'max_depth': 20, 'min_samples_split': 10, 'n_estimators': 100}         125.898729             0.065410      125.964139\n",
      "Decision Tree  0.848334   0.844900 0.848334  0.846228                       {'max_depth': 10, 'min_samples_split': 5}           9.028944             0.001837        9.030781\n",
      "\n",
      "------ Results Sorted by Total Time ------\n",
      "        Model  Accuracy  Precision   Recall  F1 Score                                                     Best Params  Training Time (s)  Prediction Time (s)  Total Time (s)\n",
      "Decision Tree  0.848334   0.844900 0.848334  0.846228                       {'max_depth': 10, 'min_samples_split': 5}           9.028944             0.001837        9.030781\n",
      "      XGBoost  0.868059   0.863590 0.868059  0.863957     {'learning_rate': 0.2, 'max_depth': 5, 'n_estimators': 100}          25.874328             0.005939       25.880267\n",
      "Random Forest  0.858942   0.853479 0.858942  0.853443 {'max_depth': 20, 'min_samples_split': 10, 'n_estimators': 100}         125.898729             0.065410      125.964139\n"
     ]
    }
   ],
   "source": [
    "# optimized classifiers and find their hyperparameters\n",
    "clf_opt = ClassifierComparisonOpt(X_train_val, y_train_val)\n",
    "clf_opt.fit_models()\n",
    "clf_opt.print_summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dc529395",
   "metadata": {},
   "source": [
    "### Finetune and compare again\n",
    "After tuning with a coarse grid we can use the result to select a finer grid and find better hyperparameters. This will only slightly improve the models, you will generally get diminishing returns the finer you tune."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4185a428",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Decision Tree ...\n",
      "Tuning Random Forest ...\n",
      "Tuning XGBoost ...\n",
      "\n",
      "------ Results Sorted by Accuracy ------\n",
      "        Model  Accuracy  Precision   Recall  F1 Score                                                     Best Params  Training Time (s)  Prediction Time (s)  Total Time (s)\n",
      "      XGBoost  0.868556   0.864049 0.868556  0.864181      {'learning_rate': 0.2, 'max_depth': 5, 'n_estimators': 75}          11.834481             0.006366       11.840847\n",
      "Random Forest  0.858942   0.853479 0.858942  0.853443 {'max_depth': 20, 'min_samples_split': 10, 'n_estimators': 100}         129.781899             0.075195      129.857094\n",
      "Decision Tree  0.849163   0.843062 0.849163  0.839388                        {'max_depth': 8, 'min_samples_split': 6}           1.778030             0.002314        1.780344\n",
      "\n",
      "------ Results Sorted by Total Time ------\n",
      "        Model  Accuracy  Precision   Recall  F1 Score                                                     Best Params  Training Time (s)  Prediction Time (s)  Total Time (s)\n",
      "Decision Tree  0.849163   0.843062 0.849163  0.839388                        {'max_depth': 8, 'min_samples_split': 6}           1.778030             0.002314        1.780344\n",
      "      XGBoost  0.868556   0.864049 0.868556  0.864181      {'learning_rate': 0.2, 'max_depth': 5, 'n_estimators': 75}          11.834481             0.006366       11.840847\n",
      "Random Forest  0.858942   0.853479 0.858942  0.853443 {'max_depth': 20, 'min_samples_split': 10, 'n_estimators': 100}         129.781899             0.075195      129.857094\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 80104 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 82392 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 68972 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Define the classifiers and their hyperparameters in a finer grid\n",
    "def get_fine_grids(use_bootstrap=True):\n",
    "    return {\n",
    "        'Decision Tree': (DecisionTreeClassifier(random_state=42), {\n",
    "            'max_depth': [8, 10, 12],\n",
    "            'min_samples_split': [4, 5, 6],\n",
    "        }),\n",
    "        'Random Forest': (RandomForestClassifier(bootstrap=use_bootstrap, random_state=42), {\n",
    "            'n_estimators': [75, 100, 150], \n",
    "            'max_depth': [15, 20, 25],\n",
    "            'min_samples_split': [8, 10, 12],\n",
    "        }),\n",
    "        'XGBoost': (XGBClassifier(eval_metric='mlogloss', random_state=42), {\n",
    "                'n_estimators': [75, 100, 150],\n",
    "                'learning_rate': [0.15, 0.2, 0.25],\n",
    "                'max_depth': [4, 5, 6]\n",
    "            })\n",
    "    }\n",
    "\n",
    "# optimized classifiers and find their hyperparameters\n",
    "clf_opt = ClassifierComparisonOpt(X_train_val, y_train_val)\n",
    "fine_grid = get_fine_grids()\n",
    "clf_opt.fit_models(fine_grid)\n",
    "clf_opt.print_summary()\n",
    "\n",
    "# Show feature importance for each model\n",
    "clf_opt.show_feature_importance()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "01f5866d",
   "metadata": {},
   "source": [
    "### Remove redundant features\n",
    "This section is pretty bad :(. If we want to remove redunant features we need to select different features for different models, which I have not implemented."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "776cdb17",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Decision Tree ...\n",
      "Tuning XGBoost ...\n",
      "Tuning Random Forest ...\n",
      "\n",
      "------ Results Sorted by Accuracy ------\n",
      "        Model  Accuracy  Precision   Recall  F1 Score                                                     Best Params  Training Time (s)  Prediction Time (s)  Total Time (s)\n",
      "      XGBoost  0.867893   0.863335 0.867893  0.863496    {'learning_rate': 0.15, 'max_depth': 5, 'n_estimators': 100}          20.872965             0.009083       20.882048\n",
      "Random Forest  0.858942   0.853479 0.858942  0.853443 {'max_depth': 20, 'min_samples_split': 10, 'n_estimators': 100}         140.562873             0.179120      140.741993\n",
      "Decision Tree  0.848666   0.843533 0.848666  0.837078                        {'max_depth': 8, 'min_samples_split': 4}           1.247936             0.001514        1.249449\n"
     ]
    }
   ],
   "source": [
    "# Remove redundant features for decision tree\n",
    "X_dt = X_train_val.drop(columns=['sex', 'native.country', 'marital.status', 'race', 'occupation', 'workclass'])\n",
    "\n",
    "# Remove redundant features for XGBoost\n",
    "X_xgb = X_train_val.drop(columns=['native.country', 'race'])\n",
    "\n",
    "# Train decision tree without redundant features\n",
    "cmp_dt = ClassifierComparisonOpt(X_dt, y_train_val)\n",
    "cmp_dt.fit_models({'Decision Tree': fine_grid['Decision Tree']})\n",
    "\n",
    "# Train XGBoost without redundant features\n",
    "cmp_xgb = ClassifierComparisonOpt(X_xgb, y_train_val)\n",
    "cmp_xgb.fit_models({'XGBoost': fine_grid['XGBoost']})\n",
    "\n",
    "# Train Random Forest, keep all features\n",
    "cmp_rf = ClassifierComparisonOpt(X_train_val, y_train_val)\n",
    "cmp_rf.fit_models({'Random Forest': fine_grid['Random Forest']})\n",
    "\n",
    "# Print results\n",
    "res = pd.concat([cmp_dt.results_df, cmp_rf.results_df, cmp_xgb.results_df])\n",
    "print(\"\\n------ Results Sorted by Accuracy ------\")\n",
    "print(res.sort_values(by='Accuracy', ascending=False).to_string(index=False))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6fbac4c8",
   "metadata": {},
   "source": [
    "### Final Models\n",
    "I have not run this code yet. I'm still not sure if we should remove some of the features."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a470fd20",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ---------------------------- Decision Tree ----------------------------\n",
    "dt = Pipeline([\n",
    "    ('dt', DecisionTreeClassifier(max_depth=8, min_samples_split=4, random_state=42)) \n",
    "])\n",
    "\n",
    "# Train the model\n",
    "dt.fit(X_train_val, y_train_val)\n",
    "y_pred = dt.predict(X_test)\n",
    "\n",
    "# Classification Report\n",
    "print(\"Classification Report:\")\n",
    "print(classification_report(y_val, y_pred, target_names=[\"Poor\", \"Rich\"]))\n",
    "\n",
    "\n",
    "# ---------------------------- Random Forest ----------------------------\n",
    "rf = Pipeline([\n",
    "    ('rf', RandomForestClassifier(max_depth=20, min_samples_split=10, n_estimators=100, bootstrap=True, random_state=42))\n",
    "])\n",
    "\n",
    "# Train the model\n",
    "rf.fit(X_train_val, y_train_val)\n",
    "y_pred = rf.predict(X_test)\n",
    "\n",
    "# Classification Report\n",
    "print(\"Classification Report:\")\n",
    "print(classification_report(y_val, y_pred, target_names=[\"Poor\", \"Rich\"]))\n",
    "\n",
    "\n",
    "# ---------------------------- Random Forest ----------------------------\n",
    "xgb = Pipeline([\n",
    "    ('xgb', XGBClassifier(learning_rate=0.15, max_depth=5, n_estimators=100, random_state=42))\n",
    "])\n",
    "\n",
    "# Train the model\n",
    "xgb.fit(X_train_val, y_train_val)\n",
    "y_pred = cmp_xgb.predict(X_test)\n",
    "\n",
    "# Classification Report\n",
    "print(\"Classification Report:\")\n",
    "print(classification_report(y_val, y_pred, target_names=[\"Poor\", \"Rich\"]))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
