{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "8b66d061",
   "metadata": {},
   "source": [
    "### Creates a correlation matrix\n",
    "This used to see if we should remove or combine some features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "557ed2b5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 180900 characters]",
      "text/plain": [
       "<Figure size 800x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 30162 entries, 1 to 32560\n",
      "Data columns (total 13 columns):\n",
      " #   Column          Non-Null Count  Dtype\n",
      "---  ------          --------------  -----\n",
      " 0   age             30162 non-null  int64\n",
      " 1   workclass       30162 non-null  int64\n",
      " 2   education.num   30162 non-null  int64\n",
      " 3   marital.status  30162 non-null  int64\n",
      " 4   occupation      30162 non-null  int64\n",
      " 5   relationship    30162 non-null  int64\n",
      " 6   race            30162 non-null  int64\n",
      " 7   sex             30162 non-null  int64\n",
      " 8   capital.gain    30162 non-null  int64\n",
      " 9   capital.loss    30162 non-null  int64\n",
      " 10  hours.per.week  30162 non-null  int64\n",
      " 11  native.country  30162 non-null  int64\n",
      " 12  income          30162 non-null  int64\n",
      "dtypes: int64(13)\n",
      "memory usage: 3.2 MB\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "import seaborn as sns\n",
    "\n",
    "\n",
    "df = pd.read_csv('./Datasets/adult.csv', comment = '#')\n",
    "\n",
    "\n",
    "# First, create a copy of the dataframe to avoid modifying the original\n",
    "df_encoded = df.copy()\n",
    "df_encoded.drop(['fnlwgt', 'education'], axis=1, inplace=True)\n",
    "\n",
    "\n",
    "# drop all rows that contain '?'\n",
    "for column in df_encoded.columns:\n",
    "    df_encoded = df_encoded[df_encoded[column] != '?']\n",
    "\n",
    "# Apply label encoding to categorical columns\n",
    "label_encoder = LabelEncoder()\n",
    "categorical_columns = ['workclass', 'marital.status', 'occupation', \n",
    "                      'relationship', 'race', 'sex', 'native.country', 'income']\n",
    "\n",
    "for column in categorical_columns:\n",
    "    df_encoded[column] = label_encoder.fit_transform(df_encoded[column])\n",
    "\n",
    "\n",
    "matrix = df_encoded.corr()\n",
    "\n",
    "plt.figure(figsize=(8,6))\n",
    "sns.heatmap(matrix, annot=True, cmap=\"coolwarm\", fmt=\".2f\", linewidths=0.5)\n",
    "plt.title(\"Correlation Heatmap\")\n",
    "plt.show()\n",
    "\n",
    "#df_encoded.head(10)\n",
    "#df_encoded.info()\n",
    "df.head(20)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e55b361b",
   "metadata": {},
   "source": [
    "### Creates a pairplot\n",
    "This demonstrates that we need a model that is appropriate for discrete data (Decision Tree, Random Forest, Naive Bayes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "8e3bd968",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 1814256 characters]",
      "text/plain": [
       "<Figure size 3250x3250 with 182 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.pairplot(df_encoded, diag_kind=\"kde\")\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
