{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-02T13:40:46.289115Z","iopub.execute_input":"2022-05-02T13:40:46.289489Z","iopub.status.idle":"2022-05-02T13:40:46.308709Z","shell.execute_reply.started":"2022-05-02T13:40:46.289386Z","shell.execute_reply":"2022-05-02T13:40:46.308023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:40:46.393341Z","iopub.execute_input":"2022-05-02T13:40:46.394291Z","iopub.status.idle":"2022-05-02T13:40:46.973103Z","shell.execute_reply.started":"2022-05-02T13:40:46.394238Z","shell.execute_reply":"2022-05-02T13:40:46.972257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Project structure\n\n- input\n    - [x] train.csv\n    - [x] test.csv\n- src\n    - [ ] create_folds.py\n    - [ ] train.py\n    - [ ] inference.py\n    - [ ] models.py\n    - [ ] config.py\n    - [ ] model_dispatcher.py\n- models\n    - [ ] model_rf.bin\n\t- [ ] model_et.bin\n- notebooks\n\t- [ ] exploration.ipynb\n\t- [ ] check_data.ipynb\n-  [ ] readme.md\n-  [ ] license","metadata":{}},{"cell_type":"code","source":"# config vars\n\ncsv_train = '../input/mnist-in-csv/mnist_train.csv'\ncsv_test = '../input/mnist-in-csv/mnist_test.csv'\n\ncsv_output = \"mnist_train_folds.csv\"","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:40:46.974592Z","iopub.execute_input":"2022-05-02T13:40:46.974846Z","iopub.status.idle":"2022-05-02T13:40:46.978606Z","shell.execute_reply.started":"2022-05-02T13:40:46.974817Z","shell.execute_reply":"2022-05-02T13:40:46.978045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(csv_train)\n#df.info(verbose=True)\n","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:40:46.979534Z","iopub.execute_input":"2022-05-02T13:40:46.980152Z","iopub.status.idle":"2022-05-02T13:40:51.718199Z","shell.execute_reply.started":"2022-05-02T13:40:46.980117Z","shell.execute_reply":"2022-05-02T13:40:51.717274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.label.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:40:51.719785Z","iopub.execute_input":"2022-05-02T13:40:51.720272Z","iopub.status.idle":"2022-05-02T13:40:51.741600Z","shell.execute_reply.started":"2022-05-02T13:40:51.720236Z","shell.execute_reply":"2022-05-02T13:40:51.740406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# src/create_folds.py\n#creating stratified kfold by target\n\nfrom sklearn import model_selection\n\nif __name__ ==\"__main__\":\n    # load train data\n    df = pd.read_csv(csv_train)\n    \n    #create kfold column filed with -1\n    df['kfold']=-1\n    \n    #randomizew rows of data\n    df = df.sample(frac=1).reset_index(drop=True)\n    \n    #fetch targets --> changed for mnist dataset target\n    y = df.label.values\n    \n    #initiate kfold class from model selection module\n    kf = model_selection.StratifiedKFold(n_splits=5)\n    \n    #fill the new kfold column\n    for f,(t_,v_) in enumerate(kf.split(X=df, y =y)):\n        df.loc[v_,'kfold'] = f\n        \n    #save new csv with kfold colun\n    df.to_csv(csv_output, index=False)\n\n","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:40:51.742886Z","iopub.execute_input":"2022-05-02T13:40:51.743210Z","iopub.status.idle":"2022-05-02T13:41:01.846664Z","shell.execute_reply.started":"2022-05-02T13:40:51.743176Z","shell.execute_reply":"2022-05-02T13:41:01.845665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# src/train.py version p76\n# other versions were shown:\n# p79 has config.py file implemented\n# p80 has an argumnet parser to run folds from the command line \n#hardcoded: input file, target, fold numbers (5)\n\nimport joblib\nimport pandas as pd\nfrom sklearn import metrics\nfrom sklearn import tree\n\ndef run(fold):\n# read the training data with folds\n    df = pd.read_csv(\"./mnist_train_folds.csv\")\n    \n# training data is where kfold is not equal to provided fold\n# also, note that we reset the index\n    df_train = df[df.kfold != fold].reset_index(drop=True)\n\n# validation data is where kfold is equal to provided fold\n    df_valid = df[df.kfold == fold].reset_index(drop=True)\n\n# drop the label column from dataframe and convert it to\n# a numpy array by using .values.\n# target is label column in the dataframe\n    x_train = df_train.drop(\"label\", axis=1).values\n    y_train = df_train.label.values\n# similarly, for validation, we have\n    x_valid = df_valid.drop(\"label\", axis=1).values\n    y_valid = df_valid.label.values\n# initialize simple decision tree classifier from sklearn\n    clf = tree.DecisionTreeClassifier()\n# fir the model on training data\n    clf.fit(x_train, y_train)\n# create predictions for validation samples\n    preds = clf.predict(x_valid)\n# calculate & print accuracy\n    accuracy = metrics.accuracy_score(y_valid, preds) \n    print(f\"Fold={fold}, Accuracy={accuracy}\")\n\n# save the model\n    #joblib.dump(clf, f\"../models/dt_{fold}.bin\")\n    joblib.dump(clf, f\"dt_{fold}.bin\")\n\nif __name__ == \"__main__\":\n    #run(fold=input('input fold: '))\n\trun(fold=0)\n\trun(fold=1)\n\trun(fold=2)\n\trun(fold=3)\n\trun(fold=4)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:41:01.847801Z","iopub.execute_input":"2022-05-02T13:41:01.848437Z","iopub.status.idle":"2022-05-02T13:42:33.634721Z","shell.execute_reply.started":"2022-05-02T13:41:01.848401Z","shell.execute_reply":"2022-05-02T13:42:33.633717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# src/model_dispatcher.py\nfrom sklearn import tree\n\nmodels = {\n    \"decision_tree_gini\": tree.DecisionTreeClassifier(criterion='gini'),\n    \"decision_tree_entropy\": tree.DecisionTreeClassifier(criterion='entropy')\n}","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:42:33.636324Z","iopub.execute_input":"2022-05-02T13:42:33.636550Z","iopub.status.idle":"2022-05-02T13:42:33.641300Z","shell.execute_reply.started":"2022-05-02T13:42:33.636523Z","shell.execute_reply":"2022-05-02T13:42:33.640342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train.py v p82 --> implementing model_dispatcher\n# adapted for use in jupytter notebook\n\nimport argparse\nimport os\nimport joblib\nimport pandas as pd\n\nfrom sklearn import metrics\n\n# import config\n# import model_dispatcher\n\ndef run(fold, model):\n    # read the training data with folds\n    df = pd.read_csv(csv_output)\n    \n    # training data is where kfold is not equal to provided fold\n    # also, note that we reset the index\n    df_train = df[df.kfold != fold].reset_index(drop=True)\n    \n    # validation data is where kfold is equal to provided fold\n    df_valid = df[df.kfold == fold].reset_index(drop=True)\n    \n    # drop the label column from dataframe and convert it to\n    # a numpy array by using .values.\n    # target is label column in the dataframe\n    x_train = df_train.drop(\"label\", axis=1).values\n    y_train = df_train.label.values\n    \n    # similarly, for validation, we have\n    x_valid = df_valid.drop(\"label\", axis=1).values\n    y_valid = df_valid.label.values\n    \n    # fetch the model from model_dispatcher\n    clf = models[model]\n    # fit the model on training data\n    \n    clf.fit(x_train, y_train)\n    \n    # create predictions for validation samples\n    preds = clf.predict(x_valid)\n    \n    # calculate & print accuracy\n    accuracy = metrics.accuracy_score(y_valid, preds)\n    print(f\"Fold={fold}, Accuracy={accuracy}\")\n    \n    # save the model\n    joblib.dump(clf, f\"c5_dt_{fold}_{model}.bin\")\n#     joblib.dump(\n#         clf,\n#         #os.path.join(config.MODEL_OUTPUT, f\"dt_{fold}.bin\")\n#         )\n\nif __name__ == \"__main__\":\n# modified for commit:\n    fold = 1\n    model_c = '2'\n    \n# modified with input():\n#     fold_c = input('type fold number 0-4:')\n#     fold = int(fold_c)\n#     model_c = input('type model #: \\n1 for gini\\n2 for entropy:')\n#     model =''\n        \n    if model_c == '1':\n        model = 'decision_tree_gini'\n    elif model_c == '2':\n        model = 'decision_tree_entropy'\n\n# original parsed version:\n#         parser = argparse.ArgumentParser()\n        \n#         parser.add_argument(\n#             \"--fold\",\n#             type=int\n#         )\n\n#         parser.add_argument(\n#             \"--model\",\n#             type=str\n#         )\n\n#         args = parser.parse_args()\n    print(f'\\nChosen parameters: \\nFold:{fold}\\nModel: {model}')\n    run(\n        fold = fold,\n        model = model\n#        fold=args.fold,\n#        model=args.model\n        )\n\n","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:42:33.643018Z","iopub.execute_input":"2022-05-02T13:42:33.643326Z","iopub.status.idle":"2022-05-02T13:42:49.301216Z","shell.execute_reply.started":"2022-05-02T13:42:33.643285Z","shell.execute_reply":"2022-05-02T13:42:49.300109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Chapter 6 - categorical variables\n\nusing cat-in-the-dat from Categorical\nFeatures Encoding Challenge from Kaggle\nthe 2nd challenge will be used","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv('../input/cat-in-the-dat-ii/train.csv', index_col='id')","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:42:49.302592Z","iopub.execute_input":"2022-05-02T13:42:49.302884Z","iopub.status.idle":"2022-05-02T13:42:52.661910Z","shell.execute_reply.started":"2022-05-02T13:42:49.302853Z","shell.execute_reply":"2022-05-02T13:42:52.660976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# thgis dataset has Nominal , Ordinal , Cyclical and Binary Categorical vars\n\ndf","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:42:52.664454Z","iopub.execute_input":"2022-05-02T13:42:52.664741Z","iopub.status.idle":"2022-05-02T13:42:53.082754Z","shell.execute_reply.started":"2022-05-02T13:42:52.664701Z","shell.execute_reply":"2022-05-02T13:42:53.081883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check target skewness\ndf.target.value_counts().plot(kind='bar')","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:42:53.083857Z","iopub.execute_input":"2022-05-02T13:42:53.084080Z","iopub.status.idle":"2022-05-02T13:42:53.300947Z","shell.execute_reply.started":"2022-05-02T13:42:53.084052Z","shell.execute_reply":"2022-05-02T13:42:53.299953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# start EDA\ndf.info()","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:42:53.302348Z","iopub.execute_input":"2022-05-02T13:42:53.302948Z","iopub.status.idle":"2022-05-02T13:42:53.755617Z","shell.execute_reply.started":"2022-05-02T13:42:53.302908Z","shell.execute_reply":"2022-05-02T13:42:53.754714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check categori8es count for each column\n\nfor col in df.columns:\n    print(df[col].value_counts())","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:42:53.757170Z","iopub.execute_input":"2022-05-02T13:42:53.757838Z","iopub.status.idle":"2022-05-02T13:42:54.447731Z","shell.execute_reply.started":"2022-05-02T13:42:53.757799Z","shell.execute_reply":"2022-05-02T13:42:54.446773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#are there NaN  in columns? which percentage?\nisnan = df.isna().sum()\n# isnan = isnan /60000 *100\nisnan","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:42:54.449187Z","iopub.execute_input":"2022-05-02T13:42:54.449443Z","iopub.status.idle":"2022-05-02T13:42:54.889692Z","shell.execute_reply.started":"2022-05-02T13:42:54.449410Z","shell.execute_reply":"2022-05-02T13:42:54.888767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# mapping some categories using a number map\n# this is Label Encoding\n#same as in sklearn.preprocessing.LabelEnconder()\n\n# example: Name: ord_2, dtype: int64\n# Freezing       142726\n# Warm           124239\n# Cold            97822\n# Boiling Hot     84790\n# Hot             67508\n# Lava Hot        64840\n\nord2_mapping = {\n    \"Freezing\": 0,\n     \"Warm\": 1,\n     \"Cold\": 2,\n     \"Boiling Hot\": 3,\n     \"Hot\": 4,\n     \"Lava Hot\": 5\n}\n\n# convert ord_2 according to the given dictionary:\n\n# df.loc[:, \"ord_2\"] = df.ord_2.map(\"ord2_mapping\")\ndf.ord_2 = df.ord_2.map(ord2_mapping)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:42:54.891310Z","iopub.execute_input":"2022-05-02T13:42:54.891586Z","iopub.status.idle":"2022-05-02T13:42:55.085769Z","shell.execute_reply.started":"2022-05-02T13:42:54.891554Z","shell.execute_reply":"2022-05-02T13:42:55.084727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.ord_2.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:42:55.086991Z","iopub.execute_input":"2022-05-02T13:42:55.087265Z","iopub.status.idle":"2022-05-02T13:42:55.104235Z","shell.execute_reply.started":"2022-05-02T13:42:55.087230Z","shell.execute_reply":"2022-05-02T13:42:55.103279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# using LabelEncoder()\n\nimport pandas as pd\nfrom sklearn import preprocessing\n\n# read the data\ndf = pd.read_csv(\"../input/cat-in-the-dat-ii/train.csv\")\n\n# fill NaN values in ord_2 column\ndf.loc[:, \"ord_2\"] = df.ord_2.fillna(\"NONE\")\n\n# initialize LabelEncoder\nlbl_enc = preprocessing.LabelEncoder()\n\n# fit label encoder and transform values on ord_2 column\n# it sill label according to alp´habetical order\n# P.S: do not use this directly. -->  fit first, then transform\ndf.loc[:, \"ord_2\"] = lbl_enc.fit_transform(df.ord_2.values)\n","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:42:55.105713Z","iopub.execute_input":"2022-05-02T13:42:55.105957Z","iopub.status.idle":"2022-05-02T13:42:57.600176Z","shell.execute_reply.started":"2022-05-02T13:42:55.105927Z","shell.execute_reply":"2022-05-02T13:42:57.599121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.ord_2.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:42:57.601489Z","iopub.execute_input":"2022-05-02T13:42:57.601749Z","iopub.status.idle":"2022-05-02T13:42:57.612864Z","shell.execute_reply.started":"2022-05-02T13:42:57.601717Z","shell.execute_reply":"2022-05-02T13:42:57.611878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# using sparse format on an ordinal binarized feature\n# using scipy's sparse method\n\nimport numpy as np\nfrom scipy import sparse\n# create our example feature matrix\nexample = np.array(\n [\n [0, 0, 1],\n [1, 0, 0],\n [1, 0, 1]\n ]\n)\n# convert numpy array to sparse CSR matrix\nsparse_example = sparse.csr_matrix(example)\n\n# compare normal and sparse info representation\nprint(f\"\\n\\n Normal array: {example.nbytes}\\n\",\n f\"Sparse data: {sparse_example.data.nbytes}\\n\\n\")\n\n# the real size of sparse format is this:\n\nprint(\"sparse data total size: \",\n sparse_example.data.nbytes +\n sparse_example.indptr.nbytes +\n sparse_example.indices.nbytes\n)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:42:57.614383Z","iopub.execute_input":"2022-05-02T13:42:57.614946Z","iopub.status.idle":"2022-05-02T13:42:57.627556Z","shell.execute_reply.started":"2022-05-02T13:42:57.614901Z","shell.execute_reply":"2022-05-02T13:42:57.626900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sparse format makes more difference when there are lots of info\n#this cell must run alone or will brake memory usage\n\nimport numpy as np\nfrom scipy import sparse\n\n# number of rows\nn_rows = 10000\n\n# number of columns\nn_cols = 100000\n\n# create random binary matrix with only 5% values as 1s\nexample = np.random.binomial(1, p=0.05, size=(n_rows, n_cols))\n\n# print size in bytes\nprint(f\"Size of dense array: {example.nbytes}\")\n\n# convert numpy array to sparse CSR matrix\nsparse_example = sparse.csr_matrix(example)\n\n# print size of this sparse matrix\nprint(f\"Size of sparse array: {sparse_example.data.nbytes}\")\nfull_size = (\n sparse_example.data.nbytes +\n sparse_example.indptr.nbytes +\n sparse_example.indices.nbytes\n)\n\n# print full size of this sparse matrix\nprint(f\"Full size of sparse array: {full_size}\")\n","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:42:57.629368Z","iopub.execute_input":"2022-05-02T13:42:57.629672Z","iopub.status.idle":"2022-05-02T13:43:22.495690Z","shell.execute_reply.started":"2022-05-02T13:42:57.629630Z","shell.execute_reply":"2022-05-02T13:43:22.494725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# research other ways of representing a sparse matrix\n\n","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:43:22.497164Z","iopub.execute_input":"2022-05-02T13:43:22.497402Z","iopub.status.idle":"2022-05-02T13:43:22.500729Z","shell.execute_reply.started":"2022-05-02T13:43:22.497373Z","shell.execute_reply":"2022-05-02T13:43:22.500132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# One Hot Encoding may take even less memory\n\nimport numpy as np\nfrom scipy import sparse\n\n# create binary matrix\nexample = np.array(\n [\n [0, 0, 0, 0, 1, 0],\n [0, 1, 0, 0, 0, 0],\n [1, 0, 1, 0, 0, 0]\n ]\n)\n\n# print size in bytes\nprint(f\"Size of dense array: {example.nbytes}\")\n\n# convert numpy array to sparse CSR matrix\nsparse_example = sparse.csr_matrix(example)\n\n# print size of this sparse matrix\nprint(f\"Size of sparse array: {sparse_example.data.nbytes}\")\nfull_size = (\n sparse_example.data.nbytes +\n sparse_example.indptr.nbytes +\n sparse_example.indices.nbytes\n)\n\n# print full size of this sparse matrix\nprint(f\"Full size of sparse array: {full_size}\")","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:43:22.501829Z","iopub.execute_input":"2022-05-02T13:43:22.502451Z","iopub.status.idle":"2022-05-02T13:43:22.536937Z","shell.execute_reply.started":"2022-05-02T13:43:22.502420Z","shell.execute_reply":"2022-05-02T13:43:22.536006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(sparse_example[:10])","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:43:22.538279Z","iopub.execute_input":"2022-05-02T13:43:22.538599Z","iopub.status.idle":"2022-05-02T13:43:22.555708Z","shell.execute_reply.started":"2022-05-02T13:43:22.538556Z","shell.execute_reply":"2022-05-02T13:43:22.555027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#testing OHE with feature array having 1001 categories\n\nimport numpy as np\nfrom sklearn import preprocessing\n\n# create random 1-d array with 1001 different categories (int)\nexample = np.random.randint(1000, size=1000000)\n\n# initialize OneHotEncoder from scikit-learn\n# keep sparse = False to get dense array\nohe1 = preprocessing.OneHotEncoder(sparse=False)\n\n# fit and transform data with dense one hot encoder\nohe1_example = ohe1.fit_transform(example.reshape(-1, 1))\n\n# print size in bytes for dense array\nprint(f\"Size of dense array: {ohe1_example.nbytes}\")\n\n# initialize OneHotEncoder from scikit-learn\n# keep sparse = True to get sparse array\nohe2 = preprocessing.OneHotEncoder(sparse=True)\n\n# fit and transform data with sparse one-hot encoder\nohe2_example = ohe2.fit_transform(example.reshape(-1, 1))\n\n# print size of this sparse matrix\nprint(f\"Size of sparse array: {ohe2_example.data.nbytes}\")\n\nfull_size = (\n    ohe2_example.data.nbytes +\n    ohe2_example.indptr.nbytes + ohe2_example.indices.nbytes\n    )\n# print full size of this sparse matrix\nprint(f\"Full size of sparse array: {full_size}\")","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:43:22.557020Z","iopub.execute_input":"2022-05-02T13:43:22.557460Z","iopub.status.idle":"2022-05-02T13:43:27.869102Z","shell.execute_reply.started":"2022-05-02T13:43:22.557429Z","shell.execute_reply":"2022-05-02T13:43:27.868143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check out One hot encode sparsed format\nprint(ohe2_example[0:10])","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:43:27.870816Z","iopub.execute_input":"2022-05-02T13:43:27.871127Z","iopub.status.idle":"2022-05-02T13:43:27.879691Z","shell.execute_reply.started":"2022-05-02T13:43:27.871087Z","shell.execute_reply":"2022-05-02T13:43:27.878506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### THE 3 METHODS to handle categorical variables\n\n - map em all according to a map dictionary and apply map()\n - conver to decimal and then to binary encode\n - convert to One hot encoder\n","metadata":{}},{"cell_type":"code","source":"df.ord_2","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:43:27.881054Z","iopub.execute_input":"2022-05-02T13:43:27.881993Z","iopub.status.idle":"2022-05-02T13:43:27.897341Z","shell.execute_reply.started":"2022-05-02T13:43:27.881951Z","shell.execute_reply":"2022-05-02T13:43:27.896301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# how to check numbers of spcific categories?\n\ndf[df.ord_2 == \"Boiling Hot\"]","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:43:27.898781Z","iopub.execute_input":"2022-05-02T13:43:27.899120Z","iopub.status.idle":"2022-05-02T13:43:27.932104Z","shell.execute_reply.started":"2022-05-02T13:43:27.899073Z","shell.execute_reply":"2022-05-02T13:43:27.931146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.groupby([\"ord_2\"])[\"id\"].count()","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:43:27.937171Z","iopub.execute_input":"2022-05-02T13:43:27.937475Z","iopub.status.idle":"2022-05-02T13:43:27.956980Z","shell.execute_reply.started":"2022-05-02T13:43:27.937440Z","shell.execute_reply":"2022-05-02T13:43:27.955929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#replace categorical with its count , using transform\n\ndf.groupby([\"ord_2\"])[\"id\"].transform(\"count\")","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:43:27.958740Z","iopub.execute_input":"2022-05-02T13:43:27.959052Z","iopub.status.idle":"2022-05-02T13:43:27.980054Z","shell.execute_reply.started":"2022-05-02T13:43:27.959012Z","shell.execute_reply":"2022-05-02T13:43:27.979411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# join 2 categories and give its appropriate count code\n# e se tiver 2 categories com mesmo count?\n\ndf.groupby(\n    [\n        \"ord_1\",\n        \"ord_2\"\n    ])[\"id\"].count().reset_index(name=\"count\")","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:43:27.983119Z","iopub.execute_input":"2022-05-02T13:43:27.983704Z","iopub.status.idle":"2022-05-02T13:43:28.068285Z","shell.execute_reply.started":"2022-05-02T13:43:27.983659Z","shell.execute_reply":"2022-05-02T13:43:28.067648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create new features:\n# NaN's will be joined altogether\n\ndf[\"new_feature\"] = (\n    df.ord_1.astype(str)\n    + \"_\"\n    + df.ord_2.astype(str)\n    )\n\ndf.new_feature","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:43:28.069826Z","iopub.execute_input":"2022-05-02T13:43:28.070156Z","iopub.status.idle":"2022-05-02T13:43:28.544287Z","shell.execute_reply.started":"2022-05-02T13:43:28.070112Z","shell.execute_reply":"2022-05-02T13:43:28.542870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:43:28.545291Z","iopub.execute_input":"2022-05-02T13:43:28.545502Z","iopub.status.idle":"2022-05-02T13:43:28.988313Z","shell.execute_reply.started":"2022-05-02T13:43:28.545475Z","shell.execute_reply":"2022-05-02T13:43:28.987333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.ord_3.fillna('NONE').value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:43:28.991477Z","iopub.execute_input":"2022-05-02T13:43:28.991718Z","iopub.status.idle":"2022-05-02T13:43:29.054752Z","shell.execute_reply.started":"2022-05-02T13:43:28.991691Z","shell.execute_reply":"2022-05-02T13:43:29.053942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # tyhis trick is to concatenate and incorporate feature categories \n# found in test data and not in train data\n# use fit transform in the final result\n# this trick wont work in live production\n# only qwhen there is a train test\n\nimport pandas as pd\nfrom sklearn import preprocessing\n\n# read training data\ntrain = pd.read_csv(\"../input/cat-in-the-dat-ii/train.csv\")\n\n#read test data\ntest = pd.read_csv(\"../input/cat-in-the-dat-ii/test.csv\")\n\n# create a fake target column for test data\n# since this column doesn't exist\ntest.loc[:, \"target\"] = -1\n\n# concatenate both training and test data\ndata = pd.concat([train, test]).reset_index(drop=True)\n\n\n# make a list of features we are interested in\n# id and target is something we should not encode\nfeatures = [x for x in train.columns if x not in [\"id\", \"target\"]]\n\n# loop over the features list\nfor feat in features:\n# create a new instance of LabelEncoder for each feature\n    lbl_enc = preprocessing.LabelEncoder()\n\n # note the trick here\n # since its categorical data, we fillna with a string\n # and we convert all the data to string type\n # so, no matter its int or float, its converted to string\n # int/float but categorical!!!\n    temp_col = data[feat].fillna(\"NONE\").astype(str).values\n # we can use fit_transform here as we do not\n # have any extra test data that we need to\n # transform on separately\n    data.loc[:, feat] = lbl_enc.fit_transform(temp_col)\n\n# split the training and test data again\ntrain = data[data.target != -1].reset_index(drop=True)\ntest = data[data.target == -1].reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:43:29.056483Z","iopub.execute_input":"2022-05-02T13:43:29.057392Z","iopub.status.idle":"2022-05-02T13:43:44.633590Z","shell.execute_reply.started":"2022-05-02T13:43:29.057354Z","shell.execute_reply":"2022-05-02T13:43:44.632249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:43:44.635061Z","iopub.execute_input":"2022-05-02T13:43:44.635653Z","iopub.status.idle":"2022-05-02T13:43:44.714135Z","shell.execute_reply.started":"2022-05-02T13:43:44.635602Z","shell.execute_reply":"2022-05-02T13:43:44.713269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# example of RARE attribution\n\ndf.ord_4 = df.ord_4.fillna(\"NONE\")\ndf.loc[\n    df[\"ord_4\"].value_counts()[df[\"ord_4\"]].values < 2000,\"ord_4\"] = \"RARE\"","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:43:44.715404Z","iopub.execute_input":"2022-05-02T13:43:44.716263Z","iopub.status.idle":"2022-05-02T13:43:44.924172Z","shell.execute_reply.started":"2022-05-02T13:43:44.716218Z","shell.execute_reply":"2022-05-02T13:43:44.923297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.ord_4.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:43:44.925696Z","iopub.execute_input":"2022-05-02T13:43:44.926482Z","iopub.status.idle":"2022-05-02T13:43:44.959180Z","shell.execute_reply.started":"2022-05-02T13:43:44.926435Z","shell.execute_reply":"2022-05-02T13:43:44.958136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create_folds.py\n# import pandas and model_selection module of scikit-learn\nimport pandas as pd\nfrom sklearn import model_selection\n\nif __name__ == \"__main__\":\n# Read training data\n    df = pd.read_csv(\"../input/cat-in-the-dat-ii/train.csv\")\n# we create a new column called kfold and fill it with -1\n    df[\"kfold\"] = -1\n\n# the next step is to randomize the rows of the data\n    df = df.sample(frac=1).reset_index(drop=True)\n\n# fetch labels\n    y = df.target.values\n\n# initiate the kfold class from model_selection module\n    kf = model_selection.StratifiedKFold(n_splits=5)\n\n# fill the new kfold column\n    for f, (t_, v_) in enumerate(kf.split(X=df, y=y)):\n        df.loc[v_, 'kfold'] = f\n\n# save the new csv with kfold column\n    df.to_csv(\"cat_train_folds.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:43:44.960896Z","iopub.execute_input":"2022-05-02T13:43:44.961670Z","iopub.status.idle":"2022-05-02T13:43:52.863049Z","shell.execute_reply.started":"2022-05-02T13:43:44.961621Z","shell.execute_reply":"2022-05-02T13:43:52.862056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check if folds are ok \ndf = pd.read_csv(\"./cat_train_folds.csv\")\ndf.kfold.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:43:52.864148Z","iopub.execute_input":"2022-05-02T13:43:52.864367Z","iopub.status.idle":"2022-05-02T13:43:55.024110Z","shell.execute_reply.started":"2022-05-02T13:43:52.864341Z","shell.execute_reply":"2022-05-02T13:43:55.023072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check if targets are well distributed, for skewness\n\nfor i in range(0,4):\n    print(df[df.kfold==i].target.value_counts())","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:43:55.025556Z","iopub.execute_input":"2022-05-02T13:43:55.025906Z","iopub.status.idle":"2022-05-02T13:43:55.124876Z","shell.execute_reply.started":"2022-05-02T13:43:55.025864Z","shell.execute_reply":"2022-05-02T13:43:55.123908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ohe_logres.py\n# this model will hot encode data \n# and use logistic regression function \n# that splits data into training and validation, \n# given a fold number, \n# handles NaN values, \n# applies one-hot encoding on all the data \n# and trains a simple Logistic Regression model\n\nimport pandas as pd\n\nfrom sklearn import linear_model\nfrom sklearn import metrics\nfrom sklearn import preprocessing\n\ndef run(fold):\n# load the full training data with folds\n    df = pd.read_csv(\"./cat_train_folds.csv\")\n\n# all columns are features except id, target and kfold columns\n    features = [f for f in df.columns if f not in (\"id\", \"target\", \"kfold\")]\n    \n# fill all NaN values with NONE\n# note that I am converting all columns to \"strings\"\n# it doesn’t matter because all are categories\n    for col in features:\n        df.loc[:, col] = df[col].astype(str).fillna(\"NONE\")\n\n# get training data using folds\n    df_train = df[df.kfold != fold].reset_index(drop=True)\n# get validation data using folds\n    df_valid = df[df.kfold == fold].reset_index(drop=True)\n\n    # initialize OneHotEncoder from scikit-learn\n\n    ohe = preprocessing.OneHotEncoder()\n\n    # fit ohe on training + validation features\n\n    full_data = pd.concat(\n    [df_train[features], df_valid[features]],\n    axis=0\n    )\n\n\n    ohe.fit(full_data[features])\n\n# transform training data\n\n    x_train = ohe.transform(df_train[features])\n# transform validation data\n\n    x_valid = ohe.transform(df_valid[features])\n\n# initialize Logistic Regression model\n    model = linear_model.LogisticRegression()\n\n# fit model on training data (ohe)\n    model.fit(x_train, df_train.target.values)\n # predict on validation data\n # we need the probability values as we are calculating AUC\n # we will use the probability of 1s\n    valid_preds = model.predict_proba(x_valid)[:, 1]\n # get roc auc score\n    auc = metrics.roc_auc_score(df_valid.target.values, valid_preds)\n # print auc\n    print(f\"Fold = {fold}, AUC = {auc}\")\n    ","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:45:07.957260Z","iopub.execute_input":"2022-05-02T13:45:07.958075Z","iopub.status.idle":"2022-05-02T13:45:07.968493Z","shell.execute_reply.started":"2022-05-02T13:45:07.958026Z","shell.execute_reply":"2022-05-02T13:45:07.967589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__ == \"__main__\":\n    for fold_ in range(5):\n        run(fold_)\n","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:45:08.853131Z","iopub.execute_input":"2022-05-02T13:45:08.853393Z","iopub.status.idle":"2022-05-02T13:46:29.958211Z","shell.execute_reply.started":"2022-05-02T13:45:08.853365Z","shell.execute_reply":"2022-05-02T13:46:29.957343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# lbl_rf.py\n# this code applies random forest, by using Label Encoder(), not OHE\n\nimport pandas as pd\nfrom sklearn import ensemble\nfrom sklearn import metrics\nfrom sklearn import preprocessing\n\ndef run(fold):\n# load the full training data with folds\n    df = pd.read_csv(\"./cat_train_folds.csv\")\n# all columns are features except id, target and kfold columns\n    features = [\n        f for f in df.columns if f not in (\"id\", \"target\", \"kfold\")\n        ]\n# fill all NaN values with NONE\n# note that I am converting all columns to \"strings\"\n# it doesnt matter because all are categories\n    for col in features:\n        df.loc[:, col] = df[col].astype(str).fillna(\"NONE\")\n\n# now its time to label encode the features\n    for col in features:\n\n# initialize LabelEncoder for each feature column\n        lbl = preprocessing.LabelEncoder()\n\n # fit label encoder on all data\n        lbl.fit(df[col])\n # transform all the data\n        df.loc[:, col] = lbl.transform(df[col])\n\n# get training data using folds\n    df_train = df[df.kfold != fold].reset_index(drop=True)\n# get validation data using folds\n    df_valid = df[df.kfold == fold].reset_index(drop=True)\n# get training data\n    x_train = df_train[features].values\n# get validation data\n    x_valid = df_valid[features].values\n# initialize random forest model\n    model = ensemble.RandomForestClassifier(n_jobs=-1)\n# fit model on training data (ohe)\n    model.fit(x_train, df_train.target.values)\n# predict on validation data\n# we need the probability values as we are calculating AUC\n# we will use the probability of 1s\n    valid_preds = model.predict_proba(x_valid)[:, 1]\n# get roc auc score\n    auc = metrics.roc_auc_score(df_valid.target.values, valid_preds)\n# print auc\n    print(f\"Fold = {fold}, AUC = {auc}\")","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:50:20.548390Z","iopub.execute_input":"2022-05-02T13:50:20.548722Z","iopub.status.idle":"2022-05-02T13:50:20.558334Z","shell.execute_reply.started":"2022-05-02T13:50:20.548668Z","shell.execute_reply":"2022-05-02T13:50:20.557668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__ == \"__main__\":\n    for fold_ in range(5):\n        run(fold_)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T13:50:35.594155Z","iopub.execute_input":"2022-05-02T13:50:35.594697Z","iopub.status.idle":"2022-05-02T13:56:14.724972Z","shell.execute_reply.started":"2022-05-02T13:50:35.594648Z","shell.execute_reply":"2022-05-02T13:56:14.723972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ohe_svd_rf.py\n# random forest model using OHE and Single Vlaue decomposition (SVD, \n# to reduce OHE matrices\n# inference in Random forest is more time consumin g and takes larger space\n# HOT encode FULL DATA\n# fit sklearns TruncatedSVD on sparse matrix, on train + valid data\n# --> reduces high dimensional sparse matrix to 120 feats before fitting random forest\nimport pandas as pd\nfrom scipy import sparse\nfrom sklearn import decomposition\nfrom sklearn import ensemble\nfrom sklearn import metrics\nfrom sklearn import preprocessing\n\ndef run(fold):\n# load the full training data with folds\n    df = pd.read_csv(\"./cat_train_folds.csv\")\n# all columns are features except id, target and kfold columns\n    features = [\n        f for f in df.columns if f not in (\"id\", \"target\", \"kfold\")\n        ]\n# fill all NaN values with NONE\n# note that I am converting all columns to \"strings\"\n# it doesnt matter because all are categories\n    for col in features:\n        df.loc[:, col] = df[col].astype(str).fillna(\"NONE\")\n# get training data using folds\n    df_train = df[df.kfold != fold].reset_index(drop=True)\n# get validation data using folds\n    df_valid = df[df.kfold == fold].reset_index(drop=True)\n\n# initialize OneHotEncoder from scikit-learn\n    ohe = preprocessing.OneHotEncoder()\n# fit ohe on training + validation features\n    full_data = pd.concat(\n        [df_train[features], df_valid[features]],\n        axis=0\n        )\n    ohe.fit(full_data[features])\n# transform training data\n    x_train = ohe.transform(df_train[features])\n# transform validation data\n    x_valid = ohe.transform(df_valid[features])\n# initialize Truncated SVD\n# we are reducing the data to 120 components\n    svd = decomposition.TruncatedSVD(n_components=120)\n# fit svd on full sparse training data\n    full_sparse = sparse.vstack((x_train, x_valid))\n    svd.fit(full_sparse)\n# transform sparse training data\n    x_train = svd.transform(x_train)\n# transform sparse validation data\n    x_valid = svd.transform(x_valid)\n# initialize random forest model\n    model = ensemble.RandomForestClassifier(n_jobs=-1)\n# fit model on training data (ohe)\n    model.fit(x_train, df_train.target.values)\n# predict on validation data\n# we need the probability values as we are calculating AUC\n# we will use the probability of 1s\n    valid_preds = model.predict_proba(x_valid)[:, 1]\n# get roc auc score\n    auc = metrics.roc_auc_score(df_valid.target.values, valid_preds)\n# print auc\n    print(f\"Fold = {fold}, AUC = {auc}\")\n","metadata":{"execution":{"iopub.status.busy":"2022-05-02T14:04:20.097203Z","iopub.execute_input":"2022-05-02T14:04:20.097761Z","iopub.status.idle":"2022-05-02T14:04:20.108586Z","shell.execute_reply.started":"2022-05-02T14:04:20.097708Z","shell.execute_reply":"2022-05-02T14:04:20.107811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# this process is very time consumind will be turned off\n\n# if __name__ == \"__main__\":\n#     for fold_ in range(5):\n#         run(fold_)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T14:04:20.475697Z","iopub.execute_input":"2022-05-02T14:04:20.475973Z","iopub.status.idle":"2022-05-02T14:50:07.318534Z","shell.execute_reply.started":"2022-05-02T14:04:20.475945Z","shell.execute_reply":"2022-05-02T14:50:07.317695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# lbl_xgb.py\n# implementing XGBoost (Gradient Boost)tre based model using LabelEncoder\nimport pandas as pd\n\nimport xgboost as xgb\nfrom sklearn import metrics\nfrom sklearn import preprocessing\n\ndef run(fold):\n# load the full training data with folds\n    df = pd.read_csv(\"./cat_train_folds.csv\")\n# all columns are features except id, target and kfold columns\n    features = [\n        f for f in df.columns if f not in (\"id\", \"target\", \"kfold\")\n        ]\n# fill all NaN values with NONE\n# note that I am converting all columns to \"strings\"\n# it doesnt matter because all are categories\n    for col in features:\n        df.loc[:, col] = df[col].astype(str).fillna(\"NONE\")\n\n# now it’s time to label encode the features\n    for col in features:\n\n# initialize LabelEncoder for each feature column\n        lbl = preprocessing.LabelEncoder()\n\n# fit label encoder on all data\n        lbl.fit(df[col])\n# transform all the data\n        df.loc[:, col] = lbl.transform(df[col])\n# get training data using folds\n    df_train = df[df.kfold != fold].reset_index(drop=True)\n# get validation data using folds\n    df_valid = df[df.kfold == fold].reset_index(drop=True)\n# get training data\n    x_train = df_train[features].values\n# get validation data\n    x_valid = df_valid[features].values\n# initialize xgboost model\n    model = xgb.XGBClassifier(\n        n_jobs=-1,\n        max_depth=7,\n        n_estimators=200\n        )\n# fit model on training data (ohe)\n    model.fit(x_train, df_train.target.values)\n# predict on validation data\n# we need the probability values as we are calculating AUC\n# we will use the probability of 1s\n    valid_preds = model.predict_proba(x_valid)[:, 1]\n# get roc auc score\n    auc = metrics.roc_auc_score(df_valid.target.values, valid_preds)\n# print auc\n    print(f\"Fold = {fold}, AUC = {auc}\")","metadata":{"execution":{"iopub.status.busy":"2022-05-02T14:51:45.377828Z","iopub.execute_input":"2022-05-02T14:51:45.379194Z","iopub.status.idle":"2022-05-02T14:51:45.393092Z","shell.execute_reply.started":"2022-05-02T14:51:45.379134Z","shell.execute_reply":"2022-05-02T14:51:45.391849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# #this run will also be  turned off for being very time consuming \n\n# if __name__ == \"__main__\":\n#     for fold_ in range(5):\n#         run(fold_)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T14:51:45.978600Z","iopub.execute_input":"2022-05-02T14:51:45.979064Z","iopub.status.idle":"2022-05-02T15:00:41.993825Z","shell.execute_reply.started":"2022-05-02T14:51:45.979030Z","shell.execute_reply":"2022-05-02T15:00:41.992781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#lets work another good categorical rich dataset\n#us data census from UCI\n\n# df = pd.read_csv('./adult.csv')","metadata":{"execution":{"iopub.status.busy":"2022-05-02T14:50:07.475830Z","iopub.status.idle":"2022-05-02T14:50:07.476592Z","shell.execute_reply.started":"2022-05-02T14:50:07.476285Z","shell.execute_reply":"2022-05-02T14:50:07.476316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# exercise: make the cross validataion\n\n# src/create_folds.py\n# creating stratified kfold by target\n\nfrom sklearn import model_selection\n\nif __name__ ==\"__main__\":\n    # load train data\n    df = pd.read_csv('../input/adult-census-income/adult.csv')\n    \n    #create kfold column filed with -1\n    df['kfold']=-1\n    \n    #randomizew rows of data\n    df = df.sample(frac=1).reset_index(drop=True)\n    \n    #fetch targets --> changed for mnist dataset target\n    y = df.income.values\n    \n    #initiate kfold class from model selection module\n    kf = model_selection.StratifiedKFold(n_splits=5)\n    \n    #fill the new kfold column\n    for f,(t_,v_) in enumerate(kf.split(X=df, y =y)):\n        df.loc[v_,'kfold'] = f\n        \n    #save new csv with kfold colun\n    df.to_csv('us_census_folds.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T15:05:09.955437Z","iopub.execute_input":"2022-05-02T15:05:09.955765Z","iopub.status.idle":"2022-05-02T15:05:10.198846Z","shell.execute_reply.started":"2022-05-02T15:05:09.955735Z","shell.execute_reply":"2022-05-02T15:05:10.197795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ohe_logres.py\nimport pandas as pd\n\nfrom sklearn import linear_model\nfrom sklearn import metrics\nfrom sklearn import preprocessing\n\ndef run(fold):\n # load the full training data with folds\n    df = pd.read_csv(\"./us_census_folds.csv\")\n    \n # list of numerical columns\n    num_cols = [\n        \"fnlwgt\",\n        \"age\",\n        \"capital.gain\",\n        \"capital.loss\",\n        \"hours.per.week\"\n        ]\n    \n     # !!!! drop numerical columns\n    df = df.drop(num_cols, axis=1)\n\n    # map targets to 0s and 1s\n    target_mapping = {\n        \"<=50K\": 0,\n        \">50K\": 1\n    }\n    df.loc[:, \"income\"] = df.income.map(target_mapping)\n    \n# all columns are features except income and kfold columns\n    features = [\n        f for f in df.columns if f not in (\"kfold\", \"income\")\n        ]\n# fill all NaN values with NONE\n# note that I am converting all columns to \"strings\"\n# it doesnt matter because all are categories\n    for col in features:\n        df.loc[:, col] = df[col].astype(str).fillna(\"NONE\")\n\n# get training data using folds\n    df_train = df[df.kfold != fold].reset_index(drop=True)\n# get validation data using folds\n    df_valid = df[df.kfold == fold].reset_index(drop=True)\n    \n# initialize OneHotEncoder from scikit-learn\n    ohe = preprocessing.OneHotEncoder()\n# fit ohe on training + validation features\n    full_data = pd.concat(\n        [df_train[features], df_valid[features]],\n        axis=0\n        )\n    ohe.fit(full_data[features])\n\n    # transform training data\n    x_train = ohe.transform(df_train[features])\n# transform validation data\n    x_valid = ohe.transform(df_valid[features])\n\n    # initialize Logistic Regression model\n    model = linear_model.LogisticRegression()\n# fit model on training data (ohe)\n    model.fit(x_train, df_train.income.values)\n# predict on validation data\n# we need the probability values as we are calculating AUC\n# we will use the probability of 1s\n    valid_preds = model.predict_proba(x_valid)[:, 1]\n\n    # get roc auc score\n    auc = metrics.roc_auc_score(df_valid.income.values, valid_preds)\n# print auc\n    print(f\"Fold = {fold}, AUC = {auc}\")\n","metadata":{"execution":{"iopub.status.busy":"2022-05-02T15:05:28.864399Z","iopub.execute_input":"2022-05-02T15:05:28.864779Z","iopub.status.idle":"2022-05-02T15:05:28.877211Z","shell.execute_reply.started":"2022-05-02T15:05:28.864739Z","shell.execute_reply":"2022-05-02T15:05:28.876486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__ == \"__main__\":\n    for fold_ in range(5):\n        run(fold_)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T15:05:31.664313Z","iopub.execute_input":"2022-05-02T15:05:31.664600Z","iopub.status.idle":"2022-05-02T15:05:34.079047Z","shell.execute_reply.started":"2022-05-02T15:05:31.664569Z","shell.execute_reply":"2022-05-02T15:05:34.078066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# trying the label encoded XGBoost without hyperparameter tuning\n\n# lbl_xgb.py\nimport pandas as pd\nimport xgboost as xgb\n\nfrom sklearn import metrics\nfrom sklearn import preprocessing\n\ndef run(fold):\n# load the full training data with folds\n    df = pd.read_csv(\"./us_census_folds.csv\")\n# list of numerical columns\n    num_cols = [\n        \"fnlwgt\",\n        \"age\",\n        \"capital.gain\",\n        \"capital.loss\",\n        \"hours.per.week\"\n        ]\n # drop numerical columns\n    df = df.drop(num_cols, axis=1)\n # map targets to 0s and 1s\n    target_mapping = {\n        \"<=50K\": 0,\n        \">50K\": 1\n        }\n    df.loc[:, \"income\"] = df.income.map(target_mapping)\n # all columns are features except kfold & income columns\n    features = [\n        f for f in df.columns if f not in (\"kfold\", \"income\")\n        ]\n\n# fill all NaN values with NONE\n# note that I am converting all columns to \"strings\"\n# it doesnt matter because all are categories\n    for col in features:\n        df.loc[:, col] = df[col].astype(str).fillna(\"NONE\")\n\n# now its time to label encode the features\n    for col in features:\n# initialize LabelEncoder for each feature column\n        lbl = preprocessing.LabelEncoder()\n# fit label encoder on all data\n        lbl.fit(df[col])\n# transform all the data\n        df.loc[:, col] = lbl.transform(df[col])\n# get training data using folds\n    df_train = df[df.kfold != fold].reset_index(drop=True)\n# get validation data using folds\n    df_valid = df[df.kfold == fold].reset_index(drop=True)\n# get training data\n    x_train = df_train[features].values\n# get validation data\n    x_valid = df_valid[features].values\n# initialize xgboost model\n    model = xgb.XGBClassifier(\n        n_jobs=-1\n        )\n# fit model on training data (ohe)\n    model.fit(x_train, df_train.income.values)\n# predict on validation data\n# we need the probability values as we are calculating AUC\n# we will use the probability of 1s\n    valid_preds = model.predict_proba(x_valid)[:, 1]\n# get roc auc score\n    auc = metrics.roc_auc_score(df_valid.income.values, valid_preds)\n# print auc\n    print(f\"Fold = {fold}, AUC = {auc}\")","metadata":{"execution":{"iopub.status.busy":"2022-05-02T15:05:53.491512Z","iopub.execute_input":"2022-05-02T15:05:53.491825Z","iopub.status.idle":"2022-05-02T15:05:53.502773Z","shell.execute_reply.started":"2022-05-02T15:05:53.491793Z","shell.execute_reply":"2022-05-02T15:05:53.502000Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__ == \"__main__\":\n    for fold_ in range(5):\n        run(fold_)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T15:05:53.981541Z","iopub.execute_input":"2022-05-02T15:05:53.982776Z","iopub.status.idle":"2022-05-02T15:06:02.097888Z","shell.execute_reply.started":"2022-05-02T15:05:53.982722Z","shell.execute_reply":"2022-05-02T15:06:02.097276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check if last model improves when max_depth =7 and n_estimators =200\ndef run(fold):\n# load the full training data with folds\n    df = pd.read_csv(\"./us_census_folds.csv\")\n# list of numerical columns\n    num_cols = [\n        \"fnlwgt\",\n        \"age\",\n        \"capital.gain\",\n        \"capital.loss\",\n        \"hours.per.week\"\n        ]\n # drop numerical columns\n    df = df.drop(num_cols, axis=1)\n # map targets to 0s and 1s\n    target_mapping = {\n        \"<=50K\": 0,\n        \">50K\": 1\n        }\n    df.loc[:, \"income\"] = df.income.map(target_mapping)\n # all columns are features except kfold & income columns\n    features = [\n        f for f in df.columns if f not in (\"kfold\", \"income\")\n        ]\n\n# fill all NaN values with NONE\n# note that I am converting all columns to \"strings\"\n# it doesnt matter because all are categories\n    for col in features:\n        df.loc[:, col] = df[col].astype(str).fillna(\"NONE\")\n\n# now its time to label encode the features\n    for col in features:\n# initialize LabelEncoder for each feature column\n        lbl = preprocessing.LabelEncoder()\n# fit label encoder on all data\n        lbl.fit(df[col])\n# transform all the data\n        df.loc[:, col] = lbl.transform(df[col])\n# get training data using folds\n    df_train = df[df.kfold != fold].reset_index(drop=True)\n# get validation data using folds\n    df_valid = df[df.kfold == fold].reset_index(drop=True)\n# get training data\n    x_train = df_train[features].values\n# get validation data\n    x_valid = df_valid[features].values\n# initialize xgboost model\n    model = xgb.XGBClassifier(\n        n_jobs=-1,\n        n_estimators =200,\n        max_depth = 7\n        )\n# fit model on training data (ohe)\n    model.fit(x_train, df_train.income.values)\n# predict on validation data\n# we need the probability values as we are calculating AUC\n# we will use the probability of 1s\n    valid_preds = model.predict_proba(x_valid)[:, 1]\n# get roc auc score\n    auc = metrics.roc_auc_score(df_valid.income.values, valid_preds)\n# print auc\n    print(f\"Fold = {fold}, AUC = {auc}\")","metadata":{"execution":{"iopub.status.busy":"2022-05-02T15:17:49.573455Z","iopub.execute_input":"2022-05-02T15:17:49.573935Z","iopub.status.idle":"2022-05-02T15:17:49.585363Z","shell.execute_reply.started":"2022-05-02T15:17:49.573884Z","shell.execute_reply":"2022-05-02T15:17:49.584413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__ == \"__main__\":\n    for fold_ in range(5):\n        run(fold_)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T15:17:58.545763Z","iopub.execute_input":"2022-05-02T15:17:58.546067Z","iopub.status.idle":"2022-05-02T15:18:14.850471Z","shell.execute_reply.started":"2022-05-02T15:17:58.546032Z","shell.execute_reply":"2022-05-02T15:18:14.849650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# now , try XGBoost including numerical features \n# without parameter tuning p127\n\n# lbl_xgb_num.py\nimport pandas as pd\nimport xgboost as xgb\nfrom sklearn import metrics\nfrom sklearn import preprocessing\n\ndef run(fold):\n # load the full training data with folds\n    df = pd.read_csv(\"./us_census_folds.csv\")\n# list of numerical columns\n    num_cols = [\n        \"fnlwgt\",\n        \"age\",\n        \"capital.gain\",\n        \"capital.loss\",\n        \"hours.per.week\"\n        ]\n # map targets to 0s and 1s\n    target_mapping = {\n        \"<=50K\": 0,\n        \">50K\": 1\n        }\n    df.loc[:, \"income\"] = df.income.map(target_mapping)\n # all columns are features except kfold & income columns\n    features = [\n        f for f in df.columns if f not in (\"kfold\", \"income\")\n        ]\n # fill all NaN values with NONE\n # note that I am converting all columns to \"strings\"\n # it doesnt matter because all are categories\n    for col in features:\n # do not encode the numerical columns\n        if col not in num_cols:\n            df.loc[:, col] = df[col].astype(str).fillna(\"NONE\")\n\n # now its time to label encode the features\n    for col in features:\n        if col not in num_cols:\n # initialize LabelEncoder for each feature column\n            lbl = preprocessing.LabelEncoder()\n\n # fit label encoder on all data\n            lbl.fit(df[col])\n # transform all the data\n            df.loc[:, col] = lbl.transform(df[col])\n # get training data using folds\n    df_train = df[df.kfold != fold].reset_index(drop=True)\n # get validation data using folds\n    df_valid = df[df.kfold == fold].reset_index(drop=True)\n # get training data\n    x_train = df_train[features].values\n # get validation data\n    x_valid = df_valid[features].values\n # initialize xgboost model\n    model = xgb.XGBClassifier(\n        n_jobs=-1\n        )\n # fit model on training data (ohe)\n    model.fit(x_train, df_train.income.values)\n # predict on validation data\n # we need the probability values as we are calculating AUC\n # we will use the probability of 1s\n    valid_preds = model.predict_proba(x_valid)[:, 1]\n # get roc auc score\n    auc = metrics.roc_auc_score(df_valid.income.values, valid_preds)\n # print auc\n    print(f\"Fold = {fold}, AUC = {auc}\")","metadata":{"execution":{"iopub.status.busy":"2022-05-02T15:23:23.180381Z","iopub.execute_input":"2022-05-02T15:23:23.180930Z","iopub.status.idle":"2022-05-02T15:23:23.194803Z","shell.execute_reply.started":"2022-05-02T15:23:23.180888Z","shell.execute_reply":"2022-05-02T15:23:23.193809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__ == \"__main__\":\n    for fold_ in range(5):\n        run(fold_)\n","metadata":{"execution":{"iopub.status.busy":"2022-05-02T15:23:24.672477Z","iopub.execute_input":"2022-05-02T15:23:24.673017Z","iopub.status.idle":"2022-05-02T15:23:34.441038Z","shell.execute_reply.started":"2022-05-02T15:23:24.672977Z","shell.execute_reply":"2022-05-02T15:23:34.439965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#label encoded categ features + numerical features + feat engineering\n# + XGBoost\n# lbl_xgb_num_feat.py p130\n\nimport itertools\nimport pandas as pd\nimport xgboost as xgb\n\nfrom sklearn import metrics\nfrom sklearn import preprocessing\n\ndef feature_engineering(df, cat_cols):\n    \"\"\"\n    This function is used for feature engineering\n    :param df: the pandas dataframe with train/test data\n    :param cat_cols: list of categorical columns\n    :return: dataframe with new features\n    \"\"\"\n # this will create all 2-combinations of values\n # in this list\n # for example:\n # list(itertools.combinations([1,2,3], 2)) will return\n # [(1, 2), (1, 3), (2, 3)]\n    combi = list(itertools.combinations(cat_cols, 2))\n    for c1, c2 in combi:\n        df.loc[\n            :,\n            c1 + \"_\" + c2\n            ] = df[c1].astype(str) + \"_\" + df[c2].astype(str)\n    return df\n\ndef run(fold):\n # load the full training data with folds\n    df = pd.read_csv(\"./us_census_folds.csv\")\n # list of numerical columns\n    num_cols = [\n         \"fnlwgt\",\n         \"age\",\n         \"capital.gain\",\n         \"capital.loss\",\n         \"hours.per.week\"\n         ]\n # map targets to 0s and 1s\n    target_mapping = {\n        \"<=50K\": 0,\n        \">50K\": 1\n         }\n    df.loc[:, \"income\"] = df.income.map(target_mapping)\n # list of categorical columns for feature engineering\n    cat_cols = [\n        c for c in df.columns if c not in num_cols\n        and c not in (\"kfold\", \"income\")\n        ]\n # add new features\n    df = feature_engineering(df, cat_cols)\n # all columns are features except kfold & income columns\n    features = [\n        f for f in df.columns if f not in (\"kfold\", \"income\")\n        ]\n # fill all NaN values with NONE\n # note that I am converting all columns to \"strings\"\n # it doesnt matter because all are categories\n    for col in features:\n # do not encode the numerical columns\n        if col not in num_cols:\n            df.loc[:, col] = df[col].astype(str).fillna(\"NONE\")\n\n # now its time to label encode the features\n    for col in features:\n        if col not in num_cols:\n # initialize LabelEncoder for each feature column\n            lbl = preprocessing.LabelEncoder()\n\n # fit label encoder on all data\n            lbl.fit(df[col])\n # transform all the data\n            df.loc[:, col] = lbl.transform(df[col])\n # get training data using folds\n    df_train = df[df.kfold != fold].reset_index(drop=True)\n # get validation data using folds\n    df_valid = df[df.kfold == fold].reset_index(drop=True)\n# get training data\n    x_train = df_train[features].values\n # get validation data\n    x_valid = df_valid[features].values\n # initialize xgboost model\n    model = xgb.XGBClassifier(\n        n_jobs=-1\n        )\n # fit model on training data (ohe)\n    model.fit(x_train, df_train.income.values)\n # predict on validation data\n # we need the probability values as we are calculating AUC\n # we will use the probability of 1s\n    valid_preds = model.predict_proba(x_valid)[:, 1]\n # get roc auc score\n    auc = metrics.roc_auc_score(df_valid.income.values, valid_preds)\n # print auc\n    print(f\"Fold = {fold}, AUC = {auc}\")\n","metadata":{"execution":{"iopub.status.busy":"2022-05-02T15:34:18.873441Z","iopub.execute_input":"2022-05-02T15:34:18.874321Z","iopub.status.idle":"2022-05-02T15:34:18.887888Z","shell.execute_reply.started":"2022-05-02T15:34:18.874279Z","shell.execute_reply":"2022-05-02T15:34:18.886942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__ == \"__main__\":\n    for fold_ in range(5):\n        run(fold_)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T15:34:19.794827Z","iopub.execute_input":"2022-05-02T15:34:19.795869Z","iopub.status.idle":"2022-05-02T15:34:43.067153Z","shell.execute_reply.started":"2022-05-02T15:34:19.795805Z","shell.execute_reply":"2022-05-02T15:34:43.066227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# run the same last model \n# but tune hyperparam max_depth to 7\n\nimport itertools\nimport pandas as pd\nimport xgboost as xgb\n\nfrom sklearn import metrics\nfrom sklearn import preprocessing\n\ndef feature_engineering(df, cat_cols):\n    \"\"\"\n    This function is used for feature engineering\n    :param df: the pandas dataframe with train/test data\n    :param cat_cols: list of categorical columns\n    :return: dataframe with new features\n    \"\"\"\n # this will create all 2-combinations of values\n # in this list\n # for example:\n # list(itertools.combinations([1,2,3], 2)) will return\n # [(1, 2), (1, 3), (2, 3)]\n    combi = list(itertools.combinations(cat_cols, 2))\n    for c1, c2 in combi:\n        df.loc[\n            :,\n            c1 + \"_\" + c2\n            ] = df[c1].astype(str) + \"_\" + df[c2].astype(str)\n    return df\n\ndef run(fold):\n # load the full training data with folds\n    df = pd.read_csv(\"./us_census_folds.csv\")\n # list of numerical columns\n    num_cols = [\n         \"fnlwgt\",\n         \"age\",\n         \"capital.gain\",\n         \"capital.loss\",\n         \"hours.per.week\"\n         ]\n # map targets to 0s and 1s\n    target_mapping = {\n        \"<=50K\": 0,\n        \">50K\": 1\n         }\n    df.loc[:, \"income\"] = df.income.map(target_mapping)\n # list of categorical columns for feature engineering\n    cat_cols = [\n        c for c in df.columns if c not in num_cols\n        and c not in (\"kfold\", \"income\")\n        ]\n # add new features\n    df = feature_engineering(df, cat_cols)\n # all columns are features except kfold & income columns\n    features = [\n        f for f in df.columns if f not in (\"kfold\", \"income\")\n        ]\n # fill all NaN values with NONE\n # note that I am converting all columns to \"strings\"\n # it doesnt matter because all are categories\n    for col in features:\n # do not encode the numerical columns\n        if col not in num_cols:\n            df.loc[:, col] = df[col].astype(str).fillna(\"NONE\")\n\n # now its time to label encode the features\n    for col in features:\n        if col not in num_cols:\n # initialize LabelEncoder for each feature column\n            lbl = preprocessing.LabelEncoder()\n\n # fit label encoder on all data\n            lbl.fit(df[col])\n # transform all the data\n            df.loc[:, col] = lbl.transform(df[col])\n # get training data using folds\n    df_train = df[df.kfold != fold].reset_index(drop=True)\n # get validation data using folds\n    df_valid = df[df.kfold == fold].reset_index(drop=True)\n# get training data\n    x_train = df_train[features].values\n # get validation data\n    x_valid = df_valid[features].values\n # initialize xgboost model\n    model = xgb.XGBClassifier(\n        n_jobs=-1,\n        max_depth = 7\n        )\n # fit model on training data (ohe)\n    model.fit(x_train, df_train.income.values)\n # predict on validation data\n # we need the probability values as we are calculating AUC\n # we will use the probability of 1s\n    valid_preds = model.predict_proba(x_valid)[:, 1]\n # get roc auc score\n    auc = metrics.roc_auc_score(df_valid.income.values, valid_preds)\n # print auc\n    print(f\"Fold = {fold}, AUC = {auc}\")","metadata":{"execution":{"iopub.status.busy":"2022-05-02T15:35:59.424440Z","iopub.execute_input":"2022-05-02T15:35:59.424953Z","iopub.status.idle":"2022-05-02T15:35:59.438115Z","shell.execute_reply.started":"2022-05-02T15:35:59.424913Z","shell.execute_reply":"2022-05-02T15:35:59.437264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__ == \"__main__\":\n    for fold_ in range(5):\n        run(fold_)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T15:36:10.397558Z","iopub.execute_input":"2022-05-02T15:36:10.398140Z","iopub.status.idle":"2022-05-02T15:36:36.982407Z","shell.execute_reply.started":"2022-05-02T15:36:10.398103Z","shell.execute_reply":"2022-05-02T15:36:36.981741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# you can also encode RARE values \n# binary featuyres\n# combine OHE and Label\n# and several other methods","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#another optyion is to use target encoding\n\n# target_encoding.py\nimport copy\nimport pandas as pd\n\nfrom sklearn import metrics\nfrom sklearn import preprocessing\nimport xgboost as xgb\n\ndef mean_target_encoding(data):\n# make a copy of dataframe\n    df = copy.deepcopy(data)\n# list of numerical columns\n    num_cols = [\n        \"fnlwgt\",\n        \"age\",\n        \"capital.gain\",\n        \"capital.loss\",\n        \"hours.per.week\"\n    ]\n    \n # map targets to 0s and 1s\n    target_mapping = {\n        \"<=50K\": 0,\n        \">50K\": 1\n    }\n    \n    df.loc[:, \"income\"] = df.income.map(target_mapping)\n\n# all columns are features except income and kfold columns\n    features = [\n        f for f in df.columns if f not in (\"kfold\", \"income\")\n        and f not in num_cols\n        ]\n # all columns are features except kfold & income columns\n    features = [\n        f for f in df.columns if f not in (\"kfold\", \"income\")\n        ]\n # fill all NaN values with NONE\n # note that I am converting all columns to \"strings\"\n # it doesnt matter because all are categories\n    for col in features:\n # do not encode the numerical columns\n        if col not in num_cols:\n            df.loc[:, col] = df[col].astype(str).fillna(\"NONE\")\n\n# now its time to label encode the features\n    for col in features:\n        if col not in num_cols:\n # initialize LabelEncoder for each feature column\n            lbl = preprocessing.LabelEncoder()\n\n # fit label encoder on all data\n            lbl.fit(df[col])\n# transform all the data\n            df.loc[:, col] = lbl.transform(df[col])\n # a list to store 5 validation dataframes\n    encoded_dfs = []\n # go over all folds\n    for fold in range(5):\n # fetch training and validation data\n        df_train = df[df.kfold != fold].reset_index(drop=True)\n        df_valid = df[df.kfold == fold].reset_index(drop=True)\n # for all feature columns, i.e. categorical columns\n        for column in features:\n # create dict of category:mean target\n            mapping_dict = dict(\n                df_train.groupby(column)[\"income\"].mean()\n            )\n # column_enc is the new column we have with mean encoding\n            df_valid.loc[\n                :, column + \"_enc\"\n            ] = df_valid[column].map(mapping_dict)\n # append to our list of encoded validation dataframes\n        encoded_dfs.append(df_valid)\n # create full data frame again and return\n    encoded_df = pd.concat(encoded_dfs, axis=0)\n    return encoded_df\n\ndef run(df, fold):\n # note that folds are same as before\n # get training data using folds\n    df_train = df[df.kfold != fold].reset_index(drop=True)\n # get validation data using folds\n    df_valid = df[df.kfold == fold].reset_index(drop=True)\n # all columns are features except income and kfold columns\n    features = [\n        f for f in df.columns if f not in (\"kfold\", \"income\")\n    ]\n # scale training data\n    x_train = df_train[features].values\n # scale validation data\n    x_valid = df_valid[features].values\n # initialize xgboost model\n    model = xgb.XGBClassifier(\n        n_jobs=-1,\n        max_depth=7\n        )\n # fit model on training data (ohe)\n    model.fit(x_train, df_train.income.values)\n # predict on validation data\n # we need the probability values as we are calculating AUC\n # we will use the probability of 1s\n    valid_preds = model.predict_proba(x_valid)[:, 1]\n # get roc auc score\n    auc = metrics.roc_auc_score(df_valid.income.values, valid_preds)\n # print auc\n    print(f\"Fold = {fold}, AUC = {auc}\")\n","metadata":{"execution":{"iopub.status.busy":"2022-05-02T15:48:13.531761Z","iopub.execute_input":"2022-05-02T15:48:13.532052Z","iopub.status.idle":"2022-05-02T15:48:13.548617Z","shell.execute_reply.started":"2022-05-02T15:48:13.532025Z","shell.execute_reply":"2022-05-02T15:48:13.547904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__ == \"__main__\":\n # read data\n    df = pd.read_csv(\"./us_census_folds.csv\")\n\n # create mean target encoded categories and\n # munge data\n    df = mean_target_encoding(df)\n # run training and validation for 5 folds\n    for fold_ in range(5):\n        run(df, fold_)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T15:48:14.318723Z","iopub.execute_input":"2022-05-02T15:48:14.319478Z","iopub.status.idle":"2022-05-02T15:48:30.253152Z","shell.execute_reply.started":"2022-05-02T15:48:14.319434Z","shell.execute_reply":"2022-05-02T15:48:30.252564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# # entity_embeddings.py p138\n# using on the cat train dataset\n\nimport os\nimport gc\nimport joblib\nimport pandas as pd\nimport numpy as np\nfrom sklearn import metrics, preprocessing\nfrom tensorflow.keras import layers\n\nfrom tensorflow.keras import optimizers\nfrom tensorflow.keras.models import Model, load_model\nfrom tensorflow.keras import callbacks\nfrom tensorflow.keras import backend as K\nfrom tensorflow.keras import utils\n\ndef create_model(data, catcols):\n    \"\"\"\n    This function returns a compiled tf.keras model\n    for entity embeddings\n    :param data: this is a pandas dataframe\n    :param catcols: list of categorical column names\n    :return: compiled tf.keras model\n    \"\"\"\n # init list of inputs for embeddings\n    inputs = []\n # init list of outputs for embeddings\n    outputs = []\n # loop over all categorical columns\n    for c in catcols:\n # find the number of unique values in the column\n        num_unique_values = int(data[c].nunique())\n    \n # simple dimension of embedding calculator\n # min size is half of the number of unique values\n # max size is 50. max size depends on the number of unique\n # categories too. 50 is quite sufficient most of the times\n # but if you have millions of unique values, you might need\n # a larger dimension\n        embed_dim = int(min(np.ceil((num_unique_values)/2), 50))\n # simple keras input layer with size 1\n        inp = layers.Input(shape=(1,))\n # add embedding layer to raw input\n # embedding size is always 1 more than unique values in input\n        out = layers.Embedding(\n            num_unique_values + 1, embed_dim, name=c\n        )(inp)\n\n # 1-d spatial dropout is the standard for emebedding layers\n # you can use it in NLP tasks too\n        out = layers.SpatialDropout1D(0.3)(out)\n # reshape the input to the dimension of embedding\n # this becomes our output layer for current feature\n        out = layers.Reshape(target_shape=(embed_dim, ))(out)\n # add input to input list\n        inputs.append(inp)\n # add output to output list\n        outputs.append(out)\n\n # concatenate all output layers\n    x = layers.Concatenate()(outputs)\n # add a batchnorm layer.\n # from here, everything is up to you\n # you can try different architectures\n # this is the architecture I like to use\n # if you have numerical features, you should add\n # them here or in concatenate layer\n    x = layers.BatchNormalization()(x)\n\n # a bunch of dense layers with dropout.\n # start with 1 or two layers only\n    x = layers.Dense(300, activation=\"relu\")(x)\n    x = layers.Dropout(0.3)(x)\n    x = layers.BatchNormalization()(x)\n\n    x = layers.Dense(300, activation=\"relu\")(x)\n    x = layers.Dropout(0.3)(x)\n    x = layers.BatchNormalization()(x)\n # using softmax and treating it as a two class problem\n # you can also use sigmoid, then you need to use only one\n # output class\n    y = layers.Dense(2, activation=\"softmax\")(x)\n # create final model\n    model = Model(inputs=inputs, outputs=y)\n # compile the model\n # we use adam and binary cross entropy.\n # feel free to use something else and see how model behaves\n    model.compile(loss='binary_crossentropy', optimizer='adam')\n    return model\n\ndef run(fold):\n # load the full training data with folds\n    df = pd.read_csv(\"./cat_train_folds.csv\")\n # all columns are features except id, target and kfold columns\n    features = [\n        f for f in df.columns if f not in (\"id\", \"target\", \"kfold\")\n        ]\n # fill all NaN values with NONE\n # note that I am converting all columns to \"strings\"\n # it doesnt matter because all are categories\n    for col in features:\n        df.loc[:, col] = df[col].astype(str).fillna(\"NONE\")\n # encode all features with label encoder individually\n # in a live setting you need to save all label encoders\n    for feat in features:\n        lbl_enc = preprocessing.LabelEncoder()\n        df.loc[:, feat] = lbl_enc.fit_transform(df[feat].values)\n # get training data using folds\n    df_train = df[df.kfold != fold].reset_index(drop=True)\n # get validation data using folds\n    df_valid = df[df.kfold == fold].reset_index(drop=True)\n # create tf.keras model\n    model = create_model(df, features)\n # our features are lists of lists\n    xtrain = [\n        df_train[features].values[:, k] for k in range(len(features))\n        ]\n    xvalid = [\n        df_valid[features].values[:, k] for k in range(len(features))\n        ]\n # fetch target columns\n    ytrain = df_train.target.values\n    yvalid = df_valid.target.values\n # convert target columns to categories\n# this is just binarization\n    ytrain_cat = utils.to_categorical(ytrain)\n    yvalid_cat = utils.to_categorical(yvalid)\n\n # fit the model\n    model.fit(xtrain,\n        ytrain_cat,\n        validation_data=(xvalid, yvalid_cat),\n        verbose=1,\n        batch_size=1024,\n        epochs=3\n        )\n # generate validation predictions\n    valid_preds = model.predict(xvalid)[:, 1]\n # print roc auc score\n    print(metrics.roc_auc_score(yvalid, valid_preds))\n # clear session to free up some GPU memory\n    K.clear_session()\n\n","metadata":{"execution":{"iopub.status.busy":"2022-05-02T16:15:27.182488Z","iopub.execute_input":"2022-05-02T16:15:27.182817Z","iopub.status.idle":"2022-05-02T16:15:27.201430Z","shell.execute_reply.started":"2022-05-02T16:15:27.182785Z","shell.execute_reply":"2022-05-02T16:15:27.200427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__ == \"__main__\":\n    run(0)\n    run(1)\n    run(2)\n    run(3)\n    run(4)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T16:15:28.591809Z","iopub.execute_input":"2022-05-02T16:15:28.592131Z","iopub.status.idle":"2022-05-02T16:23:01.767081Z","shell.execute_reply.started":"2022-05-02T16:15:28.592091Z","shell.execute_reply":"2022-05-02T16:23:01.765838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"./us_census_folds.csv\")\ndf.head().T","metadata":{"execution":{"iopub.status.busy":"2022-05-02T16:38:00.499344Z","iopub.execute_input":"2022-05-02T16:38:00.499647Z","iopub.status.idle":"2022-05-02T16:38:00.566480Z","shell.execute_reply.started":"2022-05-02T16:38:00.499615Z","shell.execute_reply":"2022-05-02T16:38:00.565492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# run TF KERAS entity embedding on us census data\n\ndef run(fold):\n # load the full training data with folds\n    df = pd.read_csv(\"./us_census_folds.csv\")\n # all columns are features except id, target and kfold columns\n    features = [\n        f for f in df.columns if f not in (\"id\", \"income\", \"kfold\")\n        ]\n    \n     # map targets to 0s and 1s\n    target_mapping = {\n        \"<=50K\": 0,\n        \">50K\": 1\n    }\n    \n    df.loc[:, \"income\"] = df.income.map(target_mapping)\n # fill all NaN values with NONE\n # note that I am converting all columns to \"strings\"\n # it doesnt matter because all are categories\n    for col in features:\n        df.loc[:, col] = df[col].astype(str).fillna(\"NONE\")\n # encode all features with label encoder individually\n # in a live setting you need to save all label encoders\n    for feat in features:\n        lbl_enc = preprocessing.LabelEncoder()\n        df.loc[:, feat] = lbl_enc.fit_transform(df[feat].values)\n # get training data using folds\n    df_train = df[df.kfold != fold].reset_index(drop=True)\n # get validation data using folds\n    df_valid = df[df.kfold == fold].reset_index(drop=True)\n # create tf.keras model\n    model = create_model(df, features)\n # our features are lists of lists\n    xtrain = [\n        df_train[features].values[:, k] for k in range(len(features))\n        ]\n    xvalid = [\n        df_valid[features].values[:, k] for k in range(len(features))\n        ]\n # fetch target columns\n    ytrain = df_train.income.values\n    yvalid = df_valid.income.values\n # convert target columns to categories\n# this is just binarization\n    ytrain_cat = utils.to_categorical(ytrain)\n    yvalid_cat = utils.to_categorical(yvalid)\n\n # fit the model\n    model.fit(xtrain,\n        ytrain_cat,\n        validation_data=(xvalid, yvalid_cat),\n        verbose=1,\n        batch_size=1024,\n        epochs=3\n        )\n # generate validation predictions\n    valid_preds = model.predict(xvalid)[:, 1]\n # print roc auc score\n    print(metrics.roc_auc_score(yvalid, valid_preds))\n # clear session to free up some GPU memory\n    K.clear_session()","metadata":{"execution":{"iopub.status.busy":"2022-05-02T16:40:55.608799Z","iopub.execute_input":"2022-05-02T16:40:55.609226Z","iopub.status.idle":"2022-05-02T16:40:55.623912Z","shell.execute_reply.started":"2022-05-02T16:40:55.609195Z","shell.execute_reply":"2022-05-02T16:40:55.622792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__ == \"__main__\":\n    run(0)\n    run(1)\n    run(2)\n    run(3)\n    run(4)","metadata":{"execution":{"iopub.status.busy":"2022-05-02T16:40:56.476102Z","iopub.execute_input":"2022-05-02T16:40:56.476642Z","iopub.status.idle":"2022-05-02T16:41:46.514565Z","shell.execute_reply.started":"2022-05-02T16:40:56.476607Z","shell.execute_reply":"2022-05-02T16:41:46.513589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}