{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2023-01-08T21:30:19.073644Z","iopub.status.busy":"2023-01-08T21:30:19.073147Z","iopub.status.idle":"2023-01-08T21:30:19.086557Z","shell.execute_reply":"2023-01-08T21:30:19.085430Z"},"papermill":{"duration":0.027109,"end_time":"2023-01-08T21:30:19.089863","exception":false,"start_time":"2023-01-08T21:30:19.062754","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd","metadata":{"papermill":{"duration":0.015869,"end_time":"2023-01-08T21:30:19.114050","exception":false,"start_time":"2023-01-08T21:30:19.098181","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-08T22:03:45.374149Z","iopub.execute_input":"2023-01-08T22:03:45.374541Z","iopub.status.idle":"2023-01-08T22:03:45.379867Z","shell.execute_reply.started":"2023-01-08T22:03:45.374511Z","shell.execute_reply":"2023-01-08T22:03:45.378696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train= pd.read_csv(\"/kaggle/input/emsi-5iir-ds/train.csv\")\ndf_test= pd.read_csv(\"/kaggle/input/emsi-5iir-ds/test.csv\")\ndf_subm= pd.read_csv(\"/kaggle/input/emsi-5iir-ds/sample_submission.csv\")","metadata":{"papermill":{"duration":15.516488,"end_time":"2023-01-08T21:30:34.637993","exception":false,"start_time":"2023-01-08T21:30:19.121505","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-08T22:03:49.608436Z","iopub.execute_input":"2023-01-08T22:03:49.608835Z","iopub.status.idle":"2023-01-08T22:04:03.160386Z","shell.execute_reply.started":"2023-01-08T22:03:49.608803Z","shell.execute_reply":"2023-01-08T22:04:03.159078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndisplay(df_train.head())\ndisplay(df_test.head())\ndisplay(df_subm.head())","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:04:59.715822Z","iopub.execute_input":"2023-01-08T22:04:59.716182Z","iopub.status.idle":"2023-01-08T22:04:59.769158Z","shell.execute_reply.started":"2023-01-08T22:04:59.716144Z","shell.execute_reply":"2023-01-08T22:04:59.768118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head()","metadata":{"execution":{"iopub.execute_input":"2023-01-08T21:30:34.654689Z","iopub.status.busy":"2023-01-08T21:30:34.653937Z","iopub.status.idle":"2023-01-08T21:30:34.689086Z","shell.execute_reply":"2023-01-08T21:30:34.687778Z"},"papermill":{"duration":0.046117,"end_time":"2023-01-08T21:30:34.691562","exception":false,"start_time":"2023-01-08T21:30:34.645445","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.info()\n","metadata":{"papermill":{"duration":1.177091,"end_time":"2023-01-08T21:30:35.876405","exception":false,"start_time":"2023-01-08T21:30:34.699314","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-08T22:05:54.829965Z","iopub.execute_input":"2023-01-08T22:05:54.830295Z","iopub.status.idle":"2023-01-08T22:05:54.913534Z","shell.execute_reply.started":"2023-01-08T22:05:54.830266Z","shell.execute_reply":"2023-01-08T22:05:54.911701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df_train.shape)\nprint(df_test.shape)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:06:54.896173Z","iopub.execute_input":"2023-01-08T22:06:54.896610Z","iopub.status.idle":"2023-01-08T22:06:54.902312Z","shell.execute_reply.started":"2023-01-08T22:06:54.896530Z","shell.execute_reply":"2023-01-08T22:06:54.900965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def val_count_df(df, column_name, sort=True):\n    value_count = df[column_name].value_counts(sort=sort).reset_index().rename(columns={column_name:\"Value Count\",\"index\":column_name}).set_index(column_name)\n    value_count[\"Percentage\"] = df[column_name].value_counts(sort=sort,normalize=True)*100\n    value_count = value_count.reset_index()\n    return value_count","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:08:23.416066Z","iopub.execute_input":"2023-01-08T22:08:23.416408Z","iopub.status.idle":"2023-01-08T22:08:23.423105Z","shell.execute_reply.started":"2023-01-08T22:08:23.416380Z","shell.execute_reply":"2023-01-08T22:08:23.421767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_count = val_count_df(df_train, \"target\")\ndisplay(target_count)\ntarget_count.set_index(\"target\").plot.pie(y=\"Value Count\", figsize=(15,5), legend=True, ylabel=\"\");","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:08:50.154215Z","iopub.execute_input":"2023-01-08T22:08:50.154541Z","iopub.status.idle":"2023-01-08T22:08:50.368416Z","shell.execute_reply.started":"2023-01-08T22:08:50.154517Z","shell.execute_reply":"2023-01-08T22:08:50.367407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef feature_27(data):\n    # This function was adapted from the following notebooks:\n    # https://www.kaggle.com/code/paultimothymooney/getting-started-with-tensorflow-decision-forests\n    for df in [data]:\n        # Extract the 10 letters of f_27 into individual features\n        for i in range(10):   # defines a number from 0 to 25 for each letter, for example if ch1 is A then get 0 and so on\n            df[f'ch{i}'] = df.f_27.str.get(i).apply(ord) - ord('A')\n            \n        \n  \n    return data","metadata":{"execution":{"iopub.execute_input":"2023-01-08T21:30:37.052677Z","iopub.status.busy":"2023-01-08T21:30:37.051881Z","iopub.status.idle":"2023-01-08T21:30:37.058397Z","shell.execute_reply":"2023-01-08T21:30:37.057266Z"},"papermill":{"duration":0.018441,"end_time":"2023-01-08T21:30:37.060624","exception":false,"start_time":"2023-01-08T21:30:37.042183","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = feature_27(df_train)\ntest_df = feature_27(df_test)\ntrain_df.columns","metadata":{"execution":{"iopub.execute_input":"2023-01-08T21:30:37.079118Z","iopub.status.busy":"2023-01-08T21:30:37.078703Z","iopub.status.idle":"2023-01-08T21:30:49.713210Z","shell.execute_reply":"2023-01-08T21:30:49.712112Z"},"papermill":{"duration":12.646471,"end_time":"2023-01-08T21:30:49.715600","exception":false,"start_time":"2023-01-08T21:30:37.069129","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df.drop(columns=['id', 'f_27'])\ntest_df = test_df.drop(columns=['id', 'f_27'])","metadata":{"execution":{"iopub.execute_input":"2023-01-08T21:30:49.922188Z","iopub.status.busy":"2023-01-08T21:30:49.921742Z","iopub.status.idle":"2023-01-08T21:30:50.236514Z","shell.execute_reply":"2023-01-08T21:30:50.235320Z"},"papermill":{"duration":0.328004,"end_time":"2023-01-08T21:30:50.239454","exception":false,"start_time":"2023-01-08T21:30:49.911450","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.columns","metadata":{"execution":{"iopub.execute_input":"2023-01-08T21:30:50.259525Z","iopub.status.busy":"2023-01-08T21:30:50.259109Z","iopub.status.idle":"2023-01-08T21:30:50.266274Z","shell.execute_reply":"2023-01-08T21:30:50.265190Z"},"papermill":{"duration":0.020158,"end_time":"2023-01-08T21:30:50.268657","exception":false,"start_time":"2023-01-08T21:30:50.248499","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from fastai.tabular.core import cont_cat_split\ncont_col, cat_col = cont_cat_split(train_df)\nprint(f'Continuous Columns: {cont_col}\\n')\nprint(f'Categorical Columns: {cat_col}')","metadata":{"execution":{"iopub.execute_input":"2023-01-08T21:30:50.288646Z","iopub.status.busy":"2023-01-08T21:30:50.288234Z","iopub.status.idle":"2023-01-08T21:30:53.304410Z","shell.execute_reply":"2023-01-08T21:30:53.302518Z"},"papermill":{"duration":3.030767,"end_time":"2023-01-08T21:30:53.308503","exception":false,"start_time":"2023-01-08T21:30:50.277736","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.execute_input":"2023-01-08T21:30:53.330254Z","iopub.status.busy":"2023-01-08T21:30:53.329248Z","iopub.status.idle":"2023-01-08T21:30:53.401463Z","shell.execute_reply":"2023-01-08T21:30:53.400227Z"},"papermill":{"duration":0.085826,"end_time":"2023-01-08T21:30:53.404381","exception":false,"start_time":"2023-01-08T21:30:53.318555","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.heatmap(abs(train_df[cont_col].corr()))","metadata":{"execution":{"iopub.execute_input":"2023-01-08T21:30:53.426150Z","iopub.status.busy":"2023-01-08T21:30:53.424916Z","iopub.status.idle":"2023-01-08T21:30:54.785914Z","shell.execute_reply":"2023-01-08T21:30:54.784576Z"},"papermill":{"duration":1.373967,"end_time":"2023-01-08T21:30:54.788412","exception":false,"start_time":"2023-01-08T21:30:53.414445","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f_cat_col = ['f_07', 'f_08', 'f_09', 'f_10', 'f_11', 'f_12', 'f_13', 'f_14', 'f_15', 'f_16', 'f_17', 'f_18', ]\nfor column in f_cat_col:    # para cada variavel na lista\n    percent = train_df[column].value_counts(normalize=True) * 100    # calculates the relative frequency of unique values in percentage\n    display(percent)\n    print('\\n')","metadata":{"execution":{"iopub.execute_input":"2023-01-08T21:30:57.124651Z","iopub.status.busy":"2023-01-08T21:30:57.124249Z","iopub.status.idle":"2023-01-08T21:30:57.281179Z","shell.execute_reply":"2023-01-08T21:30:57.279994Z"},"papermill":{"duration":0.17152,"end_time":"2023-01-08T21:30:57.284897","exception":false,"start_time":"2023-01-08T21:30:57.113377","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['f_07'] = train_df['f_07'].replace([5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], 5)\ntrain_df['f_08'] = train_df['f_08'].replace([5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16], 5)\ntrain_df['f_09'] = train_df['f_09'].replace([5, 6, 7, 8, 9, 10, 11, 12, 13, 14], 5)\ntrain_df['f_10'] = train_df['f_10'].replace([5, 6, 7, 8, 9, 10, 11, 12, 13, 14], 5)\ntrain_df['f_11'] = train_df['f_11'].replace([4, 5, 6, 7, 8, 9, 10, 11, 12, 13], 4)\ntrain_df['f_12'] = train_df['f_12'].replace([6, 7, 8, 9, 10, 11, 12, 13, 14, 16], 6)\ntrain_df['f_13'] = train_df['f_13'].replace([5, 6, 7, 8, 9, 10, 11, 12], 5)\ntrain_df['f_14'] = train_df['f_14'].replace([4, 5, 6, 7, 8, 9, 10, 11, 12, 14], 4)\ntrain_df['f_15'] = train_df['f_15'].replace([5, 6, 7, 8, 9, 10, 11, 12, 13, 14], 5)\ntrain_df['f_16'] = train_df['f_16'].replace([5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], 5)\ntrain_df['f_17'] = train_df['f_17'].replace([4, 5, 6, 7, 8, 9, 10, 11, 12, 14], 4)\ntrain_df['f_18'] = train_df['f_15'].replace([5, 6, 7, 8, 9, 10, 11, 12, 13], 5)","metadata":{"execution":{"iopub.execute_input":"2023-01-08T21:30:57.311434Z","iopub.status.busy":"2023-01-08T21:30:57.310556Z","iopub.status.idle":"2023-01-08T21:30:57.549361Z","shell.execute_reply":"2023-01-08T21:30:57.548226Z"},"papermill":{"duration":0.25517,"end_time":"2023-01-08T21:30:57.552302","exception":false,"start_time":"2023-01-08T21:30:57.297132","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for column in f_cat_col:\n    percent = train_df[column].value_counts(normalize=True) * 100\n    display(percent)\n    print('\\n')","metadata":{"execution":{"iopub.execute_input":"2023-01-08T21:30:57.577262Z","iopub.status.busy":"2023-01-08T21:30:57.576816Z","iopub.status.idle":"2023-01-08T21:30:57.730168Z","shell.execute_reply":"2023-01-08T21:30:57.728906Z"},"papermill":{"duration":0.168933,"end_time":"2023-01-08T21:30:57.733131","exception":false,"start_time":"2023-01-08T21:30:57.564198","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\nfrom sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.execute_input":"2023-01-08T21:30:57.763329Z","iopub.status.busy":"2023-01-08T21:30:57.762900Z","iopub.status.idle":"2023-01-08T21:30:57.960647Z","shell.execute_reply":"2023-01-08T21:30:57.959607Z"},"papermill":{"duration":0.215118,"end_time":"2023-01-08T21:30:57.963016","exception":false,"start_time":"2023-01-08T21:30:57.747898","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = train_df.drop(columns=['target'])    # input\ny = train_df.target    # output\n\n# split the data into training and testing\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=0, stratify=y)\n\nprint(f'X_train shape: {X_train.shape}')\nprint(f'y_train shape: {y_train.shape}')\nprint(f'-' * 27)\nprint(f'X_test shape: {X_test.shape}')\nprint(f'y_test shape: {y_test.shape}')","metadata":{"execution":{"iopub.execute_input":"2023-01-08T21:30:57.992222Z","iopub.status.busy":"2023-01-08T21:30:57.991446Z","iopub.status.idle":"2023-01-08T21:30:58.872134Z","shell.execute_reply":"2023-01-08T21:30:58.870607Z"},"papermill":{"duration":0.898835,"end_time":"2023-01-08T21:30:58.875452","exception":false,"start_time":"2023-01-08T21:30:57.976617","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tps_model = RandomForestClassifier(random_state=0)\ntps_model.fit(X_train, y_train)","metadata":{"execution":{"iopub.execute_input":"2023-01-08T21:30:58.905742Z","iopub.status.busy":"2023-01-08T21:30:58.905344Z","iopub.status.idle":"2023-01-08T21:40:24.391276Z","shell.execute_reply":"2023-01-08T21:40:24.390214Z"},"papermill":{"duration":565.517162,"end_time":"2023-01-08T21:40:24.407163","exception":false,"start_time":"2023-01-08T21:30:58.890001","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from yellowbrick.classifier import confusion_matrix\nfrom yellowbrick.classifier.rocauc import roc_auc","metadata":{"execution":{"iopub.execute_input":"2023-01-08T21:40:24.436702Z","iopub.status.busy":"2023-01-08T21:40:24.436288Z","iopub.status.idle":"2023-01-08T21:40:24.575023Z","shell.execute_reply":"2023-01-08T21:40:24.573877Z"},"papermill":{"duration":0.156526,"end_time":"2023-01-08T21:40:24.577697","exception":false,"start_time":"2023-01-08T21:40:24.421171","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(6, 6))\nconfusion_matrix(tps_model, X_train, y_train, X_test, y_test, percent=True)\nplt.tight_layout()","metadata":{"execution":{"iopub.execute_input":"2023-01-08T21:40:24.608781Z","iopub.status.busy":"2023-01-08T21:40:24.607439Z","iopub.status.idle":"2023-01-08T21:40:51.680121Z","shell.execute_reply":"2023-01-08T21:40:51.678846Z"},"papermill":{"duration":27.092182,"end_time":"2023-01-08T21:40:51.684129","exception":false,"start_time":"2023-01-08T21:40:24.591947","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(6, 6))\nroc_auc(tps_model, X_train, y_train, X_test=X_test, y_test=y_test)\nplt.tight_layout()","metadata":{"execution":{"iopub.execute_input":"2023-01-08T21:40:51.742400Z","iopub.status.busy":"2023-01-08T21:40:51.741704Z","iopub.status.idle":"2023-01-08T21:41:19.291849Z","shell.execute_reply":"2023-01-08T21:41:19.290893Z"},"papermill":{"duration":27.573074,"end_time":"2023-01-08T21:41:19.294050","exception":false,"start_time":"2023-01-08T21:40:51.720976","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission_df = pd.read_csv('/kaggle/input/emsi-5iir-ds/sample_submission.csv')\nsample_submission_df['target'] = tps_model.predict(test_df)\nsample_submission_df.to_csv('/kaggle/working/submission.csv', index=False)\n\n\nsample_submission_df","metadata":{"execution":{"iopub.execute_input":"2023-01-08T21:41:19.326568Z","iopub.status.busy":"2023-01-08T21:41:19.326157Z","iopub.status.idle":"2023-01-08T21:41:55.853899Z","shell.execute_reply":"2023-01-08T21:41:55.852621Z"},"papermill":{"duration":36.561377,"end_time":"2023-01-08T21:41:55.871057","exception":false,"start_time":"2023-01-08T21:41:19.309680","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}