{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# This notebook is derived from Kaggle Grandmaster Bojan Tunguz's NB\nhttps://www.kaggle.com/tunguz/elo-with-h2o-automl\nThis notebook merely builds on his fantastic feature engineering (194 features!) and tries to build a Deep Learning model using a new AutoML library known as Deep AutoViML. You can see its github here:\nGithub: https://github.com/AutoViML/deep_autoviml","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2022-04-04T11:43:36.785770Z","iopub.execute_input":"2022-04-04T11:43:36.786546Z","iopub.status.idle":"2022-04-04T11:43:36.830088Z","shell.execute_reply.started":"2022-04-04T11:43:36.786382Z","shell.execute_reply":"2022-04-04T11:43:36.829143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"execution":{"iopub.status.busy":"2022-04-04T11:43:36.832394Z","iopub.execute_input":"2022-04-04T11:43:36.832983Z","iopub.status.idle":"2022-04-04T11:43:36.858573Z","shell.execute_reply.started":"2022-04-04T11:43:36.832934Z","shell.execute_reply":"2022-04-04T11:43:36.857713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"../input/mitsuru-features-only-train-and-test/train_df_noindex.csv\")\ntest = pd.read_csv(\"../input/mitsuru-features-only-train-and-test/test_df_noindex.csv\")\nprint(train.shape)\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-04T11:43:36.861898Z","iopub.execute_input":"2022-04-04T11:43:36.862799Z","iopub.status.idle":"2022-04-04T11:43:50.194386Z","shell.execute_reply.started":"2022-04-04T11:43:36.862753Z","shell.execute_reply":"2022-04-04T11:43:50.193227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trf = train.sample(frac=0.01, random_state=99)\ntrf.shape","metadata":{"execution":{"iopub.status.busy":"2022-04-04T11:44:19.020105Z","iopub.execute_input":"2022-04-04T11:44:19.021055Z","iopub.status.idle":"2022-04-04T11:44:19.043924Z","shell.execute_reply.started":"2022-04-04T11:44:19.021000Z","shell.execute_reply":"2022-04-04T11:44:19.042708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#trf.to_csv('elo_train.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-04-04T11:44:24.187086Z","iopub.execute_input":"2022-04-04T11:44:24.187408Z","iopub.status.idle":"2022-04-04T11:44:24.193879Z","shell.execute_reply.started":"2022-04-04T11:44:24.187365Z","shell.execute_reply":"2022-04-04T11:44:24.192339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install deep_autoviml --upgrade","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-04-04T11:44:29.553823Z","iopub.execute_input":"2022-04-04T11:44:29.554702Z","iopub.status.idle":"2022-04-04T11:46:10.551229Z","shell.execute_reply.started":"2022-04-04T11:44:29.554658Z","shell.execute_reply":"2022-04-04T11:46:10.550054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from deep_autoviml import deep_autoviml as deepauto","metadata":{"execution":{"iopub.status.busy":"2022-04-04T11:46:14.728901Z","iopub.execute_input":"2022-04-04T11:46:14.729227Z","iopub.status.idle":"2022-04-04T11:46:19.433548Z","shell.execute_reply.started":"2022-04-04T11:46:14.729194Z","shell.execute_reply":"2022-04-04T11:46:19.432357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(test.shape)\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-04T11:46:22.287764Z","iopub.execute_input":"2022-04-04T11:46:22.288669Z","iopub.status.idle":"2022-04-04T11:46:22.487716Z","shell.execute_reply.started":"2022-04-04T11:46:22.288632Z","shell.execute_reply":"2022-04-04T11:46:22.486740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"################################################################################################\ndef EDA_find_columns_with_infinity(df):\n    \"\"\"\n    This function finds all columns in a dataframe that have inifinite values (np.inf or -np.inf)\n    It returns a list of column names. If the list is empty, it means no columns were found.\n    \"\"\"\n    add_cols = []\n    sum_cols = 0\n    for col in df.columns:\n        inf_sum1 = 0\n        inf_sum2 = 0\n        inf_sum1 = len(df[df[col]==np.inf])\n        inf_sum2 = len(df[df[col]==-np.inf])\n        if (inf_sum1 > 0) or (inf_sum2 > 0):\n            add_cols.append(col)\n            sum_cols += inf_sum1\n            sum_cols += inf_sum2\n    return add_cols\n#####################################################################################################\nimport copy\ndef FE_drop_rows_with_infinity(df, cols_list, fill_value=None):\n    \"\"\"\n    This feature engineering function will fill infinite values in your data with a fill_value.\n    You might need this function during deep_learning models where infinite values don't work.\n    You can also leave the fill_value as None which means we will drop the rows with infinity.\n    This function checks for both negative and positive infinity values to fill or remove.\n    \"\"\"\n    # first you must drop rows that have inf in them ####\n    print('    Shape of dataset initial: %s' %(df.shape[0]))\n    corr_list_copy = copy.deepcopy(cols_list)\n    init_rows = df.shape[0]\n    if fill_value:\n        for col in corr_list_copy:\n            ### Capping using the n largest value based on n given in input.\n            maxval = df[col].max()  ## what is the maximum value in this column?\n            minval = df[col].min()\n            if maxval == np.inf:\n                sorted_list = sorted(df[col].unique())\n                ### find the n_smallest values after the maximum value based on given input n\n                next_best_value_index = sorted_list.index(np.inf) - 1\n                capped_value = sorted_list[next_best_value_index]\n                df.loc[df[col]==maxval, col] =  capped_value ## maximum values are now capped\n            if minval == -np.inf:\n                sorted_list = sorted(df[col].unique())\n                ### find the n_smallest values after the maximum value based on given input n\n                next_best_value_index = sorted_list.index(-np.inf)+1\n                capped_value = sorted_list[next_best_value_index]\n                df.loc[df[col]==minval, col] =  capped_value ## maximum values are now capped\n        print('        capped all rows with infinite values in data')\n    else:\n        for col in corr_list_copy:\n            df = df[df[col]!=np.inf]\n            df = df[df[col]!=-np.inf]\n        dropped_rows = init_rows - df.shape[0]\n        print('        dropped %d rows due to infinite values in data' %dropped_rows)\n        print('    Shape of dataset after dropping rows: %s' %(df.shape[0]))\n    ###  Double check that all columns have been fixed ###############\n    cols_with_infinity = EDA_find_columns_with_infinity(df)\n    if cols_with_infinity:\n        print('There are still %d columns with infinite values. Returning...' %len(cols_with_infinity))\n    else:\n        print('There are no columns with infinite values.')\n    return df\n##################################################################################","metadata":{"execution":{"iopub.status.busy":"2022-04-04T11:46:23.242381Z","iopub.execute_input":"2022-04-04T11:46:23.242716Z","iopub.status.idle":"2022-04-04T11:46:23.258696Z","shell.execute_reply.started":"2022-04-04T11:46:23.242682Z","shell.execute_reply":"2022-04-04T11:46:23.257643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### There are 13 columns with Infinity and Negative Infinity in dataset\ninf_cols = EDA_find_columns_with_infinity(train)\nlen(inf_cols)","metadata":{"execution":{"iopub.status.busy":"2022-04-04T11:46:26.305902Z","iopub.execute_input":"2022-04-04T11:46:26.306192Z","iopub.status.idle":"2022-04-04T11:46:27.324181Z","shell.execute_reply.started":"2022-04-04T11:46:26.306161Z","shell.execute_reply":"2022-04-04T11:46:27.323036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target = \"target\"","metadata":{"execution":{"iopub.status.busy":"2022-04-04T11:46:29.644399Z","iopub.execute_input":"2022-04-04T11:46:29.644698Z","iopub.status.idle":"2022-04-04T11:46:29.650826Z","shell.execute_reply.started":"2022-04-04T11:46:29.644665Z","shell.execute_reply":"2022-04-04T11:46:29.649349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Too many inf and -inf values in train ~50% that prevents Deep Learning models from working\n### we will cap them with the next highest (or lowest) values ##","metadata":{}},{"cell_type":"code","source":"### This is going to fill infinity rows with capped values\ntrain1 = FE_drop_rows_with_infinity(train,inf_cols, True)\ntrain1.head(2)","metadata":{"execution":{"iopub.status.busy":"2022-04-04T11:46:30.836362Z","iopub.execute_input":"2022-04-04T11:46:30.836636Z","iopub.status.idle":"2022-04-04T11:46:32.275410Z","shell.execute_reply.started":"2022-04-04T11:46:30.836606Z","shell.execute_reply":"2022-04-04T11:46:32.274357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"infcols2 = EDA_find_columns_with_infinity(train1)\ninfcols2","metadata":{"execution":{"iopub.status.busy":"2022-04-04T12:14:34.200825Z","iopub.execute_input":"2022-04-04T12:14:34.201885Z","iopub.status.idle":"2022-04-04T12:14:34.444205Z","shell.execute_reply.started":"2022-04-04T12:14:34.201843Z","shell.execute_reply":"2022-04-04T12:14:34.443027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### we are going to do the same with test data ##\ntest1 = FE_drop_rows_with_infinity(test,inf_cols, True)\ntest1.head(2)","metadata":{"execution":{"iopub.status.busy":"2022-04-04T11:46:39.804328Z","iopub.execute_input":"2022-04-04T11:46:39.804616Z","iopub.status.idle":"2022-04-04T11:46:40.828893Z","shell.execute_reply.started":"2022-04-04T11:46:39.804584Z","shell.execute_reply":"2022-04-04T11:46:40.827737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train1 = train1.fillna(0)\ntest1 = test1.fillna(0)","metadata":{"execution":{"iopub.status.busy":"2022-04-04T12:16:27.659253Z","iopub.execute_input":"2022-04-04T12:16:27.659667Z","iopub.status.idle":"2022-04-04T12:16:28.242320Z","shell.execute_reply.started":"2022-04-04T12:16:27.659636Z","shell.execute_reply":"2022-04-04T12:16:28.241328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Run Deep_AutoViML using the new selected train2 and test2 data","metadata":{"execution":{"iopub.execute_input":"2021-07-02T20:26:32.42887Z","iopub.status.busy":"2021-07-02T20:26:32.428242Z","iopub.status.idle":"2021-07-02T20:26:32.434145Z","shell.execute_reply":"2021-07-02T20:26:32.433305Z","shell.execute_reply.started":"2021-07-02T20:26:32.428746Z"}}},{"cell_type":"code","source":"######   D E F A U L T S    S E T T I N G S   F O R   D E E P    A U T O  V I M L ###\nkeras_model_type =  \"big_deep\" ## always try \"fast\" first, then \"fast2\", \"auto\", etc.\n### always set early_stopping to True first and then change it to False\n#### You always need 15 max_trials to get something decent #####\n#### always set tuner to \"storm\" and then \"optuna\". \n### NLP char limit kicks off NLP processing. Feature Cross later.\nproject_name = \"Elo\"\nmodel_options = {'nlp_char_limit':50, 'cat_feat_cross_flag':False,\n                 'max_trials': 10, \"tuner\": \"storm\"}\nkeras_options = {\"patience\":10, 'class_weight': True, 'early_stopping': True, \n                 'lr_scheduler': '', \"optimizer\": ''}","metadata":{"execution":{"iopub.status.busy":"2022-04-04T12:16:28.244138Z","iopub.execute_input":"2022-04-04T12:16:28.244732Z","iopub.status.idle":"2022-04-04T12:16:28.253208Z","shell.execute_reply.started":"2022-04-04T12:16:28.244644Z","shell.execute_reply":"2022-04-04T12:16:28.252077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model, cat_vocab_dict = deepauto.fit(train1, target, keras_model_type=keras_model_type,\n                                     project_name=project_name, keras_options=keras_options,  \n                                     model_options=model_options, save_model_flag=False, use_my_model='',\n                                     model_use_case='', verbose=1)","metadata":{"execution":{"iopub.status.busy":"2022-04-04T12:16:29.298476Z","iopub.execute_input":"2022-04-04T12:16:29.298762Z","iopub.status.idle":"2022-04-04T12:17:37.643951Z","shell.execute_reply.started":"2022-04-04T12:16:29.298715Z","shell.execute_reply":"2022-04-04T12:17:37.642157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = deepauto.predict(model, project_name, test_dataset=test1,\n                                 keras_model_type=keras_model_type, \n                                 cat_vocab_dict=cat_vocab_dict)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_preds = predictions[-1]\ny_preds[:15]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = pd.Series(y_preds)\npreds.loc[pd.Series(y_preds).isnull()] = 0\npreds","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission = pd.read_csv('/kaggle/input/elo-merchant-category-recommendation/sample_submission.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission['target'] = preds.values\nsample_submission.to_csv('submission_1.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}