{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-09-04T17:15:37.531416Z","iopub.execute_input":"2021-09-04T17:15:37.531773Z","iopub.status.idle":"2021-09-04T17:15:37.542065Z","shell.execute_reply.started":"2021-09-04T17:15:37.531741Z","shell.execute_reply":"2021-09-04T17:15:37.541079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv(\"/kaggle/input/beyond-analysis/train.csv\").drop_duplicates().sample(frac=1).reset_index(drop=True)\ndf_dev = pd.read_csv(\"/kaggle/input/beyond-analysis/test.csv\")\ndf_val = pd.read_csv(\"/kaggle/input/beyond-analysis/sample_submission_random.csv\")\n\nkwn_ukn_cat = pd.concat((df_train[\"UNIQUE_IDENTIFIER\"], df_dev[\"UNIQUE_IDENTIFIER\"])).drop_duplicates().sort_values().reset_index(drop=True)\nunq_idt = pd.DataFrame(kwn_ukn_cat, columns=['UNIQUE_IDENTIFIER'])","metadata":{"execution":{"iopub.status.busy":"2021-09-04T17:15:37.660067Z","iopub.execute_input":"2021-09-04T17:15:37.660316Z","iopub.status.idle":"2021-09-04T17:15:43.896706Z","shell.execute_reply.started":"2021-09-04T17:15:37.660292Z","shell.execute_reply":"2021-09-04T17:15:43.895734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.options.display.max_columns = None\n# pd.options.display.max_rows = None\ndf_train.head(100)","metadata":{"execution":{"iopub.status.busy":"2021-09-04T17:15:43.898198Z","iopub.execute_input":"2021-09-04T17:15:43.898578Z","iopub.status.idle":"2021-09-04T17:15:43.940553Z","shell.execute_reply.started":"2021-09-04T17:15:43.898537Z","shell.execute_reply":"2021-09-04T17:15:43.939624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"code","source":"df_train.describe()","metadata":{"execution":{"iopub.status.busy":"2021-09-04T17:15:43.942438Z","iopub.execute_input":"2021-09-04T17:15:43.942786Z","iopub.status.idle":"2021-09-04T17:15:44.700430Z","shell.execute_reply.started":"2021-09-04T17:15:43.942752Z","shell.execute_reply":"2021-09-04T17:15:44.699383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Preprocessing**","metadata":{}},{"cell_type":"code","source":"CATEGORICAL_COLUMNS = [\"SEQUENCE_NO\", \"STATUS_CHECK\", \"CATEGORY_1\", \"CATEGORY_2\"]\n\n# let's typecast categorical features to a category dtype, it decreases computation time\ndf_train[CATEGORICAL_COLUMNS] = df_train[CATEGORICAL_COLUMNS].astype('category')\n# df_train[\"UNIQUE_IDENTIFIER\"] = df_train[\"UNIQUE_IDENTIFIER\"].astype('category')\ndf_dev[CATEGORICAL_COLUMNS] = df_dev[CATEGORICAL_COLUMNS].astype('category')\n# df_dev[\"UNIQUE_IDENTIFIER\"] = df_dev[\"UNIQUE_IDENTIFIER\"].astype('category')","metadata":{"execution":{"iopub.status.busy":"2021-09-04T17:15:44.702283Z","iopub.execute_input":"2021-09-04T17:15:44.702676Z","iopub.status.idle":"2021-09-04T17:15:45.232712Z","shell.execute_reply.started":"2021-09-04T17:15:44.702622Z","shell.execute_reply":"2021-09-04T17:15:45.231820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, col in enumerate(CATEGORICAL_COLUMNS):\n    plt.figure(figsize=(20, 40))\n    vl_cnt = df_train[col].value_counts()\n    plt.subplot(len(CATEGORICAL_COLUMNS), 2, 2*i+1)\n    plt.title(col+' (bar)', fontsize=20)\n    plt.bar(vl_cnt.index, vl_cnt.values)\n    plt.subplot(len(CATEGORICAL_COLUMNS), 2, 2*i+2)\n    plt.title(col+' (pie)', fontsize=20)\n    plt.pie(vl_cnt.values, labels=vl_cnt.index)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-09-04T17:15:45.234108Z","iopub.execute_input":"2021-09-04T17:15:45.234676Z","iopub.status.idle":"2021-09-04T17:15:46.781673Z","shell.execute_reply.started":"2021-09-04T17:15:45.234625Z","shell.execute_reply":"2021-09-04T17:15:46.780751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Ordinal Encoding (order matters)","metadata":{}},{"cell_type":"code","source":"# labels_cat_1 = df_train[\"CATEGORY_1\"].cat.categories.tolist()\n# labels_cat_2 = df_train[\"CATEGORY_2\"].cat.categories.tolist()\n# labels_unq_i = df_train[\"UNIQUE_IDENTIFIER\"].cat.categories.tolist()\n\n# abt_num_map = {'CATEGORY_1' : {k: v for k,v in zip(labels_cat_1,np.arange(1,len(labels_cat_1)+1))},\n#                'CATEGORY_2' : {k: v for k,v in zip(labels_cat_2,np.arange(1,len(labels_cat_2)+1))},\n#                'UNIQUE_IDENTIFIER' : {k: v for k,v in zip(labels_unq_i,np.arange(1,len(labels_unq_i)+1))}}\n# df_train['CATEGORY_1'] = df_train['CATEGORY_1'].cat.rename_categories(abt_num_map['CATEGORY_1'])\n# df_train['CATEGORY_2'] = df_train['CATEGORY_2'].cat.rename_categories(abt_num_map['CATEGORY_2'])\n# df_train['UNIQUE_IDENTIFIER'] = df_train['UNIQUE_IDENTIFIER'].cat.rename_categories(abt_num_map['UNIQUE_IDENTIFIER'])","metadata":{"execution":{"iopub.status.busy":"2021-09-04T17:15:46.783140Z","iopub.execute_input":"2021-09-04T17:15:46.783475Z","iopub.status.idle":"2021-09-04T17:15:46.787761Z","shell.execute_reply.started":"2021-09-04T17:15:46.783438Z","shell.execute_reply":"2021-09-04T17:15:46.786808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### For Y1\n\n* 4\tREVENUE\t0.198019\n* 1\tSTATUS_CHECK\t0.106235\n* 0\tSEQUENCE_NO\t0.096067\n* 5\tWINNINGS_1\t0.091930\n* 16\tPRACTICE_ENTRY_NUMBER\t0.065968\n* 11\tWITHDRAW_NUMBER\t0.060643\n* 7\tDISCOUNT\t0.049099\n\n### For Y2\n\n* 0\tSEQUENCE_NO\t0.484686\n* 1\tSTATUS_CHECK\t0.289134\n* 13\tWINNINGS_NUMBER\t0.040856\n* 12\tENTRY_NUMBER\t0.026605\n* 9\tDEPOSIT_NUMBER\t0.025906","metadata":{}},{"cell_type":"code","source":"adn_cols = ['UNIQUE_IDENTIFIER', 'REVENUE', 'WINNINGS_1', 'PRACTICE_ENTRY_NUMBER', 'WITHDRAW_NUMBER', 'DISCOUNT', 'ENTRY_NUMBER']\ndf_adn_train = df_train.loc[:, adn_cols]\ndf_adn_dev = df_dev.loc[:, adn_cols]","metadata":{"execution":{"iopub.status.busy":"2021-09-04T17:15:46.789118Z","iopub.execute_input":"2021-09-04T17:15:46.789487Z","iopub.status.idle":"2021-09-04T17:15:46.830256Z","shell.execute_reply.started":"2021-09-04T17:15:46.789451Z","shell.execute_reply":"2021-09-04T17:15:46.829131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# remove everything except first 5 and last 2 cols\ndf_train = pd.concat((df_train.iloc[:,:5], df_train.iloc[:,-2:]), axis=1)\ndf_dev = df_dev.iloc[:,:5]","metadata":{"execution":{"iopub.status.busy":"2021-09-04T17:15:46.832559Z","iopub.execute_input":"2021-09-04T17:15:46.832947Z","iopub.status.idle":"2021-09-04T17:15:46.860368Z","shell.execute_reply.started":"2021-09-04T17:15:46.832904Z","shell.execute_reply":"2021-09-04T17:15:46.859640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Remove outliers with **Interquartile range(IQR)**","metadata":{}},{"cell_type":"code","source":"non_cat_cols = df_train.select_dtypes(include=['float64']).columns.to_list()\n# stops preprocessing of Y1, Y2(last two cols)\n# non_cat_cols.pop()    # pop Y2\n# non_cat_cols.pop()    # pop Y1\n# identify outliers\nQ25 = df_train[non_cat_cols].quantile(0.25)\nQ75 = df_train[non_cat_cols].quantile(0.75)\nIQR = Q75 - Q25\nwhisker_width = 1.5\nwhisker_infimum = Q25-whisker_width*IQR\nwhisker_supremum = Q75+whisker_width*IQR\noutliers = (df_train[non_cat_cols] < whisker_infimum) | (df_train[non_cat_cols] > whisker_supremum)\n\n# capping and flooring outliers\n# df_train[non_cat_cols] = df_train[non_cat_cols].clip(lower=whisker_infimum, upper=whisker_supremum, axis=1)\n\n# remove outliers\ndf_train[non_cat_cols] = df_train[non_cat_cols][~outliers]\ndf_train = df_train.dropna()","metadata":{"execution":{"iopub.status.busy":"2021-09-04T17:15:46.861546Z","iopub.execute_input":"2021-09-04T17:15:46.861910Z","iopub.status.idle":"2021-09-04T17:15:47.058241Z","shell.execute_reply.started":"2021-09-04T17:15:46.861871Z","shell.execute_reply":"2021-09-04T17:15:47.057377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unscaled_Y = df_train.loc[:, ['Y1', 'Y2']]","metadata":{"execution":{"iopub.status.busy":"2021-09-04T17:15:47.060577Z","iopub.execute_input":"2021-09-04T17:15:47.061087Z","iopub.status.idle":"2021-09-04T17:15:47.069320Z","shell.execute_reply.started":"2021-09-04T17:15:47.061048Z","shell.execute_reply":"2021-09-04T17:15:47.068368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## IQR Robust Scaler Transform (used to scale data with `high outliers`)\n\n\n\n","metadata":{}},{"cell_type":"code","source":"non_cat_cols = df_train.select_dtypes(include=['float64']).columns.to_list()\n# calculating required quartiles\nQ25 = df_train[non_cat_cols].quantile(0.25)\nQ50 = df_train[non_cat_cols].quantile(0.50)\nQ75 = df_train[non_cat_cols].quantile(0.75)\n\ndf_train[non_cat_cols] = (df_train[non_cat_cols]-Q50) / (Q75-Q25)","metadata":{"execution":{"iopub.status.busy":"2021-09-04T17:15:47.071602Z","iopub.execute_input":"2021-09-04T17:15:47.072061Z","iopub.status.idle":"2021-09-04T17:15:47.230860Z","shell.execute_reply.started":"2021-09-04T17:15:47.072027Z","shell.execute_reply":"2021-09-04T17:15:47.230016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Z-score Normalization** (these cols have all Q25,Q50,Q75 = 0)","metadata":{}},{"cell_type":"code","source":"mean = df_adn_train.mean()\nstd = df_adn_train.std()\ndf_adn_train = (df_adn_train-mean) / std\ndf_adn_dev = (df_adn_dev-mean) / std","metadata":{"execution":{"iopub.status.busy":"2021-09-04T17:15:47.232109Z","iopub.execute_input":"2021-09-04T17:15:47.232486Z","iopub.status.idle":"2021-09-04T17:15:47.348196Z","shell.execute_reply.started":"2021-09-04T17:15:47.232452Z","shell.execute_reply":"2021-09-04T17:15:47.347192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Baysian encoding (if there are large no.of categories `'UNIQUE_IDENTIFIER'`)","metadata":{}},{"cell_type":"code","source":"category_map = {}\nfor col in CATEGORICAL_COLUMNS:\n    df_mean = df_train.groupby([col])['Y1'].mean() + df_train.groupby([col])['Y2'].mean()\n    mean_encode_map = df_mean.to_dict()\n    category_map[col] = mean_encode_map\n    df_train[col] =  df_train[col].map(mean_encode_map)\n    df_dev[col] =  df_dev[col].map(mean_encode_map)","metadata":{"execution":{"iopub.status.busy":"2021-09-04T17:15:48.750560Z","iopub.execute_input":"2021-09-04T17:15:48.750891Z","iopub.status.idle":"2021-09-04T17:15:48.827383Z","shell.execute_reply.started":"2021-09-04T17:15:48.750860Z","shell.execute_reply":"2021-09-04T17:15:48.826562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# unq_idt['UNIQUE_IDENTIFIER_ENCODED'] = unq_idt[\"UNIQUE_IDENTIFIER\"].map(category_map['UNIQUE_IDENTIFIER'])\n# # df.interpolate() fills nan intelegently\n# unq_idt['UNIQUE_IDENTIFIER_ENCODED'] = unq_idt['UNIQUE_IDENTIFIER_ENCODED'].interpolate()\n# unq_idt = dict(zip(unq_idt['UNIQUE_IDENTIFIER'], unq_idt['UNIQUE_IDENTIFIER_ENCODED']))\n# df_dev['UNIQUE_IDENTIFIER'] =  df_dev['UNIQUE_IDENTIFIER'].map(unq_idt)","metadata":{"execution":{"iopub.status.busy":"2021-09-04T17:15:49.871012Z","iopub.execute_input":"2021-09-04T17:15:49.871340Z","iopub.status.idle":"2021-09-04T17:15:49.877270Z","shell.execute_reply.started":"2021-09-04T17:15:49.871310Z","shell.execute_reply":"2021-09-04T17:15:49.876264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[adn_cols] = df_adn_train[adn_cols]\ndf_dev[adn_cols] = df_adn_dev[adn_cols]","metadata":{"execution":{"iopub.status.busy":"2021-09-04T17:15:51.005888Z","iopub.execute_input":"2021-09-04T17:15:51.006219Z","iopub.status.idle":"2021-09-04T17:15:51.256621Z","shell.execute_reply.started":"2021-09-04T17:15:51.006188Z","shell.execute_reply":"2021-09-04T17:15:51.255688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.loc[:, ['Y1']] = unscaled_Y['Y1']","metadata":{"execution":{"iopub.status.busy":"2021-09-04T17:15:51.258154Z","iopub.execute_input":"2021-09-04T17:15:51.258508Z","iopub.status.idle":"2021-09-04T17:15:51.269859Z","shell.execute_reply.started":"2021-09-04T17:15:51.258473Z","shell.execute_reply":"2021-09-04T17:15:51.268988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# so that we can take mean of categorical_cols \ndf_train[CATEGORICAL_COLUMNS] = df_train[CATEGORICAL_COLUMNS].to_numpy()\ndf_dev[CATEGORICAL_COLUMNS] = df_dev[CATEGORICAL_COLUMNS].to_numpy()\n\ndf_train = df_train.groupby('UNIQUE_IDENTIFIER').mean()\ndf_train = df_train.reset_index()\ndf_dev = df_dev.groupby('UNIQUE_IDENTIFIER').mean()\ndf_dev = df_dev.reset_index()","metadata":{"execution":{"iopub.status.busy":"2021-09-04T17:15:51.580409Z","iopub.execute_input":"2021-09-04T17:15:51.580750Z","iopub.status.idle":"2021-09-04T17:15:53.864853Z","shell.execute_reply.started":"2021-09-04T17:15:51.580718Z","shell.execute_reply":"2021-09-04T17:15:53.863979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Divide into train and dev","metadata":{}},{"cell_type":"code","source":"train=df_train.sample(frac=0.98)\ndev = df_train.drop(train.index)\nY1_train = train.pop(\"Y1\").to_numpy()\nY2_train = train.pop(\"Y2\").to_numpy()\nX_train = train.to_numpy()\n\nY1_dev = dev.pop(\"Y1\").to_numpy()\nY2_dev = dev.pop(\"Y2\").to_numpy()\nX_dev = dev.to_numpy(copy=True)\n\nX_train.shape, X_dev.shape, Y1_train.shape, Y1_dev.shape","metadata":{"execution":{"iopub.status.busy":"2021-09-04T17:15:55.225541Z","iopub.execute_input":"2021-09-04T17:15:55.225898Z","iopub.status.idle":"2021-09-04T17:15:55.280666Z","shell.execute_reply.started":"2021-09-04T17:15:55.225868Z","shell.execute_reply":"2021-09-04T17:15:55.279691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"# BATCH_SIZE = 128*8\n# n_a = [X_train.shape[1], 300, 100, 50, 1]\n# rm_prob = 0\n# steps_per_epoch = len(X_train)//BATCH_SIZE + 1","metadata":{"execution":{"iopub.status.busy":"2021-09-04T17:15:57.505813Z","iopub.execute_input":"2021-09-04T17:15:57.506142Z","iopub.status.idle":"2021-09-04T17:15:57.510054Z","shell.execute_reply.started":"2021-09-04T17:15:57.506114Z","shell.execute_reply":"2021-09-04T17:15:57.508889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def forward_prop(n_a, rm_prob=0):\n\n#     X = tf.keras.layers.Input(shape=(n_a[0],))        # (m, n_X)\n\n#     # Layer1 dense-->batchNorm-->reLu-->dropout\n#     A1 = tf.keras.layers.Dense(units=n_a[1])(X)                 # (m, n_a1)\n#     A1 = tf.keras.layers.BatchNormalization()(A1)               # (m, n_a1)\n#     A1 = tf.keras.layers.Activation(activation='relu')(A1)      # (m, n_a1)\n#     A1 = tf.keras.layers.Dropout(rm_prob)(A1)                   # (m, n_a1)\n\n#     # Layer2 dense-->batchNorm-->reLu-->dropout\n#     A2 = tf.keras.layers.Dense(units=n_a[2])(A1)                # (m, n_a2)\n#     A2 = tf.keras.layers.BatchNormalization()(A2)               # (m, n_a2)\n#     A2 = tf.keras.layers.Activation(activation='relu')(A2)      # (m, n_a2)\n#     A2 = tf.keras.layers.Dropout(rm_prob)(A2)                   # (m, n_a2)\n\n#     # Layer3 dense-->batchNorm-->reLu-->dropout\n#     A3 = tf.keras.layers.Dense(units=n_a[3])(A2)                # (m, n_a3)\n#     A3 = tf.keras.layers.BatchNormalization()(A3)               # (m, n_a3)\n#     A3 = tf.keras.layers.Activation(activation='relu')(A3)      # (m, n_a3)\n#     A3 = tf.keras.layers.Dropout(rm_prob)(A3)                   # (m, n_a3)\n\n#     # Layer4 (output)\n#     Y = tf.keras.layers.Dense(units=n_a[4], activation='linear')(A3)    # (m, 1)\n\n#     model = tf.keras.Model(inputs=X, outputs=Y)\n#     return model","metadata":{"execution":{"iopub.status.busy":"2021-09-04T17:15:57.960386Z","iopub.execute_input":"2021-09-04T17:15:57.960718Z","iopub.status.idle":"2021-09-04T17:15:57.964596Z","shell.execute_reply.started":"2021-09-04T17:15:57.960687Z","shell.execute_reply":"2021-09-04T17:15:57.963658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# MultiTasking","metadata":{}},{"cell_type":"code","source":"BATCH_SIZE = 128*8\nn_a = [X_train.shape[1], 220, 110, [55,55], [55,55], 1]\nsteps_per_epoch = len(X_train)//BATCH_SIZE + 1","metadata":{"execution":{"iopub.status.busy":"2021-09-04T17:18:54.161529Z","iopub.execute_input":"2021-09-04T17:18:54.161929Z","iopub.status.idle":"2021-09-04T17:18:54.166990Z","shell.execute_reply.started":"2021-09-04T17:18:54.161898Z","shell.execute_reply":"2021-09-04T17:18:54.165794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def forward_prop(n_a, rm_prob=0):\n\n    X = tf.keras.layers.Input(shape=(n_a[0],))        # (m, n_X)\n\n    # common-Layer1 dense-->batchNorm-->reLu-->dropout\n    A1 = tf.keras.layers.Dense(units=n_a[1])(X)                 # (m, n_a1)\n    A1 = tf.keras.layers.BatchNormalization()(A1)               # (m, n_a1)\n    A1 = tf.keras.layers.Activation(activation='relu')(A1)      # (m, n_a1)\n    A1 = tf.keras.layers.Dropout(rm_prob)(A1)                   # (m, n_a1)\n\n    # common-Layer2 dense-->batchNorm-->reLu-->dropout\n    A2 = tf.keras.layers.Dense(units=n_a[2])(A1)                # (m, n_a2)\n    A2 = tf.keras.layers.BatchNormalization()(A2)               # (m, n_a2)\n    A2 = tf.keras.layers.Activation(activation='relu')(A2)      # (m, n_a2)\n    A2 = tf.keras.layers.Dropout(rm_prob)(A2)                   # (m, n_a2)\n\n    # Y1-Layer3 dense-->batchNorm-->reLu-->dropout\n    y1_A3 = tf.keras.layers.Dense(units=n_a[3][0])(A2)                # (m, n_a3)\n    y1_A3 = tf.keras.layers.BatchNormalization()(y1_A3)               # (m, n_a3)\n    y1_A3 = tf.keras.layers.Activation(activation='relu')(y1_A3)      # (m, n_a3)\n    y1_A3 = tf.keras.layers.Dropout(rm_prob)(y1_A3)                   # (m, n_a3)\n    \n    # Y1-Layer4 dense-->batchNorm-->reLu-->dropout\n    y1_A4 = tf.keras.layers.Dense(units=n_a[4][0])(y1_A3)             # (m, n_a3)\n    y1_A4 = tf.keras.layers.BatchNormalization()(y1_A4)               # (m, n_a3)\n    y1_A4 = tf.keras.layers.Activation(activation='relu')(y1_A4)      # (m, n_a3)\n    y1_A4 = tf.keras.layers.Dropout(rm_prob)(y1_A4)                   # (m, n_a3)\n    \n    # y1-Layer4 (output)\n    Y1 = tf.keras.layers.Dense(units=n_a[5], activation='linear', name='output1')(y1_A4)    # (m, 1)\n    \n    # Y2-Layer3 dense-->batchNorm-->reLu-->dropout\n    y2_A3 = tf.keras.layers.Dense(units=n_a[3][1])(A2)                # (m, n_a4)\n    y2_A3 = tf.keras.layers.BatchNormalization()(y2_A3)               # (m, n_a4)\n    y2_A3 = tf.keras.layers.Activation(activation='relu')(y2_A3)      # (m, n_a4)\n    y2_A3 = tf.keras.layers.Dropout(rm_prob)(y2_A3)                   # (m, n_a4)\n    \n    # Y2-Layer4 dense-->batchNorm-->reLu-->dropout\n    y2_A4 = tf.keras.layers.Dense(units=n_a[3][1])(y2_A3)             # (m, n_a4)\n    y2_A4 = tf.keras.layers.BatchNormalization()(y2_A4)               # (m, n_a4)\n    y2_A4 = tf.keras.layers.Activation(activation='relu')(y2_A4)      # (m, n_a4)\n    y2_A4 = tf.keras.layers.Dropout(rm_prob)(y2_A4)                   # (m, n_a4)\n\n    # y2-Layer4 (output)\n    Y2 = tf.keras.layers.Dense(units=n_a[5], activation='linear', name='output2')(y2_A4)    # (m, 1)\n\n    model = tf.keras.Model(inputs=X, outputs=[Y1,Y2])\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-09-04T17:18:54.436602Z","iopub.execute_input":"2021-09-04T17:18:54.436976Z","iopub.status.idle":"2021-09-04T17:18:54.451972Z","shell.execute_reply.started":"2021-09-04T17:18:54.436934Z","shell.execute_reply":"2021-09-04T17:18:54.451103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = forward_prop(n_a, rm_prob=0)\nmodel.summary()\nmodel.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.00005),\n              loss=tf.keras.losses.MeanSquaredError(),\n              metrics=[tf.keras.metrics.RootMeanSquaredError()])","metadata":{"execution":{"iopub.status.busy":"2021-09-04T17:18:55.000921Z","iopub.execute_input":"2021-09-04T17:18:55.001234Z","iopub.status.idle":"2021-09-04T17:18:55.157491Z","shell.execute_reply.started":"2021-09-04T17:18:55.001205Z","shell.execute_reply":"2021-09-04T17:18:55.156701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Let's Train","metadata":{}},{"cell_type":"code","source":"train_dataset = tf.data.Dataset.from_tensor_slices((X_train,(Y1_train,Y2_train)))\ntrain_dataset = train_dataset.shuffle(buffer_size=X_train.shape[0], reshuffle_each_iteration=True)\ntrain_dataset = train_dataset.batch(batch_size=BATCH_SIZE)\ntrain_dataset = train_dataset.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)\n\ndev_dataset = tf.data.Dataset.from_tensor_slices((X_dev, (Y1_dev,Y2_dev)))\ndev_dataset = dev_dataset.shuffle(buffer_size=X_dev.shape[0], reshuffle_each_iteration=True)\ndev_dataset = dev_dataset.batch(batch_size=BATCH_SIZE)\ndev_dataset = dev_dataset.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)","metadata":{"execution":{"iopub.status.busy":"2021-09-04T17:18:56.055950Z","iopub.execute_input":"2021-09-04T17:18:56.056283Z","iopub.status.idle":"2021-09-04T17:18:56.069958Z","shell.execute_reply.started":"2021-09-04T17:18:56.056254Z","shell.execute_reply":"2021-09-04T17:18:56.068922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from datetime import datetime\ncheckpoint_dir = \"../working/checkpoints\"\ncheckpoint_filepath = checkpoint_dir + \"/cp-eph-{epoch:04d}.ckpt\"\nlogs_dir = \"../working/logs/fit/\" + datetime.now().strftime(\"%Y-%m-%d_%H:%M:%S\")","metadata":{"execution":{"iopub.status.busy":"2021-09-04T17:18:57.380821Z","iopub.execute_input":"2021-09-04T17:18:57.381159Z","iopub.status.idle":"2021-09-04T17:18:57.388094Z","shell.execute_reply.started":"2021-09-04T17:18:57.381129Z","shell.execute_reply":"2021-09-04T17:18:57.386969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prev_epoch=0\n# latest = tf.train.latest_checkpoint(checkpoint_dir)\n# if latest:\n#     model.load_weights(latest)\n#     prev_epoch = int(latest[-9:-5])\n\nCheckpoint_cbk = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_filepath, save_weights_only=True, monitor='loss', save_freq=steps_per_epoch*2)\nEarlyStopping_cbk = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=3)\nTensorBoard_cbk = tf.keras.callbacks.TensorBoard(log_dir=logs_dir, histogram_freq=1)\n\nhistory = model.fit(train_dataset, \n                    epochs=200, \n                    callbacks=[Checkpoint_cbk, EarlyStopping_cbk, TensorBoard_cbk],\n                    validation_data=dev_dataset,\n                    initial_epoch=prev_epoch)","metadata":{"scrolled":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-09-04T17:19:03.421879Z","iopub.execute_input":"2021-09-04T17:19:03.422192Z","iopub.status.idle":"2021-09-04T17:20:18.385340Z","shell.execute_reply.started":"2021-09-04T17:19:03.422161Z","shell.execute_reply":"2021-09-04T17:20:18.384568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame(history.history)","metadata":{"execution":{"iopub.status.busy":"2021-09-04T17:20:18.386823Z","iopub.execute_input":"2021-09-04T17:20:18.387116Z","iopub.status.idle":"2021-09-04T17:20:18.410861Z","shell.execute_reply.started":"2021-09-04T17:20:18.387090Z","shell.execute_reply":"2021-09-04T17:20:18.409424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y1, Y2 = model.predict(df_dev.to_numpy())","metadata":{"execution":{"iopub.status.busy":"2021-09-04T17:20:46.366240Z","iopub.execute_input":"2021-09-04T17:20:46.366682Z","iopub.status.idle":"2021-09-04T17:20:49.002902Z","shell.execute_reply.started":"2021-09-04T17:20:46.366625Z","shell.execute_reply":"2021-09-04T17:20:49.002011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"___\n___ \n___","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(20, 10))\nplt.title('Overall loss including both Y1, Y2', fontsize=20)\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])","metadata":{"execution":{"iopub.status.busy":"2021-09-04T17:20:49.006533Z","iopub.execute_input":"2021-09-04T17:20:49.006799Z","iopub.status.idle":"2021-09-04T17:20:49.203239Z","shell.execute_reply.started":"2021-09-04T17:20:49.006774Z","shell.execute_reply":"2021-09-04T17:20:49.202479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20, 15))\n# output_1\nplt.subplot(2, 2, 1)\nplt.title('output1_loss', fontsize=20)\nplt.plot(history.history['output1_loss'])\nplt.plot(history.history['val_output1_loss'])\nplt.subplot(2, 2, 2)\nplt.title('output1_root_mean_squared_error', fontsize=20)\nplt.plot(history.history['output1_root_mean_squared_error'])\nplt.plot(history.history['val_output1_root_mean_squared_error'])\n# output_2\nplt.subplot(2, 2, 3)\nplt.title('output2_loss', fontsize=20)\nplt.plot(history.history['output2_loss'])\nplt.plot(history.history['val_output2_loss'])\nplt.subplot(2, 2, 4)\nplt.title('output2_root_mean_squared_error', fontsize=20)\nplt.plot(history.history['output2_root_mean_squared_error'])\nplt.plot(history.history['val_output2_root_mean_squared_error'])\n","metadata":{"execution":{"iopub.status.busy":"2021-09-04T17:20:49.204885Z","iopub.execute_input":"2021-09-04T17:20:49.205244Z","iopub.status.idle":"2021-09-04T17:20:49.794158Z","shell.execute_reply.started":"2021-09-04T17:20:49.205208Z","shell.execute_reply":"2021-09-04T17:20:49.793354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%load_ext tensorboard\n%tensorboard --logdir \"../logs\"","metadata":{"_kg_hide-output":true,"_kg_hide-input":false,"execution":{"iopub.status.busy":"2021-09-04T17:08:06.960666Z","iopub.execute_input":"2021-09-04T17:08:06.960985Z","iopub.status.idle":"2021-09-04T17:08:11.037343Z","shell.execute_reply.started":"2021-09-04T17:08:06.960957Z","shell.execute_reply":"2021-09-04T17:08:11.036277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!tensorboard dev upload \\\n  --logdir \"../logs\" \\\n  --name \"beyond-analysis 1.0.0\" \\\n  --description \"our first project\" \\\n  --one_shot\n--yes","metadata":{"execution":{"iopub.status.busy":"2021-08-30T05:38:00.039066Z","iopub.execute_input":"2021-08-30T05:38:00.039719Z","iopub.status.idle":"2021-08-30T05:38:25.785084Z","shell.execute_reply.started":"2021-08-30T05:38:00.039683Z","shell.execute_reply":"2021-08-30T05:38:25.783809Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Submission","metadata":{}},{"cell_type":"code","source":"df_test = pd.read_csv(\"/kaggle/input/beyond-analysis/test.csv\")\ndf_submission = pd.DataFrame(df_test['UNIQUE_IDENTIFIER'].unique(), columns=['UNIQUE_IDENTIFIER'])\n\n# df_submission['Y1'] = Y1*(Q75['Y1']-Q25['Y1']) + Q50['Y1']\ndf_submission['Y1'] = Y1\ndf_submission['Y2'] = Y2*(Q75['Y2']-Q25['Y2']) + Q50['Y2']\n\ndf_submission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-09-04T17:21:14.666003Z","iopub.execute_input":"2021-09-04T17:21:14.666324Z","iopub.status.idle":"2021-09-04T17:21:16.278706Z","shell.execute_reply.started":"2021-09-04T17:21:14.666296Z","shell.execute_reply":"2021-09-04T17:21:16.277809Z"},"trusted":true},"execution_count":null,"outputs":[]}]}