{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np, pandas as pd\npd.options.display.max_columns = 100\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-09-03T11:17:16.153510Z","iopub.execute_input":"2021-09-03T11:17:16.153941Z","iopub.status.idle":"2021-09-03T11:17:17.190815Z","shell.execute_reply.started":"2021-09-03T11:17:16.153848Z","shell.execute_reply":"2021-09-03T11:17:17.189641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_history(history, epochs):\n    sns.set()\n    fig, ax = plt.subplots(1,1, figsize=(12, 4))\n    ax.plot(range(epochs), history.history['loss'], label=\"train\")\n    ax.plot(range(epochs), history.history['val_loss'], label=\"val\")\n    ax.legend()\n    ax.set_title(\"RMSE\")\n    #ax[0].axis((30, 200, 90, 95))","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:29:25.141435Z","iopub.execute_input":"2021-09-01T16:29:25.141836Z","iopub.status.idle":"2021-09-01T16:29:25.149483Z","shell.execute_reply.started":"2021-09-01T16:29:25.141801Z","shell.execute_reply":"2021-09-01T16:29:25.148379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_dir = \"../input/beyond-analysis/\"","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:29:25.151809Z","iopub.execute_input":"2021-09-01T16:29:25.152188Z","iopub.status.idle":"2021-09-01T16:29:25.169781Z","shell.execute_reply.started":"2021-09-01T16:29:25.152136Z","shell.execute_reply":"2021-09-01T16:29:25.168812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load train and test sets\ntrain = pd.read_csv(base_dir + \"train.csv\")\ntest = pd.read_csv(base_dir + \"test.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:29:25.171071Z","iopub.execute_input":"2021-09-01T16:29:25.171414Z","iopub.status.idle":"2021-09-01T16:29:31.784399Z","shell.execute_reply.started":"2021-09-01T16:29:25.171391Z","shell.execute_reply":"2021-09-01T16:29:31.783301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def unique_len(series):\n    return len(series.unique())\n\ndef deviation(series, mean):\n    return np.square(series.mean()-mean)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:29:31.785828Z","iopub.execute_input":"2021-09-01T16:29:31.786122Z","iopub.status.idle":"2021-09-01T16:29:31.791218Z","shell.execute_reply.started":"2021-09-01T16:29:31.786097Z","shell.execute_reply":"2021-09-01T16:29:31.790291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calc_customer_stats(df):\n    entry_mean = df['ENTRY'].mean()\n    revenue_mean = df['REVENUE'].mean()\n    winnings_1 = df['WINNINGS_1'].mean()\n    discount_mean = df['DISCOUNT'].mean()\n    deposit_mean = df['DEPOSIT'].mean()\n    deposit_numer_mean = df['DEPOSIT_NUMBER'].mean()\n    deposit_2_mean = df['DEPOSIT_2'].mean()\n    deposit_trials_mean = df['DEPOSIT_TRAILS'].mean()\n    entry_number_mean = df['ENTRY_NUMBER'].mean()\n    winning_number_mean = df['WINNINGS_NUMBER'].mean()\n    agg_dict = {\n        \"ENTRY\": [\"mean\",\"sum\", np.std, \"max\", \"min\", unique_len, lambda x: deviation(x, entry_mean)],\n        \"REVENUE\": [\"mean\",\"sum\", np.std, \"max\", \"min\", unique_len, lambda x: deviation(x, revenue_mean)],\n        \"WINNINGS_1\": [\"mean\",\"sum\", np.std, \"max\", \"min\", unique_len, lambda x: deviation(x, winnings_1)],\n        \"WINNINGS_2\": [\"mean\",\"sum\", np.std],\n        \"DISCOUNT\": [\"mean\",\"sum\", np.std, \"max\", \"min\", unique_len, lambda x: deviation(x, discount_mean)],\n        \"DEPOSIT\": [\"mean\",\"sum\", np.std, \"max\", \"min\", unique_len, lambda x: deviation(x, deposit_mean)],\n        \"DEPOSIT_NUMBER\": [\"mean\",\"sum\", np.std, \"max\", \"min\", unique_len, lambda x: deviation(x, deposit_numer_mean)],\n        \"DEPOSIT_2\": [\"mean\",\"sum\", np.std, \"max\", \"min\", unique_len, lambda x: deviation(x, deposit_2_mean)],\n        \"WITHDRAW\": [\"mean\",\"sum\", np.std],\n        \"WITHDRAW_NUMBER\": [\"mean\",\"sum\", np.std],\n        \"DEPOSIT_TRAILS\": [\"mean\",\"sum\", np.std, \"max\", \"min\", unique_len, lambda x: deviation(x, deposit_trials_mean)],\n        \"ENTRY_NUMBER\": [\"mean\",\"sum\", np.std, \"max\", \"min\", unique_len, lambda x: deviation(x, entry_number_mean)],\n        \"WINNINGS_NUMBER\": [\"mean\",\"sum\", np.std, \"max\", \"min\", unique_len, lambda x: deviation(x, winning_number_mean)],\n        \"PRACTICE_ENTRY\": [\"mean\",\"sum\", unique_len],\n        \"PRACTICE_WINNINGS\": [\"mean\",\"sum\", unique_len],\n        \"PRACTICE_ENTRY_NUMBER\": [\"mean\",\"sum\", unique_len],\n        \"PRACTICE_WINNINGS_NUMBER\": [\"mean\",\"sum\", unique_len]\n    }\n    \n    df_grp = df.groupby(\"UNIQUE_IDENTIFIER\").agg(agg_dict)\n    df_grp.columns = [\"_\".join(col) for col in df_grp.columns]\n    df_grp = df_grp.reset_index()\n    return df_grp","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:29:31.792385Z","iopub.execute_input":"2021-09-01T16:29:31.792621Z","iopub.status.idle":"2021-09-01T16:29:31.810801Z","shell.execute_reply.started":"2021-09-01T16:29:31.792598Z","shell.execute_reply":"2021-09-01T16:29:31.809514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"If using boosting then go down from here to LightGBM...","metadata":{}},{"cell_type":"code","source":"# preprocess train data\ntrain_ = calc_customer_stats(train)\ntemp = train.groupby(\"UNIQUE_IDENTIFIER\")[[\"Y1\",\"Y2\"]].agg(lambda x: x.unique()[0]).reset_index()\ntrain_ = train_.merge(temp, on=\"UNIQUE_IDENTIFIER\", how=\"left\")","metadata":{"execution":{"iopub.status.busy":"2021-08-29T13:03:49.109567Z","iopub.execute_input":"2021-08-29T13:03:49.109958Z","iopub.status.idle":"2021-08-29T13:04:01.341441Z","shell.execute_reply.started":"2021-08-29T13:03:49.109913Z","shell.execute_reply":"2021-08-29T13:04:01.340607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Preprocess test data\ntest_ = calc_customer_stats(test)","metadata":{"execution":{"iopub.status.busy":"2021-08-29T13:04:01.343749Z","iopub.execute_input":"2021-08-29T13:04:01.344092Z","iopub.status.idle":"2021-08-29T13:04:01.773114Z","shell.execute_reply.started":"2021-08-29T13:04:01.344055Z","shell.execute_reply":"2021-08-29T13:04:01.772278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fill NaN values which are zero std\ntrain_ = train_.fillna(value=0)\ntest_ = test_.fillna(value=0)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_","metadata":{"execution":{"iopub.status.busy":"2021-08-29T13:04:01.774551Z","iopub.execute_input":"2021-08-29T13:04:01.774907Z","iopub.status.idle":"2021-08-29T13:04:01.874492Z","shell.execute_reply.started":"2021-08-29T13:04:01.774869Z","shell.execute_reply":"2021-08-29T13:04:01.873566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_.shape, test_.shape)","metadata":{"execution":{"iopub.status.busy":"2021-08-29T13:04:01.998022Z","iopub.execute_input":"2021-08-29T13:04:01.998266Z","iopub.status.idle":"2021-08-29T13:04:02.005264Z","shell.execute_reply.started":"2021-08-29T13:04:01.998243Z","shell.execute_reply":"2021-08-29T13:04:02.004339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"markdown","source":"## DNN","metadata":{}},{"cell_type":"code","source":"X = train_.drop(['UNIQUE_IDENTIFIER', 'Y1', 'Y2'], axis=1)\ny = train_[[\"Y1\", \"Y2\"]]\nX_test = test_.drop([\"UNIQUE_IDENTIFIER\"], axis=1)","metadata":{"execution":{"iopub.status.busy":"2021-08-29T13:05:00.204325Z","iopub.execute_input":"2021-08-29T13:05:00.204662Z","iopub.status.idle":"2021-08-29T13:05:00.239056Z","shell.execute_reply.started":"2021-08-29T13:05:00.204631Z","shell.execute_reply":"2021-08-29T13:05:00.238163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Update to the latest version\n!pip install tensorflow","metadata":{"execution":{"iopub.status.busy":"2021-08-29T13:05:01.523933Z","iopub.execute_input":"2021-08-29T13:05:01.524261Z","iopub.status.idle":"2021-08-29T13:05:50.675061Z","shell.execute_reply.started":"2021-08-29T13:05:01.524231Z","shell.execute_reply":"2021-08-29T13:05:50.67412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -q -U keras-tuner","metadata":{"execution":{"iopub.status.busy":"2021-08-29T13:05:50.67669Z","iopub.execute_input":"2021-08-29T13:05:50.67702Z","iopub.status.idle":"2021-08-29T13:05:57.224331Z","shell.execute_reply.started":"2021-08-29T13:05:50.67698Z","shell.execute_reply":"2021-08-29T13:05:57.223409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import keras_tuner as kt","metadata":{"execution":{"iopub.status.busy":"2021-08-29T13:05:57.228258Z","iopub.execute_input":"2021-08-29T13:05:57.228587Z","iopub.status.idle":"2021-08-29T13:05:59.202972Z","shell.execute_reply.started":"2021-08-29T13:05:57.228535Z","shell.execute_reply":"2021-08-29T13:05:59.201653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\nfrom keras import backend as K\n\ndef model_builder(hp):\n  model = keras.Sequential()\n  model.add(keras.layers.Dense(16, input_shape=(X.shape[1], ), activation='relu'))\n    \n  # Tune the number of units in the first Dense layer\n  # Choose an optimal value between 32-512\n  hp_units = hp.Int('units1', min_value=32, max_value=512, step=32)\n  model.add(keras.layers.Dense(units=hp_units, activation='relu'))\n    \n  # Tune the dropout values\n  # Choose an optimal value between 0.1-0.5\n  hp_units = hp.Float('dropout1', min_value=0.1, max_value=0.5, step=0.01)\n  model.add(keras.layers.Dropout(hp_units))\n\n  hp_units = hp.Int('units2', min_value=32, max_value=512, step=32)\n  model.add(keras.layers.Dense(units=hp_units, activation='relu'))\n\n  hp_units = hp.Float('dropout2', min_value=0.1, max_value=0.5, step=0.01)\n  model.add(keras.layers.Dropout(hp_units))\n\n  model.add(keras.layers.Dense(2))\n\n  # Tune the learning rate for the optimizer\n  # Choose an optimal value from 0.01, 0.001, or 0.0001\n  hp_learning_rate = hp.Choice('learning_rate', values=[1e-2, 1e-3, 1e-4, 1e-5])\n    \n  # Custom loss function\n  def root_mean_squared_error(y_true, y_pred):\n      return K.sqrt(K.mean(K.square(y_pred - y_true))) \n\n  model.compile(optimizer=keras.optimizers.Adam(learning_rate=hp_learning_rate),\n                loss=root_mean_squared_error)\n\n  return model","metadata":{"execution":{"iopub.status.busy":"2021-08-29T13:05:59.207952Z","iopub.execute_input":"2021-08-29T13:05:59.208518Z","iopub.status.idle":"2021-08-29T13:05:59.229162Z","shell.execute_reply.started":"2021-08-29T13:05:59.208475Z","shell.execute_reply":"2021-08-29T13:05:59.226034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tuner = kt.Hyperband(model_builder,\n                     objective=kt.Objective(\"val_loss\", direction=\"min\"),\n                     max_epochs=20,\n                     factor=3,\n                     directory='./',\n                     project_name='kt_tuner')","metadata":{"execution":{"iopub.status.busy":"2021-08-29T13:05:59.233342Z","iopub.execute_input":"2021-08-29T13:05:59.235697Z","iopub.status.idle":"2021-08-29T13:05:59.613786Z","shell.execute_reply.started":"2021-08-29T13:05:59.235644Z","shell.execute_reply":"2021-08-29T13:05:59.612979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stop_early = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=5)","metadata":{"execution":{"iopub.status.busy":"2021-08-29T13:05:59.614979Z","iopub.execute_input":"2021-08-29T13:05:59.615318Z","iopub.status.idle":"2021-08-29T13:05:59.622798Z","shell.execute_reply.started":"2021-08-29T13:05:59.615285Z","shell.execute_reply":"2021-08-29T13:05:59.621942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tuner.search(X, y, epochs=100, validation_split=0.2, callbacks=[stop_early])","metadata":{"execution":{"iopub.status.busy":"2021-08-29T13:05:59.62587Z","iopub.execute_input":"2021-08-29T13:05:59.626116Z","iopub.status.idle":"2021-08-29T13:32:38.410803Z","shell.execute_reply.started":"2021-08-29T13:05:59.626093Z","shell.execute_reply":"2021-08-29T13:32:38.409756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get the optimal hyperparameters\nbest_hps=tuner.get_best_hyperparameters(num_trials=1)[0]\n# Build the model with the optimal hyperparameters and train it on the data for 100 epochs\nmodel = tuner.hypermodel.build(best_hps)\nhistory = model.fit(X, y, epochs=100, validation_split=0.2, verbose=1)","metadata":{"execution":{"iopub.status.busy":"2021-08-29T13:32:38.413268Z","iopub.execute_input":"2021-08-29T13:32:38.413628Z","iopub.status.idle":"2021-08-29T13:45:17.087915Z","shell.execute_reply.started":"2021-08-29T13:32:38.413589Z","shell.execute_reply":"2021-08-29T13:45:17.087034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history(history, 100)","metadata":{"execution":{"iopub.status.busy":"2021-09-03T11:17:17.192751Z","iopub.execute_input":"2021-09-03T11:17:17.193206Z","iopub.status.idle":"2021-09-03T11:17:17.296266Z","shell.execute_reply.started":"2021-09-03T11:17:17.193158Z","shell.execute_reply":"2021-09-03T11:17:17.294793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_acc_per_epoch = history.history['val_loss']\nbest_epoch = val_acc_per_epoch.index(min(val_acc_per_epoch)) + 1\nprint('Best epoch: %d' % (best_epoch,))","metadata":{"execution":{"iopub.status.busy":"2021-08-29T13:50:47.48904Z","iopub.execute_input":"2021-08-29T13:50:47.489363Z","iopub.status.idle":"2021-08-29T13:50:47.49407Z","shell.execute_reply.started":"2021-08-29T13:50:47.489333Z","shell.execute_reply":"2021-08-29T13:50:47.493159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hypermodel = tuner.hypermodel.build(best_hps)\n\n# Retrain the model\nhypermodel.fit(X, y, epochs=best_epoch, validation_split=0.2)","metadata":{"execution":{"iopub.status.busy":"2021-08-29T13:51:16.964184Z","iopub.execute_input":"2021-08-29T13:51:16.964497Z","iopub.status.idle":"2021-08-29T13:59:39.518363Z","shell.execute_reply.started":"2021-08-29T13:51:16.964467Z","shell.execute_reply":"2021-08-29T13:59:39.51751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = hypermodel.predict(X_test)","metadata":{"execution":{"iopub.status.busy":"2021-08-29T13:59:39.519813Z","iopub.execute_input":"2021-08-29T13:59:39.520179Z","iopub.status.idle":"2021-08-29T13:59:41.482839Z","shell.execute_reply.started":"2021-08-29T13:59:39.520142Z","shell.execute_reply":"2021-08-29T13:59:41.481972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_[\"Y1_NN\"] = y_pred[:,0]\ntest_[\"Y2_NN\"] = y_pred[:,1]\ntest_[[\"UNIQUE_IDENTIFIER\",\"Y1_NN\",\"Y2_NN\"]].to_csv(\"submission_NN.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-08-29T14:00:13.390778Z","iopub.execute_input":"2021-08-29T14:00:13.391147Z","iopub.status.idle":"2021-08-29T14:00:13.641448Z","shell.execute_reply.started":"2021-08-29T14:00:13.391113Z","shell.execute_reply":"2021-08-29T14:00:13.640639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## LightGBM","metadata":{}},{"cell_type":"code","source":"import lightgbm as lgb\nfrom lightgbm import plot_importance\nfrom sklearn.model_selection import KFold\nimport warnings\nwarnings.filterwarnings('ignore')\nseed = 21\nnp.random.seed(seed)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:29:36.809293Z","iopub.execute_input":"2021-09-01T16:29:36.809647Z","iopub.status.idle":"2021-09-01T16:29:38.571028Z","shell.execute_reply.started":"2021-09-01T16:29:36.809617Z","shell.execute_reply":"2021-09-01T16:29:38.569796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# preprocess train data\ntrain_ = calc_customer_stats(train)\ntemp = train.groupby(\"UNIQUE_IDENTIFIER\")[[\"STATUS_CHECK\", \"CATEGORY_1\",\"CATEGORY_2\",\"Y1\",\"Y2\"]].agg(lambda x: x.unique()[0]).reset_index()\ntrain_ = train_.merge(temp, on=\"UNIQUE_IDENTIFIER\", how=\"left\")","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:29:38.723562Z","iopub.execute_input":"2021-09-01T16:29:38.723978Z","iopub.status.idle":"2021-09-01T16:31:46.849426Z","shell.execute_reply.started":"2021-09-01T16:29:38.723943Z","shell.execute_reply":"2021-09-01T16:31:46.848515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Preprocess test data\ntest_ = calc_customer_stats(test)\ntemp = test.groupby(\"UNIQUE_IDENTIFIER\")[[\"STATUS_CHECK\", \"CATEGORY_1\", \"CATEGORY_2\"]].agg(lambda x: x.unique()[0]).reset_index()\ntest_ = test_.merge(temp, on=\"UNIQUE_IDENTIFIER\", how=\"left\")","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:31:46.850553Z","iopub.execute_input":"2021-09-01T16:31:46.850762Z","iopub.status.idle":"2021-09-01T16:33:08.042644Z","shell.execute_reply.started":"2021-09-01T16:31:46.85074Z","shell.execute_reply":"2021-09-01T16:33:08.041671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fill NaN values which are zero std\ntrain_ = train_.fillna(value=0)\ntest_ = test_.fillna(value=0)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:33:08.043982Z","iopub.execute_input":"2021-09-01T16:33:08.044263Z","iopub.status.idle":"2021-09-01T16:33:08.146788Z","shell.execute_reply.started":"2021-09-01T16:33:08.044229Z","shell.execute_reply":"2021-09-01T16:33:08.145237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = train_.drop(['UNIQUE_IDENTIFIER', 'Y1', 'Y2'], axis=1)\ny = train_[[\"Y1\", \"Y2\"]]\nX_test = test_.drop([\"UNIQUE_IDENTIFIER\"], axis=1)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:33:08.147764Z","iopub.execute_input":"2021-09-01T16:33:08.14799Z","iopub.status.idle":"2021-09-01T16:33:08.185603Z","shell.execute_reply.started":"2021-09-01T16:33:08.147963Z","shell.execute_reply":"2021-09-01T16:33:08.184467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(X.shape, X_test.shape)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:33:08.188761Z","iopub.execute_input":"2021-09-01T16:33:08.18904Z","iopub.status.idle":"2021-09-01T16:33:08.195697Z","shell.execute_reply.started":"2021-09-01T16:33:08.189017Z","shell.execute_reply":"2021-09-01T16:33:08.19434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nle1 = LabelEncoder()\nle2 = LabelEncoder()\nX_test['CATEGORY_1'] = le1.fit_transform(X_test['CATEGORY_1'])\nX_test['CATEGORY_2'] = le2.fit_transform(X_test['CATEGORY_2'])\nX['CATEGORY_1'] = le1.transform(X['CATEGORY_1'])\nX['CATEGORY_2'] = le2.transform(X['CATEGORY_2'])","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:33:08.197262Z","iopub.execute_input":"2021-09-01T16:33:08.197477Z","iopub.status.idle":"2021-09-01T16:33:08.267854Z","shell.execute_reply.started":"2021-09-01T16:33:08.197454Z","shell.execute_reply":"2021-09-01T16:33:08.266389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"If using catboost then go down from here...","metadata":{}},{"cell_type":"code","source":"FOLDS = 5\nlgb_params1 = {'bagging_fraction': 0.7446132365786178,\n               'bagging_freq': int(94.57044996418587),\n               'feature_fraction': 0.9012097906532421,\n               'feature_fraction_bynode': 0.2724293678054648,\n               'lambda_l1': 6.697079462213692,\n               'lambda_l2': 0.05709330859845584,\n               'learning_rate': 0.17588487226531224,\n               'min_data_in_leaf': int(154.15270440199714),\n               'min_sum_hessian_in_leaf': 58.37985742493366,\n               'num_leaves': int(622.1019844986934),\n        'seed': seed,\n        'feature_fraction_seed': seed,\n        'bagging_seed': seed,\n        'drop_seed': seed,\n        'data_random_seed': seed,\n        'objective': 'rmse',\n        'boosting': 'gbdt',\n        'verbosity': -1,\n        'n_jobs': -1\n}\n\nlgb_params2 = {'bagging_fraction': 0.8251097530764318,\n               'bagging_freq': int(30.596065567048885),\n               'feature_fraction': 0.49998711057091194,\n               'feature_fraction_bynode': 0.8507173745725533,\n               'lambda_l1': 7.555927148134436,\n               'lambda_l2': 7.988171568347725,\n               'learning_rate': 0.054902068760180736,\n               'min_data_in_leaf': int(797.5794978618343),\n               'min_sum_hessian_in_leaf': 18.06814076686615,\n               'num_leaves': int(529.4188648232011),\n        'seed': seed,\n        'feature_fraction_seed': seed,\n        'bagging_seed': seed,\n        'drop_seed': seed,\n        'data_random_seed': seed,\n        'objective': 'rmse',\n        'boosting': 'gbdt',\n        'verbosity': -1,\n        'n_jobs': -1}","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:33:08.269831Z","iopub.execute_input":"2021-09-01T16:33:08.27014Z","iopub.status.idle":"2021-09-01T16:33:08.278667Z","shell.execute_reply.started":"2021-09-01T16:33:08.270116Z","shell.execute_reply":"2021-09-01T16:33:08.277252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prediction arrays\nval_pred1 = np.zeros(X.shape[0])\ntest_pred1 = np.zeros(X_test.shape[0])\nval_pred2 = np.zeros(X.shape[0])\ntest_pred2 = np.zeros(X_test.shape[0])\n\n# K-fold cross valiation\nkfold = KFold(n_splits=FOLDS)\nfor fold, (train_idx, val_idx) in enumerate(kfold.split(X, y)):\n    print(\"#\"*40)\n    print(\"Training Fold - \", fold+1)\n    X_train, y_train = X.iloc[train_idx], y.iloc[train_idx]\n    X_val, y_val     = X.iloc[val_idx]  , y.iloc[val_idx]\n\n    # create lgb datasets\n    train_data1 = lgb.Dataset(X_train, y_train.iloc[:,0], categorical_feature = [\"STATUS_CHECK\", \"CATEGORY_1\", \"CATEGORY_2\"])\n    val_data1   = lgb.Dataset(X_val,   y_val.iloc[:,0],   categorical_feature = [\"STATUS_CHECK\", \"CATEGORY_1\", \"CATEGORY_2\"])\n    train_data2 = lgb.Dataset(X_train, y_train.iloc[:,1], categorical_feature = [\"STATUS_CHECK\", \"CATEGORY_1\", \"CATEGORY_2\"])\n    val_data2   = lgb.Dataset(X_val,   y_val.iloc[:,1],   categorical_feature = [\"STATUS_CHECK\", \"CATEGORY_1\", \"CATEGORY_2\"])\n\n    # Train model\n    print(\"#\"*20)\n    print(\"Training for Y1\")\n    model1 = lgb.train(params = lgb_params1,\n                     train_set = train_data1,\n                     valid_sets = [train_data1, val_data1],\n                     num_boost_round = 10000,\n                     valid_names=[\"training\", \"validation\"],\n                     early_stopping_rounds = 1000,\n                     # callbacks=[neptune_callback],\n                     verbose_eval = 200)\n    print(\"#\"*20)\n    print(\"Training for Y2\")\n    model2 = lgb.train(params = lgb_params2,\n                     train_set = train_data2,\n                     valid_sets = [train_data2, val_data2],\n                     num_boost_round = 10000,\n                     valid_names=[\"training\", \"validation\"],\n                     early_stopping_rounds = 1000,\n                     # callbacks=[neptune_callback],\n                     verbose_eval = 200)\n\n    # Plot feature importance\n    plot_importance(model1, max_num_features=25, figsize= (10, 6), title=\"For Y1\")\n    plot_importance(model2, max_num_features=25, figsize= (10, 6), title=\"For Y2\")\n    plt.show()\n    \n    # Predict validation set\n    val_pred1[val_idx] = model1.predict(X_val)\n    val_pred2[val_idx] = model2.predict(X_val)\n    \n    # Predict test set\n    test_pred1 += model1.predict(X_test) / FOLDS\n    test_pred2 += model2.predict(X_test) / FOLDS","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:34:55.880372Z","iopub.execute_input":"2021-09-01T16:34:55.880788Z","iopub.status.idle":"2021-09-01T16:40:36.960181Z","shell.execute_reply.started":"2021-09-01T16:34:55.880763Z","shell.execute_reply":"2021-09-01T16:40:36.959549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import mean_squared_error as mse\nval_rmse = np.mean([np.sqrt(mse(train_[\"Y1\"], val_pred1)), np.sqrt(mse(train_[\"Y2\"], val_pred2))])\nprint(\"OOF score:\", val_rmse)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:40:36.961189Z","iopub.execute_input":"2021-09-01T16:40:36.961509Z","iopub.status.idle":"2021-09-01T16:40:36.968144Z","shell.execute_reply.started":"2021-09-01T16:40:36.961485Z","shell.execute_reply":"2021-09-01T16:40:36.967577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_pred_LGB = pd.DataFrame({\"Y1_LGB\":val_pred1, \"Y2_LGB\":val_pred2})","metadata":{"execution":{"iopub.status.busy":"2021-09-01T09:10:32.133697Z","iopub.execute_input":"2021-09-01T09:10:32.134156Z","iopub.status.idle":"2021-09-01T09:10:32.14428Z","shell.execute_reply.started":"2021-09-01T09:10:32.134123Z","shell.execute_reply":"2021-09-01T09:10:32.143509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_[\"Y1_LGB\"] = test_pred1\ntest_[\"Y2_LGB\"] = test_pred2\ntest_[[\"UNIQUE_IDENTIFIER\", \"Y1_LGB\", \"Y2_LGB\"]].to_csv(\"submission_LGB.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:40:36.969426Z","iopub.execute_input":"2021-09-01T16:40:36.969773Z","iopub.status.idle":"2021-09-01T16:40:37.216855Z","shell.execute_reply.started":"2021-09-01T16:40:36.969746Z","shell.execute_reply":"2021-09-01T16:40:37.215447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## CatBoost","metadata":{}},{"cell_type":"code","source":"FOLDS = 5","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:40:37.218322Z","iopub.execute_input":"2021-09-01T16:40:37.218591Z","iopub.status.idle":"2021-09-01T16:40:37.22437Z","shell.execute_reply.started":"2021-09-01T16:40:37.218565Z","shell.execute_reply":"2021-09-01T16:40:37.222571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(X.shape, X_test.shape)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:40:37.22608Z","iopub.execute_input":"2021-09-01T16:40:37.226391Z","iopub.status.idle":"2021-09-01T16:40:37.24525Z","shell.execute_reply.started":"2021-09-01T16:40:37.226361Z","shell.execute_reply":"2021-09-01T16:40:37.243725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from catboost import Pool, CatBoostRegressor","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:40:37.246719Z","iopub.execute_input":"2021-09-01T16:40:37.247063Z","iopub.status.idle":"2021-09-01T16:40:37.44944Z","shell.execute_reply.started":"2021-09-01T16:40:37.24703Z","shell.execute_reply":"2021-09-01T16:40:37.447533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prediction arrays\nval_pred1 = np.zeros(X.shape[0])\ntest_pred1 = np.zeros(X_test.shape[0])\nval_pred2 = np.zeros(X.shape[0])\ntest_pred2 = np.zeros(X_test.shape[0])\n\n# K-fold cross valiation\nkfold = KFold(n_splits=FOLDS)\nfor fold, (train_idx, val_idx) in enumerate(kfold.split(X, y)):\n    print(\"#\"*60)\n    print(\"Training Fold - \", fold+1)\n    X_train, y_train = X.iloc[train_idx], y.iloc[train_idx]\n    X_val, y_val     = X.iloc[val_idx]  , y.iloc[val_idx]\n\n    # create lgb datasets\n    train_data1 = Pool(X_train, y_train.iloc[:,0], cat_features = [\"STATUS_CHECK\", \"CATEGORY_1\", \"CATEGORY_2\"])\n    val_data1   = Pool(X_val,   y_val.iloc[:,0],   cat_features = [\"STATUS_CHECK\", \"CATEGORY_1\", \"CATEGORY_2\"])\n    train_data2 = Pool(X_train, y_train.iloc[:,1], cat_features = [\"STATUS_CHECK\", \"CATEGORY_1\", \"CATEGORY_2\"])\n    val_data2   = Pool(X_val,   y_val.iloc[:,1],   cat_features = [\"STATUS_CHECK\", \"CATEGORY_1\", \"CATEGORY_2\"])\n\n    # Train model\n    print(\"#\"*20)\n    print(\"Training for Y1\")\n    model1 = CatBoostRegressor(iterations=2000, \n                              depth=3, \n                              learning_rate=0.1, \n                              loss_function='RMSE',\n                              eval_metric=\"RMSE\",\n                              use_best_model = True,\n                              early_stopping_rounds = 1000)\n    model1.fit(train_data1, eval_set=val_data1,verbose_eval=200)\n    print(\"#\"*20)\n    print(\"Training for Y2\")\n    model2 = CatBoostRegressor(iterations=7000, \n                              depth=4, \n                              learning_rate=0.05, \n                              loss_function='RMSE',\n                              eval_metric=\"RMSE\",\n                              use_best_model = True,\n                              early_stopping_rounds = 1000)\n    model2.fit(train_data2, eval_set=val_data2,verbose_eval=200)\n\n    \n    # Predict validation set\n    val_pred1[val_idx] = model1.predict(X_val)\n    val_pred2[val_idx] = model2.predict(X_val)\n    \n    # Predict test set\n    test_pred1 += model1.predict(X_test) / FOLDS\n    test_pred2 += model2.predict(X_test) / FOLDS","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:40:37.450877Z","iopub.execute_input":"2021-09-01T16:40:37.451103Z","iopub.status.idle":"2021-09-01T16:50:35.835718Z","shell.execute_reply.started":"2021-09-01T16:40:37.45108Z","shell.execute_reply":"2021-09-01T16:50:35.834026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import mean_squared_error as mse\nval_rmse = np.mean([np.sqrt(mse(train_[\"Y1\"], val_pred1)), np.sqrt(mse(train_[\"Y2\"], val_pred2))])\nprint(\"OOF score:\", val_rmse)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:50:35.837289Z","iopub.execute_input":"2021-09-01T16:50:35.837499Z","iopub.status.idle":"2021-09-01T16:50:35.844886Z","shell.execute_reply.started":"2021-09-01T16:50:35.837476Z","shell.execute_reply":"2021-09-01T16:50:35.843901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_pred_CAT = pd.DataFrame({\"Y1_CAT\":val_pred1, \"Y2_CAT\":val_pred2})","metadata":{"execution":{"iopub.status.busy":"2021-09-01T09:18:32.911198Z","iopub.execute_input":"2021-09-01T09:18:32.911507Z","iopub.status.idle":"2021-09-01T09:18:32.919972Z","shell.execute_reply.started":"2021-09-01T09:18:32.911479Z","shell.execute_reply":"2021-09-01T09:18:32.919031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_[\"Y1_CAT\"] = test_pred1\ntest_[\"Y2_CAT\"] = test_pred2\ntest_[[\"UNIQUE_IDENTIFIER\", \"Y1_CAT\", \"Y2_CAT\"]].to_csv(\"submission_CAT.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:50:35.846029Z","iopub.execute_input":"2021-09-01T16:50:35.846262Z","iopub.status.idle":"2021-09-01T16:50:36.092069Z","shell.execute_reply.started":"2021-09-01T16:50:35.846237Z","shell.execute_reply":"2021-09-01T16:50:36.090611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Ensemble","metadata":{}},{"cell_type":"code","source":"# All three models\nsub_NN = pd.read_csv(\"./submission_NN.csv\")\nsub_LGB = pd.read_csv(\"./submission_LGB.csv\")\nsub_CAT = pd.read_csv(\"./submission_CAT.csv\")\ntest_[\"Y1\"] = (sub_NN.iloc[:,1] + sub_LGB.iloc[:,1] + sub_CAT.iloc[:,1])/3\ntest_[\"Y2\"] = (sub_NN.iloc[:,2] + sub_LGB.iloc[:,2]+ sub_CAT.iloc[:,2])/3\ntest_[[\"UNIQUE_IDENTIFIER\", \"Y1\", \"Y2\"]].to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-08-31T12:36:25.243245Z","iopub.execute_input":"2021-08-31T12:36:25.243638Z","iopub.status.idle":"2021-08-31T12:36:25.815972Z","shell.execute_reply.started":"2021-08-31T12:36:25.243605Z","shell.execute_reply":"2021-08-31T12:36:25.81471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Only boosting models\nsub_LGB = pd.read_csv(\"./submission_LGB.csv\")\nsub_CAT = pd.read_csv(\"./submission_CAT.csv\")\ntest_[\"Y1\"] = (sub_LGB.iloc[:,1] + sub_CAT.iloc[:,1])/2\ntest_[\"Y2\"] = (sub_LGB.iloc[:,2]+ sub_CAT.iloc[:,2])/2\ntest_[[\"UNIQUE_IDENTIFIER\", \"Y1\", \"Y2\"]].to_csv(\"submission_boost.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:50:45.152662Z","iopub.execute_input":"2021-09-01T16:50:45.153091Z","iopub.status.idle":"2021-09-01T16:50:45.588867Z","shell.execute_reply.started":"2021-09-01T16:50:45.153054Z","shell.execute_reply":"2021-09-01T16:50:45.587787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Random Forest\nA Failed attempt!","metadata":{}},{"cell_type":"code","source":"# from sklearn.ensemble import RandomForestRegressor\n\n# X = pd.concat([train_pred_CAT, train_pred_LGB], axis=1)\n# X_test = test_[[\"Y1_CAT\", \"Y1_LGB\", \"Y2_CAT\", \"Y2_LGB\"]]\n\n# print(X.shape, X_test.shape)\n\n# # Prediction arrays\n# val_pred1 = np.zeros(X.shape[0])\n# test_pred1 = np.zeros(X_test.shape[0])\n# val_pred2 = np.zeros(X.shape[0])\n# test_pred2 = np.zeros(X_test.shape[0])\n\n# # K-fold cross valiation\n# kfold = KFold(n_splits=FOLDS)\n# for fold, (train_idx, val_idx) in enumerate(kfold.split(X, y)):\n#     print(\"#\"*60)\n#     print(\"Training Fold - \", fold+1)\n#     X_train, y_train = X.iloc[train_idx], y.iloc[train_idx]\n#     X_val, y_val     = X.iloc[val_idx]  , y.iloc[val_idx]\n\n#     # create training datasets\n#     y_train1 = y_train.iloc[:,0]\n#     y_val1   = y_val.iloc[:,0]\n#     y_train2 = y_train.iloc[:,1]\n#     y_val2   = y_val.iloc[:,1]\n\n#     # Train model\n#     print(\"#\"*20)\n#     print(\"Training for Y1\")\n#     rfr1 = RandomForestRegressor(n_estimators = 1000,\n#                             max_depth = 3,\n#                            oob_score = True,\n#                            n_jobs = -1,\n#                            random_state = seed,\n#                            max_samples = 0.6)\n#     rfr1.fit(X_train, y_train1)\n#     print(\"#\"*20)\n#     print(\"Training for Y2\")\n#     rfr2 = RandomForestRegressor(n_estimators = 2000,\n#                             max_depth = 4,\n#                            oob_score = True,\n#                            n_jobs = -1,\n#                            random_state = seed,\n#                            max_samples = 0.6)\n#     rfr2.fit(X_train, y_train2)\n\n#     # Predict validation set\n#     val_pred1[val_idx] = rfr1.predict(X_val)\n#     val_pred2[val_idx] = rfr2.predict(X_val)\n    \n#     # Predict test set\n#     test_pred1 += rfr1.predict(X_test) / FOLDS\n#     test_pred2 += rfr2.predict(X_test) / FOLDS\n\n# from sklearn.metrics import mean_squared_error as mse\n# val_rmse = np.mean([np.sqrt(mse(train_[\"Y1\"], val_pred1)), np.sqrt(mse(train_[\"Y2\"], val_pred2))])\n# print(\"OOF score:\", val_rmse)\n\n# test_[\"Y1_RFR\"] = test_pred1\n# test_[\"Y2_RFR\"] = test_pred2\n# test_[[\"UNIQUE_IDENTIFIER\", \"Y1_RFR\", \"Y2_RFR\"]].to_csv(\"submission_RFR.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T08:02:12.391054Z","iopub.execute_input":"2021-09-01T08:02:12.391453Z","iopub.status.idle":"2021-09-01T08:02:12.402863Z","shell.execute_reply.started":"2021-09-01T08:02:12.391418Z","shell.execute_reply":"2021-09-01T08:02:12.401619Z"},"trusted":true},"execution_count":null,"outputs":[]}]}