{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom matplotlib.ticker import MaxNLocator\nfrom matplotlib.colors import ListedColormap\nimport seaborn as sns\nfrom cycler import cycler\nfrom IPython.display import display\nimport datetime\nimport scipy.stats\nimport math\nimport random\nimport pickle\n\nfrom sklearn.model_selection import KFold,StratifiedKFold\nfrom sklearn.metrics import roc_auc_score, roc_curve\nfrom sklearn.calibration import CalibrationDisplay\nfrom sklearn.preprocessing import StandardScaler\n\nimport tensorflow as tf\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, LearningRateScheduler, EarlyStopping\nfrom tensorflow.keras.layers import Dense, Input, InputLayer, Add, Concatenate, Dot, BatchNormalization, Dropout\nfrom tensorflow.keras.utils import plot_model, to_categorical\n\nplt.rcParams['axes.prop_cycle'] = cycler(color=['#ffd700'] +\n                                         plt.rcParams['axes.prop_cycle'].by_key()['color'][1:])\nnp.set_printoptions(linewidth=150)","metadata":{"execution":{"iopub.status.busy":"2022-12-25T16:44:04.413425Z","iopub.execute_input":"2022-12-25T16:44:04.413885Z","iopub.status.idle":"2022-12-25T16:44:10.929477Z","shell.execute_reply.started":"2022-12-25T16:44:04.413803Z","shell.execute_reply":"2022-12-25T16:44:10.928498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_history(history, *, n_epochs=None, plot_lr=False, title=None, bottom=None, top=None):\n    \"\"\"Plot (the last n_epochs epochs of) the training history\n    \n    Plots loss and optionally val_loss and lr.\"\"\"\n    plt.figure(figsize=(15, 6))\n    from_epoch = 0 if n_epochs is None else max(len(history['loss']) - n_epochs, 0)\n    \n    # Plot training and validation losses\n    plt.plot(np.arange(from_epoch, len(history['loss'])), history['loss'][from_epoch:], label='Training loss')\n    try:\n        plt.plot(np.arange(from_epoch, len(history['loss'])), history['val_loss'][from_epoch:], label='Validation loss')\n        best_epoch = np.argmin(np.array(history['val_loss']))\n        best_val_loss = history['val_loss'][best_epoch]\n        if best_epoch >= from_epoch:\n            plt.scatter([best_epoch], [best_val_loss], c='r', label=f'Best val_loss = {best_val_loss:.5f}')\n        if best_epoch > 0:\n            almost_epoch = np.argmin(np.array(history['val_loss'])[:best_epoch])\n            almost_val_loss = history['val_loss'][almost_epoch]\n            if almost_epoch >= from_epoch:\n                plt.scatter([almost_epoch], [almost_val_loss], c='orange', label='Second best val_loss')\n    except KeyError:\n        pass\n    if bottom is not None: plt.ylim(bottom=bottom)\n    if top is not None: plt.ylim(top=top)\n    plt.gca().xaxis.set_major_locator(MaxNLocator(integer=True))\n    plt.xlabel('Epoch')\n    plt.ylabel('Loss')\n    plt.legend(loc='lower left')\n    if title is not None: plt.title(title)\n        \n    # Plot learning rate\n    if plot_lr and 'lr' in history:\n        ax2 = plt.gca().twinx()\n        ax2.plot(np.arange(from_epoch, len(history['lr'])), np.array(history['lr'][from_epoch:]), color='g', label='Learning rate')\n        ax2.set_ylabel('Learning rate')\n        ax2.legend(loc='upper right')\n        \n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-25T16:44:10.931220Z","iopub.execute_input":"2022-12-25T16:44:10.931867Z","iopub.status.idle":"2022-12-25T16:44:10.946609Z","shell.execute_reply.started":"2022-12-25T16:44:10.931823Z","shell.execute_reply":"2022-12-25T16:44:10.945715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/emsi-5iir-ds/train.csv')\ntest = pd.read_csv('/kaggle/input/emsi-5iir-ds/test.csv')\n# with open('../input/tpsmay22-keras-99778/oof_pred.pickle', 'rb') as f: train['t0'] = pickle.load(f)\n# with open('../input/tpsmay22-keras-99778/test_pred.pickle', 'rb') as f: test['t0'] = pickle.load(f)\n          \nfor df in [train, test]:\n    # Extract the 10 letters of f_27 into individual features\n    for i in range(10):\n        df[f'ch{i}'] = df.f_27.str.get(i).apply(ord) - ord('A')\n        \n    # unique_characters feature is from https://www.kaggle.com/code/cabaxiom/tps-may-22-eda-lgbm-model\n    df[\"unique_characters\"] = df.f_27.apply(lambda s: len(set(s)))\n    #df[\"unique_ints\"] = df[int_features].apply(lambda r: len(set(r)), axis=1)\n    \n    # Feature interactions: create three ternary features\n    # Every ternary feature can have the values -1, 0 and +1\n    df['i_02_21'] = (df.f_21 + df.f_02 > 5.2).astype(int) - (df.f_21 + df.f_02 < -5.3).astype(int)\n    df['i_05_22'] = (df.f_22 + df.f_05 > 5.1).astype(int) - (df.f_22 + df.f_05 < -5.4).astype(int)\n    i_00_01_26 = df.f_00 + df.f_01 + df.f_26\n    df['i_00_01_26'] = (i_00_01_26 > 5.0).astype(int) - (i_00_01_26 < -5.0).astype(int)\n    \n    #df['i_17_29'] = ((train['f_29']==1) & (train['f_17']>1)).astype(int)\n    \nfeatures = [f for f in test.columns if f != 'id' and f != 'f_27']\nfloat_features = [f for f in features if test[f].dtype == float]\nint_features = [f for f in features if test[f].dtype == int]\n\n# f30 = ['z_0', 'z_1', 'z_2']\n# for df in [train, test]:\n#     df[f30] = to_categorical(df.f_30)\n# features.remove('f_30')","metadata":{"execution":{"iopub.status.busy":"2022-12-25T16:44:10.952641Z","iopub.execute_input":"2022-12-25T16:44:10.954506Z","iopub.status.idle":"2022-12-25T16:44:38.939768Z","shell.execute_reply.started":"2022-12-25T16:44:10.954480Z","shell.execute_reply":"2022-12-25T16:44:38.938796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.layers import Activation\nfrom tensorflow.keras.utils import get_custom_objects\n\nclass Mish(Activation):\n\n    def __init__(self, activation, **kwargs):\n        super(Mish, self).__init__(activation, **kwargs)\n        self.__name__ = 'Mish'\n\n\ndef mish(inputs):\n    return inputs * tf.math.tanh(tf.math.softplus(inputs))\n\nget_custom_objects().update({'Mish': Mish(mish)})","metadata":{"execution":{"iopub.status.busy":"2022-12-25T16:44:38.945172Z","iopub.execute_input":"2022-12-25T16:44:38.945929Z","iopub.status.idle":"2022-12-25T16:44:38.973904Z","shell.execute_reply.started":"2022-12-25T16:44:38.945891Z","shell.execute_reply":"2022-12-25T16:44:38.973279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(features)\n","metadata":{"execution":{"iopub.status.busy":"2022-12-25T16:44:38.975314Z","iopub.execute_input":"2022-12-25T16:44:38.975984Z","iopub.status.idle":"2022-12-25T16:44:38.981659Z","shell.execute_reply.started":"2022-12-25T16:44:38.975948Z","shell.execute_reply":"2022-12-25T16:44:38.980646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features_a = ['f_00', 'f_01', 'f_02', 'f_03', 'f_04', 'f_05', 'f_06', \n              'f_19', 'f_20', 'f_21', 'f_22', 'f_23', 'f_24', 'f_25', 'f_26', 'f_28', \n              'f_30', 'ch7']\nfeatures_b = [f for f in features if f not in features_a]\n\ndef my_model(input_width=len(features)):\n    activation = 'Mish'\n    inputs_features_a = Input(shape=(len(features_a),))\n    inputs_features_b = Input(shape=(len(features_b),))\n    \n    DROP = 0.1\n    A=64\n    B=64\n    \n    xa1 = Dense(A, kernel_regularizer=tf.keras.regularizers.l2(35e-6),\n              activation=activation,\n             )(inputs_features_a)\n    xa2 = Dense(A, kernel_regularizer=tf.keras.regularizers.l2(35e-6),\n              activation=activation,\n             )(xa1)\n    \n    xa3 = Concatenate()([xa1, xa2])\n    \n    xa3 = Dropout(DROP)(xa3)\n    xa3 = BatchNormalization()(xa3)\n    \n    xa4 = Dense(A, kernel_regularizer=tf.keras.regularizers.l2(35e-6),\n              activation=activation,\n             )(xa3)\n      \n    xb1 = Dense(A, kernel_regularizer=tf.keras.regularizers.l2(35e-6),\n              activation=activation,\n             )(inputs_features_b)\n    xb2 = Dense(A, kernel_regularizer=tf.keras.regularizers.l2(35e-6),\n              activation=activation,\n             )(xb1)\n    \n    xb3 = Concatenate()([xb1, xb2])\n    \n    xb3 = Dropout(DROP)(xb3)\n    xb3 = BatchNormalization()(xb3)\n    \n    xb4 = Dense(A, kernel_regularizer=tf.keras.regularizers.l2(35e-6),\n              activation=activation,\n             )(xb3)\n    \n    x1 = Concatenate()([xa4, xb4])\n    x1 = Dropout(DROP)(x1)\n    x1 = BatchNormalization()(x1)\n    \n    x = Dense(A, kernel_regularizer=tf.keras.regularizers.l2(22e-6),\n              activation=activation,\n             )(x1)\n    x = Dense(A, kernel_regularizer=tf.keras.regularizers.l2(22e-6),\n              activation=activation,\n             )(x)\n    x = Dense(32, kernel_regularizer=tf.keras.regularizers.l2(22e-6),\n              activation=activation,\n             )(x) \n    x = Dense(1,\n              activation='sigmoid',\n             )(x)\n    model = Model([inputs_features_a, inputs_features_b], x)\n    \n    return model\n\nplot_model(my_model(), show_layer_names=False, show_shapes=True)","metadata":{"execution":{"iopub.status.busy":"2022-12-25T16:44:38.984849Z","iopub.execute_input":"2022-12-25T16:44:38.985699Z","iopub.status.idle":"2022-12-25T16:44:42.983581Z","shell.execute_reply.started":"2022-12-25T16:44:38.985661Z","shell.execute_reply":"2022-12-25T16:44:42.982406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EPOCHS = 300\nEPOCHS_COSINEDECAY = 300\nCYCLES = 2\nVERBOSE = 2 # set to 0 for less output, or to 2 for more output\nDIAGRAMS = True\nUSE_PLATEAU = False\nBATCH_SIZE = 2048*2\nONLY_FIRST_FOLD = True\n\n# see https://keras.io/getting_started/faq/#how-can-i-obtain-reproducible-results-using-keras-during-development\nnp.random.seed(1)\nrandom.seed(1)\ntf.random.set_seed(1)\n\ndef fit_model(X_tr, y_tr, X_va=None, y_va=None, run=0):\n    \"\"\"Scale the data, fit a model, plot the training history and optionally validate the model\n    \n    Returns a trained instance of tensorflow.keras.models.Model.\n    \n    As a side effect, updates oof[idx_va], y_va_pred, history_list and score_list.\n    \"\"\"\n    global y_va_pred\n    start_time = datetime.datetime.now()\n    \n    scaler = StandardScaler()\n    X_tr = X_tr.copy()\n    X_tr[features] = scaler.fit_transform(X_tr[features])\n    \n    if X_va is not None:\n        X_va = X_va.copy()\n        X_va[features] = scaler.transform(X_va[features])\n        validation_data = ((X_va[features_a], X_va[features_b]), y_va)\n    else:\n        validation_data = None\n\n    # Define the learning rate schedule and EarlyStopping\n    lr_start=0.01\n    if USE_PLATEAU and X_va is not None: # use early stopping\n        epochs = EPOCHS\n        lr = ReduceLROnPlateau(monitor=\"val_loss\", factor=0.7, \n                               patience=4, verbose=VERBOSE)\n        es = EarlyStopping(monitor=\"val_loss\",\n                           patience=12, \n                           verbose=1,\n                           mode=\"min\", \n                           restore_best_weights=True)\n        callbacks = [lr, es, tf.keras.callbacks.TerminateOnNaN()]\n\n    else: # use cosine learning rate decay rather than early stopping\n        epochs = EPOCHS_COSINEDECAY\n        lr_end = 0.0002\n        def cosine_decay(epoch):\n            # w decays from 1 to 0 in every cycle\n            # epoch == 0                  -> w = 1 (first epoch of cycle)\n            # epoch == epochs_per_cycle-1 -> w = 0 (last epoch of cycle)\n            epochs_per_cycle = epochs // CYCLES\n            epoch_in_cycle = epoch % epochs_per_cycle\n            if epochs_per_cycle > 1:\n                w = (1 + math.cos(epoch_in_cycle / (epochs_per_cycle-1) * math.pi)) / 2\n            else:\n                w = 1\n            return w * lr_start + (1 - w) * lr_end\n\n        lr = LearningRateScheduler(cosine_decay, verbose=0)\n        callbacks = [lr, tf.keras.callbacks.TerminateOnNaN()]\n        \n    # Construct and compile the model\n    model = my_model(len(features))\n    model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=lr_start),\n                  metrics=['AUC'],\n                  loss=tf.keras.losses.BinaryCrossentropy())\n\n    # Train the model\n    history = model.fit((X_tr[features_a], X_tr[features_b]), y_tr, \n                        validation_data=validation_data, \n                        epochs=epochs,\n                        verbose=VERBOSE,\n                        batch_size=BATCH_SIZE,\n                        shuffle=True,\n                        callbacks=callbacks)\n\n    history_list.append(history.history)\n    callbacks, es, lr, history = None, None, None, None\n    \n    if X_va is None:\n        print(f\"Training loss: {history_list[-1]['loss'][-1]:.4f}\")\n    else:\n        lastloss = f\"Training loss: {history_list[-1]['loss'][-1]:.4f} | Val loss: {history_list[-1]['val_loss'][-1]:.4f}\"\n        \n        # Inference for validation\n        y_va_pred = model.predict((X_va[features_a], X_va[features_b]), batch_size=len(X_va), verbose=0)\n        oof[idx_va] = y_va_pred.ravel()\n        \n        # Evaluation: Execution time, loss and AUC\n        score = roc_auc_score(y_va, y_va_pred)\n        print(f\"Fold {run}.{fold} | {str(datetime.datetime.now() - start_time)[-12:-7]}\"\n              f\" | {lastloss} | AUC: {score:.5f}\")\n        score_list.append(score)\n        \n        if DIAGRAMS and fold == 0 and run == 0:\n            # Plot training history\n            plot_history(history_list[-1], \n                         title=f\"Learning curve (validation AUC = {score:.5f})\",\n                         plot_lr=True)\n\n            # Plot y_true vs. y_pred\n            plt.figure(figsize=(15, 4))\n            plt.hist(y_va_pred[y_va == 0], bins=np.linspace(0, 1, 21),\n                     alpha=0.5, density=True)\n            plt.hist(y_va_pred[y_va == 1], bins=np.linspace(0, 1, 21),\n                     alpha=0.5, density=True)\n            plt.xlabel('y_pred')\n            plt.ylabel('density')\n            plt.title('OOF Predictions')\n            plt.show()\n\n    return model, scaler\n\n\nprint(f\"{len(features)} features\")\nhistory_list = []\nscore_list = []\noof = np.zeros((len(train), ), dtype=np.float32)\n#kf = KFold(n_splits=5)\nkf = StratifiedKFold(n_splits=5, shuffle=False)\nfor fold, (idx_tr, idx_va) in enumerate(kf.split(train,train.target)):\n    X_tr = train.iloc[idx_tr][features]\n    X_va = train.iloc[idx_va][features]\n    y_tr = train.iloc[idx_tr].target\n    y_va = train.iloc[idx_va].target\n    \n    model, scaler = fit_model(X_tr, y_tr, X_va, y_va)\n    if ONLY_FIRST_FOLD: break # we only need the first fold\n\nwith open('oof_pred.pickle', 'wb') as f: pickle.dump(oof, f)\nprint(f\"OOF AUC:                       {np.mean(score_list):.5f}\")","metadata":{"execution":{"iopub.status.busy":"2022-12-25T16:44:42.985130Z","iopub.execute_input":"2022-12-25T16:44:42.985462Z","iopub.status.idle":"2022-12-25T16:53:11.617971Z","shell.execute_reply.started":"2022-12-25T16:44:42.985427Z","shell.execute_reply":"2022-12-25T16:53:11.616554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12, 4))\nplt.hist(y_va_pred, bins=25, density=True)\nplt.title('Histogram of the oof predictions')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-25T16:53:11.619353Z","iopub.execute_input":"2022-12-25T16:53:11.620367Z","iopub.status.idle":"2022-12-25T16:53:11.842893Z","shell.execute_reply.started":"2022-12-25T16:53:11.620327Z","shell.execute_reply":"2022-12-25T16:53:11.841938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12, 4))\nCalibrationDisplay.from_predictions(y_va, y_va_pred, n_bins=50, strategy='quantile', ax=plt.gca())\nplt.title('Probability calibration')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-25T16:53:11.844401Z","iopub.execute_input":"2022-12-25T16:53:11.844743Z","iopub.status.idle":"2022-12-25T16:53:12.084633Z","shell.execute_reply.started":"2022-12-25T16:53:11.844709Z","shell.execute_reply":"2022-12-25T16:53:12.083701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Create submission\nVERBOSE = 0\nprint(f\"{len(features)} features\")\n\nX_tr = train.copy()\ny_tr = train.target\nX_te = test.copy()\nX_te[features] = scaler.transform(X_te[features])\n\npred_list = []\nfor seed in range(10):\n    # see https://keras.io/getting_started/faq/#how-can-i-obtain-reproducible-results-using-keras-during-development\n    np.random.seed(seed)\n    random.seed(seed)\n    tf.random.set_seed(seed)\n    model, scaler = fit_model(X_tr, y_tr, run=seed)\n    pred_list.append(model.predict((X_te[features_a], X_te[features_b]),\n                                   batch_size=len(test), verbose=0).ravel())\n    print(f\"{seed:2}\", pred_list[-1])\nprint()\nwith open('test_pred.pickle', 'wb') as f: pickle.dump(pred_list, f)\nsubmission = test[['id']].copy()\nsubmission['target'] = np.array([scipy.stats.rankdata(preds) for preds in pred_list]).mean(axis=0)\nsubmission.to_csv('submission.csv', index=False)\nsubmission","metadata":{"execution":{"iopub.status.busy":"2022-12-25T16:53:12.088128Z","iopub.execute_input":"2022-12-25T16:53:12.088413Z","iopub.status.idle":"2022-12-25T18:27:14.195611Z","shell.execute_reply.started":"2022-12-25T16:53:12.088386Z","shell.execute_reply":"2022-12-25T18:27:14.194570Z"},"trusted":true},"execution_count":null,"outputs":[]}]}