{"cells":[{"metadata":{},"cell_type":"markdown","source":"A Simple implementation of Logistic Regression with Single Layer Perceptron. The data preparation part is from [oh-my-plain-logreg](https://www.kaggle.com/superant/oh-my-plain-logreg)"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import roc_auc_score\nfrom scipy.stats import rankdata\nimport pandas as pd\nimport numpy as np\nimport scipy\nfrom keras.regularizers import l1, l2, l1_l2\nfrom keras.models import Sequential\nfrom keras.layers import Dense\nimport tensorflow as tf\nfrom keras import backend as K\nimport tensorflow as tf\nfrom keras import callbacks\nfrom keras.utils import to_categorical\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SEED = 2020\nN_Splits = 25\nVerbose = 0\nBatch_Size = 256\nEpochs = 30\nRegularization = l1_l2(l1=0.0, l2=0.0)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"D0 = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/train.csv', index_col='id')\nD_test = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/test.csv', index_col='id')\ny_train = D0['target']\nD = D0.drop(columns='target')\ntest_ids = D_test.index\nD_all = pd.concat([D, D_test])\nnum_train = len(D)\nprint(f'Data Shape : {D_all.shape}')        ","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"for col in D.columns.difference(['id']):\n    train_vals = set(D[col].dropna().unique())\n    test_vals = set(D_test[col].dropna().unique())\n\n    xor_cat_vals = train_vals ^ test_vals\n    if xor_cat_vals:\n        print(f'Replacing {len(xor_cat_vals)} values in {col}, {xor_cat_vals}')\n        D_all.loc[D_all[col].isin(xor_cat_vals), col] = 'xor'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ord_maps = {\n    'ord_0': {val: i for i, val in enumerate([1, 2, 3])},\n    'ord_1': {\n        val: i\n        for i, val in enumerate(\n            ['Novice', 'Contributor', 'Expert', 'Master', 'Grandmaster']\n        )\n    },\n    'ord_2': {\n        val: i\n        for i, val in enumerate(\n            ['Freezing', 'Cold', 'Warm', 'Hot', 'Boiling Hot', 'Lava Hot']\n        )\n    },\n    'ord_3': {val: i for i, val in enumerate(sorted(D_all['ord_3'].dropna().unique()))},\n    'ord_4': {val: i for i, val in enumerate(sorted(D_all['ord_4'].dropna().unique()))},\n    'ord_5': {val: i for i, val in enumerate(sorted(D_all['ord_5'].dropna().unique()))},\n}\n\nord_cols = pd.concat([D_all[col].map(ord_map).fillna(max(ord_map.values())//2).astype('float32') for col, ord_map in ord_maps.items()], axis=1)\nord_cols /= ord_cols.max() \nord_cols_sqr = 4*(ord_cols - 0.5)**2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oh_cols = D_all.columns.difference(ord_maps.keys())\nprint(f'OneHot encoding {len(oh_cols)} columns')\nX_oh1 = pd.get_dummies(\n    D_all[oh_cols],\n    columns=oh_cols,\n    drop_first=True,\n    dummy_na=True,\n    sparse=True,\n    dtype='int8',\n).sparse.to_coo()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_oh = scipy.sparse.hstack([X_oh1, ord_cols, ord_cols_sqr]).tocsr()\nprint(f'X_oh.shape = {X_oh.shape}')\nX_train = X_oh[:num_train]\nX_test = X_oh[num_train:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def auc(y_true, y_pred):\n    def fallback_auc(y_true, y_pred):\n        try:\n            return roc_auc_score(y_true, y_pred)\n        except:\n            return 0.5\n    return tf.py_function(fallback_auc, (y_true, y_pred), tf.double)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oof_pred_perceptron = np.zeros((X_train.shape[0]), )\ny_pred_perceptron   = np.zeros((X_test.shape[0]), )\n\nskf = StratifiedKFold(n_splits=N_Splits, shuffle=True, random_state=SEED)\n\nfor fold, (tr_ind, val_ind) in enumerate(skf.split(X_train, y_train)):\n    x_tr, x_val = X_train[tr_ind], X_train[val_ind]\n    y_tr, y_val = y_train[tr_ind], y_train[val_ind]\n    train_set = {'X':x_tr, 'y':to_categorical(y_tr)}\n    val_set   = {'X':x_val, 'y':to_categorical(y_val)}\n    model = Sequential()\n    model.add(Dense(2, activation='softmax', kernel_regularizer=Regularization, input_dim=X_train.shape[1]))\n    model.compile(optimizer='adam',\n                  loss='categorical_crossentropy',\n                  metrics=[auc],)\n    es = callbacks.EarlyStopping(monitor='val_auc', min_delta=0.001, patience=5, verbose=Verbose, mode='max', baseline=None, restore_best_weights=True)\n    sb = callbacks.ModelCheckpoint('./nn_model.w8', save_weights_only=True, save_best_only=True, verbose=Verbose)\n    annealer = callbacks.LearningRateScheduler(lambda x: 1e-3 * 0.99 ** (x+(Epochs//3)))\n    model.fit(train_set['X'],\n              train_set['y'],\n              epochs=Epochs,\n              verbose=Verbose,\n              validation_data=(val_set['X'],val_set['y']),\n              batch_size=Batch_Size,\n              callbacks=[es, sb, annealer])\n    model.load_weights('./nn_model.w8')\n    fold_pred = model.predict(val_set['X'])[:,1]\n    oof_pred_perceptron[val_ind] = fold_pred\n    y_pred_perceptron += model.predict(X_test)[:,1] / (N_Splits)\n    oof_auc_score = roc_auc_score(y_val, fold_pred)\n    print(f'fold {fold+1:02} auc score is: {oof_auc_score:.6f}')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oof_auc_score = roc_auc_score(y_train, oof_pred_perceptron)\nprint(f'SingleLayerPerceptron OOF auc score is: {oof_auc_score}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.DataFrame({'id': test_ids, 'target': y_pred_perceptron}).to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.save('oof_pred_perceptron.npy', oof_pred_perceptron)\nnp.save('y_pred_perceptron.npy',    y_pred_perceptron)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}