{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\nfrom tensorflow.keras import layers\nfrom tensorflow.keras import Sequential\nfrom tensorflow.keras import Model\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import StratifiedKFold\n\nimport os\nimport gc\nimport joblib\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn import metrics\nfrom tensorflow.keras import optimizers\nfrom tensorflow.keras.models import Model, load_model\nfrom tensorflow.keras import callbacks\nfrom tensorflow.keras import backend as K\nfrom tensorflow.keras import utils\nimport tensorflow as tf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train=pd.read_csv('../input/cat-in-the-dat-ii/train.csv')\ntest=pd.read_csv('../input/cat-in-the-dat-ii/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"training data size: \", train.shape)\nprint(\"training data size: \", test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_model(data, features):\n    inputs=[]\n    output=[]\n    \n    for c in features:\n        num_unique_values = int(data[c].nunique())\n        embed_dim = int(min(np.ceil((num_unique_values)/2), 50))\n        inp = layers.Input(shape=(1,))\n        out = layers.Embedding(num_unique_values + 1, embed_dim, name=c)(inp)\n        out = layers.SpatialDropout1D(0.3)(out)\n        out = layers.Reshape(target_shape=(embed_dim, ))(out)\n        inputs.append(inp)\n        output.append(out)\n        \n    x=layers.Concatenate()(output)\n    x = layers.BatchNormalization()(x)\n    \n    x = layers.Dense(300, activation=\"relu\")(x)\n    x = layers.Dropout(0.3)(x)\n    x = layers.BatchNormalization()(x)\n    \n    x = layers.Dense(300, activation=\"relu\")(x)\n    x = layers.Dropout(0.3)(x)\n    x = layers.BatchNormalization()(x)\n    \n    y = layers.Dense(2, activation=\"softmax\")(x)\n\n    model = Model(inputs=inputs, outputs=y)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def auc(y_true, y_pred):\n    def fallback_auc(y_true, y_pred):\n        try:\n            return metrics.roc_auc_score(y_true, y_pred)\n        except:\n            return 0.5\n    return tf.py_function(fallback_auc, (y_true, y_pred), tf.double)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test[\"target\"] = -1\ndata = pd.concat([train, test]).reset_index(drop=True)\nfeatures = [x for x in train.columns if x not in [\"id\", \"target\"]]\nfor feat in features:\n    lbl_enc = LabelEncoder()\n    data[feat] = lbl_enc.fit_transform(data[feat].fillna(\"-1\").astype(str).values)\ntrain = data[data.target != -1].reset_index(drop=True)\ntest = data[data.target == -1].reset_index(drop=True)\ntest_data = [test.loc[:, features].values[:, k] for k in range(test.loc[:, features].values.shape[1])]\noof_preds = np.zeros((len(train)))\ntest_preds = np.zeros((len(test)))\n\nskf = StratifiedKFold(n_splits=50)\nfor train_index, test_index in skf.split(train, train.target.values):\n    X_train, X_test = train.iloc[train_index, :], train.iloc[test_index, :]\n    X_train = X_train.reset_index(drop=True)\n    X_test = X_test.reset_index(drop=True)\n    y_train, y_test = X_train.target.values, X_test.target.values\n    model = create_model(data, features)\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[auc])\n    X_train = [X_train.loc[:, features].values[:, k] for k in range(X_train.loc[:, features].values.shape[1])]\n    X_test = [X_test.loc[:, features].values[:, k] for k in range(X_test.loc[:, features].values.shape[1])]\n    \n    es = callbacks.EarlyStopping(monitor='val_auc', min_delta=0.001, patience=5,\n                                 verbose=1, mode='max', baseline=None, restore_best_weights=True)\n\n    rlr = callbacks.ReduceLROnPlateau(monitor='val_auc', factor=0.5,\n                                      patience=3, min_lr=1e-6, mode='max', verbose=1)\n    \n    model.fit(X_train,\n              utils.to_categorical(y_train),\n              validation_data=(X_test, utils.to_categorical(y_test)),\n              verbose=1,\n              batch_size=1024,\n              callbacks=[es, rlr],\n              epochs=100\n             )\n    valid_fold_preds = model.predict(X_test)[:, 1]\n    test_fold_preds = model.predict(test_data)[:, 1]\n    oof_preds[test_index] = valid_fold_preds.ravel()\n    test_preds += test_fold_preds.ravel()\n    print(metrics.roc_auc_score(y_test, valid_fold_preds))\n    K.clear_session()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tf.keras.models.save_model(model, '../output/kaggle/working/', overwrite=True, include_optimizer=True, save_format=None, signatures=None, options=None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Overall AUC={}\".format(metrics.roc_auc_score(train.target.values, oof_preds)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_preds /= 50\ntest_ids = test.id.values\nprint(\"Saving submission file\")\nsubmission = pd.DataFrame.from_dict({\n    'id': test_ids,\n    'target': test_preds\n})\nsubmission.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train_df['target'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target=train_df['target']\ndel train_df['target']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['nom_0'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['nom_1'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['nom_2'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['nom_3'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(10):\n    print(train_df['nom_{}'.format(i)].value_counts())\n    print('\\n')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Encoding"},{"metadata":{"trusted":true},"cell_type":"code","source":"data=pd.concat([train_df, test_df])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cols=['bin_0', 'bin_1', 'bin_2', 'bin_3', 'bin_4']\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mode = data.filter(cols).mode()\ndata[cols]=data[cols].fillna(mode.iloc[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data['bin_3']=data['bin_3'].map({'F': 0, 'T':1})\ndata['bin_4']=data['bin_4'].map({'Y':1, 'N':0})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data[['bin_0', 'bin_1', 'bin_2']]=data[['bin_0', 'bin_1', 'bin_2']].astype('int8')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cols=[]\nfor i in range(10):\n    cols.append(\"nom_{}\".format(i))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data=data.fillna(-1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.reset_index(inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data=data.drop('id', axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data=data.drop('index', axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\nfeatures=[i for i in data.columns]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"le=LabelEncoder()\nfor i in features:\n    data[i]=le.fit_transform(data[i].astype(str))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## CH-SQAURE TEST"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data=data[:600000]\ntest_data=data[600000:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_test, y_train, y_test=train_test_split(train_data, target, test_size=.20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"classifier=RandomForestClassifier(n_estimators=1200, max_depth=100, n_jobs=-1, verbose=2, max_features='sqrt')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"classifier.fit(X_train, y_train, )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred=classifier.predict(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import f1_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fs=f1_score(y_test, pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"## Lightgbm"},{"metadata":{"trusted":true},"cell_type":"code","source":"import lightgbm as lgb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"param = {\n    'num_leaves': 31,\n    'objective': 'binary'\n    'metric': ''}\nparam['metric'] = ['auc', 'binary_logloss']","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}