{"cells":[{"metadata":{},"cell_type":"markdown","source":"In this notebook we'll try to use adversarial validation in order to see how similar/different the train and test sets are. Since we have to merge several datasets in order to get \"trainable\" data, we'll rely on other kernesl in which this has already been don. This starter version of Adversarial Validation will rely on my [Simple Linear Regression Benchmark kernel](https://www.kaggle.com/tunguz/simple-linear-regression-benchmark/) for train and test sets. "},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport lightgbm as lgb\nfrom sklearn.model_selection import KFold\nfrom sklearn import model_selection, preprocessing, metrics\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.metrics import roc_auc_score\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\nimport shap\nimport os\nprint(os.listdir(\"../input\"))\nfrom sklearn import preprocessing\nimport xgboost as xgb\nimport gc\n\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"%%time\ntrain = pd.read_csv('../input/simple-linear-regression-benchmark/new_train.csv')\ntest = pd.read_csv('../input/simple-linear-regression-benchmark/new_test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"columns = train.columns\ncolumns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.dtypes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train.sample(frac=0.1, random_state=3)\ntest = test.sample(frac=0.05, random_state=3)\n\n#train = train.sample(frac=0.2, random_state=3)\n#test = test.sample(frac=0.1, random_state=3)\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['target'] = 0\ntest['target'] = 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_test = pd.concat([train, test], axis =0)\n\n\n\n#imp = SimpleImputer(missing_values=np.nan, strategy='mean')\n#imp.fit(train_test[columns])\n#train_test[columns] = imp.transform(train_test[columns])\ntarget = train_test['target'].values\n\n\ndel train, test\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train, test = model_selection.train_test_split(train_test, test_size=0.33, random_state=42, shuffle=True)\ndel train_test\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['target']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train_y = train['target'].values.reshape(-1, 1)\n#test_y = test['target'].values.reshape(-1,1)\ntrain_y = train['target'].values\ntest_y = test['target'].values\ndel train['target'], test['target']\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = lgb.Dataset(train, label=train_y)\ntest = lgb.Dataset(test, label=test_y)\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train = train.values\n#test = test.values\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"param = {'num_leaves': 50,\n         'min_data_in_leaf': 30, \n         'objective':'binary',\n         'max_depth': 2,\n         'learning_rate': 0.2,\n         \"min_child_samples\": 20,\n         \"boosting\": \"gbdt\",\n         \"feature_fraction\": 0.9,\n         \"bagging_freq\": 1,\n         \"bagging_fraction\": 0.9 ,\n         \"bagging_seed\": 44,\n         \"metric\": 'auc',\n         \"verbosity\": -1}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_round = 50\nclf = lgb.train(param, train, num_round, valid_sets = [train, test], verbose_eval=50, early_stopping_rounds = 50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#clf = LogisticRegression()\n#clf.fit(train, train_y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#preds[:,0].shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#test_y.flatten().shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#roc_auc_score(test_y.flatten(), preds[:,1])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"AUC of 0.52 is actually pretty decent. Seems that at least at this level of analysis there is not much difference between the train and test sets. "},{"metadata":{},"cell_type":"markdown","source":"AUC of 0.56 reveals some dicrepancy between the train and test sets. Let's take a look at the feature importances:"},{"metadata":{"trusted":true},"cell_type":"code","source":"feature_imp = pd.DataFrame(sorted(zip(clf.feature_importance(),columns)), columns=['Value','Feature'])\n\nplt.figure(figsize=(20, 10))\nsns.barplot(x=\"Value\", y=\"Feature\", data=feature_imp.sort_values(by=\"Value\", ascending=False).head(20))\nplt.title('LightGBM Features')\nplt.tight_layout()\nplt.show()\nplt.savefig('lgbm_importances-01.png')\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Looks like Dew Temperature is the most distinct feature between the train and test datasets."},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}