{"cells":[{"cell_type":"markdown","metadata":{"_cell_guid":"6f8f6b79-f37b-cad6-e093-a7fbd708ea08"},"source":"This is an attempt at visualizing the magic feature(outed by Faron) in how well it separates responses. Can be used to visualize any random feature's discriminating power."},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"8a9e103d-59d3-e412-b1ac-269269f1ba23"},"outputs":[],"source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nimport time"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"b28c7a16-46c1-8579-cba7-78cec3039f94"},"outputs":[],"source":"#Borrowed from Faron's Road to 4 kernel\nDATA_DIR = \"../input\"\n\nID_COLUMN = 'Id'\nTARGET_COLUMN = 'Response'\n\nSEED = 0\nCHUNKSIZE = 50000\nNROWS = 500000\n\nTRAIN_NUMERIC = \"{0}/train_numeric.csv\".format(DATA_DIR)\nTRAIN_DATE = \"{0}/train_date.csv\".format(DATA_DIR)\n\nTEST_NUMERIC = \"{0}/test_numeric.csv\".format(DATA_DIR)\nTEST_DATE = \"{0}/test_date.csv\".format(DATA_DIR)"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"449c3c71-6648-f866-453a-5dfb4034afa4"},"outputs":[],"source":"#Borrowed from Faron's Road to 4 kernel\n\ntrain = pd.read_csv(TRAIN_NUMERIC, usecols=[ID_COLUMN, TARGET_COLUMN], nrows=NROWS)\ntest = pd.read_csv(TEST_NUMERIC, usecols=[ID_COLUMN], nrows=NROWS)\n\ntrain[\"StartTime\"] = -1\ntest[\"StartTime\"] = -1"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"a4fe35e5-079a-b8e0-9aa3-6084cb023f4e"},"outputs":[],"source":"#Borrowed from Faron's Road to 4 kernel\n\nnrows = 0\n\nfor tr, te in zip(pd.read_csv(TRAIN_DATE, chunksize=CHUNKSIZE), pd.read_csv(TEST_DATE, chunksize=CHUNKSIZE)):\n    feats = np.setdiff1d(tr.columns, [ID_COLUMN])\n\n    stime_tr = tr[feats].min(axis=1).values\n    stime_te = te[feats].min(axis=1).values\n\n    train.loc[train.Id.isin(tr.Id), 'StartTime'] = stime_tr\n    test.loc[test.Id.isin(te.Id), 'StartTime'] = stime_te\n\n    nrows += CHUNKSIZE\n    if nrows >= NROWS:\n        break"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"7a670649-5336-8bc5-6bbd-64921d371e4c"},"outputs":[],"source":"#Borrowed from Faron's Road to 4 kernel\n#HAD to change the names so they are easy to type. what can I say ¯\\_(ツ)_/¯\n\nntrain = train.shape[0]\ntrain_test = pd.concat((train, test)).reset_index(drop=True).reset_index(drop=False)\n\ntrain_test['magic1'] = train_test[ID_COLUMN].diff().fillna(9999999).astype(int)\ntrain_test['magic2'] = train_test[ID_COLUMN].iloc[::-1].diff().fillna(9999999).astype(int)\n\ntrain_test = train_test.sort_values(by=['StartTime', 'Id'], ascending=True)\n\ntrain_test['magic3'] = train_test[ID_COLUMN].diff().fillna(9999999).astype(int)\ntrain_test['magic4'] = train_test[ID_COLUMN].iloc[::-1].diff().fillna(9999999).astype(int)\n\ntrain_test = train_test.sort_values(by=['index']).drop(['index'], axis=1)\ntrain = train_test.iloc[:ntrain, :]"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"88b2d9c7-5a1b-689d-f8c5-d2d247ac0ca2"},"outputs":[],"source":"def twoplot(df, col, xaxis=None):\n    ''' scatter plot a feature split into response values as two subgraphs '''\n    if col not in df.columns.values:\n        print('ERROR: %s not a column' % col)\n    ndf = pd.DataFrame(index = df.index)\n    ndf[col] = df[col]\n    ndf[xaxis] = df[xaxis] if xaxis else df.index\n    ndf['Response'] = df['Response']\n    \n    g = sns.FacetGrid(ndf, col=\"Response\", hue=\"Response\")\n    g.map(plt.scatter, xaxis, col, alpha=.7, s=1)\n    g.add_legend();\n    \n    del ndf"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"8fa661ec-814f-b09d-f304-132b52f98634"},"outputs":[],"source":"twoplot(train, 'magic1')"},{"cell_type":"markdown","metadata":{"_cell_guid":"1da24cbb-e6e8-f052-83a2-64fb99545db0"},"source":"Doesn't look very discriminating"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"fb86893c-735e-1485-fdc6-0449b0b9c69b"},"outputs":[],"source":"twoplot(train, 'magic2')"},{"cell_type":"markdown","metadata":{"_cell_guid":"e321edf8-4469-3061-d6dc-9bbf651d3db2"},"source":"Still doesn't look very discriminating"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"9400f35e-e464-fb6b-dfc0-bf8fdca905fa"},"outputs":[],"source":"-twoplot(train, 'magic3')"},{"cell_type":"markdown","metadata":{"_cell_guid":"52efadaa-4aac-2186-c4bb-5d7a619b04cd"},"source":"The two plots are clearly different and there seems to be some signal. It's not as clear as putting a threshold to identify response 0 vs response 1. But XGboost can use this signal (as is apparent from leaderboard positions of those using it)"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"280a52e2-41a2-cf47-2559-f6798e539261"},"outputs":[],"source":"twoplot(train, 'magic4')"},{"cell_type":"markdown","metadata":{"_cell_guid":"27fec23e-b6fb-84de-325c-87f9bb9a187f"},"source":"Looks similar to magic3 in shape but the graph is skewed because there is an outlier in the left hand side plot to the far right at value 1.0-1e7"}],"metadata":{"_change_revision":0,"_is_fork":false,"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.5.2"}},"nbformat":4,"nbformat_minor":0}