{"cells":[{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"314f5ebb-87af-9b18-3226-d4c731f4ea68"},"outputs":[],"source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nfrom subprocess import check_output\nprint(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n\n# Any results you write to the current directory are saved as output."},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"3c2119de-7ada-48cf-a8d4-b422485d9e61"},"outputs":[],"source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nimport time"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"2a989190-77c7-0a47-cb97-5f21f6d88ee9"},"outputs":[],"source":"#Borrowed from Faron's Road to 4 kernel\nDATA_DIR = \"../input\"\n\nID_COLUMN = 'Id'\nTARGET_COLUMN = 'Response'\n\nSEED = 0\nCHUNKSIZE = 50000\nNROWS = 500000\n\nTRAIN_NUMERIC = \"{0}/train_numeric.csv\".format(DATA_DIR)\nTRAIN_DATE = \"{0}/train_date.csv\".format(DATA_DIR)\n\nTEST_NUMERIC = \"{0}/test_numeric.csv\".format(DATA_DIR)\nTEST_DATE = \"{0}/test_date.csv\".format(DATA_DIR)"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"4c7f5fe4-ae04-9183-21bd-a70990acb28d"},"outputs":[],"source":"#Borrowed from Faron's Road to 4 kernel\n\ntrain = pd.read_csv(TRAIN_NUMERIC, usecols=[ID_COLUMN, TARGET_COLUMN], nrows=NROWS)\ntest = pd.read_csv(TEST_NUMERIC, usecols=[ID_COLUMN], nrows=NROWS)\n\ntrain[\"StartTime\"] = -1\ntest[\"StartTime\"] = -1"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"5ee1aeb2-e8b9-3bf2-8de2-6c4e8fd59f76"},"outputs":[],"source":"#Borrowed from Faron's Road to 4 kernel\n\nnrows = 0\n\nfor tr, te in zip(pd.read_csv(TRAIN_DATE, chunksize=CHUNKSIZE), pd.read_csv(TEST_DATE, chunksize=CHUNKSIZE)):\n    feats = np.setdiff1d(tr.columns, [ID_COLUMN])\n\n    stime_tr = tr[feats].min(axis=1).values\n    stime_te = te[feats].min(axis=1).values\n\n    train.loc[train.Id.isin(tr.Id), 'StartTime'] = stime_tr\n    test.loc[test.Id.isin(te.Id), 'StartTime'] = stime_te\n\n    nrows += CHUNKSIZE\n    if nrows >= NROWS:\n        break"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"73f0950f-4511-d599-cdf7-1b5362628748"},"outputs":[],"source":"#Borrowed from Faron's Road to 4 kernel\n#HAD to change the names so they are easy to type. what can I say ¯\\_(ツ)_/¯\n\nntrain = train.shape[0]\ntrain_test = pd.concat((train, test)).reset_index(drop=True).reset_index(drop=False)\n\ntrain_test['magic1'] = train_test[ID_COLUMN].diff().fillna(9999999).astype(int)\ntrain_test['magic2'] = train_test[ID_COLUMN].iloc[::-1].diff().fillna(9999999).astype(int)\n\ntrain_test = train_test.sort_values(by=['StartTime', 'Id'], ascending=True)\n\ntrain_test['magic3'] = train_test[ID_COLUMN].diff().fillna(9999999).astype(int)\ntrain_test['magic4'] = train_test[ID_COLUMN].iloc[::-1].diff().fillna(9999999).astype(int)\n\ntrain_test = train_test.sort_values(by=['index']).drop(['index'], axis=1)\ntrain = train_test.iloc[:ntrain, :]"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"f1150feb-9dd6-9d3e-6955-4c5637e2a143"},"outputs":[],"source":"def twoplot(df, col, xaxis=None):\n    ''' scatter plot a feature split into response values as two subgraphs '''\n    if col not in df.columns.values:\n        print('ERROR: %s not a column' % col)\n    ndf = pd.DataFrame(index = df.index)\n    ndf[col] = df[col]\n    ndf[xaxis] = df[xaxis] if xaxis else df.index\n    ndf['Response'] = df['Response']\n    \n    g = sns.FacetGrid(ndf, col=\"Response\", hue=\"Response\")\n    g.map(plt.scatter, xaxis, col, alpha=.7, s=1)\n    g.add_legend();\n    \n    del ndf"}],"metadata":{"_change_revision":0,"_is_fork":false,"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.5.2"}},"nbformat":4,"nbformat_minor":0}