{"cells":[{"cell_type":"markdown","metadata":{"_cell_guid":"cb207525-2807-81af-adab-a2c27b95cab7"},"source":"Can we predict anomalies using the numerical feature density? It doesn't look like it...\n"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"74e5dd47-d604-6354-7502-4afe7b4af08e"},"outputs":[],"source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport sys"},{"cell_type":"markdown","metadata":{"_cell_guid":"e4ab94c1-f12e-cc04-9a24-1d7c9aed4879"},"source":"Author = Alex Galea   \nDate = Sept 12th, 2016"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"e45d27d0-27ff-6c42-0675-ddcb48931e76"},"outputs":[],"source":"sys.version"},{"cell_type":"markdown","metadata":{"_cell_guid":"427db241-5f74-b9c0-9775-05d765cb1a89"},"source":"**Loading the numerical dataset**:\n   "},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"9d1e6f25-b2f3-a30f-a1da-660e782fcfeb"},"outputs":[],"source":"def df_load(file, batch_size, skip = 1):\n    ''' Build dataframe by iterating over chunks. Option to skip chunks and\n        therefore read in less data. '''\n\n    reader = pd.read_csv(file, chunksize=batch_size,\n                         dtype=np.float16)\n\n    df = pd.concat((chunk for i, chunk in enumerate(reader) if i % skip == 0))\n\n    return df"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"91b6f12d-9cc6-8d96-68cd-05a708053b9f"},"outputs":[],"source":"input_path = '../input/'\ndf_num = df_load(input_path+'train_numeric.csv',\n                 batch_size=100000)"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"c1b8f01f-9cd7-a3b0-5694-6d836c3021de"},"outputs":[],"source":"df_num.head()"},{"cell_type":"markdown","metadata":{"_cell_guid":"d9548704-5529-46bc-8381-e4064dc197f2"},"source":"Get a random selection of models:"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"2382381e-4b16-cf52-9512-458044d36e8d"},"outputs":[],"source":"np.random.seed(117)\nnum_samples = 200\nrandom_picks = (np.array(np.random.random(num_samples)) * df_num.shape[1]).astype(int)\nrandom_picks = sorted(random_picks)\nprint(random_picks)"},{"cell_type":"markdown","metadata":{"_cell_guid":"c528835f-bf0e-8371-034d-93a926cb9b7c"},"source":"Plot the feature densities, comparing response = 1 targets with response = 0"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"66202e9b-f5c3-8a34-38c4-95e119c471de"},"outputs":[],"source":"for feature in random_picks[:150]:\n    fig, axes = plt.subplots(1, 2, figsize=(15, 5))\n    # Plot linear scale feature density\n    df_num.iloc[:, feature][(df_num.Response == 0)].dropna().hist(bins=50, alpha=0.4, normed=True, ax=axes[0],\n                                                         label='%d response 0 samples' % len(df_num.iloc[:, feature][(df_num.Response == 0)].dropna()))\n    df_num.iloc[:, feature][(df_num.Response == 1)].dropna().hist(bins=50, alpha=0.4, color='r', normed=True, ax=axes[0],\n                                                         label='%d response 1 samples' % len(df_num.iloc[:, feature][(df_num.Response == 1)].dropna()))\n    # Plot log scale feature density\n    df_num.iloc[:, feature][(df_num.Response == 0)].dropna().hist(bins=50, alpha=0.4, normed=True, ax=axes[1], log=True)\n    df_num.iloc[:, feature][(df_num.Response == 1)].dropna().hist(bins=50, alpha=0.4, color='r', normed=True, ax=axes[1], log=True)\n\n    axes[0].legend(loc='upper left')\n    plt.suptitle('Feature column \"%s\" histogram' % df_num.columns[feature], fontsize=15)\n    plt.show()"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"aace4901-0acd-1d15-3979-18297367be81"},"outputs":[],"source":""}],"metadata":{"_change_revision":0,"_is_fork":false,"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.5.2"}},"nbformat":4,"nbformat_minor":0}