{"cells":[{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"_kg_hide-output":false},"cell_type":"code","source":"# fastai 0.7.2\n!pip install git+https://github.com/fastai/fastai@e85667cfae2e6873b1bb026195b5d09a74dfcff9\n%load_ext autoreload\n%autoreload 2\n\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"799f8bd3e89eafd50306b578a6053ed75a3acdc6"},"cell_type":"code","source":"from fastai.imports import *\nfrom fastai.structured import *\n\nfrom pandas_summary import DataFrameSummary\nfrom sklearn.ensemble import RandomForestRegressor, RandomForestClassifier\nfrom IPython.display import display\n\nfrom sklearn import metrics","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4e5200a45795932de5ac810e5b017512a61c4885"},"cell_type":"code","source":"PATH = \"../input/\"\n\ndef display_all(df):\n    with pd.option_context(\"display.max_rows\", 1000, \"display.max_columns\", 1000): \n        display(df)\n\nimport os\nimport json\nfrom pandas.io.json import json_normalize\n\ndef load_df(csv_path, nrows=None):\n    JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']\n    types = {\n        \"fullVisitorId\": \"str\", # readme says it should be str\n        \"channelGrouping\": \"str\",\n        \"date\": \"str\",\n        \"socialEngagementType\": \"str\",\n        \"visitId\": \"int32\",\n        \"visitNumber\": \"int8\",\n        \"visitStartTime\": \"int32\",\n    }\n    \n    df = pd.read_csv(csv_path, \n                     converters={column: json.loads for column in JSON_COLUMNS}, \n                     dtype=types,\n                     nrows=nrows)\n    \n    for column in JSON_COLUMNS:\n        column_as_df = json_normalize(df[column])\n        column_as_df.columns = [f\"{column}.{subcolumn}\" for subcolumn in column_as_df.columns]\n        df = df.drop(column, axis=1).merge(column_as_df, right_index=True, left_index=True)\n    print(f\"Loaded {os.path.basename(csv_path)}. Shape: {df.shape}\")\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"06fb9fb27c5635e212ce00f2a216dd051331cb02"},"cell_type":"code","source":"def split_vals(a,n): \n    return a[:n].copy(), a[n:].copy()\n\ndef rmse(x,y):\n    return math.sqrt(((x-y)**2).mean())\n\ndef print_score(m):\n    res = [rmse(m.predict(X_train), y_train), rmse(m.predict(X_valid), y_valid),\n                m.score(X_train, y_train), m.score(X_valid, y_valid)]\n    if hasattr(m, 'oob_score_'): res.append(m.oob_score_)\n    print(res)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f3b8fdca0e3d46c62e5c01bc7fc0777d93ba4c7d"},"cell_type":"code","source":"# df_raw = pd.read_csv(f'{PATH}train_v2.csv', low_memory=False, nrows=500_000, dtype={'fullVisitorId': 'str'})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aa1d92374c339bb88a269aa8b782423f9b4b1cb5"},"cell_type":"code","source":"# display_all(df_raw.head().T)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1b142bff5ed3d615cc171ddfd0b22316996c0807"},"cell_type":"code","source":"# display_all(df_raw.describe(include=\"all\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"27711de2e1faaa7c3e0dd763aceb162799c7555a"},"cell_type":"code","source":"df_raw = load_df(f'{PATH}train_v2.csv', nrows=500_000)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cf83f7c35b979426614b08f2731e0a6638bbdf6e"},"cell_type":"markdown","source":"Data Fields\n\n    fullVisitorId - A unique identifier for each user of the Google Merchandise Store.\n    channelGrouping - The channel via which the user came to the Store.\n    date - The date on which the user visited the Store.\n    device - The specifications for the device used to access the Store.\n    geoNetwork - This section contains information about the geography of the user.\n    socialEngagementType - Engagement type, either \"Socially Engaged\" or \"Not Socially Engaged\".\n    totals - This section contains aggregate values across the session.\n    trafficSource - This section contains information about the Traffic Source from which the session originated.\n    visitId - An identifier for this session. This is part of the value usually stored as the _utmb cookie. This is only unique to the user. For a completely unique ID, you should use a combination of fullVisitorId and visitId.\n    visitNumber - The session number for this user. If this is the first session, then this is set to 1.\n    visitStartTime - The timestamp (expressed as POSIX time).\n    hits - This row and nested fields are populated for any and all types of hits. Provides a record of all page visits.\n    customDimensions - This section contains any user-level or session-level custom dimensions that are set for a session. This is a repeated field and has an entry for each dimension that is set.\n    totals - This set of columns mostly includes high-level aggregate data.\n"},{"metadata":{"trusted":true,"_uuid":"a781130a4de5223b500e10ee9252dab29fd4b3b0"},"cell_type":"code","source":"display_all(df_raw.head().T)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"56eb443f8e6ef5b9e090841c7f00c207a5461f3f"},"cell_type":"code","source":"display_all(df_raw.describe(include=\"all\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e4a1222a4fdc91656219c7df783d60b5af06e030"},"cell_type":"code","source":"const_cols = [c for c in df_raw.columns if df_raw[c].nunique(dropna=False)==1]\ndf_raw.drop(columns=const_cols + [\"customDimensions\", \"hits\"], inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3d5ca8e045d10fd3c97bc68ee4258d3c10168a1a"},"cell_type":"code","source":"df_test = load_df(f'{PATH}test_v2.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ba593a148fbbc5b61b3e5edaca96b6d3a5a3e9a1"},"cell_type":"code","source":"display_all(df_test.head().T)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a2410a4dfa4c5771a92dd7c28fd735614d1f1691"},"cell_type":"code","source":"list((set(df_raw.columns).difference(set(df_test.columns))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d4f5525b14bc45bb870d151685e6957ce73382d6"},"cell_type":"code","source":"df_raw.drop(columns=['trafficSource.campaignCode'], inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cc0daa5a1d5a618c52ca36d5bbe7da1eea513a5a"},"cell_type":"code","source":"df_test.drop(columns=const_cols + [\"customDimensions\", \"hits\"], inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"899e9b4e371fdd324848f3eb880cafd489d91139"},"cell_type":"markdown","source":"We are predicting the natural log of the sum of all transactions per user. Once the data is updated, as noted above, this will be for all users in test_v2.csv for December 1st, 2018 to January 31st, 2019."},{"metadata":{"trusted":true,"_uuid":"93430b569c24df4bc94656d04af06dac2cde3ddf"},"cell_type":"code","source":"df_raw['totals.transactionRevenue'].fillna(0, inplace=True)\ndf_raw['totals.transactionRevenue'] = df_raw['totals.transactionRevenue'].astype(float)\ndf_raw['totals.transactionRevenue'] = np.log1p(df_raw['totals.transactionRevenue'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1df6923cf3b5dd2d40edb14496670d05fdd182e9"},"cell_type":"code","source":"df_raw[\"visitStartTime\"] = pd.to_datetime(df_raw[\"visitStartTime\"], infer_datetime_format=True, unit=\"s\")\ndf_raw[\"date\"] = pd.to_datetime(df_raw[\"date\"], infer_datetime_format=True, format=\"%Y%m%d\")\nadd_datepart(df_raw, 'date')\nadd_datepart(df_raw, 'visitStartTime')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"44bf239e9fd9f60f5ae292f0cb148e2fb8a88d25"},"cell_type":"code","source":"df_raw['totals.totalTransactionRevenue'].fillna(0, inplace=True)\ndf_raw['totals.totalTransactionRevenue'] = df_raw['totals.totalTransactionRevenue'].astype('int32')\ndf_raw['totals.transactions'].fillna(0, inplace=True)\ndf_raw['totals.transactions'] = df_raw['totals.transactions'].astype('int8')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6179453d5be37bb6144317dbff51ab600ee172f3"},"cell_type":"code","source":"df_test['totals.transactionRevenue'].fillna(0, inplace=True)\ndf_test['totals.transactionRevenue'] = df_test['totals.transactionRevenue'].astype(float)\ndf_test['totals.transactionRevenue'] = np.log1p(df_test['totals.transactionRevenue'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"181ad3acd2b2dfe469695b8b31e7537ecc96fe42"},"cell_type":"code","source":"df_test[\"visitStartTime\"] = pd.to_datetime(df_test[\"visitStartTime\"], infer_datetime_format=True, unit=\"s\")\ndf_test[\"date\"] = pd.to_datetime(df_test[\"date\"], infer_datetime_format=True, format=\"%Y%m%d\")\nadd_datepart(df_test, 'date')\nadd_datepart(df_test, 'visitStartTime')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"561eb634c1f4b4e80e655b93aa9ec323295dba93"},"cell_type":"code","source":"display_all(df_raw.head().T)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"49c3069c64515181b707297b60c51f1cbc6a7945"},"cell_type":"code","source":"train_cats(df_raw)\ntrain_cats(df_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4ea57bbd1ee1045a62d3c6bd8c4fc4485417e824"},"cell_type":"code","source":"os.makedirs('tmp', exist_ok=True)\n# df_raw.to_feather('tmp/ga-raw')\n# df_test.to_feather('tmp/ga-test')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e15b66de0eaa8649785a2b4be59c1ac80e2ce1e8"},"cell_type":"code","source":"# df_raw = pd.read_feather('tmp/ga-raw')\n# df_test = pd.read_feather('tmp/ga-test')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9da94b66dfa4ed9945275af6f39622081b177c42"},"cell_type":"code","source":"# display_all(df_raw.isnull().sum().sort_index()/len(df_raw))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b08d744cb6be256e0a6d47b37bb0a21c91ba622c"},"cell_type":"code","source":"df_trn, y_trn, nas = proc_df(df_raw, 'totals.transactionRevenue')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"83e50da971fb052520af36cf25a0c178e434920f"},"cell_type":"code","source":"display_all(df_trn.head().T)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dc5720adf12e759149e7321223fc6f623a2d4bb6"},"cell_type":"markdown","source":"## See how much can we overfit - prediction without validation set"},{"metadata":{"trusted":true,"_uuid":"a8cc2514984401b845470c4fdfeaa6ef0300116d"},"cell_type":"code","source":"m = RandomForestRegressor(n_jobs=-1)\nm.fit(df_trn, y_trn)\nm.score(df_trn, y_trn)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e7ea098b1f150bf0a067963abd861a7ecdfed79c"},"cell_type":"markdown","source":"## Make it stupid, simple"},{"metadata":{"trusted":true,"_uuid":"0dc8dc8c0633d672257fa845cf9be69428912625"},"cell_type":"code","source":"train_required_ratio = 0.8\nn_trn = int(len(df_trn) * train_required_ratio)\n\nX_train, X_valid = split_vals(df_trn, n_trn)\ny_train, y_valid = split_vals(y_trn, n_trn)\nX_train.shape, X_valid.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"785a50817d4e7132fe4ece611f1a204cd2446878"},"cell_type":"code","source":"m = RandomForestRegressor(n_estimators=10, n_jobs=-1)\nm.fit(X_train, y_train)\nprint_score(m)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5dbcce517d56545ff2622b3c16b89aa0893bf453"},"cell_type":"markdown","source":"## Bagging"},{"metadata":{"trusted":true,"_uuid":"22cd6054c669139f9b4f47e11e28b0390ca01ce7"},"cell_type":"code","source":"preds = np.stack([t.predict(X_valid) for t in m.estimators_])\npreds[:,0], np.mean(preds[:,0]), y_valid[0]\npreds.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6d3604d7c671076fe03144e70e3133093d0cf04a"},"cell_type":"code","source":"plt.plot([metrics.r2_score(y_valid, np.mean(preds[:i+1], axis=0)) for i in range(10)]);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b2f18b4a999f01635735bbc158d6560a5950b30d"},"cell_type":"code","source":"m = RandomForestRegressor(n_estimators=40, n_jobs=-1, oob_score=True)\nm.fit(X_train, y_train)\nprint_score(m)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"76ab9d8b3721ce3e76fc3c722e3491188126a580"},"cell_type":"markdown","source":"No effect?"},{"metadata":{"trusted":true,"_uuid":"6876e0c52376d0ef9e6cbf46701765e41637e648"},"cell_type":"code","source":"set_rf_samples(50_000)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"49a4addcb2a020d3ff575a466ad7fce22118a9a6"},"cell_type":"code","source":"m = RandomForestRegressor(n_estimators=40, n_jobs=-1)\nm.fit(X_train, y_train)\nprint_score(m)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a61d39ae925a2e76138f46a59ff220dec7222895"},"cell_type":"code","source":"reset_rf_samples()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fc1ad7bbc4f62ac535c2236a9a3eca136e6c1a23"},"cell_type":"code","source":"m = RandomForestRegressor(n_estimators=40, min_samples_leaf=3, n_jobs=-1, oob_score=True)\nm.fit(X_train, y_train)\nprint_score(m)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ced2d13310017e79e2e497cc15913144bea2ff89"},"cell_type":"markdown","source":"# Predict on the test data"},{"metadata":{"trusted":true,"_uuid":"593be156ed89083298f1dc1512fc11261c9fa720"},"cell_type":"code","source":"df_trn, y_trn, nas = proc_df(df_raw, 'totals.transactionRevenue')\ntrain_required_ratio = 0.8\nn_trn = int(len(df_trn) * train_required_ratio)\n\nX_train, X_valid = split_vals(df_trn, n_trn)\ny_train, y_valid = split_vals(y_trn, n_trn)\n\nX_test, y_test, _ = proc_df(df_test, 'totals.transactionRevenue', nas)\nX_train.shape, X_valid.shape, X_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"639dce6238963298ec7200cdfa2514de81e32e72"},"cell_type":"code","source":"m = RandomForestRegressor(n_estimators=100, min_samples_leaf=3, n_jobs=-1, oob_score=True)\nm.fit(X_train, y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4da1c3e98605792860b6db2dee5153d80d1a4034"},"cell_type":"code","source":"# rmse(m.predict(X_train), y_train), rmse(m.predict(X_test), y_test), m.score(X_train, y_train), m.score(X_test, y_test), m.oob_score_\nprint_score(m)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"002c07588d0a10712f7427145d06e92939b2244a"},"cell_type":"code","source":"m.feature_importances_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bb2ddabe9cb590cf46679f43c3e5680505fdaea5"},"cell_type":"code","source":"X_train.columns[22]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dabc6930e67f7103fe4f76df695ccc10ddca41fc"},"cell_type":"code","source":"predictions = m.predict(X_test)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4026e986420342097dbb9471a43abd4abb5a27a3"},"cell_type":"markdown","source":"# Confidence based on tree variance"},{"metadata":{"trusted":true,"_uuid":"f5abd1e43cfda5f8f49a435ff7e01f35fd1defa8"},"cell_type":"code","source":"set_rf_samples(50000)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"994381f234b2b266b644534c5f50e9b5aef38710","scrolled":true},"cell_type":"code","source":"m = RandomForestRegressor(n_estimators=40, min_samples_leaf=3, max_features=0.5, n_jobs=-1, oob_score=True)\nm.fit(X_train, y_train)\nprint_score(m)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"837a6d02b59445d327ed6cf9280d8631fcf4076f"},"cell_type":"code","source":"raw_train, raw_valid = split_vals(df_raw, n_trn)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fa33c453fc522b5bbb16e48864f9dfab18a171fb"},"cell_type":"code","source":"x = raw_valid.copy()\nx['pred_std'] = np.std(preds, axis=0)\nx['pred'] = np.mean(preds, axis=0)\nx['totals.transactions'].value_counts().plot.barh();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f2085e9e0dddc03a3000feb145e1a8fae47564e8"},"cell_type":"code","source":"flds = ['totals.transactions', 'totals.transactionRevenue', 'pred', 'pred_std']\ntr_summ = x[flds].groupby('totals.transactions', as_index=False).mean()\ntr_summ = tr_summ[~pd.isnull(tr_summ['totals.transactionRevenue'])]\ntr_summ.plot('totals.transactions', 'totals.transactionRevenue', 'barh', xlim=(0,25));","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ebded0805296976c96d31b71ddf35f5ebb50c5e9"},"cell_type":"code","source":"tr_summ.plot('totals.transactions', 'pred', 'barh', xerr='pred_std', alpha=0.6, xlim=(0,25));","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a5c127e7bddd64e53b5fdf1c4389ed6c1fa9aad2"},"cell_type":"markdown","source":"# Feature Importance"},{"metadata":{"trusted":true,"_uuid":"e6ce91112745c592129f1d00af6a7e1462a718d3","scrolled":true},"cell_type":"code","source":"fi = rf_feat_importance(m, df_trn); fi[:30]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5daf3cb1dfeb2f588ccec416dc1bdd82e2578c06"},"cell_type":"code","source":"fi.plot('cols', 'imp', figsize=(10,6), legend=False);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b7f31c235703e8834453afe64bb05395f60766a3"},"cell_type":"code","source":"def plot_fi(fi): return fi.plot('cols', 'imp', 'barh', figsize=(12,7), legend=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c4e5d5665b306d7084ea2135ce8fdc210db7a0da"},"cell_type":"code","source":"plot_fi(fi[:30]);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3cdf6c28109bb72900ec56ac0a59b3b4ae646e82"},"cell_type":"code","source":"fi[fi.imp>0.0001]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"530cd5029437b96ba345fcb6894cb7ebf48212c5"},"cell_type":"code","source":"to_keep = fi[fi.imp>0.0001].cols; len(to_keep)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6ac07fb3b5fc9559be1a33188eb3ba3c382bd5c7"},"cell_type":"code","source":"df_keep = df_trn[to_keep].copy()\nX_train, X_valid = split_vals(df_keep, n_trn)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7513aacf2ff16584b8340f3f8a5ab3a33c623acd"},"cell_type":"code","source":"m = RandomForestRegressor(n_estimators=40, min_samples_leaf=3, max_features=0.5,\n                          n_jobs=-1, oob_score=True)\nm.fit(X_train, y_train)\nprint_score(m)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"10465a9560f7d72704ae066ea2cdf96ec6976ee9"},"cell_type":"code","source":"fi = rf_feat_importance(m, df_keep)\nplot_fi(fi);","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"69661a9aa81cac9bfb8eb2253bc523ab74cc150a"},"cell_type":"markdown","source":"## One-hot encoding"},{"metadata":{"trusted":true,"_uuid":"a857d3ec3863075613a96a5479c3813f0d715b1d"},"cell_type":"code","source":"df_trn2, y_trn, nas = proc_df(df_raw, 'totals.transactionRevenue', max_n_cat=7)\nX_train, X_valid = split_vals(df_trn2, n_trn)\n\nm = RandomForestRegressor(n_estimators=40, min_samples_leaf=3, max_features=0.5, n_jobs=-1, oob_score=True)\nm.fit(X_train, y_train)\nprint_score(m)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c1a1781cce91315d0fb5b0fd01851e3b372c6998"},"cell_type":"code","source":"fi = rf_feat_importance(m, df_trn2)\nplot_fi(fi[:25]);","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0bb3589ae34f22021a48bdb7b503f922fae19ea9"},"cell_type":"markdown","source":"Not very useful"},{"metadata":{"_uuid":"fed8b92e41cf43d71992d65e6e266197c6b6d4af"},"cell_type":"markdown","source":"# Removing redundant features"},{"metadata":{"trusted":true,"_uuid":"24dff62829f976fb4ff8fe7a6d4f3dc56ab84092"},"cell_type":"code","source":"from scipy.cluster import hierarchy as hc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fb7328c7f61910ff6475a0aa71864f96d0400b59"},"cell_type":"code","source":"df_keep = df_trn[to_keep].copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"daf0b97803a5a7244c2703d12c415db726bc2c7f"},"cell_type":"code","source":"corr = np.round(scipy.stats.spearmanr(df_keep).correlation, 4)\ncorr_condensed = hc.distance.squareform(1-corr)\nz = hc.linkage(corr_condensed, method='average')\nfig = plt.figure(figsize=(16,10))\ndendrogram = hc.dendrogram(z, labels=df_keep.columns, orientation='left', leaf_font_size=16)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d19f1247fd187f68607f7335f7d698d0bf21b764"},"cell_type":"code","source":"def get_oob(df):\n    m = RandomForestRegressor(n_estimators=30, min_samples_leaf=5, max_features=0.6, n_jobs=-1, oob_score=True)\n    x, _ = split_vals(df, n_trn)\n    m.fit(x, y_train)\n    return m.oob_score_","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b74aafd4fd90d713a2b828d8e9aceb5371c6c8e4"},"cell_type":"markdown","source":"Baseline"},{"metadata":{"trusted":true,"_uuid":"2341f13b73e1da35b6e37195fedf25653166a437"},"cell_type":"code","source":"get_oob(df_keep)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1fe573317d9878e44274f999f0c7f9bea9d0b6a4"},"cell_type":"code","source":"for c in ('totals.transactions', 'totals.totalTransactionRevenue'):\n    print(c, get_oob(df_keep.drop(c, axis=1)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0309971086b7f36d87d8a563cbf011528882ab8d"},"cell_type":"code","source":"to_drop = ['totals.transactions']\nget_oob(df_keep.drop(to_drop, axis=1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e4ce3f9697abf18710323e24fe71e9b706f2f920"},"cell_type":"code","source":"df_keep.drop(to_drop, axis=1, inplace=True)\nX_train, X_valid = split_vals(df_keep, n_trn)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3177ad80cf86eb2ff8697822a1ff3235a364bc80"},"cell_type":"code","source":"np.save('tmp/keep_cols.npy', np.array(df_keep.columns))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"541db255610d63b804f2d913b98a6d4bd077d5a9"},"cell_type":"code","source":"keep_cols = np.load('tmp/keep_cols.npy')\ndf_keep = df_trn[keep_cols]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c8eb389d21ee9efff0cf23cfe3a362b78cc463c1"},"cell_type":"code","source":"reset_rf_samples()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8cc564fec59e11060a77e087f3a9dcd7f3c72671"},"cell_type":"markdown","source":"And let's see how this model looks on the full dataset."},{"metadata":{"trusted":true,"_uuid":"c0929e89d8c4224e0cd2352e5a900453fae99dd2"},"cell_type":"code","source":"m = RandomForestRegressor(n_estimators=40, min_samples_leaf=3, max_features=0.5, n_jobs=-1, oob_score=True)\nm.fit(X_train, y_train)\nprint_score(m)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e199dd815daea2179c3431a47d7cbf5e05206f23"},"cell_type":"markdown","source":"A bit better"},{"metadata":{"_uuid":"3be24ce773a8a24a44a06742ba511c3ecd933011"},"cell_type":"markdown","source":"# Tree interpreter\n"},{"metadata":{"trusted":true,"_uuid":"31d8b408cf944cc7b0a40e76a76c7282ed22f7be"},"cell_type":"code","source":"!pip install treeinterpreter","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1417e22d7db7b4232019413e2a4e49fbaa3389ed"},"cell_type":"code","source":"from treeinterpreter import treeinterpreter as ti","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"23d37af98f0df3ba285745c3baa4e7b74c05705c"},"cell_type":"code","source":"df_train, df_valid = split_vals(df_raw[df_keep.columns], n_trn)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c308c5b07578dbe79ca0456422b9e956aa5d18d4"},"cell_type":"code","source":"row = X_valid.values[None,0]; row","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"328659c51fc6745e6c4343dc1121de05575a9ce0"},"cell_type":"code","source":"prediction, bias, contributions = ti.predict(m, row)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"68b9068f766db8c756c9b3bc0a96b7c8fc636d2e"},"cell_type":"code","source":"prediction[0], bias[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"147d85bfceda29204392f23a6fd66c7a591d00cd"},"cell_type":"code","source":"idxs = np.argsort(contributions[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dac42914069634f5402c838fa085dd2ccb49250e"},"cell_type":"code","source":"# [o for o in zip(df_keep.columns[idxs], df_valid.iloc[0][idxs], contributions[0][idxs])]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"77b95b32a52265b0af011b43a5e036d82c83ad2f"},"cell_type":"markdown","source":"# Extrapolation"},{"metadata":{"trusted":true,"_uuid":"e8428d6d12ce2825a341a42605522e20b5b97414"},"cell_type":"code","source":"df_ext = df_keep.copy()\ndf_ext['is_valid'] = 1\ndf_ext.is_valid[:n_trn] = 0\nx, y, nas = proc_df(df_ext, 'is_valid')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"89201eee31d9a70c733d725e3de94bcd748507fc"},"cell_type":"code","source":"m = RandomForestClassifier(n_estimators=40, min_samples_leaf=3, max_features=0.5, n_jobs=-1, oob_score=True)\nm.fit(x, y);\nm.oob_score_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2c3e32cc29f14bac8daaf9276692fc72b5e93a77"},"cell_type":"code","source":"fi = rf_feat_importance(m, x); fi[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5069e0e836e11576ae4a98372ca36a4f31b05b06"},"cell_type":"code","source":"feats=['totals.timeOnSite']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"86d4f67dddba441468c64a9ac16799a1a0b99dec"},"cell_type":"code","source":"(X_train[feats]/1000).describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"62adb1ad0059af77577ed6c8f3e0f76fb4de7fe0"},"cell_type":"code","source":"(X_valid[feats]/1000).describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"79158c73c49c5b4971780bcac72875f98bf7814b"},"cell_type":"code","source":"x.drop(feats, axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d09a02301ae08b12edfdd29fc8ddf750a9600d0d"},"cell_type":"code","source":"m = RandomForestClassifier(n_estimators=40, min_samples_leaf=3, max_features=0.5, n_jobs=-1, oob_score=True)\nm.fit(x, y);\nm.oob_score_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b2f64390e2df3549fc035c75d931aea7be51f14f"},"cell_type":"code","source":"fi = rf_feat_importance(m, x); fi[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"741a5ebe56d626258e460f6306544e48e8f00555"},"cell_type":"code","source":"set_rf_samples(50000)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"10f76f916debd802b80a791ebc978e12c3bb4769"},"cell_type":"code","source":"feats=['totals.timeOnSite', 'geoNetwork.country', 'totals.hits', 'totals.pageviews']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"14b3a90c4a0e0a6ed334cb736bd6c4160bf5c9f8"},"cell_type":"code","source":"X_train, X_valid = split_vals(df_keep, n_trn)\nm = RandomForestRegressor(n_estimators=40, min_samples_leaf=3, max_features=0.5, n_jobs=-1, oob_score=True)\nm.fit(X_train, y_train)\nprint_score(m)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1e330bbab248068f59b0b95554c3a576c12f46bf"},"cell_type":"code","source":"for f in feats:\n    df_subs = df_keep.drop(f, axis=1)\n    X_train, X_valid = split_vals(df_subs, n_trn)\n    m = RandomForestRegressor(n_estimators=40, min_samples_leaf=3, max_features=0.5, n_jobs=-1, oob_score=True)\n    m.fit(X_train, y_train)\n    print(f)\n    print_score(m)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f1a902e35e14c57e2f6d2e6e3cb705b73a5aee10"},"cell_type":"code","source":"reset_rf_samples()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9fa962ccd07bc9913eb4c178d5bf2b04d92b7434"},"cell_type":"code","source":"df_subs = df_keep.drop(['geoNetwork.country'], axis=1)\nX_train, X_valid = split_vals(df_subs, n_trn)\nm = RandomForestRegressor(n_estimators=40, min_samples_leaf=3, max_features=0.5, n_jobs=-1, oob_score=True)\nm.fit(X_train, y_train)\nprint_score(m)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"019bb40aaeaf17eea2c3bce41c86cb254bf121dc"},"cell_type":"markdown","source":"not much better"},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"c7726d1ab8b9502768b9f0adec1ab8742831b3c3"},"cell_type":"code","source":"plot_fi(rf_feat_importance(m, X_train));","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5febb946ea6f581310361481d902800acd362b84"},"cell_type":"code","source":"np.save('tmp/subs_cols.npy', np.array(df_subs.columns))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a444f0c295b9a41ea0bc08ac5ffb877383605a30"},"cell_type":"markdown","source":"# Our final model!"},{"metadata":{"trusted":true,"_uuid":"e2a9033691fcdb4bb4c6eb6ef6e81e3e0164f65e"},"cell_type":"code","source":"m = RandomForestRegressor(n_estimators=160, max_features=0.5, n_jobs=-1, oob_score=True)\n%time m.fit(X_train, y_train)\nprint_score(m)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0aa824be739f59f7144c145984e7b1ff6ca0df49"},"cell_type":"code","source":"X_test = X_test.drop(set(df_raw.columns).difference(set(df_subs.columns)) - {'totals.transactionRevenue'}, axis=1);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"19e6ee4d07cf109c5c597b2891a0f8fea61ab996"},"cell_type":"code","source":"predictions = m.predict(X_test)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4b768877cec697529371eb9b8ab85955022e1d0f"},"cell_type":"markdown","source":"# Prepare the submission"},{"metadata":{"trusted":true,"_uuid":"6e2d1886b913ba2dff04056fa80bb64bc1087acd"},"cell_type":"code","source":"df_actual_test = df_test.copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9b08ea17aaf7d8912f3de8b16aa56f7bcd7dd926"},"cell_type":"code","source":"actual_predicted_revenue = predictions\ndf_actual_test[\"predicted\"] = actual_predicted_revenue\n\ndf_actual_test = df_actual_test[[\"fullVisitorId\" , \"predicted\"]]\ndf_actual_test[\"fullVisitorId\"] = df_actual_test.fullVisitorId.astype('str')\ndf_actual_test[\"predicted\"] = df_actual_test.predicted.astype(np.float)\ndf_actual_test.index = df_actual_test.fullVisitorId\ndf_actual_test = df_actual_test.drop(\"fullVisitorId\",axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9616f03bee43f332c3dc7c8c8e393cc158742768"},"cell_type":"code","source":"df_actual_test[\"predicted\"].value_counts().iloc[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a264d2dc5c666e34661d1d75a2723228d051309c"},"cell_type":"code","source":"df_submission_test = pd.read_csv(filepath_or_buffer=\"../input/sample_submission_v2.csv\",index_col=\"fullVisitorId\")\ndf_submission_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cf5bc8d2c52f9673aa91455da92c6abc357b5765"},"cell_type":"code","source":"\"test shape is :{} and submission shape is : {}\".format(df_actual_test.shape , df_submission_test.shape)\nfinal_df = df_actual_test.loc[df_submission_test.index,:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"50fcfab48a18835da4f16d655eb454ef987ec0ba"},"cell_type":"code","source":"final_df = final_df[~final_df.index.duplicated(keep='first')]\nfinal_df = final_df.rename(index=str, columns={\"predicted\": \"PredictedLogRevenue\"})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"90bc62ce967c30f5e5132a0b91181f0b01a4ea54"},"cell_type":"code","source":"final_df.PredictedLogRevenue.value_counts(bins=3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f2551d1f8423588440145a25747ade9a50675cec"},"cell_type":"code","source":"final_df = final_df.fillna(0); final_df.iloc[262153]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"311766cdcaef81e88a497c15d8c9e7ed998bf133"},"cell_type":"code","source":"final_df.to_csv(\"sub.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5e218af4b80b76e353cb2b3efb9fcc88a7867dfa"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}