{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport json\nfrom pandas.io.json import json_normalize\nimport seaborn as sns\nfrom matplotlib import pyplot as plt\n\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# shared from this kernel\n# https://www.kaggle.com/julian3833/1-quick-start-read-csv-and-flatten-json-fields\n# def load_df(csv_path='../input/train.csv', nrows=None):\n#     JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']\n#     df = pd.read_csv(csv_path, dtype={'fullVisitorId': 'str'}, nrows=nrows)\n#     for column in JSON_COLUMNS:\n#         df = df.join(pd.DataFrame(df.pop(column).apply(pd.io.json.loads).values.tolist(), index=df.index))\n\n#     return df\n\ndef load_df(csv_path='../input/train.csv', nrows=None):\n    JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']\n    \n    df = pd.read_csv(csv_path, \n                     converters={column: json.loads for column in JSON_COLUMNS}, \n                     dtype={'fullVisitorId': 'str'}, # Important!!\n                     nrows=nrows)\n    \n    for column in JSON_COLUMNS:\n        column_as_df = json_normalize(df[column])\n        column_as_df.columns = [f\"{column}.{subcolumn}\" for subcolumn in column_as_df.columns]\n        df = df.drop(column, axis=1).merge(column_as_df, right_index=True, left_index=True)\n    print(f\"Loaded {os.path.basename(csv_path)}. Shape: {df.shape}\")\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"913a3edecfa39b9797095acdc180ffda070f38ba"},"cell_type":"code","source":"%%time\ndf_train = load_df(nrows=50000)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"12d3bf1768f1bf63cb0060073def947737a0212a"},"cell_type":"code","source":"df_test = load_df(nrows=50000)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4a0155a0524cf69d5c72983fbe2c56c568ddd890"},"cell_type":"markdown","source":"# Missing Plots\nA dendrogram plot from missingno clusters features based on their \"null correlation\". Transaction revenue seems to be related to the traffic sources -- if they are not null."},{"metadata":{"trusted":true,"_uuid":"c16fa520337d95fc4574ca58984bb0553a1a1961"},"cell_type":"code","source":"import missingno as msno\nmsno.matrix(df_train);\nmsno.dendrogram(df_train);","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5ac15293fdd371f0f8f069010c6e3d59ac3f4e86"},"cell_type":"markdown","source":"# Preprocessing\n1. Remove columns with only one value\n2. Convert numericals\n3. Convert dates\n4. Convert \"not available in demo dataset\" to null"},{"metadata":{"trusted":true,"_uuid":"d5c800a5ac155b6bad6ce13152a5ff894604603b"},"cell_type":"code","source":"# remove constant cols\nconstant_cols = []\nall_cols = df_train.columns\nfor col in all_cols:\n    if df_train[col].nunique() == 1:\n        constant_cols.append(col)\n        \ndf_train.drop(constant_cols, axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"41a83c914730923aa2681f557b1117420b878eb1"},"cell_type":"code","source":"# convert numericals\ndef convert_to_numerical(df, cols):\n    for col in cols:\n        df[col] = pd.to_numeric(df[col], errors='coerce')\n    return df\n\ntotal_cols = df_train.filter(like=\"totals\")\ndf_train = convert_to_numerical(df_train, total_cols)\ndf_train = convert_to_numerical(df_train, total_cols)\n\nother_numerical_cols = [\"trafficSource.adwordsClickInfo.page\", ]\ndf_train = convert_to_numerical(df_train, other_numerical_cols)\ndf_test = convert_to_numerical(df_test, other_numerical_cols)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7698960302f9b807081886d33934bcf09fd4ccb9"},"cell_type":"code","source":"# format dates\ndf_train[\"date\"] = pd.to_datetime(df_train[\"date\"], errors='coerce', format=\"%Y%m%d\")\ndf_train['visitStartTime'] = pd.to_datetime(df_train['visitStartTime'],unit='s')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e870450f2d92b3f496a43d27fa15ff707f6f7866"},"cell_type":"code","source":"# These values are considered NONE\nDEFAULT_VALUE = [\"not available in demo dataset\", \"(none)\"]\n\n# then all categoricals default values will be \"NA\"\nFILLNA_VALUE_CAT = \"NA\"\nfor col in df_train.select_dtypes(np.object).columns:\n    df_train[col].replace(DEFAULT_VALUE, value=None, inplace=True)\n    df_train[col].fillna(FILLNA_VALUE_CAT, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f2004ff467e5731ddbfea532933aa09e3e839cec"},"cell_type":"markdown","source":"# Feature Engineering\n1. Compile numerical variables\n2. Categoricals (top 100 categoricals count per person)\n3. Text features\n4. Date variables\n5. Lag variables - TODO"},{"metadata":{"trusted":true,"_uuid":"af3a967cd45eb72ea6c1770980a004fa992995b3"},"cell_type":"code","source":"FULL_ID = \"fullVisitorId\"\nlist_feature_dfs = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a8882090f35d68fa0ed8c0585833530dd06fbcc5"},"cell_type":"code","source":"# basic stats\nnumerical_cols = [\"totals.hits\", \"totals.pageviews\"]\ndf_feat_num = df_train.groupby(FULL_ID)[numerical_cols].agg([\"mean\", \"std\", \"sem\"])\ndf_feat_num.columns = [' '.join(col).strip() for col in df_feat_num.columns.values]\n\n# number of visits\ndf_num_visits = df_train.groupby(FULL_ID).size().to_frame('num_visits')\n\n# number of sessions\ndf_num_sessions = df_train.groupby(FULL_ID)[\"sessionId\"].nunique().to_frame(\"num_sessions\")\n\nlist_feature_dfs.append(df_feat_num)\nlist_feature_dfs.append(df_num_visits)\nlist_feature_dfs.append(df_num_sessions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"994ffbc083b09a5d7a4af96411ece078fd37c843"},"cell_type":"code","source":"# consider as categorical all columns that have unique values <= 100\n# this is a feature engineering parameter\nNUM_UNIQUE_VALS_TO_BE_CAT = 100\nEXEMPT_COLS = [FULL_ID, \"sessionId\", \"trafficSource.adwordsClickInfo.gclId\"]\ncategorical_cols = []\ntext_cols = []\n# all categorical-like columns\nfor col in df_train.select_dtypes(np.object).drop(EXEMPT_COLS, axis=1).columns:\n    if len(df_train[col].value_counts()) <= NUM_UNIQUE_VALS_TO_BE_CAT:\n        categorical_cols.append(col)\n    else:\n        text_cols.append(col)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"56f5574d45097c2e76c91386233f9cdb06f5fdd7"},"cell_type":"code","source":"categorical_cols","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"20c893c67365e8f4d0d72adbc55fc31477aed17d"},"cell_type":"code","source":"text_cols","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"71894e86aae02980801839bef2ea17f076fb011a"},"cell_type":"markdown","source":"## Categorical group"},{"metadata":{"trusted":true,"_uuid":"6110afef9a051156130dfc8d1c4537d27aee4f2a"},"cell_type":"code","source":"# this can be considered as a regularization parameter. \n# All values outside of the top THRESH_CAT are considered \"others\"\nTHRESH_CAT = 20\nOTHERS = \"OTHERS\"\n\ndef group_id_categorical(df, col, threshold_categorical = THRESH_CAT, others_value = OTHERS):    \n    vc = df[col].value_counts()\n\n    list_top = vc[:threshold_categorical].index\n    \n    col_series = np.where(df[col].isin(list_top), df[col], others_value)\n    id_series = df[FULL_ID]\n\n    df_group_col = pd.DataFrame({FULL_ID : id_series, col : col_series})\n    df_group_col = df_group_col.groupby([FULL_ID, col]).size().unstack()\n    \n    # naming: origin_variable.value\n    df_group_col.columns = col + \"=\" + df_group_col.columns\n    \n    return df_group_col.fillna(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d680b9af8cc3133e5011d75a9d6d3fd84b5fdf30"},"cell_type":"code","source":"for col in categorical_cols:\n    df_group_col = group_id_categorical(df_train, col)\n    list_feature_dfs.append(df_group_col)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a7786d6261e66349680d39f3edcf128513b3021b"},"cell_type":"markdown","source":"# Date Variables"},{"metadata":{"trusted":true,"_uuid":"10a4b358c882df36fc26b803b86859bcb003c27d"},"cell_type":"code","source":"df_train[\"date-month\"] = \"month-\" + df_train[\"date\"].dt.month.apply(str)\ndf_train[\"date-dayofweek\"] = \"dayofweek-\" + df_train[\"date\"].dt.day_name()\ndf_train['date-weekofmonth'] = \"weekofmonth-\" + df_train[\"date\"].apply(lambda d: str((d.day-1) // 7 + 1))\n\ndf_date_month = df_train.groupby([FULL_ID, \"date-month\"]).size().unstack().fillna(0)\ndf_date_dayofweek = df_train.groupby([FULL_ID, \"date-dayofweek\"]).size().unstack().fillna(0)\ndf_date_weekofmonth = df_train.groupby([FULL_ID, \"date-weekofmonth\"]).size().unstack().fillna(0)\n\nlist_feature_dfs.append(df_date_month)\nlist_feature_dfs.append(df_date_dayofweek)\nlist_feature_dfs.append(df_date_weekofmonth)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"08eaad83ee84466fae25a7a402248e7010f0bd2b"},"cell_type":"code","source":"df_train[\"visit-month\"] = \"visit-month-\" + df_train[\"visitStartTime\"].dt.month.apply(str)\ndf_train[\"visit-dayofweek\"] = \"visit-dayofweek-\" + df_train[\"visitStartTime\"].dt.day_name()\ndf_train['visit-weekofmonth'] = \"visit- weekofmonth-\" + df_train[\"visitStartTime\"].apply(\n    lambda d: str((d.day-1) // 7 + 1))\ndf_train[\"time-hour\"] = \"hour-\" + df_train[\"visitStartTime\"].dt.hour.apply(str)\n\ndf_visit_month = df_train.groupby([FULL_ID, \"visit-month\"]).size().unstack().fillna(0)\ndf_visit_dayofweek = df_train.groupby([FULL_ID, \"visit-dayofweek\"]).size().unstack().fillna(0)\ndf_visit_weekofmonth = df_train.groupby([FULL_ID, \"visit-weekofmonth\"]).size().unstack().fillna(0)\ndf_time_hour = df_train.groupby([FULL_ID, \"time-hour\"]).size().unstack().fillna(0)\n\nlist_feature_dfs.append(df_visit_month)\nlist_feature_dfs.append(df_visit_dayofweek)\nlist_feature_dfs.append(df_visit_weekofmonth)\nlist_feature_dfs.append(df_time_hour)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"93231e5b27db17ef6ce0a0e3c7404065a3228e4f"},"cell_type":"markdown","source":"# Lag Variables - TODO"},{"metadata":{"trusted":true,"_uuid":"47a1c6a17133f453e57203212dc3173a7d696887"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"992547d1941db851c74647f4a3c9e1fb58915fe0"},"cell_type":"markdown","source":"## Text Column"},{"metadata":{"trusted":true,"_uuid":"e5f56709caa399a046bb3ee0f8bb4b4d7ef6b972"},"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer\ncol = \"trafficSource.source\"\ndf = df_train\n\ndef group_id_text(df, col, min_df=3):\n    df_text = df.groupby(FULL_ID)[col].apply(lambda x: \"%s\" % ' '.join(x))\n    df_text_index = df_text.index\n\n    count_vec = CountVectorizer(binary=True, min_df=min_df)\n    df_text = pd.DataFrame(count_vec.fit_transform(df_text).toarray(), index=df_text_index, \n                 columns=count_vec.get_feature_names())\n    df_text.columns = col + \"=\" + df_text.columns\n    return df_text","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"01de243f14e8ceea55738a6b190dc3c1338baed9"},"cell_type":"code","source":"for col in text_cols:\n    list_feature_dfs.append(group_id_text(df_train, col))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f499c271f0d190e0d87e2b49a180b08816122833"},"cell_type":"markdown","source":"# Join them all"},{"metadata":{"trusted":true,"_uuid":"f74915dd6d314e9cb9e778d02f850ba9c3eaf22a"},"cell_type":"code","source":"from functools import reduce\ndef join_dfs(ldf, rdf):\n    return ldf.join(rdf, how='left')\n\ndf_features = reduce(join_dfs, list_feature_dfs)\n\ndf_features = df_features.fillna(0)\ndf_features.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1b0a6d833f588c78b86ef71eee770458d328aac8"},"cell_type":"code","source":"target_data = df_train.groupby(FULL_ID)[\"totals.transactionRevenue\"].sum()\n\n# log my target data\ntarget_data = np.log(1+ target_data)\n\n# reindex target data so features are aligned\ntarget_data = target_data.reindex(df_features.index)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"614cec11481942f78516f24871ec5781be4562f3"},"cell_type":"markdown","source":"# RandomForest Regressor\n1. Tuning = TODO"},{"metadata":{"trusted":true,"_uuid":"155f924c555d9569bd10fd8cb5fe0d00c4be4f28"},"cell_type":"code","source":"df_features.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cb1db2b5b58c2a01be011424da5cd3334a907c00"},"cell_type":"code","source":"from lightgbm.sklearn import LGBMRegressor\n\nmodel=LGBMRegressor(boosting_type='gbdt', max_depth=5, n_estimators=1000, \n                    max_bin=255,subsample_for_bin=50000,\n                    min_child_weight=3,min_child_samples=10,subsample=0.6,subsample_freq=1,colsample_bytree=0.6,\n                    seed=23,silent=False,nthread=-1,n_jobs=-1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3db23e4c5e227b008580d91d77208a60b126a6f3","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"# from sklearn.ensemble import RandomForestRegressor\n# params = {\"n_estimators\" : 100, \"max_depth\" : 7, \"min_samples_split\" : 0.001, \"max_features\" : \"sqrt\"}\n# rfr = RandomForestRegressor(verbose=1, n_jobs=4)\n# rfr.set_params(**params)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"68d9af3ffc54d4d5335c33e572e8f75f9bcb5ffe"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_log_error, mean_squared_error\n\nrandom_seed = 1234\nX_train, X_test, y_train, y_test = train_test_split(df_features, target_data, test_size=0.2, \n                                                    random_state = random_seed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"835efbfa42ccbe80d53f56f715e8ca0d21673376"},"cell_type":"code","source":"PATIENCE = 3\n# evaluation set\nX_train1, X_val, y_train1, y_val = train_test_split(X_train, y_train, test_size=0.2, random_state = random_seed)\n\neval_params = {\"eval_set\": (X_val, y_val), \"eval_metric\":'l2_loss', \"early_stopping_rounds\":PATIENCE}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"639e7041e8ac3acce5cf84eaa04d9fb4108f6774","scrolled":true},"cell_type":"code","source":"model.fit(X_train, y_train, **eval_params)\n\ny_preds = model.predict(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e33cf1980ae28d5710962702a207488e964207e9"},"cell_type":"code","source":"print(\"MSE:\", mean_squared_error(y_test, y_preds))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dfb503ce0b98ebb3935aa9a7c1c1286fe1415bd3"},"cell_type":"markdown","source":"- Very manual tuning. <b> You should do random search!</b>\n    - 3.1501, max_depth=7, subsample=1, colsample_bytree=1\n    - 3.3772, max_depth=7, subsample=0.8, colsample_bytree=0.8\n    - 3.2647, max_depth=7, subsample=0.6, colsample_bytree=0.6\n    \n    - 3.2742, max_depth=5, subsample=1, colsample_bytree=1\n    - 3.1380, max_depth=5, subsample=0.8, colsample_bytree=0.8\n    - **2.9920, max_depth=5, subsample=0.6, colsample_bytree=0.6**\n    \n    - 3.2010, max_depth=5, subsample=1, colsample_bytree=1\n    - 3.1321, max_depth=5, subsample=0.8, colsample_bytree=0.8\n    - 3.1340 max_depth=5, subsample=0.6, colsample_bytree=0.6"},{"metadata":{"_uuid":"7b836f8de35940c30e2255d5cd362ba98128ff58"},"cell_type":"markdown","source":" # Submission\n TODO: create functions from the above\n \n TODO: full dataset prediction and submission"},{"metadata":{"trusted":true,"_uuid":"6465175ad9958ded6011d79f2a9f901e4dc21af7"},"cell_type":"code","source":"list_test_feature_dfs = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0db0144bfad25e7cf19910346c8660ed1289ad96"},"cell_type":"code","source":"df_test = convert_to_numerical(df_test, total_cols)\ndf_test = convert_to_numerical(df_test, other_numerical_cols)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e0eeeebe1be2b968b11bb000292b6d8da5318e44"},"cell_type":"code","source":"for col in df_test.select_dtypes(np.object).columns:\n    df_test[col].replace(DEFAULT_VALUE, value=None, inplace=True)\n    df_test[col].fillna(FILLNA_VALUE_CAT, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1c50274be3bce1af89dafb537cbbc8c928b8ecb9"},"cell_type":"code","source":"# format dates\ndf_test[\"date\"] = pd.to_datetime(df_test[\"date\"], errors='coerce', format=\"%Y%m%d\")\ndf_test['visitStartTime'] = pd.to_datetime(df_test['visitStartTime'],unit='s')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a178d541420b8031f1c2c0197d2f259d1a8cde13"},"cell_type":"code","source":"# basic stats\nnumerical_cols = [\"totals.hits\", \"totals.pageviews\"]\ndf_feat_num = df_test.groupby(FULL_ID)[numerical_cols].agg([\"mean\", \"std\", \"sem\"])\ndf_feat_num.columns = [' '.join(col).strip() for col in df_feat_num.columns.values]\n\n# number of visits\ndf_num_visits = df_test.groupby(FULL_ID).size().to_frame('num_visits')\n\n# number of sessions\ndf_num_sessions = df_train.groupby(FULL_ID)[\"sessionId\"].nunique().to_frame(\"num_sessions\")\n\nlist_test_feature_dfs.append(df_feat_num)\nlist_test_feature_dfs.append(df_num_visits)\nlist_test_feature_dfs.append(df_num_sessions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b16cb48c8216a2b3827704799ffaff918b1babea"},"cell_type":"code","source":"for col in categorical_cols:\n    df_group_col = group_id_categorical(df_test, col)\n    list_test_feature_dfs.append(df_group_col)\n    \nfor col in text_cols:\n    list_test_feature_dfs.append(group_id_text(df_test, col))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fe713666400dd354f35dafddfb07920970df3161"},"cell_type":"code","source":"df_test[\"date-month\"] = \"month-\" + df_test[\"date\"].dt.month.apply(str)\ndf_test[\"date-dayofweek\"] = \"dayofweek-\" + df_test[\"date\"].dt.day_name()\ndf_test['date-weekofmonth'] = \"weekofmonth-\" + df_test[\"date\"].apply(lambda d: str((d.day-1) // 7 + 1))\n\ndf_date_month = df_test.groupby([FULL_ID, \"date-month\"]).size().unstack().fillna(0)\ndf_date_dayofweek = df_test.groupby([FULL_ID, \"date-dayofweek\"]).size().unstack().fillna(0)\ndf_date_weekofmonth = df_test.groupby([FULL_ID, \"date-weekofmonth\"]).size().unstack().fillna(0)\n\nlist_test_feature_dfs.append(df_date_month)\nlist_test_feature_dfs.append(df_date_dayofweek)\nlist_test_feature_dfs.append(df_date_weekofmonth)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"925cf0a6f0b68383a464175f5b4525157d5da254"},"cell_type":"code","source":"df_test[\"visit-month\"] = \"visit-month-\" + df_test[\"visitStartTime\"].dt.month.apply(str)\ndf_test[\"visit-dayofweek\"] = \"visit-dayofweek-\" + df_test[\"visitStartTime\"].dt.day_name()\ndf_test['visit-weekofmonth'] = \"visit- weekofmonth-\" + df_test[\"visitStartTime\"].apply(\n    lambda d: str((d.day-1) // 7 + 1))\ndf_test[\"time-hour\"] = \"hour-\" + df_test[\"visitStartTime\"].dt.hour.apply(str)\n\ndf_visit_month = df_test.groupby([FULL_ID, \"visit-month\"]).size().unstack().fillna(0)\ndf_visit_dayofweek = df_test.groupby([FULL_ID, \"visit-dayofweek\"]).size().unstack().fillna(0)\ndf_visit_weekofmonth = df_test.groupby([FULL_ID, \"visit-weekofmonth\"]).size().unstack().fillna(0)\ndf_time_hour = df_test.groupby([FULL_ID, \"time-hour\"]).size().unstack().fillna(0)\n\nlist_test_feature_dfs.append(df_visit_month)\nlist_test_feature_dfs.append(df_visit_dayofweek)\nlist_test_feature_dfs.append(df_visit_weekofmonth)\nlist_test_feature_dfs.append(df_time_hour)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4c35c3f43322d0a779b96682958a5bed67e526e9"},"cell_type":"code","source":"df_test_features = reduce(join_dfs, list_test_feature_dfs)\n\n# add columns that are not in the test set\nmissing_columns = df_features.columns.difference(df_test_features.columns)\ndf_test_features = df_test_features.reindex(columns = df_test_features.columns.tolist() + missing_columns.tolist())\n\n# fill with 0\ndf_test_features = df_test_features.fillna(0)\n\n# reorder columns\ndf_test_features = df_test_features[df_features.columns.tolist()]\n\ndf_test_features.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c539ae718b076624757243ef7891db5454307395"},"cell_type":"code","source":"y_test_preds = model.predict(df_test_features)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ba3fde1b521dce5ec3fecf264aa6611120919e9a"},"cell_type":"code","source":"df_submission = pd.DataFrame({\"PredictedLogRevenue\" : \n                             y_test_preds}, index=df_test_features.index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c9694dbd4b971e88955bd904fbde52b2434368fe"},"cell_type":"code","source":"df_sample_submission = pd.read_csv(\"../input/sample_submission.csv\", index_col=0)\ndf_sample_submission.update(df_submission)\ndf_sample_submission.reset_index(inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b33ad6aac940ff8c53c4ad9c0b37d7735cc8f8b3"},"cell_type":"code","source":"df_sample_submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fd1b25b04322d2f6f5a33d0b502f4c1592555d96"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"169c46ce306f237f8b392e0e9d94f34b1fbd154a"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}