{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport os\nimport gc\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport datetime\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\", parse_dates=[\"first_active_month\"])\ntest_df = pd.read_csv(\"../input/test.csv\", parse_dates=[\"first_active_month\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6a4a8d37c32e31eda24c8c13356916e098fae391"},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c1aea45fb58dd7c60fc507e597195a7c01532f8a"},"cell_type":"code","source":"train_df[\"feature_1\"].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"62d3129a4e86689bf50458a36ccf287f01cdde8d"},"cell_type":"code","source":"train_df[\"feature_2\"].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ccd5dfd5148aad7844f5758ea5a7047d16c558d8"},"cell_type":"code","source":"train_df[\"feature_3\"].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"53e2d8e3ed26a4628ba9867d6808428e603bbb3d"},"cell_type":"markdown","source":"So seems like feature 1,2 and 3 are cateogrical."},{"metadata":{"trusted":true,"_uuid":"92c3d9f06d6714a3b1f3d9aaa681b7241a4d74d9"},"cell_type":"code","source":"sns.set_style(\"ticks\")\nplt.figure(figsize=[6,4])\nsns.distplot(train_df[\"target\"])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5656e02436c405eea98302cd265cd218ed370e4b"},"cell_type":"code","source":"train_df.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7f77a971029b2b24a19c4c0fb7b98bf924b2794b"},"cell_type":"code","source":"test_df.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"000a4f95d9fb71ccb28700a8a65056164ffa7f51"},"cell_type":"code","source":"train_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2119259ffb57017f9a86565ba8ad763a3c6c92e0"},"cell_type":"code","source":"train_df.dtypes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c4adbc2735e119823e34bf426d8556e82afb8ced"},"cell_type":"code","source":"# from old fastai old\n# https://github.com/fastai/fastai/blob/master/old/fastai/structured.py#L76\nimport re\ndef add_datepart(df, fldname, drop=False, time=False):\n    fld = df[fldname]\n    fld_dtype = fld.dtype\n    if isinstance(fld_dtype, pd.core.dtypes.dtypes.DatetimeTZDtype):\n        fld_dtype = np.datetime64\n\n    if not np.issubdtype(fld_dtype, np.datetime64):\n        df[fldname] = fld = pd.to_datetime(fld, infer_datetime_format=True)\n    targ_pre = re.sub('[Dd]ate$', '', fldname)\n    attr = ['Year', 'Month', 'Week', 'Day', 'Dayofweek', 'Dayofyear',\n            'Is_month_end', 'Is_month_start', 'Is_quarter_end', 'Is_quarter_start', 'Is_year_end', 'Is_year_start']\n    if time: attr = attr + ['Hour', 'Minute', 'Second']\n    for n in attr: df[targ_pre + n] = getattr(fld.dt, n.lower())\n    df[targ_pre + 'Elapsed'] = fld.astype(np.int64) // 10 ** 9\n    if drop: df.drop(fldname, axis=1, inplace=True)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7dba37e08b4207be97c0a1680d333e90dbdd4953"},"cell_type":"code","source":"add_datepart(train_df, \"first_active_month\")\nadd_datepart(test_df, \"first_active_month\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d3eca1da8ad917814d8af01434477d359fd7c9e4"},"cell_type":"code","source":"train_df['elapsed_time'] = (datetime.date(2018, 2, 1) - train_df['first_active_month'].dt.date).dt.days\ntest_df['elapsed_time'] = (datetime.date(2018, 2, 1) - test_df['first_active_month'].dt.date).dt.days","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3084144624215f4f00950224af3a03cb5aba3bfa"},"cell_type":"code","source":"print(\"Train Data Time Range:\",train_df[\"first_active_month\"].min(), \"-\", train_df[\"first_active_month\"].max())\nprint(\"Test Data Time Range:\",test_df[\"first_active_month\"].min(), \"-\",  test_df[\"first_active_month\"].max())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0d905c16e089a2926ec3372e0f6ca746d877e69a"},"cell_type":"code","source":"train_df.groupby(\"first_active_month\")[\"target\"].mean().plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"04f2a118340615ffd9bbb3842839f8c9098a14d7"},"cell_type":"code","source":"set((train_df[\"first_active_month\"])).intersection(set(test_df[\"first_active_month\"]))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b049b2f9a81afd23d0317eaae53e987bdd9062af"},"cell_type":"markdown","source":"So there are some intersection between train and test data."},{"metadata":{"trusted":true,"_uuid":"6742ca563eaf06e5c3eb173b8f79e99552dfc7be"},"cell_type":"code","source":"train_card_id = train_df[\"card_id\"]\ntest_card_id = test_df[\"card_id\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"60788125612eea8b79efb7bce73aa61b982e3de0"},"cell_type":"markdown","source":"Sort data by date and get last 10% for validation split"},{"metadata":{"trusted":true,"_uuid":"3b743448d21e8d44c14f8d54609016894a8b74ae"},"cell_type":"code","source":"train_df.sort_values(\"first_active_month\", inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5a8f08edcbf82d112f5989d0a6b40051b81c98d7"},"cell_type":"code","source":"train_df.reset_index(drop=True, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9742e8a707d7b02c5a8f428522e9ff278f201a8f"},"cell_type":"code","source":"train_columns = [c for c in train_df.columns if c not in [\"first_active_month\", \"card_id\", \"target\"]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aa2901458ddf4a4825579401316e034b9edc9200"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"905285b7bd7d834d47f7b1c18306a66ce2339091"},"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(train_df[train_columns], train_df[\"target\"], test_size = 0.2, random_state=1001)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ddf7becd5827eee95fc39418d7a463879bfb1a26"},"cell_type":"code","source":"(X_train.shape, y_train.shape), (X_val.shape, y_val.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"afadbdd1b15ac282a945cc9bcf4610c13a043b0c"},"cell_type":"code","source":"from sklearn.metrics import mean_squared_error\nimport lightgbm as lgb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b57a8397683f18e02e4f11844e759704bad0c0cc"},"cell_type":"code","source":"params = {\n\"objective\" : \"regression\",\n\"metric\" : \"rmse\",\n\"num_leaves\" : 12,\n\"learning_rate\" : 0.01,\n\"bagging_fraction\" : 0.7,\n\"feature_fraction\" : 0.9,\n\"bagging_frequency\" : 4,\n\"bagging_seed\" : 1001,\n\"verbosity\" : -1\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"12eaee27cc3a522af8d1dd17051305ce07ff820b"},"cell_type":"code","source":"cat_cols = [\"feature_1\", \"feature_2\", \"feature_3\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a640ca48dee4bed5ed4a5a47e5129fc664f1f2dd"},"cell_type":"code","source":"dtrain = lgb.Dataset(X_train, label=y_train, categorical_feature=cat_cols, free_raw_data=False)\ndval = lgb.Dataset(X_val, label=y_val, categorical_feature=cat_cols, free_raw_data=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"905f85c75314ebc531d13674b5132aa16ad185b9"},"cell_type":"code","source":"evals_result = {}\nmodel_lgb = lgb.train(params, dtrain, 500, valid_sets=[dtrain,dval], valid_names=[\"train\", \"val\"], early_stopping_rounds=30, verbose_eval=100, evals_result=evals_result)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c5ee4a181850c89d8efc71a4d182763901c98c8d"},"cell_type":"code","source":"lgb.plot_importance(model_lgb, figsize=(12,10))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1c86ac74be879fc3f69cf82e6d7af457a4bd7474"},"cell_type":"code","source":"lgb.plot_importance(model_lgb, figsize=(12,10), importance_type=\"gain\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7cd2f269d23680e1a148d98286cb509edfbbbf47"},"cell_type":"code","source":"lgb.plot_metric(evals_result, \"rmse\", figsize=(12,10))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b91718202c439b45a6dc9bb31c3e1f6027f4c24f"},"cell_type":"code","source":"pred_lgb = model_lgb.predict(test_df[train_columns])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e1b522d09142f3912b609bacb24daa3ffdd13d66"},"cell_type":"code","source":"ss = pd.DataFrame({\"card_id\":test_card_id, \"target\":pred_lgb})\nss.to_csv(\"preds_starter_lgb.csv\", index=None)\nss.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"18ae1bbc84935dc56a7e4f5071cee4a81da3aafb"},"cell_type":"markdown","source":"#### xgboost"},{"metadata":{"trusted":true,"_uuid":"2c1c2f6f8676dd63a2333a7a609eff1e6244cd58"},"cell_type":"code","source":"import xgboost as xgb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aa461e308580a04d27cfcfdfe889573eca098777"},"cell_type":"code","source":"dtrain_xgb = xgb.DMatrix(X_train, label=y_train)\ndval_xgb = xgb.DMatrix(X_val, y_val)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"69bcb74624930008eceeb9c57030f4fc8f6bead0"},"cell_type":"code","source":"xgb_params = {\n        'objective': 'reg:linear',\n        'learning_rate': 0.02,\n        'max_depth': 5,\n        'subsample': 0.7,\n        'colsample_bytree': 0.9,\n        'colsample_bylevel': 0.9,\n        'n_jobs': -1,\n        \"silent\": 1\n    }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"82aa152cfdf2b056bf3160b6642a9d21cc463dcb"},"cell_type":"code","source":"evallist  = [(dtrain_xgb, 'train'), (dval_xgb, \"val\")]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5f131cb3bc61cb3f82ac1920fe1e036a4f8ec530"},"cell_type":"code","source":"model_xgb = xgb.train(xgb_params, dtrain_xgb, num_boost_round=500, evals=evallist, early_stopping_rounds=50, verbose_eval=100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c26a1a940e4a7cdc65b65587a4b35bcd94fa1599"},"cell_type":"code","source":"xgb.plot_importance(model_xgb)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"621230305b12810b25c7c5b9a38f669ef39e6fba"},"cell_type":"code","source":"preds_xgb = model_xgb.predict(xgb.DMatrix(test_df[train_columns]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"589995f668d0bba325c559a68cb6fdfb16c14554"},"cell_type":"code","source":"ss = pd.DataFrame({\"card_id\":test_card_id, \"target\":preds_xgb})\nss.to_csv(\"preds_starter_xgb.csv\", index=None)\nss.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5723e903ee45b8e4bb65dfa8b5406440423a58db"},"cell_type":"markdown","source":"#### Catboost"},{"metadata":{"trusted":true,"_uuid":"8eab4865425d4e001a447e0088aaf7126684a4f9"},"cell_type":"code","source":"from catboost import CatBoostRegressor","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"042bfeea68dfbee2464f98e05788a95d6dfb4b90"},"cell_type":"code","source":"model_cat = CatBoostRegressor(iterations=500,\n                             learning_rate=0.02,\n                             depth=6,\n                             eval_metric='RMSE',\n                             bagging_temperature = 0.9,\n                             od_type='Iter',\n                             metric_period = 100,\n                             od_wait=50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ac2b91ca75d0c0255f033c8c65748910948ac4b7"},"cell_type":"code","source":"model_cat.fit(X_train, y_train,\n             eval_set=(X_val,y_val),\n             cat_features=np.array([0,1,2]),\n             use_best_model=True,\n             verbose=100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"52c13ed083ee1b1f20736ee4d879af32b15cc8f0"},"cell_type":"code","source":"preds_cat = model_cat.predict(test_df[train_columns])  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"212b362747b8b70d230d8c78bbdf0d91dc13aedd"},"cell_type":"code","source":"pd.DataFrame(model_cat.get_feature_importance(), index=X_train[train_columns].columns, columns=[\"FeatureImportance\"]).sort_values(\"FeatureImportance\", ascending=False).plot(kind=\"barh\", legend=False, figsize=(12,10))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1173a1065453b2e7b66712db17cd58c78b45b79c"},"cell_type":"code","source":"ss = pd.DataFrame({\"card_id\":test_card_id, \"target\":preds_cat})\nss.to_csv(\"preds_starter_cat.csv\", index=None)\nss.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"aa032c2329e77d1380207553a25e716009eb96fc"},"cell_type":"markdown","source":"### Blending"},{"metadata":{"trusted":true,"_uuid":"d83e1c0ed904a901cbb1f30f9af36e34cb92266e"},"cell_type":"code","source":"ss_blend = pd.DataFrame({\"card_id\":test_card_id, \"target\":((0.33 * preds_xgb) + (0.34 * pred_lgb)+ (0.33 * preds_cat))})\nss_blend.to_csv(\"preds_starter_blend.csv\", index=None)\nss_blend.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7777576f84f6719096747423a284cd19e3716e10"},"cell_type":"markdown","source":"### To be continued..."}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}