{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nfrom sklearn.model_selection import train_test_split\nimport lightgbm as lgb\nimport xgboost as xgb\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import KFold,StratifiedKFold\nfrom sklearn.preprocessing import LabelEncoder\nimport seaborn as sns\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d9f7ce14f675a7fb562e70187ca409b14db27d3e"},"cell_type":"markdown","source":"# Read data"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"scrolled":true},"cell_type":"code","source":"train_set = pd.read_csv(\"../input/bgu-dl-assignmnt2-features-extraction/train_set.csv\")\ntest_set = pd.read_csv(\"../input/bgu-dl-assignmnt2-features-extraction/test_set.csv\")\ntarget = pd.read_csv(\"../input/bgu-dl-assignmnt2-features-extraction/target.csv\", header=None)\n\nprint(\"shape of train : \",train_set.shape)\nprint(\"shape of test : \",test_set.shape)\nprint(\"shape of target : \",target.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4f5b10436a7cc8e390e790ba7a3892ad150653bc"},"cell_type":"code","source":"train_set.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c6ac1129846997c2c2d87b32ca0d7100935a4ace"},"cell_type":"markdown","source":"# Preprocess"},{"metadata":{"trusted":true,"_uuid":"7273af8c83286a2ecc6c430be7bf2bf542cb60ec"},"cell_type":"code","source":"cat_col = ['feature_1','feature_2', 'feature_3', 'merchant_group_id', 'merchant_category_id', 'subsector_id', 'category_1',\n          'most_recent_sales_range', 'most_recent_purchases_range', 'category_4', 'city_id', 'state_id', 'category_2']\nnumeric_col = train_set.columns[~train_set.columns.isin(np.append(cat_col, ['card_id', 'first_active_month']))]\nused_col = np.concatenate((cat_col, numeric_col), axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"caf79884dfa470e2ce0aee0111152fbdb00c7948"},"cell_type":"code","source":"for col in ['category_1', 'most_recent_sales_range', 'most_recent_purchases_range', 'category_4']:    \n    lbl = LabelEncoder()\n    lbl.fit(train_set[col].unique().astype('str'))\n    train_set[col] = lbl.transform(train_set[col].astype('str'))\n    test_set[col] = lbl.transform(test_set[col].astype('str'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"79acd265a41fb43428b9c222f910f60b36360d01"},"cell_type":"code","source":"train_set = train_set.fillna(-20)\ntest_set = test_set.fillna(-20)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"565f7c63304115cdc8a6aaa3c5cddadd0b3c5ed2"},"cell_type":"markdown","source":"#  Lightgbm regression MODEL"},{"metadata":{"trusted":true,"_uuid":"d71de51418144ee7401ff31a6870591c28c1cad7"},"cell_type":"code","source":"lgb_params = {'num_leaves': 111,\n             'min_data_in_leaf': 149, \n             'objective':'regression',\n             'max_depth': 9,\n             'learning_rate': 0.005,\n             \"boosting\": \"gbdt\",\n             \"feature_fraction\": 0.7522,\n             \"bagging_freq\": 1,\n             \"bagging_fraction\": 0.7083 ,\n             \"bagging_seed\": 11,\n             \"metric\": 'rmse',\n             \"lambda_l1\": 0.2634,\n             \"random_state\": 133,\n             \"verbosity\": -1}\n\nfolds = KFold(n_splits=5, shuffle=True, random_state=24)\ntrain_predictions_lgb = np.zeros(len(train_set))\npredictions_lgb = np.zeros(len(test_set))\nfeature_importance_df = pd.DataFrame()\n\nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(train_set.values, target.values)):\n    print(\"lgb \" + str(fold_) + \"-\" * 50)\n    train_data_lgb = lgb.Dataset(train_set.iloc[trn_idx][used_col],\n                           label=target.iloc[trn_idx],\n                           categorical_feature=['feature_1','feature_2', 'feature_3'])\n    val_data_lgb = lgb.Dataset(train_set.iloc[val_idx][used_col],\n                           label=target.iloc[val_idx],\n                           categorical_feature=['feature_1','feature_2', 'feature_3'])\n\n    num_round = 10000\n    lgb_model = lgb.train(lgb_params,\n                    train_data_lgb,\n                    num_round,\n                    valid_sets = [train_data_lgb, val_data_lgb],\n                    verbose_eval=100,\n                    early_stopping_rounds = 200)\n    \n    fold_importance_df = pd.DataFrame()\n    fold_importance_df[\"feature\"] = used_col\n    fold_importance_df[\"importance\"] = lgb_model.feature_importance()\n    fold_importance_df[\"fold\"] = fold_ + 1\n    feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)\n    \n    train_predictions_lgb[val_idx] = lgb_model.predict(train_set.iloc[val_idx][used_col], num_iteration=lgb_model.best_iteration)\n    \n    predictions_lgb += lgb_model.predict(test_set[used_col], num_iteration=lgb_model.best_iteration) / folds.n_splits\n\nprint(\"CV score: {:<8.5f}\".format(mean_squared_error(train_predictions_lgb, target)**0.5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"26237018eafbf44fe54c1341daeeb4412d793b4f"},"cell_type":"code","source":"sub_df = pd.DataFrame({\"card_id\":test_set[\"card_id\"].values})\nsub_df[\"target\"] = predictions_lgb\nsub_df.to_csv(\"submit_lgb_1.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0ee98cf51711486fba6aae13288fd880d9ee3c5b"},"cell_type":"code","source":"cols = (feature_importance_df[[\"feature\", \"importance\"]]\n        .groupby(\"feature\")\n        .mean()\n        .sort_values(by=\"importance\", ascending=False)[:1000].index)\n\nbest_features = feature_importance_df.loc[feature_importance_df.feature.isin(cols)]\n\nplt.figure(figsize=(14,25))\nsns.barplot(x=\"importance\",\n            y=\"feature\",\n            data=best_features.sort_values(by=\"importance\",\n                                           ascending=False))\n\nplt.title('LightGBM Features (avg over folds)')\nplt.tight_layout()\nplt.savefig('lgbm_importances.png')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"99c7d3c5e427d540996b9645ff69cf3c1194ebc6"},"cell_type":"code","source":"sort_best_fatures = best_features.sort_values(by=\"importance\",ascending=False)\nsort_best_fatures.drop_duplicates(subset=['feature'], inplace=True)\ntop_feature = sort_best_fatures.iloc[:138,0].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bde2d8149fffdf8e8e59b0658914b240a2cfc453"},"cell_type":"code","source":"lgb_params = {'num_leaves': 111,\n             'min_data_in_leaf': 149, \n             'objective':'regression',\n             'max_depth': 9,\n             'learning_rate': 0.005,\n             \"boosting\": \"gbdt\",\n             \"feature_fraction\": 0.7522,\n             \"bagging_freq\": 1,\n             \"bagging_fraction\": 0.7083 ,\n             \"bagging_seed\": 11,\n             \"metric\": 'rmse',\n             \"lambda_l1\": 0.2634,\n             \"random_state\": 133,\n             \"verbosity\": -1}\n\nfolds = KFold(n_splits=5, shuffle=True, random_state=24)\ntrain_predictions_lgb = np.zeros(len(train_set))\npredictions_lgb = np.zeros(len(test_set))\n\nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(train_set.values, target.values)):\n    print(\"lgb \" + str(fold_) + \"-\" * 50)\n    train_data_lgb = lgb.Dataset(train_set.iloc[trn_idx][top_feature],\n                           label=target.iloc[trn_idx],\n                           categorical_feature=['feature_1'])\n    val_data_lgb = lgb.Dataset(train_set.iloc[val_idx][top_feature],\n                           label=target.iloc[val_idx],\n                           categorical_feature=['feature_1'])\n\n    num_round = 20000\n    lgb_model = lgb.train(lgb_params,\n                    train_data_lgb,\n                    num_round,\n                    valid_sets = [train_data_lgb, val_data_lgb],\n                    verbose_eval=100,\n                    early_stopping_rounds = 200)\n    \n    train_predictions_lgb[val_idx] = lgb_model.predict(train_set.iloc[val_idx][top_feature], num_iteration=lgb_model.best_iteration)\n    \n    predictions_lgb += lgb_model.predict(test_set[top_feature], num_iteration=lgb_model.best_iteration) / folds.n_splits\n\nprint(\"CV score: {:<8.5f}\".format(mean_squared_error(train_predictions_lgb, target)**0.5))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ed01edf3a451ea9b9f3394df0527aeab0566f5b9"},"cell_type":"markdown","source":"# Write result to sample submission file"},{"metadata":{"trusted":true,"_uuid":"3ae4b5456a39d955799650e8e0a8fa7935c372d9"},"cell_type":"code","source":"sub_df = pd.DataFrame({\"card_id\":test_set[\"card_id\"].values})\nsub_df[\"target\"] = predictions_lgb\nsub_df.to_csv(\"submit_lgb_2.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}