{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport datetime\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **1. Load the data**"},{"metadata":{},"cell_type":"markdown","source":"helper function to reduce memory usage"},{"metadata":{"trusted":true},"cell_type":"code","source":"def reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"read csv files and store them into data frames"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"new_transactions = pd.read_csv('/kaggle/input/elo-merchant-category-recommendation/new_merchant_transactions.csv',\n                               parse_dates=['purchase_date'])\n\nhistorical_transactions = pd.read_csv('/kaggle/input/elo-merchant-category-recommendation/historical_transactions.csv',\n                                      parse_dates=['purchase_date'])\n\ndef read_data(input_file):\n    df = pd.read_csv(input_file)\n    df['first_active_month'] = pd.to_datetime(df['first_active_month'])\n    df['elapsed_time'] = (datetime.date(2018, 2, 1) - df['first_active_month'].dt.date).dt.days\n    return df\n#_________________________________________\ntrain = read_data('/kaggle/input/elo-merchant-category-recommendation/train.csv')\ntest = read_data('/kaggle/input/elo-merchant-category-recommendation/test.csv')\n\ntarget = train['target']\nprint(target.shape)\nprint(test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# check on the top 5 from historical transactions\n\nhistorical_transactions.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 2. Preprocessing"},{"metadata":{},"cell_type":"markdown","source":"One Hot Encoding"},{"metadata":{"trusted":true},"cell_type":"code","source":"historical_transactions = pd.get_dummies(historical_transactions, columns=['category_2', 'category_3'])\nnew_transactions = pd.get_dummies(new_transactions, columns=['category_2', 'category_3'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Date processing"},{"metadata":{"trusted":true},"cell_type":"code","source":"for df in [historical_transactions, new_transactions]:\n    df['purchase_date'] = pd.to_datetime(df['purchase_date'])\n    df['year'] = df['purchase_date'].dt.year\n    df['weekofyear'] = df['purchase_date'].dt.weekofyear\n    df['month'] = df['purchase_date'].dt.month\n    df['dayofweek'] = df['purchase_date'].dt.dayofweek\n    df['weekend'] = (df.purchase_date.dt.weekday >=5).astype(int)\n    df['hour'] = df['purchase_date'].dt.hour\n    df['authorized_flag'] = df['authorized_flag'].map({'Y':1, 'N':0})\n    df['category_1'] = df['category_1'].map({'Y':1, 'N':0}) \n    #https://www.kaggle.com/c/elo-merchant-category-recommendation/discussion/73244\n    df['month_diff'] = ((datetime.datetime.today() - df['purchase_date']).dt.days)//30\n    df['month_diff'] += df['month_lag']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Reduce memory usage"},{"metadata":{"trusted":true},"cell_type":"code","source":"\nhistorical_transactions = reduce_mem_usage(historical_transactions)\nnew_transactions = reduce_mem_usage(new_transactions)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3. Feature Engineering"},{"metadata":{},"cell_type":"markdown","source":"helper function to apply aggregations on existing features to create new features"},{"metadata":{"trusted":true},"cell_type":"code","source":"def aggregate_transactions(history):\n    \n    history.loc[:, 'purchase_date'] = pd.DatetimeIndex(history['purchase_date']).\\\n                                      astype(np.int64) * 1e-9\n    \n    agg_func = {\n    'authorized_flag': ['mean'],\n    'category_1': ['sum', 'mean'],\n    'category_2_1.0': ['mean'],\n    'category_2_2.0': ['mean'],\n    'category_2_3.0': ['mean'],\n    'category_2_4.0': ['mean'],\n    'category_2_5.0': ['mean'],\n    'category_3_A': ['mean'],\n    'category_3_B': ['mean'],\n    'category_3_C': ['mean'],\n    'merchant_id': ['nunique'],\n    'merchant_category_id': ['nunique'],\n    'state_id': ['nunique'],\n    'city_id': ['nunique'],\n    'subsector_id': ['nunique'],\n    'purchase_amount': ['sum', 'mean', 'max', 'min', 'std'],\n    'installments': ['sum', 'mean', 'max', 'min', 'std'],\n    'purchase_date': [np.ptp, 'min', 'max'],\n    'month_lag': ['mean', 'max', 'min', 'std'],\n    'month_diff': ['mean'],\n    'month': ['nunique'],\n    'hour': ['nunique'],\n    'weekofyear': ['nunique'],\n    'dayofweek': ['nunique'],\n    'year': ['nunique'],\n    'authorized_flag': ['sum', 'mean'],\n    'weekend': ['sum', 'mean']\n    }\n    \n    agg_history = history.groupby(['card_id']).agg(agg_func)\n    agg_history.columns = ['_'.join(col).strip() for col in agg_history.columns.values]\n    agg_history.reset_index(inplace=True)\n    \n    df = (history.groupby('card_id')\n          .size()\n          .reset_index(name='transactions_count'))\n    \n    agg_history = pd.merge(df, agg_history, on='card_id', how='left')\n    \n    return agg_history","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"history stores aggregated results from historical transactions"},{"metadata":{"trusted":true},"cell_type":"code","source":"history = aggregate_transactions(historical_transactions)\nhistory.columns = ['hist_' + c if c != 'card_id' else c for c in history.columns]\nhistory[:5]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"new stores aggregated results from new merchant transactions"},{"metadata":{"trusted":true},"cell_type":"code","source":"new = aggregate_transactions(new_transactions)\nnew.columns = ['new_' + c if c != 'card_id' else c for c in new.columns]\nnew[:5]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 4. Combine dataframes to train and test dataframes"},{"metadata":{},"cell_type":"markdown","source":"join datasets on the common id, card_id for both train and test"},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.merge(train, history, on='card_id', how='left')\ntest = pd.merge(test, history, on='card_id', how='left')\n\ntrain = pd.merge(train, new, on='card_id', how='left')\ntest = pd.merge(test, new, on='card_id', how='left')\n\nhistory[0::5]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Impute missing values"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.impute import SimpleImputer\nmy_imputer = SimpleImputer()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# get features and remove any that have the incorrect data type for a data frame \nfeature_cols = [col for col in train.columns if col not in ['target', 'first_active_month', 'card_id']]\nX = train[feature_cols]\n\n# impute missing values\nX = my_imputer.fit_transform(X)\n\n# get the target vector\ny = train['target']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 5. Split test and training set from train dataframe"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.05, random_state=6)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#  6. Train on any regression models"},{"metadata":{},"cell_type":"markdown","source":"Import training models"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.neighbors import KNeighborsRegressor\nfrom sklearn.tree import DecisionTreeRegressor\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.ensemble import RandomForestRegressor\n\nreg_predictions = []","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Train using KNeighborsRegressor"},{"metadata":{"trusted":true},"cell_type":"code","source":"myKNeighborsReg = KNeighborsRegressor(n_neighbors = 3)\n\nmyKNeighborsReg.fit(X_train, y_train)\n\ny_predict_myKNeighborsReg = myKNeighborsReg.predict(X_test)\n\nreg_predictions.append(y_predict_myKNeighborsReg)\n\n# TODO: find and change a time stamp feature to a float","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Train using DecisionTreeRegressor"},{"metadata":{"trusted":true},"cell_type":"code","source":"myDecisionTreeReg = DecisionTreeRegressor(random_state = 5)\n\nmyDecisionTreeReg.fit(X_train, y_train)\n\ny_predict_myDecisionTreeReg = myDecisionTreeReg.predict(X_test)\n\nreg_predictions.append(y_predict_myDecisionTreeReg)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Train using LinearRegression"},{"metadata":{"trusted":true},"cell_type":"code","source":"myLinearReg = LinearRegression()\n\nmyLinearReg.fit(X_train, y_train)\n\ny_predict_myLinearReg = myLinearReg.predict(X_test)\n\nreg_predictions.append(y_predict_myLinearReg)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Train using RandomForestRegressor"},{"metadata":{"trusted":true},"cell_type":"code","source":"myRandomForestReg = RandomForestRegressor(n_estimators = 9, bootstrap = True, random_state = 3)\n\nmyRandomForestReg.fit(X_train, y_train)\n\ny_predict_myRandomForestReg = myRandomForestReg.predict(X_test)\n\nreg_predictions.append(y_predict_myRandomForestReg)\n\nprint(X.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 7. Check RMSE "},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn import metrics\n\nfor model, y_prediction in zip(['K Nearest Neighbor: ', 'Decision Tree: ', 'Linear Regression: ', 'Random Forest: '], reg_predictions):\n    mse = metrics.mean_squared_error(y_test, y_prediction)\n    rmse = np.sqrt(mse)\n    print(model + str(rmse))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 8. Dimensionality Reduction"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.decomposition import PCA\nn = 45 # (n is the number of components (new features)\n# after dimensionality reduction)\nmy_pca = PCA(n_components = n)\n# (X_Train is feature matrix of training set before DR,\n# X_Train_New is feature matrix of training set after DR):\nX_Train_new = my_pca.fit_transform(X_train)\nX_Test_new = my_pca.transform(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"reg_predictions_new = []","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Train using KNeighborsRegressor"},{"metadata":{"trusted":true},"cell_type":"code","source":"myKNeighborsReg = KNeighborsRegressor(n_neighbors = 3)\n\nmyKNeighborsReg.fit(X_Train_new, y_train)\n\ny_predict_myKNeighborsReg = myKNeighborsReg.predict(X_Test_new)\n\nreg_predictions_new.append(y_predict_myKNeighborsReg)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Train using DecisionTreeRegressor"},{"metadata":{"trusted":true},"cell_type":"code","source":"myDecisionTreeReg = DecisionTreeRegressor(random_state = 5)\n\nmyDecisionTreeReg.fit(X_Train_new, y_train)\n\ny_predict_myDecisionTreeReg = myDecisionTreeReg.predict(X_Test_new)\n\nreg_predictions_new.append(y_predict_myDecisionTreeReg)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Train using LinearRegression"},{"metadata":{"trusted":true},"cell_type":"code","source":"myLinearReg = LinearRegression()\n\nmyLinearReg.fit(X_Train_new, y_train)\n\ny_predict_myLinearReg = myLinearReg.predict(X_Test_new)\n\nreg_predictions_new.append(y_predict_myLinearReg)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Train using RandomForestRegressor"},{"metadata":{"trusted":true},"cell_type":"code","source":"myRandomForestReg = RandomForestRegressor(n_estimators = 9, bootstrap = True, random_state = 3)\n\nmyRandomForestReg.fit(X_Train_new, y_train)\n\ny_predict_myRandomForestReg = myRandomForestReg.predict(X_Test_new)\n\nreg_predictions_new.append(y_predict_myRandomForestReg)\n\nprint(X.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for model, y_prediction in zip(['K Nearest Neighbor: ', 'Decision Tree: ', 'Linear Regression: ', 'Random Forest: '], reg_predictions_new):\n    mse = metrics.mean_squared_error(y_test, y_prediction)\n    rmse = np.sqrt(mse)\n    print(model + str(rmse))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 9. Repeat steps 6, 7, and 8 on actual test dataframe"},{"metadata":{},"cell_type":"markdown","source":"Training and Testing on all new features"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_feature_cols = [col for col in test.columns if col not in ['target', 'first_active_month', 'card_id']]\nfinal_test = test[feature_cols]\nfinal_test = my_imputer.fit_transform(final_test)\n\nreg_predictions_final = {}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Train using KNeighborsRegressor"},{"metadata":{"trusted":true},"cell_type":"code","source":"myKNeighborsReg = KNeighborsRegressor(n_neighbors = 3)\n\nmyKNeighborsReg.fit(X, y)\n\ny_predict_myKNeighborsReg = myKNeighborsReg.predict(final_test)\n\nreg_predictions_final['K Nearest Neighbor: '] = y_predict_myKNeighborsReg","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Train using DecisionTreeRegressor"},{"metadata":{"trusted":true},"cell_type":"code","source":"myDecisionTreeReg = DecisionTreeRegressor(random_state = 5)\n\nmyDecisionTreeReg.fit(X, y)\n\ny_predict_myDecisionTreeReg = myDecisionTreeReg.predict(final_test)\n\nreg_predictions_final['Decision Tree: ']= y_predict_myDecisionTreeReg","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Train using LinearRegression"},{"metadata":{"trusted":true},"cell_type":"code","source":"myLinearReg = LinearRegression()\n\nmyLinearReg.fit(X, y)\n\ny_predict_myLinearReg = myLinearReg.predict(final_test)\n\nreg_predictions_final['Linear Regression: '] = y_predict_myLinearReg","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Train using RandomForestRegressor"},{"metadata":{"trusted":true},"cell_type":"code","source":"myRandomForestReg = RandomForestRegressor(n_estimators = 9, bootstrap = True, random_state = 3)\n\nmyRandomForestReg.fit(X, y)\n\ny_predict_myRandomForestReg = myRandomForestReg.predict(final_test)\n\nreg_predictions_final['Random Forest: '] = y_predict_myRandomForestReg\n\nprint(X.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for model, y_prediction in reg_predictions_final.items():\n    mse = metrics.mean_squared_error(target.iloc[:len(y_prediction)], y_prediction)\n    rmse = np.sqrt(mse)\n    print(model + str(rmse))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Training and testing using dimensionality reduction"},{"metadata":{"trusted":true},"cell_type":"code","source":"reg_predictions_final_dr = {}\n\nn = 45\n\nmy_pca = PCA(n_components = n)\n\nX_new = my_pca.fit_transform(X)\nfinal_test_new = my_pca.transform(final_test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Train using KNeighborsRegressor"},{"metadata":{"trusted":true},"cell_type":"code","source":"myKNeighborsReg = KNeighborsRegressor(n_neighbors = 3)\n\nmyKNeighborsReg.fit(X_new, y)\n\ny_predict_myKNeighborsReg = myKNeighborsReg.predict(final_test_new)\n\nreg_predictions_final_dr['K Nearest Neighbor: '] = y_predict_myKNeighborsReg","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Train using DecisionTreeRegressor"},{"metadata":{"trusted":true},"cell_type":"code","source":"myDecisionTreeReg = DecisionTreeRegressor(random_state = 5)\n\nmyDecisionTreeReg.fit(X_new, y)\n\ny_predict_myDecisionTreeReg = myDecisionTreeReg.predict(final_test_new)\n\nreg_predictions_final_dr['Decision Tree: ']= y_predict_myDecisionTreeReg","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Train using LinearRegression"},{"metadata":{"trusted":true},"cell_type":"code","source":"myLinearReg = LinearRegression()\n\nmyLinearReg.fit(X_new, y)\n\ny_predict_myLinearReg = myLinearReg.predict(final_test_new)\n\nreg_predictions_final_dr['Linear Regression: '] = y_predict_myLinearReg","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Train using RandomForestRegressor"},{"metadata":{"trusted":true},"cell_type":"code","source":"myRandomForestReg = RandomForestRegressor(n_estimators = 9, bootstrap = True, random_state = 3)\n\nmyRandomForestReg.fit(X_new, y)\n\ny_predict_myRandomForestReg = myRandomForestReg.predict(final_test_new)\n\nreg_predictions_final_dr['Random Forest: '] = y_predict_myRandomForestReg\n\nprint(X.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for model, y_prediction in reg_predictions_final_dr.items():\n    mse = metrics.mean_squared_error(target.iloc[:len(y_prediction)], y_prediction)\n    rmse = np.sqrt(mse)\n    print(model + str(rmse))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}