{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Problem statement\n\nPredict on volunteer retention in China during Covid19","metadata":{}},{"cell_type":"markdown","source":"Import libraries","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom datetime import datetime\nimport os\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2023-06-22T06:24:17.716396Z","iopub.execute_input":"2023-06-22T06:24:17.717213Z","iopub.status.idle":"2023-06-22T06:24:19.272879Z","shell.execute_reply.started":"2023-06-22T06:24:17.717166Z","shell.execute_reply":"2023-06-22T06:24:19.271913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Retrieve files","metadata":{}},{"cell_type":"code","source":"for dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n","metadata":{"execution":{"iopub.status.busy":"2023-06-22T06:24:19.274704Z","iopub.execute_input":"2023-06-22T06:24:19.275057Z","iopub.status.idle":"2023-06-22T06:24:19.281166Z","shell.execute_reply.started":"2023-06-22T06:24:19.275002Z","shell.execute_reply":"2023-06-22T06:24:19.280061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Read files","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/MLSP2023-volunteers-01/retention_train_feat.csv')\ntest = pd.read_csv('/kaggle/input/MLSP2023-volunteers-01/retention_test_feat.csv')\nsubmission = pd.read_csv('/kaggle/input/MLSP2023-volunteers-01/sample_submission_retention.csv')\ngt = pd.read_csv('/kaggle/input/MLSP2023-volunteers-01/retention_gt_train.csv')","metadata":{"execution":{"iopub.status.busy":"2023-06-22T06:24:19.282584Z","iopub.execute_input":"2023-06-22T06:24:19.282892Z","iopub.status.idle":"2023-06-22T06:24:31.012379Z","shell.execute_reply.started":"2023-06-22T06:24:19.282865Z","shell.execute_reply":"2023-06-22T06:24:31.011065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2023-06-22T06:24:31.015606Z","iopub.execute_input":"2023-06-22T06:24:31.016035Z","iopub.status.idle":"2023-06-22T06:24:31.069463Z","shell.execute_reply.started":"2023-06-22T06:24:31.015982Z","shell.execute_reply":"2023-06-22T06:24:31.066604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.iloc[:, 1:12] \ntrain","metadata":{"execution":{"iopub.status.busy":"2023-06-22T06:24:31.070793Z","iopub.execute_input":"2023-06-22T06:24:31.071224Z","iopub.status.idle":"2023-06-22T06:24:31.224904Z","shell.execute_reply.started":"2023-06-22T06:24:31.071185Z","shell.execute_reply":"2023-06-22T06:24:31.224073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count_user = train['user_id'].value_counts()\ncount_user = pd.DataFrame(count_user)\ncount_user","metadata":{"execution":{"iopub.status.busy":"2023-06-22T06:24:31.226448Z","iopub.execute_input":"2023-06-22T06:24:31.226751Z","iopub.status.idle":"2023-06-22T06:24:31.279601Z","shell.execute_reply.started":"2023-06-22T06:24:31.226727Z","shell.execute_reply":"2023-06-22T06:24:31.278554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['task_start_time'] = pd.to_datetime(train['task_start_time'])\ntrain['task_end_time'] = pd.to_datetime(train['task_end_time'])\ntrain['total_task_time'] = pd.to_numeric(train['task_end_time'] - train['task_start_time'])","metadata":{"execution":{"iopub.status.busy":"2023-06-22T06:24:31.281190Z","iopub.execute_input":"2023-06-22T06:24:31.281889Z","iopub.status.idle":"2023-06-22T06:24:32.060481Z","shell.execute_reply.started":"2023-06-22T06:24:31.281850Z","shell.execute_reply":"2023-06-22T06:24:32.059126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_gt = pd.merge(train, gt, on=\"user_id\")\ntrain_gt","metadata":{"execution":{"iopub.status.busy":"2023-06-22T06:24:32.061834Z","iopub.execute_input":"2023-06-22T06:24:32.062177Z","iopub.status.idle":"2023-06-22T06:24:32.739469Z","shell.execute_reply.started":"2023-06-22T06:24:32.062150Z","shell.execute_reply":"2023-06-22T06:24:32.738537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_gt.isna().sum().sum()","metadata":{"execution":{"iopub.status.busy":"2023-06-22T06:24:32.740717Z","iopub.execute_input":"2023-06-22T06:24:32.741133Z","iopub.status.idle":"2023-06-22T06:24:33.460762Z","shell.execute_reply.started":"2023-06-22T06:24:32.741107Z","shell.execute_reply":"2023-06-22T06:24:33.459625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_pt = train_gt.pivot_table(index='user_id')\ntrain_pt.reset_index(inplace=True)\ntrain_pt[train_pt['is_organizer'] > 0] = 1\ntrain_pt.drop('task_id', axis=1, inplace=True)\ntrain_pt.drop('user_id', axis=1, inplace=True)\ntrain_pt","metadata":{"execution":{"iopub.status.busy":"2023-06-22T06:24:33.464469Z","iopub.execute_input":"2023-06-22T06:24:33.464853Z","iopub.status.idle":"2023-06-22T06:24:33.592048Z","shell.execute_reply.started":"2023-06-22T06:24:33.464821Z","shell.execute_reply":"2023-06-22T06:24:33.590999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gt","metadata":{"execution":{"iopub.status.busy":"2023-06-22T06:24:33.593532Z","iopub.execute_input":"2023-06-22T06:24:33.593956Z","iopub.status.idle":"2023-06-22T06:24:33.606562Z","shell.execute_reply.started":"2023-06-22T06:24:33.593916Z","shell.execute_reply":"2023-06-22T06:24:33.605608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in train:\n    print(col, train[col].nunique())","metadata":{"execution":{"iopub.status.busy":"2023-06-22T06:24:33.608063Z","iopub.execute_input":"2023-06-22T06:24:33.608982Z","iopub.status.idle":"2023-06-22T06:24:34.078650Z","shell.execute_reply.started":"2023-06-22T06:24:33.608950Z","shell.execute_reply":"2023-06-22T06:24:34.077264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2023-06-22T06:24:34.080321Z","iopub.execute_input":"2023-06-22T06:24:34.080695Z","iopub.status.idle":"2023-06-22T06:24:34.788312Z","shell.execute_reply.started":"2023-06-22T06:24:34.080662Z","shell.execute_reply":"2023-06-22T06:24:34.787049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test","metadata":{"execution":{"iopub.status.busy":"2023-06-22T06:24:34.789654Z","iopub.execute_input":"2023-06-22T06:24:34.790072Z","iopub.status.idle":"2023-06-22T06:24:34.811300Z","shell.execute_reply.started":"2023-06-22T06:24:34.790044Z","shell.execute_reply":"2023-06-22T06:24:34.810307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = test.iloc[:, 1:12] \ntest","metadata":{"execution":{"iopub.status.busy":"2023-06-22T06:24:34.812766Z","iopub.execute_input":"2023-06-22T06:24:34.813075Z","iopub.status.idle":"2023-06-22T06:24:34.915430Z","shell.execute_reply.started":"2023-06-22T06:24:34.813051Z","shell.execute_reply":"2023-06-22T06:24:34.914491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in test:\n    print(col, test[col].nunique())","metadata":{"execution":{"iopub.status.busy":"2023-06-22T06:24:34.917749Z","iopub.execute_input":"2023-06-22T06:24:34.918104Z","iopub.status.idle":"2023-06-22T06:24:35.629349Z","shell.execute_reply.started":"2023-06-22T06:24:34.918076Z","shell.execute_reply":"2023-06-22T06:24:35.628102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.isna().sum().sum()","metadata":{"execution":{"iopub.status.busy":"2023-06-22T06:24:35.630678Z","iopub.execute_input":"2023-06-22T06:24:35.631001Z","iopub.status.idle":"2023-06-22T06:24:36.663473Z","shell.execute_reply.started":"2023-06-22T06:24:35.630976Z","shell.execute_reply":"2023-06-22T06:24:36.662557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['task_start_time'] = pd.to_datetime(test['task_start_time'])\ntest['task_end_time'] = pd.to_datetime(test['task_end_time'])\ntest['total_task_time'] = pd.to_numeric(test['task_end_time'] - test['task_start_time'])","metadata":{"execution":{"iopub.status.busy":"2023-06-22T06:24:36.664659Z","iopub.execute_input":"2023-06-22T06:24:36.664953Z","iopub.status.idle":"2023-06-22T06:24:37.386504Z","shell.execute_reply.started":"2023-06-22T06:24:36.664929Z","shell.execute_reply":"2023-06-22T06:24:37.385299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"execution":{"iopub.status.busy":"2023-06-22T06:24:37.388047Z","iopub.execute_input":"2023-06-22T06:24:37.388388Z","iopub.status.idle":"2023-06-22T06:24:37.401748Z","shell.execute_reply.started":"2023-06-22T06:24:37.388360Z","shell.execute_reply":"2023-06-22T06:24:37.400618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"t_gt = submission.copy()\nt_gt['user_id'] = t_gt['Id']\nt_gt.drop('Id', axis=1, inplace=True)\nt_gt","metadata":{"execution":{"iopub.status.busy":"2023-06-22T06:24:37.403401Z","iopub.execute_input":"2023-06-22T06:24:37.403900Z","iopub.status.idle":"2023-06-22T06:24:37.421459Z","shell.execute_reply.started":"2023-06-22T06:24:37.403863Z","shell.execute_reply":"2023-06-22T06:24:37.420094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_gt = pd.merge(t_gt, test, on=\"user_id\")\ntest_gt","metadata":{"execution":{"iopub.status.busy":"2023-06-22T06:24:37.422726Z","iopub.execute_input":"2023-06-22T06:24:37.423222Z","iopub.status.idle":"2023-06-22T06:24:38.316786Z","shell.execute_reply.started":"2023-06-22T06:24:37.423187Z","shell.execute_reply":"2023-06-22T06:24:38.315768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pt = test_gt.pivot_table(index='user_id',sort=False)\ntest_pt.reindex(index=t_gt.index)\ntest_pt.reset_index(inplace=True)\ntest_pt[test_pt['is_organizer'] > 0] = 1\ntest_pt.drop('task_id', axis=1, inplace=True)\ntest_pt.drop('Predicted', axis=1, inplace=True)\ntest_pt.drop('user_id', axis=1, inplace=True)\ncolumn_titles = ['is_organizer', 'latitude', 'longitude', 'total_task_time']\ndf_reorder = test_pt[column_titles]\ntest_pt = df_reorder\ntest_pt","metadata":{"execution":{"iopub.status.busy":"2023-06-22T06:24:38.318169Z","iopub.execute_input":"2023-06-22T06:24:38.318570Z","iopub.status.idle":"2023-06-22T06:24:38.431171Z","shell.execute_reply.started":"2023-06-22T06:24:38.318543Z","shell.execute_reply":"2023-06-22T06:24:38.430081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Analyse label","metadata":{}},{"cell_type":"code","source":"target = train_pt.pop('count')\n\nsns.displot(target, kde=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-22T06:24:38.432675Z","iopub.execute_input":"2023-06-22T06:24:38.433116Z","iopub.status.idle":"2023-06-22T06:24:43.738990Z","shell.execute_reply.started":"2023-06-22T06:24:38.433079Z","shell.execute_reply":"2023-06-22T06:24:43.737923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Define X and y","metadata":{}},{"cell_type":"code","source":"y = target\nX = train_pt\nX_test = test_pt","metadata":{"execution":{"iopub.status.busy":"2023-06-22T06:24:43.740323Z","iopub.execute_input":"2023-06-22T06:24:43.740663Z","iopub.status.idle":"2023-06-22T06:24:43.746873Z","shell.execute_reply.started":"2023-06-22T06:24:43.740626Z","shell.execute_reply":"2023-06-22T06:24:43.744462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X","metadata":{"execution":{"iopub.status.busy":"2023-06-22T06:24:43.748568Z","iopub.execute_input":"2023-06-22T06:24:43.748906Z","iopub.status.idle":"2023-06-22T06:24:43.783137Z","shell.execute_reply.started":"2023-06-22T06:24:43.748879Z","shell.execute_reply":"2023-06-22T06:24:43.782078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test","metadata":{"execution":{"iopub.status.busy":"2023-06-22T06:24:43.784467Z","iopub.execute_input":"2023-06-22T06:24:43.784952Z","iopub.status.idle":"2023-06-22T06:24:43.800865Z","shell.execute_reply.started":"2023-06-22T06:24:43.784925Z","shell.execute_reply":"2023-06-22T06:24:43.799949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Standardise data","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\n\nscaler = StandardScaler()\n\nfor col in X:\n    X[col] = scaler.fit_transform(X[col].values.reshape(-1,1))\n    X_test[col] = scaler.transform(X_test[col].values.reshape(-1,1))\nX","metadata":{"execution":{"iopub.status.busy":"2023-06-22T06:24:43.802659Z","iopub.execute_input":"2023-06-22T06:24:43.803327Z","iopub.status.idle":"2023-06-22T06:24:43.905756Z","shell.execute_reply.started":"2023-06-22T06:24:43.803287Z","shell.execute_reply":"2023-06-22T06:24:43.904572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Split dataset into training and validation sets","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.1,random_state=42, shuffle=y)\nX_train.shape, y_train.shape, X_val.shape, y_val.shape","metadata":{"execution":{"iopub.status.busy":"2023-06-22T06:24:43.911604Z","iopub.execute_input":"2023-06-22T06:24:43.912160Z","iopub.status.idle":"2023-06-22T06:24:44.025078Z","shell.execute_reply.started":"2023-06-22T06:24:43.912127Z","shell.execute_reply":"2023-06-22T06:24:44.023999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Define model","metadata":{}},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestRegressor\n\nmodel = RandomForestRegressor(max_depth=2,random_state=42).fit(X_train, y_train)\nprint(model.score(X_train, y_train))","metadata":{"execution":{"iopub.status.busy":"2023-06-22T06:24:44.026307Z","iopub.execute_input":"2023-06-22T06:24:44.027161Z","iopub.status.idle":"2023-06-22T06:24:45.583466Z","shell.execute_reply.started":"2023-06-22T06:24:44.027132Z","shell.execute_reply":"2023-06-22T06:24:45.582313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = model.predict(X_val)\ny_pred","metadata":{"execution":{"iopub.status.busy":"2023-06-22T06:24:45.585224Z","iopub.execute_input":"2023-06-22T06:24:45.585818Z","iopub.status.idle":"2023-06-22T06:24:45.611294Z","shell.execute_reply.started":"2023-06-22T06:24:45.585775Z","shell.execute_reply":"2023-06-22T06:24:45.610196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import mean_squared_error\nfrom math import sqrt\n\nrms = sqrt(mean_squared_error(y_val, y_pred))\nrms","metadata":{"execution":{"iopub.status.busy":"2023-06-22T06:24:45.612855Z","iopub.execute_input":"2023-06-22T06:24:45.613294Z","iopub.status.idle":"2023-06-22T06:24:45.621110Z","shell.execute_reply.started":"2023-06-22T06:24:45.613256Z","shell.execute_reply":"2023-06-22T06:24:45.620319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots()\nax.scatter(y_val, y_pred, edgecolors=(0, 0, 0))\nax.plot([y.min(), y.max()], [y.min(), y.max()], 'k--', lw=4)\nax.set_xlabel('Measured')\nax.set_ylabel('Predicted')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-22T06:24:45.622532Z","iopub.execute_input":"2023-06-22T06:24:45.623366Z","iopub.status.idle":"2023-06-22T06:24:45.868290Z","shell.execute_reply.started":"2023-06-22T06:24:45.623337Z","shell.execute_reply":"2023-06-22T06:24:45.867378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.displot(y_pred, kde=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-22T06:24:45.869379Z","iopub.execute_input":"2023-06-22T06:24:45.870217Z","iopub.status.idle":"2023-06-22T06:24:46.305333Z","shell.execute_reply.started":"2023-06-22T06:24:45.870186Z","shell.execute_reply":"2023-06-22T06:24:46.304254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_val=pd.DataFrame({'actual_value': y_val, 'predicted_value':y_pred})\ndf_val.reset_index(drop=True, inplace=True)\ndf_val","metadata":{"execution":{"iopub.status.busy":"2023-06-22T06:24:46.306759Z","iopub.execute_input":"2023-06-22T06:24:46.307145Z","iopub.status.idle":"2023-06-22T06:24:46.321942Z","shell.execute_reply.started":"2023-06-22T06:24:46.307109Z","shell.execute_reply":"2023-06-22T06:24:46.320761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Predict on test set","metadata":{}},{"cell_type":"code","source":"prediction = model.predict(X_test)\nprediction","metadata":{"execution":{"iopub.status.busy":"2023-06-22T06:24:46.323664Z","iopub.execute_input":"2023-06-22T06:24:46.324635Z","iopub.status.idle":"2023-06-22T06:24:46.395620Z","shell.execute_reply.started":"2023-06-22T06:24:46.324595Z","shell.execute_reply":"2023-06-22T06:24:46.394519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.displot(prediction, kde=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-22T06:24:46.397448Z","iopub.execute_input":"2023-06-22T06:24:46.397867Z","iopub.status.idle":"2023-06-22T06:24:48.794073Z","shell.execute_reply.started":"2023-06-22T06:24:46.397830Z","shell.execute_reply":"2023-06-22T06:24:48.792958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Prepare submission","metadata":{}},{"cell_type":"code","source":"submission['Predicted'] = prediction\nsubmission.to_csv('submission.csv', index=False)\nsubmission = pd.read_csv('submission.csv')\nsubmission","metadata":{"execution":{"iopub.status.busy":"2023-06-22T06:24:48.795402Z","iopub.execute_input":"2023-06-22T06:24:48.795736Z","iopub.status.idle":"2023-06-22T06:24:48.883578Z","shell.execute_reply.started":"2023-06-22T06:24:48.795709Z","shell.execute_reply":"2023-06-22T06:24:48.882473Z"},"trusted":true},"execution_count":null,"outputs":[]}]}