{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Problem statement\n\nPredict on volunteer retention in China during Covid19","metadata":{}},{"cell_type":"markdown","source":"Import libraries","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom datetime import datetime\nimport os\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2023-06-21T07:50:18.734135Z","iopub.execute_input":"2023-06-21T07:50:18.735046Z","iopub.status.idle":"2023-06-21T07:50:18.741400Z","shell.execute_reply.started":"2023-06-21T07:50:18.735005Z","shell.execute_reply":"2023-06-21T07:50:18.740426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Retrieve files","metadata":{}},{"cell_type":"code","source":"for dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n","metadata":{"execution":{"iopub.status.busy":"2023-06-21T07:50:18.768254Z","iopub.execute_input":"2023-06-21T07:50:18.768652Z","iopub.status.idle":"2023-06-21T07:50:18.777122Z","shell.execute_reply.started":"2023-06-21T07:50:18.768622Z","shell.execute_reply":"2023-06-21T07:50:18.775852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Read files","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/MLSP2023-volunteers-01/retention_train_feat.csv')\ntest = pd.read_csv('/kaggle/input/MLSP2023-volunteers-01/retention_test_feat.csv')\nsubmission = pd.read_csv('/kaggle/input/MLSP2023-volunteers-01/sample_submission_retention.csv')\ngt = pd.read_csv('/kaggle/input/MLSP2023-volunteers-01/retention_gt_train.csv')","metadata":{"execution":{"iopub.status.busy":"2023-06-21T07:50:18.802854Z","iopub.execute_input":"2023-06-21T07:50:18.803234Z","iopub.status.idle":"2023-06-21T07:50:26.618336Z","shell.execute_reply.started":"2023-06-21T07:50:18.803204Z","shell.execute_reply":"2023-06-21T07:50:26.617086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2023-06-21T07:50:26.621547Z","iopub.execute_input":"2023-06-21T07:50:26.622004Z","iopub.status.idle":"2023-06-21T07:50:26.646518Z","shell.execute_reply.started":"2023-06-21T07:50:26.621964Z","shell.execute_reply":"2023-06-21T07:50:26.645248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.iloc[:, 1:12] \ntrain","metadata":{"execution":{"iopub.status.busy":"2023-06-21T07:50:26.647931Z","iopub.execute_input":"2023-06-21T07:50:26.648264Z","iopub.status.idle":"2023-06-21T07:50:26.752589Z","shell.execute_reply.started":"2023-06-21T07:50:26.648235Z","shell.execute_reply":"2023-06-21T07:50:26.751457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count_user = train['user_id'].value_counts()\ncount_user = pd.DataFrame(count_user)\ncount_user","metadata":{"execution":{"iopub.status.busy":"2023-06-21T07:50:26.755206Z","iopub.execute_input":"2023-06-21T07:50:26.755567Z","iopub.status.idle":"2023-06-21T07:50:26.784655Z","shell.execute_reply.started":"2023-06-21T07:50:26.755536Z","shell.execute_reply":"2023-06-21T07:50:26.783406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['task_start_time'] = pd.to_datetime(train['task_start_time'])\ntrain['task_end_time'] = pd.to_datetime(train['task_end_time'])\ntrain['total_task_time'] = pd.to_numeric(train['task_end_time'] - train['task_start_time'])","metadata":{"execution":{"iopub.status.busy":"2023-06-21T07:50:26.786264Z","iopub.execute_input":"2023-06-21T07:50:26.786709Z","iopub.status.idle":"2023-06-21T07:50:27.602945Z","shell.execute_reply.started":"2023-06-21T07:50:26.786670Z","shell.execute_reply":"2023-06-21T07:50:27.602077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_gt = pd.merge(train, gt, on=\"user_id\")\ntrain_gt","metadata":{"execution":{"iopub.status.busy":"2023-06-21T07:50:27.604194Z","iopub.execute_input":"2023-06-21T07:50:27.604681Z","iopub.status.idle":"2023-06-21T07:50:28.137773Z","shell.execute_reply.started":"2023-06-21T07:50:27.604652Z","shell.execute_reply":"2023-06-21T07:50:28.136697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_gt.isna().sum().sum()","metadata":{"execution":{"iopub.status.busy":"2023-06-21T07:50:28.139217Z","iopub.execute_input":"2023-06-21T07:50:28.139582Z","iopub.status.idle":"2023-06-21T07:50:29.689211Z","shell.execute_reply.started":"2023-06-21T07:50:28.139553Z","shell.execute_reply":"2023-06-21T07:50:29.687989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_pt = train_gt.pivot_table(index='user_id')\ntrain_pt.reset_index(inplace=True)\ntrain_pt[train_pt['is_organizer'] > 0] = 1\ntrain_pt.drop('task_id', axis=1, inplace=True)\ntrain_pt","metadata":{"execution":{"iopub.status.busy":"2023-06-21T07:50:29.690740Z","iopub.execute_input":"2023-06-21T07:50:29.691073Z","iopub.status.idle":"2023-06-21T07:50:29.809031Z","shell.execute_reply.started":"2023-06-21T07:50:29.691045Z","shell.execute_reply":"2023-06-21T07:50:29.808000Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gt","metadata":{"execution":{"iopub.status.busy":"2023-06-21T07:50:29.810294Z","iopub.execute_input":"2023-06-21T07:50:29.810625Z","iopub.status.idle":"2023-06-21T07:50:29.823850Z","shell.execute_reply.started":"2023-06-21T07:50:29.810598Z","shell.execute_reply":"2023-06-21T07:50:29.822527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in train:\n    print(col, train[col].nunique())","metadata":{"execution":{"iopub.status.busy":"2023-06-21T07:50:29.828337Z","iopub.execute_input":"2023-06-21T07:50:29.828773Z","iopub.status.idle":"2023-06-21T07:50:30.425997Z","shell.execute_reply.started":"2023-06-21T07:50:29.828741Z","shell.execute_reply":"2023-06-21T07:50:30.424913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2023-06-21T07:50:30.427312Z","iopub.execute_input":"2023-06-21T07:50:30.427755Z","iopub.status.idle":"2023-06-21T07:50:31.923966Z","shell.execute_reply.started":"2023-06-21T07:50:30.427724Z","shell.execute_reply":"2023-06-21T07:50:31.922674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test","metadata":{"execution":{"iopub.status.busy":"2023-06-21T07:50:31.927441Z","iopub.execute_input":"2023-06-21T07:50:31.927855Z","iopub.status.idle":"2023-06-21T07:50:31.951416Z","shell.execute_reply.started":"2023-06-21T07:50:31.927824Z","shell.execute_reply":"2023-06-21T07:50:31.950083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = test.iloc[:, 1:12] \ntest","metadata":{"execution":{"iopub.status.busy":"2023-06-21T07:50:31.952938Z","iopub.execute_input":"2023-06-21T07:50:31.953388Z","iopub.status.idle":"2023-06-21T07:50:32.047066Z","shell.execute_reply.started":"2023-06-21T07:50:31.953319Z","shell.execute_reply":"2023-06-21T07:50:32.045942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in test:\n    print(col, test[col].nunique())","metadata":{"execution":{"iopub.status.busy":"2023-06-21T07:50:32.048489Z","iopub.execute_input":"2023-06-21T07:50:32.048803Z","iopub.status.idle":"2023-06-21T07:50:32.927160Z","shell.execute_reply.started":"2023-06-21T07:50:32.048776Z","shell.execute_reply":"2023-06-21T07:50:32.925959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.isna().sum().sum()","metadata":{"execution":{"iopub.status.busy":"2023-06-21T07:50:32.929711Z","iopub.execute_input":"2023-06-21T07:50:32.930175Z","iopub.status.idle":"2023-06-21T07:50:35.184253Z","shell.execute_reply.started":"2023-06-21T07:50:32.930133Z","shell.execute_reply":"2023-06-21T07:50:35.183392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['task_start_time'] = pd.to_datetime(test['task_start_time'])\ntest['task_end_time'] = pd.to_datetime(test['task_end_time'])\ntest['total_task_time'] = pd.to_numeric(test['task_end_time'] - test['task_start_time'])","metadata":{"execution":{"iopub.status.busy":"2023-06-21T07:50:35.186023Z","iopub.execute_input":"2023-06-21T07:50:35.186323Z","iopub.status.idle":"2023-06-21T07:50:36.040485Z","shell.execute_reply.started":"2023-06-21T07:50:35.186298Z","shell.execute_reply":"2023-06-21T07:50:36.039346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"execution":{"iopub.status.busy":"2023-06-21T07:50:36.041704Z","iopub.execute_input":"2023-06-21T07:50:36.042022Z","iopub.status.idle":"2023-06-21T07:50:36.057758Z","shell.execute_reply.started":"2023-06-21T07:50:36.041995Z","shell.execute_reply":"2023-06-21T07:50:36.056577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"t_gt = submission.copy()\nt_gt['user_id'] = t_gt['Id']\nt_gt.drop('Id', axis=1, inplace=True)\nt_gt","metadata":{"execution":{"iopub.status.busy":"2023-06-21T07:50:36.059216Z","iopub.execute_input":"2023-06-21T07:50:36.059579Z","iopub.status.idle":"2023-06-21T07:50:36.078655Z","shell.execute_reply.started":"2023-06-21T07:50:36.059547Z","shell.execute_reply":"2023-06-21T07:50:36.077443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_gt = pd.merge(t_gt, test, on=\"user_id\")\ntest_gt","metadata":{"execution":{"iopub.status.busy":"2023-06-21T07:50:36.080999Z","iopub.execute_input":"2023-06-21T07:50:36.081315Z","iopub.status.idle":"2023-06-21T07:50:36.792513Z","shell.execute_reply.started":"2023-06-21T07:50:36.081289Z","shell.execute_reply":"2023-06-21T07:50:36.791374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pt = test_gt.pivot_table(index='user_id',sort=False)\ntest_pt.reindex(index=t_gt.index)\ntest_pt.reset_index(inplace=True)\ntest_pt[test_pt['is_organizer'] > 0] = 1\ntest_pt.drop('task_id', axis=1, inplace=True)\ntest_pt.drop('Predicted', axis=1, inplace=True)\ncolumn_titles = ['user_id', 'is_organizer', 'latitude', 'longitude', 'total_task_time']\ndf_reorder = test_pt[column_titles]\ntest_pt = df_reorder\ntest_pt","metadata":{"execution":{"iopub.status.busy":"2023-06-21T07:50:36.794248Z","iopub.execute_input":"2023-06-21T07:50:36.794595Z","iopub.status.idle":"2023-06-21T07:50:36.910274Z","shell.execute_reply.started":"2023-06-21T07:50:36.794565Z","shell.execute_reply":"2023-06-21T07:50:36.909012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Analyse label","metadata":{}},{"cell_type":"code","source":"target = train_pt.pop('count')\n\nsns.displot(target, kde=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-21T07:50:36.911784Z","iopub.execute_input":"2023-06-21T07:50:36.912201Z","iopub.status.idle":"2023-06-21T07:50:42.675285Z","shell.execute_reply.started":"2023-06-21T07:50:36.912162Z","shell.execute_reply":"2023-06-21T07:50:42.674126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Define X and y","metadata":{}},{"cell_type":"code","source":"y = target\nX = train_pt\nX_test = test_pt","metadata":{"execution":{"iopub.status.busy":"2023-06-21T07:50:42.676855Z","iopub.execute_input":"2023-06-21T07:50:42.677168Z","iopub.status.idle":"2023-06-21T07:50:42.682375Z","shell.execute_reply.started":"2023-06-21T07:50:42.677140Z","shell.execute_reply":"2023-06-21T07:50:42.680897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X","metadata":{"execution":{"iopub.status.busy":"2023-06-21T07:50:42.683999Z","iopub.execute_input":"2023-06-21T07:50:42.684329Z","iopub.status.idle":"2023-06-21T07:50:42.717549Z","shell.execute_reply.started":"2023-06-21T07:50:42.684300Z","shell.execute_reply":"2023-06-21T07:50:42.716458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test","metadata":{"execution":{"iopub.status.busy":"2023-06-21T07:50:42.718839Z","iopub.execute_input":"2023-06-21T07:50:42.719611Z","iopub.status.idle":"2023-06-21T07:50:42.745866Z","shell.execute_reply.started":"2023-06-21T07:50:42.719574Z","shell.execute_reply":"2023-06-21T07:50:42.744253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Split dataset into training and validation sets","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.1,random_state=42, shuffle=y)\nX_train.shape, y_train.shape, X_val.shape, y_val.shape","metadata":{"execution":{"iopub.status.busy":"2023-06-21T07:50:42.747754Z","iopub.execute_input":"2023-06-21T07:50:42.748804Z","iopub.status.idle":"2023-06-21T07:50:42.763758Z","shell.execute_reply.started":"2023-06-21T07:50:42.748757Z","shell.execute_reply":"2023-06-21T07:50:42.762745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Define model","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import LinearRegression\n\nmodel = LinearRegression().fit(X_train, y_train)\nprint(model.score(X_train, y_train))","metadata":{"execution":{"iopub.status.busy":"2023-06-21T07:50:42.765098Z","iopub.execute_input":"2023-06-21T07:50:42.765474Z","iopub.status.idle":"2023-06-21T07:50:42.790456Z","shell.execute_reply.started":"2023-06-21T07:50:42.765438Z","shell.execute_reply":"2023-06-21T07:50:42.789299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = model.predict(X_val)\ny_pred","metadata":{"execution":{"iopub.status.busy":"2023-06-21T07:50:42.791966Z","iopub.execute_input":"2023-06-21T07:50:42.792746Z","iopub.status.idle":"2023-06-21T07:50:42.811322Z","shell.execute_reply.started":"2023-06-21T07:50:42.792703Z","shell.execute_reply":"2023-06-21T07:50:42.809659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import mean_squared_error\nfrom math import sqrt\n\nrms = sqrt(mean_squared_error(y_val, y_pred))\nrms","metadata":{"execution":{"iopub.status.busy":"2023-06-21T07:50:42.820767Z","iopub.execute_input":"2023-06-21T07:50:42.822060Z","iopub.status.idle":"2023-06-21T07:50:42.834011Z","shell.execute_reply.started":"2023-06-21T07:50:42.822005Z","shell.execute_reply":"2023-06-21T07:50:42.832402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots()\nax.scatter(y_val, y_pred, edgecolors=(0, 0, 0))\nax.plot([y.min(), y.max()], [y.min(), y.max()], 'k--', lw=4)\nax.set_xlabel('Measured')\nax.set_ylabel('Predicted')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-21T07:50:42.836129Z","iopub.execute_input":"2023-06-21T07:50:42.836980Z","iopub.status.idle":"2023-06-21T07:50:43.146554Z","shell.execute_reply.started":"2023-06-21T07:50:42.836938Z","shell.execute_reply":"2023-06-21T07:50:43.145439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.displot(y_pred, kde=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-21T07:50:43.147982Z","iopub.execute_input":"2023-06-21T07:50:43.148444Z","iopub.status.idle":"2023-06-21T07:50:43.798962Z","shell.execute_reply.started":"2023-06-21T07:50:43.148404Z","shell.execute_reply":"2023-06-21T07:50:43.797707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_val=pd.DataFrame({'actual_value': y_val, 'predicted_value':y_pred})\ndf_val.reset_index(drop=True, inplace=True)\ndf_val","metadata":{"execution":{"iopub.status.busy":"2023-06-21T07:50:43.800620Z","iopub.execute_input":"2023-06-21T07:50:43.800995Z","iopub.status.idle":"2023-06-21T07:50:43.817060Z","shell.execute_reply.started":"2023-06-21T07:50:43.800958Z","shell.execute_reply":"2023-06-21T07:50:43.815650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Predict on test set","metadata":{}},{"cell_type":"code","source":"prediction = model.predict(X_test)\nprediction","metadata":{"execution":{"iopub.status.busy":"2023-06-21T07:50:43.818370Z","iopub.execute_input":"2023-06-21T07:50:43.818728Z","iopub.status.idle":"2023-06-21T07:50:43.838887Z","shell.execute_reply.started":"2023-06-21T07:50:43.818699Z","shell.execute_reply":"2023-06-21T07:50:43.837573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.displot(prediction, kde=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-21T07:50:43.840145Z","iopub.execute_input":"2023-06-21T07:50:43.842154Z","iopub.status.idle":"2023-06-21T07:50:45.134408Z","shell.execute_reply.started":"2023-06-21T07:50:43.842110Z","shell.execute_reply":"2023-06-21T07:50:45.132938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Prepare submission","metadata":{}},{"cell_type":"code","source":"submission['Predicted'] = prediction\nsubmission.to_csv('submission.csv', index=False)\nsubmission = pd.read_csv('submission.csv')\nsubmission","metadata":{"execution":{"iopub.status.busy":"2023-06-21T07:50:45.136155Z","iopub.execute_input":"2023-06-21T07:50:45.137215Z","iopub.status.idle":"2023-06-21T07:50:45.269826Z","shell.execute_reply.started":"2023-06-21T07:50:45.137165Z","shell.execute_reply":"2023-06-21T07:50:45.268366Z"},"trusted":true},"execution_count":null,"outputs":[]}]}