{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Problem statement\n\nPredict on volunteers during Covid in China task 2.","metadata":{}},{"cell_type":"markdown","source":"Import files","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2023-06-26T08:01:39.536707Z","iopub.execute_input":"2023-06-26T08:01:39.537885Z","iopub.status.idle":"2023-06-26T08:01:41.112835Z","shell.execute_reply.started":"2023-06-26T08:01:39.537847Z","shell.execute_reply":"2023-06-26T08:01:41.111748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Retrieve files","metadata":{}},{"cell_type":"code","source":"for dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2023-06-26T08:01:41.118583Z","iopub.execute_input":"2023-06-26T08:01:41.121021Z","iopub.status.idle":"2023-06-26T08:01:41.133572Z","shell.execute_reply.started":"2023-06-26T08:01:41.120987Z","shell.execute_reply":"2023-06-26T08:01:41.132538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Read files","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/MLSP2023-volunteers-part2/colab_train.csv')\ntest = pd.read_csv('/kaggle/input/MLSP2023-volunteers-part2/colab_test.csv')\ntrain_edge = pd.read_csv('/kaggle/input/MLSP2023-volunteers-part2/colab_train_edge_list.csv')\ncandidates = pd.read_csv('/kaggle/input/MLSP2023-volunteers-part2/colab_test_candidate_multistep.csv')\nsubmission = pd.read_csv('/kaggle/input/MLSP2023-volunteers-part2/sample_submission_collaboration_multistep.csv')","metadata":{"execution":{"iopub.status.busy":"2023-06-26T08:01:41.138728Z","iopub.execute_input":"2023-06-26T08:01:41.141171Z","iopub.status.idle":"2023-06-26T08:01:45.150908Z","shell.execute_reply.started":"2023-06-26T08:01:41.141119Z","shell.execute_reply":"2023-06-26T08:01:45.149737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2023-06-26T08:01:45.154739Z","iopub.execute_input":"2023-06-26T08:01:45.155133Z","iopub.status.idle":"2023-06-26T08:01:45.197932Z","shell.execute_reply.started":"2023-06-26T08:01:45.155106Z","shell.execute_reply":"2023-06-26T08:01:45.197215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.task_start_date.min(), train.task_start_date.max()","metadata":{"execution":{"iopub.status.busy":"2023-06-26T08:01:45.201064Z","iopub.execute_input":"2023-06-26T08:01:45.201911Z","iopub.status.idle":"2023-06-26T08:01:45.334814Z","shell.execute_reply.started":"2023-06-26T08:01:45.201871Z","shell.execute_reply":"2023-06-26T08:01:45.333731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_edge","metadata":{"execution":{"iopub.status.busy":"2023-06-26T08:01:45.336713Z","iopub.execute_input":"2023-06-26T08:01:45.337078Z","iopub.status.idle":"2023-06-26T08:01:45.353660Z","shell.execute_reply.started":"2023-06-26T08:01:45.337045Z","shell.execute_reply":"2023-06-26T08:01:45.352212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test","metadata":{"execution":{"iopub.status.busy":"2023-06-26T08:01:45.355846Z","iopub.execute_input":"2023-06-26T08:01:45.356786Z","iopub.status.idle":"2023-06-26T08:01:45.385448Z","shell.execute_reply.started":"2023-06-26T08:01:45.356750Z","shell.execute_reply":"2023-06-26T08:01:45.384223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.task_start_date.min(), test.task_start_date.max()","metadata":{"execution":{"iopub.status.busy":"2023-06-26T08:01:45.387846Z","iopub.execute_input":"2023-06-26T08:01:45.388409Z","iopub.status.idle":"2023-06-26T08:01:45.532367Z","shell.execute_reply.started":"2023-06-26T08:01:45.388378Z","shell.execute_reply":"2023-06-26T08:01:45.531202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"candidates","metadata":{"execution":{"iopub.status.busy":"2023-06-26T08:01:45.533528Z","iopub.execute_input":"2023-06-26T08:01:45.533875Z","iopub.status.idle":"2023-06-26T08:01:45.553347Z","shell.execute_reply.started":"2023-06-26T08:01:45.533842Z","shell.execute_reply":"2023-06-26T08:01:45.552181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"execution":{"iopub.status.busy":"2023-06-26T08:01:45.558294Z","iopub.execute_input":"2023-06-26T08:01:45.561223Z","iopub.status.idle":"2023-06-26T08:01:45.577332Z","shell.execute_reply.started":"2023-06-26T08:01:45.561188Z","shell.execute_reply":"2023-06-26T08:01:45.576221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"How many tasks on average a volunteer will participate in","metadata":{}},{"cell_type":"code","source":"print(\"Average Task Count of Train User: {:.2f}\\n\".format(train.groupby('user_id')['task_id'].agg('count').rename('task_count').reset_index()['task_count'].mean()),\\\n\"Average Task Count of Test User: {:.2f}\".format(test.groupby('user_id')['task_id'].agg('count').rename('task_count').reset_index()['task_count'].mean()))","metadata":{"execution":{"iopub.status.busy":"2023-06-26T08:01:45.581251Z","iopub.execute_input":"2023-06-26T08:01:45.581551Z","iopub.status.idle":"2023-06-26T08:01:45.615110Z","shell.execute_reply.started":"2023-06-26T08:01:45.581526Z","shell.execute_reply":"2023-06-26T08:01:45.613413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Construct graphs","metadata":{}},{"cell_type":"code","source":"import networkx as nx\n\ndata = pd.concat([train, test], ignore_index=True)\nprint(data.shape)","metadata":{"execution":{"iopub.status.busy":"2023-06-26T08:01:45.616774Z","iopub.execute_input":"2023-06-26T08:01:45.617132Z","iopub.status.idle":"2023-06-26T08:01:45.783295Z","shell.execute_reply.started":"2023-06-26T08:01:45.617099Z","shell.execute_reply":"2023-06-26T08:01:45.782212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Create nodes and collect attributes","metadata":{}},{"cell_type":"code","source":"nodes = data.user_id.unique()\nnode_dict = {}\n\nfor u in nodes:\n    d = data[data.user_id==u]\n    node_dict[u] = {}\n    node_dict[u]['org_times'] = d.is_organizer.sum()    # the times of u organizing a task\n    node_dict[u]['times_of_participation'] = d.task_id.unique().shape[0] # the times of u participating in a task\n    node_dict[u]['task_longitude'] = d.longitude.mean()    # the average longitude of u's tasks\n    node_dict[u]['task_latitude'] = d.latitude.mean()      # the average latitude of u's tasks\n","metadata":{"execution":{"iopub.status.busy":"2023-06-26T08:01:45.784874Z","iopub.execute_input":"2023-06-26T08:01:45.785522Z","iopub.status.idle":"2023-06-26T08:01:50.068154Z","shell.execute_reply.started":"2023-06-26T08:01:45.785486Z","shell.execute_reply":"2023-06-26T08:01:50.066950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Encode categorigal feature, user_location","metadata":{}},{"cell_type":"code","source":"location_dict = dict([(addr, i) for i, addr in enumerate(data.user_location.unique())])\ndata['user_location'] = data.apply(lambda x: location_dict[x['user_location']], axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-06-26T08:01:50.069857Z","iopub.execute_input":"2023-06-26T08:01:50.070274Z","iopub.status.idle":"2023-06-26T08:01:58.092677Z","shell.execute_reply.started":"2023-06-26T08:01:50.070236Z","shell.execute_reply":"2023-06-26T08:01:58.091639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Then, we can add the location attributes to each node.\nfor u in nodes:\n    d = data[data.user_id == u]\n    node_dict[u]['user_location'] = d.user_location.mode()[0]","metadata":{"execution":{"iopub.status.busy":"2023-06-26T08:01:58.093935Z","iopub.execute_input":"2023-06-26T08:01:58.094338Z","iopub.status.idle":"2023-06-26T08:02:01.073509Z","shell.execute_reply.started":"2023-06-26T08:01:58.094304Z","shell.execute_reply":"2023-06-26T08:02:01.072551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create a networkx graph and add nodes.\ngraph = nx.Graph()\nfor user, attr in node_dict.items():\n    graph.add_node(user,  \n                   org_times = attr['org_times'], \n                   join_times = attr['times_of_participation'], \n                   loc = attr['user_location'],\n                   longitude = attr['task_longitude'],\n                   latitude = attr['task_latitude']\n                  )","metadata":{"execution":{"iopub.status.busy":"2023-06-26T08:02:01.074821Z","iopub.execute_input":"2023-06-26T08:02:01.075264Z","iopub.status.idle":"2023-06-26T08:02:01.090083Z","shell.execute_reply.started":"2023-06-26T08:02:01.075232Z","shell.execute_reply":"2023-06-26T08:02:01.089194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"graph","metadata":{"execution":{"iopub.status.busy":"2023-06-26T08:02:01.091540Z","iopub.execute_input":"2023-06-26T08:02:01.091886Z","iopub.status.idle":"2023-06-26T08:02:01.104223Z","shell.execute_reply.started":"2023-06-26T08:02:01.091856Z","shell.execute_reply":"2023-06-26T08:02:01.103009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Add edges to the graph","metadata":{}},{"cell_type":"code","source":"edge_dict = {}\n\nfor t in data.task_id.unique():\n    task = data[data.task_id==t]\n    users = list(task.user_id.unique())\n    \n    # for each pairs of participants, create an edge\n    for i in range(len(users)):\n        for j in range(i+1, len(users)):\n            u, v = users[i], users[j]\n            if (u, v) not in edge_dict and (v, u) not in edge_dict:\n                edge_dict[(u, v)] = 1\n            elif (u, v) in edge_dict:\n                edge_dict[(u, v)] += 1\n            else:\n                edge_dict[(v, u)] += 1\n# add edges to the networkx graph\n\nfor e, w in edge_dict.items():\n    graph.add_edge(e[0], e[1], weight=w)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-26T08:02:01.105764Z","iopub.execute_input":"2023-06-26T08:02:01.106345Z","iopub.status.idle":"2023-06-26T08:06:03.529477Z","shell.execute_reply.started":"2023-06-26T08:02:01.106309Z","shell.execute_reply":"2023-06-26T08:06:03.528476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"graph.number_of_nodes(), graph.number_of_edges()","metadata":{"execution":{"iopub.status.busy":"2023-06-26T08:06:03.531010Z","iopub.execute_input":"2023-06-26T08:06:03.531446Z","iopub.status.idle":"2023-06-26T08:06:03.540114Z","shell.execute_reply.started":"2023-06-26T08:06:03.531413Z","shell.execute_reply":"2023-06-26T08:06:03.539051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Train the model","metadata":{}},{"cell_type":"code","source":"from networkx.algorithms.link_prediction import jaccard_coefficient\n\njaccard = jaccard_coefficient(graph)","metadata":{"execution":{"iopub.status.busy":"2023-06-26T08:06:03.541708Z","iopub.execute_input":"2023-06-26T08:06:03.542410Z","iopub.status.idle":"2023-06-26T08:06:03.553535Z","shell.execute_reply.started":"2023-06-26T08:06:03.542375Z","shell.execute_reply":"2023-06-26T08:06:03.552582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Make a prediction","metadata":{}},{"cell_type":"code","source":"prediction = []\nfor link in candidates.Id:\n    u, v, t = link.split('-')\n    score = jaccard_coefficient(graph, [(int(u), int(v))])\n    prediction.append([p for i,j,p in score][0])\nprediction","metadata":{"execution":{"iopub.status.busy":"2023-06-26T08:06:03.555015Z","iopub.execute_input":"2023-06-26T08:06:03.555429Z","iopub.status.idle":"2023-06-26T08:06:06.748899Z","shell.execute_reply.started":"2023-06-26T08:06:03.555382Z","shell.execute_reply":"2023-06-26T08:06:06.747970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.displot(prediction,kde=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-26T08:06:06.750177Z","iopub.execute_input":"2023-06-26T08:06:06.751294Z","iopub.status.idle":"2023-06-26T08:06:07.463012Z","shell.execute_reply.started":"2023-06-26T08:06:06.751256Z","shell.execute_reply":"2023-06-26T08:06:07.462046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Prepare submission","metadata":{}},{"cell_type":"code","source":"submission['Predicted'] = prediction\nsubmission.to_csv('submission.csv', index=False)\nsubmission = pd.read_csv('submission.csv')\nsubmission","metadata":{"execution":{"iopub.status.busy":"2023-06-26T08:06:07.464886Z","iopub.execute_input":"2023-06-26T08:06:07.465868Z","iopub.status.idle":"2023-06-26T08:06:07.648749Z","shell.execute_reply.started":"2023-06-26T08:06:07.465836Z","shell.execute_reply":"2023-06-26T08:06:07.647844Z"},"trusted":true},"execution_count":null,"outputs":[]}]}