{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"

Social network Graph Link Prediction - Facebook Challenge

","metadata":{"id":"_ngfVeOrX7If"}},{"cell_type":"markdown","source":"### Problem statement: \nGiven a directed social graph, have to predict missing links to recommend users (Link Prediction in graph)","metadata":{"id":"BIdTcywoX7Ih"}},{"cell_type":"markdown","source":"### Data Overview\nTaken data from facebook's recruting challenge on kaggle https://www.kaggle.com/c/FacebookRecruiting \ndata contains two columns source and destination eac edge in graph \n - Data columns (total 2 columns): \n - source_node int64 \n - destination_node int64 ","metadata":{"id":"EKXSOWj3X7Ii"}},{"cell_type":"markdown","source":"### Mapping the problem into supervised learning problem:\n- Generated training samples of good and bad links from given directed graph and for each link got some features like no of followers, is he followed back, page rank, katz score, adar index, some svd fetures of adj matrix, some weight features etc. and trained ml model based on these features to predict link. \n- Some reference papers and videos : \n - https://www.cs.cornell.edu/home/kleinber/link-pred.pdf\n - https://www3.nd.edu/~dial/publications/lichtenwalter2010new.pdf\n - https://kaggle2.blob.core.windows.net/forum-message-attachments/2594/supervised_link_prediction.pdf\n - https://www.youtube.com/watch?v=2M77Hgy17cg","metadata":{"id":"frIBf5F2X7Ij"}},{"cell_type":"markdown","source":"### Business objectives and constraints: \n- No low-latency requirement.\n- Probability of prediction is useful to recommend ighest probability links","metadata":{"id":"zej3Dqi7X7Ik"}},{"cell_type":"markdown","source":"### Performance metric for supervised learning: \n- Both precision and recall is important so F1 score is good choice\n- Confusion matrix","metadata":{"id":"fSW6zbsHX7Il"}},{"cell_type":"code","source":"#Importing Libraries\n# please do go through this python notebook: \nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport csv\nimport pandas as pd#pandas to create small dataframes \nimport datetime #Convert to unix time\nimport time #Convert to unix time\n# if numpy is not installed already : pip3 install numpy\nimport numpy as np#Do aritmetic operations on arrays\n# matplotlib: used to plot graphs\nimport matplotlib\nimport matplotlib.pylab as plt\nimport seaborn as sns#Plots\nfrom matplotlib import rcParams#Size of plots \nfrom sklearn.cluster import MiniBatchKMeans, KMeans#Clustering\nimport math\nimport pickle\nimport os\n# to install xgboost: pip3 install xgboost\nimport xgboost as xgb\n\nimport warnings\nimport networkx as nx\nimport pdb\nimport pickle\nfrom pandas import HDFStore,DataFrame\nfrom pandas import read_hdf\nfrom scipy.sparse.linalg import svds, eigs\nimport gc\nfrom tqdm import tqdm","metadata":{"id":"jGe4BKx9X7Im","execution":{"iopub.status.busy":"2022-08-05T13:30:45.602849Z","iopub.execute_input":"2022-08-05T13:30:45.603359Z","iopub.status.idle":"2022-08-05T13:30:48.362065Z","shell.execute_reply.started":"2022-08-05T13:30:45.603326Z","shell.execute_reply":"2022-08-05T13:30:48.360984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PATH_TO_AFTER_EDA = '../input/facebook-data/data/after_eda'\nPATH_TO_FEATURES = '../input/facebook-data/data/fea_sample'\nPATH_TO_DATA = '../input/facebook-data/data/'","metadata":{"execution":{"iopub.status.busy":"2022-08-05T13:30:48.363711Z","iopub.execute_input":"2022-08-05T13:30:48.364Z","iopub.status.idle":"2022-08-05T13:30:48.367738Z","shell.execute_reply.started":"2022-08-05T13:30:48.363972Z","shell.execute_reply":"2022-08-05T13:30:48.366908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#reading graph\nif not os.path.isfile(PATH_TO_AFTER_EDA + '/train_woheader.csv'):\n traincsv = pd.read_csv('../input/FacebookRecruiting/train.csv')\n print(traincsv[traincsv.isna().any(1)])\n print(traincsv.info())\n print(\"Number of diplicate entries: \",sum(traincsv.duplicated()))\n traincsv.to_csv('./train_woheader.csv',header=False,index=False)\n print(\"saved the graph into file\")\nelse:\n g=nx.read_edgelist( PATH_TO_AFTER_EDA + '/train_woheader.csv',delimiter=',',create_using=nx.DiGraph(),nodetype=int)\n print(nx.info(g))","metadata":{"id":"OOi4aONAX7Iv","outputId":"b413d19f-8322-43df-c9da-d88a60d5fe1c","execution":{"iopub.status.busy":"2022-08-05T13:30:48.369081Z","iopub.execute_input":"2022-08-05T13:30:48.369469Z","iopub.status.idle":"2022-08-05T13:31:59.738185Z","shell.execute_reply.started":"2022-08-05T13:30:48.36943Z","shell.execute_reply":"2022-08-05T13:31:59.737236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> Displaying a sub graph","metadata":{"id":"ba1XVmouX7I2"}},{"cell_type":"code","source":"if not os.path.isfile('../input/linkprediction/train_woheader_sample.csv'):\n pd.read_csv('../input/FacebookRecruiting/train.csv', nrows=50).to_csv('./train_woheader_sample.csv',header=False,index=False)\n \nsubgraph=nx.read_edgelist('../input/linkprediction/train_woheader_sample.csv',delimiter=',',create_using=nx.DiGraph(),nodetype=int)\n# https://stackoverflow.com/questions/9402255/drawing-a-huge-graph-with-networkx-and-matplotlib\n\npos=nx.spring_layout(subgraph)\nnx.draw(subgraph,pos,node_color='#A0CBE2',edge_color='#00bb5e',width=1,edge_cmap=plt.cm.Blues,with_labels=True)\nplt.savefig(\"graph_sample.pdf\")\nprint(nx.info(subgraph))","metadata":{"id":"rvhGErEXX7I3","outputId":"3211434c-3826-4d98-af7e-949329c2906a","execution":{"iopub.status.busy":"2022-08-05T13:31:59.739383Z","iopub.execute_input":"2022-08-05T13:31:59.739693Z","iopub.status.idle":"2022-08-05T13:32:00.572537Z","shell.execute_reply.started":"2022-08-05T13:31:59.739659Z","shell.execute_reply":"2022-08-05T13:32:00.571756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. Exploratory Data Analysis","metadata":{"id":"QirDnZ-FX7I8"}},{"cell_type":"code","source":"# No of Unique persons \nprint(\"The number of unique persons\",len(g.nodes()))","metadata":{"id":"gdlTIj97X7I9","outputId":"2e956d78-2d25-45be-e390-fa64bd98225c","execution":{"iopub.status.busy":"2022-08-05T13:32:00.575168Z","iopub.execute_input":"2022-08-05T13:32:00.575732Z","iopub.status.idle":"2022-08-05T13:32:00.580986Z","shell.execute_reply.started":"2022-08-05T13:32:00.575676Z","shell.execute_reply":"2022-08-05T13:32:00.580085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1.1 No of followers for each person","metadata":{"id":"BdvJPYMHX7JC"}},{"cell_type":"code","source":"indegree_dist = list(dict(g.in_degree()).values())\nindegree_dist.sort()\nplt.figure(figsize=(10,6))\nplt.plot(indegree_dist)\nplt.xlabel('Index No')\nplt.ylabel('No Of Followers')\nplt.show()","metadata":{"id":"4fFEbqiSX7JD","outputId":"d4aab961-ab7b-44c8-a6b0-de6c9ed04691","execution":{"iopub.status.busy":"2022-08-05T13:32:00.583486Z","iopub.execute_input":"2022-08-05T13:32:00.584058Z","iopub.status.idle":"2022-08-05T13:32:02.719917Z","shell.execute_reply.started":"2022-08-05T13:32:00.584024Z","shell.execute_reply":"2022-08-05T13:32:02.718732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"indegree_dist = list(dict(g.in_degree()).values())\nindegree_dist.sort()\nplt.figure(figsize=(10,6))\nplt.plot(indegree_dist[0:1500000])\nplt.xlabel('Index No')\nplt.ylabel('No Of Followers')\nplt.show()","metadata":{"id":"CZPczewbX7JL","outputId":"ef79e2c1-ccc2-4d11-915d-01bee576f0a2","execution":{"iopub.status.busy":"2022-08-05T13:32:02.721492Z","iopub.execute_input":"2022-08-05T13:32:02.72184Z","iopub.status.idle":"2022-08-05T13:32:04.772143Z","shell.execute_reply.started":"2022-08-05T13:32:02.721807Z","shell.execute_reply":"2022-08-05T13:32:04.771353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.boxplot(indegree_dist)\nplt.ylabel('No Of Followers')\nplt.show()","metadata":{"id":"Wqf9qkr_X7JS","outputId":"6e880aa0-6fe3-4d22-e813-85141dab0678","execution":{"iopub.status.busy":"2022-08-05T13:32:04.773334Z","iopub.execute_input":"2022-08-05T13:32:04.773817Z","iopub.status.idle":"2022-08-05T13:32:10.467701Z","shell.execute_reply.started":"2022-08-05T13:32:04.773765Z","shell.execute_reply":"2022-08-05T13:32:10.465881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### 90-100 percentile\nfor i in range(0,11):\n print(90+i,'percentile value is',np.percentile(indegree_dist,90+i))","metadata":{"id":"5q9wSyvGX7JY","outputId":"54a2deaa-d1f0-487d-d6ab-84d1645be6ed","execution":{"iopub.status.busy":"2022-08-05T13:32:10.469109Z","iopub.execute_input":"2022-08-05T13:32:10.46941Z","iopub.status.idle":"2022-08-05T13:32:12.12703Z","shell.execute_reply.started":"2022-08-05T13:32:10.469381Z","shell.execute_reply":"2022-08-05T13:32:12.125922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"99% of data having followers of 40 only.","metadata":{"id":"9HvreJYlX7Jf"}},{"cell_type":"code","source":"### 99-100 percentile\nfor i in range(10,110,10):\n print(99+(i/100),'percentile value is',np.percentile(indegree_dist,99+(i/100)))","metadata":{"id":"mt8DA-MmX7Jg","outputId":"0ac13b00-ca01-4baf-cb93-fc50efc2959b","execution":{"iopub.status.busy":"2022-08-05T13:32:12.129246Z","iopub.execute_input":"2022-08-05T13:32:12.129566Z","iopub.status.idle":"2022-08-05T13:32:13.614839Z","shell.execute_reply.started":"2022-08-05T13:32:12.129525Z","shell.execute_reply":"2022-08-05T13:32:13.613915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%matplotlib inline\nsns.set_style('ticks')\nfig, ax = plt.subplots()\nfig.set_size_inches(11.7, 8.27)\nsns.distplot(indegree_dist, color='#16A085')\nplt.xlabel('PDF of Indegree')\nsns.despine()\n#plt.show()","metadata":{"id":"hZQed_RqX7Jk","outputId":"bb5921e4-c9f5-48b8-c0d7-963d86cc9f99","execution":{"iopub.status.busy":"2022-08-05T13:32:13.616997Z","iopub.execute_input":"2022-08-05T13:32:13.617461Z","iopub.status.idle":"2022-08-05T13:32:14.303502Z","shell.execute_reply.started":"2022-08-05T13:32:13.617416Z","shell.execute_reply":"2022-08-05T13:32:14.302512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1.2 No of people each person is following","metadata":{"id":"F_rfNRYCX7Jx"}},{"cell_type":"code","source":"outdegree_dist = list(dict(g.out_degree()).values())\noutdegree_dist.sort()\nplt.figure(figsize=(10,6))\nplt.plot(outdegree_dist)\nplt.xlabel('Index No')\nplt.ylabel('No Of people each person is following')\nplt.show()","metadata":{"id":"L-2plpMuX7Jy","outputId":"1355b363-1079-44e5-d062-ee352036d126","execution":{"iopub.status.busy":"2022-08-05T13:32:14.30516Z","iopub.execute_input":"2022-08-05T13:32:14.305581Z","iopub.status.idle":"2022-08-05T13:32:16.316958Z","shell.execute_reply.started":"2022-08-05T13:32:14.305538Z","shell.execute_reply":"2022-08-05T13:32:16.315753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"outdegree_dist = list(dict(g.out_degree()).values())\noutdegree_dist.sort()\nplt.figure(figsize=(10,6))\nplt.plot(outdegree_dist[0:1500000])\nplt.xlabel('Index No')\nplt.ylabel('No Of people each person is following')\nplt.show()","metadata":{"id":"aNaQeE5cX7J2","outputId":"c59a155f-c894-4600-9daf-35a45f7c8d4d","execution":{"iopub.status.busy":"2022-08-05T13:32:16.318843Z","iopub.execute_input":"2022-08-05T13:32:16.319651Z","iopub.status.idle":"2022-08-05T13:32:18.309546Z","shell.execute_reply.started":"2022-08-05T13:32:16.319581Z","shell.execute_reply":"2022-08-05T13:32:18.308375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.boxplot(outdegree_dist)\nplt.ylabel('No Of people each person is following')\nplt.show()","metadata":{"id":"HVYdHvLAX7J7","outputId":"da8978d6-76fe-400c-ff7d-ff60d2303efb","execution":{"iopub.status.busy":"2022-08-05T13:32:18.310714Z","iopub.execute_input":"2022-08-05T13:32:18.310999Z","iopub.status.idle":"2022-08-05T13:32:18.76416Z","shell.execute_reply.started":"2022-08-05T13:32:18.310972Z","shell.execute_reply":"2022-08-05T13:32:18.763267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### 90-100 percentile\nfor i in range(0,11):\n print(90+i,'percentile value is',np.percentile(outdegree_dist,90+i))","metadata":{"id":"FVGmz09xX7KB","outputId":"e2058803-7251-4836-df99-28075908ae99","execution":{"iopub.status.busy":"2022-08-05T13:32:18.765499Z","iopub.execute_input":"2022-08-05T13:32:18.765823Z","iopub.status.idle":"2022-08-05T13:32:20.324741Z","shell.execute_reply.started":"2022-08-05T13:32:18.765794Z","shell.execute_reply":"2022-08-05T13:32:20.32375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### 99-100 percentile\nfor i in range(10,110,10):\n print(99+(i/100),'percentile value is',np.percentile(outdegree_dist,99+(i/100)))","metadata":{"id":"zcNoTvuzX7KF","outputId":"b74b3bc9-146e-4106-b3df-af5f2a8938a0","execution":{"iopub.status.busy":"2022-08-05T13:32:20.326502Z","iopub.execute_input":"2022-08-05T13:32:20.326953Z","iopub.status.idle":"2022-08-05T13:32:21.786882Z","shell.execute_reply.started":"2022-08-05T13:32:20.32691Z","shell.execute_reply":"2022-08-05T13:32:21.785976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set_style('ticks')\nfig, ax = plt.subplots()\nfig.set_size_inches(11.7, 8.27)\nsns.distplot(outdegree_dist, color='#16A085')\nplt.xlabel('PDF of Outdegree')\nsns.despine()","metadata":{"id":"27t51jOjX7KK","outputId":"9655f989-f453-47db-be7b-732351ee3db3","execution":{"iopub.status.busy":"2022-08-05T13:32:21.787831Z","iopub.execute_input":"2022-08-05T13:32:21.788085Z","iopub.status.idle":"2022-08-05T13:32:22.442039Z","shell.execute_reply.started":"2022-08-05T13:32:21.78806Z","shell.execute_reply":"2022-08-05T13:32:22.441103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('No of persons those are not following anyone are' ,sum(np.array(outdegree_dist)==0),'and % is',\n sum(np.array(outdegree_dist)==0)*100/len(outdegree_dist) )","metadata":{"id":"9LOGY_CyX7KT","outputId":"eb5a2706-7829-4f99-c46d-84b0f7d03328","execution":{"iopub.status.busy":"2022-08-05T13:32:22.443084Z","iopub.execute_input":"2022-08-05T13:32:22.443389Z","iopub.status.idle":"2022-08-05T13:32:31.467094Z","shell.execute_reply.started":"2022-08-05T13:32:22.443361Z","shell.execute_reply":"2022-08-05T13:32:31.466103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('No of persons having zero followers are' ,sum(np.array(indegree_dist)==0),'and % is',\n sum(np.array(indegree_dist)==0)*100/len(indegree_dist) )","metadata":{"id":"hi30e3RmX7KX","outputId":"f2feffac-585d-42ed-937f-5b61e1023040","execution":{"iopub.status.busy":"2022-08-05T13:32:31.468276Z","iopub.execute_input":"2022-08-05T13:32:31.468566Z","iopub.status.idle":"2022-08-05T13:32:40.739435Z","shell.execute_reply.started":"2022-08-05T13:32:31.468538Z","shell.execute_reply":"2022-08-05T13:32:40.738521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count=0\nfor i in g.nodes():\n if len(list(g.predecessors(i)))==0 :\n if len(list(g.successors(i)))==0:\n count+=1\nprint('No of persons those are not not following anyone and also not having any followers are',count)","metadata":{"id":"rzX8vbh5X7Kc","outputId":"0fa6651f-4449-478f-a480-e289b9706afd","execution":{"iopub.status.busy":"2022-08-05T13:32:40.741722Z","iopub.execute_input":"2022-08-05T13:32:40.74213Z","iopub.status.idle":"2022-08-05T13:32:44.148309Z","shell.execute_reply.started":"2022-08-05T13:32:40.742087Z","shell.execute_reply":"2022-08-05T13:32:44.147356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1.3 both followers + following ","metadata":{"id":"J7tiBqS3X7Kh"}},{"cell_type":"code","source":"from collections import Counter\ndict_in = dict(g.in_degree())\ndict_out = dict(g.out_degree())\nd = Counter(dict_in) + Counter(dict_out)\nin_out_degree = np.array(list(d.values()))","metadata":{"id":"MNu7k3-FX7Kj","execution":{"iopub.status.busy":"2022-08-05T13:32:44.149864Z","iopub.execute_input":"2022-08-05T13:32:44.150251Z","iopub.status.idle":"2022-08-05T13:32:49.095086Z","shell.execute_reply.started":"2022-08-05T13:32:44.150215Z","shell.execute_reply":"2022-08-05T13:32:49.093992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"in_out_degree_sort = sorted(in_out_degree)\nplt.figure(figsize=(10,6))\nplt.plot(in_out_degree_sort)\nplt.xlabel('Index No')\nplt.ylabel('No Of people each person is following + followers')\nplt.show()","metadata":{"id":"ARCqfS9TX7Kn","outputId":"44fe1bc9-a591-4cff-89aa-e36c0f7165e5","execution":{"iopub.status.busy":"2022-08-05T13:32:49.096543Z","iopub.execute_input":"2022-08-05T13:32:49.096904Z","iopub.status.idle":"2022-08-05T13:32:50.738398Z","shell.execute_reply.started":"2022-08-05T13:32:49.096873Z","shell.execute_reply":"2022-08-05T13:32:50.737437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"in_out_degree_sort = sorted(in_out_degree)\nplt.figure(figsize=(10,6))\nplt.plot(in_out_degree_sort[0:1500000])\nplt.xlabel('Index No')\nplt.ylabel('No Of people each person is following + followers')\nplt.show()","metadata":{"id":"jVlgsM8hX7Ku","outputId":"28758bec-8303-4a93-f4a9-b637cb14efe0","execution":{"iopub.status.busy":"2022-08-05T13:32:50.739735Z","iopub.execute_input":"2022-08-05T13:32:50.740042Z","iopub.status.idle":"2022-08-05T13:32:52.4227Z","shell.execute_reply.started":"2022-08-05T13:32:50.740012Z","shell.execute_reply":"2022-08-05T13:32:52.421741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### 90-100 percentile\nfor i in range(0,11):\n print(90+i,'percentile value is',np.percentile(in_out_degree_sort,90+i))","metadata":{"id":"4YpW20YEX7Kz","outputId":"ba104e55-fae4-4a81-b2b6-887c804e6c6a","execution":{"iopub.status.busy":"2022-08-05T13:32:52.42413Z","iopub.execute_input":"2022-08-05T13:32:52.424556Z","iopub.status.idle":"2022-08-05T13:32:56.722939Z","shell.execute_reply.started":"2022-08-05T13:32:52.424508Z","shell.execute_reply":"2022-08-05T13:32:56.721755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### 99-100 percentile\nfor i in range(10,110,10):\n print(99+(i/100),'percentile value is',np.percentile(in_out_degree_sort,99+(i/100)))","metadata":{"id":"LclmrhGbX7K3","outputId":"43440486-b332-476b-a769-43e43e23600d","execution":{"iopub.status.busy":"2022-08-05T13:32:56.724511Z","iopub.execute_input":"2022-08-05T13:32:56.725106Z","iopub.status.idle":"2022-08-05T13:33:00.680573Z","shell.execute_reply.started":"2022-08-05T13:32:56.725041Z","shell.execute_reply":"2022-08-05T13:33:00.679748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Min of no of followers + following is',in_out_degree.min())\nprint(np.sum(in_out_degree==in_out_degree.min()),' persons having minimum no of followers + following')","metadata":{"id":"XqErF9SRX7K8","outputId":"11746c24-4d22-4466-f2bf-8dae306ee408","execution":{"iopub.status.busy":"2022-08-05T13:33:00.681673Z","iopub.execute_input":"2022-08-05T13:33:00.68196Z","iopub.status.idle":"2022-08-05T13:33:00.694938Z","shell.execute_reply.started":"2022-08-05T13:33:00.681932Z","shell.execute_reply":"2022-08-05T13:33:00.69397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Max of no of followers + following is',in_out_degree.max())\nprint(np.sum(in_out_degree==in_out_degree.max()),' persons having maximum no of followers + following')","metadata":{"id":"4rjy0p8-X7LA","outputId":"e3d20325-27e8-44f3-f0ea-18aa395e687c","execution":{"iopub.status.busy":"2022-08-05T13:33:00.696198Z","iopub.execute_input":"2022-08-05T13:33:00.696495Z","iopub.status.idle":"2022-08-05T13:33:00.709632Z","shell.execute_reply.started":"2022-08-05T13:33:00.696465Z","shell.execute_reply":"2022-08-05T13:33:00.708711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('No of persons having followers + following less than 10 are',np.sum(in_out_degree<10))","metadata":{"id":"2vkQHuVgX7LE","outputId":"c4bd961f-a5a9-45b7-d700-f8f368d6772d","execution":{"iopub.status.busy":"2022-08-05T13:33:00.711376Z","iopub.execute_input":"2022-08-05T13:33:00.711711Z","iopub.status.idle":"2022-08-05T13:33:00.719533Z","shell.execute_reply.started":"2022-08-05T13:33:00.711679Z","shell.execute_reply":"2022-08-05T13:33:00.718749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('No of weakly connected components',len(list(nx.weakly_connected_components(g))))\ncount=0\nfor i in list(nx.weakly_connected_components(g)):\n if len(i)==2:\n count+=1\nprint('weakly connected components wit 2 nodes',count)","metadata":{"id":"9YXunWI-X7LI","outputId":"00c2e979-fcfc-412f-8a02-26bac9c912ff","execution":{"iopub.status.busy":"2022-08-05T13:33:00.721062Z","iopub.execute_input":"2022-08-05T13:33:00.721359Z","iopub.status.idle":"2022-08-05T13:33:29.907442Z","shell.execute_reply.started":"2022-08-05T13:33:00.721329Z","shell.execute_reply":"2022-08-05T13:33:29.906563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Posing a problem as classification problem ","metadata":{"id":"RiPVIlFUX7LQ"}},{"cell_type":"markdown","source":"## 2.1 Generating some edges which are not present in graph for supervised learning \nGenerated Bad links from graph which are not in graph and whose shortest path is greater than 2. ","metadata":{"id":"TpDCmJroX7LS"}},{"cell_type":"code","source":"%%time\n###generating bad edges from given graph\nimport random\nif not os.path.isfile(PATH_TO_AFTER_EDA + '/missing_edges_final.p'):\n #getting all set of edges\n r = csv.reader(open(PATH_TO_AFTER_EDA + '/train_woheader.csv','r'))\n edges = dict()\n for edge in r:\n edges[(edge[0], edge[1])] = 1\n \n \n missing_edges = set([])\n while (len(missing_edges)<9437519):\n a=random.randint(1, 1862220)\n b=random.randint(1, 1862220)\n tmp = edges.get((a,b),-1)\n if tmp == -1 and a!=b:\n try:\n if nx.shortest_path_length(g,source=a,target=b) > 2: \n\n missing_edges.add((a,b))\n else:\n continue \n except: \n missing_edges.add((a,b)) \n else:\n continue\n pickle.dump(missing_edges,open('./missing_edges_final.p','wb'))\nelse:\n missing_edges = pickle.load(open(PATH_TO_AFTER_EDA + '/missing_edges_final.p','rb'))","metadata":{"id":"3K2YQD6ZX7LT","outputId":"03656e72-53e4-4239-8227-c0ffe9f535d1","execution":{"iopub.status.busy":"2022-08-05T13:33:29.908844Z","iopub.execute_input":"2022-08-05T13:33:29.909384Z","iopub.status.idle":"2022-08-05T13:33:35.307831Z","shell.execute_reply.started":"2022-08-05T13:33:29.909349Z","shell.execute_reply":"2022-08-05T13:33:35.306884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(missing_edges)","metadata":{"id":"vrjL2UAmX7LX","outputId":"8596ad2d-6c45-4c14-e377-3c06850b661d","execution":{"iopub.status.busy":"2022-08-05T13:33:35.309259Z","iopub.execute_input":"2022-08-05T13:33:35.309914Z","iopub.status.idle":"2022-08-05T13:33:35.316353Z","shell.execute_reply.started":"2022-08-05T13:33:35.309868Z","shell.execute_reply":"2022-08-05T13:33:35.315247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2.2 Training and Test data split: \nRemoved edges from Graph and used as test data and after removing used that graph for creating features for Train and test data","metadata":{"id":"LA2GGc3lX7Lb"}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nif (not os.path.isfile(PATH_TO_AFTER_EDA + '/train_pos_after_eda.csv')) and (not os.path.isfile(PATH_TO_AFTER_EDA+'/test_pos_after_eda.csv')):\n #reading total data df\n df_pos = pd.read_csv('../input/FacebookRecruiting/train.csv')\n df_neg = pd.DataFrame(list(missing_edges), columns=['source_node', 'destination_node'])\n \n print(\"Number of nodes in the graph with edges\", df_pos.shape[0])\n print(\"Number of nodes in the graph without edges\", df_neg.shape[0])\n \n #Trian test split \n #Spiltted data into 80-20 \n #positive links and negative links seperatly because we need positive training data only for creating graph \n #and for feature generation\n X_train_pos, X_test_pos, y_train_pos, y_test_pos = train_test_split(df_pos,np.ones(len(df_pos)),test_size=0.2, random_state=9)\n X_train_neg, X_test_neg, y_train_neg, y_test_neg = train_test_split(df_neg,np.zeros(len(df_neg)),test_size=0.2, random_state=9)\n \n print('='*60)\n print(\"Number of nodes in the train data graph with edges\", X_train_pos.shape[0],\"=\",y_train_pos.shape[0])\n print(\"Number of nodes in the train data graph without edges\", X_train_neg.shape[0],\"=\", y_train_neg.shape[0])\n print('='*60)\n print(\"Number of nodes in the test data graph with edges\", X_test_pos.shape[0],\"=\",y_test_pos.shape[0])\n print(\"Number of nodes in the test data graph without edges\", X_test_neg.shape[0],\"=\",y_test_neg.shape[0])\n\n #removing header and saving\n X_train_pos.to_csv('./train_pos_after_eda.csv',header=False, index=False)\n X_test_pos.to_csv('./test_pos_after_eda.csv',header=False, index=False)\n X_train_neg.to_csv('./train_neg_after_eda.csv',header=False, index=False)\n X_test_neg.to_csv('./test_neg_after_eda.csv',header=False, index=False)\nelse:\n #Graph from Traing data only \n del missing_edges","metadata":{"id":"hmcOk-nkX7Lb","outputId":"891d417c-069b-4236-e3ba-35b2bbfc252d","execution":{"iopub.status.busy":"2022-08-05T13:33:35.326467Z","iopub.execute_input":"2022-08-05T13:33:35.327093Z","iopub.status.idle":"2022-08-05T13:33:36.324844Z","shell.execute_reply.started":"2022-08-05T13:33:35.327055Z","shell.execute_reply":"2022-08-05T13:33:36.324076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if (os.path.isfile(PATH_TO_AFTER_EDA+'/train_pos_after_eda.csv')) and (os.path.isfile(PATH_TO_AFTER_EDA+'/test_pos_after_eda.csv')): \n train_graph=nx.read_edgelist(PATH_TO_AFTER_EDA+'/train_pos_after_eda.csv',delimiter=',',create_using=nx.DiGraph(),nodetype=int)\n test_graph=nx.read_edgelist(PATH_TO_AFTER_EDA+'/test_pos_after_eda.csv',delimiter=',',create_using=nx.DiGraph(),nodetype=int)\n print(nx.info(train_graph))\n print(nx.info(test_graph))\n\n # finding the unique nodes in the both train and test graphs\n train_nodes_pos = set(train_graph.nodes())\n test_nodes_pos = set(test_graph.nodes())\n\n trY_teY = len(train_nodes_pos.intersection(test_nodes_pos))\n trY_teN = len(train_nodes_pos - test_nodes_pos)\n teY_trN = len(test_nodes_pos - train_nodes_pos)\n\n print('no of people common in train and test -- ',trY_teY)\n print('no of people present in train but not present in test -- ',trY_teN)\n\n print('no of people present in test but not present in train -- ',teY_trN)\n print(' % of people not there in Train but exist in Test in total Test data are {} %'.format(teY_trN/len(test_nodes_pos)*100))","metadata":{"id":"Wj9OwIX6X7Lh","outputId":"8a11e605-e133-4b3d-c2d2-585021be9cdc","execution":{"iopub.status.busy":"2022-08-05T13:33:36.326387Z","iopub.execute_input":"2022-08-05T13:33:36.326679Z","iopub.status.idle":"2022-08-05T13:35:07.675053Z","shell.execute_reply.started":"2022-08-05T13:33:36.326641Z","shell.execute_reply":"2022-08-05T13:35:07.674194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> we have a cold start problem here","metadata":{"id":"OhpyrSuiX7Ll"}},{"cell_type":"code","source":"#final train and test data sets\nif (not os.path.isfile(PATH_TO_AFTER_EDA+'/train_after_eda.csv')) and \\\n(not os.path.isfile(PATH_TO_AFTER_EDA+'/test_after_eda.csv')) and \\\n(not os.path.isfile(PATH_TO_FEATURES+'/train_y.csv')) and \\\n(not os.path.isfile(PATH_TO_FEATURES+'/test_y.csv')) and \\\n(os.path.isfile(PATH_TO_AFTER_EDA+'/train_pos_after_eda.csv')) and \\\n(os.path.isfile(PATH_TO_AFTER_EDA+'/test_pos_after_eda.csv')) and \\\n(os.path.isfile(PATH_TO_AFTER_EDA+'/train_neg_after_eda.csv')) and \\\n(os.path.isfile(PATH_TO_AFTER_EDA+'/test_neg_after_eda.csv')):\n \n X_train_pos = pd.read_csv('../input/linkprediction/train_pos_after_eda.csv', names=['source_node', 'destination_node'])\n X_test_pos = pd.read_csv('../input/linkprediction/test_pos_after_eda.csv', names=['source_node', 'destination_node'])\n X_train_neg = pd.read_csv('../input/linkprediction/train_neg_after_eda.csv', names=['source_node', 'destination_node'])\n X_test_neg = pd.read_csv('../input/linkprediction/test_neg_after_eda.csv', names=['source_node', 'destination_node'])\n\n print('='*60)\n print(\"Number of nodes in the train data graph with edges\", X_train_pos.shape[0])\n print(\"Number of nodes in the train data graph without edges\", X_train_neg.shape[0])\n print('='*60)\n print(\"Number of nodes in the test data graph with edges\", X_test_pos.shape[0])\n print(\"Number of nodes in the test data graph without edges\", X_test_neg.shape[0])\n\n X_train = X_train_pos.append(X_train_neg,ignore_index=True)\n y_train = np.concatenate((y_train_pos,y_train_neg))\n X_test = X_test_pos.append(X_test_neg,ignore_index=True)\n y_test = np.concatenate((y_test_pos,y_test_neg))\n \n print(\"Data points in train data\",X_train.shape)\n print(\"Data points in test data\",X_test.shape)\n print(\"Shape of traget variable in train\",y_train.shape)\n print(\"Shape of traget variable in test\", y_test.shape)\n \n X_train.to_csv('./train_after_eda.csv',header=False,index=False)\n X_test.to_csv('./test_after_eda.csv',header=False,index=False)\n pd.DataFrame(y_train.astype(int)).to_csv('./train_y.csv',header=False,index=False)\n pd.DataFrame(y_test.astype(int)).to_csv('./test_y.csv',header=False,index=False)","metadata":{"id":"onultUqjX7Lm","outputId":"795b2bac-18da-4540-9343-bb69f30a6f00","execution":{"iopub.status.busy":"2022-08-05T13:35:07.676323Z","iopub.execute_input":"2022-08-05T13:35:07.67684Z","iopub.status.idle":"2022-08-05T13:35:08.127837Z","shell.execute_reply.started":"2022-08-05T13:35:07.676806Z","shell.execute_reply":"2022-08-05T13:35:08.126716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. Reading Data","metadata":{"id":"lbhaIaAphtwy"}},{"cell_type":"code","source":"if os.path.isfile(PATH_TO_AFTER_EDA+'/train_pos_after_eda.csv'):\n train_graph=nx.read_edgelist(PATH_TO_AFTER_EDA+'/train_pos_after_eda.csv',delimiter=',',create_using=nx.DiGraph(),nodetype=int)\n print(nx.info(train_graph))\nelse:\n print(\"please run the FB_EDA.ipynb or download the files from drive\")","metadata":{"id":"zyedrOo9htw3","outputId":"b2aa525a-93d3-47c3-8216-416a811bc812","execution":{"iopub.status.busy":"2022-08-05T13:35:08.12929Z","iopub.execute_input":"2022-08-05T13:35:08.129705Z","iopub.status.idle":"2022-08-05T13:36:34.137482Z","shell.execute_reply.started":"2022-08-05T13:35:08.129664Z","shell.execute_reply":"2022-08-05T13:36:34.136519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Similarity measures","metadata":{"id":"kP6nhmL7htxH"}},{"cell_type":"markdown","source":"## 2.1 Jaccard Distance:\nhttp://www.statisticshowto.com/jaccard-index/","metadata":{"id":"B63EuoBnhtxJ"}},{"cell_type":"markdown","source":"\\begin{equation}\nj = \\frac{|X\\cap Y|}{|X \\cup Y|} \n\\end{equation}","metadata":{"id":"_4i-CnlYhtxK"}},{"cell_type":"code","source":"#for followees\ndef jaccard_for_followees(a,b):\n try:\n if len(set(train_graph.successors(a))) == 0 | len(set(train_graph.successors(b))) == 0:\n return 0\n sim = (len(set(train_graph.successors(a)).intersection(set(train_graph.successors(b)))))/\\\n (len(set(train_graph.successors(a)).union(set(train_graph.successors(b)))))\n except:\n return 0\n return sim","metadata":{"id":"dH9U3WoohtxM","execution":{"iopub.status.busy":"2022-08-05T13:36:34.139169Z","iopub.execute_input":"2022-08-05T13:36:34.139483Z","iopub.status.idle":"2022-08-05T13:36:34.146677Z","shell.execute_reply.started":"2022-08-05T13:36:34.139453Z","shell.execute_reply":"2022-08-05T13:36:34.145649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#one test case\nprint(jaccard_for_followees(273084,1505602))","metadata":{"id":"Ir2ruUcPhtxS","outputId":"426a6833-1631-4024-c24a-d21ae7686472","execution":{"iopub.status.busy":"2022-08-05T13:36:34.147734Z","iopub.execute_input":"2022-08-05T13:36:34.148046Z","iopub.status.idle":"2022-08-05T13:36:34.163918Z","shell.execute_reply.started":"2022-08-05T13:36:34.147997Z","shell.execute_reply":"2022-08-05T13:36:34.163013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#node 1635354 not in graph \nprint(jaccard_for_followees(273084,1505602))","metadata":{"id":"l3LdM2R1htxY","outputId":"8ba07727-a0ab-498e-819f-0d310876191c","execution":{"iopub.status.busy":"2022-08-05T13:36:34.165543Z","iopub.execute_input":"2022-08-05T13:36:34.165938Z","iopub.status.idle":"2022-08-05T13:36:34.172393Z","shell.execute_reply.started":"2022-08-05T13:36:34.165905Z","shell.execute_reply":"2022-08-05T13:36:34.171449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#for followers\ndef jaccard_for_followers(a,b):\n try:\n if len(set(train_graph.predecessors(a))) == 0 | len(set(g.predecessors(b))) == 0:\n return 0\n sim = (len(set(train_graph.predecessors(a)).intersection(set(train_graph.predecessors(b)))))/\\\n (len(set(train_graph.predecessors(a)).union(set(train_graph.predecessors(b)))))\n return sim\n except:\n return 0","metadata":{"id":"CcOLcGr-htxc","execution":{"iopub.status.busy":"2022-08-05T13:36:34.173892Z","iopub.execute_input":"2022-08-05T13:36:34.174283Z","iopub.status.idle":"2022-08-05T13:36:34.183557Z","shell.execute_reply.started":"2022-08-05T13:36:34.174173Z","shell.execute_reply":"2022-08-05T13:36:34.182575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(jaccard_for_followers(273084,470294))","metadata":{"id":"I-pYr8pMhtxf","outputId":"7e4b4536-442a-4b0c-ae02-fb442c1955db","execution":{"iopub.status.busy":"2022-08-05T13:36:34.18486Z","iopub.execute_input":"2022-08-05T13:36:34.18528Z","iopub.status.idle":"2022-08-05T13:36:34.195851Z","shell.execute_reply.started":"2022-08-05T13:36:34.185234Z","shell.execute_reply":"2022-08-05T13:36:34.194883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#node 1635354 not in graph \nprint(jaccard_for_followees(669354,1635354))","metadata":{"id":"O0jOEa_1htxj","outputId":"1e12fabe-d990-4506-bb6b-c86b01d1b0af","execution":{"iopub.status.busy":"2022-08-05T13:36:34.196887Z","iopub.execute_input":"2022-08-05T13:36:34.197279Z","iopub.status.idle":"2022-08-05T13:36:34.207586Z","shell.execute_reply.started":"2022-08-05T13:36:34.197249Z","shell.execute_reply":"2022-08-05T13:36:34.206539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2.2 Cosine distance","metadata":{"id":"5ftUcDj8htxn"}},{"cell_type":"markdown","source":"\\begin{equation}\nCosineDistance = \\frac{|X\\cap Y|}{|X|\\cdot|Y|} \n\\end{equation}","metadata":{"id":"I9P5WGUthtxn"}},{"cell_type":"code","source":"#for followees\ndef cosine_for_followees(a,b):\n try:\n if len(set(train_graph.successors(a))) == 0 | len(set(train_graph.successors(b))) == 0:\n return 0\n sim = (len(set(train_graph.successors(a)).intersection(set(train_graph.successors(b)))))/\\\n (math.sqrt(len(set(train_graph.successors(a)))*len((set(train_graph.successors(b))))))\n return sim\n except:\n return 0","metadata":{"id":"SLAEXaOGhtxo","execution":{"iopub.status.busy":"2022-08-05T13:36:34.208753Z","iopub.execute_input":"2022-08-05T13:36:34.20904Z","iopub.status.idle":"2022-08-05T13:36:34.218682Z","shell.execute_reply.started":"2022-08-05T13:36:34.209013Z","shell.execute_reply":"2022-08-05T13:36:34.217758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(cosine_for_followees(273084,1505602))","metadata":{"id":"Cj-thm2Ohtxq","outputId":"531fceba-60f4-4e6b-97f4-f37733dc468f","execution":{"iopub.status.busy":"2022-08-05T13:36:34.219926Z","iopub.execute_input":"2022-08-05T13:36:34.220601Z","iopub.status.idle":"2022-08-05T13:36:34.230868Z","shell.execute_reply.started":"2022-08-05T13:36:34.220554Z","shell.execute_reply":"2022-08-05T13:36:34.229485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(cosine_for_followees(273084,1635354))","metadata":{"id":"qBwfIUeohtxs","outputId":"41202fc6-f4aa-4a1d-d8f6-84f960a3fbba","execution":{"iopub.status.busy":"2022-08-05T13:36:34.232125Z","iopub.execute_input":"2022-08-05T13:36:34.232701Z","iopub.status.idle":"2022-08-05T13:36:34.240513Z","shell.execute_reply.started":"2022-08-05T13:36:34.23265Z","shell.execute_reply":"2022-08-05T13:36:34.239538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def cosine_for_followers(a,b):\n try:\n \n if len(set(train_graph.predecessors(a))) == 0 | len(set(train_graph.predecessors(b))) == 0:\n return 0\n sim = (len(set(train_graph.predecessors(a)).intersection(set(train_graph.predecessors(b)))))/\\\n (math.sqrt(len(set(train_graph.predecessors(a))))*(len(set(train_graph.predecessors(b)))))\n return sim\n except:\n return 0","metadata":{"id":"eRiQ4__6htxu","execution":{"iopub.status.busy":"2022-08-05T13:36:34.242135Z","iopub.execute_input":"2022-08-05T13:36:34.242564Z","iopub.status.idle":"2022-08-05T13:36:34.249507Z","shell.execute_reply.started":"2022-08-05T13:36:34.242516Z","shell.execute_reply":"2022-08-05T13:36:34.248748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(cosine_for_followers(2,470294))","metadata":{"id":"WcsR0q-vhtxx","outputId":"f01e0558-f1e3-465f-ab14-0e4ca764f4aa","execution":{"iopub.status.busy":"2022-08-05T13:36:34.250698Z","iopub.execute_input":"2022-08-05T13:36:34.251169Z","iopub.status.idle":"2022-08-05T13:36:34.264822Z","shell.execute_reply.started":"2022-08-05T13:36:34.251138Z","shell.execute_reply":"2022-08-05T13:36:34.263768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(cosine_for_followers(669354,1635354))","metadata":{"id":"M527NDcihtx2","outputId":"8bc9607a-7262-43e2-9de8-f71d276762fc","execution":{"iopub.status.busy":"2022-08-05T13:36:34.266266Z","iopub.execute_input":"2022-08-05T13:36:34.266708Z","iopub.status.idle":"2022-08-05T13:36:34.276016Z","shell.execute_reply.started":"2022-08-05T13:36:34.266664Z","shell.execute_reply":"2022-08-05T13:36:34.275124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. Ranking Measures","metadata":{"id":"lJCnzo5Lhtx6"}},{"cell_type":"markdown","source":"https://networkx.github.io/documentation/networkx-1.10/reference/generated/networkx.algorithms.link_analysis.pagerank_alg.pagerank.html\n\nPageRank computes a ranking of the nodes in the graph G based on the structure of the incoming links.\n\n\n\nMathematical PageRanks for a simple network, expressed as percentages. (Google uses a logarithmic scale.) Page C has a higher PageRank than Page E, even though there are fewer links to C; the one link to C comes from an important page and hence is of high value. If web surfers who start on a random page have an 85% likelihood of choosing a random link from the page they are currently visiting, and a 15% likelihood of jumping to a page chosen at random from the entire web, they will reach Page E 8.1% of the time. (The 15% likelihood of jumping to an arbitrary page corresponds to a damping factor of 85%.) Without damping, all web surfers would eventually end up on Pages A, B, or C, and all other pages would have PageRank zero. In the presence of damping, Page A effectively links to all pages in the web, even though it has no outgoing links of its own.","metadata":{"id":"QoS0Yam1htx6"}},{"cell_type":"markdown","source":"## 3.1 Page Ranking\n\nhttps://en.wikipedia.org/wiki/PageRank\n","metadata":{"id":"og_cvL6whtx6"}},{"cell_type":"code","source":"if not os.path.isfile(PATH_TO_FEATURES + '/page_rank.p'):\n pr = nx.pagerank(train_graph, alpha=0.85)\n pickle.dump(pr,open('./page_rank.p','wb'))\nelse:\n pr = pickle.load(open(PATH_TO_FEATURES + '/page_rank.p','rb'))","metadata":{"id":"9B4uN8d_htx7","execution":{"iopub.status.busy":"2022-08-05T13:36:34.277684Z","iopub.execute_input":"2022-08-05T13:36:34.278299Z","iopub.status.idle":"2022-08-05T13:36:35.039357Z","shell.execute_reply.started":"2022-08-05T13:36:34.278219Z","shell.execute_reply":"2022-08-05T13:36:35.03819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('min',pr[min(pr, key=pr.get)])\nprint('max',pr[max(pr, key=pr.get)])\nprint('mean',float(sum(pr.values())) / len(pr))","metadata":{"id":"xMmB40obhtx9","outputId":"bb3d1b7a-81f9-44ab-dbe7-3214ccd47179","execution":{"iopub.status.busy":"2022-08-05T13:36:35.040955Z","iopub.execute_input":"2022-08-05T13:36:35.041371Z","iopub.status.idle":"2022-08-05T13:36:35.8267Z","shell.execute_reply.started":"2022-08-05T13:36:35.041325Z","shell.execute_reply":"2022-08-05T13:36:35.825121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#for imputing to nodes which are not there in Train data\nmean_pr = float(sum(pr.values())) / len(pr)\nprint(mean_pr)","metadata":{"id":"z4XqImvvhtx_","outputId":"992fdfad-7ff6-4626-c9ee-d9bce220a680","execution":{"iopub.status.busy":"2022-08-05T13:36:35.828389Z","iopub.execute_input":"2022-08-05T13:36:35.828782Z","iopub.status.idle":"2022-08-05T13:36:35.882083Z","shell.execute_reply.started":"2022-08-05T13:36:35.82873Z","shell.execute_reply":"2022-08-05T13:36:35.881213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. Other Graph Features","metadata":{"id":"VaSVkaW9htyB"}},{"cell_type":"markdown","source":"## 4.1 Shortest path:","metadata":{"id":"z7PSXbQvhtyC"}},{"cell_type":"markdown","source":"Getting Shortest path between twoo nodes, if nodes have direct path i.e directly connected then we are removing that edge and calculating path. ","metadata":{"id":"i-_n36C3htyC"}},{"cell_type":"code","source":"#if has direct edge then deleting that edge and calculating shortest path\ndef compute_shortest_path_length(a,b):\n p=-1\n try:\n if train_graph.has_edge(a,b):\n train_graph.remove_edge(a,b)\n p= nx.shortest_path_length(train_graph,source=a,target=b)\n train_graph.add_edge(a,b)\n else:\n p= nx.shortest_path_length(train_graph,source=a,target=b)\n return p\n except:\n return -1","metadata":{"id":"giLolc2vhtyC","execution":{"iopub.status.busy":"2022-08-05T13:36:35.88315Z","iopub.execute_input":"2022-08-05T13:36:35.883427Z","iopub.status.idle":"2022-08-05T13:36:35.893296Z","shell.execute_reply.started":"2022-08-05T13:36:35.8834Z","shell.execute_reply":"2022-08-05T13:36:35.892122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#testing\ncompute_shortest_path_length(77697, 826021)","metadata":{"id":"RO54pdcghtyE","outputId":"15ca223a-6a04-4549-d010-54619b472a9e","execution":{"iopub.status.busy":"2022-08-05T13:36:35.894776Z","iopub.execute_input":"2022-08-05T13:36:35.89507Z","iopub.status.idle":"2022-08-05T13:36:35.911171Z","shell.execute_reply.started":"2022-08-05T13:36:35.895044Z","shell.execute_reply":"2022-08-05T13:36:35.909882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#testing\ncompute_shortest_path_length(669354,1635354)","metadata":{"id":"yFc-7H2WhtyI","outputId":"6debfa4f-2067-48bc-84b3-ab86e2d9dea6","execution":{"iopub.status.busy":"2022-08-05T13:36:35.914088Z","iopub.execute_input":"2022-08-05T13:36:35.914528Z","iopub.status.idle":"2022-08-05T13:36:35.922522Z","shell.execute_reply.started":"2022-08-05T13:36:35.914496Z","shell.execute_reply":"2022-08-05T13:36:35.92141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4.2 Checking for same community","metadata":{"id":"AcZGaS85htyK"}},{"cell_type":"code","source":"#getting weekly connected edges from graph \nwcc=list(nx.weakly_connected_components(train_graph))\ndef belongs_to_same_wcc(a,b):\n index = []\n if train_graph.has_edge(b,a):\n return 1\n if train_graph.has_edge(a,b):\n for i in wcc:\n if a in i:\n index= i\n break\n if (b in index):\n train_graph.remove_edge(a,b)\n if compute_shortest_path_length(a,b)==-1:\n train_graph.add_edge(a,b)\n return 0\n else:\n train_graph.add_edge(a,b)\n return 1\n else:\n return 0\n else:\n for i in wcc:\n if a in i:\n index= i\n break\n if(b in index):\n return 1\n else:\n return 0","metadata":{"id":"nyudZqVnhtyK","execution":{"iopub.status.busy":"2022-08-05T13:36:35.924109Z","iopub.execute_input":"2022-08-05T13:36:35.924883Z","iopub.status.idle":"2022-08-05T13:36:50.469829Z","shell.execute_reply.started":"2022-08-05T13:36:35.924828Z","shell.execute_reply":"2022-08-05T13:36:50.468864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"belongs_to_same_wcc(861, 1659750)","metadata":{"id":"lrP0H2mKhtyM","outputId":"2b043a87-b460-42bf-f37e-4c04bbed6586","execution":{"iopub.status.busy":"2022-08-05T13:36:50.47137Z","iopub.execute_input":"2022-08-05T13:36:50.471715Z","iopub.status.idle":"2022-08-05T13:36:50.480946Z","shell.execute_reply.started":"2022-08-05T13:36:50.471684Z","shell.execute_reply":"2022-08-05T13:36:50.480022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"belongs_to_same_wcc(669354,1635354)","metadata":{"id":"b2gYnPhphtyO","outputId":"2005e22c-b60f-48d7-839b-650bf97cae35","execution":{"iopub.status.busy":"2022-08-05T13:36:50.48238Z","iopub.execute_input":"2022-08-05T13:36:50.482693Z","iopub.status.idle":"2022-08-05T13:36:50.495912Z","shell.execute_reply.started":"2022-08-05T13:36:50.482663Z","shell.execute_reply":"2022-08-05T13:36:50.495142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4.3 Adamic/Adar Index:\nAdamic/Adar measures is defined as inverted sum of degrees of common neighbours for given two vertices.\n$$A(x,y)=\\sum_{u \\in N(x) \\cap N(y)}\\frac{1}{log(|N(u)|)}$$","metadata":{"id":"ufbeSDV4htyR"}},{"cell_type":"code","source":"#adar index\ndef calc_adar_in(a,b):\n sum=0\n try:\n n=list(set(train_graph.successors(a)).intersection(set(train_graph.successors(b))))\n if len(n)!=0:\n for i in n:\n sum=sum+(1/np.log10(len(list(train_graph.predecessors(i)))))\n return sum\n else:\n return 0\n except:\n return 0","metadata":{"id":"rTXgIUzxhtyS","execution":{"iopub.status.busy":"2022-08-05T13:36:50.496955Z","iopub.execute_input":"2022-08-05T13:36:50.497361Z","iopub.status.idle":"2022-08-05T13:36:50.505786Z","shell.execute_reply.started":"2022-08-05T13:36:50.497331Z","shell.execute_reply":"2022-08-05T13:36:50.504912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"calc_adar_in(1,189226)","metadata":{"id":"QPGH6RjNhtyW","outputId":"2f9c0e11-02d9-4f28-d67a-65e3d4943e99","execution":{"iopub.status.busy":"2022-08-05T13:36:50.509378Z","iopub.execute_input":"2022-08-05T13:36:50.5097Z","iopub.status.idle":"2022-08-05T13:36:50.517892Z","shell.execute_reply.started":"2022-08-05T13:36:50.509669Z","shell.execute_reply":"2022-08-05T13:36:50.517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"calc_adar_in(669354,1635354)","metadata":{"id":"X4AInTn-htya","outputId":"68a0a099-2954-402f-c80f-6d436ffa1aba","execution":{"iopub.status.busy":"2022-08-05T13:36:50.519121Z","iopub.execute_input":"2022-08-05T13:36:50.51963Z","iopub.status.idle":"2022-08-05T13:36:50.527961Z","shell.execute_reply.started":"2022-08-05T13:36:50.519577Z","shell.execute_reply":"2022-08-05T13:36:50.527098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4.4 Is persion was following back:","metadata":{"id":"bS-wGC4hhtyc"}},{"cell_type":"code","source":"def follows_back(a,b):\n if train_graph.has_edge(b,a):\n return 1\n else:\n return 0","metadata":{"id":"2zwrTQuFhtyc","execution":{"iopub.status.busy":"2022-08-05T13:36:50.529161Z","iopub.execute_input":"2022-08-05T13:36:50.529528Z","iopub.status.idle":"2022-08-05T13:36:50.536239Z","shell.execute_reply.started":"2022-08-05T13:36:50.529498Z","shell.execute_reply":"2022-08-05T13:36:50.535176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"follows_back(1,189226)","metadata":{"id":"H589rvIvhtye","outputId":"ed3d8640-9834-4a95-e712-804292da70e9","execution":{"iopub.status.busy":"2022-08-05T13:36:50.53728Z","iopub.execute_input":"2022-08-05T13:36:50.53753Z","iopub.status.idle":"2022-08-05T13:36:50.548907Z","shell.execute_reply.started":"2022-08-05T13:36:50.537505Z","shell.execute_reply":"2022-08-05T13:36:50.547948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"follows_back(669354,1635354)","metadata":{"id":"xcqbV41rhtyg","outputId":"18ea6fe2-3f96-42c0-d116-ecb76ddba4b5","execution":{"iopub.status.busy":"2022-08-05T13:36:50.550284Z","iopub.execute_input":"2022-08-05T13:36:50.55069Z","iopub.status.idle":"2022-08-05T13:36:50.560436Z","shell.execute_reply.started":"2022-08-05T13:36:50.550649Z","shell.execute_reply":"2022-08-05T13:36:50.559714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4.5 Katz Centrality:\nhttps://en.wikipedia.org/wiki/Katz_centrality\n\nhttps://www.geeksforgeeks.org/katz-centrality-centrality-measure/\n Katz centrality computes the centrality for a node \n based on the centrality of its neighbors. It is a \n generalization of the eigenvector centrality. The\n Katz centrality for node `i` is\n \n$$x_i = \\alpha \\sum_{j} A_{ij} x_j + \\beta,$$\nwhere `A` is the adjacency matrix of the graph G \nwith eigenvalues $$\\lambda$$.\n\nThe parameter $$\\beta$$ controls the initial centrality and \n\n$$\\alpha < \\frac{1}{\\lambda_{max}}.$$","metadata":{"id":"oz5JlBrRhtyh"}},{"cell_type":"code","source":"if not os.path.isfile(PATH_TO_FEATURES + '/katz.p'):\n katz = nx.katz.katz_centrality(train_graph,alpha=0.005,beta=1)\n pickle.dump(katz,open('./katz.p','wb'))\nelse:\n katz = pickle.load(open(PATH_TO_FEATURES + '/katz.p','rb'))","metadata":{"id":"Kyb5jbq9htyi","execution":{"iopub.status.busy":"2022-08-05T13:36:50.561702Z","iopub.execute_input":"2022-08-05T13:36:50.561976Z","iopub.status.idle":"2022-08-05T13:36:51.413136Z","shell.execute_reply.started":"2022-08-05T13:36:50.56195Z","shell.execute_reply":"2022-08-05T13:36:51.412328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('min',katz[min(katz, key=katz.get)])\nprint('max',katz[max(katz, key=katz.get)])\nprint('mean',float(sum(katz.values())) / len(katz))","metadata":{"id":"8hHcdBCshtyj","outputId":"05f49ad4-46fe-4cf6-f32a-2fe4846b0714","execution":{"iopub.status.busy":"2022-08-05T13:36:51.414301Z","iopub.execute_input":"2022-08-05T13:36:51.4149Z","iopub.status.idle":"2022-08-05T13:36:52.200289Z","shell.execute_reply.started":"2022-08-05T13:36:51.414859Z","shell.execute_reply":"2022-08-05T13:36:52.199187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mean_katz = float(sum(katz.values())) / len(katz)\nprint(mean_katz)","metadata":{"id":"91vzcarahtyl","outputId":"99f52422-9edb-479a-d5d9-e33401160da7","execution":{"iopub.status.busy":"2022-08-05T13:36:52.201891Z","iopub.execute_input":"2022-08-05T13:36:52.202345Z","iopub.status.idle":"2022-08-05T13:36:52.253146Z","shell.execute_reply.started":"2022-08-05T13:36:52.20228Z","shell.execute_reply":"2022-08-05T13:36:52.252101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4.6 Hits Score\nThe HITS algorithm computes two numbers for a node. Authorities estimates the node value based on the incoming links. Hubs estimates the node value based on outgoing links.\n\nhttps://en.wikipedia.org/wiki/HITS_algorithm","metadata":{"id":"b95Vcu62htyn"}},{"cell_type":"code","source":"if not os.path.isfile(PATH_TO_FEATURES + '/hits.p'):\n hits = nx.hits(train_graph, max_iter=100, tol=1e-08, nstart=None, normalized=True)\n pickle.dump(hits,open('./hits.p','wb'))\nelse:\n hits = pickle.load(open(PATH_TO_FEATURES + '/hits.p','rb'))","metadata":{"id":"thswzzaJhtyn","execution":{"iopub.status.busy":"2022-08-05T13:36:52.254848Z","iopub.execute_input":"2022-08-05T13:36:52.255141Z","iopub.status.idle":"2022-08-05T13:36:53.871065Z","shell.execute_reply.started":"2022-08-05T13:36:52.255112Z","shell.execute_reply":"2022-08-05T13:36:53.870215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('min',hits[0][min(hits[0], key=hits[0].get)])\nprint('max',hits[0][max(hits[0], key=hits[0].get)])\nprint('mean',float(sum(hits[0].values())) / len(hits[0]))","metadata":{"id":"oqlm2HtThtyp","outputId":"77448253-5409-4229-f0be-b8dbc14d7f46","execution":{"iopub.status.busy":"2022-08-05T13:36:53.872133Z","iopub.execute_input":"2022-08-05T13:36:53.872533Z","iopub.status.idle":"2022-08-05T13:36:54.645503Z","shell.execute_reply.started":"2022-08-05T13:36:53.872504Z","shell.execute_reply":"2022-08-05T13:36:54.644249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 5. Featurization","metadata":{"id":"lq6zrXWIhtyr"}},{"cell_type":"markdown","source":"## 5. 1 Reading a sample of Data from both train and test","metadata":{"id":"181XzpA6htyr"}},{"cell_type":"code","source":"import random\nif os.path.isfile(PATH_TO_AFTER_EDA + '/train_after_eda.csv'):\n filename = PATH_TO_AFTER_EDA + \"/train_after_eda.csv\"\n # you uncomment this line, if you dont know the lentgh of the file name\n # here we have hardcoded the number of lines as 15100030\n # n_train = sum(1 for line in open(filename)) #number of records in file (excludes header)\n n_train = 15100028\n s = 100000 #desired sample size\n skip_train = sorted(random.sample(range(1,n_train+1),n_train-s))\n #https://stackoverflow.com/a/22259008/4084039","metadata":{"id":"OBqd3LT3htys","execution":{"iopub.status.busy":"2022-08-05T13:36:54.646994Z","iopub.execute_input":"2022-08-05T13:36:54.647513Z","iopub.status.idle":"2022-08-05T13:37:25.700219Z","shell.execute_reply.started":"2022-08-05T13:36:54.64748Z","shell.execute_reply":"2022-08-05T13:37:25.699303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if os.path.isfile(PATH_TO_AFTER_EDA + '/train_after_eda.csv'):\n filename = PATH_TO_AFTER_EDA + \"/test_after_eda.csv\"\n # you uncomment this line, if you dont know the lentgh of the file name\n # here we have hardcoded the number of lines as 3775008\n # n_test = sum(1 for line in open(filename)) #number of records in file (excludes header)\n n_test = 3775006\n s = 50000 #desired sample size\n skip_test = sorted(random.sample(range(1,n_test+1),n_test-s))\n #https://stackoverflow.com/a/22259008/4084039","metadata":{"id":"JLSEnv8Khtyu","execution":{"iopub.status.busy":"2022-08-05T13:37:25.701539Z","iopub.execute_input":"2022-08-05T13:37:25.701852Z","iopub.status.idle":"2022-08-05T13:37:32.551746Z","shell.execute_reply.started":"2022-08-05T13:37:25.701824Z","shell.execute_reply":"2022-08-05T13:37:32.550691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Number of rows in the train data file:\", n_train)\nprint(\"Number of rows we are going to elimiate in train data are\",len(skip_train))\nprint(\"Number of rows in the test data file:\", n_test)\nprint(\"Number of rows we are going to elimiate in test data are\",len(skip_test))","metadata":{"id":"cJf-fJzOhtyv","outputId":"322902a4-0420-4b99-8606-5fd0de4bbea4","execution":{"iopub.status.busy":"2022-08-05T13:37:32.553332Z","iopub.execute_input":"2022-08-05T13:37:32.55365Z","iopub.status.idle":"2022-08-05T13:37:32.559805Z","shell.execute_reply.started":"2022-08-05T13:37:32.553596Z","shell.execute_reply":"2022-08-05T13:37:32.558857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_final_train = pd.read_csv(PATH_TO_AFTER_EDA+'/train_after_eda.csv', skiprows=skip_train, names=['source_node', 'destination_node'])\ndf_final_train['indicator_link'] = pd.read_csv(PATH_TO_DATA + '/train_y.csv', skiprows=skip_train, names=['indicator_link'])\nprint(\"Our train matrix size \",df_final_train.shape)\ndf_final_train.head(2)","metadata":{"id":"USl8ZxbPhtyz","outputId":"daf2af43-3f98-4466-ad99-03bc54464714","execution":{"iopub.status.busy":"2022-08-05T13:37:32.561374Z","iopub.execute_input":"2022-08-05T13:37:32.561686Z","iopub.status.idle":"2022-08-05T13:37:43.205571Z","shell.execute_reply.started":"2022-08-05T13:37:32.561654Z","shell.execute_reply":"2022-08-05T13:37:43.203461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_final_test = pd.read_csv(PATH_TO_AFTER_EDA + '/test_after_eda.csv', skiprows=skip_test, names=['source_node', 'destination_node'])\ndf_final_test['indicator_link'] = pd.read_csv(PATH_TO_DATA + '/test_y.csv', skiprows=skip_test, names=['indicator_link'])\nprint(\"Our test matrix size \",df_final_test.shape)\ndf_final_test.head(2)","metadata":{"id":"KrJfM3ZZhty1","outputId":"1ca99e70-6d2a-45f2-f51c-fd3b1211ad20","execution":{"iopub.status.busy":"2022-08-05T13:37:43.207851Z","iopub.execute_input":"2022-08-05T13:37:43.208361Z","iopub.status.idle":"2022-08-05T13:37:45.445532Z","shell.execute_reply.started":"2022-08-05T13:37:43.208329Z","shell.execute_reply":"2022-08-05T13:37:45.444714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 5.2 Adding a set of features\n\n__we will create these each of these features for both train and test data points__\n
    \n
  1. jaccard_followers
    2. \n
  2. jaccard_followees
    3. \n
  3. cosine_followers
    4. \n
  4. cosine_followees
    5. \n
  5. num_followers_s
    6. \n
  6. num_followees_s
    7. \n
  7. num_followers_d
    8. \n
  8. num_followees_d
    9. \n
  9. inter_followers
    10. \n
  10. inter_followees
    11. \n
","metadata":{"id":"sQWRRJ6Thty5"}},{"cell_type":"code","source":"if not os.path.isfile(PATH_TO_FEATURES + '/storage_sample_stage1.h5'):\n #mapping jaccrd followers to train and test data\n df_final_train['jaccard_followers'] = df_final_train.apply(lambda row:\n jaccard_for_followers(row['source_node'],row['destination_node']),axis=1)\n df_final_test['jaccard_followers'] = df_final_test.apply(lambda row:\n jaccard_for_followers(row['source_node'],row['destination_node']),axis=1)\n\n #mapping jaccrd followees to train and test data\n df_final_train['jaccard_followees'] = df_final_train.apply(lambda row:\n jaccard_for_followees(row['source_node'],row['destination_node']),axis=1)\n df_final_test['jaccard_followees'] = df_final_test.apply(lambda row:\n jaccard_for_followees(row['source_node'],row['destination_node']),axis=1)\n \n\n #mapping jaccrd followers to train and test data\n df_final_train['cosine_followers'] = df_final_train.apply(lambda row:\n cosine_for_followers(row['source_node'],row['destination_node']),axis=1)\n df_final_test['cosine_followers'] = df_final_test.apply(lambda row:\n cosine_for_followers(row['source_node'],row['destination_node']),axis=1)\n\n #mapping jaccrd followees to train and test data\n df_final_train['cosine_followees'] = df_final_train.apply(lambda row:\n cosine_for_followees(row['source_node'],row['destination_node']),axis=1)\n df_final_test['cosine_followees'] = df_final_test.apply(lambda row:\n cosine_for_followees(row['source_node'],row['destination_node']),axis=1)","metadata":{"id":"Z0OGWQLdhty6","execution":{"iopub.status.busy":"2022-08-05T13:37:45.447124Z","iopub.execute_input":"2022-08-05T13:37:45.447412Z","iopub.status.idle":"2022-08-05T13:37:45.462992Z","shell.execute_reply.started":"2022-08-05T13:37:45.447383Z","shell.execute_reply":"2022-08-05T13:37:45.462088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def compute_features_stage1(df_final):\n #calculating no of followers followees for source and destination\n #calculating intersection of followers and followees for source and destination\n num_followers_s=[]\n num_followees_s=[]\n num_followers_d=[]\n num_followees_d=[]\n inter_followers=[]\n inter_followees=[]\n for i,row in df_final.iterrows():\n try:\n s1=set(train_graph.predecessors(row['source_node']))\n s2=set(train_graph.successors(row['source_node']))\n except:\n s1 = set()\n s2 = set()\n try:\n d1=set(train_graph.predecessors(row['destination_node']))\n d2=set(train_graph.successors(row['destination_node']))\n except:\n d1 = set()\n d2 = set()\n num_followers_s.append(len(s1))\n num_followees_s.append(len(s2))\n\n num_followers_d.append(len(d1))\n num_followees_d.append(len(d2))\n\n inter_followers.append(len(s1.intersection(d1)))\n inter_followees.append(len(s2.intersection(d2)))\n \n return num_followers_s, num_followers_d, num_followees_s, num_followees_d, inter_followers, inter_followees","metadata":{"id":"sVQ7oFvQhty8","execution":{"iopub.status.busy":"2022-08-05T13:37:45.464193Z","iopub.execute_input":"2022-08-05T13:37:45.464506Z","iopub.status.idle":"2022-08-05T13:37:45.47558Z","shell.execute_reply.started":"2022-08-05T13:37:45.464468Z","shell.execute_reply":"2022-08-05T13:37:45.47459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not os.path.isfile(PATH_TO_FEATURES + '/storage_sample_stage1.h5'):\n df_final_train['num_followers_s'], df_final_train['num_followers_d'], \\\n df_final_train['num_followees_s'], df_final_train['num_followees_d'], \\\n df_final_train['inter_followers'], df_final_train['inter_followees']= compute_features_stage1(df_final_train)\n \n df_final_test['num_followers_s'], df_final_test['num_followers_d'], \\\n df_final_test['num_followees_s'], df_final_test['num_followees_d'], \\\n df_final_test['inter_followers'], df_final_test['inter_followees']= compute_features_stage1(df_final_test)\n \n hdf = HDFStore('./storage_sample_stage1.h5')\n hdf.put('train_df',df_final_train, format='table', data_columns=True)\n hdf.put('test_df',df_final_test, format='table', data_columns=True)\n hdf.close()\nelse:\n df_final_train = read_hdf(PATH_TO_FEATURES + '/storage_sample_stage1.h5', 'train_df',mode='r')\n df_final_test = read_hdf(PATH_TO_FEATURES + '/storage_sample_stage1.h5', 'test_df',mode='r')","metadata":{"id":"HdGrj8fchty_","execution":{"iopub.status.busy":"2022-08-05T13:37:45.477079Z","iopub.execute_input":"2022-08-05T13:37:45.477436Z","iopub.status.idle":"2022-08-05T13:37:46.435554Z","shell.execute_reply.started":"2022-08-05T13:37:45.477403Z","shell.execute_reply":"2022-08-05T13:37:46.434731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 5.3 Adding new set of features\n\n__we will create these each of these features for both train and test data points__\n
    \n
  1. adar index
    2. \n
  2. is following back
    3. \n
  3. belongs to same weakly connect components
    4. \n
  4. shortest path between source and destination
    5. \n
","metadata":{"id":"abEET_cMhtzC"}},{"cell_type":"code","source":"if not os.path.isfile(PATH_TO_FEATURES+'/storage_sample_stage2.h5'):\n #mapping adar index on train\n df_final_train['adar_index'] = df_final_train.apply(lambda row: calc_adar_in(row['source_node'],row['destination_node']),axis=1)\n #mapping adar index on test\n df_final_test['adar_index'] = df_final_test.apply(lambda row: calc_adar_in(row['source_node'],row['destination_node']),axis=1)\n\n #--------------------------------------------------------------------------------------------------------\n #mapping followback or not on train\n df_final_train['follows_back'] = df_final_train.apply(lambda row: follows_back(row['source_node'],row['destination_node']),axis=1)\n\n #mapping followback or not on test\n df_final_test['follows_back'] = df_final_test.apply(lambda row: follows_back(row['source_node'],row['destination_node']),axis=1)\n\n #--------------------------------------------------------------------------------------------------------\n #mapping same component of wcc or not on train\n df_final_train['same_comp'] = df_final_train.apply(lambda row: belongs_to_same_wcc(row['source_node'],row['destination_node']),axis=1)\n\n ##mapping same component of wcc or not on train\n df_final_test['same_comp'] = df_final_test.apply(lambda row: belongs_to_same_wcc(row['source_node'],row['destination_node']),axis=1)\n \n #--------------------------------------------------------------------------------------------------------\n #mapping shortest path on train \n df_final_train['shortest_path'] = df_final_train.apply(lambda row: compute_shortest_path_length(row['source_node'],row['destination_node']),axis=1)\n #mapping shortest path on test\n df_final_test['shortest_path'] = df_final_test.apply(lambda row: compute_shortest_path_length(row['source_node'],row['destination_node']),axis=1)\n\n hdf = HDFStore('./storage_sample_stage2.h5')\n hdf.put('train_df',df_final_train, format='table', data_columns=True)\n hdf.put('test_df',df_final_test, format='table', data_columns=True)\n hdf.close()\nelse:\n df_final_train = read_hdf(PATH_TO_FEATURES+'/storage_sample_stage2.h5', 'train_df',mode='r')\n df_final_test = read_hdf(PATH_TO_FEATURES+'/storage_sample_stage2.h5', 'test_df',mode='r')","metadata":{"id":"qqmAMYiYhtzC","execution":{"iopub.status.busy":"2022-08-05T13:37:46.437061Z","iopub.execute_input":"2022-08-05T13:37:46.437486Z","iopub.status.idle":"2022-08-05T13:37:47.123244Z","shell.execute_reply.started":"2022-08-05T13:37:46.437442Z","shell.execute_reply":"2022-08-05T13:37:47.122493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 5.4 Adding new set of features\n\n__we will create these each of these features for both train and test data points__\n
    \n
  1. Weight Features\n \n
    2. \n
  2. Page Ranking of source
    3. \n
  3. Page Ranking of dest
    4. \n
  4. katz of source
    5. \n
  5. katz of dest
    6. \n
  6. hubs of source
    7. \n
  7. hubs of dest
    8. \n
  8. authorities_s of source
    9. \n
  9. authorities_s of dest
    10. \n
","metadata":{"id":"SlkZgwLOhtzF"}},{"cell_type":"markdown","source":"#### Weight Features","metadata":{"id":"B1HHbKKmhtzF"}},{"cell_type":"markdown","source":"In order to determine the similarity of nodes, an edge weight value was calculated between nodes. Edge weight decreases as the neighbor count goes up. Intuitively, consider one million people following a celebrity on a social network then chances are most of them never met each other or the celebrity. On the other hand, if a user has 30 contacts in his/her social network, the chances are higher that many of them know each other. \n`credit` - Graph-based Features for Supervised Link Prediction\nWilliam Cukierski, Benjamin Hamner, Bo Yang","metadata":{"id":"rvcX6rYihtzG"}},{"cell_type":"markdown","source":"\\begin{equation}\nW = \\frac{1}{\\sqrt{1+|X|}}\n\\end{equation}","metadata":{"id":"g365cciIhtzG"}},{"cell_type":"markdown","source":"it is directed graph so calculated Weighted in and Weighted out differently","metadata":{"id":"tK0JG-vNhtzH"}},{"cell_type":"code","source":"#weight for source and destination of each link\nWeight_in = {}\nWeight_out = {}\nfor i in tqdm(train_graph.nodes()):\n s1=set(train_graph.predecessors(i))\n w_in = 1.0/(np.sqrt(1+len(s1)))\n Weight_in[i]=w_in\n \n s2=set(train_graph.successors(i))\n w_out = 1.0/(np.sqrt(1+len(s2)))\n Weight_out[i]=w_out\n \n#for imputing with mean\nmean_weight_in = np.mean(list(Weight_in.values()))\nmean_weight_out = np.mean(list(Weight_out.values()))","metadata":{"id":"2vYQPXFWhtzH","outputId":"7e8e6d88-8bd6-45f6-f80e-82b093c18974","execution":{"iopub.status.busy":"2022-08-05T13:37:47.124316Z","iopub.execute_input":"2022-08-05T13:37:47.124578Z","iopub.status.idle":"2022-08-05T13:38:03.983642Z","shell.execute_reply.started":"2022-08-05T13:37:47.124552Z","shell.execute_reply":"2022-08-05T13:38:03.982767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not os.path.isfile(PATH_TO_FEATURES+'/storage_sample_stage3.h5'):\n #mapping to pandas train\n df_final_train['weight_in'] = df_final_train.destination_node.apply(lambda x: Weight_in.get(x,mean_weight_in))\n df_final_train['weight_out'] = df_final_train.source_node.apply(lambda x: Weight_out.get(x,mean_weight_out))\n\n #mapping to pandas test\n df_final_test['weight_in'] = df_final_test.destination_node.apply(lambda x: Weight_in.get(x,mean_weight_in))\n df_final_test['weight_out'] = df_final_test.source_node.apply(lambda x: Weight_out.get(x,mean_weight_out))\n\n\n #some features engineerings on the in and out weights\n df_final_train['weight_f1'] = df_final_train.weight_in + df_final_train.weight_out\n df_final_train['weight_f2'] = df_final_train.weight_in * df_final_train.weight_out\n df_final_train['weight_f3'] = (2*df_final_train.weight_in + 1*df_final_train.weight_out)\n df_final_train['weight_f4'] = (1*df_final_train.weight_in + 2*df_final_train.weight_out)\n\n #some features engineerings on the in and out weights\n df_final_test['weight_f1'] = df_final_test.weight_in + df_final_test.weight_out\n df_final_test['weight_f2'] = df_final_test.weight_in * df_final_test.weight_out\n df_final_test['weight_f3'] = (2*df_final_test.weight_in + 1*df_final_test.weight_out)\n df_final_test['weight_f4'] = (1*df_final_test.weight_in + 2*df_final_test.weight_out)","metadata":{"id":"yIHJb4tKhtzJ","execution":{"iopub.status.busy":"2022-08-05T13:38:03.985156Z","iopub.execute_input":"2022-08-05T13:38:03.985614Z","iopub.status.idle":"2022-08-05T13:38:03.999686Z","shell.execute_reply.started":"2022-08-05T13:38:03.985553Z","shell.execute_reply":"2022-08-05T13:38:03.998759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not os.path.isfile(PATH_TO_FEATURES+'/storage_sample_stage3.h5'):\n \n #page rank for source and destination in Train and Test\n #if anything not there in train graph then adding mean page rank \n df_final_train['page_rank_s'] = df_final_train.source_node.apply(lambda x:pr.get(x,mean_pr))\n df_final_train['page_rank_d'] = df_final_train.destination_node.apply(lambda x:pr.get(x,mean_pr))\n\n df_final_test['page_rank_s'] = df_final_test.source_node.apply(lambda x:pr.get(x,mean_pr))\n df_final_test['page_rank_d'] = df_final_test.destination_node.apply(lambda x:pr.get(x,mean_pr))\n #================================================================================\n\n #Katz centrality score for source and destination in Train and test\n #if anything not there in train graph then adding mean katz score\n df_final_train['katz_s'] = df_final_train.source_node.apply(lambda x: katz.get(x,mean_katz))\n df_final_train['katz_d'] = df_final_train.destination_node.apply(lambda x: katz.get(x,mean_katz))\n\n df_final_test['katz_s'] = df_final_test.source_node.apply(lambda x: katz.get(x,mean_katz))\n df_final_test['katz_d'] = df_final_test.destination_node.apply(lambda x: katz.get(x,mean_katz))\n #================================================================================\n\n #Hits algorithm score for source and destination in Train and test\n #if anything not there in train graph then adding 0\n df_final_train['hubs_s'] = df_final_train.source_node.apply(lambda x: hits[0].get(x,0))\n df_final_train['hubs_d'] = df_final_train.destination_node.apply(lambda x: hits[0].get(x,0))\n\n df_final_test['hubs_s'] = df_final_test.source_node.apply(lambda x: hits[0].get(x,0))\n df_final_test['hubs_d'] = df_final_test.destination_node.apply(lambda x: hits[0].get(x,0))\n #================================================================================\n\n #Hits algorithm score for source and destination in Train and Test\n #if anything not there in train graph then adding 0\n df_final_train['authorities_s'] = df_final_train.source_node.apply(lambda x: hits[1].get(x,0))\n df_final_train['authorities_d'] = df_final_train.destination_node.apply(lambda x: hits[1].get(x,0))\n\n df_final_test['authorities_s'] = df_final_test.source_node.apply(lambda x: hits[1].get(x,0))\n df_final_test['authorities_d'] = df_final_test.destination_node.apply(lambda x: hits[1].get(x,0))\n #================================================================================\n\n hdf = HDFStore('./storage_sample_stage3.h5')\n hdf.put('train_df',df_final_train, format='table', data_columns=True)\n hdf.put('test_df',df_final_test, format='table', data_columns=True)\n hdf.close()\nelse:\n df_final_train = read_hdf(PATH_TO_FEATURES+'/storage_sample_stage3.h5', 'train_df',mode='r')\n df_final_test = read_hdf(PATH_TO_FEATURES+'/storage_sample_stage3.h5', 'test_df',mode='r')","metadata":{"id":"QngEQmpThtzK","execution":{"iopub.status.busy":"2022-08-05T13:38:04.001388Z","iopub.execute_input":"2022-08-05T13:38:04.0019Z","iopub.status.idle":"2022-08-05T13:38:05.396596Z","shell.execute_reply.started":"2022-08-05T13:38:04.001857Z","shell.execute_reply":"2022-08-05T13:38:05.395839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 5.5 Adding new set of features\n\n__we will create these each of these features for both train and test data points__\n
    \n
  1. SVD features for both source and destination
    2. \n
","metadata":{"id":"12UADO69htzM"}},{"cell_type":"code","source":"def svd(x, S):\n try:\n z = sadj_dict[x]\n return S[z]\n except:\n return [0,0,0,0,0,0]","metadata":{"id":"pAK7rTIkhtzM","execution":{"iopub.status.busy":"2022-08-05T13:38:05.397706Z","iopub.execute_input":"2022-08-05T13:38:05.397979Z","iopub.status.idle":"2022-08-05T13:38:05.403994Z","shell.execute_reply.started":"2022-08-05T13:38:05.397952Z","shell.execute_reply":"2022-08-05T13:38:05.402821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#for svd features to get feature vector creating a dict node val and inedx in svd vector\nsadj_col = sorted(train_graph.nodes())\nsadj_dict = { val:idx for idx,val in enumerate(sadj_col)}","metadata":{"id":"e7JKWhS1htzO","execution":{"iopub.status.busy":"2022-08-05T13:38:05.405153Z","iopub.execute_input":"2022-08-05T13:38:05.405442Z","iopub.status.idle":"2022-08-05T13:38:07.026356Z","shell.execute_reply.started":"2022-08-05T13:38:05.405406Z","shell.execute_reply":"2022-08-05T13:38:07.025368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Adj = nx.adjacency_matrix(train_graph,nodelist=sorted(train_graph.nodes())).asfptype()","metadata":{"id":"wYDGVHEwhtzQ","execution":{"iopub.status.busy":"2022-08-05T13:42:12.648382Z","iopub.execute_input":"2022-08-05T13:42:12.648782Z","iopub.status.idle":"2022-08-05T13:42:12.681302Z","shell.execute_reply.started":"2022-08-05T13:42:12.648749Z","shell.execute_reply":"2022-08-05T13:42:12.680216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"U, s, V = svds(Adj, k = 6)\nprint('Adjacency matrix Shape',Adj.shape)\nprint('U Shape',U.shape)\nprint('V Shape',V.shape)\nprint('s Shape',s.shape)","metadata":{"id":"jMT8JniihtzT","outputId":"3f9bfb32-004f-4698-e415-469243250130","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not os.path.isfile(PATH_TO_FEATURES+'/storage_sample_stage4.h5'):\n #===================================================================================================\n \n df_final_train[['svd_u_s_1', 'svd_u_s_2','svd_u_s_3', 'svd_u_s_4', 'svd_u_s_5', 'svd_u_s_6']] = \\\n df_final_train.source_node.apply(lambda x: svd(x, U)).apply(pd.Series)\n \n df_final_train[['svd_u_d_1', 'svd_u_d_2', 'svd_u_d_3', 'svd_u_d_4', 'svd_u_d_5','svd_u_d_6']] = \\\n df_final_train.destination_node.apply(lambda x: svd(x, U)).apply(pd.Series)\n #===================================================================================================\n \n df_final_train[['svd_v_s_1','svd_v_s_2', 'svd_v_s_3', 'svd_v_s_4', 'svd_v_s_5', 'svd_v_s_6',]] = \\\n df_final_train.source_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)\n\n df_final_train[['svd_v_d_1', 'svd_v_d_2', 'svd_v_d_3', 'svd_v_d_4', 'svd_v_d_5','svd_v_d_6']] = \\\n df_final_train.destination_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)\n #===================================================================================================\n \n df_final_test[['svd_u_s_1', 'svd_u_s_2','svd_u_s_3', 'svd_u_s_4', 'svd_u_s_5', 'svd_u_s_6']] = \\\n df_final_test.source_node.apply(lambda x: svd(x, U)).apply(pd.Series)\n \n df_final_test[['svd_u_d_1', 'svd_u_d_2', 'svd_u_d_3', 'svd_u_d_4', 'svd_u_d_5','svd_u_d_6']] = \\\n df_final_test.destination_node.apply(lambda x: svd(x, U)).apply(pd.Series)\n\n #===================================================================================================\n \n df_final_test[['svd_v_s_1','svd_v_s_2', 'svd_v_s_3', 'svd_v_s_4', 'svd_v_s_5', 'svd_v_s_6',]] = \\\n df_final_test.source_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)\n\n df_final_test[['svd_v_d_1', 'svd_v_d_2', 'svd_v_d_3', 'svd_v_d_4', 'svd_v_d_5','svd_v_d_6']] = \\\n df_final_test.destination_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)\n #===================================================================================================\n\n hdf = HDFStore('./storage_sample_stage4.h5')\n hdf.put('train_df',df_final_train, format='table', data_columns=True)\n hdf.put('test_df',df_final_test, format='table', data_columns=True)\n hdf.close()","metadata":{"id":"6r5swVlDhtzU","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

Social network Graph Link Prediction - Facebook Challenge

","metadata":{"id":"naWvqMEMHglO"}},{"cell_type":"code","source":"#Importing Libraries\n# please do go through this python notebook: \nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport csv\nimport pandas as pd#pandas to create small dataframes \nimport datetime #Convert to unix time\nimport time #Convert to unix time\n# if numpy is not installed already : pip3 install numpy\nimport numpy as np#Do aritmetic operations on arrays\n# matplotlib: used to plot graphs\nimport matplotlib\nimport matplotlib.pylab as plt\nimport seaborn as sns#Plots\nfrom matplotlib import rcParams#Size of plots \nfrom sklearn.cluster import MiniBatchKMeans, KMeans#Clustering\nimport math\nimport pickle\nimport os\n# to install xgboost: pip3 install xgboost\nimport xgboost as xgb\n\nimport warnings\nimport networkx as nx\nimport pdb\nimport pickle\nfrom pandas import HDFStore,DataFrame\nfrom pandas import read_hdf\nfrom scipy.sparse.linalg import svds, eigs\nimport gc\nfrom tqdm import tqdm\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import f1_score","metadata":{"id":"9wb9N5RzHglP","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#reading\nfrom pandas import read_hdf\ndf_final_train = read_hdf(PATH_TO_FEATURES+'/storage_sample_stage4.h5', 'train_df',mode='r')\ndf_final_test = read_hdf(PATH_TO_FEATURES+'/storage_sample_stage4.h5', 'test_df',mode='r')","metadata":{"id":"XC4OJFKkHglU","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_final_train.columns","metadata":{"id":"5Gm-BHRkHglW","outputId":"fc59a58e-49e3-4bb1-83bf-ee484030c342","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train = df_final_train.indicator_link\ny_test = df_final_test.indicator_link","metadata":{"id":"XRW7VZ4AHglc","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_final_train.drop(['source_node', 'destination_node','indicator_link'],axis=1,inplace=True)\ndf_final_test.drop(['source_node', 'destination_node','indicator_link'],axis=1,inplace=True)","metadata":{"id":"4lj9M_qtHglf","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"estimators = [10,50,100,250,450]\ntrain_scores = []\ntest_scores = []\nfor i in estimators:\n clf = RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n max_depth=5, max_features='auto', max_leaf_nodes=None,\n min_impurity_decrease=0.0, min_impurity_split=None,\n min_samples_leaf=52, min_samples_split=120,\n min_weight_fraction_leaf=0.0, n_estimators=i, n_jobs=-1,random_state=25,verbose=0,warm_start=False)\n clf.fit(df_final_train,y_train)\n train_sc = f1_score(y_train,clf.predict(df_final_train))\n test_sc = f1_score(y_test,clf.predict(df_final_test))\n test_scores.append(test_sc)\n train_scores.append(train_sc)\n print('Estimators = ',i,'Train Score',train_sc,'test Score',test_sc)\nplt.plot(estimators,train_scores,label='Train Score')\nplt.plot(estimators,test_scores,label='Test Score')\nplt.xlabel('Estimators')\nplt.ylabel('Score')\nplt.title('Estimators vs score at depth of 5')","metadata":{"id":"lIEc91uVHgli","outputId":"d0441b63-7256-437a-eb13-db90a727aa31","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"depths = [3,9,11,15,20,35,50,70,130]\ntrain_scores = []\ntest_scores = []\nfor i in depths:\n clf = RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n max_depth=i, max_features='auto', max_leaf_nodes=None,\n min_impurity_decrease=0.0, min_impurity_split=None,\n min_samples_leaf=52, min_samples_split=120,\n min_weight_fraction_leaf=0.0, n_estimators=115, n_jobs=-1,random_state=25,verbose=0,warm_start=False)\n clf.fit(df_final_train,y_train)\n train_sc = f1_score(y_train,clf.predict(df_final_train))\n test_sc = f1_score(y_test,clf.predict(df_final_test))\n test_scores.append(test_sc)\n train_scores.append(train_sc)\n print('depth = ',i,'Train Score',train_sc,'test Score',test_sc)\nplt.plot(depths,train_scores,label='Train Score')\nplt.plot(depths,test_scores,label='Test Score')\nplt.xlabel('Depth')\nplt.ylabel('Score')\nplt.title('Depth vs score at depth of 5 at estimators = 115')\nplt.show()","metadata":{"id":"nZxhrTdQHglm","outputId":"53382b8e-148d-4c22-8066-d9807fb27012","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import f1_score\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import f1_score\nfrom sklearn.model_selection import RandomizedSearchCV\nfrom scipy.stats import randint as sp_randint\nfrom scipy.stats import uniform\n\nparam_dist = {\"n_estimators\":sp_randint(105,125),\n \"max_depth\": sp_randint(10,15),\n \"min_samples_split\": sp_randint(110,190),\n \"min_samples_leaf\": sp_randint(25,65)}\n\nclf = RandomForestClassifier(random_state=25,n_jobs=-1)\n\nrf_random = RandomizedSearchCV(clf, param_distributions=param_dist,\n n_iter=5,cv=10,scoring='f1',random_state=25)\n\nrf_random.fit(df_final_train,y_train)\nprint('mean test scores',rf_random.cv_results_['mean_test_score'])\n# print('mean train scores',rf_random.cv_results_['mean_train_score'])","metadata":{"id":"MrG1Lfl3Hglq","outputId":"7b6e3c97-8d66-455d-cd79-213f824719cc","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rf_random.cv_results_","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(rf_random.best_estimator_)","metadata":{"id":"hme3j_diHglu","outputId":"581ccc02-7770-4a94-d003-257af4e81570","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clf = RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n max_depth=14, max_features='auto', max_leaf_nodes=None,\n min_impurity_decrease=0.0, min_impurity_split=None,\n min_samples_leaf=28, min_samples_split=111,\n min_weight_fraction_leaf=0.0, n_estimators=121, n_jobs=-1,\n oob_score=False, random_state=25, verbose=0, warm_start=False)","metadata":{"id":"qu4KIEweHglx","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clf.fit(df_final_train,y_train)\ny_train_pred = clf.predict(df_final_train)\ny_test_pred = clf.predict(df_final_test)","metadata":{"id":"Dax83GfLHgl1","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import f1_score\nprint('Train f1 score',f1_score(y_train,y_train_pred))\nprint('Test f1 score',f1_score(y_test,y_test_pred))","metadata":{"id":"PTtAp1iTHgl4","outputId":"e2e3d725-3c4d-4374-d303-8f294b570977","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\ndef plot_confusion_matrix(test_y, predict_y):\n C = confusion_matrix(test_y, predict_y)\n \n A =(((C.T)/(C.sum(axis=1))).T)\n \n B =(C/C.sum(axis=0))\n plt.figure(figsize=(20,4))\n \n labels = [0,1]\n # representing A in heatmap format\n cmap=sns.light_palette(\"blue\")\n plt.subplot(1, 3, 1)\n sns.heatmap(C, annot=True, cmap=cmap, fmt=\".3f\", xticklabels=labels, yticklabels=labels)\n plt.xlabel('Predicted Class')\n plt.ylabel('Original Class')\n plt.title(\"Confusion matrix\")\n \n plt.subplot(1, 3, 2)\n sns.heatmap(B, annot=True, cmap=cmap, fmt=\".3f\", xticklabels=labels, yticklabels=labels)\n plt.xlabel('Predicted Class')\n plt.ylabel('Original Class')\n plt.title(\"Precision matrix\")\n \n plt.subplot(1, 3, 3)\n # representing B in heatmap format\n sns.heatmap(A, annot=True, cmap=cmap, fmt=\".3f\", xticklabels=labels, yticklabels=labels)\n plt.xlabel('Predicted Class')\n plt.ylabel('Original Class')\n plt.title(\"Recall matrix\")\n \n plt.show()","metadata":{"id":"5XfUkZYQHgl7","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Train confusion_matrix')\nplot_confusion_matrix(y_train,y_train_pred)\nprint('Test confusion_matrix')\nplot_confusion_matrix(y_test,y_test_pred)","metadata":{"id":"s6t11dhTHgl-","outputId":"189de3be-1c99-4653-f56f-12b18b0f4db7","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import roc_curve, auc\nfpr,tpr,ths = roc_curve(y_test,y_test_pred)\nauc_sc = auc(fpr, tpr)\nplt.plot(fpr, tpr, color='navy',label='ROC curve (area = %0.2f)' % auc_sc)\nplt.xlabel('False Positive Rate')\nplt.ylabel('True Positive Rate')\nplt.title('Receiver operating characteristic with test data')\nplt.legend()\nplt.show()","metadata":{"id":"61TF-VLeHgmD","outputId":"795f1b39-61c7-470f-e2d0-1fe6dc7ac5fd","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = df_final_train.columns\nimportances = clf.feature_importances_\nindices = (np.argsort(importances))[-25:]\nplt.figure(figsize=(10,12))\nplt.title('Feature Importances')\nplt.barh(range(len(indices)), importances[indices], color='r', align='center')\nplt.yticks(range(len(indices)), [features[i] for i in indices])\nplt.xlabel('Relative Importance')\nplt.show()","metadata":{"id":"HEZ7uvN6HgmK","outputId":"6f762256-54fc-4e2f-865b-22342033eb2c","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n1. Preferential Attachment \nhttp://be.amazd.com/link-prediction/
\n2. svd_dot\nhttps://storage.googleapis.com/kaggle-forum-message-attachments/2594/supervised_link_prediction.pdf
\n","metadata":{"id":"xdHW32hcHgmN"}},{"cell_type":"markdown","source":"# **Preferential Attachments**","metadata":{"id":"UTlCUFt-oHb8"}},{"cell_type":"markdown","source":"For followers\n","metadata":{"id":"IE5rtN8i0Kfr"}},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\nfrom xgboost import XGBClassifier\nfrom sklearn.calibration import CalibratedClassifierCV\nfrom sklearn.model_selection import RandomizedSearchCV\n","metadata":{"id":"mtCJ6YFP_Odq","execution":{"iopub.status.busy":"2022-08-04T14:01:35.281263Z","iopub.execute_input":"2022-08-04T14:01:35.2819Z","iopub.status.idle":"2022-08-04T14:01:35.287456Z","shell.execute_reply.started":"2022-08-04T14:01:35.281845Z","shell.execute_reply":"2022-08-04T14:01:35.286628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_confusion_matrix(test_y, predict_y):\n C = confusion_matrix(test_y, predict_y)\n print(\"Number of misclassified points \",(len(test_y)-np.trace(C))/len(test_y)*100)\n A =(((C.T)/(C.sum(axis=1))).T)\n B =(C/C.sum(axis=0))\n labels = [0,1]\n cmap=sns.light_palette(\"green\")\n print(\"-\"*50, \"Confusion matrix\", \"-\"*50)\n plt.figure(figsize=(5,5))\n sns.heatmap(C, annot=True, cmap=cmap, fmt=\".3f\", xticklabels=labels, yticklabels=labels)\n plt.xlabel('Predicted Class')\n plt.ylabel('Original Class')\n plt.show()\n\n print(\"-\"*50, \"Precision matrix\", \"-\"*50)\n plt.figure(figsize=(5,5))\n sns.heatmap(B, annot=True, cmap=cmap, fmt=\".3f\", xticklabels=labels, yticklabels=labels)\n plt.xlabel('Predicted Class')\n plt.ylabel('Original Class')\n plt.show()\n print(\"Sum of columns in precision matrix\",B.sum(axis=0))\n\n print(\"-\"*50, \"Recall matrix\" , \"-\"*50)\n plt.figure(figsize=(5,5))\n sns.heatmap(A, annot=True, cmap=cmap, fmt=\".3f\", xticklabels=labels, yticklabels=labels)\n plt.xlabel('Predicted Class')\n plt.ylabel('Original Class')\n plt.show()\n print(\"Sum of rows in precision matrix\",A.sum(axis=1))","metadata":{"id":"TT9hrfOpARip","execution":{"iopub.status.busy":"2022-08-04T14:01:35.745915Z","iopub.execute_input":"2022-08-04T14:01:35.746307Z","iopub.status.idle":"2022-08-04T14:01:35.759323Z","shell.execute_reply.started":"2022-08-04T14:01:35.746274Z","shell.execute_reply":"2022-08-04T14:01:35.758194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = {\n 'min_child_weight': [1, 5, 10],\n 'learning_rate':[0.01,0.03,0.05,0.1,0.15,0.2],\n 'n_estimators' :[100,200,500,1000,2000],\n 'gamma': [0.5, 1, 1.5, 2, 5],\n 'subsample': [0.6, 0.8, 1.0],\n 'colsample_bytree': [0.6, 0.8, 1.0],\n 'max_depth': [3, 4, 5]\n }\nxgboost = XGBClassifier() \nrandom_cfl1=RandomizedSearchCV(xgboost,param_distributions = params,verbose=10,n_jobs=-1)\nrandom_cfl1.fit(df_final_train,y_train)","metadata":{"id":"iEWArnEVBzTn","outputId":"23907d22-36ce-4343-efea-f3f04a7a8dfb","execution":{"iopub.status.busy":"2022-08-04T14:03:06.813183Z","iopub.execute_input":"2022-08-04T14:03:06.813635Z","iopub.status.idle":"2022-08-04T15:20:05.52195Z","shell.execute_reply.started":"2022-08-04T14:03:06.8136Z","shell.execute_reply":"2022-08-04T15:20:05.52041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(random_cfl1.best_params_)","metadata":{"id":"a5CsX32kV7yo","outputId":"38c1dd78-96f7-482c-e2af-35d22ff20025","execution":{"iopub.status.busy":"2022-08-04T15:20:05.524873Z","iopub.execute_input":"2022-08-04T15:20:05.525586Z","iopub.status.idle":"2022-08-04T15:20:05.534672Z","shell.execute_reply.started":"2022-08-04T15:20:05.525533Z","shell.execute_reply":"2022-08-04T15:20:05.533026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.calibration import CalibratedClassifierCV\nbest_xgb = XGBClassifier(sub_sample=0.8,learning_rate = 0.2,colsample_bytree=0.6,max_depth=5,n_estimators=1000,min_child_weight=5,gamma=5)\nbest_xgb.fit(df_final_train,y_train)\nclf = CalibratedClassifierCV(best_xgb,method=\"sigmoid\")\nclf.fit(df_final_train,y_train)","metadata":{"id":"j0hYiCPTWAp9","outputId":"d28e5e71-0c5c-420a-c782-87764cd708a8","execution":{"iopub.status.busy":"2022-08-04T15:20:05.537025Z","iopub.execute_input":"2022-08-04T15:20:05.537847Z","iopub.status.idle":"2022-08-04T15:31:52.266839Z","shell.execute_reply.started":"2022-08-04T15:20:05.537798Z","shell.execute_reply":"2022-08-04T15:31:52.265068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import log_loss\npredict_y = clf.predict_proba(df_final_train)\nprint(\"the train log loss is\",log_loss(y_train,predict_y,labels=best_xgb.classes_))\npredict_y = clf.predict_proba(df_final_test)\nprint(\"the test log loss is\",log_loss(y_test,predict_y,labels = best_xgb.classes_))\nplot_confusion_matrix(y_test,clf.predict(df_final_test))","metadata":{"id":"JMaGs6dngtMi","outputId":"3f276ba4-72ae-446b-9f93-6b36f9258340","execution":{"iopub.status.busy":"2022-08-04T15:31:52.269877Z","iopub.execute_input":"2022-08-04T15:31:52.27075Z","iopub.status.idle":"2022-08-04T15:32:02.051741Z","shell.execute_reply.started":"2022-08-04T15:31:52.270682Z","shell.execute_reply":"2022-08-04T15:32:02.050693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import f1_score\nprint(\"f1 score for train data\",f1_score(y_train,clf.predict(df_final_train)))\nprint(\"f1 score for test data\",f1_score(y_test,clf.predict(df_final_test)))","metadata":{"id":"h-caxmrvlJOb","outputId":"e5d87491-bc83-4dfd-8c40-575012489288","execution":{"iopub.status.busy":"2022-08-04T15:32:02.054976Z","iopub.execute_input":"2022-08-04T15:32:02.055384Z","iopub.status.idle":"2022-08-04T15:32:08.760451Z","shell.execute_reply.started":"2022-08-04T15:32:02.055351Z","shell.execute_reply":"2022-08-04T15:32:08.759093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import roc_curve, auc\nfpr,tpr,ths = roc_curve(y_test,clf.predict(df_final_test))\nauc_sc = auc(fpr, tpr)\nplt.plot(fpr, tpr,label = ('auc=',auc_sc))\nplt.xlabel('FPR')\nplt.ylabel('TPR')\nplt.legend()\nplt.show()","metadata":{"id":"XfaQjaqOh8Kc","outputId":"311fe76d-eac3-4c01-af6d-958f2910898e","execution":{"iopub.status.busy":"2022-08-04T15:32:08.762512Z","iopub.execute_input":"2022-08-04T15:32:08.76322Z","iopub.status.idle":"2022-08-04T15:32:11.238117Z","shell.execute_reply.started":"2022-08-04T15:32:08.763153Z","shell.execute_reply":"2022-08-04T15:32:11.2372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = df_final_train.columns\nimportances = best_xgb.feature_importances_\nindices = (np.argsort(importances))[-25:]\nplt.figure(figsize=(10,12))\nplt.title('Feature Importances')\nplt.barh(range(len(indices)), importances[indices], color='r', align='center')\nplt.yticks(range(len(indices)), [features[i] for i in indices])\nplt.xlabel('Relative Importance')\nplt.show()","metadata":{"id":"pdHn7qhrjkAh","outputId":"28bddb0b-1b83-4bb4-81d9-8b37aeb24f98","execution":{"iopub.status.busy":"2022-08-04T15:32:11.239231Z","iopub.execute_input":"2022-08-04T15:32:11.239701Z","iopub.status.idle":"2022-08-04T15:32:11.611328Z","shell.execute_reply.started":"2022-08-04T15:32:11.239669Z","shell.execute_reply":"2022-08-04T15:32:11.610319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# Conclution\n\n","metadata":{"id":"5RDAiTapkdrc"}},{"cell_type":"code","source":"from prettytable import PrettyTable\nt = PrettyTable()\nt.field_names=[\"model\",\"train_f1_Score\",\"test_f1_score\"]\nt.add_row([\"XGBoost\",\"0.9920553921274338\",\"0.9124241511382708\"])\nprint(t)","metadata":{"id":"s8TTT3Vokcdo","outputId":"f1382d20-5ad2-4992-a5b4-31009493791e","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reference","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}