{"metadata":{"colab":{"provenance":[]},"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.7.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":7288234,"sourceType":"datasetVersion","datasetId":4226600}],"dockerImageVersionId":30017,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Import Library","metadata":{"id":"jnguorlBlMyB"}},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\nimport csv\nimport pandas as pd\nimport datetime\nimport time\nimport numpy as np\nimport matplotlib\nimport matplotlib.pylab as plt\nimport seaborn as sns\nfrom matplotlib import rcParams\nfrom sklearn.cluster import MiniBatchKMeans, KMeans\nimport math\nimport pickle\nimport os\nimport xgboost as xgb\nimport networkx as nx\nimport pdb\nimport pickle\nfrom pandas import HDFStore,DataFrame\nfrom pandas import read_hdf\nfrom scipy.sparse.linalg import svds, eigs\nimport gc\nfrom tqdm import tqdm\nfrom collections import Counter\nfrom sklearn.model_selection import train_test_split\nimport random\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import f1_score, accuracy_score, precision_score, recall_score\nfrom pandas import read_hdf\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.svm import LinearSVC\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.preprocessing import OneHotEncoder","metadata":{"id":"Ne2entiLEiiJ","outputId":"bfd9492b-27c7-4f83-abd3-f617c245ea77","execution":{"iopub.status.busy":"2023-12-27T06:30:37.731001Z","iopub.execute_input":"2023-12-27T06:30:37.731273Z","iopub.status.idle":"2023-12-27T06:30:41.251350Z","shell.execute_reply.started":"2023-12-27T06:30:37.731245Z","shell.execute_reply":"2023-12-27T06:30:41.250645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset Loading","metadata":{"id":"bpBXt1PnbGyh"}},{"cell_type":"code","source":"if not os.path.isfile('train_woheader.csv'):\n    traincsv = pd.read_csv('/kaggle/input/twitter-link-dataset/twitter.csv')\n    print(traincsv[traincsv.isna().any(1)])\n    print(traincsv.info())\n    print(\"Number of diplicate entries: \",sum(traincsv.duplicated()))\n    traincsv.to_csv('train_woheader.csv',header=False,index=False)\n    print(\"saved the graph into file\")\n    g=nx.read_edgelist('train_woheader.csv',delimiter=',',create_using=nx.DiGraph(),nodetype=int)\nelse:\n    g=nx.read_edgelist('train_woheader.csv',delimiter=',',create_using=nx.DiGraph(),nodetype=int)\nif not os.path.isfile('train_woheader_sample.csv'):\n   traincsv = pd.read_csv('/kaggle/input/twitter-link-dataset/twitter.csv', nrows=50).to_csv('train_woheader_sample.csv',header=False,index=False)\nsubgraph=nx.read_edgelist('train_woheader_sample.csv',delimiter=',',create_using=nx.DiGraph(),nodetype=int)\npos=nx.spring_layout(subgraph)\nnx.draw(subgraph,pos,node_color='#A0CBE2',edge_color='#00bb5e',width=1,edge_cmap=plt.cm.Blues,with_labels=True)\nplt.savefig(\"graph_sample.pdf\")\nprint(\"Number of nodes:\", subgraph.number_of_nodes())\nprint(\"Number of edges:\", subgraph.number_of_edges())\naverage_in_degree = sum(dict(subgraph.in_degree()).values()) / subgraph.number_of_nodes()\naverage_out_degree = sum(dict(subgraph.out_degree()).values()) / subgraph.number_of_nodes()\nprint(\"Average In-Degree:\", average_in_degree)\nprint(\"Average Out-Degree:\", average_out_degree)","metadata":{"id":"_TDsS9FgbJDy","outputId":"7992afeb-9f54-4089-a05f-6b008b9f9f50","execution":{"iopub.status.busy":"2023-12-27T06:30:41.253414Z","iopub.execute_input":"2023-12-27T06:30:41.253707Z","iopub.status.idle":"2023-12-27T06:31:03.647825Z","shell.execute_reply.started":"2023-12-27T06:30:41.253676Z","shell.execute_reply":"2023-12-27T06:31:03.646925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Exploratory Data Analysis","metadata":{"id":"tpEi-1SF5nrJ"}},{"cell_type":"markdown","source":"### No. of Nodes","metadata":{"id":"qWX5EcO55rDk"}},{"cell_type":"code","source":"print(\"The number of unique persons\",len(g.nodes()))","metadata":{"id":"usfuQ_ke5ov7","outputId":"b6badd41-2e48-439f-fece-8c502a7ad20b","execution":{"iopub.status.busy":"2023-12-27T06:31:03.649602Z","iopub.execute_input":"2023-12-27T06:31:03.650016Z","iopub.status.idle":"2023-12-27T06:31:03.658231Z","shell.execute_reply.started":"2023-12-27T06:31:03.649974Z","shell.execute_reply":"2023-12-27T06:31:03.657273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### No of followers for each person","metadata":{"id":"nX3nG9ogAR8g"}},{"cell_type":"code","source":"indegree_dist = list(dict(g.in_degree()).values())\nindegree_dist.sort()\nplt.figure(figsize=(10,6))\nplt.plot(indegree_dist)\nplt.xlabel('Index No')\nplt.ylabel('No Of Followers')\nplt.show()\nindegree_dist = list(dict(g.in_degree()).values())\nindegree_dist.sort()\nplt.figure(figsize=(10,6))\nplt.plot(indegree_dist[0:1500000])\nplt.xlabel('Index No')\nplt.ylabel('No Of Followers')\nplt.show()\nplt.boxplot(indegree_dist)\nplt.ylabel('No Of Followers')\nplt.show()","metadata":{"id":"smJbTV-UASn4","outputId":"74e746ce-b2c9-42e0-8036-afbc50aa492d","execution":{"iopub.status.busy":"2023-12-27T06:31:03.659882Z","iopub.execute_input":"2023-12-27T06:31:03.660288Z","iopub.status.idle":"2023-12-27T06:31:04.306032Z","shell.execute_reply.started":"2023-12-27T06:31:03.660243Z","shell.execute_reply":"2023-12-27T06:31:04.305280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"###  probability density plot","metadata":{"id":"EIyx9KMqh6C6"}},{"cell_type":"code","source":"%matplotlib inline\nsns.set_style('ticks')\nfig, ax = plt.subplots()\nfig.set_size_inches(11.7, 8.27)\nsns.distplot(indegree_dist, color='#16A085')\nplt.xlabel('PDF of Indegree')\nsns.despine()","metadata":{"id":"rR1g71XKhya_","outputId":"74a7e283-b473-42b9-cd11-944acf2bfa84","execution":{"iopub.status.busy":"2023-12-27T06:31:04.309391Z","iopub.execute_input":"2023-12-27T06:31:04.309699Z","iopub.status.idle":"2023-12-27T06:31:04.591708Z","shell.execute_reply.started":"2023-12-27T06:31:04.309669Z","shell.execute_reply":"2023-12-27T06:31:04.590855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### No of people each person is following","metadata":{"id":"Njdenk52iDFW"}},{"cell_type":"code","source":"outdegree_dist = list(dict(g.out_degree()).values())\noutdegree_dist.sort()\nplt.figure(figsize=(10,6))\nplt.plot(outdegree_dist)\nplt.xlabel('Index No')\nplt.ylabel('No Of people each person is following')\nplt.show()\nindegree_dist = list(dict(g.in_degree()).values())\nindegree_dist.sort()\nplt.figure(figsize=(10,6))\nplt.plot(outdegree_dist[0:1500000])\nplt.xlabel('Index No')\nplt.ylabel('No Of people each person is following')\nplt.show()\nplt.boxplot(indegree_dist)\nplt.ylabel('No Of people each person is following')\nplt.show()","metadata":{"id":"gxqiqHg6h7Ix","outputId":"002d1b03-3aa1-431b-bfdb-2a9cad50fd05","execution":{"iopub.status.busy":"2023-12-27T06:31:04.594534Z","iopub.execute_input":"2023-12-27T06:31:04.594811Z","iopub.status.idle":"2023-12-27T06:31:05.203205Z","shell.execute_reply.started":"2023-12-27T06:31:04.594782Z","shell.execute_reply":"2023-12-27T06:31:05.202350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### probability density plot","metadata":{"id":"j-ebFhYqia1F"}},{"cell_type":"code","source":"sns.set_style('ticks')\nfig, ax = plt.subplots()\nfig.set_size_inches(11.7, 8.27)\nsns.distplot(outdegree_dist, color='#16A085')\nplt.xlabel('PDF of Outdegree')\nsns.despine()","metadata":{"id":"Z7jeFYhwiFHa","outputId":"6838930b-ea9a-4ce0-f851-070fac7415aa","execution":{"iopub.status.busy":"2023-12-27T06:31:05.204329Z","iopub.execute_input":"2023-12-27T06:31:05.204598Z","iopub.status.idle":"2023-12-27T06:31:05.472351Z","shell.execute_reply.started":"2023-12-27T06:31:05.204572Z","shell.execute_reply":"2023-12-27T06:31:05.471473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### No of persons those are not following anyone are","metadata":{"id":"x4RtoISOikMb"}},{"cell_type":"code","source":"print('No of persons those are not following anyone are' ,sum(np.array(outdegree_dist)==0),'and % is',\n                                sum(np.array(outdegree_dist)==0)*100/len(outdegree_dist) )","metadata":{"id":"jhjP2NRwidIX","outputId":"e6a39e4e-b979-4146-c80c-8e789b48d0a4","execution":{"iopub.status.busy":"2023-12-27T06:31:05.473661Z","iopub.execute_input":"2023-12-27T06:31:05.474133Z","iopub.status.idle":"2023-12-27T06:31:06.119362Z","shell.execute_reply.started":"2023-12-27T06:31:05.474099Z","shell.execute_reply":"2023-12-27T06:31:06.118529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### No of persons having zero followers are","metadata":{"id":"15_9slswin_u"}},{"cell_type":"code","source":"print('No of persons having zero followers are' ,sum(np.array(indegree_dist)==0),'and % is',\n                                sum(np.array(indegree_dist)==0)*100/len(indegree_dist) )","metadata":{"id":"UtENMiTBilL9","outputId":"4d705b41-3a23-4c83-aa31-645704f89ef4","execution":{"iopub.status.busy":"2023-12-27T06:31:06.120684Z","iopub.execute_input":"2023-12-27T06:31:06.120974Z","iopub.status.idle":"2023-12-27T06:31:06.748430Z","shell.execute_reply.started":"2023-12-27T06:31:06.120945Z","shell.execute_reply":"2023-12-27T06:31:06.747670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### No of persons those are not not following anyone and also not having any followers are","metadata":{"id":"QsQCprNnitxR"}},{"cell_type":"code","source":"count=0\nfor i in g.nodes():\n    if len(list(g.predecessors(i)))==0 :\n        if len(list(g.successors(i)))==0:\n            count+=1\nprint('No of persons those are not not following anyone and also not having any followers are',count)","metadata":{"id":"PLcvtU-8iqJP","outputId":"bfd6e918-9daa-4a20-e2eb-3c00b40e0132","execution":{"iopub.status.busy":"2023-12-27T06:31:06.749469Z","iopub.execute_input":"2023-12-27T06:31:06.749725Z","iopub.status.idle":"2023-12-27T06:31:06.964046Z","shell.execute_reply.started":"2023-12-27T06:31:06.749700Z","shell.execute_reply":"2023-12-27T06:31:06.963112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### both followers + following","metadata":{"id":"JXgTWgUei3Pb"}},{"cell_type":"code","source":"dict_in = dict(g.in_degree())\ndict_out = dict(g.out_degree())\nd = Counter(dict_in) + Counter(dict_out)\nin_out_degree = np.array(list(d.values()))\nin_out_degree_sort = sorted(in_out_degree)\nplt.figure(figsize=(10,6))\nplt.plot(in_out_degree_sort)\nplt.xlabel('Index No')\nplt.ylabel('No Of people each person is following + followers')\nplt.show()\nin_out_degree_sort = sorted(in_out_degree)\nplt.figure(figsize=(10,6))\nplt.plot(in_out_degree_sort[0:1500000])\nplt.xlabel('Index No')\nplt.ylabel('No Of people each person is following + followers')\nplt.show()","metadata":{"id":"yYJ6ghfhivqq","outputId":"927e3ad2-81ea-4f7f-eb36-336cd412800b","execution":{"iopub.status.busy":"2023-12-27T06:31:06.965260Z","iopub.execute_input":"2023-12-27T06:31:06.965535Z","iopub.status.idle":"2023-12-27T06:31:08.099273Z","shell.execute_reply.started":"2023-12-27T06:31:06.965507Z","shell.execute_reply":"2023-12-27T06:31:08.098216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Min and Max of no of followers + following is","metadata":{"id":"xIJibMTojOTa"}},{"cell_type":"code","source":"print('Min of no of followers + following is',in_out_degree.min())\nprint(np.sum(in_out_degree==in_out_degree.min()),' persons having minimum no of followers + following')\nprint('Max of no of followers + following is',in_out_degree.max())\nprint(np.sum(in_out_degree==in_out_degree.max()),' persons having maximum no of followers + following')\nprint('No of persons having followers + following less than 10 are',np.sum(in_out_degree<10))\nprint('No of weakly connected components',len(list(nx.weakly_connected_components(g))))\ncount=0\nfor i in list(nx.weakly_connected_components(g)):\n    if len(i)==2:\n        count+=1\nprint('weakly connected components wit 2 nodes',count)","metadata":{"id":"aqKT_JwAi_kI","outputId":"a28202a6-9dca-4619-c730-dc0666ceaebf","execution":{"iopub.status.busy":"2023-12-27T06:31:08.100408Z","iopub.execute_input":"2023-12-27T06:31:08.100695Z","iopub.status.idle":"2023-12-27T06:31:09.979687Z","shell.execute_reply.started":"2023-12-27T06:31:08.100665Z","shell.execute_reply":"2023-12-27T06:31:09.978980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Generated Bad links from graph which are not in graph and whose shortest path is greater than 2 for prediciton problem","metadata":{"id":"nX14bvtwky3C"}},{"cell_type":"code","source":"%%time\nimport random\nif not os.path.isfile('missing_edges_final.p'):\n    r = csv.reader(open('train_woheader.csv','r'))\n    edges = dict()\n    for edge in r:\n        edges[(edge[0], edge[1])] = 1\n    missing_edges = set([])\n    while (len(missing_edges)<9437519):\n        a=random.randint(1, 1862220)\n        b=random.randint(1, 1862220)\n        tmp = edges.get((a,b),-1)\n        if tmp == -1 and a!=b:\n            try:\n                if nx.shortest_path_length(g,source=a,target=b) > 2:\n                    missing_edges.add((a,b))\n                else:\n                    continue\n            except:\n                    missing_edges.add((a,b))\n        else:\n            continue\n    pickle.dump(missing_edges,open('missing_edges_final.p','wb'))\nelse:\n    missing_edges = pickle.load(open('missing_edges_final.p','rb'))","metadata":{"id":"NtAoPCCKjUTP","outputId":"b3b7e948-d040-4bdf-e646-899baee3e8e2","execution":{"iopub.status.busy":"2023-12-27T06:31:09.980690Z","iopub.execute_input":"2023-12-27T06:31:09.980997Z","iopub.status.idle":"2023-12-27T06:32:41.577299Z","shell.execute_reply.started":"2023-12-27T06:31:09.980967Z","shell.execute_reply":"2023-12-27T06:32:41.576506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Training and Test data split","metadata":{"id":"Pam37m4ceQLk"}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nif (not os.path.isfile('train_pos_after_eda.csv')) and (not os.path.isfile('test_pos_after_eda.csv')):\n    df_pos = pd.read_csv('/kaggle/input/twitter-link-dataset/twitter.csv')\n    df_neg = pd.DataFrame(list(missing_edges), columns=['source_node', 'destination_node'])\n    print(\"Number of nodes in the graph with edges\", df_pos.shape[0])\n    print(\"Number of nodes in the graph without edges\", df_neg.shape[0])\n    X_train_pos, X_test_pos, y_train_pos, y_test_pos  = train_test_split(df_pos,np.ones(len(df_pos)),test_size=0.2, random_state=9)\n    X_train_neg, X_test_neg, y_train_neg, y_test_neg  = train_test_split(df_neg,np.zeros(len(df_neg)),test_size=0.2, random_state=9)\n    print('='*60)\n    print(\"Number of nodes in the train data graph with edges\", X_train_pos.shape[0],\"=\",y_train_pos.shape[0])\n    print(\"Number of nodes in the train data graph without edges\", X_train_neg.shape[0],\"=\", y_train_neg.shape[0])\n    print('='*60)\n    print(\"Number of nodes in the test data graph with edges\", X_test_pos.shape[0],\"=\",y_test_pos.shape[0])\n    print(\"Number of nodes in the test data graph without edges\", X_test_neg.shape[0],\"=\",y_test_neg.shape[0])\n    X_train_pos.to_csv('train_pos_after_eda.csv',header=False, index=False)\n    X_test_pos.to_csv('test_pos_after_eda.csv',header=False, index=False)\n    X_train_neg.to_csv('train_neg_after_eda.csv',header=False, index=False)\n    X_test_neg.to_csv('test_neg_after_eda.csv',header=False, index=False)\nelse:\n    del missing_edges\nif (os.path.isfile('train_pos_after_eda.csv')) and (os.path.isfile('test_pos_after_eda.csv')):\n    train_graph=nx.read_edgelist('train_pos_after_eda.csv',delimiter=',',create_using=nx.DiGraph(),nodetype=int)\n    test_graph=nx.read_edgelist('test_pos_after_eda.csv',delimiter=',',create_using=nx.DiGraph(),nodetype=int)\n    train_nodes_pos = set(train_graph.nodes())\n    test_nodes_pos = set(test_graph.nodes())\n    trY_teY = len(train_nodes_pos.intersection(test_nodes_pos))\n    trY_teN = len(train_nodes_pos - test_nodes_pos)\n    teY_trN = len(test_nodes_pos - train_nodes_pos)\n    print('no of people common in train and test -- ',trY_teY)\n    print('no of people present in train but not present in test -- ',trY_teN)\n    print('no of people present in test but not present in train -- ',teY_trN)\n    print(' % of people not there in Train but exist in Test in total Test data are {} %'.format(teY_trN/len(test_nodes_pos)*100))\nif (not os.path.isfile('train_after_eda.csv')) and \\\n(not os.path.isfile('test_after_eda.csv')) and \\\n(not os.path.isfile('train_y.csv')) and \\\n(not os.path.isfile('test_y.csv')) and \\\n(os.path.isfile('train_pos_after_eda.csv')) and \\\n(os.path.isfile('test_pos_after_eda.csv')) and \\\n(os.path.isfile('train_neg_after_eda.csv')) and \\\n(os.path.isfile('test_neg_after_eda.csv')):\n    X_train_pos = pd.read_csv('train_pos_after_eda.csv', names=['source_node', 'destination_node'])\n    X_test_pos = pd.read_csv('test_pos_after_eda.csv', names=['source_node', 'destination_node'])\n    X_train_neg = pd.read_csv('train_neg_after_eda.csv', names=['source_node', 'destination_node'])\n    X_test_neg = pd.read_csv('test_neg_after_eda.csv', names=['source_node', 'destination_node'])\n    print('='*60)\n    print(\"Number of nodes in the train data graph with edges\", X_train_pos.shape[0])\n    print(\"Number of nodes in the train data graph without edges\", X_train_neg.shape[0])\n    print('='*60)\n    print(\"Number of nodes in the test data graph with edges\", X_test_pos.shape[0])\n    print(\"Number of nodes in the test data graph without edges\", X_test_neg.shape[0])\n    X_train = X_train_pos.append(X_train_neg,ignore_index=True)\n    y_train = np.concatenate((y_train_pos,y_train_neg))\n    X_test = X_test_pos.append(X_test_neg,ignore_index=True)\n    y_test = np.concatenate((y_test_pos,y_test_neg))\n    X_train.to_csv('train_after_eda.csv',header=False,index=False)\n    X_test.to_csv('test_after_eda.csv',header=False,index=False)\n    pd.DataFrame(y_train.astype(int)).to_csv('train_y.csv',header=False,index=False)\n    pd.DataFrame(y_test.astype(int)).to_csv('test_y.csv',header=False,index=False)\nif os.path.isfile('train_pos_after_eda.csv'):\n    train_graph=nx.read_edgelist('train_pos_after_eda.csv',delimiter=',',create_using=nx.DiGraph(),nodetype=int)\nelse:\n    pass\ndef jaccard_for_followees(a,b):\n    try:\n        if len(set(train_graph.successors(a))) == 0  | len(set(train_graph.successors(b))) == 0:\n            return 0\n        sim = (len(set(train_graph.successors(a)).intersection(set(train_graph.successors(b)))))/(len(set(train_graph.successors(a)).union(set(train_graph.successors(b)))))\n    except:\n        return 0\n    return sim\ndef jaccard_for_followers(a,b):\n    try:\n        if len(set(train_graph.predecessors(a))) == 0  | len(set(g.predecessors(b))) == 0:\n            return 0\n        sim = (len(set(train_graph.predecessors(a)).intersection(set(train_graph.predecessors(b)))))/(len(set(train_graph.predecessors(a)).union(set(train_graph.predecessors(b)))))\n        return sim\n    except:\n        return 0\ndef cosine_for_followees(a,b):\n    try:\n        if len(set(train_graph.successors(a))) == 0  | len(set(train_graph.successors(b))) == 0:\n            return 0\n        sim = (len(set(train_graph.successors(a)).intersection(set(train_graph.successors(b)))))/(math.sqrt(len(set(train_graph.successors(a)))*len((set(train_graph.successors(b))))))\n        return sim\n    except:\n        return 0\ndef cosine_for_followers(a,b):\n    try:\n        if len(set(train_graph.predecessors(a))) == 0  | len(set(train_graph.predecessors(b))) == 0:\n            return 0\n        sim = (len(set(train_graph.predecessors(a)).intersection(set(train_graph.predecessors(b)))))/(math.sqrt(len(set(train_graph.predecessors(a))))*(len(set(train_graph.predecessors(b)))))\n        return sim\n    except:\n        return 0\nif not os.path.isfile('page_rank.p'):\n    pr = nx.pagerank(train_graph, alpha=0.85)\n    pickle.dump(pr,open('page_rank.p','wb'))\nelse:\n    pr = pickle.load(open('page_rank.p','rb'))\nmean_pr = float(sum(pr.values())) / len(pr)\ndef compute_shortest_path_length(a,b):\n    p=-1\n    try:\n        if train_graph.has_edge(a,b):\n            train_graph.remove_edge(a,b)\n            p= nx.shortest_path_length(train_graph,source=a,target=b)\n            train_graph.add_edge(a,b)\n        else:\n            p= nx.shortest_path_length(train_graph,source=a,target=b)\n        return p\n    except:\n        return -1\nwcc=list(nx.weakly_connected_components(train_graph))\ndef belongs_to_same_wcc(a,b):\n    index = []\n    if train_graph.has_edge(b,a):\n        return 1\n    if train_graph.has_edge(a,b):\n            for i in wcc:\n                if a in i:\n                    index= i\n                    break\n            if (b in index):\n                train_graph.remove_edge(a,b)\n                if compute_shortest_path_length(a,b)==-1:\n                    train_graph.add_edge(a,b)\n                    return 0\n                else:\n                    train_graph.add_edge(a,b)\n                    return 1\n            else:\n                return 0\n    else:\n            for i in wcc:\n                if a in i:\n                    index= i\n                    break\n            if(b in index):\n                return 1\n            else:\n                return 0\ndef calc_adar_in(a,b):\n    sum=0\n    try:\n        n=list(set(train_graph.successors(a)).intersection(set(train_graph.successors(b))))\n        if len(n)!=0:\n            for i in n:\n                sum=sum+(1/np.log10(len(list(train_graph.predecessors(i)))))\n            return sum\n        else:\n            return 0\n    except:\n        return 0\ndef follows_back(a,b):\n    if train_graph.has_edge(b,a):\n        return 1\n    else:\n        return 0\nif not os.path.isfile('katz.p'):\n    katz = nx.katz.katz_centrality(train_graph,alpha=0.005,beta=1)\n    pickle.dump(katz,open('katz.p','wb'))\nelse:\n    katz = pickle.load(open('katz.p','rb'))\nmean_katz = float(sum(katz.values())) / len(katz)\nif not os.path.isfile('hits.p'):\n    hits = nx.hits(train_graph, max_iter=100, tol=1e-08, nstart=None, normalized=True)\n    pickle.dump(hits,open('hits.p','wb'))\nelse:\n    hits = pickle.load(open('hits.p','rb'))\nif os.path.isfile('train_after_eda.csv'):\n    filename = \"train_after_eda.csv\"\n    n_train =  15100028\n    s = 100000\n    skip_train = sorted(random.sample(range(1,n_train+1),n_train-s))\nif os.path.isfile('train_after_eda.csv'):\n    filename = \"test_after_eda.csv\"\n    n_test = 3775006\n    s = 50000\n    skip_test = sorted(random.sample(range(1,n_test+1),n_test-s))\ndf_final_train = pd.read_csv('train_after_eda.csv', skiprows=skip_train, names=['source_node', 'destination_node'])\ndf_final_train['indicator_link'] = pd.read_csv('train_y.csv', skiprows=skip_train, names=['indicator_link'])\ndf_final_test = pd.read_csv('test_after_eda.csv', skiprows=skip_test, names=['source_node', 'destination_node'])\ndf_final_test['indicator_link'] = pd.read_csv('test_y.csv', skiprows=skip_test, names=['indicator_link'])\nif not os.path.isfile('storage_sample_stage1.h5'):\n    df_final_train['jaccard_followers'] = df_final_train.apply(lambda row:\n                                            jaccard_for_followers(row['source_node'],row['destination_node']),axis=1)\n    df_final_test['jaccard_followers'] = df_final_test.apply(lambda row:\n                                            jaccard_for_followers(row['source_node'],row['destination_node']),axis=1)\n    df_final_train['jaccard_followees'] = df_final_train.apply(lambda row:\n                                            jaccard_for_followees(row['source_node'],row['destination_node']),axis=1)\n    df_final_test['jaccard_followees'] = df_final_test.apply(lambda row:\n                                            jaccard_for_followees(row['source_node'],row['destination_node']),axis=1)\n    df_final_train['cosine_followers'] = df_final_train.apply(lambda row:\n                                            cosine_for_followers(row['source_node'],row['destination_node']),axis=1)\n    df_final_test['cosine_followers'] = df_final_test.apply(lambda row:\n                                            cosine_for_followers(row['source_node'],row['destination_node']),axis=1)\n    df_final_train['cosine_followees'] = df_final_train.apply(lambda row:\n                                            cosine_for_followees(row['source_node'],row['destination_node']),axis=1)\n    df_final_test['cosine_followees'] = df_final_test.apply(lambda row:\n                                            cosine_for_followees(row['source_node'],row['destination_node']),axis=1)\ndef compute_features_stage1(df_final):\n    num_followers_s=[]\n    num_followees_s=[]\n    num_followers_d=[]\n    num_followees_d=[]\n    inter_followers=[]\n    inter_followees=[]\n    for i,row in df_final.iterrows():\n        try:\n            s1=set(train_graph.predecessors(row['source_node']))\n            s2=set(train_graph.successors(row['source_node']))\n        except:\n            s1 = set()\n            s2 = set()\n        try:\n            d1=set(train_graph.predecessors(row['destination_node']))\n            d2=set(train_graph.successors(row['destination_node']))\n        except:\n            d1 = set()\n            d2 = set()\n        num_followers_s.append(len(s1))\n        num_followees_s.append(len(s2))\n        num_followers_d.append(len(d1))\n        num_followees_d.append(len(d2))\n        inter_followers.append(len(s1.intersection(d1)))\n        inter_followees.append(len(s2.intersection(d2)))\n    return num_followers_s, num_followers_d, num_followees_s, num_followees_d, inter_followers, inter_followees\nif not os.path.isfile('data/fea_sample/storage_sample_stage1.h5'):\n    df_final_train['num_followers_s'], df_final_train['num_followers_d'], \\\n    df_final_train['num_followees_s'], df_final_train['num_followees_d'], \\\n    df_final_train['inter_followers'], df_final_train['inter_followees']= compute_features_stage1(df_final_train)\n    df_final_test['num_followers_s'], df_final_test['num_followers_d'], \\\n    df_final_test['num_followees_s'], df_final_test['num_followees_d'], \\\n    df_final_test['inter_followers'], df_final_test['inter_followees']= compute_features_stage1(df_final_test)\n    hdf = HDFStore('storage_sample_stage1.h5')\n    hdf.put('train_df',df_final_train, format='table', data_columns=True)\n    hdf.put('test_df',df_final_test, format='table', data_columns=True)\n    hdf.close()\nelse:\n    df_final_train = read_hdf('storage_sample_stage1.h5', 'train_df',mode='r')\n    df_final_test = read_hdf('storage_sample_stage1.h5', 'test_df',mode='r')\nif not os.path.isfile('storage_sample_stage2.h5'):\n    df_final_train['adar_index'] = df_final_train.apply(lambda row: calc_adar_in(row['source_node'],row['destination_node']),axis=1)\n    df_final_test['adar_index'] = df_final_test.apply(lambda row: calc_adar_in(row['source_node'],row['destination_node']),axis=1)\n    df_final_train['follows_back'] = df_final_train.apply(lambda row: follows_back(row['source_node'],row['destination_node']),axis=1)\n    df_final_test['follows_back'] = df_final_test.apply(lambda row: follows_back(row['source_node'],row['destination_node']),axis=1)\n    df_final_train['same_comp'] = df_final_train.apply(lambda row: belongs_to_same_wcc(row['source_node'],row['destination_node']),axis=1)\n    df_final_test['same_comp'] = df_final_test.apply(lambda row: belongs_to_same_wcc(row['source_node'],row['destination_node']),axis=1)\n    df_final_train['shortest_path'] = df_final_train.apply(lambda row: compute_shortest_path_length(row['source_node'],row['destination_node']),axis=1)\n    df_final_test['shortest_path'] = df_final_test.apply(lambda row: compute_shortest_path_length(row['source_node'],row['destination_node']),axis=1)\n    hdf = HDFStore('storage_sample_stage2.h5')\n    hdf.put('train_df',df_final_train, format='table', data_columns=True)\n    hdf.put('test_df',df_final_test, format='table', data_columns=True)\n    hdf.close()\nelse:\n    df_final_train = read_hdf('storage_sample_stage2.h5', 'train_df',mode='r')\n    df_final_test = read_hdf('storage_sample_stage2.h5', 'test_df',mode='r')\nWeight_in = {}\nWeight_out = {}\nfor i in  tqdm(train_graph.nodes()):\n    s1=set(train_graph.predecessors(i))\n    w_in = 1.0/(np.sqrt(1+len(s1)))\n    Weight_in[i]=w_in\n\n    s2=set(train_graph.successors(i))\n    w_out = 1.0/(np.sqrt(1+len(s2)))\n    Weight_out[i]=w_out\nmean_weight_in = np.mean(list(Weight_in.values()))\nmean_weight_out = np.mean(list(Weight_out.values()))\nif not os.path.isfile('storage_sample_stage3.h5'):\n    df_final_train['weight_in'] = df_final_train.destination_node.apply(lambda x: Weight_in.get(x,mean_weight_in))\n    df_final_train['weight_out'] = df_final_train.source_node.apply(lambda x: Weight_out.get(x,mean_weight_out))\n    df_final_test['weight_in'] = df_final_test.destination_node.apply(lambda x: Weight_in.get(x,mean_weight_in))\n    df_final_test['weight_out'] = df_final_test.source_node.apply(lambda x: Weight_out.get(x,mean_weight_out))\n    df_final_train['weight_f1'] = df_final_train.weight_in + df_final_train.weight_out\n    df_final_train['weight_f2'] = df_final_train.weight_in * df_final_train.weight_out\n    df_final_train['weight_f3'] = (2*df_final_train.weight_in + 1*df_final_train.weight_out)\n    df_final_train['weight_f4'] = (1*df_final_train.weight_in + 2*df_final_train.weight_out)\n    df_final_test['weight_f1'] = df_final_test.weight_in + df_final_test.weight_out\n    df_final_test['weight_f2'] = df_final_test.weight_in * df_final_test.weight_out\n    df_final_test['weight_f3'] = (2*df_final_test.weight_in + 1*df_final_test.weight_out)\n    df_final_test['weight_f4'] = (1*df_final_test.weight_in + 2*df_final_test.weight_out)\nif not os.path.isfile('storage_sample_stage3.h5'):\n    df_final_train['page_rank_s'] = df_final_train.source_node.apply(lambda x:pr.get(x,mean_pr))\n    df_final_train['page_rank_d'] = df_final_train.destination_node.apply(lambda x:pr.get(x,mean_pr))\n    df_final_test['page_rank_s'] = df_final_test.source_node.apply(lambda x:pr.get(x,mean_pr))\n    df_final_test['page_rank_d'] = df_final_test.destination_node.apply(lambda x:pr.get(x,mean_pr))\n    df_final_train['katz_s'] = df_final_train.source_node.apply(lambda x: katz.get(x,mean_katz))\n    df_final_train['katz_d'] = df_final_train.destination_node.apply(lambda x: katz.get(x,mean_katz))\n    df_final_test['katz_s'] = df_final_test.source_node.apply(lambda x: katz.get(x,mean_katz))\n    df_final_test['katz_d'] = df_final_test.destination_node.apply(lambda x: katz.get(x,mean_katz))\n    df_final_train['hubs_s'] = df_final_train.source_node.apply(lambda x: hits[0].get(x,0))\n    df_final_train['hubs_d'] = df_final_train.destination_node.apply(lambda x: hits[0].get(x,0))\n    df_final_test['hubs_s'] = df_final_test.source_node.apply(lambda x: hits[0].get(x,0))\n    df_final_test['hubs_d'] = df_final_test.destination_node.apply(lambda x: hits[0].get(x,0))\n    df_final_train['authorities_s'] = df_final_train.source_node.apply(lambda x: hits[1].get(x,0))\n    df_final_train['authorities_d'] = df_final_train.destination_node.apply(lambda x: hits[1].get(x,0))\n    df_final_test['authorities_s'] = df_final_test.source_node.apply(lambda x: hits[1].get(x,0))\n    df_final_test['authorities_d'] = df_final_test.destination_node.apply(lambda x: hits[1].get(x,0))\n    hdf = HDFStore('storage_sample_stage3.h5')\n    hdf.put('train_df',df_final_train, format='table', data_columns=True)\n    hdf.put('test_df',df_final_test, format='table', data_columns=True)\n    hdf.close()\nelse:\n    df_final_train = read_hdf('storage_sample_stage3.h5', 'train_df',mode='r')\n    df_final_test = read_hdf('storage_sample_stage3.h5', 'test_df',mode='r')\ndef svd(x, S):\n    try:\n        z = sadj_dict[x]\n        return S[z]\n    except:\n        return [0,0,0,0,0,0]\nsadj_col = sorted(train_graph.nodes())\nsadj_dict = { val:idx for idx,val in enumerate(sadj_col)}\nAdj = nx.adjacency_matrix(train_graph,nodelist=sorted(train_graph.nodes())).asfptype()\nU, s, V = svds(Adj, k = 6)\nif not os.path.isfile('storage_sample_stage4.h5'):\n    df_final_train[['svd_u_s_1', 'svd_u_s_2','svd_u_s_3', 'svd_u_s_4', 'svd_u_s_5', 'svd_u_s_6']] = \\\n    df_final_train.source_node.apply(lambda x: svd(x, U)).apply(pd.Series)\n    df_final_train[['svd_u_d_1', 'svd_u_d_2', 'svd_u_d_3', 'svd_u_d_4', 'svd_u_d_5','svd_u_d_6']] = \\\n    df_final_train.destination_node.apply(lambda x: svd(x, U)).apply(pd.Series)\n    df_final_train[['svd_v_s_1','svd_v_s_2', 'svd_v_s_3', 'svd_v_s_4', 'svd_v_s_5', 'svd_v_s_6',]] = \\\n    df_final_train.source_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)\n    df_final_train[['svd_v_d_1', 'svd_v_d_2', 'svd_v_d_3', 'svd_v_d_4', 'svd_v_d_5','svd_v_d_6']] = \\\n    df_final_train.destination_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)\n    df_final_test[['svd_u_s_1', 'svd_u_s_2','svd_u_s_3', 'svd_u_s_4', 'svd_u_s_5', 'svd_u_s_6']] = \\\n    df_final_test.source_node.apply(lambda x: svd(x, U)).apply(pd.Series)\n    df_final_test[['svd_u_d_1', 'svd_u_d_2', 'svd_u_d_3', 'svd_u_d_4', 'svd_u_d_5','svd_u_d_6']] = \\\n    df_final_test.destination_node.apply(lambda x: svd(x, U)).apply(pd.Series)\n    df_final_test[['svd_v_s_1','svd_v_s_2', 'svd_v_s_3', 'svd_v_s_4', 'svd_v_s_5', 'svd_v_s_6',]] = \\\n    df_final_test.source_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)\n    df_final_test[['svd_v_d_1', 'svd_v_d_2', 'svd_v_d_3', 'svd_v_d_4', 'svd_v_d_5','svd_v_d_6']] = \\\n    df_final_test.destination_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)\n    hdf = HDFStore('storage_sample_stage4.h5')\n    hdf.put('train_df',df_final_train, format='table', data_columns=True)\n    hdf.put('test_df',df_final_test, format='table', data_columns=True)\n    hdf.close()\nif os.path.isfile('train_pos_after_eda.csv'):\n    train_graph=nx.read_edgelist('train_pos_after_eda.csv',delimiter=',',create_using=nx.DiGraph(),nodetype=int)\nelse:\n    pass\ndef jaccard_for_followees(a,b):\n    try:\n        if len(set(train_graph.successors(a))) == 0  | len(set(train_graph.successors(b))) == 0:\n            return 0\n        sim = (len(set(train_graph.successors(a)).intersection(set(train_graph.successors(b)))))/(len(set(train_graph.successors(a)).union(set(train_graph.successors(b)))))\n    except:\n        return 0\n    return sim\ndef jaccard_for_followers(a,b):\n    try:\n        if len(set(train_graph.predecessors(a))) == 0  | len(set(g.predecessors(b))) == 0:\n            return 0\n        sim = (len(set(train_graph.predecessors(a)).intersection(set(train_graph.predecessors(b)))))/(len(set(train_graph.predecessors(a)).union(set(train_graph.predecessors(b)))))\n        return sim\n    except:\n        return 0\ndef cosine_for_followees(a,b):\n    try:\n        if len(set(train_graph.successors(a))) == 0  | len(set(train_graph.successors(b))) == 0:\n            return 0\n        sim = (len(set(train_graph.successors(a)).intersection(set(train_graph.successors(b)))))/(math.sqrt(len(set(train_graph.successors(a)))*len((set(train_graph.successors(b))))))\n        return sim\n    except:\n        return 0\ndef cosine_for_followers(a,b):\n    try:\n\n        if len(set(train_graph.predecessors(a))) == 0  | len(set(train_graph.predecessors(b))) == 0:\n            return 0\n        sim = (len(set(train_graph.predecessors(a)).intersection(set(train_graph.predecessors(b)))))/(math.sqrt(len(set(train_graph.predecessors(a))))*(len(set(train_graph.predecessors(b)))))\n        return sim\n    except:\n        return 0\nif not os.path.isfile('page_rank.p'):\n    pr = nx.pagerank(train_graph, alpha=0.85)\n    pickle.dump(pr,open('page_rank.p','wb'))\nelse:\n    pr = pickle.load(open('page_rank.p','rb'))\nmean_pr = float(sum(pr.values())) / len(pr)\ndef compute_shortest_path_length(a,b):\n    p=-1\n    try:\n        if train_graph.has_edge(a,b):\n            train_graph.remove_edge(a,b)\n            p= nx.shortest_path_length(train_graph,source=a,target=b)\n            train_graph.add_edge(a,b)\n        else:\n            p= nx.shortest_path_length(train_graph,source=a,target=b)\n        return p\n    except:\n        return -1\nwcc=list(nx.weakly_connected_components(train_graph))\ndef belongs_to_same_wcc(a,b):\n    index = []\n    if train_graph.has_edge(b,a):\n        return 1\n    if train_graph.has_edge(a,b):\n            for i in wcc:\n                if a in i:\n                    index= i\n                    break\n            if (b in index):\n                train_graph.remove_edge(a,b)\n                if compute_shortest_path_length(a,b)==-1:\n                    train_graph.add_edge(a,b)\n                    return 0\n                else:\n                    train_graph.add_edge(a,b)\n                    return 1\n            else:\n                return 0\n    else:\n            for i in wcc:\n                if a in i:\n                    index= i\n                    break\n            if(b in index):\n                return 1\n            else:\n                return 0\ndef calc_adar_in(a,b):\n    sum=0\n    try:\n        n=list(set(train_graph.successors(a)).intersection(set(train_graph.successors(b))))\n        if len(n)!=0:\n            for i in n:\n                sum=sum+(1/np.log10(len(list(train_graph.predecessors(i)))))\n            return sum\n        else:\n            return 0\n    except:\n        return 0\ndef follows_back(a,b):\n    if train_graph.has_edge(b,a):\n        return 1\n    else:\n        return 0\nif not os.path.isfile('katz.p'):\n    katz = nx.katz.katz_centrality(train_graph,alpha=0.005,beta=1)\n    pickle.dump(katz,open('katz.p','wb'))\nelse:\n    katz = pickle.load(open('katz.p','rb'))\nmean_katz = float(sum(katz.values())) / len(katz)\nif not os.path.isfile('hits.p'):\n    hits = nx.hits(train_graph, max_iter=100, tol=1e-08, nstart=None, normalized=True)\n    pickle.dump(hits,open('hits.p','wb'))\nelse:\n    hits = pickle.load(open('hits.p','rb'))\nif os.path.isfile('train_after_eda.csv'):\n    filename = \"train_after_eda.csv\"\n    n_train =  15100028\n    s = 100000\n    skip_train = sorted(random.sample(range(1,n_train+1),n_train-s))\nif os.path.isfile('train_after_eda.csv'):\n    filename = \"test_after_eda.csv\"\n    n_test = 3775006\n    s = 50000\n    skip_test = sorted(random.sample(range(1,n_test+1),n_test-s))\ndf_final_train = pd.read_csv('train_after_eda.csv', skiprows=skip_train, names=['source_node', 'destination_node'])\ndf_final_train['indicator_link'] = pd.read_csv('train_y.csv', skiprows=skip_train, names=['indicator_link'])\ndf_final_test = pd.read_csv('test_after_eda.csv', skiprows=skip_test, names=['source_node', 'destination_node'])\ndf_final_test['indicator_link'] = pd.read_csv('test_y.csv', skiprows=skip_test, names=['indicator_link'])\nif not os.path.isfile('data/fea_sample/storage_sample_stage1.h5'):\n    df_final_train['jaccard_followers'] = df_final_train.apply(lambda row:\n                                            jaccard_for_followers(row['source_node'],row['destination_node']),axis=1)\n    df_final_test['jaccard_followers'] = df_final_test.apply(lambda row:\n                                            jaccard_for_followers(row['source_node'],row['destination_node']),axis=1)\n    df_final_train['jaccard_followees'] = df_final_train.apply(lambda row:\n                                            jaccard_for_followees(row['source_node'],row['destination_node']),axis=1)\n    df_final_test['jaccard_followees'] = df_final_test.apply(lambda row:\n                                            jaccard_for_followees(row['source_node'],row['destination_node']),axis=1)\n    df_final_train['cosine_followers'] = df_final_train.apply(lambda row:\n                                            cosine_for_followers(row['source_node'],row['destination_node']),axis=1)\n    df_final_test['cosine_followers'] = df_final_test.apply(lambda row:\n                                            cosine_for_followers(row['source_node'],row['destination_node']),axis=1)\n    df_final_train['cosine_followees'] = df_final_train.apply(lambda row:\n                                            cosine_for_followees(row['source_node'],row['destination_node']),axis=1)\n    df_final_test['cosine_followees'] = df_final_test.apply(lambda row:\n                                            cosine_for_followees(row['source_node'],row['destination_node']),axis=1)\ndef compute_features_stage1(df_final):\n    num_followers_s=[]\n    num_followees_s=[]\n    num_followers_d=[]\n    num_followees_d=[]\n    inter_followers=[]\n    inter_followees=[]\n    for i,row in df_final.iterrows():\n        try:\n            s1=set(train_graph.predecessors(row['source_node']))\n            s2=set(train_graph.successors(row['source_node']))\n        except:\n            s1 = set()\n            s2 = set()\n        try:\n            d1=set(train_graph.predecessors(row['destination_node']))\n            d2=set(train_graph.successors(row['destination_node']))\n        except:\n            d1 = set()\n            d2 = set()\n        num_followers_s.append(len(s1))\n        num_followees_s.append(len(s2))\n        num_followers_d.append(len(d1))\n        num_followees_d.append(len(d2))\n        inter_followers.append(len(s1.intersection(d1)))\n        inter_followees.append(len(s2.intersection(d2)))\n    return num_followers_s, num_followers_d, num_followees_s, num_followees_d, inter_followers, inter_followees\nif not os.path.isfile('storage_sample_stage1.h5'):\n    df_final_train['num_followers_s'], df_final_train['num_followers_d'], \\\n    df_final_train['num_followees_s'], df_final_train['num_followees_d'], \\\n    df_final_train['inter_followers'], df_final_train['inter_followees']= compute_features_stage1(df_final_train)\n    df_final_test['num_followers_s'], df_final_test['num_followers_d'], \\\n    df_final_test['num_followees_s'], df_final_test['num_followees_d'], \\\n    df_final_test['inter_followers'], df_final_test['inter_followees']= compute_features_stage1(df_final_test)\n    hdf = HDFStore('storage_sample_stage1.h5')\n    hdf.put('train_df',df_final_train, format='table', data_columns=True)\n    hdf.put('test_df',df_final_test, format='table', data_columns=True)\n    hdf.close()\nelse:\n    df_final_train = read_hdf('storage_sample_stage1.h5', 'train_df',mode='r')\n    df_final_test = read_hdf('storage_sample_stage1.h5', 'test_df',mode='r')\nif not os.path.isfile('storage_sample_stage2.h5'):\n    df_final_train['adar_index'] = df_final_train.apply(lambda row: calc_adar_in(row['source_node'],row['destination_node']),axis=1)\n    df_final_test['adar_index'] = df_final_test.apply(lambda row: calc_adar_in(row['source_node'],row['destination_node']),axis=1)\n    df_final_train['follows_back'] = df_final_train.apply(lambda row: follows_back(row['source_node'],row['destination_node']),axis=1)\n    df_final_test['follows_back'] = df_final_test.apply(lambda row: follows_back(row['source_node'],row['destination_node']),axis=1)\n    df_final_train['same_comp'] = df_final_train.apply(lambda row: belongs_to_same_wcc(row['source_node'],row['destination_node']),axis=1)\n    df_final_test['same_comp'] = df_final_test.apply(lambda row: belongs_to_same_wcc(row['source_node'],row['destination_node']),axis=1)\n    df_final_train['shortest_path'] = df_final_train.apply(lambda row: compute_shortest_path_length(row['source_node'],row['destination_node']),axis=1)\n    df_final_test['shortest_path'] = df_final_test.apply(lambda row: compute_shortest_path_length(row['source_node'],row['destination_node']),axis=1)\n    hdf = HDFStore('storage_sample_stage2.h5')\n    hdf.put('train_df',df_final_train, format='table', data_columns=True)\n    hdf.put('test_df',df_final_test, format='table', data_columns=True)\n    hdf.close()\nelse:\n    df_final_train = read_hdf('storage_sample_stage2.h5', 'train_df',mode='r')\n    df_final_test = read_hdf('storage_sample_stage2.h5', 'test_df',mode='r')\nWeight_in = {}\nWeight_out = {}\nfor i in  tqdm(train_graph.nodes()):\n    s1=set(train_graph.predecessors(i))\n    w_in = 1.0/(np.sqrt(1+len(s1)))\n    Weight_in[i]=w_in\n\n    s2=set(train_graph.successors(i))\n    w_out = 1.0/(np.sqrt(1+len(s2)))\n    Weight_out[i]=w_out\nmean_weight_in = np.mean(list(Weight_in.values()))\nmean_weight_out = np.mean(list(Weight_out.values()))\nif not os.path.isfile('storage_sample_stage3.h5'):\n    df_final_train['weight_in'] = df_final_train.destination_node.apply(lambda x: Weight_in.get(x,mean_weight_in))\n    df_final_train['weight_out'] = df_final_train.source_node.apply(lambda x: Weight_out.get(x,mean_weight_out))\n    df_final_test['weight_in'] = df_final_test.destination_node.apply(lambda x: Weight_in.get(x,mean_weight_in))\n    df_final_test['weight_out'] = df_final_test.source_node.apply(lambda x: Weight_out.get(x,mean_weight_out))\n    df_final_train['weight_f1'] = df_final_train.weight_in + df_final_train.weight_out\n    df_final_train['weight_f2'] = df_final_train.weight_in * df_final_train.weight_out\n    df_final_train['weight_f3'] = (2*df_final_train.weight_in + 1*df_final_train.weight_out)\n    df_final_train['weight_f4'] = (1*df_final_train.weight_in + 2*df_final_train.weight_out)\n    df_final_test['weight_f1'] = df_final_test.weight_in + df_final_test.weight_out\n    df_final_test['weight_f2'] = df_final_test.weight_in * df_final_test.weight_out\n    df_final_test['weight_f3'] = (2*df_final_test.weight_in + 1*df_final_test.weight_out)\n    df_final_test['weight_f4'] = (1*df_final_test.weight_in + 2*df_final_test.weight_out)\nif not os.path.isfile('storage_sample_stage3.h5'):\n    df_final_train['page_rank_s'] = df_final_train.source_node.apply(lambda x:pr.get(x,mean_pr))\n    df_final_train['page_rank_d'] = df_final_train.destination_node.apply(lambda x:pr.get(x,mean_pr))\n    df_final_test['page_rank_s'] = df_final_test.source_node.apply(lambda x:pr.get(x,mean_pr))\n    df_final_test['page_rank_d'] = df_final_test.destination_node.apply(lambda x:pr.get(x,mean_pr))\n    df_final_train['katz_s'] = df_final_train.source_node.apply(lambda x: katz.get(x,mean_katz))\n    df_final_train['katz_d'] = df_final_train.destination_node.apply(lambda x: katz.get(x,mean_katz))\n    df_final_test['katz_s'] = df_final_test.source_node.apply(lambda x: katz.get(x,mean_katz))\n    df_final_test['katz_d'] = df_final_test.destination_node.apply(lambda x: katz.get(x,mean_katz))\n    df_final_train['hubs_s'] = df_final_train.source_node.apply(lambda x: hits[0].get(x,0))\n    df_final_train['hubs_d'] = df_final_train.destination_node.apply(lambda x: hits[0].get(x,0))\n    df_final_test['hubs_s'] = df_final_test.source_node.apply(lambda x: hits[0].get(x,0))\n    df_final_test['hubs_d'] = df_final_test.destination_node.apply(lambda x: hits[0].get(x,0))\n    df_final_train['authorities_s'] = df_final_train.source_node.apply(lambda x: hits[1].get(x,0))\n    df_final_train['authorities_d'] = df_final_train.destination_node.apply(lambda x: hits[1].get(x,0))\n    df_final_test['authorities_s'] = df_final_test.source_node.apply(lambda x: hits[1].get(x,0))\n    df_final_test['authorities_d'] = df_final_test.destination_node.apply(lambda x: hits[1].get(x,0))\n    hdf = HDFStore('storage_sample_stage3.h5')\n    hdf.put('train_df',df_final_train, format='table', data_columns=True)\n    hdf.put('test_df',df_final_test, format='table', data_columns=True)\n    hdf.close()\nelse:\n    df_final_train = read_hdf('storage_sample_stage3.h5', 'train_df',mode='r')\n    df_final_test = read_hdf('storage_sample_stage3.h5', 'test_df',mode='r')\ndef svd(x, S):\n    try:\n        z = sadj_dict[x]\n        return S[z]\n    except:\n        return [0,0,0,0,0,0]\nsadj_col = sorted(train_graph.nodes())\nsadj_dict = { val:idx for idx,val in enumerate(sadj_col)}\nAdj = nx.adjacency_matrix(train_graph,nodelist=sorted(train_graph.nodes())).asfptype()\nU, s, V = svds(Adj, k = 6)\nif not os.path.isfile('storage_sample_stage4.h5'):\n    df_final_train[['svd_u_s_1', 'svd_u_s_2','svd_u_s_3', 'svd_u_s_4', 'svd_u_s_5', 'svd_u_s_6']] = \\\n    df_final_train.source_node.apply(lambda x: svd(x, U)).apply(pd.Series)\n    df_final_train[['svd_u_d_1', 'svd_u_d_2', 'svd_u_d_3', 'svd_u_d_4', 'svd_u_d_5','svd_u_d_6']] = \\\n    df_final_train.destination_node.apply(lambda x: svd(x, U)).apply(pd.Series)\n    df_final_train[['svd_v_s_1','svd_v_s_2', 'svd_v_s_3', 'svd_v_s_4', 'svd_v_s_5', 'svd_v_s_6',]] = \\\n    df_final_train.source_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)\n    df_final_train[['svd_v_d_1', 'svd_v_d_2', 'svd_v_d_3', 'svd_v_d_4', 'svd_v_d_5','svd_v_d_6']] = \\\n    df_final_train.destination_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)\n    df_final_test[['svd_u_s_1', 'svd_u_s_2','svd_u_s_3', 'svd_u_s_4', 'svd_u_s_5', 'svd_u_s_6']] = \\\n    df_final_test.source_node.apply(lambda x: svd(x, U)).apply(pd.Series)\n    df_final_test[['svd_u_d_1', 'svd_u_d_2', 'svd_u_d_3', 'svd_u_d_4', 'svd_u_d_5','svd_u_d_6']] = \\\n    df_final_test.destination_node.apply(lambda x: svd(x, U)).apply(pd.Series)\n    df_final_test[['svd_v_s_1','svd_v_s_2', 'svd_v_s_3', 'svd_v_s_4', 'svd_v_s_5', 'svd_v_s_6',]] = \\\n    df_final_test.source_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)\n    df_final_test[['svd_v_d_1', 'svd_v_d_2', 'svd_v_d_3', 'svd_v_d_4', 'svd_v_d_5','svd_v_d_6']] = \\\n    df_final_test.destination_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)\n    hdf = HDFStore('storage_sample_stage4.h5')\n    hdf.put('train_df',df_final_train, format='table', data_columns=True)\n    hdf.put('test_df',df_final_test, format='table', data_columns=True)\n    hdf.close()\nif os.path.isfile('train_pos_after_eda.csv'):\n    train_graph=nx.read_edgelist('train_pos_after_eda.csv',delimiter=',',create_using=nx.DiGraph(),nodetype=int)\nelse:\n    pass\ndef jaccard_for_followees(a,b):\n    try:\n        if len(set(train_graph.successors(a))) == 0  | len(set(train_graph.successors(b))) == 0:\n            return 0\n        sim = (len(set(train_graph.successors(a)).intersection(set(train_graph.successors(b)))))/(len(set(train_graph.successors(a)).union(set(train_graph.successors(b)))))\n    except:\n        return 0\n    return sim\ndef jaccard_for_followers(a,b):\n    try:\n        if len(set(train_graph.predecessors(a))) == 0  | len(set(g.predecessors(b))) == 0:\n            return 0\n        sim = (len(set(train_graph.predecessors(a)).intersection(set(train_graph.predecessors(b)))))/(len(set(train_graph.predecessors(a)).union(set(train_graph.predecessors(b)))))\n        return sim\n    except:\n        return 0\ndef cosine_for_followees(a,b):\n    try:\n        if len(set(train_graph.successors(a))) == 0  | len(set(train_graph.successors(b))) == 0:\n            return 0\n        sim = (len(set(train_graph.successors(a)).intersection(set(train_graph.successors(b)))))/(math.sqrt(len(set(train_graph.successors(a)))*len((set(train_graph.successors(b))))))\n        return sim\n    except:\n        return 0\ndef cosine_for_followers(a,b):\n    try:\n        if len(set(train_graph.predecessors(a))) == 0  | len(set(train_graph.predecessors(b))) == 0:\n            return 0\n        sim = (len(set(train_graph.predecessors(a)).intersection(set(train_graph.predecessors(b)))))/(math.sqrt(len(set(train_graph.predecessors(a))))*(len(set(train_graph.predecessors(b)))))\n        return sim\n    except:\n        return 0\nif not os.path.isfile('page_rank.p'):\n    pr = nx.pagerank(train_graph, alpha=0.85)\n    pickle.dump(pr,open('page_rank.p','wb'))\nelse:\n    pr = pickle.load(open('page_rank.p','rb'))\nmean_pr = float(sum(pr.values())) / len(pr)\ndef compute_shortest_path_length(a,b):\n    p=-1\n    try:\n        if train_graph.has_edge(a,b):\n            train_graph.remove_edge(a,b)\n            p= nx.shortest_path_length(train_graph,source=a,target=b)\n            train_graph.add_edge(a,b)\n        else:\n            p= nx.shortest_path_length(train_graph,source=a,target=b)\n        return p\n    except:\n        return -1\nwcc=list(nx.weakly_connected_components(train_graph))\ndef belongs_to_same_wcc(a,b):\n    index = []\n    if train_graph.has_edge(b,a):\n        return 1\n    if train_graph.has_edge(a,b):\n            for i in wcc:\n                if a in i:\n                    index= i\n                    break\n            if (b in index):\n                train_graph.remove_edge(a,b)\n                if compute_shortest_path_length(a,b)==-1:\n                    train_graph.add_edge(a,b)\n                    return 0\n                else:\n                    train_graph.add_edge(a,b)\n                    return 1\n            else:\n                return 0\n    else:\n            for i in wcc:\n                if a in i:\n                    index= i\n                    break\n            if(b in index):\n                return 1\n            else:\n                return 0\ndef calc_adar_in(a,b):\n    sum=0\n    try:\n        n=list(set(train_graph.successors(a)).intersection(set(train_graph.successors(b))))\n        if len(n)!=0:\n            for i in n:\n                sum=sum+(1/np.log10(len(list(train_graph.predecessors(i)))))\n            return sum\n        else:\n            return 0\n    except:\n        return 0\ndef follows_back(a,b):\n    if train_graph.has_edge(b,a):\n        return 1\n    else:\n        return 0\nif not os.path.isfile('katz.p'):\n    katz = nx.katz.katz_centrality(train_graph,alpha=0.005,beta=1)\n    pickle.dump(katz,open('katz.p','wb'))\nelse:\n    katz = pickle.load(open('katz.p','rb'))\nmean_katz = float(sum(katz.values())) / len(katz)\nif not os.path.isfile('hits.p'):\n    hits = nx.hits(train_graph, max_iter=100, tol=1e-08, nstart=None, normalized=True)\n    pickle.dump(hits,open('hits.p','wb'))\nelse:\n    hits = pickle.load(open('hits.p','rb'))\nif os.path.isfile('train_after_eda.csv'):\n    filename = \"train_after_eda.csv\"\n    n_train =  15100028\n    s = 100000\n    skip_train = sorted(random.sample(range(1,n_train+1),n_train-s))\nif os.path.isfile('train_after_eda.csv'):\n    filename = \"test_after_eda.csv\"\n    n_test = 3775006\n    s = 50000\n    skip_test = sorted(random.sample(range(1,n_test+1),n_test-s))\ndf_final_train = pd.read_csv('train_after_eda.csv', skiprows=skip_train, names=['source_node', 'destination_node'])\ndf_final_train['indicator_link'] = pd.read_csv('train_y.csv', skiprows=skip_train, names=['indicator_link'])\ndf_final_test = pd.read_csv('test_after_eda.csv', skiprows=skip_test, names=['source_node', 'destination_node'])\ndf_final_test['indicator_link'] = pd.read_csv('test_y.csv', skiprows=skip_test, names=['indicator_link'])\nif not os.path.isfile('storage_sample_stage1.h5'):\n    df_final_train['jaccard_followers'] = df_final_train.apply(lambda row:\n                                            jaccard_for_followers(row['source_node'],row['destination_node']),axis=1)\n    df_final_test['jaccard_followers'] = df_final_test.apply(lambda row:\n                                            jaccard_for_followers(row['source_node'],row['destination_node']),axis=1)\n    df_final_train['jaccard_followees'] = df_final_train.apply(lambda row:\n                                            jaccard_for_followees(row['source_node'],row['destination_node']),axis=1)\n    df_final_test['jaccard_followees'] = df_final_test.apply(lambda row:\n                                            jaccard_for_followees(row['source_node'],row['destination_node']),axis=1)\n    df_final_train['cosine_followers'] = df_final_train.apply(lambda row:\n                                            cosine_for_followers(row['source_node'],row['destination_node']),axis=1)\n    df_final_test['cosine_followers'] = df_final_test.apply(lambda row:\n                                            cosine_for_followers(row['source_node'],row['destination_node']),axis=1)\n    df_final_train['cosine_followees'] = df_final_train.apply(lambda row:\n                                            cosine_for_followees(row['source_node'],row['destination_node']),axis=1)\n    df_final_test['cosine_followees'] = df_final_test.apply(lambda row:\n                                            cosine_for_followees(row['source_node'],row['destination_node']),axis=1)\ndef compute_features_stage1(df_final):\n    num_followers_s=[]\n    num_followees_s=[]\n    num_followers_d=[]\n    num_followees_d=[]\n    inter_followers=[]\n    inter_followees=[]\n    for i,row in df_final.iterrows():\n        try:\n            s1=set(train_graph.predecessors(row['source_node']))\n            s2=set(train_graph.successors(row['source_node']))\n        except:\n            s1 = set()\n            s2 = set()\n        try:\n            d1=set(train_graph.predecessors(row['destination_node']))\n            d2=set(train_graph.successors(row['destination_node']))\n        except:\n            d1 = set()\n            d2 = set()\n        num_followers_s.append(len(s1))\n        num_followees_s.append(len(s2))\n        num_followers_d.append(len(d1))\n        num_followees_d.append(len(d2))\n        inter_followers.append(len(s1.intersection(d1)))\n        inter_followees.append(len(s2.intersection(d2)))\n    return num_followers_s, num_followers_d, num_followees_s, num_followees_d, inter_followers, inter_followees\nif not os.path.isfile('storage_sample_stage1.h5'):\n    df_final_train['num_followers_s'], df_final_train['num_followers_d'], \\\n    df_final_train['num_followees_s'], df_final_train['num_followees_d'], \\\n    df_final_train['inter_followers'], df_final_train['inter_followees']= compute_features_stage1(df_final_train)\n    df_final_test['num_followers_s'], df_final_test['num_followers_d'], \\\n    df_final_test['num_followees_s'], df_final_test['num_followees_d'], \\\n    df_final_test['inter_followers'], df_final_test['inter_followees']= compute_features_stage1(df_final_test)\n    hdf = HDFStore('storage_sample_stage1.h5')\n    hdf.put('train_df',df_final_train, format='table', data_columns=True)\n    hdf.put('test_df',df_final_test, format='table', data_columns=True)\n    hdf.close()\nelse:\n    df_final_train = read_hdf('storage_sample_stage1.h5', 'train_df',mode='r')\n    df_final_test = read_hdf('storage_sample_stage1.h5', 'test_df',mode='r')\nif not os.path.isfile('storage_sample_stage2.h5'):\n    df_final_train['adar_index'] = df_final_train.apply(lambda row: calc_adar_in(row['source_node'],row['destination_node']),axis=1)\n    df_final_test['adar_index'] = df_final_test.apply(lambda row: calc_adar_in(row['source_node'],row['destination_node']),axis=1)\n    df_final_train['follows_back'] = df_final_train.apply(lambda row: follows_back(row['source_node'],row['destination_node']),axis=1)\n    df_final_test['follows_back'] = df_final_test.apply(lambda row: follows_back(row['source_node'],row['destination_node']),axis=1)\n    df_final_train['same_comp'] = df_final_train.apply(lambda row: belongs_to_same_wcc(row['source_node'],row['destination_node']),axis=1)\n    df_final_test['same_comp'] = df_final_test.apply(lambda row: belongs_to_same_wcc(row['source_node'],row['destination_node']),axis=1)\n    df_final_train['shortest_path'] = df_final_train.apply(lambda row: compute_shortest_path_length(row['source_node'],row['destination_node']),axis=1)\n    df_final_test['shortest_path'] = df_final_test.apply(lambda row: compute_shortest_path_length(row['source_node'],row['destination_node']),axis=1)\n    hdf = HDFStore('storage_sample_stage2.h5')\n    hdf.put('train_df',df_final_train, format='table', data_columns=True)\n    hdf.put('test_df',df_final_test, format='table', data_columns=True)\n    hdf.close()\nelse:\n    df_final_train = read_hdf('storage_sample_stage2.h5', 'train_df',mode='r')\n    df_final_test = read_hdf('storage_sample_stage2.h5', 'test_df',mode='r')\nWeight_in = {}\nWeight_out = {}\nfor i in  tqdm(train_graph.nodes()):\n    s1=set(train_graph.predecessors(i))\n    w_in = 1.0/(np.sqrt(1+len(s1)))\n    Weight_in[i]=w_in\n    s2=set(train_graph.successors(i))\n    w_out = 1.0/(np.sqrt(1+len(s2)))\n    Weight_out[i]=w_out\nmean_weight_in = np.mean(list(Weight_in.values()))\nmean_weight_out = np.mean(list(Weight_out.values()))\nif not os.path.isfile('storage_sample_stage3.h5'):\n    df_final_train['weight_in'] = df_final_train.destination_node.apply(lambda x: Weight_in.get(x,mean_weight_in))\n    df_final_train['weight_out'] = df_final_train.source_node.apply(lambda x: Weight_out.get(x,mean_weight_out))\n    df_final_test['weight_in'] = df_final_test.destination_node.apply(lambda x: Weight_in.get(x,mean_weight_in))\n    df_final_test['weight_out'] = df_final_test.source_node.apply(lambda x: Weight_out.get(x,mean_weight_out))\n    df_final_train['weight_f1'] = df_final_train.weight_in + df_final_train.weight_out\n    df_final_train['weight_f2'] = df_final_train.weight_in * df_final_train.weight_out\n    df_final_train['weight_f3'] = (2*df_final_train.weight_in + 1*df_final_train.weight_out)\n    df_final_train['weight_f4'] = (1*df_final_train.weight_in + 2*df_final_train.weight_out)\n    df_final_test['weight_f1'] = df_final_test.weight_in + df_final_test.weight_out\n    df_final_test['weight_f2'] = df_final_test.weight_in * df_final_test.weight_out\n    df_final_test['weight_f3'] = (2*df_final_test.weight_in + 1*df_final_test.weight_out)\n    df_final_test['weight_f4'] = (1*df_final_test.weight_in + 2*df_final_test.weight_out)\nif not os.path.isfile('storage_sample_stage3.h5'):\n    df_final_train['page_rank_s'] = df_final_train.source_node.apply(lambda x:pr.get(x,mean_pr))\n    df_final_train['page_rank_d'] = df_final_train.destination_node.apply(lambda x:pr.get(x,mean_pr))\n    df_final_test['page_rank_s'] = df_final_test.source_node.apply(lambda x:pr.get(x,mean_pr))\n    df_final_test['page_rank_d'] = df_final_test.destination_node.apply(lambda x:pr.get(x,mean_pr))\n    df_final_train['katz_s'] = df_final_train.source_node.apply(lambda x: katz.get(x,mean_katz))\n    df_final_train['katz_d'] = df_final_train.destination_node.apply(lambda x: katz.get(x,mean_katz))\n    df_final_test['katz_s'] = df_final_test.source_node.apply(lambda x: katz.get(x,mean_katz))\n    df_final_test['katz_d'] = df_final_test.destination_node.apply(lambda x: katz.get(x,mean_katz))\n    df_final_train['hubs_s'] = df_final_train.source_node.apply(lambda x: hits[0].get(x,0))\n    df_final_train['hubs_d'] = df_final_train.destination_node.apply(lambda x: hits[0].get(x,0))\n    df_final_test['hubs_s'] = df_final_test.source_node.apply(lambda x: hits[0].get(x,0))\n    df_final_test['hubs_d'] = df_final_test.destination_node.apply(lambda x: hits[0].get(x,0))\n    df_final_train['authorities_s'] = df_final_train.source_node.apply(lambda x: hits[1].get(x,0))\n    df_final_train['authorities_d'] = df_final_train.destination_node.apply(lambda x: hits[1].get(x,0))\n    df_final_test['authorities_s'] = df_final_test.source_node.apply(lambda x: hits[1].get(x,0))\n    df_final_test['authorities_d'] = df_final_test.destination_node.apply(lambda x: hits[1].get(x,0))\n    hdf = HDFStore('storage_sample_stage3.h5')\n    hdf.put('train_df',df_final_train, format='table', data_columns=True)\n    hdf.put('test_df',df_final_test, format='table', data_columns=True)\n    hdf.close()\nelse:\n    df_final_train = read_hdf('storage_sample_stage3.h5', 'train_df',mode='r')\n    df_final_test = read_hdf('storage_sample_stage3.h5', 'test_df',mode='r')\ndef svd(x, S):\n    try:\n        z = sadj_dict[x]\n        return S[z]\n    except:\n        return [0,0,0,0,0,0]\nsadj_col = sorted(train_graph.nodes())\nsadj_dict = { val:idx for idx,val in enumerate(sadj_col)}\nAdj = nx.adjacency_matrix(train_graph,nodelist=sorted(train_graph.nodes())).asfptype()\nU, s, V = svds(Adj, k = 6)\nif not os.path.isfile('storage_sample_stage4.h5'):\n    df_final_train[['svd_u_s_1', 'svd_u_s_2','svd_u_s_3', 'svd_u_s_4', 'svd_u_s_5', 'svd_u_s_6']] = \\\n    df_final_train.source_node.apply(lambda x: svd(x, U)).apply(pd.Series)\n    df_final_train[['svd_u_d_1', 'svd_u_d_2', 'svd_u_d_3', 'svd_u_d_4', 'svd_u_d_5','svd_u_d_6']] = \\\n    df_final_train.destination_node.apply(lambda x: svd(x, U)).apply(pd.Series)\n    df_final_train[['svd_v_s_1','svd_v_s_2', 'svd_v_s_3', 'svd_v_s_4', 'svd_v_s_5', 'svd_v_s_6',]] = \\\n    df_final_train.source_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)\n    df_final_train[['svd_v_d_1', 'svd_v_d_2', 'svd_v_d_3', 'svd_v_d_4', 'svd_v_d_5','svd_v_d_6']] = \\\n    df_final_train.destination_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)\n    df_final_test[['svd_u_s_1', 'svd_u_s_2','svd_u_s_3', 'svd_u_s_4', 'svd_u_s_5', 'svd_u_s_6']] = \\\n    df_final_test.source_node.apply(lambda x: svd(x, U)).apply(pd.Series)\n    df_final_test[['svd_u_d_1', 'svd_u_d_2', 'svd_u_d_3', 'svd_u_d_4', 'svd_u_d_5','svd_u_d_6']] = \\\n    df_final_test.destination_node.apply(lambda x: svd(x, U)).apply(pd.Series)\n    df_final_test[['svd_v_s_1','svd_v_s_2', 'svd_v_s_3', 'svd_v_s_4', 'svd_v_s_5', 'svd_v_s_6',]] = \\\n    df_final_test.source_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)\n    df_final_test[['svd_v_d_1', 'svd_v_d_2', 'svd_v_d_3', 'svd_v_d_4', 'svd_v_d_5','svd_v_d_6']] = \\\n    df_final_test.destination_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)\n    hdf = HDFStore('storage_sample_stage4.h5')\n    hdf.put('train_df',df_final_train, format='table', data_columns=True)\n    hdf.put('test_df',df_final_test, format='table', data_columns=True)\n    hdf.close()\nif os.path.isfile('train_pos_after_eda.csv'):\n    train_graph=nx.read_edgelist('train_pos_after_eda.csv',delimiter=',',create_using=nx.DiGraph(),nodetype=int)\nelse:\n    pass\ndef jaccard_for_followees(a,b):\n    try:\n        if len(set(train_graph.successors(a))) == 0  | len(set(train_graph.successors(b))) == 0:\n            return 0\n        sim = (len(set(train_graph.successors(a)).intersection(set(train_graph.successors(b)))))/(len(set(train_graph.successors(a)).union(set(train_graph.successors(b)))))\n    except:\n        return 0\n    return sim\ndef jaccard_for_followers(a,b):\n    try:\n        if len(set(train_graph.predecessors(a))) == 0  | len(set(g.predecessors(b))) == 0:\n            return 0\n        sim = (len(set(train_graph.predecessors(a)).intersection(set(train_graph.predecessors(b)))))/(len(set(train_graph.predecessors(a)).union(set(train_graph.predecessors(b)))))\n        return sim\n    except:\n        return 0\ndef cosine_for_followees(a,b):\n    try:\n        if len(set(train_graph.successors(a))) == 0  | len(set(train_graph.successors(b))) == 0:\n            return 0\n        sim = (len(set(train_graph.successors(a)).intersection(set(train_graph.successors(b)))))/(math.sqrt(len(set(train_graph.successors(a)))*len((set(train_graph.successors(b))))))\n        return sim\n    except:\n        return 0\ndef cosine_for_followers(a,b):\n    try:\n\n        if len(set(train_graph.predecessors(a))) == 0  | len(set(train_graph.predecessors(b))) == 0:\n            return 0\n        sim = (len(set(train_graph.predecessors(a)).intersection(set(train_graph.predecessors(b)))))/(math.sqrt(len(set(train_graph.predecessors(a))))*(len(set(train_graph.predecessors(b)))))\n        return sim\n    except:\n        return 0\nif not os.path.isfile('page_rank.p'):\n    pr = nx.pagerank(train_graph, alpha=0.85)\n    pickle.dump(pr,open('page_rank.p','wb'))\nelse:\n    pr = pickle.load(open('page_rank.p','rb'))\nmean_pr = float(sum(pr.values())) / len(pr)\ndef compute_shortest_path_length(a,b):\n    p=-1\n    try:\n        if train_graph.has_edge(a,b):\n            train_graph.remove_edge(a,b)\n            p= nx.shortest_path_length(train_graph,source=a,target=b)\n            train_graph.add_edge(a,b)\n        else:\n            p= nx.shortest_path_length(train_graph,source=a,target=b)\n        return p\n    except:\n        return -1\nwcc=list(nx.weakly_connected_components(train_graph))\ndef belongs_to_same_wcc(a,b):\n    index = []\n    if train_graph.has_edge(b,a):\n        return 1\n    if train_graph.has_edge(a,b):\n            for i in wcc:\n                if a in i:\n                    index= i\n                    break\n            if (b in index):\n                train_graph.remove_edge(a,b)\n                if compute_shortest_path_length(a,b)==-1:\n                    train_graph.add_edge(a,b)\n                    return 0\n                else:\n                    train_graph.add_edge(a,b)\n                    return 1\n            else:\n                return 0\n    else:\n            for i in wcc:\n                if a in i:\n                    index= i\n                    break\n            if(b in index):\n                return 1\n            else:\n                return 0\ndef calc_adar_in(a,b):\n    sum=0\n    try:\n        n=list(set(train_graph.successors(a)).intersection(set(train_graph.successors(b))))\n        if len(n)!=0:\n            for i in n:\n                sum=sum+(1/np.log10(len(list(train_graph.predecessors(i)))))\n            return sum\n        else:\n            return 0\n    except:\n        return 0\ndef follows_back(a,b):\n    if train_graph.has_edge(b,a):\n        return 1\n    else:\n        return 0\nif not os.path.isfile('katz.p'):\n    katz = nx.katz.katz_centrality(train_graph,alpha=0.005,beta=1)\n    pickle.dump(katz,open('katz.p','wb'))\nelse:\n    katz = pickle.load(open('katz.p','rb'))\nmean_katz = float(sum(katz.values())) / len(katz)\nif not os.path.isfile('hits.p'):\n    hits = nx.hits(train_graph, max_iter=100, tol=1e-08, nstart=None, normalized=True)\n    pickle.dump(hits,open('hits.p','wb'))\nelse:\n    hits = pickle.load(open('hits.p','rb'))\nif os.path.isfile('train_after_eda.csv'):\n    filename = \"train_after_eda.csv\"\n    n_train =  15100028\n    s = 100000\n    skip_train = sorted(random.sample(range(1,n_train+1),n_train-s))\nif os.path.isfile('train_after_eda.csv'):\n    filename = \"test_after_eda.csv\"\n    n_test = 3775006\n    s = 50000\n    skip_test = sorted(random.sample(range(1,n_test+1),n_test-s))\ndf_final_train = pd.read_csv('train_after_eda.csv', skiprows=skip_train, names=['source_node', 'destination_node'])\ndf_final_train['indicator_link'] = pd.read_csv('train_y.csv', skiprows=skip_train, names=['indicator_link'])\ndf_final_test = pd.read_csv('test_after_eda.csv', skiprows=skip_test, names=['source_node', 'destination_node'])\ndf_final_test['indicator_link'] = pd.read_csv('test_y.csv', skiprows=skip_test, names=['indicator_link'])\nif not os.path.isfile('storage_sample_stage1.h5'):\n    df_final_train['jaccard_followers'] = df_final_train.apply(lambda row:\n                                            jaccard_for_followers(row['source_node'],row['destination_node']),axis=1)\n    df_final_test['jaccard_followers'] = df_final_test.apply(lambda row:\n                                            jaccard_for_followers(row['source_node'],row['destination_node']),axis=1)\n    df_final_train['jaccard_followees'] = df_final_train.apply(lambda row:\n                                            jaccard_for_followees(row['source_node'],row['destination_node']),axis=1)\n    df_final_test['jaccard_followees'] = df_final_test.apply(lambda row:\n                                            jaccard_for_followees(row['source_node'],row['destination_node']),axis=1)\n    df_final_train['cosine_followers'] = df_final_train.apply(lambda row:\n                                            cosine_for_followers(row['source_node'],row['destination_node']),axis=1)\n    df_final_test['cosine_followers'] = df_final_test.apply(lambda row:\n                                            cosine_for_followers(row['source_node'],row['destination_node']),axis=1)\n    df_final_train['cosine_followees'] = df_final_train.apply(lambda row:\n                                            cosine_for_followees(row['source_node'],row['destination_node']),axis=1)\n    df_final_test['cosine_followees'] = df_final_test.apply(lambda row:\n                                            cosine_for_followees(row['source_node'],row['destination_node']),axis=1)\ndef compute_features_stage1(df_final):\n    num_followers_s=[]\n    num_followees_s=[]\n    num_followers_d=[]\n    num_followees_d=[]\n    inter_followers=[]\n    inter_followees=[]\n    for i,row in df_final.iterrows():\n        try:\n            s1=set(train_graph.predecessors(row['source_node']))\n            s2=set(train_graph.successors(row['source_node']))\n        except:\n            s1 = set()\n            s2 = set()\n        try:\n            d1=set(train_graph.predecessors(row['destination_node']))\n            d2=set(train_graph.successors(row['destination_node']))\n        except:\n            d1 = set()\n            d2 = set()\n        num_followers_s.append(len(s1))\n        num_followees_s.append(len(s2))\n        num_followers_d.append(len(d1))\n        num_followees_d.append(len(d2))\n        inter_followers.append(len(s1.intersection(d1)))\n        inter_followees.append(len(s2.intersection(d2)))\n    return num_followers_s, num_followers_d, num_followees_s, num_followees_d, inter_followers, inter_followees\nif not os.path.isfile('storage_sample_stage1.h5'):\n    df_final_train['num_followers_s'], df_final_train['num_followers_d'], \\\n    df_final_train['num_followees_s'], df_final_train['num_followees_d'], \\\n    df_final_train['inter_followers'], df_final_train['inter_followees']= compute_features_stage1(df_final_train)\n    df_final_test['num_followers_s'], df_final_test['num_followers_d'], \\\n    df_final_test['num_followees_s'], df_final_test['num_followees_d'], \\\n    df_final_test['inter_followers'], df_final_test['inter_followees']= compute_features_stage1(df_final_test)\n    hdf = HDFStore('storage_sample_stage1.h5')\n    hdf.put('train_df',df_final_train, format='table', data_columns=True)\n    hdf.put('test_df',df_final_test, format='table', data_columns=True)\n    hdf.close()\nelse:\n    df_final_train = read_hdf('storage_sample_stage1.h5', 'train_df',mode='r')\n    df_final_test = read_hdf('storage_sample_stage1.h5', 'test_df',mode='r')\nif not os.path.isfile('storage_sample_stage2.h5'):\n    df_final_train['adar_index'] = df_final_train.apply(lambda row: calc_adar_in(row['source_node'],row['destination_node']),axis=1)\n    df_final_test['adar_index'] = df_final_test.apply(lambda row: calc_adar_in(row['source_node'],row['destination_node']),axis=1)\n    df_final_train['follows_back'] = df_final_train.apply(lambda row: follows_back(row['source_node'],row['destination_node']),axis=1)\n    df_final_test['follows_back'] = df_final_test.apply(lambda row: follows_back(row['source_node'],row['destination_node']),axis=1)\n    df_final_train['same_comp'] = df_final_train.apply(lambda row: belongs_to_same_wcc(row['source_node'],row['destination_node']),axis=1)\n    df_final_test['same_comp'] = df_final_test.apply(lambda row: belongs_to_same_wcc(row['source_node'],row['destination_node']),axis=1)\n    df_final_train['shortest_path'] = df_final_train.apply(lambda row: compute_shortest_path_length(row['source_node'],row['destination_node']),axis=1)\n    df_final_test['shortest_path'] = df_final_test.apply(lambda row: compute_shortest_path_length(row['source_node'],row['destination_node']),axis=1)\n    hdf = HDFStore('storage_sample_stage2.h5')\n    hdf.put('train_df',df_final_train, format='table', data_columns=True)\n    hdf.put('test_df',df_final_test, format='table', data_columns=True)\n    hdf.close()\nelse:\n    df_final_train = read_hdf('storage_sample_stage2.h5', 'train_df',mode='r')\n    df_final_test = read_hdf('storage_sample_stage2.h5', 'test_df',mode='r')\nWeight_in = {}\nWeight_out = {}\nfor i in  tqdm(train_graph.nodes()):\n    s1=set(train_graph.predecessors(i))\n    w_in = 1.0/(np.sqrt(1+len(s1)))\n    Weight_in[i]=w_in\n    s2=set(train_graph.successors(i))\n    w_out = 1.0/(np.sqrt(1+len(s2)))\n    Weight_out[i]=w_out\nmean_weight_in = np.mean(list(Weight_in.values()))\nmean_weight_out = np.mean(list(Weight_out.values()))\nif not os.path.isfile('storage_sample_stage3.h5'):\n    df_final_train['weight_in'] = df_final_train.destination_node.apply(lambda x: Weight_in.get(x,mean_weight_in))\n    df_final_train['weight_out'] = df_final_train.source_node.apply(lambda x: Weight_out.get(x,mean_weight_out))\n    df_final_test['weight_in'] = df_final_test.destination_node.apply(lambda x: Weight_in.get(x,mean_weight_in))\n    df_final_test['weight_out'] = df_final_test.source_node.apply(lambda x: Weight_out.get(x,mean_weight_out))\n    df_final_train['weight_f1'] = df_final_train.weight_in + df_final_train.weight_out\n    df_final_train['weight_f2'] = df_final_train.weight_in * df_final_train.weight_out\n    df_final_train['weight_f3'] = (2*df_final_train.weight_in + 1*df_final_train.weight_out)\n    df_final_train['weight_f4'] = (1*df_final_train.weight_in + 2*df_final_train.weight_out)\n    df_final_test['weight_f1'] = df_final_test.weight_in + df_final_test.weight_out\n    df_final_test['weight_f2'] = df_final_test.weight_in * df_final_test.weight_out\n    df_final_test['weight_f3'] = (2*df_final_test.weight_in + 1*df_final_test.weight_out)\n    df_final_test['weight_f4'] = (1*df_final_test.weight_in + 2*df_final_test.weight_out)\nif not os.path.isfile('storage_sample_stage3.h5'):\n    df_final_train['page_rank_s'] = df_final_train.source_node.apply(lambda x:pr.get(x,mean_pr))\n    df_final_train['page_rank_d'] = df_final_train.destination_node.apply(lambda x:pr.get(x,mean_pr))\n    df_final_test['page_rank_s'] = df_final_test.source_node.apply(lambda x:pr.get(x,mean_pr))\n    df_final_test['page_rank_d'] = df_final_test.destination_node.apply(lambda x:pr.get(x,mean_pr))\n    df_final_train['katz_s'] = df_final_train.source_node.apply(lambda x: katz.get(x,mean_katz))\n    df_final_train['katz_d'] = df_final_train.destination_node.apply(lambda x: katz.get(x,mean_katz))\n    df_final_test['katz_s'] = df_final_test.source_node.apply(lambda x: katz.get(x,mean_katz))\n    df_final_test['katz_d'] = df_final_test.destination_node.apply(lambda x: katz.get(x,mean_katz))\n    df_final_train['hubs_s'] = df_final_train.source_node.apply(lambda x: hits[0].get(x,0))\n    df_final_train['hubs_d'] = df_final_train.destination_node.apply(lambda x: hits[0].get(x,0))\n    df_final_test['hubs_s'] = df_final_test.source_node.apply(lambda x: hits[0].get(x,0))\n    df_final_test['hubs_d'] = df_final_test.destination_node.apply(lambda x: hits[0].get(x,0))\n    df_final_train['authorities_s'] = df_final_train.source_node.apply(lambda x: hits[1].get(x,0))\n    df_final_train['authorities_d'] = df_final_train.destination_node.apply(lambda x: hits[1].get(x,0))\n    df_final_test['authorities_s'] = df_final_test.source_node.apply(lambda x: hits[1].get(x,0))\n    df_final_test['authorities_d'] = df_final_test.destination_node.apply(lambda x: hits[1].get(x,0))\n    hdf = HDFStore('storage_sample_stage3.h5')\n    hdf.put('train_df',df_final_train, format='table', data_columns=True)\n    hdf.put('test_df',df_final_test, format='table', data_columns=True)\n    hdf.close()\nelse:\n    df_final_train = read_hdf('storage_sample_stage3.h5', 'train_df',mode='r')\n    df_final_test = read_hdf('storage_sample_stage3.h5', 'test_df',mode='r')\ndef svd(x, S):\n    try:\n        z = sadj_dict[x]\n        return S[z]\n    except:\n        return [0,0,0,0,0,0]\nsadj_col = sorted(train_graph.nodes())\nsadj_dict = { val:idx for idx,val in enumerate(sadj_col)}\nAdj = nx.adjacency_matrix(train_graph,nodelist=sorted(train_graph.nodes())).asfptype()\nU, s, V = svds(Adj, k = 6)\nif not os.path.isfile('storage_sample_stage4.h5'):\n    df_final_train[['svd_u_s_1', 'svd_u_s_2','svd_u_s_3', 'svd_u_s_4', 'svd_u_s_5', 'svd_u_s_6']] = \\\n    df_final_train.source_node.apply(lambda x: svd(x, U)).apply(pd.Series)\n    df_final_train[['svd_u_d_1', 'svd_u_d_2', 'svd_u_d_3', 'svd_u_d_4', 'svd_u_d_5','svd_u_d_6']] = \\\n    df_final_train.destination_node.apply(lambda x: svd(x, U)).apply(pd.Series)\n    df_final_train[['svd_v_s_1','svd_v_s_2', 'svd_v_s_3', 'svd_v_s_4', 'svd_v_s_5', 'svd_v_s_6',]] = \\\n    df_final_train.source_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)\n    df_final_train[['svd_v_d_1', 'svd_v_d_2', 'svd_v_d_3', 'svd_v_d_4', 'svd_v_d_5','svd_v_d_6']] = \\\n    df_final_train.destination_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)\n    df_final_test[['svd_u_s_1', 'svd_u_s_2','svd_u_s_3', 'svd_u_s_4', 'svd_u_s_5', 'svd_u_s_6']] = \\\n    df_final_test.source_node.apply(lambda x: svd(x, U)).apply(pd.Series)\n    df_final_test[['svd_u_d_1', 'svd_u_d_2', 'svd_u_d_3', 'svd_u_d_4', 'svd_u_d_5','svd_u_d_6']] = \\\n    df_final_test.destination_node.apply(lambda x: svd(x, U)).apply(pd.Series)\n    df_final_test[['svd_v_s_1','svd_v_s_2', 'svd_v_s_3', 'svd_v_s_4', 'svd_v_s_5', 'svd_v_s_6',]] = \\\n    df_final_test.source_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)\n    df_final_test[['svd_v_d_1', 'svd_v_d_2', 'svd_v_d_3', 'svd_v_d_4', 'svd_v_d_5','svd_v_d_6']] = \\\n    df_final_test.destination_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)\n    hdf = HDFStore('storage_sample_stage4.h5')\n    hdf.put('train_df',df_final_train, format='table', data_columns=True)\n    hdf.put('test_df',df_final_test, format='table', data_columns=True)\n    hdf.close()\nif os.path.isfile('train_pos_after_eda.csv'):\n    train_graph=nx.read_edgelist('train_pos_after_eda.csv',delimiter=',',create_using=nx.DiGraph(),nodetype=int)\nelse:\n    pass\ndef jaccard_for_followees(a,b):\n    try:\n        if len(set(train_graph.successors(a))) == 0  | len(set(train_graph.successors(b))) == 0:\n            return 0\n        sim = (len(set(train_graph.successors(a)).intersection(set(train_graph.successors(b)))))/(len(set(train_graph.successors(a)).union(set(train_graph.successors(b)))))\n    except:\n        return 0\n    return sim\ndef jaccard_for_followers(a,b):\n    try:\n        if len(set(train_graph.predecessors(a))) == 0  | len(set(g.predecessors(b))) == 0:\n            return 0\n        sim = (len(set(train_graph.predecessors(a)).intersection(set(train_graph.predecessors(b)))))/(len(set(train_graph.predecessors(a)).union(set(train_graph.predecessors(b)))))\n        return sim\n    except:\n        return 0\ndef cosine_for_followees(a,b):\n    try:\n        if len(set(train_graph.successors(a))) == 0  | len(set(train_graph.successors(b))) == 0:\n            return 0\n        sim = (len(set(train_graph.successors(a)).intersection(set(train_graph.successors(b)))))/(math.sqrt(len(set(train_graph.successors(a)))*len((set(train_graph.successors(b))))))\n        return sim\n    except:\n        return 0\ndef cosine_for_followers(a,b):\n    try:\n        if len(set(train_graph.predecessors(a))) == 0  | len(set(train_graph.predecessors(b))) == 0:\n            return 0\n        sim = (len(set(train_graph.predecessors(a)).intersection(set(train_graph.predecessors(b)))))/(math.sqrt(len(set(train_graph.predecessors(a))))*(len(set(train_graph.predecessors(b)))))\n        return sim\n    except:\n        return 0\nif not os.path.isfile('page_rank.p'):\n    pr = nx.pagerank(train_graph, alpha=0.85)\n    pickle.dump(pr,open('page_rank.p','wb'))\nelse:\n    pr = pickle.load(open('page_rank.p','rb'))\nmean_pr = float(sum(pr.values())) / len(pr)\ndef compute_shortest_path_length(a,b):\n    p=-1\n    try:\n        if train_graph.has_edge(a,b):\n            train_graph.remove_edge(a,b)\n            p= nx.shortest_path_length(train_graph,source=a,target=b)\n            train_graph.add_edge(a,b)\n        else:\n            p= nx.shortest_path_length(train_graph,source=a,target=b)\n        return p\n    except:\n        return -1\nwcc=list(nx.weakly_connected_components(train_graph))\ndef belongs_to_same_wcc(a,b):\n    index = []\n    if train_graph.has_edge(b,a):\n        return 1\n    if train_graph.has_edge(a,b):\n            for i in wcc:\n                if a in i:\n                    index= i\n                    break\n            if (b in index):\n                train_graph.remove_edge(a,b)\n                if compute_shortest_path_length(a,b)==-1:\n                    train_graph.add_edge(a,b)\n                    return 0\n                else:\n                    train_graph.add_edge(a,b)\n                    return 1\n            else:\n                return 0\n    else:\n            for i in wcc:\n                if a in i:\n                    index= i\n                    break\n            if(b in index):\n                return 1\n            else:\n                return 0\ndef calc_adar_in(a,b):\n    sum=0\n    try:\n        n=list(set(train_graph.successors(a)).intersection(set(train_graph.successors(b))))\n        if len(n)!=0:\n            for i in n:\n                sum=sum+(1/np.log10(len(list(train_graph.predecessors(i)))))\n            return sum\n        else:\n            return 0\n    except:\n        return 0\ndef follows_back(a,b):\n    if train_graph.has_edge(b,a):\n        return 1\n    else:\n        return 0\nif not os.path.isfile('katz.p'):\n    katz = nx.katz.katz_centrality(train_graph,alpha=0.005,beta=1)\n    pickle.dump(katz,open('katz.p','wb'))\nelse:\n    katz = pickle.load(open('katz.p','rb'))\nmean_katz = float(sum(katz.values())) / len(katz)\nif not os.path.isfile('hits.p'):\n    hits = nx.hits(train_graph, max_iter=100, tol=1e-08, nstart=None, normalized=True)\n    pickle.dump(hits,open('hits.p','wb'))\nelse:\n    hits = pickle.load(open('hits.p','rb'))\nif os.path.isfile('train_after_eda.csv'):\n    filename = \"train_after_eda.csv\"\n    n_train =  15100028\n    s = 100000\n    skip_train = sorted(random.sample(range(1,n_train+1),n_train-s))\nif os.path.isfile('train_after_eda.csv'):\n    filename = \"test_after_eda.csv\"\n    n_test = 3775006\n    s = 50000\n    skip_test = sorted(random.sample(range(1,n_test+1),n_test-s))\ndf_final_train = pd.read_csv('train_after_eda.csv', skiprows=skip_train, names=['source_node', 'destination_node'])\ndf_final_train['indicator_link'] = pd.read_csv('train_y.csv', skiprows=skip_train, names=['indicator_link'])\ndf_final_test = pd.read_csv('test_after_eda.csv', skiprows=skip_test, names=['source_node', 'destination_node'])\ndf_final_test['indicator_link'] = pd.read_csv('test_y.csv', skiprows=skip_test, names=['indicator_link'])\nif not os.path.isfile('storage_sample_stage1.h5'):\n    df_final_train['jaccard_followers'] = df_final_train.apply(lambda row:\n                                            jaccard_for_followers(row['source_node'],row['destination_node']),axis=1)\n    df_final_test['jaccard_followers'] = df_final_test.apply(lambda row:\n                                            jaccard_for_followers(row['source_node'],row['destination_node']),axis=1)\n    df_final_train['jaccard_followees'] = df_final_train.apply(lambda row:\n                                            jaccard_for_followees(row['source_node'],row['destination_node']),axis=1)\n    df_final_test['jaccard_followees'] = df_final_test.apply(lambda row:\n                                            jaccard_for_followees(row['source_node'],row['destination_node']),axis=1)\n    df_final_train['cosine_followers'] = df_final_train.apply(lambda row:\n                                            cosine_for_followers(row['source_node'],row['destination_node']),axis=1)\n    df_final_test['cosine_followers'] = df_final_test.apply(lambda row:\n                                            cosine_for_followers(row['source_node'],row['destination_node']),axis=1)\n    df_final_train['cosine_followees'] = df_final_train.apply(lambda row:\n                                            cosine_for_followees(row['source_node'],row['destination_node']),axis=1)\n    df_final_test['cosine_followees'] = df_final_test.apply(lambda row:\n                                            cosine_for_followees(row['source_node'],row['destination_node']),axis=1)\ndef compute_features_stage1(df_final):\n    num_followers_s=[]\n    num_followees_s=[]\n    num_followers_d=[]\n    num_followees_d=[]\n    inter_followers=[]\n    inter_followees=[]\n    for i,row in df_final.iterrows():\n        try:\n            s1=set(train_graph.predecessors(row['source_node']))\n            s2=set(train_graph.successors(row['source_node']))\n        except:\n            s1 = set()\n            s2 = set()\n        try:\n            d1=set(train_graph.predecessors(row['destination_node']))\n            d2=set(train_graph.successors(row['destination_node']))\n        except:\n            d1 = set()\n            d2 = set()\n        num_followers_s.append(len(s1))\n        num_followees_s.append(len(s2))\n        num_followers_d.append(len(d1))\n        num_followees_d.append(len(d2))\n        inter_followers.append(len(s1.intersection(d1)))\n        inter_followees.append(len(s2.intersection(d2)))\n    return num_followers_s, num_followers_d, num_followees_s, num_followees_d, inter_followers, inter_followees\nif not os.path.isfile('storage_sample_stage1.h5'):\n    df_final_train['num_followers_s'], df_final_train['num_followers_d'], \\\n    df_final_train['num_followees_s'], df_final_train['num_followees_d'], \\\n    df_final_train['inter_followers'], df_final_train['inter_followees']= compute_features_stage1(df_final_train)\n    df_final_test['num_followers_s'], df_final_test['num_followers_d'], \\\n    df_final_test['num_followees_s'], df_final_test['num_followees_d'], \\\n    df_final_test['inter_followers'], df_final_test['inter_followees']= compute_features_stage1(df_final_test)\n    hdf = HDFStore('storage_sample_stage1.h5')\n    hdf.put('train_df',df_final_train, format='table', data_columns=True)\n    hdf.put('test_df',df_final_test, format='table', data_columns=True)\n    hdf.close()\nelse:\n    df_final_train = read_hdf('storage_sample_stage1.h5', 'train_df',mode='r')\n    df_final_test = read_hdf('storage_sample_stage1.h5', 'test_df',mode='r')\nif not os.path.isfile('storage_sample_stage2.h5'):\n    df_final_train['adar_index'] = df_final_train.apply(lambda row: calc_adar_in(row['source_node'],row['destination_node']),axis=1)\n    df_final_test['adar_index'] = df_final_test.apply(lambda row: calc_adar_in(row['source_node'],row['destination_node']),axis=1)\n    df_final_train['follows_back'] = df_final_train.apply(lambda row: follows_back(row['source_node'],row['destination_node']),axis=1)\n    df_final_test['follows_back'] = df_final_test.apply(lambda row: follows_back(row['source_node'],row['destination_node']),axis=1)\n    df_final_train['same_comp'] = df_final_train.apply(lambda row: belongs_to_same_wcc(row['source_node'],row['destination_node']),axis=1)\n    df_final_test['same_comp'] = df_final_test.apply(lambda row: belongs_to_same_wcc(row['source_node'],row['destination_node']),axis=1)\n    df_final_train['shortest_path'] = df_final_train.apply(lambda row: compute_shortest_path_length(row['source_node'],row['destination_node']),axis=1)\n    df_final_test['shortest_path'] = df_final_test.apply(lambda row: compute_shortest_path_length(row['source_node'],row['destination_node']),axis=1)\n    hdf = HDFStore('storage_sample_stage2.h5')\n    hdf.put('train_df',df_final_train, format='table', data_columns=True)\n    hdf.put('test_df',df_final_test, format='table', data_columns=True)\n    hdf.close()\nelse:\n    df_final_train = read_hdf('storage_sample_stage2.h5', 'train_df',mode='r')\n    df_final_test = read_hdf('storage_sample_stage2.h5', 'test_df',mode='r')\nWeight_in = {}\nWeight_out = {}\nfor i in  tqdm(train_graph.nodes()):\n    s1=set(train_graph.predecessors(i))\n    w_in = 1.0/(np.sqrt(1+len(s1)))\n    Weight_in[i]=w_in\n    s2=set(train_graph.successors(i))\n    w_out = 1.0/(np.sqrt(1+len(s2)))\n    Weight_out[i]=w_out\nmean_weight_in = np.mean(list(Weight_in.values()))\nmean_weight_out = np.mean(list(Weight_out.values()))\nif not os.path.isfile('storage_sample_stage3.h5'):\n    df_final_train['weight_in'] = df_final_train.destination_node.apply(lambda x: Weight_in.get(x,mean_weight_in))\n    df_final_train['weight_out'] = df_final_train.source_node.apply(lambda x: Weight_out.get(x,mean_weight_out))\n    df_final_test['weight_in'] = df_final_test.destination_node.apply(lambda x: Weight_in.get(x,mean_weight_in))\n    df_final_test['weight_out'] = df_final_test.source_node.apply(lambda x: Weight_out.get(x,mean_weight_out))\n    df_final_train['weight_f1'] = df_final_train.weight_in + df_final_train.weight_out\n    df_final_train['weight_f2'] = df_final_train.weight_in * df_final_train.weight_out\n    df_final_train['weight_f3'] = (2*df_final_train.weight_in + 1*df_final_train.weight_out)\n    df_final_train['weight_f4'] = (1*df_final_train.weight_in + 2*df_final_train.weight_out)\n    df_final_test['weight_f1'] = df_final_test.weight_in + df_final_test.weight_out\n    df_final_test['weight_f2'] = df_final_test.weight_in * df_final_test.weight_out\n    df_final_test['weight_f3'] = (2*df_final_test.weight_in + 1*df_final_test.weight_out)\n    df_final_test['weight_f4'] = (1*df_final_test.weight_in + 2*df_final_test.weight_out)\nif not os.path.isfile('storage_sample_stage3.h5'):\n    df_final_train['page_rank_s'] = df_final_train.source_node.apply(lambda x:pr.get(x,mean_pr))\n    df_final_train['page_rank_d'] = df_final_train.destination_node.apply(lambda x:pr.get(x,mean_pr))\n    df_final_test['page_rank_s'] = df_final_test.source_node.apply(lambda x:pr.get(x,mean_pr))\n    df_final_test['page_rank_d'] = df_final_test.destination_node.apply(lambda x:pr.get(x,mean_pr))\n    df_final_train['katz_s'] = df_final_train.source_node.apply(lambda x: katz.get(x,mean_katz))\n    df_final_train['katz_d'] = df_final_train.destination_node.apply(lambda x: katz.get(x,mean_katz))\n    df_final_test['katz_s'] = df_final_test.source_node.apply(lambda x: katz.get(x,mean_katz))\n    df_final_test['katz_d'] = df_final_test.destination_node.apply(lambda x: katz.get(x,mean_katz))\n    df_final_train['hubs_s'] = df_final_train.source_node.apply(lambda x: hits[0].get(x,0))\n    df_final_train['hubs_d'] = df_final_train.destination_node.apply(lambda x: hits[0].get(x,0))\n    df_final_test['hubs_s'] = df_final_test.source_node.apply(lambda x: hits[0].get(x,0))\n    df_final_test['hubs_d'] = df_final_test.destination_node.apply(lambda x: hits[0].get(x,0))\n    df_final_train['authorities_s'] = df_final_train.source_node.apply(lambda x: hits[1].get(x,0))\n    df_final_train['authorities_d'] = df_final_train.destination_node.apply(lambda x: hits[1].get(x,0))\n    df_final_test['authorities_s'] = df_final_test.source_node.apply(lambda x: hits[1].get(x,0))\n    df_final_test['authorities_d'] = df_final_test.destination_node.apply(lambda x: hits[1].get(x,0))\n    hdf = HDFStore('storage_sample_stage3.h5')\n    hdf.put('train_df',df_final_train, format='table', data_columns=True)\n    hdf.put('test_df',df_final_test, format='table', data_columns=True)\n    hdf.close()\nelse:\n    df_final_train = read_hdf('storage_sample_stage3.h5', 'train_df',mode='r')\n    df_final_test = read_hdf('storage_sample_stage3.h5', 'test_df',mode='r')\ndef svd(x, S):\n    try:\n        z = sadj_dict[x]\n        return S[z]\n    except:\n        return [0,0,0,0,0,0]\nsadj_col = sorted(train_graph.nodes())\nsadj_dict = { val:idx for idx,val in enumerate(sadj_col)}\nAdj = nx.adjacency_matrix(train_graph,nodelist=sorted(train_graph.nodes())).asfptype()\nU, s, V = svds(Adj, k = 6)\nif not os.path.isfile('storage_sample_stage4.h5'):\n    df_final_train[['svd_u_s_1', 'svd_u_s_2','svd_u_s_3', 'svd_u_s_4', 'svd_u_s_5', 'svd_u_s_6']] = \\\n    df_final_train.source_node.apply(lambda x: svd(x, U)).apply(pd.Series)\n    df_final_train[['svd_u_d_1', 'svd_u_d_2', 'svd_u_d_3', 'svd_u_d_4', 'svd_u_d_5','svd_u_d_6']] = \\\n    df_final_train.destination_node.apply(lambda x: svd(x, U)).apply(pd.Series)\n    df_final_train[['svd_v_s_1','svd_v_s_2', 'svd_v_s_3', 'svd_v_s_4', 'svd_v_s_5', 'svd_v_s_6',]] = \\\n    df_final_train.source_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)\n    df_final_train[['svd_v_d_1', 'svd_v_d_2', 'svd_v_d_3', 'svd_v_d_4', 'svd_v_d_5','svd_v_d_6']] = \\\n    df_final_train.destination_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)\n    df_final_test[['svd_u_s_1', 'svd_u_s_2','svd_u_s_3', 'svd_u_s_4', 'svd_u_s_5', 'svd_u_s_6']] = \\\n    df_final_test.source_node.apply(lambda x: svd(x, U)).apply(pd.Series)\n    df_final_test[['svd_u_d_1', 'svd_u_d_2', 'svd_u_d_3', 'svd_u_d_4', 'svd_u_d_5','svd_u_d_6']] = \\\n    df_final_test.destination_node.apply(lambda x: svd(x, U)).apply(pd.Series)\n    df_final_test[['svd_v_s_1','svd_v_s_2', 'svd_v_s_3', 'svd_v_s_4', 'svd_v_s_5', 'svd_v_s_6',]] = \\\n    df_final_test.source_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)\n    df_final_test[['svd_v_d_1', 'svd_v_d_2', 'svd_v_d_3', 'svd_v_d_4', 'svd_v_d_5','svd_v_d_6']] = \\\n    df_final_test.destination_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)\n    hdf = HDFStore('storage_sample_stage4.h5')\n    hdf.put('train_df',df_final_train, format='table', data_columns=True)\n    hdf.put('test_df',df_final_test, format='table', data_columns=True)\n    hdf.close()\ndf_final_train = read_hdf('storage_sample_stage4.h5', 'train_df',mode='r')\ndf_final_test = read_hdf('storage_sample_stage4.h5', 'test_df',mode='r')\ny_train = df_final_train.indicator_link\ny_test = df_final_test.indicator_link\ndf_final_train.drop(['source_node', 'destination_node','indicator_link'],axis=1,inplace=True)\ndf_final_test.drop(['source_node', 'destination_node','indicator_link'],axis=1,inplace=True)","metadata":{"id":"WYZD0gQllGO-","outputId":"9b7473b5-80f5-48eb-a3c8-fc87473f2e82","execution":{"iopub.status.busy":"2023-12-27T06:32:41.579557Z","iopub.execute_input":"2023-12-27T06:32:41.579866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Models","metadata":{}},{"cell_type":"code","source":"models = ['Decision Tree', 'SVM','Logistic Regression', 'Proposed Model']\naccuracy = []\nprecision = []\nrecall = []\nf1score = []","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Decision Tree","metadata":{}},{"cell_type":"code","source":"DTC = DecisionTreeClassifier(criterion='gini', max_depth=5, max_features='auto', max_leaf_nodes=None, min_impurity_decrease=0.0, min_impurity_split=None, min_samples_leaf=52, min_samples_split=120, min_weight_fraction_leaf=0.0, random_state=25)\nDTC.fit(df_final_train, y_train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Evaluation Measure","metadata":{}},{"cell_type":"code","source":"f1_score_DTC = f1_score(y_test,DTC.predict(df_final_test))\naccuracy_DTC = accuracy_score(y_test,DTC.predict(df_final_test))\nprecision_DTC = precision_score(y_test,DTC.predict(df_final_test))\nrecall_DTC = recall_score(y_test,DTC.predict(df_final_test))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### SVM","metadata":{}},{"cell_type":"code","source":"SVM = LinearSVC(random_state=25)\nSVM.fit(df_final_train, y_train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Evaluation Measure","metadata":{}},{"cell_type":"code","source":"f1_score_SVM = f1_score(y_test,SVM.predict(df_final_test))\naccuracy_SVM = accuracy_score(y_test,SVM.predict(df_final_test))\nprecision_SVM = precision_score(y_test,SVM.predict(df_final_test))\nrecall_SVM = recall_score(y_test,SVM.predict(df_final_test))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Logistic Regression","metadata":{}},{"cell_type":"code","source":"LR = LogisticRegression(random_state=25)\nLR.fit(df_final_train, y_train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Evaluation Measure","metadata":{}},{"cell_type":"code","source":"f1_score_LR = f1_score(y_test,LR.predict(df_final_test))\naccuracy_LR = accuracy_score(y_test,LR.predict(df_final_test))\nprecision_LR = precision_score(y_test,LR.predict(df_final_test))\nrecall_LR = recall_score(y_test,LR.predict(df_final_test))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Proposed Model","metadata":{"id":"-iuKgWtfUiIf"}},{"cell_type":"code","source":"RFC = RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini', max_depth=5, max_features='auto', max_leaf_nodes=None, min_impurity_decrease=0.0, min_impurity_split=None, min_samples_leaf=52, min_samples_split=120, min_weight_fraction_leaf=0.0, n_estimators=10, n_jobs=-1,random_state=25,verbose=0,warm_start=False)\nRFC.fit(df_final_train, y_train)\nRFC_enc = OneHotEncoder(categories='auto')\nRFC_enc.fit(RFC.apply(df_final_train))\nproposed = LogisticRegression(solver='lbfgs', max_iter=3000)\nproposed.fit(RFC_enc.transform(RFC.apply(df_final_train)), y_train)\nproposed_y_pred = proposed.predict(RFC_enc.transform(RFC.apply(df_final_test)))","metadata":{"id":"V1RoF0PRUSDT","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Evaluation Measure ","metadata":{}},{"cell_type":"code","source":"f1_score_proposed = f1_score(y_test,proposed_y_pred)\naccuracy_proposed = accuracy_score(y_test,proposed_y_pred)\nprecision_proposed = precision_score(y_test,proposed_y_pred)\nrecall_proposed = recall_score(y_test,proposed_y_pred)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Comparison Table","metadata":{}},{"cell_type":"code","source":"accuracy = [accuracy_DTC,accuracy_SVM,accuracy_LR, accuracy_proposed]\nprecision = [precision_DTC,precision_SVM,precision_LR, precision_proposed]\nrecall = [recall_DTC,recall_SVM,recall_LR, recall_proposed]\nf1score = [f1_score_DTC,f1_score_SVM,f1_score_LR, f1_score_proposed]\nmetrics_df = pd.DataFrame({'Model': models, 'Accuracy': accuracy,'Precision': precision, 'Recall': recall, 'F1 Score': f1score})\nprint(metrics_df)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Comparison Graph","metadata":{}},{"cell_type":"code","source":"metrics_df.plot(x='Model', kind='bar', figsize=(10, 6), legend=True)\nplt.title('Model Comparison')\nplt.ylabel('Metrics Score')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}