{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport random\nimport json\nfrom termcolor import colored\nfrom ipywidgets import interact, interactive, fixed, interact_manual\nimport ipywidgets as widgets\nimport plotly.express as px\nfrom plotly.offline import init_notebook_mode, iplot\nfrom plotly.graph_objs import *\ninit_notebook_mode(connected=True)         # initiate notebook for offline plot\nimport matplotlib.pyplot as plt\nimport plotly.graph_objs as go\nfrom tqdm import tqdm\nfrom bisect import bisect\n\nSEED = 42","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-06-24T05:38:09.203421Z","iopub.execute_input":"2022-06-24T05:38:09.203826Z","iopub.status.idle":"2022-06-24T05:38:09.217633Z","shell.execute_reply.started":"2022-06-24T05:38:09.203793Z","shell.execute_reply":"2022-06-24T05:38:09.216601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Importing important libraries and dataset","metadata":{}},{"cell_type":"code","source":"train_orders = pd.read_csv('../input/AI4Code/train_orders.csv')\ntrain_ancestors = pd.read_csv('../input/AI4Code/train_ancestors.csv')\nsample_submission = pd.read_csv('../input/AI4Code/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-06-24T05:38:25.821868Z","iopub.execute_input":"2022-06-24T05:38:25.822261Z","iopub.status.idle":"2022-06-24T05:38:27.802924Z","shell.execute_reply.started":"2022-06-24T05:38:25.822231Z","shell.execute_reply":"2022-06-24T05:38:27.801813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_orders.shape)\nassert train_orders.shape[0] == train_orders['id'].nunique()\ntrain_orders['# cells'] = train_orders['cell_order'].apply(lambda x: len(x.split(' ')))\ntrain_orders.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-24T05:38:40.914970Z","iopub.execute_input":"2022-06-24T05:38:40.915426Z","iopub.status.idle":"2022-06-24T05:38:41.373731Z","shell.execute_reply.started":"2022-06-24T05:38:40.915391Z","shell.execute_reply":"2022-06-24T05:38:41.372646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_ancestors.shape)\nassert train_ancestors.shape[0] == train_ancestors['id'].nunique()\nprint('# Parent notebooks:',train_ancestors['parent_id'].nunique())\nprint('# Parent notebooks not in train data:',train_ancestors[~train_ancestors['parent_id'].isin(train_orders['id'].tolist())]['parent_id'].nunique())\nprint('# Ancestor notebooks:',train_ancestors['ancestor_id'].nunique())\nprint('# Ancestor notebooks not in train data:',train_ancestors[~train_ancestors['ancestor_id'].isin(train_orders['id'].tolist())]['ancestor_id'].nunique())\nprint('Overlap between train_orders and train_ancestors datasets:',train_orders[train_orders['id'].isin(train_ancestors['id'].tolist())].shape[0]/train_orders.shape[0] * 100,'%')\ntrain_ancestors.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-24T05:38:56.488458Z","iopub.execute_input":"2022-06-24T05:38:56.488840Z","iopub.status.idle":"2022-06-24T05:38:56.857303Z","shell.execute_reply.started":"2022-06-24T05:38:56.488811Z","shell.execute_reply":"2022-06-24T05:38:56.856129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(sample_submission.shape)\nprint('\\n# Test data notebooks in train data:', sample_submission[sample_submission['id'].isin(train_orders['id'].tolist())].shape[0])\nsample_submission.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-24T05:39:16.141581Z","iopub.execute_input":"2022-06-24T05:39:16.141955Z","iopub.status.idle":"2022-06-24T05:39:16.189953Z","shell.execute_reply.started":"2022-06-24T05:39:16.141926Z","shell.execute_reply":"2022-06-24T05:39:16.188857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('\\nTop 10 ancestors with highest # latest notebooks\\n')\ntrain_ancestors.groupby(['ancestor_id']).agg({'id':'nunique'}).reset_index().sort_values(by = 'id', ascending = False).rename(columns = {'id':'# Notebooks linked'}).head(10).reset_index(drop = True)","metadata":{"execution":{"iopub.status.busy":"2022-06-24T05:39:30.704428Z","iopub.execute_input":"2022-06-24T05:39:30.704884Z","iopub.status.idle":"2022-06-24T05:39:31.095867Z","shell.execute_reply.started":"2022-06-24T05:39:30.704848Z","shell.execute_reply":"2022-06-24T05:39:31.094489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('\\nTop 10 notebooks with highest # cells\\n')\ntrain_orders.sort_values(by = '# cells', ascending = False).head(10).reset_index(drop = True)","metadata":{"execution":{"iopub.status.busy":"2022-06-24T05:39:43.839099Z","iopub.execute_input":"2022-06-24T05:39:43.839507Z","iopub.status.idle":"2022-06-24T05:39:43.887577Z","shell.execute_reply.started":"2022-06-24T05:39:43.839472Z","shell.execute_reply":"2022-06-24T05:39:43.886170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_notebook(sample_id, num_cells_to_show):\n    print('\\n# Cells in notebook:', train_orders[train_orders['id']==sample_id]['# cells'].iloc[0])\n    json_data = json.load(open('../input/AI4Code/train//' + sample_id + '.json', \"r\"))\n    assert (train_orders[train_orders['id']==sample_id]['# cells'].iloc[0]==len(json_data['source'].keys())) and (train_orders[train_orders['id']==sample_id]['# cells'].iloc[0]==len(json_data['cell_type'].keys()))\n    cell_order = train_orders[train_orders['id']==sample_id]['cell_order'].iloc[0].split(' ')\n    full_text = [json_data['source'][x] for x in json_data['source'].keys()]\n    print('\\n# Characters in entire notebook:', len('\\n'.join(full_text)),'\\n')\n    print('# Space separated words in entire notebook:', len('\\n'.join(full_text).split(' ')),'\\n')\n    print(colored('---------------------------Notebook---------------------------', attrs=['bold']),'\\n')\n    for i in range(0, len(cell_order)):\n        key = cell_order[i]\n        color = 'green' if json_data['cell_type'][key] == 'code' else 'red'\n        print(colored(json_data['cell_type'][key] + ' cell:', color), json_data['source'][key].split('\\n'),'\\n')\n        if num_cells_to_show <= i:\n            break\n\nnum_cells_list = [x for x in range(1, 1005, 10)]          \nwidgets.interactive(show_notebook, sample_id = random.sample(train_orders['id'].tolist(),5000), num_cells_to_show = num_cells_list)","metadata":{"execution":{"iopub.status.busy":"2022-06-24T05:39:55.906673Z","iopub.execute_input":"2022-06-24T05:39:55.907047Z","iopub.status.idle":"2022-06-24T05:39:56.134083Z","shell.execute_reply.started":"2022-06-24T05:39:55.907017Z","shell.execute_reply":"2022-06-24T05:39:56.132816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def count_inversions(a):\n    inversions = 0\n    sorted_so_far = []\n    for i, u in enumerate(a):  # O(N)\n        j = bisect(sorted_so_far, u)  # O(log N)\n        inversions += i - j\n        sorted_so_far.insert(j, u)  # O(N)\n    return inversions","metadata":{"execution":{"iopub.status.busy":"2022-06-24T05:40:09.129544Z","iopub.execute_input":"2022-06-24T05:40:09.129951Z","iopub.status.idle":"2022-06-24T05:40:09.136280Z","shell.execute_reply.started":"2022-06-24T05:40:09.129917Z","shell.execute_reply":"2022-06-24T05:40:09.135370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def kendall_tau(ground_truth, predictions):\n    total_inversions = 0  # total inversions in predicted ranks across all instances\n    total_2max = 0  # maximum possible inversions across all instances\n    for gt, pred in zip(ground_truth, predictions):\n        ranks = [gt.index(x) for x in pred]  # rank predicted order in terms of ground truth\n        total_inversions += count_inversions(ranks)\n        n = len(gt)\n        total_2max += n * (n - 1)\n    return 1 - 4 * total_inversions / total_2max","metadata":{"execution":{"iopub.status.busy":"2022-06-24T05:40:16.690588Z","iopub.execute_input":"2022-06-24T05:40:16.690961Z","iopub.status.idle":"2022-06-24T05:40:16.698647Z","shell.execute_reply.started":"2022-06-24T05:40:16.690931Z","shell.execute_reply":"2022-06-24T05:40:16.697691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_id = 'ba14b6fba142e9'\njson_data = json.load(open('../input/AI4Code/train//' + sample_id + '.json', \"r\"))\ncell_order = train_orders[train_orders['id']==sample_id]['cell_order'].iloc[0].split(' ')\ndummy_pred = cell_order.copy()\nrandom.Random(SEED).shuffle(dummy_pred)\ncell_order = [cell_order]\ndummy_pred = [dummy_pred]\nprint('\\nKendall Tau correlation:', kendall_tau(cell_order, dummy_pred))","metadata":{"execution":{"iopub.status.busy":"2022-06-24T05:40:25.523188Z","iopub.execute_input":"2022-06-24T05:40:25.523642Z","iopub.status.idle":"2022-06-24T05:40:25.585770Z","shell.execute_reply.started":"2022-06-24T05:40:25.523600Z","shell.execute_reply":"2022-06-24T05:40:25.584555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}