{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Random shuffle the markdown cell only to get real benchmark","metadata":{}},{"cell_type":"code","source":"import json\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nfrom scipy import sparse\nfrom tqdm import tqdm\n\npd.options.display.width = 180\npd.options.display.max_colwidth = 120\n\ndata_dir = Path('../input/AI4Code')","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-06-14T14:43:03.695790Z","iopub.execute_input":"2022-06-14T14:43:03.696237Z","iopub.status.idle":"2022-06-14T14:43:03.702245Z","shell.execute_reply.started":"2022-06-14T14:43:03.696202Z","shell.execute_reply":"2022-06-14T14:43:03.701299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUM_TRAIN = 10000\ndef read_notebook(path):\n    return (\n        pd.read_json(\n            path,\n            dtype={'cell_type': 'category', 'source': 'str'})\n        .assign(id=path.stem)\n        .rename_axis('cell_id')\n    )\n\n\npaths_train = list((data_dir / 'train').glob('*.json'))[:NUM_TRAIN]\nnotebooks_train = [\n    read_notebook(path) for path in tqdm(paths_train, desc='Train NBs')\n]\ndf = (\n    pd.concat(notebooks_train)\n    .set_index('id', append=True)\n    .swaplevel()\n    .sort_index(level='id', sort_remaining=False)\n)","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-06-14T14:43:03.725672Z","iopub.execute_input":"2022-06-14T14:43:03.726417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_orders = pd.read_csv(\n    data_dir / 'train_orders.csv',\n    index_col='id',\n    squeeze=True,\n).str.split()  # Split the string representation of cell_ids into a list","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_orders","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_ranks(base, derived):\n    return [base.index(d) for d in derived]","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_orders_ = df_orders.to_frame().join(\n    df.reset_index('cell_id').groupby('id')['cell_id'].apply(list),\n    how='right',\n)\n\nranks = {}\nfor id_, cell_order, cell_id in df_orders_.itertuples():\n    ranks[id_] = {'cell_id': cell_id, 'rank': get_ranks(cell_order, cell_id)}\n\ndf_ranks = (\n    pd.DataFrame\n    .from_dict(ranks, orient='index')\n    .rename_axis('id')\n    .apply(pd.Series.explode)\n    .set_index('cell_id', append=True)\n)","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ranks","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.merge(df, df_ranks, left_index=True, right_index=True)\ndf.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get pct_rank  rank: 1 2 3 -> 0.25 0.5 0.75\ndf['pct_rank'] = df.groupby([\"id\", \"cell_type\"])[\"rank\"].apply(lambda s: pd.Series((np.arange(len(s)) + 1) /(len(s) + 1), index=s.index))\ndf['pred'] = np.random.uniform(size=df.shape[0])\n# keep code cell in the correct order\ndf.loc[df['cell_type']=='code', \"pred\"] = df.loc[df['cell_type']=='code', \"pct_rank\"]\ndf.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from bisect import bisect\n\ndef count_inversions(a):\n    inversions = 0\n    sorted_so_far = []\n    for i, u in enumerate(a):\n        j = bisect(sorted_so_far, u)\n        inversions += i - j\n        sorted_so_far.insert(j, u)\n    return inversions\n\n\ndef kendall_tau(ground_truth, predictions):\n    total_inversions = 0\n    total_2max = 0  # twice the maximum possible inversions across all instances\n    for gt, pred in zip(ground_truth, predictions):\n        ranks = [gt.index(x) for x in pred]  # rank predicted order in terms of ground truth\n        total_inversions += count_inversions(ranks)\n        n = len(gt)\n        total_2max += n * (n - 1)\n    return 1 - 4 * total_inversions / total_2max","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# perfect rank: should be 1\ny_dummy = df.reset_index('cell_id').sort_values(\"rank\").groupby('id')['cell_id'].apply(list)\nkendall_tau(df_orders.loc[y_dummy.index], y_dummy)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# random shuffle those markdown cell\ny_dummy = df.reset_index('cell_id').sort_values(\"pred\").groupby('id')['cell_id'].apply(list)\nkendall_tau(df_orders.loc[y_dummy.index], y_dummy)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# all random shuffle: should close to 0\ndf['all_random'] = np.random.uniform(size=df.shape[0])\ny_dummy = df.reset_index('cell_id').sort_values(\"all_random\").groupby('id')['cell_id'].apply(list)\nkendall_tau(df_orders.loc[y_dummy.index], y_dummy)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submit","metadata":{}},{"cell_type":"code","source":"paths_test = list((data_dir / 'test').glob('*.json'))\nnotebooks_test = [\n    read_notebook(path) for path in tqdm(paths_test, desc='Test NBs')\n]\ndf_test = (\n    pd.concat(notebooks_test)\n    .set_index('id', append=True)\n    .swaplevel()\n    .sort_index(level='id', sort_remaining=False)\n)","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.reset_index(inplace=True)\ndf_test.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test['rank'] = list(range(len(df_test))) \ndf_test['pct_rank'] = df_test.groupby([\"id\", \"cell_type\"])[\"rank\"].apply(lambda s: pd.Series((np.arange(len(s)) + 1) /(len(s) + 1), index=s.index))\ndf_test[\"pred\"] = np.random.uniform(size=df_test.shape[0])\ndf_test.loc[df_test['cell_type']=='code', \"pred\"] = df_test.loc[df_test['cell_type']=='code', \"pct_rank\"]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df = df_test.sort_values(\"pred\").groupby(\"id\")[\"cell_id\"].apply(lambda x: \" \".join(x)).reset_index()\nsub_df.rename(columns={\"cell_id\": \"cell_order\"}, inplace=True)\nsub_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}