{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Sort the Code!\n\nThis notebook was created on a live twitch stream session. [Check out my channel here](https://www.twitch.tv/medallionstallion_)\n\n<img src=\"https://allaboutplanners.com.au/wp-content/uploads/2017/05/how-to-color-code-your-planner-using-Zooms-organized-blog-post-idea-tracking-brain-dumping-use-empty-notebook-use-empty-notes-pages-in-my-planner-min-1024x768.jpg\" width=\"500\" height=\"250\" />\n\n\nThe goal of this competition is to understand the relationship between code and comments in Python notebooks.\n\nThe task is to create an algorithm that can sort notebook cells in the correct order.\n- We are given 130,000 notebooks in the training set with the correct order\n- We need to predict on the test set the correct order\n\nLets go!","metadata":{}},{"cell_type":"markdown","source":"# Load the Data","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pylab as plt\nimport seaborn as sns\nimport json\n\nplt.style.use(\"ggplot\")\nmy_pal = sns.color_palette()","metadata":{"execution":{"iopub.status.busy":"2022-05-13T14:58:59.843931Z","iopub.execute_input":"2022-05-13T14:58:59.844654Z","iopub.status.idle":"2022-05-13T14:59:00.952840Z","shell.execute_reply.started":"2022-05-13T14:58:59.844557Z","shell.execute_reply":"2022-05-13T14:59:00.951983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"../input/AI4Code/train_orders.csv\")\nancestors = pd.read_csv(\"../input/AI4Code/train_ancestors.csv\")\nss = pd.read_csv(\"../input/AI4Code/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-05-13T14:59:00.954786Z","iopub.execute_input":"2022-05-13T14:59:00.955089Z","iopub.status.idle":"2022-05-13T14:59:02.710365Z","shell.execute_reply.started":"2022-05-13T14:59:00.955050Z","shell.execute_reply":"2022-05-13T14:59:02.709416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helper Functions\nThese functions help us load the training data from json. I also saved all the combined training data as a parquet file that we can load for quick access.","metadata":{}},{"cell_type":"code","source":"def load_example(id, is_train=True):\n    \"\"\"\n    Helper for loading json file of a training example\n    \"\"\"\n    filedir = \"train\" if is_train else \"test\"\n    with open(f\"../input/AI4Code/{filedir}/{id}.json\") as f:\n        example = json.load(f)\n    return example","metadata":{"execution":{"iopub.status.busy":"2022-05-13T14:59:02.711585Z","iopub.execute_input":"2022-05-13T14:59:02.711995Z","iopub.status.idle":"2022-05-13T14:59:02.717397Z","shell.execute_reply.started":"2022-05-13T14:59:02.711943Z","shell.execute_reply":"2022-05-13T14:59:02.716834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load an example\nexample_id = train[\"id\"].sample(1, random_state=529).values[0]\nload_example(example_id).keys()","metadata":{"execution":{"iopub.status.busy":"2022-05-13T14:59:02.718433Z","iopub.execute_input":"2022-05-13T14:59:02.719409Z","iopub.status.idle":"2022-05-13T14:59:02.753280Z","shell.execute_reply.started":"2022-05-13T14:59:02.719363Z","shell.execute_reply":"2022-05-13T14:59:02.752403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_example_df(example_id, train, ancestors):\n    \"\"\"\n    Creates a pandas dataframe of the json cells and correct order.\n    \"\"\"\n    cell_order = train.query(\"id == @example_id\")[\"cell_order\"].values[0]\n    example_df = pd.DataFrame(load_example(example_id))\n    example_df[\"id\"] = example_id\n    my_orders = {}\n\n    for idx, c in enumerate(cell_order.split(\" \")):\n        my_orders[c] = idx\n\n    example_df[\"order\"] = example_df.index.map(my_orders)\n    example_df.reset_index().rename(columns={\"index\": \"cell\"})\n\n    example_df[\"ancestor_id\"] = ancestors.query(\"id == @example_id\")[\n        \"ancestor_id\"\n    ].values[0]\n    example_df[\"parent_id\"] = ancestors.query(\"id == @example_id\")[\"parent_id\"].values[\n        0\n    ]\n    example_df = example_df.reset_index().rename(columns={\"index\": \"cell\"})\n    example_df = example_df.sort_values(\"order\").reset_index(drop=True)\n    example_df[\"id\"] = example_id\n    col_order = [\n        \"id\",\n        \"cell\",\n        \"cell_type\",\n        \"source\",\n        \"order\",\n        \"ancestor_id\",\n        \"parent_id\",\n    ]\n    example_df = example_df[col_order]\n    return example_df","metadata":{"execution":{"iopub.status.busy":"2022-05-13T14:59:02.754995Z","iopub.execute_input":"2022-05-13T14:59:02.755237Z","iopub.status.idle":"2022-05-13T14:59:02.765255Z","shell.execute_reply.started":"2022-05-13T14:59:02.755209Z","shell.execute_reply":"2022-05-13T14:59:02.764266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now we have a dataframe with all the cells, their types and the contents of them along with the correct order.","metadata":{}},{"cell_type":"code","source":"# Load the example as a dataframe\nexample_df = get_example_df(example_id, train, ancestors)\nexample_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-13T14:59:02.766496Z","iopub.execute_input":"2022-05-13T14:59:02.766748Z","iopub.status.idle":"2022-05-13T14:59:02.840112Z","shell.execute_reply.started":"2022-05-13T14:59:02.766721Z","shell.execute_reply":"2022-05-13T14:59:02.839219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Combine Data as Parquet for Fast Loading\n\nThe below function was used offline to create a combined version of the training data with all the values in a single dataframe.\n\n[Check out the dataset here](https://www.kaggle.com/datasets/robikscube/ai4code-parquet-tabular)","metadata":{}},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport json\nfrom tqdm.contrib.concurrent import process_map\n\n\ndef combine_train():\n    train = pd.read_csv(\"../input/AI4Code/train_orders.csv\")\n    ancestors = pd.read_csv(\"../input/AI4Code/train_ancestors.csv\")\n\n    # Get the list of json files\n    train_jsons = os.listdir(\"../input/AI4Code/train/\")\n    print(f\"There are {len(train_jsons)} training json files\")\n\n    all_ids = train[\"id\"].unique()\n    args = ((ids, train, ancestors) for ids in all_ids)\n    results = process_map(\n        get_example_df, args, max_workers=32, chunksize=500, total=len(all_ids)\n    )\n    all_examples = pd.concat(results).reset_index(drop=True)\n    all_examples.to_parquet(\"train_all.parquet\")","metadata":{"execution":{"iopub.status.busy":"2022-05-13T03:23:23.189358Z","iopub.execute_input":"2022-05-13T03:23:23.189658Z","iopub.status.idle":"2022-05-13T03:23:23.210852Z","shell.execute_reply.started":"2022-05-13T03:23:23.189627Z","shell.execute_reply":"2022-05-13T03:23:23.209327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_all = pd.read_parquet(\"../input/ai4code-parquet-tabular/train_all.parquet\")","metadata":{"execution":{"iopub.status.busy":"2022-05-13T03:23:26.352058Z","iopub.execute_input":"2022-05-13T03:23:26.352331Z","iopub.status.idle":"2022-05-13T03:23:49.849907Z","shell.execute_reply.started":"2022-05-13T03:23:26.352307Z","shell.execute_reply":"2022-05-13T03:23:49.848761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA of the training notebook data.\n\nSome questions to answer:\n- How many cells on average per notebook?\n- What is the breakdown of markdown vs code cells.\n- How many notebooks share the same ancestor_id","metadata":{}},{"cell_type":"code","source":"train_all['cell_type'].value_counts() \\\n    .plot(kind='barh',\n          title='Code vs. Markdown Cells in Total',\n          color=my_pal[2], figsize=(8, 5))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-13T03:29:35.973552Z","iopub.execute_input":"2022-05-13T03:29:35.973807Z","iopub.status.idle":"2022-05-13T03:29:36.368653Z","shell.execute_reply.started":"2022-05-13T03:29:35.973781Z","shell.execute_reply":"2022-05-13T03:29:36.367921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Number of Cells per id\ntrain_all['id'].value_counts() \\\n    .plot(kind='hist',\n          bins=50,\n          title='Distribution of # of Cells per Notebook')\nprint('The median number of cells per notebook is:',\n      train_all['id'].value_counts().median())\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-13T03:29:39.393321Z","iopub.execute_input":"2022-05-13T03:29:39.39357Z","iopub.status.idle":"2022-05-13T03:29:40.295739Z","shell.execute_reply.started":"2022-05-13T03:29:39.393544Z","shell.execute_reply":"2022-05-13T03:29:40.295137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There is a notebook with over 1000 cells!","metadata":{}},{"cell_type":"code","source":"train_all['id'].value_counts() \\\n    .head(50).sort_values() \\\n    .plot(kind='barh', color=my_pal[3], figsize=(8, 10),\n         title='Top 50 Notebooks by # of Cells')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-13T03:31:28.643012Z","iopub.execute_input":"2022-05-13T03:31:28.644355Z","iopub.status.idle":"2022-05-13T03:31:29.530841Z","shell.execute_reply.started":"2022-05-13T03:31:28.644286Z","shell.execute_reply":"2022-05-13T03:31:29.53002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## What's the most forked notebook?","metadata":{}},{"cell_type":"code","source":"# Find top Id\n# Find the most \"parent\"\nancestors['parent_id'].value_counts().head(20) \\\n    .sort_values() \\\n    .plot(kind='barh', figsize=(8, 8),\n          color=my_pal[1], title='Top Forked Notebooks')","metadata":{"execution":{"iopub.status.busy":"2022-05-13T03:34:10.893805Z","iopub.execute_input":"2022-05-13T03:34:10.89456Z","iopub.status.idle":"2022-05-13T03:34:11.154505Z","shell.execute_reply.started":"2022-05-13T03:34:10.894526Z","shell.execute_reply":"2022-05-13T03:34:11.153423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Find the top \"forked\" notebook \n- and print the first few cells of one of the forks.\nIt's an intoduction to machine learning notebook!","metadata":{}},{"cell_type":"code","source":"top_forked = ancestors['parent_id'].value_counts().index[0]\n# This parent id does not appear in our dataset\n# Take one of the forks of this top id\na_fork = ancestors.query('parent_id == @top_forked')['id'].values[0]\n\nprint(train_all.query('id == @a_fork')['source'].values[0])\nprint(train_all.query('id == @a_fork')['source'].values[1])","metadata":{"execution":{"iopub.status.busy":"2022-05-13T03:37:52.366009Z","iopub.execute_input":"2022-05-13T03:37:52.366357Z","iopub.status.idle":"2022-05-13T03:37:52.649327Z","shell.execute_reply.started":"2022-05-13T03:37:52.366325Z","shell.execute_reply":"2022-05-13T03:37:52.648248Z"},"trusted":true},"execution_count":null,"outputs":[]}]}