{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"AI4Code - Exploratory Data Analysis
\n
\n\nThe goal of this competition is to understand the relationship between code and comments in Python notebooks. You are challenged to reconstruct the order of markdown cells in a given notebook based on the order of the code cells, demonstrating comprehension of which natural language references which code.","metadata":{}},{"cell_type":"markdown","source":"### Importing data and libraries","metadata":{}},{"cell_type":"code","source":"# importing libraries\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n#importing dataset\nBase_dir = \"../input/AI4Code/\"\ntrain_orders = pd.read_csv(os.path.join(Base_dir, \"train_orders.csv\"))\ntrain_ancestors = pd.read_csv(os.path.join(Base_dir, \"train_ancestors.csv\")) \ntrain = pd.read_csv(\"../input/ai4code-train-dataframe/train.csv\")\ntrain.drop([\"Unnamed: 0\"], axis=1, inplace=True)\ntest = pd.read_csv(os.path.join(Base_dir, \"sample_submission.csv\"))","metadata":{"execution":{"iopub.status.busy":"2022-05-12T06:17:23.967186Z","iopub.execute_input":"2022-05-12T06:17:23.967439Z","iopub.status.idle":"2022-05-12T06:18:25.158527Z","shell.execute_reply.started":"2022-05-12T06:17:23.967412Z","shell.execute_reply":"2022-05-12T06:18:25.157717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# preparing data_orders.csv\ndict_id_type = dict(zip(train.cell_id, train.cell_type))\ndict_id_source = dict(zip(train.cell_id, train.source))\ntrain_orders.cell_order = train_orders.cell_order.apply(lambda x: x.split())\ntrain_orders = train_orders.explode(\"cell_order\")\ntrain_orders[\"cell_type\"] = train_orders.cell_order.map(dict_id_type)\ntrain_orders[\"source\"] = train_orders.cell_order.map(dict_id_source)\ndel dict_id_type, dict_id_source","metadata":{"execution":{"iopub.status.busy":"2022-05-12T06:18:25.160053Z","iopub.execute_input":"2022-05-12T06:18:25.161043Z","iopub.status.idle":"2022-05-12T06:19:09.9051Z","shell.execute_reply.started":"2022-05-12T06:18:25.161Z","shell.execute_reply":"2022-05-12T06:19:09.903887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# to help with the code vs cell statistics\ncell_type_with_orders = train_orders.groupby(\"id\").agg({\"cell_type\": lambda x: x.tolist()})\ncell_type_with_orders[\"first_type\"] = cell_type_with_orders[\"cell_type\"].apply(lambda x: x[0])\ncell_type_with_orders[\"second_type\"] = cell_type_with_orders[\"cell_type\"].apply(lambda x: x[0])\ncell_type_with_orders[\"number_markdown\"] = cell_type_with_orders[\"cell_type\"].apply(lambda x: x.count('markdown'))\ncell_type_with_orders[\"number_code\"] = cell_type_with_orders[\"cell_type\"].apply(lambda x: x.count('code'))","metadata":{"execution":{"iopub.status.busy":"2022-05-12T06:19:09.90667Z","iopub.execute_input":"2022-05-12T06:19:09.906917Z","iopub.status.idle":"2022-05-12T06:19:13.486114Z","shell.execute_reply.started":"2022-05-12T06:19:09.906874Z","shell.execute_reply":"2022-05-12T06:19:13.485157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# to help with the source code statistics\ncell_source_with_orders = train_orders.groupby(\"id\").agg({\"source\": lambda x: x.tolist()})\ncell_source_with_orders[\"first_codeblock\"] = cell_source_with_orders.source.apply(lambda x: x[0])","metadata":{"execution":{"iopub.status.busy":"2022-05-12T06:19:13.489256Z","iopub.execute_input":"2022-05-12T06:19:13.489535Z","iopub.status.idle":"2022-05-12T06:19:30.6869Z","shell.execute_reply.started":"2022-05-12T06:19:13.489507Z","shell.execute_reply":"2022-05-12T06:19:30.685891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Number of notebooks in train and test dataset","metadata":{}},{"cell_type":"code","source":"print(\"No. of jupyter notebooks in train dataset: \", len(train_orders))\nprint(\"No. of jupyter notebooks in test dataset: \", len(test))","metadata":{"execution":{"iopub.status.busy":"2022-05-12T06:19:30.688348Z","iopub.execute_input":"2022-05-12T06:19:30.688991Z","iopub.status.idle":"2022-05-12T06:19:30.694756Z","shell.execute_reply.started":"2022-05-12T06:19:30.688939Z","shell.execute_reply":"2022-05-12T06:19:30.693827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Statistics around cell_types","metadata":{}},{"cell_type":"code","source":"sns.countplot(train[\"cell_type\"])","metadata":{"execution":{"iopub.status.busy":"2022-05-12T06:19:30.696184Z","iopub.execute_input":"2022-05-12T06:19:30.696386Z","iopub.status.idle":"2022-05-12T06:19:34.508021Z","shell.execute_reply.started":"2022-05-12T06:19:30.69636Z","shell.execute_reply":"2022-05-12T06:19:34.506349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Median code cells in train dataset: \", cell_type_with_orders[\"number_code\"].median())\nprint(\"Median markdown cells in train dataset: \", cell_type_with_orders[\"number_markdown\"].median())","metadata":{"execution":{"iopub.status.busy":"2022-05-12T06:19:34.510751Z","iopub.execute_input":"2022-05-12T06:19:34.511187Z","iopub.status.idle":"2022-05-12T06:19:34.527571Z","shell.execute_reply.started":"2022-05-12T06:19:34.511145Z","shell.execute_reply":"2022-05-12T06:19:34.526134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Insight :- There are a lot more code cells than markdown cells","metadata":{}},{"cell_type":"code","source":"print(\"Most common first cell type: \", cell_type_with_orders[\"first_type\"].mode()[0])\nprint(\"Number of markdown first cell: \", len(cell_type_with_orders[cell_type_with_orders[\"first_type\"] == \"markdown\"]))\nprint(\"Number of code first cell: \", len(cell_type_with_orders[cell_type_with_orders[\"first_type\"] == \"code\"]))","metadata":{"execution":{"iopub.status.busy":"2022-05-12T06:19:34.529225Z","iopub.execute_input":"2022-05-12T06:19:34.529542Z","iopub.status.idle":"2022-05-12T06:19:34.624119Z","shell.execute_reply.started":"2022-05-12T06:19:34.529502Z","shell.execute_reply":"2022-05-12T06:19:34.622446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Insight :- Even though code cells are more common than markdown cells. Markdown cells are more commonly the first cell.","metadata":{}},{"cell_type":"code","source":"for i in range(5):\n print(cell_source_with_orders[\"first_codeblock\"].sample(1)[0])\n print(\"*****************************************************\")","metadata":{"execution":{"iopub.status.busy":"2022-05-12T06:19:34.626024Z","iopub.execute_input":"2022-05-12T06:19:34.626277Z","iopub.status.idle":"2022-05-12T06:19:34.660425Z","shell.execute_reply.started":"2022-05-12T06:19:34.626251Z","shell.execute_reply":"2022-05-12T06:19:34.6584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Insight - In a lot of these cases, after looking through a lot of them, there are two possibilities
\n- Either it is a code cell, in that case there are imports, usually the one that kaggle gives by default, import pandas as pd, import numpy as np, import os\n- Or it is the title of the notebook, with either some description or just the title","metadata":{}},{"cell_type":"code","source":"cell_type_with_orders[\"first_type_markdown\"] = cell_type_with_orders.first_type.apply(lambda x: x==\"markdown\")","metadata":{"execution":{"iopub.status.busy":"2022-05-12T06:21:56.808095Z","iopub.execute_input":"2022-05-12T06:21:56.808386Z","iopub.status.idle":"2022-05-12T06:21:56.839469Z","shell.execute_reply.started":"2022-05-12T06:21:56.808358Z","shell.execute_reply":"2022-05-12T06:21:56.838426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cell_type_with_orders[['first_type_markdown', 'number_markdown']].corr()","metadata":{"execution":{"iopub.status.busy":"2022-05-12T06:22:20.340052Z","iopub.execute_input":"2022-05-12T06:22:20.340891Z","iopub.status.idle":"2022-05-12T06:22:20.357298Z","shell.execute_reply.started":"2022-05-12T06:22:20.340846Z","shell.execute_reply":"2022-05-12T06:22:20.35551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Work in progress\nIf you find this useful, do upvote","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}