{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport json\nimport time\nimport gc\nimport plotly.graph_objects as go\nimport matplotlib.pyplot as plt\nfrom glob import glob\nfrom plotly.subplots import make_subplots","metadata":{"execution":{"iopub.status.busy":"2022-05-12T18:06:24.62384Z","iopub.execute_input":"2022-05-12T18:06:24.624132Z","iopub.status.idle":"2022-05-12T18:06:24.6563Z","shell.execute_reply.started":"2022-05-12T18:06:24.62406Z","shell.execute_reply":"2022-05-12T18:06:24.655831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = \"../input/AI4Code/\"\ntrain_path = path + 'train/'\ntest_path = path + 'test/'\ntrain_orders = pd.read_csv(path + 'train_orders.csv')\nsample_submission = pd.read_csv(path + \"sample_submission.csv\")\ntrain_ancestors = pd.read_csv(path + 'train_ancestors.csv')","metadata":{"execution":{"iopub.status.busy":"2022-05-12T18:06:24.989907Z","iopub.execute_input":"2022-05-12T18:06:24.990384Z","iopub.status.idle":"2022-05-12T18:06:27.031006Z","shell.execute_reply.started":"2022-05-12T18:06:24.990359Z","shell.execute_reply":"2022-05-12T18:06:27.029503Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

Let's see what we have in train_orders.csv file

","metadata":{}},{"cell_type":"code","source":"print(train_orders.shape)\ndisplay(train_orders.head())\nprint(\"\\n\\n\")\nprint(sample_submission.shape)\ndisplay(sample_submission.head())","metadata":{"execution":{"iopub.status.busy":"2022-05-12T18:06:27.032467Z","iopub.execute_input":"2022-05-12T18:06:27.032709Z","iopub.status.idle":"2022-05-12T18:06:27.058078Z","shell.execute_reply.started":"2022-05-12T18:06:27.032681Z","shell.execute_reply":"2022-05-12T18:06:27.056933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

As we can see there train data contain 139256 notebooks and test data contain 4 notebooks and we have 2 columns in train_orders.csv. One is \"id\" column and another is \"cell_order\" column.

\n

id columns contain id of notebook which is available in .json format. And cell_order columns contain order of cell in notebooks which is target column

","metadata":{}},{"cell_type":"code","source":"train_orders[\"path\"] = train_orders['id'].apply(lambda x: train_path+x+\".json\")","metadata":{"execution":{"iopub.status.busy":"2022-05-12T18:06:27.059201Z","iopub.execute_input":"2022-05-12T18:06:27.059419Z","iopub.status.idle":"2022-05-12T18:06:27.104207Z","shell.execute_reply.started":"2022-05-12T18:06:27.059392Z","shell.execute_reply":"2022-05-12T18:06:27.103637Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target = train_orders[\"cell_order\"].values[0]\ntarget = target.split(\" \")\ntemp_path = train_orders['path'].values[0]\nwith open(temp_path, 'r') as f:\n data = json.load(f)\ntemp_df = pd.DataFrame()\ntemp_df[\"cell_id\"] = data['cell_type'].keys()\ntemp_df[\"cell_type\"] = data[\"cell_type\"].values()\ntemp_df[\"source\"] = temp_df['cell_id'].apply(lambda x: data['source'][x])\ntemp_df[\"order\"] = temp_df['cell_id'].apply(lambda x: target.index(x))\ntemp_df","metadata":{"execution":{"iopub.status.busy":"2022-05-12T18:06:27.105905Z","iopub.execute_input":"2022-05-12T18:06:27.106689Z","iopub.status.idle":"2022-05-12T18:06:27.142777Z","shell.execute_reply.started":"2022-05-12T18:06:27.106658Z","shell.execute_reply":"2022-05-12T18:06:27.142101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

If we look at order of code cells. Code cells are already in order but markdown cells are shuffled.

\n

Means markdown cell can come at the start of the notebook or at the end of the notebook it can come between to code cells or it can come next to markdown cell too

","metadata":{}},{"cell_type":"code","source":"def create_df(path,cell_order):\n cell_order = cell_order.split(\" \")\n \n with open(path, 'r') as f:\n data = json.load(f)\n \n df = pd.DataFrame()\n df[\"cell_id\"] = data[\"cell_type\"].keys()\n df[\"cell_type\"] = data[\"cell_type\"].values()\n df[\"source\"] = df[\"cell_id\"].apply(lambda x: data[\"source\"][x])\n df[\"order\"] = df[\"cell_id\"].apply(lambda x: cell_order.index(x))\n df = df.sort_values(\"order\")\n \n return df\n\nstart = time.time()\ntrain_orders['metadata'] = train_orders.apply(lambda x: create_df(x['path'],x['cell_order']),1)\nend = time.time()\nduration = end - start\n#print(duration)","metadata":{"execution":{"iopub.status.busy":"2022-05-12T18:06:27.143804Z","iopub.execute_input":"2022-05-12T18:06:27.144024Z","iopub.status.idle":"2022-05-12T18:24:22.055739Z","shell.execute_reply.started":"2022-05-12T18:06:27.143994Z","shell.execute_reply":"2022-05-12T18:24:22.054652Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_orders[\"path\"]\ngc.collect()","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-05-12T18:24:22.070275Z","iopub.execute_input":"2022-05-12T18:24:22.070603Z","iopub.status.idle":"2022-05-12T18:24:25.210934Z","shell.execute_reply.started":"2022-05-12T18:24:22.070559Z","shell.execute_reply":"2022-05-12T18:24:25.210011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_orders.head()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-12T18:24:25.212429Z","iopub.execute_input":"2022-05-12T18:24:25.212714Z","iopub.status.idle":"2022-05-12T18:24:25.263872Z","shell.execute_reply.started":"2022-05-12T18:24:25.212683Z","shell.execute_reply":"2022-05-12T18:24:25.262947Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def c_next_to_c(df):\n \"\"\" \n df: json file in df format which contain cell_id,cell_type, source, order columns already(sorted w.r.t to order)\n return: num of times code cell came next to code cell\n \"\"\"\n array = df['cell_type'].values\n count = 0\n for idx, cell_type in enumerate(array[:-1]):\n if cell_type==\"code\":\n if cell_type == array[idx+1]:\n count+=1\n return count\ndef m_next_to_m(df):\n \"\"\" \n df: json file in df format which contain cell_id,cell_type, source, order columns already(sorted w.r.t to order)\n return: num of times markdown cell came next to markdown cell\n \"\"\"\n array = df['cell_type'].values\n count = 0\n for idx, cell_type in enumerate(array[:-1]):\n if cell_type==\"markdown\":\n if cell_type == array[idx+1]:\n count+=1\n return count\n \ndef c_next_to_m(df):\n \"\"\" \n df: json file in df format which contain cell_id,cell_type, source, order columns already(sorted w.r.t to order)\n return: num of times code cell came next to markdown cell\n \"\"\"\n array = df['cell_type'].values\n count = 0\n for idx, cell_type in enumerate(array[:-1]):\n if cell_type==\"markdown\":\n if array[idx+1] == \"code\":\n count+=1\n return count\ndef m_next_to_c(df):\n \"\"\" \n df: json file in df format which contain cell_id,cell_type, source, order columns already(sorted w.r.t to order)\n return: num of times code markdown came next to code cell\n \"\"\"\n array = df['cell_type'].values\n count = 0\n for idx, cell_type in enumerate(array[:-1]):\n if cell_type==\"code\":\n if array[idx+1] == \"markdown\":\n count+=1\n return count\n\n\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-12T18:24:25.265366Z","iopub.execute_input":"2022-05-12T18:24:25.265558Z","iopub.status.idle":"2022-05-12T18:24:25.279444Z","shell.execute_reply.started":"2022-05-12T18:24:25.265536Z","shell.execute_reply":"2022-05-12T18:24:25.278465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_orders[\"total_code_cell\"] = train_orders['metadata'].apply(lambda x: len(x[x['cell_type']=='code']))\ntrain_orders[\"total_markdown_cell\"] = train_orders['metadata'].apply(lambda x: len(x[x['cell_type']=='markdown']))\n\ntrain_orders[\"c_next_to_c\"] = train_orders['metadata'].apply(lambda x: c_next_to_c(x))\ntrain_orders[\"m_next_to_m\"] = train_orders['metadata'].apply(lambda x: m_next_to_m(x))\ntrain_orders[\"c_next_to_m\"] = train_orders['metadata'].apply(lambda x: c_next_to_m(x))\ntrain_orders[\"m_next_to_c\"] = train_orders['metadata'].apply(lambda x: m_next_to_c(x))\n\ntrain_orders[\"total_cell_next_to_cell\"] = train_orders[\"c_next_to_c\"] + train_orders[\"m_next_to_m\"] + train_orders[\"c_next_to_m\"] + train_orders[\"m_next_to_c\"]\n\ntrain_orders['n_c_next_to_c'] = train_orders[\"c_next_to_c\"] / train_orders[\"total_cell_next_to_cell\"]\ntrain_orders[\"n_m_next_to_m\"] = train_orders[\"m_next_to_m\"] / train_orders[\"total_cell_next_to_cell\"]\ntrain_orders[\"n_c_next_to_m\"] = train_orders[\"c_next_to_m\"] / train_orders[\"total_cell_next_to_cell\"]\ntrain_orders[\"n_m_next_to_c\"] = train_orders[\"m_next_to_c\"] / train_orders[\"total_cell_next_to_cell\"]\n\ntrain_orders[\"startswith_code\"] = train_orders['metadata'].apply(lambda x: 1 if x['cell_type'].values[0]==\"code\" else 0)\ntrain_orders[\"startswith_markdown\"] = train_orders['metadata'].apply(lambda x: 1 if x['cell_type'].values[0]==\"markdown\" else 0)\n\ntrain_orders[\"endswith_code\"] = train_orders['metadata'].apply(lambda x: 1 if x['cell_type'].values[-1]==\"code\" else 0)\ntrain_orders[\"endswith_markdown\"] = train_orders['metadata'].apply(lambda x: 1 if x['cell_type'].values[-1]==\"markdown\" else 0)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-12T18:24:25.281666Z","iopub.execute_input":"2022-05-12T18:24:25.281851Z","iopub.status.idle":"2022-05-12T18:26:05.271414Z","shell.execute_reply.started":"2022-05-12T18:24:25.281826Z","shell.execute_reply":"2022-05-12T18:26:05.270633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_orders[\"total_num_cell\"] = train_orders['total_code_cell'] + train_orders['total_markdown_cell']\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-12T18:26:05.272404Z","iopub.execute_input":"2022-05-12T18:26:05.272736Z","iopub.status.idle":"2022-05-12T18:26:05.278807Z","shell.execute_reply.started":"2022-05-12T18:26:05.272708Z","shell.execute_reply":"2022-05-12T18:26:05.277831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_orders.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-12T18:26:05.280054Z","iopub.execute_input":"2022-05-12T18:26:05.280253Z","iopub.status.idle":"2022-05-12T18:26:05.33974Z","shell.execute_reply.started":"2022-05-12T18:26:05.280227Z","shell.execute_reply":"2022-05-12T18:26:05.338818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA","metadata":{}},{"cell_type":"code","source":"labels = ['code_cells', \"markdown_cells\"]\ncount = [sum(train_orders[\"total_code_cell\"].values),sum(train_orders[\"total_markdown_cell\"].values)]\nfig = go.Figure(data=[go.Pie(labels=labels, values=count, pull=[0, 0.2])])\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-12T18:26:05.34089Z","iopub.execute_input":"2022-05-12T18:26:05.341225Z","iopub.status.idle":"2022-05-12T18:26:05.548817Z","shell.execute_reply.started":"2022-05-12T18:26:05.341197Z","shell.execute_reply":"2022-05-12T18:26:05.547788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

We can see in above figure that 2/3rd of train data contain code_cells and 1/3 rd of train data contain markdown_cells. But this is not enough let's see some more details

","metadata":{}},{"cell_type":"code","source":"total_kernels = 100\nstartswith_code = sum(train_orders['startswith_code'].values) * 100 / len(train_orders)\nstartswith_markdown = sum(train_orders['startswith_markdown'].values) * 100 / len(train_orders)\nendswith_code = sum(train_orders['endswith_code'].values) * 100 / len(train_orders)\nendswith_markdown = sum(train_orders['endswith_markdown'].values) * 100 / len(train_orders)\nsc_ec = sum(train_orders[train_orders[\"startswith_code\"] == 1]['endswith_code'].values) * 100 / len(train_orders[train_orders[\"startswith_code\"] == 1])\nsc_em = sum(train_orders[train_orders[\"startswith_code\"] == 1]['endswith_markdown'].values) * 100 / len(train_orders[train_orders[\"startswith_code\"] == 1])\nsm_ec = sum(train_orders[train_orders[\"startswith_markdown\"] == 1]['endswith_code'].values) * 100 / len(train_orders[train_orders[\"startswith_markdown\"] == 1])\nsm_em = sum(train_orders[train_orders[\"startswith_markdown\"] == 1]['endswith_markdown'].values) * 100 / len(train_orders[train_orders[\"startswith_markdown\"] == 1])\n\nlabels1 = ['code','markdown']\nvalues1 = [startswith_code, startswith_markdown]\n\nlabels2 = [\"code\",\"markdown\"]\nvalues2 = [endswith_code, endswith_markdown]\n\nlabels3 = [\"startswith_code_endswith_code\",\"startswith_code_endswith_markdown\", \"startswith_markdown_endswith_markdown\",\"startswith_markdown_endswith_code\"]\nvalues3 = [sc_ec, sc_em, sm_em, sm_ec]\nfig = make_subplots(rows=1,cols=2,start_cell=\"top-left\", specs=[[{\"type\":\"pie\"},{\"type\":\"pie\"}]],\n subplot_titles=[\"startswith\",\"endswith\"])\n\nfig.add_trace(go.Pie(labels=labels1, values=values1,pull=[0.02,0.02]),row=1,col=1,)\nfig.add_trace(go.Pie(labels=labels2, values=values2,pull=[0.02,0.02]),1,2)\n#fig.add_trace(go.Pie(labels=labels3, values=values3,pull=[0.02,0.02,0.02,0.02]),2,1)\nfig.update_layout(width=800,height=600)\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-12T18:26:05.55065Z","iopub.execute_input":"2022-05-12T18:26:05.551499Z","iopub.status.idle":"2022-05-12T18:26:06.165095Z","shell.execute_reply.started":"2022-05-12T18:26:05.551451Z","shell.execute_reply":"2022-05-12T18:26:06.163922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

Left side figure tell us that 2/3 of train data notebooks starts with markdown cell and 1/3 of train data notebooks starts with code cell.In right figure we can see 60.4% notebooks ends with code cell and 39.6% notebooks ends with markdown cell

","metadata":{}},{"cell_type":"code","source":"fig = make_subplots(rows=1,cols=2,start_cell=\"top-left\", specs=[[{\"type\": \"pie\"}, None]],\n subplot_titles=[\"distribution\"])\nfig.add_trace(go.Pie(labels=labels3, values=values3,pull=[0.02,0.02,0.02,0.02]),1,1)\nfig.update_layout(width=800,height=600)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-12T18:26:06.166463Z","iopub.execute_input":"2022-05-12T18:26:06.166708Z","iopub.status.idle":"2022-05-12T18:26:06.190259Z","shell.execute_reply.started":"2022-05-12T18:26:06.166684Z","shell.execute_reply":"2022-05-12T18:26:06.189089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

In 35.7% of notebooks starts with code cell and ends with code cell. 14.3% notebooks starts with code cell and ends with markdown cell. 27.3% notebooks starts with markdown cell and ends with code and 22.7% of times notebooks started with markddown cell and end with markdown cell

","metadata":{}},{"cell_type":"code","source":"c_next_to_c = sum(train_orders[\"n_c_next_to_c\"].values) / len(train_orders)\nc_next_to_m = sum(train_orders[\"n_c_next_to_m\"].values) / len(train_orders)\nm_next_to_c = sum(train_orders[\"n_m_next_to_c\"].values) / len(train_orders)\nm_next_to_m = sum(train_orders[\"n_m_next_to_m\"].values) / len(train_orders)\nlabels = ['c_next_to_c', \"c_next_to_m\",\"m_next_to_c\",\"m_next_to_m\"]\ncount = [c_next_to_c, c_next_to_m, m_next_to_c, m_next_to_m]\nfig = go.Figure(data=[go.Pie(labels=labels, values=count, pull=[0.02, 0.02, 0.02, 0.02])])\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-12T18:26:06.192164Z","iopub.execute_input":"2022-05-12T18:26:06.192464Z","iopub.status.idle":"2022-05-12T18:26:06.287088Z","shell.execute_reply.started":"2022-05-12T18:26:06.192415Z","shell.execute_reply":"2022-05-12T18:26:06.286147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

We can see that avg

\n

42.5% of times in notebooks there is code cell next to code cell.

\n

26% of time there is code cell next to markdown cell.

\n

24.4% of time there is markdown cell next to code cell

\n

7.08% of time there is markdown cell next to markdown cell

","metadata":{}},{"cell_type":"code","source":"\nids = train_orders['id'].values\n\nfig = go.Figure(data=[\n go.Bar(name=\"num_code_cells\", x=ids[:100], y=train_orders['total_code_cell'].values.tolist()[:100]),\n go.Bar(name=\"num_markdown_cells\", x=ids[:100], y=train_orders['total_markdown_cell'].values.tolist()[:100]),\n \n])\nfig.update_layout(width=800,height=500,barmode='stack')\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-12T18:26:06.288233Z","iopub.execute_input":"2022-05-12T18:26:06.288388Z","iopub.status.idle":"2022-05-12T18:26:06.323991Z","shell.execute_reply.started":"2022-05-12T18:26:06.288368Z","shell.execute_reply":"2022-05-12T18:26:06.323054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_orders.describe()","metadata":{"execution":{"iopub.status.busy":"2022-05-12T18:26:06.325314Z","iopub.execute_input":"2022-05-12T18:26:06.326435Z","iopub.status.idle":"2022-05-12T18:26:06.458512Z","shell.execute_reply.started":"2022-05-12T18:26:06.326391Z","shell.execute_reply":"2022-05-12T18:26:06.457222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count1 = train_orders['total_num_cell'].value_counts()\ncount2 = train_orders['total_code_cell'].value_counts()\ncount3 = train_orders['total_markdown_cell'].value_counts()\ndf1 = pd.DataFrame()\ndf1[\"size\"] = count1.keys()\ndf1[\"count\"]= count1.values\n\ndf2 = pd.DataFrame()\ndf2[\"size\"] = count2.keys()\ndf2[\"count\"]= count2.values\n\ndf3 = pd.DataFrame()\ndf3[\"size\"] = count3.keys()\ndf3[\"count\"]= count3.values\n\ndf1 = df1.sort_values(\"size\")\ndf2 = df2.sort_values(\"size\")\ndf3 = df3.sort_values(\"size\")\n\nx1 = df1[\"size\"].values\ny1 = df1[\"count\"].values\n\nx2 = df2[\"size\"].values\ny2 = df3[\"count\"].values\n\nx3 = df3[\"size\"].values\ny3 = df3[\"count\"].values\n\n#plt.plot(x1,y1)","metadata":{"execution":{"iopub.status.busy":"2022-05-12T18:26:06.4598Z","iopub.execute_input":"2022-05-12T18:26:06.460047Z","iopub.status.idle":"2022-05-12T18:26:06.482618Z","shell.execute_reply.started":"2022-05-12T18:26:06.460015Z","shell.execute_reply":"2022-05-12T18:26:06.481776Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Cell Distributions","metadata":{}},{"cell_type":"code","source":"fig = make_subplots(rows=2,cols=2,start_cell=\"top-left\",\n subplot_titles=[\"total_num_cell\",\"\",\"num_code\",\"num_markdown\"])\n\nfig.add_trace(\n go.Scatter(x=x1,y=y1, mode=\"lines\",name='total_num_cell'),1,1\n)\nfig.add_trace(\n go.Scatter(x=x2,y=y2, mode=\"lines\",name='num_code_cell'),2,1\n)\nfig.add_trace(\n go.Scatter(x=x3,y=y3, mode=\"lines\",name='num_markdown_cell'),2,2\n)\n\n# fig.add_trace(\n# go.Scatter(x=x1,y=y1, mode=\"lines\",name='total_num_cell'),3,1\n# )\n# fig.add_trace(\n# go.Scatter(x=x2,y=y2, mode=\"lines\",name='num_code_cell'),3,1\n# )\n# fig.add_trace(\n# go.Scatter(x=x3,y=y3, mode=\"lines\",name='num_markdown_cell'),3,1\n# )\nfig.update_layout(width=800,height=800)\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-12T18:26:06.48463Z","iopub.execute_input":"2022-05-12T18:26:06.484854Z","iopub.status.idle":"2022-05-12T18:26:06.594423Z","shell.execute_reply.started":"2022-05-12T18:26:06.48483Z","shell.execute_reply":"2022-05-12T18:26:06.593364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

In top left we can see distribution of total_num_cells in train data there is only one notebooks which contain 1005 number of cells which is highestin number of cells per notebook and there are 392 notebooks which contain only 2 cells per notebook which is lowest in number of cells per notebook

\n

There is 90% notebooks have less than 100 cells per notebook and only 10% of notebooks have more than 100 cells in noteboooks

\n

Bottem left figure is distribution of number of code cells in train data we can see ~11k notebooks contains only 1 code cell in notebook and bottem rigth figure is distribution of number of markdown cells in train_data in this also we can see there are ~11k notebooks contain only one markdown cell

\n

Don't you think distribution of number of code cells in train data is same as number of markdown cells in train data. So let's plot all three distribution in one plot.

","metadata":{}},{"cell_type":"code","source":"fig = go.Figure()\nfig.add_trace(\n go.Scatter(x=x1,y=y1, mode=\"lines\",name='total_num_cell')\n)\nfig.add_trace(\n go.Scatter(x=x2,y=y2, mode=\"lines\",name='num_code_cell')\n)\nfig.add_trace(\n go.Scatter(x=x3,y=y3, mode=\"lines\",name='num_markdown_cell')\n)\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-12T18:26:06.596169Z","iopub.execute_input":"2022-05-12T18:26:06.59654Z","iopub.status.idle":"2022-05-12T18:26:06.612285Z","shell.execute_reply.started":"2022-05-12T18:26:06.596502Z","shell.execute_reply":"2022-05-12T18:26:06.611421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

Boom 🔥🔥🔥

\n

we can see that distribution line of `number of markdown cell` is perfectly on distribution line of `number of code cell` in train data

","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}