{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\nimport seaborn as sns\nfrom tqdm import tqdm\nfrom pathlib import Path\nimport re\nimport os\n\npd.options.display.width = 180\npd.options.display.max_colwidth = 100\n\ndata_dir = Path(\"../input/AI4Code\")\n\nrc = {\"axes.spines.left\" : True,\n      \"axes.spines.right\" : False,\n      \"axes.spines.bottom\" : True,\n      \"axes.spines.top\" : False,\n      \"xtick.bottom\" : True,\n      \"xtick.labelbottom\" : True,\n      \"ytick.labelleft\" : True,\n      \"ytick.left\" : True,\n      \"figure.subplot.hspace\" : 0.7,\n    \"figure.titleweight\" : \"bold\",\n    \"axes.titleweight\" : \"bold\",\n     \"font.weight\" : \"bold\"}\nplt.rcParams.update(rc)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-06-03T15:42:26.337387Z","iopub.execute_input":"2022-06-03T15:42:26.337997Z","iopub.status.idle":"2022-06-03T15:42:27.460617Z","shell.execute_reply.started":"2022-06-03T15:42:26.337896Z","shell.execute_reply":"2022-06-03T15:42:27.459750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_notebook(path):\n    return (\n        pd.read_json(\n            path,\n            dtype={'cell_type': 'category', 'source': 'str'})\n        .assign(id=path.stem)\n        .rename_axis('cell_id')\n    )\n\n# Subset of training due to its large size\nNUM_TRAIN = 20000\npaths_train = list((data_dir / 'train').glob('*.json'))[:NUM_TRAIN]\n#paths_train = list((data_dir / 'train').glob('*.json'))\n\nnotebooks_train = [read_notebook(path) for path in tqdm(paths_train, desc='Train NBs')]\n# Get notebooks\ndf_notebooks = (pd.concat(notebooks_train).set_index('id', append=True).swaplevel().sort_index(level='id', \n                                                                                               sort_remaining=False)) \n\n# Get correct order of cells in notebooks                                                                                          \ndf_orders = pd.read_csv(data_dir/'train_orders.csv', index_col='id')\ndf_orders = df_orders.squeeze().str.split(\" \").explode().to_frame()\ndf_orders[\"rank\"] = pd.Series([np.arange(x) for x in df_orders.groupby(\"id\").count()[\"cell_order\"]]).explode().to_numpy()\n\ndf = df_notebooks.reset_index().merge(df_orders.reset_index().rename(columns = {'cell_order':'cell_id'}), \n                       how='inner', on=['id','cell_id'])\n\n# Get ancestors for notebooks\ndf_ancestors = pd.read_csv(data_dir / 'train_ancestors.csv', index_col='id')\n\n# Final combined dataframe\ndf = df.merge(df_ancestors, on=\"id\").sort_values([\"id\", \"rank\"]).set_index([\"id\", \"cell_id\"])\n\n# Dataframe for count information - Used in EDA\nmkd = df[df[\"cell_type\"] == \"markdown\"].groupby(by=[\"id\"]).count().source\ncode = df[df[\"cell_type\"] == \"code\"].groupby(by=[\"id\"]).count().source\ndf_counts = pd.concat([mkd, code], axis=1)\ndf_counts.columns = ['markdown_count', 'code_count']\ndf_counts[\"tot\"] = df_counts.markdown_count+df_counts.code_count\ndf_counts[\"ratio\"] = df_counts.code_count / df_counts.tot","metadata":{"execution":{"iopub.status.busy":"2022-06-03T15:42:27.461992Z","iopub.execute_input":"2022-06-03T15:42:27.462268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[\"cell_type\"].unique()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.isna().sum()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Most of cells in these notebooks contain code","metadata":{}},{"cell_type":"code","source":"fig1, ax = plt.subplots(figsize=(8,8))\nwedges, texts, autotexts = ax.pie([df_counts.code_count.sum(), df_counts.markdown_count.sum()], shadow=True, \n                                  explode=(0, 0.1), labels=[\"Code\", \"Markdown\"], \n                                  autopct='%1.1f%%', textprops=dict(color=\"w\"), colors=[\"#008b8b\", \"#8b0000\"])\n\nplt.setp(autotexts, size=15, weight=\"bold\")\nax.legend(wedges, [\"Code\", \"Markdown\"],\n          loc=\"center left\",\n          bbox_to_anchor=(1, 0, 0.5, 1), prop={'size': 13})\n\nax.set_title(\"Proportion of Code vs Markdown\", size=20);","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Length of notebooks (number of cells)\nMost of the notebooks have less than 160 cells. There are some notebooks having more than 200 cells","metadata":{}},{"cell_type":"code","source":"fig, axs = plt.subplots(2, figsize=(10,10), gridspec_kw={\"height_ratios\": [7, 1]})\nfig.subplots_adjust(hspace=0.2)\n\ncount = df.groupby(by=[\"id\"]).count()[\"source\"].values\naxs[0].hist(count, bins=200, color=\"#8b0000\")\n\naxs[0].set_xlabel(\"# of Cells\", weight=\"bold\")\naxs[0].set_ylabel(\"Frequency\", weight=\"bold\")\naxs[0].set_title(f\"Length of Notebooks\", size=20);\n\n# Hide axis spine, ticks\nfor txt in [\"top\",\"right\",\"left\",\"bottom\"]:\n        axs[1].spines[txt].set_visible(False)\naxs[1].xaxis.set_visible(False)\naxs[1].yaxis.set_visible(False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Shortest and longest notebooks\nEvery notebooks have more than or equal 2 cells. The longest notebook has 800 cells.","metadata":{}},{"cell_type":"code","source":"colors = sns.color_palette(\"rocket_r\", n_colors=20)\n\nfig, axs = plt.subplots(1,2, figsize=(15, 10))\n\ncount = df_counts.sort_values(\"tot\").head(10).tot\naxs[0].bar(range(10), count.values, color=colors[:10])\naxs[0].set_ylim(0, 10)\naxs[0].set_title(\"Shortest Notebooks\", size=20)\n\ncount = df_counts.sort_values(\"tot\").tail(10).tot\naxs[1].bar(range(10), count.values, color=colors[10:])\naxs[1].set_title(\"Longest Notebooks\", size=20)\n\nfor i in range(2):\n    axs[i].set_xlabel(\"Notebook\", weight=\"bold\", size=13)\n    axs[i].set_xticks([])\n    \naxs[0].set_ylabel(\"# of Cells\", weight=\"bold\", size=16)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## The first cell of notebook is often a markdown\n","metadata":{}},{"cell_type":"code","source":"# Matrix for 2-cell notebooks\nmatrix = pd.DataFrame(data = np.zeros((2,2)), columns=[\"code\", \"markdown\"], index=[\"code\", \"markdown\"])\nfor notebook in df_counts[df_counts[\"tot\"]==2].index:\n    cell1 = df.loc[notebook].cell_type[0]\n    cell2 = df.loc[notebook].cell_type[1]\n    matrix.loc[cell1, cell2] = 1 + matrix.loc[cell1, cell2]\n\n# setup the figure and axes\nfig = plt.figure(figsize=(15, 15))\n# fake data\n_x = np.arange(2)\n_y = np.arange(2)\n_xx, _yy = np.meshgrid(_x, _y)\nx, y = _xx.ravel(), _yy.ravel()\n\ntop = x + y\nbottom = np.zeros_like(top)\nwidth = depth = 1\n\n# Pie chart\nax2 = fig.add_subplot(222)\nwedges, texts, autotexts = ax2.pie(df[df[\"rank\"]==0][\"cell_type\"].value_counts(), shadow=True, \n                                  explode=(0, 0.05), \n                                  autopct='%1.1f%%', textprops=dict(color=\"w\"), colors=[\"#8b0000\", \"#008b8b\"])\nplt.setp(autotexts, size=18, weight=\"bold\")\n\nax2.legend(wedges, [\"Markdown\", \"Code\"],\n          loc=\"center left\",\n          bbox_to_anchor=(1, 0, 0.5, 1), prop={'size': 13})\n\nax2.set_title(\"What is the most common \\n first cell in a notebook?\", size=18);","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Probability a cell is code or markdown along with order of cells in notebook\n\nThe first cell is often a markdown. From 1st to 20th cell it is more likely to be a code.","metadata":{}},{"cell_type":"code","source":"most_freq_length = df_counts.tot.value_counts().idxmax()\ncount_cells = pd.DataFrame(np.zeros((2, most_freq_length)), index=[\"code\", \"markdown\"])\n\n# Grab all notebooks of average length and look at rank for types\nfor notebook in df_counts[df_counts.tot == most_freq_length].index:\n    \n    dt = df.loc[notebook, [\"cell_type\", \"rank\"]]\n    count_cells.loc[\"code\"] = count_cells.loc[\"code\"].to_numpy() + [dt.cell_type == \"code\"]\n    count_cells.loc[\"markdown\"] = count_cells.loc[\"markdown\"].to_numpy() + [dt.cell_type == \"markdown\"]\n\ncell_ratios = count_cells / np.sum(count_cells, axis=0)\n\n\n# -- Plot\nfig, ax = plt.subplots(1, figsize=(10, 15))\n\nax.barh(range(most_freq_length), np.flip(cell_ratios.loc[\"code\"]), height=0.9, color=\"#008b8b\", label=\"Code\")\nax.barh(range(most_freq_length), np.flip(cell_ratios.loc[\"markdown\"]), left=np.flip(cell_ratios.iloc[0]), \n         height=0.9, color=\"#8b0000\", label=\"Markdown\")\n\nax.set_yticks(range(21))\nax.set_yticklabels(np.flip([f\"Cell {x}\" for x in range(21)]));\n\nax.set_title(\"Distributions of cell types for the most \\n common notebook length\", size=20)\nax.legend(loc=\"upper center\", frameon=False, prop={'size': 13}, ncol=2)\n\nax.plot(np.ones(most_freq_length+1)*0.5, np.arange(-0.5, most_freq_length+0.5, 1), linestyle=\"--\", c=\"r\", alpha=0.4)\n\nfor i in range(most_freq_length):\n    \n    dt_code = cell_ratios.loc[\"code\"][most_freq_length-1-i]\n    dt_mkd = cell_ratios.loc[\"markdown\"][most_freq_length-1-i]\n    \n    if dt_code>dt_mkd:\n        ax.annotate(f\"{np.round(dt_code*100)}%\", (dt_code/2, i-0.12), c=\"w\", size=14)\n    else:\n        ax.annotate(f\"{np.round(dt_mkd*100)}%\", (dt_code + dt_mkd/2, i-0.12), c=\"w\", size=14)\n# Hide axis spine, ticks\nfor txt in [\"top\",\"right\",\"left\",\"bottom\"]:\n        ax.spines[txt].set_visible(False)\nax.xaxis.set_visible(False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Percentage of code cells in notebook\nThe histogram show that a notebook often has more than 40% of cell is code.","metadata":{}},{"cell_type":"code","source":"ratio = df_counts.sort_values(\"ratio\", ascending=False).ratio\n\nfig, axs = plt.subplots(2, figsize=(10,10), gridspec_kw={\"height_ratios\": [7, 1]})\nfig.subplots_adjust(hspace=0.2)\n\n# Histogram\naxs[0].hist(ratio, bins=200, color=\"#008b8b\");\naxs[0].set_title(\"The Percentage of Code Cells in Notebook\", size=20)\naxs[0].set_ylabel(\"Frequency\", weight=\"bold\", size=16)\n# Hide axis spine, ticks\nfor txt in [\"top\",\"right\",\"left\",\"bottom\"]:\n        axs[1].spines[txt].set_visible(False)\naxs[1].xaxis.set_visible(False)\naxs[1].yaxis.set_visible(False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Number of code cells and markdown cells in a notebook\nA notebook often has about less than 100 cells of code and less than 100 cells of markdown","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(1, figsize=(10,10))\nax.scatter(df_counts.markdown_count, df_counts.code_count, s=df_counts.tot, c=\"#8b0000\", zorder=3, alpha=0.5)\n\nax.set_xlabel(\"Markdown cells\", size=16, weight=\"bold\")\nax.set_ylabel(\"Code cells\", size=16, weight=\"bold\")\nax.grid(zorder=0)\nax.set_title(\"Code v.s Markdown\", size=24);","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Percentage of code cells\n","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(1, figsize=(10,10))\ntot = df_counts.sort_values(\"ratio\", ascending=False).tot\nax.scatter(ratio, tot, s=tot, alpha=0.1, color=\"#008b8b\")\nax.set_title(\"Percentage of Code Cells \\n  against Length of Notebook\", size=18);","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Length of cells\nThe length of cells are often less than 10000 characters. There are some having length more than 10000. Markdown often has long cell more than code\n","metadata":{}},{"cell_type":"code","source":"# Add length to df\ndf[\"length\"] = [len(x) for x in df.source]\n\n# Add length to count df\ndf_counts[\"avg_len\"] = df.groupby(\"id\").mean(\"length\")\ndf_counts[\"avg_len_code\"] = df[df.cell_type == \"code\"].groupby(\"id\").mean(\"length\")\ndf_counts[\"avg_len_mkd\"] = df[df.cell_type == \"markdown\"].groupby(\"id\").mean(\"length\")\n# For plot\ncode_length =  df[df[\"cell_type\"]==\"code\"].sort_values(\"length\", ascending=False).length.values\nmkd_length =  df[df[\"cell_type\"]==\"markdown\"].sort_values(\"length\", ascending=False).length.values\n\nfig, axs = plt.subplots(1, figsize=(14, 4))\n\naxs.boxplot(code_length, positions=[2], vert=False, patch_artist=True, boxprops=dict(facecolor=\"red\", color=\"red\"),\n             flierprops=dict(markeredgecolor=\"#008b8b\"))\naxs.set_title(\"Comparing length of Cells\", size = 20)\n\n\nbox = axs.boxplot(mkd_length, positions=[1], vert=False, patch_artist=True, boxprops=dict(facecolor=\"red\", color=\"red\"),\n             flierprops=dict(markeredgecolor=\"#8b0000\"));\n\naxs.set_yticks([1, 2], [\"Markdown\", \"Code\"], size=15);","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Compare length of code and markdown\nCode cells often have length longer than markdown cells if the length smaller than 1000.","metadata":{}},{"cell_type":"code","source":"# Same plot as above but limited x-range to (0, 1000)\n\ncode_length =  df[df[\"cell_type\"]==\"code\"].sort_values(\"length\", ascending=False).length.values\nmkd_length =  df[df[\"cell_type\"]==\"markdown\"].sort_values(\"length\", ascending=False).length.values\n\nfig, axs = plt.subplots(1, figsize=(14, 4))\n\nbplot1 = axs.boxplot(code_length, positions=[2], vert=False, patch_artist=True, widths= 0.4, \n                     boxprops=dict(facecolor=\"#008b8b\", color=\"black\", linewidth=3),\n                     capprops=dict(linestyle='-', linewidth=3),\n                     whiskerprops=dict(linestyle='-', linewidth=3),\n                     medianprops=dict(linestyle='-', linewidth=3), showfliers=False)\n\nbplot2 = axs.boxplot(mkd_length, positions=[1], vert=False, patch_artist=True, widths= 0.4, \n                     boxprops=dict(facecolor=\"#8b0000\", color=\"black\", linewidth=3),\n                     capprops=dict(linestyle='-', linewidth=3),\n                     whiskerprops=dict(linestyle='-', linewidth=3),\n                     medianprops=dict(linestyle='-', linewidth=3), showfliers=False)\naxs.set_xlim(0, 1000)\naxs.set_yticks([1, 2], [\"Markdown\", \"Code\"], size=15)\naxs.set_title(\"Comparing length of Cells\", size = 20);","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Average length of code cells and markdown cells along with notebooks length","metadata":{}},{"cell_type":"code","source":"fig = plt.figure(figsize=(13,15))\nfig.subplots_adjust(hspace=0.2)\nspec = fig.add_gridspec(2, 2)\n\nax0 = fig.add_subplot(spec[0, :])\nax0.scatter(df_counts.tot, df_counts.avg_len_mkd, c=\"#8b0000\", zorder=3, alpha=0.5, label=\"Markdown\")\nax0.scatter(df_counts.tot, df_counts.avg_len_code, c=\"#008b8b\", zorder=3, alpha=0.5, label=\"Code\")\nax0.set_xlabel(\"Length of Notebook\", size=16, weight=\"bold\")\nax0.set_ylabel(\"Avg. length of Cells\", size=16, weight=\"bold\")\nax0.legend()\n\nax1 = fig.add_subplot(spec[1, 0])\nax1.scatter(df_counts.markdown_count, df_counts.avg_len_mkd, c=\"#8b0000\", zorder=3, alpha=0.5)\nax1.set_xlabel(\"# of Markdown Cells\", size=13, weight=\"bold\")\n\nax2 = fig.add_subplot(spec[1, 1])\nax2.scatter(df_counts.code_count, df_counts.avg_len_code, c=\"#008b8b\", zorder=3, alpha=0.5)\nax2.set_xlabel(\"# of Code Cells\", size=13, weight=\"bold\");","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install langdetect\nfrom langdetect import detect_langs\n\nmarkdowns_df = df[df[\"cell_type\"]==\"markdown\"].reset_index()\n\ndf_lang = pd.DataFrame(columns=[\"Count\"])\nfailed_identifications = 0\n\n# Check notebooks\nfor i, notebook in enumerate(tqdm(markdowns_df.id.unique())):\n    \n    # Add probs to df\n    prob_df = pd.DataFrame(columns=[\"Prob\"])\n    \n    # Look at text within notebook\n    for txt in markdowns_df[markdowns_df.id == notebook].source:\n        \n        # Normalize a bit\n        txt = re.sub(r'[^\\w]', ' ', txt).strip()\n        \n        # Skip too long or too short txt\n        if len(txt) > 5000 or len(txt.split(\" \")) < 10:\n            continue\n            \n        try:\n            # Detect prob\n            lang = detect_langs(txt)\n            for l in lang:\n                if l.lang in prob_df.index:\n                    prob_df.loc[l.lang] = l.prob + prob_df.loc[l.lang]\n                else:\n                    prob_df.loc[l.lang] = l.prob\n        except:\n            failed_identifications += 1\n    \n    # Add highest prob. lang in notebook to counter\n    if len(prob_df) > 0:\n        lang = prob_df.sort_values(\"Prob\", ascending=False).index[0]\n\n        if lang in df_lang.index:\n             df_lang.loc[lang, \"Count\"] = 1 + df_lang.loc[lang, \"Count\"]\n        else:\n             df_lang.loc[lang, \"Count\"] = 1\n    \ndf_lang = df_lang.sort_values(\"Count\", ascending=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}