{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nfrom tqdm import tqdm\nimport glob\nimport plotly.express as px\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-30T09:45:24.954665Z","iopub.execute_input":"2022-05-30T09:45:24.955279Z","iopub.status.idle":"2022-05-30T09:45:26.774944Z","shell.execute_reply.started":"2022-05-30T09:45:24.955168Z","shell.execute_reply":"2022-05-30T09:45:26.773831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_order = pd.read_csv('../input/AI4Code/train_orders.csv')\ntrain_order.index = train_order.id\ntrain_order.drop(columns=['id'])","metadata":{"execution":{"iopub.status.busy":"2022-05-30T09:45:26.77888Z","iopub.execute_input":"2022-05-30T09:45:26.779333Z","iopub.status.idle":"2022-05-30T09:45:28.645175Z","shell.execute_reply.started":"2022-05-30T09:45:26.779279Z","shell.execute_reply":"2022-05-30T09:45:28.644179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"t_order = {}\nfor i,order in zip(train_order.id,train_order.cell_order):\n    t_order[i] = order","metadata":{"execution":{"iopub.status.busy":"2022-05-30T09:45:28.647089Z","iopub.execute_input":"2022-05-30T09:45:28.647422Z","iopub.status.idle":"2022-05-30T09:45:28.757389Z","shell.execute_reply.started":"2022-05-30T09:45:28.647383Z","shell.execute_reply":"2022-05-30T09:45:28.756654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Just going to randomy use 50000 notebooks for EDA.","metadata":{}},{"cell_type":"code","source":"notebooks = {}\n\nfor csv in tqdm(np.random.choice(glob.glob('../input/AI4Code/train/*'),50000,replace=False)):\n    notebooks[os.path.split(csv)[-1][:-5]] = pd.read_json(csv)","metadata":{"execution":{"iopub.status.busy":"2022-05-30T09:45:28.759316Z","iopub.execute_input":"2022-05-30T09:45:28.759613Z","iopub.status.idle":"2022-05-30T09:54:27.874625Z","shell.execute_reply.started":"2022-05-30T09:45:28.759576Z","shell.execute_reply":"2022-05-30T09:54:27.873665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for k in tqdm(notebooks):\n    notebooks[k] = notebooks[k].loc[train_order.loc[k]['cell_order'].split(\" \")]\n    notebooks[k]['cell_rank'] = range(1,1+len(notebooks[k]))","metadata":{"execution":{"iopub.status.busy":"2022-05-30T09:54:27.875759Z","iopub.execute_input":"2022-05-30T09:54:27.875987Z","iopub.status.idle":"2022-05-30T09:55:16.094531Z","shell.execute_reply.started":"2022-05-30T09:54:27.875957Z","shell.execute_reply":"2022-05-30T09:55:16.093572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(notebooks)","metadata":{"execution":{"iopub.status.busy":"2022-05-30T10:26:27.441688Z","iopub.execute_input":"2022-05-30T10:26:27.442023Z","iopub.status.idle":"2022-05-30T10:26:27.44847Z","shell.execute_reply.started":"2022-05-30T10:26:27.441988Z","shell.execute_reply":"2022-05-30T10:26:27.447571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"combined = pd.concat(notebooks.values())","metadata":{"execution":{"iopub.status.busy":"2022-05-30T09:55:16.096005Z","iopub.execute_input":"2022-05-30T09:55:16.096261Z","iopub.status.idle":"2022-05-30T09:55:26.918607Z","shell.execute_reply.started":"2022-05-30T09:55:16.096227Z","shell.execute_reply":"2022-05-30T09:55:26.917687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Languages used in notebook","metadata":{}},{"cell_type":"code","source":"#https://stackoverflow.com/a/47106810/7429803\n\n!wget https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.ftz\nimport fasttext\nmodel = fasttext.load_model('lid.176.ftz')\nprint(model.predict('الشمس تشرق'))","metadata":{"execution":{"iopub.status.busy":"2022-05-30T09:55:26.920765Z","iopub.execute_input":"2022-05-30T09:55:26.921116Z","iopub.status.idle":"2022-05-30T09:55:28.100107Z","shell.execute_reply.started":"2022-05-30T09:55:26.921065Z","shell.execute_reply":"2022-05-30T09:55:28.099338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"language = []\nfor k in tqdm(notebooks):\n    notebooks[k]['language'] = \"Python\"\n    notebooks[k].loc[notebooks[k].cell_type==\"markdown\",'language'] = notebooks[k][notebooks[k].cell_type==\"markdown\"].source.apply(func=lambda x: model.predict(x.replace(\"\\n\",\".\"))[0][0]) \n    language.append(notebooks[k][notebooks[k].cell_type==\"markdown\"].language.mode()[0])\npx.bar(pd.Series(language).value_counts()[:10],title=\"Notebook Language distribution(top 10)\")","metadata":{"execution":{"iopub.status.busy":"2022-05-30T09:55:28.101975Z","iopub.execute_input":"2022-05-30T09:55:28.102519Z","iopub.status.idle":"2022-05-30T09:59:23.880686Z","shell.execute_reply.started":"2022-05-30T09:55:28.102464Z","shell.execute_reply":"2022-05-30T09:59:23.879668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Number of languages in training subset: 57\n#### Most popular languages: english, portugese,russian, korean, japanese..\n#### Percentage of Notebooks written in English: 92.286%","metadata":{}},{"cell_type":"markdown","source":"### Distribution of length of notebooks","metadata":{}},{"cell_type":"code","source":"px.histogram([len(notebooks[k]) for k in notebooks],title=\"Distribution of Cells (Markdown and Code)\",nbins=200)","metadata":{"execution":{"iopub.status.busy":"2022-05-30T09:59:23.88243Z","iopub.execute_input":"2022-05-30T09:59:23.883141Z","iopub.status.idle":"2022-05-30T09:59:25.5592Z","shell.execute_reply.started":"2022-05-30T09:59:23.883086Z","shell.execute_reply":"2022-05-30T09:59:25.558519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"px.histogram([len(notebooks[k][notebooks[k].cell_type==\"markdown\"]) for k in notebooks],title=\"Distribution of Markdown Cells\",nbins=200)","metadata":{"execution":{"iopub.status.busy":"2022-05-30T09:59:25.561465Z","iopub.execute_input":"2022-05-30T09:59:25.562467Z","iopub.status.idle":"2022-05-30T09:59:41.494786Z","shell.execute_reply.started":"2022-05-30T09:59:25.562398Z","shell.execute_reply":"2022-05-30T09:59:41.493933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"px.histogram([len(notebooks[k][notebooks[k].cell_type==\"code\"]) for k in notebooks],title=\"Distribution of Code Cells\",nbins=200)","metadata":{"execution":{"iopub.status.busy":"2022-05-30T09:59:41.496059Z","iopub.execute_input":"2022-05-30T09:59:41.496325Z","iopub.status.idle":"2022-05-30T09:59:57.639136Z","shell.execute_reply.started":"2022-05-30T09:59:41.496293Z","shell.execute_reply":"2022-05-30T09:59:57.63822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### percentage of notebooks with no code in training subset:0\n#### Percentage of Notebooks with no markdown  in training subset: 0","metadata":{"execution":{"iopub.status.busy":"2022-05-30T09:32:36.75017Z","iopub.execute_input":"2022-05-30T09:32:36.750887Z","iopub.status.idle":"2022-05-30T09:32:36.755771Z","shell.execute_reply.started":"2022-05-30T09:32:36.750838Z","shell.execute_reply":"2022-05-30T09:32:36.754472Z"}}},{"cell_type":"markdown","source":"### Ratio of Type of Cells","metadata":{}},{"cell_type":"code","source":"px.bar(combined.cell_type.value_counts(),title=\"cell type\")","metadata":{"execution":{"iopub.status.busy":"2022-05-30T09:59:57.640882Z","iopub.execute_input":"2022-05-30T09:59:57.641256Z","iopub.status.idle":"2022-05-30T09:59:58.186401Z","shell.execute_reply.started":"2022-05-30T09:59:57.641208Z","shell.execute_reply":"2022-05-30T09:59:58.185373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Ratio of markdown:Code in the nth Cell","metadata":{}},{"cell_type":"code","source":"ctype= combined.groupby(['cell_rank','cell_type']). count().reset_index()\nctype = ctype[:300]\npx.bar(ctype,x='cell_rank',y='source',color='cell_type',title=\"Proportion of markdown and Code for each cell rank\")","metadata":{"execution":{"iopub.status.busy":"2022-05-30T09:59:58.187867Z","iopub.execute_input":"2022-05-30T09:59:58.18816Z","iopub.status.idle":"2022-05-30T09:59:59.239463Z","shell.execute_reply.started":"2022-05-30T09:59:58.188129Z","shell.execute_reply":"2022-05-30T09:59:59.238579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Code cells, Code lines with comments","metadata":{}},{"cell_type":"code","source":"combined['has_comments'] = False\ncombined['lines_with_comments'] = 0\ncombined['number_lines'] = 0\n\ncombined.loc[combined.cell_type==\"code\",'has_comments'] = combined.source[combined.cell_type==\"code\"].apply(lambda x: \"#\" in x)\ncombined.loc[combined.cell_type==\"code\",'lines_with_comments'] = combined.source[combined.cell_type==\"code\"].apply(lambda x: len([x for x in x.split('\\n') if \"#\" in x]))\ncombined.loc[combined.cell_type==\"code\",'number_lines'] = combined.source[combined.cell_type==\"code\"].apply(lambda x: len(x.split('\\n')))\n","metadata":{"execution":{"iopub.status.busy":"2022-05-30T09:59:59.241022Z","iopub.execute_input":"2022-05-30T09:59:59.241259Z","iopub.status.idle":"2022-05-30T10:00:09.260902Z","shell.execute_reply.started":"2022-05-30T09:59:59.24123Z","shell.execute_reply":"2022-05-30T10:00:09.259946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tmp = combined[combined.cell_type==\"code\"]\ntmp = tmp.number_lines[tmp.number_lines<50]\nprint(f\"Maximum lines: {combined.number_lines.max()}\")\npx.histogram(tmp,title=\"Number of lines in code cells\",nbins=100)","metadata":{"execution":{"iopub.status.busy":"2022-05-30T10:00:09.262348Z","iopub.execute_input":"2022-05-30T10:00:09.262648Z","iopub.status.idle":"2022-05-30T10:00:18.256933Z","shell.execute_reply.started":"2022-05-30T10:00:09.262611Z","shell.execute_reply":"2022-05-30T10:00:18.255413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tmp = combined[combined.cell_type==\"code\"]\ntmp = tmp[tmp.lines_with_comments>0]\npx.histogram(tmp.lines_with_comments/(tmp.number_lines+1e-10),title=\"distribution of ratio of lines with comments in code cells(0 has been omitted)\",nbins=100)","metadata":{"execution":{"iopub.status.busy":"2022-05-30T10:00:18.258672Z","iopub.execute_input":"2022-05-30T10:00:18.258928Z","iopub.status.idle":"2022-05-30T10:00:20.922878Z","shell.execute_reply.started":"2022-05-30T10:00:18.258894Z","shell.execute_reply":"2022-05-30T10:00:20.921579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"px.bar(combined.has_comments[combined.cell_type==\"code\"].value_counts(),title=\"Code cells without any comments(false)/ at least 1 comment(true)\")","metadata":{"execution":{"iopub.status.busy":"2022-05-30T10:00:20.925032Z","iopub.execute_input":"2022-05-30T10:00:20.925418Z","iopub.status.idle":"2022-05-30T10:00:21.478846Z","shell.execute_reply.started":"2022-05-30T10:00:20.925364Z","shell.execute_reply":"2022-05-30T10:00:21.477977Z"},"trusted":true},"execution_count":null,"outputs":[]}]}