{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Needed Code snippets \nSome code snippets to be familiar with\nLoading json:- \n\n```python \ncontent=json.load(open(\"file.json\")) \n```\n\nDisplay Markdown and code in different colors :- \n\n```python \nprint(f\"\\033[92m Markdown {content}\") \nprint(f\"\\033[94m code {content}\") \n``` \n\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"# Import","metadata":{}},{"cell_type":"code","source":"!pip install pycld2","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:36:47.451675Z","iopub.execute_input":"2022-06-03T13:36:47.452129Z","iopub.status.idle":"2022-06-03T13:37:28.786547Z","shell.execute_reply.started":"2022-06-03T13:36:47.452033Z","shell.execute_reply":"2022-06-03T13:37:28.785234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd \nimport json \nimport numpy as np\nimport plotly.express as px\nimport matplotlib.pyplot as plt \nimport glob \nfrom tqdm import tqdm\nimport os\nimport nltk\nfrom nltk.corpus import stopwords\nimport pycld2 as cld2","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:37:28.788959Z","iopub.execute_input":"2022-06-03T13:37:28.789374Z","iopub.status.idle":"2022-06-03T13:37:31.030098Z","shell.execute_reply.started":"2022-06-03T13:37:28.789323Z","shell.execute_reply":"2022-06-03T13:37:31.029084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Configurations","metadata":{}},{"cell_type":"code","source":"CFG={\n    \"no_of_samples\":20000,\n    \"train_orders\":\"../input/AI4Code/train_orders.csv\",\n    \"train\":\"../input/AI4Code/train\"\n}","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:37:31.031371Z","iopub.execute_input":"2022-06-03T13:37:31.031694Z","iopub.status.idle":"2022-06-03T13:37:31.035185Z","shell.execute_reply.started":"2022-06-03T13:37:31.031666Z","shell.execute_reply":"2022-06-03T13:37:31.034610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loading Data \n\nJson file format :- \n```json\n\"root\":{\n  \"cell_type\":{\n     \"cell_id\":\"markdown/code\",\n      ...\n  },\n  \"source\":{\n     \"cell_id\":\"content\" ,\n      ...\n  }\n}\n```","metadata":{}},{"cell_type":"code","source":"train_orders=pd.read_csv(CFG[\"train_orders\"])\ntrain_orders.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:37:31.036522Z","iopub.execute_input":"2022-06-03T13:37:31.037233Z","iopub.status.idle":"2022-06-03T13:37:32.823586Z","shell.execute_reply.started":"2022-06-03T13:37:31.037201Z","shell.execute_reply":"2022-06-03T13:37:32.822552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_orders[\"full_path\"]=train_orders[\"id\"].apply(lambda x:os.path.join(CFG[\"train\"],x+\".json\"))\ntrain_orders.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:37:32.824706Z","iopub.execute_input":"2022-06-03T13:37:32.825008Z","iopub.status.idle":"2022-06-03T13:37:33.034430Z","shell.execute_reply.started":"2022-06-03T13:37:32.824982Z","shell.execute_reply":"2022-06-03T13:37:33.033403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nnotebooks=[] \nfor notebook_pth in tqdm(train_orders[\"full_path\"][:CFG[\"no_of_samples\"]]): \n    notebooks.append(json.load(open(notebook_pth)))","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:37:33.035923Z","iopub.execute_input":"2022-06-03T13:37:33.036259Z","iopub.status.idle":"2022-06-03T13:39:09.843456Z","shell.execute_reply.started":"2022-06-03T13:37:33.036232Z","shell.execute_reply":"2022-06-03T13:39:09.842704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Understanding the data at a basic level \n\n\n## Structure","metadata":{"execution":{"iopub.status.busy":"2022-05-29T14:34:39.281675Z","iopub.execute_input":"2022-05-29T14:34:39.282051Z","iopub.status.idle":"2022-05-29T14:34:39.290986Z","shell.execute_reply.started":"2022-05-29T14:34:39.282018Z","shell.execute_reply":"2022-05-29T14:34:39.290054Z"}}},{"cell_type":"code","source":"notebooks[0].keys()","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:39:09.845110Z","iopub.execute_input":"2022-06-03T13:39:09.845563Z","iopub.status.idle":"2022-06-03T13:39:09.852612Z","shell.execute_reply.started":"2022-06-03T13:39:09.845519Z","shell.execute_reply":"2022-06-03T13:39:09.851559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"notebooks[0][\"cell_type\"]","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:39:09.853953Z","iopub.execute_input":"2022-06-03T13:39:09.854410Z","iopub.status.idle":"2022-06-03T13:39:09.868545Z","shell.execute_reply.started":"2022-06-03T13:39:09.854369Z","shell.execute_reply":"2022-06-03T13:39:09.867531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"notebooks[0][\"source\"]","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:39:09.869931Z","iopub.execute_input":"2022-06-03T13:39:09.870460Z","iopub.status.idle":"2022-06-03T13:39:09.884407Z","shell.execute_reply.started":"2022-06-03T13:39:09.870427Z","shell.execute_reply":"2022-06-03T13:39:09.883532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_info(notebook):\n    cell_contents=list(notebook[\"source\"].values())\n  \n    cell_types=list(notebook[\"cell_type\"].values())\n    return cell_contents,cell_types \ncontents,types=get_info(notebooks[0])","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:39:09.887864Z","iopub.execute_input":"2022-06-03T13:39:09.888347Z","iopub.status.idle":"2022-06-03T13:39:09.896420Z","shell.execute_reply.started":"2022-06-03T13:39:09.888289Z","shell.execute_reply":"2022-06-03T13:39:09.895248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# reading notebooks\nfor i in range(1,5):\n    print(\"--------------- {} notebook\".format(i))\n    contents,types=get_info(notebooks[i])\n    for type_cell,content in zip(types,contents):\n        if type_cell==\"markdown\":\n            print(f\"\\033[92m {content}\")\n        else:\n            print(f\"\\033[94m {content}\")  ","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:39:09.897574Z","iopub.execute_input":"2022-06-03T13:39:09.898011Z","iopub.status.idle":"2022-06-03T13:39:09.930585Z","shell.execute_reply.started":"2022-06-03T13:39:09.897980Z","shell.execute_reply":"2022-06-03T13:39:09.929439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"notebook_infos=list(map(get_info,notebooks))","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:39:09.931862Z","iopub.execute_input":"2022-06-03T13:39:09.932267Z","iopub.status.idle":"2022-06-03T13:39:10.176148Z","shell.execute_reply.started":"2022-06-03T13:39:09.932230Z","shell.execute_reply":"2022-06-03T13:39:10.175212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Cell position analysis before ordering","metadata":{}},{"cell_type":"code","source":"def get_position_of_stuff(classes):#tells position of markdown and code\n    code=[] \n    markdown=[]\n    i=0\n    for clas in classes:\n        if clas==\"markdown\":\n            markdown.append(i/len(classes))\n        if clas==\"code\":\n            code.append(i/len(classes))\n        i+=1 \n    return sum(code)/len(code),sum(markdown)/len(markdown)","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:39:10.177323Z","iopub.execute_input":"2022-06-03T13:39:10.177628Z","iopub.status.idle":"2022-06-03T13:39:10.183657Z","shell.execute_reply.started":"2022-06-03T13:39:10.177603Z","shell.execute_reply":"2022-06-03T13:39:10.182725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_position_of_stuff(notebook_infos[0][1])","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:39:10.184888Z","iopub.execute_input":"2022-06-03T13:39:10.185184Z","iopub.status.idle":"2022-06-03T13:39:10.196993Z","shell.execute_reply.started":"2022-06-03T13:39:10.185150Z","shell.execute_reply":"2022-06-03T13:39:10.196113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_position_of_stuff(notebook_infos[1][1])","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:39:10.198301Z","iopub.execute_input":"2022-06-03T13:39:10.199383Z","iopub.status.idle":"2022-06-03T13:39:10.209450Z","shell.execute_reply.started":"2022-06-03T13:39:10.199323Z","shell.execute_reply":"2022-06-03T13:39:10.208408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"code_positions=[] \nmarkdown_positions=[]\nfor ninfo in notebook_infos:\n    pos=get_position_of_stuff(ninfo[1]) \n    code_positions.append(pos[0])\n    markdown_positions.append(pos[1])","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:39:10.210889Z","iopub.execute_input":"2022-06-03T13:39:10.211380Z","iopub.status.idle":"2022-06-03T13:39:10.448606Z","shell.execute_reply.started":"2022-06-03T13:39:10.211322Z","shell.execute_reply":"2022-06-03T13:39:10.447459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Code position\")\n\nprint(\"mean\"+str(np.mean(code_positions)))","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:39:10.449599Z","iopub.execute_input":"2022-06-03T13:39:10.449930Z","iopub.status.idle":"2022-06-03T13:39:10.461716Z","shell.execute_reply.started":"2022-06-03T13:39:10.449900Z","shell.execute_reply":"2022-06-03T13:39:10.460600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"markdown position\")\n\nprint(\"mean\"+str(np.mean(markdown_positions)))","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:39:10.462965Z","iopub.execute_input":"2022-06-03T13:39:10.463283Z","iopub.status.idle":"2022-06-03T13:39:10.477934Z","shell.execute_reply.started":"2022-06-03T13:39:10.463254Z","shell.execute_reply":"2022-06-03T13:39:10.476939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"px.histogram(code_positions,marginal=\"box\")","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:39:10.479151Z","iopub.execute_input":"2022-06-03T13:39:10.480164Z","iopub.status.idle":"2022-06-03T13:39:11.915665Z","shell.execute_reply.started":"2022-06-03T13:39:10.480130Z","shell.execute_reply":"2022-06-03T13:39:11.914770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"px.histogram(markdown_positions,marginal=\"box\") ","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:39:11.916724Z","iopub.execute_input":"2022-06-03T13:39:11.917588Z","iopub.status.idle":"2022-06-03T13:39:12.057831Z","shell.execute_reply.started":"2022-06-03T13:39:11.917550Z","shell.execute_reply":"2022-06-03T13:39:12.056828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# We see that the markdown is placed at the last generally while the code before that before ordering lets see what happens after ordering some outliers are however there","metadata":{}},{"cell_type":"code","source":"def get_info_with_keys(notebook):\n    cell_contents=notebook[\"source\"]\n  \n    cell_types=notebook[\"cell_type\"]\n    return cell_contents,cell_types \ncontents,types=get_info_with_keys(notebooks[0])","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:39:12.059197Z","iopub.execute_input":"2022-06-03T13:39:12.059884Z","iopub.status.idle":"2022-06-03T13:39:12.064827Z","shell.execute_reply.started":"2022-06-03T13:39:12.059848Z","shell.execute_reply":"2022-06-03T13:39:12.063908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"key_with_notebook_infos=list(map(get_info_with_keys,notebooks)) \n","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:39:12.066009Z","iopub.execute_input":"2022-06-03T13:39:12.066347Z","iopub.status.idle":"2022-06-03T13:39:12.099998Z","shell.execute_reply.started":"2022-06-03T13:39:12.066301Z","shell.execute_reply":"2022-06-03T13:39:12.099041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"order_of_search=list(map(lambda x:x.split(),train_orders[\"cell_order\"])) \norder_of_search[0]","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:39:12.101407Z","iopub.execute_input":"2022-06-03T13:39:12.102107Z","iopub.status.idle":"2022-06-03T13:39:13.592087Z","shell.execute_reply.started":"2022-06-03T13:39:12.102058Z","shell.execute_reply":"2022-06-03T13:39:13.590770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_position_of_stuff(classes,according_to):\n    code=[] \n    markdown=[]\n    i=0\n    for clas in according_to:\n        if classes[clas]==\"markdown\":\n            markdown.append(i/len(according_to))\n        if classes[clas]==\"code\":\n            code.append(i/len(according_to))\n        i+=1 \n    return sum(code)/len(code),sum(markdown)/len(markdown)\nget_position_of_stuff(types,order_of_search[0])","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:39:13.593202Z","iopub.execute_input":"2022-06-03T13:39:13.593581Z","iopub.status.idle":"2022-06-03T13:39:13.604790Z","shell.execute_reply.started":"2022-06-03T13:39:13.593550Z","shell.execute_reply":"2022-06-03T13:39:13.603469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"code_positions=[] \nmarkdown_positions=[]\nfor ninfo,order in zip(key_with_notebook_infos,order_of_search):\n    pos=get_position_of_stuff(ninfo[1],order)\n    code_positions.append(pos[0])\n    markdown_positions.append(pos[1])","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:39:13.607708Z","iopub.execute_input":"2022-06-03T13:39:13.608094Z","iopub.status.idle":"2022-06-03T13:39:13.961784Z","shell.execute_reply.started":"2022-06-03T13:39:13.608062Z","shell.execute_reply":"2022-06-03T13:39:13.960787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Code position\")\n\nprint(\"mean\"+str(np.mean(code_positions)))","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:39:13.963256Z","iopub.execute_input":"2022-06-03T13:39:13.963836Z","iopub.status.idle":"2022-06-03T13:39:13.972090Z","shell.execute_reply.started":"2022-06-03T13:39:13.963792Z","shell.execute_reply":"2022-06-03T13:39:13.971116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"markdown position\")\n\nprint(\"mean\"+str(np.mean(markdown_positions)))","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:39:13.973543Z","iopub.execute_input":"2022-06-03T13:39:13.974123Z","iopub.status.idle":"2022-06-03T13:39:13.987537Z","shell.execute_reply.started":"2022-06-03T13:39:13.974057Z","shell.execute_reply":"2022-06-03T13:39:13.986396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"px.histogram(code_positions,marginal=\"box\") ","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:39:13.992388Z","iopub.execute_input":"2022-06-03T13:39:13.992910Z","iopub.status.idle":"2022-06-03T13:39:14.514573Z","shell.execute_reply.started":"2022-06-03T13:39:13.992867Z","shell.execute_reply":"2022-06-03T13:39:14.513588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"px.histogram(markdown_positions,marginal=\"box\") ","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:39:14.515759Z","iopub.execute_input":"2022-06-03T13:39:14.516077Z","iopub.status.idle":"2022-06-03T13:39:14.655426Z","shell.execute_reply.started":"2022-06-03T13:39:14.516050Z","shell.execute_reply":"2022-06-03T13:39:14.654746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# We notice that they are placed together that is almost in the same position following a bell curve however in case of markdown some unexpected outliers are found","metadata":{}},{"cell_type":"markdown","source":"# Lets see the percentage of markdown in the notebooks","metadata":{}},{"cell_type":"code","source":"def get_amount(classes):\n    return np.sum(np.array(classes)==\"markdown\")/len(classes)\nget_amount(notebook_infos[0][1])","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:39:14.656317Z","iopub.execute_input":"2022-06-03T13:39:14.656846Z","iopub.status.idle":"2022-06-03T13:39:14.662842Z","shell.execute_reply.started":"2022-06-03T13:39:14.656814Z","shell.execute_reply":"2022-06-03T13:39:14.662266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"markdown_amounts=[] \nfor info in notebook_infos:\n    markdown_amounts.append(get_amount(info[1]))","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:39:14.663972Z","iopub.execute_input":"2022-06-03T13:39:14.664459Z","iopub.status.idle":"2022-06-03T13:39:14.985774Z","shell.execute_reply.started":"2022-06-03T13:39:14.664427Z","shell.execute_reply":"2022-06-03T13:39:14.984686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"px.histogram(markdown_amounts,marginal=\"box\")","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:39:14.986939Z","iopub.execute_input":"2022-06-03T13:39:14.987263Z","iopub.status.idle":"2022-06-03T13:39:15.127849Z","shell.execute_reply.started":"2022-06-03T13:39:14.987234Z","shell.execute_reply":"2022-06-03T13:39:15.126907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.mean(markdown_amounts)","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:39:15.129038Z","iopub.execute_input":"2022-06-03T13:39:15.129390Z","iopub.status.idle":"2022-06-03T13:39:15.136889Z","shell.execute_reply.started":"2022-06-03T13:39:15.129356Z","shell.execute_reply":"2022-06-03T13:39:15.135929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"px.histogram(1-np.array(markdown_amounts),marginal=\"box\")","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:39:15.138144Z","iopub.execute_input":"2022-06-03T13:39:15.138501Z","iopub.status.idle":"2022-06-03T13:39:15.287022Z","shell.execute_reply.started":"2022-06-03T13:39:15.138472Z","shell.execute_reply":"2022-06-03T13:39:15.285694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.mean(1-np.array(markdown_amounts))","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:39:15.288863Z","iopub.execute_input":"2022-06-03T13:39:15.289284Z","iopub.status.idle":"2022-06-03T13:39:15.300443Z","shell.execute_reply.started":"2022-06-03T13:39:15.289251Z","shell.execute_reply":"2022-06-03T13:39:15.297404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Almost 33% of the notebooks are composed of markdowns","metadata":{}},{"cell_type":"markdown","source":"# Analyzing Text \n\n","metadata":{}},{"cell_type":"markdown","source":"# loading stopwords and seeing the stopword count in the markdown","metadata":{}},{"cell_type":"code","source":"stopwords=stopwords.words('english')","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:39:15.301872Z","iopub.execute_input":"2022-06-03T13:39:15.302816Z","iopub.status.idle":"2022-06-03T13:39:15.318300Z","shell.execute_reply.started":"2022-06-03T13:39:15.302767Z","shell.execute_reply":"2022-06-03T13:39:15.317164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def stopword_percent(text,stopwords):\n    stopword_count=0\n    for word in text.split(): \n        if word.lower() in stopwords:\n            stopword_count+=1 \n    return stopword_count/len(text.split())\nstopword_percent(\"Math is good and my favourite subject\",stopwords)","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:39:15.320113Z","iopub.execute_input":"2022-06-03T13:39:15.320689Z","iopub.status.idle":"2022-06-03T13:39:15.332217Z","shell.execute_reply.started":"2022-06-03T13:39:15.320644Z","shell.execute_reply":"2022-06-03T13:39:15.330633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Filter out the markdown cells","metadata":{}},{"cell_type":"code","source":"def filter_cells(cells,types,req=\"markdown\"):\n    output=[]\n    i=0\n    for typ in types: \n        if typ==req:\n            output.append(cells[i])\n        i+=1 \n    return output \nfilter_cells(notebook_infos[0][0],notebook_infos[0][1])","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:39:15.333617Z","iopub.execute_input":"2022-06-03T13:39:15.334302Z","iopub.status.idle":"2022-06-03T13:39:15.348395Z","shell.execute_reply.started":"2022-06-03T13:39:15.334242Z","shell.execute_reply":"2022-06-03T13:39:15.347316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"markdown_percents=[]\nfor ninfo in tqdm(notebook_infos): \n    filtered_cells=filter_cells(ninfo[0],ninfo[1]) \n    markdown_percents.append(np.mean(list(map(lambda x:stopword_percent(x,stopwords),filtered_cells))))","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:39:15.350019Z","iopub.execute_input":"2022-06-03T13:39:15.350727Z","iopub.status.idle":"2022-06-03T13:39:32.822015Z","shell.execute_reply.started":"2022-06-03T13:39:15.350623Z","shell.execute_reply":"2022-06-03T13:39:32.820940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plot the dist","metadata":{}},{"cell_type":"code","source":"px.histogram(markdown_percents,marginal=\"box\")","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:39:32.823489Z","iopub.execute_input":"2022-06-03T13:39:32.823837Z","iopub.status.idle":"2022-06-03T13:39:32.968662Z","shell.execute_reply.started":"2022-06-03T13:39:32.823808Z","shell.execute_reply":"2022-06-03T13:39:32.967588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# We see that generally 21% of the markdown is composed of stopwords","metadata":{}},{"cell_type":"code","source":"np.mean(markdown_percents)","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:39:32.970044Z","iopub.execute_input":"2022-06-03T13:39:32.970437Z","iopub.status.idle":"2022-06-03T13:39:32.978320Z","shell.execute_reply.started":"2022-06-03T13:39:32.970403Z","shell.execute_reply":"2022-06-03T13:39:32.977235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Language detection we see if the language is english or not","metadata":{}},{"cell_type":"code","source":"def detect_language(text_content):\n    _, _, _, detected_language = cld2.detect(text_content,  returnVectors=True)\n    return detected_language[0][2]==\"ENGLISH\"\ndetect_language(\"Деление признаков на числовые и текстовые\")","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:39:32.979802Z","iopub.execute_input":"2022-06-03T13:39:32.980114Z","iopub.status.idle":"2022-06-03T13:39:32.990308Z","shell.execute_reply.started":"2022-06-03T13:39:32.980088Z","shell.execute_reply":"2022-06-03T13:39:32.989632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"detect_language(\"Machine Learning is imporantant\")","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:39:32.991200Z","iopub.execute_input":"2022-06-03T13:39:32.991848Z","iopub.status.idle":"2022-06-03T13:39:33.003934Z","shell.execute_reply.started":"2022-06-03T13:39:32.991815Z","shell.execute_reply":"2022-06-03T13:39:33.002846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lang=[]\nfor ninfo in tqdm(notebook_infos): \n    filtered_cells=filter_cells(ninfo[0],ninfo[1]) \n    try:\n        lang+=list(map(lambda x:detect_language(x),filtered_cells))\n    except:\n        pass","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:39:33.005302Z","iopub.execute_input":"2022-06-03T13:39:33.005679Z","iopub.status.idle":"2022-06-03T13:39:40.089680Z","shell.execute_reply.started":"2022-06-03T13:39:33.005647Z","shell.execute_reply":"2022-06-03T13:39:40.088755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# In the histogram we see that a huge number of markdowns are not in english almost half as much as the number of markdowns in english","metadata":{}},{"cell_type":"code","source":"px.histogram(lang)","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:39:40.090891Z","iopub.execute_input":"2022-06-03T13:39:40.091308Z","iopub.status.idle":"2022-06-03T13:39:42.417666Z","shell.execute_reply.started":"2022-06-03T13:39:40.091276Z","shell.execute_reply":"2022-06-03T13:39:42.416795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# In this part I will be focusing on understanding and visualizing the changes in the position after ordering","metadata":{}},{"cell_type":"code","source":"infos=key_with_notebook_infos ","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:39:42.418882Z","iopub.execute_input":"2022-06-03T13:39:42.419818Z","iopub.status.idle":"2022-06-03T13:39:42.423891Z","shell.execute_reply.started":"2022-06-03T13:39:42.419783Z","shell.execute_reply":"2022-06-03T13:39:42.423096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"origin=[] \nfinishing=[]\nfor i in list(tqdm(range(len(infos)))):\n    subo=[] \n    subf=[]\n    for id_cell in infos[i][0].keys(): \n        subo.append((list((infos[i][0].keys())).index(id_cell)/len(infos[i][0].values()),1))\n        subf.append((order_of_search[i].index(id_cell)/len(infos[i][0].values()),0))\n    origin.append(subo) \n    finishing.append(subf)","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:39:42.424933Z","iopub.execute_input":"2022-06-03T13:39:42.425241Z","iopub.status.idle":"2022-06-03T13:39:45.947256Z","shell.execute_reply.started":"2022-06-03T13:39:42.425215Z","shell.execute_reply":"2022-06-03T13:39:45.946299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"color_mapping={\n    \"code\":\"blue\",\n    \"markdown\":\"green\"\n}","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:42:55.440723Z","iopub.execute_input":"2022-06-03T13:42:55.441114Z","iopub.status.idle":"2022-06-03T13:42:55.445937Z","shell.execute_reply.started":"2022-06-03T13:42:55.441080Z","shell.execute_reply":"2022-06-03T13:42:55.444987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idx=list(range(len(infos[0][0].keys())))\nfor i in idx:\n    color=color_mapping[list(infos[0][1].values())[i]]\n    plt.arrow(x=origin[0][i][0],y=origin[0][i][1],dx=finishing[0][i][0]-origin[0][i][0],dy=finishing[0][i][1]-origin[0][i][1],head_width = 0.07,color=color)\nplt.xlim(0,1)\nplt.ylim(0,1)","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:43:55.494004Z","iopub.execute_input":"2022-06-03T13:43:55.494374Z","iopub.status.idle":"2022-06-03T13:43:55.742774Z","shell.execute_reply.started":"2022-06-03T13:43:55.494343Z","shell.execute_reply":"2022-06-03T13:43:55.742029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idx=np.array(list(range(len(infos[1][0].keys()))))\nfor i in idx:\n    color=color_mapping[list(infos[1][1].values())[i]]\n    plt.arrow(x=origin[1][i][0],y=origin[1][i][1],dx=finishing[1][i][0]-origin[1][i][0],dy=finishing[1][i][1]-origin[1][i][1],head_width = 0.07,color=color)\nplt.xlim(0,1)\nplt.ylim(0,1)","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:44:31.226225Z","iopub.execute_input":"2022-06-03T13:44:31.226608Z","iopub.status.idle":"2022-06-03T13:44:31.520255Z","shell.execute_reply.started":"2022-06-03T13:44:31.226578Z","shell.execute_reply":"2022-06-03T13:44:31.519372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idx=np.array(list(range(len(infos[2][0].keys()))))\nfor i in idx:\n    color=color_mapping[list(infos[2][1].values())[i]]\n    plt.arrow(x=origin[2][i][0],y=origin[2][i][1],dx=finishing[2][i][0]-origin[2][i][0],dy=finishing[2][i][1]-origin[2][i][1],head_width = 0.07,color=color)\nplt.xlim(0,1)\nplt.ylim(0,1)","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:45:19.477586Z","iopub.execute_input":"2022-06-03T13:45:19.478030Z","iopub.status.idle":"2022-06-03T13:45:19.673508Z","shell.execute_reply.started":"2022-06-03T13:45:19.477997Z","shell.execute_reply":"2022-06-03T13:45:19.672296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idx=np.array(list(range(len(infos[3][0].keys()))))\nfor i in idx:\n    color=color_mapping[list(infos[3][1].values())[i]]\n    plt.arrow(x=origin[3][i][0],y=origin[3][i][1],dx=finishing[3][i][0]-origin[3][i][0],dy=finishing[3][i][1]-origin[3][i][1],head_width = 0.07,color=color)\nplt.xlim(0,1)\nplt.ylim(0,1)","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:45:54.396701Z","iopub.execute_input":"2022-06-03T13:45:54.397123Z","iopub.status.idle":"2022-06-03T13:45:54.842677Z","shell.execute_reply.started":"2022-06-03T13:45:54.397089Z","shell.execute_reply":"2022-06-03T13:45:54.841393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idx=np.array(list(range(len(infos[4][0].keys()))))\nfor i in idx:\n    color=color_mapping[list(infos[4][1].values())[i]]\n    plt.arrow(x=origin[4][i][0],y=origin[4][i][1],dx=finishing[4][i][0]-origin[4][i][0],dy=finishing[4][i][1]-origin[4][i][1],head_width = 0.07,color=color)\nplt.xlim(0,1)\nplt.ylim(0,1)","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:48:48.631947Z","iopub.execute_input":"2022-06-03T13:48:48.633810Z","iopub.status.idle":"2022-06-03T13:48:48.803414Z","shell.execute_reply.started":"2022-06-03T13:48:48.633731Z","shell.execute_reply":"2022-06-03T13:48:48.802678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idx=np.array(list(range(len(infos[5][0].keys()))))\nfor i in idx:\n    color=color_mapping[list(infos[5][1].values())[i]]\n    plt.arrow(x=origin[5][i][0],y=origin[5][i][1],dx=finishing[5][i][0]-origin[5][i][0],dy=finishing[5][i][1]-origin[5][i][1],head_width = 0.07,color=color)\nplt.xlim(0,1)\nplt.ylim(0,1)","metadata":{"execution":{"iopub.status.busy":"2022-06-03T13:57:57.492720Z","iopub.execute_input":"2022-06-03T13:57:57.493185Z","iopub.status.idle":"2022-06-03T13:57:57.667033Z","shell.execute_reply.started":"2022-06-03T13:57:57.493152Z","shell.execute_reply":"2022-06-03T13:57:57.666090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# The above graphs explain the mapping from the orginal notebook to the ordered notebook , each point which has y=1 that points x axis represents the position of that cell before ordering and its ordered position is when y=0 then we draw a line between those two positions markdown cells in green and code cells in blue.We are quite aware and have a strong hypotehsis that markdowns go to starting or together with the code cells at this point","metadata":{}},{"cell_type":"markdown","source":"# Measuring the amount of change after ordering \n\nA bit of innaccurate way of doing this as changing one cell also effects the next cell and all the cells after that but as for now i am using the following . \n\n$$dP=\\frac{1}{n}*\\sum_{i=1}^{n}| {F_i}_x-{O_i}_x |$$ \n\nWhere ${F_i}_x$ is the position after ordering and ${O_i}_x$ is the position before ordering and then we ofcourse take the absolute of the difference as we want measure the euclid distance .  A similar method was also used to create those graphs. n is the number of cells and then we take an average which indicates the amount of change","metadata":{}},{"cell_type":"code","source":"def get_distance(origins,finished): \n    return np.mean(np.abs(finished[...,0]-origins[...,0])) \nget_distance(np.array(origin[0]),np.array(finishing[0]))","metadata":{"execution":{"iopub.status.busy":"2022-06-03T14:29:21.455036Z","iopub.execute_input":"2022-06-03T14:29:21.455449Z","iopub.status.idle":"2022-06-03T14:29:21.464558Z","shell.execute_reply.started":"2022-06-03T14:29:21.455417Z","shell.execute_reply":"2022-06-03T14:29:21.463695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"distances=[]\nfor O,S in zip(origin,finishing): \n    distances.append(get_distance(np.array(O),np.array(S)))","metadata":{"execution":{"iopub.status.busy":"2022-06-03T14:31:42.458354Z","iopub.execute_input":"2022-06-03T14:31:42.458767Z","iopub.status.idle":"2022-06-03T14:31:43.810867Z","shell.execute_reply.started":"2022-06-03T14:31:42.458735Z","shell.execute_reply":"2022-06-03T14:31:43.809653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"px.histogram(distances)","metadata":{"execution":{"iopub.status.busy":"2022-06-03T14:31:59.596791Z","iopub.execute_input":"2022-06-03T14:31:59.597165Z","iopub.status.idle":"2022-06-03T14:31:59.733282Z","shell.execute_reply.started":"2022-06-03T14:31:59.597136Z","shell.execute_reply":"2022-06-03T14:31:59.732234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.mean(distances)","metadata":{"execution":{"iopub.status.busy":"2022-06-03T14:33:02.992760Z","iopub.execute_input":"2022-06-03T14:33:02.993168Z","iopub.status.idle":"2022-06-03T14:33:03.002129Z","shell.execute_reply.started":"2022-06-03T14:33:02.993134Z","shell.execute_reply":"2022-06-03T14:33:03.000673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# So my inital guess is almost 23% of code is reordered","metadata":{}},{"cell_type":"markdown","source":"# Thank you :) \n\n\nUpdates will be coming to this notebook . \nMind giving me feedbacks that will help me improve this kernel .\nUpvote if u like this notebook","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}