{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import ujson\nimport random\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nfrom tqdm.auto import tqdm\n\nrandom.seed(98)","metadata":{"execution":{"iopub.status.busy":"2022-05-20T13:25:54.305717Z","iopub.execute_input":"2022-05-20T13:25:54.306134Z","iopub.status.idle":"2022-05-20T13:25:54.915585Z","shell.execute_reply.started":"2022-05-20T13:25:54.306028Z","shell.execute_reply":"2022-05-20T13:25:54.914493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_folder = '../input/AI4Code/train/'\ntest_folder = '../input/AI4Code/test/'\n\ntrain_data = pd.read_csv('../input/AI4Code/train_orders.csv')\ntest_data = pd.read_csv('../input/AI4Code/train_ancestors.csv')","metadata":{"execution":{"iopub.status.busy":"2022-05-20T13:25:59.830724Z","iopub.execute_input":"2022-05-20T13:25:59.831064Z","iopub.status.idle":"2022-05-20T13:26:00.978127Z","shell.execute_reply.started":"2022-05-20T13:25:59.831032Z","shell.execute_reply":"2022-05-20T13:26:00.976813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-20T13:26:00.980412Z","iopub.execute_input":"2022-05-20T13:26:00.980759Z","iopub.status.idle":"2022-05-20T13:26:00.997241Z","shell.execute_reply.started":"2022-05-20T13:26:00.980710Z","shell.execute_reply":"2022-05-20T13:26:00.995817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_cell_order_and_json_data(file_id,display=True):\n    if isinstance(file_id,int):\n        file_id = train_data['id'].iloc[file_id]\n        \n    cell_order = train_data[train_data['id']==file_id]['cell_order'].iloc[0].split(' ')\n    file = open(train_folder+file_id+'.json')\n    json_data = ujson.load(file)\n    \n    source = json_data['source']\n    \n    # for displaying it in order\n    if display:\n        for cell_id in cell_order:\n            print(source[cell_id])\n    \n    return cell_order,json_data","metadata":{"execution":{"iopub.status.busy":"2022-05-20T13:26:09.832937Z","iopub.execute_input":"2022-05-20T13:26:09.833677Z","iopub.status.idle":"2022-05-20T13:26:09.842992Z","shell.execute_reply.started":"2022-05-20T13:26:09.833615Z","shell.execute_reply":"2022-05-20T13:26:09.841734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_cell_order_and_json_data(2);","metadata":{"execution":{"iopub.status.busy":"2022-05-20T13:26:10.080478Z","iopub.execute_input":"2022-05-20T13:26:10.081533Z","iopub.status.idle":"2022-05-20T13:26:10.118245Z","shell.execute_reply.started":"2022-05-20T13:26:10.081466Z","shell.execute_reply":"2022-05-20T13:26:10.117222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Question has to be asked<br>\nDo most notebook have please upvote line at the beginning or at the top ?<br>\nI am asking this question because this line has nothing to do with the content of the code cell.<br>\nI will consider any sentence that contains the word \"upvote\" as a cell asking for upvote.<br>\nIf upvote line is in top 20% of the cell it is top else if it is int bottom 20% it is in bottom else middle.<br>\n\nWhile we are at it we could also get some other stats using same loop<br>\nWhat is the gap between markdown cells ?<br>\nDistribution of len of markdown cells ?<br>\nWhat is the probability that markdown is at x position ?<br>\n\nAnd one hypothesis I want to test is that is longer markdown are the starting of the notebook.<br>\nI am considering markdown with more than 50 words as long<br>\n\nI will run this on random 10000 samples ","metadata":{}},{"cell_type":"code","source":"position_of_upvote = [0,0,0] # beginning, end and middle\nlen_of_markdowns = list()\nmarkdown_gap = dict()\nmarkdown_position_counter = {}\nmarkdowns_in_notebook = list()\nlonger_markdown_position = list()\nshort_markdown_position = list()\n\nrandom_list = random.sample(range(train_data.shape[0]),10_000) # random 10k samples\n\nfor i in tqdm(random_list):\n    cell_order, json_data = get_cell_order_and_json_data(i,display=False)\n    cell_type = json_data['cell_type']\n    source = json_data['source']\n    total_cell_orders = len(cell_order)\n    markdown_gap_counter = 0\n    markdown_counter = 0\n    \n    for j,cell_id in enumerate(cell_order):\n        if cell_type[cell_id] == 'markdown':\n            markdown_gap_counter += 1\n            markdown_counter += 1\n            markdown_split = source[cell_id].lower().split()\n            \n            position =j/total_cell_orders\n            markdown_len = len(markdown_split)\n            len_of_markdowns.append(markdown_len)\n            if markdown_len >= 50:\n                longer_markdown_position.append(position)\n            else:\n                short_markdown_position.append(position)\n            \n            #oth position is for counting all makrkdown cell in position x\n            if markdown_position_counter.get(j):\n                markdown_position_counter[j][0] += 1\n            else:\n                markdown_position_counter[j] = [1,0]\n            \n            if 'upvote' in source[cell_id].lower():\n                markdown_len = len(markdown_split)\n                if position <= 0.2:                 #in beginning\n                    position_of_upvote[0] +=1\n                elif position >= 0.8:               #in the end\n                    position_of_upvote[1] += 1\n                else:\n                    position_of_upvote[2] += 1          #in the middle\n        else:\n            if markdown_gap.get(markdown_gap_counter):\n                markdown_gap[markdown_gap_counter] += 1\n            else:\n                markdown_gap[markdown_gap_counter] = 1\n            markdown_gap_counter = 0\n            \n        #1st position is for counting all cells in the position\n        if markdown_position_counter.get(j):\n            markdown_position_counter[j][1] += 1\n        else:\n            markdown_position_counter[j] = [0,1]\n            \n    markdowns_in_notebook.append(markdown_counter)","metadata":{"execution":{"iopub.status.busy":"2022-05-20T13:26:56.681333Z","iopub.execute_input":"2022-05-20T13:26:56.682559Z","iopub.status.idle":"2022-05-20T13:32:03.127139Z","shell.execute_reply.started":"2022-05-20T13:26:56.682471Z","shell.execute_reply":"2022-05-20T13:32:03.126217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Count of \"Upvote ask\" at beginning end and middle of the notebook","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(15,7))\nsns.barplot(x=['beginning','end','middle'],\n            y=position_of_upvote)\nplt.title(\"Count of upvote at beginning, end and middle\")\nplt.xlabel(\"Position\")\nplt.ylabel(\"Count\");","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-20T13:32:16.341209Z","iopub.execute_input":"2022-05-20T13:32:16.341563Z","iopub.status.idle":"2022-05-20T13:32:16.556178Z","shell.execute_reply.started":"2022-05-20T13:32:16.341526Z","shell.execute_reply":"2022-05-20T13:32:16.555134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Yeah so I was expecting more upvote ask at the end, there are some middle ones too.","metadata":{}},{"cell_type":"markdown","source":"## Counter of Gap Betwwen Markdown cells","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(15,7))\nsns.barplot(x=list(markdown_gap.keys()),\n            y=list(markdown_gap.values()))\nplt.title(\"Count of gap between markdown cells\")\nplt.xlabel(\"Gap\")\nplt.ylabel(\"Count\");","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-20T13:32:21.470746Z","iopub.execute_input":"2022-05-20T13:32:21.471115Z","iopub.status.idle":"2022-05-20T13:32:21.849198Z","shell.execute_reply.started":"2022-05-20T13:32:21.471068Z","shell.execute_reply":"2022-05-20T13:32:21.848032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I was expecting 1 to be much higher but it seems like there are much more continous markdown cells.","metadata":{}},{"cell_type":"markdown","source":"## Distribution of length of markdown","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(15,7))\nlen_of_markdowns = [x for x in len_of_markdowns if x < 500]\nsns.histplot(len_of_markdowns,color='pink')\nplt.title(\"Distribution of length of markdown\");","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-20T13:32:26.905345Z","iopub.execute_input":"2022-05-20T13:32:26.906495Z","iopub.status.idle":"2022-05-20T13:32:28.409576Z","shell.execute_reply.started":"2022-05-20T13:32:26.906454Z","shell.execute_reply":"2022-05-20T13:32:28.408308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Distribution of Count of Markdowns in a Notebook","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(15,7))\nsns.histplot(markdowns_in_notebook,color='yellow')\nplt.title(\"Distribution of count of markdown in a Notebook\");","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-20T13:32:36.471791Z","iopub.execute_input":"2022-05-20T13:32:36.472159Z","iopub.status.idle":"2022-05-20T13:32:37.066208Z","shell.execute_reply.started":"2022-05-20T13:32:36.472124Z","shell.execute_reply":"2022-05-20T13:32:37.064982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Position of longer markdown","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(15,7))\nsns.histplot(longer_markdown_position,color='red')\nplt.title(\"Longer markdown position in a Notebook\");","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-20T13:32:40.716309Z","iopub.execute_input":"2022-05-20T13:32:40.716677Z","iopub.status.idle":"2022-05-20T13:32:41.017256Z","shell.execute_reply.started":"2022-05-20T13:32:40.716642Z","shell.execute_reply":"2022-05-20T13:32:41.015846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Longer markdown are much more closer to the starting as I expected.<br>\n(This could also mean that first markdown is default kaggle one, this needs to be checked)<br>\nBut one more thing is markdown are equally distributed among the other half of the notebook<br>","metadata":{}},{"cell_type":"markdown","source":"## Short Markdown positions","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(15,7))\nsns.histplot(short_markdown_position,color='green')\nplt.title(\"Short markdown position in a Notebook\");","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-20T13:32:46.340802Z","iopub.execute_input":"2022-05-20T13:32:46.341140Z","iopub.status.idle":"2022-05-20T13:32:46.781122Z","shell.execute_reply.started":"2022-05-20T13:32:46.341109Z","shell.execute_reply":"2022-05-20T13:32:46.779869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Interestingly short markdowns also follow similar patern as the longer ones","metadata":{}},{"cell_type":"markdown","source":"## Probability of markdown in a position","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(20,10))\nprobability_of_markdown = [x/y for x,y in list(markdown_position_counter.values())]\nsns.barplot(x=list(markdown_position_counter.keys()),y=probability_of_markdown)\nplt.title(\"Probability of markdown in a position\")\nplt.xticks(rotation=90)\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-20T13:32:50.925621Z","iopub.execute_input":"2022-05-20T13:32:50.926218Z","iopub.status.idle":"2022-05-20T13:33:01.198632Z","shell.execute_reply.started":"2022-05-20T13:32:50.926181Z","shell.execute_reply":"2022-05-20T13:33:01.197665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"One interesting thing here is if you see in between the pattern you will see there is a dip in the markdown cells.<br>\nwhich is sort of expected as there should be more code in the middle of the notebook.<br>","metadata":{}},{"cell_type":"markdown","source":"## I think this competition is about two question \n1. Whether there should be a code or markdown in this position?\n2. If a markdown then which markdown?\n \nI think both the questions could be answered by measuring relationship between code and markdown(just saying obvious things).<br>\nIf markdown is related to the code place it at the top of the code,or maybe below who knows I have seen notebooks explaining code after the code cell.<br>\n\nHard ones are non-code related markdown and markdown which explains concepts, maybe a good model might pick up on this as well.<br>","metadata":{}},{"cell_type":"markdown","source":"## obviously in the end, Please Upvote If you find this usefull ","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}