{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# This Notebook will show you the\n1. Overall distribution of code blocks and md blocks across all notebooks\n2. Distribution across ancestor_ids \n3. Distribution across parent_ids\n4. top 10 users with most notebboks","metadata":{}},{"cell_type":"code","source":"import os\nimport re\nimport json\nimport numpy as np\nimport pandas as pd \nfrom collections import Counter\nimport matplotlib.pyplot as plt\nfrom joblib import Parallel, delayed","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-18T06:03:13.016221Z","iopub.execute_input":"2022-05-18T06:03:13.016810Z","iopub.status.idle":"2022-05-18T06:03:13.123065Z","shell.execute_reply.started":"2022-05-18T06:03:13.016705Z","shell.execute_reply":"2022-05-18T06:03:13.121900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**#1Code blocks and Md blocks distribution**","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(\"../input/AI4Code/train_orders.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-05-18T06:03:13.124684Z","iopub.execute_input":"2022-05-18T06:03:13.124941Z","iopub.status.idle":"2022-05-18T06:03:15.155497Z","shell.execute_reply.started":"2022-05-18T06:03:13.124907Z","shell.execute_reply":"2022-05-18T06:03:15.153924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def count_code_md(filename):\n    with open(\"../input/AI4Code/train/\"+filename, 'r') as j:\n        contents = json.loads(j.read())\n        vals = list(contents[\"cell_type\"].values())\n        return len(vals), vals.count(\"code\"), vals.count(\"markdown\")","metadata":{"execution":{"iopub.status.busy":"2022-05-18T06:03:15.157787Z","iopub.execute_input":"2022-05-18T06:03:15.158137Z","iopub.status.idle":"2022-05-18T06:03:15.164040Z","shell.execute_reply.started":"2022-05-18T06:03:15.158093Z","shell.execute_reply":"2022-05-18T06:03:15.163311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, row in enumerate(df.iterrows()):\n    df.at[i, 'total_blocks'], df.at[i, 'code_blocks'], df.at[i, 'md_blocks'] = count_code_md(row[1][\"id\"]+\".json\")","metadata":{"execution":{"iopub.status.busy":"2022-05-18T06:03:15.165207Z","iopub.execute_input":"2022-05-18T06:03:15.166133Z","iopub.status.idle":"2022-05-18T06:15:22.426440Z","shell.execute_reply.started":"2022-05-18T06:03:15.166092Z","shell.execute_reply":"2022-05-18T06:15:22.424777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_blocks = df.drop([\"id\", \"cell_order\"], axis=1, inplace=False)","metadata":{"execution":{"iopub.status.busy":"2022-05-18T06:15:22.429464Z","iopub.execute_input":"2022-05-18T06:15:22.429760Z","iopub.status.idle":"2022-05-18T06:15:22.458634Z","shell.execute_reply.started":"2022-05-18T06:15:22.429725Z","shell.execute_reply":"2022-05-18T06:15:22.457110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_blocks.describe()","metadata":{"execution":{"iopub.status.busy":"2022-05-18T06:15:22.460451Z","iopub.execute_input":"2022-05-18T06:15:22.461093Z","iopub.status.idle":"2022-05-18T06:15:22.536938Z","shell.execute_reply.started":"2022-05-18T06:15:22.461051Z","shell.execute_reply":"2022-05-18T06:15:22.536286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(df_blocks[\"total_blocks\"], bins=1000)\nplt.gca().set(title='Frequency', ylabel='Frequency');","metadata":{"execution":{"iopub.status.busy":"2022-05-18T06:15:22.538174Z","iopub.execute_input":"2022-05-18T06:15:22.538959Z","iopub.status.idle":"2022-05-18T06:15:24.931583Z","shell.execute_reply.started":"2022-05-18T06:15:22.538920Z","shell.execute_reply":"2022-05-18T06:15:24.929784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(df_blocks[\"code_blocks\"], bins=200)\nplt.gca().set(title='Frequency', ylabel='Frequency');","metadata":{"execution":{"iopub.status.busy":"2022-05-18T06:15:24.934059Z","iopub.execute_input":"2022-05-18T06:15:24.934414Z","iopub.status.idle":"2022-05-18T06:15:25.409211Z","shell.execute_reply.started":"2022-05-18T06:15:24.934374Z","shell.execute_reply":"2022-05-18T06:15:25.407713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(df_blocks[\"md_blocks\"], bins=100)\nplt.gca().set(title='Frequency', ylabel='Frequency');","metadata":{"execution":{"iopub.status.busy":"2022-05-18T06:15:25.411555Z","iopub.execute_input":"2022-05-18T06:15:25.411836Z","iopub.status.idle":"2022-05-18T06:15:25.881786Z","shell.execute_reply.started":"2022-05-18T06:15:25.411804Z","shell.execute_reply":"2022-05-18T06:15:25.880199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**#2 Distribution across ancestor_ids**","metadata":{}},{"cell_type":"code","source":"df_ancestors = pd.read_csv(\"../input/AI4Code/train_ancestors.csv\")\ngb_aid = df_ancestors.groupby(\"ancestor_id\")","metadata":{"execution":{"iopub.status.busy":"2022-05-18T06:15:25.882991Z","iopub.execute_input":"2022-05-18T06:15:25.883218Z","iopub.status.idle":"2022-05-18T06:15:26.178669Z","shell.execute_reply.started":"2022-05-18T06:15:25.883192Z","shell.execute_reply":"2022-05-18T06:15:26.176722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gb_aid.count().sort_values(['id'], ascending=False)","metadata":{"execution":{"iopub.status.busy":"2022-05-18T06:15:26.187125Z","iopub.execute_input":"2022-05-18T06:15:26.187478Z","iopub.status.idle":"2022-05-18T06:15:26.398682Z","shell.execute_reply.started":"2022-05-18T06:15:26.187439Z","shell.execute_reply":"2022-05-18T06:15:26.396711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gb_aid = df_ancestors.groupby(\"ancestor_id\").size()\ngb_lst = list(gb_aid)\ngb_lst = [x for x in gb_lst if x > 5]","metadata":{"execution":{"iopub.status.busy":"2022-05-18T06:15:26.400038Z","iopub.execute_input":"2022-05-18T06:15:26.400292Z","iopub.status.idle":"2022-05-18T06:15:26.829161Z","shell.execute_reply.started":"2022-05-18T06:15:26.400261Z","shell.execute_reply":"2022-05-18T06:15:26.826822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(gb_lst, bins=100) \nplt.ylabel('Frequency')","metadata":{"execution":{"iopub.status.busy":"2022-05-18T06:15:26.830940Z","iopub.execute_input":"2022-05-18T06:15:26.831300Z","iopub.status.idle":"2022-05-18T06:15:27.153095Z","shell.execute_reply.started":"2022-05-18T06:15:26.831260Z","shell.execute_reply":"2022-05-18T06:15:27.151555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**#3 Distribution across parent_ids**","metadata":{}},{"cell_type":"code","source":"df_ancestors = pd.read_csv(\"../input/AI4Code/train_ancestors.csv\")\ngb_pid = df_ancestors.groupby(\"parent_id\")","metadata":{"execution":{"iopub.status.busy":"2022-05-18T06:15:27.154624Z","iopub.execute_input":"2022-05-18T06:15:27.155429Z","iopub.status.idle":"2022-05-18T06:15:27.440918Z","shell.execute_reply.started":"2022-05-18T06:15:27.155343Z","shell.execute_reply":"2022-05-18T06:15:27.439715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gb_pid.count().sort_values(['id'], ascending=False)","metadata":{"execution":{"iopub.status.busy":"2022-05-18T06:15:27.442961Z","iopub.execute_input":"2022-05-18T06:15:27.443863Z","iopub.status.idle":"2022-05-18T06:15:27.544607Z","shell.execute_reply.started":"2022-05-18T06:15:27.443822Z","shell.execute_reply":"2022-05-18T06:15:27.542796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gb_pid = df_ancestors.groupby(\"parent_id\").size()\ngbp_lst = list(gb_pid)\ngbp_lst = [x for x in gbp_lst if x > 5]","metadata":{"execution":{"iopub.status.busy":"2022-05-18T06:15:27.546312Z","iopub.execute_input":"2022-05-18T06:15:27.546692Z","iopub.status.idle":"2022-05-18T06:15:27.577849Z","shell.execute_reply.started":"2022-05-18T06:15:27.546659Z","shell.execute_reply":"2022-05-18T06:15:27.576856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(gbp_lst, bins=100) \nplt.ylabel('Frequency')","metadata":{"execution":{"iopub.status.busy":"2022-05-18T06:15:27.579859Z","iopub.execute_input":"2022-05-18T06:15:27.580155Z","iopub.status.idle":"2022-05-18T06:15:27.868376Z","shell.execute_reply.started":"2022-05-18T06:15:27.580123Z","shell.execute_reply":"2022-05-18T06:15:27.866445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**#4 Top 10 users with most notebboks**","metadata":{}},{"cell_type":"code","source":"def extract_user(file):\n    json_open = open(f'../input/AI4Code/train/{file}', 'r')\n    json_load = json.load(json_open)\n    json_load = '\\n'.join(json_load['source'].values())\n    res = re.findall(r'www.kaggle.com/+[a-zA-Z0-9_]+/', json_load)\n    res = set([r.split('/')[-2] for r in res])\n    res = [r for r in res if r not in ['c', 'kernels', 'learn']]\n    return res","metadata":{"execution":{"iopub.status.busy":"2022-05-18T06:15:27.869812Z","iopub.execute_input":"2022-05-18T06:15:27.870121Z","iopub.status.idle":"2022-05-18T06:15:27.877900Z","shell.execute_reply.started":"2022-05-18T06:15:27.870085Z","shell.execute_reply":"2022-05-18T06:15:27.876573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"files = os.listdir('../input/AI4Code/train')\nresult = Parallel(n_jobs=4, verbose=1)(delayed(extract_user)(file) for file in files)\nresult = sum(result, [])\ncount = Counter(result)","metadata":{"execution":{"iopub.status.busy":"2022-05-18T06:15:27.879591Z","iopub.execute_input":"2022-05-18T06:15:27.879851Z","iopub.status.idle":"2022-05-18T06:17:00.887254Z","shell.execute_reply.started":"2022-05-18T06:15:27.879818Z","shell.execute_reply":"2022-05-18T06:17:00.885781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, c in enumerate(count.most_common(10)):\n    print(i+1, c)","metadata":{"execution":{"iopub.status.busy":"2022-05-18T06:17:00.888375Z","iopub.execute_input":"2022-05-18T06:17:00.888576Z","iopub.status.idle":"2022-05-18T06:17:00.903498Z","shell.execute_reply.started":"2022-05-18T06:17:00.888551Z","shell.execute_reply":"2022-05-18T06:17:00.902305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}