{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# !pip install spacy_language_detection","metadata":{"execution":{"iopub.status.busy":"2022-06-16T17:25:38.405702Z","iopub.execute_input":"2022-06-16T17:25:38.406555Z","iopub.status.idle":"2022-06-16T17:26:10.565721Z","shell.execute_reply.started":"2022-06-16T17:25:38.406447Z","shell.execute_reply":"2022-06-16T17:26:10.564243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Generating a New Dataset\n\n<p>\n    The data provided by the competition host takes too long to read it all. It took more than 10 hours just read and concat all the files. Thus, this notebook is used to generate a new dataset in the parquet format to quickly read the data. It is impossible to work in the competition if everytime it takes 10 hours to just read the data. Moreover, I added some features which I think may help in matching markdown with code cells.\n</p>\n\n<p> \n    A lot of the code is commented to avoid duplicates because the notebook was runned many times.\n</p>\n\n<p>\n    I used the help of 2 kaggle notebooks (with some extra search) to make the scripts:\n    <ul>\n        <li><a href='https://www.kaggle.com/code/ryanholbrook/getting-started-with-ai4code'>Getting Started with AI4Code</a></li>\n        <li><a href='https://www.kaggle.com/code/andradaolteanu/ai4code-language-detection-and-model-tuning'>AI4Code - Language Detection and Model Tuning</a></li>\n    </ul>\n</p>\n\n<p>\n    This notebook generates the <a href='https://www.kaggle.com/datasets/fmakarem/ai4code-train'>AI4Code Train Dataset</a> which contains the dataframes used in this notebook.\n</p>\n    ","metadata":{}},{"cell_type":"code","source":"import json\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nfrom scipy import sparse\nfrom tqdm import tqdm\nimport os\nimport re\nimport time\nimport json\n\nimport spacy\nfrom spacy.language import Language\n# from spacy_language_detection import LanguageDetector\n\npd.options.display.width = 180\npd.options.display.max_colwidth = 120\n\n# data_dir = Path('../input/AI4Code')\ndata_path='../input/ai4code-train/train_with_features.parquet'\nlanguage_path='../input/ai4code-train/language_mapping.parquet'","metadata":{"execution":{"iopub.status.busy":"2022-06-18T10:45:23.580635Z","iopub.execute_input":"2022-06-18T10:45:23.581109Z","iopub.status.idle":"2022-06-18T10:45:34.615875Z","shell.execute_reply.started":"2022-06-18T10:45:23.581018Z","shell.execute_reply":"2022-06-18T10:45:34.614866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Regex:","metadata":{}},{"cell_type":"code","source":"variable_regex='\\w[\\d\\w_]*(?=\\s*=|\\s*,)'\nmethod_regex='(?<=\\w\\.)\\w[\\d\\w_]*'\nclass_regex='(?<=class\\s)\\w[\\d\\w_]*(?=\\s*:|\\s*\\()'\nfunc_regex='(?<=def\\s)\\w[\\d\\w_]*(?=\\s*:|\\s*\\()'\nimport_regex='(?<=import\\s)\\w+(?=\\W)|(?<=from\\s)\\w+(?=\\simport)'\n\nvariable_pattern=re.compile(variable_regex)\nmethod_pattern=re.compile(method_regex)\nclass_pattern=re.compile(class_regex)\nfunc_pattern=re.compile(func_regex)\nimport_pattern=re.compile(import_regex)","metadata":{"execution":{"iopub.status.busy":"2022-06-18T10:45:34.617752Z","iopub.execute_input":"2022-06-18T10:45:34.618351Z","iopub.status.idle":"2022-06-18T10:45:34.627775Z","shell.execute_reply.started":"2022-06-18T10:45:34.618313Z","shell.execute_reply":"2022-06-18T10:45:34.627002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"patterns=[('variable',variable_pattern),\n          ('method',method_pattern),\n          ('class',class_pattern),\n          ('func',func_pattern),\n          ('import',import_pattern)\n         ]","metadata":{"execution":{"iopub.status.busy":"2022-06-18T10:45:34.628998Z","iopub.execute_input":"2022-06-18T10:45:34.629468Z","iopub.status.idle":"2022-06-18T10:45:34.641758Z","shell.execute_reply.started":"2022-06-18T10:45:34.629437Z","shell.execute_reply":"2022-06-18T10:45:34.641097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# paths_train = list((data_dir / 'train').glob('*.json'))\n# features_df=pd.DataFrame()\n\n# for path in paths_train:\n#     nb_id=path.stem\n#     print(nb_id)\n#     with open(path) as file: \n#         print(path)\n#         data=json.load(file)\n#         print(data['cell_type'])\n#         print(data['source'])\n#     break","metadata":{"execution":{"iopub.status.busy":"2022-06-16T17:26:21.99236Z","iopub.execute_input":"2022-06-16T17:26:21.99343Z","iopub.status.idle":"2022-06-16T17:26:22.004706Z","shell.execute_reply.started":"2022-06-16T17:26:21.99338Z","shell.execute_reply":"2022-06-16T17:26:22.003809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def read_notebook(path):\n#     df_row=(\n#         pd.read_json(\n#             path,\n#             dtype={'cell_type': 'category', 'source': 'str'})\n#         .assign(id=path.stem)\n#         .rename_axis('cell_id')\n#     )\n    \n#     return df_row\n\n\n# paths_train = list((data_dir / 'train').glob('*.json'))#[:10]\n\n# notebooks_train = [\n#     read_notebook(path) for path in tqdm(paths_train, desc='Train NBs')\n# ]\n# print('started concatenation.')\n# df = (\n#     pd.concat(notebooks_train)\n#     .set_index('id', append=True)\n#     .swaplevel()\n#     .sort_index(level='id', sort_remaining=False)\n# )\n\n# print('finished reading the data')\n\n# df","metadata":{"execution":{"iopub.status.busy":"2022-06-16T17:26:22.006499Z","iopub.execute_input":"2022-06-16T17:26:22.007232Z","iopub.status.idle":"2022-06-16T17:26:22.018787Z","shell.execute_reply.started":"2022-06-16T17:26:22.007141Z","shell.execute_reply":"2022-06-16T17:26:22.017688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for pattern_name, compiled_pattern in patterns:\n#     df[pattern_name]=df_row.apply(lambda row: ','.join(compiled_pattern.findall(row['source'])) if row['cell_type']=='code' else np.nan,axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-06-16T17:26:22.020808Z","iopub.execute_input":"2022-06-16T17:26:22.021623Z","iopub.status.idle":"2022-06-16T17:26:22.037583Z","shell.execute_reply.started":"2022-06-16T17:26:22.021576Z","shell.execute_reply":"2022-06-16T17:26:22.036411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Read the Data","metadata":{}},{"cell_type":"code","source":"data_df=pd.read_parquet(data_path)\nprint('data_df successful')\nlanguage_df=pd.read_parquet(language_path)\nprint('language_df successful')\norder_df=pd.read_csv('../input/AI4Code/train_orders.csv')\nprint('order_df successful')","metadata":{"execution":{"iopub.status.busy":"2022-06-18T10:45:34.643334Z","iopub.execute_input":"2022-06-18T10:45:34.643802Z","iopub.status.idle":"2022-06-18T10:46:36.695365Z","shell.execute_reply.started":"2022-06-18T10:45:34.643771Z","shell.execute_reply":"2022-06-18T10:46:36.694504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-18T10:46:36.6965Z","iopub.execute_input":"2022-06-18T10:46:36.696828Z","iopub.status.idle":"2022-06-18T10:46:36.720559Z","shell.execute_reply.started":"2022-06-18T10:46:36.6968Z","shell.execute_reply":"2022-06-18T10:46:36.719642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"language_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-18T10:46:36.721722Z","iopub.execute_input":"2022-06-18T10:46:36.722221Z","iopub.status.idle":"2022-06-18T10:46:36.732488Z","shell.execute_reply.started":"2022-06-18T10:46:36.722181Z","shell.execute_reply":"2022-06-18T10:46:36.731498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"order_df.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Features**\n#### The code below uses the regex expressions to generate some extra features.","metadata":{}},{"cell_type":"code","source":"count=0\nnb_with_duplicate_cells={}\n\ndata_df['has_duplicates']=False\n\nfor group_index,group_df in data_df.groupby('id'):\n    duplicated_cells=group_df[group_df['source'].duplicated(keep=False)]\n    if len(duplicated_cells):\n        nb_with_duplicate_cells[group_index]=1\n        data_df.loc[duplicated_cells.index,['has_duplicates']]=True\n        count+=1\n    else:\n        nb_with_duplicate_cells[group_index]=0\n#         df.loc[group_index,['has_duplicates']]=False\n    \ncount","metadata":{"execution":{"iopub.status.busy":"2022-06-18T10:47:01.586434Z","iopub.execute_input":"2022-06-18T10:47:01.586849Z","iopub.status.idle":"2022-06-18T10:47:42.847849Z","shell.execute_reply.started":"2022-06-18T10:47:01.586811Z","shell.execute_reply":"2022-06-18T10:47:42.846723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nb_with_duplicate_cells_df=pd.DataFrame.from_dict(nb_with_duplicate_cells,orient='index').rename(columns={0:'has_duplicates'})","metadata":{"execution":{"iopub.status.busy":"2022-06-18T10:48:57.736996Z","iopub.execute_input":"2022-06-18T10:48:57.737392Z","iopub.status.idle":"2022-06-18T10:48:57.815858Z","shell.execute_reply.started":"2022-06-18T10:48:57.737359Z","shell.execute_reply":"2022-06-18T10:48:57.814753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nb_with_duplicate_cells_df","metadata":{"execution":{"iopub.status.busy":"2022-06-18T10:48:58.676138Z","iopub.execute_input":"2022-06-18T10:48:58.676515Z","iopub.status.idle":"2022-06-18T10:48:58.686684Z","shell.execute_reply.started":"2022-06-18T10:48:58.676483Z","shell.execute_reply":"2022-06-18T10:48:58.685967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_copy=df.copy()\n# start_time=time.time()\n\n# print('started with the patterns')\n\n# patterns=[('variable',variable_pattern),\n#           ('method',method_pattern),\n#           ('class',class_pattern),\n#           ('func',func_pattern),\n#           ('imported',import_pattern)\n#          ]\n\n# for pattern_name, compiled_pattern in tqdm(patterns, desc='Feature Extraction'):\n#     df_copy[pattern_name]=df_copy.apply(lambda row: ','.join(compiled_pattern.findall(row['source'])) if row['cell_type']=='code' else np.nan,axis=1)\n#     print(f'finished {pattern_name}\\n')\n    \n# print('it took: ',time.time()-start_time)\n# df_copy","metadata":{"execution":{"iopub.status.busy":"2022-06-16T07:17:35.606219Z","iopub.execute_input":"2022-06-16T07:17:35.606959Z","iopub.status.idle":"2022-06-16T07:17:35.617818Z","shell.execute_reply.started":"2022-06-16T07:17:35.606919Z","shell.execute_reply":"2022-06-16T07:17:35.616882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Language**\n\n#### The code below is used to detect the language of the code cells using Spacy.","metadata":{}},{"cell_type":"code","source":"# Language Detector Function\n# def get_lang_detector(nlp, name):\n#     return LanguageDetector()\n\n# nlp_model = spacy.load(\"en_core_web_sm\")\n\n# Language.factory(\"language_detector\", func=get_lang_detector)\n# nlp_model.add_pipe('language_detector', last=True)","metadata":{"execution":{"iopub.status.busy":"2022-06-16T17:27:29.089792Z","iopub.execute_input":"2022-06-16T17:27:29.090097Z","iopub.status.idle":"2022-06-16T17:27:30.749892Z","shell.execute_reply.started":"2022-06-16T17:27:29.090069Z","shell.execute_reply":"2022-06-16T17:27:30.748833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def apply_language_detection(row: pd.Series,nlp=nlp_model,num_char=100):\n#     string_type=row['cell_type']\n    \n#     if 'markdown' in string_type.lower():\n#         text=row['source']\n#         doc = nlp(text[:num_char])\n#         return doc._.language\n#     else:\n#         return np.nan","metadata":{"execution":{"iopub.status.busy":"2022-06-16T17:27:30.752648Z","iopub.execute_input":"2022-06-16T17:27:30.753138Z","iopub.status.idle":"2022-06-16T17:27:30.759889Z","shell.execute_reply.started":"2022-06-16T17:27:30.75309Z","shell.execute_reply":"2022-06-16T17:27:30.758915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# id_to_lang={}\n\n# i=0\n# print('start ID to language mapping')\n# for key, group_df in df_copy.groupby('id'):\n#     markdown_df=group_df[group_df['cell_type']=='markdown']\n    \n#     if len(markdown_df):\n#         source=markdown_df.reset_index().loc[0]\n    \n#     language=apply_language_detection(source)\n    \n#     if i<5: \n#         print(f'language: {language[\"language\"]} score: {language[\"score\"]}')\n        \n#         i+=1\n    \n#     id_to_lang[key]=language\n\n# print('finished mapping')","metadata":{"execution":{"iopub.status.busy":"2022-06-16T18:05:44.615229Z","iopub.execute_input":"2022-06-16T18:05:44.615686Z","iopub.status.idle":"2022-06-16T18:05:44.661032Z","shell.execute_reply.started":"2022-06-16T18:05:44.615651Z","shell.execute_reply":"2022-06-16T18:05:44.65984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# id_to_lang={i:f'{i}-th' for i in range(20)}\n# print('length: ',len(id_to_lang))\n# key=list(id_to_lang.keys())[0]\n# print('Sample: ',key, '--', id_to_lang[key])","metadata":{"execution":{"iopub.status.busy":"2022-06-16T18:10:55.820448Z","iopub.execute_input":"2022-06-16T18:10:55.82102Z","iopub.status.idle":"2022-06-16T18:10:55.827383Z","shell.execute_reply.started":"2022-06-16T18:10:55.820974Z","shell.execute_reply":"2022-06-16T18:10:55.826127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_copy.copy().reset_index().apply(lambda row: str(id_to_lang[row['id']]['language']),axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-06-16T18:11:03.805331Z","iopub.execute_input":"2022-06-16T18:11:03.805811Z","iopub.status.idle":"2022-06-16T18:11:03.824011Z","shell.execute_reply.started":"2022-06-16T18:11:03.805775Z","shell.execute_reply":"2022-06-16T18:11:03.822793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# id_to_lang_df=pd.DataFrame.from_dict(id_to_lang,orient='index')","metadata":{"execution":{"iopub.status.busy":"2022-06-16T18:18:55.504305Z","iopub.execute_input":"2022-06-16T18:18:55.504711Z","iopub.status.idle":"2022-06-16T18:18:55.516568Z","shell.execute_reply.started":"2022-06-16T18:18:55.504679Z","shell.execute_reply":"2022-06-16T18:18:55.515698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print('add the language alongside each column')\n# df_copy=df_copy.drop(columns=['lang'])\n# language_values=df_copy.copy().reset_index().apply(lambda row: id_to_lang[row['id']]['language'],axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-06-16T18:16:22.856418Z","iopub.execute_input":"2022-06-16T18:16:22.856884Z","iopub.status.idle":"2022-06-16T18:16:22.879977Z","shell.execute_reply.started":"2022-06-16T18:16:22.856847Z","shell.execute_reply":"2022-06-16T18:16:22.879121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Order of cells**","metadata":{}},{"cell_type":"code","source":"if 'cell_order' not in data_df.columns:\n    print(\"'cell_order' not in data_df columns\")\n    order_dict={row['id']:row['cell_order'].split() for index, row in order_df.iterrows()}\n\n    j=0\n    for key, val in order_dict.items():\n        if j == 2:\n            break\n        print(f'key: {key}')\n        print(f'val: {val}')\n        j+=1\n\n    print(order_dict['00001756c60be8'].index('2a9e43d6'))\n\n    cell_order=data_df.apply(lambda row: order_dict[str(row.name[0])].index(str(row.name[1])),axis=1)\n    data_df.loc[data_df.index,['cell_order']]=cell_order\n    assert not pd.isna(data_df['cell_order']).any(), \"There is a notebook cell without a 'cell_order' value which is wrong.\"\nelse:\n    print(\"'cell_order' is in data_df columns\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Save The Data**","metadata":{}},{"cell_type":"markdown","source":"#### Save the data in parquet format","metadata":{}},{"cell_type":"code","source":"# df.to_parquet('preprocessed_train.parquet')\ndata_df.to_parquet('train_with_features.parquet')\nlanguage_df.to_parquet('language_mapping.parquet')\nnb_with_duplicate_cells_df.to_parquet('nb_with_duplicate_cells.parquet')","metadata":{"execution":{"iopub.status.busy":"2022-06-18T10:49:37.947267Z","iopub.execute_input":"2022-06-18T10:49:37.947819Z","iopub.status.idle":"2022-06-18T10:49:56.24477Z","shell.execute_reply.started":"2022-06-18T10:49:37.947781Z","shell.execute_reply":"2022-06-18T10:49:56.243983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}