{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# AI4Code, A simple Neuronal Network\nHello this models will implement a simple Neuronal Net to tackle the prediction problem...\n\n\n**Competition Description...**\n\nThe goal of this competition is to understand the relationship between code and comments in Python notebooks. You are challenged to reconstruct the order of markdown cells in a given notebook based on the order of the code cells, demonstrating comprehension of which natural language references which code.","metadata":{}},{"cell_type":"code","source":"%%time\n# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nexamples = 15\ncounter = 0\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        counter += 1\n        print(os.path.join(dirname, filename))\n        if counter > examples:\n            break\n\nprint('')            \nprint('Done...')\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-13T20:43:42.624365Z","iopub.execute_input":"2022-05-13T20:43:42.624942Z","iopub.status.idle":"2022-05-13T20:44:13.432399Z","shell.execute_reply.started":"2022-05-13T20:43:42.624900Z","shell.execute_reply":"2022-05-13T20:44:13.431747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Import the requiered libraries...\nimport json\nfrom pathlib import Path\n\nfrom scipy import sparse\nfrom tqdm import tqdm\n\npd.options.display.width = 180\npd.options.display.max_colwidth = 120\n\ndata_dir = Path('../input/AI4Code')","metadata":{"execution":{"iopub.status.busy":"2022-05-13T20:44:13.433629Z","iopub.execute_input":"2022-05-13T20:44:13.434315Z","iopub.status.idle":"2022-05-13T20:44:13.439551Z","shell.execute_reply.started":"2022-05-13T20:44:13.434261Z","shell.execute_reply":"2022-05-13T20:44:13.438975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nNUM_TRAIN = 10_000\n\n# Define a function to read a json file...\ndef read_notebook(path):\n    \"\"\"\n    Read a json file and import it to a dataframe...\n    Args:\n        path (str): The filepath location to be loaded into the dataframe.\n    Returns:\n        df (dataframe): A dataframe with the json information.\n    \"\"\"\n\n    types = {'cell_type': 'category', 'source': 'str'}\n    df = pd.read_json(path, dtype = types).assign(id = path.stem).rename_axis('cell_id')\n    return df \n\n# Create a list of the locations of all the training files...\npaths_train = list((data_dir / 'train').glob('*.json'))[:NUM_TRAIN]\n\n# Create a list of dataframes from json files...\nnotebooks_train = [read_notebook(path) for path in tqdm(paths_train, desc='Train NBs')]","metadata":{"execution":{"iopub.status.busy":"2022-05-13T20:44:13.440608Z","iopub.execute_input":"2022-05-13T20:44:13.440999Z","iopub.status.idle":"2022-05-13T20:45:16.073472Z","shell.execute_reply.started":"2022-05-13T20:44:13.440944Z","shell.execute_reply":"2022-05-13T20:45:16.072541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Concat the dataframes from the notebooks_train into one frame...\n\ntrn_data = pd.concat(notebooks_train).set_index('id', append = True).swaplevel().sort_index(level = 'id', sort_remaining = False)\ntrn_data = trn_data.reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-05-13T20:45:16.083087Z","iopub.execute_input":"2022-05-13T20:45:16.083579Z","iopub.status.idle":"2022-05-13T20:45:19.469492Z","shell.execute_reply.started":"2022-05-13T20:45:16.083537Z","shell.execute_reply":"2022-05-13T20:45:19.468449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Display th efirst five to ten rows of data...\n\ntrn_data.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-05-13T20:45:19.470832Z","iopub.execute_input":"2022-05-13T20:45:19.471142Z","iopub.status.idle":"2022-05-13T20:45:19.485142Z","shell.execute_reply.started":"2022-05-13T20:45:19.471108Z","shell.execute_reply":"2022-05-13T20:45:19.484046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Display th efirst five to ten rows of data...\n\n# Get an example notebook\nnb_id = trn_data['id'][7]\nprint('Notebook:', nb_id)\n\nprint(\"The Disordered Notebook:\")\nnb = trn_data[trn_data['id'] == nb_id]\ndisplay(nb)\nprint()","metadata":{"execution":{"iopub.status.busy":"2022-05-13T20:45:19.486727Z","iopub.execute_input":"2022-05-13T20:45:19.486965Z","iopub.status.idle":"2022-05-13T20:45:19.700859Z","shell.execute_reply.started":"2022-05-13T20:45:19.486936Z","shell.execute_reply":"2022-05-13T20:45:19.699939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndf_orders = pd.read_csv('/kaggle/input/AI4Code/train_orders.csv')\ndf_orders.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-13T20:45:19.702331Z","iopub.execute_input":"2022-05-13T20:45:19.702635Z","iopub.status.idle":"2022-05-13T20:45:20.482155Z","shell.execute_reply.started":"2022-05-13T20:45:19.702590Z","shell.execute_reply":"2022-05-13T20:45:20.481319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndf_orders = pd.read_csv(data_dir / 'train_orders.csv', index_col='id',squeeze=True,).str.split()  # Split the string representation of cell_ids into a list\ndf_orders.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-13T20:45:20.483305Z","iopub.execute_input":"2022-05-13T20:45:20.483541Z","iopub.status.idle":"2022-05-13T20:45:23.322748Z","shell.execute_reply.started":"2022-05-13T20:45:20.483510Z","shell.execute_reply":"2022-05-13T20:45:23.321773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Get the correct order\ncell_order = df_orders.loc[nb_id]\n\nprint(\"The Ordered Notebook:\")\ntrn_data[trn_data['cell_id'].isin(cell_order)]","metadata":{"execution":{"iopub.status.busy":"2022-05-13T20:45:23.325798Z","iopub.execute_input":"2022-05-13T20:45:23.326029Z","iopub.status.idle":"2022-05-13T20:45:23.418484Z","shell.execute_reply.started":"2022-05-13T20:45:23.326000Z","shell.execute_reply":"2022-05-13T20:45:23.417496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n#...\nprint(cell_order)","metadata":{"execution":{"iopub.status.busy":"2022-05-13T20:45:23.421619Z","iopub.execute_input":"2022-05-13T20:45:23.421956Z","iopub.status.idle":"2022-05-13T20:45:23.428381Z","shell.execute_reply.started":"2022-05-13T20:45:23.421910Z","shell.execute_reply":"2022-05-13T20:45:23.427465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n#...\nexample = trn_data # Explain....\nexample = example.set_index(['id', 'cell_id']) # Explain.... \nexample = example.loc[nb_id, :] # Explain....\n\nexample = example.loc[cell_order, :]\ndisplay(example) # Explain....","metadata":{"execution":{"iopub.status.busy":"2022-05-13T20:45:23.430061Z","iopub.execute_input":"2022-05-13T20:45:23.430362Z","iopub.status.idle":"2022-05-13T20:45:25.335949Z","shell.execute_reply.started":"2022-05-13T20:45:23.430317Z","shell.execute_reply":"2022-05-13T20:45:25.335034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n#...\ndef get_ranks(base, derived):\n    return [base.index(d) for d in derived]\n\ncell_ranks = get_ranks(cell_order, list(example.index))\nexample.insert(0, 'rank', cell_ranks)\nexample","metadata":{"execution":{"iopub.status.busy":"2022-05-13T20:45:25.337340Z","iopub.execute_input":"2022-05-13T20:45:25.337656Z","iopub.status.idle":"2022-05-13T20:45:25.355143Z","shell.execute_reply.started":"2022-05-13T20:45:25.337612Z","shell.execute_reply":"2022-05-13T20:45:25.354023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert the df_orders to a dataframe...\ndf_orders = df_orders.to_frame()\ndf_orders.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-13T21:19:14.396152Z","iopub.execute_input":"2022-05-13T21:19:14.396747Z","iopub.status.idle":"2022-05-13T21:19:14.413734Z","shell.execute_reply.started":"2022-05-13T21:19:14.396679Z","shell.execute_reply":"2022-05-13T21:19:14.412886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Join the df_orders_ dataframe with the json dataframe...\ntrn_data_grouped = trn_data.groupby('id')['cell_id'].apply(list)\ntrn_data_grouped.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-13T21:20:25.945213Z","iopub.execute_input":"2022-05-13T21:20:25.945498Z","iopub.status.idle":"2022-05-13T21:20:26.360993Z","shell.execute_reply.started":"2022-05-13T21:20:25.945467Z","shell.execute_reply":"2022-05-13T21:20:26.359962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_orders_ = df_orders.join(trn_data_grouped, how = 'right')\ndf_orders_.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-13T21:21:40.939858Z","iopub.execute_input":"2022-05-13T21:21:40.940136Z","iopub.status.idle":"2022-05-13T21:21:40.983217Z","shell.execute_reply.started":"2022-05-13T21:21:40.940106Z","shell.execute_reply":"2022-05-13T21:21:40.982474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ranks = {} # Creates an empty dictionary of ranks...\nfor id_, cell_order, cell_id in df_orders_.itertuples():\n    ranks[id_] = {'cell_id': cell_id, 'rank': get_ranks(cell_order, cell_id)}","metadata":{"execution":{"iopub.status.busy":"2022-05-13T21:22:23.647150Z","iopub.execute_input":"2022-05-13T21:22:23.647425Z","iopub.status.idle":"2022-05-13T21:22:24.336747Z","shell.execute_reply.started":"2022-05-13T21:22:23.647395Z","shell.execute_reply":"2022-05-13T21:22:24.335635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ranks = pd.DataFrame.from_dict(ranks, orient = 'index').rename_axis('id').apply(pd.Series.explode).set_index('cell_id', append = True)\ndf_ranks.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-13T21:23:12.206172Z","iopub.execute_input":"2022-05-13T21:23:12.206493Z","iopub.status.idle":"2022-05-13T21:23:14.276166Z","shell.execute_reply.started":"2022-05-13T21:23:12.206449Z","shell.execute_reply":"2022-05-13T21:23:14.275058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}