{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n#for dirname, _, filenames in os.walk('/kaggle/input'):\n#    for filename in filenames:\n#        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-06-04T23:46:44.219082Z","iopub.execute_input":"2022-06-04T23:46:44.219828Z","iopub.status.idle":"2022-06-04T23:46:44.249469Z","shell.execute_reply.started":"2022-06-04T23:46:44.219735Z","shell.execute_reply":"2022-06-04T23:46:44.248488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This notebook preprocesses the training data into a Lemmatized DataFrame, saving the result as a pickle file that can be loaded quickly.  Most of the code is taken from the sample code, the rest the WordNetLemmatizer code found in https://www.kaggle.com/code/ilyaryabov/fastttext-sorting-with-cosine-distance-algo \n\nNote that for v2, I realized one would want the original text, so the Lemmatized version is saved as source_lem.","metadata":{}},{"cell_type":"code","source":"import json\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nfrom scipy import sparse\nfrom tqdm import tqdm\n\npd.options.display.width = 180\npd.options.display.max_colwidth = 120\n\ndata_dir = Path('../input/AI4Code')\n\n# from https://www.kaggle.com/code/ilyaryabov/fastttext-sorting-with-cosine-distance-algo\nimport re\nfrom nltk.stem import WordNetLemmatizer\n\nstemmer = WordNetLemmatizer()\n\ndef preprocess_text(document):\n        # Remove all the special characters\n        document = re.sub(r'\\W', ' ', str(document))\n\n        # remove all single characters\n        document = re.sub(r'\\s+[a-zA-Z]\\s+', ' ', document)\n\n        # Remove single characters from the start\n        document = re.sub(r'\\^[a-zA-Z]\\s+', ' ', document)\n\n        # Substituting multiple spaces with single space\n        document = re.sub(r'\\s+', ' ', document, flags=re.I)\n\n        # Removing prefixed 'b'\n        document = re.sub(r'^b\\s+', '', document)\n\n        # Converting to Lowercase\n        document = document.lower()\n        #return document\n\n        # Lemmatization\n        tokens = document.split()\n        tokens = [stemmer.lemmatize(word) for word in tokens]\n        tokens = [word for word in tokens if len(word) > 3]\n\n        preprocessed_text = ' '.join(tokens)\n        return preprocessed_text\n\n\ndef read_notebook(path):\n    df = pd.read_json(\n            path,\n            dtype={'cell_type': 'category', 'source': 'str'}).assign(id=path.stem).rename_axis('cell_id')\n    \n    df.source = df.source.apply(preprocess_text)\n    df['source_lem'] = df.source.apply(preprocess_text)\n    return df\n\npaths_train = list((data_dir / 'train').glob('*.json'))\nnotebooks_train = [\n    read_notebook(path) for path in paths_train\n]\ndf = (\n    pd.concat(notebooks_train)\n    .set_index('id', append=True)\n    .swaplevel()\n    .sort_index(level='id', sort_remaining=False)\n)\n\ndf.to_pickle(\"ai4code_preprocessed_training_data.pkl\")","metadata":{},"execution_count":null,"outputs":[]}]}