{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## The input [dataset](https://www.kaggle.com/datasets/ilyaryabov/correctly-ordered-notebooks-for-google-ai4code) with correctly ordered notebooks for Google AI4Code competition\n\n## The output [dataset](https://www.kaggle.com/datasets/ilyaryabov/fasttext-model-for-google-ai4code) with a pretrained models","metadata":{}},{"cell_type":"markdown","source":"## This notebook describes how to create a fasttext model for the Google AI4code competition","metadata":{}},{"cell_type":"code","source":"print(\"importing libraries...\")","metadata":{"execution":{"iopub.status.busy":"2022-05-17T17:27:37.312479Z","iopub.execute_input":"2022-05-17T17:27:37.312796Z","iopub.status.idle":"2022-05-17T17:27:37.322724Z","shell.execute_reply.started":"2022-05-17T17:27:37.312761Z","shell.execute_reply":"2022-05-17T17:27:37.321849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport re\nimport fasttext\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics.pairwise import cosine_similarity\nfrom nltk.stem import WordNetLemmatizer","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-17T17:27:38.532289Z","iopub.execute_input":"2022-05-17T17:27:38.532868Z","iopub.status.idle":"2022-05-17T17:27:39.963568Z","shell.execute_reply.started":"2022-05-17T17:27:38.532830Z","shell.execute_reply":"2022-05-17T17:27:39.962860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"src = '/kaggle/input/AI4Code/'\ndataset_path = '/kaggle/input/correctly-ordered-notebooks-for-google-ai4code/dataset/dataset/'","metadata":{"execution":{"iopub.status.busy":"2022-05-17T17:27:41.052578Z","iopub.execute_input":"2022-05-17T17:27:41.054681Z","iopub.status.idle":"2022-05-17T17:27:41.058599Z","shell.execute_reply.started":"2022-05-17T17:27:41.054630Z","shell.execute_reply":"2022-05-17T17:27:41.057729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stemmer = WordNetLemmatizer()\n\ndef preprocess_text(document):\n        # Remove all the special characters\n        document = re.sub(r'\\W', ' ', str(document))\n\n        # remove all single characters\n        document = re.sub(r'\\s+[a-zA-Z]\\s+', ' ', document)\n\n        # Remove single characters from the start\n        document = re.sub(r'\\^[a-zA-Z]\\s+', ' ', document)\n\n        # Substituting multiple spaces with single space\n        document = re.sub(r'\\s+', ' ', document, flags=re.I)\n\n        # Removing prefixed 'b'\n        document = re.sub(r'^b\\s+', '', document)\n\n        # Converting to Lowercase\n        document = document.lower()\n        \n        # remove digits\n        document = re.sub(r'[0-9]+', '', document)\n\n        # Lemmatization\n        tokens = document.split()\n        tokens = [stemmer.lemmatize(word) for word in tokens]\n        tokens = [word for word in tokens if len(word) > 2]\n\n        preprocessed_text = ' '.join(tokens)\n        return preprocessed_text\n    \n    \ndef preprocess_df(df):\n    return [preprocess_text(message) for message in df.source]\n","metadata":{"execution":{"iopub.status.busy":"2022-05-17T17:27:50.865414Z","iopub.execute_input":"2022-05-17T17:27:50.866149Z","iopub.status.idle":"2022-05-17T17:27:50.875299Z","shell.execute_reply.started":"2022-05-17T17:27:50.866112Z","shell.execute_reply":"2022-05-17T17:27:50.872874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# There were experiments with \n# N = 10000 (7~% of dataset) and \n# N = 140000 (100% of dataset)\n# They gave approximattly the same result\n\nN = 10000\n\nsourses = []\nprint('searching files...')\nfiles = os.listdir(dataset_path)[:N]\nprint('reading files...')\ndfs = [pd.read_csv(dataset_path + file) for file in tqdm(files)]\nprint('processing files...')\nsourses = [preprocess_df(df) for df in tqdm(dfs)]","metadata":{"execution":{"iopub.status.busy":"2022-05-17T17:27:54.432344Z","iopub.execute_input":"2022-05-17T17:27:54.433055Z","iopub.status.idle":"2022-05-17T17:28:00.343379Z","shell.execute_reply.started":"2022-05-17T17:27:54.433017Z","shell.execute_reply":"2022-05-17T17:28:00.342617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"processed_data = \"preprocessed_messages.txt\"\nprint('creating txt file...')\nwith open(processed_data, \"w\") as fout:\n    for notebook in tqdm(dfs):\n        for i in range(len(notebook)):\n            fout.write('{}\\n'.format( preprocess_text(notebook.source[i])) )\n            #print(notebook.source[i])\n    fout.close()","metadata":{"execution":{"iopub.status.busy":"2022-05-16T14:25:26.449648Z","iopub.execute_input":"2022-05-16T14:25:26.452118Z","iopub.status.idle":"2022-05-16T14:25:27.883725Z","shell.execute_reply.started":"2022-05-16T14:25:26.452042Z","shell.execute_reply":"2022-05-16T14:25:27.882849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('teaching model...')\nmodel = fasttext.train_unsupervised(input = processed_data)","metadata":{"execution":{"iopub.status.busy":"2022-05-16T14:25:30.550147Z","iopub.execute_input":"2022-05-16T14:25:30.550687Z","iopub.status.idle":"2022-05-16T14:25:38.59538Z","shell.execute_reply.started":"2022-05-16T14:25:30.550649Z","shell.execute_reply":"2022-05-16T14:25:38.593241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('saving model...')\nmodel.save_model(f'model{N}.bin')\nprint('DONE')","metadata":{"execution":{"iopub.status.busy":"2022-05-16T14:26:27.660849Z","iopub.execute_input":"2022-05-16T14:26:27.661379Z","iopub.status.idle":"2022-05-16T14:26:28.241554Z","shell.execute_reply.started":"2022-05-16T14:26:27.661342Z","shell.execute_reply":"2022-05-16T14:26:28.24069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from zipfile import ZipFile\nZipFile('model.zip', mode='w').write(f'model{N}.bin')","metadata":{"execution":{"iopub.status.busy":"2022-05-16T14:26:35.451795Z","iopub.execute_input":"2022-05-16T14:26:35.452266Z","iopub.status.idle":"2022-05-16T14:26:38.431226Z","shell.execute_reply.started":"2022-05-16T14:26:35.45223Z","shell.execute_reply":"2022-05-16T14:26:38.430328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm preprocessed_messages.txt\n!rm model*.bin","metadata":{"execution":{"iopub.status.busy":"2022-05-16T14:27:26.446807Z","iopub.execute_input":"2022-05-16T14:27:26.447406Z","iopub.status.idle":"2022-05-16T14:27:28.093415Z","shell.execute_reply.started":"2022-05-16T14:27:26.447369Z","shell.execute_reply":"2022-05-16T14:27:28.092358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## To be continued here: [Model application](https://www.kaggle.com/ilyaryabov/fastttext-sorting-with-cosine-distance-algo)","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}