{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-05-23T12:58:28.883082Z","iopub.execute_input":"2023-05-23T12:58:28.883606Z","iopub.status.idle":"2023-05-23T12:58:28.897248Z","shell.execute_reply.started":"2023-05-23T12:58:28.883565Z","shell.execute_reply":"2023-05-23T12:58:28.896134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install gensim","metadata":{"execution":{"iopub.status.busy":"2023-05-23T12:58:28.899085Z","iopub.execute_input":"2023-05-23T12:58:28.899792Z","iopub.status.idle":"2023-05-23T12:58:38.585454Z","shell.execute_reply.started":"2023-05-23T12:58:28.899764Z","shell.execute_reply":"2023-05-23T12:58:38.583348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/2023-kaggle-ai-report/kaggle_writeups_20230510.csv\")\nprint(df.sample(3))\ndf.shape","metadata":{"execution":{"iopub.status.busy":"2023-05-23T14:21:22.576818Z","iopub.execute_input":"2023-05-23T14:21:22.57723Z","iopub.status.idle":"2023-05-23T14:21:22.741294Z","shell.execute_reply.started":"2023-05-23T14:21:22.577188Z","shell.execute_reply":"2023-05-23T14:21:22.7406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.info()","metadata":{"execution":{"iopub.status.busy":"2023-05-23T14:22:54.990637Z","iopub.execute_input":"2023-05-23T14:22:54.99186Z","iopub.status.idle":"2023-05-23T14:22:55.016083Z","shell.execute_reply.started":"2023-05-23T14:22:54.991816Z","shell.execute_reply":"2023-05-23T14:22:55.015136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.describe()","metadata":{"execution":{"iopub.status.busy":"2023-05-23T14:20:04.235434Z","iopub.execute_input":"2023-05-23T14:20:04.235821Z","iopub.status.idle":"2023-05-23T14:20:04.569318Z","shell.execute_reply.started":"2023-05-23T14:20:04.23579Z","shell.execute_reply":"2023-05-23T14:20:04.56786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['Writeup']","metadata":{"execution":{"iopub.status.busy":"2023-05-23T12:58:38.779573Z","iopub.execute_input":"2023-05-23T12:58:38.779936Z","iopub.status.idle":"2023-05-23T12:58:38.791179Z","shell.execute_reply.started":"2023-05-23T12:58:38.779908Z","shell.execute_reply":"2023-05-23T12:58:38.79008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# LOWERCASING","metadata":{}},{"cell_type":"code","source":"df['Writeup'] = df['Writeup'].str.lower()\ndf['Writeup']","metadata":{"execution":{"iopub.status.busy":"2023-05-23T12:58:38.795618Z","iopub.execute_input":"2023-05-23T12:58:38.796667Z","iopub.status.idle":"2023-05-23T12:58:38.859603Z","shell.execute_reply.started":"2023-05-23T12:58:38.796593Z","shell.execute_reply":"2023-05-23T12:58:38.858873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# REMOVE HTML TAGS","metadata":{}},{"cell_type":"code","source":"import re\nprint(re.sub(':' , \" \" , re.sub(';' , ' ' , re.sub('&nbsp' , \"\" , \n       (re.sub(r'http\\S+', ' ', \n               (re.compile(r'<.*?>').sub(\"\" ,df[\"Writeup\"][0]))))))))","metadata":{"execution":{"iopub.status.busy":"2023-05-23T12:58:38.860774Z","iopub.execute_input":"2023-05-23T12:58:38.861934Z","iopub.status.idle":"2023-05-23T12:58:38.870084Z","shell.execute_reply.started":"2023-05-23T12:58:38.861904Z","shell.execute_reply":"2023-05-23T12:58:38.869067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[\"Writeup\"]","metadata":{"execution":{"iopub.status.busy":"2023-05-23T12:58:38.872489Z","iopub.execute_input":"2023-05-23T12:58:38.873805Z","iopub.status.idle":"2023-05-23T12:58:38.888165Z","shell.execute_reply.started":"2023-05-23T12:58:38.873764Z","shell.execute_reply":"2023-05-23T12:58:38.886828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# REMOVE EMOTICONS","metadata":{}},{"cell_type":"code","source":"emoj = re.compile(\"[\"\n    u\"\\U0001F600-\\U0001F64F\"  # emoticons\n    u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n    u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n    u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n    u\"\\U00002500-\\U00002BEF\"  # chinese char\n    u\"\\U00002702-\\U000027B0\"\n    u\"\\U00002702-\\U000027B0\"\n    u\"\\U000024C2-\\U0001F251\"\n    u\"\\U0001f926-\\U0001f937\"\n    u\"\\U00010000-\\U0010ffff\"\n    u\"\\u2640-\\u2642\" \n    u\"\\u2600-\\u2B55\"\n    u\"\\u200d\"\n    u\"\\u23cf\"\n    u\"\\u23e9\"\n    u\"\\u231a\"\n    u\"\\ufe0f\"  # dingbats\n    u\"\\u3030\"\n    u\"\\u2028\"\n    \"\\x08\"\n    u\"\\u200a\"\n    u\"\\u200b\"\n                  \"]+\", re.UNICODE)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T12:58:38.889729Z","iopub.execute_input":"2023-05-23T12:58:38.890065Z","iopub.status.idle":"2023-05-23T12:58:38.905668Z","shell.execute_reply.started":"2023-05-23T12:58:38.890036Z","shell.execute_reply":"2023-05-23T12:58:38.90464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\nimport pandas as pd\n\n# Assuming you have a DataFrame named 'df' with a column named 'Writeup'\n\n# Define the function to apply the code to each row\ndef clean_writeup(text):\n    cleaned_text = re.sub(':' , \" \" , re.sub(';' , ' ' , re.sub('&nbsp' , \"\" , \n       (re.sub(r'http\\S+', ' ', \n               (re.compile(r'<.*?>').sub(\"\" ,text)))))))\n    cleaned_text = emoj.sub(\"\", cleaned_text)\n    return cleaned_text\n\n# Convert the 'Writeup' column to string type\ndf[\"Writeup\"] = df[\"Writeup\"].astype(str)\n\n# Apply the function to each row of 'Writeup'\ndf[\"Writeup\"] = df[\"Writeup\"].apply(clean_writeup)\n","metadata":{"execution":{"iopub.status.busy":"2023-05-23T12:58:38.907176Z","iopub.execute_input":"2023-05-23T12:58:38.907826Z","iopub.status.idle":"2023-05-23T12:58:39.400796Z","shell.execute_reply.started":"2023-05-23T12:58:38.907794Z","shell.execute_reply":"2023-05-23T12:58:39.398862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# REMOVE STOPWORDS","metadata":{}},{"cell_type":"code","source":"# from nltk.corpus import stopwords\n# import nltk\n# nltk.download('stopwords')\n# from nltk.tokenize import word_tokenize\n\n# # Define the remove_stopwords function\n# def remove_stopwords(text):\n#     new_text = []\n#     for word in text.split():\n#         if word in stopwords.words('english'):\n#             new_text.append('')\n#         else:\n#             new_text.append(word)\n#     return \" \".join(new_text)\n\n# # Assuming you have a DataFrame named 'df' with a column named 'Writeup'\n\n# # Convert the 'Writeup' column to string type\n# df['Writeup'] = df['Writeup'].astype(str)\n\n# # Apply the remove_stopwords function to each row of 'Writeup'\n# df['Writeup'] = df['Writeup'].apply(remove_stopwords)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T12:58:39.403714Z","iopub.execute_input":"2023-05-23T12:58:39.404913Z","iopub.status.idle":"2023-05-23T12:58:39.410756Z","shell.execute_reply.started":"2023-05-23T12:58:39.40486Z","shell.execute_reply":"2023-05-23T12:58:39.40934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['Writeup']","metadata":{"execution":{"iopub.status.busy":"2023-05-23T12:58:39.412696Z","iopub.execute_input":"2023-05-23T12:58:39.413117Z","iopub.status.idle":"2023-05-23T12:58:39.42928Z","shell.execute_reply.started":"2023-05-23T12:58:39.41308Z","shell.execute_reply":"2023-05-23T12:58:39.428345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# STEMMING","metadata":{}},{"cell_type":"code","source":"import nltk\nfrom nltk.stem import PorterStemmer\nfrom nltk.tokenize import word_tokenize\n\n# Download necessary resources\nnltk.download('punkt')\n\n# Initialize the stemmer\nstemmer = PorterStemmer()\n\n# Define a function for stemming\ndef stemming(text):\n    # Tokenize the text\n    tokens = word_tokenize(text)\n    \n    # Perform stemming on each token\n    stemmed_tokens = [stemmer.stem(token) for token in tokens]\n    \n    # Join the stemmed tokens back into a sentence\n    stemmed_text = ' '.join(stemmed_tokens)\n    \n    return stemmed_text\n\n# Convert the 'Writeup' column to string type\ndf['Writeup'] = df['Writeup'].astype(str)\n\n# Apply stemming to each row of 'Writeup'\ndf['Writeup'] = df['Writeup'].apply(stemming)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T12:58:39.431511Z","iopub.execute_input":"2023-05-23T12:58:39.432943Z","iopub.status.idle":"2023-05-23T12:58:39.443496Z","shell.execute_reply.started":"2023-05-23T12:58:39.432894Z","shell.execute_reply":"2023-05-23T12:58:39.44221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['Writeup']","metadata":{"execution":{"iopub.status.busy":"2023-05-23T12:58:39.444652Z","iopub.execute_input":"2023-05-23T12:58:39.445579Z","iopub.status.idle":"2023-05-23T12:58:39.460639Z","shell.execute_reply.started":"2023-05-23T12:58:39.445513Z","shell.execute_reply":"2023-05-23T12:58:39.459603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# LEMMETIZATION","metadata":{}},{"cell_type":"code","source":"# import pandas as pd\n# from nltk.stem import WordNetLemmatizer\n# from nltk.tokenize import word_tokenize\n# import nltk\n\n# nltk.download('wordnet')\n\n# # Define the lemmatization function\n# def lemmatization(text):\n#     tokens = []\n#     lemmatizer = WordNetLemmatizer()\n#     input_str = word_tokenize(text)\n#     for word in input_str:\n#         tokens.append(lemmatizer.lemmatize(word))\n#     return \" \".join(tokens)\n\n# # Example DataFrame\n# # df = pd.DataFrame({'Writeup': ['I am running', 'She ran quickly', 'They run every day']})\n# df['Writeup'] = df['Writeup'].astype(str)\n# # Apply lemmatization to the 'Writeup' column\n# df['Writeup'] = df['Writeup'].apply(lemmatization)\n\n# # Print the updated DataFrame\n# print(df)\n","metadata":{"execution":{"iopub.status.busy":"2023-05-23T12:58:39.463802Z","iopub.execute_input":"2023-05-23T12:58:39.464313Z","iopub.status.idle":"2023-05-23T12:58:39.474028Z","shell.execute_reply.started":"2023-05-23T12:58:39.46427Z","shell.execute_reply":"2023-05-23T12:58:39.472028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df['Writeup']","metadata":{"execution":{"iopub.status.busy":"2023-05-23T12:58:39.476222Z","iopub.execute_input":"2023-05-23T12:58:39.476703Z","iopub.status.idle":"2023-05-23T12:58:39.487207Z","shell.execute_reply.started":"2023-05-23T12:58:39.476658Z","shell.execute_reply":"2023-05-23T12:58:39.485932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# WhitespaceTokenizer","metadata":{}},{"cell_type":"code","source":"from nltk.tokenize import WhitespaceTokenizer\ntk = WhitespaceTokenizer()\ndf['Writeup'] = df['Writeup'].astype(str)\ndf['Writeup'] = df['Writeup'].apply(lambda x: tk.tokenize(x))\ndf['Writeup']","metadata":{"execution":{"iopub.status.busy":"2023-05-23T12:58:39.488757Z","iopub.execute_input":"2023-05-23T12:58:39.490151Z","iopub.status.idle":"2023-05-23T12:58:40.713636Z","shell.execute_reply.started":"2023-05-23T12:58:39.49011Z","shell.execute_reply":"2023-05-23T12:58:40.712354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Clean the text in 'Writeup' column\ndf['Writeup'] = df['Writeup'].astype(str)\ndf['Writeup'] = df['Writeup'].apply(lambda x: re.sub(r\"[^a-zA-Z0-9]\", \" \", x))","metadata":{"execution":{"iopub.status.busy":"2023-05-23T12:59:05.908581Z","iopub.execute_input":"2023-05-23T12:59:05.908987Z","iopub.status.idle":"2023-05-23T12:59:06.936262Z","shell.execute_reply.started":"2023-05-23T12:59:05.908955Z","shell.execute_reply":"2023-05-23T12:59:06.935155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Word2Vec","metadata":{}},{"cell_type":"code","source":"import gensim\nfrom gensim.models import Word2Vec\nfrom nltk.tokenize import word_tokenize\ndf['Writeup'] = df['Writeup'].astype(str)\n# Tokenize the text in 'Writeup' column\ndf['Writeup'] = df['Writeup'].apply(lambda x: word_tokenize(x))\n\n# Train the Word2Vec model\nmodel = Word2Vec(window=5, min_count=2, workers=4, sg=0)\nmodel.build_vocab(df['Writeup'], progress_per=1000)\nmodel.train(df['Writeup'], total_examples=model.corpus_count, epochs=model.epochs)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T12:59:15.949288Z","iopub.execute_input":"2023-05-23T12:59:15.94972Z","iopub.status.idle":"2023-05-23T12:59:30.151925Z","shell.execute_reply.started":"2023-05-23T12:59:15.949658Z","shell.execute_reply":"2023-05-23T12:59:30.150528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"keys = model.wv.index_to_key","metadata":{"execution":{"iopub.status.busy":"2023-05-23T12:59:42.925832Z","iopub.execute_input":"2023-05-23T12:59:42.926236Z","iopub.status.idle":"2023-05-23T12:59:42.932467Z","shell.execute_reply.started":"2023-05-23T12:59:42.926203Z","shell.execute_reply":"2023-05-23T12:59:42.93099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# keys","metadata":{"execution":{"iopub.status.busy":"2023-05-23T13:00:29.44678Z","iopub.execute_input":"2023-05-23T13:00:29.447175Z","iopub.status.idle":"2023-05-23T13:00:29.452776Z","shell.execute_reply.started":"2023-05-23T13:00:29.447146Z","shell.execute_reply":"2023-05-23T13:00:29.451414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save(\"./responses.model\")","metadata":{"execution":{"iopub.status.busy":"2023-05-23T12:59:52.726229Z","iopub.execute_input":"2023-05-23T12:59:52.726612Z","iopub.status.idle":"2023-05-23T12:59:52.765279Z","shell.execute_reply.started":"2023-05-23T12:59:52.726582Z","shell.execute_reply":"2023-05-23T12:59:52.76415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.wv[\"classification\"]\nmodel.wv.most_similar(\"classification\", topn=2)","metadata":{"execution":{"iopub.status.busy":"2023-05-23T13:01:50.74973Z","iopub.execute_input":"2023-05-23T13:01:50.750138Z","iopub.status.idle":"2023-05-23T13:01:50.762016Z","shell.execute_reply.started":"2023-05-23T13:01:50.750109Z","shell.execute_reply":"2023-05-23T13:01:50.761227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[\"Date of Writeup\"] = pd.to_datetime(\n    df[\"Date of Writeup\"]\n)\n\ndf = df[\n    df['Date of Writeup'].dt.year >= 2021\n]","metadata":{"execution":{"iopub.status.busy":"2023-05-23T14:23:22.631005Z","iopub.execute_input":"2023-05-23T14:23:22.631873Z","iopub.status.idle":"2023-05-23T14:23:22.903259Z","shell.execute_reply.started":"2023-05-23T14:23:22.631833Z","shell.execute_reply":"2023-05-23T14:23:22.902116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.shape\ndf.info()","metadata":{"execution":{"iopub.status.busy":"2023-05-23T14:23:33.774363Z","iopub.execute_input":"2023-05-23T14:23:33.774823Z","iopub.status.idle":"2023-05-23T14:23:33.792727Z","shell.execute_reply.started":"2023-05-23T14:23:33.774787Z","shell.execute_reply":"2023-05-23T14:23:33.79095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df.groupby([\"Competition Launch Date\",\"Title of Competition\"])[[\"Competition URL\",\"Title of Writeup\"]].sum().reset_index()\ndf.head()\n# df.info()\n# df.shape","metadata":{"execution":{"iopub.status.busy":"2023-05-23T14:42:17.739793Z","iopub.execute_input":"2023-05-23T14:42:17.740147Z","iopub.status.idle":"2023-05-23T14:42:17.760786Z","shell.execute_reply.started":"2023-05-23T14:42:17.740116Z","shell.execute_reply":"2023-05-23T14:42:17.759215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pipe = pipeline(model=\"facebook/bart-large-mnli\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}