{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import nltk\nimport re\nimport string","metadata":{"execution":{"iopub.status.busy":"2023-04-27T07:22:50.725948Z","iopub.execute_input":"2023-04-27T07:22:50.726404Z","iopub.status.idle":"2023-04-27T07:22:51.840187Z","shell.execute_reply.started":"2023-04-27T07:22:50.726353Z","shell.execute_reply":"2023-04-27T07:22:51.838869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-04-27T07:22:53.335042Z","iopub.execute_input":"2023-04-27T07:22:53.335512Z","iopub.status.idle":"2023-04-27T07:22:53.359189Z","shell.execute_reply.started":"2023-04-27T07:22:53.335471Z","shell.execute_reply":"2023-04-27T07:22:53.358181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## get the clean data for train\nraw_train = pd.read_csv(\"/kaggle/input/ai4vozer-k1a1-supervised-classification-model/raw_train.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-04-25T07:30:14.331477Z","iopub.execute_input":"2023-04-25T07:30:14.331921Z","iopub.status.idle":"2023-04-25T07:30:15.723108Z","shell.execute_reply.started":"2023-04-25T07:30:14.331886Z","shell.execute_reply":"2023-04-25T07:30:15.721104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## get the clean data for train\ntrain = pd.read_csv(\"/kaggle/input/ai4vozer-k1a1-supervised-classification-model/clean_train.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-04-25T07:30:18.045428Z","iopub.execute_input":"2023-04-25T07:30:18.046033Z","iopub.status.idle":"2023-04-25T07:30:20.074018Z","shell.execute_reply.started":"2023-04-25T07:30:18.04598Z","shell.execute_reply":"2023-04-25T07:30:20.072501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"raw_train.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-25T07:30:24.147675Z","iopub.execute_input":"2023-04-25T07:30:24.148254Z","iopub.status.idle":"2023-04-25T07:30:24.184582Z","shell.execute_reply.started":"2023-04-25T07:30:24.148182Z","shell.execute_reply":"2023-04-25T07:30:24.183268Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import nltk\nnltk.download('punkt')\nnltk.download('stopwords')\nnltk.download('wordnet')\n!unzip /usr/share/nltk_data/corpora/wordnet.zip -d /usr/share/nltk_data/corpora/","metadata":{"execution":{"iopub.status.busy":"2023-04-27T04:49:36.390839Z","iopub.execute_input":"2023-04-27T04:49:36.392066Z","iopub.status.idle":"2023-04-27T04:49:38.166578Z","shell.execute_reply.started":"2023-04-27T04:49:36.392015Z","shell.execute_reply":"2023-04-27T04:49:38.16544Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Clean text\ndef clean_text(text):\n    text = re.sub(r'^https?:\\/\\/.*[\\r\\n]*', '', text)\n    punctuations = string.punctuation\n    text = re.sub(r'[^\\w\\s]', ' ', text)\n    text = re.sub(r'\\s+', ' ', text).strip()\n    emoji_pattern = re.compile(\"[\"\n                           u\"\\U0001F600-\\U0001F64F\"  # emoticons\n                           u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n                           u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n                           u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n                           u\"\\U00002702-\\U000027B0\"\n                           u\"\\U000024C2-\\U0001F251\"\n                           \"]+\", flags=re.UNICODE)\n    text = emoji_pattern.sub(r'', text)\n    text = re.sub(r'[^a-zA-Z0-9\\s]+', '', text)\n    return text","metadata":{"execution":{"iopub.status.busy":"2023-04-25T11:32:12.122479Z","iopub.execute_input":"2023-04-25T11:32:12.12292Z","iopub.status.idle":"2023-04-25T11:32:12.130823Z","shell.execute_reply.started":"2023-04-25T11:32:12.122884Z","shell.execute_reply":"2023-04-25T11:32:12.129505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install pyspellchecker","metadata":{"execution":{"iopub.status.busy":"2023-04-25T06:58:28.898979Z","iopub.execute_input":"2023-04-25T06:58:28.899466Z","iopub.status.idle":"2023-04-25T06:58:42.191791Z","shell.execute_reply.started":"2023-04-25T06:58:28.899424Z","shell.execute_reply":"2023-04-25T06:58:42.190223Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from spellchecker import SpellChecker\n\nspell = SpellChecker(distance = 2)\nspell.correction('chiaroscuro')","metadata":{"execution":{"iopub.status.busy":"2023-04-25T11:19:55.438013Z","iopub.execute_input":"2023-04-25T11:19:55.438476Z","iopub.status.idle":"2023-04-25T11:19:55.585828Z","shell.execute_reply.started":"2023-04-25T11:19:55.438437Z","shell.execute_reply":"2023-04-25T11:19:55.584388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"str = 'paartyy'\nlen(str)\n# help(str.count)","metadata":{"execution":{"iopub.status.busy":"2023-04-25T08:47:37.79757Z","iopub.execute_input":"2023-04-25T08:47:37.79801Z","iopub.status.idle":"2023-04-25T08:47:37.807863Z","shell.execute_reply.started":"2023-04-25T08:47:37.797971Z","shell.execute_reply":"2023-04-25T08:47:37.8056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"arr = ['one', 'stereotype', 'i', 'disagree', 'infps', 'one', 'giant', 'cause', 'i', 'a', 'cause', 'i', 'several', 'smaller', 'causes', 'i', 'actual', 'felt', 'one', 'time', 'might', 'something', 'wrong', 'inspired', 'diaphinisedbat', 'first', 'image', 'chiaroscuro', 'http', 'static', 'flickr', 'com', 'jpg', 'grownups', 'http', 'imgs', 'xkcd', 'com', 'comics', 'grownups', 'png', 'myspace', 'http', 'imgs', 'xkcd', 'com', 'comics', 'joinmyspace', 'png', 'delicious', 'http', 'imgs', 'xkcd', 'com', 'comics', 'delicious', 'png', 'dreams', 'i', 'present', 'the', 'tiniest', 'snorfer', 'happy', 'happy', 'happy', 'http', 'static', 'flickr', 'com', 'jpg', 'http', 'static', 'flickr', 'com', 'jpg', 'http', 'files', 'wordpress', 'com', 'steven', 'klein', 'jpg', 'w', 'h', 'i', 'like', 'weird', 'intx', 'that', 'make', 'sense', 'that', 'kitten', 'body', 'various', 'foods', 'infp', 'yum', 'pop', 'tarts', 'i', 'convinced', 'd', 'some', 'pet', 'peeves', 'people', 'think', 'since', 'parking', 'lot', 'common', 'driving', 'laws', 'longer', 'apply', 'cats', 'i', 'animal', 'lover', 'i', 'try', 'love', 'little', 'feline', 'devils', 'accidentally', 'forget', 'recharge', 'cellphone', 'where', 'keys', 'i', 'really', 'really', 'liked', 'lost', 'i', 'cried', 'hours', 'straight', 'series', 'finale', 'lost', 'fueled', 'imagination', 'emotions', 'it', 'helped', 'understand', 'people', 'even', 'better', 'and', 'hurley', 'that', 'page', 'best', 'source', 'imho', 'a', 'lot', 'list', 'seems', 'weaknesses', 'human', 'race', 'infps', 'infjs', 'i', 'use', 'why', 'reconsidering', 'type', 'i', 'may', 'ask', 'i', 'think', 'nfs', 'fall', 'love', 'easily', 'lots', 'unrequited', 'love', 'throughout', 'life', 'sucks', 'muchly', 'sigh', 'aw', 'man', 'got', 'bad', 'baby', 'd', 'my', 'mother', 't', 'would', 'never', 'say', 'something', 'like', 'i', 'know', 'appreciate', 'beauty', 'song', 'without', 'analyze', 'dear', 'powers', 'that', 'be', 'i', 'demand', 'enfj', 'women', 'thank', 'advance', 'sincerely', 'infp', 'male', 'when', 'i', 'stream', 'consciousness', 'mode', 'i', 'really', 'eccentric', 'sure', 'but', 'i', 'rarely', 'let', 'side', 'show', 'people', 'i', 'know', 'well', 'so', 'i', 'guess', 'i', 'know', 'maybe', 'i', 'talk', 'mine', 'but', 'quantify', 'human', 'condition', 'think', 'way', 'you', 'buy', 'copy', 'sheet', 'music', 'moonlight', 'sonata', 'you', 'idea', 'explain', 'world', 'world', 'filled', 'another', 'simmer', 'yays', 'us', 'i', 'write', 'rewrite', 'lot', 'character', 'sketches', 'i', 'something', 'really', 'amazing', 'floating', 'around', 'mental', 'ionosphere', 'i', 'need', 'flesh', 'characters', 'lot', 'it', 'set', 'multiple', 'time', 'there', 'things', 'said', 'must', 'sung', 'said', 'magic', 'lost', 'i', 'hope', 'helps', 'shrug', 'http', 'ww', 'youtube', 'com', 'watch', 'v', 'just', 'browsing', 'youtube', 'found', 'gem', 'happy', 'love', 'video', 'infp', 'male', 'i', 'also', 'highly', 'symbolic', 'love', 'love', 'love', 'i', 'also', 'get', 'far', 'description', 'come', 'man', 'throw', 'purple', 'monkey', 'something', 'every', 'i', 'fall', 'asleep', 'i', 'glad', 'people', 'quoted', 'article', 'i', 'knew', 'sparkle', 'wondering', 'someone', 'would', 'put', 'natalie', 'merchant', 'about', 'time', 'here', 'another', 'tori', 'amos', 'a', 'direct', 'quote', 'tori', 'this', 'song', 'describes', 'irreversible', 'damage', 'pain', 'lot', 'good', 'stuff', 'thread', 'like', 'double', 'stuffed', 'oreos', 'chocolate', 'covered', 'double', 'stuffed', 'oreos', 'nom', 'nom', 'nom', 'want', 'noticed', 'want', 'notice', 'noticed', 'i', 'think', 'brain', 'flipped', 'upside', 'i', 'part', 'system', 'and', 'also', 'clowns', 'pandas', 'umbrellas', 'why', 'because', 'i', 'like', 'clowns', 'pandas', 'that', 'the', 'umbrellas', 'i', 'suspect', 'things', 'get', 'messy', 'not', 'grape', 'vinaigrette', 'raspberry', 'vinaigrette', 'the', 'thing', 'i', 'decide', 'kind', 'juice', 'and', 'i', 'limited', 'fruit', 'and', 'soup', 'acceptable', 'well', 'and', 'soup', 'milk', 'based', 'soups', 'then', 'fruit', 'juice', 'infps', 'moody', 'lol', 'j', 'k', 'j', 'work', 'first', 'play', 'later', 'p', 'play', 'first', 'work', 'later', 'a', 'better', 'description', 'put', 'making', 'decisions', 'till', 'last', 'possible', 'moment', 'p', 'for', 'example', 'go', 'bookstore', 'paartyy', 'woo', 'wait', 'maybe', 'caps', 'i', 'so', 'basically', 'i', 'single', 'forever', 'sigh', 'coffee', 'helps', 'i', 'think', 'seem', 'unhappy', 'dwell', 'feelings', 'much', 'maybe', 'sense', 'unhappiness', 'comes', 'looking', 'ideal', 'want', 'feeling', 'unhappy', 'we', 'like', 'dew', 'flower', 'petals', 'get', 'close', 'see', 'true', 'beauty', 'we', 'like', 'things', 'lot', 'hard', 'find', 'we', 'anywhere', 'find', 'i', 'think', 'p', 'the', 'two', 'types', 'i', 'think', 'like', 'esfp', 'enfp', 'possibly', 'estp', 'i', 'think', 'i', 'ever', 'going', 'know', 'sure', 'i', 'narrowing', 'least', 'thanks', 'nice', 'i', 'new', 'that', 'sounds', 'like', 'something', 'say', 'about', 'years', 'ago', 'i', 'worked', 'seasonal', 'job', 'hotel', 'miles', 'friends', 'family', 'i', 'phone', 'communication', 'solely', 'letter', 'writing', 'i', 'lived', 'i', 'need', 'advice', 'i', 'get', 'lot', 'i', 'finally', 'pester', 'woman', 'enough', 'get', 'go', 'least', 'you', 'actually', 'normal', 'person', 'try', 'identify', 'areas', 'life', 'causing', 'anxiety', 'set', 'concrete', 'goals', 'remedy', 'anxiety', 'i', 'know', 'exactly', 'feeling', 'i', 'having', 'ideal', 'ah', 'man', 'i', 'want', 'swim', 'pool', 'juice', 'understand', 'understand', 'understand', 'play', 'video', 'games', 'storyline', 'know', 'love', 'know', 'what', 'tori', 'amos', 'or', 'i', 'wrong', 'idea', 'infp', 'or', 'enough', 'heard', 'here', 'one', 'favorite', 'songs', 'infp', 'song', 'silent', 'all', 'these', 'years', 'excuse', 'hi', 'everyone', 'just', 'saying', 'hi', 'new', 'etc', 'tongue', 'interesting', 'place', 'think', 'i', 'pull', 'chair', 'ok', 'start', 'talking', 'i', 'interject', 'wise', 'occasionally', 'witty', 'comment']\n\nlemmawds = [lemma.lemmatize(word) for word in arr]\n\ncorrect_word = [spell.correction(word) for word in lemmawds if (len(word) > 1 and spell.correction(word) !=None)]","metadata":{"execution":{"iopub.status.busy":"2023-04-25T09:04:23.634973Z","iopub.execute_input":"2023-04-25T09:04:23.635413Z","iopub.status.idle":"2023-04-25T09:04:30.005831Z","shell.execute_reply.started":"2023-04-25T09:04:23.635375Z","shell.execute_reply":"2023-04-25T09:04:30.004464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"correct_word","metadata":{"execution":{"iopub.status.busy":"2023-04-25T09:04:31.833023Z","iopub.execute_input":"2023-04-25T09:04:31.833462Z","iopub.status.idle":"2023-04-25T09:04:31.852732Z","shell.execute_reply.started":"2023-04-25T09:04:31.833423Z","shell.execute_reply":"2023-04-25T09:04:31.851304Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_lengthening(text):\n    pattern = re.compile(r\"(.)\\1{2,}\")\n    return pattern.sub(r\"\\1\\1\", text)","metadata":{"execution":{"iopub.status.busy":"2023-04-25T09:07:23.05035Z","iopub.execute_input":"2023-04-25T09:07:23.050767Z","iopub.status.idle":"2023-04-25T09:07:23.057129Z","shell.execute_reply.started":"2023-04-25T09:07:23.050731Z","shell.execute_reply":"2023-04-25T09:07:23.05562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a = re.search(r\"(.)\\1{1,}\", 'paartyy')\nif a:\n    print('OK')\nelse:\n    print('false')","metadata":{"execution":{"iopub.status.busy":"2023-04-25T09:27:24.419437Z","iopub.execute_input":"2023-04-25T09:27:24.419881Z","iopub.status.idle":"2023-04-25T09:27:24.428167Z","shell.execute_reply.started":"2023-04-25T09:27:24.419841Z","shell.execute_reply":"2023-04-25T09:27:24.426098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reduce_lengthening('paaaaaaartttyyyyy')","metadata":{"execution":{"iopub.status.busy":"2023-04-25T09:07:15.504428Z","iopub.execute_input":"2023-04-25T09:07:15.50487Z","iopub.status.idle":"2023-04-25T09:07:15.514131Z","shell.execute_reply.started":"2023-04-25T09:07:15.504833Z","shell.execute_reply":"2023-04-25T09:07:15.512733Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Pre-process\nfrom nltk.corpus import stopwords\nstop_words = stopwords.words('english')\nfrom nltk.stem import WordNetLemmatizer\nlemma = WordNetLemmatizer()\n\ndef pre_process(text):\n    # tokenize\n    tokenized = nltk.word_tokenize(text)\n    # stop words\n    stopwds = [word for word in tokenized if word not in stop_words]\n    # lower case\n    lowcase = [word.lower() for word in stopwds]\n    # remove number\n    rm_numbers = [word for word in lowcase if not any(d.isdigit() for d in word)]\n    # check spell     \n    correct_word = []\n    for word in rm_numbers:\n        check = re.search(r\"(.)\\1{2,}\", word)\n        if check:\n            tmp = reduce_lengthening(word)\n            correct_w = spell.correction(tmp)\n            if correct_w != None:\n                correct_word.append(correct_w)\n        else:\n            correct_word.append(word)\n    # lemma\n    lemmawds = [lemma.lemmatize(word) for word in correct_word if len(word) > 1]\n    # correct word\n\n    return ' '.join(correct_word)","metadata":{"execution":{"iopub.status.busy":"2023-04-25T11:32:46.458862Z","iopub.execute_input":"2023-04-25T11:32:46.459307Z","iopub.status.idle":"2023-04-25T11:32:46.471836Z","shell.execute_reply.started":"2023-04-25T11:32:46.459268Z","shell.execute_reply":"2023-04-25T11:32:46.470398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clean_text = raw_train['posts'].apply(lambda x: clean_text(x))","metadata":{"execution":{"iopub.status.busy":"2023-04-25T11:31:06.752255Z","iopub.execute_input":"2023-04-25T11:31:06.752751Z","iopub.status.idle":"2023-04-25T11:31:13.485811Z","shell.execute_reply.started":"2023-04-25T11:31:06.752712Z","shell.execute_reply":"2023-04-25T11:31:13.484553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"raw_train['clean'] = clean_text.apply(lambda x: pre_process(x))","metadata":{"execution":{"iopub.status.busy":"2023-04-25T11:00:07.221671Z","iopub.execute_input":"2023-04-25T11:00:07.222087Z","iopub.status.idle":"2023-04-25T11:05:22.785786Z","shell.execute_reply.started":"2023-04-25T11:00:07.222051Z","shell.execute_reply":"2023-04-25T11:05:22.783711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"raw_train.to_csv('train_cleaned2.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-04-25T11:18:28.127616Z","iopub.execute_input":"2023-04-25T11:18:28.128108Z","iopub.status.idle":"2023-04-25T11:18:29.710827Z","shell.execute_reply.started":"2023-04-25T11:18:28.128056Z","shell.execute_reply":"2023-04-25T11:18:29.709406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"raw_test = pd.read_csv(\"/kaggle/input/ai4vozer-k1a1-supervised-classification-model/raw_test.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-04-25T11:24:34.168316Z","iopub.execute_input":"2023-04-25T11:24:34.168771Z","iopub.status.idle":"2023-04-25T11:24:34.650386Z","shell.execute_reply.started":"2023-04-25T11:24:34.168736Z","shell.execute_reply":"2023-04-25T11:24:34.649089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"raw_test.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-25T11:24:52.216062Z","iopub.execute_input":"2023-04-25T11:24:52.216674Z","iopub.status.idle":"2023-04-25T11:24:52.229741Z","shell.execute_reply.started":"2023-04-25T11:24:52.216626Z","shell.execute_reply":"2023-04-25T11:24:52.228216Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"raw_test_clean = raw_test['posts'].apply(lambda x: clean_text(x))","metadata":{"execution":{"iopub.status.busy":"2023-04-25T11:32:26.652758Z","iopub.execute_input":"2023-04-25T11:32:26.653169Z","iopub.status.idle":"2023-04-25T11:32:28.875845Z","shell.execute_reply.started":"2023-04-25T11:32:26.653134Z","shell.execute_reply":"2023-04-25T11:32:28.874471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"raw_test_clean","metadata":{"execution":{"iopub.status.busy":"2023-04-25T11:32:33.37782Z","iopub.execute_input":"2023-04-25T11:32:33.378246Z","iopub.status.idle":"2023-04-25T11:32:33.388932Z","shell.execute_reply.started":"2023-04-25T11:32:33.378211Z","shell.execute_reply":"2023-04-25T11:32:33.387513Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"raw_test['clean'] = raw_test_clean.apply(lambda x: pre_process(x))","metadata":{"execution":{"iopub.status.busy":"2023-04-25T11:33:40.051017Z","iopub.execute_input":"2023-04-25T11:33:40.051467Z","iopub.status.idle":"2023-04-25T11:35:27.908637Z","shell.execute_reply.started":"2023-04-25T11:33:40.05143Z","shell.execute_reply":"2023-04-25T11:35:27.906884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"raw_test.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-25T11:36:52.104274Z","iopub.execute_input":"2023-04-25T11:36:52.104734Z","iopub.status.idle":"2023-04-25T11:36:52.122815Z","shell.execute_reply.started":"2023-04-25T11:36:52.104694Z","shell.execute_reply":"2023-04-25T11:36:52.121299Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"raw_test.to_csv(\"test_cleaned2.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-04-25T11:37:40.512101Z","iopub.execute_input":"2023-04-25T11:37:40.512549Z","iopub.status.idle":"2023-04-25T11:37:41.049625Z","shell.execute_reply.started":"2023-04-25T11:37:40.512513Z","shell.execute_reply":"2023-04-25T11:37:41.04856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import spacy\nnlp = spacy.load(\"en_core_web_sm\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install fasttext","metadata":{"execution":{"iopub.status.busy":"2023-04-23T04:19:31.903638Z","iopub.execute_input":"2023-04-23T04:19:31.904866Z","iopub.status.idle":"2023-04-23T04:19:43.853881Z","shell.execute_reply.started":"2023-04-23T04:19:31.904817Z","shell.execute_reply":"2023-04-23T04:19:43.852378Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import fasttext\n\n# # Skipgram model :\n# model = fasttext.train_unsupervised('data.txt', model='skipgram')\n\n# or, cbow model :\nmodel = fasttext.train_unsupervised('data.txt', model='cbow')","metadata":{"execution":{"iopub.status.busy":"2023-04-23T11:13:27.239997Z","iopub.execute_input":"2023-04-23T11:13:27.240547Z","iopub.status.idle":"2023-04-23T11:13:27.474232Z","shell.execute_reply.started":"2023-04-23T11:13:27.240506Z","shell.execute_reply":"2023-04-23T11:13:27.472421Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from nltk.corpus import wordnet\n\nsentence = \"I have a party woo in the brary when i leave bookstores\"\nnew_sentence = replace_synonyms(sentence)","metadata":{"execution":{"iopub.status.busy":"2023-04-27T04:52:44.099602Z","iopub.execute_input":"2023-04-27T04:52:44.100065Z","iopub.status.idle":"2023-04-27T04:52:44.106877Z","shell.execute_reply.started":"2023-04-27T04:52:44.100022Z","shell.execute_reply":"2023-04-27T04:52:44.105091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_sentence","metadata":{"execution":{"iopub.status.busy":"2023-04-27T04:52:47.075613Z","iopub.execute_input":"2023-04-27T04:52:47.076063Z","iopub.status.idle":"2023-04-27T04:52:47.083707Z","shell.execute_reply.started":"2023-04-27T04:52:47.076018Z","shell.execute_reply":"2023-04-27T04:52:47.082356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\nfrom nltk.corpus import wordnet\n\ndef replace_synonyms(sentence):\n    words = sentence.split()\n    new_sentence = []\n    for word in words:\n        synonyms = []\n        for syn in wordnet.synsets(word):\n            for lemma in syn.lemmas():\n                synonyms.append(lemma.name())\n        if synonyms:\n            new_word = random.choice(synonyms)\n            new_sentence.append(new_word)\n        else:\n            new_sentence.append(word)\n    return ' '.join(new_sentence)","metadata":{"execution":{"iopub.status.busy":"2023-04-27T04:51:58.625533Z","iopub.execute_input":"2023-04-27T04:51:58.627237Z","iopub.status.idle":"2023-04-27T04:51:58.636676Z","shell.execute_reply.started":"2023-04-27T04:51:58.627176Z","shell.execute_reply":"2023-04-27T04:51:58.635157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clean_train = pd.read_csv(\"/kaggle/input/another-data-2/train_cleaned2.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-04-27T08:05:54.144772Z","iopub.execute_input":"2023-04-27T08:05:54.145997Z","iopub.status.idle":"2023-04-27T08:05:55.248298Z","shell.execute_reply.started":"2023-04-27T08:05:54.145931Z","shell.execute_reply":"2023-04-27T08:05:55.246931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clean_train.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-27T07:36:28.714183Z","iopub.execute_input":"2023-04-27T07:36:28.71557Z","iopub.status.idle":"2023-04-27T07:36:28.746197Z","shell.execute_reply.started":"2023-04-27T07:36:28.715513Z","shell.execute_reply":"2023-04-27T07:36:28.744803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## turn data into a countvectorizer\n# from sklearn.naive_bayes import MultinomialNB\n# from sklearn.feature_extraction.text import CountVectorizer\n# count = CountVectorizer(binary=False,max_features=4000)\nfrom sklearn.feature_extraction.text import TfidfVectorizer\ntfidf = TfidfVectorizer(max_features=15000)\nX_train = clean_train['clean']\n# X_train_counted = train['cleaned'].apply(lambda x: nlp(x).vector)\n# X_train_counted = count.fit_transform(X_train.values.astype('U')).toarray()\nX_train_counted = tfidf.fit_transform(X_train.values.astype('U')).toarray()\n","metadata":{"execution":{"iopub.status.busy":"2023-04-27T08:06:09.650197Z","iopub.execute_input":"2023-04-27T08:06:09.651443Z","iopub.status.idle":"2023-04-27T08:06:15.509096Z","shell.execute_reply.started":"2023-04-27T08:06:09.651351Z","shell.execute_reply":"2023-04-27T08:06:15.507942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_trained = np.stack(X_train_counted)","metadata":{"execution":{"iopub.status.busy":"2023-04-16T07:12:25.296593Z","iopub.execute_input":"2023-04-16T07:12:25.297034Z","iopub.status.idle":"2023-04-16T07:12:25.312814Z","shell.execute_reply.started":"2023-04-16T07:12:25.296994Z","shell.execute_reply":"2023-04-16T07:12:25.311659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_counted","metadata":{"execution":{"iopub.status.busy":"2023-04-27T07:43:45.321007Z","iopub.execute_input":"2023-04-27T07:43:45.321439Z","iopub.status.idle":"2023-04-27T07:43:45.330243Z","shell.execute_reply.started":"2023-04-27T07:43:45.3214Z","shell.execute_reply":"2023-04-27T07:43:45.328981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn import preprocessing\nle = preprocessing.LabelEncoder()","metadata":{"execution":{"iopub.status.busy":"2023-04-27T07:44:09.99833Z","iopub.execute_input":"2023-04-27T07:44:09.998792Z","iopub.status.idle":"2023-04-27T07:44:10.00406Z","shell.execute_reply.started":"2023-04-27T07:44:09.998749Z","shell.execute_reply":"2023-04-27T07:44:10.002975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## fit data to model\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.discriminant_analysis import LinearDiscriminantAnalysis\ny_train = le.fit(clean_train['type'])\nprint(y_train.classes_)\ny_train = le.transform(clean_train['type'])\n# mnb = MultinomialNB(alpha = 1, fit_prior = False, class_prior = None)\n# mnb.fit(X_train_counted,y_train)\n# model = LinearDiscriminantAnalysis()\n# model.fit(X_train_counted, y_train)\nmodel = LogisticRegression(solver='liblinear',C=10.0, random_state=0)\nmodel.fit(X_train_counted, y_train)","metadata":{"execution":{"iopub.status.busy":"2023-04-27T08:06:52.963932Z","iopub.execute_input":"2023-04-27T08:06:52.964431Z","iopub.status.idle":"2023-04-27T08:07:22.818223Z","shell.execute_reply.started":"2023-04-27T08:06:52.96439Z","shell.execute_reply":"2023-04-27T08:07:22.816485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## convert cleaned_test to data of countvectorizer\ntest = pd.read_csv(\"/kaggle/input/ai4vozer-k1a1-supervised-classification-model/clean_test.csv\")\nX_test = test['cleaned']\n# X_test_counted = tfidf.transform(X_test).toarray()\nX_test_counted = test['cleaned'].apply(lambda x: nlp(x).vector)\nX_tested = np.stack(X_test_counted)","metadata":{"execution":{"iopub.status.busy":"2023-04-16T07:17:10.714393Z","iopub.execute_input":"2023-04-16T07:17:10.714807Z","iopub.status.idle":"2023-04-16T07:21:02.457478Z","shell.execute_reply.started":"2023-04-16T07:17:10.71477Z","shell.execute_reply":"2023-04-16T07:21:02.45627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## convert cleaned_test to data of countvectorizer\ntest = pd.read_csv(\"/kaggle/input/another-data-2/test_cleaned2.csv\")\nX_test = test['clean']\nX_test_counted = tfidf.transform(X_test.values.astype('U')).toarray()","metadata":{"execution":{"iopub.status.busy":"2023-04-27T08:07:39.0933Z","iopub.execute_input":"2023-04-27T08:07:39.093813Z","iopub.status.idle":"2023-04-27T08:07:41.21084Z","shell.execute_reply.started":"2023-04-27T08:07:39.093775Z","shell.execute_reply":"2023-04-27T08:07:41.20949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test","metadata":{"execution":{"iopub.status.busy":"2023-04-27T07:58:18.678464Z","iopub.execute_input":"2023-04-27T07:58:18.678865Z","iopub.status.idle":"2023-04-27T07:58:18.699911Z","shell.execute_reply.started":"2023-04-27T07:58:18.678832Z","shell.execute_reply":"2023-04-27T07:58:18.698475Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test_counted","metadata":{"execution":{"iopub.status.busy":"2023-04-27T08:07:56.057077Z","iopub.execute_input":"2023-04-27T08:07:56.057556Z","iopub.status.idle":"2023-04-27T08:07:56.066419Z","shell.execute_reply.started":"2023-04-27T08:07:56.057514Z","shell.execute_reply":"2023-04-27T08:07:56.064946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## make prediction\npred = model.predict(X_test_counted)","metadata":{"execution":{"iopub.status.busy":"2023-04-27T08:08:03.1266Z","iopub.execute_input":"2023-04-27T08:08:03.127633Z","iopub.status.idle":"2023-04-27T08:08:03.237524Z","shell.execute_reply.started":"2023-04-27T08:08:03.127591Z","shell.execute_reply":"2023-04-27T08:08:03.235665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred","metadata":{"execution":{"iopub.status.busy":"2023-04-27T08:08:06.696807Z","iopub.execute_input":"2023-04-27T08:08:06.697249Z","iopub.status.idle":"2023-04-27T08:08:06.70571Z","shell.execute_reply.started":"2023-04-27T08:08:06.697205Z","shell.execute_reply":"2023-04-27T08:08:06.704573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# form a csv file that has 2 cols: Id and Category\n## first inverse transform the prediction\npred_cat = le.inverse_transform(pred)\npred = pd.DataFrame({\"Category\":pred_cat})\npred['Id'] = test['ID']","metadata":{"execution":{"iopub.status.busy":"2023-04-27T08:08:26.521423Z","iopub.execute_input":"2023-04-27T08:08:26.52228Z","iopub.status.idle":"2023-04-27T08:08:26.530964Z","shell.execute_reply.started":"2023-04-27T08:08:26.522233Z","shell.execute_reply":"2023-04-27T08:08:26.529523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred.to_csv(\"pred7.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2023-04-27T08:08:37.008408Z","iopub.execute_input":"2023-04-27T08:08:37.008795Z","iopub.status.idle":"2023-04-27T08:08:37.017921Z","shell.execute_reply.started":"2023-04-27T08:08:37.008763Z","shell.execute_reply":"2023-04-27T08:08:37.016986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred.head(4)","metadata":{"execution":{"iopub.status.busy":"2023-04-27T08:08:30.369812Z","iopub.execute_input":"2023-04-27T08:08:30.370204Z","iopub.status.idle":"2023-04-27T08:08:30.381808Z","shell.execute_reply.started":"2023-04-27T08:08:30.370171Z","shell.execute_reply":"2023-04-27T08:08:30.38045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}