{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nfrom tqdm.autonotebook import tqdm\nimport os\nimport matplotlib.pyplot as plt\nimport json\nimport nltk\nimport re\nimport numpy as np\nfrom nltk.tokenize import word_tokenize\nfrom nltk.corpus import stopwords\nfrom nltk.stem import WordNetLemmatizer\nfrom collections import OrderedDict, Counter\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import train_test_split \nimport seaborn as sns\nimport warnings\nwarnings.filterwarnings('ignore')\n# nltk.download('stopwords')\n# nltk.download('punkt')\n# nltk.download('wordnet')","metadata":{"_uuid":"8f3d6b87-f693-46e1-ace6-528540b5f666","_cell_guid":"db41f025-8fcf-4487-b0ac-ee9f00b579f0","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\nsample_sub = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\ntrain_files_path = '../input/coleridgeinitiative-show-us-the-data/train'\ntest_files_path = '../input/coleridgeinitiative-show-us-the-data/test'","metadata":{"_uuid":"674ba341-40a8-49a5-aceb-7774f9d6724c","_cell_guid":"96ece948-a9f6-4c02-9488-e2b893446f22","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df.sample(frac=1).reset_index(drop=True)","metadata":{"_uuid":"5f27518b-1e40-495c-8644-53911300e317","_cell_guid":"affec813-edb1-4339-a001-93166a8f26da","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def Concatenate(filename, files_path=train_files_path, output='text'):\n    json_path = os.path.join(train_files_path, (filename + '.json'))\n    headings = []\n    contents = []\n    combined = []\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title'))\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n\n    all_headings = ' '.join(headings)\n    all_contents = ' '.join(contents)\n    all_data = '. '.join(combined)\n\n    if output == 'text':\n        return all_contents\n    elif output == 'head':\n        return all_headings\n    else:\n        return all_data\ntqdm.pandas()\ntrain_df['text'] = train_df['Id'].progress_apply(Concatenate)\ntqdm.pandas()\nsample_sub['text'] = sample_sub['Id'].progress_apply(Concatenate)","metadata":{"_uuid":"5effedde-d445-4ece-809d-b8ac7d7026f9","_cell_guid":"aa7f7ae1-54c7-4fa1-8966-7d2acfc140ae","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head(3)","metadata":{"_uuid":"5224b693-59e3-4f01-90cd-bd3afbb8651b","_cell_guid":"a5cb785e-e871-4422-8c28-f2b7fc9a7f7d","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df.drop(columns = ['pub_title','dataset_title', 'dataset_label','Id'], inplace = True)","metadata":{"_uuid":"7f2d7f93-3e2e-463e-9992-bd70f794f41d","_cell_guid":"4e2e57e2-5986-4054-9d4e-e70ac8d30894","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_df['cleaned_label'].value_counts())","metadata":{"_uuid":"27254d1b-3555-43c8-a09a-6506d29fa8d6","_cell_guid":"c51aeec7-b13b-4429-9f01-280bb370741f","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.isna().sum()","metadata":{"_uuid":"a437fe4d-723a-4d05-a83b-a7c30e95b1d1","_cell_guid":"25745713-4687-48ae-af2c-2d741a6b106b","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['dataset_title'].value_counts().plot(kind='bar', color='red', figsize = (24,20), fontsize = 10)\nplt.xlabel('labels')\nplt.ylabel('Total label count')\nplt.title('label count of each label type')","metadata":{"_uuid":"c5d63a9c-1f0c-4e65-a063-44cf70145c9a","_cell_guid":"06ee7460-999f-4d86-859f-1c6fd6264fc0","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def cleaning_text(text): \n    text = re.sub(\"[^a-zA-Z]\",\" \",text) \n    text = re.sub('[!@#$_]', '', text)\n    text = text.replace(\"co\",\"\")\n    text = text.replace(\"http\",\"\")\n    text = ' '.join(text.split()) \n    text = text.lower() \n    return text\n\ntrain_df['text'] = train_df['text'].progress_apply(lambda x: cleaning_text(x))\nsample_sub['text'] = sample_sub['text'].progress_apply(lambda x: cleaning_text(x))","metadata":{"_uuid":"75186668-6484-4fa8-8b3c-1bb9432f0a0a","_cell_guid":"b16c81c1-96eb-4ab7-9aee-3d869ffe7471","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df = train_df.copy()","metadata":{"_uuid":"3608c730-db20-4c93-b2cd-4b7f2fcfd546","_cell_guid":"2f54be1d-8627-4b47-aa41-688ebf5441a6","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tokenize(text):\n    token_words= word_tokenize(str(text))\n    return \" \".join(token_words)\ntrain_df['text'] = train_df['text'].progress_apply(lambda x: tokenize(x))\nsample_sub['text'] = sample_sub['text'].progress_apply(lambda x: tokenize(x))","metadata":{"_uuid":"7570cd80-6a91-4189-8010-f51ef3b21736","_cell_guid":"af906ee3-4404-4a33-af37-b8caae67bdc4","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def stopwords_clean(text):\n    stop_words = set(stopwords.words('english'))\n    no_stopword_text = [w for w in str(text).split() if not w in stop_words]\n    return \" \".join(no_stopword_text)\n\ntrain_df['text'] = train_df['text'].progress_apply(lambda x: stopwords_clean(x))\nsample_sub['text'] = sample_sub['text'].progress_apply(lambda x: stopwords_clean(x))","metadata":{"_uuid":"92d96b34-502d-4349-a42f-9d34eec0932f","_cell_guid":"bf121504-25c1-4d8d-a44e-6842b978b621","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import nltk\n# from nltk.stem import WordNetLemmatizer\n# lemma= WordNetLemmatizer() \n\n# def lemmatize_text(text):\n#     lemma_text = [lemma.lemmatize(word) for word in text]\n#     return \"\".join(lemma_text)\n\n# train_df['texts_cleaned_lemmatized'] = train_df['text_cleaned_nostop'].progress_apply(lambda x: lemmatize_text(x))\n# sample_sub['texts_cleaned_lemmatized'] = sample_sub['text_cleaned_nostop'].progress_apply(lambda x: lemmatize_text(x))","metadata":{"_uuid":"d8d876b5-6549-416a-a353-43723d10f3fd","_cell_guid":"8b5c9ecb-d7ee-4590-b551-335b1c4da06d","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['text'] = (train_df['text'].str.split().progress_apply(lambda x: OrderedDict.fromkeys(x).keys()).str.join(' '))\nsample_sub['text'] = (sample_sub['text'].str.split().progress_apply(lambda x: OrderedDict.fromkeys(x).keys()).str.join(' '))","metadata":{"_uuid":"9e4d745b-f61d-4e56-8759-397642b354ce","_cell_guid":"b8b9fb33-f55a-4765-b7a9-69e7d1581c4f","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head(3)","metadata":{"_uuid":"616e19fd-7665-434e-af22-222e7636331d","_cell_guid":"509fa9c3-8003-4119-94e8-71cd8b3ab92e","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Train_df = train_df.copy() #ML\n# df = train_df.copy()       #BERT","metadata":{"_uuid":"e9e7d78e-fea0-422a-92c7-a4f7275420e4","_cell_guid":"5e6d8321-d3ca-40a0-897e-4c708c199c94","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Train_df.info()","metadata":{"_uuid":"e3725a6f-f3ad-438b-9d6f-b5a546ac413a","_cell_guid":"3ade3ab0-fcf8-4745-982f-2aa5e207373b","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from gensim.models.doc2vec import Doc2Vec, TaggedDocument\ntext_docs = [TaggedDocument(doc.split(' '), [i]) \n             for i, doc in enumerate(Train_df.text)]\nmodel = Doc2Vec(vector_size=128, min_count=3, epochs = 30)\n#instantiate model\nmodel = Doc2Vec(vector_size=128, window=2, min_count=3, workers=8, epochs = 50)\n#build vocab\nmodel.build_vocab(text_docs)\n#train model\nmodel.train(text_docs, total_examples=model.corpus_count\n            , epochs=model.epochs)","metadata":{"_uuid":"cba6d966-8d2b-480d-898c-0856c0f41191","_cell_guid":"ff12a7f5-81cc-41dd-8631-92366c2193e9","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"text2vec = [model.infer_vector((Train_df['text'][i].split(' '))) \n            for i in range(0,len(Train_df['text']))]","metadata":{"_uuid":"5d178793-e543-47e8-9bba-fc78953507d2","_cell_guid":"c585b1ae-cca2-4f16-9309-4aa188d82d15","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_text2vec = [model.infer_vector((sample_sub['text'][i].split(' '))) \n            for i in range(0,len(sample_sub['text']))]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dtv= np.array(text2vec).tolist()\nTrain_df['text2vec'] = dtv\nTrain_df.head(2)","metadata":{"_uuid":"2d7b588a-dcf3-435b-95ca-2286256be4d5","_cell_guid":"0747caff-7cc2-40a4-99b4-2ca0e14ba181","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ttv= np.array(test_text2vec).tolist()\nsample_sub['text2vec'] = ttv\nsample_sub.head(2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sample_sub.drop(columns = ['text','cleaned_text', 'cleaned_text_tokenized'], inplace = True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sparce_matrix = text2vec\ny = Train_df.iloc[:,2].values \nX = sparce_matrix","metadata":{"_uuid":"0a592def-7459-44c2-8b0e-d4fb3b3bcbf7","_cell_guid":"433225fb-d382-4faa-bd58-0a3bb2068142","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoder = LabelEncoder()\ny = encoder.fit_transform(y)\ny[:10]\n# X[:5]","metadata":{"_uuid":"3e9c7235-e6be-40a8-91c7-c317d4de61d0","_cell_guid":"960e00a9-d7d2-471b-8f51-0349273c1fd9","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"decoded = encoder.inverse_transform(y)\ndecoded[:10]","metadata":{"_uuid":"0c963b34-29b0-400e-9c2f-f1c21b5eb368","_cell_guid":"597531ad-9acc-49eb-b2f7-c61efe7e9590","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.15, random_state=45)","metadata":{"_uuid":"f7e51254-6310-4821-b717-b8a7ee64727e","_cell_guid":"5cfc270a-7f5e-44f1-9acb-e6b776314a28","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"*Note - Accuracy is a bad metric of evaluation for the models as the classes are heavily imbalanced*","metadata":{"_uuid":"b6c8464b-1f4a-497a-aaf7-feba1da6f7c6","_cell_guid":"6eb4413e-19b0-42a5-9d3b-9c899638d776","trusted":true}},{"cell_type":"markdown","source":"**Logistic Regression Classifier**","metadata":{"_uuid":"15e0a978-0dad-4316-ab64-36f8f78e3c53","_cell_guid":"1d14ea8c-016e-4c59-b5e4-1015239b52f0","trusted":true}},{"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import confusion_matrix, accuracy_score, classification_report\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import StandardScaler\nsc = StandardScaler()\nlogistic = LogisticRegression(penalty = 'l1', solver='liblinear', C= 0.1, random_state = 45)\npipeline = Pipeline(steps=[('sc', sc),('logistic', logistic)])\npipeline.fit(X_train, y_train)","metadata":{"_uuid":"470771c6-f07b-41ce-ad0d-107f774d7ccd","_cell_guid":"3b27b48c-9352-4ebd-816a-3ad66a51c6b9","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Accuracy","metadata":{"_uuid":"874e6797-feca-427c-9edf-adb040f30174","_cell_guid":"6e350198-003d-4b15-8642-1b3c9b7acc54","trusted":true}},{"cell_type":"code","source":"y_pred = pipeline.predict(X_test)\nprint('Accuracy:',accuracy_score(y_test,y_pred))\n# print('Confusion matrix:\\n',confusion_matrix(y_test,y_pred))\n# print('Classification report:\\n',classification_report(y_test,y_pred))","metadata":{"_uuid":"24921842-9c80-4e2a-98e5-d10739145dd0","_cell_guid":"2861192c-2460-400a-8055-b2fd4351c9a9","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predicts = []\nfor i in range (4):\n    lol  = np.array(test_text2vec[i]).reshape(1, -1)\n    pred_test =  pipeline.predict(lol)\n    X = encoder.inverse_transform(pred_test)\n    X = X.tolist()[0]\n    predicts.append(X)\npredicts","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_1 = [x.lower() for x in Train_df['dataset_label'].unique()]\ntemp_2 = [x.lower() for x in Train_df['dataset_title'].unique()]\ntemp_3 = [x.lower() for x in Train_df['cleaned_label'].unique()]\n\nexisting_labels = set(temp_1 + temp_2 + temp_3)\nid_list = []\nlables_list = []\nfor index, row in tqdm(sample_sub.iterrows()):\n    sample_text = row['text']\n    row_id = row['Id']\n    temp_df = Train_df[Train_df['text'] == clean_text(sample_text)]\n    cleaned_labels = temp_df['cleaned_label'].to_list()\n    for known_label in existing_labels:\n        if known_label in sample_text.lower():\n            cleaned_labels.append(clean_text(known_label))\n    cleaned_labels = [clean_text(x) for x in cleaned_labels]\n    cleaned_labels = set(cleaned_labels)\n    lables_list.append('|'.join(cleaned_labels))\n    id_list.append(row_id)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lables_list","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_subf = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_subf.PredictionString = lables_list","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_subf['PredictionString'] = sample_subf['PredictionString'].progress_apply(lambda x: cleaning_text(x))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_subf","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_subf.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**RandomForest Classifier**","metadata":{"_uuid":"eef1b921-3d69-48b4-9541-0ee78991deb3","_cell_guid":"7be34802-4a39-4cb4-b3cd-553ce3efc46c","trusted":true}},{"cell_type":"code","source":"# from sklearn.ensemble import RandomForestClassifier\n# rfc1=RandomForestClassifier(n_estimators= 50, max_depth=15, bootstrap=True, random_state=45)\n# crfc1.fit(X_train, y_train)","metadata":{"_uuid":"c90d737b-a0dd-4b2f-afc4-3f18a673aa2a","_cell_guid":"c6189b6f-975e-416f-a126-2f4a24ea81af","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Accuracy","metadata":{"_uuid":"006f207c-3f9d-4a60-ac08-67a2084b2458","_cell_guid":"af12fb58-5400-4fb0-b3ec-a9f011ff9582","trusted":true}},{"cell_type":"code","source":"# y_pred = rfc1.predict(X_test)\n# print('Accuracy:',accuracy_score(y_test,y_pred))\n# print('Confusion matrix:\\n',confusion_matrix(y_test,y_pred))\n# print('Classification report:\\n',classification_report(y_test,y_pred))","metadata":{"_uuid":"794117a9-2924-419d-8d40-06ba27a8ed4c","_cell_guid":"e748e55a-4eeb-4f7c-91f4-b571de4319d5","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# predicts = []\n# for i in range (4):\n#     lol  = np.array(test_text2vec[i]).reshape(1, -1)\n#     pred_test =  rfc1.predict(lol)\n#     X = encoder.inverse_transform(pred_test)\n#     X = X.tolist()[0]\n#     predicts.append(X)\n# predicts","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Support Vector Machine Classifier**","metadata":{"_uuid":"8e62d5d1-ebcb-414a-8570-fc5d01d43f16","_cell_guid":"bf0bc56d-defb-4feb-8e97-790c0651227b","trusted":true}},{"cell_type":"code","source":"# from sklearn.svm import SVC\n# from sklearn.metrics import confusion_matrix, accuracy_score, classification_report\n# from sklearn.pipeline import Pipeline\n# from sklearn.preprocessing import StandardScaler\n\n# sc = StandardScaler()\n# svm = SVC(C=1,gamma='scale', kernel='linear')\n# pipe = Pipeline(steps=[('sc', sc),\n#                        ('SVM', svm)])\n\n# pipe.fit(X_train, y_train)","metadata":{"_uuid":"a618e64b-cb24-4954-83e2-5473ce152bec","_cell_guid":"87d2c38e-b771-4165-a2d7-cd7235007c4a","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Accuracy","metadata":{"_uuid":"d15f9920-aa09-4b2d-851c-52c7d742d2c9","_cell_guid":"6ef400f6-bb55-4d99-95f1-67a2f874e4d1","trusted":true}},{"cell_type":"code","source":"# y_pred = pipe.predict(X_test)\n# print('Accuracy:',accuracy_score(y_test,y_pred))\n# print('Confusion matrix:\\n',confusion_matrix(y_test,y_pred))\n# print('Classification report:\\n',classification_report(y_test,y_pred))","metadata":{"_uuid":"bedc899e-8285-4c89-b7bb-5cc84df6a0ba","_cell_guid":"9b349565-6dc8-4516-9bb9-790aca6a9af8","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# predicts = []\n# for i in range (4):\n#     lol  = np.array(test_text2vec[i]).reshape(1, -1)\n#     pred_test =  pipe.predict(lol)\n#     X = encoder.inverse_transform(pred_test)\n#     X = X.tolist()[0]\n#     predicts.append(X)\npredicts","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Naive Bayes Classifier**","metadata":{"_uuid":"31ab2d0c-d1dc-4444-8e2c-daafd8875d83","_cell_guid":"2a6f0790-bf78-4d88-9bce-7115c3dd3376","trusted":true}},{"cell_type":"code","source":"# from sklearn.naive_bayes import GaussianNB\n# gnb = GaussianNB()\n# gnb.fit(X_train, y_train)","metadata":{"_uuid":"1b84d420-ca55-4be8-96c8-edb3748b3cdf","_cell_guid":"c3a7ef23-deca-4251-b4b4-2ce9f1ffaa81","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Accuracy","metadata":{"_uuid":"c7435088-2eaa-4553-b933-37b75b22f77a","_cell_guid":"7619499a-6356-41e5-b253-96cc4aaaf6cc","trusted":true}},{"cell_type":"code","source":"# y_pred = gnb.predict(X_test)\n# from sklearn import metrics\n# print(\"Accuracy:\",metrics.accuracy_score(y_test, y_pred))","metadata":{"_uuid":"d4efec47-9142-4144-a326-e6252e1757d7","_cell_guid":"bc75fcb6-923c-4ba3-b16e-26d18ec7de86","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# predicts = []\n# for i in range (4):\n#     lol  = np.array(test_text2vec[i]).reshape(1, -1)\n#     pred_test =  gnb.predict(lol)\n#     X = encoder.inverse_transform(pred_test)\n#     X = X.tolist()[0]\n#     predicts.append(X)\n# predicts","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**BERT**","metadata":{"_uuid":"82821132-0440-429c-902b-04ab1afccbcd","_cell_guid":"4cd1b627-e798-4812-b5ed-3d10ade47938","trusted":true}},{"cell_type":"markdown","source":"Encoding the Labels","metadata":{"_uuid":"b7981305-ca20-41ad-9661-899e6d407635","_cell_guid":"17e39916-b72e-45cf-b973-8b08119e6127","trusted":true}},{"cell_type":"code","source":"# df.head(5)","metadata":{"_uuid":"9d96501a-3b78-4b67-8efc-efbc6a0ea499","_cell_guid":"f37468f0-d79e-42b7-9728-54fca04f080b","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# possible_labels = df.cleaned_label.unique()\n\n# label_dict = {}\n# for index, possible_label in enumerate(possible_labels):\n#     label_dict[possible_label] = index\n# label_dict","metadata":{"_uuid":"4785a04e-8288-4ea1-b549-7ff76e3af70a","_cell_guid":"6207f508-c918-4701-a5d6-d4595d94b6ed","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df['label'] = df.cleaned_label.replace(label_dict)","metadata":{"_uuid":"b5370bdc-dcc4-49a5-81d5-1df97cb8baaa","_cell_guid":"29de7764-9ee5-43b6-94a4-ac4e71969c1d","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Train and Validation Split","metadata":{"_uuid":"1fe28807-7615-47d7-b87e-246ac6a53e2c","_cell_guid":"0f435a1f-d748-4c10-8bdb-3bd028e7f26d","trusted":true}},{"cell_type":"code","source":"# from sklearn.model_selection import train_test_split\n\n# X_train, X_val, y_train, y_val = train_test_split(df.index.values, \n#                                                   df.label.values, \n#                                                   test_size=0.15, \n#                                                   random_state=42)\n\n# df['data_type'] = ['not_set']*df.shape[0]\n\n# df.loc[X_train, 'data_type'] = 'train'\n# df.loc[X_val, 'data_type'] = 'val'\n\n# df.groupby(['cleaned_label', 'label', 'data_type']).count()","metadata":{"_uuid":"c697dda7-3a9f-48ba-bc3f-79e400cc09d3","_cell_guid":"6339a093-588d-4ab9-895e-421016c6d18d","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"BertTokenizer and Encoding the Data","metadata":{"_uuid":"1a2f935a-38c2-4364-ac9c-c910064dd7b2","_cell_guid":"7def4b18-a8be-4a0d-9967-257b80aa64cd","trusted":true}},{"cell_type":"code","source":"# from tokenizers import Tokenizer\n# from tokenizers.models import BPE\n# from tokenizers.trainers import BpeTrainer\n# from tokenizers.pre_tokenizers import Whitespace","metadata":{"_uuid":"febf656c-e6f5-4362-8765-ab536bc2c5a0","_cell_guid":"4c370b2d-6a9a-4f9a-87d3-1825188d83e1","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install transformers\n# !pip install torchvision \n# import transformers\n# import torch\n# import torchvision\n# from torch.utils.data import TensorDataset \n# from transformers import BertForSequenceClassification","metadata":{"_uuid":"42c44a08-d79a-4087-8501-08ccec342ae1","_cell_guid":"4999d6a0-ec70-41fa-be7b-98cfc82143ec","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# tokenizer = transformers.BertTokenizer.from_pretrained('bert-base-uncased', \n#                                           do_lower_case=True)\n                                          \n# encoded_data_train = tokenizer.batch_encode_plus(\n#     df[df.data_type=='train'].cleaned_text.values, \n#     add_special_tokens=True, \n#     return_attention_mask=True, \n#     pad_to_max_length=True, \n#     max_length=256, \n#     return_tensors='pt'\n# )\n\n# encoded_data_val = tokenizer.batch_encode_plus(\n#     df[df.data_type=='val'].cleaned_text.values, \n#     add_special_tokens=True, \n#     return_attention_mask=True, \n#     pad_to_max_length=True, \n#     max_length=256, \n#     return_tensors='pt'\n# )\n\n\n# input_ids_train = encoded_data_train['input_ids']\n# attention_masks_train = encoded_data_train['attention_mask']\n# labels_train = torch.tensor(df[df.data_type=='train'].label.values)\n\n# input_ids_val = encoded_data_val['input_ids']\n# attention_masks_val = encoded_data_val['attention_mask']\n# labels_val = torch.tensor(df[df.data_type=='val'].label.values)\n\n# dataset_train = TensorDataset(input_ids_train, attention_masks_train, labels_train)\n# dataset_val = TensorDataset(input_ids_val, attention_masks_val, labels_val)","metadata":{"_uuid":"d49c8ef1-95d9-4945-a2cd-7efa7a216fb1","_cell_guid":"1613c826-de50-4b69-9988-263a8bf7d44c","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# len(dataset_train), len(dataset_val)","metadata":{"_uuid":"a79de2a7-ba98-4b70-b8f9-bfc34f867709","_cell_guid":"2ab5200e-d66f-46bd-9fca-66479c9ef310","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"BERT Pre-trained Model","metadata":{"_uuid":"4c7b0f20-28d6-472a-9fd0-0c0e0819867f","_cell_guid":"7ed402b0-1594-4c1d-a5b3-e0be3a54f8fa","trusted":true}},{"cell_type":"code","source":"# model = BertForSequenceClassification.from_pretrained(\"bert-base-uncased\",\n#                                                       num_labels=len(label_dict),\n#                                                       output_attentions=False,\n#                                                       output_hidden_states=False)","metadata":{"_uuid":"41a83c42-2aea-4d07-9432-1c720b1b0971","_cell_guid":"4d7a099c-90f6-48c1-87ee-c50c89c41728","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Data Loaders","metadata":{"_uuid":"229c8dd7-4757-4b0d-9800-8edae30b6b3c","_cell_guid":"d3749ab7-4f49-415e-9d6a-3d610cc597ce","trusted":true}},{"cell_type":"code","source":"# from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n\n# batch_size = 3\n\n# dataloader_train = DataLoader(dataset_train, \n#                               sampler=RandomSampler(dataset_train), \n#                               batch_size=batch_size)\n\n# dataloader_validation = DataLoader(dataset_val, \n#                                    sampler=SequentialSampler(dataset_val), \n#                                    batch_size=batch_size)","metadata":{"_uuid":"e42a8760-6451-45e0-b5d1-c8f63059cd96","_cell_guid":"095497ac-76b0-4caf-a9b9-9862a3606470","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Optimizer & Scheduler","metadata":{"_uuid":"2840dbd4-b152-408c-a556-52abc5bcb3fd","_cell_guid":"9fcd3855-5e72-475d-8ba3-1df447cd7dea","trusted":true}},{"cell_type":"code","source":"# from transformers import AdamW, get_linear_schedule_with_warmup\n\n# optimizer = AdamW(model.parameters(),\n#                   lr=1e-5, \n#                   eps=1e-8)\n                  \n# epochs = 5\n\n# scheduler = get_linear_schedule_with_warmup(optimizer, \n#                                             num_warmup_steps=0,\n#                                             num_training_steps=len(dataloader_train)*epochs)","metadata":{"_uuid":"37c63ce0-978c-4553-b731-57caee984ae9","_cell_guid":"77666fbb-78fd-4411-8633-f32576825ed7","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Performance Metrics","metadata":{"_uuid":"c0504090-cbb9-473f-bcb6-9dc412495486","_cell_guid":"dc6e7b8f-622b-4464-a83a-365d73ddd561","trusted":true}},{"cell_type":"code","source":"# from sklearn.metrics import f1_score\n\n# def f1_score_func(preds, labels):\n#     preds_flat = np.argmax(preds, axis=1).flatten()\n#     labels_flat = labels.flatten()\n#     return f1_score(labels_flat, preds_flat, average='weighted')\n\n# def accuracy_per_class(preds, labels):\n#     label_dict_inverse = {v: k for k, v in label_dict.items()}\n    \n#     preds_flat = np.argmax(preds, axis=1).flatten()\n#     labels_flat = labels.flatten()\n\n#     for label in np.unique(labels_flat):\n#         y_preds = preds_flat[labels_flat==label]\n#         y_true = labels_flat[labels_flat==label]\n#         print(f'Class: {label_dict_inverse[label]}')\n#         print(f'Accuracy: {len(y_preds[y_preds==label])}/{len(y_true)}\\n')","metadata":{"_uuid":"322bfd3c-ba9a-4073-a1b2-c6c9dd8172d5","_cell_guid":"662a1be9-8942-4b25-bb16-f696558270de","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Training Loop","metadata":{"_uuid":"55b75875-87c0-46d8-bd3e-4074fa974462","_cell_guid":"75e03af5-6771-4c58-8ce1-a506894b0137","trusted":true}},{"cell_type":"code","source":"# import random\n\n# seed_val = 17\n# random.seed(seed_val)\n# np.random.seed(seed_val)\n# torch.manual_seed(seed_val)\n# torch.cuda.manual_seed_all(seed_val)","metadata":{"_uuid":"966e9cad-ae18-4bcd-bba6-003507972fe2","_cell_guid":"00acedb0-e0ae-4bae-a4ad-67a2fdd4a3f3","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n# model.to(device)\n\n# print(device)","metadata":{"_uuid":"53c1907c-2908-4da6-a46e-a2a91c04644a","_cell_guid":"beb67d4f-cf57-43a5-b058-37e8802180ec","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def evaluate(dataloader_val):\n\n#     model.eval()\n    \n#     loss_val_total = 0\n#     predictions, true_vals = [], []\n    \n#     for batch in dataloader_val:\n        \n#         batch = tuple(b.to(device) for b in batch)\n        \n#         inputs = {'input_ids':      batch[0],\n#                   'attention_mask': batch[1],\n#                   'labels':         batch[2],\n#                  }\n\n#         with torch.no_grad():        \n#             outputs = model(**inputs)\n            \n#         loss = outputs[0]\n#         logits = outputs[1]\n#         loss_val_total += loss.item()\n\n#         logits = logits.detach().cpu().numpy()\n#         label_ids = inputs['labels'].cpu().numpy()\n#         predictions.append(logits)\n#         true_vals.append(label_ids)\n    \n#     loss_val_avg = loss_val_total/len(dataloader_val) \n    \n#     predictions = np.concatenate(predictions, axis=0)\n#     true_vals = np.concatenate(true_vals, axis=0)\n            \n#     return loss_val_avg, predictions, true_vals","metadata":{"_uuid":"fbd3a239-8c69-4829-9a3c-bed24964a0c9","_cell_guid":"ccf69fdb-6f7b-45c2-99d1-8a6ac7ea8b48","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for epoch in tqdm(range(1, epochs+1)):\n    \n#     model.train()\n    \n#     loss_train_total = 0\n\n#     progress_bar = tqdm(dataloader_train, desc='Epoch {:1d}'.format(epoch), leave=False, disable=False)\n#     for batch in progress_bar:\n\n#         model.zero_grad()\n        \n#         batch = tuple(b.to(device) for b in batch)\n        \n#         inputs = {'input_ids':      batch[0],\n#                   'attention_mask': batch[1],\n#                   'labels':         batch[2],\n#                  }       \n\n#         outputs = model(**inputs)\n        \n#         loss = outputs[0]\n#         loss_train_total += loss.item()\n#         loss.backward()\n\n#         torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n\n#         optimizer.step()\n#         scheduler.step()\n        \n#         progress_bar.set_postfix({'training_loss': '{:.3f}'.format(loss.item()/len(batch))})\n         \n        \n#     torch.save(model.state_dict(), f'finetuned_BERT_epoch_{epoch}.model')\n        \n#     tqdm.write(f'\\nEpoch {epoch}')\n    \n#     loss_train_avg = loss_train_total/len(dataloader_train)            \n#     tqdm.write(f'Training loss: {loss_train_avg}')\n    \n#     val_loss, predictions, true_vals = evaluate(dataloader_validation)\n#     val_f1 = f1_score_func(predictions, true_vals)\n#     tqdm.write(f'Validation loss: {val_loss}')\n#     tqdm.write(f'F1 Score (Weighted): {val_f1}')","metadata":{"_uuid":"1598465e-6dbe-492a-a248-49b4cb13cf5e","_cell_guid":"9ebebd28-7ba0-45bc-a076-17e60f5528f7","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Loading and Evaluating the Model","metadata":{"_uuid":"5e5f8e25-12a1-4792-ae66-d3c882ca2a48","_cell_guid":"a4b6bcc2-6f7b-4789-9b9a-b369facc070d","trusted":true}},{"cell_type":"code","source":"# model = BertForSequenceClassification.from_pretrained(\"bert-base-uncased\",\n#                                                       num_labels=len(label_dict),\n#                                                       output_attentions=False,\n#                                                       output_hidden_states=False)\n\n# model.to(device)","metadata":{"_uuid":"002006b4-8619-4ff3-9855-78a3c0c7b4a9","_cell_guid":"f0b2a7e0-8313-4088-8277-554dd23606f9","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model.load_state_dict(torch.load('finetuned_BERT_epoch_1.model', map_location=torch.device('cpu')))","metadata":{"_uuid":"cf27d851-ea76-442d-9177-21a195ef504a","_cell_guid":"1d454178-3b90-4853-a8e7-2c971714e20a","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# _, predictions, true_vals = evaluate(dataloader_validation)","metadata":{"_uuid":"683e74a9-e37a-4df2-aca9-39ec4c737962","_cell_guid":"9f924260-9622-4687-83c8-e0c83114d537","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# accuracy_per_class(predictions, true_vals)","metadata":{"_uuid":"053801a9-247b-4acf-a2a2-91520c1f7ff9","_cell_guid":"de89ee7f-4169-456a-b60e-40490cedbda7","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]}]}