{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-20T05:58:07.154083Z","iopub.execute_input":"2021-06-20T05:58:07.154660Z","iopub.status.idle":"2021-06-20T05:58:18.394447Z","shell.execute_reply.started":"2021-06-20T05:58:07.154572Z","shell.execute_reply":"2021-06-20T05:58:18.388817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Libraries","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport random\nimport time\nimport datetime\nfrom itertools import chain\nimport re\nimport json\nfrom tqdm.autonotebook import tqdm\nimport string\nimport collections\nfrom textblob import TextBlob\nfrom pathlib import Path\nimport collections\nimport pickle\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nfrom wordcloud import WordCloud, STOPWORDS\n\nimport nltk\nfrom nltk.probability import FreqDist\nfrom nltk.tokenize import word_tokenize\nfrom nltk.stem.wordnet import WordNetLemmatizer\n\nfrom sklearn import preprocessing\nfrom sklearn.model_selection import train_test_split, cross_val_score\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.linear_model import SGDClassifier\nfrom sklearn.svm import SVC\nfrom sklearn.naive_bayes import MultinomialNB\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.metrics import classification_report, precision_score, recall_score, f1_score, accuracy_score\nfrom sklearn.metrics import confusion_matrix, plot_confusion_matrix\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras.layers.experimental.preprocessing import TextVectorization\n\nfrom keras.preprocessing import sequence, text\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint\nfrom keras import utils\nfrom keras.models import *\nfrom keras.layers import *\nfrom keras.callbacks import *\nfrom keras.models import Sequential\nfrom keras.layers.recurrent import LSTM, GRU\nfrom keras.layers import Dense, Dropout\nfrom keras.layers.embeddings import Embedding\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.utils.vis_utils import plot_model\n\nimport en_core_web_lg\nimport spacy\nfrom spacy.lang.en import English\nfrom spacy.util import minibatch, compounding\nfrom spacy import displacy\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2021-06-20T05:58:18.396714Z","iopub.execute_input":"2021-06-20T05:58:18.397162Z","iopub.status.idle":"2021-06-20T05:58:27.161736Z","shell.execute_reply.started":"2021-06-20T05:58:18.397117Z","shell.execute_reply":"2021-06-20T05:58:27.160427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#define stopwords\nfrom nltk.corpus import stopwords\n\nstopwords_list = stopwords.words('english') + list(string.punctuation)\nstopwords_list += [\"''\", '\"\"', '...', '``']","metadata":{"execution":{"iopub.status.busy":"2021-06-20T05:58:27.165931Z","iopub.execute_input":"2021-06-20T05:58:27.166278Z","iopub.status.idle":"2021-06-20T05:58:27.185727Z","shell.execute_reply.started":"2021-06-20T05:58:27.166245Z","shell.execute_reply":"2021-06-20T05:58:27.184586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt):\n     return re.sub('[^A-Za-z0-9.]+', ' ', str(txt).lower())","metadata":{"execution":{"iopub.status.busy":"2021-06-20T05:58:27.187280Z","iopub.execute_input":"2021-06-20T05:58:27.187614Z","iopub.status.idle":"2021-06-20T05:58:27.192011Z","shell.execute_reply.started":"2021-06-20T05:58:27.187575Z","shell.execute_reply":"2021-06-20T05:58:27.191031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def text_cleaning(text, flg_stemm = False, flg_lemm = True, lst_stopwords = None):\n    '''\n    Converts all text to lower case, tokenize, remove multiple spaces, stopwords, stemming, lemmatize, \n    then convert all back to string\n    \n    text: string - name of column containing text\n    lst_stopwords: list - list of stopwords to remove\n    flg_stemm: bool - whether stemming is to be applied\n    flg_lemm: bool - whether lemmitisation is to be applied\n    '''\n    \n    #clean (convert to lowercase and remove punctuations and characters and then strip)\n    text = re.sub(r'[^\\w\\s]', '', str(text).lower().strip())\n            \n    #tokenize (convert from string to list)\n    lst_text = text.split()\n    \n    #remove Stopwords\n    if lst_stopwords is not None:\n        lst_text = [word for word in lst_text if word not in \n                    stopwords_list]\n                \n    #stemming (remove -ing, -ly, ...)\n    if flg_stemm == True:\n        ps = nltk.stem.porter.PorterStemmer()\n        lst_text = [ps.stem(word) for word in lst_text]\n                \n    #lemmatisation (convert the word into root word)\n    if flg_lemm == True:\n        lem = nltk.stem.wordnet.WordNetLemmatizer()\n        lst_text = [lem.lemmatize(word) for word in lst_text]\n            \n    #back to string from list\n    text = \" \".join(lst_text)\n    return text","metadata":{"execution":{"iopub.status.busy":"2021-06-20T05:58:27.193078Z","iopub.execute_input":"2021-06-20T05:58:27.193504Z","iopub.status.idle":"2021-06-20T05:58:27.205127Z","shell.execute_reply.started":"2021-06-20T05:58:27.193475Z","shell.execute_reply":"2021-06-20T05:58:27.204343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MAX_LENGTH = 64\nOVERLAP = 20\n    \ndef shorten_sentences(sentences):\n    \"\"\"\n    Sentences that have more than MAX_LENGTH words will be split\n    into multiple sentences with overlappings.\n    \"\"\"\n    short_sentences = []\n    for sentence in sentences:\n        words = sentence.split()\n        if len(words) > MAX_LENGTH:\n            for p in range(0, len(words), MAX_LENGTH - OVERLAP):\n                short_sentences.append(' '.join(words[p:p+MAX_LENGTH]))\n        else:\n            short_sentences.append(sentence)\n    return short_sentences","metadata":{"execution":{"iopub.status.busy":"2021-06-20T05:58:27.206153Z","iopub.execute_input":"2021-06-20T05:58:27.206574Z","iopub.status.idle":"2021-06-20T05:58:27.221800Z","shell.execute_reply.started":"2021-06-20T05:58:27.206544Z","shell.execute_reply":"2021-06-20T05:58:27.220512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# OBTAIN","metadata":{}},{"cell_type":"code","source":"#define paths\nos.listdir('/kaggle/input/coleridgeinitiative-show-us-the-data/')\ntrain_path = '../input/coleridgeinitiative-show-us-the-data/train'\ntest_path = '../input/coleridgeinitiative-show-us-the-data/test'","metadata":{"execution":{"iopub.status.busy":"2021-06-20T05:58:27.223130Z","iopub.execute_input":"2021-06-20T05:58:27.223418Z","iopub.status.idle":"2021-06-20T05:58:27.234097Z","shell.execute_reply.started":"2021-06-20T05:58:27.223377Z","shell.execute_reply":"2021-06-20T05:58:27.233212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#create a function to get the text from the JSON file and append it to the new column in table\ndef read_json_pub(filename, train_path = train_path, output = 'text'):\n    json_path = os.path.join(train_path, (filename + '.json'))\n    headings = []\n    contents = []\n    combined = []\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title'))\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n    \n    all_headings = ' '.join(headings)\n    all_contents = ' '.join(contents)\n    all_data = '. '.join(combined)\n    \n    if output == 'text':\n        return all_contents\n    elif output == 'head':\n        return all_headings\n    else:\n        return all_data","metadata":{"execution":{"iopub.status.busy":"2021-06-20T05:58:27.236494Z","iopub.execute_input":"2021-06-20T05:58:27.236790Z","iopub.status.idle":"2021-06-20T05:58:27.249380Z","shell.execute_reply.started":"2021-06-20T05:58:27.236762Z","shell.execute_reply":"2021-06-20T05:58:27.248263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#read \ntrain = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\n\n#review\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-20T05:58:27.251780Z","iopub.execute_input":"2021-06-20T05:58:27.252295Z","iopub.status.idle":"2021-06-20T05:58:27.456919Z","shell.execute_reply.started":"2021-06-20T05:58:27.252259Z","shell.execute_reply":"2021-06-20T05:58:27.456015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['cleaned_label'].nunique()","metadata":{"execution":{"iopub.status.busy":"2021-06-20T05:58:27.458952Z","iopub.execute_input":"2021-06-20T05:58:27.459242Z","iopub.status.idle":"2021-06-20T05:58:27.472657Z","shell.execute_reply.started":"2021-06-20T05:58:27.459214Z","shell.execute_reply":"2021-06-20T05:58:27.471631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create Sentences & Labels","metadata":{}},{"cell_type":"code","source":"import nltk\n\nDATA = []\nlabel_count = 0\nempty_count = 0\n\nfor idx,row in tqdm(train.iterrows()):\n    pub = \"../input/coleridgeinitiative-show-us-the-data/train/\" + row.Id + \".json\"            \n    f = open(pub)  \n    data = json.load(f)      \n\n    balanced = False\n    \n    sentences = [clean_text(sentence) for sentence in nltk.sent_tokenize(str(data))]\n    sentences = shorten_sentences(sentences) # make sentences short\n    sentences = [sentence for sentence in sentences if len(sentence) > 10] # only accept sentences with length > 10 chars\n    \n    for sentence in sentences:          \n     \n        a = re.search(row.cleaned_label.lower(), sentence)      \n        b = re.search(row.dataset_label.lower(), sentence)\n        c = re.search(row.dataset_title.lower(), sentence)\n        cleaned_label = row.cleaned_label.lower()\n        dataset_label = row.dataset_label.lower()\n        dataset_title = row.dataset_title.lower()\n        \n        if  a != None:\n            DATA.append((sentence, cleaned_label))\n            label_count = label_count + 1\n            balanced = True\n        elif b != None:\n            DATA.append((sentence, dataset_label))\n            label_count = label_count + 1\n            balanced = True\n        elif c != None:\n            DATA.append((sentence, dataset_title))\n            label_count = label_count + 1\n            balanced = True            \n        else:\n            if balanced:\n                empty_count = empty_count + 1\n                balanced = False\n    \nprint('Text with dataset:', label_count)\nprint('Text without dataset:', empty_count)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T05:58:27.473866Z","iopub.execute_input":"2021-06-20T05:58:27.474366Z","iopub.status.idle":"2021-06-20T06:20:21.783912Z","shell.execute_reply.started":"2021-06-20T05:58:27.474326Z","shell.execute_reply":"2021-06-20T06:20:21.782546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA[:5]","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:20:21.785835Z","iopub.execute_input":"2021-06-20T06:20:21.786137Z","iopub.status.idle":"2021-06-20T06:20:21.798343Z","shell.execute_reply.started":"2021-06-20T06:20:21.786100Z","shell.execute_reply":"2021-06-20T06:20:21.797386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CLASSIFIERS: RandomForestClassifier, SVC, MultinomialNB","metadata":{}},{"cell_type":"markdown","source":"# Preprocessing\n\n* Tokenize text samples into word uni+bigrams,\n* Vectorize using tf-idf encoding,\n* Select only the top 20,000 features from the vector of tokens by discarding tokens that appear fewer than 2 times and using f_classif to calculate feature importance.","metadata":{}},{"cell_type":"markdown","source":"### Train-Test-Split","metadata":{}},{"cell_type":"code","source":"#save to dataframe\ntrain_df = pd.DataFrame(DATA)\ntrain_df = train_df.rename({0: 'Sentence', 1: 'Label'}, axis = 1)\n\n#review\ntrain_df.tail(10)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:20:21.799654Z","iopub.execute_input":"2021-06-20T06:20:21.800269Z","iopub.status.idle":"2021-06-20T06:20:21.857292Z","shell.execute_reply.started":"2021-06-20T06:20:21.800222Z","shell.execute_reply":"2021-06-20T06:20:21.856286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Label'].nunique()","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:20:21.858590Z","iopub.execute_input":"2021-06-20T06:20:21.858891Z","iopub.status.idle":"2021-06-20T06:20:21.884472Z","shell.execute_reply.started":"2021-06-20T06:20:21.858861Z","shell.execute_reply":"2021-06-20T06:20:21.883465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_df['Sentence'][10])\nprint('\\n')\nprint(train_df['Label'][10])","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:20:21.885836Z","iopub.execute_input":"2021-06-20T06:20:21.886130Z","iopub.status.idle":"2021-06-20T06:20:21.892321Z","shell.execute_reply.started":"2021-06-20T06:20:21.886102Z","shell.execute_reply":"2021-06-20T06:20:21.891262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_df['Sentence'][888])\nprint('\\n')\nprint(train_df['Label'][888])","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:20:21.893803Z","iopub.execute_input":"2021-06-20T06:20:21.894111Z","iopub.status.idle":"2021-06-20T06:20:21.906418Z","shell.execute_reply.started":"2021-06-20T06:20:21.894080Z","shell.execute_reply":"2021-06-20T06:20:21.905220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_df['Sentence'][51179])\nprint('\\n')\nprint(train_df['Label'][51179])","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:20:21.908314Z","iopub.execute_input":"2021-06-20T06:20:21.908855Z","iopub.status.idle":"2021-06-20T06:20:21.916529Z","shell.execute_reply.started":"2021-06-20T06:20:21.908809Z","shell.execute_reply":"2021-06-20T06:20:21.915489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX = train_df['Sentence'].to_numpy()\ny = train_df['Label'].to_numpy()","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:20:21.918382Z","iopub.execute_input":"2021-06-20T06:20:21.918701Z","iopub.status.idle":"2021-06-20T06:20:21.925939Z","shell.execute_reply.started":"2021-06-20T06:20:21.918673Z","shell.execute_reply":"2021-06-20T06:20:21.924844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#split traing data into training a validation sets\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 123)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:20:21.927566Z","iopub.execute_input":"2021-06-20T06:20:21.928052Z","iopub.status.idle":"2021-06-20T06:20:21.956870Z","shell.execute_reply.started":"2021-06-20T06:20:21.928011Z","shell.execute_reply":"2021-06-20T06:20:21.955741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Vectorize Features","metadata":{}},{"cell_type":"code","source":"#https://developers.google.com/machine-learning/guides/text-classification/step-3\n\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nvectorizer = TfidfVectorizer(min_df = 1,                       #ignore terms that have a document frequency strictly lower than the given threshold\n                             dtype = 'int32', \n                             strip_accents = 'unicode',\n                             analyzer = 'word',\n                             token_pattern = r'\\w{1,}',\n                             ngram_range = (1, 4), \n                             use_idf = 1,                      #enable inverse-document-frequency reweighting\n                             smooth_idf = 1,                   #weights by adding one to document frequencies\n                             sublinear_tf = 1,                 #apply sublinear tf scaling\n                             stop_words = stopwords_list)\n\n#learn vocabulary from training texts and vectorize training texts\nvectorizer = vectorizer.fit(list(X_train) + list(X_test))\n\n#transform Bag-of-Words textual data to numeric\nX_train = vectorizer.transform(X_train)\nX_test = vectorizer.transform(X_test)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:20:21.958229Z","iopub.execute_input":"2021-06-20T06:20:21.958759Z","iopub.status.idle":"2021-06-20T06:20:51.593361Z","shell.execute_reply.started":"2021-06-20T06:20:21.958725Z","shell.execute_reply":"2021-06-20T06:20:51.592229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vectorizer.get_feature_names()[:20]","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:20:51.594709Z","iopub.execute_input":"2021-06-20T06:20:51.595000Z","iopub.status.idle":"2021-06-20T06:20:54.419194Z","shell.execute_reply.started":"2021-06-20T06:20:51.594972Z","shell.execute_reply":"2021-06-20T06:20:54.417988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.feature_selection import SelectKBest\nfrom sklearn.feature_selection import f_classif\n\n#select top 'k' of the vectorized features, limit on the number of features. We use the top 20K features\ntop_k = 20000\n\nselector = SelectKBest(f_classif, k = min(top_k, X_train.shape[1]))\nselector.fit(X_train, y_train)\nX_train = selector.transform(X_train).astype('float32')\nX_test = selector.transform(X_test).astype('float32')","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:20:54.420845Z","iopub.execute_input":"2021-06-20T06:20:54.421251Z","iopub.status.idle":"2021-06-20T06:21:03.367513Z","shell.execute_reply.started":"2021-06-20T06:20:54.421209Z","shell.execute_reply":"2021-06-20T06:21:03.366388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Encode Label","metadata":{}},{"cell_type":"code","source":"from sklearn import preprocessing\n\n#use the LabelEncoder to convert text labels to integers, 0, 1, 2, etc.\nencoder = preprocessing.LabelEncoder()\n\n#since we have two different data set (X_train and X_test), \n#we need to fit it on all of our data otherwise there might be some categories in the test set X_test that were not in the train set X_train \n#and we will get errors\nencoder.fit(list(y_train) + list(y_test)) \ny_train = encoder.transform(y_train)\ny_test = encoder.transform(y_test)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:21:03.371965Z","iopub.execute_input":"2021-06-20T06:21:03.372286Z","iopub.status.idle":"2021-06-20T06:21:03.538280Z","shell.execute_reply.started":"2021-06-20T06:21:03.372253Z","shell.execute_reply":"2021-06-20T06:21:03.537270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoder.classes_","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:21:03.540274Z","iopub.execute_input":"2021-06-20T06:21:03.540600Z","iopub.status.idle":"2021-06-20T06:21:03.548335Z","shell.execute_reply.started":"2021-06-20T06:21:03.540568Z","shell.execute_reply":"2021-06-20T06:21:03.547304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:21:03.549715Z","iopub.execute_input":"2021-06-20T06:21:03.550017Z","iopub.status.idle":"2021-06-20T06:21:03.561418Z","shell.execute_reply.started":"2021-06-20T06:21:03.549987Z","shell.execute_reply":"2021-06-20T06:21:03.560198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.shape","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:21:03.562778Z","iopub.execute_input":"2021-06-20T06:21:03.563083Z","iopub.status.idle":"2021-06-20T06:21:03.574200Z","shell.execute_reply.started":"2021-06-20T06:21:03.563052Z","shell.execute_reply":"2021-06-20T06:21:03.573317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:21:03.575644Z","iopub.execute_input":"2021-06-20T06:21:03.576010Z","iopub.status.idle":"2021-06-20T06:21:03.587588Z","shell.execute_reply.started":"2021-06-20T06:21:03.575982Z","shell.execute_reply":"2021-06-20T06:21:03.586381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train.shape","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:21:03.589095Z","iopub.execute_input":"2021-06-20T06:21:03.589574Z","iopub.status.idle":"2021-06-20T06:21:03.600848Z","shell.execute_reply.started":"2021-06-20T06:21:03.589504Z","shell.execute_reply":"2021-06-20T06:21:03.599830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# RandomForestClassifier","metadata":{}},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\n\n#create a tokenization a modelling pipeline\nRandomForestClassifier_model = RandomForestClassifier(n_estimators = 100, random_state = 123) #model the text\n\n#fit the pipeline to the training data\nRandomForestClassifier_model.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:21:03.602708Z","iopub.execute_input":"2021-06-20T06:21:03.603193Z","iopub.status.idle":"2021-06-20T06:30:04.507561Z","shell.execute_reply.started":"2021-06-20T06:21:03.603147Z","shell.execute_reply":"2021-06-20T06:30:04.506521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#predict \nRandomForestClassifier_yhat_train = RandomForestClassifier_model.predict(X_train)\nRandomForestClassifier_yhat_test = RandomForestClassifier_model.predict(X_test)\n\n#train accuracy \nRandomForestClassifier_train_accuracy = accuracy_score(y_train, RandomForestClassifier_yhat_train) * 100\nprint('Train accuracy: {0}'.format(RandomForestClassifier_train_accuracy))\n\n#test accuracy \nRandomForestClassifier_test_accuracy = accuracy_score(y_test, RandomForestClassifier_yhat_test) * 100\nprint('Test accuracy: {0}'.format(RandomForestClassifier_test_accuracy))","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:30:04.509026Z","iopub.execute_input":"2021-06-20T06:30:04.509336Z","iopub.status.idle":"2021-06-20T06:30:11.219004Z","shell.execute_reply.started":"2021-06-20T06:30:04.509304Z","shell.execute_reply":"2021-06-20T06:30:11.217878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#save\nfilename = 'RandomForestClassifier_model.sav'\npickle.dump(RandomForestClassifier_model, open(filename, 'wb'))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import KFold, cross_val_score\n\n#get cross validation to make sure model has gotten most of the patterns from the data correctly, \n#and its not picking up too much on the noise\nk_fold = KFold(n_splits = 5)\nRandomForestClassifier_CV = cross_val_score(RandomForestClassifier_model, X_train, y_train, \n                                            cv = k_fold, scoring = 'accuracy', n_jobs = -1).mean()\nRandomForestClassifier_CV","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:30:11.220466Z","iopub.execute_input":"2021-06-20T06:30:11.220781Z","iopub.status.idle":"2021-06-20T06:46:36.397077Z","shell.execute_reply.started":"2021-06-20T06:30:11.220749Z","shell.execute_reply":"2021-06-20T06:46:36.396056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import classification_report, precision_score, recall_score, f1_score, accuracy_score\n\n#get classification report\nprint('Model: RandomForestClassifier', '\\n', classification_report(y_test, RandomForestClassifier_yhat_test))","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:46:36.398506Z","iopub.execute_input":"2021-06-20T06:46:36.398932Z","iopub.status.idle":"2021-06-20T06:46:36.443528Z","shell.execute_reply.started":"2021-06-20T06:46:36.398899Z","shell.execute_reply":"2021-06-20T06:46:36.442481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#summary table\nsummary_table = pd.DataFrame({'Model': [],\n                              'Accuracy': [],\n                              'CV': [],\n                              'Precision': [], 'Recall': [], 'F1': []})","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:46:36.444832Z","iopub.execute_input":"2021-06-20T06:46:36.445104Z","iopub.status.idle":"2021-06-20T06:46:36.452629Z","shell.execute_reply.started":"2021-06-20T06:46:36.445078Z","shell.execute_reply":"2021-06-20T06:46:36.451582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"summary_table.loc[0] = ['CLF RandomForestClassifier',\n                        round(accuracy_score(y_test, RandomForestClassifier_yhat_test), 2),\n                        round(RandomForestClassifier_CV, 2),\n                        round(precision_score(y_test, RandomForestClassifier_yhat_test, average = 'macro'), 2), \n                        round(recall_score(y_test, RandomForestClassifier_yhat_test, average = 'macro'), 2), \n                        round(f1_score(y_test, RandomForestClassifier_yhat_test, average = 'macro'), 2)]\nsummary_table.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:46:36.454564Z","iopub.execute_input":"2021-06-20T06:46:36.455030Z","iopub.status.idle":"2021-06-20T06:46:36.511301Z","shell.execute_reply.started":"2021-06-20T06:46:36.454986Z","shell.execute_reply":"2021-06-20T06:46:36.510555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# SGDClassifier","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import SGDClassifier\n\n#create a tokenization a modelling pipeline\nSGDClassifier_model = SGDClassifier(loss = 'hinge', #use Linear SVC(SVM) model when the parameter loss is set to 'hinge'\n                                                      penalty = 'l2',\n                                                      alpha = 1e-3,\n                                                      random_state = 123,\n                                                      max_iter = 5) #model the text\n\n#fit the pipeline to the training data\nSGDClassifier_model.fit(X_train, y_train) ","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:46:36.512540Z","iopub.execute_input":"2021-06-20T06:46:36.513015Z","iopub.status.idle":"2021-06-20T06:46:46.283593Z","shell.execute_reply.started":"2021-06-20T06:46:36.512970Z","shell.execute_reply":"2021-06-20T06:46:46.282241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#predict \nSGDClassifier_yhat_train = SGDClassifier_model.predict(X_train)\nSGDClassifier_yhat_test = SGDClassifier_model.predict(X_test)\n\n#train accuracy \nSGDClassifier_train_accuracy = accuracy_score(y_train, SGDClassifier_yhat_train) * 100\nprint('Train accuracy: {0}'.format(SGDClassifier_train_accuracy))\n\n#test accuracy \nSGDClassifier_test_accuracy = accuracy_score(y_test, SGDClassifier_yhat_test) * 100\nprint('Test accuracy: {0}'.format(SGDClassifier_test_accuracy))","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:46:46.289558Z","iopub.execute_input":"2021-06-20T06:46:46.292658Z","iopub.status.idle":"2021-06-20T06:46:46.481385Z","shell.execute_reply.started":"2021-06-20T06:46:46.292566Z","shell.execute_reply":"2021-06-20T06:46:46.480436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#save\nfilename = 'SGDClassifier_model.sav'\npickle.dump(SGDClassifier_model, open(filename, 'wb'))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#get cross validation\nSGDClassifier_CV = cross_val_score(SGDClassifier_model, X_train, y_train,\n                                   cv = k_fold, scoring = 'accuracy', n_jobs=-1).mean()\nSGDClassifier_CV","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:46:46.482909Z","iopub.execute_input":"2021-06-20T06:46:46.483316Z","iopub.status.idle":"2021-06-20T06:46:58.825848Z","shell.execute_reply.started":"2021-06-20T06:46:46.483274Z","shell.execute_reply":"2021-06-20T06:46:58.825018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#get classification report\nprint('Model: Support Vector Machine', '\\n', classification_report(y_test, SGDClassifier_yhat_test))","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:46:58.827294Z","iopub.execute_input":"2021-06-20T06:46:58.827829Z","iopub.status.idle":"2021-06-20T06:46:58.864859Z","shell.execute_reply.started":"2021-06-20T06:46:58.827795Z","shell.execute_reply":"2021-06-20T06:46:58.863726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"summary_table.loc[1] = ['CLF Linear Support Vector Machine',\n                        round(accuracy_score(y_test, SGDClassifier_yhat_test), 2),\n                        round(SGDClassifier_CV, 2),\n                        round(precision_score(y_test, SGDClassifier_yhat_test, average = 'macro'), 2), \n                        round(recall_score(y_test, SGDClassifier_yhat_test, average = 'macro'), 2), \n                        round(f1_score(y_test, SGDClassifier_yhat_test, average = 'macro'), 2)]\nsummary_table.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:46:58.868445Z","iopub.execute_input":"2021-06-20T06:46:58.869021Z","iopub.status.idle":"2021-06-20T06:46:58.911520Z","shell.execute_reply.started":"2021-06-20T06:46:58.868983Z","shell.execute_reply":"2021-06-20T06:46:58.910380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# MultinomialNB","metadata":{}},{"cell_type":"code","source":"from sklearn.naive_bayes import MultinomialNB\n\n#create a tokenization a modelling pipeline\nMultinomialNB_model = MultinomialNB() #model the text\n\n#fit the pipeline to the training data\nMultinomialNB_model.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:46:58.913149Z","iopub.execute_input":"2021-06-20T06:46:58.913492Z","iopub.status.idle":"2021-06-20T06:46:59.103337Z","shell.execute_reply.started":"2021-06-20T06:46:58.913459Z","shell.execute_reply":"2021-06-20T06:46:59.102409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#predict \nMultinomialNB_yhat_train = MultinomialNB_model.predict(X_train)\nMultinomialNB_yhat_test = MultinomialNB_model.predict(X_test)\n\n#train accuracy \nMultinomialNB_train_accuracy = accuracy_score(y_train, MultinomialNB_yhat_train) * 100\nprint('Train accuracy: {0}'.format(MultinomialNB_train_accuracy))\n\n#test accuracy \nMultinomialNB_test_accuracy = accuracy_score(y_test, MultinomialNB_yhat_test) * 100\nprint('Test accuracy: {0}'.format(MultinomialNB_test_accuracy))","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:46:59.104710Z","iopub.execute_input":"2021-06-20T06:46:59.105010Z","iopub.status.idle":"2021-06-20T06:46:59.270338Z","shell.execute_reply.started":"2021-06-20T06:46:59.104981Z","shell.execute_reply":"2021-06-20T06:46:59.269284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#save\nfilename = 'MultinomialNB_model.sav'\npickle.dump(MultinomialNB_model, open(filename, 'wb'))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#get cross validation\nMultinomialNB_CV = cross_val_score(MultinomialNB_model, X_train, y_train,\n                                   cv = k_fold, scoring = 'accuracy', n_jobs=-1).mean()\nMultinomialNB_CV","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:46:59.271856Z","iopub.execute_input":"2021-06-20T06:46:59.272169Z","iopub.status.idle":"2021-06-20T06:47:00.103458Z","shell.execute_reply.started":"2021-06-20T06:46:59.272138Z","shell.execute_reply":"2021-06-20T06:47:00.102461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import classification_report, precision_score, recall_score, f1_score, accuracy_score\n\n#get classification report\nprint('Model: MultinomialNB', '\\n', classification_report(y_test, MultinomialNB_yhat_test))","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:47:00.104808Z","iopub.execute_input":"2021-06-20T06:47:00.105121Z","iopub.status.idle":"2021-06-20T06:47:00.140383Z","shell.execute_reply.started":"2021-06-20T06:47:00.105090Z","shell.execute_reply":"2021-06-20T06:47:00.139220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#update summary table\nsummary_table.loc[2] = ['CLF MultinomialNB',\n                        round(accuracy_score(y_test, MultinomialNB_yhat_test), 2),\n                        round(MultinomialNB_CV, 2),\n                        round(precision_score(y_test, MultinomialNB_yhat_test, average = 'macro'), 2), \n                        round(recall_score(y_test, MultinomialNB_yhat_test, average = 'macro'), 2), \n                        round(f1_score(y_test, MultinomialNB_yhat_test, average = 'macro'), 2)]\nsummary_table.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:47:00.141789Z","iopub.execute_input":"2021-06-20T06:47:00.142084Z","iopub.status.idle":"2021-06-20T06:47:00.182681Z","shell.execute_reply.started":"2021-06-20T06:47:00.142057Z","shell.execute_reply":"2021-06-20T06:47:00.181513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"summary_table.to_csv('clf_summary_table.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:47:00.184001Z","iopub.execute_input":"2021-06-20T06:47:00.184285Z","iopub.status.idle":"2021-06-20T06:47:00.201174Z","shell.execute_reply.started":"2021-06-20T06:47:00.184258Z","shell.execute_reply":"2021-06-20T06:47:00.200135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# PREDICTION","metadata":{}},{"cell_type":"code","source":"#get text\ntqdm.pandas()\ntrain['text'] = train['Id'].progress_apply(read_json_pub)\n\n#clean text\ntrain['text'] = train['text'].progress_apply(clean_text)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:48:46.210575Z","iopub.execute_input":"2021-06-20T06:48:46.211017Z","iopub.status.idle":"2021-06-20T06:50:53.818001Z","shell.execute_reply.started":"2021-06-20T06:48:46.210982Z","shell.execute_reply":"2021-06-20T06:50:53.816711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from functools import partial\n\n#read data\nsample_submission = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\n\n#apply the function to submission data\ntqdm.pandas()\nsample_submission['text'] = sample_submission['Id'].progress_apply(partial(read_json_pub, train_path = test_path))\n\n#review\nsample_submission.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:50:53.820049Z","iopub.execute_input":"2021-06-20T06:50:53.820513Z","iopub.status.idle":"2021-06-20T06:50:53.926800Z","shell.execute_reply.started":"2021-06-20T06:50:53.820468Z","shell.execute_reply":"2021-06-20T06:50:53.925440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_1 = [x.lower() for x in train['dataset_label'].unique()]\ntemp_2 = [x.lower() for x in train['dataset_title'].unique()]\ntemp_3 = [x.lower() for x in train['cleaned_label'].unique()]\n\nexisting_labels = set(temp_1 + temp_2 + temp_3)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:50:53.929190Z","iopub.execute_input":"2021-06-20T06:50:53.929904Z","iopub.status.idle":"2021-06-20T06:50:53.945510Z","shell.execute_reply.started":"2021-06-20T06:50:53.929846Z","shell.execute_reply":"2021-06-20T06:50:53.944662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"literal_matching = True\nRandomForestClassifier_prediction = True\nSGDClassifier_prediction = True\nMultinomialNB_prediction = True\n\nid_list = []\nlabels_list = []\n\nfor index, row in tqdm(sample_submission.iterrows()):\n\n    sample_text = row['text']\n\n    row_id = row['Id']\n    \n    #takes only the rows where train file is identical to a test file\n    temp_df = train[train['text'] == clean_text(sample_text)]\n    cleaned_labels = temp_df['cleaned_label'].to_list()\n    \n    #literal_matching \n    if literal_matching:\n        for known_label in existing_labels:\n            if known_label in sample_text.lower():    \n                cleaned_labels.append(clean_text(known_label))\n            \n        print('cleaned label:', set(cleaned_labels))   \n    \n    #classifier \n    if RandomForestClassifier_prediction:\n        \n        #extract sentences\n        sentences = [clean_text(sentence) for sentence in nltk.sent_tokenize(str(sample_text))]\n        sentences = shorten_sentences(sentences) # make sentences short\n        sentences = [sentence for sentence in sentences if len(sentence) > 10] # only accept sentences with length > 10 chars\n    \n        #predict\n        RandomForestClassifier_labels = RandomForestClassifier_model.predict(selector.transform(vectorizer.transform(sentences).toarray()))\n    \n        #get labels\n        RandomForestClassifier_labels = encoder.inverse_transform(RandomForestClassifier_labels) \n        print('RandomForestClassifier_label:', set(RandomForestClassifier_labels))\n        RandomForestClassifier_labels = set(RandomForestClassifier_labels)\n        \n    cleaned_labels += RandomForestClassifier_labels\n    print('updated cleaned label:', set(cleaned_labels))\n\n    if MultinomialNB_prediction:\n        \n        #extract sentences\n        sentences = [clean_text(sentence) for sentence in nltk.sent_tokenize(str(sample_text))]\n        sentences = shorten_sentences(sentences) # make sentences short\n        sentences = [sentence for sentence in sentences if len(sentence) > 10] # only accept sentences with length > 10 chars\n        \n        #predict\n        MultinomialNB_labels = MultinomialNB_model.predict(selector.transform(vectorizer.transform(sentences).toarray()))\n    \n        #get labels\n        MultinomialNB_labels = encoder.inverse_transform(MultinomialNB_labels) \n        print('MultinomialNB_label:', set(MultinomialNB_labels))\n        MultinomialNB_labels = set(MultinomialNB_labels)\n        \n    cleaned_labels += MultinomialNB_labels\n    print('updated cleaned label:', set(cleaned_labels))\n    \n    if SGDClassifier_prediction:\n        \n        #extract sentences\n        sentences = [clean_text(sentence) for sentence in nltk.sent_tokenize(str(sample_text))]\n        sentences = shorten_sentences(sentences) # make sentences short\n        sentences = [sentence for sentence in sentences if len(sentence) > 10] # only accept sentences with length > 10 chars\n    \n        #predict\n        SGDClassifier_labels = SGDClassifier_model.predict(selector.transform(vectorizer.transform(sentences).toarray()))\n    \n        #get labels\n        SGDClassifier_labels = encoder.inverse_transform(SGDClassifier_labels) \n        print('SGDClassifier_label:', set(SGDClassifier_labels))\n        SGDClassifier_labels = set(SGDClassifier_labels)\n        \n    cleaned_labels += SGDClassifier_labels\n    print('updated cleaned label:', set(cleaned_labels))\n    \n    cleaned_labels = set(cleaned_labels)\n    cleaned_labels = [clean_text(x) for x in cleaned_labels]    \n    labels_list.append('|'.join(cleaned_labels))\n    print('label list:', labels_list)   \n    id_list.append(row_id)\n    print('\\n')","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:50:53.947140Z","iopub.execute_input":"2021-06-20T06:50:53.947686Z","iopub.status.idle":"2021-06-20T06:51:32.349498Z","shell.execute_reply.started":"2021-06-20T06:50:53.947652Z","shell.execute_reply":"2021-06-20T06:51:32.348184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#get dataframe\nsample_submission['PredictionString'] = labels_list\nsample_submission","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:51:32.351359Z","iopub.execute_input":"2021-06-20T06:51:32.352066Z","iopub.status.idle":"2021-06-20T06:51:32.371855Z","shell.execute_reply.started":"2021-06-20T06:51:32.352017Z","shell.execute_reply":"2021-06-20T06:51:32.370596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(sample_submission['PredictionString'][0])\nprint('\\n')\nprint(sample_submission['PredictionString'][1])\nprint('\\n')\nprint(sample_submission['PredictionString'][2])\nprint('\\n')\nprint(sample_submission['PredictionString'][3])","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:51:32.373711Z","iopub.execute_input":"2021-06-20T06:51:32.374045Z","iopub.status.idle":"2021-06-20T06:51:32.387378Z","shell.execute_reply.started":"2021-06-20T06:51:32.374014Z","shell.execute_reply":"2021-06-20T06:51:32.386093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#drop 'text'\nsample_submission.drop(columns = 'text', axis = 1, inplace = True)\nsample_submission","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:51:32.389166Z","iopub.execute_input":"2021-06-20T06:51:32.389616Z","iopub.status.idle":"2021-06-20T06:51:32.411604Z","shell.execute_reply.started":"2021-06-20T06:51:32.389579Z","shell.execute_reply":"2021-06-20T06:51:32.410505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#save\nsample_submission.to_csv('submission.csv', index = False)\n\n#check\nsubmission = pd.read_csv('submission.csv')\nsubmission","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:51:32.414246Z","iopub.execute_input":"2021-06-20T06:51:32.415036Z","iopub.status.idle":"2021-06-20T06:51:32.433610Z","shell.execute_reply.started":"2021-06-20T06:51:32.414985Z","shell.execute_reply":"2021-06-20T06:51:32.432489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}