{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\n\nimport tensorflow as tf\nimport keras\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\nfrom tqdm.autonotebook import tqdm","metadata":{"execution":{"iopub.status.busy":"2021-06-20T11:48:42.998603Z","iopub.execute_input":"2021-06-20T11:48:42.998972Z","iopub.status.idle":"2021-06-20T11:48:43.103626Z","shell.execute_reply.started":"2021-06-20T11:48:42.998942Z","shell.execute_reply":"2021-06-20T11:48:43.102736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json\nimport re\nfrom wordcloud import WordCloud,STOPWORDS\nfrom collections import Counter\nfrom nltk.probability import FreqDist\nfrom functools import partial\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2021-06-20T11:48:45.469431Z","iopub.execute_input":"2021-06-20T11:48:45.469779Z","iopub.status.idle":"2021-06-20T11:48:45.830173Z","shell.execute_reply.started":"2021-06-20T11:48:45.46974Z","shell.execute_reply":"2021-06-20T11:48:45.829383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://tfhub.dev/tensorflow/bert_multi_cased_preprocess/3\n!pip install tensorflow_text\nimport tensorflow_text as text","metadata":{"execution":{"iopub.status.busy":"2021-06-20T11:47:51.118349Z","iopub.execute_input":"2021-06-20T11:47:51.118739Z","iopub.status.idle":"2021-06-20T11:47:59.03774Z","shell.execute_reply.started":"2021-06-20T11:47:51.118654Z","shell.execute_reply":"2021-06-20T11:47:59.036842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preprocessor = hub.load(\"https://tfhub.dev/tensorflow/bert_multi_cased_preprocess/3\")\n\n# Step 1: tokenize batches of text inputs.\ntext_inputs = [tf.keras.layers.Input(shape=(), dtype=tf.string),\n               ...] # This SavedModel accepts up to 2 text inputs.\ntokenize = hub.KerasLayer(preprocessor.tokenize)\ntokenized_inputs = [tokenize(segment) for segment in text_inputs]\n\n# Step 2 (optional): modify tokenized inputs.\npass\n\n# Step 3: pack input sequences for the Transformer encoder.\nseq_length = 128  # Your choice here.\nbert_pack_inputs = hub.KerasLayer(\n    preprocessor.bert_pack_inputs,\n    arguments=dict(seq_length=seq_length))  # Optional argument.\nencoder_inputs = bert_pack_inputs(tokenized_inputs)\n","metadata":{"execution":{"iopub.status.busy":"2021-06-20T11:48:19.675268Z","iopub.execute_input":"2021-06-20T11:48:19.675585Z","iopub.status.idle":"2021-06-20T11:48:19.682504Z","shell.execute_reply.started":"2021-06-20T11:48:19.675555Z","shell.execute_reply":"2021-06-20T11:48:19.681231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# reading csv files and train & test file paths\ntrain_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\nsample_sub = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\ntrain_files_path = '../input/coleridgeinitiative-show-us-the-data/train'\ntest_files_path = '../input/coleridgeinitiative-show-us-the-data/test'","metadata":{"execution":{"iopub.status.busy":"2021-06-20T11:49:00.110492Z","iopub.execute_input":"2021-06-20T11:49:00.110835Z","iopub.status.idle":"2021-06-20T11:49:00.231956Z","shell.execute_reply.started":"2021-06-20T11:49:00.110803Z","shell.execute_reply":"2021-06-20T11:49:00.2312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\nimport string\n\ndef text_cleaning(text):\n    text = ''.join([k for k in text if k not in string.punctuation])\n    text = re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()    \n    return text","metadata":{"execution":{"iopub.status.busy":"2021-06-20T11:49:02.800428Z","iopub.execute_input":"2021-06-20T11:49:02.800774Z","iopub.status.idle":"2021-06-20T11:49:02.805621Z","shell.execute_reply.started":"2021-06-20T11:49:02.80074Z","shell.execute_reply":"2021-06-20T11:49:02.804696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-20T11:49:05.081553Z","iopub.execute_input":"2021-06-20T11:49:05.081903Z","iopub.status.idle":"2021-06-20T11:49:05.10953Z","shell.execute_reply.started":"2021-06-20T11:49:05.081872Z","shell.execute_reply":"2021-06-20T11:49:05.108703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\ntest_df","metadata":{"execution":{"iopub.status.busy":"2021-06-20T11:49:08.164917Z","iopub.execute_input":"2021-06-20T11:49:08.16528Z","iopub.status.idle":"2021-06-20T11:49:08.184489Z","shell.execute_reply.started":"2021-06-20T11:49:08.165251Z","shell.execute_reply":"2021-06-20T11:49:08.183063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_append_return(filename, train_files_path=train_files_path, output='text'):\n    \"\"\"\n    Function to read json file and then return the text data from them and append to the dataframe\n    \"\"\"\n    json_path = os.path.join(train_files_path, (filename+'.json'))\n    headings = []\n    contents = []\n    combined = []\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title'))\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n    \n    all_headings = ' '.join(headings)\n    all_contents = ' '.join(contents)\n    all_data = '. '.join(combined)\n    \n    if output == 'text':\n        return all_contents\n    elif output == 'head':\n        return all_headings\n    else:\n        return all_data","metadata":{"execution":{"iopub.status.busy":"2021-06-20T11:49:10.87856Z","iopub.execute_input":"2021-06-20T11:49:10.878897Z","iopub.status.idle":"2021-06-20T11:49:10.885807Z","shell.execute_reply.started":"2021-06-20T11:49:10.878866Z","shell.execute_reply":"2021-06-20T11:49:10.884949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntqdm.pandas()\nsample_sub['text'] = sample_sub['Id'].progress_apply(partial(read_append_return, train_files_path=test_files_path))","metadata":{"execution":{"iopub.status.busy":"2021-06-20T11:49:13.461145Z","iopub.execute_input":"2021-06-20T11:49:13.461519Z","iopub.status.idle":"2021-06-20T11:49:13.59686Z","shell.execute_reply.started":"2021-06-20T11:49:13.461475Z","shell.execute_reply":"2021-06-20T11:49:13.595817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntqdm.pandas()   #tqdm is used to show any code running with a progress bar. \ntrain_df['text'] = train_df['Id'].progress_apply(read_append_return)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T11:49:16.78888Z","iopub.execute_input":"2021-06-20T11:49:16.789197Z","iopub.status.idle":"2021-06-20T11:50:16.620856Z","shell.execute_reply.started":"2021-06-20T11:49:16.789168Z","shell.execute_reply":"2021-06-20T11:50:16.619749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntqdm.pandas()\ntrain_df['text'] = train_df['text'].progress_apply(text_cleaning)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"words =list( train_df['cleaned_label'].values)\nstopwords=['ourselves', 'hers','the','of','and','in', 'between', 'yourself', 'but', 'again','of', 'there', 'about', 'once', 'during', 'out', 'very', 'having', 'with', 'they', 'own', 'an', 'be', 'some', 'for', 'do', 'its', 'yours', 'such', 'into', 'of', 'most', 'itself', 'other', 'off', 'is', 's', 'am', 'or', 'who', 'as', 'from', 'him', 'each', 'the', 'themselves', 'until', 'below', 'are', 'we', 'these', 'your', 'his', 'through', 'don', 'nor', 'me', 'were', 'her', 'more', 'himself', 'this', 'down', 'should', 'our', 'their', 'while', 'above', 'both', 'up', 'to', 'ours', 'had', 'she', 'all', 'no', 'when', 'at', 'any', 'before', 'them', 'same', 'and', 'been', 'have', 'in', 'will', 'on', 'does', 'yourselves', 'then', 'that', 'because', 'what', 'over', 'why', 'so', 'can', 'did', 'not', 'now', 'under', 'he', 'you', 'herself', 'has', 'just', 'where', 'too', 'only', 'myself', 'which', 'those', 'i', 'after', 'few', 'whom', 't', 'being', 'if', 'theirs', 'my', 'against', 'a', 'by', 'doing', 'it', 'how', 'further', 'was', 'here', 'than']\nsplit_words=[]\nfor word in words:\n    lo_w=[]\n    list_of_words=str(word).split()\n    for w in list_of_words:\n        if w not in stopwords:\n            lo_w.append(w)\n    split_words.append(lo_w)\nallwords = []\nfor wordlist in split_words:\n    allwords += wordlist","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#len(np.unique(allwords)) #187\n#np.unique(allwords)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mostcommon = FreqDist(allwords).most_common(100)\nwordcloud = WordCloud(width=1600, height=800, background_color='black', stopwords=STOPWORDS).generate(str(mostcommon))\nfig = plt.figure(figsize=(30,10), facecolor='white')\nplt.imshow(wordcloud, interpolation=\"bilinear\")\nplt.axis('off')\nplt.title('Top 100 Most Common Words in cleaned_label', fontsize=50)\nplt.tight_layout(pad=0)\nplt.show()\n\nmostcommon_small = FreqDist(allwords).most_common(25)\nx, y = zip(*mostcommon_small)\nplt.figure(figsize=(50,30))\nplt.margins(0.02)\nplt.bar(x, y)\nplt.xlabel('Words', fontsize=50)\nplt.ylabel('Frequency of Words', fontsize=50)\nplt.yticks(fontsize=40)\nplt.xticks(rotation=60, fontsize=40)\nplt.tight_layout(pad=0)\nplt.title('Freq of 25 Most Common Words in cleaned_label', fontsize=60)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Preprocessing","metadata":{}},{"cell_type":"code","source":"labels = train_df['cleaned_label'].tolist()\nprint(len(labels))\nlabels[:5]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sentences = train_df['text'].tolist()\nprint(len(sentences))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer = Tokenizer(oov_token=\"<OOV>\")\ntokenizer.fit_on_texts(sentences)\nword_index = tokenizer.word_index\nprint(len(word_index))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reverse_word_index = dict([(value, key) for (key, value) in word_index.items()])\nprint(reverse_word_index[3])\n\ndef decode_sentence(text):\n    return ' '.join([reverse_word_index.get(i, '?') for i in text])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#vocab_size = 10000\nvocab_size = len(word_index)\nembedding_dim = 16\nmax_length = 100\ntrunc_type='post'\npadding_type='post'\noov_tok = \"<OOV>\"\ntraining_size = 20000","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sentences = train_df['text'].tolist()\n#train_sentences[1]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_sentences = sample_sub['text'].tolist()\n#test_sentences[1]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sequences = tokenizer.texts_to_sequences(train_sentences)\ntrain_padded = pad_sequences(train_sequences, maxlen=max_length, padding=padding_type, truncating=trunc_type)\n\ntest_sequences = tokenizer.texts_to_sequences(test_sentences)\ntest_padded = pad_sequences(test_sequences, maxlen=max_length, padding=padding_type, truncating=trunc_type)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(decode_sentence(train_padded[0]))\n#print(train_sentences[0])\n#print(train_labels[0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels = train_df[\"cleaned_label\"].tolist()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels_sequences = tokenizer.texts_to_sequences(train_labels)\n#train_labels_sequences","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Need this block to get it to work with TensorFlow 2.x\nimport numpy as np\ntrain_padded = np.array(train_padded)\ntrain_labels = np.array(train_labels_sequences)\ntest_padded = np.array(test_padded)\n#test_labels = np.array(test_labels)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train,X_test,y_train,y_test = train_test_split(train_padded,train_labels,test_size=0.1, random_state=0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(X_train.shape,y_train.shape)\nprint(X_test.shape,y_test.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.svm import SVC\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.preprocessing import StandardScaler","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rmse_score(y_true,y_pred):\n    return np.sqrt(mean_squared_error(y_true,y_pred))\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn import preprocessing\nscaler = preprocessing.StandardScaler().fit(train_padded)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"svc = make_pipeline(StandardScaler(), SVC(kernel='linear'))\nsvc.fit(train_padded,train_labels)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict = svc.predict(test_padded)\npredict","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}