{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport os\nos.listdir('/kaggle/input/coleridgeinitiative-show-us-the-data')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Data provided to us in this competition comprises of following 4 items:\n\n- train.csv - This file comprises of publication-id and the associated data title and labels along with the cleansed label.\n- train folder - This folder provides json file for each of the id's present in the above csv file, with each of the json file describing section-details of the publication and the associated text with it.\n- sample_submission.csv - This file comprises of id, for which we must predict the data-set title.\n- test folder - This folder provides json file for each of the id's present in test.csv file and the json file describes the section details of the publication and the associated text.","metadata":{}},{"cell_type":"code","source":"import pandas as pd\ndf_train=pd.read_csv('/kaggle/input/coleridgeinitiative-show-us-the-data/train.csv')\ndf_train.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Unique Values in each column of the data:\\n\")\nfor i in df_train.columns:\n    print(str(i)+\":\" , df_train[i].nunique())\n    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.isnull().sum()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['dataset_title'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['dataset_label'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test=pd.read_csv('/kaggle/input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\ndf_test","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_input = pd.DataFrame(columns=['id','section_title','text','data_label'])\nids=df_train['Id'].values\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nfor id in ids:\n    df=pd.read_json ('../input/coleridgeinitiative-show-us-the-data/train/{}.json'.format(id))\n    for data_label in df_train[df_train['Id']==id]['dataset_label'].values:\n        new_df=df[df['text'].str.contains(data_label)].copy(deep=True)\n        new_df.loc[:,['data_label']] = data_label\n        new_df.loc[:,['id']] = id\n        new_df.reset_index(inplace=True,drop=True)\n        df_input=pd.concat([df_input, new_df], ignore_index=True)\n        df_input.reset_index(inplace=True,drop=True)\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_input","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_input[\"section_title\"][53043]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Unique Values in each column of the data:\\n\")\nfor i in df_input.columns:\n    print(str(i)+\":\" , df_input[i].nunique())\n    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_input.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_input.to_csv(\"df_input.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df=pd.read_csv(\"./df_input.csv\",index_col=\"Unnamed: 0\")\ndf.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### EDA on the given data","metadata":{}},{"cell_type":"code","source":"from wordcloud import WordCloud, ImageColorGenerator\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport nltk\nimport time\nimport pickle\nimport pyLDAvis.sklearn\nfrom collections import Counter\nfrom textblob import TextBlob\nfrom nltk.tokenize import word_tokenize\nfrom nltk.probability import FreqDist\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.decomposition import LatentDirichletAllocation, NMF\nfrom wordcloud import WordCloud, ImageColorGenerator\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm.autonotebook import tqdm\nimport string\nfrom nltk.tokenize import TweetTokenizer\nfrom nltk.corpus import stopwords\n#Tensorflow Libraries\n\n\nSTOPWORDS = set(stopwords.words('english'))\n\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\nfrom tensorflow.keras.callbacks import EarlyStopping\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Dense, Embedding, LSTM, SpatialDropout1D, Input\n\nimport re","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"words =list( df_input['data_label'].values)\nwords=[word.split() for word in words]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"allwords = []\nfor wordlist in words:\n    allwords += wordlist\n# print(allwords)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mostcommon = FreqDist(allwords).most_common(100)\nmostcommon","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wordcloud = WordCloud(width=1600, height=800, background_color='white').generate(str(mostcommon))\nfig = plt.figure(figsize=(30,10), facecolor='white')\nplt.imshow(wordcloud, interpolation=\"bilinear\")\nplt.axis('off')\nplt.title('Top 100 Most Common Words in Data Label\\n', fontsize=50)\nplt.tight_layout(pad=0)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mostcommon_small = FreqDist(allwords).most_common(25)\nx, y = zip(*mostcommon_small)\nplt.figure(figsize=(50,30))\nplt.margins(0.02)\nplt.bar(x, y)\nplt.xlabel('Words', fontsize=50)\nplt.ylabel('Frequency of Words', fontsize=50)\nplt.yticks(fontsize=40)\nplt.xticks(rotation=60, fontsize=40)\nplt.title('Freq of 25 Most Common Words in Data-Label', fontsize=60)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"words =list( df_input['section_title'].values)\nstopwords=['ourselves', 'hers','the', 'between', 'yourself', 'but', 'again','of', 'there', 'about', 'once', 'during', 'out', 'very', 'having', 'with', 'they', 'own', 'an', 'be', 'some', 'for', 'do', 'its', 'yours', 'such', 'into', 'of', 'most', 'itself', 'other', 'off', 'is', 's', 'am', 'or', 'who', 'as', 'from', 'him', 'each', 'the', 'themselves', 'until', 'below', 'are', 'we', 'these', 'your', 'his', 'through', 'don', 'nor', 'me', 'were', 'her', 'more', 'himself', 'this', 'down', 'should', 'our', 'their', 'while', 'above', 'both', 'up', 'to', 'ours', 'had', 'she', 'all', 'no', 'when', 'at', 'any', 'before', 'them', 'same', 'and', 'been', 'have', 'in', 'will', 'on', 'does', 'yourselves', 'then', 'that', 'because', 'what', 'over', 'why', 'so', 'can', 'did', 'not', 'now', 'under', 'he', 'you', 'herself', 'has', 'just', 'where', 'too', 'only', 'myself', 'which', 'those', 'i', 'after', 'few', 'whom', 't', 'being', 'if', 'theirs', 'my', 'against', 'a', 'by', 'doing', 'it', 'how', 'further', 'was', 'here', 'than']\nsplit_words=[]\nfor word in words:\n    lo_w=[]\n    list_of_words=str(word).split()\n    for w in list_of_words:\n        if w not in stopwords:\n            lo_w.append(w)\n    split_words.append(lo_w)\nallwords = []\nfor wordlist in split_words:\n    allwords += wordlist\nmostcommon = FreqDist(allwords).most_common(100)\nwordcloud = WordCloud(width=1600, height=800, background_color='white').generate(str(mostcommon))\nfig = plt.figure(figsize=(30,10), facecolor='white')\nplt.imshow(wordcloud, interpolation=\"bilinear\")\nplt.axis('off')\nplt.title('Top 100 Most Common Words in Section Title', fontsize=50)\nplt.tight_layout(pad=0)\nplt.show()\n#print(allwords)\nmostcommon_small = FreqDist(allwords).most_common(25)\nx, y = zip(*mostcommon_small)\nplt.figure(figsize=(50,30))\nplt.margins(0.02)\nplt.bar(x, y)\nplt.xlabel('Words', fontsize=50)\nplt.ylabel('Frequency of Words', fontsize=50)\nplt.yticks(fontsize=40)\nplt.xticks(rotation=60, fontsize=40)\nplt.title('Freq of 25 Most Common Words in Section-Title', fontsize=60)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model Building","metadata":{}},{"cell_type":"code","source":"#At first we need to clean the data\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\ndf_input.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Cleaning the text\n\n\ndef cleantext(data,column):\n    '''Make text lowercase, remove text in square brackets,remove links,remove punctuation\n    and remove words containing numbers.'''\n    tweets_clean = []\n    t=-1\n    for i in data[column]:\n        print()\n        i = str(i).lower()\n        i = re.sub('\\[.*?\\]', '', i)\n#         i = re.sub('https?://\\S+|www\\.\\S+', '', i)\n#         i = re.sub('<.*?>+', '', i)\n#         i = re.sub('[%s]' % re.escape(string.punctuation), '', i)\n        i = re.sub('\\n', '', i)\n#         i = re.sub('\\w*\\d\\w*', '', i)\n#         i = re.sub(r'\\$\\w*', '', i)\n        i= re.sub('[^a-zA-Z]',' ',i)\n            # remove old style retweet text \"RT\"\n#         i = re.sub(r'^RT[\\s]+', '', i)\n            # remove hyperlinks\n#         i = re.sub(r'https?:\\/\\/.*[\\r\\n]*', '', i)\n            # remove hashtags\n            # only removing the hash # sign from the word\n        i = re.sub(r'[#@]+', '', i)\n    \n    \n        tokenizer = TweetTokenizer(preserve_case=False, strip_handles=True,\n                                       reduce_len=True)\n        tweet_tokens = tokenizer.tokenize(i)\n        \n            \n            \n        text = [word for word in tweet_tokens if word not in STOPWORDS]\n        text = ' '.join(text)\n        tweets_clean.append(text)\n        t+=1\n        print(f\"{t}th sentence cleaning completed / {len(data[column])} \")\n           \n    \n\n    \n    return tweets_clean","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tqdm.pandas()\nclean_text= cleantext(df_input,\"text\")\ncleaned_text= pd.DataFrame(clean_text, columns=[\"cleaned_text\"])\ncleaned_text.head()\n%time","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_input.insert(3,\"cleaned_text\",cleaned_text[\"cleaned_text\"])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_input.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Splitting the data into dependent and independent variables to apply model.\n\nX= df_input[\"cleaned_text\"]\ny= df_input.data_label.astype(str)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y.nunique()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nle=LabelEncoder()\ny_label=le.fit_transform(y)\ndf_input[\"y_label\"]= y_label\ndf_input.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Arranging data_Label and y_label in the form of dictionary\nlis={}\nxip=zip(df_input[\"data_label\"], df_input[\"y_label\"])\nfor a,b in xip:\n    if a not in lis.keys():\n        lis[a] = b\n    else:\n        pass\n\nlis","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Re-Arranging the dictionary in the ascending order\nlabels={}\nfor k,v in lis.items():\n    labels[v]= k\n    \ny_dummy_labels={}\nfor i in sorted(labels):\n    y_dummy_labels[i] = labels[i]\n    \ny_dummy_labels\n    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.utils import to_categorical\n\nY=to_categorical(y_label,num_classes=108)\nY\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Bidirectional,Activation, Dense, Dropout, Embedding, Flatten, Conv1D, MaxPooling1D, LSTM\nfrom tensorflow.keras import utils\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initializing Hyper params\n\nembedding_dim = 300\nvocab_size=10000\npadding_type='post'\ntrunc_type='post'\noov_tok = \"<OOV>\"\nmax_length=200","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer = Tokenizer(num_words=vocab_size, oov_token=oov_tok) \ntokenizer.fit_on_texts(X)  \nword_index = tokenizer.word_index\n# print(word_index)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = tokenizer.texts_to_sequences(X) \nx = pad_sequences(x, padding = padding_type, maxlen=max_length) \n# print(len(train_sequences[0]))\n# print(len(train_padded[0]))\nprint(x)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(0,4):\n    print(\"sentence: {0}\\nsequences: {1} \".format(X.iloc[i],x[i]))\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_test, y_train, y_test = train_test_split(x, Y, test_size=0.30, random_state=1)\nprint(f\"X_train: {X_train.shape}\")\nprint(f\"y_train: {y_train.shape}\")\nprint(f\"X_test: {X_test.shape}\")\nprint(f\"y_test: {y_test.shape}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Creating model\n# Initializing Hyper params\n\nembedding_dim = 300\nvocab_size=10000\n\nmodel1=Sequential()\nmodel1.add(Embedding(vocab_size,embedding_dim,input_length=max_length))\nmodel1.add(Bidirectional(LSTM(100)))\nmodel1.add(Dropout(0.3))\nmodel1.add(Dense(256,activation = 'relu'))\nmodel1.add(Dense(108,activation='softmax')) # We have chose 108 nodes because oof the number of unique values in the y_label\nmodel1.compile(loss='categorical_crossentropy',optimizer='adam',metrics=['accuracy'])\nprint(model1.summary())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping\nimport tensorflow as tf\ntf.keras.utils.plot_model(model1)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.utils.plot_model(model1, show_shapes=True, dpi=48)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# To avoid overfitting the data we use EarlyStopping\n\n## To avoid overfitting the data we use EarlyStopping\n\nlearning_rate_reduction=ReduceLROnPlateau(monitor='val_acc',patience=3,verbose=1,factor=0.5,min_lr=0.00001)\nearly_stopping= EarlyStopping(monitor='val_loss',min_delta=0, patience=2)\nnum_epochs = 30\nhistory = model1.fit(X_train,y_train,batch_size=128,epochs=num_epochs,\n          validation_split=0.2,callbacks=[learning_rate_reduction,early_stopping])","metadata":{"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig=plt.figure(figsize=(10,5))\nplt.title(\"MODEL PERFORMANCE DURING TRAINING AND EVALUATION\")\nfig1=fig.add_subplot(121)\nplt.plot(history.history['accuracy'],color='Green')\nplt.plot(history.history[\"val_accuracy\"],color='red')\n# plt.ylim(0.90, 1.05)\nplt.xlabel(\"epoch\")\nplt.ylabel(\"accuracies\")\nplt.legend(['Train accuracy','Validation Accuracy'])\n\nfig2=fig.add_subplot(122)\nplt.plot(history.history[\"loss\"],color='g')\nplt.plot(history.history[\"val_loss\"],color='r')\n# plt.ylim(0.0, 1)\nplt.xlabel(\"epoch\")\nplt.ylabel(\"Losses\")\nplt.legend(['Train_Loss','Val_Loss'])\n\nplt.savefig(\"Model Performance.jpg\")\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nscore = model1.evaluate(X_test, y_test, batch_size=128)\nprint()\nprint(\"ACCURACY:\",score[1])\nprint(\"LOSS:\",score[0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict=np.argmax(model1.predict(X_test), axis=-1)\npredict","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"l=[]\nfor i in predict:\n    l.append(y_dummy_labels[i])\n    \npred_labels= np.array(l)\n# pred_label= y_dummy_labels[predict]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Evaluating with the performance metrics (JACCARD RULE)\n\n\n\n# Evaluate it using the metric that they use in this dataset\ndef jaccard(str1, str2): \n    a = set(str1.lower().split()) \n    b = set(str2.lower().split())\n    c = a.intersection(b)\n    return float(len(c)) / (len(a) + len(b) - len(c))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores = []\nfor i in range(pred_labels.shape[0]):\n    pred =str(pred_labels[i])\n    true = str(y_dummy_labels[np.argmax(y_test[i], axis=-1)])\n    scores.append(jaccard(pred, true))\n\nprint(f'Score: {np.mean(scores)}')\n# print(scores)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Saving the model\n# model1.save(\"../input/BI-LSTM.h5\")\nmodel1.save(\"./BI-LSTM.h5\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}