Coleridge Initiative EDA 🔍 and LSTM Model 📈
","metadata":{}},{"cell_type":"markdown","source":"#Please upvote if you find this notebook useful or interesting, I really appreciate the encouragement. Thanks!
","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n print(os.path.join(dirname, filename))\n\nimport json\nfrom tqdm import tqdm\ntqdm.pandas()\n\nfrom nltk.corpus import stopwords\nfrom unidecode import unidecode\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\nfrom wordcloud import WordCloud\n\nSTOPWORDS = set(stopwords.words('english'))\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.callbacks import EarlyStopping\nfrom keras.models import Model\nfrom keras.layers import Dense, Embedding, LSTM, SpatialDropout1D, Input\n\nimport re\n\nimport seaborn as sns","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-06-08T08:21:28.581136Z","iopub.execute_input":"2021-06-08T08:21:28.581798Z","iopub.status.idle":"2021-06-08T08:21:53.053875Z","shell.execute_reply.started":"2021-06-08T08:21:28.5817Z","shell.execute_reply":"2021-06-08T08:21:53.052942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# reading csv files and train & test file paths\ntrain_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\nsample_sub = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\ntrain_files_path = '../input/coleridgeinitiative-show-us-the-data/train'\ntest_files_path = '../input/coleridgeinitiative-show-us-the-data/test'","metadata":{"execution":{"iopub.status.busy":"2021-06-08T08:21:53.057862Z","iopub.execute_input":"2021-06-08T08:21:53.058184Z","iopub.status.idle":"2021-06-08T08:21:53.205436Z","shell.execute_reply.started":"2021-06-08T08:21:53.058151Z","shell.execute_reply":"2021-06-08T08:21:53.204454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Tabular Exploration","metadata":{}},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-08T08:21:53.208466Z","iopub.execute_input":"2021-06-08T08:21:53.208789Z","iopub.status.idle":"2021-06-08T08:21:53.239288Z","shell.execute_reply.started":"2021-06-08T08:21:53.208762Z","shell.execute_reply":"2021-06-08T08:21:53.238349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.info()","metadata":{"execution":{"iopub.status.busy":"2021-06-08T08:21:53.240528Z","iopub.execute_input":"2021-06-08T08:21:53.240795Z","iopub.status.idle":"2021-06-08T08:21:53.262059Z","shell.execute_reply.started":"2021-06-08T08:21:53.240769Z","shell.execute_reply":"2021-06-08T08:21:53.261239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here we can see that we have no null values to remove as part of the preprocessing.","metadata":{}},{"cell_type":"code","source":"for col in train_df.columns:\n print(f'We have {len(list(set(train_df[col].values)))} unique values in {col}')","metadata":{"execution":{"iopub.status.busy":"2021-06-08T08:21:53.265055Z","iopub.execute_input":"2021-06-08T08:21:53.265488Z","iopub.status.idle":"2021-06-08T08:21:53.281437Z","shell.execute_reply.started":"2021-06-08T08:21:53.265407Z","shell.execute_reply":"2021-06-08T08:21:53.280496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The above analysis shows us that:\n* We have some duplicate ID values\n* We have some duplicate pub_title values\n* We can also see that we have 130 uniqe dataset labels","metadata":{}},{"cell_type":"code","source":"# This method is taken from https://www.kaggle.com/prashansdixit/coleridge-initiative-eda-baseline-model#2.-Data-Exploration%F0%9F%94%8D\n# So please give them an upvote if you find this part useful!\n\ndef read_append_return(filename, train_files_path=train_files_path, output='text'):\n \"\"\"\n Function to read json file and then return the text data from them and append to the dataframe\n \"\"\"\n json_path = os.path.join(train_files_path, (filename+'.json'))\n headings = []\n contents = []\n combined = []\n with open(json_path, 'r') as f:\n json_decode = json.load(f)\n for data in json_decode:\n headings.append(data.get('section_title'))\n contents.append(data.get('text'))\n combined.append(data.get('section_title'))\n combined.append(data.get('text'))\n \n all_headings = ' '.join(headings)\n all_contents = ' '.join(contents)\n all_data = '. '.join(combined)\n \n if output == 'text':\n return all_contents\n elif output == 'head':\n return all_headings\n else:\n return all_data\n \ntrain_df['text'] = train_df['Id'].apply(read_append_return)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-08T08:21:53.283728Z","iopub.execute_input":"2021-06-08T08:21:53.284081Z","iopub.status.idle":"2021-06-08T08:22:54.455745Z","shell.execute_reply.started":"2021-06-08T08:21:53.284024Z","shell.execute_reply":"2021-06-08T08:22:54.454814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Target Distribution\n\nWe will look at the distribution of the target field's values.","metadata":{}},{"cell_type":"code","source":"display(train_df['cleaned_label'].value_counts())\nplt.figure(figsize=(40, 30))\nsns.set(font_scale = 2)\nsns.histplot(train_df['cleaned_label'])\nplt.xticks(rotation = 90)\nplt.show();","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-08T08:22:54.4571Z","iopub.execute_input":"2021-06-08T08:22:54.457485Z","iopub.status.idle":"2021-06-08T08:23:03.521114Z","shell.execute_reply.started":"2021-06-08T08:22:54.457443Z","shell.execute_reply":"2021-06-08T08:23:03.519766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can see from the above that there is a significant imbalance in the classes, which we should addressed before running any models.","metadata":{}},{"cell_type":"markdown","source":"# Text Preprocessing and Exploration\n\nWe will now preprocess the text and explore the results.","metadata":{}},{"cell_type":"code","source":"def clean_text(txt):\n return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower())\n\ndef preprocess_text(text):\n REPLACE_BY_SPACE_RE = re.compile(r'[(){}\\[\\]\\|@,;]')\n \n # Lowercase text\n text = text.lower()\n text = REPLACE_BY_SPACE_RE.sub(' ', text)\n text = text.replace('/', ' / ')\n \n text = unidecode(text).lower()\n text = ' '.join(word for word in text.split() if word not in STOPWORDS)\n \n return text","metadata":{"execution":{"iopub.status.busy":"2021-06-08T08:23:03.522555Z","iopub.execute_input":"2021-06-08T08:23:03.522922Z","iopub.status.idle":"2021-06-08T08:23:03.52978Z","shell.execute_reply.started":"2021-06-08T08:23:03.522887Z","shell.execute_reply":"2021-06-08T08:23:03.528695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['clean_pub_title'] = train_df['pub_title'].progress_apply(clean_text)\n# Note: I have commented out the preprocessing here because it was slowing the notebook down, so for publishing sake I have left this commented out.\ntrain_df['clean_text'] = train_df['text'].progress_apply(clean_text)","metadata":{"execution":{"iopub.status.busy":"2021-06-08T08:23:03.531323Z","iopub.execute_input":"2021-06-08T08:23:03.531764Z","iopub.status.idle":"2021-06-08T08:24:11.361114Z","shell.execute_reply.started":"2021-06-08T08:23:03.531717Z","shell.execute_reply":"2021-06-08T08:24:11.360272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df.to_csv('clean_train.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-08T08:24:11.362116Z","iopub.execute_input":"2021-06-08T08:24:11.362348Z","iopub.status.idle":"2021-06-08T08:24:11.368108Z","shell.execute_reply.started":"2021-06-08T08:24:11.362326Z","shell.execute_reply":"2021-06-08T08:24:11.365058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get word counts and unique word counts\ntrain_df['length'] = train_df['clean_text'].apply(lambda x: len(x.split(' ')))\ntrain_df['unique_word_count'] = train_df['clean_text'].apply(lambda x: len(set(x.split(' '))))","metadata":{"execution":{"iopub.status.busy":"2021-06-08T08:24:11.36991Z","iopub.execute_input":"2021-06-08T08:24:11.370207Z","iopub.status.idle":"2021-06-08T08:24:39.601441Z","shell.execute_reply.started":"2021-06-08T08:24:11.370177Z","shell.execute_reply":"2021-06-08T08:24:39.60053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Distribution plots\ndisplay(train_df['length'].describe())\nplt.figure(figsize=(40, 30))\nax = sns.distplot(train_df['length'].values)\nplt.show();\n\ndisplay(train_df['unique_word_count'].describe())\nplt.figure(figsize=(40, 30))\nax = sns.distplot(train_df['unique_word_count'].values)\nplt.show();","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-08T08:24:39.602709Z","iopub.execute_input":"2021-06-08T08:24:39.60301Z","iopub.status.idle":"2021-06-08T08:24:40.84839Z","shell.execute_reply.started":"2021-06-08T08:24:39.602978Z","shell.execute_reply":"2021-06-08T08:24:40.847681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The above cells shows, as we'd expected, that there is a large imbalance in the distribution of the words where some words appear in almost every single paper. This suggests that some use of TF-IDF might help with the classification.","metadata":{}},{"cell_type":"markdown","source":"## Word Clouds","metadata":{}},{"cell_type":"code","source":"# Define a function to plot word cloud\ndef plot_cloud(wordcloud):\n # Set figure size\n plt.figure(figsize=(40, 30))\n # Display image\n plt.imshow(wordcloud) \n # No axis details\n plt.axis(\"off\");","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-08T08:24:40.849347Z","iopub.execute_input":"2021-06-08T08:24:40.849739Z","iopub.status.idle":"2021-06-08T08:24:40.853389Z","shell.execute_reply.started":"2021-06-08T08:24:40.849702Z","shell.execute_reply":"2021-06-08T08:24:40.852776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### \"Pub Title\" Field Word Cloud","metadata":{}},{"cell_type":"code","source":"# Generate word cloud\nwordcloud = WordCloud(width = 3000, \n height = 2000, \n random_state=1, \n # background_color='salmon', \n colormap='Pastel1', \n collocations=False, \n stopwords = STOPWORDS).generate(' '.join(train_df['clean_pub_title'].values))\n# Plot\nplot_cloud(wordcloud)\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-08T08:25:53.662498Z","iopub.execute_input":"2021-06-08T08:25:53.662847Z","iopub.status.idle":"2021-06-08T08:26:07.259809Z","shell.execute_reply.started":"2021-06-08T08:25:53.662818Z","shell.execute_reply":"2021-06-08T08:26:07.259062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### \"Text\" Field Word Cloud","metadata":{}},{"cell_type":"code","source":"# Generate word cloud\nwordcloud = WordCloud(width = 3000, \n height = 2000, \n random_state=1, \n # background_color='salmon', \n colormap='Pastel1', \n collocations=False, \n stopwords = STOPWORDS).generate(' '.join(train_df['clean_text'].sample(4000).values))\n# Plot\nplot_cloud(wordcloud)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-08T08:26:07.261044Z","iopub.execute_input":"2021-06-08T08:26:07.261417Z","iopub.status.idle":"2021-06-08T08:26:59.398425Z","shell.execute_reply.started":"2021-06-08T08:26:07.261386Z","shell.execute_reply":"2021-06-08T08:26:59.396934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The above word clouds show us that there is still come cleaning required for the two fields but in general, there are some words appearing in these clouds that could be good indicators. ","metadata":{}},{"cell_type":"markdown","source":"# LSTM Model\n\nNormally, I wouldn't dive straight into a LSTM model but I have experience building these and I know they can perform well for these types of tasks.","metadata":{}},{"cell_type":"code","source":"def tokenize(text, MAX_NB_WORDS=1000, MAX_SEQUENCE_LENGTH=25, tokenizer=None):\n #text = text.apply(preprocess_text)\n\n if tokenizer is None:\n tokenizer = Tokenizer(num_words=MAX_NB_WORDS, filters=r'!\"#$%&()*+,-.:;<=>?@[\\]^_`{|}~', lower=True)\n tokenizer.fit_on_texts(text.values)\n\n # word_index = tokenizer.word_index\n\n X = tokenizer.texts_to_sequences(text.values)\n X = pad_sequences(X, maxlen=MAX_SEQUENCE_LENGTH)\n\n return X, tokenizer\n\n\ndef encode_labels(data, target_columns):\n \"\"\"\n Inputs:\n data: All the data we are provided\n target_columns: List of all the columns that are the targets for our model to predict\n Outputs:\n y: One-hot encoding of the true labels for each sample\n \"\"\"\n y = {}\n y_dummy_columns = {}\n for col in target_columns:\n dummies = pd.get_dummies(data[col])\n y_dummy_columns[col] = dummies.columns\n y[col] = dummies.values\n\n return y, y_dummy_columns\n\n\ndef prepare_data(df, feature_column, target_columns, MAX_SEQUENCE_LENGTH=25):\n X, tokenizer = tokenize(df[feature_column], MAX_SEQUENCE_LENGTH=MAX_SEQUENCE_LENGTH, MAX_NB_WORDS=1000)\n\n Y, Y_dummy_columns = encode_labels(df, target_columns)\n\n return X, Y, Y_dummy_columns, tokenizer\n","metadata":{"execution":{"iopub.status.busy":"2021-06-08T08:26:59.400198Z","iopub.execute_input":"2021-06-08T08:26:59.400498Z","iopub.status.idle":"2021-06-08T08:26:59.409385Z","shell.execute_reply.started":"2021-06-08T08:26:59.400471Z","shell.execute_reply":"2021-06-08T08:26:59.407444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train","metadata":{}},{"cell_type":"code","source":"def train_test_split(X, Y, test_size=0.1):\n # Create a mask for the train test split\n np.random.seed(123)\n mask = np.random.rand(X.shape[0]) < (1 - test_size)\n\n # Split the data into train and test\n X_train = X[mask]\n X_test = X[~mask]\n\n y_train = {}\n y_test = {}\n for key, value in Y.items():\n y_train[key] = value[mask]\n y_test[key] = value[~mask]\n\n return X_train, X_test, y_train, y_test, mask","metadata":{"execution":{"iopub.status.busy":"2021-06-08T08:26:59.411289Z","iopub.execute_input":"2021-06-08T08:26:59.411726Z","iopub.status.idle":"2021-06-08T08:26:59.428434Z","shell.execute_reply.started":"2021-06-08T08:26:59.411686Z","shell.execute_reply":"2021-06-08T08:26:59.426978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MAX_NB_WORDS = 3000 # The maximum number of words to be used. (most frequent)\nMAX_SEQUENCE_LENGTH = 25 # Max number of words in each pieice of text\nEMBEDDING_DIM = 300\n\n# Randomly order the samples\ntrain_df = train_df.sample(frac=1).reset_index(drop=True)\n\ntarget_columns = ['cleaned_label']\n\n# Get our inputs and outputs from the data\nX, Y, Y_dummy_columns, tokenizer = prepare_data(train_df, 'clean_text', target_columns, MAX_SEQUENCE_LENGTH=MAX_SEQUENCE_LENGTH)\n\n# Split the inputs and outputs into our train and test sets\nX_train, X_test, y_train, y_test, _ = train_test_split(X, Y, test_size=0.2)\n\ninput_length = X_train.shape[1]\noutput_length = y_train[next(iter(y_train))].shape[1]\noutput_name = list(y_train.keys())[0]\n\ninp = Input(shape=(input_length,))\nx = Embedding(MAX_NB_WORDS, EMBEDDING_DIM)(inp)\nx = SpatialDropout1D(0.2)(x)\nx = LSTM(100, dropout=0.2, recurrent_dropout=0.2)(x)\noutput = Dense(output_length, activation='softmax', name=output_name)(x)\n\nmodel = Model(inp, output)\n\nmodel.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n\ncallbacks = [EarlyStopping(\n monitor='val_loss', patience=3, min_delta=0.0001)]\n\nmodel.fit(X_train, y_train['cleaned_label'], epochs=100,\n batch_size=500,\n validation_split=0.1, callbacks=callbacks)","metadata":{"execution":{"iopub.status.busy":"2021-06-08T08:26:59.431177Z","iopub.execute_input":"2021-06-08T08:26:59.431613Z","iopub.status.idle":"2021-06-08T08:33:50.434395Z","shell.execute_reply.started":"2021-06-08T08:26:59.431558Z","shell.execute_reply":"2021-06-08T08:33:50.433627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Evaluate","metadata":{}},{"cell_type":"code","source":"# Evaluate it using the metric that they use in this dataset\ndef jaccard(str1, str2): \n a = set(str1.lower().split()) \n b = set(str2.lower().split())\n c = a.intersection(b)\n return float(len(c)) / (len(a) + len(b) - len(c))","metadata":{"execution":{"iopub.status.busy":"2021-06-08T08:33:50.435543Z","iopub.execute_input":"2021-06-08T08:33:50.43595Z","iopub.status.idle":"2021-06-08T08:33:50.440086Z","shell.execute_reply.started":"2021-06-08T08:33:50.435921Z","shell.execute_reply":"2021-06-08T08:33:50.439381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = model.predict(X_test)\n\nscores = []\nfor i in range(preds.shape[0]):\n pred = Y_dummy_columns['cleaned_label'].tolist()[np.argmax(preds[i])] \n true = Y_dummy_columns['cleaned_label'].tolist()[np.argmax(y_test['cleaned_label'][i])]\n scores.append(jaccard(pred, true))\n\nprint(f'Score: {np.mean(scores)}')","metadata":{"execution":{"iopub.status.busy":"2021-06-08T08:33:50.441066Z","iopub.execute_input":"2021-06-08T08:33:50.44141Z","iopub.status.idle":"2021-06-08T08:33:51.858172Z","shell.execute_reply.started":"2021-06-08T08:33:50.441384Z","shell.execute_reply":"2021-06-08T08:33:51.857047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The above score is obviously low, but I believe that with some simple tuning, and bring in the additional features we should be able to increase this quite easily. ","metadata":{"trusted":true}},{"cell_type":"markdown","source":"# Closing Remarks\n\nThis is a work in progress and I will be making regular updates to the models to improve their performance.\n\nMy next steps will be:\n* Extract words unique to the papers and feed this into the model\n* Train the LSTM on the text field.\n* Hyperparameter tuning of the LSTM model","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}