{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# !pip install scispacy\n# !pip install https://s3-us-west-2.amazonaws.com/ai2-s2-scispacy/releases/v0.4.0/en_core_sci_lg-0.4.0.tar.gz","metadata":{"execution":{"iopub.status.busy":"2021-06-10T19:47:01.474387Z","iopub.execute_input":"2021-06-10T19:47:01.474763Z","iopub.status.idle":"2021-06-10T19:47:01.480249Z","shell.execute_reply.started":"2021-06-10T19:47:01.47469Z","shell.execute_reply":"2021-06-10T19:47:01.478481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore', category=DeprecationWarning)\nwarnings.filterwarnings('ignore', category=FutureWarning)\n\nimport os\nfrom tqdm.autonotebook import tqdm\nimport json\nfrom functools import partial\nimport string\nimport re\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport tensorflow as tf\nfrom tensorflow.keras import Sequential, Input\nfrom tensorflow.keras.layers import Dense, Dropout, Conv1D, MaxPooling1D\n\nfrom sklearn.utils import resample, shuffle\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import confusion_matrix\n\nimport nltk\nimport spacy\nfrom nltk.probability import FreqDist\n\nnlp = spacy.load('en_core_web_lg', disable=['parser', 'ner'])\nnlp.max_length = 40000000\n\nos.listdir('/kaggle/input/coleridgeinitiative-show-us-the-data/')\nINPUT_SHAPE = 300\nOUTPUT_SHAPE = 2\n\ntry:\n    nlp.add_pipe(nlp.create_pipe(\"sentencizer\"))\nexcept ValueError:\n    print(\"already has pipe\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-10T19:47:01.488192Z","iopub.execute_input":"2021-06-10T19:47:01.48851Z","iopub.status.idle":"2021-06-10T19:47:19.529744Z","shell.execute_reply.started":"2021-06-10T19:47:01.488481Z","shell.execute_reply":"2021-06-10T19:47:19.528517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Citation worthiness data","metadata":{}},{"cell_type":"code","source":"cite_worthiness_path = '../input/citeworthinesstrainjsonl/cite-worthiness-scaffold-train.jsonl'\ntrain_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\nsample_sub = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\ntrain_files_path = '../input/coleridgeinitiative-show-us-the-data/train'\ntest_files_path = '../input/coleridgeinitiative-show-us-the-data/test'","metadata":{"execution":{"iopub.status.busy":"2021-06-10T19:47:19.531433Z","iopub.execute_input":"2021-06-10T19:47:19.53192Z","iopub.status.idle":"2021-06-10T19:47:19.665839Z","shell.execute_reply.started":"2021-06-10T19:47:19.531848Z","shell.execute_reply":"2021-06-10T19:47:19.664744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cite_worthiness = open(cite_worthiness_path, \"r\").read().split(\"\\n\")[:-1]\nprint(len(cite_worthiness))\nprint(json.loads(cite_worthiness[0]))","metadata":{"execution":{"iopub.status.busy":"2021-06-10T19:47:19.668214Z","iopub.execute_input":"2021-06-10T19:47:19.668738Z","iopub.status.idle":"2021-06-10T19:47:21.184683Z","shell.execute_reply.started":"2021-06-10T19:47:19.668693Z","shell.execute_reply":"2021-06-10T19:47:21.183394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"arr = []\nidx = 0\nfor citation in tqdm(cite_worthiness[:-1]):\n    c = json.loads(citation)\n    arr.append([idx, c['text'], c['cleaned_cite_text'], c['is_citation']])\n    idx+=1\ncite_columns = ['id', 'text', 'cleaned_text', 'is_citation']\nx_df  = pd.DataFrame(arr, columns=cite_columns)\nx_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-10T19:47:21.186551Z","iopub.execute_input":"2021-06-10T19:47:21.186859Z","iopub.status.idle":"2021-06-10T19:47:22.243412Z","shell.execute_reply.started":"2021-06-10T19:47:21.18683Z","shell.execute_reply":"2021-06-10T19:47:22.242074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_append_return(filename,train_files_path=train_files_path, output='text'):\n    \"\"\"\n    Function to read json file and then return the text data from them and append to the dataframe\n    \"\"\"\n    json_path = os.path.join(train_files_path, (filename+'.json'))\n    headings = []\n    contents = []\n    combined = []\n    with open(json_path, 'r') as f:\n        # load json from a single publication\n        json_decode = json.load(f)\n        # for all chapters/sections in a publication\n        for data in json_decode:\n            headings.append(data.get('section_title')) # place all headings of a publication in a list\n            contents.append(data.get('text')) # place all texts of a publication in a list\n            combined.append(data.get('section_title')) # combination of above 2\n            combined.append(data.get('text'))\n    all_headings = ' '.join(headings) # place all headings of a document in\n    all_contents = ' '.join(contents)\n    all_data = '. '.join(combined)\n    \n    if output == 'text':\n        return all_contents\n    if output == 'head':\n        return all_headings\n    else:\n        return all_data\n    ","metadata":{"execution":{"iopub.status.busy":"2021-06-10T19:47:22.245335Z","iopub.execute_input":"2021-06-10T19:47:22.246091Z","iopub.status.idle":"2021-06-10T19:47:22.25676Z","shell.execute_reply.started":"2021-06-10T19:47:22.246043Z","shell.execute_reply":"2021-06-10T19:47:22.255155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntqdm.pandas()   #tqdm is used to show any code running with a progress bar. \n \n# loop through all publications in train folder and add all te text in publication to the train_df['text'] column\ntrain_df['text'] = train_df['Id'].progress_apply(read_append_return)\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-10T19:47:22.258709Z","iopub.execute_input":"2021-06-10T19:47:22.259389Z","iopub.status.idle":"2021-06-10T19:49:00.870183Z","shell.execute_reply.started":"2021-06-10T19:47:22.259345Z","shell.execute_reply":"2021-06-10T19:49:00.86869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"worthy = x_df[x_df.is_citation == True]\nunworthy = x_df[x_df.is_citation == False]","metadata":{"execution":{"iopub.status.busy":"2021-06-10T19:49:00.872369Z","iopub.execute_input":"2021-06-10T19:49:00.873191Z","iopub.status.idle":"2021-06-10T19:49:00.91653Z","shell.execute_reply.started":"2021-06-10T19:49:00.873144Z","shell.execute_reply":"2021-06-10T19:49:00.915243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unworthy_worthy_ration = 8/2\nunworthy_downsampled = resample(unworthy,\n                               replace=False,\n                               n_samples=int(unworthy_worthy_ration*len(worthy)),\n                               random_state=1337)\nx_df_balanced = pd.concat([worthy, unworthy_downsampled])\nx_df_balanced = shuffle(x_df_balanced).reset_index()[['text', 'cleaned_text', 'is_citation']]\nx_df_balanced.head()\n\nprint(len(x_df_balanced[x_df_balanced['is_citation'] == True]))\nprint(len(x_df_balanced[x_df_balanced['is_citation'] == False]))\n\n","metadata":{"execution":{"iopub.status.busy":"2021-06-10T20:54:49.737864Z","iopub.execute_input":"2021-06-10T20:54:49.738259Z","iopub.status.idle":"2021-06-10T20:54:49.78987Z","shell.execute_reply.started":"2021-06-10T20:54:49.738227Z","shell.execute_reply":"2021-06-10T20:54:49.788312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, row in tqdm(train_df.iterrows()):\n    text = row['text']\n    label = row['dataset_label']\n    location = text.find(label)\n    start = location-400 if location > 400 else 0\n    end = location+400\n    text_slice = text[start:end]\n    doc = nlp(text_slice)\n    for sent in doc.sents:\n        if label in sent.text:\n            x_df_balanced = x_df_balanced.append({'text': sent.text, 'cleaned_text':sent.text, 'is_citation':True}, ignore_index=True)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T20:54:50.262414Z","iopub.execute_input":"2021-06-10T20:54:50.262854Z","iopub.status.idle":"2021-06-10T21:00:03.830556Z","shell.execute_reply.started":"2021-06-10T20:54:50.262822Z","shell.execute_reply":"2021-06-10T21:00:03.829063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(x_df_balanced[x_df_balanced['is_citation'] == True]))\nprint(len(x_df_balanced[x_df_balanced['is_citation'] == False]))","metadata":{"execution":{"iopub.status.busy":"2021-06-10T21:00:12.613806Z","iopub.execute_input":"2021-06-10T21:00:12.614173Z","iopub.status.idle":"2021-06-10T21:00:12.639953Z","shell.execute_reply.started":"2021-06-10T21:00:12.614143Z","shell.execute_reply":"2021-06-10T21:00:12.638746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(unworthy_downsampled) / (len(worthy) + len(unworthy_downsampled))","metadata":{"execution":{"iopub.status.busy":"2021-06-10T21:00:13.558105Z","iopub.execute_input":"2021-06-10T21:00:13.558478Z","iopub.status.idle":"2021-06-10T21:00:13.575378Z","shell.execute_reply.started":"2021-06-10T21:00:13.558444Z","shell.execute_reply":"2021-06-10T21:00:13.574068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_dataset(data_df, input_shape, output_shape):\n    X = np.zeros((len(data_df), 1, input_shape))\n    y = np.zeros((len(data_df), 1, output_shape))\n    i = 0\n    for idx, record in tqdm(data_df.iterrows()):\n        X[i] = [nlp(record['cleaned_text']).vector]\n        y[i] = [1.0, 0] if record['is_citation'] else [0, 1.0] # [1.0, 0.0] if citations else [0.0, 1.0] \n        i+=1\n    return X, y","metadata":{"execution":{"iopub.status.busy":"2021-06-10T21:00:19.011924Z","iopub.execute_input":"2021-06-10T21:00:19.012383Z","iopub.status.idle":"2021-06-10T21:00:19.020038Z","shell.execute_reply.started":"2021-06-10T21:00:19.012353Z","shell.execute_reply":"2021-06-10T21:00:19.018801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X, y= make_dataset(x_df_balanced, INPUT_SHAPE, OUTPUT_SHAPE)\nlen(X)\n","metadata":{"execution":{"iopub.status.busy":"2021-06-10T21:00:20.545672Z","iopub.execute_input":"2021-06-10T21:00:20.546149Z","iopub.status.idle":"2021-06-10T21:05:37.00129Z","shell.execute_reply.started":"2021-06-10T21:00:20.54603Z","shell.execute_reply":"2021-06-10T21:05:36.999787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"class CitationWorthinessModel:\n    def __init__(self, input_shape, output_shape, use_dropout = True, dropout_rate = 0.2, nro_hidden_layers=1):\n        self.input_shape = input_shape\n        self.output_shape = output_shape\n        self.use_dropout = use_dropout\n        self.dropout_rate = dropout_rate\n        self.nro_hidden_layers = nro_hidden_layers\n    \n    def make_model_mlp(self) -> Sequential:\n        dropout = self.use_dropout\n        model = Sequential()\n        model.add(Input(shape=(self.input_shape)))\n        for i in range(self.nro_hidden_layers):\n            if dropout:\n                model.add(Dropout(self.dropout_rate))\n            model.add(Dense(60, activation='relu'))\n        model.add(Dense(self.output_shape, activation='softmax'))\n\n        model.compile(\n            optimizer= tf.keras.optimizers.Adam(),  # Optimizer\n            # Loss function to minimize\n            loss=tf.keras.losses.BinaryCrossentropy(),\n            # List of metrics to monitor\n            metrics=[\"accuracy\"]\n        )\n        model.summary()\n        return model\n    \n    def make_model_conv(self) -> Sequential:\n        model = Sequential()\n        model.add(Input(shape=(1, self.input_shape)))\n        \n        \n        model.add(Conv1D(32, 3, padding='same', activation='relu'))\n        model.add(Conv1D(16, 3, padding='same', activation='relu'))\n        model.add(Conv1D(8, 3, padding='same', activation='relu'))\n        model.add(Conv1D(4, 3, padding='same', activation='relu'))\n        model.add(Conv1D(2, 3, padding='same', activation='relu'))\n        \n        \n        model.add(Dense(self.output_shape, activation='softmax'))\n        model.compile(\n            optimizer= tf.keras.optimizers.Adam(),  # Optimizer\n            # Loss function to minimize\n            loss=tf.keras.losses.BinaryCrossentropy(),\n            # List of metrics to monitor\n            metrics=[\"accuracy\"]\n        )\n        return model\n    \n    def train(self, X, y, epochs, batch_size, validation_data=None, verbose=0) -> None:\n        self.model = self.make_model_mlp()\n        if not validation_data:\n            self.model.fit(X, y, epochs=epochs, batch_size=batch_size, verbose=int(verbose))\n        else:\n            self.model.fit(X, y, epochs=epochs, batch_size=batch_size, validation_data=validation_data, verbose=int(verbose))\n        \n    def test(self, X, y, epochs, batch_size, verbose=0):\n        self.confusion_matrix = np.zeros(shape=(2,2))\n        kfold = KFold(n_splits=10, shuffle=True)\n        fold_nr = 1\n        for train_idx, val_idx in kfold.split(X, y):\n            print(f\"Training Fold-{fold_nr}\")\n            self.train(X[train_idx], y[train_idx],\n                       epochs=epochs, batch_size=batch_size,\n                       validation_data=(X[val_idx], y[val_idx]),\n                       verbose=verbose)\n            metrics = self.get_metrics(data=(X[val_idx], y[val_idx]), batch_size=32)\n            prediction = self.predict(data=X[val_idx])\n            self.confusion_matrix+=confusion_matrix(y_true=y[val_idx][:,0],\n                                                    y_pred=np.round()[:,0])\n            fold_nr+=1\n        \n    def get_metrics(self, data, batch_size) -> str:\n        x_val, y_val = data\n        return self.model.evaluate(x_val, y_val, batch_size=batch_size)\n        \n    def predict(self, data) -> list:\n        return self.model.predict(data)\n","metadata":{"execution":{"iopub.status.busy":"2021-06-10T21:09:51.476863Z","iopub.execute_input":"2021-06-10T21:09:51.477296Z","iopub.status.idle":"2021-06-10T21:09:51.499479Z","shell.execute_reply.started":"2021-06-10T21:09:51.477228Z","shell.execute_reply":"2021-06-10T21:09:51.498166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = CitationWorthinessModel(input_shape=INPUT_SHAPE, output_shape=OUTPUT_SHAPE, dropout_rate=0.2, nro_hidden_layers=3)\nmodel.train(X, y, epochs=50, batch_size=8, validation_data=None, verbose=1)\n","metadata":{"execution":{"iopub.status.busy":"2021-06-10T21:09:55.434147Z","iopub.execute_input":"2021-06-10T21:09:55.434595Z","iopub.status.idle":"2021-06-10T21:09:55.442597Z","shell.execute_reply.started":"2021-06-10T21:09:55.434564Z","shell.execute_reply":"2021-06-10T21:09:55.441269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model.test(X, y, epochs=50, batch_size=8, verbose=1)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T21:09:55.768179Z","iopub.execute_input":"2021-06-10T21:09:55.768607Z","iopub.status.idle":"2021-06-10T21:12:00.156224Z","shell.execute_reply.started":"2021-06-10T21:09:55.768577Z","shell.execute_reply":"2021-06-10T21:12:00.153505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_random_set = np.random.randint(len(X), size = 10)\nX_test = X[test_random_set,:]\ny_test = y[test_random_set,:]\nprint(type(X_test))\nprint(np.round(model.predict(X_test)))\ny_test","metadata":{"execution":{"iopub.status.busy":"2021-06-09T10:11:50.988636Z","iopub.status.idle":"2021-06-09T10:11:50.989165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Identifying citation worthy sentences in coleridge initiative.","metadata":{}},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()","metadata":{"execution":{"iopub.status.busy":"2021-06-09T10:11:50.990294Z","iopub.status.idle":"2021-06-09T10:11:50.990814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_1 = pd.unique(train_df['dataset_title'])\ntemp_2 = pd.unique(train_df['dataset_label'])\ntemp_3 = pd.unique(train_df['cleaned_label'])\n\n\nlabels = np.unique(np.concatenate((temp_1, temp_2, temp_3)))\nprint(len(labels))","metadata":{"execution":{"iopub.status.busy":"2021-06-09T10:11:50.991773Z","iopub.status.idle":"2021-06-09T10:11:50.992336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub['text'] = sample_sub['Id'].progress_apply(read_append_return)\nsample_sub","metadata":{"execution":{"iopub.status.busy":"2021-06-09T10:11:50.993425Z","iopub.status.idle":"2021-06-09T10:11:50.993964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def visualize_model_results(text):\n    doc = nlp(text)\n    for sent in doc.sents:\n        vector = [nlp(str(sent)).vector]\n        p = model.predict(np.array([vector,]))\n        p = np.round(p).flatten()\n        worthy = p[0] == 1\n        if worthy:\n            print(worthy, sent)\n\ntxt = sample_sub.iloc[0]['text']\nvisualize_model_results(txt)","metadata":{"execution":{"iopub.status.busy":"2021-06-09T10:11:50.995136Z","iopub.status.idle":"2021-06-09T10:11:50.995705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_sentences_and_vectors(Id):\n    text = sample_sub[sample_sub['Id'] == Id]['text'].iloc[0]\n    doc = nlp(text)\n    found_labels = []\n    for sent in doc.sents:\n        vector = [nlp(str(sent)).vector] # sentence vector\n        p = model.predict(np.array([vector,])) # worthiness prediction\n        p = np.round(p).flatten()\n        worthy = p[0] == 1\n        if worthy: #citation worthy\n            for label in labels:\n                if label in str(sent) and label not in found_labels:\n                    found_labels.append(clean_text(label))\n    return \"|\".join(found_labels)\n\nsample_sub[\"PredictionString\"] = sample_sub['Id'].progress_apply(get_sentences_and_vectors)","metadata":{"execution":{"iopub.status.busy":"2021-06-09T06:22:59.786298Z","iopub.execute_input":"2021-06-09T06:22:59.786633Z","iopub.status.idle":"2021-06-09T06:24:40.632488Z","shell.execute_reply.started":"2021-06-09T06:22:59.786603Z","shell.execute_reply":"2021-06-09T06:24:40.631676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub = sample_sub[['Id', 'PredictionString']]\nsample_sub.to_csv('submission.csv', index=False)\n\nsample_sub.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-09T06:24:40.63399Z","iopub.execute_input":"2021-06-09T06:24:40.634345Z","iopub.status.idle":"2021-06-09T06:24:40.78041Z","shell.execute_reply.started":"2021-06-09T06:24:40.634306Z","shell.execute_reply":"2021-06-09T06:24:40.779644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}