{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/coleridge-initiative-preprocessed-to-csv-data/df_train_publications.csv')\ntest = pd.read_csv('../input/coleridge-initiative-preprocessed-to-csv-data/df_test_publications.csv')\nlabels = pd.read_csv('../input/coleridge-initiative-preprocessed-to-csv-data/labels.csv')\ntrain_meta = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\n\ndef concat(column):\n    res = ' '\n    for st in column:\n        if type(st) == str:\n            res += st\n    return res\n\ntrain_gr = train.groupby('pub_id')['text'].apply(concat).reset_index()\ntrain_gr.loc[train['pub_id'].isin(train_meta['Id']), 'cleaned_label'] = train_meta.loc[train_meta['Id'].isin(train_gr['pub_id']),'cleaned_label']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## NER Feature extraction"},{"metadata":{},"cell_type":"markdown","source":"We will use spacy pre-trained model to extract some features."},{"metadata":{"trusted":true},"cell_type":"code","source":"# !python3 -m spacy download en_core_web_lg\nimport spacy\nsp_lg = spacy.load('en_core_web_lg')\n\ndocument = train_gr.loc[0, 'text']\nprint({(ent.text.strip(), ent.label_) for ent in sp_lg(document).ents})","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Here we have to decide what entities are important for us, for example, organizations, and go through the train_gr with the loop."},{"metadata":{},"cell_type":"markdown","source":"## Count Vectorizer and dimensionality reduction"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer\n\nvectorizer = CountVectorizer()\ncorpus = train_gr['text']\n\nX = vectorizer.fit_transform(corpus)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vectorizer_label = CountVectorizer()\nlabels_corpus = train_gr['cleaned_label']\ny = vectorizer_label.fit_transform(labels_corpus)\n\ny.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.decomposition import TruncatedSVD\n\nsvd = TruncatedSVD(n_components=100, n_iter=7, random_state=42)\nsvd.fit(X)\nX_tr = svd.transform(X)\n\nsvd_y = TruncatedSVD(n_components=20, n_iter=7, random_state=42)\nsvd_y.fit(y)\ny_tr = svd_y.transform(y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_tr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_tr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_tr.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_tr.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Also we can use, for example, TfIdf Vectorizer."},{"metadata":{},"cell_type":"markdown","source":"## Some pre-trained tokenizers"},{"metadata":{"trusted":true},"cell_type":"code","source":"from transformers import AutoTokenizer\ntokenizer = AutoTokenizer.from_pretrained('bert-base-cased')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"encoded_input = tokenizer(train_gr.loc[0, 'text'])\nprint(encoded_input)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Gensim Word2Vec"},{"metadata":{"trusted":true},"cell_type":"code","source":"import gensim\nw2v_model = gensim.models.Word2Vec(train_gr.loc[:10, 'text'].str.split().to_list(), size=100, min_count=1, window=5, iter=50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"w2v_model.train(train_gr.loc[:10, 'text'].str.split().to_list(), total_examples=1, epochs=50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_gr.loc[0, 'text']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"w2v_model.wv['DOT', 'Lebanon\\'s' ,'ICT' ,'training', 'program']","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}