{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\nimport re\nimport json\nimport string\nimport matplotlib.pyplot as plt\n%matplotlib inline\nfrom tqdm.autonotebook import tqdm\nfrom functools import partial\nfrom wordcloud import WordCloud, STOPWORDS\nimport nltk\nimport spacy\nnlp = spacy.load('en_core_web_lg', disable=['parser', 'ner'])\nnlp.max_length = 4000000\nfrom nltk.probability import FreqDist","metadata":{"execution":{"iopub.status.busy":"2021-06-01T05:45:44.286149Z","iopub.execute_input":"2021-06-01T05:45:44.286658Z","iopub.status.idle":"2021-06-01T05:45:56.561003Z","shell.execute_reply.started":"2021-06-01T05:45:44.286558Z","shell.execute_reply":"2021-06-01T05:45:56.559759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"../input/bigger-govt-dataset-list/data_set_26897.csv\")\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-01T05:45:58.677182Z","iopub.execute_input":"2021-06-01T05:45:58.677608Z","iopub.status.idle":"2021-06-01T05:45:58.805598Z","shell.execute_reply.started":"2021-06-01T05:45:58.677569Z","shell.execute_reply":"2021-06-01T05:45:58.804548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(df.title.unique())","metadata":{"execution":{"iopub.status.busy":"2021-06-01T05:45:59.619917Z","iopub.execute_input":"2021-06-01T05:45:59.620305Z","iopub.status.idle":"2021-06-01T05:45:59.643636Z","shell.execute_reply.started":"2021-06-01T05:45:59.620275Z","shell.execute_reply":"2021-06-01T05:45:59.642661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-01T05:46:01.577392Z","iopub.execute_input":"2021-06-01T05:46:01.577747Z","iopub.status.idle":"2021-06-01T05:46:01.696600Z","shell.execute_reply.started":"2021-06-01T05:46:01.577717Z","shell.execute_reply":"2021-06-01T05:46:01.695632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2021-05-31T22:08:03.823979Z","iopub.execute_input":"2021-05-31T22:08:03.824308Z","iopub.status.idle":"2021-05-31T22:08:03.839033Z","shell.execute_reply.started":"2021-05-31T22:08:03.824279Z","shell.execute_reply":"2021-05-31T22:08:03.838243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.dataset_label[0]","metadata":{"execution":{"iopub.status.busy":"2021-06-01T05:46:43.239276Z","iopub.execute_input":"2021-06-01T05:46:43.239666Z","iopub.status.idle":"2021-06-01T05:46:43.247129Z","shell.execute_reply.started":"2021-06-01T05:46:43.239636Z","shell.execute_reply":"2021-06-01T05:46:43.245865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"le dataset contient : \",len(train), \"élément\")\n[print(col + \":\" + str(len(train[col].unique()))) for col in train.columns ]","metadata":{"execution":{"iopub.status.busy":"2021-06-01T05:48:18.189461Z","iopub.execute_input":"2021-06-01T05:48:18.189852Z","iopub.status.idle":"2021-06-01T05:48:18.219606Z","shell.execute_reply.started":"2021-06-01T05:48:18.189804Z","shell.execute_reply":"2021-06-01T05:48:18.218275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\nsample_sub.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-01T05:48:19.124176Z","iopub.execute_input":"2021-06-01T05:48:19.124632Z","iopub.status.idle":"2021-06-01T05:48:19.146718Z","shell.execute_reply.started":"2021-06-01T05:48:19.124604Z","shell.execute_reply":"2021-06-01T05:48:19.145310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_files_path = '../input/coleridgeinitiative-show-us-the-data/train'\ntest_files_path = '../input/coleridgeinitiative-show-us-the-data/test'","metadata":{"execution":{"iopub.status.busy":"2021-06-01T05:48:21.089132Z","iopub.execute_input":"2021-06-01T05:48:21.089706Z","iopub.status.idle":"2021-06-01T05:48:21.094608Z","shell.execute_reply.started":"2021-06-01T05:48:21.089659Z","shell.execute_reply":"2021-06-01T05:48:21.093163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def json_to_text(filename, train_files_path=train_files_path, output='text'):  \n    json_path = os.path.join(train_files_path, (filename+'.json'))\n    headings = []\n    contents = []\n    combined = []\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title')) #get car dictionnaire\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n    \n    all_headings = ' '.join(headings)\n    all_contents = ' '.join(contents)\n    all_data = '. '.join(combined)\n    \n    if output == 'text':\n        return all_contents\n    elif output == 'head':\n        return all_headings\n    else:\n        return all_data","metadata":{"execution":{"iopub.status.busy":"2021-06-01T05:48:22.422484Z","iopub.execute_input":"2021-06-01T05:48:22.422870Z","iopub.status.idle":"2021-06-01T05:48:22.431398Z","shell.execute_reply.started":"2021-06-01T05:48:22.422839Z","shell.execute_reply":"2021-06-01T05:48:22.430256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def extraire_texte(filename, train_files_path=train_files_path):  \n    json_path = os.path.join(train_files_path, (filename+'.json'))\n    contents = []\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:contents.append(data.get('text'))\n\n    all_contents = ' '.join(contents)\n    return all_contents","metadata":{"execution":{"iopub.status.busy":"2021-06-01T05:48:23.873578Z","iopub.execute_input":"2021-06-01T05:48:23.873963Z","iopub.status.idle":"2021-06-01T05:48:23.881477Z","shell.execute_reply.started":"2021-06-01T05:48:23.873931Z","shell.execute_reply":"2021-06-01T05:48:23.879942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['text'] = train['Id'].apply(extraire_texte)","metadata":{"execution":{"iopub.status.busy":"2021-06-01T05:48:24.846978Z","iopub.execute_input":"2021-06-01T05:48:24.847330Z","iopub.status.idle":"2021-06-01T05:49:27.940336Z","shell.execute_reply.started":"2021-06-01T05:48:24.847298Z","shell.execute_reply":"2021-06-01T05:49:27.939172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-01T05:49:38.326451Z","iopub.execute_input":"2021-06-01T05:49:38.326865Z","iopub.status.idle":"2021-06-01T05:49:38.347004Z","shell.execute_reply.started":"2021-06-01T05:49:38.326835Z","shell.execute_reply":"2021-06-01T05:49:38.345519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.dataset_label[8000]","metadata":{"execution":{"iopub.status.busy":"2021-06-01T06:21:40.634433Z","iopub.execute_input":"2021-06-01T06:21:40.634823Z","iopub.status.idle":"2021-06-01T06:21:40.642840Z","shell.execute_reply.started":"2021-06-01T06:21:40.634775Z","shell.execute_reply":"2021-06-01T06:21:40.641304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\nprint(re.findall(\"Trends in International Mathematics and Science Study.*\",train.text[8000]))","metadata":{"execution":{"iopub.status.busy":"2021-06-01T06:21:54.773707Z","iopub.execute_input":"2021-06-01T06:21:54.774053Z","iopub.status.idle":"2021-06-01T06:21:54.784192Z","shell.execute_reply.started":"2021-06-01T06:21:54.774023Z","shell.execute_reply":"2021-06-01T06:21:54.782660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.text[10000]","metadata":{"execution":{"iopub.status.busy":"2021-06-01T06:02:00.295184Z","iopub.execute_input":"2021-06-01T06:02:00.295592Z","iopub.status.idle":"2021-06-01T06:02:00.304630Z","shell.execute_reply.started":"2021-06-01T06:02:00.295560Z","shell.execute_reply":"2021-06-01T06:02:00.303142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub['text'] = sample_sub['Id'].apply(partial(extraire_texte, train_files_path=test_files_path))","metadata":{"execution":{"iopub.status.busy":"2021-05-31T22:10:42.926361Z","iopub.execute_input":"2021-05-31T22:10:42.926708Z","iopub.status.idle":"2021-05-31T22:10:42.956261Z","shell.execute_reply.started":"2021-05-31T22:10:42.926679Z","shell.execute_reply":"2021-05-31T22:10:42.955493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-31T22:10:44.566012Z","iopub.execute_input":"2021-05-31T22:10:44.566323Z","iopub.status.idle":"2021-05-31T22:10:44.57851Z","shell.execute_reply.started":"2021-05-31T22:10:44.566295Z","shell.execute_reply":"2021-05-31T22:10:44.577334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def text_cleaning(text):\n    \n    text = ''.join([k for k in text if k not in string.punctuation]) # enlever les ponctuation\n    text = re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip() # convertir en minuscule\n    text = re.sub(\"/'+/g\", ' ', text) # enlever les caracteres speciaux \n    text = re.sub(\"\\s+\",\" \", text) # enlever les grandes espaces\n    \n    return text","metadata":{"execution":{"iopub.status.busy":"2021-05-31T22:10:46.056111Z","iopub.execute_input":"2021-05-31T22:10:46.056433Z","iopub.status.idle":"2021-05-31T22:10:46.062318Z","shell.execute_reply.started":"2021-05-31T22:10:46.056402Z","shell.execute_reply":"2021-05-31T22:10:46.061446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['text'] = train['text'].apply(text_cleaning)","metadata":{"execution":{"iopub.status.busy":"2021-05-31T22:10:47.619287Z","iopub.execute_input":"2021-05-31T22:10:47.619619Z","iopub.status.idle":"2021-05-31T22:15:05.868262Z","shell.execute_reply.started":"2021-05-31T22:10:47.619588Z","shell.execute_reply":"2021-05-31T22:15:05.867439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub['text'] = sample_sub['text'].apply(text_cleaning)","metadata":{"execution":{"iopub.status.busy":"2021-05-31T22:15:08.260365Z","iopub.execute_input":"2021-05-31T22:15:08.260754Z","iopub.status.idle":"2021-05-31T22:15:08.345024Z","shell.execute_reply.started":"2021-05-31T22:15:08.260723Z","shell.execute_reply":"2021-05-31T22:15:08.344204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from nltk.tokenize import word_tokenize\n\nwords = word_tokenize(' '.join(list( train['cleaned_label'].values))) \nstopwords=set(STOPWORDS)\nno_stopwords = [ i for i in words if i not in stopwords]","metadata":{"execution":{"iopub.status.busy":"2021-05-31T22:15:51.12629Z","iopub.execute_input":"2021-05-31T22:15:51.126681Z","iopub.status.idle":"2021-05-31T22:15:51.82491Z","shell.execute_reply.started":"2021-05-31T22:15:51.126645Z","shell.execute_reply":"2021-05-31T22:15:51.824056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mots_frequents = FreqDist(no_stopwords).most_common(100)\nwordcloud = WordCloud(width=1600, height=800, background_color='white', stopwords=STOPWORDS).generate(str(mots_frequents))\nfig = plt.figure(figsize=(30,10), facecolor='white')\nplt.imshow(wordcloud, interpolation=\"bilinear\")\nplt.axis('off')\nplt.title('les 100 mots les plus fréquents dans \"cleaned_label\"', fontsize=50)\nplt.tight_layout(pad=0)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-05-31T22:15:53.633005Z","iopub.execute_input":"2021-05-31T22:15:53.633326Z","iopub.status.idle":"2021-05-31T22:15:55.750692Z","shell.execute_reply.started":"2021-05-31T22:15:53.633295Z","shell.execute_reply":"2021-05-31T22:15:55.749776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#def prepare_text(text, nlp=nlp):\n    '''\n    Returns the text after stop-word removal and lemmatization.\n    text - Sentence to be processed\n    nlp - Spacy NLP model\n    '''\n #   doc = nlp(text)\n  #  lemma_list = [token.lemma_ for token in doc if not token.is_stop]\n   # lemmatized_sentence = ' '.join(lemma_list)\n    \n    #return lemmatized_sentence","metadata":{"execution":{"iopub.status.busy":"2021-05-31T22:15:58.486521Z","iopub.execute_input":"2021-05-31T22:15:58.486843Z","iopub.status.idle":"2021-05-31T22:15:58.492704Z","shell.execute_reply.started":"2021-05-31T22:15:58.486815Z","shell.execute_reply":"2021-05-31T22:15:58.491266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_1 = [x.lower() for x in train['dataset_label'].unique()]\nlabel_2 = [x.lower() for x in train['dataset_title'].unique()]\nlabel_3 = [x.lower() for x in train['cleaned_label'].unique()]\nlabel_4 = [x.lower() for x in df.title.unique()]\n\nlabels_existants = set(label_1 + label_2 + label_3 + label_4)\nid_list = []\nlables_list = []\nfor index, row in tqdm(sample_sub.iterrows()):\n    sample_text = row['text']\n    row_id = row['Id']\n    temp_df = train[train['text'] == text_cleaning(sample_text)]\n    cleaned_labels = temp_df['cleaned_label'].to_list()\n    for known_label in labels_existants:\n        if known_label in sample_text.lower():\n            cleaned_labels.append(text_cleaning(known_label))\n    cleaned_labels = [text_cleaning(x) for x in cleaned_labels]\n    cleaned_labels = set(cleaned_labels)\n    lables_list.append('|'.join(cleaned_labels))\n    id_list.append(row_id)","metadata":{"execution":{"iopub.status.busy":"2021-05-31T22:17:16.970066Z","iopub.execute_input":"2021-05-31T22:17:16.97038Z","iopub.status.idle":"2021-05-31T22:17:25.894385Z","shell.execute_reply.started":"2021-05-31T22:17:16.970351Z","shell.execute_reply":"2021-05-31T22:17:25.893368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"resultats = pd.DataFrame()\nresultats['Id'] = id_list\nresultats['PredictionString'] = lables_list\nresultats.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-31T22:17:25.89592Z","iopub.execute_input":"2021-05-31T22:17:25.896273Z","iopub.status.idle":"2021-05-31T22:17:25.90838Z","shell.execute_reply.started":"2021-05-31T22:17:25.896236Z","shell.execute_reply":"2021-05-31T22:17:25.907491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"resultats[\"PredictionString\"][0]","metadata":{"execution":{"iopub.status.busy":"2021-05-31T22:17:33.389391Z","iopub.execute_input":"2021-05-31T22:17:33.389745Z","iopub.status.idle":"2021-05-31T22:17:33.397768Z","shell.execute_reply.started":"2021-05-31T22:17:33.389715Z","shell.execute_reply":"2021-05-31T22:17:33.396432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-05-31T22:17:42.83689Z","iopub.execute_input":"2021-05-31T22:17:42.837207Z","iopub.status.idle":"2021-05-31T22:17:42.856234Z","shell.execute_reply.started":"2021-05-31T22:17:42.837178Z","shell.execute_reply":"2021-05-31T22:17:42.854899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"execution":{"iopub.status.busy":"2021-05-21T16:57:15.090432Z","iopub.execute_input":"2021-05-21T16:57:15.090779Z","iopub.status.idle":"2021-05-21T16:57:15.100358Z","shell.execute_reply.started":"2021-05-21T16:57:15.090741Z","shell.execute_reply":"2021-05-21T16:57:15.099423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}