{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":25925,"databundleVersionId":2089362,"sourceType":"competition"}],"dockerImageVersionId":30096,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from tqdm.notebook import tqdm\nfrom wordcloud import WordCloud\nfrom matplotlib import pyplot as plt\nfrom functools import partial\n\nimport pandas as pd\nimport numpy as np\n\nimport json\nimport re\n\npd.set_option('display.max_rows', 1000)\npd.set_option('display.max_columns', 500)\npd.set_option('display.width', 1000)\npd.set_option('display.max_colwidth', None)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-08T18:51:47.695559Z","iopub.execute_input":"2021-06-08T18:51:47.696104Z","iopub.status.idle":"2021-06-08T18:51:47.761771Z","shell.execute_reply.started":"2021-06-08T18:51:47.696033Z","shell.execute_reply":"2021-06-08T18:51:47.760723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_raw_txt(_id, path=\"train\"):\n    _d = json.loads(open(f\"{PATH}/{path}/{_id}.json\").read())\n    return \" \".join([i[\"text\"] for i in _d])\n\ndef clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()","metadata":{"execution":{"iopub.status.busy":"2021-06-08T18:52:12.845207Z","iopub.execute_input":"2021-06-08T18:52:12.845552Z","iopub.status.idle":"2021-06-08T18:52:12.850648Z","shell.execute_reply.started":"2021-06-08T18:52:12.845528Z","shell.execute_reply":"2021-06-08T18:52:12.849529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train set","metadata":{}},{"cell_type":"code","source":"PATH = \"../input/coleridgeinitiative-show-us-the-data/\"\ntqdm_params = dict(bar_format='{desc}{bar} [ {n} / {total} (remaining: {remaining}) ]', colour=\"green\")\n\ntqdm.pandas(desc=\"Text download from JSON files status:\",**tqdm_params)\ndf_train = pd.read_csv(f\"{PATH}train.csv\")\ndf_train['text'] = df_train['Id'].progress_apply(get_raw_txt)\n\nword_model = df_train[['dataset_label','dataset_title','cleaned_label']]\nword_model = word_model.values.reshape((len(df_train)*3,))\nword_model = list(np.unique(word_model))\nprint(\"Word_model created : contains all raw labels (Need to be cleaned before the next steps)\")\n\nfor i in tqdm(range(len(word_model)),desc=\"Word model cleaning using clean_text:\",**tqdm_params):\n    word_model[i] = clean_text(word_model[i])\nword_model = {i.strip() for i in set(word_model)}\n\nprint(f\"Word_model cleaned : now we have the whole list of labels from train set\\n\")\nprint(f\"Word_model unique values : \\033[1m{len(word_model)}\")\n\nwordcloud = WordCloud(max_font_size=15,background_color=\"white\",height=100,scale=3,colormap=\"Greens\")\nwordcloud = wordcloud.generate(\" \".join(word_model))\nplt.figure(figsize=(25,25))\nplt.title(\"World_model top words in labels\\n\")\nplt.imshow(wordcloud)\nplt.axis(\"off\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-08T18:52:14.404776Z","iopub.execute_input":"2021-06-08T18:52:14.405094Z","iopub.status.idle":"2021-06-08T18:53:45.301053Z","shell.execute_reply.started":"2021-06-08T18:52:14.405071Z","shell.execute_reply":"2021-06-08T18:53:45.300140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission set","metadata":{}},{"cell_type":"code","source":"pd.DataFrame(word_model).to_csv(\"wordmodel.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-06-08T18:59:27.765600Z","iopub.execute_input":"2021-06-08T18:59:27.765902Z","iopub.status.idle":"2021-06-08T18:59:27.773070Z","shell.execute_reply.started":"2021-06-08T18:59:27.765878Z","shell.execute_reply":"2021-06-08T18:59:27.772259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tqdm_params = dict(bar_format='{desc}{bar} [ {n} / {total} (remaining: {remaining}) ]', colour=\"darkgreen\")\n\ntqdm.pandas(desc=\"Text download from JSON files status :\",**tqdm_params)\nsubmission = pd.read_csv(f\"{PATH}sample_submission.csv\")\nsubmission['text'] = submission['Id'].progress_apply(partial(get_raw_txt,path=\"test\"))\n\ntqdm.pandas(desc=\"Text cleaning with clean_text function :\",**tqdm_params)\nsubmission['text'] = submission['text'].progress_apply(clean_text)\n\nsubmission['PredictionString'] = ''\nfor i in submission.index:\n    for ws in word_model:\n        if ws in submission.loc[i][\"text\"]:\n            submission.loc[i]['PredictionString'] += ws.strip() + '|'\n    submission.loc[i]['PredictionString'] = submission.loc[i]['PredictionString'][:-1]\n\nsubmission = submission.drop(\"text\",axis=1)\nsubmission.to_csv(\"submission.csv\", index = False)\nsubmission","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}