{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport re\nimport json\nimport pandas as pd\nimport numpy as np\nimport warnings\nimport csv\n\nwarnings.filterwarnings('ignore')\n\nprint(os.listdir('../input/coleridgeinitiative-show-us-the-data'))\nprint(\"Files in train directory : \\n\")\nprint(os.listdir('../input/coleridgeinitiative-show-us-the-data/train')[:5])\nprint(\"\\nFiles in test directory : \\n\")\nprint(os.listdir('../input/coleridgeinitiative-show-us-the-data/test')[:5])\n\n%load_ext autoreload\n%autoreload 2\nfrom utility_script import *","metadata":{"execution":{"iopub.status.busy":"2021-05-28T23:03:09.420739Z","iopub.execute_input":"2021-05-28T23:03:09.42113Z","iopub.status.idle":"2021-05-28T23:03:15.491309Z","shell.execute_reply.started":"2021-05-28T23:03:09.421037Z","shell.execute_reply":"2021-05-28T23:03:15.490268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Following code generates the CSV file with tagged lined","metadata":{}},{"cell_type":"code","source":"%%time\nfrom tqdm import tqdm\ndata = Data()\n# train_data = data.read_all_files(read_train=True, max_files_to_read= 3)\nfor article_id, article_json  in tqdm(data.yield_data(read_train=True), total=20000):\n    article_labels = data.assign_label_to_article_json(article_id, article_json, keep_all=False)","metadata":{"execution":{"iopub.status.busy":"2021-05-28T23:03:15.493044Z","iopub.execute_input":"2021-05-28T23:03:15.493444Z","iopub.status.idle":"2021-05-28T23:03:31.225062Z","shell.execute_reply.started":"2021-05-28T23:03:15.493406Z","shell.execute_reply":"2021-05-28T23:03:31.224348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls","metadata":{"execution":{"iopub.status.busy":"2021-05-25T23:04:56.82486Z","iopub.execute_input":"2021-05-25T23:04:56.825222Z","iopub.status.idle":"2021-05-25T23:04:57.640521Z","shell.execute_reply.started":"2021-05-25T23:04:56.825193Z","shell.execute_reply":"2021-05-25T23:04:57.639172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.read_csv('train.csv').shape","metadata":{"execution":{"iopub.status.busy":"2021-05-28T23:03:35.611216Z","iopub.execute_input":"2021-05-28T23:03:35.61184Z","iopub.status.idle":"2021-05-28T23:03:35.704492Z","shell.execute_reply.started":"2021-05-28T23:03:35.611794Z","shell.execute_reply":"2021-05-28T23:03:35.703568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.read_csv('train.csv').head(5)","metadata":{},"execution_count":null,"outputs":[]}]}