{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-22T19:03:00.281397Z","iopub.execute_input":"2021-06-22T19:03:00.281812Z","iopub.status.idle":"2021-06-22T19:03:11.521815Z","shell.execute_reply.started":"2021-06-22T19:03:00.281726Z","shell.execute_reply":"2021-06-22T19:03:11.520911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train=pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\ntrain","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:03:41.700341Z","iopub.execute_input":"2021-06-22T19:03:41.700714Z","iopub.status.idle":"2021-06-22T19:03:41.872353Z","shell.execute_reply.started":"2021-06-22T19:03:41.700683Z","shell.execute_reply":"2021-06-22T19:03:41.871331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample=pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\nsample","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:04:19.430353Z","iopub.execute_input":"2021-06-22T19:04:19.430867Z","iopub.status.idle":"2021-06-22T19:04:19.449212Z","shell.execute_reply.started":"2021-06-22T19:04:19.430836Z","shell.execute_reply":"2021-06-22T19:04:19.448459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport json\nfrom pprint import pprint\nfrom copy import deepcopy\n\nimport numpy as np\nimport pandas as pd\nfrom tqdm.notebook import tqdm","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:05:17.914657Z","iopub.execute_input":"2021-06-22T19:05:17.915004Z","iopub.status.idle":"2021-06-22T19:05:17.922789Z","shell.execute_reply.started":"2021-06-22T19:05:17.914975Z","shell.execute_reply":"2021-06-22T19:05:17.921470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dir = r\"../input/coleridgeinitiative-show-us-the-data/train\"\nfilenames = os.listdir(train_dir)\nprint(\"Number of articles retrieved from train:\", len(filenames))","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:06:49.041149Z","iopub.execute_input":"2021-06-22T19:06:49.041575Z","iopub.status.idle":"2021-06-22T19:06:49.059080Z","shell.execute_reply.started":"2021-06-22T19:06:49.041542Z","shell.execute_reply":"2021-06-22T19:06:49.057763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count = 0\nfile_exts = []\nfor dirname, _, filenames in os.walk(r\"../input/coleridgeinitiative-show-us-the-data/train\"):\n    for filename in filenames:\n        count += 1\n        file_ext = filename.split(\".\")[-1]\n        file_exts.append(file_ext)\n\nfile_ext_set = set(file_exts)\n\nprint(f\"Files: {count}\")\nprint(f\"Files extensions: {file_ext_set}\\n\\n=====================\\nFiles extension count:\\n=====================\")\nfile_ext_list = list(file_ext_set)\nfor fe in file_ext_list:\n    fe_count = file_exts.count(fe)\n    print(f\"{fe}: {fe_count}\")","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:12:00.998291Z","iopub.execute_input":"2021-06-22T19:12:00.998657Z","iopub.status.idle":"2021-06-22T19:12:01.045053Z","shell.execute_reply.started":"2021-06-22T19:12:00.998627Z","shell.execute_reply":"2021-06-22T19:12:01.044061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count = 0\nfor root, folders, filenames in os.walk(r\"../input/coleridgeinitiative-show-us-the-data/train\"):\n    print(root, folders)","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:12:44.911605Z","iopub.execute_input":"2021-06-22T19:12:44.912148Z","iopub.status.idle":"2021-06-22T19:12:44.945753Z","shell.execute_reply.started":"2021-06-22T19:12:44.912105Z","shell.execute_reply":"2021-06-22T19:12:44.944680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"json_folder_path = r\"../input/coleridgeinitiative-show-us-the-data/train\"\njson_file_name = os.listdir(json_folder_path)[0]\njson_path = os.path.join(json_folder_path, json_file_name)\n\nwith open(json_path) as json_file:\n    json_data = json.load(json_file)","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:13:27.279131Z","iopub.execute_input":"2021-06-22T19:13:27.279623Z","iopub.status.idle":"2021-06-22T19:13:27.302840Z","shell.execute_reply.started":"2021-06-22T19:13:27.279592Z","shell.execute_reply":"2021-06-22T19:13:27.301823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"json_data_df = pd.io.json.json_normalize(json_data)","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:13:47.970137Z","iopub.execute_input":"2021-06-22T19:13:47.970535Z","iopub.status.idle":"2021-06-22T19:13:47.978232Z","shell.execute_reply.started":"2021-06-22T19:13:47.970497Z","shell.execute_reply":"2021-06-22T19:13:47.977458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"json_data_df","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:14:09.295961Z","iopub.execute_input":"2021-06-22T19:14:09.296354Z","iopub.status.idle":"2021-06-22T19:14:09.310549Z","shell.execute_reply.started":"2021-06-22T19:14:09.296314Z","shell.execute_reply":"2021-06-22T19:14:09.309643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Files in folder: {len(os.listdir(json_folder_path))}\")","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:14:35.870025Z","iopub.execute_input":"2021-06-22T19:14:35.870398Z","iopub.status.idle":"2021-06-22T19:14:35.884111Z","shell.execute_reply.started":"2021-06-22T19:14:35.870362Z","shell.execute_reply":"2021-06-22T19:14:35.883212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\n\n# to process all files, uncomment the next line and comment the line below\n# list_of_files = list(os.listdir(json_folder_path))\nlist_of_files = list(os.listdir(json_folder_path))[0:500]\ntrain_df = pd.DataFrame()\n\nfor file in tqdm(list_of_files):\n    json_path = os.path.join(json_folder_path, file)\n    with open(json_path) as json_file:\n        json_data = json.load(json_file)\n    json_data_df = pd.io.json.json_normalize(json_data)\n    train_df = train_df.append(json_data_df)","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:16:44.949833Z","iopub.execute_input":"2021-06-22T19:16:44.950194Z","iopub.status.idle":"2021-06-22T19:16:48.074525Z","shell.execute_reply.started":"2021-06-22T19:16:44.950162Z","shell.execute_reply":"2021-06-22T19:16:48.072771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:17:28.815227Z","iopub.execute_input":"2021-06-22T19:17:28.815592Z","iopub.status.idle":"2021-06-22T19:17:28.826549Z","shell.execute_reply.started":"2021-06-22T19:17:28.815561Z","shell.execute_reply":"2021-06-22T19:17:28.825586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.max_colwidth', 500)\ntrain_df[['text', 'section_title']].head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:22:30.387566Z","iopub.execute_input":"2021-06-22T19:22:30.387913Z","iopub.status.idle":"2021-06-22T19:22:30.400111Z","shell.execute_reply.started":"2021-06-22T19:22:30.387885Z","shell.execute_reply":"2021-06-22T19:22:30.399348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from wordcloud import WordCloud, STOPWORDS\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm_notebook\n%matplotlib inline \nstopwords = set(STOPWORDS)\n\ndef show_wordcloud(data, title = None):\n    wordcloud = WordCloud(\n        background_color='white',\n        stopwords=stopwords,\n        max_words=200,\n        max_font_size=40, \n        scale=5,\n        random_state=1\n    ).generate(str(data))\n\n    fig = plt.figure(1, figsize=(15,15))\n    plt.axis('off')\n    if title: \n        fig.suptitle(title, fontsize=14)\n        fig.subplots_adjust(top=2.3)\n\n    plt.imshow(wordcloud)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:23:44.772377Z","iopub.execute_input":"2021-06-22T19:23:44.772720Z","iopub.status.idle":"2021-06-22T19:23:44.832958Z","shell.execute_reply.started":"2021-06-22T19:23:44.772691Z","shell.execute_reply":"2021-06-22T19:23:44.831979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_wordcloud(train_df['text'], title = 'train_df- frequent words  (500 samples)')","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:25:35.089707Z","iopub.execute_input":"2021-06-22T19:25:35.090271Z","iopub.status.idle":"2021-06-22T19:25:36.308805Z","shell.execute_reply.started":"2021-06-22T19:25:35.090228Z","shell.execute_reply":"2021-06-22T19:25:36.307992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\ndef plot_count(feature, title, df, size=1, show_percents=False):\n    f, ax = plt.subplots(1,1, figsize=(4*size,4))\n    total = float(len(df))\n    g = sns.countplot(df[feature], order = df[feature].value_counts().index[0:20], palette='Set3')\n    g.set_title(\"Number of {}\".format(title))\n    if(size > 2):\n        plt.xticks(rotation=90, size=10)\n    if(show_percents):\n        for p in ax.patches:\n            height = p.get_height()\n            ax.text(p.get_x()+p.get_width()/2.,\n                    height + 3,\n                    '{:1.2f}%'.format(100*height/total),\n                    ha=\"center\") \n    ax.set_xticklabels(ax.get_xticklabels());\n    plt.show()    \n","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:26:35.166400Z","iopub.execute_input":"2021-06-22T19:26:35.166967Z","iopub.status.idle":"2021-06-22T19:26:36.153711Z","shell.execute_reply.started":"2021-06-22T19:26:35.166918Z","shell.execute_reply":"2021-06-22T19:26:36.152698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_count('text', 'train_df - Top 20 Journals (500 samples)', train_df, 3.5)","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:27:47.581060Z","iopub.execute_input":"2021-06-22T19:27:47.581443Z","iopub.status.idle":"2021-06-22T19:27:51.268281Z","shell.execute_reply.started":"2021-06-22T19:27:47.581389Z","shell.execute_reply":"2021-06-22T19:27:51.267070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dir =  r\"../input/coleridgeinitiative-show-us-the-data/test/\"\n","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:32:26.923820Z","iopub.execute_input":"2021-06-22T19:32:26.924194Z","iopub.status.idle":"2021-06-22T19:32:26.929108Z","shell.execute_reply.started":"2021-06-22T19:32:26.924164Z","shell.execute_reply":"2021-06-22T19:32:26.927787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(test_dir)\n","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:34:02.525272Z","iopub.execute_input":"2021-06-22T19:34:02.525647Z","iopub.status.idle":"2021-06-22T19:34:02.530598Z","shell.execute_reply.started":"2021-06-22T19:34:02.525614Z","shell.execute_reply":"2021-06-22T19:34:02.529547Z"},"trusted":true},"execution_count":null,"outputs":[]}]}