{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# What does this notebook do?\nThis notebook records the initial EDA done on the raw training dataset of the Coderidge Inititative kaggle competition.\nFeel free to comment if you have any questions or suggestions. This would keep me motivated. 😄😎\n\nPS: This notebook will be updated with further progress every other day","metadata":{}},{"cell_type":"markdown","source":"## Training data checks","metadata":{}},{"cell_type":"code","source":"import os\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    print(dirname)\n    \nimport pandas as pd\n\ntrain_csv = pd.read_csv(\"/kaggle/input/coleridgeinitiative-show-us-the-data/train.csv\")\n\ntrain_csv.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv.nunique()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#How many records do we have for each label?\n\ntrain_csv.groupby(by='cleaned_label').count()\n\n#data is very disparately distribuced. We are going to need to sample records of each label","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hist_data = train_csv['cleaned_label'].value_counts(sort=True, ascending=True)\nimport matplotlib.pyplot as plt\nhist_data.plot.hist(grid=True, bins=10, rwidth=0.9,\n                   color='#607c8e')\nplt.title('Number of records by data label')\nplt.xlabel('Counts')\nplt.ylabel('Number of records')\nplt.grid(axis='y', alpha=0.75)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Lets check a sample json","metadata":{}},{"cell_type":"code","source":"import json\n\nwith open('/kaggle/input/coleridgeinitiative-show-us-the-data/train/f7e21406-f17f-45b8-b8a8-7b8a0bad0c76.json') as f:\n    train_example = json.load(f)\n    \n    \nprint(train_example)","metadata":{"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Uh oh\n\njsons appear to be too long </br>\nLets just look at how the summary csv looks like first","metadata":{}},{"cell_type":"code","source":"from wordcloud import WordCloud, STOPWORDS\nimport matplotlib.pyplot as plt\n  \n  \ncomment_words = ''\nstopwords = set(STOPWORDS)\n  \n# iterate through the csv file\nfor val in train_csv.pub_title:\n      \n    # typecaste each val to string\n    val = str(val)\n  \n    # split the value\n    tokens = val.split()\n      \n    # Converts each token into lowercase\n    for i in range(len(tokens)):\n        tokens[i] = tokens[i].lower()\n      \n    comment_words += \" \".join(tokens)+\" \"\n  \nwordcloud = WordCloud(width = 800, height = 800,\n                background_color ='white',\n                stopwords = stopwords,\n                min_font_size = 10).generate(comment_words)\n  \n# plot the WordCloud image                       \nplt.figure(figsize = (8, 8), facecolor = None)\nplt.imshow(wordcloud)\nplt.axis(\"off\")\nplt.tight_layout(pad = 0)\n  \nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from wordcloud import WordCloud, STOPWORDS\nimport matplotlib.pyplot as plt\n  \n  \ncomment_words = ''\nstopwords = set(STOPWORDS)\n  \n# iterate through the csv file\nfor val in train_csv.dataset_title:\n      \n    # typecaste each val to string\n    val = str(val)\n  \n    # split the value\n    tokens = val.split()\n      \n    # Converts each token into lowercase\n    for i in range(len(tokens)):\n        tokens[i] = tokens[i].lower()\n      \n    comment_words += \" \".join(tokens)+\" \"\n\nwordcloud = WordCloud(width = 800, height = 800,\n                background_color ='white',\n                stopwords = stopwords,\n                min_font_size = 10).generate(comment_words)\n  \n# plot the WordCloud image                       \nplt.figure(figsize = (8, 8), facecolor = None)\nplt.imshow(wordcloud)\nplt.axis(\"off\")\nplt.tight_layout(pad = 0)\n  \nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from wordcloud import WordCloud, STOPWORDS\nimport matplotlib.pyplot as plt\n  \n  \ncomment_words = ''\nstopwords = set(STOPWORDS)\n  \n# iterate through the csv file\nfor val in train_csv.cleaned_label:\n      \n    # typecaste each val to string\n    val = str(val)\n  \n    # split the value\n    tokens = val.split()\n      \n    # Converts each token into lowercase\n    for i in range(len(tokens)):\n        tokens[i] = tokens[i].lower()\n      \n    comment_words += \" \".join(tokens)+\" \"\n\nwordcloud = WordCloud(width = 800, height = 800,\n                background_color ='white',\n                stopwords = stopwords,\n                min_font_size = 10).generate(comment_words)\n  \n# plot the WordCloud image                       \nplt.figure(figsize = (8, 8), facecolor = None)\nplt.imshow(wordcloud)\nplt.axis(\"off\")\nplt.tight_layout(pad = 0)\n  \nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv.describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv['cleaned_label'].value_counts().head(50)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Now what can we check within the jsons?\n\nNot much I guess, I should start cleaning them to get them prepped for NER and the subsequent model training","metadata":{}}]}