{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\n\nfrom collections import Counter, defaultdict\nfrom spacy.lang.en import English","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nnlp=English()\ntrain_df=pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"datalabel_df=train_df.groupby('dataset_label')[['Id']].count().reset_index().rename(columns={\n    'Id': 'Number Of Publications'\n})\n\ndatalabel_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Top 30 appearing datalabels"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15, 5))\nplt.xticks( rotation='vertical')\nsns.set_theme(style=\"dark\")\nsns.barplot(data=datalabel_df.sort_values('Number Of Publications', ascending=False).head(30), \n            x='dataset_label', \n            y='Number Of Publications',\n            ci=False\n           )\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"datalabel_df.sort_values('Number Of Publications', ascending=False).head(30)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Bottom 30 refered dataset labels"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15, 5))\nplt.xticks( rotation='vertical')\nsns.set_theme(style=\"dark\")\nsns.barplot(data=datalabel_df.sort_values('Number Of Publications').head(30), \n            x='dataset_label', \n            y='Number Of Publications',\n            ci=False\n           )\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"datalabel_df.sort_values('Number Of Publications').head(30)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Looking at the most of the dataset labels it looks their topic is on\n1. Alzimers\n2. Education\n3. Covid\n4. Weather"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_tokens(dl):\n    tokens=[]\n    doc=nlp(dl)\n    for token in doc:\n        tokens.append(token)\n    return tokens\n\ndef get_word_shape(tokens):\n    word_shapes=[]\n    for token in tokens:\n        word_shapes.append(token.shape_)\n    return word_shapes\n\ndef get_num_stopwords(tokens):\n    cnt=0\n    for token in tokens:\n        if token.is_stop:\n            cnt+=1\n    return cnt\n\ndef get_stop_words(tokens):\n    stop_words=[]\n    for token in tokens:\n        if token.is_stop:\n            stop_words.append(token)\n    return stop_words\n\ndef get_num_punctuations(tokens):\n    cnt=0\n    for token in tokens:\n        if token.is_punct:\n            cnt+=1\n    return cnt\n\ndef get_punct(tokens):\n    puncts=[]\n    for token in tokens:\n        if token.is_punct:\n            puncts.append(token)\n    return puncts\n\ndef is_title(tokens):\n    return [token.is_title for token in tokens]\n\ndef is_upper(tokens):\n    return [token.is_upper for token in tokens]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"datalabel_df['tokens']=datalabel_df.dataset_label.apply(get_tokens)\ndatalabel_df['word_shape']=datalabel_df.tokens.apply(get_word_shape)\ndatalabel_df['token_len']=datalabel_df.tokens.apply(lambda x:len(x))\ndatalabel_df['num_punct']=datalabel_df.tokens.apply(get_num_punctuations)\ndatalabel_df['puncts']=datalabel_df.tokens.apply(get_punct)\ndatalabel_df['is_title']=datalabel_df.tokens.apply(is_title)\ndatalabel_df['is_upper']=datalabel_df.tokens.apply(is_upper)\ndatalabel_df['num_stopwords']=datalabel_df.tokens.apply(get_num_stopwords)\ndatalabel_df['stopwords']=datalabel_df.tokens.apply(get_stop_words)\n\n\ndatalabel_df['is_start_capital']=datalabel_df.word_shape.apply(lambda ws: ws[0][0] == 'X')\ndatalabel_df['is_start_digit']=datalabel_df.word_shape.apply(lambda ws: ws[0][0] == 'd')\n\n\ndatalabel_df=datalabel_df.sort_values('Number Of Publications', ascending=False)\ndatalabel_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Number Of tokens:', datalabel_df.token_len.nunique() )\n\nplt.figure(figsize=(15, 5))\nplt.xticks( rotation='vertical')\nsns.set_theme(style=\"dark\")\nsns.countplot(data=datalabel_df.sort_values('token_len'),\n              x='token_len')\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"datalabel_df[datalabel_df.token_len==1]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"looks like if token_len==1 --> acronynms."},{"metadata":{},"cell_type":"markdown","source":"# Starts with Capital Letter"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.title('Distribution of Labels Starting With Capital Letters')\nsns.countplot(data=datalabel_df,x='is_start_capital')\nplt.show()\n\n\nprint(\"Percent of Datasets starting with capital letters:\", 100 * datalabel_df.is_start_capital.sum()/len(datalabel_df))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Total Number Of datasets:', len(datalabel_df))\n\nprint('Number Of Datasets starting with Capital Letter:', datalabel_df.is_start_capital.sum())\nprint('Number Of Datasets starting with Digits:', datalabel_df.is_start_digit.sum())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"datalabel_df[(datalabel_df.is_start_capital==False) & (datalabel_df.is_start_digit==False)]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"datalabel_df.num_stopwords.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(datalabel_df.num_stopwords)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"datalabel_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Most Common Word Shapes"},{"metadata":{"trusted":true},"cell_type":"code","source":"wshape_freq=defaultdict(int)\nfor ws_list in datalabel_df.word_shape.values:\n    for wshape in ws_list:\n        wshape_freq[wshape]+=1\n\nwshape_df=pd.DataFrame({\n    'word_shape': list(wshape_freq.keys()),\n    'freq': list(wshape_freq.values()),\n})\n\nwshape_df['wshape_len']=wshape_df.word_shape.apply(lambda x: len(x))\nwshape_df=wshape_df.sort_values('freq')\nwshape_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(12, 5))\nplt.title('Word Shape Frequencies.')\nplt.xticks(rotation='vertical')\nsns.set_theme(style=\"dark\")\nsns.barplot(data=wshape_df, \n            x='word_shape',\n            y='freq'\n           )\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"wshape_df.sort_values('freq', ascending=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Most common unigram word-shape features \n1. Xx(m) --> Starts with Capital and followed by small\n2. X(m) --> all the words are capital\n3. x(m) --> all the words are small(this will come in conjunction with the other blocks as we observe that ~93% of cases starts with capital letters)"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.title('Word Shape Length:')\nsns.countplot(data=wshape_df, x='wshape_len')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"wshape_df[wshape_df.wshape_len>6]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Furthur Work:\n1. Bigram of wordshapes\n2. Getting Accronyms from word phrases.\n3. Overlapping datasets"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}