{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport csv\nimport matplotlib.pyplot as plt\npd.set_option('display.max_colwidth',None)# set options such that the column width is maxed.\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"data=pd.read_csv(\"../input/facebook-recruiting-iii-keyword-extraction/Train.zip\")\ndata.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.columns","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Removing Duplicates","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"data.drop_duplicates(subset=['Title', 'Body', 'Tags'],keep='first',inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.shape ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## The shape of the data decresed from 6 million to 4 million so duplicated data is about 1/3 of total data.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"data.dropna(inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(data.shape)\ndata.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer\nvec=CountVectorizer(tokenizer= lambda x:x.split(),binary=True)\ntags_vec=vec.fit_transform(data.Tags)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tags_vec.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tags=vec.get_feature_names()\ntags_data=pd.DataFrame(tags,columns=['tags'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tags_data['counts']=tags_vec.sum(axis=0).A1\ntags_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tags_sorted=tags_data.sort_values(['counts'],ascending=False)\ncounts=tags_sorted['counts'].values\nplt.plot(counts)\nplt.grid()\nplt.xlabel('tag_no')\nplt.ylabel('frequency')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(counts[:10000])\nplt.grid()\nplt.xlabel('tag_no')\nplt.ylabel('frequency')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(counts[:1000])\nplt.grid()\nplt.xlabel('tag_no')\nplt.ylabel('frequency')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(counts[:500])\nplt.grid()\nplt.xlabel('tag_no')\nplt.ylabel('frequency')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from matplotlib.pyplot import figure\nfigure(num=None, figsize=(10, 10), dpi=80, facecolor='w', edgecolor='k')\nplt.plot(counts[:100],c='b')\nplt.scatter(x=list(range(0,100,5)),y=counts[0:100:5],c='orange',label='quant 0.05')\nplt.scatter(x=list(range(0,100,25)),y=counts[0:100:25],c='m',label='quant 0.25')\nfor index,count,tag_name in zip(list(range(0,100,25)),counts[0:100:25],tags_sorted['tags'].values[0:100:25]):\n    plt.annotate(s=str(tag_name)+\", is tagged \"+str(count)+\" times\",xy=(index,count),xytext=(index+1,count+1000))\nplt.grid()\nplt.legend()\nplt.xlabel('tag_no')\nplt.ylabel('frequency')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from wordcloud import WordCloud, ImageColorGenerator\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nimport numpy as np\nwc=WordCloud().generate_from_frequencies(tags_dict)\nfig=plt.figure(figsize=(30,20))\nplt.imshow(wc)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"i=np.arange(20)\ntags_sorted['counts'].head(20).plot(kind='bar')\nplt.xticks(ticks=np.arange(20),labels=list(tags_sorted['tags'].values[:20]))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# sampling 1 million data points","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"data=data.sample(n=1000000)\ndata.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.to_csv('data_1_million.csv')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}