{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip3 install ../input/datasketch/datasketch-1.5.3-py2.py3-none-any.whl","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Import libraries"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport re\nimport json\nimport gc\nfrom itertools import repeat\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom wordcloud import WordCloud, STOPWORDS\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom datasketch import MinHash, MinHashLSHForest","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Helper functions"},{"metadata":{},"cell_type":"markdown","source":"Below clean_text function should be used to clean text as mentioned on the Evaluation page"},{"metadata":{"trusted":true},"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"source_path = '../input/coleridgeinitiative-show-us-the-data'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(f'{source_path}/train.csv')\nsample_submission_df = pd.read_csv(f'{source_path}/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Cleaning publication title"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['clean_pub_title'] = train_df.pub_title.apply(lambda x: clean_text(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## WordCloud"},{"metadata":{"trusted":true},"cell_type":"code","source":"pub_title_data = ' '.join(i for i in train_df['clean_pub_title'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"wordcloud = WordCloud(width = 3000, height = 2000, random_state=1, background_color='black', colormap='Set2', collocations=False, stopwords = STOPWORDS).generate(pub_title_data)\nplt.figure(figsize=(40, 30))\nplt.imshow(wordcloud)  \nplt.axis(\"off\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#"},{"metadata":{"trusted":true},"cell_type":"code","source":"vectorizer = CountVectorizer(stop_words='english')\ncount_m = vectorizer.fit_transform(train_df['clean_pub_title'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"count_df = pd.DataFrame({'tokens': vectorizer.get_feature_names(), 'count': count_m.toarray().sum(axis=0).tolist()})\ncount_df.sort_values(by='count', ascending=True, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"count_df.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"count_df.tail(10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Top 50 tokens with less frequency within publication title"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize = (15, 15))\nsns.pointplot(x=count_df['tokens'][:50], y=count_df['count'][:50], linestyles=\"-\")\nplt.xlabel(\"tokens\")\nplt.ylabel(\"frequency\")\nplt.xticks(rotation=90)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Top 50 tokens with high frequency within publication title"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize = (15, 15))\nsns.pointplot(x=count_df['tokens'][-50:], y=count_df['count'][-50:], color = \"green\", linestyles=\"-\")\nplt.xlabel(\"tokens\")\nplt.ylabel(\"frequency\")\nplt.xticks(rotation=90)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Calculating length of the publication title"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['pub_title_len'] = train_df.clean_pub_title.apply(lambda x: len(x))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Publication title with longest length"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f'Length: {len(train_df.iloc[train_df.pub_title_len.argmax()].clean_pub_title)}')\nprint(f'Publication title: {train_df.iloc[train_df.pub_title_len.argmax()].clean_pub_title}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"temp_df = train_df[['pub_title_len', 'clean_pub_title']].copy()\ntemp_df.sort_values(by='pub_title_len', ascending=True, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Top 50 publication title with short length"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize = (15, 15))\nsns.pointplot(x=temp_df.index[:50], y=temp_df['pub_title_len'][:50], linestyles=\"-\")\nplt.xlabel(\"row index\")\nplt.ylabel(\"title length\")\nplt.xticks(rotation=90)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Top 50 publication title with longest length"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize = (15, 15))\nsns.pointplot(x=temp_df.index[-50:], y=temp_df['pub_title_len'][-50:], linestyles=\"-\")\nplt.xlabel(\"row index\")\nplt.ylabel(\"title length\")\nplt.xticks(rotation=90)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Merge JSON files"},{"metadata":{},"cell_type":"markdown","source":"### Train - JSON to DataFrame"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_json_source = f'{source_path}/train'\ntest_json_source = f'{source_path}/test'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntemp_json_list = []\ntemp_file_name = []\ntemp_label = []\nfor json_data in os.listdir(train_json_source):\n    temp_json_list.append(pd.read_json(f'{train_json_source}/{json_data}', orient='records'))\n    with open(f'{train_json_source}/{json_data}', 'r') as f:\n        temp_data = json.load(f)\n        temp_file_name.extend(repeat(json_data.replace('.json', ''), len(temp_data)))\n        temp_label.extend(repeat(train_df.loc[train_df['Id'] == json_data.replace('.json', '')].cleaned_label.to_list()[0], len(temp_data)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_json = pd.concat(temp_json_list, ignore_index=True)\ntrain_json['file'] = temp_file_name\ntrain_json['cleaned_label'] = temp_label\ntrain_json.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del temp_json_list, temp_file_name\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_groups = train_json.groupby(train_json['file'])\ntrain_groups = train_groups.apply(lambda train_json: train_json.sort_values(by=['file']))\ntrain_groups.drop(['file'], inplace=True, axis=1)\ntrain_groups.to_csv('train_json.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_groups","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train_groups","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Test - JSON to DataFrame"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntemp_json_list = []\ntemp_file_name = []\ntemp_label = []\nfor json_data in os.listdir(test_json_source):\n    temp_json_list.append(pd.read_json(f'{test_json_source}/{json_data}', orient='records'))\n    with open(f'{test_json_source}/{json_data}', 'r') as f:\n        temp_data = json.load(f)\n        temp_file_name.extend(repeat(json_data.replace('.json', ''), len(temp_data)))\n        temp_label.extend(repeat(train_df.loc[train_df['Id'] == json_data.replace('.json', '')].cleaned_label.to_list()[0], len(temp_data)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(temp_label), len(temp_file_name)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_json = pd.concat(temp_json_list, ignore_index=True)\ntest_json['file'] = temp_file_name\ntest_json['cleaned_label'] = temp_label\ntest_json.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del temp_json_list, temp_file_name\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_json.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_groups = test_json.groupby(test_json['file'])\ntest_groups = test_groups.apply(lambda test_json: test_json.sort_values(by=['file']))\ntest_groups.drop(['file'], inplace=True, axis=1)\ntest_groups.to_csv('test_json.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_groups","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del test_groups","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_json['clean_text'] = train_json.text.apply(lambda x: clean_text(x))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### LSH"},{"metadata":{},"cell_type":"markdown","source":"* Setting number of permutations\n* Setting number of recommendations to return  \n* Setting depth of LSH Forest\n* Preparing shingles\n* MinHashing all the shingles\n* Preparing MinHashForest of MinHash\n* Indexing forest\n* Querying forest\n* Calculating jaccard similarity or cosine similarity (Post-processing)"},{"metadata":{},"cell_type":"markdown","source":"Reference: http://ekzhu.com/datasketch/"},{"metadata":{"trusted":true},"cell_type":"code","source":"class LSH:\n    def __init__(self, permutations, number_of_recommendations, depth, dataframe):\n        self.permutations = permutations\n        self.number_of_recommendations = number_of_recommendations\n        self.depth = depth\n        self.dataframe = dataframe\n        self.minhash = []\n        self.forest = None\n    \n    def minhash_data(self):\n        for title in self.dataframe['clean_text']:\n            tokens = title.split(' ')\n            min_hash = MinHash(num_perm=self.permutations)\n            for t in tokens:\n                min_hash.update(t.encode('utf-8'))\n            self.minhash.append(min_hash)\n    \n    def prepare_forest(self):\n        self.forest = MinHashLSHForest(num_perm=self.permutations, l=self.depth)\n        for i, j in enumerate(self.minhash):\n            self.forest.add(i, j)\n        self.forest.index()\n        del self.minhash\n        gc.collect()\n    \n    def query_forest(self, query, number_of_results, cosine_sim=False):\n        query_tokens = query.split(' ')\n        min_hash = MinHash(num_perm=self.permutations)\n        for i in query_tokens:\n            min_hash.update(i.encode('utf-8'))\n        result = self.forest.query(min_hash, self.number_of_recommendations)\n        if cosine_sim:\n            # print(\"Cosine Similarity\")\n            result = [(key, self.cosine_similarity(self.dataframe.iloc[key].cleaned_label, query)) for key in result]\n        else:\n            # print(\"Jaccard Similarity\")\n            result = [(key, self.jaccard_similarity(self.dataframe.iloc[key].cleaned_label, query_tokens)) for key in result]\n        result = sorted(result, key=lambda x: x[1], reverse=True)[:number_of_results]\n        iloc = [i[0] for i in result]\n        return '|'.join(set(self.dataframe.iloc[iloc].cleaned_label.to_list()))\n    \n    def jaccard_similarity(self, l1, l2):\n        intersection = len(list(set(l1).intersection(l2)))\n        union = (len(l1) + len(l2)) - intersection\n        return float(intersection) / union\n    \n    def cosine_similarity(self, string1, string2):\n        d1 = nlp(string1)\n        d2 = nlp(string2)\n        return d1.similarity(d2)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"obj = LSH(permutations=128, number_of_recommendations=20, depth=10, dataframe=train_json)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nobj.minhash_data()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nobj.prepare_forest()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nsample_submission_df['PredictionString'] = sample_submission_df.Id.apply(lambda x: obj.query_forest(train_json.loc[train_json['file'] == str(x)].cleaned_label.to_list()[0], 10))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission_df.to_csv('submission.csv', index=False)\nsample_submission_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<h3 align=\"center\" style=\"background-color:#003300;color:white;\">Thanks! More updates to come. WIP</h3> "}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}