{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# import required libraries","metadata":{}},{"cell_type":"code","source":"pip install datasketch\n","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:58:01.577458Z","iopub.execute_input":"2021-06-22T19:58:01.577974Z","iopub.status.idle":"2021-06-22T19:58:10.543513Z","shell.execute_reply.started":"2021-06-22T19:58:01.577938Z","shell.execute_reply":"2021-06-22T19:58:10.541972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json,re\nimport os\nimport numpy as np\nimport pandas as pd\nimport time\nfrom datasketch import MinHash, MinHashLSHForest","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:58:21.245476Z","iopub.execute_input":"2021-06-22T19:58:21.245907Z","iopub.status.idle":"2021-06-22T19:58:21.92375Z","shell.execute_reply.started":"2021-06-22T19:58:21.245867Z","shell.execute_reply":"2021-06-22T19:58:21.922728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# clean_text function","metadata":{}},{"cell_type":"markdown","source":"#clean_text function should be used to clean text as mentioned on the #Evaluation page","metadata":{}},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:00:30.163816Z","iopub.execute_input":"2021-06-22T19:00:30.164329Z","iopub.status.idle":"2021-06-22T19:00:30.168559Z","shell.execute_reply.started":"2021-06-22T19:00:30.164296Z","shell.execute_reply":"2021-06-22T19:00:30.16754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#function to read csv files","metadata":{}},{"cell_type":"code","source":"csvfilepath=\"../input/coleridgeinitiative-show-us-the-data/\"\ndef readcsv(filename):\n    csvfile=pd.read_csv(csvfilepath + filename)\n    return csvfile","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:01:45.595772Z","iopub.execute_input":"2021-06-22T19:01:45.596124Z","iopub.status.idle":"2021-06-22T19:01:45.601418Z","shell.execute_reply.started":"2021-06-22T19:01:45.596095Z","shell.execute_reply":"2021-06-22T19:01:45.600418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"samplefilename=\"sample_submission.csv\"\ntrainfilename=\"train.csv\"\n\ntraincsv=readcsv(trainfilename)\nsample_submission_df = readcsv(samplefilename)\ntraincsv.head(5)","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:01:48.5884Z","iopub.execute_input":"2021-06-22T19:01:48.588786Z","iopub.status.idle":"2021-06-22T19:01:48.775273Z","shell.execute_reply.started":"2021-06-22T19:01:48.588749Z","shell.execute_reply":"2021-06-22T19:01:48.773958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"traincsv['clean_pub_title'] = traincsv.pub_title.apply(lambda x: clean_text(x))","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:02:22.038119Z","iopub.execute_input":"2021-06-22T19:02:22.038724Z","iopub.status.idle":"2021-06-22T19:02:22.258157Z","shell.execute_reply.started":"2021-06-22T19:02:22.038686Z","shell.execute_reply":"2021-06-22T19:02:22.256615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"traincsv.tail(5)","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:02:25.051322Z","iopub.execute_input":"2021-06-22T19:02:25.051756Z","iopub.status.idle":"2021-06-22T19:02:25.06833Z","shell.execute_reply.started":"2021-06-22T19:02:25.051724Z","shell.execute_reply":"2021-06-22T19:02:25.067197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read json file and map it to dataframe","metadata":{}},{"cell_type":"markdown","source":"#path to JSON file","metadata":{}},{"cell_type":"code","source":"\npath_train=\"../input/coleridgeinitiative-show-us-the-data/train/\"\npath_test=\"../input/coleridgeinitiative-show-us-the-data/test/\"","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:02:16.402678Z","iopub.execute_input":"2021-06-22T19:02:16.403018Z","iopub.status.idle":"2021-06-22T19:02:16.407214Z","shell.execute_reply.started":"2021-06-22T19:02:16.402988Z","shell.execute_reply":"2021-06-22T19:02:16.406362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#read train json file","metadata":{}},{"cell_type":"code","source":"def readcsvid():\n   \n    df=pd.DataFrame()\n    dfs=pd.DataFrame()\n    file_id=[]\n    cleaned_label=[]\n    uniques = traincsv['Id'].unique() \n    for id in uniques:\n       \n        filename=id +\".json\"\n        json_file=open(os.path.join(path_train,filename))\n        dfs=pd.read_json(json_file)\n        clean_label=traincsv.loc[traincsv[\"Id\"]==id,'cleaned_label'].iloc[0]\n        dfs[\"id\"]=id\n        dfs[\"cleaned_label\"]=clean_label\n        df = pd.concat([df,dfs])\n        \n    df.to_csv(\"df_train.csv\",index=False)\n        \n    \n        \n  ","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:41:06.826162Z","iopub.execute_input":"2021-06-22T19:41:06.826546Z","iopub.status.idle":"2021-06-22T19:41:06.835132Z","shell.execute_reply.started":"2021-06-22T19:41:06.826514Z","shell.execute_reply":"2021-06-22T19:41:06.833011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# map to read test csv file to map json test data","metadata":{}},{"cell_type":"code","source":"def readtestcsvid():\n    count=0\n    df=pd.DataFrame()\n    dfs=pd.DataFrame()\n    file_id=[]\n    cleaned_label=[]\n    json_files = [pos_json for pos_json in os.listdir(path_test) if pos_json.endswith('.json')]\n    for f in json_files:\n        with open(os.path.join(path_test,f)) as json_file:\n                          \n            dfs=pd.read_json(json_file)\n            \n            dfs[\"id\"]=f.replace('.json',' ')\n            \n            df = pd.concat([df,dfs])\n                \n            \n            \n    df.to_csv(\"df_test.csv\",index=False)\n        \n    \n        ","metadata":{"execution":{"iopub.status.busy":"2021-06-22T21:35:54.763468Z","iopub.execute_input":"2021-06-22T21:35:54.763866Z","iopub.status.idle":"2021-06-22T21:35:54.771759Z","shell.execute_reply.started":"2021-06-22T21:35:54.76383Z","shell.execute_reply":"2021-06-22T21:35:54.77066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"readcsvid() #read function to map json train file\n\n","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:41:19.077144Z","iopub.execute_input":"2021-06-22T19:41:19.077576Z","iopub.status.idle":"2021-06-22T19:46:44.703183Z","shell.execute_reply.started":"2021-06-22T19:41:19.077543Z","shell.execute_reply":"2021-06-22T19:46:44.701778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"readtestcsvid()\n","metadata":{"execution":{"iopub.status.busy":"2021-06-22T21:35:58.802556Z","iopub.execute_input":"2021-06-22T21:35:58.802896Z","iopub.status.idle":"2021-06-22T21:35:58.855407Z","shell.execute_reply.started":"2021-06-22T21:35:58.802867Z","shell.execute_reply":"2021-06-22T21:35:58.854417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainjson=pd.read_csv(\"df_train.csv\")\n","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:50:16.481001Z","iopub.execute_input":"2021-06-22T19:50:16.481588Z","iopub.status.idle":"2021-06-22T19:50:24.022794Z","shell.execute_reply.started":"2021-06-22T19:50:16.481551Z","shell.execute_reply":"2021-06-22T19:50:24.021831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testjson=pd.read_csv(\"df_test.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-06-22T21:36:17.139531Z","iopub.execute_input":"2021-06-22T21:36:17.139906Z","iopub.status.idle":"2021-06-22T21:36:17.159751Z","shell.execute_reply.started":"2021-06-22T21:36:17.139874Z","shell.execute_reply":"2021-06-22T21:36:17.158527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(trainjson.shape)\nprint(testjson.shape)","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:50:51.546924Z","iopub.execute_input":"2021-06-22T19:50:51.547494Z","iopub.status.idle":"2021-06-22T19:50:51.552721Z","shell.execute_reply.started":"2021-06-22T19:50:51.547459Z","shell.execute_reply":"2021-06-22T19:50:51.551833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainjson.tail(5)","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:50:56.381553Z","iopub.execute_input":"2021-06-22T19:50:56.381895Z","iopub.status.idle":"2021-06-22T19:50:56.395829Z","shell.execute_reply.started":"2021-06-22T19:50:56.381866Z","shell.execute_reply":"2021-06-22T19:50:56.394669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ","metadata":{}},{"cell_type":"code","source":"testjson.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T21:36:22.389308Z","iopub.execute_input":"2021-06-22T21:36:22.38969Z","iopub.status.idle":"2021-06-22T21:36:22.40148Z","shell.execute_reply.started":"2021-06-22T21:36:22.389647Z","shell.execute_reply":"2021-06-22T21:36:22.400541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# Using LSH \nThis class of algorithms combines aspects of feature transformation","metadata":{}},{"cell_type":"markdown","source":"# Efficiencies can be achieved by actually using an LSH Forest algorithm for efficient searching.\n\nthe following steps for performing LSH:\n\n1.Create shingles from your available data set (e.g. unigrams, bigrams, ratings, etc.)\n\n2.Build an m×n shingle matrix where every shingle that appears in a set is marked with a 1 otherwise 0.\n\n3.Permute the rows of the shingle matrix from step 2 and build a new p×n signature matrix where the number of the row of the first shingle to appear for a set is recorded for the permutation of the signature matrix.\n\n4..Repeat permuting the rows of the input matrix times and complete filling in the p×n signature matrix.\n\n5.Choose a band size b for the number of rows you will compare between sets in the LSH matrix.","metadata":{}},{"cell_type":"code","source":"#Preprocess will split a string of text into individual tokens/shingles based on whitespace.\ndef preprocess(text):\n    text = re.sub(r'[^\\w\\s]','',text)\n    tokens = text.lower()\n    tokens = tokens.split()\n    return tokens","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:51:26.009795Z","iopub.execute_input":"2021-06-22T19:51:26.010321Z","iopub.status.idle":"2021-06-22T19:51:26.016529Z","shell.execute_reply.started":"2021-06-22T19:51:26.010264Z","shell.execute_reply":"2021-06-22T19:51:26.01527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label=[]\n\nlabel=trainjson['cleaned_label'].unique().tolist()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:51:29.567968Z","iopub.execute_input":"2021-06-22T19:51:29.568512Z","iopub.status.idle":"2021-06-22T19:51:29.596812Z","shell.execute_reply.started":"2021-06-22T19:51:29.568465Z","shell.execute_reply":"2021-06-22T19:51:29.595701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shingles=[]\n\n\nfor t in label:\n    shingles.append(preprocess(t))\n","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:51:34.809408Z","iopub.execute_input":"2021-06-22T19:51:34.809774Z","iopub.status.idle":"2021-06-22T19:51:34.814674Z","shell.execute_reply.started":"2021-06-22T19:51:34.809744Z","shell.execute_reply":"2021-06-22T19:51:34.813496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Choose your parameters","metadata":{}},{"cell_type":"code","source":"#Number of Permutations\npermutations = 128\n\n#Number of Recommendations to return\nnum_recommendations = 1","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:51:38.435928Z","iopub.execute_input":"2021-06-22T19:51:38.436448Z","iopub.status.idle":"2021-06-22T19:51:38.440398Z","shell.execute_reply.started":"2021-06-22T19:51:38.436414Z","shell.execute_reply":"2021-06-22T19:51:38.439582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create Minhash Forest for Queries\nIn order to create the Minhash Forest, we will execute the following steps:\n\n1. Pass in a dataframe with every string you want to query.\n\n2. Preprocess a string of text using our preprocessing step above.\n\n3. Set the number of permutations in your MinHash.\n\n4. MinHash the string on all of your shingles in the string.\n5. Store the MinHash of the string.\n6. Repeat 2-5 for all strings in your dataframe.\n7. Build a forest of all the MinHashed strings.\nI8. ndex your forest to make it searchable.\n","metadata":{}},{"cell_type":"code","source":"def get_forest(data, perms):\n    start_time = time.time()\n    \n    minhash = []\n    \n    for text in data['text']:\n        tokens = preprocess(str(text))\n        m = MinHash(num_perm=perms)\n        for s in tokens:\n            m.update(s.encode('utf8'))\n        minhash.append(m)\n        \n    forest = MinHashLSHForest(num_perm=perms)\n    \n    for i,m in enumerate(minhash):\n        forest.add(i,m)\n        \n    forest.index()\n    \n    print('It took %s seconds to build forest.' %(time.time()-start_time))\n    \n    return forest","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:51:42.338033Z","iopub.execute_input":"2021-06-22T19:51:42.338436Z","iopub.status.idle":"2021-06-22T19:51:42.346361Z","shell.execute_reply.started":"2021-06-22T19:51:42.338403Z","shell.execute_reply":"2021-06-22T19:51:42.345186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluate Queries\nIn order to query the forest that was built, we will follow the steps below:\n\nPreprocess your text into shingles.\nSet the same number of permutations for your MinHash as was used to build the forest.\nCreate your MinHash on the text using all your shingles.\nQuery the forest with your MinHash and return the number of requested recommendations.\nProvide the titles of each conference paper recommended.\n","metadata":{}},{"cell_type":"code","source":"def predict(text, database, perms, num_results, forest):\n    \n    \n    tokens = preprocess(text)\n    m = MinHash(num_perm=perms)\n    for s in tokens:\n        m.update(s.encode('utf8'))\n        \n    idx_array = np.array(forest.query(m, num_results))\n    if len(idx_array) == 0:\n        return None,None # if your query is empty, return none\n    \n    result = database.iloc[idx_array]['text']\n    file=database.iloc[idx_array]['id']\n   \n    \n    return result,file","metadata":{"execution":{"iopub.status.busy":"2021-06-22T21:29:03.797429Z","iopub.execute_input":"2021-06-22T21:29:03.798084Z","iopub.status.idle":"2021-06-22T21:29:03.807492Z","shell.execute_reply.started":"2021-06-22T21:29:03.798041Z","shell.execute_reply":"2021-06-22T21:29:03.80635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Testing Our Recommendation Engine on json publication\nWe will start by loading the CSV containing all the publication \n\nFinally, we can query any string of text such as a title or general topic to return a list of recommendations.  Naturally, we get the exact prediction\nas one of our recommendations.\n\n","metadata":{}},{"cell_type":"code","source":"\nforest = get_forest(trainjson, permutations)","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:59:01.755165Z","iopub.execute_input":"2021-06-22T19:59:01.755756Z","iopub.status.idle":"2021-06-22T20:39:29.32496Z","shell.execute_reply.started":"2021-06-22T19:59:01.755705Z","shell.execute_reply":"2021-06-22T20:39:29.32395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_recommendations = 5\nresult=[]\nfilename=[]\nstart_time = time.time()\nfor t in label:\n    res,f=predict(t,trainjson, permutations, num_recommendations, forest)\n    result.append(res)\n    filename.append(f)\nprint('It took %s seconds to query forest.' %(time.time()-start_time))","metadata":{"execution":{"iopub.status.busy":"2021-06-22T21:29:07.970006Z","iopub.execute_input":"2021-06-22T21:29:07.970401Z","iopub.status.idle":"2021-06-22T21:29:08.561487Z","shell.execute_reply.started":"2021-06-22T21:29:07.970366Z","shell.execute_reply":"2021-06-22T21:29:08.56034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"\n\npredict_df = pd.DataFrame(list(zip(result, filename)), columns = ['predicted_string','filename'])","metadata":{"execution":{"iopub.status.busy":"2021-06-22T22:12:53.525906Z","iopub.execute_input":"2021-06-22T22:12:53.526509Z","iopub.status.idle":"2021-06-22T22:12:53.532256Z","shell.execute_reply.started":"2021-06-22T22:12:53.526472Z","shell.execute_reply":"2021-06-22T22:12:53.531207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict_df.head()\n      ","metadata":{"execution":{"iopub.status.busy":"2021-06-22T22:39:13.020118Z","iopub.execute_input":"2021-06-22T22:39:13.02052Z","iopub.status.idle":"2021-06-22T22:39:13.038201Z","shell.execute_reply.started":"2021-06-22T22:39:13.020486Z","shell.execute_reply":"2021-06-22T22:39:13.037386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict_df['predicted_string'] = predict_df.predicted_string.apply(lambda x: clean_text(x))\nsample_submission_df.['PredictionString']=sample_submission_df.Id.apply(lambda x:train_json.loc[train_json['file'] == sample_submission_df['Id']].cleaned_label","metadata":{"execution":{"iopub.status.busy":"2021-06-22T22:46:44.497112Z","iopub.execute_input":"2021-06-22T22:46:44.497741Z","iopub.status.idle":"2021-06-22T22:46:44.504042Z","shell.execute_reply.started":"2021-06-22T22:46:44.497691Z","shell.execute_reply":"2021-06-22T22:46:44.50292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission_df.to_csv('submission.csv', index=False)\nsample_submission_df","metadata":{"execution":{"iopub.status.busy":"2021-06-22T20:39:29.408376Z","iopub.status.idle":"2021-06-22T20:39:29.408856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}