{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport re\nimport numpy as np\nfrom tqdm import tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/train.csv\",nrows=10000)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_text(filename, test=False):\n    if test:\n        df = pd.read_json('../input/coleridgeinitiative-show-us-the-data/test/{}.json'.format(filename))\n    else:\n        df = pd.read_json('../input/coleridgeinitiative-show-us-the-data/train/{}.json'.format(filename))\n    text = \" \".join(list(df['text']))\n    return text\n\ndf_train['text'] = df_train['Id'].apply(get_text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# text = 'I created this project on datasets indian families of my datasets of comon datasets indian families'\n# entity = 'datasets indian families'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def convert_tokens(x,m,train=True):\n    df = pd.DataFrame()\n    text = x['text'].replace('\\uf0b7','').split()\n    entity = x['dataset_label']\n    \n    ##\n    tokens=[]\n    k=0\n    for i,x in enumerate(text):\n\n        if k==0:\n            if x==entity.split()[0]:\n                entity_len = len(entity.split())\n                if entity == ' '.join(text[i:i+entity_len]):\n                    tokens.extend(['o-dataset']*len(entity.split()))\n                    k = entity_len\n                    #print('k updated')\n                else:\n                    #print(x,'o1')\n                    tokens.append('o')\n            else:\n                #print(x,'o2')\n                tokens.append('o')\n\n\n        k = max(0,k-1)\n            \n    k=0\n    sentence_hash=[]\n    for i in range(0,len(text),290):\n        sentence_hash.extend([f'sentence#{k}']* len(text[i:i+290]))\n        k+=1\n      \n    df['token']=list(map(str,text))\n    df['sentence#'] = sentence_hash\n    df['sentence'] = f'sentence{m}'\n    df['entity'] = tokens\n    \n#     if train:\n#         def decide(x):\n#             for k in label:\n#                 if x.find(k)!=-1:\n#                     return 'o-dataset'\n\n#             return 'o'\n#         label = x['dataset_label'].split()\n#         df['entity'] = df['token'].map(lambda x : decide(x))    \n    \n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final = pd.DataFrame()\nfor i,row in tqdm(df_train.iterrows()):\n    df = convert_tokens(row,i)\n    final = final.append(df,ignore_index=True)\n\n##making some correction of bad tagging    \n# masks = (final['entity'].shift()==final['entity'])|(final['entity']==final['entity'].shift(-1))    \n# final.loc[~masks,'entity'] = 'o'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final.to_pickle(\"show_us_the_data_train_ner.pkl\")\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}