{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import json\nimport glob\nimport numpy as np\nimport pandas as pd","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train=pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\ntrain","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Add all the file names to a list"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Path to the JSON files\ntrain_files = glob.glob(\"../input/coleridgeinitiative-show-us-the-data/train/*.json\")\ntest_files = glob.glob(\"../input/coleridgeinitiative-show-us-the-data/test/*.json\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# #compare one json with the train data\n    \n    #there is no pub_title....\n    #the keywords dont seem to popup in the ele file\n    #duplicate articles ?"},{"metadata":{"trusted":true},"cell_type":"code","source":"    \n    ele=train_files[1]\n    print(ele)\n    pdele=pd.read_json(ele)\n    idnr=ele.split('/')[-1:][0].split('.')[0]\n    pdele['Id']=idnr\n    print(train[train.Id==idnr],pdele)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Loading all the JSON files\n\n #articles are splitted in paragraphs\n "},{"metadata":{"trusted":true},"cell_type":"code","source":"# load json file using pandas\ndef readjson(data_files,variab):\n    df_train = pd.DataFrame()\n    for count,ele in enumerate(data_files,len(data_files)):\n        #df_train = pd.concat([df_train, pd.read_json(ele)])\n        pdele=pd.read_json(ele).reset_index()\n        idnr=ele.split('/')[-1:][0].split('.')[0]\n        pdele['Id']=idnr\n        df_train=df_train.append(pdele)\n    df_train.to_csv(variab+'.csv',index=False)\n    print(df_train.head())\n    return df_train\n\ntesttxt=readjson(test_files,'test')\ntraintxt=readjson(train_files,'train')\ntesttxt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"traintxt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submit=testtxt.merge(train,how='inner',left_on='Id',right_on='Id')\nsubmit","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submit=submit.groupby('Id').max().reset_index()[['Id','cleaned_label']]\nsubmit.columns=['Id','PredictionString']\nsubmit.to_csv('submission.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"testtxt.groupby('Id').max()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}