{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"The purpose of this notebook is to illustrate how to upload data from the coleridge-initiative-data dataset quickly\n","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport pickle\nimport re\nimport glob\n\ndef clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df = pd.read_csv('../input/coleridge-initiative-data/submission.csv', index_col=0)\nsubmission_df['PredictionString'] = None\nindata = open('../input/coleridge-initiative-data/confirmed_databases.pickle','rb')\ndatabase_dict = pickle.load(indata)\nindata.close()\ndatasets_titles = database_dict['original'] #Fetch the original set of databases (and labels) that we know about\n#Test data has to be built\ntest_files = glob.glob(\"../input/coleridgeinitiative-show-us-the-data/test/*.json\")\ndf_test_publications = pd.DataFrame()\n\nfor test_file in test_files:\n    file_data = pd.read_json(test_file)\n    file_data.insert(0,'pub_id', test_file.split('/')[-1].split('.')[0])\n    df_test_publications = pd.concat([df_test_publications, file_data])\n\ndf_test_publications.to_csv(\"df_test_publications.csv\",index=False)\ndf_test_publications['text'] = df_test_publications['text'].str.lower()\ndf_train_publications = pd.read_csv('../input/coleridge-initiative-data/df_train_publications.csv')\n\ndf_train_publications.head(3)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_publications.head(3)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = []\nfor index in submission_df.index:\n    print(index)\n    publication_text = df_test_publications[df_test_publications['pub_id'] == index].text.str.cat(sep='\\n').lower()\n    label = []\n    for dataset_title in datasets_titles:\n        if dataset_title in publication_text:\n            label.append(clean_text(dataset_title))\n    labels.append('|'.join(label))\n\nsubmission_df['PredictionString'] = labels\n\nsubmission_df.to_csv('submission.csv')\n\nsubmission_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}