{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport os\nimport json\nimport re","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\", 'This pattern has match groups')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Read Train Data\ndf_train=pd.read_csv('/kaggle/input/coleridgeinitiative-show-us-the-data/train.csv')\ndf_train.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Read Test Data\ndf_test=pd.read_csv('/kaggle/input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\ndf_test.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Read Train Jsons to extract Text \ndf_input = pd.DataFrame(columns=['id','section_title','text','data_label'])\nids=df_train['Id'].values\nfor id in ids:\n    df=pd.read_json ('../input/coleridgeinitiative-show-us-the-data/train/{}.json'.format(id))\n    \n    for data_label in df_train[df_train['Id']==id]['dataset_label'].values:\n        new_df=df[df['text'].str.contains(data_label)].copy(deep=True)\n        new_df.loc[:,['data_label']] = data_label\n        new_df.loc[:,['id']] = id\n        new_df.reset_index(inplace=True,drop=True)\n        df_input=pd.concat([df_input, new_df], ignore_index=True)\n        df_input.reset_index(inplace=True,drop=True)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_input.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Read Test Jsons\ndf_test_input = pd.DataFrame(columns=['id','section_title','text'])\nids=df_test['Id'].values\nfor id in ids:\n    df=pd.read_json ('../input/coleridgeinitiative-show-us-the-data/test/{}.json'.format(id))\n    \n    df.loc[:,['id']] = id\n    df.reset_index(inplace=True,drop=True)\n    df_test_input=pd.concat([df_test_input, df], ignore_index=True)\n    df_test_input.reset_index(inplace=True,drop=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv', index_col=0)\nsubmission_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Create list of target Dataset Labels\ndatasets_titles = [str(x).lower() for x in df_input['data_label'].unique()]\nlabels = []","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### String matching if Dataset label is present in Journal Text\nfor index in submission_df.index:\n    publication_text = df_test_input[df_test_input['id'] == index].text.str.cat(sep='\\n').lower()\n    \n    label = []\n    for dataset_title in datasets_titles:\n        if dataset_title in publication_text:\n            label.append(clean_text(dataset_title))\n    labels.append('|'.join(label))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Replace Nulls with top 2 frequent dataset labels in Training set\nsubmission_df['PredictionString'] = labels\nsubmission_df['PredictionString'].fillna('alzheimer s disease neuroimaging initiative adni|adni', inplace=True)\nsubmission_df.to_csv('submission.csv')\n\nsubmission_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}