{"cells":[{"metadata":{},"cell_type":"markdown","source":"Here is a function I made in order to delete leaked data from the training set. \n\nIt removes all the matches from the training set that appear also in the test set.\n\nIt inputs two DataFrame : df_train (input data set) and df_test (test data set).\n","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\ndef concat_row(r):\n    if r['WTeamID'] < r['LTeamID']:\n        res = str(r['Season'])+\"_\"+str(r['WTeamID'])+\"_\"+str(r['LTeamID'])\n    else:\n        res = str(r['Season'])+\"_\"+str(r['LTeamID'])+\"_\"+str(r['WTeamID'])\n    return res\n\n# Delete leaked from train\ndef delete_leaked_from_df_train(df_train, df_test):\n    df_train['Concats'] = df_train.apply(concat_row, axis=1)\n    df_train_duplicates = df_train[df_train['Concats'].isin(df_test['ID'].unique())]\n    df_train_idx = df_train_duplicates.index.values\n    df_train = df_train.drop(df_train_idx)\n    df_train = df_train.drop('Concats', axis=1)\n    \n    return df_train \n\ndef read_data(inFile, sep=','):\n    df_op = pd.read_csv(filepath_or_buffer=inFile, low_memory=False, encoding='utf-8', sep=sep)\n    return df_op","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PATH = \"/kaggle/input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/\"\ndf_test = read_data(PATH+\"MSampleSubmissionStage1_2020.csv\")\ndf_train = read_data(PATH+\"MDataFiles_Stage1/MNCAATourneyCompactResults.csv\")\n\nprint(\"SIZE TRAIN BEFORE :\")\nprint(df_train.shape)\ndf_train = delete_leaked_from_df_train(df_train, df_test)\nprint()\nprint(\"SIZE TRAIN AFTER :\")\nprint(df_train.shape)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}