{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport sklearn\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.ensemble import HistGradientBoostingRegressor\nfrom sklearn.metrics import mean_squared_error, r2_score\nimport pyarrow as pa\nimport pyarrow.parquet as pq","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-08-10T12:37:32.348438Z","iopub.execute_input":"2023-08-10T12:37:32.348850Z","iopub.status.idle":"2023-08-10T12:37:34.396754Z","shell.execute_reply.started":"2023-08-10T12:37:32.348816Z","shell.execute_reply":"2023-08-10T12:37:34.395749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file_path = '/kaggle/input/c/air-toxicity-and-chronic-respiratory-diseases-us/'\ntrain = pd.read_csv(file_path+'train.csv')\ntest  = pd.read_csv(file_path+'test.csv')\nprint(\"Test and train data loaded\")\nlead = pd.read_parquet(file_path+'supplementary_LEAD.parquet')\nhaps = pd.read_parquet(file_path+'supplementary_HAPS.parquet')\nnox  = pd.read_parquet(file_path+'supplementary_NONOxNOy.parquet')\nvocs = pd.read_parquet(file_path+'supplementary_VOCS.parquet')\nprint(\"Supplementary files loaded\")","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2023-08-10T12:37:34.399119Z","iopub.execute_input":"2023-08-10T12:37:34.400187Z","iopub.status.idle":"2023-08-10T12:37:42.088840Z","shell.execute_reply.started":"2023-08-10T12:37:34.400143Z","shell.execute_reply":"2023-08-10T12:37:42.087905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def explore(dataframe):\n    print(dataframe.head())\n    print()\n    print(dataframe.describe())\n    print()\n    print(dataframe['Parameter.Name'].unique())\n    print()\n    print(dataframe['Parameter.Name'].value_counts())","metadata":{"execution":{"iopub.status.busy":"2023-08-10T12:37:42.089838Z","iopub.execute_input":"2023-08-10T12:37:42.090190Z","iopub.status.idle":"2023-08-10T12:37:42.096843Z","shell.execute_reply.started":"2023-08-10T12:37:42.090160Z","shell.execute_reply":"2023-08-10T12:37:42.095465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def save_as_parquet(dataframe,filename):\n    table = pa.Table.from_pandas(dataframe)\n    output = \"/kaggle/working/\"\n    open(output+filename,'w')\n    pq.write_table(table,output+filename)\n    print(filename,'saved to /kaggle/working')\n\n#processing the supplements takes quite a bit time, so it's broken down into multiple functions\n#this function removes the Max.Value for each data point and returns either the mean or median\n#for a year's worth of data.\ndef supplement_preprocessing(dataframe,metric):\n    supplement = dataframe.copy()\n    np.parameter_list = supplement['Parameter.Name'].unique()\n    supplement.drop(columns='X1st.Max.Value', inplace=True)\n    supplement.rename(columns={'Date.Local':'Year','Parameter.Name':'Parameter'}, inplace=True)\n    supplement['Year'] = supplement['Year'].astype(str)\n    supplement['Year'] = supplement['Year'].str[0:4]\n    supplement['Year'] = supplement['Year'].astype(int)\n    \n    if metric.lower() == 'mean':\n        supplement = supplement.groupby(by=['State.Name','Parameter','Year']).mean()\n        return np.parameter_list, supplement\n    elif metric.lower() == 'median':\n        supplement = supplement.groupby(by=['State.Name','Parameter','Year']).median()\n        return np.parameter_list, supplement\n    else:\n        print(\"Invalid metric. Please enter mean or median to indicate which metric the polutant parameter should be evaluated by.\")    \n\n#When processing an individual parameter, this function extracts the target parameter from the rest of the data set and formats it into it's own column\n#named after the parameter, to prepare\ndef parameter_preprocessing(dataframe,parameter):\n    supplement = dataframe.copy()\n    supplement.rename(columns={'Arithmetic.Mean':parameter}, inplace=True)\n    supplement.query(\"Parameter == @parameter\", inplace=True)\n    supplement = supplement.reset_index()\n    supplement.drop(axis=1,columns=['Parameter'], inplace = True)\n    return supplement\n    \ndef sklearn_train(input_data,y,phase,seed):\n    X = input_data.copy()\n    X.drop(columns=['ID','State.Name','Incidence'], inplace=True)\n    np.results[phase] = {}\n    X = pd.get_dummies(X)\n    \n    X_train, X_val, y_train, y_val, = train_test_split(X, y, random_state = seed)\n    \n    y_val.drop(columns='Incidence', inplace=True)\n    \n    model = HistGradientBoostingRegressor()\n    \n    model.fit(X_train, y_train)\n    y_pred = model.predict(X_val)\n\n    mse = mean_squared_error(y_val, y_pred)\n    r2 = r2_score(y_val, y_pred)\n    \n    if phase == \"Check\" or phase == 'Train':\n        np.results[phase]['r2'] = r2\n    elif phase == 'Final':\n        print('---Histogram Gradient Boosting Regressor---')\n        print('Features:',input_data.columns[5:])\n        print('mse:',mse)\n        print('r2:',r2)\n        return model","metadata":{"execution":{"iopub.status.busy":"2023-08-10T12:37:42.099913Z","iopub.execute_input":"2023-08-10T12:37:42.100307Z","iopub.status.idle":"2023-08-10T12:37:42.123358Z","shell.execute_reply.started":"2023-08-10T12:37:42.100275Z","shell.execute_reply":"2023-08-10T12:37:42.121391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def single_parameter_test(dataframe,train_data,test_data,parameter,seed):\n    X = parameter_preprocessing(dataframe,parameter)\n    train_data = train_data.merge(X, on=['State.Name','Year'],how = 'left')\n    test_data = test_data.merge(X, on=['State.name','Year'],how = 'left')\n    \n#This function will take preprocessed supplements and run through each parameter\n#with the sklearn regressor to identify if the next parameter is reducing error\n#or increasing error. Parameters that improve the score are kept, parameters\n#that are not are dropped.\ndef iterative_parameter_testing(dataframe,train_data,test_data,parameter_list,seed):\n    for parameter in np.parameter_list:\n        X = parameter_preprocessing(dataframe,parameter)\n        #initial run to create starting data. At present, it preserves the initial parameter\n        if parameter == np.parameter_list[0] and 'Train' not in np.results:\n            train_data = train_data.merge(X, on=['State.Name','Year'],how = 'left')\n            test_data = test_data.merge(X, on=['State.Name','Year'],how='left')\n            sklearn_train(train_data,y,'Train',seed)\n            #print(parameter,'loaded')\n            continue\n            \n        check = train_data.merge(X, on=['State.Name','Year'],how ='left')\n        sklearn_train(check,y,'Check',seed)\n        if np.results['Check']['r2'] > np.results['Train']['r2']:\n            train_data = check\n            test_data = test_data.merge(X, on=['State.Name','Year'],how='left')\n            np.results['Train']['r2']=np.results['Check']['r2']\n            #print(parameter,\"loaded\")\n        #else:\n            #print(parameter,'not loaded')\n            \n    print('--Supplement loaded--')\n    \n    return train_data, test_data","metadata":{"execution":{"iopub.status.busy":"2023-08-10T13:53:24.263399Z","iopub.execute_input":"2023-08-10T13:53:24.263800Z","iopub.status.idle":"2023-08-10T13:53:24.276842Z","shell.execute_reply.started":"2023-08-10T13:53:24.263770Z","shell.execute_reply":"2023-08-10T13:53:24.275181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#y = train['Incidence'].copy()\n#y = y.astype(float)\n\n#supplements = [lead,haps,nox,vocs]\n\n#for seed in range(1,11):\n#    np.results = {}\n#    train_data = train.copy()\n#    test_data = test.copy()\n#    for item in supplements:\n#        parameter_list,parameter_data = supplement_preprocessing(item,'mean')\n#        train_data, test_data = iterative_parameter_testing(parameter_data,train_data,test_data,parameter_list,seed)\n#    model = sklearn_train(train_data,y,'Final',seed)\n#    print('Seed:',seed)\n    \n    \n#save_as_parquet(train_data,'train_temp.parquet')\n#save_as_parquet(test_data,'train_temp.parquet')","metadata":{"execution":{"iopub.status.busy":"2023-08-10T13:58:10.132644Z","iopub.execute_input":"2023-08-10T13:58:10.133148Z","iopub.status.idle":"2023-08-10T14:44:53.819880Z","shell.execute_reply.started":"2023-08-10T13:58:10.133116Z","shell.execute_reply":"2023-08-10T14:44:53.818740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.results = {}\n\ny = train['Incidence'].copy()\ny = y.astype(float)\n\ntrain_data = train.copy()\ntest_data = test.copy()\n\nsupplements = [lead,haps,nox,vocs]\n\nseed = 3\n\nfor item in supplements:\n    parameter_list,parameter_data = supplement_preprocessing(item,'mean')\n    train_data, test_data = iterative_parameter_testing(parameter_data,train_data,test_data,parameter_list,seed)","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2023-08-10T12:37:42.145745Z","iopub.execute_input":"2023-08-10T12:37:42.147216Z","iopub.status.idle":"2023-08-10T12:42:26.280363Z","shell.execute_reply.started":"2023-08-10T12:37:42.147141Z","shell.execute_reply":"2023-08-10T12:42:26.278791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"save_as_parquet(train_data,'train_temp.parquet')\nsave_as_parquet(test_data,'train_temp.parquet')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = train_data.copy()\n    \nmodel = sklearn_train(X,y,'Final',seed)\n\nX = test_data.copy()\n\nX.drop(columns=['ID','State.Name'],inplace=True)\nX = pd.get_dummies(X)\npred = model.predict(X)\n\noutput = pd.DataFrame(pred, columns=['Incidence'],index=test['ID'])\noutput.to_csv('/kaggle/working/submission.csv',index=True)","metadata":{"execution":{"iopub.status.busy":"2023-08-10T12:42:26.281872Z","iopub.execute_input":"2023-08-10T12:42:26.282264Z","iopub.status.idle":"2023-08-10T12:42:27.351377Z","shell.execute_reply.started":"2023-08-10T12:42:26.282232Z","shell.execute_reply":"2023-08-10T12:42:27.350161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#pip install pycaret","metadata":{"_kg_hide-output":true,"_kg_hide-input":false,"scrolled":true,"execution":{"iopub.status.busy":"2023-08-10T12:42:27.352608Z","iopub.execute_input":"2023-08-10T12:42:27.352922Z","iopub.status.idle":"2023-08-10T12:42:27.359451Z","shell.execute_reply.started":"2023-08-10T12:42:27.352894Z","shell.execute_reply":"2023-08-10T12:42:27.358045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#from pycaret.regression import RegressionExperiment\n\n#pycaret_X = train_data.copy()\n#pycaret_X.drop(columns=['ID','State.Name','Year'])\n\n#exp = RegressionExperiment()\n#exp.setup(train_data, target='Incidence',session_id=1)\n\n#best = exp.compare_models()","metadata":{"execution":{"iopub.status.busy":"2023-08-10T12:42:27.361000Z","iopub.execute_input":"2023-08-10T12:42:27.361368Z","iopub.status.idle":"2023-08-10T12:42:27.375873Z","shell.execute_reply.started":"2023-08-10T12:42:27.361336Z","shell.execute_reply":"2023-08-10T12:42:27.374390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#exp.tune_model(best)","metadata":{"execution":{"iopub.status.busy":"2023-08-10T12:42:27.379644Z","iopub.execute_input":"2023-08-10T12:42:27.380410Z","iopub.status.idle":"2023-08-10T12:42:27.390501Z","shell.execute_reply.started":"2023-08-10T12:42:27.380356Z","shell.execute_reply":"2023-08-10T12:42:27.388348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#exp.plot_model(best, plot = 'feature')","metadata":{"execution":{"iopub.status.busy":"2023-08-10T12:42:27.392489Z","iopub.execute_input":"2023-08-10T12:42:27.392892Z","iopub.status.idle":"2023-08-10T12:42:27.405547Z","shell.execute_reply.started":"2023-08-10T12:42:27.392861Z","shell.execute_reply":"2023-08-10T12:42:27.404048Z"},"trusted":true},"execution_count":null,"outputs":[]}]}