{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Getting Started","metadata":{}},{"cell_type":"markdown","source":"## Example of (a very simple) Predictive Model","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport os\nfrom sklearn.ensemble import RandomForestRegressor","metadata":{"execution":{"iopub.status.busy":"2023-07-24T15:25:45.611515Z","iopub.execute_input":"2023-07-24T15:25:45.612192Z","iopub.status.idle":"2023-07-24T15:25:47.628622Z","shell.execute_reply.started":"2023-07-24T15:25:45.612148Z","shell.execute_reply":"2023-07-24T15:25:47.627564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file_path_train = \"/kaggle/input/air-toxicity-and-chronic-respiratory-diseases-us/train.csv\"\nfile_path_test = \"/kaggle/input/air-toxicity-and-chronic-respiratory-diseases-us/test.csv\"\nfile_path_sample_submission = \"/kaggle/input/air-toxicity-and-chronic-respiratory-diseases-us/sample_submission.csv\"\nfile_path_submission = \"submission.csv\"","metadata":{"execution":{"iopub.status.busy":"2023-07-24T15:25:47.630556Z","iopub.execute_input":"2023-07-24T15:25:47.630883Z","iopub.status.idle":"2023-07-24T15:25:47.636559Z","shell.execute_reply.started":"2023-07-24T15:25:47.630856Z","shell.execute_reply":"2023-07-24T15:25:47.635248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_supplementary(filename, parammeter):\n    file_path_supplementary_LEAD = os.path.join(\"/kaggle/input/air-toxicity-and-chronic-respiratory-diseases-us/\", filename)\n    supplementary = pd.read_parquet(file_path_supplementary_LEAD)\n    supplementary = supplementary[supplementary['Parameter.Name'] == parammeter]\n    supplementary.drop([\"Parameter.Name\", \"X1st.Max.Value\"], axis=1, inplace=True)\n    supplementary['Date.Local'] = supplementary['Date.Local'].apply(lambda x: x.year)\n    supplementary.rename(columns={'Date.Local': 'Year', 'Arithmetic.Mean': parammeter}, inplace=True)\n    supplementary = pd.DataFrame(supplementary.groupby([\"State.Name\",\"Year\"])[parammeter].median())\n    return supplementary","metadata":{"execution":{"iopub.status.busy":"2023-07-24T15:25:47.637812Z","iopub.execute_input":"2023-07-24T15:25:47.638157Z","iopub.status.idle":"2023-07-24T15:25:47.655996Z","shell.execute_reply.started":"2023-07-24T15:25:47.638109Z","shell.execute_reply":"2023-07-24T15:25:47.654518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def input_supplementary(input_data):\n    df = read_supplementary(input_data.iloc[0].file, input_data.iloc[0].parameter)\n    for i in range(1, input_data.shape[0]):\n        df1 = read_supplementary(input_data.iloc[i].file, input_data.iloc[i].parameter)\n        df = pd.merge(df, df1, on=[\"State.Name\", \"Year\"], how='outer')\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-07-24T15:25:47.658723Z","iopub.execute_input":"2023-07-24T15:25:47.659772Z","iopub.status.idle":"2023-07-24T15:25:47.673942Z","shell.execute_reply.started":"2023-07-24T15:25:47.659732Z","shell.execute_reply":"2023-07-24T15:25:47.672545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def supp_feature(supplementary, state, year, lag):\n    supplementary_state = supplementary[supplementary.index.get_level_values('State.Name') == state]\n    return pd.concat([pd.Series(index = ['State.Name', 'Year'], data = [state, year]), \n                    supplementary_state[(supplementary_state.index.get_level_values('Year') >= year-lag)&\n                               (supplementary_state.index.get_level_values('Year') <= year)] \\\n    .median()])","metadata":{"execution":{"iopub.status.busy":"2023-07-24T15:25:47.675527Z","iopub.execute_input":"2023-07-24T15:25:47.676729Z","iopub.status.idle":"2023-07-24T15:25:47.689023Z","shell.execute_reply.started":"2023-07-24T15:25:47.676590Z","shell.execute_reply":"2023-07-24T15:25:47.687652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def age_cleaning(x):\n    return int(x.split(\"-\")[0])/5-13 ","metadata":{"execution":{"iopub.status.busy":"2023-07-24T15:25:47.690875Z","iopub.execute_input":"2023-07-24T15:25:47.691287Z","iopub.status.idle":"2023-07-24T15:25:47.704512Z","shell.execute_reply.started":"2023-07-24T15:25:47.691251Z","shell.execute_reply":"2023-07-24T15:25:47.703207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_data = pd.DataFrame({\n    \"file\" : [\"supplementary_NONOxNOy.parquet\", \"supplementary_LEAD.parquet\", \"supplementary_VOCS.parquet\", \n              \"supplementary_VOCS.parquet\", \"supplementary_HAPS.parquet\",\n              \"supplementary_HAPS.parquet\"],\n    \"parameter\": [\"Nitric oxide (NO)\", \"Lead (TSP) STP\", \"Toluene\", \n                  \"Total NMOC (non-methane organic compound)\", \"Benzene\", \"Ethylene dichloride\"]\n})","metadata":{"execution":{"iopub.status.busy":"2023-07-24T15:25:47.705972Z","iopub.execute_input":"2023-07-24T15:25:47.706463Z","iopub.status.idle":"2023-07-24T15:25:47.725539Z","shell.execute_reply.started":"2023-07-24T15:25:47.706428Z","shell.execute_reply":"2023-07-24T15:25:47.724545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = list({'Age','Nitric oxide (NO)',\n       'Lead (TSP) STP', 'Toluene',\n       'Total NMOC (non-methane organic compound)', 'Benzene',\n       'Ethylene dichloride'})","metadata":{"execution":{"iopub.status.busy":"2023-07-24T15:25:47.727526Z","iopub.execute_input":"2023-07-24T15:25:47.727993Z","iopub.status.idle":"2023-07-24T15:25:47.740453Z","shell.execute_reply.started":"2023-07-24T15:25:47.727950Z","shell.execute_reply":"2023-07-24T15:25:47.739341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"supplementary = input_supplementary(input_data)","metadata":{"execution":{"iopub.status.busy":"2023-07-24T15:25:47.742401Z","iopub.execute_input":"2023-07-24T15:25:47.743211Z","iopub.status.idle":"2023-07-24T15:26:21.504190Z","shell.execute_reply.started":"2023-07-24T15:25:47.743164Z","shell.execute_reply":"2023-07-24T15:26:21.502962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(file_path_train)\ntrain['Age'] = train['Age'].map(age_cleaning)","metadata":{"execution":{"iopub.status.busy":"2023-07-24T15:26:21.510107Z","iopub.execute_input":"2023-07-24T15:26:21.510469Z","iopub.status.idle":"2023-07-24T15:26:21.538262Z","shell.execute_reply.started":"2023-07-24T15:26:21.510439Z","shell.execute_reply":"2023-07-24T15:26:21.537410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"locations = sorted(set(supplementary.index.map(lambda x: x[0])))","metadata":{"execution":{"iopub.status.busy":"2023-07-24T15:26:21.539617Z","iopub.execute_input":"2023-07-24T15:26:21.540266Z","iopub.status.idle":"2023-07-24T15:26:21.546518Z","shell.execute_reply.started":"2023-07-24T15:26:21.540233Z","shell.execute_reply":"2023-07-24T15:26:21.545237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"supplementary = pd.DataFrame([supp_feature(supplementary, loc, y, 10) for loc in locations\n                   for y in range(1990, 2019)])","metadata":{"execution":{"iopub.status.busy":"2023-07-24T15:26:21.547978Z","iopub.execute_input":"2023-07-24T15:26:21.548387Z","iopub.status.idle":"2023-07-24T15:26:25.794591Z","shell.execute_reply.started":"2023-07-24T15:26:21.548356Z","shell.execute_reply":"2023-07-24T15:26:25.793172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_supp = pd.merge(train, supplementary, on=[\"State.Name\", \"Year\"], how='left')\ndel(train)\ndel(locations)","metadata":{"execution":{"iopub.status.busy":"2023-07-24T15:26:25.796104Z","iopub.execute_input":"2023-07-24T15:26:25.796461Z","iopub.status.idle":"2023-07-24T15:26:25.810572Z","shell.execute_reply.started":"2023-07-24T15:26:25.796428Z","shell.execute_reply":"2023-07-24T15:26:25.809226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = train_supp['Incidence']","metadata":{"execution":{"iopub.status.busy":"2023-07-24T15:26:25.812533Z","iopub.execute_input":"2023-07-24T15:26:25.813012Z","iopub.status.idle":"2023-07-24T15:26:25.822629Z","shell.execute_reply.started":"2023-07-24T15:26:25.812967Z","shell.execute_reply":"2023-07-24T15:26:25.821408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = train_supp[features]","metadata":{"execution":{"iopub.status.busy":"2023-07-24T15:26:25.823750Z","iopub.execute_input":"2023-07-24T15:26:25.824397Z","iopub.status.idle":"2023-07-24T15:26:25.843041Z","shell.execute_reply.started":"2023-07-24T15:26:25.824357Z","shell.execute_reply":"2023-07-24T15:26:25.841481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = X.interpolate(method=\"spline\", order=3)","metadata":{"execution":{"iopub.status.busy":"2023-07-24T15:26:25.844834Z","iopub.execute_input":"2023-07-24T15:26:25.845363Z","iopub.status.idle":"2023-07-24T15:26:26.036671Z","shell.execute_reply.started":"2023-07-24T15:26:25.845315Z","shell.execute_reply":"2023-07-24T15:26:26.034675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import h2o\nfrom h2o.automl import H2OAutoML\nh2o.init()","metadata":{"execution":{"iopub.status.busy":"2023-07-24T15:26:26.038565Z","iopub.execute_input":"2023-07-24T15:26:26.039603Z","iopub.status.idle":"2023-07-24T15:26:35.620536Z","shell.execute_reply.started":"2023-07-24T15:26:26.039552Z","shell.execute_reply":"2023-07-24T15:26:35.619251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_supp.to_csv('train.csv')\n\ntrain_data = h2o.import_file(\"train.csv\")\n#test_data = h2o.import_file(path=data_folder + \"/test.csv\")\n#df_samp = pd.read_csv('/kaggle/input/playground-series-s3e19/sample_submission.csv')   ","metadata":{"execution":{"iopub.status.busy":"2023-07-24T15:26:35.622464Z","iopub.execute_input":"2023-07-24T15:26:35.623852Z","iopub.status.idle":"2023-07-24T15:26:37.329826Z","shell.execute_reply.started":"2023-07-24T15:26:35.623800Z","shell.execute_reply":"2023-07-24T15:26:37.328441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initialize H2O AutoML\naml = H2OAutoML(max_models=10, seed=42)\n\n# Train the AutoML model on your data\naml.train(y=\"Incidence\", training_frame=train_data)","metadata":{"execution":{"iopub.status.busy":"2023-07-24T15:26:37.332946Z","iopub.execute_input":"2023-07-24T15:26:37.334294Z","iopub.status.idle":"2023-07-24T15:28:39.188397Z","shell.execute_reply.started":"2023-07-24T15:26:37.334242Z","shell.execute_reply":"2023-07-24T15:28:39.186829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2023-07-24T15:28:39.190136Z","iopub.execute_input":"2023-07-24T15:28:39.190577Z","iopub.status.idle":"2023-07-24T15:28:39.206209Z","shell.execute_reply.started":"2023-07-24T15:28:39.190536Z","shell.execute_reply":"2023-07-24T15:28:39.204575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"aml.explain(train_data)","metadata":{"execution":{"iopub.status.busy":"2023-07-24T15:30:03.084046Z","iopub.execute_input":"2023-07-24T15:30:03.084557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv(file_path_test)\ntest['Age'] =  test['Age'].map(age_cleaning)","metadata":{"execution":{"iopub.status.busy":"2023-07-24T15:28:39.208899Z","iopub.execute_input":"2023-07-24T15:28:39.210103Z","iopub.status.idle":"2023-07-24T15:28:39.242480Z","shell.execute_reply.started":"2023-07-24T15:28:39.210049Z","shell.execute_reply":"2023-07-24T15:28:39.241192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_supp = pd.merge(test, supplementary, on=[\"State.Name\", \"Year\"], how='left')\n","metadata":{"execution":{"iopub.status.busy":"2023-07-24T15:28:39.244784Z","iopub.execute_input":"2023-07-24T15:28:39.245689Z","iopub.status.idle":"2023-07-24T15:28:39.263748Z","shell.execute_reply.started":"2023-07-24T15:28:39.245641Z","shell.execute_reply":"2023-07-24T15:28:39.262604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_supp.to_csv('test.csv')\n\ntest_data = h2o.import_file(\"test.csv\")\n","metadata":{"execution":{"iopub.status.busy":"2023-07-24T15:28:39.269812Z","iopub.execute_input":"2023-07-24T15:28:39.273104Z","iopub.status.idle":"2023-07-24T15:28:39.700372Z","shell.execute_reply.started":"2023-07-24T15:28:39.273047Z","shell.execute_reply":"2023-07-24T15:28:39.699065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = aml.predict(test_data)\n\n# Convert predictions to a pandas DataFrame\npredictions_df = predictions.as_data_frame()","metadata":{"execution":{"iopub.status.busy":"2023-07-24T15:28:39.702190Z","iopub.execute_input":"2023-07-24T15:28:39.702664Z","iopub.status.idle":"2023-07-24T15:28:41.335914Z","shell.execute_reply.started":"2023-07-24T15:28:39.702619Z","shell.execute_reply":"2023-07-24T15:28:41.334796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['Incidence']=predictions_df.predict\ntest[['ID','Incidence']].to_csv('subm.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2023-07-24T15:28:41.337652Z","iopub.execute_input":"2023-07-24T15:28:41.338922Z","iopub.status.idle":"2023-07-24T15:28:41.371722Z","shell.execute_reply.started":"2023-07-24T15:28:41.338873Z","shell.execute_reply":"2023-07-24T15:28:41.370803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = X.interpolate(method=\"spline\", order=3)","metadata":{"execution":{"iopub.status.busy":"2023-07-24T15:28:41.373647Z","iopub.execute_input":"2023-07-24T15:28:41.374560Z","iopub.status.idle":"2023-07-24T15:28:41.382205Z","shell.execute_reply.started":"2023-07-24T15:28:41.374514Z","shell.execute_reply":"2023-07-24T15:28:41.381074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = X.fillna(0)","metadata":{"execution":{"iopub.status.busy":"2023-07-24T15:28:41.383809Z","iopub.execute_input":"2023-07-24T15:28:41.385693Z","iopub.status.idle":"2023-07-24T15:28:41.393727Z","shell.execute_reply.started":"2023-07-24T15:28:41.385592Z","shell.execute_reply":"2023-07-24T15:28:41.392451Z"},"trusted":true},"execution_count":null,"outputs":[]}]}