{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def toDF(pm25_filename,temp_filename,wind_filename):\n  pm25_df = pd.read_csv('/kaggle/input/2110446-data-science-and-data-engineering-2021/DS_kaggle_edited_v3/'+pm25_filename, sep='\\t', header=None, skiprows=10)\n  pm25_df['Time'] = pd.to_datetime({'year': pm25_df[0], 'month': pm25_df[1], 'day':pm25_df[2],'hour': pm25_df[3]})\n  pm25_df=pm25_df.drop([0,1,2,3,5,6],axis=1)\n  pm25_df['Time'] = pm25_df['Time'].dt.tz_localize('UTC').dt.tz_convert('Asia/Bangkok')\n  pm25_df['Time'] = pm25_df['Time'].dt.tz_localize(None)\n  pm25_df.columns = ['PM25', 'Time']\n  pm25_df = pm25_df.set_index('Time')\n  pm25_df = pm25_df[~pm25_df.index.duplicated(keep='first')]\n  pm25_df.index = pd.DatetimeIndex(pm25_df.index)\n\n  temp_df = pd.read_csv('/kaggle/input/2110446-data-science-and-data-engineering-2021/DS_kaggle_edited_v3/'+temp_filename)\n  temp_df['datetime'] = pd.to_datetime(temp_df['datetime'])\n  temp_df.set_index(temp_df['datetime'],inplace=True)\n  temp_df.drop(columns={'datetime','lat','long'},inplace=True)\n  temp_df.columns = ['Temp']\n  temp_df = temp_df.resample('h').pad()\n\n  wind_df = pd.read_csv('/kaggle/input/2110446-data-science-and-data-engineering-2021/DS_kaggle_edited_v3/'+wind_filename)\n  wind_df['datetime'] = pd.to_datetime(wind_df['datetime'])\n  wind_df.set_index(wind_df['datetime'],inplace=True)\n  wind_df.drop(columns={'datetime','lat','long'},inplace=True)\n  wind_df.columns = ['WindDir', 'WindSpeed']\n  wind_df = wind_df.resample('h').bfill()\n\n  df = pm25_df.merge(temp_df,left_index=True,right_index=True)\n  df = df.merge(wind_df,left_index=True,right_index=True)\n\n  df = df[['Temp','WindSpeed','WindDir','PM25']]\n\n  return df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def toDFtest(pm25_filename,temp_filename,wind_filename):\n  pm25_df = pd.read_csv('/kaggle/input/2110446-data-science-and-data-engineering-2021/DS_kaggle_edited_v3/'+pm25_filename)\n  pm25_df.columns = ['Time', 'PM25']\n  pm25_df['Time'] = pd.to_datetime(pm25_df['Time'], infer_datetime_format=True)\n  pm25_df = pm25_df.set_index('Time')\n  pm25_df = pm25_df[~pm25_df.index.duplicated(keep='first')]\n  pm25_df.index = pd.DatetimeIndex(pm25_df.index)\n\n  temp_df = pd.read_csv('/kaggle/input/2110446-data-science-and-data-engineering-2021/DS_kaggle_edited_v3/'+temp_filename)\n  temp_df['datetime'] = pd.to_datetime(temp_df['datetime'])\n  temp_df.set_index(temp_df['datetime'],inplace=True)\n  temp_df.drop(columns={'datetime','lat','long'},inplace=True)\n  temp_df.columns = ['Temp']\n  temp_df = temp_df.resample('h').bfill()\n\n  wind_df = pd.read_csv('/kaggle/input/2110446-data-science-and-data-engineering-2021/DS_kaggle_edited_v3/'+wind_filename)\n  wind_df['datetime'] = pd.to_datetime(wind_df['datetime'])\n  wind_df.set_index(wind_df['datetime'],inplace=True)\n  wind_df.drop(columns={'datetime','lat','long'},inplace=True)\n  wind_df.columns = ['WindDir', 'WindSpeed']\n  wind_df = wind_df.resample('h').bfill()\n\n  df = pm25_df.merge(temp_df,left_index=True,right_index=True)\n  df = df.merge(wind_df,left_index=True,right_index=True)\n\n  df = df[['Temp','WindSpeed','WindDir','PM25']]\n\n\n  return df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bkk_train = toDF('Bangkok/Train/Bangkok.txt', 'Bangkok/Train/3H_temperature_Bangkok.csv', 'Bangkok/Train/3H_wind_Bangkok.csv')\nctb_train = toDF('Chanthaburi/Train/Chanthaburi.txt', 'Chanthaburi/Train/3H_temperature_Chanthaburi.csv', 'Chanthaburi/Train/3H_wind_Chanthaburi.csv')\ncnx_train = toDF('Chiang Mai/Train/Chiang_Mai.txt', 'Chiang Mai/Train/3H_temperature_Chiang Mai.csv' , 'Chiang Mai/Train/3H_wind_Chiang Mai.csv')\nkcb_train = toDF('Kanchanaburi/Train/Kanchanaburi.txt', 'Kanchanaburi/Train/3H_temperature_Kanchanaburi.csv', 'Kanchanaburi/Train/3H_wind_Kanchanaburi.csv')\nkk_train  = toDF('Khon Kaen/Train/Khon_Kaen.txt', 'Khon Kaen/Train/3H_temperature_Khon Kaen.csv', 'Khon Kaen/Train/3H_wind_Khon Kaen.csv')\nskl_train = toDF('Songkhla/Train/Songkhla.txt', 'Songkhla/Train/3H_temperature_Songkhla.csv', 'Songkhla/Train/3H_wind_Songkhla.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bkk_test = toDFtest('Bangkok/Test/Bangkok (Thailand timezone).csv', 'Bangkok/Test/3H_temperature_Bangkok.csv', 'Bangkok/Test/3H_wind_Bangkok.csv')\nctb_test = toDFtest('Chanthaburi/Test/Chanthaburi (Thailand timezone).csv', 'Chanthaburi/Test/3H_temperature_Chanthaburi.csv', 'Chanthaburi/Test/3H_wind_Chanthaburi.csv')\ncnx_test = toDFtest('Chiang Mai/Test/Chiang Mai(Thailand timezone).csv', 'Chiang Mai/Test/3H_temperature_Chiang Mai.csv', 'Chiang Mai/Test/3H_wind_Chiang Mai.csv')\nkcb_test = toDFtest('Kanchanaburi/Test/Kanchanaburi(Thailand timezone).csv', 'Kanchanaburi/Test/3H_temperature_Kanchanaburi.csv', 'Kanchanaburi/Test/3H_wind_Kanchanaburi.csv')\nkk_test  = toDFtest('Khon Kaen/Test/Khon Kaen(Thailand timezone).csv', 'Khon Kaen/Test/3H_temperature_Khon Kaen.csv', 'Khon Kaen/Test/3H_wind_Khon Kaen.csv')\nskl_test = toDFtest('Songkhla/Test/Songkhla(Thailand timezone).csv', 'Songkhla/Test/3H_temperature_Songkhla.csv', 'Songkhla/Test/3H_wind_Songkhla.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom numpy import array\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nfrom tensorflow import keras\nimport seaborn as sns;\nimport math\n\nfrom sklearn import preprocessing\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.preprocessing import StandardScaler\n\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import mean_squared_error\n\nfrom keras import optimizers\nfrom keras.models import Sequential\n\nfrom keras.layers import LSTM\nfrom keras.layers.core import Dense, Dropout, Activation\n\nfrom keras.callbacks import ModelCheckpoint\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kk_train.describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kkmean = kk_train.PM25.mean()\nkksd = kk_train.PM25.std()\n\nkk_train.PM25[abs(kk_train.PM25)>(kkmean + 3*kksd)]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kk_train = kk_train.drop(kk_train[kk_train.PM25>(kkmean + 3*kksd)].index)\nkk_train.describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kcb_train.describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kcbmean = kcb_train.PM25.mean()\nkcbsd = kcb_train.PM25.std()\n\nkcb_train.PM25[abs(kcb_train.PM25)>(kcbmean + 3*kcbsd)]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kcb_train = kcb_train.drop(kcb_train.PM25[abs(kcb_train.PM25)>(kcbmean + 3*kcbsd)].index)\nkcb_train.describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnx_train.describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnxmean = cnx_train.PM25.mean()\ncnxsd = cnx_train.PM25.std()\n\ncnx_train.PM25[abs(cnx_train.PM25)>(cnxmean + 3*cnxsd)]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnx_train = cnx_train.drop(cnx_train.PM25[abs(cnx_train.PM25)>(cnxmean + 3*cnxsd)].index)\ncnx_train.describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bkk_train.describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bkkmean = bkk_train.PM25.mean()\nbkksd = bkk_train.PM25.std()\n\nbkk_train.PM25[abs(bkk_train.PM25)>(bkkmean + 3*bkksd)]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bkk_train = bkk_train.drop(bkk_train.PM25[abs(bkk_train.PM25)>(bkkmean + 3*bkksd)].index)\nbkk_train.describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ctb_train.describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ctbmean = ctb_train.PM25.mean()\nctbsd = ctb_train.PM25.std()\n\nctb_train.PM25[abs(ctb_train.PM25)>(ctbmean + 3*ctbsd)]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ctb_train = ctb_train.drop(ctb_train.PM25[abs(ctb_train.PM25)>(ctbmean + 3*ctbsd)].index)\nctb_train.describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skl_train.describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sklmean = skl_train.PM25.mean()\nsklsd = skl_train.PM25.std()\n\nskl_train.PM25[abs(skl_train.PM25)>(sklmean + 3*sklsd)]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skl_train = skl_train.drop(skl_train.PM25[abs(skl_train.PM25)>(sklmean + 3*sklsd)].index)\nskl_train.describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def normalize_df(df):\n    min_max_scaler = preprocessing.MinMaxScaler()\n    df['Temp'] = min_max_scaler.fit_transform(df.Temp.values.reshape(-1,1))\n    df['WindSpeed'] = min_max_scaler.fit_transform(df.WindSpeed.values.reshape(-1,1))\n    df['WindDir'] = min_max_scaler.fit_transform(df.WindDir.values.reshape(-1,1))\n    df['PM25'] = min_max_scaler.fit_transform(df.PM25.values.reshape(-1,1))\n    return df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bkk_train = normalize_df(bkk_train)\nctb_train = normalize_df(ctb_train)\ncnx_train = normalize_df(cnx_train)\nkcb_train = normalize_df(kcb_train)\nkk_train = normalize_df(kk_train)\nskl_train = normalize_df(skl_train)\nbkk_train","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bkk_test = normalize_df(bkk_test)\nctb_test = normalize_df(ctb_test)\ncnx_test = normalize_df(cnx_test)\nkcb_test = normalize_df(kcb_test)\nkk_test = normalize_df(kk_test)\nskl_test = normalize_df(skl_test)\nbkk_test","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prepare_data(df_train, df_test):\n    train_data = df_train.to_numpy()\n    test_data = df_test.to_numpy()\n    \n    train = []\n    for i in range(len(train_data)-4): \n        train.append(train_data[i: i + 4]) \n    train = np.array(train)\n    \n    test = []\n    for i in range(len(test_data) - 4): \n        test.append(test_data[i: i + 4]) \n    test = np.array(test)\n    \n    x_train = train[:, :-1] \n    y_train = train[:, -1][:,-1]\n    \n    x_test = test[:, :-1] \n    y_test = test[:, -1][:,-1]\n    return [x_train,y_train,x_test,y_test]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bkk_X_train, bkk_y_train, bkk_X_test, bkk_y_test = prepare_data(bkk_train, bkk_test)\nctb_X_train, ctb_y_train, ctb_X_test, ctb_y_test = prepare_data(ctb_train, ctb_test)\ncnx_X_train, cnx_y_train, cnx_X_test, cnx_y_test = prepare_data(cnx_train, cnx_test)\nkcb_X_train, kcb_y_train, kcb_X_test, kcb_y_test = prepare_data(kcb_train, kcb_test)\nkk_X_train, kk_y_train, kk_X_test, kk_y_test = prepare_data(kk_train, kk_test)\nskl_X_train, skl_y_train, skl_X_test, skl_y_test = prepare_data(skl_train, skl_test)\n#print (X_train[0], y_train[0])\n#print(skl_X_train.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from numpy.random import seed\nimport tensorflow\nseed(45)\ntensorflow.random.set_seed(123)\nn_features = 4\nprev_days = 4\ndef build_model(layers):\n    p=0.1\n    model = Sequential()\n    model.add(LSTM(50, activation='relu', return_sequences=True, input_shape=(prev_days, n_features)))\n    model.add(Dropout(p)) \n    model.add(LSTM(50, activation='relu'))\n    model.add(Dropout(p))\n    model.add(Dense(1,activation='linear'))\n    model.compile(loss='mse', optimizer='adam', metrics=['mse']) \n    return model\n\nbkk_model = build_model([n_features, prev_days, 1])\ncnx_model = build_model([n_features, prev_days, 1])\nctb_model = build_model([n_features, prev_days, 1])\nkcb_model = build_model([n_features, prev_days, 1])\nkk_model = build_model([n_features, prev_days, 1])\nskl_model = build_model([n_features, prev_days, 1])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def denormalize_df(df, normalized_value): \n    df = df['PM25'].values.reshape(-1,1)\n    normalized_value = normalized_value.reshape(-1,1)\n\n    min_max_scaler = preprocessing.MinMaxScaler()\n    _ = min_max_scaler.fit_transform(df)\n    denorm = min_max_scaler.inverse_transform(normalized_value)\n    return denorm","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filename_list = ['bkk_pm25_forecast.h5', 'ctb_pm25_forecast.h5', 'cnx_pm25_forecast.h5', 'kcb_pm25_forecast.h5', 'kk_pm25_forecast.h5', 'skl_pm25_forecast.h5']\n\nfilepath = '../output/kaggle/working/bkk_pm25_forecast.h5'\ncheckpoint = ModelCheckpoint(filepath, monitor='val_loss', verbose=1, save_best_only=True, mode='min')\ncallbacks_list = [checkpoint]\nhist = bkk_model.fit(bkk_X_train, bkk_y_train, validation_data=(bkk_X_test, bkk_y_test), epochs=20 ,verbose = 1)\n\n# def train_model(filename):\n#     filepath = '../output/kaggle/working/' + filename\n#     checkpoint = ModelCheckpoint(filepath, monitor='val_loss', verbose=1, save_best_only=True, mode='min')\n#     callbacks_list = [checkpoint]\n#     hist = model.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=20 ,verbose = 1)\n#     model.save_weights(filename)\n    \n# for filename in filename_list:\n#     print(\"==========================================\")\n#     train_model(filename)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bkk_model.save_weights('bkk_pm25_forecast.h5')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bkk_model.load_weights('bkk_pm25_forecast.h5')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bkk_yhat = bkk_model.predict(bkk_X_test)\nfor i in range(5):\n    print(bkk_yhat[i])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_bkk_train = toDF('Bangkok/Train/Bangkok.txt', 'Bangkok/Train/3H_temperature_Bangkok.csv', 'Bangkok/Train/3H_wind_Bangkok.csv')\nnew_bkk_test = toDFtest('Bangkok/Test/Bangkok (Thailand timezone).csv', 'Bangkok/Test/3H_temperature_Bangkok.csv', 'Bangkok/Test/3H_wind_Bangkok.csv')\n\n#new_bkk_train.PM25[abs(new_bkk_train.PM25)>(bkkmean + 5*bkksd)]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# new_bkk_train = new_bkk_train.drop(new_bkk_train.PM25[abs(new_bkk_train.PM25)>(bkkmean + 5*bkksd)].index)\n# new_bkk_train.describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bkk_new_pred = denormalize_df(new_bkk_train, bkk_yhat)\nbkk_newy_test = denormalize_df(new_bkk_test, bkk_y_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import mean_squared_error\nprint(\"RMSE : \",math.sqrt(mean_squared_error(bkk_newy_test, bkk_new_pred))) #RMSE\nprint(bkk_newy_test.mean())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from matplotlib.pyplot import figure\nfigure(num=None, figsize=(18, 6), dpi=80, facecolor='w', edgecolor='k')\nplt.title(\"BKK Prediction\")\nplt.plot(bkk_newy_test, color='g',label='ground truth')\nplt.plot(bkk_new_pred, alpha=.7, color='r',label='predict')\nplt.legend(loc=\"upper right\")\n\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filepath = '../output/kaggle/working/ctb_pm25_forecast.h5'\ncheckpoint = ModelCheckpoint(filepath, monitor='val_loss', verbose=1, save_best_only=True, mode='min')\ncallbacks_list = [checkpoint]\nhist = ctb_model.fit(ctb_X_train, ctb_y_train, validation_data=(ctb_X_test, ctb_y_test), epochs=20 ,verbose = 1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ctb_model.save_weights('ctb_pm25_forecast.h5')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ctb_model.load_weights('ctb_pm25_forecast.h5')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ctb_yhat = ctb_model.predict(ctb_X_test)\nfor i in range(5):\n    print(ctb_yhat[i])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_ctb_train = toDF('Chanthaburi/Train/Chanthaburi.txt', 'Chanthaburi/Train/3H_temperature_Chanthaburi.csv', 'Chanthaburi/Train/3H_wind_Chanthaburi.csv')\nnew_ctb_test = toDFtest('Chanthaburi/Test/Chanthaburi (Thailand timezone).csv', 'Chanthaburi/Test/3H_temperature_Chanthaburi.csv', 'Chanthaburi/Test/3H_wind_Chanthaburi.csv')\n\n#new_ctb_train.PM25[abs(new_ctb_train.PM25)>(ctbmean + 3*ctbsd)]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# new_ctb_train = new_ctb_train.drop(new_ctb_train.PM25[abs(new_ctb_train.PM25)>(ctbmean + 3*ctbsd)].index)\n# new_ctb_train.describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ctb_new_pred = denormalize_df(new_ctb_train, ctb_yhat)\nctb_newy_test = denormalize_df(new_ctb_test, ctb_y_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import mean_squared_error\nprint(\"RMSE : \",math.sqrt(mean_squared_error(ctb_newy_test, ctb_new_pred))) #RMSE\nprint(ctb_newy_test.mean())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from matplotlib.pyplot import figure\nfigure(num=None, figsize=(18, 6), dpi=80, facecolor='w', edgecolor='k')\nplt.title(\"Chanthaburi Prediction\")\nplt.plot(ctb_newy_test, color='g',label='ground truth')\nplt.plot(ctb_new_pred, alpha=.7, color='r',label='predict')\nplt.legend(loc=\"upper right\")\n\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filepath = '../output/kaggle/working/cnx_pm25_forecast.h5'\ncheckpoint = ModelCheckpoint(filepath, monitor='val_loss', verbose=1, save_best_only=True, mode='min')\ncallbacks_list = [checkpoint]\nhist = cnx_model.fit(cnx_X_train, cnx_y_train, validation_data=(cnx_X_test, cnx_y_test), epochs=20 ,verbose = 1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnx_model.save_weights('cnx_pm25_forecast.h5')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnx_model.load_weights('cnx_pm25_forecast.h5')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnx_yhat = cnx_model.predict(cnx_X_test)\nfor i in range(5):\n    print(cnx_yhat[i])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_cnx_train = toDF('Chiang Mai/Train/Chiang_Mai.txt', 'Chiang Mai/Train/3H_temperature_Chiang Mai.csv' , 'Chiang Mai/Train/3H_wind_Chiang Mai.csv')\nnew_cnx_test = toDFtest('Chiang Mai/Test/Chiang Mai(Thailand timezone).csv', 'Chiang Mai/Test/3H_temperature_Chiang Mai.csv', 'Chiang Mai/Test/3H_wind_Chiang Mai.csv')\n\n#new_cnx_train.PM25[abs(new_cnx_train.PM25)>(cnxmean + 5*cnxsd)]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# new_cnx_train = new_cnx_train.drop(new_cnx_train.PM25[abs(new_cnx_train.PM25)>(cnxmean + 5*cnxsd)].index)\n# new_cnx_train.describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnx_new_pred = denormalize_df(new_cnx_train, cnx_yhat)\ncnx_newy_test = denormalize_df(new_cnx_test, cnx_y_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import mean_squared_error\nprint(\"RMSE : \",math.sqrt(mean_squared_error(cnx_newy_test, cnx_new_pred))) #RMSE\nprint(cnx_newy_test.mean())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from matplotlib.pyplot import figure\nfigure(num=None, figsize=(18, 6), dpi=80, facecolor='w', edgecolor='k')\nplt.title(\"Chaing Mai Prediction\")\nplt.plot(cnx_newy_test, color='g',label='ground truth')\nplt.plot(cnx_new_pred, alpha=.7, color='r',label='predict')\nplt.legend(loc=\"upper right\")\n\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filepath = '../output/kaggle/working/kcb_pm25_forecast.h5'\ncheckpoint = ModelCheckpoint(filepath, monitor='val_loss', verbose=1, save_best_only=True, mode='min')\ncallbacks_list = [checkpoint]\nhist = kcb_model.fit(kcb_X_train, kcb_y_train, validation_data=(kcb_X_test, kcb_y_test), epochs=20 ,verbose = 1)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kcb_model.save_weights('kcb_pm25_forecast.h5')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kcb_model.load_weights('kcb_pm25_forecast.h5')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kcb_yhat = kcb_model.predict(kcb_X_test)\nfor i in range(5):\n    print(kcb_yhat[i])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_kcb_train = toDF('Kanchanaburi/Train/Kanchanaburi.txt', 'Kanchanaburi/Train/3H_temperature_Kanchanaburi.csv', 'Kanchanaburi/Train/3H_wind_Kanchanaburi.csv')\nnew_kcb_test = toDFtest('Kanchanaburi/Test/Kanchanaburi(Thailand timezone).csv', 'Kanchanaburi/Test/3H_temperature_Kanchanaburi.csv', 'Kanchanaburi/Test/3H_wind_Kanchanaburi.csv')\n\n#new_kcb_train.PM25[abs(new_kcb_train.PM25)>(kcbmean + 3*kcbsd)]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#new_kcb_train.PM25[new_kcb_train.PM25>(kcbmean + 3*kcbsd)] = 3*kcbsd\nnew_kcb_train = new_kcb_train.drop(new_kcb_train.PM25[abs(new_kcb_train.PM25)>(kcbmean + 3*kcbsd)].index)\n\nnew_kcb_train.describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kcb_new_pred = denormalize_df(new_kcb_train, kcb_yhat)\nkcb_newy_test = denormalize_df(new_kcb_test, kcb_y_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import mean_squared_error\nprint(\"RMSE : \",math.sqrt(mean_squared_error(kcb_newy_test, kcb_new_pred))) #RMSE\nprint(kcb_newy_test.mean())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from matplotlib.pyplot import figure\nfigure(num=None, figsize=(18, 6), dpi=80, facecolor='w', edgecolor='k')\nplt.title(\"Kanchanaburi Prediction\")\nplt.plot(kcb_newy_test, color='g',label='ground truth')\nplt.plot(kcb_new_pred, alpha=.7, color='r',label='predict')\nplt.legend(loc=\"upper right\")\n\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filepath = '../output/kaggle/working/kk_pm25_forecast.h5'\ncheckpoint = ModelCheckpoint(filepath, monitor='val_loss', verbose=1, save_best_only=True, mode='min')\ncallbacks_list = [checkpoint]\nhist = kk_model.fit(kk_X_train, kk_y_train, validation_data=(kk_X_test, kk_y_test), epochs=20 ,verbose = 1)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kk_model.save_weights('kk_pm25_forecast.h5')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kk_model.load_weights('kk_pm25_forecast.h5')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kk_yhat = kk_model.predict(kk_X_test)\nfor i in range(5):\n    print(kk_yhat[i])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_kk_train  = toDF('Khon Kaen/Train/Khon_Kaen.txt', 'Khon Kaen/Train/3H_temperature_Khon Kaen.csv', 'Khon Kaen/Train/3H_wind_Khon Kaen.csv')\nnew_kk_test  = toDFtest('Khon Kaen/Test/Khon Kaen(Thailand timezone).csv', 'Khon Kaen/Test/3H_temperature_Khon Kaen.csv', 'Khon Kaen/Test/3H_wind_Khon Kaen.csv')\n\n#new_kk_train.PM25[abs(new_kk_train.PM25)>(kkmean + 4*kksd)]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#new_kk_train.PM25[new_kk_train.PM25>(kkmean + 3*kksd)] = 3*kksd\n\nnew_kk_train = new_kk_train.drop(new_kk_train.PM25[abs(new_kk_train.PM25)>(kkmean + 3*kksd)].index)\n\nnew_kk_train.describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kk_new_pred = denormalize_df(new_kk_train, kk_yhat)\nkk_newy_test = denormalize_df(new_kk_test, kk_y_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import mean_squared_error\nprint(\"RMSE : \",math.sqrt(mean_squared_error(kk_newy_test, kk_new_pred))) #RMSE\nprint(kk_newy_test.mean())\nprint(kk_new_pred.mean())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from matplotlib.pyplot import figure\nfigure(num=None, figsize=(18, 6), dpi=80, facecolor='w', edgecolor='k')\nplt.title(\"Khon Kaen Prediction\")\nplt.plot(kk_newy_test, color='g',label='ground truth')\nplt.plot(kk_new_pred, alpha=.7, color='r',label='predict')\nplt.legend(loc=\"upper right\")\n\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filepath = '../output/kaggle/working/skl_pm25_forecast.h5'\ncheckpoint = ModelCheckpoint(filepath, monitor='val_loss', verbose=1, save_best_only=True, mode='min')\ncallbacks_list = [checkpoint]\nhist = skl_model.fit(skl_X_train, skl_y_train, validation_data=(skl_X_test, skl_y_test), epochs=20 ,verbose = 1)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skl_model.save_weights('skl_pm25_forecast.h5')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skl_model.load_weights('skl_pm25_forecast.h5')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skl_yhat = skl_model.predict(skl_X_test)\nfor i in range(5):\n    print(skl_yhat[i])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_skl_train = toDF('Songkhla/Train/Songkhla.txt', 'Songkhla/Train/3H_temperature_Songkhla.csv', 'Songkhla/Train/3H_wind_Songkhla.csv')\nnew_skl_test = toDFtest('Songkhla/Test/Songkhla(Thailand timezone).csv', 'Songkhla/Test/3H_temperature_Songkhla.csv', 'Songkhla/Test/3H_wind_Songkhla.csv')\n\n#new_skl_train.PM25[abs(new_skl_train.PM25)>(sklmean + 3*sklsd)]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# new_skl_train.PM25[new_skl_train.PM25>(sklmean + 3*sklsd)] = 3*sklsd\n# new_skl_train = new_skl_train.drop(new_skl_train.PM25[abs(new_skl_train.PM25)>(sklmean + 3*sklsd)].index)\n\n# new_skl_train.describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skl_new_pred = denormalize_df(new_skl_train, skl_yhat)\nskl_newy_test = denormalize_df(new_skl_test, skl_y_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import mean_squared_error\nprint(\"RMSE : \",math.sqrt(mean_squared_error(skl_newy_test, skl_new_pred))) #RMSE\nprint(skl_newy_test.mean())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from matplotlib.pyplot import figure\nfigure(num=None, figsize=(18, 6), dpi=80, facecolor='w', edgecolor='k')\nplt.title(\"Songkhla Prediction\")\nplt.plot(skl_newy_test, color='g',label='ground truth')\nplt.plot(skl_new_pred, alpha=.7, color='r',label='predict')\nplt.legend(loc=\"upper right\")\n\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output = pd.DataFrame()\noutput_test = pd.DataFrame()\ndf_ctb_new_pred = pd.DataFrame(ctb_new_pred)\ndf_ctb_newy_test = pd.DataFrame(ctb_newy_test)\nfor i in range(1112):\n    i *= 6\n    output = pd.concat([output, df_ctb_new_pred[i+4:i+76]])\n    output_test = pd.concat([output_test, df_ctb_newy_test[i+4:i+76]])\noutput","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_cnx_new_pred = pd.DataFrame(cnx_new_pred)\ndf_cnx_newy_test = pd.DataFrame(cnx_newy_test)\nfor i in range(1112):\n    i *= 6\n    output = pd.concat([output, df_cnx_new_pred[i+4:i+76]])\n    output_test = pd.concat([output_test, df_cnx_newy_test[i+4:i+76]])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_kcb_new_pred = pd.DataFrame(kcb_new_pred)\ndf_kcb_newy_test = pd.DataFrame(kcb_newy_test)\nfor i in range(1112):\n    i *= 6\n    output = pd.concat([output, df_kcb_new_pred[i+4:i+76]])\n    output_test = pd.concat([output_test, df_kcb_newy_test[i+4:i+76]])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_bkk_new_pred = pd.DataFrame(bkk_new_pred)\ndf_bkk_newy_test = pd.DataFrame(bkk_newy_test)\nfor i in range(1112):\n    i *= 6\n    output = pd.concat([output, df_bkk_new_pred[i+4:i+76]])\n    output_test = pd.concat([output_test, df_bkk_newy_test[i+4:i+76]])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_kk_new_pred = pd.DataFrame(kk_new_pred)\ndf_kk_newy_test = pd.DataFrame(kk_newy_test)\nfor i in range(1110):\n    i *= 6\n    output = pd.concat([output, df_kk_new_pred[i+4:i+76]])\n    output_test = pd.concat([output_test, df_kk_newy_test[i+4:i+76]])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_skl_new_pred = pd.DataFrame(skl_new_pred)\ndf_skl_newy_test = pd.DataFrame(skl_newy_test)\nfor i in range(1127):\n    i *= 6\n    output = pd.concat([output, df_skl_new_pred[i+4:i+76]])\n    output_test = pd.concat([output_test, df_skl_newy_test[i+4:i+76]])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import r2_score, mean_squared_error\n\ndef r2_rmse( p,r ):\n    r2 = r2_score( p[0], r[0] )\n    rmse = np.sqrt( mean_squared_error( p[0], r[0] ) )\n    return pd.Series( dict(  r2 = r2, rmse = rmse ) )\n\nr2_rmse(output, output_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame({'Id': list(range(0, len(output))), 'Predicted': output.to_numpy().flatten()})\ndf.to_csv('/kaggle/working/submission.csv', index=False)\ndf = pd.DataFrame({'Id': list(range(0, len(output))), 'Predicted': output_test.to_numpy().flatten()})\ndf.to_csv('/kaggle/working/realdata.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}