{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\n\nPATH = \"/kaggle/input/applications-of-deep-learning-wustlfall-2021/city/\"\nPATH_TRAIN = os.path.join(PATH, \"train.csv\")\nPATH_TEST = os.path.join(PATH, \"test.csv\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-12-06T19:15:26.528315Z","iopub.execute_input":"2021-12-06T19:15:26.529354Z","iopub.status.idle":"2021-12-06T19:15:26.534253Z","shell.execute_reply.started":"2021-12-06T19:15:26.529313Z","shell.execute_reply":"2021-12-06T19:15:26.533476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv(PATH_TRAIN)\ndf_test = pd.read_csv(PATH_TEST)\n\ndf_train['filename'] = df_train.id.astype(str) + \".jpg\"\ndf_test['filename'] = df_test.id.astype(str) + \".jpg\"","metadata":{"execution":{"iopub.status.busy":"2021-12-06T19:15:26.536046Z","iopub.execute_input":"2021-12-06T19:15:26.536449Z","iopub.status.idle":"2021-12-06T19:15:26.608135Z","shell.execute_reply.started":"2021-12-06T19:15:26.536417Z","shell.execute_reply":"2021-12-06T19:15:26.60729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\n\nimport tensorflow.keras\nimport pandas as pd\nimport sklearn as sk\nimport tensorflow as tf\nimport numpy as np\nimport os\nimport keras_preprocessing\nfrom keras_preprocessing import image\nfrom keras_preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.applications.resnet_v2 import ResNet152V2\nfrom tensorflow.keras.applications.vgg16 import VGG16\nfrom tensorflow.keras.applications.vgg19 import VGG19\nfrom tensorflow.keras.applications.xception import Xception\nfrom tensorflow.keras.layers import Input, Dropout, Reshape\nfrom tensorflow.keras.layers import Dense, GlobalAveragePooling2D\nfrom tensorflow.keras.layers import LeakyReLU, PReLU\nfrom tensorflow.keras.layers import UpSampling2D, Conv2D, MaxPooling2D\nfrom tensorflow.keras.layers import Flatten, BatchNormalization\nfrom tensorflow.keras.layers import Activation, ZeroPadding2D\nfrom tensorflow.keras.models import Sequential, Model, load_model, save_model\nfrom tensorflow.keras.initializers import he_normal\nfrom tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau\nfrom tensorflow.keras.callbacks import EarlyStopping\nfrom tensorflow.keras.metrics import RootMeanSquaredError\nfrom tensorflow.keras.optimizers import Adam\nfrom sklearn.model_selection import KFold","metadata":{"execution":{"iopub.status.busy":"2021-12-06T19:15:26.609171Z","iopub.execute_input":"2021-12-06T19:15:26.609371Z","iopub.status.idle":"2021-12-06T19:15:32.373736Z","shell.execute_reply.started":"2021-12-06T19:15:26.609344Z","shell.execute_reply":"2021-12-06T19:15:32.372813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nWIDTH = 150\nHEIGHT = 150\n\n# Split into train/test\nx_train, x_test, y_train, y_test = train_test_split(    \n    df_train['filename'].values, df_train['sqft'].values, test_size=0.1) \n\ndf_train_cut = pd.DataFrame({\"filename\":x_train,\"sqft\":y_train})\ndf_validate_cut = pd.DataFrame({\"filename\":x_test,\"sqft\":y_test})\nprint(f\"Training shape: {df_train_cut.shape}\")\nprint(f\"Validate shape: {df_validate_cut.shape}\")\n\ntraining_datagen = ImageDataGenerator(\n  rescale = 1./255,\n  horizontal_flip=True,\n  # vertical_flip=True,\n  fill_mode='nearest')\n\ntrain_generator = training_datagen.flow_from_dataframe(\n        dataframe=df_train_cut,\n        directory=PATH,\n        x_col=\"filename\",\n        y_col=\"sqft\",\n        target_size=(HEIGHT, WIDTH),\n        batch_size=32, # Keeping the training batch size small USUALLY increases performance\n        class_mode='raw')\n\nvalidation_datagen = ImageDataGenerator(rescale = 1./255)\n\nval_generator = validation_datagen.flow_from_dataframe(\n        dataframe=df_validate_cut,\n        directory=PATH,\n        x_col=\"filename\",\n        y_col=\"sqft\",\n        target_size=(HEIGHT, WIDTH),\n        batch_size=256, # Make the validation batch size as large as you have memory for\n        class_mode='raw')","metadata":{"execution":{"iopub.status.busy":"2021-12-06T19:15:32.375017Z","iopub.execute_input":"2021-12-06T19:15:32.375241Z","iopub.status.idle":"2021-12-06T19:15:52.802536Z","shell.execute_reply.started":"2021-12-06T19:15:32.375214Z","shell.execute_reply":"2021-12-06T19:15:52.801751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.applications import NASNetMobile\nfrom tensorflow.keras.applications.resnet_v2 import ResNet50V2, ResNet152V2, ResNet101V2\nfrom tensorflow.keras.applications import InceptionResNetV2\nfrom tensorflow.keras.applications import MobileNetV2\nfrom tensorflow.keras.applications import DenseNet121\n\nWIDTH = 150\nHEIGHT = 150\n\ndef build_resnet():\n    input_tensor = Input(shape=(HEIGHT, WIDTH, 3))\n    base_model = ResNet152V2(\n      include_top=False, weights=None, input_tensor=input_tensor,\n      classifier_activation='relu'\n  )\n\n    x=base_model.output\n    x=GlobalAveragePooling2D()(x)\n\n    x = Dense(1024,activation='relu')(x)\n    x = Dense(1024,activation='relu')(x)\n\n    output=Dense(1,activation='linear')(x)\n    model_resnet=Model(inputs=input_tensor,outputs=output)\n    return model_resnet\n\ndef build_vgg():\n    input_tensor = Input(shape=(HEIGHT, WIDTH, 3))\n    base_model = VGG16(\n      include_top=False, weights=None, input_tensor=input_tensor,\n      classifier_activation='relu'\n  )\n\n    x=base_model.output\n    x=GlobalAveragePooling2D()(x)\n\n    x = Dense(1024,activation='relu')(x)\n    x = Dense(1024,activation='relu')(x)\n\n    output=Dense(1,activation='linear')(x)\n    model_vgg=Model(inputs=input_tensor,outputs=output)\n    return model_vgg\n\ndef build_xception():\n    input_tensor = Input(shape=(HEIGHT, WIDTH, 3))\n    base_model = Xception(\n      include_top=False, weights=None, input_tensor=input_tensor,\n      classifier_activation='relu'\n  )\n\n    x=base_model.output\n    x=GlobalAveragePooling2D()(x)\n\n    x = Dense(1024,activation='relu')(x)\n    x= Dense(1024,activation='relu')(x)\n\n    output=Dense(1,activation='linear')(x)\n    model_xception=Model(inputs=input_tensor,outputs=output)\n    return model_xception","metadata":{"execution":{"iopub.status.busy":"2021-12-06T19:34:01.106034Z","iopub.execute_input":"2021-12-06T19:34:01.106774Z","iopub.status.idle":"2021-12-06T19:34:01.120861Z","shell.execute_reply.started":"2021-12-06T19:34:01.106734Z","shell.execute_reply":"2021-12-06T19:34:01.120077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import KFold\nfrom sklearn.metrics import mean_squared_error, log_loss\n\nx = df_train[\"filename\"].values\ny = df_train[\"sqft\"].values\nx_submit = df_test[\"filename\"].values\n\noos_y = []\noos_pred = []\n\nmodels = [build_resnet(),build_xception()]\ndataset_blend_train = np.zeros((x.shape[0], len(models)))\ndataset_blend_test = np.zeros((x_submit.shape[0], len(models)))\nsubmit_datagen = ImageDataGenerator(rescale = 1./255)\nsubmit_generator = submit_datagen.flow_from_dataframe(\n        dataframe=df_test,\n        directory=PATH,\n        x_col=\"filename\",\n        batch_size = 1,\n        shuffle = False,\n        target_size=(HEIGHT, WIDTH),\n        class_mode=None)\nsubmit_generator.reset()\n\n# **change the kfold here**\nkf = KFold(3)\nfolds = list(kf.split(x))\nfold=0\n\nfor j, model in enumerate(models):\n    print(\"Model: {} : {}\".format(j, model))\n    fold_sums = np.zeros((x_submit.shape[0], len(folds)))\n    total_rmse = 0\n    for i, (train, test) in enumerate(folds):\n    #training set\n        train_cut = df_train.loc[train]\n    #testing set\n        val_cut = df_train.loc[test]\n\n        training_datagen = ImageDataGenerator(\n      rescale = 1./255,\n      horizontal_flip=True,\n      fill_mode='nearest')\n\n        train_generator = training_datagen.flow_from_dataframe(\n          dataframe= train_cut,\n          directory=PATH,\n          x_col=\"filename\",\n          y_col=\"sqft\",\n          target_size=(HEIGHT, WIDTH),\n          batch_size=32,\n          class_mode='raw')\n\n        validation_datagen = ImageDataGenerator(rescale = 1./255)\n\n        val_generator = validation_datagen.flow_from_dataframe(\n          dataframe=val_cut,\n          directory=PATH,\n          x_col=\"filename\",\n          y_col=\"sqft\",\n          target_size=(HEIGHT, WIDTH),\n          batch_size=256, # Make the validation batch size as large as you have memory for\n          class_mode='raw')  \n    \n        STEP_SIZE_VALID=val_generator.n//val_generator.batch_size\n\n        model.compile(loss = 'mean_squared_error', optimizer=Adam(learning_rate=1e-5), metrics=[RootMeanSquaredError(name=\"rmse\")])\n        monitor = EarlyStopping(monitor='val_loss', min_delta=1e-3, patience=20, verbose=1, mode='min',\n            restore_best_weights=True)\n    # model_checkpoint1=ModelCheckpoint(\n    #     os.path.join(PATH, 'blend{}_bestmodel.h5'.format(j)),\n    #     monitor=\"val_loss\",save_best_only=True,mode=\"min\",verbose=1)\n    # model_checkpoint2=ModelCheckpoint(os.path.join(PATH, 'resnet152_latestmodel.h5'),monitor=\"val_loss\",save_best_only=False,mode=\"min\",\n    #                                       save_weights_only=True, verbose=0)\n        lr_reduce = ReduceLROnPlateau(monitor='val_loss',factor=0.6,patience=20,verbose=1,mode='auto',min_lr=1e-7)\n        callback = [monitor,lr_reduce]\n        \n        # **change epochs here**\n        model.fit(train_generator, epochs=1, steps_per_epoch=250, validation_data = val_generator, \n              callbacks=callback, verbose = 1, validation_steps=STEP_SIZE_VALID)\n\n        pred = model.predict(val_generator)\n        dataset_blend_train[test, j] = pred[:, 0]\n        pred2 = model.predict(submit_generator,steps=len(df_test))\n        fold_sums[:, i] = pred2[:, 0]\n\n        rmse = np.sqrt(mean_squared_error(val_cut[\"sqft\"].values, pred[:,0]))\n        total_rmse+=rmse\n        print(\"Fold #{}: rmse={}\".format(i,rmse))\n  \n    print(\"Model{}: Mean rmse={}\".format(j,total_rmse/len(folds)))\n    dataset_blend_test[:, j] = fold_sums.mean(1)\n\nprint()\nprint(\"Blending models.\")\n\nfrom sklearn.ensemble import BaggingRegressor\nfrom sklearn.svm import SVR\nblend = BaggingRegressor(SVR())\nblend.fit(dataset_blend_train, y)\nsubmit_data = blend.predict(dataset_blend_test)\nbest_df_submit = pd.DataFrame({\"id\":df_test['id'],'sqft':submit_data.flatten()})\nbest_df_submit.to_csv(\"/kaggle/working/submit.csv\",index = False)\nbest_df_submit.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-06T19:34:44.824815Z","iopub.execute_input":"2021-12-06T19:34:44.825451Z"},"trusted":true},"execution_count":null,"outputs":[]}]}