{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":7966698,"sourceType":"datasetVersion","datasetId":4687299}],"dockerImageVersionId":30673,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## ST456 Deep Learning Project","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport tensorflow_hub as hub\nimport os\nimport gc\nimport random\nimport matplotlib.pyplot as plt\nimport zipfile\nimport keras\nfrom sklearn.model_selection import train_test_split\nfrom keras import layers\n\n#from keras import ops\nfrom keras.models import Sequential\nfrom keras.layers import Activation, Dropout, Flatten, Dense, BatchNormalization, GlobalMaxPooling2D\nfrom keras.preprocessing import image\nfrom keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Dropout\nfrom keras.optimizers import Adam\nfrom keras.metrics import MeanSquaredError, RootMeanSquaredError\nfrom keras.utils import to_categorical\nfrom keras import backend as K\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom sklearn.model_selection import train_test_split\nfrom tqdm import tqdm\n\nprint(tf.config.list_physical_devices('GPU'))","metadata":{"execution":{"iopub.status.busy":"2024-04-02T23:57:28.338495Z","iopub.execute_input":"2024-04-02T23:57:28.339127Z","iopub.status.idle":"2024-04-02T23:57:28.347264Z","shell.execute_reply.started":"2024-04-02T23:57:28.339097Z","shell.execute_reply":"2024-04-02T23:57:28.346414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_path = '/kaggle/input/processed-galaxy-data/compressed_img_target.npz'","metadata":{"execution":{"iopub.status.busy":"2024-04-02T23:57:28.483955Z","iopub.execute_input":"2024-04-02T23:57:28.484221Z","iopub.status.idle":"2024-04-02T23:57:28.488120Z","shell.execute_reply.started":"2024-04-02T23:57:28.484199Z","shell.execute_reply":"2024-04-02T23:57:28.487207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load cropped, compressed images\nloaded_arrays = np.load(data_path)\n\ndata = loaded_arrays['images']\ntarget = loaded_arrays['target']\n\nbatch_size = 32 * 2  # 2 GPUs\n\nprint(data.shape)","metadata":{"execution":{"iopub.status.busy":"2024-04-02T23:57:28.803735Z","iopub.execute_input":"2024-04-02T23:57:28.804018Z","iopub.status.idle":"2024-04-02T23:57:40.729843Z","shell.execute_reply.started":"2024-04-02T23:57:28.803995Z","shell.execute_reply":"2024-04-02T23:57:40.728959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.getcwd()","metadata":{"execution":{"iopub.status.busy":"2024-04-02T23:57:40.731708Z","iopub.execute_input":"2024-04-02T23:57:40.732363Z","iopub.status.idle":"2024-04-02T23:57:40.738742Z","shell.execute_reply.started":"2024-04-02T23:57:40.732328Z","shell.execute_reply":"2024-04-02T23:57:40.737759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rgb_to_grayscale(images):\n    # convert RGB to grayscale using the formula: Y = 0.299*R + 0.587*G + 0.114*B\n    grayscale_images = np.dot(images[..., :3], [0.299, 0.587, 0.114])\n    grayscale_images = np.squeeze(grayscale_images)\n\n    return grayscale_images\n\n# Define RMSE loss function\ndef rmse_custom(y_true, y_pred):\n    return K.sqrt(K.mean(K.square(y_pred - y_true)))\n","metadata":{"execution":{"iopub.status.busy":"2024-04-02T23:57:40.739918Z","iopub.execute_input":"2024-04-02T23:57:40.740220Z","iopub.status.idle":"2024-04-02T23:57:40.750257Z","shell.execute_reply.started":"2024-04-02T23:57:40.740188Z","shell.execute_reply":"2024-04-02T23:57:40.749537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train/validation/test split due to data augmentation\n# 72% train, 8% validation, 20% test\nX_train, X_test, y_train, y_test = train_test_split(data, target, test_size=0.2, random_state = 42)\nX_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.1, random_state=42)\n\n# Normalize pixel values to range [0, 1]\nX_train = X_train.astype('float32') / 255\nX_test = X_test.astype('float32') / 255\nX_val = X_val.astype('float32') / 255\n\ny_train = y_train.astype('float32')\ny_test = y_test.astype('float32')\ny_val = y_val.astype('float32')","metadata":{"execution":{"iopub.status.busy":"2024-04-02T23:57:40.752428Z","iopub.execute_input":"2024-04-02T23:57:40.753008Z","iopub.status.idle":"2024-04-02T23:57:42.186591Z","shell.execute_reply.started":"2024-04-02T23:57:40.752976Z","shell.execute_reply":"2024-04-02T23:57:42.185804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.shape, X_val.shape, X_test.shape","metadata":{"execution":{"iopub.status.busy":"2024-04-02T23:57:42.187683Z","iopub.execute_input":"2024-04-02T23:57:42.187994Z","iopub.status.idle":"2024-04-02T23:57:42.194367Z","shell.execute_reply.started":"2024-04-02T23:57:42.187970Z","shell.execute_reply":"2024-04-02T23:57:42.193411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_augmentation = ImageDataGenerator(\n    rotation_range=360,\n    width_shift_range=0.2,\n    height_shift_range=0.2,\n    shear_range=0.2,\n    zoom_range=0.2,\n    horizontal_flip=True,\n    fill_mode='nearest'\n)\n\ntrain_datagen = data_augmentation.flow(X_train, y_train, batch_size=batch_size)\nvalidation_datagen = ImageDataGenerator().flow(X_val, y_val, batch_size=batch_size)\n","metadata":{"execution":{"iopub.status.busy":"2024-04-02T23:57:42.195574Z","iopub.execute_input":"2024-04-02T23:57:42.195849Z","iopub.status.idle":"2024-04-02T23:57:42.205952Z","shell.execute_reply.started":"2024-04-02T23:57:42.195821Z","shell.execute_reply":"2024-04-02T23:57:42.205138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Recording the effects of constraint coefficient on performance of base CNN model\n\nAs we increase the force of the decision tree constraints on the customised loss function, how does this impact model performance?","metadata":{}},{"cell_type":"code","source":"# these regularization parameters correspond to the galaxy-zoo decision tree structure\ndef custom_loss(y_true, y_pred, l=0.001):\n    c1 = tf.abs(tf.reduce_sum(y_pred[:, 0:3], axis=1) - 1) # Class 1 constraint\n    c2 = tf.abs(tf.reduce_sum(y_pred[:, 13:15], axis=1) - 1) # Class 6 constraint\n    c3 = tf.abs((y_pred[:, 3] + y_pred[:, 4]) - y_pred[:, 1]) # Class 2 constraint\n    c4 = tf.abs((y_pred[:, 15] + y_pred[:, 16] + y_pred[:, 17]) - y_pred[:, 10]) # Class 7 constraint\n    c5 = tf.abs(tf.reduce_sum(y_pred[:, 18:25], axis=1) - y_pred[:, 13]) # Class 8 constraint\n    c6 = tf.abs(tf.reduce_sum(y_pred[:, 25:28], axis=1) - y_pred[:, 3]) # Class 9 constraint\n    c7 = tf.abs(tf.reduce_sum(y_pred[:, 5:7], axis=1) - y_pred[:, 4]) # Class 3 constraint\n    c8 = tf.abs(tf.reduce_sum(y_pred[:, 7:9], axis=1) - y_pred[:, 4]) # Class 4 constraint\n    c9 = tf.abs(tf.reduce_sum(y_pred[:, 28:31], axis=1) - y_pred[:, 7]) # Class 10 constraint\n    c10 = tf.abs(tf.reduce_sum(y_pred[:, 9:13], axis=1) - y_pred[:, 7] - y_pred[:, 8]) # Class 5 constraint\n    c11 = tf.abs(tf.reduce_sum(y_pred[:, 31:37], axis=1) - y_pred[:, 7]) # Class 11 constraint\n    \n    # l (lambda) coefficient controls the force of penalties\n    c_loss = l*(c1+c2+c3+c4+c5+c6+c7+c8+c9+c10+c11)\n\n    # base loss (mean squared error)\n    mse_loss = tf.reduce_mean(tf.square(y_true - y_pred))\n\n    # combine base loss with constraints\n    return mse_loss + c_loss\n\ndef custom_loss_wrapper(l=0.001):\n    def loss(y_true, y_pred):\n        return custom_loss(y_true, y_pred, l=l)\n    return loss","metadata":{"execution":{"iopub.status.busy":"2024-04-02T23:57:42.207016Z","iopub.execute_input":"2024-04-02T23:57:42.207300Z","iopub.status.idle":"2024-04-02T23:57:42.221948Z","shell.execute_reply.started":"2024-04-02T23:57:42.207277Z","shell.execute_reply":"2024-04-02T23:57:42.221105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Build CNN model\n# to use two GPUs in parallel\nstrategy = tf.distribute.MirroredStrategy()\nprint('Number of devices: {}'.format(strategy.num_replicas_in_sync))","metadata":{"execution":{"iopub.status.busy":"2024-04-02T23:57:42.223849Z","iopub.execute_input":"2024-04-02T23:57:42.224130Z","iopub.status.idle":"2024-04-02T23:57:42.239759Z","shell.execute_reply.started":"2024-04-02T23:57:42.224107Z","shell.execute_reply":"2024-04-02T23:57:42.238796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"l = [0.001, 0.01, 0.1, 0.2, 0.5, 1]\nnum_epochs = 30\nresults = {'lambda':[], 'rmse':[]}\n\n# so we can keep resetting the weights\n\nfig, axes = plt.subplots(3, 2, figsize=(12, 8), sharex='col', sharey='row')\naxes = axes.flatten()\n\n# Loop through lambda values\nfor i, lamda in enumerate(l):\n    print(f'Training Lambda = {lamda}')\n    print('-'*200)\n    \n    with strategy.scope():\n        model = Sequential()\n        model.add(Conv2D(32, (3, 3), activation='relu', input_shape=(64, 64, 3)))\n        model.add(MaxPooling2D((2, 2)))\n        model.add(Conv2D(64, (3, 3), activation='relu'))\n        model.add(MaxPooling2D((2, 2)))\n        model.add(Conv2D(128, (3, 3), activation='relu'))\n        model.add(MaxPooling2D((2, 2)))\n        model.add(Conv2D(128, (3, 3), activation='relu'))\n        model.add(MaxPooling2D((2, 2)))\n        model.add(Flatten())\n        model.add(Dense(512, activation='relu'))\n        model.add(Dense(37, activation='sigmoid'))\n\n        model.compile(optimizer='adam', loss=custom_loss_wrapper(l=lamda), metrics=[RootMeanSquaredError()])\n\n        # Train the model\n        history = model.fit(\n            train_datagen,\n            steps_per_epoch=len(X_train) // 32,\n            epochs=num_epochs,\n            validation_data=validation_datagen,\n            validation_steps=len(X_val) // 32\n        )\n    \n    # Visualize loss and metric by training epoch\n    train_metric = history.history['root_mean_squared_error']\n    val_metric = history.history['val_root_mean_squared_error']\n    epochs = range(1, len(train_metric) + 1)\n    \n    # Plot training and validation metric\n    axes[i].plot(epochs, train_metric, marker='o', color='b', label='Training RMSE')\n    axes[i].plot(epochs, val_metric, marker='o', color='r', label='Validation RMSE')\n    axes[i].set_title(f'$\\lambda={lamda}$') \n    axes[i].set_xlabel('Epochs') \n    axes[i].set_ylabel('RMSE')\n    plt.tight_layout()\n    \n    # Save plot\n    plt.savefig(f'plot_l_{lamda}.png')\n    \n    # Save model\n    model_path = f'base_model_lambda_{lamda}.keras'\n    model.save(model_path)\n    \n    # Evaluate model on test data\n    test_loss, test_rmse = model.evaluate(X_test, y_test)\n    \n    # Store results\n    results['rmse'].append(test_rmse)\n    results['lambda'].append(lamda)\n","metadata":{"execution":{"iopub.status.busy":"2024-04-02T19:22:23.963642Z","iopub.execute_input":"2024-04-02T19:22:23.964052Z","iopub.status.idle":"2024-04-02T19:31:14.632359Z","shell.execute_reply.started":"2024-04-02T19:22:23.964018Z","shell.execute_reply":"2024-04-02T19:31:14.630958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_df = pd.DataFrame(results)\nresults_df.to_csv('reg_effect.csv', index=False)\nresults_df.head()","metadata":{},"execution_count":null,"outputs":[]}]}