{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import time, os, sys\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nfrom tensorflow.keras import backend as K\nfrom tensorflow.keras import layers, metrics, losses, callbacks, regularizers\nfrom tensorflow.python.client import device_lib\nfrom sklearn.model_selection import train_test_split\nfrom kaggle_datasets import KaggleDatasets\nplt.rc('figure', autolayout=True)\nplt.rc('axes', labelweight='bold', labelsize='large',titleweight='bold', titlesize=18, titlepad=10)\nplt.rc('image', cmap='magma')\nprint(\"Python version: {}\".format(sys.version_info))\nprint(\"Numpy version: {}\".format(np.version.version))\nprint(\"Tensorflow version: {}\".format(tf.__version__))","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":4.557406,"end_time":"2021-08-25T08:12:51.175525","exception":false,"start_time":"2021-08-25T08:12:46.618119","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-09-03T09:24:13.749282Z","iopub.execute_input":"2021-09-03T09:24:13.749818Z","iopub.status.idle":"2021-09-03T09:24:20.560598Z","shell.execute_reply.started":"2021-09-03T09:24:13.749735Z","shell.execute_reply":"2021-09-03T09:24:20.55964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Download dataset\n# ---------------------------\n!unzip -q -n ../input/galaxy-zoo-the-galaxy-challenge/images_training_rev1.zip -d ../temp/\n!unzip -q -n ../input/galaxy-zoo-the-galaxy-challenge/images_test_rev1.zip -d ../temp/     \nlabels_pd = pd.read_csv('../input/galaxy-zoo-the-galaxy-challenge/training_solutions_rev1.zip',compression='zip')\n\nfor dirname, _, filenames in os.walk('../input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n        \nprint(\"Dataset loading done!\")","metadata":{"execution":{"iopub.status.busy":"2021-09-03T09:32:21.542716Z","iopub.execute_input":"2021-09-03T09:32:21.543096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Init hardware (CPU/GPU/TPU)\n# ---------------------------\nstrategy = tf.distribute.get_strategy() # default distribution strategy in Tensorflow. Works on CPU and single GPU.\nprint('Running on CPU/GPU')\n    ","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2021-09-03T09:30:39.856854Z","iopub.execute_input":"2021-09-03T09:30:39.85723Z","iopub.status.idle":"2021-09-03T09:30:45.990966Z","shell.execute_reply.started":"2021-09-03T09:30:39.857199Z","shell.execute_reply":"2021-09-03T09:30:45.989848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Dataset preprocessing\n# ---------------------\nBATCH_SIZE = 64\nAUTOTUNE = tf.data.AUTOTUNE\ndef get_image_and_features(image_path, training=True):\n    image = tf.io.decode_image(tf.io.read_file(image_path), dtype=tf.dtypes.float32)\n    image = tf.image.resize_with_crop_or_pad(image, 160, 160) \n    if training:\n        label = tf.strings.split(image_path, os.path.sep)[3] # take the galaxy number from image path\n        galaxyID = int(tf.strings.substr(label, 0, tf.strings.length(label)-4)) # remove .jpg extension\n        galaxy_row = labels_pd.loc[labels_pd['GalaxyID'] == galaxyID]\n        features = tf.cast(galaxy_row.iloc[0,1:].values, tf.float32)\n        return image, features\n    else:\n        return image\n    \ndef dataset_preprocessing(images_path, training=True, tpu=False):\n      \n    images_path_ds = tf.data.Dataset.from_tensor_slices(images_path)\n    if training:\n        dataset = images_path_ds.map(lambda x: tf.py_function(func=get_image_and_features, inp=[x], Tout=(tf.float32,tf.float32)),num_parallel_calls=AUTOTUNE)\n    else:\n        dataset = images_path_ds.map(lambda x: tf.py_function(func=get_image_and_features, inp=[x,False], Tout=tf.float32),num_parallel_calls=AUTOTUNE)\n        \n    #dataset = images_path_ds.map(get_image_and_features, num_parallel_calls=tf.data.AUTOTUNE)\n    dataset = dataset.shuffle(2048) if training else dataset\n    dataset = dataset.batch(BATCH_SIZE) # Set batch size\n    dataset = dataset.prefetch(AUTOTUNE) # Add dataset prefetch() operations to reduce read latency while training the model\n    return dataset\n\nimages_path = tf.io.gfile.glob('../temp/images_training_rev1/*')\nseed = 54\ntf.random.set_seed(seed)\nimages_path = tf.random.shuffle(images_path)\nsamples_size = len(images_path)\nprint(\"Number of total samples: {}\".format(samples_size))\n\n# Split train validation sets\nsplitIndex = int(np.floor(samples_size*0.8))\nimages_path_train = images_path[:splitIndex]\nimages_path_val = images_path[splitIndex:]\n     \ntrain_dataset = dataset_preprocessing(images_path_train)\nprint('Number of training batches: %d' % tf.data.experimental.cardinality(train_dataset))\n\nval_dataset = dataset_preprocessing(images_path_val)\nprint('Number of validation batches: %d' % tf.data.experimental.cardinality(val_dataset))\n\nprint(\"Dataset preprocessing done!\")","metadata":{"papermill":{"duration":3.715617,"end_time":"2021-08-25T08:13:17.136783","exception":false,"start_time":"2021-08-25T08:13:13.421166","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-09-03T09:13:47.392399Z","iopub.execute_input":"2021-09-03T09:13:47.392825Z","iopub.status.idle":"2021-09-03T09:13:47.966568Z","shell.execute_reply.started":"2021-09-03T09:13:47.392792Z","shell.execute_reply":"2021-09-03T09:13:47.965368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot example\n# --------------------\ndef plot_example(dataset, rows=2, cols=3):\n    fig, axes = plt.subplots(rows, cols, figsize=(12, 6))\n    images = list(dataset.take(1).as_numpy_iterator())[0][0]\n    labels = list(dataset.take(1).as_numpy_iterator())[0][1]\n    for i in range(rows):\n        for j in range(cols):\n            axes[i,j].grid(False)\n            axes[i,j].axis('off')\n            axes[i,j].imshow(images[cols*i+j, :])\n    plt.show()\n    return images[0].shape, labels[0].shape\nimage_shape, features_num = plot_example(train_dataset)\nprint(\"Images shape is: {}\".format(image_shape))\nprint(\"Features length is: {}\".format(features_num))\n","metadata":{"papermill":{"duration":2.318482,"end_time":"2021-08-25T08:13:19.492136","exception":false,"start_time":"2021-08-25T08:13:17.173654","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-09-03T09:13:55.254816Z","iopub.execute_input":"2021-09-03T09:13:55.255257Z","iopub.status.idle":"2021-09-03T09:14:30.314743Z","shell.execute_reply.started":"2021-09-03T09:13:55.255221Z","shell.execute_reply":"2021-09-03T09:14:30.313697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Model\n# ---------------------\ndef create_model(input_shape, use_augmentation=False):\n    model = tf.keras.models.Sequential(name=\"galaxyClassifier\", layers=[\n        layers.Conv2D(filters=16,kernel_size=(6, 6),activation='relu',input_shape=input_shape),\n        layers.BatchNormalization(),\n        layers.MaxPooling2D(pool_size=(2, 2)),\n        layers.Dropout(0.1),\n        layers.Conv2D(filters=32,kernel_size=(5, 5),activation='relu'),\n        layers.BatchNormalization(),\n        layers.MaxPooling2D(pool_size=(2, 2)),\n        layers.Dropout(0.1),\n        layers.Conv2D(filters=64,kernel_size=(3, 3),activation='relu'),\n        layers.BatchNormalization(),\n        layers.MaxPooling2D(pool_size=(2, 2)),\n        layers.Dropout(0.1),\n        layers.Conv2D(filters=128,kernel_size=(3, 3),activation='relu'),\n        layers.BatchNormalization(),\n        layers.MaxPooling2D(pool_size=(2, 2)),\n        layers.Dropout(0.1),\n        layers.Flatten(),\n        layers.Dense(512, activation='relu'),\n        layers.Dropout(0.2),\n        layers.Dense(256, activation='relu'),\n        layers.Dropout(0.2),\n        layers.Dense(37, activation='sigmoid')\n    ])\n\n    if use_augmentation:\n        data_augmentation = tf.keras.models.Sequential(name='augmentation', layers=[\n            layers.experimental.preprocessing.RandomFlip(\"horizontal_and_vertical\", input_shape=input_shape),\n            layers.experimental.preprocessing.RandomRotation(0.3),\n            layers.experimental.preprocessing.RandomZoom(height_factor=0.2, width_factor=0.2),\n            layers.experimental.preprocessing.RandomContrast(0.05)])\n        model = tf.keras.models.Sequential(name=\"galaxyClassifier\", layers=[\n            data_augmentation,\n            model])\n        \n    return model\n\nwith strategy.scope(): \n    image_shape = (160,160,3)\n    model = create_model(image_shape, True)\n    optimizer = tf.keras.optimizers.Adam(learning_rate=1e-3) # Optimizer\n    loss_func = losses.MeanSquaredError() # Loss function\n    model.compile(loss=loss_func, optimizer=optimizer, metrics=[tf.keras.metrics.RootMeanSquaredError()])\n    \nmodel.summary()","metadata":{"papermill":{"duration":0.449897,"end_time":"2021-08-25T08:13:19.983119","exception":false,"start_time":"2021-08-25T08:13:19.533222","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-09-03T09:01:33.799326Z","iopub.execute_input":"2021-09-03T09:01:33.800053Z","iopub.status.idle":"2021-09-03T09:01:35.447883Z","shell.execute_reply.started":"2021-09-03T09:01:33.800001Z","shell.execute_reply":"2021-09-03T09:01:35.446719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training\n# ---------------------\nnum_epochs = 50\nverbose = True\n# Callbacks\nreduce_lr = callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=7, min_lr=1e-5)\ndef decay_schedule(epoch, lr):\n    return lr * 0.8 if (epoch % 10 == 0) and (epoch != 0) else lr\nlr_scheduler = callbacks.LearningRateScheduler(decay_schedule)\nearly_stop = callbacks.EarlyStopping(monitor='val_loss',mode='min',verbose=1,patience=7)\ncheckpoint = callbacks.ModelCheckpoint('best_model', save_best_only=True, monitor='val_accuracy', mode='max')\ncallbacksInUse = []\n\nprint('------- Training -------')\nstart = time.time()\nhistory = model.fit(train_dataset, validation_data=val_dataset, epochs=num_epochs, callbacks=callbacksInUse, use_multiprocessing=True, verbose=verbose)\nend = time.time()\nprint(\"Total training took {:.2f} hours.\".format((end - start)/3600))\n\n# Plot learning curves\nmetrics = history.history\nfig, axes = plt.subplots(1, 2, figsize=(12,6))\naxes[0].plot(metrics['root_mean_squared_error'], label='train_accuracy')\naxes[0].plot(metrics['val_root_mean_squared_error'], label='val_accuracy')\naxes[0].set_xlabel('Epoch')\naxes[0].set_ylabel('RMSE')\naxes[0].legend(loc='upper right')\naxes[1].plot(metrics['loss'], label='train_loss')\naxes[1].plot(metrics['val_loss'], label='val_loss')\naxes[1].set_xlabel('Epoch')\naxes[1].set_ylabel('Loss')\naxes[1].legend(loc='upper right')\nplt.show()","metadata":{"papermill":{"duration":395.850426,"end_time":"2021-08-25T08:19:55.874824","exception":false,"start_time":"2021-08-25T08:13:20.024398","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-09-03T09:01:43.483005Z","iopub.execute_input":"2021-09-03T09:01:43.483448Z","iopub.status.idle":"2021-09-03T09:01:44.68386Z","shell.execute_reply.started":"2021-09-03T09:01:43.483401Z","shell.execute_reply":"2021-09-03T09:01:44.680146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Evaluation\n# -------------------\nresults = model.evaluate(val_dataset, verbose=1)\nprint(\"Test RMSE: {:.3f}\".format(results[1]))","metadata":{"papermill":{"duration":2.634409,"end_time":"2021-08-25T08:20:00.056127","exception":false,"start_time":"2021-08-25T08:19:57.421718","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-08-29T16:43:23.378379Z","iopub.execute_input":"2021-08-29T16:43:23.378729Z","iopub.status.idle":"2021-08-29T16:43:23.405612Z","shell.execute_reply.started":"2021-08-29T16:43:23.378697Z","shell.execute_reply":"2021-08-29T16:43:23.403429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save model\n# -------------------\noutPath = model.name+\".h5\"\nmodel.save(outPath)\nprint(\"Model is saved: {}\".format(outPath))\n\n#model.load_weights('../input/models/galaxyClassifier1.h5')\n#print(\"Model is loaded succesfully!\")","metadata":{"papermill":{"duration":1.581862,"end_time":"2021-08-25T08:20:07.413816","exception":false,"start_time":"2021-08-25T08:20:05.831954","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-09-01T16:32:16.562437Z","iopub.execute_input":"2021-09-01T16:32:16.562963Z","iopub.status.idle":"2021-09-01T16:32:16.979975Z","shell.execute_reply.started":"2021-09-01T16:32:16.562923Z","shell.execute_reply":"2021-09-01T16:32:16.978994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prediction\n# -------------------\ntest_images_path = tf.io.gfile.glob('../temp/images_test_rev1/*')\ntest_dataset = dataset_preprocessing(test_images_path,False)\nprint('Number of test batches: %d' % tf.data.experimental.cardinality(test_dataset))\nprint('------- Prediction -------')\nresult = model.predict(test_dataset, verbose=1)\nprint(\"result shape: {}\".format(result.shape))\n","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:32:16.981295Z","iopub.execute_input":"2021-09-01T16:32:16.981831Z","iopub.status.idle":"2021-09-01T16:35:58.122062Z","shell.execute_reply.started":"2021-09-01T16:32:16.981787Z","shell.execute_reply":"2021-09-01T16:35:58.121106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Kaggle submission\n# -------------------   \n# Option 1: \npredictions = np.vstack(np.array(result))\ngalaxyIDs = np.array([v.split('/')[3].split('.')[0] for v in test_images_path]).reshape(len(test_images_path),1)\nsubmission = pd.DataFrame(np.hstack((galaxyIDs, predictions)), columns=labels_pd.columns)\nsubmission = submission.sort_values(by=['GalaxyID'])\n\n# Option 2: not good why?\n#submission = pd.read_csv('../input/galaxy-zoo-the-galaxy-challenge/all_ones_benchmark.zip')\n#submission.iloc[:, 1:] = result\n\nprint(\"submission data shape: {}\".format(submission.shape))\nsubmission.to_csv('submission.csv', index=False)\nprint(\"Submission file is saved succesfully!\")","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:39:53.506465Z","iopub.execute_input":"2021-09-01T16:39:53.506875Z","iopub.status.idle":"2021-09-01T16:40:01.006412Z","shell.execute_reply.started":"2021-09-01T16:39:53.50684Z","shell.execute_reply":"2021-09-01T16:40:01.005432Z"},"trusted":true},"execution_count":null,"outputs":[]}]}