{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install Augmentor","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from string import Template\nfrom zipfile import ZipFile\nfrom os import path, mkdir, makedirs\nimport pandas as pd\nfrom shutil import copy\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport Augmentor\nfrom tqdm import tqdm\nfrom sklearn.model_selection import train_test_split\nimport seaborn as sns","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Filter"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.style.use('seaborn')\n\nCOMPETITION_NAME = \"galaxy-zoo-the-galaxy-challenge\"\nDATA_PATH = \"/kaggle/input/galaxy-zoo-the-galaxy-challenge/44352/\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#  Para a classificação das galáxias, o dataset fornecido pelo galaxy challenge vem com 37 classes.\n#\n#  Para reduzir a quantidade classes, filtramos as classes que desejamos e e copiamos cada classe par\n#  sua devida pasta. Usaremos imagens apenas com indices de respostas maiores de 90%.\n# - completely-rounded: Class7.1\n# - in-between: 7.2\n# - cigar-shaped: Class7.3\n# - on-edge: Class2.1\n# - spiral-barred: Class3.1 && Class4.1\n# - spiral: Class3.2 && Class4.1\n\n#%% Loading csv and adjusting the dataframe\noriginal_training_data = pd.read_csv(DATA_PATH + \"training_solutions_rev1.csv\")\n\n# Pandas read GalaxyID has float, converts it back to string.\noriginal_training_data[\"GalaxyID\"] = original_training_data[\"GalaxyID\"].astype(str)\n\n# Better column naming\ncolumns_mapper = {\n    \"GalaxyID\": \"GalaxyID\",\n    \"Class7.1\": \"completely_round\",\n    \"Class7.2\": \"in_between\",\n    \"Class7.3\": \"cigar_shaped\",\n    \"Class2.1\": \"on_edge\",\n    \"Class4.1\": \"has_signs_of_spiral\",\n    \"Class3.1\": \"spiral_barred\",\n    \"Class3.2\": \"spiral\",\n}\n\ncolumns = list(columns_mapper.values())\ngalaxies_df = original_training_data.rename(columns=columns_mapper)[columns]\ngalaxies_df.set_index(\"GalaxyID\", inplace=True)\ngalaxies_df.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ### Criar DataFrames para cada classe\n#%% Simple function to plot each class data\n\ndef plot_distribution(df, column):\n    print(\"Items: \" + str(df.shape[0]))\n    sns.distplot(df[column])\n    plt.xlabel(\"% Votes\")\n    plt.title('Distribution - ' + column)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"completely_round_df = galaxies_df.sort_values(by=\"completely_round\", ascending=False)[0:7000]\ncompletely_round_df[\"type\"] = \"completely_round\"\ncompletely_round_df = completely_round_df[[\"type\", \"completely_round\"]]\n\nplot_distribution(completely_round_df, \"completely_round\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"in_between_df = galaxies_df.sort_values(by=\"in_between\", ascending=False)[0:6000]\nin_between_df[\"type\"] = \"in_between\"\n\n# filters\nbigger_than_completely_round = (\n    in_between_df[\"in_between\"] > in_between_df[\"completely_round\"]\n)\nbigger_than_cigar_shaped = in_between_df[\"in_between\"] > in_between_df[\"cigar_shaped\"]\n\nin_between_df = in_between_df[bigger_than_completely_round & bigger_than_cigar_shaped]\nin_between_df = in_between_df[[\"type\", \"in_between\"]]\nplot_distribution(in_between_df, \"in_between\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cigar_shaped_df = galaxies_df.sort_values(by=\"cigar_shaped\", ascending=False)[0:1550]\ncigar_shaped_df[\"type\"] = \"cigar_shaped\"\n\n# filters\nbigger_than_in_between = cigar_shaped_df[\"cigar_shaped\"] > cigar_shaped_df[\"in_between\"]\nbigger_than_on_edge = cigar_shaped_df[\"cigar_shaped\"] > cigar_shaped_df[\"on_edge\"]\n\ncigar_shaped_df = cigar_shaped_df[bigger_than_in_between & bigger_than_on_edge]\ncigar_shaped_df = cigar_shaped_df[[\"type\", \"cigar_shaped\"]]\n\nplot_distribution(cigar_shaped_df, \"cigar_shaped\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"on_edge_df = galaxies_df.sort_values(by=\"on_edge\", ascending=False)[0:5000]\non_edge_df[\"type\"] = \"on_edge\"\non_edge_df = on_edge_df[[\"type\", \"on_edge\"]]\nplot_distribution(on_edge_df, \"on_edge\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"spiral_barred_df = galaxies_df.sort_values(\n    by=[\"spiral_barred\", \"has_signs_of_spiral\"], ascending=False\n)[0:4500]\n\nspiral_barred_filter = spiral_barred_df['spiral'] < spiral_barred_df['spiral_barred']\nspiral_barred_df = spiral_barred_df[spiral_barred_filter]\nspiral_barred_df[\"type\"] = \"spiral_barred\"\nspiral_barred_df = spiral_barred_df[[\"type\", \"spiral_barred\"]]\nplot_distribution(spiral_barred_df, \"spiral_barred\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"spiral_df = galaxies_df.sort_values(\n    by=[\"spiral\", \"has_signs_of_spiral\"], ascending=False\n)[0:8000]\nspiral_df[\"type\"] = \"spiral\"\nspiral_df = spiral_df[[\"type\", \"spiral\"]]\nplot_distribution(spiral_df, \"spiral\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#%% Generate a single dataframe with all galaxies from each class\ndfs = [\n    completely_round_df,\n    in_between_df,\n    cigar_shaped_df,\n    on_edge_df,\n    spiral_barred_df,\n    spiral_df,\n]\n\n\n# Merge and drop and possible duplicates\nmerged_dfs = pd.concat(dfs, sort=False)\nmerged_dfs.reset_index(inplace=True)\nmerged_dfs.drop_duplicates(subset=\"GalaxyID\", inplace=True)\nmerged_dfs.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Split the datafrane between train and test\ntrain_df, validation_df = train_test_split(merged_dfs, test_size=0.2)\n#%% plot distribuition\ndef plot_info_set(df, name):\n    countings = df.groupby(\"type\").count().to_dict()[\"GalaxyID\"]\n    labels = list(countings.keys())\n    values = list(countings.values())\n    index = np.arange(len(labels))\n    plt.bar(index, values)\n    plt.title(name)\n    plt.xticks(index, labels, rotation=30)\n    plt.show()\n\n\nplot_info_set(train_df, \"Train dataset\")\nplot_info_set(validation_df, \"Test dataset\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Augment "},{"metadata":{"trusted":true},"cell_type":"code","source":"ZOOM_FACTOR=1.6\nDIMEN=70\nFILTERED_DATA_PATH = \"/data/filtered/\"\nDATASETS_PATH = \"/data/sets/\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def copy_files_of_set(df, dataset):\n    print(\"Copying filtered files of \" + dataset)\n    if path.isdir(FILTERED_DATA_PATH + dataset) is False:\n        makedirs(FILTERED_DATA_PATH + dataset, exist_ok=True)\n\n    src_path = Template(DATA_PATH + \"images_training_rev1/$name.jpg\")\n\n    for index, image in df.iterrows():\n        dest_path = FILTERED_DATA_PATH + dataset + '/' + image['type']\n        source_img = src_path.substitute(name=image[\"GalaxyID\"])\n\n        if path.isdir(dest_path) is False:\n            mkdir(dest_path)\n\n        copy(source_img, dest_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"copy_files_of_set(train_df, \"training\")\ncopy_files_of_set(validation_df, \"validation\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def resize_and_zoom(dataset):\n    p = Augmentor.Pipeline(FILTERED_DATA_PATH + dataset, DATASETS_PATH + dataset)\n    p.zoom(probability=1, max_factor=ZOOM_FACTOR, min_factor=ZOOM_FACTOR)\n    p.resize(probability=1, width=DIMEN, height=DIMEN)\n    p.process()\n    \ndef augment_set(n, dataset = \"\"):\n    p = Augmentor.Pipeline(FILTERED_DATA_PATH + \"training/\" + dataset, DATASETS_PATH + \"training/\" + dataset)\n    p.zoom(probability=1, max_factor=ZOOM_FACTOR, min_factor=ZOOM_FACTOR)\n    p.rotate_random_90(probability=0.2)\n    p.flip_top_bottom(probability=0.5)\n    p.flip_left_right(probability=0.5)\n    p.random_contrast(probability=0.5, min_factor=0.7, max_factor=1.5)\n    p.random_brightness(probability=0.5, min_factor=0.7, max_factor=1.8)\n    p.resize(probability=1, width=DIMEN, height=DIMEN)\n    p.sample(n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"resize_and_zoom(\"training\")\nresize_and_zoom(\"validation\")\n# augment_set(2500, \"cigar_shaped\")\n# augment_set(1500, \"spiral_barred\")\naugment_set(n = 15000)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\nfrom keras.models import Sequential, Model\nfrom keras.layers import Conv2D, MaxPooling2D\nfrom keras.layers import Activation, Dropout, Flatten, Dense, Input\nfrom keras.optimizers import rmsprop, Adam\nfrom keras import regularizers\nfrom keras.callbacks import ModelCheckpoint\nimport matplotlib.pyplot as plt\nfrom os import path, mkdir","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMAGE_SIZE = (DIMEN, DIMEN)\nINPUT_SHAPE = (DIMEN, DIMEN, 3)\n\nBATCH_SIZE = 32\nTRAIN_DIR = DATASETS_PATH + \"training\"\nVALIDATION_DIR = DATASETS_PATH + \"validation\"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"model = Sequential()\n\nmodel.add(Conv2D(32, (3, 3), input_shape=INPUT_SHAPE))\nmodel.add(Activation(\"relu\"))\n\nmodel.add(Conv2D(32, (3, 3)))\nmodel.add(Activation(\"relu\"))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel.add(Conv2D(32, (3, 3)))\nmodel.add(Activation(\"relu\"))\n\nmodel.add(Conv2D(32, (3, 3)))\nmodel.add(Activation(\"relu\"))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\n\n\nmodel.add(Conv2D(64, (3, 3)))\nmodel.add(Activation(\"relu\"))\n\n# model.add(Conv2D(64, (3, 3)))\nmodel.add(Conv2D(64, (3, 3), kernel_regularizer=regularizers.l2(0.015)))\nmodel.add(Activation(\"relu\"))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\nmodel.add(Dropout(0.5))\n\nmodel.add(Flatten())\n# model.add(Dense(64))\nmodel.add(Dense(64, kernel_regularizer=regularizers.l2(0.015)))\nmodel.add(Activation(\"relu\"))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(6))\nmodel.add(Activation(\"softmax\"))\n\nmodel.compile(Adam(), loss=\"sparse_categorical_crossentropy\", metrics=[\"accuracy\"])\n\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"datagen = ImageDataGenerator()\n\ntrain_generator = datagen.flow_from_directory(\n    TRAIN_DIR, class_mode=\"sparse\", target_size=IMAGE_SIZE, batch_size=BATCH_SIZE\n)\n\nvalidation_generator = datagen.flow_from_directory(\n    VALIDATION_DIR, class_mode=\"sparse\", target_size=IMAGE_SIZE, batch_size=BATCH_SIZE\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if path.isdir(\"/weights\") is False:\n    mkdir(\"/weights\")\n\ntrains_steps = train_generator.n // train_generator.batch_size\nvalidation_steps = validation_generator.n // validation_generator.batch_size\n\nmodel_checkpoint = ModelCheckpoint(\n    \"/weights/weights{epoch:08d}.h5\", save_weights_only=True, period=5\n)\n\nfit_result = model.fit_generator(\n    train_generator,\n    steps_per_epoch=trains_steps,\n    validation_data=validation_generator,\n    validation_steps=validation_steps,\n    epochs=80,\n    callbacks=[model_checkpoint],\n)\n\nmodel.save_weights(\"/weights/final_epoch.h5\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#%%\n# Accuracy\n\nplt.plot(fit_result.history[\"acc\"])\nplt.plot(fit_result.history[\"val_acc\"])\nplt.title(\"Model accuracy\")\nplt.ylabel(\"Accuracy\")\nplt.xlabel(\"Epoch\")\nplt.legend([\"Train\", \"Test\"], loc=\"upper left\")\nplt.show()\n\n# Loss\nplt.plot(fit_result.history[\"loss\"])\nplt.plot(fit_result.history[\"val_loss\"])\nplt.title(\"Model loss\")\nplt.ylabel(\"Loss\")\nplt.xlabel(\"Epoch\")\nplt.legend([\"Train\", \"Test\"], loc=\"upper left\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Predict"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import classification_report, confusion_matrix\nimport numpy as np\nimport seaborn as sns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prediction_generator = datagen.flow_from_directory(\n    VALIDATION_DIR, class_mode=\"sparse\", target_size=IMAGE_SIZE, batch_size=1, shuffle=False,\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predict_result = model.predict_generator(prediction_generator, prediction_generator.n)\ny_predicts = np.argmax(predict_result, axis=1)\nclasses_labels = list(prediction_generator.class_indices.keys())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(classification_report(prediction_generator.classes, y_predicts, target_names=classes_labels))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.heatmap(confusion_matrix(prediction_generator.classes, y_predicts), xticklabels=classes_labels, yticklabels=classes_labels, annot=True,fmt='.5g') ","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}