{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Classificação morfológica de galáxias"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install Augmentor","execution_count":null,"outputs":[]},{"metadata":{"pycharm":{"is_executing":false,"name":"#%% Imports\n"},"trusted":true},"cell_type":"code","source":"from string import Template\nfrom zipfile import ZipFile\nfrom os import path, mkdir\nimport pandas as pd\nfrom shutil import copy, rmtree\nimport matplotlib.pyplot as plt\nimport numpy as np\nfrom skimage import io\nimport Augmentor\nfrom tqdm.autonotebook import tqdm\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{"pycharm":{"name":"#%% md\n"}},"cell_type":"markdown","source":"### Unzip do dataset"},{"metadata":{"pycharm":{"name":"#%% md\n"}},"cell_type":"markdown","source":" \n### Carregar dados das galáxias e ajustar DataFrames para filtrar\n\n#### Separar galáxias por classe\n\n Para a classificação das galáxias, o dataset fornecido pelo galaxy challenge vem com 37 classes.\n \n Para reduzir a quantidade classes, filtramos as classes que desejamos e e copiamos cada classe par\n sua devida pasta. Usaremos imagens apenas com indices de respostas maiores de 90%.\n- completely-rounded: Class7.1\n- in-between: 7.2\n- cigar-shaped: Class7.3\n- on-edge: Class2.1\n- spiral-barred: Class3.1 && Class4.1\n- spiral: Class3.2 && Class4.1"},{"metadata":{"pycharm":{"is_executing":false,"name":"#%% Create Base DataFrame for filtering images for each class\n"},"trusted":true},"cell_type":"code","source":"original_training_data = pd.read_csv('/kaggle/input/galaxy-zoo-the-galaxy-challenge/44352/training_solutions_rev1.csv')\n\n# Pandas coloca o galaxyID como float, convertemos novamente para string.\noriginal_training_data['GalaxyID'] = original_training_data['GalaxyID'].astype(str)\n\ncolumns_mapper = {\n    'GalaxyID': 'GalaxyID',\n    'Class7.1': 'completely_round',\n    'Class7.2': 'in_between',\n    'Class7.3': 'cigar_shaped',\n    'Class2.1': 'on_edge',\n    'Class4.1': 'has_signs_of_spiral',\n    'Class3.1': 'spiral_barred',\n    'Class3.2': 'spiral'\n}\ncolumns = list(columns_mapper.values())\ntraining_df = original_training_data.rename(columns=columns_mapper)[columns]\ntraining_df.set_index('GalaxyID', inplace=True)\ntraining_df.head(10)","execution_count":null,"outputs":[]},{"metadata":{"pycharm":{"name":"#%% md\n"}},"cell_type":"markdown","source":"### Criar DataFrames para cada classe"},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_distribution(df, column):\n    plt.plot(list(df[column]))\n    plt.title('Distribution of data')\n    plt.ylabel('% Votes')\n    plt.legend([column], loc='upper left')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# DataFrames of each class\ncompletely_round_df = training_df.sort_values(by = 'completely_round', ascending= False)[0:5000]\ncompletely_round_df['type'] = 'completely_round'\ncompletely_round_df = completely_round_df[['type', 'completely_round']]\nplot_distribution(completely_round_df, 'completely_round')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"in_between_df = training_df.sort_values(by = 'in_between', ascending= False)[0:3600]\nin_between_df['type'] = 'in_between'\n\n# filters\nbigger_than_completely_round = in_between_df['in_between'] > in_between_df['completely_round']\nbigger_than_cigar_shaped = in_between_df['in_between'] > in_between_df['cigar_shaped']\n\nin_between_df = in_between_df[bigger_than_completely_round & bigger_than_cigar_shaped]\nin_between_df = in_between_df[['type', 'in_between']]\nplot_distribution(in_between_df, 'in_between')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cigar_shaped_df = training_df.sort_values(by = 'cigar_shaped', ascending= False)[0:1550]\ncigar_shaped_df['type'] = 'cigar_shaped'\n\n# filters\nbigger_than_in_between = cigar_shaped_df['cigar_shaped'] > cigar_shaped_df['in_between']\nbigger_than_on_edge = cigar_shaped_df['cigar_shaped'] > cigar_shaped_df['on_edge']\n\ncigar_shaped_df = cigar_shaped_df[bigger_than_in_between & bigger_than_on_edge]\ncigar_shaped_df = cigar_shaped_df[['type', 'cigar_shaped']]\nplot_distribution(cigar_shaped_df, 'cigar_shaped')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"on_edge_df = training_df.sort_values(by = 'on_edge', ascending= False)[0:5100]\non_edge_df['type'] = 'on_edge'\non_edge_df = on_edge_df[['type', 'on_edge']]\nplot_distribution(on_edge_df, 'on_edge')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"spiral_barred_df = training_df.sort_values(by = ['spiral_barred', 'has_signs_of_spiral'], ascending= False)[0:3300]\nspiral_barred_df['type'] = 'spiral_barred'\nspiral_barred_df = spiral_barred_df[['type', 'spiral_barred']]\nplot_distribution(spiral_barred_df, 'spiral_barred')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"spiral_df = training_df.sort_values(by = ['spiral', 'has_signs_of_spiral'], ascending= False)[0:5000]\nspiral_df['type'] = 'spiral'\nspiral_df = spiral_df[['type', 'spiral']]\nplot_distribution(spiral_df, 'spiral')","execution_count":null,"outputs":[]},{"metadata":{"pycharm":{"is_executing":false,"name":"#%% Create DataFrames for each class\n"},"trusted":true},"cell_type":"code","source":"dfs = [\n    completely_round_df,\n    in_between_df,\n    cigar_shaped_df,\n    on_edge_df,\n    spiral_barred_df,\n    spiral_df\n]\n\n\n# Merge and drop and possible duplicates\nmerged_dfs = pd.concat(dfs, sort=False)\nmerged_dfs.reset_index(inplace = True)\nmerged_dfs.drop_duplicates(subset='GalaxyID', inplace = True)\n\n\ntrain_merged_df, test_merged_df = train_test_split(merged_dfs, test_size=0.2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_merged_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_merged_df.shape","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":true},"cell_type":"code","source":"def plot_info_set(df, name):\n    countings = df.groupby('type').count().to_dict()['GalaxyID']\n    labels = list(countings.keys())\n    values = list(countings.values())\n\n    fig1, ax1 = plt.subplots()\n    ax1.pie(values, labels=labels, autopct='%1.1f%%', startangle=90)\n    ax1.axis('equal')\n    fig1.suptitle(name)\n    plt.tight_layout()\n    plt.show()\n\n    index = np.arange(len(labels))\n\n    plt.bar(index, values)\n\n    plt.xticks(index, labels, rotation=30)\n    plt.show()\n\nplot_info_set(train_merged_df, 'Train dataset')\nplot_info_set(test_merged_df, 'Test dataset')\n","execution_count":null,"outputs":[]},{"metadata":{"pycharm":{"name":"#%% md\n"}},"cell_type":"markdown","source":"### Copiar arquivos para as devidas pastas"},{"metadata":{"trusted":true},"cell_type":"code","source":"rmtree('/training', ignore_errors=True)\nrmtree('/training_dataset', ignore_errors=True)\nrmtree('/test', ignore_errors=True)\nrmtree('/test_dataset', ignore_errors=True)","execution_count":null,"outputs":[]},{"metadata":{"pycharm":{"is_executing":false,"name":"#%% Copy files based on class\n"},"trusted":true},"cell_type":"code","source":"def copy_files_of_set(df, dest_folder):\n    pbar = tqdm(total=df.shape[0], desc=\"Copying images\", unit=\" Images\")\n    if path.isdir(dest_folder) is False:\n        mkdir(dest_folder)\n\n    src_path = Template('/kaggle/input/galaxy-zoo-the-galaxy-challenge/44352/images_training_rev1/$name.jpg')\n\n    for index, image in df.iterrows():\n        dest_path = Template('/$path/$folder/').substitute(path=dest_folder, folder=image['type'])\n        source_img = src_path.substitute(name=image['GalaxyID'])\n        if path.isdir(dest_path) is False:\n            mkdir(dest_path)\n        copy(source_img, dest_path)\n        pbar.update(1)\n    pbar.close()\ncopy_files_of_set(train_merged_df, '/training')\ncopy_files_of_set(test_merged_df, '/test')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"p = Augmentor.Pipeline(\"/training\", \"../training_dataset\")\np.zoom(probability=1, max_factor=1.4, min_factor=1.4)\np.resize(probability=1, width=70, height=70)\np.process()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"p = Augmentor.Pipeline(\"/test\", \"../test_dataset\")\np.zoom(probability=1, max_factor=1.4, min_factor=1.4)\np.resize(probability=1, width=70, height=70)\np.process()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"p = Augmentor.Pipeline('/training/', '../training_dataset/')\n# p = Augmentor.Pipeline('../data/training/cigar_shaped', '../../training_augmented/cigar_shaped')\np.zoom(probability=1, max_factor=1.4, min_factor=1.4)\np.rotate_random_90(probability=0.2)\np.flip_top_bottom(probability=0.5)\np.flip_left_right(probability=0.5)\np.random_contrast(probability=0.5, min_factor=0.7, max_factor=1.5)\np.random_brightness(probability=0.5, min_factor=0.7, max_factor=1.8)\np.resize(probability=1, width=70, height=70)\np.sample(10000)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\nfrom keras.models import Sequential\nfrom keras.layers import Conv2D, MaxPooling2D\nfrom keras.layers import Activation, Dropout, Flatten, Dense\nfrom keras.optimizers import rmsprop, Adam\nfrom keras import regularizers\nfrom keras.callbacks import ModelCheckpoint","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_size = (70,70)\ntrain_dir = '/training_dataset'\ntest_dir = '/test_dataset'\nbatch_size = 32\n\ndatagen = ImageDataGenerator()\n\ntrain_generator = datagen.flow_from_directory(\n    train_dir,\n    class_mode='categorical',\n    target_size=image_size,\n    batch_size=batch_size,\n)\n\nvalidation_generator = datagen.flow_from_directory(\n    test_dir,\n    class_mode='categorical',\n    target_size=image_size,\n    batch_size=batch_size,\n)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = Sequential()\n\nmodel.add(Conv2D(32,(3, 3), input_shape=(image_size[0], image_size[0], 3)))\nmodel.add(Activation('relu'))\n\nmodel.add(Conv2D(32, (3, 3)))\nmodel.add(Activation('relu'))\nmodel.add(MaxPooling2D(pool_size=(2,2)))\n\nmodel.add(Conv2D(32, (3, 3)))\nmodel.add(Activation('relu'))\n\nmodel.add(Conv2D(32, (3, 3)))\nmodel.add(Activation('relu'))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\n\n\nmodel.add(Conv2D(64, (3, 3)))\nmodel.add(Activation('relu'))\n\nmodel.add(Conv2D(64, (3, 3), kernel_regularizer=regularizers.l2(0.015)))\nmodel.add(Activation('relu'))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\nmodel.add(Dropout(0.5))\n\nmodel.add(Flatten())\nmodel.add(Dense(64, kernel_regularizer=regularizers.l2(0.015)))\nmodel.add(Activation('relu'))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(6))\nmodel.add(Activation('sigmoid'))\n\nmodel.compile(Adam(lr=0.0001), loss=\"categorical_crossentropy\", metrics=[\"accuracy\"])\n\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if path.isdir('/weights') is False:\n    mkdir('/weights')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trains_steps = train_generator.n // train_generator.batch_size\nvalidation_steps = validation_generator.n // validation_generator.batch_size\nmodel_checkpoint = ModelCheckpoint('/weights/weights{epoch:08d}.h5', save_weights_only=True, period=5)\n\nfit_result = model.fit_generator(\n    train_generator,\n    steps_per_epoch=trains_steps,\n    validation_data =validation_generator,\n    validation_steps=validation_steps,\n    epochs=71,\n    callbacks=[model_checkpoint]\n)\n\nmodel.save_weights('/weights/final_epoch.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Accuracy\nplt.plot(list(range(25,80)), fit_result.history['acc'][25:80])\nplt.plot(list(range(25,80)), fit_result.history['val_acc'][25:80])\nplt.title('Model accuracy')\nplt.ylabel('Accuracy')\nplt.xlabel('Epoch')\nplt.legend(['Train', 'Test'], loc='upper left')\nplt.show()\n\n# Loss\nplt.plot(list(range(25,80)), fit_result.history['loss'][25:80])\nplt.plot(list(range(25,80)), fit_result.history['val_loss'][25:80])\nplt.title('Model loss')\nplt.ylabel('Loss')\nplt.xlabel('Epoch')\nplt.legend(['Train', 'Test'], loc='upper left')\nplt.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.3"},"pycharm":{"stem_cell":{"cell_type":"raw","metadata":{"collapsed":false},"source":[]}}},"nbformat":4,"nbformat_minor":1}