{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# GitHub:\n\n### https://github.com/clbcabral/PonyGE2/blob/master/src/fitness/paper_mnist.py","metadata":{}},{"cell_type":"code","source":"#from fitness.base_ff_classes.base_ff import base_ff\nfrom tensorflow.keras import datasets, layers, models, callbacks, optimizers\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras import backend as K\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelBinarizer\nimport numpy as np\nimport pandas as pd\nimport re, csv\nimport tensorflow as tf","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def transform_data(train, test):\n    \n    y_train = np.array(train.label)\n    train.drop(['label'], axis = 1, inplace = True) # Removing labels from dataframe\n\n    # Converting images into 28x28 matrix (original proportions)\n    x_train = np.array(train).reshape(train.shape[0], 28, 28)\n    x_test = np.array(test).reshape(test.shape[0], 28, 28)\n    x_train = np.array(x_train).reshape((-1, 28, 28, 1))\n    x_test = np.array(x_test).reshape((-1, 28, 28, 1))\n    \n    # Normalization\n    x_train = x_train.astype(\"float\") / 255.0\n    x_test = x_test.astype(\"float\") / 255.0\n    \n    # Transform labels into a binary vector (10 classes)\n    y_train = to_categorical(y_train)\n    lb = LabelBinarizer()\n    y_train = lb.fit_transform(y_train)\n    \n    return x_train, y_train, x_test","metadata":{"execution":{"iopub.status.busy":"2022-08-17T14:01:50.133276Z","iopub.execute_input":"2022-08-17T14:01:50.133969Z","iopub.status.idle":"2022-08-17T14:01:50.141096Z","shell.execute_reply.started":"2022-08-17T14:01:50.133931Z","shell.execute_reply":"2022-08-17T14:01:50.140177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# F1 Score metric function\ndef f1_score(y_true, y_pred):\n    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n    precision = true_positives / (predicted_positives + K.epsilon())\n    recall = true_positives / (possible_positives + K.epsilon())\n    f1_val = 2 * (precision * recall) / (precision + recall + K.epsilon())\n    return f1_val\n\ndef build_model(phenotype):\n\n    # Detect and init the TPU\n    #tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect()\n    # Instantiate a distribution strategy\n    #tpu_strategy = tf.distribute.experimental.TPUStrategy(tpu)\n\n    #with tpu_strategy.scope():\n\n        model = models.Sequential()\n        \n        '''# Data Augmentation\n        data_augmentation_layer = tf.keras.Sequential([\n          tf.keras.layers.RandomRotation(0.15, input_shape=(28, 28, 1)),\n          tf.keras.layers.RandomZoom(0.18),                                \n        ], name='data_augmentation')\n        model.add(data_augmentation_layer)\n        '''\n\n        filter_size = 28\n        nconv, npool, nfc, nfcneuron = [int(i) for i in re.findall('\\d+', phenotype.split('lr-')[0])]\n        has_dropout = 'dropout' in phenotype\n        has_batch_normalization = 'bnorm' in phenotype\n        has_pool = 'pool' in phenotype\n        learning_rate = float(phenotype.split('lr-')[1])\n\n        try:\n            \n            # Pooling\n            for i in range(npool):\n            \n                # Convolutions\n                for j in range(nconv):\n            \n                    model.add(layers.Conv2D(filter_size, (3, 3), activation='relu', padding='same', input_shape=(28, 28, 1)))\n\n                    # Duplicate number of filters for each two convolutions\n                    if (((i + j) % 2) == 1): filter_size = filter_size * 2\n\n                    # Add batch normalization\n                    if has_batch_normalization:\n                        model.add(layers.BatchNormalization())\n\n                # Add pooling\n                if has_pool:\n                    model.add(layers.MaxPooling2D(pool_size=(2, 2)))\n                    # Add dropout\n                    if has_dropout:\n                        model.add(layers.Dropout(0.25))\n\n            model.add(layers.Flatten())\n\n            # fully connected\n            for i in range(nfc):\n                model.add(layers.Dense(nfcneuron))\n                model.add(layers.Activation('relu'))\n\n            if has_dropout:\n                model.add(layers.Dropout(0.5))\n\n            model.add(layers.Dense(10, activation='softmax'))\n            \n            model.summary()\n\n        except Exception as ex:\n            # Some NN topologies are invalid\n            print(ex)\n            return None\n\n        opt = optimizers.Adam(lr=learning_rate)\n        model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy', f1_score])\n        return model","metadata":{"execution":{"iopub.status.busy":"2022-08-17T14:01:51.879635Z","iopub.execute_input":"2022-08-17T14:01:51.880135Z","iopub.status.idle":"2022-08-17T14:01:51.895618Z","shell.execute_reply.started":"2022-08-17T14:01:51.880080Z","shell.execute_reply":"2022-08-17T14:01:51.894743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nBuild model sem o fenótipo\n(Apenas para comparar. Não precisa rodar)\n'''\ndef build_model():\n\n    # Arquitetura: (((conv*3)bnorm-pool-)*2)fc*2*128*lr-0.001\n    nfcneuron = 128\n    filter_size = 28\n    learning_rate = 0.001\n\n    model = models.Sequential()\n            \n    model.add(layers.Conv2D(filter_size, (3, 3), activation='relu', padding='same', input_shape=(28, 28, 1)))\n    model.add(layers.BatchNormalization())\n    model.add(layers.Conv2D(filter_size, (3, 3), activation='relu', padding='same'))\n    model.add(layers.BatchNormalization())\n    model.add(layers.Conv2D(filter_size*2, (3, 3), activation='relu', padding='same'))\n    model.add(layers.BatchNormalization())\n    model.add(layers.MaxPooling2D(pool_size=(2, 2)))\n    #model.add(layers.Dropout(0.25))\n\n    model.add(layers.Conv2D(filter_size*2, (3, 3), activation='relu', padding='same'))\n    model.add(layers.BatchNormalization())\n    model.add(layers.Conv2D(filter_size*4, (3, 3), activation='relu', padding='same'))\n    model.add(layers.BatchNormalization())\n    model.add(layers.Conv2D(filter_size*4, (3, 3), activation='relu', padding='same'))\n    model.add(layers.BatchNormalization())\n    model.add(layers.MaxPooling2D(pool_size=(2, 2)))\n    #model.add(layers.Dropout(0.25))\n\n    model.add(layers.Flatten())\n\n    model.add(layers.Dense(nfcneuron))\n    model.add(layers.Activation('relu'))\n    #model.add(layers.Dropout(0.5))\n    \n    model.add(layers.Dense(nfcneuron))\n    model.add(layers.Activation('relu'))\n\n    model.add(layers.Dense(10, activation='softmax'))\n\n    opt = optimizers.Adam(learning_rate=learning_rate)\n\n    model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy', f1_score])\n        \n    return model\n\nm1 = build_model('(((conv*3)bnorm-pool-)*2)fc*2*128*lr-0.001')\nm2 = build_model2()\nm1.summary() == m2.summary()","metadata":{"execution":{"iopub.status.busy":"2022-08-17T13:31:59.329867Z","iopub.execute_input":"2022-08-17T13:31:59.330336Z","iopub.status.idle":"2022-08-17T13:31:59.341472Z","shell.execute_reply.started":"2022-08-17T13:31:59.330297Z","shell.execute_reply":"2022-08-17T13:31:59.340781Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Arquitetura da CNN \"(((conv\\*3)bnorm-pool-)\\*2)fc\\*2\\*128\\*lr-0.001\":\n\n### https://drive.google.com/file/d/18GKkwXMY7N3o3ie303iZYKLQnXpzVl5J/view?usp=sharing","metadata":{}},{"cell_type":"code","source":"def train_model(model, x_train, y_train):\n\n    batch_size = 128\n    \n    train_ds = tf.data.Dataset.from_tensor_slices((x_train, y_train)).batch(batch_size, drop_remainder=True)\n    \n    # sem validação. monitorando a acurácia\n    es = callbacks.EarlyStopping(monitor='accuracy', mode='max', verbose=1, patience=10, baseline=0.5)\n    \n    model.fit(train_ds, epochs=70, batch_size=batch_size, callbacks=[es])\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2022-08-17T14:01:54.848029Z","iopub.execute_input":"2022-08-17T14:01:54.848775Z","iopub.status.idle":"2022-08-17T14:01:54.855947Z","shell.execute_reply.started":"2022-08-17T14:01:54.848741Z","shell.execute_reply":"2022-08-17T14:01:54.855108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Run:","metadata":{}},{"cell_type":"code","source":"# Carrega o dataset\ntrain = pd.read_csv('../input/digit-recognizer/train.csv')\ntest = pd.read_csv('../input/digit-recognizer/test.csv')\n\n# Subsets\nx_train, y_train, x_test = transform_data(train, test)","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-08-17T14:01:56.679947Z","iopub.execute_input":"2022-08-17T14:01:56.680532Z","iopub.status.idle":"2022-08-17T14:02:01.060657Z","shell.execute_reply.started":"2022-08-17T14:01:56.680493Z","shell.execute_reply":"2022-08-17T14:02:01.059789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Monta o modelo\nPHENOTYPE = '(((conv*3)bnorm-pool-)*2)fc*2*128*lr-0.001' # 0.99400 test acc\nPHENOTYPE_2 = '(((conv*3)bnorm-pool-dropout)*2)fc*2*128*lr-0.001' # 0.99196 test acc\n\nmodel = build_model(PHENOTYPE)","metadata":{"execution":{"iopub.status.busy":"2022-08-17T14:02:01.062499Z","iopub.execute_input":"2022-08-17T14:02:01.062792Z","iopub.status.idle":"2022-08-17T14:02:01.223905Z","shell.execute_reply.started":"2022-08-17T14:02:01.062757Z","shell.execute_reply":"2022-08-17T14:02:01.223192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Treinar modelo\nmodel = train_model(model, x_train, y_train)\n\n# Predizer subset de teste\nyhat = model.predict(x_test)\nyhat = [np.argmax(y) for y in yhat]","metadata":{"execution":{"iopub.status.busy":"2022-08-17T14:02:03.780641Z","iopub.execute_input":"2022-08-17T14:02:03.780922Z","iopub.status.idle":"2022-08-17T14:06:29.692859Z","shell.execute_reply.started":"2022-08-17T14:02:03.780892Z","shell.execute_reply":"2022-08-17T14:06:29.691952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nindex = 2\nplt.figure(figsize=(3,3))\nplt.imshow(x_test[index], cmap = 'Greys')\nplt.title(yhat[index])","metadata":{"execution":{"iopub.status.busy":"2022-08-17T14:17:59.798769Z","iopub.execute_input":"2022-08-17T14:17:59.799034Z","iopub.status.idle":"2022-08-17T14:17:59.991413Z","shell.execute_reply.started":"2022-08-17T14:17:59.799006Z","shell.execute_reply":"2022-08-17T14:17:59.990682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(8, 8, figsize = (15, 15))\naxes = axes.ravel()\n\nfor i in range(64):\n    axes[i].imshow(x_test[i], cmap = 'gray')\n    axes[i].set_title(yhat[i])\n    axes[i].axis('off')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-17T14:18:05.240614Z","iopub.execute_input":"2022-08-17T14:18:05.240886Z","iopub.status.idle":"2022-08-17T14:18:07.822134Z","shell.execute_reply.started":"2022-08-17T14:18:05.240857Z","shell.execute_reply":"2022-08-17T14:18:07.821002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame()\nids = []\n\nfor i, y in enumerate(yhat):\n    #print(i+1, y)\n    ids.append(i+1)\n    \nsubmission['ImageId'] = ids\nsubmission['Label'] = yhat","metadata":{"execution":{"iopub.status.busy":"2022-08-17T14:07:07.700437Z","iopub.execute_input":"2022-08-17T14:07:07.701172Z","iopub.status.idle":"2022-08-17T14:07:07.737133Z","shell.execute_reply.started":"2022-08-17T14:07:07.701099Z","shell.execute_reply":"2022-08-17T14:07:07.736417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)\nsubmission","metadata":{"execution":{"iopub.status.busy":"2022-08-17T14:07:11.119709Z","iopub.execute_input":"2022-08-17T14:07:11.120289Z","iopub.status.idle":"2022-08-17T14:07:11.161259Z","shell.execute_reply.started":"2022-08-17T14:07:11.120251Z","shell.execute_reply":"2022-08-17T14:07:11.160388Z"},"trusted":true},"execution_count":null,"outputs":[]}]}