{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport sys\nimport os\nimport cv2","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-12-15T17:56:40.660354Z","iopub.execute_input":"2021-12-15T17:56:40.660962Z","iopub.status.idle":"2021-12-15T17:56:40.878813Z","shell.execute_reply.started":"2021-12-15T17:56:40.660856Z","shell.execute_reply":"2021-12-15T17:56:40.878124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.preprocessing import image\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.callbacks import LearningRateScheduler, ModelCheckpoint, EarlyStopping, ReduceLROnPlateau\nfrom tensorflow.keras.callbacks import Callback\nfrom tensorflow.keras.regularizers import l2\nfrom tensorflow.keras import optimizers\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.applications.xception import Xception\nfrom tensorflow.keras.layers import *","metadata":{"execution":{"iopub.status.busy":"2021-12-15T17:56:41.631554Z","iopub.execute_input":"2021-12-15T17:56:41.632013Z","iopub.status.idle":"2021-12-15T17:56:46.899216Z","shell.execute_reply.started":"2021-12-15T17:56:41.631974Z","shell.execute_reply":"2021-12-15T17:56:46.898416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# В setup выносим основные настройки: так удобнее их перебирать в дальнейшем.\nRANDOM_SEED          = 100\n\nEPOCHS               = 1  # эпох на обучение\nBATCH_SIZE           = 64 # test2 -128\nLR                   = 0.0001\nVAL_SPLIT            = 0.15 # сколько данных выделяем на тест = 15%\n\nCLASS_NUM            = 2  \nIMG_SIZE             = 224 # какого размера подаем изображения в сеть\nIMG_CHANNELS         = 3   # у RGB 3 канала\ninput_shape          = (IMG_SIZE, IMG_SIZE, IMG_CHANNELS)\n\nPATH = \"../input/dog-vs-cat/dvc/\" # рабочая директория","metadata":{"execution":{"iopub.status.busy":"2021-12-15T17:56:46.902954Z","iopub.execute_input":"2021-12-15T17:56:46.90337Z","iopub.status.idle":"2021-12-15T17:56:46.910613Z","shell.execute_reply.started":"2021-12-15T17:56:46.903339Z","shell.execute_reply":"2021-12-15T17:56:46.909227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Аугментация данных\ntrain_datagen = ImageDataGenerator(\n    rescale=1. / 255, #коэффициент масштабирования. По умолчанию - Нет. Если Нет или 0, \n    #масштабирование не применяется, в противном случае мы умножаем данные на предоставленное значение \n    #(после применения всех остальных преобразований).\n    \n    rotation_range = 5, #угол случайных поворотов\n    width_shift_range = 0.1, #диапазон сдвига ширины\n    height_shift_range = 0.1, #диапазон сдвига высоты\n    #brightness_range =  [0.5, 1.5], # сдвиг яркости\n    validation_split = VAL_SPLIT, # Доля выборки для валидации \n    horizontal_flip = True  # отражение по горизонтали\n    )\n\ntest_datagen = ImageDataGenerator(rescale=1. / 255)","metadata":{"execution":{"iopub.status.busy":"2021-12-15T17:56:46.911632Z","iopub.execute_input":"2021-12-15T17:56:46.911861Z","iopub.status.idle":"2021-12-15T17:56:46.925852Z","shell.execute_reply.started":"2021-12-15T17:56:46.911829Z","shell.execute_reply":"2021-12-15T17:56:46.924817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Генерация данных\ntrain_generator = train_datagen.flow_from_directory(\n    PATH,#+'train/',      # директория где расположены папки с картинками \n    target_size=(IMG_SIZE, IMG_SIZE),\n    batch_size=BATCH_SIZE,\n    class_mode='categorical',\n    shuffle=True, seed=RANDOM_SEED,\n    subset='training') # set as training data\n\ntest_generator = train_datagen.flow_from_directory(\n    PATH,#+'train/',    \n    target_size=(IMG_SIZE, IMG_SIZE),\n    batch_size=BATCH_SIZE,\n    class_mode='categorical',\n    shuffle=True, seed=RANDOM_SEED,\n    subset='validation') # set as validation data","metadata":{"execution":{"iopub.status.busy":"2021-12-15T17:56:46.928052Z","iopub.execute_input":"2021-12-15T17:56:46.92845Z","iopub.status.idle":"2021-12-15T17:57:12.987414Z","shell.execute_reply.started":"2021-12-15T17:56:46.928414Z","shell.execute_reply":"2021-12-15T17:57:12.985834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import keras\nimport keras.backend as K\nimport numpy as np\nfrom tensorflow.python.ops import math_ops\n\ndef custom_loss(y_true, y_pred):\n    diff = math_ops.squared_difference(y_pred, y_true)  #squared difference\n    loss = K.mean(diff, axis=-1) #mean over last dimension\n    loss = loss / 10.0\n    return loss","metadata":{"execution":{"iopub.status.busy":"2021-12-15T17:57:17.262793Z","iopub.execute_input":"2021-12-15T17:57:17.263251Z","iopub.status.idle":"2021-12-15T17:57:17.279025Z","shell.execute_reply.started":"2021-12-15T17:57:17.263185Z","shell.execute_reply":"2021-12-15T17:57:17.278001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def softmax(z):\n    \"\"\"Softmax function\"\"\"\n    return K.exp(z) / K.sum(K.exp(z))","metadata":{"execution":{"iopub.status.busy":"2021-12-15T17:57:25.910895Z","iopub.execute_input":"2021-12-15T17:57:25.911515Z","iopub.status.idle":"2021-12-15T17:57:25.915948Z","shell.execute_reply.started":"2021-12-15T17:57:25.91146Z","shell.execute_reply":"2021-12-15T17:57:25.915057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#custom_loss([2.0,2.0],[2.0,3.0])","metadata":{"execution":{"iopub.status.busy":"2021-12-15T17:57:26.829926Z","iopub.execute_input":"2021-12-15T17:57:26.830177Z","iopub.status.idle":"2021-12-15T17:57:26.833882Z","shell.execute_reply.started":"2021-12-15T17:57:26.830149Z","shell.execute_reply":"2021-12-15T17:57:26.833165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import keras.backend as K\nfrom tensorflow.python.ops import math_ops\n\ndef my_c_crossentropy0(y_true, y_pred):\n    r = np.zeros(len(y_true))\n    #r = y_true\n    for j in range(y_pred[0].shape[0]):    # для каждого класса\n        r -= y_true[:,j] * K.log(y_pred[:,j])      \n\n    r = tf.convert_to_tensor(r, dtype=tf.float64)\n\n    return r","metadata":{"execution":{"iopub.status.busy":"2021-12-15T17:57:27.719431Z","iopub.execute_input":"2021-12-15T17:57:27.719717Z","iopub.status.idle":"2021-12-15T17:57:27.725685Z","shell.execute_reply.started":"2021-12-15T17:57:27.719687Z","shell.execute_reply":"2021-12-15T17:57:27.724834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import keras.backend as K\nfrom tensorflow.python.ops import math_ops\n\ndef my_c_crossentropy(y_true, y_pred):\n    r = -K.sum(y_true * K.log(y_pred), axis=1) # сумма по классам\n    return r","metadata":{"execution":{"iopub.status.busy":"2021-12-15T17:57:28.50662Z","iopub.execute_input":"2021-12-15T17:57:28.507033Z","iopub.status.idle":"2021-12-15T17:57:28.512922Z","shell.execute_reply.started":"2021-12-15T17:57:28.506993Z","shell.execute_reply":"2021-12-15T17:57:28.51196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\n\ndef my_softmax_loss(y_true, y_pred):\n    \n    #y_pred = tf.matmul(x, W) + b\n    numerators = tf.reduce_sum(y_true * tf.exp(y_pred), axis=1)\n    denominators = tf.reduce_sum(tf.exp(y_pred), axis=1)\n    loss = - tf.reduce_sum(K.log(numerators / denominators))\n\n    return loss","metadata":{"execution":{"iopub.status.busy":"2021-12-15T17:57:29.508253Z","iopub.execute_input":"2021-12-15T17:57:29.508535Z","iopub.status.idle":"2021-12-15T17:57:29.514322Z","shell.execute_reply.started":"2021-12-15T17:57:29.508504Z","shell.execute_reply":"2021-12-15T17:57:29.513669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def my_circle_loss(y_true, y_pred):\n    \n    #y_pred = tf.matmul(x, W) + b\n    numerators = tf.reduce_sum(y_true * tf.exp(y_pred), axis=1)\n    denominators = tf.reduce_sum(tf.exp(y_pred), axis=1)\n    loss_softmax = - tf.reduce_sum(K.log(numerators / denominators))\n    \n    class_freqs = tf.reduce_sum(y_true, axis=0, keepdims=True)\n    class_freqs = tf.transpose(class_freqs)\n    \n    centres = tf.matmul(tf.transpose(y_true), y_pred)\n    centres = tf.divide(centres, class_freqs)\n    repeated_centres = tf.matmul(y_true, centres)\n    \n    sq_distances = tf.square(tf.norm(y_pred - repeated_centres, axis=1))\n    loss_centre = tf.reduce_sum(sq_distances)\n    \n    loss = loss_softmax + (2/2) * loss_centre \n\n    return loss\n","metadata":{"execution":{"iopub.status.busy":"2021-12-15T18:22:08.122775Z","iopub.execute_input":"2021-12-15T18:22:08.123036Z","iopub.status.idle":"2021-12-15T18:22:08.130161Z","shell.execute_reply.started":"2021-12-15T18:22:08.123005Z","shell.execute_reply":"2021-12-15T18:22:08.129439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_true = np.array([ [0.2, 0.8, 0.1], [0.1, 0.8, 0.1], [1, 0.01, 0.01], [0.05, 0.95, 0.01]])\ny_pred = np.array([ [0.1, 0.8, 0.1], [0.1, 0.8, 0.2], [0.05, 0.95, 0.01], [1, 0.01, 0.01]])\ny_true = tf.convert_to_tensor(y_true, dtype=tf.float64)\ny_pred = tf.convert_to_tensor(y_pred, dtype=tf.float64)\nmy_c_crossentropy(y_true, y_pred)","metadata":{"execution":{"iopub.status.busy":"2021-12-15T18:22:09.007051Z","iopub.execute_input":"2021-12-15T18:22:09.00745Z","iopub.status.idle":"2021-12-15T18:22:09.019171Z","shell.execute_reply.started":"2021-12-15T18:22:09.007415Z","shell.execute_reply":"2021-12-15T18:22:09.018396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"my_circle_loss(y_true, y_pred)","metadata":{"execution":{"iopub.status.busy":"2021-12-15T18:22:15.994861Z","iopub.execute_input":"2021-12-15T18:22:15.995113Z","iopub.status.idle":"2021-12-15T18:22:16.005852Z","shell.execute_reply.started":"2021-12-15T18:22:15.995085Z","shell.execute_reply":"2021-12-15T18:22:16.00509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"my_softmax_loss(y_true, y_pred)","metadata":{"execution":{"iopub.status.busy":"2021-12-15T18:22:16.562795Z","iopub.execute_input":"2021-12-15T18:22:16.563033Z","iopub.status.idle":"2021-12-15T18:22:16.569716Z","shell.execute_reply.started":"2021-12-15T18:22:16.563007Z","shell.execute_reply":"2021-12-15T18:22:16.568983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"softmax(my_c_crossentropy(y_true, y_pred))","metadata":{"execution":{"iopub.status.busy":"2021-12-15T17:57:38.449304Z","iopub.execute_input":"2021-12-15T17:57:38.449567Z","iopub.status.idle":"2021-12-15T17:57:38.460119Z","shell.execute_reply.started":"2021-12-15T17:57:38.449532Z","shell.execute_reply":"2021-12-15T17:57:38.459224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#custom_loss2(y_true, y_pred)","metadata":{"execution":{"iopub.status.busy":"2021-12-15T17:57:40.867907Z","iopub.execute_input":"2021-12-15T17:57:40.868427Z","iopub.status.idle":"2021-12-15T17:57:40.872665Z","shell.execute_reply.started":"2021-12-15T17:57:40.868388Z","shell.execute_reply":"2021-12-15T17:57:40.871072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_true = np.array([ [0.2, 0.8, 0.1], [0.1, 0.8, 0.1], [1, 0.01, 0.01], [0.05, 0.95, 0.01]])\ny_pred = np.array([ [0.1, 0.8, 0.1], [0.1, 0.8, 0.2], [0.05, 0.95, 0.01], [1, 0.01, 0.01]])\nmy_c_crossentropy(y_true, y_pred)","metadata":{"execution":{"iopub.status.busy":"2021-12-15T17:57:41.893272Z","iopub.execute_input":"2021-12-15T17:57:41.893905Z","iopub.status.idle":"2021-12-15T17:57:41.907297Z","shell.execute_reply.started":"2021-12-15T17:57:41.893868Z","shell.execute_reply":"2021-12-15T17:57:41.906474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.losses.categorical_crossentropy(y_true,y_pred)","metadata":{"execution":{"iopub.status.busy":"2021-12-15T17:57:42.915386Z","iopub.execute_input":"2021-12-15T17:57:42.915923Z","iopub.status.idle":"2021-12-15T17:57:42.933856Z","shell.execute_reply.started":"2021-12-15T17:57:42.915886Z","shell.execute_reply":"2021-12-15T17:57:42.933152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Строим модель\n#Загружаем предобученную сеть Xception:\nbase_model = Xception(weights='imagenet', include_top=False, input_shape = input_shape)\n\n#base_model.trainable = False\n#base_model.summary()\n#Устанавливаем новую «голову» (head):\nx = base_model.output\nx = GlobalAveragePooling2D()(x)  #Pooling слой\nx = BatchNormalization()(x) #добавим Batch нормализацию\nx = Dense(256, activation='elu')(x) # полносвязный слой с активацией elu\nx = Dropout(0.25)(x) # полносвязный слой с вероятность отключения нейронов в слое\npredictions = Dense(CLASS_NUM, activation='elu')(x) #логистический слой c 2 классами\n\nmodel = Model(inputs=base_model.input, outputs=predictions)\n#model.compile(loss=\"categorical_crossentropy\", optimizer=optimizers.Adam(lr=LR), metrics=[\"accuracy\"])\n#model.compile(loss=custom_loss2, optimizer=optimizers.Adam(lr=LR), metrics=[\"accuracy\"])\n#model.compile(loss=my_c_crossentropy, optimizer=optimizers.Adam(lr=LR), metrics=[\"accuracy\"])\n\n#model.compile(loss=my_softmax_loss, optimizer=optimizers.Adam(lr=LR), metrics=[\"accuracy\"])\nmodel.compile(loss=my_circle_loss, optimizer=optimizers.Adam(lr=LR), metrics=[\"accuracy\"])\n\n","metadata":{"execution":{"iopub.status.busy":"2021-12-15T18:23:22.279246Z","iopub.execute_input":"2021-12-15T18:23:22.279532Z","iopub.status.idle":"2021-12-15T18:23:23.358458Z","shell.execute_reply.started":"2021-12-15T18:23:22.2795Z","shell.execute_reply":"2021-12-15T18:23:23.357723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Добавим ModelCheckpoint. Эта функция позволяет сохранять прогресс обучения модели,\n#чтобы в нужный момент можно было его подгрузить и дообучить модель.\ncheckpoint = ModelCheckpoint('dvc.hdf5' , monitor = ['val_accuracy'] , verbose = 1  , mode = 'max')\n\n#останоить обучение, когда метрика перестает улучшаться.\nearlystop = EarlyStopping(monitor = 'val_accuracy', patience = 4, restore_best_weights = True)\n\n#Снижайть скорость обучения, когда метрика перестает улучшаться.\nreduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.25, patience=2, min_lr=0.0000001, verbose=1, mode='auto')\n    \ncallbacks_list = [checkpoint, earlystop, reduce_lr]","metadata":{"execution":{"iopub.status.busy":"2021-12-15T18:23:23.359999Z","iopub.execute_input":"2021-12-15T18:23:23.360271Z","iopub.status.idle":"2021-12-15T18:23:23.367502Z","shell.execute_reply.started":"2021-12-15T18:23:23.360224Z","shell.execute_reply":"2021-12-15T18:23:23.366611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_model.trainable = True","metadata":{"execution":{"iopub.status.busy":"2021-12-15T18:23:40.296859Z","iopub.execute_input":"2021-12-15T18:23:40.297362Z","iopub.status.idle":"2021-12-15T18:23:40.30619Z","shell.execute_reply.started":"2021-12-15T18:23:40.297323Z","shell.execute_reply":"2021-12-15T18:23:40.305531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#model.load_weights('../input/test12-0/test12(3).hdf5')","metadata":{"execution":{"iopub.status.busy":"2021-12-15T18:23:41.700727Z","iopub.execute_input":"2021-12-15T18:23:41.701323Z","iopub.status.idle":"2021-12-15T18:23:41.704462Z","shell.execute_reply.started":"2021-12-15T18:23:41.701284Z","shell.execute_reply":"2021-12-15T18:23:41.703756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Обучаем модель\nhistory = model.fit_generator(\n        train_generator,\n        steps_per_epoch = len(train_generator),\n        validation_data = test_generator, \n        validation_steps = len(test_generator),\n        epochs = EPOCHS,\n        callbacks = callbacks_list\n)","metadata":{"execution":{"iopub.status.busy":"2021-12-15T18:23:42.648113Z","iopub.execute_input":"2021-12-15T18:23:42.648814Z","iopub.status.idle":"2021-12-15T18:32:13.513219Z","shell.execute_reply.started":"2021-12-15T18:23:42.648776Z","shell.execute_reply":"2021-12-15T18:32:13.512494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#softmax loss \nimport tensorflow as tf\n\ndef softmax_loss(y_true, W, b, x):\n    \n    y_pred = tf.matmul(x, W) + b\n    numerators = tf.reduce_sum(y_true * tf.exp(y_pred), axis=1)\n    denominators = tf.reduce_sum(tf.exp(y_pred), axis=1)\n    loss = - tf.reduce_sum(tf.log(numerators / denominators))\n\n    return loss","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Center Loss\nimport tensorflow as tf\n\ndef circle_loss(W, b, lamda_center):\n    \n    def inner(y_true, x):\n        y_pred = tf.matmul(x, W) + b\n        numerators = tf.reduce_sum(y_true * tf.exp(y_pred), axis=1)\n        denominators = tf.reduce_sum(tf.exp(y_pred), axis=1)\n        loss_softmax = - tf.reduce_sum(tf.log(numerators / denominators))\n\n        class_freqs = tf.reduce_sum(y_true, axis=0, keepdims=True)\n        class_freqs = tf.transpose(class_freqs)\n\n        centres = tf.matmul(tf.transpose(y_true), x)\n        centres = tf.divide(centres, class_freqs)\n        repeated_centres = tf.matmul(y_true, centres)\n\n        sq_distances = tf.square(tf.norm(x - repeated_centres, axis=1))\n        loss_centre = tf.reduce_sum(sq_distances)\n\n        loss = loss_softmax + (lambda_center/2) * loss_centre \n\n        return loss\n    return inner","metadata":{"execution":{"iopub.status.busy":"2021-12-15T18:13:30.048506Z","iopub.execute_input":"2021-12-15T18:13:30.049197Z","iopub.status.idle":"2021-12-15T18:13:30.056949Z","shell.execute_reply.started":"2021-12-15T18:13:30.049157Z","shell.execute_reply":"2021-12-15T18:13:30.055975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\n\n#A-Softmax (aka SphereFace)\n\ndef SphereFaceLoss(W, x, m):\n    def inner(y_true, x):\n    \n        # replace 0 => 1 and 1=> m in y_true\n        M = (m-1) * y_true + 1\n    \n        # consider normalized weight matrix\n        normalized_W, norms = tf.linalg.normalize(W, axis=0)\n    \n        # get dot products (projections)\n        y_pred = x * normalized_W\n\n        # W . x = ||W||*||x||*cos(theta)\n        # but ||W|| = 1\n        # so (W . x) / ||x|| = cos(theta) \n    \n        cos_theta, norm_x = tf.linalg.normalize(y_pred, axis=1)    \n        theta = tf.acos(cos_theta)\n    \n        # multiply theta by appropriate margin\n        new_theta = theta * M\n        new_cos_theta = tf.cos(new_theta)\n        new_y_pred = norm_x * new_cos_theta\n\n        # the following part is the same as softmax loss\n        numerators = tf.reduce_sum(y_true * tf.exp(new_y_pred), axis=1)\n        denominators = tf.reduce_sum(tf.exp(new_y_pred), axis=1)\n        loss = - tf.reduce_sum(tf.log(numerators / denominators))\n\n        return loss\n    return inner","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Large Margin Cosine Loss (aka CosFace)\ndef CosFaceLoss(W, m, s):\n    def inner(y_true, x):\n        # replace 0 => 1 and 1=> m in y_true\n        y_true = tf.cast(y_true, dtype=tf.float32)\n        M = m * y_true\n\n        # W . x = ||W|| * ||x|| * cos(theta)\n        # so (W . x) / (||W|| * ||x||) = cos(theta)\n\n        dot_product = tf.matmul(x, W)\n        cos_theta, cos_theta_norm = tf.linalg.normalize(dot_product,axis=0)\n\n        # re-scale the cosines by a hyper-parameter s\n        # and subtract appropriate margin\n        y_pred = s * cos_theta - M\n\n        # the following part is the same as softmax loss\n        numerators = tf.reduce_sum(y_true * tf.exp(y_pred), axis=1)\n        denominators = tf.reduce_sum(tf.exp(y_pred), axis=1)\n        loss = - tf.reduce_sum(tf.math.log(numerators/denominators))\n        return loss\n    return inner","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Additive Angular Margin Loss (aka ArcFace)\nimport tensorflow as tf\n\ndef ArcFaceLoss(W, m):\n    def inner(y_true, x):\n        # replace 0 => 1 and 1=> m in y_true\n        M = (m-1) * y_true + 1\n    \n        # consider normalized weight matrix and feature vectors\n        normalized_W, norms_w = tf.linalg.normalize(W, axis=0)\n        normalized_x, norms_x = tf.linalg.normalize(x, axis=0)\n        \n        # W . x = ||W||*||x||*cos(theta)\n        # but ||W|| = 1 and ||x|| = 1\n        # so (W . x) = cos(theta) \n        cos_theta = normalized_x * normalized_W\n    \n        theta = tf.acos(cos_theta)\n    \n        # add appropriate margin to theta\n        new_theta = theta + M\n        new_cos_theta = tf.cos(new_theta)\n    \n        # re-scale the cosines by a hyper-parameter s\n        y_pred = s * new_cos_theta     \n\n        # the following part is the same as softmax loss\n        numerators = tf.reduce_sum(y_true * tf.exp(y_pred), axis=1)\n        denominators = tf.reduce_sum(tf.exp(y_pred), axis=1)\n        loss = - tf.reduce_sum(tf.log(numerators / denominators))\n    \n        return loss\n\n    return inner","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import FileLink\nFileLink(r'./dvc.hdf5')","metadata":{"execution":{"iopub.status.busy":"2021-12-15T06:11:42.764273Z","iopub.status.idle":"2021-12-15T06:11:42.76499Z","shell.execute_reply.started":"2021-12-15T06:11:42.764718Z","shell.execute_reply":"2021-12-15T06:11:42.76475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import keras\nloss = keras.losses.SparseCategoricalCrossentropy()","metadata":{"execution":{"iopub.status.busy":"2021-12-15T05:35:28.350567Z","iopub.execute_input":"2021-12-15T05:35:28.351235Z","iopub.status.idle":"2021-12-15T05:35:28.354716Z","shell.execute_reply.started":"2021-12-15T05:35:28.351196Z","shell.execute_reply":"2021-12-15T05:35:28.353978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss([1.1],[1.1])","metadata":{"execution":{"iopub.status.busy":"2021-12-15T05:36:44.868347Z","iopub.execute_input":"2021-12-15T05:36:44.869068Z","iopub.status.idle":"2021-12-15T05:36:44.880068Z","shell.execute_reply.started":"2021-12-15T05:36:44.869031Z","shell.execute_reply":"2021-12-15T05:36:44.879248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!git clone https://github.com/4uiiurz1/keras-arcface.git","metadata":{"execution":{"iopub.status.busy":"2021-12-15T05:21:22.182314Z","iopub.execute_input":"2021-12-15T05:21:22.182742Z","iopub.status.idle":"2021-12-15T05:21:23.397235Z","shell.execute_reply.started":"2021-12-15T05:21:22.182708Z","shell.execute_reply":"2021-12-15T05:21:23.396463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nsys.path.append('./keras-arcface')","metadata":{"execution":{"iopub.status.busy":"2021-12-15T05:24:16.123488Z","iopub.execute_input":"2021-12-15T05:24:16.124071Z","iopub.status.idle":"2021-12-15T05:24:16.127589Z","shell.execute_reply.started":"2021-12-15T05:24:16.124031Z","shell.execute_reply":"2021-12-15T05:24:16.126859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras import backend as K\nfrom keras.layers import Layer\nfrom keras import regularizers\n\nimport tensorflow as tf","metadata":{"execution":{"iopub.status.busy":"2021-12-15T05:30:52.822595Z","iopub.execute_input":"2021-12-15T05:30:52.822855Z","iopub.status.idle":"2021-12-15T05:30:52.826968Z","shell.execute_reply.started":"2021-12-15T05:30:52.822825Z","shell.execute_reply":"2021-12-15T05:30:52.82597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2021-12-15T05:49:03.094659Z","iopub.execute_input":"2021-12-15T05:49:03.094914Z","iopub.status.idle":"2021-12-15T05:49:03.101706Z","shell.execute_reply.started":"2021-12-15T05:49:03.094885Z","shell.execute_reply":"2021-12-15T05:49:03.100709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label = Input(shape=(10,))\n\n#Строим модель\n#Загружаем предобученную сеть Xception:\nbase_model = Xception(weights='imagenet', include_top=False, input_shape = input_shape)\n\n#base_model.trainable = False\n#base_model.summary()\n#Устанавливаем новую «голову» (head):\nx = base_model.output\nx = GlobalAveragePooling2D()(x)  #Pooling слой\nx = BatchNormalization()(x) #добавим Batch нормализацию\nx = Dense(256, activation='elu')(x) # полносвязный слой с активацией elu\nx = Dropout(0.25)(x) # полносвязный слой с вероятность отключения нейронов в слое\nx = Dense(512, kernel_initializer='he_normal')(x)\nx = BatchNormalization()(x)\noutput = ArcFace(n_classes=2)([x, label])\n\nmodel = Model([input, label], output)\n\nmodel.compile(loss='categorical_crossentropy',\n              optimizer=Adam(),\n              metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2021-12-15T05:56:46.766208Z","iopub.execute_input":"2021-12-15T05:56:46.766494Z","iopub.status.idle":"2021-12-15T05:56:48.071312Z","shell.execute_reply.started":"2021-12-15T05:56:46.766457Z","shell.execute_reply":"2021-12-15T05:56:48.070229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nx = BatchNormalization()(x)\nx = Dropout(0.5)(x)\nx = Flatten()(x)\nx = Dense(512, kernel_initializer='he_normal')(x)\nx = BatchNormalization()(x)\noutput = ArcFace(num_classes=10)([x, label])\n\n\n\nmodel = Model([input, label], output)\n\nmodel.compile(loss='categorical_crossentropy',\n              optimizer=Adam(),\n              metrics=['accuracy'])\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_img = '../input/test-dvc/test1/1.jpg'","metadata":{"execution":{"iopub.status.busy":"2021-12-15T18:33:18.883318Z","iopub.execute_input":"2021-12-15T18:33:18.884136Z","iopub.status.idle":"2021-12-15T18:33:18.887674Z","shell.execute_reply.started":"2021-12-15T18:33:18.884093Z","shell.execute_reply":"2021-12-15T18:33:18.886858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from matplotlib import pyplot\nfrom matplotlib.image import imread\nimage = imread(test_img)\n# plot raw pixel data\npyplot.imshow(image)\n# show the figure\npyplot.show()\nimage = cv2.imread(test_img)[..., ::-1]\nimage3 = cv2.resize(image, (224, 224))/255\npred = model.predict(image3[None, ...])\n#print(pred)\npredict = np.argmax(pred, axis=-1)\n#print(predictions)\nif predict == 0:\n    print('it`s cat')\nelse:\n    print('it`s dog') ","metadata":{"execution":{"iopub.status.busy":"2021-12-15T18:33:19.132122Z","iopub.execute_input":"2021-12-15T18:33:19.132426Z","iopub.status.idle":"2021-12-15T18:33:19.394274Z","shell.execute_reply.started":"2021-12-15T18:33:19.132396Z","shell.execute_reply":"2021-12-15T18:33:19.393533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"       \n","metadata":{"execution":{"iopub.status.busy":"2021-12-12T15:54:48.17059Z","iopub.execute_input":"2021-12-12T15:54:48.170916Z","iopub.status.idle":"2021-12-12T15:54:48.247308Z","shell.execute_reply.started":"2021-12-12T15:54:48.170881Z","shell.execute_reply":"2021-12-12T15:54:48.245311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Посмотрим на данные\n!ls ../tmp/dogs-vs-cats/","metadata":{"execution":{"iopub.status.busy":"2021-12-12T13:54:11.464875Z","iopub.execute_input":"2021-12-12T13:54:11.465192Z","iopub.status.idle":"2021-12-12T13:54:12.224813Z","shell.execute_reply.started":"2021-12-12T13:54:11.465161Z","shell.execute_reply":"2021-12-12T13:54:12.223745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2021-12-12T13:54:42.80256Z","iopub.execute_input":"2021-12-12T13:54:42.80287Z","iopub.status.idle":"2021-12-12T13:54:42.808483Z","shell.execute_reply.started":"2021-12-12T13:54:42.802841Z","shell.execute_reply":"2021-12-12T13:54:42.807485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2021-12-12T13:55:09.923677Z","iopub.execute_input":"2021-12-12T13:55:09.923991Z","iopub.status.idle":"2021-12-12T13:55:10.89536Z","shell.execute_reply.started":"2021-12-12T13:55:09.923942Z","shell.execute_reply":"2021-12-12T13:55:10.894333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from matplotlib import pyplot\nfrom matplotlib.image import imread\n# define location of dataset\nfolder = train_dir + '/'\n# plot first few images\nfor i in range(9):\n# define subplot\n    pyplot.subplot(330 + 1 + i)\n# define filename\n    filename = folder + 'dog.' + str(i) + '.jpg'\n    \n# load image pixels\n    #print(filename)\n    \n    image = imread(filename)\n# plot raw pixel data\n    pyplot.imshow(image)\n    # show the figure\n\npyplot.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-12T13:55:48.214347Z","iopub.execute_input":"2021-12-12T13:55:48.214696Z","iopub.status.idle":"2021-12-12T13:55:49.159178Z","shell.execute_reply.started":"2021-12-12T13:55:48.21466Z","shell.execute_reply":"2021-12-12T13:55:49.158313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_file_list = os.listdir('../tmp/dogs-vs-cats/train')\nprint('Num of images:', len(train_file_list))\n\ndog_filenames = [fn for fn in train_file_list if fn.startswith('dog')]\ncat_filenames = [fn for fn in train_file_list if fn.startswith('cat')]\n\ndataset_filenames = train_test_split(\n    dog_filenames, cat_filenames, test_size=0.1, shuffle=True, random_state=42\n)\n\n\ntrain_dog, valid_dog, train_cat, valid_cat  = [fns for fns in dataset_filenames]\n\n\ntrain_dog_total, valid_dog_total, train_cat_total, valid_cat_total = [len(fns) for fns in dataset_filenames]\ntrain_total = train_dog_total + train_cat_total\nvalid_total = valid_dog_total + valid_cat_total\nprint('Train: {}, test: {}'.format(train_total, valid_total))","metadata":{"execution":{"iopub.status.busy":"2021-12-12T13:56:51.36617Z","iopub.execute_input":"2021-12-12T13:56:51.366502Z","iopub.status.idle":"2021-12-12T13:56:51.412015Z","shell.execute_reply.started":"2021-12-12T13:56:51.366466Z","shell.execute_reply":"2021-12-12T13:56:51.411122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls -la '../tmp/dogs-vs-cats/train'\n","metadata":{"execution":{"iopub.status.busy":"2021-12-12T14:00:49.222352Z","iopub.execute_input":"2021-12-12T14:00:49.222667Z","iopub.status.idle":"2021-12-12T14:00:50.499543Z","shell.execute_reply.started":"2021-12-12T14:00:49.222637Z","shell.execute_reply":"2021-12-12T14:00:50.498479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Строим модель\n#Загружаем предобученную сеть Xception:\nbase_model = Xception(weights='imagenet', include_top=False, input_shape = input_shape)\n\n#base_model.trainable = False\n#base_model.summary()\n#Устанавливаем новую «голову» (head):\nx = base_model.output\nx = GlobalAveragePooling2D()(x)  #Pooling слой\nx = BatchNormalization()(x) #добавим Batch нормализацию\nx = Dense(256, activation='elu')(x) # полносвязный слой с активацией elu\nx = Dropout(0.25)(x) # полносвязный слой с вероятность отключения нейронов в слое\npredictions = Dense(CLASS_NUM, activation='softmax')(x) #логистический слой c 3 классами\n\n\nmodel = Model(inputs=base_model.input, outputs=predictions)\nmodel.compile(loss=\"categorical_crossentropy\", optimizer=optimizers.Adam(lr=LR), metrics=[\"accuracy\"])\n\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Добавим ModelCheckpoint. Эта функция позволяет сохранять прогресс обучения модели,\n#чтобы в нужный момент можно было его подгрузить и дообучить модель.\ncheckpoint = ModelCheckpoint('dvc.hdf5' , monitor = ['val_accuracy'] , verbose = 1  , mode = 'max')\n\n#останоить обучение, когда метрика перестает улучшаться.\nearlystop = EarlyStopping(monitor = 'val_accuracy', patience = 4, restore_best_weights = True)\n\n#Снижайть скорость обучения, когда метрика перестает улучшаться.\nreduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.25, patience=2, min_lr=0.0000001, verbose=1, mode='auto')\n    \ncallbacks_list = [checkpoint, earlystop, reduce_lr]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nprint('Tensorflow   :', tf.__version__)\nprint('Keras        :', tf.keras.__version__)","metadata":{"execution":{"iopub.status.busy":"2021-12-20T07:06:54.157444Z","iopub.execute_input":"2021-12-20T07:06:54.157714Z","iopub.status.idle":"2021-12-20T07:06:55.144098Z","shell.execute_reply.started":"2021-12-20T07:06:54.157684Z","shell.execute_reply":"2021-12-20T07:06:55.143352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! pip install neptune-client neptune-tensorflow-keras tensorflow==2.6.2","metadata":{"execution":{"iopub.status.busy":"2021-12-20T07:07:02.144416Z","iopub.execute_input":"2021-12-20T07:07:02.144695Z","iopub.status.idle":"2021-12-20T07:07:31.425286Z","shell.execute_reply.started":"2021-12-20T07:07:02.144662Z","shell.execute_reply":"2021-12-20T07:07:31.424365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nimport neptune.new as neptune\nfrom neptune.new.integrations.tensorflow_keras import NeptuneCallback\n\nrun = neptune.init(project='common/tf-keras-integration',\napi_token='ANONYMOUS')\n\nmnist = tf.keras.datasets.mnist\n(x_train, y_train), (x_test, y_test) = mnist.load_data()\nx_train, x_test = x_train / 255.0, x_test / 255.0\n\ny_train = tf.one_hot(y_train, 10)\ny_test = tf.one_hot(y_test, 10)\n\nmodel_input = tf.keras.Input((28, 28))\nflat = tf.keras.layers.Flatten()(model_input)\ndense_1 = tf.keras.layers.Dense(32, activation=tf.keras.activations.relu)(flat)\ndense_2 = tf.keras.layers.Dense(10, activation='sigmoid')(dense_1)\ndense_2.trainable = False\n\nmodel = tf.keras.models.Model(inputs=model_input, outputs=[dense_1, dense_2])\nmodel.build(input_shape=(28, 28))\nmodel.summary()\n\ndef CosFaceLoss(W, m, s):\n    def inner(y_true, x):\n        # replace 0 => 1 and 1=> m in y_true\n        y_true = tf.cast(y_true, dtype=tf.float32)\n        M = m * y_true\n\n        # W . x = ||W|| * ||x|| * cos(theta)\n        # so (W . x) / (||W|| * ||x||) = cos(theta)\n\n        dot_product = tf.matmul(x, W)\n        cos_theta, cos_theta_norm = tf.linalg.normalize(dot_product,axis=0)\n\n        # re-scale the cosines by a hyper-parameter s\n        # and subtract appropriate margin\n        y_pred = s * cos_theta - M\n\n        # the following part is the same as softmax loss\n        numerators = tf.reduce_sum(y_true * tf.exp(y_pred), axis=1)\n        denominators = tf.reduce_sum(tf.exp(y_pred), axis=1)\n        loss = - tf.reduce_sum(tf.math.log(numerators/denominators))\n        return loss\n    return inner\n\ndef ArcFaceLoss(W, m, s):\n    def inner(y_true, x):\n        # replace 0 => 1 and 1=> m in y_true\n        M = (m-1) * y_true + 1\n        \n        # W . x = ||W||*||x||*cos(theta)\n        # but ||W|| = 1 and ||x|| = 1\n        # so (W . x) = cos(theta) \n        dot_product = tf.matmul(x, W)\n        cos_theta,cos_theta_norms = tf.linalg.normalize(dot_product,axis=0)\n\n        theta = tf.acos(cos_theta)\n    \n        # add appropriate margin to theta\n        new_theta = theta + M\n        new_cos_theta = tf.cos(new_theta)\n    \n        # re-scale the cosines by a hyper-parameter s\n        y_pred = s * new_cos_theta     \n\n        # the following part is the same as softmax loss\n        numerators = tf.reduce_sum(y_true * tf.exp(y_pred), axis=1)\n        denominators = tf.reduce_sum(tf.exp(y_pred), axis=1)\n        loss = - tf.reduce_sum(tf.log(numerators / denominators))\n    \n        return loss\n\n    return inner\n\n\ndef dummy_loss(ytrue, ypred):\n    return tf.constant([0])\n\n# uncomment the loss of your choice and run the script\n# multiple times as needed\nloss_func = CosFaceLoss(W=model.layers[-1].weights[0], m=10.0, s=10.0)\n# loss_func = ArcFaceLoss(W=model.layers[-1].weights[0], m=0.2, s=10.0)\n\nmodel.compile(optimizer='adam', metrics=['accuracy'],\n              loss=[loss_func, dummy_loss])\n\nneptune_cbk = NeptuneCallback(run=run, base_namespace='metrics')\n\nmodel.fit(x_train,y_train,epochs=5,batch_size=64,callbacks=[neptune_cbk])","metadata":{"execution":{"iopub.status.busy":"2021-12-20T06:55:20.088835Z","iopub.execute_input":"2021-12-20T06:55:20.089184Z","iopub.status.idle":"2021-12-20T06:55:24.174207Z","shell.execute_reply.started":"2021-12-20T06:55:20.089081Z","shell.execute_reply":"2021-12-20T06:55:24.172914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}