{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"prepare data","metadata":{}},{"cell_type":"code","source":"import zipfile\nimport os\nimport sys\nimport random\nimport shutil\n\n\nntst = .1\nif not os.path.isdir(\"/kaggle/working/dataset/train/0\"):\n    print(\"unzipping\")\n    with zipfile.ZipFile('/kaggle/input/dogs-vs-cats/train.zip',\"r\") as z:\n        z.extractall()    \n\n    print(\"creating folders\")\n    os.makedirs(\"/kaggle/working/dataset/train/0\")\n    os.makedirs(\"/kaggle/working/dataset/train/1\")\n    os.makedirs(\"/kaggle/working/dataset/test/0\")\n    os.makedirs(\"/kaggle/working/dataset/test/1\")\n    os.makedirs(\"/kaggle/working/out\")\n\n    print(\"reorganizing data\")\n    imfn_all = os.listdir(\"/kaggle/working/train/\")\n    imfn_dogs = [imfn for imfn in imfn_all if 'dog' in imfn]\n    imfn_cats = [imfn for imfn in imfn_all if 'cat' in imfn]\n    random.shuffle(imfn_dogs)\n    random.shuffle(imfn_cats)\n\n    ntst_dogs = int(ntst*len(imfn_dogs))\n    ntst_cats = int(ntst*len(imfn_cats))\n\n    for i in range(len(imfn_dogs)):\n        if i<ntst_dogs:\n            shutil.move(\"/kaggle/working/train/{}\".format(imfn_dogs[i]),\"/kaggle/working/dataset/test/0/\")\n        else:\n            shutil.move(\"/kaggle/working/train/{}\".format(imfn_dogs[i]),\"/kaggle/working/dataset/train/0/\")\n\n    for i in range(len(imfn_cats)):\n        if i<ntst_dogs:\n            shutil.move(\"/kaggle/working/train/{}\".format(imfn_cats[i]),\"/kaggle/working/dataset/test/1/\")\n        else:\n            shutil.move(\"/kaggle/working/train/{}\".format(imfn_cats[i]),\"/kaggle/working/dataset/train/1/\")\n\n    for dirname, _, filenames in os.walk('/kaggle/working/dataset'):\n        print(dirname,len(filenames))","metadata":{"execution":{"iopub.status.busy":"2022-12-15T20:42:33.029185Z","iopub.status.idle":"2022-12-15T20:42:33.029910Z","shell.execute_reply.started":"2022-12-15T20:42:33.029658Z","shell.execute_reply":"2022-12-15T20:42:33.029682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"experiments","metadata":{}},{"cell_type":"code","source":"import sklearn\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport numpy as np\nimport os\nfrom sklearn import metrics\nfrom sklearn.metrics import roc_curve\nfrom matplotlib import pyplot\nfrom matplotlib.image import imread\n\nclass Const:\n    def __init__(self):\n        self.path_trn='/kaggle/working/dataset/train'\n        self.path_tst='/kaggle/working/dataset/test'\n        self.path_out='/kaggle/working/out'\n        self.path_model=\"{}/w.h5\".format(self.path_out)\n\n        self.sz=(128, 128)\n        self.is_overfitting=False\n        self.is_aug=True\n\n        self.dropout=0.25\n        self.lr=.01\n        self.decay_factor=.5\n        self.decay_min_lr=.00001\n\n        self.batch_sz=20,\n        self.steps=100,\n        self.epochs=50,\n        self.bsz_infer=100\n        self.val_split=.2\n        self.estop_pateience=10\n\ndef viz(const):\n    for c in range(2):\n        print(\"dog\" if c==0 else \"cat\")\n        pth=\"{}/{}\".format(const.path_trn,c)\n        fn_lst=os.listdir(pth)\n        random.shuffle(fn_lst)\n        for i,fn in enumerate(fn_lst[:9]):\n            pyplot.subplot(330 + 1 + i)\n            im = imread(\"{}/{}/{}\".format(const.path_trn , c , fn))\n            pyplot.imshow(im)\n        pyplot.show()\n\ndef stats(ctrn,class_weights,const):\n    cls = np.unique(ctrn)\n    ntotal = sum([len(os.listdir(\"{}/{}\".format(const.path_trn, c))) for c in cls])\n    ntrn = int(ntotal * (1 - const.val_split))\n    nval = int(ntotal * const.val_split)\n    ntst = sum([len(os.listdir(\"{}/{}\".format(const.path_tst, c))) for c in cls])\n    print(\"class weights\", class_weights, \"ntrain\", ntrn, \"nval\", nval, \\\n          \"ntst\", ntst)\n    return ntrn,nval,ntst\n\n\ndef get_gtrn(path, sz, bsz, validation_split, is_overfitting=False, is_aug=True):\n    if is_aug:\n        data_gen = tf.keras.preprocessing.image.ImageDataGenerator(\n            horizontal_flip = True,\n            validation_split=validation_split,\n            rescale=1./255,\n            rotation_range=15,\n            width_shift_range=0.1,\n            height_shift_range=0.1,\n            shear_range=0.1,\n            zoom_range=0.2\n        )\n    else:\n        data_gen = tf.keras.preprocessing.image.ImageDataGenerator(rescale=1./255,validation_split=validation_split)\n    tgen = data_gen.flow_from_directory(path, target_size=sz,batch_size=20, class_mode='categorical', subset='training')\n    if is_overfitting:\n        vgen=tgen\n    else:\n        vgen = data_gen.flow_from_directory(path, target_size=sz,batch_size=20, class_mode='categorical',subset='validation')\n    return tgen, vgen\n\n\ndef get_gtst(path, sz,bsz):\n    data_gen = tf.keras.preprocessing.image.ImageDataGenerator(rescale=1./255)\n    gen = data_gen.flow_from_directory(path, target_size=sz,batch_size=20, class_mode='categorical')\n    return gen\n\n\ndef load_model(sz,const):\n    input = tf.keras.Input(shape=(sz[0], sz[1], 3))\n\n    out=tf.keras.layers.Conv2D(kernel_size=(3, 3), filters=32, activation=\"relu\")(input)\n    out=tf.keras.layers.BatchNormalization()(out)\n    out=tf.keras.layers.MaxPooling2D()(out)\n#    out=tf.keras.layers.Dropout(const.dropout)(out)\n\n    out=tf.keras.layers.Conv2D(kernel_size=(3, 3), filters=64, activation=\"relu\")(out)\n    out=tf.keras.layers.BatchNormalization()(out)\n    out=tf.keras.layers.MaxPooling2D()(out)\n#    out=tf.keras.layers.Dropout(const.dropout)(out)\n\n    out=tf.keras.layers.Conv2D(kernel_size=(3, 3), filters=128, activation=\"relu\")(out)\n    out=tf.keras.layers.BatchNormalization()(out)\n    out=tf.keras.layers.MaxPooling2D()(out)\n#    out=tf.keras.layers.Dropout(const.dropout)(out)\n\n    out=tf.keras.layers.Flatten()(out)\n    out=tf.keras.layers.Dense(512, activation='relu')(out)\n    out=tf.keras.layers.BatchNormalization()(out)\n#    out=tf.keras.layers.Dropout(const.dropout)(out)\n    out=tf.keras.layers.Dense(2, activation='softmax')(out)\n\n    model=tf.keras.models.Model(inputs=input, outputs=out)\n    loss = tf.keras.losses.CategoricalCrossentropy()\n    opt = tf.keras.optimizers.RMSprop(learning_rate=const.lr)\n    model.compile(optimizer=opt, loss=loss, metrics=['accuracy'])\n\n    model.summary()\n    return model\n\n\ndef train(const,expr_name=\"\"):\n#    tf.debugging.set_log_device_placement(True)\n\n    print(\"\\nloading model\",flush=True)\n    model = load_model(const.sz,const)\n\n    # prepare and balance data loaders\n    print(\"\\nloading training data\",flush=True)\n    gtrn, gval = get_gtrn(const.path_trn,const.sz, const.batch_sz, validation_split=const.val_split, \\\n                               is_overfitting=const.is_overfitting, is_aug=const.is_aug)\n    ctrn = gtrn.classes\n    class_weights = sklearn.utils.compute_class_weight(class_weight='balanced', classes=np.unique(ctrn), y=ctrn)\n    class_weights = dict(zip(np.unique(ctrn), class_weights))\n\n    # data stats\n    print(\"data:\",end=\" \",flush=True)\n    print(const.path_trn)\n    stats(ctrn,class_weights,const)\n\n    # train\n    print(\"\\ntraining\",flush=True)\n    ncpu = os. cpu_count()\n    print(\"ncpu\",ncpu)\n    estop=tf.keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=const.estop_pateience, \\\n                                           verbose=2, mode=\"min\", restore_best_weights=True)\n    decay = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', patience=2, verbose=1, \\\n                                                 factor=const.decay_factor, min_lr=const.decay_min_lr)\n    checkpoint = tf.keras.callbacks.ModelCheckpoint(\"{}/w_{}.h5\".format(c.path_out, expr_name), \\\n                                                    monitor='loss', verbose=1, save_best_only=True, mode='min')\n\n    history = model.fit(gtrn, validation_data=gval, epochs=const.epochs, \\\n                        class_weight=class_weights, callbacks=[estop, decay,checkpoint],\n                        use_multiprocessing=True,workers=ncpu)\n    model.save_weights(const.path_model)\n    \n    # training history visualization\n    print(\"visualizing training process\",flush=True)\n    plt.plot(history.history['accuracy'])\n    plt.plot(history.history['val_accuracy'])\n    plt.title('acc')\n    plt.ylabel('acc')\n    plt.xlabel('epoch')\n    plt.legend(['trn', 'val'], loc='upper left')\n    #plt.savefig(\"{}/acc.jpg\".format(const.path_out))\n    plt.show()\n    plt.close()\n\n    plt.plot(history.history['loss'])\n    plt.plot(history.history['val_loss'])\n    plt.title('loss')\n    plt.ylabel('loss')\n    plt.xlabel('e')\n    plt.legend(['trn', 'val'], loc='upper left')\n    #plt.savefig(\"{}/loss.jpg\".format(const.path_out))\n    plt.show()\n    plt.close()\n\n    \ndef eval(const):\n    model=load_model(const.sz,const)\n    model.load_weights(const.path_model)\n    gtst=get_gtst(const.path_tst if not const.is_overfitting else const.path_trn,const.sz,const.batch_sz)\n    test_scores = model.evaluate(x=gtst, verbose=2)\n    print(\"loss\",test_scores[0],\"acc\",test_scores[1])\n\ndef test(const):\n    model = load_model(const.sz,const)\n    model.load_weights(const.path_model)\n\n    data_gen = tf.keras.preprocessing.image.ImageDataGenerator(rescale=1./255)\n    gen = data_gen.flow_from_directory(\n        const.path_tst if not const.is_overfitting else const.path_trn, \n        target_size=(128,128),\n        batch_size=1, \n        class_mode='categorical', \n        shuffle=False)\n    \n    P = model.predict(gen,verbose=1)\n    \n    print(\"dogs\")\n    fpr, rec, _ = roc_curve(gen.classes, P[:,0],pos_label=0)\n    plt.plot(fpr,rec)\n    plt.ylabel('rec')\n    plt.xlabel('fpr')\n    plt.show()\n    plt.close()\n    \n    print(\"cats\")\n    fpr, rec, _ = roc_curve(gen.classes, P[:,1],pos_label=1)\n    plt.plot(fpr,rec)\n    plt.ylabel('rec')\n    plt.xlabel('fpr')\n    plt.show()\n    plt.close()\n    \n    print(\"\\nperformance statistics:\")\n    for c in range(2):\n        rec,fpr=[],[]\n        for th in [.98,.95]+[1./(th+1) for th in range(1,10)]+[.05,.02,0]:\n            data={'tp':0, 'fp':0, 'tn':0, 'fn':0}\n            for i in range(P.shape[0]):\n                if gen.classes[i]==c:\n                    if P[i, c]>=th:\n                        data['tp']+=1\n                    else:\n                        data['fn']+=1\n                else:\n                    if P[i, c]>=th:\n                        data['fp']+=1\n                    else:\n                        data['tn']+=1\n            pos = data['tp'] + data['fn']\n            neg = data['tn'] + data['fp']\n\n            rec += [float(data['tp'])/pos]\n            fpr += [float(data['fp'])/neg]\n            print(\"class {} th {:.2f} rec {:.3f} fpr {:.3f} pos {} neg {} tp {} fp {} tn {} fn {}\".format(c,th,rec[-1],fpr[-1],pos,neg,data['tp'],data['fp'],data['tn'],data['fn']))\n        print(\"\\n\")","metadata":{"execution":{"iopub.status.busy":"2022-12-15T20:56:32.978203Z","iopub.execute_input":"2022-12-15T20:56:32.978866Z","iopub.status.idle":"2022-12-15T20:56:33.033318Z","shell.execute_reply.started":"2022-12-15T20:56:32.978819Z","shell.execute_reply":"2022-12-15T20:56:33.032247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"experiments","metadata":{}},{"cell_type":"code","source":"if __name__=='__main__':\n    \"\"\"\n    start: subset , im size 32x32 , overfit (im size 32x32) , rmsprop\n    res: trn loss plateaue\n    \"\"\"\n    expr_name = \"e10_sz32_drop0_bn0_aug0_overfit1_ntrn1K\"\n    c = Const()\n    c.path_trn = \"/kaggle/working/dataset/train1000\"\n    c.epochs = 10\n    c.dropout = 0\n    c.is_overfitting = True\n    c.is_aug = False\n    c.sz = (32, 32)\n    c.path_model = \"{}/w_{}.h5\".format(c.path_out, expr_name)\n    #print(expr_name)\n    #train(c)\n    #eval(c)\n\n    \"\"\"\n    changes: all data\n    res: trn loss plateaue\n    \"\"\"\n    expr_name = \"e50_sz32_drop0_bn0_aug0_overfit1_ntrnALL\"\n    c = Const()\n    c.path_trn = \"/kaggle/working/dataset/train\"\n    c.epochs = 50\n    c.dropout = 0\n    c.is_overfitting = True\n    c.is_aug = False\n    c.sz = (32, 32)\n    c.path_model = \"{}/w_{}.h5\".format(c.path_out, expr_name)\n    #print(expr_name)\n    #train(c)\n    #eval(c)\n    \n    \"\"\"\n    changes: im size 128x128 , non overfit , aug\n    res: trn loss plateaue\n    \"\"\"\n    expr_name = \"e100_drop0_bn1_aug1_overfit0_ntrnALL\"\n    c = Const()\n    c.path_train = \"/kaggle/working/dataset/train\"\n    c.epochs = 100\n    c.dropout = 0\n    c.is_overfitting = False\n    c.is_aug = True\n    c.path_model = \"{}/w_{}.h5\".format(c.path_out, expr_name)\n    #print(expr_name)\n    #train(c)\n    #eval(c)\n    \n    \"\"\"\n    changes: overfit , dropout p=.25\n    res: trn loss plateaue\n    \"\"\"\n    expr_name = \"e50_sz128_drop25_bn0_aug1_overfit1_ntrnALL\"\n    c = Const()\n    c.path_train = \"/kaggle/working/dataset/train\"\n    c.epochs = 50\n    c.dropout = .25\n    c.is_overfitting = True\n    c.is_aug = True\n    c.path_model = \"{}/w_{}.h5\".format(c.path_out, expr_name)\n    #print(expr_name)\n    #train(c)\n    #eval(c)\n    \n    \"\"\"\n    prev: e100_drop0_bn1_aug1_overfit0_ntrnALL\n    changes: overfit , bn\n    res: trn & val losses converges in 15e\n    \"\"\"\n    expr_name = \"e50_sz128_drop0_bn1_aug1_overfit1_ntrnALL\"\n    c = Const()\n    c.path_train = \"/kaggle/working/dataset/train\"\n    c.epochs = 50\n    c.dropout = 0\n    c.is_overfitting = True\n    c.is_aug = True\n    c.path_model = \"{}/w_{}.h5\".format(c.path_out, expr_name)\n    #print(expr_name)\n    #train(c)\n    #eval(c)\n    \n    \"\"\"\n    changes: eval test\n    \"\"\"\n    expr_name = \"e50_drop0_bn1_aug1_overfit1_ntrnALL\"\n    c = Const()\n    c.path_train = \"/kaggle/working/dataset/train\"\n    c.epochs = 50\n    c.dropout = 0\n    c.is_overfitting = False\n    c.is_aug = True\n    c.path_model = \"{}/w_{}.h5\".format(c.path_out, expr_name)\n    #print(expr_name)\n    #train(c,expr_name)\n    #eval(c)\n    \n    \"\"\"\n    changes: +roc\n    \"\"\"\n    expr_name = \"e50_drop0_bn1_aug1_overfit1_ntrnALL\"\n    c = Const()\n    c.path_train = \"/kaggle/working/dataset/train\"\n    c.path_train = \"/kaggle/working/dataset/test\"\n    c.epochs = 50\n    c.dropout = 0\n    c.is_overfitting = False\n    c.is_aug = True\n    c.path_model = \"{}/w_{}.h5\".format(c.path_out, expr_name)\n    print(expr_name)\n    viz(c)\n    train(c,expr_name)\n    eval(c)\n    test(c)","metadata":{"execution":{"iopub.status.busy":"2022-12-15T20:56:39.504895Z","iopub.execute_input":"2022-12-15T20:56:39.505305Z","iopub.status.idle":"2022-12-16T01:02:51.634182Z","shell.execute_reply.started":"2022-12-15T20:56:39.505265Z","shell.execute_reply":"2022-12-16T01:02:51.632041Z"},"trusted":true},"execution_count":null,"outputs":[]}]}