{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"from PIL import Image\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport cv2\nfrom sklearn.model_selection import train_test_split, StratifiedShuffleSplit\nimport tensorflow as tf\nfrom tensorflow.python.framework import ops\nimport math\nimport glob\nimport zipfile\nimport tensorflow_datasets as tfds\n# from skimage.transform import resize   # for resizing images\n\nAUTOTUNE = tf.data.experimental.AUTOTUNE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# constants:\nBATCH_SIZE=64\n#BATCH_SIZE=56\ntarget_image_dim = 224\n#target_image_dim = 336","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"local_zip = '/kaggle/input/cassava-disease/train.zip'\nzip_ref = zipfile.ZipFile(local_zip, 'r')\nzip_ref.extractall('/tmp')\nzip_ref.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainDIR = '/tmp/train/'\nvalDIR = '/tmp/val/'\ntestDIR = '/tmp/test/'\nif (not(os.path.exists(valDIR))):\n    os.mkdir(valDIR)\n    for cx in os.listdir(trainDIR):\n        os.mkdir(os.path.join(valDIR,cx))\nif (not(os.path.exists(testDIR))):\n    os.mkdir(testDIR)\n    for cx in os.listdir(trainDIR):\n        os.mkdir(os.path.join(testDIR,cx))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cxnames = os.listdir(trainDIR)\nnumSamples = [len(os.listdir(os.path.join(trainDIR,c)))  for c in cxnames]\nnumClasses = len(cxnames)\nfor j in range(numClasses):\n    print('class %s: %d/%d' % (cxnames[j],numSamples[j],sum(numSamples)))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Actually let's grab the data from TFDS \ncassava, info =  tfds.load('cassava', as_supervised=True, with_info=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"info","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset,val_dataset,test_dataset = cassava['train'], cassava['validation'], cassava['test']\nnum_train_examples= info.splits['train'].num_examples\nnum_val_examples= info.splits['validation'].num_examples\nnum_test_examples= info.splits['test'].num_examples\nprint((num_train_examples,num_val_examples,num_test_examples))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Same number of training images as Kaggle ds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def convert_and_resize(image,label):\n    image = tf.image.convert_image_dtype(image,tf.float32) # cast and normalize the image to [0,1]\n    image = tf.image.resize_with_pad(image, target_image_dim, target_image_dim)\n    return (image,label)\n    \ndef augment(image,label):\n    image = tf.image.convert_image_dtype(image,tf.float32)\n    image = tf.image.resize_with_pad(image, target_image_dim, target_image_dim)\n    image = tf.image.random_crop(image,size=(target_image_dim//2,target_image_dim//2,3))\n    image = tf.image.resize_with_pad(image, target_image_dim, target_image_dim)\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_saturation(image,0.0,3.0)\n    image = tf.image.random_brightness(image,0.3)\n    image = tf.image.random_contrast(image,0.5,2.0)\n    return (image,label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def killImage(image,label):\n    image = tf.image.convert_image_dtype(image,tf.float32)\n    image = tf.image.resize_with_pad(image, target_image_dim, target_image_dim)\n    image =  tf.image.adjust_brightness(image,-2)\n    return (image,label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Build BATCHES:\naugmented_train_batches = (\n    train_dataset\n    .cache()\n    .shuffle(num_train_examples//4)\n    .map(augment, num_parallel_calls=AUTOTUNE)\n    .batch(BATCH_SIZE)\n    .prefetch(AUTOTUNE)\n) \nnon_augmented_train_batches = (\n    train_dataset\n    .cache()\n    .shuffle(num_train_examples//4)\n    .map(convert_and_resize, num_parallel_calls=AUTOTUNE)\n    .batch(BATCH_SIZE)\n    .prefetch(AUTOTUNE)\n)\nkilled_train_batches = (\n    train_dataset\n    .cache()\n    .shuffle(num_train_examples//4)\n    .map(killImage, num_parallel_calls=AUTOTUNE)\n    .batch(BATCH_SIZE)\n    .prefetch(AUTOTUNE)\n)\nvalidation_batches = (\n    val_dataset\n    .map(convert_and_resize, num_parallel_calls=AUTOTUNE)\n    .batch(2*BATCH_SIZE)\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_KilledModel(optimizer='sgd',lr=1e-2):\n    model = tf.keras.Sequential([\n        tf.keras.layers.GlobalAveragePooling3D(input_shape=(target_image_dim,target_image_dim, 3)),\n        tf.keras.layers.Dense(numClasses, activation='softmax')\n    ])\n    model.compile(optimizer=tf.keras.optimizers.SGD(learning_rate=1e-5),\n                  loss='sparse_categorical_crossentropy',metrics=['accuracy'])\n    return model\n# doesn't work: cannot pool channels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_KilledModel():\n    model = tf.keras.Sequential([\n        tf.keras.layers.GlobalAveragePooling2D(input_shape=(target_image_dim,target_image_dim, 3)),\n        tf.keras.layers.Lambda(lambda x: tf.reduce_mean(x, axis=1, keepdims=True)),\n        tf.keras.layers.Dense(numClasses, activation='softmax')\n    ])\n    model.compile(optimizer=tf.keras.optimizers.SGD(learning_rate=1e-3),\n                  loss='sparse_categorical_crossentropy',metrics=['accuracy'])\n    return model\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_model_LogisticRegression():\n    model = tf.keras.Sequential([\n        tf.keras.layers.Flatten(input_shape=(target_image_dim,target_image_dim, 3)),\n        tf.keras.layers.Dense(numClasses, activation='softmax')\n    ])\n    model.compile(optimizer='adam',\n                 loss='sparse_categorical_crossentropy',\n                 metrics=['accuracy'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_model_FullyConnected():\n    model = tf.keras.Sequential([\n      tf.keras.layers.Flatten(input_shape=(target_image_dim, target_image_dim, 3)),\n      tf.keras.layers.Dense(1024, activation='relu'),\n      tf.keras.layers.Dense(256, activation='relu'),\n      tf.keras.layers.Dense(numClasses, activation='softmax')\n    ])\n    model.compile(optimizer = 'adam',\n                loss='sparse_categorical_crossentropy',#tf.losses.SparseCategoricalCrossentropy(from_logits=True),\n                metrics=['accuracy'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_model_TFIP24():\n    model = tf.keras.models.Sequential([\n        # Note the input shape is the desired size of the image, with 3 bytes color\n        # This is the first convolution\n        tf.keras.layers.Conv2D(64, (3,3), activation='relu', input_shape=(target_image_dim, target_image_dim, 3)),\n        tf.keras.layers.MaxPooling2D(2, 2),\n        # The second convolution\n        tf.keras.layers.Conv2D(64, (3,3), activation='relu'),\n        tf.keras.layers.MaxPooling2D(2,2),\n        # The third convolution\n        tf.keras.layers.Conv2D(128, (3,3), activation='relu'),\n        tf.keras.layers.MaxPooling2D(2,2),\n        # The fourth convolution\n        tf.keras.layers.Conv2D(128, (3,3), activation='relu'),\n        tf.keras.layers.MaxPooling2D(2,2),\n        # Flatten the results to feed into a DNN\n        tf.keras.layers.Flatten(),\n        tf.keras.layers.Dropout(0.5),\n        # 512 neuron hidden layer\n        tf.keras.layers.Dense(512, activation='relu'),\n        tf.keras.layers.Dense(numClasses, activation='softmax')\n    ])\n    model.compile(loss = 'sparse_categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\ndef plotLossAcc(myHistory):\n    acc = myHistory.history['accuracy']\n    val_acc = myHistory.history['val_accuracy']\n    loss = myHistory.history['loss']\n    val_loss = myHistory.history['val_loss']\n\n    epochs = range(len(acc))\n    fig=plt.figure(figsize=(8,6), dpi= 80, facecolor='w', edgecolor='k')\n    plt.plot(epochs, acc, 'ro-', label='Training accuracy')\n    plt.plot(epochs, val_acc, 'bo-', label='Validation accuracy')\n    plt.title('Training and validation accuracy')\n    plt.legend(loc=0)\n    plt.figure()\n    plt.show()\n    fig2=plt.figure(figsize=(8,6), dpi= 80, facecolor='w', edgecolor='k')\n    plt.plot(epochs, loss, 'ro-', label='Training loss')\n    plt.plot(epochs, val_loss, 'bo-', label='Validation loss')\n    plt.title('Training and validation loss')\n    plt.legend(loc=0)\n    plt.figure()\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# define Inception-based transfer learning model here?\n'''\nfrom tensorflow.keras.applications.inception_v3 import InceptionV3\n\n!wget --no-check-certificate \\\n    https://storage.googleapis.com/mledu-datasets/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5 \\\n    -O /tmp/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5\nlocal_weights_file = '/tmp/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5'\npre_trained_model = InceptionV3(input_shape = (target_image_dim, target_image_dim, 3), include_top=False, weights=None)\npre_trained_model.load_weights(local_weights_file)\nfor layer in pre_trained_model.layers:\n    layer.trainable = False\n    '''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#pre_trained_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_20 = make_model_TFIP24()\nmodel_20.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_20 = make_model_TFIP24()\nhist_20 = model_20.fit(non_augmented_train_batches, epochs=50, validation_data=validation_batches)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_20.save(\"cassava_20.h5\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plotLossAcc(hist_20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_21 = make_model_TFIP24()\nhist_21 = model_21.fit(augmented_train_batches, epochs=50, validation_data=validation_batches)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_21.save(\"cassava_21.h5\")\nplotLossAcc(hist_21)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#model_26 = make_model_TFIP24()\n#model_26.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#hist_26 = model_26.fit(augmented_train_batches, epochs=50, validation_data=validation_batches)\n#model_26.save(\"cassava_26.h5\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#plotLossAcc(hist_26)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_00 = make_model_LogisticRegression()\nmodel_00.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"hist_00 = model_00.fit(non_augmented_train_batches, epochs=50, validation_data=validation_batches)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_00.save(\"cassava_00.h5\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plotLossAcc(hist_00)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_01 = make_model_LogisticRegression()\nhist_01 = model_01.fit(augmented_train_batches, epochs=50, validation_data=validation_batches)\nmodel_01.save(\"cassava_01.h5\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plotLossAcc(hist_01)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_0K0 = make_KilledModel()\nmodel_0K0.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# BATCH_SIZE=56\nhist_0K0 = model_0K0.fit(killed_train_batches, epochs=50, validation_data=validation_batches)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_0K0.save(\"cassava_0K0.h5\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plotLossAcc(hist_0K0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_0K = make_KilledModel()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_0K.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_0K2 = make_KilledModel()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_0K2.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_0K2 = make_KilledModel()\nhist_0K2 = model_0K2.fit(killed_train_batches, epochs=50, validation_data=validation_batches)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(89*64)\nprint(num_train_examples)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"5656/8/7","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_without_aug = make_model_TFIP24()\n\nno_aug_history = model_without_aug.fit(non_augmented_train_batches, epochs=10, validation_data=validation_batches)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"65536*3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_with_aug = make_model_TFIP()\n\nyes_aug_history = model_with_aug.fit(augmented_train_batches, epochs=10, validation_data=validation_batches)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset, info =  tfds.load('mnist', as_supervised=True, with_info=True)\ntrain_dataset, test_dataset = dataset['train'], dataset['test']\n\nnum_train_examples= info.splits['train'].num_examples","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef convert(image, label):\n    image = tf.image.convert_image_dtype(image,tf.float32) # cast and normalize the image to [0,1]\n    return (image,label)\n\ndef convert_and_resize(image,label):\n    (image,label) = convert(image,label)\n    image = tf.image.resize_with_pad(image, 150, 150)\n    return (image,label)\n    \ndef augment(image,label):\n    (image,label) = convert_and_resize(image,label)\n    image = tf.image.random_crop(image, size=[28, 28, 1]) # Random crop back to 28x28\n    image = tf.image.random_brightness(image, max_delta=0.5) # Random brightness\n    return (image,label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 64\n# Only use a subset of the data so it's easier to overfit, for this tutorial\nNUM_EXAMPLES = 2048","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTOTUNE = tf.data.experimental.AUTOTUNE\n\naugmented_train_batches = (\n    train_dataset\n    # Only train on a subset, so you can quickly see the effect.\n    .take(NUM_EXAMPLES)\n    .cache()\n    .shuffle(num_train_examples//4)\n    # The augmentation is added here.\n    .map(augment, num_parallel_calls=AUTOTUNE)\n    .batch(BATCH_SIZE)\n    .prefetch(AUTOTUNE)\n) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"non_augmented_train_batches = (\n    train_dataset\n    # Only train on a subset, so you can quickly see the effect.\n    .take(NUM_EXAMPLES)\n    .cache()\n    .shuffle(num_train_examples//4)\n    # No augmentation.\n    .map(convert, num_parallel_calls=AUTOTUNE)\n    .batch(BATCH_SIZE)\n    .prefetch(AUTOTUNE)\n) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"validation_batches = (\n    test_dataset\n    .map(convert, num_parallel_calls=AUTOTUNE)\n    .batch(2*BATCH_SIZE)\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_model():\n    model = tf.keras.Sequential([\n      tf.keras.layers.Flatten(input_shape=(28, 28, 1)),\n      tf.keras.layers.Dense(4096, activation='relu'),\n      tf.keras.layers.Dense(4096, activation='relu'),\n      tf.keras.layers.Dense(10)\n    ])\n    model.compile(optimizer = 'adam',\n                loss=tf.losses.SparseCategoricalCrossentropy(from_logits=True),\n                metrics=['accuracy'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_without_aug = make_model()\n\nno_aug_history = model_without_aug.fit(non_augmented_train_batches, epochs=5, validation_data=validation_batches)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_with_aug = make_model()\n\naug_history = model_with_aug.fit(augmented_train_batches, epochs=5, validation_data=validation_batches)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}