{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -q chitra","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\nfrom glob import glob\nfrom tqdm import tqdm\nfrom PIL import Image\nfrom IPython.display import display\nimport random\n\nimport tensorflow as tf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from chitra.datagenerator import Dataset\nfrom chitra.trainer import create_cnn, Trainer\nfrom chitra.dataloader import Clf\n\nfrom chitra.core import IMAGENET_LABELS","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTOTUNE = tf.data.experimental.AUTOTUNE\n\n@tf.function\ndef rescale(image, label):\n    image = image / 127.5 - 1.0\n    return image, label\n\n\ndef tensor_to_image(tensor):\n    return Image.fromarray(tensor.numpy().astype('uint8'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SPLIT_PCT = 0.9\nCAT_SPLIT = 0.05\nSHUFFLE_SPLIT = True\n\nIMG_SIZE = (224, 224)\nBS = 8","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tf.keras.backend.clear_session()\ntf.config.optimizer.set_jit(True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dir = '/kaggle/input/Imagenette-comp/train'\nlen(glob(f'{train_dir}/*/*'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CLASS_MAP = {'c1d6f6c4': 0, 'c1d6fa84': 1, 'c1d6fc6e': 2, 'c1d6fd90': 3, 'c1d6fe94': 4, 'c1d6ff98': 5, 'c1d70092': 6, 'c1d70196': 7, 'c1d702ae': 8, 'c1d70420': 9}\n\nlabels = [e[0] for e in sorted(CLASS_MAP.items(), key=lambda x: x[1])]\n\n\ntensor_dict = {}\nfor k,v in CLASS_MAP.items():\n    key = tf.constant(k)\n    tensor_dict[key.ref()] = v","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"folders = glob(f'{train_dir}/*');print('total folders:', len(folders))\ntrain_files, val_files = [], []\nfor folder in tqdm(folders):\n    filenames = glob(f'{folder}/*')\n    idx = int(len(filenames) * CAT_SPLIT)\n    if SHUFFLE_SPLIT: random.shuffle(filenames)\n    \n    train_files.extend(filenames[idx:])\n    val_files.extend(filenames[:idx])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train_files), len(val_files)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_files_set = set(val_files)\nfor train_file in train_files:\n    if train_file in val_files_set:\n        print('problem!')\n        raise Exception('duplicate')\n\ndel val_files_set","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"keys = list(CLASS_MAP.keys())\nvals = list(CLASS_MAP.values())\n\nkeys_tensor = keys  #tf.constant(keys)\nvals_tensor = vals  #tf.constant(vals)\n\ntable_init = tf.lookup.KeyValueTensorInitializer(\n                    keys_tensor, vals_tensor)\n\nlookup_class_to_idx = tf.lookup.StaticHashTable(table_init, -1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"@tf.function\ndef encode_label(filename):\n    label = tf.strings.split(filename, '/')[-2]\n    \n    label = lookup_class_to_idx.lookup(\n                    label) if lookup_class_to_idx else label\n    return label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"@tf.function\ndef load_data(filename):\n    image = tf.image.decode_jpeg(tf.io.read_file(filename), channels=3)\n    image = tf.image.resize(image, IMG_SIZE)\n    return image, encode_label(filename)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"@tf.function\ndef rescale(image, label):\n    return (image/127.5-1.0), label\n\n\n@tf.function\ndef augment(image, label):\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_flip_up_down(image)\n    image = tf.image.random_hue(image, 0.1)\n    \n    return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dl = tf.data.Dataset.from_tensor_slices(train_files).shuffle(2048*2).map(load_data, AUTOTUNE).map(augment, AUTOTUNE)\nval_dl = tf.data.Dataset.from_tensor_slices(val_files).map(load_data, AUTOTUNE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(8):\n    image, label = next(iter(train_dl))\n    display(tensor_to_image(image))\n    print(label.numpy())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dl = train_dl.map(rescale, AUTOTUNE).batch(BS).prefetch(AUTOTUNE)\nval_dl = val_dl.map(rescale, AUTOTUNE).batch(BS).prefetch(AUTOTUNE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = create_cnn('xception', 10, 0.3, weights=None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# trainer = Trainer(train_ds, model)\n# trainer.compile2(BS, tf.keras.optimizers.SGD(1, momentum=0.9,nesterov=True), lr_range=(1e-4, 1),\n#                  loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n#                  metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.compile(tf.keras.optimizers.SGD(momentum=0.9, nesterov=True),\n              tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n              metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(train_dl,\n          epochs=50,\n          validation_data = val_dl,\n          callbacks=[tf.keras.callbacks.EarlyStopping(patience=4)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"?tf.keras.callbacks.EarlyStopping","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv('../input/Imagenette-comp/sample_submission.csv');df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_files = tf.data.Dataset.from_tensor_slices(glob('/kaggle/input/Imagenette-comp/test/*'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"@tf.function\ndef load_image(filename):\n    image = tf.image.decode_jpeg(tf.io.read_file(filename), channels=3)\n    image = tf.image.resize(image, (224,224))\n    image = image/127.5 - 1.0\n    return filename, image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"next(iter(test_files.batch(2))).numpy().tolist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dl = test_files.map(load_image, AUTOTUNE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict_classes(images):\n    preds = model.predict(images)\n    preds = tf.math.softmax(preds)\n    classes = tf.math.argmax(preds, axis=1).numpy()\n    return classes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for filename, img in test_dl.batch(4).take(1):\n    print(filename.numpy())\n    classes = predict_classes(img)\n    \n    print(classes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TEST_BS = 32\nsteps = len(glob('/kaggle/input/Imagenette-comp/test/*'))//TEST_BS","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result = []\nresult_files = []\nfor data in tqdm(test_dl.batch(TEST_BS), total=steps):\n    images = data[1]\n    filenames = data[0].numpy().tolist()\n    classes = predict_classes(images)\n    \n    result.extend(classes.tolist())\n    result_files.extend(filenames)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predlist = []\nfor e in result:\n    class_name = labels[e]\n    predlist.append(class_name)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"basename = os.path.basename\ndf['Id'] = list(map(lambda x: basename(x.decode()).split('.')[0] ,result_files))\ndf['Category'] = predlist","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.to_csv('submission.csv', index=False)\ndf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}