{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install py7zr\n","metadata":{"execution":{"iopub.status.busy":"2023-10-16T12:29:18.753169Z","iopub.execute_input":"2023-10-16T12:29:18.753673Z","iopub.status.idle":"2023-10-16T12:29:33.409319Z","shell.execute_reply.started":"2023-10-16T12:29:18.753638Z","shell.execute_reply":"2023-10-16T12:29:33.408217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import py7zr\nimport os\n\n# Define the paths to the .7z files\ntrain_7z_path = '/kaggle/input/cifar-10/train.7z'\ntest_7z_path = '/kaggle/input/cifar-10/test.7z'\n\n# Define the extraction paths\nextraction_path = '/kaggle/working/'\n\n# Unzip the train dataset\nwith py7zr.SevenZipFile(train_7z_path, mode='r') as z:\n    z.extractall(path=extraction_path)\n\n# Unzip the test dataset\nwith py7zr.SevenZipFile(test_7z_path, mode='r') as z:\n    z.extractall(path=extraction_path)","metadata":{"execution":{"iopub.status.busy":"2023-10-16T12:30:50.670621Z","iopub.execute_input":"2023-10-16T12:30:50.671051Z","iopub.status.idle":"2023-10-16T12:44:46.887083Z","shell.execute_reply.started":"2023-10-16T12:30:50.67101Z","shell.execute_reply":"2023-10-16T12:44:46.88609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nimport tensorflow_hub as hub\n\nimport tensorflow_datasets as tfds\n\nimport time\n\nfrom PIL import Image\nimport requests\nfrom io import BytesIO\n\nimport matplotlib.pyplot as plt\nimport numpy as np\n\nimport os\nimport pathlib","metadata":{"execution":{"iopub.status.busy":"2023-10-17T09:26:54.797683Z","iopub.execute_input":"2023-10-17T09:26:54.798464Z","iopub.status.idle":"2023-10-17T09:27:11.523915Z","shell.execute_reply.started":"2023-10-17T09:26:54.798406Z","shell.execute_reply":"2023-10-17T09:27:11.522968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf_cifar10_labels = ['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck']","metadata":{"execution":{"iopub.status.busy":"2023-10-17T09:29:42.096346Z","iopub.execute_input":"2023-10-17T09:29:42.097265Z","iopub.status.idle":"2023-10-17T09:29:42.101857Z","shell.execute_reply.started":"2023-10-17T09:29:42.097233Z","shell.execute_reply":"2023-10-17T09:29:42.100954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load model into KerasLayer\nmodel_url = \"https://tfhub.dev/google/bit/m-r50x1/1\"\nmodule = hub.KerasLayer(model_url)\ndef preprocess_image(image):\n  image = np.array(image)\n  # reshape into shape [batch_size, height, width, num_channels]\n  img_reshaped = tf.reshape(image, [1, image.shape[0], image.shape[1], image.shape[2]])\n  # Use `convert_image_dtype` to convert to floats in the [0,1] range.\n  image = tf.image.convert_image_dtype(img_reshaped, tf.float32)  \n  return image","metadata":{"execution":{"iopub.status.busy":"2023-10-17T09:29:49.246104Z","iopub.execute_input":"2023-10-17T09:29:49.246433Z","iopub.status.idle":"2023-10-17T09:30:09.211691Z","shell.execute_reply.started":"2023-10-17T09:29:49.246406Z","shell.execute_reply":"2023-10-17T09:30:09.210738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_name = 'cifar10'\nds, info = tfds.load(name=dataset_name, split=['train'], with_info=True)\nds = ds[0]\nnum_examples = info.splits['train'].num_examples\nNUM_CLASSES = 10","metadata":{"execution":{"iopub.status.busy":"2023-10-17T09:30:14.657095Z","iopub.execute_input":"2023-10-17T09:30:14.657431Z","iopub.status.idle":"2023-10-17T09:31:13.649374Z","shell.execute_reply.started":"2023-10-17T09:30:14.657403Z","shell.execute_reply":"2023-10-17T09:31:13.648432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_split = 0.9\nnum_train = int(train_split * num_examples)\nds_train = ds.take(num_train)\nds_test = ds.skip(num_train)\n\nDATASET_NUM_TRAIN_EXAMPLES = num_examples","metadata":{"execution":{"iopub.status.busy":"2023-10-17T09:31:35.16636Z","iopub.execute_input":"2023-10-17T09:31:35.166734Z","iopub.status.idle":"2023-10-17T09:31:35.184977Z","shell.execute_reply.started":"2023-10-17T09:31:35.166705Z","shell.execute_reply":"2023-10-17T09:31:35.183929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Detect GPUs, if available\ngpus = tf.config.experimental.list_physical_devices('GPU')\nif gpus:\n    try:\n        # Currently, memory growth needs to be the same across GPUs\n        for gpu in gpus:\n            tf.config.experimental.set_memory_growth(gpu, True)\n        logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n        print(len(gpus), \"Physical GPUs,\", len(logical_gpus), \"Logical GPUs\")\n    except RuntimeError as e:\n        print(e)\n# Define the MirroredStrategy for two GPUs\nstrategy = tf.distribute.MirroredStrategy(devices=['/GPU:0', '/GPU:1'])\n\nprint(\"Number of devices: {}\".format(strategy.num_replicas_in_sync))","metadata":{"execution":{"iopub.status.busy":"2023-10-17T09:31:38.377082Z","iopub.execute_input":"2023-10-17T09:31:38.377431Z","iopub.status.idle":"2023-10-17T09:31:38.754032Z","shell.execute_reply.started":"2023-10-17T09:31:38.377401Z","shell.execute_reply":"2023-10-17T09:31:38.753077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Add new head to the BiT model\n\nclass MyBiTModel(tf.keras.Model):\n  \"\"\"BiT with a new head.\"\"\"\n\n  def __init__(self, num_classes, module):\n    super().__init__()\n\n    self.num_classes = num_classes\n    self.head = tf.keras.layers.Dense(num_classes, kernel_initializer='zeros')\n    self.bit_model = module\n  \n  def call(self, images):\n    # No need to cut head off since we are using feature extractor model\n    bit_embedding = self.bit_model(images)\n    return self.head(bit_embedding)","metadata":{"execution":{"iopub.status.busy":"2023-10-17T09:31:41.816701Z","iopub.execute_input":"2023-10-17T09:31:41.817405Z","iopub.status.idle":"2023-10-17T09:31:41.823943Z","shell.execute_reply.started":"2023-10-17T09:31:41.817368Z","shell.execute_reply":"2023-10-17T09:31:41.822691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"RESIZE_TO = 160\nCROP_TO = 128\nSCHEDULE_LENGTH = 10000\nSCHEDULE_BOUNDARIES = [3000, 6000, 9000]","metadata":{"execution":{"iopub.status.busy":"2023-10-17T09:31:44.823909Z","iopub.execute_input":"2023-10-17T09:31:44.824229Z","iopub.status.idle":"2023-10-17T09:31:44.828548Z","shell.execute_reply.started":"2023-10-17T09:31:44.824203Z","shell.execute_reply":"2023-10-17T09:31:44.827574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create data pipelines for training and testing:\nBATCH_SIZE = 1024\nSCHEDULE_LENGTH = SCHEDULE_LENGTH * 512 / BATCH_SIZE\n\nSTEPS_PER_EPOCH = 10\n\ndef cast_to_tuple(features):\n  return (features['image'], features['label'])\n  \ndef preprocess_train(features):\n  # Apply random crops and horizontal flips for all tasks \n  # except those for which cropping or flipping destroys the label semantics\n  # (e.g. predict orientation of an object)\n  features['image'] = tf.image.random_flip_left_right(features['image'])\n  features['image'] = tf.image.resize(features['image'], [RESIZE_TO, RESIZE_TO])\n  features['image'] = tf.image.random_crop(features['image'], [CROP_TO, CROP_TO, 3])\n  features['image'] = tf.cast(features['image'], tf.float32) / 255.0\n  return features\n\ndef preprocess_test(features):\n  features['image'] = tf.image.resize(features['image'], [RESIZE_TO, RESIZE_TO])\n  features['image'] = tf.cast(features['image'], tf.float32) / 255.0\n  return features","metadata":{"execution":{"iopub.status.busy":"2023-10-17T09:31:51.037916Z","iopub.execute_input":"2023-10-17T09:31:51.038229Z","iopub.status.idle":"2023-10-17T09:31:51.04476Z","shell.execute_reply.started":"2023-10-17T09:31:51.038202Z","shell.execute_reply":"2023-10-17T09:31:51.043921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pipeline_train = (ds_train\n                  .shuffle(10000)\n                  .repeat(int(SCHEDULE_LENGTH * BATCH_SIZE / DATASET_NUM_TRAIN_EXAMPLES * STEPS_PER_EPOCH) + 1 + 50)  # repeat dataset_size / num_steps\n                  .map(preprocess_train, num_parallel_calls=8)\n                  .batch(BATCH_SIZE)\n                  .map(cast_to_tuple)  # for keras model.fit\n                  .prefetch(2))\n\npipeline_test = (ds_test.map(preprocess_test, num_parallel_calls=1)\n                  .map(cast_to_tuple)  # for keras model.fit\n                  .batch(BATCH_SIZE)\n                  .prefetch(2))","metadata":{"execution":{"iopub.status.busy":"2023-10-17T09:31:55.235551Z","iopub.execute_input":"2023-10-17T09:31:55.235881Z","iopub.status.idle":"2023-10-17T09:31:55.411036Z","shell.execute_reply.started":"2023-10-17T09:31:55.235846Z","shell.execute_reply":"2023-10-17T09:31:55.410127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.callbacks import Callback\n\n# Define a custom callback to stop training when accuracy exceeds the threshold\nclass CustomEarlyStopping(Callback):\n    def __init__(self, monitor='accuracy', value=0.945, verbose=1):\n        super(CustomEarlyStopping, self).__init__()\n        self.monitor = monitor\n        self.value = value\n        self.verbose = verbose\n\n    def on_epoch_end(self, epoch, logs=None):\n        current_accuracy = logs.get(self.monitor)\n        if current_accuracy is None:\n            raise ValueError(f\"Metric '{self.monitor}' is not available. Available metrics: {','.join(logs.keys())}\")\n\n        if current_accuracy >= self.value:\n            if self.verbose > 0:\n                print(f\"\\nReached {self.monitor} >= {self.value}, stopping training.\")\n            self.model.stop_training = True","metadata":{"execution":{"iopub.status.busy":"2023-10-16T22:59:24.464163Z","iopub.execute_input":"2023-10-16T22:59:24.464487Z","iopub.status.idle":"2023-10-16T22:59:24.472079Z","shell.execute_reply.started":"2023-10-16T22:59:24.464461Z","shell.execute_reply":"2023-10-16T22:59:24.470992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    model = MyBiTModel(num_classes=NUM_CLASSES, module=module)\n    \n    lr = 0.001 * BATCH_SIZE / 512 \n\n    # Decay learning rate by a factor of 10 at SCHEDULE_BOUNDARIES.\n    lr_schedule = tf.keras.optimizers.schedules.PiecewiseConstantDecay(boundaries=SCHEDULE_BOUNDARIES, \n                                                                       values=[lr, lr*0.1, lr*0.001, lr*0.0001])\n    optimizer = tf.keras.optimizers.Adam(learning_rate=lr_schedule)\n\n    loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n    model.compile(optimizer=optimizer,\n              loss=loss_fn,\n              metrics=['accuracy'])\n# Fine-tune model\nhistory = model.fit(\n    pipeline_train,\n    batch_size=BATCH_SIZE,\n    steps_per_epoch=STEPS_PER_EPOCH,\n    epochs= 35,  # TODO: replace with `epochs=10` here to shorten fine-tuning for tutorial if you wish\n    validation_data=pipeline_test, # here we are only using \n    # this data to evaluate our performance\n)","metadata":{"execution":{"iopub.status.busy":"2023-10-17T09:50:20.522159Z","iopub.execute_input":"2023-10-17T09:50:20.522533Z","iopub.status.idle":"2023-10-17T10:06:50.877767Z","shell.execute_reply.started":"2023-10-17T09:50:20.522504Z","shell.execute_reply":"2023-10-17T10:06:50.876828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# z = [str(i+1)+'.png' for i in range(300000)]\ndef decode_img(img):\n  # convert the compressed string to a 3D uint8 tensor\n  img = tf.image.decode_png(img, channels=3)\n  return img  \n\ndef process_path(file_path):\n  # load the raw data from the file as a string\n  img = tf.io.read_file(file_path)\n  img = decode_img(img)\n  features = {'image': img}\n  return features","metadata":{"execution":{"iopub.status.busy":"2023-10-17T10:07:01.430585Z","iopub.execute_input":"2023-10-17T10:07:01.430947Z","iopub.status.idle":"2023-10-17T10:07:01.437394Z","shell.execute_reply.started":"2023-10-17T10:07:01.430918Z","shell.execute_reply":"2023-10-17T10:07:01.436394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_ds = tf.data.Dataset.list_files('/kaggle/working/test/*', shuffle = False)\ntest_ds = list_ds.map(process_path, num_parallel_calls=tf.data.experimental.AUTOTUNE)\n# Create a list of image IDs (without the file extension)","metadata":{"execution":{"iopub.status.busy":"2023-10-17T10:07:04.530814Z","iopub.execute_input":"2023-10-17T10:07:04.531149Z","iopub.status.idle":"2023-10-17T10:07:07.52876Z","shell.execute_reply.started":"2023-10-17T10:07:04.531123Z","shell.execute_reply":"2023-10-17T10:07:07.527806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process_path_(file_path):\n  label = tf.strings.split(file_path, os.sep)[-1]\n  return label\nt = list_ds.map(process_path_)\nIDs = []\nfor f in t.take(300000):\n    IDs.append(int(f.numpy()[:-4]))","metadata":{"execution":{"iopub.status.busy":"2023-10-17T10:07:10.127066Z","iopub.execute_input":"2023-10-17T10:07:10.128517Z","iopub.status.idle":"2023-10-17T10:08:14.357168Z","shell.execute_reply.started":"2023-10-17T10:07:10.12847Z","shell.execute_reply":"2023-10-17T10:08:14.356132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def cast_to_tuple_(features):\n    return (features['image'])","metadata":{"execution":{"iopub.status.busy":"2023-10-17T10:08:17.847282Z","iopub.execute_input":"2023-10-17T10:08:17.847946Z","iopub.status.idle":"2023-10-17T10:08:17.85648Z","shell.execute_reply.started":"2023-10-17T10:08:17.847905Z","shell.execute_reply":"2023-10-17T10:08:17.855452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = (test_ds.map(preprocess_test, num_parallel_calls=1)\n                  .map(cast_to_tuple_)  # for keras model.fit\n                  .batch(BATCH_SIZE)\n                  .prefetch(2))","metadata":{"execution":{"iopub.status.busy":"2023-10-17T10:08:25.044069Z","iopub.execute_input":"2023-10-17T10:08:25.044391Z","iopub.status.idle":"2023-10-17T10:08:25.09621Z","shell.execute_reply.started":"2023-10-17T10:08:25.044364Z","shell.execute_reply":"2023-10-17T10:08:25.09529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict_label = model.predict(x = test)","metadata":{"execution":{"iopub.status.busy":"2023-10-17T10:08:29.815541Z","iopub.execute_input":"2023-10-17T10:08:29.815894Z","iopub.status.idle":"2023-10-17T10:20:07.402481Z","shell.execute_reply.started":"2023-10-17T10:08:29.815867Z","shell.execute_reply":"2023-10-17T10:20:07.401421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nclass_names = ['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck']\nprediction = np.argmax(predict_label, axis=1)\nsubmission = pd.DataFrame(columns=['id', 'label'], dtype=str)\nsubmission['label'] = [class_names[int(i)] for i in prediction]\nsubmission['id'] = IDs\n\nsubmission.sort_values(by=['id'])\nsubmission.to_csv('submission.csv', index=None)","metadata":{"execution":{"iopub.status.busy":"2023-10-17T10:20:13.131115Z","iopub.execute_input":"2023-10-17T10:20:13.131474Z","iopub.status.idle":"2023-10-17T10:20:13.790727Z","shell.execute_reply.started":"2023-10-17T10:20:13.131422Z","shell.execute_reply":"2023-10-17T10:20:13.789619Z"},"trusted":true},"execution_count":null,"outputs":[]}]}