{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":30559,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install py7zr\n","metadata":{"execution":{"iopub.status.busy":"2023-12-07T10:38:04.266714Z","iopub.execute_input":"2023-12-07T10:38:04.266958Z","iopub.status.idle":"2023-12-07T10:38:20.324676Z","shell.execute_reply.started":"2023-12-07T10:38:04.266936Z","shell.execute_reply":"2023-12-07T10:38:20.323715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import py7zr\nimport os\n\n# Define the paths to the .7z files\ntrain_7z_path = '/kaggle/input/cifar-10/train.7z'\ntest_7z_path = '/kaggle/input/cifar-10/test.7z'\n\n# Define the extraction paths\nextraction_path = '/kaggle/working/'\n\n# Unzip the train dataset\nwith py7zr.SevenZipFile(train_7z_path, mode='r') as z:\n    z.extractall(path=extraction_path)\n\n# Unzip the test dataset\nwith py7zr.SevenZipFile(test_7z_path, mode='r') as z:\n    z.extractall(path=extraction_path)","metadata":{"execution":{"iopub.status.busy":"2023-12-07T10:38:58.923573Z","iopub.execute_input":"2023-12-07T10:38:58.924468Z","iopub.status.idle":"2023-12-07T10:58:42.769629Z","shell.execute_reply.started":"2023-12-07T10:38:58.924429Z","shell.execute_reply":"2023-12-07T10:58:42.768712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nimport tensorflow_hub as hub\n\nimport tensorflow_datasets as tfds\n\nimport time\n\nfrom PIL import Image\nimport requests\nfrom io import BytesIO\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport os\nimport pathlib","metadata":{"execution":{"iopub.status.busy":"2023-12-07T10:58:55.005124Z","iopub.execute_input":"2023-12-07T10:58:55.005515Z","iopub.status.idle":"2023-12-07T10:59:07.111578Z","shell.execute_reply.started":"2023-12-07T10:58:55.005464Z","shell.execute_reply":"2023-12-07T10:59:07.110751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf_cifar10_labels = ['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck']","metadata":{"execution":{"iopub.status.busy":"2023-12-07T10:59:31.794851Z","iopub.execute_input":"2023-12-07T10:59:31.795460Z","iopub.status.idle":"2023-12-07T10:59:31.800475Z","shell.execute_reply.started":"2023-12-07T10:59:31.795426Z","shell.execute_reply":"2023-12-07T10:59:31.799345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load model into KerasLayer\nmodel_url = \"https://tfhub.dev/google/bit/m-r50x1/1\"\nmodule = hub.KerasLayer(model_url)\ndef preprocess_image(image):\n  image = np.array(image)\n  # reshape into shape [batch_size, height, width, num_channels]\n  img_reshaped = tf.reshape(image, [1, image.shape[0], image.shape[1], image.shape[2]])\n  # Use `convert_image_dtype` to convert to floats in the [0,1] range.\n  image = tf.image.convert_image_dtype(img_reshaped, tf.float32)  \n  return image","metadata":{"execution":{"iopub.status.busy":"2023-12-07T10:59:35.527617Z","iopub.execute_input":"2023-12-07T10:59:35.528314Z","iopub.status.idle":"2023-12-07T10:59:57.037078Z","shell.execute_reply.started":"2023-12-07T10:59:35.528278Z","shell.execute_reply":"2023-12-07T10:59:57.036246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_name = 'cifar10'\nds, info = tfds.load(name=dataset_name, split=['train'], with_info=True)\nds = ds[0]\nnum_examples = info.splits['train'].num_examples\nNUM_CLASSES = 10","metadata":{"execution":{"iopub.status.busy":"2023-12-07T11:00:02.856136Z","iopub.execute_input":"2023-12-07T11:00:02.856498Z","iopub.status.idle":"2023-12-07T11:00:59.444782Z","shell.execute_reply.started":"2023-12-07T11:00:02.856456Z","shell.execute_reply":"2023-12-07T11:00:59.444033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_split = 0.9\nnum_train = int(train_split * num_examples)\nds_train = ds.take(num_train)\nds_test = ds.skip(num_train)\n\nDATASET_NUM_TRAIN_EXAMPLES = num_examples","metadata":{"execution":{"iopub.status.busy":"2023-12-07T11:02:28.587088Z","iopub.execute_input":"2023-12-07T11:02:28.587841Z","iopub.status.idle":"2023-12-07T11:02:28.602716Z","shell.execute_reply.started":"2023-12-07T11:02:28.587804Z","shell.execute_reply":"2023-12-07T11:02:28.601808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Detect GPUs, if available\ngpus = tf.config.experimental.list_physical_devices('GPU')\nif gpus:\n    try:\n        # Currently, memory growth needs to be the same across GPUs\n        for gpu in gpus:\n            tf.config.experimental.set_memory_growth(gpu, True)\n        logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n        print(len(gpus), \"Physical GPUs,\", len(logical_gpus), \"Logical GPUs\")\n    except RuntimeError as e:\n        print(e)\n# Define the MirroredStrategy for two GPUs\nstrategy = tf.distribute.MirroredStrategy(devices=['/GPU:0', '/GPU:1'])\n\nprint(\"Number of devices: {}\".format(strategy.num_replicas_in_sync))","metadata":{"execution":{"iopub.status.busy":"2023-12-07T11:02:31.796751Z","iopub.execute_input":"2023-12-07T11:02:31.797229Z","iopub.status.idle":"2023-12-07T11:02:32.131171Z","shell.execute_reply.started":"2023-12-07T11:02:31.797189Z","shell.execute_reply":"2023-12-07T11:02:32.130206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Add new head to the BiT model\n\nclass MyBiTModel(tf.keras.Model):\n  \"\"\"BiT with a new head.\"\"\"\n\n  def __init__(self, num_classes, module):\n    super().__init__()\n\n    self.num_classes = num_classes\n    self.head = tf.keras.layers.Dense(num_classes, kernel_initializer='zeros')\n    self.bit_model = module\n  \n  def call(self, images):\n    # No need to cut head off since we are using feature extractor model\n    bit_embedding = self.bit_model(images)\n    return self.head(bit_embedding)","metadata":{"execution":{"iopub.status.busy":"2023-12-07T11:02:36.615966Z","iopub.execute_input":"2023-12-07T11:02:36.616476Z","iopub.status.idle":"2023-12-07T11:02:36.622925Z","shell.execute_reply.started":"2023-12-07T11:02:36.616435Z","shell.execute_reply":"2023-12-07T11:02:36.621956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"RESIZE_TO = 160\nCROP_TO = 128\nSCHEDULE_LENGTH = 10000\nSCHEDULE_BOUNDARIES = [3000, 6000, 9000]","metadata":{"execution":{"iopub.status.busy":"2023-12-07T11:02:40.177749Z","iopub.execute_input":"2023-12-07T11:02:40.178482Z","iopub.status.idle":"2023-12-07T11:02:40.182806Z","shell.execute_reply.started":"2023-12-07T11:02:40.178450Z","shell.execute_reply":"2023-12-07T11:02:40.181871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create data pipelines for training and testing:\nBATCH_SIZE = 1024\nSCHEDULE_LENGTH = SCHEDULE_LENGTH * 512 / BATCH_SIZE\n\nSTEPS_PER_EPOCH = 10\n\ndef cast_to_tuple(features):\n  return (features['image'], features['label'])\n  \ndef preprocess_train(features):\n  # Apply random crops and horizontal flips for all tasks \n  # except those for which cropping or flipping destroys the label semantics\n  # (e.g. predict orientation of an object)\n  features['image'] = tf.image.random_flip_left_right(features['image'])\n  features['image'] = tf.image.resize(features['image'], [RESIZE_TO, RESIZE_TO])\n  features['image'] = tf.image.random_crop(features['image'], [CROP_TO, CROP_TO, 3])\n  features['image'] = tf.cast(features['image'], tf.float32) / 255.0\n  return features\n\ndef preprocess_test(features):\n  features['image'] = tf.image.resize(features['image'], [RESIZE_TO, RESIZE_TO])\n  features['image'] = tf.cast(features['image'], tf.float32) / 255.0\n  return features","metadata":{"execution":{"iopub.status.busy":"2023-12-07T11:02:45.232339Z","iopub.execute_input":"2023-12-07T11:02:45.233305Z","iopub.status.idle":"2023-12-07T11:02:45.240623Z","shell.execute_reply.started":"2023-12-07T11:02:45.233270Z","shell.execute_reply":"2023-12-07T11:02:45.239633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pipeline_train = (ds_train\n                  .shuffle(10000)\n                  .repeat(int(SCHEDULE_LENGTH * BATCH_SIZE / DATASET_NUM_TRAIN_EXAMPLES * STEPS_PER_EPOCH) + 1 + 50)  # repeat dataset_size / num_steps\n                  .map(preprocess_train, num_parallel_calls=8)\n                  .batch(BATCH_SIZE)\n                  .map(cast_to_tuple)  # for keras model.fit\n                  .prefetch(2))\n\npipeline_test = (ds_test.map(preprocess_test, num_parallel_calls=1)\n                  .map(cast_to_tuple)  # for keras model.fit\n                  .batch(BATCH_SIZE)\n                  .prefetch(2))","metadata":{"execution":{"iopub.status.busy":"2023-12-07T11:02:47.907011Z","iopub.execute_input":"2023-12-07T11:02:47.907694Z","iopub.status.idle":"2023-12-07T11:02:48.149096Z","shell.execute_reply.started":"2023-12-07T11:02:47.907660Z","shell.execute_reply":"2023-12-07T11:02:48.148049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.callbacks import Callback\n\n# Define a custom callback to stop training when accuracy exceeds the threshold\nclass CustomEarlyStopping(Callback):\n    def __init__(self, monitor='accuracy', value=0.945, verbose=1):\n        super(CustomEarlyStopping, self).__init__()\n        self.monitor = monitor\n        self.value = value\n        self.verbose = verbose\n\n    def on_epoch_end(self, epoch, logs=None):\n        current_accuracy = logs.get(self.monitor)\n        if current_accuracy is None:\n            raise ValueError(f\"Metric '{self.monitor}' is not available. Available metrics: {','.join(logs.keys())}\")\n\n        if current_accuracy >= self.value:\n            if self.verbose > 0:\n                print(f\"\\nReached {self.monitor} >= {self.value}, stopping training.\")\n            self.model.stop_training = True","metadata":{"execution":{"iopub.status.busy":"2023-12-07T11:02:59.736753Z","iopub.execute_input":"2023-12-07T11:02:59.737112Z","iopub.status.idle":"2023-12-07T11:02:59.745388Z","shell.execute_reply.started":"2023-12-07T11:02:59.737080Z","shell.execute_reply":"2023-12-07T11:02:59.744480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    model = MyBiTModel(num_classes=NUM_CLASSES, module=module)\n    \n    lr = 0.001 * BATCH_SIZE / 512 \n\n    # Decay learning rate by a factor of 10 at SCHEDULE_BOUNDARIES.\n    lr_schedule = tf.keras.optimizers.schedules.PiecewiseConstantDecay(boundaries=SCHEDULE_BOUNDARIES, \n                                                                       values=[lr, lr*0.1, lr*0.001, lr*0.0001])\n    optimizer = tf.keras.optimizers.Adam(learning_rate=lr_schedule)\n\n    loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n    model.compile(optimizer=optimizer,\n              loss=loss_fn,\n              metrics=['accuracy'])\n# Fine-tune model\nhistory = model.fit(\n    pipeline_train,\n    batch_size=BATCH_SIZE,\n    steps_per_epoch=STEPS_PER_EPOCH,\n    epochs= 35,  # TODO: replace with `epochs=10` here to shorten fine-tuning for tutorial if you wish\n    validation_data=pipeline_test, # here we are only using \n    # this data to evaluate our performance\n)","metadata":{"execution":{"iopub.status.busy":"2023-12-07T11:03:15.739579Z","iopub.execute_input":"2023-12-07T11:03:15.740024Z","iopub.status.idle":"2023-12-07T11:24:12.654166Z","shell.execute_reply.started":"2023-12-07T11:03:15.739984Z","shell.execute_reply":"2023-12-07T11:24:12.653224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# z = [str(i+1)+'.png' for i in range(300000)]\ndef decode_img(img):\n  # convert the compressed string to a 3D uint8 tensor\n  img = tf.image.decode_png(img, channels=3)\n  return img  \n\ndef process_path(file_path):\n  # load the raw data from the file as a string\n  img = tf.io.read_file(file_path)\n  img = decode_img(img)\n  features = {'image': img}\n  return features","metadata":{"execution":{"iopub.status.busy":"2023-12-07T11:24:18.024089Z","iopub.execute_input":"2023-12-07T11:24:18.024966Z","iopub.status.idle":"2023-12-07T11:24:18.030306Z","shell.execute_reply.started":"2023-12-07T11:24:18.024932Z","shell.execute_reply":"2023-12-07T11:24:18.029328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_ds = tf.data.Dataset.list_files('/kaggle/working/test/*', shuffle = False)\ntest_ds = list_ds.map(process_path, num_parallel_calls=tf.data.experimental.AUTOTUNE)\n# Create a list of image IDs (without the file extension)","metadata":{"execution":{"iopub.status.busy":"2023-12-07T11:24:21.648980Z","iopub.execute_input":"2023-12-07T11:24:21.649440Z","iopub.status.idle":"2023-12-07T11:24:22.979292Z","shell.execute_reply.started":"2023-12-07T11:24:21.649402Z","shell.execute_reply":"2023-12-07T11:24:22.978536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process_path_(file_path):\n  label = tf.strings.split(file_path, os.sep)[-1]\n  return label\nt = list_ds.map(process_path_)\nIDs = []\nfor f in t.take(300000):\n    IDs.append(int(f.numpy()[:-4]))","metadata":{"execution":{"iopub.status.busy":"2023-12-07T11:24:26.023268Z","iopub.execute_input":"2023-12-07T11:24:26.024124Z","iopub.status.idle":"2023-12-07T11:25:22.250509Z","shell.execute_reply.started":"2023-12-07T11:24:26.024088Z","shell.execute_reply":"2023-12-07T11:25:22.249422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def cast_to_tuple_(features):\n    return (features['image'])","metadata":{"execution":{"iopub.status.busy":"2023-12-07T11:25:27.952462Z","iopub.execute_input":"2023-12-07T11:25:27.953303Z","iopub.status.idle":"2023-12-07T11:25:27.957467Z","shell.execute_reply.started":"2023-12-07T11:25:27.953269Z","shell.execute_reply":"2023-12-07T11:25:27.956521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = (test_ds.map(preprocess_test, num_parallel_calls=1)\n                  .map(cast_to_tuple_)  # for keras model.fit\n                  .batch(BATCH_SIZE)\n                  .prefetch(2))","metadata":{"execution":{"iopub.status.busy":"2023-12-07T11:25:30.069377Z","iopub.execute_input":"2023-12-07T11:25:30.070091Z","iopub.status.idle":"2023-12-07T11:25:30.126550Z","shell.execute_reply.started":"2023-12-07T11:25:30.070060Z","shell.execute_reply":"2023-12-07T11:25:30.125598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict_label = model.predict(x = test)","metadata":{"execution":{"iopub.status.busy":"2023-12-07T11:25:32.642294Z","iopub.execute_input":"2023-12-07T11:25:32.643225Z","iopub.status.idle":"2023-12-07T11:41:42.842398Z","shell.execute_reply.started":"2023-12-07T11:25:32.643186Z","shell.execute_reply":"2023-12-07T11:41:42.841393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nclass_names = ['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck']\nprediction = np.argmax(predict_label, axis=1)\nsubmission = pd.DataFrame(columns=['id', 'label'], dtype=str)\nsubmission['label'] = [class_names[int(i)] for i in prediction]\nsubmission['id'] = IDs\n\nsubmission.sort_values(by=['id'])\nsubmission.to_csv('submission.csv', index=None)","metadata":{"execution":{"iopub.status.busy":"2023-12-07T11:42:25.290412Z","iopub.execute_input":"2023-12-07T11:42:25.291114Z","iopub.status.idle":"2023-12-07T11:42:26.193262Z","shell.execute_reply.started":"2023-12-07T11:42:25.291079Z","shell.execute_reply":"2023-12-07T11:42:26.192264Z"},"trusted":true},"execution_count":null,"outputs":[]}]}