{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Extracting the CIFAR images from the 7z files is time consuming, especially the test set images which takes about 18-25 minutes to extract. To save GPU time, I will convert the images into numpy arrays and save them in the `.npz` format. The `npz` files can then be used across multiple notebooks. \n\nThere will be two `.npz` files in the output of this notebook - `train.npz` and `test.npz` containing the training and test images respectively. Each of these files have two keys:\n- `images`: a 4D array of images (batch_size x width x height x color_channels)\n- `ids`: a 1D array containing the ids of the images (indexed from 1). The IDs in the `train.npz` will be needed to map the images to their appropriate labels. ","metadata":{}},{"cell_type":"code","source":"!pip install py7zr","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-09-05T17:44:59.144174Z","iopub.execute_input":"2023-09-05T17:44:59.144639Z","iopub.status.idle":"2023-09-05T17:45:15.408354Z","shell.execute_reply.started":"2023-09-05T17:44:59.144603Z","shell.execute_reply":"2023-09-05T17:45:15.406972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os,shutil\nimport re\nimport numpy as np\n\nfrom PIL import Image\nfrom pathlib import Path\nfrom py7zr import unpack_7zarchive\nfrom typing import Tuple\nshutil.register_unpack_format('7zip', ['.7z'], unpack_7zarchive)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-09-05T17:45:15.411145Z","iopub.execute_input":"2023-09-05T17:45:15.411627Z","iopub.status.idle":"2023-09-05T17:45:15.590106Z","shell.execute_reply.started":"2023-09-05T17:45:15.411582Z","shell.execute_reply":"2023-09-05T17:45:15.588997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_folder = Path('/kaggle/input/cifar-10')\ntmp_folder_loc = Path('/tmp/')","metadata":{"execution":{"iopub.status.busy":"2023-09-05T17:45:15.591710Z","iopub.execute_input":"2023-09-05T17:45:15.592067Z","iopub.status.idle":"2023-09-05T17:45:15.597185Z","shell.execute_reply.started":"2023-09-05T17:45:15.592037Z","shell.execute_reply":"2023-09-05T17:45:15.596175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_data_from_extracted(folder:Path) -> Tuple[np.ndarray]:\n    '''\n    Returns the images and their corresponding ids as arrays\n    '''\n    img_list = []\n    id_list = []\n    \n    for img_file in os.listdir(folder):\n        img_list.append(\n            np.array(Image.open(folder/img_file))\n        )\n        id_list.append(int(re.findall(r'([0-9]+)',img_file)[0]))\n        \n    return np.array(img_list),np.array(id_list)","metadata":{"execution":{"iopub.status.busy":"2023-09-05T17:45:15.599836Z","iopub.execute_input":"2023-09-05T17:45:15.600196Z","iopub.status.idle":"2023-09-05T17:45:15.611482Z","shell.execute_reply.started":"2023-09-05T17:45:15.600167Z","shell.execute_reply":"2023-09-05T17:45:15.610440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time \n# extract the train images\nshutil.unpack_archive(input_folder/'train.7z',tmp_folder_loc,format='7zip')","metadata":{"execution":{"iopub.status.busy":"2023-09-05T17:45:15.613149Z","iopub.execute_input":"2023-09-05T17:45:15.613501Z","iopub.status.idle":"2023-09-05T17:46:12.283446Z","shell.execute_reply.started":"2023-09-05T17:45:15.613470Z","shell.execute_reply":"2023-09-05T17:46:12.282219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# load images in training set and save in npz format\ntrain_images, train_ids = load_data_from_extracted(tmp_folder_loc/'train')\nnp.savez_compressed('train.npz',images=train_images,ids=train_ids)","metadata":{"execution":{"iopub.status.busy":"2023-09-05T17:46:12.285002Z","iopub.execute_input":"2023-09-05T17:46:12.285454Z","iopub.status.idle":"2023-09-05T17:46:34.462213Z","shell.execute_reply.started":"2023-09-05T17:46:12.285412Z","shell.execute_reply":"2023-09-05T17:46:34.461039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time \n# extract the train images\nshutil.unpack_archive(input_folder/'test.7z',tmp_folder_loc,format='7zip')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# load images in test set and save in npz format\ntest_images, test_ids = load_data_from_extracted(tmp_folder_loc/'test')\nnp.savez_compressed('test.npz',images=test_images,ids=test_ids)","metadata":{"execution":{"iopub.status.busy":"2023-09-05T17:04:15.660457Z","iopub.execute_input":"2023-09-05T17:04:15.660823Z","iopub.status.idle":"2023-09-05T17:04:15.925979Z","shell.execute_reply.started":"2023-09-05T17:04:15.660797Z","shell.execute_reply":"2023-09-05T17:04:15.924194Z"},"trusted":true},"execution_count":null,"outputs":[]}],"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}}