{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# import zipfile\n# import glob\n\n# zip_file = glob.glob('../input/dogs-vs-cats/*.zip')  \n# # print(zip_file)\n\n# def extract_zip(file):\n#     with zipfile.ZipFile(file,\"r\") as zip_ref:\n#         zip_ref.extractall(\"temp\")\n        \n# for files in zip_file:\n#     extract_zip(files)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T19:11:34.633812Z","iopub.execute_input":"2021-08-26T19:11:34.634375Z","iopub.status.idle":"2021-08-26T19:11:34.640441Z","shell.execute_reply.started":"2021-08-26T19:11:34.634250Z","shell.execute_reply":"2021-08-26T19:11:34.639554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install efficientnet_pytorch","metadata":{"execution":{"iopub.status.busy":"2021-08-26T19:11:34.642192Z","iopub.execute_input":"2021-08-26T19:11:34.642747Z","iopub.status.idle":"2021-08-26T19:11:43.559485Z","shell.execute_reply.started":"2021-08-26T19:11:34.642709Z","shell.execute_reply":"2021-08-26T19:11:43.558543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport re\nimport numpy as np\nfrom torch.utils.data import Dataset\nfrom PIL import Image\nimport torch\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nimport torch.nn.functional as F\nfrom torch import nn, optim\nfrom torch.utils.data import DataLoader\nfrom tqdm import tqdm\nfrom efficientnet_pytorch import EfficientNet\n\nimport sklearn\nimport pandas as pd\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import log_loss","metadata":{"execution":{"iopub.status.busy":"2021-08-26T19:37:38.953643Z","iopub.execute_input":"2021-08-26T19:37:38.954037Z","iopub.status.idle":"2021-08-26T19:37:39.320154Z","shell.execute_reply.started":"2021-08-26T19:37:38.953998Z","shell.execute_reply":"2021-08-26T19:37:39.319296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CatDog(Dataset):\n    def __init__(self, root, transform=None):\n        self.images = os.listdir(root)\n        self.images.sort(key=lambda x: int(re.findall(r\"\\d+\", x)[0]))\n        self.root = root\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.images)\n\n    def __getitem__(self, index):\n        file = self.images[index]\n        img = np.array(Image.open(os.path.join(self.root, file)))\n\n        if self.transform is not None:\n            img = self.transform(image=img)[\"image\"]\n\n        if \"dog\" in file:\n            label = 1\n        elif \"cat\" in file:\n            label = 0\n        else:\n            label = -1\n\n        return img, label","metadata":{"execution":{"iopub.status.busy":"2021-08-26T19:11:46.771229Z","iopub.execute_input":"2021-08-26T19:11:46.771549Z","iopub.status.idle":"2021-08-26T19:11:46.780540Z","shell.execute_reply.started":"2021-08-26T19:11:46.771515Z","shell.execute_reply":"2021-08-26T19:11:46.779246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Config ","metadata":{}},{"cell_type":"code","source":"DEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nNUM_WORKERS = 4\nBATCH_SIZE = 5\nPIN_MEMORY = True\nLOAD_MODEL = True\nSAVE_MODEL = True\nCHECKPOINT_FILE = \"b7.pth.tar\"\nWEIGHT_DECAY = 1e-4\nLEARNING_RATE = 1e-4\nNUM_EPOCHS = 1\n\n# Augumentation\nbasic_transform = A.Compose(\n    [\n        A.Resize(height=448, width=448),\n        A.Normalize(\n            mean=[0.485, 0.456, 0.406],\n            std=[0.229, 0.224, 0.225],\n            max_pixel_value=255.0,\n        ),\n        ToTensorV2(),\n    ]\n)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T19:11:46.782165Z","iopub.execute_input":"2021-08-26T19:11:46.782589Z","iopub.status.idle":"2021-08-26T19:11:46.831180Z","shell.execute_reply.started":"2021-08-26T19:11:46.782554Z","shell.execute_reply":"2021-08-26T19:11:46.830225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Đây là một kĩ thuật cơ bản khác trong Transfer Learning là **Feature Extraction**. Mình sẽ extract các feature từ các pretrain model(ở đây là EfficientNetB7), sau đó cho các feature qua một classifier bất kì. \nCách này chắc chắn sẽ nhanh hơn Fine Tuning vì phải không phải train thêm mạng CNN. Tuy nhiên trong nhiều trường hợp kết quả sẽ không bằng được Fine Tuning.\nMọi người cố gắng hiểu ý tưởng của kĩ thuật này để phân biệt với Fine Tuing.","metadata":{}},{"cell_type":"code","source":"def save_feature_vectors(model, loader, output_size=(1, 1), file=\"trainb7\"):\n    model.eval()\n    images, labels = [], []\n\n    for idx, (x, y) in enumerate(tqdm(loader)):\n        x = x.to(DEVICE)\n\n        with torch.no_grad():\n            features = model.extract_features(x)\n            features = F.adaptive_avg_pool2d(features, output_size=output_size)\n        images.append(features.reshape(x.shape[0], -1).detach().cpu().numpy())\n        labels.append(y.numpy())\n\n    np.save(f\"X_{file}.npy\", np.concatenate(images, axis=0))\n    np.save(f\"y_{file}.npy\", np.concatenate(labels, axis=0))\n    model.train()","metadata":{"execution":{"iopub.status.busy":"2021-08-26T19:11:55.793294Z","iopub.execute_input":"2021-08-26T19:11:55.793628Z","iopub.status.idle":"2021-08-26T19:11:55.802051Z","shell.execute_reply.started":"2021-08-26T19:11:55.793597Z","shell.execute_reply":"2021-08-26T19:11:55.800987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = EfficientNet.from_pretrained(\"efficientnet-b7\")\nmodel._fc = nn.Linear(2560, 1)\ntrain_dataset = CatDog(root=\"../input/cats-and-dogs-embedded-data/train/train\", transform = basic_transform)\ntest_dataset = CatDog(root=\"../input/cats-and-dogs-embedded-data/test/test\", transform = basic_transform)\n\ntrain_loader = DataLoader(\n        train_dataset,\n        shuffle=True,\n        batch_size = BATCH_SIZE,\n        num_workers = NUM_WORKERS,\n        pin_memory=True,\n        )\ntest_loader = DataLoader(\n    test_dataset,\n    shuffle=False,\n    batch_size= BATCH_SIZE,\n    num_workers= NUM_WORKERS,\n    )\n\nmodel = model.to(DEVICE)\n\nsave_feature_vectors(model, train_loader, output_size=(1, 1), file=\"train_b7\")\nsave_feature_vectors(model, test_loader, output_size=(1, 1), file=\"test_b7\")","metadata":{"execution":{"iopub.status.busy":"2021-08-26T19:11:58.487432Z","iopub.execute_input":"2021-08-26T19:11:58.487769Z","iopub.status.idle":"2021-08-26T19:34:08.956114Z","shell.execute_reply.started":"2021-08-26T19:11:58.487738Z","shell.execute_reply":"2021-08-26T19:34:08.954822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Sau khi đã có vector các feature thì đưa vào clasifier. Có thể thử với Logistic Regression, SVC, Tree-based(i.e. Random Forest)","metadata":{}},{"cell_type":"code","source":"X = np.load(f'../input/data-features/X_train_b7.npy')\ny = np.load(f'../input/data-features/y_train_b7.npy')\n\n# Split data and train classifier\nprint(f\"Training data shape: {X.shape}, labels shape: {y.shape}\")\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.001, random_state=1337)\nclf = LogisticRegression(max_iter=2000)\nclf.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T19:40:14.890687Z","iopub.execute_input":"2021-08-26T19:40:14.891084Z","iopub.status.idle":"2021-08-26T19:40:22.146063Z","shell.execute_reply.started":"2021-08-26T19:40:14.891039Z","shell.execute_reply":"2021-08-26T19:40:22.145097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Validation\nval_preds= clf.predict_proba(X_val)[:,1]\nprint(f\"On validation set:\")\nprint(f\"Accuracy: {clf.score(X_val, y_val)}\")\nprint(f\"LOG LOSS: {log_loss(y_val, val_preds)} \")\nprint(\"%--------------------------------------------------%\")\n\n# Predict\nprint(\"Getting predictions for test set\")\nX_test = np.load(f'../input/data-features/X_test_b7.npy')\nX_test_preds = clf.predict_proba(X_test)[:,1]\ndf = pd.DataFrame({'id': np.arange(1, 12501), 'label': np.clip(X_test_preds, 0.005, 0.995)})\ndf.to_csv(f\"mysubmission.csv\", index=False)\nprint(\"Done getting predictions!\")","metadata":{"execution":{"iopub.status.busy":"2021-08-26T19:42:14.563360Z","iopub.execute_input":"2021-08-26T19:42:14.563689Z","iopub.status.idle":"2021-08-26T19:42:14.824502Z","shell.execute_reply.started":"2021-08-26T19:42:14.563661Z","shell.execute_reply":"2021-08-26T19:42:14.823118Z"},"trusted":true},"execution_count":null,"outputs":[]}]}