{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install tez\n!pip install timm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-03-25T15:50:49.796055Z","iopub.execute_input":"2022-03-25T15:50:49.796647Z","iopub.status.idle":"2022-03-25T15:51:07.051852Z","shell.execute_reply.started":"2022-03-25T15:50:49.796608Z","shell.execute_reply":"2022-03-25T15:51:07.050958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\nimport albumentations\nimport numpy as np\nimport pandas as pd\nimport timm\nimport torch\nimport torch.nn as nn\nfrom sklearn import metrics, model_selection\n\nfrom tez import Tez, TezConfig\nfrom tez.callbacks import EarlyStopping\nfrom tez.utils import seed_everything","metadata":{"execution":{"iopub.status.busy":"2022-03-25T15:51:22.163721Z","iopub.execute_input":"2022-03-25T15:51:22.164007Z","iopub.status.idle":"2022-03-25T15:51:25.151483Z","shell.execute_reply.started":"2022-03-25T15:51:22.163978Z","shell.execute_reply":"2022-03-25T15:51:25.150422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class args:\n    input = \"../input/digit-recognizer/\"\n    model_name = \"resnet50\"\n    learning_rate = 1e-2\n    batch_size = 64\n    epochs = 25\n    output = \".\"\n    accumulation_steps = 1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DigitRecognizerDataset:\n    def __init__(self, df, augmentations):\n        self.df = df\n        self.targets = df.label.values\n        self.df = self.df.drop(columns=[\"label\"])\n        self.augmentations = augmentations\n\n        self.images = self.df.to_numpy(dtype=np.float32).reshape((-1, 28, 28))\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, item):\n        targets = self.targets[item]\n        image = self.images[item]\n        image = np.expand_dims(image, axis=0)\n\n        return {\n            \"image\": torch.tensor(image, dtype=torch.float),\n            \"targets\": torch.tensor(targets, dtype=torch.long),\n        }","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DigitRecognizerModel(nn.Module):\n    def __init__(self, model_name, num_classes, learning_rate, n_train_steps):\n        super().__init__()\n\n        self.learning_rate = learning_rate\n        self.n_train_steps = n_train_steps\n        self.model = timm.create_model(\n            model_name,\n            pretrained=True,\n            in_chans=1,\n            num_classes=num_classes,\n        )\n\n    def monitor_metrics(self, outputs, targets):\n        device = targets.get_device()\n        outputs = np.argmax(outputs.cpu().detach().numpy(), axis=1)\n        targets = targets.cpu().detach().numpy()\n        acc = metrics.accuracy_score(targets, outputs)\n        acc = torch.tensor(acc, device=device)\n        return {\"accuracy\": acc}\n\n    def optimizer_scheduler(self):\n        opt = torch.optim.SGD(\n            self.parameters(),\n            lr=self.learning_rate,\n            momentum=0.9,\n        )\n        sch = torch.optim.lr_scheduler.ReduceLROnPlateau(\n            opt,\n            factor=0.5,\n            patience=2,\n            verbose=True,\n            mode=\"max\",\n            threshold=1e-4,\n        )\n        return opt, sch\n\n    def forward(self, image, targets=None):\n        x = self.model(image)\n        if targets is not None:\n            loss = nn.CrossEntropyLoss()(x, targets)\n            metrics = self.monitor_metrics(x, targets)\n            return x, loss, metrics\n        return x, 0, {}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seed_everything(42)\n\ndf = pd.read_csv(os.path.join(args.input, \"train.csv\"))\ntest_df = pd.read_csv(os.path.join(args.input, \"test.csv\"))\ntest_df.loc[:, \"label\"] = 0  # Fake label\n\ntrain_aug = albumentations.Compose(\n    [\n        albumentations.Normalize(\n            mean=[0.485, 0.456, 0.406],\n            std=[0.229, 0.224, 0.225],\n            max_pixel_value=255.0,\n            p=1.0,\n        ),\n    ],\n    p=1.0,\n)\n\nvalid_aug = albumentations.Compose(\n    [\n        albumentations.Normalize(\n            mean=[0.485, 0.456, 0.406],\n            std=[0.229, 0.224, 0.225],\n            max_pixel_value=255.0,\n            p=1.0,\n        ),\n    ],\n    p=1.0,\n)\n\ntrain_df, valid_df = model_selection.train_test_split(\n    df,\n    test_size=0.2,\n    random_state=42,\n    stratify=df[\"label\"].values,\n)\n\ntrain_dataset = DigitRecognizerDataset(\n    df=train_df,\n    augmentations=train_aug,\n)\nvalid_dataset = DigitRecognizerDataset(\n    df=valid_df,\n    augmentations=valid_aug,\n)\ntest_dataset = DigitRecognizerDataset(\n    df=test_df,\n    augmentations=valid_aug,\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_train_steps = int(len(train_dataset) / args.batch_size / args.accumulation_steps * args.epochs)\nmodel = DigitRecognizerModel(\n    model_name=args.model_name,\n    num_classes=df.label.nunique(),\n    learning_rate=args.learning_rate,\n    n_train_steps=n_train_steps,\n)\n\nmodel = Tez(model)\nconfig = TezConfig(\n    training_batch_size=args.batch_size,\n    validation_batch_size=2 * args.batch_size,\n    test_batch_size=2 * args.batch_size,\n    gradient_accumulation_steps=args.accumulation_steps,\n    epochs=args.epochs,\n    step_scheduler_after=\"epoch\",\n    step_scheduler_metric=\"valid_accuracy\",\n    fp16=True,\n)\n\nes = EarlyStopping(\n    monitor=\"valid_accuracy\",\n    model_path=os.path.join(args.output, \"model.bin\"),\n    patience=10,\n    mode=\"max\",\n    save_weights_only=True,\n)\n\nmodel.fit(\n    train_dataset,\n    valid_dataset=valid_dataset,\n    callbacks=[es],\n    config=config,\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.load(os.path.join(args.output, \"model.bin\"), weights_only=True)\n\npreds_iter = model.predict(test_dataset)\nfinal_preds = []\nfor preds in preds_iter:\n    final_preds.append(preds)\nfinal_preds = np.vstack(final_preds)\nfinal_preds = np.argmax(final_preds, axis=1)\n\ndf = pd.DataFrame(\n    {\n        \"ImageId\": np.arange(1, len(test_dataset) + 1),\n        \"Label\": final_preds,\n    }\n)\ndf.to_csv(os.path.join(args.output, \"submission.csv\"), index=False)","metadata":{},"execution_count":null,"outputs":[]}]}