{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install ../input/pytsfile/pyts-0.12.0-py3-none-any.whl","metadata":{"_uuid":"452f586e-93f8-429b-8659-7ea347076206","_cell_guid":"e5c4cb71-d1b0-4168-8d53-b7a1effe9b6e","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-01-28T03:11:25.671743Z","iopub.execute_input":"2022-01-28T03:11:25.67202Z","iopub.status.idle":"2022-01-28T03:11:33.766643Z","shell.execute_reply.started":"2022-01-28T03:11:25.671982Z","shell.execute_reply":"2022-01-28T03:11:33.765822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torchvision\nimport os\nimport requests\nimport torch\ndef create_pretrained_resnet():\n    #model = torchvision.models.resnet50(pretrained=False)\n    #return model\n    \n    checkpoint = torch.load('../input/file-with-resnet/resnet50-19c8e357.pth')\n    model = torchvision.models.resnet50(pretrained=False)\n    model.load_state_dict(checkpoint)\n    return model\n#os.environ['TORCH_HOME'] = '../input/file-with-resnet/hub'\n#model = torchvision.models.resnet50(pretrained=True)","metadata":{"execution":{"iopub.status.busy":"2022-01-28T03:19:05.495012Z","iopub.execute_input":"2022-01-28T03:19:05.495277Z","iopub.status.idle":"2022-01-28T03:19:06.140015Z","shell.execute_reply.started":"2022-01-28T03:19:05.495246Z","shell.execute_reply":"2022-01-28T03:19:06.139036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nimport pandas as pd\nimport numpy as np\nimport random\nimport torch\nTRAIN_ROWS = 10000\nDEVICE = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nBATCH_SIZE = 32\nIMAGE_SIZE=75\n\npath='/kaggle/input/g-research-crypto-forecasting/supplemental_train.csv'\n# path='/kaggle/input/g-research-crypto-forecasting/train.csv'\ndf = pd.read_csv(path, low_memory=False)\n#df = pd.read_csv(\"train.csv\", nrows=10000)\ndf = df.iloc[-TRAIN_ROWS:]\n\n\n    \n\ndf = df.fillna(df.mean())\ndf\n\ndef transform_target(df):\n    new_col_name =\"Target\"\n    df.loc[df[new_col_name] >= 0, new_col_name] = 1\n    df.loc[df[new_col_name] < 0, new_col_name] = 0\n    return df\n    \n    \n\ndef expand_to_ignoring_asset_it_row(df, scaler):\n    Asset_ID = df[\"Asset_ID\"]\n    dm = pd.get_dummies(Asset_ID)\n    df, scaler = standartize_df(df, scaler)\n    \n    columns = [\"Count\", \"Open\", \"High\", \"Low\", \"Close\", \"Volume\", \"VWAP\"]\n    new_columns = []\n    for one in range(14):\n        for column in columns:\n            factor = dm.get(one, 0)\n            #df[\"{}_{}\".format(column, one)] = df[\"{}\".format(column)] * factor\n            \n            _data = df[\"{}\".format(column)] * factor\n            _d = pd.DataFrame(_data, columns=[\"{}_{}\".format(column, one)])\n            new_columns.append(_d)\n    \n    \n            \n    del df[\"Asset_ID\"]\n    del df[\"timestamp\"]\n    for column in columns:\n        del df[\"{}\".format(column)]\n    \n    \n    df = pd.concat([df]+new_columns,axis=1)\n    \n    return df, scaler\n    \n    \n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\ndef add_hyperfeatures_to_df(df):\n    original_columns = list(df.columns)\n    for c in original_columns:\n        df[\"{}_ewm_mean\".format(c)] = df[c].ewm(com=10).mean()\n        df[\"{}_ewm_std\".format(c)] = df[c].ewm(com=10).std()\n        df[\"{}_ewm_cov\".format(c)] = df[c].ewm(com=10).cov()\n\n        windows = {10,30}\n        shifts = [0]\n        for base_size in windows:\n            continue\n            for shift in shifts:\n                df[\"shift_{}_{}_mean_{}\".format(shift, c, base_size)] = (\n                    df[c].shift(shift).rolling(window=base_size).mean()\n                )\n                # df[\"{}_var_{}\".format(c, base_size)] = (\n                #    df[c].rolling(window=base_size).var()\n                # )\n                df[\"shift_{}_{}_skew_{}\".format(shift, c, base_size)] = (\n                    df[c].shift(shift).rolling(window=base_size).skew()\n                )\n                df[\"shift_{}_{}_kurt_{}\".format(shift, c, base_size)] = (\n                    df[c].shift(shift).rolling(window=base_size).kurt()\n                )\n                df[\"shift_{}_{}_std_{}\".format(shift, c, base_size)] = (\n                    df[c].shift(shift).rolling(window=base_size).std()\n                )\n                # df[\"{}_cov_{}\".format(c, base_size)] = df[c].rolling(window=base_size).cov()\n\n    df = df.fillna(df.mean())\n    return df\n\n\ndef standartize_df(df, scaler=None):\n    from sklearn.preprocessing import StandardScaler\n    if scaler:\n        df[:] = scaler.transform(df[:])\n    else:\n        scaler = StandardScaler()\n        df[:] = scaler.fit_transform(df[:])\n    return df, scaler\n    \n\ndef create_dataframe(\n    df, scaler=None, with_hyperf=False\n) -> pd.DataFrame:\n    df, scaler = expand_to_ignoring_asset_it_row(df, scaler)\n    \n    if with_hyperf:\n        base_columns = df.columns\n\n        list_of_dataframes_input = []\n        list_of_dataframes_output = []\n        chunk_size=2000\n        for start in range(0, df.shape[0], chunk_size):\n            df_subset = df.iloc[start:start + chunk_size]\n            list_of_dataframes_input.append(df_subset)\n        df = None\n        for df_subset in list_of_dataframes_input:\n            list_of_dataframes_output.append(add_hyperfeatures_to_df(df_subset))\n\n        df = pd.concat(list_of_dataframes_output)\n\n        df = df.drop(columns=base_columns)\n    \n    df = df.fillna(df.mean())\n    df = df.fillna(0)\n    return df, scaler\n\n\n\n\n\n\n\n\n\n\n\n\n\nimport pandas as pd\nimport numpy as np\nfrom pyts.image import GramianAngularField, MarkovTransitionField\nfrom torch.utils.data import Dataset\nfrom typing import Callable, Tuple\nimport torch\nfrom torch import Tensor, nn, optim\nimport time\nimport copy\nimport cv2\nfrom tqdm import tqdm\nclass InputDataset(Dataset):\n    \n    @staticmethod\n    def transform_to_imagetensor(x):\n        gadf = GramianAngularField(image_size=IMAGE_SIZE, method='difference')\n        x = gadf.fit_transform(x)\n        res = []\n        for img in x:\n            img = np.float32(img)\n            color_channeled_image = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)\n            color_channeled_image = np.moveaxis(color_channeled_image, 2, 0)\n            res.append(color_channeled_image)\n        \n        return np.array(res)\n    \n    def __init__(\n        self, df_x: pd.DataFrame, df_y: pd.DataFrame, device: torch.DeviceObjType, shuffle=None\n    ):\n        assert shuffle is not None\n        self.len = len(df_x)\n        assert self.len == len(df_y)\n        self.device = device\n        if shuffle:\n            idx = np.random.permutation(df_x.index)\n            df_x = df_x.reindex(idx)\n            df_y = df_y.reindex(idx)\n        \n        self.x = df_x\n        self.x = InputDataset.transform_to_imagetensor(self.x)\n        self.y = df_y\n        self.y = self.y.to_numpy()\n\n    \n\n    def __getitem__(self, index: int) -> Tuple[Tensor, Tensor]:\n        x = self.x[index]\n        y = self.y[index]\n\n        x = torch.from_numpy(x)\n        y = torch.from_numpy(np.array([y]))\n        x = x.to(self.device).float()\n        y = y.to(self.device).long()\n        return x, y[0]\n\n    def __len__(self):\n        return self.len\n    \ndef train_model(model, dataloaders, criterion, optimizer, num_epochs=25, is_inception=False):\n    since = time.time()\n\n    val_acc_history = []\n\n    best_model_wts = copy.deepcopy(model.state_dict())\n    best_acc = 0.0\n    \n    losses = []\n    class LossRecord:\n        val_loss = None\n        train_loss = None\n        epoch = None\n    \n    terminate_training = False\n    for epoch in range(num_epochs):\n        if terminate_training:\n            break\n            \n        loss_record = LossRecord()\n        loss_record.epoch = epoch\n        \n        loss_diff = 9999999999\n        \n        # Each epoch has a training and validation phase\n        for phase in ['train', 'val']:\n            if phase == 'train':\n                model.train()  # Set model to training mode\n            else:\n                model.eval()   # Set model to evaluate mode\n\n            running_loss = 0.0\n            running_corrects = 0\n\n            # Iterate over data.\n            for inputs, labels in tqdm(dataloaders[phase]):\n                # zero the parameter gradients\n                optimizer.zero_grad()\n\n                # forward\n                # track history if only in train\n                with torch.set_grad_enabled(phase == 'train'):\n                    # Get model outputs and calculate loss\n                    # Special case for inception because in training it has an auxiliary output. In train\n                    #   mode we calculate the loss by summing the final output and the auxiliary output\n                    #   but in testing we only consider the final output.\n                    if is_inception and phase == 'train':\n                        # From https://discuss.pytorch.org/t/how-to-optimize-inception-model-with-auxiliary-classifiers/7958\n                        outputs, aux_outputs = model(inputs)\n                        loss1 = criterion(outputs, labels)\n                        loss2 = criterion(aux_outputs, labels)\n                        loss = loss1 + 0.4*loss2\n                    else:\n                        outputs = model(inputs)\n                        loss = criterion(outputs, labels)\n\n                    _, preds = torch.max(outputs, 1)\n\n                    # backward + optimize only if in training phase\n                    if phase == 'train':\n                        loss.backward()\n                        optimizer.step()\n\n                # statistics\n                running_loss += loss.item() * inputs.size(0)\n                running_corrects += torch.sum(preds == labels.data)\n\n            epoch_loss = running_loss / len(dataloaders[phase].dataset)\n            epoch_acc = running_corrects.double() / len(dataloaders[phase].dataset)\n\n            #print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc))\n            if phase == 'train':\n                loss_record.train_loss = epoch_loss\n                \n            \n            if phase == 'val':\n                if best_acc < epoch_acc:\n                    best_acc = epoch_acc\n                    best_model_wts = copy.deepcopy(model.state_dict())\n                \n                if False:\n                    terminate_training=True\n            \n\n            \"\"\"\n            # deep copy the model\n            if phase == 'val' and epoch_acc > best_acc and epoch > 0:\n                best_acc = epoch_acc\n                best_model_wts = copy.deepcopy(model.state_dict())\n            if phase == 'val':\n                val_acc_history.append(epoch_acc)\n            \"\"\"\n            \n            \n\n        #print()\n\n    time_elapsed = time.time() - since\n    print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))\n    print('Best val Acc: {:4f}'.format(best_acc))\n\n    # load best model weights\n    model.load_state_dict(best_model_wts)\n    import os \n    try:\n        os.remove(\"best_model\")\n    except:\n        pass\n    torch.save(model.state_dict(), \"best_model\")\n    \n    return model, val_acc_history\n\n\ndf = transform_target(df)\nY = df[\"Target\"]\ndel df[\"Target\"]\n\ndf, scaler = create_dataframe(df)\n\ndf\n\nY.to_numpy()\n\nvalidation_size = int(len(df) / 5)\ndataset_train = InputDataset(df_x=df.iloc[:-validation_size],df_y=Y.iloc[:-validation_size], device=DEVICE, shuffle=True)\ndataset_valid = InputDataset(df_x=df.iloc[-validation_size:],df_y=Y.iloc[-validation_size:], device=DEVICE, shuffle=False)\n\n\ntrain_loader1 = torch.utils.data.DataLoader(\n    dataset=dataset_train, batch_size=BATCH_SIZE, drop_last=True\n)\nvalidation_loader1 = torch.utils.data.DataLoader(\n    dataset=dataset_valid, batch_size=BATCH_SIZE, drop_last=True\n)\n\ndataloaders_dict = {'train':train_loader1, 'val':validation_loader1}\n\nmodel = create_pretrained_resnet()\nmodel.to(DEVICE)\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.SGD(model.parameters(), lr=0.00003, momentum=0.008)\n\n\n\n\nmodel, val_acc = train_model(model, dataloaders_dict, criterion, optimizer, num_epochs=10, is_inception=False)\n\nmodel = create_pretrained_resnet()\nmodel.load_state_dict(torch.load(\"best_model\"))\nmodel.eval()\nmodel.to(DEVICE)\n\ndef count_of_numbers(number, lst):\n    c = 0\n    for i in lst:\n        if i == number:\n            c+=1\n    return c\n\n\ndef predict(model, test_df):\n    dataset_predict = InputDataset(df_x=test_df,df_y=test_df, device=DEVICE, shuffle=False)\n    result_loader = torch.utils.data.DataLoader(\n        dataset=dataset_predict, batch_size=BATCH_SIZE, drop_last=False\n    )\n    res = []\n    for inputs, labels in tqdm(result_loader):\n        with torch.no_grad():\n            outputs = model(inputs)\n            _, preds = torch.max(outputs, 1)\n            numpy_preds = preds.cpu().detach().numpy()\n            python_preds = list(numpy_preds)\n            python_preds_rounded = []\n            for i in python_preds:\n                if i >0:\n                    python_preds_rounded.append(1)\n                else:\n                    python_preds_rounded.append(0)\n            res.extend(python_preds_rounded)\n    \n    \n    return res\n    \n\n\nimport gresearch_crypto\nenv = gresearch_crypto.make_env()\n\n# Training data is in the competition dataset as usual\n\niter_test = env.iter_test()\nfor (test_df, sample_prediction_df) in iter_test:\n    del test_df['row_id']\n    base_columns = test_df.columns\n    # raise Exception(base_columns)\n    test_df, scaler = create_dataframe(test_df, scaler)\n    sample_prediction_df['Target'] = predict(model, test_df)\n    env.predict(sample_prediction_df)\n\nfname = \"best_model\"\nimport os\nif os.path.exists(fname):\n  os.remove(fname)\nelse:\n  print(\"{} file does not exist\".format(fname))\n    \nprint(\"Done 76421497\")\n","metadata":{},"execution_count":null,"outputs":[]}]}