{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## [The address of the paper](https://dl.acm.org/doi/10.1145/3394486.3403118)","metadata":{}},{"cell_type":"markdown","source":"# Load and prepare the data","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import RobustScaler\nimport gresearch_crypto\nimport gc\nimport os\nimport traceback\nimport random\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport torch\nfrom scipy.sparse import linalg\nfrom torch.autograd import Variable\nimport torch.optim as optim\nimport torch.nn as nn\nfrom torch.nn import init\nimport numbers\nimport torch.nn.functional as F\nimport argparse\nimport time\nimport datatable as dt\nfrom torch.utils.data import Dataset, DataLoader\nimport torch.autograd as autograd\nfrom sklearn.preprocessing import MinMaxScaler\nimport math\n\nDEBUG = False","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-01T11:39:16.370307Z","iopub.execute_input":"2022-02-01T11:39:16.370643Z","iopub.status.idle":"2022-02-01T11:39:18.728742Z","shell.execute_reply.started":"2022-02-01T11:39:16.370556Z","shell.execute_reply":"2022-02-01T11:39:18.728128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/g-research-crypto-forecasting/train.csv').set_index(\"timestamp\")\nassets = pd.read_csv('../input/g-research-crypto-forecasting/asset_details.csv')\n#for assets sorting \nassets_order = pd.read_csv('../input/g-research-crypto-forecasting/supplemental_train.csv').Asset_ID[:14]\nassets_order = dict((t,i) for i,t in enumerate(assets_order))\n\nif DEBUG:\n    train = train[10000000:]\n\ntrain[['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP','Target']] = \\\ntrain[['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP','Target']].astype(np.float32)\n\ntrain['Target'] = train['Target'].fillna(0)","metadata":{"execution":{"iopub.status.busy":"2022-02-01T11:39:18.730196Z","iopub.execute_input":"2022-02-01T11:39:18.730813Z","iopub.status.idle":"2022-02-01T11:40:38.661291Z","shell.execute_reply.started":"2022-02-01T11:39:18.730777Z","shell.execute_reply":"2022-02-01T11:40:38.660146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"VWAP_max = np.max(train[np.isfinite(train.VWAP)].VWAP)\nVWAP_min = np.min(train[np.isfinite(train.VWAP)].VWAP)\nprint(VWAP_max, \"\\n\", VWAP_min)\n\ntrain['VWAP'] = np.nan_to_num(train.VWAP, posinf=VWAP_max, neginf=VWAP_min)","metadata":{"execution":{"iopub.status.busy":"2022-02-01T11:40:38.663109Z","iopub.execute_input":"2022-02-01T11:40:38.663394Z","iopub.status.idle":"2022-02-01T11:40:41.328135Z","shell.execute_reply.started":"2022-02-01T11:40:38.663362Z","shell.execute_reply":"2022-02-01T11:40:41.327124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = train[['Asset_ID', 'Target']].copy()\n\ntimes = dict((t,i) for i,t in enumerate(df.index.unique()))\ndf['id'] = df.index.map(times)\ndf['id'] = df['id'].astype(str) + '_' + df['Asset_ID'].astype(str)\nids = df.id.copy()\n\ndel df","metadata":{"execution":{"iopub.status.busy":"2022-02-01T11:40:41.329588Z","iopub.execute_input":"2022-02-01T11:40:41.329865Z","iopub.status.idle":"2022-02-01T11:41:49.776707Z","shell.execute_reply.started":"2022-02-01T11:40:41.329833Z","shell.execute_reply":"2022-02-01T11:41:49.775915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scale_features = train.columns.drop(['Asset_ID','Target'])\nRS = RobustScaler()\ntrain[scale_features] = RS.fit_transform(train[scale_features])","metadata":{"execution":{"iopub.status.busy":"2022-02-01T11:41:49.779968Z","iopub.execute_input":"2022-02-01T11:41:49.780498Z","iopub.status.idle":"2022-02-01T11:41:56.034991Z","shell.execute_reply.started":"2022-02-01T11:41:49.780454Z","shell.execute_reply":"2022-02-01T11:41:56.034104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ind = train.index.unique()\n\ndef reindex(df):\n    df = df.reindex(range(ind[0],ind[-1]+60,60),method='nearest')\n    df = df.fillna(method=\"ffill\").fillna(method=\"bfill\")\n    return df\n\ntrain=train.groupby('Asset_ID').apply(reindex).reset_index(0, drop=True).sort_index()\ntrain.shape","metadata":{"execution":{"iopub.status.busy":"2022-02-01T11:41:56.036186Z","iopub.execute_input":"2022-02-01T11:41:56.036437Z","iopub.status.idle":"2022-02-01T11:42:12.441608Z","shell.execute_reply.started":"2022-02-01T11:41:56.036405Z","shell.execute_reply":"2022-02-01T11:42:12.440818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Matching records and marking generated rows as 'non-real'\n\ntrain['group_num'] = train.index.map(times)\ntrain = train.dropna(subset=['group_num'])\ntrain['group_num'] = train['group_num'].astype('int')\n\ntrain['id'] = train['group_num'].astype(str) + '_' + train['Asset_ID'].astype(str)\n\ntrain['is_real'] = train.id.isin(ids)*1\ntrain = train.drop('id', axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-02-01T11:42:12.44275Z","iopub.execute_input":"2022-02-01T11:42:12.443002Z","iopub.status.idle":"2022-02-01T11:43:57.841449Z","shell.execute_reply.started":"2022-02-01T11:42:12.442972Z","shell.execute_reply":"2022-02-01T11:43:57.840376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Features values for 'non-real' rows are set to zeros\n\nfeatures = train.columns.drop(['Asset_ID','group_num','is_real'])\ntrain.loc[train.is_real==0, features]=0.","metadata":{"execution":{"iopub.status.busy":"2022-02-01T11:43:57.842921Z","iopub.execute_input":"2022-02-01T11:43:57.843254Z","iopub.status.idle":"2022-02-01T11:44:01.139319Z","shell.execute_reply.started":"2022-02-01T11:43:57.843212Z","shell.execute_reply":"2022-02-01T11:44:01.138362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['asset_order'] = train.Asset_ID.map(assets_order) \ntrain=train.sort_values(by=['group_num', 'asset_order'])\ntrain.head(20)","metadata":{"execution":{"iopub.status.busy":"2022-02-01T11:44:01.140997Z","iopub.execute_input":"2022-02-01T11:44:01.141363Z","iopub.status.idle":"2022-02-01T11:44:11.274519Z","shell.execute_reply.started":"2022-02-01T11:44:01.141315Z","shell.execute_reply":"2022-02-01T11:44:11.27392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"id_weight = dict(zip(assets.Asset_ID, assets.Weight))\nasset_id = list(train[['Asset_ID']].iloc[0:14, 0])\nweight_14 = []\nfor i in asset_id:\n    weight_14.append(id_weight.get(i))\nweight_14","metadata":{"execution":{"iopub.status.busy":"2022-02-01T11:44:11.275737Z","iopub.execute_input":"2022-02-01T11:44:11.276198Z","iopub.status.idle":"2022-02-01T11:44:12.064483Z","shell.execute_reply.started":"2022-02-01T11:44:11.276167Z","shell.execute_reply":"2022-02-01T11:44:12.063531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets = train['Target'].to_numpy().reshape(-1, 14)\n\nfeatures = train.columns.drop(['Asset_ID', 'Target', 'group_num','is_real'])\ntrain = train[features]\n\ntrain=np.array(train)\ntrain = train.reshape(-1,14,train.shape[-1])\ntrain.shape","metadata":{"execution":{"iopub.status.busy":"2022-02-01T11:44:12.065693Z","iopub.execute_input":"2022-02-01T11:44:12.065918Z","iopub.status.idle":"2022-02-01T11:44:14.537087Z","shell.execute_reply.started":"2022-02-01T11:44:12.065891Z","shell.execute_reply":"2022-02-01T11:44:14.536254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_val = train[:-len(train)//10], train[-len(train)//10:]\ny_train, y_val = targets[:-len(train)//10], targets[-len(train)//10:]","metadata":{"execution":{"iopub.status.busy":"2022-02-01T11:44:14.538787Z","iopub.execute_input":"2022-02-01T11:44:14.539278Z","iopub.status.idle":"2022-02-01T11:44:14.545271Z","shell.execute_reply.started":"2022-02-01T11:44:14.539235Z","shell.execute_reply":"2022-02-01T11:44:14.544452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = X_train.transpose((0, 2, 1))\nX_val = X_val.transpose((0, 2, 1))","metadata":{"execution":{"iopub.status.busy":"2022-02-01T11:44:14.546453Z","iopub.execute_input":"2022-02-01T11:44:14.546677Z","iopub.status.idle":"2022-02-01T11:44:14.558082Z","shell.execute_reply.started":"2022-02-01T11:44:14.546653Z","shell.execute_reply":"2022-02-01T11:44:14.557396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(X_train.shape)\nprint(X_val.shape)\nprint(y_train.shape)\nprint(y_val.shape)","metadata":{"execution":{"iopub.status.busy":"2022-02-01T11:44:14.561189Z","iopub.execute_input":"2022-02-01T11:44:14.56156Z","iopub.status.idle":"2022-02-01T11:44:14.57217Z","shell.execute_reply.started":"2022-02-01T11:44:14.561529Z","shell.execute_reply":"2022-02-01T11:44:14.571183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# prepare dataset, metric and dataset","metadata":{}},{"cell_type":"code","source":"class CustomDataset(Dataset):\n    def __init__(self, x_set, y_set, length):\n            self.x, self.y = x_set, y_set\n            self.length = length\n            self.size = len(x_set)\n    def __len__(self): \n        return int(len(self.x)- self.length)\n    def __getitem__(self, idx):\n        window_x=[]\n        window_y=[]\n        end_ind = idx + self.length\n        if end_ind <= self.size:\n            window_x.append(self.x[idx : end_ind])\n            window_y.append(self.y[end_ind])\n        else:\n            window_x.append(self.x[self.size - idx : self.size - idx + self.length])\n            window_y.append(self.y[self.size - idx + self.length])\n\n        return np.array(window_x).transpose((0, 2, 3, 1)).squeeze(), np.array(window_y).squeeze()\n\n# train_data = CustomDataset(x, y, length=15)\n# train_loader = DataLoader(dataset=train_data, batch_size=batch_size, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2022-02-01T11:44:14.573289Z","iopub.execute_input":"2022-02-01T11:44:14.573512Z","iopub.status.idle":"2022-02-01T11:44:14.583095Z","shell.execute_reply.started":"2022-02-01T11:44:14.573487Z","shell.execute_reply":"2022-02-01T11:44:14.582244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# util.py\n\ndef load_dataset(batch_size, valid_batch_size=None, test_batch_size=None):\n    data = {}\n\n    train_data = CustomDataset(X_train, y_train, length = WINDOW_SIZE)\n    data['train_loader'] = DataLoader(dataset=train_data, batch_size=batch_size, shuffle=True)\n    val_data = CustomDataset(X_val, y_val, length = WINDOW_SIZE)\n    data['val_loader'] = DataLoader(dataset=val_data, batch_size=valid_batch_size, shuffle=False)\n    data['y_val'] = np.expand_dims(y_val, axis=1)\n\n    return data\n\n\ndef masked_mse(preds, labels, null_val=np.nan):\n    if np.isnan(null_val):\n        mask = ~torch.isnan(labels)\n    else:\n        mask = (labels != null_val)\n    mask = mask.float()\n    mask /= torch.mean((mask))\n    mask = torch.where(torch.isnan(mask), torch.zeros_like(mask), mask)\n    loss = (preds - labels) ** 2\n    loss = loss * mask\n    loss = torch.where(torch.isnan(loss), torch.zeros_like(loss), loss)\n    return torch.mean(loss)\n\n\ndef masked_rmse(preds, labels, null_val=np.nan):\n    return torch.sqrt(masked_mse(preds=preds, labels=labels, null_val=null_val))\n\n\ndef masked_mae(preds, labels, null_val=np.nan):\n    if np.isnan(null_val):\n        mask = ~torch.isnan(labels)\n    else:\n        mask = (labels != null_val)\n    mask = mask.float()\n    mask /= torch.mean(mask)\n    mask = torch.where(torch.isnan(mask), torch.zeros_like(mask), mask)\n    loss = torch.abs(preds - labels)\n    loss = loss * mask\n    loss = torch.where(torch.isnan(loss), torch.zeros_like(loss), loss)\n    return torch.mean(loss)\n\n\ndef masked_mape(preds, labels, null_val=np.nan):\n    if np.isnan(null_val):\n        mask = ~torch.isnan(labels)\n    else:\n        mask = (labels != null_val)\n    mask = mask.float()\n    mask /= torch.mean(mask)\n    mask = torch.where(torch.isnan(mask), torch.zeros_like(mask), mask)\n    loss = torch.abs(preds - labels) / labels\n    loss = loss * mask\n    loss = torch.where(torch.isnan(loss), torch.zeros_like(loss), loss)\n    return torch.mean(loss)\n\n\ndef mae(pred, real):\n    return np.mean(np.mean(np.abs(pred - real)), axis = 1)\n\n\ndef metric(pred, real):\n    mae = masked_mae(pred, real, 0.0).item()\n    mape = masked_mape(pred, real, 0.0).item()\n    rmse = masked_rmse(pred, real, 0.0).item()\n    return mae, mape, rmse\n\ndef weighted_correlation(x, y):\n    corrs = []\n    for i in range(x.size(0)):\n        w = torch.ravel(torch.Tensor(weight_14).to(device))\n        a = torch.ravel(x[i])\n        b = torch.ravel(y[i])\n        sum_w = torch.sum(w)\n        mean_a = torch.sum(a * w) / sum_w\n        mean_b = torch.sum(b * w) / sum_w\n        var_a = torch.sum(w * torch.square(a - mean_a)) / sum_w\n        var_b = torch.sum(w * torch.square(b - mean_b)) / sum_w\n        cov = torch.sum((a * b * w)) / torch.sum(w) - mean_a * mean_b\n        corr = cov / torch.sqrt(var_a * var_b)\n        corrs.append(corr)\n    return torch.mean(torch.Tensor(corrs).to(device))","metadata":{"execution":{"iopub.status.busy":"2022-02-01T11:44:14.584547Z","iopub.execute_input":"2022-02-01T11:44:14.585039Z","iopub.status.idle":"2022-02-01T11:44:14.615323Z","shell.execute_reply.started":"2022-02-01T11:44:14.584994Z","shell.execute_reply":"2022-02-01T11:44:14.61446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class masked_mse(nn.Module):\n    def __init__(self):\n        super().__init__()\n\n    def forward(self, y_pred, y_true):\n        loss = torch.zeros([1]).to(device)\n        for i in range(y_pred.size(0)):\n            mask = torch.not_equal(y_true[i], 0.)\n            y_true_masked = torch.masked_select(y_true[i], mask)\n            y_pred_masked = torch.masked_select(y_pred[i], mask)\n            loss += F.mse_loss(y_true_masked, y_pred_masked)\n        return loss","metadata":{"execution":{"iopub.status.busy":"2022-02-01T11:44:14.616796Z","iopub.execute_input":"2022-02-01T11:44:14.617233Z","iopub.status.idle":"2022-02-01T11:44:14.632335Z","shell.execute_reply.started":"2022-02-01T11:44:14.617187Z","shell.execute_reply":"2022-02-01T11:44:14.631408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class nconv(nn.Module):\n    def __init__(self):\n        super(nconv,self).__init__()\n\n    def forward(self,x, A):\n        x = torch.einsum('ncwl,vw->ncvl',(x,A))\n        return x.contiguous()\n\nclass dy_nconv(nn.Module):\n    def __init__(self):\n        super(dy_nconv,self).__init__()\n\n    def forward(self,x, A):\n        x = torch.einsum('ncvl,nvwl->ncwl',(x,A))\n        return x.contiguous()\n\nclass linear(nn.Module):\n    def __init__(self,c_in,c_out,bias=True):\n        super(linear,self).__init__()\n        self.mlp = torch.nn.Conv2d(c_in, c_out, kernel_size=(1, 1), padding=(0,0), stride=(1,1), bias=bias)\n\n    def forward(self,x):\n        return self.mlp(x)\n\n\nclass prop(nn.Module):\n    def __init__(self,c_in,c_out,gdep,dropout,alpha):\n        super(prop, self).__init__()\n        self.nconv = nconv()\n        self.mlp = linear(c_in,c_out)\n        self.gdep = gdep\n        self.dropout = dropout\n        self.alpha = alpha\n\n    def forward(self,x,adj):\n        adj = adj + torch.eye(adj.size(0)).to(x.device)\n        d = adj.sum(1)\n        h = x\n        dv = d\n        a = adj / dv.view(-1, 1)\n        for i in range(self.gdep):\n            h = self.alpha*x + (1-self.alpha)*self.nconv(h,a)\n        ho = self.mlp(h)\n        return ho\n\n\nclass mixprop(nn.Module):\n    def __init__(self,c_in,c_out,gdep,dropout,alpha):\n        super(mixprop, self).__init__()\n        self.nconv = nconv()\n        self.mlp = linear((gdep+1)*c_in,c_out)\n        self.gdep = gdep\n        self.dropout = dropout\n        self.alpha = alpha\n\n\n    def forward(self,x,adj):\n        adj = adj + torch.eye(adj.size(0)).to(x.device)\n        d = adj.sum(1)\n        h = x\n        out = [h]\n        a = adj / d.view(-1, 1)\n        for i in range(self.gdep):\n            h = self.alpha*x + (1-self.alpha)*self.nconv(h,a)\n            out.append(h)\n        ho = torch.cat(out,dim=1)\n        ho = self.mlp(ho)\n        return ho\n\nclass dy_mixprop(nn.Module):\n    def __init__(self,c_in,c_out,gdep,dropout,alpha):\n        super(dy_mixprop, self).__init__()\n        self.nconv = dy_nconv()\n        self.mlp1 = linear((gdep+1)*c_in,c_out)\n        self.mlp2 = linear((gdep+1)*c_in,c_out)\n\n        self.gdep = gdep\n        self.dropout = dropout\n        self.alpha = alpha\n        self.lin1 = linear(c_in,c_in)\n        self.lin2 = linear(c_in,c_in)\n\n\n    def forward(self,x):\n        #adj = adj + torch.eye(adj.size(0)).to(x.device)\n        #d = adj.sum(1)\n        x1 = torch.tanh(self.lin1(x))\n        x2 = torch.tanh(self.lin2(x))\n        adj = self.nconv(x1.transpose(2,1),x2)\n        adj0 = torch.softmax(adj, dim=2)\n        adj1 = torch.softmax(adj.transpose(2,1), dim=2)\n\n        h = x\n        out = [h]\n        for i in range(self.gdep):\n            h = self.alpha*x + (1-self.alpha)*self.nconv(h,adj0)\n            out.append(h)\n        ho = torch.cat(out,dim=1)\n        ho1 = self.mlp1(ho)\n\n\n        h = x\n        out = [h]\n        for i in range(self.gdep):\n            h = self.alpha * x + (1 - self.alpha) * self.nconv(h, adj1)\n            out.append(h)\n        ho = torch.cat(out, dim=1)\n        ho2 = self.mlp2(ho)\n\n        return ho1+ho2\n\n\n\nclass dilated_1D(nn.Module):\n    def __init__(self, cin, cout, dilation_factor=2):\n        super(dilated_1D, self).__init__()\n        self.tconv = nn.ModuleList()\n        self.kernel_set = [2,3,6,7]\n        self.tconv = nn.Conv2d(cin,cout,(1,7),dilation=(1,dilation_factor))\n\n    def forward(self,input):\n        x = self.tconv(input)\n        return x\n\nclass dilated_inception(nn.Module):\n    def __init__(self, cin, cout, dilation_factor=2):\n        super(dilated_inception, self).__init__()\n        self.tconv = nn.ModuleList()\n        self.kernel_set = [2,3,6,7]\n        cout = int(cout/len(self.kernel_set))\n        for kern in self.kernel_set:\n            self.tconv.append(nn.Conv2d(cin,cout,(1,kern),dilation=(1,dilation_factor)))\n\n    def forward(self,input):\n        x = []\n        for i in range(len(self.kernel_set)):\n            x.append(self.tconv[i](input))\n        for i in range(len(self.kernel_set)):\n            x[i] = x[i][...,-x[-1].size(3):]\n        x = torch.cat(x,dim=1)\n        return x\n\n\nclass graph_constructor(nn.Module):\n    def __init__(self, nnodes, k, dim, device, alpha=3, static_feat=None):\n        super(graph_constructor, self).__init__()\n        self.nnodes = nnodes\n        if static_feat is not None:\n            xd = static_feat.shape[1]\n            self.lin1 = nn.Linear(xd, dim)\n            self.lin2 = nn.Linear(xd, dim)\n        else:\n            self.emb1 = nn.Embedding(nnodes, dim)\n            self.emb2 = nn.Embedding(nnodes, dim)\n            self.lin1 = nn.Linear(dim,dim)\n            self.lin2 = nn.Linear(dim,dim)\n\n        self.device = device\n        self.k = k\n        self.dim = dim\n        self.alpha = alpha\n        self.static_feat = static_feat\n\n    def forward(self, idx):\n        if self.static_feat is None:\n            nodevec1 = self.emb1(idx)\n            nodevec2 = self.emb2(idx)\n        else:\n            nodevec1 = self.static_feat[idx,:]\n            nodevec2 = nodevec1\n\n        nodevec1 = torch.tanh(self.alpha*self.lin1(nodevec1))\n        nodevec2 = torch.tanh(self.alpha*self.lin2(nodevec2))\n\n        a = torch.mm(nodevec1, nodevec2.transpose(1,0))-torch.mm(nodevec2, nodevec1.transpose(1,0))\n        adj = F.relu(torch.tanh(self.alpha*a))\n        mask = torch.zeros(idx.size(0), idx.size(0)).to(self.device)\n        mask.fill_(float('0'))\n        s1,t1 = (adj + torch.rand_like(adj)*0.01).topk(self.k,1)\n        mask.scatter_(1,t1,s1.fill_(1))\n        adj = adj*mask\n        return adj\n\n    def fullA(self, idx):\n        if self.static_feat is None:\n            nodevec1 = self.emb1(idx)\n            nodevec2 = self.emb2(idx)\n        else:\n            nodevec1 = self.static_feat[idx,:]\n            nodevec2 = nodevec1\n\n        nodevec1 = torch.tanh(self.alpha*self.lin1(nodevec1))\n        nodevec2 = torch.tanh(self.alpha*self.lin2(nodevec2))\n\n        a = torch.mm(nodevec1, nodevec2.transpose(1,0))-torch.mm(nodevec2, nodevec1.transpose(1,0))\n        adj = F.relu(torch.tanh(self.alpha*a))\n        return adj\n\nclass graph_global(nn.Module):\n    def __init__(self, nnodes, k, dim, device, alpha=3, static_feat=None):\n        super(graph_global, self).__init__()\n        self.nnodes = nnodes\n        self.A = nn.Parameter(torch.randn(nnodes, nnodes).to(device), requires_grad=True).to(device)\n\n    def forward(self, idx):\n        return F.relu(self.A)\n\n\nclass graph_undirected(nn.Module):\n    def __init__(self, nnodes, k, dim, device, alpha=3, static_feat=None):\n        super(graph_undirected, self).__init__()\n        self.nnodes = nnodes\n        if static_feat is not None:\n            xd = static_feat.shape[1]\n            self.lin1 = nn.Linear(xd, dim)\n        else:\n            self.emb1 = nn.Embedding(nnodes, dim)\n            self.lin1 = nn.Linear(dim,dim)\n\n        self.device = device\n        self.k = k\n        self.dim = dim\n        self.alpha = alpha\n        self.static_feat = static_feat\n\n    def forward(self, idx):\n        if self.static_feat is None:\n            nodevec1 = self.emb1(idx)\n            nodevec2 = self.emb1(idx)\n        else:\n            nodevec1 = self.static_feat[idx,:]\n            nodevec2 = nodevec1\n\n        nodevec1 = torch.tanh(self.alpha*self.lin1(nodevec1))\n        nodevec2 = torch.tanh(self.alpha*self.lin1(nodevec2))\n\n        a = torch.mm(nodevec1, nodevec2.transpose(1,0))\n        adj = F.relu(torch.tanh(self.alpha*a))\n        mask = torch.zeros(idx.size(0), idx.size(0)).to(self.device)\n        mask.fill_(float('0'))\n        s1,t1 = adj.topk(self.k,1)\n        mask.scatter_(1,t1,s1.fill_(1))\n        adj = adj*mask\n        return adj\n\n\n\nclass graph_directed(nn.Module):\n    def __init__(self, nnodes, k, dim, device, alpha=3, static_feat=None):\n        super(graph_directed, self).__init__()\n        self.nnodes = nnodes\n        if static_feat is not None:\n            xd = static_feat.shape[1]\n            self.lin1 = nn.Linear(xd, dim)\n            self.lin2 = nn.Linear(xd, dim)\n        else:\n            self.emb1 = nn.Embedding(nnodes, dim)\n            self.emb2 = nn.Embedding(nnodes, dim)\n            self.lin1 = nn.Linear(dim,dim)\n            self.lin2 = nn.Linear(dim,dim)\n\n        self.device = device\n        self.k = k\n        self.dim = dim\n        self.alpha = alpha\n        self.static_feat = static_feat\n\n    def forward(self, idx):\n        if self.static_feat is None:\n            nodevec1 = self.emb1(idx)\n            nodevec2 = self.emb2(idx)\n        else:\n            nodevec1 = self.static_feat[idx,:]\n            nodevec2 = nodevec1\n\n        nodevec1 = torch.tanh(self.alpha*self.lin1(nodevec1))\n        nodevec2 = torch.tanh(self.alpha*self.lin2(nodevec2))\n\n        a = torch.mm(nodevec1, nodevec2.transpose(1,0))\n        adj = F.relu(torch.tanh(self.alpha*a))\n        mask = torch.zeros(idx.size(0), idx.size(0)).to(self.device)\n        mask.fill_(float('0'))\n        s1,t1 = adj.topk(self.k,1)\n        mask.scatter_(1,t1,s1.fill_(1))\n        adj = adj*mask\n        return adj\n\n\nclass LayerNorm(nn.Module):\n    __constants__ = ['normalized_shape', 'weight', 'bias', 'eps', 'elementwise_affine']\n    def __init__(self, normalized_shape, eps=1e-5, elementwise_affine=True):\n        super(LayerNorm, self).__init__()\n        if isinstance(normalized_shape, numbers.Integral):\n            normalized_shape = (normalized_shape,)\n        self.normalized_shape = tuple(normalized_shape)\n        self.eps = eps\n        self.elementwise_affine = elementwise_affine\n        if self.elementwise_affine:\n            self.weight = nn.Parameter(torch.Tensor(*normalized_shape))\n            self.bias = nn.Parameter(torch.Tensor(*normalized_shape))\n        else:\n            self.register_parameter('weight', None)\n            self.register_parameter('bias', None)\n        self.reset_parameters()\n\n\n    def reset_parameters(self):\n        if self.elementwise_affine:\n            init.ones_(self.weight)\n            init.zeros_(self.bias)\n\n    def forward(self, input, idx):\n        if self.elementwise_affine:\n            return F.layer_norm(input, tuple(input.shape[1:]), self.weight[:,idx,:], self.bias[:,idx,:], self.eps)\n        else:\n            return F.layer_norm(input, tuple(input.shape[1:]), self.weight, self.bias, self.eps)\n\n    def extra_repr(self):\n        return '{normalized_shape}, eps={eps}, ' \\\n            'elementwise_affine={elementwise_affine}'.format(**self.__dict__)\n","metadata":{"execution":{"iopub.status.busy":"2022-02-01T11:44:14.634681Z","iopub.execute_input":"2022-02-01T11:44:14.635014Z","iopub.status.idle":"2022-02-01T11:44:15.073288Z","shell.execute_reply.started":"2022-02-01T11:44:14.634984Z","shell.execute_reply":"2022-02-01T11:44:15.072379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class gtnet(nn.Module):\n    def __init__(self, gcn_true, buildA_true, gcn_depth, num_nodes, device, predefined_A=None, static_feat=None, dropout=0.3, subgraph_size=20, node_dim=40, dilation_exponential=1, conv_channels=32, residual_channels=32, skip_channels=64, end_channels=128, seq_length=12, in_dim=2, out_dim=12, layers=3, propalpha=0.05, tanhalpha=3, layer_norm_affline=True):\n        super(gtnet, self).__init__()\n        self.gcn_true = gcn_true\n        self.buildA_true = buildA_true\n        self.num_nodes = num_nodes\n        self.dropout = dropout\n        self.predefined_A = predefined_A\n        self.filter_convs = nn.ModuleList()\n        self.gate_convs = nn.ModuleList()\n        self.residual_convs = nn.ModuleList()\n        self.skip_convs = nn.ModuleList()\n        self.gconv1 = nn.ModuleList()\n        self.gconv2 = nn.ModuleList()\n        self.norm = nn.ModuleList()\n        self.start_conv = nn.Conv2d(in_channels=in_dim,\n                                    out_channels=residual_channels,\n                                    kernel_size=(1, 1))\n        self.gc = graph_constructor(num_nodes, subgraph_size, node_dim, device, alpha=tanhalpha, static_feat=static_feat)\n\n        self.seq_length = seq_length\n        kernel_size = 7\n        if dilation_exponential>1:\n            self.receptive_field = int(1+(kernel_size-1)*(dilation_exponential**layers-1)/(dilation_exponential-1))\n        else:\n            self.receptive_field = layers*(kernel_size-1) + 1\n\n        for i in range(1):\n            if dilation_exponential>1:\n                rf_size_i = int(1 + i*(kernel_size-1)*(dilation_exponential**layers-1)/(dilation_exponential-1))\n            else:\n                rf_size_i = i*layers*(kernel_size-1)+1\n            new_dilation = 1\n            for j in range(1,layers+1):\n                if dilation_exponential > 1:\n                    rf_size_j = int(rf_size_i + (kernel_size-1)*(dilation_exponential**j-1)/(dilation_exponential-1))\n                else:\n                    rf_size_j = rf_size_i+j*(kernel_size-1)\n\n                self.filter_convs.append(dilated_inception(residual_channels, conv_channels, dilation_factor=new_dilation))\n                self.gate_convs.append(dilated_inception(residual_channels, conv_channels, dilation_factor=new_dilation))\n                self.residual_convs.append(nn.Conv2d(in_channels=conv_channels,\n                                                    out_channels=residual_channels,\n                                                 kernel_size=(1, 1)))\n                if self.seq_length>self.receptive_field:\n                    self.skip_convs.append(nn.Conv2d(in_channels=conv_channels,\n                                                    out_channels=skip_channels,\n                                                    kernel_size=(1, self.seq_length-rf_size_j+1)))\n                else:\n                    self.skip_convs.append(nn.Conv2d(in_channels=conv_channels,\n                                                    out_channels=skip_channels,\n                                                    kernel_size=(1, self.receptive_field-rf_size_j+1)))\n\n                if self.gcn_true:\n                    self.gconv1.append(mixprop(conv_channels, residual_channels, gcn_depth, dropout, propalpha))\n                    self.gconv2.append(mixprop(conv_channels, residual_channels, gcn_depth, dropout, propalpha))\n\n                if self.seq_length>self.receptive_field:\n                    self.norm.append(LayerNorm((residual_channels, num_nodes, self.seq_length - rf_size_j + 1),elementwise_affine=layer_norm_affline))\n                else:\n                    self.norm.append(LayerNorm((residual_channels, num_nodes, self.receptive_field - rf_size_j + 1),elementwise_affine=layer_norm_affline))\n\n                new_dilation *= dilation_exponential\n\n        self.layers = layers\n        self.end_conv_1 = nn.Conv2d(in_channels=skip_channels,\n                                             out_channels=end_channels,\n                                             kernel_size=(1,1),\n                                             bias=True)\n        self.end_conv_2 = nn.Conv2d(in_channels=end_channels,\n                                             out_channels=out_dim,\n                                             kernel_size=(1,1),\n                                             bias=True)\n        if self.seq_length > self.receptive_field:\n            self.skip0 = nn.Conv2d(in_channels=in_dim, out_channels=skip_channels, kernel_size=(1, self.seq_length), bias=True)\n            self.skipE = nn.Conv2d(in_channels=residual_channels, out_channels=skip_channels, kernel_size=(1, self.seq_length-self.receptive_field+1), bias=True)\n\n        else:\n            self.skip0 = nn.Conv2d(in_channels=in_dim, out_channels=skip_channels, kernel_size=(1, self.receptive_field), bias=True)\n            self.skipE = nn.Conv2d(in_channels=residual_channels, out_channels=skip_channels, kernel_size=(1, 1), bias=True)\n\n\n        self.idx = torch.arange(self.num_nodes).to(device)\n        \n        self.sig = torch.nn.Sigmoid()\n\n\n    def forward(self, input, idx=None):\n        seq_len = input.size(3)\n        assert seq_len==self.seq_length, 'input sequence length not equal to preset sequence length'\n\n        if self.seq_length<self.receptive_field:\n            input = nn.functional.pad(input,(self.receptive_field-self.seq_length,0,0,0))\n\n\n\n        if self.gcn_true:\n            if self.buildA_true:\n                if idx is None:\n                    adp = self.gc(self.idx)\n                else:\n                    adp = self.gc(idx)\n            else:\n                adp = self.predefined_A\n\n        x = self.start_conv(input)\n        skip = self.skip0(F.dropout(input, self.dropout, training=self.training))\n        for i in range(self.layers):\n            residual = x\n            filter = self.filter_convs[i](x)\n            filter = torch.tanh(filter)\n            gate = self.gate_convs[i](x)\n            gate = torch.sigmoid(gate)\n            x = filter * gate\n            x = F.dropout(x, self.dropout, training=self.training)\n            s = x\n            s = self.skip_convs[i](s)\n            skip = s + skip\n            if self.gcn_true:\n                x = self.gconv1[i](x, adp)+self.gconv2[i](x, adp.transpose(1,0))\n            else:\n                x = self.residual_convs[i](x)\n\n            x = x + residual[:, :, :, -x.size(3):]\n            if idx is None:\n                x = self.norm[i](x,self.idx)\n            else:\n                x = self.norm[i](x,idx)\n\n        skip = self.skipE(x) + skip\n        x = F.relu(skip)\n        x = F.relu(self.end_conv_1(x))\n        x = self.end_conv_2(x)\n        # x = self.sig(x + 1e-8)\n        return x","metadata":{"execution":{"iopub.status.busy":"2022-02-01T11:44:15.074962Z","iopub.execute_input":"2022-02-01T11:44:15.075403Z","iopub.status.idle":"2022-02-01T11:44:15.11512Z","shell.execute_reply.started":"2022-02-01T11:44:15.075366Z","shell.execute_reply":"2022-02-01T11:44:15.114173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Optim(object):\n\n    def _makeOptimizer(self):\n        if self.method == 'sgd':\n            self.optimizer = optim.SGD(self.params, lr=self.lr, weight_decay=self.lr_decay)\n        elif self.method == 'adagrad':\n            self.optimizer = optim.Adagrad(self.params, lr=self.lr, weight_decay=self.lr_decay)\n        elif self.method == 'adadelta':\n            self.optimizer = optim.Adadelta(self.params, lr=self.lr, weight_decay=self.lr_decay)\n        elif self.method == 'adam':\n            self.optimizer = optim.Adam(self.params, lr=self.lr, weight_decay=self.lr_decay)\n        else:\n            raise RuntimeError(\"Invalid optim method: \" + self.method)\n\n    def __init__(self, params, method, lr, clip, lr_decay=1, start_decay_at=None):\n        self.params = params  # careful: params may be a generator\n        self.last_ppl = None\n        self.lr = lr\n        self.clip = clip\n        self.method = method\n        self.lr_decay = lr_decay\n        self.start_decay_at = start_decay_at\n        self.start_decay = False\n\n        self._makeOptimizer()\n\n    def step(self):\n        # Compute gradients norm.\n        grad_norm = 0\n        if self.clip is not None:\n            torch.nn.utils.clip_grad_norm_(self.params, self.clip)\n\n        # for param in self.params:\n        #     grad_norm += math.pow(param.grad.data.norm(), 2)\n        #\n        # grad_norm = math.sqrt(grad_norm)\n        # if grad_norm > 0:\n        #     shrinkage = self.max_grad_norm / grad_norm\n        # else:\n        #     shrinkage = 1.\n        #\n        # for param in self.params:\n        #     if shrinkage < 1:\n        #         param.grad.data.mul_(shrinkage)\n        self.optimizer.step()\n        return  grad_norm\n\n    # decay learning rate if val perf does not improve or we hit the start_decay_at limit\n    def updateLearningRate(self, ppl, epoch):\n        if self.start_decay_at is not None and epoch >= self.start_decay_at:\n            self.start_decay = True\n        if self.last_ppl is not None and ppl > self.last_ppl:\n            self.start_decay = True\n\n        if self.start_decay:\n            self.lr = self.lr * self.lr_decay\n            print(\"Decaying learning rate to %g\" % self.lr)\n        #only decay for one epoch\n        self.start_decay = False\n\n        self.last_ppl = ppl\n\n        self._makeOptimizer()","metadata":{"execution":{"iopub.status.busy":"2022-02-01T11:44:15.116519Z","iopub.execute_input":"2022-02-01T11:44:15.116951Z","iopub.status.idle":"2022-02-01T11:44:15.13275Z","shell.execute_reply.started":"2022-02-01T11:44:15.116914Z","shell.execute_reply":"2022-02-01T11:44:15.132087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir save","metadata":{"execution":{"iopub.status.busy":"2022-02-01T11:44:15.133732Z","iopub.execute_input":"2022-02-01T11:44:15.134373Z","iopub.status.idle":"2022-02-01T11:44:15.960737Z","shell.execute_reply.started":"2022-02-01T11:44:15.134339Z","shell.execute_reply":"2022-02-01T11:44:15.959663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# training","metadata":{}},{"cell_type":"code","source":"def evaluate(dataloader, model, evaluateL2, evaluateL1):\n    model.eval()\n    total_loss = 0\n    total_loss_l1 = 0\n    n_samples = 0\n    predict = None\n    test = None\n\n    for iter, (X, Y) in enumerate(dataloader['val_loader']):\n        X = torch.Tensor(X.float()).to(args.device)\n        Y = torch.Tensor(Y.float()).squeeze().to(args.device)\n        with torch.no_grad():\n            output = model(X)\n        output = torch.squeeze(output)\n        if len(output.shape)==1:\n            output = output.unsqueeze(dim=0)\n        if predict is None:\n            predict = output\n            test = Y\n        else:\n            predict = torch.cat((predict, output))\n            test = torch.cat((test, Y))\n\n        total_loss += evaluateL2(output, Y).item()\n        total_loss_l1 += evaluateL1(output, Y).item()\n        n_samples += (output.size(0))\n\n    rse = math.sqrt(total_loss / n_samples)\n    rae = (total_loss_l1 / n_samples)\n\n    predict = predict.data.cpu().numpy()\n    Ytest = test.data.cpu().numpy()\n    sigma_p = (predict).std(axis=0)\n    sigma_g = (Ytest).std(axis=0)\n    mean_p = predict.mean(axis=0)\n    mean_g = Ytest.mean(axis=0)\n    index = (sigma_g != 0)\n    correlation = ((predict - mean_p) * (Ytest - mean_g)).mean(axis=0) / (sigma_p * sigma_g)\n    correlation = (correlation[index]).mean()\n    return rse, rae, correlation\n\n\ndef train(dataloader, model, criterion, optim):\n    model.train()\n    total_loss = 0\n    n_samples = 0\n    iter = 0\n    for iter, (X, Y) in enumerate(dataloader['train_loader']):\n        model.zero_grad()\n        optim.optimizer.zero_grad()\n        X = torch.Tensor(X.float()).to(args.device)\n        Y = torch.Tensor(Y.float()).squeeze().to(args.device)\n        if iter % args.step_size == 0:\n            perm = np.random.permutation(range(args.num_nodes))\n        num_sub = int(args.num_nodes / args.num_split)\n\n        for j in range(args.num_split):\n            if j != args.num_split - 1:\n                id = perm[j * num_sub:(j + 1) * num_sub]\n            else:\n                id = perm[j * num_sub:]\n            id = torch.tensor(id).to(device)\n            tx = X[:, :, id, :]\n            ty = Y[:, id]\n            output = model(tx,id)\n            output = torch.squeeze(output)\n            loss = criterion(output, ty) \n#             with autograd.detect_anomaly():\n#                 loss.backward()\n            loss.backward()\n            total_loss += loss.item()\n            n_samples += (output.size(0))\n            grad_norm = optim.step()\n\n        if iter%700==0:\n            print('iter:{:3d} | loss: {:.8f} '.format(iter, loss.item()/(output.size(0))))\n        iter += 1\n    return total_loss / n_samples","metadata":{"execution":{"iopub.status.busy":"2022-02-01T11:44:15.962796Z","iopub.execute_input":"2022-02-01T11:44:15.963632Z","iopub.status.idle":"2022-02-01T11:44:15.986054Z","shell.execute_reply.started":"2022-02-01T11:44:15.963589Z","shell.execute_reply":"2022-02-01T11:44:15.985138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SEED = 42\nN_ASSETS = 14\nWINDOW_SIZE = 15\nBATCH = 30\nEPOCH = 2\nDEVICE = 'cpu'","metadata":{"execution":{"iopub.status.busy":"2022-02-01T11:44:15.987306Z","iopub.execute_input":"2022-02-01T11:44:15.987969Z","iopub.status.idle":"2022-02-01T11:44:15.996574Z","shell.execute_reply.started":"2022-02-01T11:44:15.987937Z","shell.execute_reply":"2022-02-01T11:44:15.995634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"parser = argparse.ArgumentParser(description='PyTorch Time series forecasting')\n\nparser.add_argument('--log_interval', type=int, default=2000, metavar='N',\n                    help='report interval')\nparser.add_argument('--seed', type=int, default=42, help='seed')\nparser.add_argument('--save', type=str, default='save/model.pt',\n                    help='path to save the final model')\nparser.add_argument('--optim', type=str, default='adam')\nparser.add_argument('--Masked', type=bool, default=False)\nparser.add_argument('--normalize', type=int, default=2)\nparser.add_argument('--device',type=str,default='cuda:0',help='device')\nparser.add_argument('--gcn_true', type=bool, default=True, help='whether to add graph convolution layer')\nparser.add_argument('--buildA_true', type=bool, default=True, help='whether to construct adaptive adjacency matrix')\nparser.add_argument('--gcn_depth',type=int,default=2,help='graph convolution depth')\nparser.add_argument('--num_nodes',type=int,default=14,help='number of nodes/variables')\nparser.add_argument('--dropout',type=float,default=0.3,help='dropout rate')\nparser.add_argument('--subgraph_size',type=int,default=14,help='k')\nparser.add_argument('--node_dim',type=int,default=40,help='dim of nodes')\nparser.add_argument('--dilation_exponential',type=int,default=2,help='dilation exponential')\nparser.add_argument('--conv_channels',type=int,default=8,help='convolution channels')\nparser.add_argument('--residual_channels',type=int,default=8,help='residual channels')\nparser.add_argument('--skip_channels',type=int,default=8,help='skip channels')\nparser.add_argument('--end_channels',type=int,default=16,help='end channels')\nparser.add_argument('--in_dim',type=int,default=8,help='inputs dimension')\nparser.add_argument('--seq_in_len',type=int,default=15,help='input sequence length')\nparser.add_argument('--seq_out_len',type=int,default=1,help='output sequence length')\nparser.add_argument('--horizon', type=int, default=3)\nparser.add_argument('--layers',type=int,default=4,help='number of layers')\n\nparser.add_argument('--batch_size',type=int,default=32,help='batch size')\nparser.add_argument('--lr',type=float,default=0.0001,help='learning rate')\nparser.add_argument('--weight_decay',type=float,default=0.00001,help='weight decay rate')\n\nparser.add_argument('--clip',type=int,default=5,help='clip')\n\nparser.add_argument('--propalpha',type=float,default=0.05,help='prop alpha')\nparser.add_argument('--tanhalpha',type=float,default=3,help='tanh alpha')\n\nparser.add_argument('--epochs',type=int,default=1,help='epochs')\nparser.add_argument('--num_split',type=int,default=1,help='number of splits for graphs')\nparser.add_argument('--step_size',type=int,default=100,help='step_size')\n\nargs = parser.parse_args(args=['--seed', str(SEED), '--epochs', str(EPOCH), '--device', DEVICE, '--num_nodes', str(N_ASSETS), '--seq_in_len', str(WINDOW_SIZE), '--batch_size', str(BATCH)])","metadata":{"execution":{"iopub.status.busy":"2022-02-01T11:44:15.997711Z","iopub.execute_input":"2022-02-01T11:44:15.998258Z","iopub.status.idle":"2022-02-01T11:44:16.022907Z","shell.execute_reply.started":"2022-02-01T11:44:15.998221Z","shell.execute_reply":"2022-02-01T11:44:16.022071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(args.device)\nrandom.seed(args.seed)\nnp.random.seed(args.seed)\ntorch.manual_seed(args.seed)","metadata":{"execution":{"iopub.status.busy":"2022-02-01T11:44:16.024126Z","iopub.execute_input":"2022-02-01T11:44:16.024345Z","iopub.status.idle":"2022-02-01T11:44:16.039096Z","shell.execute_reply.started":"2022-02-01T11:44:16.02431Z","shell.execute_reply":"2022-02-01T11:44:16.038179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def main():\n\n    dataloader = load_dataset(args.batch_size, args.batch_size, args.batch_size)\n    \n    model = gtnet(args.gcn_true, args.buildA_true, args.gcn_depth, args.num_nodes,\n                  device, dropout=args.dropout, subgraph_size=args.subgraph_size,\n                  node_dim=args.node_dim, dilation_exponential=args.dilation_exponential,\n                  conv_channels=args.conv_channels, residual_channels=args.residual_channels,\n                  skip_channels=args.skip_channels, end_channels= args.end_channels,\n                  seq_length=args.seq_in_len, in_dim=args.in_dim, out_dim=args.seq_out_len,\n                  layers=args.layers, propalpha=args.propalpha, tanhalpha=args.tanhalpha, layer_norm_affline=False)\n    model = model.to(device)\n    \n#     with open('../input/mtgnngcf/save/model.pt', 'rb') as f:\n#         model = torch.load(f)\n\n    print(args)\n    print('The recpetive field size is', model.receptive_field)\n    nParams = sum([p.nelement() for p in model.parameters()])\n    print('Number of model parameters is', nParams, flush=True)\n\n    if args.Masked:\n        criterion = masked_mse().to(device)\n    else:\n        criterion = nn.MSELoss(size_average=False).to(device)\n    evaluateL2 = nn.MSELoss(size_average=False).to(device)\n    evaluateL1 = nn.L1Loss(size_average=False).to(device)\n\n\n    best_val = 1e8\n    optim = Optim(\n        model.parameters(), args.optim, args.lr, args.clip, lr_decay=args.weight_decay\n    )\n\n    # At any point you can hit Ctrl + C to break out of training early.\n    try:\n        print('begin training')\n        for epoch in range(1, args.epochs + 1):\n            epoch_start_time = time.time()\n            train_loss = train(dataloader, model, criterion, optim)\n            val_loss, val_rae, val_corr = evaluate(dataloader, model, evaluateL2, evaluateL1)\n            print(\n                '| end of epoch {:3d} | time: {:5.2f}s | train_loss {:5.4f} | valid rse {:5.4f} | valid rae {:5.4f} | valid corr  {:5.4f}'.format(\n                    epoch, (time.time() - epoch_start_time), train_loss, val_loss, val_rae, val_corr), flush=True)\n            # Save the model if the validation loss is the best we've seen so far.\n\n            if val_loss < best_val:\n                with open(args.save, 'wb') as f:\n                    torch.save(model, f)\n                best_val = val_loss\n            if epoch % 5 == 0:\n                test_acc, test_rae, test_corr = evaluate(dataloader, model, evaluateL2, evaluateL1)\n                print(\"test rse {:5.4f} | test rae {:5.4f} | test corr {:5.4f}\".format(test_acc, test_rae, test_corr), flush=True)\n\n    except KeyboardInterrupt:\n        print('-' * 89)\n        print('Exiting from training early')\n\n    # Load the best saved model.\n    with open(args.save, 'rb') as f:\n        model = torch.load(f)\n\n    vtest_acc, vtest_rae, vtest_corr = evaluate(dataloader, model, evaluateL2, evaluateL1)\n    test_acc, test_rae, test_corr = evaluate(dataloader, model, evaluateL2, evaluateL1)\n    print(\"final test rse {:5.4f} | test rae {:5.4f} | test corr {:5.4f}\".format(test_acc, test_rae, test_corr))\n    return vtest_acc, vtest_rae, vtest_corr, test_acc, test_rae, test_corr","metadata":{"execution":{"iopub.status.busy":"2022-02-01T11:44:16.040286Z","iopub.execute_input":"2022-02-01T11:44:16.040833Z","iopub.status.idle":"2022-02-01T11:44:16.059637Z","shell.execute_reply.started":"2022-02-01T11:44:16.0408Z","shell.execute_reply":"2022-02-01T11:44:16.058901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# debug\n# torch.autograd.set_detect_anomaly(True)\n\nvacc = []\nvrae = []\nvcorr = []\nacc = []\nrae = []\ncorr = []\nfor i in range(1):\n    val_acc, val_rae, val_corr, test_acc, test_rae, test_corr = main()\n    vacc.append(val_acc)\n    vrae.append(val_rae)\n    vcorr.append(val_corr)\n    acc.append(test_acc)\n    rae.append(test_rae)\n    corr.append(test_corr)\nprint('\\n\\n')\nprint('10 runs average')\nprint('\\n\\n')\nprint(\"valid\\trse\\trae\\tcorr\")\nprint(\"mean\\t{:5.4f}\\t{:5.4f}\\t{:5.4f}\".format(np.mean(vacc), np.mean(vrae), np.mean(vcorr)))\nprint(\"std\\t{:5.4f}\\t{:5.4f}\\t{:5.4f}\".format(np.std(vacc), np.std(vrae), np.std(vcorr)))\nprint('\\n\\n')\nprint(\"test\\trse\\trae\\tcorr\")\nprint(\"mean\\t{:5.4f}\\t{:5.4f}\\t{:5.4f}\".format(np.mean(acc), np.mean(rae), np.mean(corr)))\nprint(\"std\\t{:5.4f}\\t{:5.4f}\\t{:5.4f}\".format(np.std(acc), np.std(rae), np.std(corr)))","metadata":{"execution":{"iopub.status.busy":"2022-02-01T11:44:16.060693Z","iopub.execute_input":"2022-02-01T11:44:16.061407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Submission","metadata":{}},{"cell_type":"code","source":"with open(args.save, 'rb') as f:\n    model = torch.load(f)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"placeholder=pd.read_csv('../input/g-research-crypto-forecasting/supplemental_train.csv')[:15*14]\nplaceholder[scale_features] = RS.transform(placeholder[scale_features])\nplaceholder['asset_order'] = placeholder.Asset_ID.map(assets_order) \ntest_sample = np.array(placeholder[features])\ntest_sample = test_sample.reshape(-1,14,test_sample.shape[-1])\ntest_sample = np.expand_dims(test_sample, axis=0)\ntest_sample.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for test gap filling\nexample = pd.read_csv('../input/g-research-crypto-forecasting/example_test.csv')[:14]\nexample['asset_order'] = example.Asset_ID.map(assets_order) \nexample = example[['Asset_ID','asset_order']]\nexample","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"env = gresearch_crypto.make_env()\niter_test = env.iter_test()\n\nfor (test_df, sample_prediction_df) in iter_test:\n        \n    test_df['VWAP'] =np.nan_to_num(test_df.VWAP, posinf=VWAP_max, neginf=VWAP_min)\n    test_df[scale_features] = RS.transform(test_df[scale_features])\n    \n    test_data = test_df.merge(example, how='outer', on='Asset_ID').sort_values('asset_order')\n\n    test = np.array(test_data[features].fillna(0))\n    test = test.reshape(-1,1,14,test.shape[-1])\n    test_sample = np.hstack([test_sample, test])[:,-15:]\n    input_x = torch.Tensor(test_sample).to(args.device).transpose(1, 3)\n    y_pred = model(input_x).cpu().detach().numpy()\n    y_pred = y_pred.squeeze().reshape(-1, 1).squeeze()\n    \n    test_data['Target'] = y_pred\n        \n    for _, row in test_df.iterrows(): \n        try:\n            sample_prediction_df.loc[sample_prediction_df['row_id'] == row['row_id'], 'Target'] = test_data.loc[test_data['row_id'] == row['row_id'], 'Target'].item()\n        except:\n            sample_prediction_df.loc[sample_prediction_df['row_id'] == row['row_id'], 'Target'] = 0\n          \n    env.predict(sample_prediction_df)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}