{"cells":[{"metadata":{},"cell_type":"markdown","source":"# fastai v2 Kernel Starter Code\n\nThe goal of this kernel is to show how to train a neural network using fastai 2.0 for this Kaggle Competition"},{"metadata":{},"cell_type":"markdown","source":"## Grabbing the Library"},{"metadata":{},"cell_type":"markdown","source":"First we need to enable internet access within this kernel and then `!pip install git+https://github.com/fastai/fastai_dev             > /dev/null` the dev repository for us to import from."},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install git+https://github.com/fastai/fastai_dev > /dev/null","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We're going to need a variety of imports, most importantly the `tabular.core` module for building the dataset (the rest deal with training the model)\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"from fastai2.data.all import *\nfrom fastai2.tabular.model import *\nfrom fastai2.optimizer import *\nfrom fastai2.learner import *\nfrom fastai2.metrics import *\nfrom fastai2.callback.all import *","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We'll also use some functionality from the `_40_tabular_core_alt` notebook for memory usage"},{"metadata":{"trusted":true},"cell_type":"code","source":"class _TabIloc:\n    \"Get/set rows by iloc and cols by name\"\n    def __init__(self,to): self.to = to\n    def __getitem__(self, idxs):\n        df = self.to.items\n        if isinstance(idxs,tuple):\n            rows,cols = idxs\n            cols = df.columns.isin(cols) if is_listy(cols) else df.columns.get_loc(cols)\n        else: rows,cols = idxs,slice(None)\n        return self.to.new(df.iloc[rows, cols])\n    \nclass Tabular(CollBase, GetAttr, FilteredBase):\n    \"A `DataFrame` wrapper that knows which cols are cont/cat/y, and returns rows in `__getitem__`\"\n    _default='items'\n    def __init__(self, df, procs=None, cat_names=None, cont_names=None, y_names=None, type_y=Category, splits=None, do_setup=True):\n        if splits is None: splits=[range_of(df)]\n        df = df.iloc[sum(splits, [])].copy()\n        super().__init__(df)\n        \n        self.y_names = L(y_names)\n        if type_y is not None: procs = L(procs) + getattr(type_y, 'create', noop)\n        self.cat_names,self.cont_names,self.procs = L(cat_names),L(cont_names),Pipeline(procs, as_item=True)\n        self.split = len(splits[0])\n        if do_setup: self.setup()\n\n    def subset(self, i): return self.new(self.items[slice(0,self.split) if i==0 else slice(self.split,len(self))])\n    def copy(self): self.items = self.items.copy(); return self\n    def new(self, df): return type(self)(df, do_setup=False, type_y=None, **attrdict(self, 'procs','cat_names','cont_names','y_names'))\n    def show(self, max_n=10, **kwargs): display_df(self.all_cols[:max_n])\n    def setup(self): self.procs.setup(self)\n    def process(self): self.procs(self)\n    def iloc(self): return _TabIloc(self)\n    def targ(self): return self.items[self.y_names]\n    def all_col_names (self): return self.cat_names + self.cont_names + self.y_names\n    def n_subsets(self): return 2\n\nproperties(Tabular,'iloc','targ','all_col_names','n_subsets')\n\nclass TabularPandas(Tabular):\n    def transform(self, cols, f): self[cols] = self[cols].transform(f)\n        \ndef _add_prop(cls, nm):\n    @property\n    def f(o): return o[list(getattr(o,nm+'_names'))]\n    @f.setter\n    def fset(o, v): o[getattr(o,nm+'_names')] = v\n    setattr(cls, nm+'s', f)\n    setattr(cls, nm+'s', fset)\n\n_add_prop(Tabular, 'cat')\n_add_prop(Tabular, 'cont')\n_add_prop(Tabular, 'y')\n_add_prop(Tabular, 'all_col')\n\nclass TabularProc(InplaceTransform):\n    \"Base class to write a non-lazy tabular processor for dataframes\"\n    def setup(self, items=None):\n        super().setup(getattr(items,'train',items))\n        # Procs are called as soon as data is available\n        return self(items.items if isinstance(items,DataSource) else items)\n    \ndef _apply_cats (voc, add, c): return c.cat.codes+add if is_categorical_dtype(c) else c.map(voc[c.name].o2i)\ndef _decode_cats(voc, c): return c.map(dict(enumerate(voc[c.name].items)))\n\nclass Categorify(TabularProc):\n    \"Transform the categorical variables to that type.\"\n    order = 1\n    def setups(self, to):\n        self.classes = {n:CategoryMap(to.iloc[:,n].items, add_na=(n in to.cat_names)) for n in to.cat_names}\n    def encodes(self, to): to.transform(to.cat_names, partial(_apply_cats, self.classes, 1))\n    def decodes(self, to): to.transform(to.cat_names, partial(_decode_cats, self.classes))\n    def __getitem__(self,k): return self.classes[k]\n    \n@Categorize\ndef setups(self, to:Tabular): \n    if len(to.y_names) > 0: self.vocab = CategoryMap(to.iloc[:,to.y_names[0]].items)\n    return self(to)\n\n@Categorize\ndef encodes(self, to:Tabular): \n    to.transform(to.y_names, partial(_apply_cats, {n: self.vocab for n in to.y_names}, 0))\n    return to\n  \n@Categorize\ndef decodes(self, to:Tabular): \n    to.transform(to.y_names, partial(_decode_cats, {n: self.vocab for n in to.y_names}))\n    return to\n\nclass Normalize(TabularProc):\n    \"Normalize the continuous variables.\"\n    order = 2\n    def setups(self, dsrc): self.means,self.stds = dsrc.conts.mean(),dsrc.conts.std(ddof=0)+1e-7\n    def encodes(self, to): to.conts = (to.conts-self.means) / self.stds\n    def decodes(self, to): to.conts = (to.conts*self.stds ) + self.means\n        \nclass FillStrategy:\n    \"Namespace containing the various filling strategies.\"\n    def median  (c,fill): return c.median()\n    def constant(c,fill): return fill\n    def mode    (c,fill): return c.dropna().value_counts().idxmax()\n    \nclass FillMissing(TabularProc):\n    \"Fill the missing values in continuous columns.\"\n    def __init__(self, fill_strategy=FillStrategy.median, add_col=True, fill_vals=None):\n        if fill_vals is None: fill_vals = defaultdict(int)\n        store_attr(self, 'fill_strategy,add_col,fill_vals')\n\n    def setups(self, dsrc):\n        self.na_dict = {n:self.fill_strategy(dsrc[n], self.fill_vals[n])\n                        for n in pd.isnull(dsrc.conts).any().keys()}\n\n    def encodes(self, to):\n        missing = pd.isnull(to.conts)\n        for n in missing.any().keys():\n            assert n in self.na_dict, f\"nan values in `{n}` but not in setup training set\"\n            to[n].fillna(self.na_dict[n], inplace=True)\n            if self.add_col:\n                to.loc[:,n+'_na'] = missing[n]\n                if n+'_na' not in to.cat_names: to.cat_names.append(n+'_na')\n                    \nclass ReadTabBatch(ItemTransform):\n    def __init__(self, to): self.to = to\n    # TODO: use float for cont targ\n    def encodes(self, to): return tensor(to.cats).long(),tensor(to.conts).float(), tensor(to.targ)\n\n    def decodes(self, o):\n        cats,conts,targs = to_np(o)\n        vals = np.concatenate([cats,conts,targs], axis=1)\n        df = pd.DataFrame(vals, columns=self.to.all_col_names)\n        to = self.to.new(df)\n        to = self.to.procs.decode(to)\n        return to\n    \n@typedispatch\ndef show_batch(x: Tabular, y, its, max_n=10, ctxs=None):\n    x.show()\n    \n@delegates()\nclass TabDataLoader(TfmdDL):\n    do_item = noops\n    def __init__(self, dataset, bs=16, shuffle=False, after_batch=None, num_workers=0, **kwargs):\n        after_batch = L(after_batch)+ReadTabBatch(dataset)\n        super().__init__(dataset, bs=bs, shuffle=shuffle, after_batch=after_batch, num_workers=num_workers, **kwargs)\n\n    def create_batch(self, b): return self.dataset.iloc[b]\n\nTabularPandas._dl_type = TabDataLoader","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Setting Up Our Data"},{"metadata":{},"cell_type":"markdown","source":"Let's make a `Path` object to our data and combine the `train.csv` with the `building_metadata.csv` to grab some more information about these meter readings. For simplicity we will use the first 1000 samples from the training set. For the `DataFrame` preperation please see ryches Kernel [here](https://www.kaggle.com/ryches/simple-lgbm-solution)"},{"metadata":{"trusted":true},"cell_type":"code","source":"path = Path('/kaggle/input/ashrae-energy-prediction')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv(path/'train.csv', nrows=3000)\ntrain = df.iloc[:2000]\ntest = df.iloc[2000:]\nbldg = pd.read_csv(path/'building_metadata.csv')\nweather_train = pd.read_csv(path/\"weather_train.csv\")\nweather_test = pd.read_csv(path/\"weather_test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train), len(test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Next we'll get rid of any missing `y` variables"},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train[np.isfinite(train['meter_reading'])]\ntest = test[np.isfinite(test['meter_reading'])]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Then we'll merge our data with the provided metadata"},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train.merge(bldg, left_on = 'building_id', right_on = 'building_id', how = 'left')\ntest = test.merge(bldg, left_on = 'building_id', right_on = 'building_id', how = 'left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train.merge(weather_train, left_on = ['site_id', 'timestamp'], right_on = ['site_id', 'timestamp'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"There is some type discrepencies in the test data so we need to convert it first"},{"metadata":{"trusted":true},"cell_type":"code","source":"test = test.merge(weather_train, left_on = ['site_id', 'timestamp'], right_on = ['site_id', 'timestamp'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del weather_train, weather_test, bldg","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train[\"timestamp\"] = pd.to_datetime(train[\"timestamp\"])\ntrain[\"hour\"] = train[\"timestamp\"].dt.hour\ntrain[\"day\"] = train[\"timestamp\"].dt.day\ntrain[\"weekend\"] = train[\"timestamp\"].dt.weekday\ntrain[\"month\"] = train[\"timestamp\"].dt.month\ntest[\"timestamp\"] = pd.to_datetime(test[\"timestamp\"])\ntest[\"hour\"] = test[\"timestamp\"].dt.hour\ntest[\"day\"] = test[\"timestamp\"].dt.day\ntest[\"weekend\"] = test[\"timestamp\"].dt.weekday\ntest[\"month\"] = test[\"timestamp\"].dt.month","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.drop('timestamp', axis=1, inplace=True)\ntest.drop('timestamp', axis=1, inplace=True)\ntrain['meter_reading'] = np.log1p(train['meter_reading'])\ntest['meter_reading'] = np.log1p(test['meter_reading'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Making the DataBunch"},{"metadata":{},"cell_type":"markdown","source":"Next, just like in fastai v1 we need to declare a few things. Specifically our Categorical and Continuous variables, our preprocessors (Normalization, Categorification, and FillMissing), along with how we want to split our data. `fastai` v2 now includes a `RandomSplitter` which is similar to `.split_by_rand_pct()` but now we can specify a custom range for our data (hence `range_of(train)`)"},{"metadata":{"trusted":true},"cell_type":"code","source":"cat_vars = [\"building_id\", \"primary_use\", \"hour\", \"day\", \"weekend\", \"month\", \"meter\"]\ncont_vars = [\"square_feet\", \"year_built\", \"air_temperature\", \"cloud_coverage\",\n              \"dew_temperature\"]\ndep_var = 'meter_reading'","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":false,"trusted":true},"cell_type":"code","source":"procs = [Normalize, Categorify, FillMissing]\nsplits = RandomSplitter()(range_of(train))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now that those are defined, we can create a `TabularPandas` object by passing in our dataframe, the `procs`, our variables, what our `y` is, and how we want to split our data. `fastai` v2 is built on a Pipeline structure where first we dictate what we want to do, then we call the databunch (the high-level API is not done yet so we have nothing similar to directly DataBunching an object)"},{"metadata":{"trusted":true},"cell_type":"code","source":"to = TabularPandas(train, procs, cat_vars, cont_vars, y_names=dep_var, type_y=Float, splits=splits)\nto_test = TabularPandas(test, procs, cat_vars, cont_vars, y_names=dep_var, type_y=Float)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"If we look at what `to` actually is, we can see what looks to be a bunch of batches of our data aligned into a dataframe that can easily be read!"},{"metadata":{"trusted":true},"cell_type":"code","source":"to","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We can then also easily look at our training and validation datasets by calling `.train` or `.valid`"},{"metadata":{"trusted":true},"cell_type":"code","source":"to.train","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"From here we can create our DataBunch object one of two ways. We can either directly do a `dbch = to.databunch()`, *or* we can take it one step further and apply custom works to some dataloaders. First let's look at the basic version"},{"metadata":{"trusted":true},"cell_type":"code","source":"dbch = to.databunch()\ndbch.valid_dl.show_batch()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now let's try doing this the second method. We can increase our batch size since the validation set is much smaller than our training dataset. We can also specify a few options with our training dataset too. To do this, we will need to create `TabDataLoaders` to, well, load the data!\n\nWe pass in a dataset, a batch size, our `num_workers`, along with if we want to shuffle our dataset and drop the last batch if it does not evenly split. You should always want to do this with the **training** dataset but not the validation. Defaultly they are both set to `False`"},{"metadata":{"trusted":true},"cell_type":"code","source":"trn_dl = TabDataLoader(to.train, bs=64, num_workers=0)\nval_dl = TabDataLoader(to.valid, bs=128, num_workers=0, shuffle=False, drop_last=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dl = TabDataLoader(to_test, bs=128, num_workers=0, shuffle=False, drop_last=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Lastly we can create a `DataBunch` object by calling `DataBunch()` and passing in our two `DataLoaders`"},{"metadata":{"trusted":true},"cell_type":"code","source":"dbunch = DataBunch(trn_dl, val_dl)\ndbunch.valid_dl.show_batch()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"As you can see there are a *lot* of ways we can customize our DataBunch's now"},{"metadata":{},"cell_type":"markdown","source":"## Training the Model"},{"metadata":{},"cell_type":"markdown","source":"First we need to create a `TabularModel` that needs an embedding matrix size, how many continuous variables to expect, the number of possible outputs (classes), and how big we want our layers. To pass in the embedding matrix sizes, we can use `get_emb_sz` onto a `TabularPandas` object"},{"metadata":{},"cell_type":"markdown","source":"First let's define our embedding size rule of thumb, along with our `get_emb_sz` function"},{"metadata":{"trusted":true},"cell_type":"code","source":"def emb_sz_rule(n_cat): \n    \"Rule of thumb to pick embedding size corresponding to `n_cat`\"\n    return min(600, round(1.6 * n_cat**0.56))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _one_emb_sz(classes, n, sz_dict=None):\n    \"Pick an embedding size for `n` depending on `classes` if not given in `sz_dict`.\"\n    sz_dict = ifnone(sz_dict, {})\n    n_cat = len(classes[n])\n    sz = sz_dict.get(n, int(emb_sz_rule(n_cat)))  # rule of thumb\n    return n_cat,sz","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_emb_sz(to, sz_dict=None):\n    \"Get default embedding size from `TabularPreprocessor` `proc` or the ones in `sz_dict`\"\n    return [_one_emb_sz(to.procs.classes, n, sz_dict) for n in to.cat_names]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now we pass in our `TabularPandas` object, `to`"},{"metadata":{"trusted":true},"cell_type":"code","source":"emb_szs = get_emb_sz(to); print(emb_szs)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The last piece of the puzzle we need is our basic `TabularModel`"},{"metadata":{"trusted":true},"cell_type":"code","source":"class TabularModel(Module):\n    \"Basic model for tabular data.\"\n    def __init__(self, emb_szs, n_cont, out_sz, layers, ps=None, embed_p=0., y_range=None, use_bn=True, bn_final=False):\n        ps = ifnone(ps, [0]*len(layers))\n        if not is_listy(ps): ps = [ps]*len(layers)\n        self.embeds = nn.ModuleList([Embedding(ni, nf) for ni,nf in emb_szs])\n        self.emb_drop = nn.Dropout(embed_p)\n        self.bn_cont = nn.BatchNorm1d(n_cont)\n        n_emb = sum(e.embedding_dim for e in self.embeds)\n        self.n_emb,self.n_cont,self.y_range = n_emb,n_cont,y_range\n        sizes = [n_emb + n_cont] + layers + [out_sz]\n        actns = [nn.ReLU(inplace=True) for _ in range(len(sizes)-2)] + [None]\n        _layers = [BnDropLin(sizes[i], sizes[i+1], bn=use_bn and i!=0, p=p, act=a)\n                       for i,(p,a) in enumerate(zip([0.]+ps,actns))]\n        if bn_final: _layers.append(nn.BatchNorm1d(sizes[-1]))\n        self.layers = nn.Sequential(*_layers)\n    \n    def forward(self, x_cat, x_cont):\n        if self.n_emb != 0:\n            x = [e(x_cat[:,i]) for i,e in enumerate(self.embeds)]\n            x = torch.cat(x, 1)\n            x = self.emb_drop(x)\n        if self.n_cont != 0:\n            x_cont = self.bn_cont(x_cont)\n            x = torch.cat([x, x_cont], 1) if self.n_emb != 0 else x_cont\n        x = self.layers(x)\n        if self.y_range is not None:\n            x = (self.y_range[1]-self.y_range[0]) * torch.sigmoid(x) + self.y_range[0]\n        return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"If you noticed, most of what changed with the v2 API is focused on the dataloading / DataBunch creation. The rest of this Kernel sould look very familiar to fastai users"},{"metadata":{"trusted":true},"cell_type":"code","source":"model = TabularModel(emb_szs, len(to.cont_names), 1, [1000,500]); model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now we can define our optimization function and create our `Learner`"},{"metadata":{"trusted":true},"cell_type":"code","source":"opt_func = partial(Adam, wd=0.01, eps=1e-5)\nlearn = Learner(dbunch, model, MSELossFlat(), opt_func=opt_func)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.fit_one_cycle(30)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"To get our predictions we just simply pass in that test dataloader from earlier"},{"metadata":{"trusted":true},"cell_type":"code","source":"to_test = TabularPandas(test, procs, cat_vars, cont_vars, y_names=dep_var, type_y=Float)\ntest_dl = TabDataLoader(to_test, bs=128, shuffle=False, drop_last=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds, _ = learn.get_preds(dl=test_dl) \npreds = np.expm1(preds.numpy())\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame(columns=['row_id', 'meter_reading'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Just for example. In reality for this competition you would want to use: `submission['row_id'] = test['row_id']`"},{"metadata":{"trusted":true},"cell_type":"code","source":"submission['row_id'] = test['building_id']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission['meter_reading'] = preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Hope this helps you get started! :)\n\n- muellerzr"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}