{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport mxnet as mx # mxnet module\nfrom mxnet import autograd, gluon, init, nd\nfrom mxnet.gluon import data as gdata, loss as gloss, utils as gutils, nn\nimport time\nimport math","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/Kannada-MNIST/train.csv')\ntest = pd.read_csv('../input/Kannada-MNIST/test.csv')\n\ntrain.shape, test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"split = int(train.shape[0] * 0.9) # split the train data into train and valid with a ratio\nX_train = nd.array(train.iloc[:split, 1:].values.astype(dtype=np.uint8)).reshape(split, 28, 28, 1)\ny_train = nd.array(train.iloc[:split, 0].values.astype(dtype=np.int32)).reshape(split,)\nX_valid = nd.array(train.iloc[split:, 1:].values.astype(dtype=np.uint8)).reshape(train.shape[0] - split, 28, 28, 1)\ny_valid = nd.array(train.iloc[split:, 0].values.astype(dtype=np.int32)).reshape(train.shape[0] - split,)\nX_train.shape, X_valid.shape, y_train.shape, y_valid.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# construct the array dataset\ntrain_data = gdata.dataset.ArrayDataset(X_train, y_train)\nvalid_data = gdata.dataset.ArrayDataset(X_valid, y_valid)\n\n# batch data\nbatch_size = 1000\n# transform data for a large dataset\ntransform_train = gdata.vision.transforms.Compose([\n    gdata.vision.transforms.Resize(35),\n    gdata.vision.transforms.RandomResizedCrop(28, scale=(0.8, 1.0), ratio=(1.0, 1.0)),\n    gdata.vision.transforms.RandomFlipLeftRight(),\n    gdata.vision.transforms.ToTensor(),\n    gdata.vision.transforms.Normalize(0.0819, 0.2412)\n])\ntransform_valid = gdata.vision.transforms.Compose([\n    gdata.vision.transforms.ToTensor(),\n    gdata.vision.transforms.Normalize(0.0819, 0.2412)\n])\ntrain_data = train_data.transform_first(transform_train)\nvalid_data = valid_data.transform_first(transform_valid)\ntrain_iter = gdata.DataLoader(train_data, batch_size, shuffle=True, num_workers=0)\nvalid_iter = gdata.DataLoader(valid_data, batch_size, shuffle=False, num_workers=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# build a CNN\ndef get_net(ctx):\n    net = nn.HybridSequential()\n    net.add(\n        # first\n        nn.Conv2D(64, kernel_size=3, padding=1),\n        nn.BatchNorm(),\n        nn.LeakyReLU(0.1),\n        nn.Conv2D(64, kernel_size=3, padding=1),\n        nn.BatchNorm(),\n        nn.LeakyReLU(0.1),\n        nn.Conv2D(64, kernel_size=5, padding=2),\n        nn.BatchNorm(),\n        nn.LeakyReLU(0.1),\n        nn.MaxPool2D(pool_size=2),\n        nn.Dropout(0.2),\n        # second\n        nn.Conv2D(128, kernel_size=3, padding=1),\n        nn.BatchNorm(),\n        nn.LeakyReLU(0.1),\n        nn.Conv2D(128, kernel_size=3, padding=1),\n        nn.BatchNorm(),\n        nn.LeakyReLU(0.1),\n        nn.Conv2D(128, kernel_size=5, padding=2),\n        nn.BatchNorm(),\n        nn.LeakyReLU(0.1),\n        nn.MaxPool2D(pool_size=2),\n        nn.Dropout(0.2),\n        # third\n        nn.Conv2D(256, kernel_size=3, padding=1),\n        nn.BatchNorm(),\n        nn.LeakyReLU(0.1),\n        nn.Conv2D(256, kernel_size=3, padding=1),\n        nn.BatchNorm(),\n        nn.LeakyReLU(0.1),\n        nn.MaxPool2D(pool_size=2),\n        nn.Dropout(0.2),\n        # fourth\n        nn.Flatten(),\n        nn.Dense(256),\n        nn.BatchNorm(),\n        nn.LeakyReLU(0.1),\n        nn.Dense(10)\n    )\n    net.initialize(ctx=ctx, init=init.Xavier())\n    return net","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _get_batch(batch, ctx):\n    \"\"\"Return features and labels on ctx.\"\"\"\n    features, labels = batch\n    if labels.dtype != features.dtype:\n        labels = labels.astype(features.dtype)\n    return (gutils.split_and_load(features, ctx),\n            gutils.split_and_load(labels, ctx), features.shape[0])\n\ndef evaluate_accuracy(data_iter, net, ctx=[mx.cpu()]):\n    \"\"\"Evaluate accuracy of a model on the given data set.\"\"\"\n    if isinstance(ctx, mx.Context):\n        ctx = [ctx]\n    acc_sum, n = nd.array([0]), 0\n    for batch in data_iter:\n        features, labels, _ = _get_batch(batch, ctx)\n        for X, y in zip(features, labels):\n            y = y.astype('float32')\n            acc_sum += (net(X).argmax(axis=1) == y).sum().copyto(mx.cpu())\n            n += y.size\n        acc_sum.wait_to_read()\n    return acc_sum.asscalar() / n\n\ndef train(net, train_iter, valid_iter, split, batch_size, num_epochs, ctx):\n    \"\"\"Train the CNN\"\"\"\n    iterations_per_epoch = math.ceil(split / batch_size)\n    schedule = mx.lr_scheduler.FactorScheduler(step=20 * iterations_per_epoch, factor=0.1)\n    rmsprop_optim = mx.optimizer.RMSProp(lr_scheduler=schedule)\n    trainer = gluon.Trainer(net.collect_params(), optimizer=rmsprop_optim)\n    loss = gloss.SoftmaxCrossEntropyLoss()\n    for epoch in range(num_epochs):\n        train_l_sum, train_acc_sum, n, start = 0.0, 0.0, 0, time.time()\n        for X, y in train_iter:\n            y = y.astype('float32').as_in_context(ctx)\n            with autograd.record():\n                y_hat  = net(X.as_in_context(ctx))\n                l = loss(y_hat, y).sum()\n            l.backward()\n            trainer.step(batch_size)\n            train_l_sum += l.asscalar()\n            train_acc_sum += (y_hat.argmax(axis=1) == y).sum().asscalar()\n            n += y.size\n        time_s = \"time %.2f sec\" % (time.time() - start)\n        if valid_iter is not None:\n            valid_acc = evaluate_accuracy(valid_iter, net, ctx)\n            epoch_s = ('epoch %d, loss %f, train acc %f, valid acc %f, ' %\n                      (epoch + 1, train_l_sum / n, train_acc_sum / n, valid_acc))\n        else:\n            epoch_s = ('epoch %d, loss %f, train acc %f, ' %\n                      (epoch + 1, train_l_sum / n, train_acc_sum / n))\n        print(epoch_s + time_s + ', lr ' + str(trainer.learning_rate))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ctx, num_epochs = mx.gpu(0), 50\nnet = get_net(ctx)\nnet.hybridize()\ntrain(net, train_iter, valid_iter, split, batch_size, num_epochs, ctx)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test = nd.array(test.iloc[:, 1:].values.astype(dtype=np.uint8)).reshape(test.shape[0], 28, 28, 1)\ntest_data = gdata.dataset.ArrayDataset(X_test)\ntest_data = test_data.transform_first(transform_valid)\ntest_iter = gdata.DataLoader(test_data, batch_size, shuffle=False, num_workers=0)\n\npreds = []\nfor X in test_iter:\n    y_hat = net(X.as_in_context(ctx))\n    preds.extend(y_hat.argmax(axis=1).astype(int).asnumpy())\nsample_submission = pd.read_csv('../input/Kannada-MNIST/sample_submission.csv')\nsample_submission['label'] = pd.Series(preds)\nsubmission = pd.concat([sample_submission['id'], sample_submission['label']], axis=1)\nsubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}