{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"from numpy.random import seed\nseed(1)\nimport tensorflow as tf\ntf.compat.v1.set_random_seed\nimport random as rn\nrn.seed(4)\n\nimport numpy as np\nimport keras\nimport sys\nimport pandas as pd\nimport matplotlib\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Activation, Dropout, Flatten, Conv2D, MaxPooling2D\nfrom keras.utils import to_categorical\nimport matplotlib.pyplot as plt\nfrom keras.layers import BatchNormalization\nfrom keras.constraints import unit_norm\nfrom keras import regularizers\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import LearningRateScheduler\nimport matplotlib.cm as cm\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv(\"../input/digit-recognizer/train.csv\")\n#print(df_train.head())\ndf_test = pd.read_csv(\"../input/digit-recognizer/test.csv\")\n\n#(train_x, train_y), (test_x, test_y) = mnist.load_data(label_mode='fine')\n\n\ndtr = df_train.values\ndtest = df_test.values\ntrain_x = dtr[:,1:]\ntrain_y = dtr[:,0]\n#val_x = dtr[9000:,:-1]\n#val_y = dtr[9000:,-1]\ntest_x = dtest[:,:]\n#test_y = dtest[:,-1]\n\n\n#Preprocessing\n#train_x = train_x.reshape((train_x.shape[0], 3, -1))\n#test_x = test_x.reshape((test_x.shape[0], 3, -1))\n#val_x = val_x.reshape((test_x.shape[0], 1, -1))\n\ntrain_x = train_x/255.0\ntest_x = test_x/255.0\n#val_x = val_x/255.0\n#print(train_x.shape)\n#train_x = np.concatenate([train_x[:, i, None].reshape(train_x.shape[0], -1, 1) for i in range(3)], axis=2)\n#test_x = np.concatenate([test_x[:, i, None].reshape(test_x.shape[0], -1, 1) for i in range(3)], axis=2)\n#val_x = np.concatenate([val_x[:, i, None].reshape(val_x.shape[0], -1, 1) for i in range(3)], axis=2)\n\ntrain_x = train_x.reshape((train_x.shape[0],28,28,1))\ntest_x = test_x.reshape((test_x.shape[0],28,28,1))\n\n\nprint(train_x.shape,test_x.shape)\n\n\ntrain_y = to_categorical(train_y)\nprint(train_y.shape)\n#test_y = to_categorical(test_y)\n\n#plt.imsave('filename.png', train_x[0].reshape(28,28), cmap=cm.gray)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def lrcheck(epoch):\n    lr = 0.001\n    if epoch > 75:\n        lr = 0.0005\n    if epoch > 100:\n        lr = 0.0003\n    return lr\n\n\n\nweight_decay = 1e-4\nmodel = Sequential()\nmodel.add(Conv2D(32, (3,3), padding='same', kernel_regularizer=regularizers.l2(weight_decay), input_shape=(28,28,1)))\nmodel.add(Activation('elu'))\nmodel.add(BatchNormalization())\nmodel.add(Conv2D(32, (3,3), padding='same', kernel_regularizer=regularizers.l2(weight_decay)))\nmodel.add(Activation('elu'))\nmodel.add(BatchNormalization())\nmodel.add(MaxPooling2D(pool_size=(2,2)))\nmodel.add(Dropout(0.2))\n \nmodel.add(Conv2D(64, (3,3), padding='same', kernel_regularizer=regularizers.l2(weight_decay)))\nmodel.add(Activation('elu'))\nmodel.add(BatchNormalization())\nmodel.add(Conv2D(64, (3,3), padding='same', kernel_regularizer=regularizers.l2(weight_decay)))\nmodel.add(Activation('elu'))\nmodel.add(BatchNormalization())\nmodel.add(MaxPooling2D(pool_size=(2,2)))\nmodel.add(Dropout(0.3))\n \nmodel.add(Conv2D(128, (3,3), padding='same', kernel_regularizer=regularizers.l2(weight_decay)))\nmodel.add(Activation('elu'))\nmodel.add(BatchNormalization())\nmodel.add(Conv2D(128, (3,3), padding='same', kernel_regularizer=regularizers.l2(weight_decay)))\nmodel.add(Activation('elu'))\nmodel.add(BatchNormalization())\nmodel.add(MaxPooling2D(pool_size=(2,2)))\nmodel.add(Dropout(0.4))\n \nmodel.add(Flatten())\nmodel.add(Dense(10, activation = 'softmax'))\n \nmodel.summary()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''datagen = ImageDataGenerator(\n        featurewise_center=False,  # set input mean to 0 over the dataset\n        samplewise_center=False,  # set each sample mean to 0\n        featurewise_std_normalization=False,  # divide inputs by std of the dataset\n        samplewise_std_normalization=False,  # divide each input by its std\n        zca_whitening=False,  # apply ZCA whitening\n        zca_epsilon=1e-06,  # epsilon for ZCA whitening\n        rotation_range= 0,  # randomly rotate images in the range (degrees, 0 to 180)\n        # randomly shift images horizontally (fraction of total width)\n        width_shift_range=0.1,\n    # randomly shift images vertically (fraction of total height)\n    height_shift_range=0.1,\n    shear_range=0.,  # set range for random shear\n    zoom_range=0.,  # set range for random zoom\n    channel_shift_range=0.,  # set range for random channel shifts\n    # set mode for filling points outside the input boundaries\n    fill_mode='nearest',\n    cval=0.,  # value used for fill_mode = \"constant\"\n    horizontal_flip=True,  # randomly flip images\n    vertical_flip=False,  # randomly flip images\n        # set rescaling factor (applied before any other transformation)\n    rescale=None,\n        # set function that will be applied on each input\n    preprocessing_function=None,\n        # image data format, either \"channels_first\" or \"channels_last\"\n    data_format=None,\n        # fraction of images reserved for validation (strictly between 0 and 1)\n    validation_split=0.0)\n\n    # Compute quantities required for feature-wise normalization\n    # (std, mean, and principal components if ZCA whitening is applied).\ndatagen.fit(train_x)'''\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"    # Fit the model on the batches generated by datagen.flow().\n''' history = model.fit_generator(datagen.flow(train_x, train_y,\n                                     batch_size = 400),\n                                     epochs = 50,\n                                     workers=4,\n                                     steps_per_epoch = 120)'''\n\nbatch_size = 400\n\nopt = keras.optimizers.RMSprop(lr=0.001,decay=1e-6)\nmodel.compile(loss='categorical_crossentropy', optimizer = opt, metrics=['accuracy'])\n#history = model.fit_generator(datagen.flow(train_x, train_y, batch_size= batch_size),\\\n#                   steps_per_epoch = 1000, epochs = 40,\\\n#                    verbose = 1, callbacks = [LearningRateScheduler(lrcheck)])\nhistory = model.fit(train_x,train_y, validation_split = 0.10, batch_size = batch_size, epochs = 3,verbose = 1, callbacks = [LearningRateScheduler(lrcheck)], shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# list all data in history\nprint(history.history.keys())\n# summarize history for accuracy\nplt.plot(history.history['accuracy'])\nplt.plot(history.history['val_accuracy'])\nplt.title('model accuracy')\nplt.ylabel('Accuracy')\nplt.xlabel('Epoch')\nplt.legend(['Train', 'Val'], loc='upper left')\nplt.savefig('accuracy.png')\nplt.show()\n\n\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('Model loss') \nplt.ylabel('Loss')\nplt.xlabel('Epoch')\nplt.legend(['Train', 'Val'], loc='upper left')\nplt.savefig('loss.png')\nplt.show()\n\ndef convert(train_y):\n    target = np.zeros((len(train_y),1), dtype = int)\n    for i in range(len(train_y)):\n        target[i][0] = int(np.argmax(train_y[i]))\n    return target\n\nprediction = model.predict(test_x)\ntarget = convert(prediction)\n\n\nnp.savetxt(\"predictions.txt\", target, delimiter = '\\n')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}