{"nbformat_minor": 1, "cells": [{"source": ["This is my first Kaggle Kernel. Any feedback is appreciated."], "metadata": {"_uuid": "299c001add010ee5b86bc44a040a2b28053c61e3", "_cell_guid": "6087ceba-b80e-4bbb-88d8-05d7ed814d58"}, "cell_type": "markdown"}, {"outputs": [], "source": ["import numpy as np\n", "import os\n", "import keras\n", "import pandas as pd\n", "import matplotlib.pyplot as plt"], "metadata": {"_uuid": "0201d4025abd21aa500a498d1ef1c90a2edbdc2b", "collapsed": true, "_cell_guid": "8564265e-9876-487e-8b6c-b5cfb0d27025"}, "cell_type": "code", "execution_count": null}, {"outputs": [], "source": ["#Import Keras\n", "import numpy as np\n", "import keras\n", "from keras.models import Sequential\n", "from keras.layers import Dense, Dropout, Flatten\n", "from keras.layers import Conv2D, MaxPooling2D\n", "from keras.optimizers import SGD,Adam\n", "from keras.losses import categorical_crossentropy\n", "from keras.metrics import categorical_accuracy\n", "from keras.regularizers import l2\n", "from keras.layers.normalization import BatchNormalization\n", "from keras.callbacks import ModelCheckpoint\n", "from sklearn.metrics import accuracy_score\n", "from keras.preprocessing.image import ImageDataGenerator\n", "from keras import backend as K\n", "K.set_image_dim_ordering('tf')\n", "K.set_image_data_format('channels_last')\n"], "metadata": {"_uuid": "76789851c2cc069b0f3b2021b6ef546dd34be597", "collapsed": true, "_cell_guid": "6c0665f4-966d-4d5a-a91a-4e0bc58279df"}, "cell_type": "code", "execution_count": null}, {"outputs": [], "source": ["# let's get the mnist data\n", "test_set=pd.read_csv(r'test.csv')\n", "train_set=pd.read_csv('train.csv')\n", "test_set=test_set.as_matrix()\n", "train_set=train_set.as_matrix()\n", "\n", "\n", "\n", "print('train set {0}, test set {1}'.format(train_set.shape,test_set.shape))"], "metadata": {"_uuid": "55ad06d3feeadf28ab6604f42829b8d47e6eef39", "collapsed": true, "_cell_guid": "8fc22fac-77e9-4c22-9f24-8d1df15b848f"}, "cell_type": "code", "execution_count": null}, {"outputs": [], "source": ["#split into labels and images\n", "labels = train_set[:,0]\n", "imgs = train_set[:,1:]\n", "\n", "#split training data for training and validation\n", "ntrain = 40000\n", "train_labels,val_labels=labels[0:ntrain],labels[ntrain:]\n", "train_imgs,val_imgs=imgs[0:ntrain,:],imgs[ntrain:,:]\n", "\n", "n=1\n", "print('true label: {}'.format(labels[n]) )\n", "plt.imshow(imgs[n,:].reshape((28,28) ) )\n", "plt.show()\n", "\n", "\n", "print(  'img data shape: train set {0} validation set {1}'.format(\\\n", "     train_imgs.shape,val_imgs.shape)) \n", "print(  'label data shape: train set {0} validation set {1}'.format( \\\n", "    train_labels.shape,val_labels.shape))\n", "print('data type: {0}, max val: {1}, min val: {2}'.format(train_imgs.dtype,train_imgs.max(),train_imgs.min()) )\n", "print('data mean {0}, data std {1}'.format(np.mean(train_imgs) , np.std(train_imgs) ) )"], "metadata": {"_uuid": "24d527a9e220ac4af07b6e443a27b2d1cfb7eb37", "collapsed": true, "_cell_guid": "18451b91-0e58-43c3-b91a-32b4e99e466d"}, "cell_type": "code", "execution_count": null}, {"outputs": [], "source": ["# reshape data for CNN\n", "train_imgs_k,val_imgs_k = np.reshape(train_imgs,(train_imgs.shape[0],28,28,1)),np.reshape(val_imgs,(val_imgs.shape[0],28,28,1))\n", "test_imgs_k = np.reshape(test_set,(test_set.shape[0],28,28,1))\n", "train_labels_k,val_labels_k=\\\n", "    keras.utils.to_categorical(train_labels, num_classes=10),keras.utils.to_categorical(val_labels, num_classes=10)\n", "\n", "print('imgs reshaped for CNN train/val: {0} / {1}'.format(train_imgs_k.shape,val_imgs_k.shape) )\n", "print('one-hot labels for CNN train/val: {0} / {1}'.format(train_labels_k.shape,val_labels_k.shape) )\n", "\n"], "metadata": {"_uuid": "f6abc4973e9d3705eac32a67eefdbc48c6e5a079", "collapsed": true, "_cell_guid": "8b658fbd-6f18-4a28-a91e-78514ef0f7c1"}, "cell_type": "code", "execution_count": null}, {"outputs": [], "source": ["#Define some usefull functions\n", "def get_CNN(img_rows, img_cols,LossF ,Metrics,Optimizer=Adam(1e-5),DropP1=0.,DropP2=0.,reg=0.000,batch_norm=False,Activation='relu'):\n", "    print('Optimizer: {0}, DropPerct: {1}, Loss: {2}, reg: {3} batch norm: {4}'.format(Optimizer, (DropP1,DropP2), LossF, reg, batch_norm))\n", "    L2 = l2(reg)\n", "    \n", "    \n", "    model = Sequential()\n", "    # input: 100x100 images with 3 channels -> (100, 100, 3) tensors.\n", "    # this applies 32 convolution filters of size 3x3 each.\n", "    model.add(Conv2D(32, (3, 3), activation=Activation,kernel_regularizer=L2, input_shape=(img_rows, img_cols, 1)))\n", "    model.add(Conv2D(32, (3, 3), activation=Activation,kernel_regularizer=L2))\n", "    model.add(Conv2D(32, (3, 3), activation=Activation,kernel_regularizer=L2))\n", "    model.add(MaxPooling2D(pool_size=(2, 2)))\n", "    if batch_norm: model.add(BatchNormalization())\n", "    model.add(Dropout(DropP1))\n", "\n", "    model.add(Conv2D(64, (3, 3), activation=Activation,kernel_regularizer=L2))\n", "    model.add(Conv2D(64, (3, 3), activation=Activation,kernel_regularizer=L2))\n", "    model.add(Conv2D(32, (3, 3), activation=Activation,kernel_regularizer=L2))\n", "    model.add(MaxPooling2D(pool_size=(2, 2)))\n", "    if batch_norm: model.add(BatchNormalization())\n", "    model.add(Dropout(DropP1))\n", "\n", "    model.add(Flatten())\n", "    model.add(Dense(256, activation=Activation,kernel_regularizer=L2))\n", "    model.add(Dense(256, activation=Activation,kernel_regularizer=L2))\n", "    if batch_norm: model.add(BatchNormalization())\n", "    model.add(Dropout(DropP2))\n", "    model.add(Dense(10, activation='softmax',kernel_regularizer=L2))\n", "    \n", "    model.compile(loss='categorical_crossentropy', optimizer=Optimizer)\n", "    return model\n", "\n", "def evaluate(X,y,model,S=0):\n", "    print '\\n validation scores: \\n'\n", "    scores = model.evaluate(X,y, batch_size=1, verbose=0)\n", "    if S: txtfile = open('model_eval{0}.txt'.format(i),'w')\n", "    if S: txtfile.write('MODEL EVAL - Patients {0} \\r\\n'.format(patients))\n", "    for score,metric in zip(scores,model.metrics_names):\n", "        print '{0} score: {1}'.format(metric,score)\n", "        if S: txtfile.write('{0} score: {1} \\r\\n'.format(metric,score))\n", "    if S: txtfile.close() \n", "    return scores,model.metrics_names\n", "\n", "def plot_loss(hist):\n", "    val_loss = hist['val_loss']\n", "    loss = hist['loss']\n", "\n", "    f, (ax1, ax2) = plt.subplots(1, 2, sharey=True)\n", "    ax1.plot(loss)\n", "    ax1.set_xlabel('epochs')\n", "    ax1.set_ylabel('loss ')\n", "    ax1.set_title('training')\n", "    \n", "    ax2.plot(val_loss)\n", "    ax2.set_title('validation')\n", "    ax2.set_xlabel('epochs')\n", "    ax2.set_ylabel('loss ')\n", "    \n", "    plt.show()\n", "    return\n", "\n", "def plot_loss_tail(hist):\n", "    val_loss = hist['val_loss'][-20:-1]\n", "    loss = hist['loss'][-20:-1]\n", "    f, (ax1, ax2) = plt.subplots(1, 2, sharey=True)\n", "    ax1.plot(loss)\n", "    ax1.set_xlabel('epochs')\n", "    ax1.set_ylabel('loss ')\n", "    ax1.set_title('training')\n", "    \n", "    ax2.plot(val_loss)\n", "    ax2.set_title('validation')\n", "    ax2.set_xlabel('epochs')\n", "    ax2.set_ylabel('loss ')\n", "    \n", "    plt.show()\n", "    return"], "metadata": {"_uuid": "bbcb51b281c1e4b3e42621d6db790277a54108da", "collapsed": true, "_cell_guid": "ca34c428-46e8-459d-accc-6374ee59133e"}, "cell_type": "code", "execution_count": null}, {"outputs": [], "source": [], "metadata": {"_uuid": "a6ff3d281e5a75bac10deb16cf206606c8201db6", "collapsed": true, "_cell_guid": "2158fbd1-efe9-4e65-8632-322aeb2c8a1c"}, "cell_type": "code", "execution_count": null}, {"outputs": [], "source": ["#Set model Parameters\n", "img_rows,img_cols = 28,28 # image resolution\n", "LossF=categorical_crossentropy\n", "Metrics=[categorical_accuracy]\n", "Activation = 'relu'\n", "batchNorm= True\n", "Lr = 1.0e-1\n", "Optim= SGD(lr=Lr, decay=1e-6, momentum=0.9, nesterov=True) # Adam(lr) \n", "cnn_model = get_CNN(img_rows,img_cols,LossF ,Metrics,Optimizer=Optim,DropP1=0.04,DropP2=0.4,reg=0.0005,batch_norm=batchNorm,Activation=Activation)\n", "\n", "#Set training parameters\n", "N_epochs = 250\n", "Batch_Size = 1500 # adjust to fit gpu limitations\n", "reduce_lr1=keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=12, verbose=1, mode='auto', epsilon=0.01, cooldown=1000, min_lr=0)\n", "reduce_lr2=keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=12, verbose=1, mode='auto', epsilon=0.005, cooldown=1000, min_lr=0)\n", "reduce_lr3=keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=12, verbose=1, mode='auto', epsilon=0.001, cooldown=1000, min_lr=0)\n", "\n", "\n", "early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=0., patience=15, verbose=1, mode='auto')\n", "model_checkpoint = ModelCheckpoint('cnn.hdf5', monitor='val_loss', save_best_only=True)\n", "CALLBACKS = [early_stop,model_checkpoint,reduce_lr1,reduce_lr2,reduce_lr3]\n", "\n", "\n"], "metadata": {"_uuid": "0df89c9f51c409d93730e57ff080448a0e5b1ee3", "scrolled": true, "collapsed": true, "_cell_guid": "7c9a8f0e-c1db-464e-b6cf-c5dad7262740"}, "cell_type": "code", "execution_count": null}, {"outputs": [], "source": ["#set parameters for on-line data augmentation\n", "train_datagen = ImageDataGenerator(featurewise_center=False,\n", "    samplewise_center=False,\n", "    featurewise_std_normalization=False,\n", "    samplewise_std_normalization=False,\n", "    zca_whitening=False,\n", "    zca_epsilon=1e-6,\n", "    rotation_range=5.,\n", "    width_shift_range=.05,\n", "    height_shift_range=.05,\n", "    shear_range=20*(3.1416/180),\n", "    zoom_range=0.05,\n", "    data_format=\"channels_last\")\n", "\n", "\n", "train_datagen.fit(train_imgs_k)\n"], "metadata": {"_uuid": "991023678823c122d55186852d8ddb24b276b282", "collapsed": true, "_cell_guid": "a1e14d98-c523-46de-ba6c-77f31070099c"}, "cell_type": "code", "execution_count": null}, {"outputs": [], "source": ["'''\n", "history = cnn_model.fit(train_imgs_k, train_labels_k, batch_size=Batch_Size, epochs=N_epochs,\\\n", "                 verbose=True, shuffle=True,callbacks=CALLBACKS,\\\n", "                 validation_data=(val_imgs_k,val_labels_k))\n", "'''"], "metadata": {"_uuid": "46ccc109bc47e28ab3ebc8205018239e61114e05", "collapsed": true, "_cell_guid": "9278f584-eed8-4259-a3be-3f873d690211"}, "cell_type": "code", "execution_count": null}, {"outputs": [], "source": ["# Train model with data (using on line augmentations)\n", "Steps = len(train_imgs_k)/Batch_Size\n", "hist=cnn_model.fit_generator(train_datagen.flow(train_imgs_k,train_labels_k, batch_size=Batch_Size,shuffle=True), steps_per_epoch=Steps, epochs=N_epochs,\\\n", "                 verbose=True,callbacks=CALLBACKS,\\\n", "                 validation_data=(val_imgs_k,val_labels_k))\n"], "metadata": {"_uuid": "c229056a4f44000d47e651581adda741bce199e7", "collapsed": true, "_cell_guid": "76c3f950-7936-4533-a7ac-6a29a8f682a8"}, "cell_type": "code", "execution_count": null}, {"outputs": [], "source": ["#Plot Training Loss\n", "plot_loss(hist.history )\n", "plot_loss_tail(hist.history )"], "metadata": {"_uuid": "0f0db678046e0c3a972edaac2277d4ba66a299d4", "collapsed": true, "_cell_guid": "76135ee1-45b4-486b-b19b-d32b2fefd8dc"}, "cell_type": "code", "execution_count": null}, {"outputs": [], "source": ["# Sample a few predictions for sanity check\n", "def sample_results(imgs,labels,model,n=10):\n", "    N = imgs.shape[0]\n", "    samples = np.random.randint(0,high=N,size=n)\n", "    y_pred = model.predict(imgs)\n", "    for s in samples:\n", "        img,lbl,pred=imgs[s],labels[s],y_pred[s]\n", "        print('y_true: {0}, y_pred: {1}'.format(np.argmax(lbl),np.argmax(pred) ) )\n", "        \n", "sample_results(val_imgs_k,val_labels_k,cnn_model)"], "metadata": {"_uuid": "f2ba5ac27005edfe516deea4595ddc557e076671", "scrolled": true, "collapsed": true, "_cell_guid": "7b223aec-5e76-47a5-8d8e-24501773d42c"}, "cell_type": "code", "execution_count": null}, {"outputs": [], "source": ["# Generate prediction accuracy for validation data\n", "pred_proba = cnn_model.predict(val_imgs_k)\n", "y_pred=np.argmax(pred_proba,axis=1)\n", "y_true = np.argmax(val_labels_k,axis=1)\n", "cnn_accuracy = accuracy_score(y_true, y_pred, normalize=True)\n", "print(cnn_accuracy)"], "metadata": {"_uuid": "18efc8746311daa126de3d7ffe1d4305a773f4f1", "collapsed": true, "_cell_guid": "f018298b-4cda-4572-94e6-836e2a5224da"}, "cell_type": "code", "execution_count": null}, {"outputs": [], "source": ["#predict labels for test images\n", "pred_proba_test = cnn_model.predict(test_imgs_k)\n", "y_pred_test=np.argmax(pred_proba_test,axis=1)\n"], "metadata": {"_uuid": "555f06f88e727b29ec5a8bb5bde7584c1ec9f8c0", "collapsed": true, "_cell_guid": "4397f5cc-c521-479f-8103-70211a8eb5c5"}, "cell_type": "code", "execution_count": null}, {"outputs": [], "source": ["# Write to file\n", "fid = open('mnist_sub.csv','w')\n", "fid.write('ImageId,Label\\r\\n')\n", "for i,l in enumerate(y_pred_test):\n", "    fid.write('{0},{1}\\r\\n'.format(i+1,l))\n", "fid.close()"], "metadata": {"_uuid": "ff0d58965d2f8ac05d4a342bb4be848223d62018", "collapsed": true, "_cell_guid": "cdb5abd3-e66f-4aa2-9aea-3d45ed5945dc"}, "cell_type": "code", "execution_count": null}], "metadata": {"kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"}, "anaconda-cloud": {}, "language_info": {"mimetype": "text/x-python", "version": "3.6.3", "nbconvert_exporter": "python", "file_extension": ".py", "pygments_lexer": "ipython3", "codemirror_mode": {"version": 3, "name": "ipython"}, "name": "python"}}, "nbformat": 4}