{"metadata": {"kernelspec": {"language": "python", "name": "python3", "display_name": "Python 3"}, "language_info": {"nbconvert_exporter": "python", "pygments_lexer": "ipython3", "codemirror_mode": {"version": 3, "name": "ipython"}, "file_extension": ".py", "version": "3.6.3", "name": "python", "mimetype": "text/x-python"}, "anaconda-cloud": {}}, "nbformat_minor": 1, "cells": [{"cell_type": "markdown", "metadata": {}, "source": ["This is my first Kaggle kernel. Any feedback is greatly appreciated :)\n", "\n", "*Cheers, \n", "Andrei*"]}, {"cell_type": "code", "metadata": {"_cell_guid": "6098d392-2988-443c-a67f-8e1f00076934", "_uuid": "ac8b12e519b20c568b007f778c5052e8b4a041a1"}, "source": ["# First, let's import some usefull packages\n", "import numpy as np\n", "import os\n", "import keras\n", "import pandas as pd\n", "import matplotlib.pyplot as plt\n", "\n", "\n", "from keras.models import Sequential\n", "from keras.layers import Dense, Dropout, Flatten\n", "from keras.layers import Conv2D, MaxPooling2D\n", "from keras.optimizers import SGD,Adam\n", "from keras.losses import categorical_crossentropy\n", "from keras.metrics import categorical_accuracy\n", "from keras.regularizers import l2\n", "from keras.layers.normalization import BatchNormalization\n", "from keras.callbacks import ModelCheckpoint\n", "from sklearn.metrics import accuracy_score\n", "from keras.preprocessing.image import ImageDataGenerator\n", "from keras import backend as K\n", "K.set_image_dim_ordering('tf')\n", "K.set_image_data_format('channels_last')\n"], "execution_count": null, "outputs": []}, {"cell_type": "code", "metadata": {}, "source": ["os.chdir('../input')\n", "print(os.listdir(os.getcwd()))"], "execution_count": null, "outputs": []}, {"cell_type": "code", "metadata": {"_cell_guid": "7b77aed1-e003-4c79-9c1e-5a9ac12368d6", "_uuid": "e82af61d441f5db664754fd8f325d2f06ae87060"}, "source": ["# let's get the mnist data\n", "test_set=pd.read_csv(r'test.csv')\n", "train_set=pd.read_csv('train.csv')\n", "test_set=test_set.as_matrix()\n", "train_set=train_set.as_matrix()\n", "\n", "\n", "# spit out the shape of the numpy arrays\n", "print('train set {0}, test set {1}'.format(train_set.shape,test_set.shape))"], "execution_count": null, "outputs": []}, {"cell_type": "code", "metadata": {"_cell_guid": "49a34425-1c9b-407f-9bbf-07927c988e45", "_uuid": "927728ccddfc7d1ce27bea7cb477bb7b6b9dce50"}, "source": ["#split into labels and images\n", "labels = train_set[:,0]\n", "imgs = train_set[:,1:]\n", "\n", "#split training data for training and validation\n", "ntrain = 40000\n", "train_labels,val_labels=labels[0:ntrain],labels[ntrain:]\n", "train_imgs,val_imgs=imgs[0:ntrain,:],imgs[ntrain:,:]\n", "\n", "# Lets view one of the images and its label\n", "n=1\n", "print('true label: {}'.format(labels[n]) )\n", "plt.imshow(imgs[n,:].reshape((28,28) ) )\n", "plt.show()\n", "\n"], "execution_count": null, "outputs": []}, {"cell_type": "code", "metadata": {}, "source": ["# Lets get a simple overview of the data we're working with.\n", "print(  'img data shape: train set {0} validation set {1}'.format(\\\n", "     train_imgs.shape,val_imgs.shape)) \n", "print(  'label data shape: train set {0} validation set {1}'.format( \\\n", "    train_labels.shape,val_labels.shape))\n", "print('data type: {0}, max val: {1}, min val: {2}'.format(train_imgs.dtype,train_imgs.max(),train_imgs.min()) )\n", "print('data mean {0}, data std {1}'.format(np.mean(train_imgs) , np.std(train_imgs) ) )"], "execution_count": null, "outputs": []}, {"cell_type": "code", "metadata": {"_cell_guid": "5f3112d6-52b9-4f22-93c6-7bc4b723e48c", "_uuid": "99d984a3e1801ba988209f77df8afa0610e11a51"}, "source": ["# reshape data for CNN\n", "train_imgs_k,val_imgs_k = \\\n", "    np.reshape(train_imgs,(train_imgs.shape[0],28,28,1)),np.reshape(val_imgs,(val_imgs.shape[0],28,28,1))\n", "test_imgs_k = \\\n", "    np.reshape(test_set,(test_set.shape[0],28,28,1))\n", "train_labels_k,val_labels_k=\\\n", "    keras.utils.to_categorical(train_labels, num_classes=10),keras.utils.to_categorical(val_labels, num_classes=10)\n", "\n", "print('imgs reshaped for CNN train/val: {0} / {1}'.format(train_imgs_k.shape,val_imgs_k.shape) )\n", "print('one-hot labels for CNN train/val: {0} / {1}'.format(train_labels_k.shape,val_labels_k.shape) )\n", "\n"], "execution_count": null, "outputs": []}, {"cell_type": "code", "metadata": {"_cell_guid": "7b857bb4-2084-4958-bb3b-28697904b52b", "_uuid": "f1e6331a27b7eeec1ae2ff1e090930640370a097"}, "source": ["# Define a model getter with easily tunable parameters along with some usefull functions\n", "def get_CNN(img_rows, img_cols,LossF ,Metrics,Optimizer=Adam(1e-5),DropP1=0.,DropP2=0.,reg=0.000,batch_norm=False,Activation='relu'):\n", "    print('Optimizer: {0}, DropPerct: {1}, Loss: {2}, reg: {3} batch norm: {4}'.format(Optimizer, (DropP1,DropP2), LossF, reg, batch_norm))\n", "    L2 = l2(reg)\n", "    \n", "    \n", "    model = Sequential()\n", "    # input: 100x100 images with 3 channels -> (100, 100, 3) tensors.\n", "    # this applies 32 convolution filters of size 3x3 each.\n", "    model.add(Conv2D(32, (3, 3), activation=Activation,kernel_regularizer=L2, input_shape=(img_rows, img_cols, 1)))\n", "    if batch_norm: model.add(BatchNormalization())\n", "    model.add(Conv2D(32, (3, 3), activation=Activation,kernel_regularizer=L2))\n", "    if batch_norm: model.add(BatchNormalization())\n", "    model.add(Conv2D(32, (3, 3), activation=Activation,kernel_regularizer=L2))\n", "    if batch_norm: model.add(BatchNormalization())\n", "    model.add(MaxPooling2D(pool_size=(2, 2)))\n", "    model.add(Dropout(DropP1))\n", "\n", "    model.add(Conv2D(64, (3, 3), activation=Activation,kernel_regularizer=L2))\n", "    if batch_norm: model.add(BatchNormalization())\n", "    model.add(Conv2D(64, (3, 3), activation=Activation,kernel_regularizer=L2))\n", "    if batch_norm: model.add(BatchNormalization())\n", "    model.add(Conv2D(32, (3, 3), activation=Activation,kernel_regularizer=L2))\n", "    if batch_norm: model.add(BatchNormalization())\n", "    model.add(MaxPooling2D(pool_size=(2, 2)))\n", "    if batch_norm: model.add(BatchNormalization())\n", "    model.add(Dropout(DropP1))\n", "\n", "    model.add(Flatten())\n", "    model.add(Dense(256, activation=Activation,kernel_regularizer=L2))\n", "    model.add(Dense(256, activation=Activation,kernel_regularizer=L2))\n", "    if batch_norm: model.add(BatchNormalization())\n", "    model.add(Dropout(DropP2))\n", "    model.add(Dense(10, activation='softmax',kernel_regularizer=L2))\n", "    \n", "    model.compile(loss='categorical_crossentropy', optimizer=Optimizer)\n", "    return model\n", "\n", "def evaluate(X,y,model,S=0):\n", "    print ('\\n validation scores: \\n')\n", "    scores = model.evaluate(X,y, batch_size=1, verbose=0)\n", "    if S: txtfile = open('model_eval{0}.txt'.format(i),'w')\n", "    if S: txtfile.write('MODEL EVAL - Patients {0} \\r\\n'.format(patients))\n", "    for score,metric in zip(scores,model.metrics_names):\n", "        print ('{0} score: {1}'.format(metric,score) )\n", "        if S: txtfile.write('{0} score: {1} \\r\\n'.format(metric,score))\n", "    if S: txtfile.close() \n", "    return scores,model.metrics_names\n", "\n", "def plot_loss(hist):\n", "    val_loss = hist['val_loss']\n", "    loss = hist['loss']\n", "\n", "    f, (ax1, ax2) = plt.subplots(1, 2, sharey=True)\n", "    ax1.plot(loss)\n", "    ax1.set_xlabel('epochs')\n", "    ax1.set_ylabel('loss ')\n", "    ax1.set_title('training')\n", "    \n", "    ax2.plot(val_loss)\n", "    ax2.set_title('validation')\n", "    ax2.set_xlabel('epochs')\n", "    ax2.set_ylabel('loss ')\n", "    \n", "    plt.show()\n", "    return\n", "\n", "def plot_loss_tail(hist):\n", "    val_loss = hist['val_loss'][-20:-1]\n", "    loss = hist['loss'][-20:-1]\n", "    f, (ax1, ax2) = plt.subplots(1, 2, sharey=True)\n", "    ax1.plot(loss)\n", "    ax1.set_xlabel('epochs')\n", "    ax1.set_ylabel('loss ')\n", "    ax1.set_title('training')\n", "    \n", "    ax2.plot(val_loss)\n", "    ax2.set_title('validation')\n", "    ax2.set_xlabel('epochs')\n", "    ax2.set_ylabel('loss ')\n", "    \n", "    plt.show()\n", "    return"], "execution_count": null, "outputs": []}, {"cell_type": "code", "metadata": {"scrolled": true, "_cell_guid": "023d69c1-baef-4758-862b-36be27d1ed0a", "_uuid": "4386a62df2a740dac988c1d28760837c98c40210"}, "source": ["#Set model Parameters\n", "img_rows,img_cols = 28,28 # image resolution\n", "\n", "\n", "LossF=categorical_crossentropy # Set the metric for the model's optimization\n", "Metrics=[categorical_accuracy] # calculate model accuracy to make sure it's going in the right direction\n", "Activation = 'relu' # Nonlinear activation\n", "batchNorm= True \n", "Lr = 1.0e-1 # Set learning rate\n", "Optim= SGD(lr=Lr, decay=1e-6, momentum=0.9, nesterov=True) # Adam(lr) \n", "cnn_model = get_CNN(img_rows,img_cols,LossF ,Metrics,Optimizer=Optim,DropP1=0.04,DropP2=0.4,reg=0.0005,batch_norm=batchNorm,Activation=Activation)\n", "\n", "#Set training parameters\n", "N_epochs = 15 # Set low for demo purposes\n", "Batch_Size = 1500 # adjust to fit gpu limitations\n", "\n", "# Set a few callbacks to reduce the learning rate once the model starts to level off\n", "reduce_lr1=keras.callbacks.ReduceLROnPlateau(monitor='val_loss', \\\n", "    factor=0.5, patience=12, verbose=1, mode='auto', epsilon=0.01, cooldown=1000, min_lr=0)\n", "reduce_lr2=keras.callbacks.ReduceLROnPlateau(monitor='val_loss', \\\n", "    factor=0.5, patience=12, verbose=1, mode='auto', epsilon=0.005, cooldown=1000, min_lr=0)\n", "reduce_lr3=keras.callbacks.ReduceLROnPlateau(monitor='val_loss', \\\n", "    factor=0.5, patience=12, verbose=1, mode='auto', epsilon=0.001, cooldown=1000, min_lr=0)\n", "\n", "#stop training before model starts to overfit\n", "early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', \\\n", "    min_delta=0., patience=15, verbose=1, mode='auto') \n", "\n", "# If you're training locally, define a checkpoint to save the model. \n", "# That way, if your model crashes, all is not lost\n", "\n", "#model_checkpoint = ModelCheckpoint('cnn.hdf5', monitor='val_loss', save_best_only=True) \n", "\n", "\n", "CALLBACKS = [early_stop,reduce_lr1,reduce_lr2,reduce_lr3]#+[model_checkpoint]"], "execution_count": null, "outputs": []}, {"cell_type": "code", "metadata": {"_cell_guid": "4ab87d58-3290-467c-9000-45dc99fc4350", "_uuid": "7a18092639908390c2c335b5f25749f5335b025d"}, "source": ["#Define a data generator to augment the training set each epoch\n", "train_datagen = ImageDataGenerator(featurewise_center=False,\n", "    samplewise_center=False,\n", "    featurewise_std_normalization=False,\n", "    samplewise_std_normalization=False,\n", "    zca_whitening=False,\n", "    zca_epsilon=1e-6,\n", "    rotation_range=5.,\n", "    width_shift_range=.05,\n", "    height_shift_range=.05,\n", "    shear_range=20*(3.1416/180),\n", "    zoom_range=0.05,\n", "    data_format=\"channels_last\")\n", "\n", "\n", "train_datagen.fit(train_imgs_k)\n"], "execution_count": null, "outputs": []}, {"cell_type": "code", "metadata": {"_cell_guid": "3c8d813c-7148-4e21-9f40-17bccbbff9c4", "_uuid": "206d1c40dcf3e90cf783a14666328801932e02c0"}, "source": ["# Train the model\n", "Steps = len(train_imgs_k)/Batch_Size\n", "hist=cnn_model.fit_generator(train_datagen.flow(train_imgs_k,train_labels_k, \\\n", "                batch_size=Batch_Size,shuffle=True), steps_per_epoch=Steps, epochs=N_epochs,\\\n", "                 verbose=True,callbacks=CALLBACKS,validation_data=(val_imgs_k,val_labels_k))\n"], "execution_count": null, "outputs": []}, {"cell_type": "code", "metadata": {"_cell_guid": "6ee934cf-1da9-41fe-9272-cf20cc78f61b", "_uuid": "f8f34a09162c7beb00c351c2bf18e361e6565ca5", "collapsed": true}, "source": ["# Plot the progression of the loss function\n", "hist.history\n", "plot_loss(hist.history )\n", "plot_loss_tail(hist.history )"], "execution_count": null, "outputs": []}, {"cell_type": "code", "metadata": {"scrolled": true, "_cell_guid": "6306415a-b432-43e5-a924-938fe4abd051", "_uuid": "11479b8265007fc173facc9cd48d866713e8e61a", "collapsed": true}, "source": ["# Sample a few of the data points along with the predicted label as a sanity check\n", "def sample_results(imgs,labels,model,n=10):\n", "    N = imgs.shape[0]\n", "    samples = np.random.randint(0,high=N,size=n)\n", "    y_pred = model.predict(imgs)\n", "    for s in samples:\n", "        img,lbl,pred=imgs[s],labels[s],y_pred[s]\n", "        print('y_true: {0}, y_pred: {1}'.format(np.argmax(lbl),np.argmax(pred) ) )\n", "        \n", "sample_results(val_imgs_k,val_labels_k,cnn_model)"], "execution_count": null, "outputs": []}, {"cell_type": "code", "metadata": {"_cell_guid": "f1cb1595-203b-472e-967f-60cfc4dc3813", "_uuid": "722700182cb732b5b2b52efeec64fa094f38a965", "collapsed": true}, "source": ["# evaluate the accuracy on the validation/tuning set\n", "pred_proba = cnn_model.predict(val_imgs_k)\n", "y_pred=np.argmax(pred_proba,axis=1)\n", "y_true = np.argmax(val_labels_k,axis=1)\n", "cnn_accuracy = accuracy_score(y_true, y_pred, normalize=True)\n", "print(cnn_accuracy)"], "execution_count": null, "outputs": []}, {"cell_type": "code", "metadata": {"_cell_guid": "f3784410-f615-4724-85e7-27cbc4d9664a", "_uuid": "383b8f63bf2f7e421dcfd725837690d2e6df4c96", "collapsed": true}, "source": ["#predict labels for each image in the test set\n", "pred_proba_test = cnn_model.predict(test_imgs_k)\n", "y_pred_test=np.argmax(pred_proba_test,axis=1)\n"], "execution_count": null, "outputs": []}, {"cell_type": "code", "metadata": {"_cell_guid": "d99ace76-0680-4343-ba8a-cd038ed2e645", "_uuid": "bd4abe7e212c95b78f5c0d456deb211accf68795", "collapsed": true}, "source": ["# Save the results if running at home\n", "# Write to file\n", "fid = open('mnist_sub.csv','w')\n", "fid.write('ImageId,Label\\r\\n')\n", "for i,l in enumerate(y_pred_test):\n", "    fid.write('{0},{1}\\r\\n'.format(i+1,l))\n", "fid.close()\n"], "execution_count": null, "outputs": []}, {"cell_type": "code", "metadata": {"_cell_guid": "5347f5ce-6e3f-47b3-9487-74a015afde08", "_uuid": "7d0b4469998f2ea23d5eb1306c286a248b4c5041", "collapsed": true}, "source": [], "execution_count": null, "outputs": []}, {"cell_type": "code", "metadata": {"_cell_guid": "e800e091-de7a-4274-a5ee-66844a96e6c7", "_uuid": "c876f9faf40b0a5c7ba9ce19dff202566ae04d0e", "collapsed": true}, "source": [], "execution_count": null, "outputs": []}], "nbformat": 4}