{"cells":[{"metadata":{"trusted":true,"_uuid":"6171210353fb9e0f7fda125c9f9fcb4937e2bdbc","collapsed":true},"cell_type":"code","source":"#################################################################################################\n\n# MNIST - from Machine Learning to Neural Networks, from Neural Networks to Deep Learning\n\n################################################################################################\n\n# author Alexandre Gazagnes\n# update 08/22/18\n# commit v6\n\n# Machine Learning : \n#   first dataset tour\n#   dummy and naive approch\n#   KNN\n#   GridSearchCV and meta parametres\n#   first 'pooling' method\n\n# Neural Networks : \n#   previous features\n#   MLP\n#   learning rate\n#   various data augmentation strategies (shift, flip, add nosise)\n#   activation function\n#   momuntum\n\n# Deep Learning :\n#   previous features\n#   convolutional layes\n#   dropout\n#   batch normalization\n#   loss function\n#   optimization\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"####\n# first we have import, plotting and logging instructions \n####\n\n# import \nimport os, sys, logging, random, time\nfrom itertools import product\nfrom collections import Iterable\n\nimport pandas as pd\nimport numpy as np\n\nfrom matplotlib import pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.decomposition import PCA\nfrom sklearn.preprocessing import StandardScaler, Normalizer, MinMaxScaler\nfrom sklearn.linear_model import Perceptron\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.neural_network import MLPClassifier\nfrom sklearn.svm import LinearSVC, NuSVC\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.dummy import DummyClassifier\nfrom sklearn.metrics import accuracy_score, accuracy_score\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.utils import shuffle\n\nimport tensorflow as tf\n\nimport fastai\n\nfrom keras.utils.np_utils import to_categorical\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D, BatchNormalization\nfrom keras.optimizers import Adam\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import LearningRateScheduler\n\n# constants\nTEST_SIZE = 0.3\nCV        = 5 \n\n# logging and warnings\n# logger = logging.getLogger()\n# logger.setLevel(logging.CRITICAL)\nl = logging.INFO\nlogging.basicConfig(level=l, format=\"%(levelname)s : %(message)s\")\ninfo = logging.info\n# import warnings\n# warnings.filterwarnings('ignore')\n\n# graph spec\n%matplotlib inline\n# get_ipython().magic('matplotlib inline')\nsns.set()\n\n# input data files are available in the \"../input/\" directory.\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b04c4041b9a54e22601641d254bbe1160cdb23ba","collapsed":true},"cell_type":"code","source":"####\n# then we can init our dataframe objects\n####\n\ntrain   = pd.read_csv(\"../input/train.csv\")\ntest    = pd.read_csv(\"../input/test.csv\")\nsample  = pd.read_csv(\"../input/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7cec456fe661004c19ce9f8a6905b8d3a26465cf"},"cell_type":"code","source":"# just for control\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6ac09e26b7098f2fd2ea3f283b6db4033db190c8"},"cell_type":"code","source":"# just for control\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a4ef0a36592f33fea08668c6b1f724583898dfa4"},"cell_type":"code","source":"# just for control\nsample.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"496d27f138e31529a92fd8f73c8fc16f78419ed0"},"cell_type":"code","source":"# let's see more about our df\nprint(train.ndim)\nprint(train.shape)\nprint(train.max().max())\nprint(train.min().min())\nprint(train.isna().any().any())\nprint(train.isna().any().describe())\nprint((785-1) **0.5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3d908f2eb6acc900ab48d3ab085061e63ba41574"},"cell_type":"code","source":"# about our labels\nprint(train.label.value_counts(normalize=True))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d9ce973b2f7355a979899e975f58715dae122a11"},"cell_type":"code","source":"####\n# it is better to have a visual repr of numbers\n####\n\ndef show_numbers(df, n=10, dim=28) : \n    fig, ax = plt.subplots(1, n,  figsize=(25,15))\n    for i in range(n) : \n        df_img = np.array(df.iloc[i, :]).reshape(dim, dim)\n        ax[i].imshow(df_img, cmap=\"gray\")\n        \n####\n\nshow_numbers(train.iloc[:20, 1:])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1986397cbe28a47a54fc9b947b333abd636341f2"},"cell_type":"code","source":"# 10 samples of each\nfor i in range(10) : \n    _df = (train.loc[train.label == i, :]).iloc[:10, :].drop(\"label\", axis=1)\n    show_numbers(_df, 10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e0b3f3ef353f51cf0a821beae51ceeea329cfd17"},"cell_type":"code","source":"####\n# we now are going to test a dummy and a a naive model to define our worst performance level\n####\n\n# split X and y\nX, y = train.drop(\"label\", axis=1), train.label\nprint(X.shape)\nprint(y.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"997eb7eb92409bdd550294eb86f6add94b1d1a3c"},"cell_type":"code","source":"# we will be able to deal with a 42000 * 800 matrix but it is a little bit messy\n# it is better to sample this dataframe\n\ndef reduce_size(X,y=None,batch_size = 0.3) : \n    size = int(batch_size *len(X))\n    if isinstance(y, Iterable) : \n        X,y = shuffle(X,y)\n        X = X.iloc[:size, :]\n        y = y.iloc[:size]\n        return X,y\n    else : \n        X = shuffle(X)\n        X = X.iloc[:size, :]\n        return X\n    \n####\n\nX,y = reduce_size(X,y,0.05)\nprint(X.shape, y.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d06a8186294dc737f0b154c984fee1b3772425ff"},"cell_type":"code","source":"####\n# then we can separate test and train data\n####\n\nX_train, X_test, y_train, y_test = train_test_split(X,y, test_size=TEST_SIZE)\nprint([i.shape for i in (X_train, X_test, y_train, y_test)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5d8924238522c37c9545f6cd646459a368460e25"},"cell_type":"code","source":"####\n# using DummyClassifier from sklearn\n####\n\ndef dummy_score(X_train, X_test, y_train, y_test) : \n    dummy = DummyClassifier()\n    dummy.fit(X_train, y_train)\n    y_pred = dummy.predict(X_test)\n    return accuracy_score(y_test, y_pred)\n\n####\n\nnp.array([dummy_score(X_train, X_test, y_train, y_test) for i in range(100)]).mean().round(2)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true,"_uuid":"97bcd85ac26a45cb6e5b573b86cbd360e8833cd4","collapsed":true},"cell_type":"code","source":"# ok, our dummy classifer do not give us a good representation of our basic performance...","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e10443cfbcef78551befc7149ed6b5f3c9fb4459"},"cell_type":"code","source":"######\n# let's try a brutal GridSearchCV on KNN\n######\n\n# params_all = dict(  n_neighbors=[1,2,3,4,5,6,7,8,9,10], algorithm=['auto', 'ball_tree', 'kd_tree', 'brute'], \n#                     metric=['euclidean', 'manhattan', 'chebyshev', 'minkowski', 'wminkowski', 'seuclidean', 'mahalanobis'],\n#                     p = [1,2])\n\nparams_dict = dict(n_neighbors=[1,3,5], algorithm=['auto'])\ngrid = GridSearchCV(KNeighborsClassifier(), params_dict, cv=CV)\ngrid.fit(X_train, y_train)\ny_pred = grid.predict(X_test)\naccuracy_score(y_test, y_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a8cbcf1f306accb0361b71da40cbc17e2d9052e8"},"cell_type":"code","source":"# ok, and so on...\nprint(grid.best_score_, grid.best_params_)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"27632cc022dee59073af5bea8d8d9768fc4f8ecc","_kg_hide-output":true,"trusted":true,"collapsed":true},"cell_type":"code","source":"# ok, we can now say that our basic performance is about 0.85-0.90","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"collapsed":true},"cell_type":"code","source":"# let's try various algo\nparams_dict = dict(  n_neighbors=[1,3,5], algorithm=['auto', 'ball_tree', 'kd_tree', 'brute'])\ngrid = GridSearchCV(KNeighborsClassifier(), params_dict, cv=CV)\ngrid.fit(X_train, y_train)\ny_pred = grid.predict(X_test)\naccuracy_score(y_test, y_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"47435af4696041a88f00e556e445201c69199524"},"cell_type":"code","source":"print(grid.best_params_)\nprint(grid.best_score_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aed553110258f53711de2febb37fa4eb49c5929a","collapsed":true},"cell_type":"code","source":"####\n# we are able to reduce the number of examples, but what about feature's space ?\n# is it possible to have a much smaller dimension space, without loosing a lot of information?\n####\n\n# define a pooling function\ndef pooling_vector(v) : \n    size = len(v)**0.5\n    if size %2  : raise ValueError(\"not implemented\")\n    size = int(size)\n    _size = int((size/2))    \n    \n    old_mat = np.array(v).reshape(size,size)\n    new_mat = np.zeros(_size**2).reshape(_size, _size)\n\n    idx = range(len(old_mat[0]))\n    for i,j in product(idx, idx) : \n        if (i%2 or j%2): continue   \n        n = np.array([  old_mat[i,j],old_mat[i,j+1],\n                            old_mat[i+1,j],old_mat[i+1,j+1]]).max()\n        new_mat[int(i/2), int(j/2)] = n\n    return new_mat.flatten()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f241d9dbe08e4d9e10395b5281bbf157611e0e61"},"cell_type":"code","source":"# just have a look, and try this new function\n\n# create a test vector/matrix\ntest_vect = np.zeros(28**2).reshape(28,28)\ntest_vect[:,19] = test_vect[11,:] = 127\n\n# test_vect is a matrix full of 0, with a cross of 127 in the middle \ntest_vect = test_vect.flatten()\nprint(len(test_vect))\n\n# apply this transformation \nnew_vect = pooling_vector(test_vect)\nprint(len(new_vect))\n\n# control that our cross of 888 is still there\nprint(new_vect.reshape(14,14))\nprint(784/4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b7db695fb96d4c6c857fe6098eca074de80531eb"},"cell_type":"code","source":"# we can control visualy the action of pooling\n_test_vect = test_vect.reshape(28,28)\n_new_vect = new_vect.reshape(14,14)\nfig, ax = plt.subplots(1, 2,  figsize=(15,8))\nax[0].imshow(_test_vect)\nax[1].imshow(_new_vect)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8f7d570cb45104954e772443a68c8814fd643426"},"cell_type":"code","source":"# not bad but if we try with a real number, it is much more better\n\nnum_before = np.array(X_train.iloc[0, :]).reshape(28,28)\nnumb_after = pooling_vector(np.array(X_train.iloc[0, :])).reshape(14,14) \nfig, ax = plt.subplots(1, 2,  figsize=(15,8))\nax[0].imshow(num_before, cmap=\"gray\")\nax[1].imshow(numb_after, cmap=\"gray\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9dd085a52ceaf12aef2f5deac1777cf07dcf4793"},"cell_type":"code","source":"# Ok for one vector of features, but for the entire dataset? \nnew_X = pd.DataFrame(0, index = X.index, columns=range(int(len(X.columns)/4)))\nnew_X.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c557e9dd2ea5aabe6f1cf501685e22ec96266aba"},"cell_type":"code","source":"# control shape \nprint(new_X.shape)\nprint(new_X.shape[1] * 4)\nprint(X.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"15b2fd627142beef9596e107e36a0cd7043eab4b"},"cell_type":"code","source":"# awfull way to proceed, but easily understadable \nfor i in X.index : new_X.loc[i, :] = pooling_vector(X.loc[i, :])\nnew_X.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dc1de659375aa0d521945791eb99f630e58542f6"},"cell_type":"code","source":"# just have a look\nprint(new_X.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"60b3c210fd47db36dd84aaa8059ceeb9831d35f2"},"cell_type":"code","source":"# are indexes still good?\n(new_X.index == y.index).all()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"293a63ac78b0d79ab0bf5ed63549f4ed92920010"},"cell_type":"code","source":"# change our dataframe\nX = new_X.copy()\n\nX_train, X_test, y_train, y_test = train_test_split(X,y, test_size=TEST_SIZE)\nprint([i.shape for i in (X_train, X_test, y_train, y_test)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8e6170d9f63170d56ec985dd6a002f511b5c9c37"},"cell_type":"code","source":"# compute loss of accuracy, dividing features space by 4 !  \n\nparams_dict = dict(n_neighbors=[1,3,5], algorithm=['auto'])\ngrid = GridSearchCV(KNeighborsClassifier(), params_dict, cv=CV)\ngrid.fit(X_train, y_train)\ny_pred = grid.predict(X_test)\naccuracy_score(y_test, y_pred)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2cbee4d19e007598d1fbb2a2390e7c5373eb97d2","_kg_hide-output":true,"trusted":true,"collapsed":true},"cell_type":"code","source":"# ok, we can say that the performance is still good but the model is much more faster!","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9bf0ab3cc70ed6e041a16724c8ba3a2bfefd6192","collapsed":true},"cell_type":"code","source":"# function to pool a entire dataframe in one command\ndef pooling_dataframe(X) : \n    if \"label\" not in X.columns :\n        new_X = pd.DataFrame(0, index = X.index, columns=range(int(len(X.columns)/4)))\n        for i in X.index : new_X.loc[i, :] = pooling_vector(X.loc[i, :])\n        # new_X = X.apply(pooling_vector, axis=1)\n    else : \n        y = X.label\n        X = X.drop(\"label\", axis=1)\n        new_X = pd.DataFrame(0, index = X.index, columns=range(int(len(X.columns)/4)))\n        for i in X.index : new_X.loc[i, :] = pooling_vector(X.loc[i, :])\n        # new_X = X.apply(pooling_vector, axis=1)\n        new_X[\"label\"] = y\n    return new_X","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fbbde18396fd64f18cb02c2b34e1fe9c16ffd463"},"cell_type":"code","source":"####\n# we now are able to compute much more faster, so we can increase our example numbers...\n####\n\n_train = train.copy()\nprint(_train.shape)\n_train = reduce_size(_train, batch_size=0.3)\nX, y = _train.drop(\"label\", axis=1), _train.label\nprint(X.shape, y.shape)\n\nX_before = X.copy()\nX = pooling_dataframe(X)\nX_after = X.copy()\n\nX_train, X_test, y_train, y_test = train_test_split(X,y, test_size=TEST_SIZE)\nprint([i.shape for i in (X_train, X_test, y_train, y_test)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f7acb83dcbbd46830cd25f7c7a24ccb74e2445d5"},"cell_type":"code","source":"# control our lost information\nshow_numbers(X_before, dim=28)\nshow_numbers(X_after, dim=14)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"662c9afa2dfd6de03cab1a193f4093d237524950","collapsed":true},"cell_type":"code","source":"# interlude\nassert (1+1)==2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"68bde7140d432e7014b05403f3e38144c9eb89db"},"cell_type":"code","source":"# and try to evaluate global performance.\n\nparams_dict = dict(n_neighbors=[1,3,5], algorithm=['auto'])\ngrid = GridSearchCV(KNeighborsClassifier(), params_dict, cv=CV)\ngrid.fit(X_train, y_train)\ny_pred = grid.predict(X_test)\naccuracy_score(y_test, y_pred)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"97079abb78c0ac927eb88c0f7e9b92cb75cd1c67","_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"# and we are happy to observe a really big step in our performance quest!\nprint(grid.best_score_, grid.best_params_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ecd912d3cd1fd07f6738af924cf97408c8db7f19"},"cell_type":"code","source":"# and if we reduce drasitcly our train dataset vs our test_dataset\n\nX, y = train.drop(\"label\", axis=1), train.label\nprint(X.shape, y.shape)\n\nX,y = reduce_size(X,y, 0.3)\nprint(X.shape, y.shape)\n\nX = pooling_dataframe(X)\n\nX_train, X_test, y_train, y_test = train_test_split(X,y, test_size=0.7)\nprint([i.shape for i in (X_train, X_test, y_train, y_test)])\n\nparams_dict = dict(n_neighbors=[1,3,5], algorithm=['auto'])\ngrid = GridSearchCV(KNeighborsClassifier(), params_dict, cv=CV)\ngrid.fit(X_train, y_train)\ny_pred = grid.predict(X_test)\naccuracy_score(y_test, y_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cd76b252d5ef905441a4b5a206cfe98bfab1523d"},"cell_type":"code","source":"print(grid.best_score_, grid.best_params_)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":false,"trusted":true,"_uuid":"6959d46791e9985d3170f6b99f2370fbb297097b","collapsed":true},"cell_type":"code","source":"####\n# do not forget to submit :/\n####\n\n# test.head()\n# test.shape\n\n# X = test\n# X = pooling_dataframe(X)\n# y_pred = grid.predict(X)\n\n# subs_data = pd.DataFrame({'Label':y_pred})\n# subs_data.index+=1\n# subs_data.index.name='ImageId'\n# subs_data.to_csv('../input/submission0.csv',index=True)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true,"_uuid":"391059f4125d8c80aef2b54ae95407fd4e243d74","collapsed":true},"cell_type":"code","source":"# we could try to find out the best KNN configuration, searchingg the best meta\n# parametres and trying to upgrade our accuracy score 0.001 by 0.0001\n# but let's go deeper and try to have a look at our first Neural Networks model : MLP","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"06a2806c2242184121d79e8fde19ca933903cf38"},"cell_type":"code","source":"####\n# We are now having a look at Neural Networks, specialy the easiest one : MLP multi layer perceptron\n####\n\n# let's draw our worst model\n\nX, y = train.drop(\"label\", axis=1), train.label\nX,y = reduce_size(X,y, 0.3)\n\nX_train = pooling_dataframe(X_train)\nX_train, X_test, y_train, y_test = train_test_split(X,y, test_size=TEST_SIZE)\n\nmlp = MLPClassifier()\nmlp.fit(X_train, y_train)\ny_pred = mlp.predict(X_test)\naccuracy_score(y_test, y_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"437ec125eb7b39f8d4c48e3a7079992532a8a805"},"cell_type":"code","source":"# not so good, let's add a GridSearchCV\ngrid = GridSearchCV(MLPClassifier(), {}, cv=CV)\ngrid.fit(X_train, y_train)\ny_pred = grid.predict(X_test)\naccuracy_score(y_test, y_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f26792d52ccf4fbfaeb7658ec80ccec2ba2580b2"},"cell_type":"code","source":"print(grid.best_score_, grid.best_params_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"132f56e36f473d6ba9b9fc1b4f37c8d1b2be19bc"},"cell_type":"code","source":"# let's try one specific model\nmlp = MLPClassifier((150, 150, 20))\ngrid = GridSearchCV(mlp, {}, cv=CV)\ngrid.fit(X_train, y_train)\ny_pred = grid.predict(X_test)\naccuracy_score(y_test, y_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e44ecc3653446e53e7906cfc0a624fea67d418b4"},"cell_type":"code","source":"print(grid.best_score_, grid.best_params_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dd0a0b22925dba90f3745e2d020c024d09af1745"},"cell_type":"code","source":"# second one \nmlp = MLPClassifier((200,200,200))\ngrid = GridSearchCV(mlp, {}, cv=CV)\ngrid.fit(X_train, y_train)\ny_pred = grid.predict(X_test)\naccuracy_score(y_test, y_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c8225818e1a31be5b092f2891c92785eaee77387"},"cell_type":"code","source":"print(grid.best_score_, grid.best_params_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"111a170bb25282a08dd397a2277573fe082a7e23"},"cell_type":"code","source":"# and just to try what about a MLP with full dataset, all featuress? \n\nX, y = train.drop(\"label\", axis=1), train.label\nprint(X.shape, y.shape)\n\n# X,y = reduce_size(X,y, 0.3)\n# print(X.shape, y.shape)\n# X = pooling_dataframe(X)\n\nX_train, X_test, y_train, y_test = train_test_split(X,y, test_size=TEST_SIZE)\nprint([i.shape for i in (X_train, X_test, y_train, y_test)])\n\nmlp = MLPClassifier((200, 200, 200, 100, 50))\ngrid = GridSearchCV(mlp, {}, cv=CV)\ngrid.fit(X_train, y_train)\ny_pred = grid.predict(X_test)\naccuracy_score(y_test, y_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b3a9c2a0f4d430382b9e07b69b853d4afacd9a3d"},"cell_type":"code","source":"print(grid.best_score_, grid.best_params_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a3c4f72a11e4e8fda095be91bdcc51dc72811eef"},"cell_type":"code","source":"# same thing just with our pooling function\n\nX, y = train.drop(\"label\", axis=1), train.label\nprint(X.shape, y.shape)\n\nX = pooling_dataframe(X)\nprint(X.shape, y.shape)\nX_train, X_test, y_train, y_test = train_test_split(X,y, test_size=TEST_SIZE)\nprint([i.shape for i in (X_train, X_test, y_train, y_test)])\n\nmlp = MLPClassifier((200, 200, 200, 100, 50))\ngrid = GridSearchCV(mlp, {}, cv=CV)\ngrid.fit(X_train, y_train)\ny_pred = grid.predict(X_test)\naccuracy_score(y_test, y_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3ffe2b3bad35cd904df1ef470600d85c361db567"},"cell_type":"code","source":"print(grid.best_score_, grid.best_params_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"acf0eb1e44e682b382da47561e294c50c3af3564"},"cell_type":"code","source":"####\n# we are now focus on data augmentation, first thing first : data augmentation fo every number\n####\n\ndef shift_left(v, n=2, dim=28) :\n    old_matrix = np.array(v).reshape(dim,dim)\n    new_matrix = np.zeros(dim**2).reshape(dim,dim)\n    new_matrix[:, :-n]   = old_matrix[:,n:]\n    new_matrix[:, -n:]     = left  = old_matrix[:, :n]\n    return new_matrix.flatten()    \n\ndef shift_right(v, n=2, dim=28) : \n    old_matrix = np.array(v).reshape(dim,dim)\n    new_matrix = np.zeros(dim**2).reshape(dim,dim)\n    new_matrix[:, n:]   = old_matrix[:,:-n]\n    new_matrix[:, :n] = left  = old_matrix[:, -n:]\n    return new_matrix.flatten()\n\ndef shift_up(v, n=2, dim=28):\n    old_matrix = np.array(v).reshape(dim,dim)\n    new_matrix = np.zeros(dim**2).reshape(dim,dim)\n    new_matrix[:-n, :]   = old_matrix[n:,:]\n    new_matrix[-n:, :] = left  = old_matrix[:n,: ]\n    return new_matrix.flatten()\n\ndef shift_down(v, n=2, dim=28) : \n    old_matrix = np.array(v).reshape(dim,dim)\n    new_matrix = np.zeros(dim**2).reshape(dim,dim)\n    new_matrix[n:, :]   = old_matrix[:-n,:]\n    new_matrix[:n, :] = old_matrix[-n:,: ]\n    return new_matrix.flatten()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c572fe56e4569358fa29855e8d33906cc162d7a3"},"cell_type":"code","source":"# control shift action \nfig, ax = plt.subplots(1,5, figsize=(25,15))\nv = train.iloc[0, 1:]\nax[0].imshow(np.array(v).reshape(28,28), cmap=\"gray\")\nax[1].imshow(np.array(shift_left(v, n=3)).reshape(28,28), cmap=\"gray\")\nax[2].imshow(np.array(shift_right(v, n=3)).reshape(28,28), cmap=\"gray\")\nax[3].imshow(np.array(shift_up(v, n=3)).reshape(28,28), cmap=\"gray\")\nax[4].imshow(np.array(shift_down(v, n=3)).reshape(28,28), cmap=\"gray\")\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8b213577c99069ab9b9480213e3b348510ad784c","collapsed":true},"cell_type":"code","source":"####\n# then 0,1 and 8 can be \"flipped\" horizontaly or verticaly\n####\n\ndef flip_verticaly(v, dim=28) : \n    old_matrix = np.array(v).reshape(dim,dim)\n    new_matrix = np.zeros(dim**2).reshape(dim,dim)\n    for i in range(dim) : \n        new_matrix[i, :]   = old_matrix[(dim-1)-i,:]\n    return new_matrix.flatten()\n\ndef flip_honrizontaly(v, dim=28) : \n    old_matrix = np.array(v).reshape(dim,dim)\n    new_matrix = np.zeros(dim**2).reshape(dim,dim)\n    for i in range(dim) : \n        new_matrix[:, i]   = old_matrix[:,(dim-1)-i]\n    return new_matrix.flatten()\n\ndef flip_both(v, dim=28) :\n    return flip_honrizontaly(flip_verticaly(v,dim=dim), dim=dim)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"98e371b3bc885c2e1b7d526f5bd779f9d349695e"},"cell_type":"code","source":"# control flip action \nfor i in [0, 8, 1] : \n    fig, ax = plt.subplots(1,8, figsize=(20,10))\n    v = (train.loc[train.label==i, :]).iloc[0, 1:]\n    v2 = (train.loc[train.label==i, :]).iloc[1, 1:]\n    ax[0].imshow(np.array(v).reshape(28,28), cmap=\"gray\")\n    ax[1].imshow(np.array(flip_verticaly(v)).reshape(28,28), cmap=\"gray\")\n    ax[2].imshow(np.array(flip_honritontaly(v)).reshape(28,28), cmap=\"gray\")\n    ax[3].imshow(np.array(flip_both(v)).reshape(28,28), cmap=\"gray\")\n    ax[4].imshow(np.array(v2).reshape(28,28), cmap=\"gray\")\n    ax[5].imshow(np.array(flip_verticaly(v2)).reshape(28,28), cmap=\"gray\")\n    ax[6].imshow(np.array(flip_honritontaly(v2)).reshape(28,28), cmap=\"gray\")\n    ax[7].imshow(np.array(flip_both(v2)).reshape(28,28), cmap=\"gray\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4e6940d02b095374693d0ea3fd0797cde16c3751","collapsed":true},"cell_type":"code","source":"####\n# we now combine shift and flip, not on one vect but on one dataframe\n####\n\ndef shift_dataframe(df, y=None, dim=28) : \n    if (\"label\" in df.columns) and (not isinstance(y, Iterable)) : \n        label = df.label \n        _df = df.drop(\"label\", axis=1)\n        tdf = _df.copy()\n    elif (\"label\" not in df.columns) and (isinstance(y, Iterable))  :\n        _df = df.copy()\n        tdf = _df.copy()         \n    elif (\"label\" not in df.columns) and (not isinstance(y, Iterable)):\n        raise ValueError(\"Error\")\n    \n    for transf in [shift_right, shift_left, shift_up, shift_down] :     \n        _df = _df.append(tdf.apply(transf, axis=1, dim=dim), ignore_index=True)\n                      \n    if (\"label\" in df.columns) and (not isinstance(y, Iterable)) : \n        df = _df.copy()\n        df[\"label\"] = 5 * list(label.values)\n        return df\n    elif (\"label\" not in df.columns) and (isinstance(y, Iterable))  :\n        df = _df.copy()\n        y = 5 * list(y.values)\n        return df, y          \n                                          \ndef flip_dataframe(df, y=None, dim=28) :     \n    mask = ((df.label == 0) | (df.label ==1) | (df.label == 8))\n    _df = df.loc[mask, :]\n    df = df.loc[~mask, :]\n\n    if (\"label\" in df.columns) and (not isinstance(y, Iterable)) : \n        _label = _df.label                            \n        _df = df.drop(\"label\", axis=1)\n        tdf = _df.copy()\n    elif (\"label\" not in df.columns) and (isinstance(y, Iterable))  :\n        _df = df.copy()\n        tdf = _df.copy()\n        y = y.loc[~mask] \n        _y = y.loc[mask]\n    elif (\"label\" not in df.columns) and (not isinstance(y, Iterable)):\n        raise ValueError(\"Error\")                                          \n                                        \n    for transf in [flip_verticaly, flip_honrizontaly, flip_both] :     \n        _df = _df.append(tdf.apply(transf, axis=1, dim=dim), ignore_index=True)\n               \n    if (\"label\" in df.columns) and (not isinstance(y, Iterable)) : \n        lab = list(df.label.values)\n        lab.extend(4 * list(_label.values))\n        df = df.drop(\"label\", axis=1).append(_df, ignore_index=True)\n        df[\"label\"] = lab\n        return df\n    elif (\"label\" not in df.columns) and (isinstance(y, Iterable))  :\n        df = _df.copy()\n        y = list(y.values)                                  \n        y.extend(4 * list(_y.values))\n        return df, y    \n                                          \ndef augment_dataframe(df,y=None, dim=28) : \n    return flip_dataframe(shift_dataframe(df,y=y,dim=dim),y=y, dim=dim)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7cf90bfd648d67e8062b4d7475f67cc660ca2f6a"},"cell_type":"code","source":"# control shift \ndf = train.iloc[:2, :]\ndf= shift_dataframe(df,dim=28)\nl = len(df)\ndf = df.drop(\"label\", axis=1)\n\nfig, ax = plt.subplots(1,l, figsize = (20,10))\nfor i,_ in enumerate(df.index) : \n    ax[i].imshow(df.iloc[i, :].values.reshape(28,28), cmap=\"gray\")\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b1b63a107e0ce4986ac0424075253f863f0cab65"},"cell_type":"code","source":"# control flip\ndf = train.iloc[:2, :]\ndf= flip_dataframe(df,dim=28)\nl = len(df)\nprint(l)\ndf = df.drop(\"label\", axis=1)\n\nfig, ax = plt.subplots(1,l, figsize = (20,10))\nfor i,_ in enumerate(df.index) : \n    ax[i].imshow(df.iloc[i, :].values.reshape(28,28), cmap=\"gray\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7c6a439915c55dc6090260f16eb7461b124ce449"},"cell_type":"code","source":"# control both\ndf = train.iloc[:2, :]\ndf= augment_dataframe(df)\nl = len(df)\nprint(list(df.label))\ndf = df.drop(\"label\", axis=1)\nl == 2 * 5 * 4\nfig, ax = plt.subplots(1,l, figsize = (40,20))\nfor i,_ in enumerate(df.index) : \n    ax[i].imshow(df.iloc[i, :].values.reshape(28,28), cmap=\"gray\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e67922a50cc15c353f59531171177b72cea68636"},"cell_type":"code","source":"def perturb_vector(v, rate=0.02, hard_method=True):\n    \n    if hard_method  : method = lambda i : 0 if i >128 else 255\n    else :            method = lambda i : np.random.randint(0,255)\n        \n    l, n= len(v), int(len(v) * rate)\n    idxs = n * [1] ; idxs.extend((l-n )*[0])\n    idxs = shuffle(idxs)\n    v = np.array([(j if not i else method(j)) for i, j in zip(idxs,v)])\n    return v","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eabc2dd7dc79d150de9325aca95208991940a189"},"cell_type":"code","source":"# control this fucntion\nv = np.arange(128-50, 128+50)\nv2 = perturb_vector(v)\nv2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0f7ddfe776c8a9c64ab5f960ec6b724afa73fdad"},"cell_type":"code","source":"def perturb_dataframe(X, rate=0.02, hard_method=True) : \n    if \"label\" not in X.index :\n        new_X = X.copy()\n        new_X = X.apply(perturb_vector, axis=1, rate=rate, hard_method=hard_method)\n        return new_X\n    else : \n        y = X.label\n        X = X.drop(\"label\", axis=1)\n        new_X = X.copy()\n        new_X = X.apply(perturb_vector, axis=1, rate=rate, hard_method=hard_method)\n        new_X[\"label\"] = y\n        return new_X\n        \n# FACTORIZATION !!!!","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ac9108f9def9011abcc95a3336cc9bf2575523e9"},"cell_type":"code","source":"# we can show the \"noise\" created with or without 'hard_method'\n\ndf = train.iloc[:10].drop(\"label\", axis=1)\nshow_numbers(df)\ndf = perturb_dataframe(df, hard_method=True)\nshow_numbers(df)\n\ndf = train.iloc[:10].drop(\"label\", axis=1)\ndf = perturb_dataframe(df, hard_method=False)\nshow_numbers(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b06efe5d33ee19eb2618c04ac23a33a41cb8cfd0"},"cell_type":"code","source":"# we now are able to perturb before or after pooling\n\ndf = train.iloc[:10].drop(\"label\", axis=1)\nshow_numbers(df,dim=28)\ndf = pooling_dataframe(df)\nshow_numbers(df,dim=14)\n_df = perturb_dataframe(df, hard_method=True)\nshow_numbers(_df, dim=14)\n_df = perturb_dataframe(df, hard_method=False)\nshow_numbers(_df, dim=14)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"54a62715b7d2848e0dc052769c2c19b6535b4b98"},"cell_type":"code","source":"df = train.iloc[:10].drop(\"label\", axis=1)\nshow_numbers(df,dim=28)\ndf = perturb_dataframe(df, hard_method=True)\nshow_numbers(df, dim=28)\ndf = pooling_dataframe(df)\nshow_numbers(df,dim=14)\n\ndf = train.iloc[:10].drop(\"label\", axis=1)\nshow_numbers(df,dim=28)\ndf = perturb_dataframe(df, hard_method=False)\nshow_numbers(df, dim=28)\ndf = pooling_dataframe(df)\nshow_numbers(df,dim=14)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"95136632b9c1ed5c70bf406798f545015301d7af"},"cell_type":"code","source":"# we can that perturbing and then pooling is not good method, neither using \"soft\" method\n# we can just pool and then use pertubating hard method","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"28f91e05614e9ec83c2d34006ee7e5d1cc0d9ea8"},"cell_type":"code","source":"# lets now try this perturb without pooling\n\nX, y = train.drop(\"label\", axis=1), train.label\nprint(X.shape, y.shape)\n\nX,y = reduce_size(X,y, 0.3)\nprint(X.shape, y.shape)\n# X = pooling_dataframe(X)\n\nX_train, X_test, y_train, y_test = train_test_split(X,y, test_size=TEST_SIZE)\nprint([i.shape for i in (X_train, X_test, y_train, y_test)])\nX_train = perturb_dataframe(X_train, hard_method=True)\n\nmlp = MLPClassifier((200, 200, 200, 100, 50))\ngrid = GridSearchCV(mlp, {}, cv=CV)\ngrid.fit(X_train, y_train)\ny_pred = grid.predict(X_test)\naccuracy_score(y_test, y_pred)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6e7116c040ef58dcccc1b267310e21ba3a725674"},"cell_type":"code","source":"# lets now try this pooling and then perturb \n\nX, y = train.drop(\"label\", axis=1), train.label\nprint(X.shape, y.shape)\n\nX,y = reduce_size(X,y, 0.3)\nprint(X.shape, y.shape)\nX = pooling_dataframe(X)\n\nX_train, X_test, y_train, y_test = train_test_split(X,y, test_size=TEST_SIZE)\nprint([i.shape for i in (X_train, X_test, y_train, y_test)])\nX_train = perturb_dataframe(X_train, hard_method=True)\n\nmlp = MLPClassifier((200, 200, 200, 100, 50))\ngrid = GridSearchCV(mlp, {}, cv=CV)\ngrid.fit(X_train, y_train)\ny_pred = grid.predict(X_test)\naccuracy_score(y_test, y_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"033f2e504bc60606d2f7c0815b860eef6e08c8d6"},"cell_type":"code","source":"# finally let see the infulence of data augmentation \n\n_train = shift_dataframe(train)\nX, y = _train.drop(\"label\", axis=1), _train.label\nprint(X.shape, y.shape)\n\nX,y = reduce_size(X,y, 0.3)\nprint(X.shape, y.shape)\nX = pooling_dataframe(X)\n\nX_train, X_test, y_train, y_test = train_test_split(X,y, test_size=TEST_SIZE)\nprint([i.shape for i in (X_train, X_test, y_train, y_test)])\nX_train = perturb_dataframe(X_train, hard_method=True)\n\nmlp = MLPClassifier((200, 200, 200, 100, 50))\ngrid = GridSearchCV(mlp, {}, cv=CV)\ngrid.fit(X_train, y_train)\ny_pred = grid.predict(X_test)\naccuracy_score(y_test, y_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"31e01458ac4e2e7cc66a22fd474fe7d60adaae57","_kg_hide-output":true,"collapsed":true},"cell_type":"code","source":"# we could of course try to find out the best MLP configuration ever but again, but even if we could reach 0.96, 0.97 accuracy score\n# it is better to go depper and to have look at Deep learning and scpecialy Convolutional Neural Networks","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d73926406619c851586ae4b11031f22d39ad2070"},"cell_type":"code","source":"####\n# we now are going to use our preprocessing tools in a 'real' deep learning networks\n# we will strat with a very simple CNN model, of 6 hidden layers, build with tensor flow\n# then we will build much more complex/performant model using keras\n####\n\n# classic\n_train = train.copy()\nprint(_train.shape)\n# _train = reduce_size(_train, batch_size=0.3)\nprint(_train.shape)\nX, y = _train.drop(\"label\", axis=1), _train.label\nprint(X.shape, y.shape)\nX_train, X_test, y_train, y_test = train_test_split(X,y, test_size=0.15)\nprint([i.shape for i in (X_train, X_test, y_train, y_test)])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e8e9e4cada4bdc70ec376c572aa1badf7039eefb"},"cell_type":"code","source":"# preprocessing of our train set \n_train = X_train.copy()\nprint(_train.shape)\nprint(y_train.shape)\n_train[\"label\"] = y_train\n_train = shift_dataframe(_train)\nprint(_train.shape)\n# _train = flip_dataframe(_train)\nX_train, y_train = _train.drop(\"label\", axis=1), _train.label\nprint(X_train.shape, y_train.shape)\nX_train = perturb_dataframe(X_train, hard_method=True)\n\nX_test_len = len(X_test)\nX_train_len = len(X_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"96054edadb480ec079c8cb8838846328eb18f801"},"cell_type":"code","source":"# now we need to reshape \nX_train = X_train.values.reshape(-1, 28, 28, 1)\nX_test  = X_test.values.reshape(-1, 28, 28, 1)\ny_train = y_train.values.reshape(-1, 1)\ny_test  = y_test.values.reshape(-1, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8df114af5c8b5e2ab912c8c2b7648995c954cc13","collapsed":true},"cell_type":"code","source":"# split and reshape X and y\n# X = train.values[:,1:].reshape(-1, 28, 28, 1)\n# y = train.values[:,:1].reshape(-1, 1)\n\n# split test train\n\n# test_size = 0.4\n# L = len(train)\n# X_test_len = int(L*test_size)\n# X_train_len = L - X_test_len\n\n# X_train, X_test = X[: N,:,:,:], X[N:,:,:,:]\n# y_train, y_test = y[:N, :], y[N:, :]\n\n# L = 40000\n# N = 2000\n\n# X_train = train.values[X_test_len:,1:].reshape(-1, 28, 28, 1)\n# y_train = train.values[X_test_len:,:1].reshape(-1, 1)\n\n# X_test = train.values[:X_test_len,1:].reshape(-1, 28, 28, 1)\n# y_test = train.values[:X_test_len,:1].reshape(-1, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"02da75efebd1dc9a6b59d38a0de4c5b3af85a696","collapsed":true},"cell_type":"code","source":"# then we need to define our placesholders\n\nx = tf.placeholder(dtype=tf.float32, shape=[None, 28, 28, 1])\ny = tf.placeholder(dtype=tf.float32, shape=[None, 10])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"991f6e55753d55d76dd2fb8d0f61a04ed16c51ec"},"cell_type":"code","source":"#####\n# here we draw our layer model\n####\n\n_layer = x\n_filters = [128, 128, 128, 256, 256, 512]\n\nfor f in _filters: \n    _layer = tf.layers.conv2d(_layer, filters=f, kernel_size=3, padding=\"SAME\")\n    _layer = tf.nn.relu(_layer)\n    _layer = tf.nn.max_pool(_layer, ksize=(1, 2, 2, 1), strides=(1,2,2,1), padding=\"SAME\")\n    _layer = tf.nn.dropout(_layer, 0.5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d4f117d15ead7f3ac9a222e7212b7a9fccd73ec6","collapsed":true},"cell_type":"code","source":"####\n# we just chnage the last layer and the output layer\n####\n\n_layer = tf.layers.conv2d(_layer, filters=1024, kernel_size=3, padding=\"SAME\")\n_layer = tf.nn.relu(_layer)\n_layer = tf.nn.max_pool(_layer, ksize=(1, 2, 2, 1), strides=(1,2,2,1), padding=\"SAME\")\n_layer = tf.reshape(_layer, [-1, 1024])\n\nout = tf.layers.dense(_layer, units=10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aa3295f9f6df38ca7fb2425dd9977e70523e6610","collapsed":true},"cell_type":"code","source":"####\n# our loss function\n####\n\nloss = tf.reduce_mean(0.5 * tf.nn.softmax_cross_entropy_with_logits_v2(logits=out, labels=y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1459b99a60930effaf16341e0cf4b689ce62d9c9","collapsed":true},"cell_type":"code","source":"####\n#our optimizer\n####\n\noptimizer = tf.train.AdamOptimizer().minimize(loss)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fe42b2f3ca648663dbb9481eaef6d6e8b085d5cd","collapsed":true},"cell_type":"code","source":"#####\n# our batch parser\n#####\n\ndef next_batch(batch_size, training):\n    array_x = []\n    array_y = []\n    \n    for i in range(batch_size):\n        index = np.random.randint(0, X_train_len)\n        array_x.append(X_train[index])\n        array_y.append(np.eye(10)[y_train[index]].reshape(10))\n    return { x: array_x, y: array_y}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2722ff6c4eec28e30ef895235519436120863825","collapsed":true},"cell_type":"code","source":"####\n# our global params\n####\n\nepochs  = 12000 # 100, 500, 1000, 3000, 8000, 12000\nsess    = tf.Session()\nsess.run(tf.global_variables_initializer())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"21bc76c3fc22bbc0033f020b8015d52cce261399","collapsed":true},"cell_type":"code","source":"####\n# finally the training session\n####\n\nfor i in range(epochs):\n    train_batch = next_batch(128, True)\n    _ = sess.run([optimizer], feed_dict=train_batch)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c4a006dd916821c4af612394cfa09b506d8a788b"},"cell_type":"code","source":"# we can compute accuracy\n\ntest_pred = np.array([np.argmax(item) for item in sess.run([out], feed_dict={x:X_test})[0]]).reshape(-1, 1)\nprint(np.sum(np.equal(y_test, test_pred))/len(test_pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"96d8e7493eebb99e74a0fb5cb5d5a87ab1ab0539"},"cell_type":"code","source":"test_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0b4bf6d20c7cf6301a0937714459eaeb82945177"},"cell_type":"code","source":"y_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"84a6b8207093386ba6d39aaa4a0e6076bab6f09c"},"cell_type":"code","source":"\"\"\"\ndata = train.copy()\ndel train\ndata = input_data.read_data_sets(DATA_DIR, one_hot=True)\ndata = input_data.read_data_sets(\"../input\", one_hot=True)\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"199910da7051c0fcf440ea43a4bd1dae6aff3c3a","collapsed":true},"cell_type":"code","source":"\"\"\"\ny_true = tf.placeholder (tf.float32, [None, 10])\ny_pred = tf.matmul(x, W)\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"15ec29f489f9c8238c0a90f4c181196edd3bf83e","collapsed":true},"cell_type":"code","source":"\"\"\"\nx = tf.placeholder(tf.float32, [None, 784])\nW = tf.Variable(tf.zeros([784, 10]))\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e703ff133b6e158f60d8dd361ca6bf194c5a0a07","collapsed":true},"cell_type":"code","source":"\"\"\"\ncross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=y_pred, labels=y_true))\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a5f1d8c4ec6e2779a993c0c625487ed47bd4e35c","collapsed":true},"cell_type":"code","source":"\"\"\"\ngd_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)\ncorrect_mask = tf.equal(tf.argmax(y_pred, 1), tf.argmax(y_true, 1))\naccuracy = tf.reduce_mean(tf.cast(correct_mask, tf.float32))\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6f5af742016c2cea4ae4f4c8094af0ef839f0ee9","collapsed":true},"cell_type":"code","source":"\"\"\"\nwith tf.Session() as sess:\n    sess.run(tf.global_variables_initializer())\n    for _ in range(NUM_STEPS):\n        batch_xs, batch_ys = data.train.next_batch(MINIBATCH_SIZE)\n        sess.run(gd_step, feed_dict={x: batch_xs, y_true: batch_ys})\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f3bf4175cd956ff16dda046fc434ff5b1a559d31","collapsed":true},"cell_type":"code","source":"\"\"\"\nans = sess.run(accuracy, feed_dict={x: data.test.images,y_true: data.test.labels})\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f97211957f07c60db190dccf1e7ce5e8b510754b","collapsed":true},"cell_type":"code","source":"\"\"\"\nprint(\"Accuracy: {:.4}\".format(ans*100))\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ec70ddcef6c409d026da4f7b49896fca149e0c98","collapsed":true},"cell_type":"code","source":"# CNN","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a763215816d9bc72da6c0222f8aa45259617b548","collapsed":true},"cell_type":"code","source":"def weight_variable(shape):\n    initial = tf.truncated_normal(shape, stddev=0.1)\n    return tf.Variable(initial)\n\n\ndef bias_variable(shape):\n    initial = tf.constant(0.1, shape=shape)\n    return tf.Variable(initial)\n\n\ndef conv2d(x, W):\n    return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')\n\n\ndef max_pool_2x2(x):\n    return tf.nn.max_pool(x, ksize=[1, 2, 2, 1],\n                          strides=[1, 2, 2, 1], padding='SAME')\n\n\ndef conv_layer(input_, shape):\n    W = weight_variable(shape)\n    b = bias_variable([shape[3]])\n    return tf.nn.relu(conv2d(input_, W) + b)\n\n\ndef full_layer(input_, size):\n    in_size = int(input_.get_shape()[1])\n    W = weight_variable([in_size, size])\n    b = bias_variable([size])\n    return tf.matmul(input_, W) + b","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1404b3226eef7f8783f32e8cda2d9bb44302d18b","collapsed":true},"cell_type":"code","source":"\"\"\"\nDATA_DIR = '/tmp/data'\nMINIBATCH_SIZE = 50\nSTEPS = 5000\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"76538308b285d381511a46baa5ec92ca17dfce52","collapsed":true},"cell_type":"code","source":"\"\"\"\nmnist = input_data.read_data_sets(DATA_DIR, one_hot=True)\nx = tf.placeholder(tf.float32, shape=[None, 784])\ny_ = tf.placeholder(tf.float32, shape=[None, 10])\nx_image = tf.reshape(x, [-1, 28, 28, 1])\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f95cd638de55f84020ca40960495c655695eff34","collapsed":true},"cell_type":"code","source":"\"\"\"\nconv1 = conv_layer(x_image, shape=[5, 5, 1, 32])\nconv1_pool = max_pool_2x2(conv1)\n\nconv2 = conv_layer(conv1_pool, shape=[5, 5, 32, 64])\nconv2_pool = max_pool_2x2(conv2)\n\nconv2_flat = tf.reshape(conv2_pool, [-1, 7*7*64])\nfull_1 = tf.nn.relu(full_layer(conv2_flat, 1024))\n\"\"\"\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"631ce0dd9ed2ca9814ccd085da355ce39d75245c","collapsed":true},"cell_type":"code","source":"\"\"\"\nkeep_prob = tf.placeholder(tf.float32)\nfull1_drop = tf.nn.dropout(full_1, keep_prob=keep_prob)\ny_conv = full_layer(full1_drop, 10)\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8954df8d24323b3614f89591a0eb75e073f29353","collapsed":true},"cell_type":"code","source":"\"\"\"\ncross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=y_conv, labels=y_))\ntrain_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)\ncorrect_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1))\naccuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ef2a62bb8e1196081f8e37071e24d87678005a94","collapsed":true},"cell_type":"code","source":"\"\"\"\nwith tf.Session() as sess:\n    sess.run(tf.global_variables_initializer())\n\n    for i in range(STEPS):\n        batch = mnist.train.next_batch(MINIBATCH_SIZE)\n\n        if i % 100 == 0:\n            train_accuracy = sess.run(accuracy, feed_dict={x: batch[0], y_: batch[1],\n                                                           keep_prob: 1.0})\n            print(\"step {}, training accuracy {}\".format(i, train_accuracy))\n\n        sess.run(train_step, feed_dict={x: batch[0], y_: batch[1], keep_prob: 0.5})\n\n    X = mnist.test.images.reshape(10, 1000, 784)\n    Y = mnist.test.labels.reshape(10, 1000, 10)\n    test_accuracy = np.mean(\n        [sess.run(accuracy, feed_dict={x: X[i], y_: Y[i], keep_prob: 1.0}) for i in range(10)])\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"50f4807497e78bca2b4e86fde9eccf4c059858c6","collapsed":true},"cell_type":"code","source":"\"\"\"\nprint(\"test accuracy: {}\".format(test_accuracy))\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"07bc93f99e15b48136f1c6707e9c7127400e60f5"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"8f19fb11ecb1da8c8ea085f9c0f90f3206817825"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"df5124156f1e01a85866b5b77aa0239da67a9f3b"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}