{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nimport functools\nimport keras\nfrom keras import backend as K\nimport PIL\nimport IPython\nimport math\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom keras.layers import Dense, Input\nfrom keras.layers import Conv2D, Flatten, Lambda\nfrom keras.layers import Reshape, Conv2DTranspose\nfrom keras.models import Model\nimport pandas as pd\nfrom keras.losses import mse, binary_crossentropy,sparse_categorical_crossentropy\nfrom keras.utils import plot_model\nimport  logging\nlogging.basicConfig(filename='out.log',level=logging.INFO)\nlogger=logging.getLogger()\nimport math\n\n\nimport  datetime\ndate_depart=datetime.datetime.now()\nprint(os.listdir(\"../input\"))\nduree_max=datetime.timedelta(hours=5,minutes=45)\ndate_limite= date_depart+duree_max\nepochs=int(1e8)\nval_split=0.1\nload_weights=False\nfrac_gen_epoch=4\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"train_df=pd.read_csv(\"../input/digit-recognizer/train.csv\",dtype=\"int16\")\ntrain_df=train_df.sample(frac=1)\nval_size=int(np.ceil( len(train_df)*val_split) )\nXtrain=train_df.values[:-val_size,1:]\nXtrain=Xtrain.reshape(-1,28,28,1)\nYtrain=train_df.label.values[:-val_size]\ntrain_df.info()\ntrain_df.head()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fa01c61faa5368f95a5633952195929236e42088"},"cell_type":"code","source":"n_fig=10\n\nsample=np.random.choice(Xtrain.shape[0],n_fig)\n\n\nfig=plt.figure(figsize=(15,15), dpi=100)\nlignes=1\ncols=n_fig//lignes\nif cols==0:\n    cols=n_fig\nelif n_fig%lignes!=0:\n    cols+=1\nfor i,idx in enumerate(sample):\n    \n    \n    plt.subplot(lignes,cols,i+1)\n    plt.imshow(Xtrain[idx,...,0],cmap=\"Greys\", interpolation=\"nearest\",vmin=-5, vmax=260)\n    plt.title(Ytrain[idx])\n\n    \nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"541d062f2c5fcb8fdcf26156741d319cc9d0dd82"},"cell_type":"code","source":"Xval=train_df.values[-val_size:,1:]\nXval=Xval.reshape(-1,28,28,1)\nYval=train_df.label.values[-val_size:]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0a1dfd75ff2604dfc1ae13f3363eade439f76d9a","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"test_df=pd.read_csv(\"../input/digit-recognizer/test.csv\",dtype=\"int16\")\nXtest=test_df.values\nXtest=Xtest.reshape(-1,28,28,1)\ntest_df.info()\ntest_df.head()\n","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"297e447c7a0f65790d806bc57d9c3c93525369fc"},"cell_type":"code","source":"class termination_date(keras.callbacks.Callback ):\n    def __init__(self,end_date):\n        self.end_date=end_date\n    def on_epoch_end(self, batch, logs=None):\n        if datetime.datetime.now()>self.end_date:\n            self.model.stop_training = True\n            logging.info(\"end date\")\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5e9e83eb455a6c84519a70aa82eef5e079aa7a68"},"cell_type":"code","source":"def covariance(x):\n    n=K.get_variable_shape(x)[0]\n    xm=x-K.mean(x,axis=0)\n    K.get_variable_shape(x)\n    return K.dot(K.transpose(xm),xm)/n\ndef corellation(x):\n    return covariance(x)/K.var(x,axis=0)\n\ndef corellation_mid(x):\n    n=K.get_variable_shape(x)[1]    \n    return corellation(x)-np.identity(n)\n\ndef corellation_regul(*args,**kvargs):    \n    return lambda x: keras.regularizers.l1_l2(*args,**kvargs)(corellation_mid(x))\ndef regul_combi(l1corr=0.01,l2corr=0.01,*args,**kvargs):\n    reg=keras.regularizers.l1_l2(*args,**kvargs)\n    return lambda x: reg(x)+corellation_regul(l1corr,l2corr)(x)\n\n    \n    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6ede82da2c89f42b47cf88cb643ddb4967a19a96","scrolled":true},"cell_type":"code","source":"basel1=1e-5\nbasel2=2e-3\nreguls=keras.regularizers.l1_l2(basel1,2e-3)\nregulsfc=keras.regularizers.l1_l2(basel1,2e-3)\nregulsfc1=regul_combi(1e-3,2e-1,basel1,basel2*20)\ninp=Input(shape=(28,28,1), name=\"image\", dtype=\"float32\")\n\ninp1=keras.layers.ZeroPadding2D(padding=2)(inp)\ninp2=keras.layers.GaussianNoise(10)(inp1)\n\ninp3=keras.layers.Lambda( lambda x:x/256\n                        )(inp2)\nc1pre=Conv2D(12,3,\n          strides=1,\n          padding='same',\n          kernel_regularizer=None, \n          activation=\"selu\",\n          name=\"c1pre\"\n         )(inp3)\n\nc1pre1=keras.layers.concatenate([c1pre,inp3])\nc1pre1=keras.layers.SpatialDropout2D(0.2)(c1pre1)\nc1=Conv2D(80,7,\n          strides=1,\n          padding='same',\n          kernel_regularizer=None, \n          activation=\"selu\",\n          name=\"c1\"\n         )(c1pre1)\n\nc2=Conv2D(100,1,\n          strides=1,\n          kernel_regularizer=reguls,            \n          activation=\"selu\",\n           padding='same',\n          name=\"c2\"\n         )(c1)\nc2c=keras.layers.concatenate([c2,c1pre1])\nc2c=keras.layers.BatchNormalization()(c2c)\nc2c=keras.layers.SpatialDropout2D(0.32)(c2c)\n\nc3=Conv2D(80,5,\n          strides=2,\n          kernel_regularizer=reguls, \n          activation=\"selu\",\n           padding='same',\n          name=\"c3\"\n         )(c2c)\nc3a=Conv2D(60,1,\n          strides=1,\n          kernel_regularizer=reguls, \n          activation=\"selu\",\n           padding='same',\n          name=\"c3a\"\n         )(c3)\nc4=Conv2D(80,5,\n          strides=2,\n          kernel_regularizer=reguls, \n          activation=\"selu\",\n           padding='same',\n          name=\"c4\",\n          \n         )(c3a)\nc4a=Conv2D(80,1,\n          strides=1,\n          kernel_regularizer=reguls, \n          activation=\"selu\",\n           padding='same',\n          name=\"c4a\",\n          \n         )(c4)\nc4a1=keras.layers.concatenate([c4a,c4])\nc4a2=keras.layers.SpatialDropout2D(0.3)(c4a1)\nc5=Conv2D(200,5,\n          strides=2,\n          kernel_regularizer=reguls, \n          activation=\"selu\",\n           padding='same',\n          name=\"c5\",\n          \n         )(c4a2)\n\n\npool_c2=keras.layers.GlobalMaxPooling2D(name=\"pooling_c2\")(c2)\npool=keras.layers.GlobalMaxPooling2D(name=\"pooling_c5\")(c5)\npool_c=keras.layers.concatenate([pool_c2,pool])\npool_c=keras.layers.BatchNormalization()(pool_c)\n\npool_c=keras.layers.Dropout(0.3)(pool_c)\n\nhidd=Dense(200,\n        activation=\"selu\", \n        kernel_regularizer=regulsfc,\n        name=\"hidd\"\n       )(pool_c)\n\no=Dense(10,\n        activation=\"softmax\", \n        kernel_regularizer=regulsfc1,\n        name=\"softmax\"\n       )(hidd)\nmodel=Model(inputs=inp,outputs=o)\n\noptimizer=keras.optimizers.Adam(clipnorm=1. , clipvalue=1.)\ntop2acc=functools.partial(keras.metrics.sparse_top_k_categorical_accuracy,k=2)\ntop2acc.__name__=\"sparse_top_2_categorical_accuracy\"\n\n\nmodel.compile(optimizer,\n              loss=\"sparse_categorical_crossentropy\",\n              metrics=[\"sparse_categorical_accuracy\",\"sparse_categorical_crossentropy\",top2acc]\n             )\n\n\nif load_weights:\n    model.load_weights(\"../input/digit-test/digits.h5\", by_name=True, skip_mismatch=True, reshape=False)\n    if os.path.exists(\"digits.h5\"):\n        model.load_weights(\"digits.h5\", by_name=True, skip_mismatch=True, reshape=False)\nmodel.summary()\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a9702668fc06f101c4a9f954a54a1201493513af"},"cell_type":"code","source":"plot_model(model,show_shapes=True, show_layer_names=True)\nIPython.display.Image(filename='model.png')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b2fc588a3f385070332e606da7410bfa6be72b55"},"cell_type":"code","source":"fichier_modele=\"digits.h5\"\nbatch_size=512\n\nval_batch_size=batch_size\n\n\nsteps_per_epoch=Xtrain.shape[0]/batch_size*2.5/frac_gen_epoch\nvalidation_steps=Xval.shape[0]/val_batch_size\nsteps_per_epoch=int(steps_per_epoch)\nvalidation_steps=int(validation_steps)+2\n\ndef noise_degrade(im, scale=40):\n    im+=np.random.normal(loc=0.0, scale=scale, size=im.shape)\n    im+=np.random.uniform(-scale*2,scale*2, size=im.shape)\n    return im\n\nnoise_degrade_r=functools.partial(noise_degrade,scale=5)\nimg_gen_base=keras.preprocessing.image.ImageDataGenerator()\nimg_gen_aug=keras.preprocessing.image.ImageDataGenerator(zoom_range=[0.95,1.2],\n                                                     rotation_range=10,\n                                                    brightness_range=[0.8,1.2],\n                                                         rescale=0,\n                                                     shear_range=0.02,\n                                                     width_shift_range=0.02,\n                                                     height_shift_range=0.02,\n                                                            preprocessing_function=noise_degrade,\n                                                     validation_split=0.1)\n\nimg_gen_aug_fine1=keras.preprocessing.image.ImageDataGenerator(zoom_range=[0.99,1.05],\n                                                     rotation_range=1,\n                                                    brightness_range=[0.95,1.05],\n                                                         rescale=0,\n                                                     shear_range=0.005,\n                                                     width_shift_range=0.005,\n                                                     height_shift_range=0.005,\n                                                              preprocessing_function=noise_degrade_r,\n                                                     validation_split=0.1)\nimg_gen_aug_fine=keras.preprocessing.image.ImageDataGenerator(zoom_range=[0.99,1.05],\n                                                     rotation_range=1,\n                                                    brightness_range=[0.95,1.05],\n                                                         rescale=0,\n                                                     shear_range=0.005,\n                                                     width_shift_range=0.005,\n                                                     height_shift_range=0.005,\n                                                              \n                                                     validation_split=0.1)\n\n\n\n\n\ncallbacks=[\n        keras.callbacks.ReduceLROnPlateau(monitor='val_sparse_categorical_accuracy',\n                                          patience=10,\n                                          min_delta=0.0005,\n                                          #factor=0.2,\n                                          min_lr=1e-6,\n                                          verbose=1,\n                                          cooldown=20\n\n                                          ),\n        keras.callbacks.ModelCheckpoint(monitor='val_sparse_categorical_accuracy',\n                                        filepath=fichier_modele,\n                                        verbose=1,\n                                        save_best_only=True,\n                                        period=20),\n        keras.callbacks.EarlyStopping(monitor='val_sparse_categorical_accuracy',\n                                      patience=100,\n                                         \n                                          verbose=1,\n                                          restore_best_weights=True\n\n                                          ),\n         keras.callbacks.CSVLogger(\"train.csv\", separator=',', append=True),\n         termination_date(date_limite)\n\n        ]\ndef resc_gen(genbase):\n    for X,Y in genbase:\n      \n        yield (X-128,Y)\n\n\ntrain_gen=img_gen_aug.flow(Xtrain, Ytrain, batch_size=batch_size, shuffle=True)\ntrain_gen_fine=img_gen_aug_fine.flow(Xtrain, Ytrain, batch_size=batch_size, shuffle=True)\ntrain_gen_fine1=img_gen_aug_fine.flow(Xtrain, Ytrain, batch_size=batch_size, shuffle=True)\nval_gen=img_gen_base.flow(Xval, Yval, batch_size=val_batch_size, shuffle=True)\n\n#train_gen=resc_gen(train_gen)\n#train_gen_fine=resc_gen(train_gen_fine)\n#train_gen_fine1=resc_gen(train_gen_fine1)\n\n\n#val_gen=resc_gen(val_gen)\n\n\nfig=plt.figure(figsize=(15,7), dpi=100)\nti=next(train_gen)[0]\nn_fig=12\nfor i in range(n_fig):\n    plt.subplot(2,n_fig,i+1)\n    plt.imshow(ti[i,...,0],cmap=\"Greys\", interpolation=\"nearest\",vmin=-5, vmax=260)\n    plt.subplot(2,n_fig,i+n_fig+1)\n    plt.imshow(Xtrain[i,...,0],cmap=\"Greys\", interpolation=\"nearest\",vmin=-5, vmax=260)\n    \nplt.show()\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f4475d58bbde4779494bebfe99777a14078388b2"},"cell_type":"code","source":"hist_pre=model.fit(Xtrain, Ytrain,\n          epochs=epochs,\n          verbose=2,\n          batch_size=batch_size,\n          callbacks=callbacks+[keras.callbacks.EarlyStopping(monitor='val_sparse_categorical_accuracy',\n                          patience=20,\n                              \n                              verbose=1,\n                              restore_best_weights=True\n\n                              )],\n          validation_data=(Xval,Yval),\n)\n\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b94d396de6c36a6687754440acb4f17a09801919"},"cell_type":"code","source":"hist=model.fit_generator(train_gen,\n          epochs=epochs,\n          initial_epoch=hist_pre.epoch[-1]+1,\n           steps_per_epoch= steps_per_epoch,\n          callbacks=callbacks,\n          verbose=2,\n        validation_data=(Xval,Yval),\n        #validation_steps=validation_steps\n                        )\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a0c4d01aeb0cb82a506d761e107ca7bede4baeca"},"cell_type":"code","source":"hist_fine1=model.fit_generator(train_gen_fine1,\n          epochs=epochs,\n          initial_epoch=hist.epoch[-1]+1,\n           steps_per_epoch= steps_per_epoch,\n          callbacks=callbacks,\n          verbose=2,\n        validation_data=(Xval,Yval),\n        \n                        )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"23a0ce719ab0f46006c52fb7e851f036625a55cb"},"cell_type":"code","source":"\nhist_fine=model.fit_generator(train_gen_fine,\n          epochs=epochs,\n          initial_epoch=hist_fine1.epoch[-1]+1,\n           steps_per_epoch= steps_per_epoch,\n          callbacks=callbacks,\n          verbose=2,\n        validation_data=(Xval,Yval),\n        #validation_steps=validation_steps\n                        )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dff28a6601954ae5085304d88ec5c9c8b58cc623"},"cell_type":"code","source":"\n\n\nhist_post=model.fit(Xtrain, Ytrain,\n          epochs=epochs,\n          verbose=2,\n          batch_size=batch_size,\n          initial_epoch=hist_fine.epoch[-1]+1,\n           callbacks=callbacks+[keras.callbacks.EarlyStopping(monitor='val_sparse_categorical_accuracy',\n                          patience=20,\n                              \n                              verbose=1,\n                              restore_best_weights=True\n\n                              )],\n          validation_data=(Xval,Yval)\n)\n\n\nmodel.save(fichier_modele)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5c60eaa3e739c80d7d2ff43f2206b5d01eb52439"},"cell_type":"code","source":"\nfor m,e in zip(model.metrics_names,\n               model.evaluate(Xval, Yval,batch_size=batch_size) ):\n               print (m,e)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0edf7b65855802bffcb7ac77378b9c72ad6eca1f"},"cell_type":"code","source":"fig=plt.figure(figsize=(15,7), dpi=100)\nplt.subplot(\"211\")\ntrain_history=hist.history\nfor k in train_history.keys():\n    train_history[k]=hist_pre.history[k]+ train_history[k]+hist_fine.history[k]+hist_post.history[k]\n    \n    \nfor k in train_history.keys():\n    if \"acc\" in k:\n        plt.plot(train_history[k],label=k)\nplt.ylim(0.7,1)\nplt.legend()\nplt.subplot(\"212\")\nplt.yscale(\"log\")\nfor k in train_history.keys():\n    if \"loss\" in k:\n        plt.plot(train_history[k],label=k)\nplt.legend()\n\nplt.ylim(top=0.8)\nfig.savefig(\"graph.png\",dpi=200,transparent=False)\n#IPython.display.Image(filename=\"graph.png\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4db06d94e5d8b06f493c763e5fbe09332ac4ccda"},"cell_type":"code","source":"preds=model.predict(Xtest,batch_size=256, verbose=1,)\nbaseindex=pd.RangeIndex(start=1, stop=preds.shape[0]+1, step=1, name='ImageId')\nsub=pd.DataFrame( data=preds.argmax(axis=-1),\n                 index=baseindex, \n                 columns=[\"Label\"], dtype=\"int32\", copy=True)\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bb0d8b4dcd7a80fdce7d7862554ca8c69d75014a"},"cell_type":"code","source":"n_fig=8*4\nsample=sub.sample(n=n_fig)\nsample.Label\n\nfig=plt.figure(figsize=(15,15), dpi=100)\nti=next(train_gen)[0]\nlignes=4\ncols=n_fig//lignes\nif n_fig%lignes!=0:\n    cols+=1\nfor i in range(n_fig):\n    label=sample.Label.iloc[i]\n    idx=sample.index.values[i]\n    plt.subplot(lignes,cols,i+1)\n    plt.imshow(Xtest[idx,...,0],cmap=\"Greys\", interpolation=\"nearest\",vmin=-127, vmax=128)\n    plt.title(label)\n\n    \nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"50cec072dd2ffa85767fa9ac06db4673a073d108"},"cell_type":"code","source":"sub.to_csv(\"submission.csv\",index_label=\"ImageId\")\nsub.info()\nsub\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}