{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.layers import *\nfrom keras.models import Model\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Layer Visualization Code"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Visualize Layer Keras\n\nfrom __future__ import print_function\n\nimport time\nfrom PIL import Image as pil_image\nfrom keras.preprocessing.image import save_img\nfrom keras import layers\nfrom keras import backend as K\n\n\ndef normalize(x):\n    \"\"\"utility function to normalize a tensor.\n\n    # Arguments\n        x: An input tensor.\n\n    # Returns\n        The normalized input tensor.\n    \"\"\"\n    return x / (K.sqrt(K.mean(K.square(x))) + K.epsilon())\n\n\ndef deprocess_image(x):\n    \"\"\"utility function to convert a float array into a valid uint8 image.\n\n    # Arguments\n        x: A numpy-array representing the generated image.\n\n    # Returns\n        A processed numpy-array, which could be used in e.g. imshow.\n    \"\"\"\n    # normalize tensor: center on 0., ensure std is 0.25\n    x -= x.mean()\n    x /= (x.std() + K.epsilon())\n    x *= 0.25\n\n    # clip to [0, 1]\n    x += 0.5\n    x = np.clip(x, 0, 1)\n\n    # convert to RGB array\n    x *= 255\n    if K.image_data_format() == 'channels_first':\n        x = x.transpose((1, 2, 0))\n    x = np.clip(x, 0, 255).astype('uint8')\n    return x\n\n\ndef process_image(x, former):\n    \"\"\"utility function to convert a valid uint8 image back into a float array.\n       Reverses `deprocess_image`.\n\n    # Arguments\n        x: A numpy-array, which could be used in e.g. imshow.\n        former: The former numpy-array.\n                Need to determine the former mean and variance.\n\n    # Returns\n        A processed numpy-array representing the generated image.\n    \"\"\"\n    if K.image_data_format() == 'channels_first':\n        x = x.transpose((2, 0, 1))\n    return (x / 255 - 0.5) * 4 * former.std() + former.mean()\n\n\ndef visualize_layer(model,\n                    layer_name,\n                    step=1.,\n                    epochs=15,\n                    upscaling_steps=9,\n                    upscaling_factor=1.2,\n                    output_dim=(412, 412),\n                    filter_range=(0, None)):\n    \"\"\"Visualizes the most relevant filters of one conv-layer in a certain model.\n\n    # Arguments\n        model: The model containing layer_name.\n        layer_name: The name of the layer to be visualized.\n                    Has to be a part of model.\n        step: step size for gradient ascent.\n        epochs: Number of iterations for gradient ascent.\n        upscaling_steps: Number of upscaling steps.\n                         Starting image is in this case (80, 80).\n        upscaling_factor: Factor to which to slowly upgrade\n                          the image towards output_dim.\n        output_dim: [img_width, img_height] The output image dimensions.\n        filter_range: Tupel[lower, upper]\n                      Determines the to be computed filter numbers.\n                      If the second value is `None`,\n                      the last filter will be inferred as the upper boundary.\n    \"\"\"\n\n    def _generate_filter_image(input_img,\n                               layer_output,\n                               filter_index):\n        \"\"\"Generates image for one particular filter.\n\n        # Arguments\n            input_img: The input-image Tensor.\n            layer_output: The output-image Tensor.\n            filter_index: The to be processed filter number.\n                          Assumed to be valid.\n\n        #Returns\n            Either None if no image could be generated.\n            or a tuple of the image (array) itself and the last loss.\n        \"\"\"\n        s_time = time.time()\n\n        # we build a loss function that maximizes the activation\n        # of the nth filter of the layer considered\n        if K.image_data_format() == 'channels_first':\n            loss = K.mean(layer_output[:, filter_index, :, :])\n        else:\n            loss = K.mean(layer_output[:, :, :, filter_index])\n\n        # we compute the gradient of the input picture wrt this loss\n        grads = K.gradients(loss, input_img)[0]\n\n        # normalization trick: we normalize the gradient\n        grads = normalize(grads)\n\n        # this function returns the loss and grads given the input picture\n        iterate = K.function([input_img], [loss, grads])\n\n        # we start from a gray image with some random noise\n        intermediate_dim = tuple(\n            int(x / (upscaling_factor ** upscaling_steps)) for x in output_dim)\n        if K.image_data_format() == 'channels_first':\n            input_img_data = np.random.random(\n                (1, 3, intermediate_dim[0], intermediate_dim[1]))\n        else:\n            input_img_data = np.random.random(\n                (1, intermediate_dim[0], intermediate_dim[1], 3))\n        input_img_data = (input_img_data - 0.5) * 20 + 128\n\n        # Slowly upscaling towards the original size prevents\n        # a dominating high-frequency of the to visualized structure\n        # as it would occur if we directly compute the 412d-image.\n        # Behaves as a better starting point for each following dimension\n        # and therefore avoids poor local minima\n        for up in reversed(range(upscaling_steps)):\n            # we run gradient ascent for e.g. 20 steps\n            for _ in range(epochs):\n                loss_value, grads_value = iterate([input_img_data])\n                input_img_data += grads_value * step\n\n                # some filters get stuck to 0, we can skip them\n                if loss_value <= K.epsilon():\n                    return None\n\n            # Calculate upscaled dimension\n            intermediate_dim = tuple(\n                int(x / (upscaling_factor ** up)) for x in output_dim)\n            # Upscale\n            img = deprocess_image(input_img_data[0])\n            img = np.array(pil_image.fromarray(img).resize(intermediate_dim,\n                                                           pil_image.BICUBIC))\n            input_img_data = np.expand_dims(\n                process_image(img, input_img_data[0]), 0)\n\n        # decode the resulting input image\n        img = deprocess_image(input_img_data[0])\n        e_time = time.time()\n        print('Costs of filter {:3}: {:5.0f} ( {:4.2f}s )'.format(filter_index,\n                                                                  loss_value,\n                                                                  e_time - s_time))\n        return img, loss_value\n\n    def _draw_filters(filters, n=None):\n        \"\"\"Draw the best filters in a nxn grid.\n\n        # Arguments\n            filters: A List of generated images and their corresponding losses\n                     for each processed filter.\n            n: dimension of the grid.\n               If none, the largest possible square will be used\n        \"\"\"\n        if n is None:\n            n = int(np.floor(np.sqrt(len(filters))))\n\n        # the filters that have the highest loss are assumed to be better-looking.\n        # we will only keep the top n*n filters.\n        filters.sort(key=lambda x: x[1], reverse=True)\n        filters = filters[:n * n]\n\n        # build a black picture with enough space for\n        # e.g. our 8 x 8 filters of size 412 x 412, with a 5px margin in between\n        MARGIN = 5\n        width = n * output_dim[0] + (n - 1) * MARGIN\n        height = n * output_dim[1] + (n - 1) * MARGIN\n        stitched_filters = np.zeros((width, height, 3), dtype='uint8')\n\n        # fill the picture with our saved filters\n        for i in range(n):\n            for j in range(n):\n                img, _ = filters[i * n + j]\n                width_margin = (output_dim[0] + MARGIN) * i\n                height_margin = (output_dim[1] + MARGIN) * j\n                stitched_filters[\n                    width_margin: width_margin + output_dim[0],\n                    height_margin: height_margin + output_dim[1], :] = img\n\n        # save the result to disk\n        save_img('vgg_{0:}_{1:}x{1:}.png'.format(layer_name, n), stitched_filters)\n\n    # this is the placeholder for the input images\n    assert len(model.inputs) == 1\n    input_img = model.inputs[0]\n\n    # get the symbolic outputs of each \"key\" layer (we gave them unique names).\n    layer_dict = dict([(layer.name, layer) for layer in model.layers[1:]])\n\n    output_layer = layer_dict[layer_name]\n    assert isinstance(output_layer, layers.Conv2D)\n\n    # Compute to be processed filter range\n    filter_lower = filter_range[0]\n    filter_upper = (filter_range[1]\n                    if filter_range[1] is not None\n                    else len(output_layer.get_weights()[1]))\n    assert(filter_lower >= 0\n           and filter_upper <= len(output_layer.get_weights()[1])\n           and filter_upper > filter_lower)\n    print('Compute filters {:} to {:}'.format(filter_lower, filter_upper))\n\n    # iterate through each filter and generate its corresponding image\n    processed_filters = []\n    for f in range(filter_lower, filter_upper):\n        img_loss = _generate_filter_image(input_img, output_layer.output, f)\n\n        if img_loss is not None:\n            processed_filters.append(img_loss)\n\n    print('{} filter processed.'.format(len(processed_filters)))\n    # Finally draw and store the best filters to disk\n    _draw_filters(processed_filters)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nimport seaborn as sns\nfrom sklearn.metrics import confusion_matrix\n%matplotlib inline\ndef plot_confusion_matrix(y_true, y_pred):\n    mtx = confusion_matrix(y_true, y_pred)\n    fig, ax = plt.subplots(figsize=(8,8))\n    sns.heatmap(mtx, annot=True, fmt='d', linewidths=.5,  cbar=False, ax=ax)\n    #  square=True,\n    plt.ylabel('Label')\n    plt.xlabel('Prediction')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data=pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\ntest_data=pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y=train_data[\"label\"]\nX=train_data.copy()\ndel X[\"label\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plt.imshow(X.values[0].reshape((28,28)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# display(X,y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SIZE=32","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def reshape32(img):\n    img=img.reshape((28,28))\n    img=np.pad(img,((SIZE-28)//2,(SIZE-28)//2))\n    img=img.reshape((SIZE,SIZE,1))\n#     img=np.stack((img,)*3, axis=-1)\n    return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plt.imshow(reshape32(X.values[12]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_X=[]\nfor i,img in enumerate(X.values):\n    new_X.append(reshape32(img))\nnew_X=np.array(new_X)\nnew_X[new_X<50]=0\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plt.imshow(new_X[12])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\ntrain_X,val_X,train_y,val_y = train_test_split(new_X/255,y,test_size=0.1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"inp=Input(shape=(32,32,1))\n\n\nmodel = Conv2D(filters=32, kernel_size=(2, 2), padding='SAME', activation='relu',name=\"conv32\")(inp)\nmodel = Conv2D(filters=32, kernel_size=(2, 2), padding='SAME', activation='relu')(model)\nmodel = Conv2D(filters=32, kernel_size=(2, 2), padding='SAME', activation='relu')(model)\nmodel = BatchNormalization(momentum=0.15)(model)\nmodel = MaxPool2D(pool_size=(2, 2))(model)\nmodel = Conv2D(filters=32, kernel_size=(3, 3), padding='SAME', activation='relu')(model)\nmodel = Dropout(rate=0.3)(model)\n\nmodel = Conv2D(filters=64, kernel_size=(5, 5), padding='SAME', activation='relu',name=\"conv64\")(model)\nmodel = Conv2D(filters=64, kernel_size=(5, 5), padding='SAME', activation='relu')(model)\nmodel = Conv2D(filters=64, kernel_size=(5, 5), padding='SAME', activation='relu')(model)\nmodel = BatchNormalization(momentum=0.15)(model)\nmodel = MaxPool2D(pool_size=(2, 2))(model)\nmodel = Conv2D(filters=64, kernel_size=(5, 5), padding='SAME', activation='relu')(model)\nmodel = BatchNormalization(momentum=0.15)(model)\nmodel = Dropout(rate=0.2)(model)\n\nmodel = Conv2D(filters=128, kernel_size=(7, 7), padding='SAME', activation='relu',name=\"conv128\")(model)\nmodel = Conv2D(filters=128, kernel_size=(7, 7), padding='SAME', activation='relu')(model)\nmodel = Conv2D(filters=128, kernel_size=(7, 7), padding='SAME', activation='relu')(model)\nmodel = BatchNormalization(momentum=0.15)(model)\nmodel = MaxPool2D(pool_size=(2, 2))(model)\nmodel = Conv2D(filters=128, kernel_size=(7, 7), padding='SAME', activation='relu')(model)\nmodel = BatchNormalization(momentum=0.15)(model)\nmodel = Dropout(rate=0.2)(model)\n\n\n\nmodel = Conv2D(filters=256, kernel_size=(15, 15), padding='SAME', activation='relu',name=\"conv256\")(model)\nmodel = Conv2D(filters=256, kernel_size=(15, 15), padding='SAME', activation='relu')(model)\nmodel = Conv2D(filters=256, kernel_size=(15, 15), padding='SAME', activation='relu')(model)\nmodel = BatchNormalization(momentum=0.15)(model)\nmodel = MaxPool2D(pool_size=(2, 2))(model)\n# model = Conv2D(filters=256, kernel_size=(2, 2), padding='SAME', activation='relu')(model)\n# model = BatchNormalization(momentum=0.15)(model)\nmodel = Dropout(rate=0.2)(model)\n\n\nmy_x=Flatten()(model)\n# my_x=Dense(1024,activation='relu',kernel_initializer='he_uniform',bias_initializer='zeros')(my_x)\n# my_x=Dropout(0.2)(my_x)\n\n# my_x=Dense(512,activation='relu',kernel_initializer='he_uniform',bias_initializer='zeros')(my_x)\n# my_x=Dropout(0.2)(my_x)\n\nmy_x=Dense(256,activation='relu',kernel_initializer='he_uniform',bias_initializer='zeros')(my_x)\nmy_x=Dropout(0.2)(my_x)\n\nmy_x=Dense(128,activation='relu',kernel_initializer='he_uniform',bias_initializer='zeros')(my_x)\nmy_x=Dropout(0.2)(my_x)\n\n\nmy_x=Dense(64,activation='relu',name='my')(my_x)\nmy_x=Dropout(0.2)(my_x)\n\nmy_x=Dense(32,activation='relu',name='my2')(my_x)\nmy_x=Dropout(0.2)(my_x)\n\n\n\npreds=Dense(10,activation='softmax',kernel_initializer='he_uniform',bias_initializer='zeros',name='output')(my_x)\n\nmy_model=Model(inputs=inp,outputs=preds)\n\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# my_model=new_model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for layer in my_model.layers:\n#     if layer.name == 'output' or layer.name == 'my' :\n#         layer.trainable=True\n#     else:\n#         layer.trainable=True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.optimizers import Adadelta\nmy_model.compile(optimizer=Adadelta(),loss='categorical_crossentropy',metrics=['accuracy','mse'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"my_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.callbacks import ReduceLROnPlateau\n\nrlrp = ReduceLROnPlateau(monitor='loss', factor=0.1, patience=2, min_delta=1E-30,verbose=1)\nhistory=my_model.fit(x=train_X,y=pd.get_dummies(train_y),validation_data=(val_X,pd.get_dummies(val_y)),epochs=100, batch_size=1024,callbacks=[rlrp])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for layer in my_model.layers:\n    print(layer.name,)\n    if 'conv' not in layer.name:\n        continue\n    filters, biases = layer.get_weights()\n    \n    filters, biases = layer.get_weights()\n    f_min, f_max = filters.min(), filters.max()\n    filters = (filters - f_min) / (f_max - f_min)\n    n_filters, ix = 6, 1\n    for i in range(n_filters):\n        f = filters[:, :, :, i]\n        for j in range(1):\n            ax = plt.subplot(n_filters, 3, ix)\n            ax.set_xticks([])\n            ax.set_yticks([])\n            plt.imshow(f[:, :, j], cmap='gray')\n            ix += 1\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds=my_model.predict(train_X)\nprint(\"Training Accuracy : \",accuracy_score(train_y,np.argmax(preds,axis=1)))\n\n\n\npreds=my_model.predict(val_X)\nprint(\"Validation Accuracy : \",accuracy_score(val_y,np.argmax(preds,axis=1)))\nplot_confusion_matrix(val_y,np.argmax(preds,axis=1))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# #mis predicted\n\nfor i in range(len(val_X)):\n    if np.argmax(my_model.predict(val_X[i].reshape(1,32,32,1)),axis=1)!=val_y.values[i]:\n        (plt.imshow(  val_X[i].reshape(32,32),))\n        plt.show()\n        print(\"Label : \",val_y.values[i])\n        print(\"Prediction : \",np.argmax(my_model.predict(val_X[i].reshape(1,32,32,1)),axis=1))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_X=[]\nfor i,img in enumerate(test_data.values):\n    z=reshape32(img)\n    test_X.append(z)\ntest_X=np.array(test_X)\ntest_X[test_X<50]=0\ntest_X=test_X/255","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sol=np.argmax(my_model.predict((test_X)),axis=1)\ndf=pd.DataFrame(sol)\ndf.index+=1\ndf.to_csv(\"/kaggle/working/sol_final.csv\",index=True,header=[\"Label\"],index_label=[\"ImageId\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}