{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import warnings\nwarnings.simplefilter(\"ignore\",category=FutureWarning)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as mpatches\n%matplotlib inline\nfrom sklearn.model_selection import train_test_split\n\nfrom scipy.stats import norm\n\nimport tensorflow as tf\nimport keras\nfrom keras.layers import *\nfrom keras.models import *\nfrom keras.losses import *\nfrom keras.callbacks import *\nfrom keras.optimizers import *\nfrom keras import metrics\nfrom keras import backend as K   # 'generic' backend so code works with either tensorflow or theano\n\nK.clear_session()\n\nnp.random.seed(237)\nprint(keras.__version__) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/digit-recognizer/train.csv')\n# train_orig = pd.read_csv('../input/digit-recognizer/train.csv')\ntest = pd.read_csv('../input/digit-recognizer/test.csv')\n# test_orig = pd.read_csv('../input/digit-recognizer/test.csv')\n\ntrain.tail()\n      ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# test_orig['label'] = 11\n# testCols = test_orig.columns.tolist()\n# testCols = testCols[-1:] + testCols[:-1]\n# test_orig = test_orig[testCols]\n# combined.tail()\n# 検証/テストサンプルとして5000枚のランダムな画像を用意する\n# valid = combined.sample(n = 5000, random_state = 555)\n# train_after = combined.loc[~combined.index.isin(valid.index)]\n\n# 余分なデータを消去\n# del train_orig, test_orig, combined\n\n#valid.head()\nX_train = train.drop(['label'], axis = 1)\n# X_valid = test_orig.drop(['label'], axis = 1)\n\n# labelを取得\ny_train = train['label']\n# y_valid = valid['label']\n# X_train, X_test, y_train, y_test = train_test_split(x_train,Y_train,test_size=0.1,shuffle=True)\n\n# 正規化を行う\nX_train = X_train.astype('float32') / 255.\nX_train = X_train.values.reshape(-1,28,28,1)\n\nX_test = test.astype('float32') / 255.\nX_test = X_test.values.reshape(-1,28,28,1)\n\n# num_classes = 10\n\n# y_train = keras.utils.to_categorical(y_train, num_classes)\n\n# X_valid = X_valid.astype('float32') / 255.\n# X_valid = X_valid.values.reshape(-1,28,28,1)\nplt.figure(1)\nplt.subplot(221)\nplt.imshow(X_train[13][:,:,0])\n\nplt.subplot(222)\nplt.imshow(X_train[690][:,:,0])\n\nplt.subplot(223)\nplt.imshow(X_train[2375][:,:,0])\n\nplt.subplot(224)\nplt.imshow(X_test[3360][:,:,0])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def sampling(args):\n    z_mean, z_log_var = args\n    batch = K.shape(z_mean)[0]\n    dim = K.int_shape(z_mean)[1]\n    epsilon = K.random_normal(shape=(batch, dim))\n    return z_mean + K.exp(0.5 * z_log_var) * epsilon\nlatent_dim = 2\ninputs = Input(shape=(28,28,1), name='encoder_input')\n# x = Conv2D(16, (3, 3), activation='relu', padding='same')(inputs)\n# x = MaxPooling2D((2, 2), padding='same')(x)\n# x = Conv2D(8, (3, 3), activation='relu', padding='same')(x)\n# x = MaxPooling2D((2, 2), padding='same')(x)\n# x = Conv2D(8, (3, 3), activation='relu', padding='same')(x)\n# x = MaxPooling2D((2, 2), padding='same',name='encoded')(x)\nx = Conv2D(32, 3, activation=\"relu\", padding=\"same\")(inputs)\nx = Conv2D(64, 3, activation=\"relu\", strides=(2,2), padding=\"same\")(x)\nx = Conv2D(64, 3, padding='same', activation='relu')(x)\nx = Conv2D(64, 3, padding='same', activation='relu')(x)\nshape = K.int_shape(x)\nx = Flatten()(x)\nx = Dense(32, activation='relu')(x)\n\nz_mean = Dense(latent_dim, name='z_mean')(x)\nz_log_var = Dense(latent_dim, name='z_log_var')(x)\n\nz = Lambda(sampling, output_shape=(latent_dim,), name='z')([z_mean, z_log_var])\n\nencoder = Model(inputs, [z_mean, z_log_var, z], name='encoder')\nencoder.summary()\n\nlatent_inputs = Input(shape=(latent_dim,), name='z_sampling')\n# x = Dense(shape[1] * shape[2] * shape[3], activation='relu')(latent_inputs)\n# x = Reshape((shape[1], shape[2], shape[3]))(x)\n# x = Conv2D(8, (3, 3), activation='relu', padding='same')(x)\n# x = UpSampling2D((2, 2))(x)\n# x = Conv2D(8, (3, 3), activation='relu', padding='same')(x)\n# x = UpSampling2D((2, 2))(x)\n# x = Conv2D(16, (3, 3), activation='relu')(x)\n# x = UpSampling2D((2, 2))(x)\n# x = Dense(7 * 7 * 64, activation=\"relu\")(latent_inputs)\n# x = Reshape((7, 7, 64))(x)\n# x = Conv2DTranspose(64, 3, activation=\"relu\", strides=2, padding=\"same\")(x)\n# x = Conv2DTranspose(32, 3, activation=\"relu\", strides=2, padding=\"same\")(x)\n# outputs = Conv2DTranspose(1, 3, activation=\"sigmoid\", padding=\"same\")(x)\n# outputs = Conv2D(1, (3, 3), activation='sigmoid', padding='same')(x)\nx = Dense(np.prod(shape[1:]),activation='relu')(latent_inputs)\nx = Reshape(shape[1:])(x)\nx = Conv2DTranspose(32, 3,padding='same',activation='relu',strides=(2, 2))(x)\noutputs = Conv2D(1, 3,padding='same', activation='sigmoid')(x)\n\n# instantiate decoder model\ndecoder = Model(latent_inputs, outputs, name='decoder')\ndecoder.summary()\n\n# instantiate VAE model\noutputs = decoder(encoder(inputs)[2])\nvae = Model(inputs, outputs, name='vae')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Compute VAE loss\nreconstruction_loss = binary_crossentropy(K.flatten(inputs), K.flatten(outputs))\nreconstruction_loss *= 28 * 28\nkl_loss = 1 + z_log_var - K.square(z_mean) - K.exp(z_log_var)\nkl_loss = K.sum(kl_loss, axis=-1)\nkl_loss *= -0.5\nvae_loss = K.mean(reconstruction_loss + kl_loss)\nvae.add_loss(vae_loss)\nvae.compile(optimizer='rmsprop')\nhistory=vae.fit(X_train,shuffle=True,epochs=10, batch_size=16,validation_data=(X_train, None))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(type(history))\n\nprint(type(history.history))\n\nprint(history.history.keys())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"a = vae.evaluate(X_test, y=None)\nprint(a)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_label_clusters(encoder, decoder, data, labels):\n    z_mean, _, _ = encoder.predict(data)\n    plt.figure(figsize=(12, 10))\n    plt.scatter(z_mean[:, 0], z_mean[:, 1], c=labels)\n    plt.colorbar()\n    plt.xlabel(\"z[0]\")\n    plt.ylabel(\"z[1]\")\n    plt.show()\n\nX_train = np.expand_dims(X_train, -1).astype(\"float32\") / 255\n\nplot_label_clusters(encoder, decoder, X_train, y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_latent(input_img, y):\n    # display a n*n 2D manifold of digits\n    n = 30\n    digit_size = 28\n    scale = 2.0\n    figsize = 15\n    figure = np.zeros((digit_size * n, digit_size * n))\n    grid_x = np.linspace(-scale, scale, n)\n    grid_y = np.linspace(-scale, scale, n)[::-1]\n\n    for i, yi in enumerate(grid_y):\n        for j, xi in enumerate(grid_x):\n            z_sample = np.array([[xi, yi]])\n            x_decoded = decoder.predict(z_sample)\n            digit = x_decoded[0].reshape(digit_size, digit_size)\n            figure[\n                i * digit_size : (i + 1) * digit_size,\n                j * digit_size : (j + 1) * digit_size,\n            ] = digit\n\n    plt.figure(figsize=(figsize, figsize))\n    start_range = digit_size // 2\n    end_range = n * digit_size + start_range + 1\n    pixel_range = np.arange(start_range, end_range, digit_size)\n    sample_range_x = np.round(grid_x, 1)\n    sample_range_y = np.round(grid_y, 1)\n    plt.xticks(pixel_range, sample_range_x)\n    plt.yticks(pixel_range, sample_range_y)\n    plt.xlabel(\"z[0]\")\n    plt.ylabel(\"z[1]\")\n    plt.imshow(figure, cmap=\"Greys_r\")\n    plt.show()\n\n\nplot_latent(encoder, decoder)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(X_test.shape)\n\npredictions = vae.predict(X_test)\n\nprint(type(predictions))\n\nprint(predictions.shape)\n\nprint(predictions)\n\nprint(predictions.sum())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results = np.argmax(predictions,axis = 1)\nresults = results.flatten()\n\nresults = results[results<10]\nresults = results[0:28000]\nresults = pd.Series(results,name=\"Label\")\nprint(results)\nprint(type(results))\nprint(results.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.concat([pd.Series(range(1,28001),name = \"ImageID\"),results],axis = 1)\n\nsubmission.to_csv(\"vae_submission.csv\",index=False)\nsubmission","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}