{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nimport tensorflow as tf\nimport keras\nfrom keras import backend as K\nfrom keras import models\nfrom keras import layers\nfrom keras.layers import Layer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_data(batch_size=10000, input_size=20, attention_index=5):\n    '''\n    batch_size : Total batch size\n    input_size : Total input size, values are random number except at the attention_index\n    attention_index : value is 0 or 1 at the attention index\n    '''\n    \n    train_x = np.random.randn(batch_size, input_size)\n    train_y = np.random.randint(2, size=(batch_size, 1))\n    train_x[:, attention_index] = train_y[:, 0]\n    return train_x, train_y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class AttentionLayer(Layer):\n    def __init__(self, output_dim, **kwargs):\n        self.output_dim = output_dim\n        super(AttentionLayer, self).__init__(**kwargs)\n        \n    def build(self, input_shape):\n        self.att_vec = self.add_weight(\n            name = 'attention_vector', \n            shape = (input_shape[1], ),\n            initializer = 'normal',\n            trainable = True\n        )\n        super(AttentionLayer, self).build(input_shape)\n\n    def call(self, x):\n        y = tf.math.multiply(self.att_vec, x)\n        y = keras.activations.softmax(y, axis=-1)\n        return y\n\n    def compute_output_shape(self, input_shape):\n        return (input_shape[0], self.output_dim)\n\n# Make Training Set\ntrain_x, train_y = make_data()\ninput_dims = train_x.shape[1]\n\n# Make Net\ninput_layer = layers.Input(shape=(input_dims,))\nattention_vector = AttentionLayer(input_dims)(input_layer)\ny = layers.dot([attention_vector, input_layer], axes=1)\ny = layers.Dense(1, activation='sigmoid', use_bias=False)(y)\n\n# Train \nmodel = models.Model(input_layer, y)\nmodel.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\nmodel.summary()\nmodel.fit(train_x, train_y, epochs=20, batch_size=32, validation_split=0.2, verbose=1)\n\n# Evaluation\ntest_x, test_y = make_data()\ntrue = test_y\npred = model.predict(test_x)\npred = np.around(pred)\nscore = (1-np.sum(np.abs(pred-true))/len(true))*100\nprint('score: %f %%'%(score))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Calculate Attention vector\nattention_layer = model.layers[1]\nfunc = K.function([model.input], [attention_layer.output])\noutput = func([test_x])[0]\nattention_vector = np.mean(output, axis=0)\n\n# Show Attention vector\nplt.title('Attention Vector')\nplt.bar(np.arange(len(attention_vector))+0.3, attention_vector, width=0.6)\nplt.xlabel('index')\nplt.ylabel('Attention ratio')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}