{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import keras\nimport numpy as np\nimport pandas as pd\nimport os\nimport random\nimport matplotlib.pyplot as plt\n\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Загружаем данные"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/Kannada-MNIST/train.csv\")\nvalidate = pd.read_csv(\"../input/Kannada-MNIST/Dig-MNIST.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_X = np.array(train.drop(\"label\",axis=1),dtype=np.float32)\ntrain_Y = np.array(train[[\"label\"]],dtype=np.int32)\nval_X = np.array(validate.drop(\"label\",axis=1),dtype=np.float32)\nval_Y = np.array(validate[[\"label\"]],dtype=np.int32)\ntrain_X = np.reshape(train_X,(-1, 28, 28, 1))\nval_X = np.reshape(val_X,(-1, 28, 28, 1))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Нормализуем"},{"metadata":{"trusted":true},"cell_type":"code","source":"all_x = np.concatenate((train_X, val_X))  # / 255. * 2 - 1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Делим"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_X, val_X, train_Y, val_Y = train_test_split(all_x, np.concatenate((train_Y, val_Y)), shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Строим (коммунизм) сеть"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.layers import Conv2D, BatchNormalization, Dense, Flatten, Dropout, AveragePooling2D, Activation","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"1. Натренируйте 30 эпох сверточную нейронную сеть LeNet-5 на датасете конкурса https://www.kaggle.com/c/Kannada-MNIST/overview с использованием стохастического градиентного спуска и размером батча 16. Какая точность на валидации?  [1 балл]"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.callbacks import ModelCheckpoint, CSVLogger, LearningRateScheduler\nfilepath=\"best-weights_1.hdf5\"\ncheckpoint = ModelCheckpoint(filepath, monitor='val_accuracy', verbose=1, save_best_only=True, mode='max')\ncsv_logger = CSVLogger('log_1.csv', append=True, separator=';')\ncallbacks = [checkpoint, csv_logger]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = keras.Sequential()\nmodel.add(Conv2D(6, (5, 5), strides=(1, 1), padding='same', activation='tanh', input_shape=(28, 28, 1)))\nmodel.add(BatchNormalization())\nmodel.add(AveragePooling2D(pool_size=(2, 2), strides=2, padding='same'))\nmodel.add(Activation('tanh'))\nmodel.add(Conv2D(16, (5, 5), strides=(1, 1), padding='same', activation='tanh'))\nmodel.add(BatchNormalization())\nmodel.add(AveragePooling2D(pool_size=(2, 2), strides=2, padding='same'))\nmodel.add(Activation('tanh'))\nmodel.add(Conv2D(120, (5, 5), strides=(1, 1), padding='same', activation='tanh'))\nmodel.add(BatchNormalization())\nmodel.add(Flatten())\nmodel.add(Dense(84, activation='tanh'))\nmodel.add(Dense(10, activation='softmax'))\nmodel.compile(optimizer='sgd', loss=\"sparse_categorical_crossentropy\", metrics=[\"accuracy\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(train_X, train_Y, batch_size=16, epochs=30, validation_data=(val_X, val_Y), callbacks=callbacks)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"2. Попробуйте поменять learning_rate градиетного спуска. Что изменилось?  [1 балл]"},{"metadata":{"trusted":true},"cell_type":"code","source":"model = keras.Sequential()\nmodel.add(Conv2D(6, (5, 5), strides=(1, 1), padding='same', activation='tanh', input_shape=(28, 28, 1)))\nmodel.add(BatchNormalization())\nmodel.add(AveragePooling2D(pool_size=(2, 2), strides=2, padding='same'))\nmodel.add(Activation('tanh'))\nmodel.add(Conv2D(16, (5, 5), strides=(1, 1), padding='same', activation='tanh'))\nmodel.add(BatchNormalization())\nmodel.add(AveragePooling2D(pool_size=(2, 2), strides=2, padding='same'))\nmodel.add(Activation('tanh'))\nmodel.add(Conv2D(120, (5, 5), strides=(1, 1), padding='same', activation='tanh'))\nmodel.add(BatchNormalization())\nmodel.add(Flatten())\nmodel.add(Dense(84, activation='tanh'))\nmodel.add(Dense(10, activation='softmax'))\nmodel.compile(optimizer='sgd', loss=\"sparse_categorical_crossentropy\", metrics=[\"accuracy\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.callbacks import ModelCheckpoint, CSVLogger, LearningRateScheduler\nfilepath=\"best_weights_2.hdf5\"\ncheckpoint = ModelCheckpoint(filepath, monitor='val_accuracy', verbose=1, save_best_only=True, mode='max')\nscheduler = LearningRateScheduler(lambda e: 0.01 if e < 5 else 0.001 if e < 10 else 0.0001 if e < 15 else 0.00001 if e < 20 else 0.0000001)\ncsv_logger = CSVLogger('log_2.csv', append=True, separator=';')\ncallbacks = [checkpoint, scheduler, csv_logger]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(train_X, train_Y, batch_size=16, epochs=30, validation_data=(val_X, val_Y), callbacks=callbacks)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Переучите вашу самую лучшую (или одну из лучших) архитетектур с SGD оптимизаторами, SGD с моментом, Adadelta, RMSProb, Adam. Сохраните CSV лог обучения и валидации. Постройте кривые обучения. Объясните результат."},{"metadata":{"trusted":true},"cell_type":"code","source":"model.load_weights(\"best-weights_1.hdf5\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_csv = pd.read_csv(\"../input/Kannada-MNIST/test.csv\")\nX_test = np.array(test_csv.drop(\"id\",axis=1),dtype=np.float32)\nX_test = np.reshape(X_test,(-1, 28, 28, 1))\nX_test = X_test\nresults = model.predict(X_test)\nresults = np.argmax(results, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv(\"../input/Kannada-MNIST/sample_submission.csv\")\nsubmission['label'] = results\nsubmission.to_csv(\"submission.csv\",index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Продолжим"},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_model():\n    model = keras.Sequential()\n    model.add(Conv2D(6, (5, 5), strides=(1, 1), padding='same', activation='tanh', input_shape=(28, 28, 1)))\n    model.add(AveragePooling2D(pool_size=(2, 2), strides=2, padding='same'))\n    model.add(Activation('tanh'))\n    model.add(Conv2D(16, (5, 5), strides=(1, 1), padding='same', activation='tanh'))\n    model.add(AveragePooling2D(pool_size=(2, 2), strides=2, padding='same'))\n    model.add(Activation('tanh'))\n    model.add(Conv2D(120, (5, 5), strides=(1, 1), padding='same', activation='tanh'))\n    model.add(Flatten())\n    model.add(Dense(84, activation='tanh'))\n    model.add(Dense(10, activation='softmax'))\n    return model\n\ndef get_callbacks(index):\n    filepath=\"best_weights_{}.hdf5\".format(index)\n    checkpoint = ModelCheckpoint(filepath, monitor='val_accuracy', verbose=1, save_best_only=True, mode='max')\n    scheduler = LearningRateScheduler(lambda e: 0.01 if e < 5 else 0.001 if e < 10 else 0.0001 if e < 15 else 0.00001 if e < 20 else 0.0000001)\n    csv_logger = CSVLogger('log_{}.csv'.format(index), append=True, separator=';')\n    callbacks = [checkpoint, scheduler, csv_logger]\n    return callbacks","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras import optimizers","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model = create_model()\n# model.compile(optimizer='RMSprop', loss=\"sparse_categorical_crossentropy\", metrics=[\"accuracy\"])\n# callbacks = get_callbacks(3)\n# model.fit(train_X, train_Y, batch_size=16, epochs=30, validation_data=(val_X, val_Y), callbacks=callbacks)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model = create_model()\n# model.compile(optimizer='Adagrad', loss=\"sparse_categorical_crossentropy\", metrics=[\"accuracy\"])\n# callbacks = get_callbacks(4)\n# model.fit(train_X, train_Y, batch_size=16, epochs=30, validation_data=(val_X, val_Y), callbacks=callbacks)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model = create_model()\n# model.compile(optimizer='Adadelta', loss=\"sparse_categorical_crossentropy\", metrics=[\"accuracy\"])\n# callbacks = get_callbacks(5)\n# model.fit(train_X, train_Y, batch_size=16, epochs=30, validation_data=(val_X, val_Y), callbacks=callbacks)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model = create_model()\n# model.compile(optimizer='Adam', loss=\"sparse_categorical_crossentropy\", metrics=[\"accuracy\"])\n# callbacks = get_callbacks(6)\n# model.fit(train_X, train_Y, batch_size=16, epochs=30, validation_data=(val_X, val_Y), callbacks=callbacks)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model = create_model()\n# model.compile(optimizer='Adamax', loss=\"sparse_categorical_crossentropy\", metrics=[\"accuracy\"])\n# callbacks = get_callbacks(7)\n# model.fit(train_X, train_Y, batch_size=16, epochs=30, validation_data=(val_X, val_Y), callbacks=callbacks)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model = create_model()\n# model.compile(optimizer='Nadam', loss=\"sparse_categorical_crossentropy\", metrics=[\"accuracy\"])\n# callbacks = get_callbacks(8)\n# model.fit(train_X, train_Y, batch_size=16, epochs=30, validation_data=(val_X, val_Y), callbacks=callbacks)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model.load_weights(\"best_weights_4.hdf5\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# test_csv = pd.read_csv(\"../input/Kannada-MNIST/test.csv\")\n# X_test = np.array(test_csv.drop(\"id\",axis=1),dtype=np.float32)\n# X_test = np.reshape(X_test,(-1, 28, 28, 1))\n# X_test = X_test\n# results = model.predict(X_test)\n# results = np.argmax(results, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# results","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# submission = pd.read_csv(\"../input/Kannada-MNIST/sample_submission.csv\")\n# submission['label'] = results\n# submission.to_csv(\"submission.csv\",index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}