{"cells":[{"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":34},"colab_type":"code","executionInfo":{"elapsed":1149,"status":"ok","timestamp":1572728188184,"user":{"displayName":"Сергей Шутов","photoUrl":"https://lh3.googleusercontent.com/a-/AAuE7mC52V9L4-7fxM2dtV6Vdr8h1wPJqF8P9xvd3LTINg=s64","userId":"01883329426007109825"},"user_tz":-180},"id":"FuhU-ueV1fi2","outputId":"d391c8a8-f938-49ff-cbc7-0ed4ac897407","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\n\nfrom sklearn.model_selection import KFold\n\nimport tensorflow as tf\n\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense, Flatten, Dropout, Conv2D, MaxPooling2D\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras import regularizers\n\n#tf.debugging.set_log_device_placement(True)\n\nprint(tf.config.experimental.list_physical_devices('CPU'))\nprint(tf.config.experimental.list_physical_devices('GPU'))\nprint(tf.__version__)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data = pd.read_csv('../input/bird-or-aircraft-dafe-open/train_x.csv', index_col=0, header=None)\ntrain_labels = pd.read_csv('../input/bird-or-aircraft-dafe-open/train_y.csv', index_col=0)\ntest_data = pd.read_csv('../input/bird-or-aircraft-dafe-open/test_x.csv', index_col=0, header=None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data.shape, train_labels.shape, test_data.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Check classes balance\n\ntrain_labels['target'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Convert to numpy arrays\n\ntrain_data = train_data.to_numpy()\ntest_data = test_data.to_numpy()\n\ntrain_labels = train_labels.to_numpy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_sample_image(sample):\n    image = sample.reshape(32, 32, 3)\n    return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Sample image\n\nplt.figure(figsize=(2, 2))\nplt.imshow(get_sample_image(train_data[555]))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Sample images \n\nplt.figure(figsize=(8, 8))\nfor i in range(25):\n    plt.subplot(5, 5, i+1)\n    plt.xticks([])\n    plt.yticks([])\n    plt.imshow(get_sample_image(train_data[i]))\n    plt.xlabel('plane' if train_labels[i] else 'bird')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Normalize and reshape to 32 x 32 x 3 (RGB)\n\ntrain_data = train_data / 255\ntrain_data = train_data.reshape(train_data.shape[0], 32, 32, 3)","execution_count":null,"outputs":[]},{"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":34},"colab_type":"code","executionInfo":{"elapsed":3254,"status":"ok","timestamp":1572728190696,"user":{"displayName":"Сергей Шутов","photoUrl":"https://lh3.googleusercontent.com/a-/AAuE7mC52V9L4-7fxM2dtV6Vdr8h1wPJqF8P9xvd3LTINg=s64","userId":"01883329426007109825"},"user_tz":-180},"id":"dyCuBpfn1fjd","outputId":"f6fda4b9-dcdd-49bf-f2df-0cbd4e43efa0","trusted":true},"cell_type":"code","source":"# Convolutional NN\n\ndef build_model():\n    \n    model = Sequential([\n        Conv2D(16, 3, padding='same', activation='relu', input_shape=(32, 32, 3)),\n        MaxPooling2D(),\n        Conv2D(32, 3, padding='same', activation='relu'),\n        MaxPooling2D(),\n        Conv2D(64, 3, padding='same', activation='relu'),\n        MaxPooling2D(),\n        Flatten(),\n        Dropout(0.5),\n        Dense(512, activation='relu', kernel_regularizer=regularizers.l1_l2(0.0001)),\n        Dense(1, activation='sigmoid')\n    ])\n    \n    model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":34},"colab_type":"code","executionInfo":{"elapsed":1379,"status":"ok","timestamp":1572728188476,"user":{"displayName":"Сергей Шутов","photoUrl":"https://lh3.googleusercontent.com/a-/AAuE7mC52V9L4-7fxM2dtV6Vdr8h1wPJqF8P9xvd3LTINg=s64","userId":"01883329426007109825"},"user_tz":-180},"id":"t7KzOMEp1fjE","outputId":"935b0874-f174-49f5-f733-18b6623dc8d1","trusted":true},"cell_type":"code","source":"# Training with KFold validation\n\nkf = KFold(n_splits=5, shuffle=True, random_state=42)\n\nepochs_num = 150\nall_loss = []\nall_accuracy = []\n\ni = 0\nfor train_index, val_index in kf.split(train_data):\n    i += 1\n    print('Processing fold #', i)\n    \n    X_train = train_data[train_index] \n    y_train = train_labels[train_index]\n\n    X_val = train_data[val_index]\n    y_val = train_labels[val_index]\n    \n    model = build_model()\n    history = model.fit(X_train, y_train,\n                        epochs=epochs_num, \n                        batch_size=128,\n                        validation_data=(X_val, y_val),\n                        verbose=0)\n    \n    loss_history = history.history['val_loss']\n    all_loss.append(loss_history)\n\n    accuracy_history = history.history['val_accuracy']\n    all_accuracy.append(accuracy_history)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Exponential moving average function\n\ndef smooth_curve(points, factor=0.9):\n    smoothed_points = []\n    for point in points:\n        if smoothed_points:\n            previous = smoothed_points[-1]\n            smoothed_points.append(previous * factor + point * (1 - factor))\n        else:\n            smoothed_points.append(point)\n    return smoothed_points","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plots\n\nn_from = 0\nn_to = epochs_num\n    \nplt.figure(figsize=(20, 10))\n\n# Accuracy plots\nplt.subplot(2, 4, 1)\nfor i in range(5):\n    sns.lineplot(np.arange(n_to - n_from), all_accuracy[i][n_from:n_to]);\n\nplt.subplot(2, 4, 2)\nfor i in range(5):\n    sns.lineplot(np.arange(n_to - n_from), smooth_curve(all_accuracy[i][n_from:n_to], factor=0.7));\n\nplt.subplot(2, 4, 3)\nall_accuracy_np = np.array(all_accuracy)\nsns.lineplot(np.arange(n_to - n_from), all_accuracy_np.mean(axis=0)[n_from:n_to]);\n\nplt.subplot(2, 4, 4)\nsns.lineplot(np.arange(n_to - n_from), smooth_curve(all_accuracy_np.mean(axis=0)[n_from:n_to], factor=0.7));\n\n# Loss plots\nplt.subplot(2, 4, 5)\nfor i in range(5):\n    sns.lineplot(np.arange(n_to - n_from), all_loss[i][n_from:n_to]);\n\nplt.subplot(2, 4, 6)\nfor i in range(5):\n    sns.lineplot(np.arange(n_to - n_from), smooth_curve(all_loss[i][n_from:n_to], factor=0.7));\n\nplt.subplot(2, 4, 7)\nall_loss_np = np.array(all_loss)\nsns.lineplot(np.arange(n_to - n_from), all_loss_np.mean(axis=0)[n_from:n_to]);\n\nplt.subplot(2, 4, 8)\nsns.lineplot(np.arange(n_to - n_from), smooth_curve(all_loss_np.mean(axis=0)[n_from:n_to], factor=0.7));\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Final train\n\nepochs_num = 75\n\nX_train = train_data\ny_train = train_labels\n\nmodel = build_model()\nmodel.fit(X_train, y_train, epochs=epochs_num, batch_size=128, verbose=0);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Normalize and reshape to 32 x 32 x 3 (RGB)\n\nX_test = test_data / 255\nX_test = X_test.reshape(test_data.shape[0], 32, 32, 3)","execution_count":null,"outputs":[]},{"metadata":{"colab":{},"colab_type":"code","id":"GfXYCGoS1fkC","trusted":true},"cell_type":"code","source":"# Make Predictions\n\ny_pred = model.predict(X_test)\n\ny_pred[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Show some predictions\n\nplt.figure(figsize=(8, 8))\nfor i in range(25):\n    plt.subplot(5, 5, i+1)\n    plt.xticks([])\n    plt.yticks([])\n    plt.imshow(get_sample_image(test_data[i]))\n    plt.xlabel('plane' if y_pred[i] >= 0.5 else 'bird')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Make submission\n\nsubmission = pd.DataFrame({'id': range(test_data.shape[0]),\n                           'target': (y_pred >= 0.5).astype('int').flatten()\n                          })","execution_count":null,"outputs":[]},{"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":34},"colab_type":"code","executionInfo":{"elapsed":963,"status":"ok","timestamp":1572728604151,"user":{"displayName":"Сергей Шутов","photoUrl":"https://lh3.googleusercontent.com/a-/AAuE7mC52V9L4-7fxM2dtV6Vdr8h1wPJqF8P9xvd3LTINg=s64","userId":"01883329426007109825"},"user_tz":-180},"id":"oX7dGRcy1fkP","outputId":"488cdc8d-d94c-407b-9cea-3cb4f1d03083","trusted":true},"cell_type":"code","source":"submission.tail(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"colab":{"collapsed_sections":[],"name":"mnist-tf.ipynb","provenance":[]},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.5"}},"nbformat":4,"nbformat_minor":1}