{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Import Libraries","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\nimport seaborn as sns\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nimport tensorflow as tf\n\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras import metrics\nfrom tensorflow.keras import backend as K","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Reading the folder architecture of Kaggle to get the dataset path.\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 訓練データとテストデータを読み込む\nmnist_train = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\nmnist_test = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# データの行と列の数を見る\nprint(mnist_train.shape, mnist_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 前から5つの要素を見る\nmnist_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 各列ごとの要約統計量を見る。個数、平均、標準偏差、最小値、1/4分位数、中央値、3/4分位数、最大値)\nmnist_train.describe()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## データのクリーニングと正規化","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# データの欠損がないかを見る\nmnist_train.isna().any().any()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### データの正規化","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# データを入力と出力の特徴量に分割して学習させることで、何を取り入れて何を捨てるかに基づいてモデルを学習させることができる。\nmnist_train_data = mnist_train.loc[:, \"pixel0\":]\nmnist_train_label = mnist_train.loc[:, \"label\"]\n\n# 画像の配列を最大値で除算して0-1の範囲になるようにします. \n# 画像のピクセルの値の範囲は255なので, ここでは255とします. \nmnist_train_data = mnist_train_data/255.0\nmnist_test = mnist_test/255.0","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 数字を配列で可視化","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# グラフを見る \ndigit_array = mnist_train.loc[3, \"pixel0\":]\narr = np.array(digit_array) \n\n#.reshape(a, (28,28))\nimage_array = np.reshape(arr, (28,28))\n\ndigit_img = plt.imshow(image_array, cmap=plt.cm.binary)\nplt.colorbar(digit_img)\nprint(\"IMAGE LABEL: {}\".format(mnist_train.loc[3, \"label\"]))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# MINIST DigitsでのPCAの実装\n## 説明変数の中に、お互いに相関関係にある複数の説明変数が存在すると、モデルはそれらの説明変数の影響を強く受けてバイアスがかかった状態となってしまいます。その状態を解消するのが主成分分析(PCA)です。\n\n## 主成分分析は「相関関係にある複数の説明変数」を相関関係の少ない説明変数にまとめます。これにより説明変数の相関によるバイアスが軽減され、モデルはより適切な学習を行えるようになります。\n## https://logics-of-blue.com/principal-components-analysis/\n\n### 1) 手動でのアプローチを使用:","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"#### a) データの標準化","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\n\nstandardized_scalar = StandardScaler()\nstandardized_data = standardized_scalar.fit_transform(mnist_train_data)\nstandardized_data.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### b) 共分散行列 S(dxd) の計算","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"cov_matrix = np.matmul(standardized_data.T, standardized_data)\ncov_matrix.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### c) 固有値と固有ベクトルの計算","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from scipy.linalg import eigh\n\nlambdas, vectors = eigh(cov_matrix, eigvals=(782, 783))\nvectors.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vectors = vectors.T\nvectors.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### d) 単位ベクトルU1=V1と新しい座標を計算する","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"new_coordinates = np.matmul(vectors, standardized_data.T)\nprint(new_coordinates.shape)\nnew_coordinates = np.vstack((new_coordinates, mnist_train_label)).T","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_new = pd.DataFrame(new_coordinates, columns=[\"f1\", \"f2\", \"labels\"])\ndf_new.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### e) seabornを使ってFacetGridをプロットする（facet_grid を利用することで、複数のグラフを一枚のプロットに表示させることができる。）","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.FacetGrid(df_new, hue=\"labels\", size=6).map(plt.scatter, \"f1\", \"f2\").add_legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 2) Sci-kit Learnライブラリーを使用(https://qiita.com/supersaiakujin/items/138c0d8e6511735f1f45)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn import decomposition\n\npca = decomposition.PCA()\npca.n_components = 2\npca_data = pca.fit_transform(standardized_data)\npca_data.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pca_data = np.vstack((pca_data.T, mnist_train_label)).T","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_PCA = pd.DataFrame(new_coordinates, columns=[\"f1\", \"f2\", \"labels\"])\ndf_PCA.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.FacetGrid(df_new, hue=\"labels\", size=12).map(plt.scatter, \"f1\", \"f2\").add_legend()\nplt.savefig(\"PCA_FacetGrid.png\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## PCA次元削減(https://aiacademy.jp/media/?p=256)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# 関数の意味http://yshampei.hatenablog.com/entry/2017/11/18/180402 https://qiita.com/Sa_qiita/items/fc61f776cef657242e69\npca.n_components = 784\npca_data = pca.fit_transform(standardized_data)\npercent_variance_retained = pca.explained_variance_ / np.sum(pca.explained_variance_)\n\ncum_variance_retained = np.cumsum(percent_variance_retained)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nplt.figure(1, figsize=(10, 6))\nplt.clf()\nplt.plot(cum_variance_retained, linewidth=2)\nplt.axis(\"tight\")\nplt.grid()\nplt.xlabel(\"number of compoments\")\nplt.ylabel(\"cumulative variance retained\")\nplt.savefig(\"pca_cumulative_variance.png\")\nplt.show()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### データ処理","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# グラフでどの数字がどれだけあるか見る\nsns.countplot(mnist_train.label)\nprint(list(mnist_train.label.value_counts().sort_index()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# データフレームを配列に変換\nmnist_train_data = np.array(mnist_train_data)\nmnist_train_label = np.array(mnist_train_label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# さっきのトレーニングデータの形を28×28×1のグレースケールに再編成\nmnist_train_data = mnist_train_data.reshape(mnist_train_data.shape[0], 28, 28, 1)\nprint(mnist_train_data.shape, mnist_train_label.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### モデルを作る\n> 入力層、隠れ層、出力層がそれぞれ一つずつあるニューラルネットワークで学習させる","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# 必要なライブラリを読み込む\n# TensorFlow is Google's open source AI framework and we are using is here to build model.\n# Keras is built on top of Tensorflow and gives us\n# NO MORE GEEKY STUFF, Know more about them here:  https://www.tensorflow.org     https://keras.io\n\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense, Dropout, Lambda, Flatten, BatchNormalization\nfrom tensorflow.keras.layers import Conv2D, MaxPool2D, AvgPool2D\nfrom tensorflow.keras.optimizers import Adadelta\nfrom keras.utils.np_utils import to_categorical\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau\nfrom tensorflow.keras.callbacks import LearningRateScheduler","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### トレーニングデータのラベルをエンコード","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# ラベルをエンコードしてクラスラベルにし、それをカテゴリ変数として変換する\nnclasses = mnist_train_label.max() - mnist_train_label.min() + 1\nmnist_train_label = to_categorical(mnist_train_label, num_classes = nclasses)\nprint(\"Shape of ytrain after encoding: \", mnist_train_label.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 一連のモデルをつくる","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model(input_shape=(28, 28, 1)):\n    model = Sequential()\n    model.add(Conv2D(32, kernel_size = 3, activation='relu', input_shape = input_shape))\n    model.add(BatchNormalization())\n    model.add(Conv2D(32, kernel_size = 3, activation='relu'))\n    model.add(BatchNormalization())\n    model.add(Conv2D(32, kernel_size = 5, strides=2, padding='same', activation='relu'))\n    model.add(BatchNormalization())\n    model.add(Dropout(0.4))\n\n    model.add(Conv2D(64, kernel_size = 3, activation='relu'))\n    model.add(BatchNormalization())\n    model.add(Conv2D(64, kernel_size = 3, activation='relu'))\n    model.add(BatchNormalization())\n    model.add(Conv2D(64, kernel_size = 5, strides=2, padding='same', activation='relu'))\n    model.add(BatchNormalization())\n    model.add(Dropout(0.4))\n\n    model.add(Conv2D(128, kernel_size = 4, activation='relu'))\n    model.add(BatchNormalization())\n    model.add(Flatten())\n    model.add(Dropout(0.4))\n    model.add(Dense(10, activation='softmax'))\n    return model\n\n    \ndef compile_model(model, optimizer='adam', loss='categorical_crossentropy'):\n    model.compile(optimizer=optimizer, loss=loss, metrics=[\"accuracy\"])\n    \n    \ndef train_model(model, train, test, epochs, split):\n    history = model.fit(train, test, shuffle=True, epochs=epochs, validation_split=split)\n    return history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# モデルの構築、コンパイル、訓練を行うために構築された上記の関数を使用してモデルを訓練する\ncnn_model = build_model((28, 28, 1))\ncompile_model(cnn_model, 'adam', 'categorical_crossentropy')\n\n# モデルを好きなだけエポック「一つの訓練データを何回繰り返して学習させるか」してトレーニングするが、、80以上でトレーニングしても何の役にも立ちませんし、最終的には過学習を増加させることがわかった\nmodel_history = train_model(cnn_model, mnist_train_data, mnist_train_label, 80, 0.2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### モデルの性能分析","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_model_performance(metric, validations_metric):\n    plt.plot(model_history.history[metric],label = str('Training ' + metric))\n    plt.plot(model_history.history[validations_metric],label = str('Validation ' + metric))\n    plt.legend()\n    plt.savefig(str(metric + '_plot.png'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_model_performance('accuracy', 'val_accuracy')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_model_performance('loss', 'val_loss')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### テストデータの変換","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# 訓練データと同様にテストデータを配列に変換する\nmnist_test_arr = np.array(mnist_test)\nmnist_test_arr = mnist_test_arr.reshape(mnist_test_arr.shape[0], 28, 28, 1)\nprint(mnist_test_arr.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 予想と提出","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# テストデータの予測\npredictions = cnn_model.predict(mnist_test_arr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 確信度の一番高いものを判定してリストに追加「https://www.tensorflow.org/tutorials/keras/classification?hl=ja」\npredictions_test = []\n\nfor i in predictions:\n    predictions_test.append(np.argmax(i))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 提出\nsubmission =  pd.DataFrame({\n        \"ImageId\": mnist_test.index+1,\n        \"Label\": predictions_test\n    })\n\nsubmission.to_csv('my_submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 関数の説明　https://keras.io/ja/models/sequential/\n### http://yshampei.hatenablog.com/entry/2017/11/18/180402 https://qiita.com/Sa_qiita/items/fc61f776cef657242e69\n","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## 参考スクリプト　https://www.kaggle.com/blurredmachine/mnist-classification-eda-pca-cnn-99-7-score","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}