{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"CNN for Mnist get 99+% accuracy","metadata":{}},{"cell_type":"markdown","source":"# 导入相关常用库","metadata":{"ExecuteTime":{"end_time":"2021-04-14T03:21:33.734602Z","start_time":"2021-04-14T03:21:33.728617Z"}}},{"cell_type":"markdown","source":"## 数据分析和清洗库","metadata":{"ExecuteTime":{"end_time":"2021-04-14T05:25:16.570052Z","start_time":"2021-04-14T05:25:16.564067Z"}}},{"cell_type":"code","source":"# 导入数据分析常用库，该部分模块可用于数据集的分析和可视化\n%matplotlib inline\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"ExecuteTime":{"end_time":"2021-04-14T07:07:39.686418Z","start_time":"2021-04-14T07:07:38.149505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 机器学习相关库","metadata":{}},{"cell_type":"code","source":"# 导入sklearn相关库+(xgboost)\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.model_selection import train_test_split, GridSearchCV\nfrom sklearn.pipeline import make_pipeline, Pipeline\n\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.ensemble import RandomForestClassifier\nfrom xgboost import XGBClassifier","metadata":{"ExecuteTime":{"end_time":"2021-04-14T07:07:40.18709Z","start_time":"2021-04-14T07:07:39.688387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 深度学习相关库","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.layers import Dense\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.optimizers import SGD\n\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras.layers import Dense, Flatten, Dropout, Conv2D, MaxPool2D\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.optimizers import RMSprop\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, ModelCheckpoint","metadata":{"ExecuteTime":{"end_time":"2021-04-14T07:07:43.251857Z","start_time":"2021-04-14T07:07:40.189049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 数据处理","metadata":{"ExecuteTime":{"end_time":"2021-04-14T03:29:16.285161Z","start_time":"2021-04-14T03:29:16.258552Z"}}},{"cell_type":"markdown","source":"## 导入数据集","metadata":{}},{"cell_type":"code","source":"# 导入csv数据\ndf_train = pd.read_csv(\"../input/digit-recognizer/train.csv\")\ndf_test = pd.read_csv(\"../input/digit-recognizer/test.csv\")","metadata":{"ExecuteTime":{"end_time":"2021-04-14T07:07:47.154421Z","start_time":"2021-04-14T07:07:43.253852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 查看数据情况","metadata":{}},{"cell_type":"code","source":"# 查看数据集相关信息\n# 通过info()函数可以得知数据集没有为None的情况，可以跳过数据清洗环节\ndf_train.info()\ndf_test.info()","metadata":{"ExecuteTime":{"end_time":"2021-04-14T07:07:47.317986Z","start_time":"2021-04-14T07:07:47.156416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 查看数据，(42000, 785)，42000个样本，每个样本为28*28=785个特征，即28*28像素的图片，每一个值为0~255\nprint(df_train.shape)","metadata":{"ExecuteTime":{"end_time":"2021-04-14T07:07:47.324966Z","start_time":"2021-04-14T07:07:47.319982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 创建样本数据集","metadata":{}},{"cell_type":"code","source":"# 转换成narray y/255.0 将数据缩放到0~1之间\nX = np.array(df_train.drop(['label'], axis=1)).astype(np.float32)/255.0\ny = np.array(df_train['label'])","metadata":{"ExecuteTime":{"end_time":"2021-04-14T07:07:47.918378Z","start_time":"2021-04-14T07:07:47.327963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 数据可视化分析","metadata":{}},{"cell_type":"code","source":"# 打印部分手写体图片\nfig, axes = plt.subplots(5, 5, figsize=(10, 10))\naxes = axes.flatten()\n\nfor ax, x, label in zip(axes, X[:25], y[:25]):\n    img = x.reshape(28, 28)\n    ax.set_xticks([])\n    ax.set_yticks([])\n    ax.imshow(img, cmap='Greys')\n    ax.set_title(label)","metadata":{"ExecuteTime":{"end_time":"2021-04-14T07:07:48.981536Z","start_time":"2021-04-14T07:07:47.922369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 拆分训练数据和测试数据","metadata":{}},{"cell_type":"code","source":"# 拆分训练集和测试集\nX_train, X_test, y_train, y_test = train_test_split(X, y, shuffle=True, random_state=10)","metadata":{"ExecuteTime":{"end_time":"2021-04-14T07:07:49.513154Z","start_time":"2021-04-14T07:07:48.984566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 建模+模型训练+模型评估","metadata":{"ExecuteTime":{"end_time":"2021-04-14T03:30:12.780424Z","start_time":"2021-04-14T03:30:12.76461Z"}}},{"cell_type":"markdown","source":"## 逻辑回归","metadata":{}},{"cell_type":"code","source":"# 构建逻辑回归模型\n# model = LogisticRegression(max_iter=1000)\n# model.fit(X_train, y_train)\n\n# train_score = model.score(X_train, y_train)\n# test_score = model.score(X_test, y_test)\n# print('Train score:{} Test score:{}'.format(train_score, test_score))\n# Train score:0.9464444444444444 Test score:0.9188571428571428\n# executed in 39.7s, finished 08:23:30 2021-04-13","metadata":{"ExecuteTime":{"end_time":"2021-04-14T07:07:49.518103Z","start_time":"2021-04-14T07:07:49.514143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 随机森林","metadata":{"ExecuteTime":{"end_time":"2021-04-14T06:33:15.622099Z","start_time":"2021-04-14T06:33:15.619106Z"}}},{"cell_type":"code","source":"# 构建随机森林模型 RandomForestClassifier\n# model = RandomForestClassifier()\n# model.fit(X_train, y_train)\n\n# train_score = model.score(X_train, y_train)\n# test_score = model.score(X_test, y_test)\n# print('Train score:{} Test score:{}'.format(train_score, test_score))\n# Train score:1.0 Test score:0.9648571428571429\n# executed in 22.5s, finished 08:23:57 2021-04-13","metadata":{"ExecuteTime":{"end_time":"2021-04-14T07:07:49.525083Z","start_time":"2021-04-14T07:07:49.521096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## xgboost","metadata":{}},{"cell_type":"code","source":"# xgboost XGBRFClassifier\n# model = XGBClassifier(use_label_encoder=False)\n# model.fit(X_train, y_train)\n\n# train_score = model.score(X_train, y_train)\n# test_score = model.score(X_test, y_test)\n# print('Train score:{} Test score:{}'.format(train_score, test_score))\n# Train score:1.0 Test score:0.9737142857142858\n# executed in 3m 56s, finished 09:13:44 2021-04-13","metadata":{"ExecuteTime":{"end_time":"2021-04-14T07:07:49.533062Z","start_time":"2021-04-14T07:07:49.528076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 全连接神经网络","metadata":{"ExecuteTime":{"end_time":"2021-04-14T06:33:56.822829Z","start_time":"2021-04-14T06:33:56.818843Z"}}},{"cell_type":"code","source":"# 可以发现XGBClassifier耗时更久，但得到了更好的准确度\n# 如下使用深度学习构建神经网络处理手写体分类\n\n# sgd = SGD(learning_rate=.01, momentum=.9)\n\n# model = Sequential()\n# model.add(Dense(units=64, activation='relu', input_dim=(28*28)))\n# model.add(Dense(units=128, activation='relu'))\n# model.add(Dense(units=10, activation='softmax'))\n\n# model.compile(optimizer=sgd, loss='sparse_categorical_crossentropy', metrics=['accuracy'])\n# model.fit(\n#     x=X_train, y=y_train,\n#     batch_size=128,\n#     epochs=20\n# )\n\n# # 模型评估\n# loss, accuracy = model.evaluate(X_test, y_test)\n# print('loss:{} accuracy:{}'.format(loss, accuracy))\n# train:loss: 0.0473 - accuracy: 0.9863\n# test:loss:0.10506495967241271 accuracy:0.9676190614700317\n# executed in 23.5s, finished 09:08:35 2021-04-13","metadata":{"ExecuteTime":{"end_time":"2021-04-14T07:07:49.540043Z","start_time":"2021-04-14T07:07:49.536054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"raw","source":"从上面可以看出，xgboost的效果更好，但训练的时间也更长，神经网络和随机森林效果都不错，在测试数据集上准确率达到了96%+，minist数据集是一份较简单数据集，或许可以有更好的表现，现在我们通过卷积神经网络探索一下是否可以继续提升准确率","metadata":{}},{"cell_type":"markdown","source":"## 卷积神经网络","metadata":{}},{"cell_type":"markdown","source":"### 转换必要的数据格式转换","metadata":{}},{"cell_type":"code","source":"# 先将y转换成categorical格式，即2-->[0010000000]\nY_train = y_train.copy()\nY_test = y_test.copy()\n\ny_train = to_categorical(y_train, num_classes = 10)\ny_test = to_categorical(y_test, num_classes = 10)\n\nX_train = X_train.reshape(-1,28,28,1)\nX_test = X_test.reshape(-1, 28, 28, 1)","metadata":{"ExecuteTime":{"end_time":"2021-04-14T07:07:49.55301Z","start_time":"2021-04-14T07:07:49.544033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 创建模型网络","metadata":{}},{"cell_type":"code","source":"# 创建卷积神经网络\nmodel = Sequential()\n\nmodel.add(Conv2D(filters = 32, kernel_size = (5,5),padding = 'Same', \n                 activation ='relu', input_shape = (28,28,1)))\nmodel.add(Conv2D(filters = 32, kernel_size = (5,5),padding = 'Same', \n                 activation ='relu'))\nmodel.add(MaxPool2D(pool_size=(2,2)))\nmodel.add(Dropout(0.25))\n\nmodel.add(Conv2D(filters = 64, kernel_size = (3,3),padding = 'Same', \n                 activation ='relu'))\nmodel.add(Conv2D(filters = 64, kernel_size = (3,3),padding = 'Same', \n                 activation ='relu'))\nmodel.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))\nmodel.add(Dropout(0.25))\n\nmodel.add(Flatten())\nmodel.add(Dense(256, activation = \"relu\"))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(10, activation = \"softmax\"))","metadata":{"ExecuteTime":{"end_time":"2021-04-14T07:07:51.391534Z","start_time":"2021-04-14T07:07:49.556001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 编译","metadata":{}},{"cell_type":"code","source":"# 模型编译\noptimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)\nmodel.compile(optimizer = optimizer , loss = \"categorical_crossentropy\", metrics=[\"accuracy\"])","metadata":{"ExecuteTime":{"end_time":"2021-04-14T07:07:51.430427Z","start_time":"2021-04-14T07:07:51.393491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 数据增强","metadata":{}},{"cell_type":"code","source":"# 创建图片增强生成器\ndatagen = ImageDataGenerator(\n    featurewise_center=False,  # set input mean to 0 over the dataset\n    samplewise_center=False,  # set each sample mean to 0\n    featurewise_std_normalization=False,  # divide inputs by std of the dataset\n    samplewise_std_normalization=False,  # divide each input by its std\n    zca_whitening=False,  # apply ZCA whitening\n    rotation_range=10,  # randomly rotate images in the range (degrees, 0 to 180)\n    zoom_range = 0.1, # Randomly zoom image \n    width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)\n    height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)\n    horizontal_flip=False,  # randomly flip images\n    vertical_flip=False  # randomly flip images\n)\n\nbatch_size = 32\nepochs = 30\n\ntrain_gen = datagen.flow(\n    x=X_train, y=y_train,\n    batch_size=batch_size\n)\n\ntest_gen = datagen.flow(\n    x=X_test, y=y_test,\n    batch_size=batch_size\n)","metadata":{"ExecuteTime":{"end_time":"2021-04-14T07:07:51.439366Z","start_time":"2021-04-14T07:07:51.432385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 回调函数","metadata":{}},{"cell_type":"code","source":"# 当标准评估停止提升时，降低学习速率。\n# 当学习停止时，模型总是会受益于降低 2-10 倍的学习速率\n# 这个回调函数监测一个数据并且当这个数据在一定「有耐心」的训练轮之后还没有进步，那么学习速率就会被降低\n# Set a learning rate annealer\nreduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.5,\n                              patience=2, verbose=1, min_lr=0.00001)\n\n# model_check = ModelCheckpoint(\n#     filepath='D:/desktop/ep{epoch:03d}-loss{loss:.3f}-val_loss{val_loss:.3f}_accuracy{accuracy:.4f}_val_accuracy{accuracy:.4f}.h5',\n#     save_best_only=True\n# )","metadata":{"ExecuteTime":{"end_time":"2021-04-14T07:07:51.446369Z","start_time":"2021-04-14T07:07:51.441362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 模型训练","metadata":{}},{"cell_type":"code","source":"# 模型训练\nhistory = model.fit(\n    x=X_train,\n    y=y_train,\n    batch_size=batch_size,\n    epochs=2, # 可以跑30轮得到99+%得准确率\n#     epochs=epochs,\n#   callbacks=[reduce_lr, model_check],\n    callbacks=[reduce_lr],\n    validation_data=(X_test, y_test),\n    steps_per_epoch=X_train.shape[0]//batch_size+1,\n)\n# train:31414/31500- ETA: 0s - loss: 0.0281 - accuracy: 0.9926 - \n# test:val_loss: 0.0412 - val_accuracy: 0.9914\n# executed in 6m 28s, finished 13:03:05 2021-04-13","metadata":{"ExecuteTime":{"end_time":"2021-04-14T07:09:55.103233Z","start_time":"2021-04-14T07:09:16.811435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 模型评估：混淆矩阵","metadata":{}},{"cell_type":"code","source":"# 卷积神经网络可以获得99%+的准确率，效果达到可用水平，接下来通过混淆矩阵查看具体识别错误的情况\ncm = confusion_matrix(y_true=Y_test, y_pred=np.argmax(model.predict(X_test), axis=1))","metadata":{"ExecuteTime":{"end_time":"2021-04-14T07:10:01.201732Z","start_time":"2021-04-14T07:09:59.468366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cm_df = pd.DataFrame(cm)\n\nplt.figure(figsize=(10, 7))\nsns.heatmap(data=cm_df, cmap=plt.cm.Blues, center=0.5, annot=True, fmt='.0f', linewidths=.3)","metadata":{"ExecuteTime":{"end_time":"2021-04-14T07:10:03.13933Z","start_time":"2021-04-14T07:10:02.577828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 可视化erros","metadata":{}},{"cell_type":"code","source":"# 获取模型错误识别的数据\ny_pred = np.argmax(model.predict(X_test), axis=1)\nerros = (y_pred - Y_test) != 0\n\nX_erros = X_test[erros]\ny_erros = Y_test[erros]\ny_erros_pred = y_pred[erros]","metadata":{"ExecuteTime":{"end_time":"2021-04-14T07:10:23.163279Z","start_time":"2021-04-14T07:10:21.29093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 画出部分错误识别的例子\nfig, axes = plt.subplots(5, 5, figsize=(20, 20))\naxes = axes.flatten()\nfor ax, x, y, y_pred in zip(axes, X_erros, y_erros, y_erros_pred):\n    x = x.reshape(28, 28)\n    ax.imshow(x, cmap='Greys')\n    ax.set_title('right:{} wrong:{}'.format(y, y_pred))\n    ax.set_xticks([])\n    ax.set_yticks([])","metadata":{"ExecuteTime":{"end_time":"2021-04-14T07:10:25.946838Z","start_time":"2021-04-14T07:10:24.776935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 可以看出，这部分的确是因为手写非常异常导致的错误，或许直接由人都无法分辨\n# 通过上面我们可以得到通过cnn模型可以获取最佳的效果，我们可以直接将该模型进行保存","metadata":{"ExecuteTime":{"end_time":"2021-04-14T07:11:19.107236Z","start_time":"2021-04-14T07:11:19.104243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 模型保存","metadata":{}},{"cell_type":"code","source":"# 模型保存\n# model.save('D:/desktop/mnist_99.3_v1.h5')","metadata":{"ExecuteTime":{"end_time":"2021-04-14T07:11:17.524806Z","start_time":"2021-04-14T07:11:17.521815Z"},"trusted":true},"execution_count":null,"outputs":[]}]}