{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# cnn在minist数据集得分99+%","metadata":{}},{"cell_type":"code","source":"# 导入数据分析常用库，该部分模块可用于数据集的分析和可视化\n%matplotlib inline\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"ExecuteTime":{"end_time":"2021-04-14T01:21:35.097458Z","start_time":"2021-04-14T01:21:35.075755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 导入sklearn相关库+(xgboost)\nfrom sklearn.model_selection import train_test_split, GridSearchCV\nfrom sklearn.pipeline import make_pipeline, Pipeline\n\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.ensemble import RandomForestClassifier\nfrom xgboost import XGBClassifier","metadata":{"ExecuteTime":{"end_time":"2021-04-14T01:21:36.025564Z","start_time":"2021-04-14T01:21:36.021481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"# 导入csv数据\n# df_train = pd.read_csv(r'F:\\Python\\datasets\\kaggle\\started\\digit-recognizer\\train.csv')\n# df_test = pd.read_csv(r'F:\\Python\\datasets\\kaggle\\started\\digit-recognizer\\test.csv')\n\ndf_train = pd.read_csv(\"../input/digit-recognizer/train.csv\")\ndf_test = pd.read_csv(\"../input/digit-recognizer/test.csv\")","metadata":{"ExecuteTime":{"end_time":"2021-04-14T01:21:40.485155Z","start_time":"2021-04-14T01:21:36.975935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 查看数据集相关信息\n# 通过info()函数可以得知数据集没有为None的情况，可以跳过数据清洗环节\ndf_train.info()\ndf_test.info()","metadata":{"ExecuteTime":{"end_time":"2021-04-14T01:21:43.018447Z","start_time":"2021-04-14T01:21:42.943478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 查看数据，(42000, 785)，42000个样本，每个样本为28*28=785个特征，即28*28像素的图片，每一个值为0~255\nprint(df_train.shape)","metadata":{"ExecuteTime":{"end_time":"2021-04-14T01:21:44.587108Z","start_time":"2021-04-14T01:21:44.583145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 转换成narray y/255.0 将数据缩放到0~1之间\nX = np.array(df_train.drop(['label'], axis=1)).astype(np.float32)/255.0\ny = np.array(df_train['label'])","metadata":{"ExecuteTime":{"end_time":"2021-04-14T01:21:48.240985Z","start_time":"2021-04-14T01:21:47.785832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 打印部分手写体图片\nfig, axes = plt.subplots(5, 5, figsize=(10, 10))\naxes = axes.flatten()\n\nfor ax, x, label in zip(axes, X[:25], y[:25]):\n    img = x.reshape(28, 28)\n    ax.set_xticks([])\n    ax.set_yticks([])\n    ax.imshow(img, cmap='Greys')\n    ax.set_title(label)","metadata":{"ExecuteTime":{"end_time":"2021-04-14T01:02:19.380997Z","start_time":"2021-04-14T01:02:18.368462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 拆分训练集和测试集\nX_train, X_test, y_train, y_test = train_test_split(X, y, shuffle=True, random_state=10)","metadata":{"ExecuteTime":{"end_time":"2021-04-14T01:21:52.658752Z","start_time":"2021-04-14T01:21:52.187955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 构建逻辑回归模型\nmodel = LogisticRegression(max_iter=1000)\nmodel.fit(X_train, y_train)\n\ntrain_score = model.score(X_train, y_train)\ntest_score = model.score(X_test, y_test)\nprint('Train score:{} Test score:{}'.format(train_score, test_score))\n# Train score:0.9473015873015873 Test score:0.9161904761904762\n# executed in 39.7s, finished 08:23:30 2021-04-13","metadata":{"ExecuteTime":{"end_time":"2021-04-13T08:23:30.659504Z","start_time":"2021-04-13T08:22:50.940717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 构建随机森林模型 RandomForestClassifier\nmodel = RandomForestClassifier()\nmodel.fit(X_train, y_train)\n\ntrain_score = model.score(X_train, y_train)\ntest_score = model.score(X_test, y_test)\nprint('Train score:{} Test score:{}'.format(train_score, test_score))\n# Train score:1.0 Test score:0.9642857142857143\n# executed in 22.5s, finished 08:23:57 2021-04-13","metadata":{"ExecuteTime":{"end_time":"2021-04-13T08:23:57.474087Z","start_time":"2021-04-13T08:23:34.962253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# xgboost XGBRFClassifier\nmodel = XGBClassifier(use_label_encoder=False)\nmodel.fit(X_train, y_train)\n\ntrain_score = model.score(X_train, y_train)\ntest_score = model.score(X_test, y_test)\nprint('Train score:{} Test score:{}'.format(train_score, test_score))\n# Train score:1.0 Test score:0.974\n# executed in 3m 56s, finished 09:13:44 2021-04-13","metadata":{"ExecuteTime":{"end_time":"2021-04-13T09:13:44.0097Z","start_time":"2021-04-13T09:09:47.708019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 可以发现XGBClassifier耗时更久，但得到了更好的准确度\n# 如下使用深度学习构建神经网络处理手写体分类\nfrom tensorflow.keras.layers import Dense\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.optimizers import SGD\n\nsgd = SGD(learning_rate=.01, momentum=.9)\n\nmodel = Sequential()\nmodel.add(Dense(units=64, activation='relu', input_dim=(28*28)))\nmodel.add(Dense(units=128, activation='relu'))\nmodel.add(Dense(units=10, activation='softmax'))\n\nmodel.compile(optimizer=sgd, loss='sparse_categorical_crossentropy', metrics=['accuracy'])\nmodel.fit(\n    x=X_train, y=y_train,\n    batch_size=128,\n    epochs=20\n)\n\n# 模型评估\nloss, accuracy = model.evaluate(X_test, y_test)\nprint('loss:{} accuracy:{}'.format(loss, accuracy))\n# loss:0.10486899316310883 accuracy:0.966952383518219\n# executed in 23.5s, finished 09:08:35 2021-04-13","metadata":{"ExecuteTime":{"end_time":"2021-04-13T09:08:35.173751Z","start_time":"2021-04-13T09:08:11.684322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"raw","source":"从上面可以看出，xgboost的效果更好，但训练的时间也更长，神经网络和随机森林效果都不错，在测试数据集上准确率达到了96%+，minist数据集是一份较简单数据集，或许可以有更好的表现，现在我们通过卷积神经网络探索一下是否可以继续提升准确率","metadata":{}},{"cell_type":"code","source":"# 先将y转换成categorical格式，即2-->[0010000000]\nfrom tensorflow.keras.utils import to_categorical\nY_train = y_train.copy()\nY_test = y_test.copy()\n\ny_train = to_categorical(y_train, num_classes = 10)\ny_test = to_categorical(y_test, num_classes = 10)","metadata":{"ExecuteTime":{"end_time":"2021-04-14T01:21:57.779771Z","start_time":"2021-04-14T01:21:57.761091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.layers import Dense, Flatten, Dropout, Conv2D, MaxPool2D\nfrom tensorflow.keras.models import Sequential\n\n# 创建卷积神经网络\n\nmodel = Sequential()\n\nmodel.add(Conv2D(filters = 32, kernel_size = (5,5),padding = 'Same', \n                 activation ='relu', input_shape = (28,28,1)))\nmodel.add(Conv2D(filters = 32, kernel_size = (5,5),padding = 'Same', \n                 activation ='relu'))\nmodel.add(MaxPool2D(pool_size=(2,2)))\nmodel.add(Dropout(0.25))\n\nmodel.add(Conv2D(filters = 64, kernel_size = (3,3),padding = 'Same', \n                 activation ='relu'))\nmodel.add(Conv2D(filters = 64, kernel_size = (3,3),padding = 'Same', \n                 activation ='relu'))\nmodel.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))\nmodel.add(Dropout(0.25))\n\nmodel.add(Flatten())\nmodel.add(Dense(256, activation = \"relu\"))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(10, activation = \"softmax\"))","metadata":{"ExecuteTime":{"end_time":"2021-04-14T01:03:20.706166Z","start_time":"2021-04-14T01:03:15.397385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.optimizers import RMSprop\n\n# 模型编译\noptimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)\nmodel.compile(optimizer = optimizer , loss = \"categorical_crossentropy\", metrics=[\"accuracy\"])","metadata":{"ExecuteTime":{"end_time":"2021-04-14T01:03:24.143931Z","start_time":"2021-04-14T01:03:24.090863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 按照卷积神经网络更新样本数据的shape\nX_train = X_train.reshape(-1,28,28,1)\nX_test = X_test.reshape(-1, 28, 28, 1)","metadata":{"ExecuteTime":{"end_time":"2021-04-14T01:22:50.000337Z","start_time":"2021-04-14T01:22:49.996346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.preprocessing.image import ImageDataGenerator\n\n# 创建图片增强生成器\ndatagen = ImageDataGenerator(\n    featurewise_center=False,  # set input mean to 0 over the dataset\n    samplewise_center=False,  # set each sample mean to 0\n    featurewise_std_normalization=False,  # divide inputs by std of the dataset\n    samplewise_std_normalization=False,  # divide each input by its std\n    zca_whitening=False,  # apply ZCA whitening\n    rotation_range=10,  # randomly rotate images in the range (degrees, 0 to 180)\n    zoom_range = 0.1, # Randomly zoom image \n    width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)\n    height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)\n    horizontal_flip=False,  # randomly flip images\n    vertical_flip=False  # randomly flip images\n)\n\nbatch_size = 32\nepochs = 30\n\ntrain_gen = datagen.flow(\n    x=X_train, y=y_train,\n    batch_size=batch_size\n)\n\ntest_gen = datagen.flow(\n    x=X_test, y=y_test,\n    batch_size=batch_size\n)","metadata":{"ExecuteTime":{"end_time":"2021-04-14T01:03:38.402379Z","start_time":"2021-04-14T01:03:38.386757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.callbacks import ReduceLROnPlateau, ModelCheckpoint\n# 当标准评估停止提升时，降低学习速率。\n# 当学习停止时，模型总是会受益于降低 2-10 倍的学习速率\n# 这个回调函数监测一个数据并且当这个数据在一定「有耐心」的训练轮之后还没有进步，那么学习速率就会被降低\n# Set a learning rate annealer\nreduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.5,\n                              patience=2, verbose=1, min_lr=0.00001)\n\n# model_check = ModelCheckpoint(\n#     filepath='D:/desktop/ep{epoch:03d}-loss{loss:.3f}-val_loss{val_loss:.3f}_accuracy{accuracy:.4f}_val_accuracy{accuracy:.4f}.h5',\n#     save_best_only=True\n# )","metadata":{"ExecuteTime":{"end_time":"2021-04-14T01:04:15.684621Z","start_time":"2021-04-14T01:04:15.679623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 模型训练\nhistory = model.fit(\n    x=X_train,\n    y=y_train,\n    batch_size=batch_size,\n    epochs=2, # 如果再本地可以使用高性能gpu，可以设置epochs=30，以导到99%+的准确率\n#     epochs=epochs,\n#   callbacks=[reduce_lr, model_check],\n    callbacks=[reduce_lr],\n    validation_data=(X_test, y_test),\n    steps_per_epoch=X_train.shape[0]//batch_size+1,\n)\n# train:31414/31500- ETA: 0s - loss: 0.0281 - accuracy: 0.9926 - \n# test:val_loss: 0.0412 - val_accuracy: 0.9914\n# executed in 6m 28s, finished 13:03:05 2021-04-13","metadata":{"ExecuteTime":{"end_time":"2021-04-14T01:13:56.604457Z","start_time":"2021-04-14T01:04:16.861746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 卷积神经网络可以获得99%+的准确率，效果达到可用水平，接下来通过混淆矩阵查看具体识别错误的情况\nfrom sklearn.metrics import confusion_matrix\n\ncm = confusion_matrix(y_true=Y_test, y_pred=np.argmax(model.predict(X_test), axis=1))","metadata":{"ExecuteTime":{"end_time":"2021-04-14T01:24:14.124427Z","start_time":"2021-04-14T01:24:12.318846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cm_df = pd.DataFrame(cm)\n\nplt.figure(figsize=(10, 7))\nsns.heatmap(data=cm_df, cmap=plt.cm.Blues, center=0.5, annot=True, fmt='.0f', linewidths=.3)","metadata":{"ExecuteTime":{"end_time":"2021-04-14T01:28:06.713763Z","start_time":"2021-04-14T01:28:06.293839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 获取模型错误识别的数据\ny_pred = np.argmax(model.predict(X_test), axis=1)\nerros = (y_pred - Y_test) != 0\n\nX_erros = X_test[erros]\ny_erros = Y_test[erros]\ny_erros_pred = y_pred[erros]","metadata":{"ExecuteTime":{"end_time":"2021-04-14T01:37:43.109052Z","start_time":"2021-04-14T01:37:41.319553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 画出部分错误识别的例子\nfig, axes = plt.subplots(5, 5, figsize=(20, 20))\naxes = axes.flatten()\nfor ax, x, y, y_pred in zip(axes, X_erros, y_erros, y_erros_pred):\n    x = x.reshape(28, 28)\n    ax.imshow(x, cmap='Greys')\n    ax.set_title('right:{} wrong:{}'.format(y, y_pred))\n    ax.set_xticks([])\n    ax.set_yticks([])","metadata":{"ExecuteTime":{"end_time":"2021-04-14T01:40:01.621891Z","start_time":"2021-04-14T01:40:00.77592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 可以看出，这部分的确是因为手写非常异常导致的错误，或许直接由人都无法分辨\n# 通过上面我们可以得到通过cnn模型可以获取最佳的效果，我们可以直接将该模型进行保存","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 模型保存\nmodel.save(\"../output/kaggle/working/mnist_99_cnn_model.h5\")","metadata":{"ExecuteTime":{"end_time":"2021-04-14T01:42:34.153819Z","start_time":"2021-04-14T01:42:33.824384Z"},"trusted":true},"execution_count":null,"outputs":[]}]}