{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n        \n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-06-08T15:55:03.661471Z","iopub.execute_input":"2022-06-08T15:55:03.661865Z","iopub.status.idle":"2022-06-08T15:55:03.670197Z","shell.execute_reply.started":"2022-06-08T15:55:03.661829Z","shell.execute_reply":"2022-06-08T15:55:03.669207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#  1.数据处理","metadata":{}},{"cell_type":"markdown","source":">  pip install py7zr -i http://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com","metadata":{}},{"cell_type":"code","source":"from py7zr import unpack_7zarchive\nimport shutil\nshutil.register_unpack_format('7zip',['.7z'],unpack_7zarchive)","metadata":{"execution":{"iopub.status.busy":"2022-06-08T15:57:59.789002Z","iopub.execute_input":"2022-06-08T15:57:59.789493Z","iopub.status.idle":"2022-06-08T15:58:00.002535Z","shell.execute_reply.started":"2022-06-08T15:57:59.789451Z","shell.execute_reply":"2022-06-08T15:58:00.001713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shutil.unpack_archive('../input/cifar-10/train.7z', '/kaggle/temp/')#解压","metadata":{"execution":{"iopub.status.busy":"2022-06-08T15:59:16.099761Z","iopub.execute_input":"2022-06-08T15:59:16.100126Z","iopub.status.idle":"2022-06-08T16:00:05.088378Z","shell.execute_reply.started":"2022-06-08T15:59:16.100096Z","shell.execute_reply":"2022-06-08T16:00:05.087519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels = pd.read_csv(\"../input/cifar-10/trainLabels.csv\", header=\"infer\")\n\nclasses = train_labels['label'].unique()\nprint(classes)   #打印lable","metadata":{"execution":{"iopub.status.busy":"2022-06-08T16:01:02.607060Z","iopub.execute_input":"2022-06-08T16:01:02.607439Z","iopub.status.idle":"2022-06-08T16:01:02.655245Z","shell.execute_reply.started":"2022-06-08T16:01:02.607407Z","shell.execute_reply":"2022-06-08T16:01:02.654347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not os.path.exists(\"/kaggle/temp/valid\"):\n    os.mkdir(\"/kaggle/temp/valid\")  #创建valid文件夹\n    \nparent_path_train = \"/kaggle/temp/train\"\nparent_path_valid = \"/kaggle/temp/valid\"\nparent_path_test = \"/kaggle/temp/test\"\n\nfor class1 in classes:\n    path_train = os.path.join(parent_path_train,class1)\n    if not os.path.exists(path_train):\n        os.mkdir(path_train)\n    path_valid = os.path.join(parent_path_valid,class1)\n    if not os.path.exists(path_valid):\n        os.mkdir(path_valid)\n        \nfor (int_ind,row) in train_labels.iterrows():\n    id = str(row[\"id\"])+\".png\"\n    source_path = os.path.join(parent_path_train,id)\n    \n    p=np.random.random()\n    #随机取值\n    if p<=0.8:\n        target_path = os.path.join(parent_path_train,row[\"label\"],id)\n        os.replace(source_path, target_path)\n    else:\n        target_path = os.path.join(parent_path_valid,row[\"label\"],id)\n        os.replace(source_path, target_path)","metadata":{"execution":{"iopub.status.busy":"2022-06-08T16:01:04.150389Z","iopub.execute_input":"2022-06-08T16:01:04.151323Z","iopub.status.idle":"2022-06-08T16:01:08.494704Z","shell.execute_reply.started":"2022-06-08T16:01:04.151284Z","shell.execute_reply":"2022-06-08T16:01:08.493717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2.模型搭建","metadata":{}},{"cell_type":"code","source":"import keras\nfrom keras.datasets import cifar10\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Activation, Flatten\nfrom keras.layers import Conv2D, MaxPooling2D\nimport os\n\nimport numpy as np\n\nimport seaborn as sns\nimport matplotlib\nimport matplotlib.pyplot as plt\n\nfrom sklearn.metrics import confusion_matrix, classification_report\nimport itertools\nimport os\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nimport tensorflow as tf\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras.layers import BatchNormalization\nfrom tensorflow.keras.regularizers import l2\nfrom tensorflow.keras.layers import Dense, Dropout, Activation,Flatten,Conv2D,MaxPooling2D\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.datasets import cifar10 \nimport warnings\nwarnings.filterwarnings('ignore')   #忽略warning\n\ntf.__version__\n%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2022-06-08T16:01:12.049410Z","iopub.execute_input":"2022-06-08T16:01:12.049783Z","iopub.status.idle":"2022-06-08T16:01:12.063359Z","shell.execute_reply.started":"2022-06-08T16:01:12.049753Z","shell.execute_reply":"2022-06-08T16:01:12.062586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model=Sequential()\nmodel.add( Conv2D(filters=32, kernel_size=(3,3), strides=(1,1), padding='valid', activation=None, use_bias=False, input_shape=(32,32,3)) )\nmodel.add(BatchNormalization())\nmodel.add(Activation('relu'))\n\nmodel.add( Conv2D(filters=48, kernel_size=(3,3), strides=(1,1), padding='valid', activation=None, use_bias=False) )\nmodel.add(BatchNormalization())\nmodel.add(Activation('relu'))\n\nmodel.add( Conv2D(filters=64, kernel_size=(3,3), strides=(1,1), padding='valid', activation=None, use_bias=False) )\nmodel.add(BatchNormalization())\nmodel.add(Activation('relu'))\n\nmodel.add( Conv2D(filters=80, kernel_size=(3,3), strides=(1,1), padding='valid', activation=None, use_bias=False) )\nmodel.add(BatchNormalization())\nmodel.add(Activation('relu'))\n\nmodel.add( Conv2D(filters=96, kernel_size=(3,3), strides=(1,1), padding='valid', activation=None, use_bias=False) )\nmodel.add(BatchNormalization())\nmodel.add(Activation('relu'))\n\nmodel.add( Conv2D(filters=112, kernel_size=(3,3), strides=(1,1), padding='valid', activation=None, use_bias=False) )\nmodel.add(BatchNormalization())\nmodel.add(Activation('relu'))\n\nmodel.add( Conv2D(filters=128, kernel_size=(3,3), strides=(1,1), padding='valid', activation=None, use_bias=False) )\nmodel.add(BatchNormalization())\nmodel.add(Activation('relu'))\n\nmodel.add( Conv2D(filters=144, kernel_size=(3,3), strides=(1,1), padding='valid', activation=None, use_bias=False) )\nmodel.add(BatchNormalization())\nmodel.add(Activation('relu'))\n\nmodel.add( Conv2D(filters=160, kernel_size=(3,3), strides=(1,1), padding='valid', activation=None, use_bias=False) )\nmodel.add(BatchNormalization())\nmodel.add(Activation('relu'))\n\nmodel.add( Conv2D(filters=176, kernel_size=(3,3), strides=(1,1), padding='valid', activation=None, use_bias=False) )\nmodel.add(BatchNormalization())\nmodel.add(Activation('relu'))\n\nmodel.add(Flatten())\n\nmodel.add(Dense(units=10))\nmodel.add(BatchNormalization())\nmodel.add(Activation('softmax'))\nmodel.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2022-06-08T16:01:14.014284Z","iopub.execute_input":"2022-06-08T16:01:14.014639Z","iopub.status.idle":"2022-06-08T16:01:14.232398Z","shell.execute_reply.started":"2022-06-08T16:01:14.014610Z","shell.execute_reply":"2022-06-08T16:01:14.231611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3.模型训练","metadata":{}},{"cell_type":"code","source":"train_datagen = ImageDataGenerator(featurewise_center=False,\n                             samplewise_center=False,\n                             featurewise_std_normalization=False,\n                             samplewise_std_normalization=False,\n                             zca_whitening=False,\n                             rotation_range=10,\n                             zoom_range=0.1,\n                             width_shift_range=0.1,\n                             height_shift_range=0.1,\n                             horizontal_flip=False,\n                             vertical_flip=False,\n                             rescale=1./255)\nvalid_datagen = ImageDataGenerator()\n\ntrain_generator = train_datagen.flow_from_directory(directory='/kaggle/temp/train/', shuffle=True, target_size=(32,32),batch_size=128)\nvalid_generator = valid_datagen.flow_from_directory(directory='/kaggle/temp/valid/', shuffle=True, target_size=(32,32),batch_size=128)","metadata":{"execution":{"iopub.status.busy":"2022-06-08T16:01:16.231079Z","iopub.execute_input":"2022-06-08T16:01:16.231447Z","iopub.status.idle":"2022-06-08T16:01:17.468175Z","shell.execute_reply.started":"2022-06-08T16:01:16.231418Z","shell.execute_reply":"2022-06-08T16:01:17.466404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#迭代50次\nmodel.fit(train_generator,epochs=50, validation_data=valid_generator,steps_per_epoch=train_generator.n//train_generator.batch_size,\n         validation_steps= valid_generator.n//valid_generator.batch_size,workers=8,use_multiprocessing=True)","metadata":{"execution":{"iopub.status.busy":"2022-06-08T16:03:52.095487Z","iopub.execute_input":"2022-06-08T16:03:52.096203Z","iopub.status.idle":"2022-06-08T16:33:41.099541Z","shell.execute_reply.started":"2022-06-08T16:03:52.096163Z","shell.execute_reply":"2022-06-08T16:33:41.098552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4.测试","metadata":{}},{"cell_type":"code","source":"from py7zr import unpack_7zarchive\nimport shutil\nshutil.register_unpack_format('7zip',['.7z'],unpack_7zarchive)","metadata":{"execution":{"iopub.status.busy":"2022-06-08T16:49:23.873043Z","iopub.execute_input":"2022-06-08T16:49:23.873404Z","iopub.status.idle":"2022-06-08T16:49:23.896795Z","shell.execute_reply.started":"2022-06-08T16:49:23.873366Z","shell.execute_reply":"2022-06-08T16:49:23.895700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shutil.unpack_archive('/kaggle/input/cifar-10/test.7z','/kaggle/temp/test')\nshutil.unregister_unpack_format('7zip')","metadata":{"execution":{"iopub.status.busy":"2022-06-08T16:49:31.569506Z","iopub.execute_input":"2022-06-08T16:49:31.569872Z","iopub.status.idle":"2022-06-08T17:04:49.923759Z","shell.execute_reply.started":"2022-06-08T16:49:31.569842Z","shell.execute_reply":"2022-06-08T17:04:49.922931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#加载测试集图像数据\ntest_datagen = ImageDataGenerator(rescale=1./255)\n\ntest_gen = test_datagen.flow_from_directory(directory='/kaggle/temp/test',target_size=(32,32),batch_size=64,class_mode=None,shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2022-06-08T17:06:57.133076Z","iopub.execute_input":"2022-06-08T17:06:57.133442Z","iopub.status.idle":"2022-06-08T17:07:03.666935Z","shell.execute_reply.started":"2022-06-08T17:06:57.133413Z","shell.execute_reply":"2022-06-08T17:07:03.666121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 5.预测","metadata":{}},{"cell_type":"code","source":"test_gen.reset()\npredictions_vecs = model.predict(test_gen)\npredictions_final = np.argmax(predictions_vecs, axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-06-08T17:07:06.313206Z","iopub.execute_input":"2022-06-08T17:07:06.313560Z","iopub.status.idle":"2022-06-08T17:08:28.297828Z","shell.execute_reply.started":"2022-06-08T17:07:06.313530Z","shell.execute_reply":"2022-06-08T17:08:28.296974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#print(type(train_generator.class_indices))\nprint(train_generator.class_indices)\n\nclasses = {value:key for (key,value) in train_generator.class_indices.items()}\nprint(classes)\n\npredicted_classes=np.empty(shape=300000,dtype=np.dtype('U20'))\n\nind=0\nfor i in predictions_final.tolist():\n    predicted_classes[ind]=classes[i]\n    ind=ind+1\n    \nfilenames_wo_ext = []\nfor fname in test_gen.filenames:\n    filenames_wo_ext.append(int(fname.split(sep=\"/\")[1].split(sep=\".\")[0])-1)\n\npredicted_classes_final = np.empty(shape=300000,dtype=np.dtype('U20'))\npredicted_classes_final[filenames_wo_ext]=predicted_classes","metadata":{"execution":{"iopub.status.busy":"2022-06-08T17:08:34.860647Z","iopub.execute_input":"2022-06-08T17:08:34.861013Z","iopub.status.idle":"2022-06-08T17:08:35.305382Z","shell.execute_reply.started":"2022-06-08T17:08:34.860984Z","shell.execute_reply":"2022-06-08T17:08:35.304565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv('../input/cifar-10/sampleSubmission.csv',header='infer')\n#sub.info()\nsub['label'] = predicted_classes_final\nsub.to_csv('submission.csv',index=False)\nsub","metadata":{"execution":{"iopub.status.busy":"2022-06-08T17:08:37.534365Z","iopub.execute_input":"2022-06-08T17:08:37.534739Z","iopub.status.idle":"2022-06-08T17:08:38.168336Z","shell.execute_reply.started":"2022-06-08T17:08:37.534709Z","shell.execute_reply":"2022-06-08T17:08:38.167515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}