{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/working/dog-breed-identification'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-07-05T13:36:31.373884Z","iopub.execute_input":"2021-07-05T13:36:31.374175Z","iopub.status.idle":"2021-07-05T13:36:31.384225Z","shell.execute_reply.started":"2021-07-05T13:36:31.374106Z","shell.execute_reply":"2021-07-05T13:36:31.383190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install d2lzh","metadata":{"execution":{"iopub.status.busy":"2021-07-05T13:36:31.385605Z","iopub.execute_input":"2021-07-05T13:36:31.386282Z","iopub.status.idle":"2021-07-05T13:36:44.365446Z","shell.execute_reply.started":"2021-07-05T13:36:31.386237Z","shell.execute_reply":"2021-07-05T13:36:44.364451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp -r /kaggle/input/dog-breed-identification /kaggle/working/dog-breed-identification","metadata":{"execution":{"iopub.status.busy":"2021-07-05T13:36:44.368858Z","iopub.execute_input":"2021-07-05T13:36:44.369208Z","iopub.status.idle":"2021-07-05T13:38:27.551184Z","shell.execute_reply.started":"2021-07-05T13:36:44.369174Z","shell.execute_reply":"2021-07-05T13:38:27.550070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pwd","metadata":{"execution":{"iopub.status.busy":"2021-07-05T13:38:27.556292Z","iopub.execute_input":"2021-07-05T13:38:27.558312Z","iopub.status.idle":"2021-07-05T13:38:28.244459Z","shell.execute_reply.started":"2021-07-05T13:38:27.558265Z","shell.execute_reply":"2021-07-05T13:38:28.243558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import collections\nimport d2lzh as d2l\nimport math\nfrom mxnet import autograd, gluon, init, nd\nfrom mxnet.gluon import data as gdata, loss as gloss, model_zoo, nn\nimport os\nimport shutil\nimport time\nimport zipfile\nimport tensorflow as tf\nfrom tensorflow import keras\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport os\nimport shutil\nimport collections\nimport math\nimport random","metadata":{"execution":{"iopub.status.busy":"2021-07-05T13:38:28.246024Z","iopub.execute_input":"2021-07-05T13:38:28.246363Z","iopub.status.idle":"2021-07-05T13:38:35.021416Z","shell.execute_reply.started":"2021-07-05T13:38:28.246325Z","shell.execute_reply":"2021-07-05T13:38:35.020584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reorg_train_valid(data_dir, train_dir, input_dir, valid_ratio, idx_label):\n    # 训练集中数量最少一类的狗的样本数\n    min_n_train_per_label = (\n        collections.Counter(idx_label.values()).most_common()[:-2:-1][0][1])\n    # 验证集中每类狗的样本数\n    n_valid_per_label = math.floor(min_n_train_per_label * valid_ratio)\n    label_count = {}\n    for train_file in os.listdir(os.path.join(data_dir, train_dir)):\n        idx = train_file.split('.')[0]\n        label = idx_label[idx]\n        d2l.mkdir_if_not_exist([data_dir, input_dir, 'train_valid', label])\n        shutil.copy(os.path.join(data_dir, train_dir, train_file),\n                    os.path.join(data_dir, input_dir, 'train_valid', label))\n        if label not in label_count or label_count[label] < n_valid_per_label:\n            d2l.mkdir_if_not_exist([data_dir, input_dir, 'valid', label])\n            shutil.copy(os.path.join(data_dir, train_dir, train_file),\n                        os.path.join(data_dir, input_dir, 'valid', label))\n            label_count[label] = label_count.get(label, 0) + 1\n        else:\n            d2l.mkdir_if_not_exist([data_dir, input_dir, 'train', label])\n            shutil.copy(os.path.join(data_dir, train_dir, train_file),\n                        os.path.join(data_dir, input_dir, 'train', label))\ndef reorg_dog_data(data_dir, label_file, train_dir, test_dir, input_dir,\n                   valid_ratio):\n    # 读取训练数据标签\n    with open(os.path.join(data_dir, label_file), 'r') as f:\n        # 跳过文件头行（栏名称）\n        lines = f.readlines()[1:]\n        tokens = [l.rstrip().split(',') for l in lines]\n        idx_label = dict(((idx, label) for idx, label in tokens))\n    reorg_train_valid(data_dir, train_dir, input_dir, valid_ratio, idx_label)\n    # 整理测试集\n    d2l.mkdir_if_not_exist([data_dir, input_dir, 'test', 'unknown'])\n    for test_file in os.listdir(os.path.join(data_dir, test_dir)):\n        shutil.copy(os.path.join(data_dir, test_dir, test_file),\n                    os.path.join(data_dir, input_dir, 'test', 'unknown'))","metadata":{"execution":{"iopub.status.busy":"2021-07-05T13:38:35.022776Z","iopub.execute_input":"2021-07-05T13:38:35.023140Z","iopub.status.idle":"2021-07-05T13:38:35.038617Z","shell.execute_reply.started":"2021-07-05T13:38:35.023102Z","shell.execute_reply":"2021-07-05T13:38:35.036307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dir = '/kaggle/working/dog-breed-identification'\nlabel_file, train_dir, test_dir = 'labels.csv', 'train', 'test'\ninput_dir, batch_size, valid_ratio = 'train_valid_test', 128, 0.1\nreorg_dog_data(data_dir, label_file, train_dir, test_dir, input_dir,valid_ratio)","metadata":{"execution":{"iopub.status.busy":"2021-07-05T13:38:35.041771Z","iopub.execute_input":"2021-07-05T13:38:35.042272Z","iopub.status.idle":"2021-07-05T13:38:39.327798Z","shell.execute_reply.started":"2021-07-05T13:38:35.042234Z","shell.execute_reply":"2021-07-05T13:38:39.326938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def transform_train(imgpath,label):\n    # 随机对图像裁剪出面积为原图像面积0.08~1倍、且高和宽之比在3/4~4/3的图像，再放缩为高和\n    # 宽均为224像素的新图像\n    feature=tf.io.read_file(imgpath)\n    feature = tf.image.decode_jpeg(feature,channels=3)\n    feature = tf.image.resize(feature, size=[400, 400])\n    seed=random.randint(8,100)/100\n    feature = tf.image.random_crop(feature, size=[int(seed*feature.shape[0]), int(seed*feature.shape[1]), 3])\n    feature = tf.image.resize(feature, size=[224, 224])\n    feature = tf.image.random_flip_left_right(feature)\n    feature = tf.image.random_flip_up_down(feature)\n    # 标准化\n    feature = tf.divide(feature, 255.)\n    # 正则化\n    mean = tf.convert_to_tensor([0.485, 0.456, 0.406])\n    std = tf.convert_to_tensor([0.229, 0.224, 0.225])\n    feature = tf.divide(tf.subtract(feature, mean), std)\n    #feature = tf.image.per_image_standardization(feature)\n    #print(feature,label)\n    return tf.image.convert_image_dtype(feature, tf.float32),label\ndef transform_test(imgpath,label):\n    feature=tf.io.read_file(imgpath)\n    feature = tf.image.decode_jpeg(feature,channels=3)\n    feature = tf.image.resize(feature, [224, 224])\n    feature = tf.divide(feature, 255.)\n    # feature = tf.image.per_image_standardization(feature)\n    mean = tf.convert_to_tensor([0.485, 0.456, 0.406])\n    std = tf.convert_to_tensor([0.229, 0.224, 0.225])\n    feature = tf.divide(tf.subtract(feature, mean), std)\n    return feature,label","metadata":{"execution":{"iopub.status.busy":"2021-07-05T13:38:39.329725Z","iopub.execute_input":"2021-07-05T13:38:39.330159Z","iopub.status.idle":"2021-07-05T13:38:39.353555Z","shell.execute_reply.started":"2021-07-05T13:38:39.330117Z","shell.execute_reply":"2021-07-05T13:38:39.343614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pathlib\ndata_root=\"/kaggle/working/dog-breed-identification/train_valid_test\"\ntrain_data_root = pathlib.Path(data_root+\"/train\")\nvalid_data_root = pathlib.Path(data_root+\"/valid\")\ntrain_valid_data_root = pathlib.Path(data_root+\"/train_valid\")\ntest_data_root = pathlib.Path(data_root+\"/test\")\nlabel_names = sorted(item.name for item in train_data_root.glob('*/') if item.is_dir())\nlabel_to_index = dict((name, index) for index, name in enumerate(label_names))\n\ntrain_all_image_paths = [str(path) for path in list(train_data_root.glob('*/*'))]\nvalid_all_image_paths = [str(path) for path in list(valid_data_root.glob('*/*'))]\ntrain_valid_all_image_paths = [str(path) for path in list(train_valid_data_root.glob('*/*'))]\ntest_all_image_paths = [str(path) for path in list(test_data_root.glob('*/*'))]\n\n\ntrain_all_image_labels = [label_to_index[pathlib.Path(path).parent.name] for path in train_all_image_paths]\nvalid_all_image_labels = [label_to_index[pathlib.Path(path).parent.name] for path in valid_all_image_paths]\ntrain_valid_all_image_labels = [label_to_index[pathlib.Path(path).parent.name] for path in train_valid_all_image_paths]\ntest_all_image_labels = [-1 for i in range(len(test_all_image_paths))]\nprint(\"First 10 images indices: \", train_valid_all_image_labels[:10])\nprint(\"First 10 labels indices: \", train_valid_all_image_labels[:10])","metadata":{"execution":{"iopub.status.busy":"2021-07-05T13:38:39.357284Z","iopub.execute_input":"2021-07-05T13:38:39.358691Z","iopub.status.idle":"2021-07-05T13:38:40.001960Z","shell.execute_reply.started":"2021-07-05T13:38:39.358536Z","shell.execute_reply":"2021-07-05T13:38:40.001133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds = tf.data.Dataset.from_tensor_slices((train_all_image_paths, train_all_image_labels)).map(transform_train).shuffle(len(train_all_image_paths)).batch(batch_size)\nvalid_ds = tf.data.Dataset.from_tensor_slices((valid_all_image_paths, valid_all_image_labels)).map(transform_train).shuffle(len(valid_all_image_paths)).batch(batch_size)\ntrain_valid_ds = tf.data.Dataset.from_tensor_slices((train_valid_all_image_paths, train_valid_all_image_labels)).map(transform_train).shuffle(len(train_valid_all_image_paths)).batch(batch_size)\ntest_ds = tf.data.Dataset.from_tensor_slices((test_all_image_paths, test_all_image_labels)).map(transform_test).shuffle(len(test_all_image_paths)).batch(batch_size)","metadata":{"execution":{"iopub.status.busy":"2021-07-05T13:38:40.005607Z","iopub.execute_input":"2021-07-05T13:38:40.007572Z","iopub.status.idle":"2021-07-05T13:38:45.654427Z","shell.execute_reply.started":"2021-07-05T13:38:40.007532Z","shell.execute_reply":"2021-07-05T13:38:45.653577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds","metadata":{"execution":{"iopub.status.busy":"2021-07-05T13:38:45.655753Z","iopub.execute_input":"2021-07-05T13:38:45.656115Z","iopub.status.idle":"2021-07-05T13:38:45.664629Z","shell.execute_reply.started":"2021-07-05T13:38:45.656071Z","shell.execute_reply":"2021-07-05T13:38:45.663876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.applications import ResNet50\nnet=ResNet50(\n    input_shape=(224, 224, 3),\n    weights='imagenet',\n    include_top=False\n)\nmodel = tf.keras.Sequential([\n    net,\n    tf.keras.layers.GlobalAveragePooling2D(),\n    tf.keras.layers.Dense(256, activation='relu',dtype=tf.float32),\n    tf.keras.layers.Dropout(.5),\n    tf.keras.layers.Dense(len(label_names), activation='softmax',dtype=tf.float32)\n])\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-07-05T13:38:45.666005Z","iopub.execute_input":"2021-07-05T13:38:45.666567Z","iopub.status.idle":"2021-07-05T13:38:48.573696Z","shell.execute_reply.started":"2021-07-05T13:38:45.666530Z","shell.execute_reply":"2021-07-05T13:38:48.572819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr = 0.1\nlr_decay = 0.01\n\ndef scheduler(epoch):\n    if epoch < 10:\n        return lr\n    else:\n        return lr * tf.math.exp(lr_decay * (10 - epoch))\n\ncallback = tf.keras.callbacks.LearningRateScheduler(scheduler)\n\nmodel.compile(optimizer=keras.optimizers.SGD(learning_rate=lr, momentum=0.9),\n        loss='sparse_categorical_crossentropy')","metadata":{"execution":{"iopub.status.busy":"2021-07-05T13:38:48.577585Z","iopub.execute_input":"2021-07-05T13:38:48.579630Z","iopub.status.idle":"2021-07-05T13:38:48.621906Z","shell.execute_reply.started":"2021-07-05T13:38:48.579585Z","shell.execute_reply":"2021-07-05T13:38:48.617913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.fit(train_ds, epochs=1 , validation_data=valid_ds,  callbacks=[callback])","metadata":{"execution":{"iopub.status.busy":"2021-07-05T13:38:48.626430Z","iopub.execute_input":"2021-07-05T13:38:48.629012Z","iopub.status.idle":"2021-07-05T13:40:40.868597Z","shell.execute_reply.started":"2021-07-05T13:38:48.628944Z","shell.execute_reply":"2021-07-05T13:40:40.867753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer=keras.optimizers.SGD(learning_rate=lr, momentum=0.9),\n        loss='sparse_categorical_crossentropy')\nmodel.fit(train_valid_ds, epochs=1 , callbacks=[callback])","metadata":{"execution":{"iopub.status.busy":"2021-07-05T13:40:40.870234Z","iopub.execute_input":"2021-07-05T13:40:40.870590Z","iopub.status.idle":"2021-07-05T13:42:32.519624Z","shell.execute_reply.started":"2021-07-05T13:40:40.870552Z","shell.execute_reply":"2021-07-05T13:42:32.518860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"probabilities=model.predict(test_ds)\npredictions=np.argmax(probabilities, axis=-1)","metadata":{"execution":{"iopub.status.busy":"2021-07-05T13:42:32.521083Z","iopub.execute_input":"2021-07-05T13:42:32.521421Z","iopub.status.idle":"2021-07-05T13:43:19.552294Z","shell.execute_reply.started":"2021-07-05T13:42:32.521386Z","shell.execute_reply":"2021-07-05T13:43:19.551397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(probabilities)","metadata":{"execution":{"iopub.status.busy":"2021-07-05T13:43:19.553696Z","iopub.execute_input":"2021-07-05T13:43:19.554302Z","iopub.status.idle":"2021-07-05T13:43:19.568073Z","shell.execute_reply.started":"2021-07-05T13:43:19.554262Z","shell.execute_reply":"2021-07-05T13:43:19.567290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/working/dog-breed-identification/sample_submission.csv')\n\nfor i, c in enumerate(df.columns[1:]):\n    df[c] = probabilities[:,i]\n\ndf.to_csv('submission.csv', index=None)","metadata":{"execution":{"iopub.status.busy":"2021-07-05T13:43:19.569291Z","iopub.execute_input":"2021-07-05T13:43:19.569910Z","iopub.status.idle":"2021-07-05T13:43:22.822447Z","shell.execute_reply.started":"2021-07-05T13:43:19.569871Z","shell.execute_reply":"2021-07-05T13:43:22.821560Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rf /kaggle/working/dog-breed-identification","metadata":{"execution":{"iopub.status.busy":"2021-07-05T13:43:22.823747Z","iopub.execute_input":"2021-07-05T13:43:22.824101Z","iopub.status.idle":"2021-07-05T13:43:25.161923Z","shell.execute_reply.started":"2021-07-05T13:43:22.824065Z","shell.execute_reply":"2021-07-05T13:43:25.160834Z"},"trusted":true},"execution_count":null,"outputs":[]}]}