{"cells":[{"metadata":{},"cell_type":"markdown","source":"* First to download the cifar-10 data https://www.kaggle.com/c/cifar-10/data and unzip the \"test.7z\" and \"train.7z\" in the same folder.\n* Unzip the data will take a loooong time!!!\n* Then learn to convert the data to hdf5 http://docs.h5py.org/en/stable/\n* keras supports using hdf5 for the model input: https://keras.io/io_utils/\n* Using keras in TPU train with hdf5\n* I got a score 0.71720 in this code\n* The history can be viewed in lastest version","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# I use this code in my computer in order to convert, it should be work in your computer \nfrom glob import glob\nimport numpy as np\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport pandas as pd\npf = pd.read_csv(\"trainLabels.csv\")\npf\n\nc2i = {category:id for id,category in enumerate(set(pf[\"label\"]))}\ni2c = np.asarray(list(set(pf[\"label\"])))\nnp.save(\"int2categroy.npy\",i2c)\nc2i,i2c\n\n# build the one hot matrix\ntrain_label = np.zeros([len(pf[\"label\"]),len(i2c)],dtype=\"uint8\")\nfor id, label in enumerate(pf[\"label\"]):\n    train_label[id][c2i[label]] = 1\ntrain_label\n\ndef read_image(path):\n    image = keras.preprocessing.image.load_img(path)\n    return keras.preprocessing.image.img_to_array(image)\n\ntrain_data = np.zeros([len(glob(\"train/*\")),32,32,3],dtype=\"uint8\")\nfor p in tqdm(glob(\"train/*\")):\n    id = int(p.split(\"/\")[-1].split(\".\")[0]) - 1 \n    train_data[id] = read_image(p)\n    \n# convert to train.hdf5\nimport h5py\ntrain_f = h5py.File(\"train.hdf5\",\"w\")\ntrain_f.create_dataset(\"train_data\",data=train_data)\ntrain_f.create_dataset(\"train_label\",data=train_label)\n\ntrain_f.keys(),train_f[\"train_data\"],train_f[\"train_label\"]\n\nplt.imshow(train_f[\"train_data\"][0]) # check the label\n\ntrain_f.close()\n\n# convert to train.hdf5\nimport h5py\ntest_f = h5py.File(\"test.hdf5\",\"w\")\ntest_f.create_dataset(\"test_data\",data=test_data)\n\ntest_f.keys(),test_f[\"test_data\"]\n\ntest_f.close()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"I have converted the data to hdf5 and upload it for training.","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport numpy as np\nimport h5py\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_f = h5py.File(\"../input/cifar10-compete/train.hdf5\",\"r\")\ntrain_data = train_f[\"train_data\"]\ntrain_label = train_f[\"train_label\"]\ntrain_data,train_label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(train_data[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport keras \nfrom keras.layers import *\nfrom keras.models import Model\nfrom keras.applications.densenet import preprocess_input\nfrom keras.applications import DenseNet121\nfrom keras.preprocessing.image import ImageDataGenerator","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://github.com/keras-team/keras/tree/master/examples\n# data augment\n# if we dont use the augment, the validation accuracy will be bad\ndatagen = ImageDataGenerator(\n    # set input mean to 0 over the dataset\n    featurewise_center=False,\n    # set each sample mean to 0\n    samplewise_center=False,\n    # divide inputs by std of dataset\n    featurewise_std_normalization=False,\n    # divide each input by its std\n    samplewise_std_normalization=False,\n    # apply ZCA whitening\n    zca_whitening=False,\n    # epsilon for ZCA whitening\n    zca_epsilon=1e-06,\n    # randomly rotate images in the range (deg 0 to 180)\n    rotation_range=0,\n    # randomly shift images horizontally\n    width_shift_range=0.1,\n    # randomly shift images vertically\n    height_shift_range=0.1,\n    # set range for random shear\n    shear_range=0.,\n    # set range for random zoom\n    zoom_range=0.,\n    # set range for random channel shifts\n    channel_shift_range=0.,\n    # set mode for filling points outside the input boundaries\n    fill_mode='nearest',\n    # value used for fill_mode = \"constant\"\n    cval=0.,\n    # randomly flip images\n    horizontal_flip=True,\n    # randomly flip images\n    vertical_flip=False,\n    # set rescaling factor (applied before any other transformation)\n    rescale=1./255,\n    # set function that will be applied on each input\n    preprocessing_function=None,\n    # image data format, either \"channels_first\" or \"channels_last\"\n    data_format=None,\n    # fraction of images reserved for validation (strictly between 0 and 1)\n    validation_split=0.0)\ndatagen.fit(train_data)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"How can I train a Keras model on TPU? https://keras.io/getting_started/faq/#how-can-i-train-a-keras-model-on-tpu","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection\nprint('Running on TPU: ', tpu.cluster_spec().as_dict()['worker'])\n\ntf.config.experimental_connect_to_cluster(tpu)\ntf.tpu.experimental.initialize_tpu_system(tpu)\nstrategy = tf.distribute.experimental.TPUStrategy(tpu)\nprint('Replicas: ', strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    input = Input(train_data[0].shape)\n    img_net = DenseNet121(weights=None, input_shape = train_data[0].shape, include_top=False)\n#     if using transfer learning then fixing the img_net \n#     img_net.trainable = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# we can adding a normalize layer after input layer, but I have normalize in ImageDataGenerator\n# with strategy.scope():\n#     def normalize(x):\n#         return x/255\n# #         return preprocess_input(x)\n#     Normalize = Lambda(normalize)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    x = input\n#     x = Normalize(x)\n    x = img_net(x)\n    x = Flatten()(x)\n    x = Dense(512,activation=\"relu\")(x)\n    x = Dropout(0.5)(x)\n    x = Dense(10,activation=\"softmax\")(x)\n    output = x\n    model = Model(input,output)\n    model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    model.compile(\"adam\",loss=\"categorical_crossentropy\",metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    history = model.fit_generator(datagen.flow(train_data,train_label,batch_size=256),epochs=25) \n# if you want to get a better result \n# https://github.com/keras-team/keras/blob/master/examples/cifar10_cnn.py\n# https://github.com/keras-team/keras/blob/master/examples/cifar10_resnet.py","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history_df = pd.DataFrame(history.history)\nhistory_df[[\"loss\"]].plot()\nhistory_df[[\"accuracy\"]].plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.save_weights(\"cifar10_Densenet121_.weight\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Prediction\ntest_f = h5py.File(\"../input/cifar10-compete/test.hdf5\",\"r\")\ntest_data = test_f[\"test_data\"]\ntest_data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_label = np.argmax(model.predict(test_data[:]/255),-1) # test_data[:]/255 will comsuming lots of memory. In this time is ok\n# so adding a normalize layer after input layer can prevent it","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# convert the int to categroy\ni2c = np.load(\"../input/cifar10-compete/int2categroy.npy\")\ni2c","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_label = [i2c[i] for i in test_label]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"answers = pd.DataFrame({\"id\":np.arange(1,300000+1),\"label\":test_label})\nanswers","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"answers.to_csv(\"answers.csv\",index=0) # Finally, download the answers.csv and summit it. ","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}