{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Kaggle playground: cat-in-the-dat-ii Resnet50"},{"metadata":{},"cell_type":"raw","source":"network: resnet50\nlearning late: 1e-2"},{"metadata":{},"cell_type":"markdown","source":"## Run Setting"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train setting\nTRAIN_MAP_GEN = True\nTRAIN_DATA_NAME = \"train_data_ary.npy\"\nTRAIN_DATA_PATH = \"../input/workingdata/\" + TRAIN_DATA_NAME\n\n# test setting\nTEST_MAP_GEN = True\nTEST_DATA_NAME = \"test_data_ary.npy\"\nTEST_DATA_PATH = \"../input/workingdata/\" + TEST_DATA_NAME\n\n\n\n# training or load traind parameter\nTRAIN_ENB = True\nMODEL_FILE = \"model.yaml\"\nWEIGHTS_FILE = \"resnet50.h5\"\nHISTORY_FILE = \"history_dict\"\n\n# fit setting\nEPOCHS = 100\nSTEPS_PER_EPOCH = 100\nPOS_DATA_PER_STEP = 1000\nNEG_DATA_PER_STEP = 1000\nLR = 1e-2\nPOS_COEF = 2.0\nNEG_COEF = 1.0\n\n# set random seed\nimport random\nrandom.seed(1)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## data pre process"},{"metadata":{"trusted":true},"cell_type":"code","source":"##########\n# open pandas csv\n##########\nimport pandas as pd\nsample_submission = pd.read_csv(\"../input/cat-in-the-dat-ii/sample_submission.csv\")\ntest = pd.read_csv(\"../input/cat-in-the-dat-ii/test.csv\")\ntrain = pd.read_csv(\"../input/cat-in-the-dat-ii/train.csv\")\ndisplay(train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# downsampling\n\n# check pos/neg data num\ndisplay(train[train.target == 0].shape, train[train.target == 1].shape)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##########\n# load train data\n##########\nfrom IPython.display import display\nimport pandas as pd\nimport numpy as np\n\ntrain_data = train.iloc[:,1:24] # reject id,target\ntrain_id = np.array(train.iloc[:,0].astype('int32'))\ntrain_labels = np.array(train.iloc[:,24].astype('int32'))\n\ndisplay(train_labels)\ndisplay(np.min(train_labels),np.max(train_labels),np.mean(train_labels))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##########\n# load test data\n##########\ntest_data = test.iloc[:,1:24] # reject id,target\ntest_id = np.array(test.iloc[:,0].astype('int32'))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##########\n# convert string to num mapping array\n##########\nfrom keras.preprocessing.text import Tokenizer\nimport numpy as np\n\n##############\n# 文字列を適当な数値に変換\n# NaNは-1に変換\n##############\nword_file = 'word_index.txt'\ndef to_int_ary(str_ary):\n    tokenizer = Tokenizer(split=\"\\n\")\n    tokenizer.fit_on_texts(str_ary) # 数値のマッピング生成\n\n    with open(word_file, 'a') as f:\n        print(tokenizer.word_index, file=f)\n    \n    seq = tokenizer.texts_to_sequences(str_ary) # 文字列を数値に変換\n    seq_ary = np.array(seq).flatten()\n    return seq_ary\n\ndef num_map_gen(data):\n    with open(word_file, 'w') as f:\n        print(\"# word index file\", file=f)\n\n    data_ary = []\n    for column_name, item in data.iteritems():\n        with open(word_file, 'a') as f:\n            print(column_name, file=f)\n        \n        if(item.dtype == 'object'):\n            data_ary.append(to_int_ary(item.astype('str')))\n        elif(item.dtype == 'float64'):\n            data_ary.append(item.fillna(-1).astype('float64'))\n        else:\n            raise Exception(\"error, data type is unknown. %s\" % item.dtype)\n\n    # (23, N) => (N, 23), N個の23要素のデータ列\n    return np.array(data_ary).transpose(1,0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Select Map Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data_ary = []\nif(TRAIN_MAP_GEN): # regenerate\n    train_data_ary = num_map_gen(train_data)\n    # Save train data\n    np.save(TRAIN_DATA_NAME, train_data_ary)\nelse: # load\n    train_data_ary = np.load(TRAIN_DATA_NAME)\n","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"_kg_hide-output":false,"trusted":true},"cell_type":"code","source":"def separate_pn(data, labels, ids):\n    # get 0(neg) / 1(pos) True/False mask list\n    pos_mask = (labels == 1)\n    neg_mask = (labels == 0)\n\n\n    # get pos/neg data/labels\n    # data: (112323, 1, 1, 23), labels: (112323)\n    pos_data = data[pos_mask]\n    pos_labels = labels[pos_mask]\n    neg_data = data[neg_mask]\n    neg_labels = labels[neg_mask]        \n    pos_ids = ids[pos_mask]\n    neg_ids = ids[neg_mask]\n    \n    return [[pos_ids, pos_data, pos_labels], [neg_ids, neg_data, neg_labels]]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pos_set, neg_set = separate_pn(train_data_ary, train_labels, train_id)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model Gen"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.layers import Input, Dense\nfrom keras.models import Model\nfrom keras.utils import to_categorical\nfrom keras.layers import Input, Dense\nfrom keras.models import Model\nfrom keras.layers import GlobalAveragePooling2D\n\n# input shape\ninput_shape = shape=(32,32,1,)\n\n# shape = (W, H, BATCH(NONE))\ninput_layer = Input(input_shape)\n\n#resnet50\nfrom keras.applications.resnet50 import ResNet50\nresnet50_model = ResNet50(include_top=False,\n                 weights=None,\n                 input_tensor=input_layer,\n                 input_shape=input_shape,\n                 pooling=None,\n                 classes=2)\nx = resnet50_model.output\nx = GlobalAveragePooling2D()(x)\nresnet50_prediction = Dense(1, activation = 'sigmoid')(x)\n\nprediction = resnet50_prediction\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## fit data proc"},{"metadata":{"trusted":true},"cell_type":"code","source":"# (N,23) => (N,32,32,1)\ndef data_padding(data):\n    from keras.preprocessing import sequence\n\n    # (N,23) => (N, 32)\n    data_tmp = sequence.pad_sequences(data, maxlen=32,\n                                      padding=\"post\", truncating=\"post\",\n                                      dtype=\"float64\", value=0.0)\n\n    # (N,32) => (N,1,32,1)\n    data_tmp = data_tmp.reshape(-1,1,32,1)\n    \n    # (N,1,32,1) => (N,32,32,1)\n    data_tmp = sequence.pad_sequences(data_tmp, maxlen=32,\n                                      padding=\"post\", truncating=\"post\",\n                                      dtype=\"float64\", value=0.0)\n\n    # (N,32,32,1)\n    return data_tmp\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# downsampling generator\ndef data_shuffle(pos_set, neg_set):\n        # get pos/neg data/labels\n        # data: (112323, 1, 1, 23), labels: (112323)\n        pos_ids = pos_set[0]\n        pos_data = pos_set[1]\n        pos_labels = pos_set[2]\n        neg_ids = neg_set[0]\n        neg_data = neg_set[1]\n        neg_labels = neg_set[2]        \n\n        # random select index 0...len(pos_labels) num: POS_DATA_PER_STEP\n        pos_selected_index = random.sample(list(range(len(pos_labels))), POS_DATA_PER_STEP)\n        neg_selected_index = random.sample(list(range(len(neg_labels))), NEG_DATA_PER_STEP)                          \n        \n        pos_selected_ids = pos_ids[pos_selected_index]\n        pos_selected_data = pos_data[pos_selected_index]\n        pos_selected_labels = pos_labels[pos_selected_index]\n        neg_selected_ids = neg_ids[neg_selected_index]\n        neg_selected_data = neg_data[neg_selected_index]\n        neg_selected_labels = neg_labels[neg_selected_index]\n\n        # data: (20000, 1, 1, 23), labels: (20000,)\n        selected_ids = np.concatenate([pos_selected_ids, neg_selected_ids])\n        selected_data = np.concatenate([pos_selected_data, neg_selected_data])\n        selected_labels = np.concatenate([pos_selected_labels, neg_selected_labels])\n\n        p = np.random.permutation(len(selected_labels))\n        shuffled_ids = selected_ids[p]\n        shuffled_data = selected_data[p]\n        shuffled_labels = selected_labels[p]\n        \n        return [shuffled_ids, shuffled_data, shuffled_labels]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# pos/neg_set:\n# [0]: ids (N, 1) / [1]: data (N, 23) / [2]: labels (N, 1) \ndef data_callback(pos_set, neg_set, dc_call_num):\n    while True:\n        ids, data, labels = data_shuffle(pos_set, neg_set)\n        pad_data = data_padding(data)\n\n        if(dc_call_num[0] < 5):\n            np.savetxt('pad_data0_' + str(dc_call_num[0]) + '.csv', pad_data[0][0])\n            np.savetxt('pad_data1_' + str(dc_call_num[0]) + '.csv', pad_data[1][0])\n            np.savetxt('pad_data2_' + str(dc_call_num[0]) + '.csv', pad_data[2][0])\n            np.savetxt('pad_labels_' + str(dc_call_num[0]) + '.csv', labels[0:9])\n            np.savetxt('pad_ids_' + str(dc_call_num[0]) + '.csv', ids[0:9])\n        \n        dc_call_num[0] = dc_call_num[0] + 1\n        \n        yield((pad_data, labels))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Loss Function"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras import backend as K\n\n# -(y_true * log(y_pred) + (1 - y_true) * log(1-y_pred))\ndef binary_cost_cross_entropy(y_true, y_pred):\n    #L = - (y_true * K.log(y_pred) * POS_COEF + (1 - y_true) * K.log(1-y_pred) * NEG_COEF)\n    L = - (y_true * K.log(y_pred)  + (1 - y_true) * K.log(1-y_pred) )\n\n    mean_L = K.mean(L)\n    return mean_L\n    # loss=K.mean(y_pred)\n    # return loss","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Learning"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.layers import Input, Dense\nfrom keras.models import Model\nfrom keras.utils import to_categorical\n\nmodel = Model(inputs=input_layer, outputs=prediction)\ndisplay(model.summary())\n\nfrom keras.optimizers import Adam\nmodel.compile(optimizer=Adam(lr=LR),\n#              loss=binary_cost_cross_entropy,\n              loss='binary_crossentropy',\n              metrics=['accuracy'])\n\ndc_call_num = [0]\n# pos: 487677, neg: 112323 => 1epoch: (1,000 + 1,000) x40\nhistory = model.fit_generator(data_callback(pos_set, neg_set, dc_call_num),\n                              steps_per_epoch=STEPS_PER_EPOCH, epochs=EPOCHS)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Save"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Save Model\nwith open(MODEL_FILE, \"w\") as yaml_file:\n    yaml_file.write(model.to_yaml())\n\nmodel.save_weights(WEIGHTS_FILE)\n\n# Save history\nimport pickle\nwith open(HISTORY_FILE, 'wb') as f:\n    pickle.dump(history.history, f)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## History Plot"},{"metadata":{"trusted":true},"cell_type":"code","source":"hist = history.history\n\nimport matplotlib.pyplot as plt\nfig, (axL, axR) = plt.subplots(ncols=2, figsize=(10,4))\n\n# Plot the loss in the history\naxL.plot(hist['loss'],label=\"loss for training\")\naxL.set_title('model loss')\naxL.set_xlabel('epoch')\naxL.set_ylabel('loss')\naxL.legend(loc='upper right')\n\n# Plot the acc in the history\naxR.plot(hist['accuracy'],label=\"loss for training\")\naxR.set_title('model accuracy')\naxR.set_xlabel('epoch')\naxR.set_ylabel('accuracy')\naxR.legend(loc='upper right')\n\ndisplay(fig)\n\nfig.savefig('./history.png')\nplt.close()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# release memory"},{"metadata":{"trusted":true},"cell_type":"code","source":"del train\ndel train_data\ndel train_labels\ndel train_id\ndel train_data_ary\ndel pos_set\ndel neg_set","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## test"},{"metadata":{},"cell_type":"markdown","source":"### test data map gen"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data_ary = []\nif(TEST_MAP_GEN):\n    test_data_nmap = num_map_gen(test_data)\n    test_data_ary = np.array(test_data_nmap)\n    np.save(TEST_DATA_NAME, test_data_ary)\n\nelse:\n    test_data_ary = np.load(TEST_DATA_PATH)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### predict"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\n\ntest_data_pad = data_padding(test_data_ary)\npredict = model.predict(test_data_pad)\ndisplay(predict)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### create submission file"},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame({'id':test_id,'target':predict.flatten()})\ndisplay(submission)\nsubmission.to_csv(\"submision.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Debug"},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# weight取得\n# display(model.layers[2].name)\n# display(model.layers[2].get_weights()[0]) # [0]: weights, [1]: bias\n    \n# from keras import backend as K\n# l_in = 0\n# l_out = 4\n# display(model.layers[l_in].name)\n# display(model.layers[l_out].name)\n\n\n# get_layer_output = K.function([model.layers[l_in].input],\n#                               [model.layers[l_out].output])\n# layer_output = get_layer_output(train_data_pad[0:2])\n# display(train_data_pad[0])\n# display(layer_output)\n# display(model.predict(train_data_pad[0:9]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# binary\n# predict = model.predict(data_padding(train_data_ary))\n\n# categorical\n#predict = model.predict(train_data_pad).argmax(axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# display(predict.shape)\n# display(np.min(predict.flatten()))\n# display(np.max(predict))\n# display(np.mean(predict))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Result Memo"},{"metadata":{},"cell_type":"markdown","source":"## simple full connection\n* flatten/64 relu/64 relu/1softmax: 0.1872\n* flatten/4  relu/4  relu/1softmax: 0.1872\n* flatten/256relu/256relu/1softmax: 0.1872\n### softmaxは出力の合計が1なので、2値分類では使えない\n\n\n* flatten/256relu/256relu/1relu   : 0.8128\n* flatten/256relu/256relu/1sigmoid: 0.7290\n* flatten/256leaky/256leaky/1sig  : 0.7301\n### sigmoidは0-1の実数\n\n\n### binary_crossentropy, sigmoid, dense(1) だとpredict結果が0.xxxになるので、2値分類っぽくならない\n### => categorical_crossentropy, softmax, dense(2) のcategoricalで[0,1]を出力し、argmaxで結果を得る\n###    => sigmoidも0.5以上以下で分類すればよいだけだった、学習した結果、どの結果も0になりやすいのは学習のさせ方が悪いようだ。\n###       データセットが0に偏ったデータになっていることによる影響がありそうだ。"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.4"}},"nbformat":4,"nbformat_minor":4}