{"cells":[{"cell_type":"code","metadata":{"_cell_guid":"a64f73aa-ad9d-4dbe-a265-112abd9db6f9","collapsed":true,"_uuid":"bfb57a2184d8d27882228f4a62abcf91d367bffd"},"outputs":[],"execution_count":null,"source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom glob import glob\nimport os\nfrom skimage.io import imread\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\n# dsb_data_dir = os.path.join('.', 'input')\ndsb_data_dir = os.path.join('../input/data-science-bowl-2018/')\nstage_label = 'stage1'"},{"cell_type":"code","metadata":{"_cell_guid":"8119704a-96fb-41d6-accc-6353898b1157","_uuid":"9d536a162df3387c4ceb4ff41b4024eeeb8e8fee"},"outputs":[],"execution_count":null,"source":"train_labels = pd.read_csv(os.path.join(dsb_data_dir,'{}_train_labels.csv'.format(stage_label)))\n# train_labels = pd.read_csv('../input/data-science-bowl-2018/stage1_train_labels.csv')\ntrain_labels['EncodedPixels'] = train_labels['EncodedPixels'].map(lambda ep: [int(x) for x in ep.split(' ')])\ntrain_labels.sample(3)"},{"cell_type":"code","metadata":{"_cell_guid":"475316a9-0b0b-4d98-93a9-a79da9f782c2","_uuid":"0ab5e9737931d9a5436f53d6ff5330e26906b6e4"},"outputs":[],"execution_count":null,"source":"# all_images = glob(os.path.join(dsb_data_dir, 'stage1_*', '*', '*', '*'))\nall_images = glob(os.path.join(dsb_data_dir, 'stage1_*', '*', '*', '*'))\nimg_df = pd.DataFrame({'path': all_images})\nimg_id = lambda in_path: in_path.split('/')[-3]\nimg_type = lambda in_path: in_path.split('/')[-2]\nimg_group = lambda in_path: in_path.split('/')[-4].split('_')[1]\nimg_stage = lambda in_path: in_path.split('/')[-4].split('_')[0]\nimg_df['ImageId'] = img_df['path'].map(img_id)\nimg_df['ImageType'] = img_df['path'].map(img_type)\nimg_df['TrainingSplit'] = img_df['path'].map(img_group)\nimg_df['Stage'] = img_df['path'].map(img_stage)\nimg_df.sample(2)"},{"cell_type":"code","metadata":{"_cell_guid":"c585dbbb-ea7e-451a-881a-f84d3c8d642a","_uuid":"3b4788c443cc2d11ccaccb9765cf1f9e6893bde7"},"outputs":[],"execution_count":null,"source":"%%time\ntrain_df = img_df.query('TrainingSplit==\"train\"')\ntrain_rows = []\ngroup_cols = ['Stage', 'ImageId']\nfor n_group, n_rows in train_df.groupby(group_cols):\n    c_row = {col_name: col_value for col_name, col_value in zip(group_cols, n_group)}\n    c_row['masks'] = n_rows.query('ImageType == \"masks\"')['path'].values.tolist()\n    c_row['images'] = n_rows.query('ImageType == \"images\"')['path'].values.tolist()\n    train_rows += [c_row]\ntrain_img_df = pd.DataFrame(train_rows)    \nIMG_CHANNELS = 3\ndef read_and_stack(in_img_list):\n    return np.sum(np.stack([imread(c_img) for c_img in in_img_list], 0), 0)/255.0\ntrain_img_df['images'] = train_img_df['images'].map(read_and_stack).map(lambda x: x[:,:,:IMG_CHANNELS])\ntrain_img_df['masks'] = train_img_df['masks'].map(read_and_stack).map(lambda x: x.astype(int))\ntrain_img_df.sample(1)"},{"cell_type":"code","metadata":{"_cell_guid":"c6c1d73b-3829-4f15-9463-5abb87f37fd9","_uuid":"99d1598ba6f4e1d495fd31409755b8b3865ea193"},"outputs":[],"execution_count":null,"source":"n_img = 6\nfig, m_axs = plt.subplots(2, n_img, figsize = (12, 4))\nfor (_, c_row), (c_im, c_lab) in zip(train_img_df.sample(n_img).iterrows(), \n                                     m_axs.T):\n    c_im.imshow(c_row['images'])\n    c_im.axis('off')\n    c_im.set_title('Microscope')\n    \n    c_lab.imshow(c_row['masks'])\n    c_lab.axis('off')\n    c_lab.set_title('Labeled')"},{"cell_type":"code","metadata":{"_cell_guid":"105fe26b-e5dc-4f14-9c2e-e1510a879065","_uuid":"2f3b295ae6526eae5ed36538c53ac4514c75ac27"},"outputs":[],"execution_count":null,"source":"train_img_df['Red'] = train_img_df['images'].map(lambda x: np.mean(x[:,:,0]))\ntrain_img_df['Green'] = train_img_df['images'].map(lambda x: np.mean(x[:,:,1]))\ntrain_img_df['Blue'] = train_img_df['images'].map(lambda x: np.mean(x[:,:,2]))\ntrain_img_df['Gray'] = train_img_df['images'].map(lambda x: np.mean(x))\ntrain_img_df['Red-Blue'] = train_img_df['images'].map(lambda x: np.mean(x[:,:,0]-x[:,:,2]))\nsns.pairplot(train_img_df[['Gray', 'Red', 'Green', 'Blue', 'Red-Blue']])"},{"cell_type":"code","metadata":{"_cell_guid":"c445ec1a-cc4c-4914-ae28-8742f1f7af02","_uuid":"3ba32adb6d798e065ce1da9871040a184e7ca747"},"outputs":[],"execution_count":null,"source":"train_img_df['images'].map(lambda x: x.shape).value_counts()"},{"cell_type":"code","metadata":{"_cell_guid":"4acb6583-6b8c-4aa4-a40e-b2424eb820e9","_uuid":"bf855373ea38b3f723832b72ad9b9aeccd5e3889"},"outputs":[],"execution_count":null,"source":"from keras.models import Sequential\nfrom keras.layers import BatchNormalization, Conv2D, UpSampling2D, Lambda\nsimple_cnn = Sequential()\nsimple_cnn.add(BatchNormalization(input_shape = (None, None, IMG_CHANNELS), \n                                  name = 'NormalizeInput'))\nsimple_cnn.add(Conv2D(8, kernel_size = (3,3), padding = 'same'))\nsimple_cnn.add(Conv2D(8, kernel_size = (3,3), padding = 'same'))\n# use dilations to get a slightly larger field of view\nsimple_cnn.add(Conv2D(16, kernel_size = (3,3), dilation_rate = 2, padding = 'same'))\nsimple_cnn.add(Conv2D(16, kernel_size = (3,3), dilation_rate = 2, padding = 'same'))\nsimple_cnn.add(Conv2D(32, kernel_size = (3,3), dilation_rate = 3, padding = 'same'))\n\n# the final processing\nsimple_cnn.add(Conv2D(16, kernel_size = (1,1), padding = 'same'))\nsimple_cnn.add(Conv2D(1, kernel_size = (1,1), padding = 'same', activation = 'sigmoid'))\nsimple_cnn.summary()"},{"cell_type":"code","metadata":{"_cell_guid":"1e7da713-13d6-4561-b274-212ff00b0d2d","collapsed":true,"_uuid":"b7250038c8519901e2d7d64c967ae68328293396"},"outputs":[],"execution_count":null,"source":"from keras import backend as K\nsmooth = 1.\ndef dice_coef(y_true, y_pred):\n    y_true_f = K.flatten(y_true)\n    y_pred_f = K.flatten(y_pred)\n    intersection = K.sum(y_true_f * y_pred_f)\n    return (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)\ndef dice_coef_loss(y_true, y_pred):\n    return -dice_coef(y_true, y_pred)\nsimple_cnn.compile(optimizer = 'adam', \n                   loss = dice_coef_loss, \n                   metrics = [dice_coef, 'acc', 'mse'])"},{"cell_type":"code","metadata":{"_cell_guid":"5fc1363c-9dcd-4f75-9a6f-9fa370252945","_uuid":"c934f87a1eba28b915f27ff36a10f9774377b2c2"},"outputs":[],"execution_count":null,"source":"def simple_gen():\n    while True:\n        for _, c_row in train_img_df.iterrows():\n            yield np.expand_dims(c_row['images'],0), np.expand_dims(np.expand_dims(c_row['masks'],-1),0)\n\nsimple_cnn.fit_generator(simple_gen(), \n                         steps_per_epoch=train_img_df.shape[0],\n#                       epochs = 3)\n                        epochs = 1)"},{"cell_type":"code","metadata":{"_cell_guid":"931444fa-3608-4b41-97d9-ea17a7b09c3e","_uuid":"2f2f82728f4ad52a099a78fa67a002a72b92c7c9"},"outputs":[],"execution_count":null,"source":"%%time\ntest_df = img_df.query('TrainingSplit==\"test\"')\ntest_rows = []\ngroup_cols = ['Stage', 'ImageId']\nfor n_group, n_rows in test_df.groupby(group_cols):\n    c_row = {col_name: col_value for col_name, col_value in zip(group_cols, n_group)}\n    c_row['images'] = n_rows.query('ImageType == \"images\"')['path'].values.tolist()\n    test_rows += [c_row]\ntest_img_df = pd.DataFrame(test_rows)    \n\ntest_img_df['images'] = test_img_df['images'].map(read_and_stack).map(lambda x: x[:,:,:IMG_CHANNELS])\nprint(test_img_df.shape[0], 'images to process')\ntest_img_df.sample(1)"},{"cell_type":"code","metadata":{"_cell_guid":"576594a0-46e9-4e2f-9065-cf2988a38d1c","_uuid":"4f0c0bd57ec86f40539f4eaf078d270bcf75e876"},"outputs":[],"execution_count":null,"source":"%%time\ntest_img_df['masks'] = test_img_df['images'].map(lambda x: simple_cnn.predict(np.expand_dims(x, 0))[0, :, :, 0])"},{"cell_type":"code","metadata":{"_cell_guid":"a1687b4a-a503-4dbf-af09-f2a4f409cdae","scrolled":true,"_uuid":"07bd7302329ee7d0d8fe2f71b61259b9bd324861"},"outputs":[],"execution_count":null,"source":"n_img = 3\nfrom skimage.morphology import closing, opening, disk\ndef clean_img(x):\n    return opening(closing(x, disk(1)), disk(3))\nfig, m_axs = plt.subplots(3, n_img, figsize = (12, 6))\nfor (_, d_row), (c_im, c_lab, c_clean) in zip(test_img_df.sample(n_img).iterrows(), \n                                     m_axs):\n    c_im.imshow(d_row['images'])\n    c_im.axis('off')\n    c_im.set_title('Microscope')\n    \n    c_lab.imshow(d_row['masks'])\n    c_lab.axis('off')\n    c_lab.set_title('Predicted')\n    \n    c_clean.imshow(clean_img(d_row['masks']))\n    c_clean.axis('off')\n    c_clean.set_title('Clean')"},{"cell_type":"code","metadata":{"_cell_guid":"a2707d10-a832-4e11-9c35-c8a23b8a500b","collapsed":true,"_uuid":"1797b72c02c00bc985896e8c1dc0eb432d6d1fa6"},"outputs":[],"execution_count":null,"source":"from skimage.morphology import label # label regions\ndef rle_encoding(x):\n    '''\n    x: numpy array of shape (height, width), 1 - mask, 0 - background\n    Returns run length as list\n    '''\n    dots = np.where(x.T.flatten()==1)[0] # .T sets Fortran order down-then-right\n    run_lengths = []\n    prev = -2\n    for b in dots:\n        if (b>prev+1): run_lengths.extend((b+1, 0))\n        run_lengths[-1] += 1\n        prev = b\n    return run_lengths\n\ndef prob_to_rles(x, cut_off = 0.5):\n    lab_img = label(x>cut_off)\n    if lab_img.max()<1:\n        lab_img[0,0] = 1 # ensure at least one prediction per image\n    for i in range(1, lab_img.max()+1):\n        yield rle_encoding(lab_img==i)"},{"cell_type":"code","metadata":{"_cell_guid":"b92bceb7-cb00-4772-9d4d-2e97b8205fa4","collapsed":true,"_uuid":"20d4160520cd2eb82e61f8090550e1a4645511e5"},"outputs":[],"execution_count":null,"source":"_, train_rle_row = next(train_img_df.tail(5).iterrows()) \ntrain_row_rles = list(prob_to_rles(train_rle_row['masks']))"},{"cell_type":"code","metadata":{"_cell_guid":"17ec2682-9b01-406b-b062-2e395edb606c","collapsed":true,"_uuid":"fbd8c1ca58387f3178bc96b587194e5604daa51c"},"outputs":[],"execution_count":null,"source":"tl_rles = train_labels.query('ImageId==\"{ImageId}\"'.format(**train_rle_row))['EncodedPixels']"},{"cell_type":"code","metadata":{"_cell_guid":"f7fb53a5-2b04-41f1-9032-52d457eb77b6","_uuid":"5a6a5e248c97f57204d962323309cf61f6b599c8"},"outputs":[],"execution_count":null,"source":"match, mismatch = 0, 0\nfor img_rle, train_rle in zip(sorted(train_row_rles, key = lambda x: x[0]), \n                             sorted(tl_rles, key = lambda x: x[0])):\n    for i_x, i_y in zip(img_rle, train_rle):\n        if i_x == i_y:\n            match += 1\n        else:\n            mismatch += 1\nprint('Matches: %d, Mismatches: %d, Accuracy: %2.1f%%' % (match, mismatch, 100.0*match/(match+mismatch)))"},{"cell_type":"code","metadata":{"_cell_guid":"dceb9bd7-e48b-4dcb-91b7-c69ec7fb72dc","collapsed":true,"_uuid":"93414543127f1720002cd906e6aad9d95812c36c"},"outputs":[],"execution_count":null,"source":"test_img_df['rles'] = test_img_df['masks'].map(clean_img).map(lambda x: list(prob_to_rles(x)))"},{"cell_type":"code","metadata":{"_cell_guid":"5e081eb8-0179-4d7e-a276-8071f3f632b1","_uuid":"efb56d21a665044654a035453b1f5845098a6cee"},"outputs":[],"execution_count":null,"source":"out_pred_list = []\nfor _, c_row in test_img_df.iterrows():\n    for c_rle in c_row['rles']:\n        out_pred_list+=[dict(ImageId=c_row['ImageId'], \n                             EncodedPixels = ' '.join(np.array(c_rle).astype(str)))]\nout_pred_df = pd.DataFrame(out_pred_list)\nprint(out_pred_df.shape[0], 'regions found for', test_img_df.shape[0], 'images')\nout_pred_df.sample(3)"},{"cell_type":"code","metadata":{"_cell_guid":"e2292f82-a641-4db1-9a7d-4686fc7432ab","collapsed":true,"_uuid":"a498a80446966d564e9e5ec656e80a4f17319b0f"},"outputs":[],"execution_count":null,"source":"out_pred_df[['ImageId', 'EncodedPixels']].to_csv('predictions.csv', index = False)"},{"cell_type":"code","metadata":{"_cell_guid":"0b532b95-d80d-4102-94af-e295b3dc7c81","collapsed":true,"_uuid":"51497011f2657853900b8f29e823978f06b42f38"},"outputs":[],"execution_count":null,"source":""}],"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"version":"3.6.4","file_extension":".py","nbconvert_exporter":"python","pygments_lexer":"ipython3"}},"nbformat":4,"nbformat_minor":1}