{"cells":[{"metadata":{"_cell_guid":"6e3b64bd-be17-4427-925a-fb2815c4423b","_uuid":"6983bc6f737961280d16cb6632bfd0384e14be6b","trusted":true},"cell_type":"code","source":"import os\nimport random\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom skimage.io import imread, imread_collection\nfrom skimage.filters import scharr\n\nfrom sklearn.cluster import KMeans\n\nfrom tqdm import tqdm\n\nfrom subprocess import check_output\nprint(check_output([\"ls\", \"../input/\"]).decode(\"utf8\"))\n\nnp.seterr(divide='ignore', invalid='ignore')\nnp.random.seed(131)\n%matplotlib inline","execution_count":1,"outputs":[]},{"metadata":{"_cell_guid":"b9d97e79-af90-4a2b-8fa6-12a1893927f3","_uuid":"db4d820718669c8529ff79afc133e7e510bb6ff7"},"cell_type":"markdown","source":"# <center> Utility function"},{"metadata":{"_cell_guid":"4b352454-6a94-4727-97cc-96e8dd9f8d15","_uuid":"3f3d9d7420d0d68842de18b2f1a8b03a8aa74e35","collapsed":true,"trusted":true},"cell_type":"code","source":"TRAIN_PATH = '../input/stage1_train'\nTEST_PATH = '../input/stage1_test'\nTRAIN_IMAGE_PATTERN = \"%s/{}/images/{}.png\" % TRAIN_PATH\nTRAIN_MASK_PATTERN = \"%s/{}/masks/*.png\" % TRAIN_PATH\nTEST_IMAGE_PATTERN = \"%s/{}/images/{}.png\" % TEST_PATH","execution_count":2,"outputs":[]},{"metadata":{"_cell_guid":"bed34915-a814-4a28-81d8-6b57a269d7d7","_uuid":"1bd3f3b82ac1f44892723dc6b3df18be2a328186","collapsed":true,"trusted":true},"cell_type":"code","source":"def read_img(img_id, flag_train=True):\n    if flag_train:\n        img_path = TRAIN_IMAGE_PATTERN.format(img_id, img_id)\n    else:\n        img_path = TEST_IMAGE_PATTERN.format(img_id, img_id)\n    img = imread(img_path)\n    img = img[:, :, :3]\n    return img","execution_count":3,"outputs":[]},{"metadata":{"_cell_guid":"4b1be6f9-17ee-437a-8ce5-951b09660ee4","_uuid":"e3ed465cfc4ad04a5ca632c56a4baf229081d25b","collapsed":true,"trusted":true},"cell_type":"code","source":"def masks_reader(mask_id):\n    mask_path = TRAIN_MASK_PATTERN.format(mask_id, mask_id)\n    masks = imread_collection(mask_path).concatenate()\n    return masks","execution_count":4,"outputs":[]},{"metadata":{"_cell_guid":"2d1dcd0b-0776-4616-9e3a-e9d9c7f51a74","_uuid":"027e6faa4c1e580e3c259827f5d68893fbe5d634","collapsed":true,"trusted":true},"cell_type":"code","source":"def get_spector(ids):\n    img = read_img(ids)\n    masks = masks_reader(ids)\n    spector = [[],[],[]]\n    for m in masks:\n        i = img.copy()\n        ir, ig, ib = i[:,:,0], i[:,:,1], i[:,:,2]\n        ir, ig, ib, m = ir.flatten(), ig.flatten(), ib.flatten(), m.flatten()\n        ir, ig, ib = ir[m>0], ig[m>0], ib[m>0]\n        spector[0].extend(ir.tolist())\n        spector[1].extend(ig.tolist())\n        spector[2].extend(ib.tolist())\n    return spector","execution_count":5,"outputs":[]},{"metadata":{"_cell_guid":"89683d8d-3538-4cb4-9f8f-acc7e2d6dc89","_uuid":"294ad3aed0b039bd01c6bf5631a0905c9593e314","collapsed":true,"trusted":true},"cell_type":"code","source":"def read_concat_mask(mask_id):\n    masks = masks_reader(mask_id)\n    _,height, width = masks.shape\n    num_masks = masks.shape[0]\n    mask = np.zeros((height, width), np.uint32)\n    for index in range(0, num_masks):\n        mask[masks[index] > 0] = 1\n    return mask","execution_count":6,"outputs":[]},{"metadata":{"_cell_guid":"9f259902-6dd3-4bd5-815b-18fe82bb16a6","_uuid":"afdacbbfe9d9c553761273f899522caebfb38267","collapsed":true,"trusted":true},"cell_type":"code","source":"def read_contur_mask(mask_id):\n    masks = masks_reader(mask_id)\n    _,height, width = masks.shape\n    num_masks = masks.shape[0]\n    mask = np.zeros((height, width), np.uint32)\n    c = 125\n    for index in range(0, num_masks):\n        dm = scharr(masks[index])\n        mask[dm > 0] = c\n        if c!=255:\n            c += 1\n        else:\n            c = 125\n    return mask","execution_count":7,"outputs":[]},{"metadata":{"_cell_guid":"1b40e0d3-da8b-4fd7-8f33-8da639d1ba53","_uuid":"046db3e314b85ea9a47491ec2fdc7ab1ceacb99e","collapsed":true,"trusted":true},"cell_type":"code","source":"def image_ids(root_dir, ignore=[]):\n    ids = []\n    for id in os.listdir(root_dir):\n        if id in ignore:\n            print('Skipping ID:', id)\n        else:\n            ids.append(id)\n    return ids","execution_count":8,"outputs":[]},{"metadata":{"_cell_guid":"2acfc461-da5a-4c24-b642-e722a96ea5dc","_uuid":"124cc6b1d9aa710b0dbde664714478a13f399855","collapsed":true,"trusted":true},"cell_type":"code","source":"def visualizer(imgs, n=4, figsize=(16,16), title=''):\n    fig = plt.figure(figsize=figsize)\n\n    n_samples = list(range(len(imgs)))\n\n    for i in range(int(n**2)):\n        try:\n            rsample = random.choice(n_samples)\n            n_samples.remove(rsample)\n            img = imgs[rsample]\n            ax = fig.add_subplot(n,n,i+1)\n            ax.imshow(img)\n            ax.axis('off')\n        except IndexError:\n            pass\n    fig.suptitle(title)","execution_count":9,"outputs":[]},{"metadata":{"_cell_guid":"c5c6413f-a189-41d3-ba8c-1eede568db00","_uuid":"578eac46744fab3ff28a3c5311ce1a3b6137effd","collapsed":true,"trusted":true},"cell_type":"code","source":"def contur_visualizer(imgs, contur, n=2, figsize=(16,16), title=''):\n    fig = plt.figure(figsize=figsize)\n\n    n_samples = list(range(len(imgs)))\n\n    for i in range(int(n**2)):\n        try:\n            rsample = random.choice(n_samples)\n            n_samples.remove(rsample)\n            img = imgs[rsample]\n            cont = contur[rsample]\n            ax = fig.add_subplot(n,n,i+1)\n            ax.imshow(img)\n            ax.imshow(cont, alpha=0.5)\n            ax.axis('off')\n        except IndexError:\n            pass\n    fig.suptitle(title)\n    plt.savefig('contur.png')","execution_count":10,"outputs":[]},{"metadata":{"_cell_guid":"ffe4f686-0c9a-4ddb-8b3c-3e7e08cf5294","_uuid":"81414b4b3aad9b106758f2a2e556562aa508c275"},"cell_type":"markdown","source":"# <center> Reading and visualisation data"},{"metadata":{"_cell_guid":"f72a37d5-89db-4ffe-bcf0-6fe1073be50d","_uuid":"228ad04fbfb41f29e7bf43e38041753107208268","collapsed":true,"trusted":true},"cell_type":"code","source":"train_img_ids = image_ids(TRAIN_PATH)\ntest_img_ids = image_ids(TEST_PATH)","execution_count":11,"outputs":[]},{"metadata":{"_cell_guid":"87f20ab7-f261-4409-8f11-ccf44653632a","_uuid":"43d5ebdaa8d1d0f5a82182f01306dcb00e197cf0","trusted":true},"cell_type":"code","source":"train_image = [read_img(i) for i in tqdm(train_img_ids, desc='Reading train image')]\ntest_image = [read_img(i, flag_train=False) for i in tqdm(test_img_ids, desc='Reading test image')]","execution_count":12,"outputs":[]},{"metadata":{"_cell_guid":"9696f959-9e69-4d40-87cc-f50219d8b2f7","_uuid":"01740b7ad2de8938bb95ae620ba7dbe9799f3623","trusted":true},"cell_type":"code","source":"visualizer(train_image)","execution_count":13,"outputs":[]},{"metadata":{"_cell_guid":"628cde7f-7766-4560-838d-986956608f4d","_uuid":"a8dd6ce6299e2357a59bd5ac8f5e22c3e9314dcc"},"cell_type":"markdown","source":"At the part of data shows above you can see that the data set has image in gray scale and a color."},{"metadata":{"_cell_guid":"0162034f-2bb7-4dc9-a794-ff7f190850ed","_uuid":"8f9589e6273dbd43eb04f68e3ce8c37e4577e6a3"},"cell_type":"markdown","source":"# <center> Filtering training dataset"},{"metadata":{"_cell_guid":"29d37918-fa20-4f5d-87ef-1b41e251c494","_uuid":"06bd656afd0d04b86506c3fe724c811ea8df1293"},"cell_type":"markdown","source":"Let's try to break the data into several classes, which are based on the color features of the images. And I do this in two stages, at each step the KMeans algorithm is used."},{"metadata":{"_cell_guid":"8f4fbd12-f395-4d4b-a4a2-7f5ceed38729","_uuid":"b2215d13701adbe3a2e5a17c39149af0cce5aa14","collapsed":true,"trusted":true},"cell_type":"code","source":"def get_color_state1(imgs):\n    color_state = []\n    for img in imgs:\n        g = np.mean(img[:,:,0])\n        grm = np.mean(img[:,:,1]-img[:,:,0])\n        grs = np.std(img[:,:,1]-img[:,:,0])\n        color_state.append([g,grm, grs])\n    return color_state    ","execution_count":14,"outputs":[]},{"metadata":{"_cell_guid":"f93eaab2-45d8-4925-9e55-236bcb09ed20","_uuid":"d8220827be3c87ef7c3cd325487fbf5caee91f84","collapsed":true,"trusted":true},"cell_type":"code","source":"train_cs1 = get_color_state1(train_image)\ntest_cs1 = get_color_state1(test_image)","execution_count":15,"outputs":[]},{"metadata":{"_cell_guid":"12fb3df0-b067-4146-bea3-a787bcf75eb1","_uuid":"e91ec8f4ec0dadb8aa7332002974a36d6900d13d","collapsed":true,"trusted":true},"cell_type":"code","source":"X_tr = train_cs1\nX_te = test_cs1\n\nkmeans = KMeans(n_clusters=4).fit(X_tr)\ntrain_cl1 = np.argmin(kmeans.transform(X_tr), -1)\ntest_cl1 = np.argmin(kmeans.transform(X_te), -1)","execution_count":16,"outputs":[]},{"metadata":{"_cell_guid":"4bcdc855-b91d-4459-8141-b3ae18cf619b","_uuid":"2314c175870b5f3a1d2bae3aa840ed15e95504bb"},"cell_type":"markdown","source":"At the first stage I get 4 clusters. In the zero cluster, only images with a gray scale are obtained, and they can be used for training as a single whole. The first and second will be considered in the second stage. The third cluster can be removed from the training data, because we do not have examples of this cluster in the test sample."},{"metadata":{"_cell_guid":"95fc8d07-6d0c-41c8-8814-9744ee622810","_uuid":"baa85f4c87f4442cf68ca6ac50daf59e588312df","scrolled":false,"trusted":true},"cell_type":"code","source":"for j in range(4):\n    train_img_cl = []\n    for i in range(len(train_cl1)):\n        if train_cl1[i]==j:\n            train_img_cl.append(train_image[i])\n    visualizer(train_img_cl, title='Cluster '+str(j))","execution_count":17,"outputs":[]},{"metadata":{"_cell_guid":"5c4f7da2-3191-49da-a5fa-59dc3147d732","_uuid":"71789029b077f9c854e26c46e819580bc7343a4c","scrolled":false,"trusted":true},"cell_type":"code","source":"for j in range(4):\n    test_img_cl = []\n    for i in range(len(test_cl1)):\n        if test_cl1[i]==j:\n            test_img_cl.append(test_image[i])\n    visualizer(test_img_cl, title='Cluster '+str(j))","execution_count":18,"outputs":[]},{"metadata":{"_cell_guid":"a3f0eccf-f207-4f42-a756-51391de40caf","_uuid":"148bd43fd88993ffd668a4054dbc17d7f8ed99e7","trusted":true},"cell_type":"code","source":"utest_cl1 = np.unique(test_cl1)\nfreez_cl1 = set([0])&set(utest_cl1)\nutest_cl1","execution_count":19,"outputs":[]},{"metadata":{"_cell_guid":"126c4519-36d5-4080-8080-362acb78121b","_uuid":"5e3289e260f78f390f26244123b3c92742c38e51","collapsed":true,"trusted":true},"cell_type":"code","source":"train_image1 = []\ntrain_ids1 = []\ntrain_clout = {\n    'ids':[],\n    'image':[],\n    'cluster':[],\n}\nfor i in range(len(train_cl1)):\n    if train_cl1[i] in (set(utest_cl1)^set(freez_cl1)):\n        train_image1.append(train_image[i])\n        train_ids1.append(train_img_ids[i])\n    elif train_cl1[i] in freez_cl1:\n        train_clout['ids'].append(train_img_ids[i])\n        train_clout['image'].append(train_image[i])\n        train_clout['cluster'].append(train_cl1[i])","execution_count":20,"outputs":[]},{"metadata":{"_cell_guid":"4bb991ca-5ad0-408e-8d96-df462ecf69d2","_uuid":"e2d361e5c23c257b775905cb0b51ea0d6bef9465","collapsed":true,"trusted":true},"cell_type":"code","source":"test_image1 = []\ntest_ids1 = []\ntest_clout = {\n    'ids':[],\n    'image':[],\n    'cluster':[],\n}\nfor i in range(len(test_cl1)):\n    if test_cl1[i] in (set(utest_cl1)^set(freez_cl1)):\n        test_image1.append(test_image[i])\n        test_ids1.append(test_img_ids[i])\n    elif test_cl1[i] in freez_cl1:\n        test_clout['ids'].append(test_img_ids[i])\n        test_clout['image'].append(test_image[i])\n        test_clout['cluster'].append(test_cl1[i])","execution_count":21,"outputs":[]},{"metadata":{"_cell_guid":"7064e92e-142f-44bd-ba22-d330e0e53c5c","_uuid":"8473ce209f43b137cd9aa4055a522a95488a726d"},"cell_type":"markdown","source":"In the second stage, I get 2 clusters, and then I add only the zero cluster to the training data and delete the noise images."},{"metadata":{"_cell_guid":"8d4e7d7c-b755-4f22-a8c0-65661bfbf0c9","_uuid":"0fe534e2baa3ae3434ec3e04a4a79de09af4a018","collapsed":true,"trusted":true},"cell_type":"code","source":"def get_color_state2(imgs):\n    color_state = []\n    for img in imgs:\n        r = np.mean(img[:,:,0])\n        b = np.mean(img[:,:,2])\n\n        rs = np.std(img[:,:,0])\n        bs = np.std(img[:,:,2])\n        color_state.append([r, rs, b, bs])\n    return color_state  ","execution_count":22,"outputs":[]},{"metadata":{"_cell_guid":"fddd983e-d370-4190-ae18-a67971575b7e","_uuid":"8b79baecf2cabdddc06a60efbf66467dfd0d32f9","collapsed":true,"trusted":true},"cell_type":"code","source":"train_cs2 = get_color_state2(train_image1)\ntest_cs2 = get_color_state2(test_image1)","execution_count":23,"outputs":[]},{"metadata":{"_cell_guid":"5b97f4be-b6e3-4cde-9b44-0548e778e6d7","_uuid":"55ac8289824296d56f61994c95175dfb54ee4606","collapsed":true,"trusted":true},"cell_type":"code","source":"X_tr = train_cs2\nX_te = test_cs2\n\nkmeans = KMeans(n_clusters=2).fit(X_tr)\ntrain_cl2 = np.argmin(kmeans.transform(X_tr), -1)\ntest_cl2 = np.argmin(kmeans.transform(X_te), -1)","execution_count":24,"outputs":[]},{"metadata":{"_cell_guid":"f46d5907-21b7-4d8a-8512-04beadea8843","_uuid":"518bb7811e41268915afe0be8590ec3c5848dd1d","trusted":true},"cell_type":"code","source":"utest_cl2 = np.unique(test_cl2)\nfreez_cl2 = set([0])&set(utest_cl2)\nutest_cl2","execution_count":25,"outputs":[]},{"metadata":{"_cell_guid":"bfe4c922-dd23-4946-9a19-bf39e0d2b354","_uuid":"9b38237512291d9d4173d0f08ec5066665d58eb4","collapsed":true,"trusted":true},"cell_type":"code","source":"for i in range(len(train_cl2)):\n    if train_cl2[i] in freez_cl2:\n        train_clout['ids'].append(train_ids1[i])\n        train_clout['image'].append(train_image1[i])\n        train_clout['cluster'].append(train_cl2[i]+1)","execution_count":26,"outputs":[]},{"metadata":{"_cell_guid":"9f80495e-3361-4462-9c2d-4387ca3cbac0","_uuid":"90d94c9266c1028209e6de9d00215c1504741ef4","collapsed":true,"trusted":true},"cell_type":"code","source":"for i in range(len(test_cl2)):\n    if test_cl2[i] in freez_cl2:\n        test_clout['ids'].append(test_ids1[i])\n        test_clout['image'].append(test_image1[i])\n        test_clout['cluster'].append(test_cl2[i]+1)","execution_count":27,"outputs":[]},{"metadata":{"_cell_guid":"0e049d1e-b382-41f1-a04c-bb3e2b4943af","_uuid":"a046c750e70e9c88f0aad721f2de96951d3bdc81","scrolled":false,"trusted":true},"cell_type":"code","source":"for j in range(2):\n    train_img_cl = []\n    for i in range(len(train_clout['cluster'])):\n        if train_clout['cluster'][i]==j:\n            train_img_cl.append(train_clout['image'][i])\n    visualizer(train_img_cl, title='Cluster '+str(j))","execution_count":28,"outputs":[]},{"metadata":{"_cell_guid":"e318849f-643d-4087-b7c2-a8dc005e3c98","_uuid":"3528d0883a4fd55f740e2e850f2411ce4959129d","scrolled":false,"trusted":true},"cell_type":"code","source":"for j in range(2):\n    test_img_cl = []\n    for i in range(len(test_clout['cluster'])):\n        if test_clout['cluster'][i]==j:\n            test_img_cl.append(test_clout['image'][i])\n    visualizer(test_img_cl, title='Cluster '+str(j))","execution_count":29,"outputs":[]},{"metadata":{"_cell_guid":"55668f87-e092-4a9f-80bc-27e77f866415","_uuid":"0185e1c5933d18901fb1aaeb7c7c5a854923eb1c","collapsed":true},"cell_type":"markdown","source":" # <center> Spector of nucleus"},{"metadata":{"_cell_guid":"66cde07b-3c3c-484e-81b9-cb0951643226","_uuid":"71b253e33b29a76af778c5852c78ac9db7e9adad","trusted":true},"cell_type":"code","source":"spector = {\n    0:[],\n    1: [[],[],[]],\n}\nfor i, c in tqdm(zip(train_clout['ids'], train_clout['cluster']), desc='Get Spetor'):\n    if c==0:\n        spector[c].extend(get_spector(i)[0])\n    else:\n        spector[c][0].extend(get_spector(i)[0])\n        spector[c][1].extend(get_spector(i)[1])\n        spector[c][2].extend(get_spector(i)[2])","execution_count":30,"outputs":[]},{"metadata":{"_cell_guid":"53ef2985-0519-4881-8471-7731e8268d13","_uuid":"de4beb7578a3f60d2b8d547ae782fa7f09653c01","trusted":true},"cell_type":"code","source":"sns.distplot(spector[0]);","execution_count":31,"outputs":[]},{"metadata":{"_cell_guid":"f7aaf424-49f7-4f59-9960-89ad51aec2de","_uuid":"fee6c2d1a41d8f423c394f50285093d620504e95","trusted":true},"cell_type":"code","source":"sns.distplot(spector[1][0], color='red', label='r')\nsns.distplot(spector[1][1], color='green', label='g')\nsns.distplot(spector[1][2], color='blue', label='b')\nplt.legend();","execution_count":32,"outputs":[]},{"metadata":{"_cell_guid":"405cb830-0866-4c24-986e-acd12e38495e","_uuid":"4db9d2a3cc59ff87211f0b2b4c5c974cf3b3f7d6"},"cell_type":"markdown","source":"At the plots above you can see the color distribution for nucleuses in various cluster."},{"metadata":{"_cell_guid":"198f56f6-2027-4d75-a76f-650a6abf200f","_uuid":"ff695d24bbf0360e957c9fa94c1fbe58e3c9125e"},"cell_type":"markdown","source":"# <center> Masks"},{"metadata":{"_cell_guid":"aed87f6d-a3ad-4b80-90a5-ee29854d9735","_uuid":"21b7fa51a28948b4b6ace0b11aaf3683ecda1eaf"},"cell_type":"markdown","source":"In many open works on the Kaggle mask, neural network and training strategy are implemented without attention to the boundaries between the nucleus, which can lead to the fusion of several masks of nucleus into one. Therefore, the result obtained will not be completely correct for the task.\nIn the images below you can see that quite a few cores have a common border and some nucleus are located one above the other."},{"metadata":{"_cell_guid":"2fca6f5b-4bb3-45e3-8d0b-4cd2c65d79c9","_uuid":"ade40a22c067003c8b9149015a9fc452829625ca","trusted":true},"cell_type":"code","source":"train_clout['contur'] = []\nfor i in tqdm(train_clout['ids'], desc='Add Masks'):\n    train_clout['contur'].append(read_contur_mask(i))","execution_count":33,"outputs":[]},{"metadata":{"_cell_guid":"f1fc41e7-541b-416c-9b66-22dcf0b58e60","_uuid":"5f8915831bb9f8989477f675e809dcbdfa04e169","trusted":true},"cell_type":"code","source":"contur_visualizer(train_clout['image'], train_clout['contur'])","execution_count":34,"outputs":[]},{"metadata":{"_cell_guid":"01e5a055-456e-45db-b1b6-13bba6c06e68","_uuid":"9ad8fb72236bfa698206d095898300dad83d3c19"},"cell_type":"markdown","source":"# <center> Conclusion"},{"metadata":{"_cell_guid":"47fb7568-5e83-4d05-b671-bd21e8b8a4d9","_uuid":"f5166e96d2e29934ee86b4b080a3f0d03718fdda"},"cell_type":"markdown","source":"In this article, I tried to remove redundant data from a set of training materials and demonstrate the need to take into account the boundaries. In my opinion, for this task will be better to use r-cnn, which is able to determine the position of the nucleus and make a mask for it."}],"metadata":{"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}