{"cells":[{"metadata":{"_uuid":"835798ac8d54a3d99b9c9a7071bb13c9a2441a5a"},"cell_type":"markdown","source":"# Grouping images by hue, nuclei number and nuclei overlapping\n\nThis is an attempt to group the training images by generating some features and grouping them by using the K-means algorithm.\n\nThe goal was to group the images by color levels (hue and value) and level of nuclei detection difficulty. We estimate the level of difficulty by measuring the total number of masks and the total number of isolated masks in the union of masks.\n\nI hope it can be used as a starting point for those interested in trying to partition the data themselves."},{"metadata":{"trusted":false,"_uuid":"38189ceb0d52c25ad55e258fb59774f1e0a93237"},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport glob\nimport os\nimport tqdm\nfrom matplotlib import pyplot as plt\nfrom skimage.io import imread \nfrom skimage.color import rgb2hsv\nfrom sklearn.cluster import KMeans\nfrom skimage.morphology import label as sk_label\n\nid_list = glob.glob(os.path.join('..','input','stage1_train','[0-9a-z]*'))\nid_list = [ os.path.basename(i) for i in id_list ]\nprint('Number of train images: %s'%(len(id_list)))","execution_count":1,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"cd903d2717895e3dd58d5853ee45952ee5decf6c"},"cell_type":"code","source":"def count_masks(id, height, width):\n    mask_list = glob.glob(os.path.join('..','input','stage1_train',id,'masks','*.png'))\n    mask_list = [ os.path.basename(i) for i in mask_list]\n    mask = np.zeros((height,width))\n    n_masks = len(mask_list)\n    for i,m in enumerate(mask_list):\n        mask = np.maximum(mask, imread(os.path.join('..','input','stage1_train',id,'masks',m)))\n    mask = mask>0\n    return n_masks, sk_label(mask).max()","execution_count":2,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"f3a343d71376e278859016d11324b4ebdf003bdb"},"cell_type":"code","source":"image_size   = np.empty((len(id_list),2))\nimage_hsv    = np.empty((len(id_list),3))\nimage_masks  = np.empty((len(id_list)))\nimage_masks2 = np.empty((len(id_list)))\nimages = []\nfor i,ids in tqdm.tqdm(enumerate(id_list)):\n    img = imread(os.path.join('..','input','stage1_train',ids,'images',ids+'.png'))\n    images.extend([img])\n    assert img.shape[2]==4\n    image_size[i,...] = img.shape[0:2]\n    img_hsv = rgb2hsv(img[...,0:3])\n    image_hsv[i,...] = [np.mean(img_hsv[:,:,0])  ,np.mean(img_hsv[:,:,1]) , np.mean(img_hsv[:,:,2])]\n    image_masks[i], image_masks2[i] = count_masks(ids, img.shape[0], img.shape[1])\n\ndf = pd.DataFrame({'Id':id_list,\n                   'images':images,\n                   'height':image_size[:,0],\n                   'width':image_size[:,1],\n                   'H':image_hsv[:,0],\n                   'S':image_hsv[:,1],\n                   'V':image_hsv[:,2],\n                   'nmask':image_masks,\n                   'nmask2':image_masks2})","execution_count":3,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"4ffaeda5edf7204fcb57caf8a973dd2eca490e2f"},"cell_type":"code","source":"df['mask_diff'] = (df['nmask'] - df['nmask2'])/df['nmask']\ndf['nmask_norm'] = df['nmask']/80\nkmeans = KMeans(n_clusters=8,random_state=2018).fit(np.array(df[['nmask_norm','H','V','mask_diff']]))\ndf['cluster'] = kmeans.labels_","execution_count":4,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"7134cf9fcb03d15465fa40edfe36d72f25a5568b"},"cell_type":"code","source":"df.groupby('cluster').mean()","execution_count":5,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"335a5adfbf416fddb2417dc121027475f5644ee1"},"cell_type":"code","source":"def show_cluster(cl):\n    df_e = df[df['cluster']==cl].reset_index()\n    plt.figure(figsize=(20,2*(df_e.shape[0]//8+1)))\n    for i in range(df_e.shape[0]):\n        plt.subplot(df_e.shape[0]//8+1,8,i+1)\n        plt.imshow(df_e['images'][i])\n    plt.show()","execution_count":6,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"ac4cea5676a3327bdcb2fd357f7c48d6dcd4f4f2"},"cell_type":"code","source":"show_cluster(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"42f7049e0ccb3c49aaa993ccd44f4102f02c0b96"},"cell_type":"code","source":"show_cluster(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"e0ce50cba5481df450d6a9d306c8d7c676eeb16c"},"cell_type":"code","source":"show_cluster(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"3d564ee84105d1eb7fd3777a9ae87393366e2d7e"},"cell_type":"code","source":"show_cluster(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"063db350671c85a0c4f4e670ddd4a4dc30bbdd8b"},"cell_type":"code","source":"show_cluster(4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"c49b9c7c9a75de97bcfc7514681204cfe8985a57"},"cell_type":"code","source":"show_cluster(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"fabf0dceb764484e6439bff19220c75bdce9dbb1"},"cell_type":"code","source":"show_cluster(6)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"0ab28c3220364d1f9a6e7493838c8a49c016f183"},"cell_type":"code","source":"show_cluster(7)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.5.2"}},"nbformat":4,"nbformat_minor":1}