{"cells":[{"metadata":{"_cell_guid":"0d502410-bcf2-439e-b3f0-2d8f259b25dc","_uuid":"f13325e6a8ffc706824aeacc90696db4166bceba"},"cell_type":"markdown","source":"
\n\n
VISUALIZATION AND LOADER DATASET
\n
2018 Data Science Bowl
\n
Pedro D. Marrero Fernandez
\n
pdmf@cin.ufpe.br
\n
https://www.kaggle.com/c/data-science-bowl-2018/discussion/47770
\n
:), :(, :/
\n
"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":false,"collapsed":true},"cell_type":"code","source":"import os\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport sys\nimport PIL.Image\nimport scipy.misc\nimport cv2\nimport skimage.color as skcolor\nimport skimage.util as skutl\nimport random\nimport csv\nimport matplotlib.pyplot as plt\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"6420dfb3-60c0-4c97-a74c-7f0dfde53de0","_uuid":"ae0f2592be680fc731c4d7d9711c4a270bd1732a"},"cell_type":"markdown","source":"**DATA LOADER FOR DATA SCIENCE BOWL**"},{"metadata":{"collapsed":true,"_uuid":"091b62fd7d3839071cfbef4e60dd6c06752e4cff","_cell_guid":"2adec244-9659-449a-a81f-f8ac086df916","trusted":false},"cell_type":"code","source":"#Class dsxbProvide\nclass dsxbProvide(object):\n '''\n Mnagement for Data Science Bowl image dataset\n https://www.kaggle.com/c/data-science-bowl-2018/data\n ''' \n \n @classmethod\n def create(\n cls, \n base_folder,\n sub_folder, \n id_file_name,\n folders_image='images',\n folders_masks='masks',\n ):\n '''\n Factory function that create an instance of dsxbProvide and load the data form disk.\n '''\n provide = cls(base_folder, sub_folder, id_file_name, folders_image, folders_masks)\n provide._load_folders();\n return provide\n \n def __init__(self,\n base_folder, \n sub_folder, \n id_file_name,\n folders_image='images',\n folders_masks='masks',\n ):\n self.base_folder = base_folder\n self.sub_folders = sub_folder\n self.folders_image = folders_image\n self.folders_masks = folders_masks\n self.id_file_name = id_file_name\n self.index = 0\n self.data = [] \n \n def __len__(self):\n return len(self.data)\n\n def __getitem__(self, i): \n #check index\n if i<0 and i>len(self.data): raise ValueError('Index outside range');\n self.index = i;\n image_pathname = self.data[i][1]; \n masks_pathname = self.data[i][2]; \n #load image\n image = np.array(self._loadimage(image_pathname), dtype=np.uint8) \n #load mask\n label = np.zeros( (image.shape[0], image.shape[1], len(masks_pathname) ) )\n for k,pathname in enumerate(masks_pathname):\n mask = np.array(self._loadimage(pathname), dtype=np.uint8)\n label[:,:,k] = mask*(k+1) \n return image, label\n \n\n def _load_folders(self):\n '''\n load file patch for disk\n ''' \n self.data = [] \n folder_path = os.path.join(self.base_folder, self.sub_folders )\n in_id_path = os.path.join(self.base_folder, self.id_file_name )\n ext = 'png' \n with open(in_id_path) as csvfile: \n id_files = csv.reader(csvfile) \n head = True\n id_current = ''\n data = {}\n for row in id_files: \n if head: head=False; continue;\n id_data = row[0]\n rl_code = [int(x) for x in row[1].split(' ')] \n if id_data != id_current: \n id_current = id_data \n images_path = os.path.join(folder_path, id_data, self.folders_image)\n masks_path = os.path.join(folder_path, id_data, self.folders_masks) \n masks_files = [ os.path.join(masks_path,f) \\\n for f in sorted(os.listdir(masks_path)) \\\n if f.split('.')[-1] == ext ];\n image_file = os.path.join(images_path, '{}.{}'.format(id_data,ext) ) \n data[id_current] = (id_current, image_file, masks_files, [], []) \n data[id_current][3].append(rl_code)\n # to array \n self.data = [ v for k,v in data.items()]\n\n\n def _loadimage(self, pathname):\n '''\n Load image using pathname\n '''\n if os.path.exists(pathname):\n try:\n image = PIL.Image.open(pathname)\n image.load()\n except IOError as e:\n raise ValueError('IOError: Trying to load \"%s\": %s' % (pathname, e.message) ) \n else:\n raise ValueError('\"%s\" not found' % pathname)\n \n return image;\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false,"collapsed":true},"cell_type":"code","source":"pathdataset = '../input'\nnamedataset = '.'\nmetadata = 'stage1_train_labels.csv'\npathname = os.path.join(pathdataset, namedataset);\npathmetadata = os.path.join(pathdataset, namedataset, metadata)\n\nbase_folder = pathname\nsub_folder = 'stage1_train'\nid_file_name = metadata\nfolders_image='images'\nfolders_masks='masks'\n\ndataloader = dsxbProvide.create(\n base_folder, \n sub_folder, \n id_file_name, \n folders_image, \n folders_masks\n )\n\nprint('Load dataset')\nprint('Total:',len(dataloader))\nprint(':)')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"070b71e6-efac-45ef-97de-7f9d7b375460","_uuid":"da965a710b03ad08172eaba2d408dbf79f18c224"},"cell_type":"markdown","source":"**VISUALIZATION**"},{"metadata":{"collapsed":true,"_uuid":"7f86ba1db4b14c87c028d85040a7593dfb2d86b4","_cell_guid":"894f6efa-8bf8-476e-b7b3-cf2427700fd5","trusted":false},"cell_type":"code","source":"\ndef torgb(im):\n if len(im.shape)==2:\n im = np.expand_dims(im, axis=2) \n im = np.concatenate( (im,im,im), axis=2 )\n return im\n \ndef setcolor(im, mask, color):\n \n tmp=im.copy()\n tmp=np.reshape( tmp, (-1, im.shape[2]) ) \n mask = np.reshape( mask, (-1,1)) \n tmp[ np.where(mask>0)[0] ,:] = color\n im=np.reshape( tmp, (im.shape) )\n return im\n\ndef lincomb(im1,im2,mask, alpha):\n \n #im = np.zeros( (im1.shape[0], im1.shape[1], 3) )\n im = im1.copy() \n \n row, col = np.where(mask>0)\n for i in range( len(row) ):\n r,c = row[i],col[i]\n #print(r,c)\n im[r,c,0] = im1[r,c,0]*(1-alpha) + im2[r,c,0]*(alpha)\n im[r,c,1] = im1[r,c,1]*(1-alpha) + im2[r,c,1]*(alpha)\n im[r,c,2] = im1[r,c,2]*(1-alpha) + im2[r,c,2]*(alpha)\n return im\n\ndef makebackgroundcell(labels):\n ch = labels.shape[2]\n cmap = plt.get_cmap('jet_r')\n imlabel = np.zeros( (labels.shape[0], labels.shape[1], 3) ) \n for i in range(ch):\n mask = labels[:,:,i]\n color = cmap(float(i)/ch)\n imlabel = setcolor(imlabel,mask,color[:3])\n return imlabel\n\ndef makeedgecell(labels):\n ch = labels.shape[2]\n cmap = plt.get_cmap('jet_r')\n imedge = np.zeros( (labels.shape[0], labels.shape[1], 3) ) \n for i in range(ch):\n mask = labels[:,:,i]\n color = cmap(float(i)/ch)\n mask = mask.astype(np.uint8)\n _,contours,_ = cv2.findContours(mask, cv2.RETR_LIST,cv2.CHAIN_APPROX_NONE )\n for cnt in contours: cv2.drawContours(imedge, cnt, -1, color[:3], 1)\n return imedge\n\ndef makeimagecell(image, labels, alphaback=0.3, alphaedge=0.3):\n \n imagecell = image.copy()\n imagecell = imagecell - np.min(imagecell)\n imagecell = imagecell / np.max(imagecell) \n imagecell = torgb(imagecell)\n \n mask = np.sum(labels, axis=2)\n imagecellbackground = makebackgroundcell(labels)\n imagecelledge = makeedgecell(labels)\n maskedge = np.sum(imagecelledge, axis=2)\n \n imagecell = lincomb(imagecell,imagecellbackground, mask, alphaback )\n imagecell = lincomb(imagecell,imagecelledge, maskedge, alphaedge )\n \n return imagecell\n \n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"04ec2f4f-34eb-4db4-a10c-5ea25aea56fa","_uuid":"f9159332b4aff975e0e1dcc966eaf5a8411cae9d","trusted":false,"collapsed":true},"cell_type":"code","source":"#k = np.random.randint( len(dataloader) )\nimage, labels = dataloader[ 538 ]\n\nplt.figure( figsize=(16,16))\nplt.subplot(311)\nimagecell = makeimagecell(image, labels, alphaback=0.3, alphaedge=0.3)\nplt.imshow( imagecell )\nplt.title('Image+Background+Edge')\nplt.axis('off')\nplt.ioff()\nplt.subplot(312)\nimagecell = makeimagecell(image, labels, alphaback=1.0, alphaedge=0.0)\nplt.imshow( imagecell )\nplt.title('Image+Background')\nplt.axis('off')\nplt.ioff()\nplt.subplot(313)\nimagecell = makeimagecell(image, labels, alphaback=0.0, alphaedge=1.0)\nplt.imshow( imagecell )\nplt.title('Image+Edge')\nplt.axis('off')\nplt.ioff()\n\nplt.show() ","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"efc73019-3430-476f-92c3-ba8a68a6951c","_uuid":"ec49fd6cd4974b5926b1a601d0282a935dcef30e","trusted":false,"collapsed":true},"cell_type":"code","source":"def display_samplas(dataloader, row=3,col=3, alphaback=0.3, alphaedge=0.2):\n \"\"\"\n Display random data from dataset\n For debug only\n \"\"\"\n fig, ax = plt.subplots(row, col, figsize=(8,8), sharex=True, sharey=True,\n subplot_kw={'adjustable': 'box-forced'}) \n for i in range(row):\n for j in range(col): \n k = np.random.randint( len(dataloader) )\n image, labels = dataloader[ k ]\n imagecell = makeimagecell(image, labels, alphaback=alphaback, alphaedge=alphaedge) \n ax[i,j].imshow(imagecell)\n ax[i,j].set_title('Image Idx: %d' % (k,))\n for a in ax.ravel():\n a.set_axis_off()\n plt.tight_layout()\n plt.show()\n\nnp.random.seed(4)\ndisplay_samplas(dataloader,row=4, col=4)","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"1411ee30e5a9bc2f6fc9aa0da9e9124c3f2273e0","_cell_guid":"57e3e8ad-3ef5-49a5-bdc5-ccb3dcfa3449","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}