{"cells":[{"metadata":{"_cell_guid":"c9684708-1878-4c5d-90fb-c416a121f488","_uuid":"eaa818ef76d6469cdf5be255a0f30b8bd550129b"},"cell_type":"markdown","source":"# What this kernel is about:\nThere are visualisations of diferent image types present in train & test datasets, code is mainly from https://www.kaggle.com/mpware/stage1-eda-microscope-image-types-clustering\nMain impact  of this kernel is creating a mosaic from train and test data. \nSkip to part 5 to see complited mosaics. \nUPD. In the comment section you can find csv with: original img id, cluster, big picture id to use on your own"},{"metadata":{"_cell_guid":"f66f4f0e-b132-47a9-8cdf-f399c1babac3","_uuid":"2004e029151c992242095bc461dd20d9079b8051"},"cell_type":"markdown","source":"## 1. Imports"},{"metadata":{"collapsed":true,"_uuid":"cf5797d32d566bc2e4d5629f32acdf34e7cca23b","_cell_guid":"0f9eeae6-d93d-4a46-b0e3-5ac22b586d34","trusted":true},"cell_type":"code","source":"# Import necessary modules and set global constants and variables. \n      \nimport pandas as pd                 \nimport numpy as np                                       \nfrom sklearn.cluster import KMeans\nfrom scipy.ndimage.morphology import binary_fill_holes\nimport cv2                         # To read and manipulate images\nimport os                          # For filepath, directory handling\nimport sys                         # System-specific parameters and functions\nimport tqdm                        # Use smart progress meter\nimport seaborn as sns              # For pairplots\nimport matplotlib.pyplot as plt    # Python 2D plotting library\nimport matplotlib.cm as cm         # Color map\n%matplotlib inline ","execution_count":1,"outputs":[]},{"metadata":{"_cell_guid":"2773f3ee-3624-47de-a221-27d3d35a7c86","_uuid":"f6a6bcd85997c34f019ba6e289509fe439708fd8","trusted":true},"cell_type":"code","source":"# Global constants.\nTRAIN_DIR = '../input/data-science-bowl-2018/stage1_train'\nTEST_DIR = '../input/data-science-bowl-2018/stage1_test'\nIMG_DIR_NAME = 'images'   # Folder name including the image\nMASK_DIR_NAME = 'masks'   # Folder name including the masks\n    \n\n# Display working/train/test directories.\nprint('TRAIN_DIR = {}'.format(TRAIN_DIR))\nprint('TEST_DIR = {}'.format(TEST_DIR))","execution_count":2,"outputs":[]},{"metadata":{"_cell_guid":"95ccafe1-49de-4aa2-99ae-9baed1726c82","_uuid":"2c0fe8415f14bf5d7109100d6c29bc3e4a5c05da"},"cell_type":"markdown","source":"## 2. Functions"},{"metadata":{"collapsed":true,"_uuid":"48012f9861aafd59b1e5b4d6ef94bf27463c8be2","_cell_guid":"390c1f3f-a92b-4e40-be90-930c24379c9a","trusted":true},"cell_type":"code","source":"# Collection of methods for data operations. Implemented are functions to read  \n# images/masks from files and to read basic properties of the train/test data sets.\n\ndef read_image(filepath, color_mode=cv2.IMREAD_COLOR, target_size=None,space='bgr'):\n    \"\"\"Read an image from a file and resize it.\"\"\"\n    img = cv2.imread(filepath, color_mode)\n    if target_size: \n        img = cv2.resize(img, target_size, interpolation = cv2.INTER_AREA)\n    if space == 'hsv':\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)\n    return img\n\ndef read_train_data_properties(train_dir, img_dir_name, mask_dir_name):\n    \"\"\"Read basic properties of training images and masks\"\"\"\n    tmp = []\n    for i,dir_name in enumerate(next(os.walk(train_dir))[1]):\n\n        img_dir = os.path.join(train_dir, dir_name, img_dir_name)\n        mask_dir = os.path.join(train_dir, dir_name, mask_dir_name) \n        num_masks = len(next(os.walk(mask_dir))[2])\n        img_name = next(os.walk(img_dir))[2][0]\n        img_name_id = os.path.splitext(img_name)[0]\n        img_path = os.path.join(img_dir, img_name)\n        mask_path = os.path.join(train_dir,dir_name,FULL_MASK_DIR_NAME,img_name_id+'_mask.png')\n        img_shape = read_image(img_path).shape\n        tmp.append(['{}'.format(img_name_id), img_shape[0], img_shape[1],\n                    img_shape[0]/img_shape[1], img_shape[2], num_masks,\n                    img_path, mask_dir,mask_path])\n\n    train_df = pd.DataFrame(tmp, columns = ['img_id', 'img_height', 'img_width',\n                                            'img_ratio', 'num_channels', \n                                            'num_masks', 'image_path', 'mask_dir','mask_path'])\n    return train_df\n\n\ndef read_test_data_properties(test_dir, img_dir_name):\n    \"\"\"Read basic properties of test images.\"\"\"\n    tmp = []\n    for i,dir_name in enumerate(next(os.walk(test_dir))[1]):\n\n        img_dir = os.path.join(test_dir, dir_name, img_dir_name)\n        img_name = next(os.walk(img_dir))[2][0]\n        img_name_id = os.path.splitext(img_name)[0]\n        img_path = os.path.join(img_dir, img_name)\n        img_shape = read_image(img_path).shape\n        tmp.append(['{}'.format(img_name_id), img_shape[0], img_shape[1],\n                    img_shape[0]/img_shape[1], img_shape[2], img_path])\n\n    test_df = pd.DataFrame(tmp, columns = ['img_id', 'img_height', 'img_width',\n                                           'img_ratio', 'num_channels', 'image_path'])\n    return test_df\n\ndef load_raw_data(image_size=(256, 256), space = 'bgr',load_mask=True):\n    \"\"\"Load raw data.\"\"\"\n    # Python lists to store the training images/masks and test images.\n    x_train, y_train, x_test = [],[],[]\n\n    # Read and resize train images/masks. \n    print('Loading and resizing train images and masks ...')\n    sys.stdout.flush()\n    for i, filename in tqdm.tqdm(enumerate(train_df['image_path']), total=len(train_df)):\n        img = read_image(train_df['image_path'].loc[i], target_size=image_size,space = space)\n        if load_mask:\n            mask = read_image(train_df['mask_path'].loc[i],\n                              color_mode=cv2.IMREAD_GRAYSCALE,\n                              target_size=image_size)\n            #mask = read_mask(train_df['mask_dir'].loc[i], target_size=image_size)\n            y_train.append(mask)\n        x_train.append(img)\n        \n    # Read and resize test images. \n    print('Loading and resizing test images ...')\n    sys.stdout.flush()\n    for i, filename in tqdm.tqdm(enumerate(test_df['image_path']), total=len(test_df)):\n        img = read_image(test_df['image_path'].loc[i], target_size=image_size,space=space)\n        x_test.append(img)\n\n    # Transform lists into 4-dim numpy arrays.\n    x_train = np.array(x_train)\n    #if load_mask:\n    y_train = np.array(y_train)\n    #y_train = np.expand_dims(np.array(y_train), axis=4)\n    x_test = np.array(x_test)\n    print('Data loaded')\n    if load_mask:\n        return x_train, y_train, x_test\n    else:\n        return x_train, x_test\n\ndef get_domimant_colors(img, top_colors=1):\n    \"\"\"Return dominant image color\"\"\"\n    img_l = img.reshape((img.shape[0] * img.shape[1], img.shape[2]))\n    clt = KMeans(n_clusters = top_colors)\n    clt.fit(img_l)\n    # grab the number of different clusters and create a histogram\n    # based on the number of pixels assigned to each cluster\n    numLabels = np.arange(0, len(np.unique(clt.labels_)) + 1)\n    (hist, _) = np.histogram(clt.labels_, bins = numLabels)\n    # normalize the histogram, such that it sums to one\n    hist = hist.astype(\"float\")\n    hist /= hist.sum()\n    return clt.cluster_centers_, hist\n\ndef cluster_images_by_hsv():\n    \"\"\"Clusterization based on hsv colors. Adds 'hsv_cluster' column to tables\"\"\"\n    print('Loading data')\n    x_train_hsv,x_test_hsv = load_raw_data(image_size=None,space='hsv',load_mask=False)\n    x_hsv = np.concatenate([x_train_hsv,x_test_hsv])\n    print('Calculating dominant hsv for each image')\n    dominant_hsv = []\n    for img in tqdm.tqdm(x_hsv):\n        res1, res2 = get_domimant_colors(img,top_colors=1)\n        dominant_hsv.append(res1.squeeze())\n    print('Calculating clusters')\n    kmeans = KMeans(n_clusters=3).fit(dominant_hsv)\n    train_df['HSV_CLUSTER'] = kmeans.predict(dominant_hsv[:len(x_train_hsv)])\n    test_df['HSV_CLUSTER'] = kmeans.predict(dominant_hsv[len(x_train_hsv):])\n    print('Images clustered')\n    return None\n\ndef plot_images(selected_images_df,images_rows=4,images_cols=8,plot_figsize=4):\n    \"\"\"Plot image_rows*image_cols of selected images. Used to visualy check clusterization\"\"\"\n    f, axarr = plt.subplots(images_rows,images_cols,figsize=(plot_figsize*images_cols,images_rows*plot_figsize))\n    for row in range(images_rows):\n        for col in range(images_cols):\n            if (row*images_cols + col) < selected_images_df.shape[0]:\n                image_path = selected_images_df['image_path'].iloc[row*images_cols + col]\n            else:\n                continue\n            img = read_image(image_path)\n            height, width, l = img.shape\n            ax = axarr[row,col]\n            ax.axis('off')\n            ax.set_title(\"%dx%d\"%(width, height))\n            ax.imshow(img)\n","execution_count":3,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"2bd1cca92b79eacbfceb7c32f221ca033559d51c","_cell_guid":"86a4264b-ef9d-42c3-bb1d-030b2fc36b48","trusted":true},"cell_type":"code","source":"# Basic properties of images/masks. \n# train_df = read_train_data_properties(TRAIN_DIR, IMG_DIR_NAME, MASK_DIR_NAME)\n# test_df = read_test_data_properties(TEST_DIR, IMG_DIR_NAME)\n# cluster_images_by_hsv()\n# train_df.to_csv('./train_df.csv',index=False)\n# test_df.to_csv('./test_df.csv',index=False)\n\n# We don't need to compute everything (especially clusters) every time. simly load them\ntrain_df = pd.read_csv('../input/test-train-df/train_df.csv')\ntest_df = pd.read_csv('../input/test-train-df/test_df.csv')\n\n# we need to change filepath from my filesystem to kaggle filesystem\ntrain_change_filepath = lambda x: '../input/data-science-bowl-2018/stage1_train/{0}/images/{0}.png'.format(x.split('/')[-1][:-4])\ntest_change_filepath = lambda x: '../input/data-science-bowl-2018/stage1_test/{0}/images/{0}.png'.format(x.split('/')[-1][:-4])\ntrain_df.image_path = train_df.image_path.map(train_change_filepath)\ntrain_df.drop(['mask_dir','mask_path'],inplace=True,axis = 1)\ntest_df.image_path = test_df.image_path.map(test_change_filepath)","execution_count":4,"outputs":[]},{"metadata":{"_cell_guid":"67d70624-a35b-4275-afba-b77838884ab3","_uuid":"f29eb4f99510bc6dbc8beb29b4ff7bf594173236","trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":5,"outputs":[]},{"metadata":{"_cell_guid":"cc4db64b-ad0f-4e57-a238-8214e3c0ef6e","_uuid":"10374cdde824f2013f0e177a47221d10e479efc8"},"cell_type":"markdown","source":"\n## 3. Train & Test clusters visualization"},{"metadata":{"_cell_guid":"79cdaa1b-1962-4c3d-bdd6-f9db1619e852","_uuid":"a308905dc9a02b870ce40640432e7dc858788d92"},"cell_type":"markdown","source":"#### Train data"},{"metadata":{"_cell_guid":"22aacd27-f56b-4b1a-8eb5-f11c1c00d842","_uuid":"363b67c51f3cf70ddcea3978b1f7220ab902b825","trusted":true},"cell_type":"code","source":"for idx in range(3):\n    print(\"Images in cluster {}: {}\".format(idx,train_df[train_df['HSV_CLUSTER'] == idx].shape[0]))","execution_count":6,"outputs":[]},{"metadata":{"_cell_guid":"819302cc-ea76-4f63-b053-6baac3dbf840","_uuid":"aea46e509cfb5dc59dfe17b678d3c65a49f333e3","trusted":true},"cell_type":"code","source":"plot_images(train_df[train_df['HSV_CLUSTER'] == 0],2,4)","execution_count":7,"outputs":[]},{"metadata":{"_cell_guid":"c91cee9b-f627-4bac-ba48-29a7ae8f7c07","_uuid":"c4ae77c65a13c3b1cbfa63d98b22b36664c670f3","trusted":true},"cell_type":"code","source":"plot_images(train_df[train_df['HSV_CLUSTER'] == 1],2,4)","execution_count":8,"outputs":[]},{"metadata":{"_cell_guid":"ca4d09b8-2e6c-4a79-9ed3-72c7cd935007","_uuid":"63fd8bcb7fd30a5795b62a62bbd5341d17afca14","trusted":true},"cell_type":"code","source":"plot_images(train_df[train_df['HSV_CLUSTER'] == 2],2,4)","execution_count":9,"outputs":[]},{"metadata":{"_cell_guid":"f712275e-783b-4c67-a0f4-92cfc90d2418","_uuid":"0c51621cff96a8a22795e7057f3a359f8232e8eb"},"cell_type":"markdown","source":"#### Test data"},{"metadata":{"_cell_guid":"cf20ee54-97fb-406d-a639-36c29db6d58d","_uuid":"06aed3f8ef9fa59817ab624dca501d38b8a8a6e3","trusted":true},"cell_type":"code","source":"for idx in range(3):\n    print(\"Images in cluster {}: {}\".format(idx,test_df[test_df['HSV_CLUSTER'] == idx].shape[0]))","execution_count":10,"outputs":[]},{"metadata":{"_cell_guid":"868e290d-eda4-432d-9926-f62750e4d3a5","_uuid":"f83f868d2508b71c0ca460debf34c75e6fd36d52","trusted":true},"cell_type":"code","source":"plot_images(test_df[test_df['HSV_CLUSTER'] == 0],2,4)","execution_count":11,"outputs":[]},{"metadata":{"_cell_guid":"83e6003f-73f9-49bd-836e-7a9c8effd1e2","_uuid":"067078102648556fc2dae427fd7abdeaaef9f492","trusted":true},"cell_type":"code","source":"plot_images(test_df[test_df['HSV_CLUSTER'] == 2],2,4)","execution_count":12,"outputs":[]},{"metadata":{"_cell_guid":"fbf0bb67-06ce-4bc0-8193-defb6855e5bb","_uuid":"aab50510e4183e8e6cf0b0f35ea81f0a8de0cb40","trusted":true},"cell_type":"code","source":"plot_images(test_df[test_df['HSV_CLUSTER'] == 1],2,4)","execution_count":13,"outputs":[]},{"metadata":{"_cell_guid":"b2253418-ee14-416d-8dd2-dd847244fad8","_uuid":"0cf1aaa7185a0b12d39632670a951c56e45a1378"},"cell_type":"markdown","source":"-----\n## 4. Load & Preprocess data"},{"metadata":{"_cell_guid":"a9914bc2-75fe-4d74-be50-fa27d09a7c8e","_uuid":"1ba2bf62202492f04753bd8d723e6a82c19e5674","trusted":true},"cell_type":"code","source":"# Read images/masks from files and resize them. Each image and mask \n# is stored as a 3-dim array where the number of channels is 3 and 1, respectively.\nx_train, x_test = load_raw_data(load_mask=False,image_size=None)","execution_count":14,"outputs":[]},{"metadata":{"_cell_guid":"e12f22a3-c668-4f96-940b-c616a27a66c5","_uuid":"2b63a840501e0def6f18c2ff7d5acbebcc5b3aa8","trusted":true},"cell_type":"code","source":"x_train.shape","execution_count":15,"outputs":[]},{"metadata":{"_cell_guid":"4e82641b-7a23-4e6c-b8c2-ecdefe2411e1","_uuid":"97ece476a7a17283da12b34f500979d9f5b0d67f"},"cell_type":"markdown","source":"## 5. Mosaic hypotesis\nLets try to make big images from 4 small images"},{"metadata":{"collapsed":true,"_uuid":"517d2cc9b274d38f6407c9c3e18a8613f46f5469","_cell_guid":"d9dcebaa-bc86-425e-b69f-01e019173f61","trusted":true},"cell_type":"code","source":"from sklearn.neighbors import NearestNeighbors\n# nn == Nearest Neighbors in the comments","execution_count":16,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"b28dc11d0fa963bb883c0ef2e5cd0475ff892d71","_cell_guid":"81f83c76-4f30-40e5-8471-050891b4d7bd","trusted":true},"cell_type":"code","source":"def combine_images(data,indexes):\n    \"\"\" Combines img from data using indexes as follows:\n        0 1\n        2 3 \n    \"\"\"\n    up = np.hstack([data[indexes[0]],data[indexes[1]]])\n    down = np.hstack([data[indexes[2]],data[indexes[3]]])\n    full = np.vstack([up,down])\n    return full\n\ndef make_mosaic(data,return_connectivity = False, plot_images = False,external_df = None):\n    \"\"\"Find images with simular borders and combine them to one big image\"\"\"\n    if external_df is not None:\n        external_df['mosaic_idx'] = np.nan\n        external_df['mosaic_position'] = np.nan\n        # print(external_df.head())\n    \n    # extract borders from images\n    borders = []\n    for x in data:\n        borders.extend([x[0,:,:].flatten(),x[-1,:,:].flatten(),\n                        x[:,0,:].flatten(),x[:,-1,:].flatten()])\n    borders = np.array(borders)\n\n    # prepare df with all data\n    lens = np.array([len(border) for border in borders])\n    img_idx = list(range(len(data)))*4\n    img_idx.sort()\n    position = ['up','down','left','right']*len(data)\n    nn = [None]*len(position)\n    df = pd.DataFrame(data=np.vstack([img_idx,position,borders,lens,nn]).T,\n                      columns=['img_idx','position','border','len','nn'])\n    uniq_lens = df['len'].unique()\n    \n    for idx,l in enumerate(uniq_lens):\n        # fit NN on borders of certain size with 1 neighbor\n        nn = NearestNeighbors(n_neighbors=1).fit(np.stack(df[df.len == l]['border'].values))\n        distances, neighbors = nn.kneighbors()\n        real_neighbor = np.array([None]*len(neighbors))\n        distances, neighbors = distances.flatten(),neighbors.flatten()\n\n        # if many borders are close to one, we want to take only the closest\n        uniq_neighbors = np.unique(neighbors)\n\n        # difficult to understand but works :c\n        for un_n in uniq_neighbors:\n            # min distance for borders with same nn\n            min_index = list(distances).index(distances[neighbors == un_n].min())\n            # check that min is double-sided\n            double_sided = distances[neighbors[min_index]] == distances[neighbors == un_n].min()\n            if double_sided and distances[neighbors[min_index]] < 1000:\n                real_neighbor[min_index] = neighbors[min_index]\n                real_neighbor[neighbors[min_index]] = min_index\n        indexes = df[df.len == l].index\n        for idx2,r_n in enumerate(real_neighbor):\n            if r_n is not None:\n                df['nn'].iloc[indexes[idx2]] = indexes[r_n]\n    \n    # img connectivity graph. \n    img_connectivity = {}\n    for img in df.img_idx.unique():\n        slc = df[df['img_idx'] == img]\n        img_nn = {}\n\n        # get near images_id & position\n        for nn_border,position in zip(slc[slc['nn'].notnull()]['nn'],\n                                      slc[slc['nn'].notnull()]['position']):\n\n            # filter obvious errors when we try to connect bottom of one image to bottom of another\n            # my hypotesis is that images were simply cut, without rotation\n            if position == df.iloc[nn_border]['position']:\n                continue\n            img_nn[position] = df.iloc[nn_border]['img_idx']\n        img_connectivity[img] = img_nn\n\n    imgs = []\n    indexes = set()\n    mosaic_idx = 0\n    \n    # errors in connectivity are filtered \n    good_img_connectivity = {}\n    for k,v in img_connectivity.items():\n        if v.get('down') is not None:\n            if v.get('right') is not None:\n                # need down right image\n                # check if both right and down image are connected to the same image in the down right corner\n                if (img_connectivity[v['right']].get('down') is not None) and img_connectivity[v['down']].get('right') is not None:\n                    if img_connectivity[v['right']]['down'] == img_connectivity[v['down']]['right']:\n                        v['down_right'] = img_connectivity[v['right']]['down']\n                        temp_indexes = [k,v['right'],v['down'],v['down_right']]\n                        if (len(np.unique(temp_indexes)) < 4) or (len(indexes.intersection(temp_indexes)) > 0):\n                            continue\n                        # надо тут фильтровать что они не одинаковые\n                        good_img_connectivity[k] = temp_indexes\n                        indexes.update(temp_indexes)\n                        imgs.append(combine_images(data,temp_indexes))\n                        if external_df is not None:\n                            external_df['mosaic_idx'].iloc[temp_indexes] = mosaic_idx\n                            external_df['mosaic_position'].iloc[temp_indexes] = ['up_left','up_right','down_left','down_right']\n                            mosaic_idx += 1\n                        continue\n            if v.get('left') is not None:\n                # need down left image\n                if img_connectivity[v['left']].get('down') is not None and img_connectivity[v['down']].get('left') is not None:\n                    if img_connectivity[v['left']]['down'] == img_connectivity[v['down']]['left']:\n                        v['down_left'] = img_connectivity[v['left']]['down']\n                        temp_indexes = [v['left'],k,v['down_left'],v['down']]\n                        if (len(np.unique(temp_indexes)) < 4) or (len(indexes.intersection(temp_indexes)) > 0):\n                            continue\n                        good_img_connectivity[k] = temp_indexes\n                        indexes.update(temp_indexes)\n                        imgs.append(combine_images(data,temp_indexes))\n                        \n                        if external_df is not None:\n                            external_df['mosaic_idx'].iloc[temp_indexes] = mosaic_idx\n                            external_df['mosaic_position'].iloc[temp_indexes] = ['up_left','up_right','down_left','down_right']\n                            \n                            mosaic_idx += 1 \n                        continue\n        if v.get('up') is not None:\n            if v.get('right') is not None:\n                # need up right image\n                if img_connectivity[v['right']].get('up') is not None and img_connectivity[v['up']].get('right') is not None:\n                    if img_connectivity[v['right']]['up'] == img_connectivity[v['up']]['right']:\n                        v['up_right'] = img_connectivity[v['right']]['up']\n                        temp_indexes = [v['up'],v['up_right'],k,v['right']]\n                        if (len(np.unique(temp_indexes)) < 4) or (len(indexes.intersection(temp_indexes)) > 0):\n                            continue\n                        good_img_connectivity[k] = temp_indexes\n                        indexes.update(temp_indexes)\n                        imgs.append(combine_images(data,temp_indexes))\n                        \n                        if external_df is not None:\n                            external_df['mosaic_idx'].iloc[temp_indexes] = mosaic_idx\n                            external_df['mosaic_position'].iloc[temp_indexes] = ['up_left','up_right','down_left','down_right']\n                            \n                            mosaic_idx += 1 \n                        continue\n            if v.get('left') is not None:\n                # need up left image\n                if img_connectivity[v['left']].get('up') is not None and img_connectivity[v['up']].get('left') is not None:\n                    if img_connectivity[v['left']]['up'] == img_connectivity[v['up']]['left']:\n                        v['up_left'] = img_connectivity[v['left']]['up']\n                        temp_indexes = [v['up_left'],v['up'],v['left'],k]\n                        if (len(np.unique(temp_indexes)) < 4) or (len(indexes.intersection(temp_indexes)) > 0):\n                            continue\n                        good_img_connectivity[k] = temp_indexes\n                        indexes.update(temp_indexes)\n                        imgs.append(combine_images(data,temp_indexes))\n                        \n                        if external_df is not None:\n                            external_df['mosaic_idx'].iloc[temp_indexes] = mosaic_idx\n                            external_df['mosaic_position'].iloc[temp_indexes] = ['up_left','up_right','down_left','down_right']\n                            \n                            mosaic_idx += 1 \n                        continue\n\n    # same images are present 4 times (one for every piece) so we need to filter them\n    print('Images before filtering: {}'.format(np.shape(imgs)))\n    \n    # can use np. unique only on images of one size, flatten first, then select\n    flattened = np.array([i.flatten() for i in imgs])\n    uniq_lens = np.unique([i.shape for i in flattened])\n    filtered_imgs = []\n    for un_l in uniq_lens:\n        filtered_imgs.extend(np.unique(np.array([i for i in imgs if i.flatten().shape == un_l]),axis=0))\n        \n    filtered_imgs = np.array(filtered_imgs)\n    print('Images after filtering: {}'.format(np.shape(filtered_imgs)))\n    \n    if return_connectivity:\n        print(good_img_connectivity)\n    \n    if plot_images:\n        for i in filtered_imgs:\n            plt.imshow(i)\n            plt.show()\n            \n    # list of not combined images. return if you need\n    not_combined = list(set(range(len(data))) - indexes)\n    \n    if external_df is not None:\n        #un_mos_id = external_df[external_df.mosaic_idx.notnull()].mosaic_idx.unique()\n        #mos_dict = {k:v for k,v in zip(un_mos_id,range(len(un_mos_id)))}\n        #external_df.mosaic_idx = external_df.mosaic_idx.map(mos_dict)\n        ## print(temp.mosaic_idx.shape[0])\n        ## print(len(temp.mosaic_idx[temp.mosaic_idx.isnull()] ))\n        ## print(len(list(range(temp.mosaic_idx.shape[0]-len(temp.mosaic_idx[temp.mosaic_idx.isnull()]),\n        ##                     temp.mosaic_idx.shape[0]))))\n        external_df.loc[external_df[external_df['mosaic_idx'].isnull()].index,'mosaic_idx'] = range(\n            int(np.nanmax(external_df.mosaic_idx.unique())) + 1,\n            int(np.nanmax(external_df.mosaic_idx.unique())) + 1 + len(external_df.mosaic_idx[external_df.mosaic_idx.isnull()]))\n        external_df['mosaic_idx'] = external_df['mosaic_idx'].astype(np.int32)\n        if return_connectivity:\n            return filtered_imgs, external_df, good_img_connectivity\n        else:\n            return filtered_imgs, external_df\n    if return_connectivity:\n        return filtered_imgs,good_img_connectivity\n    else:\n        return filtered_imgs","execution_count":17,"outputs":[]},{"metadata":{"_cell_guid":"8f564f65-5b51-480d-9c6a-26e46b01b49f","_uuid":"a5b5096596783231271afb01190e5529e30ec680","trusted":true},"cell_type":"code","source":"make_mosaic(x_test,return_connectivity=True,plot_images=True);","execution_count":18,"outputs":[]},{"metadata":{"_cell_guid":"ebd24711-3bce-45d1-a758-981c4aec649e","_uuid":"2b4eeb4d13c385dfd3a95c424e889054c2b61dec","trusted":true},"cell_type":"code","source":"make_mosaic(x_train,return_connectivity=False,plot_images=True);","execution_count":19,"outputs":[]},{"metadata":{"_cell_guid":"ba1d6fe7-bc88-4b1a-8988-2ff6f217a86b","_uuid":"f93a7f03b1d2df9d8603ba792fc5dbb997497f5b","trusted":true},"cell_type":"code","source":"## This is how connectivity graph look like\nmake_mosaic(x_test,return_connectivity=True,plot_images=False);","execution_count":20,"outputs":[]},{"metadata":{"_cell_guid":"e3df3678-67dd-4e84-bd61-66a5cc6ba189","_uuid":"b3a6b46ce5931049c4559f5d875614897f76b70c","trusted":true},"cell_type":"code","source":"# code which makes csv with clusters and mosaic ids for test data\nimgs, data_frame = make_mosaic(x_test,return_connectivity=False,plot_images=False,external_df=test_df);\ndata_frame[['img_id','HSV_CLUSTER','mosaic_idx','mosaic_position']].head(20)","execution_count":24,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"75f61b78bcc0fc3d916fb9d7008db717dd30e273"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"varInspector":{"window_display":false,"position":{"width":"498px","left":"869px","top":"109px","height":"174px","right":"20px"},"cols":{"lenType":16,"lenName":16,"lenVar":40},"types_to_exclude":["module","function","builtin_function_or_method","instance","_Feature"],"kernels_config":{"r":{"varRefreshCmd":"cat(var_dic_list()) ","library":"var_list.r","delete_cmd_postfix":") ","delete_cmd_prefix":"rm("},"python":{"varRefreshCmd":"print(var_dic_list())","library":"var_list.py","delete_cmd_postfix":"","delete_cmd_prefix":"del "}},"oldHeight":152,"varInspector_section_display":"block"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}