{"cells":[{"metadata":{"_uuid":"30c9b23447ec0ca28aae0fa7ec53d947cca76283","_cell_guid":"6cf46447-f01c-4f43-bbc6-3794f7d43981"},"cell_type":"markdown","source":"# 2018 Data Science Bowl\n\n## Objective : Find the nuclei in divergent images to advance in medical discovery"},{"metadata":{"_uuid":"1fe96d79e8e242ba23a684e0580c885c7546f974","_cell_guid":"148ea66c-9604-4113-aabe-f192dd397443"},"cell_type":"markdown","source":"### Importing necessary libraries and datasets"},{"metadata":{"_uuid":"4be8b12457163f8e51a5d227f52ae41bb2297205","_cell_guid":"570edd30-615d-4215-bad1-5e29eb79932d","trusted":true,"collapsed":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport glob\nimport cv2\nimport math\nimport seaborn as sns\nimport json\n\nsns.set()\nsns.set_palette(\"husl\")\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nTRAIN_PATH = '../input/stage1_train/'\nTEST_PATH = '../input/stage1_test/'\n\nRANDOM_SEED=75\n\nOUTPUT_PATH = './'\nCONTOUR_EXTRACT_MODE = cv2.RETR_TREE\n\n# Any results you write to the current directory are saved as output.","execution_count":1,"outputs":[]},{"metadata":{"_uuid":"e861e4365aaeee3ae906f1a63e8f45a6d5c6fb7f","_cell_guid":"c25ed36d-9100-4b8a-bd8e-a6c833b178f2","trusted":true,"collapsed":true},"cell_type":"code","source":"# Ensuring that only the folders are picked up and any junk files in the same path are ignored\n\ntrain_ids = [x for x in os.listdir(TRAIN_PATH) if os.path.isdir(TRAIN_PATH+x)]\ntest_ids = [x for x in os.listdir(TEST_PATH) if os.path.isdir(TEST_PATH+x)]","execution_count":2,"outputs":[]},{"metadata":{"_uuid":"ccaa8c0bbb510358a10746fdf265ec46446a787e","_cell_guid":"f28479d9-c298-4ece-a427-87ff48b31204","trusted":true},"cell_type":"code","source":"# Create a pandas dataframe combining all images and marking them as train or test. \n# It is a way we can do a comparison across all images.\n\ndf = pd.DataFrame({'id':train_ids,'train_or_test':'train'})\ndf = df.append(pd.DataFrame({'id':test_ids,'train_or_test':'test'}))\n\ndf.groupby(['train_or_test']).count()","execution_count":3,"outputs":[]},{"metadata":{"_uuid":"ac49cf904442a625f9af63343c1760ffc7965291","_cell_guid":"bee7d7bc-3860-4c67-9cd1-271485c0e274"},"cell_type":"markdown","source":"There are 670 training images and 65 test images, which makes the test-set less than 10% of the training set, and this sounds OK for our analysis purpose."},{"metadata":{"_uuid":"0dfbded88a8c509a22614aea94bd65687f95714c","_cell_guid":"177128d1-f26b-4cd4-bbce-4c93ff7266c4"},"cell_type":"markdown","source":"### Building the paths for the individual images"},{"metadata":{"_uuid":"b1f4c288e9127634d808f81ca74644085fbd3ae6","_cell_guid":"704d7074-6075-4e18-a9fe-c0cae5df6efb","trusted":true,"collapsed":true},"cell_type":"code","source":"df['path'] = df.apply(lambda x:'../input/stage1_{}/{}/images/{}.png'.format(x[1],x[0],x[0]), axis=1)","execution_count":4,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"f9cf3fa9838964809a524c14062ba9dc15902449","_cell_guid":"93fcdb79-71ff-4106-be4c-58c354ab882d","trusted":true},"cell_type":"code","source":"from sklearn.cluster import KMeans\n\ndef centroid_histogram(clt):\n    # grab the number of different clusters and create a histogram\n    # based on the number of pixels assigned to each cluster\n    numLabels = np.arange(0, len(np.unique(clt.labels_)) + 1)\n    (hist, _) = np.histogram(clt.labels_, bins = numLabels)\n\n    # normalize the histogram, such that it sums to one\n    hist = hist.astype(\"float\")\n    hist /= hist.sum()\n\n    # return the histogram\n    return hist","execution_count":5,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"0bc1573326ff18d4df4c3fb632a16c21dd4970ef"},"cell_type":"code","source":"def get_image_info(path, clusters=2):\n    image = cv2.imread(path)\n    height,width,_ = image.shape\n    image = cv2.cvtColor(image, cv2.COLOR_BGRA2RGB)\n    image = image.reshape((image.shape[0] * image.shape[1], 3))\n    clt = KMeans(n_clusters = clusters)\n    clt.fit(image)\n    hist = centroid_histogram(clt)\n    \n    bg_idx, fg_idx = 0, clusters-1\n    if hist[bg_idx] < hist[fg_idx]:\n        bg_idx, fg_idx = clusters-1, 0\n    \n    bg_red, bg_green, bg_blue = clt.cluster_centers_[bg_idx]\n    fg_red, fg_green, fg_blue = clt.cluster_centers_[fg_idx]\n    \n    bg_color = sum(clt.cluster_centers_[bg_idx])/3\n    fg_color = sum(clt.cluster_centers_[fg_idx])/3\n    max_color_pct = hist[bg_idx]\n    min_color_pct = hist[fg_idx]\n    \n    return (pd.Series([height,width,\n                       bg_red, bg_green, bg_blue, bg_color,\n                       fg_red, fg_green, fg_blue, fg_color,\n                       hist[bg_idx],hist[fg_idx],\n                       fg_color < bg_color]))","execution_count":6,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"30d55c75b064f42e7cf08cba5193236e40e8088f"},"cell_type":"code","source":"image_info = os.path.join(OUTPUT_PATH,'images.json')\n\nif os.path.isfile(image_info):\n    with open(image_info, 'r') as datafile:\n        data = json.load(datafile)\n        df = pd.read_json(path_or_buf=data, orient='records')\n        data = None\nelse:\n    names = ['height','width',\n             'bg_red', 'bg_green', 'bg_blue','bg_color',\n             'fg_red', 'fg_green', 'fg_blue','fg_color',\n             'bg_color_pct','fg_color_pct','invert']\n\n    df[names] = df['path'].apply(lambda x: get_image_info(x))\n    df['shape'] = df[['height','width']].apply(lambda x: '{:04d}x{:04d}'.format(x[0], x[1]), axis=1)\n\n    with open(image_info, 'w') as outfile:\n        json.dump(df.to_json(orient='records'), outfile)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"31560c4771c7cf1dab6a02c3f5f23e072123bbad"},"cell_type":"code","source":"len(df['shape'].unique()),len(df['width'].unique()), len(df['height'].unique())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"11bbe433753db4df922fdd7b5dd9aa80d47ed550"},"cell_type":"markdown","source":"# Image Distribution\n\nNow, we will have a look how the data is distributed/spread across all the images."},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"d55cb9489da5dfd0f4e779cebe0926d17b64bb62"},"cell_type":"code","source":"%matplotlib inline\nfrom matplotlib import pyplot as plt\n\nagg = df[['shape','train_or_test','id']].groupby(['shape','train_or_test']).count().unstack()\nagg.columns = agg.columns.droplevel()\n\nagg.plot.barh(stacked=True,figsize=(15,5))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"767787f7108057c357602f74fc4702527d1c4baa"},"cell_type":"code","source":"agg[agg['train'].isnull()]","execution_count":null,"outputs":[]}],"metadata":{"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}