{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport glob\nfrom tqdm import tqdm\n\nimport numpy as np\nimport pandas as pd\nimport xarray as xr\n\nfrom skimage.transform import resize\nfrom skimage import io\n\nimport matplotlib.pyplot as plt\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ls ../input/galaxy-zoo-the-galaxy-challenge/44352/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_DIR = '../input/galaxy-zoo-the-galaxy-challenge/44352/'","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"df = pd.read_csv(os.path.join(DATA_DIR, 'training_solutions_rev1.csv'))\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Based on https://www.kaggle.com/helmehelmuto/keras-cnn\n# but uses float32 to save memory/space\n\nORIG_SHAPE = (424,424)\nCROP_SIZE = (256,256)\nIMG_SHAPE = (64,64)\n\ndef get_image(path, x1,y1, shape, crop_size):\n    x = plt.imread(path)\n    x = x[x1:x1+crop_size[0], y1:y1+crop_size[1]]\n    x = resize(x, shape)\n    return x\n    \ndef get_train_data(dataframe, shape=IMG_SHAPE, crop_size=CROP_SIZE):\n    x1 = (ORIG_SHAPE[0]-CROP_SIZE[0])//2\n    y1 = (ORIG_SHAPE[1]-CROP_SIZE[1])//2\n   \n    sel = dataframe.values\n    ids = sel[:,0].astype(int).astype(str)\n    y_batch = sel[:,1:].astype(np.float32)\n    x_batch = []\n    for i in tqdm(ids):\n        x = get_image(DATA_DIR + 'images_training_rev1/'+i+'.jpg', x1, y1, shape=shape, crop_size=crop_size)\n        x_batch.append(x.astype(np.float32))\n    x_batch = np.array(x_batch)\n    return x_batch, y_batch\n\n%time X_train, y_train = get_train_data(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train.shape, y_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.pcolormesh(X_train[1,:,:,0])\nplt.colorbar()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train.max()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ds = xr.Dataset({\n    'image_train': (('sample', 'x', 'y', 'channel'), X_train),\n    'label_train': (('sample', 'feature'), y_train)\n}).astype(np.float32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ds.nbytes / 1e9  # GB","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%time ds.to_netcdf('galaxy_train.nc')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ls -lh ./galaxy_train.nc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}