{"cells":[{"metadata":{},"cell_type":"markdown","source":"Train and predict with single step iteration. For multiple iterated predictions perform single step prediction looping back to predict subsequent step. Loss (and Accuracy) need to be updated for not directed prediction, but forward iteration of prediction since this is a many-to-one problem."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\nfrom IPython.display import clear_output\nimport pandas as pd\nfrom PIL import Image\nimport time\nimport math\nfrom tqdm.auto import tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nprint(sys.version)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tf.__version__","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SIZE = 25","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Rules for updating**"},{"metadata":{"trusted":true},"cell_type":"code","source":"# from http://jakevdp.github.io/blog/2013/08/07/conways-game-of-life/\ndef life_step_1(X):\n    \"\"\"Game of life step using generator expressions\"\"\"\n    nbrs_count = sum(np.roll(np.roll(X, i, 0), j, 1)\n                     for i in (-1, 0, 1) for j in (-1, 0, 1)\n                     if (i != 0 or j != 0))\n    return (nbrs_count == 3) | (X & (nbrs_count == 2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def draw_image(img):\n    img = Image.fromarray(np.uint8(img) * 255)\n    return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_animate(arr):\n    clear_output(wait=True)\n    plt.imshow(draw_image(arr), cmap='gray')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# WORKING WITH THE DATA\n# parts based on https://www.kaggle.com/candaceng/understanding-the-problem-and-eda","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Load data**"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/conways-reverse-game-of-life-2020/train.csv')\ntest_df = pd.read_csv('/kaggle/input/conways-reverse-game-of-life-2020/test.csv')\nprint(train_df.shape)\nprint(test_df.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"See count distribution for number of iteration steps. They are fairly uniformly distributed"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.groupby(['delta']).size()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1, 1, figsize=(6,6))\nnum_bins = 50\n\n#plot final cell distr after time steps\nfor delta in range(1,6):\n    mask = train_df.loc[train_df['delta'] == delta]\n    counts = mask.iloc[:, 627:][mask.iloc[:, 627:] == 1].count(axis=1).values\n    ax.hist(counts, num_bins, density=True, label=f'{delta}', alpha=0.5)\n\nax.set_xlabel('Number of Alive Cells')\nax.set_ylabel('Probability density')\nax.legend(prop={'size': 10})    \nfig.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# SELECTING ONE SAMPLE TO VISUALIZE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_sample = train_df.sample()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_start = train_sample.loc[:, train_sample.columns.str.startswith('start')]\nsample_stop = train_sample.loc[:, train_sample.columns.str.startswith('stop')]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"start_arr = np.asarray(sample_start).reshape(25, 25)\nstop_arr = np.asarray(sample_stop).reshape(25, 25)\n# time step \ntime_step = train_sample['delta'].values[0]\nprint(time_step)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_comp_step(arr1, arr2, step):\n    fig, ax = plt.subplots(1,2, figsize=(12,12))\n    ax[0].imshow(draw_image(arr1), cmap='gray')\n    ax[0].set_title('start')\n    ax[0].axis('off')\n    ax[1].imshow(draw_image(arr2), cmap='gray')\n    ax[1].set_title(f'stop after: {step}')\n    ax[1].axis('off')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_comp_step(start_arr, stop_arr, time_step)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Display as individual frames**"},{"metadata":{"trusted":true},"cell_type":"code","source":"updated_arr = np.copy(start_arr)\nsteps = []\nsteps.append(updated_arr)\nfor x in range(time_step):\n    updated_arr = life_step_1(updated_arr)\n    steps.append(updated_arr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, m_axs = plt.subplots(1, len(steps), figsize = (10,20))\nfor c_ax, c_row in zip(m_axs.flatten(), steps):\n    c_ax.imshow(c_row, cmap='gray')\n    c_ax.axis('off')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Consider only single step for now"},{"metadata":{"trusted":true},"cell_type":"code","source":"# CREATING SINGLE STEP DATASET","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"single_step_df = train_df.loc[train_df['delta'] == 1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1, 1, figsize=(6,6))\nnum_bins = 50\n\ndelta = 1\ncounts = single_step_df.iloc[:, 2:627][single_step_df.iloc[:, 2:627] == 1].count(axis=1).values\nax.hist(counts, num_bins, density=True, label=f'start', alpha=0.5)\ncounts = single_step_df.iloc[:, 627:][single_step_df.iloc[:, 627:] == 1].count(axis=1).values\nax.hist(counts, num_bins, density=True, label=f'stop', alpha=0.5)\n\nax.set_xlabel('Number of Alive Cells')\nax.set_ylabel('Probability density')\nax.legend(prop={'size': 10})    \nfig.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, m_axs = plt.subplots(5, 2, figsize=(12,12))\nfor i, (c_ax, c_row) in enumerate(zip(m_axs.flatten(), single_step_df.sample(5).iterrows())):\n    \n    m_axs[i,0].imshow(np.asarray(c_row[1][627:]).reshape(25,25).astype('uint8'))\n    m_axs[i,0].set_title(c_row[0])\n    m_axs[i,0].axis('off')\n    \n    m_axs[i,1].imshow(np.asarray(c_row[1][2:627]).reshape(25,25).astype('uint8'))\n    m_axs[i,1].set_title(c_row[0])\n    m_axs[i,1].axis('off')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Resize to simulate periodic boundary. Check on single slice"},{"metadata":{"trusted":true},"cell_type":"code","source":"idx = 13970\nsingle_img = np.asarray(single_step_df.loc[idx][2:627]).reshape(25,25).astype('uint8')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"single_img_33 = np.pad(single_img, (3,3), 'wrap')\nsingle_img_33 = np.pad(single_img_33, (1,1), constant_values=(0,0))\n#single_img_32 = single_img_33[:-1, :-1]\nsingle_img_33.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots()\nplt.imshow(single_img_33)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"end_single_img_33 = life_step_1(single_img_33)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"single_img_33[4:-4, 4:-4].shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots()\nplt.imshow(end_single_img_33[4:-4, 4:-4])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_comp(arr1, arr2, *args, labels=['']):\n    lst=[]\n    lst.append(arr1)\n    lst.append(arr2)\n    for arg in args:\n        lst.append(arg)\n    n = len(lst)\n    if labels == ['']:\n        labels = labels * n\n    fig, ax = plt.subplots(1,n, figsize=(12,12))\n    for idx in range(n):\n        ax[idx].imshow(draw_image(lst[idx]), cmap='gray')\n        ax[idx].set_title(labels[idx])\n        ax[idx].axis('off')\n\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_comp(end_single_img_33[4:-4, 4:-4], np.asarray(single_step_df.loc[idx][627:]).reshape(25,25).astype('uint8'), labels = ['end process','end orig'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Above, we return the exact same result without periodic boundary required. "},{"metadata":{"trusted":true},"cell_type":"code","source":"def preprocess(arr): #resize to 33x33\n    arr = np.pad(arr, (3,3), 'wrap')\n    arr = np.pad(arr, (1,1), constant_values=(0,0))\n    return arr#[:-1, :-1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preprocess(single_img).shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def postprocess(arr): # returns shape (25, 25) from shape (33,33). same for tensor, check shapes though\n    return arr[4:-4, 4:-4]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"start_key = ['start_' + str(i) for i in range(625)]\nstop_key = ['stop_' + str(i) for i in range(625)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Input, Conv2D, Activation, Conv2DTranspose, \\\n    concatenate, Dropout, Lambda, MaxPooling2D, BatchNormalization, AveragePooling2D\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.losses import BinaryCrossentropy\nfrom tensorflow.data import Dataset\nfrom tensorflow.keras import backend as K","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"K.image_data_format()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# need to use tensor not numpy\n# https://stackoverflow.com/questions/49192051/converting-tensor-to-np-array-using-k-eval-in-keras-returns-invalidargumenterr","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Simulate more single step data"},{"metadata":{"trusted":true},"cell_type":"code","source":"probs = [[0.90, 0.10],\n         [0.80, 0.20],\n         [0.70, 0.30],\n         [0.60, 0.40],\n         [0.50, 0.50],\n         [0.40, 0.60],\n         [0.30, 0.70],\n         [0.20, 0.80],\n         [0.10, 0.90]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"single_step_df.shape # current amount","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_single_step_df = pd.DataFrame(columns=list(train_df.columns), dtype=np.int64)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nfor i in range(10000):\n    arr = np.random.choice([0,1], (SIZE, SIZE), p=probs[np.random.choice(len(probs))])#probs[np.random.choice(len(probs))]\n    #warm-up, based on Kaggle desccription\n    for j in range(5):\n        arr = life_step_1(arr)\n    # 1 interation\n    update_arr = life_step_1(arr)\n\n    new_row = np.concatenate((np.array([len(single_step_df) + int(i+1)]).reshape(1,-1), np.array([1]).reshape(1,-1), arr.reshape(-1, 625).round(0).astype('uint8'), update_arr.reshape(-1, 625).round(0).astype('uint8')), axis=1)\n    new_single_step_df = new_single_step_df.append(pd.DataFrame(new_row, columns=list(train_df.columns)), ignore_index=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_single_step_df = single_step_df.append(new_single_step_df, ignore_index=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_single_step_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1, 1, figsize=(6,6))\nnum_bins = 50\n\ncounts = single_step_df.iloc[:, 2:][single_step_df.iloc[:, 2:] == 1].count(axis=1).values\nax.hist(counts, num_bins, density=True, label=f'original', alpha=0.5)\ncounts = new_single_step_df.iloc[:, 2:][new_single_step_df.iloc[:, 2:] == 1].count(axis=1).values\nax.hist(counts, num_bins, density=True, label=f'new', alpha=0.5)\n\nax.set_xlabel('Number of Alive Cells')\nax.set_ylabel('Probability density')\nax.legend(prop={'size': 10})    \nfig.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Get any 0 images"},{"metadata":{"trusted":true},"cell_type":"code","source":"counts = single_step_df.iloc[:, 2:][single_step_df.iloc[:, 2:] == 1].count(axis=1).values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, m_axs = plt.subplots(5, 2, figsize=(12,12))\nfor i, (c_ax, c_row) in enumerate(zip(m_axs.flatten(), new_single_step_df.sample(5).iterrows())):\n    \n    m_axs[i,0].imshow(np.asarray(c_row[1][627:]).reshape(25,25).astype('uint8'))\n    m_axs[i,0].set_title(c_row[0])\n    m_axs[i,0].axis('off')\n    \n    m_axs[i,1].imshow(np.asarray(c_row[1][2:627]).reshape(25,25).astype('uint8'))\n    m_axs[i,1].set_title(c_row[0])\n    m_axs[i,1].axis('off')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"single_step_df = new_single_step_df.copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainY = single_step_df.loc[:, single_step_df.columns.str.startswith('start')].values.reshape(-1, 25, 25)\ntrainX = single_step_df.loc[:, single_step_df.columns.str.startswith('stop')].values.reshape(-1, 25, 25)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainX.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainX = np.array([preprocess(xi) for xi in trainX]).astype(np.float32)\ntrainX = np.expand_dims(trainX, axis=-1).astype(np.float32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainX.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainY = np.array([preprocess(xi) for xi in trainY]).astype(np.float32)\ntrainY = np.expand_dims(trainY, axis=-1).astype(np.float32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainY.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def life_step_1_tensor(X):\n    \"\"\"Game of life step using generator expressions\"\"\"\n    nbrs_count = tf.stack([tf.roll(tf.roll(X, i, 0), j, 1)\n                     for i in (-1, 0, 1) for j in (-1, 0, 1)\n                     if (i != 0 or j != 0)])\n    nbrs_count = tf.squeeze(K.sum(nbrs_count, axis=0, keepdims=True), axis=0)\n    nbrs_count = (nbrs_count == 3) | ((X == 1) & (nbrs_count == 2))\n    return tf.cast(nbrs_count, dtype=tf.uint32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def postprocess_tensor(arr): # returns shape (25, 25) from shape (31,31). same for tensor, check shapes though\n    return arr[4:-4, 4:-4,:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def wrap_pad(t, extra_dims):\n    s = tf.shape(t)\n    m = tf.constant([extra_dims[0], extra_dims[1]])\n    d = tf.constant([1, 3, 3, s.numpy()[-1]])\n    t = tf.tile(t, d)[:, s[1]-m[0]:m[0]-s[1], s[2]-m[1]:m[1]-s[2], :]\n    paddings = tf.constant([[0,0],[1, 1],[1, 1], [0,0]])\n    t = tf.pad(t, paddings, 'CONSTANT')\n    return t","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def life_step_1_tensor_sum(X):\n    \"\"\"Game of life step using generator expressions\"\"\"\n    nbrs_count = tf.stack([tf.roll(tf.roll(X, i, 0), j, 1)\n                     for i in (-1, 0, 1) for j in (-1, 0, 1)\n                     if (i != 0 or j != 0)])\n    nbrs_count = tf.squeeze(K.sum(nbrs_count, axis=0, keepdims=True), axis=0)\n    return tf.cast(nbrs_count, dtype=tf.float16)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"t = trainY[0, :, :, :]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"t1 = life_step_1_tensor(t)\nprint(t1.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_comp(postprocess_tensor(t1)[:,:,0], postprocess_tensor(trainX[0, :, :])[:,:,0], labels = ['end process','end target'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"t_sum = life_step_1_tensor_sum(t)\nfig, axs = plt.subplots()\nplt.imshow(postprocess_tensor(t_sum)[:,:,0].numpy().astype(np.uint16))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def custom_loss(y_actual,y_pred):\n    \n#     pred_fwd = y_pred\n#     actual_fwd = y_actual\n    y_actual_f = tf.cast(K.flatten(y_actual), tf.float32)\n    y_pred_f = tf.cast(K.flatten(y_pred), tf.float32)\n    bce = BinaryCrossentropy(from_logits=False)\n    bce(y_actual_f, y_pred_f)\n    #focal_tversky_2d(y_actual, y_pred)\n    return bce(y_actual_f, y_pred_f)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def dice_coef_loss(y_true, y_pred):\n    H, W, C = y_true.shape[1:]\n    smooth = 1e-5\n    pred_flat = tf.reshape(y_pred, [-1, H * W * C])\n    true_flat = tf.reshape(y_true, [-1, H * W * C])\n    intersection = 2 * tf.reduce_sum(pred_flat * true_flat, axis=1) + smooth\n    denominator = tf.reduce_sum(pred_flat, axis=1) + tf.reduce_sum(true_flat, axis=1) + smooth\n    loss = 1 - tf.reduce_mean(intersection / denominator)\n    return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def focal_tversky_2d(y_true, y_pred, alpha=0.7, gamma=0.75):\n    H, W, C = y_true.shape[1:]\n    smooth = 1e-5\n    y_pred_pos = tf.reshape(y_pred, [-1, H * W * C])\n    y_true_pos = tf.reshape(y_true, [-1, H * W * C])\n    true_pos = tf.reduce_sum(y_true_pos * y_pred_pos, axis=1)\n    false_neg = tf.reduce_sum(y_true_pos * (1 - y_pred_pos), axis=1)\n    false_pos = tf.reduce_sum((1 - y_true_pos) * y_pred_pos, axis=1)\n    tversky = (true_pos + smooth) / (true_pos + alpha * false_neg + (1 - alpha) * false_pos + smooth)\n    loss = 1 - tf.reduce_mean(tversky)\n    loss = tf.pow(loss, gamma)\n    return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def crglnet_v1():\n\n    inputShape = (33, 33, 1)\n\n    inputs = Input(inputShape)\n    \n#     inputs = Lambda(lambda x: wrap_pad(x, (3,3)))(inputs)# to change shape of input\n    \n    c1 = Conv2D(64, (3, 3), activation='elu', padding='same')(inputs)\n    \n    c2 = Conv2D(64, (3, 3), activation='elu', padding='same')(c1)\n\n    c3 = Conv2D(128, (3, 3), activation='elu', padding='same')(c2)\n\n    c4 = Conv2D(64, (3, 3), activation='elu', padding='same')(c3)\n    \n    c5 = Conv2D(64, (3, 3), activation='elu', padding='same')(c4)\n\n    c6 = Conv2D(1, (1, 1), activation='sigmoid')(c5)\n\n    model = Model(inputs=[inputs], outputs=[c6])\n    \n    return model\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def crglnet_v2():\n\n    inputShape = (33, 33, 1)\n\n    inputs = Input(inputShape)\n    \n#     inputs = Lambda(lambda x: wrap_pad(x, (3,3)))(inputs)# to change shape of input\n    \n    c1 = Conv2D(32, (3, 3), activation='elu', padding='same')(inputs)\n    \n    c2 = Conv2D(64, (3, 3), activation='elu', padding='same')(c1)\n\n    c3 = Conv2D(128, (5, 5), activation='elu', padding='same')(c2)\n    \n    c4 = concatenate([Conv2D(64, (3, 3), activation='elu', padding='same')(c3), c2])\n    \n    c5 = concatenate([Conv2D(32, (3, 3), activation='elu', padding='same')(c4), c1])\n\n    c6 = Conv2D(64, (3, 3), activation='elu', padding='same')(c5)\n    \n    c7 = Conv2D(32, (3, 3), activation='elu', padding='same')(c6)\n\n    c8 = Conv2D(1, (1, 1), activation='sigmoid')(c7)\n\n    model = Model(inputs=[inputs], outputs=[c8])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def crglnet_v3():\n\n    inputShape = (33, 33, 1)\n\n    inputs = Input(inputShape)\n    \n    c1 = Conv2D(32, (3, 3), activation='relu', padding='same')(inputs)\n    \n    c2 = Conv2D(64, (3, 3), activation='relu', padding='same')(c1)\n\n    c3 = Conv2D(128, (5, 5), activation='relu', padding='same')(c2)\n    \n    c4 = Conv2D(128, (5, 5), activation='relu', padding='same')(c3)\n    \n    c5 = concatenate([Conv2D(32, (3, 3), activation='relu', padding='same')(c4), c1])\n\n    c6 = Conv2D(64, (3, 3), activation='relu', padding='same')(c5)\n    \n    c7 = Conv2D(32, (3, 3), activation='relu', padding='same')(c6)\n\n    c8 = Conv2D(1, (1, 1), activation='sigmoid')(c7)\n\n    model = Model(inputs=[inputs], outputs=[c8])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def crglnet_v4():\n\n    image_inputShape = (33, 33, 1)\n    #feat_image_inputShape = (33, 33, 1)\n    \n    image_inputs = Input(image_inputShape)\n    feature_inputs = Lambda(lambda x: life_step_1_tensor_sum(x))(image_inputs)\n    \n    bn1 = BatchNormalization()(image_inputs)\n    ci1 = Conv2D(32, (3, 3), activation='elu', padding='same')(bn1)\n    \n    bn2 = BatchNormalization()(feature_inputs)  \n    cf1 = Conv2D(32, (3, 3), activation='elu', padding='same')(bn2)\n    \n    concat_layer= concatenate([image_inputs, feature_inputs])\n#     bn3 = BatchNormalization()(concat_layer) \n    \n#     inputs = Lambda(lambda x: wrap_pad(x, (3,3)))(inputs)# to change shape of input\n    \n    c1 = Conv2D(32, (3, 3), activation='elu', padding='same')(concat_layer)\n    \n    c2 = Conv2D(64, (3, 3), activation='elu', padding='same')(c1)\n\n    c3 = Conv2D(128, (5, 5), activation='elu', padding='same')(c2)\n    \n    c4 = concatenate([Conv2D(64, (3, 3), activation='elu', padding='same')(c3), c2])\n    \n    c5 = concatenate([Conv2D(32, (3, 3), activation='elu', padding='same')(c4), c1])\n\n    c6 = Conv2D(64, (3, 3), activation='elu', padding='same')(c5)\n    \n    c7 = Conv2D(32, (3, 3), activation='elu', padding='same')(c6)\n\n    c8 = Conv2D(1, (1, 1), activation='sigmoid')(c7)\n\n    model = Model(inputs=[image_inputs], outputs=[c8])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = crglnet_v4()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"opt = Adam(lr=1e-3)\nmodel.compile(optimizer=opt, loss=custom_loss, metrics=['accuracy', tf.keras.metrics.MeanIoU(num_classes=2)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_x, test_x, train_y, test_y = train_test_split(trainX, trainY, test_size=0.2, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#https://stackoverflow.com/questions/58947679/no-gradients-provided-for-any-variable-in-tensorflow2-0\ndef step(train_x, true_y):\n    loss = []\n    with tf.GradientTape() as tape:\n\n        # Make prediction\n        pred_y = model(train_x)\n        # Calculate loss\n        # sends a batch\n        pred_y = tf.cast(pred_y,tf.float32)\n        # This is incorrect - cannot compute gradient with binary rule. Needs probs\n#         thresh = tf.constant([0.5], dtype=tf.float32)\n#         pred_fwd = tf.cast(tf.where(pred_y>thresh,tf.constant(1),tf.constant(0)), tf.uint32)\n#         pred_fwd = tf.map_fn(life_step_1_tensor, pred_fwd, fn_output_signature=tf.uint32)\n#         true_fwd = tf.map_fn(life_step_1_tensor, true_y, fn_output_signature=tf.uint32)\n#         test_loss = custom_loss(true_fwd, pred_fwd)\n        model_loss = custom_loss(true_y, pred_y)# + test_loss\n        loss.append(model_loss.numpy())\n        \n    \n    # Calculate gradients\n    model_gradients = tape.gradient(model_loss, model.trainable_variables)\n    # Update model\n    opt.apply_gradients(zip(model_gradients, model.trainable_variables))\n    return np.mean(loss)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Training loop\nepochs = 20\nbatch_size = 32\nbat_per_epoch = math.floor(len(train_x) / batch_size)\nepoch_loss = []\nval_acc = []\nfor epoch in tqdm(range(epochs)):\n    step_loss = []\n    for i in tqdm(range(bat_per_epoch)):\n        n = i*batch_size\n        step_loss.append(step(train_x[n:n+batch_size], train_y[n:n+batch_size]))\n        \n    epoch_loss.append(np.mean(step_loss))\n    val_acc.append(model.evaluate(test_x, test_y, verbose=0)[1])\n        \n    print(f'Epoch: {epoch}, Loss: {epoch_loss[epoch]}, Val_acc: {val_acc[epoch]}')\n#model.save('model_iterative_1step_40epochs')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Calculate accuracy\nmodel.compile(optimizer=opt, loss=custom_loss, metrics=['accuracy', tf.keras.metrics.MeanIoU(num_classes=2)]) # Compile just for evaluation\nprint(model.evaluate(test_x, test_y, verbose=0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = model.predict(test_x)\npreds.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds_thresh = tf.where(preds>0.5,1,0)\npred_fwd = tf.map_fn(life_step_1_tensor, preds_thresh, fn_output_signature=tf.uint32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"actual_fwd = tf.map_fn(life_step_1_tensor, test_y, fn_output_signature=tf.uint32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"idx = 1001\nlabels = ['given end','pred start', 'end process','end target']\nplot_comp(postprocess_tensor(test_x[idx, :, :, :])[:, :, 0], postprocess_tensor(preds_thresh[idx, :, :, :])[:, :, 0], postprocess_tensor(pred_fwd[idx,:,:,:])[:,:,0], postprocess_tensor(actual_fwd[idx,:,:,:])[:,:,0], labels = labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax1 = plt.subplots()\ncolor = 'tab:red'\nax1.plot(np.arange(0, epochs), epoch_loss, label=\"custom_loss\", color=color)\nax1.set_ylabel(\"Loss\", color=color)\nax1.tick_params(axis='y', labelcolor=color)\n\nax2 = ax1.twinx()\n\ncolor = 'tab:blue'\nax2.plot(np.arange(0, epochs), val_acc, label=\"val_acc\")\nax2.set_ylabel(\"Val_Acc\", color=color)\nax2.tick_params(axis='y', labelcolor=color)\n\nplt.title(\"Training Loss\")\nplt.xlabel(\"Epoch #\")\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# MAKE PREDICTIONS ITERATIVELY"},{"metadata":{"trusted":true},"cell_type":"code","source":"start_features = [f for f in train_df.columns if \"start\" in f]\nstop_features = [f for f in train_df.columns if \"stop\" in f]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.DataFrame()\nm = tf.keras.metrics.Accuracy()\naccuracies = []\nt1 = tqdm(range(1,6), desc=f'Delta: ')\nfor delta in t1:\n    m.reset_states()\n    \n    t1.set_description(f'Delta: {delta}')\n    t1.refresh()\n    \n    test_data_iter = train_df.loc[train_df['delta'] == delta]\n    tmp_sub = test_data_iter[[\"id\"]].copy()\n    \n    testY = test_data_iter.loc[:, test_data_iter.columns.str.startswith('start')].values.reshape(-1, 25, 25)\n    testX = test_data_iter.loc[:, test_data_iter.columns.str.startswith('stop')].values.reshape(-1, 25, 25)\n    \n    testX = np.array([preprocess(xi) for xi in testX]).astype(np.float32)\n    testX = np.expand_dims(testX, axis=-1).astype(np.float32)\n    \n    testY = np.array([preprocess(xi) for xi in testY]).astype(np.float32)\n    testY = np.expand_dims(testY, axis=-1).astype(np.float32)\n    \n    t2 = tqdm(range(delta))\n    for i in t2:\n        if i == 0:\n            preds = model.predict(testX)\n        else:\n            preds = tf.where(preds>0.5,1,0)\n            preds = model.predict(preds)\n            \n    preds = tf.cast(tf.where(preds>0.5,1,0), tf.uint32)\n    m.update_state(preds, testY)\n    acc = m.result().numpy()\n    print(f'Accuracy: {acc}')\n    accuracies.append(acc) \n    \n    preds = preds[:, 4:-4, 4:-4,:].numpy()\n    tmp = pd.DataFrame(preds.reshape(-1, 625).astype(np.uint8), columns=start_features, index=tmp_sub['id'])\n    tmp_sub = tmp_sub.join(tmp)\n    sub = sub.append(tmp_sub)\nsub.sort_index(inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f'Mean accuracy: {np.array(accuracies).mean()}')\nprint(f'LB score estimate from training: {1 - np.array(accuracies).mean()}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# TEST DATA PREDICTION FOR SUBMISSION\nsub = pd.DataFrame()\nm = tf.keras.metrics.Accuracy()\naccuracies = []\nt1 = tqdm(range(1,6), desc=f'Delta: ')\nfor delta in t1:\n    m.reset_states()\n    \n    t1.set_description(f'Delta: {delta}')\n    t1.refresh()\n    \n    test_data_iter = test_df.loc[test_df['delta'] == delta]\n    tmp_sub = test_data_iter[[\"id\"]].copy()\n    tmp_sub.set_index(tmp_sub['id'].values)\n    \n    testX = test_data_iter.loc[:, test_data_iter.columns.str.startswith('stop')].values.reshape(-1, 25, 25)\n    \n    testX = np.array([preprocess(xi) for xi in testX]).astype(np.float32)\n    testX = np.expand_dims(testX, axis=-1).astype(np.float32)\n    \n    t2 = tqdm(range(delta))\n    for i in t2:\n        if i == 0:\n            preds = model.predict(testX)\n        else:\n            preds = tf.where(preds>0.5,1,0)\n            preds = model.predict(preds)\n            \n    preds = tf.cast(tf.where(preds>0.5,1,0), tf.uint32)   \n    preds = preds[:, 4:-4, 4:-4,:].numpy()\n    tmp = pd.DataFrame(preds.reshape(-1, 625).astype(np.uint8), columns=start_features, index=tmp_sub['id'].values)\n    tmp.insert(loc = 0, column='id', value=tmp_sub['id'].values)\n    sub = sub.append(tmp)\nsub.sort_index(inplace = True)\nsub.reset_index(drop = True, inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}