{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"#identify and set up directories\nprint(os.getcwd())\nwork_dir = os.getcwd()\n\nchp_id = \"cnn\"\nprint(work_dir)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd  # data frame operations  \nimport sklearn\nimport plotly\nimport plotly.graph_objs as go\nimport time\nimport numpy as np\nimport os\nimport sys\nimport re # regular expressions\nimport scipy\nimport cv2\nimport seaborn as sns  # pretty plotting, including heat map\nfrom functools import partial\nfrom sklearn.preprocessing import StandardScaler, MinMaxScaler\nfrom sklearn.model_selection import train_test_split\nfrom matplotlib.backends.backend_pdf import PdfPages\n\n\n# Python ≥3.5 is required\nassert sys.version_info >= (3, 5)\n# Scikit-Learn ≥0.20 is required\nassert sklearn.__version__ >= \"0.20\"\n\n\n# TensorFlow ≥2.0 is required\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom keras import backend as K\nassert tf.__version__ >= \"2.0\"\n\n# To plot pretty figures\n%matplotlib inline\nimport matplotlib as mpl\nimport matplotlib.pyplot as plt\nmpl.rc('axes', labelsize=14)\nmpl.rc('xtick', labelsize=12)\nmpl.rc('ytick', labelsize=12)\n\n#Set enviorment varaibles\nrandom_seed=1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!unzip ../input/dogs-vs-cats-redux-kernels-edition/train.zip -d train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!unzip ../input/dogs-vs-cats-redux-kernels-edition/test.zip -d test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"outdir = work_dir +'/cats_dogs_arrays'\nos.mkdir('/kaggle/working/cats_dogs_arrays')\nprint(outdir)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_dir_name = work_dir +'/train/train'\nprint(image_dir_name)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.mkdir('/kaggle/working/tmp')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Standard Functions\n# Sorting of file names facilitated by\ndef tryint(s):\n    try:\n        return int(s)\n    except:\n        return s\n\ndef alphanum_key(s):\n    \"\"\" Turn a string into a list of string and number chunks.\n        \"z23a\" -> [\"z\", 23, \"a\"]\n    \"\"\"\n    return [ tryint(c) for c in re.split('([0-9]+)', s) ]\n\ndef sort_nicely(l):\n    \"\"\" Sort the given list in the way that humans expect.\n    \"\"\"\n    l.sort(key=alphanum_key)\n    \n# Generate list of file names, excluding hidden files    \ndef directory_list (dir_name,str1):\n    start_list = os.listdir(dir_name)\n    end_list = []\n    for file in start_list:\n        if (not file.startswith(str1)):\n            end_list.append(file) \n    end_list.sort(key = alphanum_key)        \n    return(end_list)        \n\ncat_file_names = directory_list(image_dir_name, \"cat\")\ndog_file_names = directory_list(image_dir_name, \"dog\") ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_image_dir_name = work_dir +'/test/test'\nprint(test_image_dir_name)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#S3a Define Function to Create CNN - Soure Geron Chap14\ndef save_fig(fig_id, tight_layout=True, fig_extension=\"png\", resolution=300):\n    path = os.path.join(work_dir +'/', fig_id + \".\" + fig_extension)\n    print(\"Saving figure\", fig_id)\n    if tight_layout:\n        plt.tight_layout()\n    plt.savefig(path, format=fig_extension, dpi=resolution)\n\ndef plot_image(image):\n    plt.imshow(image, cmap=\"gray\", interpolation=\"nearest\")\n    plt.axis(\"off\")\n\ndef plot_color_image(image):\n    plt.imshow(image, interpolation=\"nearest\")\n    plt.axis(\"off\")\n\ndef feature_map_size(input_size, kernel_size, strides=1, padding=\"SAME\"):\n    if padding == \"SAME\":\n        return (input_size - 1) // strides + 1\n    else:\n        return (input_size - kernel_size) // strides + 1\n\ndef dist_plot(var1, var2, var3):\n    tmp_plt=sns.countplot(var1, palette=\"Blues\").set_title(var2)\n    tmp_fig = tmp_plt.get_figure()\n    tmp_fig.savefig(var3 + \".png\", \n        bbox_inches = 'tight', dpi=None, facecolor='w', edgecolor='b', \n        orientation='portrait', papertype=None, format=None, \n        transparent=True, pad_inches=0.25)\n    return(tmp_plt)\n\ndef pad_before_and_padded_size(input_size, kernel_size, strides=1):\n    fmap_size = feature_map_size(input_size, kernel_size, strides)\n    padded_size = max((fmap_size - 1) * strides + kernel_size, input_size)\n    pad_before = (padded_size - input_size) // 2\n    return pad_before, padded_size\n\ndef manual_same_padding(images, kernel_size, strides=1):\n    if kernel_size == 1:\n        return images.astype(np.float32)\n    batch_size, height, width, channels = images.shape\n    top_pad, padded_height = pad_before_and_padded_size(height, kernel_size, strides)\n    left_pad, padded_width  = pad_before_and_padded_size(width, kernel_size, strides)\n    padded_shape = [batch_size, padded_height, padded_width, channels]\n    padded_images = np.zeros(padded_shape, dtype=np.float32)\n    padded_images[:, top_pad:height+top_pad, left_pad:width+left_pad, :] = images\n    return padded_images\n#Tensorboard Logs\nroot_logdir = os.path.join(os.curdir, \"tf_logs\")\ndef get_run_logdir():\n    import time\n    run_id = time.strftime(\"run_%Y_%m_%d-%H_%M_%S\")\n    return os.path.join(root_logdir, run_id)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(dog_file_names))\nprint(len(cat_file_names))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#   Convert image to numpy array. 3 channels for color  and 1 converted to grayscale\n#   Info on npy binary format for saving numpy arrays https://towardsdatascience.com/\ndef parse_grayscale(image_file_path):\n    image = cv2.imread(image_file_path, cv2.IMREAD_GRAYSCALE)\n    return(image)\n    \ndef parse_color(image_file_path):\n    image = cv2.imread(image_file_path, cv2.IMREAD_COLOR)\n    # Default cv2 is BGR... need RGB\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    return(image)\n  \ndef parse_grayscale_and_resize(image_file_path, size = (64, 64)):\n    image = cv2.imread(image_file_path, cv2.IMREAD_GRAYSCALE)\n    image = cv2.resize(image, size)\n    return(image)\n\ndef parse_color_and_resize(image_file_path, size = (64, 64)):\n    image = cv2.imread(image_file_path, cv2.IMREAD_COLOR)\n    # Default cv2 is BGR... need RGB\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    image = cv2.resize(image, size)\n    return(image)  \n    \ndef show_grayscale_image(image):\n    plt.imshow(image, cmap = 'gray') \n    plt.axis('off')\n    plt.show()\n\ndef show_color_image(image):\n    plt.imshow(image) \n    plt.axis('off')\n    plt.show()   ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Examine dimensions of original raster images \nprint(len(dog_file_names))\nprint(len(cat_file_names))\ncats_shapes = []\nfor ifile in range(len(cat_file_names)):\n    image_file_path = os.path.join(image_dir_name, cat_file_names[ifile])\n    image = parse_color(image_file_path)\n    cats_shapes.append(image.shape)\n#print('\\n\\nCat image file shapes:\\n')    \n#print(cats_shapes)    \n\ndogs_shapes = []\nfor ifile in range(len(dog_file_names)):\n    image_file_path = os.path.join(image_dir_name, dog_file_names[ifile])\n    image = parse_color(image_file_path)\n    dogs_shapes.append(image.shape)    \n#print('\\n\\nDog image file shapes:\\n') \n#print(dogs_shapes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create Numpy Image Arrays\n#----------------------------------------------------------------------\nprint('\\nProcessing image files to 64x64 color or grayscale arrays')\n\n# Create cats_1000_64_64_1 and numpy array for 12500 cat images in grayscale\ncats_1000_64_64_1 = np.zeros((12500, 64, 64, 1))  \nfor ifile in range(len(cat_file_names)):\n    image_file_path = os.path.join(image_dir_name, cat_file_names[ifile])\n    image = parse_grayscale_and_resize(image_file_path, size = (64, 64))\n    cats_1000_64_64_1[ifile,:,:,0] = image\n       \n# Create dogs_1000_64_64_1 and numpy array for 12500 dog images in grayscale   \ndogs_1000_64_64_1 = np.zeros((12500, 64, 64, 1))  \nfor ifile in range(len(dog_file_names)):\n    image_file_path = os.path.join(image_dir_name, dog_file_names[ifile])\n    image = parse_grayscale_and_resize(image_file_path, size = (64, 64))\n    dogs_1000_64_64_1[ifile,:,:,0] = image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.save(os.path.join(outdir, 'cats_1000_64_64_1.npy'), cats_1000_64_64_1)\nnp.save(os.path.join(outdir, 'dogs_1000_64_64_1.npy'), dogs_1000_64_64_1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#get the final test data ready here\n#want to see if there is enough space\n\npath = test_image_dir_name\n#os.listdir(path)\n\nX_test1 = []\nid_line = []\ndef create_test1_data(path):\n    for p in os.listdir(path):\n        id_line.append(p.split(\".\")[0])\n        img_array = cv2.imread(os.path.join(path,p),cv2.IMREAD_GRAYSCALE)\n        new_img_array = cv2.resize(img_array, dsize=(64, 64))\n        X_test1.append(new_img_array)\ncreate_test1_data(path)\nX_test1 = np.array(X_test1).reshape(-1,64,64,1)\nX_test1 = X_test1/255","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.listdir(outdir) # returns list","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"RANDOM_SEED=1\n\n#Reset Graphs for Tensorboard\ndef reset_graph(seed= RANDOM_SEED):\n    tf.reset_default_graph()\n    tf.set_random_seed(seed)\n    np.random.seed(seed)\n    \n    \n#Save images to working directory\ndef save_fig(fig_id, tight_layout=True):\n    path = os.path.join(work_dir, \"images\", chp_id, fig_id + \".png\")\n    print(\"Saving figure\", fig_id)\n    if tight_layout:\n        plt.tight_layout()\n    plt.savefig(path, format='png', dpi=300)\n    \n\n#Randomly Sort Batches\ndef shuffle_batch(X, y, batch_size):\n    rnd_idx = np.random.permutation(len(X))\n    n_batches = len(X) // batch_size\n    for batch_idx in np.array_split(rnd_idx, n_batches):\n        X_batch, y_batch = X[batch_idx], y[batch_idx]\n        yield X_batch, y_batch\n        \n\nfrom matplotlib import pyplot as plt  # for display of images\ndef show_grayscale_image(image):\n    plt.imshow(image, cmap='gray')\n    plt.axis('off')\n    plt.show()\n    \n    \n#Check distribtion of test , valid and train\ndef dist_plot(var1, var2, var3):\n    tmp_plt=sns.countplot(var1, palette=\"Blues\").set_title(var2)\n    tmp_fig = tmp_plt.get_figure()\n    tmp_fig.savefig(var3 + \".png\", \n        bbox_inches = 'tight', dpi=None, facecolor='w', edgecolor='b', \n        orientation='portrait', papertype=None, format=None, \n        transparent=True, pad_inches=0.25, frameon=None)\n    return(tmp_plt)\n  \n  #Optimize memory\ndef get_model_params():\n    gvars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)\n    return {gvar.op.name: value for gvar, value in zip(gvars, tf.get_default_session().run(gvars))}\n\ndef restore_model_params(model_params):\n    gvar_names = list(model_params.keys())\n    assign_ops = {gvar_name: tf.get_default_graph().get_operation_by_name(gvar_name + \"/Assign\")\n                  for gvar_name in gvar_names}\n    init_values = {gvar_name: assign_op.inputs[1] for gvar_name, assign_op in assign_ops.items()}\n    feed_dict = {init_values[gvar_name]: model_params[gvar_name] for gvar_name in gvar_names}\n    tf.get_default_session().run(assign_ops, feed_dict=feed_dict)\n        \n#S4 Set enviorment varaibles\n\nheight = 64\nwidth = 64  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# CatsDogs  dataset # \n# Documentation on npy binary format for saving numpy arrays for later use\n#     https://towardsdatascience.com/why-you-should-start-using-npy-file-more-often-df2a13cc0161\n# Under the working directory, data files are in directory cats_dogs_64_128 \n# Read in cats and dogs grayscale 64x64 files to create training data\ncats_1000_64_64_1 = np.load(outdir+'/cats_1000_64_64_1.npy')\ndogs_1000_64_64_1 = np.load(outdir+'/dogs_1000_64_64_1.npy')\n\nprint(\"Shape of cat data: \",cats_1000_64_64_1.shape)\nprint(\"Shape of dog data: \",dogs_1000_64_64_1.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Examine first cat and first dog grayscale images\nshow_grayscale_image(cats_1000_64_64_1[0,:,:,0])\nshow_grayscale_image(dogs_1000_64_64_1[0,:,:,0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#S6 Create modeling dataset - stack cat and dog array\nX_cat_dog= np.concatenate((cats_1000_64_64_1, dogs_1000_64_64_1), axis = 0) \n#Drop last column in array will add back after scaling process\nX_cat_dog=X_cat_dog[:,:,:,-1]\nX_cat_dog.shape\n\n#Assign labels\ny_cat_dog = np.concatenate((np.zeros((12500), dtype = np.int32), \n                            np.ones((12500), dtype = np.int32)), axis = 0)\n#S7 Split Train, Validate and Test\nX_train, X_test_ds, y_train, y_test_ds= train_test_split(X_cat_dog, y_cat_dog, \n                                                         test_size=0.5, random_state= random_seed)\nX_test, X_valid, y_test, y_valid = train_test_split(X_test_ds, y_test_ds, \n                                                    test_size=0.30, random_state = random_seed)\n#S8 Scale images/numpy array\nX_mean = X_train.mean(axis=0, keepdims=True)\nX_std = X_train.std(axis=0, keepdims=True) + 1e-7\nX_train = (X_train - X_mean) / X_std\nX_valid = (X_valid - X_mean) / X_std\nX_test = (X_test - X_mean) / X_std\n\nX_train = X_train[..., np.newaxis]\nX_valid = X_valid[..., np.newaxis]\nX_test = X_test[..., np.newaxis]\n\n#Review Distribution\nprint(X_train.shape)\nprint(X_test.shape)\nprint(X_valid.shape)\nprint(y_train.shape)\nprint(y_test.shape)\nprint(y_valid.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#S9 Check distribtion of test , valid and train\ncd_plt_trn=dist_plot(y_train, 'Train', \"TrainDistCatDog\")\ncd_plt_trn.get_figure().show()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cd_plt_tst=dist_plot(y_test, 'Test', \"TestDistCatDog\")\ncd_plt_tst.get_figure().show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cd_plt_vld=dist_plot(y_valid, 'Valid', \"ValidDistCatDog\")\ncd_plt_vld.get_figure().show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Compile Model 1\nmodel = keras.models.Sequential([\n    keras.layers.Conv2D(filters=64, kernel_size=7, activation='relu', padding='SAME', input_shape=[64, 64, 1]),\n    keras.layers.MaxPooling2D(pool_size=2),\n    keras.layers.Conv2D(filters=128, kernel_size=3, activation='relu', padding='SAME'),\n    keras.layers.Conv2D(filters=128, kernel_size=3, activation='relu', padding='SAME'),\n    keras.layers.MaxPooling2D(pool_size=2),\n    keras.layers.Conv2D(filters=256, kernel_size=3, activation='relu', padding='SAME'),\n    keras.layers.Conv2D(filters=256, kernel_size=3, activation='relu', padding='SAME'),\n    keras.layers.MaxPooling2D(pool_size=2),\n    keras.layers.Flatten(),\n    keras.layers.Dense(units=128, activation='relu'),\n    keras.layers.Dropout(0.5),\n    keras.layers.Dense(units=64, activation='relu'),\n    keras.layers.Dropout(0.5),\n    #keras.layers.Dense(units=2, activation='softmax'),\n    keras.layers.Dense(1, activation='sigmoid'),\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#S11 Clear and Reset log\nkeras.backend.clear_session()\nnp.random.seed(1)\ntf.random.set_seed(1)\n#Reset Log Directory\nrun_logdir = get_run_logdir()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Execution with early Stopping Model 1\nstart_time_train = time.process_time()\ntensorboard_cb = keras.callbacks.TensorBoard(run_logdir)\ncheckpoint_cb = keras.callbacks.ModelCheckpoint(work_dir+\"/tmp/my_keras_model.h5\", save_best_only=True)\nearly_stopping_cb=keras.callbacks.EarlyStopping(monitor='val_loss', mode ='min', min_delta=1, patience = 75)\n#optimizer = keras.optimizers.Nadam(lr=1e-4, beta_1=0.9, beta_2=0.999)\noptimizer = keras.optimizers.RMSprop(lr=1e-4, rho=0.9)\nn_epochs = 100\n\nmodel.compile(loss='binary_crossentropy', optimizer =optimizer, metrics=[\"accuracy\"])\nhistory = model.fit(X_train, y_train, epochs=n_epochs, \n                    validation_data=[X_test, y_test],\n                    callbacks=[checkpoint_cb, tensorboard_cb, early_stopping_cb])\nscore = model.evaluate(X_valid, y_valid)\nX_new = X_test[:10] # pretend we have new images\ny_pred = model.predict(X_new)\nend_time_train = time.process_time()\nm1_time_train = end_time_train-start_time_train\nprint(m1_time_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Model Summary \nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# View History\nhistory.params\npd.DataFrame(history.history).plot(figsize=(8, 5))\nplt.grid(True)\nplt.gca().set_ylim(0, 1)\n#save_fig(\"keras_learning_curves_plot\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create Predicted Probabilties\ny_proba = model.predict(X_valid)\ny_proba.round(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Create Predicted Value\ny_pred = model.predict_classes(X_valid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#View actual to predicted\nprint(\"Predicted classes:\", np.reshape(y_pred[:20], (1, 20)))\nprint(\"Actual classes:   \", y_valid[:20])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Create Kaggle Submission\npredictions = model.predict(X_test1)\npredicted_val = [int(round(p[0])) for p in predictions]\nsubmission_df = pd.DataFrame({'id':id_line, 'label':predicted_val})\nsubmission_df.to_csv(\"submission1.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#define swish\ndef swish(x):\n    return K.sigmoid(x) * x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Model #2\n#same as model 1, but replace 'relu' with swish function\nmodel2 = keras.models.Sequential([\n    keras.layers.Conv2D(filters=64, kernel_size=7, activation=swish, padding='SAME', input_shape=[64, 64, 1]),\n    keras.layers.MaxPooling2D(pool_size=2),\n    keras.layers.Conv2D(filters=128, kernel_size=3, activation=swish, padding='SAME'),\n    keras.layers.Conv2D(filters=128, kernel_size=3, activation=swish, padding='SAME'),\n    keras.layers.MaxPooling2D(pool_size=2),\n    keras.layers.Conv2D(filters=256, kernel_size=3, activation=swish, padding='SAME'),\n    keras.layers.Conv2D(filters=256, kernel_size=3, activation=swish, padding='SAME'),\n    keras.layers.MaxPooling2D(pool_size=2),\n    keras.layers.Flatten(),\n    keras.layers.Dense(units=128, activation=swish),\n    keras.layers.Dropout(0.5),\n    keras.layers.Dense(units=64, activation=swish),\n    keras.layers.Dropout(0.5),\n    #keras.layers.Dense(units=2, activation='softmax'),\n    keras.layers.Dense(1, activation='sigmoid'),\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Clear and Reset log Model #2\nkeras.backend.clear_session()\nnp.random.seed(1)\ntf.random.set_seed(1)\n#Reset Log Directory\nrun_logdir = get_run_logdir()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Execution with early Stopping Model #2\nstart_time_train = time.process_time()\ntensorboard_cb = keras.callbacks.TensorBoard(run_logdir)\ncheckpoint_cb = keras.callbacks.ModelCheckpoint(work_dir+\"/tmp/my_keras_model.h5\", save_best_only=True)\nearly_stopping_cb=keras.callbacks.EarlyStopping(monitor='val_loss', mode ='min', min_delta=1, patience=75)\n#optimizer = keras.optimizers.Nadam(lr=1e-4, beta_1=0.9, beta_2=0.999)\noptimizer = keras.optimizers.RMSprop(lr=1e-4, rho=0.9)\nn_epochs = 100\n\nmodel2.compile(loss='binary_crossentropy', optimizer =optimizer, metrics=[\"accuracy\"])\nhistory = model2.fit(X_train, y_train, epochs=n_epochs, \n                    validation_data=[X_test, y_test],\n                    callbacks=[checkpoint_cb, tensorboard_cb, early_stopping_cb])\nscore2 = model2.evaluate(X_valid, y_valid)\nX_new = X_test[:10] # pretend we have new images\ny_pred_2 = model2.predict(X_new)\nend_time_train = time.process_time()\nm2_time_train = end_time_train-start_time_train\nprint(m2_time_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Model Summary \nmodel2.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# View History Model #2\nhistory.params\npd.DataFrame(history.history).plot(figsize=(8, 5))\nplt.grid(True)\nplt.gca().set_ylim(0, 1)\n#save_fig(\"keras_learning_curves_plot\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#S14 Create Predicted Probabilties Model #2\ny_proba2 = model2.predict(X_valid)\ny_proba2.round(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Create Predicted Value Model #2\ny_pred2 = model2.predict_classes(X_valid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#View actual to predicted Model #2\nprint(\"Predicted classes:\", np.reshape(y_pred2[:20], (1, 20)))\nprint(\"Actual classes:   \", y_valid[:20])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Create Kaggle Submission Model #2\npredictions = model2.predict(X_test1)\npredicted_val = [int(round(p[0])) for p in predictions]\nsubmission_df = pd.DataFrame({'id':id_line, 'label':predicted_val})\nsubmission_df.to_csv(\"submission2.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Compile Model 3 same as model 1, but with strides = 2 instead of pooling\nmodel3 = keras.models.Sequential([\n    keras.layers.Conv2D(filters=64, kernel_size=7,strides=(2, 2), activation='relu', padding='SAME', input_shape=[64, 64, 1]),\n    #keras.layers.MaxPooling2D(pool_size=2),\n    keras.layers.Conv2D(filters=128, kernel_size=3,strides=(2, 2), activation='relu', padding='SAME'),\n    keras.layers.Conv2D(filters=128, kernel_size=3,strides=(2, 2), activation='relu', padding='SAME'),\n    #keras.layers.MaxPooling2D(pool_size=2),\n    keras.layers.Conv2D(filters=256, kernel_size=3, strides=(2, 2),activation='relu', padding='SAME'),\n    keras.layers.Conv2D(filters=256, kernel_size=3,strides=(2, 2), activation='relu', padding='SAME'),\n    #keras.layers.MaxPooling2D(pool_size=2),\n    keras.layers.Flatten(),\n    keras.layers.Dense(units=128, activation='relu'),\n    keras.layers.Dropout(0.5),\n    keras.layers.Dense(units=64, activation='relu'),\n    keras.layers.Dropout(0.5),\n    #keras.layers.Dense(units=2, activation='softmax'),\n    keras.layers.Dense(1, activation='sigmoid'),\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#S11 Clear and Reset log\nkeras.backend.clear_session()\nnp.random.seed(1)\ntf.random.set_seed(1)\n#Reset Log Directory\nrun_logdir = get_run_logdir()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Execution with early Stopping Model 3\nstart_time_train = time.process_time()\ntensorboard_cb = keras.callbacks.TensorBoard(run_logdir)\ncheckpoint_cb = keras.callbacks.ModelCheckpoint(work_dir+\"/tmp/my_keras_model.h5\", save_best_only=True)\nearly_stopping_cb=keras.callbacks.EarlyStopping(monitor='val_loss', mode ='min', min_delta=1, patience=75)\n#optimizer = keras.optimizers.Nadam(lr=1e-4, beta_1=0.9, beta_2=0.999)\noptimizer = keras.optimizers.RMSprop(lr=1e-4, rho=0.9)\nn_epochs = 100\n\nmodel3.compile(loss='binary_crossentropy', optimizer =optimizer, metrics=[\"accuracy\"])\nhistory = model3.fit(X_train, y_train, epochs=n_epochs, \n                    validation_data=[X_test, y_test],\n                    callbacks=[checkpoint_cb, tensorboard_cb, early_stopping_cb])\nscore3 = model3.evaluate(X_valid, y_valid)\nX_new = X_test[:10] # pretend we have new images\ny_pred3 = model3.predict(X_new)\nend_time_train = time.process_time()\nm3_time_train = end_time_train-start_time_train\nprint(m3_time_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Model Summary \nmodel3.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# View History\nhistory.params\npd.DataFrame(history.history).plot(figsize=(8, 5))\nplt.grid(True)\nplt.gca().set_ylim(0, 1)\n#save_fig(\"keras_learning_curves_plot\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create Predicted Probabilties Model #3\ny_proba3 = model3.predict(X_valid)\ny_proba3.round(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Create Predicted Value Model #3\ny_pred3 = model3.predict_classes(X_valid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#View actual to predicted Model #3\nprint(\"Predicted classes:\", np.reshape(y_pred3[:20], (1, 20)))\nprint(\"Actual classes:   \", y_valid[:20])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Create Kaggle Submission Model #3\npredictions = model3.predict(X_test1)\npredicted_val = [int(round(p[0])) for p in predictions]\nsubmission_df = pd.DataFrame({'id':id_line, 'label':predicted_val})\nsubmission_df.to_csv(\"submission3.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Model 4 same as Model 2, swish, but with 2 strides, no pooling\nmodel4 = keras.models.Sequential([\n    keras.layers.Conv2D(filters=64, kernel_size=7, strides=(2, 2),activation=swish, padding='SAME', input_shape=[64, 64, 1]),\n    #keras.layers.MaxPooling2D(pool_size=2),\n    keras.layers.Conv2D(filters=128, kernel_size=3, strides=(2, 2),activation=swish, padding='SAME'),\n    keras.layers.Conv2D(filters=128, kernel_size=3, strides=(2, 2),activation=swish, padding='SAME'),\n    #keras.layers.MaxPooling2D(pool_size=2),\n    keras.layers.Conv2D(filters=256, kernel_size=3, strides=(2, 2),activation=swish, padding='SAME'),\n    keras.layers.Conv2D(filters=256, kernel_size=3, strides=(2, 2),activation=swish, padding='SAME'),\n    #keras.layers.MaxPooling2D(pool_size=2),\n    keras.layers.Flatten(),\n    keras.layers.Dense(units=128, activation=swish),\n    keras.layers.Dropout(0.5),\n    keras.layers.Dense(units=64, activation=swish),\n    keras.layers.Dropout(0.5),\n    #keras.layers.Dense(units=2, activation='softmax'),\n    keras.layers.Dense(1, activation='sigmoid'),\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Clear and Reset log Model #4\nkeras.backend.clear_session()\nnp.random.seed(1)\ntf.random.set_seed(1)\n#Reset Log Directory\nrun_logdir = get_run_logdir()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Execution with early Stopping Model #4\nstart_time_train = time.process_time()\ntensorboard_cb = keras.callbacks.TensorBoard(run_logdir)\ncheckpoint_cb = keras.callbacks.ModelCheckpoint(work_dir+\"/tmp/my_keras_model.h5\", save_best_only=True)\nearly_stopping_cb=keras.callbacks.EarlyStopping(monitor='val_loss', mode ='min', min_delta=1, patience=75)\n#optimizer = keras.optimizers.Nadam(lr=1e-4, beta_1=0.9, beta_2=0.999)\noptimizer = keras.optimizers.RMSprop(lr=1e-4, rho=0.9)\nn_epochs = 100\n\nmodel4.compile(loss='binary_crossentropy', optimizer =optimizer, metrics=[\"accuracy\"])\nhistory = model4.fit(X_train, y_train, epochs=n_epochs, \n                    validation_data=[X_test, y_test],\n                    callbacks=[checkpoint_cb, tensorboard_cb, early_stopping_cb])\nscore4 = model4.evaluate(X_valid, y_valid)\nX_new = X_test[:10] # pretend we have new images\ny_pred_4 = model4.predict(X_new)\nend_time_train = time.process_time()\nm4_time_train = end_time_train-start_time_train\nprint(m4_time_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Model Summary \nmodel4.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# View History Model #4\nhistory.params\npd.DataFrame(history.history).plot(figsize=(8, 5))\nplt.grid(True)\nplt.gca().set_ylim(0, 1)\n#save_fig(\"keras_learning_curves_plot\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create Predicted Probabilties Model #4\ny_proba4 = model4.predict(X_valid)\ny_proba4.round(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Create Predicted Value Model #4\ny_pred4 = model4.predict_classes(X_valid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#View actual to predicted Model #4\nprint(\"Predicted classes:\", np.reshape(y_pred4[:20], (1, 20)))\nprint(\"Actual classes:   \", y_valid[:20])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Create Kaggle Submission Model #4\npredictions = model4.predict(X_test1)\npredicted_val = [int(round(p[0])) for p in predictions]\nsubmission_df = pd.DataFrame({'id':id_line, 'label':predicted_val})\nsubmission_df.to_csv(\"submission4.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#print model train times and scores\nprint('Model 1 Time = ',m1_time_train, ', Score = ', score)\nprint('Model 2 Time = ',m2_time_train, ', Score = ', score2)\nprint('Model 3 Time = ',m3_time_train, ', Score = ', score3)\nprint('Model 4 Time = ',m4_time_train, ', Score = ', score4)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}