{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# Setting package umum \nimport pandas as pd\nimport pandas_profiling as pp\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport seaborn as sns\nfrom tqdm import tqdm_notebook as tqdm\nimport time\nimport tensorflow as tf\n%matplotlib inline\n\nfrom matplotlib.pylab import rcParams\n# For every plotting cell use this\n# grid = gridspec.GridSpec(n_row,n_col)\n# ax = plt.subplot(grid[i])\n# fig, axes = plt.subplots()\nrcParams['figure.figsize'] = [10,5]\nplt.style.use('fivethirtyeight') \nsns.set_style('whitegrid')\n\nimport warnings\nwarnings.filterwarnings('ignore')\nfrom tqdm import tqdm\n\npd.set_option('display.max_rows', 50)\npd.set_option('display.max_columns', 50)\npd.options.display.float_format = '{:.5f}'.format\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"### Try Keras\nfrom keras import Sequential\nfrom keras.layers import Dense\n\n# Define model\nmodel = Sequential([Dense(units=1, input_shape=[1])])\n\n# Compile model\nmodel.compile(optimizer='sgd', loss='mean_squared_error')\n\n# Define data\nxs = np.array([-1, 0, 1, 2, 3, 4], dtype=float)\nys = np.array([-3, -1, 1, 3, 5, 7], dtype=float)\n\n# Train model\nmodel.fit(xs, ys, epochs=500)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Predict\nmodel.predict([6])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Exercise 1 : House Pirce Question","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"### House price\nfrom keras import Sequential\nfrom keras.layers import Dense\n\n# Define model\nmodel = Sequential()\nmodel.add(Dense(units=1, input_shape=[1]))\n\n# Compile model\nmodel.compile(optimizer='sgd', loss='mean_squared_error')\n\n# Define data\nxs = np.array([0,1,2,3,4,5,6], dtype=float)\nys = np.array([0.5,1,1.5,2,2.5,3,3.5], dtype=float)\n\n# Train model\nmodel.fit(xs, ys, epochs=500)\n\n# Predict\nmodel.predict([7])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Fashion MNIST","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"### Load data\nfrom keras.datasets import fashion_mnist\n(train_img, train_lab), (test_img, test_lab) = fashion_mnist.load_data()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Plot one image\nplt.imshow(train_img[11]) ;","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Normalize data\ntrain_img = train_img / 225\ntest_img = test_img / 225","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Make keras model\nimport tensorflow as tf\nfrom keras import Sequential\nfrom keras.layers import Flatten,Dense\nimport time\n\nstart = time.time()\n\n# Set early stopping\nclass EarlyStop(tf.keras.callbacks.Callback) :\n    \n    def __init__(self, threshold) :\n        self.thres = threshold\n        \n    def on_epoch_end(self, epoch, logs={}) :\n        if (logs.get('accuracy')>self.thres) :\n            print('\\nReached',self.thres*100,'Accuracy so stop train!')\n            self.model.stop_training=True\n            \ncallbacks = EarlyStop(0.9)\n\n# Define model\nmodel = Sequential()\nmodel.add(Flatten(input_shape=(28,28)))\nmodel.add(Dense(512, activation=tf.nn.relu))\nmodel.add(Dense(512, activation=tf.nn.relu))\nmodel.add(Dense(10, activation=tf.nn.softmax))\n\n# Compile model\nmodel.compile(optimizer=tf.optimizers.Adam(),\n              loss='sparse_categorical_crossentropy',\n              metrics=['accuracy'])\n\n# Train model\nmodel.fit(train_img, train_lab, epochs=10, callbacks=[callbacks])\n\nend = time.time()\nprint('Time Used :',(end-start)/60)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Evaluate model\nmodel.evaluate(test_img, test_lab)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Predict \npred = model.predict(test_img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### See the predicted label\nnp.argmax(pred[0])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Digit MNIST","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"### Load data\nfrom keras.datasets import mnist\n(train_img, train_lab), (test_img, test_lab) = mnist.load_data()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Normalize data\ntrain_img = train_img / 225\ntest_img = test_img / 225","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### See the shape of the data\nprint(train_img.shape)\nprint(len(set(train_lab)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Make keras model\nimport tensorflow as tf\nfrom keras import Sequential\nfrom keras.layers import Flatten,Dense\nimport time\n\nstart = time.time()\n\n# Set early stopping\nclass EarlyStop(tf.keras.callbacks.Callback) :\n    \n    def __init__(self, threshold) :\n        self.thres = threshold\n        \n    def on_epoch_end(self, epoch, logs={}) :\n        if (logs.get('accuracy')>self.thres) :\n            print('\\nReached',self.thres*100,'Accuracy so stop train!')\n            self.model.stop_training=True\n            \ncallbacks = EarlyStop(0.99)\n\n# Define model\nmodel = Sequential()\nmodel.add(Flatten(input_shape=(28,28)))\nmodel.add(Dense(512, activation=tf.nn.relu))\nmodel.add(Dense(512, activation=tf.nn.relu))\nmodel.add(Dense(10, activation=tf.nn.softmax))\n\n# Compile model\nmodel.compile(optimizer=tf.optimizers.Adam(),\n              loss='sparse_categorical_crossentropy',\n              metrics=['accuracy'])\n\n# Train model\nmodel.fit(train_img, train_lab, epochs=10, callbacks=[callbacks])\n\nend = time.time()\nprint('Time Used :',(end-start)/60)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Cat vs Dog","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"### Download the data\n!wget --no-check-certificate \\\n  https://storage.googleapis.com/mledu-datasets/cats_and_dogs_filtered.zip \\\n  -O /tmp/cats_and_dogs_filtered.zip","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Unzip the data\nimport zipfile\nimport time\n\nstart = time.time()\n\ndata_zip = '/tmp/cats_and_dogs_filtered.zip'\ndata_ref = zipfile.ZipFile(data_zip, 'r')\ndata_ref.extractall()\ndata_ref.close()\nprint('Unzip Data Completed')\n\nend = time.time()\nprint('Time Used :',(end-start)/60)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Set directory path for data train and validation\nimport os\nbase_dir = '/kaggle/working/cats_and_dogs_filtered'\ntrain_dir = os.path.join(base_dir, 'train')\nval_dir = os.path.join(base_dir, 'validation')\n\n# Directory with our training cat/dog pictures\ntrain_cats_dir = os.path.join(train_dir, 'cats')\ntrain_dogs_dir = os.path.join(train_dir, 'dogs')\n\n# Directory with our validation cat/dog pictures\nval_cats_dir = os.path.join(val_dir, 'cats')\nval_dogs_dir = os.path.join(val_dir, 'dogs')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Get the name of the file for train and validation\ntrain_cat_fn = os.listdir(train_cats_dir)\ntrain_dog_fn = os.listdir(train_dogs_dir)\n\nval_cat_fn = os.listdir(val_cats_dir)\nval_dog_fn = os.listdir(val_dogs_dir)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Image example\nimport matplotlib.image as mpimg\nrcParams['figure.figsize'] = [10,5]\nplt.style.use('fivethirtyeight') \nsns.set_style('whitegrid')\ngrid = gridspec.GridSpec(1,2)\n\n# Cat image\nax = plt.subplot(grid[0])\ncat_img = mpimg.imread(os.path.join(train_cats_dir, train_cat_fn[0]))\nax.imshow(cat_img)\nax.axis('off') ;\n\n# Dog image\nax = plt.subplot(grid[1])\ndog_img = mpimg.imread(os.path.join(train_dogs_dir, train_dog_fn[0]))\nax.imshow(dog_img)\nax.axis('off') ;","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Image size varied","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"### Preprocess the image\nfrom keras.preprocessing.image import ImageDataGenerator\n\n# Define generator\ntrain_datagen = ImageDataGenerator(rescale=1/255)\nval_datagen = ImageDataGenerator(rescale=1/255)\n\n# Define flow for train gen\ntrain_gen = train_datagen.flow_from_directory(train_dir,\n                                              batch_size=20,\n                                              class_mode='binary',\n                                              target_size=(150,150))\n\n# Define flow for validation gen\nval_gen = val_datagen.flow_from_directory(val_dir,\n                                          batch_size=20,\n                                          class_mode='binary',\n                                          target_size=(150,150))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Make keras model\nimport tensorflow as tf\nimport keras\nfrom keras import Sequential\nfrom keras.layers import Conv2D, MaxPooling2D, Flatten, Dense\nimport time\n\nstart = time.time()\n\n# Set early stopping\nclass EarlyStop(tf.keras.callbacks.Callback) :\n    \n    def __init__(self, threshold) :\n        self.thres = threshold\n        \n    def on_epoch_end(self, epoch, logs={}) :\n        if (logs.get('val_accuracy')>self.thres) :\n            print('\\nReached',self.thres*100,' Validation Accuracy so stop train!')\n            self.model.stop_training=True\n            \ncallbacks = EarlyStop(0.72)\n\n# Define model\nmodel = Sequential()\nmodel.add(Conv2D(16, (3,3), activation='relu', input_shape=(150,150,3)))\nmodel.add(MaxPooling2D(2,2))\nmodel.add(Conv2D(32, (3,3), activation='relu', input_shape=(150,150,3)))\nmodel.add(MaxPooling2D(2,2))\nmodel.add(Conv2D(64, (3,3), activation='relu', input_shape=(150,150,3)))\nmodel.add(MaxPooling2D(2,2))\n\nmodel.add(Flatten())\nmodel.add(Dense(512, activation='relu'))\nmodel.add(Dense(1, activation='sigmoid'))\n\n# Compile model\nmodel.compile(optimizer=tf.keras.optimizers.RMSprop(lr=0.001),\n              loss='binary_crossentropy',\n              metrics=['accuracy'])\n\n# Train model\nhistory = model.fit_generator(train_gen, \n                              validation_data=val_gen,\n                              steps_per_epoch=100,\n                              epochs=20,\n                              validation_steps=50,\n                              verbose=1,\n                              callbacks=[callbacks])\n\nend = time.time()\nprint('Time Used :',(end-start)/60)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Try predict some image\nfrom keras.preprocessing.image import load_img, img_to_array\nfrom random import sample\nlist_cat_fn = [os.path.join(val_cats_dir, fn) for fn in sample(val_cat_fn, 5)]\nlist_dog_fn = [os.path.join(val_dogs_dir, fn) for fn in sample(val_dog_fn, 5)]\nlist_fn = list_cat_fn + list_dog_fn\n\n# Iteration to predict\nfor fn in list_fn :\n    \n    # Load and preprocess image\n    img = load_img(fn, target_size=(150,150))\n    vect_img = img_to_array(img)\n    vect_img = np.expand_dims(vect_img, axis=0)\n    ready_img = np.vstack([vect_img])\n    \n    # Predict\n    classes = model.predict(ready_img, batch_size=10)\n    if classes == 0 :\n        print(fn,'is a cat')\n    else :\n        print(fn,'is a dog')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Representation of the image based on model\nfrom keras.preprocessing.image import load_img, img_to_array\nimport keras\nfrom random import choice\nlist_cat_fn = [os.path.join(val_cats_dir, fn) for fn in sample(val_cat_fn, 5)]\nlist_dog_fn = [os.path.join(val_dogs_dir, fn) for fn in sample(val_dog_fn, 5)]\nlist_fn = list_cat_fn + list_dog_fn\nsucc_out = [layer.output for layer in model.layers[1:]]\n\n# Define viz model\nviz_model = keras.models.Model(inputs=model.input, outputs=succ_out)\n\n# Pick one image randomly\nimg = load_img(choice(list_fn), target_size=(150,150))\nx = img_to_array(img)\nx   = x.reshape((1,) + x.shape)  \nx = x / 255\n\n# Predicting\nsuccessive_feature_maps = viz_model.predict(x)\n\n# List of layer name\nlayer_names = [layer.name for layer in model.layers]\n\n# Plot the visualization on each layer\nfor layer_name, feature_map in zip(layer_names, successive_feature_maps):\n  \n  if len(feature_map.shape) == 4:\n    \n    #-------------------------------------------\n    # Just do this for the conv / maxpool layers, not the fully-connected layers\n    #-------------------------------------------\n    n_features = feature_map.shape[-1]  # number of features in the feature map\n    size       = feature_map.shape[ 1]  # feature map shape (1, size, size, n_features)\n    \n    # We will tile our images in this matrix\n    display_grid = np.zeros((size, size * n_features))\n    \n    #-------------------------------------------------\n    # Postprocess the feature to be visually palatable\n    #-------------------------------------------------\n    for i in range(n_features):\n      x  = feature_map[0, :, :, i]\n      x -= x.mean()\n      x /= x.std ()\n      x *=  64\n      x += 128\n      x  = np.clip(x, 0, 255).astype('uint8')\n      display_grid[:, i * size : (i + 1) * size] = x # Tile each filter into a horizontal grid\n\n    #-----------------\n    # Display the grid\n    #-----------------\n\n    scale = 20. / n_features\n    plt.figure( figsize=(scale * n_features, scale) )\n    plt.title ( layer_name )\n    plt.grid  ( False )\n    plt.imshow( display_grid, aspect='auto', cmap='viridis' ) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Plot the performance of the model\nrcParams['figure.figsize'] = [10,8]\nplt.style.use('fivethirtyeight') \nsns.set_style('whitegrid')\ngrid = gridspec.GridSpec(2,1)\n\n# Set the variable \nacc      = history.history[     'accuracy' ]\nval_acc  = history.history[ 'val_accuracy' ]\nloss     = history.history[    'loss' ]\nval_loss = history.history['val_loss' ]\nepo      = list(range(len(acc)))\n\n# Plot the loss\nax = plt.subplot(grid[0])\nax.plot(epo, loss, label='Train Loss')\nax.plot(epo, val_loss, label='Validation Loss')\nax.set_title('Training and Validation Loss')\nax.legend() ;\n\n# Plot the loss\nax = plt.subplot(grid[1])\nax.plot(epo, acc, label='Train Acc')\nax.plot(epo, val_acc, label='Validation Acc')\nax.set_title('Training and Validation Accuracy')\nax.legend() ;\n\nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#-----------------------------------------------------------\n# Retrieve a list of list results on training and test data\n# sets for each training epoch\n#-----------------------------------------------------------\nacc      = history.history[     'accuracy' ]\nval_acc  = history.history[ 'val_accuracy' ]\nloss     = history.history[    'loss' ]\nval_loss = history.history['val_loss' ]\n\nepochs   = range(len(acc)) # Get number of epochs\n\n#------------------------------------------------\n# Plot training and validation accuracy per epoch\n#------------------------------------------------\nplt.plot  ( epochs,     acc )\nplt.plot  ( epochs, val_acc )\nplt.title ('Training and validation accuracy')\nplt.figure()\n\n#------------------------------------------------\n# Plot training and validation loss per epoch\n#------------------------------------------------\nplt.plot  ( epochs,     loss )\nplt.plot  ( epochs, val_loss )\nplt.title ('Training and validation loss'   )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Load full data\nimport zipfile\nimport time\n\n# Unzip train data\nstart = time.time()\n\ntrain_zip = '/kaggle/input/dogs-vs-cats/train.zip'\ntrain_ref = zipfile.ZipFile(train_zip, 'r')\ntrain_ref.extractall()\ntrain_ref.close()\nprint('Unzip Train Completed')\n\nend = time.time()\nprint('Time Used :',(end-start)/60)\n\n# Unzip test data\nstart = time.time()\n\ntest_zip = '/kaggle/input/dogs-vs-cats/test1.zip'\ntest_ref = zipfile.ZipFile(test_zip, 'r')\ntest_ref.extractall()\ntest_ref.close()\nprint('Unzip Test Completed')\n\nend = time.time()\nprint('Time Used :',(end-start)/60)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Make directory for ImageGenerator\nimport os\nTRAINING_DIR = '/kaggle/training/'\nTRAINING_CAT_DIR = '/kaggle/training/cat/'\nTRAINING_DOG_DIR = '/kaggle/training/dog/'\nos.mkdir(TRAINING_DIR)\nos.mkdir(TRAINING_CAT_DIR)\nos.mkdir(TRAINING_DOG_DIR)\n\nVALIDATION_DIR = '/kaggle/validation/'\nVALIDATION_CAT_DIR = '/kaggle/validation/cat/'\nVALIDATION_DOG_DIR = '/kaggle/validation/dog/'\nos.mkdir(VALIDATION_DIR)\nos.mkdir(VALIDATION_CAT_DIR)\nos.mkdir(VALIDATION_DOG_DIR)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### List file name in train dataset\nSOURCE_DIR = '/kaggle/working/train/'\nlist_fn = os.listdir(SOURCE_DIR)\nlist_cat_fn = [fn for fn in list_fn if 'cat' in fn]\nlist_dog_fn = [fn for fn in list_fn if 'dog' in fn]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Split data\nimport random\nfrom shutil import copyfile\nSPLIT_PROP = 0.8\nlist_class = ['cat','dog']\nlist_name_fn = [list_cat_fn, list_dog_fn]\n\nfor i,c in enumerate(tqdm(list_class)) :\n            \n    # Splitting\n    list_class_fn = list_name_fn[i]\n    pure_random = random.sample(list_class_fn, len(list_class_fn))\n    random_train = pure_random[:int(SPLIT_PROP*len(pure_random))]\n    random_val = pure_random[int(SPLIT_PROP*len(pure_random)):]\n    \n    # Insert into new train dir\n    TRAIN_CLASS_DIR = os.path.join(TRAINING_DIR,c)\n    for f in random_train :\n        copyfile(os.path.join(SOURCE_DIR,f), os.path.join(TRAIN_CLASS_DIR,f))\n        \n    # Insert into new valid dir\n    VALID_CLASS_DIR = os.path.join(VALIDATION_DIR,c)\n    for f in random_val :\n        copyfile(os.path.join(SOURCE_DIR,f), os.path.join(VALID_CLASS_DIR,f))\n        \n    del pure_random, random_train, random_val","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Make pre train model\nfrom tensorflow.keras.applications.resnet50 import ResNet50\nfrom tensorflow.keras.applications.resnet50 import preprocess_input,decode_predictions\n\nHEIGHT = 150\nWIDTH = 150\nbase_model = ResNet50(weights='imagenet', \n                 include_top=False, \n                 input_shape=(HEIGHT, WIDTH, 3))\n\nprec_input = tf.keras.applications.resnet50.preprocess_input\ndecode = tf.keras.applications.resnet50.decode_predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Define parameter\nBATCH_SIZE=10\nEPOCHS = 10","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Preprocess the image\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\n\n# Define generator\ntrain_datagen = ImageDataGenerator(rescale=1/255, preprocessing_function=prec_input)\nval_datagen = ImageDataGenerator(rescale=1/255, preprocessing_function=prec_input)\n\n# Define flow for train gen\ntrain_gen = train_datagen.flow_from_directory(TRAINING_DIR,\n                                              batch_size=BATCH_SIZE,\n                                              interpolation='bicubic',\n                                              class_mode='categorical',\n                                              shuffle=True,\n                                              target_size=(HEIGHT, WIDTH))\n\n# Define flow for validation gen\nval_gen = val_datagen.flow_from_directory(VALIDATION_DIR,\n                                          batch_size=BATCH_SIZE,\n                                          interpolation='bicubic',\n                                          class_mode='categorical',\n                                          shuffle=False,\n                                          target_size=(HEIGHT, WIDTH))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Add layer and dont train the layer before\nfrom tensorflow.keras.layers import Dense, Activation, Flatten, Dropout, GlobalAveragePooling2D\nfrom tensorflow.keras import Sequential, Model\nfrom tensorflow.keras.optimizers import SGD, Adam, RMSprop\nfrom tensorflow.keras.metrics import Precision\nfrom tensorflow.keras.activations import swish\nfc_layers = []\nnum_classes = 2\ndropout = 0.2\n\ndef make_model() :\n    \n    # Freeze layer\n    for l in base_model.layers :\n        l.trainable = False\n\n    # Make new model\n    model = Sequential()\n    model.add(base_model)\n    model.add(GlobalAveragePooling2D())\n    model.add(Flatten())\n    for fc in fc_layers:\n\n        # New FC layer, random init\n        model.add(Dense(fc, activation=swish))\n        model.add(Dropout(dropout))\n\n    model.add(Dense(num_classes, activation='softmax'))\n\n    # Compile model\n    model.compile(Adam(), loss='categorical_crossentropy', metrics=['acc'])\n    \n    return model\n\n\nmodel = make_model()\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit_generator(train_gen,\n                              epochs=EPOCHS,\n                              verbose=1,\n                              validation_data=val_gen)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# DEFINE A KERAS MODEL TO CLASSIFY CATS V DOGS\n# USE AT LEAST 3 CONVOLUTION LAYERS\nmodel = tf.keras.models.Sequential([\n    tf.keras.layers.Conv2D(16, (3,3), activation='relu', input_shape=(150, 150, 3)),\n    tf.keras.layers.MaxPooling2D(2,2),\n    tf.keras.layers.Conv2D(32, (3,3), activation='relu'),\n    tf.keras.layers.MaxPooling2D(2,2), \n    tf.keras.layers.Conv2D(64, (3,3), activation='relu'), \n    tf.keras.layers.MaxPooling2D(2,2),\n    tf.keras.layers.Flatten(), \n    tf.keras.layers.Dense(512, activation='relu'), \n    tf.keras.layers.Dense(2, activation='softmax')  \n\n# YOUR CODE HERE\n])\n\nmodel.compile(optimizer=RMSprop(lr=0.001), loss='categorical_crossentropy', metrics=['acc'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit_generator(train_gen,\n                              epochs=EPOCHS,\n                              verbose=1,\n                              validation_data=val_gen)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Human vs Horse","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"### Get data\n!wget --no-check-certificate https://storage.googleapis.com/laurencemoroney-blog.appspot.com/horse-or-human.zip -O /tmp/horse-or-human.zip\n!wget --no-check-certificate https://storage.googleapis.com/laurencemoroney-blog.appspot.com/validation-horse-or-human.zip -O /tmp/validation-horse-or-human.zip ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Extract the zip file\nimport zipfile\nimport time\n\n# Unzip train data\ndata_zip = '/tmp/horse-or-human.zip'\ndata_ref = zipfile.ZipFile(data_zip, 'r')\ndata_ref.extractall('/training')\ndata_ref.close()\nprint('Unzip Train Data Completed')\n\n# Unzip valid data\nvalid_zip = '/tmp/validation-horse-or-human.zip'\nvalid_ref = zipfile.ZipFile(valid_zip, 'r')\nvalid_ref.extractall('/validating')\nvalid_ref.close()\nprint('Unzip Valid Data Completed')\n\n%time","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Check the proportion of classes in train and valid dataset\nTRAIN_HORSE_DIR = '/training/horses'\nTRAIN_HUMAN_DIR = '/training/humans'\nVAL_HORSE_DIR = '/validating/horses'\nVAL_HUMAN_DIR = '/validating/humans'\n\nprint('Total obs on horse class in train :',len(os.listdir(TRAIN_HORSE_DIR)))\nprint('Total obs on human class in train :',len(os.listdir(TRAIN_HUMAN_DIR)))\nprint('Total obs on horse class in validation :',len(os.listdir(VAL_HORSE_DIR)))\nprint('Total obs on human class in validation :',len(os.listdir(VAL_HUMAN_DIR)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Define pretrained model\nfrom tensorflow.keras.applications.inception_v3 import InceptionV3\nfrom tensorflow.keras.applications.inception_v3 import preprocess_input,decode_predictions\nBATCH_SIZE = 20\nINPUT_SIZE = (150,150)\n\n# Get local weight\n!wget --no-check-certificate \\\n    https://storage.googleapis.com/mledu-datasets/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5 \\\n    -O /tmp/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5\n    \n# Define inception model\nbase_model = InceptionV3(weights=None, \n                         include_top=False, \n                         input_shape=(INPUT_SIZE[0], INPUT_SIZE[1], 3))\n\n# Load local weight\nlocal_weight_file = '/tmp/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5'\nbase_model.load_weights(local_weight_file)\n\n# Freeze all layer\nbase_model.trainable = False\n\n# Summary of the model\nbase_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Get specific layer from pre-trained model \nlast_layer = base_model.get_layer('mixed7')\nlast_output = last_layer.output\n\nprint('Last layer shape :',last_layer.output_shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Set the image generator\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nTRAIN_DIR = '/training'\nVALID_DIR = '/validating'\n\n# Make image generator for training dataset\ntrain_datagen = ImageDataGenerator(rescale = 1./255.,\n                                   rotation_range = 40,\n                                   width_shift_range = 0.2,\n                                   height_shift_range = 0.2,\n                                   shear_range = 0.2,\n                                   zoom_range = 0.2,\n                                   horizontal_flip = True)\n\ntrain_gen = train_datagen.flow_from_directory(TRAIN_DIR,\n                                              batch_size = BATCH_SIZE,\n                                              class_mode = 'binary', \n                                              target_size = INPUT_SIZE)  \n\n# Make image generator for validation dataset\nval_datagen = ImageDataGenerator(rescale = 1/255. )\n\nval_gen =  val_datagen.flow_from_directory(VALID_DIR,\n                                            batch_size  = BATCH_SIZE,\n                                            class_mode  = 'binary', \n                                            target_size = INPUT_SIZE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Define new model\nfrom tensorflow.keras.layers import Dense, Activation, Flatten, Dropout, GlobalAveragePooling2D\nfrom tensorflow.keras import Sequential, Model\nfrom tensorflow.keras.optimizers import SGD, Adam, RMSprop\nfrom tensorflow.keras.metrics import Precision\nfrom tensorflow.keras.activations import swish\\\n\n# model = Sequential()\n# model.add(base_model)\n# model.add(Flatten())\n# model.add(Dense(1024, activation='relu'))\n# model.add(Dropout(0.2))\n# model.add(Dense(1, activation='sigmoid'))\n\n# Use this when you not using the whole pre trained model\nx = Flatten()(last_output)\nx = Dense(1024, activation='relu')(x)\nx = Dropout(0.2)(x)\nx = Dense(1, activation='sigmoid')(x)\nmodel = Model(base_model.input, x)\n\n# Compile model\nmodel.compile(optimizer = RMSprop(lr=0.0001),\n              loss = 'binary_crossentropy',\n              metrics = 'accuracy')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Define early callback based on metrics\nclass EarlyStop(tf.keras.callbacks.Callback) :\n    \n    def __init__(self, threshold) :\n        self.thres = threshold\n        \n    def on_epoch_end(self, epoch, logs={}) :\n        if (logs.get('accuracy')>self.thres) :\n            print('\\nReached',self.thres*100,'Accuracy so stop train!')\n            self.model.stop_training=True\n            \ncallbacks = EarlyStop(0.99)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Train model\nEPOCHS = 99\nhistory = model.fit_generator(train_gen,\n                              validation_data=val_gen,\n                              steps_per_epoch=100,\n                              validation_steps=50,\n                              epochs=EPOCHS,\n                              callbacks=callbacks,\n                              verbose=1)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Rock Paper Scissors","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"### Function to read the data\ndef get_data(path) :\n    # Read the csv\n    df = pd.read_csv(path)\n    \n    # Get all the label\n    from tensorflow.keras.utils import to_categorical\n    label = to_categorical(np.array(df['label']))\n    \n    # Get array of image\n    image = []\n    for i in range(len(df)) :\n        split_img = np.array(np.split(df.iloc[i,1:].values, 28))\n        image.append(split_img)\n    \n    return np.array(image), label\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Get data\ntrain_img, train_label = get_data('../input/sign-language-mnist/sign_mnist_train.csv')\nval_img, val_label = get_data('../input/sign-language-mnist/sign_mnist_test.csv')\n\n%time","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Shape of the data\nprint('Train image shape :',train_img.shape)\nprint('Train label shape :',train_label.shape)\nprint('Valid image shape :',val_img.shape)\nprint('Valid label shape :',val_label.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Make the image generator\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nBATCH_SIZE = 32\nINPUT_SIZE = (28,28)\nTRAIN_DIR = '/training'\nVALID_DIR = '/validating'\n\n# Make image generator for training dataset\ntrain_img = np.expand_dims(train_img, 3)\ntrain_datagen = ImageDataGenerator(rescale = 1./255.,\n                                   rotation_range = 40,\n                                   width_shift_range = 0.2,\n                                   height_shift_range = 0.2,\n                                   shear_range = 0.2,\n                                   zoom_range = 0.2,\n                                   horizontal_flip = True)\n\ntrain_gen = train_datagen.flow(x=train_img, y=train_label,\n                              batch_size = BATCH_SIZE)  \n\n# Make image generator for validation dataset\nval_img = np.expand_dims(val_img, 3)\nval_datagen = ImageDataGenerator(rescale = 1/255. )\n\nval_gen = val_datagen.flow(x=val_img, y=val_label,\n                              batch_size = BATCH_SIZE) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Define new model\nfrom tensorflow.keras.layers import Dense, Activation, Flatten, Dropout, MaxPooling2D, Conv2D\nfrom tensorflow.keras import Sequential, Model\nfrom tensorflow.keras.optimizers import SGD, Adam, RMSprop\nfrom tensorflow.keras.metrics import Precision\nfrom tensorflow.keras.activations import swish\\\n\nmodel = Sequential()\nmodel.add(Conv2D(64, (3,3), activation='relu', input_shape=(INPUT_SIZE[0], INPUT_SIZE[1], 1)))\nmodel.add(MaxPooling2D(2,2))\nmodel.add(Conv2D(128, (3,3), activation='relu'))\nmodel.add(MaxPooling2D(2,2))\nmodel.add(Flatten())\nmodel.add(Dense(256, activation='relu'))\nmodel.add(Dropout(0.2))\nmodel.add(Dense(25, activation='softmax'))\n\n# Compile model\nmodel.compile(optimizer = Adam(),\n              loss = 'categorical_crossentropy',\n              metrics = 'accuracy')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Define early callback based on metrics\nclass EarlyStop(tf.keras.callbacks.Callback) :\n    \n    def __init__(self, threshold) :\n        self.thres = threshold\n        \n    def on_epoch_end(self, epoch, logs={}) :\n        if (logs.get('val_accuracy')>self.thres) :\n            print('\\nReached',self.thres*100,'Accuracy so stop train!')\n            self.model.stop_training=True\n            \ncallbacks = EarlyStop(0.9)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Train model\nEPOCHS = 50\nhistory = model.fit_generator(train_gen,\n                              validation_data=val_gen,\n                              steps_per_epoch=train_img.shape[0] // BATCH_SIZE,\n                              validation_steps=val_img.shape[0] // BATCH_SIZE,\n                              epochs=EPOCHS,\n                              callbacks=callbacks,\n                              verbose=1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Evaluate model\nmodel.evaluate(val_img, val_label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Plot the performance of the model\nrcParams['figure.figsize'] = [10,8]\nplt.style.use('fivethirtyeight') \nsns.set_style('whitegrid')\ngrid = gridspec.GridSpec(2,1)\n\n# Get the metrics and loss\nacc      = history.history[     'accuracy' ]\nval_acc  = history.history[ 'val_accuracy' ]\nloss     = history.history[    'loss' ]\nval_loss = history.history['val_loss' ]\nepo   = range(len(acc)) # Get number of epochs\n\n# Plot the loss\nax = plt.subplot(grid[0])\nax.plot(epo, loss, label='Train Loss')\nax.plot(epo, val_loss, label='Validation Loss')\nax.set_title('Training and Validation Loss')\nax.legend() ;\n\n# Plot the acccuracy\nax = plt.subplot(grid[1])\nax.plot(epo, acc, label='Train Acc')\nax.plot(epo, val_acc, label='Validation Acc')\nax.set_title('Training and Validation Accuracy')\nax.legend() ;","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# BBC News","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"### Get data\n!wget --no-check-certificate \\\n    https://storage.googleapis.com/laurencemoroney-blog.appspot.com/bbc-text.csv \\\n    -O /tmp/bbc-text.csv\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Load data\ndata = pd.read_csv('/tmp/bbc-text.csv')\nlabel = list(data['category'])\nsentences_raw = list(data['text'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Print the first expected output\nprint(len(sentences_raw))\nprint(sentences_raw[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Remove stopwords from data\nfrom nltk.tokenize import word_tokenize\nstopwords = [ \"a\", \"about\", \"above\", \"after\", \"again\", \"against\", \"all\", \"am\", \"an\", \"and\", \"any\", \"are\", \"as\", \"at\", \"be\", \"because\", \"been\", \"before\", \"being\", \"below\", \"between\", \"both\", \"but\", \"by\", \"could\", \"did\", \"do\", \"does\", \"doing\", \"down\", \"during\", \"each\", \"few\", \"for\", \"from\", \"further\", \"had\", \"has\", \"have\", \"having\", \"he\", \"he'd\", \"he'll\", \"he's\", \"her\", \"here\", \"here's\", \"hers\", \"herself\", \"him\", \"himself\", \"his\", \"how\", \"how's\", \"i\", \"i'd\", \"i'll\", \"i'm\", \"i've\", \"if\", \"in\", \"into\", \"is\", \"it\", \"it's\", \"its\", \"itself\", \"let's\", \"me\", \"more\", \"most\", \"my\", \"myself\", \"nor\", \"of\", \"on\", \"once\", \"only\", \"or\", \"other\", \"ought\", \"our\", \"ours\", \"ourselves\", \"out\", \"over\", \"own\", \"same\", \"she\", \"she'd\", \"she'll\", \"she's\", \"should\", \"so\", \"some\", \"such\", \"than\", \"that\", \"that's\", \"the\", \"their\", \"theirs\", \"them\", \"themselves\", \"then\", \"there\", \"there's\", \"these\", \"they\", \"they'd\", \"they'll\", \"they're\", \"they've\", \"this\", \"those\", \"through\", \"to\", \"too\", \"under\", \"until\", \"up\", \"very\", \"was\", \"we\", \"we'd\", \"we'll\", \"we're\", \"we've\", \"were\", \"what\", \"what's\", \"when\", \"when's\", \"where\", \"where's\", \"which\", \"while\", \"who\", \"who's\", \"whom\", \"why\", \"why's\", \"with\", \"would\", \"you\", \"you'd\", \"you'll\", \"you're\", \"you've\", \"your\", \"yours\", \"yourself\", \"yourselves\" ]\nsentences = []\n\nfor t in tqdm(sentences_raw) :\n    tokenize_text = word_tokenize(t)\n    list_text = [i for i in tokenize_text if i not in stopwords]\n    sentences.append((\" \").join(list_text))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Tokenize the text data\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\ntokenizer = Tokenizer(oov_token='<OOV>',\n                      #num_words to specify maximum number of token based on frequency\n                     )\n\n# Get the token dict from data\ntokenizer.fit_on_texts(sentences)\nword_index = tokenizer.word_index\nprint(len(word_index))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Pad the text data\nsequences = tokenizer.texts_to_sequences(sentences)\npadded = pad_sequences(sequences, padding='post', truncating='pre', maxlen=None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Print the second output\nprint(padded[0])\nprint(padded.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Tokenize the label data\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\ntokenizer_label = Tokenizer()\n\n# Get the token dict from data\ntokenizer_label.fit_on_texts(label)\nlabel_word_index = tokenizer_label.word_index\nlabel_seq = tokenizer_label.texts_to_sequences(label)\n\nprint(label_seq)\nprint(label_word_index)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# BBC News Prediction","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"### Get data\n!wget --no-check-certificate \\\n    https://storage.googleapis.com/laurencemoroney-blog.appspot.com/bbc-text.csv \\\n    -O /tmp/bbc-text.csv\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Load data\ndata = pd.read_csv('/tmp/bbc-text.csv')\nlabel = list(data['category'])\nsentences_raw = list(data['text'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Remove stopwords from data\nfrom nltk.tokenize import word_tokenize\nstopwords = [ \"a\", \"about\", \"above\", \"after\", \"again\", \"against\", \"all\", \"am\", \"an\", \"and\", \"any\", \"are\", \"as\", \"at\", \"be\", \"because\", \"been\", \"before\", \"being\", \"below\", \"between\", \"both\", \"but\", \"by\", \"could\", \"did\", \"do\", \"does\", \"doing\", \"down\", \"during\", \"each\", \"few\", \"for\", \"from\", \"further\", \"had\", \"has\", \"have\", \"having\", \"he\", \"he'd\", \"he'll\", \"he's\", \"her\", \"here\", \"here's\", \"hers\", \"herself\", \"him\", \"himself\", \"his\", \"how\", \"how's\", \"i\", \"i'd\", \"i'll\", \"i'm\", \"i've\", \"if\", \"in\", \"into\", \"is\", \"it\", \"it's\", \"its\", \"itself\", \"let's\", \"me\", \"more\", \"most\", \"my\", \"myself\", \"nor\", \"of\", \"on\", \"once\", \"only\", \"or\", \"other\", \"ought\", \"our\", \"ours\", \"ourselves\", \"out\", \"over\", \"own\", \"same\", \"she\", \"she'd\", \"she'll\", \"she's\", \"should\", \"so\", \"some\", \"such\", \"than\", \"that\", \"that's\", \"the\", \"their\", \"theirs\", \"them\", \"themselves\", \"then\", \"there\", \"there's\", \"these\", \"they\", \"they'd\", \"they'll\", \"they're\", \"they've\", \"this\", \"those\", \"through\", \"to\", \"too\", \"under\", \"until\", \"up\", \"very\", \"was\", \"we\", \"we'd\", \"we'll\", \"we're\", \"we've\", \"were\", \"what\", \"what's\", \"when\", \"when's\", \"where\", \"where's\", \"which\", \"while\", \"who\", \"who's\", \"whom\", \"why\", \"why's\", \"with\", \"would\", \"you\", \"you'd\", \"you'll\", \"you're\", \"you've\", \"your\", \"yours\", \"yourself\", \"yourselves\" ]\nsentences = []\n\nfor t in tqdm(sentences_raw) :\n    tokenize_text = word_tokenize(t)\n    list_text = [i for i in tokenize_text if i not in stopwords]\n    sentences.append((\" \").join(list_text))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lab_tokenizer.word_index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Define variable for tokenizing and modelling\n# Reduce this parameter to reduce overfitting\nVOCAB_SIZE = 1000\nEMBEDDING_DIM = 16\nMAX_LENGTH = 120\n\nTRUNC_TYPE = 'post'\nPADDING_TYPE = 'post'\nOOV_TOK = '<OOV>'\nTRAIN_PROP = .8","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Split dataset\ntrain_size = int(TRAIN_PROP * len(sentences))\ntrain_sentences = sentences[:train_size]\ntrain_label = label[:train_size]\nval_sentences = sentences[train_size:]\nval_label = label[train_size:]\n\nprint(train_size)\nprint(len(train_sentences))\nprint(len(train_label))\nprint(len(val_sentences))\nprint(len(val_label))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Tokenize the train sentences\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\ntext_tokenizer = Tokenizer(oov_token=OOV_TOK, num_words=VOCAB_SIZE)\n\n# Get the token dict from data\ntext_tokenizer.fit_on_texts(train_sentences)\nword_index = text_tokenizer.word_index\n\n# Pad the data\ntrain_sequences = text_tokenizer.texts_to_sequences(train_sentences)\ntrain_padded = pad_sequences(train_sequences, padding=PADDING_TYPE, truncating=TRUNC_TYPE, maxlen=MAX_LENGTH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Pad the validation data\nval_sequences = text_tokenizer.texts_to_sequences(val_sentences)\nval_padded = pad_sequences(val_sequences, padding=PADDING_TYPE, truncating=TRUNC_TYPE, maxlen=MAX_LENGTH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Tokenize the label\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\nlab_tokenizer = Tokenizer()\n\n# Get the token dict from data\nlab_tokenizer.fit_on_texts(train_label)\n\n# Get label sequences\ntrain_label_seq = np.array(lab_tokenizer.texts_to_sequences(train_label))\nval_label_seq = np.array(lab_tokenizer.texts_to_sequences(val_label))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Make simple Embedding MLP model\nfrom tensorflow.keras.layers import Dense, Embedding, GlobalAveragePooling1D\nfrom tensorflow.keras import Sequential, Model\nfrom tensorflow.keras.optimizers import SGD, Adam, RMSprop\nfrom tensorflow.keras.metrics import Precision\n\nmodel = Sequential()\nmodel.add(Embedding(VOCAB_SIZE, EMBEDDING_DIM, input_length=MAX_LENGTH))\nmodel.add(GlobalAveragePooling1D())\nmodel.add(Dense(24, activation='relu'))\nmodel.add(Dense(6, activation='softmax'))\n\n# Compile model\nmodel.compile(optimizer = Adam(),\n              loss = 'sparse_categorical_crossentropy',\n              metrics = 'accuracy')\n\n# Summary of the model\nmodel.summary()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Train the model\nnum_epochs = 30\nhistory = model.fit(train_padded,\n                    train_label_seq,\n                    epochs=num_epochs,\n                    validation_data=(val_padded, val_label_seq),\n                    verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Plot the performance of the model\nrcParams['figure.figsize'] = [10,8]\nplt.style.use('fivethirtyeight') \nsns.set_style('whitegrid')\ngrid = gridspec.GridSpec(2,1)\n\n# Get the metrics and loss\nacc      = history.history[     'accuracy' ]\nval_acc  = history.history[ 'val_accuracy' ]\nloss     = history.history[    'loss' ]\nval_loss = history.history['val_loss' ]\nepo   = range(len(acc)) # Get number of epochs\n\n# Plot the loss\nax = plt.subplot(grid[0])\nax.plot(epo, loss, label='Train Loss')\nax.plot(epo, val_loss, label='Validation Loss')\nax.set_title('Training and Validation Loss')\nax.legend() ;\n\n# Plot the acccuracy\nax = plt.subplot(grid[1])\nax.plot(epo, acc, label='Train Acc')\nax.plot(epo, val_acc, label='Validation Acc')\nax.set_title('Training and Validation Accuracy')\nax.legend() ;","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Make dictionary to reverse the number from tokenizing to text\nreverse_word_index = dict([(value, key) for (key, value) in word_index.items()])\n\ndef decode_sentence(text):\n    return ' '.join([reverse_word_index.get(i, '?') for i in text])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Get the embedding weight for visualization\ne = model.layers[0]\nweights = e.get_weights()[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Save the weight\nimport io\n\nout_v = io.open('vecs.tsv', 'w', encoding='utf-8')\nout_m = io.open('meta.tsv', 'w', encoding='utf-8')\nfor word_num in range(1, VOCAB_SIZE):\n    word = reverse_word_index[word_num]\n    embeddings = weights[word_num]\n    out_m.write(word + \"\\n\")\n    out_v.write('\\t'.join([str(x) for x in embeddings]) + \"\\n\")\nout_v.close()\nout_m.close()\n\n# Use the file to make embedding visualization at https://projector.tensorflow.org/","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Stanford","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"### Get data\n!wget --no-check-certificate \\\n    https://storage.googleapis.com/laurencemoroney-blog.appspot.com/training_cleaned.csv \\\n    -O /tmp/training_cleaned.csv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Load the data\ndata = pd.read_csv('/tmp/training_cleaned.csv', names=['label','id','time','query','handle','text'])\ndata = data.sample(frac=1).reset_index(drop=True)\nsentences = list(data['text'])\nlabel = list(data['label'].map({0:0, 4:1}))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Define variable for tokenizing and modelling\nEMBEDDING_DIM = 100\nMAX_LENGTH = 16\nTRUNC_TYPE = 'post'\nPADDING_TYPE = 'post'\nOOV_TOK = '<OOV>'\nTRAIN_PROP = .8","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Split dataset\ntrain_size = int(TRAIN_PROP * len(sentences))\ntrain_sentences = sentences[:train_size]\ntrain_label = label[:train_size]\nval_sentences = sentences[train_size:]\nval_label = label[train_size:]\n\nprint(train_size)\nprint(len(train_sentences))\nprint(len(train_label))\nprint(len(val_sentences))\nprint(len(val_label))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Tokenize the train sentences\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\ntext_tokenizer = Tokenizer(oov_token=OOV_TOK)\n\n# Get the token dict from data\ntext_tokenizer.fit_on_texts(train_sentences)\nword_index = text_tokenizer.word_index\n\n# Pad the data\ntrain_sequences = text_tokenizer.texts_to_sequences(train_sentences)\ntrain_padded = pad_sequences(train_sequences, padding=PADDING_TYPE, truncating=TRUNC_TYPE, maxlen=MAX_LENGTH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Pad the validation data\nval_sequences = text_tokenizer.texts_to_sequences(val_sentences)\nval_padded = pad_sequences(val_sequences, padding=PADDING_TYPE, truncating=TRUNC_TYPE, maxlen=MAX_LENGTH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Define vocab size\nVOCAB_SIZE = len(word_index) + 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Get weight for embedding matrix in model\n!wget --no-check-certificate \\\n    https://storage.googleapis.com/laurencemoroney-blog.appspot.com/glove.6B.100d.txt \\\n    -O /tmp/glove.6B.100d.txt\n    \n# Load the weight\nembeddings_index = {};\nwith open('/tmp/glove.6B.100d.txt') as f:\n    for line in f:\n        values = line.split();\n        word = values[0];\n        coefs = np.asarray(values[1:], dtype='float32');\n        embeddings_index[word] = coefs;\n\n# Make weight matrix\nembeddings_matrix = np.zeros((VOCAB_SIZE, EMBEDDING_DIM));\nfor word, i in word_index.items():\n    embedding_vector = embeddings_index.get(word);\n    if embedding_vector is not None:\n        embeddings_matrix[i] = embedding_vector;","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Make model\nfrom tensorflow.keras.layers import Dense, Embedding, GlobalAveragePooling1D, GlobalMaxPooling1D, Conv1D, LSTM, Bidirectional, GRU\nfrom tensorflow.keras import Sequential, Model\nfrom tensorflow.keras.optimizers import SGD, Adam, RMSprop\nfrom tensorflow.keras.metrics import Precision\n\n# Make simple embedding model\nmodel_simple = Sequential()\nmodel_simple.add(Embedding(VOCAB_SIZE, EMBEDDING_DIM, input_length=MAX_LENGTH,\n                    weights=[embeddings_matrix], trainable=False))\nmodel_simple.add(GlobalAveragePooling1D())\nmodel_simple.add(Dense(128, activation='relu'))\nmodel_simple.add(Dense(1, activation='sigmoid'))\n\nmodel_simple.compile(optimizer = Adam(),\n              loss = 'binary_crossentropy',\n              metrics = 'accuracy')\n\n# Make single LSTM model\nmodel_single_lstm = Sequential()\nmodel_single_lstm.add(Embedding(VOCAB_SIZE, EMBEDDING_DIM, input_length=MAX_LENGTH,\n                    weights=[embeddings_matrix], trainable=False))\nmodel_single_lstm.add(Bidirectional(LSTM(64)))\nmodel_single_lstm.add(Dense(128, activation='relu'))\nmodel_single_lstm.add(Dense(1, activation='sigmoid'))\n\nmodel_single_lstm.compile(optimizer = Adam(),\n              loss = 'binary_crossentropy',\n              metrics = 'accuracy')\n\n# Make single GRU\nmodel_single_gru = Sequential()\nmodel_single_gru.add(Embedding(VOCAB_SIZE, EMBEDDING_DIM, input_length=MAX_LENGTH,\n                    weights=[embeddings_matrix], trainable=False))\nmodel_single_gru.add(Bidirectional(GRU(64)))\nmodel_single_gru.add(Dense(128, activation='relu'))\nmodel_single_gru.add(Dense(1, activation='sigmoid'))\n\nmodel_single_gru.compile(optimizer = Adam(),\n              loss = 'binary_crossentropy',\n              metrics = 'accuracy')\n\n# Make single Conv\nmodel_single_conv = Sequential()\nmodel_single_conv.add(Embedding(VOCAB_SIZE, EMBEDDING_DIM, input_length=MAX_LENGTH,\n                    weights=[embeddings_matrix], trainable=False))\nmodel_single_conv.add(Conv1D(64, 5, activation='relu'))\nmodel_single_conv.add(GlobalMaxPooling1D())\nmodel_single_conv.add(Dense(128, activation='relu'))\nmodel_single_conv.add(Dense(1, activation='sigmoid'))\n\nmodel_single_conv.compile(optimizer = Adam(),\n              loss = 'binary_crossentropy',\n              metrics = 'accuracy')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Plot the performance of the model\ndef plot_history(history) :\n    rcParams['figure.figsize'] = [10,8]\n    plt.style.use('fivethirtyeight') \n    sns.set_style('whitegrid')\n    grid = gridspec.GridSpec(2,1)\n\n    # Get the metrics and loss\n    acc      = history.history[     'accuracy' ]\n    val_acc  = history.history[ 'val_accuracy' ]\n    loss     = history.history[    'loss' ]\n    val_loss = history.history['val_loss' ]\n    epo   = range(len(acc)) # Get number of epochs\n\n    # Plot the loss\n    ax = plt.subplot(grid[0])\n    ax.plot(epo, loss, label='Train Loss')\n    ax.plot(epo, val_loss, label='Validation Loss')\n    ax.set_title('Training and Validation Loss')\n    ax.legend() ;\n\n    # Plot the acccuracy\n    ax = plt.subplot(grid[1])\n    ax.plot(epo, acc, label='Train Acc')\n    ax.plot(epo, val_acc, label='Validation Acc')\n    ax.set_title('Training and Validation Accuracy')\n    ax.legend() ;","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Train simple model\nnum_epochs = 5\nhistory_simple = model_simple.fit(train_padded,\n                    np.array(train_label),\n                    epochs=num_epochs,\n                    validation_data=(val_padded, np.array(val_label)),\n                    verbose=1)\n\n%time","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Plot simple model performance\nplot_history(history_simple)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Train single lstm model\nnum_epochs = 10\nhistory_single_lstm = model_single_lstm.fit(train_padded,\n                    np.array(train_label),\n                    epochs=num_epochs,\n                    validation_data=(val_padded, np.array(val_label)),\n                    verbose=1)\n\n%time","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Plot simple model performance\nplot_history(history_single_lstm)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Train single gru model\nnum_epochs = 5\nhistory_single_gru = model_single_gru.fit(train_padded,\n                    np.array(train_label),\n                    epochs=num_epochs,\n                    validation_data=(val_padded, np.array(val_label)),\n                    verbose=1)\n\n%time","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Plot simple model performance\nplot_history(history_single_gru)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Train single conv model\nnum_epochs = 5\nhistory_single_conv = model_single_conv.fit(train_padded,\n                    np.array(train_label),\n                    epochs=num_epochs,\n                    validation_data=(val_padded, np.array(val_label)),\n                    verbose=1)\n\n%time","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Plot simple model performance\nplot_history(history_single_conv)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Make complicated model\nfrom tensorflow.keras.layers import Dense, Embedding, Dropout, MaxPooling1D, GlobalAveragePooling1D, GlobalMaxPooling1D, Conv1D, LSTM, Bidirectional, GRU\nfrom tensorflow.keras import Sequential, Model\nfrom tensorflow.keras.optimizers import SGD, Adam, RMSprop\nfrom tensorflow.keras.metrics import Precision\n\n# Make simple embedding model\nmodel = Sequential()\nmodel.add(Embedding(VOCAB_SIZE, EMBEDDING_DIM, input_length=MAX_LENGTH,\n                    weights=[embeddings_matrix], trainable=False))\nmodel.add(Dropout(0.2))\nmodel.add(Conv1D(64, 5, activation='relu'))\nmodel.add(MaxPooling1D(4))\nmodel.add(LSTM(64))\nmodel.add(Dense(1, activation='sigmoid'))\n\nmodel.compile(optimizer = Adam(),\n              loss = 'binary_crossentropy',\n              metrics = 'accuracy')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Train single conv model\nnum_epochs = 5\nhistory = model.fit(train_padded,\n                    np.array(train_label),\n                    epochs=num_epochs,\n                    validation_data=(val_padded, np.array(val_label)),\n                    verbose=1)\n\n%time","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Plot simple model performance\nplot_history(history)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Shakespeare","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"### Get data\n!wget --no-check-certificate \\\n    https://storage.googleapis.com/laurencemoroney-blog.appspot.com/sonnets.txt \\\n    -O /tmp/sonnets.txt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Load the data\ndata = open('/tmp/sonnets.txt').read()\ncorpus = data.lower().split('\\n')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Define variable for tokenizing and modelling\nEMBEDDING_DIM = 100\nTRUNC_TYPE = 'post'\nPADDING_TYPE = 'pre'\nOOV_TOK = '<OOV>'\nTRAIN_PROP = .8","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Make the input sequences for prediciton word\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\ntokenizer = Tokenizer()\n\n# Tokenize the corpus\ntokenizer.fit_on_texts(corpus)\nword_index = tokenizer.word_index\nVOCAB_SIZE = len(word_index) + 1\n\n# Make input sequences\ntrain_seq = []\nfor text in corpus :\n    token_list = tokenizer.texts_to_sequences([text])[0]\n    \n    for i in range(1, len(token_list)) :\n        gram = token_list[:i+1]\n        train_seq.append(gram)\n        \n# Pad the sequences\nMAX_LENGTH = np.max([len(seq) for seq in train_seq])\ntrain_padded = pad_sequences(train_seq, padding=PADDING_TYPE, maxlen=MAX_LENGTH)\n\n# Split to train data and label\ntrain_data, train_label = train_padded[:,:-1], train_padded[:,-1]\n\n# One hot label\nfrom tensorflow.keras.utils import to_categorical\ntrain_label = to_categorical(train_label, num_classes=VOCAB_SIZE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Get weight for embedding matrix in model\n!wget --no-check-certificate \\\n    https://storage.googleapis.com/laurencemoroney-blog.appspot.com/glove.6B.100d.txt \\\n    -O /tmp/glove.6B.100d.txt\n    \n# Load the weight\nembeddings_index = {};\nwith open('/tmp/glove.6B.100d.txt') as f:\n    for line in f:\n        values = line.split();\n        word = values[0];\n        coefs = np.asarray(values[1:], dtype='float32');\n        embeddings_index[word] = coefs;\n\n# Make weight matrix\nembeddings_matrix = np.zeros((VOCAB_SIZE, EMBEDDING_DIM));\nfor word, i in word_index.items():\n    embedding_vector = embeddings_index.get(word);\n    if embedding_vector is not None:\n        embeddings_matrix[i] = embedding_vector;","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Make model\nfrom tensorflow.keras.layers import Dense, Embedding, TimeDistributed, Dropout, GlobalAveragePooling1D, GlobalMaxPooling1D, Conv1D, LSTM, Bidirectional, GRU\nfrom tensorflow.keras import Sequential, Model\nfrom tensorflow.keras.regularizers import l1, l2\nfrom tensorflow.keras.optimizers import SGD, Adam, RMSprop\nfrom tensorflow.keras.metrics import Precision\n\n# Make simple embedding model\nmodel = Sequential()\nmodel.add(Embedding(VOCAB_SIZE, EMBEDDING_DIM, input_length=MAX_LENGTH-1,\n                    weights=[embeddings_matrix], trainable=True))\nmodel.add(Bidirectional(LSTM(256, return_sequences=True)))\nmodel.add(Dropout(0.2))\nmodel.add(Bidirectional(LSTM(128)))\nmodel.add(Dense(VOCAB_SIZE // 2, activation='relu', kernel_regularizer=l2(0.01)))\nmodel.add(Dense(VOCAB_SIZE, activation='softmax'))\n\nmodel.compile(optimizer = Adam(),\n              loss = 'categorical_crossentropy',\n              metrics = 'accuracy')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Train model\nnum_epochs = 100\nhistory = model.fit(train_data,\n                    train_label,\n                    epochs=num_epochs,\n                    verbose=1)\n\n%time","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Plot the performance of the model\ndef plot_history(history) :\n    rcParams['figure.figsize'] = [10,8]\n    plt.style.use('fivethirtyeight') \n    sns.set_style('whitegrid')\n    grid = gridspec.GridSpec(2,1)\n\n    # Get the metrics and loss\n    acc      = history.history[     'accuracy' ]\n    loss     = history.history[    'loss' ]\n    epo   = range(len(acc)) # Get number of epochs\n\n    # Plot the loss\n    ax = plt.subplot(grid[0])\n    ax.plot(epo, loss, label='Train Loss')\n    ax.set_title('Training Loss')\n    ax.legend() ;\n\n    # Plot the acccuracy\n    ax = plt.subplot(grid[1])\n    ax.plot(epo, acc, label='Train Acc')\n    ax.set_title('Training Accuracy')\n    ax.legend() ;","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Plot model performance - LSTM no Bidirection\nplot_history(history)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"seed_text = \"Help me Obi Wan Kenobi, you're my only hope\"\nnext_words = 100\n  \nfor _ in range(next_words):\n    token_list = tokenizer.texts_to_sequences([seed_text])[0]\n    token_list = pad_sequences([token_list], maxlen=MAX_LENGTH-1, padding='pre')\n    predicted = model.predict_classes(token_list, verbose=0)\n    output_word = \"\"\n    for word, index in tokenizer.word_index.items():\n        if index == predicted:\n            output_word = word\n            break\n    seed_text += \" \" + output_word\nprint(seed_text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}