{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport tensorflow as tf\nimport seaborn as sns\nimport cv2, shutil, os\nfrom sklearn.preprocessing import StandardScaler\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.preprocessing.text import Tokenizer\n#import zipfile module\nfrom zipfile import ZipFile\nimport matplotlib.pyplot as plt\nfrom tensorflow.keras import datasets,layers,models\nfrom keras.models import Sequential, load_model\nfrom sklearn.model_selection import train_test_split","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-02-02T10:25:31.756434Z","iopub.execute_input":"2023-02-02T10:25:31.757693Z","iopub.status.idle":"2023-02-02T10:25:44.115324Z","shell.execute_reply.started":"2023-02-02T10:25:31.757549Z","shell.execute_reply":"2023-02-02T10:25:44.114262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_history(history):\n    plt.plot(history.history['accuracy'])\n    plt.plot(history.history['val_accuracy'])\n    plt.title('model accuracy')\n    plt.ylabel('accuracy')\n    plt.xlabel('epoch')\n    plt.legend(['train', 'test'], loc='upper left')\n    plt.show()\n    # summarize history for loss\n    plt.plot(history.history['loss'])\n    plt.plot(history.history['val_loss'])\n    plt.title('model loss')\n    plt.ylabel('loss')\n    plt.xlabel('epoch')\n    plt.legend(['train', 'test'], loc='upper left')\n    plt.show()\n\ndef plot_train(train_images,class_names,train_labels):\n    plt.figure(figsize=(20,10))\n    for i in range(5):\n        plt.subplot(5,2,i+1)\n        plt.xticks([])\n        plt.yticks([])\n        plt.grid(False)\n        plt.imshow(train_images[i],cmap=plt.cm.binary)\n        # The CIFAR labels happen to be arrays,# which is why you need the extra index\n        plt.xlabel(class_names[train_labels[i]])\n        plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:25:44.135634Z","iopub.execute_input":"2023-02-02T10:25:44.135948Z","iopub.status.idle":"2023-02-02T10:25:44.145505Z","shell.execute_reply.started":"2023-02-02T10:25:44.135921Z","shell.execute_reply":"2023-02-02T10:25:44.144435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.python.client import device_lib\nprint(device_lib.list_local_devices())","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2023-02-02T10:22:51.835824Z","iopub.execute_input":"2023-02-02T10:22:51.836840Z","iopub.status.idle":"2023-02-02T10:22:56.857285Z","shell.execute_reply.started":"2023-02-02T10:22:51.836802Z","shell.execute_reply":"2023-02-02T10:22:56.854280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# configuration to resolve performance issue\ninit = tf.compat.v1.global_variables_initializer()\nconfig = tf.compat.v1.ConfigProto(intra_op_parallelism_threads=4,\n                       inter_op_parallelism_threads=4,\n                       allow_soft_placement=True)","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:25:49.723387Z","iopub.execute_input":"2023-02-02T10:25:49.723802Z","iopub.status.idle":"2023-02-02T10:25:55.035446Z","shell.execute_reply.started":"2023-02-02T10:25:49.723766Z","shell.execute_reply":"2023-02-02T10:25:55.034393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **PERCEPTRON**\n<img src=\"https://i0.wp.com/datascientest.com/wp-content/uploads/2021/04/illu_perceptron_blog-138.png?w=1024&ssl=1\" alt=\"Alternative text\" width=500px height=500px />","metadata":{}},{"cell_type":"markdown","source":"Un Perceptron est un neurone artificiel, et donc une unité de réseau de neurones. Il effectue des calculs pour détecter des caractéristiques ou des tendances dans les données d’entrée.\nIl s’agit d’un algorithme pour l’apprentissage supervisé de classificateurs binaires. C’est cet algorithme qui permet aux neurones artificiels d’apprendre et de traiter les éléments d’un ensemble de données.","metadata":{}},{"cell_type":"markdown","source":"En réalité, le Perceptron est une fonction mathématique. Les données d’entrée (x) sont multipliées par les coefficients de poids (w). Le résultat produit est une valeur.Cette valeur peut être positive ou négative. Le neurone artificiel s’active si la valeur est positive. Il ne s’active donc que si le poids calculé des données d’entrée dépasse un certain seuil.\nLe résultat prédit est comparé avec le résultat connu. En cas de différence, l’erreur est rétropropagée afin de permettre d’ajuster les poids.\n","metadata":{}},{"cell_type":"code","source":"def step_function(x):\n    if x<0:\n        return 0\n    else:\n        return 1\n\n\ntraining_set = [((0, 0), 0), ((0, 1), 1), ((1, 0), 1), ((1, 1), 1)]\n\n# ploting data points using seaborn (Seaborn requires dataframe)\nplt.figure(0)\n\nx1 = [training_set[i][0][0] for i in range(4)]\nx2 = [training_set[i][0][1] for i in range(4)]\ny = [training_set[i][1] for i in range(4)]\n\ndf = pd.DataFrame(\n    {'x1': x1,\n     'x2': x2,\n     'y': y\n    })\n    \nsns.lmplot(\"x1\", \"x2\", data=df, hue='y', fit_reg=False, markers=[\"o\", \"s\"])\n\n\n# parameter initialization\nw = np.random.rand(2)\nerrors = [] \neta = .5\nepoch = 30\nb = 0\n\n\n# Learning\nfor i in range(epoch):\n    for x, y in training_set:\n      # u = np.dot(x , w) +b\n        u = sum(x*w) + b\n        \n        error = y - step_function(u) \n      \n        errors.append(error) \n        for index, value in enumerate(x):\n            #print(w[index])\n            w[index] += eta * error * value\n            b += eta*error\n   \n        # produce all decision boundaries\n            a = [0,-b/w[1]]\n            c = [-b/w[0],0]\n            \"\"\"plt.figure(1)\n            plt.plot(a,c)\"\"\"\n\n            \n            \n# final decision boundary\na = [0,-b/w[1]]\nc = [-b/w[0],0]\nplt.plot(a,c)\n","metadata":{"execution":{"iopub.status.busy":"2023-02-02T09:54:03.875917Z","iopub.execute_input":"2023-02-02T09:54:03.876338Z","iopub.status.idle":"2023-02-02T09:54:04.430488Z","shell.execute_reply.started":"2023-02-02T09:54:03.876291Z","shell.execute_reply":"2023-02-02T09:54:04.429569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **MULTILAYER PERCEPTRON**\n<img src=\"https://miro.medium.com/max/720/1*MF1q2Q3fbpYlXX8fZUiwpA.webp\" alt=\"Alternative text\" width=500px height=500px />","metadata":{}},{"cell_type":"markdown","source":"Le perceptron multicouche a été développé pour remédier à cette limitation. Il s'agit d'un réseau neuronal dans lequel la correspondance entre les entrées et les sorties n'est pas linéaire.\nUn perceptron multicouche comporte des couches d'entrée et de sortie, et une ou plusieurs couches cachées avec de nombreux neurones empilés. Et alors que dans le perceptron, le neurone doit avoir une fonction d'activation qui impose un seuil, comme ReLU ou sigmoïde, les neurones d'un perceptron multicouche peuvent utiliser n'importe quelle fonction d'activation arbitraire.","metadata":{}},{"cell_type":"markdown","source":"# **MNIST DIGITS**\n<img src=\"https://camo.githubusercontent.com/01c057a753e92a9bc70b8c45d62b295431851c09cffadf53106fc0aea7e2843f/687474703a2f2f692e7974696d672e636f6d2f76692f3051493378675875422d512f687164656661756c742e6a7067\" alt=\"Alternative text\" width=500px height=500px />\n","metadata":{}},{"cell_type":"code","source":"# the data, shuffled and split between a train and test sets \nclass_names = [str(i) for i in range(10)]\n(X_train, y_train), (X_test, y_test) = datasets.mnist.load_data()\nplot_train(X_train,class_names,y_train)\n# for each image we have a (28*28) vector\n# we will convert the (28*28) vector into single dimensional vector of 1 * 784 \nX_train = X_train.reshape(X_train.shape[0], X_train.shape[1]*X_train.shape[2])/255.\nX_test = X_test.reshape(X_test.shape[0], X_test.shape[1]*X_test.shape[2])/255.","metadata":{"execution":{"iopub.status.busy":"2023-02-02T09:54:04.431897Z","iopub.execute_input":"2023-02-02T09:54:04.432730Z","iopub.status.idle":"2023-02-02T09:54:06.067316Z","shell.execute_reply.started":"2023-02-02T09:54:04.432691Z","shell.execute_reply":"2023-02-02T09:54:06.066108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Sequential()\nmodel.add(layers.Dense(units=512,activation='elu',input_shape=(784,)))\nmodel.add(layers.Dense(units=512,activation='elu'))\nmodel.add(layers.Dense(units=64,activation='elu'))\nmodel.add(layers.Dense(units=10,activation='softmax'))\nmodel.compile(optimizer='adam',loss='sparse_categorical_crossentropy',metrics=['accuracy'])\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-02-02T09:54:06.068826Z","iopub.execute_input":"2023-02-02T09:54:06.069247Z","iopub.status.idle":"2023-02-02T09:54:06.191122Z","shell.execute_reply.started":"2023-02-02T09:54:06.069208Z","shell.execute_reply":"2023-02-02T09:54:06.190189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with tf.device('/gpu:0'):\n    history = model.fit(X_train, y_train, epochs=10,validation_data = (X_test,y_test))","metadata":{"execution":{"iopub.status.busy":"2023-02-02T09:54:06.192506Z","iopub.execute_input":"2023-02-02T09:54:06.193063Z","iopub.status.idle":"2023-02-02T09:54:58.777150Z","shell.execute_reply.started":"2023-02-02T09:54:06.193027Z","shell.execute_reply":"2023-02-02T09:54:58.776182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history(history)","metadata":{"execution":{"iopub.status.busy":"2023-02-02T09:54:58.778895Z","iopub.execute_input":"2023-02-02T09:54:58.779271Z","iopub.status.idle":"2023-02-02T09:54:59.214684Z","shell.execute_reply.started":"2023-02-02T09:54:58.779232Z","shell.execute_reply":"2023-02-02T09:54:59.213628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with tf.device('/gpu:0'):\n    predicted_classes = model.predict(X_test)\n    predicted_classes = np.argmax(predicted_classes,axis = 1)\n    test_accuracy = model.evaluate(X_test, y_test)[1]\n    print(\"Accuracy on test images =  \",test_accuracy*100,\"%\")\nplt.figure(figsize=(20,10))\nfor i in range(10):\n    plt.subplot(2,5,i+1)\n    plt.xticks([])\n    plt.yticks([])\n    plt.grid(False)\n    plt.imshow(X_test[i].reshape((28,28)),cmap=plt.cm.binary)\n    # The CIFAR labels happen to be arrays,# which is why you need the extra index\n    plt.xlabel(class_names[predicted_classes[i]])\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-02T09:54:59.219994Z","iopub.execute_input":"2023-02-02T09:54:59.220368Z","iopub.status.idle":"2023-02-02T09:55:01.814823Z","shell.execute_reply.started":"2023-02-02T09:54:59.220337Z","shell.execute_reply":"2023-02-02T09:55:01.813247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **CONVOLUTIONAL NEURAL NETWORK**\n<img src=\"https://miro.medium.com/max/720/1*uAeANQIOQPqWZnnuH-VEyw.jpeg\" alt=\"Alternative text\" width=500px height=500px />","metadata":{}},{"cell_type":"markdown","source":"Un réseau neuronal convolutif (ConvNet/CNN) est un algorithme d'apprentissage profond capable de prendre une image en entrée, d'attribuer une importance (poids et biais apprenables) à divers aspects/objets de l'image, et de les différencier les uns des autres.\nUne image n'est rien d'autre qu'une matrice de valeurs de pixels. Alors pourquoi ne pas simplement aplatir l'image (par exemple, une matrice d'image 3x3 en un vecteur 9x1) et l'envoyer à un perceptron multi-niveaux à des fins de classification ? \n","metadata":{}},{"cell_type":"markdown","source":"Dans le cas d'images binaires extrêmement basiques, la méthode pourrait afficher un score de précision moyen lors de la prédiction des classes, mais n'aurait que peu ou pas de précision lorsqu'il s'agit d'images complexes présentant des dépendances de pixels.\nUn ConvNet est capable de capturer avec succès les dépendances spatiales et temporelles d'une image grâce à l'application de filtres pertinents. L'architecture s'adapte mieux à l'ensemble des données d'image grâce à la réduction du nombre de paramètres impliqués et à la réutilisation des poids. En d'autres termes, le réseau peut être entraîné à mieux comprendre la sophistication de l'image.\n","metadata":{}},{"cell_type":"markdown","source":"**CONVOLUTION**","metadata":{}},{"cell_type":"markdown","source":"L'objectif de l'opération de convolution est d'extraire les caractéristiques de haut niveau, telles que les bords, de l'image d'entrée. Les ConvNets ne doivent pas être limités à une seule couche de convolution. Conventionnellement, la première couche de convolution est responsable de la capture des caractéristiques de bas niveau telles que les bords, la couleur, l'orientation du gradient, etc. Avec l'ajout de couches, l'architecture s'adapte également aux caractéristiques de haut niveau, nous donnant un réseau qui a une compréhension globale des images dans l'ensemble de données, comme nous le ferions.\n\nL'opération donne deux types de résultats : l'un dans lequel la dimension de la caractéristique convoluée est réduite par rapport à l'entrée, et l'autre dans lequel la dimension est augmentée ou reste la même. Ceci est fait en appliquant Valid Padding dans le cas du premier, ou Same Padding dans le cas du second. \n","metadata":{}},{"cell_type":"markdown","source":"<img src=\"https://miro.medium.com/max/640/1*nYf_cUIHFEWU1JXGwnz-Ig.gif\" alt=\"Alternative text\" width=200px height=100px />","metadata":{}},{"cell_type":"markdown","source":"**POOLING**","metadata":{}},{"cell_type":"markdown","source":"Comme la couche convolutionnelle, la couche de mise en commun est responsable de la réduction de la taille spatiale de la caractéristique convoluée. Cela permet de diminuer la puissance de calcul nécessaire au traitement des données par réduction de la dimensionnalité. En outre, elle est utile pour extraire les caractéristiques dominantes qui sont invariantes en termes de rotation et de position, ce qui permet de maintenir le processus d'entraînement efficace du modèle.\n\nIl existe deux types de mise en commun : Max Pooling et Average Pooling. La mise en commun maximale renvoie la valeur maximale de la partie de l'image couverte par le noyau. D'autre part, la mise en commun moyenne renvoie la moyenne de toutes les valeurs de la partie de l'image couverte par le noyau.\n\nLe Max Pooling fonctionne également comme un suppresseur de bruit. Il élimine complètement les activations bruyantes et effectue également un débruitage ainsi qu'une réduction de la dimensionnalité. D'autre part, la mise en commun moyenne effectue simplement une réduction de la dimensionnalité en tant que mécanisme de suppression du bruit. Par conséquent, nous pouvons dire que le Max Pooling est beaucoup plus performant que le Average Pooling.","metadata":{}},{"cell_type":"markdown","source":"<img src=\"https://miro.medium.com/max/640/1*uoWYsCV5vBU8SHFPAPao-w.gif\" alt=\"Alternative text\" width=500px height=100px />","metadata":{}},{"cell_type":"markdown","source":"# **CNN - FASHION MNIST**\n<img src=\"https://storage.googleapis.com/tfds-data/visualization/fig/fashion_mnist-3.0.1.png\" alt=\"Alternative text\" width=500px height=500px />","metadata":{}},{"cell_type":"code","source":"(train_images,train_labels), (test_images,test_labels) = datasets.fashion_mnist.load_data()\nclass_names = ['T-shirt/top',' Trouser','Pullover','Dress','Coat','Sandal','Shirt','Sneaker','Bag','Ankle boot']\n# Normalize pixel values to be between 0 and 1\ntrain_images, test_images = train_images/255.0, test_images/255.0","metadata":{"execution":{"iopub.status.busy":"2023-02-02T09:55:01.822199Z","iopub.execute_input":"2023-02-02T09:55:01.826049Z","iopub.status.idle":"2023-02-02T09:55:03.359330Z","shell.execute_reply.started":"2023-02-02T09:55:01.825984Z","shell.execute_reply":"2023-02-02T09:55:03.358190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**SOME DISPLAY**","metadata":{}},{"cell_type":"code","source":"plot_train(train_images,class_names,train_labels)","metadata":{"execution":{"iopub.status.busy":"2023-02-02T09:55:03.361125Z","iopub.execute_input":"2023-02-02T09:55:03.361551Z","iopub.status.idle":"2023-02-02T09:55:03.982113Z","shell.execute_reply.started":"2023-02-02T09:55:03.361510Z","shell.execute_reply":"2023-02-02T09:55:03.980544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#SPLIT IN TRAIN VALIDATION\ninput_shape = list(train_images.shape[1:])\ninput_shape.append(1)\nshape = [-1]\nshape.extend(input_shape)\ntrain_images_resh = train_images.reshape(-1,28,28,1)\nX_train, X_val, Y_train, Y_val = train_test_split(train_images_resh, train_labels, test_size = 0.3, random_state=43)","metadata":{"execution":{"iopub.status.busy":"2023-02-02T09:55:03.984023Z","iopub.execute_input":"2023-02-02T09:55:03.984450Z","iopub.status.idle":"2023-02-02T09:55:04.493036Z","shell.execute_reply.started":"2023-02-02T09:55:03.984408Z","shell.execute_reply":"2023-02-02T09:55:04.491452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Sequential()\nmodel.add(layers.Conv2D(256,(3,3),input_shape = input_shape,padding='same'))\nmodel.add(layers.MaxPooling2D((2,2)))\nmodel.add(layers.Conv2D(256,(3,3),activation=\"elu\",padding='same'))\nmodel.add(layers.MaxPooling2D((2,2)))\nmodel.add(layers.Conv2D(256,(3,3),activation=\"elu\",padding='same'))\nmodel.add(layers.MaxPooling2D((2,2)))\n\nmodel.add(layers.Flatten())\nmodel.add(layers.Dense(units=64,activation='elu'))\nmodel.add(layers.BatchNormalization())\nmodel.add(layers.Dropout(0.2))\nmodel.add(layers.Dense(units=64,activation='elu'))\nmodel.add(layers.BatchNormalization())\nmodel.add(layers.Dropout(0.2))\nmodel.add(layers.Dense(units=64,activation='elu'))\nmodel.add(layers.BatchNormalization())\nmodel.add(layers.Dropout(0.2))\n\nmodel.add(layers.Dense(units=32,activation='elu'))\nmodel.add(layers.Dense(units=10,activation='softmax'))\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-02-02T09:55:04.494526Z","iopub.execute_input":"2023-02-02T09:55:04.494927Z","iopub.status.idle":"2023-02-02T09:55:04.637348Z","shell.execute_reply.started":"2023-02-02T09:55:04.494888Z","shell.execute_reply":"2023-02-02T09:55:04.636252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**FIT ON VALIDATION AND TRAIN DATA with GPU**","metadata":{}},{"cell_type":"code","source":"with tf.device('/gpu:0'):\n    model.compile(optimizer='adam',loss='sparse_categorical_crossentropy',metrics=['accuracy'])\n    history = model.fit(X_train, Y_train, epochs=20,validation_data = (X_val,Y_val))","metadata":{"execution":{"iopub.status.busy":"2023-02-02T09:55:04.640564Z","iopub.execute_input":"2023-02-02T09:55:04.640925Z","iopub.status.idle":"2023-02-02T10:00:27.886030Z","shell.execute_reply.started":"2023-02-02T09:55:04.640897Z","shell.execute_reply":"2023-02-02T10:00:27.884742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history(history)","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:00:27.887741Z","iopub.execute_input":"2023-02-02T10:00:27.888143Z","iopub.status.idle":"2023-02-02T10:00:28.372458Z","shell.execute_reply.started":"2023-02-02T10:00:27.888097Z","shell.execute_reply":"2023-02-02T10:00:28.371433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**MAKE PREDICTION ON TEST IMAGES**","metadata":{}},{"cell_type":"code","source":"with tf.device('/gpu:0'):\n    test_images = test_images.reshape(-1,28,28,1)\n    predicted_classes = model.predict(test_images)\n    predicted_classes = np.argmax(predicted_classes,axis = 1)\n    test_accuracy = model.evaluate(test_images, test_labels)[1]\n    print(\"Accuracy on test images =  \",test_accuracy*100,\"%\")\nplt.figure(figsize=(20,10))\nfor i in range(10):\n    plt.subplot(2,5,i+1)\n    plt.xticks([])\n    plt.yticks([])\n    plt.grid(False)\n    plt.imshow(test_images[i],cmap=plt.cm.binary)\n    # The CIFAR labels happen to be arrays,# which is why you need the extra index\n    plt.xlabel(class_names[predicted_classes[i]])\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:00:28.377244Z","iopub.execute_input":"2023-02-02T10:00:28.379802Z","iopub.status.idle":"2023-02-02T10:00:32.013875Z","shell.execute_reply.started":"2023-02-02T10:00:28.379761Z","shell.execute_reply":"2023-02-02T10:00:32.012415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **OVER FITTING**","metadata":{}},{"cell_type":"markdown","source":"Surajustement : On dit qu'un modèle statistique est surajusté lorsque le modèle ne fait pas de prédictions précises sur les données de test. Lorsqu'un modèle est entraîné avec autant de données, il commence à apprendre à partir du bruit et des entrées de données inexactes dans notre ensemble de données. Et lorsque le test avec les données de test entraîne une variance élevée. Alors le modèle ne catégorise pas les données correctement, à cause de trop de détails et de bruit. Les causes de l'overfitting sont les méthodes non paramétriques et non linéaires, car ces types d'algorithmes d'apprentissage automatique ont plus de liberté dans la construction du modèle basé sur l'ensemble de données et peuvent donc vraiment construire des modèles irréalistes. ","metadata":{}},{"cell_type":"markdown","source":"<img src=\"https://media.geeksforgeeks.org/wp-content/cdn-uploads/20190523171258/overfitting_2.png\" alt=\"Alternative text\" width=500px height=500px />","metadata":{}},{"cell_type":"markdown","source":"1.     Augmenter les données de formation.\n1.     Réduire la complexité du modèle.\n1.     Arrêt précoce pendant la phase de formation (surveillez la perte pendant la période de formation ; dès que la perte commence à augmenter, arrêtez la formation).\n1.     Régularisation Ridge et Régularisation Lasso\n1.     Utilisez le dropout pour les réseaux neuronaux afin de lutter contre l'overfitting.","metadata":{}},{"cell_type":"markdown","source":"# **CNN - CIFAR10**\n<img src=\"https://miro.medium.com/max/720/1*OSvbuPLy0PSM2nZ62SbtlQ.png\" alt=\"Alternative text\" width=500px height=500px />","metadata":{}},{"cell_type":"code","source":"(train_images,train_labels), (test_images,test_labels) = datasets.cifar10.load_data()\nclass_names = ['airplane','automobile','bird','cat','deer','dog','frog','horse','ship','truck']\n# Normalize pixel values to be between 0 and 1\ntrain_images, test_images = train_images/255.0, test_images/255.0","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:00:32.020704Z","iopub.execute_input":"2023-02-02T10:00:32.021277Z","iopub.status.idle":"2023-02-02T10:00:37.685456Z","shell.execute_reply.started":"2023-02-02T10:00:32.021223Z","shell.execute_reply":"2023-02-02T10:00:37.684319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_train(train_images,class_names,train_labels[:,0])","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:00:37.686976Z","iopub.execute_input":"2023-02-02T10:00:37.687396Z","iopub.status.idle":"2023-02-02T10:00:38.100410Z","shell.execute_reply.started":"2023-02-02T10:00:37.687348Z","shell.execute_reply":"2023-02-02T10:00:38.098950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#SPLIT IN TRAIN VALIDATION\ninput_shape = list(train_images.shape[1:])\nshape = [-1]\nshape.extend(input_shape)\ntrain_images_resh = train_images.reshape(shape)\nX_train, X_val, Y_train, Y_val = train_test_split(train_images_resh, train_labels, test_size = 0.2, random_state=43)","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:00:38.107022Z","iopub.execute_input":"2023-02-02T10:00:38.108131Z","iopub.status.idle":"2023-02-02T10:00:39.544294Z","shell.execute_reply.started":"2023-02-02T10:00:38.108066Z","shell.execute_reply":"2023-02-02T10:00:39.543195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Nous nous sommes concentrés sur le Dropout et le Early stopping:   \n1. La couche dropout met aléatoirement les unités d'entrée à 0 avec une fréquence de taux à chaque étape de la période d'apprentissage, ce qui permet d'éviter le surajustement. Les entrées qui ne sont pas mises à 0 sont mises à l'échelle par 1/(1 - taux) de sorte que la somme de toutes les entrées reste inchangée.\n1. L'early stopping :  une technique d'optimisation utilisée pour réduire l'overfitting sans compromettre la précision du modèle. L'idée principale de l'arrêt précoce est d'arrêter la formation avant qu'un modèle ne commence à s'adapter de manière excessive.","metadata":{}},{"cell_type":"code","source":"callback = tf.keras.callbacks.EarlyStopping(patience=4, restore_best_weights=True)\nmodel = Sequential()\nmodel.add(layers.Conv2D(64,(3,3),input_shape=input_shape,padding='same'))\nmodel.add(layers.BatchNormalization())\n\nmodel.add(layers.Conv2D(64,(3,3),input_shape=input_shape,padding='same'))\nmodel.add(layers.BatchNormalization())\nmodel.add(layers.MaxPooling2D((2,2)))\n\nmodel.add(layers.Conv2D(128,(3,3),input_shape=input_shape,padding='same'))\nmodel.add(layers.BatchNormalization())\n\nmodel.add(layers.Conv2D(128,(3,3),input_shape=input_shape,padding='same'))\nmodel.add(layers.BatchNormalization())\nmodel.add(layers.MaxPooling2D((2,2)))\n\nmodel.add(layers.Conv2D(256,(3,3),activation=\"relu\",padding='same'))\nmodel.add(layers.BatchNormalization())\n\n\nmodel.add(layers.Conv2D(256,(3,3),activation=\"relu\",padding='same'))\nmodel.add(layers.BatchNormalization())\nmodel.add(layers.MaxPooling2D((2,2)))\n\nmodel.add(layers.Conv2D(512,(3,3),activation=\"relu\",padding='same'))\nmodel.add(layers.BatchNormalization())\n\nmodel.add(layers.Conv2D(512,(3,3),activation=\"relu\",padding='same'))\nmodel.add(layers.BatchNormalization())\nmodel.add(layers.MaxPooling2D((2,2)))\n\nmodel.add(layers.Flatten())\nmodel.add(layers.Dense(units=1024,activation='relu'))\nmodel.add(layers.BatchNormalization())\nmodel.add(layers.Dropout(0.25))\nmodel.add(layers.Dense(units=1024,activation='relu'))\nmodel.add(layers.BatchNormalization())\nmodel.add(layers.Dropout(0.25))\n\nmodel.add(layers.Dense(units=10,activation='softmax'))\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:00:39.546004Z","iopub.execute_input":"2023-02-02T10:00:39.546377Z","iopub.status.idle":"2023-02-02T10:00:39.823177Z","shell.execute_reply.started":"2023-02-02T10:00:39.546338Z","shell.execute_reply":"2023-02-02T10:00:39.822110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**FIT ON VALIDATION AND TRAIN DATA**","metadata":{}},{"cell_type":"code","source":"with tf.device('/gpu:1'):\n    model.compile(optimizer='adam',loss='sparse_categorical_crossentropy',metrics=['accuracy'])\n    history = model.fit(X_train, Y_train, epochs=50,validation_data = (X_val,Y_val),batch_size=1000)","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:00:39.824753Z","iopub.execute_input":"2023-02-02T10:00:39.825580Z","iopub.status.idle":"2023-02-02T10:10:06.052801Z","shell.execute_reply.started":"2023-02-02T10:00:39.825539Z","shell.execute_reply":"2023-02-02T10:10:06.051742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history(history)\n","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:10:06.054567Z","iopub.execute_input":"2023-02-02T10:10:06.055219Z","iopub.status.idle":"2023-02-02T10:10:06.464363Z","shell.execute_reply.started":"2023-02-02T10:10:06.055167Z","shell.execute_reply":"2023-02-02T10:10:06.463437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**PREDICT ON TEST IMAGES**","metadata":{}},{"cell_type":"code","source":"with tf.device('/gpu:0'):\n    test_images = test_images.reshape(shape)\n    predicted_classes = model.predict(test_images)\n    predicted_classes = np.argmax(predicted_classes,axis = 1)\n    test_accuracy = model.evaluate(test_images, test_labels)\n    print(\"Accuracy on test images = \",test_accuracy)\nplt.figure(figsize=(20,10))\nfor i in range(10):\n    plt.subplot(2,5,i+1)\n    plt.xticks([])\n    plt.yticks([])\n    plt.grid(False)\n    plt.imshow(test_images[i].reshape(32,32,3),cmap=plt.cm.binary)\n    # The CIFAR labels happen to be arrays,# which is why you need the extra index\n    plt.xlabel(class_names[predicted_classes[i]])\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:10:06.465913Z","iopub.execute_input":"2023-02-02T10:10:06.466291Z","iopub.status.idle":"2023-02-02T10:10:15.783459Z","shell.execute_reply.started":"2023-02-02T10:10:06.466238Z","shell.execute_reply":"2023-02-02T10:10:15.782037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **IMAGE AUGMENTATION WITH IMGAUG**\n<img src=\"https://editor.analyticsvidhya.com/uploads/84333flip.jpg\" alt=\"Alternative text\" width=500px height=500px />","metadata":{}},{"cell_type":"markdown","source":"On dit d'un réseau neuronal convolutif qui peut classer de manière robuste des objets même s'ils sont placés dans des orientations différentes qu'il possède la propriété appelée invariance. Plus précisément, un réseau neuronal convolutif peut être invariant par rapport à la translation, au point de vue, à la taille ou à l'éclairage (ou à une combinaison de ces éléments).\n\nC'est essentiellement la prémisse de l'augmentation des données. Dans le scénario du monde réel, nous pouvons avoir un ensemble de données d'images prises dans un ensemble limité de conditions. Mais, notre application cible peut exister dans une variété de conditions, telles que l'orientation différente, l'emplacement, l'échelle, la luminosité, etc. Nous tenons compte de ces situations en formant notre réseau neuronal avec des données supplémentaires modifiées de manière synthétique.","metadata":{}},{"cell_type":"code","source":"#load images\nimport glob\nimport imgaug.augmenters as iaa\nimages = []\nimages_path = glob.glob('/kaggle/input/images/*.jpg')\nfor img_path in images_path:\n    img = cv2.imread(img_path)\n    img = img[...,::-1]\n    images.append(img)","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:10:15.785989Z","iopub.execute_input":"2023-02-02T10:10:15.786520Z","iopub.status.idle":"2023-02-02T10:10:17.002406Z","shell.execute_reply.started":"2023-02-02T10:10:15.786468Z","shell.execute_reply":"2023-02-02T10:10:17.001374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 2. Image Augmentation\naugmentation = iaa.Sequential([\n    # 1. Flip\n    iaa.Fliplr(0.5),\n    iaa.Flipud(0.5),\n    # 2. Affine\n    iaa.Affine(translate_percent={\"x\": (-0.2, 0.2), \"y\": (-0.2, 0.2)},\n               rotate=(-30, 30),\n               scale=(0.5, 1.5)),\n    # 3. Multiply\n    iaa.Multiply((0.8, 1.2)),\n    # 4. Linearcontrast\n    iaa.LinearContrast((0.6, 1.4)),\n    # Perform methods below only sometimes\n    iaa.Sometimes(0.5,\n        # 5. GaussianBlur\n        iaa.GaussianBlur((0.0, 3.0))\n        )\n])","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:10:17.003792Z","iopub.execute_input":"2023-02-02T10:10:17.004354Z","iopub.status.idle":"2023-02-02T10:10:17.019898Z","shell.execute_reply.started":"2023-02-02T10:10:17.004316Z","shell.execute_reply":"2023-02-02T10:10:17.019041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 3. Show Images\ncnt = 0\nwhile cnt<5:\n    augmented_images = augmentation(images=images)\n    for img in augmented_images:\n        cnt+=1\n        plt.imshow(img)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:10:17.021725Z","iopub.execute_input":"2023-02-02T10:10:17.022135Z","iopub.status.idle":"2023-02-02T10:10:19.456835Z","shell.execute_reply.started":"2023-02-02T10:10:17.022101Z","shell.execute_reply":"2023-02-02T10:10:19.455914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **CNN with data Augmentation - CATS VS DOGS** \n<img src=\"http://intelligencesquared.com/wp-content/uploads/3-10.png\" alt=\"Alternative text\" width=500px height=500px />","metadata":{}},{"cell_type":"markdown","source":"**import data**\n","metadata":{}},{"cell_type":"code","source":"URL = 'https://storage.googleapis.com/mledu-datasets/cats_and_dogs_filtered.zip'\npath_to_zip = tf.keras.utils.get_file('cats_and_dogs.zip', origin=URL, extract=True)\nPATH = os.path.join(os.path.dirname(path_to_zip), 'cats_and_dogs_filtered')","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:27:08.341696Z","iopub.execute_input":"2023-02-02T10:27:08.342467Z","iopub.status.idle":"2023-02-02T10:27:09.784095Z","shell.execute_reply.started":"2023-02-02T10:27:08.342427Z","shell.execute_reply":"2023-02-02T10:27:09.782984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dir = os.path.join(PATH, 'train')\nvalidation_dir = os.path.join(PATH, 'validation')","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:27:14.032992Z","iopub.execute_input":"2023-02-02T10:27:14.034264Z","iopub.status.idle":"2023-02-02T10:27:14.042989Z","shell.execute_reply.started":"2023-02-02T10:27:14.034220Z","shell.execute_reply":"2023-02-02T10:27:14.041136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Setup data inputs\nimport os\ntrain_dir ='./train/'\nfiles = []\nfor dirname, _, filenames in os.walk(train_dir):\n    for filename in filenames:\n        files.append(filename)\n\n#create directories to separate dogs and cats\ndog_dir = \"./train/dog/\"\ncat_dir = \"./train/cat/\"\n\nif not os.path.exists(dog_dir):\n    os.mkdir(dog_dir)\n    \nif not os.path.exists(cat_dir):\n    os.mkdir(cat_dir)\n\nclasses = [\"dog\", \"cat\"]\ntargets = []\n#create targets 1 for dog and 0 for cat\nfor file in files:\n    if \"dog\" in file:\n        targets.append(1)\n        #shutil.move(train_dir+file, dog_dir + file)\n    else:\n        targets.append(0)\n        #shutil.move(train_dir+file, cat_dir + file)\n","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:27:19.526580Z","iopub.execute_input":"2023-02-02T10:27:19.527806Z","iopub.status.idle":"2023-02-02T10:27:19.571493Z","shell.execute_reply.started":"2023-02-02T10:27:19.527750Z","shell.execute_reply":"2023-02-02T10:27:19.570524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMAGE_SHAPE = (32,32)\nBATCH_SIZE = 128\ntrain = tf.keras.preprocessing.image_dataset_from_directory(train_dir,\n                                          seed=123,\n                                          image_size=IMAGE_SHAPE,\n                                          batch_size=BATCH_SIZE)\n\nvalid_data = tf.keras.preprocessing.image_dataset_from_directory(validation_dir,\n                                              image_size=IMAGE_SHAPE,\n                                              batch_size=BATCH_SIZE,\n                                              seed=123,)","metadata":{"execution":{"iopub.status.busy":"2023-02-02T11:05:31.028364Z","iopub.execute_input":"2023-02-02T11:05:31.028788Z","iopub.status.idle":"2023-02-02T11:05:32.333464Z","shell.execute_reply.started":"2023-02-02T11:05:31.028751Z","shell.execute_reply":"2023-02-02T11:05:32.332164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**SOME DISPLAY**","metadata":{}},{"cell_type":"code","source":"class_names = train.class_names\nplt.figure(figsize=(10, 10))\nfor images, labels in train.take(1):\n    for i in range(5):\n        ax = plt.subplot(3, 3, i + 1)\n        plt.imshow(images[i].numpy().astype(\"uint8\"))\n        plt.title(class_names[labels[i]])\n        plt.axis(\"off\")","metadata":{"execution":{"iopub.status.busy":"2023-02-02T11:05:36.942334Z","iopub.execute_input":"2023-02-02T11:05:36.942737Z","iopub.status.idle":"2023-02-02T11:05:38.446329Z","shell.execute_reply.started":"2023-02-02T11:05:36.942703Z","shell.execute_reply":"2023-02-02T11:05:38.445309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**DATA AUGMENTATION**","metadata":{}},{"cell_type":"code","source":"def augment(image,label):\n    img = tf.cast(image, tf.float32)\n    img = tf.image.rot90(image) \n    img = tf.image.adjust_saturation(img, 3)\n    img = tf.image.adjust_brightness(img, 0.4) # Random brightness\n    return image,label\n","metadata":{"execution":{"iopub.status.busy":"2023-02-02T11:05:42.380849Z","iopub.execute_input":"2023-02-02T11:05:42.381303Z","iopub.status.idle":"2023-02-02T11:05:42.387699Z","shell.execute_reply.started":"2023-02-02T11:05:42.381268Z","shell.execute_reply":"2023-02-02T11:05:42.386076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prepare(ds, shuffle=False, augment=False):\n    batch_size = 32  \n    if shuffle:\n        ds = ds.shuffle(1000)\n        # Batch all datasets.\n        ds = ds.batch(batch_size)\n    # Use data augmentation only on the training set.\n    if augment:\n        ds = ds.map(augment, num_parallel_calls=tf.data.AUTOTUNE)\n        # Use buffered prefetching on all datasets.\n    return ds.prefetch(buffer_size=tf.data.AUTOTUNE)\n\ntrainDS = prepare(train,shuffle = False,augment = augment)\n","metadata":{"execution":{"iopub.status.busy":"2023-02-02T11:05:46.263767Z","iopub.execute_input":"2023-02-02T11:05:46.264880Z","iopub.status.idle":"2023-02-02T11:05:46.320960Z","shell.execute_reply.started":"2023-02-02T11:05:46.264844Z","shell.execute_reply":"2023-02-02T11:05:46.319974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainDS","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:28:08.270392Z","iopub.execute_input":"2023-02-02T10:28:08.270828Z","iopub.status.idle":"2023-02-02T10:28:08.280034Z","shell.execute_reply.started":"2023-02-02T10:28:08.270793Z","shell.execute_reply":"2023-02-02T10:28:08.278547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**PREPARE MODEL**","metadata":{}},{"cell_type":"code","source":"#create model\nmodel = Sequential()\nmodel.add(layers.Conv2D(256,(3,3),input_shape = (32,32,3), data_format='channels_last',padding='same'))\nmodel.add(layers.MaxPooling2D())\nmodel.add(layers.Conv2D(256,(3,3),activation=\"elu\",padding='same'))\nmodel.add(layers.MaxPooling2D())\nmodel.add(layers.Conv2D(256,(3,3),activation=\"elu\",padding='same'))\nmodel.add(layers.MaxPooling2D())\n\nmodel.add(layers.Flatten())\nmodel.add(layers.Dense(units=64,activation='elu'))\nmodel.add(layers.BatchNormalization())\nmodel.add(layers.Dropout(0.2))\nmodel.add(layers.Dense(units=64,activation='elu'))\nmodel.add(layers.BatchNormalization())\nmodel.add(layers.Dropout(0.2))\nmodel.add(layers.Dense(units=64,activation='elu'))\nmodel.add(layers.BatchNormalization())\nmodel.add(layers.Dropout(0.2))\n\nmodel.add(layers.Dense(units=32,activation='elu'))\nmodel.add(layers.Dense(units=10,activation='sigmoid'))\n#model.summary()","metadata":{"execution":{"iopub.status.busy":"2023-02-02T11:05:54.494909Z","iopub.execute_input":"2023-02-02T11:05:54.495342Z","iopub.status.idle":"2023-02-02T11:05:54.624087Z","shell.execute_reply.started":"2023-02-02T11:05:54.495306Z","shell.execute_reply":"2023-02-02T11:05:54.623074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**TRAIN MODEL**","metadata":{}},{"cell_type":"code","source":"#train\nwith tf.device('/gpu:0'):\n  model.compile(optimizer='adam',loss='sparse_categorical_crossentropy',metrics=['accuracy'])\n  history = model.fit(trainDS, epochs=10,validation_data = valid_data)","metadata":{"execution":{"iopub.status.busy":"2023-02-02T11:06:00.908325Z","iopub.execute_input":"2023-02-02T11:06:00.908759Z","iopub.status.idle":"2023-02-02T11:09:46.684799Z","shell.execute_reply.started":"2023-02-02T11:06:00.908724Z","shell.execute_reply":"2023-02-02T11:09:46.683578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**PLOT ACCURACY AND LOSS**","metadata":{}},{"cell_type":"code","source":"plot_history(history)\n","metadata":{"execution":{"iopub.status.busy":"2023-02-02T11:09:51.285964Z","iopub.execute_input":"2023-02-02T11:09:51.286362Z","iopub.status.idle":"2023-02-02T11:09:51.703746Z","shell.execute_reply.started":"2023-02-02T11:09:51.286329Z","shell.execute_reply":"2023-02-02T11:09:51.702633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **NLP WITH RNN**","metadata":{}},{"cell_type":"markdown","source":"**Data Preparation**","metadata":{}},{"cell_type":"code","source":"patent = pd.read_csv(\"/kaggle/input/patent-abstracts/neural_network_patent_query.csv\")\npatent.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:33:31.731489Z","iopub.execute_input":"2023-02-02T10:33:31.732575Z","iopub.status.idle":"2023-02-02T10:33:31.851712Z","shell.execute_reply.started":"2023-02-02T10:33:31.732534Z","shell.execute_reply":"2023-02-02T10:33:31.850762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Tokenize**\nTokenization is used in natural language processing to split paragraphs and sentences into smaller units that can be more easily assigned meaning.\n<img src=\"https://cdn.analyticsvidhya.com/wp-content/uploads/2019/11/tokenization.png\" alt=\"Alternative text\" width=500px height=500px />\n","metadata":{}},{"cell_type":"code","source":"tokenizer = Tokenizer(\n    num_words=None,\n    filters='#$%&()*+-<=>@[\\\\]^_`{|}~\\t\\n',\n    lower = False,\n    split=' '\n)\nabst = patent['patent_abstract'].values\ntokenizer.fit_on_texts(abst)\nsequences = tokenizer.texts_to_sequences(abst)\nsequences[0][:10]","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:33:36.834072Z","iopub.execute_input":"2023-02-02T10:33:36.834506Z","iopub.status.idle":"2023-02-02T10:33:37.409411Z","shell.execute_reply.started":"2023-02-02T10:33:36.834470Z","shell.execute_reply":"2023-02-02T10:33:37.408114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Give the network a sequence of words and train it to predict the next word**","metadata":{}},{"cell_type":"code","source":"features = []\nlabels = []\n\ntraining_length = 5\n\n# Iterate through the sequences of tokens\nfor seq in sequences:\n    for i in range(len(seq)-training_length):\n        extract = seq[i:i+training_length+1]\n        # Set the features and label\n        features.append(extract[:-1])\n        labels.append(extract[-1])\nfeatures = np.array(features)\nfeatures.shape","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:33:41.231344Z","iopub.execute_input":"2023-02-02T10:33:41.231775Z","iopub.status.idle":"2023-02-02T10:33:42.498668Z","shell.execute_reply.started":"2023-02-02T10:33:41.231736Z","shell.execute_reply":"2023-02-02T10:33:42.497268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_words = len(tokenizer.word_index)+1","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:33:43.908571Z","iopub.execute_input":"2023-02-02T10:33:43.909773Z","iopub.status.idle":"2023-02-02T10:33:43.915004Z","shell.execute_reply.started":"2023-02-02T10:33:43.909730Z","shell.execute_reply":"2023-02-02T10:33:43.913247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.layers import LSTM, Dense, Dropout, Masking, Embedding,Bidirectional","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:33:48.588321Z","iopub.execute_input":"2023-02-02T10:33:48.589460Z","iopub.status.idle":"2023-02-02T10:33:48.594241Z","shell.execute_reply.started":"2023-02-02T10:33:48.589418Z","shell.execute_reply":"2023-02-02T10:33:48.593010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"# download glove and unzip it in Notebook.\n!wget http://nlp.stanford.edu/data/glove.6B.zip\n!unzip glove*.zip\"\"\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def embedding_for_vocab(filepath, word_index,embedding_dim):\n    vocab_size = len(word_index) + 1\n    # Adding again 1 because of reserved 0 index\n    embedding_matrix_vocab = np.zeros((vocab_size,embedding_dim))\n    with open(filepath,encoding=\"utf8\") as f:\n        for line in f:\n            word, *vector = line.split()\n            if word in word_index:\n                idx = word_index[word]\n                embedding_matrix_vocab[idx] = np.array(vector, dtype=np.float32)[:embedding_dim]\n    return embedding_matrix_vocab\nembedding_matrix = embedding_for_vocab('./glove.6B.300d.txt', tokenizer.word_index,300)","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:33:50.621320Z","iopub.execute_input":"2023-02-02T10:33:50.621768Z","iopub.status.idle":"2023-02-02T10:34:00.645059Z","shell.execute_reply.started":"2023-02-02T10:33:50.621731Z","shell.execute_reply":"2023-02-02T10:34:00.643919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**RNN - LSTM layer**\nAn LSTM layer learns long-term dependencies between time steps in time series and sequence data. The state of the layer consists of the hidden state (also known as the output state) and the cell state. The hidden state at time step t contains the output of the LSTM layer for this time step.\n<img src=\"https://miro.medium.com/max/674/1*jikKbzFXCq-IYnFZankIMg.png\" alt=\"Alternative text\" width=500px height=500px />","metadata":{}},{"cell_type":"code","source":"num_words = len(tokenizer.word_index)+1\ndef embedding_for_vocab(filepath, word_index,embedding_dim):\n    vocab_size = len(word_index) + 1\n    # Adding again 1 because of reserved 0 index\n    embedding_matrix_vocab = np.zeros((vocab_size,embedding_dim))\n    with open(filepath,encoding=\"utf8\") as f:\n        for line in f:\n            word, *vector = line.split()\n            if word in word_index:\n                idx = word_index[word]\n                embedding_matrix_vocab[idx] = np.array(vector, dtype=np.float32)[:embedding_dim]\n    return embedding_matrix_vocab\nembedding_matrix = embedding_for_vocab('./glove.6B.300d.txt', tokenizer.word_index,300)\n","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:34:00.647360Z","iopub.execute_input":"2023-02-02T10:34:00.647771Z","iopub.status.idle":"2023-02-02T10:34:09.327074Z","shell.execute_reply.started":"2023-02-02T10:34:00.647733Z","shell.execute_reply":"2023-02-02T10:34:09.325819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Building a Recurrent Neural Network**","metadata":{}},{"cell_type":"code","source":"from keras.models import Sequential\nfrom tensorflow.compat.v1.keras.layers import CuDNNLSTM\nfrom keras.layers import LSTM, Dense, Dropout, Masking, Embedding\ndef create_nlp(embedding_matrix,num_words,training_length):\n    model = Sequential()\n    # Embedding layer\n\n    if embedding_matrix != []:\n        model.add(\n        Embedding(input_dim = num_words,\n                  input_length = training_length,\n                  output_dim=np.shape(embedding_matrix)[1],\n                  weights=[embedding_matrix],\n                  trainable=False,\n                  mask_zero=False))\n    else:\n        model.add(\n        Embedding(input_dim=num_words,\n                  input_length = training_length,\n                  output_dim=100,\n                  mask_zero=False))\n        \n\n    # Recurrent layer\n    model.add(CuDNNLSTM(4, return_sequences=False))\n    model.add(Dropout(0.02))\n    model.add(Dense(128, activation='elu'))\n\n    # Output layer\n    model.add(Dense(num_words, activation='softmax'))\n    return model\n\n","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:52:37.862505Z","iopub.execute_input":"2023-02-02T10:52:37.863790Z","iopub.status.idle":"2023-02-02T10:52:37.873687Z","shell.execute_reply.started":"2023-02-02T10:52:37.863736Z","shell.execute_reply":"2023-02-02T10:52:37.872633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train, x_val, y_train, y_val = train_test_split(features, np.array(labels), test_size = 0.2, random_state=43)","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:38:34.296656Z","iopub.execute_input":"2023-02-02T10:38:34.298095Z","iopub.status.idle":"2023-02-02T10:38:34.424783Z","shell.execute_reply.started":"2023-02-02T10:38:34.298045Z","shell.execute_reply":"2023-02-02T10:38:34.423603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = create_nlp(embedding_matrix,num_words,training_length)\n# Compile the model\nmodel.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])\nprint(model.summary())","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:52:42.984836Z","iopub.execute_input":"2023-02-02T10:52:42.985277Z","iopub.status.idle":"2023-02-02T10:52:49.208053Z","shell.execute_reply.started":"2023-02-02T10:52:42.985243Z","shell.execute_reply":"2023-02-02T10:52:49.206758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with tf.device('/gpu:1'):\n    history = model.fit(x_train,y_train,epochs=100,batch_size=5000,validation_data=(x_val,y_val))","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:52:55.825049Z","iopub.execute_input":"2023-02-02T10:52:55.825850Z","iopub.status.idle":"2023-02-02T11:03:49.479052Z","shell.execute_reply.started":"2023-02-02T10:52:55.825809Z","shell.execute_reply":"2023-02-02T11:03:49.478035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history(history)","metadata":{"execution":{"iopub.status.busy":"2023-02-02T11:04:06.667322Z","iopub.execute_input":"2023-02-02T11:04:06.667743Z","iopub.status.idle":"2023-02-02T11:04:07.082557Z","shell.execute_reply.started":"2023-02-02T11:04:06.667708Z","shell.execute_reply":"2023-02-02T11:04:07.081539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wd_idx = tokenizer.word_index\nidx = tokenizer.index_word\nx, y = x_train[25],y_train[25]\nphrase = ' '.join(idx[w] for w in x)\nphrase = 'This invention is a novel'\nimport sys,time\nsys.stdout.write(phrase)\nfor i in range(50):\n    u = phrase.split(' ')[i:i+training_length]\n    tmp = np.array([wd_idx[w] for w in u]).reshape((1,-1))\n    pred = idx[np.argmax(model.predict(tmp))]\n    phrase+=' '+pred\n    sys.stdout.write(' '+pred)\n    time.sleep(0.2)","metadata":{"execution":{"iopub.status.busy":"2023-02-02T11:04:40.205246Z","iopub.execute_input":"2023-02-02T11:04:40.205658Z","iopub.status.idle":"2023-02-02T11:04:52.365545Z","shell.execute_reply.started":"2023-02-02T11:04:40.205599Z","shell.execute_reply":"2023-02-02T11:04:52.364391Z"},"trusted":true},"execution_count":null,"outputs":[]}]}