{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":65471,"databundleVersionId":7211464,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import IPython.display import display\nimport csv \nfrom PIL import image \nfrom scipy.ndimage import rotate \n%matplotlip inline\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training letters images and labels files\nletters_training_images_file_path = \"Datasets/Arabic - letters - classification Dataset CSV/training images.zip\"\nletters_training_labels_file_path = \"Datasets/Arabic - letters - classification Dataset CSV/training labels.zip\"\n# Testing letters images and labels files\nletters_testing_images_file_path = \"Datasets/Arabic - letters - classification Dataset CSV/testing images.zip\"\nletters_testing_labels_file_path = \"Datasets/Arabic - letters - classification Dataset CSV/testing labels.zip\"\n\n# Loading dataset into dataframes\ntraining_letters_images = pd.read_csv(letters_training_images_file_path, compression='zip', header=None)\ntraining_letters_labels = pd.read_csv(letters_training_labels_file_path, compression='zip', header=None)\ntesting_letters_images = pd.read_csv(letters_testing_images_file_path, compression='zip', header=None)\ntesting_letters_labels = pd.read_csv(letters_testing_labels_file_path, compression='zip', header=None)\n\n# print statistics about the dataset\nprint(\"There are %d training arabic letter .\" %training_letters_images.shape[0])\nprint(\"There are %d testing arabic letter .\" %testing_letters_images.shape[0])\ntraining_letters_images.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# Training digits images and labels files\ndigits_training_images_file_path = \"Arabic Handwritten Digits Dataset CSV/training images.zip\"\ndigits_training_labels_file_path = \"Arabic letters Digits Dataset CSV/training labels.zip\"\n# Testing digits images and labels files\ndigits_testing_images_file_path = \"Arabic letters Digits Dataset CSV/testing images.zip\"\ndigits_testing_labels_file_path = \"Arabic letters Digits Dataset CSV/testing labels.zip\"\n\n# Loading dataset into dataframes\ntraining_digits_images = pd.read_csv(digits_training_images_file_path, compression='zip', header=None)\ntraining_digits_labels = pd.read_csv(digits_training_labels_file_path, compression='zip', header=None)\ntesting_digits_images = pd.read_csv(digits_testing_images_file_path, compression='zip', header=None)\ntesting_digits_labels = pd.read_csv(digits_testing_labels_file_path, compression='zip', header=None)\n\n# print statistics about the dataset\nprint(\"There are %d training arabic digit images of 64x64 pixels.\" %training_digits_images.shape[0])\nprint(\"There are %d testing arabic digit images of 64x64 pixels.\" %testing_digits_images.shape[0])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def convert_values_to_image(image_values, display=False):\n  image_array = np.asarray(image_values)\n  image_array = image_array.reshape(64, 64).astype('uint8')\n  # The original dataset is reflected so we will flip it then rotate for a better view only.\n  image_array = np.flip(image_array, 0)\n  image_array = rotate(image_array, -90)\n  new_image = Image.fromarray(image_array)\n  if display == True:\n    new_image.show()\n  return new_image","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"convert_values_to_image(training_letters_images.loc[0], True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"convert_values_to_image(training_letters_images.loc[12], True)\n     ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_digits_images_scaled = training_digits_images.values.astype('float32')/255\ntraining_digits_labels = training_digits_labels.values.astype('int32')\ntesting_digits_images_scaled = testing_digits_images.values.astype('float32')/255\ntesting_digits_labels = testing_digits_labels.values.astype('int32')\n\ntraining_letters_images_scaled = training_letters_images.values.astype('float32')/255\ntraining_letters_labels = training_letters_labels.values.astype('int32')\ntesting_letters_images_scaled = testing_letters_images.values.astype('float32')/255\ntesting_letters_labels = testing_letters_labels.values.astype('int32')\n     \n\nprint(\"Training images of digits after scaling\")\nprint(training_digits_images_scaled.shape)\ntraining_digits_images_scaled[0:5]\n     ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Training images of letters after scaling\")\nprint(training_letters_images_scaled.shape)\ntraining_letters_images_scaled[0:5]\n     ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.utils import to_categorical\n\n# one hot encoding\n# number of classes = 10 (digits classes) + 28 (arabic alphabet classes)\nnumber_of_classes = 38\ntraining_letters_labels_encoded = to_categorical(training_letters_labels, num_classes=number_of_classes)\ntesting_letters_labels_encoded = to_categorical(testing_letters_labels, num_classes=number_of_classes)\ntraining_digits_labels_encoded = to_categorical(training_digits_labels, num_classes=number_of_classes)\ntesting_digits_labels_encoded = to_categorical(testing_digits_labels, num_classes=number_of_classes)\n     ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(training_digits_labels_encoded)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# reshape input digit images to 64x64x1\ntraining_digits_images_scaled = training_digits_images_scaled.reshape([-1, 64, 64, 1])\ntesting_digits_images_scaled = testing_digits_images_scaled.reshape([-1, 64, 64, 1])\n\n# reshape input letter images to 64x64x1\ntraining_letters_images_scaled = training_letters_images_scaled.reshape([-1, 64, 64, 1])\ntesting_letters_images_scaled = testing_letters_images_scaled.reshape([-1, 64, 64, 1])\n\nprint(training_digits_images_scaled.shape, training_digits_labels_encoded.shape, testing_digits_images_scaled.shape, testing_digits_labels_encoded.shape)\nprint(training_letters_images_scaled.shape, training_letters_labels_encoded.shape, testing_letters_images_scaled.shape, testing_letters_labels_encoded.shape)\n     \n    training_data_images = np.concatenate((training_digits_images_scaled, training_letters_images_scaled), axis=0) \ntraining_data_labels = np.concatenate((training_digits_labels_encoded, training_letters_labels_encoded), axis=0)\nprint(\"Total Training images are {} images of shape\".format(training_data_images.shape[0]))\nprint(training_data_images.shape, training_data_labels.shape)\n\n\ntesting_data_images = np.concatenate((testing_digits_images_scaled, testing_letters_images_scaled), axis=0) \ntesting_data_labels = np.concatenate((testing_digits_labels_encoded, testing_letters_labels_encoded), axis=0)\nprint(\"Total Testing images are {} images of shape\".format(testing_data_images.shape[0]))\nprint(testing_data_images.shape, testing_data_labels.shape)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.models import Sequential\nfrom keras.layers import Conv2D, MaxPooling2D, GlobalAveragePooling2D, BatchNormalization, Dropout, Dense\n\ndef create_model(optimizer='adam', kernel_initializer='he_normal', activation='relu'):\n  # create model\n  model = Sequential()\n  model.add(Conv2D(filters=16, kernel_size=3, padding='same', input_shape=(64, 64, 1), kernel_initializer=kernel_initializer, activation=activation))\n  model.add(BatchNormalization())\n  model.add(MaxPooling2D(pool_size=2))\n  model.add(Dropout(0.2))\n\n  model.add(Conv2D(filters=32, kernel_size=3, padding='same', kernel_initializer=kernel_initializer, activation=activation))\n  model.add(BatchNormalization())\n  model.add(MaxPooling2D(pool_size=2))\n  model.add(Dropout(0.2))\n\n  model.add(Conv2D(filters=64, kernel_size=3, padding='same', kernel_initializer=kernel_initializer, activation=activation))\n  model.add(BatchNormalization())\n  model.add(MaxPooling2D(pool_size=2))\n  model.add(Dropout(0.2))\n\n  model.add(Conv2D(filters=128, kernel_size=3, padding='same', kernel_initializer=kernel_initializer, activation=activation))\n  model.add(BatchNormalization())\n  model.add(MaxPooling2D(pool_size=2))\n  model.add(Dropout(0.2))\n  model.add(GlobalAveragePooling2D())\n  \n  #Fully connected final layer\nmodel.add(Dense(38, activation='softmax'))\n\n  # Compile model\n  model.compile(loss='categorical_crossentropy', metrics=['accuracy'], optimizer=optimizer)\n  return model\n     ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = create_model()\nmodel.summary()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://pypi.python.org/pypi/pydot\n!apt-get -qq install -y graphviz && pip install -q pydot\nimport pydot\n     from keras.utils import plot_model\n\nplot_model(model, to_file=\"model.png\", show_shapes=True)\nfrom IPython.display import Image as IPythonImage\ndisplay(IPythonImage('model.png'))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fix random seed for reproducibility\nseed = 7\nnp.random.seed(seed)\n\n# define the grid search parameters\noptimizer = ['RMSprop', 'Adam', 'Adagrad', 'Nadam']\nkernel_initializer = ['normal', 'uniform']\nactivation = ['relu', 'linear', 'tanh']\n\nparam_grid = dict(optimizer=optimizer, kernel_initializer=kernel_initializer, activation=activation)\n\n# count number of different parameters values combinations\nparameters_number = 1\nfor x in param_grid:\n  parameters_number = parameters_number * len(param_grid[x]) \nprint(\"Number of different parameter combinations = {}\".format(parameters_number))\n     epochs = 5\nbatch_size = 20 # 20 divides the training data samples\n\n#creating the models with different hyperparameters\nfor a,b,c in [(x,y,z) for x in optimizer for z in activation for y in kernel_initializer]:\n    params = {'optimizer' : a , 'kernel_initializer' : b , 'activation' : c}\n    print(params)\n    curr_model = create_model(a, b, c)\n    curr_model.fit(training_data_images, training_data_labels, \n                    validation_data=(testing_data_images, testing_data_labels),\n                    epochs=epochs, batch_size=batch_size, verbose=1)\n    print(\"=============================================================================\")\n     ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = create_model(optimizer='A', kernel_initializer='uniform', activation='relu')\n     from keras.callbacks import ModelCheckpoint  \n\n# using checkpoints to save model weights to be used later instead of training again on the same epochs.\ncheckpointer = ModelCheckpoint(filepath='weights.hdf5', verbose=1, save_best_only=True)\nhistory = model.fit(training_data_images, training_data_labels, \n                    validation_data=(testing_data_images, testing_data_labels),\n                    epochs=10, batch_size=20, verbose=1, callbacks=[checkpointer])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\ndef plot_loss_accuracy(history):\n  # Loss Curves\n  plt.figure(figsize=[8,6])\n  plt.plot(history.history['loss'],'r',linewidth=3.0)\n  plt.plot(history.history['val_loss'],'b',linewidth=3.0)\n  plt.legend(['Training loss', 'Validation Loss'],fontsize=18)\n  plt.xlabel('Epochs ',fontsize=16)\n  plt.ylabel('Loss',fontsize=16)\n  plt.title('Loss Curves',fontsize=16)\n\n  # Accuracy Curves\n  plt.figure(figsize=[8,6])\n  plt.plot(history.history['acc'],'r',linewidth=3.0)\n  plt.plot(history.history['val_acc'],'b',linewidth=3.0)\n  plt.legend(['Training Accuracy', 'Validation Accuracy'],fontsize=18)\n  plt.xlabel('Epochs ',fontsize=16)\n  plt.ylabel('Accuracy',fontsize=16)\n  plt.title('Accuracy Curves',fontsize=16) ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_loss_accuracy(history)\n     model.load_weights('weights.hdf5')\n    # Final evaluation of the model\nmetrics = model.evaluate(testing_data_images, testing_data_labels, verbose=1)\nprint(\"Test Accuracy: {}\".format(metrics[1]))\nprint(\"Test Loss: {}\".format(metrics[0]))\n     epochs = 20\nbatch_size = 20\n\ncheckpointer = ModelCheckpoint(filepath='weights.hdf5', verbose=1, save_best_only=True)\n\nhistory = model.fit(training_data_images, training_data_labels, \n                    validation_data=(testing_data_images, testing_data_labels),\n                    epochs=epochs, batch_size=batch_size, verbose=1, callbacks=[checkpointer])\n          \nmodel.load_weights('weights.hdf5')\nplot_loss_accuracy(history)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Final evaluation of the model\nmetrics = model.evaluate(testing_data_images, testing_data_labels, verbose=1)\nprint(\"Test Accuracy: {}\".format(metrics[1]))\nprint(\"Test Loss: {}\".format(metrics[0]))\n     ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.models import model_from_yaml\n     \n\nmodel_yaml = model.to_yaml()\nwith open(\"model.yaml\", \"w\") as yaml_file:\n    yaml_file.write(model_yaml)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# serialize weights to HDF5\nmodel.save_weights(\"model.h5\")\nprint(\"Saved model to disk\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load YAML and create model\nyaml_file = open('model.yaml', 'r')\nloaded_model_yaml = yaml_file.read()\nyaml_file.close()\nloaded_model = model_from_yaml(loaded_model_yaml)\n# load weights into new model\nloaded_model.load_weights(\"model.h5\")\nprint(\"Loaded model from disk\")\n \n# compile the loaded model\nloaded_model.compile(loss='binary_crossentropy', optimizer='A', metrics=['accuracy'])\n     ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"baseline_model = Sequential()\nbaseline_model.add(Conv2D(filters=16, kernel_size=3, padding='same', input_shape=(64, 64, 1), activation='relu')) # Input layer\nbaseline_model.add(GlobalAveragePooling2D())\nbaseline_model.add(Dense(38, activation = 'softmax')) # Output layer => output dimension = 38 as it is multi-class\n\n# Compile the baseline model\nbaseline_model.compile(loss='categorical_crossentropy', metrics=['accuracy'], optimizer='Adam')\n\n# Fit the baseline model with training dataset\nepochs = 5\nbatch_size = 20\n\nbaseline_model.fit(training_data_images, training_data_labels, \n                  validation_data=(testing_data_images, testing_data_labels),\n                  epochs=epochs, batch_size=batch_size, verbose=1)\n\n# Test the baseline model\nbaseline_metrics = baseline_model.evaluate(testing_data_images, testing_data_labels, verbose=1)\nprint(\"Baseline Model Test Accuracy: {}\".format(baseline_metrics[1]))\nprint(\"Baseline Model Test Loss: {}\".format(baseline_metrics[0]))\ndef get_predicted_classes(model, data, labels=None):\n  image_predictions = model.predict(data)\n  predicted_classes = np.argmax(image_predictions, axis=1)\n  true_classes = np.argmax(labels, axis=1)\n  return predicted_classes, true_classes\nfrom sklearn.metrics import classification_report\n\ndef get_classification_report(y_true, y_pred):\n  print(classification_report(y_true, y_pred))\n     y_pred, y_true = get_predicted_classes(model, testing_data_images, testing_data_labels)\nget_classification_report(y_true, y_pred)\n     y_pred, y_true = get_predicted_classes(baseline_model, testing_data_images, testing_data_labels)\nget_classification_report(y_true, y_pred)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_non_zero_index_from_one_hot_encoding(one_hot_encoding):\n  non_zero_index = np.where(one_hot_encoding == 1)[0]\n  assert(len(non_zero_index) == 1)\n  non_zero_index = non_zero_index[0]\n  return non_zero_index\n     \n\ndef convert_categorical_label_to_real_label(categorical_label):\n  real_labels = []\n  real_labels.extend([x for x in range(10)])\n  real_labels.extend(['أ', 'ب', 'ت', 'ث', 'ج', 'ح', 'خ', 'د', 'ذ', 'ر', 'ز', 'س', 'ش', 'ص', 'ض', 'ط', 'ظ', 'ع', 'غ', 'ف', 'ق', 'ك', 'ل', 'م', 'ن', 'ه', 'و', 'ى'])\n  return real_labels[categorical_label]\n     \n\ndef show_example(sample_index):\n  y_pred, y_true = get_predicted_classes(model, testing_data_images[[sample_index]], testing_data_labels[[sample_index]])\n  non_zero_index = get_non_zero_index_from_one_hot_encoding(testing_data_labels[sample_index])\n  y_true = y_true[0]\n  y_pred = y_pred[0]\n  assert y_true == non_zero_index\n  true_label = convert_categorical_label_to_real_label(y_true)\n  predicted_label = convert_categorical_label_to_real_label(y_pred)\n  print(\"The following image has the written character '{}' but the model predicted it as '{}'\".format(true_label, predicted_label))\n  return true_label == predicted_label","metadata":{},"execution_count":null,"outputs":[]}]}