{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport random\nimport pandas as pd\nimport numpy as np\nimport datetime\nfrom datetime import datetime\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPooling2D\nfrom keras.utils.np_utils import to_categorical\nfrom keras.utils import np_utils\n\n\nfrom sklearn.model_selection import train_test_split, cross_val_score,GridSearchCV\nfrom sklearn.metrics import confusion_matrix\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"RANDOM_SEED = 1\nSET_FIT_INTERCEPT = True\n\nos.environ['PYTHONHASHSEED'] = str(RANDOM_SEED)\nrandom.seed(RANDOM_SEED)\n\nnp.random.seed(RANDOM_SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/train.csv\")\ntest = pd.read_csv(\"../input/test.csv\")\n\ntrain_data = np.array(train)\ntest_data = np.array(test)\n\n# drop the label (target) of train dataset\nX_train_data = (train.values[:,1:]).astype(np.float32)\n\n# get the label (target) of train dataset \ny_train_data = (train.values[:,0]).astype(np.int32)\n\n# test dataset does not contain a label\nX_test_data = (test.values).astype(np.float32) \n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_test, y_train, y_test = train_test_split(X_train_data, y_train_data, test_size = .2, random_state=RANDOM_SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = X_train.reshape(-1, 28, 28, 1)\nX_test = X_test.reshape(-1, 28, 28, 1)\nprint(X_train.shape, y_train.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = X_train.astype('float32')/255\nX_test = X_test.astype('float32')/255 ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Integer-valued labels:')\nprint(y_train[:10])\n\n# one-hot encode the labels\ny_train = np_utils.to_categorical(y_train, 10)\ny_test = np_utils.to_categorical(y_test, 10)\n\n# print first ten (one-hot) training labels\nprint('One-hot labels:')\nprint(y_train[:10])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# examine the shape of the loaded data\nprint('train_data shape:', X_train.shape)\nprint('test_data shape:', X_test.shape, '\\n')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# X_train = tf.keras.utils.normalize(X_train, axis = 1)\n# X_test = tf.keras.utils.normalize(X_test, axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data = test_data.reshape(-1,28,28,1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('response count\\n', train.label.value_counts())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_plot = train.drop('label',axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(14,12))\nfor digit_num in range(0,70):\n    plt.subplot(7,10,digit_num+1)\n    grid_data = train_plot.iloc[digit_num].as_matrix().reshape(28,28)  # reshape from 1d to 2d pixel array\n    plt.imshow(grid_data, interpolation = \"none\", cmap = \"afmhot\")\n    plt.xticks([])\n    plt.yticks([])\nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"optimizers = ['Adamax']\nnum_nodes = [256]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import ParameterGrid\nparameters = {\n'optimizer': optimizers,\n'num_nodes' : num_nodes\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"parameterList = list(ParameterGrid(parameters))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_nodes_list = []\noptimizer_list = []\ntrain_acc_list = []\ntest_acc_list = []\nproc_list = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.callbacks import ModelCheckpoint   \nfrom keras.callbacks import ReduceLROnPlateau\n\n# train the model\n#checkpointer = ModelCheckpoint(filepath='mnist.model.best.hdf5', \n#                               verbose=1, save_best_only=True)\nreduce_lr = ReduceLROnPlateau(monitor='loss', factor=0.3, verbose=1,\n                              patience=2, min_lr=0.00000001)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"optim = parameterList[0]['optimizer']\nnumNodes = parameterList[0]['num_nodes']\nprint(\"Optimizer: \", optim)\nprint(\"Number of Nodes: \", numNodes)\n\nmodel1 = Sequential([\n    Conv2D(filters=64, kernel_size=3, padding='same', activation='relu',input_shape=(28,28,1)),\n    Conv2D(filters=64, kernel_size=3, padding='same', activation='relu'),\n    Conv2D(filters=128, kernel_size=3, padding='same', activation='relu'),\n    MaxPooling2D(pool_size=2),\n    Conv2D(filters=128, kernel_size=3, padding='same', activation='relu'),\n    Conv2D(filters=192, kernel_size=3, padding='same', activation='relu'),\n    MaxPooling2D(pool_size=2),\n    Conv2D(filters=192, kernel_size=5, padding='same', activation='relu'),\n    MaxPooling2D(pool_size=2, padding='same'),\n    Flatten(),\n    Dense(numNodes, activation='relu'),\n    Dense(10, activation='softmax'),\n    ])\n\nmodel1.compile(\n    optimizer=optim,\n    loss='categorical_crossentropy',\n    metrics=['accuracy'],\n    )\n\nstart = datetime.now()\nmodel1.fit(X_train, y_train, batch_size=100, epochs=50,validation_data=(X_test, y_test), callbacks=[reduce_lr],verbose=1, shuffle=True)\nend = datetime.now()\nprocessing_time = end-start\n\nmodel1.summary()\n\nscore_train,acc_train = model1.evaluate(X_train,y_train)\nscore_test,acc_test = model1.evaluate(X_test,y_test)\n\nprint(\"Training Accuracy: \", acc_train)\nprint(\"Test Accuracy: \", acc_test)\nprint(\"Processing Time: \", processing_time)\nnum_nodes_list.append(numNodes)\noptimizer_list.append(optim)\ntrain_acc_list.append(acc_train)\ntest_acc_list.append(acc_test)\nproc_list.append(processing_time)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"performance_df = pd.DataFrame(columns = [\"Combination\", \n                                         \"Optimizer\",\n                                         \"Number of Nodes\",\n                                         \"Processing Time\", \n                                         \"Train Accuracy\", \n                                         \"Test Accuracy\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"performance_df['Combination'] = ParameterGrid(parameters)\nperformance_df['Optimizer'] = optimizer_list\nperformance_df['Number of Nodes'] = num_nodes_list\nperformance_df['Processing Time'] = proc_list\nperformance_df['Train Accuracy'] = train_acc_list\nperformance_df['Test Accuracy'] =  test_acc_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"performance_df = performance_df.sort_values(by='Test Accuracy',ascending=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"performance_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results = model1.predict_classes(test_data,verbose=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results = pd.Series(results,name=\"Label\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.savetxt('results.csv',\n           np.c_[range(1,len(test)+1),results],\n           delimiter=',',\n           header='ImageId,Label',\n           comments= '',\n           fmt = '%d'\n          )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}