{"cells":[{"metadata":{},"cell_type":"markdown","source":"## <center>1st Kaggle competition : digit recognizer</center>"},{"metadata":{},"cell_type":"markdown","source":"### Librairies "},{"metadata":{"trusted":true},"cell_type":"code","source":"import warnings \nwarnings.simplefilter(action='ignore', category=FutureWarning)\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix, accuracy_score, classification_report\nfrom scipy.stats import randint as sp_randint\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.model_selection import RandomizedSearchCV\nfrom sklearn.datasets import load_digits\nfrom sklearn.ensemble import RandomForestClassifier\nimport random\nfrom xgboost import XGBClassifier\nfrom time import time\nfrom keras.utils import np_utils, to_categorical\nfrom keras.layers.core import Dense, Activation, Dropout\nfrom keras.models import Sequential\nfrom keras.wrappers.scikit_learn import KerasClassifier\nfrom keras.constraints import maxnorm\nfrom keras.callbacks import ReduceLROnPlateau","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Import data"},{"metadata":{"trusted":true},"cell_type":"code","source":"data = pd.read_csv('../input/digit-recognizer/train.csv')\ntest = pd.read_csv('../input/digit-recognizer/test.csv')\ndata.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data exploration "},{"metadata":{},"cell_type":"markdown","source":"### Check if null"},{"metadata":{"trusted":true},"cell_type":"code","source":"data.isnull().any().sum()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> ### Data visualization "},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data=(data.drop(columns='label')).values\nlabeled_data_num=(data.label).values\nlabeled_data=to_categorical(labeled_data_num)\ntrain_data0 = train_data.reshape(train_data.shape[0], 28, 28)\nplt.figure(figsize=(13,13))\nfor i in range(5, 9):\n    plt.subplot(450 + (i+1))\n    plt.imshow(train_data0[i], cmap=plt.get_cmap('gray'))\n    plt.title(labeled_data_num[i])\n    plt.axis('off')\nplt.show","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Target distribution"},{"metadata":{"trusted":true},"cell_type":"code","source":"data.label.value_counts().sort_index().plot(kind='bar')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Machine learning "},{"metadata":{},"cell_type":"markdown","source":"### Random forest"},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(data.drop(columns='label'), data.label, test_size=0.2, random_state=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"rfc  = RandomForestClassifier(n_estimators = 300,criterion= 'gini', max_depth= 5, max_features=9)\nrfc.fit(X_train, y_train)\nrfc_pred = rfc.predict(X_test)\nprint('Accuracy of Random Forest model:', accuracy_score(y_test, rfc_pred))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### XGBoost "},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(data.drop(columns='label'), data.label, test_size=0.2, random_state=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xgb=XGBClassifier(objective='multi:softmax', num_class=10, \n        n_jobs=-1,booster=\"gbtree\",tree_method = \"hist\",\n        grow_policy = \"depthwise\")\nxgb.fit(X_train, y_train)\nxgb_pred = xgb.predict(X_test)\nprint('Accuracy of XGboost model:', accuracy_score(y_test, xgb_pred))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Deep learning : neural network "},{"metadata":{},"cell_type":"markdown","source":"### Pre-processing "},{"metadata":{"trusted":true},"cell_type":"code","source":"#Import and split data\nX_train = (data.drop(columns='label').values).astype('float32')\n#Preprocessing : converting labels into categorical\ny_train = np_utils.to_categorical(data.label.values.astype('int32'))\n#Preprocessing : normalization\nscale = np.max(X_train)\nX_train /= scale\nmean = np.std(X_train)\nX_train -= mean\n\ninput_dim = X_train.shape[1]\nnb_classes = y_train.shape[1]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Grid search for epochs and batch size"},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_model():\n\t# create model\n\tmodel = Sequential()\n\tmodel.add(Dense(128, input_dim=input_dim,kernel_initializer='he_uniform',kernel_constraint=maxnorm(1)))\n\tmodel.add(Activation('softplus'))\n\tmodel.add(Dropout(0.1))\n\tmodel.add(Dense(128, kernel_initializer='he_normal',kernel_constraint=maxnorm(2)))\n\tmodel.add(Activation('relu'))\n\tmodel.add(Dropout(0.1))\n\tmodel.add(Dense(nb_classes))\n\tmodel.add(Activation('softmax'))\n\tmodel.compile(loss='categorical_crossentropy', optimizer='Adamax', metrics=['accuracy'])\n\treturn model\nmodel = KerasClassifier(build_fn=create_model, verbose=0)\n# define the grid search parameters\nbatch_size = [20, 50, 100]\nepochs = [20, 50, 80, 100]\nparam_grid = dict(batch_size=batch_size, epochs=epochs)\ngrid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1, cv=3)\ngrid_result = grid.fit(X_train, y_train)\n# summarize results\nprint(\"Best batch size and epochs: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\nmeans = grid_result.cv_results_['mean_test_score']\nstds = grid_result.cv_results_['std_test_score']\nparams = grid_result.cv_results_['params']\nfor mean, stdev, param in zip(means, stds, params):\n    print(\"%f (%f) with: %r\" % (mean, stdev, param))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Grid search for optimizer"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Grid search for optimizer\ndef create_model(optimizer='adam'):\n\t# create model\n\tmodel = Sequential()\n\tmodel.add(Dense(128, input_dim=input_dim,kernel_initializer='he_uniform',\n                    kernel_constraint=maxnorm(2)))\n\tmodel.add(Activation('softplus'))\n\tmodel.add(Dropout(0.1))\n\tmodel.add(Dense(128,kernel_initializer='he_normal',kernel_constraint=maxnorm(1)))\n\tmodel.add(Activation('relu'))\n\tmodel.add(Dropout(0.1))\n\tmodel.add(Dense(nb_classes))\n\tmodel.add(Activation('softmax'))\n\t# Compile model\n\tmodel.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])\n\treturn model\nmodel = KerasClassifier(build_fn=create_model, epochs=100, batch_size=100, verbose=0)\n# define the grid search parameters\noptimizer = ['SGD', 'RMSprop', 'Adagrad', 'Adadelta', 'Adam', 'Adamax', 'Nadam']\nparam_grid = dict(optimizer=optimizer)\ngrid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1, cv=3)\ngrid_result = grid.fit(X_train, y_train)\n\n# summarize results\nprint(\"Best optimizer: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\nmeans = grid_result.cv_results_['mean_test_score']\nstds = grid_result.cv_results_['std_test_score']\nparams = grid_result.cv_results_['params']\nfor mean, stdev, param in zip(means, stds, params):\n    print(\"%f (%f) with: %r\" % (mean, stdev, param))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Grid search for activation"},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_model(activation='relu'):\n\t# create model\n\tmodel = Sequential()\n\tmodel.add(Dense(128, input_dim=input_dim,kernel_initializer='he_uniform',\n                kernel_constraint=maxnorm(2), activation=activation))\n\tmodel.add(Dropout(0.1))\n\tmodel.add(Dense(128,kernel_initializer='lecun_uniform',kernel_constraint=maxnorm(1)))\n\tmodel.add(Activation('relu'))\n\tmodel.add(Dropout(0.1))\n\tmodel.add(Dense(nb_classes))\n\tmodel.add(Activation('softplus'))\n\t# Compile model\n\tmodel.compile(loss='categorical_crossentropy', optimizer='Adamax', metrics=['accuracy'])\n\treturn model\nmodel = KerasClassifier(build_fn=create_model, epochs=100, batch_size=100, verbose=0)\n# define the grid search parameters\nactivation = ['softmax', 'softplus', 'softsign', 'relu', 'tanh', 'sigmoid', 'hard_sigmoid', 'linear']\nparam_grid = dict(activation=activation)\ngrid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1, cv=3)\ngrid_result = grid.fit(X_train, y_train)\n\n# summarize results\nprint(\"Best activation 1: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\nmeans = grid_result.cv_results_['mean_test_score']\nstds = grid_result.cv_results_['std_test_score']\nparams = grid_result.cv_results_['params']\nfor mean, stdev, param in zip(means, stds, params):\n    print(\"%f (%f) with: %r\" % (mean, stdev, param))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_model(activation='relu'):\n\t# create model\n\tmodel = Sequential()\n\tmodel.add(Dense(128, input_dim=input_dim,kernel_initializer='he_uniform',\n                kernel_constraint=maxnorm(2)))\n\tmodel.add(Activation('softplus'))\n\tmodel.add(Dropout(0.1))\n\tmodel.add(Dense(128,kernel_initializer='he_normal',kernel_constraint=maxnorm(1),\n                    activation=activation))\n\tmodel.add(Dropout(0.1))\n\tmodel.add(Dense(nb_classes))\n\tmodel.add(Activation('softmax'))\n\t# Compile model\n\tmodel.compile(loss='categorical_crossentropy', optimizer='Adamax', metrics=['accuracy'])\n\treturn model\nmodel = KerasClassifier(build_fn=create_model, epochs=100, batch_size=100, verbose=0)\n# define the grid search parameters\nactivation = ['softmax', 'softplus', 'softsign', 'relu', 'tanh', 'sigmoid', 'hard_sigmoid', 'linear']\nparam_grid = dict(activation=activation)\ngrid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1, cv=3)\ngrid_result = grid.fit(X_train, y_train)\n\n# summarize results\nprint(\"Best activation 2: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\nmeans = grid_result.cv_results_['mean_test_score']\nstds = grid_result.cv_results_['std_test_score']\nparams = grid_result.cv_results_['params']\nfor mean, stdev, param in zip(means, stds, params):\n    print(\"%f (%f) with: %r\" % (mean, stdev, param))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_model(activation='relu'):\n\t# create model\n\tmodel = Sequential()\n\tmodel.add(Dense(128, input_dim=input_dim,kernel_initializer='he_uniform',\n                kernel_constraint=maxnorm(2)))\n\tmodel.add(Activation('softplus'))\n\tmodel.add(Dropout(0.1))\n\tmodel.add(Dense(128,kernel_initializer='lecun_uniform',kernel_constraint=maxnorm(1)))\n\tmodel.add(Activation('relu'))\n\tmodel.add(Dropout(0.1))\n\tmodel.add(Dense(nb_classes,activation=activation))\n\t# Compile model\n\tmodel.compile(loss='categorical_crossentropy', optimizer='Adamax', metrics=['accuracy'])\n\treturn model\nmodel = KerasClassifier(build_fn=create_model, epochs=100, batch_size=100, verbose=0)\n# define the grid search parameters\nactivation = ['softmax', 'softplus', 'softsign', 'relu', 'tanh', 'sigmoid', 'hard_sigmoid', 'linear']\nparam_grid = dict(activation=activation)\ngrid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1, cv=3)\ngrid_result = grid.fit(X_train, y_train)\n\n# summarize results\nprint(\"Best activation 3: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\nmeans = grid_result.cv_results_['mean_test_score']\nstds = grid_result.cv_results_['std_test_score']\nparams = grid_result.cv_results_['params']\nfor mean, stdev, param in zip(means, stds, params):\n    print(\"%f (%f) with: %r\" % (mean, stdev, param))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Grid search for init mode "},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_model(init_mode='uniform'):\n\t# create model\n\tmodel = Sequential()\n\tmodel.add(Dense(128, input_dim=input_dim,kernel_initializer=init_mode,\n                kernel_constraint=maxnorm(2)))\n\tmodel.add(Activation('softplus'))\n\tmodel.add(Dropout(0.1))\n\tmodel.add(Dense(128,kernel_initializer='he_normal',kernel_constraint=maxnorm(1)))\n\tmodel.add(Activation('relu'))\n\tmodel.add(Dropout(0.1))\n\tmodel.add(Dense(nb_classes))\n\tmodel.add(Activation('softmax'))\n\t# Compile model\n\tmodel.compile(loss='categorical_crossentropy', optimizer='Adamax', metrics=['accuracy'])\n\treturn model\nmodel = KerasClassifier(build_fn=create_model, epochs=100, batch_size=100, verbose=0)\n# define the grid search parameters\ninit_mode = ['uniform', 'lecun_uniform', 'normal', 'zero', 'glorot_normal', 'glorot_uniform', 'he_normal', 'he_uniform']\nparam_grid = dict(init_mode=init_mode)\ngrid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1, cv=3)\ngrid_result = grid.fit(X_train, y_train)\n\n# summarize results\nprint(\"Best init mode: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\nmeans = grid_result.cv_results_['mean_test_score']\nstds = grid_result.cv_results_['std_test_score']\nparams = grid_result.cv_results_['params']\nfor mean, stdev, param in zip(means, stds, params):\n    print(\"%f (%f) with: %r\" % (mean, stdev, param))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_model(init_mode='uniform'):\n\t# create model\n\tmodel = Sequential()\n\tmodel.add(Dense(128, input_dim=input_dim,kernel_initializer='he_uniform',\n                kernel_constraint=maxnorm(2)))\n\tmodel.add(Activation('softplus'))\n\tmodel.add(Dropout(0.1))\n\tmodel.add(Dense(128,kernel_initializer=init_mode,kernel_constraint=maxnorm(1)))\n\tmodel.add(Activation('relu'))\n\tmodel.add(Dropout(0.1))\n\tmodel.add(Dense(nb_classes))\n\tmodel.add(Activation('softmax'))\n\t# Compile model\n\tmodel.compile(loss='categorical_crossentropy', optimizer='Adamax', metrics=['accuracy'])\n\treturn model\nmodel = KerasClassifier(build_fn=create_model, epochs=100, batch_size=100, verbose=0)\n# define the grid search parameters\ninit_mode = ['uniform', 'lecun_uniform', 'normal', 'zero', 'glorot_normal', 'glorot_uniform', 'he_normal', 'he_uniform']\nparam_grid = dict(init_mode=init_mode)\ngrid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1, cv=3)\ngrid_result = grid.fit(X_train, y_train)\n\n# summarize results\nprint(\"Best init mode: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\nmeans = grid_result.cv_results_['mean_test_score']\nstds = grid_result.cv_results_['std_test_score']\nparams = grid_result.cv_results_['params']\nfor mean, stdev, param in zip(means, stds, params):\n    print(\"%f (%f) with: %r\" % (mean, stdev, param))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Applying final model "},{"metadata":{"scrolled":false,"trusted":true},"cell_type":"code","source":"#Import and split data\nX_train = (data.drop(columns='label').values).astype('float32')\nX_test = (pd.read_csv('../input/digit-recognizer/test.csv').values).astype('float32')\n#Preprocessing : converting labels into categorical\ny_train = np_utils.to_categorical(data.label.values.astype('int32'))\n#Preprocessing : normalization\nscale = np.max(X_train)\nX_train /= scale\nX_test /= scale\nmean = np.std(X_train)\nX_train -= mean\nX_test -= mean\n\ninput_dim = X_train.shape[1]\nnb_classes = y_train.shape[1]\n#Model\ninput_dim = X_train.shape[1]\nnb_classes = y_train.shape[1]\nmodel = Sequential()\nmodel.add(Dense(128, input_dim=input_dim,kernel_initializer='he_uniform',kernel_constraint=maxnorm(1)))\nmodel.add(Activation('softplus'))\nmodel.add(Dropout(0.1))\nmodel.add(Dense(128, kernel_initializer='he_normal',kernel_constraint=maxnorm(2)))\nmodel.add(Activation('relu'))\nmodel.add(Dropout(0.1))\nmodel.add(Dense(nb_classes))\nmodel.add(Activation('softmax'))\nmodel.compile(loss='categorical_crossentropy', optimizer='Adamax', metrics=['accuracy'])\nprint(\"Training...\")\nmodel.fit(X_train, y_train, epochs=100, batch_size=100, validation_split=0.1, verbose=2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Generating predictions "},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Generating test predictions...\")\npreds = model.predict_classes(X_test, verbose=0)\n\ndef write_preds(preds, fname):\n    pd.DataFrame({\"ImageId\": list(range(1,len(preds)+1)), \"Label\": preds}).to_csv(fname, index=False, header=True)\n\nwrite_preds(preds, \"keras-nlp.csv\")\nprint(\"Predictions available\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.10"}},"nbformat":4,"nbformat_minor":1}