{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-05-28T04:29:19.663227Z","iopub.execute_input":"2021-05-28T04:29:19.663592Z","iopub.status.idle":"2021-05-28T04:29:19.692861Z","shell.execute_reply.started":"2021-05-28T04:29:19.663514Z","shell.execute_reply":"2021-05-28T04:29:19.69202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/Kannada-MNIST/train.csv', index_col=False)\ntest = pd.read_csv('../input/Kannada-MNIST/test.csv', index_col=False)\nvalidate = pd.read_csv('../input/Kannada-MNIST/Dig-MNIST.csv', index_col = False)","metadata":{"execution":{"iopub.status.busy":"2021-05-28T04:29:19.694271Z","iopub.execute_input":"2021-05-28T04:29:19.694533Z","iopub.status.idle":"2021-05-28T04:29:24.853525Z","shell.execute_reply.started":"2021-05-28T04:29:19.694506Z","shell.execute_reply":"2021-05-28T04:29:24.85272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# copy the 3 datasets and separate X, y\n# Turn df into array and scale the X_data between 0 and 1\nX_train = train.copy().drop('label', axis = 1).values\nX_train = X_train.astype('float32') / 255.0\ny_train = train.label\n\nX_test = test.copy().drop('id', axis = 1).values\nX_test = X_test.astype('float32')/255.0\n\nX_valid = validate.copy().drop('label', axis = 1).values\nX_valid = X_valid.astype('float32')/255.0\ny_valid = validate.label\n\nprint('X_train data:{}'.format(X_train.shape))\nprint('y_train data:{}'.format(y_train.shape))\nprint('X_test data:{}'.format(X_test.shape))\nprint('X_valid data:{}'.format(X_valid.shape))\nprint('y_valid data:{}'.format(y_valid.shape))","metadata":{"execution":{"iopub.status.busy":"2021-05-28T04:29:24.855388Z","iopub.execute_input":"2021-05-28T04:29:24.855674Z","iopub.status.idle":"2021-05-28T04:29:25.284251Z","shell.execute_reply.started":"2021-05-28T04:29:24.855648Z","shell.execute_reply":"2021-05-28T04:29:25.283158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\npd.set_option('display.max_columns', 800)\nfrom datetime import datetime\nimport os\n\n# Convert all object features to category Features\nimport pandas.api.types as ptypes\n\n# Metrics for classification\nfrom sklearn.metrics import accuracy_score\n\n# Tensoreflow and keras packages\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import backend as K\nfrom tensorflow.keras.utils import to_categorical\n\nfrom tensorflow.keras import models\nfrom tensorflow.keras.models import Sequential, Model, load_model\nfrom tensorflow.keras.layers import Dense, BatchNormalization, Flatten, Reshape, Input,\\\n                                    Conv2D, MaxPooling2D, Dropout, UpSampling2D, ActivityRegularization\n\nfrom tensorflow.keras import optimizers\nfrom tensorflow.keras.optimizers import Adadelta, RMSprop,SGD,Adam\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras.initializers import RandomNormal, Constant\nfrom tensorflow.keras.callbacks import EarlyStopping\n\n# Visualization\nimport matplotlib as mpl\nimport matplotlib.pyplot as plt\n\nnp.random.seed(42)\ntf.random.set_seed(42)\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\n#To display the plots\n%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2021-05-28T04:29:25.286356Z","iopub.execute_input":"2021-05-28T04:29:25.286967Z","iopub.status.idle":"2021-05-28T04:29:30.672327Z","shell.execute_reply.started":"2021-05-28T04:29:25.286921Z","shell.execute_reply":"2021-05-28T04:29:30.671541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rounded_accuracy(y_true, y_pred):\n    return keras.metrics.binary_accuracy(tf.round(y_true), tf.round(y_pred))","metadata":{"execution":{"iopub.status.busy":"2021-05-28T04:29:30.673766Z","iopub.execute_input":"2021-05-28T04:29:30.674079Z","iopub.status.idle":"2021-05-28T04:29:30.679522Z","shell.execute_reply.started":"2021-05-28T04:29:30.674045Z","shell.execute_reply":"2021-05-28T04:29:30.678578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Model (2) - Denoising Autoencoder\n# Create an encoder\nkeras.backend.clear_session()\nstart = datetime.now()\ntf.random.set_seed(42)\nnp.random.seed(42)\n\ninput_layer = Input(shape = (784,))\n# Building Encoder\ne1=Dropout(0.5)(input_layer)\ne2=Dense(100, activation = 'selu')(e1)\ne3=Dense(30, activation = 'selu')(e2)\n\n# Building Decoder and Output\nd1 = Dense(30, activation = 'selu')(e3)\nd2 = Dense(100, activation = 'selu')(d1)\noutput = Dense(784,activation=\"sigmoid\")(d2)\n\ndenoise_ae = Model(inputs = input_layer, outputs = output)\n#optimizer = Adam(learning_rate = 0.0025)\ndenoise_ae.compile(loss=\"binary_crossentropy\", optimizer='rmsprop', metrics = [rounded_accuracy])\nhistory2 = denoise_ae.fit(X_train, X_train, epochs=20,\n                         validation_data=(X_valid, X_valid))\nend = datetime.now()\nruntime_2 = end - start\nprint('Denoising Autoencoder Runtime:', runtime_2)","metadata":{"execution":{"iopub.status.busy":"2021-05-28T04:29:30.681044Z","iopub.execute_input":"2021-05-28T04:29:30.681527Z","iopub.status.idle":"2021-05-28T04:31:13.911388Z","shell.execute_reply.started":"2021-05-28T04:29:30.68149Z","shell.execute_reply":"2021-05-28T04:31:13.91022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save and reload the encoder for evaluation and TSNE\ndenoising_encoder = Model(inputs = input_layer, outputs = e3)\n# save the encoder to file\ndenoising_encoder.save('denoising_encoder.h5')\n# Load the saved encoder back\ndenoising_encoder = load_model('denoising_encoder.h5')","metadata":{"execution":{"iopub.status.busy":"2021-05-28T04:31:13.912836Z","iopub.execute_input":"2021-05-28T04:31:13.913246Z","iopub.status.idle":"2021-05-28T04:31:13.967474Z","shell.execute_reply.started":"2021-05-28T04:31:13.913206Z","shell.execute_reply":"2021-05-28T04:31:13.966761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Using encoder to extract 30 important features\nX_train_encoded2 = denoising_encoder.predict(X_train)\nX_test_encoded2 = denoising_encoder.predict(X_test)\nX_valid_encoded2 = denoising_encoder.predict(X_valid)\nprint(X_test_encoded2.shape)","metadata":{"execution":{"iopub.status.busy":"2021-05-28T04:31:13.968761Z","iopub.execute_input":"2021-05-28T04:31:13.969109Z","iopub.status.idle":"2021-05-28T04:31:16.077913Z","shell.execute_reply.started":"2021-05-28T04:31:13.969073Z","shell.execute_reply":"2021-05-28T04:31:16.076886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Using Base Model encoder's output as feature extraction and dimensionality reduction input\n# for XGB Classifier to do prediction\nstart = datetime.now()\nimport xgboost as xgb\nfrom xgboost import XGBClassifier\nmodel2 = XGBClassifier(eval_metric='mlogloss') \nmodel2.fit(X_train_encoded2, y_train)\ny_pred2 = model2.predict(X_test_encoded2)\nend = datetime.now()\nruntime = end - start\nprint(runtime)","metadata":{"execution":{"iopub.status.busy":"2021-05-28T04:31:45.460015Z","iopub.execute_input":"2021-05-28T04:31:45.460342Z","iopub.status.idle":"2021-05-28T05:19:08.647969Z","shell.execute_reply.started":"2021-05-28T04:31:45.460312Z","shell.execute_reply":"2021-05-28T05:19:08.647031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Predict test data\nprint(y_pred2.shape)\nprint(y_pred2[:50])","metadata":{"execution":{"iopub.status.busy":"2021-05-28T05:22:40.310286Z","iopub.execute_input":"2021-05-28T05:22:40.310637Z","iopub.status.idle":"2021-05-28T05:22:40.318841Z","shell.execute_reply.started":"2021-05-28T05:22:40.310604Z","shell.execute_reply":"2021-05-28T05:22:40.317621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission = pd.read_csv('../input/Kannada-MNIST/sample_submission.csv')\nsubmission = sample_submission.copy()\nsubmission['label']=y_pred2\nsubmission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-05-28T05:22:44.300944Z","iopub.execute_input":"2021-05-28T05:22:44.301263Z","iopub.status.idle":"2021-05-28T05:22:44.330424Z","shell.execute_reply.started":"2021-05-28T05:22:44.301235Z","shell.execute_reply":"2021-05-28T05:22:44.329659Z"},"trusted":true},"execution_count":null,"outputs":[]}]}