{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"32b05b55ff6c09cb46fb28e1639a6f3ccff10bfc"},"cell_type":"code","source":"# Load require libraries\n\n# Plotting\nimport matplotlib.image as mpimg\nimport matplotlib.pyplot as plt\nimport matplotlib\n%matplotlib inline\n\n# Neural Network","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"markdown","source":"### Load dataset"},{"metadata":{"trusted":true,"_uuid":"64647d83e831e3dde55182f33c31bf9287684c70"},"cell_type":"code","source":"dataTrainRaw = pd.read_csv( \"../input/train.csv\" )\ndataTestRaw  = pd.read_csv( \"../input/test.csv\" )\n\ndataTrainFeatRaw = (dataTrainRaw.iloc[:,1:].values).astype( 'float32' )\ndataTrainLabels  = dataTrainRaw.iloc[:,0].values.astype('int32')\ndataTestFeatRaw  = dataTestRaw.values.astype('float32')\n\nprint( \"Train Features : \", dataTrainFeatRaw.shape )\nprint( \"Train Labels   : \", dataTrainLabels.shape )\nprint( \"Test Features  : \", dataTestFeatRaw.shape )","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e9148355668a37b76aa9379fe157e9b4044c1a0e"},"cell_type":"markdown","source":"### Create a Cross validation test"},{"metadata":{"trusted":true,"_uuid":"72731d41166992c85a2da91b09dfe81955713284"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\ndataCvTrainFeatRaw, dataCvTestFeatRaw, dataCvTrainLabels, dataCvTestLabels = train_test_split( dataTrainFeatRaw, dataTrainLabels, test_size = 0.10 )\n\nprint( \"Cross Validation: \" )\nprint( \"    Train Features : \", dataCvTrainFeatRaw.shape )\nprint( \"    Train Labels   : \", dataCvTrainLabels.shape )\nprint( \"    Test Features  : \", dataCvTestFeatRaw.shape )\nprint( \"    Test Labels    : \", dataCvTestLabels.shape )\n\nfrom keras.utils import to_categorical\ndataCvTrainOnehot = to_categorical( dataCvTrainLabels, num_classes = 10 )\ndataCvTestOnehot  = to_categorical( dataCvTestLabels, num_classes = 10 )\ndataTrainOnehot   = to_categorical( dataTrainLabels, num_classes = 10 )","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1a38b88930bed7aeb07ce7c39e05973d59be56ea"},"cell_type":"markdown","source":"### Feature Enginnering 1: Normalize data"},{"metadata":{"trusted":true,"_uuid":"2b2fe65543fdc49b6b1c2f8261c93859d231eb3c"},"cell_type":"code","source":"# Feature Engineering\nfrom sklearn.preprocessing import StandardScaler\n\ndataStandardizer = StandardScaler()\ndataStandardizer.fit( dataTrainFeatRaw )\n\ndataCvTrainFeatStd = dataStandardizer.transform( dataCvTrainFeatRaw )\ndataCvTestFeatStd  = dataStandardizer.transform( dataCvTestFeatRaw )\n\nprint( \"Before standardization: \" )\nprint( \"    Train:        Mean = {:+9.6f}    SD = {:9.6f}\".format(np.mean(dataCvTrainFeatRaw),np.std(dataCvTrainFeatRaw)) )\nprint( \"    Test :        Mean = {:+9.6f}    SD = {:9.6f}\".format(np.mean(dataCvTestFeatRaw),np.std(dataCvTestFeatRaw)) )\nprint( \"After standardization: \" )\nprint( \"    Train:        Mean = {:+9.6f}    SD = {:9.6f}\".format(np.mean(dataCvTrainFeatStd),np.std(dataCvTrainFeatStd)) )\nprint( \"    Test :        Mean = {:+9.6f}    SD = {:9.6f}\".format(np.mean(dataCvTestFeatStd),np.std(dataCvTestFeatStd)) )","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1ce4b5531c47aac3abbd9f2d0dfe1aff822b655b"},"cell_type":"markdown","source":"### Deep learning: Make some utility functions"},{"metadata":{"trusted":true,"_uuid":"4f12d17e1d7c3b440acb32fffd26d6dfd7fa2ef3"},"cell_type":"code","source":"from keras.models import Sequential\nfrom keras.layers import Dense\nfrom keras.optimizers import Adam ,RMSprop\n\n# Utility function to create models\ndef make_model( inDim, outDim, hDims = [] ):\n    model = Sequential()\n    if not hDims:\n        model.add( Dense( outDim, input_dim = inDim, activation = 'softmax') )\n    else:\n        model.add( Dense( hDims[0], input_dim = inDim, activation = \"relu\" ) )\n        for i in range(1,len(hDims)):\n            model.add( Dense( hDims[i], activation = \"relu\" ) )\n        model.add( Dense( outDim, activation = \"softmax\") )\n    model.compile( optimizer = \"Adam\", loss = \"categorical_crossentropy\", metrics = [\"accuracy\"] )\n    \n    return model\n\n# For training models and ploting results\ndef train_model( model, trainX, trainY, valX, valY, epochs = 50, verbose = 2, batch_size = 32 ):\n#     print( \"Train    X: \", trainX.shape, \"  Y: \", trainY.shape )\n#     print( \"Train    X: \", valX.shape, \"  Y: \", valY.shape )\n    valData = (valX,valY)\n    if valX is None:\n        valData = None\n    trainInfo = model.fit( trainX, trainY, validation_data = valData, epochs = epochs, verbose = verbose, batch_size = batch_size )\n    \n    # Plot Results\n    trainHistory = trainInfo.history\n    trainHistory.keys()\n\n    fig, ax = plt.subplots( 1, 2, figsize = (12,4) )\n\n    ax[0].plot( trainHistory['loss'], label = \"Train\" )\n    if valX is not None:\n        ax[0].plot( trainHistory['val_loss'], label = \"Validation\" )\n    ax[0].set_title( \"Loss\" )\n    ax[0].set_xlabel( \"Epochs\" )\n    ax[0].set_ylabel( \"Loss\" )\n    ax[0].grid( True )\n    ax[0].legend()\n\n    ax[1].plot( trainHistory['acc'], label = \"Train\" )\n    if valX is not None:\n        ax[1].plot( trainHistory['val_acc'], label = \"Validation\" )\n    ax[1].set_title( \"Accuracy\" )\n    ax[1].set_xlabel( \"Epochs\" )\n    ax[1].set_ylabel( \"Accuracy\" )\n    ax[1].grid( True )\n    ax[1].legend()\n    \n    return trainInfo","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"65fa8bde1f1b0da9b2b5abb528f9cae8d1db1a68"},"cell_type":"markdown","source":"### Baseline Model: A simple DNN"},{"metadata":{"trusted":true,"_uuid":"4dae32b530d02e42e807457e22dedb61c7c93d37"},"cell_type":"code","source":"baselineModel1 = make_model( dataCvTrainFeatRaw.shape[1], 10, [32,32] )\ntrain_model( baselineModel1, dataCvTrainFeatRaw, dataCvTrainOnehot, dataCvTestFeatRaw, dataCvTestOnehot )","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f0217f00299030e93d52fff29f03eda4baf7d984"},"cell_type":"markdown","source":"### Baseline Model: A little bit deeper DNN"},{"metadata":{"trusted":true,"_uuid":"473326c821b9a117c84d92244519fd9ecbb53b9d"},"cell_type":"code","source":"baselineModel2 = make_model( dataCvTrainFeatRaw.shape[1], 10, [64,64,32,32] )\ntrain_model( baselineModel2, dataCvTrainFeatRaw, dataCvTrainOnehot, dataCvTestFeatRaw, dataCvTestOnehot )","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0aff99bcbf820d52fdc2e23ff2885a2bba3d0c6d"},"cell_type":"markdown","source":"### Feature Enginnering 1: Deep Learning with Data Normalization"},{"metadata":{"_uuid":"404243899fe56af3b77886f0ced451fbc7bfd002"},"cell_type":"markdown","source":"#### Small Model"},{"metadata":{"trusted":true,"_uuid":"198245c5764662520726286e25229df1f41efca4"},"cell_type":"code","source":"normModel1 = make_model( dataCvTrainFeatStd.shape[1], 10, [32,32] )\ntrain_model( normModel1, dataCvTrainFeatStd, dataCvTrainOnehot, dataCvTestFeatStd, dataCvTestOnehot )","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"34d3c4304ec037c282c516991e5fa7c48e20de3a"},"cell_type":"markdown","source":"#### Deep model"},{"metadata":{"trusted":true,"_uuid":"e07afb1648373d35a678a3d1203c47d626a4528d"},"cell_type":"code","source":"normModel2 = make_model( dataCvTrainFeatStd.shape[1], 10, [64,64,32,32] )\ntrain_model( normModel2, dataCvTrainFeatStd, dataCvTrainOnehot, dataCvTestFeatStd, dataCvTestOnehot )","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f002c2fd24728026b3659a64292a42970ee790eb"},"cell_type":"markdown","source":"### Feature Engineering 2: LDA"},{"metadata":{"trusted":true,"_uuid":"f497f3090476a7ae7bc69c35a37b987f28547c45"},"cell_type":"code","source":"from sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA\n# from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis as QDA\n\nlda = LDA()\nlda.fit( dataCvTrainFeatStd, dataCvTrainLabels )\n\ndataCvTrainFeatLDA = lda.transform( dataCvTrainFeatStd )\ndataCvTestFeatLDA  = lda.transform( dataCvTestFeatStd )\n\nprint( \"Shape after LDA:    Train = \", dataCvTrainFeatLDA.shape, \"    Test = \", dataCvTestFeatLDA.shape )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cf824cd3f39ccc67791262462fb735d550105e57"},"cell_type":"markdown","source":"#### Small Model"},{"metadata":{"trusted":true,"_uuid":"61bab082de28f79129563ebfd17eb3ad6372509d"},"cell_type":"code","source":"ldaModel1 = make_model( dataCvTrainFeatLDA.shape[1], 10, [32,32] )\ntrain_model( ldaModel1, dataCvTrainFeatLDA, dataCvTrainOnehot, dataCvTestFeatLDA, dataCvTestOnehot )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"43ed760bbbb10ba9fb3ae629afc2861786a58b88"},"cell_type":"code","source":"ldaModel2 = make_model( dataCvTrainFeatLDA.shape[1], 10, [64,64,32,32] )\ntrain_model( ldaModel2, dataCvTrainFeatLDA, dataCvTrainOnehot, dataCvTestFeatLDA, dataCvTestOnehot )","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"83033a77e668362a574c3a5685737784b022dbab"},"cell_type":"markdown","source":"### Comparing the performance of different models"},{"metadata":{"trusted":true,"_uuid":"f0f3a59a420f9616c786891939de448e95c604b8"},"cell_type":"code","source":"from sklearn.metrics import accuracy_score\n\nacc_baselineModel1 = accuracy_score( dataCvTestLabels, np.argmax(baselineModel1.predict(dataCvTestFeatRaw),1) );\nacc_baselineModel2 = accuracy_score( dataCvTestLabels, np.argmax(baselineModel2.predict(dataCvTestFeatRaw),1) );\n\nacc_normModel1     = accuracy_score( dataCvTestLabels, np.argmax(normModel1.predict(dataCvTestFeatStd),1) );\nacc_normModel2     = accuracy_score( dataCvTestLabels, np.argmax(normModel2.predict(dataCvTestFeatStd),1) );\n\nacc_ldaModel1      = accuracy_score( dataCvTestLabels, np.argmax(ldaModel1.predict(dataCvTestFeatLDA),1) );\nacc_ldaModel2      = accuracy_score( dataCvTestLabels, np.argmax(ldaModel2.predict(dataCvTestFeatLDA),1) );\n\nacc_ldaPlain       = accuracy_score( dataCvTestLabels, lda.predict(dataCvTestFeatStd) )\n\n\nprint( \"Results\" )\nprint( \"Baseline Models: \" )\nprint( \"    Small Network    = \", acc_baselineModel1 )\nprint( \"    Large Network    = \", acc_baselineModel2 )\nprint( \"Models with data normalization: \" )\nprint( \"    Small Network    = \", acc_normModel1 )\nprint( \"    Large Network    = \", acc_normModel2 )\nprint( \"LDA based Models: \" )\nprint( \"    Small Network    = \", acc_ldaModel1 )\nprint( \"    Large Network    = \", acc_ldaModel2 )\nprint( \"Plain LDA            = \", acc_ldaPlain )\n\n\ndata = [ acc_baselineModel1, acc_baselineModel2, acc_normModel1, acc_normModel2, acc_ldaModel1, acc_ldaModel2, acc_ldaPlain ]\nfig, ax = plt.subplots( figsize = (12,6) )\nmodelLabels = ( \"Baseline: Small\", \"Baseline: Large\", \"Norm: Small\", \"Norm: Large\", \"LDA: Small\", \"LDA: Large\", \"LDA: Plain\" )\nax.barh( np.arange(len(data)), data )\nax.set_yticks( np.arange(len(data)) )\nax.set_yticklabels( modelLabels )\nax.set_xticks( np.arange(0.0,1.05,0.1) )\nax.invert_yaxis()\nax.set_xlabel( \"Accuracy\" )\nax.set_title( \"Accuracy of Models\" )\nax.grid()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c813aa8210aa34cb9e87e5d8bf08fbc0bd3b6a2c"},"cell_type":"markdown","source":"### Combining with deep CNN"},{"metadata":{"trusted":true,"_uuid":"dcb2ae9f7f2e2b5b19dcea983af2f73dee948612"},"cell_type":"code","source":"from keras.models import Sequential, Model\nfrom keras.layers import Dense, Dropout, Flatten, Dense, BatchNormalization, Input, Concatenate\nfrom keras.optimizers import Adam\nfrom keras.layers import Convolution2D, MaxPooling2D, Reshape, Lambda","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2e3e1e1c1611b361217eca5f7e64302093c37fbb"},"cell_type":"code","source":"def make_cnn_model():\n#     model = Sequential( [\n#         Lambda( lambda x: (x-dataStandardizer.mean_)/(dataStandardizer.scale_), \n#                            input_shape = (784,), output_shape = (784,) ),\n#         Reshape( (28,28,1) ),\n#         Convolution2D( 32, (5,5), activation = 'relu' ),\n#         BatchNormalization( axis = 1 ),\n#         Convolution2D( 32, (5,5), activation = 'relu' ),\n#         MaxPooling2D(),\n#         BatchNormalization( axis = 1 ),\n#         Flatten(),\n#         Dense( 128, activation = 'tanh' ),\n#         Dense( 10, activation = 'softmax' )\n#     ])\n    \n    inputImg = Input( shape = (784,) )\n    inputLda = Input( shape = (9,) )\n    \n    imgBranch = Lambda( lambda x:(x-dataStandardizer.mean_)/(dataStandardizer.scale_), input_shape=(784,), output_shape=(784,) )(inputImg)\n    imgBranch = Reshape( (28,28,1) )(imgBranch)\n    imgBranch = Convolution2D( 32, (5,5), activation = 'relu' )(imgBranch)\n    imgBranch = BatchNormalization( axis = 1 )(imgBranch)\n    imgBranch = Convolution2D( 32, (5,5), activation = 'relu' )(imgBranch)\n    imgBranch = MaxPooling2D()(imgBranch)\n    imgBranch = BatchNormalization( axis = 1 )(imgBranch)\n    imgBranch = Convolution2D( 64, (3,3), activation = 'relu' )(imgBranch)\n    imgBranch = BatchNormalization( axis = 1 )(imgBranch)\n    imgBranch = Convolution2D( 64, (3,3), activation = 'relu' )(imgBranch)\n    imgBranch = MaxPooling2D()(imgBranch)\n    imgBranch = BatchNormalization( axis = 1 )(imgBranch)\n    \n    imgBranch = Flatten()(imgBranch)\n    \n    merged10 = Concatenate()( [imgBranch,inputLda] )\n    merged11 = Dense( 512, activation = 'softmax' )(merged10)\n    merged12 = Dense( 512, activation = 'relu' )(merged10)\n    merged13 = Dense( 512, activation = 'tanh' )(merged10)\n    \n    merged20 = Concatenate()( [merged11,merged12,merged13,inputLda] )\n    merged20 = BatchNormalization()(merged20)\n    merged21 = Dense( 128, activation = 'softmax' )(merged20)\n    merged22 = Dense( 128, activation = 'relu' )(merged20)\n    merged23 = Dense( 128, activation = 'tanh' )(merged20)\n    \n    merged30 = Concatenate()( [merged21,merged22,merged23,inputLda] )\n    merged30 = BatchNormalization()(merged30)\n    merged31 = Dense( 128 )(merged30)\n    \n    output = Dense( 10, activation = 'softmax' )(merged30)\n    \n    model = Model( inputs = [inputImg,inputLda], outputs = output )\n    \n    model.compile( Adam(), loss = 'categorical_crossentropy', metrics = ['accuracy'] )\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a52632459820e1db65e98bfe8985c165db090e50"},"cell_type":"code","source":"# dataStandardizer.mean_\ncnnModel = make_cnn_model()\ncnnModel.summary()\n# train_model( cnnModel, dataCvTrainFeatRaw, dataCvTrainOnehot, dataCvTestFeatRaw, dataCvTestOnehot, epochs = 5, verbose = 1 )\ntrain_model( cnnModel, [ dataCvTrainFeatRaw, dataCvTrainFeatLDA], dataCvTrainOnehot, \n                       [ dataCvTestFeatRaw, dataCvTestFeatLDA], dataCvTestOnehot, epochs = 10, verbose = 1 )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bcf7405c4ecfda67239c04ad490162a33228113f"},"cell_type":"code","source":"# from sklearn.metrics import accuracy_score\n# acc_cnn            = accuracy_score( dataCvTestLabels, np.argmax(cnnModel.predict([dataCvTestFeatRaw,dataCvTestFeatLDA])) )","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"43a2b76a951f68936adf4f9d229b4fda7cd657cd"},"cell_type":"markdown","source":"### Compare all models"},{"metadata":{"trusted":true,"_uuid":"3bc5b5446734aa1468bb1d0b88f8c7ca0cb03074"},"cell_type":"code","source":"from sklearn.metrics import accuracy_score\n\nacc_baselineModel1 = accuracy_score( dataCvTestLabels, np.argmax(baselineModel1.predict(dataCvTestFeatRaw),1) );\nacc_baselineModel2 = accuracy_score( dataCvTestLabels, np.argmax(baselineModel2.predict(dataCvTestFeatRaw),1) );\n\nacc_normModel1     = accuracy_score( dataCvTestLabels, np.argmax(normModel1.predict(dataCvTestFeatStd),1) );\nacc_normModel2     = accuracy_score( dataCvTestLabels, np.argmax(normModel2.predict(dataCvTestFeatStd),1) );\n\nacc_ldaModel1      = accuracy_score( dataCvTestLabels, np.argmax(ldaModel1.predict(dataCvTestFeatLDA),1) );\nacc_ldaModel2      = accuracy_score( dataCvTestLabels, np.argmax(ldaModel2.predict(dataCvTestFeatLDA),1) );\n\nacc_ldaPlain       = accuracy_score( dataCvTestLabels, lda.predict(dataCvTestFeatStd) )\n\n# acc_cnn            = accuracy_score( dataCvTestLabels, np.argmax(cnnModel.predict([dataCvTestFeatRaw,dataCvTestFeatLDA) )\n\n\nprint( \"Results\" )\nprint( \"Baseline Models: \" )\nprint( \"    Small Network    = \", acc_baselineModel1 )\nprint( \"    Large Network    = \", acc_baselineModel2 )\nprint( \"Models with data normalization: \" )\nprint( \"    Small Network    = \", acc_normModel1 )\nprint( \"    Large Network    = \", acc_normModel2 )\nprint( \"LDA based Models: \" )\nprint( \"    Small Network    = \", acc_ldaModel1 )\nprint( \"    Large Network    = \", acc_ldaModel2 )\nprint( \"Plain LDA            = \", acc_ldaPlain )\n# print( \"CNN Model            = \", acc_cnn )                                                                                   \n                                                                                   \n\n\ndata = [ acc_baselineModel1, acc_baselineModel2, acc_normModel1, acc_normModel2, acc_ldaModel1, acc_ldaModel2, acc_ldaPlain ]\nfig, ax = plt.subplots( figsize = (12,6) )\nmodelLabels = ( \"Baseline: Small\", \"Baseline: Large\", \"Norm: Small\", \"Norm: Large\", \"LDA: Small\", \"LDA: Large\", \"LDA: Plain\" )\nax.barh( np.arange(len(data)), data )\nax.set_yticks( np.arange(len(data)) )\nax.set_yticklabels( modelLabels )\nax.set_xticks( np.arange(0.0,1.05,0.1) )\nax.invert_yaxis()\nax.set_xlabel( \"Accuracy\" )\nax.set_title( \"Accuracy of Models\" )\nax.grid()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"672c5f7d85c6ac684d59498f3c66a126eaa8bb94"},"cell_type":"markdown","source":"### Train on full data"},{"metadata":{"trusted":true,"_uuid":"759844a4a3d46512dd74a6804f5adc1ea6fc5637"},"cell_type":"code","source":"lda = LDA()\nlda.fit( dataTrainFeatRaw, dataTrainLabels )\ndataTrainFeatLDA = lda.transform( dataTrainFeatRaw )\ndataTestFeatLDA  = lda.transform( dataTestFeatRaw )\n\ncnnModel = make_cnn_model()\n# cnnModel.fit( [dataTrainFeatRaw,dataTrainFeatLDA], dataTrainOnehot , epochs = 3, verbose = 1, batch_size = 128 )\ntrain_model( cnnModel, [ dataTrainFeatRaw, dataTrainFeatLDA], dataTrainOnehot, \n                       None, None, epochs = 50, verbose = 1 )\nclassProb = cnnModel.predict( [dataTestFeatRaw,dataTestFeatLDA], verbose = 0 )\npredictions = classProb.argmax( axis = -1 )\nsubmissions=pd.DataFrame({\"ImageId\": list(range(1,len(predictions)+1)),\n                         \"Label\": predictions})\nsubmissions.to_csv(\"results.csv\", index=False, header=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2c72a4163885bbeb017ed2bb448bc280d9ba1c88"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}