{"cells":[{"metadata":{"_uuid":"573c4ba72827bd1c899836c48eab695fc82eec4f"},"cell_type":"markdown","source":"# >99.4% model + PCA visualizations + Error analysis"},{"metadata":{"_uuid":"37cecca84cf9e1caf4a1105fa0cb134971ee018c"},"cell_type":"markdown","source":"This notebook is focusing on data analysis, visualizations ans error analysis. Brought to you by eLearn:inga"},{"metadata":{"_uuid":"0053e64a2b00517c2503853b7a6897b91bdfa262"},"cell_type":"markdown","source":"## Data exploration"},{"metadata":{"trusted":true,"_uuid":"33f6733e92372ced2f03e673bb25da3b0b769d09"},"cell_type":"code","source":"%matplotlib inline\n\n# For simple vectorized calculations\nimport numpy as np\n\n# Mainly data handling and representation\nimport pandas as pd\n\n# Models\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Conv2D, Flatten, MaxPooling2D, BatchNormalization, Activation, Dropout\nfrom keras.losses import categorical_crossentropy\nfrom keras.optimizers import Adam\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras import Model\n\n# Data preparation\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.decomposition import PCA\n\n# Plotting and display\nfrom IPython.display import display\nfrom matplotlib import pyplot as plt\n\nnp.random.seed(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"494adb48d23004bb668b725c687b7529dc5d0981"},"cell_type":"code","source":"# Path of the file to read.\ntrain_file_path = '../input/train.csv'\n\n# Read the file\ndigit_data_orig = pd.read_csv(train_file_path)\n\n# The shape of the data\ndigit_data_orig.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cca7782c3e5de2f94c9ea0da88b06cdc05c2ff9b"},"cell_type":"code","source":"# Separate the label from the data\ny_orig = digit_data_orig.iloc[:, 0].values.reshape(-1,1)\n\nm = y_orig.shape[0]\nprint(\"The number of images: m = {}\".format(m))\n\n# There are 42000 images so 42000 labels\nprint(\"The shape of y: {}\".format(y_orig.shape))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e7386583012c9379b3709b1b61e08b39f19bf4e3"},"cell_type":"code","source":"# One-hot encode the categorical values\ndef one_hot_encode_categories(y):\n    \n    encoder = OneHotEncoder(handle_unknown='ignore', sparse=False)\n\n    y_one_hot = pd.DataFrame(encoder.fit_transform(y), columns=encoder.get_feature_names())\n        \n    return y_one_hot, encoder","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true,"_uuid":"8816929ca35ba9033699c3a653cfebb2d6786af9"},"cell_type":"code","source":"# One hot encode y\ny_one_hot, encoder = one_hot_encode_categories(y_orig)\n\n# Strip the category names\ny_one_hot.columns = pd.DataFrame(y_one_hot.columns)[0].apply(lambda x: x[-1])\n\n# The number of categories\nn_y = y_one_hot.shape[1]\nprint(\"The number of categories: n_y = {}\".format(n_y))\n\n# A few examples\nprint(y_orig[0:10])\ny_one_hot.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e6ef0deb7915a1e03d0e03b6ba0d8540d99f94a2"},"cell_type":"code","source":"# Let's see how many examples are ther from each category\ndisplay(pd.DataFrame(y_one_hot.sum(axis=0)).transpose())\n\n# Plot the values\nplt.bar(y_one_hot.columns, y_one_hot.sum(axis=0))\n\n# There is approximately the same number of examples there are from each category","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dd2822d759be76a947d7c811c8927829bf0c342b"},"cell_type":"code","source":"# Separate the image data\nX_orig = digit_data_orig.iloc[:, 1:].values.reshape(-1,1)\n\nprint(\"The shape of X without reshaping: {}\".format(X_orig.shape))\n\n# There are 42000 images and 64x64 pixel each image which is 32928000 total","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c6811dba1ba5f26e9ea97e28d9ff7c86ff51e3b7"},"cell_type":"code","source":"# Let's reshape the images and view some\nX_reshaped = X_orig.reshape(-1,28,28)\n\ndef plot_sample_images(X, y, images_to_show=10, random=True):\n\n    fig = plt.figure(1)\n\n    images_to_show = min(X.shape[0], images_to_show)\n\n    # Set the canvas based on the numer of images\n    fig.set_size_inches(18.5, images_to_show * 0.3)\n\n    # Generate random integers (non repeating)\n    if random == True:\n        idx = np.random.choice(range(X.shape[0]), images_to_show, replace=False)\n    else:\n        idx = np.arange(images_to_show)\n        \n    # Print the images with labels\n    for i in range(images_to_show):\n        plt.subplot(images_to_show/10 + 1, 10, i+1)\n        plt.title(str(y[idx[i]]))\n        plt.imshow(X[idx[i], :, :], cmap='Greys')\n        \n\n# Choose how many images you would like to see\nimages_to_show = 30\n\nplot_sample_images(X_reshaped, y_orig, images_to_show=images_to_show)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"159788b021c31cc2e9a323611d04885de1d5fe72"},"cell_type":"code","source":"# The number of X features are 28*28 = 784\nn_x = 28*28\n\nprint(\"The number of X features are: n_x = {}\".format(n_x))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2a47d02380128f6f69d3ae782512e98a16f2de6d"},"cell_type":"markdown","source":""},{"metadata":{"scrolled":true,"trusted":true,"_uuid":"c2de6d6ef2b5e2b625b4a728cf8ff5c455f17882"},"cell_type":"code","source":"# Scale the image pixel values from 0-255 to 0-1 range so the neural net can to converge faster\nX_scaled = X_reshaped / 255\n\nprint(\"Original scale: {} - {}\".format(X_reshaped.min(), X_reshaped.max()))\nprint(\"New scale: {} - {}\".format(X_scaled.min(), X_scaled.max()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8ded6e0ba0f5ffd704ddc198c78da8b4d1d06f8b"},"cell_type":"code","source":"X = X_scaled.reshape(-1, 28, 28, 1)\ny = y_one_hot","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9f6d8cbfd81e9772aad615cbd33a16673d8647b1"},"cell_type":"markdown","source":"## Simple model definition"},{"metadata":{"trusted":true,"_uuid":"bdad677fbfc163f34a3e53d2605ae00195273fd8"},"cell_type":"code","source":"# We can train a model using directly the images, let's first do that","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1ab648e367b6f280d317a58ad8f40362d7fc3cc7"},"cell_type":"code","source":"def model_definition():\n    # Define a simple model in Keras\n    model = Sequential()\n\n    # Add layers to the model\n\n    # Add convolutional layer\n    model.add(Conv2D(50, kernel_size=(5,5), input_shape=(28,28,1)))\n\n    # Add ReLu activation function\n    model.add(Activation('relu'))\n\n    # Add dropout layer for generalization\n    model.add(Dropout(0.05))\n\n    # Add maxpool layer\n    model.add(MaxPooling2D(pool_size=(2, 2), strides=None, padding='valid', data_format=None))\n\n    # Add batch normalization to help learning and avoid vanishing or exploding gradient\n    model.add(BatchNormalization())\n\n    # Add convolutional layer\n    model.add(Conv2D(50, kernel_size=(3,3)))\n\n    # Add ReLu activation function\n    model.add(Activation('relu'))\n\n    # Add dropout layer for generalization\n    model.add(Dropout(0.05))\n\n    # Maxpool layer\n    model.add(MaxPooling2D(pool_size=(2, 2), strides=None, padding='valid', data_format=None))\n\n    # Add batch normalization to help learning and avoid vanishing or exploding gradient\n    model.add(BatchNormalization())\n\n    # Add flatten layer to get 1d data for dense layer\n    model.add(Flatten())\n\n    # Dense layer\n    model.add(Dense(100, input_dim=650))\n    \n    # Add ReLu activation function\n    model.add(Activation('relu'))\n\n    # Dense layer\n    model.add(Dense(10))\n    \n    # Add sigmoid activation function to get values beteween 0-1\n    model.add(Activation('softmax'))\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fed66ff3ceb679c7c9a1a3d7245f1931c8bb7f33"},"cell_type":"code","source":"model = model_definition()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6f1aab601240e87d74a6d3e1097b08f538d18720"},"cell_type":"code","source":"# Define the hyperparameters\n\nbatch_size = 32\nepochs = 100","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4ab23104074a55ce771658d33e70b1fd7984f388"},"cell_type":"code","source":"# Define the loss function, this is a categorical cross entropy\nloss = categorical_crossentropy","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b69eb82b7c19665d681ee23b68348e5e7b63518f"},"cell_type":"code","source":"# Define the optimizer\noptimizer = Adam(lr=0.0005)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"962b41487304be2f94dcbc8f6e55ccdf52f8975b"},"cell_type":"code","source":"# Compile the model\nmodel.compile(loss=loss, optimizer=optimizer, metrics=[\"categorical_accuracy\"])","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":true,"_uuid":"25299af7363cdf505f7c39bd1d2639b02be1fec4"},"cell_type":"code","source":"# Let's see the model configuration\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":true,"_uuid":"8b957f262e11f07d854de09a01a49bcb6b7ccc37"},"cell_type":"code","source":"# Split the data into train and validation parts\ntrain_X, val_X, train_y, val_y = train_test_split(X, y.values, random_state=1)\n\nprint(\"The train image shape: {}\".format(train_X.shape))\nprint(\"The train label shape: {}\".format(train_y.shape))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dd79a39fe24f7117ade137ba9a4d3078e192e805"},"cell_type":"markdown","source":"### Image augmentation"},{"metadata":{"trusted":true,"_uuid":"ee9bf6265250094876b0e7d8d1edd096cce68dd8"},"cell_type":"code","source":"# Define the augmentation properties\ngenerator = ImageDataGenerator(#featurewise_center=True,\n                               #samplewise_center=True,\n                               #featurewise_std_normalization=True,\n                               #samplewise_std_normalization=True,\n                               #zca_whitening=False,\n                               #zca_epsilon=1e-06,\n                               rotation_range=10,\n                               width_shift_range=0.1,\n                               height_shift_range=0.1,\n                               #brightness_range=None,\n                               shear_range=5,\n                               zoom_range=0.1,\n                               cval=0.0,)\n\n# Fit the augmentation to the images\ngenerator.fit(X)\n\nX_augmented, y_augmented = generator.flow(train_X, train_y, batch_size=batch_size).next()\n\n# Plot some augmented images\nplot_sample_images(X_augmented[:10,:,:,0], encoder.inverse_transform(y_augmented)[:10,0], 10)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true,"_uuid":"9e57623e78e1d823cc80ab891c1d88e57cd2a9c9"},"cell_type":"code","source":"# Let's train the model\nhistory = model.fit_generator(generator.flow(train_X, train_y, batch_size=batch_size),\n                              steps_per_epoch=len(train_X) / batch_size,\n                              validation_data=[val_X, val_y],\n                              epochs=epochs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"42b4793973ba5fa5a2578f0e70792f0e7dfb71b3"},"cell_type":"code","source":"def plot_history(history):# Plot the loss and accuracy\n    # Format the train history\n    history_df = pd.DataFrame(history.history, columns=history.history.keys())\n\n    \n    # Plot the accuracy\n    fig = plt.figure()\n    fig.set_size_inches(18.5, 10)\n    ax = plt.subplot(211)\n    ax.plot(history_df[\"categorical_accuracy\"], label=\"categorical_accuracy\")\n    ax.plot(history_df[\"val_categorical_accuracy\"], label=\"val_categorical_accuracy\")\n    ax.legend()\n    plt.title('Score during training.')\n    plt.xlabel('Training step')\n    plt.ylabel('Accuracy')\n    plt.grid(b=True, which='major', axis='both')\n    \n    # Plot the loss\n    ax = plt.subplot(212)\n    ax.plot(history_df[\"loss\"], label=\"loss\")\n    ax.plot(history_df[\"val_loss\"], label=\"val_loss\")\n    ax.legend()\n    plt.title('Loss during training.')\n    plt.xlabel('Training step')\n    plt.ylabel('Loss')\n    plt.grid(b=True, which='major', axis='both')\n    \n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":true,"_uuid":"ead2fb7f73f4516602d37e782af1530040fc8713"},"cell_type":"code","source":"plot_history(history)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"94d740a488d15c197afe1234650f917ff10d7cf0"},"cell_type":"code","source":"# The result seems to be good, so let's train the data on the whole train dataset\n\n# Reinitialize the model\n#model = model_definition()\n\n# Compile the model\nmodel.compile(loss=loss, optimizer=optimizer, metrics=[\"categorical_accuracy\"])\n\n# Train the model on the full train data\nfinal_history = model.fit_generator(generator.flow(X, y, batch_size=batch_size),\n                                              steps_per_epoch=len(X) / batch_size,\n                                              epochs=epochs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"55b4d86b9e6838e2f9b9c17db0514bfe2bb35c47"},"cell_type":"code","source":"def plot_history(history):# Plot the loss and accuracy\n    # Format the train history\n    history_df = pd.DataFrame(history.history, columns=history.history.keys())\n    display(history_df)\n    \n    fig = plt.figure()\n    ax = plt.subplot(111)\n    for i in range(history_df.shape[1]):\n        ax.plot(history_df.iloc[:,i], label=history_df.columns[i])\n    ax.legend()\n    plt.title('Score() and loss during training.')\n    plt.xlabel('Training step')\n    plt.ylabel('Accuracy and Loss')\n    plt.grid(b=True, which='major', axis='both')\n    plt.show()\n\nplot_history(final_history)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6e041d01f08b6e4f338e6928fcc76f8542b5e649"},"cell_type":"markdown","source":"### Import the test data"},{"metadata":{"trusted":true,"_uuid":"f2c5e2a06c102f99992b540c5631c45b933246c6"},"cell_type":"code","source":"# Path of the file to read.\ntrain_file_path = '../input/test.csv'\n\n# Read the file\ntest_data_orig = pd.read_csv(train_file_path)\n\n# The shape of the data\ntest_data_orig.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"84dae9795b2694a79156f4cb3c41bbc8181a8c7e"},"cell_type":"markdown","source":"### Transform the test data"},{"metadata":{"trusted":true,"_uuid":"9a588bbfbec36dec0a9a4c6e49c7edd8aaaf10d2"},"cell_type":"code","source":"# Let's reshape the images and view some\nX_test_reshaped = test_data_orig.values.reshape(-1, 28, 28, 1)\nprint(X_test_reshaped.shape)\n# Choose how many images you would like to see\nimages_to_show = 30\n\nplot_sample_images(X_test_reshaped[:, :, :, 0], np.zeros((X_test_reshaped.shape[0])), images_to_show=images_to_show)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1d17b18db1eba421f9b5dd44437b58c37bdebaae"},"cell_type":"code","source":"# Scale the image pixel values from 0-255 to 0-1 range so the neural net can to converge faster\nX_test_scaled = X_test_reshaped / 255\n\n# The final X_test\nX_test = X_test_scaled","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d173301300e3524fe4a7034c0da64f7cd14e2262"},"cell_type":"markdown","source":"### Predict the output values"},{"metadata":{"trusted":true,"_uuid":"2f2846a3179db8ea624a9714f8a9b8cf6663d140"},"cell_type":"code","source":"# Predict the labels\ntest_preds_unscaled = model.predict(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"212cc63fe08a0c16bf57ec62443171ba0e341414"},"cell_type":"code","source":"# Inverse transform the predictions to the original scale\ntest_preds = encoder.inverse_transform(test_preds_unscaled)[:,0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"02a49fce2e754987b8663412cdb6efa9119067ee"},"cell_type":"code","source":"# Save the predictions\noutput = pd.DataFrame({'ImageId': range(1, test_preds.shape[0] + 1),\n                       'Label': test_preds})\n\noutput.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5c15ec3401b489d812711bba21617b253e4d237d"},"cell_type":"markdown","source":"### Show the learned visualization by PCA"},{"metadata":{"trusted":true,"_uuid":"3b5cfc1fde27a30425f97c25f2dc09d81049c184"},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true,"_uuid":"85c0f3ee3df0df8a224335ea996ffb26b83ecba4"},"cell_type":"code","source":"# Get the output of the last activation function\nlayer_name = 'dense_4'\n\n# Define an intermediate model\nintermediate_layer_model = Model(inputs=model.input,\n                                 outputs=model.get_layer(layer_name).output)\n\n# Calculate the values of the intermediate model\nintermediate_output = intermediate_layer_model.predict(X)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f4f22e55449b4dd5191e75618ea18c77c53363c8"},"cell_type":"code","source":"PCA_transformer = PCA()\n        \n# Fit and transform\nactivation_7_PCA = PCA_transformer.fit_transform(intermediate_output)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":true,"_uuid":"02babb391f23fc92541e48c701c58bd3a007a739"},"cell_type":"code","source":"# %matplotlib inline\nnumber_of_points = 10000\n\nx1 = activation_7_PCA[:number_of_points, 0]\nx2 = activation_7_PCA[:number_of_points, 1]\n\nfig = plt.figure()\nfig.set_size_inches(18.5, 10)\n\nplt.scatter(x=x1,\n            y=x2,\n            c=(y_orig[:number_of_points])[:, 0],\n            cmap=\"tab10\")\n\nax = plt.subplot(111)\n\nfor i in range(number_of_points):\n    if not i % 100:\n        ax.annotate(str(y_orig[i]), (x1[i], x2[i]))\n\nplt.title('Last layer visualization using PCA 2D')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":true,"_uuid":"5100cb33ae0e9a412aff93557cd0de71bf2b4b0c"},"cell_type":"code","source":"\nfrom mpl_toolkits.mplot3d import Axes3D\n\nnumber_of_points = 1000\n\nx1 = activation_7_PCA[:number_of_points, 0]\nx2 = activation_7_PCA[:number_of_points, 1]\nx3 = activation_7_PCA[:number_of_points, 2]\n\nfig = plt.figure()\nfig.set_size_inches(10, 10)\n\n\n\nax = plt.subplot(111, projection='3d')\n\nax.scatter(xs=x1,\n            ys=x2,\n            zs=x3,\n            c=(y_orig[:number_of_points])[:,0],\n            cmap=\"tab10\",)\n\nfor i in range(number_of_points):\n    if not i % 10:\n        ax.text(x1[i], x2[i], x3[i], str(y_orig[i]))\n\nplt.title('Last layer visualization using PCA 3D')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"86ee310909c040eac8021f5b6d251ae843913ae4"},"cell_type":"markdown","source":"### Error analysis"},{"metadata":{"trusted":true,"_uuid":"523564f4dd1fd9f76e216788d73b0ebcad3a4608"},"cell_type":"code","source":"augmented_data_batch = 100000\nX_aug, y_aug_real_unscaled = generator.flow(X, y.values, batch_size=augmented_data_batch).next()\n\n# Print examples when the model made bad decisions\ny_aug_preds_unscaled = model.predict(X_aug)\n# Inverse transform the predictions to the original scale\ny_aug_preds = encoder.inverse_transform(y_aug_preds_unscaled)\ny_aug_real = encoder.inverse_transform(y_aug_real_unscaled)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"18e3f694b9bd15040c3145ac19fa2d603410f6fd"},"cell_type":"code","source":"y_aug_all = pd.DataFrame([y_aug_preds[:,0], y_aug_real[:,0]]).transpose()\ny_aug_all.columns = [\"y predicted\", \"y real\"]\ny_aug_all.head(10)\nplot_sample_images(X_aug[:10, :, :, 0], y_aug_real, 10, random=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9718ee1d6f586de82b2141118ad5b006c601d278"},"cell_type":"code","source":"pred_errors = y_aug_all[y_aug_all['y predicted'] != y_aug_all['y real']]\ntotal_errors = pred_errors.shape[0]\n\nprint(\"The total number of errors from {} augmented image: {}\".format(augmented_data_batch, total_errors))\nprint(\"Which is {0:.3f}%\".format(total_errors/augmented_data_batch * 100))","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true,"_uuid":"fc7df5cac52e9173e9293b448dbe73bc1ef7e090"},"cell_type":"code","source":"errors_by_category = []\nerror_count_by_category = []\n\nfor i in range(n_y):\n    \n    errors_by_category.append(pred_errors[pred_errors[\"y real\"] == i])\n\n    error_count_by_category.append(errors_by_category[i].shape[0])\n\nerror_count_by_category_df = pd.DataFrame(error_count_by_category).transpose()\n\nprint(\"Number of errors by category: \")\ndisplay(error_count_by_category_df)\n\nfig = plt.figure()\nax = plt.subplot(111)\nplt.bar(x=error_count_by_category_df.columns, height=error_count_by_category_df.values[0]/total_errors * 100)\n\nplt.title('Percentage of error by category')\nplt.xlabel('Categories')\nplt.ylabel('Percentage of error (%)')\n#plt.xticks(range(n_y))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3f98decddd73ab0ba418555b06215fb2588c73c9"},"cell_type":"code","source":"for errors in errors_by_category:\n    plot_sample_images(X_aug[errors.index, :, :, 0], errors[\"y predicted\"].values, 10, random=False)\n    plt.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}