{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport os\nimport random\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport pathlib\nfrom sklearn.preprocessing import StandardScaler, RobustScaler\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nwarnings.filterwarnings(action=\"ignore\",category=DeprecationWarning)\nwarnings.filterwarnings(action=\"ignore\",category=FutureWarning)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f'tf={tf.__version__}, keras={keras.__version__}')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"SEED = 31\nEPOCHS = 1000\nBATCH_SIZE = 1000\nVALIDATION_SPLIT = 0.01\nTARGET = 'scalar_coupling_constant'\nPREDICTORS = [\n    'molecule_atom_index_0_dist_mean_div',\n    'molecule_atom_index_0_dist_max_div',\n    'molecule_atom_index_1_dist_max_div',\n    'molecule_atom_index_0_dist_std_div',\n    'molecule_atom_index_0_dist_min_div',\n    'molecule_atom_index_1_dist_mean_div',\n    'molecule_atom_index_1_dist_std_div',\n    'molecule_atom_1_dist_std_diff',\n    'molecule_atom_index_0_dist_std_diff',\n    'molecule_atom_index_0_dist_mean_diff',\n    'molecule_atom_index_1_dist_max_diff',\n    'molecule_atom_index_0_dist_max_diff',\n    'molecule_type_0_dist_std_diff',\n    'molecule_atom_index_1_dist_mean_diff',\n    'molecule_atom_index_1_dist_std_diff',\n    'molecule_atom_1_dist_min_div',\n    'molecule_atom_1_dist_min_diff',\n    'type_0',\n    'type_1',\n    'molecule_type_dist_min',\n    'molecule_type_dist_mean',\n    'molecule_type_0_dist_std',\n    'dist_to_type_1_mean',\n    'dist',\n    'molecule_type_dist_max',\n    'dist_x',\n    'dist_y',\n    'dist_z'\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    \nseed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"file_folder = '../input/champs-scalar-coupling-preprocess'\ntrain = pd.read_csv(f'{file_folder}/train.csv')\ntest = pd.read_csv(f'{file_folder}/test.csv')\nprint(f'train={train.shape}, test={test.shape}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train = train_whole.sample(frac=0.99)\n#validation = train_whole.drop(train.index)\n#print(f'train={train.shape}, validation={validation.shape}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Eval function"},{"metadata":{"trusted":true},"cell_type":"code","source":"def group_mean_log_mae(y_true, y_pred, types, floor=1e-9):\n    \"\"\"\n    Fast metric computation for this competition: https://www.kaggle.com/c/champs-scalar-coupling\n    Code is from this kernel: https://www.kaggle.com/uberkinder/efficient-metric\n    \"\"\"\n    maes = (y_true-y_pred).abs().groupby(types).mean()\n    maes = np.log(maes.map(lambda x: max(x, floor)))\n    print(maes)\n    return maes.mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train = train[TARGET]\nx_train = train[PREDICTORS]\nx_test = test[PREDICTORS]\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Normalize features"},{"metadata":{"trusted":true},"cell_type":"code","source":"scaler = RobustScaler().fit(x_train.values)\nnorm = scaler.transform(x_train.values)\nx_train = pd.DataFrame(norm, index=x_train.index, columns=x_train.columns)\nnorm = scaler.transform(x_test.values)\nx_test = pd.DataFrame(norm, index=x_test.index, columns=x_test.columns)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Neural net"},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model():\n  model = keras.Sequential([\n    layers.Dense(64, activation=tf.nn.relu, input_shape=[len(x_train.keys())]),\n    layers.Dense(64, activation=tf.nn.relu),\n    layers.Dense(64, activation=tf.nn.relu),\n    layers.Dense(64, activation=tf.nn.relu),\n    layers.Dense(64, activation=tf.nn.relu),\n    layers.Dense(64, activation=tf.nn.relu),\n    layers.Dense(64, activation=tf.nn.relu),\n    layers.Dense(64, activation=tf.nn.relu),\n    layers.Dense(1)\n  ])\n  optimizer = tf.keras.optimizers.RMSprop(0.001)\n  model.compile(loss='mean_absolute_error',\n                optimizer=optimizer,\n                metrics=['mean_absolute_error', 'mean_squared_error'])\n  return model\n\n\nmodel = build_model()\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# sanity check model is producing output of desired type and shape\n\nexample_batch = x_train[:10]\nexample_result = model.predict(example_batch)\nexample_result","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train the model"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\n# Display training progress by printing a single dot for each completed epoch\nclass PrintDot(keras.callbacks.Callback):\n  def on_epoch_end(self, epoch, logs):\n    if epoch % 100 == 0: print('')\n    print('.', end='')\n\n\n# The patience parameter is the amount of epochs to check for improvement\nearly_stop = keras.callbacks.EarlyStopping(monitor='val_loss', patience=10)    \n\nhistory = model.fit(\n  x_train, y_train,\n  epochs=EPOCHS, validation_split = VALIDATION_SPLIT, verbose=0, batch_size=BATCH_SIZE,\n  callbacks=[early_stop, PrintDot()])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"hist = pd.DataFrame(history.history)\nhist['epoch'] = history.epoch\nhist.tail()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_history(history):\n  hist = pd.DataFrame(history.history)\n  hist['epoch'] = history.epoch\n\n  plt.figure()\n  plt.xlabel('Epoch')\n  plt.ylabel('Mean Abs Error [MPG]')\n  plt.plot(hist['epoch'], hist['mean_absolute_error'],\n           label='Train Error')\n  plt.plot(hist['epoch'], hist['val_mean_absolute_error'],\n           label = 'Val Error')\n  plt.ylim([0,5])\n  plt.legend()\n\n  #plt.figure()\n  #plt.xlabel('Epoch')\n  #plt.ylabel('Mean Square Error [$MPG^2$]')\n  #plt.plot(hist['epoch'], hist['mean_squared_error'],label='Train Error')\n  #plt.plot(hist['epoch'], hist['val_mean_squared_error'],label = 'Val Error')\n  #plt.ylim([0,20])\n  #plt.legend()\n  plt.show()\n\n\nplot_history(history)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Early stopping"},{"metadata":{"trusted":true},"cell_type":"code","source":"#model = build_model()\n\n# The patience parameter is the amount of epochs to check for improvement\n#early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', patience=10)\n\n#history = model.fit(x_train, y_train, epochs=EPOCHS, batch_size=BATCH_SIZE,\n #                   validation_split = VALIDATION_SPLIT, verbose=0, callbacks=[early_stop, PrintDot()])\n\n#plot_history(history)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#loss, mae, mse = model.evaluate(test, y_test, verbose=0)\n\n#print(\"Testing set Mean Abs Error: {:5.2f} MPG\".format(mae))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred_train = model.predict(x_train, batch_size=BATCH_SIZE).flatten()\ngmlm = group_mean_log_mae(y_train, y_pred_train, train['type'])\nprint('group_mean_log_mae={}'.format(gmlm))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = model.predict(x_test, batch_size=BATCH_SIZE).flatten()\nprint(preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame({'id': test['id'], 'scalar_coupling_constant': preds})\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)\ntrain = pd.DataFrame({'id': train['id'], 'type': train['type'], TARGET: train[TARGET], 'pred': y_pred_train})\ntrain.to_csv('train.csv', index=False)\nprint(os.listdir(\".\"))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}