{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\n# os.system('pip install pandas==0.24.1')\n\nimport pandas as pd\nprint(pd.__version__)\nSEED = 26\nLR = 1e-4","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import warnings\nimport logging\n\nwarnings.filterwarnings(\"ignore\")\nwarnings.filterwarnings(action=\"ignore\",category=DeprecationWarning)\nwarnings.filterwarnings(action=\"ignore\",category=FutureWarning)\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport os\nimport math\nimport gc\nimport copy\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler, MinMaxScaler\n\nnp.random.seed(SEED)\nfrom tensorflow import set_random_seed\n\nset_random_seed(SEED)\n\nfrom keras.layers import Dense, Input, Activation\nfrom keras.layers import BatchNormalization,Add,Dropout\nfrom keras.optimizers import Adam\nfrom keras.models import Model, load_model\nfrom keras import callbacks\nfrom keras import backend as K\n\ndef read_pickle(path):\n    with open(path, \"rb\") as f:\n        obj = pickle.load(f)\n    return obj\n\n\ndef add_contributions(train, test):\n    scalar_coupling_contributions = pd.read_csv(\n        f\"{DATA_PATH}/scalar_coupling_contributions.csv\"\n    )\n    gc.collect()\n    len_train = len(train)\n    df = pd.concat([train, test], axis=0, sort=True)\n    del train, test\n    gc.collect()\n\n    df = pd.merge(\n        df,\n        scalar_coupling_contributions,\n        on=[\"molecule_name\", \"atom_index_0\", \"atom_index_1\", \"type\"],\n        how=\"left\",\n    )\n    # energy = pd.read_csv(f\"{DATA_DIR}/potential_energy.csv\")\n    # df = df.merge(\n    #     energy, left_on=[\"molecule_name\"], right_on=[\"molecule_name\"], how=\"left\"\n    # )\n    # del energy\n\n    train = df.iloc[:len_train, :].reset_index(drop=True)\n    test = df.iloc[len_train:, :].reset_index(drop=True)\n    del df\n    gc.collect()\n    return train, test\n\n\ndef select_features_for_type(\n        type_,\n        imp_file=\"../input/cached-champs/lgb_feat_importance_full.csv\",\n        top_n=900,\n        imp_threshold=100,\n):\n    df = pd.read_csv(imp_file)\n    df = df[\n        (df[\"type\"] == type_)\n        & (df[\"importance\"] > 0)\n        & (df[\"importance\"] > imp_threshold)\n    ]\n    df = df.sort_values(\"importance\", ascending=False)\n    return list(df[\"feature\"].values[:top_n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(os.listdir(\"../input\"))\n# feats\nFILETRAIN1 = \"../input/cached-champs/k_cached_processed_train2.pkl\"\nFILETEST1 = \"../input/cached-champs/k_cached_processed_test2.pkl\"\nFILETRAIN = \"../input/cached-champs/k_cached_processed_train_inv_pvals.pkl\"\nFILETEST = \"../input/cached-champs/k_cached_processed_test_inv_pvals.pkl\"\nepoch_n = 2200\nverbose = 1\nbatch_size = 2048\nLINEAR_TOP = 27\nLGB_TOP = 17\nTARGETS = ['fc', 'sd', 'pso', 'dso']\n\nlogger = logging.getLogger('chimps gona win')\nlogger.setLevel(logging.DEBUG)\nimport pickle\nDATA_PATH = '../input/champs-scalar-coupling'\nSUBMISSIONS_PATH = './'\n# use atomic numbers to recode atomic names\nATOMIC_NUMBERS = {\n    'H': 1,\n    'C': 6,\n    'N': 7,\n    'O': 8,\n    'F': 9\n}\nlogger.debug('starting... this is going to be awesome')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"types_dict = {\n    \"1JHC\": 0,\n    \"2JHH\": 3,\n    \"1JHN\": 1,\n    \"2JHN\": 4,\n    \"2JHC\": 2,\n    \"3JHH\": 6,\n    \"3JHC\": 5,\n    \"3JHN\": 7,\n}\n\n\natom_features = [\n    \"atom_2\",\n    \"atom_3\",\n    \"atom_4\",\n    \"atom_5\",\n    \"atom_6\",\n    \"atom_7\",\n    \"atom_8\",\n    \"atom_9\",\n    # \"overth\",\n]\n\nfc_feats = [\n    \"fc_preds_type\",\n    \"fc_preds_akira\",\n    \"fc_preds_akira2\",\n    \"fc_preds_akira3\",\n    \"fc_preds_akira4\",\n]\n\ntypes_to_run = [\"1JHN\", \"1JHC\", \"2JHH\", \"2JHN\", \"2JHC\", \"3JHH\", \"3JHC\", \"3JHN\"]\nall_feats = []\nfor mol_type in types_to_run:\n    i = types_dict[mol_type]\n    linear_feats = select_features_for_type(\n        i, top_n=LINEAR_TOP, imp_threshold=0,\n        imp_file=\"../input/cached-champs/feats_inv_pvals.csv\"\n    )\n    lgb_feats = select_features_for_type(\n        i, top_n=LGB_TOP, imp_threshold=0,\n    )\n    feats = linear_feats + lgb_feats\n    all_feats.append(feats)\n    # print(i, mol_type, feats)\nall_feats = np.unique(list(np.concatenate(all_feats)) + atom_features + fc_feats)\n# all_feats = [col for col in all_feats if 'fc_preds' not in col]\nprint(all_feats)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dtypes = {\n    'molecule_name': 'category',\n    'atom_index_0': 'int8',\n    'atom_index_1': 'int8',\n    'type': 'category',\n    'scalar_coupling_constant': 'float32'\n}\ntrain_csv = pd.read_csv(f'{DATA_PATH}/train.csv', index_col='id', dtype=train_dtypes)\n# train_csv['molecule_index'] = train_csv.molecule_name.str.replace('dsgdb9nsd_', '').astype('int32')\ncols = ['molecule_name', 'atom_index_0', 'atom_index_1', 'type']\ntrain_csv = train_csv[cols]\ntrain_csv.head(10)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\ntest_csv = pd.read_csv(f'{DATA_PATH}/test.csv', index_col='id', dtype=train_dtypes)\ntest_csv['molecule_index'] = test_csv['molecule_name'].str.replace('dsgdb9nsd_', '').astype('int32')\ncols = [col for col in cols if 'scalar_coupling_constant' not in col]\ntest_csv = test_csv[cols]\ngc.collect()\nprint(train_csv.shape)\nprint(test_csv.shape)\ntest_csv.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv, test_csv = add_contributions(train_csv, test_csv)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cols = [\"molecule_name\", \"atom_index_0\", \"atom_index_1\"]\ntrain_csv = train_csv.drop(cols, axis=1)\ntest_csv = test_csv.drop(cols, axis=1)\ntrain_csv.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plug extra data\ntr = read_pickle(FILETRAIN)\ntr = tr.fillna(0)\ntrain_ix = train_csv.index\ntr.index = train_ix\ntrain_csv = pd.concat([train_csv, tr], axis=1)\ntrain_csv.index = train_ix\ntrain_csv = train_csv[[col for col in train_csv.columns if col in list(all_feats) + TARGETS + [\"type\"]]]\ndel tr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"te = read_pickle(FILETEST)\nte = te.fillna(0)\ntest_ix = test_csv.index\nte.index = test_ix\ntest_csv = pd.concat([test_csv, te], axis=1)\ntest_csv.index = test_ix\ntest_csv = test_csv[[col for col in test_csv.columns if col in list(all_feats) + [\"type\"]]]\ndel te, test_ix, train_ix\ngc.collect()\nprint(train_csv.shape)\nprint(test_csv.shape)\ntest_csv.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plug extra data\ntr = read_pickle(FILETRAIN1)\ntr = tr.fillna(0)[[col for col in tr.columns if col not in train_csv.columns]]\ntrain_ix = train_csv.index\ntr.index = train_ix\ntrain_csv = pd.concat([train_csv, tr], axis=1)\ntrain_csv.index = train_ix\ndel tr\ntrain_csv = train_csv[list(all_feats) + TARGETS + [\"type\"]]\ngc.collect()\nte = read_pickle(FILETEST1)\nte = te.fillna(0)[[col for col in te.columns if col not in test_csv.columns]]\ntest_ix = test_csv.index\nte.index = test_ix\ntest_csv = pd.concat([test_csv, te], axis=1)\ntest_csv.index = test_ix\ndel te, test_ix, train_ix\ngc.collect()\ntest_csv = test_csv[list(all_feats) + [\"type\"]]\ntest_csv.head(10)\nprint(train_csv.shape)\nprint(test_csv.shape)\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# test_csv = test_csv[list(all_feats)]\n# train_csv = train_csv[list(all_feats) + ['scalar_coupling_constant']]\ngc.collect()\n# train_csv = reduce_mem_usage(train_csv)\n# test_csv = reduce_mem_usage(test_csv)\nprint(len(train_csv.columns), len(np.unique(train_csv.columns)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_nn_model(input_shape):\n    inp = Input(shape=(input_shape,))\n    x = Dense(2048, activation=\"relu\")(inp)\n    x = BatchNormalization()(x)\n    x = Dropout(0.1)(x)\n    x = Dense(1024, activation=\"relu\")(x)\n    x = BatchNormalization()(x)\n    x = Dense(512, activation=\"relu\")(x)\n    x = BatchNormalization()(x)\n    out = Dense(4, activation=\"linear\")(x)  \n    model = Model(inputs=inp, outputs=[out])\n    return model\n\ndef plot_history(history, label):\n    plt.plot(history.history['loss'])\n    plt.plot(history.history['val_loss'])\n    plt.title('Loss for %s' % label)\n    plt.ylabel('Loss')\n    plt.xlabel('Epoch')\n    _= plt.legend(['Train','Validation'], loc='upper left')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Set up GPU preferences\nconfig = tf.ConfigProto( device_count = {'GPU': 1 , 'CPU': 2} ) \nconfig.gpu_options.allow_growth = True\nconfig.gpu_options.per_process_gpu_memory_fraction = 0.6\nsess = tf.Session(config=config) \nK.set_session(sess)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from datetime import datetime\nfrom sklearn.preprocessing import OneHotEncoder\n\ncv_score = []\ncv_score_total = 0\n\n# Set to True if we want to train from scratch.  False will reuse saved models as a starting point.\nretrain = True\n\nstart_time = datetime.now()\ntest_prediction = np.zeros(len(test_csv))\n\nclass FeatureTransformer:\n    def transform(self, dataset, ohe_features=[], continuous_features=[]):\n        ohe_df = OneHotEncoder().fit_transform(dataset.loc[:, ohe_features]).toarray()\n        skews = dataset.loc[:, continuous_features].skew().to_dict()\n        # for column, skew in skews.items():\n        # if skew > 1 and 'fc_preds' not in column:\n        #     dataset[column] = np.log1p(dataset[column])\n\n        return np.concatenate(\n            [\n                StandardScaler().fit_transform(dataset.loc[:, continuous_features]),\n                ohe_df,\n            ],\n            axis=1,\n        )\n    \n# types_to_run = [\"1JHN\"]   # quick test\nfor mol_type in types_to_run:\n    model_name_wrt = \"/kaggle/working/molecule_model_%s.hdf5\" % mol_type\n    model_name_rd = \"../input/nncont-seed-25/molecule_model_%s.hdf5\" % mol_type\n\n    if mol_type==\"3JHC\":\n        linear = 16\n        lgb = 16\n    else:\n        linear = LINEAR_TOP\n        lgb = LGB_TOP\n    # n_feats=30\n    linear_feats = select_features_for_type(\n        types_dict[mol_type], top_n=linear, imp_threshold=0,\n        imp_file=\"../input/cached-champs/feats_inv_pvals.csv\"\n    )\n    lgb_feats = select_features_for_type(\n        types_dict[mol_type], top_n=lgb, imp_threshold=0,\n    )\n    \n    distance_features = list(np.unique(linear_feats + lgb_feats + fc_feats))\n    input_features = list(np.unique(distance_features + atom_features))\n    print(input_features)\n    df_train_ = train_csv.loc[train_csv[\"type\"] == mol_type, input_features + TARGETS]\n    df_test_ = test_csv.loc[test_csv[\"type\"] == mol_type, input_features]\n    msg = f\"training {mol_type}, out of {types_to_run}, train shape: {df_train_.shape}, LR: {LR}\"\n    print(msg)\n    logger.debug(msg)\n    # Standard Scaler from sklearn does seem to work better here than other Scalers\n    input_data = FeatureTransformer().transform(\n        dataset=pd.concat(\n            [df_train_.loc[:, input_features], df_test_.loc[:, input_features]]\n        ),\n        ohe_features=atom_features,\n        continuous_features=distance_features,  # + fc_feats\n    )\n    target_data = df_train_.loc[:, TARGETS].values\n\n    # Simple split to provide us a validation set to do our CV checks with\n    train_index, cv_index = train_test_split(\n        np.arange(len(df_train_)), random_state=SEED, test_size=0.1\n    )\n    # Split all our input and targets by train and cv indexes\n    train_target = target_data[train_index]\n    cv_target = target_data[cv_index]\n    train_input = input_data[train_index]\n    cv_input = input_data[cv_index]\n    test_input = input_data[len(df_train_) :, :]\n\n    # Build the Neural Net\n    nn_model = create_nn_model(train_input.shape[1])\n\n    # If retrain==False, then we load a previous saved model as a starting point.\n    retrain = False\n    if not retrain:\n        nn_model = load_model(model_name_rd)\n\n    nn_model.compile(loss=\"mae\", optimizer=Adam(lr=LR))\n\n    # Callback for Early Stopping... May want to raise the min_delta for small numbers of epochs\n    es = callbacks.EarlyStopping(\n        monitor=\"val_loss\",\n        min_delta=0.0001,\n        patience=50,\n        verbose=1,\n        mode=\"auto\",\n        restore_best_weights=True,\n    )\n    # Callback for Reducing the Learning Rate... when the monitor levels out for 'patience' epochs, then the LR is reduced\n    rlr = callbacks.ReduceLROnPlateau(\n        monitor=\"val_loss\", factor=0.5, patience=30, min_lr=1e-7, mode=\"auto\", verbose=1\n    )\n    # Save the best value of the model for future use\n    sv_mod = callbacks.ModelCheckpoint(\n        model_name_wrt, monitor=\"val_loss\", save_best_only=True, period=1\n    )\n    history = nn_model.fit(\n        train_input,\n        [train_target],\n        validation_data=(cv_input, [cv_target]),\n        callbacks=[es, rlr, sv_mod],\n        epochs=epoch_n,\n        batch_size=batch_size,\n        verbose=0,\n    )\n\n    cv_predict = nn_model.predict(cv_input).sum(axis=1)\n    plot_history(history, mol_type)\n    accuracy = np.mean(np.abs(cv_target.sum(axis=1) - cv_predict))\n    print(np.log(accuracy))\n    cv_score.append(np.log(accuracy))\n    cv_score_total += np.log(accuracy)\n\n    # Predict on the test data set using our trained model\n    test_predict = nn_model.predict(test_input).sum(axis=1)\n\n    # for each molecule type we'll grab the predicted values\n    test_prediction[test_csv[\"type\"] == mol_type] = test_predict\n    K.clear_session()\n\ncv_score_total /= len(types_to_run)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submit = pd.read_csv(f'{DATA_PATH}/sample_submission.csv')\ndef submits(predictions):\n    submit[\"scalar_coupling_constant\"] = predictions\n    submit.to_csv(\"/kaggle/working/nnetCont_sub.csv\", index=False)\nsubmits(test_prediction)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print ('Total training time: ', datetime.now() - start_time)\n\ni=0\nfor mol_type in types_to_run: \n    print(mol_type,\": cv score is \",cv_score[i])\n    i+=1\nprint(\"total cv score is\",cv_score_total)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submit = pd.read_csv(f'{DATA_PATH}/sample_submission.csv')\ndef submits(predictions):\n    submit[\"scalar_coupling_constant\"] = predictions\n    submit.to_csv(f\"/kaggle/working/nnetCont_sub_{round(cv_score_total, 4)}.csv\", index=False)\nsubmits(test_prediction)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}