{"cells":[{"metadata":{},"cell_type":"markdown","source":"In this notebook, I have taken insipiration from the brute force engineering notebook. This notebook got a public lb score of -0.56337. However, it has to be run locally."},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":false},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport lightgbm as lgb\n#from sklearn.preprocessing import StandardScaler\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.model_selection import train_test_split\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport os\nfrom sklearn.ensemble import RandomForestRegressor\nprint(os.listdir(\"../PredictingMolecularproperties/\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"code","source":"df_train = pd.read_csv('../PredictingMolecularproperties/train.csv')\n\ndf_test = pd.read_csv('../PredictingMolecularproperties/test.csv')\nconstant_data = pd.read_csv('../PredictingMolecularproperties/scalar_coupling_contributions.csv')\nstructures = pd.read_csv('../PredictingMolecularproperties/structures.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Importing data"},{"metadata":{"trusted":false},"cell_type":"code","source":"ids = df_test[\"id\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"df_train.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"df_train.type.unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"from tqdm import tqdm_notebook as tqdm\natomic_radius = {'H':0.38, 'C':0.77, 'N':0.75, 'O':0.73, 'F':0.71} # Without fudge factor\n\nfudge_factor = 0.05\natomic_radius = {k:v + fudge_factor for k,v in atomic_radius.items()}\nprint(atomic_radius)\n\nelectronegativity = {'H':2.2, 'C':2.55, 'N':3.04, 'O':3.44, 'F':3.98}\n\n#structures = pd.read_csv(structures, dtype={'atom_index':np.int8})\n\natoms = structures['atom'].values\natoms_en = [electronegativity[x] for x in tqdm(atoms)]\natoms_rad = [atomic_radius[x] for x in tqdm(atoms)]\n\nstructures['EN'] = atoms_en\nstructures['rad'] = atoms_rad\n\ndisplay(structures.head())\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"i_atom = structures['atom_index'].values\np = structures[['x', 'y', 'z']].values\np_compare = p\nm = structures['molecule_name'].values\nm_compare = m\nr = structures['rad'].values\nr_compare = r\n\nsource_row = np.arange(len(structures))\nmax_atoms = 28\n\nbonds = np.zeros((len(structures)+1, max_atoms+1), dtype=np.int8)\nbond_dists = np.zeros((len(structures)+1, max_atoms+1), dtype=np.float32)\n\nprint('Calculating the bonds')\n\nfor i in tqdm(range(max_atoms-1)):\n    p_compare = np.roll(p_compare, -1, axis=0)\n    m_compare = np.roll(m_compare, -1, axis=0)\n    r_compare = np.roll(r_compare, -1, axis=0)\n    \n    mask = np.where(m == m_compare, 1, 0) #Are we still comparing atoms in the same molecule?\n    dists = np.linalg.norm(p - p_compare, axis=1) * mask\n    r_bond = r + r_compare\n    \n    bond = np.where(np.logical_and(dists > 0.0001, dists < r_bond), 1, 0)\n    \n    source_row = source_row\n    target_row = source_row + i + 1 #Note: Will be out of bounds of bonds array for some values of i\n    target_row = np.where(np.logical_or(target_row > len(structures), mask==0), len(structures), target_row) #If invalid target, write to dummy row\n    \n    source_atom = i_atom\n    target_atom = i_atom + i + 1 #Note: Will be out of bounds of bonds array for some values of i\n    target_atom = np.where(np.logical_or(target_atom > max_atoms, mask==0), max_atoms, target_atom) #If invalid target, write to dummy col\n    \n    bonds[(source_row, target_atom)] = bond\n    bonds[(target_row, source_atom)] = bond\n    bond_dists[(source_row, target_atom)] = dists\n    bond_dists[(target_row, source_atom)] = dists\n\nbonds = np.delete(bonds, axis=0, obj=-1) #Delete dummy row\nbonds = np.delete(bonds, axis=1, obj=-1) #Delete dummy col\nbond_dists = np.delete(bond_dists, axis=0, obj=-1) #Delete dummy row\nbond_dists = np.delete(bond_dists, axis=1, obj=-1) #Delete dummy col\n\nprint('Counting and condensing bonds')\n\nbonds_numeric = [[i for i,x in enumerate(row) if x] for row in tqdm(bonds)]\nbond_lengths = [[dist for i,dist in enumerate(row) if i in bonds_numeric[j]] for j,row in enumerate(tqdm(bond_dists))]\nbond_lengths_mean = [ np.mean(x) for x in bond_lengths]\nn_bonds = [len(x) for x in bonds_numeric]\n\n#bond_data = {'bond_' + str(i):col for i, col in enumerate(np.transpose(bonds))}\n#bond_data.update({'bonds_numeric':bonds_numeric, 'n_bonds':n_bonds})\n\nbond_data = {'n_bonds':n_bonds, 'bond_lengths_mean': bond_lengths_mean }\nbond_df = pd.DataFrame(bond_data)\nstructures = structures.join(bond_df)\ndisplay(structures.head(20))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Calculating bond data, taken from another notebook."},{"metadata":{"trusted":false},"cell_type":"code","source":"del bond_data\ndel bond_df\ndel bonds_numeric\ndel bond_lengths\ndel bond_lengths_mean\ndel i_atom\ndel p, p_compare\ndel m, m_compare\ndel r, r_compare\ndel source_row\ndel bonds\ndel dists\ndel atoms\ndel atoms_en\ndel atoms_rad\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"y = df_train['scalar_coupling_constant']","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"len_train = df_train.shape[0]\ncombined = pd.concat([df_train,df_test],sort=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"structures['molecule_name_atom'] = structures['molecule_name'] + \"_\" + structures['atom_index'].apply(str)\ncombined['molecule_name_atom0'] = combined['molecule_name'] + \"_\" + combined['atom_index_0'].apply(str)\ncombined_f = pd.merge(combined, structures, how = 'left', left_on='molecule_name_atom0', right_on='molecule_name_atom')\n#combined_f","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"combined_f.drop(\"molecule_name_atom0\", axis=1, inplace=True)\ncombined_f.drop(\"molecule_name_y\", axis=1, inplace=True)\ncombined_f.drop(\"atom_index\", axis=1, inplace=True)\ncombined_f.drop(\"molecule_name_atom\", axis=1, inplace=True)\n\ncombined_f.rename(columns = {\"x\":\"x0\",\n                \"EN\":\"EN0\",\n                \"rad\":\"rad0\",\n                \"n_bonds\":\"n_bonds0\",\n                \"bond_lengths_mean\":\"bond_lengths_mean0\",\n                 \"y\":\"y0\",\n                  \"z\":\"z0\",\n                  \"atom\":\"atom0\",\n                  \"molecule_name_x\":\"molecule_name\"\n                 }, inplace=True)\n#combined_f","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"combined_f['molecule_name_atom1'] = combined_f['molecule_name'] + \"_\" + combined_f['atom_index_1'].apply(str)\ncombined = pd.merge(combined_f, structures, how = 'left', left_on='molecule_name_atom1', right_on='molecule_name_atom')\n#combined","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false},"cell_type":"code","source":"combined.drop(\"molecule_name_atom1\", axis=1, inplace=True)\ncombined.drop(\"molecule_name_y\", axis=1, inplace=True)\ncombined.drop(\"atom_index\", axis=1, inplace=True)\ncombined.drop(\"molecule_name_atom\", axis=1, inplace=True)\n\ncombined.rename(columns = {\"x\":\"x1\",\n                 \"EN\":\"EN1\",\n                \"rad\":\"rad1\",\n                \"n_bonds\":\"n_bonds1\",\n                \"bond_lengths_mean\":\"bond_lengths_mean1\",\n                 \"y\":\"y1\",\n                  \"z\":\"z1\",\n                  \"atom\":\"atom1\",\n                  \"molecule_name_x\":\"molecule_name\"\n                 }, inplace=True)\ncombined","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Combining structures.csv"},{"metadata":{"trusted":false},"cell_type":"code","source":"combined.info()\ndel combined_f","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"combined[\"atom1\"].unique()","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false},"cell_type":"code","source":"pos_0 = combined[['x0', 'y0', 'z0']].values\npos_1 = combined[['x1', 'y1', 'z1']].values\n\ncombined['dist'] = np.linalg.norm(pos_0 - pos_1, axis=1)\ncombined['dist_x'] = (combined[\"x0\"] - combined[\"x1\"])**2\ncombined['dist_y'] = (combined[\"y0\"] - combined[\"y1\"])**2\ncombined['dist_z'] = (combined[\"z0\"] - combined[\"z1\"])**2\n\n\ncombined[\"atom_indexes\"] = combined[\"atom_index_0\"] + combined[\"atom_index_1\"]\ncombined[\"distance^2\"] = (combined[\"dist\"])**2\ncombined[\"distance_sqrt\"] = np.sqrt(combined[\"dist\"])\ncombined[\"1/distance^3\"] = 1 / (combined[\"dist\"] + 1)**3\ncombined[\"distance^3\"] = (combined[\"dist\"])**3\ncombined['type_0'] = combined['type'].apply(lambda x: x[0])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Few variations of the distance feature."},{"metadata":{"scrolled":true,"trusted":false},"cell_type":"code","source":"combined['num_molecule_bonds'] = combined.groupby('molecule_name')['id'].transform('count')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"combined['dist_mean'] = combined.groupby('molecule_name')['dist'].transform('mean')\ncombined['dist_min'] = combined.groupby('molecule_name')['dist'].transform('min')\ncombined['dist_max'] = combined.groupby('molecule_name')['dist'].transform('max')\ncombined['dist_std'] = combined.groupby('molecule_name')['dist'].transform('std')\n\ncombined['atom_0_num_bonds'] = combined.groupby(['molecule_name', 'atom_index_0'])['id'].transform('count')\ncombined['atom_1_num_bonds'] = combined.groupby(['molecule_name', 'atom_index_1'])['id'].transform('count')\n\ncombined[\"bf_dist_grouped_type\"] = combined.groupby(['molecule_name', 'type'])[\"dist\"].transform('mean')\ncombined[\"bf_dist_grouped_type\"] = combined.groupby(['molecule_name', 'type'])[\"dist\"].transform('max')\ncombined[\"bf_dist_grouped_type\"] = combined.groupby(['molecule_name', 'type'])[\"dist\"].transform('min')\ncombined[\"bf_dist_grouped_type\"] = combined.groupby(['molecule_name', 'type'])[\"dist\"].transform('std')\n\ncombined[f\"bf_dist_grouped_n_bonds\"] = combined.groupby(['molecule_name', 'n_bonds1'])[\"dist\"].transform('mean')\ncombined[f\"bf_dist_grouped_n_bonds\"] = combined.groupby(['molecule_name', 'n_bonds1'])[\"dist\"].transform('max')\ncombined[f\"bf_dist_grouped_n_bonds\"] = combined.groupby(['molecule_name', 'n_bonds1'])[\"dist\"].transform('min')\ncombined[f\"bf_dist_grouped_n_bonds\"] = combined.groupby(['molecule_name', 'n_bonds1'])[\"dist\"].transform('std')\n        \ncombined[f\"bf_dist_grouped_atom\"] = combined.groupby(['molecule_name', 'atom1'])[\"dist\"].transform('mean')\ncombined[f\"bf_dist_grouped_atom1\"] = combined.groupby(['molecule_name', 'atom1'])[\"dist\"].transform('max')\ncombined[f\"bf_dist_grouped_atom1\"] = combined.groupby(['molecule_name', 'atom1'])[\"dist\"].transform('min')\ncombined[f\"bf_dist_grouped_atom1\"] = combined.groupby(['molecule_name', 'atom1'])[\"dist\"].transform('std')\n        ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Few features taken from the brute force engineering notebook."},{"metadata":{"trusted":false},"cell_type":"code","source":"combined['dist/maxdist'] = combined['dist'] / combined['dist_max']\ncombined['ENs/dist'] = (combined['EN0'] + combined['EN1']) / combined['dist']\ncombined['rads/dist'] = (combined['rad0'] + combined['rad1']) / combined['dist']\ncombined['ENS'] = combined['EN0'] + combined['EN1']\n#combined['atom_num_bonds'] = combined['atom1'] + str(combined['n_bonds1'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"numerical = [\"EN0\", \"rad0\", \"bond_lengths_mean0\", \"EN1\", \"rad1\", \"bond_lengths_mean1\", \"1/distance^3\", \"ENS\"]\nfor col in numerical:\n    for col2 in numerical:\n        if col != col2:\n            combined[f\"bf_{col}_/_{col2}\"] = combined[col] / combined[col2]\n            combined[f\"bf_{col}_*_{col2}\"] = combined[col] * combined[col2]\n            combined[f\"bf_{col}_+_{col2}\"] = combined[col] + combined[col2]\n            combined[f\"bf_{col}_-_{col2}\"] = combined[col] - combined[col2]\n            \n        combined[f\"bf_{col}_grouped_type\"] = combined.groupby(['molecule_name', 'type'])[col].transform('mean')\n        combined[f\"bf_{col}_grouped_type\"] = combined.groupby(['molecule_name', 'type'])[col].transform('max')\n        combined[f\"bf_{col}_grouped_type\"] = combined.groupby(['molecule_name', 'type'])[col].transform('min')\n        combined[f\"bf_{col}_grouped_type\"] = combined.groupby(['molecule_name', 'type'])[col].transform('std')\n\n        combined[f\"bf_{col}_grouped_n_bonds\"] = combined.groupby(['molecule_name', 'n_bonds1'])[col].transform('mean')\n        combined[f\"bf_{col}_grouped_n_bonds\"] = combined.groupby(['molecule_name', 'n_bonds1'])[col].transform('max')\n        combined[f\"bf_{col}_grouped_n_bonds\"] = combined.groupby(['molecule_name', 'n_bonds1'])[col].transform('min')\n        combined[f\"bf_{col}_grouped_n_bonds\"] = combined.groupby(['molecule_name', 'n_bonds1'])[col].transform('std')\n        \n        combined[f\"bf_{col}_grouped_n_bonds\"] = combined.groupby(['molecule_name', 'atom1'])[col].transform('mean')\n        combined[f\"bf_{col}_grouped_n_bonds\"] = combined.groupby(['molecule_name', 'atom1'])[col].transform('max')\n        combined[f\"bf_{col}_grouped_n_bonds\"] = combined.groupby(['molecule_name', 'atom1'])[col].transform('min')\n        combined[f\"bf_{col}_grouped_n_bonds\"] = combined.groupby(['molecule_name', 'atom1'])[col].transform('std')\n        \n\ncombined.info()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"In the above code, instead of calculating only the mean, min, max and standard deviation, I have taken a few features and applied the 4 arithmetic operations on them. I was only able to use a few coloumns, since the number of features goes on increasing and is roughly approximated to n!, where n is the number of features in the 'numerical' array.\nI am running this on my personal computer, but if we could add more features, I think the score might go down. "},{"metadata":{"trusted":false},"cell_type":"code","source":"def reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2\n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)\n            else:\n                c_prec = df[col].apply(lambda x: np.finfo(x).precision).max()\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max and c_prec == np.finfo(np.float16).precision:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max and c_prec == np.finfo(np.float32).precision:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"I wasn't able to use the reduce_mem_usage method as it transformed the data in an awkward way, which could not be used by the lightgbm model."},{"metadata":{"trusted":false},"cell_type":"code","source":"combined.drop(\"id\", axis=1, inplace=True)\ncombined.drop(\"n_bonds0\", axis=1, inplace=True)\ncombined.drop(\"atom0\", axis=1, inplace=True)\ncombined.drop(\"molecule_name\", axis=1, inplace=True)\n\ncombined[\"atom_indexes\"] = combined[\"atom_indexes\"].apply(str)\ncombined.drop(\"atom_index_0\", axis=1, inplace=True)\ncombined.drop(\"atom_index_1\", axis=1, inplace=True)\ncombined.drop(\"scalar_coupling_constant\", axis=1, inplace=True)\n#print(combined.head())\n#combined = reduce_mem_usage(combined)\nprint(combined.head())\ncombined=pd.get_dummies(combined)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Applying one-hot encoding for all categorical features."},{"metadata":{"trusted":false},"cell_type":"code","source":"\ndf_train=combined[:len_train]\ndf_test=combined[len_train:]\ndel len_train\ndel combined\ndel structures","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"x = df_train\n#x = np.array(x)\n#x = x.reshape((-1, 1))\ny_fc = constant_data['fc']\n#y_sd = constant_data['sd']\n#y_pso = constant_data['pso']\n#y_dso = constant_data['dso']\nx_predict = df_test\n#x_predict = np.array(x_predict)\n#x_predict = x_predict.reshape((-1,1))\nx_train, x_valid, y_train, y_valid = train_test_split(x, y, train_size=0.8)\n\n#df_train.info()\ndel constant_data","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false},"cell_type":"code","source":"df_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"df_train_1JHN = df_train[df_train[\"type_1JHN\"] == 1]\ny_fc_1JHN = y_fc[df_train[\"type_1JHN\"] == 1]\ny_1JHN = y[df_train[\"type_1JHN\"] == 1]\n\ndf_train_1JHC_1 = df_train[(df_train[\"type_1JHC\"] == 1) & (df_train[\"dist\"] > 1.065)]\ny_fc_1JHC_1 = y_fc[(df_train[\"type_1JHC\"] == 1) & (df_train[\"dist\"] > 1.065)]\ny_1JHC_1 = y[(df_train[\"type_1JHC\"] == 1) & (df_train[\"dist\"] > 1.065)]\n\ndf_train_1JHC_2 = df_train[(df_train[\"type_1JHC\"] == 1) & (df_train[\"dist\"] <= 1.065)]\ny_fc_1JHC_2 = y_fc[(df_train[\"type_1JHC\"] == 1) & (df_train[\"dist\"] <= 1.065)]\ny_1JHC_2 = y[(df_train[\"type_1JHC\"] == 1) & (df_train[\"dist\"] <= 1.065)]\n\ndf_train_2JHC = df_train[df_train[\"type_2JHC\"] == 1]\ny_fc_2JHC = y_fc[df_train[\"type_2JHC\"] == 1]\ny_2JHC = y[df_train[\"type_2JHC\"] == 1]\n\ndf_train_3JHH = df_train[df_train[\"type_3JHH\"] == 1]\ny_fc_3JHH = y_fc[df_train[\"type_3JHH\"] == 1]\ny_3JHH = y[df_train[\"type_3JHH\"] == 1]\n\ndf_train_3JHC = df_train[df_train[\"type_3JHC\"] == 1]\ny_fc_3JHC = y_fc[df_train[\"type_3JHC\"] == 1]\ny_3JHC = y[df_train[\"type_3JHC\"] == 1]\n\ndf_train_2JHH = df_train[df_train[\"type_2JHH\"] == 1]\ny_fc_2JHH = y_fc[df_train[\"type_2JHH\"] == 1]\ny_2JHH = y[df_train[\"type_2JHH\"] == 1]\n\ndf_train_3JHN = df_train[df_train[\"type_3JHN\"] == 1]\ny_fc_3JHN = y_fc[df_train[\"type_3JHN\"] == 1]\ny_3JHN = y[df_train[\"type_3JHN\"] == 1]\n\ndf_train_2JHN = df_train[df_train[\"type_2JHN\"] == 1]\ny_fc_2JHN = y_fc[df_train[\"type_2JHN\"] == 1]\ny_2JHN = y[df_train[\"type_2JHN\"] == 1]\n","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false},"cell_type":"code","source":"df_train_1JHN.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"df_train_1JHN.drop([\"type_1JHC\", \"type_1JHN\", \"type_2JHC\", \"type_2JHH\", \"type_2JHN\", \"type_3JHC\", \"type_3JHH\", \"type_3JHN\"], axis=1, inplace=True)\ndf_train_1JHC_1.drop([\"type_1JHC\", \"type_1JHN\", \"type_2JHC\", \"type_2JHH\", \"type_2JHN\", \"type_3JHC\", \"type_3JHH\", \"type_3JHN\"], axis=1, inplace=True)\ndf_train_1JHC_2.drop([\"type_1JHC\", \"type_1JHN\", \"type_2JHC\", \"type_2JHH\", \"type_2JHN\", \"type_3JHC\", \"type_3JHH\", \"type_3JHN\"], axis=1, inplace=True)\ndf_train_2JHC.drop([\"type_1JHC\", \"type_1JHN\", \"type_2JHC\", \"type_2JHH\", \"type_2JHN\", \"type_3JHC\", \"type_3JHH\", \"type_3JHN\"], axis=1, inplace=True)\ndf_train_3JHH.drop([\"type_1JHC\", \"type_1JHN\", \"type_2JHC\", \"type_2JHH\", \"type_2JHN\", \"type_3JHC\", \"type_3JHH\", \"type_3JHN\"], axis=1, inplace=True)\ndf_train_3JHC.drop([\"type_1JHC\", \"type_1JHN\", \"type_2JHC\", \"type_2JHH\", \"type_2JHN\", \"type_3JHC\", \"type_3JHH\", \"type_3JHN\"], axis=1, inplace=True)\ndf_train_2JHH.drop([\"type_1JHC\", \"type_1JHN\", \"type_2JHC\", \"type_2JHH\", \"type_2JHN\", \"type_3JHC\", \"type_3JHH\", \"type_3JHN\"], axis=1, inplace=True)\ndf_train_3JHN.drop([\"type_1JHC\", \"type_1JHN\", \"type_2JHC\", \"type_2JHH\", \"type_2JHN\", \"type_3JHC\", \"type_3JHH\", \"type_3JHN\"], axis=1, inplace=True)\ndf_train_2JHN.drop([\"type_1JHC\", \"type_1JHN\", \"type_2JHC\", \"type_2JHH\", \"type_2JHN\", \"type_3JHC\", \"type_3JHH\", \"type_3JHN\"], axis=1, inplace=True)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"df_test.insert(0, 'ID', range(1, 1 + len(df_test)))\n\n\ndf_test_1JHN = df_test[df_test[\"type_1JHN\"] == 1]\n\ndf_test_1JHC_1 = df_test[(df_test[\"type_1JHC\"] == 1) & (df_test[\"dist\"] > 1.065)]\n\ndf_test_1JHC_2 = df_test[(df_test[\"type_1JHC\"] == 1) & (df_test[\"dist\"] <= 1.065)]\n\ndf_test_2JHC = df_test[df_test[\"type_2JHC\"] == 1]\n\ndf_test_3JHH = df_test[df_test[\"type_3JHH\"] == 1]\n\ndf_test_3JHC = df_test[df_test[\"type_3JHC\"] == 1]\n\ndf_test_2JHH = df_test[df_test[\"type_2JHH\"] == 1]\n\ndf_test_3JHN = df_test[df_test[\"type_3JHN\"] == 1]\n\ndf_test_2JHN = df_test[df_test[\"type_2JHN\"] == 1]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"df_test_1JHN.drop([\"type_1JHC\", \"type_1JHN\", \"type_2JHC\", \"type_2JHH\", \"type_2JHN\", \"type_3JHC\", \"type_3JHH\", \"type_3JHN\"], axis=1, inplace=True)\ndf_test_1JHC_1.drop([\"type_1JHC\", \"type_1JHN\", \"type_2JHC\", \"type_2JHH\", \"type_2JHN\", \"type_3JHC\", \"type_3JHH\", \"type_3JHN\"], axis=1, inplace=True)\ndf_test_1JHC_2.drop([\"type_1JHC\", \"type_1JHN\", \"type_2JHC\", \"type_2JHH\", \"type_2JHN\", \"type_3JHC\", \"type_3JHH\", \"type_3JHN\"], axis=1, inplace=True)\ndf_test_2JHC.drop([\"type_1JHC\", \"type_1JHN\", \"type_2JHC\", \"type_2JHH\", \"type_2JHN\", \"type_3JHC\", \"type_3JHH\", \"type_3JHN\"], axis=1, inplace=True)\ndf_test_3JHH.drop([\"type_1JHC\", \"type_1JHN\", \"type_2JHC\", \"type_2JHH\", \"type_2JHN\", \"type_3JHC\", \"type_3JHH\", \"type_3JHN\"], axis=1, inplace=True)\ndf_test_3JHC.drop([\"type_1JHC\", \"type_1JHN\", \"type_2JHC\", \"type_2JHH\", \"type_2JHN\", \"type_3JHC\", \"type_3JHH\", \"type_3JHN\"], axis=1, inplace=True)\ndf_test_2JHH.drop([\"type_1JHC\", \"type_1JHN\", \"type_2JHC\", \"type_2JHH\", \"type_2JHN\", \"type_3JHC\", \"type_3JHH\", \"type_3JHN\"], axis=1, inplace=True)\ndf_test_3JHN.drop([\"type_1JHC\", \"type_1JHN\", \"type_2JHC\", \"type_2JHH\", \"type_2JHN\", \"type_3JHC\", \"type_3JHH\", \"type_3JHN\"], axis=1, inplace=True)\ndf_test_2JHN.drop([\"type_1JHC\", \"type_1JHN\", \"type_2JHC\", \"type_2JHH\", \"type_2JHN\", \"type_3JHC\", \"type_3JHH\", \"type_3JHN\"], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"I have created separate training and testing sets for each type so that it would be easier to make separate models."},{"metadata":{},"cell_type":"markdown","source":"\n\n<br>\n<br>\n<br>\n<br>\n<br>\n\n\n\n<b><b><b>MODELS</b></b></b>\n<br>\n<br>\n<br>\n<br>\n<br>\n<br>\n<br>\n\n"},{"metadata":{"trusted":false},"cell_type":"code","source":"def train_type_wise(train_data, y_fc_type, y_type):\n    d_train_fc = lgb.Dataset(train_data, label=y_fc_type)\n    params = {}\n    params['learning_rate'] = 0.2\n    params['boosting_type'] = 'gbdt'\n    params['objective'] = 'regression'\n    params['metric'] = 'mae'\n    params['num_leaves'] = 10\n    params['min_data'] = 50\n    params['max_depth'] = 10\n\n    clf_fc = lgb.train(params, d_train_fc, 2000)\n    \n    del d_train_fc\n    del params\n    \n    train_data[\"fc\"] = clf_fc.predict(train_data)\n    #del clf_fc\n    \n    \n    d_train = lgb.Dataset(train_data, label=y_type)\n    params = {'num_leaves': 128,\n          'min_child_samples': 79,\n          'objective': 'regression',\n          'max_depth': 9,\n          'learning_rate': 0.2,\n          \"boosting_type\": \"gbdt\",\n          \"metric\": 'mae',\n          \"verbosity\": -1,\n          'reg_alpha': 0.1,\n          'reg_lambda': 0.3,\n         }\n    cl = lgb.train(params, d_train, 2000)\n    \n    train_data[\"ridge\"] = cl.predict(train_data)\n\n    del d_train\n    del params\n    #del cl\n\n    \n    d_train = lgb.Dataset(train_data, label=y_type)\n    params = {\n                'boosting_type': 'gbdt',\n                'objective': 'regression',\n                'metric': 'mae',\n                'learning_rate': 0.2,\n                'num_leaves': 20, \n                'reg_alpha': 0.5, \n                'reg_lambda': 0.5, \n                'nthread': 4, \n                'device': 'cpu',\n                'min_child_samples': 45\n            }\n\n    clf = lgb.train(params, d_train, 2000)\n    return clf_fc, cl, clf\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"So we are making 3 models per type. The first model predicts the fc constant from the scalar_coupling_contributions file. Then the fc predictions make an additional feature for the second model which predicts the scalar coupling constant. The output of this as well forms an additional feature which is used by the final model to make the final predictions. \n\nIdeally, one of the last 2 models should be a different kind of model, like a neural network or maybe even another tree-based model using a different library. However, I was unable to use neural networks successfully."},{"metadata":{"trusted":false},"cell_type":"code","source":"model_1JHN_fc, model_1JHN_1, model_1JHN_2 = train_type_wise(df_train_1JHN, y_fc_1JHN, y_1JHN)\n\nmodel_1JHC_1_fc, model_1JHC_1_1, model_1JHC_1_2 = train_type_wise(df_train_1JHC_1, y_fc_1JHC_1, y_1JHC_1)\n\nmodel_1JHC_2_fc, model_1JHC_2_1, model_1JHC_2_2 = train_type_wise(df_train_1JHC_2, y_fc_1JHC_2, y_1JHC_2)\n\nmodel_2JHC_fc, model_2JHC_1, model_2JHC_2 = train_type_wise(df_train_2JHC, y_fc_2JHC, y_2JHC)\n\nmodel_3JHH_fc, model_3JHH_1, model_3JHH_2 = train_type_wise(df_train_3JHH, y_fc_3JHH, y_3JHH)\n\nmodel_3JHC_fc, model_3JHC_1, model_3JHC_2 = train_type_wise(df_train_3JHC, y_fc_3JHC, y_3JHC)\n\nmodel_2JHH_fc, model_2JHH_1, model_2JHH_2 = train_type_wise(df_train_2JHH, y_fc_2JHH, y_2JHH)\n\nmodel_3JHN_fc, model_3JHN_1, model_3JHN_2 = train_type_wise(df_train_3JHN, y_fc_3JHN, y_3JHN)\n\nmodel_2JHN_fc, model_2JHN_1, model_2JHN_2 = train_type_wise(df_train_2JHN, y_fc_2JHN, y_2JHN)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def predict_type_wise(test_data, model_fc, model_1, model_2):\n    x_pred = test_data.drop(\"ID\", axis=1)\n    test_data[\"fc\"] = model_fc.predict(x_pred)\n    x_pred = test_data.drop(\"ID\", axis=1)\n    test_data[\"ridge\"] = model_1.predict(x_pred)\n    x_pred = test_data.drop(\"ID\", axis=1)\n    test_data[\"predictions\"] = model_2.predict(x_pred)\n    return test_data","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false},"cell_type":"code","source":"predictions_1JHN = predict_type_wise(df_test_1JHN, model_1JHN_fc, model_1JHN_1, model_1JHN_2)\n\npredictions_1JHC_1 = predict_type_wise(df_test_1JHC_1, model_1JHC_1_fc, model_1JHC_1_1, model_1JHC_1_2)\n\npredictions_1JHC_2 = predict_type_wise(df_test_1JHC_2, model_1JHC_2_fc, model_1JHC_2_1, model_1JHC_2_2)\n\npredictions_2JHC = predict_type_wise(df_test_2JHC, model_2JHC_fc, model_2JHC_1, model_2JHC_2)\n\npredictions_3JHH = predict_type_wise(df_test_3JHH, model_3JHH_fc, model_3JHH_1, model_3JHH_2)\n\npredictions_3JHC = predict_type_wise(df_test_3JHC, model_3JHC_fc, model_3JHC_1, model_3JHC_2)\n\npredictions_2JHH = predict_type_wise(df_test_2JHH, model_2JHH_fc, model_2JHH_1, model_2JHH_2)\n\npredictions_3JHN = predict_type_wise(df_test_3JHN, model_3JHN_fc, model_3JHN_1, model_3JHN_2)\n\npredictions_2JHN = predict_type_wise(df_test_2JHN, model_2JHN_fc, model_2JHN_1, model_2JHN_2)\n","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false},"cell_type":"code","source":"predicted_data = pd.concat([predictions_1JHN, predictions_1JHC_1, predictions_1JHC_2, predictions_2JHC, predictions_3JHH, predictions_3JHC, predictions_2JHH, predictions_3JHN, predictions_2JHN], ignore_index=True)\n\npredicted_data","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"final_data = pd.merge(df_test, predicted_data, how='left', left_on=\"ID\", right_on=\"ID\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"final_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"#prediction_array = np.array(predictions_scalar)\n#temp = prediction_array.flat\n#predictions = list(temp)\n#print(predictions)\npredictions = final_data[\"predictions\"]\nmy_sub = pd.DataFrame({'id':ids, 'scalar_coupling_constant':predictions})\nmy_sub.to_csv('submission.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Since this was the first time I used lightgbm, I wasn't able to tune any of the parameter and pretty much copied them from other notebooks. I have also not used the QM9 dataset, which I think may be very beneficial.\n\nPlease upvote if you found it helpful. "}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.3"}},"nbformat":4,"nbformat_minor":1}