{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\n\nimport pandas as pd\nimport numpy as np\n\nimport math\nimport gc\nimport copy\nimport os\n\nfrom sklearn.model_selection import KFold, train_test_split\nfrom sklearn.metrics import mean_absolute_error\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom lightgbm import LGBMRegressor","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#设置数据输入路径和导出路径\nDATA_PATH = '../input'\nSUBMISSIONS_PATH = './'\n#用原子序数代替原子名称\nATOMIC_NUMBERS = {\n    'H': 1,\n    'C': 6,\n    'N': 7,\n    'O': 8,\n    'F': 9\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#设置显示参数\npd.set_option('display.max_colwidth', -1)\npd.set_option('display.max_rows', 120)\npd.set_option('display.max_columns', 120)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#显示当前路径下的文件夹\nos.listdir(DATA_PATH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#定义减小消耗内存函数\ndef reduce_mem_usage(df, verbose=False):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv = pd.read_csv('../input/read-giba/giba_train.csv')\ntest_csv = pd.read_csv('../input/read-giba/giba_test.csv')\ntrain_csv.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_csv.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#减小消耗内存，防止处理数据时内存不足\ntrain_csv = reduce_mem_usage(train_csv,verbose = True)\ntest_csv = reduce_mem_usage(test_csv,verbose = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#使用新的数值型特征molecule_index代替字符型molecule_name\ntrain_csv['molecule_index'] = train_csv.molecule_name.str.replace('dsgdb9nsd_', '').astype('int32')\ntest_csv['molecule_index'] = test_csv.molecule_name.str.replace('dsgdb9nsd_', '').astype('int32')\n#删除molecule_name列\ntrain_csv = train_csv.drop(columns = 'molecule_name',axis = 1)\ntest_csv = test_csv.drop(columns = 'molecule_name',axis = 1)\ntrain_csv.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_csv.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_csv = pd.read_csv(f'{DATA_PATH}/champs-scalar-coupling/sample_submission.csv', index_col='id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#设置QM9数据集使用的特征\nqm9_columns = ['mulliken_min', 'mulliken_max', 'mulliken_atom_0', 'mulliken_atom_1']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Load QM9 features...\")\ndata_qm9 = pd.read_pickle('../input/quantum-machine-9-qm9/data.covs.pickle')\ndata_qm9.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#删除不使用的特征列\ndata_qm9 = data_qm9.drop(columns = ['type', 'linear', 'atom_index_0', 'atom_index_1', \n                                    'scalar_coupling_constant', 'U', 'G', 'H', \n                                    'mulliken_mean', 'r2', 'U0','rc_A','rc_B',\n                                    'rc_C', 'mu', 'alpha', 'homo','lumo', 'gap',\n                                    'zpve', 'Cv', 'freqs_min', 'freqs_max', 'freqs_mean',], axis=1)\ndata_qm9 = reduce_mem_usage(data_qm9,verbose=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_qm9.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#data_qm9.set_index('id',inplace = True)\n#使用新的数值型特征molecule_index代替字符型molecule_name\ndata_qm9['molecule_index'] = data_qm9.molecule_name.str.replace('dsgdb9nsd_', '').astype('int32')\n#删除molecule_name列\ndata_qm9 = data_qm9.drop(columns = 'molecule_name',axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_qm9.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv = pd.merge(train_csv, data_qm9, how='left', on=['molecule_index','id'])\ntest_csv = pd.merge(test_csv, data_qm9, how='left', on=['molecule_index','id'])\n#删除data_qm9文件，防止内存不足\ndel data_qm9\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv.set_index('id',inplace = True)\n#train_csv = train_csv.drop(['molecule_name'],axis =1)\ntrain_csv.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_csv.set_index('id',inplace = True)\n#test_csv = test_csv.drop(['molecule_name'],axis =1)\ntest_csv.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"structures_dtypes = {\n    'molecule_name': 'category',\n    'atom_index': 'int8',\n    'atom': 'category',\n    'x': 'float32',\n    'y': 'float32',\n    'z': 'float32'\n}\nstructures_csv = pd.read_csv(f'{DATA_PATH}/champs-scalar-coupling/structures.csv', dtype=structures_dtypes)\nstructures_csv['molecule_index'] = structures_csv.molecule_name.str.replace('dsgdb9nsd_', '').astype('int32')\nstructures_csv = structures_csv[['molecule_index', 'atom_index', 'atom', 'x', 'y', 'z']]\nstructures_csv['atom'] = structures_csv['atom'].replace(ATOMIC_NUMBERS).astype('int8')\nstructures_csv.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Shape: ', structures_csv.shape)\nprint('Total: ', structures_csv.memory_usage().sum())\nstructures_csv.memory_usage()\nstructures_csv = reduce_mem_usage(structures_csv,verbose = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#将所有的缺失数据填充为0\ntrain_csv = train_csv.fillna(0)\ntest_csv = test_csv.fillna(0)\nstructures_csv = structures_csv.fillna(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv.to_csv(f'{SUBMISSIONS_PATH}/fin_train.csv')\ntest_csv.to_csv(f'{SUBMISSIONS_PATH}/fin_test.csv')\nstructures_csv.to_csv(f'{SUBMISSIONS_PATH}/fin_structures.csv')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.1"}},"nbformat":4,"nbformat_minor":1}