\ndef reduce_mem_usage(df, verbose=True):\n numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n start_mem = df.memory_usage().sum() / 1024**2
\n for col in df.columns:\n col_type = df[col].dtypes\n if col_type in numerics:\n c_min = df[col].min()\n c_max = df[col].max()\n if str(col_type)[:3] == 'int':\n if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n df[col] = df[col].astype(np.int8)\n elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n df[col] = df[col].astype(np.int16)\n elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n df[col] = df[col].astype(np.int32)\n elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n df[col] = df[col].astype(np.int64)
\n else:\n if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n df[col] = df[col].astype(np.float16)\n elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n df[col] = df[col].astype(np.float32)\n else:\n df[col] = df[col].astype(np.float64)
\n end_mem = df.memory_usage().sum() / 1024**2\n if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n return df\n
\ntrain = reduce_mem_usage(pd.read_csv('../input/train.csv'))\ntest = reduce_mem_usage(pd.read_csv('../input/test.csv'))\nmulliken = reduce_mem_usage(pd.read_csv('../input/mulliken_charges.csv'))\ndipole = reduce_mem_usage(pd.read_csv('../input/dipole_moments.csv'))\nstructure = reduce_mem_usage(pd.read_csv('../input/structures.csv'))\npotential = reduce_mem_usage(pd.read_csv('../input/potential_energy.csv'))\nmagnetic = reduce_mem_usage(pd.read_csv('../input/magnetic_shielding_tensors.csv'))\nscaler = reduce_mem_usage(pd.read_csv('../input/scalar_coupling_contributions.csv'))\nsub = reduce_mem_usage(pd.read_csv('../input/sample_submission.csv'))\n
Mem. usage decreased to 106.62 Mb (50.0% reduction)\nMem. usage decreased to 52.57 Mb (45.0% reduction)\nMem. usage decreased to 16.09 Mb (54.2% reduction)\nMem. usage decreased to 1.13 Mb (56.2% reduction)\nMem. usage decreased to 51.74 Mb (52.1% reduction)\nMem. usage decreased to 0.81 Mb (37.5% reduction)\nMem. usage decreased to 39.49 Mb (69.3% reduction)\nMem. usage decreased to 115.50 Mb (59.4% reduction)\nMem. usage decreased to 11.95 Mb (68.7% reduction)
", "messages": [ { "id": 540562, "postDate": "2019-05-31T16:48:54.183Z", "content": "\ndef reduce_mem_usage(df, verbose=True):\n numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n start_mem = df.memory_usage().sum() / 1024**2
\n for col in df.columns:\n col_type = df[col].dtypes\n if col_type in numerics:\n c_min = df[col].min()\n c_max = df[col].max()\n if str(col_type)[:3] == 'int':\n if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n df[col] = df[col].astype(np.int8)\n elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n df[col] = df[col].astype(np.int16)\n elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n df[col] = df[col].astype(np.int32)\n elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n df[col] = df[col].astype(np.int64)
\n else:\n if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n df[col] = df[col].astype(np.float16)\n elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n df[col] = df[col].astype(np.float32)\n else:\n df[col] = df[col].astype(np.float64)
\n end_mem = df.memory_usage().sum() / 1024**2\n if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n return df\n
\ntrain = reduce_mem_usage(pd.read_csv('../input/train.csv'))\ntest = reduce_mem_usage(pd.read_csv('../input/test.csv'))\nmulliken = reduce_mem_usage(pd.read_csv('../input/mulliken_charges.csv'))\ndipole = reduce_mem_usage(pd.read_csv('../input/dipole_moments.csv'))\nstructure = reduce_mem_usage(pd.read_csv('../input/structures.csv'))\npotential = reduce_mem_usage(pd.read_csv('../input/potential_energy.csv'))\nmagnetic = reduce_mem_usage(pd.read_csv('../input/magnetic_shielding_tensors.csv'))\nscaler = reduce_mem_usage(pd.read_csv('../input/scalar_coupling_contributions.csv'))\nsub = reduce_mem_usage(pd.read_csv('../input/sample_submission.csv'))\n
Mem. usage decreased to 106.62 Mb (50.0% reduction)\nMem. usage decreased to 52.57 Mb (45.0% reduction)\nMem. usage decreased to 16.09 Mb (54.2% reduction)\nMem. usage decreased to 1.13 Mb (56.2% reduction)\nMem. usage decreased to 51.74 Mb (52.1% reduction)\nMem. usage decreased to 0.81 Mb (37.5% reduction)\nMem. usage decreased to 39.49 Mb (69.3% reduction)\nMem. usage decreased to 115.50 Mb (59.4% reduction)\nMem. usage decreased to 11.95 Mb (68.7% reduction)
", "rawMarkdown": "```\ndef reduce_mem_usage(df, verbose=True):\n numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n start_mem = df.memory_usage().sum() / 1024**2 \n for col in df.columns:\n col_type = df[col].dtypes\n if col_type in numerics:\n c_min = df[col].min()\n c_max = df[col].max()\n if str(col_type)[:3] == 'int':\n if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n df[col] = df[col].astype(np.int8)\n elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n df[col] = df[col].astype(np.int16)\n elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n df[col] = df[col].astype(np.int32)\n elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n df[col] = df[col].astype(np.int64) \n else:\n if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n df[col] = df[col].astype(np.float16)\n elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n df[col] = df[col].astype(np.float32)\n else:\n df[col] = df[col].astype(np.float64) \n end_mem = df.memory_usage().sum() / 1024**2\n if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n return df\n```\n\n\n```\ntrain = reduce_mem_usage(pd.read_csv('../input/train.csv'))\ntest = reduce_mem_usage(pd.read_csv('../input/test.csv'))\nmulliken = reduce_mem_usage(pd.read_csv('../input/mulliken_charges.csv'))\ndipole = reduce_mem_usage(pd.read_csv('../input/dipole_moments.csv'))\nstructure = reduce_mem_usage(pd.read_csv('../input/structures.csv'))\npotential = reduce_mem_usage(pd.read_csv('../input/potential_energy.csv'))\nmagnetic = reduce_mem_usage(pd.read_csv('../input/magnetic_shielding_tensors.csv'))\nscaler = reduce_mem_usage(pd.read_csv('../input/scalar_coupling_contributions.csv'))\nsub = reduce_mem_usage(pd.read_csv('../input/sample_submission.csv'))\n```\n\nMem. usage decreased to 106.62 Mb (50.0% reduction)\nMem. usage decreased to 52.57 Mb (45.0% reduction)\nMem. usage decreased to 16.09 Mb (54.2% reduction)\nMem. usage decreased to 1.13 Mb (56.2% reduction)\nMem. usage decreased to 51.74 Mb (52.1% reduction)\nMem. usage decreased to 0.81 Mb (37.5% reduction)\nMem. usage decreased to 39.49 Mb (69.3% reduction)\nMem. usage decreased to 115.50 Mb (59.4% reduction)\nMem. usage decreased to 11.95 Mb (68.7% reduction)", "votes": 11 }, { "id": 544332, "postDate": "2019-06-05T11:56:57.987Z", "content": "Thanks again. But could you please recognize the original author of this code snippet @gemartin and this is the link of his original work: https://www.kaggle.com/gemartin/load-data-reduce-memory-usage
", "rawMarkdown": "Thanks again. But could you please recognize the original author of this code snippet @gemartin and this is the link of his original work: https://www.kaggle.com/gemartin/load-data-reduce-memory-usage", "votes": 9, "replies": [ { "id": 545871, "postDate": "2019-06-06T01:44:25.597Z", "content": "Thanks for highlighting it!
", "rawMarkdown": "Thanks for highlighting it!", "votes": 1 } ] }, { "id": 554187, "postDate": "2019-06-17T05:52:53.213Z", "content": "Thanks for the method, very useful for this problem in my 8 GB RAM laptop.
", "rawMarkdown": "Thanks for the method, very useful for this problem in my 8 GB RAM laptop." }, { "id": 550698, "postDate": "2019-06-12T00:00:01.960Z", "content": "So this reduced the data frames, but I still cannot merge them; i'm getting a memory error.
\n\nAnyone have any insights on how to merge the entire data set,
\n\nAre you all parallel processing?
", "rawMarkdown": "So this reduced the data frames, but I still cannot merge them; i'm getting a memory error.\n\nAnyone have any insights on how to merge the entire data set, \n\nAre you all parallel processing? " }, { "id": 544707, "postDate": "2019-06-05T19:40:13.857Z", "content": "This appears in almost every competition but thanks for reposting.
", "rawMarkdown": "This appears in almost every competition but thanks for reposting." }, { "id": 544260, "postDate": "2019-06-05T10:20:34.437Z", "content": "A nice approach and useful function. Thanks for sharing.
", "rawMarkdown": "A nice approach and useful function. Thanks for sharing." }, { "id": 546410, "postDate": "2019-06-06T14:56:54.477Z", "content": "I didn’t know what to deal with this data.\nThanks for your kernel!
", "rawMarkdown": "I didn’t know what to deal with this data.\nThanks for your kernel!", "isDeleted": true } ], "comments": [ { "id": 544332, "author_name": "FGPC", "author_url": "", "post_date": "2019-06-05T11:56:57.987000", "content": "Thanks again. But could you please recognize the original author of this code snippet @gemartin and this is the link of his original work: https://www.kaggle.com/gemartin/load-data-reduce-memory-usage
", "votes": 9, "replies": [ { "id": 545871, "author_name": "Ashish Gupta", "author_url": "", "post_date": "2019-06-06T01:44:25.597000", "content": "Thanks for highlighting it!
", "votes": 1, "replies": [] } ] }, { "id": 554187, "author_name": "Tomás Olarte", "author_url": "", "post_date": "2019-06-17T05:52:53.213000", "content": "Thanks for the method, very useful for this problem in my 8 GB RAM laptop.
", "votes": 0, "replies": [] }, { "id": 550698, "author_name": "arsosa", "author_url": "", "post_date": "2019-06-12T00:00:01.960000", "content": "So this reduced the data frames, but I still cannot merge them; i'm getting a memory error.
\n\nAnyone have any insights on how to merge the entire data set,
\n\nAre you all parallel processing?
", "votes": 0, "replies": [] }, { "id": 544707, "author_name": "RNA", "author_url": "", "post_date": "2019-06-05T19:40:13.857000", "content": "This appears in almost every competition but thanks for reposting.
", "votes": 0, "replies": [] }, { "id": 544260, "author_name": "Murat Ozturkmen", "author_url": "", "post_date": "2019-06-05T10:20:34.437000", "content": "A nice approach and useful function. Thanks for sharing.
", "votes": 0, "replies": [] }, { "id": 546410, "author_name": "", "author_url": "", "post_date": "2019-06-06T14:56:54.477000", "content": "I didn’t know what to deal with this data.\nThanks for your kernel!
", "votes": 0, "replies": [] } ], "raw_markdown_by_id": { "540562": "```\ndef reduce_mem_usage(df, verbose=True):\n numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n start_mem = df.memory_usage().sum() / 1024**2 \n for col in df.columns:\n col_type = df[col].dtypes\n if col_type in numerics:\n c_min = df[col].min()\n c_max = df[col].max()\n if str(col_type)[:3] == 'int':\n if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n df[col] = df[col].astype(np.int8)\n elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n df[col] = df[col].astype(np.int16)\n elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n df[col] = df[col].astype(np.int32)\n elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n df[col] = df[col].astype(np.int64) \n else:\n if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n df[col] = df[col].astype(np.float16)\n elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n df[col] = df[col].astype(np.float32)\n else:\n df[col] = df[col].astype(np.float64) \n end_mem = df.memory_usage().sum() / 1024**2\n if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n return df\n```\n\n\n```\ntrain = reduce_mem_usage(pd.read_csv('../input/train.csv'))\ntest = reduce_mem_usage(pd.read_csv('../input/test.csv'))\nmulliken = reduce_mem_usage(pd.read_csv('../input/mulliken_charges.csv'))\ndipole = reduce_mem_usage(pd.read_csv('../input/dipole_moments.csv'))\nstructure = reduce_mem_usage(pd.read_csv('../input/structures.csv'))\npotential = reduce_mem_usage(pd.read_csv('../input/potential_energy.csv'))\nmagnetic = reduce_mem_usage(pd.read_csv('../input/magnetic_shielding_tensors.csv'))\nscaler = reduce_mem_usage(pd.read_csv('../input/scalar_coupling_contributions.csv'))\nsub = reduce_mem_usage(pd.read_csv('../input/sample_submission.csv'))\n```\n\nMem. usage decreased to 106.62 Mb (50.0% reduction)\nMem. usage decreased to 52.57 Mb (45.0% reduction)\nMem. usage decreased to 16.09 Mb (54.2% reduction)\nMem. usage decreased to 1.13 Mb (56.2% reduction)\nMem. usage decreased to 51.74 Mb (52.1% reduction)\nMem. usage decreased to 0.81 Mb (37.5% reduction)\nMem. usage decreased to 39.49 Mb (69.3% reduction)\nMem. usage decreased to 115.50 Mb (59.4% reduction)\nMem. usage decreased to 11.95 Mb (68.7% reduction)", "544332": "Thanks again. But could you please recognize the original author of this code snippet @gemartin and this is the link of his original work: https://www.kaggle.com/gemartin/load-data-reduce-memory-usage", "554187": "Thanks for the method, very useful for this problem in my 8 GB RAM laptop.", "550698": "So this reduced the data frames, but I still cannot merge them; i'm getting a memory error.\n\nAnyone have any insights on how to merge the entire data set, \n\nAre you all parallel processing? ", "544707": "This appears in almost every competition but thanks for reposting.", "544260": "A nice approach and useful function. Thanks for sharing.", "546410": "I didn’t know what to deal with this data.\nThanks for your kernel!" } }