{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import pandas as pd","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7bac5177d4eaf6fa9cfb14ee1d7c1cfee43166e8"},"cell_type":"code","source":"hist = pd.read_csv('../input/historical_transactions.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3b9942f9d840175b3fb9a717aa59d64ebaf6db6e"},"cell_type":"code","source":"hist.info(memory_usage='deep')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a5ef69e26da9908afc595e309d569be33fbe4a9e"},"cell_type":"markdown","source":"13.1 GB is huge memory foot print. Let's see if we can smartly use dtypes and reduce size. 29,112,361 total rows and 14 columns."},{"metadata":{"trusted":true,"_uuid":"a08add2cb38bf5fb8158d693766c9a9a154b74dd"},"cell_type":"code","source":"hist.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c5c5139a1c89f0bf7e54656fe2fd587dad2392ad"},"cell_type":"code","source":"hist.authorized_flag.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"50d7caa32d9fc26cdc1a0d8f73d9b5a84eae58fc"},"cell_type":"markdown","source":"Convert to category type as it will have less memory need."},{"metadata":{"trusted":true,"_uuid":"339a65d366ce6d43afd7da37310b5214a8b16f1a"},"cell_type":"code","source":"hist.authorized_flag = hist.authorized_flag.astype('category')\nhist.info(memory_usage='deep')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"14bae4cd34e531e501f69b71a6e1206758b309ef"},"cell_type":"markdown","source":"1.8GB reduction. On right path.. Let's see how far we can optimise this."},{"metadata":{"trusted":true,"_uuid":"448354c439e7f44443204c53c8249a610a37f01c"},"cell_type":"code","source":"hist.category_1.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f5efbc134de6e603e27706637c35dfd0c1d9b192"},"cell_type":"code","source":"hist.category_1 = hist.category_1.astype('category')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d2c33543f268b2fa5ed5e8cb851ec063ed8008dc"},"cell_type":"code","source":"hist.category_2.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"add06ecc1f2194dbeb3375b27440902de226c5ed"},"cell_type":"markdown","source":"It is tempting to change it to int. However if we see the data dictionary, it is just anonymised category like other categories. So 1.0 does not necessarily mean better/worst than 2.0. So it is similar to category 1. So let's use 'category' dtype."},{"metadata":{"trusted":true,"_uuid":"67759fe930a16181d2de18356c7f249aec205329"},"cell_type":"code","source":"hist.category_2 = hist.category_2.astype('category')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0d35b9e82d1de51c206e6668a4ebe8f42df0f6d9"},"cell_type":"code","source":"hist.category_3.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"32a74388248725496d0584ea4a72993a78504b00"},"cell_type":"code","source":"hist.category_3 = hist.category_3.astype('category')\nhist.info(memory_usage= 'deep')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b2e55323e5e97adc43957b4851b1c134e526ff04"},"cell_type":"code","source":"hist.city_id.max(),hist.city_id.min(),sum(hist.city_id.isnull())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"47511dac5f38005a57eff14024756b818010cd46"},"cell_type":"code","source":"hist.city_id = hist.city_id.astype('int16')\nhist.city_id.max(),hist.city_id.min(),sum(hist.city_id.isnull())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"566b4de5b78b13508695172506d5fb16a4b32a0b"},"cell_type":"code","source":"hist.installments.max(),hist.installments.min(),sum(hist.installments.isnull())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9b72739f6c8ff381b7326092d2eae160c1d8f56f"},"cell_type":"code","source":"hist.installments = hist.installments.astype('int16')\nhist.installments.max(),hist.installments.min(),sum(hist.installments.isnull())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"187ca40b6f004492ff205bda1c1998da174dcfcf"},"cell_type":"code","source":"hist.month_lag.max(),hist.month_lag.min(),sum(hist.month_lag.isnull())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0f342cacdb13e0ef6b63c7da3dc827e4a67c5f29"},"cell_type":"code","source":"hist.month_lag = hist.month_lag.astype('int16')\nhist.month_lag.max(),hist.month_lag.min(),sum(hist.month_lag.isnull())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dee7041d87e405bbc1032ebfbbaed7bc7b3773b4"},"cell_type":"code","source":"hist.state_id.max(),hist.state_id.min(),sum(hist.state_id.isnull())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"07237ee7b65083c0ba66910ebf17ae4d7570a9e2"},"cell_type":"code","source":"hist.state_id = hist.state_id.astype('int16')\nhist.state_id.max(),hist.state_id.min(),sum(hist.state_id.isnull())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"acc145fba66f7864f67474a2be7cca4cbc0eeea7"},"cell_type":"code","source":"hist.subsector_id.max(),hist.subsector_id.min(),sum(hist.subsector_id.isnull())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9456ab3b7ff75f643d7461c60633296b0407cf49"},"cell_type":"code","source":"hist.subsector_id = hist.subsector_id.astype('int8')\nhist.subsector_id.max(),hist.subsector_id.min(),sum(hist.subsector_id.isnull())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"96faff75cb83a46b35bbfcc7f2974ac8259918d7"},"cell_type":"code","source":"hist.info(memory_usage='deep')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"14e31fa7ac21d53bdf1f1ad5bd50cbd214ccb0d4"},"cell_type":"markdown","source":"6.7 GB... Almost 50% reduction. There is no more straight downcast options available.  The remaining option is card_id and merchant_id. They are anonymised string objects. One option to make these workable is replace sequential integers for each uique ids (both merchant and card ids). Slice these mapping of objects and int mappings in other dataframes which can be later used for joining with other datasets."},{"metadata":{"trusted":true,"_uuid":"b5759181981d76b9991f3ffa3634e20e0d08ccf6"},"cell_type":"code","source":"unique_merchants = pd.DataFrame(hist.merchant_id.value_counts())\nunique_merchants.reset_index(inplace = True)\nunique_merchants.reset_index(inplace = True)\nunique_merchants.columns = ['merchant_id_code','merchant_id','count']\nunique_merchants.drop(columns = ['count'],inplace = True)\nunique_merchants.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"86fe3a9c08e4e5828d91f0453e899b5ee5cc9858"},"cell_type":"code","source":"sum(hist.merchant_id.isnull())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ef14a174a07c134d3249d1b31c0cf4e0508c1d6c"},"cell_type":"markdown","source":"1. merge unique_merchants with hist df. drop merchant_id from hist df. fill nulls with -1 and make the merchant_id_code as int."},{"metadata":{"trusted":true,"_uuid":"5f57242f59516a48fa0ca534116412b6c35f71aa"},"cell_type":"code","source":"hist = hist.merge(unique_merchants, on = 'merchant_id',how = 'left')\nhist.drop(columns = ['merchant_id'],inplace = True)\nhist.merchant_id_code.fillna(value = -1,inplace = True)\nhist.merchant_id_code = hist.merchant_id_code.astype('int32')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"05418844d0e47941471933e29e312f251612a7a1","trusted":true},"cell_type":"code","source":"unique_cards = pd.DataFrame(hist.card_id.value_counts())\nunique_cards.reset_index(inplace = True)\nunique_cards.reset_index(inplace = True)\nunique_cards.columns = ['card_id_code','card_id','count']\nunique_cards.drop(columns = ['count'],inplace = True)\nunique_cards.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bda7495d7db637a07c1bea818f58ce0fbc04b8b8"},"cell_type":"code","source":"hist = hist.merge(unique_cards, on = 'card_id',how = 'left')\nhist.drop(columns = ['card_id'],inplace = True)\nhist.card_id_code.fillna(value=-1,inplace = True)\nhist.card_id_code = hist.card_id_code.astype('int32')\nhist.info(memory_usage='deep')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"858c40a86ec5100977670633871056d74a08a227"},"cell_type":"code","source":"unique_merchants.info(memory_usage='deep')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b7bfc9e2a7f79c112b90b537d8d2267513e8e58b"},"cell_type":"code","source":"unique_cards.info(memory_usage='deep')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1f0043c538474f40c39a46ae94c565f0d054d899"},"cell_type":"markdown","source":"13.1 GB df is converted into 3.3 gb df and 2 dfs with 25mb. So total reduction of 74% reduction !!! That is pretty good progress. Now time to do EDA....."},{"metadata":{"trusted":true,"_uuid":"d6e32b95ed3d10054067df343988964a4cd558d6"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}