{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport gzip","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# reading offer details\noffers = pd.read_csv('../input/acquire-valued-shoppers-challenge/transactions.csv.gz', compression='gzip')\nprint(offers.shape)\noffers.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#file = gz.open('../input/acquire-valued-shoppers-challenge/transactions.csv.gz')\nwith gzip.open('../input/acquire-valued-shoppers-challenge/transactions.csv.gz', 'rb', compresslevel=9) as f:\n    file_content = f.read()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#checking how many unique values are there in each column\noffers.nunique()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from datetime import datetime\n\nloc_offers = \"../input/acquire-valued-shoppers-challenge/offers.csv.gz\"\nloc_transactions = \"../input/acquire-valued-shoppers-challenge/transactions.csv.gz\"\nloc_reduced = \"reduced2.csv\" # will be created\n\ndef reduce_data(loc_offers, loc_transactions, loc_reduced):\n\n  start = datetime.now()\n  #get all categories on offer in a dict\n  offers = {}\n  for e, line in enumerate(open(loc_offers)):\n    offers[ line.split(\",\")[1] ] = 1\n  #open output file\n  with open(loc_reduced, \"wb\") as outfile:\n    #go through transactions file and reduce\n    reduced = 0\n    for e, line in enumerate(open(loc_transactions)):\n      if e == 0:\n        outfile.write( line ) #print header\n      else:\n        #only write when category in offers dict\n          if line.split(\",\")[3] in offers:\n            outfile.write( line )\n            reduced += 1\n      #progress\n      if e % 5000000 == 0:\n        print (e, reduced, datetime.now() - start)\n  print (e, reduced, datetime.now() - start)\n\nreduce_data(loc_offers, loc_transactions, loc_reduced)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}