{"nbformat": 4, "cells": [{"cell_type": "markdown", "metadata": {"_uuid": "5f8b3a429f0111c68dd0ebe99d277d3b2dc48f49", "_cell_guid": "5f83ed1d-b36f-428d-8340-507a7066dc2c"}, "source": ["This is a simple example to read lines of a csv file with a condition"]}, {"outputs": [], "cell_type": "code", "metadata": {"_uuid": "9b34a95a7a5213826b3e15292e69c70e8e975889", "_kg_hide-input": true, "scrolled": true, "collapsed": true, "_cell_guid": "3d120c78-15d3-4c34-b656-70633e124f4c"}, "execution_count": null, "source": ["import numpy as np\n", "import os\n", "import pandas as pd\n", "\n", "PATH = \"../input\"\n", "\n", "dtypes = {\n", "          'id' : np.dtype('int64'),\n", "          'store_nbr': np.dtype('int64'),\n", "          'item_nbr': np.dtype('int64'),\n", "          'unit_sales': np.dtype('float64'),\n", "          'onpromotion': np.dtype('O')}\n", "\n", "def load_train_data(data_path=PATH, file_name=\"train.csv\", condition=None, dtypes=None):\n", "    df = pd.DataFrame( )\n", "    csv_path = os.path.join(data_path, file_name)\n", "    reader = pd.read_csv(csv_path, parse_dates=['date'], dtype=dtypes , iterator=True,  chunksize=10000000)\n", "    return pd.concat([chunk.query(condition) for chunk in reader])\n", "   \n", "        \n", "train = load_train_data(data_path=PATH, file_name=\"train.csv\", condition=\"item_nbr == 103665\", dtypes=dtypes)\n", "\n", "train.shape"]}, {"outputs": [], "cell_type": "code", "metadata": {"_uuid": "fb8d48a81182d619f8a3cdfac4e4c52f1db3abe8", "collapsed": true, "_cell_guid": "04fed9bc-ab06-4181-b6dc-54a3a88c1cc3"}, "execution_count": null, "source": ["train.head()"]}, {"outputs": [], "cell_type": "code", "metadata": {"_uuid": "31bda1581ccbe411ce2d64942c0685eb0d11e458", "collapsed": true, "_cell_guid": "c0cbca69-f478-4fc0-8a4a-a86d06442042"}, "execution_count": null, "source": ["import gc\n", "gc.collect()"]}], "metadata": {"language_info": {"name": "python", "codemirror_mode": {"name": "ipython", "version": 3}, "nbconvert_exporter": "python", "version": "3.6.3", "mimetype": "text/x-python", "pygments_lexer": "ipython3", "file_extension": ".py"}, "kernelspec": {"name": "python3", "language": "python", "display_name": "Python 3"}}, "nbformat_minor": 1}