{"cells":[{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"cb859a34-e609-ab94-e766-352ae02dd953"},"outputs":[],"source":"import numpy as np\nimport pandas as pd\nfrom random import random\nfrom itertools import chain\nimport scipy\nimport scipy.sparse as spar\n\n\ndef sparse_dummies(categories):\n    num_rows = categories.shape[0]\n    num_cats = len(categories.values.categories)\n    categories = categories.reset_index(drop=True)[categories.values.codes > -1]\n    data = np.ones(categories.shape[0])\n    return spar.csr_matrix(\n        (data, (categories.index.values, categories.values.codes)),\n        shape=(num_rows, num_cats))\n\n\ndef one_hot(df):\n    df = df.apply(lambda col: col.astype('category'), axis=0)\n    cat_counts = df.apply(lambda col: len(col.values.categories), axis=0)\n    header = list(chain.from_iterable(\n    [[colname] * ncats for colname, ncats in zip(cat_counts.index, cat_counts.values)]))\n    dummy_matrices = list( sparse_dummies(df[col]) for col in df.columns )\n    mtx = spar.hstack(dummy_matrices)\n    return header, mtx.tocsr()\n\n\ndef save_ohe(filename, header, matrix):\n    np.savez(filename,\n        header=header,\n        indices=matrix.indices,\n        indptr=matrix.indptr,\n        shape=matrix.shape)\n\n\ndef load_ohe(filename):\n    loader = np.load(filename)\n    header = loader['header']\n    indices = loader['indices']\n    indptr = loader['indptr']\n    data = np.ones(len(indices))\n    mtx = csr_matrix((data, indices, indptr), shape = loader['shape'])\n    return header, mtx\n\n\ndef chunker(seq, size):\n    return (seq[pos:pos + size] for pos in range(0, len(seq), size))"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"8ab2bbec-ed61-edd5-bae7-17d0a42a617a"},"outputs":[],"source":"cat_path = '../input/train_categorical.csv'\n\nchunksize = 500\nnrows = 10000\n\ncolumns = pd.read_csv(\n\tcat_path,\n\tindex_col=0,\n\tnrows = 1,\n\tdtype=str).columns.values\n\nprint(len(columns))\n\nheaders = []\nmatrices = []\nprogress = 0\n# read file in vertical chunks\n#for col_subset in chunker(columns, chunksize):\n\n#\tusecols = np.append(['Id'], col_subset)\n#\tdf = pd.read_csv(\n#\t\tcat_path,\n#\t\tindex_col=0,\n#\t\tdtype=str,\n#\t\tusecols=usecols,\n #       nrows=nrows\n#\t)\n\n\n\n#\theader, mtx = one_hot(df)\n#\tmatrices.append(mtx)\n#\theaders.append(header)\n\n#\tprogress += chunksize\n#\tprint(\"progress = {}: {}\".format(progress, mtx.shape))\n\n\n\n#full_header = np.hstack(headers)\n#full_mtx = hstack(matrices).tocsr()\n\n\ndf = pd.read_csv(\n\t\tcat_path,\n\t\tindex_col=0,\n\t\tdtype=str,\n        nrows=nrows\n)\n\nfull_header, full_mtx = one_hot(df)\n\nprint(full_header.shape)\nprint(full_mtx.shape)\n\n\nwith open('train_cat_sparse', 'w') as resfile:\n\tsave_ohe(resfile, full_header, full_mtx)"}],"metadata":{"_change_revision":0,"_is_fork":false,"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.5.2"}},"nbformat":4,"nbformat_minor":0}