{"cells":[{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"f900a536-cb80-2665-d915-41fb4ff68a3b"},"outputs":[],"source":"import pandas as pd\nimport numpy as np\n\nfiles =['sample_submission.csv','test_categorical.csv','test_date.csv','test_numeric.csv','train_categorical.csv','train_date.csv','train_numeric.csv']\n#Get my first sumission in... hooked now :)\nsub = pd.read_csv('../input/sample_submission.csv')\n#sub.to_csv('sample_submission.csv', index=False)\nprint(len(sub))"},{"cell_type":"markdown","metadata":{"_cell_guid":"6dd49588-0838-9452-bb86-132008f7bfd3"},"source":"TO DO:\n======\n\nLoad files in chunks and append to sparse matrix or dictionary\n----------------"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"79e31ff9-7205-f88e-2c72-56ce9987a736"},"outputs":[],"source":"from IPython.display import display, HTML\nimport time; start_time = time.time()\nimport math\nfrom sklearn.linear_model import PassiveAggressiveRegressor\nfrom sklearn.feature_extraction.text import HashingVectorizer\nfrom sklearn.externals import joblib\nfrom scipy import sparse\n\nd = {}\ndf_train = pd.read_csv('../input/train_categorical.csv', \n                       low_memory=False, #remove warning\n                       encoding=\"ISO-8859-1\", \n                       #usecols=[], \n                       #dtype={}, \n                       iterator=True, \n                       chunksize=10000) #change to memory capacity\ni = 0\nfor df in df_train:\n    #start sparce matrix or dictionary\n    #joblib.dump(model.best_estimator_, 'clf' + str(i) + '.pkl', compress = 1)\n    #model = joblib.load('clf' + str(i) + '.pkl')\n    display(df.head(3))\n    d = df[:10].to_dict(orient='dict')\n    #display(d)\n    #d = df[:10].to_sparse()\n    #display(d)\n    d = sparse.coo_matrix(df[:10])\n    display(d)\n    break"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"f5152b2d-bd3c-374b-15bd-34df0ae915a7"},"outputs":[],"source":"id_test = []\ny_pred = []\nreg = PassiveAggressiveRegressor(warm_start=True, random_state=123)\ndf_train = pd.read_csv('../input/train_numeric.csv', iterator=True, chunksize=50000)\ni = 0\nfor df in df_train:\n    print(i)\n    df = df.fillna(-1.0)\n    col = [x for x in df.columns if x not in ['Id','Response']]\n    model = reg.fit(df[col], df['Response'])\n    if i > 10:\n        break\n    i += 1\ndf_train = ''\n\ndf_test = pd.read_csv('../input/test_numeric.csv', iterator=True, chunksize=50000)\nfor df in df_test:\n    df = df.fillna(-1.0)\n    col = [x for x in df.columns if x not in ['Id','Response']]\n    id_test += list(df['Id'].values)\n    y_pred += list(model.predict(df[col]))\ndf_test = ''\nfor i in range(len(y_pred)):\n    if y_pred[i]<0.3:\n        y_pred[i] = 0\n    else:\n        y_pred[i] = 1       \npd.DataFrame({\"Id\": id_test, \"Response\": y_pred}).to_csv('submission_par.csv',index=False)\nprint(\"Retreat! : %s minutes ---\" % round(((time.time() - start_time)/60),2))"}],"metadata":{"_change_revision":0,"_is_fork":false,"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.5.2"}},"nbformat":4,"nbformat_minor":0}