We keeping in mind, there are mostly categorical values with such unique numbers of them:
\n\ntrain.csv \nAgencia_ID: # of unique = 552 \nCanal_ID: # of unique = 9\nRuta_SAK: # of unique = 3603 \nCliente_ID: # of unique = 880604\nProducto_ID: # of unique = 1799\nDemanda_uni_equil will be used as a target. We will iterate through the file to be able to store all data in the memory.
\n\nDue to the fact, that there are a lot clients, that column will droped.
\n\nimport numpy as np\nimport pandas as pd\nimport math\nimport sklearn\n\nfrom sklearn.cross_validation import cross_val_score\nfrom subprocess import check_output\n\nfrom sklearn.metrics import make_scorer, mean_squared_error\nfrom sklearn.cross_validation import train_test_split\nfrom sklearn.preprocessing import normalize\n\nfrom sklearn.linear_model import SGDRegressor\n\nfrom sklearn.preprocessing import OneHotEncoder\n\n # rmsle - error function used in LB\n def rmsle_func(actual, predicted):\n return np.sqrt(msle(actual, predicted))\n def msle(actual, predicted):\n return np.mean(sle(actual, predicted))\n def sle(actual, predicted):\n return (np.power(np.log(np.array(actual)+1) - \n np.log(np.array(predicted)+1), 2))\n# to create post only non negative values\n def nonnegative(x):\n if x > 0:\n return x\n else: \n return 0\n\n# to decrease memory usage: \ndtypes = {'Semana' : 'int32',\n 'Agencia_ID' :'int32',\n 'Canal_ID' : 'int32',\n 'Ruta_SAK' : 'int32',\n 'Cliente-ID' : 'int32',\n 'Producto_ID':'int32',\n 'Venta_hoy':'float32',\n 'Venta_uni_hoy': 'int32',\n 'Dev_uni_proxima':'int32',\n 'Dev_proxima':'float32',\n 'Demanda_uni_equil':'int32'}\n\nmodel = SGDRegressor(loss='squared_loss', penalty='l2', alpha=0.0001, \n fit_intercept=True, n_iter=10, shuffle=True, verbose=0, \n epsilon=0.1, learning_rate='invscaling', \n eta0=0.01, power_t=0.25, warm_start=True, average=False)\n\nfrom sklearn.feature_extraction import FeatureHasher\nh = FeatureHasher(n_features=8000, input_type = 'string') #8000 - the number of total unique values over all data\n\n\ndf_train = pd.read_csv('train.csv', dtype = dtypes, usecols=["Semana", "Agencia_ID", "Canal_ID", 'Ruta_SAK',\n 'Producto_ID','Demanda_uni_equil'], chunksize=1500)\n\n# use astype('str') in python 3\ni = 1\nnum = 100# to limit train size\n#pd.concat([train, pd.get_dummies(train['Semana'],sparse=True)], axis=1, join_axes=[train.index])\nfor chunk in df_train:\n if i < num :\n X_chunk = h.fit_transform(chunk[["Semana", "Agencia_ID", "Canal_ID", 'Ruta_SAK', 'Producto_ID']].astype('string').as_matrix())\n y_chunk = np.ravel(chunk[['Demanda_uni_equil']].as_matrix())\n\n model.partial_fit(X_chunk, y_chunk)\n i = i + 1\n elif i == num:\n X_chunk = h.fit_transform(chunk[["Semana", "Agencia_ID", "Canal_ID", 'Ruta_SAK','Producto_ID']].astype('string').values)\n y_chunk = np.ravel(chunk[['Demanda_uni_equil']].values)\n\n print ('rmsle: ', rmsle_func(y_chunk, model.predict(X_chunk)))\n print ('RMSE ', math.sqrt(sklearn.metrics.mean_squared_error(y_chunk, model.predict(X_chunk))))\n i = i + 1\n else:\n break\nprint 'Finished the fitting'\n\n#predict\n\n X_test = pd.read_csv('../input/test.csv',dtype = dtypes,usecols=['id', "Semana", "Agencia_ID", "Canal_ID", 'Ruta_SAK',\n 'Producto_ID'])\nids = X_test['id']\nX_test.drop(['id'], axis =1, inplace = True)\n\ny_predicted = model.predict(h.fit_transform(X_test.astype('string').values))\n\n\n\n# submission \nsubmission = pd.DataFrame({"id":ids, "Demanda_uni_equil": y_predicted})\n\ny_predicted = map(nonnegative, y_predicted)\n\nsubmission = pd.DataFrame({"id":ids, "Demanda_uni_equil": y_predicted})\ncols = ['id',"Demanda_uni_equil"]\nsubmission = submission[cols]\nsubmission.to_csv("submission.csv", index=False)\n\n\nprint('Completed!')\nWe keeping in mind, there are mostly categorical values with such unique numbers of them:
\n\ntrain.csv \nAgencia_ID: # of unique = 552 \nCanal_ID: # of unique = 9\nRuta_SAK: # of unique = 3603 \nCliente_ID: # of unique = 880604\nProducto_ID: # of unique = 1799\nDemanda_uni_equil will be used as a target. We will iterate through the file to be able to store all data in the memory.
\n\nDue to the fact, that there are a lot clients, that column will droped.
\n\nimport numpy as np\nimport pandas as pd\nimport math\nimport sklearn\n\nfrom sklearn.cross_validation import cross_val_score\nfrom subprocess import check_output\n\nfrom sklearn.metrics import make_scorer, mean_squared_error\nfrom sklearn.cross_validation import train_test_split\nfrom sklearn.preprocessing import normalize\n\nfrom sklearn.linear_model import SGDRegressor\n\nfrom sklearn.preprocessing import OneHotEncoder\n\n # rmsle - error function used in LB\n def rmsle_func(actual, predicted):\n return np.sqrt(msle(actual, predicted))\n def msle(actual, predicted):\n return np.mean(sle(actual, predicted))\n def sle(actual, predicted):\n return (np.power(np.log(np.array(actual)+1) - \n np.log(np.array(predicted)+1), 2))\n# to create post only non negative values\n def nonnegative(x):\n if x > 0:\n return x\n else: \n return 0\n\n# to decrease memory usage: \ndtypes = {'Semana' : 'int32',\n 'Agencia_ID' :'int32',\n 'Canal_ID' : 'int32',\n 'Ruta_SAK' : 'int32',\n 'Cliente-ID' : 'int32',\n 'Producto_ID':'int32',\n 'Venta_hoy':'float32',\n 'Venta_uni_hoy': 'int32',\n 'Dev_uni_proxima':'int32',\n 'Dev_proxima':'float32',\n 'Demanda_uni_equil':'int32'}\n\nmodel = SGDRegressor(loss='squared_loss', penalty='l2', alpha=0.0001, \n fit_intercept=True, n_iter=10, shuffle=True, verbose=0, \n epsilon=0.1, learning_rate='invscaling', \n eta0=0.01, power_t=0.25, warm_start=True, average=False)\n\nfrom sklearn.feature_extraction import FeatureHasher\nh = FeatureHasher(n_features=8000, input_type = 'string') #8000 - the number of total unique values over all data\n\n\ndf_train = pd.read_csv('train.csv', dtype = dtypes, usecols=["Semana", "Agencia_ID", "Canal_ID", 'Ruta_SAK',\n 'Producto_ID','Demanda_uni_equil'], chunksize=1500)\n\n# use astype('str') in python 3\ni = 1\nnum = 100# to limit train size\n#pd.concat([train, pd.get_dummies(train['Semana'],sparse=True)], axis=1, join_axes=[train.index])\nfor chunk in df_train:\n if i < num :\n X_chunk = h.fit_transform(chunk[["Semana", "Agencia_ID", "Canal_ID", 'Ruta_SAK', 'Producto_ID']].astype('string').as_matrix())\n y_chunk = np.ravel(chunk[['Demanda_uni_equil']].as_matrix())\n\n model.partial_fit(X_chunk, y_chunk)\n i = i + 1\n elif i == num:\n X_chunk = h.fit_transform(chunk[["Semana", "Agencia_ID", "Canal_ID", 'Ruta_SAK','Producto_ID']].astype('string').values)\n y_chunk = np.ravel(chunk[['Demanda_uni_equil']].values)\n\n print ('rmsle: ', rmsle_func(y_chunk, model.predict(X_chunk)))\n print ('RMSE ', math.sqrt(sklearn.metrics.mean_squared_error(y_chunk, model.predict(X_chunk))))\n i = i + 1\n else:\n break\nprint 'Finished the fitting'\n\n#predict\n\n X_test = pd.read_csv('../input/test.csv',dtype = dtypes,usecols=['id', "Semana", "Agencia_ID", "Canal_ID", 'Ruta_SAK',\n 'Producto_ID'])\nids = X_test['id']\nX_test.drop(['id'], axis =1, inplace = True)\n\ny_predicted = model.predict(h.fit_transform(X_test.astype('string').values))\n\n\n\n# submission \nsubmission = pd.DataFrame({"id":ids, "Demanda_uni_equil": y_predicted})\n\ny_predicted = map(nonnegative, y_predicted)\n\nsubmission = pd.DataFrame({"id":ids, "Demanda_uni_equil": y_predicted})\ncols = ['id',"Demanda_uni_equil"]\nsubmission = submission[cols]\nsubmission.to_csv("submission.csv", index=False)\n\n\nprint('Completed!')\n