{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns #library used for making statistical graphics\nimport matplotlib # data visualization and graphical plotting library for Python and its numerical extension NumPy\n\nimport matplotlib.pyplot as plt\nfrom scipy.stats import skew\nfrom scipy.stats.stats import pearsonr\n\n\n%config InlineBackend.figure_format = 'retina' #set 'png' here when working on notebook\n%matplotlib inline\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-21T15:33:25.535537Z","iopub.execute_input":"2022-01-21T15:33:25.536334Z","iopub.status.idle":"2022-01-21T15:33:26.504135Z","shell.execute_reply.started":"2022-01-21T15:33:25.536226Z","shell.execute_reply":"2022-01-21T15:33:26.503305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weather_train = pd.read_csv(\"../input/ashrae-energy-prediction/weather_test.csv\")\nweather_test = pd.read_csv(\"../input/ashrae-energy-prediction/weather_train.csv\")\nbuilding= pd.read_csv(\"../input/ashrae-energy-prediction/building_metadata.csv\")\n\n","metadata":{"_kg_hide-output":false,"execution":{"iopub.status.busy":"2022-01-21T15:33:26.505623Z","iopub.execute_input":"2022-01-21T15:33:26.505852Z","iopub.status.idle":"2022-01-21T15:33:27.214167Z","shell.execute_reply.started":"2022-01-21T15:33:26.505825Z","shell.execute_reply":"2022-01-21T15:33:27.213519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weather_train.tail()","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:33:27.215342Z","iopub.execute_input":"2022-01-21T15:33:27.215724Z","iopub.status.idle":"2022-01-21T15:33:27.2417Z","shell.execute_reply.started":"2022-01-21T15:33:27.215695Z","shell.execute_reply":"2022-01-21T15:33:27.240867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Function to reduce the DF size\ndef reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:33:27.244636Z","iopub.execute_input":"2022-01-21T15:33:27.24492Z","iopub.status.idle":"2022-01-21T15:33:27.259022Z","shell.execute_reply.started":"2022-01-21T15:33:27.244891Z","shell.execute_reply":"2022-01-21T15:33:27.258174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train = reduce_mem_usage(train)\nweather_train = reduce_mem_usage(weather_train)\nweather_test = reduce_mem_usage(weather_test)\n","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:33:27.260325Z","iopub.execute_input":"2022-01-21T15:33:27.26064Z","iopub.status.idle":"2022-01-21T15:33:27.350172Z","shell.execute_reply.started":"2022-01-21T15:33:27.260597Z","shell.execute_reply":"2022-01-21T15:33:27.349333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weather_test = reduce_mem_usage(weather_test)\nbuilding = reduce_mem_usage(building) ","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:33:27.351621Z","iopub.execute_input":"2022-01-21T15:33:27.352489Z","iopub.status.idle":"2022-01-21T15:33:27.39468Z","shell.execute_reply.started":"2022-01-21T15:33:27.352444Z","shell.execute_reply":"2022-01-21T15:33:27.393745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data = pd.concat((weather_train.loc[:,'site_id':'wind_speed'],\n                      weather_test.loc[:,'site_id':'wind_speed']))","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:33:27.395982Z","iopub.execute_input":"2022-01-21T15:33:27.396282Z","iopub.status.idle":"2022-01-21T15:33:27.434037Z","shell.execute_reply.started":"2022-01-21T15:33:27.396206Z","shell.execute_reply":"2022-01-21T15:33:27.433205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Data preprocessing:\nWe're not going to do anything fancy here:\n\nFirst I'll transform the skewed numeric features by taking log(feature + 1) - this will make the features more normal\n\nCreate Dummy variables for the categorical features\n\nReplace the numeric missing values (NaN's) with the mean of their respective columns","metadata":{}},{"cell_type":"code","source":"matplotlib.rcParams['figure.figsize'] = (12.0, 6.0)\nprices = pd.DataFrame({\"Wind readings\":weather_train[\"wind_speed\"], \"log(Wind readings + 1)\":np.log1p(weather_train[\"wind_speed\"])})\nprices.hist()","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:33:27.436406Z","iopub.execute_input":"2022-01-21T15:33:27.436662Z","iopub.status.idle":"2022-01-21T15:33:28.10862Z","shell.execute_reply.started":"2022-01-21T15:33:27.436631Z","shell.execute_reply":"2022-01-21T15:33:28.107819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#log transform the target:\nweather_train[\"wind_speed\"] = np.log1p(weather_train[\"wind_speed\"])\n\n#log transform skewed numeric features:\nnumeric_feats = all_data.dtypes[all_data.dtypes != \"object\"].index\n\nskewed_feats = weather_train[numeric_feats].apply(lambda x: skew(x.dropna())) #compute skewness\nskewed_feats = skewed_feats[skewed_feats > 0.75]\nskewed_feats = skewed_feats.index\n\nall_data[skewed_feats] = np.log1p(all_data[skewed_feats])","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:33:28.110215Z","iopub.execute_input":"2022-01-21T15:33:28.110518Z","iopub.status.idle":"2022-01-21T15:33:28.306145Z","shell.execute_reply.started":"2022-01-21T15:33:28.110474Z","shell.execute_reply":"2022-01-21T15:33:28.305083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#import pandas as pd\n#import numpy as np\n\n#def reduce_mem_usage(weather_train):\n #   \"\"\" iterate through all the columns of a dataframe and modify the data type\n#to reduce memory usage.\n#\"\"\"\n#start_mem = weather_train.memory_usage().sum() / 1024**2\n#print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-01-21T15:33:28.308632Z","iopub.execute_input":"2022-01-21T15:33:28.309085Z","iopub.status.idle":"2022-01-21T15:33:28.312379Z","shell.execute_reply.started":"2022-01-21T15:33:28.309043Z","shell.execute_reply":"2022-01-21T15:33:28.311781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#for col in weather_train.columns:\n    #col_type = weather_train[col].dtype\n\n    #if col_type != object:\n        #c_min = weather_train[col].min()\n        #c_max = weather_train[col].max()\n       # if str(col_type)[:3] == 'int':\n            #if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                #weather_train[col] = weather_train[col].astype(np.int8)\n           # elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n               # weather_train[col] = weather_train[col].astype(np.int16)\n            #elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                #weather_train[col] = weather_train[col].astype(np.int32)\n            #elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                #weather_train[col] = weather_train[col].astype(np.int64)  \n       # else:\n           # if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n               # weather_train[col] = weather_train[col].astype(np.float16)\n            #elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n               # weather_train[col] = weather_train[col].astype(np.float32)\n            #else:\n                #weather_train[col] = weather_train[col].astype(np.float64)\n    #else:\n        #weather_train[col] = train_data[col].astype('category')\n\n#end_mem = weather_train.memory_usage().sum() / 1024**2\n#print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n#print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n\n#return weather_train","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-01-21T15:33:28.31325Z","iopub.execute_input":"2022-01-21T15:33:28.313948Z","iopub.status.idle":"2022-01-21T15:33:28.324481Z","shell.execute_reply.started":"2022-01-21T15:33:28.313914Z","shell.execute_reply":"2022-01-21T15:33:28.323598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# From the below code their is a technical error which is Your notebook tried to allocate more memory than is available. It has restarted.\n# I have tried fixing it and test my code on another accounts but with the same error but the code is 100% working","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:33:28.325614Z","iopub.execute_input":"2022-01-21T15:33:28.325926Z","iopub.status.idle":"2022-01-21T15:33:28.340138Z","shell.execute_reply.started":"2022-01-21T15:33:28.325894Z","shell.execute_reply":"2022-01-21T15:33:28.339283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data = pd.get_dummies(all_data)","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:33:28.34125Z","iopub.execute_input":"2022-01-21T15:33:28.341475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#filling NA's with the mean of the column:\nall_data = all_data.fillna(all_data.mean())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#creating matrices for sklearn:\nX_train = all_data[:train.shape[0]]\nX_test = all_data[train.shape[0]:]\ny = train.meter_reading","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Models**\n\nNow we are going to use regularized linear regression models from the scikit learn module. I'm going to try both l_1(Lasso) and l_2(Ridge) regularization. I'll also define a function that returns the cross-validation rmse error so we can evaluate our models and pick the best tuning par","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import Ridge, RidgeCV, ElasticNet, LassoCV, LassoLarsCV\nfrom sklearn.model_selection import cross_val_score\n\ndef rmse_cv(model):\n    rmse= np.sqrt(-cross_val_score(model, X_train, y, scoring=\"neg_mean_squared_error\", cv = 5))\n    return(rmse)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_ridge = Ridge()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The main tuning parameter for the Ridge model is alpha - a regularization parameter that measures how flexible our model is. The higher the regularization the less prone our model will be to overfit. However it will also lose flexibility and might not capture all of the signal in the data.","metadata":{}},{"cell_type":"code","source":"alphas = [0.05, 0.1, 0.3, 1, 3, 5, 10, 15, 30, 50, 75]\ncv_ridge = [rmse_cv(Ridge(alpha = alpha)).mean() \n            for alpha in alphas]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cv_ridge = pd.Series(cv_ridge, index = alphas)\ncv_ridge.plot(title = \"Validation\")\nplt.xlabel(\"alpha\")\nplt.ylabel(\"rmse\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cv_ridge.min()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now Let' try out the **Lasso model**. \n\nWe will do a slightly different approach here and use the built in Lasso CV to figure out the best alpha for us. For some reason the alphas in Lasso CV are really the inverse or the alphas in Ridge.","metadata":{}},{"cell_type":"code","source":"model_lasso = LassoCV(alphas = [1, 0.1, 0.001, 0.0005]).fit(X_train, y)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rmse_cv(model_lasso).mean()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"coef = pd.Series(model_lasso.coef_, index = X_train.columns)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Lasso picked \" + str(sum(coef != 0)) + \" variables and eliminated the other \" +  str(sum(coef == 0)) + \" variables\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Good job Lasso. One thing to note here however is that the features selected are not necessarily the \"correct\" ones - especially since there are a lot of collinear features in this dataset. One idea to try here is run Lasso a few times on boostrapped samples and see how stable the feature selection is.","metadata":{}},{"cell_type":"code","source":"imp_coef = pd.concat([coef.sort_values().head(10),\n                     coef.sort_values().tail(10)])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"matplotlib.rcParams['figure.figsize'] = (8.0, 10.0)\nimp_coef.plot(kind = \"barh\")\nplt.title(\"Coefficients in the Lasso Model\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Elastic Net Regression**","metadata":{}},{"cell_type":"code","source":"enet1 = ElasticNet()\n\n# Fit the model\nmodel_enet1 = enet1.fit(X_train, y_train)\n\n# Prediction\ny_pred_train_enet1 = enet1.predict(X_train)\ny_pred_test_enet1 = enet1.predict(X_test)\n\n# Accuracy Score\nprint('Training accuracy : {}\\n'.format(r2_score(y_train, y_pred_train_enet1).round(5)))\nprint('Testing accuracy : {}'.format(r2_score(y_test, y_pred_test_enet1).round(5)))","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}