{"cells":[{"metadata":{"_cell_guid":"3b543ba3-19c5-1d80-5366-1742c46cb54c","_uuid":"598bf440f6c9b4e68c0e987ebf96110b44c29d88"},"cell_type":"markdown","source":"# LASSO regression on a simulated data set\n\nYou wouldn't typically think of using a neural network toolkit to do a lasso regression, but it appears it works just fine! This example uses is based on a simulated regression example with regressors $x_1, x_2, x_3$, where only the $x_1$ has an effect on the response $y$. The data generating equation is:\n\n$$\ny = 1.1 + 3.2 * x_1 + \\epsilon\n$$\nwhere $\\epsilon \\sim N(0, 7.5)$.\n","outputs":[],"execution_count":null},{"metadata":{"_cell_guid":"307d3306-7d8e-18e0-82ab-d8108dc2834f","_uuid":"7dbb03f4854c643aa8a0d72b0ed79dfb03368061","trusted":true},"cell_type":"code","source":"%env KERAS_BACKEND=theano\nimport theano\n\nimport numpy as np\nfrom numpy.random import normal\nfrom numpy.random import seed\nimport matplotlib.pyplot as plt\n\nimport keras\nfrom keras.layers import Input, Dense\nfrom keras.models import Model\nfrom keras.regularizers import l1, l2","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"cf0c27f4-ca13-b7d4-95ca-9c4e0062952e","_uuid":"3a317351d703df6bb37e3f6cb8728b28b2399dfb","trusted":true},"cell_type":"code","source":"# Simulating the data\nseed(44234)\nN = 1000\n\nx = normal(size = N).astype(np.float32).reshape((N, 1))\nz = normal(size = N).astype(np.float32).reshape((N, 1))\nw = normal(size = N).astype(np.float32).reshape((N, 1))\nX = np.concatenate((x, w, z), axis = 1)\nepsilon = normal(size = N, scale = 7.5)\n\ny = 1.1 + 3.2 * X[:, 0] + epsilon\ny = y.astype(np.float32).reshape((N, 1))\n\nplt.scatter(X[:, 0], y[:, 0])\nplt.title(\"y vs x_1: positive relationship\")\nplt.show()\n\nplt.scatter(X[:, 1], y[:, 0])\nplt.title(\"y vs x_2: no relationship\")\nplt.show()\n\nplt.scatter(X[:, 2], y[:, 0])\nplt.title(\"y vs x_3: no relationship\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"87be038f-e4a5-0bf4-d864-508de2cc14af","_uuid":"6ad24c52a665240416e8c123d1350f3e326ea7e0"},"cell_type":"markdown","source":"# Ordinary regression estimation with stochastic gradient descent\nWithout regularization, the regression below estimates all coeficients with non-zero values, though the estimates for $\\beta_2$ and $\\beta_3$ are relatively small.","outputs":[],"execution_count":null},{"metadata":{"_cell_guid":"367b0390-8f7e-45fd-dea2-ae5ef16dce4c","_uuid":"14e971c63163924a514614fb573cee7b7f89b0d0","trusted":true},"cell_type":"code","source":"input_x = Input(shape = (3,))\nlin_fn = Dense(1)(input_x)\nyx_model = Model(inputs = input_x, outputs = lin_fn)\nyx_model.compile(loss = 'mean_squared_error', optimizer = \"sgd\")\nyx_model.fit(X, y, epochs = 10000, batch_size = 500, verbose = 0)\nweights = yx_model.get_weights()\nb = weights[0]\nprint(\"Coefficients for x_1, x_2, x_3 are %.3f, %.3f, %.3f, respectively\" %\n      (b[0], b[1], b[2]))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"ab6f9118-01ab-047b-88b4-5ab795d5d2ea","_uuid":"e9aadb5f665af46403dbb1843614826284cf5631"},"cell_type":"markdown","source":"# LASSO estimation with stochastic gradient descent\n\nLet's set the l1 regularization parameter something greater than zero and see what happens...","outputs":[],"execution_count":null},{"metadata":{"_cell_guid":"ea2ef19f-85df-0584-f61e-6b537a28cbd9","_uuid":"4b4c8b2cbfb518fa2caac43ad85242a2e14bd67f","trusted":true},"cell_type":"code","source":"lambda_val = .7\nlasso_input = Input(shape = (3,))\nlasso_fn = Dense(1, kernel_regularizer = l1(lambda_val))(lasso_input)\nlasso_model = Model(inputs = lasso_input, outputs = lasso_fn)\nlasso_model.compile(loss = 'mean_squared_error', optimizer = \"sgd\")\nlasso_model.fit(X, y, epochs = 10000, batch_size = 500, verbose = 0)\nweights = lasso_model.get_weights()\nb = weights[0]\nprint(\"Coefficients for x_1, x_2, x_3 are %.3f, %.3f, %.3f, respectively\" %\n      (b[0], b[1], b[2]))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"7e0aacf0-01e3-7301-8c94-75c36aba3bf0","_uuid":"8723b8574a74a9eee97afe37501d8e596b23e47e"},"cell_type":"markdown","source":"*Note that the second two variables are practically zeroed out.*\n\nSo there you have it: for all practical purposes, LASSO using a neural network toolkit!","outputs":[],"execution_count":null}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"_change_revision":0,"_is_fork":false,"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}