{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport seaborn as sns\nimport statsmodels.formula.api as smf\nimport statsmodels.api as sm\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport patsy\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report, RocCurveDisplay\nfrom sklearn.tree import DecisionTreeClassifier, plot_tree\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.ensemble import GradientBoostingClassifier\nfrom sklearn.ensemble import VotingClassifier\nfrom pandas import get_dummies\n","metadata":{"execution":{"iopub.status.busy":"2023-07-10T07:15:54.297213Z","iopub.execute_input":"2023-07-10T07:15:54.297632Z","iopub.status.idle":"2023-07-10T07:15:58.160231Z","shell.execute_reply.started":"2023-07-10T07:15:54.297599Z","shell.execute_reply":"2023-07-10T07:15:58.159067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Applied Machine Learning (with Python)\n## Exploring Our Data\n\nWe can start by importing our training data, understanding how it's constructed and what we're aiming to achieve.","metadata":{}},{"cell_type":"code","source":"titanic_train = pd.read_csv('/kaggle/input/titanic/train.csv', index_col=0)\ntitanic_train","metadata":{"execution":{"iopub.status.busy":"2023-07-10T07:15:58.162301Z","iopub.execute_input":"2023-07-10T07:15:58.162685Z","iopub.status.idle":"2023-07-10T07:15:58.234392Z","shell.execute_reply.started":"2023-07-10T07:15:58.162654Z","shell.execute_reply":"2023-07-10T07:15:58.233231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"You'll notice that unlike traditional spreadsheets, you can't \"click into\" your data here! Any change we make to our data has to be programatic - what are the benefits and challenges?","metadata":{}},{"cell_type":"markdown","source":"If you do want to make changes to your data, you'll have to be explicit: we use the '.loc' command to pick a specific value based on the X and Y axis.","metadata":{}},{"cell_type":"code","source":"titanic_train.loc[1,'Survived']","metadata":{"execution":{"iopub.status.busy":"2023-07-10T07:15:58.235747Z","iopub.execute_input":"2023-07-10T07:15:58.236510Z","iopub.status.idle":"2023-07-10T07:15:58.245601Z","shell.execute_reply.started":"2023-07-10T07:15:58.236478Z","shell.execute_reply":"2023-07-10T07:15:58.244301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"How would you go about change that first value to someone who had survived?","metadata":{}},{"cell_type":"code","source":"titanic_train.loc[1,'Survived'] = 1\n\ntitanic_train","metadata":{"execution":{"iopub.status.busy":"2023-07-10T07:16:12.902328Z","iopub.execute_input":"2023-07-10T07:16:12.902738Z","iopub.status.idle":"2023-07-10T07:16:12.925018Z","shell.execute_reply.started":"2023-07-10T07:16:12.902704Z","shell.execute_reply":"2023-07-10T07:16:12.923980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now, as we've changed our data, lets return to our original - can you re-import the dataset?","metadata":{}},{"cell_type":"code","source":"file_location = '/kaggle/input/titanic/train.csv'\n\ntitanic_train = pd.read_csv(file_location, index_col=0)\ntitanic_train\n","metadata":{"execution":{"iopub.status.busy":"2023-07-07T08:12:43.691883Z","iopub.execute_input":"2023-07-07T08:12:43.695195Z","iopub.status.idle":"2023-07-07T08:12:43.740990Z","shell.execute_reply.started":"2023-07-07T08:12:43.695162Z","shell.execute_reply":"2023-07-07T08:12:43.740027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The function we've called , 'pd.read_csv', incidates this is part of the open source Pandas library, a core part of the Python data science ecosystem - that means it comes bundled with a bunch of useful functions!","metadata":{}},{"cell_type":"code","source":"titanic_train.describe()","metadata":{"execution":{"iopub.status.busy":"2023-07-07T08:12:43.742296Z","iopub.execute_input":"2023-07-07T08:12:43.743397Z","iopub.status.idle":"2023-07-07T08:12:43.797908Z","shell.execute_reply.started":"2023-07-07T08:12:43.743353Z","shell.execute_reply":"2023-07-07T08:12:43.796992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Remember what we're trying to achieve: understanding how the data we have can help us understand the data we don't.  Correlation is a good place to start.","metadata":{}},{"cell_type":"code","source":"titanic_train[['Survived','Age']].corr()","metadata":{"execution":{"iopub.status.busy":"2023-07-07T08:12:43.799634Z","iopub.execute_input":"2023-07-07T08:12:43.800356Z","iopub.status.idle":"2023-07-07T08:12:43.814509Z","shell.execute_reply.started":"2023-07-07T08:12:43.800319Z","shell.execute_reply":"2023-07-07T08:12:43.813459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# can you apply a similar technique to the other columns?\ntitanic_train[['Survived','Pclass','Age','Fare']].corr()","metadata":{"execution":{"iopub.status.busy":"2023-07-07T08:12:43.816463Z","iopub.execute_input":"2023-07-07T08:12:43.817233Z","iopub.status.idle":"2023-07-07T08:12:43.834070Z","shell.execute_reply.started":"2023-07-07T08:12:43.817194Z","shell.execute_reply":"2023-07-07T08:12:43.832982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's now briefly go back to our slides and discuss linear models.","metadata":{}},{"cell_type":"markdown","source":"## Predictions with Linear Models","metadata":{}},{"cell_type":"markdown","source":"We identified that there was a link between the paid fare, and survival - lets build a model to examine how those relate.","metadata":{}},{"cell_type":"code","source":"mod = smf.ols(formula='Survived ~ Fare', data=titanic_train)\nres = mod.fit()\nres.summary()","metadata":{"execution":{"iopub.status.busy":"2023-07-07T08:12:49.822899Z","iopub.execute_input":"2023-07-07T08:12:49.823401Z","iopub.status.idle":"2023-07-07T08:12:49.855774Z","shell.execute_reply.started":"2023-07-07T08:12:49.823362Z","shell.execute_reply":"2023-07-07T08:12:49.854682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# can you add passenger class to the model?\n\nmod = smf.ols(formula='Survived ~ Pclass + Fare', data=titanic_train)\nres = mod.fit()\nres.summary()","metadata":{"execution":{"iopub.status.busy":"2023-07-10T07:28:40.237387Z","iopub.execute_input":"2023-07-10T07:28:40.238125Z","iopub.status.idle":"2023-07-10T07:28:40.268322Z","shell.execute_reply.started":"2023-07-10T07:28:40.238086Z","shell.execute_reply":"2023-07-10T07:28:40.267101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's look at our output again - what does it tell you about how our variables relate, and how well our model fits our data?","metadata":{}},{"cell_type":"markdown","source":"## Making our first predictions\nWe've fit a model to our data! Now, lets use Kaggle as intended and try an initial submission.","metadata":{}},{"cell_type":"code","source":"test_vals = pd.read_csv('/kaggle/input/titanic/test.csv', index_col=0)\ntest_vals","metadata":{"execution":{"iopub.status.busy":"2023-07-10T07:29:49.160267Z","iopub.execute_input":"2023-07-10T07:29:49.160709Z","iopub.status.idle":"2023-07-10T07:29:49.192820Z","shell.execute_reply.started":"2023-07-10T07:29:49.160675Z","shell.execute_reply":"2023-07-10T07:29:49.191613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Meaningful Test Values\n* Here, we've been given a \"test set\" that is selected at random from our full dataset\n* How much do you think this matches reality?\n","metadata":{}},{"cell_type":"code","source":"# Can you create a set of predictions using our linear model?\n\nprediction = res.predict(test_vals)\nprediction","metadata":{"execution":{"iopub.status.busy":"2023-07-07T08:12:49.934752Z","iopub.execute_input":"2023-07-07T08:12:49.935450Z","iopub.status.idle":"2023-07-07T08:12:49.947288Z","shell.execute_reply.started":"2023-07-07T08:12:49.935419Z","shell.execute_reply":"2023-07-07T08:12:49.946006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"You'll notice we have a challenge with our linear model: it doesn't predict specific outcomes, but a *linear* outcome - this is referrred to as a regressor model, in contrast to a classifier model.\n\nAs an interim solution, as easy fix is to set a \"threshold\"... but how we set our threshold will affect our prediction.\n\nSo what should our threshold be here? Let's try a few approaches and see how it affects our results.","metadata":{}},{"cell_type":"code","source":"prediction.plot.hist()","metadata":{"execution":{"iopub.status.busy":"2023-07-07T08:12:49.948986Z","iopub.execute_input":"2023-07-07T08:12:49.949402Z","iopub.status.idle":"2023-07-07T08:12:50.180021Z","shell.execute_reply.started":"2023-07-07T08:12:49.949355Z","shell.execute_reply":"2023-07-07T08:12:50.178954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"threshold = 0.5","metadata":{"execution":{"iopub.status.busy":"2023-07-07T08:12:50.181468Z","iopub.execute_input":"2023-07-07T08:12:50.184104Z","iopub.status.idle":"2023-07-07T08:12:50.187943Z","shell.execute_reply.started":"2023-07-07T08:12:50.184070Z","shell.execute_reply":"2023-07-07T08:12:50.187189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_vals['prediction'] = prediction > threshold\ntest_vals['prediction']  = test_vals['prediction'].astype(int)\ntest_vals['prediction']","metadata":{"execution":{"iopub.status.busy":"2023-07-07T08:12:50.189100Z","iopub.execute_input":"2023-07-07T08:12:50.189383Z","iopub.status.idle":"2023-07-07T08:12:50.203983Z","shell.execute_reply.started":"2023-07-07T08:12:50.189358Z","shell.execute_reply":"2023-07-07T08:12:50.202972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now we've produced our very first set of predictions! [Let's try and submit them](https://www.kaggle.com/competitions/titanic), and see how we do.","metadata":{}},{"cell_type":"code","source":"example_submission = test_vals[['prediction']].rename(columns={'prediction': 'Survived'})\nexample_submission.to_csv('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2023-07-07T08:12:50.205757Z","iopub.execute_input":"2023-07-07T08:12:50.206948Z","iopub.status.idle":"2023-07-07T08:12:50.218547Z","shell.execute_reply.started":"2023-07-07T08:12:50.206870Z","shell.execute_reply":"2023-07-07T08:12:50.217601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"How did you do?","metadata":{}},{"cell_type":"markdown","source":"To understand our predictions a little more, we can look at how many positive and negative predictions we had ... how does that compare to what you'd expect?","metadata":{}},{"cell_type":"code","source":"print('Total entries:' + str(len(test_vals)))","metadata":{"execution":{"iopub.status.busy":"2023-07-07T08:12:50.219655Z","iopub.execute_input":"2023-07-07T08:12:50.220340Z","iopub.status.idle":"2023-07-07T08:12:50.225562Z","shell.execute_reply.started":"2023-07-07T08:12:50.220311Z","shell.execute_reply":"2023-07-07T08:12:50.224822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_vals['prediction'].sum()","metadata":{"execution":{"iopub.status.busy":"2023-07-07T08:12:50.226519Z","iopub.execute_input":"2023-07-07T08:12:50.227438Z","iopub.status.idle":"2023-07-07T08:12:50.238304Z","shell.execute_reply.started":"2023-07-07T08:12:50.227397Z","shell.execute_reply":"2023-07-07T08:12:50.237584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## From Linear Models to AI\n\nThis might look miles away from \"proper AI\" - but is it really?\n\nHow different is [text](https://www.kaggle.com/code/cdeotte/titanic-using-name-only-0-81818)?\n\nHow about pictures?\n\nWhat about audio?\n\n","metadata":{}},{"cell_type":"code","source":"# Load the data\nmnist_train = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\n\n# Normalize the data\nmnist_train = mnist_train / 255.0\n","metadata":{"execution":{"iopub.status.busy":"2023-07-07T08:12:50.590640Z","iopub.execute_input":"2023-07-07T08:12:50.591089Z","iopub.status.idle":"2023-07-07T08:12:55.037878Z","shell.execute_reply.started":"2023-07-07T08:12:50.591050Z","shell.execute_reply":"2023-07-07T08:12:55.036998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example = mnist_train.drop(columns=['label']).loc[0].values.reshape(28,28)\ng = plt.imshow(example)","metadata":{"execution":{"iopub.status.busy":"2023-07-07T08:12:55.038809Z","iopub.execute_input":"2023-07-07T08:12:55.039216Z","iopub.status.idle":"2023-07-07T08:12:55.347862Z","shell.execute_reply.started":"2023-07-07T08:12:55.039149Z","shell.execute_reply":"2023-07-07T08:12:55.346802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"What about audio?","metadata":{}},{"cell_type":"code","source":"import IPython.display as ipd  # To play sound in the notebook\nfname = '../input/freesound-audio-tagging/audio_train/' + '00044347.wav'   # Hi-hat\nipd.Audio(fname)","metadata":{"execution":{"iopub.status.busy":"2023-07-07T08:12:55.349044Z","iopub.execute_input":"2023-07-07T08:12:55.349350Z","iopub.status.idle":"2023-07-07T08:12:55.409593Z","shell.execute_reply.started":"2023-07-07T08:12:55.349323Z","shell.execute_reply":"2023-07-07T08:12:55.408644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Using scipy\nfrom scipy.io import wavfile\nimport matplotlib.pyplot as plt\nrate, data = wavfile.read(fname)\nplt.figure(figsize=(16, 4))\nplt.plot(data[:500], '.'); plt.plot(data[:500], '-');","metadata":{"execution":{"iopub.status.busy":"2023-07-07T08:12:55.410826Z","iopub.execute_input":"2023-07-07T08:12:55.411801Z","iopub.status.idle":"2023-07-07T08:12:55.732281Z","shell.execute_reply.started":"2023-07-07T08:12:55.411767Z","shell.execute_reply":"2023-07-07T08:12:55.731064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The core ML challenge is, more often than not, how to fix your data to these increasingly squiggly lines - and with differnet model types and more computing power, machine learning and fit any number of unpredictable patterns! \n\nYou can explore this a little further with the tool below.  ","metadata":{}},{"cell_type":"code","source":"\nfrom ipywidgets import interact\nfrom fastai.basics import *\n\n\ndef plot_function(f, title=None, min=-2.1, max=2.1, color='r', ylim=None):\n    x = torch.linspace(min,max, 100)[:,None]\n    if ylim: plt.ylim(ylim)\n    plt.plot(x, f(x), color)\n    if title is not None: plt.title(title)\n\ndef f(x): return 3*x**2 + 2*x + 1\n\ndef quad(a, b, c, x): return a*x**2 + b*x + c\ndef mk_quad(a,b,c): return partial(quad, a,b,c)\n\ndef noise(x, scale): return np.random.normal(scale=scale, size=x.shape)\ndef add_noise(x, mult, add): return x * (1+noise(x,mult)) + noise(x,add)\ndef mae(preds, acts): return (torch.abs(preds-acts)).mean()\n\nnp.random.seed(42)\n\nx = torch.linspace(-2, 2, steps=20)[:,None]\ny = add_noise(f(x), 0.15, 1.5)\n\n\nfrom ipywidgets import interactive, VBox\nimport matplotlib.pyplot as plt\n\ndef plot_quad(a, b, c):\n    f = mk_quad(a, b, c)\n    plt.scatter(x, y)\n    loss = mae(f(x), y)\n    fig = plot_function(f, ylim=(-3,12), title=f\"MAE: {loss:.2f}\")\n    fig\n\na_slider = interactive(plot_quad, a=(0, 2, 0.1), b=(0, 2, 0.1), c=(0, 2, 0.1))\nvb = VBox([a_slider])\nvb\n","metadata":{"execution":{"iopub.status.busy":"2023-07-07T08:12:55.733835Z","iopub.execute_input":"2023-07-07T08:12:55.734279Z","iopub.status.idle":"2023-07-07T08:12:59.894083Z","shell.execute_reply.started":"2023-07-07T08:12:55.734241Z","shell.execute_reply":"2023-07-07T08:12:59.892892Z"},"trusted":true},"execution_count":null,"outputs":[]}]}