{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\npd.set_option('display.float_format', lambda x: '%.3f' % x)\npd.set_option('display.max_columns', None)\n\nimport random\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.metrics import mean_squared_error, r2_score\n\nimport os\nprint(os.listdir(\"../input\"))\n\nimport seaborn as sns\nsns.set(style=\"darkgrid\")\nimport matplotlib.pyplot as plt\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# *** Read Data ***\ndf = pd.read_csv('../input/train.csv')\ndfTest  = pd.read_csv('../input/test.csv')\ndfSub   = pd.read_csv('../input/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# *** Data View ***\nprint(df.head())\nprint(dfSub.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# *** First Data Impression ***\nprint('Train Data Info')\nprint(df.info())\nprint('\\nTest Data Info \\n')\nprint(dfTest.info())\nprint('\\nSubmission Data Info \\n')\nprint(dfSub.info())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# *** Check Missing Values ***\nprint('Missing Values in Training Data')\nprint(df.isnull().sum())\nprint('Missing Values in Test Data')\nprint(df.isnull().sum())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"No Missing Values in either train or test data"},{"metadata":{"trusted":true},"cell_type":"code","source":"# *** Check Unique Values ***\nprint('Unique Values in Training Data')\nprint(df.nunique())\nprint('Unique Values in Test Data')\nprint(dfTest.nunique())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- 85,003 molecules in train data and 45,772 molecules in test data\n- 8 types of coupling in both training and test data"},{"metadata":{"trusted":true},"cell_type":"code","source":"# *** Explore Trainin Data ***\n# scalar_coupling_constant\ndf['scalar_coupling_constant'].describe(percentiles = [0.25, 0.5, 0.75, 0.9, 0.95, 0.99])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plot the Distribution\nsns.boxplot(df['scalar_coupling_constant'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- 50% of the coupling constant values are in between -0.2 to 2.2\n- Some values even crosses 200"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Check the dot plot to make the distribution even more clear\nsns.stripplot(x = df['scalar_coupling_constant'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.distplot(df['scalar_coupling_constant'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- With this probability distribution graph, the actual data nature has been revealed completely\n- Apart from the two very obvious peak, there is a very small third peak at the end of dostribution around 200"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Distribution of scalar_coupling_constant by molecule_name\ndf.groupby('molecule_name')['scalar_coupling_constant'].mean().reset_index().head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.groupby('molecule_name')['scalar_coupling_constant'].mean().reset_index()['scalar_coupling_constant']. \\\n        describe(percentiles = [0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.99])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.distplot(df.groupby('molecule_name')['scalar_coupling_constant'].mean().reset_index()['scalar_coupling_constant'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- Almost All the mean vaues of coupling constant for a molecule are in between 10 to 30\n- From the distribution table also, it is very much clear"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Lets Explore the Atom Index Combination in Detail\n(df['atom_index_0'].map(str) + '--' + df['atom_index_1'].map(str)).nunique()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- There are 513 index combination\n- Let's check the distribution of count"},{"metadata":{"trusted":true},"cell_type":"code","source":"(df['atom_index_0'].map(str) + '--' + df['atom_index_1'].map(str)).value_counts().nlargest(10) ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- It is evident that the top 10 combinations are originationg from 9 to 12\n- Now I am interested in lookin the bottom ten also"},{"metadata":{"trusted":true},"cell_type":"code","source":"(df['atom_index_0'].map(str) + '--' + df['atom_index_1'].map(str)).value_counts().nsmallest(10) ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- It looks very rare for the same index pairs to come together\n> - Lets verify this "},{"metadata":{"trusted":true},"cell_type":"code","source":"df[df['atom_index_0'].map(str) == df['atom_index_1'].map(str)].shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- We can conclude that same pair of indexes never come together\n- Let's check the complete distribution of index pair count"},{"metadata":{"trusted":true},"cell_type":"code","source":"# *** Distribution of Index 0 and 1\ndf['atom_index_0'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df['atom_index_0'].value_counts(normalize = True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- Top 12 index 0 accounts for 90% of the data"},{"metadata":{"trusted":true},"cell_type":"code","source":"df['atom_index_1'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df['atom_index_1'].value_counts(normalize = True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- Top 12 index 1 accounts for 80% of the data\n- Top indexes by count varies in index 0 and index 1"},{"metadata":{"trusted":true},"cell_type":"code","source":"countDist = (df['atom_index_0'].map(str) + '--' + df['atom_index_1'].map(str)).value_counts().reset_index(). \\\n             rename(columns = {0 : 'combinationCount'})\nsns.distplot(countDist['combinationCount'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df['molecule_name'].value_counts().nlargest(10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Looks Like the counts doesn't vary much. let's check:"},{"metadata":{"trusted":true},"cell_type":"code","source":"df['molecule_name'].value_counts().reset_index()['molecule_name']. \\\n                         describe(percentiles = [0.25, 0.5, 0.75, 0.9, 0.95, 0.99])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ax = sns.boxplot(x=df['molecule_name'].value_counts().reset_index()['molecule_name'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- The median count is somewhere between 50 and 60.\n- From the above table it is 54.\n- Hardly a little more than 1% of molecules have occured more than 100 "},{"metadata":{"trusted":true},"cell_type":"code","source":"# *** Simple Linear Regression Model ***\ndf['randomNum'] = df['id'].map(lambda x : random.uniform(0, 1))\ndfTrain = df.query('randomNum <= 0.7')\ndfValid  = df.query('randomNum > 0.7')\ndel dfTrain['randomNum']\ndel dfValid['randomNum']\ndel df['randomNum'] ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- We will use simple risk based features"},{"metadata":{"trusted":true},"cell_type":"code","source":"molecule_name_risk = dfTrain.groupby('molecule_name')['scalar_coupling_constant'].mean().reset_index()\nmolecule_name_risk.rename(columns = {'scalar_coupling_constant' : 'molecule_name_risk'}, inplace = True)\n\ndfTrain = pd.merge(dfTrain, molecule_name_risk, on = 'molecule_name')\n\nindex_0_risk = dfTrain.groupby('atom_index_0')['scalar_coupling_constant'].mean().reset_index()\nindex_0_risk.rename(columns = {'scalar_coupling_constant' : 'index_0_risk'}, inplace = True)\n\ndfTrain = pd.merge(dfTrain, index_0_risk, on = 'atom_index_0')\n\nindex_1_risk = dfTrain.groupby('atom_index_1')['scalar_coupling_constant'].mean().reset_index()\nindex_1_risk.rename(columns = {'scalar_coupling_constant' : 'index_1_risk'}, inplace = True)\n\ndfTrain = pd.merge(dfTrain, index_1_risk, on = 'atom_index_1')\n\ntype_risk = dfTrain.groupby('type')['scalar_coupling_constant'].mean().reset_index()\ntype_risk.rename(columns = {'scalar_coupling_constant' : 'type_risk'}, inplace = True)\n\ndfTrain = pd.merge(dfTrain, type_risk, on = 'type')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dfTrain.drop(['molecule_name', 'atom_index_0', 'atom_index_1', 'type'], axis = 1, inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dfValid = pd.merge(dfValid, molecule_name_risk, on = 'molecule_name')\ndfValid = pd.merge(dfValid, index_0_risk, on = 'atom_index_0')\ndfValid = pd.merge(dfValid, index_1_risk, on = 'atom_index_1')\ndfValid = pd.merge(dfValid, type_risk, on = 'type')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dfValid.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(dfValid.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dfValid.drop(['molecule_name', 'atom_index_0', 'atom_index_1', 'type'], axis = 1, inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"molecule_name_risk.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(dfTrain.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lmReg = LinearRegression()\nlmReg.fit(dfTrain.drop(['id', 'scalar_coupling_constant'], axis = 1), dfTrain['scalar_coupling_constant'])\ndfValid['scalar_coupling_constant_pred'] = lmReg.predict(dfValid.drop(['id', 'scalar_coupling_constant'], axis = 1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dfValid['predDiff'] = dfValid['scalar_coupling_constant'] - dfValid['scalar_coupling_constant_pred'] ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dfValid['predDiff'].describe(percentiles = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.95, 0.99])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dfValid['scalar_coupling_constant'].describe(percentiles = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.95, 0.99])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# The coefficients\nprint('Coefficients: \\n', lmReg.coef_)\n# The mean squared error\nprint(\"Mean squared error: %.2f\"\n      % mean_squared_error(dfValid['scalar_coupling_constant'], dfValid['scalar_coupling_constant_pred'] ))\n# Explained variance score: 1 is perfect prediction\nprint('Variance score: %.2f' % r2_score(dfValid['scalar_coupling_constant'], dfValid['scalar_coupling_constant_pred']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plot outputs\nsns.boxplot(x=\"variable\", y=\"value\", data=pd.melt(dfValid[['scalar_coupling_constant', 'scalar_coupling_constant_pred']]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ax = sns.lineplot(data = dfValid[['scalar_coupling_constant', 'scalar_coupling_constant_pred']])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#dfTest = pd.merge(dfTest, molecule_name_risk, on = 'molecule_name')\ndfTest = pd.merge(dfTest, index_0_risk, on = 'atom_index_0')\ndfTest = pd.merge(dfTest, index_1_risk, on = 'atom_index_1')\ndfTest = pd.merge(dfTest, type_risk, on = 'type')\ndfTest['molecule_name_risk'] = 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dfTest['scalar_coupling_constant_pred'] = lmReg.predict(dfTest.drop(['id', 'molecule_name', 'atom_index_0', 'atom_index_1', \\\n                                                                    'type'], axis = 1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dfSub['scalar_coupling_constant'] = dfTest['scalar_coupling_constant_pred']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dfSub.to_csv('sample_submission.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- Submission Score : 2.09\n- Build Model on Complete Training Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"molecule_name_risk = df.groupby('molecule_name')['scalar_coupling_constant'].mean().reset_index()\nmolecule_name_risk.rename(columns = {'scalar_coupling_constant' : 'molecule_name_risk'}, inplace = True)\n\ndf = pd.merge(df, molecule_name_risk, on = 'molecule_name')\n\nindex_0_risk = df.groupby('atom_index_0')['scalar_coupling_constant'].mean().reset_index()\nindex_0_risk.rename(columns = {'scalar_coupling_constant' : 'index_0_risk'}, inplace = True)\n\ndf = pd.merge(df, index_0_risk, on = 'atom_index_0')\n\nindex_1_risk = df.groupby('atom_index_1')['scalar_coupling_constant'].mean().reset_index()\nindex_1_risk.rename(columns = {'scalar_coupling_constant' : 'index_1_risk'}, inplace = True)\n\ndf = pd.merge(df, index_1_risk, on = 'atom_index_1')\n\ntype_risk = df.groupby('type')['scalar_coupling_constant'].mean().reset_index()\ntype_risk.rename(columns = {'scalar_coupling_constant' : 'type_risk'}, inplace = True)\n\ndf = pd.merge(df, type_risk, on = 'type')\n\ndf.drop(['molecule_name', 'atom_index_0', 'atom_index_1', 'type'], axis = 1, inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dfTest  = pd.read_csv('../input/test.csv')\ndfTest = pd.merge(dfTest, index_0_risk, on = 'atom_index_0')\ndfTest = pd.merge(dfTest, index_1_risk, on = 'atom_index_1')\ndfTest = pd.merge(dfTest, type_risk, on = 'type')\ndfTest['molecule_name_risk'] = 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dfTest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dfTest.drop(['id', 'molecule_name', 'atom_index_0', 'atom_index_1', 'type'], axis = 1).head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lmReg = LinearRegression()\nlmReg.fit(df.drop(['id', 'scalar_coupling_constant'], axis = 1), df['scalar_coupling_constant'])\ndfTest['scalar_coupling_constant_pred'] = lmReg.predict(dfTest.drop(['id', 'molecule_name', \\\n                                                        'atom_index_0', 'atom_index_1', 'type'], axis = 1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dfSub['scalar_coupling_constant'] = dfTest['scalar_coupling_constant_pred']\ndfSub.to_csv('sample_submission.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dfSub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}