{"cells":[{"metadata":{"_uuid":"03dcd8faee9e5f130ca546e978cd7adc31fa69cb"},"cell_type":"markdown","source":"# How far can you go using only train.csv and test.csv ?\n### In this kernel, I will tackle the problem the easiest way : \n- Using only the features available in the train.csv and test.csv files\n- Using a Linear Regression as model\n\nI'll also tackle the outliers issue a bit.\n\n### Any feedback is always appreciated ! "},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":false,"_kg_hide-output":false},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nfrom sklearn.svm import  OneClassSVM\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import mean_squared_error, accuracy_score\nfrom sklearn.linear_model import LinearRegression\nimport lightgbm as lgb\n\nsns.set_style('whitegrid')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"67285ac775235dcbb93ee7acc847fb0aa3a43339"},"cell_type":"markdown","source":"### Loading Data"},{"metadata":{"trusted":true,"_uuid":"1707293c54feb5e41e32c397803810642b1ce0a6"},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")\ntest_df = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6f0ae97c8eae2e7411464e514271812f4b607404"},"cell_type":"code","source":"train_df.info()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"55af8c760485aade5a95d7bfcb046fc1a2946df4"},"cell_type":"markdown","source":"Nothing is missing"},{"metadata":{"trusted":true,"_uuid":"972924c0c1b32bd8e379f5d0e66b353f53f4ccae"},"cell_type":"code","source":"test_df.info()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f12ddd1c360555078000c2df0eec76f4f557fc5a"},"cell_type":"markdown","source":"We have one data missing, let us fill it."},{"metadata":{"trusted":true,"_uuid":"2aaf0d64692fb06f835f74c8df4432c98e824d9f"},"cell_type":"code","source":"test_df['first_active_month'] = test_df['first_active_month'].fillna('2018-01-01')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e85e30098bf20c17a33fd5a3510d46c92d3ccad6"},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f9307b25c72c38333a735a751e49bb1bee3b0d13"},"cell_type":"code","source":"print('Size of train dataset :', len(train_df))\nprint('Size of test dataset :', len(test_df))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"191d96fee8bd33cc35bed2336e751701e859589e"},"cell_type":"markdown","source":"### Let us check the repartition of the 3 features"},{"metadata":{"trusted":true,"_uuid":"8e1a1b9db1525f10670c3c2f36780f76a2dc2e6e"},"cell_type":"code","source":"plt.figure(figsize=(10, 7))\nsns.countplot(train_df['feature_1'])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a12e7e69a51903064bb952d697ddf3dee93840a2"},"cell_type":"code","source":"plt.figure(figsize=(10, 7))\nsns.countplot(train_df['feature_2'])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8d40c26006311c22be824306de58098e11976897"},"cell_type":"code","source":"plt.figure(figsize=(10, 7))\nsns.countplot(train_df['feature_3'])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"75693298987677fb31dcc99bf845aac854267f2f"},"cell_type":"markdown","source":"Features seem pretty balanced.\n"},{"metadata":{"_uuid":"c284ce61345dd4e08ae1a9503e2449ca6d5d5b57"},"cell_type":"markdown","source":"### First active month"},{"metadata":{"trusted":true,"_uuid":"7abf6b3a2f4dadbc280d09380d37a832041b08d2"},"cell_type":"code","source":"train_df['first_active_month'] = pd.to_datetime(train_df['first_active_month'])\ntest_df['first_active_month'] = pd.to_datetime(test_df['first_active_month'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c374d94c29948ac54dfbb302ce3b4f6d3b0f2587"},"cell_type":"code","source":"print(\"Newest client :\", max( max(train_df['first_active_month']), max(test_df['first_active_month'])))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"999eb3d75b51803bbbcb38c2f303ff2221982a6f"},"cell_type":"code","source":"first_active_day = pd.to_timedelta(train_df['first_active_month']).apply(lambda x: x.days)\nfirst_active_day_test = pd.to_timedelta(test_df['first_active_month']).apply(lambda x: x.days)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1d9829d683c5b498254a3025770f080f84ed3d3b"},"cell_type":"code","source":"newest = max(max(first_active_day), max(first_active_day_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2adc4688a91983f7fe5f538bfadc5aedf9b2488c"},"cell_type":"code","source":"ancienety = (newest - first_active_day).astype(int) // 30\nancienety_test = (newest - first_active_day_test).astype(int) // 30","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"5dc3354508b7ad399f058c76dd7b943d86dbe798"},"cell_type":"code","source":"plt.figure(figsize=(12, 8))\nsns.distplot(ancienety)\nplt.xlabel('Ancienety (in month)')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3a69ad2c3b4f6a7419e1184ee06e91df8f552b30"},"cell_type":"code","source":"train_df['ancienety'] = ancienety\ntest_df['ancienety'] = ancienety_test","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0faa9cbc42809eb2e87a22ede05f8c83ff3cab4c"},"cell_type":"markdown","source":"### Now to the distribution of the target"},{"metadata":{"trusted":true,"_uuid":"4b4d24c37fc6bd563ca1fd015aa98213dc316035"},"cell_type":"code","source":"plt.figure(figsize=(10, 7))\nsns.distplot(train_df['target'])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"648289a8d1383569f81a243a6714b63e890c6cf7"},"cell_type":"markdown","source":"It's basically a Gaussian with some outliers at on the left. Let's take a closer look at those outliers."},{"metadata":{"trusted":true,"_uuid":"5b28a3b54a4f785599abaf275925e92a2065c7b5"},"cell_type":"code","source":"df_outlier = train_df[train_df['target'] < -30]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ebe18a5615466dc8ecd1d3088ea1371bd313e340"},"cell_type":"code","source":"print('Number of outliers :', len(df_outlier))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"25e4d1cfbec93043066efba1cf16fde23786d745"},"cell_type":"code","source":"df_outlier.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0d8bcc62a7fab3ea32845021313a256d3a68cc29"},"cell_type":"code","source":"df_inlier = train_df[train_df['target'] > -30]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"708bab271a9541a5935a0067488df9ed8ea5380d"},"cell_type":"markdown","source":"Nothing can be said so far, except that they'll play an important role in the future."},{"metadata":{"_uuid":"acbaa3a84209128da935212afa5ba40164844177"},"cell_type":"markdown","source":"## First prediction :\n> Using a linear regression"},{"metadata":{"_uuid":"e0117e9858902bdcdb4e2a49c29939c4f98b26d0"},"cell_type":"markdown","source":"### Linear Regression"},{"metadata":{"trusted":true,"_uuid":"b796802f7195b5b902ac0ae210ebf9c910b3ca50"},"cell_type":"code","source":"X_train = np.array(train_df[['feature_1', 'feature_2', 'feature_3', 'ancienety']])\nX_test = np.array(test_df[['feature_1', 'feature_2', 'feature_3', 'ancienety']])\ny_train = np.array(train_df['target'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fc25c818a1136666e641602c70a105d23bc50ef1"},"cell_type":"code","source":"lin_reg = LinearRegression()\nlin_reg.fit(X_train, y_train)\ny_pred = lin_reg.predict(X_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"02925215871275fd2bc9c12a8b23aa6d322393d0"},"cell_type":"code","source":"print('RMSE on train data :', np.sqrt(mean_squared_error(y_train, y_pred)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e74a766a4dd5a8cd2bbb1f183b490dfd6e70d4fa"},"cell_type":"code","source":"y_test = lin_reg.predict(X_test)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c81ff61619c791c4c7fe2022cfc023e5beb1295f"},"cell_type":"markdown","source":"### Linear Regression without Outliers"},{"metadata":{"trusted":true,"_uuid":"3eec6c685824712588155d6836c467503f61df4d"},"cell_type":"code","source":"X_inlier = np.array(df_inlier[['feature_1', 'feature_2', 'feature_3', 'ancienety']])\ny_inlier = np.array(df_inlier['target'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dec26add8ee6c96439f8b1daa2889ee43386a063"},"cell_type":"code","source":"lin_reg = LinearRegression()\nlin_reg.fit(X_inlier, y_inlier)\ny_pred = lin_reg.predict(X_inlier)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4715e7aac2f8031139c6f1df04b4b41d4ab188fe"},"cell_type":"code","source":"print('RMSE on inliers :', np.sqrt(mean_squared_error(y_inlier, y_pred)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f6e5b94ff3c226b2d295afaf8950e1c2904e686b"},"cell_type":"markdown","source":"#### Outliers seem to play quite a big role, it could be useful to detect them before applying a classifier."},{"metadata":{"_uuid":"3e22402f5882c247f5b630612a2afd91d20572c8"},"cell_type":"markdown","source":"## Can we detect the Outliers ?\n\nI'm using the One Class SVM from sklearn.\n> https://scikit-learn.org/stable/modules/generated/sklearn.svm.OneClassSVM.html"},{"metadata":{"trusted":true,"_uuid":"eb3134ca3ad572387a8b291fc187e65449a840b6"},"cell_type":"code","source":"X_outlier = np.array(df_outlier[['feature_1', 'feature_2', 'feature_3', 'ancienety']])\ny_outlier = np.ones(X_outlier.shape[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cd5219fc4b782596f51ae108dcbf882319b1f8ee"},"cell_type":"code","source":"clf = OneClassSVM(nu=0.01, kernel=\"rbf\", gamma=0.1)\nclf.fit(X_inlier[:50000])  #It takes a while so I don't take all the inliers","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"38d898c9997784eb98137be81ce14a48a842d6dc"},"cell_type":"code","source":"y_pred_inlier = clf.predict(X_inlier)\ny_pred_outlier = clf.predict(X_outlier)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a6f9b8249e719f58be0c3187e480fa1434e43242"},"cell_type":"code","source":"print(\"Accuracy on inliers :\", accuracy_score(y_pred_inlier, np.ones(y_inlier.size)))\nprint(\"Accuracy on outliers :\", accuracy_score(y_pred_outlier, - np.ones(y_outlier.size)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6f69b1974891f122dd8fd03af42c8e9544cd8c4b"},"cell_type":"markdown","source":"#### No surprise, the answer is no. But it could probably be done with more features !"},{"metadata":{"trusted":true,"_uuid":"51327f2f250b2662835c77bc360884dc11fbd0dc"},"cell_type":"markdown","source":"## Submission"},{"metadata":{"trusted":true,"_uuid":"fe805c41f51155cfa578df8d14f20def66ff946d"},"cell_type":"code","source":"submission = pd.DataFrame({\"card_id\": test_df[\"card_id\"].values})\nsubmission[\"target\"] = y_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0dd199c42d64a0345308aa03794037858164702c"},"cell_type":"code","source":"submission.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5192874a8d30781e409a822e822ae5e454c105a8"},"cell_type":"markdown","source":"This simple model scored 3.925, which is way better than I expected."},{"metadata":{"trusted":true,"_uuid":"95d5f7da83eaec010ebe712efc1e8f9a5bf14159"},"cell_type":"markdown","source":"## Following work :\n- Taking the other stuff into account !\n\n#### Thanks for reading !"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}