{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport datetime as dt\n%matplotlib inline\nimport matplotlib.pyplot as plt\nimport scipy\nimport scipy.stats as stats\nimport seaborn as sns\nimport sklearn.linear_model as lm\nfrom sklearn.preprocessing import OneHotEncoder, LabelEncoder\npd.options.display.max_rows = 8\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-09-23T15:45:22.526338Z","iopub.execute_input":"2021-09-23T15:45:22.527922Z","iopub.status.idle":"2021-09-23T15:45:23.734485Z","shell.execute_reply.started":"2021-09-23T15:45:22.527664Z","shell.execute_reply":"2021-09-23T15:45:23.733526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import chardet\nwith open('/kaggle/input/2021-uplift-modeling/x5-uplift-valid/data/train.csv', 'rb') as f:\n    result = chardet.detect(f.read())  # or readline if the file is large\n\ntrain = pd.read_csv('/kaggle/input/2021-uplift-modeling/x5-uplift-valid/data/train.csv', encoding=result['encoding'])\ntrain_purch = pd.read_csv('/kaggle/input/2021-uplift-modeling/x5-uplift-valid/train_purch/train_purch.csv', encoding=result['encoding'])\n\ntest = pd.read_csv('/kaggle/input/2021-uplift-modeling/x5-uplift-valid/data/test.csv', encoding=result['encoding'])\ntest_purch = pd.read_csv('/kaggle/input/2021-uplift-modeling/x5-uplift-valid/test_purch/test_purch.csv', encoding=result['encoding'])\n\nclients = pd.read_csv('/kaggle/input/2021-uplift-modeling/x5-uplift-valid/data/clients2.csv', encoding=result['encoding'])\nproducts = pd.read_csv('/kaggle/input/2021-uplift-modeling/x5-uplift-valid/data/products.csv', encoding=result['encoding'])","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:45:23.736303Z","iopub.execute_input":"2021-09-23T15:45:23.736574Z","iopub.status.idle":"2021-09-23T15:46:28.277415Z","shell.execute_reply.started":"2021-09-23T15:45:23.736544Z","shell.execute_reply":"2021-09-23T15:46:28.276603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:46:28.278807Z","iopub.execute_input":"2021-09-23T15:46:28.279124Z","iopub.status.idle":"2021-09-23T15:46:28.301497Z","shell.execute_reply.started":"2021-09-23T15:46:28.279084Z","shell.execute_reply":"2021-09-23T15:46:28.300621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:46:28.304459Z","iopub.execute_input":"2021-09-23T15:46:28.305363Z","iopub.status.idle":"2021-09-23T15:46:28.346615Z","shell.execute_reply.started":"2021-09-23T15:46:28.305311Z","shell.execute_reply":"2021-09-23T15:46:28.345772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Подготовка данных","metadata":{}},{"cell_type":"code","source":"clients['first_issue_date'] = pd.to_datetime(clients['first_issue_date'], errors=\"coerce\").dt.date\nclients['first_redeem_date'] = pd.to_datetime(clients['first_redeem_date'], errors=\"coerce\").dt.date","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:46:28.347758Z","iopub.execute_input":"2021-09-23T15:46:28.348079Z","iopub.status.idle":"2021-09-23T15:46:28.716495Z","shell.execute_reply.started":"2021-09-23T15:46:28.348048Z","shell.execute_reply":"2021-09-23T15:46:28.715639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = train.join(clients.set_index('client_id'), on='client_id').drop('client_id.1', axis = 1)\n#df_train = df_train.drop('client_id.1', axis = 1)\ndf_test = test.join(clients.set_index('client_id'), on='client_id').drop('client_id.1', axis = 1)\n\n#df_train.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:46:28.717723Z","iopub.execute_input":"2021-09-23T15:46:28.717959Z","iopub.status.idle":"2021-09-23T15:46:29.139317Z","shell.execute_reply.started":"2021-09-23T15:46:28.717931Z","shell.execute_reply":"2021-09-23T15:46:29.138448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:46:29.140635Z","iopub.execute_input":"2021-09-23T15:46:29.140873Z","iopub.status.idle":"2021-09-23T15:46:29.152940Z","shell.execute_reply.started":"2021-09-23T15:46:29.140845Z","shell.execute_reply":"2021-09-23T15:46:29.152108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_purch_prod = train_purch.join(products.set_index('product_id'), on='product_id')\ntrain_purch_prod.head(3)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:46:29.154079Z","iopub.execute_input":"2021-09-23T15:46:29.154332Z","iopub.status.idle":"2021-09-23T15:46:36.172916Z","shell.execute_reply.started":"2021-09-23T15:46:29.154273Z","shell.execute_reply":"2021-09-23T15:46:36.172081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_purch_prod = test_purch.join(products.set_index('product_id'), on='product_id')\ntest_purch_prod.head(3)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:46:36.174112Z","iopub.execute_input":"2021-09-23T15:46:36.174344Z","iopub.status.idle":"2021-09-23T15:46:39.526533Z","shell.execute_reply.started":"2021-09-23T15:46:36.174312Z","shell.execute_reply":"2021-09-23T15:46:39.525961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_purch_prod['purch_popular_product'] =  np.where(train_purch_prod['product_id'] == '4009f09b04', 1 , 0)\ntest_purch_prod['purch_popular_product'] =  np.where(test_purch_prod['product_id'] == '4009f09b04', 1 , 0)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:46:39.528842Z","iopub.execute_input":"2021-09-23T15:46:39.529553Z","iopub.status.idle":"2021-09-23T15:46:43.367052Z","shell.execute_reply.started":"2021-09-23T15:46:39.529516Z","shell.execute_reply":"2021-09-23T15:46:43.366384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_purch_prod['transaction_datetime'] = pd.to_datetime(train_purch_prod['transaction_datetime'], errors=\"coerce\")\ntest_purch_prod['transaction_datetime'] = pd.to_datetime(test_purch_prod['transaction_datetime'], errors=\"coerce\")","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:46:43.368357Z","iopub.execute_input":"2021-09-23T15:46:43.369117Z","iopub.status.idle":"2021-09-23T15:46:55.084401Z","shell.execute_reply.started":"2021-09-23T15:46:43.369070Z","shell.execute_reply":"2021-09-23T15:46:55.083571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Сгруппируем наборы данных train_purch, test_purch по client_id и store_id","metadata":{}},{"cell_type":"code","source":"train_purch_group = train_purch_prod.groupby(['client_id', 'store_id']).agg({\n    'regular_points_received': 'sum', 'express_points_received' : 'sum', 'regular_points_spent' : 'sum', 'express_points_spent' : 'sum',\n    'product_quantity' : 'sum', 'trn_sum_from_iss' : 'sum', 'trn_sum_from_red': 'sum', 'purch_popular_product': 'sum', 'is_alcohol': 'sum', 'is_own_trademark': 'sum'   \n    })","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:46:55.086537Z","iopub.execute_input":"2021-09-23T15:46:55.087174Z","iopub.status.idle":"2021-09-23T15:47:02.316100Z","shell.execute_reply.started":"2021-09-23T15:46:55.087107Z","shell.execute_reply":"2021-09-23T15:47:02.314826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_purch_group = train_purch_group.reset_index()","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:47:02.317372Z","iopub.execute_input":"2021-09-23T15:47:02.317749Z","iopub.status.idle":"2021-09-23T15:47:02.354787Z","shell.execute_reply.started":"2021-09-23T15:47:02.317714Z","shell.execute_reply":"2021-09-23T15:47:02.353834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_purch_group = test_purch_prod.groupby(['client_id', 'store_id']).agg({\n    'regular_points_received': 'sum', 'express_points_received' : 'sum', 'regular_points_spent' : 'sum', 'express_points_spent' : 'sum',\n    'product_quantity' : 'sum', 'trn_sum_from_iss' : 'sum', 'trn_sum_from_red': 'sum', 'purch_popular_product': 'sum', 'is_alcohol': 'sum', 'is_own_trademark': 'sum'  \n    }).reset_index()","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:47:02.355955Z","iopub.execute_input":"2021-09-23T15:47:02.356185Z","iopub.status.idle":"2021-09-23T15:47:05.371694Z","shell.execute_reply.started":"2021-09-23T15:47:02.356158Z","shell.execute_reply":"2021-09-23T15:47:05.370826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Вычислим частоту совершения покупок и дополним этой информацией сгруппированные наборы данных","metadata":{}},{"cell_type":"code","source":"def frequency_purch (X):\n    purch = X.groupby(['client_id', 'transaction_id', 'transaction_datetime']).agg({ 'product_quantity': 'sum' }).reset_index(col_level=1)\n    purch = purch.groupby(['client_id']).agg({ 'transaction_datetime': ['first', 'last'], 'transaction_id': 'count' }).reset_index(col_level=1)\n    purch.columns = purch.columns.get_level_values(1) \n    \n    purch = purch.rename(columns = {'first': 'first_transaction_date', 'last': 'last_transaction_date', 'count': 'transaction_count'}) \n\n    purch['last_transaction_date'] = pd.to_datetime(purch['last_transaction_date'], errors=\"coerce\")\n    purch['first_transaction_date'] = pd.to_datetime(purch['first_transaction_date'], errors=\"coerce\")\n\n    purch['period_purch'] = abs(purch['last_transaction_date'] - purch['first_transaction_date'])\n    purch['period_purch'] = purch['period_purch'].dt.days\n\n    purch['frequency'] = round(purch['period_purch'] / purch['transaction_count'])\n    \n    return purch","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:47:05.373444Z","iopub.execute_input":"2021-09-23T15:47:05.373787Z","iopub.status.idle":"2021-09-23T15:47:05.384220Z","shell.execute_reply.started":"2021-09-23T15:47:05.373746Z","shell.execute_reply":"2021-09-23T15:47:05.383109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"purch_train = frequency_purch(train_purch)\npurch_test = frequency_purch(test_purch)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:47:05.386088Z","iopub.execute_input":"2021-09-23T15:47:05.386401Z","iopub.status.idle":"2021-09-23T15:47:56.759026Z","shell.execute_reply.started":"2021-09-23T15:47:05.386362Z","shell.execute_reply":"2021-09-23T15:47:56.758156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"purch_train.head(3)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:47:56.760345Z","iopub.execute_input":"2021-09-23T15:47:56.760566Z","iopub.status.idle":"2021-09-23T15:47:56.773085Z","shell.execute_reply.started":"2021-09-23T15:47:56.760541Z","shell.execute_reply":"2021-09-23T15:47:56.772210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_purch_group = train_purch_group.join(purch_train.set_index('client_id'), on='client_id')\ntest_purch_group = test_purch_group.join(purch_test.set_index('client_id'), on='client_id')\ntrain_purch_group.head(3)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:47:56.774849Z","iopub.execute_input":"2021-09-23T15:47:56.775189Z","iopub.status.idle":"2021-09-23T15:47:57.057025Z","shell.execute_reply.started":"2021-09-23T15:47:56.775148Z","shell.execute_reply":"2021-09-23T15:47:57.056336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = train_purch_group.join(df_train.set_index('client_id'), on='client_id')\n","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:47:57.058122Z","iopub.execute_input":"2021-09-23T15:47:57.058390Z","iopub.status.idle":"2021-09-23T15:47:57.297994Z","shell.execute_reply.started":"2021-09-23T15:47:57.058362Z","shell.execute_reply":"2021-09-23T15:47:57.297090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = test_purch_group.join(df_test.set_index('client_id'), on='client_id')","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:47:57.299388Z","iopub.execute_input":"2021-09-23T15:47:57.299638Z","iopub.status.idle":"2021-09-23T15:47:57.386989Z","shell.execute_reply.started":"2021-09-23T15:47:57.299610Z","shell.execute_reply":"2021-09-23T15:47:57.386221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head(3)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:47:57.388277Z","iopub.execute_input":"2021-09-23T15:47:57.388529Z","iopub.status.idle":"2021-09-23T15:47:57.415965Z","shell.execute_reply.started":"2021-09-23T15:47:57.388500Z","shell.execute_reply":"2021-09-23T15:47:57.415161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = df_train.drop(['first_redeem_date'], axis = 1)\ndf_test = df_test.drop(['first_redeem_date'], axis = 1)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:47:57.417086Z","iopub.execute_input":"2021-09-23T15:47:57.417326Z","iopub.status.idle":"2021-09-23T15:47:57.623945Z","shell.execute_reply.started":"2021-09-23T15:47:57.417301Z","shell.execute_reply":"2021-09-23T15:47:57.622975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#data_test.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:47:57.625248Z","iopub.execute_input":"2021-09-23T15:47:57.625480Z","iopub.status.idle":"2021-09-23T15:47:57.629917Z","shell.execute_reply.started":"2021-09-23T15:47:57.625437Z","shell.execute_reply":"2021-09-23T15:47:57.628908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#data_train.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:47:57.631183Z","iopub.execute_input":"2021-09-23T15:47:57.631401Z","iopub.status.idle":"2021-09-23T15:47:57.641201Z","shell.execute_reply.started":"2021-09-23T15:47:57.631375Z","shell.execute_reply":"2021-09-23T15:47:57.640478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_train = df_train.copy()","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:47:57.642641Z","iopub.execute_input":"2021-09-23T15:47:57.642859Z","iopub.status.idle":"2021-09-23T15:47:57.694395Z","shell.execute_reply.started":"2021-09-23T15:47:57.642834Z","shell.execute_reply":"2021-09-23T15:47:57.693674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_train.columns","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:47:57.695659Z","iopub.execute_input":"2021-09-23T15:47:57.695868Z","iopub.status.idle":"2021-09-23T15:47:57.701530Z","shell.execute_reply.started":"2021-09-23T15:47:57.695842Z","shell.execute_reply":"2021-09-23T15:47:57.700772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_attribs = ['regular_points_received', 'express_points_received',  'regular_points_spent', \n               'express_points_spent', 'trn_sum_from_iss', 'trn_sum_from_red',\n               'purch_popular_product', 'is_own_trademark', 'transaction_count','frequency', 'age']","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:47:57.702943Z","iopub.execute_input":"2021-09-23T15:47:57.703207Z","iopub.status.idle":"2021-09-23T15:47:57.714350Z","shell.execute_reply.started":"2021-09-23T15:47:57.703180Z","shell.execute_reply":"2021-09-23T15:47:57.713692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"date_attribs = ['first_transaction_date', 'last_transaction_date', 'first_issue_date']","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:47:57.720800Z","iopub.execute_input":"2021-09-23T15:47:57.721555Z","iopub.status.idle":"2021-09-23T15:47:57.726636Z","shell.execute_reply.started":"2021-09-23T15:47:57.721521Z","shell.execute_reply":"2021-09-23T15:47:57.725925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_attribs = ['client_id', 'store_id', 'gender']","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:47:57.728106Z","iopub.execute_input":"2021-09-23T15:47:57.728558Z","iopub.status.idle":"2021-09-23T15:47:57.742717Z","shell.execute_reply.started":"2021-09-23T15:47:57.728513Z","shell.execute_reply":"2021-09-23T15:47:57.741873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_train.drop(date_attribs + cat_attribs, axis=1).describe()","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:47:57.743922Z","iopub.execute_input":"2021-09-23T15:47:57.744365Z","iopub.status.idle":"2021-09-23T15:47:58.053946Z","shell.execute_reply.started":"2021-09-23T15:47:57.744321Z","shell.execute_reply":"2021-09-23T15:47:58.053084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corr_matrix = data_train.corr(method = 'spearman')\nprint(corr_matrix[\"target\"].sort_values(ascending=False))","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:47:58.055316Z","iopub.execute_input":"2021-09-23T15:47:58.055536Z","iopub.status.idle":"2021-09-23T15:47:59.829942Z","shell.execute_reply.started":"2021-09-23T15:47:58.055511Z","shell.execute_reply":"2021-09-23T15:47:59.829109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(9, 10))\nnum = ['product_quantity', 'trn_sum_from_iss', 'trn_sum_from_red',\n        'purch_popular_product','transaction_count', 'period_purch', 'frequency', 'age', 'is_own_trademark', 'purch_popular_product']\nhm = sns.heatmap(data_train[num].corr(),annot=True,)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:47:59.831248Z","iopub.execute_input":"2021-09-23T15:47:59.831449Z","iopub.status.idle":"2021-09-23T15:48:01.111068Z","shell.execute_reply.started":"2021-09-23T15:47:59.831424Z","shell.execute_reply":"2021-09-23T15:48:01.110392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(9, 10))\nran = ['product_quantity', 'trn_sum_from_iss', 'trn_sum_from_red', 'is_alcohol',\n        'purch_popular_product','transaction_count', 'period_purch', 'frequency', 'treatment_flg', 'age', 'is_own_trademark', 'purch_popular_product']\nhm = sns.heatmap(data_train[ran].corr(method = 'spearman'), annot=True)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:01.112030Z","iopub.execute_input":"2021-09-23T15:48:01.112758Z","iopub.status.idle":"2021-09-23T15:48:03.351235Z","shell.execute_reply.started":"2021-09-23T15:48:01.112708Z","shell.execute_reply":"2021-09-23T15:48:03.350247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Исследуем значимость корреляций нескольких величин","metadata":{}},{"cell_type":"code","source":"rho, pval = scipy.stats.spearmanr(data_train['product_quantity'], data_train['trn_sum_from_iss']) \nrho, pval","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:03.353096Z","iopub.execute_input":"2021-09-23T15:48:03.353628Z","iopub.status.idle":"2021-09-23T15:48:03.489530Z","shell.execute_reply.started":"2021-09-23T15:48:03.353584Z","shell.execute_reply":"2021-09-23T15:48:03.488543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Т.к. pval<0.05, то коэффициент корреляции является значимым","metadata":{}},{"cell_type":"code","source":"rho, pval = scipy.stats.spearmanr(data_train['is_own_trademark'], data_train['trn_sum_from_iss']) \nrho, pval","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:03.490770Z","iopub.execute_input":"2021-09-23T15:48:03.490991Z","iopub.status.idle":"2021-09-23T15:48:03.615323Z","shell.execute_reply.started":"2021-09-23T15:48:03.490966Z","shell.execute_reply":"2021-09-23T15:48:03.614419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rho, pval = scipy.stats.spearmanr(data_train['period_purch'], data_train['frequency']) \nrho, pval","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:03.616560Z","iopub.execute_input":"2021-09-23T15:48:03.616798Z","iopub.status.idle":"2021-09-23T15:48:03.708362Z","shell.execute_reply.started":"2021-09-23T15:48:03.616769Z","shell.execute_reply":"2021-09-23T15:48:03.707380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rho, pval = scipy.stats.spearmanr(data_train['is_own_trademark'], data_train['purch_popular_product']) \nrho, pval","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:03.709900Z","iopub.execute_input":"2021-09-23T15:48:03.710254Z","iopub.status.idle":"2021-09-23T15:48:03.791171Z","shell.execute_reply.started":"2021-09-23T15:48:03.710211Z","shell.execute_reply":"2021-09-23T15:48:03.790320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rho, pval = scipy.stats.spearmanr(data_train['treatment_flg'], data_train['trn_sum_from_iss']) \nrho, pval","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:03.792685Z","iopub.execute_input":"2021-09-23T15:48:03.792930Z","iopub.status.idle":"2021-09-23T15:48:03.901007Z","shell.execute_reply.started":"2021-09-23T15:48:03.792905Z","shell.execute_reply":"2021-09-23T15:48:03.900061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"В данном случае коэффициент корреляции не является значимым","metadata":{}},{"cell_type":"code","source":"rho, pval = scipy.stats.spearmanr(data_train['age'], data_train['trn_sum_from_iss']) \nrho, pval","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:03.902319Z","iopub.execute_input":"2021-09-23T15:48:03.902566Z","iopub.status.idle":"2021-09-23T15:48:04.025211Z","shell.execute_reply.started":"2021-09-23T15:48:03.902523Z","shell.execute_reply":"2021-09-23T15:48:04.024553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Коэффициент корреляции является значимым","metadata":{}},{"cell_type":"markdown","source":"Удалим  из набора данных атрибуты 'period_purch', 'product_quantity'","metadata":{}},{"cell_type":"code","source":"data_train = data_train.drop(['period_purch', 'product_quantity', 'treatment_flg'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:04.026278Z","iopub.execute_input":"2021-09-23T15:48:04.026494Z","iopub.status.idle":"2021-09-23T15:48:04.067367Z","shell.execute_reply.started":"2021-09-23T15:48:04.026469Z","shell.execute_reply":"2021-09-23T15:48:04.066429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#data_train.shape","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:04.069117Z","iopub.execute_input":"2021-09-23T15:48:04.069354Z","iopub.status.idle":"2021-09-23T15:48:04.073498Z","shell.execute_reply.started":"2021-09-23T15:48:04.069328Z","shell.execute_reply":"2021-09-23T15:48:04.072518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_test = df_test.drop(['period_purch', 'product_quantity'], axis=1).copy()","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:04.074707Z","iopub.execute_input":"2021-09-23T15:48:04.074950Z","iopub.status.idle":"2021-09-23T15:48:04.113232Z","shell.execute_reply.started":"2021-09-23T15:48:04.074921Z","shell.execute_reply":"2021-09-23T15:48:04.112563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Исследуем набор данных на наличие выбросов и удалим некоторые из них","metadata":{}},{"cell_type":"code","source":"f, ax = plt.subplots(figsize=(6, 6))\nfig = sns.boxplot(x='gender', y='trn_sum_from_iss', data=data_train)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:04.114315Z","iopub.execute_input":"2021-09-23T15:48:04.115063Z","iopub.status.idle":"2021-09-23T15:48:04.801218Z","shell.execute_reply.started":"2021-09-23T15:48:04.115007Z","shell.execute_reply":"2021-09-23T15:48:04.800605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f, ax = plt.subplots(figsize=(6, 6))\nfig = sns.boxplot(x='gender', y='trn_sum_from_red', data=data_train)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:04.802238Z","iopub.execute_input":"2021-09-23T15:48:04.802973Z","iopub.status.idle":"2021-09-23T15:48:05.671470Z","shell.execute_reply.started":"2021-09-23T15:48:04.802937Z","shell.execute_reply":"2021-09-23T15:48:05.670778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#f, ax = plt.subplots(figsize=(6, 6))\n#fig = sns.boxplot(x='gender', y='is_own_trademark', data=data_train)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:05.672559Z","iopub.execute_input":"2021-09-23T15:48:05.674387Z","iopub.status.idle":"2021-09-23T15:48:05.678422Z","shell.execute_reply.started":"2021-09-23T15:48:05.674349Z","shell.execute_reply":"2021-09-23T15:48:05.677471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f, ax = plt.subplots(figsize=(6, 6))\nfig = sns.boxplot(x='gender', y='transaction_count', data=data_train)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:05.679598Z","iopub.execute_input":"2021-09-23T15:48:05.680449Z","iopub.status.idle":"2021-09-23T15:48:06.282560Z","shell.execute_reply.started":"2021-09-23T15:48:05.680415Z","shell.execute_reply":"2021-09-23T15:48:06.281756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#f, ax = plt.subplots(figsize=(6, 6))\n#fig = sns.boxplot(x='gender', y='is_alcohol', data=data_train)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:06.283792Z","iopub.execute_input":"2021-09-23T15:48:06.284039Z","iopub.status.idle":"2021-09-23T15:48:06.288180Z","shell.execute_reply.started":"2021-09-23T15:48:06.284010Z","shell.execute_reply":"2021-09-23T15:48:06.287228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#f, ax = plt.subplots(figsize=(6, 6))\n#fig = sns.boxplot(x='gender', y='regular_points_received', data=data_train)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:06.289863Z","iopub.execute_input":"2021-09-23T15:48:06.290115Z","iopub.status.idle":"2021-09-23T15:48:06.300364Z","shell.execute_reply.started":"2021-09-23T15:48:06.290086Z","shell.execute_reply":"2021-09-23T15:48:06.299572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#f, ax = plt.subplots(figsize=(6, 6))\n#fig = sns.boxplot(x='gender', y='express_points_received', data=data_train)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:06.301698Z","iopub.execute_input":"2021-09-23T15:48:06.302250Z","iopub.status.idle":"2021-09-23T15:48:06.312524Z","shell.execute_reply.started":"2021-09-23T15:48:06.302215Z","shell.execute_reply":"2021-09-23T15:48:06.311573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#data_train[['purch_popular_product',  'frequency']].plot.box()","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:06.314026Z","iopub.execute_input":"2021-09-23T15:48:06.314302Z","iopub.status.idle":"2021-09-23T15:48:06.323579Z","shell.execute_reply.started":"2021-09-23T15:48:06.314272Z","shell.execute_reply":"2021-09-23T15:48:06.322501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_train['age'].plot.box()","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:06.325063Z","iopub.execute_input":"2021-09-23T15:48:06.325300Z","iopub.status.idle":"2021-09-23T15:48:06.538222Z","shell.execute_reply.started":"2021-09-23T15:48:06.325274Z","shell.execute_reply":"2021-09-23T15:48:06.537289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"col_outlier = ['trn_sum_from_iss', 'purch_popular_product', 'is_own_trademark', 'transaction_count', 'age']","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:06.539405Z","iopub.execute_input":"2021-09-23T15:48:06.539704Z","iopub.status.idle":"2021-09-23T15:48:06.544673Z","shell.execute_reply.started":"2021-09-23T15:48:06.539674Z","shell.execute_reply":"2021-09-23T15:48:06.543535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q1 = data_train['age'].quantile(0.99)\n#q2 = data_train['trn_sum_from_iss'].quantile(0.99)\n#q3 = data_train['purch_popular_product'].quantile(0.99)\n#q4 = data_train['frequency'].quantile(0.99)\n#q5 = data_train['is_own_trademark'].quantile(0.99)\n#q6 = data_train['transaction_count'].quantile(0.99)\n#q7 = data_train['trn_sum_from_red'].quantile(0.99)\n\n\ndata_train = data_train[data_train['age'] < q1]\ndata_train = data_train[data_train['age'] > 16]\n#data_train = data_train[data_train['trn_sum_from_iss'] < q2]\n#data_train = data_train[data_train['purch_popular_product'] < q3]\n#data_train = data_train[data_train['frequency'] < q4]\n#data_train = data_train[data_train['is_own_trademark'] < q5]\n#data_train = data_train[data_train['transaction_count'] < q6]\n#data_train = data_train[data_train['trn_sum_from_red'] < q7]","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:06.546832Z","iopub.execute_input":"2021-09-23T15:48:06.547689Z","iopub.status.idle":"2021-09-23T15:48:06.657593Z","shell.execute_reply.started":"2021-09-23T15:48:06.547640Z","shell.execute_reply":"2021-09-23T15:48:06.656950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%matplotlib inline\nimport matplotlib.pyplot as plt\ndata_train[['trn_sum_from_iss','purch_popular_product', 'age', 'transaction_count', 'is_own_trademark', 'trn_sum_from_red']].hist(bins=50, figsize=(20,15))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:06.658964Z","iopub.execute_input":"2021-09-23T15:48:06.659428Z","iopub.status.idle":"2021-09-23T15:48:08.394132Z","shell.execute_reply.started":"2021-09-23T15:48:06.659398Z","shell.execute_reply":"2021-09-23T15:48:08.393246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train = data_train['target'].copy()\nX_train = data_train.drop(columns=['target'], axis=1).copy()","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:08.395340Z","iopub.execute_input":"2021-09-23T15:48:08.395604Z","iopub.status.idle":"2021-09-23T15:48:08.467726Z","shell.execute_reply.started":"2021-09-23T15:48:08.395576Z","shell.execute_reply":"2021-09-23T15:48:08.466768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.shape, y_train.shape","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:08.468938Z","iopub.execute_input":"2021-09-23T15:48:08.469186Z","iopub.status.idle":"2021-09-23T15:48:08.474889Z","shell.execute_reply.started":"2021-09-23T15:48:08.469157Z","shell.execute_reply":"2021-09-23T15:48:08.474002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import StandardScaler","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:08.476500Z","iopub.execute_input":"2021-09-23T15:48:08.477058Z","iopub.status.idle":"2021-09-23T15:48:08.487546Z","shell.execute_reply.started":"2021-09-23T15:48:08.477025Z","shell.execute_reply":"2021-09-23T15:48:08.486756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomDateAttrs(BaseEstimator, TransformerMixin):\n    def __init__(self):\n        pass\n\n    def fit(self, X, y=None):\n        return self\n\n    def transform(self, X):\n        X['year_first_transaction'] = X['first_transaction_date'].apply(lambda d: d.year)\n        X['month_first_transaction'] = X['first_transaction_date'].apply(lambda d: d.month)\n        X['day_first_transaction'] = X['first_transaction_date'].apply(lambda d: d.day)\n\n        X['year_last_transaction'] = X['last_transaction_date'].apply(lambda d: d.year)\n        X['month_last_transaction'] = X['last_transaction_date'].apply(lambda d: d.month)\n        X['day_last_transaction'] = X['last_transaction_date'].apply(lambda d: d.day)\n\n        X['year_first_issue'] = X['first_issue_date'].apply(lambda d: d.year)\n        X['month_first_issue'] = X['first_issue_date'].apply(lambda d: d.month)\n        X['day_first_issue'] = X['first_issue_date'].apply(lambda d: d.day)\n\n        X = X.drop(['first_transaction_date', 'last_transaction_date', 'first_issue_date'], axis =1)\n        return X","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:08.489088Z","iopub.execute_input":"2021-09-23T15:48:08.489837Z","iopub.status.idle":"2021-09-23T15:48:08.500441Z","shell.execute_reply.started":"2021-09-23T15:48:08.489804Z","shell.execute_reply":"2021-09-23T15:48:08.499577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomObjectAttrs(BaseEstimator, TransformerMixin):\n    def __init__(self):\n        pass\n\n    def fit(self, X, y=None):\n        return self\n\n    def transform(self, X):\n        X['client_id_cod'] = pd.factorize(X['client_id'])[0]\n        X['store_id'] = pd.factorize(X['store_id'])[0]\n        X['gender'] = pd.factorize(X['gender'])[0]\n        return X","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:08.501786Z","iopub.execute_input":"2021-09-23T15:48:08.502252Z","iopub.status.idle":"2021-09-23T15:48:08.513199Z","shell.execute_reply.started":"2021-09-23T15:48:08.502219Z","shell.execute_reply":"2021-09-23T15:48:08.512151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.compose import ColumnTransformer\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import OrdinalEncoder\n\nfull_pipeline = ColumnTransformer([\n        (\"cat\", CustomObjectAttrs(), cat_attribs),\n        (\"num\", SimpleImputer(strategy=\"median\"), num_attribs + ['is_alcohol']),\n        (\"date\", CustomDateAttrs(), date_attribs)\n    ])","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:08.514621Z","iopub.execute_input":"2021-09-23T15:48:08.514952Z","iopub.status.idle":"2021-09-23T15:48:08.586024Z","shell.execute_reply.started":"2021-09-23T15:48:08.514924Z","shell.execute_reply":"2021-09-23T15:48:08.585022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_attribs = ['regular_points_received', 'express_points_received',  'regular_points_spent', \n               'express_points_spent', 'trn_sum_from_iss', 'trn_sum_from_red',\n               'purch_popular_product', 'is_own_trademark', 'transaction_count','frequency', 'age']","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:08.587417Z","iopub.execute_input":"2021-09-23T15:48:08.587651Z","iopub.status.idle":"2021-09-23T15:48:08.594428Z","shell.execute_reply.started":"2021-09-23T15:48:08.587624Z","shell.execute_reply":"2021-09-23T15:48:08.593585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.shape","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:08.597218Z","iopub.execute_input":"2021-09-23T15:48:08.597545Z","iopub.status.idle":"2021-09-23T15:48:08.604409Z","shell.execute_reply.started":"2021-09-23T15:48:08.597509Z","shell.execute_reply":"2021-09-23T15:48:08.603555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_transformed = full_pipeline.fit_transform(X_train)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:08.605865Z","iopub.execute_input":"2021-09-23T15:48:08.606168Z","iopub.status.idle":"2021-09-23T15:48:27.602450Z","shell.execute_reply.started":"2021-09-23T15:48:08.606131Z","shell.execute_reply":"2021-09-23T15:48:27.601789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_transformed[:,3]","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:27.603435Z","iopub.execute_input":"2021-09-23T15:48:27.603963Z","iopub.status.idle":"2021-09-23T15:48:27.610192Z","shell.execute_reply.started":"2021-09-23T15:48:27.603930Z","shell.execute_reply":"2021-09-23T15:48:27.609290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_test.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:27.611274Z","iopub.execute_input":"2021-09-23T15:48:27.611488Z","iopub.status.idle":"2021-09-23T15:48:27.640688Z","shell.execute_reply.started":"2021-09-23T15:48:27.611463Z","shell.execute_reply":"2021-09-23T15:48:27.640066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test_transformed = full_pipeline.transform(data_test)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:27.641644Z","iopub.execute_input":"2021-09-23T15:48:27.642307Z","iopub.status.idle":"2021-09-23T15:48:35.552885Z","shell.execute_reply.started":"2021-09-23T15:48:27.642267Z","shell.execute_reply":"2021-09-23T15:48:35.552254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" X_test_transformed.shape","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:35.553970Z","iopub.execute_input":"2021-09-23T15:48:35.554681Z","iopub.status.idle":"2021-09-23T15:48:35.559605Z","shell.execute_reply.started":"2021-09-23T15:48:35.554638Z","shell.execute_reply":"2021-09-23T15:48:35.558761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" X_test_transformed","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:35.560598Z","iopub.execute_input":"2021-09-23T15:48:35.560871Z","iopub.status.idle":"2021-09-23T15:48:35.573933Z","shell.execute_reply.started":"2021-09-23T15:48:35.560844Z","shell.execute_reply":"2021-09-23T15:48:35.573142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_columns = ['client_id', 'store_id', 'gender', 'client_cod']+ num_attribs + ['is_alcohol', 'year_first_transaction', 'month_first_transaction',\n                                                    'day_first_transaction', 'year_last_transaction', 'month_last_transaction', 'day_last_transaction',\n                                                    'year_first_issue', 'month_first_issue', 'day_first_issue'] ","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:35.575049Z","iopub.execute_input":"2021-09-23T15:48:35.575377Z","iopub.status.idle":"2021-09-23T15:48:35.583085Z","shell.execute_reply.started":"2021-09-23T15:48:35.575345Z","shell.execute_reply":"2021-09-23T15:48:35.582559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_transformed = pd.DataFrame(X_train_transformed, columns=feature_columns)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:35.584167Z","iopub.execute_input":"2021-09-23T15:48:35.584788Z","iopub.status.idle":"2021-09-23T15:48:35.912504Z","shell.execute_reply.started":"2021-09-23T15:48:35.584756Z","shell.execute_reply":"2021-09-23T15:48:35.911600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test_transformed = pd.DataFrame(X_test_transformed, columns=feature_columns)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:35.913898Z","iopub.execute_input":"2021-09-23T15:48:35.914146Z","iopub.status.idle":"2021-09-23T15:48:36.059767Z","shell.execute_reply.started":"2021-09-23T15:48:35.914117Z","shell.execute_reply":"2021-09-23T15:48:36.059026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test_transformed.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:36.060766Z","iopub.execute_input":"2021-09-23T15:48:36.061110Z","iopub.status.idle":"2021-09-23T15:48:36.087171Z","shell.execute_reply.started":"2021-09-23T15:48:36.061078Z","shell.execute_reply":"2021-09-23T15:48:36.086248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Разделим набор Train на наборы Train и Val в соотношении 90:10","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_train_transformed, X_val_transformed, Y_train, Y_val = train_test_split(X_train_transformed, y_train, test_size=0.1, random_state=2018)\n\nprint('Shape of x_train:', X_train_transformed.shape)\nprint('Shape of x_val:', X_val_transformed.shape)\nprint('Shape of y_train:', Y_train.shape)\nprint('Shape of y_val:', Y_val.shape)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:36.088428Z","iopub.execute_input":"2021-09-23T15:48:36.088692Z","iopub.status.idle":"2021-09-23T15:48:37.006200Z","shell.execute_reply.started":"2021-09-23T15:48:36.088663Z","shell.execute_reply":"2021-09-23T15:48:37.005441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Стандартизаци и нормализация данных","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler, MinMaxScaler\n\nscaler = MinMaxScaler()\nscaler.fit(X_train_transformed.drop('client_id', axis = 1))\n\nX_train_scaled = scaler.transform(X_train_transformed.drop('client_id', axis = 1))\nX_val_scaled = scaler.transform(X_val_transformed.drop('client_id', axis = 1))\nX_test_scaled = scaler.transform(X_test_transformed.drop('client_id', axis = 1 ))","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:37.007306Z","iopub.execute_input":"2021-09-23T15:48:37.007565Z","iopub.status.idle":"2021-09-23T15:48:39.757296Z","shell.execute_reply.started":"2021-09-23T15:48:37.007536Z","shell.execute_reply":"2021-09-23T15:48:39.756317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Нейронная сеть","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tensorflow.python.keras.layers import Input, Dense","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:39.758635Z","iopub.execute_input":"2021-09-23T15:48:39.758953Z","iopub.status.idle":"2021-09-23T15:48:44.823977Z","shell.execute_reply.started":"2021-09-23T15:48:39.758912Z","shell.execute_reply":"2021-09-23T15:48:44.823120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def act(x):\n    return keras.activations.elu(x, alpha=0.1)\n    #return keras.activations.relu(x, alpha=0.05)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:44.825415Z","iopub.execute_input":"2021-09-23T15:48:44.825769Z","iopub.status.idle":"2021-09-23T15:48:44.830738Z","shell.execute_reply.started":"2021-09-23T15:48:44.825727Z","shell.execute_reply":"2021-09-23T15:48:44.829861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"he_inint = keras.initializers.VarianceScaling(\n    scale=1.0, mode=\"fan_avg\", distribution=\"truncated_normal\", seed=None\n)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:44.832025Z","iopub.execute_input":"2021-09-23T15:48:44.832331Z","iopub.status.idle":"2021-09-23T15:48:44.846236Z","shell.execute_reply.started":"2021-09-23T15:48:44.832293Z","shell.execute_reply":"2021-09-23T15:48:44.845072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = keras.Sequential(\n    [\n        keras.Input(shape=X_train_scaled.shape[1]),\n        layers.Dense(16, kernel_initializer = he_inint, activation=act),\n        #layers.BatchNormalization(),\n        layers.Dense(16, kernel_initializer = he_inint, activation=act),\n        layers.Dropout(0.5),\n        layers.Dense(8, kernel_initializer = he_inint, activation=act),\n        layers.Dropout(0.5),\n        layers.Dense(1, activation=\"sigmoid\"),\n    ]\n)\n\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:44.847657Z","iopub.execute_input":"2021-09-23T15:48:44.847904Z","iopub.status.idle":"2021-09-23T15:48:44.972632Z","shell.execute_reply.started":"2021-09-23T15:48:44.847875Z","shell.execute_reply":"2021-09-23T15:48:44.971694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#opt = keras.optimizers.SGD(learning_rate=0.03, momentum=0.1, nesterov=True, name=\"SGD\")\n\nmodel.compile(optimizer = 'rmsprop', loss = \"binary_crossentropy\", metrics=[\"accuracy\"])\n\nmodel.fit(X_train_scaled, Y_train, validation_data = (X_val_scaled, Y_val), epochs=60, batch_size=200)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:48:44.974208Z","iopub.execute_input":"2021-09-23T15:48:44.974563Z","iopub.status.idle":"2021-09-23T15:50:53.845426Z","shell.execute_reply.started":"2021-09-23T15:48:44.974517Z","shell.execute_reply":"2021-09-23T15:50:53.844823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(model.history.history['loss'])\nplt.plot(model.history.history['val_loss'])\nplt.title(\"Model's Training & Validation loss across apochs\")\nplt.ylabel('Loss')\nplt.xlabel('Epochs')\nplt.legend(['Train', 'Validation'], loc='upper right')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:50:53.848681Z","iopub.execute_input":"2021-09-23T15:50:53.849174Z","iopub.status.idle":"2021-09-23T15:50:54.286011Z","shell.execute_reply.started":"2021-09-23T15:50:53.849135Z","shell.execute_reply":"2021-09-23T15:50:54.285189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(model.history.history['accuracy'])\nplt.plot(model.history.history['val_accuracy'])\nplt.title(\"Model's Training & Validation accuracy across apochs\")\nplt.ylabel('Accuracy')\nplt.xlabel('Epochs')\nplt.legend(['Train', 'Validation'], loc='upper right')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:50:54.287327Z","iopub.execute_input":"2021-09-23T15:50:54.287566Z","iopub.status.idle":"2021-09-23T15:50:54.520469Z","shell.execute_reply.started":"2021-09-23T15:50:54.287530Z","shell.execute_reply":"2021-09-23T15:50:54.519597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prediction=model.predict(X_test_scaled)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:50:54.529429Z","iopub.execute_input":"2021-09-23T15:50:54.529766Z","iopub.status.idle":"2021-09-23T15:50:57.863012Z","shell.execute_reply.started":"2021-09-23T15:50:54.529715Z","shell.execute_reply":"2021-09-23T15:50:57.862202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Cassifiers","metadata":{}},{"cell_type":"code","source":"data_train_drop = data_train.drop_duplicates(subset=['client_id']).copy()\ndata_test_drop = data_test.drop_duplicates(subset=['client_id']).copy()","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:50:57.865438Z","iopub.execute_input":"2021-09-23T15:50:57.865801Z","iopub.status.idle":"2021-09-23T15:50:58.026144Z","shell.execute_reply.started":"2021-09-23T15:50:57.865759Z","shell.execute_reply":"2021-09-23T15:50:58.025227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train_drop = data_train_drop['target'].copy()\nX_train_drop = data_train_drop.drop(columns=['store_id','target'], axis=1).copy()","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:50:58.027182Z","iopub.execute_input":"2021-09-23T15:50:58.027396Z","iopub.status.idle":"2021-09-23T15:50:58.070615Z","shell.execute_reply.started":"2021-09-23T15:50:58.027371Z","shell.execute_reply":"2021-09-23T15:50:58.069719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomObjectAttrs1(BaseEstimator, TransformerMixin):\n    def __init__(self):\n        pass\n\n    def fit(self, X, y=None):\n        return self\n\n    def transform(self, X):\n        X['client_id_cod'] = pd.factorize(X['client_id'])[0]\n        #X['store_id'] = pd.factorize(X['store_id'])[0]\n        X['gender'] = pd.factorize(X['gender'])[0]\n        return X","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:50:58.071939Z","iopub.execute_input":"2021-09-23T15:50:58.072199Z","iopub.status.idle":"2021-09-23T15:50:58.079785Z","shell.execute_reply.started":"2021-09-23T15:50:58.072163Z","shell.execute_reply":"2021-09-23T15:50:58.078122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_pipeline1 = ColumnTransformer([\n        (\"cat\", CustomObjectAttrs1(), ['client_id', 'gender']),\n        (\"num\", SimpleImputer(strategy=\"median\"), num_attribs + ['is_alcohol']),\n        (\"date\", CustomDateAttrs(), date_attribs)\n    ])","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:50:58.081794Z","iopub.execute_input":"2021-09-23T15:50:58.082717Z","iopub.status.idle":"2021-09-23T15:50:58.092806Z","shell.execute_reply.started":"2021-09-23T15:50:58.082672Z","shell.execute_reply":"2021-09-23T15:50:58.091865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_drop = full_pipeline1.fit_transform(X_train_drop)\nX_test_drop = full_pipeline1.fit_transform(data_test_drop)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:50:58.094184Z","iopub.execute_input":"2021-09-23T15:50:58.094617Z","iopub.status.idle":"2021-09-23T15:51:07.526103Z","shell.execute_reply.started":"2021-09-23T15:50:58.094571Z","shell.execute_reply":"2021-09-23T15:51:07.525227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_columns = ['client_id', 'gender', 'client_id_cod'] + num_attribs + ['is_alcohol', 'year_first_transaction', 'month_first_transaction',\n                                                    'day_first_transaction', 'year_last_transaction', 'month_last_transaction', 'day_last_transaction',\n                                                    'year_first_issue', 'month_first_issue', 'day_first_issue'] ","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:51:07.527593Z","iopub.execute_input":"2021-09-23T15:51:07.527908Z","iopub.status.idle":"2021-09-23T15:51:07.534205Z","shell.execute_reply.started":"2021-09-23T15:51:07.527869Z","shell.execute_reply":"2021-09-23T15:51:07.533188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_drop = pd.DataFrame(X_train_drop, columns=feature_columns)\nX_test_drop = pd.DataFrame(X_test_drop, columns=feature_columns)\n\n\n\nfrom sklearn.model_selection import train_test_split\n\nX_train_drop, X_val_drop, Y_train_drop, Y_val_drop = train_test_split(X_train_drop, y_train_drop, test_size=0.1, random_state=2018)\n\nprint('Shape of x_train:', X_train_drop.shape)\nprint('Shape of x_val:', X_val_drop.shape)\nprint('Shape of y_train:', Y_train_drop.shape)\nprint('Shape of y_val:', Y_val_drop.shape)\n","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:51:07.535442Z","iopub.execute_input":"2021-09-23T15:51:07.535788Z","iopub.status.idle":"2021-09-23T15:51:07.873089Z","shell.execute_reply.started":"2021-09-23T15:51:07.535746Z","shell.execute_reply":"2021-09-23T15:51:07.871827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler, MinMaxScaler\n\nscaler = StandardScaler()\nscaler.fit(X_train_drop.drop('client_id', axis = 1))\n\nX_train_drop_sc = scaler.transform(X_train_drop.drop('client_id', axis = 1))\nX_val_drop_sc = scaler.transform(X_val_drop.drop('client_id', axis = 1))\nX_test_drop_sc = scaler.transform(X_test_drop.drop('client_id', axis = 1))","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:56:21.392334Z","iopub.execute_input":"2021-09-23T15:56:21.393191Z","iopub.status.idle":"2021-09-23T15:56:22.436272Z","shell.execute_reply.started":"2021-09-23T15:56:21.393132Z","shell.execute_reply":"2021-09-23T15:56:22.435614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.ensemble import BaggingClassifier\nfrom sklearn.tree import DecisionTreeClassifier\n\nbag_clf = BaggingClassifier(\n    DecisionTreeClassifier(random_state=42), n_estimators=500,\n    max_samples=100, bootstrap=True, n_jobs=-1, random_state=42)\nbag_clf.fit(X_train_drop_sc, Y_train_drop)\ny_pred = bag_clf.predict(X_val_drop_sc)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:57:33.231438Z","iopub.execute_input":"2021-09-23T15:57:33.232389Z","iopub.status.idle":"2021-09-23T15:57:40.904755Z","shell.execute_reply.started":"2021-09-23T15:57:33.232338Z","shell.execute_reply":"2021-09-23T15:57:40.903437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score\nfrom sklearn.metrics import roc_auc_score\n\nprint('acc: ', accuracy_score(Y_val_drop, y_pred))\nprint('gini: ', 2*roc_auc_score(Y_val_drop, y_pred) - 1)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:57:40.907076Z","iopub.execute_input":"2021-09-23T15:57:40.907452Z","iopub.status.idle":"2021-09-23T15:57:40.921943Z","shell.execute_reply.started":"2021-09-23T15:57:40.907407Z","shell.execute_reply":"2021-09-23T15:57:40.921267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tree_clf = DecisionTreeClassifier(random_state=42)\ntree_clf.fit(X_train_drop_sc, Y_train_drop)\ny_pred_tree = tree_clf.predict(X_val_drop_sc)\n\nprint('acc: ', accuracy_score(Y_val_drop, y_pred_tree))\nprint('gini: ', 2*roc_auc_score(Y_val_drop, y_pred_tree) - 1)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:57:46.286504Z","iopub.execute_input":"2021-09-23T15:57:46.287337Z","iopub.status.idle":"2021-09-23T15:57:48.875392Z","shell.execute_reply.started":"2021-09-23T15:57:46.287295Z","shell.execute_reply":"2021-09-23T15:57:48.874523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\nfrom sklearn.model_selection import cross_val_predict\nfrom sklearn.metrics import roc_curve\n\nforest_clf = RandomForestClassifier(n_estimators=100, random_state=4)\nforest_clf.fit(X_train_drop_sc, Y_train_drop)\ny_pred_forest = forest_clf.predict(X_val_drop_sc)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T15:57:53.106279Z","iopub.execute_input":"2021-09-23T15:57:53.106888Z","iopub.status.idle":"2021-09-23T15:58:29.905143Z","shell.execute_reply.started":"2021-09-23T15:57:53.106840Z","shell.execute_reply":"2021-09-23T15:58:29.904231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred_forest = forest_clf.predict(X_val_drop_sc)\n\nprint('acc: ', accuracy_score(Y_val_drop, y_pred_forest))\nprint('gini: ', 2*roc_auc_score(Y_val_drop, y_pred_forest) - 1)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T16:00:04.167809Z","iopub.execute_input":"2021-09-23T16:00:04.168605Z","iopub.status.idle":"2021-09-23T16:00:04.858603Z","shell.execute_reply.started":"2021-09-23T16:00:04.168562Z","shell.execute_reply":"2021-09-23T16:00:04.857578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_probas_forest = cross_val_predict(forest_clf, X_train_drop_sc, Y_train_drop, cv=3,\n                                    method=\"predict_proba\")\n\nfpr_forest, tpr_forest, thresholds_forest = roc_curve(Y_train_drop, y_probas_forest[:, 1])","metadata":{"execution":{"iopub.status.busy":"2021-09-23T16:00:21.228045Z","iopub.execute_input":"2021-09-23T16:00:21.229146Z","iopub.status.idle":"2021-09-23T16:01:26.628441Z","shell.execute_reply.started":"2021-09-23T16:00:21.229095Z","shell.execute_reply":"2021-09-23T16:01:26.627509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_roc_curve(fpr, tpr, label=None):\n    plt.plot(fpr, tpr, linewidth=2, label=label)\n    plt.plot([0, 1], [0, 1], 'k--')\n    plt.axis([0, 1, 0, 1])\n    plt.xlabel('False Positive Rate', fontsize=16)\n    plt.ylabel('True Positive Rate', fontsize=16)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T16:01:26.630697Z","iopub.execute_input":"2021-09-23T16:01:26.631027Z","iopub.status.idle":"2021-09-23T16:01:26.637119Z","shell.execute_reply.started":"2021-09-23T16:01:26.630984Z","shell.execute_reply":"2021-09-23T16:01:26.636505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(8, 6))\nplot_roc_curve(fpr_forest, tpr_forest, \"Random Forest\")\nplt.legend(loc=\"lower right\", fontsize=16)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-09-23T16:01:26.638200Z","iopub.execute_input":"2021-09-23T16:01:26.639019Z","iopub.status.idle":"2021-09-23T16:01:26.864138Z","shell.execute_reply.started":"2021-09-23T16:01:26.638969Z","shell.execute_reply":"2021-09-23T16:01:26.863218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#y_pred = bag_clf.predict_proba(X_test_drop_sc)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T16:03:28.306799Z","iopub.execute_input":"2021-09-23T16:03:28.307608Z","iopub.status.idle":"2021-09-23T16:03:30.552861Z","shell.execute_reply.started":"2021-09-23T16:03:28.307554Z","shell.execute_reply":"2021-09-23T16:03:30.551818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#out_df1 = pd.DataFrame(y_pred[:,1])\n#out_df1.columns = [\"pred\"]\n#out_df1[\"client_id\"] = X_test_drop.client_id.values\n#out_df1 = out_df1.reindex(columns=[\"client_id\"] + list(out_df1.columns[:-1]))\n#out_df = out_df.drop_duplicates(subset=['client_id'])","metadata":{"execution":{"iopub.status.busy":"2021-09-23T16:03:35.694036Z","iopub.execute_input":"2021-09-23T16:03:35.694348Z","iopub.status.idle":"2021-09-23T16:03:35.709741Z","shell.execute_reply.started":"2021-09-23T16:03:35.694315Z","shell.execute_reply":"2021-09-23T16:03:35.708961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#out_df1.to_csv(\"cz.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T16:03:57.319019Z","iopub.execute_input":"2021-09-23T16:03:57.319617Z","iopub.status.idle":"2021-09-23T16:03:57.504293Z","shell.execute_reply.started":"2021-09-23T16:03:57.319566Z","shell.execute_reply":"2021-09-23T16:03:57.503592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"out_df = pd.DataFrame(prediction)\nout_df.columns = [\"pred\"]\nout_df[\"client_id\"] = data_test.client_id.values\nout_df = out_df.reindex(columns=[\"client_id\"] + list(out_df.columns[:-1]))\nout_df = out_df.drop_duplicates(subset=['client_id'])\nout_df.to_csv(\"cz.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-09-23T16:19:11.816632Z","iopub.execute_input":"2021-09-23T16:19:11.817333Z","iopub.status.idle":"2021-09-23T16:19:12.057042Z","shell.execute_reply.started":"2021-09-23T16:19:11.817289Z","shell.execute_reply":"2021-09-23T16:19:12.055942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"out_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-23T16:19:29.275108Z","iopub.execute_input":"2021-09-23T16:19:29.275745Z","iopub.status.idle":"2021-09-23T16:19:29.287196Z","shell.execute_reply.started":"2021-09-23T16:19:29.275706Z","shell.execute_reply":"2021-09-23T16:19:29.286400Z"},"trusted":true},"execution_count":null,"outputs":[]}]}