{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-01-23T05:21:45.819202Z","iopub.execute_input":"2023-01-23T05:21:45.819788Z","iopub.status.idle":"2023-01-23T05:21:45.854785Z","shell.execute_reply.started":"2023-01-23T05:21:45.819713Z","shell.execute_reply":"2023-01-23T05:21:45.853692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%matplotlib inline\nimport cv2\nimport numpy as np\nimport matplotlib.pyplot as plt\n\nfrom sklearn.model_selection import train_test_split, cross_validate, cross_val_predict\nfrom sklearn.metrics import accuracy_score, classification_report\n\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.ensemble import RandomForestClassifier, StackingClassifier, BaggingClassifier\nfrom sklearn.svm import SVC, LinearSVC\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.model_selection import GridSearchCV, cross_val_score","metadata":{"execution":{"iopub.status.busy":"2023-01-23T06:48:10.143815Z","iopub.execute_input":"2023-01-23T06:48:10.144251Z","iopub.status.idle":"2023-01-23T06:48:10.154279Z","shell.execute_reply.started":"2023-01-23T06:48:10.144219Z","shell.execute_reply":"2023-01-23T06:48:10.15266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = np.loadtxt('../input/digit-recognizer/train.csv', delimiter=',', skiprows=1)\ntest = np.loadtxt('../input/digit-recognizer/test.csv', delimiter=',', skiprows=1)","metadata":{"execution":{"iopub.status.busy":"2023-01-23T05:58:28.022674Z","iopub.execute_input":"2023-01-23T05:58:28.023079Z","iopub.status.idle":"2023-01-23T05:58:56.091419Z","shell.execute_reply.started":"2023-01-23T05:58:28.02305Z","shell.execute_reply":"2023-01-23T05:58:56.090099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# сохраняем разметку в отдельную переменную\ntrain_label = train[:, 0]\n# приводим размерность к удобному для обаботки виду\ntrain_img = np.resize(train[:, 1:], (train.shape[0], 28, 28))\ntest_img = np.resize(test, (test.shape[0], 28, 28))","metadata":{"execution":{"iopub.status.busy":"2023-01-23T05:59:43.353269Z","iopub.execute_input":"2023-01-23T05:59:43.354425Z","iopub.status.idle":"2023-01-23T05:59:43.541243Z","shell.execute_reply.started":"2023-01-23T05:59:43.354377Z","shell.execute_reply":"2023-01-23T05:59:43.540285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Визуализируем исходные данные","metadata":{}},{"cell_type":"code","source":"fig = plt.figure(figsize=(20, 10))\nfor i, img in enumerate(train_img[0:5], 1):\n    subplot = fig.add_subplot(1, 7, i)\n    plt.imshow(img, cmap='gray');\n    subplot.set_title('%s' % train_label[i - 1]);","metadata":{"execution":{"iopub.status.busy":"2023-01-23T05:59:45.027136Z","iopub.execute_input":"2023-01-23T05:59:45.02757Z","iopub.status.idle":"2023-01-23T05:59:45.5134Z","shell.execute_reply.started":"2023-01-23T05:59:45.027535Z","shell.execute_reply":"2023-01-23T05:59:45.512541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Вычисляем X и Y составляющие градиента с помощью оператора Собеля","metadata":{}},{"cell_type":"code","source":"train_sobel_x = np.zeros_like(train_img)\ntrain_sobel_y = np.zeros_like(train_img)\nfor i in range(len(train_img)):\n    train_sobel_x[i] = cv2.Sobel(train_img[i], cv2.CV_64F, dx=1, dy=0, ksize=3)\n    train_sobel_y[i] = cv2.Sobel(train_img[i], cv2.CV_64F, dx=0, dy=1, ksize=3)","metadata":{"execution":{"iopub.status.busy":"2023-01-23T05:59:46.97353Z","iopub.execute_input":"2023-01-23T05:59:46.973932Z","iopub.status.idle":"2023-01-23T05:59:47.589178Z","shell.execute_reply.started":"2023-01-23T05:59:46.973899Z","shell.execute_reply":"2023-01-23T05:59:47.58796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_sobel_x = np.zeros_like(test_img)\ntest_sobel_y = np.zeros_like(test_img)\nfor i in range(len(test_img)):\n    test_sobel_x[i] = cv2.Sobel(test_img[i], cv2.CV_64F, dx=1, dy=0, ksize=3)\n    test_sobel_y[i] = cv2.Sobel(test_img[i], cv2.CV_64F, dx=0, dy=1, ksize=3)","metadata":{"execution":{"iopub.status.busy":"2023-01-23T05:59:47.591783Z","iopub.execute_input":"2023-01-23T05:59:47.592155Z","iopub.status.idle":"2023-01-23T05:59:48.004453Z","shell.execute_reply.started":"2023-01-23T05:59:47.592122Z","shell.execute_reply":"2023-01-23T05:59:48.003028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Вычисляем угол и длину вектора градиента","metadata":{}},{"cell_type":"code","source":"train_g, train_theta = cv2.cartToPolar(train_sobel_x, train_sobel_y)\ntest_g, test_theta = cv2.cartToPolar(test_sobel_x, test_sobel_y)","metadata":{"execution":{"iopub.status.busy":"2023-01-23T06:00:05.55242Z","iopub.execute_input":"2023-01-23T06:00:05.552921Z","iopub.status.idle":"2023-01-23T06:00:05.845049Z","shell.execute_reply.started":"2023-01-23T06:00:05.552893Z","shell.execute_reply":"2023-01-23T06:00:05.844142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(20, 10))\nfor i, img in enumerate(train_g[:5], 1):\n    subplot = fig.add_subplot(1, 7, i)\n    plt.imshow(img, cmap='gray');\n    subplot.set_title('%s' % train_label[i - 1]);\n    subplot = fig.add_subplot(3, 7, i)\n    plt.hist(train_theta[i - 1].flatten(),\n             bins=16, weights=train_g[i - 1].flatten())","metadata":{"execution":{"iopub.status.busy":"2023-01-23T06:00:10.519484Z","iopub.execute_input":"2023-01-23T06:00:10.519978Z","iopub.status.idle":"2023-01-23T06:00:11.706452Z","shell.execute_reply.started":"2023-01-23T06:00:10.519948Z","shell.execute_reply":"2023-01-23T06:00:11.704374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Вычисляем гистограммы градиентов","metadata":{}},{"cell_type":"code","source":"# Гистограммы вычисляются с учетом длины вектора градиента\ntrain_hist = np.zeros((len(train_img), 16))\nfor i in range(len(train_img)):\n    hist, borders = np.histogram(train_theta[i],\n                                 bins=16,\n                                 range=(0., 2. * np.pi),\n                                 weights=train_g[i])\n    train_hist[i] = hist","metadata":{"execution":{"iopub.status.busy":"2023-01-23T06:00:27.176969Z","iopub.execute_input":"2023-01-23T06:00:27.177381Z","iopub.status.idle":"2023-01-23T06:00:30.957315Z","shell.execute_reply.started":"2023-01-23T06:00:27.177347Z","shell.execute_reply":"2023-01-23T06:00:30.956024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_hist = np.zeros((len(test_img), 16))\nfor i in range(len(test_img)):\n    hist, borders = np.histogram(test_theta[i],\n                                 bins=16,\n                                 range=(0., 2. * np.pi),\n                                 weights=test_g[i])\n    test_hist[i] = hist","metadata":{"execution":{"iopub.status.busy":"2023-01-23T06:00:31.391234Z","iopub.execute_input":"2023-01-23T06:00:31.391669Z","iopub.status.idle":"2023-01-23T06:00:33.9207Z","shell.execute_reply.started":"2023-01-23T06:00:31.391633Z","shell.execute_reply":"2023-01-23T06:00:33.919749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# По умолчанию используется L2 норма\ntrain_hist = train_hist / np.linalg.norm(train_hist, axis=1)[:, None]\ntest_hist = test_hist / np.linalg.norm(test_hist, axis=1)[:, None]","metadata":{"execution":{"iopub.status.busy":"2023-01-23T06:01:12.772693Z","iopub.execute_input":"2023-01-23T06:01:12.773036Z","iopub.status.idle":"2023-01-23T06:01:12.786326Z","shell.execute_reply.started":"2023-01-23T06:01:12.773009Z","shell.execute_reply":"2023-01-23T06:01:12.784997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Разбиваем выборку на обучение и валидацию и строим различные модели классификации, выбираем лучший скор","metadata":{}},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(train_hist, train_label, test_size=0.2, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2023-01-23T06:01:27.049776Z","iopub.execute_input":"2023-01-23T06:01:27.05027Z","iopub.status.idle":"2023-01-23T06:01:27.066891Z","shell.execute_reply.started":"2023-01-23T06:01:27.050226Z","shell.execute_reply":"2023-01-23T06:01:27.06482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## RandomForestClassifier","metadata":{}},{"cell_type":"code","source":"rfc=RandomForestClassifier(random_state=42)\n\nparam_grid = { \n    'n_estimators': [5, 10, 20, 30],\n    'max_features': ['auto', 'sqrt', 'log2'],\n    'max_depth' : [4, 5, 6, 7, 8, 10, 12, 15],\n    'criterion' :['gini', 'entropy']\n}","metadata":{"execution":{"iopub.status.busy":"2023-01-23T06:02:10.019248Z","iopub.execute_input":"2023-01-23T06:02:10.019758Z","iopub.status.idle":"2023-01-23T06:02:10.028196Z","shell.execute_reply.started":"2023-01-23T06:02:10.019729Z","shell.execute_reply":"2023-01-23T06:02:10.026116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CV_rfc = GridSearchCV(estimator=rfc, param_grid=param_grid)\nCV_rfc.fit(X_train, y_train)\nCV_rfc.best_params_","metadata":{"execution":{"iopub.status.busy":"2023-01-23T06:02:11.410446Z","iopub.execute_input":"2023-01-23T06:02:11.410854Z","iopub.status.idle":"2023-01-23T06:33:04.645662Z","shell.execute_reply.started":"2023-01-23T06:02:11.410821Z","shell.execute_reply":"2023-01-23T06:33:04.64438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Проба увеличение количества estimators\nRFC = RandomForestClassifier(n_estimators=150, max_depth=15, criterion='gini', max_features='auto')\nRFC.fit(X_train, y_train)\nrfc_pred = RFC.predict(X_test)\nprint(accuracy_score(y_test, rfc_pred))\nprint(classification_report(y_test, rfc_pred))","metadata":{"execution":{"iopub.status.busy":"2023-01-23T06:35:35.107151Z","iopub.execute_input":"2023-01-23T06:35:35.107611Z","iopub.status.idle":"2023-01-23T06:35:57.705254Z","shell.execute_reply.started":"2023-01-23T06:35:35.107584Z","shell.execute_reply":"2023-01-23T06:35:57.703907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## BaggingClassifier","metadata":{}},{"cell_type":"code","source":"clf = BaggingClassifier(base_estimator=DecisionTreeClassifier(),\n                        n_estimators=150, max_samples=0.5, max_features=0.5, random_state=42).fit(X_train, y_train)\nclf_pred = clf.predict(X_test)\nprint(accuracy_score(y_test, clf_pred))\nprint(classification_report(y_test, clf_pred))","metadata":{"execution":{"iopub.status.busy":"2023-01-23T06:49:31.746924Z","iopub.execute_input":"2023-01-23T06:49:31.747311Z","iopub.status.idle":"2023-01-23T06:49:59.436306Z","shell.execute_reply.started":"2023-01-23T06:49:31.747271Z","shell.execute_reply":"2023-01-23T06:49:59.434894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## StackingClassifier","metadata":{}},{"cell_type":"code","source":"SC = StackingClassifier([\n    ('lr', LogisticRegression(solver='saga')),\n    ('rf', RandomForestClassifier(),\n    'lsc', LinearSVC())])\nSC.fit(X_train, y_train)\nsc_pred = SC.predict(X_test)\nprint(accuracy_score(y_test, sc_pred))\nprint(classification_report(y_test, sc_pred))","metadata":{"execution":{"iopub.status.busy":"2023-01-23T06:49:59.438184Z","iopub.execute_input":"2023-01-23T06:49:59.438881Z","iopub.status.idle":"2023-01-23T06:51:28.655588Z","shell.execute_reply.started":"2023-01-23T06:49:59.43885Z","shell.execute_reply":"2023-01-23T06:51:28.654757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## XGB","metadata":{}},{"cell_type":"code","source":"import xgboost as xgb\n\nxgb_clf = xgb.XGBClassifier()","metadata":{"execution":{"iopub.status.busy":"2023-01-23T07:06:08.343154Z","iopub.execute_input":"2023-01-23T07:06:08.343598Z","iopub.status.idle":"2023-01-23T07:06:08.349676Z","shell.execute_reply.started":"2023-01-23T07:06:08.343568Z","shell.execute_reply":"2023-01-23T07:06:08.348446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xgb_clf.fit(X_train, y_train)\nxgb_pred = xgb_clf.predict(X_test)\nscores = cross_val_score(xgb_clf, X_train, y_train, cv=5)\nprint(\"Mean cross-validation score: %.2f\" % scores.mean())\nprint(accuracy_score(y_test, xgb_pred))","metadata":{"execution":{"iopub.status.busy":"2023-01-23T07:06:09.492603Z","iopub.execute_input":"2023-01-23T07:06:09.493177Z","iopub.status.idle":"2023-01-23T07:09:37.747462Z","shell.execute_reply.started":"2023-01-23T07:06:09.493139Z","shell.execute_reply":"2023-01-23T07:09:37.746246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Визуализируем предсказания","metadata":{}},{"cell_type":"code","source":"fig = plt.figure(figsize=(20, 10))\nfor i, img in enumerate(test_img[0:5], 1):\n    subplot = fig.add_subplot(1, 7, i)\n    plt.imshow(img, cmap='gray');\n    subplot.set_title('%s' % xgb_pred[i - 1]);","metadata":{"execution":{"iopub.status.busy":"2023-01-23T07:10:30.864312Z","iopub.execute_input":"2023-01-23T07:10:30.866088Z","iopub.status.idle":"2023-01-23T07:10:31.334116Z","shell.execute_reply.started":"2023-01-23T07:10:30.865977Z","shell.execute_reply":"2023-01-23T07:10:31.333314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('submit.txt', 'w') as dst:\n    dst.write('ImageId,Label\\n')\n    for i, p in enumerate(sc_pred, 1):\n        dst.write('%s,%s\\n' % (i, p))","metadata":{"execution":{"iopub.status.busy":"2023-01-23T07:11:08.109378Z","iopub.execute_input":"2023-01-23T07:11:08.110024Z","iopub.status.idle":"2023-01-23T07:11:08.122532Z","shell.execute_reply.started":"2023-01-23T07:11:08.109986Z","shell.execute_reply":"2023-01-23T07:11:08.121191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}