{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport matplotlib\nimport matplotlib.pyplot as plt\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.svm import SVC\nfrom sklearn.metrics import precision_recall_fscore_support\nfrom sklearn.model_selection import GridSearchCV","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv')\ntest = pd.read_csv('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f7a528588eb5ff9dfdaf71e867e29fb69406d764"},"cell_type":"code","source":"train.columns.difference(test.columns)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"06607efd718ee53732e412667633d8df77ae1267"},"cell_type":"code","source":"target = 'label'\nlabels = train[target]\ntrain = train.drop(target, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"63665a3228e2146f964765155a60f91e0308376d"},"cell_type":"code","source":"train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"54623ef0796ad49e8225142b45429795c805dc32"},"cell_type":"code","source":"some_digit = train.values[50].reshape(28,28)\nplt.imshow(some_digit, cmap = matplotlib.cm.binary, interpolation=\"nearest\")\nplt.axis(\"off\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7cb16b3a756e962e72b12fd3e8871691c146766e"},"cell_type":"markdown","source":"Deskewing data before using an SVC is provably better"},{"metadata":{"trusted":true,"_uuid":"7276b99a6d10499a6fa779186229677961061479"},"cell_type":"code","source":"def std_scaled(data, scaler, is_train):\n    # Don't scale row Id\n    preds = data.select_dtypes(include=[np.int, np.float]).columns.tolist()\n    if is_train:\n        for pred in preds:\n            data[pred] = scaler.fit_transform(data[[pred]])\n    else:\n        for pred in preds:\n            data[pred] = scaler.transform(data[[pred]])            \n    return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6b3c7dc973ffe655816c124a75c9b43693159942"},"cell_type":"code","source":"%%capture\nscaler = StandardScaler()\ntrain_scaled = std_scaled(train, scaler, True)\ntest_scaled = std_scaled(test, scaler, False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f313d8373c5467bb3b01f24e39d63ff206e03ccc"},"cell_type":"code","source":"train_scaled[:1]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a890077ab38bd43682e719be23fc35c96ac71df3"},"cell_type":"markdown","source":"Both svm.SVC and linear_model.SGDClassifier use an SVC, but the latter doesn't expose slack variable sum C and basis function degree for tuning, nor does it use the kernel trick. It is much faster though.\n\nTime Complexities:\nSVC: O(M^2N) to O(M^3N)\nSGDClassifier: O(MN)"},{"metadata":{"trusted":true,"_uuid":"8991c4377c188f18d34e18098ce05d13748ab1af"},"cell_type":"code","source":"svc = SVC(C=1, kernel='poly', degree=9, coef0=1, decision_function_shape='ovo')\nsvc.fit(train_scaled, labels)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b5442ff32101686bdd89c558bba5cc4f5b2eb114"},"cell_type":"markdown","source":"See if the model does not totally underfit the training data."},{"metadata":{"trusted":true,"_uuid":"b83ec9a207f7c493fa8fdd1656b117cfba2b1246"},"cell_type":"code","source":"some_data = train_scaled[:10]\nsome_labels = labels[:10]\nprint(\"Prediction:\", list(svc.predict(some_data)))\nprint(\"Actual lab:\", list(some_labels))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b3d178a60c52f56e4313490cf5e881e54b4ab2fc"},"cell_type":"code","source":"def accuracy(preds, labs):\n    n = len(preds)\n    if n != len(labs):\n        raise ValueError\n    wrong = 0\n    for i in range(n):\n        if preds[i] != labs[i]:\n            wrong += 1\n    return (n-wrong) / n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6dfaa089401274fdca2ee2439598cd08e644af1e"},"cell_type":"code","source":"svc_preds = list(svc.predict(train_scaled))\nlabs = list(labels)\nprint(\"SVC accuracy on train set: \" + str(accuracy(svc_preds, labs)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"277aed092a06b0a4fa55a53b3ddc98945ba357e5"},"cell_type":"code","source":"# F1 is a harmonic mean of precision and recall, so it's higher if both are good\nF1 = precision_recall_fscore_support(labs, svc_preds)[2]\nF1","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"af7fc22d5f3a7d669cfacd03e51dd6fb97bf6248"},"cell_type":"markdown","source":"If the model looks promising tune it."},{"metadata":{"trusted":true,"_uuid":"e6a93b7cf20685f32bcf23845c30af56368f305b"},"cell_type":"code","source":"# GridSearchCV for slack variable sum (C) and basis function (degree)\n# param_grid = [{'C': [0.1, 1, 5]}]\n# grid_search = GridSearchCV(svc, param_grid, n_jobs=-1, cv=5)\n# grid_search.fit(train_scaled, labels)\n# grid_search.best_params_\n# C=1 on train is best","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"33294a211bde8d970dcd45331363e57846fb7407"},"cell_type":"code","source":"# Submit\npredictions = svc.predict(test).tolist()\ntest['ImageId'] = range(1, len(test)+1)\nsubmission = pd.DataFrame({'ImageId': test.ImageId, target: predictions})\nsubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"638aca0cc86c2c35cbacd5a8f9867a0caef9558f"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}