{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import sys\n!cp ../input/rapids/rapids.0.15.0 /opt/conda/envs/rapids.tar.gz\n!cd /opt/conda/envs/ && tar -xzvf rapids.tar.gz > /dev/null\nsys.path = [\"/opt/conda/envs/rapids/lib/python3.7/site-packages\"] + sys.path\nsys.path = [\"/opt/conda/envs/rapids/lib/python3.7\"] + sys.path\nsys.path = [\"/opt/conda/envs/rapids/lib\"] + sys.path\n!cp /opt/conda/envs/rapids/lib/libxgboost.so /opt/conda/lib/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install --use-feature=2020-resolver https://s3-us-west-2.amazonaws.com/xgboost-nightly-builds/xgboost-1.3.0_SNAPSHOT%2B00b0ad1293b4fa74d6aca5da4e9ab7a9d16777f0-py3-none-manylinux2010_x86_64.whl\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import xgboost as xgb\nxgb.__version__","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport cudf\nimport cupy as cp\nfrom cuml.neighbors import KNeighborsClassifier\nfrom cuml.linear_model import LogisticRegression\nfrom sklearn.model_selection import KFold, train_test_split\nfrom sklearn.metrics import roc_auc_score, log_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission = pd.read_csv('/kaggle/input/dogs-vs-cats-redux-kernels-edition/sample_submission.csv')\nsample_submission.tail()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_image_list = np.load('../input/cats-and-dogs-embedded-data/cats_and_dogs_1/train_image_list.npy')\ntest_image_list = np.load('../input/cats-and-dogs-embedded-data/cats_and_dogs_1/test_image_list.npy')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ids = [int(x.split('.')[0]) for x in test_image_list]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target = np.load('../input/cats-and-dogs-embedded-data/cats_and_dogs_1/target.npy')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_EB7_ns = np.load('../input/cats-and-dogs-embedded-data/cats_and_dogs_4/cats_and_dogs_4/train_EB7_ns.npy')\ntest_EB7_ns = np.load('../input/cats-and-dogs-embedded-data/cats_and_dogs_4/cats_and_dogs_4/test_EB7_ns.npy')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_oof = np.zeros(train_EB7_ns.shape[0])\ny_test = 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NUM_FOLDS = 20\nkf = KFold(n_splits=NUM_FOLDS, shuffle=True, random_state=137)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nfor f, (train_ind, val_ind) in enumerate(kf.split(train_EB7_ns, target)):\n    print(f)\n    X_train, X_val = train_EB7_ns[train_ind], train_EB7_ns[val_ind]\n    y_train, y_val = target[train_ind], target[val_ind]\n    lr = LogisticRegression(C=0.026, max_iter=10000)\n    lr.fit(X_train, y_train)\n    val_preds = lr.predict_proba(X_val)[:,1]\n    test_preds = lr.predict_proba(test_EB7_ns)[:,1]\n    y_oof[val_ind] = val_preds\n    y_test += test_preds/NUM_FOLDS","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"roc_auc_score(target, y_oof)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"log_loss(target, y_oof)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_oof_xgb_1 = np.zeros(train_EB7_ns.shape[0])\ny_test_xgb_1 = 0\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nxgb_params = {\n    'eta': 0.05,\n    'max_depth': 4,\n    'subsample': 0.8,\n    'colsample_bytree': 0.6,\n    'alpha': 0.01,\n    'lambda': 1.00,\n    'gamma' : 0.02,\n    'max_bin': 256,\n    'objective': 'reg:logistic',\n    'eval_metric': 'auc',\n    'verbosity': 0,\n    'tree_method': 'gpu_hist', \n    'predictor': 'gpu_predictor'\n}\n\ndtest = xgb.DMatrix(test_EB7_ns)\n\nfor f, (train_ind, val_ind) in enumerate(kf.split(train_EB7_ns, target)):\n    print(f)\n    X_train, X_val = train_EB7_ns[train_ind], train_EB7_ns[val_ind]\n    y_train, y_val = target[train_ind], target[val_ind]\n    \n    dtrain = xgb.DMatrix(X_train, y_train)\n    dval = xgb.DMatrix(X_val, y_val)\n    \n    clf = xgb.train(xgb_params, dtrain, num_boost_round=100)\n    val_preds = clf.predict(dval)\n    test_preds = clf.predict(dtest)\n    y_oof_xgb_1[val_ind] = val_preds\n    y_test_xgb_1 += test_preds/NUM_FOLDS\n    \nprint(roc_auc_score(target, y_oof_xgb_1))\nprint(log_loss(target, y_oof_xgb_1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ny_oof_knn = np.zeros(train_EB7_ns.shape[0])\ny_test_knn = 0\n\nkf = KFold(n_splits=NUM_FOLDS, shuffle=True, random_state=137)\n\nfor f, (train_ind, val_ind) in enumerate(kf.split(train_EB7_ns, target)):\n    print(f)\n    X_train, X_val = train_EB7_ns[train_ind], train_EB7_ns[val_ind]\n    y_train, y_val = target[train_ind], target[val_ind]\n    knc = KNeighborsClassifier(n_neighbors=400)\n    knc.fit(X_train, y_train)\n    val_preds = knc.predict_proba(X_val)[:,1]\n    test_preds = knc.predict_proba(test_EB7_ns)[:,1]\n    y_oof_knn[val_ind] = val_preds\n    y_test_knn += test_preds/NUM_FOLDS\n    \nprint(roc_auc_score(target, y_oof_knn))\nprint(log_loss(target, y_oof_knn))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(roc_auc_score(target, 0.9*y_oof+0.1*y_oof_knn))\nprint(log_loss(target, 0.9*y_oof+0.1*y_oof_knn))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission['id'] = ids\nsample_submission['label'] = y_test\nsample_submission.to_csv('submission_lr.csv', index=False)\nsample_submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission['id'] = ids\nsample_submission['label'] = y_test_xgb_1\nsample_submission.to_csv('submission_xgb_1.csv', index=False)\nsample_submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission['label'] = y_test_knn\nsample_submission.to_csv('submission_knn.csv', index=False)\nsample_submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission['label'] = 0.9*y_test + 0.1* y_test_knn\nsample_submission.to_csv('submission_blend_1.csv', index=False)\nsample_submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission['label'] = 0.95*y_test + 0.05* y_test_knn\nsample_submission.to_csv('submission_blend_2.csv', index=False)\nsample_submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission['label'] = 0.99*y_test + 0.01* y_test_knn\nsample_submission.to_csv('submission_blend_3.csv', index=False)\nsample_submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission['id'] = ids\nsample_submission['label'] = y_test**1.001\nsample_submission.to_csv('submission_lr_1.csv', index=False)\nsample_submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission['id'] = ids\nsample_submission['label'] = y_test**1.005\nsample_submission.to_csv('submission_lr_2.csv', index=False)\nsample_submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission['id'] = ids\nsample_submission['label'] = y_test**1.01\nsample_submission.to_csv('submission_lr_3.csv', index=False)\nsample_submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission['id'] = ids\nsample_submission['label'] = y_test**0.999\nsample_submission.to_csv('submission_lr_4.csv', index=False)\nsample_submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission['id'] = ids\nsample_submission['label'] = y_test**0.995\nsample_submission.to_csv('submission_lr_5.csv', index=False)\nsample_submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission['id'] = ids\nsample_submission['label'] = y_test**0.99\nsample_submission.to_csv('submission_lr_6.csv', index=False)\nsample_submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission['label'] = 0.5*y_test_xgb_1 + 0.5* y_test\nsample_submission.to_csv('submission_blend_4.csv', index=False)\nsample_submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission['label'] = (0.5*y_test_xgb_1 + 0.5* y_test)**0.99\nsample_submission.to_csv('submission_blend_5.csv', index=False)\nsample_submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission['label'] = (0.5*y_test_xgb_1 + 0.5* y_test)**1.01\nsample_submission.to_csv('submission_blend_6.csv', index=False)\nsample_submission.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}