{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Cassava Disease Classification using Label Propagation\n I tried Label Propagion in Cassava Disease Classification.\n\n Label Propagation: [Semi-Supervised Learning Using Gaussian Fields and Harmonic Functions](http://www.aaai.org/Papers/ICML/2003/ICML03-118.pdf)\n \n## Experiment\n1. Extracte image FEATURE by DenseNet\n2. Construct a kNN graph using train and extra data\n3. Add pseudo labels to the extra data using Label Propagation\n4. Learn using LightGBM and evaluate CV\n\nFirst, I tried learning using only labeled data(train).\nNext, I tried Label Propagation and compared the results.\n\n## Result\nLabeled lada only: CV-0.69059 , LB-0.69668\n\nLabel Propagation: CV-0.54667 , LB-0.53907\n\n\nHmmm. It did not go well.\n"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport glob\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm import tqdm\nfrom joblib import Parallel, delayed\nfrom PIL import Image\n\nfrom sklearn import svm, tree, neighbors, ensemble\nfrom sklearn.cluster import KMeans\nfrom sklearn import model_selection, feature_selection \nfrom sklearn.metrics import confusion_matrix, classification_report, roc_curve, roc_auc_score\nfrom sklearn.metrics import auc, accuracy_score, precision_recall_curve, f1_score\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold, train_test_split\nfrom sklearn.decomposition import PCA\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.semi_supervised import LabelPropagation\n\nimport scipy.stats\n\nimport lightgbm as lgb\nimport xgboost as xgb\nimport catboost as ctb\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Feature extraction by DenseNet"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"from keras.applications.densenet import preprocess_input, DenseNet121\nfrom keras.models import Model\nfrom keras.layers import GlobalAveragePooling2D, Input, Lambda, AveragePooling1D\nimport keras.backend as K\n\nIMG_SIZE = 256\n\nmodel_input = Input((IMG_SIZE,IMG_SIZE,3))\nbackbone = DenseNet121(input_tensor = model_input, \n                       weights=\"../input/densenet-keras/DenseNet-BC-121-32-no-top.h5\",\n                       include_top = False)\nx = backbone.output\nx = GlobalAveragePooling2D()(x)\nx = Lambda(lambda x: K.expand_dims(x,axis = -1))(x)\nx = AveragePooling1D(4)(x)\nmodel_output = Lambda(lambda x: x[:,:,0])(x)\n\nm = Model(model_input,model_output)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_image(path):\n    img = Image.open(path)\n    img = img.resize((IMG_SIZE, IMG_SIZE))\n    #preprocessing\n    return np.asarray(img)\n\nbatch_size = 32\nlabels = ['cgm', 'cmd', 'healthy', 'cbb', 'cbsd']\n\n#train\ntrain  = pd.DataFrame(columns=['target'] + ['DenseNet_%d'%(i) for i in range(256)], dtype='float32')\nfor lb in range(len(labels)):\n    image_files = glob.glob('../input/cassava-disease/train/train/' + labels[lb] + '/*.jpg')\n    n_batches = len(image_files) // batch_size + 1\n    \n    for i in tqdm(range(n_batches)):\n        batch = image_files[i*batch_size:(i+1)*batch_size]\n        batch_images = np.zeros((len(batch), IMG_SIZE, IMG_SIZE, 3))\n        for i,p in enumerate(batch):\n            batch_images[i] = load_image(p)\n        preds = m.predict(batch_images)\n        for i,p in enumerate(batch):\n            image_id = p.split('/')[-1]\n            train.loc[image_id, 'target'] = lb\n            train.loc[image_id, 1:] = preds[i]\n            \n#test\nsubmission = pd.read_csv('../input/cassava-disease/sample_submission_file.csv')\ntest = pd.DataFrame(index=submission['Id'], columns=['DenseNet_%d'%(i) for i in range(256)])\nimage_files = glob.glob('../input/cassava-disease/test/test/0/*.jpg')\nn_batches = len(image_files) // batch_size + 1\n\nfor i in tqdm(range(n_batches)):\n    batch = image_files[i*batch_size:(i+1)*batch_size]\n    batch_images = np.zeros((len(batch), IMG_SIZE, IMG_SIZE, 3))\n    for i,p in enumerate(batch):\n        batch_images[i] = load_image(p)\n    preds = m.predict(batch_images)\n    for i,p in enumerate(batch):\n        image_id = p.split('/')[-1]\n        test.loc[image_id, :] = preds[i]\n\n#extra\nextra = pd.DataFrame(columns=['DenseNet_%d'%(i) for i in range(256)])\nimage_files = glob.glob('../input/cassava-disease/extraimages/extraimages/*.jpg')\nn_batches = len(image_files) // batch_size + 1\n\nfor i in tqdm(range(n_batches)):\n    batch = image_files[i*batch_size:(i+1)*batch_size]\n    batch_images = np.zeros((len(batch), IMG_SIZE, IMG_SIZE, 3))\n    for i,p in enumerate(batch):\n        batch_images[i] = load_image(p)\n    preds = m.predict(batch_images)\n    for i,p in enumerate(batch):\n        image_id = p.split('/')[-1]\n        extra.loc[image_id, :] = preds[i]\n\nprint(\"train:\", train.shape)\nprint(\"test:\", test.shape)\nprint(\"extra:\", extra.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# When learing only with labeled data"},{"metadata":{"trusted":true},"cell_type":"code","source":"def run_lgb(params, X_train, X_test, extra=None, n_splits=5):\n    verbose_eval = 1000\n    num_rounds = 200000\n    early_stop = 1000\n    \n    kf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=1337)\n    \n    oof_train = np.zeros((X_train.shape[0]))\n    oof_test  = np.zeros((X_test.shape[0], n_splits))\n    \n    i = 0\n    for train_idx, valid_idx in kf.split(X_train, X_train['target']):\n        X_tr = X_train.iloc[train_idx].drop('target', axis=1)\n        X_val = X_train.iloc[valid_idx].drop('target', axis=1)\n        \n        y_tr = X_train.iloc[train_idx]['target']\n        y_val = X_train.iloc[valid_idx]['target']\n        \n        if extra is not None:\n            X_tr = pd.concat([X_tr, extra.drop('target', axis=1)], sort=False)\n            y_tr = pd.concat([y_tr, extra['target']], sort=False)\n        \n        d_train = lgb.Dataset(data=X_tr, label=y_tr)\n        d_valid = lgb.Dataset(data=X_val, label=y_val)\n        \n        watchlist = [d_train, d_valid]\n        model = lgb.train(params,\n                          train_set=d_train,\n                          num_boost_round=num_rounds,\n                          valid_sets=watchlist,\n                          early_stopping_rounds=early_stop, verbose_eval=verbose_eval)\n        valid_pred = model.predict(X_val.values, num_iteration=model.best_iteration)\n        test_pred = model.predict(X_test.values, num_iteration=model.best_iteration)\n        \n        oof_train[valid_idx] = np.argmax(valid_pred, axis=1)\n        oof_test[:,i] = np.argmax(test_pred, axis=1)\n        i += 1\n    return model, oof_train, oof_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lgb_params = {\n    \"objective\" : \"multiclass\",\n    \"num_class\" : len(labels),\n    \"metric\" : \"multi_logloss\",\n    \"boosting\": 'gbdt',\n    \"max_depth\" : -1,\n    \"num_leaves\" : 30,\n    \"learning_rate\" : 0.01,\n    \"bagging_freq\": 5,\n    \"bagging_fraction\" : 0.85,\n    \"feature_fraction\" : 0.4,\n#    \"min_data_in_leaf\": 20,\n#    \"min_sum_heassian_in_leaf\": 10,\n    \"tree_learner\": \"serial\",\n    \"boost_from_average\": \"false\",\n    \"bagging_seed\" : 10,\n    \"verbosity\" : 1,\n    'seed': 1337,\n#    'max_bin':100,\n    'lambda_l1': 2.622427756417558,\n    'lambda_l2': 2.624427931714477,\n}\n\nmodel_lgb, oof_train_lgb, oof_test_lgb = run_lgb(lgb_params, train, test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"CV Score:\", accuracy_score(train['target'], oof_train_lgb))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/cassava-disease/sample_submission_file.csv')\nsubmission['Category'] = scipy.stats.mode(oof_test_lgb, axis=1)[0].astype('int').flatten()\nsubmission['Category'] = submission['Category'].apply(lambda x: labels[x])\nsubmission.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission_lgb.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Label propagation by kneighors_graph"},{"metadata":{"trusted":true},"cell_type":"code","source":"tmp = pd.concat([train, extra], sort=False)\ntmp = tmp.fillna(-1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label_prop = LabelPropagation(kernel='knn', n_neighbors=2)\n#label_prop = LabelPropagation(n_neighbors=2)\nlabel_prop.fit(tmp.drop('target', axis=1), tmp['target'])\n\npreds = label_prop.predict(tmp.drop('target', axis=1))\npd.value_counts(preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"extra_ = extra.copy().astype('float32')\nextra_['target'] = preds[tmp['target']==-1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_ex, oof_train_ex, oof_test_ex = run_lgb(lgb_params, train, test, extra=extra_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"CV Score:\", accuracy_score(train['target'], oof_train_ex))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/cassava-disease/sample_submission_file.csv')\nsubmission['Category'] = scipy.stats.mode(oof_test_ex, axis=1)[0].astype('int').flatten()\nsubmission['Category'] = submission['Category'].apply(lambda x: labels[x])\nsubmission.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission_lp.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}