{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"scrolled":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\nimport pickle\n# There are more imports later, when needed","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Input processing\n------------------------------------\n\nFor this, I will be using the output of the EDA notebook, which can be found [here](https://www.kaggle.com/shubhayudas6/team-vsm-eda)"},{"metadata":{},"cell_type":"markdown","source":"### Load in the dataset and the mapping"},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"train = pd.read_csv('../input/Malware-detection-NS/train.csv', low_memory=False)\ntest = pd.read_csv('../input/Malware-detection-NS/test.csv', low_memory=False)\n\nwith open('../input/categorical-data-mapping/convertor.sav', 'rb') as fp:\n    CategoricalMapping = pickle.load(fp)\n\nOutputCol = 'HasDetections'\nUniqueCol = 'MachineIdentifier'\n\ntrain.shape, test.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Get the list of features to drop\n--------------------------------------------\n\nThis comes from the EDA notebook. I didn't flick them from nowhere!\n\nDrop these features from the test and the train datasets. This will be followed with processing the categorical features."},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"DropColList = [\n 'ProductName',\n 'PuaMode',\n 'Census_ProcessorClass',\n 'DefaultBrowsersIdentifier',\n 'Census_IsFlightingInternal',\n 'Census_InternalBatteryType',\n 'Census_ThresholdOptIn',\n 'Census_IsWIMBootEnabled',\n 'SmartScreen',\n 'IsBeta',\n 'Census_IsFlightsDisabled',\n 'SMode',\n 'Census_IsPortableOperatingSystem',\n 'AutoSampleOptIn',\n 'UacLuaenable',\n 'Census_IsVirtualDevice',\n 'HasTpm',\n 'AVProductsInstalled',\n 'AVProductsEnabled',\n 'OsSuite',\n 'Census_ProcessorManufacturerIdentifier',\n 'Census_OSArchitecture',\n 'Census_OSBuildNumber',\n 'CityIdentifier',\n 'Census_FirmwareVersionIdentifier',\n 'Census_OEMModelIdentifier',\n 'Census_ProcessorModelIdentifier',\n 'GeoNameIdentifier',\n 'Census_PowerPlatformRoleName',\n 'AppVersion',\n 'AvSigVersion',\n 'CountryIdentifier',\n 'LocaleEnglishNameIdentifier',\n 'OsVer',\n 'Census_MDC2FormFactor',\n 'Census_DeviceFamily',\n 'Census_OSSkuName',\n 'Census_FlightRing', \n 'Census_InternalPrimaryDisplayResolutionVertical',\n 'Census_InternalPrimaryDisplayResolutionHorizontal'\n]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Now let me deal with categorical data"},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"def DtypeFixer(data):\n    dataset = data.copy(deep=True)\n    \n    actuallyBinary = ['IsProtected',\n                      'Firewall',\n                      'Wdft_IsGamer',\n                      'Census_IsAlwaysOnAlwaysConnectedCapable',\n                      'IsSxsPassiveMode',\n                    'Census_HasOpticalDiskDrive',\n                    'Census_IsSecureBootEnabled',\n                    'Census_IsPenCapable',\n                    'AVProductsEnabled',\n                    'Census_ProcessorManufacturerIdentifier',\n                    'RtpStateBitfield',\n                    'OsSuite',\n                    ]\n\n    \n    # These go to 64 bit float, if they aren't\n    actuallyFloat = ['Census_ProcessorCoreCount',\n                     'Census_PrimaryDiskTotalCapacity',\n                     'Census_SystemVolumeTotalCapacity',\n                     'Census_TotalPhysicalRAM',\n                     'Census_InternalPrimaryDiagonalDisplaySizeInInches',\n                     'Census_InternalBatteryNumberOfCharges'\n                    ]\n\n\n    dataset[actuallyBinary] = dataset[actuallyBinary].fillna(method='bfill', axis=0)\n    dataset[actuallyBinary] = dataset[actuallyBinary].astype(np.int16)\n\n    dataset[actuallyFloat] = dataset[actuallyFloat].astype(float)\n\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"def CategoricalFixer(data):\n    dataset = data.copy(deep=True)\n    \n    for feature in CategoricalMapping:\n        if feature in dataset.columns.values:\n            dataset[feature] = dataset[feature].astype('category')\n\n            NAreplacement = CategoricalMapping[feature][\"na\"]\n            buckets = CategoricalMapping[feature][\"fixers\"]\n\n            if NAreplacement not in dataset[feature].unique():\n                dataset[feature] = dataset[feature].cat.add_categories(NAreplacement)\n\n            dataset[feature] = dataset[feature].fillna(NAreplacement)\n\n            for bucket in buckets:\n                name = list(bucket.keys())[0]\n                mapping = bucket[name]\n                dataset[feature] = dataset[feature].apply(lambda x: name if x in mapping else x)\n            \n            dataset[feature] = dataset[feature].astype('object')\n\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"def Preprocess(data, training=False):\n    dataset = data.copy(deep=True)\n    \n    if training:\n        dataset = dataset.drop(labels=[UniqueCol], axis=1)\n        dataset[OutputCol] = dataset[OutputCol].astype(np.int16)\n\n    dataset = DtypeFixer(dataset)\n    dataset = CategoricalFixer(dataset)\n    dataset = dataset.drop(labels=DropColList, axis=1)\n    \n    categorical_cols = dataset.select_dtypes(include='object').columns\n    for feature in categorical_cols:\n        dataset[feature] = dataset[feature].apply(str)\n    \n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"train = Preprocess(train, training=True)\ntest = Preprocess(test)\n\ntrain.info()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Dealing with missing data"},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"categorical_cols = train.select_dtypes(include='object').columns.values\nnumerical_cols = train.select_dtypes(exclude='object').columns.values\n\ntrain[categorical_cols] = train[categorical_cols].fillna('NOTA', axis=1)\ntest[categorical_cols] = test[categorical_cols].fillna('NOTA', axis=1)\n\ntrain[numerical_cols] = train[numerical_cols].fillna(method='bfill', axis=1)\n\nnumerical_cols = np.delete(numerical_cols, np.where(numerical_cols == OutputCol))\ntest[numerical_cols] = test[numerical_cols].fillna(method='bfill', axis=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preparation for training\n-----------------------------------------------\n\nIn this block, only the training data must be touched."},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"import scipy.sparse as scsparse\n\nimport seaborn as sns\nfrom matplotlib.pyplot import show\n\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Check condition of class imbalance"},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"print(train[OutputCol].value_counts())\nsns.countplot(y=OutputCol, data=train)\nshow()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### One hot encode all the categorical data"},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"y = train[OutputCol].values\ntrain = train.drop(labels=[OutputCol], axis=1)\n\ncategorical_cols = train.select_dtypes(include='object').columns\nnumerical_cols = train.select_dtypes(exclude='object').columns\n\nsparseNumeric = scsparse.csr_matrix(train[numerical_cols])\n\nOHE = OneHotEncoder(handle_unknown='ignore')\nencodedTrain = OHE.fit_transform(train[categorical_cols])\n\nX = scsparse.hstack([sparseNumeric, encodedTrain])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Fix class imbalance"},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"from imblearn.over_sampling import SMOTE, RandomOverSampler","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.15, random_state=6, stratify=y)\n\nX_train.shape, X_test.shape, y_train.shape, y_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"oversampler = RandomOverSampler(sampling_strategy=0.2, random_state=1)\n\nX_train, y_train = oversampler.fit_resample(X_train, y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"X.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model training"},{"metadata":{},"cell_type":"markdown","source":"Wanted to stick with models that I understood, but then everyone was topping the leaderboard. So ventured into trees. Ugh."},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\nfrom sklearn.naive_bayes import BernoulliNB\nfrom sklearn.svm import LinearSVC\n\nfrom lightgbm import LGBMClassifier\n\nfrom sklearn.metrics import roc_auc_score, plot_confusion_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"splitter = StratifiedKFold(n_splits=8, shuffle=True, random_state=8)\nsplitter.get_n_splits(X_train, y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"# Some other models to try\n\n# 0.5642757541490503 on test set, not submitted, maybe use for ensemble?\n# model = BernoulliNB(alpha=1e-6)   \n\n# 0.61, submitted. Our favourite, but not the best here\n# model = LinearSVC(C=0.4, class_weight=\"balanced\", penalty=\"l1\", dual=False, max_iter=10000, random_state=66, verbose=2)\n\n# Was irritated, so went to so many estimators and leaves. I have no clue how to tune this lol. Increase size continuously?\n# But we go as deep as we can. Unlike our knowledge from the AI511 course. Or am I just too lazy?\n\nmodel = None\nfor train_index, val_index in splitter.split(X_train, y_train):\n    X_train_split, X_val = X_train[train_index], X_train[val_index]\n    y_train_split, y_val = y_train[train_index], y_train[val_index]\n    \n    model = LGBMClassifier(\n        num_leaves=64,\n        n_estimators=1000,\n        learning_rate=0.07,\n        objective='binary',\n        class_weight={0: 1, 1:3},\n        subsample_for_bin=20000,\n        min_split_gain=5e-4,\n        subsample=0.8,\n        reg_alpha=0.25,\n        reg_lambda=0.1,\n        random_state=4,\n        n_jobs=-1,\n        silent=False\n    )\n\n    model.fit(X_train_split, y_train_split, eval_set=[(X_val, y_val)], eval_metric='auc', early_stopping_rounds=50)\n\nmodel","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Evaluation"},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"predictions = model.predict_proba(X_test)[:, 1]\nroc_auc_score(y_test, predictions)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Get predictions on test data"},{"metadata":{"trusted":true},"cell_type":"code","source":"identifiers = test[UniqueCol]\ntest = test.drop(labels=[UniqueCol], axis=1)\nfor col in test.columns:\n    test[col] = test[col].astype(train[col].dtypes.name)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sparseNumericTest = scsparse.csr_matrix(test[numerical_cols])\nencodedTest = OHE.transform(test[categorical_cols])\n\ntestX = scsparse.csr_matrix(scsparse.hstack([sparseNumericTest, encodedTest]))\ntestX","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"testPredictions = model.predict_proba(testX)[:, -1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"output = pd.DataFrame([identifiers, testPredictions], [UniqueCol, OutputCol]).transpose()\noutput.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}