{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport plotly.express as px\nimport plotly.graph_objs as go\n\nimport plotly\nplotly.offline.init_notebook_mode(connected=True)\n\n#Ignore warnings\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-08-29T22:19:06.543758Z","iopub.execute_input":"2022-08-29T22:19:06.544357Z","iopub.status.idle":"2022-08-29T22:19:06.559357Z","shell.execute_reply.started":"2022-08-29T22:19:06.544198Z","shell.execute_reply":"2022-08-29T22:19:06.558030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"![](https://upload.wikimedia.org/wikipedia/commons/thumb/0/01/Avtoteka_logo.jpg/440px-Avtoteka_logo.jpg)https://www.wikiwand.com/en/Avito.ru","metadata":{}},{"cell_type":"markdown","source":"#AVITO.ru\n\n\"Avito is a Russian classified advertisements website with sections devoted to general goods for sale, jobs, real estate, personals, cars for sale, and services. Avito is the most popular classifieds site in Russia and is the biggest classifieds site in the world.\"\n\nhttps://en.wikipedia.org/wiki/Avito.ru\n\nThe Hunt for Prohibited Content\n\nPredict which ads contain illicit content\n\n\"AVITO.ru is one of the top-5 Russian websites. Since its inception in 2007, millions of Russians have used the service to buy and sell goods and services, such as home electronics, clothing, cars and real estate. In addition, thousands of Russian businesses use AVITO.ru as a sales channel for new and old goods as well as services.\"\n\n\"One of our biggest challenges is to cope with our volume and growth while continuing to provide the same exceptional customer experience. To address that challenge, we have one of the best data science teams analyzing tons of raw data to make our content cleaner, more relevant and selling faster.\"\n\nhttps://www.kaggle.com/competitions/avito-prohibited-content","metadata":{}},{"cell_type":"code","source":"from plotly.subplots import make_subplots\nfrom plotly import graph_objects as go\n\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.model_selection import train_test_split\n\nfrom sklearn.metrics import f1_score\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.metrics import classification_report\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import precision_score\nfrom sklearn.metrics import recall_score\n\nfrom xgboost import XGBClassifier , XGBRegressor\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.neural_network import MLPClassifier\n# from sklearn.neighbors import KNeighborsClassifier\n# from sklearn.naive_bayes import GaussianNB\n\nfrom sklearn.experimental import enable_iterative_imputer\nfrom sklearn.impute import IterativeImputer\nfrom xgboost import XGBClassifier , XGBRegressor\n\nimport shap","metadata":{"execution":{"iopub.status.busy":"2022-08-29T22:19:21.072443Z","iopub.execute_input":"2022-08-29T22:19:21.072884Z","iopub.status.idle":"2022-08-29T22:19:25.855931Z","shell.execute_reply.started":"2022-08-29T22:19:21.072850Z","shell.execute_reply":"2022-08-29T22:19:25.854532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#All script by Shirsh Mall https://www.kaggle.com/code/shirshmall/search-for-higgs-boson-decay-modes","metadata":{}},{"cell_type":"code","source":"!unzip -n ../input/avito-prohibited-content/avito_train.zip","metadata":{"execution":{"iopub.status.busy":"2022-08-29T22:19:29.582891Z","iopub.execute_input":"2022-08-29T22:19:29.583748Z","iopub.status.idle":"2022-08-29T22:20:09.676271Z","shell.execute_reply.started":"2022-08-29T22:19:29.583703Z","shell.execute_reply":"2022-08-29T22:20:09.674884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!unzip -n ../input/avito-prohibited-content/avito_test.zip","metadata":{"execution":{"iopub.status.busy":"2022-08-29T22:20:23.669733Z","iopub.execute_input":"2022-08-29T22:20:23.670250Z","iopub.status.idle":"2022-08-29T22:20:35.771048Z","shell.execute_reply.started":"2022-08-29T22:20:23.670203Z","shell.execute_reply":"2022-08-29T22:20:35.769373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Eight years ago it was tsv\n\nsampleSubmission.tsv- a sample submission file in the correct format produced by our introductory code.\n\nhttps://www.kaggle.com/competitions/avito-prohibited-content/data","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv('../input/avito-prohibited-content/sample_submission.csv')\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-29T22:20:45.593838Z","iopub.execute_input":"2022-08-29T22:20:45.594298Z","iopub.status.idle":"2022-08-29T22:20:46.039100Z","shell.execute_reply.started":"2022-08-29T22:20:45.594257Z","shell.execute_reply":"2022-08-29T22:20:46.037739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import zipfile\ntraining= zipfile.ZipFile('../input/avito-prohibited-content/avito_train.zip')\ntraining.extractall()\ntest= zipfile.ZipFile('../input/avito-prohibited-content/avito_test.zip')\ntest.extractall()","metadata":{"execution":{"iopub.status.busy":"2022-08-29T22:20:53.119828Z","iopub.execute_input":"2022-08-29T22:20:53.120266Z","iopub.status.idle":"2022-08-29T22:21:35.880456Z","shell.execute_reply.started":"2022-08-29T22:20:53.120228Z","shell.execute_reply":"2022-08-29T22:21:35.878767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#When I ran the 1st unzip it returned tsv file. On the snippet below the table started with itemid.\n\nSo I changed EventID to itemid.","metadata":{}},{"cell_type":"code","source":"#Code by Shirsh Mall https://www.kaggle.com/code/shirshmall/search-for-higgs-boson-decay-modes\n\nData = pd.read_csv( './avito_train.tsv', sep='\\t', error_bad_lines=False )\nData_itemid = Data[['itemid']]#Original Data_Event_ID = Data[['EventId']]\n#Data_Label = Data[['Label']]\n\nData_Test = pd.read_csv( './avito_test.tsv', sep='\\t', error_bad_lines=False )\nData_Test_itemid = Data_Test[['itemid']]\n\nData.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-29T22:21:45.541331Z","iopub.execute_input":"2022-08-29T22:21:45.541869Z","iopub.status.idle":"2022-08-29T22:23:13.660751Z","shell.execute_reply.started":"2022-08-29T22:21:45.541828Z","shell.execute_reply":"2022-08-29T22:23:13.659557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#word cloud\nfrom wordcloud import WordCloud, ImageColorGenerator\ntext = \" \".join(str(each) for each in Data.category)\n# Create and generate a word cloud image:\nwordcloud = WordCloud(max_words=200,colormap='Set3', background_color=\"black\").generate(text)\nplt.figure(figsize=(10,6))\nplt.figure(figsize=(15,10))\n# Display the generated image:\nplt.imshow(wordcloud, interpolation='Bilinear')\nplt.axis(\"off\")\nplt.figure(1,figsize=(12, 12))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-29T22:23:20.329121Z","iopub.execute_input":"2022-08-29T22:23:20.329940Z","iopub.status.idle":"2022-08-29T22:23:46.520354Z","shell.execute_reply.started":"2022-08-29T22:23:20.329879Z","shell.execute_reply":"2022-08-29T22:23:46.519270Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code by Shirsh Mall https://www.kaggle.com/code/shirshmall/search-for-higgs-boson-decay-modes\n\nData.drop(columns=['itemid'], inplace=True)\nData_Test.drop(columns=['itemid'], inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-29T22:24:02.551236Z","iopub.execute_input":"2022-08-29T22:24:02.551671Z","iopub.status.idle":"2022-08-29T22:24:03.492846Z","shell.execute_reply.started":"2022-08-29T22:24:02.551612Z","shell.execute_reply":"2022-08-29T22:24:03.491530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Replacing Invalid Values is NaN for futher analysis","metadata":{}},{"cell_type":"code","source":"#Code by Shirsh Mall https://www.kaggle.com/code/shirshmall/search-for-higgs-boson-decay-modes\n\nData.replace(to_replace=-999.000, value=np.nan, inplace=True)\nData_Test.replace(to_replace=-999.000, value=np.nan, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-29T22:24:08.740805Z","iopub.execute_input":"2022-08-29T22:24:08.742041Z","iopub.status.idle":"2022-08-29T22:24:10.052055Z","shell.execute_reply.started":"2022-08-29T22:24:08.741975Z","shell.execute_reply":"2022-08-29T22:24:10.050872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Missing Values","metadata":{}},{"cell_type":"code","source":"#Code by Shirsh Mall https://www.kaggle.com/code/shirshmall/search-for-higgs-boson-decay-modes\n\ndef missing( df ):\n    high_missing_values_features = []\n    missing_values_features_less_5 = []\n    missing_values_features_btw_5_and_40 = []\n    missing_values_features = []\n    for column in df.columns.values:\n        print( \"Feature:- \", column )\n        percent_missing = df[column].isnull().sum()/df[column].shape[0]*100\n        print('Percentage of Missing Values:- ', percent_missing )\n    \n        if percent_missing>=25:\n            high_missing_values_features.append( column )\n        elif percent_missing<=40:\n            missing_values_features_less_5.append( column )\n        elif percent_missing>5 & percent_missing<=40:\n            missing_values_features_btw_5_and_25.append( column )\n        print(\"--------------------------------------------------------------\")\n        print(\"\\n\")\n    return missing_values_features_less_5, missing_values_features_btw_5_and_40,  high_missing_values_features","metadata":{"execution":{"iopub.status.busy":"2022-08-29T22:24:24.534063Z","iopub.execute_input":"2022-08-29T22:24:24.534512Z","iopub.status.idle":"2022-08-29T22:24:24.543339Z","shell.execute_reply.started":"2022-08-29T22:24:24.534455Z","shell.execute_reply":"2022-08-29T22:24:24.541881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Data.isna().sum()","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-08-29T22:24:32.975997Z","iopub.execute_input":"2022-08-29T22:24:32.976396Z","iopub.status.idle":"2022-08-29T22:24:34.542767Z","shell.execute_reply.started":"2022-08-29T22:24:32.976364Z","shell.execute_reply":"2022-08-29T22:24:34.541572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_values_features_less_5, missing_values_features_btw_5_and_40,  high_missing_values_features = missing( Data )","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-08-29T22:24:44.733537Z","iopub.execute_input":"2022-08-29T22:24:44.733952Z","iopub.status.idle":"2022-08-29T22:24:46.289884Z","shell.execute_reply.started":"2022-08-29T22:24:44.733920Z","shell.execute_reply":"2022-08-29T22:24:46.288313Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code by Shirsh Mall https://www.kaggle.com/code/shirshmall/search-for-higgs-boson-decay-modes\n\nData.drop(columns=high_missing_values_features,inplace=True)\nData_Test.replace(to_replace=-999.000, value=np.nan, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-29T22:25:09.965949Z","iopub.execute_input":"2022-08-29T22:25:09.966432Z","iopub.status.idle":"2022-08-29T22:25:11.132948Z","shell.execute_reply.started":"2022-08-29T22:25:09.966393Z","shell.execute_reply":"2022-08-29T22:25:11.132003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Correlation Matrix\npx.imshow( Data.corr(),color_continuous_scale='RdBu_r', width=800, height=600 )#original 1200/1000","metadata":{"_kg_hide-output":false,"execution":{"iopub.status.busy":"2022-08-29T22:25:16.453251Z","iopub.execute_input":"2022-08-29T22:25:16.453676Z","iopub.status.idle":"2022-08-29T22:25:18.501566Z","shell.execute_reply.started":"2022-08-29T22:25:16.453642Z","shell.execute_reply":"2022-08-29T22:25:18.500389Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Target: is_blocked\n\nis_blocked – Boolean target variable. This is the column to predict.\n\nhttps://www.kaggle.com/competitions/avito-prohibited-content/data","metadata":{}},{"cell_type":"markdown","source":"#Checking Target Feature","metadata":{}},{"cell_type":"code","source":"Data['is_blocked'].unique()","metadata":{"execution":{"iopub.status.busy":"2022-08-29T22:26:06.838746Z","iopub.execute_input":"2022-08-29T22:26:06.839176Z","iopub.status.idle":"2022-08-29T22:26:06.875128Z","shell.execute_reply.started":"2022-08-29T22:26:06.839142Z","shell.execute_reply":"2022-08-29T22:26:06.873690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Data['is_blocked'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-29T22:26:15.276944Z","iopub.execute_input":"2022-08-29T22:26:15.278048Z","iopub.status.idle":"2022-08-29T22:26:15.324010Z","shell.execute_reply.started":"2022-08-29T22:26:15.277993Z","shell.execute_reply":"2022-08-29T22:26:15.322979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code by Shirsh Mall https://www.kaggle.com/code/shirshmall/search-for-higgs-boson-decay-modes\n\ncounts_signal = Data['is_blocked'].value_counts()[1]\ncounts_background = Data['is_blocked'].value_counts()[0]\nplt.pie( [counts_signal, counts_background], labels=['one ','zero'] )\nprint( Data['is_blocked'].value_counts() )\nprint('-----------------------------')","metadata":{"execution":{"iopub.status.busy":"2022-08-29T22:26:20.981316Z","iopub.execute_input":"2022-08-29T22:26:20.981738Z","iopub.status.idle":"2022-08-29T22:26:21.230092Z","shell.execute_reply.started":"2022-08-29T22:26:20.981705Z","shell.execute_reply":"2022-08-29T22:26:21.228308Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#It's already zero/one","metadata":{}},{"cell_type":"code","source":"#Save for next time. It's already zero/one\n\nData['is_blocked'] = Data['is_blocked'].replace(to_replace='s', value=1)\nData['is_blocked'] = Data['is_blocked'].replace(to_replace='b', value=0)","metadata":{"execution":{"iopub.status.busy":"2022-08-29T22:26:38.108326Z","iopub.execute_input":"2022-08-29T22:26:38.109123Z","iopub.status.idle":"2022-08-29T22:26:38.192011Z","shell.execute_reply.started":"2022-08-29T22:26:38.109073Z","shell.execute_reply":"2022-08-29T22:26:38.190833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Encoding","metadata":{}},{"cell_type":"code","source":"#Code by Bizen https://www.kaggle.com/hiro5299834/tps-apr-2021-deebtables/notebook\n\nTARGET = 'is_blocked' #Target could Not be float otherwise will result in valueError: Unknown label type: 'continuous'. Even after the encoding.\n\nlabel_cols = ['category', 'subcategory', 'title', 'description', 'attrs']\nnumerical_cols = ['price', 'phones_cnt', 'emails_cnt', 'urls_cnt', 'close_hours']","metadata":{"execution":{"iopub.status.busy":"2022-08-29T22:26:45.340644Z","iopub.execute_input":"2022-08-29T22:26:45.341749Z","iopub.status.idle":"2022-08-29T22:26:45.347372Z","shell.execute_reply.started":"2022-08-29T22:26:45.341710Z","shell.execute_reply":"2022-08-29T22:26:45.346192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code by Bizen https://www.kaggle.com/hiro5299834/tps-apr-2021-deebtables/notebook That's 404 page\n#https://www.kaggle.com/code/hiro5299834/tps-apr-2021-pseudo-labeling-voting-ensemble\n\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler\nfrom sklearn.preprocessing import LabelEncoder\n\ndef label_encoder(c):\n    le = LabelEncoder()\n    return le.fit_transform(c)\n\nscaler = StandardScaler()\n\nlabel_encoded_df = Data[label_cols].apply(label_encoder)\nnumerical_df= pd.DataFrame(scaler.fit_transform(Data[numerical_cols]), columns=numerical_cols)\n#numerical_df = Data[numerical_cols]\ntarget_df = Data[TARGET]\n\nData = pd.concat([label_encoded_df, numerical_df, target_df], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-29T22:26:52.629102Z","iopub.execute_input":"2022-08-29T22:26:52.630221Z","iopub.status.idle":"2022-08-29T22:28:12.411348Z","shell.execute_reply.started":"2022-08-29T22:26:52.630178Z","shell.execute_reply":"2022-08-29T22:28:12.409822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Data.dtypes","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-08-29T22:28:15.680110Z","iopub.execute_input":"2022-08-29T22:28:15.680935Z","iopub.status.idle":"2022-08-29T22:28:15.693319Z","shell.execute_reply.started":"2022-08-29T22:28:15.680876Z","shell.execute_reply":"2022-08-29T22:28:15.691416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split( Data.drop(['is_blocked'], axis=1), Data[['is_blocked']], test_size=0.25, random_state=10, stratify=Data[['is_blocked']])","metadata":{"execution":{"iopub.status.busy":"2022-08-29T22:28:26.260171Z","iopub.execute_input":"2022-08-29T22:28:26.260675Z","iopub.status.idle":"2022-08-29T22:28:46.505432Z","shell.execute_reply.started":"2022-08-29T22:28:26.260634Z","shell.execute_reply":"2022-08-29T22:28:46.504154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Train Test Split\n\nAnd Imputing High Missing Values","metadata":{}},{"cell_type":"code","source":"#Code by Shirsh Mall https://www.kaggle.com/code/shirshmall/search-for-higgs-boson-decay-modes\n\nxgb_imputer = IterativeImputer( estimator=XGBRegressor(), max_iter=20, random_state=10, skip_complete=True ) \nxgb_imputer.fit( X_train )\nX_train = pd.DataFrame(xgb_imputer.transform( X_train ), columns=X_train.columns.values)\nX_test = pd.DataFrame(xgb_imputer.transform( X_test ), columns=X_train.columns.values)\n\n# Data_Test = pd.DataFrame(xgb_imputer.transform( Data_Test ), columns=X_train.columns.values)","metadata":{"execution":{"iopub.status.busy":"2022-08-29T22:28:49.732796Z","iopub.execute_input":"2022-08-29T22:28:49.733198Z","iopub.status.idle":"2022-08-29T22:28:54.616329Z","shell.execute_reply.started":"2022-08-29T22:28:49.733167Z","shell.execute_reply":"2022-08-29T22:28:54.615056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code by Shirsh Mall https://www.kaggle.com/code/shirshmall/search-for-higgs-boson-decay-modes\n\nX_train.to_csv('preprocessed_X_train.csv')\nX_test.to_csv('preprocessed_X_test.csv')\ny_train.to_csv('preprocessed_X_train.csv')\ny_test.to_csv('preprocessed_X_test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-08-29T22:28:58.693867Z","iopub.execute_input":"2022-08-29T22:28:58.694287Z","iopub.status.idle":"2022-08-29T22:29:49.059252Z","shell.execute_reply.started":"2022-08-29T22:28:58.694253Z","shell.execute_reply":"2022-08-29T22:29:49.058066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Applying ANN Model","metadata":{}},{"cell_type":"code","source":"#Code by Shirsh Mall https://www.kaggle.com/code/shirshmall/search-for-higgs-boson-decay-modes\n\nimport tensorflow as tf\nfrom keras import Sequential\nfrom keras.layers import Dense\nfrom keras.layers import BatchNormalization\nfrom keras.layers import Dropout\nfrom keras.layers import InputLayer\nfrom keras.callbacks import EarlyStopping\nimport keras_tuner as kt","metadata":{"execution":{"iopub.status.busy":"2022-08-29T22:29:52.347795Z","iopub.execute_input":"2022-08-29T22:29:52.348744Z","iopub.status.idle":"2022-08-29T22:30:00.173655Z","shell.execute_reply.started":"2022-08-29T22:29:52.348686Z","shell.execute_reply":"2022-08-29T22:30:00.172016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code by Shirsh Mall https://www.kaggle.com/code/shirshmall/search-for-higgs-boson-decay-modes\n\n# Plotting Loss vs epochs graph, Accuaracy vs epochs graph\ndef Plot_Loss_Accuracy( history, epochs ):\n    loss = history.history['loss']\n    val_loss = history.history['val_loss']\n\n    epochs = np.linspace(1,epochs,epochs)\n    epochs\n\n    fig = go.Figure(data=go.Scatter( x=epochs, y=loss, name='loss' ))\n    fig.add_trace( go.Scatter( x=epochs, y=val_loss, name='val_loss' ) )\n    fig.update_layout( title=\"Loss\", xaxis_title='epochs' )\n    fig.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-29T22:30:02.655941Z","iopub.execute_input":"2022-08-29T22:30:02.656629Z","iopub.status.idle":"2022-08-29T22:30:02.669263Z","shell.execute_reply.started":"2022-08-29T22:30:02.656565Z","shell.execute_reply":"2022-08-29T22:30:02.667780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code by Shirsh Mall https://www.kaggle.com/code/shirshmall/search-for-higgs-boson-decay-modes\n\nauc_roc_metric = tf.keras.metrics.AUC(curve='ROC')\nprecision_metric = tf.keras.metrics.Precision()\n\n# Neural network\nmodel = Sequential()\nmodel.add(Dense(100, input_dim=X_train.shape[1], activation='relu'))\nmodel.add(Dense(50, activation='relu'))\nmodel.add(Dense(1, activation='sigmoid'))\nmodel.compile(loss='binary_crossentropy' , optimizer='adam', metrics=[precision_metric,auc_roc_metric])","metadata":{"execution":{"iopub.status.busy":"2022-08-29T22:30:07.894213Z","iopub.execute_input":"2022-08-29T22:30:07.894679Z","iopub.status.idle":"2022-08-29T22:30:08.064068Z","shell.execute_reply.started":"2022-08-29T22:30:07.894643Z","shell.execute_reply":"2022-08-29T22:30:08.063027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Epochs. Here we go Started 22:19 ended 23:09","metadata":{}},{"cell_type":"code","source":"history = model.fit(X_train, y_train , validation_data=(X_test,y_test), epochs=10, batch_size=32)","metadata":{"execution":{"iopub.status.busy":"2022-08-29T22:30:13.954689Z","iopub.execute_input":"2022-08-29T22:30:13.959592Z","iopub.status.idle":"2022-08-29T23:33:40.397714Z","shell.execute_reply.started":"2022-08-29T22:30:13.959447Z","shell.execute_reply":"2022-08-29T23:33:40.395762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Plot_Loss_Accuracy( history, epochs=10 )","metadata":{"execution":{"iopub.status.busy":"2022-08-29T23:33:49.290146Z","iopub.execute_input":"2022-08-29T23:33:49.290648Z","iopub.status.idle":"2022-08-29T23:33:49.321300Z","shell.execute_reply.started":"2022-08-29T23:33:49.290597Z","shell.execute_reply":"2022-08-29T23:33:49.319903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Keras Tuner ANN Model","metadata":{}},{"cell_type":"code","source":"#Code by Shirsh Mall https://www.kaggle.com/code/shirshmall/search-for-higgs-boson-decay-modes\n\ndef hypertuning_model(hp):\n    \n    model = Sequential()\n    model.add( InputLayer( input_shape=X_train.shape[1] ))\n              \n    number_of_layers = hp.Int('No_of_Layers',min_value=2,max_value=6)\n    for num_layer in range(1, number_of_layers+1 ):\n        model.add( Dense( hp.Int('layer'+str(num_layer), min_value=10, max_value=510, step=100), activation='relu'))\n        # model.add( Dropout( hp.Choice('rate',values=[0.05,0.1,0.2,0.3] )))\n        model.add( BatchNormalization() )\n        \n    model.add(Dense(1, activation='sigmoid'))\n    model.compile(loss='binary_crossentropy', metrics=['accuracy', auc_roc_metric ],\n                  optimizer=tf.keras.optimizers.Adam(learning_rate=hp.Choice('learning_rate', values=[1e-1,1e-2, 1e-3, 1e-4])))\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2022-08-29T23:33:58.945831Z","iopub.execute_input":"2022-08-29T23:33:58.946768Z","iopub.status.idle":"2022-08-29T23:33:58.957008Z","shell.execute_reply.started":"2022-08-29T23:33:58.946721Z","shell.execute_reply":"2022-08-29T23:33:58.955428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hyper_tuner_model = kt.Hyperband( hypertuning_model, objective='val_loss',overwrite=True, max_epochs=10,seed=10 )","metadata":{"execution":{"iopub.status.busy":"2022-08-29T23:34:05.281523Z","iopub.execute_input":"2022-08-29T23:34:05.281960Z","iopub.status.idle":"2022-08-29T23:34:05.384001Z","shell.execute_reply.started":"2022-08-29T23:34:05.281925Z","shell.execute_reply":"2022-08-29T23:34:05.382506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Epochs started 23:10. At 24:24 we're still running trial 4!","metadata":{}},{"cell_type":"code","source":"#Code by Shirsh Mall https://www.kaggle.com/code/shirshmall/search-for-higgs-boson-decay-modes\n\ncall_back_ES = EarlyStopping(monitor='val_loss',patience=3)\ncall_back_MCP = tf.keras.callbacks.ModelCheckpoint(filepath=\"./\", monitor='val_loss', save_best_only=False, save_weights_only=False, mode='auto', save_freq='epoch')\nhyper_tuner_model.search( X_train,y_train, validation_data=(X_test,y_test), epochs=10, callbacks=[call_back_ES,call_back_MCP] )","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code by Shirsh Mall https://www.kaggle.com/code/shirshmall/search-for-higgs-boson-decay-modes\n\nbest_hyperparameters = hyper_tuner_model.get_best_hyperparameters(num_trials=1)[0]\nbest_hyperparameters.values","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code by Shirsh Mall https://www.kaggle.com/code/shirshmall/search-for-higgs-boson-decay-modes\n\nn_best_models = hyper_tuner_model.get_best_models(num_models=1)[0]\nn_best_models.summary()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code by Shirsh Mall https://www.kaggle.com/code/shirshmall/search-for-higgs-boson-decay-modes\n\nfinal_model = hyper_tuner_model.hypermodel.build( best_hyperparameters )\nhistory = final_model.fit(X_train,y_train, validation_data=(X_test,y_test),epochs=10 )","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Plot_Loss_Accuracy( history, epochs=10 )","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Neural Network using Pytorch","metadata":{}},{"cell_type":"code","source":"#Code by Shirsh Mall https://www.kaggle.com/code/shirshmall/search-for-higgs-boson-decay-modes\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Basic Parameters","metadata":{}},{"cell_type":"code","source":"#Code by Shirsh Mall https://www.kaggle.com/code/shirshmall/search-for-higgs-boson-decay-modes\n\nEPOCHS = 20\nBATCH_SIZE = 64\nLEARNING_RATE = 0.001","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code by Shirsh Mall https://www.kaggle.com/code/shirshmall/search-for-higgs-boson-decay-modes\n\nclass Train(Dataset):\n    \n    def __init__(self, X_data, y_data):\n        self.X_data = X_data\n        self.y_data = y_data\n        \n    def __getitem__(self, index):\n        return self.X_data[index], self.y_data[index]\n        \n    def __len__ (self):\n        return len(self.X_data)\n\n\n## test data    \nclass Test(Dataset):\n    \n    def __init__(self, X_data):\n        self.X_data = X_data\n        \n    def __getitem__(self, index):\n        return self.X_data[index]\n        \n    def __len__ (self):\n        return len(self.X_data)\n    \n########################\n\ntrain_data = Train(torch.FloatTensor(X_train.values), torch.FloatTensor(y_train.values))\ntest_data = Test(torch.FloatTensor(X_test.values))\n\n########################\n\ntrain_loader = DataLoader(dataset=train_data, batch_size=BATCH_SIZE, shuffle=True)\ntest_loader = DataLoader(dataset=test_data, batch_size=1)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Neural Network","metadata":{}},{"cell_type":"code","source":"#Code by Shirsh Mall https://www.kaggle.com/code/shirshmall/search-for-higgs-boson-decay-modes\n\nclass NN_Model(nn.Module):\n    def __init__(self):\n        super(NN_Model, self).__init__()\n        # Number of input features is 129.\n        self.layer_1 = nn.Linear(X_train.shape[1], 200) \n        self.layer_2 = nn.Linear(200, 100)\n        self.layer_out = nn.Linear(100, 1) \n        \n        self.relu = nn.ReLU()\n        self.dropout = nn.Dropout(p=0.1)\n       \n        \n    def forward(self, inputs):\n        x = self.relu(self.layer_1(inputs))\n        x = self.relu(self.layer_2(x))\n        x = self.dropout(x)\n        x = self.layer_out(x)\n        \n        return x\n    \n##############\n\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nprint(device)\n\n##############\n\nmodel = NN_Model()\nmodel.to(device)\nprint(model)\ncriterion = nn.BCEWithLogitsLoss()\noptimizer = optim.Adam(model.parameters(), lr=LEARNING_RATE)\n\n##############\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Metric","metadata":{}},{"cell_type":"code","source":"#Code by Shirsh Mall https://www.kaggle.com/code/shirshmall/search-for-higgs-boson-decay-modes\n\ndef binary_acc(y_pred, y_test):\n    y_pred_tag = torch.round(torch.sigmoid(y_pred))\n    \n    correct_results_sum = (y_pred_tag == y_test).sum().float()\n    acc = correct_results_sum/y_test.shape[0]\n    acc = torch.round(acc * 100)\n    \n    return acc","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Train Neural Network","metadata":{}},{"cell_type":"code","source":"#Code by Shirsh Mall https://www.kaggle.com/code/shirshmall/search-for-higgs-boson-decay-modes\n\nlosss = []\nepoch_losss = []\nepoch_accc = []\naccs = []\nmodel.train()\nfor e in range(1, EPOCHS+1):\n    epoch_loss = 0\n    epoch_acc = 0\n    for X_batch, y_batch in train_loader:\n        X_batch, y_batch = X_batch.to(device), y_batch.to(device)\n        optimizer.zero_grad()\n        \n        y_pred = model(X_batch)\n        \n        loss = criterion(y_pred, y_batch)\n        acc = binary_acc(y_pred, y_batch)\n        \n        loss.backward()\n        optimizer.step()\n        \n        epoch_loss += loss.item()\n        epoch_acc += acc.item()\n        losss.append(epoch_loss)\n        accs.append(epoch_acc)\n    epoch_losss.append(epoch_loss/len(train_loader))\n    epoch_accc.append(epoch_acc/len(train_loader))\n\n    print(f'Epoch {e+0:03}: | Loss: {epoch_loss/len(train_loader):.5f} | Acc: {epoch_acc/len(train_loader):.3f}')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Plot Loss and Accuracy","metadata":{}},{"cell_type":"code","source":"#Code by Shirsh Mall https://www.kaggle.com/code/shirshmall/search-for-higgs-boson-decay-modes\n\nplt.plot(epoch_losss, label='loss')\nplt.legend()\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code by Shirsh Mall https://www.kaggle.com/code/shirshmall/search-for-higgs-boson-decay-modes\n\nplt.plot(epoch_accc, label='accuracy')\nplt.legend()\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code by Shirsh Mall https://www.kaggle.com/code/shirshmall/search-for-higgs-boson-decay-modes\n\ny_pred_list = []\nmodel.eval()\nwith torch.no_grad():\n    for X_batch in test_loader:\n        X_batch = X_batch.to(device)\n        y_test_pred = model(X_batch)\n        y_test_pred = torch.sigmoid(y_test_pred)\n        y_pred_list.append(y_test_pred.cpu().numpy())\n\ny_pred_list = [a.squeeze().tolist() for a in y_pred_list]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Acknowledgements:\n\nShirsh Mall https://www.kaggle.com/code/shirshmall/search-for-higgs-boson-decay-modes\n\nBizen https://www.kaggle.com/code/hiro5299834/tps-apr-2021-pseudo-labeling-voting-ensemble","metadata":{}}]}