{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-22T11:04:33.381587Z","iopub.execute_input":"2022-12-22T11:04:33.382052Z","iopub.status.idle":"2022-12-22T11:04:33.391804Z","shell.execute_reply.started":"2022-12-22T11:04:33.382019Z","shell.execute_reply":"2022-12-22T11:04:33.390519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_path = \"/kaggle/input/convolve-epoch1/train.json\"\nser = pd.read_json(data_path,typ=\"series\")\nser.to_frame('count')\ndf = pd.DataFrame({'count':ser})\ndf","metadata":{"execution":{"iopub.status.busy":"2022-12-22T11:04:33.410191Z","iopub.execute_input":"2022-12-22T11:04:33.411272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nohe = LabelEncoder()\ndf['count'] = ohe.fit_transform(df['count'])\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.astype('str')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['count'][0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = []\nfor i in range(4152659):\n    if(df['count'][i]==1):\n        x.append(1)    \n    else:\n        x.append(0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ded = pd.Series(x,name='Label')\nok = pd.DataFrame(ded)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ok","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ok['Label'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json\nx = []\nwith open(\"/kaggle/input/convolve-epoch1/train.json\",\"r\") as file:\n    jsonData = json.load(file)\n    for i in jsonData:\n        x.append(i)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"s = pd.Series(x,name='Log')\nst =s.to_frame()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"newdf = pd.concat([st,ok],axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"newdf","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"newdf.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"backup=newdf.copy()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"newdf[\"c_or_i\"] = newdf[\"Log\"].str.split().str[-1]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"newdf=newdf.replace('interrupt','50')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"newdf['New']=newdf.c_or_i.apply(lambda x: np.where(x.isdigit(),x,'1'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"newdf['New'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"newdf['New']=newdf['New'].astype(int)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"newdf['New'][newdf['New'] > 51] = 1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"newdf['New'][newdf['New'] < 50] = 1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"newdf['New'][newdf['New'] == 50] = 0","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"newdf=newdf.drop('c_or_i',axis='columns')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"newdf['err_id']=newdf['Log'].str.split().str.get(0).astype(int)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"newdf['err_msg']=newdf['Log'].str.split().str.get(2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"newdf['err_r']=newdf['err_msg'].str.split('-').str.get(0)\nnewdf['err_m']=newdf['err_msg'].str.split('-').str.get(1)\nnewdf['err_n']=newdf['err_msg'].str.split('-').str.get(2)\nnewdf['err_cj']=newdf['err_msg'].str.split('-').str.get(3)\nnewdf['err_u']=newdf['err_msg'].str.split('-').str.get(4)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"newdf=newdf.dropna()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"newdf['err_r']=newdf['err_r'].str.replace('R','').astype(int)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"newdf['err_m']=newdf['err_m'].str.replace('M','').astype(int)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"newdf['err_n']=newdf['err_n'].str.replace('N','')\nnewdf['err_n']=newdf['err_n'].str.replace('A','10')\nnewdf['err_n']=newdf['err_n'].str.replace('B','11')\nnewdf['err_n']=newdf['err_n'].str.replace('C','12')\nnewdf['err_n']=newdf['err_n'].str.replace('D','13')\nnewdf['err_n']=newdf['err_n'].str.replace('E','14')\nnewdf['err_n']=newdf['err_n'].str.replace('F','15')\nnewdf['err_n']=newdf['err_n'].str.replace('L','2')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"newdf['err_n']=newdf['err_n'].astype(int)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# newdf['err_n'] = pd.to_numeric(newdf['err_n'], errors='coerce')\n# newdf = newdf[newdf['err_n'].notna()].astype({'err_n': int})","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"newdf['err_cj']=newdf['err_cj'].str.replace('C:J','')\nnewdf['err_cj']=newdf['err_cj'].str.replace('I:J','')\nnewdf['err_cj']=newdf['err_cj'].str.replace('U18','23')\nnewdf['err_cj']=newdf['err_cj'].str.replace('U','')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"newdf['err_cj']=newdf['err_cj'].astype(int)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"newdf['err_u']=newdf['err_u'].str.replace('U','')\nnewdf['err_u']=newdf['err_u'].str.replace('11','2')\nnewdf['err_u']=newdf['err_u'].str.replace('C','3')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"newdf['err_u']=newdf['err_u'].astype(int)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# newdf['err_cj'] = pd.to_numeric(newdf['err_cj'], errors='coerce')\n# newdf = newdf[newdf['err_cj'].notna()].astype({'err_cj': int})","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"newdf","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"newdf['Label'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.utils import resample\n# #create two different dataframe of majority and minority class \n# df_majority = newdf[(newdf['Label']==1)] \n# df_minority = newdf[(newdf['Label']==0)] \n# # upsample minority class\n# df_minority_upsampled = resample(df_minority, \n#                                  replace=True,    # sample with replacement\n#                                  n_samples= 3892561, # to match majority class\n#                                  random_state=42)  # reproducible results\n# # Combine majority class with upsampled minority class\n# df_upsampled = pd.concat([df_minority_upsampled, df_majority])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_upsampled","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_upsampled.corr()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from imblearn.over_sampling import SMOTE\n# Resampling the minority class. The strategy can be changed as required.\nsm = SMOTE(sampling_strategy='minority', random_state=42)\n# Fit the model to generate the data.\noversampled_X, oversampled_Y = sm.fit_resample(newdf.drop(['err_id','Label','Log','err_msg'], axis=1), newdf['Label'])\noversampled = pd.concat([pd.DataFrame(oversampled_Y), pd.DataFrame(oversampled_X)], axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oversampled","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oversampled.corr()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"newdf.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#newdf['Label'].value_counts()\n# type(newdf['Log'][0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv(\"/kaggle/input/convolve-epoch1/test.csv\")\ntest","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['err_id']=test[' Log'].str.split().str.get(0).astype(int)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test[\"c_or_i\"] = test[\" Log\"].str.split().str[-1]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test=test.replace('interrupt','50')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['New']=test.c_or_i.apply(lambda x: np.where(x.isdigit(),x,'1'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['New']=test['New'].astype(int)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['New'][test['New'] > 51] = 1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['New'][test['New'] < 50] = 1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['New'][test['New'] == 50] = 0","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test=test.drop('c_or_i',axis='columns')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['err_msg']=test[' Log'].str.split().str.get(2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['err_r']=test['err_msg'].str.split('-').str.get(0)\ntest['err_m']=test['err_msg'].str.split('-').str.get(1)\ntest['err_n']=test['err_msg'].str.split('-').str.get(2)\ntest['err_cj']=test['err_msg'].str.split('-').str.get(3)\ntest['err_u']=test['err_msg'].str.split('-').str.get(4)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['New'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test=test.fillna(0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['err_r']=test['err_r'].str.replace('R','')\ntest['err_r']=test['err_r'].str.replace('NULL','0')\ntest['err_r']=test['err_r'].str.replace('UNKNOWN_LOCATION','0')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['err_r']=test['err_r'].astype(int)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['err_m']=test['err_m'].str.replace('M','')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test=test.fillna(0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['err_m']=test['err_m'].astype(int)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['err_n']=test['err_n'].str.replace('N','')\ntest['err_n']=test['err_n'].str.replace('A','10')\ntest['err_n']=test['err_n'].str.replace('B','11')\ntest['err_n']=test['err_n'].str.replace('C','12')\ntest['err_n']=test['err_n'].str.replace('D','13')\ntest['err_n']=test['err_n'].str.replace('E','14')\ntest['err_n']=test['err_n'].str.replace('F','15')\ntest['err_n']=test['err_n'].str.replace('L','2')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test=test.fillna(0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['err_n']=test['err_n'].astype(int)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['err_cj']=test['err_cj'].str.replace('C:J','')\ntest['err_cj']=test['err_cj'].str.replace('I:J','')\ntest['err_cj']=test['err_cj'].str.replace('U18','23')\ntest['err_cj']=test['err_cj'].str.replace('U','')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test=test.fillna(0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['err_cj']=test['err_cj'].astype(int)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['err_u']=test['err_u'].str.replace('U','')\ntest['err_u']=test['err_u'].str.replace('11','2')\ntest['err_u']=test['err_u'].str.replace('C','3')\ntest['err_u']=test['err_u'].str.replace('A','0')\ntest['err_u']=test['err_u'].str.replace('E','0')\ntest['err_u']=test['err_u'].str.replace('D','0')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test=test.fillna(0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['err_u']=test['err_u'].astype(int)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#test =test.astype('str')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test.Log[595299]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#newdf =newdf.astype('str')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = oversampled.drop(['Label'],axis='columns')\n# x = df_upsampled.drop(['Label','Log','err_msg'],axis='columns')\ny = oversampled.Label","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train,x_test,y_train,y_test = train_test_split(x,y,test_size=0.01,random_state=42,shuffle=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.svm import SVC\n# model = SVC(C=10)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.ensemble import RandomForestClassifier\n# rfc = RandomForestClassifier(n_estimators=100)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.preprocessing import OneHotEncoder","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# oh = OneHotEncoder()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# newdf['err_r'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for i in range(4152659):\n#     oh.fit(newdf['err_id'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test=test.drop(['ID',' Log','err_msg','err_id'],axis='columns')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# rfc.fit(x_train, y_train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# rfc.score(x_test, y_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.tree import DecisionTreeClassifier\n# knn = KNeighborsClassifier(n_neighbors = 3)\n# knn.fit(x_train, y_train)\n# predictions = knn.predict(X_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# knn.score(x_test, y_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.tree import DecisionTreeClassifier\ndecision_tree = DecisionTreeClassifier()\ndecision_tree.fit(x_train, y_train)\npredictions = decision_tree.predict(X_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"decision_tree.score(x_test, y_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from imblearn.ensemble import BalancedBaggingClassifier\n# from sklearn.tree import DecisionTreeClassifier\n# #Create an instance\n# classifier = BalancedBaggingClassifier(base_estimator=DecisionTreeClassifier(),\n#                                 sampling_strategy='not majority',\n#                                 replacement=False,\n#                                 random_state=42)\n# classifier.fit(x_train, y_train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# classifier.score(x_test,y_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# predictions = classifier.predict(X_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# predictions = rfc.predict(X_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output = pd.DataFrame({'ID': test.ID, 'Label': predictions})","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output['Label']=output['Label'].astype(str)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output['Label']=output['Label'].str.replace('0','abnormal')\noutput['Label']=output['Label'].str.replace('1','normal')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output['Label'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output.to_csv('file2.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}