{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Libraries","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-10T09:17:41.526862Z","iopub.execute_input":"2022-12-10T09:17:41.527724Z","iopub.status.idle":"2022-12-10T09:18:21.607182Z","shell.execute_reply.started":"2022-12-10T09:17:41.527661Z","shell.execute_reply":"2022-12-10T09:18:21.605853Z"}}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport json\nfrom lightgbm import LGBMClassifier\nfrom sklearn import model_selection, preprocessing, linear_model, metrics\nfrom textblob import Word\nfrom sklearn.feature_extraction.text import CountVectorizer\nimport re","metadata":{"execution":{"iopub.status.busy":"2022-12-19T10:03:25.622235Z","iopub.execute_input":"2022-12-19T10:03:25.623195Z","iopub.status.idle":"2022-12-19T10:03:28.833874Z","shell.execute_reply.started":"2022-12-19T10:03:25.623071Z","shell.execute_reply":"2022-12-19T10:03:28.832903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom tqdm import tqdm\nfrom sklearn.model_selection import train_test_split\nimport tensorflow as tf\nfrom keras.models import Sequential\nfrom keras.layers.recurrent import LSTM, GRU,SimpleRNN\nfrom keras.layers.core import Dense, Activation, Dropout\nfrom keras.layers.embeddings import Embedding\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.utils import np_utils\nfrom sklearn import preprocessing, decomposition, model_selection, metrics, pipeline\nfrom keras.layers import GlobalMaxPooling1D, Conv1D, MaxPooling1D, Flatten, Bidirectional, SpatialDropout1D\nfrom keras.preprocessing import sequence, text\nfrom keras.callbacks import EarlyStopping\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\nfrom plotly import graph_objs as go\nimport plotly.express as px\nimport plotly.figure_factory as ff","metadata":{"execution":{"iopub.status.busy":"2022-12-19T10:03:32.452311Z","iopub.execute_input":"2022-12-19T10:03:32.452675Z","iopub.status.idle":"2022-12-19T10:03:37.674525Z","shell.execute_reply.started":"2022-12-19T10:03:32.452645Z","shell.execute_reply":"2022-12-19T10:03:37.673026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n    # set: this is always the case on Kaggle.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2022-12-19T08:24:02.552307Z","iopub.execute_input":"2022-12-19T08:24:02.552705Z","iopub.status.idle":"2022-12-19T08:24:02.569914Z","shell.execute_reply.started":"2022-12-19T08:24:02.552674Z","shell.execute_reply":"2022-12-19T08:24:02.568706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loading Json","metadata":{}},{"cell_type":"code","source":"file = open('/kaggle/input/convolve-prev/cnv/train.json')\ndata = json.load(file)","metadata":{"execution":{"iopub.status.busy":"2022-12-19T10:03:39.630960Z","iopub.execute_input":"2022-12-19T10:03:39.631635Z","iopub.status.idle":"2022-12-19T10:03:49.938701Z","shell.execute_reply.started":"2022-12-19T10:03:39.631594Z","shell.execute_reply":"2022-12-19T10:03:49.937667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = list(data.keys())\ntarget = list(data.values())\nprint(labels[0],target[0])","metadata":{"execution":{"iopub.status.busy":"2022-12-19T10:03:52.725999Z","iopub.execute_input":"2022-12-19T10:03:52.726740Z","iopub.status.idle":"2022-12-19T10:03:52.936901Z","shell.execute_reply.started":"2022-12-19T10:03:52.726702Z","shell.execute_reply":"2022-12-19T10:03:52.935778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#file.close()\nlabels = 0\ntarget = 0\ndata = 0","metadata":{"execution":{"iopub.status.busy":"2022-12-19T10:05:00.307575Z","iopub.execute_input":"2022-12-19T10:05:00.307942Z","iopub.status.idle":"2022-12-19T10:05:00.467273Z","shell.execute_reply.started":"2022-12-19T10:05:00.307909Z","shell.execute_reply":"2022-12-19T10:05:00.466375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.DataFrame()\ntrain['Log'] = labels\n\ntrain['target'] = target","metadata":{"execution":{"iopub.status.busy":"2022-12-19T10:04:03.699617Z","iopub.execute_input":"2022-12-19T10:04:03.700714Z","iopub.status.idle":"2022-12-19T10:04:05.229264Z","shell.execute_reply.started":"2022-12-19T10:04:03.700676Z","shell.execute_reply":"2022-12-19T10:04:05.228272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/convolve-prev/cnv/test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-12-19T10:05:04.885133Z","iopub.execute_input":"2022-12-19T10:05:04.886292Z","iopub.status.idle":"2022-12-19T10:05:06.597862Z","shell.execute_reply.started":"2022-12-19T10:05:04.886243Z","shell.execute_reply":"2022-12-19T10:05:06.596932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.options.display.max_colwidth = 1000\ntest['Log'] = test['Log'].str.lstrip()\ntest['Log'] = test['Log'].str.rstrip()\n#test[test['Log'].str.startswith('N')]\nremoved_letters = train['Log'].str.replace('[a-zA-Z]','')\nremoved_letters = removed_letters.apply(lambda s: re.sub(r'\\W+', ' ', s))","metadata":{"execution":{"iopub.status.busy":"2022-12-19T09:15:19.844610Z","iopub.execute_input":"2022-12-19T09:15:19.845565Z","iopub.status.idle":"2022-12-19T09:16:38.150976Z","shell.execute_reply.started":"2022-12-19T09:15:19.845515Z","shell.execute_reply":"2022-12-19T09:16:38.150130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"k = int(removed_letters[0].split()[15].split('.')[-1])\nremoved_letters[15]\n","metadata":{"execution":{"iopub.status.busy":"2022-12-19T09:26:37.508210Z","iopub.execute_input":"2022-12-19T09:26:37.508588Z","iopub.status.idle":"2022-12-19T09:26:37.515037Z","shell.execute_reply.started":"2022-12-19T09:26:37.508557Z","shell.execute_reply":"2022-12-19T09:26:37.514181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"removed_letters = removed_letters.str.rstrip()\nremoved_letters = removed_letters.str.lstrip()\nfinal = []\nstore = []\nfor i in range (1,len(removed_letters)):\n    #print(int(removed_letters[i-1].split()[3].split('.')[-1]),int(removed_letters[i].split()[3].split('.')[-1]))\n    try:\n        k = int(removed_letters[i-1].split()[15].split('.')[-1])\n        k_n = int(removed_letters[i].split()[15].split('.')[-1])\n    except:\n        store.append((removed_letters[i-1],removed_letters[i]))\n    if k_n - k < 0:\n        k_n *= 10 \n    final.append(k_n - k)","metadata":{"execution":{"iopub.status.busy":"2022-12-19T09:32:48.548704Z","iopub.execute_input":"2022-12-19T09:32:48.549630Z","iopub.status.idle":"2022-12-19T09:33:29.972528Z","shell.execute_reply.started":"2022-12-19T09:32:48.549600Z","shell.execute_reply":"2022-12-19T09:33:29.971640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#store","metadata":{"execution":{"iopub.status.busy":"2022-12-19T09:36:42.388343Z","iopub.execute_input":"2022-12-19T09:36:42.389234Z","iopub.status.idle":"2022-12-19T09:36:42.395809Z","shell.execute_reply.started":"2022-12-19T09:36:42.389195Z","shell.execute_reply":"2022-12-19T09:36:42.394309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test Data","metadata":{}},{"cell_type":"code","source":"train['Log'] = train['Log'].apply(lambda x: x.lower())\ntest['Log'] = test['Log'].apply(lambda x: x.lower())","metadata":{"execution":{"iopub.status.busy":"2022-12-19T08:24:45.926976Z","iopub.execute_input":"2022-12-19T08:24:45.929432Z","iopub.status.idle":"2022-12-19T08:24:48.263643Z","shell.execute_reply.started":"2022-12-19T08:24:45.929380Z","shell.execute_reply":"2022-12-19T08:24:48.262565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Text Preprocessing","metadata":{}},{"cell_type":"markdown","source":"## Removing digits","metadata":{}},{"cell_type":"code","source":"train['Log'] = train['Log'].str.replace('\\d','')\ntest['Log'] = test['Log'].str.replace('\\d','')","metadata":{"execution":{"iopub.status.busy":"2022-12-19T06:16:17.703686Z","iopub.execute_input":"2022-12-19T06:16:17.704078Z","iopub.status.idle":"2022-12-19T06:16:23.305057Z","shell.execute_reply.started":"2022-12-19T06:16:17.704038Z","shell.execute_reply":"2022-12-19T06:16:23.302905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Removing Delimeters","metadata":{}},{"cell_type":"code","source":"train['Log'] = train['Log'].apply(lambda s: re.sub(r'\\W+', ' ', s))\ntest['Log'] = test['Log'].apply(lambda s: re.sub(r'\\W+', ' ', s))","metadata":{"execution":{"iopub.status.busy":"2022-12-19T08:24:59.935334Z","iopub.execute_input":"2022-12-19T08:24:59.935727Z","iopub.status.idle":"2022-12-19T08:25:53.975223Z","shell.execute_reply.started":"2022-12-19T08:24:59.935694Z","shell.execute_reply":"2022-12-19T08:25:53.974148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train-Test Split","metadata":{}},{"cell_type":"code","source":"def roc_auc(predictions,target):\n    '''\n    This methods returns the AUC Score when given the Predictions\n    and Labels\n    '''\n    \n    fpr, tpr, thresholds = metrics.roc_curve(target, predictions)\n    roc_auc = metrics.auc(fpr, tpr)\n    return roc_auc","metadata":{"execution":{"iopub.status.busy":"2022-12-19T10:05:24.914985Z","iopub.execute_input":"2022-12-19T10:05:24.915763Z","iopub.status.idle":"2022-12-19T10:05:24.923748Z","shell.execute_reply.started":"2022-12-19T10:05:24.915725Z","shell.execute_reply":"2022-12-19T10:05:24.922781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = train[\"Log\"]\nconditions = [(train['target'] == 'normal'),\n             (train['target'] == 'abnormal')]\nvalues = [0,1]\ntrain['target'] = np.select(conditions,values)\nY = train[\"target\"]\nxtrain, xvalid, ytrain, yvalid = train_test_split(train.Log.values, train.target.values, \n                                                  stratify=train.target.values, \n                                                  random_state=42, \n                                                  test_size=0.2, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2022-12-19T10:05:27.795792Z","iopub.execute_input":"2022-12-19T10:05:27.796173Z","iopub.status.idle":"2022-12-19T10:05:29.655348Z","shell.execute_reply.started":"2022-12-19T10:05:27.796138Z","shell.execute_reply":"2022-12-19T10:05:29.654300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ytrain","metadata":{"execution":{"iopub.status.busy":"2022-12-19T10:05:59.567440Z","iopub.execute_input":"2022-12-19T10:05:59.568004Z","iopub.status.idle":"2022-12-19T10:05:59.579379Z","shell.execute_reply.started":"2022-12-19T10:05:59.567944Z","shell.execute_reply":"2022-12-19T10:05:59.578329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Tokenising ","metadata":{}},{"cell_type":"code","source":"# using keras tokenizer here\n#! pip install keras\nfrom keras.preprocessing import sequence, text\ntoken = text.Tokenizer(num_words=None)\nmax_len = 200\ntoken.fit_on_texts(list(xtrain) + list(xvalid))\nxtrain_seq = token.texts_to_sequences(xtrain)\nxvalid_seq = token.texts_to_sequences(xvalid)\n\n#zero pad the sequences\nxtrain_pad = sequence.pad_sequences(xtrain_seq, maxlen=max_len)\nxvalid_pad = sequence.pad_sequences(xvalid_seq, maxlen=max_len)\n\nword_index = token.word_index","metadata":{"execution":{"iopub.status.busy":"2022-12-19T10:06:04.594222Z","iopub.execute_input":"2022-12-19T10:06:04.594585Z","iopub.status.idle":"2022-12-19T10:10:15.562658Z","shell.execute_reply.started":"2022-12-19T10:06:04.594552Z","shell.execute_reply":"2022-12-19T10:10:15.561607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xtrain_seq = 0\nxvalid_seq = 0","metadata":{"execution":{"iopub.status.busy":"2022-12-19T10:11:10.641139Z","iopub.execute_input":"2022-12-19T10:11:10.641885Z","iopub.status.idle":"2022-12-19T10:11:11.702346Z","shell.execute_reply.started":"2022-12-19T10:11:10.641848Z","shell.execute_reply":"2022-12-19T10:11:11.701304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training Model","metadata":{}},{"cell_type":"code","source":"embeddings_index = {}\nf = open('/kaggle/input/glove840b300dtxt/glove.840B.300d.txt','r',encoding='utf-8')\nfor line in tqdm(f):\n    values = line.split(' ')\n    word = values[0]\n    coefs = np.asarray([float(val) for val in values[1:]])\n    embeddings_index[word] = coefs\nf.close()\n\nprint('Found %s word vectors.' % len(embeddings_index))","metadata":{"execution":{"iopub.status.busy":"2022-12-19T10:18:07.501934Z","iopub.execute_input":"2022-12-19T10:18:07.502327Z","iopub.status.idle":"2022-12-19T10:22:23.790218Z","shell.execute_reply.started":"2022-12-19T10:18:07.502293Z","shell.execute_reply":"2022-12-19T10:22:23.789176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"embedding_matrix = np.zeros((len(word_index) + 1, 300))\nfor word, i in tqdm(word_index.items()):\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None:\n        embedding_matrix[i] = embedding_vector","metadata":{"execution":{"iopub.status.busy":"2022-12-19T10:22:40.174194Z","iopub.execute_input":"2022-12-19T10:22:40.174554Z","iopub.status.idle":"2022-12-19T10:22:41.630572Z","shell.execute_reply.started":"2022-12-19T10:22:40.174524Z","shell.execute_reply":"2022-12-19T10:22:41.628887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Sequential()\nmodel.add(Embedding(len(word_index) + 1,\n                     300,\n                     weights=[embedding_matrix],\n                     input_length=max_len,\n                     trainable=False))\n\nmodel.add(LSTM(100, dropout=0.3, recurrent_dropout=0.3))\nmodel.add(Dense(1, activation='sigmoid'))\nmodel.compile(loss='binary_crossentropy', optimizer='adam',metrics=['accuracy'])\n    \nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-12-19T10:22:45.590390Z","iopub.execute_input":"2022-12-19T10:22:45.590775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.fit(xtrain_pad,ytrain)","metadata":{"execution":{"iopub.status.busy":"2022-12-19T10:01:44.781563Z","iopub.execute_input":"2022-12-19T10:01:44.782623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predicting on Test","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/convolve-epoch1/test.csv')\ndf['Label'] = model.predict(test_vec)\n","metadata":{"execution":{"iopub.status.busy":"2022-12-10T10:16:46.092337Z","iopub.execute_input":"2022-12-10T10:16:46.092732Z","iopub.status.idle":"2022-12-10T10:16:47.42114Z","shell.execute_reply.started":"2022-12-10T10:16:46.092702Z","shell.execute_reply":"2022-12-10T10:16:47.419925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Drop Logs and index","metadata":{}},{"cell_type":"code","source":"df = df.drop(['Log'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-12-10T10:17:44.207542Z","iopub.execute_input":"2022-12-10T10:17:44.207953Z","iopub.status.idle":"2022-12-10T10:17:44.222677Z","shell.execute_reply.started":"2022-12-10T10:17:44.207921Z","shell.execute_reply":"2022-12-10T10:17:44.220864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Form a CSV","metadata":{}},{"cell_type":"code","source":"df.to_csv('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-12-10T10:17:47.621833Z","iopub.execute_input":"2022-12-10T10:17:47.622239Z","iopub.status.idle":"2022-12-10T10:17:48.669966Z","shell.execute_reply.started":"2022-12-10T10:17:47.622208Z","shell.execute_reply":"2022-12-10T10:17:48.668726Z"},"trusted":true},"execution_count":null,"outputs":[]}]}