{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%load_ext autoreload\n%autoreload 2\n\nfrom utility_script import *\n\nimport os\nimport re\nimport json\nimport pandas as pd\nimport numpy as np\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-12T09:35:29.382111Z","iopub.execute_input":"2021-06-12T09:35:29.382556Z","iopub.status.idle":"2021-06-12T09:35:37.048277Z","shell.execute_reply.started":"2021-06-12T09:35:29.382408Z","shell.execute_reply":"2021-06-12T09:35:37.046811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mymodel = MyBERT(train=True)\nmymodel.compile()","metadata":{"execution":{"iopub.status.busy":"2021-06-12T09:33:49.492974Z","iopub.execute_input":"2021-06-12T09:33:49.493668Z","iopub.status.idle":"2021-06-12T09:34:13.101573Z","shell.execute_reply.started":"2021-06-12T09:33:49.493625Z","shell.execute_reply":"2021-06-12T09:34:13.10014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = Data()\ntrain_df = data.read_train_from_file() #nrows= 100\nprint('train_df shape before ', train_df.shape)\ntrain_df_1 = train_df.loc[train_df['dataset_found']].groupby(['dataset_name', 'section_title']).head(500).reset_index(drop=True)\ntrain_df_2 = train_df.loc[~train_df['dataset_found']].sample(frac=1)\nprint(train_df_1.shape)\nprint(train_df_2.shape)\ntrain_df = train_df_1.append(train_df_2)\ntrain_df = train_df.sample(frac=1).reset_index(drop=True)\nprint('train_df shape after dropping shit ', train_df.shape)\ntrain_df.head(2)","metadata":{"execution":{"iopub.status.busy":"2021-06-12T09:35:37.054748Z","iopub.execute_input":"2021-06-12T09:35:37.057885Z","iopub.status.idle":"2021-06-12T09:37:36.984243Z","shell.execute_reply.started":"2021-06-12T09:35:37.057833Z","shell.execute_reply":"2021-06-12T09:37:36.982924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(chunk):\n    train_encodings = mymodel.fast_tokenizer(chunk[data.lines_col].tolist(), max_length=MAX_LENGTH_PER_SENTENCE, padding=True, truncation=True, return_offsets_mapping=True,is_split_into_words=True, return_tensors='np') #return_offsets_mapping=True # ,\n    train_labels = data.encode_tags(chunk[data.labels_col], train_encodings)\n    mymodel.train_model(train_encodings, tf.convert_to_tensor(np.array(train_labels)), epochs = 2)\n\n\ndef train_model_large_df(df, iterations = None):\n    df.dropna(inplace=True)\n    i = 0\n    exception_count = 0\n    for chunk in np.array_split(df, int(df.shape[0]/128)):\n        try:\n            train_model(chunk)\n        except:\n            exception_count += 1\n            pass\n    print(\"total exception_count\", exception_count)\n    #     op = mymodel.model(train_encodings)\n    #     get_dataset_names_from_outputs(op)\n#         break\n#         i += 1\n#         if i > 2:\n#             break","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nMAX_LENGTH_PER_SENTENCE=256\ntrain_model_large_df(train_df)\n# val_labels = encode_tags(val_tags, val_encodings)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# n = 4877\n# test_example = [chunk['lines'].loc[n]]\n# test_labels = [chunk['labels'].loc[n]]\n# train_encodings = mymodel.fast_tokenizer(test_example, max_length=MAX_LENGTH_PER_SENTENCE, padding=True, truncation=True, return_offsets_mapping=True,is_split_into_words=True, return_tensors='np') #return_offsets_mapping=True # ,\n# train_labels = data.encode_tags(test_labels, train_encodings)\n# mymodel.train_model(train_encodings, tf.convert_to_tensor(np.array(train_labels)), epochs = 2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mymodel.model.save_pretrained(\"model_file\")\nmymodel.tokenizer.save_pretrained(\"tokenizer_file\")\n# mymodel.fast_tokenizer.save_pretrained(\"fast_tokenizer_file\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls /kaggle/working/model_file","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#TODO:\n# in data class, take from class 0 only if length > 3","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.columns","metadata":{"execution":{"iopub.status.busy":"2021-05-28T23:22:03.621689Z","iopub.execute_input":"2021-05-28T23:22:03.622008Z","iopub.status.idle":"2021-05-28T23:22:03.697715Z","shell.execute_reply.started":"2021-05-28T23:22:03.621961Z","shell.execute_reply":"2021-05-28T23:22:03.696153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head(2)","metadata":{"execution":{"iopub.status.busy":"2021-06-03T12:42:21.122907Z","iopub.execute_input":"2021-06-03T12:42:21.123319Z","iopub.status.idle":"2021-06-03T12:42:21.199049Z","shell.execute_reply.started":"2021-06-03T12:42:21.123285Z","shell.execute_reply":"2021-06-03T12:42:21.198191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape","metadata":{"execution":{"iopub.status.busy":"2021-06-03T12:42:05.231949Z","iopub.execute_input":"2021-06-03T12:42:05.232285Z","iopub.status.idle":"2021-06-03T12:42:05.303403Z","shell.execute_reply.started":"2021-06-03T12:42:05.232255Z","shell.execute_reply":"2021-06-03T12:42:05.302251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.loc[train_df['dataset_found']].groupby('section_title').count().sort_values(by='lines',ascending=False)","metadata":{"execution":{"iopub.status.busy":"2021-05-28T23:33:47.774548Z","iopub.execute_input":"2021-05-28T23:33:47.774876Z","iopub.status.idle":"2021-05-28T23:33:47.874219Z","shell.execute_reply.started":"2021-05-28T23:33:47.774842Z","shell.execute_reply":"2021-05-28T23:33:47.872719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df.loc[train_df['dataset_found']]['lines'].apply(lambda x: len(x)).value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-05-28T23:26:34.286181Z","iopub.execute_input":"2021-05-28T23:26:34.286679Z","iopub.status.idle":"2021-05-28T23:26:34.391954Z","shell.execute_reply.started":"2021-05-28T23:26:34.286643Z","shell.execute_reply":"2021-05-28T23:26:34.390994Z"},"trusted":true},"execution_count":null,"outputs":[]}]}