{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install  tensorflow-text \n!pip install tf-models-official  ","metadata":{"execution":{"iopub.status.busy":"2022-10-30T20:32:47.084576Z","iopub.execute_input":"2022-10-30T20:32:47.085084Z","iopub.status.idle":"2022-10-30T20:35:05.708674Z","shell.execute_reply.started":"2022-10-30T20:32:47.084969Z","shell.execute_reply":"2022-10-30T20:35:05.706603Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport nltk \nimport shutil\n\n\nimport tensorflow as tf\nimport tensorflow_hub as hub\nimport tensorflow_text as text\nfrom official.nlp import optimization  # to create AdamW optimizer\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\n\ntf.get_logger().setLevel('ERROR')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-10-30T20:36:22.520001Z","iopub.execute_input":"2022-10-30T20:36:22.520546Z","iopub.status.idle":"2022-10-30T20:36:28.767870Z","shell.execute_reply.started":"2022-10-30T20:36:22.520491Z","shell.execute_reply":"2022-10-30T20:36:28.766403Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data","metadata":{}},{"cell_type":"code","source":"path = \"../input/feedback-prize-english-language-learning/\"\ntrain = pd.read_csv(path+\"train.csv\")\ntest = pd.read_csv(path+\"test.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-10-30T20:37:02.450537Z","iopub.execute_input":"2022-10-30T20:37:02.450954Z","iopub.status.idle":"2022-10-30T20:37:02.593687Z","shell.execute_reply.started":"2022-10-30T20:37:02.450922Z","shell.execute_reply":"2022-10-30T20:37:02.592406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-30T20:37:08.882997Z","iopub.execute_input":"2022-10-30T20:37:08.883598Z","iopub.status.idle":"2022-10-30T20:37:08.923223Z","shell.execute_reply.started":"2022-10-30T20:37:08.883527Z","shell.execute_reply":"2022-10-30T20:37:08.921949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2022-10-30T20:37:09.805100Z","iopub.execute_input":"2022-10-30T20:37:09.805821Z","iopub.status.idle":"2022-10-30T20:37:09.834416Z","shell.execute_reply.started":"2022-10-30T20:37:09.805775Z","shell.execute_reply":"2022-10-30T20:37:09.833075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"No null values ","metadata":{}},{"cell_type":"markdown","source":"## MODEL ","metadata":{}},{"cell_type":"code","source":"# Choose a BERT model to fine-tune\nbert_model_name = 'small_bert/bert_en_uncased_L-4_H-512_A-8' \n\nmap_name_to_handle = {\n    'bert_en_uncased_L-12_H-768_A-12':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/3',\n    'bert_en_cased_L-12_H-768_A-12':\n        'https://tfhub.dev/tensorflow/bert_en_cased_L-12_H-768_A-12/3',\n    'bert_multi_cased_L-12_H-768_A-12':\n        'https://tfhub.dev/tensorflow/bert_multi_cased_L-12_H-768_A-12/3',\n    'small_bert/bert_en_uncased_L-2_H-128_A-2':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-128_A-2/1',\n    'small_bert/bert_en_uncased_L-2_H-256_A-4':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-256_A-4/1',\n    'small_bert/bert_en_uncased_L-2_H-512_A-8':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-512_A-8/1',\n    'small_bert/bert_en_uncased_L-2_H-768_A-12':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-768_A-12/1',\n    'small_bert/bert_en_uncased_L-4_H-128_A-2':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-128_A-2/1',\n    'small_bert/bert_en_uncased_L-4_H-256_A-4':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-256_A-4/1',\n    'small_bert/bert_en_uncased_L-4_H-512_A-8':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-512_A-8/1',\n    'small_bert/bert_en_uncased_L-4_H-768_A-12':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-768_A-12/1',\n    'small_bert/bert_en_uncased_L-6_H-128_A-2':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-128_A-2/1',\n    'small_bert/bert_en_uncased_L-6_H-256_A-4':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-256_A-4/1',\n    'small_bert/bert_en_uncased_L-6_H-512_A-8':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-512_A-8/1',\n    'small_bert/bert_en_uncased_L-6_H-768_A-12':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-768_A-12/1',\n    'small_bert/bert_en_uncased_L-8_H-128_A-2':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-128_A-2/1',\n    'small_bert/bert_en_uncased_L-8_H-256_A-4':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-256_A-4/1',\n    'small_bert/bert_en_uncased_L-8_H-512_A-8':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-512_A-8/1',\n    'small_bert/bert_en_uncased_L-8_H-768_A-12':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-768_A-12/1',\n    'small_bert/bert_en_uncased_L-10_H-128_A-2':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-128_A-2/1',\n    'small_bert/bert_en_uncased_L-10_H-256_A-4':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-256_A-4/1',\n    'small_bert/bert_en_uncased_L-10_H-512_A-8':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-512_A-8/1',\n    'small_bert/bert_en_uncased_L-10_H-768_A-12':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-768_A-12/1',\n    'small_bert/bert_en_uncased_L-12_H-128_A-2':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-128_A-2/1',\n    'small_bert/bert_en_uncased_L-12_H-256_A-4':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-256_A-4/1',\n    'small_bert/bert_en_uncased_L-12_H-512_A-8':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-512_A-8/1',\n    'small_bert/bert_en_uncased_L-12_H-768_A-12':\n        'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-768_A-12/1',\n    'albert_en_base':\n        'https://tfhub.dev/tensorflow/albert_en_base/2',\n    'electra_small':\n        'https://tfhub.dev/google/electra_small/2',\n    'electra_base':\n        'https://tfhub.dev/google/electra_base/2',\n    'experts_pubmed':\n        'https://tfhub.dev/google/experts/bert/pubmed/2',\n    'experts_wiki_books':\n        'https://tfhub.dev/google/experts/bert/wiki_books/2',\n    'talking-heads_base':\n        'https://tfhub.dev/tensorflow/talkheads_ggelu_bert_en_base/1',\n}\n\n## Text inputs need to be transformed to numeric token ids and arranged in several Tensors \n##before being input to BERT. TensorFlow Hub provides a matching preprocessing model for \n## each of the BERT models \n\nmap_model_to_preprocess = {\n    'bert_en_uncased_L-12_H-768_A-12':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'bert_en_cased_L-12_H-768_A-12':\n        'https://tfhub.dev/tensorflow/bert_en_cased_preprocess/3',\n    'small_bert/bert_en_uncased_L-2_H-128_A-2':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-2_H-256_A-4':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-2_H-512_A-8':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-2_H-768_A-12':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-4_H-128_A-2':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-4_H-256_A-4':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-4_H-512_A-8':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-4_H-768_A-12':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-6_H-128_A-2':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-6_H-256_A-4':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-6_H-512_A-8':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-6_H-768_A-12':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-8_H-128_A-2':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-8_H-256_A-4':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-8_H-512_A-8':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-8_H-768_A-12':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-10_H-128_A-2':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-10_H-256_A-4':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-10_H-512_A-8':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-10_H-768_A-12':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-12_H-128_A-2':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-12_H-256_A-4':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-12_H-512_A-8':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'small_bert/bert_en_uncased_L-12_H-768_A-12':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'bert_multi_cased_L-12_H-768_A-12':\n        'https://tfhub.dev/tensorflow/bert_multi_cased_preprocess/3',\n    'albert_en_base':\n        'https://tfhub.dev/tensorflow/albert_en_preprocess/3',\n    'electra_small':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'electra_base':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'experts_pubmed':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'experts_wiki_books':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n    'talking-heads_base':\n        'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3',\n}\n\ntfhub_handle_encoder = map_name_to_handle[bert_model_name]\ntfhub_handle_preprocess = map_model_to_preprocess[bert_model_name]\n\nprint(f'BERT model selected           : {tfhub_handle_encoder}')\nprint(f'Preprocess model auto-selected: {tfhub_handle_preprocess}')","metadata":{"execution":{"iopub.status.busy":"2022-10-30T20:37:16.794577Z","iopub.execute_input":"2022-10-30T20:37:16.795207Z","iopub.status.idle":"2022-10-30T20:37:16.814798Z","shell.execute_reply.started":"2022-10-30T20:37:16.795153Z","shell.execute_reply":"2022-10-30T20:37:16.813296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bert_preprocess_model = hub.KerasLayer(tfhub_handle_preprocess)","metadata":{"execution":{"iopub.status.busy":"2022-10-30T20:37:18.186752Z","iopub.execute_input":"2022-10-30T20:37:18.187152Z","iopub.status.idle":"2022-10-30T20:37:20.690720Z","shell.execute_reply.started":"2022-10-30T20:37:18.187121Z","shell.execute_reply":"2022-10-30T20:37:20.689189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Let's try the preprocessing model on some text and see the output:\n\ntext_test = ['this is such an amazing movie!']\ntext_preprocessed = bert_preprocess_model(text_test)\n\nprint(f'Keys       : {list(text_preprocessed.keys())}')\nprint(f'Shape      : {text_preprocessed[\"input_word_ids\"].shape}')\nprint(f'Word Ids   : {text_preprocessed[\"input_word_ids\"][0, :12]}')\nprint(f'Input Mask : {text_preprocessed[\"input_mask\"][0, :12]}')\nprint(f'Type Ids   : {text_preprocessed[\"input_type_ids\"][0, :12]}')","metadata":{"execution":{"iopub.status.busy":"2022-10-30T20:37:20.692760Z","iopub.execute_input":"2022-10-30T20:37:20.693174Z","iopub.status.idle":"2022-10-30T20:37:20.924599Z","shell.execute_reply.started":"2022-10-30T20:37:20.693138Z","shell.execute_reply":"2022-10-30T20:37:20.923290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#bert_model = hub.KerasLayer(tfhub_handle_encoder)\n\ndef build_classifier_model():\n    text_input = tf.keras.layers.Input(shape=(), dtype=tf.string, name='text')\n    preprocessing_layer = hub.KerasLayer(tfhub_handle_preprocess, name='preprocessing')\n    \n    encoder_inputs = preprocessing_layer(text_input)\n    encoder = hub.KerasLayer(tfhub_handle_encoder, trainable=True, name='BERT_encoder')\n    outputs = encoder(encoder_inputs)\n    net = outputs['pooled_output']\n    net = tf.keras.layers.Dropout(0.1)(net)\n    \n    #net = tf.keras.layers.Dense(6, activation=None, name='classifier')(net)\n    #Y1 output\n    #y1_output = tf.keras.layers.Dense(units = 1, activation = \"linear\", name = \"y1_output\")(net)\n    y1_outputclass = tf.keras.layers.Dense(units = 1, activation = \"linear\", name = \"y1_outputclass\")(net)\n\n    #Y2 output\n    #y2_output = tf.keras.layers.Dense(units = 1, activation = \"linear\", name = \"y2_output\")(net)\n    y2_outputclass = tf.keras.layers.Dense(units = 1, activation = \"linear\", name = \"y2_outputclass\")(net)\n\n    #Y3 output\n    #y3_output = tf.keras.layers.Dense(units = 1, activation = \"linear\", name = \"y3_output\")(net)\n    y3_outputclass = tf.keras.layers.Dense(units = 1, activation = \"linear\", name = \"y3_outputclass\")(net)\n\n\n    #Y4 output\n    #y4_output = tf.keras.layers.Dense(units = 1, activation = \"linear\", name = \"y4_output\")(net)\n    y4_outputclass = tf.keras.layers.Dense(units = 1, activation = \"linear\", name = \"y4_outputclass\")(net)\n\n    #Y5 output\n    #y5_output = tf.keras.layers.Dense(units = 1, activation = \"linear\", name = \"y5_output\")(net)\n    y5_outputclass = tf.keras.layers.Dense(units = 1, activation = \"linear\", name = \"y5_outputclass\")(net)\n\n\n    #Y6 output\n    #y6_output = tf.keras.layers.Dense(units = 1, activation = \"linear\", name = \"y6_output\")(net)\n    y6_outputclass = tf.keras.layers.Dense(units = 1, activation = \"softmax\", name = \"y6_outputclass\")(net)\n\n    return tf.keras.Model(text_input, outputs = [y1_outputclass, y2_outputclass,y3_outputclass, y4_outputclass, y5_outputclass, y6_outputclass])","metadata":{"execution":{"iopub.status.busy":"2022-10-30T20:37:25.393615Z","iopub.execute_input":"2022-10-30T20:37:25.394049Z","iopub.status.idle":"2022-10-30T20:37:32.793815Z","shell.execute_reply.started":"2022-10-30T20:37:25.394015Z","shell.execute_reply":"2022-10-30T20:37:32.792644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classifier_model = build_classifier_model()\nbert_raw_result = classifier_model(tf.constant(text_test))\nprint(tf.sigmoid(bert_raw_result))","metadata":{"execution":{"iopub.status.busy":"2022-10-30T20:37:32.795630Z","iopub.execute_input":"2022-10-30T20:37:32.796030Z","iopub.status.idle":"2022-10-30T20:37:42.669431Z","shell.execute_reply.started":"2022-10-30T20:37:32.795987Z","shell.execute_reply":"2022-10-30T20:37:42.668074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.utils.plot_model(classifier_model, show_shapes=False)\n","metadata":{"execution":{"iopub.status.busy":"2022-10-30T20:37:42.670755Z","iopub.execute_input":"2022-10-30T20:37:42.671400Z","iopub.status.idle":"2022-10-30T20:37:44.077613Z","shell.execute_reply.started":"2022-10-30T20:37:42.671366Z","shell.execute_reply":"2022-10-30T20:37:44.076217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AUTOTUNE = tf.data.AUTOTUNE\nbatch_size = 128\nseed = 42\n\nraw_train_ds = tf.data.TextLineDataset(\n   '../input/feedback-prize-english-language-learning/train.csv',)\n\ntrain_ds = raw_train_ds.cache().prefetch(buffer_size=AUTOTUNE)\n\nval_ds = tf.data.TextLineDataset(\n    '../input/feedback-prize-english-language-learning/train.csv',)\n\nval_ds = val_ds.cache().prefetch(buffer_size=AUTOTUNE)\n\ntest_ds = tf.data.TextLineDataset(\n    '../input/feedback-prize-english-language-learning/test.csv')\n\ntest_ds = test_ds.cache().prefetch(buffer_size=AUTOTUNE)","metadata":{"execution":{"iopub.status.busy":"2022-10-30T20:37:44.080636Z","iopub.execute_input":"2022-10-30T20:37:44.081029Z","iopub.status.idle":"2022-10-30T20:37:44.137234Z","shell.execute_reply.started":"2022-10-30T20:37:44.080995Z","shell.execute_reply":"2022-10-30T20:37:44.136290Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss = tf.keras.losses.mean_absolute_error\n\nmetrics = tf.metrics.Accuracy()\n\nepochs = 2\nsteps_per_epoch = tf.data.experimental.cardinality(train_ds).numpy()\nnum_train_steps = steps_per_epoch * epochs\nnum_warmup_steps = int(0.1*num_train_steps)\n\ninit_lr = 3e-5\n\"\"\"optimizer = optimization.create_optimizer(init_lr=init_lr,\n                                          num_train_steps=num_train_steps,\n                                          num_warmup_steps=num_warmup_steps,\n                                          \n                                          optimizer_type='adamw')\"\"\"\n\nclassifier_model.compile(loss = 'mean_absolute_error', optimizer = 'adam', metrics = ['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2022-10-30T20:38:07.162052Z","iopub.execute_input":"2022-10-30T20:38:07.162528Z","iopub.status.idle":"2022-10-30T20:38:07.193553Z","shell.execute_reply.started":"2022-10-30T20:38:07.162490Z","shell.execute_reply":"2022-10-30T20:38:07.192678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X =train['full_text'] \ny = train.drop(columns=['full_text','text_id'])\n\nprint(f'Training model with {tfhub_handle_encoder}')\nhistory = classifier_model.fit(x=X, y =y, epochs=epochs)","metadata":{"execution":{"iopub.status.busy":"2022-10-30T20:38:13.952476Z","iopub.execute_input":"2022-10-30T20:38:13.953125Z","iopub.status.idle":"2022-10-30T20:57:04.625769Z","shell.execute_reply.started":"2022-10-30T20:38:13.953087Z","shell.execute_reply":"2022-10-30T20:57:04.624305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ypred =classifier_model.predict([test['full_text']])","metadata":{"execution":{"iopub.status.busy":"2022-10-28T20:06:27.299003Z","iopub.status.idle":"2022-10-28T20:06:27.299789Z","shell.execute_reply.started":"2022-10-28T20:06:27.299556Z","shell.execute_reply":"2022-10-28T20:06:27.299581Z"},"trusted":true},"execution_count":null,"outputs":[]}]}