{"metadata":{"kernelspec":{"name":"ir","display_name":"R","language":"R"},"language_info":{"name":"R","codemirror_mode":"r","pygments_lexer":"r","mimetype":"text/x-r-source","file_extension":".r","version":"4.0.5"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"library(tidyverse)\nlibrary(reticulate)\nlibrary(keras)\nlibrary(tensorflow)\nlibrary(data.table)\nnp <- import(\"numpy\")\nlibrary(foreach)\nlibrary(doParallel)\ninstall.packages(\"tfdatasets\")\nlibrary(tfdatasets)","metadata":{"execution":{"iopub.status.busy":"2023-07-09T16:34:17.401340Z","iopub.execute_input":"2023-07-09T16:34:17.403537Z","iopub.status.idle":"2023-07-09T16:34:57.687407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tensorflow::tf_version()\ntf$keras$backend$set_floatx('float32')\ntensorflow::tf_gpu_configured()","metadata":{"execution":{"iopub.status.busy":"2023-07-09T15:36:21.485178Z","iopub.execute_input":"2023-07-09T15:36:21.520351Z","iopub.status.idle":"2023-07-09T15:36:28.496996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_terms <- fread(\"/kaggle/input/cafa-5-protein-function-prediction/Train/train_terms.tsv\",sep=\"\\t\")\ntrain_ids <- np$load(\"/kaggle/input/protbert-embeddings-for-cafa5/train_ids.npy\")","metadata":{"execution":{"iopub.status.busy":"2023-07-09T16:34:59.621240Z","iopub.execute_input":"2023-07-09T16:34:59.651029Z","iopub.status.idle":"2023-07-09T16:35:04.047982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_cores <- 4\ncl <- makeCluster(num_cores)\nregisterDoParallel(cl)\n\ntargets <- function(n_labels_to_consider) {\n  vec_freqCount <- train_terms %>% \n    group_by(term) %>% \n    summarize(n = n()) %>% \n    arrange(desc(n)) \n\n  labels_to_consider <- vec_freqCount$term[1:n_labels_to_consider] \n\n  vec_train_protein_ids <- train_ids\n  train_size <- length(vec_train_protein_ids)\n  Y <- array(0, dim = c(train_size, n_labels_to_consider))\n\n  series_train_protein_ids <- setNames(as.vector(vec_train_protein_ids), seq_along(vec_train_protein_ids))\n  train_terms_smaller <- subset(train_terms, term %in% labels_to_consider)\n\n  for (i in 1:dim(Y)[2]) {\n    m <- train_terms_smaller$term %in% labels_to_consider[i]\n    Y[, i] <- as.numeric(series_train_protein_ids %in% train_terms_smaller$EntryID[m])\n    if (i %% 10 == 0) {\n      #print(paste(i, \" \", sum(m)))\n    }\n  }\n    return(Y)\n}","metadata":{"execution":{"iopub.status.busy":"2023-07-09T16:35:05.427883Z","iopub.execute_input":"2023-07-09T16:35:05.432984Z","iopub.status.idle":"2023-07-09T16:35:11.607425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" \n# Define the U-Net model\nProUnet_1d <- function(input_shape) {\n  reshape_layer <- layer_reshape(target_shape = c(input_shape, 1))\n  \n  inputs <- layer_input(shape = input_shape)\n  x <- reshape_layer(inputs)\n  \n  conv1 <- x %>%\n    layer_conv_1d(filters = 64, kernel_size = 3, activation = 'relu', padding = 'same') %>%\n    layer_conv_1d(filters = 64, kernel_size = 3, activation = 'relu', padding = 'same')\n  \n  pool1 <- conv1 %>% layer_max_pooling_1d(pool_size = 2)\n  \n  conv2 <- pool1 %>%\n    layer_conv_1d(filters = 128, kernel_size = 3, activation = 'relu', padding = 'same') %>%\n    layer_conv_1d(filters = 128, kernel_size = 3, activation = 'relu', padding = 'same')\n  \n  pool2 <- conv2 %>% layer_max_pooling_1d(pool_size = 2)\n  \n  # Bottom\n  conv3 <- pool2 %>%\n    layer_conv_1d(filters = 256, kernel_size = 3, activation = 'relu', padding = 'same') %>%\n    layer_conv_1d(filters = 256, kernel_size = 3, activation = 'relu', padding = 'same')\n  \n  # Expanding path\n  up2 <- conv3 %>%\n    layer_upsampling_1d(size = 2) %>%\n    {layer_concatenate(inputs = list(conv2, .), axis = 2, trainable = TRUE)} %>%\n    layer_conv_1d(filters = 128, kernel_size = 3, activation = 'relu', padding = 'same') %>%\n    layer_conv_1d(filters = 128, kernel_size = 3, activation = 'relu', padding = 'same')\n  \n  up1 <- up2 %>%\n    layer_upsampling_1d(size = 2) %>%\n    {layer_concatenate(inputs = list(conv1, .), axis = 2, trainable = TRUE)} %>%\n    layer_conv_1d(filters = 64, kernel_size = 3, activation = 'relu', padding = 'same') %>%\n    layer_conv_1d(filters = 64, kernel_size = 3, activation = 'relu', padding = 'same')\n  \n  # Output\n  outputs <- up1 %>% layer_conv_1d(filters = 1, kernel_size = 1, activation = 'sigmoid', padding = 'same') %>%\n     layer_flatten()\n  \n  # Create the model\n  model <- keras_model(inputs, outputs)\n  \n  return(model)\n}\n\n# Test the model\ninput_shape <- c(1024)  # Input shape for the model\n","metadata":{"execution":{"iopub.status.busy":"2023-07-07T16:25:34.259159Z","iopub.execute_input":"2023-07-07T16:25:34.263032Z","iopub.status.idle":"2023-07-07T16:25:34.287635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"residual_block <- function(inputs, filters, strides = 1, use_bias = FALSE) {\n  x <- inputs\n  shortcut <- inputs\nfilters <- 32\nstrides = 1\nuse_bias <- FALSE\n    shortcut <- layer_conv_1d(filters = filters, kernel_size = 1, strides = strides, padding = \"valid\", use_bias = use_bias)(shortcut)\n    shortcut <- layer_batch_normalization()(shortcut) \n  \n    x <- layer_conv_1d(filters = filters, kernel_size = 3, strides = strides, padding = \"same\", use_bias = use_bias)(x)\n  x <- layer_batch_normalization()(x)\n  x <- layer_activation(activation = \"relu\")(x)\n  \n  x <- layer_conv_1d(filters = filters, kernel_size = 3, strides = 1, padding = \"same\", use_bias = use_bias)(x)\n  x <- layer_batch_normalization()(x)\n  \n  x <- layer_add(list(x, shortcut), trainable = TRUE) \n  x <- layer_activation(activation = \"relu\")(x)\n    \n  return(x)\n}\n","metadata":{"execution":{"iopub.status.busy":"2023-07-07T16:25:36.499078Z","iopub.execute_input":"2023-07-07T16:25:36.501833Z","iopub.status.idle":"2023-07-07T16:25:36.526553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"resnet_model <- function(input_shape, num_classes) {\n  reshape_layer <- layer_reshape(target_shape = c(input_shape, 1))\n    inputs <- layer_input(shape = input_shape)\n  x <- reshape_layer(inputs)\n  \n  x <- layer_conv_1d(filters = 64, kernel_size = 3, strides = 2, padding = \"same\", use_bias = FALSE)(x)\n  x <- layer_batch_normalization()(x)\n  x <- layer_activation(activation = \"relu\")(x)\n  x <- layer_max_pooling_1d(pool_size = 3, strides = 2, padding = \"same\")(x)\n \n  x <- residual_block(x, filters = 64, strides = 1)\n  x <- residual_block(x, filters = 64, strides = 1)\n  \n  x <- residual_block(x, filters = 128, strides = 2)\n  x <- residual_block(x, filters = 128, strides = 1)\n  \n  x <- residual_block(x, filters = 256, strides = 2)\n  x <- residual_block(x, filters = 256, strides = 1)\n  \n  x <- residual_block(x, filters = 512, strides = 2)\n  x <- residual_block(x, filters = 512, strides = 1)\n  \n  x <- layer_average_pooling_1d(pool_size = 3)(x)\n  x <- layer_flatten()(x)\n  \n  outputs <- layer_dense(units = num_classes, activation='linear')(x)\n  \n  model <- keras_model(inputs, outputs)\n     \n    model$compile(\n  optimizer = \"adam\",\n  loss = \"mean_squared_error\",\n  metrics = c(\"mse\")\n)\n  \n  return(model)\n}","metadata":{"execution":{"iopub.status.busy":"2023-07-07T16:26:41.830067Z","iopub.execute_input":"2023-07-07T16:26:41.832289Z","iopub.status.idle":"2023-07-07T16:26:41.849703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" MultiLayerPerceptron <- function(input_dim, num_classes) {\n  \n  model <- keras_model_sequential()\n  \n  model %>%\n    layer_dense(units = 1012, input_shape = input_dim) %>%\n    layer_activation(\"tanh\") %>%\n    layer_dense(units = 864) %>%\n    layer_activation(\"tanh\") %>%\n    layer_dense(units = 712) %>%\n    layer_activation(\"tanh\") %>%\n    layer_dense(units = num_classes,activation='linear')\n      \n    model$compile(\n  optimizer = \"adam\",\n  loss = \"mean_squared_error\",\n  metrics = c(\"mse\")\n         \n)\n     return(model)\n  \n}\n\n ","metadata":{"execution":{"iopub.status.busy":"2023-07-07T16:26:32.408069Z","iopub.execute_input":"2023-07-07T16:26:32.413034Z","iopub.status.idle":"2023-07-07T16:26:32.435943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CNN1D <- function(input_dim, num_classes) {\n  model <- keras_model_sequential()\n  \n  model %>%\n    layer_reshape(target_shape = c(input_dim, 1)) %>%\n    layer_conv_1d(filters = 3, kernel_size = 3, padding = \"same\", activation = \"relu\", input_shape = c(1, input_dim)) %>%\n    layer_max_pooling_1d(pool_size = 2, strides = 2) %>%\n    layer_conv_1d(filters = 8, kernel_size = 3, padding = \"same\", activation = \"relu\") %>%\n    layer_max_pooling_1d(pool_size = 2, strides = 2) %>%\n    layer_flatten() %>%\n    layer_dense(units = 128, activation = \"relu\") %>%\n    layer_dense(units = num_classes, activation = \"linear\")\n    \n  \n  \n  model$compile(\n    optimizer = \"adam\",\n    loss = \"mean_squared_error\",\n    metrics = c(\"mse\")\n  )\n  \n  return(model)\n}\n","metadata":{"execution":{"iopub.status.busy":"2023-07-07T16:26:51.372421Z","iopub.execute_input":"2023-07-07T16:26:51.373940Z","iopub.status.idle":"2023-07-07T16:26:51.386957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"split_data <- function(proportion, n_labels_to_consider){\n    embeds <- np$load(\"/kaggle/input/protbert-embeddings-for-cafa5/train_embeddings.npy\")\n    train_split <- sample(nrow(embeds), size = proportion * nrow(embeds))\n    target <- targets(n_labels_to_consider)\n    embeds_tr <- embeds[train_split,]\n    targets_tr <- target[train_split,]\n    embeds_te <- embeds[-train_split,]\n    targets_te <- target[-train_split,]\n    return(list(list(embeds_tr,targets_tr),list(embeds_te,targets_te)))\n\n}","metadata":{"execution":{"iopub.status.busy":"2023-07-09T16:35:14.142466Z","iopub.execute_input":"2023-07-09T16:35:14.153264Z","iopub.status.idle":"2023-07-09T16:35:14.174204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data <- split_data(0.8,600)\nrm(train_terms, train_ids)\ninvisible(gc())","metadata":{"execution":{"iopub.status.busy":"2023-07-09T16:35:19.090940Z","iopub.execute_input":"2023-07-09T16:35:19.092600Z","iopub.status.idle":"2023-07-09T16:36:35.848178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_0_w <- round((1 - mean(train_data[[1]][[2]] == 0)) * 100)\nclass_1_w <- 100 - class_0_w","metadata":{"execution":{"iopub.status.busy":"2023-07-09T16:41:53.772829Z","iopub.execute_input":"2023-07-09T16:41:53.774484Z","iopub.status.idle":"2023-07-09T16:41:54.260945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nAttentionMLPLayer <- function(layer_input ,units , input_dim , activation = 'relu', use_bias = TRUE) {\n  \n    Att_layer <- layer_input %>%\n    layer_reshape(target_shape = c(input_dim, 1)) %>%\n    layer_conv_1d(filters = 3, kernel_size = 3, padding = \"same\", activation = \"relu\", input_shape = c(1, input_dim)) %>%\n    layer_max_pooling_1d(pool_size = 2, strides = 2) %>%\n    layer_conv_1d(filters = 8, kernel_size = 3, padding = \"same\", activation = \"relu\") %>%\n    layer_max_pooling_1d(pool_size = 2, strides = 2) %>%\n    layer_flatten() %>% \n    layer_dense(units = input_dim, activation = \"relu\" )\n    layer_multiply <- layer_multiply(list(layer_input, Att_layer),trainable = TRUE)\n    layer_output <- layer_dense(units = units, activation = activation, use_bias = use_bias)(layer_multiply)\n    return(layer_output)\n    \n}\n\n\nHiddenMLPLayer <- function(input_layer ,out_units, dropout_rate , activation = 'relu', use_bias = TRUE) {\n    \n    layer_Hidd <-  input_layer %>% \n            layer_activation(\"gelu\") %>% \n             layer_batch_normalization()  %>%\n             layer_dropout(rate = dropout_rate) %>% \n             layer_dense(units = out_units)    \n    return(layer_Hidd)\n    \n}\n\nMLP_out <- function(final_layer, num_classes) {\n\n  \n  OUT_layer <- final_layer %>%\n    layer_dense(units = 712) %>%\n    layer_activation(\"tanh\") %>%\n    layer_dense(units = num_classes,activation='linear')\n      \n    return(OUT_layer)\n}\n\ninput_shape <- c(1024L)\nnum_classes  <- 600L\ninput_layer <- layer_input(shape = input_shape)\n\nAtt_layer <- AttentionMLPLayer(layer = input_layer ,units = 400 ,input_dim = 1024 , activation = 'relu', use_bias = TRUE)\nHidd_layer <- HiddenMLPLayer(input_layer ,out_units = 400, dropout_rate = 0.3 , activation = 'relu', use_bias = TRUE)\nlayer_add <- layer_add(list(Att_layer, Hidd_layer), trainable = TRUE) \nlayer_output <- MLP_out(final_layer = layer_add, num_classes = num_classes)\nmodel <- keras_model(inputs = input_layer, outputs = layer_output)","metadata":{"execution":{"iopub.status.busy":"2023-07-09T16:25:50.006850Z","iopub.execute_input":"2023-07-09T16:25:50.023071Z","iopub.status.idle":"2023-07-09T16:25:50.275454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"callbacks_list <- list(\n  callback_early_stopping(monitor = \"val_loss\",\n                          min_delta = 1e-4,\n                          patience = 5,\n                          verbose = 1,\n                          mode = \"max\"),\n  callback_reduce_lr_on_plateau(monitor = \"val_loss\",\n                                factor = 0.5,\n                                patience = 5,\n                                verbose = 1,\n                                mode = \"max\")\n)\nmodel$compile(\n    optimizer = \"adam\",\n    loss = \"mean_squared_error\",\n    metrics = c(\"mse\")\n  )\nclass_0_w <- round((1 - mean(train_data[[1]][[2]] == 0)) * 100)\nclass_1_w <- 100 - class_1_w","metadata":{"execution":{"iopub.status.busy":"2023-07-09T16:29:58.289856Z","iopub.execute_input":"2023-07-09T16:29:58.291639Z","iopub.status.idle":"2023-07-09T16:29:58.411890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history <- model %>% fit(train_data[[1]][[1]],\n                         train_data[[1]][[2]] , \n                         batch_size = 128, \n                         epochs = 5,\n                        validation_data = train_data[[2]],\n                        callbacks = callbacks_list ,\n                        class_weights = list(\"0\"= class_0_w,\"1\"= class_1_w))\nprint(history)\nplot(history)\ncat(\"Best val loss: \", min(history$metrics$val_loss))\nsave_model_hdf5(model, \"SPROF.h5\")\nrm(model,history)\ninvisible(gc())","metadata":{"execution":{"iopub.status.busy":"2023-07-09T16:30:01.119919Z","iopub.execute_input":"2023-07-09T16:30:01.123702Z","iopub.status.idle":"2023-07-09T16:32:14.619692Z"},"trusted":true},"execution_count":null,"outputs":[]}]}