{ "id": 433329, "title": "Selection submission candidates", "url": "/competitions/cafa-5-protein-function-prediction/discussion/433329", "author_name": "", "post_date": "2023-08-21T09:25:48.139252700Z", "votes": 5, "comment_count": 4, "views": 0, "content": "

IIt’s time to select the best submission.
\nTo do this, you need:

\n

Take from testsuperset protein that:
\nHas not anotation in one of 3 subantologies
\nFill this protein during the private timeline.
\nI use the history period from 2023-01-01 to 2023-02-01
\nAnd 2023-02-01 to 2023-03-01
\nAnd calculate the local CAFA metric for all your submission files.

\n

To calculate local cafa metrics, you can use a code like this:
\nhttps://www.kaggle.com/code/alexandervc/pytorch-keras-etc-3-blend-cafa-metric-etc

\n

To find data for this, you can use the dataset:
\nhttps://www.kaggle.com/datasets/liudacheldieva/my-dataset

\n

Notebook:
\nhttps://www.kaggle.com/liudacheldieva/test-superset

\n

How to extract a dataset from testsuperset
\nwith no notation n 1 sub ontology at 2023-01-01 and with
\nannotation at 2023-02-01:
\nYou can get the date by subtracting set from the date 2023-02-01
\nand set on the date 2023-01-01:

\n

date you can see in line 4:

\n
column  GO    AC\n
\n

0 €\u0003}q NaN NaN NaN
\n1 AC A2A6H3; B2RY07; NaN NaN A2A6H3
\n2 DT 23-OCT-2007, integrated into UniProtKB/Sw… NaN NaN NaN
\n3 DT 11-SEP-2007, sequence version 2. NaN NaN NaN
\n4 DT 28-JUN-2023, entry version 97. NaN NaN NaN
\n… … … … …
\n13038322 SQ SEQUENCE 73 AA; 8481 MW; 2B13935E3AEF… NaN NaN NaN
\n13038323 MPALHIEDLP EKEKLKMEVE QLRKEVKLQR QQVSKCSE… NaN NaN NaN
\n13038324 KNPFKEKGSC VIS NaN NaN NaN
\n13038325 // NaN NaN NaN
\n13038326 ræ\u0003 NaN NaN NaN

\n

\"\"

\n

The object has columns: column, AC, GO

\n

\"\"

\n

Task:

\n\n

\"\"

\n

\"\"

\n

file preprocess:
\nVersion 276

\n

f = pd.read_csv('/kaggle/input/cafa-submission/tmp5500.txt' , sep = '\\t', names = [\"column\"], encoding= 'unicode_escape')
\nf = f[f[\"column\"].notnull()]
\nGO = f.copy()

\n

GO['GO']=f[f[\"column\"].str.contains(\"DR GO; GO:\")][\"column\"].str.split(';').str[1].str.strip()
\nGO['datetime']=pd.to_datetime(GO[[\"GO\",\"entry version\",\"AC\",\"DT\"]][\"DT\"])
\nGO['DT']=f[f[\"column\"].str.contains(\"entry version\")][\"column\"].str.split(',').str[1].str.split('\"DT ').str[1]
\nGO['AC']=f[f[\"column\"].str.contains(\"AC \")][\"column\"].str.split(';').str[0].str.split('AC ').str[1].str.strip()

\n

Up to today version of uniprot in dataset Version 276 and tier

\n

code that download uniprot is hear:
\nimport numpy as np # linear algebra
\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
\nimport os
\nfrom tqdm import tqdm
\nimport joblib
\nimport requests, sys, json
\nimport json
\nimport shutil
\nfrom kaggle_secrets import UserSecretsClient
\nimport pathlib

\n

protein from test superset:

\n

а = pd.read_csv('/kaggle/input/cafa-submission/df_test_seq.tsv', sep=\"\\t\" ) #141865
\nа = а[[\"Id\"]]
\nfiles_to_delete = './*.txt'

\n

Change below

\n

#
\nsecrets = UserSecretsClient()
\nos.environ['KAGGLE_USERNAME'] = secrets.get_secret(\"KAGGLE_USERNAME\")
\nos.environ['KAGGLE_KEY'] = secrets.get_secret(\"KAGGLE_KEY\")
\nos.makedirs('/kaggle/dataset/', exist_ok=True)

\n

Change below

\n

meta = dict(
\n id=\"liudacheldieva/my-dataset\",
\n title=\"My brand new dataset\",#dataset name
\n isPrivate=True,
\n licenses=[dict(name=\"other\")]
\n)
\nwith open('/kaggle/dataset/dataset-metadata.json', 'w') as f:
\n json.dump(meta, f)

\n

os.makedirs('/kaggle/dataset/my-directory', exist_ok=True)

\n

start index to load protein to dataset

\n

i=5000 #20000 280000

\n

def get_url(url, **kwargs):
\n response = requests.get(url, **kwargs)
\n if not response.ok:
\n response.raise_for_status()
\n sys.exit()
\n return response

\n

print(1)
\nfor lo in range(0,100):
\n try:
\n uniprot_data_proteins = {}
\n i=i+1
\n start = i
\n for id in list(а[\"Id\"])[start:]:
\n r = get_url(f'https://rest.uniprot.org/unisave/{id}?format=txt')
\n if i%100 ==0 and i>0:
\n print(i)
\n if i%500 ==0 and i>0: #500
\n print(i)
\n joblib.dump(uniprot_data_proteins, f'tmp{i}.txt')
\n uniprot_data_proteins = {}
\n uniprot_data_proteins[str(id)] = r.text
\n i+=1

\n
     i% ==  i>: #\n       dirname, _, filenames  .walk():\n         filename  filenames:\n          (..join(dirname, filename))\n          src = ..join(dirname, filename)\n          dst = f\n          shutil.copy2(src, dst)\n      !kaggle datasets version -p  -m  \n\n      files_list = pathlib.Path(.getcwd()).glob(files_to_delete)\n       file_path  files_list:\n        .(file_path)\n\njoblib.(uniprot_data_proteins, f)\n
\n

except:
\n pass

", "messages": [ { "id": "2400871", "postDate": "08/21/2023 09:25:48", "content": "

IIt’s time to select the best submission.
\nTo do this, you need:

\n

Take from testsuperset protein that:
\nHas not anotation in one of 3 subantologies
\nFill this protein during the private timeline.
\nI use the history period from 2023-01-01 to 2023-02-01
\nAnd 2023-02-01 to 2023-03-01
\nAnd calculate the local CAFA metric for all your submission files.

\n

To calculate local cafa metrics, you can use a code like this:
\nhttps://www.kaggle.com/code/alexandervc/pytorch-keras-etc-3-blend-cafa-metric-etc

\n

To find data for this, you can use the dataset:
\nhttps://www.kaggle.com/datasets/liudacheldieva/my-dataset

\n

Notebook:
\nhttps://www.kaggle.com/liudacheldieva/test-superset

\n

How to extract a dataset from testsuperset
\nwith no notation n 1 sub ontology at 2023-01-01 and with
\nannotation at 2023-02-01:
\nYou can get the date by subtracting set from the date 2023-02-01
\nand set on the date 2023-01-01:

\n

date you can see in line 4:

\n
column  GO    AC\n
\n

0 €\u0003}q NaN NaN NaN
\n1 AC A2A6H3; B2RY07; NaN NaN A2A6H3
\n2 DT 23-OCT-2007, integrated into UniProtKB/Sw… NaN NaN NaN
\n3 DT 11-SEP-2007, sequence version 2. NaN NaN NaN
\n4 DT 28-JUN-2023, entry version 97. NaN NaN NaN
\n… … … … …
\n13038322 SQ SEQUENCE 73 AA; 8481 MW; 2B13935E3AEF… NaN NaN NaN
\n13038323 MPALHIEDLP EKEKLKMEVE QLRKEVKLQR QQVSKCSE… NaN NaN NaN
\n13038324 KNPFKEKGSC VIS NaN NaN NaN
\n13038325 // NaN NaN NaN
\n13038326 ræ\u0003 NaN NaN NaN

\n

\"\"

\n

The object has columns: column, AC, GO

\n

\"\"

\n

Task:

\n\n

\"\"

\n

\"\"

\n

file preprocess:
\nVersion 276

\n

f = pd.read_csv('/kaggle/input/cafa-submission/tmp5500.txt' , sep = '\\t', names = [\"column\"], encoding= 'unicode_escape')
\nf = f[f[\"column\"].notnull()]
\nGO = f.copy()

\n

GO['GO']=f[f[\"column\"].str.contains(\"DR GO; GO:\")][\"column\"].str.split(';').str[1].str.strip()
\nGO['datetime']=pd.to_datetime(GO[[\"GO\",\"entry version\",\"AC\",\"DT\"]][\"DT\"])
\nGO['DT']=f[f[\"column\"].str.contains(\"entry version\")][\"column\"].str.split(',').str[1].str.split('\"DT ').str[1]
\nGO['AC']=f[f[\"column\"].str.contains(\"AC \")][\"column\"].str.split(';').str[0].str.split('AC ').str[1].str.strip()

\n

Up to today version of uniprot in dataset Version 276 and tier

\n

code that download uniprot is hear:
\nimport numpy as np # linear algebra
\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
\nimport os
\nfrom tqdm import tqdm
\nimport joblib
\nimport requests, sys, json
\nimport json
\nimport shutil
\nfrom kaggle_secrets import UserSecretsClient
\nimport pathlib

\n

protein from test superset:

\n

а = pd.read_csv('/kaggle/input/cafa-submission/df_test_seq.tsv', sep=\"\\t\" ) #141865
\nа = а[[\"Id\"]]
\nfiles_to_delete = './*.txt'

\n

Change below

\n

#
\nsecrets = UserSecretsClient()
\nos.environ['KAGGLE_USERNAME'] = secrets.get_secret(\"KAGGLE_USERNAME\")
\nos.environ['KAGGLE_KEY'] = secrets.get_secret(\"KAGGLE_KEY\")
\nos.makedirs('/kaggle/dataset/', exist_ok=True)

\n

Change below

\n

meta = dict(
\n id=\"liudacheldieva/my-dataset\",
\n title=\"My brand new dataset\",#dataset name
\n isPrivate=True,
\n licenses=[dict(name=\"other\")]
\n)
\nwith open('/kaggle/dataset/dataset-metadata.json', 'w') as f:
\n json.dump(meta, f)

\n

os.makedirs('/kaggle/dataset/my-directory', exist_ok=True)

\n

start index to load protein to dataset

\n

i=5000 #20000 280000

\n

def get_url(url, **kwargs):
\n response = requests.get(url, **kwargs)
\n if not response.ok:
\n response.raise_for_status()
\n sys.exit()
\n return response

\n

print(1)
\nfor lo in range(0,100):
\n try:
\n uniprot_data_proteins = {}
\n i=i+1
\n start = i
\n for id in list(а[\"Id\"])[start:]:
\n r = get_url(f'https://rest.uniprot.org/unisave/{id}?format=txt')
\n if i%100 ==0 and i>0:
\n print(i)
\n if i%500 ==0 and i>0: #500
\n print(i)
\n joblib.dump(uniprot_data_proteins, f'tmp{i}.txt')
\n uniprot_data_proteins = {}
\n uniprot_data_proteins[str(id)] = r.text
\n i+=1

\n
     i% ==  i>: #\n       dirname, _, filenames  .walk():\n         filename  filenames:\n          (..join(dirname, filename))\n          src = ..join(dirname, filename)\n          dst = f\n          shutil.copy2(src, dst)\n      !kaggle datasets version -p  -m  \n\n      files_list = pathlib.Path(.getcwd()).glob(files_to_delete)\n       file_path  files_list:\n        .(file_path)\n\njoblib.(uniprot_data_proteins, f)\n
\n

except:
\n pass

", "rawMarkdown": "IIt’s time to select the best submission.\nTo do this, you need:\n\nTake from testsuperset protein that:\nHas not anotation in one of 3 subantologies\nFill this protein during the private timeline.\nI use the history period from 2023-01-01 to 2023-02-01\nAnd 2023-02-01 to 2023-03-01\nAnd calculate the local CAFA metric for all your submission files.\n\nTo calculate local cafa metrics, you can use a code like this:\nhttps://www.kaggle.com/code/alexandervc/pytorch-keras-etc-3-blend-cafa-metric-etc\n\nTo find data for this, you can use the dataset:\nhttps://www.kaggle.com/datasets/liudacheldieva/my-dataset\n\nNotebook:\nhttps://www.kaggle.com/liudacheldieva/test-superset\n\nHow to extract a dataset from testsuperset\nwith no notation n 1 sub ontology at 2023-01-01 and with \nannotation at 2023-02-01:\nYou can get the date by subtracting set from the date 2023-02-01 \nand set on the date 2023-01-01:\n\ndate you can see in line 4:\n\n\n\tcolumn\tGO\tDT\tAC\n0\t€\u0003}q\tNaN\tNaN\tNaN\n1\tAC A2A6H3; B2RY07;\tNaN\tNaN\tA2A6H3\n2\tDT 23-OCT-2007, integrated into UniProtKB/Sw...\tNaN\tNaN\tNaN\n3\tDT 11-SEP-2007, sequence version 2.\tNaN\tNaN\tNaN\n4\t**DT 28-JUN-2023**, entry version 97.\tNaN\tNaN\tNaN\n...\t...\t...\t...\t...\n13038322\tSQ SEQUENCE 73 AA; 8481 MW; 2B13935E3AEF...\tNaN\tNaN\tNaN\n13038323\tMPALHIEDLP EKEKLKMEVE QLRKEVKLQR QQVSKCSE...\tNaN\tNaN\tNaN\n13038324\tKNPFKEKGSC VIS\tNaN\tNaN\tNaN\n13038325\t//\tNaN\tNaN\tNaN\n13038326\træ\u0003\tNaN\tNaN\tNaN\n\n![](https://www.googleapis.com/download/storage/v1/b/kaggle-forum-message-attachments/o/inbox%2F4172517%2F9e25c1a4d0c58ef29f3c27bf12afb46c%2F2023-08-21%20%2012.30.51.png?generation=1692614485431109&alt=media)\n\nThe object has columns: column, AC, GO\n\n![](https://www.googleapis.com/download/storage/v1/b/kaggle-forum-message-attachments/o/inbox%2F4172517%2Fb122fa78f22cadc59d7482391c00cd97%2F2023-08-21%20%2013.39.19.png?generation=1692615405041785&alt=media)\n\nTask:\n* Create a new datetime column that is equal to the date from the column column.\n* Set the filter datetime column from 2023-01-01 to 2023-02-01\n* Create a new branch column that is equal to 'CCO' MFO' 'BPO' .\n* Set a filter on the number of GOs on the date 2023-01-01 in one of the 'CCO' MFO' 'BPO' ==0\n* Set a filter on the number of GOs on the date 2023-02-01 in one of the 'CCO' MFO' 'BPO' >0\n* Save to private2023-01-01_2023-02-01.tsv\n* Calculate competition metric for proteins and one of 'CCO' MFO' 'BPO' From submission.tsv AND one of 'CCO' MFO' 'BPO'\n* Compare competition metrics for all your submission.tsv and pick three for Private\n\n![](https://www.googleapis.com/download/storage/v1/b/kaggle-forum-message-attachments/o/inbox%2F4172517%2F005f3b6baf246c90e792e27d3bfe3128%2F2023-08-21%20%2012.30.51.png?generation=1692610603508016&alt=media)\n\n![](https://www.googleapis.com/download/storage/v1/b/kaggle-forum-message-attachments/o/inbox%2F4172517%2Fe17968b7914aa9d85dada22e35ff710d%2F2023-08-21%20%2012.33.58.png?generation=1692610623457422&alt=media)\n\nfile preprocess:\nVersion 276 \n\nf = pd.read_csv('/kaggle/input/cafa-submission/tmp5500.txt' , sep = '\\t', names = [\"column\"], encoding= 'unicode_escape')\nf = f[f[\"column\"].notnull()]\nGO = f.copy()\n\nGO['GO']=f[f[\"column\"].str.contains(\"DR GO; GO:\")][\"column\"].str.split(';').str[1].str.strip()\nGO['datetime']=pd.to_datetime(GO[[\"GO\",\"entry version\",\"AC\",\"DT\"]][\"DT\"])\nGO['DT']=f[f[\"column\"].str.contains(\"entry version\")][\"column\"].str.split(',').str[1].str.split('\"DT ').str[1]\nGO['AC']=f[f[\"column\"].str.contains(\"AC \")][\"column\"].str.split(';').str[0].str.split('AC ').str[1].str.strip() \n\nUp to today version of uniprot in dataset Version 276 and tier\n\ncode that download uniprot is hear:\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nfrom tqdm import tqdm\nimport joblib\nimport requests, sys, json\nimport json\nimport shutil\nfrom kaggle_secrets import UserSecretsClient\nimport pathlib\n#protein from test superset:\nа = pd.read_csv('/kaggle/input/cafa-submission/df_test_seq.tsv', sep=\"\\t\" ) #141865\nа = а[[\"Id\"]]\nfiles_to_delete = './*.txt' \n# Change below\n#\nsecrets = UserSecretsClient()\nos.environ['KAGGLE_USERNAME'] = secrets.get_secret(\"KAGGLE_USERNAME\")\nos.environ['KAGGLE_KEY'] = secrets.get_secret(\"KAGGLE_KEY\")\nos.makedirs('/kaggle/dataset/', exist_ok=True)\n# Change below\nmeta = dict(\n id=\"liudacheldieva/my-dataset\",\n title=\"My brand new dataset\",#dataset name\n isPrivate=True,\n licenses=[dict(name=\"other\")]\n)\nwith open('/kaggle/dataset/dataset-metadata.json', 'w') as f:\n json.dump(meta, f)\n\nos.makedirs('/kaggle/dataset/my-directory', exist_ok=True)\n\n#start index to load protein to dataset\ni=5000 #20000 280000\n\ndef get_url(url, **kwargs):\n response = requests.get(url, **kwargs)\n if not response.ok:\n response.raise_for_status()\n sys.exit()\n return response\n\nprint(1)\nfor lo in range(0,100):\n try:\n uniprot_data_proteins = {}\n i=i+1\n start = i\n for id in list(а[\"Id\"])[start:]:\n r = get_url(f'https://rest.uniprot.org/unisave/{id}?format=txt') \n if i%100 ==0 and i>0: \n print(i) \n if i%500 ==0 and i>0: #500\n print(i)\n joblib.dump(uniprot_data_proteins, f'tmp{i}.txt')\n uniprot_data_proteins = {}\n uniprot_data_proteins[str(id)] = r.text \n i+=1\n \n if i%1000 ==0 and i>0: #5000\n for dirname, _, filenames in os.walk('/kaggle/working'):\n for filename in filenames:\n print(os.path.join(dirname, filename))\n src = os.path.join(dirname, filename)\n dst = f'/kaggle/dataset/{filename}'\n shutil.copy2(src, dst)\n !kaggle datasets version -p \"/kaggle/dataset\" -m \"Updated via notebook\" --dir-mode zip\n \n files_list = pathlib.Path(os.getcwd()).glob(files_to_delete)\n for file_path in files_list:\n os.remove(file_path)\n \n joblib.dump(uniprot_data_proteins, f'tmp{i}.txt')\n except:\n pass", "votes": null }, { "id": "2400903", "postDate": "08/21/2023 09:59:13", "content": "

It may be a good idea to make a notebook out of this post.

", "rawMarkdown": "It may be a good idea to make a notebook out of this post.", "votes": null }, { "id": "2400914", "postDate": "08/21/2023 10:05:39", "content": "

You comment on every discussion, but it brings nothing.

", "rawMarkdown": "You comment on every discussion, but it brings nothing.", "votes": null }, { "id": "2400974", "postDate": "08/21/2023 10:58:38", "content": "

I think it's good idea to make a notebook out of this post :)

\n

I create a notebook:
\n https://www.kaggle.com/liudacheldieva/test-superset

", "rawMarkdown": "I think it's good idea to make a notebook out of this post :)\n\nI create a notebook:\n https://www.kaggle.com/liudacheldieva/test-superset", "votes": null }, { "id": "2401001", "postDate": "08/21/2023 11:22:56", "content": "

Comments by Tilii are very valuable and interesting !
\nMay be even the most valuable in all bioinformatics related competitions.
\nAt least for me.

", "rawMarkdown": "Comments by Tilii are very valuable and interesting ! \nMay be even the most valuable in all bioinformatics related competitions. \nAt least for me.", "votes": null } ], "comments": [ { "id": 2400903, "author_name": "tilii7", "author_url": "", "post_date": "08/21/2023 09:59:13", "content": "

It may be a good idea to make a notebook out of this post.

", "votes": null, "replies": [ { "id": 2400914, "author_name": "bejeweled", "author_url": "", "post_date": "08/21/2023 10:05:39", "content": "

You comment on every discussion, but it brings nothing.

", "votes": null, "replies": [ { "id": 2400974, "author_name": "", "author_url": "", "post_date": "08/21/2023 10:58:38", "content": "

I think it's good idea to make a notebook out of this post :)

\n

I create a notebook:
\n https://www.kaggle.com/liudacheldieva/test-superset

", "votes": null, "replies": [] }, { "id": 2401001, "author_name": "alexandervc", "author_url": "", "post_date": "08/21/2023 11:22:56", "content": "

Comments by Tilii are very valuable and interesting !
\nMay be even the most valuable in all bioinformatics related competitions.
\nAt least for me.

", "votes": null, "replies": [] } ] } ] } ], "raw_markdown_by_id": { "2400871": "IIt’s time to select the best submission.\nTo do this, you need:\n\nTake from testsuperset protein that:\nHas not anotation in one of 3 subantologies\nFill this protein during the private timeline.\nI use the history period from 2023-01-01 to 2023-02-01\nAnd 2023-02-01 to 2023-03-01\nAnd calculate the local CAFA metric for all your submission files.\n\nTo calculate local cafa metrics, you can use a code like this:\nhttps://www.kaggle.com/code/alexandervc/pytorch-keras-etc-3-blend-cafa-metric-etc\n\nTo find data for this, you can use the dataset:\nhttps://www.kaggle.com/datasets/liudacheldieva/my-dataset\n\nNotebook:\nhttps://www.kaggle.com/liudacheldieva/test-superset\n\nHow to extract a dataset from testsuperset\nwith no notation n 1 sub ontology at 2023-01-01 and with \nannotation at 2023-02-01:\nYou can get the date by subtracting set from the date 2023-02-01 \nand set on the date 2023-01-01:\n\ndate you can see in line 4:\n\n\n\tcolumn\tGO\tDT\tAC\n0\t€\u0003}q\tNaN\tNaN\tNaN\n1\tAC A2A6H3; B2RY07;\tNaN\tNaN\tA2A6H3\n2\tDT 23-OCT-2007, integrated into UniProtKB/Sw...\tNaN\tNaN\tNaN\n3\tDT 11-SEP-2007, sequence version 2.\tNaN\tNaN\tNaN\n4\t**DT 28-JUN-2023**, entry version 97.\tNaN\tNaN\tNaN\n...\t...\t...\t...\t...\n13038322\tSQ SEQUENCE 73 AA; 8481 MW; 2B13935E3AEF...\tNaN\tNaN\tNaN\n13038323\tMPALHIEDLP EKEKLKMEVE QLRKEVKLQR QQVSKCSE...\tNaN\tNaN\tNaN\n13038324\tKNPFKEKGSC VIS\tNaN\tNaN\tNaN\n13038325\t//\tNaN\tNaN\tNaN\n13038326\træ\u0003\tNaN\tNaN\tNaN\n\n![](https://www.googleapis.com/download/storage/v1/b/kaggle-forum-message-attachments/o/inbox%2F4172517%2F9e25c1a4d0c58ef29f3c27bf12afb46c%2F2023-08-21%20%2012.30.51.png?generation=1692614485431109&alt=media)\n\nThe object has columns: column, AC, GO\n\n![](https://www.googleapis.com/download/storage/v1/b/kaggle-forum-message-attachments/o/inbox%2F4172517%2Fb122fa78f22cadc59d7482391c00cd97%2F2023-08-21%20%2013.39.19.png?generation=1692615405041785&alt=media)\n\nTask:\n* Create a new datetime column that is equal to the date from the column column.\n* Set the filter datetime column from 2023-01-01 to 2023-02-01\n* Create a new branch column that is equal to 'CCO' MFO' 'BPO' .\n* Set a filter on the number of GOs on the date 2023-01-01 in one of the 'CCO' MFO' 'BPO' ==0\n* Set a filter on the number of GOs on the date 2023-02-01 in one of the 'CCO' MFO' 'BPO' >0\n* Save to private2023-01-01_2023-02-01.tsv\n* Calculate competition metric for proteins and one of 'CCO' MFO' 'BPO' From submission.tsv AND one of 'CCO' MFO' 'BPO'\n* Compare competition metrics for all your submission.tsv and pick three for Private\n\n![](https://www.googleapis.com/download/storage/v1/b/kaggle-forum-message-attachments/o/inbox%2F4172517%2F005f3b6baf246c90e792e27d3bfe3128%2F2023-08-21%20%2012.30.51.png?generation=1692610603508016&alt=media)\n\n![](https://www.googleapis.com/download/storage/v1/b/kaggle-forum-message-attachments/o/inbox%2F4172517%2Fe17968b7914aa9d85dada22e35ff710d%2F2023-08-21%20%2012.33.58.png?generation=1692610623457422&alt=media)\n\nfile preprocess:\nVersion 276 \n\nf = pd.read_csv('/kaggle/input/cafa-submission/tmp5500.txt' , sep = '\\t', names = [\"column\"], encoding= 'unicode_escape')\nf = f[f[\"column\"].notnull()]\nGO = f.copy()\n\nGO['GO']=f[f[\"column\"].str.contains(\"DR GO; GO:\")][\"column\"].str.split(';').str[1].str.strip()\nGO['datetime']=pd.to_datetime(GO[[\"GO\",\"entry version\",\"AC\",\"DT\"]][\"DT\"])\nGO['DT']=f[f[\"column\"].str.contains(\"entry version\")][\"column\"].str.split(',').str[1].str.split('\"DT ').str[1]\nGO['AC']=f[f[\"column\"].str.contains(\"AC \")][\"column\"].str.split(';').str[0].str.split('AC ').str[1].str.strip() \n\nUp to today version of uniprot in dataset Version 276 and tier\n\ncode that download uniprot is hear:\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nfrom tqdm import tqdm\nimport joblib\nimport requests, sys, json\nimport json\nimport shutil\nfrom kaggle_secrets import UserSecretsClient\nimport pathlib\n#protein from test superset:\nа = pd.read_csv('/kaggle/input/cafa-submission/df_test_seq.tsv', sep=\"\\t\" ) #141865\nа = а[[\"Id\"]]\nfiles_to_delete = './*.txt' \n# Change below\n#\nsecrets = UserSecretsClient()\nos.environ['KAGGLE_USERNAME'] = secrets.get_secret(\"KAGGLE_USERNAME\")\nos.environ['KAGGLE_KEY'] = secrets.get_secret(\"KAGGLE_KEY\")\nos.makedirs('/kaggle/dataset/', exist_ok=True)\n# Change below\nmeta = dict(\n id=\"liudacheldieva/my-dataset\",\n title=\"My brand new dataset\",#dataset name\n isPrivate=True,\n licenses=[dict(name=\"other\")]\n)\nwith open('/kaggle/dataset/dataset-metadata.json', 'w') as f:\n json.dump(meta, f)\n\nos.makedirs('/kaggle/dataset/my-directory', exist_ok=True)\n\n#start index to load protein to dataset\ni=5000 #20000 280000\n\ndef get_url(url, **kwargs):\n response = requests.get(url, **kwargs)\n if not response.ok:\n response.raise_for_status()\n sys.exit()\n return response\n\nprint(1)\nfor lo in range(0,100):\n try:\n uniprot_data_proteins = {}\n i=i+1\n start = i\n for id in list(а[\"Id\"])[start:]:\n r = get_url(f'https://rest.uniprot.org/unisave/{id}?format=txt') \n if i%100 ==0 and i>0: \n print(i) \n if i%500 ==0 and i>0: #500\n print(i)\n joblib.dump(uniprot_data_proteins, f'tmp{i}.txt')\n uniprot_data_proteins = {}\n uniprot_data_proteins[str(id)] = r.text \n i+=1\n \n if i%1000 ==0 and i>0: #5000\n for dirname, _, filenames in os.walk('/kaggle/working'):\n for filename in filenames:\n print(os.path.join(dirname, filename))\n src = os.path.join(dirname, filename)\n dst = f'/kaggle/dataset/{filename}'\n shutil.copy2(src, dst)\n !kaggle datasets version -p \"/kaggle/dataset\" -m \"Updated via notebook\" --dir-mode zip\n \n files_list = pathlib.Path(os.getcwd()).glob(files_to_delete)\n for file_path in files_list:\n os.remove(file_path)\n \n joblib.dump(uniprot_data_proteins, f'tmp{i}.txt')\n except:\n pass", "2400903": "It may be a good idea to make a notebook out of this post.", "2400914": "You comment on every discussion, but it brings nothing.", "2400974": "I think it's good idea to make a notebook out of this post :)\n\nI create a notebook:\n https://www.kaggle.com/liudacheldieva/test-superset", "2401001": "Comments by Tilii are very valuable and interesting ! \nMay be even the most valuable in all bioinformatics related competitions. \nAt least for me." }, "source": "meta" }