Mercurial > repos > pimarin > data_manager_bakta
changeset 19:c90380f8bbbc draft
planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_build_bakta_database commit 8eb28a93e6f2688bb2f3f85aea0389e1b1148816-dirty
| author | pimarin |
|---|---|
| date | Fri, 13 Jan 2023 14:11:30 +0000 |
| parents | 04bee0f935a2 |
| children | b165d8c97c15 |
| files | data_manager/bakta_build_database.py data_manager/bakta_build_database.xml test-data/bakta_test.loc test-data/bakta_test_data_manager.json test-data/bakta_test_data_manager_test2.json test-data/db-versions.json |
| diffstat | 6 files changed, 61 insertions(+), 31 deletions(-) [+] |
line wrap: on
line diff
--- a/data_manager/bakta_build_database.py Wed Nov 09 15:49:56 2022 +0000 +++ b/data_manager/bakta_build_database.py Fri Jan 13 14:11:30 2023 +0000 @@ -3,12 +3,10 @@ import json import os import sys -# import subprocess import tarfile from datetime import datetime from pathlib import Path -# implement pip as a subprocess: -# subprocess.check_call([sys.executable, '-m', 'pip', 'install', 'requests']) + import requests @@ -21,7 +19,8 @@ def __init__(self, data_table_name="bakta_database", db_name=Path.cwd().joinpath("db"), - db_version="latest"): + db_version="latest", + test_mode=False): self.bakta_table_list = None self.db_url = None self.data_table_entry = None @@ -29,6 +28,8 @@ self.db_name = db_name self.db_version = db_version self.DB_VERSIONS_URL = 'https://raw.githubusercontent.com/oschwengers/bakta/master/db-versions.json' + self.DB_TEST_URL = 'https://zenodo.org/record/7360542/files/db-versions.json' + self.test_mode = test_mode def get_data_table_format(self): """ @@ -44,8 +45,10 @@ def fetch_db_versions(self, db_version="latest"): """ - List bakta database info depending of the db_version selected + List bakta database info related to the db_version selected """ + if self.test_mode is True: + self.DB_VERSIONS_URL = self.DB_TEST_URL try: with requests.get(self.DB_VERSIONS_URL) as resp: versions = json.loads(resp.content) @@ -59,16 +62,6 @@ db_date_list.append(datetime.strptime(db_dic["date"], '%Y-%m-%d').date()) filtered_version = max(versions, key=lambda x: x['date']) - elif db_version == "test": - filtered_version = {"date": "date_test", - "major": "0", - "minor": "0", - "doi": "10.5281/zenodo.7197299", - "record": "7197299", - "md5": "8b0250c17078742fc12207d4efb0fc1a", - "software-min": {"major": "0", - "minor": "0"} - } else: filtered_version = None for item in versions: @@ -83,15 +76,15 @@ self.db_version = db_version return filtered_version - def get_data_manager(self, bakta_database_info, output_path): + def get_data_manager(self, bakta_database_info): self.bakta_table_list = self.get_data_table_format() bakta_value = f"V{bakta_database_info['major']}." \ f"{bakta_database_info['minor']}_" \ f"{bakta_database_info['date']}" tool_version = str(f"{bakta_database_info['software-min']['major']}." f"{bakta_database_info['software-min']['minor']}") - data_info = dict(value=bakta_database_info['record'], - dbkey=bakta_value, + data_info = dict(value=bakta_value, + dbkey=bakta_database_info['record'], bakta_version=tool_version, path="db") self.bakta_table_list["data_tables"][self.data_table_name] = [data_info] @@ -108,13 +101,15 @@ def __init__(self, db_dir=Path.cwd(), db_name="bakta", - tarball_name="db.tar.gz"): + tarball_name="db.tar.gz", + test_mode=False): super().__init__() self.md5 = None self.db_dir = db_dir self.db_name = db_name self.tarball_name = tarball_name self.tarball_path = None + self.test_mode = test_mode def download(self): self.db_name = f'{self.db_name}_{self.db_version}' @@ -212,20 +207,13 @@ def main(): all_args = parse_arguments() - with open(all_args.data_manager_json) as fh: params = json.load(fh) target_dir = params['output_data'][0]['extra_files_path'] os.makedirs(target_dir) # init the class to download bakta db - bakta_upload = InstallBaktaDatabase() - # extract the version - if all_args.test is True: - bakta_db = bakta_upload.fetch_db_versions( - db_version="test") - else: - bakta_db = bakta_upload.fetch_db_versions( - db_version=all_args.database_version) + bakta_upload = InstallBaktaDatabase(test_mode=all_args.test) + bakta_db = bakta_upload.fetch_db_versions(db_version=all_args.database_version) # update the path for galaxy bakta_upload.db_dir = target_dir # download the database @@ -235,7 +223,7 @@ # untar db bakta_upload.untar() # make the data_manager metadata - bakta_data_manager = bakta_upload.get_data_manager(bakta_database_info=bakta_db, output_path=target_dir) + bakta_data_manager = bakta_upload.get_data_manager(bakta_database_info=bakta_db) with open(all_args.data_manager_json, 'w') as fh: json.dump(bakta_data_manager, fh, sort_keys=True)
--- a/data_manager/bakta_build_database.xml Wed Nov 09 15:49:56 2022 +0000 +++ b/data_manager/bakta_build_database.xml Fri Jan 13 14:11:30 2023 +0000 @@ -27,11 +27,18 @@ <data name="output_file" format="data_manager_json"/> </outputs> <tests> - <!-- Test download --> + <!-- Test 1 with version 1.0 --> <test expect_num_outputs="1"> <param name="test_data_manager" value="--test"/> + <param name="database_select" value="1.0"/> <output name="output_file" value="bakta_test_data_manager.json" /> </test> + <!-- Test 2 with the latest option --> + <test expect_num_outputs="1"> + <param name="test_data_manager" value="--test"/> + <param name="database_select" value="latest"/> + <output name="output_file" value="bakta_test_data_manager_test2.json" /> + </test> </tests> <help><![CDATA[ Download specific version of Bakta database <https://github.com/oschwengers/bakta#database>
--- a/test-data/bakta_test.loc Wed Nov 09 15:49:56 2022 +0000 +++ b/test-data/bakta_test.loc Fri Jan 13 14:11:30 2023 +0000 @@ -5,3 +5,11 @@ # # for example 7197299 V0.0_date_test 0.0 ${__HERE__} +7197299 V1.0_2022-10-12 1.4 /tmp/tmpxrkfnuec/galaxy-dev/tool-data/bakta_database/7197299 +7360139 V2.0_2022-11-25 1.5 /tmp/tmpxrkfnuec/galaxy-dev/tool-data/bakta_database/7360139 +7197299 V1.0_2022-10-12 1.4 /tmp/tmpwe9n4gyg/galaxy-dev/tool-data/bakta_database/7197299 +7360139 V2.0_2022-11-25 1.5 /tmp/tmpwe9n4gyg/galaxy-dev/tool-data/bakta_database/7360139 +V1.0_2022-10-12 7197299 1.4 /tmp/tmpw5w3lbzc/galaxy-dev/tool-data/bakta_database/V1.0_2022-10-12 +V2.0_2022-11-25 7360139 1.5 /tmp/tmpw5w3lbzc/galaxy-dev/tool-data/bakta_database/V2.0_2022-11-25 +V1.0_2022-10-12 7197299 1.4 /tmp/tmpi5flu1x0/galaxy-dev/tool-data/bakta_database/V1.0_2022-10-12 +V2.0_2022-11-25 7360139 1.5 /tmp/tmpi5flu1x0/galaxy-dev/tool-data/bakta_database/V2.0_2022-11-25
--- a/test-data/bakta_test_data_manager.json Wed Nov 09 15:49:56 2022 +0000 +++ b/test-data/bakta_test_data_manager.json Fri Jan 13 14:11:30 2023 +0000 @@ -1,1 +1,1 @@ -{"data_tables": {"bakta_database": [{"bakta_version": "0.0", "dbkey": "V0.0_date_test", "path": "db", "value": "7197299"}]}} \ No newline at end of file +{"data_tables": {"bakta_database": [{"bakta_version": "1.4", "dbkey": "7197299", "path": "db", "value": "V1.0_2022-10-12"}]}} \ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/bakta_test_data_manager_test2.json Fri Jan 13 14:11:30 2023 +0000 @@ -0,0 +1,1 @@ +{"data_tables": {"bakta_database": [{"bakta_version": "1.5", "dbkey": "7360139", "path": "db", "value": "V2.0_2022-11-25"}]}} \ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/db-versions.json Fri Jan 13 14:11:30 2023 +0000 @@ -0,0 +1,26 @@ +[ + { + "date": "2022-10-12", + "major": 1, + "minor": 0, + "doi": "10.5281/zenodo.7197299", + "record": "7197299", + "md5": "8b0250c17078742fc12207d4efb0fc1a", + "software-min": { + "major": 1, + "minor": 4 + } + }, + { + "date": "2022-11-25", + "major": 2, + "minor": 0, + "doi": "10.5281/zenodo.7360139", + "record": "7360139", + "md5": "ebdb799a6bd97e56ca359db781ab8bab", + "software-min": { + "major": 1, + "minor": 5 + } + } +]