Mercurial > repos > ieguinoa > ena_upload
comparison extract_tables.py @ 25:758e5bc32bbb draft
Uploaded
| author | ieguinoa |
|---|---|
| date | Thu, 19 Nov 2020 10:21:04 +0000 |
| parents | d7a88a8a8f5f |
| children |
comparison
equal
deleted
inserted
replaced
| 24:6c82d674873b | 25:758e5bc32bbb |
|---|---|
| 1 import argparse | 1 import argparse |
| 2 import json | 2 import json |
| 3 import os | |
| 4 import pathlib | 3 import pathlib |
| 5 | |
| 6 from datetime import datetime | 4 from datetime import datetime |
| 7 | 5 |
| 6 FILE_FORMAT = 'fastq' | |
| 7 | |
| 8 parser = argparse.ArgumentParser() | 8 parser = argparse.ArgumentParser() |
| 9 parser.add_argument('--studies',dest='studies_json_path', required=True) | 9 parser.add_argument('--studies', dest='studies_json_path', required=True) |
| 10 parser.add_argument('--out_dir',dest='out_path', required=True) | 10 parser.add_argument('--out_dir', dest='out_path', required=True) |
| 11 parser.add_argument('--action', dest='action', required=True) | |
| 11 args = parser.parse_args() | 12 args = parser.parse_args() |
| 12 | 13 |
| 14 with open(args.studies_json_path, 'r') as studies_json_file: | |
| 15 studies_dict = json.load(studies_json_file) | |
| 16 studies_table = open(pathlib.Path(args.out_path) / 'studies.tsv', 'w') | |
| 17 studies_table.write('\t'.join(['alias', 'status', 'accession', 'title', 'study_type', | |
| 18 'study_abstract', 'pubmed_id', 'submission_date']) + '\n') | |
| 19 samples_table = open(pathlib.Path(args.out_path) / 'samples.tsv', 'w') | |
| 20 experiments_table = open(pathlib.Path(args.out_path) / 'experiments.tsv', 'w') | |
| 21 experiments_table.write('\t'.join(['alias', 'status', 'accession', 'title', 'study_alias', | |
| 22 'sample_alias', 'design_description', 'library_name', | |
| 23 'library_strategy', 'library_source', 'library_selection', | |
| 24 'library_layout', 'insert_size', | |
| 25 'library_construction_protocol', 'platform', 'instrument_model', | |
| 26 'submission_date']) + '\n') | |
| 27 runs_table = open(pathlib.Path(args.out_path) / 'runs.tsv', 'w') | |
| 28 runs_table.write('\t'.join(['alias', 'status', 'accession', 'experiment_alias', 'file_name', | |
| 29 'file_format', 'file_checksum', 'submission_date']) + '\n') | |
| 13 | 30 |
| 14 with open(args.studies_json_path,'r') as studies_json_file: | 31 action = args.action |
| 15 studies_dict = json.load(studies_json_file) | |
| 16 | |
| 17 studies_table = open(pathlib.Path(args.out_path) / 'studies.tsv', 'w') | |
| 18 studies_table.write('\t'.join(['alias','status','accession','title','study_type','study_abstract','pubmed_id','submission_date']) + '\n') | |
| 19 | |
| 20 samples_table = open(pathlib.Path(args.out_path) / 'samples.tsv', 'w') | |
| 21 | |
| 22 experiments_table = open(pathlib.Path(args.out_path) / 'experiments.tsv', 'w') | |
| 23 experiments_table.write('\t'.join(['alias','status','accession','title','study_alias','sample_alias','design_description','library_name','library_strategy','library_source','library_selection','library_layout','insert_size','library_construction_protocol','platform','instrument_model','submission_date'])+ '\n') | |
| 24 | |
| 25 runs_table = open(pathlib.Path(args.out_path) / 'runs.tsv', 'w') | |
| 26 runs_table.write('\t'.join(['alias','status','accession','experiment_alias','file_name','file_format','file_checksum','submission_date'])+ '\n') | |
| 27 | |
| 28 action = 'add' | |
| 29 viral_submission = False | |
| 30 | 32 |
| 31 dt_oobj = datetime.now(tz=None) | 33 dt_oobj = datetime.now(tz=None) |
| 32 timestamp = dt_oobj.strftime("%Y%m%d_%H:%M:%S") | 34 timestamp = dt_oobj.strftime("%Y%m%d_%H:%M:%S") |
| 33 for study_index, study in enumerate(studies_dict): | 35 for study_index, study in enumerate(studies_dict): |
| 34 study_alias = 'study_'+str(study_index)+'_'+timestamp | 36 study_alias = 'study_' + str(study_index) + '_' + timestamp |
| 35 studies_table.write('\t'.join([study_alias,action,'ENA_accession',study['title'], study['type'],study['abstract'],study['pubmed_id'],'ENA_submission_data'])) | 37 studies_table.write('\t'.join([study_alias, action, 'ENA_accession', study['title'], |
| 36 if "geo_location" in study['samples'][0].keys(): # sample belongs to a viral sample | 38 study['type'], study['abstract'], study['pubmed_id'], |
| 37 samples_table.write('\t'.join(['alias','status','accession','title','scientific_name','taxon_id','sample_description','collection_date','geographic_location','host_common_name','host_subject_id','host_health_state','host_sex','host_scientific_name','collector_name','collecting_institution','isolate','submission_date']) + '\n') | 39 'ENA_submission_data'])) |
| 40 if "geo_location" in study['samples'][0].keys(): # sample belongs to a viral sample | |
| 41 samples_table.write('\t'.join(['alias', 'status', 'accession', 'title', 'scientific_name', | |
| 42 'taxon_id', 'sample_description', 'collection_date', | |
| 43 'geographic_location', 'host_common_name', 'host_subject_id', | |
| 44 'host_health_state', 'host_sex', 'host_scientific_name', | |
| 45 'collector_name', 'collecting_institution', 'isolate', | |
| 46 'submission_date']) + '\n') | |
| 38 else: | 47 else: |
| 39 samples_table.write('\t'.join(['alias','status','accession','title','scientific_name','taxon_id','sample_description','submission_date']) + '\n') | 48 samples_table.write('\t'.join(['alias', 'status', 'accession', 'title', 'scientific_name', |
| 40 for sample_index,sample in enumerate(study['samples']): | 49 'taxon_id', 'sample_description', 'submission_date']) + '\n') |
| 41 sample_alias = 'sample_'+str(sample_index)+'_'+timestamp | 50 for sample_index, sample in enumerate(study['samples']): |
| 42 if "geo_location" in sample.keys(): # sample belongs to a viral sample | 51 sample_alias = 'sample_' + str(sample_index) + '_' + timestamp |
| 52 if "geo_location" in sample.keys(): # sample belongs to a viral sample | |
| 43 if sample['collector_name'] == '': | 53 if sample['collector_name'] == '': |
| 44 sample['collector_name'] = 'unknown' | 54 sample['collector_name'] = 'unknown' |
| 45 samples_table.write('\t'.join([sample_alias,action,'ena_accession',sample['title'],sample['tax_name'], sample['tax_id'],sample['description'],sample['collection_date'],sample['geo_location'],sample['host_common_name'],sample['host_subject_id'],sample['host_health_state'],sample['host_sex'],sample['host_scientific_name'],sample['collector_name'],sample['collecting_institution'],sample['isolate'],'ENA_submission_date'])+ '\n') | 55 samples_table.write('\t'.join([sample_alias, action, 'ena_accession', sample['title'], |
| 56 sample['tax_name'], sample['tax_id'], | |
| 57 sample['description'], sample['collection_date'], | |
| 58 sample['geo_location'], sample['host_common_name'], | |
| 59 sample['host_subject_id'], sample['host_health_state'], | |
| 60 sample['host_sex'], sample['host_scientific_name'], | |
| 61 sample['collector_name'], | |
| 62 sample['collecting_institution'], sample['isolate'], | |
| 63 'ENA_submission_date']) + '\n') | |
| 46 else: | 64 else: |
| 47 samples_table.write('\t'.join([sample_alias,action,'ena_accession',sample['title'],sample['tax_name'], sample['tax_id'],sample['description'],'ENA_submission_date'])+ '\n') | 65 samples_table.write('\t'.join([sample_alias, action, 'ena_accession', sample['title'], |
| 48 for exp_index,exp in enumerate(sample['experiments']): | 66 sample['tax_name'], sample['tax_id'], |
| 49 exp_alias = 'experiment_'+str(exp_index)+'.'+str(sample_index)+'_'+timestamp | 67 sample['description'], 'ENA_submission_date']) + '\n') |
| 50 lib_alias = 'library_'+str(exp_index)+'_'+str(sample_index) | 68 for exp_index, exp in enumerate(sample['experiments']): |
| 51 experiments_table.write('\t'.join([exp_alias,action,'accession_ena',exp['title'],study_alias,sample_alias,exp['experiment_design'],lib_alias,exp['library_strategy'],exp['library_source'],exp['library_selection'],exp['library_layout'].lower(),exp['insert_size'],exp['library_construction_protocol'],exp['platform'],exp['instrument_model'],'submission_date_ENA']) + '\n') | 69 exp_alias = 'experiment_' + str(exp_index) + '.' + str(sample_index) + '_' + timestamp |
| 70 lib_alias = 'library_' + str(exp_index) + '_' + str(sample_index) | |
| 71 experiments_table.write('\t'.join([exp_alias, action, 'accession_ena', exp['title'], | |
| 72 study_alias, sample_alias, exp['experiment_design'], | |
| 73 lib_alias, exp['library_strategy'], | |
| 74 exp['library_source'], exp['library_selection'], | |
| 75 exp['library_layout'].lower(), exp['insert_size'], | |
| 76 exp['library_construction_protocol'], | |
| 77 exp['platform'], exp['instrument_model'], | |
| 78 'submission_date_ENA']) + '\n') | |
| 52 run_index = 0 | 79 run_index = 0 |
| 53 # exp['runs'] is a list of lists | 80 # exp['runs'] is a list of lists |
| 54 for run in exp['runs']: | 81 for run in exp['runs']: |
| 55 run_index += 1 | 82 run_index += 1 |
| 56 run_alias = '.'.join(['run_'+str(run_index),str(exp_index),str(sample_index)]) + '_' +timestamp | 83 run_alias = '.'.join(['run_' + str(run_index), str(exp_index), str(sample_index)]) \ |
| 84 + '_' + timestamp | |
| 57 for file_entry in run: | 85 for file_entry in run: |
| 58 file_format = 'fastq' | 86 runs_table.write('\t'.join([run_alias, action, 'ena_run_accession', exp_alias, |
| 59 runs_table.write('\t'.join([run_alias,action,'ena_run_accession',exp_alias,file_entry,file_format,'file_checksum','submission_date_ENA']) + '\n') | 87 file_entry, FILE_FORMAT, 'file_checksum', |
| 88 'submission_date_ENA']) + '\n') | |
| 60 | 89 |
| 61 studies_table.close() | 90 studies_table.close() |
| 62 samples_table.close() | 91 samples_table.close() |
| 63 experiments_table.close() | 92 experiments_table.close() |
| 64 runs_table.close() | 93 runs_table.close() |