|
0
|
1 #!/usr/bin/env python
|
|
|
2 # ref: https://galaxyproject.org/admin/tools/data-managers/how-to/define/
|
|
|
3
|
|
|
4 # Rewritten by H.E. Cicada Brokaw Dennis from a source downloaded from the toolshed and
|
|
|
5 # other example code on the web.
|
|
|
6 # This now allows downloading of a user selected library
|
|
|
7 # but only from the CTAT Genome Resource Library website.
|
|
|
8 # Ultimately we might want to allow the user to specify any location
|
|
|
9 # from which to download.
|
|
|
10 # Users can create or download other libraries and use this tool to add them if they don't want
|
|
|
11 # to add them by hand.
|
|
|
12
|
|
|
13 import argparse
|
|
|
14 import os
|
|
|
15 #import tarfile
|
|
|
16 #import urllib
|
|
|
17 import subprocess
|
|
|
18
|
|
|
19 # Comment out the following line when testing without galaxy package.
|
|
|
20 from galaxy.util.json import to_json_string
|
|
|
21 # The following is not being used, but leaving as info
|
|
|
22 # in case we ever want to get input values using json.
|
|
|
23 # from galaxy.util.json import from_json_string
|
|
|
24
|
|
|
25 # datetime.now() is used to create the unique_id
|
|
|
26 from datetime import datetime
|
|
|
27
|
|
|
28 # The FileListParser is used by get_ctat_genome_filenames(),
|
|
|
29 # which is called by the Data Manager interface (.xml file) to get
|
|
|
30 # the filenames that are available online at broadinstitute.org
|
|
|
31 # Not sure best way to do it.
|
|
|
32 # This object uses HTMLParser to look through the html
|
|
|
33 # searching for the filenames within anchor tags.
|
|
|
34 import urllib2
|
|
|
35 from HTMLParser import HTMLParser
|
|
|
36
|
|
|
37 _CTAT_ResourceLib_URL = 'https://data.broadinstitute.org/Trinity/CTAT_RESOURCE_LIB/'
|
|
|
38 _CTAT_BuildDir_Name = 'ctat_genome_lib_build_dir'
|
|
|
39 _CTAT_ResourceLib_DisplayNamePrefix = 'CTAT_GenomeResourceLib_'
|
|
|
40 _CTAT_ResourceLib_DefaultGenome = 'Unspecified_Genome'
|
|
|
41 _NumBytesNeededForBuild = 64424509440 # 60 Gigabytes. FIX - This might not be correct.
|
|
|
42 _Download_TestFile = "write_testfile.txt"
|
|
|
43 _DownloadSuccessFile = 'download_succeeded.txt'
|
|
|
44
|
|
|
45 class FileListParser(HTMLParser):
|
|
|
46 def __init__(self):
|
|
|
47 # Have to use direct call to super class rather than using super():
|
|
|
48 # super(FileListParser, self).__init__()
|
|
|
49 # because HTMLParser is an "old style" class and its inheritance chain does not include object.
|
|
|
50 HTMLParser.__init__(self)
|
|
|
51 self.urls = set()
|
|
|
52 def handle_starttag(self, tag, attrs):
|
|
|
53 # Look for filename references in anchor tags and add them to urls.
|
|
|
54 if tag == "a":
|
|
|
55 # The tag is an anchor tag.
|
|
|
56 for attribute in attrs:
|
|
|
57 # print "Checking: {:s}".format(str(attribute))
|
|
|
58 if attribute[0] == "href":
|
|
|
59 # Does the href have a tar.gz in it?
|
|
|
60 if ("tar.gz" in attribute[1]) and ("md5" not in attribute[1]):
|
|
|
61 # Add the value to urls.
|
|
|
62 self.urls.add(attribute[1])
|
|
|
63 # End of class FileListParser
|
|
|
64
|
|
|
65 def get_ctat_genome_urls():
|
|
|
66 # open the url and retrieve the urls of the files in the directory.
|
|
|
67 resource = urllib2.urlopen(_CTAT_ResourceLib_URL)
|
|
|
68 theHTML = resource.read()
|
|
|
69 filelist_parser = FileListParser()
|
|
|
70 filelist_parser.feed(theHTML)
|
|
|
71 # For dynamic options need to return an interable with contents that are tuples with 3 items.
|
|
|
72 # Item one is a string that is the display name put into the option list.
|
|
|
73 # Item two is the value that is put into the parameter associated with the option list.
|
|
|
74 # Item three is a True or False value, indicating whether the item is selected.
|
|
|
75 options = []
|
|
|
76 for i, url in enumerate(filelist_parser.urls):
|
|
|
77 # The urls look like:
|
|
|
78 # https://data.broadinstitute.org/Trinity/CTAT_RESOURCE_LIB/GRCh37_v19_CTAT_lib_Feb092018.plug-n-play.tar.gz
|
|
|
79 # https://data.broadinstitute.org/Trinity/CTAT_RESOURCE_LIB/Mouse_M16_CTAT_lib_Feb202018.source_data.tar.gz
|
|
|
80 filename = url.split("/")[-1]
|
|
|
81 if filename.split("_")[0] != "Mouse":
|
|
|
82 # Take out the mouse genome options for now.
|
|
|
83 # The mouse genome option is not handled correctly yet
|
|
|
84 options.append((filename, url, i == 0))
|
|
|
85 # return a tuple of the urls
|
|
4
|
86 print "The list being returned as options is:"
|
|
|
87 print "{:s}\n".format(str(options))
|
|
0
|
88 return options
|
|
|
89
|
|
|
90 # The following was used by the example program to get input parameters through the json.
|
|
|
91 # Just leaving here for reference.
|
|
|
92 # We are getting all of our parameter values through command line arguments.
|
|
|
93 #def get_reference_id_name(params):
|
|
|
94 # genome_id = params['param_dict']['genome_id']
|
|
|
95 # genome_name = params['param_dict']['genome_name']
|
|
|
96 # return genome_id, genome_name
|
|
|
97 #
|
|
|
98 #def get_url(params):
|
|
|
99 # trained_url = params['param_dict']['trained_url']
|
|
|
100 # return trained_url
|
|
|
101
|
|
|
102 def download_from_BroadInst(source, destination, force_download):
|
|
|
103 # Input Parameters
|
|
|
104 # source is the full URL of the file we want to download.
|
|
|
105 # It should look something like:
|
|
|
106 # https://data.broadinstitute.org/Trinity/CTAT_RESOURCE_LIB/GRCh37_v19_CTAT_lib_Feb092018.plug-n-play.tar.gz
|
|
|
107 # destination is the location where the source file will be unarchived.
|
|
|
108 # Relative paths are expanded using the current working directory, so within Galaxy,
|
|
|
109 # it is best to send in absolute fully specified path names so you know to where
|
|
|
110 # the source file going to be extracted.
|
|
|
111 # force_download will cause a new download and extraction to occur, even if the destination
|
|
|
112 # has a file in it indicating that a previous download succeeded.
|
|
|
113 #
|
|
|
114 # Returns the following:
|
|
|
115 # return (downloaded_directory, download_has_source_data, genome_build_directory, lib_was_downloaded)
|
|
|
116 # downloaded_directory
|
|
|
117 # The directory which was created as a subdirectory of the destination directory
|
|
|
118 # when the download occurred, or if there was no download,
|
|
|
119 # possibly the same directory as destination, if that is where the data resides.
|
|
|
120 # download_has_source_data
|
|
|
121 # Is a boolean indicating whether the source file was "source_data" or was "plug-n-play".
|
|
|
122 # genome_build_directory
|
|
|
123 # The directory where the genome resource library is or where it should be built.
|
|
|
124 # It can be the same as the downloaded directory, but is sometimes a subdirectory of it.
|
|
|
125 # lib_was_downloaded
|
|
|
126 # Since it doesn't always do the download, the function returns whether download occurred.
|
|
|
127 lib_was_downloaded = False
|
|
|
128
|
|
4
|
129 print "In download_from_BroadInst(). The source_url is:\n\t{:s}".format(str(source))
|
|
|
130
|
|
0
|
131 # Get the root filename of the Genome Directory.
|
|
|
132 src_filename = source.split("/")[-1]
|
|
|
133 root_genome_dirname = src_filename.split(".")[0]
|
|
|
134 # If the src_filename indicates it is a source file, as opposed to plug-n-play,
|
|
|
135 # then we may need to do some post processing on it.
|
|
|
136 type_of_download = src_filename.split(".")[1]
|
|
|
137 download_has_source_data = (type_of_download == "source_data")
|
|
|
138
|
|
|
139 # We want to make sure that destination is absolute fully specified path.
|
|
|
140 cannonical_destination = os.path.realpath(destination)
|
|
|
141 if os.path.exists(cannonical_destination):
|
|
|
142 if not os.path.isdir(cannonical_destination):
|
|
|
143 raise ValueError("The destination is not a directory: " + \
|
|
|
144 "{:s}".format(cannonical_destination))
|
|
|
145 # else all is good. It is a directory.
|
|
|
146 else:
|
|
|
147 # We need to create it.
|
|
|
148 try:
|
|
|
149 os.makedirs(cannonical_destination)
|
|
|
150 except os.error:
|
|
|
151 print "ERROR: Trying to create the following directory path:"
|
|
|
152 print "\t{:s}".format(cannonical_destination)
|
|
|
153 raise
|
|
|
154
|
|
|
155 # Make sure the directory now exists and we can write to it.
|
|
|
156 if not os.path.exists(cannonical_destination):
|
|
|
157 # It should have been created, but if it doesn't exist at this point
|
|
|
158 # in the code, something is wrong. Raise an error.
|
|
|
159 raise OSError("The destination directory could not be created: " + \
|
|
|
160 "{:s}".format(cannonical_destination))
|
|
|
161 test_writing_file = "{:s}/{:s}".format(cannonical_destination, _Download_TestFile)
|
|
|
162 try:
|
|
|
163 filehandle = open(test_writing_file, "w")
|
|
|
164 filehandle.write("Testing writing to this file.")
|
|
|
165 filehandle.close()
|
|
|
166 os.remove(test_writing_file)
|
|
|
167 except IOError:
|
|
|
168 print "The destination directory could not be written into: " + \
|
|
|
169 "{:s}".format(cannonical_destination)
|
|
|
170 raise
|
|
|
171
|
|
|
172 # Get the list of files in the directory,
|
|
|
173 # We use it to check for a previous download or extraction among other things.
|
|
|
174 orig_files_in_destdir = set(os.listdir(cannonical_destination))
|
|
|
175 # See whether the file has been downloaded already.
|
|
|
176 download_success_file_path = "{:s}/{:s}".format(cannonical_destination, _DownloadSuccessFile)
|
|
|
177 if ((_DownloadSuccessFile not in orig_files_in_destdir) \
|
|
|
178 or (root_genome_dirname not in orig_files_in_destdir) \
|
|
|
179 or force_download):
|
|
|
180 # Check whether there is enough space on the device for the library.
|
|
|
181 statvfs = os.statvfs(cannonical_destination)
|
|
|
182 # fs_size = statvfs.f_frsize * statvfs.f_blocks # Size of filesystem in bytes
|
|
|
183 # num_free_bytes = statvfs.f_frsize * statvfs.f_bfree # Actual number of free bytes
|
|
|
184 num_avail_bytes = statvfs.f_frsize * statvfs.f_bavail # Number of free bytes that ordinary users
|
|
|
185 # are allowed to use (excl. reserved space)
|
|
|
186 if (num_avail_bytes < _NumBytesNeededForBuild):
|
|
|
187 raise OSError("There is insufficient space ({:s} bytes)".format(str(num_avail_bytes)) + \
|
|
|
188 " on the device of the destination directory: " + \
|
|
|
189 "{:s}".format(cannonical_destination))
|
|
|
190
|
|
|
191 #Previous code to download and untar. Not using anymore.
|
|
|
192 #full_filepath = os.path.join(destination, src_filename)
|
|
|
193 #
|
|
|
194 #Download ref: https://dzone.com/articles/how-download-file-python
|
|
|
195 #f = urllib2.urlopen(source)
|
|
|
196 #data = f.read()
|
|
|
197 #with open(full_filepath, 'wb') as code:
|
|
|
198 # code.write(data)
|
|
|
199 #
|
|
|
200 #Another way to download:
|
|
|
201 #try:
|
|
|
202 # urllib.urlretrieve(url=source, filename=full_filepath)
|
|
|
203 #
|
|
|
204 #Then untar the file.
|
|
|
205 #try:
|
|
|
206 # tarfile.open(full_filepath, mode='r:*').extractall()
|
|
|
207
|
|
|
208 if (_DownloadSuccessFile in orig_files_in_destdir):
|
|
|
209 # Since we are redoing the download,
|
|
|
210 # the success file needs to be removed
|
|
|
211 # until the download has succeeded.
|
|
|
212 os.remove(download_success_file_path)
|
|
|
213 # We want to transfer and untar the file without storing the tar file, because that
|
|
|
214 # adds all that much more space to the needed amount of free space on the disk.
|
|
|
215 # Use subprocess to pipe the output of curl into tar.
|
|
|
216 command = "curl {:s} | tar -xzvf - -C {:s}".format(source, cannonical_destination)
|
|
|
217 try: # to send the command that downloads and extracts the file.
|
|
|
218 command_output = subprocess.check_output(command, shell=True)
|
|
|
219 # FIX - not sure check_output is what we want to use. If we want to have an error raised on
|
|
|
220 # any problem, maybe we should not be checking output.
|
|
|
221 except subprocess.CalledProcessError:
|
|
|
222 print "ERROR: Trying to run the following command:\n\t{:s}".format(command)
|
|
|
223 raise
|
|
|
224 else:
|
|
|
225 lib_was_downloaded = True
|
|
|
226
|
|
|
227 # Some code to help us if errors occur.
|
|
|
228 print "\n*******************************\nFinished download and extraction."
|
|
|
229 subprocess.check_call("ls -lad {:s}/*".format(cannonical_destination), shell=True)
|
|
|
230 subprocess.check_call("ls -lad {:s}/*/*".format(cannonical_destination), shell=True)
|
|
|
231
|
|
|
232 newfiles_in_destdir = set(os.listdir(cannonical_destination)) - orig_files_in_destdir
|
|
|
233 if (root_genome_dirname not in newfiles_in_destdir):
|
|
|
234 # Perhaps it has a different name than what we expected it to be.
|
|
|
235 # It will be the file that was not in the directory
|
|
|
236 # before we did the download and extraction.
|
|
|
237 found_filename = None
|
|
|
238 if len(newfiles_in_destdir) == 1:
|
|
|
239 found_filename = newfiles_in_destdir[0]
|
|
|
240 else:
|
|
|
241 for filename in newfiles_in_destdir:
|
|
|
242 # In most cases, there will only be one new file, but some OS's might have created
|
|
|
243 # other files in the directory.
|
|
|
244 # Look for the directory that was downloaded and extracted.
|
|
|
245 # The correct file's name should be a substring of the tar file that was downloaded.
|
|
|
246 if filename in src_filename:
|
|
|
247 found_filename = filename
|
|
|
248 if found_filename is not None:
|
|
|
249 root_genome_dirname = found_filename
|
|
|
250
|
|
|
251 downloaded_directory = "{:s}/{:s}".format(cannonical_destination, root_genome_dirname)
|
|
|
252
|
|
|
253 if (os.path.exists(downloaded_directory)):
|
|
|
254 try:
|
|
|
255 # Create a file to indicate that the download succeeded.
|
|
|
256 subprocess.check_call("touch {:s}".format(download_success_file_path), shell=True)
|
|
|
257 except IOError:
|
|
|
258 print "The download_success file could not be created: " + \
|
|
|
259 "{:s}".format(download_success_file_path)
|
|
|
260 raise
|
|
|
261 # Look for the build directory, or specify the path where it should be placed.
|
|
|
262 if len(os.listdir(downloaded_directory)) == 1:
|
|
|
263 # Then that one file is a subdirectory that should be the downloaded_directory.
|
|
|
264 subdir_filename = os.listdir(downloaded_directory)[0]
|
|
|
265 genome_build_directory = "{:s}/{:s}".format(downloaded_directory, subdir_filename)
|
|
|
266 else:
|
|
|
267 genome_build_directory = "{:s}/{:s}".format(downloaded_directory, _CTAT_BuildDir_Name)
|
|
|
268 else:
|
|
|
269 raise ValueError("ERROR: Could not find the extracted file in the destination directory:" + \
|
|
|
270 "\n\t{:s}".format(cannonical_destination))
|
|
|
271
|
|
|
272 return (downloaded_directory, download_has_source_data, genome_build_directory, lib_was_downloaded)
|
|
|
273
|
|
|
274 def gmap_the_library(genome_build_directory):
|
|
|
275 # This is the processing that needs to happen for gmap-fusion to work.
|
|
|
276 # genome_build_directory should normally be a fully specified path,
|
|
|
277 # though it should work if it is relative.
|
|
|
278 command = "gmap_build -D {:s}/ -d ref_genome.fa.gmap -k 13 {:s}/ref_genome.fa".format( \
|
|
|
279 genome_build_directory, genome_build_directory)
|
|
|
280 try: # to send the gmap_build command.
|
|
|
281 command_output = subprocess.check_output(command, shell=True)
|
|
|
282 except subprocess.CalledProcessError:
|
|
|
283 print "ERROR: While trying to run the gmap_build command on the library:\n\t{:s}".format(command)
|
|
|
284 raise
|
|
|
285 finally:
|
|
|
286 # Some code to help us if errors occur.
|
|
|
287 print "\n*******************************\nAfter running gmap_build."
|
|
|
288 if os.path.exists(genome_build_directory):
|
|
|
289 print "\nBuild Directory {:s}:".format(genome_build_directory)
|
|
|
290 subprocess.check_call("ls -la {:s}".format(genome_build_directory), shell=True)
|
|
|
291 dir_entries = os.listdir(genome_build_directory)
|
|
|
292 for entry in dir_entries:
|
|
|
293 entry_path = "{:s}/{:s}".format(genome_build_directory, entry)
|
|
|
294 print "\nDirectory {:s}:".format(entry_path)
|
|
|
295 subprocess.check_call("ls -la {:s}".format(entry_path), shell=True)
|
|
|
296 if os.path.isdir(entry_path):
|
|
|
297 subdir_entries = os.listdir(entry_path)
|
|
|
298 for subdir_entry in subdir_entries:
|
|
|
299 subdir_entry_path = "{:s}/{:s}".format(entry_path, subdir_entry)
|
|
|
300 print "\nDirectory {:s}:".format(subdir_entry_path)
|
|
|
301 subprocess.check_call("ls -la {:s}".format(subdir_entry_path), shell=True)
|
|
|
302 else:
|
|
|
303 print "Genome Build Directory does not exist:\n\t{:s}".format(genome_build_directory)
|
|
|
304 print "*******************************"
|
|
|
305
|
|
|
306 def build_the_library(genome_source_directory, genome_build_directory, build, gmap_build):
|
|
|
307 """ genome_source_directory is the location of the source_data needed to build the library.
|
|
|
308 Normally it is fully specified, but could be relative.
|
|
|
309 genome_build_directory is the location where the library will be built.
|
|
|
310 It can be relative to the current working directory or an absolute path.
|
|
|
311 build specifies whether to run prep_genome_lib.pl even if it was run before.
|
|
|
312 gmap_build specifies whether to run gmap_build or not.
|
|
|
313
|
|
|
314 Following was the old way to do it. Before FusionFilter 0.5.0.
|
|
|
315 prep_genome_lib.pl \
|
|
|
316 --genome_fa ref_genome.fa \
|
|
|
317 --gtf ref_annot.gtf \
|
|
|
318 --blast_pairs blast_pairs.gene_syms.outfmt6.gz \
|
|
|
319 --fusion_annot_lib fusion_lib.dat.gz
|
|
|
320 --output_dir ctat_genome_lib_build_dir
|
|
|
321 index_pfam_domain_info.pl \
|
|
|
322 --pfam_domains PFAM.domtblout.dat.gz \
|
|
|
323 --genome_lib_dir ctat_genome_lib_build_dir
|
|
|
324 gmap_build -D ctat_genome_lib_build_dir -d ref_genome.fa.gmap -k 13 ctat_genome_lib_build_dir/ref_genome.fa"
|
|
|
325 """
|
|
|
326 if (genome_source_directory != "" ) and build:
|
|
|
327 if os.path.exists(genome_source_directory):
|
|
|
328 os.chdir(genome_source_directory)
|
|
|
329 # FIX - look for a fusion_annot_lib and include it, else omit it.
|
|
|
330 command = "prep_genome_lib.pl --genome_fa ref_genome.fa --gtf ref_annot.gtf " + \
|
|
|
331 "--fusion_annot_lib CTAT_HumanFusionLib.v0.1.0.dat.gz " + \
|
|
|
332 "--annot_filter_rule AnnotFilterRule.pm " + \
|
|
|
333 "--pfam_db PFAM.domtblout.dat.gz " + \
|
|
|
334 "--output_dir {:s} ".format(genome_build_directory)
|
|
|
335 if gmap_build:
|
|
|
336 command += "--gmap_build "
|
|
|
337 try: # to send the prep_genome_lib command.
|
|
|
338 command_output = subprocess.check_call(command, shell=True)
|
|
|
339 except subprocess.CalledProcessError:
|
|
|
340 print "ERROR: While trying to run the prep_genome_lib.pl command " + \
|
|
|
341 "on the CTAT Genome Resource Library:\n\t{:s}".format(command)
|
|
|
342 raise
|
|
|
343 finally:
|
|
|
344 # Some code to help us if errors occur.
|
|
|
345 print "*******************************"
|
|
|
346 if os.path.exists(genome_build_directory):
|
|
|
347 print "\nSource Directory {:s}:".format(genome_source_directory)
|
|
|
348 subprocess.check_call("ls -la {:s}".format(genome_source_directory), shell=True)
|
|
|
349 dir_entries = os.listdir(genome_source_directory)
|
|
|
350 for entry in dir_entries:
|
|
|
351 entry_path = "{:s}/{:s}".format(genome_source_directory, entry)
|
|
|
352 print "\nDirectory {:s}:".format(entry_path)
|
|
|
353 subprocess.check_call("ls -la {:s}".format(entry_path), shell=True)
|
|
|
354 else:
|
|
|
355 print "Genome Source Directory does not exist:\n\t{:s}".format(genome_source_directory)
|
|
|
356 if os.path.exists(genome_build_directory):
|
|
|
357 print "\nBuild Directory {:s}:".format(genome_build_directory)
|
|
|
358 subprocess.check_call("ls -la {:s}".format(genome_build_directory), shell=True)
|
|
|
359 dir_entries = os.listdir(genome_build_directory)
|
|
|
360 for entry in dir_entries:
|
|
|
361 entry_path = "{:s}/{:s}".format(genome_build_directory, entry)
|
|
|
362 print "\nDirectory {:s}:".format(entry_path)
|
|
|
363 subprocess.check_call("ls -la {:s}".format(entry_path), shell=True)
|
|
|
364 if os.path.isdir(entry_path):
|
|
|
365 subdir_entries = os.listdir(entry_path)
|
|
|
366 for subdir_entry in subdir_entries:
|
|
|
367 subdir_entry_path = "{:s}/{:s}".format(entry_path, subdir_entry)
|
|
|
368 print "\nDirectory {:s}:".format(subdir_entry_path)
|
|
|
369 subprocess.check_call("ls -la {:s}".format(subdir_entry_path), shell=True)
|
|
|
370 else:
|
|
|
371 print "Genome Build Directory does not exist:\n\t{:s}".format(genome_build_directory)
|
|
|
372 print "*******************************"
|
|
|
373 else:
|
|
|
374 raise ValueError("Cannot build the CTAT Genome Resource Library. " + \
|
|
|
375 "The source directory does not exist:\n\t{:s}".format(genome_source_directory))
|
|
|
376 elif gmap_build:
|
|
|
377 gmap_the_library(genome_build_directory)
|
|
|
378
|
|
|
379 def main():
|
|
|
380 #Parse Command Line
|
|
|
381 parser = argparse.ArgumentParser()
|
|
|
382 parser.add_argument('-s', '--source_url', default="", \
|
|
|
383 help='This is the url of a file with the data. They come from https://data.broadinstitute.org/Trinity/CTAT_RESOURCE_LIB/.')
|
|
|
384 parser.add_argument('-n', '--display_name', default="", \
|
|
|
385 help='Is used as the display name for the entry of this Genome Resource Library in the data table.')
|
|
|
386 parser.add_argument('-p', '--destination_path', \
|
|
|
387 help='Full path of the CTAT Resource Library location or destination, either where it is, or where it will be placed.')
|
|
|
388 parser.add_argument('-o', '--output_filename', \
|
|
|
389 help='Name of the output file, where the json dictionary will be written.')
|
|
|
390 parser.add_argument('-f', '--force_download',
|
|
|
391 help='Forces download of the Genome Resource Library, even if previously downloaded.', action="store_true")
|
|
|
392 parser.add_argument('-b', '--build',
|
|
|
393 help='Forces build/rebuild the Genome Resource Library, even if previously built. ' + \
|
|
|
394 'Must have downloaded source_data for this to work.', action="store_true")
|
|
|
395 parser.add_argument('-m', '--gmap_build',
|
|
|
396 help='Must be selected if you want the library to be gmapped. ' + \
|
|
|
397 'Will force gmap_build of the Genome Resource Library, even if previously gmapped.', action="store_true")
|
|
|
398 args = parser.parse_args()
|
|
|
399
|
|
|
400 # All of the input parameters are written by default to the output file prior to
|
|
|
401 # this program being called.
|
|
|
402 # But I do not get input values from the json file, but rather from command line.
|
|
|
403 # Just leaving the following code as a comment, in case it might be useful to someone later.
|
|
|
404 # params = from_json_string(open(filename).read())
|
|
|
405 # target_directory = params['output_data'][0]['extra_files_path']
|
|
|
406 # os.mkdir(target_directory)
|
|
|
407
|
|
4
|
408 print "The value of source_url argument is:\n\t{:s}".format(str(args.source_url))
|
|
|
409
|
|
0
|
410 # FIX - not sure the lib_was_downloaded actually serves a purpose...
|
|
|
411 lib_was_downloaded = False
|
|
|
412 download_has_source_data = False
|
|
|
413 # If we do not download the directory, the destination_path should be the
|
|
|
414 # location of the genome resource library.
|
|
|
415 downloaded_directory = None
|
|
|
416 # FIX - look inside of the args.destination_path to see if the build directory is inside it or is it.
|
|
|
417 genome_build_directory = None
|
|
|
418 # FIX - need to make sure we are handling all "possible" combinations of arguments.
|
|
|
419 # Probably would be good if we could simplify/remove some of them.
|
|
|
420 if (args.source_url != ""):
|
|
|
421 downloaded_directory, download_has_source_data, genome_build_directory, lib_was_downloaded = \
|
|
|
422 download_from_BroadInst(source=args.source_url, \
|
|
|
423 destination=args.destination_path, \
|
|
|
424 force_download=args.force_download)
|
|
|
425 else:
|
|
|
426 genome_build_directory = args.destination_path
|
|
|
427 if not os.path.exists(genome_build_directory):
|
|
|
428 raise ValueError("Cannot find the CTAT Genome Resource Library. " + \
|
|
|
429 "The directory does not exist:\n\t{:s}".format(genome_build_directory))
|
|
|
430 # else:
|
|
|
431 # FIX - Check if there is an actual CTAT Genome Resource Lib there.
|
|
|
432 # _CTAT_BuildDir_Name
|
|
|
433
|
|
|
434 print "\nThe location of the CTAT Genome Resource Library is {:s}.\n".format(genome_build_directory)
|
|
|
435
|
|
|
436 # Take out builds for testing.
|
|
|
437 # FIX - We should leave a file indicating build success the same way we do for download success.
|
|
|
438 if (download_has_source_data or args.build or args.gmap_build) :
|
|
|
439 build_the_library(downloaded_directory, genome_build_directory, args.build, args.gmap_build)
|
|
|
440 elif (args.gmap_build):
|
|
|
441 gmap_the_library(genome_build_directory)
|
|
|
442
|
|
|
443 if (args.source_url != None) and (args.source_url != ""):
|
|
|
444 # Get the name out of the source's filename.
|
|
|
445 source_filename_root = args.source_url.split("/")[-1].split(".")[0]
|
|
|
446
|
|
|
447 # Determine the display_name for the library.
|
|
|
448 if (args.display_name is None) or (args.display_name == ""):
|
|
|
449 if (source_filename_root != None) and (source_filename_root != ""):
|
|
|
450 # Get the name out of the source filename.
|
|
|
451 display_name = _CTAT_ResourceLib_DisplayNamePrefix + source_filename_root
|
|
|
452 else:
|
|
|
453 display_name = _CTAT_ResourceLib_DisplayNamePrefix + _CTAT_ResourceLib_DefaultGenome
|
|
|
454 print "WARNING: We do not have a genome name. Using a default name, that might not be correct."
|
|
|
455 else:
|
|
|
456 display_name = _CTAT_ResourceLib_DisplayNamePrefix + args.display_name
|
|
|
457 display_name = display_name.replace(" ","_")
|
|
|
458 print "The Genome Name will be set to: {:s}\n".format(display_name)
|
|
|
459
|
|
|
460 # Create a unique_id for the library.
|
|
|
461 datetime_stamp = datetime.now().strftime("_%Y_%m_%d_%H_%M_%S_%f")
|
|
|
462 if (source_filename_root != None) and (source_filename_root != ""):
|
|
|
463 unique_id = source_filename_root + datetime_stamp
|
|
|
464 elif (downloaded_directory != None) and (downloaded_directory != ""):
|
|
|
465 unique_id = os.path.basename(downloaded_directory).split(".")[0]
|
|
|
466 else:
|
|
|
467 unique_id = _CTAT_ResourceLib_DefaultGenome + datetime_stamp
|
|
|
468
|
|
|
469 print "The Resource Lib's display_name will be set to: {:s}\n".format(display_name)
|
|
|
470 print "Its unique_id will be set to: {:s}\n".format(unique_id)
|
|
|
471 print "Its dir_path will be set to: {:s}\n".format(genome_build_directory)
|
|
|
472
|
|
|
473 data_manager_dict = {}
|
|
|
474 data_manager_dict['data_tables'] = {}
|
|
|
475 data_manager_dict['data_tables']['ctat_genome_resource_libs'] = []
|
|
|
476 data_table_entry = dict(value=unique_id, name=display_name, path=genome_build_directory)
|
|
|
477 data_manager_dict['data_tables']['ctat_genome_resource_libs'].append(data_table_entry)
|
|
|
478
|
|
|
479 # Temporarily the output file's dictionary is written for debugging:
|
|
|
480 print "The dictionary for the output file is:\n\t{:s}".format(str(data_manager_dict))
|
|
|
481 # Save info to json file. This is used to transfer data from the DataManager tool, to the data manager,
|
|
|
482 # which then puts it into the correct .loc file (I think).
|
|
|
483 # Comment out the following line when testing without galaxy package.
|
|
|
484 open(args.output_filename, 'wb').write(to_json_string(data_manager_dict))
|
|
|
485
|
|
|
486 if __name__ == "__main__":
|
|
|
487 main()
|
