|
0
|
1 #!/usr/bin/env python
|
|
|
2 # ref: https://galaxyproject.org/admin/tools/data-managers/how-to/define/
|
|
|
3
|
|
|
4 # Rewritten by H.E. Cicada Brokaw Dennis from a source downloaded from the toolshed and
|
|
|
5 # other example code on the web.
|
|
|
6 # This now allows downloading of a user selected library
|
|
|
7 # but only from the CTAT Genome Resource Library website.
|
|
|
8 # Ultimately we might want to allow the user to specify any location
|
|
|
9 # from which to download.
|
|
|
10 # Users can create or download other libraries and use this tool to add them if they don't want
|
|
|
11 # to add them by hand.
|
|
|
12
|
|
|
13 import argparse
|
|
|
14 import os
|
|
|
15 #import tarfile
|
|
|
16 #import urllib
|
|
|
17 import subprocess
|
|
|
18
|
|
|
19 # Comment out the following line when testing without galaxy package.
|
|
|
20 from galaxy.util.json import to_json_string
|
|
|
21 # The following is not being used, but leaving as info
|
|
|
22 # in case we ever want to get input values using json.
|
|
|
23 # from galaxy.util.json import from_json_string
|
|
|
24
|
|
|
25 # datetime.now() is used to create the unique_id
|
|
|
26 from datetime import datetime
|
|
|
27
|
|
|
28 # The FileListParser is used by get_ctat_genome_filenames(),
|
|
|
29 # which is called by the Data Manager interface (.xml file) to get
|
|
|
30 # the filenames that are available online at broadinstitute.org
|
|
|
31 # Not sure best way to do it.
|
|
|
32 # This object uses HTMLParser to look through the html
|
|
|
33 # searching for the filenames within anchor tags.
|
|
|
34 import urllib2
|
|
|
35 from HTMLParser import HTMLParser
|
|
|
36
|
|
|
37 _CTAT_ResourceLib_URL = 'https://data.broadinstitute.org/Trinity/CTAT_RESOURCE_LIB/'
|
|
8
|
38 _CTAT_MutationIndex_URL = 'https://data.broadinstitute.org/Trinity/CTAT/mutation/'
|
|
6
|
39 _CTAT_Build_dirname = 'ctat_genome_lib_build_dir'
|
|
0
|
40 _CTAT_ResourceLib_DisplayNamePrefix = 'CTAT_GenomeResourceLib_'
|
|
|
41 _CTAT_ResourceLib_DefaultGenome = 'Unspecified_Genome'
|
|
6
|
42 _CTAT_HumanFusionLib_FilenamePrefix = 'CTAT_HumanFusionLib'
|
|
|
43 _CTAT_RefGenome_Filename = 'ref_genome.fa'
|
|
7
|
44 _CTAT_MouseGenome_Prefix = 'Mouse'
|
|
|
45 _CTAT_HumanGenome_Prefix = 'GRCh'
|
|
8
|
46 _NumBytesNeededForBuild = 66571993088 # 62 Gigabytes. FIX - This might not be correct.
|
|
|
47 _NumBytesNeededForIndexes = 21474836480 # 20 Gigabytes. FIX - This might not be correct.
|
|
0
|
48 _Download_TestFile = "write_testfile.txt"
|
|
|
49 _DownloadSuccessFile = 'download_succeeded.txt'
|
|
8
|
50 _LibBuiltSuccessFile = 'build_succeeded.txt'
|
|
|
51 _MutationDownloadSuccessFile = 'mutation_index_download_succeeded.txt'
|
|
0
|
52
|
|
|
53 class FileListParser(HTMLParser):
|
|
|
54 def __init__(self):
|
|
|
55 # Have to use direct call to super class rather than using super():
|
|
|
56 # super(FileListParser, self).__init__()
|
|
|
57 # because HTMLParser is an "old style" class and its inheritance chain does not include object.
|
|
|
58 HTMLParser.__init__(self)
|
|
|
59 self.urls = set()
|
|
|
60 def handle_starttag(self, tag, attrs):
|
|
|
61 # Look for filename references in anchor tags and add them to urls.
|
|
|
62 if tag == "a":
|
|
|
63 # The tag is an anchor tag.
|
|
|
64 for attribute in attrs:
|
|
|
65 # print "Checking: {:s}".format(str(attribute))
|
|
|
66 if attribute[0] == "href":
|
|
|
67 # Does the href have a tar.gz in it?
|
|
|
68 if ("tar.gz" in attribute[1]) and ("md5" not in attribute[1]):
|
|
|
69 # Add the value to urls.
|
|
|
70 self.urls.add(attribute[1])
|
|
|
71 # End of class FileListParser
|
|
|
72
|
|
|
73 def get_ctat_genome_urls():
|
|
|
74 # open the url and retrieve the urls of the files in the directory.
|
|
|
75 resource = urllib2.urlopen(_CTAT_ResourceLib_URL)
|
|
|
76 theHTML = resource.read()
|
|
|
77 filelist_parser = FileListParser()
|
|
|
78 filelist_parser.feed(theHTML)
|
|
|
79 # For dynamic options need to return an interable with contents that are tuples with 3 items.
|
|
|
80 # Item one is a string that is the display name put into the option list.
|
|
|
81 # Item two is the value that is put into the parameter associated with the option list.
|
|
|
82 # Item three is a True or False value, indicating whether the item is selected.
|
|
|
83 options = []
|
|
|
84 for i, url in enumerate(filelist_parser.urls):
|
|
5
|
85 # The urls should look like:
|
|
0
|
86 # https://data.broadinstitute.org/Trinity/CTAT_RESOURCE_LIB/GRCh37_v19_CTAT_lib_Feb092018.plug-n-play.tar.gz
|
|
|
87 # https://data.broadinstitute.org/Trinity/CTAT_RESOURCE_LIB/Mouse_M16_CTAT_lib_Feb202018.source_data.tar.gz
|
|
8
|
88 # But in actuality, they are coming in looking like:
|
|
5
|
89 # GRCh37_v19_CTAT_lib_Feb092018.plug-n-play.tar.gz
|
|
|
90 # Mouse_M16_CTAT_lib_Feb202018.source_data.tar.gz
|
|
|
91 # Write code to handle both situations, or an ftp: url.
|
|
|
92 if (url.split(":")[0] == "http") or (url.split(":")[0] == "https") or (url.split(":")[0] == "ftp"):
|
|
|
93 full_url_path = url
|
|
|
94 else:
|
|
|
95 # Assume the path is relative to the page location.
|
|
|
96 full_url_path = "{:s}/{:s}".format(_CTAT_ResourceLib_URL, url)
|
|
0
|
97 filename = url.split("/")[-1]
|
|
8
|
98 # if filename.split("_")[0] != _CTAT_MouseGenome_Prefix:
|
|
|
99 # # Don't put in the mouse genome options for now.
|
|
|
100 # # The mouse genome option is not handled correctly yet
|
|
|
101 # options.append((filename, full_url_path, i == 0))
|
|
|
102 # Mouse genomes should work now (we hope) - FIX - still not tested.
|
|
|
103 options.append((filename, full_url_path, i == 0))
|
|
|
104 options.sort() # So the list will be in alphabetical order.
|
|
|
105 # return a tuple of the urls
|
|
|
106 print "The list being returned as options is:"
|
|
|
107 print "{:s}\n".format(str(options))
|
|
|
108 return options
|
|
5
|
109
|
|
8
|
110 def get_mutation_index_urls():
|
|
|
111 # open the url and retrieve the urls of the files in the directory.
|
|
|
112 resource = urllib2.urlopen(_CTAT_MutationIndex_URL)
|
|
|
113 theHTML = resource.read()
|
|
|
114 filelist_parser = FileListParser()
|
|
|
115 filelist_parser.feed(theHTML)
|
|
|
116 # For dynamic options need to return an interable with contents that are tuples with 3 items.
|
|
|
117 # Item one is a string that is the display name put into the option list.
|
|
|
118 # Item two is the value that is put into the parameter associated with the option list.
|
|
|
119 # Item three is a True or False value, indicating whether the item is selected.
|
|
|
120 options = []
|
|
|
121 for i, url in enumerate(filelist_parser.urls):
|
|
|
122 # The urls should look like:
|
|
|
123 # https://data.broadinstitute.org/Trinity/CTAT/mutation/mc7.tar.gz
|
|
|
124 # https://data.broadinstitute.org/Trinity/CTAT/mutation/hg19.tar.gz
|
|
|
125 # But in actuality, they are coming in looking like:
|
|
|
126 # hg19.tar.gz
|
|
|
127 # mc7.tar.gz
|
|
|
128 # Write code to handle both situations, or an ftp: url.
|
|
|
129 if (url.split(":")[0] == "http") or (url.split(":")[0] == "https") or (url.split(":")[0] == "ftp"):
|
|
|
130 full_url_path = url
|
|
|
131 else:
|
|
|
132 # Assume the path is relative to the page location.
|
|
|
133 full_url_path = "{:s}/{:s}".format(_CTAT_MutationIndex_URL, url)
|
|
|
134 filename = url.split("/")[-1]
|
|
|
135 options.append((filename, full_url_path, i == 0))
|
|
5
|
136 options.sort() # So the list will be in alphabetical order.
|
|
0
|
137 # return a tuple of the urls
|
|
4
|
138 print "The list being returned as options is:"
|
|
|
139 print "{:s}\n".format(str(options))
|
|
0
|
140 return options
|
|
|
141
|
|
|
142 # The following was used by the example program to get input parameters through the json.
|
|
|
143 # Just leaving here for reference.
|
|
|
144 # We are getting all of our parameter values through command line arguments.
|
|
|
145 #def get_reference_id_name(params):
|
|
|
146 # genome_id = params['param_dict']['genome_id']
|
|
|
147 # genome_name = params['param_dict']['genome_name']
|
|
|
148 # return genome_id, genome_name
|
|
|
149 #
|
|
|
150 #def get_url(params):
|
|
|
151 # trained_url = params['param_dict']['trained_url']
|
|
|
152 # return trained_url
|
|
|
153
|
|
8
|
154 # The following procedure is used to help with debugging and for user information.
|
|
6
|
155 def print_directory_contents(dir_path, num_levels):
|
|
|
156 if num_levels > 0:
|
|
|
157 if os.path.exists(dir_path) and os.path.isdir(dir_path):
|
|
|
158 print "\nDirectory {:s}:".format(dir_path)
|
|
|
159 subprocess.call("ls -la {:s} 2>&1".format(dir_path), shell=True)
|
|
|
160 else:
|
|
|
161 print "Path either does not exist, or is not a directory:\n\t{:s}.".format(dir_path)
|
|
|
162 if num_levels > 1:
|
|
8
|
163 if os.path.exists(dir_path) and os.path.isdir(dir_path):
|
|
|
164 for filename in os.listdir(dir_path):
|
|
|
165 filename_path = "{:s}/{:s}".format(dir_path, filename)
|
|
|
166 if os.path.exists(filename_path) and os.path.isdir(filename_path):
|
|
|
167 print_directory_contents(filename_path, num_levels-1)
|
|
|
168 else:
|
|
|
169 print "Path either does not exist, or is not a directory:\n\t{:s}.".format(dir_path)
|
|
6
|
170
|
|
0
|
171 def download_from_BroadInst(source, destination, force_download):
|
|
|
172 # Input Parameters
|
|
|
173 # source is the full URL of the file we want to download.
|
|
|
174 # It should look something like:
|
|
|
175 # https://data.broadinstitute.org/Trinity/CTAT_RESOURCE_LIB/GRCh37_v19_CTAT_lib_Feb092018.plug-n-play.tar.gz
|
|
|
176 # destination is the location where the source file will be unarchived.
|
|
|
177 # Relative paths are expanded using the current working directory, so within Galaxy,
|
|
|
178 # it is best to send in absolute fully specified path names so you know to where
|
|
|
179 # the source file going to be extracted.
|
|
|
180 # force_download will cause a new download and extraction to occur, even if the destination
|
|
|
181 # has a file in it indicating that a previous download succeeded.
|
|
|
182 #
|
|
|
183 # Returns the following:
|
|
|
184 # return (downloaded_directory, download_has_source_data, genome_build_directory, lib_was_downloaded)
|
|
|
185 # downloaded_directory
|
|
|
186 # The directory which was created as a subdirectory of the destination directory
|
|
|
187 # when the download occurred, or if there was no download,
|
|
|
188 # possibly the same directory as destination, if that is where the data resides.
|
|
|
189 # download_has_source_data
|
|
|
190 # Is a boolean indicating whether the source file was "source_data" or was "plug-n-play".
|
|
|
191 # genome_build_directory
|
|
|
192 # The directory where the genome resource library is or where it should be built.
|
|
|
193 # It can be the same as the downloaded directory, but is sometimes a subdirectory of it.
|
|
|
194 # lib_was_downloaded
|
|
|
195 # Since it doesn't always do the download, the function returns whether download occurred.
|
|
|
196 lib_was_downloaded = False
|
|
7
|
197 if len(source.split(":")) == 1:
|
|
8
|
198 # Then we were given a source_url without a leading https: or similar.
|
|
7
|
199 # Assume we only were given the filename and that it exists at _CTAT_ResourceLib_URL.
|
|
|
200 source = "{:s}/{:s}".format(_CTAT_ResourceLib_URL, source)
|
|
8
|
201 # else we might want to check that it is one of "http", "ftp", "file" or other accepted url starts.
|
|
0
|
202
|
|
4
|
203 print "In download_from_BroadInst(). The source_url is:\n\t{:s}".format(str(source))
|
|
|
204
|
|
0
|
205 # Get the root filename of the Genome Directory.
|
|
|
206 src_filename = source.split("/")[-1]
|
|
|
207 root_genome_dirname = src_filename.split(".")[0]
|
|
|
208 # If the src_filename indicates it is a source file, as opposed to plug-n-play,
|
|
|
209 # then we may need to do some post processing on it.
|
|
|
210 type_of_download = src_filename.split(".")[1]
|
|
7
|
211 print "The file to be extracted is {:s}".format(src_filename)
|
|
|
212 print "The type of download is {:s}".format(type_of_download)
|
|
0
|
213 download_has_source_data = (type_of_download == "source_data")
|
|
|
214
|
|
|
215 # We want to make sure that destination is absolute fully specified path.
|
|
|
216 cannonical_destination = os.path.realpath(destination)
|
|
|
217 if os.path.exists(cannonical_destination):
|
|
|
218 if not os.path.isdir(cannonical_destination):
|
|
|
219 raise ValueError("The destination is not a directory: " + \
|
|
|
220 "{:s}".format(cannonical_destination))
|
|
|
221 # else all is good. It is a directory.
|
|
|
222 else:
|
|
|
223 # We need to create it.
|
|
|
224 try:
|
|
|
225 os.makedirs(cannonical_destination)
|
|
|
226 except os.error:
|
|
|
227 print "ERROR: Trying to create the following directory path:"
|
|
|
228 print "\t{:s}".format(cannonical_destination)
|
|
|
229 raise
|
|
|
230
|
|
|
231 # Make sure the directory now exists and we can write to it.
|
|
|
232 if not os.path.exists(cannonical_destination):
|
|
|
233 # It should have been created, but if it doesn't exist at this point
|
|
|
234 # in the code, something is wrong. Raise an error.
|
|
|
235 raise OSError("The destination directory could not be created: " + \
|
|
|
236 "{:s}".format(cannonical_destination))
|
|
7
|
237 test_writing_file = "{:s}/{:s}.{:s}".format(cannonical_destination, root_genome_dirname, _Download_TestFile)
|
|
0
|
238 try:
|
|
|
239 filehandle = open(test_writing_file, "w")
|
|
|
240 filehandle.write("Testing writing to this file.")
|
|
|
241 filehandle.close()
|
|
|
242 os.remove(test_writing_file)
|
|
|
243 except IOError:
|
|
|
244 print "The destination directory could not be written into: " + \
|
|
|
245 "{:s}".format(cannonical_destination)
|
|
|
246 raise
|
|
|
247
|
|
|
248 # Get the list of files in the directory,
|
|
|
249 # We use it to check for a previous download or extraction among other things.
|
|
|
250 orig_files_in_destdir = set(os.listdir(cannonical_destination))
|
|
|
251 # See whether the file has been downloaded already.
|
|
8
|
252 # FIX - Try looking one or two directories above, as well as current directory,
|
|
|
253 # and maybe one directory below,
|
|
|
254 # for the download success file?
|
|
|
255 # Not sure about this though...
|
|
7
|
256 download_success_file = "{:s}.{:s}".format(root_genome_dirname, _DownloadSuccessFile)
|
|
|
257 download_success_file_path = "{:s}/{:s}".format(cannonical_destination, download_success_file)
|
|
|
258 if ((download_success_file not in orig_files_in_destdir) \
|
|
0
|
259 or (root_genome_dirname not in orig_files_in_destdir) \
|
|
|
260 or force_download):
|
|
|
261 # Check whether there is enough space on the device for the library.
|
|
|
262 statvfs = os.statvfs(cannonical_destination)
|
|
|
263 # fs_size = statvfs.f_frsize * statvfs.f_blocks # Size of filesystem in bytes
|
|
|
264 # num_free_bytes = statvfs.f_frsize * statvfs.f_bfree # Actual number of free bytes
|
|
|
265 num_avail_bytes = statvfs.f_frsize * statvfs.f_bavail # Number of free bytes that ordinary users
|
|
|
266 # are allowed to use (excl. reserved space)
|
|
|
267 if (num_avail_bytes < _NumBytesNeededForBuild):
|
|
|
268 raise OSError("There is insufficient space ({:s} bytes)".format(str(num_avail_bytes)) + \
|
|
|
269 " on the device of the destination directory: " + \
|
|
|
270 "{:s}".format(cannonical_destination))
|
|
|
271
|
|
|
272 #Previous code to download and untar. Not using anymore.
|
|
|
273 #full_filepath = os.path.join(destination, src_filename)
|
|
|
274 #
|
|
|
275 #Download ref: https://dzone.com/articles/how-download-file-python
|
|
|
276 #f = urllib2.urlopen(source)
|
|
|
277 #data = f.read()
|
|
|
278 #with open(full_filepath, 'wb') as code:
|
|
|
279 # code.write(data)
|
|
|
280 #
|
|
|
281 #Another way to download:
|
|
|
282 #try:
|
|
|
283 # urllib.urlretrieve(url=source, filename=full_filepath)
|
|
|
284 #
|
|
|
285 #Then untar the file.
|
|
|
286 #try:
|
|
|
287 # tarfile.open(full_filepath, mode='r:*').extractall()
|
|
|
288
|
|
7
|
289 if (download_success_file in orig_files_in_destdir):
|
|
0
|
290 # Since we are redoing the download,
|
|
|
291 # the success file needs to be removed
|
|
|
292 # until the download has succeeded.
|
|
|
293 os.remove(download_success_file_path)
|
|
|
294 # We want to transfer and untar the file without storing the tar file, because that
|
|
|
295 # adds all that much more space to the needed amount of free space on the disk.
|
|
|
296 # Use subprocess to pipe the output of curl into tar.
|
|
6
|
297 command = "curl --silent {:s} | tar -xzf - -C {:s}".format(source, cannonical_destination)
|
|
0
|
298 try: # to send the command that downloads and extracts the file.
|
|
|
299 command_output = subprocess.check_output(command, shell=True)
|
|
|
300 # FIX - not sure check_output is what we want to use. If we want to have an error raised on
|
|
|
301 # any problem, maybe we should not be checking output.
|
|
|
302 except subprocess.CalledProcessError:
|
|
|
303 print "ERROR: Trying to run the following command:\n\t{:s}".format(command)
|
|
|
304 raise
|
|
|
305 else:
|
|
|
306 lib_was_downloaded = True
|
|
|
307
|
|
|
308 # Some code to help us if errors occur.
|
|
|
309 print "\n*******************************\nFinished download and extraction."
|
|
6
|
310 print_directory_contents(cannonical_destination, 2)
|
|
|
311 print "*******************************\n"
|
|
0
|
312
|
|
|
313 newfiles_in_destdir = set(os.listdir(cannonical_destination)) - orig_files_in_destdir
|
|
|
314 if (root_genome_dirname not in newfiles_in_destdir):
|
|
|
315 # Perhaps it has a different name than what we expected it to be.
|
|
|
316 # It will be the file that was not in the directory
|
|
|
317 # before we did the download and extraction.
|
|
|
318 found_filename = None
|
|
|
319 if len(newfiles_in_destdir) == 1:
|
|
|
320 found_filename = newfiles_in_destdir[0]
|
|
|
321 else:
|
|
|
322 for filename in newfiles_in_destdir:
|
|
|
323 # In most cases, there will only be one new file, but some OS's might have created
|
|
|
324 # other files in the directory.
|
|
|
325 # Look for the directory that was downloaded and extracted.
|
|
|
326 # The correct file's name should be a substring of the tar file that was downloaded.
|
|
|
327 if filename in src_filename:
|
|
|
328 found_filename = filename
|
|
|
329 if found_filename is not None:
|
|
|
330 root_genome_dirname = found_filename
|
|
|
331
|
|
|
332 downloaded_directory = "{:s}/{:s}".format(cannonical_destination, root_genome_dirname)
|
|
|
333
|
|
|
334 if (os.path.exists(downloaded_directory)):
|
|
|
335 try:
|
|
|
336 # Create a file to indicate that the download succeeded.
|
|
|
337 subprocess.check_call("touch {:s}".format(download_success_file_path), shell=True)
|
|
|
338 except IOError:
|
|
|
339 print "The download_success file could not be created: " + \
|
|
|
340 "{:s}".format(download_success_file_path)
|
|
|
341 raise
|
|
|
342 # Look for the build directory, or specify the path where it should be placed.
|
|
|
343 if len(os.listdir(downloaded_directory)) == 1:
|
|
|
344 # Then that one file is a subdirectory that should be the downloaded_directory.
|
|
6
|
345 # That is how the plug-n-play directories are structured.
|
|
0
|
346 subdir_filename = os.listdir(downloaded_directory)[0]
|
|
|
347 genome_build_directory = "{:s}/{:s}".format(downloaded_directory, subdir_filename)
|
|
|
348 else:
|
|
6
|
349 # In this case, we have source_data in the directory. The default will be to create
|
|
|
350 # the build directory in the downloaded_directory with the default _CTAT_Build_dirname.
|
|
|
351 # In this case, this directory will not exist yet until the library is built.
|
|
|
352 genome_build_directory = "{:s}/{:s}".format(downloaded_directory, _CTAT_Build_dirname)
|
|
0
|
353 else:
|
|
|
354 raise ValueError("ERROR: Could not find the extracted file in the destination directory:" + \
|
|
|
355 "\n\t{:s}".format(cannonical_destination))
|
|
|
356
|
|
|
357 return (downloaded_directory, download_has_source_data, genome_build_directory, lib_was_downloaded)
|
|
|
358
|
|
|
359 def gmap_the_library(genome_build_directory):
|
|
|
360 # This is the processing that needs to happen for gmap-fusion to work.
|
|
|
361 # genome_build_directory should normally be a fully specified path,
|
|
6
|
362 # though this function should work even if it is relative.
|
|
|
363 # The command prints messages out to stderr, even when there is not an error,
|
|
|
364 # so route stderr to stdout. Otherwise, galaxy thinks an error occurred.
|
|
|
365 command = "gmap_build -D {:s}/ -d ref_genome.fa.gmap -k 13 {:s}/ref_genome.fa 2>&1".format( \
|
|
0
|
366 genome_build_directory, genome_build_directory)
|
|
|
367 try: # to send the gmap_build command.
|
|
|
368 command_output = subprocess.check_output(command, shell=True)
|
|
|
369 except subprocess.CalledProcessError:
|
|
|
370 print "ERROR: While trying to run the gmap_build command on the library:\n\t{:s}".format(command)
|
|
|
371 raise
|
|
|
372 finally:
|
|
|
373 # Some code to help us if errors occur.
|
|
|
374 print "\n*******************************\nAfter running gmap_build."
|
|
6
|
375 print_directory_contents(genome_build_directory, 2)
|
|
|
376 print "*******************************\n"
|
|
0
|
377
|
|
8
|
378 def download_mutation_indexes(source_url, genome_build_directory, force_download):
|
|
|
379 print "\n*****************************************************************"
|
|
|
380 print "* The real mutation indexes have not yet been created. Just testing. *"
|
|
|
381 print "*****************************************************************\n"
|
|
|
382 # It is assumed that this procedure is only called with a valid genome_build_directory.
|
|
|
383 # No checks are made to see whether it exists, whether we can write to it, etc.
|
|
|
384 index_was_downloaded = False
|
|
|
385 if len(source_url.split(":")) == 1:
|
|
|
386 # Then we were given a source_url without a leading https: or similar.
|
|
|
387 # Assume we only were given the filename and that it exists at _CTAT_MutationIndex_URL.
|
|
|
388 source_url = "{:s}/{:s}".format(_CTAT_MutationIndex_URL, source_url)
|
|
|
389
|
|
|
390 print "In download_mutation_indexes(). The source_url is:\n\t{:s}".format(str(source_url))
|
|
|
391
|
|
|
392 # Get the root filename of the Genome Directory.
|
|
|
393 src_filename = source.split("/")[-1]
|
|
|
394 root_genome_dirname = src_filename.split(".")[0]
|
|
|
395 print "The mutation index file to be downloaded and extracted is {:s}".format(src_filename)
|
|
|
396
|
|
|
397 # Get the list of files in the directory,
|
|
|
398 # We use it to check for a previous download or extraction among other things.
|
|
|
399 orig_files_in_destdir = set(os.listdir(genome_build_directory))
|
|
|
400 # See whether the index file has been downloaded already.
|
|
|
401 download_success_file = "{:s}.{:s}".format(root_genome_dirname, _MutationDownloadSuccessFile)
|
|
|
402 download_success_file_path = "{:s}/{:s}".format(genome_build_directory, download_success_file)
|
|
|
403 if ((download_success_file not in orig_files_in_destdir) or force_download):
|
|
|
404 # Check whether there is enough space on the device for the library.
|
|
|
405 statvfs = os.statvfs(genome_build_directory)
|
|
|
406 # fs_size = statvfs.f_frsize * statvfs.f_blocks # Size of filesystem in bytes
|
|
|
407 # num_free_bytes = statvfs.f_frsize * statvfs.f_bfree # Actual number of free bytes
|
|
|
408 num_avail_bytes = statvfs.f_frsize * statvfs.f_bavail # Number of free bytes that ordinary users
|
|
|
409 # are allowed to use (excl. reserved space)
|
|
|
410 if (num_avail_bytes < _NumBytesNeededForIndexes):
|
|
|
411 raise OSError("There is insufficient space ({:s} bytes)".format(str(num_avail_bytes)) + \
|
|
|
412 " for the indexes on the device of the destination directory: " + \
|
|
|
413 "{:s}".format(cannonical_destination))
|
|
|
414 if (download_success_file in orig_files_in_destdir):
|
|
|
415 # Since we are redoing the download,
|
|
|
416 # the success file needs to be removed
|
|
|
417 # until the download has succeeded.
|
|
|
418 os.remove(download_success_file_path)
|
|
|
419 # We want to transfer and untar the file without storing the tar file, because that
|
|
|
420 # adds all that much more space to the needed amount of free space on the disk.
|
|
|
421 # Use subprocess to pipe the output of curl into tar.
|
|
|
422 command = "curl --silent {:s} | tar -xzf - -C {:s}".format(source_url, genome_build_directory)
|
|
|
423 try: # to send the command that downloads and extracts the file.
|
|
|
424 command_output = subprocess.check_output(command, shell=True)
|
|
|
425 # FIX - not sure check_output is what we want to use. If we want to have an error raised on
|
|
|
426 # any problem, maybe we should not be checking output.
|
|
|
427 except subprocess.CalledProcessError:
|
|
|
428 print "ERROR: Trying to run the following command:\n\t{:s}".format(command)
|
|
|
429 raise
|
|
|
430 else:
|
|
|
431 index_was_downloaded = True
|
|
|
432 # Some code to help us if errors occur.
|
|
|
433 print "/n*********************************************************"
|
|
|
434 print "* Finished download and extraction of Mutation Indexes. *"
|
|
|
435 print_directory_contents(genome_build_directory, 2)
|
|
|
436 print "*********************************************************\n"
|
|
|
437 try:
|
|
|
438 # Create a file to indicate that the download succeeded.
|
|
|
439 subprocess.check_call("touch {:s}".format(download_success_file_path), shell=True)
|
|
|
440 except IOError:
|
|
|
441 print "The download_success file could not be created: " + \
|
|
|
442 "{:s}".format(download_success_file_path)
|
|
|
443 raise
|
|
|
444 return index_was_downloaded
|
|
|
445
|
|
0
|
446 def build_the_library(genome_source_directory, genome_build_directory, build, gmap_build):
|
|
|
447 """ genome_source_directory is the location of the source_data needed to build the library.
|
|
|
448 Normally it is fully specified, but could be relative.
|
|
|
449 genome_build_directory is the location where the library will be built.
|
|
|
450 It can be relative to the current working directory or an absolute path.
|
|
|
451 build specifies whether to run prep_genome_lib.pl even if it was run before.
|
|
|
452 gmap_build specifies whether to run gmap_build or not.
|
|
|
453
|
|
|
454 Following was the old way to do it. Before FusionFilter 0.5.0.
|
|
|
455 prep_genome_lib.pl \
|
|
|
456 --genome_fa ref_genome.fa \
|
|
|
457 --gtf ref_annot.gtf \
|
|
|
458 --blast_pairs blast_pairs.gene_syms.outfmt6.gz \
|
|
|
459 --fusion_annot_lib fusion_lib.dat.gz
|
|
|
460 --output_dir ctat_genome_lib_build_dir
|
|
|
461 index_pfam_domain_info.pl \
|
|
|
462 --pfam_domains PFAM.domtblout.dat.gz \
|
|
|
463 --genome_lib_dir ctat_genome_lib_build_dir
|
|
|
464 gmap_build -D ctat_genome_lib_build_dir -d ref_genome.fa.gmap -k 13 ctat_genome_lib_build_dir/ref_genome.fa"
|
|
|
465 """
|
|
7
|
466
|
|
8
|
467 # Get the root filename of the Genome Directory.
|
|
|
468 src_filename = genome_source_directory.split("/")[-1]
|
|
|
469 root_genome_dirname = src_filename.split(".")[0]
|
|
7
|
470 print "Building the CTAT Genome Resource Library from source data at:\n\t{:s}".format(genome_source_directory)
|
|
8
|
471 # See whether the library has been built already. The success file is written into the source directory.
|
|
|
472 files_in_sourcedir = set(os.listdir(genome_source_directory))
|
|
|
473 build_success_file = "{:s}.{:s}".format(root_genome_dirname, _LibBuiltSuccessFile)
|
|
|
474 build_success_file_path = "{:s}/{:s}".format(genome_source_directory, build_success_file)
|
|
|
475 if (genome_source_directory != "" ) and \
|
|
|
476 ((build_success_file not in files_in_sourcedir) or build):
|
|
0
|
477 if os.path.exists(genome_source_directory):
|
|
|
478 os.chdir(genome_source_directory)
|
|
8
|
479 if (build_success_file in files_in_sourcedir):
|
|
|
480 # Since we are redoing the build,
|
|
|
481 # the success file needs to be removed
|
|
|
482 # until the build has succeeded.
|
|
|
483 os.remove(build_success_file_path)
|
|
6
|
484 # Create the command that builds the Genome Resource Library form the source data.
|
|
0
|
485 command = "prep_genome_lib.pl --genome_fa ref_genome.fa --gtf ref_annot.gtf " + \
|
|
|
486 "--pfam_db PFAM.domtblout.dat.gz " + \
|
|
8
|
487 "--output_dir {:s} ".format(genome_build_directory)
|
|
6
|
488 found_HumanFusionLib = False
|
|
|
489 HumanFusionLib_filename = "NoFileFound"
|
|
|
490 for filename in os.listdir(genome_source_directory):
|
|
|
491 # At the time this was written, the filename was CTAT_HumanFusionLib.v0.1.0.dat.gz
|
|
|
492 # We only check the prefix, in case other versions are used later.
|
|
|
493 # I assume there is only one in the directory, but if there are more than one,
|
|
|
494 # the later one, alphabetically, will be used.
|
|
|
495 if filename.split(".")[0] == _CTAT_HumanFusionLib_FilenamePrefix:
|
|
|
496 found_HumanFusionLib = True
|
|
|
497 filename_of_HumanFusionLib = filename
|
|
|
498 if found_HumanFusionLib:
|
|
|
499 # The mouse genomes do not have a fusion_annot_lib
|
|
|
500 # so only add the following for Human genomes.
|
|
|
501 command += "--fusion_annot_lib {:s} ".format(filename_of_HumanFusionLib) + \
|
|
|
502 "--annot_filter_rule AnnotFilterRule.pm "
|
|
0
|
503 if gmap_build:
|
|
|
504 command += "--gmap_build "
|
|
6
|
505 # Send stderr of the command to stdout, because some functions may write to stderr,
|
|
|
506 # even though no error has occurred. We will depend on error code return in order
|
|
|
507 # to know if an error occurred.
|
|
|
508 command += " 2>&1"
|
|
0
|
509 try: # to send the prep_genome_lib command.
|
|
|
510 command_output = subprocess.check_call(command, shell=True)
|
|
|
511 except subprocess.CalledProcessError:
|
|
|
512 print "ERROR: While trying to run the prep_genome_lib.pl command " + \
|
|
|
513 "on the CTAT Genome Resource Library:\n\t{:s}".format(command)
|
|
|
514 raise
|
|
|
515 finally:
|
|
|
516 # Some code to help us if errors occur.
|
|
6
|
517 print "\n*******************************"
|
|
|
518 print "Contents of Genome Source Directory {:s}:".format(genome_source_directory)
|
|
|
519 print_directory_contents(genome_source_directory, 2)
|
|
|
520 print "\nContents of Genome Build Directory {:s}:".format(genome_build_directory)
|
|
|
521 print_directory_contents(genome_build_directory, 2)
|
|
|
522 print "*******************************\n"
|
|
0
|
523 else:
|
|
|
524 raise ValueError("Cannot build the CTAT Genome Resource Library. " + \
|
|
|
525 "The source directory does not exist:\n\t{:s}".format(genome_source_directory))
|
|
|
526 elif gmap_build:
|
|
|
527 gmap_the_library(genome_build_directory)
|
|
8
|
528 try:
|
|
|
529 # Create a file to indicate that the build succeeded.
|
|
|
530 subprocess.check_call("touch {:s}".format(build_success_file_path), shell=True)
|
|
|
531 except IOError:
|
|
|
532 print "The download_success file could not be created: " + \
|
|
|
533 "{:s}".format(build_success_file_path)
|
|
|
534 raise
|
|
0
|
535
|
|
6
|
536 def search_for_genome_build_dir(top_dir_path):
|
|
|
537 # If we do not download the directory, the topdir_path could be the
|
|
|
538 # location of the genome resource library, but we also want to allow the
|
|
|
539 # user to give the same value for top_dir_path that they do when a
|
|
|
540 # build happens, so we need to handle all three cases:
|
|
|
541 # 1) Is the top_dir_path the build directory,
|
|
|
542 # 2) or is it inside of the given directory,
|
|
|
543 # 3) or is it inside a subdirectory of the given directory.
|
|
|
544 # The source_data downloads are built to a directory named _CTAT_Build_dirname,
|
|
|
545 # and the plug-n-play downloads contain a sub-directory named _CTAT_Build_dirname.
|
|
7
|
546 # We also look for the genome name and return that, if we find it in the
|
|
|
547 # directory name of the directory holding the build directory.
|
|
|
548 top_dir_full_path = os.path.realpath(top_dir_path)
|
|
6
|
549 genome_build_directory = None
|
|
7
|
550 genome_name_from_dirname = None
|
|
6
|
551 print_warning = False
|
|
|
552
|
|
7
|
553 if not os.path.exists(top_dir_full_path):
|
|
6
|
554 raise ValueError("Cannot find the CTAT Genome Resource Library. " + \
|
|
7
|
555 "The given directory does not exist:\n\t{:s}".format(top_dir_full_path))
|
|
|
556 elif not os.path.isdir(top_dir_full_path):
|
|
6
|
557 raise ValueError("Cannot find the CTAT Genome Resource Library. " + \
|
|
7
|
558 "The given directory is not a directory:\n\t{:s}".format(top_dir_full_path))
|
|
|
559 if top_dir_full_path.split("/")[-1] == _CTAT_Build_dirname:
|
|
|
560 print "Build directory is: {:s}".format(top_dir_full_path)
|
|
6
|
561 # The top_dir_path is the path to the genome_build_directory.
|
|
7
|
562 genome_build_directory = top_dir_full_path
|
|
6
|
563 else:
|
|
|
564 # Look for it inside of the top_dir_path directory.
|
|
7
|
565 print "Looking inside of: {:s}".format(top_dir_full_path)
|
|
|
566 top_dir_contents = os.listdir(top_dir_full_path)
|
|
6
|
567 if (_CTAT_Build_dirname in top_dir_contents):
|
|
|
568 # The genome_build_directory is inside of the top_dir_path directory.
|
|
|
569 print "1. Found it."
|
|
7
|
570 genome_build_directory = "{:s}/{:s}".format(top_dir_full_path,_CTAT_Build_dirname)
|
|
6
|
571 else:
|
|
|
572 # Find all subdirectories containing the _CTAT_Build_dirname or the _CTAT_RefGenome_Filename.
|
|
|
573 # Look down the directory tree two levels.
|
|
|
574 build_dirs_in_subdirs = list()
|
|
|
575 subdirs_with_genome_files = list()
|
|
|
576 build_dirs_in_sub_subdirs = list()
|
|
|
577 sub_subdirs_with_genome_files = list()
|
|
7
|
578 subdirs = [entry for entry in top_dir_contents if (os.path.isdir("{:s}/{:s}".format(top_dir_full_path,entry)))]
|
|
6
|
579 for subdir in subdirs:
|
|
7
|
580 subdir_path = "{:s}/{:s}".format(top_dir_full_path, subdir)
|
|
6
|
581 subdir_path_contents = os.listdir(subdir_path)
|
|
|
582 # print "Is it one of:\n\t" + "\n\t".join(subdir_path_contents)
|
|
|
583 if (_CTAT_Build_dirname in subdir_path_contents):
|
|
|
584 # The genome_build_directory is inside of the subdir_path directory.
|
|
|
585 print "2a, Found one."
|
|
|
586 build_dirs_in_subdirs.append("{:s}/{:s}".format(subdir_path, _CTAT_Build_dirname))
|
|
|
587 if (_CTAT_RefGenome_Filename in subdir_path_contents):
|
|
|
588 subdirs_with_genome_files.append(subdir_path)
|
|
|
589 # Since we are already looping, loop through all dirs one level deeper as well.
|
|
|
590 sub_subdirs = [entry for entry in subdir_path_contents if (os.path.isdir("{:s}/{:s}".format(subdir_path,entry)))]
|
|
|
591 for sub_subdir in sub_subdirs:
|
|
|
592 sub_subdir_path = "{:s}/{:s}".format(subdir_path, sub_subdir)
|
|
|
593 sub_subdir_path_contents = os.listdir(sub_subdir_path)
|
|
|
594 # print "Is it one of:\n\t" + "\n\t".join(sub_subdir_path_contents)
|
|
|
595 if (_CTAT_Build_dirname in sub_subdir_path_contents):
|
|
|
596 # The genome_build_directory is inside of the sub_subdir_path directory.
|
|
|
597 print "3a. Found one."
|
|
|
598 build_dirs_in_sub_subdirs.append("{:s}/{:s}".format(sub_subdir_path, _CTAT_Build_dirname))
|
|
|
599 if (_CTAT_RefGenome_Filename in sub_subdir_path_contents):
|
|
|
600 sub_subdirs_with_genome_files.append(sub_subdir_path)
|
|
|
601 # Hopefully there is one and only one found build directory.
|
|
|
602 # If none are found we check for a directory containing the genome reference file,
|
|
|
603 # but the build process sometimes causes more than one directory to have a copy,
|
|
|
604 # so finding that file is not a sure thing.
|
|
|
605 if (len(build_dirs_in_subdirs) + len(build_dirs_in_sub_subdirs)) > 1:
|
|
|
606 print "\n***************************************"
|
|
|
607 print "Found multiple CTAT Genome Resource Libraries " + \
|
|
7
|
608 "in the given directory:\n\t{:s}".format(top_dir_full_path)
|
|
|
609 print_directory_contents(top_dir_full_path, 2)
|
|
6
|
610 print "***************************************\n"
|
|
|
611 raise ValueError("Found multiple CTAT Genome Resource Libraries " + \
|
|
7
|
612 "in the given directory:\n\t{:s}".format(top_dir_full_path))
|
|
6
|
613 elif len(build_dirs_in_subdirs) == 1:
|
|
|
614 # The genome_build_directory is inside of the subdir_path directory.
|
|
|
615 print "2b, Found it."
|
|
|
616 genome_build_directory = build_dirs_in_subdirs[0]
|
|
|
617 elif len(build_dirs_in_sub_subdirs) == 1:
|
|
|
618 # The genome_build_directory is inside of the subdir_path directory.
|
|
|
619 print "3b, Found it."
|
|
|
620 genome_build_directory = build_dirs_in_sub_subdirs[0]
|
|
|
621 elif (len(sub_subdirs_with_genome_files) + len(subdirs_with_genome_files)) > 1:
|
|
|
622 print "\n***************************************"
|
|
|
623 print "Unable to find CTAT Genome Resource Library " + \
|
|
7
|
624 "in the given directory:\n\t{:s}".format(top_dir_full_path)
|
|
6
|
625 print "And multiple directories contain {:s}".format(_CTAT_RefGenome_Filename)
|
|
7
|
626 print_directory_contents(top_dir_full_path, 2)
|
|
6
|
627 print "***************************************\n"
|
|
|
628 raise ValueError("Unable to find CTAT Genome Resource Library " + \
|
|
7
|
629 "in the given directory:\n\t{:s}".format(top_dir_full_path))
|
|
6
|
630 elif (len(sub_subdirs_with_genome_files) == 1):
|
|
|
631 print "3c, Maybe found it."
|
|
|
632 genome_build_directory = sub_subdirs_with_genome_files[0]
|
|
|
633 print_warning = True
|
|
|
634 elif (len(subdirs_with_genome_files) == 1):
|
|
|
635 print "2c, Maybe found it."
|
|
|
636 genome_build_directory = subdirs_with_genome_files[0]
|
|
|
637 print_warning = True
|
|
|
638 elif (_CTAT_RefGenome_Filename in top_dir_contents):
|
|
|
639 print "1c. Maybe found it."
|
|
7
|
640 genome_build_directory = top_dir_full_path
|
|
6
|
641 print_warning = True
|
|
|
642 else:
|
|
|
643 print "\n***************************************"
|
|
|
644 print "Unable to find CTAT Genome Resource Library " + \
|
|
7
|
645 "in the given directory:\n\t{:s}".format(top_dir_full_path)
|
|
|
646 print_directory_contents(top_dir_full_path, 2)
|
|
6
|
647 print "***************************************\n"
|
|
|
648 raise ValueError("Unable to find CTAT Genome Resource Library " + \
|
|
7
|
649 "in the given directory:\n\t{:s}".format(top_dir_full_path))
|
|
6
|
650 # end else
|
|
|
651 # Check if the CTAT Genome Resource Lib has anything in it (and specifically ref_genome.fa).
|
|
|
652 if (genome_build_directory is None):
|
|
|
653 print "\n***************************************"
|
|
|
654 print "Cannot find the CTAT Genome Resource Library " + \
|
|
7
|
655 "in the given directory:\n\t{:s}".format(top_dir_full_path)
|
|
|
656 print_directory_contents(top_dir_full_path, 2)
|
|
6
|
657 print "***************************************\n"
|
|
|
658 raise ValueError("Cannot find the CTAT Genome Resource Library " + \
|
|
7
|
659 "in the given directory:\n\t{:s}".format(top_dir_full_path))
|
|
|
660 else:
|
|
|
661 if (_CTAT_RefGenome_Filename not in os.listdir(genome_build_directory)):
|
|
|
662 print "\n***************************************"
|
|
|
663 print "\nWARNING: Cannot find Genome Reference file {:s}".format(_CTAT_RefGenome_Filename) + \
|
|
|
664 "in the genome build directory:\n\t{:s}".format(genome_build_directory)
|
|
|
665 print_directory_contents(genome_build_directory, 2)
|
|
|
666 print "***************************************\n"
|
|
|
667 if print_warning and genome_build_directory:
|
|
|
668 print "\n***************************************"
|
|
|
669 print "\nWARNING: Cannot find the CTAT Genome Resource Library," + \
|
|
|
670 "but found a {:s} file, so set its directory as the library.".format(_CTAT_RefGenome_Filename)
|
|
|
671 print "This my not be the correct directory:\n\t{:s}".format(genome_build_directory)
|
|
|
672 print_directory_contents(genome_build_directory, 2)
|
|
|
673 print "***************************************\n"
|
|
6
|
674 return genome_build_directory
|
|
|
675
|
|
7
|
676 def find_genome_name_in_path(path):
|
|
|
677 # The form of the genome name in directory names (if present in the path) looks like:
|
|
|
678 # GRCh37_v19_CTAT_lib_Feb092018
|
|
|
679 # Mouse_M16_CTAT_lib_Feb202018
|
|
|
680 genome_name = None
|
|
|
681 if (path is not None) and (path != ""):
|
|
|
682 for element in path.split("/"):
|
|
|
683 # print "Looking for genome name in {:s}.".format(element)
|
|
|
684 if (element[0:len(_CTAT_MouseGenome_Prefix)] == _CTAT_MouseGenome_Prefix) \
|
|
|
685 or (element[0:len(_CTAT_HumanGenome_Prefix)] == _CTAT_HumanGenome_Prefix):
|
|
|
686 # Remove any extension that might be in the filename.
|
|
|
687 genome_name = element.split(".")[0]
|
|
|
688 return genome_name
|
|
|
689
|
|
0
|
690 def main():
|
|
|
691 #Parse Command Line
|
|
|
692 parser = argparse.ArgumentParser()
|
|
6
|
693 parser.add_argument('-s', '--source_url', default='', \
|
|
|
694 help='This is the url of a file with the data. ' + \
|
|
|
695 'They come from https://data.broadinstitute.org/Trinity/CTAT_RESOURCE_LIB/.')
|
|
|
696 parser.add_argument('-n', '--display_name', default='', \
|
|
0
|
697 help='Is used as the display name for the entry of this Genome Resource Library in the data table.')
|
|
|
698 parser.add_argument('-o', '--output_filename', \
|
|
|
699 help='Name of the output file, where the json dictionary will be written.')
|
|
8
|
700 parser.add_argument('-d', '--force_download', \
|
|
6
|
701 help='Forces download of the Genome Resource Library, even if previously downloaded.', action='store_true')
|
|
8
|
702 parser.add_argument('-b', '--build', \
|
|
0
|
703 help='Forces build/rebuild the Genome Resource Library, even if previously built. ' + \
|
|
6
|
704 'Must have downloaded source_data for this to work.', action='store_true')
|
|
8
|
705 parser.add_argument('-g', '--gmap_build', \
|
|
0
|
706 help='Must be selected if you want the library to be gmapped. ' + \
|
|
6
|
707 'Will force gmap_build of the Genome Resource Library, even if previously gmapped.', action='store_true')
|
|
8
|
708 parser.add_argument('-m', '--download_mutation_indexes', default='', \
|
|
|
709 help='Set to the url of the mutation indexes for the Library. ' + \
|
|
|
710 'Will download mutation indexes into the Genome Resource Library.', action='store_true')
|
|
|
711 parser.add_argument('-f', '--force_mutation_indexes_download', \
|
|
|
712 help='Forces the mutation indexes to download, ' + \
|
|
|
713 'even if previously downloaded to this Library.', action='store_true')
|
|
6
|
714 requiredNamed = parser.add_argument_group('required named arguments')
|
|
|
715 requiredNamed.add_argument('-p', '--destination_path', required=True, \
|
|
|
716 help='Full path of the CTAT Resource Library location or destination, either where it is, or where it will be placed.')
|
|
0
|
717 args = parser.parse_args()
|
|
|
718
|
|
|
719 # All of the input parameters are written by default to the output file prior to
|
|
|
720 # this program being called.
|
|
|
721 # But I do not get input values from the json file, but rather from command line.
|
|
|
722 # Just leaving the following code as a comment, in case it might be useful to someone later.
|
|
|
723 # params = from_json_string(open(filename).read())
|
|
|
724 # target_directory = params['output_data'][0]['extra_files_path']
|
|
|
725 # os.mkdir(target_directory)
|
|
|
726
|
|
4
|
727 print "The value of source_url argument is:\n\t{:s}".format(str(args.source_url))
|
|
|
728
|
|
6
|
729 # FIX - not sure lib_was_downloaded actually serves a purpose...
|
|
8
|
730 # The original intent was to check whether an attempted download actually succeeded before proceeding,
|
|
|
731 # but I believe that in those situations, currently, exceptions are raised.
|
|
|
732 # FIX - Need to double check that. Sometimes, although we are told to download, the function
|
|
|
733 # could find that the files are already there, successfully downloaded from a prior attempt,
|
|
|
734 # and does not re-download them.
|
|
0
|
735 lib_was_downloaded = False
|
|
8
|
736 lib_was_built = False
|
|
0
|
737 download_has_source_data = False
|
|
|
738 downloaded_directory = None
|
|
|
739 genome_build_directory = None
|
|
|
740 # FIX - need to make sure we are handling all "possible" combinations of arguments.
|
|
|
741 # Probably would be good if we could simplify/remove some of them.
|
|
6
|
742 # But I think the current interface is using them all.
|
|
0
|
743 if (args.source_url != ""):
|
|
|
744 downloaded_directory, download_has_source_data, genome_build_directory, lib_was_downloaded = \
|
|
|
745 download_from_BroadInst(source=args.source_url, \
|
|
|
746 destination=args.destination_path, \
|
|
|
747 force_download=args.force_download)
|
|
|
748 else:
|
|
9
|
749 if (args.build):
|
|
|
750 # Then the user wants to build the directory from the data
|
|
|
751 # in the location that was given in destination_path.
|
|
|
752 downloaded_directory = args.destination_path
|
|
6
|
753 genome_build_directory = search_for_genome_build_dir(args.destination_path)
|
|
0
|
754
|
|
9
|
755 print "\nThe location of the downloaded_directory is {:s}.\n".format(str(downloaded_directory))
|
|
0
|
756 print "\nThe location of the CTAT Genome Resource Library is {:s}.\n".format(genome_build_directory)
|
|
|
757
|
|
|
758 # FIX - We should leave a file indicating build success the same way we do for download success.
|
|
8
|
759 # To take out builds for testing, comment out the lines that do the building.
|
|
|
760 # The command that builds the ctat genome library also has an option for building the gmap indexes.
|
|
|
761 # That is why the gmap_build value is sent to build_the_library(), but if we are not building the
|
|
|
762 # library, the user might still be asking for a gmap_build. That is done after rechecking for the
|
|
|
763 # genome_build_directory.
|
|
9
|
764 if (downloaded_directory is not None) and (download_has_source_data or args.build):
|
|
7
|
765 build_the_library(downloaded_directory, genome_build_directory, True, args.gmap_build)
|
|
8
|
766 lib_was_built = True
|
|
9
|
767 elif downloaded_directory is None:
|
|
|
768 print "No directory was downloaded and there is no source data, " + \
|
|
|
769 "so the Resource Library was not built (it may already be built)."
|
|
6
|
770 # The following looks to see if the library actually exists after the build,
|
|
|
771 # and raises an error if it cannot find the library files.
|
|
|
772 # The reassignment of genome_build_directory should be superfluous,
|
|
8
|
773 # since genome_build_directory should already point to the correct directory,
|
|
6
|
774 # unless I made a mistake in the build code.
|
|
|
775 genome_build_directory = search_for_genome_build_dir(genome_build_directory)
|
|
|
776
|
|
8
|
777 if (args.gmap_build and not lib_was_built):
|
|
|
778 # If we did not build the genome resource library
|
|
|
779 # the user might still be asking for a gmap_build.
|
|
|
780 gmap_the_library(genome_build_directory)
|
|
|
781
|
|
|
782 if (args.download_mutation_indexes != ""):
|
|
|
783 download_mutation_indexes(source_url=args.download_mutation_indexes, \
|
|
|
784 genome_build_directory=genome_build_directory, \
|
|
|
785 force_download=args.force_mutation_indexes_download)
|
|
|
786
|
|
7
|
787 # Need to get the genome name.
|
|
|
788 genome_name = find_genome_name_in_path(args.source_url)
|
|
|
789 if genome_name is None:
|
|
|
790 genome_name = find_genome_name_in_path(genome_build_directory)
|
|
|
791 if genome_name is None:
|
|
|
792 genome_name = find_genome_name_in_path(downloaded_directory)
|
|
|
793 if genome_name is None:
|
|
|
794 genome_name = find_genome_name_in_path(args.destination_path)
|
|
|
795 if genome_name is None:
|
|
|
796 genome_name = find_genome_name_in_path(args.display_name)
|
|
|
797 if genome_name is None:
|
|
|
798 genome_name = _CTAT_ResourceLib_DefaultGenome
|
|
|
799 print "WARNING: We could not find a genome name in any of the directory paths."
|
|
0
|
800
|
|
|
801 # Determine the display_name for the library.
|
|
|
802 if (args.display_name is None) or (args.display_name == ""):
|
|
7
|
803 # Create the display_name from the genome_name.
|
|
|
804 display_name = _CTAT_ResourceLib_DisplayNamePrefix + genome_name
|
|
0
|
805 else:
|
|
|
806 display_name = _CTAT_ResourceLib_DisplayNamePrefix + args.display_name
|
|
|
807 display_name = display_name.replace(" ","_")
|
|
|
808
|
|
|
809 # Create a unique_id for the library.
|
|
|
810 datetime_stamp = datetime.now().strftime("_%Y_%m_%d_%H_%M_%S_%f")
|
|
8
|
811 unique_id = genome_name + "." + datetime_stamp
|
|
0
|
812
|
|
6
|
813 print "The Genome Resource Library's display_name will be set to: {:s}\n".format(display_name)
|
|
0
|
814 print "Its unique_id will be set to: {:s}\n".format(unique_id)
|
|
|
815 print "Its dir_path will be set to: {:s}\n".format(genome_build_directory)
|
|
|
816
|
|
|
817 data_manager_dict = {}
|
|
|
818 data_manager_dict['data_tables'] = {}
|
|
|
819 data_manager_dict['data_tables']['ctat_genome_resource_libs'] = []
|
|
|
820 data_table_entry = dict(value=unique_id, name=display_name, path=genome_build_directory)
|
|
|
821 data_manager_dict['data_tables']['ctat_genome_resource_libs'].append(data_table_entry)
|
|
|
822
|
|
|
823 # Temporarily the output file's dictionary is written for debugging:
|
|
|
824 print "The dictionary for the output file is:\n\t{:s}".format(str(data_manager_dict))
|
|
|
825 # Save info to json file. This is used to transfer data from the DataManager tool, to the data manager,
|
|
|
826 # which then puts it into the correct .loc file (I think).
|
|
|
827 # Comment out the following line when testing without galaxy package.
|
|
|
828 open(args.output_filename, 'wb').write(to_json_string(data_manager_dict))
|
|
|
829
|
|
|
830 if __name__ == "__main__":
|
|
|
831 main()
|
