|
0
|
1 #!/usr/bin/env python
|
|
|
2 # ref: https://galaxyproject.org/admin/tools/data-managers/how-to/define/
|
|
|
3
|
|
|
4 # Rewritten by H.E. Cicada Brokaw Dennis from a source downloaded from the toolshed and
|
|
|
5 # other example code on the web.
|
|
|
6 # This now allows downloading of a user selected library
|
|
|
7 # but only from the CTAT Genome Resource Library website.
|
|
|
8 # Ultimately we might want to allow the user to specify any location
|
|
|
9 # from which to download.
|
|
|
10 # Users can create or download other libraries and use this tool to add them if they don't want
|
|
|
11 # to add them by hand.
|
|
|
12
|
|
|
13 import argparse
|
|
|
14 import os
|
|
|
15 #import tarfile
|
|
|
16 #import urllib
|
|
|
17 import subprocess
|
|
|
18
|
|
|
19 # Comment out the following line when testing without galaxy package.
|
|
|
20 from galaxy.util.json import to_json_string
|
|
|
21 # The following is not being used, but leaving as info
|
|
|
22 # in case we ever want to get input values using json.
|
|
|
23 # from galaxy.util.json import from_json_string
|
|
|
24
|
|
|
25 # datetime.now() is used to create the unique_id
|
|
|
26 from datetime import datetime
|
|
|
27
|
|
|
28 # The FileListParser is used by get_ctat_genome_filenames(),
|
|
|
29 # which is called by the Data Manager interface (.xml file) to get
|
|
|
30 # the filenames that are available online at broadinstitute.org
|
|
|
31 # Not sure best way to do it.
|
|
|
32 # This object uses HTMLParser to look through the html
|
|
|
33 # searching for the filenames within anchor tags.
|
|
|
34 import urllib2
|
|
|
35 from HTMLParser import HTMLParser
|
|
|
36
|
|
|
37 _CTAT_ResourceLib_URL = 'https://data.broadinstitute.org/Trinity/CTAT_RESOURCE_LIB/'
|
|
6
|
38 _CTAT_Build_dirname = 'ctat_genome_lib_build_dir'
|
|
0
|
39 _CTAT_ResourceLib_DisplayNamePrefix = 'CTAT_GenomeResourceLib_'
|
|
|
40 _CTAT_ResourceLib_DefaultGenome = 'Unspecified_Genome'
|
|
6
|
41 _CTAT_HumanFusionLib_FilenamePrefix = 'CTAT_HumanFusionLib'
|
|
|
42 _CTAT_RefGenome_Filename = 'ref_genome.fa'
|
|
0
|
43 _NumBytesNeededForBuild = 64424509440 # 60 Gigabytes. FIX - This might not be correct.
|
|
|
44 _Download_TestFile = "write_testfile.txt"
|
|
|
45 _DownloadSuccessFile = 'download_succeeded.txt'
|
|
|
46
|
|
|
47 class FileListParser(HTMLParser):
|
|
|
48 def __init__(self):
|
|
|
49 # Have to use direct call to super class rather than using super():
|
|
|
50 # super(FileListParser, self).__init__()
|
|
|
51 # because HTMLParser is an "old style" class and its inheritance chain does not include object.
|
|
|
52 HTMLParser.__init__(self)
|
|
|
53 self.urls = set()
|
|
|
54 def handle_starttag(self, tag, attrs):
|
|
|
55 # Look for filename references in anchor tags and add them to urls.
|
|
|
56 if tag == "a":
|
|
|
57 # The tag is an anchor tag.
|
|
|
58 for attribute in attrs:
|
|
|
59 # print "Checking: {:s}".format(str(attribute))
|
|
|
60 if attribute[0] == "href":
|
|
|
61 # Does the href have a tar.gz in it?
|
|
|
62 if ("tar.gz" in attribute[1]) and ("md5" not in attribute[1]):
|
|
|
63 # Add the value to urls.
|
|
|
64 self.urls.add(attribute[1])
|
|
|
65 # End of class FileListParser
|
|
|
66
|
|
|
67 def get_ctat_genome_urls():
|
|
|
68 # open the url and retrieve the urls of the files in the directory.
|
|
|
69 resource = urllib2.urlopen(_CTAT_ResourceLib_URL)
|
|
|
70 theHTML = resource.read()
|
|
|
71 filelist_parser = FileListParser()
|
|
|
72 filelist_parser.feed(theHTML)
|
|
|
73 # For dynamic options need to return an interable with contents that are tuples with 3 items.
|
|
|
74 # Item one is a string that is the display name put into the option list.
|
|
|
75 # Item two is the value that is put into the parameter associated with the option list.
|
|
|
76 # Item three is a True or False value, indicating whether the item is selected.
|
|
|
77 options = []
|
|
|
78 for i, url in enumerate(filelist_parser.urls):
|
|
5
|
79 # The urls should look like:
|
|
0
|
80 # https://data.broadinstitute.org/Trinity/CTAT_RESOURCE_LIB/GRCh37_v19_CTAT_lib_Feb092018.plug-n-play.tar.gz
|
|
|
81 # https://data.broadinstitute.org/Trinity/CTAT_RESOURCE_LIB/Mouse_M16_CTAT_lib_Feb202018.source_data.tar.gz
|
|
5
|
82 # But is actuality, they are coming in looking like:
|
|
|
83 # GRCh37_v19_CTAT_lib_Feb092018.plug-n-play.tar.gz
|
|
|
84 # Mouse_M16_CTAT_lib_Feb202018.source_data.tar.gz
|
|
|
85 # Write code to handle both situations, or an ftp: url.
|
|
|
86 if (url.split(":")[0] == "http") or (url.split(":")[0] == "https") or (url.split(":")[0] == "ftp"):
|
|
|
87 full_url_path = url
|
|
|
88 else:
|
|
|
89 # Assume the path is relative to the page location.
|
|
|
90 full_url_path = "{:s}/{:s}".format(_CTAT_ResourceLib_URL, url)
|
|
0
|
91 filename = url.split("/")[-1]
|
|
5
|
92
|
|
0
|
93 if filename.split("_")[0] != "Mouse":
|
|
|
94 # Take out the mouse genome options for now.
|
|
|
95 # The mouse genome option is not handled correctly yet
|
|
5
|
96 options.append((filename, full_url_path, i == 0))
|
|
|
97 options.sort() # So the list will be in alphabetical order.
|
|
0
|
98 # return a tuple of the urls
|
|
4
|
99 print "The list being returned as options is:"
|
|
|
100 print "{:s}\n".format(str(options))
|
|
0
|
101 return options
|
|
|
102
|
|
|
103 # The following was used by the example program to get input parameters through the json.
|
|
|
104 # Just leaving here for reference.
|
|
|
105 # We are getting all of our parameter values through command line arguments.
|
|
|
106 #def get_reference_id_name(params):
|
|
|
107 # genome_id = params['param_dict']['genome_id']
|
|
|
108 # genome_name = params['param_dict']['genome_name']
|
|
|
109 # return genome_id, genome_name
|
|
|
110 #
|
|
|
111 #def get_url(params):
|
|
|
112 # trained_url = params['param_dict']['trained_url']
|
|
|
113 # return trained_url
|
|
|
114
|
|
6
|
115 def print_directory_contents(dir_path, num_levels):
|
|
|
116 if num_levels > 0:
|
|
|
117 if os.path.exists(dir_path) and os.path.isdir(dir_path):
|
|
|
118 print "\nDirectory {:s}:".format(dir_path)
|
|
|
119 subprocess.call("ls -la {:s} 2>&1".format(dir_path), shell=True)
|
|
|
120 else:
|
|
|
121 print "Path either does not exist, or is not a directory:\n\t{:s}.".format(dir_path)
|
|
|
122 if num_levels > 1:
|
|
|
123 for filename in os.listdir(dir_path):
|
|
|
124 filename_path = "{:s}/{:s}".format(dir_path, filename)
|
|
|
125 if os.path.exists(filename_path) and os.path.isdir(filename_path):
|
|
|
126 print_directory_contents(filename_path, num_levels-1)
|
|
|
127
|
|
0
|
128 def download_from_BroadInst(source, destination, force_download):
|
|
|
129 # Input Parameters
|
|
|
130 # source is the full URL of the file we want to download.
|
|
|
131 # It should look something like:
|
|
|
132 # https://data.broadinstitute.org/Trinity/CTAT_RESOURCE_LIB/GRCh37_v19_CTAT_lib_Feb092018.plug-n-play.tar.gz
|
|
|
133 # destination is the location where the source file will be unarchived.
|
|
|
134 # Relative paths are expanded using the current working directory, so within Galaxy,
|
|
|
135 # it is best to send in absolute fully specified path names so you know to where
|
|
|
136 # the source file going to be extracted.
|
|
|
137 # force_download will cause a new download and extraction to occur, even if the destination
|
|
|
138 # has a file in it indicating that a previous download succeeded.
|
|
|
139 #
|
|
|
140 # Returns the following:
|
|
|
141 # return (downloaded_directory, download_has_source_data, genome_build_directory, lib_was_downloaded)
|
|
|
142 # downloaded_directory
|
|
|
143 # The directory which was created as a subdirectory of the destination directory
|
|
|
144 # when the download occurred, or if there was no download,
|
|
|
145 # possibly the same directory as destination, if that is where the data resides.
|
|
|
146 # download_has_source_data
|
|
|
147 # Is a boolean indicating whether the source file was "source_data" or was "plug-n-play".
|
|
|
148 # genome_build_directory
|
|
|
149 # The directory where the genome resource library is or where it should be built.
|
|
|
150 # It can be the same as the downloaded directory, but is sometimes a subdirectory of it.
|
|
|
151 # lib_was_downloaded
|
|
|
152 # Since it doesn't always do the download, the function returns whether download occurred.
|
|
|
153 lib_was_downloaded = False
|
|
|
154
|
|
4
|
155 print "In download_from_BroadInst(). The source_url is:\n\t{:s}".format(str(source))
|
|
|
156
|
|
0
|
157 # Get the root filename of the Genome Directory.
|
|
|
158 src_filename = source.split("/")[-1]
|
|
|
159 root_genome_dirname = src_filename.split(".")[0]
|
|
|
160 # If the src_filename indicates it is a source file, as opposed to plug-n-play,
|
|
|
161 # then we may need to do some post processing on it.
|
|
|
162 type_of_download = src_filename.split(".")[1]
|
|
|
163 download_has_source_data = (type_of_download == "source_data")
|
|
|
164
|
|
|
165 # We want to make sure that destination is absolute fully specified path.
|
|
|
166 cannonical_destination = os.path.realpath(destination)
|
|
|
167 if os.path.exists(cannonical_destination):
|
|
|
168 if not os.path.isdir(cannonical_destination):
|
|
|
169 raise ValueError("The destination is not a directory: " + \
|
|
|
170 "{:s}".format(cannonical_destination))
|
|
|
171 # else all is good. It is a directory.
|
|
|
172 else:
|
|
|
173 # We need to create it.
|
|
|
174 try:
|
|
|
175 os.makedirs(cannonical_destination)
|
|
|
176 except os.error:
|
|
|
177 print "ERROR: Trying to create the following directory path:"
|
|
|
178 print "\t{:s}".format(cannonical_destination)
|
|
|
179 raise
|
|
|
180
|
|
|
181 # Make sure the directory now exists and we can write to it.
|
|
|
182 if not os.path.exists(cannonical_destination):
|
|
|
183 # It should have been created, but if it doesn't exist at this point
|
|
|
184 # in the code, something is wrong. Raise an error.
|
|
|
185 raise OSError("The destination directory could not be created: " + \
|
|
|
186 "{:s}".format(cannonical_destination))
|
|
|
187 test_writing_file = "{:s}/{:s}".format(cannonical_destination, _Download_TestFile)
|
|
|
188 try:
|
|
|
189 filehandle = open(test_writing_file, "w")
|
|
|
190 filehandle.write("Testing writing to this file.")
|
|
|
191 filehandle.close()
|
|
|
192 os.remove(test_writing_file)
|
|
|
193 except IOError:
|
|
|
194 print "The destination directory could not be written into: " + \
|
|
|
195 "{:s}".format(cannonical_destination)
|
|
|
196 raise
|
|
|
197
|
|
|
198 # Get the list of files in the directory,
|
|
|
199 # We use it to check for a previous download or extraction among other things.
|
|
|
200 orig_files_in_destdir = set(os.listdir(cannonical_destination))
|
|
|
201 # See whether the file has been downloaded already.
|
|
|
202 download_success_file_path = "{:s}/{:s}".format(cannonical_destination, _DownloadSuccessFile)
|
|
|
203 if ((_DownloadSuccessFile not in orig_files_in_destdir) \
|
|
|
204 or (root_genome_dirname not in orig_files_in_destdir) \
|
|
|
205 or force_download):
|
|
|
206 # Check whether there is enough space on the device for the library.
|
|
|
207 statvfs = os.statvfs(cannonical_destination)
|
|
|
208 # fs_size = statvfs.f_frsize * statvfs.f_blocks # Size of filesystem in bytes
|
|
|
209 # num_free_bytes = statvfs.f_frsize * statvfs.f_bfree # Actual number of free bytes
|
|
|
210 num_avail_bytes = statvfs.f_frsize * statvfs.f_bavail # Number of free bytes that ordinary users
|
|
|
211 # are allowed to use (excl. reserved space)
|
|
|
212 if (num_avail_bytes < _NumBytesNeededForBuild):
|
|
|
213 raise OSError("There is insufficient space ({:s} bytes)".format(str(num_avail_bytes)) + \
|
|
|
214 " on the device of the destination directory: " + \
|
|
|
215 "{:s}".format(cannonical_destination))
|
|
|
216
|
|
|
217 #Previous code to download and untar. Not using anymore.
|
|
|
218 #full_filepath = os.path.join(destination, src_filename)
|
|
|
219 #
|
|
|
220 #Download ref: https://dzone.com/articles/how-download-file-python
|
|
|
221 #f = urllib2.urlopen(source)
|
|
|
222 #data = f.read()
|
|
|
223 #with open(full_filepath, 'wb') as code:
|
|
|
224 # code.write(data)
|
|
|
225 #
|
|
|
226 #Another way to download:
|
|
|
227 #try:
|
|
|
228 # urllib.urlretrieve(url=source, filename=full_filepath)
|
|
|
229 #
|
|
|
230 #Then untar the file.
|
|
|
231 #try:
|
|
|
232 # tarfile.open(full_filepath, mode='r:*').extractall()
|
|
|
233
|
|
|
234 if (_DownloadSuccessFile in orig_files_in_destdir):
|
|
|
235 # Since we are redoing the download,
|
|
|
236 # the success file needs to be removed
|
|
|
237 # until the download has succeeded.
|
|
|
238 os.remove(download_success_file_path)
|
|
|
239 # We want to transfer and untar the file without storing the tar file, because that
|
|
|
240 # adds all that much more space to the needed amount of free space on the disk.
|
|
|
241 # Use subprocess to pipe the output of curl into tar.
|
|
6
|
242 command = "curl --silent {:s} | tar -xzf - -C {:s}".format(source, cannonical_destination)
|
|
0
|
243 try: # to send the command that downloads and extracts the file.
|
|
|
244 command_output = subprocess.check_output(command, shell=True)
|
|
|
245 # FIX - not sure check_output is what we want to use. If we want to have an error raised on
|
|
|
246 # any problem, maybe we should not be checking output.
|
|
|
247 except subprocess.CalledProcessError:
|
|
|
248 print "ERROR: Trying to run the following command:\n\t{:s}".format(command)
|
|
|
249 raise
|
|
|
250 else:
|
|
|
251 lib_was_downloaded = True
|
|
|
252
|
|
|
253 # Some code to help us if errors occur.
|
|
|
254 print "\n*******************************\nFinished download and extraction."
|
|
6
|
255 print_directory_contents(cannonical_destination, 2)
|
|
|
256 print "*******************************\n"
|
|
0
|
257
|
|
|
258 newfiles_in_destdir = set(os.listdir(cannonical_destination)) - orig_files_in_destdir
|
|
|
259 if (root_genome_dirname not in newfiles_in_destdir):
|
|
|
260 # Perhaps it has a different name than what we expected it to be.
|
|
|
261 # It will be the file that was not in the directory
|
|
|
262 # before we did the download and extraction.
|
|
|
263 found_filename = None
|
|
|
264 if len(newfiles_in_destdir) == 1:
|
|
|
265 found_filename = newfiles_in_destdir[0]
|
|
|
266 else:
|
|
|
267 for filename in newfiles_in_destdir:
|
|
|
268 # In most cases, there will only be one new file, but some OS's might have created
|
|
|
269 # other files in the directory.
|
|
|
270 # Look for the directory that was downloaded and extracted.
|
|
|
271 # The correct file's name should be a substring of the tar file that was downloaded.
|
|
|
272 if filename in src_filename:
|
|
|
273 found_filename = filename
|
|
|
274 if found_filename is not None:
|
|
|
275 root_genome_dirname = found_filename
|
|
|
276
|
|
|
277 downloaded_directory = "{:s}/{:s}".format(cannonical_destination, root_genome_dirname)
|
|
|
278
|
|
|
279 if (os.path.exists(downloaded_directory)):
|
|
|
280 try:
|
|
|
281 # Create a file to indicate that the download succeeded.
|
|
|
282 subprocess.check_call("touch {:s}".format(download_success_file_path), shell=True)
|
|
|
283 except IOError:
|
|
|
284 print "The download_success file could not be created: " + \
|
|
|
285 "{:s}".format(download_success_file_path)
|
|
|
286 raise
|
|
|
287 # Look for the build directory, or specify the path where it should be placed.
|
|
|
288 if len(os.listdir(downloaded_directory)) == 1:
|
|
|
289 # Then that one file is a subdirectory that should be the downloaded_directory.
|
|
6
|
290 # That is how the plug-n-play directories are structured.
|
|
0
|
291 subdir_filename = os.listdir(downloaded_directory)[0]
|
|
|
292 genome_build_directory = "{:s}/{:s}".format(downloaded_directory, subdir_filename)
|
|
|
293 else:
|
|
6
|
294 # In this case, we have source_data in the directory. The default will be to create
|
|
|
295 # the build directory in the downloaded_directory with the default _CTAT_Build_dirname.
|
|
|
296 # In this case, this directory will not exist yet until the library is built.
|
|
|
297 genome_build_directory = "{:s}/{:s}".format(downloaded_directory, _CTAT_Build_dirname)
|
|
0
|
298 else:
|
|
|
299 raise ValueError("ERROR: Could not find the extracted file in the destination directory:" + \
|
|
|
300 "\n\t{:s}".format(cannonical_destination))
|
|
|
301
|
|
|
302 return (downloaded_directory, download_has_source_data, genome_build_directory, lib_was_downloaded)
|
|
|
303
|
|
|
304 def gmap_the_library(genome_build_directory):
|
|
|
305 # This is the processing that needs to happen for gmap-fusion to work.
|
|
|
306 # genome_build_directory should normally be a fully specified path,
|
|
6
|
307 # though this function should work even if it is relative.
|
|
|
308 # The command prints messages out to stderr, even when there is not an error,
|
|
|
309 # so route stderr to stdout. Otherwise, galaxy thinks an error occurred.
|
|
|
310 command = "gmap_build -D {:s}/ -d ref_genome.fa.gmap -k 13 {:s}/ref_genome.fa 2>&1".format( \
|
|
0
|
311 genome_build_directory, genome_build_directory)
|
|
|
312 try: # to send the gmap_build command.
|
|
|
313 command_output = subprocess.check_output(command, shell=True)
|
|
|
314 except subprocess.CalledProcessError:
|
|
|
315 print "ERROR: While trying to run the gmap_build command on the library:\n\t{:s}".format(command)
|
|
|
316 raise
|
|
|
317 finally:
|
|
|
318 # Some code to help us if errors occur.
|
|
|
319 print "\n*******************************\nAfter running gmap_build."
|
|
6
|
320 print_directory_contents(genome_build_directory, 2)
|
|
|
321 print "*******************************\n"
|
|
0
|
322
|
|
|
323 def build_the_library(genome_source_directory, genome_build_directory, build, gmap_build):
|
|
|
324 """ genome_source_directory is the location of the source_data needed to build the library.
|
|
|
325 Normally it is fully specified, but could be relative.
|
|
|
326 genome_build_directory is the location where the library will be built.
|
|
|
327 It can be relative to the current working directory or an absolute path.
|
|
|
328 build specifies whether to run prep_genome_lib.pl even if it was run before.
|
|
|
329 gmap_build specifies whether to run gmap_build or not.
|
|
|
330
|
|
|
331 Following was the old way to do it. Before FusionFilter 0.5.0.
|
|
|
332 prep_genome_lib.pl \
|
|
|
333 --genome_fa ref_genome.fa \
|
|
|
334 --gtf ref_annot.gtf \
|
|
|
335 --blast_pairs blast_pairs.gene_syms.outfmt6.gz \
|
|
|
336 --fusion_annot_lib fusion_lib.dat.gz
|
|
|
337 --output_dir ctat_genome_lib_build_dir
|
|
|
338 index_pfam_domain_info.pl \
|
|
|
339 --pfam_domains PFAM.domtblout.dat.gz \
|
|
|
340 --genome_lib_dir ctat_genome_lib_build_dir
|
|
|
341 gmap_build -D ctat_genome_lib_build_dir -d ref_genome.fa.gmap -k 13 ctat_genome_lib_build_dir/ref_genome.fa"
|
|
|
342 """
|
|
|
343 if (genome_source_directory != "" ) and build:
|
|
|
344 if os.path.exists(genome_source_directory):
|
|
|
345 os.chdir(genome_source_directory)
|
|
6
|
346 # Create the command that builds the Genome Resource Library form the source data.
|
|
0
|
347 command = "prep_genome_lib.pl --genome_fa ref_genome.fa --gtf ref_annot.gtf " + \
|
|
|
348 "--pfam_db PFAM.domtblout.dat.gz " + \
|
|
6
|
349 "--output_dir {:s}".format(genome_build_directory)
|
|
|
350 found_HumanFusionLib = False
|
|
|
351 HumanFusionLib_filename = "NoFileFound"
|
|
|
352 for filename in os.listdir(genome_source_directory):
|
|
|
353 # At the time this was written, the filename was CTAT_HumanFusionLib.v0.1.0.dat.gz
|
|
|
354 # We only check the prefix, in case other versions are used later.
|
|
|
355 # I assume there is only one in the directory, but if there are more than one,
|
|
|
356 # the later one, alphabetically, will be used.
|
|
|
357 if filename.split(".")[0] == _CTAT_HumanFusionLib_FilenamePrefix:
|
|
|
358 found_HumanFusionLib = True
|
|
|
359 filename_of_HumanFusionLib = filename
|
|
|
360 if found_HumanFusionLib:
|
|
|
361 # The mouse genomes do not have a fusion_annot_lib
|
|
|
362 # so only add the following for Human genomes.
|
|
|
363 command += "--fusion_annot_lib {:s} ".format(filename_of_HumanFusionLib) + \
|
|
|
364 "--annot_filter_rule AnnotFilterRule.pm "
|
|
0
|
365 if gmap_build:
|
|
|
366 command += "--gmap_build "
|
|
6
|
367 # Send stderr of the command to stdout, because some functions may write to stderr,
|
|
|
368 # even though no error has occurred. We will depend on error code return in order
|
|
|
369 # to know if an error occurred.
|
|
|
370 command += " 2>&1"
|
|
0
|
371 try: # to send the prep_genome_lib command.
|
|
|
372 command_output = subprocess.check_call(command, shell=True)
|
|
|
373 except subprocess.CalledProcessError:
|
|
|
374 print "ERROR: While trying to run the prep_genome_lib.pl command " + \
|
|
|
375 "on the CTAT Genome Resource Library:\n\t{:s}".format(command)
|
|
|
376 raise
|
|
|
377 finally:
|
|
|
378 # Some code to help us if errors occur.
|
|
6
|
379 print "\n*******************************"
|
|
|
380 print "Contents of Genome Source Directory {:s}:".format(genome_source_directory)
|
|
|
381 print_directory_contents(genome_source_directory, 2)
|
|
|
382 print "\nContents of Genome Build Directory {:s}:".format(genome_build_directory)
|
|
|
383 print_directory_contents(genome_build_directory, 2)
|
|
|
384 print "*******************************\n"
|
|
0
|
385 else:
|
|
|
386 raise ValueError("Cannot build the CTAT Genome Resource Library. " + \
|
|
|
387 "The source directory does not exist:\n\t{:s}".format(genome_source_directory))
|
|
|
388 elif gmap_build:
|
|
|
389 gmap_the_library(genome_build_directory)
|
|
|
390
|
|
6
|
391 def search_for_genome_build_dir(top_dir_path):
|
|
|
392 # If we do not download the directory, the topdir_path could be the
|
|
|
393 # location of the genome resource library, but we also want to allow the
|
|
|
394 # user to give the same value for top_dir_path that they do when a
|
|
|
395 # build happens, so we need to handle all three cases:
|
|
|
396 # 1) Is the top_dir_path the build directory,
|
|
|
397 # 2) or is it inside of the given directory,
|
|
|
398 # 3) or is it inside a subdirectory of the given directory.
|
|
|
399 # The source_data downloads are built to a directory named _CTAT_Build_dirname,
|
|
|
400 # and the plug-n-play downloads contain a sub-directory named _CTAT_Build_dirname.
|
|
|
401 genome_build_directory = None
|
|
|
402 print_warning = False
|
|
|
403
|
|
|
404 if not os.path.exists(top_dir_path):
|
|
|
405 raise ValueError("Cannot find the CTAT Genome Resource Library. " + \
|
|
|
406 "The given directory does not exist:\n\t{:s}".format(top_dir_path))
|
|
|
407 elif not os.path.isdir(top_dir_path):
|
|
|
408 raise ValueError("Cannot find the CTAT Genome Resource Library. " + \
|
|
|
409 "The given directory is not a directory:\n\t{:s}".format(top_dir_path))
|
|
|
410 if top_dir_path.split("/")[-1] == _CTAT_Build_dirname:
|
|
|
411 print "Build directory is: {:s}".format(top_dir_path)
|
|
|
412 # The top_dir_path is the path to the genome_build_directory.
|
|
|
413 genome_build_directory = top_dir_path
|
|
|
414 else:
|
|
|
415 # Look for it inside of the top_dir_path directory.
|
|
|
416 print "Looking inside of: {:s}".format(top_dir_path)
|
|
|
417 top_dir_contents = os.listdir(top_dir_path)
|
|
|
418 if (_CTAT_Build_dirname in top_dir_contents):
|
|
|
419 # The genome_build_directory is inside of the top_dir_path directory.
|
|
|
420 print "1. Found it."
|
|
|
421 genome_build_directory = "{:s}/{:s}".format(top_dir_path,_CTAT_Build_dirname)
|
|
|
422 else:
|
|
|
423 # Find all subdirectories containing the _CTAT_Build_dirname or the _CTAT_RefGenome_Filename.
|
|
|
424 # Look down the directory tree two levels.
|
|
|
425 build_dirs_in_subdirs = list()
|
|
|
426 subdirs_with_genome_files = list()
|
|
|
427 build_dirs_in_sub_subdirs = list()
|
|
|
428 sub_subdirs_with_genome_files = list()
|
|
|
429 subdirs = [entry for entry in top_dir_contents if (os.path.isdir("{:s}/{:s}".format(top_dir_path,entry)))]
|
|
|
430 for subdir in subdirs:
|
|
|
431 subdir_path = "{:s}/{:s}".format(top_dir_path, subdir)
|
|
|
432 subdir_path_contents = os.listdir(subdir_path)
|
|
|
433 # print "Is it one of:\n\t" + "\n\t".join(subdir_path_contents)
|
|
|
434 if (_CTAT_Build_dirname in subdir_path_contents):
|
|
|
435 # The genome_build_directory is inside of the subdir_path directory.
|
|
|
436 print "2a, Found one."
|
|
|
437 build_dirs_in_subdirs.append("{:s}/{:s}".format(subdir_path, _CTAT_Build_dirname))
|
|
|
438 if (_CTAT_RefGenome_Filename in subdir_path_contents):
|
|
|
439 subdirs_with_genome_files.append(subdir_path)
|
|
|
440 # Since we are already looping, loop through all dirs one level deeper as well.
|
|
|
441 sub_subdirs = [entry for entry in subdir_path_contents if (os.path.isdir("{:s}/{:s}".format(subdir_path,entry)))]
|
|
|
442 for sub_subdir in sub_subdirs:
|
|
|
443 sub_subdir_path = "{:s}/{:s}".format(subdir_path, sub_subdir)
|
|
|
444 sub_subdir_path_contents = os.listdir(sub_subdir_path)
|
|
|
445 # print "Is it one of:\n\t" + "\n\t".join(sub_subdir_path_contents)
|
|
|
446 if (_CTAT_Build_dirname in sub_subdir_path_contents):
|
|
|
447 # The genome_build_directory is inside of the sub_subdir_path directory.
|
|
|
448 print "3a. Found one."
|
|
|
449 build_dirs_in_sub_subdirs.append("{:s}/{:s}".format(sub_subdir_path, _CTAT_Build_dirname))
|
|
|
450 if (_CTAT_RefGenome_Filename in sub_subdir_path_contents):
|
|
|
451 sub_subdirs_with_genome_files.append(sub_subdir_path)
|
|
|
452 # Hopefully there is one and only one found build directory.
|
|
|
453 # If none are found we check for a directory containing the genome reference file,
|
|
|
454 # but the build process sometimes causes more than one directory to have a copy,
|
|
|
455 # so finding that file is not a sure thing.
|
|
|
456 if (len(build_dirs_in_subdirs) + len(build_dirs_in_sub_subdirs)) > 1:
|
|
|
457 print "\n***************************************"
|
|
|
458 print "Found multiple CTAT Genome Resource Libraries " + \
|
|
|
459 "in the given directory:\n\t{:s}".format(top_dir_path)
|
|
|
460 print_directory_contents(top_dir_path, 2)
|
|
|
461 print "***************************************\n"
|
|
|
462 raise ValueError("Found multiple CTAT Genome Resource Libraries " + \
|
|
|
463 "in the given directory:\n\t{:s}".format(top_dir_path))
|
|
|
464 elif len(build_dirs_in_subdirs) == 1:
|
|
|
465 # The genome_build_directory is inside of the subdir_path directory.
|
|
|
466 print "2b, Found it."
|
|
|
467 genome_build_directory = build_dirs_in_subdirs[0]
|
|
|
468 elif len(build_dirs_in_sub_subdirs) == 1:
|
|
|
469 # The genome_build_directory is inside of the subdir_path directory.
|
|
|
470 print "3b, Found it."
|
|
|
471 genome_build_directory = build_dirs_in_sub_subdirs[0]
|
|
|
472 elif (len(sub_subdirs_with_genome_files) + len(subdirs_with_genome_files)) > 1:
|
|
|
473 print "\n***************************************"
|
|
|
474 print "Unable to find CTAT Genome Resource Library " + \
|
|
|
475 "in the given directory:\n\t{:s}".format(top_dir_path)
|
|
|
476 print "And multiple directories contain {:s}".format(_CTAT_RefGenome_Filename)
|
|
|
477 print_directory_contents(top_dir_path, 2)
|
|
|
478 print "***************************************\n"
|
|
|
479 raise ValueError("Unable to find CTAT Genome Resource Library " + \
|
|
|
480 "in the given directory:\n\t{:s}".format(top_dir_path))
|
|
|
481 elif (len(sub_subdirs_with_genome_files) == 1):
|
|
|
482 print "3c, Maybe found it."
|
|
|
483 genome_build_directory = sub_subdirs_with_genome_files[0]
|
|
|
484 print_warning = True
|
|
|
485 elif (len(subdirs_with_genome_files) == 1):
|
|
|
486 print "2c, Maybe found it."
|
|
|
487 genome_build_directory = subdirs_with_genome_files[0]
|
|
|
488 print_warning = True
|
|
|
489 elif (_CTAT_RefGenome_Filename in top_dir_contents):
|
|
|
490 print "1c. Maybe found it."
|
|
|
491 genome_build_directory = top_dir_path
|
|
|
492 print_warning = True
|
|
|
493 else:
|
|
|
494 print "\n***************************************"
|
|
|
495 print "Unable to find CTAT Genome Resource Library " + \
|
|
|
496 "in the given directory:\n\t{:s}".format(top_dir_path)
|
|
|
497 print_directory_contents(top_dir_path, 2)
|
|
|
498 print "***************************************\n"
|
|
|
499 raise ValueError("Unable to find CTAT Genome Resource Library " + \
|
|
|
500 "in the given directory:\n\t{:s}".format(top_dir_path))
|
|
|
501 # end else
|
|
|
502 # Check if the CTAT Genome Resource Lib has anything in it (and specifically ref_genome.fa).
|
|
|
503 if (genome_build_directory is None):
|
|
|
504 print "\n***************************************"
|
|
|
505 print "Cannot find the CTAT Genome Resource Library " + \
|
|
|
506 "in the given directory:\n\t{:s}".format(top_dir_path)
|
|
|
507 print_directory_contents(top_dir_path, 2)
|
|
|
508 print "***************************************\n"
|
|
|
509 raise ValueError("Cannot find the CTAT Genome Resource Library " + \
|
|
|
510 "in the given directory:\n\t{:s}".format(top_dir_path))
|
|
|
511 elif (_CTAT_RefGenome_Filename not in os.listdir(genome_build_directory)):
|
|
|
512 print "\n***************************************"
|
|
|
513 print "\nWARNING: Cannot find Genome Reference file {:s}".format(_CTAT_RefGenome_Filename) + \
|
|
|
514 "in the genome build directory:\n\t{:s}".format(genome_build_directory)
|
|
|
515 print_directory_contents(genome_build_directory, 2)
|
|
|
516 print "***************************************\n"
|
|
|
517 if print_warning and genome_build_directory:
|
|
|
518 print "\n***************************************"
|
|
|
519 print "\nWARNING: Cannot find the CTAT Genome Resource Library," + \
|
|
|
520 "but found a {:s} file, so set its directory as the library.".format(_CTAT_RefGenome_Filename)
|
|
|
521 print "This my not be the correct directory:\n\t{:s}".format(genome_build_directory)
|
|
|
522 print_directory_contents(genome_build_directory, 2)
|
|
|
523 print "***************************************\n"
|
|
|
524 return genome_build_directory
|
|
|
525
|
|
0
|
526 def main():
|
|
|
527 #Parse Command Line
|
|
|
528 parser = argparse.ArgumentParser()
|
|
6
|
529 parser.add_argument('-s', '--source_url', default='', \
|
|
|
530 help='This is the url of a file with the data. ' + \
|
|
|
531 'They come from https://data.broadinstitute.org/Trinity/CTAT_RESOURCE_LIB/.')
|
|
|
532 parser.add_argument('-n', '--display_name', default='', \
|
|
0
|
533 help='Is used as the display name for the entry of this Genome Resource Library in the data table.')
|
|
|
534 parser.add_argument('-o', '--output_filename', \
|
|
|
535 help='Name of the output file, where the json dictionary will be written.')
|
|
|
536 parser.add_argument('-f', '--force_download',
|
|
6
|
537 help='Forces download of the Genome Resource Library, even if previously downloaded.', action='store_true')
|
|
0
|
538 parser.add_argument('-b', '--build',
|
|
|
539 help='Forces build/rebuild the Genome Resource Library, even if previously built. ' + \
|
|
6
|
540 'Must have downloaded source_data for this to work.', action='store_true')
|
|
0
|
541 parser.add_argument('-m', '--gmap_build',
|
|
|
542 help='Must be selected if you want the library to be gmapped. ' + \
|
|
6
|
543 'Will force gmap_build of the Genome Resource Library, even if previously gmapped.', action='store_true')
|
|
|
544 requiredNamed = parser.add_argument_group('required named arguments')
|
|
|
545 requiredNamed.add_argument('-p', '--destination_path', required=True, \
|
|
|
546 help='Full path of the CTAT Resource Library location or destination, either where it is, or where it will be placed.')
|
|
0
|
547 args = parser.parse_args()
|
|
|
548
|
|
|
549 # All of the input parameters are written by default to the output file prior to
|
|
|
550 # this program being called.
|
|
|
551 # But I do not get input values from the json file, but rather from command line.
|
|
|
552 # Just leaving the following code as a comment, in case it might be useful to someone later.
|
|
|
553 # params = from_json_string(open(filename).read())
|
|
|
554 # target_directory = params['output_data'][0]['extra_files_path']
|
|
|
555 # os.mkdir(target_directory)
|
|
|
556
|
|
4
|
557 print "The value of source_url argument is:\n\t{:s}".format(str(args.source_url))
|
|
|
558
|
|
6
|
559 # FIX - not sure lib_was_downloaded actually serves a purpose...
|
|
0
|
560 lib_was_downloaded = False
|
|
|
561 download_has_source_data = False
|
|
|
562 downloaded_directory = None
|
|
|
563 genome_build_directory = None
|
|
|
564 # FIX - need to make sure we are handling all "possible" combinations of arguments.
|
|
|
565 # Probably would be good if we could simplify/remove some of them.
|
|
6
|
566 # But I think the current interface is using them all.
|
|
0
|
567 if (args.source_url != ""):
|
|
|
568 downloaded_directory, download_has_source_data, genome_build_directory, lib_was_downloaded = \
|
|
|
569 download_from_BroadInst(source=args.source_url, \
|
|
|
570 destination=args.destination_path, \
|
|
|
571 force_download=args.force_download)
|
|
|
572 else:
|
|
6
|
573 genome_build_directory = search_for_genome_build_dir(args.destination_path)
|
|
0
|
574
|
|
|
575 print "\nThe location of the CTAT Genome Resource Library is {:s}.\n".format(genome_build_directory)
|
|
|
576
|
|
|
577 # FIX - We should leave a file indicating build success the same way we do for download success.
|
|
6
|
578 # To take out builds for testing, coment out the next four lines.
|
|
0
|
579 if (download_has_source_data or args.build or args.gmap_build) :
|
|
|
580 build_the_library(downloaded_directory, genome_build_directory, args.build, args.gmap_build)
|
|
|
581 elif (args.gmap_build):
|
|
|
582 gmap_the_library(genome_build_directory)
|
|
|
583
|
|
6
|
584 # The following looks to see if the library actually exists after the build,
|
|
|
585 # and raises an error if it cannot find the library files.
|
|
|
586 # The reassignment of genome_build_directory should be superfluous,
|
|
|
587 # unless I made a mistake in the build code.
|
|
|
588 # FIX - need to get the genome name from the directory name, if there was no download.
|
|
|
589 #genome_build_directory, genome_name_from_dirname = search_for_genome_build_dir(genome_build_directory)
|
|
|
590 genome_build_directory = search_for_genome_build_dir(genome_build_directory)
|
|
|
591
|
|
|
592 source_filename_root = None
|
|
0
|
593 if (args.source_url != None) and (args.source_url != ""):
|
|
|
594 # Get the name out of the source's filename.
|
|
|
595 source_filename_root = args.source_url.split("/")[-1].split(".")[0]
|
|
|
596
|
|
|
597 # Determine the display_name for the library.
|
|
|
598 if (args.display_name is None) or (args.display_name == ""):
|
|
|
599 if (source_filename_root != None) and (source_filename_root != ""):
|
|
6
|
600 # Create the display_name from the source_filename_root.
|
|
0
|
601 display_name = _CTAT_ResourceLib_DisplayNamePrefix + source_filename_root
|
|
|
602 else:
|
|
|
603 display_name = _CTAT_ResourceLib_DisplayNamePrefix + _CTAT_ResourceLib_DefaultGenome
|
|
6
|
604 print "WARNING: We do not have a genome name."
|
|
0
|
605 else:
|
|
|
606 display_name = _CTAT_ResourceLib_DisplayNamePrefix + args.display_name
|
|
|
607 display_name = display_name.replace(" ","_")
|
|
|
608
|
|
|
609 # Create a unique_id for the library.
|
|
|
610 datetime_stamp = datetime.now().strftime("_%Y_%m_%d_%H_%M_%S_%f")
|
|
|
611 if (source_filename_root != None) and (source_filename_root != ""):
|
|
|
612 unique_id = source_filename_root + datetime_stamp
|
|
|
613 elif (downloaded_directory != None) and (downloaded_directory != ""):
|
|
|
614 unique_id = os.path.basename(downloaded_directory).split(".")[0]
|
|
|
615 else:
|
|
|
616 unique_id = _CTAT_ResourceLib_DefaultGenome + datetime_stamp
|
|
|
617
|
|
6
|
618 print "The Genome Resource Library's display_name will be set to: {:s}\n".format(display_name)
|
|
0
|
619 print "Its unique_id will be set to: {:s}\n".format(unique_id)
|
|
|
620 print "Its dir_path will be set to: {:s}\n".format(genome_build_directory)
|
|
|
621
|
|
|
622 data_manager_dict = {}
|
|
|
623 data_manager_dict['data_tables'] = {}
|
|
|
624 data_manager_dict['data_tables']['ctat_genome_resource_libs'] = []
|
|
|
625 data_table_entry = dict(value=unique_id, name=display_name, path=genome_build_directory)
|
|
|
626 data_manager_dict['data_tables']['ctat_genome_resource_libs'].append(data_table_entry)
|
|
|
627
|
|
|
628 # Temporarily the output file's dictionary is written for debugging:
|
|
|
629 print "The dictionary for the output file is:\n\t{:s}".format(str(data_manager_dict))
|
|
|
630 # Save info to json file. This is used to transfer data from the DataManager tool, to the data manager,
|
|
|
631 # which then puts it into the correct .loc file (I think).
|
|
|
632 # Comment out the following line when testing without galaxy package.
|
|
|
633 open(args.output_filename, 'wb').write(to_json_string(data_manager_dict))
|
|
|
634
|
|
|
635 if __name__ == "__main__":
|
|
|
636 main()
|
