|
0
|
1 #!/usr/bin/env python
|
|
|
2 # ref: https://galaxyproject.org/admin/tools/data-managers/how-to/define/
|
|
|
3
|
|
|
4 # Rewritten by H.E. Cicada Brokaw Dennis from a source downloaded from the toolshed and
|
|
|
5 # other example code on the web.
|
|
|
6 # This now allows downloading of a user selected library
|
|
|
7 # but only from the CTAT Genome Resource Library website.
|
|
|
8 # Ultimately we might want to allow the user to specify any location
|
|
|
9 # from which to download.
|
|
|
10 # Users can create or download other libraries and use this tool to add them if they don't want
|
|
|
11 # to add them by hand.
|
|
|
12
|
|
|
13 import argparse
|
|
|
14 import os
|
|
|
15 #import tarfile
|
|
|
16 #import urllib
|
|
|
17 import subprocess
|
|
|
18
|
|
|
19 # Comment out the following line when testing without galaxy package.
|
|
|
20 from galaxy.util.json import to_json_string
|
|
|
21 # The following is not being used, but leaving as info
|
|
|
22 # in case we ever want to get input values using json.
|
|
|
23 # from galaxy.util.json import from_json_string
|
|
|
24
|
|
|
25 # datetime.now() is used to create the unique_id
|
|
|
26 from datetime import datetime
|
|
|
27
|
|
|
28 # The FileListParser is used by get_ctat_genome_filenames(),
|
|
|
29 # which is called by the Data Manager interface (.xml file) to get
|
|
|
30 # the filenames that are available online at broadinstitute.org
|
|
|
31 # Not sure best way to do it.
|
|
|
32 # This object uses HTMLParser to look through the html
|
|
|
33 # searching for the filenames within anchor tags.
|
|
|
34 import urllib2
|
|
|
35 from HTMLParser import HTMLParser
|
|
|
36
|
|
|
37 _CTAT_ResourceLib_URL = 'https://data.broadinstitute.org/Trinity/CTAT_RESOURCE_LIB/'
|
|
|
38 _CTAT_BuildDir_Name = 'ctat_genome_lib_build_dir'
|
|
|
39 _CTAT_ResourceLib_DisplayNamePrefix = 'CTAT_GenomeResourceLib_'
|
|
|
40 _CTAT_ResourceLib_DefaultGenome = 'Unspecified_Genome'
|
|
|
41 _NumBytesNeededForBuild = 64424509440 # 60 Gigabytes. FIX - This might not be correct.
|
|
|
42 _Download_TestFile = "write_testfile.txt"
|
|
|
43 _DownloadSuccessFile = 'download_succeeded.txt'
|
|
|
44
|
|
|
45 class FileListParser(HTMLParser):
|
|
|
46 def __init__(self):
|
|
|
47 # Have to use direct call to super class rather than using super():
|
|
|
48 # super(FileListParser, self).__init__()
|
|
|
49 # because HTMLParser is an "old style" class and its inheritance chain does not include object.
|
|
|
50 HTMLParser.__init__(self)
|
|
|
51 self.urls = set()
|
|
|
52 def handle_starttag(self, tag, attrs):
|
|
|
53 # Look for filename references in anchor tags and add them to urls.
|
|
|
54 if tag == "a":
|
|
|
55 # The tag is an anchor tag.
|
|
|
56 for attribute in attrs:
|
|
|
57 # print "Checking: {:s}".format(str(attribute))
|
|
|
58 if attribute[0] == "href":
|
|
|
59 # Does the href have a tar.gz in it?
|
|
|
60 if ("tar.gz" in attribute[1]) and ("md5" not in attribute[1]):
|
|
|
61 # Add the value to urls.
|
|
|
62 self.urls.add(attribute[1])
|
|
|
63 # End of class FileListParser
|
|
|
64
|
|
|
65 def get_ctat_genome_urls():
|
|
|
66 # open the url and retrieve the urls of the files in the directory.
|
|
|
67 resource = urllib2.urlopen(_CTAT_ResourceLib_URL)
|
|
|
68 theHTML = resource.read()
|
|
|
69 filelist_parser = FileListParser()
|
|
|
70 filelist_parser.feed(theHTML)
|
|
|
71 # For dynamic options need to return an interable with contents that are tuples with 3 items.
|
|
|
72 # Item one is a string that is the display name put into the option list.
|
|
|
73 # Item two is the value that is put into the parameter associated with the option list.
|
|
|
74 # Item three is a True or False value, indicating whether the item is selected.
|
|
|
75 options = []
|
|
|
76 for i, url in enumerate(filelist_parser.urls):
|
|
5
|
77 # The urls should look like:
|
|
0
|
78 # https://data.broadinstitute.org/Trinity/CTAT_RESOURCE_LIB/GRCh37_v19_CTAT_lib_Feb092018.plug-n-play.tar.gz
|
|
|
79 # https://data.broadinstitute.org/Trinity/CTAT_RESOURCE_LIB/Mouse_M16_CTAT_lib_Feb202018.source_data.tar.gz
|
|
5
|
80 # But is actuality, they are coming in looking like:
|
|
|
81 # GRCh37_v19_CTAT_lib_Feb092018.plug-n-play.tar.gz
|
|
|
82 # Mouse_M16_CTAT_lib_Feb202018.source_data.tar.gz
|
|
|
83 # Write code to handle both situations, or an ftp: url.
|
|
|
84 if (url.split(":")[0] == "http") or (url.split(":")[0] == "https") or (url.split(":")[0] == "ftp"):
|
|
|
85 full_url_path = url
|
|
|
86 else:
|
|
|
87 # Assume the path is relative to the page location.
|
|
|
88 full_url_path = "{:s}/{:s}".format(_CTAT_ResourceLib_URL, url)
|
|
0
|
89 filename = url.split("/")[-1]
|
|
5
|
90
|
|
0
|
91 if filename.split("_")[0] != "Mouse":
|
|
|
92 # Take out the mouse genome options for now.
|
|
|
93 # The mouse genome option is not handled correctly yet
|
|
5
|
94 options.append((filename, full_url_path, i == 0))
|
|
|
95 options.sort() # So the list will be in alphabetical order.
|
|
0
|
96 # return a tuple of the urls
|
|
4
|
97 print "The list being returned as options is:"
|
|
|
98 print "{:s}\n".format(str(options))
|
|
0
|
99 return options
|
|
|
100
|
|
|
101 # The following was used by the example program to get input parameters through the json.
|
|
|
102 # Just leaving here for reference.
|
|
|
103 # We are getting all of our parameter values through command line arguments.
|
|
|
104 #def get_reference_id_name(params):
|
|
|
105 # genome_id = params['param_dict']['genome_id']
|
|
|
106 # genome_name = params['param_dict']['genome_name']
|
|
|
107 # return genome_id, genome_name
|
|
|
108 #
|
|
|
109 #def get_url(params):
|
|
|
110 # trained_url = params['param_dict']['trained_url']
|
|
|
111 # return trained_url
|
|
|
112
|
|
|
113 def download_from_BroadInst(source, destination, force_download):
|
|
|
114 # Input Parameters
|
|
|
115 # source is the full URL of the file we want to download.
|
|
|
116 # It should look something like:
|
|
|
117 # https://data.broadinstitute.org/Trinity/CTAT_RESOURCE_LIB/GRCh37_v19_CTAT_lib_Feb092018.plug-n-play.tar.gz
|
|
|
118 # destination is the location where the source file will be unarchived.
|
|
|
119 # Relative paths are expanded using the current working directory, so within Galaxy,
|
|
|
120 # it is best to send in absolute fully specified path names so you know to where
|
|
|
121 # the source file going to be extracted.
|
|
|
122 # force_download will cause a new download and extraction to occur, even if the destination
|
|
|
123 # has a file in it indicating that a previous download succeeded.
|
|
|
124 #
|
|
|
125 # Returns the following:
|
|
|
126 # return (downloaded_directory, download_has_source_data, genome_build_directory, lib_was_downloaded)
|
|
|
127 # downloaded_directory
|
|
|
128 # The directory which was created as a subdirectory of the destination directory
|
|
|
129 # when the download occurred, or if there was no download,
|
|
|
130 # possibly the same directory as destination, if that is where the data resides.
|
|
|
131 # download_has_source_data
|
|
|
132 # Is a boolean indicating whether the source file was "source_data" or was "plug-n-play".
|
|
|
133 # genome_build_directory
|
|
|
134 # The directory where the genome resource library is or where it should be built.
|
|
|
135 # It can be the same as the downloaded directory, but is sometimes a subdirectory of it.
|
|
|
136 # lib_was_downloaded
|
|
|
137 # Since it doesn't always do the download, the function returns whether download occurred.
|
|
|
138 lib_was_downloaded = False
|
|
|
139
|
|
4
|
140 print "In download_from_BroadInst(). The source_url is:\n\t{:s}".format(str(source))
|
|
|
141
|
|
0
|
142 # Get the root filename of the Genome Directory.
|
|
|
143 src_filename = source.split("/")[-1]
|
|
|
144 root_genome_dirname = src_filename.split(".")[0]
|
|
|
145 # If the src_filename indicates it is a source file, as opposed to plug-n-play,
|
|
|
146 # then we may need to do some post processing on it.
|
|
|
147 type_of_download = src_filename.split(".")[1]
|
|
|
148 download_has_source_data = (type_of_download == "source_data")
|
|
|
149
|
|
|
150 # We want to make sure that destination is absolute fully specified path.
|
|
|
151 cannonical_destination = os.path.realpath(destination)
|
|
|
152 if os.path.exists(cannonical_destination):
|
|
|
153 if not os.path.isdir(cannonical_destination):
|
|
|
154 raise ValueError("The destination is not a directory: " + \
|
|
|
155 "{:s}".format(cannonical_destination))
|
|
|
156 # else all is good. It is a directory.
|
|
|
157 else:
|
|
|
158 # We need to create it.
|
|
|
159 try:
|
|
|
160 os.makedirs(cannonical_destination)
|
|
|
161 except os.error:
|
|
|
162 print "ERROR: Trying to create the following directory path:"
|
|
|
163 print "\t{:s}".format(cannonical_destination)
|
|
|
164 raise
|
|
|
165
|
|
|
166 # Make sure the directory now exists and we can write to it.
|
|
|
167 if not os.path.exists(cannonical_destination):
|
|
|
168 # It should have been created, but if it doesn't exist at this point
|
|
|
169 # in the code, something is wrong. Raise an error.
|
|
|
170 raise OSError("The destination directory could not be created: " + \
|
|
|
171 "{:s}".format(cannonical_destination))
|
|
|
172 test_writing_file = "{:s}/{:s}".format(cannonical_destination, _Download_TestFile)
|
|
|
173 try:
|
|
|
174 filehandle = open(test_writing_file, "w")
|
|
|
175 filehandle.write("Testing writing to this file.")
|
|
|
176 filehandle.close()
|
|
|
177 os.remove(test_writing_file)
|
|
|
178 except IOError:
|
|
|
179 print "The destination directory could not be written into: " + \
|
|
|
180 "{:s}".format(cannonical_destination)
|
|
|
181 raise
|
|
|
182
|
|
|
183 # Get the list of files in the directory,
|
|
|
184 # We use it to check for a previous download or extraction among other things.
|
|
|
185 orig_files_in_destdir = set(os.listdir(cannonical_destination))
|
|
|
186 # See whether the file has been downloaded already.
|
|
|
187 download_success_file_path = "{:s}/{:s}".format(cannonical_destination, _DownloadSuccessFile)
|
|
|
188 if ((_DownloadSuccessFile not in orig_files_in_destdir) \
|
|
|
189 or (root_genome_dirname not in orig_files_in_destdir) \
|
|
|
190 or force_download):
|
|
|
191 # Check whether there is enough space on the device for the library.
|
|
|
192 statvfs = os.statvfs(cannonical_destination)
|
|
|
193 # fs_size = statvfs.f_frsize * statvfs.f_blocks # Size of filesystem in bytes
|
|
|
194 # num_free_bytes = statvfs.f_frsize * statvfs.f_bfree # Actual number of free bytes
|
|
|
195 num_avail_bytes = statvfs.f_frsize * statvfs.f_bavail # Number of free bytes that ordinary users
|
|
|
196 # are allowed to use (excl. reserved space)
|
|
|
197 if (num_avail_bytes < _NumBytesNeededForBuild):
|
|
|
198 raise OSError("There is insufficient space ({:s} bytes)".format(str(num_avail_bytes)) + \
|
|
|
199 " on the device of the destination directory: " + \
|
|
|
200 "{:s}".format(cannonical_destination))
|
|
|
201
|
|
|
202 #Previous code to download and untar. Not using anymore.
|
|
|
203 #full_filepath = os.path.join(destination, src_filename)
|
|
|
204 #
|
|
|
205 #Download ref: https://dzone.com/articles/how-download-file-python
|
|
|
206 #f = urllib2.urlopen(source)
|
|
|
207 #data = f.read()
|
|
|
208 #with open(full_filepath, 'wb') as code:
|
|
|
209 # code.write(data)
|
|
|
210 #
|
|
|
211 #Another way to download:
|
|
|
212 #try:
|
|
|
213 # urllib.urlretrieve(url=source, filename=full_filepath)
|
|
|
214 #
|
|
|
215 #Then untar the file.
|
|
|
216 #try:
|
|
|
217 # tarfile.open(full_filepath, mode='r:*').extractall()
|
|
|
218
|
|
|
219 if (_DownloadSuccessFile in orig_files_in_destdir):
|
|
|
220 # Since we are redoing the download,
|
|
|
221 # the success file needs to be removed
|
|
|
222 # until the download has succeeded.
|
|
|
223 os.remove(download_success_file_path)
|
|
|
224 # We want to transfer and untar the file without storing the tar file, because that
|
|
|
225 # adds all that much more space to the needed amount of free space on the disk.
|
|
|
226 # Use subprocess to pipe the output of curl into tar.
|
|
|
227 command = "curl {:s} | tar -xzvf - -C {:s}".format(source, cannonical_destination)
|
|
|
228 try: # to send the command that downloads and extracts the file.
|
|
|
229 command_output = subprocess.check_output(command, shell=True)
|
|
|
230 # FIX - not sure check_output is what we want to use. If we want to have an error raised on
|
|
|
231 # any problem, maybe we should not be checking output.
|
|
|
232 except subprocess.CalledProcessError:
|
|
|
233 print "ERROR: Trying to run the following command:\n\t{:s}".format(command)
|
|
|
234 raise
|
|
|
235 else:
|
|
|
236 lib_was_downloaded = True
|
|
|
237
|
|
|
238 # Some code to help us if errors occur.
|
|
|
239 print "\n*******************************\nFinished download and extraction."
|
|
|
240 subprocess.check_call("ls -lad {:s}/*".format(cannonical_destination), shell=True)
|
|
|
241 subprocess.check_call("ls -lad {:s}/*/*".format(cannonical_destination), shell=True)
|
|
|
242
|
|
|
243 newfiles_in_destdir = set(os.listdir(cannonical_destination)) - orig_files_in_destdir
|
|
|
244 if (root_genome_dirname not in newfiles_in_destdir):
|
|
|
245 # Perhaps it has a different name than what we expected it to be.
|
|
|
246 # It will be the file that was not in the directory
|
|
|
247 # before we did the download and extraction.
|
|
|
248 found_filename = None
|
|
|
249 if len(newfiles_in_destdir) == 1:
|
|
|
250 found_filename = newfiles_in_destdir[0]
|
|
|
251 else:
|
|
|
252 for filename in newfiles_in_destdir:
|
|
|
253 # In most cases, there will only be one new file, but some OS's might have created
|
|
|
254 # other files in the directory.
|
|
|
255 # Look for the directory that was downloaded and extracted.
|
|
|
256 # The correct file's name should be a substring of the tar file that was downloaded.
|
|
|
257 if filename in src_filename:
|
|
|
258 found_filename = filename
|
|
|
259 if found_filename is not None:
|
|
|
260 root_genome_dirname = found_filename
|
|
|
261
|
|
|
262 downloaded_directory = "{:s}/{:s}".format(cannonical_destination, root_genome_dirname)
|
|
|
263
|
|
|
264 if (os.path.exists(downloaded_directory)):
|
|
|
265 try:
|
|
|
266 # Create a file to indicate that the download succeeded.
|
|
|
267 subprocess.check_call("touch {:s}".format(download_success_file_path), shell=True)
|
|
|
268 except IOError:
|
|
|
269 print "The download_success file could not be created: " + \
|
|
|
270 "{:s}".format(download_success_file_path)
|
|
|
271 raise
|
|
|
272 # Look for the build directory, or specify the path where it should be placed.
|
|
|
273 if len(os.listdir(downloaded_directory)) == 1:
|
|
|
274 # Then that one file is a subdirectory that should be the downloaded_directory.
|
|
|
275 subdir_filename = os.listdir(downloaded_directory)[0]
|
|
|
276 genome_build_directory = "{:s}/{:s}".format(downloaded_directory, subdir_filename)
|
|
|
277 else:
|
|
|
278 genome_build_directory = "{:s}/{:s}".format(downloaded_directory, _CTAT_BuildDir_Name)
|
|
|
279 else:
|
|
|
280 raise ValueError("ERROR: Could not find the extracted file in the destination directory:" + \
|
|
|
281 "\n\t{:s}".format(cannonical_destination))
|
|
|
282
|
|
|
283 return (downloaded_directory, download_has_source_data, genome_build_directory, lib_was_downloaded)
|
|
|
284
|
|
|
285 def gmap_the_library(genome_build_directory):
|
|
|
286 # This is the processing that needs to happen for gmap-fusion to work.
|
|
|
287 # genome_build_directory should normally be a fully specified path,
|
|
|
288 # though it should work if it is relative.
|
|
|
289 command = "gmap_build -D {:s}/ -d ref_genome.fa.gmap -k 13 {:s}/ref_genome.fa".format( \
|
|
|
290 genome_build_directory, genome_build_directory)
|
|
|
291 try: # to send the gmap_build command.
|
|
|
292 command_output = subprocess.check_output(command, shell=True)
|
|
|
293 except subprocess.CalledProcessError:
|
|
|
294 print "ERROR: While trying to run the gmap_build command on the library:\n\t{:s}".format(command)
|
|
|
295 raise
|
|
|
296 finally:
|
|
|
297 # Some code to help us if errors occur.
|
|
|
298 print "\n*******************************\nAfter running gmap_build."
|
|
|
299 if os.path.exists(genome_build_directory):
|
|
|
300 print "\nBuild Directory {:s}:".format(genome_build_directory)
|
|
|
301 subprocess.check_call("ls -la {:s}".format(genome_build_directory), shell=True)
|
|
|
302 dir_entries = os.listdir(genome_build_directory)
|
|
|
303 for entry in dir_entries:
|
|
|
304 entry_path = "{:s}/{:s}".format(genome_build_directory, entry)
|
|
|
305 print "\nDirectory {:s}:".format(entry_path)
|
|
|
306 subprocess.check_call("ls -la {:s}".format(entry_path), shell=True)
|
|
|
307 if os.path.isdir(entry_path):
|
|
|
308 subdir_entries = os.listdir(entry_path)
|
|
|
309 for subdir_entry in subdir_entries:
|
|
|
310 subdir_entry_path = "{:s}/{:s}".format(entry_path, subdir_entry)
|
|
|
311 print "\nDirectory {:s}:".format(subdir_entry_path)
|
|
|
312 subprocess.check_call("ls -la {:s}".format(subdir_entry_path), shell=True)
|
|
|
313 else:
|
|
|
314 print "Genome Build Directory does not exist:\n\t{:s}".format(genome_build_directory)
|
|
|
315 print "*******************************"
|
|
|
316
|
|
|
317 def build_the_library(genome_source_directory, genome_build_directory, build, gmap_build):
|
|
|
318 """ genome_source_directory is the location of the source_data needed to build the library.
|
|
|
319 Normally it is fully specified, but could be relative.
|
|
|
320 genome_build_directory is the location where the library will be built.
|
|
|
321 It can be relative to the current working directory or an absolute path.
|
|
|
322 build specifies whether to run prep_genome_lib.pl even if it was run before.
|
|
|
323 gmap_build specifies whether to run gmap_build or not.
|
|
|
324
|
|
|
325 Following was the old way to do it. Before FusionFilter 0.5.0.
|
|
|
326 prep_genome_lib.pl \
|
|
|
327 --genome_fa ref_genome.fa \
|
|
|
328 --gtf ref_annot.gtf \
|
|
|
329 --blast_pairs blast_pairs.gene_syms.outfmt6.gz \
|
|
|
330 --fusion_annot_lib fusion_lib.dat.gz
|
|
|
331 --output_dir ctat_genome_lib_build_dir
|
|
|
332 index_pfam_domain_info.pl \
|
|
|
333 --pfam_domains PFAM.domtblout.dat.gz \
|
|
|
334 --genome_lib_dir ctat_genome_lib_build_dir
|
|
|
335 gmap_build -D ctat_genome_lib_build_dir -d ref_genome.fa.gmap -k 13 ctat_genome_lib_build_dir/ref_genome.fa"
|
|
|
336 """
|
|
|
337 if (genome_source_directory != "" ) and build:
|
|
|
338 if os.path.exists(genome_source_directory):
|
|
|
339 os.chdir(genome_source_directory)
|
|
|
340 # FIX - look for a fusion_annot_lib and include it, else omit it.
|
|
|
341 command = "prep_genome_lib.pl --genome_fa ref_genome.fa --gtf ref_annot.gtf " + \
|
|
|
342 "--fusion_annot_lib CTAT_HumanFusionLib.v0.1.0.dat.gz " + \
|
|
|
343 "--annot_filter_rule AnnotFilterRule.pm " + \
|
|
|
344 "--pfam_db PFAM.domtblout.dat.gz " + \
|
|
|
345 "--output_dir {:s} ".format(genome_build_directory)
|
|
|
346 if gmap_build:
|
|
|
347 command += "--gmap_build "
|
|
|
348 try: # to send the prep_genome_lib command.
|
|
|
349 command_output = subprocess.check_call(command, shell=True)
|
|
|
350 except subprocess.CalledProcessError:
|
|
|
351 print "ERROR: While trying to run the prep_genome_lib.pl command " + \
|
|
|
352 "on the CTAT Genome Resource Library:\n\t{:s}".format(command)
|
|
|
353 raise
|
|
|
354 finally:
|
|
|
355 # Some code to help us if errors occur.
|
|
|
356 print "*******************************"
|
|
|
357 if os.path.exists(genome_build_directory):
|
|
|
358 print "\nSource Directory {:s}:".format(genome_source_directory)
|
|
|
359 subprocess.check_call("ls -la {:s}".format(genome_source_directory), shell=True)
|
|
|
360 dir_entries = os.listdir(genome_source_directory)
|
|
|
361 for entry in dir_entries:
|
|
|
362 entry_path = "{:s}/{:s}".format(genome_source_directory, entry)
|
|
|
363 print "\nDirectory {:s}:".format(entry_path)
|
|
|
364 subprocess.check_call("ls -la {:s}".format(entry_path), shell=True)
|
|
|
365 else:
|
|
|
366 print "Genome Source Directory does not exist:\n\t{:s}".format(genome_source_directory)
|
|
|
367 if os.path.exists(genome_build_directory):
|
|
|
368 print "\nBuild Directory {:s}:".format(genome_build_directory)
|
|
|
369 subprocess.check_call("ls -la {:s}".format(genome_build_directory), shell=True)
|
|
|
370 dir_entries = os.listdir(genome_build_directory)
|
|
|
371 for entry in dir_entries:
|
|
|
372 entry_path = "{:s}/{:s}".format(genome_build_directory, entry)
|
|
|
373 print "\nDirectory {:s}:".format(entry_path)
|
|
|
374 subprocess.check_call("ls -la {:s}".format(entry_path), shell=True)
|
|
|
375 if os.path.isdir(entry_path):
|
|
|
376 subdir_entries = os.listdir(entry_path)
|
|
|
377 for subdir_entry in subdir_entries:
|
|
|
378 subdir_entry_path = "{:s}/{:s}".format(entry_path, subdir_entry)
|
|
|
379 print "\nDirectory {:s}:".format(subdir_entry_path)
|
|
|
380 subprocess.check_call("ls -la {:s}".format(subdir_entry_path), shell=True)
|
|
|
381 else:
|
|
|
382 print "Genome Build Directory does not exist:\n\t{:s}".format(genome_build_directory)
|
|
|
383 print "*******************************"
|
|
|
384 else:
|
|
|
385 raise ValueError("Cannot build the CTAT Genome Resource Library. " + \
|
|
|
386 "The source directory does not exist:\n\t{:s}".format(genome_source_directory))
|
|
|
387 elif gmap_build:
|
|
|
388 gmap_the_library(genome_build_directory)
|
|
|
389
|
|
|
390 def main():
|
|
|
391 #Parse Command Line
|
|
|
392 parser = argparse.ArgumentParser()
|
|
|
393 parser.add_argument('-s', '--source_url', default="", \
|
|
|
394 help='This is the url of a file with the data. They come from https://data.broadinstitute.org/Trinity/CTAT_RESOURCE_LIB/.')
|
|
|
395 parser.add_argument('-n', '--display_name', default="", \
|
|
|
396 help='Is used as the display name for the entry of this Genome Resource Library in the data table.')
|
|
|
397 parser.add_argument('-p', '--destination_path', \
|
|
|
398 help='Full path of the CTAT Resource Library location or destination, either where it is, or where it will be placed.')
|
|
|
399 parser.add_argument('-o', '--output_filename', \
|
|
|
400 help='Name of the output file, where the json dictionary will be written.')
|
|
|
401 parser.add_argument('-f', '--force_download',
|
|
|
402 help='Forces download of the Genome Resource Library, even if previously downloaded.', action="store_true")
|
|
|
403 parser.add_argument('-b', '--build',
|
|
|
404 help='Forces build/rebuild the Genome Resource Library, even if previously built. ' + \
|
|
|
405 'Must have downloaded source_data for this to work.', action="store_true")
|
|
|
406 parser.add_argument('-m', '--gmap_build',
|
|
|
407 help='Must be selected if you want the library to be gmapped. ' + \
|
|
|
408 'Will force gmap_build of the Genome Resource Library, even if previously gmapped.', action="store_true")
|
|
|
409 args = parser.parse_args()
|
|
|
410
|
|
|
411 # All of the input parameters are written by default to the output file prior to
|
|
|
412 # this program being called.
|
|
|
413 # But I do not get input values from the json file, but rather from command line.
|
|
|
414 # Just leaving the following code as a comment, in case it might be useful to someone later.
|
|
|
415 # params = from_json_string(open(filename).read())
|
|
|
416 # target_directory = params['output_data'][0]['extra_files_path']
|
|
|
417 # os.mkdir(target_directory)
|
|
|
418
|
|
4
|
419 print "The value of source_url argument is:\n\t{:s}".format(str(args.source_url))
|
|
|
420
|
|
0
|
421 # FIX - not sure the lib_was_downloaded actually serves a purpose...
|
|
|
422 lib_was_downloaded = False
|
|
|
423 download_has_source_data = False
|
|
|
424 # If we do not download the directory, the destination_path should be the
|
|
|
425 # location of the genome resource library.
|
|
|
426 downloaded_directory = None
|
|
|
427 # FIX - look inside of the args.destination_path to see if the build directory is inside it or is it.
|
|
|
428 genome_build_directory = None
|
|
|
429 # FIX - need to make sure we are handling all "possible" combinations of arguments.
|
|
|
430 # Probably would be good if we could simplify/remove some of them.
|
|
|
431 if (args.source_url != ""):
|
|
|
432 downloaded_directory, download_has_source_data, genome_build_directory, lib_was_downloaded = \
|
|
|
433 download_from_BroadInst(source=args.source_url, \
|
|
|
434 destination=args.destination_path, \
|
|
|
435 force_download=args.force_download)
|
|
|
436 else:
|
|
|
437 genome_build_directory = args.destination_path
|
|
|
438 if not os.path.exists(genome_build_directory):
|
|
|
439 raise ValueError("Cannot find the CTAT Genome Resource Library. " + \
|
|
|
440 "The directory does not exist:\n\t{:s}".format(genome_build_directory))
|
|
|
441 # else:
|
|
|
442 # FIX - Check if there is an actual CTAT Genome Resource Lib there.
|
|
|
443 # _CTAT_BuildDir_Name
|
|
|
444
|
|
|
445 print "\nThe location of the CTAT Genome Resource Library is {:s}.\n".format(genome_build_directory)
|
|
|
446
|
|
|
447 # Take out builds for testing.
|
|
|
448 # FIX - We should leave a file indicating build success the same way we do for download success.
|
|
|
449 if (download_has_source_data or args.build or args.gmap_build) :
|
|
|
450 build_the_library(downloaded_directory, genome_build_directory, args.build, args.gmap_build)
|
|
|
451 elif (args.gmap_build):
|
|
|
452 gmap_the_library(genome_build_directory)
|
|
|
453
|
|
|
454 if (args.source_url != None) and (args.source_url != ""):
|
|
|
455 # Get the name out of the source's filename.
|
|
|
456 source_filename_root = args.source_url.split("/")[-1].split(".")[0]
|
|
|
457
|
|
|
458 # Determine the display_name for the library.
|
|
|
459 if (args.display_name is None) or (args.display_name == ""):
|
|
|
460 if (source_filename_root != None) and (source_filename_root != ""):
|
|
|
461 # Get the name out of the source filename.
|
|
|
462 display_name = _CTAT_ResourceLib_DisplayNamePrefix + source_filename_root
|
|
|
463 else:
|
|
|
464 display_name = _CTAT_ResourceLib_DisplayNamePrefix + _CTAT_ResourceLib_DefaultGenome
|
|
|
465 print "WARNING: We do not have a genome name. Using a default name, that might not be correct."
|
|
|
466 else:
|
|
|
467 display_name = _CTAT_ResourceLib_DisplayNamePrefix + args.display_name
|
|
|
468 display_name = display_name.replace(" ","_")
|
|
|
469 print "The Genome Name will be set to: {:s}\n".format(display_name)
|
|
|
470
|
|
|
471 # Create a unique_id for the library.
|
|
|
472 datetime_stamp = datetime.now().strftime("_%Y_%m_%d_%H_%M_%S_%f")
|
|
|
473 if (source_filename_root != None) and (source_filename_root != ""):
|
|
|
474 unique_id = source_filename_root + datetime_stamp
|
|
|
475 elif (downloaded_directory != None) and (downloaded_directory != ""):
|
|
|
476 unique_id = os.path.basename(downloaded_directory).split(".")[0]
|
|
|
477 else:
|
|
|
478 unique_id = _CTAT_ResourceLib_DefaultGenome + datetime_stamp
|
|
|
479
|
|
|
480 print "The Resource Lib's display_name will be set to: {:s}\n".format(display_name)
|
|
|
481 print "Its unique_id will be set to: {:s}\n".format(unique_id)
|
|
|
482 print "Its dir_path will be set to: {:s}\n".format(genome_build_directory)
|
|
|
483
|
|
|
484 data_manager_dict = {}
|
|
|
485 data_manager_dict['data_tables'] = {}
|
|
|
486 data_manager_dict['data_tables']['ctat_genome_resource_libs'] = []
|
|
|
487 data_table_entry = dict(value=unique_id, name=display_name, path=genome_build_directory)
|
|
|
488 data_manager_dict['data_tables']['ctat_genome_resource_libs'].append(data_table_entry)
|
|
|
489
|
|
|
490 # Temporarily the output file's dictionary is written for debugging:
|
|
|
491 print "The dictionary for the output file is:\n\t{:s}".format(str(data_manager_dict))
|
|
|
492 # Save info to json file. This is used to transfer data from the DataManager tool, to the data manager,
|
|
|
493 # which then puts it into the correct .loc file (I think).
|
|
|
494 # Comment out the following line when testing without galaxy package.
|
|
|
495 open(args.output_filename, 'wb').write(to_json_string(data_manager_dict))
|
|
|
496
|
|
|
497 if __name__ == "__main__":
|
|
|
498 main()
|
