|
1
|
1 """
|
|
|
2 Provides mapping between extensions and datatypes, mime-types, etc.
|
|
|
3 """
|
|
|
4 import os, sys, tempfile, threading, logging
|
|
|
5 import data, tabular, interval, images, sequence, qualityscore, genetics, xml, coverage, tracks, chrominfo, binary, assembly, ngsindex, wsf, annotation, ontology, termenrichment
|
|
|
6 import galaxy.util
|
|
|
7 from galaxy.util.odict import odict
|
|
|
8 from display_applications.application import DisplayApplication
|
|
|
9
|
|
|
10 class ConfigurationError( Exception ):
|
|
|
11 pass
|
|
|
12
|
|
|
13 class Registry( object ):
|
|
|
14 def __init__( self ):
|
|
|
15 self.log = logging.getLogger(__name__)
|
|
|
16 self.log.addHandler( logging.NullHandler() )
|
|
|
17 self.datatypes_by_extension = {}
|
|
|
18 self.mimetypes_by_extension = {}
|
|
|
19 self.datatype_converters = odict()
|
|
|
20 # Converters defined in local datatypes_conf.xml
|
|
|
21 self.converters = []
|
|
|
22 # Converters defined in datatypes_conf.xml included in installed tool shed repositories.
|
|
|
23 self.proprietary_converters = []
|
|
|
24 self.converter_deps = {}
|
|
|
25 self.available_tracks = []
|
|
|
26 self.set_external_metadata_tool = None
|
|
|
27 self.sniff_order = []
|
|
|
28 self.upload_file_formats = []
|
|
|
29 # Datatype elements defined in local datatypes_conf.xml that contain display applications.
|
|
|
30 self.display_app_containers = []
|
|
|
31 # Datatype elements in datatypes_conf.xml included in installed
|
|
|
32 # tool shed repositories that contain display applications.
|
|
|
33 self.proprietary_display_app_containers = []
|
|
|
34 # Map a display application id to a display application
|
|
|
35 self.display_applications = odict()
|
|
|
36 # The following 2 attributes are used in the to_xml_file()
|
|
|
37 # method to persist the current state into an xml file.
|
|
|
38 self.display_path_attr = None
|
|
|
39 self.converters_path_attr = None
|
|
|
40 # The 'default' converters_path defined in local datatypes_conf.xml
|
|
|
41 self.converters_path = None
|
|
|
42 # The 'default' display_path defined in local datatypes_conf.xml
|
|
|
43 self.display_applications_path = None
|
|
|
44 self.inherit_display_application_by_class = []
|
|
|
45 # Keep a list of imported proprietary datatype class modules.
|
|
|
46 self.imported_modules = []
|
|
|
47 self.datatype_elems = []
|
|
|
48 self.sniffer_elems = []
|
|
|
49 self.xml_filename = None
|
|
|
50 def load_datatypes( self, root_dir=None, config=None, deactivate=False, override=True ):
|
|
|
51 """
|
|
|
52 Parse a datatypes XML file located at root_dir/config. If deactivate is True, an installed tool shed
|
|
|
53 repository that includes proprietary datatypes is being deactivated, so appropriate loaded datatypes
|
|
|
54 will be removed from the registry. The value of override will be False when a tool shed repository is
|
|
|
55 being installed. Since installation is occurring after the datatypes registry has been initialized, its
|
|
|
56 contents cannot be overridden by new introduced conflicting data types.
|
|
|
57 """
|
|
|
58 def __import_module( full_path, datatype_module ):
|
|
|
59 sys.path.insert( 0, full_path )
|
|
|
60 imported_module = __import__( datatype_module )
|
|
|
61 sys.path.pop( 0 )
|
|
|
62 return imported_module
|
|
|
63 if root_dir and config:
|
|
|
64 handling_proprietary_datatypes = False
|
|
|
65 # Parse datatypes_conf.xml
|
|
|
66 tree = galaxy.util.parse_xml( config )
|
|
|
67 root = tree.getroot()
|
|
|
68 # Load datatypes and converters from config
|
|
|
69 if deactivate:
|
|
|
70 self.log.debug( 'Deactivating datatypes from %s' % config )
|
|
|
71 else:
|
|
|
72 self.log.debug( 'Loading datatypes from %s' % config )
|
|
|
73 registration = root.find( 'registration' )
|
|
|
74 # Set default paths defined in local datatypes_conf.xml.
|
|
|
75 if not self.converters_path:
|
|
|
76 self.converters_path_attr = registration.get( 'converters_path', 'lib/galaxy/datatypes/converters' )
|
|
|
77 self.converters_path = os.path.join( root_dir, self.converters_path_attr )
|
|
|
78 if not os.path.isdir( self.converters_path ):
|
|
|
79 raise ConfigurationError( "Directory does not exist: %s" % self.converters_path )
|
|
|
80 if not self.display_applications_path:
|
|
|
81 self.display_path_attr = registration.get( 'display_path', 'display_applications' )
|
|
|
82 self.display_applications_path = os.path.join( root_dir, self.display_path_attr )
|
|
|
83 # Proprietary datatype's <registration> tag may have special attributes, proprietary_converter_path and proprietary_display_path.
|
|
|
84 proprietary_converter_path = registration.get( 'proprietary_converter_path', None )
|
|
|
85 proprietary_display_path = registration.get( 'proprietary_display_path', None )
|
|
|
86 if proprietary_converter_path or proprietary_display_path and not handling_proprietary_datatypes:
|
|
|
87 handling_proprietary_datatypes = True
|
|
|
88 for elem in registration.findall( 'datatype' ):
|
|
|
89 try:
|
|
|
90 extension = elem.get( 'extension', None )
|
|
|
91 dtype = elem.get( 'type', None )
|
|
|
92 type_extension = elem.get( 'type_extension', None )
|
|
|
93 mimetype = elem.get( 'mimetype', None )
|
|
|
94 display_in_upload = elem.get( 'display_in_upload', False )
|
|
|
95 make_subclass = galaxy.util.string_as_bool( elem.get( 'subclass', False ) )
|
|
|
96 # Proprietary datatypes included in installed tool shed repositories will include two special attributes
|
|
|
97 # (proprietary_path and proprietary_datatype_module) if they depend on proprietary datatypes classes.
|
|
|
98 proprietary_path = elem.get( 'proprietary_path', None )
|
|
|
99 proprietary_datatype_module = elem.get( 'proprietary_datatype_module', None )
|
|
|
100 if proprietary_path or proprietary_datatype_module and not handling_proprietary_datatypes:
|
|
|
101 handling_proprietary_datatypes = True
|
|
|
102 if deactivate:
|
|
|
103 # We are deactivating an installed tool shed repository, so eliminate the
|
|
|
104 # datatype elem from the in-memory list of datatype elems.
|
|
|
105 for in_memory_elem in self.datatype_elems:
|
|
|
106 in_memory_extension = in_memory_elem.get( 'extension', None )
|
|
|
107 if in_memory_extension == extension:
|
|
|
108 in_memory_dtype = elem.get( 'type', None )
|
|
|
109 in_memory_type_extension = elem.get( 'type_extension', None )
|
|
|
110 in_memory_mimetype = elem.get( 'mimetype', None )
|
|
|
111 in_memory_display_in_upload = elem.get( 'display_in_upload', False )
|
|
|
112 in_memory_make_subclass = galaxy.util.string_as_bool( elem.get( 'subclass', False ) )
|
|
|
113 if in_memory_dtype == dtype and in_memory_type_extension == type_extension and in_memory_mimetype == mimetype \
|
|
|
114 and in_memory_display_in_upload == display_in_upload and in_memory_make_subclass == make_subclass:
|
|
|
115 self.datatype_elems.remove( in_memory_elem )
|
|
|
116 else:
|
|
|
117 # Keep an in-memory list of datatype elems to enable persistence.
|
|
|
118 self.datatype_elems.append( elem )
|
|
|
119 if extension and extension in self.datatypes_by_extension and deactivate:
|
|
|
120 # We are deactivating an installed tool shed repository, so eliminate the datatype from the registry.
|
|
|
121 # TODO: Handle deactivating datatype converters, etc before removing from self.datatypes_by_extension.
|
|
|
122 self.log.debug( "Removing datatype with extension '%s' from the registry." % extension )
|
|
|
123 del self.datatypes_by_extension[ extension ]
|
|
|
124 can_process_datatype = False
|
|
|
125 else:
|
|
|
126 can_process_datatype = ( extension and ( dtype or type_extension ) ) and ( extension not in self.datatypes_by_extension or override )
|
|
|
127 if can_process_datatype:
|
|
|
128 if dtype:
|
|
|
129 fields = dtype.split( ':' )
|
|
|
130 datatype_module = fields[0]
|
|
|
131 datatype_class_name = fields[1]
|
|
|
132 datatype_class = None
|
|
|
133 if proprietary_path and proprietary_datatype_module:
|
|
|
134 # We need to change the value of sys.path, so do it in a way that is thread-safe.
|
|
|
135 lock = threading.Lock()
|
|
|
136 lock.acquire( True )
|
|
|
137 try:
|
|
|
138 imported_module = __import_module( proprietary_path, proprietary_datatype_module )
|
|
|
139 if imported_module not in self.imported_modules:
|
|
|
140 self.imported_modules.append( imported_module )
|
|
|
141 if hasattr( imported_module, datatype_class_name ):
|
|
|
142 datatype_class = getattr( imported_module, datatype_class_name )
|
|
|
143 except Exception, e:
|
|
|
144 full_path = os.path.join( full_path, proprietary_datatype_module )
|
|
|
145 self.log.debug( "Exception importing proprietary code file %s: %s" % ( str( full_path ), str( e ) ) )
|
|
|
146 finally:
|
|
|
147 lock.release()
|
|
|
148 if datatype_class is None:
|
|
|
149 # The datatype class name must be contained in one of the datatype modules in the Galaxy distribution.
|
|
|
150 fields = datatype_module.split( '.' )
|
|
|
151 module = __import__( fields.pop(0) )
|
|
|
152 for mod in fields:
|
|
|
153 module = getattr( module, mod )
|
|
|
154 datatype_class = getattr( module, datatype_class_name )
|
|
|
155 elif type_extension:
|
|
|
156 datatype_class = self.datatypes_by_extension[type_extension].__class__
|
|
|
157 if make_subclass:
|
|
|
158 datatype_class = type( datatype_class_name, (datatype_class,), {} )
|
|
|
159 if extension in self.datatypes_by_extension:
|
|
|
160 self.log.warning( "Overriding conflicting datatype with extension '%s', using datatype from %s." % ( extension, config ) )
|
|
|
161 self.datatypes_by_extension[ extension ] = datatype_class()
|
|
|
162 if mimetype is None:
|
|
|
163 # Use default mime type as per datatype spec
|
|
|
164 mimetype = self.datatypes_by_extension[ extension ].get_mime()
|
|
|
165 self.mimetypes_by_extension[ extension ] = mimetype
|
|
|
166 if hasattr( datatype_class, "get_track_type" ):
|
|
|
167 self.available_tracks.append( extension )
|
|
|
168 if display_in_upload:
|
|
|
169 self.upload_file_formats.append( extension )
|
|
|
170 # Max file size cut off for setting optional metadata
|
|
|
171 self.datatypes_by_extension[ extension ].max_optional_metadata_filesize = elem.get( 'max_optional_metadata_filesize', None )
|
|
|
172 for converter in elem.findall( 'converter' ):
|
|
|
173 # Build the list of datatype converters which will later be loaded into the calling app's toolbox.
|
|
|
174 converter_config = converter.get( 'file', None )
|
|
|
175 target_datatype = converter.get( 'target_datatype', None )
|
|
|
176 depends_on = converter.get( 'depends_on', None )
|
|
|
177 if depends_on and target_datatype:
|
|
|
178 if extension not in self.converter_deps:
|
|
|
179 self.converter_deps[extension] = {}
|
|
|
180 self.converter_deps[extension][target_datatype] = depends_on.split(',')
|
|
|
181 if converter_config and target_datatype:
|
|
|
182 #if imported_modules:
|
|
|
183 if proprietary_converter_path:
|
|
|
184 self.proprietary_converters.append( ( converter_config, extension, target_datatype ) )
|
|
|
185 else:
|
|
|
186 self.converters.append( ( converter_config, extension, target_datatype ) )
|
|
|
187 for composite_file in elem.findall( 'composite_file' ):
|
|
|
188 # add composite files
|
|
|
189 name = composite_file.get( 'name', None )
|
|
|
190 if name is None:
|
|
|
191 self.log.warning( "You must provide a name for your composite_file (%s)." % composite_file )
|
|
|
192 optional = composite_file.get( 'optional', False )
|
|
|
193 mimetype = composite_file.get( 'mimetype', None )
|
|
|
194 self.datatypes_by_extension[extension].add_composite_file( name, optional=optional, mimetype=mimetype )
|
|
|
195 for display_app in elem.findall( 'display' ):
|
|
|
196 #if imported_modules:
|
|
|
197 if proprietary_display_path:
|
|
|
198 if elem not in self.proprietary_display_app_containers:
|
|
|
199 self.proprietary_display_app_containers.append( elem )
|
|
|
200 else:
|
|
|
201 if elem not in self.display_app_containers:
|
|
|
202 self.display_app_containers.append( elem )
|
|
|
203 elif not deactivate:
|
|
|
204 # A new tool shed repository that contains proprietary datatypes is being installed, and since installation
|
|
|
205 # is occurring after the datatypes registry has been initialized, its contents cannot be overridden by new
|
|
|
206 # introduced conflicting data types.
|
|
|
207 self.log.warning( "Ignoring conflicting datatype with extension '%s' from %s." % ( extension, config ) )
|
|
|
208 except Exception, e:
|
|
|
209 if deactivate:
|
|
|
210 self.log.warning( "Error deactivating datatype with extension '%s': %s" % ( extension, str( e ) ) )
|
|
|
211 else:
|
|
|
212 self.log.warning( "Error loading datatype with extension '%s': %s" % ( extension, str( e ) ) )
|
|
|
213 # Load datatype sniffers from the config
|
|
|
214 sniffers = root.find( 'sniffers' )
|
|
|
215 if sniffers:
|
|
|
216 for elem in sniffers.findall( 'sniffer' ):
|
|
|
217 # Keep an in-memory list of sniffer elems to enable persistence.
|
|
|
218 self.sniffer_elems.append( elem )
|
|
|
219 dtype = elem.get( 'type', None )
|
|
|
220 if dtype:
|
|
|
221 try:
|
|
|
222 fields = dtype.split( ":" )
|
|
|
223 datatype_module = fields[0]
|
|
|
224 datatype_class_name = fields[1]
|
|
|
225 module = None
|
|
|
226 #if imported_modules:
|
|
|
227 if handling_proprietary_datatypes:
|
|
|
228 # See if one of the imported modules contains the datatype class name.
|
|
|
229 for imported_module in self.imported_modules:
|
|
|
230 if hasattr( imported_module, datatype_class_name ):
|
|
|
231 module = imported_module
|
|
|
232 break
|
|
|
233 if module is None:
|
|
|
234 # The datatype class name must be contained in one of the datatype modules in the Galaxy distribution.
|
|
|
235 module = __import__( datatype_module )
|
|
|
236 for comp in datatype_module.split( '.' )[ 1: ]:
|
|
|
237 module = getattr( module, comp )
|
|
|
238 aclass = getattr( module, datatype_class_name )()
|
|
|
239 if deactivate:
|
|
|
240 for sniffer_class in self.sniff_order:
|
|
|
241 if sniffer_class.__class__ == aclass.__class__:
|
|
|
242 self.sniff_order.remove( sniffer_class )
|
|
|
243 break
|
|
|
244 self.log.debug( "Deactivated sniffer for datatype '%s'" % dtype )
|
|
|
245 else:
|
|
|
246 # See if we have a conflicting sniffer already loaded.
|
|
|
247 conflict = False
|
|
|
248 for conflict_loc, sniffer_class in enumerate( self.sniff_order ):
|
|
|
249 if sniffer_class.__class__ == aclass.__class__:
|
|
|
250 # We have a conflicting sniffer, so replace the one previously loaded.
|
|
|
251 conflict = True
|
|
|
252 if override:
|
|
|
253 del self.sniff_order[ conflict_loc ]
|
|
|
254 self.log.debug( "Replaced conflicting sniffer for datatype '%s'" % dtype )
|
|
|
255 break
|
|
|
256 if conflict:
|
|
|
257 if override:
|
|
|
258 self.sniff_order.append( aclass )
|
|
|
259 self.log.debug( "Loaded sniffer for datatype '%s'" % dtype )
|
|
|
260 else:
|
|
|
261 self.sniff_order.append( aclass )
|
|
|
262 self.log.debug( "Loaded sniffer for datatype '%s'" % dtype )
|
|
|
263 except Exception, exc:
|
|
|
264 if deactivate:
|
|
|
265 self.log.warning( "Error deactivating sniffer for datatype '%s': %s" % ( dtype, str( exc ) ) )
|
|
|
266 else:
|
|
|
267 self.log.warning( "Error appending sniffer for datatype '%s' to sniff_order: %s" % ( dtype, str( exc ) ) )
|
|
|
268 self.upload_file_formats.sort()
|
|
|
269 # Persist the xml form of the registry into a temporary file so that it
|
|
|
270 # can be loaded from the command line by tools and set_metadata processing.
|
|
|
271 self.to_xml_file()
|
|
|
272 # Default values.
|
|
|
273 if not self.datatypes_by_extension:
|
|
|
274 self.datatypes_by_extension = {
|
|
|
275 ## required for galaxy-obo:
|
|
|
276 'obo' : ontology.Obo(),
|
|
|
277 'owl' : ontology.Owl(),
|
|
|
278 'ontology' : ontology.Ontology(),
|
|
|
279 'gaf' : annotation.Gaf(),
|
|
|
280 'terf' : termenrichment.TerfTab(),
|
|
|
281
|
|
|
282 'ab1' : binary.Ab1(),
|
|
|
283 'axt' : sequence.Axt(),
|
|
|
284 'bam' : binary.Bam(),
|
|
|
285 'bed' : interval.Bed(),
|
|
|
286 'blastxml' : xml.BlastXml(),
|
|
|
287 'coverage' : coverage.LastzCoverage(),
|
|
|
288 'customtrack' : interval.CustomTrack(),
|
|
|
289 'csfasta' : sequence.csFasta(),
|
|
|
290 'fasta' : sequence.Fasta(),
|
|
|
291 'eland' : tabular.Eland(),
|
|
|
292 'fastq' : sequence.Fastq(),
|
|
|
293 'fastqsanger' : sequence.FastqSanger(),
|
|
|
294 'gtf' : interval.Gtf(),
|
|
|
295 'gff' : interval.Gff(),
|
|
|
296 'gff3' : interval.Gff3(),
|
|
|
297 'genetrack' : tracks.GeneTrack(),
|
|
|
298 'interval' : interval.Interval(),
|
|
|
299 'laj' : images.Laj(),
|
|
|
300 'lav' : sequence.Lav(),
|
|
|
301 'maf' : sequence.Maf(),
|
|
|
302 'pileup' : tabular.Pileup(),
|
|
|
303 'qualsolid' : qualityscore.QualityScoreSOLiD(),
|
|
|
304 'qualsolexa' : qualityscore.QualityScoreSolexa(),
|
|
|
305 'qual454' : qualityscore.QualityScore454(),
|
|
|
306 'sam' : tabular.Sam(),
|
|
|
307 'scf' : binary.Scf(),
|
|
|
308 'sff' : binary.Sff(),
|
|
|
309 'tabular' : tabular.Tabular(),
|
|
|
310 'taxonomy' : tabular.Taxonomy(),
|
|
|
311 'txt' : data.Text(),
|
|
|
312 'wig' : interval.Wiggle(),
|
|
|
313 'xml' : xml.GenericXml(),
|
|
|
314 }
|
|
|
315 self.mimetypes_by_extension = {
|
|
|
316 'ab1' : 'application/octet-stream',
|
|
|
317 'axt' : 'text/plain',
|
|
|
318 'bam' : 'application/octet-stream',
|
|
|
319 'bed' : 'text/plain',
|
|
|
320 'blastxml' : 'application/xml',
|
|
|
321 'customtrack' : 'text/plain',
|
|
|
322 'csfasta' : 'text/plain',
|
|
|
323 'eland' : 'application/octet-stream',
|
|
|
324 'fasta' : 'text/plain',
|
|
|
325 'fastq' : 'text/plain',
|
|
|
326 'fastqsanger' : 'text/plain',
|
|
|
327 'gtf' : 'text/plain',
|
|
|
328 'gff' : 'text/plain',
|
|
|
329 'gff3' : 'text/plain',
|
|
|
330 'interval' : 'text/plain',
|
|
|
331 'laj' : 'text/plain',
|
|
|
332 'lav' : 'text/plain',
|
|
|
333 'maf' : 'text/plain',
|
|
|
334 'memexml' : 'application/xml',
|
|
|
335 'pileup' : 'text/plain',
|
|
|
336 'qualsolid' : 'text/plain',
|
|
|
337 'qualsolexa' : 'text/plain',
|
|
|
338 'qual454' : 'text/plain',
|
|
|
339 'sam' : 'text/plain',
|
|
|
340 'scf' : 'application/octet-stream',
|
|
|
341 'sff' : 'application/octet-stream',
|
|
|
342 'tabular' : 'text/plain',
|
|
|
343 'taxonomy' : 'text/plain',
|
|
|
344 'txt' : 'text/plain',
|
|
|
345 'wig' : 'text/plain',
|
|
|
346 'xml' : 'application/xml',
|
|
|
347 }
|
|
|
348 # super supertype fix for input steps in workflows.
|
|
|
349 if 'data' not in self.datatypes_by_extension:
|
|
|
350 self.datatypes_by_extension['data'] = data.Data()
|
|
|
351 self.mimetypes_by_extension['data'] = 'application/octet-stream'
|
|
|
352 # Default values - the order in which we attempt to determine data types is critical
|
|
|
353 # because some formats are much more flexibly defined than others.
|
|
|
354 if len(self.sniff_order) < 1:
|
|
|
355 self.sniff_order = [
|
|
|
356 binary.Bam(),
|
|
|
357 binary.Sff(),
|
|
|
358 xml.BlastXml(),
|
|
|
359 xml.GenericXml(),
|
|
|
360 sequence.Maf(),
|
|
|
361 sequence.Lav(),
|
|
|
362 sequence.csFasta(),
|
|
|
363 qualityscore.QualityScoreSOLiD(),
|
|
|
364 qualityscore.QualityScore454(),
|
|
|
365 sequence.Fasta(),
|
|
|
366 sequence.Fastq(),
|
|
|
367 interval.Wiggle(),
|
|
|
368 images.Html(),
|
|
|
369 sequence.Axt(),
|
|
|
370 interval.Bed(),
|
|
|
371 interval.CustomTrack(),
|
|
|
372 interval.Gtf(),
|
|
|
373 interval.Gff(),
|
|
|
374 interval.Gff3(),
|
|
|
375 tabular.Pileup(),
|
|
|
376 interval.Interval(),
|
|
|
377 tabular.Sam(),
|
|
|
378 tabular.Eland()
|
|
|
379 ]
|
|
|
380 def append_to_sniff_order():
|
|
|
381 # Just in case any supported data types are not included in the config's sniff_order section.
|
|
|
382 for ext in self.datatypes_by_extension:
|
|
|
383 datatype = self.datatypes_by_extension[ext]
|
|
|
384 included = False
|
|
|
385 for atype in self.sniff_order:
|
|
|
386 if isinstance(atype, datatype.__class__):
|
|
|
387 included = True
|
|
|
388 break
|
|
|
389 if not included:
|
|
|
390 self.sniff_order.append(datatype)
|
|
|
391 append_to_sniff_order()
|
|
|
392 def get_available_tracks(self):
|
|
|
393 return self.available_tracks
|
|
|
394 def get_mimetype_by_extension(self, ext, default = 'application/octet-stream' ):
|
|
|
395 """Returns a mimetype based on an extension"""
|
|
|
396 try:
|
|
|
397 mimetype = self.mimetypes_by_extension[ext]
|
|
|
398 except KeyError:
|
|
|
399 #datatype was never declared
|
|
|
400 mimetype = default
|
|
|
401 self.log.warning('unknown mimetype in data factory %s' % ext)
|
|
|
402 return mimetype
|
|
|
403 def get_datatype_by_extension(self, ext ):
|
|
|
404 """Returns a datatype based on an extension"""
|
|
|
405 try:
|
|
|
406 builder = self.datatypes_by_extension[ext]
|
|
|
407 except KeyError:
|
|
|
408 builder = data.Text()
|
|
|
409 return builder
|
|
|
410 def change_datatype(self, data, ext, set_meta = True ):
|
|
|
411 data.extension = ext
|
|
|
412 # call init_meta and copy metadata from itself. The datatype
|
|
|
413 # being converted *to* will handle any metadata copying and
|
|
|
414 # initialization.
|
|
|
415 if data.has_data():
|
|
|
416 data.set_size()
|
|
|
417 data.init_meta( copy_from=data )
|
|
|
418 if set_meta:
|
|
|
419 #metadata is being set internally
|
|
|
420 data.set_meta( overwrite = False )
|
|
|
421 data.set_peek()
|
|
|
422 return data
|
|
|
423 def old_change_datatype(self, data, ext):
|
|
|
424 """Creates and returns a new datatype based on an existing data and an extension"""
|
|
|
425 newdata = factory(ext)(id=data.id)
|
|
|
426 for key, value in data.__dict__.items():
|
|
|
427 setattr(newdata, key, value)
|
|
|
428 newdata.ext = ext
|
|
|
429 return newdata
|
|
|
430 def load_datatype_converters( self, toolbox, installed_repository_dict=None, deactivate=False ):
|
|
|
431 """
|
|
|
432 If deactivate is False, add datatype converters from self.converters or self.proprietary_converters
|
|
|
433 to the calling app's toolbox. If deactivate is True, eliminates relevant converters from the calling
|
|
|
434 app's toolbox.
|
|
|
435 """
|
|
|
436 if installed_repository_dict:
|
|
|
437 # Load converters defined by datatypes_conf.xml included in installed tool shed repository.
|
|
|
438 converters = self.proprietary_converters
|
|
|
439 else:
|
|
|
440 # Load converters defined by local datatypes_conf.xml.
|
|
|
441 converters = self.converters
|
|
|
442 for elem in converters:
|
|
|
443 tool_config = elem[0]
|
|
|
444 source_datatype = elem[1]
|
|
|
445 target_datatype = elem[2]
|
|
|
446 if installed_repository_dict:
|
|
|
447 converter_path = installed_repository_dict[ 'converter_path' ]
|
|
|
448 config_path = os.path.join( converter_path, tool_config )
|
|
|
449 else:
|
|
|
450 config_path = os.path.join( self.converters_path, tool_config )
|
|
|
451 try:
|
|
|
452 converter = toolbox.load_tool( config_path )
|
|
|
453 if installed_repository_dict:
|
|
|
454 # If the converter is included in an installed tool shed repository, set the tool
|
|
|
455 # shed related tool attributes.
|
|
|
456 converter.tool_shed = installed_repository_dict[ 'tool_shed' ]
|
|
|
457 converter.repository_name = installed_repository_dict[ 'repository_name' ]
|
|
|
458 converter.repository_owner = installed_repository_dict[ 'repository_owner' ]
|
|
|
459 converter.installed_changeset_revision = installed_repository_dict[ 'installed_changeset_revision' ]
|
|
|
460 converter.old_id = converter.id
|
|
|
461 # The converter should be included in the list of tools defined in tool_dicts.
|
|
|
462 tool_dicts = installed_repository_dict[ 'tool_dicts' ]
|
|
|
463 for tool_dict in tool_dicts:
|
|
|
464 if tool_dict[ 'id' ] == converter.id:
|
|
|
465 converter.guid = tool_dict[ 'guid' ]
|
|
|
466 converter.id = tool_dict[ 'guid' ]
|
|
|
467 break
|
|
|
468 if deactivate:
|
|
|
469 del toolbox.tools_by_id[ converter.id ]
|
|
|
470 if source_datatype in self.datatype_converters:
|
|
|
471 del self.datatype_converters[ source_datatype ][ target_datatype ]
|
|
|
472 self.log.debug( "Deactivated converter: %s", converter.id )
|
|
|
473 else:
|
|
|
474 toolbox.tools_by_id[ converter.id ] = converter
|
|
|
475 if source_datatype not in self.datatype_converters:
|
|
|
476 self.datatype_converters[ source_datatype ] = odict()
|
|
|
477 self.datatype_converters[ source_datatype ][ target_datatype ] = converter
|
|
|
478 self.log.debug( "Loaded converter: %s", converter.id )
|
|
|
479 except Exception, e:
|
|
|
480 if deactivate:
|
|
|
481 self.log.exception( "Error deactivating converter (%s): %s" % ( config_path, str( e ) ) )
|
|
|
482 else:
|
|
|
483 self.log.exception( "Error loading converter (%s): %s" % ( config_path, str( e ) ) )
|
|
|
484 def load_display_applications( self, installed_repository_dict=None, deactivate=False ):
|
|
|
485 """
|
|
|
486 If deactivate is False, add display applications from self.display_app_containers or
|
|
|
487 self.proprietary_display_app_containers to appropriate datatypes. If deactivate is
|
|
|
488 True, eliminates relevant display applications from appropriate datatypes.
|
|
|
489 """
|
|
|
490 if installed_repository_dict:
|
|
|
491 # Load display applications defined by datatypes_conf.xml included in installed tool shed repository.
|
|
|
492 datatype_elems = self.proprietary_display_app_containers
|
|
|
493 else:
|
|
|
494 # Load display applications defined by local datatypes_conf.xml.
|
|
|
495 datatype_elems = self.display_app_containers
|
|
|
496 for elem in datatype_elems:
|
|
|
497 extension = elem.get( 'extension', None )
|
|
|
498 for display_app in elem.findall( 'display' ):
|
|
|
499 display_file = display_app.get( 'file', None )
|
|
|
500 if installed_repository_dict:
|
|
|
501 display_path = installed_repository_dict[ 'display_path' ]
|
|
|
502 display_file_head, display_file_tail = os.path.split( display_file )
|
|
|
503 config_path = os.path.join( display_path, display_file_tail )
|
|
|
504 else:
|
|
|
505 config_path = os.path.join( self.display_applications_path, display_file )
|
|
|
506 try:
|
|
|
507 inherit = galaxy.util.string_as_bool( display_app.get( 'inherit', 'False' ) )
|
|
|
508 display_app = DisplayApplication.from_file( config_path, self )
|
|
|
509 if display_app:
|
|
|
510 if display_app.id in self.display_applications:
|
|
|
511 if deactivate:
|
|
|
512 del self.display_applications[ display_app.id ]
|
|
|
513 else:
|
|
|
514 # If we already loaded this display application, we'll use the first one loaded.
|
|
|
515 display_app = self.display_applications[ display_app.id ]
|
|
|
516 elif installed_repository_dict:
|
|
|
517 # If the display application is included in an installed tool shed repository,
|
|
|
518 # set the tool shed related tool attributes.
|
|
|
519 display_app.tool_shed = installed_repository_dict[ 'tool_shed' ]
|
|
|
520 display_app.repository_name = installed_repository_dict[ 'repository_name' ]
|
|
|
521 display_app.repository_owner = installed_repository_dict[ 'repository_owner' ]
|
|
|
522 display_app.installed_changeset_revision = installed_repository_dict[ 'installed_changeset_revision' ]
|
|
|
523 display_app.old_id = display_app.id
|
|
|
524 # The display application should be included in the list of tools defined in tool_dicts.
|
|
|
525 tool_dicts = installed_repository_dict[ 'tool_dicts' ]
|
|
|
526 for tool_dict in tool_dicts:
|
|
|
527 if tool_dict[ 'id' ] == display_app.id:
|
|
|
528 display_app.guid = tool_dict[ 'guid' ]
|
|
|
529 display_app.id = tool_dict[ 'guid' ]
|
|
|
530 break
|
|
|
531 if deactivate:
|
|
|
532 del self.display_applications[ display_app.id ]
|
|
|
533 del self.datatypes_by_extension[ extension ].display_applications[ display_app.id ]
|
|
|
534 if inherit and ( self.datatypes_by_extension[ extension ], display_app ) in self.inherit_display_application_by_class:
|
|
|
535 self.inherit_display_application_by_class.remove( ( self.datatypes_by_extension[ extension ], display_app ) )
|
|
|
536 self.log.debug( "Deactivated display application '%s' for datatype '%s'." % ( display_app.id, extension ) )
|
|
|
537 else:
|
|
|
538 self.display_applications[ display_app.id ] = display_app
|
|
|
539 self.datatypes_by_extension[ extension ].add_display_application( display_app )
|
|
|
540 if inherit and ( self.datatypes_by_extension[ extension ], display_app ) not in self.inherit_display_application_by_class:
|
|
|
541 self.inherit_display_application_by_class.append( ( self.datatypes_by_extension[ extension ], display_app ) )
|
|
|
542 self.log.debug( "Loaded display application '%s' for datatype '%s', inherit=%s." % ( display_app.id, extension, inherit ) )
|
|
|
543 except Exception, e:
|
|
|
544 if deactivate:
|
|
|
545 self.log.exception( "Error deactivating display application (%s): %s" % ( config_path, str( e ) ) )
|
|
|
546 else:
|
|
|
547 self.log.exception( "Error loading display application (%s): %s" % ( config_path, str( e ) ) )
|
|
|
548 # Handle display_application subclass inheritance.
|
|
|
549 for extension, d_type1 in self.datatypes_by_extension.iteritems():
|
|
|
550 for d_type2, display_app in self.inherit_display_application_by_class:
|
|
|
551 current_app = d_type1.get_display_application( display_app.id, None )
|
|
|
552 if current_app is None and isinstance( d_type1, type( d_type2 ) ):
|
|
|
553 self.log.debug( "Adding inherited display application '%s' to datatype '%s'" % ( display_app.id, extension ) )
|
|
|
554 d_type1.add_display_application( display_app )
|
|
|
555 def load_external_metadata_tool( self, toolbox ):
|
|
|
556 """Adds a tool which is used to set external metadata"""
|
|
|
557 # We need to be able to add a job to the queue to set metadata. The queue will currently only accept jobs with an associated
|
|
|
558 # tool. We'll create a special tool to be used for Auto-Detecting metadata; this is less than ideal, but effective
|
|
|
559 # Properly building a tool without relying on parsing an XML file is near impossible...so we'll create a temporary file
|
|
|
560 tool_xml_text = """
|
|
|
561 <tool id="__SET_METADATA__" name="Set External Metadata" version="1.0.1" tool_type="set_metadata">
|
|
|
562 <type class="SetMetadataTool" module="galaxy.tools"/>
|
|
|
563 <action module="galaxy.tools.actions.metadata" class="SetMetadataToolAction"/>
|
|
|
564 <command>$__SET_EXTERNAL_METADATA_COMMAND_LINE__</command>
|
|
|
565 <inputs>
|
|
|
566 <param format="data" name="input1" type="data" label="File to set metadata on."/>
|
|
|
567 <param name="__ORIGINAL_DATASET_STATE__" type="hidden" value=""/>
|
|
|
568 <param name="__SET_EXTERNAL_METADATA_COMMAND_LINE__" type="hidden" value=""/>
|
|
|
569 </inputs>
|
|
|
570 </tool>
|
|
|
571 """
|
|
|
572 tmp_name = tempfile.NamedTemporaryFile()
|
|
|
573 tmp_name.write( tool_xml_text )
|
|
|
574 tmp_name.flush()
|
|
|
575 set_meta_tool = toolbox.load_tool( tmp_name.name )
|
|
|
576 toolbox.tools_by_id[ set_meta_tool.id ] = set_meta_tool
|
|
|
577 self.set_external_metadata_tool = set_meta_tool
|
|
|
578 self.log.debug( "Loaded external metadata tool: %s", self.set_external_metadata_tool.id )
|
|
|
579 def get_converters_by_datatype(self, ext):
|
|
|
580 """Returns available converters by source type"""
|
|
|
581 converters = odict()
|
|
|
582 source_datatype = type(self.get_datatype_by_extension(ext))
|
|
|
583 for ext2, dict in self.datatype_converters.items():
|
|
|
584 converter_datatype = type(self.get_datatype_by_extension(ext2))
|
|
|
585 if issubclass(source_datatype, converter_datatype):
|
|
|
586 converters.update(dict)
|
|
|
587 #Ensure ext-level converters are present
|
|
|
588 if ext in self.datatype_converters.keys():
|
|
|
589 converters.update(self.datatype_converters[ext])
|
|
|
590 return converters
|
|
|
591 def get_converter_by_target_type(self, source_ext, target_ext):
|
|
|
592 """Returns a converter based on source and target datatypes"""
|
|
|
593 converters = self.get_converters_by_datatype(source_ext)
|
|
|
594 if target_ext in converters.keys():
|
|
|
595 return converters[target_ext]
|
|
|
596 return None
|
|
|
597 def find_conversion_destination_for_dataset_by_extensions( self, dataset, accepted_formats, converter_safe = True ):
|
|
|
598 """Returns ( target_ext, existing converted dataset )"""
|
|
|
599 for convert_ext in self.get_converters_by_datatype( dataset.ext ):
|
|
|
600 if isinstance( self.get_datatype_by_extension( convert_ext ), accepted_formats ):
|
|
|
601 converted_dataset = dataset.get_converted_files_by_type( convert_ext )
|
|
|
602 if converted_dataset:
|
|
|
603 ret_data = converted_dataset
|
|
|
604 elif not converter_safe:
|
|
|
605 continue
|
|
|
606 else:
|
|
|
607 ret_data = None
|
|
|
608 return ( convert_ext, ret_data )
|
|
|
609 return ( None, None )
|
|
|
610 def get_composite_extensions( self ):
|
|
|
611 return [ ext for ( ext, d_type ) in self.datatypes_by_extension.iteritems() if d_type.composite_type is not None ]
|
|
|
612 def get_upload_metadata_params( self, context, group, tool ):
|
|
|
613 """Returns dict of case value:inputs for metadata conditional for upload tool"""
|
|
|
614 rval = {}
|
|
|
615 for ext, d_type in self.datatypes_by_extension.iteritems():
|
|
|
616 inputs = []
|
|
|
617 for meta_name, meta_spec in d_type.metadata_spec.iteritems():
|
|
|
618 if meta_spec.set_in_upload:
|
|
|
619 help_txt = meta_spec.desc
|
|
|
620 if not help_txt or help_txt == meta_name:
|
|
|
621 help_txt = ""
|
|
|
622 inputs.append( '<param type="text" name="%s" label="Set metadata value for "%s"" value="%s" help="%s"/>' % ( meta_name, meta_name, meta_spec.default, help_txt ) )
|
|
|
623 rval[ ext ] = "\n".join( inputs )
|
|
|
624 if 'auto' not in rval and 'txt' in rval: #need to manually add 'auto' datatype
|
|
|
625 rval[ 'auto' ] = rval[ 'txt' ]
|
|
|
626 return rval
|
|
|
627 @property
|
|
|
628 def integrated_datatypes_configs( self ):
|
|
|
629 if self.xml_filename and os.path.isfile( self.xml_filename ):
|
|
|
630 return self.xml_filename
|
|
|
631 self.to_xml_file()
|
|
|
632 return self.xml_filename
|
|
|
633 def to_xml_file( self ):
|
|
|
634 if self.xml_filename is not None:
|
|
|
635 # If persisted previously, attempt to remove
|
|
|
636 # the temporary file in which we were written.
|
|
|
637 try:
|
|
|
638 os.unlink( self.xml_filename )
|
|
|
639 except:
|
|
|
640 pass
|
|
|
641 self.xml_filename = None
|
|
|
642 fd, filename = tempfile.mkstemp()
|
|
|
643 self.xml_filename = os.path.abspath( filename )
|
|
|
644 if self.converters_path_attr:
|
|
|
645 converters_path_str = ' converters_path="%s"' % self.converters_path_attr
|
|
|
646 else:
|
|
|
647 converters_path_str = ''
|
|
|
648 if self.display_path_attr:
|
|
|
649 display_path_str = ' display_path="%s"' % self.display_path_attr
|
|
|
650 else:
|
|
|
651 display_path_str = ''
|
|
|
652 os.write( fd, '<?xml version="1.0"?>\n' )
|
|
|
653 os.write( fd, '<datatypes>\n' )
|
|
|
654 os.write( fd, '<registration%s%s>\n' % ( converters_path_str, display_path_str ) )
|
|
|
655 for elem in self.datatype_elems:
|
|
|
656 os.write( fd, '%s' % galaxy.util.xml_to_string( elem ) )
|
|
|
657 os.write( fd, '</registration>\n' )
|
|
|
658 os.write( fd, '<sniffers>\n' )
|
|
|
659 for elem in self.sniffer_elems:
|
|
|
660 os.write( fd, '%s' % galaxy.util.xml_to_string( elem ) )
|
|
|
661 os.write( fd, '</sniffers>\n' )
|
|
|
662 os.write( fd, '</datatypes>\n' )
|
|
|
663 os.close( fd )
|
|
|
664 os.chmod( self.xml_filename, 0644 )
|
