Mercurial > repos > yating-l > jbrowsearchivecreator

comparison TrackHub.py @ 25:31a41ce128cc draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload for repository https://github.com/Yating-L/jbrowse-archive-creator.git commit 691e5366893905d30943a3cb8cdfb6341f0f5362-dirty
author yating-l
date Fri, 13 Oct 2017 12:44:31 -0400
parents bb6fdccef474
children 127037c49bc8
comparison
equal deleted inserted replaced
24:bb6fdccef474 25:31a41ce128cc
3 import os 3 import os
4 import subprocess 4 import subprocess
5 import shutil 5 import shutil
6 import zipfile 6 import zipfile
7 import json 7 import json
8 import utils 8 import tempfile
9 import logging
10
11 from datatypes.Datatype import Datatype
12 from apollo.ApolloInstance import ApolloInstance
13 from tracks.TrackStyles import TrackStyles
14 from util import subtools
15 from util import santitizer
9 16
10 17
11 class TrackHub: 18 class TrackHub:
12 def __init__(self, inputFiles, reference, outputDirect, tool_dir, genome, extra_files_path, metaData, jbrowse_host): 19 def __init__(self, inputFastaFile, apollo_user, outputFile, extra_files_path, tool_directory, trackType, apollo_host):
13 self.input_files = inputFiles.tracks 20
14 self.outfile = outputDirect 21 self.rootAssemblyHub = None
15 self.outfolder = extra_files_path 22
16 self.out_path = os.path.join(extra_files_path, 'myHub') 23 self.mySpecieFolderPath = None
17 self.reference = reference 24
18 self.tool_dir = tool_dir 25 # Store intermediate files, will be removed if not in debug mode
19 self.metaData = metaData 26 self.myTracksFolderPath = None
20 self.raw = os.path.join(self.out_path, 'raw') 27
21 self.json = os.path.join(self.out_path, 'json') 28 # Store binary files: Bam, BigWig
22 self.jbrowse_host = jbrowse_host 29 self.myBinaryFolderPath = None
23 try: 30
24 if os.path.exists(self.json): 31 self.tool_directory = tool_directory
25 shutil.rmtree(self.json) 32 self.trackType = trackType
26 os.makedirs(self.json) 33 self.reference_genome = inputFastaFile
27 except OSError as e: 34 self.genome_name = inputFastaFile.assembly_id
28 print "Cannot create json folder error({0}): {1}".format(e.errno, e.strerror) 35 self.extra_files_path = extra_files_path
29 else: 36 self.outputFile = outputFile
30 print "Create jbrowse folder {}".format(self.out_path) 37 self.chromSizesFile = None
38
39 # Set up apollo
40 self.apollo = ApolloInstance(apollo_host)
41 self.apollo_user = apollo_user
42
43 # Set all the missing variables of this class, and create physically the folders/files
44 self.rootAssemblyHub = self.__createAssemblyHub__(extra_files_path=extra_files_path)
45 # Init the Datatype
46 Datatype.pre_init(self.reference_genome, self.chromSizesFile,
47 self.extra_files_path, self.tool_directory,
48 self.mySpecieFolderPath, self.myTracksFolderPath, self.myBinaryFolderPath, self.trackType)
49
50 self._prepareRefseq()
51 self.trackList = os.path.join(self.mySpecieFolderPath, "trackList.json")
52 self._createTrackList()
53
54 self.myTrackStyle = TrackStyles(self.tool_directory, self.mySpecieFolderPath, self.trackList)
55 #self.cssFolderPath = os.path.join(self.mySpecieFolderPath, 'css')
56 #self.cssFilePath = os.path.join(self.cssFolderPath, 'custom_track_styles.css')
57 self.logger = logging.getLogger(__name__)
58
31 59
32 def createHub(self): 60
33 self.prepareRefseq() 61 def addTrack(self, trackDbObject):
34 for input_file in self.input_files: 62 if trackDbObject['dataType'].lower() == 'bam':
35 self.addTrack(input_file) 63 #new_track = subprocess.Popen(['echo', trackDbObject['options']], stdout=subprocess.PIPE)
36 self.indexName() 64 #subprocess.call(['add-track-json.pl', json_file], stdin=new_track.stdout)
37 slink = self.makeArchive() 65 subtools.add_track_json(self.trackList, trackDbObject['options'])
38 self.outHtml(slink) 66 #subtools.add_track_json(self.trackList, trackDbObject['track_json'])
67 elif trackDbObject['dataType'].lower() == 'bigwig':
68 subtools.add_track_json(self.trackList, trackDbObject['options'])
69 else:
70 if trackDbObject['trackType'] == 'HTMLFeatures':
71 self._customizeHTMLFeature(trackDbObject)
72 subtools.flatfile_to_json(trackDbObject['trackDataURL'], trackDbObject['dataType'], trackDbObject['trackType'], trackDbObject['trackLabel'], self.mySpecieFolderPath, trackDbObject['options'])
73
74
75 def terminate(self, debug=False):
76 """ Write html file """
77 self._indexName()
78 if not debug:
79 self._removeRaw()
80 self._makeArchive()
39 print "Success!\n" 81 print "Success!\n"
40 82
41 def prepareRefseq(self): 83
42 try: 84 def _customizeHTMLFeature(self, trackDbObject):
85 if trackDbObject['options']:
86 subfeatures = trackDbObject['options'].get('subfeatureClasses')
87 feature_color = trackDbObject['options']['feature_color']
88 if subfeatures:
89 for key, value in subfeatures.items():
90 self.myTrackStyle.addCustomColor(value, feature_color)
91 else:
92 customizedFeature = santitizer.sanitize_name(trackDbObject['trackLabel'])
93 clientConfig = json.loads(trackDbObject['options']['clientConfig'])
94 clientConfig['renderClassName'] = customizedFeature
95 trackDbObject['options']['clientConfig'] = json.dumps(clientConfig)
96 self.myTrackStyle.addCustomColor(customizedFeature, feature_color)
97
98 def _removeRaw(self):
99 if os.path.exists(self.myTracksFolderPath):
100 shutil.rmtree(self.myTracksFolderPath)
101
102 def _createTrackList(self):
103 if not os.path.exists(self.trackList):
104 os.mknod(self.trackList)
105
106 def _prepareRefseq(self):
107 subtools.prepare_refseqs(self.reference_genome.false_path, self.mySpecieFolderPath)
108 #try:
43 #print os.path.join(self.tool_dir, 'prepare-refseqs.pl') + ", '--fasta', " + self.reference +", '--out', self.json])" 109 #print os.path.join(self.tool_dir, 'prepare-refseqs.pl') + ", '--fasta', " + self.reference +", '--out', self.json])"
44 subprocess.call(['prepare-refseqs.pl', '--fasta', self.reference, '--out', self.json]) 110 #subprocess.call(['prepare-refseqs.pl', '--fasta', self.reference_genome.false_path, '--out', self.mySpecieFolderPath])
45 except OSError as e: 111 #except OSError as e:
46 print "Cannot prepare reference error({0}): {1}".format(e.errno, e.strerror) 112 #print "Cannot prepare reference error({0}): {1}".format(e.errno, e.strerror)
47 #TODO: hard coded the bam and bigwig tracks. Need to allow users to customize the settings 113
48 def addTrack(self, track): 114 def _indexName(self):
49 #print "false_path" , track['false_path'] 115 #subprocess.call(['generate-names.pl', '-v', '--out', self.mySpecieFolderPath])
50 if track['false_path'] in self.metaData.keys(): 116 subtools.generate_names(self.mySpecieFolderPath)
51 metadata = self.metaData[track['false_path']]
52 else:
53 metadata = {}
54 self.SetMetadata(track, metadata)
55 if track['dataType'] == 'bam':
56 self.Bam(track, metadata)
57 # print "add bam track\n"
58 elif track['dataType'] == 'bigwig':
59 #create trackList.json if not exist
60 self.createTrackList()
61 json_file = os.path.join(self.json, "trackList.json")
62 bigwig_file = os.path.join(self.raw, track['fileName'])
63 subprocess.call(['add-bw-track.pl', '--label', metadata['label'], '--bw_url', bigwig_file, '--pos_color', metadata['style']['pos_color'], '--neg_color', metadata['style']['neg_color'], '--plot', 'JBrowse/View/Track/Wiggle/XYPlot', '--out', json_file, '--in', json_file])
64 else:
65 flat_file = os.path.join(self.raw, track['fileName'])
66 if track['dataType'] == 'bed':
67 subprocess.call(['flatfile-to-json.pl', '--bed', flat_file, '--trackType', metadata['type'], '--trackLabel', metadata['label'], '--Config', '{"category" : "%s"}' % metadata['category'], '--clientConfig', '{"color" : "%s"}' % metadata['color'], '--out', self.json])
68 elif track['dataType'] == 'bedSpliceJunctions' or track['dataType'] == 'gtf' or track['dataType'] == 'blastxml':
69 subprocess.call(['flatfile-to-json.pl', '--gff', flat_file, '--trackType', metadata['type'], '--trackLabel', metadata['label'], '--Config', '{"glyph": "JBrowse/View/FeatureGlyph/Segments", "category" : "%s"}' % metadata['category'], '--clientConfig', '{"color" : "%s"}' % metadata['color'], '--out', self.json])
70 elif track['dataType'] == 'gff3_transcript':
71 subprocess.call(['flatfile-to-json.pl', '--gff', flat_file, '--trackType', metadata['type'], '--trackLabel', metadata['label'], '--Config', '{"transcriptType": "transcript", "category" : "%s"}' % metadata['category'], '--clientConfig', '{"color" : "%s"}' % metadata['color'], '--out', self.json])
72 else:
73 subprocess.call(['flatfile-to-json.pl', '--gff', flat_file, '--trackType', metadata['type'], '--trackLabel', metadata['label'], '--Config', '{"category" : "%s"}' % metadata['category'], '--clientConfig', '{"color" : "%s"}' % metadata['color'], '--out', self.json])
74
75 def indexName(self):
76 subprocess.call(['generate-names.pl', '-v', '--out', self.json])
77 print "finished name index \n" 117 print "finished name index \n"
78 118
79 def makeArchive(self): 119 def _outHtml(self, host_name):
80 file_dir = os.path.abspath(self.outfile) 120 with open(self.outputFile, 'w') as htmlfile:
81 source_dir = os.path.dirname(file_dir) 121 htmlstr = 'The new Organism "%s" is created on Apollo: <br>' % self.genome_name
82 folder_name = os.path.basename(self.outfolder) 122 jbrowse_hub = '<li><a href = "%s" target="_blank">View JBrowse Hub on Apollo</a></li>' % host_name
83 source_name = os.path.basename(self.out_path) 123 htmlstr += jbrowse_hub
84 source = os.path.join(source_dir, folder_name, source_name) 124 htmlfile.write(htmlstr)
85 slink = source.replace('/', '_')
86 slink = os.path.join('/var/www/html/JBrowse-1.12.1/data', slink)
87 try:
88 if os.path.islink(slink):
89 os.unlink(slink)
90 except OSError as oserror:
91 print "Cannot create symlink to the data({0}): {1}".format(oserror.errno, oserror.strerror)
92 os.symlink(source, slink)
93 return slink
94
95 def outHtml(self, slink):
96 with open(self.outfile, 'w') as htmlfile:
97 htmlstr = 'The JBrowse Hub is created: <br>'
98 url = self.jbrowse_host + "/JBrowse-1.12.1/index.html?data=%s"
99 jbrowse_hub = '<li><a href = "%s" target="_blank">View JBrowse Hub</a></li>' % url
100 link_name = os.path.basename(slink)
101 relative_path = os.path.join('data', link_name + '/json')
102 htmlstr += jbrowse_hub % relative_path
103 htmlfile.write(htmlstr)
104 125
105 def createTrackList(self): 126 def _makeArchive(self):
106 trackList = os.path.join(self.json, "trackList.json") 127 self.apollo.loadHubToApollo(self.apollo_user, self.genome_name, self.mySpecieFolderPath, admin=True)
107 if not os.path.exists(trackList): 128 apollo_host = self.apollo.getHost()
108 os.mknod(trackList) 129 self._outHtml(apollo_host)
109 130
110 def Bam(self, track, metadata):
111 #create trackList.json if not exist
112 self.createTrackList()
113 json_file = os.path.join(self.json, "trackList.json")
114 bam_track = dict()
115 bam_track['type'] = 'JBrowse/View/Track/Alignments2'
116 bam_track['storeClass'] = 'JBrowse/Store/SeqFeature/BAM'
117 bam_track['urlTemplate'] = os.path.join('../raw', track['fileName'])
118 bam_track['baiUrlTemplate'] = os.path.join('../raw', track['index'])
119 bam_track['label'] = metadata['label']
120 bam_track['category'] = metadata['category']
121 bam_track = json.dumps(bam_track)
122 #Use add-track-json.pl to add bam track to json file
123 new_track = subprocess.Popen(['echo', bam_track], stdout=subprocess.PIPE)
124 subprocess.call(['add-track-json.pl', json_file], stdin=new_track.stdout)
125 '''
126 def BigWig(self, track, metadata):
127 #create trackList.json if not exist
128 self.createTrackList()
129 json_file = os.path.join(self.json, "trackList.json")
130 bigwig_track = dict()
131 bigwig_track['urlTemplate'] = os.path.join('../raw', track['fileName'])
132 bigwig_track['type'] = 'JBrowse/View/Track/Wiggle/XYPlot'
133 bigwig_track['storeClass'] = 'JBrowse/Store/SeqFeature/BigWig'
134 bigwig_track['label'] = metadata['label']
135 bigwig_track['style'] = metadata['style']
136 bigwig_track['category'] = metadata['category']
137 bigwig_track = json.dumps(bigwig_track)
138 #Use add-track-json.pl to add bigwig track to json file
139 new_track = subprocess.Popen(['echo', bigwig_track], stdout=subprocess.PIPE)
140 #output = new_track.communicate()[0]
141 subprocess.call(['add-track-json.pl', json_file], stdin=new_track.stdout)
142 '''
143 def BigWig
144 #If the metadata is not set, use the default value
145 def SetMetadata(self, track, metadata):
146 if 'label' not in metadata.keys() or metadata['label'] == '':
147 metadata['label'] = track['fileName']
148 if 'color' not in metadata.keys() or metadata['color'] == '':
149 metadata['color'] = "#daa520"
150 if track['dataType'] == 'bigwig':
151 if 'style' not in metadata.keys():
152 metadata['style'] = {}
153 if 'pos_color' not in metadata['style'] or metadata['style']['pos_color'] == '':
154 metadata['style']['pos_color'] = "#FFA600"
155 if 'neg_color' not in metadata['style'] or metadata['style']['neg_color'] == '':
156 metadata['style']['neg_color'] = "#005EFF"
157 if 'category' not in metadata.keys() or metadata['category'] == '':
158 metadata['category'] = "Default group"
159 if track['dataType'] == 'blastxml':
160 metadata['type'] = "G-OnRamp_plugin/BlastAlignment"
161 elif track['dataType'] == 'bigpsl':
162 metadata['type'] = "G-OnRamp_plugin/BlatAlignment"
163 elif track['dataType'] == 'gff3_transcript' or track['dataType'] == 'gff3_mrna':
164 metadata['type'] = "G-OnRamp_plugin/GenePred"
165 else:
166 metadata['type'] = "CanvasFeatures"
167 131
132 def __createAssemblyHub__(self, extra_files_path):
133 # Get all necessaries infos first
134 # 2bit file creation from input fasta
168 135
136 # baseNameFasta = os.path.basename(fasta_file_name)
137 # suffixTwoBit, extensionTwoBit = os.path.splitext(baseNameFasta)
138 # nameTwoBit = suffixTwoBit + '.2bit'
139 twoBitFile = tempfile.NamedTemporaryFile(bufsize=0)
140 subtools.faToTwoBit(self.reference_genome.false_path, twoBitFile.name)
169 141
170 142 # Generate the twoBitInfo
143 twoBitInfoFile = tempfile.NamedTemporaryFile(bufsize=0)
144 subtools.twoBitInfo(twoBitFile.name, twoBitInfoFile.name)
171 145
146 # Then we get the output to generate the chromSizes
147 self.chromSizesFile = tempfile.NamedTemporaryFile(bufsize=0, suffix=".chrom.sizes")
148 subtools.sortChromSizes(twoBitInfoFile.name, self.chromSizesFile.name)
172 149
150 # We can get the biggest scaffold here, with chromSizesFile
151 with open(self.chromSizesFile.name, 'r') as chrom_sizes:
152 # TODO: Check if exists
153 self.default_pos = chrom_sizes.readline().split()[0]
173 154
155 # TODO: Manage to put every fill Function in a file dedicated for reading reasons
156 # Create the root directory
157 myHubPath = os.path.join(extra_files_path, "myHub")
158 if not os.path.exists(myHubPath):
159 os.makedirs(myHubPath)
160
161 # Create the specie folder
162 # TODO: Generate the name depending on the specie
163 mySpecieFolderPath = os.path.join(myHubPath, self.genome_name)
164 if not os.path.exists(mySpecieFolderPath):
165 os.makedirs(mySpecieFolderPath)
166 self.mySpecieFolderPath = mySpecieFolderPath
167
168 # We create the 2bit file while we just created the specie folder
169 #self.twoBitName = self.genome_name + ".2bit"
170 #self.two_bit_final_path = os.path.join(self.mySpecieFolderPath, self.twoBitName)
171 #shutil.copyfile(twoBitFile.name, self.two_bit_final_path)
172
173 # Create the folder tracks into the specie folder
174 tracksFolderPath = os.path.join(mySpecieFolderPath, "raw")
175 if not os.path.exists(tracksFolderPath):
176 os.makedirs(tracksFolderPath)
177 self.myTracksFolderPath = tracksFolderPath
178
179 myBinaryFolderPath = os.path.join(mySpecieFolderPath, 'bbi')
180 if not os.path.exists(myBinaryFolderPath):
181 os.makedirs(myBinaryFolderPath)
182 self.myBinaryFolderPath = myBinaryFolderPath
183
184 return myHubPath