diff TrackHub.py @ 25:31a41ce128cc draft

planemo upload for repository https://github.com/Yating-L/jbrowse-archive-creator.git commit 691e5366893905d30943a3cb8cdfb6341f0f5362-dirty

--- a/TrackHub.py	Wed Jul 12 12:55:27 2017 -0400
+++ b/TrackHub.py	Fri Oct 13 12:44:31 2017 -0400
@@ -5,169 +5,180 @@
 import shutil
 import zipfile
 import json
-import utils
+import tempfile
+import logging
+
+from datatypes.Datatype import Datatype
+from apollo.ApolloInstance import ApolloInstance
+from tracks.TrackStyles import TrackStyles
+from util import subtools
+from util import santitizer
 
 
 class TrackHub:
-    def __init__(self, inputFiles, reference, outputDirect, tool_dir, genome, extra_files_path, metaData, jbrowse_host):
-        self.input_files = inputFiles.tracks
-        self.outfile = outputDirect
-        self.outfolder = extra_files_path
-        self.out_path = os.path.join(extra_files_path, 'myHub')
-        self.reference = reference
-        self.tool_dir = tool_dir
-        self.metaData = metaData
-        self.raw = os.path.join(self.out_path, 'raw')
-        self.json = os.path.join(self.out_path, 'json')
-        self.jbrowse_host = jbrowse_host
-        try: 
-            if os.path.exists(self.json):
-                shutil.rmtree(self.json)
-            os.makedirs(self.json)
-        except OSError as e:
-            print "Cannot create json folder error({0}): {1}".format(e.errno, e.strerror)
-        else:
-            print "Create jbrowse folder {}".format(self.out_path)
+    def __init__(self, inputFastaFile, apollo_user, outputFile, extra_files_path, tool_directory, trackType, apollo_host):
+        
+        self.rootAssemblyHub = None
+
+        self.mySpecieFolderPath = None
+
+        # Store intermediate files, will be removed if not in debug mode
+        self.myTracksFolderPath = None
+
+        # Store binary files: Bam, BigWig
+        self.myBinaryFolderPath = None
+
+        self.tool_directory = tool_directory
+        self.trackType = trackType
+        self.reference_genome = inputFastaFile
+        self.genome_name = inputFastaFile.assembly_id
+        self.extra_files_path = extra_files_path
+        self.outputFile = outputFile
+        self.chromSizesFile = None
+
+        # Set up apollo
+        self.apollo = ApolloInstance(apollo_host)
+        self.apollo_user = apollo_user
+        
+        # Set all the missing variables of this class, and create physically the folders/files
+        self.rootAssemblyHub = self.__createAssemblyHub__(extra_files_path=extra_files_path)
+        # Init the Datatype 
+        Datatype.pre_init(self.reference_genome, self.chromSizesFile,
+                          self.extra_files_path, self.tool_directory,
+                          self.mySpecieFolderPath, self.myTracksFolderPath, self.myBinaryFolderPath, self.trackType)  
+
+        self._prepareRefseq()
+        self.trackList = os.path.join(self.mySpecieFolderPath, "trackList.json")
+        self._createTrackList()
+        
+        self.myTrackStyle = TrackStyles(self.tool_directory, self.mySpecieFolderPath, self.trackList)
+        #self.cssFolderPath = os.path.join(self.mySpecieFolderPath, 'css')
+        #self.cssFilePath = os.path.join(self.cssFolderPath, 'custom_track_styles.css')
+        self.logger = logging.getLogger(__name__)
+
     
-    def createHub(self):
-        self.prepareRefseq()
-        for input_file in self.input_files:
-            self.addTrack(input_file)
-        self.indexName()
-        slink = self.makeArchive()
-        self.outHtml(slink)
+
+    def addTrack(self, trackDbObject):
+        if trackDbObject['dataType'].lower() == 'bam':
+            #new_track = subprocess.Popen(['echo', trackDbObject['options']], stdout=subprocess.PIPE)
+            #subprocess.call(['add-track-json.pl', json_file], stdin=new_track.stdout)
+            subtools.add_track_json(self.trackList, trackDbObject['options'])
+            #subtools.add_track_json(self.trackList, trackDbObject['track_json'])
+        elif trackDbObject['dataType'].lower() == 'bigwig':
+            subtools.add_track_json(self.trackList, trackDbObject['options'])
+        else: 
+            if trackDbObject['trackType'] == 'HTMLFeatures':
+                self._customizeHTMLFeature(trackDbObject)
+            subtools.flatfile_to_json(trackDbObject['trackDataURL'], trackDbObject['dataType'], trackDbObject['trackType'], trackDbObject['trackLabel'], self.mySpecieFolderPath, trackDbObject['options'])
+
+
+    def terminate(self, debug=False):
+        """ Write html file """
+        self._indexName()
+        if not debug:
+            self._removeRaw()
+        self._makeArchive()
         print "Success!\n"
-    
-    def prepareRefseq(self):
-        try:
+
+
+    def _customizeHTMLFeature(self, trackDbObject):
+        if trackDbObject['options']:
+            subfeatures = trackDbObject['options'].get('subfeatureClasses')
+            feature_color = trackDbObject['options']['feature_color']
+            if subfeatures:
+                for key, value in subfeatures.items():
+                    self.myTrackStyle.addCustomColor(value, feature_color)
+            else:
+                customizedFeature = santitizer.sanitize_name(trackDbObject['trackLabel'])
+                clientConfig = json.loads(trackDbObject['options']['clientConfig'])
+                clientConfig['renderClassName'] = customizedFeature
+                trackDbObject['options']['clientConfig'] = json.dumps(clientConfig)
+                self.myTrackStyle.addCustomColor(customizedFeature, feature_color)
+                  
+    def _removeRaw(self):
+        if os.path.exists(self.myTracksFolderPath):
+            shutil.rmtree(self.myTracksFolderPath)
+
+    def _createTrackList(self):
+        if not os.path.exists(self.trackList):
+            os.mknod(self.trackList)   
+
+    def _prepareRefseq(self):
+        subtools.prepare_refseqs(self.reference_genome.false_path, self.mySpecieFolderPath)
+        #try:
             #print os.path.join(self.tool_dir, 'prepare-refseqs.pl') + ", '--fasta', " + self.reference +", '--out', self.json])"
-            subprocess.call(['prepare-refseqs.pl', '--fasta', self.reference, '--out', self.json])
-        except OSError as e:
-            print "Cannot prepare reference error({0}): {1}".format(e.errno, e.strerror)
-    #TODO: hard coded the bam and bigwig tracks. Need to allow users to customize the settings
-    def addTrack(self, track):
-        #print "false_path" , track['false_path']
-        if track['false_path'] in self.metaData.keys():
-            metadata = self.metaData[track['false_path']]
-        else:
-            metadata = {}
-        self.SetMetadata(track, metadata)
-        if track['dataType'] == 'bam':
-            self.Bam(track, metadata)
-           # print "add bam track\n"
-        elif track['dataType'] == 'bigwig':
-            #create trackList.json if not exist
-            self.createTrackList()
-            json_file = os.path.join(self.json, "trackList.json")
-            bigwig_file = os.path.join(self.raw, track['fileName'])
-            subprocess.call(['add-bw-track.pl', '--label', metadata['label'], '--bw_url', bigwig_file, '--pos_color', metadata['style']['pos_color'], '--neg_color', metadata['style']['neg_color'], '--plot', 'JBrowse/View/Track/Wiggle/XYPlot', '--out', json_file, '--in', json_file])
-        else: 
-            flat_file = os.path.join(self.raw, track['fileName'])
-            if track['dataType'] == 'bed':
-                subprocess.call(['flatfile-to-json.pl', '--bed', flat_file, '--trackType', metadata['type'], '--trackLabel', metadata['label'], '--Config', '{"category" : "%s"}' % metadata['category'], '--clientConfig', '{"color" : "%s"}' % metadata['color'], '--out', self.json])
-            elif track['dataType'] == 'bedSpliceJunctions' or track['dataType'] == 'gtf' or track['dataType'] == 'blastxml':
-                subprocess.call(['flatfile-to-json.pl', '--gff', flat_file, '--trackType', metadata['type'], '--trackLabel', metadata['label'], '--Config', '{"glyph": "JBrowse/View/FeatureGlyph/Segments", "category" : "%s"}' % metadata['category'], '--clientConfig', '{"color" : "%s"}' % metadata['color'], '--out', self.json])
-            elif track['dataType'] == 'gff3_transcript':
-                subprocess.call(['flatfile-to-json.pl', '--gff', flat_file, '--trackType', metadata['type'], '--trackLabel', metadata['label'], '--Config', '{"transcriptType": "transcript", "category" : "%s"}' % metadata['category'], '--clientConfig', '{"color" : "%s"}' % metadata['color'], '--out', self.json])
-            else:
-                subprocess.call(['flatfile-to-json.pl', '--gff', flat_file, '--trackType', metadata['type'], '--trackLabel', metadata['label'], '--Config', '{"category" : "%s"}' % metadata['category'], '--clientConfig', '{"color" : "%s"}' % metadata['color'], '--out', self.json])
-            
-    def indexName(self):
-        subprocess.call(['generate-names.pl', '-v', '--out', self.json])
+            #subprocess.call(['prepare-refseqs.pl', '--fasta', self.reference_genome.false_path, '--out', self.mySpecieFolderPath])
+        #except OSError as e:
+            #print "Cannot prepare reference error({0}): {1}".format(e.errno, e.strerror)
+
+    def _indexName(self):
+        #subprocess.call(['generate-names.pl', '-v', '--out', self.mySpecieFolderPath])
+        subtools.generate_names(self.mySpecieFolderPath)
         print "finished name index \n"
 
-    def makeArchive(self):
-        file_dir = os.path.abspath(self.outfile)
-        source_dir = os.path.dirname(file_dir)
-        folder_name = os.path.basename(self.outfolder)
-        source_name = os.path.basename(self.out_path)
-        source = os.path.join(source_dir, folder_name, source_name)
-        slink = source.replace('/', '_')
-        slink = os.path.join('/var/www/html/JBrowse-1.12.1/data', slink)
-        try:
-            if os.path.islink(slink):
-                os.unlink(slink)
-        except OSError as oserror:
-            print "Cannot create symlink to the data({0}): {1}".format(oserror.errno, oserror.strerror)
-        os.symlink(source, slink)
-        return slink
-    
-    def outHtml(self, slink):
-        with open(self.outfile, 'w') as htmlfile:
-            htmlstr = 'The JBrowse Hub is created: <br>'
-            url = self.jbrowse_host + "/JBrowse-1.12.1/index.html?data=%s"
-            jbrowse_hub = '<li><a href = "%s" target="_blank">View JBrowse Hub</a></li>' % url
-            link_name = os.path.basename(slink)
-            relative_path = os.path.join('data', link_name + '/json')
-            htmlstr += jbrowse_hub % relative_path
-            htmlfile.write(htmlstr)
+    def _outHtml(self, host_name):
+        with open(self.outputFile, 'w') as htmlfile:
+            htmlstr = 'The new Organism "%s" is created on Apollo: <br>' % self.genome_name
+            jbrowse_hub = '<li><a href = "%s" target="_blank">View JBrowse Hub on Apollo</a></li>' % host_name
+            htmlstr += jbrowse_hub
+            htmlfile.write(htmlstr)     
+
+    def _makeArchive(self):
+        self.apollo.loadHubToApollo(self.apollo_user, self.genome_name, self.mySpecieFolderPath, admin=True)
+        apollo_host = self.apollo.getHost()
+        self._outHtml(apollo_host)
+        
+
+    def __createAssemblyHub__(self, extra_files_path):
+        # Get all necessaries infos first
+        # 2bit file creation from input fasta
+
+        # baseNameFasta = os.path.basename(fasta_file_name)
+        # suffixTwoBit, extensionTwoBit = os.path.splitext(baseNameFasta)
+        # nameTwoBit = suffixTwoBit + '.2bit'
+        twoBitFile = tempfile.NamedTemporaryFile(bufsize=0)
+        subtools.faToTwoBit(self.reference_genome.false_path, twoBitFile.name)
+
+        # Generate the twoBitInfo
+        twoBitInfoFile = tempfile.NamedTemporaryFile(bufsize=0)
+        subtools.twoBitInfo(twoBitFile.name, twoBitInfoFile.name)
+
+        # Then we get the output to generate the chromSizes
+        self.chromSizesFile = tempfile.NamedTemporaryFile(bufsize=0, suffix=".chrom.sizes")
+        subtools.sortChromSizes(twoBitInfoFile.name, self.chromSizesFile.name) 
 
-    def createTrackList(self):
-        trackList = os.path.join(self.json, "trackList.json")
-        if not os.path.exists(trackList):
-            os.mknod(trackList)
-    
-    def Bam(self, track, metadata):
-        #create trackList.json if not exist
-        self.createTrackList()
-        json_file = os.path.join(self.json, "trackList.json")
-        bam_track = dict()
-        bam_track['type'] = 'JBrowse/View/Track/Alignments2'
-        bam_track['storeClass'] = 'JBrowse/Store/SeqFeature/BAM'
-        bam_track['urlTemplate'] = os.path.join('../raw', track['fileName'])
-        bam_track['baiUrlTemplate'] = os.path.join('../raw', track['index'])
-        bam_track['label'] = metadata['label']
-        bam_track['category'] = metadata['category']
-        bam_track = json.dumps(bam_track)
-        #Use add-track-json.pl to add bam track to json file
-        new_track = subprocess.Popen(['echo', bam_track], stdout=subprocess.PIPE)
-        subprocess.call(['add-track-json.pl', json_file], stdin=new_track.stdout)
-    '''
-    def BigWig(self, track, metadata):
-        #create trackList.json if not exist
-        self.createTrackList()
-        json_file = os.path.join(self.json, "trackList.json")
-        bigwig_track = dict()
-        bigwig_track['urlTemplate'] = os.path.join('../raw', track['fileName'])
-        bigwig_track['type'] = 'JBrowse/View/Track/Wiggle/XYPlot'
-        bigwig_track['storeClass'] = 'JBrowse/Store/SeqFeature/BigWig'
-        bigwig_track['label'] = metadata['label']
-        bigwig_track['style'] = metadata['style']
-        bigwig_track['category'] = metadata['category']
-        bigwig_track = json.dumps(bigwig_track)
-        #Use add-track-json.pl to add bigwig track to json file
-        new_track = subprocess.Popen(['echo', bigwig_track], stdout=subprocess.PIPE)
-        #output = new_track.communicate()[0]
-        subprocess.call(['add-track-json.pl', json_file], stdin=new_track.stdout)
-    '''
-    def BigWig
-    #If the metadata is not set, use the default value
-    def SetMetadata(self, track, metadata):
-        if 'label' not in metadata.keys() or metadata['label'] == '':
-            metadata['label'] = track['fileName']
-        if 'color' not in metadata.keys() or metadata['color'] == '':
-            metadata['color'] = "#daa520"
-        if track['dataType'] == 'bigwig':
-            if 'style' not in metadata.keys():
-                metadata['style'] = {}
-            if 'pos_color' not in metadata['style'] or metadata['style']['pos_color'] == '':
-                metadata['style']['pos_color'] = "#FFA600"
-            if 'neg_color' not in metadata['style'] or metadata['style']['neg_color'] == '':
-                metadata['style']['neg_color'] = "#005EFF"
-        if 'category' not in metadata.keys() or metadata['category'] == '':
-            metadata['category'] = "Default group"
-        if track['dataType'] == 'blastxml':
-            metadata['type'] = "G-OnRamp_plugin/BlastAlignment"
-        elif track['dataType'] == 'bigpsl':
-            metadata['type'] = "G-OnRamp_plugin/BlatAlignment"
-        elif track['dataType'] == 'gff3_transcript' or track['dataType'] == 'gff3_mrna':
-            metadata['type'] = "G-OnRamp_plugin/GenePred"
-        else:
-            metadata['type'] = "CanvasFeatures"
+        # We can get the biggest scaffold here, with chromSizesFile
+        with open(self.chromSizesFile.name, 'r') as chrom_sizes:
+            # TODO: Check if exists
+            self.default_pos = chrom_sizes.readline().split()[0]
+
+        # TODO: Manage to put every fill Function in a file dedicated for reading reasons
+        # Create the root directory
+        myHubPath = os.path.join(extra_files_path, "myHub")
+        if not os.path.exists(myHubPath):
+            os.makedirs(myHubPath)
+
+        # Create the specie folder
+        # TODO: Generate the name depending on the specie
+        mySpecieFolderPath = os.path.join(myHubPath, self.genome_name)
+        if not os.path.exists(mySpecieFolderPath):
+            os.makedirs(mySpecieFolderPath)
+        self.mySpecieFolderPath = mySpecieFolderPath
 
-
+        # We create the 2bit file while we just created the specie folder
+        #self.twoBitName = self.genome_name + ".2bit"
+        #self.two_bit_final_path = os.path.join(self.mySpecieFolderPath, self.twoBitName)
+        #shutil.copyfile(twoBitFile.name, self.two_bit_final_path)
 
-   
+        # Create the folder tracks into the specie folder
+        tracksFolderPath = os.path.join(mySpecieFolderPath, "raw")
+        if not os.path.exists(tracksFolderPath):
+            os.makedirs(tracksFolderPath)
+        self.myTracksFolderPath = tracksFolderPath
 
+        myBinaryFolderPath = os.path.join(mySpecieFolderPath, 'bbi')
+        if not os.path.exists(myBinaryFolderPath):
+            os.makedirs(myBinaryFolderPath)
+        self.myBinaryFolderPath = myBinaryFolderPath
 
-
+        return myHubPath