# HG changeset patch # User iuc # Date 1394538129 14400 # Node ID c2645201ffaec4b0544fddec9dc0890489667771 # Parent e67da4f2c9bfbdfdf5b7d115622f3ecce13428cc Uploaded diff -r e67da4f2c9bf -r c2645201ffae gatk2_annotations.txt.sample --- a/gatk2_annotations.txt.sample Sat Jan 18 07:00:26 2014 -0500 +++ b/gatk2_annotations.txt.sample Tue Mar 11 07:42:09 2014 -0400 @@ -1,30 +1,45 @@ #unique_id name gatk_value tools_valid_for -AlleleBalance AlleleBalance AlleleBalance UnifiedGenotyper,VariantAnnotator,VariantRecalibrator -AlleleBalanceBySample AlleleBalanceBySample AlleleBalanceBySample UnifiedGenotyper,VariantAnnotator,VariantRecalibrator -BaseCounts BaseCounts BaseCounts UnifiedGenotyper,VariantAnnotator,VariantRecalibrator -BaseQualityRankSumTest BaseQualityRankSumTest BaseQualityRankSumTest UnifiedGenotyper,VariantAnnotator,VariantRecalibrator -ChromosomeCounts ChromosomeCounts ChromosomeCounts UnifiedGenotyper,VariantAnnotator,VariantRecalibrator -DepthOfCoverage DepthOfCoverage DepthOfCoverage UnifiedGenotyper,VariantAnnotator,VariantRecalibrator -DepthPerAlleleBySample DepthPerAlleleBySample DepthPerAlleleBySample UnifiedGenotyper,VariantAnnotator,VariantRecalibrator -FisherStrand FisherStrand FisherStrand UnifiedGenotyper,VariantAnnotator,VariantRecalibrator -GCContent GCContent GCContent UnifiedGenotyper,VariantAnnotator,VariantRecalibrator -HaplotypeScore HaplotypeScore HaplotypeScore UnifiedGenotyper,VariantAnnotator,VariantRecalibrator -HardyWeinberg HardyWeinberg HardyWeinberg UnifiedGenotyper,VariantAnnotator,VariantRecalibrator -HomopolymerRun HomopolymerRun HomopolymerRun UnifiedGenotyper,VariantAnnotator,VariantRecalibrator -InbreedingCoeff InbreedingCoeff InbreedingCoeff UnifiedGenotyper,VariantAnnotator,VariantRecalibrator +#http://gatkforums.broadinstitute.org/discussion/1268/how-should-i-interpret-vcf-files-produced-by-the-gatk +AlleleBalance AlleleBalance AB UnifiedGenotyper,VariantAnnotator,VariantRecalibrator,HaplotypeCaller +AlleleBalanceBySample AlleleBalanceBySample AlleleBalanceBySample UnifiedGenotyper,VariantAnnotator,VariantRecalibrator,HaplotypeCaller +BaseCounts BaseCounts BaseCounts UnifiedGenotyper,VariantAnnotator,VariantRecalibrator,HaplotypeCaller +BaseQualityRankSumTest BaseQualityRankSumTest BaseQualityRankSumTest UnifiedGenotyper,VariantAnnotator,VariantRecalibrator,HaplotypeCaller +ChromosomeCounts ChromosomeCounts ChromosomeCounts UnifiedGenotyper,VariantAnnotator,VariantRecalibrator,HaplotypeCaller +Coverage Coverage Coverage UnifiedGenotyper,VariantAnnotator,VariantRecalibrator,HaplotypeCaller +DepthPerAlleleBySample DepthPerAlleleBySample DepthPerAlleleBySample UnifiedGenotyper,VariantAnnotator,VariantRecalibrator,HaplotypeCaller +FisherStrand FisherStrand FisherStrand UnifiedGenotyper,VariantAnnotator,VariantRecalibrator,HaplotypeCaller +GCContent GCContent GCContent UnifiedGenotyper,VariantAnnotator,VariantRecalibrator,HaplotypeCaller +HaplotypeScore HaplotypeScore HaplotypeScore UnifiedGenotyper,VariantAnnotator,VariantRecalibrator,HaplotypeCaller +HardyWeinberg HardyWeinberg HardyWeinberg UnifiedGenotyper,VariantAnnotator,VariantRecalibrator,HaplotypeCaller +HomopolymerRun HomopolymerRun HomopolymerRun UnifiedGenotyper,VariantAnnotator,VariantRecalibrator,HaplotypeCaller +InbreedingCoeff InbreedingCoeff InbreedingCoeff UnifiedGenotyper,VariantAnnotator,VariantRecalibrator,HaplotypeCaller IndelType IndelType IndelType UnifiedGenotyper,VariantAnnotator,VariantRecalibrator -LowMQ LowMQ LowMQ UnifiedGenotyper,VariantAnnotator,VariantRecalibrator -MVLikelihoodRatio MVLikelihoodRatio MVLikelihoodRatio UnifiedGenotyper,VariantAnnotator,VariantRecalibrator -MappingQualityRankSumTest MappingQualityRankSumTest MappingQualityRankSumTest UnifiedGenotyper,VariantAnnotator,VariantRecalibrator -MappingQualityZero MappingQualityZero MappingQualityZero UnifiedGenotyper,VariantAnnotator,VariantRecalibrator -MappingQualityZeroBySample MappingQualityZeroBySample MappingQualityZeroBySample UnifiedGenotyper,VariantAnnotator,VariantRecalibrator -MappingQualityZeroFraction MappingQualityZeroFraction MappingQualityZeroFraction UnifiedGenotyper,VariantAnnotator,VariantRecalibrator -NBaseCount NBaseCount NBaseCount UnifiedGenotyper,VariantAnnotator,VariantRecalibrator -QualByDepth QualByDepth QualByDepth UnifiedGenotyper,VariantAnnotator,VariantRecalibrator -RMSMappingQuality RMSMappingQuality RMSMappingQuality UnifiedGenotyper,VariantAnnotator,VariantRecalibrator -ReadDepthAndAllelicFractionBySample ReadDepthAndAllelicFractionBySample ReadDepthAndAllelicFractionBySample UnifiedGenotyper,VariantAnnotator,VariantRecalibrator -ReadPosRankSumTest ReadPosRankSumTest ReadPosRankSumTest UnifiedGenotyper,VariantAnnotator,VariantRecalibrator -SampleList SampleList SampleList UnifiedGenotyper,VariantAnnotator,VariantRecalibrator -SnpEff SnpEff SnpEff VariantAnnotator,VariantRecalibrator -SpanningDeletions SpanningDeletions SpanningDeletions UnifiedGenotyper,VariantAnnotator,VariantRecalibrator -TechnologyComposition TechnologyComposition TechnologyComposition UnifiedGenotyper,VariantAnnotator,VariantRecalibrator +LowMQ LowMQ LowMQ UnifiedGenotyper,VariantAnnotator,VariantRecalibrator,HaplotypeCaller +MVLikelihoodRatio MVLikelihoodRatio MVLikelihoodRatio VariantAnnotator,VariantRecalibrator,HaplotypeCaller +MappingQualityRankSumTest MappingQualityRankSumTest MappingQualityRankSumTest UnifiedGenotyper,VariantAnnotator,VariantRecalibrator,HaplotypeCaller +MappingQualityZero MappingQualityZero MappingQualityZero UnifiedGenotyper,VariantAnnotator,VariantRecalibrator,HaplotypeCaller +MappingQualityZeroBySample MappingQualityZeroBySample MappingQualityZeroBySample UnifiedGenotyper,VariantAnnotator,VariantRecalibrator,HaplotypeCaller +NBaseCount NBaseCount NBaseCount UnifiedGenotyper,VariantAnnotator,VariantRecalibrator,HaplotypeCaller +QualByDepth QualByDepth QualByDepth UnifiedGenotyper,VariantAnnotator,VariantRecalibrator,HaplotypeCaller +RMSMappingQuality RMSMappingQuality RMSMappingQuality UnifiedGenotyper,VariantAnnotator,VariantRecalibrator,HaplotypeCaller +ReadPosRankSumTest ReadPosRankSumTest ReadPosRankSumTest UnifiedGenotyper,VariantAnnotator,VariantRecalibrator,HaplotypeCaller +SampleList SampleList SampleList UnifiedGenotyper,VariantAnnotator,VariantRecalibrator,HaplotypeCaller +SnpEff SnpEff SnpEff VariantAnnotator,VariantRecalibrator,HaplotypeCaller +SpanningDeletions SpanningDeletions SpanningDeletions UnifiedGenotyper,VariantAnnotator,VariantRecalibrator,HaplotypeCaller +VariantType VariantType VariantType UnifiedGenotyper,VariantAnnotator,VariantRecalibrator,HaplotypeCaller +AlleleCount Allele count in genotypes, for each ALT allele (AC) AC UnifiedGenotyper,VariantAnnotator,VariantRecalibrator,HaplotypeCaller +AlleleFrequency Allele Frequency, for each ALT allele (AF) AF UnifiedGenotyper,VariantAnnotator,VariantRecalibrator,HaplotypeCaller +AlleleNumber Total number of alleles in called genotypes (AN) AN UnifiedGenotyper,VariantAnnotator,VariantRecalibrator,HaplotypeCaller +Coverage Unfiltered depth over all samples (DP) DP UnifiedGenotyper,VariantAnnotator,VariantRecalibrator,HaplotypeCaller +Dels Dels Dels UnifiedGenotyper,VariantAnnotator,VariantRecalibrator +MQ RMS Mapping Quality MQ UnifiedGenotyper,VariantAnnotator,VariantRecalibrator,HaplotypeCaller +MQ0 Mapping Quality Zero MQ0 UnifiedGenotyper,VariantAnnotator,VariantRecalibrator,HaplotypeCaller +BaseQualityRankSumTest BaseQualityRankSumTest BaseQualityRankSumTest UnifiedGenotyper,VariantAnnotator,VariantRecalibrator,HaplotypeCaller +MappingQualityRankSumTest MappingQualityRankSumTest MappingQualityRankSumTest UnifiedGenotyper,VariantAnnotator,VariantRecalibrator,HaplotypeCaller +ReadPosRankSumTest ReadPosRankSumTest ReadPosRankSumTest UnifiedGenotyper,VariantAnnotator,VariantRecalibrator,HaplotypeCaller +HaplotypeScore HaplotypeScore HaplotypeScore UnifiedGenotyper,VariantAnnotator,VariantRecalibrator,HaplotypeCaller +QualByDepth QualByDepth QD UnifiedGenotyper,VariantAnnotator,VariantRecalibrator,HaplotypeCaller +VQSLOD Variant quality score recalibration VQSLOD UnifiedGenotyper,VariantAnnotator,VariantRecalibrator,HaplotypeCaller +FisherStrand FisherStrand FS UnifiedGenotyper,VariantAnnotator,VariantRecalibrator,HaplotypeCaller +StrandBias Strand Bias evidence (higher SB, more bias, more false positive calls) SB UnifiedGenotyper,VariantAnnotator,VariantRecalibrator,HaplotypeCaller + diff -r e67da4f2c9bf -r c2645201ffae gatk2_macros.xml --- a/gatk2_macros.xml Sat Jan 18 07:00:26 2014 -0500 +++ b/gatk2_macros.xml Tue Mar 11 07:42:09 2014 -0400 @@ -3,6 +3,7 @@ gatk2 samtools + picard GATK2_PATH GATK2_SITE_OPTIONS diff -r e67da4f2c9bf -r c2645201ffae gatk2_wrapper.py --- a/gatk2_wrapper.py Sat Jan 18 07:00:26 2014 -0500 +++ b/gatk2_wrapper.py Tue Mar 11 07:42:09 2014 -0400 @@ -7,7 +7,6 @@ import sys, optparse, os, tempfile, subprocess, shutil from binascii import unhexlify -from string import Template GALAXY_EXT_TO_GATK_EXT = { 'gatk_interval':'intervals', 'bam_index':'bam.bai', 'gatk_dbsnp':'dbSNP', 'picard_interval_list':'interval_list' } #items not listed here will use the galaxy extension as-is GALAXY_EXT_TO_GATK_FILE_TYPE = GALAXY_EXT_TO_GATK_EXT #for now, these are the same, but could be different if needed @@ -19,6 +18,7 @@ if tmp_dir and os.path.exists( tmp_dir ): shutil.rmtree( tmp_dir ) + def gatk_filename_from_galaxy( galaxy_filename, galaxy_ext, target_dir = None, prefix = None ): suffix = GALAXY_EXT_TO_GATK_EXT.get( galaxy_ext, galaxy_ext ) if prefix is None: @@ -29,36 +29,39 @@ os.symlink( galaxy_filename, gatk_filename ) return gatk_filename + def gatk_filetype_argument_substitution( argument, galaxy_ext ): return argument % dict( file_type = GALAXY_EXT_TO_GATK_FILE_TYPE.get( galaxy_ext, galaxy_ext ) ) + def open_file_from_option( filename, mode = 'rb' ): if filename: return open( filename, mode = mode ) return None + def html_report_from_directory( html_out, dir ): html_out.write( '\n\nGalaxy - GATK Output\n\n\n

\n

\n\n\n' ) -def index_bam_files( bam_filenames, tmp_dir ): + +def index_bam_files( bam_filenames ): for bam_filename in bam_filenames: bam_index_filename = "%s.bai" % bam_filename if not os.path.exists( bam_index_filename ): #need to index this bam file stderr_name = tempfile.NamedTemporaryFile( prefix = "bam_index_stderr" ).name command = 'samtools index %s %s' % ( bam_filename, bam_index_filename ) - proc = subprocess.Popen( args=command, shell=True, stderr=open( stderr_name, 'wb' ) ) - return_code = proc.wait() - if return_code: + try: + subprocess.check_call( args=command, shell=True, stderr=open( stderr_name, 'wb' ) ) + except: for line in open( stderr_name ): print >> sys.stderr, line - os.unlink( stderr_name ) #clean up - cleanup_before_exit( tmp_dir ) raise Exception( "Error indexing BAM file" ) - os.unlink( stderr_name ) #clean up + finally: + os.unlink( stderr_name ) def __main__(): #Parse Command Line @@ -74,8 +77,7 @@ parser.add_option( '-e', '--phone_home', dest='phone_home', action='store', type="string", default='STANDARD', help='What kind of GATK run report should we generate(NO_ET|STANDARD|STDOUT)' ) parser.add_option( '-K', '--gatk_key', dest='gatk_key', action='store', type="string", default=None, help='What kind of GATK run report should we generate(NO_ET|STANDARD|STDOUT)' ) (options, args) = parser.parse_args() - - tmp_dir = tempfile.mkdtemp( prefix='tmp-gatk-' ) + if options.pass_through_options: cmd = ' '.join( options.pass_through_options ) else: @@ -87,42 +89,50 @@ elif options.max_jvm_heap_fraction is not None: cmd = cmd.replace( 'java ', 'java -XX:DefaultMaxRAMFraction=%s -XX:+UseParallelGC ' % ( options.max_jvm_heap_fraction ), 1 ) bam_filenames = [] - if options.datasets: - for ( dataset_arg, filename, galaxy_ext, prefix ) in options.datasets: - gatk_filename = gatk_filename_from_galaxy( filename, galaxy_ext, target_dir = tmp_dir, prefix = prefix ) - if dataset_arg: - cmd = '%s %s "%s"' % ( cmd, gatk_filetype_argument_substitution( dataset_arg, galaxy_ext ), gatk_filename ) - if galaxy_ext == "bam": - bam_filenames.append( gatk_filename ) - index_bam_files( bam_filenames, tmp_dir ) - #set up stdout and stderr output options - stdout = open_file_from_option( options.stdout, mode = 'wb' ) - stderr = open_file_from_option( options.stderr, mode = 'wb' ) - #if no stderr file is specified, we'll use our own - if stderr is None: - stderr = tempfile.NamedTemporaryFile( prefix="gatk-stderr-", dir=tmp_dir ) - - proc = subprocess.Popen( args=cmd, stdout=stdout, stderr=stderr, shell=True, cwd=tmp_dir ) - return_code = proc.wait() - - if return_code: - stderr_target = sys.stderr - else: - stderr_target = sys.stdout - stderr.flush() - stderr.seek(0) - while True: - chunk = stderr.read( CHUNK_SIZE ) - if chunk: - stderr_target.write( chunk ) + tmp_dir = tempfile.mkdtemp( prefix='tmp-gatk-' ) + try: + if options.datasets: + for ( dataset_arg, filename, galaxy_ext, prefix ) in options.datasets: + gatk_filename = gatk_filename_from_galaxy( filename, galaxy_ext, target_dir = tmp_dir, prefix = prefix ) + if dataset_arg: + cmd = '%s %s "%s"' % ( cmd, gatk_filetype_argument_substitution( dataset_arg, galaxy_ext ), gatk_filename ) + if galaxy_ext == "bam": + bam_filenames.append( gatk_filename ) + if galaxy_ext == 'fasta': + subprocess.check_call( 'samtools faidx "%s"' % gatk_filename, shell=True ) + subprocess.check_call( 'java -jar %s R=%s O=%s QUIET=true' % ( os.path.join(os.environ['JAVA_JAR_PATH'], 'CreateSequenceDictionary.jar'), gatk_filename, os.path.splitext(gatk_filename)[0] + '.dict' ), shell=True ) + index_bam_files( bam_filenames ) + #set up stdout and stderr output options + stdout = open_file_from_option( options.stdout, mode = 'wb' ) + stderr = open_file_from_option( options.stderr, mode = 'wb' ) + #if no stderr file is specified, we'll use our own + if stderr is None: + stderr = tempfile.NamedTemporaryFile( prefix="gatk-stderr-", dir=tmp_dir ) + + proc = subprocess.Popen( args=cmd, stdout=stdout, stderr=stderr, shell=True, cwd=tmp_dir ) + return_code = proc.wait() + + if return_code: + stderr_target = sys.stderr else: - break - stderr.close() + stderr_target = sys.stdout + stderr.flush() + stderr.seek(0) + while True: + chunk = stderr.read( CHUNK_SIZE ) + if chunk: + stderr_target.write( chunk ) + else: + break + stderr.close() + finally: + cleanup_before_exit( tmp_dir ) + #generate html reports if options.html_report_from_directory: for ( html_filename, html_dir ) in options.html_report_from_directory: html_report_from_directory( open( html_filename, 'wb' ), html_dir ) - - cleanup_before_exit( tmp_dir ) + -if __name__=="__main__": __main__() +if __name__ == "__main__": + __main__() diff -r e67da4f2c9bf -r c2645201ffae haplotype_caller.xml --- a/haplotype_caller.xml Sat Jan 18 07:00:26 2014 -0500 +++ b/haplotype_caller.xml Tue Mar 11 07:42:09 2014 -0400 @@ -158,7 +158,7 @@ - + @@ -191,7 +191,7 @@ - + diff -r e67da4f2c9bf -r c2645201ffae tool_dependencies.xml --- a/tool_dependencies.xml Sat Jan 18 07:00:26 2014 -0500 +++ b/tool_dependencies.xml Tue Mar 11 07:42:09 2014 -0400 @@ -15,6 +15,38 @@ - + + + + + + + + + + + + + + + https://github.com/bgruening/download_store/raw/master/gatk2_R_deps/colorspace_1.2-4.tar.gz + https://github.com/bgruening/download_store/raw/master/gatk2_R_deps/stringr_0.6.2.tar.gz + https://github.com/bgruening/download_store/raw/master/gatk2_R_deps/RColorBrewer_1.0-5.tar.gz + https://github.com/bgruening/download_store/raw/master/gatk2_R_deps/dichromat_2.0-0.tar.gz + https://github.com/bgruening/download_store/raw/master/gatk2_R_deps/munsell_0.4.2.tar.gz + https://github.com/bgruening/download_store/raw/master/gatk2_R_deps/labeling_0.2.tar.gz + https://github.com/bgruening/download_store/raw/master/gatk2_R_deps/plyr_1.8.1.tar.gz + https://github.com/bgruening/download_store/raw/master/gatk2_R_deps/digest_0.6.4.tar.gz + https://github.com/bgruening/download_store/raw/master/gatk2_R_deps/gtable_0.1.2.tar.gz + https://github.com/bgruening/download_store/raw/master/gatk2_R_deps/reshape2_1.2.2.tar.gz + https://github.com/bgruening/download_store/raw/master/gatk2_R_deps/scales_0.2.3.tar.gz + https://github.com/bgruening/download_store/raw/master/gatk2_R_deps/proto_0.3-10.tar.gz + https://github.com/bgruening/download_store/raw/master/gatk2_R_deps/ggplot2_0.9.3.1.tar.gz + + + + + R depencies for GATK2. + diff -r e67da4f2c9bf -r c2645201ffae unified_genotyper.xml --- a/unified_genotyper.xml Sat Jan 18 07:00:26 2014 -0500 +++ b/unified_genotyper.xml Tue Mar 11 07:42:09 2014 -0400 @@ -22,7 +22,7 @@ \$GATK2_SITE_OPTIONS ## according to http://www.broadinstitute.org/gatk/guide/article?id=1975 - --num_cpu_threads_per_data_thread 6 + --num_cpu_threads_per_data_thread 1 #if $reference_source.reference_source_selector != "history": -R "${reference_source.ref_file.fields.path}" diff -r e67da4f2c9bf -r c2645201ffae variant_recalibrator.xml --- a/variant_recalibrator.xml Sat Jan 18 07:00:26 2014 -0500 +++ b/variant_recalibrator.xml Tue Mar 11 07:42:09 2014 -0400 @@ -63,15 +63,12 @@ --maxIterations "${analysis_param_type.max_iterations}" --numKMeans "${analysis_param_type.num_k_means}" --stdThreshold "${analysis_param_type.std_threshold}" - --qualThreshold "${analysis_param_type.qual_threshold}" --shrinkage "${analysis_param_type.shrinkage}" --dirichlet "${analysis_param_type.dirichlet}" --priorCounts "${analysis_param_type.prior_counts}" - #if str( $analysis_param_type.bad_variant_selector.bad_variant_selector_type ) == 'percent': - --percentBadVariants "${analysis_param_type.bad_variant_selector.percent_bad_variants}" - #else: - --minNumBadVariants "${analysis_param_type.bad_variant_selector.min_num_bad_variants}" - #end if + + --minNumBadVariants "${analysis_param_type.min_num_bad_variants}" + --target_titv "${analysis_param_type.target_titv}" #for $tranche in [ $tranche.strip() for $tranche in str( $analysis_param_type.ts_tranche ).split( ',' ) if $tranche.strip() ] --TStranche "${tranche}" @@ -83,7 +80,6 @@ #end if --ignore_filter "${ignore_filter_name}" #end for - --ts_filter_level "${analysis_param_type.ts_filter_level}" ' #end if @@ -100,7 +96,7 @@ - + @@ -114,7 +110,7 @@ - + @@ -324,26 +320,17 @@ - - - - - + + + + - - - - - - - - - - - - + + + @@ -360,7 +347,6 @@ - @@ -410,7 +396,6 @@ maxIterations The maximum number of VBEM iterations to be performed in variational Bayes algorithm. Procedure will normally end when convergence is detected. numKMeans The number of k-means iterations to perform in order to initialize the means of the Gaussians in the Gaussian mixture model. stdThreshold If a variant has annotations more than -std standard deviations away from mean then don't use it for building the Gaussian mixture model. - qualThreshold If a known variant has raw QUAL value less than -qual then don't use it for building the Gaussian mixture model. shrinkage The shrinkage parameter in variational Bayes algorithm. dirichlet The dirichlet parameter in variational Bayes algorithm. priorCounts The number of prior counts to use in variational Bayes algorithm. @@ -423,7 +408,6 @@ path_to_Rscript The path to your implementation of Rscript. For Broad users this is maybe /broad/tools/apps/R-2.6.0/bin/Rscript rscript_file The output rscript file generated by the VQSR to aid in visualization of the input data and learned model path_to_resources Path to resources folder holding the Sting R scripts. - ts_filter_level The truth sensitivity level at which to start filtering, used here to indicate filtered variants in plots @CITATION_SECTION@