|
0
|
1 #!/usr/bin/perl -w
|
|
|
2 #
|
|
|
3 # $RCSfile: AnalyzeSDFilesData.pl,v $
|
|
|
4 # $Date: 2015/02/28 20:46:04 $
|
|
|
5 # $Revision: 1.27 $
|
|
|
6 #
|
|
|
7 # Author: Manish Sud <msud@san.rr.com>
|
|
|
8 #
|
|
|
9 # Copyright (C) 2015 Manish Sud. All rights reserved.
|
|
|
10 #
|
|
|
11 # This file is part of MayaChemTools.
|
|
|
12 #
|
|
|
13 # MayaChemTools is free software; you can redistribute it and/or modify it under
|
|
|
14 # the terms of the GNU Lesser General Public License as published by the Free
|
|
|
15 # Software Foundation; either version 3 of the License, or (at your option) any
|
|
|
16 # later version.
|
|
|
17 #
|
|
|
18 # MayaChemTools is distributed in the hope that it will be useful, but without
|
|
|
19 # any warranty; without even the implied warranty of merchantability of fitness
|
|
|
20 # for a particular purpose. See the GNU Lesser General Public License for more
|
|
|
21 # details.
|
|
|
22 #
|
|
|
23 # You should have received a copy of the GNU Lesser General Public License
|
|
|
24 # along with MayaChemTools; if not, see <http://www.gnu.org/licenses/> or
|
|
|
25 # write to the Free Software Foundation Inc., 59 Temple Place, Suite 330,
|
|
|
26 # Boston, MA, 02111-1307, USA.
|
|
|
27 #
|
|
|
28
|
|
|
29 use strict;
|
|
|
30 use FindBin; use lib "$FindBin::Bin/../lib";
|
|
|
31 use Getopt::Long;
|
|
|
32 use File::Basename;
|
|
|
33 use Text::ParseWords;
|
|
|
34 use Benchmark;
|
|
|
35 use FileUtil;
|
|
|
36 use SDFileUtil;
|
|
|
37 use TextUtil;
|
|
|
38 use StatisticsUtil;
|
|
|
39
|
|
|
40 my($ScriptName, %Options, $StartTime, $EndTime, $TotalTime);
|
|
|
41
|
|
|
42 # Autoflush STDOUT
|
|
|
43 $| = 1;
|
|
|
44
|
|
|
45 # Starting message...
|
|
|
46 $ScriptName = basename($0);
|
|
|
47 print "\n$ScriptName: Starting...\n\n";
|
|
|
48 $StartTime = new Benchmark;
|
|
|
49
|
|
|
50 # Get the options and setup script...
|
|
|
51 SetupScriptUsage();
|
|
|
52 if ($Options{help} || @ARGV < 1) {
|
|
|
53 die GetUsageFromPod("$FindBin::Bin/$ScriptName");
|
|
|
54 }
|
|
|
55
|
|
|
56 my(@SDFilesList);
|
|
|
57 @SDFilesList = ExpandFileNames(\@ARGV, "sd sdf");
|
|
|
58
|
|
|
59 print "Processing options...\n";
|
|
|
60 my(%OptionsInfo);
|
|
|
61 ProcessOptions();
|
|
|
62
|
|
|
63 # Collect information about SD files...
|
|
|
64 print "Checking input SD file(s)...\n";
|
|
|
65 my(%SDFilesInfo);
|
|
|
66 RetrieveSDFilesInfo();
|
|
|
67 ProcessSDFilesDataLabelsInfo();
|
|
|
68
|
|
|
69 # Generate output files...
|
|
|
70 my($FileIndex);
|
|
|
71 if (@SDFilesList > 1) {
|
|
|
72 print "\nProcessing SD files...\n";
|
|
|
73 }
|
|
|
74 for $FileIndex (0 .. $#SDFilesList) {
|
|
|
75 if ($SDFilesInfo{FileOkay}[$FileIndex]) {
|
|
|
76 print "\nProcessing file $SDFilesList[$FileIndex]...\n";
|
|
|
77 AnalyzeSDFile($FileIndex);
|
|
|
78 }
|
|
|
79 }
|
|
|
80 print "\n$ScriptName:Done...\n\n";
|
|
|
81
|
|
|
82 $EndTime = new Benchmark;
|
|
|
83 $TotalTime = timediff ($EndTime, $StartTime);
|
|
|
84 print "Total time: ", timestr($TotalTime), "\n";
|
|
|
85
|
|
|
86 ###############################################################################
|
|
|
87
|
|
|
88 # Analyze data...
|
|
|
89 sub AnalyzeSDFile {
|
|
|
90 my($Index) = @_;
|
|
|
91 my($SDFile, $DataLabel, $DataValue, @DataLabelsToAnalyze, %DataFieldValuesToAnalyzeMap);
|
|
|
92
|
|
|
93 $SDFile = $SDFilesList[$Index];
|
|
|
94 @DataLabelsToAnalyze = @{$SDFilesInfo{UniqueDataLabelsToAnalyze}[$Index]};
|
|
|
95 %DataFieldValuesToAnalyzeMap = ();
|
|
|
96 for $DataLabel (@DataLabelsToAnalyze) {
|
|
|
97 @{$DataFieldValuesToAnalyzeMap{$DataLabel}} = ();
|
|
|
98 }
|
|
|
99
|
|
|
100 # Collect appropriate data field label values for analysis...
|
|
|
101 my($CmpdString, @CmpdLines, %DataFieldValues, $CmpdCount, $InvalidCmpdCount, @InvalidCmpdDataLabels);
|
|
|
102 open SDFILE, "$SDFile" or die "Error: Can't open $SDFile: $! \n";
|
|
|
103 $CmpdCount = 0;
|
|
|
104 $InvalidCmpdCount = 0;
|
|
|
105 while ($CmpdString = ReadCmpdString(\*SDFILE)) {
|
|
|
106 $CmpdCount++;
|
|
|
107 @CmpdLines = split "\n", $CmpdString;
|
|
|
108 %DataFieldValues = GetCmpdDataHeaderLabelsAndValues(\@CmpdLines);
|
|
|
109 @InvalidCmpdDataLabels = ();
|
|
|
110 DATALABEL: for $DataLabel (@DataLabelsToAnalyze) {
|
|
|
111 if (exists $DataFieldValues{$DataLabel}) {
|
|
|
112 $DataValue = $DataFieldValues{$DataLabel};
|
|
|
113 if ($OptionsInfo{CheckData}) {
|
|
|
114 if (!IsNumerical($DataValue)) {
|
|
|
115 push @InvalidCmpdDataLabels, $DataLabel;
|
|
|
116 next DATALABEL;
|
|
|
117 }
|
|
|
118 }
|
|
|
119 push @{$DataFieldValuesToAnalyzeMap{$DataLabel}}, $DataValue;
|
|
|
120 }
|
|
|
121 }
|
|
|
122 if (@InvalidCmpdDataLabels) {
|
|
|
123 $InvalidCmpdCount++;
|
|
|
124 if ($OptionsInfo{DetailLevel} >=4 ) {
|
|
|
125 print "Compound record $CmpdCount contains ", scalar(@InvalidCmpdDataLabels)," non-numerical or empty value(s) for data field(s) - ", JoinWords(\@InvalidCmpdDataLabels, ", ", 0)," - to be analyzed:\n$CmpdString \n";
|
|
|
126 }
|
|
|
127 elsif ($OptionsInfo{DetailLevel} >= 3) {
|
|
|
128 print "Compound record $CmpdCount contains ", scalar(@InvalidCmpdDataLabels)," non-numerical or empty value(s) for data field(s) - ", JoinWords(\@InvalidCmpdDataLabels, ", ", 0)," - to be analyzed...\n";
|
|
|
129 }
|
|
|
130 elsif ($OptionsInfo{DetailLevel} >= 2) {
|
|
|
131 print "Compound record $CmpdCount contains ", scalar(@InvalidCmpdDataLabels)," non-numerical or empty value(s) for data field to be analyzed...\n";
|
|
|
132 }
|
|
|
133 }
|
|
|
134 }
|
|
|
135 if ($InvalidCmpdCount && ($OptionsInfo{DetailLevel} >= 1)) {
|
|
|
136 print "Non-numerical or empty data present in $InvalidCmpdCount compound record(s)...\n";
|
|
|
137 }
|
|
|
138 close SDFILE;
|
|
|
139
|
|
|
140 # Perform the analysis...
|
|
|
141 my(@SpecifiedFunctionNames, $SpecifiedFunction);
|
|
|
142 @SpecifiedFunctionNames = ();
|
|
|
143
|
|
|
144 for $SpecifiedFunction (@{$OptionsInfo{SpecifiedStatisticalFunctions}}) {
|
|
|
145 if ($SpecifiedFunction !~ /^(Covariance|Correlation|Frequency|Rsquare|StandardScores|StandardScoresN)$/i) {
|
|
|
146 push @SpecifiedFunctionNames, $OptionsInfo{SpecifiedStatisticalFunctionsMap}{lc($SpecifiedFunction)};
|
|
|
147 }
|
|
|
148 }
|
|
|
149 if (@SpecifiedFunctionNames) {
|
|
|
150 PerformAnalysis($Index, \@SpecifiedFunctionNames, \%DataFieldValuesToAnalyzeMap)
|
|
|
151 }
|
|
|
152 if (exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{covariance}) || exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{correlation}) || exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{rsquare})) {
|
|
|
153 if ($OptionsInfo{AllDataLabelPairs} || $OptionsInfo{CommonDataLabelPairs}) {
|
|
|
154 PerformMatrixAnalysis($Index, \%DataFieldValuesToAnalyzeMap);
|
|
|
155 }
|
|
|
156 else {
|
|
|
157 # Perform pairwise analysis for specified columns and write out calculated values - correlation
|
|
|
158 # rsquare, or covariance - in the same file.
|
|
|
159 PerformDataLabelPairAnalysis($Index, \%DataFieldValuesToAnalyzeMap);
|
|
|
160 }
|
|
|
161 }
|
|
|
162 if (exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{standardscores}) || exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{standardscoresn}) ) {
|
|
|
163 PerformStandardScoresAnalysis($Index, \%DataFieldValuesToAnalyzeMap);
|
|
|
164 }
|
|
|
165 if (exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{frequency})) {
|
|
|
166 PerformFrequencyAnalysis($Index, \%DataFieldValuesToAnalyzeMap);
|
|
|
167 }
|
|
|
168
|
|
|
169 }
|
|
|
170
|
|
|
171 # Calculate values for various statistical functions...
|
|
|
172 sub PerformAnalysis {
|
|
|
173 my($Index, $SpecifiedFunctionNamesRef, $DataValuesToAnalyzeMapRef) = @_;
|
|
|
174 my($NewTextFile, $Line, $SpecifiedFunction, $Label, @ColLabels, @DataLabelsToAnalyze);
|
|
|
175
|
|
|
176 $NewTextFile = $SDFilesInfo{NewTextFileRoot}[$Index] . $OptionsInfo{FileNameMode} . "." . $SDFilesInfo{NewTextFileExt}[$Index];
|
|
|
177
|
|
|
178 print "Generating new text file $NewTextFile...\n";
|
|
|
179 open NEWTEXTFILE, ">$NewTextFile" or die "Error: Can't open $NewTextFile: $! \n";
|
|
|
180
|
|
|
181 # Write out column labels...
|
|
|
182 @ColLabels = ();
|
|
|
183 push @ColLabels, "DataLabel";
|
|
|
184 for $SpecifiedFunction (@{$SpecifiedFunctionNamesRef}) {
|
|
|
185 $Label = $SpecifiedFunction;
|
|
|
186 if ($SpecifiedFunction =~ /^(KLargest|KSmallest)$/i) {
|
|
|
187 my($KthValue);
|
|
|
188 $KthValue = ($SpecifiedFunction =~ /^KLargest$/i) ? $OptionsInfo{KLargest} : $OptionsInfo{KSmallest};
|
|
|
189 $Label = AddNumberSuffix($KthValue) . "$SpecifiedFunction";
|
|
|
190 $Label =~ s/K//g;
|
|
|
191 }
|
|
|
192 elsif ($SpecifiedFunction =~ /^TrimMean$/i) {
|
|
|
193 $Label = "${SpecifiedFunction}($OptionsInfo{TrimFraction})";
|
|
|
194 }
|
|
|
195 push @ColLabels, $Label;
|
|
|
196 }
|
|
|
197 $Line = JoinWords(\@ColLabels, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
|
|
|
198 print NEWTEXTFILE "$Line\n";
|
|
|
199
|
|
|
200 # Go over each column to be analyzed...
|
|
|
201 @DataLabelsToAnalyze = @{$SDFilesInfo{DataLabelsToAnalyze}[$Index]};
|
|
|
202
|
|
|
203 # Turn off "strict"; otherwise, invoking statistical functions using function name string
|
|
|
204 # is problematic.
|
|
|
205 no strict;
|
|
|
206
|
|
|
207 my($DataValuesRef, $DataLabel, $Value, @RowValues, %CalculatedValues);
|
|
|
208 %CalculatedValues = ();
|
|
|
209 for $DataLabel (@DataLabelsToAnalyze) {
|
|
|
210 @RowValues = ();
|
|
|
211 # Setup column id...
|
|
|
212 push @RowValues, $DataLabel;
|
|
|
213 $DataValuesRef = \@{$DataValuesToAnalyzeMapRef->{$DataLabel}};
|
|
|
214 FUNCTIONNAME: for $SpecifiedFunction (@{$SpecifiedFunctionNamesRef}) {
|
|
|
215 $Value = "";
|
|
|
216 if (!@{$DataValuesToAnalyzeMapRef->{$DataLabel}}) {
|
|
|
217 # Invalid column values...
|
|
|
218 push @RowValues, $Value;
|
|
|
219 next FUNCTIONNAME;
|
|
|
220 }
|
|
|
221 if ($SpecifiedFunction =~ /^Count$/i) {
|
|
|
222 $Value = @{$DataValuesToAnalyzeMapRef->{$DataLabel}};
|
|
|
223 }
|
|
|
224 elsif ($SpecifiedFunction =~ /^KLargest$/i) {
|
|
|
225 $Value = &$SpecifiedFunction($DataValuesRef, $OptionsInfo{KLargest});
|
|
|
226 }
|
|
|
227 elsif ($SpecifiedFunction =~ /^KSmallest$/i) {
|
|
|
228 $Value = &$SpecifiedFunction($DataValuesRef, $OptionsInfo{KSmallest});
|
|
|
229 }
|
|
|
230 elsif ($SpecifiedFunction =~ /^StandardDeviation$/i) {
|
|
|
231 if (exists($CalculatedValues{$DataLabel}{StandardDeviation})) {
|
|
|
232 $Value = $CalculatedValues{$DataLabel}{StandardDeviation};
|
|
|
233 }
|
|
|
234 else {
|
|
|
235 $Value = &$SpecifiedFunction($DataValuesRef);
|
|
|
236 $CalculatedValues{$DataLabel}{StandardDeviation} = $Value;
|
|
|
237 }
|
|
|
238 }
|
|
|
239 elsif ($SpecifiedFunction =~ /^StandardError$/i) {
|
|
|
240 if (!exists($CalculatedValues{$DataLabel}{StandardDeviation})) {
|
|
|
241 $Value = StandardDeviation($DataValuesRef);
|
|
|
242 $CalculatedValues{$DataLabel}{StandardDeviation} = $Value;
|
|
|
243 }
|
|
|
244 if (defined $CalculatedValues{$DataLabel}{StandardDeviation}) {
|
|
|
245 $Value = &$SpecifiedFunction($CalculatedValues{$DataLabel}{StandardDeviation}, @{$DataValuesToAnalyzeMapRef->{$DataLabel}});
|
|
|
246 }
|
|
|
247 }
|
|
|
248 elsif ($SpecifiedFunction =~ /^TrimMean$/i) {
|
|
|
249 $Value = &$SpecifiedFunction($DataValuesRef, $OptionsInfo{TrimFraction});
|
|
|
250 }
|
|
|
251 else {
|
|
|
252 $Value = &$SpecifiedFunction($DataValuesRef);
|
|
|
253 }
|
|
|
254 # Format the output value. And add zero to get rid of tariling zeros...
|
|
|
255 $Value = (defined($Value) && length($Value)) ? (sprintf("%.$OptionsInfo{Precision}f", $Value) + 0) : "";
|
|
|
256 push @RowValues, $Value;
|
|
|
257 }
|
|
|
258 $Line = JoinWords(\@RowValues, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
|
|
|
259 print NEWTEXTFILE "$Line\n";
|
|
|
260 }
|
|
|
261 close NEWTEXTFILE;
|
|
|
262 }
|
|
|
263
|
|
|
264 # Calculate covariance, correlation, rsquare for specified data field label pairs....
|
|
|
265 sub PerformDataLabelPairAnalysis {
|
|
|
266 my($Index, $DataValuesToAnalyzeMapRef) = @_;
|
|
|
267 my($NewTextFile, @ColLabels, $Line, $CalculateCorrelation, $CalculateRSquare, $CalculateCovariance);
|
|
|
268
|
|
|
269 $CalculateCorrelation = exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{correlation}) ? 1 : 0;
|
|
|
270 $CalculateRSquare = exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{rsquare}) ? 1 : 0;
|
|
|
271 $CalculateCovariance = exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{covariance}) ? 1 : 0;
|
|
|
272
|
|
|
273 $NewTextFile = $SDFilesInfo{NewTextFileRoot}[$Index] . "DataFieldPairsAnalysis." . $SDFilesInfo{NewTextFileExt}[$Index];
|
|
|
274 print "Generating new text file $NewTextFile...\n";
|
|
|
275 open NEWTEXTFILE, ">$NewTextFile" or die "Error: Can't open $NewTextFile: $! \n";
|
|
|
276
|
|
|
277 # Write out the column labels...
|
|
|
278 @ColLabels = ();
|
|
|
279 push @ColLabels, ("DataLabel1", "DataLabel2");
|
|
|
280 if ($CalculateCorrelation || $CalculateRSquare) {
|
|
|
281 push @ColLabels, "Correlation";
|
|
|
282 if ($CalculateRSquare) {
|
|
|
283 push @ColLabels, "RSquare";
|
|
|
284 }
|
|
|
285 }
|
|
|
286 if ($CalculateCovariance) {
|
|
|
287 push @ColLabels, "Covariance";
|
|
|
288 }
|
|
|
289 $Line = JoinWords(\@ColLabels, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
|
|
|
290 print NEWTEXTFILE "$Line\n";
|
|
|
291
|
|
|
292 # Go over each data field pair...
|
|
|
293 my($CorrelationValue, $RSquareValue, $CovarianceValue, $LabelIndex, $DataLabel1, $DataLabel2, $DataValues1, $DataValues2, @DataLabelPairs1ToAnalyze, @DataLabelPairs2ToAnalyze, @RowValues, $Value);
|
|
|
294
|
|
|
295 @DataLabelPairs1ToAnalyze = @{$SDFilesInfo{DataLabelPairs1ToAnalyze}[$Index]};
|
|
|
296 @DataLabelPairs2ToAnalyze = @{$SDFilesInfo{DataLabelPairs2ToAnalyze}[$Index]};
|
|
|
297 for $LabelIndex (0 .. $#DataLabelPairs1ToAnalyze) {
|
|
|
298 @RowValues = ();
|
|
|
299 $DataLabel1 = $DataLabelPairs1ToAnalyze[$LabelIndex];
|
|
|
300 $DataLabel2 = $DataLabelPairs2ToAnalyze[$LabelIndex];
|
|
|
301 $DataValues1 = \@{$DataValuesToAnalyzeMapRef->{$DataLabel1}};
|
|
|
302 $DataValues2 = \@{$DataValuesToAnalyzeMapRef->{$DataLabel2}};
|
|
|
303
|
|
|
304 # Setup column ids...
|
|
|
305 push @RowValues, $DataLabel1;
|
|
|
306 push @RowValues, $DataLabel2;
|
|
|
307
|
|
|
308 if (@$DataValues1 != @$DataValues2) {
|
|
|
309 # Print a warning...
|
|
|
310 warn "Warning: Skipping analysis for data field pair $DataLabel1, $DataLabel2: Number of valid data values must be same.\n";
|
|
|
311 if ($CalculateCorrelation || $CalculateRSquare) {
|
|
|
312 push @RowValues, "";
|
|
|
313 if ($CalculateRSquare) {
|
|
|
314 push @RowValues, "";
|
|
|
315 }
|
|
|
316 }
|
|
|
317 if ($CalculateCovariance) {
|
|
|
318 push @RowValues, "";
|
|
|
319 }
|
|
|
320 }
|
|
|
321 else {
|
|
|
322 # Calculate appropriate value...
|
|
|
323 if ($CalculateCorrelation || $CalculateRSquare) {
|
|
|
324 $CorrelationValue = Correlation($DataValues1, $DataValues2);
|
|
|
325 $Value = (defined($CorrelationValue) && length($CorrelationValue)) ? (sprintf("%.$OptionsInfo{Precision}f", $CorrelationValue) + 0) : "";
|
|
|
326 push @RowValues, $Value;
|
|
|
327 if ($CalculateRSquare) {
|
|
|
328 $RSquareValue = (defined($CorrelationValue) && length($CorrelationValue)) ? ($CorrelationValue ** 2) : "";
|
|
|
329 $Value = (length($RSquareValue)) ? (sprintf("%.$OptionsInfo{Precision}f", $RSquareValue) + 0) : "";
|
|
|
330 push @RowValues, $Value;
|
|
|
331 }
|
|
|
332 }
|
|
|
333 if ($CalculateCovariance) {
|
|
|
334 $CovarianceValue = Covariance($DataValues1, $DataValues2);
|
|
|
335 $Value = (defined($CovarianceValue) && length($CovarianceValue)) ? (sprintf("%.$OptionsInfo{Precision}f", $CovarianceValue) + 0) : "";
|
|
|
336 push @RowValues, $Value;
|
|
|
337 }
|
|
|
338 }
|
|
|
339 $Line = JoinWords(\@RowValues, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
|
|
|
340 print NEWTEXTFILE "$Line\n";
|
|
|
341 }
|
|
|
342 close NEWTEXTFILE;
|
|
|
343 }
|
|
|
344
|
|
|
345 # Generate histogram numbers...
|
|
|
346 sub PerformFrequencyAnalysis {
|
|
|
347 my($Index, $DataValuesToAnalyzeMapRef) = @_;
|
|
|
348 my($NewTextFile, $ColLabel, @ColLabels, @RowValues, $Line, $DataLabel, @DataLabelsToAnalyze, $DataValuesRef, $BinValue, $FrequencyValue, $Value, %FrequencyMap);
|
|
|
349
|
|
|
350 @DataLabelsToAnalyze = @{$SDFilesInfo{DataLabelsToAnalyze}[$Index]};
|
|
|
351 for $DataLabel (@DataLabelsToAnalyze) {
|
|
|
352 $NewTextFile = $SDFilesInfo{NewTextFileRoot}[$Index] . $DataLabel . "FrequencyAnalysis." . $SDFilesInfo{NewTextFileExt}[$Index];
|
|
|
353 print "Generating new text file $NewTextFile...\n";
|
|
|
354 open NEWTEXTFILE, ">$NewTextFile" or die "Error: Can't open $NewTextFile: $! \n";
|
|
|
355
|
|
|
356 # Write out the column labels...
|
|
|
357 @ColLabels = ();
|
|
|
358 push @ColLabels , ("Bins", "Frequency");
|
|
|
359 $Line = JoinWords(\@ColLabels, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
|
|
|
360 print NEWTEXTFILE "$Line\n";
|
|
|
361
|
|
|
362 #Calculate and write out frequency values...
|
|
|
363 %FrequencyMap = ();
|
|
|
364 $DataValuesRef = \@{$DataValuesToAnalyzeMapRef->{$DataLabel}};
|
|
|
365 if (@$DataValuesRef) {
|
|
|
366 if (@{$OptionsInfo{BinRange}}) {
|
|
|
367 %FrequencyMap = Frequency($DataValuesRef, \@{$OptionsInfo{BinRange}});
|
|
|
368 }
|
|
|
369 else {
|
|
|
370 %FrequencyMap = Frequency($DataValuesRef, $OptionsInfo{NumOfBins});
|
|
|
371 }
|
|
|
372 }
|
|
|
373 for $BinValue (sort { $a <=> $b } keys %FrequencyMap) {
|
|
|
374 $FrequencyValue = $FrequencyMap{$BinValue};
|
|
|
375
|
|
|
376 @RowValues = ();
|
|
|
377 $Value = (length($BinValue)) ? (sprintf("%.$OptionsInfo{Precision}f", $BinValue) + 0) : "";
|
|
|
378 push @RowValues, $Value;
|
|
|
379 $Value = (length($FrequencyValue)) ? (sprintf("%.$OptionsInfo{Precision}f", $FrequencyValue) + 0) : "";
|
|
|
380 push @RowValues, $Value;
|
|
|
381
|
|
|
382 $Line = JoinWords(\@RowValues, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
|
|
|
383 print NEWTEXTFILE "$Line\n";
|
|
|
384 }
|
|
|
385 close NEWTEXTFILE;
|
|
|
386 }
|
|
|
387 }
|
|
|
388
|
|
|
389 # Calculate covariance, correlation/rsquare matrices....
|
|
|
390 sub PerformMatrixAnalysis {
|
|
|
391 my($Index, $DataValuesToAnalyzeMapRef) = @_;
|
|
|
392 my($CorrelationTextFile, $CovarianceTextFile, $RSquareTextFile, $CalculateCorrelation, $CalculateRSquare, $CalculateCovariance);
|
|
|
393
|
|
|
394 $CalculateCorrelation = exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{correlation}) ? 1 : 0;
|
|
|
395 $CalculateRSquare = exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{rsquare}) ? 1 : 0;
|
|
|
396 $CalculateCovariance = exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{covariance}) ? 1 : 0;
|
|
|
397
|
|
|
398 $CorrelationTextFile = $SDFilesInfo{NewTextFileRoot}[$Index] . "CorrelationMatrix." . $SDFilesInfo{NewTextFileExt}[$Index];
|
|
|
399 $RSquareTextFile = $SDFilesInfo{NewTextFileRoot}[$Index] . "RSquareMatrix." . $SDFilesInfo{NewTextFileExt}[$Index];
|
|
|
400 $CovarianceTextFile = $SDFilesInfo{NewTextFileRoot}[$Index] . "CovarianceMatrix." . $SDFilesInfo{NewTextFileExt}[$Index];
|
|
|
401
|
|
|
402 my($TextFilesList, $Delimiter);
|
|
|
403 $TextFilesList = "";
|
|
|
404 if ($CalculateCorrelation || $CalculateRSquare) {
|
|
|
405 $TextFilesList = $CorrelationTextFile;
|
|
|
406 if ($CalculateRSquare) {
|
|
|
407 $TextFilesList .= ", $CorrelationTextFile";
|
|
|
408 }
|
|
|
409 }
|
|
|
410 $Delimiter = length($TextFilesList) ? "," : "";
|
|
|
411 if ($CalculateCovariance) {
|
|
|
412 $TextFilesList .= "${Delimiter} ${CorrelationTextFile}";
|
|
|
413 }
|
|
|
414 if ($TextFilesList =~ /\,/) {
|
|
|
415 print "Generating new text files $TextFilesList...\n"
|
|
|
416 }
|
|
|
417 else {
|
|
|
418 print "Generating new text file $TextFilesList...\n"
|
|
|
419 }
|
|
|
420 if ($CalculateCorrelation || $CalculateRSquare) {
|
|
|
421 open CORRELATIONTEXTFILE, ">$CorrelationTextFile" or die "Error: Can't open $CorrelationTextFile: $! \n";
|
|
|
422 if ($CalculateRSquare) {
|
|
|
423 open RSQUARETEXTFILE, ">$RSquareTextFile" or die "Error: Can't open $RSquareTextFile: $! \n";
|
|
|
424 }
|
|
|
425 }
|
|
|
426 if ($CalculateCovariance) {
|
|
|
427 open COVARIANCETEXTFILE, ">$CovarianceTextFile" or die "Error: Can't open $CovarianceTextFile: $! \n";
|
|
|
428 }
|
|
|
429
|
|
|
430 my($Line, $Value, $CorrelationValue, $RSquareValue, $CovarianceValue, $DataLabel, $DataLabel1, $DataLabel2, $DataValuesRef1, $DataValuesRef2, @ColLabels, @CovarianceRowValues, @CorrelationRowValues, @RSquareRowValues);
|
|
|
431
|
|
|
432 # Write out the column labels...
|
|
|
433 @ColLabels = ();
|
|
|
434 push @ColLabels, @{$SDFilesInfo{AllDataLabels}[$Index]};
|
|
|
435 $Line = JoinWords(\@ColLabels, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
|
|
|
436 if ($CalculateCorrelation || $CalculateRSquare) {
|
|
|
437 print CORRELATIONTEXTFILE "$Line\n";
|
|
|
438 if ($CalculateRSquare) {
|
|
|
439 print RSQUARETEXTFILE "$Line\n";
|
|
|
440 }
|
|
|
441 }
|
|
|
442 if ($CalculateCovariance) {
|
|
|
443 print COVARIANCETEXTFILE "$Line\n";
|
|
|
444 }
|
|
|
445
|
|
|
446 # Due to symmetric nature of these matrices, only one half needs to be
|
|
|
447 # calculated. So, just calculate the lower half and copy it to upper half...
|
|
|
448 my(%CorrelationMatrixMap, %RSquareMatrixMap, %CovarianceMatrixMap, $LabelIndex1, $LabelIndex2, @DataLabelsToAnalyze);
|
|
|
449
|
|
|
450 %CorrelationMatrixMap = (); %RSquareMatrixMap = (); %CovarianceMatrixMap = ();
|
|
|
451 @DataLabelsToAnalyze = ();
|
|
|
452 @DataLabelsToAnalyze = $OptionsInfo{AllDataLabelPairs} ? @{$SDFilesInfo{AllDataLabels}[$Index]} : @{$SDFilesInfo{CommonDataLabels}[$Index]};
|
|
|
453
|
|
|
454 for $LabelIndex1 (0 .. (@DataLabelsToAnalyze - 1)) {
|
|
|
455 $DataLabel1 = $DataLabelsToAnalyze[$LabelIndex1];
|
|
|
456 for $LabelIndex2 (0 .. $LabelIndex1) {
|
|
|
457 $DataLabel2 = $DataLabelsToAnalyze[$LabelIndex2];
|
|
|
458 $DataValuesRef1 = \@{$DataValuesToAnalyzeMapRef->{$DataLabel1}};
|
|
|
459 $DataValuesRef2 = \@{$DataValuesToAnalyzeMapRef->{$DataLabel2}};
|
|
|
460 if ($CalculateCorrelation || $CalculateRSquare) {
|
|
|
461 $CorrelationValue = Correlation($DataValuesRef1, $DataValuesRef2);
|
|
|
462 $CorrelationValue = (defined($CorrelationValue) && length($CorrelationValue)) ? (sprintf("%.$OptionsInfo{Precision}f", $CorrelationValue) + 0) : "";
|
|
|
463 $CorrelationMatrixMap{$DataLabel1}{$DataLabel2} = $CorrelationValue;
|
|
|
464 if ($DataLabel1 ne $DataLabel2) {
|
|
|
465 $CorrelationMatrixMap{$DataLabel2}{$DataLabel1} = $CorrelationValue;
|
|
|
466 }
|
|
|
467 if ($CalculateRSquare) {
|
|
|
468 $RSquareValue = (defined($CorrelationValue) && length($CorrelationValue)) ? ($CorrelationValue ** 2) : "";
|
|
|
469 $RSquareValue = (length($RSquareValue)) ? (sprintf("%.$OptionsInfo{Precision}f", $RSquareValue) + 0) : "";
|
|
|
470 $RSquareMatrixMap{$DataLabel1}{$DataLabel2} = $RSquareValue;
|
|
|
471 if ($DataLabel1 ne $DataLabel2) {
|
|
|
472 $RSquareMatrixMap{$DataLabel2}{$DataLabel1} = $RSquareValue;
|
|
|
473 }
|
|
|
474 }
|
|
|
475 }
|
|
|
476 if ($CalculateCovariance) {
|
|
|
477 $CovarianceValue = Covariance($DataValuesRef1, $DataValuesRef2);
|
|
|
478 $CovarianceValue = (defined($CovarianceValue) && length($CovarianceValue)) ? (sprintf("%.$OptionsInfo{Precision}f", $CovarianceValue) + 0) : "";
|
|
|
479 $CovarianceMatrixMap{$DataLabel1}{$DataLabel2} = $CovarianceValue;
|
|
|
480 if ($DataLabel1 ne $DataLabel2) {
|
|
|
481 $CovarianceMatrixMap{$DataLabel2}{$DataLabel1} = $CovarianceValue;
|
|
|
482 }
|
|
|
483 }
|
|
|
484 }
|
|
|
485 }
|
|
|
486
|
|
|
487 # Write out the matrices...
|
|
|
488 for $LabelIndex1 (0 .. (@DataLabelsToAnalyze - 1)) {
|
|
|
489 $DataLabel1 = $DataLabelsToAnalyze[$LabelIndex1];
|
|
|
490 @CorrelationRowValues = ();
|
|
|
491 @RSquareRowValues = ();
|
|
|
492 @CovarianceRowValues = ();
|
|
|
493 if ($CalculateCorrelation || $CalculateRSquare) {
|
|
|
494 push @CorrelationRowValues, $DataLabel1;
|
|
|
495 if ($CalculateRSquare) {
|
|
|
496 push @RSquareRowValues, $DataLabel1;
|
|
|
497 }
|
|
|
498 }
|
|
|
499 if ($CalculateCovariance) {
|
|
|
500 push @CovarianceRowValues, $DataLabel;
|
|
|
501 }
|
|
|
502 for $LabelIndex2 (0 .. (@DataLabelsToAnalyze - 1)) {
|
|
|
503 $DataLabel2 = $DataLabelsToAnalyze[$LabelIndex2];
|
|
|
504 if ($CalculateCorrelation || $CalculateRSquare) {
|
|
|
505 push @CorrelationRowValues, $CorrelationMatrixMap{$DataLabel1}{$DataLabel2};
|
|
|
506 if ($CalculateRSquare) {
|
|
|
507 push @RSquareRowValues, $RSquareMatrixMap{$DataLabel1}{$DataLabel2};
|
|
|
508 }
|
|
|
509 }
|
|
|
510 if ($CalculateCovariance) {
|
|
|
511 push @CovarianceRowValues, $CovarianceMatrixMap{$DataLabel1}{$DataLabel2};
|
|
|
512 }
|
|
|
513 }
|
|
|
514 if ($CalculateCorrelation || $CalculateRSquare) {
|
|
|
515 $Line = JoinWords(\@CorrelationRowValues, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
|
|
|
516 print CORRELATIONTEXTFILE "$Line\n";
|
|
|
517 if ($CalculateRSquare) {
|
|
|
518 $Line = JoinWords(\@RSquareRowValues, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
|
|
|
519 print RSQUARETEXTFILE "$Line\n";
|
|
|
520 }
|
|
|
521 }
|
|
|
522 if ($CalculateCovariance) {
|
|
|
523 $Line = JoinWords(\@CovarianceRowValues, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
|
|
|
524 print COVARIANCETEXTFILE "$Line\n";
|
|
|
525 }
|
|
|
526 }
|
|
|
527 if ($CalculateCorrelation || $CalculateRSquare) {
|
|
|
528 close CORRELATIONTEXTFILE;
|
|
|
529 if ($CalculateRSquare) {
|
|
|
530 close RSQUARETEXTFILE;
|
|
|
531 }
|
|
|
532 }
|
|
|
533 if ($CalculateCovariance) {
|
|
|
534 close COVARIANCETEXTFILE;
|
|
|
535 }
|
|
|
536 }
|
|
|
537
|
|
|
538 # Calculate standard scores...
|
|
|
539 sub PerformStandardScoresAnalysis {
|
|
|
540 my($Index, $DataValuesToAnalyzeMapRef) = @_;
|
|
|
541 my($StandardScores, $StandardScoresN, $NewTextFile, @ColLabels, $Label, $NewLine);
|
|
|
542
|
|
|
543 $StandardScores = exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{standardscores}) ? 1 : 0;
|
|
|
544 $StandardScoresN = exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{standardscoresn}) ? 1 : 0;
|
|
|
545
|
|
|
546 $NewTextFile = $SDFilesInfo{NewTextFileRoot}[$Index] . "StandardScores." . $SDFilesInfo{NewTextFileExt}[$Index];
|
|
|
547 print "Generating new text file $NewTextFile...\n";
|
|
|
548 open NEWTEXTFILE, ">$NewTextFile" or die "Error: Can't open $NewTextFile: $! \n";
|
|
|
549
|
|
|
550 my($DataLabel, @DataLabelsToAnalyze);
|
|
|
551 # Write out column labels...
|
|
|
552 @ColLabels = ();
|
|
|
553 @DataLabelsToAnalyze = @{$SDFilesInfo{DataLabelsToAnalyze}[$Index]};
|
|
|
554 for $DataLabel (@DataLabelsToAnalyze) {
|
|
|
555 if ($StandardScores) {
|
|
|
556 push @ColLabels, "${DataLabel}\(StandardScores)";
|
|
|
557 }
|
|
|
558 if ($StandardScoresN) {
|
|
|
559 push @ColLabels, "${DataLabel}\(StandardScoresN)";
|
|
|
560 }
|
|
|
561 }
|
|
|
562 $NewLine = JoinWords(\@ColLabels, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
|
|
|
563 print NEWTEXTFILE "$NewLine\n";
|
|
|
564
|
|
|
565 # Go over each column to be analyzed and calculate standard deviation
|
|
|
566 # and mean values...
|
|
|
567 my($DataValuesRef, %StandardDeviationMap, %StandardDeviationNMap, %MeanMap);
|
|
|
568 %StandardDeviationMap = ();
|
|
|
569 %StandardDeviationNMap = ();
|
|
|
570 %MeanMap = ();
|
|
|
571 for $DataLabel (@DataLabelsToAnalyze) {
|
|
|
572 $DataValuesRef = \@{$DataValuesToAnalyzeMapRef->{$DataLabel}};
|
|
|
573 if (!exists($MeanMap{$DataLabel})) {
|
|
|
574 $MeanMap{$DataLabel} = Mean($DataValuesRef);
|
|
|
575 }
|
|
|
576 if ($StandardScores) {
|
|
|
577 if (!exists($StandardDeviationMap{$DataLabel})) {
|
|
|
578 $StandardDeviationMap{$DataLabel} = StandardDeviation($DataValuesRef);
|
|
|
579 }
|
|
|
580 }
|
|
|
581 if ($StandardScoresN) {
|
|
|
582 if (!exists($StandardDeviationNMap{$DataLabel})) {
|
|
|
583 $StandardDeviationNMap{$DataLabel} = StandardDeviationN($DataValuesRef);
|
|
|
584 }
|
|
|
585 }
|
|
|
586 }
|
|
|
587 #
|
|
|
588 # Go over each data field and calculate standard scores for each column
|
|
|
589 # using (x[i] - mean) / (n - 1) for StandardScores and (x[i] - mean) / n
|
|
|
590 # for StandardScoresN; write out the calculated values as well...
|
|
|
591
|
|
|
592 my($SDFile, $Value, $ValueOkay, $ScoreValue, @RowValues, $CmpdString, @CmpdLines, %DataFieldValues);
|
|
|
593 $SDFile = $SDFilesList[$Index];
|
|
|
594
|
|
|
595 open SDFILE, "$SDFile" or die "Error: Can't open $SDFile: $! \n";
|
|
|
596 while ($CmpdString = ReadCmpdString(\*SDFILE)) {
|
|
|
597 @CmpdLines = split "\n", $CmpdString;
|
|
|
598 %DataFieldValues = GetCmpdDataHeaderLabelsAndValues(\@CmpdLines);
|
|
|
599 @RowValues = ();
|
|
|
600 for $DataLabel (@DataLabelsToAnalyze) {
|
|
|
601 $Value = "";
|
|
|
602 if (exists $DataFieldValues{$DataLabel}) {
|
|
|
603 $Value = $DataFieldValues{$DataLabel};
|
|
|
604 }
|
|
|
605 $ValueOkay = ($OptionsInfo{CheckData} && !IsNumerical($Value)) ? 0 : 1;
|
|
|
606 if ($StandardScores) {
|
|
|
607 $ScoreValue = $ValueOkay ? (($Value - $MeanMap{$DataLabel})/$StandardDeviationMap{$DataLabel}) : "";
|
|
|
608 $ScoreValue = (defined($ScoreValue) && length($ScoreValue)) ? (sprintf("%.$OptionsInfo{Precision}f", $ScoreValue) + 0) : "";
|
|
|
609 push @RowValues, $ScoreValue;
|
|
|
610 }
|
|
|
611 if ($StandardScoresN) {
|
|
|
612 $ScoreValue = $ValueOkay ? (($Value - $MeanMap{$DataLabel})/$StandardDeviationNMap{$DataLabel}) : "";
|
|
|
613 $ScoreValue = (defined($ScoreValue) && length($ScoreValue)) ? (sprintf("%.$OptionsInfo{Precision}f", $ScoreValue) + 0) : "";
|
|
|
614 push @RowValues, $ScoreValue;
|
|
|
615 }
|
|
|
616 }
|
|
|
617 $NewLine = JoinWords(\@RowValues, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
|
|
|
618 print NEWTEXTFILE "$NewLine\n";
|
|
|
619 }
|
|
|
620 close SDFILE;
|
|
|
621 close NEWTEXTFILE;
|
|
|
622
|
|
|
623 }
|
|
|
624
|
|
|
625 # Make sure the specified data field labels exists in SD files...
|
|
|
626 sub ProcessSDFilesDataLabelsInfo {
|
|
|
627 my($Index, $DataFieldIndex, $SDFile, $DataLabel, @DataLabelsToAnalyze, %UniqueDataLabelsToAnalyzeMap);
|
|
|
628
|
|
|
629 @{$SDFilesInfo{DataLabelsToAnalyze}} = ();
|
|
|
630 @{$SDFilesInfo{DataLabelPairs1ToAnalyze}} = ();
|
|
|
631 @{$SDFilesInfo{DataLabelPairs2ToAnalyze}} = ();
|
|
|
632 @{$SDFilesInfo{UniqueDataLabelsToAnalyze}} = ();
|
|
|
633
|
|
|
634 FILELIST: for $Index (0 .. $#SDFilesList) {
|
|
|
635 $SDFile = $SDFilesList[$Index];
|
|
|
636
|
|
|
637 @{$SDFilesInfo{DataLabelsToAnalyze}[$Index]} = ();
|
|
|
638 @{$SDFilesInfo{DataLabelPairs1ToAnalyze}[$Index]} = ();
|
|
|
639 @{$SDFilesInfo{DataLabelPairs2ToAnalyze}[$Index]} = ();
|
|
|
640 @{$SDFilesInfo{UniqueDataLabelsToAnalyze}[$Index]} = ();
|
|
|
641
|
|
|
642 %UniqueDataLabelsToAnalyzeMap = ();
|
|
|
643
|
|
|
644 if ($SDFilesInfo{FileOkay}[$Index]) {
|
|
|
645 @DataLabelsToAnalyze = ();
|
|
|
646 if (@{$OptionsInfo{SpecifiedDataLabels}}) {
|
|
|
647 for $DataLabel (@{$OptionsInfo{SpecifiedDataLabels}}) {
|
|
|
648 if (exists($SDFilesInfo{AllDataLabelsMap}[$Index]{$DataLabel})) {
|
|
|
649 push @DataLabelsToAnalyze, $DataLabel;
|
|
|
650 }
|
|
|
651 }
|
|
|
652 }
|
|
|
653 elsif (defined($OptionsInfo{DataFields}) && $OptionsInfo{DataFields} =~ /^All$/i) {
|
|
|
654 push @DataLabelsToAnalyze, @{$SDFilesInfo{AllDataLabels}[$Index]};
|
|
|
655 }
|
|
|
656 else {
|
|
|
657 push @DataLabelsToAnalyze, @{$SDFilesInfo{CommonDataLabels}[$Index]};
|
|
|
658 }
|
|
|
659 if (@DataLabelsToAnalyze) {
|
|
|
660 push @{$SDFilesInfo{DataLabelsToAnalyze}[$Index]}, @DataLabelsToAnalyze;
|
|
|
661 # Set up unique data field label map as well...
|
|
|
662 for $DataLabel (@DataLabelsToAnalyze) {
|
|
|
663 if (!exists $UniqueDataLabelsToAnalyzeMap{$DataLabel}) {
|
|
|
664 $UniqueDataLabelsToAnalyzeMap{$DataLabel} = $DataLabel;
|
|
|
665 }
|
|
|
666 }
|
|
|
667 }
|
|
|
668 else {
|
|
|
669 warn "Warning: Ignoring file $SDFile: None of the data field labels specified, @{$OptionsInfo{SpecifiedDataLabels}}, using \"--datafields\" option exist.\n";
|
|
|
670 $SDFilesInfo{FileOkay}[$Index] = 0;
|
|
|
671 next FILELIST;
|
|
|
672 }
|
|
|
673 if (!$OptionsInfo{Overwrite} && exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{frequency})) {
|
|
|
674 # Make sure specific frequency files don't exist...
|
|
|
675 my($FrequencyFile);
|
|
|
676 for $DataLabel (@DataLabelsToAnalyze) {
|
|
|
677 $FrequencyFile = $SDFilesInfo{NewTextFileRoot}[$Index] . $SDFilesInfo{AllDataLabelsMap}[$Index]{$DataLabel} . "FrequencyAnalysis." . $SDFilesInfo{NewTextFileExt}[$Index];
|
|
|
678 if (-e $FrequencyFile) {
|
|
|
679 warn "Warning: Ignoring file $SDFile: The file $FrequencyFile already exists.\n";
|
|
|
680 $SDFilesInfo{FileOkay}[$Index] = 0;
|
|
|
681 next FILELIST;
|
|
|
682 }
|
|
|
683 }
|
|
|
684 }
|
|
|
685 # Setup specified data field label pairs...
|
|
|
686 if (exists $OptionsInfo{SpecifiedStatisticalFunctionsMap}{correlation} || exists $OptionsInfo{SpecifiedStatisticalFunctionsMap}{covariance} || exists $OptionsInfo{SpecifiedStatisticalFunctionsMap}{rsquare}) {
|
|
|
687 my(@DataLabelPairsToAnalyze, $DataLabel1, $DataLabel2);
|
|
|
688 if (@{$OptionsInfo{SpecifiedDataLabelPairs}}) {
|
|
|
689 # Make sure both data field labels exist...
|
|
|
690 my($DataFieldIndex);
|
|
|
691 for ($DataFieldIndex = 0; (($DataFieldIndex + 1) < @{$OptionsInfo{SpecifiedDataLabelPairs}}); $DataFieldIndex += 2 ) {
|
|
|
692 $DataLabel1 = $OptionsInfo{SpecifiedDataLabelPairs}[$DataFieldIndex];
|
|
|
693 $DataLabel2 = $OptionsInfo{SpecifiedDataLabelPairs}[$DataFieldIndex + 1];
|
|
|
694 if (exists($SDFilesInfo{AllDataLabelsMap}[$Index]{$DataLabel1}) && exists($SDFilesInfo{AllDataLabelsMap}[$Index]{$DataLabel2})) {
|
|
|
695 push @DataLabelPairsToAnalyze, ($DataLabel1, $DataLabel2);
|
|
|
696 }
|
|
|
697 }
|
|
|
698 }
|
|
|
699 elsif ($OptionsInfo{AllDataLabelPairs}) {
|
|
|
700 for $DataLabel1 (@{$SDFilesInfo{AllDataLabels}[$Index]}) {
|
|
|
701 for $DataLabel2 (@{$SDFilesInfo{AllDataLabels}[$Index]}) {
|
|
|
702 push @DataLabelPairsToAnalyze, ($DataLabel1, $DataLabel2);
|
|
|
703 }
|
|
|
704 }
|
|
|
705 }
|
|
|
706 else {
|
|
|
707 for $DataLabel1 (@{$SDFilesInfo{CommonDataLabels}[$Index]}) {
|
|
|
708 for $DataLabel2 (@{$SDFilesInfo{CommonDataLabels}[$Index]}) {
|
|
|
709 push @DataLabelPairsToAnalyze, ($DataLabel1, $DataLabel2);
|
|
|
710 }
|
|
|
711 }
|
|
|
712 }
|
|
|
713 if (@DataLabelPairsToAnalyze) {
|
|
|
714 if (@DataLabelPairsToAnalyze % 2) {
|
|
|
715 warn "Warning: Ignoring file $SDFile: Invalid number values specified using \"--datafieldpairs\" option: It must contain even number of valid values.\n";
|
|
|
716 $SDFilesInfo{FileOkay}[$Index] = 0;
|
|
|
717 next FILELIST;
|
|
|
718 }
|
|
|
719 else {
|
|
|
720 for ($DataFieldIndex = 0; $DataFieldIndex < @DataLabelPairsToAnalyze; $DataFieldIndex += 2) {
|
|
|
721 push @{$SDFilesInfo{DataLabelPairs1ToAnalyze}[$Index]}, $DataLabelPairsToAnalyze[$DataFieldIndex];
|
|
|
722 push @{$SDFilesInfo{DataLabelPairs2ToAnalyze}[$Index]}, $DataLabelPairsToAnalyze[$DataFieldIndex + 1];
|
|
|
723 }
|
|
|
724 # Set up unique data field labe map as well...
|
|
|
725 for $DataLabel (@DataLabelPairsToAnalyze) {
|
|
|
726 if (!exists $UniqueDataLabelsToAnalyzeMap{$DataLabel}) {
|
|
|
727 $UniqueDataLabelsToAnalyzeMap{$DataLabel} = $DataLabel;
|
|
|
728 }
|
|
|
729 }
|
|
|
730 }
|
|
|
731 }
|
|
|
732 }
|
|
|
733 # Setup unique data field label array...
|
|
|
734 push @{$SDFilesInfo{UniqueDataLabelsToAnalyze}[$Index]}, (sort keys %UniqueDataLabelsToAnalyzeMap);
|
|
|
735 }
|
|
|
736 }
|
|
|
737 }
|
|
|
738
|
|
|
739 # Retrieve information about input SD files...
|
|
|
740 sub RetrieveSDFilesInfo {
|
|
|
741 my($SDFile, $Index, $FileDir, $FileExt, $FileName, $OutFile, $OutFileRoot, $OutFileExt, $CmpdCount);
|
|
|
742
|
|
|
743 %SDFilesInfo = ();
|
|
|
744
|
|
|
745 @{$SDFilesInfo{FileOkay}} = ();
|
|
|
746 @{$SDFilesInfo{CmpdCount}} = ();
|
|
|
747 @{$SDFilesInfo{NewTextFileRoot}} = ();
|
|
|
748 @{$SDFilesInfo{NewTextFileExt}} = ();
|
|
|
749
|
|
|
750 @{$SDFilesInfo{AllDataFieldLabels}} = ();
|
|
|
751 @{$SDFilesInfo{AllDataFieldLabelsMap}} = ();
|
|
|
752 @{$SDFilesInfo{CommonDataLabels}} = ();
|
|
|
753
|
|
|
754 FILELIST: for $Index (0 .. $#SDFilesList) {
|
|
|
755 $SDFile = $SDFilesList[$Index];
|
|
|
756
|
|
|
757 $SDFilesInfo{FileOkay}[$Index] = 0;
|
|
|
758
|
|
|
759 $SDFilesInfo{CmpdCount}[$Index] = 0;
|
|
|
760 $SDFilesInfo{NewTextFileRoot}[$Index] = "";
|
|
|
761 $SDFilesInfo{NewTextFileExt}[$Index] = "";
|
|
|
762
|
|
|
763 @{$SDFilesInfo{AllDataLabels}[$Index]} = ();
|
|
|
764 %{$SDFilesInfo{AllDataLabelsMap}[$Index]} = ();
|
|
|
765 @{$SDFilesInfo{CommonDataLabels}[$Index]} = ();
|
|
|
766
|
|
|
767 if (!(-e $SDFile)) {
|
|
|
768 warn "Warning: Ignoring file $SDFile: It doesn't exist\n";
|
|
|
769 next FILELIST;
|
|
|
770 }
|
|
|
771 if (!CheckFileType($SDFile, "sd sdf")) {
|
|
|
772 warn "Warning: Ignoring file $SDFile: It's not a SD file\n";
|
|
|
773 next FILELIST;
|
|
|
774 }
|
|
|
775
|
|
|
776 # Generate appropriate name for the new text files...
|
|
|
777 $FileDir = ""; $FileName = ""; $FileExt = "";
|
|
|
778 ($FileDir, $FileName, $FileExt) = ParseFileName($SDFile);
|
|
|
779 $OutFileExt = "csv";
|
|
|
780 if ($Options{outdelim} =~ /^tab$/i) {
|
|
|
781 $OutFileExt = "tsv";
|
|
|
782 }
|
|
|
783 if ($Options{root} && (@SDFilesList == 1)) {
|
|
|
784 my ($RootFileDir, $RootFileName, $RootFileExt) = ParseFileName($Options{root});
|
|
|
785 if ($RootFileName && $RootFileExt) {
|
|
|
786 $FileName = $RootFileName;
|
|
|
787 }
|
|
|
788 else {
|
|
|
789 $FileName = $Options{root};
|
|
|
790 }
|
|
|
791 $OutFileRoot = $FileName;
|
|
|
792 }
|
|
|
793 else {
|
|
|
794 $OutFileRoot = $FileName;
|
|
|
795 }
|
|
|
796 $OutFile = $OutFileRoot . $OptionsInfo{FileNameMode} . ".$OutFileExt";
|
|
|
797
|
|
|
798 if (!$OptionsInfo{Overwrite}) {
|
|
|
799 if (-e $OutFile) {
|
|
|
800 warn "Warning: Ignoring file $SDFile: The file $OutFile already exists\n";
|
|
|
801 next FILELIST;
|
|
|
802 }
|
|
|
803 if (exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{covariance}) || exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{correlation}) || exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{rsquare})) {
|
|
|
804 if ($OptionsInfo{AllDataLabelPairs}) {
|
|
|
805 if (exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{covariance}) && (-e "${OutFileRoot}CovarianceMatrix.${FileExt}")) {
|
|
|
806 warn "Warning: Ignoring file $SDFile: The file ${OutFileRoot}Covariance.${FileExt} already exists.\n";
|
|
|
807 next FILELIST;
|
|
|
808 }
|
|
|
809 if (exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{correlation}) && (-e "${OutFileRoot}CorrelationMatrix.${FileExt}")) {
|
|
|
810 warn "Warning: Ignoring file $SDFile: The file ${OutFileRoot}CorrelationMatrix.${FileExt} already exists.\n";
|
|
|
811 next FILELIST;
|
|
|
812 }
|
|
|
813 if (exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{rsquare}) && (-e "${OutFileRoot}RSquareMatrix.${FileExt}")) {
|
|
|
814 warn "Warning: Ignoring file $SDFile: The file ${OutFileRoot}RSquareMatrix.${FileExt} already exists.\n";
|
|
|
815 next FILELIST;
|
|
|
816 }
|
|
|
817 }
|
|
|
818 else {
|
|
|
819 if (-e "${OutFileRoot}ColumnPairsAnalysis.${FileExt}") {
|
|
|
820 warn "Warning: Ignoring file $SDFile: The file ${OutFileRoot}ColumnPairsAnalysis.${FileExt} already exists.\n";
|
|
|
821 next FILELIST;
|
|
|
822 }
|
|
|
823 }
|
|
|
824 }
|
|
|
825 if (exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{standardscores}) && (-e "${OutFileRoot}StandardScores.${FileExt}")) {
|
|
|
826 warn "Warning: Ignoring file $SDFile: The file ${OutFileRoot}StandardScores.${FileExt} already exists.\n";
|
|
|
827 next FILELIST;
|
|
|
828 }
|
|
|
829 }
|
|
|
830
|
|
|
831 if (!open SDFILE, "$SDFile") {
|
|
|
832 warn "Warning: Ignoring file $SDFile: Couldn't open it: $! \n";
|
|
|
833 next FILELIST;
|
|
|
834 }
|
|
|
835
|
|
|
836 my($CmpdCount, $Label, $DataFieldLabelsRef, $CommonDataFieldLabelsRef, @DataFieldLabels, @CommonDataFieldLabels);
|
|
|
837 $CmpdCount = 0;
|
|
|
838 @DataFieldLabels = ();
|
|
|
839 @CommonDataFieldLabels = ();
|
|
|
840 ($CmpdCount, $DataFieldLabelsRef, $CommonDataFieldLabelsRef) = GetAllAndCommonCmpdDataHeaderLabels(\*SDFILE);
|
|
|
841 push @DataFieldLabels, @{$DataFieldLabelsRef};
|
|
|
842 push @CommonDataFieldLabels, @{$CommonDataFieldLabelsRef};
|
|
|
843 close SDFILE;
|
|
|
844
|
|
|
845 $SDFilesInfo{FileOkay}[$Index] = 1;
|
|
|
846 $SDFilesInfo{NewTextFileRoot}[$Index] = "$OutFileRoot";
|
|
|
847 $SDFilesInfo{NewTextFileExt}[$Index] = "$OutFileExt";
|
|
|
848
|
|
|
849 $SDFilesInfo{CmpdCount}[$Index] = $CmpdCount;
|
|
|
850 push @{$SDFilesInfo{AllDataLabels}[$Index]}, @DataFieldLabels;
|
|
|
851 push @{$SDFilesInfo{CommonDataLabels}[$Index]}, @CommonDataFieldLabels;
|
|
|
852 for $Label (@DataFieldLabels) {
|
|
|
853 $SDFilesInfo{AllDataLabelsMap}[$Index]{$Label} = $Label;
|
|
|
854 }
|
|
|
855 }
|
|
|
856 }
|
|
|
857
|
|
|
858 # Process option values...
|
|
|
859 sub ProcessOptions {
|
|
|
860 %OptionsInfo = ();
|
|
|
861
|
|
|
862 $OptionsInfo{Mode} = $Options{mode};
|
|
|
863
|
|
|
864 $OptionsInfo{DataFields} = defined $Options{datafields} ? $Options{datafields} : undef;
|
|
|
865
|
|
|
866 $OptionsInfo{DetailLevel} = $Options{detail};
|
|
|
867
|
|
|
868 # Setup supported statistical functions...
|
|
|
869 my($SupportedFunction, @SupportedStatisticaFunctions, %SupportedStatisticaFunctionsMap);
|
|
|
870
|
|
|
871 %SupportedStatisticaFunctionsMap = ();
|
|
|
872 @SupportedStatisticaFunctions = qw(Average AverageDeviation Correlation Count Covariance GeometricMean Frequency HarmonicMean KLargest KSmallest Kurtosis Maximum Minimum Mean Median Mode RSquare Skewness Sum SumOfSquares StandardDeviation StandardDeviationN StandardError StandardScores StandardScoresN TrimMean Variance VarianceN);
|
|
|
873
|
|
|
874 for $SupportedFunction (@SupportedStatisticaFunctions) {
|
|
|
875 $SupportedStatisticaFunctionsMap{lc($SupportedFunction)} = $SupportedFunction;
|
|
|
876 }
|
|
|
877
|
|
|
878 # Setup a list of functions to use for analysis...
|
|
|
879 my($SpecifiedFunction);
|
|
|
880
|
|
|
881 %{$OptionsInfo{SpecifiedStatisticalFunctionsMap}} = ();
|
|
|
882 @{$OptionsInfo{SpecifiedStatisticalFunctions}} = ();
|
|
|
883
|
|
|
884 # Check mode values...
|
|
|
885 if ($Options{mode} =~ /^DescriptiveStatisticsBasic$/i ) {
|
|
|
886 $OptionsInfo{FileNameMode} = "DescriptiveStatisticsBasic";
|
|
|
887 @{$OptionsInfo{SpecifiedStatisticalFunctions}} = qw(Count Maximum Minimum Mean Median StandardDeviation StandardError Variance Sum);
|
|
|
888 }
|
|
|
889 elsif ($Options{mode} =~ /^DescriptiveStatisticsAll$/i ) {
|
|
|
890 $OptionsInfo{FileNameMode} = "DescriptiveStatisticsAll";
|
|
|
891 @{$OptionsInfo{SpecifiedStatisticalFunctions}} = qw(Count Maximum Minimum Mean GeometricMean HarmonicMean TrimMean Median Mode StandardDeviation Kurtosis Skewness StandardError Variance RSquare Frequency KLargest KSmallest Sum);
|
|
|
892 }
|
|
|
893 elsif ($Options{mode} =~ /^All$/i ) {
|
|
|
894 $OptionsInfo{FileNameMode} = "AllStatistics";
|
|
|
895 @{$OptionsInfo{SpecifiedStatisticalFunctions}} = @SupportedStatisticaFunctions;
|
|
|
896 }
|
|
|
897 else {
|
|
|
898 $OptionsInfo{FileNameMode} = "SpecifiedStatistics";
|
|
|
899
|
|
|
900 # Comma delimited list of functions...
|
|
|
901 my($Mode, @SpecifiedFunctions, @UnsupportedSpecifiedFunctions);
|
|
|
902
|
|
|
903 $Mode = $Options{mode};
|
|
|
904 $Mode =~ s/ //g;
|
|
|
905 @SpecifiedFunctions = split ",", $Mode;
|
|
|
906 @UnsupportedSpecifiedFunctions = ();
|
|
|
907 for $SpecifiedFunction (@SpecifiedFunctions) {
|
|
|
908 if (exists($SupportedStatisticaFunctionsMap{lc($SpecifiedFunction)})) {
|
|
|
909 push @{$OptionsInfo{SpecifiedStatisticalFunctions}}, $SpecifiedFunction;
|
|
|
910 }
|
|
|
911 else {
|
|
|
912 push @UnsupportedSpecifiedFunctions, $SpecifiedFunction;
|
|
|
913 }
|
|
|
914 }
|
|
|
915 if (@UnsupportedSpecifiedFunctions) {
|
|
|
916 if (@UnsupportedSpecifiedFunctions > 1) {
|
|
|
917 warn "Error: The values specified - ", JoinWords(\@UnsupportedSpecifiedFunctions, ", ", 0)," - for option \"-m --mode\" are not valid.\n";
|
|
|
918 }
|
|
|
919 else {
|
|
|
920 warn "Error: The value specified, @UnsupportedSpecifiedFunctions , for option \"-m --mode\" is not valid.\n";
|
|
|
921 }
|
|
|
922 die "Allowed values:", JoinWords(\@SupportedStatisticaFunctions, ", ", 0), "\n";
|
|
|
923 }
|
|
|
924 }
|
|
|
925
|
|
|
926 FUNCTION: for $SpecifiedFunction (@{$OptionsInfo{SpecifiedStatisticalFunctions}}) {
|
|
|
927 if (exists $OptionsInfo{SpecifiedStatisticalFunctionsMap}{lc($SpecifiedFunction)} ) {
|
|
|
928 next FUNCTION;
|
|
|
929 }
|
|
|
930 $OptionsInfo{SpecifiedStatisticalFunctionsMap}{lc($SpecifiedFunction)} = $SupportedStatisticaFunctionsMap{lc($SpecifiedFunction)};
|
|
|
931 }
|
|
|
932
|
|
|
933 # Setup delimiter and quotes...
|
|
|
934 $OptionsInfo{OutDelim} = ($Options{outdelim} =~ /tab/i ) ? "\t" : (($Options{outdelim} =~ /semicolon/i) ? "\;" : "\,");
|
|
|
935 $OptionsInfo{OutQuote} = ($Options{quote} =~ /yes/i ) ? 1 : 0;
|
|
|
936
|
|
|
937 $OptionsInfo{Overwrite} = defined $Options{overwrite} ? $Options{overwrite} : undef;
|
|
|
938 $OptionsInfo{Root} = defined $Options{root} ? $Options{root} : undef;
|
|
|
939
|
|
|
940 # Setup miscellaneous options...
|
|
|
941 $OptionsInfo{CheckData} = $Options{fast} ? 0 : 1;
|
|
|
942 $OptionsInfo{Precision} = $Options{precision};
|
|
|
943
|
|
|
944 $OptionsInfo{KLargest} = $Options{klargest};
|
|
|
945 $OptionsInfo{KSmallest} = $Options{ksmallest};
|
|
|
946
|
|
|
947 $OptionsInfo{TrimFraction} = $Options{trimfraction};
|
|
|
948
|
|
|
949 # Setup frequency bin values...
|
|
|
950 $OptionsInfo{NumOfBins} = 10;
|
|
|
951 @{$OptionsInfo{BinRange}} = ();
|
|
|
952 if ($Options{frequencybins} =~ /\,/) {
|
|
|
953 my($BinValue, @SpecifiedBinRange);
|
|
|
954 @SpecifiedBinRange = split /\,/, $Options{frequencybins};
|
|
|
955 if (@SpecifiedBinRange < 2) {
|
|
|
956 die "Error: The value specified, $Options{frequencybins}, for option \"--frequencybins\" is not valid: Must contain at least two values. \n";
|
|
|
957 }
|
|
|
958 for $BinValue (@SpecifiedBinRange) {
|
|
|
959 if (!IsNumerical($BinValue)) {
|
|
|
960 die "Error: The value specified, $Options{frequencybins}, for option \"--frequencybins\" is not valid: Contains non numeric values. \n";
|
|
|
961 }
|
|
|
962 }
|
|
|
963 my($Index1, $Index2);
|
|
|
964 for $Index1 (0 .. $#SpecifiedBinRange) {
|
|
|
965 for $Index2 (($Index1 + 1) .. $#SpecifiedBinRange) {
|
|
|
966 if ($SpecifiedBinRange[$Index1] >= $SpecifiedBinRange[$Index2]) {
|
|
|
967 die "Error: The value specified, $Options{frequencybins}, for option \"--frequencybins\" is not valid: Must contain values in ascending order. \n";
|
|
|
968 }
|
|
|
969 }
|
|
|
970 }
|
|
|
971 push @{$OptionsInfo{BinRange}}, @SpecifiedBinRange;
|
|
|
972 }
|
|
|
973 else {
|
|
|
974 $OptionsInfo{NumOfBins} = $Options{frequencybins};
|
|
|
975 if (!IsPositiveInteger($OptionsInfo{NumOfBins})) {
|
|
|
976 die "Error: The value specified, $Options{frequencybins}, for option \"--frequencybins\" is not valid. Allowed values: positive integer or \"number,number,[number]...\". \n";
|
|
|
977 }
|
|
|
978 }
|
|
|
979
|
|
|
980 # Setup specified data field labels...
|
|
|
981 @{$OptionsInfo{SpecifiedDataLabels}} = ();
|
|
|
982 if (defined $Options{datafields} && $Options{datafields} !~ /^(All|Common)$/i ) {
|
|
|
983 my(@SpecifiedValues) = split ",", $Options{datafields};
|
|
|
984 push @{$OptionsInfo{SpecifiedDataLabels}}, @SpecifiedValues;
|
|
|
985 }
|
|
|
986 @{$OptionsInfo{SpecifiedDataLabelPairs}} = ();
|
|
|
987 $OptionsInfo{AllDataLabelPairs} = (defined($Options{datafieldpairs}) && $Options{datafieldpairs} =~ /^AllPairs$/i) ? 1 : 0;
|
|
|
988 $OptionsInfo{CommonDataLabelPairs} = (defined($Options{datafieldpairs}) && $Options{datafieldpairs} =~ /^CommonPairs$/i) ? 1 : 0;
|
|
|
989 if (defined($Options{datafieldpairs}) && !$OptionsInfo{AllDataLabelPairs} && !$OptionsInfo{CommonDataLabelPairs}) {
|
|
|
990 my(@SpecifiedValues) = split ",", $Options{datafieldpairs};
|
|
|
991 if (@SpecifiedValues % 2) {
|
|
|
992 die "Error: Invalid number of values specified using \"--datafieldpairs\" option: It must contain even number of values.\n";
|
|
|
993 }
|
|
|
994 push @{$OptionsInfo{SpecifiedDataLabelPairs}}, @SpecifiedValues;
|
|
|
995 }
|
|
|
996
|
|
|
997 }
|
|
|
998
|
|
|
999 # Setup script usage and retrieve command line arguments specified using various options...
|
|
|
1000 sub SetupScriptUsage {
|
|
|
1001
|
|
|
1002 # Retrieve all the options...
|
|
|
1003 %Options = ();
|
|
|
1004 $Options{detail} = 0;
|
|
|
1005 $Options{datafields} = "Common";
|
|
|
1006 $Options{datafieldpairs} = "CommonPairs";
|
|
|
1007 $Options{frequencybins} = 10;
|
|
|
1008 $Options{klargest} = 2;
|
|
|
1009 $Options{ksmallest} = 2;
|
|
|
1010 $Options{mode} = "DescriptiveStatisticsBasic";
|
|
|
1011 $Options{outdelim} = "comma";
|
|
|
1012 $Options{precision} = 2;
|
|
|
1013 $Options{quote} = "yes";
|
|
|
1014 $Options{trimfraction} = 0.1;
|
|
|
1015
|
|
|
1016 if (!GetOptions(\%Options, "datafields=s", "datafieldpairs=s", "detail|d=i", "frequencybins=s", "fast|f", "help|h", "klargest=i", "ksmallest=i", "mode|m=s", "outdelim=s", "overwrite|o", "precision|p=i", "quote|q=s", "root|r=s", "trimfraction=f", "workingdir|w=s")) {
|
|
|
1017 die "\nTo get a list of valid options and their values, use \"$ScriptName -h\" or\n\"perl -S $ScriptName -h\" command and try again...\n";
|
|
|
1018 }
|
|
|
1019 if ($Options{workingdir}) {
|
|
|
1020 if (! -d $Options{workingdir}) {
|
|
|
1021 die "Error: The value specified, $Options{workingdir}, for option \"-w --workingdir\" is not a directory name.\n";
|
|
|
1022 }
|
|
|
1023 chdir $Options{workingdir} or die "Error: Couldn't chdir $Options{workingdir}: $! \n";
|
|
|
1024 }
|
|
|
1025 if (!IsInteger($Options{detail})) {
|
|
|
1026 die "Error: The value specified, $Options{detail}, for option \"-d --detail\" is not valid. Allowed values: >= 0\n";
|
|
|
1027 }
|
|
|
1028 if ($Options{outdelim} !~ /^(comma|semicolon|tab)$/i) {
|
|
|
1029 die "Error: The value specified, $Options{outdelim}, for option \"--outdelim\" is not valid. Allowed values: comma, tab, or semicolon\n";
|
|
|
1030 }
|
|
|
1031 if ($Options{quote} !~ /^(yes|no)$/i) {
|
|
|
1032 die "Error: The value specified, $Options{quote}, for option \"-q --quote\" is not valid. Allowed values: yes or no\n";
|
|
|
1033 }
|
|
|
1034 if (!IsPositiveInteger($Options{precision})) {
|
|
|
1035 die "Error: The value specified, $Options{precision}, for option \"-p --precision\" is not valid. Allowed values: > 0 \n";
|
|
|
1036 }
|
|
|
1037 if (!IsPositiveInteger($Options{klargest})) {
|
|
|
1038 die "Error: The value specified, $Options{klargest}, for option \"--klargest\" is not valid. Allowed values: > 0 \n";
|
|
|
1039 }
|
|
|
1040 if (!IsPositiveInteger($Options{ksmallest})) {
|
|
|
1041 die "Error: The value specified, $Options{ksmallest}, for option \"--ksmallest\" is not valid. Allowed values: > 0 \n";
|
|
|
1042 }
|
|
|
1043 if (IsFloat($Options{trimfraction})) {
|
|
|
1044 if ($Options{trimfraction} <= 0 || $Options{trimfraction} >= 1.0) {
|
|
|
1045 die "Error: The value specified, $Options{trimfraction}, for option \"--trimfraction\" is not valid. Allowed values: > 0 and < 1.0\n";
|
|
|
1046 }
|
|
|
1047 }
|
|
|
1048 else {
|
|
|
1049 die "Error: The value specified, $Options{trimfraction}, for option \"--trimfraction\" is not valid. Allowed values: > 0 and < 1.0\n";
|
|
|
1050 }
|
|
|
1051 }
|
|
|
1052
|
|
|
1053 __END__
|
|
|
1054
|
|
|
1055 =head1 NAME
|
|
|
1056
|
|
|
1057 AnalyzeSDFilesData.pl - Analyze numerical data field values in SDFile(s)
|
|
|
1058
|
|
|
1059 =head1 SYNOPSIS
|
|
|
1060
|
|
|
1061 AnalyzeSDFilesData.pl SDFile(s)...
|
|
|
1062
|
|
|
1063 AnalyzeSDFilesData.pl [B<--datafields> "fieldlabel,[fieldlabel,...]" | All]
|
|
|
1064 [B<--datafieldpairs> "fieldlabel,fieldlabel,[fieldlabel,fieldlabel,...]" | AllPairs] [B<-d, --detail> infolevel]
|
|
|
1065 [B<-f, --fast>] [B<--frequencybins> number | "number,number,[number,...]"]
|
|
|
1066 [B<-h, --help>] [B<--klargest> number] [B<--ksmallest> number]
|
|
|
1067 [B<-m, --mode> DescriptiveStatisticsBasic | DescriptiveStatisticsAll | All | "function1, [function2,...]"]
|
|
|
1068 [B<--trimfraction> number] [B<-w, --workingdir> dirname] SDFiles(s)...
|
|
|
1069
|
|
|
1070 =head1 DESCRIPTION
|
|
|
1071
|
|
|
1072 Analyze numerical data field values in I<SDFile(s)> using a combination of various statistical
|
|
|
1073 functions; Non-numerical values are simply ignored. For I<Correlation, RSquare, and
|
|
|
1074 Covariance> analysis, the count of valid values in specified data field pairs must be same;
|
|
|
1075 otherwise, column data field pair is ignored. The file names are separated by space.The valid file
|
|
|
1076 extensions are I<.sdf> and I<.sd>. All other file names are ignored. All the SD files in a
|
|
|
1077 current directory can be specified either by I<*.sdf> or the current directory name.
|
|
|
1078
|
|
|
1079 =head1 OPTIONS
|
|
|
1080
|
|
|
1081 =over 4
|
|
|
1082
|
|
|
1083 =item B<--datafields> I<"fieldlabel,[fieldlabel,...]" | Common | All>
|
|
|
1084
|
|
|
1085 Data fields to use for analysis. Possible values: list of comma separated data field
|
|
|
1086 labels, data fields common to all records, or all data fields. Default value: I<Common>.
|
|
|
1087 Examples:
|
|
|
1088
|
|
|
1089 ALogP,MolWeight,EC50
|
|
|
1090 "MolWeight,PSA"
|
|
|
1091
|
|
|
1092 =item B<--datafieldpairs> I<"fieldlabel,fieldlabel,[fieldlabel,fieldlabel,...]" | CommonPairs | AllPairs>
|
|
|
1093
|
|
|
1094 This value is mode specific and is only used for I<Correlation, PearsonCorrelation, or
|
|
|
1095 Covariance> value of B<-m, --mode> option. It specifies data field label pairs to use
|
|
|
1096 for data analysis during I<Correlation> and I<Covariance> calculations. Possible values:
|
|
|
1097 comma delimited list of data field label pairs, data field label pairs common to all records,
|
|
|
1098 or all data field pairs. Default value:I<CommonPairs>. Example:
|
|
|
1099
|
|
|
1100 MolWeight,EC50,NumN+O,PSA
|
|
|
1101
|
|
|
1102 For I<AllPairs> value of B<--datafieldpairs> option, all data field label pairs are used for
|
|
|
1103 I<Correlation> and I<Covariance> calculations.
|
|
|
1104
|
|
|
1105 =item B<-d, --detail> I<infolevel>
|
|
|
1106
|
|
|
1107 Level of information to print about column values being ignored. Default: I<0>. Possible values:
|
|
|
1108 0, 1, 2, 3, or 4.
|
|
|
1109
|
|
|
1110 =item B<-f, --fast>
|
|
|
1111
|
|
|
1112 In this mode, all the data field values specified for analysis are assumed to contain numerical
|
|
|
1113 data and no checking is performed before analysis. By default, only numerical data is
|
|
|
1114 used for analysis; other types of column data is ignored.
|
|
|
1115
|
|
|
1116 =item B<--frequencybins> I<number | "number,number,[number,...]">
|
|
|
1117
|
|
|
1118 Specify number of bins or bin range to use for frequency analysis. Default value: I<10>
|
|
|
1119
|
|
|
1120 Number of bins value along with the smallest and largest value for a column is used to
|
|
|
1121 group the column values into different groups.
|
|
|
1122
|
|
|
1123 The bin range list is used to group values for a column into different groups; It must contain
|
|
|
1124 values in ascending order. Examples:
|
|
|
1125
|
|
|
1126 10,20,30
|
|
|
1127 0.1,0.2,0.3,0.4,0.5
|
|
|
1128
|
|
|
1129 The frequency value calculated for a specific bin corresponds to all the column values
|
|
|
1130 which are greater than the previous bin value and less than or equal to the current bin value.
|
|
|
1131
|
|
|
1132 =item B<-h, --help>
|
|
|
1133
|
|
|
1134 Print this help message.
|
|
|
1135
|
|
|
1136 =item B<--klargest> I<number>
|
|
|
1137
|
|
|
1138 Kth largest value to find by I<KLargest> function. Default value: I<2>. Valid values: positive
|
|
|
1139 integers.
|
|
|
1140
|
|
|
1141 =item B<--ksmallest> I<number>
|
|
|
1142
|
|
|
1143 Kth smallest value to find by I<KSmallest> function. Default values: I<2>. Valid values: positive
|
|
|
1144 integers.
|
|
|
1145
|
|
|
1146 =item B<-m, --mode> I<DescriptiveStatisticsBasic | DescriptiveStatisticsAll | All | "function1, [function2,...]">
|
|
|
1147
|
|
|
1148 Specify how to analyze data in SDFile(s): calculate basic or all descriptive statistics; or
|
|
|
1149 use a comma delimited list of supported statistical functions. Possible values:
|
|
|
1150 I<DescriptiveStatisticsBasic | DescriptiveStatisticsAll | "function1,[function2]...">. Default
|
|
|
1151 value: I<DescriptiveStatisticsBasic>
|
|
|
1152
|
|
|
1153 I<DescriptiveStatisticsBasic> includes these functions: I<Count, Maximum, Minimum, Mean,
|
|
|
1154 Median, Sum, StandardDeviation, StandardError, Variance>.
|
|
|
1155
|
|
|
1156 I<DescriptiveStatisticsAll>, in addition to I<DescriptiveStatisticsBasic> functions, includes:
|
|
|
1157 I<GeometricMean, Frequency, HarmonicMean, KLargest, KSmallest, Kurtosis, Mode, RSquare,
|
|
|
1158 Skewness, TrimMean>.
|
|
|
1159
|
|
|
1160 I<All> uses complete list of supported functions: I<Average, AverageDeviation, Correlation,
|
|
|
1161 Count, Covariance, GeometricMean, Frequency, HarmonicMean, KLargest, KSmallest, Kurtosis,
|
|
|
1162 Maximum, Minimum, Mean, Median, Mode, RSquare, Skewness, Sum,
|
|
|
1163 SumOfSquares, StandardDeviation, StandardDeviationN, StandardError, StandardScores,
|
|
|
1164 StandardScoresN, TrimMean, Variance, VarianceN>. The function names ending with N
|
|
|
1165 calculate corresponding values assuming an entire population instead of a population sample.
|
|
|
1166 Here are the formulas for these functions:
|
|
|
1167
|
|
|
1168 Average: See Mean
|
|
|
1169
|
|
|
1170 AverageDeviation: SUM( ABS(x[i] - Xmean) ) / n
|
|
|
1171
|
|
|
1172 Correlation: See Pearson Correlation
|
|
|
1173
|
|
|
1174 Covariance: SUM( (x[i] - Xmean)(y[i] - Ymean) ) / n
|
|
|
1175
|
|
|
1176 GeometricMean: NthROOT( PRODUCT(x[i]) )
|
|
|
1177
|
|
|
1178 HarmonicMean: 1 / ( SUM(1/x[i]) / n )
|
|
|
1179
|
|
|
1180 Mean: SUM( x[i] ) / n
|
|
|
1181
|
|
|
1182 Median: Xsorted[(n - 1)/2 + 1] for even values of n; (Xsorted[n/2] + Xsorted[n/2 + 1])/2
|
|
|
1183 for odd values of n.
|
|
|
1184
|
|
|
1185 Kurtosis: [ {n(n + 1)/(n - 1)(n - 2)(n - 3)} SUM{ ((x[i] - Xmean)/STDDEV)^4 } ] -
|
|
|
1186 {3((n - 1)^2)}/{(n - 2)(n-3)}
|
|
|
1187
|
|
|
1188 PearsonCorrelation: SUM( (x[i] - Xmean)(y[i] - Ymean) ) / SQRT( SUM( (x[i] - Xmean)^2 )
|
|
|
1189 (SUM( (y[i] - Ymean)^2 )) )
|
|
|
1190
|
|
|
1191 RSquare: PearsonCorrelation^2
|
|
|
1192
|
|
|
1193 Skewness: {n/(n - 1)(n - 2)} SUM{ ((x[i] - Xmean)/STDDEV)^3 }
|
|
|
1194
|
|
|
1195 StandardDeviation: SQRT ( SUM( (x[i] - Mean)^2 ) / (n - 1) )
|
|
|
1196
|
|
|
1197 StandardDeviationN: SQRT ( SUM( (x[i] - Mean)^2 ) / n )
|
|
|
1198
|
|
|
1199 StandardError: StandardDeviation / SQRT( n )
|
|
|
1200
|
|
|
1201 StandardScore: (x[i] - Mean) / (n - 1)
|
|
|
1202
|
|
|
1203 StandardScoreN: (x[i] - Mean) / n
|
|
|
1204
|
|
|
1205 Variance: SUM( (x[i] - Xmean)^2 / (n - 1) )
|
|
|
1206
|
|
|
1207 VarianceN: SUM( (x[i] - Xmean)^2 / n )
|
|
|
1208
|
|
|
1209 =item B<-o, --overwrite>
|
|
|
1210
|
|
|
1211 Overwrite existing files.
|
|
|
1212
|
|
|
1213 =item B<--outdelim> I<comma | tab | semicolon>
|
|
|
1214
|
|
|
1215 Output text file delimiter. Possible values: I<comma, tab, or semicolon>
|
|
|
1216 Default value: I<comma>.
|
|
|
1217
|
|
|
1218 =item B<-p, --precision> I<number>
|
|
|
1219
|
|
|
1220 Precision of calculated values in the output file. Default: up to I<2> decimal places.
|
|
|
1221 Valid values: positive integers.
|
|
|
1222
|
|
|
1223 =item B<-q, --quote> I<yes | no>
|
|
|
1224
|
|
|
1225 Put quotes around column values in output text file. Possible values: I<yes or
|
|
|
1226 no>. Default value: I<yes>.
|
|
|
1227
|
|
|
1228 =item B<-r, --root> I<rootname>
|
|
|
1229
|
|
|
1230 New text file name is generated using the root: <Root>.<Ext>. Default new file
|
|
|
1231 name: <InitialSDFileName><Mode>.<Ext>. Based on the specified analysis,
|
|
|
1232 <Mode> corresponds to one of these values: DescriptiveStatisticsBasic,
|
|
|
1233 DescriptiveStatisticsAll, AllStatistics, SpecifiedStatistics, Covariance, Correlation,
|
|
|
1234 Frequency, or StandardScores. The csv, and tsv <Ext> values are used for
|
|
|
1235 comma/semicolon, and tab delimited text files respectively. This option is ignored for
|
|
|
1236 multiple input files.
|
|
|
1237
|
|
|
1238 =item B<--trimfraction> I<number>
|
|
|
1239
|
|
|
1240 Fraction of data to exclude from the top and bottom of the data set during
|
|
|
1241 I<TrimMean> calculation. Default value: I<0.1> Valid values: > 0 and < 1.
|
|
|
1242
|
|
|
1243 =item B<-w --workingdir> I<text>
|
|
|
1244
|
|
|
1245 Location of working directory. Default: current directory.
|
|
|
1246
|
|
|
1247 =back
|
|
|
1248
|
|
|
1249 =head1 EXAMPLES
|
|
|
1250
|
|
|
1251 To calculate basic statistics for data in all common data fields and generate a
|
|
|
1252 NewSample1DescriptiveStatisticsBasic.csv file, type:
|
|
|
1253
|
|
|
1254 % AnalyzeSDFilesData.pl -o -r NewSample1 Sample1.sdf
|
|
|
1255
|
|
|
1256 To calculate basic statistics for MolWeight data field and generate a
|
|
|
1257 NewSample1DescriptiveStatisticsBasic.csv file, type:
|
|
|
1258
|
|
|
1259 % AnalyzeSDFilesData.pl --datafields MolWeight -o -r NewSample1
|
|
|
1260 Sample1.sdf
|
|
|
1261
|
|
|
1262 To calculate all available statistics for MolWeight data field and all data field pairs,
|
|
|
1263 and generate NewSample1DescriptiveStatisticsAll.csv, NewSample1CorrelationMatrix.csv,
|
|
|
1264 NewSample1CorrelationMatrix.csv, and NewSample1MolWeightFrequencyAnalysis.csv
|
|
|
1265 files, type:
|
|
|
1266
|
|
|
1267 % AnalyzeSDFilesData.pl -m DescriptiveStatisticsAll --datafields
|
|
|
1268 MolWeight -o --datafieldpairs AllPairs -r NewSample1 Sample1.sdf
|
|
|
1269
|
|
|
1270 To compute frequency distribution of MolWeight data field into five bins and
|
|
|
1271 generate NewSample1MolWeightFrequencyAnalysis.csv, type:
|
|
|
1272
|
|
|
1273 % AnalyzeSDFilesData.pl -m Frequency --frequencybins 5 --datafields
|
|
|
1274 MolWeight -o -r NewSample1 Sample1.sdf
|
|
|
1275
|
|
|
1276 To compute frequency distribution of data in MolWeight data field into specified bin range
|
|
|
1277 values, and generate NewSample1MolWeightFrequencyAnalysis.csv, type:
|
|
|
1278
|
|
|
1279 % AnalyzeSDFilesData.pl -m Frequency --frequencybins "100,200,400"
|
|
|
1280 --datafields MolWeight -o -r NewSample1 Sample1.sdf
|
|
|
1281
|
|
|
1282 To calculate all available statistics for data in all data fields and pairs, type:
|
|
|
1283
|
|
|
1284 % AnalyzeSDFilesData.pl -m All --datafields All --datafieldpairs
|
|
|
1285 AllPairs -o -r NewSample1 Sample1.sdf
|
|
|
1286
|
|
|
1287 =head1 AUTHOR
|
|
|
1288
|
|
|
1289 Manish Sud <msud@san.rr.com>
|
|
|
1290
|
|
|
1291 =head1 SEE ALSO
|
|
|
1292
|
|
|
1293 FilterSDFiles.pl, InfoSDFiles.pl, SplitSDFiles.pl, MergeTextFilesWithSD.pl
|
|
|
1294
|
|
|
1295 =head1 COPYRIGHT
|
|
|
1296
|
|
|
1297 Copyright (C) 2015 Manish Sud. All rights reserved.
|
|
|
1298
|
|
|
1299 This file is part of MayaChemTools.
|
|
|
1300
|
|
|
1301 MayaChemTools is free software; you can redistribute it and/or modify it under
|
|
|
1302 the terms of the GNU Lesser General Public License as published by the Free
|
|
|
1303 Software Foundation; either version 3 of the License, or (at your option)
|
|
|
1304 any later version.
|
|
|
1305
|
|
|
1306 =cut
|
