       1 #!/usr/bin/perl -w
       2 #
       3 # $RCSfile: SortTextFiles.pl,v $
       4 # $Date: 2015/02/28 20:46:21 $
       5 # $Revision: 1.36 $
       6 #
       7 # Author: Manish Sud <msud@san.rr.com>
       8 #
       9 # Copyright (C) 2015 Manish Sud. All rights reserved.
      10 #
      11 # This file is part of MayaChemTools.
      12 #
      13 # MayaChemTools is free software; you can redistribute it and/or modify it under
      14 # the terms of the GNU Lesser General Public License as published by the Free
      15 # Software Foundation; either version 3 of the License, or (at your option) any
      16 # later version.
      17 #
      18 # MayaChemTools is distributed in the hope that it will be useful, but without
      19 # any warranty; without even the implied warranty of merchantability of fitness
      20 # for a particular purpose.  See the GNU Lesser General Public License for more
      21 # details.
      22 #
      23 # You should have received a copy of the GNU Lesser General Public License
      24 # along with MayaChemTools; if not, see <http://www.gnu.org/licenses/> or
      25 # write to the Free Software Foundation Inc., 59 Temple Place, Suite 330,
      26 # Boston, MA, 02111-1307, USA.
      27 #
      28
      29 use strict;
      30 use FindBin; use lib "$FindBin::Bin/../lib";
      31 use Getopt::Long;
      32 use File::Basename;
      33 use Text::ParseWords;
      34 use Benchmark;
      35 use FileUtil;
      36 use TextUtil;
      37
      38 my($ScriptName, %Options, $StartTime, $EndTime, $TotalTime);
      39
      40 # Autoflush STDOUT
      41 $| = 1;
      42
      43 # Starting message...
      44 $ScriptName = basename($0);
      45 print "\n$ScriptName: Starting...\n\n";
      46 $StartTime = new Benchmark;
      47
      48 # Get the options and setup script...
      49 SetupScriptUsage();
      50 if ($Options{help} || @ARGV < 1) {
      51   die GetUsageFromPod("$FindBin::Bin/$ScriptName");
      52 }
      53
      54 my(@TextFilesList);
      55 @TextFilesList = ExpandFileNames(\@ARGV, "csv tsv");
      56
      57 print "Processing options...\n";
      58 my(%OptionsInfo);
      59 ProcessOptions();
      60
      61 print "Checking input text file(s)...\n";
      62 my(%TextFilesInfo);
      63 RetrieveTextFilesInfo();
      64 ProcessColumnsInfo();
      65
      66 # Generate output files...
      67 my($FileIndex);
      68 if (@TextFilesList > 1) {
      69   print "\nProcessing text files...\n";
      70 }
      71 for $FileIndex (0 .. $#TextFilesList) {
      72   if ($TextFilesInfo{FileOkay}[$FileIndex]) {
      73     print "\nProcessing file $TextFilesList[$FileIndex]...\n";
      74     SortTextFile($FileIndex);
      75   }
      76 }
      77 print "\n$ScriptName:Done...\n\n";
      78
      79 $EndTime = new Benchmark;
      80 $TotalTime = timediff ($EndTime, $StartTime);
      81 print "Total time: ", timestr($TotalTime), "\n";
      82
      83 ###############################################################################
      84
      85 # Sort it out...
      86 sub SortTextFile {
      87   my($Index) = @_;
      88   my($TextFile, $NewTextFile, $KeyCol, $Line, $KeyColValue, $InDelim, @ColLabels, @LineWords);
      89
      90   $TextFile = $TextFilesList[$Index];
      91   $InDelim = $TextFilesInfo{InDelim}[$Index];
      92   $NewTextFile = $TextFilesInfo{OutFile}[$Index];
      93   $KeyCol = $TextFilesInfo{KeyColNum}[$Index];
      94   @ColLabels = @{$TextFilesInfo{ColLabels}[$Index]};
      95
      96   print "Generating new Text file $NewTextFile...\n";
      97   open NEWTEXTFILE, ">$NewTextFile" or die "Error: Couldn't open $NewTextFile: $! \n";
      98   open TEXTFILE, "$TextFile" or die "Error: Can't open $TextFile: $! \n";
      99
     100   # Skip over column labels from old file...
     101   $Line = GetTextLine(\*TEXTFILE);
     102
     103   # Add column lablels in new file...
     104   $Line = JoinWords(\@ColLabels, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
     105   print NEWTEXTFILE "$Line\n";
     106
     107   # Go over all rows and store the lines using key value as hash...
     108   my(%KeyToLinesMap, @InvalidDataLines, $LineCount);
     109
     110   %KeyToLinesMap = ();
     111   @InvalidDataLines = ();
     112   $LineCount = 1;
     113   TEXTLINE: while ($Line = GetTextLine(\*TEXTFILE)) {
     114     @LineWords = quotewords($InDelim, 0, $Line);
     115     $LineCount++;
     116     if ($KeyCol < @LineWords) {
     117       $KeyColValue = $LineWords[$KeyCol];
     118       if (!IsNotEmpty($KeyColValue)) {
     119         $Line = JoinWords(\@LineWords, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
     120         push @InvalidDataLines, $Line;
     121         if ($OptionsInfo{DetailLevel} >= 3 ) {
     122           print "Ignoring line $LineCount: Contains empty value for key column $ColLabels[$KeyCol]: $Line\n";
     123         }
     124         elsif ($OptionsInfo{DetailLevel} >= 2) {
     125           print "Ignoring line $LineCount: Contains empty value for key column $ColLabels[$KeyCol]...\n";
     126         }
     127         next TEXTLINE;
     128       }
     129       if ($OptionsInfo{KeyData} =~ /^numeric$/i) {
     130         if (!IsFloat($KeyColValue)) {
     131           $Line = JoinWords(\@LineWords, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
     132           push @InvalidDataLines, $Line;
     133           if ($OptionsInfo{DetailLevel} >= 3 ) {
     134             print "Line number $LineCount: Contains non-numerical value for key column $ColLabels[$KeyCol]: $Line\n";
     135           }
     136           elsif ($OptionsInfo{DetailLevel} >= 2) {
     137             print "Line number $LineCount: Contains non-numerical value for key column $ColLabels[$KeyCol]...\n";
     138           }
     139           next TEXTLINE;
     140         }
     141       }
     142       if (exists($KeyToLinesMap{$KeyColValue})) {
     143         # Append to existing line...
     144         $Line = JoinWords(\@LineWords, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
     145         $KeyToLinesMap{$KeyColValue} .= "\n" . $Line;
     146       }
     147       else {
     148         $Line = JoinWords(\@LineWords, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
     149         $KeyToLinesMap{$KeyColValue} = $Line;
     150       }
     151     }
     152   }
     153   if ($OptionsInfo{Sort} =~ /^ascending$/i) {
     154     if ($OptionsInfo{KeyData} =~ /^alphanumeric$/i) {
     155       for $KeyColValue (sort { lc($a) cmp lc($b) } keys %KeyToLinesMap ) {
     156         print NEWTEXTFILE "$KeyToLinesMap{$KeyColValue}\n";
     157       }
     158     }
     159     else {
     160       for $KeyColValue (sort { $a <=> $b } keys %KeyToLinesMap ) {
     161         print NEWTEXTFILE "$KeyToLinesMap{$KeyColValue}\n";
     162       }
     163     }
     164   }
     165   else {
     166     if ($OptionsInfo{KeyData} =~ /^alphanumeric$/i) {
     167       for $KeyColValue (sort { lc($b) cmp lc($a) } keys %KeyToLinesMap ) {
     168         print NEWTEXTFILE "$KeyToLinesMap{$KeyColValue}\n";
     169       }
     170     }
     171     else {
     172       for $KeyColValue (sort { $b <=> $a } keys %KeyToLinesMap ) {
     173         print NEWTEXTFILE "$KeyToLinesMap{$KeyColValue}\n";
     174       }
     175     }
     176   }
     177   # Write out the lines with invalid data...
     178   if (@InvalidDataLines) {
     179     print "Placing ", scalar(@InvalidDataLines)," line(s) with invalid column key data at the end...\n";
     180     for $Line (@InvalidDataLines) {
     181       print NEWTEXTFILE "$Line\n";
     182     }
     183   }
     184   close NEWTEXTFILE;
     185   close TEXTFILE;
     186
     187 }
     188
     189 # Retrieve information about input text files...
     190 sub RetrieveTextFilesInfo {
     191   my($Index, $TextFile, $FileDir, $FileName, $FileExt, $InDelim, $Line, @ColLabels, $OutFileRoot,  $OutFile, $ColNum, $ColLabel);
     192
     193   %TextFilesInfo = ();
     194
     195   @{$TextFilesInfo{FileOkay}} = ();
     196   @{$TextFilesInfo{ColCount}} = ();
     197   @{$TextFilesInfo{ColLabels}} = ();
     198   @{$TextFilesInfo{ColLabelToNumMap}} = ();
     199   @{$TextFilesInfo{InDelim}} = ();
     200   @{$TextFilesInfo{OutFile}} = ();
     201
     202   FILELIST: for $Index (0 .. $#TextFilesList) {
     203     $TextFile = $TextFilesList[$Index];
     204
     205     $TextFilesInfo{FileOkay}[$Index] = 0;
     206     $TextFilesInfo{ColCount}[$Index] = 0;
     207     $TextFilesInfo{InDelim}[$Index] = "";
     208     $TextFilesInfo{OutFile}[$Index] = "";
     209     @{$TextFilesInfo{ColLabels}[$Index]} = ();
     210     %{$TextFilesInfo{ColLabelToNumMap}[$Index]} = ();
     211
     212     if (!(-e $TextFile)) {
     213       warn "Warning: Ignoring file $TextFile: It doesn't exist\n";
     214       next FILELIST;
     215     }
     216     if (!CheckFileType($TextFile, "csv tsv")) {
     217       warn "Warning: Ignoring file $TextFile: It's not a csv or tsv file\n";
     218       next FILELIST;
     219     }
     220     ($FileDir, $FileName, $FileExt) = ParseFileName($TextFile);
     221     if ($FileExt =~ /^tsv$/i) {
     222       $InDelim = "\t";
     223     }
     224     else {
     225       $InDelim = "\,";
     226       if ($Options{indelim} !~ /^(comma|semicolon)$/i) {
     227         warn "Warning: Ignoring file $TextFile: The value specified, $Options{indelim}, for option \"--indelim\" is not valid for csv files\n";
     228         next FILELIST;
     229       }
     230       if ($Options{indelim} =~ /^semicolon$/i) {
     231         $InDelim = "\;";
     232       }
     233     }
     234
     235     if (!open TEXTFILE, "$TextFile") {
     236       warn "Warning: Ignoring file $TextFile: Couldn't open it: $! \n";
     237       next FILELIST;
     238     }
     239
     240     $Line = GetTextLine(\*TEXTFILE);
     241     @ColLabels = quotewords($InDelim, 0, $Line);
     242     close TEXTFILE;
     243
     244     $FileDir = ""; $FileName = ""; $FileExt = "";
     245     ($FileDir, $FileName, $FileExt) = ParseFileName($TextFile);
     246     $FileExt = "csv";
     247     if ($Options{outdelim} =~ /^tab$/i) {
     248       $FileExt = "tsv";
     249     }
     250     if ($Options{root} && (@TextFilesList == 1)) {
     251       my ($RootFileDir, $RootFileName, $RootFileExt) = ParseFileName($Options{root});
     252       if ($RootFileName && $RootFileExt) {
     253         $FileName = $RootFileName;
     254       }
     255       else {
     256         $FileName = $Options{root};
     257       }
     258       $OutFileRoot = $FileName;
     259     }
     260     else {
     261       $OutFileRoot = $FileName . "SortedByColumn";
     262     }
     263
     264     $OutFile = $OutFileRoot . ".$FileExt";
     265     if (lc($OutFile) eq lc($TextFile)) {
     266       warn "Warning: Ignoring file $TextFile:Output file name, $OutFile, is same as input text file name, $TextFile\n";
     267       next FILELIST;
     268     }
     269     if (!$Options{overwrite}) {
     270       if (-e $OutFile) {
     271         warn "Warning: Ignoring file $TextFile: The file $OutFile already exists\n";
     272         next FILELIST;
     273       }
     274     }
     275
     276     $TextFilesInfo{FileOkay}[$Index] = 1;
     277     $TextFilesInfo{InDelim}[$Index] = $InDelim;
     278     $TextFilesInfo{OutFile}[$Index] = "$OutFile";
     279
     280     $TextFilesInfo{ColCount}[$Index] = @ColLabels;
     281     push @{$TextFilesInfo{ColLabels}[$Index]}, @ColLabels;
     282     for $ColNum (0 .. $#ColLabels) {
     283       $ColLabel = $ColLabels[$ColNum];
     284       $TextFilesInfo{ColLabelToNumMap}[$Index]{$ColLabel} = $ColNum;
     285     }
     286   }
     287
     288 }
     289
     290 # Make sure specified key column are okay...
     291 sub ProcessColumnsInfo {
     292   my($Index, $TextFile, $SpecifiedKeyCol);
     293
     294   @{$TextFilesInfo{KeyColNum}} = ();
     295
     296   $SpecifiedKeyCol = $OptionsInfo{SpecifiedKeyCol};
     297
     298   FILELIST: for $Index (0 .. $#TextFilesList) {
     299     $TextFile = $TextFilesList[$Index];
     300     $TextFilesInfo{KeyColNum}[$Index] = 0;
     301
     302     if ($TextFilesInfo{FileOkay}[$Index]) {
     303       my($KeyColNum, $KeyColValid);
     304
     305       $KeyColNum = 0;
     306       $KeyColValid = 1;
     307       if ($SpecifiedKeyCol) {
     308         if ($OptionsInfo{Mode} =~ /^colnum$/i) {
     309           if ($SpecifiedKeyCol <= $TextFilesInfo{ColCount}[$Index]) {
     310             $KeyColNum = $SpecifiedKeyCol - 1;
     311           }
     312           else {
     313             $KeyColValid = 0;
     314           }
     315         }
     316         else {
     317           if (exists($TextFilesInfo{ColLabelToNumMap}[$Index]{$SpecifiedKeyCol})) {
     318             $KeyColNum =  $TextFilesInfo{ColLabelToNumMap}[$Index]{$SpecifiedKeyCol};
     319           }
     320           else {
     321             $KeyColValid = 0;
     322           }
     323         }
     324       }
     325       if ($KeyColValid) {
     326         $TextFilesInfo{KeyColNum}[$Index] = $KeyColNum;
     327       }
     328       else {
     329         warn "Warning: Ignoring file $TextFile: Column key specified, $SpecifiedKeyCol, using \"k --key\" option doesn't exist\n";
     330         $TextFilesInfo{FileOkay}[$Index] = 0;
     331       }
     332     }
     333   }
     334 }
     335
     336 # Process option values...
     337 sub ProcessOptions {
     338   %OptionsInfo = ();
     339
     340   $OptionsInfo{Mode} = $Options{mode};
     341
     342   $OptionsInfo{DetailLevel} = $Options{detail};
     343
     344   $OptionsInfo{Sort} = $Options{sort};
     345   $OptionsInfo{KeyData} = $Options{keydata};
     346
     347   $OptionsInfo{InDelim} = $Options{indelim};
     348
     349   $OptionsInfo{OutDelim} = ($Options{outdelim} =~ /^tab$/i ) ? "\t" : (($Options{outdelim} =~ /^semicolon$/i) ? "\;" : "\,");
     350   $OptionsInfo{OutQuote} = ($Options{quote} =~ /^yes$/i) ? 1 : 0;
     351
     352   $OptionsInfo{Overwrite} = defined $Options{overwrite} ? $Options{overwrite} : undef;
     353   $OptionsInfo{Root} = defined $Options{root} ? $Options{root} : undef;
     354
     355   $OptionsInfo{Key} = defined $Options{key} ? $Options{key} : undef;
     356   $OptionsInfo{SpecifiedKeyCol} = "";
     357
     358   if (defined $Options{key}) {
     359     $OptionsInfo{SpecifiedKeyCol} = $Options{key};
     360     if ($Options{mode} =~ /^colnum$/i) {
     361       if (!IsPositiveInteger($OptionsInfo{SpecifiedKeyCol})) {
     362         die "Error: Invalid value $Options{key} specified using \"-k --key\" option: Allowed values: > 0\n";
     363       }
     364     }
     365   }
     366 }
     367
     368 # Setup script usage  and retrieve command line arguments specified using various options...
     369 sub SetupScriptUsage {
     370   %Options = ();
     371
     372   $Options{detail} = 1;
     373   $Options{mode} = "colnum";
     374   $Options{sort} = "ascending";
     375   $Options{keydata} = "numeric";
     376   $Options{indelim} = "comma";
     377   $Options{outdelim} = "comma";
     378   $Options{quote} = "yes";
     379   if (!GetOptions(\%Options, "detail|d=i", "help|h", "indelim=s", "key|k=s", "keydata=s", "mode|m=s", "outdelim=s", "overwrite|o", "quote|q=s", "root|r=s", "sort|s=s", "workingdir|w=s")) {
     380     die "\nTo get a list of valid options and their values, use \"$ScriptName -h\" or\n\"perl -S $ScriptName -h\" command and try again...\n";
     381   }
     382   if ($Options{workingdir}) {
     383     if (! -d $Options{workingdir}) {
     384       die "Error: The value specified, $Options{workingdir}, for option \"-w --workingdir\" is not a directory name.\n";
     385     }
     386     chdir $Options{workingdir} or die "Error: Couldn't chdir $Options{workingdir}: $! \n";
     387   }
     388   if ($Options{mode} !~ /^(colnum|collabel)$/i) {
     389     die "Error: The value specified, $Options{mode}, for option \"-m --mode\" is not valid. Allowed values: colnum or collabel\n";
     390   }
     391   if ($Options{keydata} !~ /^(numeric|alphanumeric)$/i) {
     392     die "Error: The value specified, $Options{keydata}, for option \"--keydata\" is not valid. Allowed values: numeric or alphanumeric\n";
     393   }
     394   if ($Options{indelim} !~ /^(comma|semicolon)$/i) {
     395     die "Error: The value specified, $Options{indelim}, for option \"--indelim\" is not valid. Allowed values: comma or semicolon\n";
     396   }
     397   if ($Options{outdelim} !~ /^(comma|semicolon|tab)$/i) {
     398     die "Error: The value specified, $Options{outdelim}, for option \"--outdelim\" is not valid. Allowed values: comma, tab, or semicolon\n";
     399   }
     400   if ($Options{quote} !~ /^(yes|no)$/i) {
     401     die "Error: The value specified, $Options{quote}, for option \"-q --quote\" is not valid. Allowed values: yes or no\n";
     402   }
     403   if ($Options{sort} !~ /^(ascending|descending)$/i) {
     404     die "Error: The value specified, $Options{sort}, for option \"-s --sort\" is not valid. Allowed values: ascending or descending\n";
     405   }
     406   if (!IsPositiveInteger($Options{detail})) {
     407     die "Error: The value specified, $Options{detail}, for option \"-d --detail\" is not valid. Allowed values: > 0\n";
     408   }
     409 }
     410