diff mayachemtools/docs/modules/man3/Lexer.3 @ 0:73ae111cf86f draft

Uploaded

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/mayachemtools/docs/modules/man3/Lexer.3	Wed Jan 20 11:55:01 2016 -0500
@@ -0,0 +1,293 @@
+.\" Automatically generated by Pod::Man 2.25 (Pod::Simple 3.22)
+.\"
+.\" Standard preamble:
+.\" ========================================================================
+.de Sp \" Vertical space (when we can't use .PP)
+.if t .sp .5v
+.if n .sp
+..
+.de Vb \" Begin verbatim text
+.ft CW
+.nf
+.ne \\$1
+..
+.de Ve \" End verbatim text
+.ft R
+.fi
+..
+.\" Set up some character translations and predefined strings.  \*(-- will
+.\" give an unbreakable dash, \*(PI will give pi, \*(L" will give a left
+.\" double quote, and \*(R" will give a right double quote.  \*(C+ will
+.\" give a nicer C++.  Capital omega is used to do unbreakable dashes and
+.\" therefore won't be available.  \*(C` and \*(C' expand to `' in nroff,
+.\" nothing in troff, for use with C<>.
+.tr \(*W-
+.ds C+ C\v'-.1v'\h'-1p'\s-2+\h'-1p'+\s0\v'.1v'\h'-1p'
+.ie n \{\
+.    ds -- \(*W-
+.    ds PI pi
+.    if (\n(.H=4u)&(1m=24u) .ds -- \(*W\h'-12u'\(*W\h'-12u'-\" diablo 10 pitch
+.    if (\n(.H=4u)&(1m=20u) .ds -- \(*W\h'-12u'\(*W\h'-8u'-\"  diablo 12 pitch
+.    ds L" ""
+.    ds R" ""
+.    ds C` ""
+.    ds C' ""
+'br\}
+.el\{\
+.    ds -- \|\(em\|
+.    ds PI \(*p
+.    ds L" ``
+.    ds R" ''
+'br\}
+.\"
+.\" Escape single quotes in literal strings from groff's Unicode transform.
+.ie \n(.g .ds Aq \(aq
+.el       .ds Aq '
+.\"
+.\" If the F register is turned on, we'll generate index entries on stderr for
+.\" titles (.TH), headers (.SH), subsections (.SS), items (.Ip), and index
+.\" entries marked with X<> in POD.  Of course, you'll have to process the
+.\" output yourself in some meaningful fashion.
+.ie \nF \{\
+.    de IX
+.    tm Index:\\$1\t\\n%\t"\\$2"
+..
+.    nr % 0
+.    rr F
+.\}
+.el \{\
+.    de IX
+..
+.\}
+.\"
+.\" Accent mark definitions (@(#)ms.acc 1.5 88/02/08 SMI; from UCB 4.2).
+.\" Fear.  Run.  Save yourself.  No user-serviceable parts.
+.    \" fudge factors for nroff and troff
+.if n \{\
+.    ds #H 0
+.    ds #V .8m
+.    ds #F .3m
+.    ds #[ \f1
+.    ds #] \fP
+.\}
+.if t \{\
+.    ds #H ((1u-(\\\\n(.fu%2u))*.13m)
+.    ds #V .6m
+.    ds #F 0
+.    ds #[ \&
+.    ds #] \&
+.\}
+.    \" simple accents for nroff and troff
+.if n \{\
+.    ds ' \&
+.    ds ` \&
+.    ds ^ \&
+.    ds , \&
+.    ds ~ ~
+.    ds /
+.\}
+.if t \{\
+.    ds ' \\k:\h'-(\\n(.wu*8/10-\*(#H)'\'\h"|\\n:u"
+.    ds ` \\k:\h'-(\\n(.wu*8/10-\*(#H)'\`\h'|\\n:u'
+.    ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'^\h'|\\n:u'
+.    ds , \\k:\h'-(\\n(.wu*8/10)',\h'|\\n:u'
+.    ds ~ \\k:\h'-(\\n(.wu-\*(#H-.1m)'~\h'|\\n:u'
+.    ds / \\k:\h'-(\\n(.wu*8/10-\*(#H)'\z\(sl\h'|\\n:u'
+.\}
+.    \" troff and (daisy-wheel) nroff accents
+.ds : \\k:\h'-(\\n(.wu*8/10-\*(#H+.1m+\*(#F)'\v'-\*(#V'\z.\h'.2m+\*(#F'.\h'|\\n:u'\v'\*(#V'
+.ds 8 \h'\*(#H'\(*b\h'-\*(#H'
+.ds o \\k:\h'-(\\n(.wu+\w'\(de'u-\*(#H)/2u'\v'-.3n'\*(#[\z\(de\v'.3n'\h'|\\n:u'\*(#]
+.ds d- \h'\*(#H'\(pd\h'-\w'~'u'\v'-.25m'\f2\(hy\fP\v'.25m'\h'-\*(#H'
+.ds D- D\\k:\h'-\w'D'u'\v'-.11m'\z\(hy\v'.11m'\h'|\\n:u'
+.ds th \*(#[\v'.3m'\s+1I\s-1\v'-.3m'\h'-(\w'I'u*2/3)'\s-1o\s+1\*(#]
+.ds Th \*(#[\s+2I\s-2\h'-\w'I'u*3/5'\v'-.3m'o\v'.3m'\*(#]
+.ds ae a\h'-(\w'a'u*4/10)'e
+.ds Ae A\h'-(\w'A'u*4/10)'E
+.    \" corrections for vroff
+.if v .ds ~ \\k:\h'-(\\n(.wu*9/10-\*(#H)'\s-2\u~\d\s+2\h'|\\n:u'
+.if v .ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'\v'-.4m'^\v'.4m'\h'|\\n:u'
+.    \" for low resolution devices (crt and lpr)
+.if \n(.H>23 .if \n(.V>19 \
+\{\
+.    ds : e
+.    ds 8 ss
+.    ds o a
+.    ds d- d\h'-1'\(ga
+.    ds D- D\h'-1'\(hy
+.    ds th \o'bp'
+.    ds Th \o'LP'
+.    ds ae ae
+.    ds Ae AE
+.\}
+.rm #[ #] #H #V #F C
+.\" ========================================================================
+.\"
+.IX Title "LEXER 1"
+.TH LEXER 1 "2015-03-29" "perl v5.14.2" "MayaChemTools"
+.\" For nroff, turn off justification.  Always turn off hyphenation; it makes
+.\" way too many mistakes in technical documents.
+.if n .ad l
+.nh
+.SH "NAME"
+Parsers::Lexer
+.SH "SYNOPSIS"
+.IX Header "SYNOPSIS"
+use Parsers::Lexer;
+.PP
+use Parsers::Lexer qw(:all);
+.SH "DESCRIPTION"
+.IX Header "DESCRIPTION"
+\&\fBLexer\fR class provides the following methods:
+.PP
+new, GetLex, Lex, Next, Peek, StringifyLexer
+.PP
+The object oriented chained \fBLexer\fR is implemented based on examples available in
+Higher-order Perl [ Ref 126 ] book by Mark J. Dominus. It is designed to be used
+both in standalone mode or as a base class for \fBYYLexer\fR.
+.PP
+A chained lexer is created by generating a lexer for for the first specified token
+specification using specified input and chaining it with other lexers generated for all
+subsequent token specifications. The lexer generated for the first token specification
+uses input iterator to retrieve any available input text; the subsequent chained lexeres
+for rest of the token specifications use lexers generated for previous token
+specifications to get next input, which might be unmatched input text or a reference
+to an array containing token and  matched text pair.
+.SS "\s-1METHODS\s0"
+.IX Subsection "METHODS"
+.IP "\fBnew\fR" 4
+.IX Item "new"
+.Vb 1
+\&    $Lexer = new Parsers::Lexer($Input, @TokensSpec);
+.Ve
+.Sp
+Using specified \fIInput\fR and \fITokensSpec\fR, \fBnew\fR method generates a new lexer
+and returns a reference to newly created \fBLexer\fR object.
+.Sp
+Example:
+.Sp
+.Vb 10
+\&    # Tokens specifications supplied by the caller. It\*(Aqs an array containing references
+\&    # to arrays with each containing TokenLabel and TokenMatchRegex pair along with
+\&    # an option reference to code to be executed after a matched.
+\&    #
+\&    @LexerTokensSpec = (
+\&        [ \*(AqLETTER\*(Aq, qr/[a\-zA\-Z]/ ],
+\&        [ \*(AqNUMBER\*(Aq, qr/\ed+/ ],
+\&        [ \*(AqSPACE\*(Aq, qr/[ ]*/,
+\&            sub { my($This, $TokenLabel, $MatchedText) = @_; return \*(Aq\*(Aq; }
+\&        ],
+\&        [ \*(AqNEWLINE\*(Aq, qr/(?:\er\en|\er|\en)/,
+\&            sub { my($This, $TokenLabel, $MatchedText) = @_;  return "\en"; }
+\&        ],
+\&        [ \*(AqCHAR\*(Aq, qr/./ ]
+\&    );
+\&
+\&    # Input string...
+\&    $InputText = \*(Aqy = 3 + 4\*(Aq;
+\&    $Lexer = new Parsers::Lexer($InputText, @LexerTokensSpec);
+\&
+\&    # Process input stream...
+\&    while (defined($Token = $Lexer\->Lex())) {
+\&        print "Token: " . ((ref $Token) ? "@{$Token}" : "$Token") . "\en";
+\&    }
+\&
+\&    # Input file...
+\&    $InputFile = "Input.txt";
+\&    open INPUTFILE, "$InputFile" or die "Couldn\*(Aqt open $InputFile: $!\en";
+\&    $Lexer = new Parsers::Lexer(\e*INPUTFILE, @LexerTokensSpec);
+\&
+\&    # Input file iterator...
+\&    $InputFile = "TestSimpleCalcParser.txt";
+\&    open INPUTFILE, "$InputFile" or die "Couldn\*(Aqt open $InputFile: $!\en";
+\&    $InputIterator = sub { return <INPUTFILE>; };
+\&    $Lexer = new Parsers::Lexer($InputIterator, @LexerTokensSpec);
+\&
+\&    @LexerTokensSpec = (
+\&        [ \*(AqVAR\*(Aq, qr/[[:alpha:]]+/ ],
+\&        [ \*(AqNUM\*(Aq, qr/\ed+/ ],
+\&        [ \*(AqOP\*(Aq, qr/[\-+=\e/]/,
+\&            sub { my($This, $Label, $Value) = @_;
+\&                $Value .= "; ord: " . ord $Value;
+\&                return [$Label, $Value];
+\&            }
+\&        ],
+\&        [ \*(AqNEWLINE\*(Aq, qr/(?:\er\en|\er|\en)/, sub { return [$_[1], \*(AqNewLine\*(Aq]; } ],
+\&        [ \*(AqSPACE\*(Aq, qr/\es*/, sub { return [$_[1], \*(AqSpace\*(Aq]; } ],
+\&    );
+\&
+\&    # Look ahead without removing...
+\&    $Token = $Lexer\->Lex(\*(AqPeek\*(Aq);
+\&    if (defined $Token && ref $Token) {
+\&        print "PEEK: Token: @{$Token}\en\en";
+\&    }
+\&
+\&    # Process input stream...
+\&    while (defined($Token = $Lexer\->Lex())) {
+\&        print "Token: " . ((ref $Token) ? "@{$Token}" : "$Token") . "\en";
+\&    }
+.Ve
+.IP "\fBGetLex\fR" 4
+.IX Item "GetLex"
+.Vb 1
+\&    $LexerRef = $Lexer\->GetLex();
+.Ve
+.Sp
+Returns a refernece to \fILexer\fR method to the caller for use in a specific \fBYYLexer\fR.
+.IP "\fBLex\fR" 4
+.IX Item "Lex"
+.Vb 7
+\&    $TokenRefOrText = $Lexer\->Lex($Mode);
+\&    if (ref $TokenRefOrText) {
+\&        ($TokenLabel, $TokenValue) = @{$TokenRefOrText};
+\&    }
+\&    else {
+\&        $TokenText = $TokenRefOrText;
+\&    }
+.Ve
+.Sp
+Get next available token label and value pair as an array reference or unrecognized
+text from input stream by either removing it from the input or simply peeking ahead
+and without removing it from the input stream.
+.Sp
+Possible \fIMode\fR values: \fIPeek, Next\fR. Default: \fINext\fR.
+.IP "\fBNext\fR" 4
+.IX Item "Next"
+.Vb 1
+\&    $TokenRefOrText = $Lexer\->Next();
+.Ve
+.Sp
+Get next available token label and value pair as an array reference or unrecognized
+text from input stream by removing it from the input stream.
+.IP "\fBPeek\fR" 4
+.IX Item "Peek"
+.Vb 1
+\&    $TokenRefOrText = $Lexer\->Peek();
+.Ve
+.Sp
+Get next available token label and value pair as an array reference or unrecognized
+text from input stream by by simply peeking ahead and without removing it from the
+input stream.
+.IP "\fBStringifyLexer\fR" 4
+.IX Item "StringifyLexer"
+.Vb 1
+\&    $LexerString = $Lexer\->StringifyLexer();
+.Ve
+.Sp
+Returns a string containing information about \fILexer\fR object.
+.SH "AUTHOR"
+.IX Header "AUTHOR"
+Manish Sud <[e-mail]>
+.SH "SEE ALSO"
+.IX Header "SEE ALSO"
+YYLexer.pm, SimpleCalcYYLexer.pm, SimpleCalcParser.yy
+.SH "COPYRIGHT"
+.IX Header "COPYRIGHT"
+Copyright (C) 2015 Manish Sud. All rights reserved.
+.PP
+This file is part of MayaChemTools.
+.PP
+MayaChemTools is free software; you can redistribute it and/or modify it under
+the terms of the \s-1GNU\s0 Lesser General Public License as published by the Free
+Software Foundation; either version 3 of the License, or (at your option)
+any later version.