view mayachemtools/docs/modules/man3/Lexer.3 @ 0:73ae111cf86f draft

Uploaded

.\" Automatically generated by Pod::Man 2.25 (Pod::Simple 3.22)
.\"
.\" Standard preamble:
.\" ========================================================================
.de Sp \" Vertical space (when we can't use .PP)
.if t .sp .5v
.if n .sp
..
.de Vb \" Begin verbatim text
.ft CW
.nf
.ne \\$1
..
.de Ve \" End verbatim text
.ft R
.fi
..
.\" Set up some character translations and predefined strings.  \*(-- will
.\" give an unbreakable dash, \*(PI will give pi, \*(L" will give a left
.\" double quote, and \*(R" will give a right double quote.  \*(C+ will
.\" give a nicer C++.  Capital omega is used to do unbreakable dashes and
.\" therefore won't be available.  \*(C` and \*(C' expand to `' in nroff,
.\" nothing in troff, for use with C<>.
.tr \(*W-
.ds C+ C\v'-.1v'\h'-1p'\s-2+\h'-1p'+\s0\v'.1v'\h'-1p'
.ie n \{\
.    ds -- \(*W-
.    ds PI pi
.    if (\n(.H=4u)&(1m=24u) .ds -- \(*W\h'-12u'\(*W\h'-12u'-\" diablo 10 pitch
.    if (\n(.H=4u)&(1m=20u) .ds -- \(*W\h'-12u'\(*W\h'-8u'-\"  diablo 12 pitch
.    ds L" ""
.    ds R" ""
.    ds C` ""
.    ds C' ""
'br\}
.el\{\
.    ds -- \|\(em\|
.    ds PI \(*p
.    ds L" ``
.    ds R" ''
'br\}
.\"
.\" Escape single quotes in literal strings from groff's Unicode transform.
.ie \n(.g .ds Aq \(aq
.el       .ds Aq '
.\"
.\" If the F register is turned on, we'll generate index entries on stderr for
.\" titles (.TH), headers (.SH), subsections (.SS), items (.Ip), and index
.\" entries marked with X<> in POD.  Of course, you'll have to process the
.\" output yourself in some meaningful fashion.
.ie \nF \{\
.    de IX
.    tm Index:\\$1\t\\n%\t"\\$2"
..
.    nr % 0
.    rr F
.\}
.el \{\
.    de IX
..
.\}
.\"
.\" Accent mark definitions (@(#)ms.acc 1.5 88/02/08 SMI; from UCB 4.2).
.\" Fear.  Run.  Save yourself.  No user-serviceable parts.
.    \" fudge factors for nroff and troff
.if n \{\
.    ds #H 0
.    ds #V .8m
.    ds #F .3m
.    ds #[ \f1
.    ds #] \fP
.\}
.if t \{\
.    ds #H ((1u-(\\\\n(.fu%2u))*.13m)
.    ds #V .6m
.    ds #F 0
.    ds #[ \&
.    ds #] \&
.\}
.    \" simple accents for nroff and troff
.if n \{\
.    ds ' \&
.    ds ` \&
.    ds ^ \&
.    ds , \&
.    ds ~ ~
.    ds /
.\}
.if t \{\
.    ds ' \\k:\h'-(\\n(.wu*8/10-\*(#H)'\'\h"|\\n:u"
.    ds ` \\k:\h'-(\\n(.wu*8/10-\*(#H)'\`\h'|\\n:u'
.    ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'^\h'|\\n:u'
.    ds , \\k:\h'-(\\n(.wu*8/10)',\h'|\\n:u'
.    ds ~ \\k:\h'-(\\n(.wu-\*(#H-.1m)'~\h'|\\n:u'
.    ds / \\k:\h'-(\\n(.wu*8/10-\*(#H)'\z\(sl\h'|\\n:u'
.\}
.    \" troff and (daisy-wheel) nroff accents
.ds : \\k:\h'-(\\n(.wu*8/10-\*(#H+.1m+\*(#F)'\v'-\*(#V'\z.\h'.2m+\*(#F'.\h'|\\n:u'\v'\*(#V'
.ds 8 \h'\*(#H'\(*b\h'-\*(#H'
.ds o \\k:\h'-(\\n(.wu+\w'\(de'u-\*(#H)/2u'\v'-.3n'\*(#[\z\(de\v'.3n'\h'|\\n:u'\*(#]
.ds d- \h'\*(#H'\(pd\h'-\w'~'u'\v'-.25m'\f2\(hy\fP\v'.25m'\h'-\*(#H'
.ds D- D\\k:\h'-\w'D'u'\v'-.11m'\z\(hy\v'.11m'\h'|\\n:u'
.ds th \*(#[\v'.3m'\s+1I\s-1\v'-.3m'\h'-(\w'I'u*2/3)'\s-1o\s+1\*(#]
.ds Th \*(#[\s+2I\s-2\h'-\w'I'u*3/5'\v'-.3m'o\v'.3m'\*(#]
.ds ae a\h'-(\w'a'u*4/10)'e
.ds Ae A\h'-(\w'A'u*4/10)'E
.    \" corrections for vroff
.if v .ds ~ \\k:\h'-(\\n(.wu*9/10-\*(#H)'\s-2\u~\d\s+2\h'|\\n:u'
.if v .ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'\v'-.4m'^\v'.4m'\h'|\\n:u'
.    \" for low resolution devices (crt and lpr)
.if \n(.H>23 .if \n(.V>19 \
\{\
.    ds : e
.    ds 8 ss
.    ds o a
.    ds d- d\h'-1'\(ga
.    ds D- D\h'-1'\(hy
.    ds th \o'bp'
.    ds Th \o'LP'
.    ds ae ae
.    ds Ae AE
.\}
.rm #[ #] #H #V #F C
.\" ========================================================================
.\"
.IX Title "LEXER 1"
.TH LEXER 1 "2015-03-29" "perl v5.14.2" "MayaChemTools"
.\" For nroff, turn off justification.  Always turn off hyphenation; it makes
.\" way too many mistakes in technical documents.
.if n .ad l
.nh
.SH "NAME"
Parsers::Lexer
.SH "SYNOPSIS"
.IX Header "SYNOPSIS"
use Parsers::Lexer;
.PP
use Parsers::Lexer qw(:all);
.SH "DESCRIPTION"
.IX Header "DESCRIPTION"
\&\fBLexer\fR class provides the following methods:
.PP
new, GetLex, Lex, Next, Peek, StringifyLexer
.PP
The object oriented chained \fBLexer\fR is implemented based on examples available in
Higher-order Perl [ Ref 126 ] book by Mark J. Dominus. It is designed to be used
both in standalone mode or as a base class for \fBYYLexer\fR.
.PP
A chained lexer is created by generating a lexer for for the first specified token
specification using specified input and chaining it with other lexers generated for all
subsequent token specifications. The lexer generated for the first token specification
uses input iterator to retrieve any available input text; the subsequent chained lexeres
for rest of the token specifications use lexers generated for previous token
specifications to get next input, which might be unmatched input text or a reference
to an array containing token and  matched text pair.
.SS "\s-1METHODS\s0"
.IX Subsection "METHODS"
.IP "\fBnew\fR" 4
.IX Item "new"
.Vb 1
\&    $Lexer = new Parsers::Lexer($Input, @TokensSpec);
.Ve
.Sp
Using specified \fIInput\fR and \fITokensSpec\fR, \fBnew\fR method generates a new lexer
and returns a reference to newly created \fBLexer\fR object.
.Sp
Example:
.Sp
.Vb 10
\&    # Tokens specifications supplied by the caller. It\*(Aqs an array containing references
\&    # to arrays with each containing TokenLabel and TokenMatchRegex pair along with
\&    # an option reference to code to be executed after a matched.
\&    #
\&    @LexerTokensSpec = (
\&        [ \*(AqLETTER\*(Aq, qr/[a\-zA\-Z]/ ],
\&        [ \*(AqNUMBER\*(Aq, qr/\ed+/ ],
\&        [ \*(AqSPACE\*(Aq, qr/[ ]*/,
\&            sub { my($This, $TokenLabel, $MatchedText) = @_; return \*(Aq\*(Aq; }
\&        ],
\&        [ \*(AqNEWLINE\*(Aq, qr/(?:\er\en|\er|\en)/,
\&            sub { my($This, $TokenLabel, $MatchedText) = @_;  return "\en"; }
\&        ],
\&        [ \*(AqCHAR\*(Aq, qr/./ ]
\&    );
\&
\&    # Input string...
\&    $InputText = \*(Aqy = 3 + 4\*(Aq;
\&    $Lexer = new Parsers::Lexer($InputText, @LexerTokensSpec);
\&
\&    # Process input stream...
\&    while (defined($Token = $Lexer\->Lex())) {
\&        print "Token: " . ((ref $Token) ? "@{$Token}" : "$Token") . "\en";
\&    }
\&
\&    # Input file...
\&    $InputFile = "Input.txt";
\&    open INPUTFILE, "$InputFile" or die "Couldn\*(Aqt open $InputFile: $!\en";
\&    $Lexer = new Parsers::Lexer(\e*INPUTFILE, @LexerTokensSpec);
\&
\&    # Input file iterator...
\&    $InputFile = "TestSimpleCalcParser.txt";
\&    open INPUTFILE, "$InputFile" or die "Couldn\*(Aqt open $InputFile: $!\en";
\&    $InputIterator = sub { return <INPUTFILE>; };
\&    $Lexer = new Parsers::Lexer($InputIterator, @LexerTokensSpec);
\&
\&    @LexerTokensSpec = (
\&        [ \*(AqVAR\*(Aq, qr/[[:alpha:]]+/ ],
\&        [ \*(AqNUM\*(Aq, qr/\ed+/ ],
\&        [ \*(AqOP\*(Aq, qr/[\-+=\e/]/,
\&            sub { my($This, $Label, $Value) = @_;
\&                $Value .= "; ord: " . ord $Value;
\&                return [$Label, $Value];
\&            }
\&        ],
\&        [ \*(AqNEWLINE\*(Aq, qr/(?:\er\en|\er|\en)/, sub { return [$_[1], \*(AqNewLine\*(Aq]; } ],
\&        [ \*(AqSPACE\*(Aq, qr/\es*/, sub { return [$_[1], \*(AqSpace\*(Aq]; } ],
\&    );
\&
\&    # Look ahead without removing...
\&    $Token = $Lexer\->Lex(\*(AqPeek\*(Aq);
\&    if (defined $Token && ref $Token) {
\&        print "PEEK: Token: @{$Token}\en\en";
\&    }
\&
\&    # Process input stream...
\&    while (defined($Token = $Lexer\->Lex())) {
\&        print "Token: " . ((ref $Token) ? "@{$Token}" : "$Token") . "\en";
\&    }
.Ve
.IP "\fBGetLex\fR" 4
.IX Item "GetLex"
.Vb 1
\&    $LexerRef = $Lexer\->GetLex();
.Ve
.Sp
Returns a refernece to \fILexer\fR method to the caller for use in a specific \fBYYLexer\fR.
.IP "\fBLex\fR" 4
.IX Item "Lex"
.Vb 7
\&    $TokenRefOrText = $Lexer\->Lex($Mode);
\&    if (ref $TokenRefOrText) {
\&        ($TokenLabel, $TokenValue) = @{$TokenRefOrText};
\&    }
\&    else {
\&        $TokenText = $TokenRefOrText;
\&    }
.Ve
.Sp
Get next available token label and value pair as an array reference or unrecognized
text from input stream by either removing it from the input or simply peeking ahead
and without removing it from the input stream.
.Sp
Possible \fIMode\fR values: \fIPeek, Next\fR. Default: \fINext\fR.
.IP "\fBNext\fR" 4
.IX Item "Next"
.Vb 1
\&    $TokenRefOrText = $Lexer\->Next();
.Ve
.Sp
Get next available token label and value pair as an array reference or unrecognized
text from input stream by removing it from the input stream.
.IP "\fBPeek\fR" 4
.IX Item "Peek"
.Vb 1
\&    $TokenRefOrText = $Lexer\->Peek();
.Ve
.Sp
Get next available token label and value pair as an array reference or unrecognized
text from input stream by by simply peeking ahead and without removing it from the
input stream.
.IP "\fBStringifyLexer\fR" 4
.IX Item "StringifyLexer"
.Vb 1
\&    $LexerString = $Lexer\->StringifyLexer();
.Ve
.Sp
Returns a string containing information about \fILexer\fR object.
.SH "AUTHOR"
.IX Header "AUTHOR"
Manish Sud <[e-mail]>
.SH "SEE ALSO"
.IX Header "SEE ALSO"
YYLexer.pm, SimpleCalcYYLexer.pm, SimpleCalcParser.yy
.SH "COPYRIGHT"
.IX Header "COPYRIGHT"
Copyright (C) 2015 Manish Sud. All rights reserved.
.PP
This file is part of MayaChemTools.
.PP
MayaChemTools is free software; you can redistribute it and/or modify it under
the terms of the \s-1GNU\s0 Lesser General Public License as published by the Free
Software Foundation; either version 3 of the License, or (at your option)
any later version.