comparison mayachemtools/docs/modules/man3/Lexer.3 @ 0:73ae111cf86f draft

Uploaded
author deepakjadmin
date Wed, 20 Jan 2016 11:55:01 -0500
parents
children
comparison
equal deleted inserted replaced
-1:000000000000 0:73ae111cf86f
1 .\" Automatically generated by Pod::Man 2.25 (Pod::Simple 3.22)
2 .\"
3 .\" Standard preamble:
4 .\" ========================================================================
5 .de Sp \" Vertical space (when we can't use .PP)
6 .if t .sp .5v
7 .if n .sp
8 ..
9 .de Vb \" Begin verbatim text
10 .ft CW
11 .nf
12 .ne \\$1
13 ..
14 .de Ve \" End verbatim text
15 .ft R
16 .fi
17 ..
18 .\" Set up some character translations and predefined strings. \*(-- will
19 .\" give an unbreakable dash, \*(PI will give pi, \*(L" will give a left
20 .\" double quote, and \*(R" will give a right double quote. \*(C+ will
21 .\" give a nicer C++. Capital omega is used to do unbreakable dashes and
22 .\" therefore won't be available. \*(C` and \*(C' expand to `' in nroff,
23 .\" nothing in troff, for use with C<>.
24 .tr \(*W-
25 .ds C+ C\v'-.1v'\h'-1p'\s-2+\h'-1p'+\s0\v'.1v'\h'-1p'
26 .ie n \{\
27 . ds -- \(*W-
28 . ds PI pi
29 . if (\n(.H=4u)&(1m=24u) .ds -- \(*W\h'-12u'\(*W\h'-12u'-\" diablo 10 pitch
30 . if (\n(.H=4u)&(1m=20u) .ds -- \(*W\h'-12u'\(*W\h'-8u'-\" diablo 12 pitch
31 . ds L" ""
32 . ds R" ""
33 . ds C` ""
34 . ds C' ""
35 'br\}
36 .el\{\
37 . ds -- \|\(em\|
38 . ds PI \(*p
39 . ds L" ``
40 . ds R" ''
41 'br\}
42 .\"
43 .\" Escape single quotes in literal strings from groff's Unicode transform.
44 .ie \n(.g .ds Aq \(aq
45 .el .ds Aq '
46 .\"
47 .\" If the F register is turned on, we'll generate index entries on stderr for
48 .\" titles (.TH), headers (.SH), subsections (.SS), items (.Ip), and index
49 .\" entries marked with X<> in POD. Of course, you'll have to process the
50 .\" output yourself in some meaningful fashion.
51 .ie \nF \{\
52 . de IX
53 . tm Index:\\$1\t\\n%\t"\\$2"
54 ..
55 . nr % 0
56 . rr F
57 .\}
58 .el \{\
59 . de IX
60 ..
61 .\}
62 .\"
63 .\" Accent mark definitions (@(#)ms.acc 1.5 88/02/08 SMI; from UCB 4.2).
64 .\" Fear. Run. Save yourself. No user-serviceable parts.
65 . \" fudge factors for nroff and troff
66 .if n \{\
67 . ds #H 0
68 . ds #V .8m
69 . ds #F .3m
70 . ds #[ \f1
71 . ds #] \fP
72 .\}
73 .if t \{\
74 . ds #H ((1u-(\\\\n(.fu%2u))*.13m)
75 . ds #V .6m
76 . ds #F 0
77 . ds #[ \&
78 . ds #] \&
79 .\}
80 . \" simple accents for nroff and troff
81 .if n \{\
82 . ds ' \&
83 . ds ` \&
84 . ds ^ \&
85 . ds , \&
86 . ds ~ ~
87 . ds /
88 .\}
89 .if t \{\
90 . ds ' \\k:\h'-(\\n(.wu*8/10-\*(#H)'\'\h"|\\n:u"
91 . ds ` \\k:\h'-(\\n(.wu*8/10-\*(#H)'\`\h'|\\n:u'
92 . ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'^\h'|\\n:u'
93 . ds , \\k:\h'-(\\n(.wu*8/10)',\h'|\\n:u'
94 . ds ~ \\k:\h'-(\\n(.wu-\*(#H-.1m)'~\h'|\\n:u'
95 . ds / \\k:\h'-(\\n(.wu*8/10-\*(#H)'\z\(sl\h'|\\n:u'
96 .\}
97 . \" troff and (daisy-wheel) nroff accents
98 .ds : \\k:\h'-(\\n(.wu*8/10-\*(#H+.1m+\*(#F)'\v'-\*(#V'\z.\h'.2m+\*(#F'.\h'|\\n:u'\v'\*(#V'
99 .ds 8 \h'\*(#H'\(*b\h'-\*(#H'
100 .ds o \\k:\h'-(\\n(.wu+\w'\(de'u-\*(#H)/2u'\v'-.3n'\*(#[\z\(de\v'.3n'\h'|\\n:u'\*(#]
101 .ds d- \h'\*(#H'\(pd\h'-\w'~'u'\v'-.25m'\f2\(hy\fP\v'.25m'\h'-\*(#H'
102 .ds D- D\\k:\h'-\w'D'u'\v'-.11m'\z\(hy\v'.11m'\h'|\\n:u'
103 .ds th \*(#[\v'.3m'\s+1I\s-1\v'-.3m'\h'-(\w'I'u*2/3)'\s-1o\s+1\*(#]
104 .ds Th \*(#[\s+2I\s-2\h'-\w'I'u*3/5'\v'-.3m'o\v'.3m'\*(#]
105 .ds ae a\h'-(\w'a'u*4/10)'e
106 .ds Ae A\h'-(\w'A'u*4/10)'E
107 . \" corrections for vroff
108 .if v .ds ~ \\k:\h'-(\\n(.wu*9/10-\*(#H)'\s-2\u~\d\s+2\h'|\\n:u'
109 .if v .ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'\v'-.4m'^\v'.4m'\h'|\\n:u'
110 . \" for low resolution devices (crt and lpr)
111 .if \n(.H>23 .if \n(.V>19 \
112 \{\
113 . ds : e
114 . ds 8 ss
115 . ds o a
116 . ds d- d\h'-1'\(ga
117 . ds D- D\h'-1'\(hy
118 . ds th \o'bp'
119 . ds Th \o'LP'
120 . ds ae ae
121 . ds Ae AE
122 .\}
123 .rm #[ #] #H #V #F C
124 .\" ========================================================================
125 .\"
126 .IX Title "LEXER 1"
127 .TH LEXER 1 "2015-03-29" "perl v5.14.2" "MayaChemTools"
128 .\" For nroff, turn off justification. Always turn off hyphenation; it makes
129 .\" way too many mistakes in technical documents.
130 .if n .ad l
131 .nh
132 .SH "NAME"
133 Parsers::Lexer
134 .SH "SYNOPSIS"
135 .IX Header "SYNOPSIS"
136 use Parsers::Lexer;
137 .PP
138 use Parsers::Lexer qw(:all);
139 .SH "DESCRIPTION"
140 .IX Header "DESCRIPTION"
141 \&\fBLexer\fR class provides the following methods:
142 .PP
143 new, GetLex, Lex, Next, Peek, StringifyLexer
144 .PP
145 The object oriented chained \fBLexer\fR is implemented based on examples available in
146 Higher-order Perl [ Ref 126 ] book by Mark J. Dominus. It is designed to be used
147 both in standalone mode or as a base class for \fBYYLexer\fR.
148 .PP
149 A chained lexer is created by generating a lexer for for the first specified token
150 specification using specified input and chaining it with other lexers generated for all
151 subsequent token specifications. The lexer generated for the first token specification
152 uses input iterator to retrieve any available input text; the subsequent chained lexeres
153 for rest of the token specifications use lexers generated for previous token
154 specifications to get next input, which might be unmatched input text or a reference
155 to an array containing token and matched text pair.
156 .SS "\s-1METHODS\s0"
157 .IX Subsection "METHODS"
158 .IP "\fBnew\fR" 4
159 .IX Item "new"
160 .Vb 1
161 \& $Lexer = new Parsers::Lexer($Input, @TokensSpec);
162 .Ve
163 .Sp
164 Using specified \fIInput\fR and \fITokensSpec\fR, \fBnew\fR method generates a new lexer
165 and returns a reference to newly created \fBLexer\fR object.
166 .Sp
167 Example:
168 .Sp
169 .Vb 10
170 \& # Tokens specifications supplied by the caller. It\*(Aqs an array containing references
171 \& # to arrays with each containing TokenLabel and TokenMatchRegex pair along with
172 \& # an option reference to code to be executed after a matched.
173 \& #
174 \& @LexerTokensSpec = (
175 \& [ \*(AqLETTER\*(Aq, qr/[a\-zA\-Z]/ ],
176 \& [ \*(AqNUMBER\*(Aq, qr/\ed+/ ],
177 \& [ \*(AqSPACE\*(Aq, qr/[ ]*/,
178 \& sub { my($This, $TokenLabel, $MatchedText) = @_; return \*(Aq\*(Aq; }
179 \& ],
180 \& [ \*(AqNEWLINE\*(Aq, qr/(?:\er\en|\er|\en)/,
181 \& sub { my($This, $TokenLabel, $MatchedText) = @_; return "\en"; }
182 \& ],
183 \& [ \*(AqCHAR\*(Aq, qr/./ ]
184 \& );
185 \&
186 \& # Input string...
187 \& $InputText = \*(Aqy = 3 + 4\*(Aq;
188 \& $Lexer = new Parsers::Lexer($InputText, @LexerTokensSpec);
189 \&
190 \& # Process input stream...
191 \& while (defined($Token = $Lexer\->Lex())) {
192 \& print "Token: " . ((ref $Token) ? "@{$Token}" : "$Token") . "\en";
193 \& }
194 \&
195 \& # Input file...
196 \& $InputFile = "Input.txt";
197 \& open INPUTFILE, "$InputFile" or die "Couldn\*(Aqt open $InputFile: $!\en";
198 \& $Lexer = new Parsers::Lexer(\e*INPUTFILE, @LexerTokensSpec);
199 \&
200 \& # Input file iterator...
201 \& $InputFile = "TestSimpleCalcParser.txt";
202 \& open INPUTFILE, "$InputFile" or die "Couldn\*(Aqt open $InputFile: $!\en";
203 \& $InputIterator = sub { return <INPUTFILE>; };
204 \& $Lexer = new Parsers::Lexer($InputIterator, @LexerTokensSpec);
205 \&
206 \& @LexerTokensSpec = (
207 \& [ \*(AqVAR\*(Aq, qr/[[:alpha:]]+/ ],
208 \& [ \*(AqNUM\*(Aq, qr/\ed+/ ],
209 \& [ \*(AqOP\*(Aq, qr/[\-+=\e/]/,
210 \& sub { my($This, $Label, $Value) = @_;
211 \& $Value .= "; ord: " . ord $Value;
212 \& return [$Label, $Value];
213 \& }
214 \& ],
215 \& [ \*(AqNEWLINE\*(Aq, qr/(?:\er\en|\er|\en)/, sub { return [$_[1], \*(AqNewLine\*(Aq]; } ],
216 \& [ \*(AqSPACE\*(Aq, qr/\es*/, sub { return [$_[1], \*(AqSpace\*(Aq]; } ],
217 \& );
218 \&
219 \& # Look ahead without removing...
220 \& $Token = $Lexer\->Lex(\*(AqPeek\*(Aq);
221 \& if (defined $Token && ref $Token) {
222 \& print "PEEK: Token: @{$Token}\en\en";
223 \& }
224 \&
225 \& # Process input stream...
226 \& while (defined($Token = $Lexer\->Lex())) {
227 \& print "Token: " . ((ref $Token) ? "@{$Token}" : "$Token") . "\en";
228 \& }
229 .Ve
230 .IP "\fBGetLex\fR" 4
231 .IX Item "GetLex"
232 .Vb 1
233 \& $LexerRef = $Lexer\->GetLex();
234 .Ve
235 .Sp
236 Returns a refernece to \fILexer\fR method to the caller for use in a specific \fBYYLexer\fR.
237 .IP "\fBLex\fR" 4
238 .IX Item "Lex"
239 .Vb 7
240 \& $TokenRefOrText = $Lexer\->Lex($Mode);
241 \& if (ref $TokenRefOrText) {
242 \& ($TokenLabel, $TokenValue) = @{$TokenRefOrText};
243 \& }
244 \& else {
245 \& $TokenText = $TokenRefOrText;
246 \& }
247 .Ve
248 .Sp
249 Get next available token label and value pair as an array reference or unrecognized
250 text from input stream by either removing it from the input or simply peeking ahead
251 and without removing it from the input stream.
252 .Sp
253 Possible \fIMode\fR values: \fIPeek, Next\fR. Default: \fINext\fR.
254 .IP "\fBNext\fR" 4
255 .IX Item "Next"
256 .Vb 1
257 \& $TokenRefOrText = $Lexer\->Next();
258 .Ve
259 .Sp
260 Get next available token label and value pair as an array reference or unrecognized
261 text from input stream by removing it from the input stream.
262 .IP "\fBPeek\fR" 4
263 .IX Item "Peek"
264 .Vb 1
265 \& $TokenRefOrText = $Lexer\->Peek();
266 .Ve
267 .Sp
268 Get next available token label and value pair as an array reference or unrecognized
269 text from input stream by by simply peeking ahead and without removing it from the
270 input stream.
271 .IP "\fBStringifyLexer\fR" 4
272 .IX Item "StringifyLexer"
273 .Vb 1
274 \& $LexerString = $Lexer\->StringifyLexer();
275 .Ve
276 .Sp
277 Returns a string containing information about \fILexer\fR object.
278 .SH "AUTHOR"
279 .IX Header "AUTHOR"
280 Manish Sud <msud@san.rr.com>
281 .SH "SEE ALSO"
282 .IX Header "SEE ALSO"
283 YYLexer.pm, SimpleCalcYYLexer.pm, SimpleCalcParser.yy
284 .SH "COPYRIGHT"
285 .IX Header "COPYRIGHT"
286 Copyright (C) 2015 Manish Sud. All rights reserved.
287 .PP
288 This file is part of MayaChemTools.
289 .PP
290 MayaChemTools is free software; you can redistribute it and/or modify it under
291 the terms of the \s-1GNU\s0 Lesser General Public License as published by the Free
292 Software Foundation; either version 3 of the License, or (at your option)
293 any later version.