|
0
|
1 .\" Automatically generated by Pod::Man 2.25 (Pod::Simple 3.22)
|
|
|
2 .\"
|
|
|
3 .\" Standard preamble:
|
|
|
4 .\" ========================================================================
|
|
|
5 .de Sp \" Vertical space (when we can't use .PP)
|
|
|
6 .if t .sp .5v
|
|
|
7 .if n .sp
|
|
|
8 ..
|
|
|
9 .de Vb \" Begin verbatim text
|
|
|
10 .ft CW
|
|
|
11 .nf
|
|
|
12 .ne \\$1
|
|
|
13 ..
|
|
|
14 .de Ve \" End verbatim text
|
|
|
15 .ft R
|
|
|
16 .fi
|
|
|
17 ..
|
|
|
18 .\" Set up some character translations and predefined strings. \*(-- will
|
|
|
19 .\" give an unbreakable dash, \*(PI will give pi, \*(L" will give a left
|
|
|
20 .\" double quote, and \*(R" will give a right double quote. \*(C+ will
|
|
|
21 .\" give a nicer C++. Capital omega is used to do unbreakable dashes and
|
|
|
22 .\" therefore won't be available. \*(C` and \*(C' expand to `' in nroff,
|
|
|
23 .\" nothing in troff, for use with C<>.
|
|
|
24 .tr \(*W-
|
|
|
25 .ds C+ C\v'-.1v'\h'-1p'\s-2+\h'-1p'+\s0\v'.1v'\h'-1p'
|
|
|
26 .ie n \{\
|
|
|
27 . ds -- \(*W-
|
|
|
28 . ds PI pi
|
|
|
29 . if (\n(.H=4u)&(1m=24u) .ds -- \(*W\h'-12u'\(*W\h'-12u'-\" diablo 10 pitch
|
|
|
30 . if (\n(.H=4u)&(1m=20u) .ds -- \(*W\h'-12u'\(*W\h'-8u'-\" diablo 12 pitch
|
|
|
31 . ds L" ""
|
|
|
32 . ds R" ""
|
|
|
33 . ds C` ""
|
|
|
34 . ds C' ""
|
|
|
35 'br\}
|
|
|
36 .el\{\
|
|
|
37 . ds -- \|\(em\|
|
|
|
38 . ds PI \(*p
|
|
|
39 . ds L" ``
|
|
|
40 . ds R" ''
|
|
|
41 'br\}
|
|
|
42 .\"
|
|
|
43 .\" Escape single quotes in literal strings from groff's Unicode transform.
|
|
|
44 .ie \n(.g .ds Aq \(aq
|
|
|
45 .el .ds Aq '
|
|
|
46 .\"
|
|
|
47 .\" If the F register is turned on, we'll generate index entries on stderr for
|
|
|
48 .\" titles (.TH), headers (.SH), subsections (.SS), items (.Ip), and index
|
|
|
49 .\" entries marked with X<> in POD. Of course, you'll have to process the
|
|
|
50 .\" output yourself in some meaningful fashion.
|
|
|
51 .ie \nF \{\
|
|
|
52 . de IX
|
|
|
53 . tm Index:\\$1\t\\n%\t"\\$2"
|
|
|
54 ..
|
|
|
55 . nr % 0
|
|
|
56 . rr F
|
|
|
57 .\}
|
|
|
58 .el \{\
|
|
|
59 . de IX
|
|
|
60 ..
|
|
|
61 .\}
|
|
|
62 .\"
|
|
|
63 .\" Accent mark definitions (@(#)ms.acc 1.5 88/02/08 SMI; from UCB 4.2).
|
|
|
64 .\" Fear. Run. Save yourself. No user-serviceable parts.
|
|
|
65 . \" fudge factors for nroff and troff
|
|
|
66 .if n \{\
|
|
|
67 . ds #H 0
|
|
|
68 . ds #V .8m
|
|
|
69 . ds #F .3m
|
|
|
70 . ds #[ \f1
|
|
|
71 . ds #] \fP
|
|
|
72 .\}
|
|
|
73 .if t \{\
|
|
|
74 . ds #H ((1u-(\\\\n(.fu%2u))*.13m)
|
|
|
75 . ds #V .6m
|
|
|
76 . ds #F 0
|
|
|
77 . ds #[ \&
|
|
|
78 . ds #] \&
|
|
|
79 .\}
|
|
|
80 . \" simple accents for nroff and troff
|
|
|
81 .if n \{\
|
|
|
82 . ds ' \&
|
|
|
83 . ds ` \&
|
|
|
84 . ds ^ \&
|
|
|
85 . ds , \&
|
|
|
86 . ds ~ ~
|
|
|
87 . ds /
|
|
|
88 .\}
|
|
|
89 .if t \{\
|
|
|
90 . ds ' \\k:\h'-(\\n(.wu*8/10-\*(#H)'\'\h"|\\n:u"
|
|
|
91 . ds ` \\k:\h'-(\\n(.wu*8/10-\*(#H)'\`\h'|\\n:u'
|
|
|
92 . ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'^\h'|\\n:u'
|
|
|
93 . ds , \\k:\h'-(\\n(.wu*8/10)',\h'|\\n:u'
|
|
|
94 . ds ~ \\k:\h'-(\\n(.wu-\*(#H-.1m)'~\h'|\\n:u'
|
|
|
95 . ds / \\k:\h'-(\\n(.wu*8/10-\*(#H)'\z\(sl\h'|\\n:u'
|
|
|
96 .\}
|
|
|
97 . \" troff and (daisy-wheel) nroff accents
|
|
|
98 .ds : \\k:\h'-(\\n(.wu*8/10-\*(#H+.1m+\*(#F)'\v'-\*(#V'\z.\h'.2m+\*(#F'.\h'|\\n:u'\v'\*(#V'
|
|
|
99 .ds 8 \h'\*(#H'\(*b\h'-\*(#H'
|
|
|
100 .ds o \\k:\h'-(\\n(.wu+\w'\(de'u-\*(#H)/2u'\v'-.3n'\*(#[\z\(de\v'.3n'\h'|\\n:u'\*(#]
|
|
|
101 .ds d- \h'\*(#H'\(pd\h'-\w'~'u'\v'-.25m'\f2\(hy\fP\v'.25m'\h'-\*(#H'
|
|
|
102 .ds D- D\\k:\h'-\w'D'u'\v'-.11m'\z\(hy\v'.11m'\h'|\\n:u'
|
|
|
103 .ds th \*(#[\v'.3m'\s+1I\s-1\v'-.3m'\h'-(\w'I'u*2/3)'\s-1o\s+1\*(#]
|
|
|
104 .ds Th \*(#[\s+2I\s-2\h'-\w'I'u*3/5'\v'-.3m'o\v'.3m'\*(#]
|
|
|
105 .ds ae a\h'-(\w'a'u*4/10)'e
|
|
|
106 .ds Ae A\h'-(\w'A'u*4/10)'E
|
|
|
107 . \" corrections for vroff
|
|
|
108 .if v .ds ~ \\k:\h'-(\\n(.wu*9/10-\*(#H)'\s-2\u~\d\s+2\h'|\\n:u'
|
|
|
109 .if v .ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'\v'-.4m'^\v'.4m'\h'|\\n:u'
|
|
|
110 . \" for low resolution devices (crt and lpr)
|
|
|
111 .if \n(.H>23 .if \n(.V>19 \
|
|
|
112 \{\
|
|
|
113 . ds : e
|
|
|
114 . ds 8 ss
|
|
|
115 . ds o a
|
|
|
116 . ds d- d\h'-1'\(ga
|
|
|
117 . ds D- D\h'-1'\(hy
|
|
|
118 . ds th \o'bp'
|
|
|
119 . ds Th \o'LP'
|
|
|
120 . ds ae ae
|
|
|
121 . ds Ae AE
|
|
|
122 .\}
|
|
|
123 .rm #[ #] #H #V #F C
|
|
|
124 .\" ========================================================================
|
|
|
125 .\"
|
|
|
126 .IX Title "LEXER 1"
|
|
|
127 .TH LEXER 1 "2015-03-29" "perl v5.14.2" "MayaChemTools"
|
|
|
128 .\" For nroff, turn off justification. Always turn off hyphenation; it makes
|
|
|
129 .\" way too many mistakes in technical documents.
|
|
|
130 .if n .ad l
|
|
|
131 .nh
|
|
|
132 .SH "NAME"
|
|
|
133 Parsers::Lexer
|
|
|
134 .SH "SYNOPSIS"
|
|
|
135 .IX Header "SYNOPSIS"
|
|
|
136 use Parsers::Lexer;
|
|
|
137 .PP
|
|
|
138 use Parsers::Lexer qw(:all);
|
|
|
139 .SH "DESCRIPTION"
|
|
|
140 .IX Header "DESCRIPTION"
|
|
|
141 \&\fBLexer\fR class provides the following methods:
|
|
|
142 .PP
|
|
|
143 new, GetLex, Lex, Next, Peek, StringifyLexer
|
|
|
144 .PP
|
|
|
145 The object oriented chained \fBLexer\fR is implemented based on examples available in
|
|
|
146 Higher-order Perl [ Ref 126 ] book by Mark J. Dominus. It is designed to be used
|
|
|
147 both in standalone mode or as a base class for \fBYYLexer\fR.
|
|
|
148 .PP
|
|
|
149 A chained lexer is created by generating a lexer for for the first specified token
|
|
|
150 specification using specified input and chaining it with other lexers generated for all
|
|
|
151 subsequent token specifications. The lexer generated for the first token specification
|
|
|
152 uses input iterator to retrieve any available input text; the subsequent chained lexeres
|
|
|
153 for rest of the token specifications use lexers generated for previous token
|
|
|
154 specifications to get next input, which might be unmatched input text or a reference
|
|
|
155 to an array containing token and matched text pair.
|
|
|
156 .SS "\s-1METHODS\s0"
|
|
|
157 .IX Subsection "METHODS"
|
|
|
158 .IP "\fBnew\fR" 4
|
|
|
159 .IX Item "new"
|
|
|
160 .Vb 1
|
|
|
161 \& $Lexer = new Parsers::Lexer($Input, @TokensSpec);
|
|
|
162 .Ve
|
|
|
163 .Sp
|
|
|
164 Using specified \fIInput\fR and \fITokensSpec\fR, \fBnew\fR method generates a new lexer
|
|
|
165 and returns a reference to newly created \fBLexer\fR object.
|
|
|
166 .Sp
|
|
|
167 Example:
|
|
|
168 .Sp
|
|
|
169 .Vb 10
|
|
|
170 \& # Tokens specifications supplied by the caller. It\*(Aqs an array containing references
|
|
|
171 \& # to arrays with each containing TokenLabel and TokenMatchRegex pair along with
|
|
|
172 \& # an option reference to code to be executed after a matched.
|
|
|
173 \& #
|
|
|
174 \& @LexerTokensSpec = (
|
|
|
175 \& [ \*(AqLETTER\*(Aq, qr/[a\-zA\-Z]/ ],
|
|
|
176 \& [ \*(AqNUMBER\*(Aq, qr/\ed+/ ],
|
|
|
177 \& [ \*(AqSPACE\*(Aq, qr/[ ]*/,
|
|
|
178 \& sub { my($This, $TokenLabel, $MatchedText) = @_; return \*(Aq\*(Aq; }
|
|
|
179 \& ],
|
|
|
180 \& [ \*(AqNEWLINE\*(Aq, qr/(?:\er\en|\er|\en)/,
|
|
|
181 \& sub { my($This, $TokenLabel, $MatchedText) = @_; return "\en"; }
|
|
|
182 \& ],
|
|
|
183 \& [ \*(AqCHAR\*(Aq, qr/./ ]
|
|
|
184 \& );
|
|
|
185 \&
|
|
|
186 \& # Input string...
|
|
|
187 \& $InputText = \*(Aqy = 3 + 4\*(Aq;
|
|
|
188 \& $Lexer = new Parsers::Lexer($InputText, @LexerTokensSpec);
|
|
|
189 \&
|
|
|
190 \& # Process input stream...
|
|
|
191 \& while (defined($Token = $Lexer\->Lex())) {
|
|
|
192 \& print "Token: " . ((ref $Token) ? "@{$Token}" : "$Token") . "\en";
|
|
|
193 \& }
|
|
|
194 \&
|
|
|
195 \& # Input file...
|
|
|
196 \& $InputFile = "Input.txt";
|
|
|
197 \& open INPUTFILE, "$InputFile" or die "Couldn\*(Aqt open $InputFile: $!\en";
|
|
|
198 \& $Lexer = new Parsers::Lexer(\e*INPUTFILE, @LexerTokensSpec);
|
|
|
199 \&
|
|
|
200 \& # Input file iterator...
|
|
|
201 \& $InputFile = "TestSimpleCalcParser.txt";
|
|
|
202 \& open INPUTFILE, "$InputFile" or die "Couldn\*(Aqt open $InputFile: $!\en";
|
|
|
203 \& $InputIterator = sub { return <INPUTFILE>; };
|
|
|
204 \& $Lexer = new Parsers::Lexer($InputIterator, @LexerTokensSpec);
|
|
|
205 \&
|
|
|
206 \& @LexerTokensSpec = (
|
|
|
207 \& [ \*(AqVAR\*(Aq, qr/[[:alpha:]]+/ ],
|
|
|
208 \& [ \*(AqNUM\*(Aq, qr/\ed+/ ],
|
|
|
209 \& [ \*(AqOP\*(Aq, qr/[\-+=\e/]/,
|
|
|
210 \& sub { my($This, $Label, $Value) = @_;
|
|
|
211 \& $Value .= "; ord: " . ord $Value;
|
|
|
212 \& return [$Label, $Value];
|
|
|
213 \& }
|
|
|
214 \& ],
|
|
|
215 \& [ \*(AqNEWLINE\*(Aq, qr/(?:\er\en|\er|\en)/, sub { return [$_[1], \*(AqNewLine\*(Aq]; } ],
|
|
|
216 \& [ \*(AqSPACE\*(Aq, qr/\es*/, sub { return [$_[1], \*(AqSpace\*(Aq]; } ],
|
|
|
217 \& );
|
|
|
218 \&
|
|
|
219 \& # Look ahead without removing...
|
|
|
220 \& $Token = $Lexer\->Lex(\*(AqPeek\*(Aq);
|
|
|
221 \& if (defined $Token && ref $Token) {
|
|
|
222 \& print "PEEK: Token: @{$Token}\en\en";
|
|
|
223 \& }
|
|
|
224 \&
|
|
|
225 \& # Process input stream...
|
|
|
226 \& while (defined($Token = $Lexer\->Lex())) {
|
|
|
227 \& print "Token: " . ((ref $Token) ? "@{$Token}" : "$Token") . "\en";
|
|
|
228 \& }
|
|
|
229 .Ve
|
|
|
230 .IP "\fBGetLex\fR" 4
|
|
|
231 .IX Item "GetLex"
|
|
|
232 .Vb 1
|
|
|
233 \& $LexerRef = $Lexer\->GetLex();
|
|
|
234 .Ve
|
|
|
235 .Sp
|
|
|
236 Returns a refernece to \fILexer\fR method to the caller for use in a specific \fBYYLexer\fR.
|
|
|
237 .IP "\fBLex\fR" 4
|
|
|
238 .IX Item "Lex"
|
|
|
239 .Vb 7
|
|
|
240 \& $TokenRefOrText = $Lexer\->Lex($Mode);
|
|
|
241 \& if (ref $TokenRefOrText) {
|
|
|
242 \& ($TokenLabel, $TokenValue) = @{$TokenRefOrText};
|
|
|
243 \& }
|
|
|
244 \& else {
|
|
|
245 \& $TokenText = $TokenRefOrText;
|
|
|
246 \& }
|
|
|
247 .Ve
|
|
|
248 .Sp
|
|
|
249 Get next available token label and value pair as an array reference or unrecognized
|
|
|
250 text from input stream by either removing it from the input or simply peeking ahead
|
|
|
251 and without removing it from the input stream.
|
|
|
252 .Sp
|
|
|
253 Possible \fIMode\fR values: \fIPeek, Next\fR. Default: \fINext\fR.
|
|
|
254 .IP "\fBNext\fR" 4
|
|
|
255 .IX Item "Next"
|
|
|
256 .Vb 1
|
|
|
257 \& $TokenRefOrText = $Lexer\->Next();
|
|
|
258 .Ve
|
|
|
259 .Sp
|
|
|
260 Get next available token label and value pair as an array reference or unrecognized
|
|
|
261 text from input stream by removing it from the input stream.
|
|
|
262 .IP "\fBPeek\fR" 4
|
|
|
263 .IX Item "Peek"
|
|
|
264 .Vb 1
|
|
|
265 \& $TokenRefOrText = $Lexer\->Peek();
|
|
|
266 .Ve
|
|
|
267 .Sp
|
|
|
268 Get next available token label and value pair as an array reference or unrecognized
|
|
|
269 text from input stream by by simply peeking ahead and without removing it from the
|
|
|
270 input stream.
|
|
|
271 .IP "\fBStringifyLexer\fR" 4
|
|
|
272 .IX Item "StringifyLexer"
|
|
|
273 .Vb 1
|
|
|
274 \& $LexerString = $Lexer\->StringifyLexer();
|
|
|
275 .Ve
|
|
|
276 .Sp
|
|
|
277 Returns a string containing information about \fILexer\fR object.
|
|
|
278 .SH "AUTHOR"
|
|
|
279 .IX Header "AUTHOR"
|
|
|
280 Manish Sud <msud@san.rr.com>
|
|
|
281 .SH "SEE ALSO"
|
|
|
282 .IX Header "SEE ALSO"
|
|
|
283 YYLexer.pm, SimpleCalcYYLexer.pm, SimpleCalcParser.yy
|
|
|
284 .SH "COPYRIGHT"
|
|
|
285 .IX Header "COPYRIGHT"
|
|
|
286 Copyright (C) 2015 Manish Sud. All rights reserved.
|
|
|
287 .PP
|
|
|
288 This file is part of MayaChemTools.
|
|
|
289 .PP
|
|
|
290 MayaChemTools is free software; you can redistribute it and/or modify it under
|
|
|
291 the terms of the \s-1GNU\s0 Lesser General Public License as published by the Free
|
|
|
292 Software Foundation; either version 3 of the License, or (at your option)
|
|
|
293 any later version.
|
