|
0
|
1 NAME
|
|
|
2 Parsers::YYLexer
|
|
|
3
|
|
|
4 SYNOPSIS
|
|
|
5 use Parseres::YYLexer;
|
|
|
6
|
|
|
7 use Parsers::YYLexer qw(:all);
|
|
|
8
|
|
|
9 DESCRIPTION
|
|
|
10 YYLexer class provides the following methods:
|
|
|
11
|
|
|
12 new, GetYYLex, Next, Peek, SetupYYTabFile, StringifyYYLexer, YYLex
|
|
|
13
|
|
|
14 Parsers::YYLexer class is derived from Parsers::Lexer base class, which
|
|
|
15 provides all the underlying lexer functionality. YYLexer class is
|
|
|
16 designed to be used with yyparse code generated by running byacc on a
|
|
|
17 parsers defined using parser definition ParserName.yy file.
|
|
|
18
|
|
|
19 *YYTabFile* containing mapping of token labels to integers must be
|
|
|
20 explicitly specified by the caller. This file is processed during new
|
|
|
21 method invocation and mapping of token labels to integers is loaded in a
|
|
|
22 hash to be used later by YYLex method to return token number and text
|
|
|
23 pairs to the parser.
|
|
|
24
|
|
|
25 METHODS
|
|
|
26 new
|
|
|
27 $YYLexer = new Parsers::YYLexer($Input, @YYLexerTokensSpec);
|
|
|
28
|
|
|
29 Using specified *Input* and *YYLexerTokensSpec*, new method
|
|
|
30 generates a new YYLexer and returns a reference to newly created
|
|
|
31 YYLexer object.
|
|
|
32
|
|
|
33 Examples:
|
|
|
34
|
|
|
35 # Tokens specifications supplied by the caller. It's an array containing references
|
|
|
36 # to arrays with each containing TokenLabel and TokenMatchRegex pair along with
|
|
|
37 # an option reference to code to be executed after a matched.
|
|
|
38 #
|
|
|
39 @LexerTokensSpec = (
|
|
|
40 [ 'LETTER', qr/[a-zA-Z]/ ],
|
|
|
41 [ 'NUMBER', qr/\d+/ ],
|
|
|
42 [ 'SPACE', qr/[ ]*/,
|
|
|
43 sub { my($This, $TokenLabel, $MatchedText) = @_; return ''; }
|
|
|
44 ],
|
|
|
45 [ 'NEWLINE', qr/(?:\r\n|\r|\n)/,
|
|
|
46 sub { my($This, $TokenLabel, $MatchedText) = @_; return "\n"; }
|
|
|
47 ],
|
|
|
48 [ 'CHAR', qr/./ ]
|
|
|
49 );
|
|
|
50
|
|
|
51 # Input string...
|
|
|
52 $InputText = 'y = 3 + 4';
|
|
|
53
|
|
|
54 $YLexer = new Parsers::YYLexer($InputText, @LexerTokensSpec);
|
|
|
55
|
|
|
56 # Setup default token table file...
|
|
|
57 $YYTabFile = "Parsers/SimpleCalcParser.tab.ph";
|
|
|
58 $This->SetupYYTabFile($YYTabFile);
|
|
|
59
|
|
|
60 # Process input stream...
|
|
|
61 ($TokenNumber, $TokenText) = $YYLexer->Lex();
|
|
|
62 print "TokenNumber: $TokenNumber; TokenText: $TokenText\n";
|
|
|
63
|
|
|
64 # Input file...
|
|
|
65 $InputFile = "Input.txt";
|
|
|
66 open INPUTFILE, "$InputFile" or die "Couldn't open $InputFile: $!\n";
|
|
|
67 $Lexer = new Parsers::YYLexer(\*INPUTFILE, @LexerTokensSpec);
|
|
|
68
|
|
|
69 # Input file iterator...
|
|
|
70 $InputFile = "TestSimpleCalcParser.txt";
|
|
|
71 open INPUTFILE, "$InputFile" or die "Couldn't open $InputFile: $!\n";
|
|
|
72 $InputIterator = sub { return <INPUTFILE>; };
|
|
|
73 $Lexer = new Parsers::YYLexer($InputIterator, @LexerTokensSpec);
|
|
|
74
|
|
|
75 # Usage with code generated by byacc from a parser definition
|
|
|
76 # file SimpleCalcParser.yy...
|
|
|
77
|
|
|
78 $InputText = "3 + 4 +6\nx=3\ny=5\nx+y\nx+z\n";
|
|
|
79
|
|
|
80 $YYLexer = new Parsers::YYLexer($InputText,@LexerTokensSpec);
|
|
|
81
|
|
|
82 $YYLex = $YYLexer->GetYYLex();
|
|
|
83
|
|
|
84 $YYTabFile = "Parsers/SimpleCalcParser.tab.ph";
|
|
|
85 $YYLexer->SetupYYTabFile($YYTabFile);
|
|
|
86
|
|
|
87 $Debug = 0;
|
|
|
88 $SimpleCalcParser = new Parsers::SimpleCalcParser($YYLex,
|
|
|
89 \&Parsers::SimpleCalcParser::yyerror, $Debug);
|
|
|
90
|
|
|
91 $Value = $SimpleCalcParser->yyparse();
|
|
|
92 print "Value = " . (defined($Value) ? "$Value" : "Undefined") . "\n";
|
|
|
93
|
|
|
94 GetYYLex
|
|
|
95 $YYLex = $YYLexer->GetYYLex();
|
|
|
96
|
|
|
97 Returns a curried verson of YYLexer as YYLex: yyparse in parser
|
|
|
98 generated by byacc expects it to call without passing any argument
|
|
|
99 for the *YYLexer* object.
|
|
|
100
|
|
|
101 Next
|
|
|
102 ($TokenNumber, $TokenText) = $YYLexer->Next();
|
|
|
103
|
|
|
104 Returns next available TokenNumber and any matched TokenText from
|
|
|
105 input stream by removing it from the input stream. Token number and
|
|
|
106 text of zero corresponds to end of input (EOI).
|
|
|
107
|
|
|
108 Peek
|
|
|
109 ($TokenNumber, $TokenText) = $YYLexer->Peek();
|
|
|
110
|
|
|
111 Returns next available TokenNumber and any matched TokenText from
|
|
|
112 input stream by simply looking ahead and without removing it from
|
|
|
113 the input stream. Token number and text of zero corresponds to end
|
|
|
114 of input (EOI).
|
|
|
115
|
|
|
116 SetupYYTabFile
|
|
|
117 $YYLexer = $YYLexer->SetupYYTabFile($YYTabFile);
|
|
|
118
|
|
|
119 Processes token labels to integers data map in specified *YYTabFile*
|
|
|
120 and returns *YYLexer*.
|
|
|
121
|
|
|
122 Notes:
|
|
|
123
|
|
|
124 . YYTabFile must be a complete path or available through @INC path in the
|
|
|
125 same directory where this package is located.
|
|
|
126 . Name of YYTabFile might start with any valid sub directory name in @INC
|
|
|
127 For example, "Parsers/<YYTablFile>" implies the tab file in parsers sub
|
|
|
128 directory under MayaChemTools lib directory as it would be already in @INC
|
|
|
129 path.
|
|
|
130 . YYTabFile must be explicitly set by the caller. The default YYTabFile name,
|
|
|
131 y.tab.ph, generated by byacc is not used implicitly to avoid confusion
|
|
|
132 among multiple distinct instances of YYLexer.
|
|
|
133 . YYTabFile is generated by byacc during its usage with -d options and
|
|
|
134 contains mapping of token codes to token names/labels. YYLexer used this
|
|
|
135 file to map token labels to token codes before returning token code and
|
|
|
136 value pair back to yyparse function used by byacc.
|
|
|
137 . User defined token numbers start from 257
|
|
|
138 . Token number for any user defined token EOI is mapped to its value before
|
|
|
139 default token number of 0 for EOI.
|
|
|
140
|
|
|
141 The format of YYTabFile generated by byacc during generation of parser code in
|
|
|
142 Perl code is:
|
|
|
143
|
|
|
144 ... ...
|
|
|
145 $NUMBER=257;
|
|
|
146 $ADDOP=258;
|
|
|
147 $SUBOP=259;
|
|
|
148 ... ..
|
|
|
149
|
|
|
150 YYLex
|
|
|
151 ($TokenNumber, $TokenText) = $YYLexer->YYLex();
|
|
|
152 ($TokenNumber, $TokenText) = $YYLexer->YYLex($Mode);
|
|
|
153
|
|
|
154 Returns available TokenNumber and any matched TokenText from input
|
|
|
155 stream by either removing it from the input stream or by simply
|
|
|
156 looking ahead and without removing it from the input stream. Token
|
|
|
157 number and text of zero corresponds to end of input (EOI).
|
|
|
158
|
|
|
159 Possible *Mode* values: *Peek, Next*. Default: *Next*.
|
|
|
160
|
|
|
161 *YYLex* is designed to be used with yyparse code generated by
|
|
|
162 running byacc on a parsers defined using parser definition
|
|
|
163 ParserName.yy file.
|
|
|
164
|
|
|
165 Notes:
|
|
|
166
|
|
|
167 . Token label and value pairs returned by Lexer from by base class, which
|
|
|
168 can't be mapped to token labels specified in YYTabFile are ignored.
|
|
|
169 . Token text of length 1 returned by Lexer from base class without a
|
|
|
170 corresponding explicit token label, which can't be mapped to a token
|
|
|
171 number using Perl ord function, is ignored.
|
|
|
172
|
|
|
173 StringifyYYLexer
|
|
|
174 $YYLexerString = $YYLexer->StringifyYYLexer();
|
|
|
175
|
|
|
176 Returns a string containing information about *YYLexer* object.
|
|
|
177
|
|
|
178 AUTHOR
|
|
|
179 Manish Sud <msud@san.rr.com>
|
|
|
180
|
|
|
181 SEE ALSO
|
|
|
182 Lexer.pm, SimpleCalcYYLexer.pm, SimpleCalcParser.yy
|
|
|
183
|
|
|
184 COPYRIGHT
|
|
|
185 Copyright (C) 2015 Manish Sud. All rights reserved.
|
|
|
186
|
|
|
187 This file is part of MayaChemTools.
|
|
|
188
|
|
|
189 MayaChemTools is free software; you can redistribute it and/or modify it
|
|
|
190 under the terms of the GNU Lesser General Public License as published by
|
|
|
191 the Free Software Foundation; either version 3 of the License, or (at
|
|
|
192 your option) any later version.
|
|
|
193
|
