|
1
|
1 <html>
|
|
|
2 <head>
|
|
|
3 <title>MayaChemTools:Code:Parsers::Lexer.pm</title>
|
|
|
4 <meta http-equiv="content-type" content="text/html;charset=utf-8">
|
|
|
5 <link rel="stylesheet" type="text/css" href="../../../css/MayaChemToolsCode.css">
|
|
|
6 </head>
|
|
|
7 <body leftmargin="20" rightmargin="20" topmargin="10" bottommargin="10">
|
|
|
8 <br/>
|
|
|
9 <center>
|
|
|
10 <a href="http://www.mayachemtools.org" title="MayaChemTools Home"><img src="../../../images/MayaChemToolsLogo.gif" border="0" alt="MayaChemTools"></a>
|
|
|
11 </center>
|
|
|
12 <br/>
|
|
|
13 <pre>
|
|
|
14 <a name="package-Parsers::Lexer-"></a> 1 <span class="k">package </span><span class="i">Parsers::Lexer</span><span class="sc">;</span>
|
|
|
15 2 <span class="c">#</span>
|
|
|
16 3 <span class="c"># $RCSfile: Lexer.pm,v $</span>
|
|
|
17 4 <span class="c"># $Date: 2015/02/28 20:50:55 $</span>
|
|
|
18 5 <span class="c"># $Revision: 1.10 $</span>
|
|
|
19 6 <span class="c">#</span>
|
|
|
20 7 <span class="c"># Author: Manish Sud <msud@san.rr.com></span>
|
|
|
21 8 <span class="c">#</span>
|
|
|
22 9 <span class="c"># Copyright (C) 2015 Manish Sud. All rights reserved.</span>
|
|
|
23 10 <span class="c">#</span>
|
|
|
24 11 <span class="c"># This file is part of MayaChemTools.</span>
|
|
|
25 12 <span class="c">#</span>
|
|
|
26 13 <span class="c"># MayaChemTools is free software; you can redistribute it and/or modify it under</span>
|
|
|
27 14 <span class="c"># the terms of the GNU Lesser General Public License as published by the Free</span>
|
|
|
28 15 <span class="c"># Software Foundation; either version 3 of the License, or (at your option) any</span>
|
|
|
29 16 <span class="c"># later version.</span>
|
|
|
30 17 <span class="c">#</span>
|
|
|
31 18 <span class="c"># MayaChemTools is distributed in the hope that it will be useful, but without</span>
|
|
|
32 19 <span class="c"># any warranty; without even the implied warranty of merchantability of fitness</span>
|
|
|
33 20 <span class="c"># for a particular purpose. See the GNU Lesser General Public License for more</span>
|
|
|
34 21 <span class="c"># details.</span>
|
|
|
35 22 <span class="c">#</span>
|
|
|
36 23 <span class="c"># You should have received a copy of the GNU Lesser General Public License</span>
|
|
|
37 24 <span class="c"># along with MayaChemTools; if not, see <http://www.gnu.org/licenses/> or</span>
|
|
|
38 25 <span class="c"># write to the Free Software Foundation Inc., 59 Temple Place, Suite 330,</span>
|
|
|
39 26 <span class="c"># Boston, MA, 02111-1307, USA.</span>
|
|
|
40 27 <span class="c">#</span>
|
|
|
41 28
|
|
|
42 29 <span class="k">use</span> <span class="w">strict</span><span class="sc">;</span>
|
|
|
43 30 <span class="k">use</span> <span class="w">Carp</span><span class="sc">;</span>
|
|
|
44 31 <span class="k">use</span> <span class="w">Exporter</span><span class="sc">;</span>
|
|
|
45 32 <span class="k">use</span> <span class="w">Scalar::Util</span> <span class="s">(</span><span class="s">)</span><span class="sc">;</span>
|
|
|
46 33
|
|
|
47 34 <span class="k">use</span> <span class="w">vars</span> <span class="q">qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS)</span><span class="sc">;</span>
|
|
|
48 35
|
|
|
49 36 <span class="i">@ISA</span> = <span class="q">qw(Exporter)</span><span class="sc">;</span>
|
|
|
50 37 <span class="i">@EXPORT</span> = <span class="q">qw()</span><span class="sc">;</span>
|
|
|
51 38 <span class="i">@EXPORT_OK</span> = <span class="q">qw()</span><span class="sc">;</span>
|
|
|
52 39
|
|
|
53 40 <span class="i">%EXPORT_TAGS</span> = <span class="s">(</span><span class="w">all</span> <span class="cm">=></span> <span class="s">[</span><span class="i">@EXPORT</span><span class="cm">,</span> <span class="i">@EXPORT_OK</span><span class="s">]</span><span class="s">)</span><span class="sc">;</span>
|
|
|
54 41
|
|
|
55 42 <span class="c"># Setup class variables...</span>
|
|
|
56 43 <span class="k">my</span><span class="s">(</span><span class="i">$ClassName</span><span class="s">)</span><span class="sc">;</span>
|
|
|
57 44 <span class="i">_InitializeClass</span><span class="s">(</span><span class="s">)</span><span class="sc">;</span>
|
|
|
58 45
|
|
|
59 46 <span class="c"># Overload Perl functions...</span>
|
|
|
60 47 <span class="k">use</span> <span class="w">overload</span> <span class="q">'""'</span> <span class="cm">=></span> <span class="q">'StringifyLexer'</span><span class="sc">;</span>
|
|
|
61 48
|
|
|
62 49 <span class="c"># Class constructor...</span>
|
|
|
63 <a name="new-"></a> 50 <span class="k">sub </span><span class="m">new</span> <span class="s">{</span>
|
|
|
64 51 <span class="k">my</span><span class="s">(</span><span class="i">$Class</span><span class="cm">,</span> <span class="i">$Input</span><span class="cm">,</span> <span class="i">@TokensSpec</span><span class="s">)</span> = <span class="i">@_</span><span class="sc">;</span>
|
|
|
65 52
|
|
|
66 53 <span class="c"># Initialize object...</span>
|
|
|
67 54 <span class="k">my</span> <span class="i">$This</span> = <span class="s">{</span><span class="s">}</span><span class="sc">;</span>
|
|
|
68 55 <span class="k">bless</span> <span class="i">$This</span><span class="cm">,</span> <span class="k">ref</span><span class="s">(</span><span class="i">$Class</span><span class="s">)</span> || <span class="i">$Class</span><span class="sc">;</span>
|
|
|
69 56 <span class="i">$This</span><span class="i">->_InitializeLexer</span><span class="s">(</span><span class="s">)</span><span class="sc">;</span>
|
|
|
70 57
|
|
|
71 58 <span class="i">$This</span><span class="i">->_ValidateParametersAndGenerateLexer</span><span class="s">(</span><span class="i">$Input</span><span class="cm">,</span> <span class="i">@TokensSpec</span><span class="s">)</span><span class="sc">;</span>
|
|
|
72 59
|
|
|
73 60 <span class="k">return</span> <span class="i">$This</span><span class="sc">;</span>
|
|
|
74 61 <span class="s">}</span>
|
|
|
75 62
|
|
|
76 63
|
|
|
77 64 <span class="c"># Initialize class ...</span>
|
|
|
78 <a name="_InitializeClass-"></a> 65 <span class="k">sub </span><span class="m">_InitializeClass</span> <span class="s">{</span>
|
|
|
79 66 <span class="c">#Class name...</span>
|
|
|
80 67 <span class="i">$ClassName</span> = <span class="w">__PACKAGE__</span><span class="sc">;</span>
|
|
|
81 68 <span class="s">}</span>
|
|
|
82 69
|
|
|
83 70 <span class="c"># Initialize object data...</span>
|
|
|
84 71 <span class="c">#</span>
|
|
|
85 <a name="_InitializeLexer-"></a> 72 <span class="k">sub </span><span class="m">_InitializeLexer</span> <span class="s">{</span>
|
|
|
86 73 <span class="k">my</span><span class="s">(</span><span class="i">$This</span><span class="s">)</span> = <span class="i">@_</span><span class="sc">;</span>
|
|
|
87 74
|
|
|
88 75 <span class="c"># Input parameter used by lexer to retrieve text to be lexed. Supported parameter types:</span>
|
|
|
89 76 <span class="c"># . Reference to input iterator function</span>
|
|
|
90 77 <span class="c"># . Reference to an open file handle</span>
|
|
|
91 78 <span class="c"># . Text string</span>
|
|
|
92 79 <span class="c">#</span>
|
|
|
93 80 <span class="i">$This</span>->{<span class="w">Input</span>} = <span class="k">undef</span><span class="sc">;</span>
|
|
|
94 81
|
|
|
95 82 <span class="c"># Type of input paramater determined using Perl ref function:</span>
|
|
|
96 83 <span class="c"># . InputIterator - ref returns CODE</span>
|
|
|
97 84 <span class="c"># . FileStream - ref return GLOB and fileno is valid</span>
|
|
|
98 85 <span class="c"># . String - ref return an empty string</span>
|
|
|
99 86 <span class="c">#</span>
|
|
|
100 87 <span class="i">$This</span>->{<span class="w">InputType</span>} = <span class="q">''</span><span class="sc">;</span>
|
|
|
101 88
|
|
|
102 89 <span class="c"># Tokens specifications supplied by the caller. It's an array containing references</span>
|
|
|
103 90 <span class="c"># to arrays with each containing TokenLabel and TokenMatchRegex pair along with</span>
|
|
|
104 91 <span class="c"># an option reference to code to be executed after a matched.</span>
|
|
|
105 92 <span class="c">#</span>
|
|
|
106 93 <span class="c"># For example:</span>
|
|
|
107 94 <span class="c">#</span>
|
|
|
108 95 <span class="c"># @LexerTokensSpec = (</span>
|
|
|
109 96 <span class="c"># [ 'LETTER', qr/[a-zA-Z]/ ],</span>
|
|
|
110 97 <span class="c"># [ 'NUMBER', qr/\d+/ ],</span>
|
|
|
111 98 <span class="c"># [ 'SPACE', qr/[ ]*/, sub { my($This, $TokenLabel, $MatchedText) = @_; return ''; } ],</span>
|
|
|
112 99 <span class="c"># [ 'NEWLINE', qr/(?:\r\n|\r|\n)/, sub { my($This, $TokenLabel, $MatchedText) = @_; return "\n"; } ],</span>
|
|
|
113 100 <span class="c"># [ 'CHAR', qr/[\.]/ ],</span>
|
|
|
114 101 <span class="c"># );</span>
|
|
|
115 102 <span class="c">#</span>
|
|
|
116 103 <span class="i">@</span>{<span class="i">$This</span>->{<span class="w">TokensSpec</span>}} = <span class="s">(</span><span class="s">)</span><span class="sc">;</span>
|
|
|
117 104
|
|
|
118 105 <span class="c"># Refernce to chained lexer...</span>
|
|
|
119 106 <span class="i">$This</span>->{<span class="w">ChainedLexer</span>} = <span class="k">undef</span><span class="sc">;</span>
|
|
|
120 107
|
|
|
121 108 <span class="k">return</span> <span class="i">$This</span><span class="sc">;</span>
|
|
|
122 109 <span class="s">}</span>
|
|
|
123 110
|
|
|
124 111 <span class="c"># Validate input parameters and generate a chained lexer...</span>
|
|
|
125 112 <span class="c">#</span>
|
|
|
126 <a name="_ValidateParametersAndGenerateLexer-"></a> 113 <span class="k">sub </span><span class="m">_ValidateParametersAndGenerateLexer</span> <span class="s">{</span>
|
|
|
127 114 <span class="k">my</span><span class="s">(</span><span class="i">$This</span><span class="cm">,</span> <span class="i">$Input</span><span class="cm">,</span> <span class="i">@TokensSpec</span><span class="s">)</span> = <span class="i">@_</span><span class="sc">;</span>
|
|
|
128 115
|
|
|
129 116 <span class="c">#</span>
|
|
|
130 117 <span class="c"># Validate input to be lexed...</span>
|
|
|
131 118 <span class="k">if</span> <span class="s">(</span>!<span class="k">defined</span> <span class="i">$Input</span><span class="s">)</span> <span class="s">{</span>
|
|
|
132 119 <span class="w">croak</span> <span class="q">"Error: ${ClassName}->new: Object can't be instantiated: Input is not defined. Supported values: a reference to input iterator function, a reference to an open file handle or a text string..."</span><span class="sc">;</span>
|
|
|
133 120 <span class="s">}</span>
|
|
|
134 121 <span class="i">$This</span>->{<span class="w">Input</span>} = <span class="i">$Input</span><span class="sc">;</span>
|
|
|
135 122
|
|
|
136 123 <span class="c"># Check input parameter type...</span>
|
|
|
137 124 <span class="k">my</span><span class="s">(</span><span class="i">$InputType</span><span class="s">)</span><span class="sc">;</span>
|
|
|
138 125
|
|
|
139 126 <span class="i">$InputType</span> = <span class="k">ref</span> <span class="i">$Input</span><span class="sc">;</span>
|
|
|
140 127 <span class="k">if</span> <span class="s">(</span><span class="i">$InputType</span> =~ <span class="q">/CODE/i</span><span class="s">)</span> <span class="s">{</span>
|
|
|
141 128 <span class="c"># Input iterator...</span>
|
|
|
142 129 <span class="i">$This</span>->{<span class="w">InputType</span>} = <span class="q">"InputIterator"</span><span class="sc">;</span>
|
|
|
143 130 <span class="s">}</span>
|
|
|
144 131 <span class="k">elsif</span> <span class="s">(</span><span class="i">$InputType</span> =~ <span class="q">/GLOB/i</span> && <span class="k">defined</span> <span class="k">fileno</span> <span class="i">$Input</span><span class="s">)</span> <span class="s">{</span>
|
|
|
145 132 <span class="c"># Input stream...</span>
|
|
|
146 133 <span class="i">$This</span>->{<span class="w">InputType</span>} = <span class="q">"FileStream"</span><span class="sc">;</span>
|
|
|
147 134 <span class="s">}</span>
|
|
|
148 135 <span class="k">elsif</span> <span class="s">(</span><span class="i">$InputType</span><span class="s">)</span> <span class="s">{</span>
|
|
|
149 136 <span class="c"># Perl ref function returns nonempty string for all other references...</span>
|
|
|
150 137 <span class="w">croak</span> <span class="q">"Error: ${ClassName}->new: Object can't be instantiated: Invalid input parameter type specified. Supported parameter types: a reference to input iterator function, a reference to an open file handle or a text string..."</span><span class="sc">;</span>
|
|
|
151 138 <span class="s">}</span>
|
|
|
152 139 <span class="k">else</span> <span class="s">{</span>
|
|
|
153 140 <span class="c"># Input string...</span>
|
|
|
154 141 <span class="i">$This</span>->{<span class="w">InputType</span>} = <span class="q">"String"</span><span class="sc">;</span>
|
|
|
155 142 <span class="s">}</span>
|
|
|
156 143
|
|
|
157 144 <span class="c"># Check tokens specifications...</span>
|
|
|
158 145 <span class="k">if</span> <span class="s">(</span>!<span class="i">@TokensSpec</span><span class="s">)</span> <span class="s">{</span>
|
|
|
159 146 <span class="w">croak</span> <span class="q">"Error: ${ClassName}->new: TokensSpec is not defined or the array doesn't contain any values. Supported values: a reference to an array containg token label, regular expression to match and an option reference to function to modify matched values..."</span><span class="sc">;</span>
|
|
|
160 147 <span class="s">}</span>
|
|
|
161 148 <span class="i">@</span>{<span class="i">$This</span>->{<span class="w">TokensSpec</span>}} = <span class="i">@TokensSpec</span><span class="sc">;</span>
|
|
|
162 149
|
|
|
163 150 <span class="i">$This</span><span class="i">->_GenerateLexer</span><span class="s">(</span><span class="i">$Input</span><span class="cm">,</span> <span class="i">@TokensSpec</span><span class="s">)</span><span class="sc">;</span>
|
|
|
164 151
|
|
|
165 152 <span class="k">return</span> <span class="i">$This</span><span class="sc">;</span>
|
|
|
166 153 <span class="s">}</span>
|
|
|
167 154
|
|
|
168 155 <span class="c"># Generate a lexer using reference to an input iterator function, an open file</span>
|
|
|
169 156 <span class="c"># handle or an input string passed as first parameter by the caller along</span>
|
|
|
170 157 <span class="c"># with token specifications as second paramater...</span>
|
|
|
171 158 <span class="c">#</span>
|
|
|
172 <a name="_GenerateLexer-"></a> 159 <span class="k">sub </span><span class="m">_GenerateLexer</span> <span class="s">{</span>
|
|
|
173 160 <span class="k">my</span><span class="s">(</span><span class="i">$This</span><span class="cm">,</span> <span class="i">$Input</span><span class="cm">,</span> <span class="i">@TokensSpec</span><span class="s">)</span> = <span class="i">@_</span><span class="sc">;</span>
|
|
|
174 161
|
|
|
175 162 <span class="k">if</span> <span class="s">(</span><span class="i">$This</span>->{<span class="w">InputType</span>} =~ <span class="q">/^InputIterator$/i</span><span class="s">)</span> <span class="s">{</span>
|
|
|
176 163 <span class="i">$This</span><span class="i">->_GenerateInputIteratorLexer</span><span class="s">(</span><span class="i">$Input</span><span class="cm">,</span> <span class="i">@TokensSpec</span><span class="s">)</span><span class="sc">;</span>
|
|
|
177 164 <span class="s">}</span>
|
|
|
178 165 <span class="k">elsif</span> <span class="s">(</span><span class="i">$This</span>->{<span class="w">InputType</span>} =~ <span class="q">/^FileStream$/i</span><span class="s">)</span> <span class="s">{</span>
|
|
|
179 166 <span class="i">$This</span><span class="i">->_GenerateInputFileStreamLexer</span><span class="s">(</span><span class="i">$Input</span><span class="cm">,</span> <span class="i">@TokensSpec</span><span class="s">)</span><span class="sc">;</span>
|
|
|
180 167 <span class="s">}</span>
|
|
|
181 168 <span class="k">elsif</span> <span class="s">(</span><span class="i">$This</span>->{<span class="w">InputType</span>} =~ <span class="q">/^String$/i</span><span class="s">)</span> <span class="s">{</span>
|
|
|
182 169 <span class="i">$This</span><span class="i">->_GenerateInputStringLexer</span><span class="s">(</span><span class="i">$Input</span><span class="cm">,</span> <span class="i">@TokensSpec</span><span class="s">)</span><span class="sc">;</span>
|
|
|
183 170 <span class="s">}</span>
|
|
|
184 171 <span class="k">else</span> <span class="s">{</span>
|
|
|
185 172 <span class="w">croak</span> <span class="q">"Error: ${ClassName}->new: Object can't be instantiated: Invalid input parameter type specified. Supported parameter types: a reference to input iterator function, a reference to an open file handle or a text string..."</span><span class="sc">;</span>
|
|
|
186 173 <span class="s">}</span>
|
|
|
187 174
|
|
|
188 175 <span class="k">return</span> <span class="i">$This</span><span class="sc">;</span>
|
|
|
189 176 <span class="s">}</span>
|
|
|
190 177
|
|
|
191 178 <span class="c"># Generate a lexer using specifed input iterator...</span>
|
|
|
192 179 <span class="c">#</span>
|
|
|
193 <a name="_GenerateInputIteratorLexer-"></a> 180 <span class="k">sub </span><span class="m">_GenerateInputIteratorLexer</span> <span class="s">{</span>
|
|
|
194 181 <span class="k">my</span><span class="s">(</span><span class="i">$This</span><span class="cm">,</span> <span class="i">$InputIteratorRef</span><span class="cm">,</span> <span class="i">@TokensSpec</span><span class="s">)</span> = <span class="i">@_</span><span class="sc">;</span>
|
|
|
195 182
|
|
|
196 183 <span class="i">$This</span><span class="i">->_GenerateChainedLexer</span><span class="s">(</span><span class="i">$InputIteratorRef</span><span class="cm">,</span> <span class="i">@TokensSpec</span><span class="s">)</span><span class="sc">;</span>
|
|
|
197 184
|
|
|
198 185 <span class="k">return</span> <span class="i">$This</span><span class="sc">;</span>
|
|
|
199 186 <span class="s">}</span>
|
|
|
200 187
|
|
|
201 188 <span class="c"># Generate a lexer using specifed input file stream reference...</span>
|
|
|
202 189 <span class="c">#</span>
|
|
|
203 <a name="_GenerateInputFileStreamLexer-"></a> 190 <span class="k">sub </span><span class="m">_GenerateInputFileStreamLexer</span> <span class="s">{</span>
|
|
|
204 191 <span class="k">my</span><span class="s">(</span><span class="i">$This</span><span class="cm">,</span> <span class="i">$FileHandleRef</span><span class="cm">,</span> <span class="i">@TokensSpec</span><span class="s">)</span> = <span class="i">@_</span><span class="sc">;</span>
|
|
|
205 192
|
|
|
206 193 <span class="c"># Iterator is a annoymous function reference and Perl keeps $FileHandleRef</span>
|
|
|
207 194 <span class="c"># in scope during its execution.</span>
|
|
|
208 195
|
|
|
209 196 <span class="i">$This</span><span class="i">->_GenerateChainedLexer</span><span class="s">(</span> <span class="k">sub</span> <span class="s">{</span> <span class="k">return</span> <span class="q"><$FileHandleRef></span><span class="sc">;</span> <span class="s">}</span><span class="cm">,</span> <span class="i">@TokensSpec</span><span class="s">)</span><span class="sc">;</span>
|
|
|
210 197
|
|
|
211 198 <span class="k">return</span> <span class="i">$This</span><span class="sc">;</span>
|
|
|
212 199 <span class="s">}</span>
|
|
|
213 200
|
|
|
214 201 <span class="c"># Generate a lexer using specifed input string...</span>
|
|
|
215 202 <span class="c">#</span>
|
|
|
216 <a name="_GenerateInputStringLexer-"></a> 203 <span class="k">sub </span><span class="m">_GenerateInputStringLexer</span> <span class="s">{</span>
|
|
|
217 204 <span class="k">my</span><span class="s">(</span><span class="i">$This</span><span class="cm">,</span> <span class="i">$Text</span><span class="cm">,</span> <span class="i">@TokensSpec</span><span class="s">)</span> = <span class="i">@_</span><span class="sc">;</span>
|
|
|
218 205 <span class="k">my</span><span class="s">(</span><span class="i">@InputText</span><span class="s">)</span> = <span class="s">(</span><span class="i">$Text</span><span class="s">)</span><span class="sc">;</span>
|
|
|
219 206
|
|
|
220 207 <span class="c"># Iterator is a annoymous function reference and Perl keeps @InputText</span>
|
|
|
221 208 <span class="c"># in scope during its execution.</span>
|
|
|
222 209
|
|
|
223 210 <span class="i">$This</span><span class="i">->_GenerateChainedLexer</span><span class="s">(</span> <span class="k">sub</span> <span class="s">{</span> <span class="k">return</span> <span class="k">shift</span> <span class="i">@InputText</span><span class="sc">;</span> <span class="s">}</span><span class="cm">,</span> <span class="i">@TokensSpec</span><span class="s">)</span><span class="sc">;</span>
|
|
|
224 211
|
|
|
225 212 <span class="k">return</span> <span class="i">$This</span><span class="sc">;</span>
|
|
|
226 213 <span class="s">}</span>
|
|
|
227 214
|
|
|
228 215 <span class="c"># Get next available token label and value pair as an array reference or unrecognized</span>
|
|
|
229 216 <span class="c"># text from input stream by either removing it from the input or simply peeking ahead...</span>
|
|
|
230 217 <span class="c">#</span>
|
|
|
231 218 <span class="c"># Supported mode values: Peek, Next. Default: Next</span>
|
|
|
232 219 <span class="c">#</span>
|
|
|
233 <a name="Lex-"></a> 220 <span class="k">sub </span><span class="m">Lex</span> <span class="s">{</span>
|
|
|
234 221 <span class="k">my</span><span class="s">(</span><span class="i">$This</span><span class="cm">,</span> <span class="i">$Mode</span><span class="s">)</span> = <span class="i">@_</span><span class="sc">;</span>
|
|
|
235 222
|
|
|
236 223 <span class="k">return</span> <span class="i">$This</span>->{<span class="w">ChainedLexer</span>}-><span class="s">(</span><span class="i">$Mode</span><span class="s">)</span>
|
|
|
237 224 <span class="s">}</span>
|
|
|
238 225
|
|
|
239 226 <span class="c"># Get next available token label and value pair as an array reference or unrecognized</span>
|
|
|
240 227 <span class="c"># text from input stream by either removing it from the input stream...</span>
|
|
|
241 228 <span class="c">#</span>
|
|
|
242 <a name="Next-"></a> 229 <span class="k">sub </span><span class="m">Next</span> <span class="s">{</span>
|
|
|
243 230 <span class="k">my</span><span class="s">(</span><span class="i">$This</span><span class="s">)</span> = <span class="i">@_</span><span class="sc">;</span>
|
|
|
244 231
|
|
|
245 232 <span class="k">return</span> <span class="i">$This</span><span class="i">->Lex</span><span class="s">(</span><span class="s">)</span><span class="sc">;</span>
|
|
|
246 233 <span class="s">}</span>
|
|
|
247 234
|
|
|
248 235 <span class="c"># Get next available token label and value pair as an array reference or unrecognized</span>
|
|
|
249 236 <span class="c"># text from input stream by simply peeking ahead and without removing it from the input</span>
|
|
|
250 237 <span class="c"># stream..</span>
|
|
|
251 238 <span class="c">#</span>
|
|
|
252 <a name="Peek-"></a> 239 <span class="k">sub </span><span class="m">Peek</span> <span class="s">{</span>
|
|
|
253 240 <span class="k">my</span><span class="s">(</span><span class="i">$This</span><span class="s">)</span> = <span class="i">@_</span><span class="sc">;</span>
|
|
|
254 241
|
|
|
255 242 <span class="k">return</span> <span class="i">$This</span><span class="i">->Lex</span><span class="s">(</span><span class="q">'Peek'</span><span class="s">)</span>
|
|
|
256 243 <span class="s">}</span>
|
|
|
257 244
|
|
|
258 245 <span class="c"># Get a reference to lexer method to be used by the caller...</span>
|
|
|
259 246 <span class="c">#</span>
|
|
|
260 <a name="GetLex-"></a> 247 <span class="k">sub </span><span class="m">GetLex</span> <span class="s">{</span>
|
|
|
261 248 <span class="k">my</span><span class="s">(</span><span class="i">$This</span><span class="s">)</span> = <span class="i">@_</span><span class="sc">;</span>
|
|
|
262 249
|
|
|
263 250 <span class="k">return</span> <span class="k">sub</span> <span class="s">{</span> <span class="i">$This</span><span class="i">->Lex</span><span class="s">(</span><span class="s">)</span><span class="sc">;</span> <span class="s">}</span><span class="sc">;</span>
|
|
|
264 251 <span class="s">}</span>
|
|
|
265 252
|
|
|
266 253 <span class="c"># The chained lexer generation is implemented based on examples in Higher-order Perl</span>
|
|
|
267 254 <span class="c"># [ Ref 126 ] book.</span>
|
|
|
268 255 <span class="c">#</span>
|
|
|
269 256 <span class="c"># Generate a lexer using specified input iterator and chaining it with other lexers generated</span>
|
|
|
270 257 <span class="c"># for all token specifications. The lexer generated for first token specification uses input</span>
|
|
|
271 258 <span class="c"># iterator to retrieve any available input text; the subsequent chained lexeres for rest</span>
|
|
|
272 259 <span class="c"># of the tokens use lexers generated for previous token specifications to get next input</span>
|
|
|
273 260 <span class="c"># which might be unmatched input text or a reference to an array containing token and</span>
|
|
|
274 261 <span class="c"># matched text pair.</span>
|
|
|
275 262 <span class="c">#</span>
|
|
|
276 <a name="_GenerateChainedLexer-"></a> 263 <span class="k">sub </span><span class="m">_GenerateChainedLexer</span> <span class="s">{</span>
|
|
|
277 264 <span class="k">my</span><span class="s">(</span><span class="i">$This</span><span class="cm">,</span> <span class="i">$InputIteratorRef</span><span class="cm">,</span> <span class="i">@TokensSpec</span><span class="s">)</span> = <span class="i">@_</span><span class="sc">;</span>
|
|
|
278 265 <span class="k">my</span><span class="s">(</span><span class="i">$TokenSpecRef</span><span class="cm">,</span> <span class="i">$ChainedLexer</span><span class="s">)</span><span class="sc">;</span>
|
|
|
279 266
|
|
|
280 267 <span class="i">$ChainedLexer</span> = <span class="k">undef</span><span class="sc">;</span>
|
|
|
281 268 <span class="k">for</span> <span class="i">$TokenSpecRef</span> <span class="s">(</span><span class="i">@TokensSpec</span><span class="s">)</span> <span class="s">{</span>
|
|
|
282 269 <span class="i">$ChainedLexer</span> = <span class="k">defined</span> <span class="i">$ChainedLexer</span> ? <span class="i">$This</span><span class="i">->_GenerateLexerForToken</span><span class="s">(</span><span class="i">$ChainedLexer</span><span class="cm">,</span> <span class="i">@</span>{<span class="i">$TokenSpecRef</span>}<span class="s">)</span> <span class="co">:</span> <span class="i">$This</span><span class="i">->_GenerateLexerForToken</span><span class="s">(</span><span class="i">$InputIteratorRef</span><span class="cm">,</span> <span class="i">@</span>{<span class="i">$TokenSpecRef</span>}<span class="s">)</span><span class="sc">;</span>
|
|
|
283 270 <span class="s">}</span>
|
|
|
284 271
|
|
|
285 272 <span class="i">$This</span>->{<span class="w">ChainedLexer</span>} = <span class="i">$ChainedLexer</span><span class="sc">;</span>
|
|
|
286 273
|
|
|
287 274 <span class="k">return</span> <span class="i">$This</span><span class="sc">;</span>
|
|
|
288 275 <span class="s">}</span>
|
|
|
289 276
|
|
|
290 277
|
|
|
291 278 <span class="c"># Generate a lexer using specifed token specification using specified input or</span>
|
|
|
292 279 <span class="c"># input retrieved using another token lexer. The lexer retrieving input from the</span>
|
|
|
293 280 <span class="c"># specified input stream is at the bottom of the chain.</span>
|
|
|
294 281 <span class="c">#</span>
|
|
|
295 <a name="_GenerateLexerForToken-"></a> 282 <span class="k">sub </span><span class="m">_GenerateLexerForToken</span> <span class="s">{</span>
|
|
|
296 283 <span class="k">my</span><span class="s">(</span><span class="i">$This</span><span class="cm">,</span> <span class="i">$InputIteratorOrLexer</span><span class="cm">,</span> <span class="i">$TokenLabel</span><span class="cm">,</span> <span class="i">$RegexPattern</span><span class="cm">,</span> <span class="i">$TokenMatchActionRef</span><span class="s">)</span> = <span class="i">@_</span><span class="sc">;</span>
|
|
|
297 284 <span class="k">my</span><span class="s">(</span><span class="i">$TokenMatchAndSplitRef</span><span class="cm">,</span> <span class="i">$InputBuffer</span><span class="cm">,</span> <span class="i">@ProcessedTokens</span><span class="s">)</span><span class="sc">;</span>
|
|
|
298 285
|
|
|
299 286 <span class="c"># Input buffer for a specific lexer in chained lexers containing unprocessed</span>
|
|
|
300 287 <span class="c"># text for token specifications retrieved from a downstrean lexer or intial</span>
|
|
|
301 288 <span class="c"># input...</span>
|
|
|
302 289 <span class="c">#</span>
|
|
|
303 290 <span class="i">$InputBuffer</span> = <span class="q">""</span><span class="sc">;</span>
|
|
|
304 291
|
|
|
305 292 <span class="c"># @ProcessedTokens contains either references to an array containing token label</span>
|
|
|
306 293 <span class="c"># and matched text or any unmatched input text string...</span>
|
|
|
307 294 <span class="c">#</span>
|
|
|
308 295 <span class="i">@ProcessedTokens</span> = <span class="s">(</span><span class="s">)</span><span class="sc">;</span>
|
|
|
309 296
|
|
|
310 297 <span class="c"># Setup a default annoymous function reference to generate an array reference</span>
|
|
|
311 298 <span class="c"># containing $Token and text matched to $RegexPattern.</span>
|
|
|
312 299 <span class="c">#</span>
|
|
|
313 300 <span class="i">$TokenMatchActionRef</span> = <span class="k">defined</span> <span class="i">$TokenMatchActionRef</span> ? <span class="i">$TokenMatchActionRef</span> <span class="co">:</span> <span class="k">sub</span> <span class="s">{</span> <span class="k">my</span><span class="s">(</span><span class="i">$This</span><span class="cm">,</span> <span class="i">$Label</span><span class="cm">,</span> <span class="i">$MatchedText</span><span class="s">)</span> = <span class="i">@_</span><span class="sc">;</span> <span class="k">return</span> <span class="s">[</span><span class="i">$Label</span><span class="cm">,</span> <span class="i">$MatchedText</span><span class="s">]</span> <span class="s">}</span><span class="sc">;</span>
|
|
|
314 301
|
|
|
315 302 <span class="c"># Setup an annoymous function to match and split input text using $RegexPattern for</span>
|
|
|
316 303 <span class="c"># a specific token during its lexer invocation in chained lexers.</span>
|
|
|
317 304 <span class="c">#</span>
|
|
|
318 305 <span class="c"># The usage of parenthesis around $RegexPattern during split allows capturing of matched</span>
|
|
|
319 306 <span class="c"># text, which is subsequently processed to retrieve matched $Token values. The split function</span>
|
|
|
320 307 <span class="c"># inserts a "" separator in the returned array as first entry whenever $InputText starts with</span>
|
|
|
321 308 <span class="c"># $RegexPattern. $InputText is returned as the only element for no match.</span>
|
|
|
322 309 <span class="c">#</span>
|
|
|
323 310 <span class="i">$TokenMatchAndSplitRef</span> = <span class="k">sub</span> <span class="s">{</span> <span class="k">my</span><span class="s">(</span><span class="i">$InputText</span><span class="s">)</span> = <span class="i">@_</span><span class="sc">;</span> <span class="k">return</span> <span class="k">split</span> <span class="q">/($RegexPattern)/</span><span class="cm">,</span> <span class="i">$InputText</span><span class="sc">;</span> <span class="s">}</span><span class="sc">;</span>
|
|
|
324 311
|
|
|
325 312 <span class="c"># Setup a lexer for $TokenLabel as an annoymous function and return its reference to caller</span>
|
|
|
326 313 <span class="c"># which in turns chains the lexers for all $Tokens before returning a reference to a lexer</span>
|
|
|
327 314 <span class="c"># at top of the lexer chain.</span>
|
|
|
328 315 <span class="c">#</span>
|
|
|
329 316 <span class="c"># Perl maintains scope of all variables defined with in the scope of the current function</span>
|
|
|
330 317 <span class="c"># during invocation of annoymous function even after the return call.</span>
|
|
|
331 318 <span class="c">#</span>
|
|
|
332 319 <span class="k">return</span> <span class="k">sub</span> <span class="s">{</span>
|
|
|
333 320 <span class="k">my</span><span class="s">(</span><span class="i">$Mode</span><span class="s">)</span> = <span class="i">@_</span><span class="sc">;</span>
|
|
|
334 321
|
|
|
335 322 <span class="c"># Currenly supported value for mode: Peek, Next</span>
|
|
|
336 323 <span class="c">#</span>
|
|
|
337 324 <span class="i">$Mode</span> = <span class="k">defined</span> <span class="i">$Mode</span> ? <span class="i">$Mode</span> <span class="co">:</span> <span class="q">'Next'</span><span class="sc">;</span>
|
|
|
338 325
|
|
|
339 326 <span class="k">while</span> <span class="s">(</span><span class="i">@ProcessedTokens</span> == <span class="n">0</span> && <span class="k">defined</span> <span class="i">$InputBuffer</span> <span class="s">)</span> <span class="s">{</span>
|
|
|
340 327 <span class="c"># Get any new input....</span>
|
|
|
341 328 <span class="k">my</span> <span class="i">$NewInput</span> = <span class="i">$InputIteratorOrLexer</span>-><span class="s">(</span><span class="s">)</span><span class="sc">;</span>
|
|
|
342 329
|
|
|
343 330 <span class="k">if</span> <span class="s">(</span><span class="k">ref</span> <span class="i">$NewInput</span><span class="s">)</span> <span class="s">{</span>
|
|
|
344 331 <span class="c"># Input is an array reference containing matched token and text returned by</span>
|
|
|
345 332 <span class="c"># a chained lexer downstream lexer...</span>
|
|
|
346 333 <span class="c">#</span>
|
|
|
347 334 <span class="c"># Match $RegexPattern in available buffer text to retieve any matched text</span>
|
|
|
348 335 <span class="c"># for current $Token. $Separator might be "": $RegexPattern is at start of</span>
|
|
|
349 336 <span class="c"># of $InputBuffer</span>
|
|
|
350 337 <span class="c">#</span>
|
|
|
351 338 <span class="c"># Process input buffer containing text to be matched for the current lexer</span>
|
|
|
352 339 <span class="c"># which didn't get processed earlier during @NewTokens > 2 while loop:</span>
|
|
|
353 340 <span class="c"># no match for current lexer or more input available. It maintains order</span>
|
|
|
354 341 <span class="c"># of token matching in input stream.</span>
|
|
|
355 342 <span class="c">#</span>
|
|
|
356 343 <span class="k">my</span><span class="s">(</span><span class="i">$Separator</span><span class="cm">,</span> <span class="i">$MatchedTokenRefOrText</span><span class="s">)</span><span class="sc">;</span>
|
|
|
357 344
|
|
|
358 345 <span class="s">(</span><span class="i">$Separator</span><span class="cm">,</span> <span class="i">$MatchedTokenRefOrText</span><span class="s">)</span> = <span class="i">$TokenMatchAndSplitRef</span>-><span class="s">(</span><span class="i">$InputBuffer</span><span class="s">)</span><span class="sc">;</span>
|
|
|
359 346 <span class="k">if</span> <span class="s">(</span><span class="k">defined</span> <span class="i">$MatchedTokenRefOrText</span><span class="s">)</span> <span class="s">{</span>
|
|
|
360 347 <span class="i">$MatchedTokenRefOrText</span> = <span class="i">$TokenMatchActionRef</span>-><span class="s">(</span><span class="i">$This</span><span class="cm">,</span> <span class="i">$TokenLabel</span><span class="cm">,</span> <span class="i">$MatchedTokenRefOrText</span><span class="s">)</span><span class="sc">;</span>
|
|
|
361 348 <span class="s">}</span>
|
|
|
362 349
|
|
|
363 350 <span class="c"># Collect valid token references or text...</span>
|
|
|
364 351 <span class="k">push</span> <span class="i">@ProcessedTokens</span><span class="cm">,</span> <span class="k">grep</span> <span class="s">{</span> <span class="k">defined</span> <span class="i">$_</span> && <span class="i">$_</span> <span class="k">ne</span> <span class="q">""</span> <span class="s">}</span> <span class="s">(</span><span class="i">$Separator</span><span class="cm">,</span> <span class="i">$MatchedTokenRefOrText</span><span class="cm">,</span> <span class="i">$NewInput</span><span class="s">)</span><span class="sc">;</span>
|
|
|
365 352
|
|
|
366 353 <span class="c"># Empty put buffer...</span>
|
|
|
367 354 <span class="i">$InputBuffer</span> = <span class="q">""</span><span class="sc">;</span>
|
|
|
368 355
|
|
|
369 356 <span class="c"># Get out of the loop as processed token refererences and/or text are available...</span>
|
|
|
370 357 <span class="k">last</span><span class="sc">;</span>
|
|
|
371 358 <span class="s">}</span>
|
|
|
372 359
|
|
|
373 360 <span class="c"># Process input retrieved from downstream lexer or input iterator which hasn't</span>
|
|
|
374 361 <span class="c"># been processed into tokens..</span>
|
|
|
375 362 <span class="k">if</span> <span class="s">(</span><span class="k">defined</span> <span class="i">$NewInput</span><span class="s">)</span> <span class="s">{</span>
|
|
|
376 363 <span class="i">$InputBuffer</span> .= <span class="i">$NewInput</span><span class="sc">;</span>
|
|
|
377 364 <span class="s">}</span>
|
|
|
378 365
|
|
|
379 366 <span class="c"># Retrieve any matched tokens from available input for the current lexer...</span>
|
|
|
380 367 <span class="c">#</span>
|
|
|
381 368 <span class="k">my</span><span class="s">(</span><span class="i">@NewTokens</span><span class="s">)</span> = <span class="i">$TokenMatchAndSplitRef</span>-><span class="s">(</span><span class="i">$InputBuffer</span><span class="s">)</span><span class="sc">;</span>
|
|
|
382 369
|
|
|
383 370 <span class="k">while</span> <span class="s">(</span> <span class="i">@NewTokens</span> > <span class="n">2</span> || <span class="i">@NewTokens</span> && !<span class="k">defined</span> <span class="i">$NewInput</span><span class="s">)</span> <span class="s">{</span>
|
|
|
384 371 <span class="c"># Scenario 1: Complete match</span>
|
|
|
385 372 <span class="c"># @NewTokens > 2 : Availability of separator, matched token text, separator.</span>
|
|
|
386 373 <span class="c"># The separator might correspond to token for a token for upstream lexer followed</span>
|
|
|
387 374 <span class="c"># by matched token from current lexer. It ends up getting passed to upsrteam</span>
|
|
|
388 375 <span class="c"># lexer for processing.</span>
|
|
|
389 376 <span class="c">#</span>
|
|
|
390 377 <span class="c"># Scenario 2: No more input available from iterator or downstream lexer</span>
|
|
|
391 378 <span class="c"># @NewTokens <= 2 and no more input implies any left over text in buffer. And</span>
|
|
|
392 379 <span class="c"># it ends up getting passed to upsrteam for processing.</span>
|
|
|
393 380 <span class="c">#</span>
|
|
|
394 381
|
|
|
395 382 <span class="c"># Take off any unprocessed input text that doesn't match off the buffer: It'll be</span>
|
|
|
396 383 <span class="c"># passed to upstream chained lexer for processing...</span>
|
|
|
397 384 <span class="c">#</span>
|
|
|
398 385 <span class="k">push</span> <span class="i">@ProcessedTokens</span><span class="cm">,</span> <span class="k">shift</span> <span class="i">@NewTokens</span><span class="sc">;</span>
|
|
|
399 386
|
|
|
400 387 <span class="k">if</span> <span class="s">(</span><span class="i">@NewTokens</span><span class="s">)</span> <span class="s">{</span>
|
|
|
401 388 <span class="k">my</span> <span class="i">$MatchedTokenText</span> = <span class="k">shift</span> <span class="i">@NewTokens</span><span class="sc">;</span>
|
|
|
402 389 <span class="k">push</span> <span class="i">@ProcessedTokens</span><span class="cm">,</span> <span class="i">$TokenMatchActionRef</span>-><span class="s">(</span><span class="i">$This</span><span class="cm">,</span> <span class="i">$TokenLabel</span><span class="cm">,</span> <span class="i">$MatchedTokenText</span><span class="s">)</span><span class="sc">;</span>
|
|
|
403 390 <span class="s">}</span>
|
|
|
404 391 <span class="s">}</span>
|
|
|
405 392
|
|
|
406 393 <span class="c"># Retrieve any leftover text from NewTokens and put it back into InputBuffer for</span>
|
|
|
407 394 <span class="c"># processing by current lexer. All token references have been taken out....</span>
|
|
|
408 395 <span class="c">#</span>
|
|
|
409 396 <span class="i">$InputBuffer</span> = <span class="q">""</span><span class="sc">;</span>
|
|
|
410 397 <span class="k">if</span> <span class="s">(</span><span class="i">@NewTokens</span><span class="s">)</span> <span class="s">{</span>
|
|
|
411 398 <span class="i">$InputBuffer</span> = <span class="k">join</span> <span class="q">""</span><span class="cm">,</span> <span class="i">@NewTokens</span><span class="sc">;</span>
|
|
|
412 399 <span class="s">}</span>
|
|
|
413 400
|
|
|
414 401 <span class="k">if</span> <span class="s">(</span>!<span class="k">defined</span> <span class="i">$NewInput</span><span class="s">)</span> <span class="s">{</span>
|
|
|
415 402 <span class="c"># No more input from the downstream lexer...</span>
|
|
|
416 403 <span class="i">$InputBuffer</span> = <span class="k">undef</span><span class="sc">;</span>
|
|
|
417 404 <span class="s">}</span>
|
|
|
418 405
|
|
|
419 406 <span class="c"># Clean up any empty strings from ProcessedTokens containing token</span>
|
|
|
420 407 <span class="c"># references or text...</span>
|
|
|
421 408 <span class="i">@ProcessedTokens</span> = <span class="k">grep</span> <span class="s">{</span> <span class="i">$_</span> <span class="k">ne</span> <span class="q">""</span> <span class="s">}</span> <span class="i">@ProcessedTokens</span><span class="sc">;</span>
|
|
|
422 409
|
|
|
423 410 <span class="s">}</span>
|
|
|
424 411
|
|
|
425 412 <span class="c"># Return reference to an array containing token and matched text or just unmatched input text...</span>
|
|
|
426 413 <span class="k">my</span> <span class="i">$TokenRefOrText</span> = <span class="k">undef</span><span class="sc">;</span>
|
|
|
427 414
|
|
|
428 415 <span class="k">if</span> <span class="s">(</span><span class="i">@ProcessedTokens</span><span class="s">)</span> <span class="s">{</span>
|
|
|
429 416 <span class="c"># Get first available reference either by just peeking or removing it from the list</span>
|
|
|
430 417 <span class="c"># of available tokens...</span>
|
|
|
431 418 <span class="i">$TokenRefOrText</span> = <span class="s">(</span><span class="i">$Mode</span> =~ <span class="q">/^Peek$/i</span><span class="s">)</span> ? <span class="i">$ProcessedTokens</span>[<span class="n">0</span>] <span class="co">:</span> <span class="k">shift</span> <span class="i">@ProcessedTokens</span><span class="sc">;</span>
|
|
|
432 419 <span class="s">}</span>
|
|
|
433 420
|
|
|
434 421 <span class="k">return</span> <span class="i">$TokenRefOrText</span><span class="sc">;</span>
|
|
|
435 422 <span class="s">}</span><span class="sc">;</span>
|
|
|
436 423 <span class="s">}</span>
|
|
|
437 424
|
|
|
438 425 <span class="c"># Is it a lexer object?</span>
|
|
|
439 <a name="_IsLexer-"></a> 426 <span class="k">sub </span><span class="m">_IsLexer</span> <span class="s">{</span>
|
|
|
440 427 <span class="k">my</span><span class="s">(</span><span class="i">$Object</span><span class="s">)</span> = <span class="i">@_</span><span class="sc">;</span>
|
|
|
441 428
|
|
|
442 429 <span class="k">return</span> <span class="s">(</span><span class="i">Scalar::Util::blessed</span><span class="s">(</span><span class="i">$Object</span><span class="s">)</span> && <span class="i">$Object</span><span class="i">->isa</span><span class="s">(</span><span class="i">$ClassName</span><span class="s">)</span><span class="s">)</span> ? <span class="n">1</span> <span class="co">:</span> <span class="n">0</span><span class="sc">;</span>
|
|
|
443 430 <span class="s">}</span>
|
|
|
444 431
|
|
|
445 432 <span class="c"># Return a string containing information about lexer...</span>
|
|
|
446 <a name="StringifyLexer-"></a> 433 <span class="k">sub </span><span class="m">StringifyLexer</span> <span class="s">{</span>
|
|
|
447 434 <span class="k">my</span><span class="s">(</span><span class="i">$This</span><span class="s">)</span> = <span class="i">@_</span><span class="sc">;</span>
|
|
|
448 435 <span class="k">my</span><span class="s">(</span><span class="i">$LexerString</span><span class="s">)</span><span class="sc">;</span>
|
|
|
449 436
|
|
|
450 437 <span class="i">$LexerString</span> = <span class="q">"Lexer: PackageName: $ClassName; "</span> . <span class="i">$This</span><span class="i">->_GetLexerInfoString</span><span class="s">(</span><span class="s">)</span><span class="sc">;</span>
|
|
|
451 438
|
|
|
452 439 <span class="k">return</span> <span class="i">$LexerString</span><span class="sc">;</span>
|
|
|
453 440 <span class="s">}</span>
|
|
|
454 441
|
|
|
455 442 <span class="c"># Return a string containing information about lexer...</span>
|
|
|
456 <a name="_GetLexerInfoString-"></a> 443 <span class="k">sub </span><span class="m">_GetLexerInfoString</span> <span class="s">{</span>
|
|
|
457 444 <span class="k">my</span><span class="s">(</span><span class="i">$This</span><span class="s">)</span> = <span class="i">@_</span><span class="sc">;</span>
|
|
|
458 445 <span class="k">my</span><span class="s">(</span><span class="i">$LexerInfoString</span><span class="cm">,</span> <span class="i">$TokensSpec</span><span class="cm">,</span> <span class="i">$TokenSpec</span><span class="cm">,</span> <span class="i">$TokenLabel</span><span class="cm">,</span> <span class="i">$TokenMatchRegex</span><span class="cm">,</span> <span class="i">$TokenMatchAction</span><span class="s">)</span><span class="sc">;</span>
|
|
|
459 446
|
|
|
460 447 <span class="i">$LexerInfoString</span> = <span class="q">"InputType: $This->{InputType}"</span><span class="sc">;</span>
|
|
|
461 448
|
|
|
462 449 <span class="k">if</span> <span class="s">(</span><span class="i">$This</span>->{<span class="w">InputType</span>} =~ <span class="q">/^String$/i</span><span class="s">)</span> <span class="s">{</span>
|
|
|
463 450 <span class="i">$LexerInfoString</span> .= <span class="q">"; InputString: $This->{Input}"</span><span class="sc">;</span>
|
|
|
464 451 <span class="s">}</span>
|
|
|
465 452
|
|
|
466 453 <span class="i">$TokensSpec</span> = <span class="q">"TokensSpecifications: <None>"</span><span class="sc">;</span>
|
|
|
467 454 <span class="k">if</span> <span class="s">(</span><span class="i">@</span>{<span class="i">$This</span>->{<span class="w">TokensSpec</span>}}<span class="s">)</span> <span class="s">{</span>
|
|
|
468 455 <span class="i">$TokensSpec</span> = <span class="q">"TokensSpecifications: < [Label, MatchRegex, MatchAction]:"</span><span class="sc">;</span>
|
|
|
469 456 <span class="k">for</span> <span class="i">$TokenSpec</span> <span class="s">(</span><span class="i">@</span>{<span class="i">$This</span>->{<span class="w">TokensSpec</span>}}<span class="s">)</span> <span class="s">{</span>
|
|
|
470 457 <span class="s">(</span><span class="i">$TokenLabel</span><span class="cm">,</span> <span class="i">$TokenMatchRegex</span><span class="s">)</span> = <span class="i">@</span>{<span class="i">$TokenSpec</span>}<span class="sc">;</span>
|
|
|
471 458 <span class="i">$TokenMatchAction</span> = <span class="s">(</span><span class="i">@</span>{<span class="i">$TokenSpec</span>} == <span class="n">3</span><span class="s">)</span> ? <span class="q">"$TokenSpec->[2]"</span> <span class="co">:</span> <span class="q">"undefined"</span><span class="sc">;</span>
|
|
|
472 459 <span class="i">$TokensSpec</span> .= <span class="q">" [$TokenLabel, $TokenMatchRegex, $TokenMatchAction]"</span><span class="sc">;</span>
|
|
|
473 460 <span class="s">}</span>
|
|
|
474 461 <span class="i">$TokensSpec</span> .= <span class="q">" >"</span><span class="sc">;</span>
|
|
|
475 462 <span class="s">}</span>
|
|
|
476 463
|
|
|
477 464 <span class="i">$LexerInfoString</span> .= <span class="q">"; $TokensSpec"</span><span class="sc">;</span>
|
|
|
478 465
|
|
|
479 466 <span class="k">return</span> <span class="i">$LexerInfoString</span><span class="sc">;</span>
|
|
|
480 467 <span class="s">}</span>
|
|
|
481 468
|
|
|
482 <a name="EOF-"></a></pre>
|
|
|
483 <p> </p>
|
|
|
484 <br />
|
|
|
485 <center>
|
|
|
486 <img src="../../../images/h2o2.png">
|
|
|
487 </center>
|
|
|
488 </body>
|
|
|
489 </html>
|
