Mercurial > repos > deepakjadmin > mayatool3_test2

comparison docs/modules/html/code/Lexer.html @ 0:4816e4a8ae95 draft default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Uploaded
author deepakjadmin
date Wed, 20 Jan 2016 09:23:18 -0500
parents
children
comparison
equal deleted inserted replaced
-1:000000000000 0:4816e4a8ae95
1 <html>
2 <head>
3 <title>MayaChemTools:Code:Parsers::Lexer.pm</title>
4 <meta http-equiv="content-type" content="text/html;charset=utf-8">
5 <link rel="stylesheet" type="text/css" href="../../../css/MayaChemToolsCode.css">
6 </head>
7 <body leftmargin="20" rightmargin="20" topmargin="10" bottommargin="10">
8 <br/>
9 <center>
10 <a href="http://www.mayachemtools.org" title="MayaChemTools Home"><img src="../../../images/MayaChemToolsLogo.gif" border="0" alt="MayaChemTools"></a>
11 </center>
12 <br/>
13 <pre>
14 <a name="package-Parsers::Lexer-"></a> 1 <span class="k">package </span><span class="i">Parsers::Lexer</span><span class="sc">;</span>
15 2 <span class="c">#</span>
16 3 <span class="c"># $RCSfile: Lexer.pm,v $</span>
17 4 <span class="c"># $Date: 2015/02/28 20:50:55 $</span>
18 5 <span class="c"># $Revision: 1.10 $</span>
19 6 <span class="c">#</span>
20 7 <span class="c"># Author: Manish Sud &lt;msud@san.rr.com&gt;</span>
21 8 <span class="c">#</span>
22 9 <span class="c"># Copyright (C) 2015 Manish Sud. All rights reserved.</span>
23 10 <span class="c">#</span>
24 11 <span class="c"># This file is part of MayaChemTools.</span>
25 12 <span class="c">#</span>
26 13 <span class="c"># MayaChemTools is free software; you can redistribute it and/or modify it under</span>
27 14 <span class="c"># the terms of the GNU Lesser General Public License as published by the Free</span>
28 15 <span class="c"># Software Foundation; either version 3 of the License, or (at your option) any</span>
29 16 <span class="c"># later version.</span>
30 17 <span class="c">#</span>
31 18 <span class="c"># MayaChemTools is distributed in the hope that it will be useful, but without</span>
32 19 <span class="c"># any warranty; without even the implied warranty of merchantability of fitness</span>
33 20 <span class="c"># for a particular purpose. See the GNU Lesser General Public License for more</span>
34 21 <span class="c"># details.</span>
35 22 <span class="c">#</span>
36 23 <span class="c"># You should have received a copy of the GNU Lesser General Public License</span>
37 24 <span class="c"># along with MayaChemTools; if not, see &lt;http://www.gnu.org/licenses/&gt; or</span>
38 25 <span class="c"># write to the Free Software Foundation Inc., 59 Temple Place, Suite 330,</span>
39 26 <span class="c"># Boston, MA, 02111-1307, USA.</span>
40 27 <span class="c">#</span>
41 28
42 29 <span class="k">use</span> <span class="w">strict</span><span class="sc">;</span>
43 30 <span class="k">use</span> <span class="w">Carp</span><span class="sc">;</span>
44 31 <span class="k">use</span> <span class="w">Exporter</span><span class="sc">;</span>
45 32 <span class="k">use</span> <span class="w">Scalar::Util</span> <span class="s">(</span><span class="s">)</span><span class="sc">;</span>
46 33
47 34 <span class="k">use</span> <span class="w">vars</span> <span class="q">qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS)</span><span class="sc">;</span>
48 35
49 36 <span class="i">@ISA</span> = <span class="q">qw(Exporter)</span><span class="sc">;</span>
50 37 <span class="i">@EXPORT</span> = <span class="q">qw()</span><span class="sc">;</span>
51 38 <span class="i">@EXPORT_OK</span> = <span class="q">qw()</span><span class="sc">;</span>
52 39
53 40 <span class="i">%EXPORT_TAGS</span> = <span class="s">(</span><span class="w">all</span> <span class="cm">=&gt;</span> <span class="s">[</span><span class="i">@EXPORT</span><span class="cm">,</span> <span class="i">@EXPORT_OK</span><span class="s">]</span><span class="s">)</span><span class="sc">;</span>
54 41
55 42 <span class="c"># Setup class variables...</span>
56 43 <span class="k">my</span><span class="s">(</span><span class="i">$ClassName</span><span class="s">)</span><span class="sc">;</span>
57 44 <span class="i">_InitializeClass</span><span class="s">(</span><span class="s">)</span><span class="sc">;</span>
58 45
59 46 <span class="c"># Overload Perl functions...</span>
60 47 <span class="k">use</span> <span class="w">overload</span> <span class="q">&#39;&quot;&quot;&#39;</span> <span class="cm">=&gt;</span> <span class="q">&#39;StringifyLexer&#39;</span><span class="sc">;</span>
61 48
62 49 <span class="c"># Class constructor...</span>
63 <a name="new-"></a> 50 <span class="k">sub </span><span class="m">new</span> <span class="s">{</span>
64 51 <span class="k">my</span><span class="s">(</span><span class="i">$Class</span><span class="cm">,</span> <span class="i">$Input</span><span class="cm">,</span> <span class="i">@TokensSpec</span><span class="s">)</span> = <span class="i">@_</span><span class="sc">;</span>
65 52
66 53 <span class="c"># Initialize object...</span>
67 54 <span class="k">my</span> <span class="i">$This</span> = <span class="s">{</span><span class="s">}</span><span class="sc">;</span>
68 55 <span class="k">bless</span> <span class="i">$This</span><span class="cm">,</span> <span class="k">ref</span><span class="s">(</span><span class="i">$Class</span><span class="s">)</span> || <span class="i">$Class</span><span class="sc">;</span>
69 56 <span class="i">$This</span><span class="i">-&gt;_InitializeLexer</span><span class="s">(</span><span class="s">)</span><span class="sc">;</span>
70 57
71 58 <span class="i">$This</span><span class="i">-&gt;_ValidateParametersAndGenerateLexer</span><span class="s">(</span><span class="i">$Input</span><span class="cm">,</span> <span class="i">@TokensSpec</span><span class="s">)</span><span class="sc">;</span>
72 59
73 60 <span class="k">return</span> <span class="i">$This</span><span class="sc">;</span>
74 61 <span class="s">}</span>
75 62
76 63
77 64 <span class="c"># Initialize class ...</span>
78 <a name="_InitializeClass-"></a> 65 <span class="k">sub </span><span class="m">_InitializeClass</span> <span class="s">{</span>
79 66 <span class="c">#Class name...</span>
80 67 <span class="i">$ClassName</span> = <span class="w">__PACKAGE__</span><span class="sc">;</span>
81 68 <span class="s">}</span>
82 69
83 70 <span class="c"># Initialize object data...</span>
84 71 <span class="c">#</span>
85 <a name="_InitializeLexer-"></a> 72 <span class="k">sub </span><span class="m">_InitializeLexer</span> <span class="s">{</span>
86 73 <span class="k">my</span><span class="s">(</span><span class="i">$This</span><span class="s">)</span> = <span class="i">@_</span><span class="sc">;</span>
87 74
88 75 <span class="c"># Input parameter used by lexer to retrieve text to be lexed. Supported parameter types:</span>
89 76 <span class="c"># . Reference to input iterator function</span>
90 77 <span class="c"># . Reference to an open file handle</span>
91 78 <span class="c"># . Text string</span>
92 79 <span class="c">#</span>
93 80 <span class="i">$This</span>-&gt;{<span class="w">Input</span>} = <span class="k">undef</span><span class="sc">;</span>
94 81
95 82 <span class="c"># Type of input paramater determined using Perl ref function:</span>
96 83 <span class="c"># . InputIterator - ref returns CODE</span>
97 84 <span class="c"># . FileStream - ref return GLOB and fileno is valid</span>
98 85 <span class="c"># . String - ref return an empty string</span>
99 86 <span class="c">#</span>
100 87 <span class="i">$This</span>-&gt;{<span class="w">InputType</span>} = <span class="q">&#39;&#39;</span><span class="sc">;</span>
101 88
102 89 <span class="c"># Tokens specifications supplied by the caller. It&#39;s an array containing references</span>
103 90 <span class="c"># to arrays with each containing TokenLabel and TokenMatchRegex pair along with</span>
104 91 <span class="c"># an option reference to code to be executed after a matched.</span>
105 92 <span class="c">#</span>
106 93 <span class="c"># For example:</span>
107 94 <span class="c">#</span>
108 95 <span class="c"># @LexerTokensSpec = (</span>
109 96 <span class="c"># [ &#39;LETTER&#39;, qr/[a-zA-Z]/ ],</span>
110 97 <span class="c"># [ &#39;NUMBER&#39;, qr/\d+/ ],</span>
111 98 <span class="c"># [ &#39;SPACE&#39;, qr/[ ]*/, sub { my($This, $TokenLabel, $MatchedText) = @_; return &#39;&#39;; } ],</span>
112 99 <span class="c"># [ &#39;NEWLINE&#39;, qr/(?:\r\n|\r|\n)/, sub { my($This, $TokenLabel, $MatchedText) = @_; return &quot;\n&quot;; } ],</span>
113 100 <span class="c"># [ &#39;CHAR&#39;, qr/[\.]/ ],</span>
114 101 <span class="c"># );</span>
115 102 <span class="c">#</span>
116 103 <span class="i">@</span>{<span class="i">$This</span>-&gt;{<span class="w">TokensSpec</span>}} = <span class="s">(</span><span class="s">)</span><span class="sc">;</span>
117 104
118 105 <span class="c"># Refernce to chained lexer...</span>
119 106 <span class="i">$This</span>-&gt;{<span class="w">ChainedLexer</span>} = <span class="k">undef</span><span class="sc">;</span>
120 107
121 108 <span class="k">return</span> <span class="i">$This</span><span class="sc">;</span>
122 109 <span class="s">}</span>
123 110
124 111 <span class="c"># Validate input parameters and generate a chained lexer...</span>
125 112 <span class="c">#</span>
126 <a name="_ValidateParametersAndGenerateLexer-"></a> 113 <span class="k">sub </span><span class="m">_ValidateParametersAndGenerateLexer</span> <span class="s">{</span>
127 114 <span class="k">my</span><span class="s">(</span><span class="i">$This</span><span class="cm">,</span> <span class="i">$Input</span><span class="cm">,</span> <span class="i">@TokensSpec</span><span class="s">)</span> = <span class="i">@_</span><span class="sc">;</span>
128 115
129 116 <span class="c">#</span>
130 117 <span class="c"># Validate input to be lexed...</span>
131 118 <span class="k">if</span> <span class="s">(</span>!<span class="k">defined</span> <span class="i">$Input</span><span class="s">)</span> <span class="s">{</span>
132 119 <span class="w">croak</span> <span class="q">&quot;Error: ${ClassName}-&gt;new: Object can&#39;t be instantiated: Input is not defined. Supported values: a reference to input iterator function, a reference to an open file handle or a text string...&quot;</span><span class="sc">;</span>
133 120 <span class="s">}</span>
134 121 <span class="i">$This</span>-&gt;{<span class="w">Input</span>} = <span class="i">$Input</span><span class="sc">;</span>
135 122
136 123 <span class="c"># Check input parameter type...</span>
137 124 <span class="k">my</span><span class="s">(</span><span class="i">$InputType</span><span class="s">)</span><span class="sc">;</span>
138 125
139 126 <span class="i">$InputType</span> = <span class="k">ref</span> <span class="i">$Input</span><span class="sc">;</span>
140 127 <span class="k">if</span> <span class="s">(</span><span class="i">$InputType</span> =~ <span class="q">/CODE/i</span><span class="s">)</span> <span class="s">{</span>
141 128 <span class="c"># Input iterator...</span>
142 129 <span class="i">$This</span>-&gt;{<span class="w">InputType</span>} = <span class="q">&quot;InputIterator&quot;</span><span class="sc">;</span>
143 130 <span class="s">}</span>
144 131 <span class="k">elsif</span> <span class="s">(</span><span class="i">$InputType</span> =~ <span class="q">/GLOB/i</span> &amp;&amp; <span class="k">defined</span> <span class="k">fileno</span> <span class="i">$Input</span><span class="s">)</span> <span class="s">{</span>
145 132 <span class="c"># Input stream...</span>
146 133 <span class="i">$This</span>-&gt;{<span class="w">InputType</span>} = <span class="q">&quot;FileStream&quot;</span><span class="sc">;</span>
147 134 <span class="s">}</span>
148 135 <span class="k">elsif</span> <span class="s">(</span><span class="i">$InputType</span><span class="s">)</span> <span class="s">{</span>
149 136 <span class="c"># Perl ref function returns nonempty string for all other references...</span>
150 137 <span class="w">croak</span> <span class="q">&quot;Error: ${ClassName}-&gt;new: Object can&#39;t be instantiated: Invalid input parameter type specified. Supported parameter types: a reference to input iterator function, a reference to an open file handle or a text string...&quot;</span><span class="sc">;</span>
151 138 <span class="s">}</span>
152 139 <span class="k">else</span> <span class="s">{</span>
153 140 <span class="c"># Input string...</span>
154 141 <span class="i">$This</span>-&gt;{<span class="w">InputType</span>} = <span class="q">&quot;String&quot;</span><span class="sc">;</span>
155 142 <span class="s">}</span>
156 143
157 144 <span class="c"># Check tokens specifications...</span>
158 145 <span class="k">if</span> <span class="s">(</span>!<span class="i">@TokensSpec</span><span class="s">)</span> <span class="s">{</span>
159 146 <span class="w">croak</span> <span class="q">&quot;Error: ${ClassName}-&gt;new: TokensSpec is not defined or the array doesn&#39;t contain any values. Supported values: a reference to an array containg token label, regular expression to match and an option reference to function to modify matched values...&quot;</span><span class="sc">;</span>
160 147 <span class="s">}</span>
161 148 <span class="i">@</span>{<span class="i">$This</span>-&gt;{<span class="w">TokensSpec</span>}} = <span class="i">@TokensSpec</span><span class="sc">;</span>
162 149
163 150 <span class="i">$This</span><span class="i">-&gt;_GenerateLexer</span><span class="s">(</span><span class="i">$Input</span><span class="cm">,</span> <span class="i">@TokensSpec</span><span class="s">)</span><span class="sc">;</span>
164 151
165 152 <span class="k">return</span> <span class="i">$This</span><span class="sc">;</span>
166 153 <span class="s">}</span>
167 154
168 155 <span class="c"># Generate a lexer using reference to an input iterator function, an open file</span>
169 156 <span class="c"># handle or an input string passed as first parameter by the caller along</span>
170 157 <span class="c"># with token specifications as second paramater...</span>
171 158 <span class="c">#</span>
172 <a name="_GenerateLexer-"></a> 159 <span class="k">sub </span><span class="m">_GenerateLexer</span> <span class="s">{</span>
173 160 <span class="k">my</span><span class="s">(</span><span class="i">$This</span><span class="cm">,</span> <span class="i">$Input</span><span class="cm">,</span> <span class="i">@TokensSpec</span><span class="s">)</span> = <span class="i">@_</span><span class="sc">;</span>
174 161
175 162 <span class="k">if</span> <span class="s">(</span><span class="i">$This</span>-&gt;{<span class="w">InputType</span>} =~ <span class="q">/^InputIterator$/i</span><span class="s">)</span> <span class="s">{</span>
176 163 <span class="i">$This</span><span class="i">-&gt;_GenerateInputIteratorLexer</span><span class="s">(</span><span class="i">$Input</span><span class="cm">,</span> <span class="i">@TokensSpec</span><span class="s">)</span><span class="sc">;</span>
177 164 <span class="s">}</span>
178 165 <span class="k">elsif</span> <span class="s">(</span><span class="i">$This</span>-&gt;{<span class="w">InputType</span>} =~ <span class="q">/^FileStream$/i</span><span class="s">)</span> <span class="s">{</span>
179 166 <span class="i">$This</span><span class="i">-&gt;_GenerateInputFileStreamLexer</span><span class="s">(</span><span class="i">$Input</span><span class="cm">,</span> <span class="i">@TokensSpec</span><span class="s">)</span><span class="sc">;</span>
180 167 <span class="s">}</span>
181 168 <span class="k">elsif</span> <span class="s">(</span><span class="i">$This</span>-&gt;{<span class="w">InputType</span>} =~ <span class="q">/^String$/i</span><span class="s">)</span> <span class="s">{</span>
182 169 <span class="i">$This</span><span class="i">-&gt;_GenerateInputStringLexer</span><span class="s">(</span><span class="i">$Input</span><span class="cm">,</span> <span class="i">@TokensSpec</span><span class="s">)</span><span class="sc">;</span>
183 170 <span class="s">}</span>
184 171 <span class="k">else</span> <span class="s">{</span>
185 172 <span class="w">croak</span> <span class="q">&quot;Error: ${ClassName}-&gt;new: Object can&#39;t be instantiated: Invalid input parameter type specified. Supported parameter types: a reference to input iterator function, a reference to an open file handle or a text string...&quot;</span><span class="sc">;</span>
186 173 <span class="s">}</span>
187 174
188 175 <span class="k">return</span> <span class="i">$This</span><span class="sc">;</span>
189 176 <span class="s">}</span>
190 177
191 178 <span class="c"># Generate a lexer using specifed input iterator...</span>
192 179 <span class="c">#</span>
193 <a name="_GenerateInputIteratorLexer-"></a> 180 <span class="k">sub </span><span class="m">_GenerateInputIteratorLexer</span> <span class="s">{</span>
194 181 <span class="k">my</span><span class="s">(</span><span class="i">$This</span><span class="cm">,</span> <span class="i">$InputIteratorRef</span><span class="cm">,</span> <span class="i">@TokensSpec</span><span class="s">)</span> = <span class="i">@_</span><span class="sc">;</span>
195 182
196 183 <span class="i">$This</span><span class="i">-&gt;_GenerateChainedLexer</span><span class="s">(</span><span class="i">$InputIteratorRef</span><span class="cm">,</span> <span class="i">@TokensSpec</span><span class="s">)</span><span class="sc">;</span>
197 184
198 185 <span class="k">return</span> <span class="i">$This</span><span class="sc">;</span>
199 186 <span class="s">}</span>
200 187
201 188 <span class="c"># Generate a lexer using specifed input file stream reference...</span>
202 189 <span class="c">#</span>
203 <a name="_GenerateInputFileStreamLexer-"></a> 190 <span class="k">sub </span><span class="m">_GenerateInputFileStreamLexer</span> <span class="s">{</span>
204 191 <span class="k">my</span><span class="s">(</span><span class="i">$This</span><span class="cm">,</span> <span class="i">$FileHandleRef</span><span class="cm">,</span> <span class="i">@TokensSpec</span><span class="s">)</span> = <span class="i">@_</span><span class="sc">;</span>
205 192
206 193 <span class="c"># Iterator is a annoymous function reference and Perl keeps $FileHandleRef</span>
207 194 <span class="c"># in scope during its execution.</span>
208 195
209 196 <span class="i">$This</span><span class="i">-&gt;_GenerateChainedLexer</span><span class="s">(</span> <span class="k">sub</span> <span class="s">{</span> <span class="k">return</span> <span class="q">&lt;$FileHandleRef&gt;</span><span class="sc">;</span> <span class="s">}</span><span class="cm">,</span> <span class="i">@TokensSpec</span><span class="s">)</span><span class="sc">;</span>
210 197
211 198 <span class="k">return</span> <span class="i">$This</span><span class="sc">;</span>
212 199 <span class="s">}</span>
213 200
214 201 <span class="c"># Generate a lexer using specifed input string...</span>
215 202 <span class="c">#</span>
216 <a name="_GenerateInputStringLexer-"></a> 203 <span class="k">sub </span><span class="m">_GenerateInputStringLexer</span> <span class="s">{</span>
217 204 <span class="k">my</span><span class="s">(</span><span class="i">$This</span><span class="cm">,</span> <span class="i">$Text</span><span class="cm">,</span> <span class="i">@TokensSpec</span><span class="s">)</span> = <span class="i">@_</span><span class="sc">;</span>
218 205 <span class="k">my</span><span class="s">(</span><span class="i">@InputText</span><span class="s">)</span> = <span class="s">(</span><span class="i">$Text</span><span class="s">)</span><span class="sc">;</span>
219 206
220 207 <span class="c"># Iterator is a annoymous function reference and Perl keeps @InputText</span>
221 208 <span class="c"># in scope during its execution.</span>
222 209
223 210 <span class="i">$This</span><span class="i">-&gt;_GenerateChainedLexer</span><span class="s">(</span> <span class="k">sub</span> <span class="s">{</span> <span class="k">return</span> <span class="k">shift</span> <span class="i">@InputText</span><span class="sc">;</span> <span class="s">}</span><span class="cm">,</span> <span class="i">@TokensSpec</span><span class="s">)</span><span class="sc">;</span>
224 211
225 212 <span class="k">return</span> <span class="i">$This</span><span class="sc">;</span>
226 213 <span class="s">}</span>
227 214
228 215 <span class="c"># Get next available token label and value pair as an array reference or unrecognized</span>
229 216 <span class="c"># text from input stream by either removing it from the input or simply peeking ahead...</span>
230 217 <span class="c">#</span>
231 218 <span class="c"># Supported mode values: Peek, Next. Default: Next</span>
232 219 <span class="c">#</span>
233 <a name="Lex-"></a> 220 <span class="k">sub </span><span class="m">Lex</span> <span class="s">{</span>
234 221 <span class="k">my</span><span class="s">(</span><span class="i">$This</span><span class="cm">,</span> <span class="i">$Mode</span><span class="s">)</span> = <span class="i">@_</span><span class="sc">;</span>
235 222
236 223 <span class="k">return</span> <span class="i">$This</span>-&gt;{<span class="w">ChainedLexer</span>}-&gt;<span class="s">(</span><span class="i">$Mode</span><span class="s">)</span>
237 224 <span class="s">}</span>
238 225
239 226 <span class="c"># Get next available token label and value pair as an array reference or unrecognized</span>
240 227 <span class="c"># text from input stream by either removing it from the input stream...</span>
241 228 <span class="c">#</span>
242 <a name="Next-"></a> 229 <span class="k">sub </span><span class="m">Next</span> <span class="s">{</span>
243 230 <span class="k">my</span><span class="s">(</span><span class="i">$This</span><span class="s">)</span> = <span class="i">@_</span><span class="sc">;</span>
244 231
245 232 <span class="k">return</span> <span class="i">$This</span><span class="i">-&gt;Lex</span><span class="s">(</span><span class="s">)</span><span class="sc">;</span>
246 233 <span class="s">}</span>
247 234
248 235 <span class="c"># Get next available token label and value pair as an array reference or unrecognized</span>
249 236 <span class="c"># text from input stream by simply peeking ahead and without removing it from the input</span>
250 237 <span class="c"># stream..</span>
251 238 <span class="c">#</span>
252 <a name="Peek-"></a> 239 <span class="k">sub </span><span class="m">Peek</span> <span class="s">{</span>
253 240 <span class="k">my</span><span class="s">(</span><span class="i">$This</span><span class="s">)</span> = <span class="i">@_</span><span class="sc">;</span>
254 241
255 242 <span class="k">return</span> <span class="i">$This</span><span class="i">-&gt;Lex</span><span class="s">(</span><span class="q">&#39;Peek&#39;</span><span class="s">)</span>
256 243 <span class="s">}</span>
257 244
258 245 <span class="c"># Get a reference to lexer method to be used by the caller...</span>
259 246 <span class="c">#</span>
260 <a name="GetLex-"></a> 247 <span class="k">sub </span><span class="m">GetLex</span> <span class="s">{</span>
261 248 <span class="k">my</span><span class="s">(</span><span class="i">$This</span><span class="s">)</span> = <span class="i">@_</span><span class="sc">;</span>
262 249
263 250 <span class="k">return</span> <span class="k">sub</span> <span class="s">{</span> <span class="i">$This</span><span class="i">-&gt;Lex</span><span class="s">(</span><span class="s">)</span><span class="sc">;</span> <span class="s">}</span><span class="sc">;</span>
264 251 <span class="s">}</span>
265 252
266 253 <span class="c"># The chained lexer generation is implemented based on examples in Higher-order Perl</span>
267 254 <span class="c"># [ Ref 126 ] book.</span>
268 255 <span class="c">#</span>
269 256 <span class="c"># Generate a lexer using specified input iterator and chaining it with other lexers generated</span>
270 257 <span class="c"># for all token specifications. The lexer generated for first token specification uses input</span>
271 258 <span class="c"># iterator to retrieve any available input text; the subsequent chained lexeres for rest</span>
272 259 <span class="c"># of the tokens use lexers generated for previous token specifications to get next input</span>
273 260 <span class="c"># which might be unmatched input text or a reference to an array containing token and</span>
274 261 <span class="c"># matched text pair.</span>
275 262 <span class="c">#</span>
276 <a name="_GenerateChainedLexer-"></a> 263 <span class="k">sub </span><span class="m">_GenerateChainedLexer</span> <span class="s">{</span>
277 264 <span class="k">my</span><span class="s">(</span><span class="i">$This</span><span class="cm">,</span> <span class="i">$InputIteratorRef</span><span class="cm">,</span> <span class="i">@TokensSpec</span><span class="s">)</span> = <span class="i">@_</span><span class="sc">;</span>
278 265 <span class="k">my</span><span class="s">(</span><span class="i">$TokenSpecRef</span><span class="cm">,</span> <span class="i">$ChainedLexer</span><span class="s">)</span><span class="sc">;</span>
279 266
280 267 <span class="i">$ChainedLexer</span> = <span class="k">undef</span><span class="sc">;</span>
281 268 <span class="k">for</span> <span class="i">$TokenSpecRef</span> <span class="s">(</span><span class="i">@TokensSpec</span><span class="s">)</span> <span class="s">{</span>
282 269 <span class="i">$ChainedLexer</span> = <span class="k">defined</span> <span class="i">$ChainedLexer</span> ? <span class="i">$This</span><span class="i">-&gt;_GenerateLexerForToken</span><span class="s">(</span><span class="i">$ChainedLexer</span><span class="cm">,</span> <span class="i">@</span>{<span class="i">$TokenSpecRef</span>}<span class="s">)</span> <span class="co">:</span> <span class="i">$This</span><span class="i">-&gt;_GenerateLexerForToken</span><span class="s">(</span><span class="i">$InputIteratorRef</span><span class="cm">,</span> <span class="i">@</span>{<span class="i">$TokenSpecRef</span>}<span class="s">)</span><span class="sc">;</span>
283 270 <span class="s">}</span>
284 271
285 272 <span class="i">$This</span>-&gt;{<span class="w">ChainedLexer</span>} = <span class="i">$ChainedLexer</span><span class="sc">;</span>
286 273
287 274 <span class="k">return</span> <span class="i">$This</span><span class="sc">;</span>
288 275 <span class="s">}</span>
289 276
290 277
291 278 <span class="c"># Generate a lexer using specifed token specification using specified input or</span>
292 279 <span class="c"># input retrieved using another token lexer. The lexer retrieving input from the</span>
293 280 <span class="c"># specified input stream is at the bottom of the chain.</span>
294 281 <span class="c">#</span>
295 <a name="_GenerateLexerForToken-"></a> 282 <span class="k">sub </span><span class="m">_GenerateLexerForToken</span> <span class="s">{</span>
296 283 <span class="k">my</span><span class="s">(</span><span class="i">$This</span><span class="cm">,</span> <span class="i">$InputIteratorOrLexer</span><span class="cm">,</span> <span class="i">$TokenLabel</span><span class="cm">,</span> <span class="i">$RegexPattern</span><span class="cm">,</span> <span class="i">$TokenMatchActionRef</span><span class="s">)</span> = <span class="i">@_</span><span class="sc">;</span>
297 284 <span class="k">my</span><span class="s">(</span><span class="i">$TokenMatchAndSplitRef</span><span class="cm">,</span> <span class="i">$InputBuffer</span><span class="cm">,</span> <span class="i">@ProcessedTokens</span><span class="s">)</span><span class="sc">;</span>
298 285
299 286 <span class="c"># Input buffer for a specific lexer in chained lexers containing unprocessed</span>
300 287 <span class="c"># text for token specifications retrieved from a downstrean lexer or intial</span>
301 288 <span class="c"># input...</span>
302 289 <span class="c">#</span>
303 290 <span class="i">$InputBuffer</span> = <span class="q">&quot;&quot;</span><span class="sc">;</span>
304 291
305 292 <span class="c"># @ProcessedTokens contains either references to an array containing token label</span>
306 293 <span class="c"># and matched text or any unmatched input text string...</span>
307 294 <span class="c">#</span>
308 295 <span class="i">@ProcessedTokens</span> = <span class="s">(</span><span class="s">)</span><span class="sc">;</span>
309 296
310 297 <span class="c"># Setup a default annoymous function reference to generate an array reference</span>
311 298 <span class="c"># containing $Token and text matched to $RegexPattern.</span>
312 299 <span class="c">#</span>
313 300 <span class="i">$TokenMatchActionRef</span> = <span class="k">defined</span> <span class="i">$TokenMatchActionRef</span> ? <span class="i">$TokenMatchActionRef</span> <span class="co">:</span> <span class="k">sub</span> <span class="s">{</span> <span class="k">my</span><span class="s">(</span><span class="i">$This</span><span class="cm">,</span> <span class="i">$Label</span><span class="cm">,</span> <span class="i">$MatchedText</span><span class="s">)</span> = <span class="i">@_</span><span class="sc">;</span> <span class="k">return</span> <span class="s">[</span><span class="i">$Label</span><span class="cm">,</span> <span class="i">$MatchedText</span><span class="s">]</span> <span class="s">}</span><span class="sc">;</span>
314 301
315 302 <span class="c"># Setup an annoymous function to match and split input text using $RegexPattern for</span>
316 303 <span class="c"># a specific token during its lexer invocation in chained lexers.</span>
317 304 <span class="c">#</span>
318 305 <span class="c"># The usage of parenthesis around $RegexPattern during split allows capturing of matched</span>
319 306 <span class="c"># text, which is subsequently processed to retrieve matched $Token values. The split function</span>
320 307 <span class="c"># inserts a &quot;&quot; separator in the returned array as first entry whenever $InputText starts with</span>
321 308 <span class="c"># $RegexPattern. $InputText is returned as the only element for no match.</span>
322 309 <span class="c">#</span>
323 310 <span class="i">$TokenMatchAndSplitRef</span> = <span class="k">sub</span> <span class="s">{</span> <span class="k">my</span><span class="s">(</span><span class="i">$InputText</span><span class="s">)</span> = <span class="i">@_</span><span class="sc">;</span> <span class="k">return</span> <span class="k">split</span> <span class="q">/($RegexPattern)/</span><span class="cm">,</span> <span class="i">$InputText</span><span class="sc">;</span> <span class="s">}</span><span class="sc">;</span>
324 311
325 312 <span class="c"># Setup a lexer for $TokenLabel as an annoymous function and return its reference to caller</span>
326 313 <span class="c"># which in turns chains the lexers for all $Tokens before returning a reference to a lexer</span>
327 314 <span class="c"># at top of the lexer chain.</span>
328 315 <span class="c">#</span>
329 316 <span class="c"># Perl maintains scope of all variables defined with in the scope of the current function</span>
330 317 <span class="c"># during invocation of annoymous function even after the return call.</span>
331 318 <span class="c">#</span>
332 319 <span class="k">return</span> <span class="k">sub</span> <span class="s">{</span>
333 320 <span class="k">my</span><span class="s">(</span><span class="i">$Mode</span><span class="s">)</span> = <span class="i">@_</span><span class="sc">;</span>
334 321
335 322 <span class="c"># Currenly supported value for mode: Peek, Next</span>
336 323 <span class="c">#</span>
337 324 <span class="i">$Mode</span> = <span class="k">defined</span> <span class="i">$Mode</span> ? <span class="i">$Mode</span> <span class="co">:</span> <span class="q">&#39;Next&#39;</span><span class="sc">;</span>
338 325
339 326 <span class="k">while</span> <span class="s">(</span><span class="i">@ProcessedTokens</span> == <span class="n">0</span> &amp;&amp; <span class="k">defined</span> <span class="i">$InputBuffer</span> <span class="s">)</span> <span class="s">{</span>
340 327 <span class="c"># Get any new input....</span>
341 328 <span class="k">my</span> <span class="i">$NewInput</span> = <span class="i">$InputIteratorOrLexer</span>-&gt;<span class="s">(</span><span class="s">)</span><span class="sc">;</span>
342 329
343 330 <span class="k">if</span> <span class="s">(</span><span class="k">ref</span> <span class="i">$NewInput</span><span class="s">)</span> <span class="s">{</span>
344 331 <span class="c"># Input is an array reference containing matched token and text returned by</span>
345 332 <span class="c"># a chained lexer downstream lexer...</span>
346 333 <span class="c">#</span>
347 334 <span class="c"># Match $RegexPattern in available buffer text to retieve any matched text</span>
348 335 <span class="c"># for current $Token. $Separator might be &quot;&quot;: $RegexPattern is at start of</span>
349 336 <span class="c"># of $InputBuffer</span>
350 337 <span class="c">#</span>
351 338 <span class="c"># Process input buffer containing text to be matched for the current lexer</span>
352 339 <span class="c"># which didn&#39;t get processed earlier during @NewTokens &gt; 2 while loop:</span>
353 340 <span class="c"># no match for current lexer or more input available. It maintains order</span>
354 341 <span class="c"># of token matching in input stream.</span>
355 342 <span class="c">#</span>
356 343 <span class="k">my</span><span class="s">(</span><span class="i">$Separator</span><span class="cm">,</span> <span class="i">$MatchedTokenRefOrText</span><span class="s">)</span><span class="sc">;</span>
357 344
358 345 <span class="s">(</span><span class="i">$Separator</span><span class="cm">,</span> <span class="i">$MatchedTokenRefOrText</span><span class="s">)</span> = <span class="i">$TokenMatchAndSplitRef</span>-&gt;<span class="s">(</span><span class="i">$InputBuffer</span><span class="s">)</span><span class="sc">;</span>
359 346 <span class="k">if</span> <span class="s">(</span><span class="k">defined</span> <span class="i">$MatchedTokenRefOrText</span><span class="s">)</span> <span class="s">{</span>
360 347 <span class="i">$MatchedTokenRefOrText</span> = <span class="i">$TokenMatchActionRef</span>-&gt;<span class="s">(</span><span class="i">$This</span><span class="cm">,</span> <span class="i">$TokenLabel</span><span class="cm">,</span> <span class="i">$MatchedTokenRefOrText</span><span class="s">)</span><span class="sc">;</span>
361 348 <span class="s">}</span>
362 349
363 350 <span class="c"># Collect valid token references or text...</span>
364 351 <span class="k">push</span> <span class="i">@ProcessedTokens</span><span class="cm">,</span> <span class="k">grep</span> <span class="s">{</span> <span class="k">defined</span> <span class="i">$_</span> &amp;&amp; <span class="i">$_</span> <span class="k">ne</span> <span class="q">&quot;&quot;</span> <span class="s">}</span> <span class="s">(</span><span class="i">$Separator</span><span class="cm">,</span> <span class="i">$MatchedTokenRefOrText</span><span class="cm">,</span> <span class="i">$NewInput</span><span class="s">)</span><span class="sc">;</span>
365 352
366 353 <span class="c"># Empty put buffer...</span>
367 354 <span class="i">$InputBuffer</span> = <span class="q">&quot;&quot;</span><span class="sc">;</span>
368 355
369 356 <span class="c"># Get out of the loop as processed token refererences and/or text are available...</span>
370 357 <span class="k">last</span><span class="sc">;</span>
371 358 <span class="s">}</span>
372 359
373 360 <span class="c"># Process input retrieved from downstream lexer or input iterator which hasn&#39;t</span>
374 361 <span class="c"># been processed into tokens..</span>
375 362 <span class="k">if</span> <span class="s">(</span><span class="k">defined</span> <span class="i">$NewInput</span><span class="s">)</span> <span class="s">{</span>
376 363 <span class="i">$InputBuffer</span> .= <span class="i">$NewInput</span><span class="sc">;</span>
377 364 <span class="s">}</span>
378 365
379 366 <span class="c"># Retrieve any matched tokens from available input for the current lexer...</span>
380 367 <span class="c">#</span>
381 368 <span class="k">my</span><span class="s">(</span><span class="i">@NewTokens</span><span class="s">)</span> = <span class="i">$TokenMatchAndSplitRef</span>-&gt;<span class="s">(</span><span class="i">$InputBuffer</span><span class="s">)</span><span class="sc">;</span>
382 369
383 370 <span class="k">while</span> <span class="s">(</span> <span class="i">@NewTokens</span> &gt; <span class="n">2</span> || <span class="i">@NewTokens</span> &amp;&amp; !<span class="k">defined</span> <span class="i">$NewInput</span><span class="s">)</span> <span class="s">{</span>
384 371 <span class="c"># Scenario 1: Complete match</span>
385 372 <span class="c"># @NewTokens &gt; 2 : Availability of separator, matched token text, separator.</span>
386 373 <span class="c"># The separator might correspond to token for a token for upstream lexer followed</span>
387 374 <span class="c"># by matched token from current lexer. It ends up getting passed to upsrteam</span>
388 375 <span class="c"># lexer for processing.</span>
389 376 <span class="c">#</span>
390 377 <span class="c"># Scenario 2: No more input available from iterator or downstream lexer</span>
391 378 <span class="c"># @NewTokens &lt;= 2 and no more input implies any left over text in buffer. And</span>
392 379 <span class="c"># it ends up getting passed to upsrteam for processing.</span>
393 380 <span class="c">#</span>
394 381
395 382 <span class="c"># Take off any unprocessed input text that doesn&#39;t match off the buffer: It&#39;ll be</span>
396 383 <span class="c"># passed to upstream chained lexer for processing...</span>
397 384 <span class="c">#</span>
398 385 <span class="k">push</span> <span class="i">@ProcessedTokens</span><span class="cm">,</span> <span class="k">shift</span> <span class="i">@NewTokens</span><span class="sc">;</span>
399 386
400 387 <span class="k">if</span> <span class="s">(</span><span class="i">@NewTokens</span><span class="s">)</span> <span class="s">{</span>
401 388 <span class="k">my</span> <span class="i">$MatchedTokenText</span> = <span class="k">shift</span> <span class="i">@NewTokens</span><span class="sc">;</span>
402 389 <span class="k">push</span> <span class="i">@ProcessedTokens</span><span class="cm">,</span> <span class="i">$TokenMatchActionRef</span>-&gt;<span class="s">(</span><span class="i">$This</span><span class="cm">,</span> <span class="i">$TokenLabel</span><span class="cm">,</span> <span class="i">$MatchedTokenText</span><span class="s">)</span><span class="sc">;</span>
403 390 <span class="s">}</span>
404 391 <span class="s">}</span>
405 392
406 393 <span class="c"># Retrieve any leftover text from NewTokens and put it back into InputBuffer for</span>
407 394 <span class="c"># processing by current lexer. All token references have been taken out....</span>
408 395 <span class="c">#</span>
409 396 <span class="i">$InputBuffer</span> = <span class="q">&quot;&quot;</span><span class="sc">;</span>
410 397 <span class="k">if</span> <span class="s">(</span><span class="i">@NewTokens</span><span class="s">)</span> <span class="s">{</span>
411 398 <span class="i">$InputBuffer</span> = <span class="k">join</span> <span class="q">&quot;&quot;</span><span class="cm">,</span> <span class="i">@NewTokens</span><span class="sc">;</span>
412 399 <span class="s">}</span>
413 400
414 401 <span class="k">if</span> <span class="s">(</span>!<span class="k">defined</span> <span class="i">$NewInput</span><span class="s">)</span> <span class="s">{</span>
415 402 <span class="c"># No more input from the downstream lexer...</span>
416 403 <span class="i">$InputBuffer</span> = <span class="k">undef</span><span class="sc">;</span>
417 404 <span class="s">}</span>
418 405
419 406 <span class="c"># Clean up any empty strings from ProcessedTokens containing token</span>
420 407 <span class="c"># references or text...</span>
421 408 <span class="i">@ProcessedTokens</span> = <span class="k">grep</span> <span class="s">{</span> <span class="i">$_</span> <span class="k">ne</span> <span class="q">&quot;&quot;</span> <span class="s">}</span> <span class="i">@ProcessedTokens</span><span class="sc">;</span>
422 409
423 410 <span class="s">}</span>
424 411
425 412 <span class="c"># Return reference to an array containing token and matched text or just unmatched input text...</span>
426 413 <span class="k">my</span> <span class="i">$TokenRefOrText</span> = <span class="k">undef</span><span class="sc">;</span>
427 414
428 415 <span class="k">if</span> <span class="s">(</span><span class="i">@ProcessedTokens</span><span class="s">)</span> <span class="s">{</span>
429 416 <span class="c"># Get first available reference either by just peeking or removing it from the list</span>
430 417 <span class="c"># of available tokens...</span>
431 418 <span class="i">$TokenRefOrText</span> = <span class="s">(</span><span class="i">$Mode</span> =~ <span class="q">/^Peek$/i</span><span class="s">)</span> ? <span class="i">$ProcessedTokens</span>[<span class="n">0</span>] <span class="co">:</span> <span class="k">shift</span> <span class="i">@ProcessedTokens</span><span class="sc">;</span>
432 419 <span class="s">}</span>
433 420
434 421 <span class="k">return</span> <span class="i">$TokenRefOrText</span><span class="sc">;</span>
435 422 <span class="s">}</span><span class="sc">;</span>
436 423 <span class="s">}</span>
437 424
438 425 <span class="c"># Is it a lexer object?</span>
439 <a name="_IsLexer-"></a> 426 <span class="k">sub </span><span class="m">_IsLexer</span> <span class="s">{</span>
440 427 <span class="k">my</span><span class="s">(</span><span class="i">$Object</span><span class="s">)</span> = <span class="i">@_</span><span class="sc">;</span>
441 428
442 429 <span class="k">return</span> <span class="s">(</span><span class="i">Scalar::Util::blessed</span><span class="s">(</span><span class="i">$Object</span><span class="s">)</span> &amp;&amp; <span class="i">$Object</span><span class="i">-&gt;isa</span><span class="s">(</span><span class="i">$ClassName</span><span class="s">)</span><span class="s">)</span> ? <span class="n">1</span> <span class="co">:</span> <span class="n">0</span><span class="sc">;</span>
443 430 <span class="s">}</span>
444 431
445 432 <span class="c"># Return a string containing information about lexer...</span>
446 <a name="StringifyLexer-"></a> 433 <span class="k">sub </span><span class="m">StringifyLexer</span> <span class="s">{</span>
447 434 <span class="k">my</span><span class="s">(</span><span class="i">$This</span><span class="s">)</span> = <span class="i">@_</span><span class="sc">;</span>
448 435 <span class="k">my</span><span class="s">(</span><span class="i">$LexerString</span><span class="s">)</span><span class="sc">;</span>
449 436
450 437 <span class="i">$LexerString</span> = <span class="q">&quot;Lexer: PackageName: $ClassName; &quot;</span> . <span class="i">$This</span><span class="i">-&gt;_GetLexerInfoString</span><span class="s">(</span><span class="s">)</span><span class="sc">;</span>
451 438
452 439 <span class="k">return</span> <span class="i">$LexerString</span><span class="sc">;</span>
453 440 <span class="s">}</span>
454 441
455 442 <span class="c"># Return a string containing information about lexer...</span>
456 <a name="_GetLexerInfoString-"></a> 443 <span class="k">sub </span><span class="m">_GetLexerInfoString</span> <span class="s">{</span>
457 444 <span class="k">my</span><span class="s">(</span><span class="i">$This</span><span class="s">)</span> = <span class="i">@_</span><span class="sc">;</span>
458 445 <span class="k">my</span><span class="s">(</span><span class="i">$LexerInfoString</span><span class="cm">,</span> <span class="i">$TokensSpec</span><span class="cm">,</span> <span class="i">$TokenSpec</span><span class="cm">,</span> <span class="i">$TokenLabel</span><span class="cm">,</span> <span class="i">$TokenMatchRegex</span><span class="cm">,</span> <span class="i">$TokenMatchAction</span><span class="s">)</span><span class="sc">;</span>
459 446
460 447 <span class="i">$LexerInfoString</span> = <span class="q">&quot;InputType: $This-&gt;{InputType}&quot;</span><span class="sc">;</span>
461 448
462 449 <span class="k">if</span> <span class="s">(</span><span class="i">$This</span>-&gt;{<span class="w">InputType</span>} =~ <span class="q">/^String$/i</span><span class="s">)</span> <span class="s">{</span>
463 450 <span class="i">$LexerInfoString</span> .= <span class="q">&quot;; InputString: $This-&gt;{Input}&quot;</span><span class="sc">;</span>
464 451 <span class="s">}</span>
465 452
466 453 <span class="i">$TokensSpec</span> = <span class="q">&quot;TokensSpecifications: &lt;None&gt;&quot;</span><span class="sc">;</span>
467 454 <span class="k">if</span> <span class="s">(</span><span class="i">@</span>{<span class="i">$This</span>-&gt;{<span class="w">TokensSpec</span>}}<span class="s">)</span> <span class="s">{</span>
468 455 <span class="i">$TokensSpec</span> = <span class="q">&quot;TokensSpecifications: &lt; [Label, MatchRegex, MatchAction]:&quot;</span><span class="sc">;</span>
469 456 <span class="k">for</span> <span class="i">$TokenSpec</span> <span class="s">(</span><span class="i">@</span>{<span class="i">$This</span>-&gt;{<span class="w">TokensSpec</span>}}<span class="s">)</span> <span class="s">{</span>
470 457 <span class="s">(</span><span class="i">$TokenLabel</span><span class="cm">,</span> <span class="i">$TokenMatchRegex</span><span class="s">)</span> = <span class="i">@</span>{<span class="i">$TokenSpec</span>}<span class="sc">;</span>
471 458 <span class="i">$TokenMatchAction</span> = <span class="s">(</span><span class="i">@</span>{<span class="i">$TokenSpec</span>} == <span class="n">3</span><span class="s">)</span> ? <span class="q">&quot;$TokenSpec-&gt;[2]&quot;</span> <span class="co">:</span> <span class="q">&quot;undefined&quot;</span><span class="sc">;</span>
472 459 <span class="i">$TokensSpec</span> .= <span class="q">&quot; [$TokenLabel, $TokenMatchRegex, $TokenMatchAction]&quot;</span><span class="sc">;</span>
473 460 <span class="s">}</span>
474 461 <span class="i">$TokensSpec</span> .= <span class="q">&quot; &gt;&quot;</span><span class="sc">;</span>
475 462 <span class="s">}</span>
476 463
477 464 <span class="i">$LexerInfoString</span> .= <span class="q">&quot;; $TokensSpec&quot;</span><span class="sc">;</span>
478 465
479 466 <span class="k">return</span> <span class="i">$LexerInfoString</span><span class="sc">;</span>
480 467 <span class="s">}</span>
481 468
482 <a name="EOF-"></a></pre>
483 <p>&nbsp;</p>
484 <br />
485 <center>
486 <img src="../../../images/h2o2.png">
487 </center>
488 </body>
489 </html>