1
2
3
4
5 package parse
6
7 import (
8 "fmt"
9 "testing"
10 )
11
12
13 var itemName = map[itemType]string{
14 itemError: "error",
15 itemBool: "bool",
16 itemChar: "char",
17 itemCharConstant: "charconst",
18 itemComment: "comment",
19 itemComplex: "complex",
20 itemDeclare: ":=",
21 itemEOF: "EOF",
22 itemField: "field",
23 itemIdentifier: "identifier",
24 itemLeftDelim: "left delim",
25 itemLeftParen: "(",
26 itemNumber: "number",
27 itemPipe: "pipe",
28 itemRawString: "raw string",
29 itemRightDelim: "right delim",
30 itemRightParen: ")",
31 itemSpace: "space",
32 itemString: "string",
33 itemVariable: "variable",
34
35
36 itemDot: ".",
37 itemBlock: "block",
38 itemDefine: "define",
39 itemElse: "else",
40 itemIf: "if",
41 itemEnd: "end",
42 itemNil: "nil",
43 itemRange: "range",
44 itemTemplate: "template",
45 itemWith: "with",
46 }
47
48 func (i itemType) String() string {
49 s := itemName[i]
50 if s == "" {
51 return fmt.Sprintf("item%d", int(i))
52 }
53 return s
54 }
55
56 type lexTest struct {
57 name string
58 input string
59 items []item
60 }
61
62 func mkItem(typ itemType, text string) item {
63 return item{
64 typ: typ,
65 val: text,
66 }
67 }
68
69 var (
70 tDot = mkItem(itemDot, ".")
71 tBlock = mkItem(itemBlock, "block")
72 tEOF = mkItem(itemEOF, "")
73 tFor = mkItem(itemIdentifier, "for")
74 tLeft = mkItem(itemLeftDelim, "{{")
75 tLpar = mkItem(itemLeftParen, "(")
76 tPipe = mkItem(itemPipe, "|")
77 tQuote = mkItem(itemString, `"abc \n\t\" "`)
78 tRange = mkItem(itemRange, "range")
79 tRight = mkItem(itemRightDelim, "}}")
80 tRpar = mkItem(itemRightParen, ")")
81 tSpace = mkItem(itemSpace, " ")
82 raw = "`" + `abc\n\t\" ` + "`"
83 rawNL = "`now is{{\n}}the time`"
84 tRawQuote = mkItem(itemRawString, raw)
85 tRawQuoteNL = mkItem(itemRawString, rawNL)
86 )
87
88 var lexTests = []lexTest{
89 {"empty", "", []item{tEOF}},
90 {"spaces", " \t\n", []item{mkItem(itemText, " \t\n"), tEOF}},
91 {"text", `now is the time`, []item{mkItem(itemText, "now is the time"), tEOF}},
92 {"text with comment", "hello-{{/* this is a comment */}}-world", []item{
93 mkItem(itemText, "hello-"),
94 mkItem(itemComment, "/* this is a comment */"),
95 mkItem(itemText, "-world"),
96 tEOF,
97 }},
98 {"punctuation", "{{,@% }}", []item{
99 tLeft,
100 mkItem(itemChar, ","),
101 mkItem(itemChar, "@"),
102 mkItem(itemChar, "%"),
103 tSpace,
104 tRight,
105 tEOF,
106 }},
107 {"parens", "{{((3))}}", []item{
108 tLeft,
109 tLpar,
110 tLpar,
111 mkItem(itemNumber, "3"),
112 tRpar,
113 tRpar,
114 tRight,
115 tEOF,
116 }},
117 {"empty action", `{{}}`, []item{tLeft, tRight, tEOF}},
118 {"for", `{{for}}`, []item{tLeft, tFor, tRight, tEOF}},
119 {"block", `{{block "foo" .}}`, []item{
120 tLeft, tBlock, tSpace, mkItem(itemString, `"foo"`), tSpace, tDot, tRight, tEOF,
121 }},
122 {"quote", `{{"abc \n\t\" "}}`, []item{tLeft, tQuote, tRight, tEOF}},
123 {"raw quote", "{{" + raw + "}}", []item{tLeft, tRawQuote, tRight, tEOF}},
124 {"raw quote with newline", "{{" + rawNL + "}}", []item{tLeft, tRawQuoteNL, tRight, tEOF}},
125 {"numbers", "{{1 02 0x14 0X14 -7.2i 1e3 1E3 +1.2e-4 4.2i 1+2i 1_2 0x1.e_fp4 0X1.E_FP4}}", []item{
126 tLeft,
127 mkItem(itemNumber, "1"),
128 tSpace,
129 mkItem(itemNumber, "02"),
130 tSpace,
131 mkItem(itemNumber, "0x14"),
132 tSpace,
133 mkItem(itemNumber, "0X14"),
134 tSpace,
135 mkItem(itemNumber, "-7.2i"),
136 tSpace,
137 mkItem(itemNumber, "1e3"),
138 tSpace,
139 mkItem(itemNumber, "1E3"),
140 tSpace,
141 mkItem(itemNumber, "+1.2e-4"),
142 tSpace,
143 mkItem(itemNumber, "4.2i"),
144 tSpace,
145 mkItem(itemComplex, "1+2i"),
146 tSpace,
147 mkItem(itemNumber, "1_2"),
148 tSpace,
149 mkItem(itemNumber, "0x1.e_fp4"),
150 tSpace,
151 mkItem(itemNumber, "0X1.E_FP4"),
152 tRight,
153 tEOF,
154 }},
155 {"characters", `{{'a' '\n' '\'' '\\' '\u00FF' '\xFF' '本'}}`, []item{
156 tLeft,
157 mkItem(itemCharConstant, `'a'`),
158 tSpace,
159 mkItem(itemCharConstant, `'\n'`),
160 tSpace,
161 mkItem(itemCharConstant, `'\''`),
162 tSpace,
163 mkItem(itemCharConstant, `'\\'`),
164 tSpace,
165 mkItem(itemCharConstant, `'\u00FF'`),
166 tSpace,
167 mkItem(itemCharConstant, `'\xFF'`),
168 tSpace,
169 mkItem(itemCharConstant, `'本'`),
170 tRight,
171 tEOF,
172 }},
173 {"bools", "{{true false}}", []item{
174 tLeft,
175 mkItem(itemBool, "true"),
176 tSpace,
177 mkItem(itemBool, "false"),
178 tRight,
179 tEOF,
180 }},
181 {"dot", "{{.}}", []item{
182 tLeft,
183 tDot,
184 tRight,
185 tEOF,
186 }},
187 {"nil", "{{nil}}", []item{
188 tLeft,
189 mkItem(itemNil, "nil"),
190 tRight,
191 tEOF,
192 }},
193 {"dots", "{{.x . .2 .x.y.z}}", []item{
194 tLeft,
195 mkItem(itemField, ".x"),
196 tSpace,
197 tDot,
198 tSpace,
199 mkItem(itemNumber, ".2"),
200 tSpace,
201 mkItem(itemField, ".x"),
202 mkItem(itemField, ".y"),
203 mkItem(itemField, ".z"),
204 tRight,
205 tEOF,
206 }},
207 {"keywords", "{{range if else end with}}", []item{
208 tLeft,
209 mkItem(itemRange, "range"),
210 tSpace,
211 mkItem(itemIf, "if"),
212 tSpace,
213 mkItem(itemElse, "else"),
214 tSpace,
215 mkItem(itemEnd, "end"),
216 tSpace,
217 mkItem(itemWith, "with"),
218 tRight,
219 tEOF,
220 }},
221 {"variables", "{{$c := printf $ $hello $23 $ $var.Field .Method}}", []item{
222 tLeft,
223 mkItem(itemVariable, "$c"),
224 tSpace,
225 mkItem(itemDeclare, ":="),
226 tSpace,
227 mkItem(itemIdentifier, "printf"),
228 tSpace,
229 mkItem(itemVariable, "$"),
230 tSpace,
231 mkItem(itemVariable, "$hello"),
232 tSpace,
233 mkItem(itemVariable, "$23"),
234 tSpace,
235 mkItem(itemVariable, "$"),
236 tSpace,
237 mkItem(itemVariable, "$var"),
238 mkItem(itemField, ".Field"),
239 tSpace,
240 mkItem(itemField, ".Method"),
241 tRight,
242 tEOF,
243 }},
244 {"variable invocation", "{{$x 23}}", []item{
245 tLeft,
246 mkItem(itemVariable, "$x"),
247 tSpace,
248 mkItem(itemNumber, "23"),
249 tRight,
250 tEOF,
251 }},
252 {"pipeline", `intro {{echo hi 1.2 |noargs|args 1 "hi"}} outro`, []item{
253 mkItem(itemText, "intro "),
254 tLeft,
255 mkItem(itemIdentifier, "echo"),
256 tSpace,
257 mkItem(itemIdentifier, "hi"),
258 tSpace,
259 mkItem(itemNumber, "1.2"),
260 tSpace,
261 tPipe,
262 mkItem(itemIdentifier, "noargs"),
263 tPipe,
264 mkItem(itemIdentifier, "args"),
265 tSpace,
266 mkItem(itemNumber, "1"),
267 tSpace,
268 mkItem(itemString, `"hi"`),
269 tRight,
270 mkItem(itemText, " outro"),
271 tEOF,
272 }},
273 {"declaration", "{{$v := 3}}", []item{
274 tLeft,
275 mkItem(itemVariable, "$v"),
276 tSpace,
277 mkItem(itemDeclare, ":="),
278 tSpace,
279 mkItem(itemNumber, "3"),
280 tRight,
281 tEOF,
282 }},
283 {"2 declarations", "{{$v , $w := 3}}", []item{
284 tLeft,
285 mkItem(itemVariable, "$v"),
286 tSpace,
287 mkItem(itemChar, ","),
288 tSpace,
289 mkItem(itemVariable, "$w"),
290 tSpace,
291 mkItem(itemDeclare, ":="),
292 tSpace,
293 mkItem(itemNumber, "3"),
294 tRight,
295 tEOF,
296 }},
297 {"field of parenthesized expression", "{{(.X).Y}}", []item{
298 tLeft,
299 tLpar,
300 mkItem(itemField, ".X"),
301 tRpar,
302 mkItem(itemField, ".Y"),
303 tRight,
304 tEOF,
305 }},
306 {"trimming spaces before and after", "hello- {{- 3 -}} -world", []item{
307 mkItem(itemText, "hello-"),
308 tLeft,
309 mkItem(itemNumber, "3"),
310 tRight,
311 mkItem(itemText, "-world"),
312 tEOF,
313 }},
314 {"trimming spaces before and after comment", "hello- {{- /* hello */ -}} -world", []item{
315 mkItem(itemText, "hello-"),
316 mkItem(itemComment, "/* hello */"),
317 mkItem(itemText, "-world"),
318 tEOF,
319 }},
320
321 {"badchar", "#{{\x01}}", []item{
322 mkItem(itemText, "#"),
323 tLeft,
324 mkItem(itemError, "unrecognized character in action: U+0001"),
325 }},
326 {"unclosed action", "{{", []item{
327 tLeft,
328 mkItem(itemError, "unclosed action"),
329 }},
330 {"EOF in action", "{{range", []item{
331 tLeft,
332 tRange,
333 mkItem(itemError, "unclosed action"),
334 }},
335 {"unclosed quote", "{{\"\n\"}}", []item{
336 tLeft,
337 mkItem(itemError, "unterminated quoted string"),
338 }},
339 {"unclosed raw quote", "{{`xx}}", []item{
340 tLeft,
341 mkItem(itemError, "unterminated raw quoted string"),
342 }},
343 {"unclosed char constant", "{{'\n}}", []item{
344 tLeft,
345 mkItem(itemError, "unterminated character constant"),
346 }},
347 {"bad number", "{{3k}}", []item{
348 tLeft,
349 mkItem(itemError, `bad number syntax: "3k"`),
350 }},
351 {"unclosed paren", "{{(3}}", []item{
352 tLeft,
353 tLpar,
354 mkItem(itemNumber, "3"),
355 mkItem(itemError, `unclosed left paren`),
356 }},
357 {"extra right paren", "{{3)}}", []item{
358 tLeft,
359 mkItem(itemNumber, "3"),
360 tRpar,
361 mkItem(itemError, `unexpected right paren U+0029 ')'`),
362 }},
363
364
365
366
367 {"long pipeline deadlock", "{{|||||}}", []item{
368 tLeft,
369 tPipe,
370 tPipe,
371 tPipe,
372 tPipe,
373 tPipe,
374 tRight,
375 tEOF,
376 }},
377 {"text with bad comment", "hello-{{/*/}}-world", []item{
378 mkItem(itemText, "hello-"),
379 mkItem(itemError, `unclosed comment`),
380 }},
381 {"text with comment close separated from delim", "hello-{{/* */ }}-world", []item{
382 mkItem(itemText, "hello-"),
383 mkItem(itemError, `comment ends before closing delimiter`),
384 }},
385
386
387 {"unmatched right delimiter", "hello-{.}}-world", []item{
388 mkItem(itemText, "hello-{.}}-world"),
389 tEOF,
390 }},
391 }
392
393
394 func collect(t *lexTest, left, right string) (items []item) {
395 l := lex(t.name, t.input, left, right, true)
396 for {
397 item := l.nextItem()
398 items = append(items, item)
399 if item.typ == itemEOF || item.typ == itemError {
400 break
401 }
402 }
403 return
404 }
405
406 func equal(i1, i2 []item, checkPos bool) bool {
407 if len(i1) != len(i2) {
408 return false
409 }
410 for k := range i1 {
411 if i1[k].typ != i2[k].typ {
412 return false
413 }
414 if i1[k].val != i2[k].val {
415 return false
416 }
417 if checkPos && i1[k].pos != i2[k].pos {
418 return false
419 }
420 if checkPos && i1[k].line != i2[k].line {
421 return false
422 }
423 }
424 return true
425 }
426
427 func TestLex(t *testing.T) {
428 for _, test := range lexTests {
429 items := collect(&test, "", "")
430 if !equal(items, test.items, false) {
431 t.Errorf("%s: got\n\t%+v\nexpected\n\t%v", test.name, items, test.items)
432 }
433 }
434 }
435
436
437 var lexDelimTests = []lexTest{
438 {"punctuation", "$$,@%{{}}@@", []item{
439 tLeftDelim,
440 mkItem(itemChar, ","),
441 mkItem(itemChar, "@"),
442 mkItem(itemChar, "%"),
443 mkItem(itemChar, "{"),
444 mkItem(itemChar, "{"),
445 mkItem(itemChar, "}"),
446 mkItem(itemChar, "}"),
447 tRightDelim,
448 tEOF,
449 }},
450 {"empty action", `$$@@`, []item{tLeftDelim, tRightDelim, tEOF}},
451 {"for", `$$for@@`, []item{tLeftDelim, tFor, tRightDelim, tEOF}},
452 {"quote", `$$"abc \n\t\" "@@`, []item{tLeftDelim, tQuote, tRightDelim, tEOF}},
453 {"raw quote", "$$" + raw + "@@", []item{tLeftDelim, tRawQuote, tRightDelim, tEOF}},
454 }
455
456 var (
457 tLeftDelim = mkItem(itemLeftDelim, "$$")
458 tRightDelim = mkItem(itemRightDelim, "@@")
459 )
460
461 func TestDelims(t *testing.T) {
462 for _, test := range lexDelimTests {
463 items := collect(&test, "$$", "@@")
464 if !equal(items, test.items, false) {
465 t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items)
466 }
467 }
468 }
469
470 var lexPosTests = []lexTest{
471 {"empty", "", []item{{itemEOF, 0, "", 1}}},
472 {"punctuation", "{{,@%#}}", []item{
473 {itemLeftDelim, 0, "{{", 1},
474 {itemChar, 2, ",", 1},
475 {itemChar, 3, "@", 1},
476 {itemChar, 4, "%", 1},
477 {itemChar, 5, "#", 1},
478 {itemRightDelim, 6, "}}", 1},
479 {itemEOF, 8, "", 1},
480 }},
481 {"sample", "0123{{hello}}xyz", []item{
482 {itemText, 0, "0123", 1},
483 {itemLeftDelim, 4, "{{", 1},
484 {itemIdentifier, 6, "hello", 1},
485 {itemRightDelim, 11, "}}", 1},
486 {itemText, 13, "xyz", 1},
487 {itemEOF, 16, "", 1},
488 }},
489 {"trimafter", "{{x -}}\n{{y}}", []item{
490 {itemLeftDelim, 0, "{{", 1},
491 {itemIdentifier, 2, "x", 1},
492 {itemRightDelim, 5, "}}", 1},
493 {itemLeftDelim, 8, "{{", 2},
494 {itemIdentifier, 10, "y", 2},
495 {itemRightDelim, 11, "}}", 2},
496 {itemEOF, 13, "", 2},
497 }},
498 {"trimbefore", "{{x}}\n{{- y}}", []item{
499 {itemLeftDelim, 0, "{{", 1},
500 {itemIdentifier, 2, "x", 1},
501 {itemRightDelim, 3, "}}", 1},
502 {itemLeftDelim, 6, "{{", 2},
503 {itemIdentifier, 10, "y", 2},
504 {itemRightDelim, 11, "}}", 2},
505 {itemEOF, 13, "", 2},
506 }},
507 }
508
509
510
511 func TestPos(t *testing.T) {
512 for _, test := range lexPosTests {
513 items := collect(&test, "", "")
514 if !equal(items, test.items, true) {
515 t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items)
516 if len(items) == len(test.items) {
517
518 for i := range items {
519 if !equal(items[i:i+1], test.items[i:i+1], true) {
520 i1 := items[i]
521 i2 := test.items[i]
522 t.Errorf("\t#%d: got {%v %d %q %d} expected {%v %d %q %d}",
523 i, i1.typ, i1.pos, i1.val, i1.line, i2.typ, i2.pos, i2.val, i2.line)
524 }
525 }
526 }
527 }
528 }
529 }
530
531
532 func TestShutdown(t *testing.T) {
533
534 const text = "erroneous{{define}}{{else}}1234"
535 lexer := lex("foo", text, "{{", "}}", false)
536 _, err := New("root").parseLexer(lexer)
537 if err == nil {
538 t.Fatalf("expected error")
539 }
540
541 token, ok := <-lexer.items
542 if ok {
543 t.Fatalf("input was not drained; got %v", token)
544 }
545 }
546
547
548
549 func (t *Tree) parseLexer(lex *lexer) (tree *Tree, err error) {
550 defer t.recover(&err)
551 t.ParseName = t.Name
552 t.startParse(nil, lex, map[string]*Tree{})
553 t.parse()
554 t.add()
555 t.stopParse()
556 return t, nil
557 }
558
View as plain text