feat: Implement the Wings tokenizer (#4)

Produces essentially the same token stream as the that found in
   x3d_import, except includes comment and whitespace tokens, which
   can be ignored in the parser.
   Resolves #3.

Reviewed-on: #4
Co-authored-by: Glen Whitney <glen@studioinfinity.org>
Co-committed-by: Glen Whitney <glen@studioinfinity.org>

src/example.civet

@@ -1,2 +1,550 @@
 {convert} from ./index.js
-console.log convert 'while (10) cows\nmoo'
+hartPoly := `
+#VRML V1.0 ascii
+
+Separator {
+  DEF Title Info {
+    string "test"
+    }
+  DEF SceneInfo Info {
+    string "(c) George W. Hart, 1996.  george@li.net"
+    string "Dept. Computer Science, Hofstra University."
+    string "Freely distributable for noncommercial purposes."
+    }
+  DEF BackgroundColor Info { # blue sky
+    string "0.2 0.5 1"
+    }
+  DEF Viewer Info {
+    string "examiner"
+    }
+  ShapeHints {
+    vertexOrdering UNKNOWN_ORDERING
+    creaseAngle    0
+    }
+  DirectionalLight {
+    direction -.5 -1 0
+    intensity 0.75
+    }
+  DirectionalLight {
+    direction .5 1 0
+    intensity 0.75
+    }
+  Separator {
+    Coordinate3 {
+	point [
+	-4.2301 0 0.081329,
+	-4.0293 -0.61803 1.1328,
+	-4.0293 0.61803 1.1328,
+	-3.9982 -0.71364 -0.90092,
+	-3.9982 0.71364 -0.90092,
+	-3.7663 0 -1.8832,
+	-3.6541 -1.7727 0.90091,
+	-3.6541 1.7727 0.90091,
+	-3.623 -1.8683 -1.1328,
+	-3.623 1.8683 -1.1328,
+	-3.4222 -1.0591 2.115,
+	-3.4222 1.0591 2.115,
+	-3.4222 -2.4864 -0.08133,
+	-3.4222 2.4864 -0.08133,
+	-3.3911 -1.1547 -2.115,
+	-3.3911 1.1547 -2.115,
+	-3.047 -2.2138 1.8832,
+	-3.047 2.2138 1.8832,
+	-3.0159 0 -2.8654,
+	-2.8965 2.8683 -1.1328,
+	-2.8965 -2.8683 -1.1328,
+	-2.8151 -0.61803 3.0973,
+	-2.8151 0.61803 3.0973,
+	-2.8151 -2.9274 0.90092,
+	-2.8151 2.9274 0.90092,
+	-2.6408 -1.1547 -3.0973,
+	-2.6408 1.1547 -3.0973,
+	-2.44 -1.7727 2.8654,
+	-2.44 1.7727 2.8654,
+	-2.1461 -2.8683 -2.115,
+	-2.1461 2.8683 -2.115,
+	-2.0648 -2.9274 2.115,
+	-2.0648 2.9274 2.115,
+	-1.9645 0 3.7471,
+	-1.9142 3.582 -0.90092,
+	-1.9142 -3.582 -0.90092,
+	-1.9142 2.1547 -3.0973,
+	-1.9142 -2.1547 -3.0973,
+	-1.8329 3.6411 1.1328,
+	-1.8329 -3.6411 1.1328,
+	-1.5893 1.1547 -3.7471,
+	-1.5893 -1.1547 -3.7471,
+	-1.4577 2.4864 3.0973,
+	-1.4577 -2.4864 3.0973,
+	-1.3072 4.023 0.08133,
+	-1.3072 -4.023 0.08133,
+	-1.2141 0 -3.979,
+	-1.1639 3.582 -1.8832,
+	-1.1639 -3.582 -1.8832,
+	-0.98225 -0.71364 3.979,
+	-0.98225 0.71364 3.979,
+	-0.93198 2.8683 -2.8654,
+	-0.93198 -2.8683 -2.8654,
+	-0.65733 4.023 1.1328,
+	-0.65733 -4.023 1.1328,
+	-0.60706 1.8683 3.7471,
+	-0.60706 -1.8683 3.7471,
+	-0.5568 4.023 -0.90092,
+	-0.5568 -4.023 -0.90092,
+	-0.37518 1.1547 -3.979,
+	-0.37518 -1.1547 -3.979,
+	-0.28214 2.8683 3.0973,
+	-0.28214 -2.8683 3.0973,
+	-0.050265 3.582 2.115,
+	-0.050265 -3.582 2.115,
+	0 0 4.2109,
+	0 0 -4.2109,
+	0.050265 3.582 -2.115,
+	0.050265 -3.582 -2.115,
+	0.28214 2.8683 -3.0973,
+	0.28214 -2.8683 -3.0973,
+	0.37518 1.1547 3.979,
+	0.37518 -1.1547 3.979,
+	0.5568 4.023 0.90092,
+	0.5568 -4.023 0.90092,
+	0.60706 1.8683 -3.7471,
+	0.60706 -1.8683 -3.7471,
+	0.65733 4.023 -1.1328,
+	0.65733 -4.023 -1.1328,
+	0.93198 2.8683 2.8654,
+	0.93198 -2.8683 2.8654,
+	0.98225 -0.71364 -3.979,
+	0.98225 0.71364 -3.979,
+	1.1639 3.582 1.8832,
+	1.1639 -3.582 1.8832,
+	1.2141 0 3.979,
+	1.3072 4.023 -0.08133,
+	1.3072 -4.023 -0.08133,
+	1.4577 2.4864 -3.0973,
+	1.4577 -2.4864 -3.0973,
+	1.5893 1.1547 3.7471,
+	1.5893 -1.1547 3.7471,
+	1.8329 3.6411 -1.1328,
+	1.8329 -3.6411 -1.1328,
+	1.9142 2.1547 3.0973,
+	1.9142 -2.1547 3.0973,
+	1.9142 3.582 0.90092,
+	1.9142 -3.582 0.90092,
+	1.9645 0 -3.7471,
+	2.0648 -2.9274 -2.115,
+	2.0648 2.9274 -2.115,
+	2.1461 -2.8683 2.115,
+	2.1461 2.8683 2.115,
+	2.44 -1.7727 -2.8654,
+	2.44 1.7727 -2.8654,
+	2.6408 -1.1547 3.0973,
+	2.6408 1.1547 3.0973,
+	2.8151 -2.9274 -0.90092,
+	2.8151 2.9274 -0.90092,
+	2.8151 -0.61803 -3.0973,
+	2.8151 0.61803 -3.0973,
+	2.8965 2.8683 1.1328,
+	2.8965 -2.8683 1.1328,
+	3.0159 0 2.8654,
+	3.047 -2.2138 -1.8832,
+	3.047 2.2138 -1.8832,
+	3.3911 -1.1547 2.115,
+	3.3911 1.1547 2.115,
+	3.4222 -2.4864 0.08133,
+	3.4222 2.4864 0.08133,
+	3.4222 -1.0591 -2.115,
+	3.4222 1.0591 -2.115,
+	3.623 -1.8683 1.1328,
+	3.623 1.8683 1.1328,
+	3.6541 -1.7727 -0.90091,
+	3.6541 1.7727 -0.90091,
+	3.7663 0 1.8832,
+	3.9982 -0.71364 0.90092,
+	3.9982 0.71364 0.90092,
+	4.0293 -0.61803 -1.1328,
+	4.0293 0.61803 -1.1328,
+	4.2301 0 -0.081329,
+	]
+      }
+    Material {
+      diffuseColor .9 .4 0.3   # 3-sided
+      }
+    IndexedFaceSet {
+      coordIndex [
+	2,0,1,-1,
+	13,19,9,-1,
+	33,22,21,-1,
+	36,40,26,-1,
+	53,44,38,-1,
+	55,61,42,-1,
+	86,92,77,-1,
+	88,75,69,-1,
+	90,106,94,-1,
+	123,119,111,-1,
+	12,8,20,-1,
+	25,41,37,-1,
+	39,45,54,-1,
+	56,43,62,-1,
+	87,78,93,-1,
+	89,70,76,-1,
+	91,95,105,-1,
+	110,109,98,-1,
+	122,112,118,-1,
+	131,129,130,-1,
+	]
+      }
+   Material {
+      diffuseColor .4 .5 0.9   # 4-sided
+      }
+   IndexedFaceSet {
+      coordIndex [
+	2,11,17,7,-1,
+	4,5,3,0,-1,
+	5,4,9,15,-1,
+	11,22,28,17,-1,
+	17,28,42,32,-1,
+	18,5,15,26,-1,
+	24,13,7,17,-1,
+	24,17,32,38,-1,
+	30,19,34,47,-1,
+	33,49,65,50,-1,
+	36,30,47,51,-1,
+	47,57,77,67,-1,
+	50,65,71,55,-1,
+	51,47,67,69,-1,
+	53,63,83,73,-1,
+	57,47,34,44,-1,
+	63,61,79,83,-1,
+	65,85,90,71,-1,
+	83,79,94,102,-1,
+	86,73,83,96,-1,
+	88,100,115,104,-1,
+	96,83,102,111,-1,
+	100,92,108,115,-1,
+	115,108,119,125,-1,
+	6,12,23,16,-1,
+	16,23,39,31,-1,
+	20,29,48,35,-1,
+	27,16,31,43,-1,
+	29,37,52,48,-1,
+	35,48,58,45,-1,
+	48,52,70,68,-1,
+	58,48,68,78,-1,
+	60,41,46,66,-1,
+	62,64,84,80,-1,
+	64,54,74,84,-1,
+	76,60,66,81,-1,
+	80,84,101,95,-1,
+	81,66,82,98,-1,
+	84,97,112,101,-1,
+	87,97,84,74,-1,
+	93,99,114,107,-1,
+	99,89,103,114,-1,
+	105,116,126,113,-1,
+	114,103,109,120,-1,
+	114,124,118,107,-1,
+	116,122,127,126,-1,
+	124,114,120,129,-1,
+	131,128,126,127,-1,
+	6,16,10,1,-1,
+	14,8,3,5,-1,
+	16,27,21,10,-1,
+	25,14,5,18,-1,
+	56,72,65,49,-1,
+	66,46,40,59,-1,
+	72,91,85,65,-1,
+	82,66,59,75,-1,
+	113,126,117,106,-1,
+	121,110,104,115,-1,
+	126,128,123,117,-1,
+	130,121,115,125,-1,
+	]
+      }
+    Material {
+      diffuseColor .9 .9 0.3   # 6-sided
+      }
+    IndexedFaceSet {
+      coordIndex [
+	2,1,10,21,22,11,-1,
+	13,9,4,0,2,7,-1,
+	19,13,24,38,44,34,-1,
+	22,33,50,55,42,28,-1,
+	36,26,15,9,19,30,-1,
+	40,36,51,69,75,59,-1,
+	53,38,32,42,61,63,-1,
+	61,55,71,90,94,79,-1,
+	86,77,57,44,53,73,-1,
+	88,69,67,77,92,100,-1,
+	92,86,96,111,119,108,-1,
+	123,111,102,94,106,117,-1,
+	12,20,35,45,39,23,-1,
+	25,37,29,20,8,14,-1,
+	37,41,60,76,70,52,-1,
+	39,54,64,62,43,31,-1,
+	56,62,80,95,91,72,-1,
+	70,89,99,93,78,68,-1,
+	78,87,74,54,45,58,-1,
+	87,93,107,118,112,97,-1,
+	89,76,81,98,109,103,-1,
+	109,110,121,130,129,120,-1,
+	112,122,116,105,95,101,-1,
+	129,131,127,122,118,124,-1,
+	75,88,104,110,98,82,-1,
+	131,130,125,119,123,128,-1,
+	8,12,6,1,0,3,-1,
+	33,21,27,43,56,49,-1,
+	41,25,18,26,40,46,-1,
+	91,105,113,106,90,85,-1,
+	]
+      }
+    Material {
+      diffuseColor 0 0 0     # black edges
+      }
+    IndexedLineSet {
+      coordIndex [
+	0,1,-1,
+	0,2,-1,
+	0,3,-1,
+	0,4,-1,
+	1,2,-1,
+	1,6,-1,
+	1,10,-1,
+	2,7,-1,
+	2,11,-1,
+	3,5,-1,
+	3,8,-1,
+	4,5,-1,
+	4,9,-1,
+	5,14,-1,
+	5,15,-1,
+	5,18,-1,
+	6,12,-1,
+	6,16,-1,
+	7,13,-1,
+	7,17,-1,
+	8,12,-1,
+	8,14,-1,
+	8,20,-1,
+	9,13,-1,
+	9,15,-1,
+	9,19,-1,
+	10,16,-1,
+	10,21,-1,
+	11,17,-1,
+	11,22,-1,
+	12,20,-1,
+	12,23,-1,
+	13,19,-1,
+	13,24,-1,
+	14,25,-1,
+	15,26,-1,
+	16,23,-1,
+	16,27,-1,
+	16,31,-1,
+	17,24,-1,
+	17,28,-1,
+	17,32,-1,
+	18,25,-1,
+	18,26,-1,
+	19,30,-1,
+	19,34,-1,
+	20,29,-1,
+	20,35,-1,
+	21,22,-1,
+	21,27,-1,
+	21,33,-1,
+	22,28,-1,
+	22,33,-1,
+	23,39,-1,
+	24,38,-1,
+	25,37,-1,
+	25,41,-1,
+	26,36,-1,
+	26,40,-1,
+	27,43,-1,
+	28,42,-1,
+	29,37,-1,
+	29,48,-1,
+	30,36,-1,
+	30,47,-1,
+	31,39,-1,
+	31,43,-1,
+	32,38,-1,
+	32,42,-1,
+	33,49,-1,
+	33,50,-1,
+	34,44,-1,
+	34,47,-1,
+	35,45,-1,
+	35,48,-1,
+	36,40,-1,
+	36,51,-1,
+	37,41,-1,
+	37,52,-1,
+	38,44,-1,
+	38,53,-1,
+	39,45,-1,
+	39,54,-1,
+	40,46,-1,
+	40,59,-1,
+	41,46,-1,
+	41,60,-1,
+	42,55,-1,
+	42,61,-1,
+	43,56,-1,
+	43,62,-1,
+	44,53,-1,
+	44,57,-1,
+	45,54,-1,
+	45,58,-1,
+	46,66,-1,
+	47,51,-1,
+	47,57,-1,
+	47,67,-1,
+	48,52,-1,
+	48,58,-1,
+	48,68,-1,
+	49,56,-1,
+	49,65,-1,
+	50,55,-1,
+	50,65,-1,
+	51,69,-1,
+	52,70,-1,
+	53,63,-1,
+	53,73,-1,
+	54,64,-1,
+	54,74,-1,
+	55,61,-1,
+	55,71,-1,
+	56,62,-1,
+	56,72,-1,
+	57,77,-1,
+	58,78,-1,
+	59,66,-1,
+	59,75,-1,
+	60,66,-1,
+	60,76,-1,
+	61,63,-1,
+	61,79,-1,
+	62,64,-1,
+	62,80,-1,
+	63,83,-1,
+	64,84,-1,
+	65,71,-1,
+	65,72,-1,
+	65,85,-1,
+	66,81,-1,
+	66,82,-1,
+	67,69,-1,
+	67,77,-1,
+	68,70,-1,
+	68,78,-1,
+	69,75,-1,
+	69,88,-1,
+	70,76,-1,
+	70,89,-1,
+	71,90,-1,
+	72,91,-1,
+	73,83,-1,
+	73,86,-1,
+	74,84,-1,
+	74,87,-1,
+	75,82,-1,
+	75,88,-1,
+	76,81,-1,
+	76,89,-1,
+	77,86,-1,
+	77,92,-1,
+	78,87,-1,
+	78,93,-1,
+	79,83,-1,
+	79,94,-1,
+	80,84,-1,
+	80,95,-1,
+	81,98,-1,
+	82,98,-1,
+	83,96,-1,
+	83,102,-1,
+	84,97,-1,
+	84,101,-1,
+	85,90,-1,
+	85,91,-1,
+	86,92,-1,
+	86,96,-1,
+	87,93,-1,
+	87,97,-1,
+	88,100,-1,
+	88,104,-1,
+	89,99,-1,
+	89,103,-1,
+	90,94,-1,
+	90,106,-1,
+	91,95,-1,
+	91,105,-1,
+	92,100,-1,
+	92,108,-1,
+	93,99,-1,
+	93,107,-1,
+	94,102,-1,
+	94,106,-1,
+	95,101,-1,
+	95,105,-1,
+	96,111,-1,
+	97,112,-1,
+	98,109,-1,
+	98,110,-1,
+	99,114,-1,
+	100,115,-1,
+	101,112,-1,
+	102,111,-1,
+	103,109,-1,
+	103,114,-1,
+	104,110,-1,
+	104,115,-1,
+	105,113,-1,
+	105,116,-1,
+	106,113,-1,
+	106,117,-1,
+	107,114,-1,
+	107,118,-1,
+	108,115,-1,
+	108,119,-1,
+	109,110,-1,
+	109,120,-1,
+	110,121,-1,
+	111,119,-1,
+	111,123,-1,
+	112,118,-1,
+	112,122,-1,
+	113,126,-1,
+	114,120,-1,
+	114,124,-1,
+	115,121,-1,
+	115,125,-1,
+	116,122,-1,
+	116,126,-1,
+	117,123,-1,
+	117,126,-1,
+	118,122,-1,
+	118,124,-1,
+	119,123,-1,
+	119,125,-1,
+	120,129,-1,
+	121,130,-1,
+	122,127,-1,
+	123,128,-1,
+	124,129,-1,
+	125,130,-1,
+	126,127,-1,
+	126,128,-1,
+	127,131,-1,
+	128,131,-1,
+	129,130,-1,
+	129,131,-1,
+	130,131,-1,
+	]
+      }
+    }
+  }
+`
+console.log convert hartPoly

src/index.civet

@@ -3,21 +3,28 @@ moo from ../deps/moo.js
 type Tree = {[key:string]: string | Tree}
 
 lexer := moo.compile
-   WS:      /[ \t]+/
-   comment: /\/\/.*?$/
-   number:  /0|[1-9][0-9]*/
-   string:  /"(?:\\["\\]|[^\n"\\])*"/
-   lparen:  '('
-   rparen:  ')'
-   keyword: 'while if else moo cows'.split(' ')
-   NL:
-      match: /\n/
+   comment:    /#.*?$/
+   whitespace:
+      match:   /\s+/
       lineBreaks: true
+   string:
+      match:   /"(?:\\"|[^"])*"/
+      lineBreaks: true
+   number:     /[\d.-][\d.x]*/
+   word:       /[^\d.-\W]\w*/
+   comma:      ','
+   obrace:     '{'
+   cbrace:     '}'
+   obracket:   '['
+   cbracket:   ']'
+   oparen:     '('
+   cparen:     ')'
 
 export function tree97(vrml1: string): Tree
    tree: Tree := {}
    for tok, index of lexer.reset vrml1
-      if tok.type then tree[`${index}`] = {tok.type, tok.value}
+      if tok.type and tok.type !== 'whitespace' and tok.type !== 'comment'
+         tree[`${index}`] = {tok.type, tok.value}
    tree
 
 function render(t: string | Tree): string
@@ -26,11 +33,13 @@ function render(t: string | Tree): string
    result .= ''
    for item of Object.values t
       if typeof item is 'string' then result += item + ' '
-      else switch item.type
-         'WS' // ignore
-         'NL'
+      else
+         typ := item.type
+         result += render(typ)
+         if typeof typ === 'string' and
+               ['cbrace', 'cbracket', 'cparen'].includes(typ)
             result += "\n"
-         else result += render(item.type) + ' '
+         else result += ' '
    result
 
 export function convert(vrml1: string): string