From 8186038efb765ab4cd4c7985660ba684b9e18cf0 Mon Sep 17 00:00:00 2001 From: Glen Whitney Date: Sat, 2 Sep 2023 18:13:14 +0000 Subject: [PATCH] feat: Implement the Wings tokenizer (#4) Produces essentially the same token stream as the that found in x3d_import, except includes comment and whitespace tokens, which can be ignored in the parser. Resolves #3. Reviewed-on: https://code.studioinfinity.org/glen/vrml1to97/pulls/4 Co-authored-by: Glen Whitney Co-committed-by: Glen Whitney --- src/example.civet | 550 +++++++++++++++++++++++++++++++++++++++++++++- src/index.civet | 37 ++-- 2 files changed, 572 insertions(+), 15 deletions(-) diff --git a/src/example.civet b/src/example.civet index 6f689ba..18190a1 100644 --- a/src/example.civet +++ b/src/example.civet @@ -1,2 +1,550 @@ {convert} from ./index.js -console.log convert 'while (10) cows\nmoo' +hartPoly := ` +#VRML V1.0 ascii + +Separator { + DEF Title Info { + string "test" + } + DEF SceneInfo Info { + string "(c) George W. Hart, 1996. george@li.net" + string "Dept. Computer Science, Hofstra University." + string "Freely distributable for noncommercial purposes." + } + DEF BackgroundColor Info { # blue sky + string "0.2 0.5 1" + } + DEF Viewer Info { + string "examiner" + } + ShapeHints { + vertexOrdering UNKNOWN_ORDERING + creaseAngle 0 + } + DirectionalLight { + direction -.5 -1 0 + intensity 0.75 + } + DirectionalLight { + direction .5 1 0 + intensity 0.75 + } + Separator { + Coordinate3 { + point [ + -4.2301 0 0.081329, + -4.0293 -0.61803 1.1328, + -4.0293 0.61803 1.1328, + -3.9982 -0.71364 -0.90092, + -3.9982 0.71364 -0.90092, + -3.7663 0 -1.8832, + -3.6541 -1.7727 0.90091, + -3.6541 1.7727 0.90091, + -3.623 -1.8683 -1.1328, + -3.623 1.8683 -1.1328, + -3.4222 -1.0591 2.115, + -3.4222 1.0591 2.115, + -3.4222 -2.4864 -0.08133, + -3.4222 2.4864 -0.08133, + -3.3911 -1.1547 -2.115, + -3.3911 1.1547 -2.115, + -3.047 -2.2138 1.8832, + -3.047 2.2138 1.8832, + -3.0159 0 -2.8654, + -2.8965 2.8683 -1.1328, + -2.8965 -2.8683 -1.1328, + -2.8151 -0.61803 3.0973, + -2.8151 0.61803 3.0973, + -2.8151 -2.9274 0.90092, + -2.8151 2.9274 0.90092, + -2.6408 -1.1547 -3.0973, + -2.6408 1.1547 -3.0973, + -2.44 -1.7727 2.8654, + -2.44 1.7727 2.8654, + -2.1461 -2.8683 -2.115, + -2.1461 2.8683 -2.115, + -2.0648 -2.9274 2.115, + -2.0648 2.9274 2.115, + -1.9645 0 3.7471, + -1.9142 3.582 -0.90092, + -1.9142 -3.582 -0.90092, + -1.9142 2.1547 -3.0973, + -1.9142 -2.1547 -3.0973, + -1.8329 3.6411 1.1328, + -1.8329 -3.6411 1.1328, + -1.5893 1.1547 -3.7471, + -1.5893 -1.1547 -3.7471, + -1.4577 2.4864 3.0973, + -1.4577 -2.4864 3.0973, + -1.3072 4.023 0.08133, + -1.3072 -4.023 0.08133, + -1.2141 0 -3.979, + -1.1639 3.582 -1.8832, + -1.1639 -3.582 -1.8832, + -0.98225 -0.71364 3.979, + -0.98225 0.71364 3.979, + -0.93198 2.8683 -2.8654, + -0.93198 -2.8683 -2.8654, + -0.65733 4.023 1.1328, + -0.65733 -4.023 1.1328, + -0.60706 1.8683 3.7471, + -0.60706 -1.8683 3.7471, + -0.5568 4.023 -0.90092, + -0.5568 -4.023 -0.90092, + -0.37518 1.1547 -3.979, + -0.37518 -1.1547 -3.979, + -0.28214 2.8683 3.0973, + -0.28214 -2.8683 3.0973, + -0.050265 3.582 2.115, + -0.050265 -3.582 2.115, + 0 0 4.2109, + 0 0 -4.2109, + 0.050265 3.582 -2.115, + 0.050265 -3.582 -2.115, + 0.28214 2.8683 -3.0973, + 0.28214 -2.8683 -3.0973, + 0.37518 1.1547 3.979, + 0.37518 -1.1547 3.979, + 0.5568 4.023 0.90092, + 0.5568 -4.023 0.90092, + 0.60706 1.8683 -3.7471, + 0.60706 -1.8683 -3.7471, + 0.65733 4.023 -1.1328, + 0.65733 -4.023 -1.1328, + 0.93198 2.8683 2.8654, + 0.93198 -2.8683 2.8654, + 0.98225 -0.71364 -3.979, + 0.98225 0.71364 -3.979, + 1.1639 3.582 1.8832, + 1.1639 -3.582 1.8832, + 1.2141 0 3.979, + 1.3072 4.023 -0.08133, + 1.3072 -4.023 -0.08133, + 1.4577 2.4864 -3.0973, + 1.4577 -2.4864 -3.0973, + 1.5893 1.1547 3.7471, + 1.5893 -1.1547 3.7471, + 1.8329 3.6411 -1.1328, + 1.8329 -3.6411 -1.1328, + 1.9142 2.1547 3.0973, + 1.9142 -2.1547 3.0973, + 1.9142 3.582 0.90092, + 1.9142 -3.582 0.90092, + 1.9645 0 -3.7471, + 2.0648 -2.9274 -2.115, + 2.0648 2.9274 -2.115, + 2.1461 -2.8683 2.115, + 2.1461 2.8683 2.115, + 2.44 -1.7727 -2.8654, + 2.44 1.7727 -2.8654, + 2.6408 -1.1547 3.0973, + 2.6408 1.1547 3.0973, + 2.8151 -2.9274 -0.90092, + 2.8151 2.9274 -0.90092, + 2.8151 -0.61803 -3.0973, + 2.8151 0.61803 -3.0973, + 2.8965 2.8683 1.1328, + 2.8965 -2.8683 1.1328, + 3.0159 0 2.8654, + 3.047 -2.2138 -1.8832, + 3.047 2.2138 -1.8832, + 3.3911 -1.1547 2.115, + 3.3911 1.1547 2.115, + 3.4222 -2.4864 0.08133, + 3.4222 2.4864 0.08133, + 3.4222 -1.0591 -2.115, + 3.4222 1.0591 -2.115, + 3.623 -1.8683 1.1328, + 3.623 1.8683 1.1328, + 3.6541 -1.7727 -0.90091, + 3.6541 1.7727 -0.90091, + 3.7663 0 1.8832, + 3.9982 -0.71364 0.90092, + 3.9982 0.71364 0.90092, + 4.0293 -0.61803 -1.1328, + 4.0293 0.61803 -1.1328, + 4.2301 0 -0.081329, + ] + } + Material { + diffuseColor .9 .4 0.3 # 3-sided + } + IndexedFaceSet { + coordIndex [ + 2,0,1,-1, + 13,19,9,-1, + 33,22,21,-1, + 36,40,26,-1, + 53,44,38,-1, + 55,61,42,-1, + 86,92,77,-1, + 88,75,69,-1, + 90,106,94,-1, + 123,119,111,-1, + 12,8,20,-1, + 25,41,37,-1, + 39,45,54,-1, + 56,43,62,-1, + 87,78,93,-1, + 89,70,76,-1, + 91,95,105,-1, + 110,109,98,-1, + 122,112,118,-1, + 131,129,130,-1, + ] + } + Material { + diffuseColor .4 .5 0.9 # 4-sided + } + IndexedFaceSet { + coordIndex [ + 2,11,17,7,-1, + 4,5,3,0,-1, + 5,4,9,15,-1, + 11,22,28,17,-1, + 17,28,42,32,-1, + 18,5,15,26,-1, + 24,13,7,17,-1, + 24,17,32,38,-1, + 30,19,34,47,-1, + 33,49,65,50,-1, + 36,30,47,51,-1, + 47,57,77,67,-1, + 50,65,71,55,-1, + 51,47,67,69,-1, + 53,63,83,73,-1, + 57,47,34,44,-1, + 63,61,79,83,-1, + 65,85,90,71,-1, + 83,79,94,102,-1, + 86,73,83,96,-1, + 88,100,115,104,-1, + 96,83,102,111,-1, + 100,92,108,115,-1, + 115,108,119,125,-1, + 6,12,23,16,-1, + 16,23,39,31,-1, + 20,29,48,35,-1, + 27,16,31,43,-1, + 29,37,52,48,-1, + 35,48,58,45,-1, + 48,52,70,68,-1, + 58,48,68,78,-1, + 60,41,46,66,-1, + 62,64,84,80,-1, + 64,54,74,84,-1, + 76,60,66,81,-1, + 80,84,101,95,-1, + 81,66,82,98,-1, + 84,97,112,101,-1, + 87,97,84,74,-1, + 93,99,114,107,-1, + 99,89,103,114,-1, + 105,116,126,113,-1, + 114,103,109,120,-1, + 114,124,118,107,-1, + 116,122,127,126,-1, + 124,114,120,129,-1, + 131,128,126,127,-1, + 6,16,10,1,-1, + 14,8,3,5,-1, + 16,27,21,10,-1, + 25,14,5,18,-1, + 56,72,65,49,-1, + 66,46,40,59,-1, + 72,91,85,65,-1, + 82,66,59,75,-1, + 113,126,117,106,-1, + 121,110,104,115,-1, + 126,128,123,117,-1, + 130,121,115,125,-1, + ] + } + Material { + diffuseColor .9 .9 0.3 # 6-sided + } + IndexedFaceSet { + coordIndex [ + 2,1,10,21,22,11,-1, + 13,9,4,0,2,7,-1, + 19,13,24,38,44,34,-1, + 22,33,50,55,42,28,-1, + 36,26,15,9,19,30,-1, + 40,36,51,69,75,59,-1, + 53,38,32,42,61,63,-1, + 61,55,71,90,94,79,-1, + 86,77,57,44,53,73,-1, + 88,69,67,77,92,100,-1, + 92,86,96,111,119,108,-1, + 123,111,102,94,106,117,-1, + 12,20,35,45,39,23,-1, + 25,37,29,20,8,14,-1, + 37,41,60,76,70,52,-1, + 39,54,64,62,43,31,-1, + 56,62,80,95,91,72,-1, + 70,89,99,93,78,68,-1, + 78,87,74,54,45,58,-1, + 87,93,107,118,112,97,-1, + 89,76,81,98,109,103,-1, + 109,110,121,130,129,120,-1, + 112,122,116,105,95,101,-1, + 129,131,127,122,118,124,-1, + 75,88,104,110,98,82,-1, + 131,130,125,119,123,128,-1, + 8,12,6,1,0,3,-1, + 33,21,27,43,56,49,-1, + 41,25,18,26,40,46,-1, + 91,105,113,106,90,85,-1, + ] + } + Material { + diffuseColor 0 0 0 # black edges + } + IndexedLineSet { + coordIndex [ + 0,1,-1, + 0,2,-1, + 0,3,-1, + 0,4,-1, + 1,2,-1, + 1,6,-1, + 1,10,-1, + 2,7,-1, + 2,11,-1, + 3,5,-1, + 3,8,-1, + 4,5,-1, + 4,9,-1, + 5,14,-1, + 5,15,-1, + 5,18,-1, + 6,12,-1, + 6,16,-1, + 7,13,-1, + 7,17,-1, + 8,12,-1, + 8,14,-1, + 8,20,-1, + 9,13,-1, + 9,15,-1, + 9,19,-1, + 10,16,-1, + 10,21,-1, + 11,17,-1, + 11,22,-1, + 12,20,-1, + 12,23,-1, + 13,19,-1, + 13,24,-1, + 14,25,-1, + 15,26,-1, + 16,23,-1, + 16,27,-1, + 16,31,-1, + 17,24,-1, + 17,28,-1, + 17,32,-1, + 18,25,-1, + 18,26,-1, + 19,30,-1, + 19,34,-1, + 20,29,-1, + 20,35,-1, + 21,22,-1, + 21,27,-1, + 21,33,-1, + 22,28,-1, + 22,33,-1, + 23,39,-1, + 24,38,-1, + 25,37,-1, + 25,41,-1, + 26,36,-1, + 26,40,-1, + 27,43,-1, + 28,42,-1, + 29,37,-1, + 29,48,-1, + 30,36,-1, + 30,47,-1, + 31,39,-1, + 31,43,-1, + 32,38,-1, + 32,42,-1, + 33,49,-1, + 33,50,-1, + 34,44,-1, + 34,47,-1, + 35,45,-1, + 35,48,-1, + 36,40,-1, + 36,51,-1, + 37,41,-1, + 37,52,-1, + 38,44,-1, + 38,53,-1, + 39,45,-1, + 39,54,-1, + 40,46,-1, + 40,59,-1, + 41,46,-1, + 41,60,-1, + 42,55,-1, + 42,61,-1, + 43,56,-1, + 43,62,-1, + 44,53,-1, + 44,57,-1, + 45,54,-1, + 45,58,-1, + 46,66,-1, + 47,51,-1, + 47,57,-1, + 47,67,-1, + 48,52,-1, + 48,58,-1, + 48,68,-1, + 49,56,-1, + 49,65,-1, + 50,55,-1, + 50,65,-1, + 51,69,-1, + 52,70,-1, + 53,63,-1, + 53,73,-1, + 54,64,-1, + 54,74,-1, + 55,61,-1, + 55,71,-1, + 56,62,-1, + 56,72,-1, + 57,77,-1, + 58,78,-1, + 59,66,-1, + 59,75,-1, + 60,66,-1, + 60,76,-1, + 61,63,-1, + 61,79,-1, + 62,64,-1, + 62,80,-1, + 63,83,-1, + 64,84,-1, + 65,71,-1, + 65,72,-1, + 65,85,-1, + 66,81,-1, + 66,82,-1, + 67,69,-1, + 67,77,-1, + 68,70,-1, + 68,78,-1, + 69,75,-1, + 69,88,-1, + 70,76,-1, + 70,89,-1, + 71,90,-1, + 72,91,-1, + 73,83,-1, + 73,86,-1, + 74,84,-1, + 74,87,-1, + 75,82,-1, + 75,88,-1, + 76,81,-1, + 76,89,-1, + 77,86,-1, + 77,92,-1, + 78,87,-1, + 78,93,-1, + 79,83,-1, + 79,94,-1, + 80,84,-1, + 80,95,-1, + 81,98,-1, + 82,98,-1, + 83,96,-1, + 83,102,-1, + 84,97,-1, + 84,101,-1, + 85,90,-1, + 85,91,-1, + 86,92,-1, + 86,96,-1, + 87,93,-1, + 87,97,-1, + 88,100,-1, + 88,104,-1, + 89,99,-1, + 89,103,-1, + 90,94,-1, + 90,106,-1, + 91,95,-1, + 91,105,-1, + 92,100,-1, + 92,108,-1, + 93,99,-1, + 93,107,-1, + 94,102,-1, + 94,106,-1, + 95,101,-1, + 95,105,-1, + 96,111,-1, + 97,112,-1, + 98,109,-1, + 98,110,-1, + 99,114,-1, + 100,115,-1, + 101,112,-1, + 102,111,-1, + 103,109,-1, + 103,114,-1, + 104,110,-1, + 104,115,-1, + 105,113,-1, + 105,116,-1, + 106,113,-1, + 106,117,-1, + 107,114,-1, + 107,118,-1, + 108,115,-1, + 108,119,-1, + 109,110,-1, + 109,120,-1, + 110,121,-1, + 111,119,-1, + 111,123,-1, + 112,118,-1, + 112,122,-1, + 113,126,-1, + 114,120,-1, + 114,124,-1, + 115,121,-1, + 115,125,-1, + 116,122,-1, + 116,126,-1, + 117,123,-1, + 117,126,-1, + 118,122,-1, + 118,124,-1, + 119,123,-1, + 119,125,-1, + 120,129,-1, + 121,130,-1, + 122,127,-1, + 123,128,-1, + 124,129,-1, + 125,130,-1, + 126,127,-1, + 126,128,-1, + 127,131,-1, + 128,131,-1, + 129,130,-1, + 129,131,-1, + 130,131,-1, + ] + } + } + } +` +console.log convert hartPoly diff --git a/src/index.civet b/src/index.civet index 670d0d0..1787c12 100644 --- a/src/index.civet +++ b/src/index.civet @@ -3,21 +3,28 @@ moo from ../deps/moo.js type Tree = {[key:string]: string | Tree} lexer := moo.compile - WS: /[ \t]+/ - comment: /\/\/.*?$/ - number: /0|[1-9][0-9]*/ - string: /"(?:\\["\\]|[^\n"\\])*"/ - lparen: '(' - rparen: ')' - keyword: 'while if else moo cows'.split(' ') - NL: - match: /\n/ + comment: /#.*?$/ + whitespace: + match: /\s+/ lineBreaks: true + string: + match: /"(?:\\"|[^"])*"/ + lineBreaks: true + number: /[\d.-][\d.x]*/ + word: /[^\d.-\W]\w*/ + comma: ',' + obrace: '{' + cbrace: '}' + obracket: '[' + cbracket: ']' + oparen: '(' + cparen: ')' export function tree97(vrml1: string): Tree tree: Tree := {} for tok, index of lexer.reset vrml1 - if tok.type then tree[`${index}`] = {tok.type, tok.value} + if tok.type and tok.type !== 'whitespace' and tok.type !== 'comment' + tree[`${index}`] = {tok.type, tok.value} tree function render(t: string | Tree): string @@ -26,11 +33,13 @@ function render(t: string | Tree): string result .= '' for item of Object.values t if typeof item is 'string' then result += item + ' ' - else switch item.type - 'WS' // ignore - 'NL' + else + typ := item.type + result += render(typ) + if typeof typ === 'string' and + ['cbrace', 'cbracket', 'cparen'].includes(typ) result += "\n" - else result += render(item.type) + ' ' + else result += ' ' result export function convert(vrml1: string): string