feat: support DEF and USE syntax to share subtrees (#18)

Reviewed-on: #18
Co-authored-by: Glen Whitney <glen@studioinfinity.org>
Co-committed-by: Glen Whitney <glen@studioinfinity.org>

README.md

@@ -1,13 +1,26 @@
 # vrml1to97
 
-JavaScript converter from VRML 1.0 to VRML97 file format, based on Wings 3D
+JavaScript converter from VRML 1.0 to VRML97 file format.
-conversion logic.
 
-Essentially, this is a JavaScript reimplementation of the algorithm of the
+This converter was originally a JavaScript reimplementation of the algorithm
-"token rearranger" found in the
+of the "token rearranger" found in the
 [Wings 3D x3d importer](https://github.com/dgud/wings/blob/master/plugins_src/import_export/x3d_import.erl)
 (which was written in Erlang).
 
+However, from version 0.3, it adds support for a significantly larger subset of
+VRML 1.0. Overall, it recognizes and converts
+
+  * Title, SceneInfo, BackgroundColor, and View "Info" nodes [this node type
+    was removed in VRML97].
+  * All Light nodes
+  * Grouping nodes including Transform, Separator, Group, Switch, and WWWAnchor
+  * ShapeHints
+  * Rotations
+  * All shape nodes including Cube, Cone, Cylinder, Sphere, IndexedFaceSet,
+    IndexedLineSet, PointSet, Coordinate3, and Normal
+  * All material nodes including Material, TextureCoordinate2, and Texture2
+  * DEF and USE constructs to share subtrees
+
 ## Usage
 
 From an es6 module under Node (for example)
@@ -44,8 +57,8 @@ Currently this package exports just two functions:
    actually check that its input is VRML 1, so its behavior is undefined
    if given anything but valid VRML 1 syntax). Returns VRML 97 syntax for
    the same scene, as nearly as it can translate. Note that not all of
-   VRML 1 is recognized, and some of the translations may be somewhat
+   VRML 1 is recognized (see above for a list of constructs that should
-   approximate.
+   be handled), and some of the translations may be somewhat approximate.
 
    If the optional second argument `source` is supplied, `convert` adds
    a comment indicating that the original vrml1 came from the specified
@@ -66,10 +79,12 @@ Currently this package exports just two functions:
 ## Conversion notes
 
 One sort of geometry common to VRML 1 and VRML97 is the IndexedFaceSet. These
-often used to render solids. Indeed, the default in VRML97 is to assume they
+entities are often used to render solids. Indeed, the default in VRML97 is to
-do represent a solid, with the normals pointing outward. Unusual visual effects
+assume they do represent a solid, with the normals pointing outward. Unusual
-ensue if the normals are not properly directed (basically, you see through the
+visual effects ensue if the normals are not properly directed (basically, you
-"front" of the solid and see the "backs" of the faces on the opposite side).
+see through the "front" of the solid and see the "backs" of the faces on the
+opposite side).
+
 As a result, unless the input VRML 1 explicitly includes an explicit
 vertexOrdering of CLOCKWISE or COUNTERCLOCKWISE, the default solid treatment
 will be turned off in the VRML97 output, meaning that all faces will be

package.json5

@@ -1,6 +1,6 @@
 {
   name: 'vrml1to97',
-  version: '0.2.4',
+  version: '0.3.0',
   description: 'JavaScript converter from VRML 1 to VRML97',
   scripts: {
     test: 'echo "Error: no test specified" && exit 1',
@@ -40,7 +40,7 @@
   },
   type: 'module',
   devDependencies: {
-    '@danielx/civet': '^0.6.71',
+    '@danielx/civet': '^0.6.72',
     '@types/moo': '^0.5.9',
     'http-server': '^14.1.1',
     json5: '^2.2.3',

pnpm-lock.yaml

@@ -11,8 +11,8 @@ dependencies:
 
 devDependencies:
   '@danielx/civet':
-    specifier: ^0.6.71
+    specifier: ^0.6.72
-    version: 0.6.71(typescript@5.3.3)
+    version: 0.6.72(typescript@5.3.3)
   '@types/moo':
     specifier: ^0.5.9
     version: 0.5.9
@@ -35,8 +35,8 @@ packages:
       '@jridgewell/trace-mapping': 0.3.9
     dev: true
 
-  /@danielx/civet@0.6.71(typescript@5.3.3):
+  /@danielx/civet@0.6.72(typescript@5.3.3):
-    resolution: {integrity: sha512-piOoHtJARe6YqRiXN02Ryb+nLU9JwU8TQLknvPwmlaNm6krdKN+X9dM+C9D4LRoAnAySC2wVshR0wf7YDIUV1Q==}
+    resolution: {integrity: sha512-jumnIbXbdFs0ZiKN62fmD+p8QGi+E0jmtc02dKz9wIIoPkODsa4XXlBrS5BRR5fr3w5d3ah8Vq7gWt+DL9Wa0Q==}
     engines: {node: '>=19 || ^18.6.0 || ^16.17.0'}
     hasBin: true
     peerDependencies:

src/index.civet

@@ -2,6 +2,7 @@ moo from ../deps/moo.js
 import type {Lexer, Token} from ../deps/moo.d.ts
 
 type Tree = {[key:string]: (string | Tree)[]}
+type DefTree = {_definitions?: Tree} & Tree
 
 lexer := moo.compile
    comment:    /#.*?$/
@@ -86,100 +87,34 @@ function addWorldParameter(name: string, value: string, tree: Tree): void
      children.push world
   world.WorldInfo.push(` ${name}`, ` ${value}`)
 
-function parse(stream: Lexer, tree: Tree = {}): Tree
+// Current best way to re-use a pattern
+matches := (str: string, pat: RegExp) => pat.test(str)
+operator matches
+GroupNode := /(?:Transform)?Separator|Group|Switch|WWWAnchor/
+SetNode := /IndexedFaceSet|IndexedLineSet|PointSet/
+LightNode := /Light$/
+
+function parse(stream: Lexer, tree: DefTree = {}): DefTree
    held .= filtered stream  // for lookahead
+   currentDefinition .= ''  // if we have just started a DEF
 
    while next := filtered stream
       unless held then break
+      // May need to know the current definition, but it clears each round
+      lastDefinition := currentDefinition
+      currentDefinition = ''
+
       switch data :=
             {nt: next.type, nv: next.value, ht: held.type, hv: held.value}
-         {ht: 'word', nt: 'obrace'}
+         {ht: 'word', hv: 'Info'}
-            switch held.value
+            // Here the meaning is entirely dependent on what DEF we are in
-               /(?:Transform)?Separator|Group|Switch|WWWAnchor/
+            if next.type !== 'obrace'
-                  parent :=
+               // Not sure what this construct is, so ignore it
-                     held.value.endsWith('Separator') ? 'Transform' : 'Group'
+               held = next
-                  {children, ...context} := tree
+               continue
-                  subTree := parse stream, context
-                  if newKids := subTree.children
-                     addChild `${parent} { children [
-                        ${renderList newKids} ] }\n`, tree
-               'ShapeHints'
-                  subTree := parse stream
-                  if 'vertexOrdering' in subTree
-                     tree.ccw = [
-                        subTree.vertexOrdering[0] is 'ccw' ? 'true' : 'false']
-                  if 'creaseAngle' in subTree
-                     tree.creaseAngle = subTree.creaseAngle
-               'Coordinate3'
-                  tree.Coordinate = toksUntilClose stream
-               'Normal'
-                  tree.Normal = toksUntilClose stream
-               'TextureCoordinate2'
-                  tree.TextureCoordinate = toksUntilClose stream
-               'Texture2'
-                  tree.Texture = translatedToksUntilClose stream
-               'Material'
-                  tree.Material = translatedToksUntilClose stream
-               'Cube'
-                  dims := width: '', height: '', depth: ''
-                  findNumbersAtTopLevel stream, dims
-                  params := [`size ${dims.width} ${dims.height} ${dims.depth}`]
-                  addShape 'Box', params, tree
-               'Cone'
-                  dims := bottomRadius: '', height: ''
-                  findNumbersAtTopLevel stream, dims
-                  params := [`bottomRadius ${dims.bottomRadius}`,
-                             `height ${dims.height}`]
-                  addShape 'Cone', params, tree
-               'Cylinder'
-                  dims := radius: '', height: ''
-                  findNumbersAtTopLevel stream, dims
-                  params := [`radius ${dims.radius} height ${dims.height}`]
-                  addShape 'Cylinder', params, tree
-               'Sphere'
-                  dims := radius: ''
-                  findNumbersAtTopLevel stream, dims
-                  addShape 'Sphere', [`radius ${dims.radius}`], tree
-               /IndexedFaceSet|IndexedLineSet|PointSet/
-                  isFaces := held.value is 'IndexedFaceSet'
-                  contents := translatedToksUntilClose stream
-                  params := []
-                  if 'Coordinate' in tree
-                     params.push "coord Coordinate {\n",
-                        ...tree.Coordinate, " }\n"
-                  if 'Normal' in tree
-                     params.push "normal Normal {\n",
-                        ...tree.Normal, " }\n"
-                  if isFaces and 'TextureCoordinate' in tree
-                     params.push "texCoord TextureCoordinate {\n",
-                        ...tree.TextureCoordinate, " }\n"
-                  if isFaces and 'creaseAngle' in tree
-                     params.push `creaseAngle ${tree.creaseAngle[0]}`
-                  if isFaces
-                     if 'ccw' in tree then params.push `ccw ${tree.ccw[0]}`
-                     else params.push 'solid false'
-                  params.push ...contents
-                  addShape held.value, params, tree
-               /Light$/
             contents := toksUntilClose stream
-                  addChild {[held.value]: contents}, tree
-               else
-                  parse stream // discard the subgroup
             held = filtered stream
-         {ht: 'word', hv: 'vertexOrdering', nt: 'word', nv: 'COUNTERCLOCKWISE'}
+            switch lastDefinition
-            tree.vertexOrdering = ['ccw']
-            held = filtered stream
-         {ht: 'word', hv: 'vertexOrdering', nt: 'word', nv: 'CLOCKWISE'}
-            tree.vertexOrdering = ['cw']
-            held = filtered stream
-         {ht: 'word', hv: 'creaseAngle', nt: 'number'}
-            tree.creaseAngle = [ next.value ]
-            held = filtered stream
-         {ht: 'word', nt: 'word', nv: 'Info'}
-            opener := filtered stream
-            if opener and opener.type === 'obrace'
-               contents := toksUntilClose stream
-               switch held.value
                'BackgroundColor'
                   // Find the first string token and assume it is the color
                   stok := contents.find .type[0] === 'string'
@@ -213,13 +148,157 @@ function parse(stream: Lexer, tree: Tree = {}): Tree
                      |> .map((x) => x in tr ? tr[x] : x.toUpperCase())
                      |> .join ' '
                   addChild `NavigationInfo { type [ ${viewers} ] }\n`, tree
+         {ht: 'word', hv: 'DEF'}
+            unless next.type is 'word'
+               // Don't understand this construction, ignore the DEF
+               held = next
+               continue
+            currentDefinition = next.value
             held = filtered stream
-            else held = opener
+            if held?.type is 'word'
+               clause := `DEF ${currentDefinition}`
+               role .= held.value
+               switch role
+                  matches GroupNode
+                     role = 'Child'
+                  'ShapeHints'
+                     role = 'Dummy' // will fill in when we parse the ShapeHints
+                  'Coordinate3'
+                     role = 'Coordinate'
+                  'TextureCoordinate2'
+                     role = 'TextureCoordinate'
+                  'Texture2'
+                     role = 'Texture'
+                  /Cube|Cone|Cylinder|Sphere/
+                     role = 'Shape'
+                  matches SetNode
+                     role = 'Shape'
+                  matches LightNode
+                     role = 'Child'
+               (tree._definitions ??= {})[currentDefinition] = [role]
+               unless role is 'Info'
+                  // Info nodes don't directly generate a node in translation
+                  addChild clause, tree
+         {ht: 'word', hv: 'USE'}
+            unless next.type is 'word'
+               // Don't understand this construction, ignore the USE
+               held = next
+               continue
+            known := tree._definitions
+            if known and next.value in known
+               role := known[next.value][0]
+               clause := `USE ${next.value}`
+               switch role
+                  'Child'
+                     addChild clause, tree
+                  'Shape'
+                     addShape clause, [], tree
+                  {}
+                     mergeTree role, tree
                   else
+                     tree[role] = [clause]
+            else
+               console.error 'USE of unDEFined identifier', next.value
+            held = filtered stream
+         {ht: 'word', nt: 'obrace'}
+            switch held.value
+               matches GroupNode
+                  parent :=
+                     held.value.endsWith('Separator') ? 'Transform' : 'Group'
+                  {children, ...context} := tree
+                  subTree := parse stream, context
+                  if newKids := subTree.children
+                     newChild .= `${parent} {\n   `
+                     if 'Rotation' in subTree
+                        newChild += renderList subTree.Rotation
+                        newChild += "\n   "
+                     newChild += `children [\n      ${renderList newKids} ] }\n`
+                     addChild newChild, tree
+               'ShapeHints'
+                  subTree := parse stream
+                  hints: Tree := {}
+                  if 'vertexOrdering' in subTree
+                     hints.ccw = [
+                        subTree.vertexOrdering[0] is 'ccw' ? 'TRUE' : 'FALSE']
+                  if 'creaseAngle' in subTree
+                     hints.creaseAngle = subTree.creaseAngle
+                  mergeTree hints, tree
+                  if lastDefinition and tree._definitions
+                     tree._definitions[lastDefinition] = [hints]
+               'Rotation'
+                  tree.Rotation = toksUntilClose stream
+               'Coordinate3'
+                  tree.Coordinate = toksUntilClose stream
+               'Normal'
+                  tree.Normal = toksUntilClose stream
+               'TextureCoordinate2'
+                  tree.TextureCoordinate = toksUntilClose stream
+               'Texture2'
+                  tree.Texture = translatedToksUntilClose stream
+               'Material'
+                  tree.Material = translatedToksUntilClose stream
+               'Cube'
+                  dims := width: '', height: '', depth: ''
+                  findNumbersAtTopLevel stream, dims
+                  params := [`size ${dims.width} ${dims.height} ${dims.depth}`]
+                  addShape 'Box', params, tree
+               'Cone'
+                  dims := bottomRadius: '', height: ''
+                  findNumbersAtTopLevel stream, dims
+                  params := [`bottomRadius ${dims.bottomRadius}`,
+                             `height ${dims.height}`]
+                  addShape 'Cone', params, tree
+               'Cylinder'
+                  dims := radius: '', height: ''
+                  findNumbersAtTopLevel stream, dims
+                  params := [`radius ${dims.radius} height ${dims.height}`]
+                  addShape 'Cylinder', params, tree
+               'Sphere'
+                  dims := radius: ''
+                  findNumbersAtTopLevel stream, dims
+                  addShape 'Sphere', [`radius ${dims.radius}`], tree
+               matches SetNode
+                  isFaces := held.value is 'IndexedFaceSet'
+                  contents := translatedToksUntilClose stream
+                  params := []
+                  if 'Coordinate' in tree
+                     params.push "coord Coordinate {\n",
+                        ...tree.Coordinate, " }\n"
+                  if 'Normal' in tree
+                     params.push "normal Normal {\n",
+                        ...tree.Normal, " }\n"
+                  if isFaces and 'TextureCoordinate' in tree
+                     params.push "texCoord TextureCoordinate {\n",
+                        ...tree.TextureCoordinate, " }\n"
+                  if isFaces and 'creaseAngle' in tree
+                     params.push `creaseAngle ${tree.creaseAngle[0]}`
+                  if isFaces
+                     if 'ccw' in tree then params.push `ccw ${tree.ccw[0]}`
+                     else params.push 'solid FALSE'
+                  params.push ...contents
+                  addShape held.value, params, tree
+               matches LightNode
+                  contents := toksUntilClose stream
+                  addChild {[held.value]: contents}, tree
+               else
+                  parse stream // discard the subgroup
+            held = filtered stream
+         {ht: 'word', hv: 'vertexOrdering', nt: 'word'}
+            switch next.value
+               'COUNTERCLOCKWISE'
+                  tree.vertexOrdering = ['ccw']
+               'CLOCKWISE'
+                  tree.vertexOrdering = ['cw']
+            held = filtered stream
+         {ht: 'word', hv: 'creaseAngle', nt: 'number'}
+            tree.creaseAngle = [ next.value ]
+            held = filtered stream
+         else
+            console.error 'Ignoring unparseable token', held
             held = next // ignore unknown words
       if not held or held.type === 'cbrace' then break
    if held and held.type !== 'cbrace'
-      console.log 'Oddly ended up with held', held
+      console.error 'Incomplete parse, symbol at end:', held
    tree
 
 function addShape(nodeType: string, params: (string | Tree)[], tree: Tree): void
@@ -233,9 +312,15 @@ function addShape(nodeType: string, params: (string | Tree)[], tree: Tree): void
          shape.Shape.push "texture ImageTexture {\n",
             ...tree.Texture, " }\n"
       shape.Shape.push " }\n"
-   shape.Shape.push `geometry ${nodeType} {\n`, ...params, " }\n"
+   geometry: (string|Tree)[] := [`geometry ${nodeType}`]
+   if params.length then geometry.push "  {\n", ...params, " }\n"
+   shape.Shape.push ...geometry
    addChild shape, tree
 
+function mergeTree(subtree: Tree, tree: Tree): void
+   for key, value in subtree
+      tree[key] = value
+
 function render(t: string | Tree): string
    if typeof t is 'string' then return t
    if 'children' in t then return renderList t.children