feat: install moo and use in toy example (#2)

I failed to find a satisfactory way to compile and import moo.js with
  fixed specifiers, so finally I just gave up and patched the distributed
  moo code to be an es6 module. Very ugly but it works.

Resolves #1.

Reviewed-on: #2
Co-authored-by: Glen Whitney <glen@studioinfinity.org>
Co-committed-by: Glen Whitney <glen@studioinfinity.org>

.gitignore

@@ -1,6 +1,7 @@
 # Object files
 build
 dist
+deps
 
 # Editor backups
 *~

README.md

@@ -22,7 +22,7 @@ or from a script
 
 ```js
 (async () => {
-   const vrml1to97 = await import('vrml1to97')
+   const vrml1to97 = await import('./dist/vrml1to97/index.js')
    const vrml1spec = '# VRML 1.0 ....'
    const vrml97spec = vrml1to97.convert(vrml1spec)
 })()

etc/streamToString.js

@@ -0,0 +1,10 @@
+export async function streamToString(stream) {
+   // lets have a ReadableStream as a stream variable
+   const chunks = [];
+
+   for await (const chunk of stream) {
+      chunks.push(Buffer.from(chunk));
+   }
+
+   return Buffer.concat(chunks).toString("utf-8");
+}

etc/testscript.html

@@ -5,8 +5,8 @@
     <title>vrml1to97 test</title>
     <script>
       (async () => {
-        const vrml1to97 = await import('../dist/index.js')
-        console.log(vrml1to97.convert('test'))
+        const vrml1to97 = await import('../dist/vrml1to97/index.js')
+        console.log(vrml1to97.convert('if (moo) cows // hurray'))
       })()
     </script>
   </head>

etc/vrml1to97.js

@@ -1,18 +1,8 @@
 #!/usr/bin/env node
 
-import {convert} from './index.js'
+import {convert} from './vrml1to97/index.js'
 import {stdin, argv} from 'node:process'
-
-async function streamToString(stream) {
-   // lets have a ReadableStream as a stream variable
-   const chunks = [];
-
-   for await (const chunk of stream) {
-      chunks.push(Buffer.from(chunk));
-   }
-
-   return Buffer.concat(chunks).toString("utf-8");
-}
+import {streamToString} from './streamToString.js'
 
 if (argv.length > 2) {
    console.log('Usage: vrml1to97 < old.wrl > shinynew.wrl')

package.json5

@@ -4,11 +4,21 @@
   description: 'JavaScript converter from VRML 1 to VRML97',
   scripts: {
     test: 'echo "Error: no test specified" && exit 1',
-    build_ts: 'civet -c -o build/.ts src/*civet',
-    build_js: 'tsc',
+    // Use civet to create .ts files from the source:
+    build_ts: 'mkdir -p build && civet -c -o build/.ts src/*civet',
+    // These next three steps get all dependencies into one place for convenience:
+    build_deps1: 'mkdir -p deps',
+    // Strip the UMD header so that es6 can import:
+    build_deps2: 'node tools/modulize.mjs < node_modules/moo/moo.js > deps/moo.js',
+    // Give Typescript the info it needs:
+    build_deps3: 'cp node_modules/@types/moo/index.d.ts deps/moo.d.ts',
+    // Use the Typescript compiler to create the final .js files:
+    build_js: 'tsc && mkdir -p dist/deps && cp deps/*.js dist/deps',
+    // And finally add the executable and minimal package.json
     build_etc: 'cp etc/*.js* dist',
     build: 'pnpm --sequential /build_/',
-    try: 'pnpm build && node dist/example.js',
+    try: 'pnpm build && node dist/vrml1to97/example.js',
+    clean: 'rm -r build dist deps'
   },
   keywords: [
     'javascript',
@@ -27,7 +37,11 @@
   },
   devDependencies: {
     '@danielx/civet': '^0.6.26',
+    '@types/moo': '^0.5.6',
     'http-server': '^14.1.1',
     typescript: '^5.2.2',
   },
+  dependencies: {
+    moo: '^0.5.2',
+  },
 }

pnpm-lock.yaml

@@ -4,10 +4,18 @@ settings:
   autoInstallPeers: true
   excludeLinksFromLockfile: false
 
+dependencies:
+  moo:
+    specifier: ^0.5.2
+    version: 0.5.2
+
 devDependencies:
   '@danielx/civet':
     specifier: ^0.6.26
     version: 0.6.26
+  '@types/moo':
+    specifier: ^0.5.6
+    version: 0.5.6
   http-server:
     specifier: ^14.1.1
     version: 14.1.1
@@ -48,6 +56,10 @@ packages:
       '@jridgewell/sourcemap-codec': 1.4.15
     dev: true
 
+  /@types/moo@0.5.6:
+    resolution: {integrity: sha512-Q60hZhulhl2Ox4LjbJvhH+HzsKrwzLPjEB8dZw0fK1MH2HyOLe6LDou68yTfsWasxGv7DPZe5VNM5vgpzOa2nw==}
+    dev: true
+
   /ansi-styles@4.3.0:
     resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==}
     engines: {node: '>=8'}
@@ -233,6 +245,10 @@ packages:
       minimist: 1.2.8
     dev: true
 
+  /moo@0.5.2:
+    resolution: {integrity: sha512-iSAJLHYKnX41mKcJKjqvnAN9sf0LMDTXDEvFv+ffuRR9a1MIuXLjMNL6EsnDHSkKLTWNqQQ5uo61P4EbU4NU+Q==}
+    dev: false
+
   /ms@2.1.3:
     resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==}
     dev: true

src/example.civet

@@ -1,2 +1,2 @@
 {convert} from ./index.js
-console.log convert 'bar'
+console.log convert 'while (10) cows\nmoo'

src/index.civet

@@ -1,14 +1,37 @@
+moo from ../deps/moo.js
+
 type Tree = {[key:string]: string | Tree}
 
+lexer := moo.compile
+   WS:      /[ \t]+/
+   comment: /\/\/.*?$/
+   number:  /0|[1-9][0-9]*/
+   string:  /"(?:\\["\\]|[^\n"\\])*"/
+   lparen:  '('
+   rparen:  ')'
+   keyword: 'while if else moo cows'.split(' ')
+   NL:
+      match: /\n/
+      lineBreaks: true
+
 export function tree97(vrml1: string): Tree
-   {converted: 'foo'}
+   tree: Tree := {}
+   for tok, index of lexer.reset vrml1
+      if tok.type then tree[`${index}`] = {tok.type, tok.value}
+   tree
 
 function render(t: string | Tree): string
    if typeof t is 'string'
       return t
-   if result := t.converted
-      render result
-   else '<Conversion failed>'
+   result .= ''
+   for item of Object.values t
+      if typeof item is 'string' then result += item + ' '
+      else switch item.type
+         'WS' // ignore
+         'NL'
+            result += "\n"
+         else result += render(item.type) + ' '
+   result
 
 export function convert(vrml1: string): string
    render tree97 vrml1

tools/modulize.mjs

@@ -0,0 +1,9 @@
+import {stdin, argv} from 'node:process'
+import {streamToString} from '../etc/streamToString.js'
+
+const preamble = `// ES6 Module converted from https://github.com/no-context/moo
+   export default (((f) => f())(function () {
+`
+const umd = await streamToString(stdin)
+const realStart = umd.indexOf(" 'use strict'")
+console.log(preamble + umd.substring(realStart))

tsconfig.json

@@ -5,8 +5,8 @@
     "forceConsistentCasingInFileNames": true,
     "esModuleInterop": true,
     "declaration": true,
-    "rootDir": "./build",
-    "outDir": "./dist"
+    "rootDir": "build",
+    "outDir": "dist/vrml1to97"
   },
   "ts-node": {
     "transpileOnly": true,