From f789ed94fdce08c849420cb24ba372b5fac566d8 Mon Sep 17 00:00:00 2001 From: Glen Whitney Date: Sun, 28 Feb 2021 22:21:56 -0800 Subject: [PATCH 1/8] feat: Add OCaml code generation Also start using nailgun to speed up code generation. Resolves #6. --- .drone.yml | 16 +++++++++++ .gitignore | 6 +++++ bin/fosgen | 15 ++++++++++- bin/generate_test_code | 2 +- bin/let_sun_shine | 26 ++++++++++++++++++ editor/Generation.esv | 1 + tests/basic.spt | 2 +- trans/fostr.str | 1 + trans/haskell.str | 2 +- trans/ocaml.str | 61 ++++++++++++++++++++++++++++++++++++++++++ 10 files changed, 128 insertions(+), 4 deletions(-) create mode 100755 bin/let_sun_shine create mode 100644 trans/ocaml.str diff --git a/.drone.yml b/.drone.yml index 452f163..fdeea06 100644 --- a/.drone.yml +++ b/.drone.yml @@ -29,6 +29,18 @@ steps: - java -jar /drone/lib/spt/org.metaborg.spt.cmd/target/org.metaborg.spt.cmd* -l . -s /drone/lib/spt/org.metaborg.meta.lang.spt -t tests - mkdir -p lib - curl -o lib/sunshine.jar -L 'http://artifacts.metaborg.org/service/local/artifact/maven/redirect?r=snapshots&g=org.metaborg&a=org.metaborg.sunshine2&v=LATEST' + - name: setup_gen + image: maven + volumes: + - name: m2 + path: /root/.m2 + commands: + - git clone https://github.com/facebook/nailgun.git + - cd nailgun + - make + - cd ../bin + - ln -s ../nailgun/nailgun-client/target/ng . + - cd .. - bin/fosgen tests/emit_sum.fos - name: extract_tests image: xonsh/xonsh @@ -55,6 +67,10 @@ steps: image: haskell commands: - bin/run_tests runghc hs + - name: ocaml_tests + image: ocaml/opam + commands: + - bin/run_tests ocaml ml volumes: - name: lib diff --git a/.gitignore b/.gitignore index 80a4a3c..35e6d95 100644 --- a/.gitignore +++ b/.gitignore @@ -12,10 +12,16 @@ .pydevproject +a.out + *.aterm /site +bin/ng tests/extracted/* tests/*.js tests/*.py tests/*.hs +tests/*.ml +tests/*.cmi +tests/*.cmo adhoc* diff --git a/bin/fosgen b/bin/fosgen index c5af48b..910c7d9 100755 --- a/bin/fosgen +++ b/bin/fosgen @@ -5,6 +5,7 @@ erro() { printf "%s\n" "$*" >&2; } ##### Set defaults: SUPPRESS_ERR=YES +USE_NAILGUN=YES LANGUAGE=Python ##### Extract command line options: @@ -14,18 +15,23 @@ do -h|--help) echo echo "Usage:" - echo " fosgen [-d] [-l LANGUAGE] FILE" + echo " fosgen [-d] [-j] [-l LANGUAGE] FILE" echo echo "Writes to standard output the code generated from the fostr" echo "program in FILE, targeting the specified LANGUAGE (which" echo "defaults to Python)." echo echo "The -d option writes diagnostic output to standard error." + echo "The -j option uses the Spoofax Sunshine JAR directly, rather" + echo "than via nailgun." exit ;; -d) SUPPRESS_ERR='' ;; + -j) + USE_NAILGUN='' + ;; -l) shift LANGUAGE="$1" @@ -67,5 +73,12 @@ then exec 2>/dev/null fi +if [[ $USE_NAILGUN ]] +then + $BINDIR/let_sun_shine + $BINDIR/ng sunshine transform -p $PROJDIR -n $LANGUAGE -i $PROGRAM + exit $? +fi + java -jar $SUNJAR transform -p $PROJDIR -l $PROJDIR -l $MVN_REPO -n $LANGUAGE -i $PROGRAM exit $? diff --git a/bin/generate_test_code b/bin/generate_test_code index 2c443d6..0d151a8 100755 --- a/bin/generate_test_code +++ b/bin/generate_test_code @@ -4,7 +4,7 @@ failed=0 for dir in tests/extracted/*; do for file in $dir/*.fos; do - for language in Python Javascript Haskell; do + for language in Python Javascript Haskell OCaml; do echo bin/fosgen -l ${language%.*} $file ... bin/fosgen -l $language $file if [[ $? -ne 0 ]]; then diff --git a/bin/let_sun_shine b/bin/let_sun_shine new file mode 100755 index 0000000..aa8df53 --- /dev/null +++ b/bin/let_sun_shine @@ -0,0 +1,26 @@ +#!/bin/bash + +BINDIR=$(dirname $BASH_SOURCE) + +if $BINDIR/ng sunshine --help +then + # echo "sun already shining." +else + # echo "disperse the clouds." + SUNJAR="$BINDIR/../lib/sunshine.jar" + PROJDIR="$BINDIR/.." + if [[ ! $MVN_REPO ]]; then + MVN_REPO="$HOME/.m2/repository" + fi + if [[ ! -d $MVN_REPO ]]; then + MVN_REPO="/root/.m2/repository" + fi + if [[ ! -d $MVN_REPO ]]; then + echo "Cannot find your Maven repository. Please set environment variable" + echo "MVN_REPO to its full path and re-run." + exit 1 + fi + java -jar $SUNJAR server >/dev/null 2>&1 & + sleep 5 + $BINDIR/ng sunshine load -l $PROJDIR -l $MVN_REPO +fi diff --git a/editor/Generation.esv b/editor/Generation.esv index 3d71fe2..9e5051c 100644 --- a/editor/Generation.esv +++ b/editor/Generation.esv @@ -4,3 +4,4 @@ menus action: "Python" = to-python action: "Javascript" = to-javascript action: "Haskell" = to-haskell + action: "OCaml" = to-ocaml diff --git a/tests/basic.spt b/tests/basic.spt index 215c83f..592150b 100644 --- a/tests/basic.spt +++ b/tests/basic.spt @@ -108,7 +108,7 @@ again. That way we can use the general (left-associative) parse to TopLevel( Gets(Gets(Gets(Gets(DefGets(LitString("'Two and '")),Int("2")), LitString("' make '")),Sum(Int("2"),Int("2"))), - EscString("\"./n\""))) + EscString("\".\n\""))) /** writes Two and 2 make 4. **/ diff --git a/trans/fostr.str b/trans/fostr.str index d9a585a..91194d6 100644 --- a/trans/fostr.str +++ b/trans/fostr.str @@ -6,6 +6,7 @@ imports pp outline analysis + ocaml haskell javascript python diff --git a/trans/haskell.str b/trans/haskell.str index 0e26050..0b08716 100644 --- a/trans/haskell.str +++ b/trans/haskell.str @@ -10,7 +10,7 @@ rules rule. C) We will use bottomup-para to traverse the full AST with the transformation from A so that we have access to the original expression - (and get get the Statix-associated type when we need to). + (and can get the Statix-associated type when we need to). Hence the transformation in (A) must actually take a pair of an (original) term and a term with value strings at each child, and be certain to return a value string. diff --git a/trans/ocaml.str b/trans/ocaml.str new file mode 100644 index 0000000..11fcdca --- /dev/null +++ b/trans/ocaml.str @@ -0,0 +1,61 @@ +module ocaml +imports libstrategolib signatures/- util signature/TYPE analysis + +/* Note will use bottomup-para to traverse the full AST so that + we have access to the original expression (and can get the + Statix-associated type when we need to). + + This means that every one of our local rules must take a pair + of an original term and a term with every child replaced by + its generated code. + */ + +rules + ml: (_, TopLevel(x)) -> $[(* fostr preamble *) + type stream = { getS: string -> stream } + let rec stdio = { + getS = (fun s -> print_string s; stdio) + };; + (* End of preamble *) + + [x]] + + ml: (_, Stream()) -> $[stdio] + ml: (_, Int(x)) -> x + ml: (_, LitString(x)) -> $[{|[x]|}] + ml: (_, EscString(x)) -> x + ml: (_, Sum(x,y)) -> $[[x] + [y]] + ml: (Gets(_,yn), Gets(x, y)) + -> $[([x]).getS ([(yn,y)])] + ml: (To(xn,_), To(x, y)) + -> $[let _fto = ([x]) in (ignore (([y]).getS ([(xn,"_fto")])); _fto)] + ml: (_, Terminate(x)) -> x + ml: (_, Sequence(l)) -> l + + ml_seq: [x] -> x + ml_seq: [x | xs ] -> $[ignore ([x]); +[xs]] + + /* One drawback of using paramorphism is we have to handle lists + explicitly: + */ + ml: (_, []) -> [] + ml: (_, [x | xs]) -> [x | xs] + + /* Another drawback of using paramorphism is at the very leaves we have + to undouble the tuple: + */ + ml: (x, x) -> x where x + + ml_str: (node, code) -> $[[node]([code])] + +strategies + + ml_string_cast = get-type; (?INT() < !"string_of_int" + !"") + + ocaml = bottomup-para(try(ml)) + + // Interface ocaml code generation with editor services and file system + to-ocaml: (selected, _, _, path, project-path) -> (filename, result) + with filename := path + ; result := selected -- 2.34.1 From d1c3ed34814e1e853581820523d08dfb09d18a28 Mon Sep 17 00:00:00 2001 From: Glen Whitney Date: Mon, 1 Mar 2021 09:16:50 -0800 Subject: [PATCH 2/8] fix: try different image for compiling Nailgun client Also mention OCaml code generation in docs --- .drone.yml | 2 +- tests/basic.spt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.drone.yml b/.drone.yml index fdeea06..fb12161 100644 --- a/.drone.yml +++ b/.drone.yml @@ -30,7 +30,7 @@ steps: - mkdir -p lib - curl -o lib/sunshine.jar -L 'http://artifacts.metaborg.org/service/local/artifact/maven/redirect?r=snapshots&g=org.metaborg&a=org.metaborg.sunshine2&v=LATEST' - name: setup_gen - image: maven + image: silkeh/clang volumes: - name: m2 path: /root/.m2 diff --git a/tests/basic.spt b/tests/basic.spt index 592150b..a5e98c0 100644 --- a/tests/basic.spt +++ b/tests/basic.spt @@ -59,8 +59,8 @@ For example, this snippet generates the following Python: It generates nearly identical code in this simple example for Javascript (just with `"Hello, world!"` in place of `r'Hello, world!'`), although it generates a different -preamble defining Stdio for each language. (Currently, Haskell code -generation is also supported.) +preamble defining Stdio for each language. (Currently, Haskell and OCaml +code generation are also supported.) There's not much to break down in such a tiny program as this, but let's do it. The prefix operator `<<<` could be read as "the default stream receives...", -- 2.34.1 From e3ff1f259e0b39de110f432a153a761a3d406eba Mon Sep 17 00:00:00 2001 From: Glen Whitney Date: Mon, 1 Mar 2021 09:29:46 -0800 Subject: [PATCH 3/8] fix: OK, try the gcc image, see if that will do it --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index fb12161..eea527f 100644 --- a/.drone.yml +++ b/.drone.yml @@ -30,7 +30,7 @@ steps: - mkdir -p lib - curl -o lib/sunshine.jar -L 'http://artifacts.metaborg.org/service/local/artifact/maven/redirect?r=snapshots&g=org.metaborg&a=org.metaborg.sunshine2&v=LATEST' - name: setup_gen - image: silkeh/clang + image: gcc volumes: - name: m2 path: /root/.m2 -- 2.34.1 From 87a6fbc64299ea218c31476ee691308e0591069f Mon Sep 17 00:00:00 2001 From: Glen Whitney Date: Mon, 1 Mar 2021 09:50:33 -0800 Subject: [PATCH 4/8] fix: eliminate empty then block in bash script --- bin/let_sun_shine | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/let_sun_shine b/bin/let_sun_shine index aa8df53..84dfffe 100755 --- a/bin/let_sun_shine +++ b/bin/let_sun_shine @@ -5,6 +5,7 @@ BINDIR=$(dirname $BASH_SOURCE) if $BINDIR/ng sunshine --help then # echo "sun already shining." + : else # echo "disperse the clouds." SUNJAR="$BINDIR/../lib/sunshine.jar" -- 2.34.1 From b1f87428f56d05872f451e6af8d70e5c469883c1 Mon Sep 17 00:00:00 2001 From: Glen Whitney Date: Mon, 1 Mar 2021 10:06:44 -0800 Subject: [PATCH 5/8] feat: -d option to fosgen now enables nailgun diagnostics --- .drone.yml | 2 +- bin/fosgen | 7 ++++++- bin/let_sun_shine | 21 +++++++++++++++++---- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/.drone.yml b/.drone.yml index eea527f..813be68 100644 --- a/.drone.yml +++ b/.drone.yml @@ -41,7 +41,7 @@ steps: - cd ../bin - ln -s ../nailgun/nailgun-client/target/ng . - cd .. - - bin/fosgen tests/emit_sum.fos + - bin/fosgen -d tests/emit_sum.fos - name: extract_tests image: xonsh/xonsh commands: diff --git a/bin/fosgen b/bin/fosgen index 910c7d9..b03656b 100755 --- a/bin/fosgen +++ b/bin/fosgen @@ -75,7 +75,12 @@ fi if [[ $USE_NAILGUN ]] then - $BINDIR/let_sun_shine + if [[ $SUPPRESS_ERR ]] + then + $BINDIR/let_sun_shine + else + $BINDIR/let_sun_shine noisy + fi $BINDIR/ng sunshine transform -p $PROJDIR -n $LANGUAGE -i $PROGRAM exit $? fi diff --git a/bin/let_sun_shine b/bin/let_sun_shine index 84dfffe..cb3c67a 100755 --- a/bin/let_sun_shine +++ b/bin/let_sun_shine @@ -1,13 +1,21 @@ #!/bin/bash +# Helper for fosgen, not intended to be used directly +# With an argument, print diagnostic output + BINDIR=$(dirname $BASH_SOURCE) if $BINDIR/ng sunshine --help then - # echo "sun already shining." - : + if [[ $1 ]] + then + echo "sun already shining." + fi else - # echo "disperse the clouds." + if [[ $1 ]] + then + echo "disperse the clouds." + fi SUNJAR="$BINDIR/../lib/sunshine.jar" PROJDIR="$BINDIR/.." if [[ ! $MVN_REPO ]]; then @@ -21,7 +29,12 @@ else echo "MVN_REPO to its full path and re-run." exit 1 fi - java -jar $SUNJAR server >/dev/null 2>&1 & + if [[ $1 ]] + then + java -jar $SUNJAR server & + else + java -jar $SUNJAR server >/dev/null 2>&1 & + fi sleep 5 $BINDIR/ng sunshine load -l $PROJDIR -l $MVN_REPO fi -- 2.34.1 From dc19df8811ef795ffb0ed589da456b93f64f07db Mon Sep 17 00:00:00 2001 From: Glen Whitney Date: Mon, 1 Mar 2021 10:21:33 -0800 Subject: [PATCH 6/8] test: Move fosgen test into a step where java is available --- .drone.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.drone.yml b/.drone.yml index 813be68..fad4086 100644 --- a/.drone.yml +++ b/.drone.yml @@ -41,7 +41,6 @@ steps: - cd ../bin - ln -s ../nailgun/nailgun-client/target/ng . - cd .. - - bin/fosgen -d tests/emit_sum.fos - name: extract_tests image: xonsh/xonsh commands: @@ -53,7 +52,8 @@ steps: path: /drone/lib - name: m2 path: /root/.m2 - commands: + commands: # Note we first make sure that fosgen is working + - bin/fosgen -d tests/emit_sum.fos - bin/generate_test_code - name: python_tests image: python:slim -- 2.34.1 From 221d85ebbe1713287e20fb1e789e15e9c3526221 Mon Sep 17 00:00:00 2001 From: Glen Whitney Date: Mon, 1 Mar 2021 10:39:18 -0800 Subject: [PATCH 7/8] test: add permissions to extracted test directory The hope is this will prevent the permission errors in the OCaml tests --- .drone.yml | 1 + bin/extract_tests.xsh | 1 + 2 files changed, 2 insertions(+) diff --git a/.drone.yml b/.drone.yml index fad4086..e1363ac 100644 --- a/.drone.yml +++ b/.drone.yml @@ -70,6 +70,7 @@ steps: - name: ocaml_tests image: ocaml/opam commands: + - ls -als tests/extracted - bin/run_tests ocaml ml volumes: diff --git a/bin/extract_tests.xsh b/bin/extract_tests.xsh index 7cf131a..1d8c59f 100644 --- a/bin/extract_tests.xsh +++ b/bin/extract_tests.xsh @@ -19,6 +19,7 @@ EXP = 'expect' for path in TEST_LIST: destdir = pf"{DESTINATION}/{path.stem}" mkdir -p @(destdir) + chmod ugo+rwx @(destdir) contents = path.read_text() tests = re.split(r'test\s*(.+?)\s*\[\[.*?\n', contents)[1:] testit = iter(tests) -- 2.34.1 From 3effe65f0185eda809ca030f9e9bc86df5287c42 Mon Sep 17 00:00:00 2001 From: Glen Whitney Date: Mon, 1 Mar 2021 11:14:53 -0800 Subject: [PATCH 8/8] test: try to get ocaml initialized by opam for testing --- .drone.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.drone.yml b/.drone.yml index e1363ac..90b928b 100644 --- a/.drone.yml +++ b/.drone.yml @@ -71,6 +71,8 @@ steps: image: ocaml/opam commands: - ls -als tests/extracted + - opam init + - eval $(opam env) - bin/run_tests ocaml ml volumes: -- 2.34.1