From d2ba26a53e5b96c0f6d313db9bfc9e2469968087 Mon Sep 17 00:00:00 2001 From: Glen Whitney Date: Thu, 18 Feb 2021 19:41:54 -0800 Subject: [PATCH 1/4] feat: Add double-quoted string constants with escapes Resolves #20. --- syntax/fostr.sdf3 | 1 + tests/hw2.fos | 1 + trans/haskell.str | 1 + trans/javascript.str | 1 + trans/python.str | 1 + trans/statics.stx | 9 ++++++--- 6 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 tests/hw2.fos diff --git a/syntax/fostr.sdf3 b/syntax/fostr.sdf3 index c346eee..8624b58 100644 --- a/syntax/fostr.sdf3 +++ b/syntax/fostr.sdf3 @@ -39,6 +39,7 @@ context-free syntax Ex.Int = INT Ex.LitString = STRING_LITERAL + Ex.EscString = STRING Ex.Stream = Ex.Sum = < + > {left} Ex.Gets = [[Ex] << [Ex]] {left} diff --git a/tests/hw2.fos b/tests/hw2.fos new file mode 100644 index 0000000..c177d31 --- /dev/null +++ b/tests/hw2.fos @@ -0,0 +1 @@ +<<< "Hello,\t\tworld!\n\n" diff --git a/trans/haskell.str b/trans/haskell.str index fc75a7b..0e26050 100644 --- a/trans/haskell.str +++ b/trans/haskell.str @@ -39,6 +39,7 @@ rules hs: (_, Stream()) -> "StdIO" hs: (_, Int(x)) -> x hs: (_, LitString(x)) -> x + hs: (_, EscString(x)) -> x hs: (_, Sum(x, y)) -> $[([x] + [y])] hs: (Gets(_, xn), Gets(s, x)) -> v diff --git a/trans/javascript.str b/trans/javascript.str index 1fd2900..3a847e2 100644 --- a/trans/javascript.str +++ b/trans/javascript.str @@ -17,6 +17,7 @@ rules js: Stream() -> $[Stdio] js: Int(x) -> x js: LitString(x) -> x + js: EscString(x) -> x js: Sum(x,y) -> $[[x] + [y]] js: Gets(x, y) -> $[[x].gets([y])] js: To(x, y) -> $[to([x],[y])] diff --git a/trans/python.str b/trans/python.str index 068f05a..6449417 100644 --- a/trans/python.str +++ b/trans/python.str @@ -19,6 +19,7 @@ rules py: Stream() -> $[Stdio] py: Int(x) -> x py: LitString(x) -> $[r[x]] + py: EscString(x) -> x py: Sum(x,y) -> $[[x] + [y]] py: Gets(x, y) -> $[[x].gets([y])] py: To(x, y) -> $[to([x],[y])] diff --git a/trans/statics.stx b/trans/statics.stx index ef63c8d..340a1df 100644 --- a/trans/statics.stx +++ b/trans/statics.stx @@ -184,16 +184,19 @@ constructor was trivial: Now typing literals is straightforward: ```statix +{! "\git docs/statix_works:trans/statics.stx" extract: + start: '(.*ty_Ex.Int.*\s*)' + stop: '/. ../' +!} +``` **/ - /** md */ ty_Ex(Int(_)) = INT(). ty_Ex(LitString(_)) = STRING(). + ty_Ex(EscString(_)) = STRING(). ty_Ex(e@Stream()) = STREAM(). - /* **/ /** md -``` Finally we get to the binary operators, and here we use the pattern found in recent versions of the From f9c6e04c8ccb1b2dfc5d766755701d518597ae3a Mon Sep 17 00:00:00 2001 From: Glen Whitney Date: Fri, 19 Feb 2021 08:08:24 -0800 Subject: [PATCH 2/4] docs: Finally get the tour to start from the real helloworld Also improves the testing situation for the features to date. Resolves #17. --- tests/basic.spt | 151 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 101 insertions(+), 50 deletions(-) diff --git a/tests/basic.spt b/tests/basic.spt index 494cafb..215c83f 100644 --- a/tests/basic.spt +++ b/tests/basic.spt @@ -16,15 +16,21 @@ Title: A whirlwind tour of fostr ## Whirlwind tour -fostr is just in its infancy, so it's not yet even ready for -Hello, World. The best we can offer now is this little snippet -that writes the sum of the ASCII codes for 'H', 'W', and '!' to standard output: +There seems only to be one way to start a tour like this. So here goes: ```fostr **/ -/** md */ test emit_sum [[ -stream << 72 + 87 + 33 +/** md */ test hello_world [[ +<<< 'Hello, world!' ]] /* **/ +parse to TopLevel(DefGets(LitString("'Hello, world!'"))) +/** writes +Hello, world!**/ + +// Prior proto-hello-world, no longer in the tour. +test emit_sum [[ +stream << 72 + 87 + 33 +]] parse to TopLevel(Gets(Stream(), Sum(Sum(Int("72"), Int("87")), Int("33")))) /** writes 192**/ @@ -33,7 +39,7 @@ parse to TopLevel(Gets(Stream(), Sum(Sum(Int("72"), Int("87")), Int("33")))) ``` At the moment, there are only two ways to run a file containing fostr code -(you can find the above in `tests/emit_sum.fos`). They both start by +(you can find the above in `tests/hw.fos`). They both start by cloning this fostr project. Then, either: 1. Open the project in Eclipse and build it, visit your program file, @@ -46,30 +52,70 @@ cloning this fostr project. Then, either: For example, this snippet generates the following Python: ```python -{! ../tests/emit_sum.py extract: +{! ../tests/hw.py extract: start: 'Stdio\s=' !} ``` -(which writes "192" to standard output); it also generates identical code in -this simple example for -Javascript, although it generates a different preamble defining Stdio in each -case. (Haskell code generation is also currently supported.) +It generates nearly identical code in +this simple example for Javascript (just with `"Hello, world!"` +in place of `r'Hello, world!'`), although it generates a different +preamble defining Stdio for each language. (Currently, Haskell code +generation is also supported.) + +There's not much to break down in such a tiny program as this, but let's do +it. The prefix operator `<<<` could be read as "the default stream receives...", +and unsurprisingly in a main program the default stream is standard input and +output. And `'Hello, world!'` is a literal string constant; what you see is +what you get. The only detail to know is that such constants must occur +within a single line of your source file. So depending on how you +ran the program and how closely you looked at its output, +you may have noticed this program does not write a newline at the end +of its message. Nothing is ever implicitly sent to a stream. So if you want +newlines, you should switch to a (double-quoted) string that allows +the usual array of escape sequences: + +```fostr +**/ + +/** md */ test hello_esc_world [[ +<<< "Hello,\t\tworld!\n\n" +]] /* **/ +parse to TopLevel(DefGets(EscString("\"Hello,\t\tworld!\n\n\""))) +/** writes +Hello, world! + +**/ + +/** md +``` +(We threw in two of each so you could clearly see them in the output if +you run this program.) ### Everything has a value As mentioned in the [Introduction](../README.md), everything in a fostr program (including the entire program itself) is an expression and has -a value. So what's the value of that expression above? Well, appropriately -enough, `stream` is our -first example of a stream, and for convenience, the value of a stream -receiving an item is (usually) just the stream back again. The `<<` operator -is also left-associative, so that way we can chain insertions into a stream: +a value. So what's the value of that expression above? Well, for convenience, +the value of a stream receiving an item is (generally) just the stream back +again. That way we can use the general (left-associative) +`_stream_ << _value_` operator to chain insertions into a stream: ```fostr **/ -/** md */ test emit_twice [[ -stream << 72 + 87 + 33 << 291 +/** md */ test state_obvious [[ +<<< 'Two and ' << 2 << ' make ' << 2+2 << ".\n" ]] /* **/ +parse to TopLevel( + Gets(Gets(Gets(Gets(DefGets(LitString("'Two and '")),Int("2")), + LitString("' make '")),Sum(Int("2"),Int("2"))), + EscString("\"./n\""))) +/** writes +Two and 2 make 4. +**/ + +test emit_twice [[ +stream << 72 + 87 + 33 << 291 +]] parse to TopLevel( Gets(Gets(Stream(), Sum(Sum(Int("72"), Int("87")), Int("33"))), Int("291"))) /** writes @@ -77,24 +123,28 @@ parse to TopLevel( /** md ``` -Running this program produces a nice palindromic output: "192291". And because sometimes you want to emphasize the value and propagate that instead of the stream, you can also write these expressions "the other way" -with `>>`; both forms return the first argument, so the following writes "824": +with `>>>` for sending to the default stream or `>>` in general; these forms +(generally) return the value sent, so the following writes "824": ```fostr **/ /** md */ test enters_twice [[ -(7 + 8 >> stream + 9) >> stream +(7 + 8 >> stream + 9) >>> ]] /* **/ parse to TopLevel( - To(Sum(Sum(Int("7"), To(Int("8"), Stream())), Int("9")), Stream())) + DefTo(Sum(Sum(Int("7"), To(Int("8"), Stream())), Int("9")))) /** writes 824**/ /** md ``` +Two things are worth noting here: the default stream can always be referred to +directly via the identifier `stream`, and the precedences of `<<` and `>>` are +different so that generally full expressions go to a stream with `<<` but +just individual terms are sent with `>>`. ### Layout in fostr @@ -104,13 +154,13 @@ lines are indented from the start of the initial line: **/ /** md */ test receive_enter_break [[ -stream << +<<< 7 - + 8 >> stream + + 8 >>> + 9 ]] /* **/ parse to TopLevel( - Gets(Stream(), Sum(Sum(Int("7"), To(Int("8"), Stream())), Int("9")))) + DefGets(Sum(Sum(Int("7"), DefTo(Int("8"))), Int("9")))) /** writes 824**/ @@ -121,8 +171,8 @@ parse to TopLevel( **/ /** md */ test enter_receive_bad_continuation [[ -(7 + 8 >> stream + 9) ->> (stream << 9 + 2) +(7 + 8 >>> + 9) +>> (<<< 9 + 2) ]] /* **/ parse fails @@ -145,16 +195,17 @@ lines are evaluated in sequence. For example, the program **/ /** md */ test emit_thrice [[ - stream << 72 + 87 - stream << 88 + <<< 72 + 87 + <<< 88 + 96 - 99 + 12 >> - stream + 99 + 12 + >>> + ]] /* **/ parse to TopLevel(Sequence([ - Gets(Stream(), Sum(Int("72"), Int("87"))), - Gets(Stream(), Sum(Int("88"), Int("96"))), - Sum(Int("99"), To(Int("12"), Stream())) + DefGets(Sum(Int("72"), Int("87"))), + DefGets(Sum(Int("88"), Int("96"))), + Sum(Int("99"), DefTo(Int("12"))) ])) /** writes 15918412**/ @@ -169,10 +220,10 @@ in sequence align at the left; e.g., the following fails to parse: **/ /** md */ test emit_thrice_bad_alignment [[ - stream << 72 + 87 -stream << 88 + <<< 72 + 87 +<<< 88 + 96 - 99 + 12 >> stream + 99 + 12 >>> ]] /* **/ parse fails @@ -187,23 +238,23 @@ are so terminated. So the following is OK: **/ /** md */ test emit_several [[ - stream << 1 + 2; 3 >> stream - (4 + 5) >> stream; stream << 6; - stream << 7 - stream << 8 + <<< 1 + 2; 3 >>> + (4 + 5) >>>; stream << 6; + <<< 7 + <<< 8 + (9+10); - 11 + 12 >> stream; 13 >> stream - >> stream + 11 + 12 >>>; 13 >>> + >>> ]] /* **/ parse to TopLevel(Sequence([ - ISequence(Prior([Terminate(Gets(Stream(), Sum(Int("1"), Int("2"))))]), - To(Int("3"), Stream())), - ISequence(Prior([Terminate(To(Sum(Int("4"), Int("5")), Stream()))]), + ISequence(Prior([Terminate(DefGets(Sum(Int("1"), Int("2"))))]), + DefTo(Int("3"))), + ISequence(Prior([Terminate(DefTo(Sum(Int("4"), Int("5"))))]), Terminate(Gets(Stream(), Int("6")))), - Gets(Stream(), Int("7")), - Terminate(Gets(Stream(), Sum(Int("8"), Sum(Int("9"), Int("10"))))), - ISequence(Prior([Terminate(Sum(Int("11"), To(Int("12"), Stream())))]), - To(To(Int("13"), Stream()), Stream())) + DefGets(Int("7")), + Terminate(DefGets(Sum(Int("8"), Sum(Int("9"), Int("10"))))), + ISequence(Prior([Terminate(Sum(Int("11"), DefTo(Int("12"))))]), + DefTo(DefTo(Int("13")))) ])) /** writes 3396727121313**/ From cc89ad1e93a0e5da1080e30e0c45af53a850ffd0 Mon Sep 17 00:00:00 2001 From: Glen Whitney Date: Mon, 1 Mar 2021 20:40:35 +0000 Subject: [PATCH 3/4] Add OCaml code generation (#24) Also start using nailgun to speed up code generation. Resolves #6. Co-authored-by: Glen Whitney Reviewed-on: https://code.studioinfinity.org/glen/fostr/pulls/24 Co-Authored-By: Glen Whitney Co-Committed-By: Glen Whitney --- .drone.yml | 23 ++++++++++++++-- .gitignore | 6 +++++ bin/extract_tests.xsh | 1 + bin/fosgen | 20 +++++++++++++- bin/generate_test_code | 2 +- bin/let_sun_shine | 40 +++++++++++++++++++++++++++ editor/Generation.esv | 1 + tests/basic.spt | 6 ++--- trans/fostr.str | 1 + trans/haskell.str | 2 +- trans/ocaml.str | 61 ++++++++++++++++++++++++++++++++++++++++++ 11 files changed, 155 insertions(+), 8 deletions(-) create mode 100755 bin/let_sun_shine create mode 100644 trans/ocaml.str diff --git a/.drone.yml b/.drone.yml index 452f163..90b928b 100644 --- a/.drone.yml +++ b/.drone.yml @@ -29,7 +29,18 @@ steps: - java -jar /drone/lib/spt/org.metaborg.spt.cmd/target/org.metaborg.spt.cmd* -l . -s /drone/lib/spt/org.metaborg.meta.lang.spt -t tests - mkdir -p lib - curl -o lib/sunshine.jar -L 'http://artifacts.metaborg.org/service/local/artifact/maven/redirect?r=snapshots&g=org.metaborg&a=org.metaborg.sunshine2&v=LATEST' - - bin/fosgen tests/emit_sum.fos + - name: setup_gen + image: gcc + volumes: + - name: m2 + path: /root/.m2 + commands: + - git clone https://github.com/facebook/nailgun.git + - cd nailgun + - make + - cd ../bin + - ln -s ../nailgun/nailgun-client/target/ng . + - cd .. - name: extract_tests image: xonsh/xonsh commands: @@ -41,7 +52,8 @@ steps: path: /drone/lib - name: m2 path: /root/.m2 - commands: + commands: # Note we first make sure that fosgen is working + - bin/fosgen -d tests/emit_sum.fos - bin/generate_test_code - name: python_tests image: python:slim @@ -55,6 +67,13 @@ steps: image: haskell commands: - bin/run_tests runghc hs + - name: ocaml_tests + image: ocaml/opam + commands: + - ls -als tests/extracted + - opam init + - eval $(opam env) + - bin/run_tests ocaml ml volumes: - name: lib diff --git a/.gitignore b/.gitignore index 80a4a3c..35e6d95 100644 --- a/.gitignore +++ b/.gitignore @@ -12,10 +12,16 @@ .pydevproject +a.out + *.aterm /site +bin/ng tests/extracted/* tests/*.js tests/*.py tests/*.hs +tests/*.ml +tests/*.cmi +tests/*.cmo adhoc* diff --git a/bin/extract_tests.xsh b/bin/extract_tests.xsh index 7cf131a..1d8c59f 100644 --- a/bin/extract_tests.xsh +++ b/bin/extract_tests.xsh @@ -19,6 +19,7 @@ EXP = 'expect' for path in TEST_LIST: destdir = pf"{DESTINATION}/{path.stem}" mkdir -p @(destdir) + chmod ugo+rwx @(destdir) contents = path.read_text() tests = re.split(r'test\s*(.+?)\s*\[\[.*?\n', contents)[1:] testit = iter(tests) diff --git a/bin/fosgen b/bin/fosgen index c5af48b..b03656b 100755 --- a/bin/fosgen +++ b/bin/fosgen @@ -5,6 +5,7 @@ erro() { printf "%s\n" "$*" >&2; } ##### Set defaults: SUPPRESS_ERR=YES +USE_NAILGUN=YES LANGUAGE=Python ##### Extract command line options: @@ -14,18 +15,23 @@ do -h|--help) echo echo "Usage:" - echo " fosgen [-d] [-l LANGUAGE] FILE" + echo " fosgen [-d] [-j] [-l LANGUAGE] FILE" echo echo "Writes to standard output the code generated from the fostr" echo "program in FILE, targeting the specified LANGUAGE (which" echo "defaults to Python)." echo echo "The -d option writes diagnostic output to standard error." + echo "The -j option uses the Spoofax Sunshine JAR directly, rather" + echo "than via nailgun." exit ;; -d) SUPPRESS_ERR='' ;; + -j) + USE_NAILGUN='' + ;; -l) shift LANGUAGE="$1" @@ -67,5 +73,17 @@ then exec 2>/dev/null fi +if [[ $USE_NAILGUN ]] +then + if [[ $SUPPRESS_ERR ]] + then + $BINDIR/let_sun_shine + else + $BINDIR/let_sun_shine noisy + fi + $BINDIR/ng sunshine transform -p $PROJDIR -n $LANGUAGE -i $PROGRAM + exit $? +fi + java -jar $SUNJAR transform -p $PROJDIR -l $PROJDIR -l $MVN_REPO -n $LANGUAGE -i $PROGRAM exit $? diff --git a/bin/generate_test_code b/bin/generate_test_code index 2c443d6..0d151a8 100755 --- a/bin/generate_test_code +++ b/bin/generate_test_code @@ -4,7 +4,7 @@ failed=0 for dir in tests/extracted/*; do for file in $dir/*.fos; do - for language in Python Javascript Haskell; do + for language in Python Javascript Haskell OCaml; do echo bin/fosgen -l ${language%.*} $file ... bin/fosgen -l $language $file if [[ $? -ne 0 ]]; then diff --git a/bin/let_sun_shine b/bin/let_sun_shine new file mode 100755 index 0000000..cb3c67a --- /dev/null +++ b/bin/let_sun_shine @@ -0,0 +1,40 @@ +#!/bin/bash + +# Helper for fosgen, not intended to be used directly +# With an argument, print diagnostic output + +BINDIR=$(dirname $BASH_SOURCE) + +if $BINDIR/ng sunshine --help +then + if [[ $1 ]] + then + echo "sun already shining." + fi +else + if [[ $1 ]] + then + echo "disperse the clouds." + fi + SUNJAR="$BINDIR/../lib/sunshine.jar" + PROJDIR="$BINDIR/.." + if [[ ! $MVN_REPO ]]; then + MVN_REPO="$HOME/.m2/repository" + fi + if [[ ! -d $MVN_REPO ]]; then + MVN_REPO="/root/.m2/repository" + fi + if [[ ! -d $MVN_REPO ]]; then + echo "Cannot find your Maven repository. Please set environment variable" + echo "MVN_REPO to its full path and re-run." + exit 1 + fi + if [[ $1 ]] + then + java -jar $SUNJAR server & + else + java -jar $SUNJAR server >/dev/null 2>&1 & + fi + sleep 5 + $BINDIR/ng sunshine load -l $PROJDIR -l $MVN_REPO +fi diff --git a/editor/Generation.esv b/editor/Generation.esv index 3d71fe2..9e5051c 100644 --- a/editor/Generation.esv +++ b/editor/Generation.esv @@ -4,3 +4,4 @@ menus action: "Python" = to-python action: "Javascript" = to-javascript action: "Haskell" = to-haskell + action: "OCaml" = to-ocaml diff --git a/tests/basic.spt b/tests/basic.spt index 215c83f..a5e98c0 100644 --- a/tests/basic.spt +++ b/tests/basic.spt @@ -59,8 +59,8 @@ For example, this snippet generates the following Python: It generates nearly identical code in this simple example for Javascript (just with `"Hello, world!"` in place of `r'Hello, world!'`), although it generates a different -preamble defining Stdio for each language. (Currently, Haskell code -generation is also supported.) +preamble defining Stdio for each language. (Currently, Haskell and OCaml +code generation are also supported.) There's not much to break down in such a tiny program as this, but let's do it. The prefix operator `<<<` could be read as "the default stream receives...", @@ -108,7 +108,7 @@ again. That way we can use the general (left-associative) parse to TopLevel( Gets(Gets(Gets(Gets(DefGets(LitString("'Two and '")),Int("2")), LitString("' make '")),Sum(Int("2"),Int("2"))), - EscString("\"./n\""))) + EscString("\".\n\""))) /** writes Two and 2 make 4. **/ diff --git a/trans/fostr.str b/trans/fostr.str index d9a585a..91194d6 100644 --- a/trans/fostr.str +++ b/trans/fostr.str @@ -6,6 +6,7 @@ imports pp outline analysis + ocaml haskell javascript python diff --git a/trans/haskell.str b/trans/haskell.str index 0e26050..0b08716 100644 --- a/trans/haskell.str +++ b/trans/haskell.str @@ -10,7 +10,7 @@ rules rule. C) We will use bottomup-para to traverse the full AST with the transformation from A so that we have access to the original expression - (and get get the Statix-associated type when we need to). + (and can get the Statix-associated type when we need to). Hence the transformation in (A) must actually take a pair of an (original) term and a term with value strings at each child, and be certain to return a value string. diff --git a/trans/ocaml.str b/trans/ocaml.str new file mode 100644 index 0000000..11fcdca --- /dev/null +++ b/trans/ocaml.str @@ -0,0 +1,61 @@ +module ocaml +imports libstrategolib signatures/- util signature/TYPE analysis + +/* Note will use bottomup-para to traverse the full AST so that + we have access to the original expression (and can get the + Statix-associated type when we need to). + + This means that every one of our local rules must take a pair + of an original term and a term with every child replaced by + its generated code. + */ + +rules + ml: (_, TopLevel(x)) -> $[(* fostr preamble *) + type stream = { getS: string -> stream } + let rec stdio = { + getS = (fun s -> print_string s; stdio) + };; + (* End of preamble *) + + [x]] + + ml: (_, Stream()) -> $[stdio] + ml: (_, Int(x)) -> x + ml: (_, LitString(x)) -> $[{|[x]|}] + ml: (_, EscString(x)) -> x + ml: (_, Sum(x,y)) -> $[[x] + [y]] + ml: (Gets(_,yn), Gets(x, y)) + -> $[([x]).getS ([(yn,y)])] + ml: (To(xn,_), To(x, y)) + -> $[let _fto = ([x]) in (ignore (([y]).getS ([(xn,"_fto")])); _fto)] + ml: (_, Terminate(x)) -> x + ml: (_, Sequence(l)) -> l + + ml_seq: [x] -> x + ml_seq: [x | xs ] -> $[ignore ([x]); +[xs]] + + /* One drawback of using paramorphism is we have to handle lists + explicitly: + */ + ml: (_, []) -> [] + ml: (_, [x | xs]) -> [x | xs] + + /* Another drawback of using paramorphism is at the very leaves we have + to undouble the tuple: + */ + ml: (x, x) -> x where x + + ml_str: (node, code) -> $[[node]([code])] + +strategies + + ml_string_cast = get-type; (?INT() < !"string_of_int" + !"") + + ocaml = bottomup-para(try(ml)) + + // Interface ocaml code generation with editor services and file system + to-ocaml: (selected, _, _, path, project-path) -> (filename, result) + with filename := path + ; result := selected From 7feddbcfbedce0ed2623ccc6f7b338c41bdf78e7 Mon Sep 17 00:00:00 2001 From: Glen Whitney Date: Sat, 13 Mar 2021 19:30:23 +0000 Subject: [PATCH 4/4] Extraction from streams (#25) Add the ! postfix operator and !! expression. Also add the ++ string concatenation operator. Also allow specification of standard input in the test scheme. Resolves #7, #18. Co-authored-by: Glen Whitney Reviewed-on: https://code.studioinfinity.org/glen/fostr/pulls/25 Co-Authored-By: Glen Whitney Co-Committed-By: Glen Whitney --- bin/extract_tests.xsh | 8 ++++++++ bin/run_tests | 10 ++++++++-- mkdocs.yml | 2 +- syntax/fostr.sdf3 | 5 ++++- tests/basic.spt | 36 +++++++++++++++++++++++++++++++++++- trans/desugar.str | 1 + trans/haskell.str | 10 ++++++++++ trans/javascript.str | 23 ++++++++++++++++++++--- trans/ocaml.str | 11 ++++++++--- trans/python.str | 4 ++++ trans/statics.stx | 7 +++++++ 11 files changed, 106 insertions(+), 11 deletions(-) diff --git a/bin/extract_tests.xsh b/bin/extract_tests.xsh index 1d8c59f..422acb1 100644 --- a/bin/extract_tests.xsh +++ b/bin/extract_tests.xsh @@ -13,6 +13,9 @@ DESTINATION = 'tests/extracted' # Extension for extracted files: EXT = 'fos' +# Extension for desired input: +INP = 'in' + # Extension for expectations: EXP = 'expect' @@ -34,6 +37,11 @@ for path in TEST_LIST: expath = destdir / f"{name}.{EXT}" expath.write_text(example) echo Wrote @(expath) + im = re.search(r'/\*\*\s+accepts.*?\n([\s\S]*?)\*\*/', details[em.end():]) + if im: + ipath = destdir / f"{name}.{INP}" + ipath.write_text(im[1]) + echo " ...and" @(ipath) xm = re.search(r'/\*\*\s+writes.*?\n([\s\S]*?)\*\*/', details[em.end():]) if xm: xpath = destdir / f"{name}.{EXP}" diff --git a/bin/run_tests b/bin/run_tests index 90ccaa7..da49485 100755 --- a/bin/run_tests +++ b/bin/run_tests @@ -9,8 +9,14 @@ diffed=0 for dir in tests/extracted/*; do for file in $dir/*.$ext; do ((total++)) - $command $file > $file.out - if [[ $? -ne 0 ]]; then + if [[ -f ${file%.*}.in ]]; then + cat ${file%.*}.in | $command $file > $file.out + result=$? + else + $command $file > $file.out + result=$? + fi + if [[ $result -ne 0 ]]; then echo ERROR: $command $file failed. ((failed++)) else diff --git a/mkdocs.yml b/mkdocs.yml index e316a98..c2d5242 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -9,7 +9,7 @@ plugins: - search - semiliterate: ignore_folders: [target, lib] - exclude_extensions: ['.o', '.hi'] + exclude_extensions: ['.o', '.hi', '.cmi', '.cmo'] extract_standard_markdown: terminate: theme: diff --git a/syntax/fostr.sdf3 b/syntax/fostr.sdf3 index 8624b58..f782466 100644 --- a/syntax/fostr.sdf3 +++ b/syntax/fostr.sdf3 @@ -42,10 +42,13 @@ context-free syntax Ex.EscString = STRING Ex.Stream = Ex.Sum = < + > {left} + Ex.Concat = < ++ > {left} Ex.Gets = [[Ex] << [Ex]] {left} Ex.DefGets = [<<< [Ex]] Ex.To = [[Ex] >> [Ex]] {left} Ex.DefTo = [[Ex] >>>] + Ex.Emits = <!> + Ex.DefEmits = Ex = <()> {bracket} @@ -53,7 +56,7 @@ context-free priorities Ex.To > Ex.DefTo - > Ex.Sum + > {Ex.Sum Ex.Concat} > Ex.DefGets > Ex.Gets, diff --git a/tests/basic.spt b/tests/basic.spt index a5e98c0..3d5a4a6 100644 --- a/tests/basic.spt +++ b/tests/basic.spt @@ -293,4 +293,38 @@ test emit_several_default [[ >>> ]] parse succeeds /** writes -3399677527121313*/ +3399677527121313**/ + +/** md +### Streams are bidirectional + +So far we have only sent items to a stream. But we can extract them from +streams as well, with the `!` postfix operator. `!!` all by itself abbreviates +`stream!`, i.e., extraction from the standard stream. For example, + +```fostr +**/ + +/** md */ test custom_hw [[ +<<< "What is your name?\n" +<<< 'Hello, ' ++ !! +]] /* **/ +parse to TopLevel(Sequence([ + DefGets(EscString("\"What is your name?\n\"")), + DefGets(Concat(LitString("'Hello, '"),DefEmits())) +])) +/** accepts +Kilroy +**/ +/** writes +What is your name? +Hello, Kilroy +**/ + +/** md +``` + +queries users for their name and then writes a customized greeting. It also +illustrates the use of `++` for string concatenation, as opposed to `+` for +(numerical) addition. +**/ diff --git a/trans/desugar.str b/trans/desugar.str index 7276f38..b22ebd8 100644 --- a/trans/desugar.str +++ b/trans/desugar.str @@ -17,6 +17,7 @@ rules defStream: DefGets(x) -> Gets(Stream(), x) defStream: DefTo(x) -> To(x, Stream()) + defStream: DefEmits() -> Emits(Stream()) strategies diff --git a/trans/haskell.str b/trans/haskell.str index 0b08716..0e0eb15 100644 --- a/trans/haskell.str +++ b/trans/haskell.str @@ -23,6 +23,7 @@ rules import System.IO data IOStream = StdIO + -- Danger: These currently assume the stream is StdIO gets :: Show b => a -> b -> IO a gets s d = do putStr(show d) @@ -33,6 +34,10 @@ rules putStr(d) return s + emit s = do + l <- getLine + return (l ++ "\n") + main = do [()]return [val]] @@ -41,6 +46,7 @@ rules hs: (_, LitString(x)) -> x hs: (_, EscString(x)) -> x hs: (_, Sum(x, y)) -> $[([x] + [y])] + hs: (_, Concat(x, y)) -> $[([x] ++ [y])] hs: (Gets(_, xn), Gets(s, x)) -> v with v := "_fostr_get" @@ -52,6 +58,10 @@ rules hs_gets: (s, xn, x ) -> $[[s] [xn] [x]] hs_getOp = get-type; (?STRING() < !"`getsStr`" + !"`gets`") + hs: (_, Emits(s)) -> v + with v := "_fostr_emitted" + ; [$[[v] <- emit [s]]] + hs: (_, Terminate(x)) -> $[[x];;] hs: (_, Sequence(l)) -> l /* One drawback of using paramorphism is we have to handle lists diff --git a/trans/javascript.str b/trans/javascript.str index 3a847e2..9b596ad 100644 --- a/trans/javascript.str +++ b/trans/javascript.str @@ -3,24 +3,41 @@ imports libstrategolib signatures/- util rules js: TopLevel(x) -> $[// Fostr preamble + const _fostr_readline = require('readline'); + const _fostr_events = require('events'); + const _fostr_rl = _fostr_readline.createInterface({input: process.stdin}); const Stdio = { - gets: v => { process.stdout.write(String(v)); return Stdio; }, + gets: v => { process.stdout.write(String(v)); return Stdio; }, + emit: async () => { + const [line] = await _fostr_events.once(_fostr_rl, 'line'); + return line + "\n"; } } function to(data, strm) { strm.gets(data); return data; } + + const _fostr_body = async () => { // End of preamble - [x]] + [x] + + // Fostr coda + _fostr_rl.close() + } + _fostr_body(); + ] + with line := "[line]" js: Stream() -> $[Stdio] js: Int(x) -> x js: LitString(x) -> x js: EscString(x) -> x - js: Sum(x,y) -> $[[x] + [y]] + js: Sum(x, y) -> $[[x] + [y]] + js: Concat(x, y) -> $[[x] + [y]] js: Gets(x, y) -> $[[x].gets([y])] js: To(x, y) -> $[to([x],[y])] + js: Emits(x) -> $[(await [x].emit())] js: Terminate(x) -> x js: Sequence(l) -> l diff --git a/trans/ocaml.str b/trans/ocaml.str index 11fcdca..049dbf4 100644 --- a/trans/ocaml.str +++ b/trans/ocaml.str @@ -12,9 +12,10 @@ imports libstrategolib signatures/- util signature/TYPE analysis rules ml: (_, TopLevel(x)) -> $[(* fostr preamble *) - type stream = { getS: string -> stream } + type stream = { getS: string -> stream; emitS: unit -> string } let rec stdio = { - getS = (fun s -> print_string s; stdio) + getS = (fun s -> print_string s; stdio); + emitS = (fun () -> (read_line ()) ^ "\n"); };; (* End of preamble *) @@ -24,11 +25,15 @@ rules ml: (_, Int(x)) -> x ml: (_, LitString(x)) -> $[{|[x]|}] ml: (_, EscString(x)) -> x - ml: (_, Sum(x,y)) -> $[[x] + [y]] + ml: (_, Sum(x, y)) -> $[[x] + [y]] + ml: (_, Concat(x, y)) -> $[[x] ^ [y]] + ml: (Gets(_,yn), Gets(x, y)) -> $[([x]).getS ([(yn,y)])] ml: (To(xn,_), To(x, y)) -> $[let _fto = ([x]) in (ignore (([y]).getS ([(xn,"_fto")])); _fto)] + ml: (_, Emits(s)) -> $[[s].emitS ()] + ml: (_, Terminate(x)) -> x ml: (_, Sequence(l)) -> l diff --git a/trans/python.str b/trans/python.str index 6449417..f42b1cb 100644 --- a/trans/python.str +++ b/trans/python.str @@ -8,6 +8,8 @@ rules def gets(self, v): print(v, file=sys.stdout, end='') return self + def emit(self): + return input() + "\n" # Python inconsistently strips when using input def to(data,strm): strm.gets(data) return data @@ -21,8 +23,10 @@ rules py: LitString(x) -> $[r[x]] py: EscString(x) -> x py: Sum(x,y) -> $[[x] + [y]] + py: Concat(x,y) -> $[[x] + [y]] py: Gets(x, y) -> $[[x].gets([y])] py: To(x, y) -> $[to([x],[y])] + py: Emits(x) -> $[[x].emit()] py: Terminate(x) -> $[[x];] py: Sequence(l) -> l diff --git a/trans/statics.stx b/trans/statics.stx index 340a1df..ce99bf7 100644 --- a/trans/statics.stx +++ b/trans/statics.stx @@ -222,6 +222,13 @@ This pattern lets us specify error messages. type_Ex(e2) == STREAM() | error $[Items may only be sent to Streams.]@e2. /* **/ + ty_Ex(Concat(e1, e2)) = STRING() :- + type_Ex(e1) == STRING() | error $[Expression [e1] not String in concat.]@e1, + type_Ex(e2) == STRING() | error $[Expression [e2] not String in concat.]@e2. + + ty_Ex(Emits(e)) = STRING() :- // At the moment, only stream is stdio + type_Ex(e) == STREAM() | error $[Only Streams may emit items.]@e. + /** md ```