feat: Allow expressions to be terminated/sequenced by ;

Note that ultimately a terminated sequence may have
   a slightly different semantics (applying streams
   to `_|_`, most likely) but for now they don't.

bin/extract_tests.xsh

@@ -23,8 +23,10 @@ for path in TEST_LIST:
   tests = re.split(r'test\s*(.+?)\s*\[\[.*?\n', contents)[1:]
   testit = iter(tests)
   for name, details in zip(testit, testit):
-    pfm = re.search(r'\n\s*\]\].*?parse\s*fails', details)
+    pfm  = re.search(r'\n\s*\]\][\s\S]*?parse\s*fails', details)
     if pfm: continue   # skip examples that don't parse
+    ntfm = re.search(r'\n\s*\]\].*?don.t.test', details)
+    if ntfm: continue  # explicit skip
     em = re.search(r'\n\s*\]\]', details)
     if not em: continue
     example = details[:em.start()+1]

editor/Syntax.esv

@@ -20,6 +20,7 @@ menus
     
     action: "Format"          = editor-format (source)
     action: "Show parsed AST" = debug-show-aterm (source)
+    action: "Desugar AST"     = debug-desugar-fostr (source)
 
 views
   

syntax/fostr.sdf3

@@ -10,16 +10,24 @@ context-free start-symbols
 
 context-free sorts
 
-  Start LineSeq Line Ex
+  Start LineSeq Line OptTermEx TermExLst TermEx Ex
 
 context-free syntax
   
   Start.TopLevel   = LineSeq
 
-  LineSeq          = <<ln:Ex>> {layout(offside ln)}
+  LineSeq          = Line
-  LineSeq.Sequence = sq:Ex+    {layout(align-list sq)}
+  LineSeq.Sequence = sq:Line+            {layout(align-list sq)}
 
-  Ex+         = Ex+ ln:Ex {layout(offside ln)}
+  Line             = OptTermEx
+  Line.ISequence   = TermExLst OptTermEx {layout(0.first.line == 1.first.line)}
+
+  TermExLst.Prior  = TermEx+
+
+  OptTermEx        = ex:Ex               {layout(offside ex)}
+  OptTermEx        = te:TermEx           {layout(offside te)}
+
+  TermEx.Terminate = <<Ex>;>
 
   Ex.Int      = INT
   Ex.Stream   = <stream>
@@ -36,8 +44,4 @@ context-free priorities
   > Ex.Gets,
 
   // prevent cycle: no singletons
-  LineSeq.Sequence <0> .> Ex+       = Ex,
+  LineSeq.Sequence <0> .> Line+ = Line
-
-  // flat: No LineSeq immediately in LineSeq
-  Ex+ = Ex <0>     .> LineSeq.Sequence,
-  Ex+ = Ex+ Ex <1> .> LineSeq.Sequence

tests/basic.spt

@@ -14,8 +14,8 @@ that writes the sum of the ASCII codes for 'H', 'W', and '!' to standard output:
 
 /** md */ test emit_sum [[
 stream << 72 + 87 + 33
-]]/* **/ parse to TopLevel(Gets(Stream(),
+]] /* **/
-                                Sum(Sum(Int("72"), Int("87")), Int("33"))))
+parse to TopLevel(Gets(Stream(), Sum(Sum(Int("72"), Int("87")), Int("33"))))
 /** writes
 192**/
 
@@ -59,7 +59,8 @@ is also left-associative, so that way we can chain insertions into a stream:
 
 /** md */ test emit_twice [[
 stream << 72 + 87 + 33 << 291
-]]/* **/ parse to TopLevel(
+]] /* **/
+parse to TopLevel(
   Gets(Gets(Stream(), Sum(Sum(Int("72"), Int("87")), Int("33"))), Int("291")))
 /** writes
 192291**/
@@ -76,7 +77,8 @@ with `>>`; both forms return the first argument, so the following writes "824":
 
 /** md */ test enters_twice [[
 (7 + 8 >> stream + 9) >> stream
-]]/* **/ parse to TopLevel(
+]] /* **/
+parse to TopLevel(
   To(Sum(Sum(Int("7"), To(Int("8"), Stream())), Int("9")), Stream()))
 /** writes
 824**/
@@ -96,7 +98,8 @@ stream <<
   7
   + 8 >> stream
   + 9
-]]/* **/ parse to TopLevel(
+]] /* **/
+parse to TopLevel(
   Gets(Stream(), Sum(Sum(Int("7"), To(Int("8"), Stream())), Int("9"))))
 /** writes
 824**/
@@ -107,15 +110,17 @@ stream <<
 ```fostr
 **/
 
-/** md */ test enter_receive_bad_break [[
+/** md */ test enter_receive_bad_continuation [[
 (7 + 8 >> stream + 9)
 >> (stream << 9 + 2)
-]] /* **/ parse fails
+]] /* **/
+parse fails
 
 /* Extra tests not in the tour */
 test enter_receive [[
 (7 + 8 >> stream + 9) >> (stream << 9 + 2)
-]]/* **/ parse to TopLevel(
+]] /* **/
+parse to TopLevel(
   To(Sum(Sum(Int("7"),To(Int("8"),Stream())),Int("9")),
      Gets(Stream(),Sum(Int("9"),Int("2")))))
 /** writes
@@ -135,18 +140,20 @@ lines are evaluated in sequence. For example, the program
     + 96
   99 + 12 >>
           stream
-]] /* **/ parse to TopLevel( Sequence(
+]] /* **/
-    [ Gets(Stream(), Sum(Int("72"), Int("87")))
+parse to TopLevel(Sequence([
-    , Gets(Stream(), Sum(Int("88"), Int("96")))
+  Gets(Stream(), Sum(Int("72"), Int("87"))),
-    , Sum(Int("99"), To(Int("12"), Stream()))]))
+  Gets(Stream(), Sum(Int("88"), Int("96"))),
+  Sum(Int("99"), To(Int("12"), Stream()))
+]))
 /** writes
 15918412**/
 
 /** md
 ```
 
-will write 15918412. fostr enforces that successive expressions in sequence
+will write 15918412. The fostr parser enforces that successive expressions
-must line up at the left, i.e., the following will not parse:
+in sequence align at the left; e.g., the following fails to parse:
 
 ```fostr
 **/
@@ -156,8 +163,61 @@ must line up at the left, i.e., the following will not parse:
 stream << 88
     + 96
   99 + 12 >> stream
-]] /* **/ parse fails
+]] /* **/
+parse fails
+
+/** md
+```
+
+Note you can optionally terminate an expression in a sequence with a semicolon,
+and you may place multiple expressions on a single line if the earlier one(s)
+are so terminated. So the following is OK:
+
+```fostr
+**/
+
+/** md */ test emit_several [[
+  stream << 1 + 2; 3 >> stream
+  (4 + 5) >> stream; stream << 6;
+  stream << 7
+  stream << 8
+    + (9+10);
+  11 + 12 >> stream; 13 >> stream
+                        >> stream
+]] /* **/
+parse to TopLevel(Sequence([
+  ISequence(Prior([Terminate(Gets(Stream(), Sum(Int("1"), Int("2"))))]),
+            To(Int("3"), Stream())),
+  ISequence(Prior([Terminate(To(Sum(Int("4"), Int("5")), Stream()))]),
+            Terminate(Gets(Stream(), Int("6")))),
+  Gets(Stream(), Int("7")),
+  Terminate(Gets(Stream(), Sum(Int("8"), Sum(Int("9"), Int("10"))))),
+  ISequence(Prior([Terminate(Sum(Int("11"), To(Int("12"), Stream())))]),
+            To(To(Int("13"), Stream()), Stream()))
+]))
+/** writes
+3396727121313**/
 
 /** md
 ```
 **/
+
+test emit_several_desugar [[
+  stream << 1 + 2; 3 >> stream
+  (4 + 5) >> stream; stream << 6;
+  stream << 7
+  stream << 8
+    + (9+10);
+  11 + 12 >> stream; 13 >> stream
+                        >> stream
+]] /* don't test */
+run desugar-fostr to TopLevel(Sequence([
+  Terminate(Gets(Stream(), Sum(Int("1"), Int("2")))),
+  To(Int("3"), Stream()),
+  Terminate(To(Sum(Int("4"), Int("5")), Stream())),
+  Terminate(Gets(Stream(), Int("6"))),
+  Gets(Stream(), Int("7")),
+  Terminate(Gets(Stream(), Sum(Int("8"), Sum(Int("9"), Int("10"))))),
+  Terminate(Sum(Int("11"), To(Int("12"), Stream()))),
+  To(To(Int("13"), Stream()), Stream())
+]))

tests/emit_several.fos

@@ -0,0 +1,7 @@
+stream << 1 + 2; 3 >> stream
+(4 + 5) >> stream; stream << 6;
+stream << 7 
+stream << 8
+  + (9+10);
+11 + 12 >> stream; 13 >> stream
+                      >> stream

trans/analysis.str

@@ -9,6 +9,7 @@ imports
   injections/-
 
   libspoofax/term/origin
+  desugar
 
 rules // Analysis
 
@@ -19,7 +20,8 @@ rules // Analysis
   // multi-file analysis
 //  editor-analyze = stx-editor-analyze(pre-analyze, post-analyze|"statics", "projectOk", "fileOk")
 
-  pre-analyze  = origin-track-forced(explicate-injections-fostr-Start)
+  pre-analyze  = desugar-fostr
+                 ; origin-track-forced(explicate-injections-fostr-Start)
   post-analyze = origin-track-forced(implicate-injections-fostr-Start)
 
 rules // Editor Services
@@ -31,16 +33,21 @@ rules // Editor Services
 rules // Debugging
 
   // Prints the abstract syntax ATerm of a selection.
-  debug-show-aterm: (selected, _, _, path, project-path) -> (filename, result)
+  debug-show-aterm: (sel, _, _, path, projp) -> (filename, result)
     with filename := <guarantee-extension(|"aterm")> path
-       ; result   := selected
+       ; result   := sel
+
+  // Prints the desugared abstract syntax ATerm of a selection.
+  debug-desugar-fostr: (sel, _, _, path, projp) -> (filename, result)
+    with filename := <guarantee-extension(|"desugared.aterm")> path
+       ; result   := <desugar-fostr> sel
 
   // Prints the pre-analyzed abstract syntax ATerm of a selection.
-  debug-show-pre-analyzed: (selected, _, _, path, project-path) -> (filename, result)
+  debug-show-pre-analyzed: (sel, _, _, path, projp) -> (filename, result)
     with filename := <guarantee-extension(|"pre-analyzed.aterm")> path
-       ; result   := <pre-analyze> selected
+       ; result   := <pre-analyze> sel
 
   // Prints the analyzed annotated abstract syntax ATerm of a selection.
-  debug-show-analyzed: (selected, _, _, path, project-path) -> (filename, result)
+  debug-show-analyzed: (sel, _, _, path, projp) -> (filename, result)
     with filename := <guarantee-extension(|"analyzed.aterm")> path
-       ; result   := selected
+       ; result   := sel

trans/desugar.str

@@ -0,0 +1,22 @@
+module desugar
+imports libstrategolib signatures/-
+
+rules
+  /* ISequence() and Prior() are just noise for more expressions in sequence,
+     put in to get the layout rules right. So we remove them and collapse
+     all occurrence of them into one big Sequence() call on a list.
+     
+     This is slightly tricky because there might not be any Sequence() call
+     at the top level, but yet an ISequence(). So we do it in two passes,
+     first converting ISequence()s to Sequence()s, and then collapsing
+     Sequence()s.
+  */
+  deISe:      ISequence(Prior(l),x) -> Sequence(<conc>(l, [x]))
+
+  enList: x -> [x]
+  seqFlatten: Sequence(l) -> Sequence(<mapconcat(?Sequence(<id>) <+ enList)>l)
+  
+strategies
+
+  desugar-fostr = bottomup(try(deISe <+ seqFlatten))
+ 

trans/haskell.str

@@ -1,10 +1,5 @@
 module haskell
 imports libstrategolib signatures/- util
-
-signature
-  constructors
-    TopLevel: Ex -> Ex
-
 rules
   /* Approach: Generate code from the bottom up.
      At every node, we create a pair of the implementation and
@@ -28,16 +23,16 @@ rules
   hs: Stream()             -> ("StdIO", "")
   hs: Int(x)               -> (x, "")
   hs: Sum( (c, p), (d, q)) -> ($[([c] + [d])], <conc-strings>(p,q))
-  hs: Gets((c, p), (d, q)) -> <hsget>(c,d,<conc-strings>(p,q),<newname>"fosgt")
 
+  hs: Gets((c, p), (d, q)) -> <hsget>(c,d,<conc-strings>(p,q),<newname>"fosgt")
   hsget: (s, x, p, v) -> (v, <concat-strings>[p, $[[v] <- [s] `gets` [x]],
                                               "\n"])
 
   hs: To(  (c, p), (d, q)) -> <hsto>(c,d,<conc-strings>(p,q),<newname>"fosto")
-
   hsto: (x, s, p, v) -> (v, <concat-strings>[p, $[let [v] = [x]], "\n",
                                              $[[s] `gets` [v]], "\n"])
 
+  hs: Terminate((c,p))     -> ($[[c];;], p)
   hs: Sequence(l)          -> (<last; Fst>l, <map(Snd); concat-strings>l)
 
 strategies

trans/javascript.str

@@ -1,10 +1,6 @@
 module javascript
 imports libstrategolib signatures/- util
 
-signature
-  constructors
-    TopLevel: Ex -> Ex
-
 rules
   js: TopLevel(x) -> $[const Stdio = {
     gets: v => { process.stdout.write(String(v)); return Stdio; },
@@ -20,6 +16,7 @@ rules
   js: Sum(x,y)     -> $[[x] + [y]]
   js: Gets(x, y)   -> $[[x].gets([y])]
   js: To(x, y)     -> $[to([x],[y])]
+  js: Terminate(x) -> x
   js: Sequence(l)  -> <join(|";\n")>l
 
 strategies

trans/python.str

@@ -1,10 +1,5 @@
 module python
 imports libstrategolib signatures/- util
-
-signature
-  constructors
-    TopLevel: Ex -> Ex
-
 rules
 
   py: TopLevel(x) -> $[import sys
@@ -23,6 +18,7 @@ rules
   py: Sum(x,y)     -> $[[x] + [y]]
   py: Gets(x, y)   -> $[[x].gets([y])]
   py: To(x, y)     -> $[to([x],[y])]
+  py: Terminate(x) -> $[[x];]
   py: Sequence(l)  -> <join(|"\n")>l
 
 strategies