feat: Add inclusion syntax for extracted content

Also updated the documentation, using the extraction syntax.

  Resolves #2.

mkdocs_semiliterate/plugin.py

@@ -8,15 +8,88 @@ you will have immediate access to all of the following extensions. (Note that th
 documentation assumes a familiarity with the
 [usage](https://athackst.github.io/mkdocs-simple-plugin/mkdocs_simple_plugin/plugin/)
 of the `simple` plugin.)
+"""
+
+from mkdocs import utils
+from mkdocs.config import config_options
+from mkdocs_simple_plugin.plugin import SimplePlugin, StreamExtract, LazyFile
+
+import re
+import yaml
+
+
+class StreamInclusion(StreamExtract):
+    """ md  An extension of the StreamExtract class which adds
 
 ### Inclusion syntax
 
-To be documented when implemented.
+While extracting content from a file (because it matches one of the
+`semiliterate` patterns, rather than just one of the `include_extensions`),
+an unescaped expression of the form
+
+`{! FILENAME YAML !}`
+
+(which may span multiple lines) will trigger file inclusion. The FILENAME may
+be a bare word, in which case it cannot contain whitespace, or it may be
+enclosed in single or double quotes. Note that FILENAME is interpreted relative
+to the directory in which the file containing the `{! .. !}` expression
+resides. The YAML is interpreted exactly as the extraction options to a
+`semiliterate` item as
+[documented](https://athackst.github.io/mkdocs-simple-plugin/mkdocs_simple_plugin/plugin/index.html#plugin_usage)
+for the `simple` extension. The text extracted from FILENAME
+is interpolated at the current location in the file currently being written.
+Recursive inclusion is supported.
+
+Note that the `{! ... !}` directive must be in lines that semiliterate would
+normally copy. That is, it does not examine lines before the `start` regexp
+is encountered, or after the `terminate` regexp, or between instances of
+`stop` and `start`. It also doesn't check any text written from lines that
+match these special expressions. Moreover, on such normally-transcribed lines,
+it's the text **after** the application of any semiliterate replacements that
+is checked for `{! ... !}`.
      """
 
-import mkdocs
+    include_open = re.compile(r'''(?<![`\\])(\{\!\s*)([\s'"])''')
-from mkdocs.config import config_options
+    include_quoted_file = re.compile(
-from mkdocs_simple_plugin.plugin import SimplePlugin
+        r'''(['"])(?P<fn>.*?)\1\s+(?P<yml>[\s\S]*?)\s\!\}''')
+    include_bare_file = re.compile(r'\s(?P<fn>.*?)\s+(?P<yml>[\s\S]*?)\s\!\}')
+
+    def extract_line(self, line):
+        """Copy line to the output stream, applying all specified replacements
+           and handling inclusion syntax.
+        """
+        line = self.replace_line(line)
+        include_match = StreamInclusion.include_open.search(line)
+        if not include_match:
+            self.transcribe(line)
+            return
+        # OK, we have found (the start of) an inclusion and must process it
+        preamble = line[:include_match.start()]
+        remainder = line[include_match.end(1):]
+        body_pattern = StreamInclusion.include_quoted_file
+        if include_match[2].isspace():
+            body_pattern = StreamInclusion.include_bare_file
+        body_match = body_pattern.search(remainder)
+        if not body_match:
+            for extra_line in self.input_stream:
+                remainder += self.replace_line(extra_line)
+                body_match = body_pattern.search(remainder)
+                if body_match:
+                    break
+        if not body_match:
+            raise EOFError("semiliterate: End of file while scanning for `!}`")
+        include_path = self.include_root + '/' + body_match['fn']
+        new_root = re.match(r'(.*)/', include_path)[1]
+        include_parameters = yaml.safe_load(body_match['yml'])
+        with open(include_path) as include_file:
+            self.transcribe(preamble)
+            inclusion = StreamInclusion(
+                include_file, self.output_stream, include_root=new_root,
+                **include_parameters)
+            if inclusion.productive():
+                self.wrote_something = True
+        self.transcribe(remainder[body_match.end():])
+
 
 class SemiliteratePlugin(SimplePlugin):
     """ md   An extension of the mkdocs-simple-plugin
@@ -26,12 +99,13 @@ class SemiliteratePlugin(SimplePlugin):
 behavior as compared to `simple`. They are described in this section, with their
 default values in parentheses at the beginning of each entry.
 
-copy_standard_markdown
+{! plugin.py ---
-:   (false) Whether to add MkDocs' list of standard Markdown extensions to the
+  start: '[*]SimplePlugin.config_scheme'
-    `include_extensions` parameter so that Markdown files will be directly copied
+  terminate: '^\s*\)'
-    to the docsite. Note that the `simple` behavior corresponds to a _true_ value
+  replace:
-    for `copy_standard_markdown`, but `semiliterate` still incorporates all
+  - ['\(.(.*)., config_options.Type\(.*, default=(.*)\)\)', '\1\n:  (\2)']
-    standard Markdown files because of the `extract_standard_markdown` default.
+  - '^\s*#(.*\s*)$'
+!}
 
 extract_standard_markdown
 :   (true) To be implemented and documented
@@ -40,9 +114,52 @@ extract_standard_markdown
     config_scheme = (
         *SimplePlugin.config_scheme,
         ('copy_standard_markdown', config_options.Type(bool, default=False))
+        # Whether to add MkDocs' list of standard Markdown extensions to the
+        #    `include_extensions` parameter so that Markdown files will be
+        #    directly copied to the docsite. Note that the `simple` behavior
+        #    corresponds to a _true_ value for `copy_standard_markdown`, but
+        #    `semiliterate` still incorporates all standard Markdown files
+        #    because of the `extract_standard_markdown` default.
     )
 
     def copy_doc_files(self, destination_directory):
         if not self.config['copy_standard_markdown']:
             self.include_extensions = self.config['include_extensions']
         return super().copy_doc_files(destination_directory)
+
+    ## FIXME: This method is copied from simple, just to insert a control
+    ## over what class is used to do the extraction. Try to get this inserted as
+    ## the method of the same name in simple.
+    def extract_from(self, from_directory, name, destination_directory):
+        """Extract content from the file in _from_directory_ named _name_
+           to a file or files in _destination_directory_, as specified by
+           the semiliterate parameters.
+        """
+        new_paths = []
+        original = "{}/{}".format(from_directory, name)
+        for item in self.semiliterate:
+            name_match = item['pattern'].search(name)
+            if name_match:
+                new_name = (name[:name_match.start(name_match.lastindex)]
+                            + '.md'
+                            + name[name_match.end(name_match.lastindex):])
+                if 'destination' in item:
+                    new_name = name_match.expand(item['destination'])
+                new_file = LazyFile(destination_directory, new_name)
+                with open(original) as original_file:
+                    utils.log.debug(
+                        "mkdocs-simple-plugin: Scanning {} ...".format(original))
+                    productive = self.try_extraction(
+                        original_file, from_directory, new_file, **item)
+                    new_file.close()
+                    if productive:
+                        new_path = "{}/{}".format(destination_directory,
+                                                  new_name)
+                        utils.log.debug(
+                            "        ... extracted {}".format(new_path))
+                        new_paths.append((original, new_path))
+        return new_paths
+
+    def try_extraction(self, original_file, root, new_file, **kwargs):
+        return StreamInclusion(
+            original_file, new_file, include_root=root, **kwargs).productive()

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = mkdocs-semiliterate
-version = 0.0.3
+version = 0.0.4
 
 [options]
 packages = mkdocs_semiliterate