chore: Update to mkdocs-simple-plugin v1.0

Also updates from pep517.build to the `build` package, and bumps the
  semiliterate version number.
  Resolves #14.

mkdocs_semiliterate/plugin.py

@@ -12,7 +12,9 @@ of the `simple` plugin.)
 
 from mkdocs import utils
 from mkdocs.config import config_options
-from mkdocs_simple_plugin.plugin import SimplePlugin, StreamExtract
+from mkdocs_simple_plugin.semiliterate import (
+    Semiliterate, LazyFile, ExtractionPattern, StreamExtract, get_line)
+from mkdocs_simple_plugin.plugin import SimplePlugin
 
 import os
 import re
@@ -21,6 +23,37 @@ import tempfile
 import yaml
 
 
+class FlextractionPattern(ExtractionPattern):
+    r""" Extends ExtractionPattern to add ensure_line argument
+to replace_line method.
+    """
+    # Following must be identical to ExtractionPattern.replace_line,
+    # except as marked:
+    def replace_line(self, line, ensure_line=True):
+        """Apply the specified replacements to the line and return it."""
+        if not self.replace:
+            return line
+        for item in self.replace:
+            pattern = item[0] if isinstance(item, tuple) else item
+            match_object = pattern.search(line)
+            if match_object:
+                # CHANGES HERE
+                replaced = False
+                replacement = ''
+                if isinstance(item, tuple):
+                    replacement = match_object.expand(item[1])
+                    replaced = True
+                elif match_object.lastindex:
+                    replacement = match_object[match_object.lastindex]
+                    replaced = True
+                if replaced and ensure_line:
+                    replacement = get_line(replacement)
+                return replacement
+                # END OF CHANGES
+        # Otherwise, just return the line.
+        return line
+
+
 class StreamInclusion(StreamExtract):
     r""" md  An extension of the StreamExtract class which adds
 
@@ -39,9 +72,9 @@ to the directory in which the file containing the `{! .. !}` expression
 resides. The YAML is interpreted exactly as the extraction options to a
 `semiliterate` item as
 [documented](https://athackst.github.io/mkdocs-simple-plugin/mkdocs_simple_plugin/plugin/index.html#semiliterate)
-for the `simple` extension. The text extracted from FILENAME
-is interpolated at the current location in the file currently being written.
-Recursive inclusion is supported.
+for the `simple` extension, subject to the extensions below. The text
+extracted from FILENAME is interpolated at the current location in the file
+currently being written. Recursive inclusion is supported.
 
 The simplest example of such an inclusion directive is just
 `{! boilerplate.md !}`, which (because of the conventions for extraction
@@ -52,7 +85,7 @@ For an example that uses more of the extraction parameters, the current
 version number of mkdocs-semiliterate is extracted into the
 [Overview](../README.md) of this documentation via
 
-` {! ../README.md extract: { start: 'repo:.*(\{!.*!\})' }
+` {! ../README.md extract: { start: 'repo:.*(\{!.*!\})', stop: '' }
 terminate: Rationale
 !}`
 
@@ -81,11 +114,30 @@ is checked for `{! ... !}`.
         r'''(['"])(?P<fn>.*?)\1\s+(?P<yml>[\s\S]*?)\s?\!\}''')
     include_bare_file = re.compile(r'\s(?P<fn>.*?)\s+(?P<yml>[\s\S]*?)\s?\!\}')
 
-    def extract_line(self, line):
+    def __init__(self, input_stream, output_stream, include_root,
+                 ensurelines=True, terminate=None, patterns=None, **kwargs):
+        if terminate and not hasattr(terminate, 'search'):
+            terminate = re.compile(terminate)
+        # Unfortunately, "simple" has now moved the pattern parsing into
+        # Semiliterate, so we need to reiterate the code for that here:
+        if patterns is None:
+            if 'extract' in kwargs:
+                extract = kwargs.pop('extract')
+                if isinstance(extract, dict):
+                    extract = [extract]
+                patterns = [FlextractionPattern(**p) for p in extract]
+            else:
+                patterns = [FlextractionPattern()]
+        super().__init__(input_stream, output_stream,
+                         terminate, patterns, **kwargs)
+        self.include_root = include_root
+        self.ensure_lines = ensurelines
+
+    def extract_line(self, line, extraction_pattern):
         """Copy line to the output stream, applying all specified replacements
            and handling inclusion syntax.
         """
-        line = self.replace_line(line)
+        line = extraction_pattern.replace_line(line, self.ensure_lines)
         include_match = StreamInclusion.include_open.search(line)
         if not include_match:
             self.transcribe(line)
@@ -102,7 +154,8 @@ is checked for `{! ... !}`.
         body_match = body_pattern.search(remainder)
         if not body_match:
             for extra_line in self.input_stream:
-                remainder += self.replace_line(extra_line)
+                remainder += extraction_pattern.replace_line(extra_line,
+                                                             self.ensure_lines)
                 body_match = body_pattern.search(remainder)
                 if body_match:
                     break
@@ -183,12 +236,36 @@ and that file is extracted from.
         with open(include_path) as include_file:
             self.transcribe(preamble)
             inclusion = StreamInclusion(
-                include_file, self.output_stream, include_root=new_root,
+                include_file, self.output_stream, new_root,
                 **include_parameters)
             if inclusion.extract():
                 self.wrote_something = True
         self.transcribe(remainder[body_match.end():])
 
+    # ## The following has to be identical to StreamExtract.check_pattern
+    # ## except for the marked bit handling ensure_lines
+    def check_pattern(self, pattern, line, emit_last=True):
+        """Check if pattern is contained in line.
+
+        If _pattern_ is not false-y and is contained in _line_,
+        returns true (and if the _emit_last_ flag is true,
+        emits the last group of the match if any). Otherwise,
+        check_pattern does nothing but return false.
+        """
+        if not pattern:
+            return False
+        match_object = pattern.search(line)
+        if not match_object:
+            return False
+        if match_object.lastindex and emit_last:
+            # CHANGES HERE
+            # self.transcribe(get_line(match_object[match_object.lastindex]))
+            to_emit = match_object[match_object.lastindex]
+            if self.ensure_lines:
+                to_emit = get_line(to_emit)
+            self.transcribe(to_emit)
+        return True
+
 
 class SemiliteratePlugin(SimplePlugin):
     r""" md   An extension of the mkdocs-simple-plugin
@@ -230,21 +307,21 @@ terminate: '^\s*\)'
         *altered_config_scheme,
         ('exclude_extensions',
          config_options.Type(list, default=['.o'])),
-        # Files whose name contains a string in this list will not be processed
-        #    by `semiliterate`, regardless of whether they might match
-        #    `include_extensions`, the `semiliterate` patterns, or standard
-        #    Markdown.
+        #    Files whose name contains a string in this list will not be
+        #    processed by `semiliterate`, regardless of whether they might
+        #    match `include_extensions`, the `semiliterate` patterns, or
+        #    standard Markdown.
         ('copy_standard_markdown',
          config_options.Type(bool, default=False)),
-        # Whether to add MkDocs' list of standard Markdown extensions to the
-        #    `include_extensions` parameter so that Markdown files will be
+        #    Whether to add MkDocs' list of standard Markdown extensions to
+        #    the `include_extensions` parameter so that Markdown files will be
         #    directly copied to the docsite. Note that the `simple` behavior
         #    corresponds to a _true_ value for `copy_standard_markdown`, but
         #    `semiliterate` still incorporates all standard Markdown files
         #    because of the following `extract_standard_markdown` parameter.
         ('extract_standard_markdown',
          config_options.Type(dict, default={}))
-        # If the `enable` key of this dict parameter is true
+        #    If the `enable` key of this dict parameter is true
         #    (it defaults to the opposite of `copy_standard_markdown`),
         #    it adds a semiliterate block causing extraction (and hence
         #    include-directive processing) from all standard Markdown files
@@ -258,7 +335,14 @@ terminate: '^\s*\)'
         #    inclusion-directive processing) from standard Markdown files.
     )
 
-    def build_docs(self):
+    def on_config(self, config, **kwargs):
+        # Since we have extensions in Demiliterate, suppress the semiliterate
+        # configuration until we handle it ourselves:
+        semi = self.config['semiliterate']
+        self.config['semiliterate'] = []
+        new_config = super().on_config(config, **kwargs)
+        self.config['semiliterate'] = semi
+        self.semiliterate = [Demiliterate(**item) for item in semi]
         self.exclude_extensions = self.config['exclude_extensions']
         dflt_enable = False
         if not self.config['copy_standard_markdown']:
@@ -266,24 +350,83 @@ terminate: '^\s*\)'
             dflt_enable = True
         if self.config['extract_standard_markdown'].get('enable', dflt_enable):
             ext_pat = '|'.join(re.escape(s) for s in utils.markdown_extensions)
-            self.semiliterate.append(dict(
-                pattern=re.compile(f"^(.*(?:{ext_pat}))$"),
-                destination=r'\1',
-                **self.config['extract_standard_markdown']))
-        paths = []
-        for root, directories, files in os.walk("."):
-            if self.in_include_directory(root):
-                document_root = self.build_docs_dir + root[1:]
-                for f in files:
-                    if any(ext in f for ext in self.exclude_extensions):
-                        continue
-                    paths.extend(self.copy_file(root, f, document_root))
-                    paths.extend(self.extract_from(root, f, document_root))
-            directories[:] = [d for d in directories
-                              if self.in_search_directory(d, root)]
-        return paths
+            self.semiliterate.append(
+                Demiliterate(
+                    pattern=re.compile(f"^(.*(?:{ext_pat}))$"),
+                    destination=r'\1',
+                    **self.config['extract_standard_markdown']))
+        return new_config
 
-    def try_extraction(self, original_file, root, new_file, **kwargs):
-        extraction = StreamInclusion(
-            original_file, new_file, include_root=root, **kwargs)
-        return extraction.extract()
+    def in_extensions(self, file):
+        if any(ext in file for ext in self.exclude_extensions):
+            return False
+        return super().in_extensions(file)
+
+    def extract_from(self, from_directory, name, to_directory):
+        if any(ext in name for ext in self.exclude_extensions):
+            return False
+        return super().extract_from(from_directory, name, to_directory)
+
+
+class Demiliterate(Semiliterate):
+    r""" md  Extends Semiliterate to use StreamInclusion, not StreamExtract
+
+semiliterate.ensurelines
+:   (true) Guarantees that a newline is trancribed for each line of the input,
+    even if a start, stop, terminate, or replacement pattern would have
+    suppressed the newline. Note this can be set separately for each block
+    (i.e. filename pattern) within the semiliterate parameter. The default
+    setting corresponds to the `simple` behavior, so setting this to "false"
+    allows you to suppress newlines with judicious use of these patterns.
+    """
+
+    def __init__(
+            self,
+            pattern,
+            destination=None,
+            terminate=None,
+            ensurelines=True,
+            extract=[]):
+        super().__init__(pattern, destination, terminate)
+        self.ensure_lines = ensurelines
+        if isinstance(extract, dict):
+            extract = [extract]
+        self.patterns = [FlextractionPattern(**p) for p in extract]
+        if len(self.patterns) == 0:
+            self.patterns = [FlextractionPattern()]
+
+    # Note this needs to be identical to Semiliterate.try_extraction
+    # except with StreamInclusion in place of StreamExtract in the
+    # `extraction =` line, and a couple of options as noted below:
+    def try_extraction(
+            self,
+            from_directory,
+            from_file,
+            destination_directory,
+            **kwargs):
+        """Try to extract documentation from file with name.
+
+        Returns True if extraction was successful.
+        """
+        to_file = self.filenname_match(from_file)
+        if not to_file:
+            return False
+        from_file_path = os.path.join(from_directory, from_file)
+        try:
+            with open(from_file_path) as original_file:
+                utils.log.debug(
+                    "mkdocs-simple-plugin: Scanning {}...".format(from_file))
+                # extraction = StreamExtract(
+                extraction = StreamInclusion(
+                    input_stream=original_file,
+                    output_stream=LazyFile(destination_directory, to_file),
+                    include_root=from_directory,  # ## ADDED
+                    ensurelines=self.ensure_lines,  # ## ADDED
+                    terminate=self.terminate,
+                    patterns=self.patterns,
+                    **kwargs)
+                return extraction.extract()
+        except BaseException:
+            utils.log.error("mkdocs-simple-plugin: could not build {}".format(
+                from_file_path))
+        return False