.drone.yml

@@ -91,7 +91,7 @@ steps:
 # [repository site](https://code.studioinfinity.org/glen/mkdocs-semiliterate).
 # Pull requests are welcome as well. If you're new to contributing to open-source
 # projects, `mkdocs-simple-plugin` has a very nice
-# [tutorial](https://althack.dev/mkdocs-simple-plugin/v{! setup.cfg { extract: {start: 'mkdocs~=', stop: '(\d*\.\d*\.?\d*)'}, ensurelines: false} !}/CONTRIBUTING/).
+# [tutorial](https://althack.dev/mkdocs-simple-plugin/v{! setup.cfg { extract: {start: 'mkdocs~=', stop: '(\d*\.\d*\.?\d*)'}} !}/CONTRIBUTING/).
 #
 # ### Publishing
 #

mkdocs.yml

@@ -7,15 +7,21 @@ plugins:
 - search
 - semiliterate:
     merge_docs_dir: false
-    ignore_folders: [build, dist, tests, semiliterate, .venv]
+    ignore: [build, dist, tests, semiliterate, .venv]
-    ignore_hidden: false
+    include: [LICENSE, '.png']
-    include_extensions: [LICENSE, '.png']
+    copy: true
     extract_standard_markdown:
       extract:
         replace: [['^(.*)<!-- repo: -->.*<!-- site:(.*?) -->(.*\s*)$', '\1\2\3']]
+      ensurelines: false
     semiliterate:
       - pattern: '\.py$'
-        extract: {start: '"""\smd', stop: '"""'}
+        extract:
+          - start: '"""\smd'
+            stop: '"""'
+          - start: '#\sstart-md'
+            stop: '#\send-md'
+            replace: ['^# (.*\s*)$']
       - pattern: '.drone.yml'
         destination: 'drone_develop.md'
         extract:

mkdocs_semiliterate/plugin.py

@@ -7,8 +7,7 @@ It accepts all of the same parameters, so `mkdocs` will still work as before,
 and you will have immediate access to all of the following extensions.
 (Note that this documentation assumes a familiarity with the
 [usage](https://althack.dev/mkdocs-simple-plugin/v{! ../setup.cfg {
-  extract: {start: 'mkdocs~=', stop: '(\d*\.\d*\.?\d*)'},
+  extract: {start: 'mkdocs~=', stop: '(\d*\.\d*\.?\d*)'}
-  ensurelines: false
 } !}/mkdocs_simple_plugin/plugin/)
 of the `simple` plugin.)
 """
@@ -16,9 +15,8 @@ of the `simple` plugin.)
 from mkdocs import utils
 from mkdocs.config import config_options
 from mkdocs_simple_plugin.semiliterate import (
-    Semiliterate, LazyFile, ExtractionPattern, StreamExtract,
+    Semiliterate, LazyFile, ExtractionPattern, StreamExtract)
-    get_line, get_match)
+from mkdocs_simple_plugin.simple import (Simple, SimplePath)
-from mkdocs_simple_plugin.simple import Simple
 from mkdocs_simple_plugin.plugin import SimplePlugin
 
 import os
@@ -29,44 +27,52 @@ import tempfile
 import yaml
 
 
-class FlextractionPattern(ExtractionPattern):
+class BorrowFile:
-    r""" Extends ExtractionPattern to add ensure_line argument
+    """Just forwards to another stream, and never closes it
-to replace_line method.
+    Except it also has its own value of ensurelines.
     """
-    # Following must be identical to ExtractionPattern.replace_line,
+    def __init__(self, other, ensurelines=True):
-    # except as marked:
+        self.other = other
-    def replace_line(self, line, ensure_line=True):
+        self.file_name = other.file_name
-        """Apply the specified replacements to the line and return it."""
+        self.ensurelines = ensurelines
-        # Process trimming
+
-        if self._trim:
+    def write_just(self, arg):
-            line = line[self._trim:]
+        self.other.write_just(arg)
-        # Process inline content regex
+
-        if self._content:
+    def write(self, arg):
-            match_object = get_match(self._content, line)
+        if (self.ensurelines and not arg.endswith("\n")):
-            if match_object.lastindex:
+            arg += "\n"
-                return match_object[match_object.lastindex]
+        self.other.write_just(arg)
-        # Perform replace operations:
+
-        if not self.replace:
+    def close(self):
-            return line
+        return None
-        for item in self.replace:
+
-            pattern = item[0] if isinstance(item, tuple) else item
+
-            match_object = pattern.search(line)
+class LazierFile(LazyFile):
-            if match_object:
+    """Just like LazyFile, except with ensurelines parameter
-                # CHANGES HERE
+    The parameter controls whether every call to write guarantees a newline
-                replaced = False
+    is written.
-                replacement = ''
+    """
-                if isinstance(item, tuple):
+    def __init__(self, directory: str, name: str, ensurelines=True):
-                    replacement = match_object.expand(item[1])
+        super().__init__(directory, name)
-                    replaced = True
+        self.ensurelines = ensurelines
-                elif match_object.lastindex:
+
-                    replacement = match_object[match_object.lastindex]
+    # Basically identical to LazyFile write, so take care to sync
-                    replaced = True
+    def write_just(self, arg: str) -> None:
-                if replaced and ensure_line:
+        """ create and write exactly arg to file, no newlines added"""
-                    replacement = get_line(replacement)
+        if arg is None:
-                return replacement
+            return
-                # END OF CHANGES
+        if self.file_object is None:
-        # Otherwise, just return the line.
+            filename = os.path.join(self.file_directory, self.file_name)
-        return line
+            os.makedirs(self.file_directory, exist_ok=True)
+            self.file_object = open(filename, 'w+')
+        self.file_object.write(arg)
+
+    def write(self, arg: str) -> None:
+        if self.ensurelines:
+            super().write(arg)
+        else:
+            self.write_just(arg)
 
 
 class StreamInclusion(StreamExtract):
@@ -75,7 +81,7 @@ class StreamInclusion(StreamExtract):
 ### Inclusion syntax
 
 While extracting content from a file (because it matches one of the
-`semiliterate` patterns, rather than just one of the `include_extensions`),
+`semiliterate` patterns, rather than just `include`),
 an unescaped expression of the form
 
 `{! FILENAME YAML !}`
@@ -128,32 +134,31 @@ is checked for `{! ... !}`.
     include_bare_file = re.compile(r'\s(?P<fn>.*?)\s+(?P<yml>[\s\S]*?)\s?\!\}')
 
     def __init__(self, input_stream, output_stream, include_root,
-                 ensurelines=True, terminate=None, patterns=None, **kwargs):
+                 terminate=None, patterns=None, extract=None, **kwargs):
         if terminate and not hasattr(terminate, 'search'):
             terminate = re.compile(terminate)
-        # Unfortunately, "simple" has now moved the pattern parsing into
+        if patterns is None and extract:
-        # Semiliterate, so we need to reiterate the code for that here:
+            # Sadly we must reiterate the pattern parsing here, because of
-        if patterns is None:
+            # inclusions not coming from a [DS]emiliterate object
-            if 'extract' in kwargs:
+            extractions = []
-                extract = kwargs.pop('extract')
             if isinstance(extract, dict):
                 extract = [extract]
-                patterns = [FlextractionPattern(**p) for p in extract]
+            for extract_params in extract:
-            else:
+                extractions.append(ExtractionPattern(**extract_params))
-                patterns = [FlextractionPattern()]
+            if len(extractions) > 0:
+                patterns = extractions
         super().__init__(input_stream, output_stream,
                          terminate, patterns, **kwargs)
         self.include_root = include_root
-        self.ensure_lines = ensurelines
 
     def extract_line(self, line, extraction_pattern):
         """Copy line to the output stream, applying all specified replacements
            and handling inclusion syntax.
         """
-        line = extraction_pattern.replace_line(line, self.ensure_lines)
+        line = extraction_pattern.replace_line(line)
         include_match = StreamInclusion.include_open.search(line)
         if not include_match:
-            self.transcribe(line)
+            self.output_stream.write(line)
             return
         # OK, we have found (the start of) an inclusion and must process it
         preamble = line[:include_match.start()]
@@ -167,8 +172,7 @@ is checked for `{! ... !}`.
         body_match = body_pattern.search(remainder)
         if not body_match:
             for extra_line in self.input_stream:
-                remainder += extraction_pattern.replace_line(extra_line,
+                remainder += extraction_pattern.replace_line(extra_line)
-                                                             self.ensure_lines)
                 body_match = body_pattern.search(remainder)
                 if body_match:
                     break
@@ -251,7 +255,7 @@ anticipated by the `{%- block ... %}` directives placed by the theme writer.
         if gitextract:
             (write_handle, include_path) = tempfile.mkstemp()
             utils.log.info(
-                f"semiliterate: extracting {filename} to {include_path}")
+                f"semiliterate: git extracting {filename} to {include_path}")
             contents = subprocess.check_output(['git', 'show', filename])
             os.write(write_handle, contents)
             os.close(write_handle)
@@ -274,35 +278,16 @@ anticipated by the `{%- block ... %}` directives placed by the theme writer.
         if not include_parameters:
             include_parameters = {}
         with open(include_path) as include_file:
-            self.transcribe(preamble)
+            self.output_stream.write(preamble)
             inclusion = StreamInclusion(
-                include_file, self.output_stream, new_root,
+                include_file,
+                BorrowFile(self.output_stream,
+                           include_parameters.get('ensurelines', True)),
+                new_root,
                 **include_parameters)
             if inclusion.extract():
                 self.wrote_something = True
-        self.transcribe(remainder[body_match.end():])
+        self.output_stream.write(remainder[body_match.end():])
-
-    # ## The following has to be identical to StreamExtract.try_extract_match
-    # ## except for the marked bit handling ensure_lines
-    def try_extract_match(
-            self,
-            match_object: re.Match,
-            emit_last: bool = True) -> bool:
-        """Extract match into output.
-
-        If _match_object_ is not false-y, returns true.
-        If extract flag is true, emits the last group of the match if any.
-        """
-        if not match_object:
-            return False
-        if match_object.lastindex and emit_last:
-            # CHANGES HERE
-            # self.transcribe(get_line(match_object[match_object.lastindex]))
-            to_emit = match_object[match_object.lastindex]
-            if self.ensure_lines:
-                to_emit = get_line(to_emit)
-            self.transcribe(to_emit)
-        return True
 
 
 class SemiliteratePlugin(SimplePlugin):
@@ -320,26 +305,22 @@ default values in parentheses at the beginning of each entry.
   - ['config_options.Type.*?default=([^\)]*)', ':  (\1)']
   - '^\s*#(.*\s*)$'
 terminate: '^\s*\)'
-!}
-{! plugin.py extract:
-  start: 'r["]{3}Extend'
-  stop: '["]{3}'
 !}
     """
     super_config_scheme = SimplePlugin.config_scheme
     config_scheme = (
         # Note documentation of each new parameter **follows** the parameter.
         *super_config_scheme,
-        ('exclude_extensions',
+        ('exclude',
          config_options.Type(list, default=['.o'])),
         #    Files whose name contains a string in this list will not be
         #    processed by `semiliterate`, regardless of whether they might
-        #    match `include_extensions`, the `semiliterate` patterns, or
+        #    match `include`, the `semiliterate` patterns, or
         #    standard Markdown.
         ('copy_standard_markdown',
          config_options.Type(bool, default=False)),
         #    Whether to add MkDocs' list of standard Markdown extensions to
-        #    the `include_extensions` parameter so that Markdown files will be
+        #    the `include` parameter so that Markdown files will be
         #    directly copied to the docsite. Note that the `simple` behavior
         #    corresponds to a _true_ value for `copy_standard_markdown`, but
         #    `semiliterate` still incorporates all standard Markdown files
@@ -365,19 +346,10 @@ terminate: '^\s*\)'
     )
 
     def on_config(self, config, **kwargs):
-        r""" md
-### Adjusting the mkdocs theme
-
-`semiliterate` also makes it possible to add generated files to the mkdocs
-theme. It does this by detecting if a `theme.custom_dir` parameter has been set
-in the mkdocs configuration, and if so, it adds the corresponding directory
-in the generated docs dir to the theme search path. (Note this means that
-files in the corresponding subdirectory of your project will be copied into
-the resulting doc site unless their names start with a '.')
-        """
         # Save the include extensions before SimplePlugin modifies them:
-        self.config['saved_includes'] = self.config['include_extensions']
+        saved_includes = self.config['include']
         new_config = super().on_config(config, **kwargs)
+        self.config['saved_includes'] = saved_includes
         cfpath = os.path.dirname(config.config_file_path)
         self.custom_dir = None
         for themedir in config['theme'].dirs:
@@ -385,7 +357,7 @@ the resulting doc site unless their names start with a '.')
             if common == cfpath:
                 self.custom_dir = os.path.relpath(themedir, cfpath)
                 newthemedir = os.path.join(
-                    self.config['build_docs_dir'], self.custom_dir)
+                    self.config['build_dir'], self.custom_dir)
                 utils.log.debug(
                     'mkdocs-semiliterate: found theme.custom_dir = '
                     + self.custom_dir
@@ -418,18 +390,18 @@ the resulting doc site unless their names start with a '.')
 
 class Semisimple(Simple):
     """Mkdocs Semisimple Plugin"""
-    def __init__(self, semiliterate, exclude_extensions, saved_includes,
+    def __init__(self, semiliterate, exclude, saved_includes,
                  copy_standard_markdown, extract_standard_markdown,
                  extract_on_copy, **kwargs):
         # Since we have extensions in Demiliterate, suppress the semiliterate
         # configuration until we handle it ourselves:
         super().__init__(semiliterate=[], **kwargs)
         self.semiliterate = [Demiliterate(**item) for item in semiliterate]
-        self.exclude_extensions = exclude_extensions
+        self.exclude = exclude
         self.extract_on_copy = extract_on_copy
         dflt_enable = False
         if not copy_standard_markdown:
-            self.copy_glob = set(saved_includes)
+            self.doc_glob = set(saved_includes)
             dflt_enable = True
         if extract_standard_markdown.get('enable', dflt_enable):
             ext_pat = '|'.join(re.escape(s) for s in utils.markdown_extensions)
@@ -439,44 +411,71 @@ class Semisimple(Simple):
                     destination=r'\1',
                     **extract_standard_markdown))
 
-    def should_copy_file(self, file):
+    def is_doc_file(self, name: str) -> bool:
-        if any(ext in file for ext in self.exclude_extensions):
+        if any(ext in name for ext in self.exclude):
             return False
-        return super().should_copy_file(file)
+        return super().is_doc_file(name)
 
-    def try_extract(self, from_directory, name, to_directory):
+    def try_extract(self, from_dir: str, name: str, to_dir: str) -> list:
-        if any(ext in name for ext in self.exclude_extensions):
+        if any(ext in name for ext in self.exclude):
-            return False
+            return []
-        if not self.extract_on_copy and self.should_copy_file(name):
+        if not self.extract_on_copy and self.is_doc_file(name):
-            return False
+            return []
-        return super().try_extract(from_directory, name, to_directory)
+        return super().try_extract(from_dir, name, to_dir)
 
     # Had to override this because the simple version hardcoded that if a file
     # was copied, it could not be extracted. So check carefully for changes in
-    # simple. Only the lines between # # START and # # END differ.
+    # simple. Only the line marked # REMOVED was commented out
-    def build_docs(self) -> list:
+    def build_docs(
+            self,
+            dirty=False,
+            last_build_time=None,
+            do_copy=False) -> list:
         """Build the docs directory from workspace files."""
         paths = []
         files = self.get_files()
         for file in files:
             if not os.path.isfile(file):
                 continue
+            if dirty and last_build_time and (
+                    os.path.getmtime(file) <= last_build_time):
+                continue
             from_dir = os.path.dirname(file)
             name = os.path.basename(file)
             build_prefix = os.path.normpath(
                 os.path.join(self.build_dir, from_dir))
 
-            # # START
+            doc_paths = self.get_doc_file(
-            copied = self.try_copy_file(from_dir, name, build_prefix)
+                from_dir, name, build_prefix, True)
-            extracted = self.try_extract(from_dir, name, build_prefix)
+            if doc_paths:
-            if (copied or extracted):
+                paths.append(
-                paths.append(file)
+                    SimplePath(
-            # # END
+                        output_root=".",
+                        output_relpath=os.path.relpath(path=file, start="."),
+                        input_path=file)
+                )
+                utils.log.info("mkdocs-semiliterate: Added %s", file)
+                # continue    # REMOVED
+
+            extracted_paths = self.try_extract(from_dir, name, build_prefix)
+            for path in extracted_paths:
+                paths.append(
+                    SimplePath(
+                        output_root=self.build_dir,
+                        output_relpath=os.path.relpath(
+                            path=path,
+                            start=self.build_dir),
+                        input_path=file))
+                utils.log.info(
+                    "mkdocs-semiliterate: Extracted %s -> %s", file, path)
+            if extracted_paths:
+                continue
+
         return paths
 
 
 class Demiliterate(Semiliterate):
-    r"""Extends Semiliterate to use StreamInclusion, not StreamExtract
+    r""" md Extends Semiliterate to use StreamInclusion, not StreamExtract
 
 semiliterate.ensurelines
 :   (true) Guarantees that a newline is trancribed for each line of the input,
@@ -489,18 +488,13 @@ semiliterate.ensurelines
 
     def __init__(
             self,
-            pattern,
+            pattern: str,
-            destination=None,
+            destination: str = None,
-            terminate=None,
+            terminate: str = None,
-            ensurelines=True,
+            extract: list = None,
-            extract=[]):
+            ensurelines=True):
-        super().__init__(pattern, destination, terminate)
+        super().__init__(pattern, destination, terminate, extract)
-        self.ensure_lines = ensurelines
+        self.ensurelines = ensurelines
-        if isinstance(extract, dict):
-            extract = [extract]
-        self.patterns = [FlextractionPattern(**p) for p in extract]
-        if len(self.patterns) == 0:
-            self.patterns = [FlextractionPattern()]
 
     # Note that this has diverged noticeably from the
     # Semiliterate.try_extraction method that it overrides, especially
@@ -513,37 +507,53 @@ semiliterate.ensurelines
             from_directory,
             from_file,
             destination_directory,
-            **kwargs):
+            **kwargs) -> list:
         """Try to extract documentation from file with name.
 
-        Returns True if extraction was successful.
+        Args:
+            from_directory (str): The source directory
+            from_file (str): The source filename within directory
+            destination_directory (str): The destination directory
+
+        Returns a list of extracted files.
         """
         to_file = self.filename_match(from_file)
         if not to_file:
-            return False
+            return []
         from_file_path = os.path.join(from_directory, from_file)
         to_file_path = os.path.join(destination_directory, to_file)  # ## ADDED
         (destination_directory, to_file) = os.path.split(to_file_path)  # ADDED
         try:
             with open(from_file_path) as original_file:
                 utils.log.debug(
-                    f"mkdocs-semiliterate: In {from_directory}, "
+                    f"mkdocs-semiliterate: Scanning {from_file_path}... ")
-                    + f"scanning {from_file}...")
                 # extraction = StreamExtract(
                 extraction = StreamInclusion(
                     input_stream=original_file,
-                    output_stream=LazyFile(destination_directory, to_file),
+                    output_stream=LazierFile(
+                        destination_directory, to_file, self.ensurelines),
                     include_root=from_directory,  # ## ADDED
-                    ensurelines=self.ensure_lines,  # ## ADDED
                     terminate=self.terminate,
-                    patterns=self.patterns,
+                    patterns=self.extractions,
                     **kwargs)
                 return extraction.extract()
         except (UnicodeDecodeError) as error:
-            utils.log.info("mkdocs-semiliterate: skipping  %s\n %s",
+            utils.log.debug("mkdocs-semiliterate: Skipped  %s", from_file_path)
+            utils.log.debug(
+                "mkdocs-semiliterate: Error details: %s", str(error))
+        except (OSError, IOError) as error:
+            utils.log.error("mkdocs-semiliterate: could not build %s\n %s",
                             from_file_path, str(error))
-        except BaseException as error:
+        return []
-            utils.log.error(
+
-                f"mkdocs-semiliterate: could not build {from_file_path}:\n  "
+# start-md
-                + str(error))
+# ### Adjusting the mkdocs theme
-        return False
+
+# `semiliterate` also makes it possible to add generated files to the mkdocs
+# theme. It does this by detecting if a `theme.custom_dir` parameter has been
+# set in the mkdocs configuration, and if so, it adds the corresponding
+# directory in the generated docs dir to the theme search path. (Note this
+# means that
+# files in the corresponding subdirectory of your project will be copied into
+# the resulting doc site unless their names start with a '.')
+# end-md

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = mkdocs-semiliterate
-version = 0.7.1
+version = 0.8.0
 description = Extension of mkdocs-simple-plugin adding easy content inclusion
 long_description = file: README.md
 long_description_content_type = text/markdown
@@ -25,7 +25,7 @@ license = Apache-2.0
 packages = mkdocs_semiliterate
 install_requires =
   mkdocs~=1.4
-  mkdocs-simple-plugin==2.1.2
+  mkdocs-simple-plugin==3.2.0
 
 [options.entry_points]
 mkdocs.plugins =

tests/fixtures/extract-inclusion/mkdocs.yml

@@ -2,9 +2,9 @@ site_name: Enable extraction from included files
 docs_dir: refsite # dummy
 plugins:
 - semiliterate:
-    ignore_folders: [refsite]
+    ignore: [refsite]
     merge_docs_dir: false
-    include_extensions: ['.txt']
+    include: ['.txt']
     extract_on_copy: true
     semiliterate:
     - pattern: '\.txt'

tests/fixtures/full-inclusion/mkdocs.yml

@@ -2,5 +2,5 @@ site_name: Full inclusion
 docs_dir: refsite # dummy
 plugins:
 - semiliterate:
-    ignore_folders: [refsite, snippet]
+    ignore: [refsite, snippet]
     merge_docs_dir: false

tests/fixtures/git-inclusion/mkdocs.yml

@@ -2,5 +2,5 @@ site_name: Full inclusion
 docs_dir: refsite # dummy
 plugins:
 - semiliterate:
-    ignore_folders: [refsite, snippet]
+    ignore: [refsite, snippet]
     merge_docs_dir: false

tests/fixtures/no-extract-inclusion/mkdocs.yml

@@ -2,5 +2,5 @@ site_name: Do not extract from included files
 docs_dir: refsite # dummy
 plugins:
 - semiliterate:
-    ignore_folders: [refsite]
+    ignore: [refsite]
     merge_docs_dir: false

tests/fixtures/quoted-filename/mkdocs.yml

@@ -2,5 +2,5 @@ site_name: Full inclusion
 docs_dir: refsite # dummy
 plugins:
 - semiliterate:
-    ignore_folders: [refsite, snippet]
+    ignore: [refsite, snippet]
     merge_docs_dir: false

tests/fixtures/recursive-inclusion/mkdocs.yml

@@ -2,7 +2,7 @@ site_name: Custom semiliterate
 docs_dir: refsite # dummy
 plugins:
 - semiliterate:
-    ignore_folders: [refsite]
+    ignore: [refsite]
     merge_docs_dir: false
     copy_standard_markdown: true
     semiliterate:

tests/fixtures/sibling-destination/mkdocs.yml

@@ -2,7 +2,7 @@ site_name: Enable extraction from included files
 docs_dir: refsite # dummy
 plugins:
 - semiliterate:
-    ignore_folders: [refsite]
+    ignore: [refsite]
     merge_docs_dir: false
     semiliterate:
     - pattern: 'raw\.txt'

tests/fixtures/theme-modification/mkdocs.yml

@@ -5,10 +5,10 @@ theme:
   custom_dir: doc_theme/
 plugins:
 - semiliterate:
-    ignore_folders: [refsite]
+    ignore: [refsite]
-    ignore_hidden: false
     merge_docs_dir: false
-    include_extensions: []
+    include: []
     semiliterate:
       - pattern: '[.](base).generator$' # Amend readthedocs theme
         destination: '\1.html'
+        ensurelines: false