feat: Allow semiliterate to adjust the theme

The mechanism here is to shadow a specified `custom.theme_dir` in the
  generated docs directory, and then ignore generated files therein (since they
  will be being used by the theme to render the pages, and are not actually
  part of the documentation).

mkdocs_semiliterate/plugin.py

@@ -19,6 +19,7 @@ from mkdocs_simple_plugin.plugin import SimplePlugin
 import os
 import re
 import subprocess
+import sys
 import tempfile
 import yaml
 
@@ -164,9 +165,8 @@ is checked for `{! ... !}`.
             utils.log.error(errmsg)
             raise EOFError(errmsg)
         filename = body_match['fn']
-        gitextract = False
         r""" md
-### Double-quoted filenames and Git extraction
+### Double-quoted filenames and special extraction
 
 Standard Python escape sequences in double-quoted filenames are interpreted
 as usual; for example you can write
@@ -179,9 +179,13 @@ as usual; for example you can write
 to include a file whose name (`snippet/Say "Don't"`, in this case) has both
 double and single quotes.
 
-Further, `semiliterate` supports a special escape to extract a file from the
-Git archive of the project (presuming it is under Git version control) and then
-include content from that file. For example, you could write
+Further, `semiliterate` supports some special escape sequences for
+doublequoted file names to include text from other places than current files
+in the project tree:
+
+`\git`: extracts a version of a file from the Git archive of the project
+(presuming it is under Git version control) and then
+includes content from that file. For example, you could write
 ```
 {! ../tests/fixtures/git-inclusion/README.md extract:
   start: '(.*!.*)'
@@ -206,14 +210,32 @@ form
 
 is that the output of `git show SPECIFIER` is written to a temporary file,
 and that file is extracted from.
+
+`\syspath`: searches for the remainder of the doublequoted filename in the
+python `sys.path` and includes content from the found file. For example, you
+could write
+```
+{! ../tests/fixtures/theme-modification/doc_theme/.base.generator extract:
+  start: '(.*!.*)'
+!}
+```
+to create a version of the "base.html" of the "readthedocs" theme with
+additional content at the very top of the body. As the example suggests, this
+mechanism is primarily useful to tweak an mkdocs theme in ways not
+anticipated by the `{%- block ... %}` directives placed by the theme writer.
 """
+        gitextract = False
+        syspathextract = False
         if doublequoted:
             if filename[:5] == r'\git ':
                 gitextract = True
                 filename = filename[5:]
+            elif filename[:9] == r'\syspath ':
+                syspathextract = True
+                filename = filename[9:]
             filename = (filename.encode('latin-1', 'backslashreplace')
                                 .decode('unicode-escape'))
-        include_path = self.include_root + '/' + filename
+        include_path = os.path.join(self.include_root, filename)
         if gitextract:
             (write_handle, include_path) = tempfile.mkstemp()
             utils.log.info(
@@ -221,7 +243,13 @@ and that file is extracted from.
             contents = subprocess.check_output(['git', 'show', filename])
             os.write(write_handle, contents)
             os.close(write_handle)
-        new_root = re.match(r'(.*)/', include_path)[1]
+        if syspathextract:
+            for dirname in sys.path:
+                candidate = os.path.join(dirname, filename)
+                if os.path.isfile(candidate):
+                    include_path = candidate
+                    break
+        new_root = os.path.dirname(include_path)
         try:
             include_parameters = yaml.safe_load(body_match['yml'])
         except Exception as err:
@@ -300,6 +328,10 @@ default values in parentheses at the beginning of each entry.
   - ['config_options.Type.*?default=([^\)]*)', ':  (\1)']
   - '^\s*#(.*\s*)$'
 terminate: '^\s*\)'
+!}
+{! plugin.py extract:
+  start: 'r["]{3}Extend'
+  stop: '["]{3}'
 !}
     """
     config_scheme = (
@@ -359,8 +391,42 @@ terminate: '^\s*\)'
                     pattern=re.compile(f"^(.*(?:{ext_pat}))$"),
                     destination=r'\1',
                     **self.config['extract_standard_markdown']))
+        r""" md
+### Adjusting the mkdocs theme
+
+`semiliterate` also makes it possible to add generated files to the mkdocs
+theme. It does this by detecting if a `theme.custom_dir` parameter has been set
+in the mkdocs configuration, and if so, it adds the corresponding directory
+in the generated docs dir to the theme search path. (Note this means that
+files in the corresponding subdirectory of your project will be copied into
+the resulting doc site unless their names start with a '.')
+        """
+        cfpath = os.path.dirname(config.config_file_path)
+        self.custom_dir = None
+        for themedir in config['theme'].dirs:
+            common = os.path.commonpath([cfpath, themedir])
+            if common == cfpath:
+                self.custom_dir = os.path.relpath(themedir, cfpath)
+                newthemedir = os.path.join(self.build_docs_dir, self.custom_dir)
+                utils.log.debug(
+                    'mkdocs-semiliterate: found theme.custom_dir = '
+                    + self.custom_dir
+                    + f"; adding theme directory {newthemedir}")
+                config['theme'].dirs.insert(0, newthemedir)
+                break
         return new_config
 
+    def on_files(self, files, config):
+        # If we designated a subdirectory for the theme, ignore files in it
+        if self.custom_dir:
+            sources = files.src_paths
+            for path in sources:
+                if path.startswith(self.custom_dir):
+                    utils.log.debug(
+                        f"mkdocs-semiliterate: ignoring {path} "
+                        + f"from theme directory {self.custom_dir}")
+                    files.remove(sources[path])
+
     def in_extensions(self, file):
         if any(ext in file for ext in self.exclude_extensions):
             return False
@@ -375,7 +441,7 @@ terminate: '^\s*\)'
 
 
 class Demiliterate(Semiliterate):
-    r""" md  Extends Semiliterate to use StreamInclusion, not StreamExtract
+    r"""Extends Semiliterate to use StreamInclusion, not StreamExtract
 
 semiliterate.ensurelines
 :   (true) Guarantees that a newline is trancribed for each line of the input,

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = mkdocs-semiliterate
-version = 0.5.0
+version = 0.6.0
 description = Extension of mkdocs-simple-plugin adding easy content inclusion
 long_description = file: README.md
 long_description_content_type = text/markdown

tests/fixtures/theme-modification/README.md

@@ -0,0 +1,3 @@
+# Test of syspath extraction
+
+Hopefully Kilroy magically visited above.

tests/fixtures/theme-modification/doc_theme/.base.generator

@@ -0,0 +1,4 @@
+{! "\syspath mkdocs/themes/readthedocs/base.html"
+extract:
+  replace: [ ['([<]body.*$)', '\1\n<p>Kilroy was here.</p>\n'] ]
+!}

tests/fixtures/theme-modification/mkdocs.yml

@@ -0,0 +1,13 @@
+site_name: syspath inclusion
+docs_dir: refsite # dummy
+theme:
+  name: readthedocs
+  custom_dir: doc_theme/
+plugins:
+- semiliterate:
+    ignore_folders: [refsite, snippet]
+    merge_docs_dir: false
+    include_extensions: []
+    semiliterate:
+      - pattern: '[.](base).generator$' # Amend readthedocs theme
+        destination: '\1.html'

tests/fixtures/theme-modification/refsite/404.html.cropped

@@ -0,0 +1,89 @@
+<!DOCTYPE html>
+<html class="writer-html5" lang="en" >
+<head>
+    <meta charset="utf-8" />
+    <meta http-equiv="X-UA-Compatible" content="IE=edge" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>syspath inclusion</title>
+    
+    <!--[if lt IE 9]>
+    <![endif]-->
+</head>
+
+<body class="wy-body-for-nav" role="document">
+<p>Kilroy was here.</p>
+
+  <div class="wy-grid-for-nav">
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side stickynav">
+    <div class="wy-side-scroll">
+      <div class="wy-side-nav-search">
+          <a href="/." class="icon icon-home"> syspath inclusion
+        </a>
+      </div>
+
+      <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
+              <ul>
+                <li class="toctree-l1"><a class="reference internal" href="/.">Test of syspath extraction</a>
+                </li>
+              </ul>
+      </div>
+    </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+      <nav class="wy-nav-top" role="navigation" aria-label="Mobile navigation menu">
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="/.">syspath inclusion</a>
+        
+      </nav>
+      <div class="wy-nav-content">
+        <div class="rst-content"><div role="navigation" aria-label="breadcrumbs navigation">
+  <ul class="wy-breadcrumbs">
+    <li><a href="/." class="icon icon-home" alt="Docs"></a> &raquo;</li>
+    <li class="wy-breadcrumbs-aside">
+    </li>
+  </ul>
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+            <div class="section" itemprop="articleBody">
+              
+
+  <h1 id="404-page-not-found">404</h1>
+
+  <p><strong>Page not found</strong></p>
+
+
+            </div>
+          </div><footer>
+
+  <hr/>
+
+  <div role="contentinfo">
+    <!-- Copyright etc -->
+  </div>
+
+  Built with <a href="https://www.mkdocs.org/">MkDocs</a> using a <a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+</footer>
+          
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+
+  <div class="rst-versions" role="note" aria-label="Versions">
+  <span class="rst-current-version" data-toggle="rst-current-version">
+    
+    
+    
+  </span>
+</div>
+        window.onload = function () {
+            SphinxRtdTheme.Navigation.enable(true);
+        };
+
+</body>
+</html>
+

tests/fixtures/theme-modification/refsite/css/theme_extra.css

@@ -0,0 +1,191 @@
+/*
+ * Wrap inline code samples otherwise they shoot of the side and
+ * can't be read at all.
+ *
+ * https://github.com/mkdocs/mkdocs/issues/313
+ * https://github.com/mkdocs/mkdocs/issues/233
+ * https://github.com/mkdocs/mkdocs/issues/834
+ */
+.rst-content code {
+    white-space: pre-wrap;
+    word-wrap: break-word;
+    padding: 2px 5px;
+}
+
+/**
+ * Make code blocks display as blocks and give them the appropriate
+ * font size and padding.
+ *
+ * https://github.com/mkdocs/mkdocs/issues/855
+ * https://github.com/mkdocs/mkdocs/issues/834
+ * https://github.com/mkdocs/mkdocs/issues/233
+ */
+.rst-content pre code {
+    white-space: pre;
+    word-wrap: normal;
+    display: block;
+    padding: 12px;
+    font-size: 12px;
+}
+
+/**
+ * Fix code colors
+ *
+ * https://github.com/mkdocs/mkdocs/issues/2027
+ */
+.rst-content code {
+    color: #E74C3C;
+}
+
+.rst-content pre code {
+    color: #000;
+    background: #f8f8f8;
+}
+
+/*
+ * Fix link colors when the link text is inline code.
+ *
+ * https://github.com/mkdocs/mkdocs/issues/718
+ */
+a code {
+    color: #2980B9;
+}
+a:hover code {
+    color: #3091d1;
+}
+a:visited code {
+    color: #9B59B6;
+}
+
+/*
+ * The CSS classes from highlight.js seem to clash with the
+ * ReadTheDocs theme causing some code to be incorrectly made
+ * bold and italic.
+ *
+ * https://github.com/mkdocs/mkdocs/issues/411
+ */
+pre .cs, pre .c {
+    font-weight: inherit;
+    font-style: inherit;
+}
+
+/*
+ * Fix some issues with the theme and non-highlighted code
+ * samples. Without and highlighting styles attached the
+ * formatting is broken.
+ *
+ * https://github.com/mkdocs/mkdocs/issues/319
+ */
+.rst-content .no-highlight {
+    display: block;
+    padding: 0.5em;
+    color: #333;
+}
+
+
+/*
+ * Additions specific to the search functionality provided by MkDocs
+ */
+
+.search-results {
+    margin-top: 23px;
+}
+
+.search-results article {
+    border-top: 1px solid #E1E4E5;
+    padding-top: 24px;
+}
+
+.search-results article:first-child {
+    border-top: none;
+}
+
+form .search-query {
+    width: 100%;
+    border-radius: 50px;
+    padding: 6px 12px;  /* csslint allow: box-model */
+    border-color: #D1D4D5;
+}
+
+/*
+ * Improve inline code blocks within admonitions.
+ *
+ * https://github.com/mkdocs/mkdocs/issues/656
+ */
+ .rst-content .admonition code {
+    color: #404040;
+    border: 1px solid #c7c9cb;
+    border: 1px solid rgba(0, 0, 0, 0.2);
+    background: #f8fbfd;
+    background: rgba(255, 255, 255, 0.7);
+}
+
+/*
+ * Account for wide tables which go off the side.
+ * Override borders to avoid weirdness on narrow tables.
+ *
+ * https://github.com/mkdocs/mkdocs/issues/834
+ * https://github.com/mkdocs/mkdocs/pull/1034
+ */
+.rst-content .section .docutils {
+    width: 100%;
+    overflow: auto;
+    display: block;
+    border: none;
+}
+
+td, th {
+    border: 1px solid #e1e4e5 !important; /* csslint allow: important */
+    border-collapse: collapse;
+}
+
+/*
+ * Without the following amendments, the navigation in the theme will be
+ * slightly cut off. This is due to the fact that the .wy-nav-side has a
+ * padding-bottom of 2em, which must not necessarily align with the font-size of
+ * 90 % on the .rst-current-version container, combined with the padding of 12px
+ * above and below. These amendments fix this in two steps: First, make sure the
+ * .rst-current-version container has a fixed height of 40px, achieved using
+ * line-height, and then applying a padding-bottom of 40px to this container. In
+ * a second step, the items within that container are re-aligned using flexbox.
+ *
+ * https://github.com/mkdocs/mkdocs/issues/2012
+ */
+ .wy-nav-side {
+    padding-bottom: 40px;
+}
+
+/*
+ * The second step of above amendment: Here we make sure the items are aligned
+ * correctly within the .rst-current-version container. Using flexbox, we
+ * achieve it in such a way that it will look like the following:
+ *
+ * [No repo_name]
+ *         Next >>                    // On the first page
+ * << Previous     Next >>            // On all subsequent pages
+ *
+ * [With repo_name]
+ *    <repo_name>        Next >>      // On the first page
+ * <repo_name>  << Previous  Next >>  // On all subsequent pages
+ *
+ * https://github.com/mkdocs/mkdocs/issues/2012
+ */
+.rst-versions .rst-current-version {
+    padding: 0 12px;
+    display: flex;
+    font-size: initial;
+    justify-content: space-between;
+    align-items: center;
+    line-height: 40px;
+}
+
+/*
+ * Please note that this amendment also involves removing certain inline-styles
+ * from the file ./mkdocs/themes/readthedocs/versions.html.
+ *
+ * https://github.com/mkdocs/mkdocs/issues/2012
+ */
+.rst-current-version span {
+    flex: 1;
+    text-align: center;
+}

tests/fixtures/theme-modification/refsite/index.html.cropped

@@ -0,0 +1,97 @@
+<!DOCTYPE html>
+<html class="writer-html5" lang="en" >
+<head>
+    <meta charset="utf-8" />
+    <meta http-equiv="X-UA-Compatible" content="IE=edge" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="description" content="None" />
+    <title>syspath inclusion</title>
+    
+        // Current page data
+        var mkdocs_page_name = "Test of syspath extraction";
+        var mkdocs_page_input_path = "README.md";
+        var mkdocs_page_url = null;
+    
+    <!--[if lt IE 9]>
+    <![endif]-->
+</head>
+
+<body class="wy-body-for-nav" role="document">
+<p>Kilroy was here.</p>
+
+  <div class="wy-grid-for-nav">
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side stickynav">
+    <div class="wy-side-scroll">
+      <div class="wy-side-nav-search">
+          <a href="." class="icon icon-home"> syspath inclusion
+        </a>
+      </div>
+
+      <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
+              <ul class="current">
+                <li class="toctree-l1 current"><a class="reference internal current" href=".">Test of syspath extraction</a>
+    <ul class="current">
+    </ul>
+                </li>
+              </ul>
+      </div>
+    </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+      <nav class="wy-nav-top" role="navigation" aria-label="Mobile navigation menu">
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href=".">syspath inclusion</a>
+        
+      </nav>
+      <div class="wy-nav-content">
+        <div class="rst-content"><div role="navigation" aria-label="breadcrumbs navigation">
+  <ul class="wy-breadcrumbs">
+    <li><a href="." class="icon icon-home" alt="Docs"></a> &raquo;</li>
+      <li>Test of syspath extraction</li>
+    <li class="wy-breadcrumbs-aside">
+    </li>
+  </ul>
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+            <div class="section" itemprop="articleBody">
+              
+                <h1 id="test-of-syspath-extraction">Test of syspath extraction</h1>
+<p>Hopefully Kilroy magically visited above.</p>
+              
+            </div>
+          </div><footer>
+
+  <hr/>
+
+  <div role="contentinfo">
+    <!-- Copyright etc -->
+  </div>
+
+  Built with <a href="https://www.mkdocs.org/">MkDocs</a> using a <a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+</footer>
+          
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+
+  <div class="rst-versions" role="note" aria-label="Versions">
+  <span class="rst-current-version" data-toggle="rst-current-version">
+    
+    
+    
+  </span>
+</div>
+        window.onload = function () {
+            SphinxRtdTheme.Navigation.enable(true);
+        };
+
+</body>
+</html>
+
+<!--
+-->
+

tests/fixtures/theme-modification/refsite/js/html5shiv.min.js

@@ -0,0 +1,4 @@
+/**
+* @preserve HTML5 Shiv 3.7.3 | @afarkas @jdalton @jon_neal @rem | MIT/GPL2 Licensed
+*/
+!function(a,b){function c(a,b){var c=a.createElement("p"),d=a.getElementsByTagName("head")[0]||a.documentElement;return c.innerHTML="x<style>"+b+"</style>",d.insertBefore(c.lastChild,d.firstChild)}function d(){var a=t.elements;return"string"==typeof a?a.split(" "):a}function e(a,b){var c=t.elements;"string"!=typeof c&&(c=c.join(" ")),"string"!=typeof a&&(a=a.join(" ")),t.elements=c+" "+a,j(b)}function f(a){var b=s[a[q]];return b||(b={},r++,a[q]=r,s[r]=b),b}function g(a,c,d){if(c||(c=b),l)return c.createElement(a);d||(d=f(c));var e;return e=d.cache[a]?d.cache[a].cloneNode():p.test(a)?(d.cache[a]=d.createElem(a)).cloneNode():d.createElem(a),!e.canHaveChildren||o.test(a)||e.tagUrn?e:d.frag.appendChild(e)}function h(a,c){if(a||(a=b),l)return a.createDocumentFragment();c=c||f(a);for(var e=c.frag.cloneNode(),g=0,h=d(),i=h.length;i>g;g++)e.createElement(h[g]);return e}function i(a,b){b.cache||(b.cache={},b.createElem=a.createElement,b.createFrag=a.createDocumentFragment,b.frag=b.createFrag()),a.createElement=function(c){return t.shivMethods?g(c,a,b):b.createElem(c)},a.createDocumentFragment=Function("h,f","return function(){var n=f.cloneNode(),c=n.createElement;h.shivMethods&&("+d().join().replace(/[\w\-:]+/g,function(a){return b.createElem(a),b.frag.createElement(a),'c("'+a+'")'})+");return n}")(t,b.frag)}function j(a){a||(a=b);var d=f(a);return!t.shivCSS||k||d.hasCSS||(d.hasCSS=!!c(a,"article,aside,dialog,figcaption,figure,footer,header,hgroup,main,nav,section{display:block}mark{background:#FF0;color:#000}template{display:none}")),l||i(a,d),a}var k,l,m="3.7.3",n=a.html5||{},o=/^<|^(?:button|map|select|textarea|object|iframe|option|optgroup)$/i,p=/^(?:a|b|code|div|fieldset|h1|h2|h3|h4|h5|h6|i|label|li|ol|p|q|span|strong|style|table|tbody|td|th|tr|ul)$/i,q="_html5shiv",r=0,s={};!function(){try{var a=b.createElement("a");a.innerHTML="<xyz></xyz>",k="hidden"in a,l=1==a.childNodes.length||function(){b.createElement("a");var a=b.createDocumentFragment();return"undefined"==typeof a.cloneNode||"undefined"==typeof a.createDocumentFragment||"undefined"==typeof a.createElement}()}catch(c){k=!0,l=!0}}();var t={elements:n.elements||"abbr article aside audio bdi canvas data datalist details dialog figcaption figure footer header hgroup main mark meter nav output picture progress section summary template time video",version:m,shivCSS:n.shivCSS!==!1,supportsUnknownElements:l,shivMethods:n.shivMethods!==!1,type:"default",shivDocument:j,createElement:g,createDocumentFragment:h,addElements:e};a.html5=t,j(b),"object"==typeof module&&module.exports&&(module.exports=t)}("undefined"!=typeof window?window:this,document);

tests/fixtures/theme-modification/refsite/js/theme_extra.js

@@ -0,0 +1,8 @@
+/*
+ * Assign 'docutils' class to tables so styling and
+ * JavaScript behavior is applied.
+ *
+ * https://github.com/mkdocs/mkdocs/issues/2028
+ */
+
+$('div.rst-content table').addClass('docutils');

tests/fixtures/theme-modification/refsite/sitemap.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+    <url>
+         <loc>None</loc>
+         <lastmod>2022-08-09</lastmod>
+         <changefreq>daily</changefreq>
+    </url>
+</urlset>