mkdocs_semiliterate/plugin.py

@@ -19,6 +19,7 @@ from mkdocs_simple_plugin.plugin import SimplePlugin
 import os
 import re
 import subprocess
+import sys
 import tempfile
 import yaml
 
@@ -164,9 +165,8 @@ is checked for `{! ... !}`.
             utils.log.error(errmsg)
             raise EOFError(errmsg)
         filename = body_match['fn']
-        gitextract = False
         r""" md
-### Double-quoted filenames and Git extraction
+### Double-quoted filenames and special extraction
 
 Standard Python escape sequences in double-quoted filenames are interpreted
 as usual; for example you can write
@@ -179,9 +179,13 @@ as usual; for example you can write
 to include a file whose name (`snippet/Say "Don't"`, in this case) has both
 double and single quotes.
 
-Further, `semiliterate` supports a special escape to extract a file from the
-Git archive of the project (presuming it is under Git version control) and then
-include content from that file. For example, you could write
+Further, `semiliterate` supports some special escape sequences for
+doublequoted file names to include text from other places than current files
+in the project tree:
+
+`\git`: extracts a version of a file from the Git archive of the project
+(presuming it is under Git version control) and then
+includes content from that file. For example, you could write
 ```
 {! ../tests/fixtures/git-inclusion/README.md extract:
   start: '(.*!.*)'
@@ -206,14 +210,32 @@ form
 
 is that the output of `git show SPECIFIER` is written to a temporary file,
 and that file is extracted from.
+
+`\syspath`: searches for the remainder of the doublequoted filename in the
+python `sys.path` and includes content from the found file. For example, you
+could write
+```
+{! ../tests/fixtures/theme-modification/doc_theme/.base.generator extract:
+  start: '(.*!.*)'
+!}
+```
+to create a version of the "base.html" of the "readthedocs" theme with
+additional content at the very top of the body. As the example suggests, this
+mechanism is primarily useful to tweak an mkdocs theme in ways not
+anticipated by the `{%- block ... %}` directives placed by the theme writer.
 """
+        gitextract = False
+        syspathextract = False
         if doublequoted:
             if filename[:5] == r'\git ':
                 gitextract = True
                 filename = filename[5:]
+            elif filename[:9] == r'\syspath ':
+                syspathextract = True
+                filename = filename[9:]
             filename = (filename.encode('latin-1', 'backslashreplace')
                                 .decode('unicode-escape'))
-        include_path = self.include_root + '/' + filename
+        include_path = os.path.join(self.include_root, filename)
         if gitextract:
             (write_handle, include_path) = tempfile.mkstemp()
             utils.log.info(
@@ -221,7 +243,13 @@ and that file is extracted from.
             contents = subprocess.check_output(['git', 'show', filename])
             os.write(write_handle, contents)
             os.close(write_handle)
-        new_root = re.match(r'(.*)/', include_path)[1]
+        if syspathextract:
+            for dirname in sys.path:
+                candidate = os.path.join(dirname, filename)
+                if os.path.isfile(candidate):
+                    include_path = candidate
+                    break
+        new_root = os.path.dirname(include_path)
         try:
             include_parameters = yaml.safe_load(body_match['yml'])
         except Exception as err:
@@ -300,6 +328,10 @@ default values in parentheses at the beginning of each entry.
   - ['config_options.Type.*?default=([^\)]*)', ':  (\1)']
   - '^\s*#(.*\s*)$'
 terminate: '^\s*\)'
+!}
+{! plugin.py extract:
+  start: 'r["]{3}Extend'
+  stop: '["]{3}'
 !}
     """
     config_scheme = (
@@ -359,8 +391,42 @@ terminate: '^\s*\)'
                     pattern=re.compile(f"^(.*(?:{ext_pat}))$"),
                     destination=r'\1',
                     **self.config['extract_standard_markdown']))
+        r""" md
+### Adjusting the mkdocs theme
+
+`semiliterate` also makes it possible to add generated files to the mkdocs
+theme. It does this by detecting if a `theme.custom_dir` parameter has been set
+in the mkdocs configuration, and if so, it adds the corresponding directory
+in the generated docs dir to the theme search path. (Note this means that
+files in the corresponding subdirectory of your project will be copied into
+the resulting doc site unless their names start with a '.')
+        """
+        cfpath = os.path.dirname(config.config_file_path)
+        self.custom_dir = None
+        for themedir in config['theme'].dirs:
+            common = os.path.commonpath([cfpath, themedir])
+            if common == cfpath:
+                self.custom_dir = os.path.relpath(themedir, cfpath)
+                newthemedir = os.path.join(self.build_docs_dir, self.custom_dir)
+                utils.log.debug(
+                    'mkdocs-semiliterate: found theme.custom_dir = '
+                    + self.custom_dir
+                    + f"; adding theme directory {newthemedir}")
+                config['theme'].dirs.insert(0, newthemedir)
+                break
         return new_config
 
+    def on_files(self, files, config):
+        # If we designated a subdirectory for the theme, ignore files in it
+        if self.custom_dir:
+            sources = files.src_paths
+            for path in sources:
+                if path.startswith(self.custom_dir):
+                    utils.log.debug(
+                        f"mkdocs-semiliterate: ignoring {path} "
+                        + f"from theme directory {self.custom_dir}")
+                    files.remove(sources[path])
+
     def in_extensions(self, file):
         if any(ext in file for ext in self.exclude_extensions):
             return False
@@ -375,7 +441,7 @@ terminate: '^\s*\)'
 
 
 class Demiliterate(Semiliterate):
-    r""" md  Extends Semiliterate to use StreamInclusion, not StreamExtract
+    r"""Extends Semiliterate to use StreamInclusion, not StreamExtract
 
 semiliterate.ensurelines
 :   (true) Guarantees that a newline is trancribed for each line of the input,

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = mkdocs-semiliterate
-version = 0.5.0
+version = 0.6.0
 description = Extension of mkdocs-simple-plugin adding easy content inclusion
 long_description = file: README.md
 long_description_content_type = text/markdown

tests/fixtures/theme-modification/README.md

@@ -0,0 +1,3 @@
+# Test of syspath extraction
+
+Hopefully Kilroy magically visited above.

tests/fixtures/theme-modification/doc_theme/.base.generator

@@ -0,0 +1,4 @@
+{! "\syspath mkdocs/themes/readthedocs/base.html"
+extract:
+  replace: [ ['([<]body.*$)', '\1\n<p>Kilroy was here.</p>\n'] ]
+!}

tests/fixtures/theme-modification/mkdocs.yml

@@ -0,0 +1,13 @@
+site_name: syspath inclusion
+docs_dir: refsite # dummy
+theme:
+  name: readthedocs
+  custom_dir: doc_theme/
+plugins:
+- semiliterate:
+    ignore_folders: [refsite, snippet]
+    merge_docs_dir: false
+    include_extensions: []
+    semiliterate:
+      - pattern: '[.](base).generator$' # Amend readthedocs theme
+        destination: '\1.html'

tests/fixtures/theme-modification/refsite/404.html.cropped

@@ -0,0 +1,89 @@
+<!DOCTYPE html>
+<html class="writer-html5" lang="en" >
+<head>
+    <meta charset="utf-8" />
+    <meta http-equiv="X-UA-Compatible" content="IE=edge" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>syspath inclusion</title>
+    
+    <!--[if lt IE 9]>
+    <![endif]-->
+</head>
+
+<body class="wy-body-for-nav" role="document">
+<p>Kilroy was here.</p>
+
+  <div class="wy-grid-for-nav">
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side stickynav">
+    <div class="wy-side-scroll">
+      <div class="wy-side-nav-search">
+          <a href="/." class="icon icon-home"> syspath inclusion
+        </a>
+      </div>
+
+      <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
+              <ul>
+                <li class="toctree-l1"><a class="reference internal" href="/.">Test of syspath extraction</a>
+                </li>
+              </ul>
+      </div>
+    </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+      <nav class="wy-nav-top" role="navigation" aria-label="Mobile navigation menu">
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="/.">syspath inclusion</a>
+        
+      </nav>
+      <div class="wy-nav-content">
+        <div class="rst-content"><div role="navigation" aria-label="breadcrumbs navigation">
+  <ul class="wy-breadcrumbs">
+    <li><a href="/." class="icon icon-home" alt="Docs"></a> &raquo;</li>
+    <li class="wy-breadcrumbs-aside">
+    </li>
+  </ul>
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+            <div class="section" itemprop="articleBody">
+              
+
+  <h1 id="404-page-not-found">404</h1>
+
+  <p><strong>Page not found</strong></p>
+
+
+            </div>
+          </div><footer>
+
+  <hr/>
+
+  <div role="contentinfo">
+    <!-- Copyright etc -->
+  </div>
+
+  Built with <a href="https://www.mkdocs.org/">MkDocs</a> using a <a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+</footer>
+          
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+
+  <div class="rst-versions" role="note" aria-label="Versions">
+  <span class="rst-current-version" data-toggle="rst-current-version">
+    
+    
+    
+  </span>
+</div>
+        window.onload = function () {
+            SphinxRtdTheme.Navigation.enable(true);
+        };
+
+</body>
+</html>
+

tests/fixtures/theme-modification/refsite/css/theme_extra.css

@@ -0,0 +1,191 @@
+/*
+ * Wrap inline code samples otherwise they shoot of the side and
+ * can't be read at all.
+ *
+ * https://github.com/mkdocs/mkdocs/issues/313
+ * https://github.com/mkdocs/mkdocs/issues/233
+ * https://github.com/mkdocs/mkdocs/issues/834
+ */
+.rst-content code {
+    white-space: pre-wrap;
+    word-wrap: break-word;
+    padding: 2px 5px;
+}
+
+/**
+ * Make code blocks display as blocks and give them the appropriate
+ * font size and padding.
+ *
+ * https://github.com/mkdocs/mkdocs/issues/855
+ * https://github.com/mkdocs/mkdocs/issues/834
+ * https://github.com/mkdocs/mkdocs/issues/233
+ */
+.rst-content pre code {
+    white-space: pre;
+    word-wrap: normal;
+    display: block;
+    padding: 12px;
+    font-size: 12px;
+}
+
+/**
+ * Fix code colors
+ *
+ * https://github.com/mkdocs/mkdocs/issues/2027
+ */
+.rst-content code {
+    color: #E74C3C;
+}
+
+.rst-content pre code {
+    color: #000;
+    background: #f8f8f8;
+}
+
+/*
+ * Fix link colors when the link text is inline code.
+ *
+ * https://github.com/mkdocs/mkdocs/issues/718
+ */
+a code {
+    color: #2980B9;
+}
+a:hover code {
+    color: #3091d1;
+}
+a:visited code {
+    color: #9B59B6;
+}
+
+/*
+ * The CSS classes from highlight.js seem to clash with the
+ * ReadTheDocs theme causing some code to be incorrectly made
+ * bold and italic.
+ *
+ * https://github.com/mkdocs/mkdocs/issues/411
+ */
+pre .cs, pre .c {
+    font-weight: inherit;
+    font-style: inherit;
+}
+
+/*
+ * Fix some issues with the theme and non-highlighted code
+ * samples. Without and highlighting styles attached the
+ * formatting is broken.
+ *
+ * https://github.com/mkdocs/mkdocs/issues/319
+ */
+.rst-content .no-highlight {
+    display: block;
+    padding: 0.5em;
+    color: #333;
+}
+
+
+/*
+ * Additions specific to the search functionality provided by MkDocs
+ */
+
+.search-results {
+    margin-top: 23px;
+}
+
+.search-results article {
+    border-top: 1px solid #E1E4E5;
+    padding-top: 24px;
+}
+
+.search-results article:first-child {
+    border-top: none;
+}
+
+form .search-query {
+    width: 100%;
+    border-radius: 50px;
+    padding: 6px 12px;  /* csslint allow: box-model */
+    border-color: #D1D4D5;
+}
+
+/*
+ * Improve inline code blocks within admonitions.
+ *
+ * https://github.com/mkdocs/mkdocs/issues/656
+ */
+ .rst-content .admonition code {
+    color: #404040;
+    border: 1px solid #c7c9cb;
+    border: 1px solid rgba(0, 0, 0, 0.2);
+    background: #f8fbfd;
+    background: rgba(255, 255, 255, 0.7);
+}
+
+/*
+ * Account for wide tables which go off the side.
+ * Override borders to avoid weirdness on narrow tables.
+ *
+ * https://github.com/mkdocs/mkdocs/issues/834
+ * https://github.com/mkdocs/mkdocs/pull/1034
+ */
+.rst-content .section .docutils {
+    width: 100%;
+    overflow: auto;
+    display: block;
+    border: none;
+}
+
+td, th {
+    border: 1px solid #e1e4e5 !important; /* csslint allow: important */
+    border-collapse: collapse;
+}
+
+/*
+ * Without the following amendments, the navigation in the theme will be
+ * slightly cut off. This is due to the fact that the .wy-nav-side has a
+ * padding-bottom of 2em, which must not necessarily align with the font-size of
+ * 90 % on the .rst-current-version container, combined with the padding of 12px
+ * above and below. These amendments fix this in two steps: First, make sure the
+ * .rst-current-version container has a fixed height of 40px, achieved using
+ * line-height, and then applying a padding-bottom of 40px to this container. In
+ * a second step, the items within that container are re-aligned using flexbox.
+ *
+ * https://github.com/mkdocs/mkdocs/issues/2012
+ */
+ .wy-nav-side {
+    padding-bottom: 40px;
+}
+
+/*
+ * The second step of above amendment: Here we make sure the items are aligned
+ * correctly within the .rst-current-version container. Using flexbox, we
+ * achieve it in such a way that it will look like the following:
+ *
+ * [No repo_name]
+ *         Next >>                    // On the first page
+ * << Previous     Next >>            // On all subsequent pages
+ *
+ * [With repo_name]
+ *    <repo_name>        Next >>      // On the first page
+ * <repo_name>  << Previous  Next >>  // On all subsequent pages
+ *
+ * https://github.com/mkdocs/mkdocs/issues/2012
+ */
+.rst-versions .rst-current-version {
+    padding: 0 12px;
+    display: flex;
+    font-size: initial;
+    justify-content: space-between;
+    align-items: center;
+    line-height: 40px;
+}
+
+/*
+ * Please note that this amendment also involves removing certain inline-styles
+ * from the file ./mkdocs/themes/readthedocs/versions.html.
+ *
+ * https://github.com/mkdocs/mkdocs/issues/2012
+ */
+.rst-current-version span {
+    flex: 1;
+    text-align: center;
+}

tests/fixtures/theme-modification/refsite/index.html.cropped

@@ -0,0 +1,97 @@
+<!DOCTYPE html>
+<html class="writer-html5" lang="en" >
+<head>
+    <meta charset="utf-8" />
+    <meta http-equiv="X-UA-Compatible" content="IE=edge" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="description" content="None" />
+    <title>syspath inclusion</title>
+    
+        // Current page data
+        var mkdocs_page_name = "Test of syspath extraction";
+        var mkdocs_page_input_path = "README.md";
+        var mkdocs_page_url = null;
+    
+    <!--[if lt IE 9]>
+    <![endif]-->
+</head>
+
+<body class="wy-body-for-nav" role="document">
+<p>Kilroy was here.</p>
+
+  <div class="wy-grid-for-nav">
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side stickynav">
+    <div class="wy-side-scroll">
+      <div class="wy-side-nav-search">
+          <a href="." class="icon icon-home"> syspath inclusion
+        </a>
+      </div>
+
+      <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
+              <ul class="current">
+                <li class="toctree-l1 current"><a class="reference internal current" href=".">Test of syspath extraction</a>
+    <ul class="current">
+    </ul>
+                </li>
+              </ul>
+      </div>
+    </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+      <nav class="wy-nav-top" role="navigation" aria-label="Mobile navigation menu">
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href=".">syspath inclusion</a>
+        
+      </nav>
+      <div class="wy-nav-content">
+        <div class="rst-content"><div role="navigation" aria-label="breadcrumbs navigation">
+  <ul class="wy-breadcrumbs">
+    <li><a href="." class="icon icon-home" alt="Docs"></a> &raquo;</li>
+      <li>Test of syspath extraction</li>
+    <li class="wy-breadcrumbs-aside">
+    </li>
+  </ul>
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+            <div class="section" itemprop="articleBody">
+              
+                <h1 id="test-of-syspath-extraction">Test of syspath extraction</h1>
+<p>Hopefully Kilroy magically visited above.</p>
+              
+            </div>
+          </div><footer>
+
+  <hr/>
+
+  <div role="contentinfo">
+    <!-- Copyright etc -->
+  </div>
+
+  Built with <a href="https://www.mkdocs.org/">MkDocs</a> using a <a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+</footer>
+          
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+
+  <div class="rst-versions" role="note" aria-label="Versions">
+  <span class="rst-current-version" data-toggle="rst-current-version">
+    
+    
+    
+  </span>
+</div>
+        window.onload = function () {
+            SphinxRtdTheme.Navigation.enable(true);
+        };
+
+</body>
+</html>
+
+<!--
+-->
+

tests/fixtures/theme-modification/refsite/js/html5shiv.min.js

@@ -0,0 +1,4 @@
+/**
+* @preserve HTML5 Shiv 3.7.3 | @afarkas @jdalton @jon_neal @rem | MIT/GPL2 Licensed
+*/
+!function(a,b){function c(a,b){var c=a.createElement("p"),d=a.getElementsByTagName("head")[0]||a.documentElement;return c.innerHTML="x<style>"+b+"</style>",d.insertBefore(c.lastChild,d.firstChild)}function d(){var a=t.elements;return"string"==typeof a?a.split(" "):a}function e(a,b){var c=t.elements;"string"!=typeof c&&(c=c.join(" ")),"string"!=typeof a&&(a=a.join(" ")),t.elements=c+" "+a,j(b)}function f(a){var b=s[a[q]];return b||(b={},r++,a[q]=r,s[r]=b),b}function g(a,c,d){if(c||(c=b),l)return c.createElement(a);d||(d=f(c));var e;return e=d.cache[a]?d.cache[a].cloneNode():p.test(a)?(d.cache[a]=d.createElem(a)).cloneNode():d.createElem(a),!e.canHaveChildren||o.test(a)||e.tagUrn?e:d.frag.appendChild(e)}function h(a,c){if(a||(a=b),l)return a.createDocumentFragment();c=c||f(a);for(var e=c.frag.cloneNode(),g=0,h=d(),i=h.length;i>g;g++)e.createElement(h[g]);return e}function i(a,b){b.cache||(b.cache={},b.createElem=a.createElement,b.createFrag=a.createDocumentFragment,b.frag=b.createFrag()),a.createElement=function(c){return t.shivMethods?g(c,a,b):b.createElem(c)},a.createDocumentFragment=Function("h,f","return function(){var n=f.cloneNode(),c=n.createElement;h.shivMethods&&("+d().join().replace(/[\w\-:]+/g,function(a){return b.createElem(a),b.frag.createElement(a),'c("'+a+'")'})+");return n}")(t,b.frag)}function j(a){a||(a=b);var d=f(a);return!t.shivCSS||k||d.hasCSS||(d.hasCSS=!!c(a,"article,aside,dialog,figcaption,figure,footer,header,hgroup,main,nav,section{display:block}mark{background:#FF0;color:#000}template{display:none}")),l||i(a,d),a}var k,l,m="3.7.3",n=a.html5||{},o=/^<|^(?:button|map|select|textarea|object|iframe|option|optgroup)$/i,p=/^(?:a|b|code|div|fieldset|h1|h2|h3|h4|h5|h6|i|label|li|ol|p|q|span|strong|style|table|tbody|td|th|tr|ul)$/i,q="_html5shiv",r=0,s={};!function(){try{var a=b.createElement("a");a.innerHTML="<xyz></xyz>",k="hidden"in a,l=1==a.childNodes.length||function(){b.createElement("a");var a=b.createDocumentFragment();return"undefined"==typeof a.cloneNode||"undefined"==typeof a.createDocumentFragment||"undefined"==typeof a.createElement}()}catch(c){k=!0,l=!0}}();var t={elements:n.elements||"abbr article aside audio bdi canvas data datalist details dialog figcaption figure footer header hgroup main mark meter nav output picture progress section summary template time video",version:m,shivCSS:n.shivCSS!==!1,supportsUnknownElements:l,shivMethods:n.shivMethods!==!1,type:"default",shivDocument:j,createElement:g,createDocumentFragment:h,addElements:e};a.html5=t,j(b),"object"==typeof module&&module.exports&&(module.exports=t)}("undefined"!=typeof window?window:this,document);

tests/fixtures/theme-modification/refsite/js/theme_extra.js

@@ -0,0 +1,8 @@
+/*
+ * Assign 'docutils' class to tables so styling and
+ * JavaScript behavior is applied.
+ *
+ * https://github.com/mkdocs/mkdocs/issues/2028
+ */
+
+$('div.rst-content table').addClass('docutils');

tests/fixtures/theme-modification/refsite/sitemap.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+    <url>
+         <loc>None</loc>
+         <lastmod>2022-08-09</lastmod>
+         <changefreq>daily</changefreq>
+    </url>
+</urlset>