diff --git a/.drone.yml b/.drone.yml index 93b304c..f61cefe 100644 --- a/.drone.yml +++ b/.drone.yml @@ -24,8 +24,8 @@ steps: ### develop # You can build the distribution with # ``` - - pip install pep517 - - python -m pep517.build . + - pip install build + - python -m build . # ``` # That will produce a `.whl` file in the `dist` subdirectory. ### diff --git a/README.md b/README.md index ff88169..41c53e0 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # MkDocs semiliterate Plugin This plugin for [MkDocs](http://mkdocs.org) is an extension of Allison Thackston's excellent [mkdocs-simple-plugin](https://athackst.github.io/mkdocs-simple-plugin). It allows you to include content from one file into another (via `{! ... !}` syntax), using exactly the same extraction specification that the `simple` plugin already uses for identifying documentation in source files. - + ## Rationale diff --git a/mkdocs_semiliterate/plugin.py b/mkdocs_semiliterate/plugin.py index 54f2c0b..08e86d0 100644 --- a/mkdocs_semiliterate/plugin.py +++ b/mkdocs_semiliterate/plugin.py @@ -12,7 +12,9 @@ of the `simple` plugin.) from mkdocs import utils from mkdocs.config import config_options -from mkdocs_simple_plugin.plugin import SimplePlugin, StreamExtract +from mkdocs_simple_plugin.semiliterate import ( + Semiliterate, LazyFile, ExtractionPattern, StreamExtract, get_line) +from mkdocs_simple_plugin.plugin import SimplePlugin import os import re @@ -21,6 +23,37 @@ import tempfile import yaml +class FlextractionPattern(ExtractionPattern): + r""" Extends ExtractionPattern to add ensure_line argument +to replace_line method. + """ + # Following must be identical to ExtractionPattern.replace_line, + # except as marked: + def replace_line(self, line, ensure_line=True): + """Apply the specified replacements to the line and return it.""" + if not self.replace: + return line + for item in self.replace: + pattern = item[0] if isinstance(item, tuple) else item + match_object = pattern.search(line) + if match_object: + # CHANGES HERE + replaced = False + replacement = '' + if isinstance(item, tuple): + replacement = match_object.expand(item[1]) + replaced = True + elif match_object.lastindex: + replacement = match_object[match_object.lastindex] + replaced = True + if replaced and ensure_line: + replacement = get_line(replacement) + return replacement + # END OF CHANGES + # Otherwise, just return the line. + return line + + class StreamInclusion(StreamExtract): r""" md An extension of the StreamExtract class which adds @@ -39,9 +72,9 @@ to the directory in which the file containing the `{! .. !}` expression resides. The YAML is interpreted exactly as the extraction options to a `semiliterate` item as [documented](https://athackst.github.io/mkdocs-simple-plugin/mkdocs_simple_plugin/plugin/index.html#semiliterate) -for the `simple` extension. The text extracted from FILENAME -is interpolated at the current location in the file currently being written. -Recursive inclusion is supported. +for the `simple` extension, subject to the extensions below. The text +extracted from FILENAME is interpolated at the current location in the file +currently being written. Recursive inclusion is supported. The simplest example of such an inclusion directive is just `{! boilerplate.md !}`, which (because of the conventions for extraction @@ -52,7 +85,7 @@ For an example that uses more of the extraction parameters, the current version number of mkdocs-semiliterate is extracted into the [Overview](../README.md) of this documentation via -` {! ../README.md extract: { start: 'repo:.*(\{!.*!\})' } +` {! ../README.md extract: { start: 'repo:.*(\{!.*!\})', stop: '' } terminate: Rationale !}` @@ -81,11 +114,30 @@ is checked for `{! ... !}`. r'''(['"])(?P.*?)\1\s+(?P[\s\S]*?)\s?\!\}''') include_bare_file = re.compile(r'\s(?P.*?)\s+(?P[\s\S]*?)\s?\!\}') - def extract_line(self, line): + def __init__(self, input_stream, output_stream, include_root, + ensurelines=True, terminate=None, patterns=None, **kwargs): + if terminate and not hasattr(terminate, 'search'): + terminate = re.compile(terminate) + # Unfortunately, "simple" has now moved the pattern parsing into + # Semiliterate, so we need to reiterate the code for that here: + if patterns is None: + if 'extract' in kwargs: + extract = kwargs.pop('extract') + if isinstance(extract, dict): + extract = [extract] + patterns = [FlextractionPattern(**p) for p in extract] + else: + patterns = [FlextractionPattern()] + super().__init__(input_stream, output_stream, + terminate, patterns, **kwargs) + self.include_root = include_root + self.ensure_lines = ensurelines + + def extract_line(self, line, extraction_pattern): """Copy line to the output stream, applying all specified replacements and handling inclusion syntax. """ - line = self.replace_line(line) + line = extraction_pattern.replace_line(line, self.ensure_lines) include_match = StreamInclusion.include_open.search(line) if not include_match: self.transcribe(line) @@ -102,7 +154,8 @@ is checked for `{! ... !}`. body_match = body_pattern.search(remainder) if not body_match: for extra_line in self.input_stream: - remainder += self.replace_line(extra_line) + remainder += extraction_pattern.replace_line(extra_line, + self.ensure_lines) body_match = body_pattern.search(remainder) if body_match: break @@ -183,12 +236,36 @@ and that file is extracted from. with open(include_path) as include_file: self.transcribe(preamble) inclusion = StreamInclusion( - include_file, self.output_stream, include_root=new_root, + include_file, self.output_stream, new_root, **include_parameters) if inclusion.extract(): self.wrote_something = True self.transcribe(remainder[body_match.end():]) + # ## The following has to be identical to StreamExtract.check_pattern + # ## except for the marked bit handling ensure_lines + def check_pattern(self, pattern, line, emit_last=True): + """Check if pattern is contained in line. + + If _pattern_ is not false-y and is contained in _line_, + returns true (and if the _emit_last_ flag is true, + emits the last group of the match if any). Otherwise, + check_pattern does nothing but return false. + """ + if not pattern: + return False + match_object = pattern.search(line) + if not match_object: + return False + if match_object.lastindex and emit_last: + # CHANGES HERE + # self.transcribe(get_line(match_object[match_object.lastindex])) + to_emit = match_object[match_object.lastindex] + if self.ensure_lines: + to_emit = get_line(to_emit) + self.transcribe(to_emit) + return True + class SemiliteratePlugin(SimplePlugin): r""" md An extension of the mkdocs-simple-plugin @@ -230,21 +307,21 @@ terminate: '^\s*\)' *altered_config_scheme, ('exclude_extensions', config_options.Type(list, default=['.o'])), - # Files whose name contains a string in this list will not be processed - # by `semiliterate`, regardless of whether they might match - # `include_extensions`, the `semiliterate` patterns, or standard - # Markdown. + # Files whose name contains a string in this list will not be + # processed by `semiliterate`, regardless of whether they might + # match `include_extensions`, the `semiliterate` patterns, or + # standard Markdown. ('copy_standard_markdown', config_options.Type(bool, default=False)), - # Whether to add MkDocs' list of standard Markdown extensions to the - # `include_extensions` parameter so that Markdown files will be + # Whether to add MkDocs' list of standard Markdown extensions to + # the `include_extensions` parameter so that Markdown files will be # directly copied to the docsite. Note that the `simple` behavior # corresponds to a _true_ value for `copy_standard_markdown`, but # `semiliterate` still incorporates all standard Markdown files # because of the following `extract_standard_markdown` parameter. ('extract_standard_markdown', config_options.Type(dict, default={})) - # If the `enable` key of this dict parameter is true + # If the `enable` key of this dict parameter is true # (it defaults to the opposite of `copy_standard_markdown`), # it adds a semiliterate block causing extraction (and hence # include-directive processing) from all standard Markdown files @@ -258,7 +335,14 @@ terminate: '^\s*\)' # inclusion-directive processing) from standard Markdown files. ) - def build_docs(self): + def on_config(self, config, **kwargs): + # Since we have extensions in Demiliterate, suppress the semiliterate + # configuration until we handle it ourselves: + semi = self.config['semiliterate'] + self.config['semiliterate'] = [] + new_config = super().on_config(config, **kwargs) + self.config['semiliterate'] = semi + self.semiliterate = [Demiliterate(**item) for item in semi] self.exclude_extensions = self.config['exclude_extensions'] dflt_enable = False if not self.config['copy_standard_markdown']: @@ -266,24 +350,83 @@ terminate: '^\s*\)' dflt_enable = True if self.config['extract_standard_markdown'].get('enable', dflt_enable): ext_pat = '|'.join(re.escape(s) for s in utils.markdown_extensions) - self.semiliterate.append(dict( - pattern=re.compile(f"^(.*(?:{ext_pat}))$"), - destination=r'\1', - **self.config['extract_standard_markdown'])) - paths = [] - for root, directories, files in os.walk("."): - if self.in_include_directory(root): - document_root = self.build_docs_dir + root[1:] - for f in files: - if any(ext in f for ext in self.exclude_extensions): - continue - paths.extend(self.copy_file(root, f, document_root)) - paths.extend(self.extract_from(root, f, document_root)) - directories[:] = [d for d in directories - if self.in_search_directory(d, root)] - return paths + self.semiliterate.append( + Demiliterate( + pattern=re.compile(f"^(.*(?:{ext_pat}))$"), + destination=r'\1', + **self.config['extract_standard_markdown'])) + return new_config - def try_extraction(self, original_file, root, new_file, **kwargs): - extraction = StreamInclusion( - original_file, new_file, include_root=root, **kwargs) - return extraction.extract() + def in_extensions(self, file): + if any(ext in file for ext in self.exclude_extensions): + return False + return super().in_extensions(file) + + def extract_from(self, from_directory, name, to_directory): + if any(ext in name for ext in self.exclude_extensions): + return False + return super().extract_from(from_directory, name, to_directory) + + +class Demiliterate(Semiliterate): + r""" md Extends Semiliterate to use StreamInclusion, not StreamExtract + +semiliterate.ensurelines +: (true) Guarantees that a newline is trancribed for each line of the input, + even if a start, stop, terminate, or replacement pattern would have + suppressed the newline. Note this can be set separately for each block + (i.e. filename pattern) within the semiliterate parameter. The default + setting corresponds to the `simple` behavior, so setting this to "false" + allows you to suppress newlines with judicious use of these patterns. + """ + + def __init__( + self, + pattern, + destination=None, + terminate=None, + ensurelines=True, + extract=[]): + super().__init__(pattern, destination, terminate) + self.ensure_lines = ensurelines + if isinstance(extract, dict): + extract = [extract] + self.patterns = [FlextractionPattern(**p) for p in extract] + if len(self.patterns) == 0: + self.patterns = [FlextractionPattern()] + + # Note this needs to be identical to Semiliterate.try_extraction + # except with StreamInclusion in place of StreamExtract in the + # `extraction =` line, and a couple of options as noted below: + def try_extraction( + self, + from_directory, + from_file, + destination_directory, + **kwargs): + """Try to extract documentation from file with name. + + Returns True if extraction was successful. + """ + to_file = self.filenname_match(from_file) + if not to_file: + return False + from_file_path = os.path.join(from_directory, from_file) + try: + with open(from_file_path) as original_file: + utils.log.debug( + "mkdocs-simple-plugin: Scanning {}...".format(from_file)) + # extraction = StreamExtract( + extraction = StreamInclusion( + input_stream=original_file, + output_stream=LazyFile(destination_directory, to_file), + include_root=from_directory, # ## ADDED + ensurelines=self.ensure_lines, # ## ADDED + terminate=self.terminate, + patterns=self.patterns, + **kwargs) + return extraction.extract() + except BaseException: + utils.log.error("mkdocs-simple-plugin: could not build {}".format( + from_file_path)) + return False diff --git a/setup.cfg b/setup.cfg index fffab8d..8a5ab98 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = mkdocs-semiliterate -version = 0.2.0 +version = 0.3.0 description = Extension of mkdocs-simple-plugin adding easy content inclusion long_description = file: README.md long_description_content_type = text/markdown diff --git a/tests/fixtures/full-inclusion/refsite/404.html b/tests/fixtures/full-inclusion/refsite/404.html.cropped similarity index 80% rename from tests/fixtures/full-inclusion/refsite/404.html rename to tests/fixtures/full-inclusion/refsite/404.html.cropped index a5b03b3..df35a94 100644 --- a/tests/fixtures/full-inclusion/refsite/404.html +++ b/tests/fixtures/full-inclusion/refsite/404.html.cropped @@ -7,17 +7,8 @@ - Full inclusion - - - - - - - - @@ -52,11 +43,8 @@

Documentation built with MkDocs.

- -