diff --git a/.drone.yml b/.drone.yml index 7b4477d..6966c84 100644 --- a/.drone.yml +++ b/.drone.yml @@ -91,7 +91,7 @@ steps: # [repository site](https://code.studioinfinity.org/glen/mkdocs-semiliterate). # Pull requests are welcome as well. If you're new to contributing to open-source # projects, `mkdocs-simple-plugin` has a very nice -# [tutorial](https://althack.dev/mkdocs-simple-plugin/v{! setup.cfg { extract: {start: 'mkdocs~=', stop: '(\d*\.\d*\.?\d*)'}, ensurelines: false} !}/CONTRIBUTING/). +# [tutorial](https://althack.dev/mkdocs-simple-plugin/v{! setup.cfg { extract: {start: 'mkdocs~=', stop: '(\d*\.\d*\.?\d*)'}} !}/CONTRIBUTING/). # # ### Publishing # diff --git a/mkdocs.yml b/mkdocs.yml index a4dd21a..58f1a42 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -7,15 +7,21 @@ plugins: - search - semiliterate: merge_docs_dir: false - ignore_folders: [build, dist, tests, semiliterate, .venv] - ignore_hidden: false - include_extensions: [LICENSE, '.png'] + ignore: [build, dist, tests, semiliterate, .venv] + include: [LICENSE, '.png'] + copy: true extract_standard_markdown: extract: replace: [['^(.*).*(.*\s*)$', '\1\2\3']] + ensurelines: false semiliterate: - pattern: '\.py$' - extract: {start: '"""\smd', stop: '"""'} + extract: + - start: '"""\smd' + stop: '"""' + - start: '#\sstart-md' + stop: '#\send-md' + replace: ['^# (.*\s*)$'] - pattern: '.drone.yml' destination: 'drone_develop.md' extract: diff --git a/mkdocs_semiliterate/plugin.py b/mkdocs_semiliterate/plugin.py index b04c838..6e368ad 100644 --- a/mkdocs_semiliterate/plugin.py +++ b/mkdocs_semiliterate/plugin.py @@ -7,8 +7,7 @@ It accepts all of the same parameters, so `mkdocs` will still work as before, and you will have immediate access to all of the following extensions. (Note that this documentation assumes a familiarity with the [usage](https://althack.dev/mkdocs-simple-plugin/v{! ../setup.cfg { - extract: {start: 'mkdocs~=', stop: '(\d*\.\d*\.?\d*)'}, - ensurelines: false + extract: {start: 'mkdocs~=', stop: '(\d*\.\d*\.?\d*)'} } !}/mkdocs_simple_plugin/plugin/) of the `simple` plugin.) """ @@ -16,9 +15,8 @@ of the `simple` plugin.) from mkdocs import utils from mkdocs.config import config_options from mkdocs_simple_plugin.semiliterate import ( - Semiliterate, LazyFile, ExtractionPattern, StreamExtract, - get_line, get_match) -from mkdocs_simple_plugin.simple import Simple + Semiliterate, LazyFile, ExtractionPattern, StreamExtract) +from mkdocs_simple_plugin.simple import (Simple, SimplePath) from mkdocs_simple_plugin.plugin import SimplePlugin import os @@ -29,44 +27,52 @@ import tempfile import yaml -class FlextractionPattern(ExtractionPattern): - r""" Extends ExtractionPattern to add ensure_line argument -to replace_line method. +class BorrowFile: + """Just forwards to another stream, and never closes it + Except it also has its own value of ensurelines. """ - # Following must be identical to ExtractionPattern.replace_line, - # except as marked: - def replace_line(self, line, ensure_line=True): - """Apply the specified replacements to the line and return it.""" - # Process trimming - if self._trim: - line = line[self._trim:] - # Process inline content regex - if self._content: - match_object = get_match(self._content, line) - if match_object.lastindex: - return match_object[match_object.lastindex] - # Perform replace operations: - if not self.replace: - return line - for item in self.replace: - pattern = item[0] if isinstance(item, tuple) else item - match_object = pattern.search(line) - if match_object: - # CHANGES HERE - replaced = False - replacement = '' - if isinstance(item, tuple): - replacement = match_object.expand(item[1]) - replaced = True - elif match_object.lastindex: - replacement = match_object[match_object.lastindex] - replaced = True - if replaced and ensure_line: - replacement = get_line(replacement) - return replacement - # END OF CHANGES - # Otherwise, just return the line. - return line + def __init__(self, other, ensurelines=True): + self.other = other + self.file_name = other.file_name + self.ensurelines = ensurelines + + def write_just(self, arg): + self.other.write_just(arg) + + def write(self, arg): + if (self.ensurelines and not arg.endswith("\n")): + arg += "\n" + self.other.write_just(arg) + + def close(self): + return None + + +class LazierFile(LazyFile): + """Just like LazyFile, except with ensurelines parameter + The parameter controls whether every call to write guarantees a newline + is written. + """ + def __init__(self, directory: str, name: str, ensurelines=True): + super().__init__(directory, name) + self.ensurelines = ensurelines + + # Basically identical to LazyFile write, so take care to sync + def write_just(self, arg: str) -> None: + """ create and write exactly arg to file, no newlines added""" + if arg is None: + return + if self.file_object is None: + filename = os.path.join(self.file_directory, self.file_name) + os.makedirs(self.file_directory, exist_ok=True) + self.file_object = open(filename, 'w+') + self.file_object.write(arg) + + def write(self, arg: str) -> None: + if self.ensurelines: + super().write(arg) + else: + self.write_just(arg) class StreamInclusion(StreamExtract): @@ -75,7 +81,7 @@ class StreamInclusion(StreamExtract): ### Inclusion syntax While extracting content from a file (because it matches one of the -`semiliterate` patterns, rather than just one of the `include_extensions`), +`semiliterate` patterns, rather than just `include`), an unescaped expression of the form `{! FILENAME YAML !}` @@ -128,32 +134,31 @@ is checked for `{! ... !}`. include_bare_file = re.compile(r'\s(?P.*?)\s+(?P[\s\S]*?)\s?\!\}') def __init__(self, input_stream, output_stream, include_root, - ensurelines=True, terminate=None, patterns=None, **kwargs): + terminate=None, patterns=None, extract=None, **kwargs): if terminate and not hasattr(terminate, 'search'): terminate = re.compile(terminate) - # Unfortunately, "simple" has now moved the pattern parsing into - # Semiliterate, so we need to reiterate the code for that here: - if patterns is None: - if 'extract' in kwargs: - extract = kwargs.pop('extract') - if isinstance(extract, dict): - extract = [extract] - patterns = [FlextractionPattern(**p) for p in extract] - else: - patterns = [FlextractionPattern()] + if patterns is None and extract: + # Sadly we must reiterate the pattern parsing here, because of + # inclusions not coming from a [DS]emiliterate object + extractions = [] + if isinstance(extract, dict): + extract = [extract] + for extract_params in extract: + extractions.append(ExtractionPattern(**extract_params)) + if len(extractions) > 0: + patterns = extractions super().__init__(input_stream, output_stream, terminate, patterns, **kwargs) self.include_root = include_root - self.ensure_lines = ensurelines def extract_line(self, line, extraction_pattern): """Copy line to the output stream, applying all specified replacements and handling inclusion syntax. """ - line = extraction_pattern.replace_line(line, self.ensure_lines) + line = extraction_pattern.replace_line(line) include_match = StreamInclusion.include_open.search(line) if not include_match: - self.transcribe(line) + self.output_stream.write(line) return # OK, we have found (the start of) an inclusion and must process it preamble = line[:include_match.start()] @@ -167,8 +172,7 @@ is checked for `{! ... !}`. body_match = body_pattern.search(remainder) if not body_match: for extra_line in self.input_stream: - remainder += extraction_pattern.replace_line(extra_line, - self.ensure_lines) + remainder += extraction_pattern.replace_line(extra_line) body_match = body_pattern.search(remainder) if body_match: break @@ -251,7 +255,7 @@ anticipated by the `{%- block ... %}` directives placed by the theme writer. if gitextract: (write_handle, include_path) = tempfile.mkstemp() utils.log.info( - f"semiliterate: extracting {filename} to {include_path}") + f"semiliterate: git extracting {filename} to {include_path}") contents = subprocess.check_output(['git', 'show', filename]) os.write(write_handle, contents) os.close(write_handle) @@ -274,35 +278,16 @@ anticipated by the `{%- block ... %}` directives placed by the theme writer. if not include_parameters: include_parameters = {} with open(include_path) as include_file: - self.transcribe(preamble) + self.output_stream.write(preamble) inclusion = StreamInclusion( - include_file, self.output_stream, new_root, + include_file, + BorrowFile(self.output_stream, + include_parameters.get('ensurelines', True)), + new_root, **include_parameters) if inclusion.extract(): self.wrote_something = True - self.transcribe(remainder[body_match.end():]) - - # ## The following has to be identical to StreamExtract.try_extract_match - # ## except for the marked bit handling ensure_lines - def try_extract_match( - self, - match_object: re.Match, - emit_last: bool = True) -> bool: - """Extract match into output. - - If _match_object_ is not false-y, returns true. - If extract flag is true, emits the last group of the match if any. - """ - if not match_object: - return False - if match_object.lastindex and emit_last: - # CHANGES HERE - # self.transcribe(get_line(match_object[match_object.lastindex])) - to_emit = match_object[match_object.lastindex] - if self.ensure_lines: - to_emit = get_line(to_emit) - self.transcribe(to_emit) - return True + self.output_stream.write(remainder[body_match.end():]) class SemiliteratePlugin(SimplePlugin): @@ -320,26 +305,22 @@ default values in parentheses at the beginning of each entry. - ['config_options.Type.*?default=([^\)]*)', ': (\1)'] - '^\s*#(.*\s*)$' terminate: '^\s*\)' -!} -{! plugin.py extract: - start: 'r["]{3}Extend' - stop: '["]{3}' !} """ super_config_scheme = SimplePlugin.config_scheme config_scheme = ( # Note documentation of each new parameter **follows** the parameter. *super_config_scheme, - ('exclude_extensions', + ('exclude', config_options.Type(list, default=['.o'])), # Files whose name contains a string in this list will not be # processed by `semiliterate`, regardless of whether they might - # match `include_extensions`, the `semiliterate` patterns, or + # match `include`, the `semiliterate` patterns, or # standard Markdown. ('copy_standard_markdown', config_options.Type(bool, default=False)), # Whether to add MkDocs' list of standard Markdown extensions to - # the `include_extensions` parameter so that Markdown files will be + # the `include` parameter so that Markdown files will be # directly copied to the docsite. Note that the `simple` behavior # corresponds to a _true_ value for `copy_standard_markdown`, but # `semiliterate` still incorporates all standard Markdown files @@ -365,19 +346,10 @@ terminate: '^\s*\)' ) def on_config(self, config, **kwargs): - r""" md -### Adjusting the mkdocs theme - -`semiliterate` also makes it possible to add generated files to the mkdocs -theme. It does this by detecting if a `theme.custom_dir` parameter has been set -in the mkdocs configuration, and if so, it adds the corresponding directory -in the generated docs dir to the theme search path. (Note this means that -files in the corresponding subdirectory of your project will be copied into -the resulting doc site unless their names start with a '.') - """ # Save the include extensions before SimplePlugin modifies them: - self.config['saved_includes'] = self.config['include_extensions'] + saved_includes = self.config['include'] new_config = super().on_config(config, **kwargs) + self.config['saved_includes'] = saved_includes cfpath = os.path.dirname(config.config_file_path) self.custom_dir = None for themedir in config['theme'].dirs: @@ -385,7 +357,7 @@ the resulting doc site unless their names start with a '.') if common == cfpath: self.custom_dir = os.path.relpath(themedir, cfpath) newthemedir = os.path.join( - self.config['build_docs_dir'], self.custom_dir) + self.config['build_dir'], self.custom_dir) utils.log.debug( 'mkdocs-semiliterate: found theme.custom_dir = ' + self.custom_dir @@ -418,18 +390,18 @@ the resulting doc site unless their names start with a '.') class Semisimple(Simple): """Mkdocs Semisimple Plugin""" - def __init__(self, semiliterate, exclude_extensions, saved_includes, + def __init__(self, semiliterate, exclude, saved_includes, copy_standard_markdown, extract_standard_markdown, extract_on_copy, **kwargs): # Since we have extensions in Demiliterate, suppress the semiliterate # configuration until we handle it ourselves: super().__init__(semiliterate=[], **kwargs) self.semiliterate = [Demiliterate(**item) for item in semiliterate] - self.exclude_extensions = exclude_extensions + self.exclude = exclude self.extract_on_copy = extract_on_copy dflt_enable = False if not copy_standard_markdown: - self.copy_glob = set(saved_includes) + self.doc_glob = set(saved_includes) dflt_enable = True if extract_standard_markdown.get('enable', dflt_enable): ext_pat = '|'.join(re.escape(s) for s in utils.markdown_extensions) @@ -439,44 +411,71 @@ class Semisimple(Simple): destination=r'\1', **extract_standard_markdown)) - def should_copy_file(self, file): - if any(ext in file for ext in self.exclude_extensions): + def is_doc_file(self, name: str) -> bool: + if any(ext in name for ext in self.exclude): return False - return super().should_copy_file(file) + return super().is_doc_file(name) - def try_extract(self, from_directory, name, to_directory): - if any(ext in name for ext in self.exclude_extensions): - return False - if not self.extract_on_copy and self.should_copy_file(name): - return False - return super().try_extract(from_directory, name, to_directory) + def try_extract(self, from_dir: str, name: str, to_dir: str) -> list: + if any(ext in name for ext in self.exclude): + return [] + if not self.extract_on_copy and self.is_doc_file(name): + return [] + return super().try_extract(from_dir, name, to_dir) # Had to override this because the simple version hardcoded that if a file # was copied, it could not be extracted. So check carefully for changes in - # simple. Only the lines between # # START and # # END differ. - def build_docs(self) -> list: + # simple. Only the line marked # REMOVED was commented out + def build_docs( + self, + dirty=False, + last_build_time=None, + do_copy=False) -> list: """Build the docs directory from workspace files.""" paths = [] files = self.get_files() for file in files: if not os.path.isfile(file): continue + if dirty and last_build_time and ( + os.path.getmtime(file) <= last_build_time): + continue from_dir = os.path.dirname(file) name = os.path.basename(file) build_prefix = os.path.normpath( os.path.join(self.build_dir, from_dir)) - # # START - copied = self.try_copy_file(from_dir, name, build_prefix) - extracted = self.try_extract(from_dir, name, build_prefix) - if (copied or extracted): - paths.append(file) - # # END + doc_paths = self.get_doc_file( + from_dir, name, build_prefix, True) + if doc_paths: + paths.append( + SimplePath( + output_root=".", + output_relpath=os.path.relpath(path=file, start="."), + input_path=file) + ) + utils.log.info("mkdocs-semiliterate: Added %s", file) + # continue # REMOVED + + extracted_paths = self.try_extract(from_dir, name, build_prefix) + for path in extracted_paths: + paths.append( + SimplePath( + output_root=self.build_dir, + output_relpath=os.path.relpath( + path=path, + start=self.build_dir), + input_path=file)) + utils.log.info( + "mkdocs-semiliterate: Extracted %s -> %s", file, path) + if extracted_paths: + continue + return paths class Demiliterate(Semiliterate): - r"""Extends Semiliterate to use StreamInclusion, not StreamExtract + r""" md Extends Semiliterate to use StreamInclusion, not StreamExtract semiliterate.ensurelines : (true) Guarantees that a newline is trancribed for each line of the input, @@ -489,18 +488,13 @@ semiliterate.ensurelines def __init__( self, - pattern, - destination=None, - terminate=None, - ensurelines=True, - extract=[]): - super().__init__(pattern, destination, terminate) - self.ensure_lines = ensurelines - if isinstance(extract, dict): - extract = [extract] - self.patterns = [FlextractionPattern(**p) for p in extract] - if len(self.patterns) == 0: - self.patterns = [FlextractionPattern()] + pattern: str, + destination: str = None, + terminate: str = None, + extract: list = None, + ensurelines=True): + super().__init__(pattern, destination, terminate, extract) + self.ensurelines = ensurelines # Note that this has diverged noticeably from the # Semiliterate.try_extraction method that it overrides, especially @@ -513,37 +507,53 @@ semiliterate.ensurelines from_directory, from_file, destination_directory, - **kwargs): + **kwargs) -> list: """Try to extract documentation from file with name. - Returns True if extraction was successful. + Args: + from_directory (str): The source directory + from_file (str): The source filename within directory + destination_directory (str): The destination directory + + Returns a list of extracted files. """ to_file = self.filename_match(from_file) if not to_file: - return False + return [] from_file_path = os.path.join(from_directory, from_file) to_file_path = os.path.join(destination_directory, to_file) # ## ADDED (destination_directory, to_file) = os.path.split(to_file_path) # ADDED try: with open(from_file_path) as original_file: utils.log.debug( - f"mkdocs-semiliterate: In {from_directory}, " - + f"scanning {from_file}...") + f"mkdocs-semiliterate: Scanning {from_file_path}... ") # extraction = StreamExtract( extraction = StreamInclusion( input_stream=original_file, - output_stream=LazyFile(destination_directory, to_file), + output_stream=LazierFile( + destination_directory, to_file, self.ensurelines), include_root=from_directory, # ## ADDED - ensurelines=self.ensure_lines, # ## ADDED terminate=self.terminate, - patterns=self.patterns, + patterns=self.extractions, **kwargs) return extraction.extract() except (UnicodeDecodeError) as error: - utils.log.info("mkdocs-semiliterate: skipping %s\n %s", - from_file_path, str(error)) - except BaseException as error: - utils.log.error( - f"mkdocs-semiliterate: could not build {from_file_path}:\n " - + str(error)) - return False + utils.log.debug("mkdocs-semiliterate: Skipped %s", from_file_path) + utils.log.debug( + "mkdocs-semiliterate: Error details: %s", str(error)) + except (OSError, IOError) as error: + utils.log.error("mkdocs-semiliterate: could not build %s\n %s", + from_file_path, str(error)) + return [] + +# start-md +# ### Adjusting the mkdocs theme + +# `semiliterate` also makes it possible to add generated files to the mkdocs +# theme. It does this by detecting if a `theme.custom_dir` parameter has been +# set in the mkdocs configuration, and if so, it adds the corresponding +# directory in the generated docs dir to the theme search path. (Note this +# means that +# files in the corresponding subdirectory of your project will be copied into +# the resulting doc site unless their names start with a '.') +# end-md diff --git a/setup.cfg b/setup.cfg index e8d3d4c..07eb2b0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = mkdocs-semiliterate -version = 0.7.1 +version = 0.8.0 description = Extension of mkdocs-simple-plugin adding easy content inclusion long_description = file: README.md long_description_content_type = text/markdown @@ -25,7 +25,7 @@ license = Apache-2.0 packages = mkdocs_semiliterate install_requires = mkdocs~=1.4 - mkdocs-simple-plugin==2.1.2 + mkdocs-simple-plugin==3.2.0 [options.entry_points] mkdocs.plugins = diff --git a/tests/fixtures/extract-inclusion/mkdocs.yml b/tests/fixtures/extract-inclusion/mkdocs.yml index a193299..a5b3b13 100644 --- a/tests/fixtures/extract-inclusion/mkdocs.yml +++ b/tests/fixtures/extract-inclusion/mkdocs.yml @@ -2,9 +2,9 @@ site_name: Enable extraction from included files docs_dir: refsite # dummy plugins: - semiliterate: - ignore_folders: [refsite] + ignore: [refsite] merge_docs_dir: false - include_extensions: ['.txt'] + include: ['.txt'] extract_on_copy: true semiliterate: - pattern: '\.txt' diff --git a/tests/fixtures/full-inclusion/mkdocs.yml b/tests/fixtures/full-inclusion/mkdocs.yml index bd3e416..14f8365 100644 --- a/tests/fixtures/full-inclusion/mkdocs.yml +++ b/tests/fixtures/full-inclusion/mkdocs.yml @@ -2,5 +2,5 @@ site_name: Full inclusion docs_dir: refsite # dummy plugins: - semiliterate: - ignore_folders: [refsite, snippet] + ignore: [refsite, snippet] merge_docs_dir: false diff --git a/tests/fixtures/git-inclusion/mkdocs.yml b/tests/fixtures/git-inclusion/mkdocs.yml index bd3e416..14f8365 100644 --- a/tests/fixtures/git-inclusion/mkdocs.yml +++ b/tests/fixtures/git-inclusion/mkdocs.yml @@ -2,5 +2,5 @@ site_name: Full inclusion docs_dir: refsite # dummy plugins: - semiliterate: - ignore_folders: [refsite, snippet] + ignore: [refsite, snippet] merge_docs_dir: false diff --git a/tests/fixtures/no-extract-inclusion/mkdocs.yml b/tests/fixtures/no-extract-inclusion/mkdocs.yml index 45d9e5e..cf3e5e0 100644 --- a/tests/fixtures/no-extract-inclusion/mkdocs.yml +++ b/tests/fixtures/no-extract-inclusion/mkdocs.yml @@ -2,5 +2,5 @@ site_name: Do not extract from included files docs_dir: refsite # dummy plugins: - semiliterate: - ignore_folders: [refsite] + ignore: [refsite] merge_docs_dir: false diff --git a/tests/fixtures/quoted-filename/mkdocs.yml b/tests/fixtures/quoted-filename/mkdocs.yml index bd3e416..14f8365 100644 --- a/tests/fixtures/quoted-filename/mkdocs.yml +++ b/tests/fixtures/quoted-filename/mkdocs.yml @@ -2,5 +2,5 @@ site_name: Full inclusion docs_dir: refsite # dummy plugins: - semiliterate: - ignore_folders: [refsite, snippet] + ignore: [refsite, snippet] merge_docs_dir: false diff --git a/tests/fixtures/recursive-inclusion/mkdocs.yml b/tests/fixtures/recursive-inclusion/mkdocs.yml index a334947..81a6fb1 100644 --- a/tests/fixtures/recursive-inclusion/mkdocs.yml +++ b/tests/fixtures/recursive-inclusion/mkdocs.yml @@ -2,7 +2,7 @@ site_name: Custom semiliterate docs_dir: refsite # dummy plugins: - semiliterate: - ignore_folders: [refsite] + ignore: [refsite] merge_docs_dir: false copy_standard_markdown: true semiliterate: diff --git a/tests/fixtures/sibling-destination/mkdocs.yml b/tests/fixtures/sibling-destination/mkdocs.yml index e32d41e..a73d305 100644 --- a/tests/fixtures/sibling-destination/mkdocs.yml +++ b/tests/fixtures/sibling-destination/mkdocs.yml @@ -2,7 +2,7 @@ site_name: Enable extraction from included files docs_dir: refsite # dummy plugins: - semiliterate: - ignore_folders: [refsite] + ignore: [refsite] merge_docs_dir: false semiliterate: - pattern: 'raw\.txt' diff --git a/tests/fixtures/theme-modification/mkdocs.yml b/tests/fixtures/theme-modification/mkdocs.yml index cd8949d..e3c0461 100644 --- a/tests/fixtures/theme-modification/mkdocs.yml +++ b/tests/fixtures/theme-modification/mkdocs.yml @@ -5,10 +5,10 @@ theme: custom_dir: doc_theme/ plugins: - semiliterate: - ignore_folders: [refsite] - ignore_hidden: false + ignore: [refsite] merge_docs_dir: false - include_extensions: [] + include: [] semiliterate: - pattern: '[.](base).generator$' # Amend readthedocs theme destination: '\1.html' + ensurelines: false \ No newline at end of file