From 4392e297421a75c3f3d5f7add3ce84a05a4051d7 Mon Sep 17 00:00:00 2001 From: MinchinWeb Date: Sat, 29 May 2021 18:21:45 -0600 Subject: [PATCH] Use implicit namespace plugins for import and export (#1216) * behavior outline * FIrst pass at allow external plugins * remove template exporter * Add listing of active plugins to '--version' output * Documentation for plugins * [Docs] add custom imports and exporters to site TOC * [Docs] better linewrapping * enforce positive initial linewrap Check column widths update gitignore throw error when linewrap too small simply check for large enough linewrap value * delete unused error message * PR feedback make exception more informative update check_linewrap signature in src and test make check_linewrap a free function * delete unused function * delete else..pass block * newline for make format * Include dates_exporter * Use Base classes for importer and exporters. * [Docs] improve documentation of custom Importers and Exporters * [Testing] separate run with external plugin! * basic behavior test * prototype unittest for JSON Exporter test for unimplemented method * make format delete unused imports * Remove 'importer' or 'exporter' from filenames where not needed * [Test] run different tests with or without the external plugins installed * [Test] move test rot13 plugin into git tree from https://github.com/MinchinWeb/jrnl-rot13-exporter/commit/0dc912af8265ef4ef4743fdc439d89025259d8b2 * consolidate demo plugins to common package * [Docs] name page for plugins * [Docs] include the sample plug in code files directly * style fixes * [test] determine whether to run external plug in tests based on installed packages * improved code documentation * style fixes for GitHub actions * Convert "short" and "pretty" (and "default") formaters to plugins further to https://github.com/jrnl-org/jrnl/pull/1177 * more code clean up tests pass locally...now for GitHub... * [tests] dynamically determine jrnl version for plugin tests * [GitHub Actions] direct install of testing plugins * Remove template code * [plugins] meta --> collector * [Docs] create scripted entries using an custom importer * (closer to) being able to run behave tests outside project root directory * We already know when exporter to use Don't re-calculate it! * [Tests] don't name test plugin 'testing" If so named, pip won't install it. * [Test] run behave tests with test plugins outside project root * [Test] behave tests pass locally * [Docs] fix typo * [GitHub Actions] run test commands from poetry's shell * black-ify code * [GitHub Actions] move downstream (rather than up) to run tests * [GitHub Actions] set shell to poetry * [GitHub Workflows] Manually activate virtual environment * [GitHub Actions] Skip Windows & Python 3.8 Can't seem to find Python exe? * [GiotHub Actions] explicitly use virtual env * [GitHub Actions] create virutal env directly * [GitHub Actions] better activate of Windows virtual env * [GitHub Actions] create virtual env on Mac * [Github Actions] install wheel and upgrade pip * [GitHub Actions] skip virtual environments altogether * [GitHub Actions] change directory for behave test * Remove Windows exclusions from CI as per note -- they should be working now Co-authored-by: Suhas Co-authored-by: Micah Jerome Ellison --- .../workflows/testing_external_plugins.yaml | 57 ++++++ docs/formats.md | 5 +- docs/plugins.md | 181 ++++++++++++++++++ docs_theme/requirements.txt | 3 +- features/environment.py | 47 ++++- features/format.feature | 4 + features/plugins.feature | 86 +++++++++ features/steps/core.py | 56 ++---- features/steps/export_steps.py | 8 + features/steps/override.py | 8 +- features/write.feature | 2 + jrnl/args.py | 4 +- jrnl/behave_testing.py | 47 +++++ jrnl/commands.py | 24 ++- jrnl/contrib/exporter/.gitkeep | 0 jrnl/contrib/importer/.gitkeep | 0 jrnl/jrnl.py | 14 +- jrnl/plugins/__init__.py | 46 +---- jrnl/plugins/{text_exporter.py => base.py} | 71 +++++-- jrnl/plugins/collector.py | 106 ++++++++++ .../{dates_exporter.py => exporter/dates.py} | 8 +- .../{fancy_exporter.py => exporter/fancy.py} | 21 +- .../{json_exporter.py => exporter/json.py} | 9 +- .../markdown.py} | 6 +- jrnl/plugins/exporter/pretty.py | 20 ++ jrnl/plugins/exporter/short.py | 20 ++ .../{tag_exporter.py => exporter/tag.py} | 10 +- jrnl/plugins/exporter/text.py | 21 ++ .../{xml_exporter.py => exporter/xml.py} | 39 ++-- .../{yaml_exporter.py => exporter/yaml.py} | 13 +- .../{jrnl_importer.py => importer/jrnl.py} | 11 +- jrnl/plugins/template.py | 142 -------------- jrnl/plugins/template_exporter.py | 43 ----- jrnl/plugins/util.py | 15 +- jrnl/templates/sample.template | 18 -- mkdocs.yml | 4 + poetry.lock | 16 ++ tests/external_plugins_src/README.md | 9 + .../jrnl/contrib/exporter/custom_json.py | 37 ++++ .../jrnl/contrib/exporter/flag.py | 25 +++ .../jrnl/contrib/exporter/rot13.py | 15 ++ .../jrnl/contrib/importer/simple_json.py | 50 +++++ tests/external_plugins_src/setup.py | 35 ++++ tests/test_plugin.py | 46 +++++ tests/unit/test_display.py | 2 +- 45 files changed, 1021 insertions(+), 383 deletions(-) create mode 100644 .github/workflows/testing_external_plugins.yaml create mode 100644 docs/plugins.md create mode 100644 features/plugins.feature create mode 100644 jrnl/behave_testing.py create mode 100644 jrnl/contrib/exporter/.gitkeep create mode 100644 jrnl/contrib/importer/.gitkeep rename jrnl/plugins/{text_exporter.py => base.py} (55%) create mode 100644 jrnl/plugins/collector.py rename jrnl/plugins/{dates_exporter.py => exporter/dates.py} (86%) rename jrnl/plugins/{fancy_exporter.py => exporter/fancy.py} (85%) rename jrnl/plugins/{json_exporter.py => exporter/json.py} (92%) rename jrnl/plugins/{markdown_exporter.py => exporter/markdown.py} (95%) create mode 100644 jrnl/plugins/exporter/pretty.py create mode 100644 jrnl/plugins/exporter/short.py rename jrnl/plugins/{tag_exporter.py => exporter/tag.py} (85%) create mode 100644 jrnl/plugins/exporter/text.py rename jrnl/plugins/{xml_exporter.py => exporter/xml.py} (92%) rename jrnl/plugins/{yaml_exporter.py => exporter/yaml.py} (93%) rename jrnl/plugins/{jrnl_importer.py => importer/jrnl.py} (78%) delete mode 100644 jrnl/plugins/template.py delete mode 100644 jrnl/plugins/template_exporter.py delete mode 100644 jrnl/templates/sample.template create mode 100644 tests/external_plugins_src/README.md create mode 100644 tests/external_plugins_src/jrnl/contrib/exporter/custom_json.py create mode 100644 tests/external_plugins_src/jrnl/contrib/exporter/flag.py create mode 100644 tests/external_plugins_src/jrnl/contrib/exporter/rot13.py create mode 100644 tests/external_plugins_src/jrnl/contrib/importer/simple_json.py create mode 100644 tests/external_plugins_src/setup.py create mode 100644 tests/test_plugin.py diff --git a/.github/workflows/testing_external_plugins.yaml b/.github/workflows/testing_external_plugins.yaml new file mode 100644 index 00000000..51da56b6 --- /dev/null +++ b/.github/workflows/testing_external_plugins.yaml @@ -0,0 +1,57 @@ +name: Testing + +on: + push: + branches: [ develop, release ] + paths: + - 'jrnl/**' + - 'features/**' + - 'tests/**' + - 'poetry.lock' + - 'pyproject.toml' + pull_request: + branches: [ develop ] + paths: + - 'jrnl/**' + - 'features/**' + - 'tests/**' + - 'poetry.lock' + - 'pyproject.toml' + +jobs: + test-namespace-plugins: + if: > + ! contains(github.event.head_commit.message, '[ci skip]') + runs-on: ${{ matrix.os }} + strategy: + matrix: + python-version: [ 3.7, 3.8, 3.9 ] + os: [ ubuntu-latest, macos-latest, windows-latest ] + exclude: # Added for GitHub Actions PR problem 2020-12-19 -- remove later! + - os: windows-latest + python-version: 3.9 + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install pip setuptools wheel --upgrade + python -m pip install . + python -m pip install ./tests/external_plugins_src/ + python -m pip install pytest behave + # installed test plugins aren't recognized by "behave" if run from the + # project's root folder + + - name: Test with pytest + if: success() || failure() + run: pytest --junitxml=reports/pytest/results.xml + + - name: Test with behave + if: success() || failure() + run: cd features && behave --no-skipped --format progress2 --junit --junit-directory ../reports/behave diff --git a/docs/formats.md b/docs/formats.md index 99b3e69b..90030045 100644 --- a/docs/formats.md +++ b/docs/formats.md @@ -13,7 +13,10 @@ used alone (e.g. `jrnl --format json`) to display all entries from the selected This page shows examples of all the built-in formats, but since `jrnl` supports adding more formats through plugins, you may have more available on your system. Please see -`jrnl --help` for a list of which formats are available on your system. +`jrnl --version` for a list of which formats are available on your system. Note +that plugins can also override built-in formats, so review your installed +plugins if your output does not match what is listed here. You can also [write +your own plugins](./plugins.md) to create custom formats. Any of these formats can be used interchangeably, and are only grouped into "display", "data", and "report" formats below for convenience. diff --git a/docs/plugins.md b/docs/plugins.md new file mode 100644 index 00000000..a3ce0aa6 --- /dev/null +++ b/docs/plugins.md @@ -0,0 +1,181 @@ + + +# Extending jrnl + +*jrnl* can be extended with custom importers and exporters. + +Note that custom importers and exporters can be given the same name as a +built-in importer or exporter to override it. + +Custom Importers and Exporters are traditional Python packages, and are +installed (into *jrnl*) simply by installing them so they are available to the +Python interpreter that is running *jrnl*. + +Exporter are also used as "formatters" when entries are written to the command +line. + +## Rational + +I added this feature because *jrnl* was overall working well for me, but I +found myself maintaining a private fork so I could have a slightly customized +export format. Implementing (import and) export plugins was seen as a way to +maintain my custom exporter without the need to maintaining my private fork. + +This implementation tries to keep plugins as light as possible, and as free of +boilerplate code as reasonable. As well, internal importers and exporters are +implemented in almost exactly the same way as custom importers and exporters, +and so it is hoped that plugins can be moved from "contributed" to "internal" +easily, or that internal plugins can serve as a base and/or a demonstration for +external plugins. + +-- @MinchinWeb, May 2021 + +## Entry Class + +Both the Importers and the Exporters work on the `Entry` class. Below is a +(selective) description of the class, it's properties and functions: + +- **Entry** (class) at `jrnl.Entry.Entry`. + - **title** (string): a single line that represents a entry's title. + - **date** (datetime.datetime): the date and time assigned to an entry. + - **body** (string): the main body of the entry. Can be basically any + length. *jrnl* assumes no particular structure here. + - **starred** (boolean): is an entry starred? Presumably, starred entries + are of particular importance. + - **tags** (list of strings): the tags attached to an entry. Each tag + includes the pre-facing "tag symbol". + - **\_\_init\_\_(journal, date=None, text="", starred=False)**: contractor + method + - **journal** (*jrnl.Journal.Journal*): a link to an existing Journal + class. Mainly used to access it's configuration. + - **date** (datetime.datetime) + - **text** (string): assumed to include both the title and the body. + When the title, body, or tags of an entry are requested, this text + will the parsed to determine the tree. + - **starred** (boolean) + +Entries also have "advanced" metadata if they are using the DayOne backend, but +we'll ignore that for the purposes of this demo. + +## Custom Importer + +If you have a (custom) datasource that you want to import into your jrnl +(perhaps like a blog export), you can write a custom importer to do this. + +An importer takes the source data, turns it into Entries and then appends those +entries to a Journal. Here is a basic Importer, assumed to be provided with a +nicely formatted JSON file: + +~~~ python +{% + include-markdown "../tests/external_plugins_src/jrnl/contrib/importer/simple_json.py" + comments=false +%} +~~~ + +Note that the above is very minimal, doesn't do any error checking, and doesn't +try to import all possible entry metadata. + +Another potential use of a custom importer is to effectively create a scripted +entry creator. For example, maybe each day you want to create a journal entry +that contains the answers to specific questions; you could create a custom +"importer" that would ask you the questions, and then create an entry containing +the answers provided. + +Some implementation notes: + +- The importer class must be named **Importer**, and should sub-class + **jrnl.plugins.base.BaseImporter**. +- The importer module must be within the **jrnl.contrib.importer** namespace. +- The importer must not have any `__init__.py` files in the base directories + (but you can have one for your importer base directory if it is in a + directory rather than a single file). +- The importer must be installed as a Python package available to the same + Python interpreter running jrnl. +- The importer must expose at least the following the following members: + - **version** (string): the version of the plugin. Displayed to help the + user debug their installations. + - **names** (list of strings): these are the "names" that can be passed to + the CLI to involve your importer. If you specify one used by a built-in + plugin, it will overwrite it (effectively making the built-in one + unavailable). + - **import_(journal, input=None)**: the actual importer. Must append + entries to the journal passed to it. It is recommended to accept either a + filename or standard input as a source. + +## Custom Exporter + +Custom exporters are useful to make *jrnl*'s data available to other programs. +One common usecase would to generate the input to be used by a static site +generator or blogging engine. + +An exporter take either a whole journal or a specific entry and exports it. +Below is a basic JSON Exporter; note that a more extensive JSON exporter is +included in *jrnl* and so this (if installed) would override the built in +exporter. + +~~~ python +{% + include-markdown "../tests/external_plugins_src/jrnl/contrib/exporter/custom_json.py" + comments=false +%} +~~~ + +Note that the above is very minimal, doesn't do any error checking, and doesn't +export all entry metadata. + +Some implementation notes: + +- the exporter class must be named **Exporter** and should sub-class + **jrnl.plugins.base.BaseExporter**. +- the exporter module must be within the **jrnl.contrib.exporter** namespace. +- The exporter must not have any `__init__.py` files in the base directories + (but you can have one for your exporter base directory if it is in a + directory rather than a single file). +- The exporter must be installed as a Python package available to the same + Python interpreter running jrnl. +- the exporter should expose at least the following the following members + (there are a few more you will need to define if you don't subclass + `jrnl.plugins.base.BaseExporter`): + - **version** (string): the version of the plugin. Displayed to help the + user debug their installations. + - **names** (list of strings): these are the "names" that can be passed to + the CLI to invole your exporter. If you specific one used by a built-in + plugin, it will overwrite it (effectively making the built-in one + unavailable). + - **extension** (string): the file extention used on exported entries. + - **export_entry(entry)**: given an entry, returns a string of the formatted, + exported entry. + - **export_journal(journal)**: (optional) given a journal, returns a string + of the formatted, exported entries of the journal. If not implemented, + *jrnl* will call **export_entry()** on each entry in turn and then + concatenate the results together. + +### Special Exporters + +There are a few "special" exporters, in that they are called by *jrnl* in +situations other than a traditional export. They are: + +- **short** -- called by `jrnl --short`. Displays each entry on a single line. + The default is to print the timestamp of the entry, followed by the title. + The built-in (default) plugin is at `jrnl.plugins.exporter.short`. +- **default** -- called when a different format is not specified. The built-in + (default) plugin is at `jrnl.plugins.exporter.pretty`. + +## Development Tips + +- Editable installs (`pip install -e ...`) don't seem to play nice with + the namespace layout. If your plugin isn't appearing, try a non-editable + install of both *jrnl* and your plugin. +- If you run *jrnl* from the main project root directory (the one that contains + *jrnl*'s source code), namespace plugins won't be recognized. This is (I + suspect) because the Python interpreter will find your *jrnl* source directory + (which doesn't contain your namespace plugins) before it find your + "site-packages" directory (i.e. installed packages, which will recognize + namespace packages). +- Don't name your plugin file "testing.py" or it won't be installed (at least + automatically) by pip. +- For examples, you can look to the *jrnl*'s internal importers and exporters. + As well, there are some basic external examples included in *jrnl*'s git repo + at `tests/external_plugins_src` (including the example code above). diff --git a/docs_theme/requirements.txt b/docs_theme/requirements.txt index 33dff294..56d32c9e 100644 --- a/docs_theme/requirements.txt +++ b/docs_theme/requirements.txt @@ -1 +1,2 @@ -mkdocs==1.1 +mkdocs==1.1.2 +mkdocs-include-markdown-plugin==2.8.0 diff --git a/features/environment.py b/features/environment.py index f4baab34..766f9576 100644 --- a/features/environment.py +++ b/features/environment.py @@ -1,9 +1,17 @@ import os +from pathlib import Path import shutil from jrnl.os_compat import on_windows +try: + from jrnl.contrib.exporter import flag as testing_exporter +except ImportError: + testing_exporter = None + CWD = os.getcwd() +HERE = Path(__file__).resolve().parent +TARGET_CWD = HERE.parent # project root folder # @see https://behave.readthedocs.io/en/latest/tutorial.html#debug-on-error-in-case-of-step-failures BEHAVE_DEBUG_ON_ERROR = False @@ -15,6 +23,8 @@ def setup_debug_on_error(userdata): def before_all(context): + # always start in project root directory + os.chdir(TARGET_CWD) setup_debug_on_error(context.config.userdata) @@ -27,10 +37,10 @@ def before_all(context): def clean_all_working_dirs(): - if os.path.exists("test.txt"): - os.remove("test.txt") + if os.path.exists(HERE / "test.txt"): + os.remove(HERE / "test.txt") for folder in ("configs", "journals", "cache"): - working_dir = os.path.join("features", folder) + working_dir = HERE / folder if os.path.exists(working_dir): shutil.rmtree(working_dir) @@ -46,20 +56,28 @@ def before_feature(context, feature): feature.skip("Skipping on Windows") return + if "skip_only_with_external_plugins" in feature.tags and testing_exporter is None: + feature.skip("Requires test external plugins installed") + return + + if "skip_no_external_plugins" in feature.tags and testing_exporter: + feature.skip("Skipping with external plugins installed") + return + def before_scenario(context, scenario): """Before each scenario, backup all config and journal test data.""" # Clean up in case something went wrong clean_all_working_dirs() for folder in ("configs", "journals"): - original = os.path.join("features", "data", folder) - working_dir = os.path.join("features", folder) + original = HERE / "data" / folder + working_dir = HERE / folder if not os.path.exists(working_dir): os.mkdir(working_dir) for filename in os.listdir(original): - source = os.path.join(original, filename) + source = original / filename if os.path.isdir(source): - shutil.copytree(source, os.path.join(working_dir, filename)) + shutil.copytree(source, (working_dir / filename)) else: shutil.copy2(source, working_dir) @@ -73,11 +91,22 @@ def before_scenario(context, scenario): scenario.skip("Skipping on Windows") return + if ( + "skip_only_with_external_plugins" in scenario.effective_tags + and testing_exporter is None + ): + scenario.skip("Requires test external plugins installed") + return + + if "skip_no_external_plugins" in scenario.effective_tags and testing_exporter: + scenario.skip("Skipping with external plugins installed") + return + def after_scenario(context, scenario): """After each scenario, restore all test data and remove working_dirs.""" - if os.getcwd() != CWD: - os.chdir(CWD) + if os.getcwd() != TARGET_CWD: + os.chdir(TARGET_CWD) # only clean up if debugging is off and the scenario passed if BEHAVE_DEBUG_ON_ERROR and scenario.status != "failed": diff --git a/features/format.feature b/features/format.feature index f935e8c8..e369d7b3 100644 --- a/features/format.feature +++ b/features/format.feature @@ -26,6 +26,7 @@ Feature: Custom formats | basic_folder | | basic_dayone | + @skip_no_external_plugins Scenario Outline: JSON format Given we use the config ".yaml" And we use the password "test" if prompted @@ -48,6 +49,7 @@ Feature: Custom formats | basic_folder | | basic_dayone | + @skip_no_external_plugins Scenario: Exporting dayone to json Given we use the config "dayone.yaml" When we run "jrnl --export json" @@ -91,6 +93,7 @@ Feature: Custom formats | basic_folder | | basic_dayone | + @skip_no_external_plugins Scenario Outline: Exporting using filters should only export parts of the journal Given we use the config ".yaml" And we use the password "test" if prompted @@ -112,6 +115,7 @@ Feature: Custom formats | basic_folder | | basic_dayone | + @skip # template exporters have been removed Scenario Outline: Exporting using custom templates Given we use the config ".yaml" And we load template "sample.template" diff --git a/features/plugins.feature b/features/plugins.feature new file mode 100644 index 00000000..8b86328b --- /dev/null +++ b/features/plugins.feature @@ -0,0 +1,86 @@ +Feature: Functionality of Importer and Exporter Plugins + + @skip_no_external_plugins + Scenario Outline: List buildin plugin names in --version + Given We use the config "basic_onefile.yaml" + When We run "jrnl --version" + Then the output should contain pyproject.toml version + And The output should contain " : from jrnl..." + And the output should not contain ".contrib." + + Examples: + | plugin_name | version | source | type | filename | + | jrnl | | plugins | importer | jrnl | + | boxed | | plugins | exporter | fancy | + | dates | | plugins | exporter | dates | + | default | | plugins | exporter | pretty | + | fancy | | plugins | exporter | fancy | + | json | | plugins | exporter | json | + | markdown | | plugins | exporter | markdown | + | md | | plugins | exporter | markdown | + | pretty | | plugins | exporter | pretty | + | short | | plugins | exporter | short | + | tags | | plugins | exporter | tag | + | text | | plugins | exporter | text | + | txt | | plugins | exporter | text | + | xml | | plugins | exporter | xml | + | yaml | | plugins | exporter | yaml | + + @skip_only_with_external_plugins + Scenario Outline: List external plugin names in --version + Given We use the config "basic_onefile.yaml" + When We run "jrnl --version" + Then the output should contain pyproject.toml version + And The output should contain " : from jrnl..." + Examples: + | plugin_name | version | source | type | filename | + | jrnl | | plugins | importer | jrnl | + | json | v1.0.0 | contrib | importer | simple_json | + | boxed | | plugins | exporter | fancy | + | dates | | plugins | exporter | dates | + | default | | plugins | exporter | pretty | + | fancy | | plugins | exporter | fancy | + | json | v1.0.0 | contrib | exporter | custom_json | + | markdown | | plugins | exporter | markdown | + | md | | plugins | exporter | markdown | + | pretty | | plugins | exporter | pretty | + | rot13 | v1.0.0 | contrib | exporter | rot13 | + | short | | plugins | exporter | short | + | tags | | plugins | exporter | tag | + | testing | v0.0.1 | contrib | exporter | flag | + | text | | plugins | exporter | text | + | txt | v1.0.0 | contrib | exporter | rot13 | + | xml | | plugins | exporter | xml | + | yaml | | plugins | exporter | yaml | + + @skip_only_with_external_plugins + Scenario Outline: Do not list overridden plugin names in --version + Given We use the config "basic_onefile.yaml" + When We run "jrnl --version" + Then the output should contain pyproject.toml version + And the output should not contain " : from jrnl..." + + Examples: + | plugin_name | version | source | type | filename | + | json | | plugins | exporter | json | + | txt | | plugins | exporter | text | + + + @skip_only_with_external_plugins + Scenario Outline: JSON format + Given we use the config ".yaml" + And we use the password "test" if prompted + When we run "jrnl --format json" + Then we should get no error + And the output should be parsable as json + And "entries" in the json output should have 3 elements + And entry 1 should not have an array "tags" + And entry 2 should not have an array "tags" + And entry 3 should not have an array "tags" + + Examples: configs + | config | + | basic_onefile | + | basic_encrypted | + | basic_folder | + | basic_dayone | diff --git a/features/steps/core.py b/features/steps/core.py index 2ad5bcc4..48140164 100644 --- a/features/steps/core.py +++ b/features/steps/core.py @@ -3,7 +3,6 @@ import ast from collections import defaultdict -from jrnl.args import parse_args import os from pathlib import Path import re @@ -14,20 +13,23 @@ from behave import given from behave import then from behave import when import keyring - import toml import yaml from yaml.loader import SafeLoader - -import jrnl.time from jrnl import Journal from jrnl import __version__ from jrnl import plugins +from jrnl.args import parse_args +from jrnl.behave_testing import _mock_getpass +from jrnl.behave_testing import _mock_input +from jrnl.behave_testing import _mock_time_parse from jrnl.cli import cli from jrnl.config import load_config from jrnl.os_compat import split_args -from jrnl.override import apply_overrides, _recursively_apply +from jrnl.override import _recursively_apply +from jrnl.override import apply_overrides +import jrnl.time try: import parsedatetime.parsedatetime_consts as pdt @@ -279,42 +281,6 @@ def extension_editor_file(context, suffix): assert filename_suffix == suffix -def _mock_getpass(inputs): - def prompt_return(prompt=""): - if type(inputs) == str: - return inputs - try: - return next(inputs) - except StopIteration: - raise KeyboardInterrupt - - return prompt_return - - -def _mock_input(inputs): - def prompt_return(prompt=""): - try: - val = next(inputs) - print(prompt, val) - return val - except StopIteration: - raise KeyboardInterrupt - - return prompt_return - - -def _mock_time_parse(context): - original_parse = jrnl.time.parse - if "now" not in context: - return original_parse - - def wrapper(input, *args, **kwargs): - input = context.now if input == "now" else input - return original_parse(input, *args, **kwargs) - - return wrapper - - @when('we run "{command}" and enter') @when('we run "{command}" and enter nothing') @when('we run "{command}" and enter "{inputs}"') @@ -461,8 +427,9 @@ def run(context, command, text=""): @given('we load template "{filename}"') def load_template(context, filename): full_path = os.path.join("features/data/templates", filename) + exporter = plugins.template_exporter.__exporter_from_file(full_path) - plugins.__exporter_types[exporter.names[0]] = exporter + plugins.collector.__exporter_types[exporter.names[0]] = exporter @when('we set the keyring password of "{journal}" to "{password}"') @@ -540,6 +507,11 @@ def check_output_version_inline(context): @then('the output should contain "{text}" or "{text2}"') def check_output_inline(context, text=None, text2=None): text = text or context.text + if "" in text: + pyproject = (Path(__file__) / ".." / ".." / ".." / "pyproject.toml").resolve() + pyproject_contents = toml.load(pyproject) + pyproject_version = pyproject_contents["tool"]["poetry"]["version"] + text = text.replace("", pyproject_version) out = context.stdout_capture.getvalue() assert (text and text in out) or (text2 and text2 in out) diff --git a/features/steps/export_steps.py b/features/steps/export_steps.py index 8141dc36..d52e8b4d 100644 --- a/features/steps/export_steps.py +++ b/features/steps/export_steps.py @@ -89,6 +89,14 @@ def entry_array_count(context, entry_number, name, items_number): assert len(out_json["entries"][entry_number - 1][name]) == items_number +@then('entry {entry_number:d} should not have an array "{name}"') +def entry_not_array_item(context, entry_number, name): + # note that entry_number is 1-indexed. + out = context.stdout_capture.getvalue() + out_json = json.loads(out) + assert name not in out_json["entries"][entry_number - 1] + + @then("the output should be a valid XML string") def assert_valid_xml_string(context): output = context.stdout_capture.getvalue() diff --git a/features/steps/override.py b/features/steps/override.py index ff1760ed..8e509e88 100644 --- a/features/steps/override.py +++ b/features/steps/override.py @@ -1,11 +1,11 @@ -from jrnl.jrnl import run from unittest import mock -# from __future__ import with_statement -from jrnl.args import parse_args from behave import then -from features.steps.core import _mock_getpass, _mock_time_parse +from jrnl.args import parse_args +from jrnl.behave_testing import _mock_getpass +from jrnl.behave_testing import _mock_time_parse +from jrnl.jrnl import run @then("the editor {editor} should have been called") diff --git a/features/write.feature b/features/write.feature index eb22e480..08766a4d 100644 --- a/features/write.feature +++ b/features/write.feature @@ -183,6 +183,8 @@ Feature: Writing new entries. And we run "jrnl -until 1980" Then the output should be "1979-05-01 09:00 Being born hurts." + # the testing plugins override the JSON exporter + @skip_no_external_plugins Scenario: Writing into Dayone adds extended metadata Given we use the config "dayone.yaml" When we run "jrnl 01 may 1979: Being born hurts." diff --git a/jrnl/args.py b/jrnl/args.py index c8bd7743..9fc44c1f 100644 --- a/jrnl/args.py +++ b/jrnl/args.py @@ -12,9 +12,9 @@ from .commands import postconfig_list from .commands import preconfig_diagnostic from .commands import preconfig_version from .output import deprecated_cmd -from .plugins import EXPORT_FORMATS -from .plugins import IMPORT_FORMATS from .plugins import util +from .plugins.collector import EXPORT_FORMATS +from .plugins.collector import IMPORT_FORMATS class WrappingFormatter(argparse.RawTextHelpFormatter): diff --git a/jrnl/behave_testing.py b/jrnl/behave_testing.py new file mode 100644 index 00000000..c31a7010 --- /dev/null +++ b/jrnl/behave_testing.py @@ -0,0 +1,47 @@ +# Copyright (C) 2012-2021 jrnl contributors +# License: https://www.gnu.org/licenses/gpl-3.0.html + +""" +Certain functions to support the *behave* test suite. + +They are placed here so they are importable in multiple places, as otherwise +imports fail when running the suite outside of the project's root folder. + +""" +import jrnl.time + + +def _mock_getpass(inputs): + def prompt_return(prompt=""): + if type(inputs) == str: + return inputs + try: + return next(inputs) + except StopIteration: + raise KeyboardInterrupt + + return prompt_return + + +def _mock_input(inputs): + def prompt_return(prompt=""): + try: + val = next(inputs) + print(prompt, val) + return val + except StopIteration: + raise KeyboardInterrupt + + return prompt_return + + +def _mock_time_parse(context): + original_parse = jrnl.time.parse + if "now" not in context: + return original_parse + + def wrapper(input, *args, **kwargs): + input = context.now if input == "now" else input + return original_parse(input, *args, **kwargs) + + return wrapper diff --git a/jrnl/commands.py b/jrnl/commands.py index 07ca0767..8da64499 100644 --- a/jrnl/commands.py +++ b/jrnl/commands.py @@ -28,6 +28,12 @@ def preconfig_diagnostic(_): def preconfig_version(_): from jrnl import __title__ from jrnl import __version__ + from jrnl.plugins.collector import ( + IMPORT_FORMATS, + EXPORT_FORMATS, + get_exporter, + get_importer, + ) version_str = f"""{__title__} version {__version__} @@ -37,6 +43,22 @@ This is free software, and you are welcome to redistribute it under certain conditions; for details, see: https://www.gnu.org/licenses/gpl-3.0.html""" print(version_str) + print() + print("Active Plugins:") + print(" Importers:") + for importer in IMPORT_FORMATS: + importer_class = get_importer(importer) + print( + f" {importer} : {importer_class.version} from", + f"{importer_class().class_path()}", + ) + print(" Exporters:") + for exporter in EXPORT_FORMATS: + exporter_class = get_exporter(exporter) + # print(f" {exporter} : {exporter_class.version} from {exporter_class().class_path()}") + print(f" {exporter} : ", end="") + print(f"{exporter_class.version} from ", end="") + print(f"{exporter_class().class_path()}") def postconfig_list(config, **kwargs): @@ -47,7 +69,7 @@ def postconfig_list(config, **kwargs): def postconfig_import(args, config, **kwargs): from .Journal import open_journal - from .plugins import get_importer + from .plugins.collector import get_importer # Requires opening the journal journal = open_journal(args.journal_name, config) diff --git a/jrnl/contrib/exporter/.gitkeep b/jrnl/contrib/exporter/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/jrnl/contrib/importer/.gitkeep b/jrnl/contrib/importer/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/jrnl/jrnl.py b/jrnl/jrnl.py index 2d06115d..7b5d17ea 100644 --- a/jrnl/jrnl.py +++ b/jrnl/jrnl.py @@ -324,19 +324,17 @@ def _delete_search_results(journal, old_entries, **kwargs): def _display_search_results(args, journal, **kwargs): if args.short or args.export == "short": - print(journal.pprint(short=True)) - - elif args.export == "pretty": - print(journal.pprint()) + print(plugins.collector.get_exporter("short").export(journal)) elif args.tags: - print(plugins.get_exporter("tags").export(journal)) + print(plugins.collector.get_exporter("tags").export(journal)) elif args.export: - exporter = plugins.get_exporter(args.export) + exporter = plugins.collector.get_exporter(args.export) print(exporter.export(journal, args.filename)) elif kwargs["config"].get("display_format"): - exporter = plugins.get_exporter(kwargs["config"]["display_format"]) + exporter = plugins.collector.get_exporter(kwargs["config"]["display_format"]) print(exporter.export(journal, args.filename)) else: - print(journal.pprint()) + # print(journal.pprint()) + print(plugins.collector.get_exporter("default").export(journal)) diff --git a/jrnl/plugins/__init__.py b/jrnl/plugins/__init__.py index da6199fb..8a3eb693 100644 --- a/jrnl/plugins/__init__.py +++ b/jrnl/plugins/__init__.py @@ -1,48 +1,4 @@ # encoding: utf-8 + # Copyright (C) 2012-2021 jrnl contributors # License: https://www.gnu.org/licenses/gpl-3.0.html - -from .fancy_exporter import FancyExporter -from .jrnl_importer import JRNLImporter -from .json_exporter import JSONExporter -from .markdown_exporter import MarkdownExporter -from .tag_exporter import TagExporter -from .dates_exporter import DatesExporter -from .template_exporter import __all__ as template_exporters -from .text_exporter import TextExporter -from .xml_exporter import XMLExporter -from .yaml_exporter import YAMLExporter - -__exporters = [ - JSONExporter, - MarkdownExporter, - TagExporter, - DatesExporter, - TextExporter, - XMLExporter, - YAMLExporter, - FancyExporter, -] + template_exporters -__importers = [JRNLImporter] - -__exporter_types = {name: plugin for plugin in __exporters for name in plugin.names} -__exporter_types["pretty"] = None -__exporter_types["short"] = None -__importer_types = {name: plugin for plugin in __importers for name in plugin.names} - -EXPORT_FORMATS = sorted(__exporter_types.keys()) -IMPORT_FORMATS = sorted(__importer_types.keys()) - - -def get_exporter(format): - for exporter in __exporters: - if hasattr(exporter, "names") and format in exporter.names: - return exporter - return None - - -def get_importer(format): - for importer in __importers: - if hasattr(importer, "names") and format in importer.names: - return importer - return None diff --git a/jrnl/plugins/text_exporter.py b/jrnl/plugins/base.py similarity index 55% rename from jrnl/plugins/text_exporter.py rename to jrnl/plugins/base.py index c9eaaf14..772d9a3a 100644 --- a/jrnl/plugins/text_exporter.py +++ b/jrnl/plugins/base.py @@ -2,6 +2,11 @@ # Copyright (C) 2012-2021 jrnl contributors # License: https://www.gnu.org/licenses/gpl-3.0.html +""" +Base class for Importers and Exporters. +""" + + import os import re import unicodedata @@ -10,16 +15,36 @@ from jrnl.color import ERROR_COLOR from jrnl.color import RESET_COLOR -class TextExporter: - """This Exporter can convert entries and journals into text files.""" +class BaseImporter: + """Base Importer class (to sub-class)""" - names = ["text", "txt"] - extension = "txt" + # names = ["jrnl"] + # version = __version__ + + @classmethod + def class_path(cls): + return cls.__module__ + + @staticmethod + def import_(journal, input=None): + raise NotImplementedError + + +class BaseExporter: + """Base Exporter class (to sub-class)""" + + # names = ["text", "txt"] + # extension = "txt" + # version = __version__ + + @classmethod + def class_path(cls): + return cls.__module__ @classmethod def export_entry(cls, entry): """Returns a string representation of a single entry.""" - return str(entry) + raise NotImplementedError @classmethod def export_journal(cls, journal): @@ -32,9 +57,16 @@ class TextExporter: try: with open(path, "w", encoding="utf-8") as f: f.write(cls.export_journal(journal)) - return f"[Journal exported to {path}]" + return ( + f"[Journal '{journal.name}' exported (as a single file) to {path}]" + ) except IOError as e: return f"[{ERROR_COLOR}ERROR{RESET_COLOR}: {e.filename} {e.strerror}]" + except NotImplementedError: + return ( + f"[{ERROR_COLOR}ERROR{RESET_COLOR}: This exporter doesn't support " + "exporting as a single file.]" + ) @classmethod def make_filename(cls, entry): @@ -45,16 +77,23 @@ class TextExporter: @classmethod def write_files(cls, journal, path): """Exports a journal into individual files for each entry.""" - for entry in journal.entries: - try: - full_path = os.path.join(path, cls.make_filename(entry)) - with open(full_path, "w", encoding="utf-8") as f: - f.write(cls.export_entry(entry)) - except IOError as e: - return "[{2}ERROR{3}: {0} {1}]".format( - e.filename, e.strerror, ERROR_COLOR, RESET_COLOR - ) - return "[Journal exported to {}]".format(path) + try: + for entry in journal.entries: + try: + full_path = os.path.join(path, cls.make_filename(entry)) + with open(full_path, "w", encoding="utf-8") as f: + f.write(cls.export_entry(entry)) + except IOError as e: + return "[{2}ERROR{3}: {0} {1}]".format( + e.filename, e.strerror, ERROR_COLOR, RESET_COLOR + ) + except NotImplementedError: + return ( + f"[{ERROR_COLOR}ERROR{RESET_COLOR}: This exporter doesn't support " + "exporting as individual files.]" + ) + else: + return f"[Journal '{journal.name}' exported (as multiple files) to {path}]" def _slugify(string): """Slugifies a string. diff --git a/jrnl/plugins/collector.py b/jrnl/plugins/collector.py new file mode 100644 index 00000000..c848c508 --- /dev/null +++ b/jrnl/plugins/collector.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python +# encoding: utf-8 +# Copyright (C) 2012-2021 jrnl contributors +# License: https://www.gnu.org/licenses/gpl-3.0.html + +""" +Code relating to the collecting of plugins and distributing calls to them. + +In particular, the code here collects the list of imports and exporters, both +internal and external, and tells the main program which plugins are available. +Actual calling of the plugins is done directly and works because given plugin +functions are importable/callable at predetermined (code) locations. + +Internal plugins are located in the `jrnl.plugins` namespace, and external +plugins are located in the `jrnl.contrib` namespace. + +This file was originally called "meta", using that title in the reflexive sense; +i.e. it is the collection of code that allows plugins to deal with themselves. +""" + +import importlib +import pkgutil + +import jrnl.contrib.exporter +import jrnl.contrib.importer +import jrnl.plugins.exporter +import jrnl.plugins.importer + +__exporters_builtin = list( + pkgutil.iter_modules( + jrnl.plugins.exporter.__path__, jrnl.plugins.exporter.__name__ + "." + ) +) +__exporters_contrib = list( + pkgutil.iter_modules( + jrnl.contrib.exporter.__path__, jrnl.contrib.exporter.__name__ + "." + ) +) + +__importers_builtin = list( + pkgutil.iter_modules( + jrnl.plugins.importer.__path__, jrnl.plugins.importer.__name__ + "." + ) +) +__importers_contrib = list( + pkgutil.iter_modules( + jrnl.contrib.importer.__path__, jrnl.contrib.importer.__name__ + "." + ) +) + +__exporter_types_builtin = { + name: importlib.import_module(plugin.name) + for plugin in __exporters_builtin + for name in importlib.import_module(plugin.name).Exporter.names +} +__exporter_types_contrib = { + name: importlib.import_module(plugin.name) + for plugin in __exporters_contrib + for name in importlib.import_module(plugin.name).Exporter.names +} + + +__importer_types_builtin = { + name: importlib.import_module(plugin.name) + for plugin in __importers_builtin + for name in importlib.import_module(plugin.name).Importer.names +} +__importer_types_contrib = { + name: importlib.import_module(plugin.name) + for plugin in __importers_contrib + for name in importlib.import_module(plugin.name).Importer.names +} + +__exporter_types = { + **__exporter_types_builtin, + **__exporter_types_contrib, +} +__importer_types = { + **__importer_types_builtin, + **__importer_types_contrib, +} + +EXPORT_FORMATS = sorted(__exporter_types.keys()) +"""list of stings: all available export formats.""" +IMPORT_FORMATS = sorted(__importer_types.keys()) +"""list of stings: all available import formats.""" + + +def get_exporter(format): + """ + Given an export format, returns the (callable) class of the corresponding exporter. + """ + try: + return __exporter_types[format].Exporter + except (AttributeError, KeyError): + return None + + +def get_importer(format): + """ + Given an import format, returns the (callable) class of the corresponding importer. + """ + try: + return __importer_types[format].Importer + except (AttributeError, KeyError): + return None diff --git a/jrnl/plugins/dates_exporter.py b/jrnl/plugins/exporter/dates.py similarity index 86% rename from jrnl/plugins/dates_exporter.py rename to jrnl/plugins/exporter/dates.py index e032b652..9ed0c70e 100644 --- a/jrnl/plugins/dates_exporter.py +++ b/jrnl/plugins/exporter/dates.py @@ -1,16 +1,20 @@ # encoding: utf-8 # Copyright (C) 2012-2021 jrnl contributors # License: https://www.gnu.org/licenses/gpl-3.0.html + from collections import Counter -from .text_exporter import TextExporter +from jrnl.plugins.base import BaseExporter + +from ... import __version__ -class DatesExporter(TextExporter): +class Exporter(BaseExporter): """This Exporter lists dates and their respective counts, for heatingmapping etc.""" names = ["dates"] extension = "dates" + version = __version__ @classmethod def export_entry(cls, entry): diff --git a/jrnl/plugins/fancy_exporter.py b/jrnl/plugins/exporter/fancy.py similarity index 85% rename from jrnl/plugins/fancy_exporter.py rename to jrnl/plugins/exporter/fancy.py index 2cb27eca..7bfe16f0 100644 --- a/jrnl/plugins/fancy_exporter.py +++ b/jrnl/plugins/exporter/fancy.py @@ -2,17 +2,21 @@ # Copyright (C) 2012-2021 jrnl contributors # License: https://www.gnu.org/licenses/gpl-3.0.html -from jrnl.exception import JrnlError + from textwrap import TextWrapper -from .text_exporter import TextExporter +from jrnl.plugins.base import BaseExporter +from jrnl.plugins.util import check_provided_linewrap_viability + +from ... import __version__ -class FancyExporter(TextExporter): +class Exporter(BaseExporter): """This Exporter can convert entries and journals into text with unicode box drawing characters.""" names = ["fancy", "boxed"] extension = "txt" + version = __version__ # Top border of the card border_a = "┎" @@ -79,14 +83,3 @@ class FancyExporter(TextExporter): def export_journal(cls, journal): """Returns a unicode representation of an entire journal.""" return "\n".join(cls.export_entry(entry) for entry in journal) - - -def check_provided_linewrap_viability(linewrap, card, journal): - if len(card[0]) > linewrap: - width_violation = len(card[0]) - linewrap - raise JrnlError( - "LineWrapTooSmallForDateFormat", - config_linewrap=linewrap, - columns=width_violation, - journal=journal, - ) diff --git a/jrnl/plugins/json_exporter.py b/jrnl/plugins/exporter/json.py similarity index 92% rename from jrnl/plugins/json_exporter.py rename to jrnl/plugins/exporter/json.py index 666d9a3d..ab714a0e 100644 --- a/jrnl/plugins/json_exporter.py +++ b/jrnl/plugins/exporter/json.py @@ -4,15 +4,18 @@ import json -from .text_exporter import TextExporter -from .util import get_tags_count +from jrnl.plugins.base import BaseExporter +from jrnl.plugins.util import get_tags_count + +from ... import __version__ -class JSONExporter(TextExporter): +class Exporter(BaseExporter): """This Exporter can convert entries and journals into json.""" names = ["json"] extension = "json" + version = __version__ @classmethod def entry_to_dict(cls, entry): diff --git a/jrnl/plugins/markdown_exporter.py b/jrnl/plugins/exporter/markdown.py similarity index 95% rename from jrnl/plugins/markdown_exporter.py rename to jrnl/plugins/exporter/markdown.py index 11f748b6..87a28393 100644 --- a/jrnl/plugins/markdown_exporter.py +++ b/jrnl/plugins/exporter/markdown.py @@ -8,15 +8,17 @@ import sys from jrnl.color import RESET_COLOR from jrnl.color import WARNING_COLOR +from jrnl.plugins.base import BaseExporter -from .text_exporter import TextExporter +from ... import __version__ -class MarkdownExporter(TextExporter): +class Exporter(BaseExporter): """This Exporter can convert entries and journals into Markdown.""" names = ["md", "markdown"] extension = "md" + version = __version__ @classmethod def export_entry(cls, entry, to_multifile=True): diff --git a/jrnl/plugins/exporter/pretty.py b/jrnl/plugins/exporter/pretty.py new file mode 100644 index 00000000..e44eb872 --- /dev/null +++ b/jrnl/plugins/exporter/pretty.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +# encoding: utf-8 +# Copyright (C) 2012-2021 jrnl contributors +# License: https://www.gnu.org/licenses/gpl-3.0.html + +from jrnl.plugins.base import BaseExporter + +from ... import __version__ + + +class Exporter(BaseExporter): + """Pretty print journal""" + + names = ["pretty", "default"] + extension = "txt" + version = __version__ + + @classmethod + def export_journal(cls, journal): + return journal.pprint() diff --git a/jrnl/plugins/exporter/short.py b/jrnl/plugins/exporter/short.py new file mode 100644 index 00000000..c9c21a8a --- /dev/null +++ b/jrnl/plugins/exporter/short.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +# encoding: utf-8 +# Copyright (C) 2012-2021 jrnl contributors +# License: https://www.gnu.org/licenses/gpl-3.0.html + +from jrnl.plugins.base import BaseExporter + +from ... import __version__ + + +class Exporter(BaseExporter): + """Short export -- i.e. single line date and title""" + + names = ["short"] + extension = "txt" + version = __version__ + + @classmethod + def export_journal(cls, journal): + return journal.pprint(short=True) diff --git a/jrnl/plugins/tag_exporter.py b/jrnl/plugins/exporter/tag.py similarity index 85% rename from jrnl/plugins/tag_exporter.py rename to jrnl/plugins/exporter/tag.py index 1153fa01..f33ab999 100644 --- a/jrnl/plugins/tag_exporter.py +++ b/jrnl/plugins/exporter/tag.py @@ -2,15 +2,19 @@ # Copyright (C) 2012-2021 jrnl contributors # License: https://www.gnu.org/licenses/gpl-3.0.html -from .text_exporter import TextExporter -from .util import get_tags_count + +from jrnl.plugins.base import BaseExporter +from jrnl.plugins.util import get_tags_count + +from ... import __version__ -class TagExporter(TextExporter): +class Exporter(BaseExporter): """This Exporter can lists the tags for entries and journals, exported as a plain text file.""" names = ["tags"] extension = "tags" + version = __version__ @classmethod def export_entry(cls, entry): diff --git a/jrnl/plugins/exporter/text.py b/jrnl/plugins/exporter/text.py new file mode 100644 index 00000000..20339922 --- /dev/null +++ b/jrnl/plugins/exporter/text.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python +# encoding: utf-8 +# Copyright (C) 2012-2021 jrnl contributors +# License: https://www.gnu.org/licenses/gpl-3.0.html + +from jrnl.plugins.base import BaseExporter + +from ... import __version__ + + +class Exporter(BaseExporter): + """This Exporter can convert entries and journals into text files.""" + + names = ["text", "txt"] + extension = "txt" + version = __version__ + + @classmethod + def export_entry(cls, entry): + """Returns a string representation of a single entry.""" + return str(entry) diff --git a/jrnl/plugins/xml_exporter.py b/jrnl/plugins/exporter/xml.py similarity index 92% rename from jrnl/plugins/xml_exporter.py rename to jrnl/plugins/exporter/xml.py index 9901f4b7..69bf0225 100644 --- a/jrnl/plugins/xml_exporter.py +++ b/jrnl/plugins/exporter/xml.py @@ -4,30 +4,18 @@ from xml.dom import minidom -from .json_exporter import JSONExporter -from .util import get_tags_count +from jrnl.plugins.base import BaseExporter +from jrnl.plugins.util import get_tags_count + +from ... import __version__ -class XMLExporter(JSONExporter): +class Exporter(BaseExporter): """This Exporter can convert entries and journals into XML.""" names = ["xml"] extension = "xml" - - @classmethod - def export_entry(cls, entry, doc=None): - """Returns an XML representation of a single entry.""" - doc_el = doc or minidom.Document() - entry_el = doc_el.createElement("entry") - for key, value in cls.entry_to_dict(entry).items(): - elem = doc_el.createElement(key) - elem.appendChild(doc_el.createTextNode(value)) - entry_el.appendChild(elem) - if not doc: - doc_el.appendChild(entry_el) - return doc_el.toprettyxml() - else: - return entry_el + version = __version__ @classmethod def entry_to_xml(cls, entry, doc): @@ -44,6 +32,21 @@ class XMLExporter(JSONExporter): entry_el.appendChild(doc.createTextNode(entry.fulltext)) return entry_el + @classmethod + def export_entry(cls, entry, doc=None): + """Returns an XML representation of a single entry.""" + doc_el = doc or minidom.Document() + entry_el = doc_el.createElement("entry") + for key, value in cls.entry_to_dict(entry).items(): + elem = doc_el.createElement(key) + elem.appendChild(doc_el.createTextNode(value)) + entry_el.appendChild(elem) + if not doc: + doc_el.appendChild(entry_el) + return doc_el.toprettyxml() + else: + return entry_el + @classmethod def export_journal(cls, journal): """Returns an XML representation of an entire journal.""" diff --git a/jrnl/plugins/yaml_exporter.py b/jrnl/plugins/exporter/yaml.py similarity index 93% rename from jrnl/plugins/yaml_exporter.py rename to jrnl/plugins/exporter/yaml.py index 887fdaf1..ff76790c 100644 --- a/jrnl/plugins/yaml_exporter.py +++ b/jrnl/plugins/exporter/yaml.py @@ -9,15 +9,17 @@ import sys from jrnl.color import ERROR_COLOR from jrnl.color import RESET_COLOR from jrnl.color import WARNING_COLOR +from jrnl.plugins.base import BaseExporter -from .text_exporter import TextExporter +from ... import __version__ -class YAMLExporter(TextExporter): +class Exporter(BaseExporter): """This Exporter can convert entries and journals into Markdown formatted text with YAML front matter.""" names = ["yaml"] extension = "md" + version = __version__ @classmethod def export_entry(cls, entry, to_multifile=True): @@ -132,9 +134,10 @@ class YAMLExporter(TextExporter): def export_journal(cls, journal): """Returns an error, as YAML export requires a directory as a target.""" print( - "{}ERROR{}: YAML export must be to individual files. Please specify a directory to export to.".format( - ERROR_COLOR, RESET_COLOR + ( + f"[{ERROR_COLOR}ERROR{RESET_COLOR}: YAML export must be to " + "individual files. Please specify a directory to export to.]" ), file=sys.stderr, ) - return + raise NotImplementedError diff --git a/jrnl/plugins/jrnl_importer.py b/jrnl/plugins/importer/jrnl.py similarity index 78% rename from jrnl/plugins/jrnl_importer.py rename to jrnl/plugins/importer/jrnl.py index 214fc70b..c403e4f0 100644 --- a/jrnl/plugins/jrnl_importer.py +++ b/jrnl/plugins/importer/jrnl.py @@ -4,11 +4,16 @@ import sys +from jrnl.plugins.base import BaseImporter -class JRNLImporter: +from ... import __version__ + + +class Importer(BaseImporter): """This plugin imports entries from other jrnl files.""" names = ["jrnl"] + version = __version__ @staticmethod def import_(journal, input=None): @@ -27,7 +32,9 @@ class JRNLImporter: journal.import_(other_journal_txt) new_cnt = len(journal.entries) print( - "[{} imported to {} journal]".format(new_cnt - old_cnt, journal.name), + "[{} entries imported to '{}' journal]".format( + new_cnt - old_cnt, journal.name + ), file=sys.stderr, ) journal.write() diff --git a/jrnl/plugins/template.py b/jrnl/plugins/template.py deleted file mode 100644 index cb852471..00000000 --- a/jrnl/plugins/template.py +++ /dev/null @@ -1,142 +0,0 @@ -# Copyright (C) 2012-2021 jrnl contributors -# License: https://www.gnu.org/licenses/gpl-3.0.html - -import re - -import yaml - -VAR_RE = r"[_a-zA-Z][a-zA-Z0-9_]*" -EXPRESSION_RE = r"[\[\]():.a-zA-Z0-9_]*" -PRINT_RE = r"{{ *(.+?) *}}" -START_BLOCK_RE = r"{% *(if|for) +(.+?) *%}" -END_BLOCK_RE = r"{% *end(for|if) *%}" -FOR_RE = r"{{% *for +({varname}) +in +([^%]+) *%}}".format(varname=VAR_RE) -IF_RE = r"{% *if +(.+?) *%}" -BLOCK_RE = r"{% *block +(.+?) *%}((?:.|\n)+?){% *endblock *%}" -INCLUDE_RE = r"{% *include +(.+?) *%}" - - -class Template: - def __init__(self, template): - self.template = template - self.clean_template = None - self.blocks = {} - - @classmethod - def from_file(cls, filename): - with open(filename) as f: - front_matter, body = f.read().strip("-\n").split("---", 2) - front_matter = yaml.load(front_matter, Loader=yaml.SafeLoader) - template = cls(body) - template.__dict__.update(front_matter) - return template - - def render(self, **vars): - if self.clean_template is None: - self._get_blocks() - return self._expand(self.clean_template, **vars) - - def render_block(self, block, **vars): - if self.clean_template is None: - self._get_blocks() - return self._expand(self.blocks[block], **vars) - - def _eval_context(self, vars): - import asteval - - e = asteval.Interpreter(use_numpy=False, writer=None) - e.symtable.update(vars) - e.symtable["__last_iteration"] = vars.get("__last_iteration", False) - return e - - def _get_blocks(self): - def s(match): - name, contents = match.groups() - self.blocks[name] = self._strip_single_nl(contents) - return "" - - self.clean_template = re.sub(BLOCK_RE, s, self.template, flags=re.MULTILINE) - - def _expand(self, template, **vars): - stack = sorted( - [ - (m.start(), 1, m.groups()[0]) - for m in re.finditer(START_BLOCK_RE, template) - ] - + [ - (m.end(), -1, m.groups()[0]) - for m in re.finditer(END_BLOCK_RE, template) - ] - ) - - last_nesting, nesting = 0, 0 - start = 0 - result = "" - block_type = None - if not stack: - return self._expand_vars(template, **vars) - - for pos, indent, typ in stack: - nesting += indent - if nesting == 1 and last_nesting == 0: - block_type = typ - result += self._expand_vars(template[start:pos], **vars) - start = pos - elif nesting == 0 and last_nesting == 1: - if block_type == "if": - result += self._expand_cond(template[start:pos], **vars) - elif block_type == "for": - result += self._expand_loops(template[start:pos], **vars) - elif block_type == "block": - result += self._save_block(template[start:pos], **vars) - start = pos - last_nesting = nesting - - result += self._expand_vars(template[stack[-1][0] :], **vars) - return result - - def _expand_vars(self, template, **vars): - safe_eval = self._eval_context(vars) - expanded = re.sub( - INCLUDE_RE, lambda m: self.render_block(m.groups()[0], **vars), template - ) - return re.sub(PRINT_RE, lambda m: str(safe_eval(m.groups()[0])), expanded) - - def _expand_cond(self, template, **vars): - start_block = re.search(IF_RE, template, re.M) - end_block = list(re.finditer(END_BLOCK_RE, template, re.M))[-1] - expression = start_block.groups()[0] - sub_template = self._strip_single_nl( - template[start_block.end() : end_block.start()] - ) - - safe_eval = self._eval_context(vars) - if safe_eval(expression): - return self._expand(sub_template) - return "" - - def _strip_single_nl(self, template, strip_r=True): - if template[0] == "\n": - template = template[1:] - if strip_r and template[-1] == "\n": - template = template[:-1] - return template - - def _expand_loops(self, template, **vars): - start_block = re.search(FOR_RE, template, re.M) - end_block = list(re.finditer(END_BLOCK_RE, template, re.M))[-1] - var_name, iterator = start_block.groups() - sub_template = self._strip_single_nl( - template[start_block.end() : end_block.start()], strip_r=False - ) - - safe_eval = self._eval_context(vars) - - result = "" - items = safe_eval(iterator) - for idx, var in enumerate(items): - vars[var_name] = var - vars["__last_iteration"] = idx == len(items) - 1 - result += self._expand(sub_template, **vars) - del vars[var_name] - return self._strip_single_nl(result) diff --git a/jrnl/plugins/template_exporter.py b/jrnl/plugins/template_exporter.py deleted file mode 100644 index d2e5ce3e..00000000 --- a/jrnl/plugins/template_exporter.py +++ /dev/null @@ -1,43 +0,0 @@ -# encoding: utf-8 -# Copyright (C) 2012-2021 jrnl contributors -# License: https://www.gnu.org/licenses/gpl-3.0.html - -from glob import glob -import os - -from .template import Template -from .text_exporter import TextExporter - - -class GenericTemplateExporter(TextExporter): - """This Exporter can convert entries and journals into text files.""" - - @classmethod - def export_entry(cls, entry): - """Returns a string representation of a single entry.""" - vars = {"entry": entry, "tags": entry.tags} - return cls.template.render_block("entry", **vars) - - @classmethod - def export_journal(cls, journal): - """Returns a string representation of an entire journal.""" - vars = {"journal": journal, "entries": journal.entries, "tags": journal.tags} - return cls.template.render_block("journal", **vars) - - -def __exporter_from_file(template_file): - """Create a template class from a file""" - name = os.path.basename(template_file).replace(".template", "") - template = Template.from_file(template_file) - return type( - str(f"{name.title()}Exporter"), - (GenericTemplateExporter,), - {"names": [name], "extension": template.extension, "template": template}, - ) - - -__all__ = [] - -# Factory pattern to create Exporter classes for all available templates -for template_file in glob("jrnl/templates/*.template"): - __all__.append(__exporter_from_file(template_file)) diff --git a/jrnl/plugins/util.py b/jrnl/plugins/util.py index ae49a2a8..3c596e78 100644 --- a/jrnl/plugins/util.py +++ b/jrnl/plugins/util.py @@ -2,11 +2,13 @@ # Copyright (C) 2012-2021 jrnl contributors # License: https://www.gnu.org/licenses/gpl-3.0.html +from jrnl.exception import JrnlError + def get_tags_count(journal): """Returns a set of tuples (count, tag) for all tags present in the journal.""" # Astute reader: should the following line leave you as puzzled as me the first time - # I came across this construction, worry not and embrace the ensuing moment of enlightment. + # I came across this construction, worry not and embrace the ensuing moment of enlightenment. tags = [tag for entry in journal.entries for tag in set(entry.tags)] # To be read: [for entry in journal.entries: for tag in set(entry.tags): tag] tag_counts = {(tags.count(tag), tag) for tag in tags} @@ -24,3 +26,14 @@ def oxford_list(lst): return lst[0] + " or " + lst[1] else: return ", ".join(lst[:-1]) + ", or " + lst[-1] + + +def check_provided_linewrap_viability(linewrap, card, journal): + if len(card[0]) > linewrap: + width_violation = len(card[0]) - linewrap + raise JrnlError( + "LineWrapTooSmallForDateFormat", + config_linewrap=linewrap, + columns=width_violation, + journal=journal, + ) diff --git a/jrnl/templates/sample.template b/jrnl/templates/sample.template deleted file mode 100644 index 983d6af3..00000000 --- a/jrnl/templates/sample.template +++ /dev/null @@ -1,18 +0,0 @@ ---- -extension: txt ---- - -{% block journal %} -{% for entry in entries %} -{% include entry %} -{% endfor %} - -{% endblock %} - -{% block entry %} -{{ entry.title }} -{{ "-" * len(entry.title) }} - -{{ entry.body }} - -{% endblock %} diff --git a/mkdocs.yml b/mkdocs.yml index 65a515da..cbd8db10 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -12,6 +12,9 @@ extra_css: - assets/highlight.css markdown_extensions: - admonition +plugins: + - include-markdown + - search repo_url: https://github.com/jrnl-org/jrnl/ edit_uri: edit/develop/docs/ site_author: jrnl contributors @@ -24,4 +27,5 @@ nav: - Privacy and Security: privacy-and-security.md - Formats: formats.md - Advanced Usage: advanced.md + - Custom Importers & Exporters: plugins.md - Recipes: recipes.md diff --git a/poetry.lock b/poetry.lock index 94b251d3..3e6fd010 100644 --- a/poetry.lock +++ b/poetry.lock @@ -447,6 +447,18 @@ watchdog = ">=2.0" [package.extras] i18n = ["babel (>=2.9.0)"] +[[package]] +name = "mkdocs-include-markdown-plugin" +version = "2.8.0" +description = "Mkdocs Markdown includer plugin." +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.extras] +dev = ["bump2version (==1.0.1)", "flake8 (==3.8.4)", "flake8-implicit-str-concat (==0.2.0)", "flake8-print (==4.0.0)", "isort (==5.6.4)", "pre-commit (==2.9.2)", "pytest (==6.1.2)", "pytest-cov (==2.10.1)", "pyupgrade (==2.9.0)", "yamllint (==1.25.0)"] +test = ["pytest (==6.1.2)", "pytest-cov (==2.10.1)"] + [[package]] name = "mypy-extensions" version = "0.4.3" @@ -1139,6 +1151,10 @@ mkdocs = [ {file = "mkdocs-1.2.1-py3-none-any.whl", hash = "sha256:11141126e5896dd9d279b3e4814eb488e409a0990fb638856255020406a8e2e7"}, {file = "mkdocs-1.2.1.tar.gz", hash = "sha256:6e0ea175366e3a50d334597b0bc042b8cebd512398cdd3f6f34842d0ef524905"}, ] +mkdocs-include-markdown-plugin = [ + {file = "mkdocs_include_markdown_plugin-2.8.0-py3-none-any.whl", hash = "sha256:29b7d40da2945414f4dcb4c39eac004da6a644433f10d9da0dd5e331e50a5dbf"}, + {file = "mkdocs_include_markdown_plugin-2.8.0.tar.gz", hash = "sha256:a4171b1f8a5cb4e2e05f2989ca47f4825ed0723021af7a3a871f8abe7cb91ba0"}, +] mypy-extensions = [ {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, diff --git a/tests/external_plugins_src/README.md b/tests/external_plugins_src/README.md new file mode 100644 index 00000000..a6674f4e --- /dev/null +++ b/tests/external_plugins_src/README.md @@ -0,0 +1,9 @@ +# Rot13 Custom Exporter for Jrnl + +This is a custom exporter to demostrate how to write customer exporters for +[jrnl](https://github.com/jrnl-org/jrnl). It is also used by *jrnl* in its +tests to ensure the feature works as expected. + +This plugin applies a [Caeser +cipher](https://en.wikipedia.org/wiki/Caesar_cipher) (specifically the +[ROT13](https://en.wikipedia.org/wiki/ROT13)) to output text. diff --git a/tests/external_plugins_src/jrnl/contrib/exporter/custom_json.py b/tests/external_plugins_src/jrnl/contrib/exporter/custom_json.py new file mode 100644 index 00000000..7f816bb8 --- /dev/null +++ b/tests/external_plugins_src/jrnl/contrib/exporter/custom_json.py @@ -0,0 +1,37 @@ +# pelican\contrib\exporter\custom_json.py +import json + +from jrnl.plugins.base import BaseExporter + +__version__ = "v1.0.0" + + +class Exporter(BaseExporter): + """ + This basic Exporter can convert entries and journals into JSON. + """ + + names = ["json"] + extension = "json" + version = __version__ + + @classmethod + def entry_to_dict(cls, entry): + return { + "title": entry.title, + "body": entry.body, + "date": entry.date.strftime("%Y-%m-%d"), + } + + @classmethod + def export_entry(cls, entry): + """Returns a json representation of a single entry.""" + return json.dumps(cls.entry_to_dict(entry), indent=2) + "\n" + + @classmethod + def export_journal(cls, journal): + """Returns a json representation of an entire journal.""" + result = { + "entries": [cls.entry_to_dict(e) for e in journal.entries], + } + return json.dumps(result, indent=2) diff --git a/tests/external_plugins_src/jrnl/contrib/exporter/flag.py b/tests/external_plugins_src/jrnl/contrib/exporter/flag.py new file mode 100644 index 00000000..a7b1c6a2 --- /dev/null +++ b/tests/external_plugins_src/jrnl/contrib/exporter/flag.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python +# encoding: utf-8 +# Copyright (C) 2012-2021 jrnl contributors +# License: https://www.gnu.org/licenses/gpl-3.0.html + +""" +Exporter for testing and experimentation purposes. + +It is not called "testing" because then it's not installed. + +The presence of this plugin is also used as a "switch" by the test suite to +decide on whether or not to run the "vanilla" test suite, or the test suite +for external plugins. + +The `export_entry` and `export_journal` methods are both purposely not +implemented to confirm behavior on plugins that don't implement them. +""" + +from jrnl.plugins.base import BaseExporter + + +class Exporter(BaseExporter): + names = ["testing", "test"] + version = "v0.0.1" + extension = "test" diff --git a/tests/external_plugins_src/jrnl/contrib/exporter/rot13.py b/tests/external_plugins_src/jrnl/contrib/exporter/rot13.py new file mode 100644 index 00000000..d7893f60 --- /dev/null +++ b/tests/external_plugins_src/jrnl/contrib/exporter/rot13.py @@ -0,0 +1,15 @@ +import codecs + +from jrnl.plugins.base import BaseExporter + +__version__ = "v1.0.0" + + +class Exporter(BaseExporter): + names = ["rot13", "txt"] + extension = "txt" + version = __version__ + + @classmethod + def export_entry(cls, entry): + return codecs.encode(str(entry), "rot_13") diff --git a/tests/external_plugins_src/jrnl/contrib/importer/simple_json.py b/tests/external_plugins_src/jrnl/contrib/importer/simple_json.py new file mode 100644 index 00000000..b1d27858 --- /dev/null +++ b/tests/external_plugins_src/jrnl/contrib/importer/simple_json.py @@ -0,0 +1,50 @@ +# pelican\contrib\importer\sample_json.py +import json +import sys + +from jrnl import Entry +from jrnl.plugins.base import BaseImporter + +__version__ = "v1.0.0" + + +class Importer(BaseImporter): + """JSON Importer for jrnl.""" + + names = ["json"] + version = __version__ + + @staticmethod + def import_(journal, input=None): + """ + Given a nicely formatted JSON file, will add the + contained Entries to the journal. + """ + + old_cnt = len(journal.entries) + if input: + with open(input, "r", encoding="utf-8") as f: + data = json.loads(f) + else: + try: + data = sys.stdin.read() + except KeyboardInterrupt: + print( + "[Entries NOT imported into journal.]", + file=sys.stderr, + ) + sys.exit(0) + + for json_entry in data: + raw = json_entry["title"] + "/n" + json_entry["body"] + date = json_entry["date"] + entry = Entry.Entry(journal, date, raw) + journal.entries.append(entry) + + new_cnt = len(journal.entries) + print( + "[{} entries imported to '{}' journal]".format( + new_cnt - old_cnt, journal.name + ), + file=sys.stderr, + ) diff --git a/tests/external_plugins_src/setup.py b/tests/external_plugins_src/setup.py new file mode 100644 index 00000000..820b5022 --- /dev/null +++ b/tests/external_plugins_src/setup.py @@ -0,0 +1,35 @@ +import os +import re + +try: + from setuptools import setup +except ImportError: + from distutils.core import setup + +base_dir = os.path.dirname(os.path.abspath(__file__)) + + +def get_version(filename="jrnl/contrib/exporter/rot13.py"): + with open(os.path.join(base_dir, filename), encoding="utf-8") as initfile: + for line in initfile.readlines(): + m = re.match("__version__ *= *['\"](.*)['\"]", line) + if m: + return m.group(1) + + +setup( + name="jrnl-demo-plugins", + version=get_version(), + description="Demonstration custom plugins for jrnl", + long_description="\n\n".join([open(os.path.join(base_dir, "README.md")).read()]), + long_description_content_type="text/markdown", + author="W. Minchin", + author_email="w_minchin@hotmail.com", + url="https://github.com/jrnl-org/jrnl/tree/develop/tests/external_plugins_src", + packages=["jrnl", "jrnl.contrib", "jrnl.contrib.exporter", "jrnl.contrib.importer"], + include_package_data=True, + install_requires=[ + "jrnl", + ], + zip_safe=False, # use wheels instead +) diff --git a/tests/test_plugin.py b/tests/test_plugin.py new file mode 100644 index 00000000..cd71effd --- /dev/null +++ b/tests/test_plugin.py @@ -0,0 +1,46 @@ +from datetime import date +import json + +import pytest + +from jrnl import Entry +from jrnl import Journal +from jrnl.plugins.exporter import json as json_exporter + +try: + from jrnl.contrib.exporter import testing as testing_exporter +except: + testing_exporter = None + + +if testing_exporter: + + @pytest.fixture() + def create_entry(): + entry = Entry.Entry( + journal=Journal.Journal(), + text="This is the entry text", + date=date(year=2001, month=1, day=1), + starred=True, + ) + yield entry + + class TestBaseExporter(testing_exporter.Exporter): + def test_unimplemented_export(self, create_entry): + entry = create_entry + with pytest.raises(NotImplementedError): + self.export_entry(entry) + + class TestJsonExporter(json_exporter.Exporter): + def test_json_exporter_name(self): + assert "json" in self.names + + def test_export_entry(self, create_entry): + entry = create_entry + fake_uuid = "ewqf09-432p9p0433-243209" # generated by mashing keys + entry.uuid = fake_uuid + exported = self.export_entry(entry) + deserialized_export = json.loads(exported) + assert deserialized_export["title"] == "This is the entry text" + assert deserialized_export["date"] == "2001-01-01" + assert "uuid" in deserialized_export.keys() diff --git a/tests/unit/test_display.py b/tests/unit/test_display.py index 921d1631..72b4554c 100644 --- a/tests/unit/test_display.py +++ b/tests/unit/test_display.py @@ -9,7 +9,7 @@ from jrnl.jrnl import _display_search_results # fmt: off # see: https://github.com/psf/black/issues/664 -@pytest.mark.parametrize("export_format", [ "pretty", "short","markdown"]) +@pytest.mark.parametrize("export_format", [ "pretty", "short", "markdown"]) #fmt: on @mock.patch.object(argparse, "Namespace", return_value={"export": "markdown", "filename": "irrele.vant"}) def test_export_format(mock_args, export_format):