From 6f1dd6077ed8d96d16e652f304499cc718b96d83 Mon Sep 17 00:00:00 2001 From: Manuel Ebert Date: Mon, 21 Jul 2014 13:30:10 +0900 Subject: [PATCH 1/6] Improves dateutil parsing Closes #133 Fixes #183, #185, #228 --- jrnl/Journal.py | 53 ++++++------------------------------------------- jrnl/time.py | 49 +++++++++++++++++++++++++++++++++++++++++++++ jrnl/util.py | 5 ++--- setup.py | 2 +- 4 files changed, 58 insertions(+), 51 deletions(-) create mode 100644 jrnl/time.py diff --git a/jrnl/Journal.py b/jrnl/Journal.py index 0eb0ce5c..2dab065a 100644 --- a/jrnl/Journal.py +++ b/jrnl/Journal.py @@ -4,12 +4,10 @@ from __future__ import absolute_import from . import Entry from . import util +from . import time import codecs -try: import parsedatetime.parsedatetime_consts as pdt -except ImportError: import parsedatetime as pdt import re from datetime import datetime -import dateutil import sys try: from Crypto.Cipher import AES @@ -34,9 +32,6 @@ class Journal(object): } self.config.update(kwargs) # Set up date parser - consts = pdt.Constants(usePyICU=False) - consts.DOWParseStyle = -1 # "Monday" will be either today or the last Monday - self.dateparse = pdt.Calendar(consts) self.key = None # used to decrypt and encrypt the journal self.search_tags = None # Store tags we're highlighting self.name = name @@ -212,8 +207,9 @@ class Journal(object): If strict is True, all tags must be present in an entry. If false, the entry is kept if any tag is present.""" self.search_tags = set([tag.lower() for tag in tags]) - end_date = self.parse_date(end_date) - start_date = self.parse_date(start_date) + end_date = time.parse(end_date, inclusive=True) + start_date = time.parse(start_date) + # If strict mode is on, all tags have to be present in entry tagged = self.search_tags.issubset if strict else self.search_tags.intersection result = [ @@ -239,43 +235,6 @@ class Journal(object): e.body = '' self.entries = result - def parse_date(self, date_str): - """Parses a string containing a fuzzy date and returns a datetime.datetime object""" - if not date_str: - return None - elif isinstance(date_str, datetime): - return date_str - - try: - date = dateutil.parser.parse(date_str) - flag = 1 if date.hour == 0 and date.minute == 0 else 2 - date = date.timetuple() - except: - date, flag = self.dateparse.parse(date_str) - - if not flag: # Oops, unparsable. - try: # Try and parse this as a single year - year = int(date_str) - return datetime(year, 1, 1) - except ValueError: - return None - except TypeError: - return None - - if flag is 1: # Date found, but no time. Use the default time. - date = datetime(*date[:3], hour=self.config['default_hour'], minute=self.config['default_minute']) - else: - date = datetime(*date[:6]) - - # Ugly heuristic: if the date is more than 4 weeks in the future, we got the year wrong. - # Rather then this, we would like to see parsedatetime patched so we can tell it to prefer - # past dates - dt = datetime.now() - date - if dt.days < -28: - date = date.replace(date.year - 1) - - return date - def new_entry(self, raw, date=None, sort=True): """Constructs a new entry from some raw text input. If a date is given, it will parse and use this, otherwise scan for a date in the input first.""" @@ -289,7 +248,7 @@ class Journal(object): if not date: if title.find(": ") > 0: starred = "*" in title[:title.find(": ")] - date = self.parse_date(title[:title.find(": ")]) + date = time.parse(title[:title.find(": ")], default_hour=self.config['default_hour'], default_minute=self.config['default_minute']) if date or starred: # Parsed successfully, strip that from the raw text title = title[title.find(": ")+1:].strip() elif title.strip().startswith("*"): @@ -299,7 +258,7 @@ class Journal(object): starred = True title = title[:-1].strip() if not date: # Still nothing? Meh, just live in the moment. - date = self.parse_date("now") + date = time.parse("now") entry = Entry.Entry(self, date, title, body, starred=starred) entry.modified = True self.entries.append(entry) diff --git a/jrnl/time.py b/jrnl/time.py new file mode 100644 index 00000000..a0a1a807 --- /dev/null +++ b/jrnl/time.py @@ -0,0 +1,49 @@ +from datetime import datetime +from dateutil.parser import parse as dateparse +try: import parsedatetime.parsedatetime_consts as pdt +except ImportError: import parsedatetime as pdt + +DEFAULT_FUTURE = datetime(datetime.now().year, 12, 31, 23, 59, 59) +DEFAULT_PAST = datetime(datetime.now().year, 1, 1, 0, 0) + +consts = pdt.Constants(usePyICU=False) +consts.DOWParseStyle = -1 # "Monday" will be either today or the last Monday +CALENDAR = pdt.Calendar(consts) + + +def parse(date_str, inclusive=False, default_hour=None, default_minute=None): + """Parses a string containing a fuzzy date and returns a datetime.datetime object""" + if not date_str: + return None + elif isinstance(date_str, datetime): + return date_str + + try: + date = dateparse(date_str, default=DEFAULT_FUTURE if inclusive else DEFAULT_PAST) + flag = 1 if date.hour == date.minute == 0 else 2 + date = date.timetuple() + except ValueError: + date, flag = CALENDAR.parse(date_str) + + if not flag: # Oops, unparsable. + try: # Try and parse this as a single year + year = int(date_str) + return datetime(year, 1, 1) + except ValueError: + return None + except TypeError: + return None + + if flag is 1: # Date found, but no time. Use the default time. + date = datetime(*date[:3], hour=default_hour or 0, minute=default_minute or 0) + else: + date = datetime(*date[:6]) + + # Ugly heuristic: if the date is more than 4 weeks in the future, we got the year wrong. + # Rather then this, we would like to see parsedatetime patched so we can tell it to prefer + # past dates + dt = datetime.now() - date + if dt.days < -28: + date = date.replace(date.year - 1) + + return date diff --git a/jrnl/util.py b/jrnl/util.py index 166a0023..b06113c2 100644 --- a/jrnl/util.py +++ b/jrnl/util.py @@ -2,10 +2,8 @@ # encoding: utf-8 import sys import os -from tzlocal import get_localzone import getpass as gp import keyring -import pytz import json if "win32" in sys.platform: import colorama @@ -24,12 +22,14 @@ STDOUT = sys.stdout TEST = False __cached_tz = None + def getpass(prompt="Password: "): if not TEST: return gp.getpass(prompt) else: return py23_input(prompt) + def get_password(validator, keychain=None, max_attempts=3): pwd_from_keychain = keychain and get_keychain(keychain) password = pwd_from_keychain or getpass() @@ -150,4 +150,3 @@ def byte2int(b): """Converts a byte to an integer. This is equivalent to ord(bs[0]) on Python 2 and bs[0] on Python 3.""" return ord(b)if PY2 else b - diff --git a/setup.py b/setup.py index 72cf6f97..2a4f0f30 100644 --- a/setup.py +++ b/setup.py @@ -84,7 +84,7 @@ setup( "six>=1.6.1", "tzlocal>=1.1", "keyring>=3.3", - "python-dateutil>=2.2" + "python-dateutil==1.5" ] + [p for p, cond in conditional_dependencies.items() if cond], extras_require = { "encrypted": "pycrypto>=2.6" From 6a16edb67b18bea020f6b6a90156b7bdc83ddb08 Mon Sep 17 00:00:00 2001 From: Manuel Ebert Date: Mon, 21 Jul 2014 13:30:25 +0900 Subject: [PATCH 2/6] Introduces `-on` option Fixes #246 --- jrnl/cli.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/jrnl/cli.py b/jrnl/cli.py index ceb37c57..af60bbe4 100644 --- a/jrnl/cli.py +++ b/jrnl/cli.py @@ -34,6 +34,7 @@ def parse_args(args=None): reading = parser.add_argument_group('Reading', 'Specifying either of these parameters will display posts of your journal') reading.add_argument('-from', dest='start_date', metavar="DATE", help='View entries after this date') reading.add_argument('-until', '-to', dest='end_date', metavar="DATE", help='View entries before this date') + reading.add_argument('-on', dest='on_date', metavar="DATE", help='View entries on this date') reading.add_argument('-and', dest='strict', action="store_true", help='Filter by tags using AND (default: OR)') reading.add_argument('-starred', dest='starred', action="store_true", help='Show only starred entries') reading.add_argument('-n', dest='limit', default=None, metavar="N", help="Shows the last n entries matching the filter. '-n 3' and '-3' have the same effect.", nargs="?", type=int) @@ -41,7 +42,7 @@ def parse_args(args=None): exporting = parser.add_argument_group('Export / Import', 'Options for transmogrifying your journal') exporting.add_argument('--short', dest='short', action="store_true", help='Show only titles or line containing the search tags') exporting.add_argument('--tags', dest='tags', action="store_true", help='Returns a list of all tags and number of occurences') - exporting.add_argument('--export', metavar='TYPE', dest='export', choices=['text','txt','markdown','md','json'], help='Export your journal. TYPE can be json, markdown, or text.', default=False, const=None) + exporting.add_argument('--export', metavar='TYPE', dest='export', choices=['text', 'txt', 'markdown', 'md', 'json'], help='Export your journal. TYPE can be json, markdown, or text.', default=False, const=None) exporting.add_argument('-o', metavar='OUTPUT', dest='output', help='Optionally specifies output file when using --export. If OUTPUT is a directory, exports each entry into an individual file instead.', default=False, const=None) exporting.add_argument('--encrypt', metavar='FILENAME', dest='encrypt', help='Encrypts your existing journal with a new password', nargs='?', default=False, const=None) exporting.add_argument('--decrypt', metavar='FILENAME', dest='decrypt', help='Decrypts your journal and stores it in plain text', nargs='?', default=False, const=None) @@ -57,7 +58,7 @@ def guess_mode(args, config): if args.decrypt is not False or args.encrypt is not False or args.export is not False or any((args.short, args.tags, args.edit)): compose = False export = True - elif any((args.start_date, args.end_date, args.limit, args.strict, args.starred)): + elif any((args.start_date, args.end_date, args.on_date, args.limit, args.strict, args.starred)): # Any sign of displaying stuff? compose = False elif args.text and all(word[0] in config['tagsymbols'] for word in u" ".join(args.text).split()): @@ -203,6 +204,8 @@ def run(manual_args=None): journal.write() else: old_entries = journal.entries + if args.on_date: + args.start_date = args.end_date = args.on_date journal.filter(tags=args.text, start_date=args.start_date, end_date=args.end_date, strict=args.strict, From 1822f50c7073a76ae7d36971ddad1e291b8beba7 Mon Sep 17 00:00:00 2001 From: Manuel Ebert Date: Mon, 21 Jul 2014 13:30:36 +0900 Subject: [PATCH 3/6] Tests for date parsing and `-on` --- features/core.feature | 7 +++++++ features/steps/core.py | 6 ++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/features/core.feature b/features/core.feature index 61a6bf57..3a9b9c7c 100644 --- a/features/core.feature +++ b/features/core.feature @@ -20,6 +20,13 @@ Feature: Basic reading and writing to a journal When we run "jrnl -n 1" Then the output should contain "2013-07-23 09:00 A cold and stormy day." + Scenario: Filtering for dates + Given we use the config "basic.json" + When we run "jrnl -on 2013-06-10 --short" + Then the output should be "2013-06-10 15:40 Life is good." + When we run "jrnl -on 'june 6 2013' --short" + Then the output should be "2013-06-10 15:40 Life is good." + Scenario: Emoji support Given we use the config "basic.json" When we run "jrnl 23 july 2013: 🌞 sunny day. Saw an 🐘" diff --git a/features/steps/core.py b/features/steps/core.py index cdde7613..c4aa2f59 100644 --- a/features/steps/core.py +++ b/features/steps/core.py @@ -110,9 +110,11 @@ def check_output_field_key(context, field, key): assert field in out_json assert key in out_json[field] + @then('the output should be') -def check_output(context): - text = context.text.strip().splitlines() +@then('the output should be "{text}"') +def check_output(context, text=None): + text = (text or context.text).strip().splitlines() out = context.stdout_capture.getvalue().strip().splitlines() assert len(text) == len(out), "Output has {} lines (expected: {})".format(len(out), len(text)) for line_text, line_out in zip(text, out): From 7f7f00e91bca40ede49c8c8b1cedd44986bc166d Mon Sep 17 00:00:00 2001 From: Manuel Ebert Date: Mon, 21 Jul 2014 13:30:39 +0900 Subject: [PATCH 4/6] Version bump --- CHANGELOG.md | 4 ++++ jrnl/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3629c27..c8c12a17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ Changelog ========= +### 1.9 (July 21, 2014) + +* __1.9.0__ Improved: Greatly improved date parsing. Also added an `-on` option for filtering + ### 1.8 (May 22, 2014) * __1.8.7__ Fixed: -from and -to filters are inclusive (thanks to @grplyler) diff --git a/jrnl/__init__.py b/jrnl/__init__.py index 516b60f5..b68d8ee8 100644 --- a/jrnl/__init__.py +++ b/jrnl/__init__.py @@ -8,7 +8,7 @@ jrnl is a simple journal application for your command line. from __future__ import absolute_import __title__ = 'jrnl' -__version__ = '1.8.7' +__version__ = '1.9.0' __author__ = 'Manuel Ebert' __license__ = 'MIT License' __copyright__ = 'Copyright 2013 - 2014 Manuel Ebert' From 316374922b2b1862022fa6e638f262f7666a75bc Mon Sep 17 00:00:00 2001 From: Manuel Ebert Date: Mon, 21 Jul 2014 14:22:07 +0900 Subject: [PATCH 5/6] Fix bug when omitting day in parsing --- jrnl/time.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/jrnl/time.py b/jrnl/time.py index a0a1a807..41503517 100644 --- a/jrnl/time.py +++ b/jrnl/time.py @@ -18,12 +18,19 @@ def parse(date_str, inclusive=False, default_hour=None, default_minute=None): elif isinstance(date_str, datetime): return date_str - try: - date = dateparse(date_str, default=DEFAULT_FUTURE if inclusive else DEFAULT_PAST) - flag = 1 if date.hour == date.minute == 0 else 2 - date = date.timetuple() - except ValueError: - date, flag = CALENDAR.parse(date_str) + default_date = DEFAULT_FUTURE if inclusive else DEFAULT_PAST + date = None + while not date: + try: + date = dateparse(date_str, default=default_date) + flag = 1 if date.hour == date.minute == 0 else 2 + date = date.timetuple() + except Exception as e: + if e.args[0] == 'day is out of range for month': + y, m, d, H, M, S = default_date.timetuple()[:6] + default_date = datetime(y, m, d - 1, H, M, S) + else: + date, flag = CALENDAR.parse(date_str) if not flag: # Oops, unparsable. try: # Try and parse this as a single year @@ -45,5 +52,4 @@ def parse(date_str, inclusive=False, default_hour=None, default_minute=None): dt = datetime.now() - date if dt.days < -28: date = date.replace(date.year - 1) - return date From 5799be7f1ced7e31054caa8ba3793ca61cf33e40 Mon Sep 17 00:00:00 2001 From: Manuel Ebert Date: Mon, 21 Jul 2014 14:22:16 +0900 Subject: [PATCH 6/6] Conditional dateutil for python 3 --- setup.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 2a4f0f30..f0015be2 100644 --- a/setup.py +++ b/setup.py @@ -69,7 +69,9 @@ conditional_dependencies = { "pyreadline>=2.0": not readline_available and "win32" in sys.platform, "readline>=6.2": not readline_available and "win32" not in sys.platform, "colorama>=0.2.5": "win32" in sys.platform, - "argparse>=1.1.0": sys.version.startswith("2.6") + "argparse>=1.1.0": sys.version.startswith("2.6"), + "python-dateutil==1.5": sys.version.startswith("2."), + "python-dateutil>=2.2": sys.version.startswith("3."), } @@ -84,7 +86,6 @@ setup( "six>=1.6.1", "tzlocal>=1.1", "keyring>=3.3", - "python-dateutil==1.5" ] + [p for p, cond in conditional_dependencies.items() if cond], extras_require = { "encrypted": "pycrypto>=2.6"