diff --git a/features/data/configs/empty_folder.json b/features/data/configs/empty_folder.json
new file mode 100644
index 00000000..d270db03
--- /dev/null
+++ b/features/data/configs/empty_folder.json
@@ -0,0 +1,14 @@
+{
+  "default_hour": 9,
+  "timeformat": "%Y-%m-%d %H:%M",
+  "linewrap": 80,
+  "encrypt": false,
+  "editor": "",
+  "default_minute": 0,
+  "highlight": true,
+  "password": "",
+  "journals": {
+    "default": "features/journals/empty_folder"
+  },
+  "tagsymbols": "@"
+}
diff --git a/features/environment.py b/features/environment.py
index c5b96721..bbf59226 100644
--- a/features/environment.py
+++ b/features/environment.py
@@ -12,6 +12,14 @@ def before_scenario(context, scenario):
     context.messages = StringIO()
     jrnl.util.STDERR = context.messages
     jrnl.util.TEST = True
+
+    # Clean up in case something went wrong
+    for folder in ("configs", "journals"):
+        working_dir = os.path.join("features", folder)
+        if os.path.exists(working_dir):
+            shutil.rmtree(working_dir)
+
+
     for folder in ("configs", "journals"):
         original = os.path.join("features", "data", folder)
         working_dir = os.path.join("features", folder)
@@ -30,4 +38,5 @@ def after_scenario(context, scenario):
     context.messages = None
     for folder in ("configs", "journals"):
         working_dir = os.path.join("features", folder)
-        shutil.rmtree(working_dir)
+        if os.path.exists(working_dir):
+            shutil.rmtree(working_dir)
diff --git a/features/regression.feature b/features/regression.feature
index c4cf41e7..da0ef277 100644
--- a/features/regression.feature
+++ b/features/regression.feature
@@ -1,8 +1,15 @@
 Feature: Zapped bugs should stay dead.
 
     Scenario: Writing an entry does not print the entire journal
+        # https://github.com/maebert/jrnl/issues/87
         Given we use the config "basic.json"
         When we run "jrnl 23 july 2013: A cold and stormy day. I ate crisps on the sofa."
         Then we should see the message "Entry added"
         When we run "jrnl -n 1"
         Then the output should not contain "Life is good"
+
+    Scenario: Opening an folder that's not a DayOne folder gives a nice error message
+        Given we use the config "empty_folder.json"
+        When we run "jrnl Herro"
+        Then we should get an error
+        Then we should see the message "is a directory, but doesn't seem to be a DayOne journal either"
diff --git a/features/steps/core.py b/features/steps/core.py
index 66e6a07f..d805e12d 100644
--- a/features/steps/core.py
+++ b/features/steps/core.py
@@ -53,16 +53,28 @@ def run_with_input(context, command, inputs=None):
     args = _parse_args(command)
     buffer = StringIO(text.strip())
     jrnl.util.STDIN = buffer
-    jrnl.cli(args)
+    try:
+        jrnl.cli(args or None)
+        context.exit_status = 0
+    except SystemExit as e:
+        context.exit_status = e.code
 
 @when('we run "{command}"')
 def run(context, command):
     args = _parse_args(command)
-    jrnl.cli(args or None)
+    try:
+        jrnl.cli(args or None)
+        context.exit_status = 0
+    except SystemExit as e:
+        context.exit_status = e.code
 
+@then('we should get an error')
+def has_error(context):
+    assert context.exit_status is not 0
 
 @then('we should get no error')
 def no_error(context):
+    assert context.exit_status is 0
     assert context.failed is False
 
 @then('the output should be parsable as json')
@@ -121,7 +133,7 @@ def check_output_not_inline(context, text):
 @then('we should see the message "{text}"')
 def check_message(context, text):
     out = context.messages.getvalue()
-    assert text in out
+    assert text in out, [text, out]
 
 @then('the journal should contain "{text}"')
 @then('journal "{journal_name}" should contain "{text}"')