diff --git a/tests/bdd/features/format.feature b/tests/bdd/features/format.feature
index d4856949..18a65a92 100644
--- a/tests/bdd/features/format.feature
+++ b/tests/bdd/features/format.feature
@@ -584,12 +584,12 @@ Feature: Custom formats
         Given we use the config "format_md.yaml"
         And we create a cache directory
         When we run "jrnl longtitle --format markdown --file {cache_dir}"
-        Then the output should contain "Journal exported to"
+        Then the cache directory should contain 2 files
         And we should get no error
     
     Scenario: Export entries in text format with a title longer than max file name length.
         Given we use the config "format_text.yaml"
         And we create a cache directory
         When we run "jrnl longtitle --format text --file {cache_dir}"
-        Then the output should contain "Journal exported to"
+        Then the cache directory should contain 2 files
         And we should get no error
diff --git a/tests/lib/helpers.py b/tests/lib/helpers.py
index 39a24f71..347a5983 100644
--- a/tests/lib/helpers.py
+++ b/tests/lib/helpers.py
@@ -17,6 +17,18 @@ def does_directory_contain_files(file_list, directory_path):
     return True
 
 
+def does_directory_contain_n_files(directory_path, number):
+    count = 0
+    if not os.path.isdir(directory_path):
+        return False
+
+    for path in os.scandir(directory_path):
+        if path.is_file():
+            count += 1
+
+    return int(number) == count
+
+
 def parse_should_or_should_not(should_or_should_not):
     if should_or_should_not == "should":
         return True
diff --git a/tests/lib/then_steps.py b/tests/lib/then_steps.py
index 4efde7a5..51b145d1 100644
--- a/tests/lib/then_steps.py
+++ b/tests/lib/then_steps.py
@@ -13,6 +13,7 @@ from ruamel.yaml import YAML
 from jrnl.config import scope_config
 from tests.lib.helpers import assert_equal_tags_ignoring_order
 from tests.lib.helpers import does_directory_contain_files
+from tests.lib.helpers import does_directory_contain_n_files
 from tests.lib.helpers import get_nested_val
 from tests.lib.helpers import parse_should_or_should_not
 
@@ -201,6 +202,11 @@ def assert_dir_contains_files(file_list, cache_dir):
     assert does_directory_contain_files(file_list, cache_dir["path"])
 
 
+@then(parse("the cache directory should contain {number} files"))
+def assert_dir_contains_n_files(cache_dir, number):
+    assert does_directory_contain_n_files(cache_dir["path"], number)
+
+
 @then(parse("the journal directory should contain\n{file_list}"))
 def journal_directory_should_contain(config_on_disk, file_list):
     scoped_config = scope_config(config_on_disk, "default")