diff --git a/libagent/device/ui.py b/libagent/device/ui.py
index 2dfb8ed..baf0ecf 100644
--- a/libagent/device/ui.py
+++ b/libagent/device/ui.py
@@ -28,7 +28,8 @@ class UI:
                                                   default_pinentry)
         self.options_getter = create_default_options_getter()
         self.device_name = device_type.__name__
-        self.cached_passphrase_ack = None
+        self.cached_passphrase_ack = util.ExpiringCache(
+            seconds=float(config.get('cache_expiry_seconds', 'inf')))
 
     def get_pin(self, _code=None):
         """Ask the user for (scrambled) PIN."""
diff --git a/libagent/gpg/__init__.py b/libagent/gpg/__init__.py
index f6e263c..79f6019 100644
--- a/libagent/gpg/__init__.py
+++ b/libagent/gpg/__init__.py
@@ -249,8 +249,6 @@ def run_agent(device_type):
         pubkey_bytes = keyring.export_public_keys(env=env)
         device_type.ui = device.ui.UI(device_type=device_type,
                                       config=vars(args))
-        device_type.ui.cached_passphrase_ack = util.ExpiringCache(
-            seconds=float(args.cache_expiry_seconds))
         handler = agent.Handler(device=device_type(),
                                 pubkey_bytes=pubkey_bytes)
 
@@ -318,7 +316,5 @@ def main(device_type):
 
     args = parser.parse_args()
     device_type.ui = device.ui.UI(device_type=device_type, config=vars(args))
-    device_type.ui.cached_passphrase_ack = util.ExpiringCache(
-        seconds=float(args.cache_expiry_seconds))
 
     return args.func(device_type=device_type, args=args)
diff --git a/libagent/ssh/__init__.py b/libagent/ssh/__init__.py
index 1933720..10cc029 100644
--- a/libagent/ssh/__init__.py
+++ b/libagent/ssh/__init__.py
@@ -274,8 +274,6 @@ def main(device_type):
 
     # override default PIN/passphrase entry tools (relevant for TREZOR/Keepkey):
     device_type.ui = device.ui.UI(device_type=device_type, config=vars(args))
-    device_type.ui.cached_passphrase_ack = util.ExpiringCache(
-        args.cache_expiry_seconds)
 
     conn = JustInTimeConnection(
         conn_factory=lambda: client.Client(device_type()),