diff --git a/libagent/device/ui.py b/libagent/device/ui.py
index 525c4e8..4006fec 100644
--- a/libagent/device/ui.py
+++ b/libagent/device/ui.py
@@ -23,6 +23,7 @@ class UI:
                                                   default_pinentry)
         self.options_getter = create_default_options_getter()
         self.device_name = device_type.__name__
+        self.cached_passphrase_ack = None
 
     def get_pin(self, _code=None):
         """Ask the user for (scrambled) PIN."""
@@ -41,12 +42,19 @@ class UI:
 
     def get_passphrase(self):
         """Ask the user for passphrase."""
-        return interact(
-            title='{} passphrase'.format(self.device_name),
-            prompt='Passphrase:',
-            description=None,
-            binary=self.passphrase_entry_binary,
-            options=self.options_getter())
+        passphrase = None
+        if self.cached_passphrase_ack:
+            passphrase = self.cached_passphrase_ack.get()
+        if passphrase is None:
+            passphrase = interact(
+                title='{} passphrase'.format(self.device_name),
+                prompt='Passphrase:',
+                description=None,
+                binary=self.passphrase_entry_binary,
+                options=self.options_getter())
+        if self.cached_passphrase_ack:
+            self.cached_passphrase_ack.set(passphrase)
+        return passphrase
 
     def button_request(self, _code=None):
         """Called by TrezorClient when device interaction is required."""
diff --git a/libagent/gpg/__init__.py b/libagent/gpg/__init__.py
index ee19836..0204fe4 100644
--- a/libagent/gpg/__init__.py
+++ b/libagent/gpg/__init__.py
@@ -249,7 +249,7 @@ def run_agent(device_type):
         pubkey_bytes = keyring.export_public_keys(env=env)
         device_type.ui = device.ui.UI(device_type=device_type,
                                       config=vars(args))
-        device_type.cached_passphrase_ack = util.ExpiringCache(
+        device_type.ui.cached_passphrase_ack = util.ExpiringCache(
             seconds=float(args.cache_expiry_seconds))
         handler = agent.Handler(device=device_type(),
                                 pubkey_bytes=pubkey_bytes)
@@ -318,7 +318,7 @@ def main(device_type):
 
     args = parser.parse_args()
     device_type.ui = device.ui.UI(device_type=device_type, config=vars(args))
-    device_type.cached_passphrase_ack = util.ExpiringCache(
+    device_type.ui.cached_passphrase_ack = util.ExpiringCache(
         seconds=float(args.cache_expiry_seconds))
 
     return args.func(device_type=device_type, args=args)
diff --git a/libagent/ssh/__init__.py b/libagent/ssh/__init__.py
index ce60a65..c73f78d 100644
--- a/libagent/ssh/__init__.py
+++ b/libagent/ssh/__init__.py
@@ -275,7 +275,7 @@ def main(device_type):
 
     # override default PIN/passphrase entry tools (relevant for TREZOR/Keepkey):
     device_type.ui = device.ui.UI(device_type=device_type, config=vars(args))
-    device_type.cached_passphrase_ack = util.ExpiringCache(
+    device_type.ui.cached_passphrase_ack = util.ExpiringCache(
         args.cache_expiry_seconds)
 
     conn = JustInTimeConnection(