diff --git a/libfprint/fpi-ssm.c b/libfprint/fpi-ssm.c
index 6e56e448..3cc39a7f 100644
--- a/libfprint/fpi-ssm.c
+++ b/libfprint/fpi-ssm.c
@@ -338,7 +338,10 @@ fpi_ssm_start_subsm (FpiSsm *parent, FpiSsm *child)
 {
   BUG_ON (parent->timeout);
   child->parentsm = parent;
-  g_clear_pointer (&parent->timeout, g_source_destroy);
+
+  fpi_ssm_clear_delayed_action (parent);
+  fpi_ssm_clear_delayed_action (child);
+
   fpi_ssm_start (child, __subsm_complete);
 }