diff --git a/lib/src/keypress_handler.rs b/lib/src/keypress_handler.rs
index 172e89b..7561918 100644
--- a/lib/src/keypress_handler.rs
+++ b/lib/src/keypress_handler.rs
@@ -167,6 +167,7 @@ pub async fn keypress_handler(
                 let position_in_queue = queue
                     .iter()
                     .enumerate()
+                    .filter(|(_, queued)| queued.source_half == event.source)
                     .find(|(_, queued)| queued.source_button == event.source_button)
                     .map(|(i, _)| i);