diff --git a/src/peft/tuners/tuners_utils.py b/src/peft/tuners/tuners_utils.py
index 7f0b1a77ba..75711b127a 100644
--- a/src/peft/tuners/tuners_utils.py
+++ b/src/peft/tuners/tuners_utils.py
@@ -530,10 +530,9 @@ def inject_adapter(
             and (len(peft_config.target_modules) >= MIN_TARGET_MODULES_FOR_OPTIMIZATION)
             and (peft_config.peft_type != PeftType.IA3)
         ):
+            suffixes = tuple("." + suffix for suffix in peft_config.target_modules)
             names_no_target = [
-                name
-                for name in key_list
-                if not any((name == suffix) or name.endswith("." + suffix) for suffix in peft_config.target_modules)
+                name for name in key_list if (name not in peft_config.target_modules) and not name.endswith(suffixes)
             ]
             new_target_modules = _find_minimal_target_modules(peft_config.target_modules, names_no_target)
             if len(new_target_modules) < len(peft_config.target_modules):
@@ -543,10 +542,10 @@ def inject_adapter(
         # MATCHING & CREATING MODULES #
         ###############################
 
-        existing_adapter_map = {}
+        existing_adapter_prefixes = []
         for key, module in named_modules:
             if isinstance(module, BaseTunerLayer):
-                existing_adapter_map[key] = module
+                existing_adapter_prefixes.append(key + ".")
 
         # TODO: check if this the most robust way
         module_names: set[str] = set()
@@ -560,8 +559,8 @@ def inject_adapter(
 
             # It is possible that we're adding an additional adapter, so if we encounter a key that clearly belongs to a
             # previous adapter we can skip here since we don't want to interfere with adapter internals.
-            for adapter_key in existing_adapter_map:
-                if key.startswith(adapter_key + "."):
+            for adapter_key in existing_adapter_prefixes:
+                if key.startswith(adapter_key):
                     excluded_modules.append(key)
                     break