diff --git a/src/common/settings.h b/src/common/settings.h
index b52d0d1d0f9c6862051227edee53b959b50dae92..ee9e0b5a15f4d28451c3fb9d69a4fa548c0241f2 100644
--- a/src/common/settings.h
+++ b/src/common/settings.h
@@ -572,8 +572,6 @@ struct Values {
 
     BasicSetting<bool> emulate_analog_keyboard{false, "emulate_analog_keyboard"};
     BasicSetting<bool> keyboard_enabled{false, "keyboard_enabled"};
-    KeyboardKeysRaw keyboard_keys;
-    KeyboardModsRaw keyboard_mods;
 
     BasicSetting<bool> debug_pad_enabled{false, "debug_pad_enabled"};
     ButtonsRaw debug_pad_buttons;
diff --git a/src/core/hid/emulated_devices.cpp b/src/core/hid/emulated_devices.cpp
index 0d840a00352baeb7e61e7cf5ed1e46abbe4db227..45e0bd80dbe414b7a7ad7642c8148bdd4f861d97 100644
--- a/src/core/hid/emulated_devices.cpp
+++ b/src/core/hid/emulated_devices.cpp
@@ -29,13 +29,29 @@ void EmulatedDevices::ReloadInput() {
                    mouse_button_devices.begin(),
                    Common::Input::CreateDevice<Common::Input::InputDevice>);
 
-    std::transform(Settings::values.keyboard_keys.begin(), Settings::values.keyboard_keys.end(),
-                   keyboard_devices.begin(),
-                   Common::Input::CreateDeviceFromString<Common::Input::InputDevice>);
+    std::size_t key_index = 0;
+    for (auto& keyboard_device : keyboard_devices) {
+        // Keyboard keys are only mapped on port 1, pad 0
+        Common::ParamPackage keyboard_params;
+        keyboard_params.Set("engine", "keyboard");
+        keyboard_params.Set("button", static_cast<int>(key_index));
+        keyboard_params.Set("port", 1);
+        keyboard_params.Set("pad", 0);
+        keyboard_device = Common::Input::CreateDevice<Common::Input::InputDevice>(keyboard_params);
+        key_index++;
+    }
 
-    std::transform(Settings::values.keyboard_mods.begin(), Settings::values.keyboard_mods.end(),
-                   keyboard_modifier_devices.begin(),
-                   Common::Input::CreateDeviceFromString<Common::Input::InputDevice>);
+    key_index = 0;
+    for (auto& keyboard_device : keyboard_modifier_devices) {
+        // Keyboard moddifiers are only mapped on port 1, pad 1
+        Common::ParamPackage keyboard_params;
+        keyboard_params.Set("engine", "keyboard");
+        keyboard_params.Set("button", static_cast<int>(key_index));
+        keyboard_params.Set("port", 1);
+        keyboard_params.Set("pad", 1);
+        keyboard_device = Common::Input::CreateDevice<Common::Input::InputDevice>(keyboard_params);
+        key_index++;
+    }
 
     for (std::size_t index = 0; index < mouse_button_devices.size(); ++index) {
         if (!mouse_button_devices[index]) {
diff --git a/src/input_common/drivers/keyboard.cpp b/src/input_common/drivers/keyboard.cpp
index 328fe1ac16756abea0accf7d29a886cc6654ca5c..23b0c0ccf311ab18b6225ea687a4f49ae21c96bf 100644
--- a/src/input_common/drivers/keyboard.cpp
+++ b/src/input_common/drivers/keyboard.cpp
@@ -13,15 +13,26 @@ constexpr PadIdentifier key_identifier = {
     .port = 0,
     .pad = 0,
 };
-constexpr PadIdentifier modifier_identifier = {
+constexpr PadIdentifier keyboard_key_identifier = {
     .guid = Common::UUID{Common::INVALID_UUID},
-    .port = 0,
+    .port = 1,
+    .pad = 0,
+};
+constexpr PadIdentifier keyboard_modifier_identifier = {
+    .guid = Common::UUID{Common::INVALID_UUID},
+    .port = 1,
     .pad = 1,
 };
 
 Keyboard::Keyboard(const std::string& input_engine_) : InputEngine(input_engine_) {
+    // Keyboard is broken into 3 diferent sets:
+    // key: Unfiltered intended for controllers.
+    // keyboard_key: Allows only Settings::NativeKeyboard::Keys intended for keyboard emulation.
+    // keyboard_modifier: Allows only Settings::NativeKeyboard::Modifiers intended for keyboard
+    // emulation.
     PreSetController(key_identifier);
-    PreSetController(modifier_identifier);
+    PreSetController(keyboard_key_identifier);
+    PreSetController(keyboard_modifier_identifier);
 }
 
 void Keyboard::PressKey(int key_code) {
@@ -32,35 +43,50 @@ void Keyboard::ReleaseKey(int key_code) {
     SetButton(key_identifier, key_code, false);
 }
 
-void Keyboard::SetModifiers(int key_modifiers) {
+void Keyboard::PressKeyboardKey(int key_index) {
+    if (key_index == Settings::NativeKeyboard::None) {
+        return;
+    }
+    SetButton(keyboard_key_identifier, key_index, true);
+}
+
+void Keyboard::ReleaseKeyboardKey(int key_index) {
+    if (key_index == Settings::NativeKeyboard::None) {
+        return;
+    }
+    SetButton(keyboard_key_identifier, key_index, false);
+}
+
+void Keyboard::SetKeyboardModifiers(int key_modifiers) {
     for (int i = 0; i < 32; ++i) {
         bool key_value = ((key_modifiers >> i) & 0x1) != 0;
-        SetButton(modifier_identifier, i, key_value);
+        SetButton(keyboard_modifier_identifier, i, key_value);
         // Use the modifier to press the key button equivalent
         switch (i) {
         case Settings::NativeKeyboard::LeftControl:
-            SetButton(key_identifier, Settings::NativeKeyboard::LeftControlKey, key_value);
+            SetButton(keyboard_key_identifier, Settings::NativeKeyboard::LeftControlKey, key_value);
             break;
         case Settings::NativeKeyboard::LeftShift:
-            SetButton(key_identifier, Settings::NativeKeyboard::LeftShiftKey, key_value);
+            SetButton(keyboard_key_identifier, Settings::NativeKeyboard::LeftShiftKey, key_value);
             break;
         case Settings::NativeKeyboard::LeftAlt:
-            SetButton(key_identifier, Settings::NativeKeyboard::LeftAltKey, key_value);
+            SetButton(keyboard_key_identifier, Settings::NativeKeyboard::LeftAltKey, key_value);
             break;
         case Settings::NativeKeyboard::LeftMeta:
-            SetButton(key_identifier, Settings::NativeKeyboard::LeftMetaKey, key_value);
+            SetButton(keyboard_key_identifier, Settings::NativeKeyboard::LeftMetaKey, key_value);
             break;
         case Settings::NativeKeyboard::RightControl:
-            SetButton(key_identifier, Settings::NativeKeyboard::RightControlKey, key_value);
+            SetButton(keyboard_key_identifier, Settings::NativeKeyboard::RightControlKey,
+                      key_value);
             break;
         case Settings::NativeKeyboard::RightShift:
-            SetButton(key_identifier, Settings::NativeKeyboard::RightShiftKey, key_value);
+            SetButton(keyboard_key_identifier, Settings::NativeKeyboard::RightShiftKey, key_value);
             break;
         case Settings::NativeKeyboard::RightAlt:
-            SetButton(key_identifier, Settings::NativeKeyboard::RightAltKey, key_value);
+            SetButton(keyboard_key_identifier, Settings::NativeKeyboard::RightAltKey, key_value);
             break;
         case Settings::NativeKeyboard::RightMeta:
-            SetButton(key_identifier, Settings::NativeKeyboard::RightMetaKey, key_value);
+            SetButton(keyboard_key_identifier, Settings::NativeKeyboard::RightMetaKey, key_value);
             break;
         default:
             // Other modifier keys should be pressed with PressKey since they stay enabled until
diff --git a/src/input_common/drivers/keyboard.h b/src/input_common/drivers/keyboard.h
index 2ab92fd6c29d81fab48c45b6ef6884e077790fc0..ad123b136808055010d2a7f20a0db3e24d9f4a47 100644
--- a/src/input_common/drivers/keyboard.h
+++ b/src/input_common/drivers/keyboard.h
@@ -28,11 +28,23 @@ public:
      */
     void ReleaseKey(int key_code);
 
+    /**
+     * Sets the status of the keyboard key to pressed
+     * @param key_index index of the key to press
+     */
+    void PressKeyboardKey(int key_index);
+
+    /**
+     * Sets the status of the keyboard key to released
+     * @param key_index index of the key to release
+     */
+    void ReleaseKeyboardKey(int key_index);
+
     /**
      * Sets the status of all keyboard modifier keys
      * @param key_modifiers the code of the key to release
      */
-    void SetModifiers(int key_modifiers);
+    void SetKeyboardModifiers(int key_modifiers);
 
     /// Sets all keys to the non pressed state
     void ReleaseAllKeys();
diff --git a/src/input_common/input_mapping.cpp b/src/input_common/input_mapping.cpp
index 0ffc710281afbbe489a2d47078b21d918e135cd7..0eeeff372524878475f54b8f92ed63c9aad850f7 100644
--- a/src/input_common/input_mapping.cpp
+++ b/src/input_common/input_mapping.cpp
@@ -28,6 +28,10 @@ void MappingFactory::RegisterInput(const MappingData& data) {
     if (!is_enabled) {
         return;
     }
+    if (!IsDriverValid(data)) {
+        return;
+    }
+
     switch (input_type) {
     case Polling::InputType::Button:
         RegisterButton(data);
@@ -168,4 +172,25 @@ void MappingFactory::RegisterMotion(const MappingData& data) {
     input_queue.Push(new_input);
 }
 
+bool MappingFactory::IsDriverValid(const MappingData& data) const {
+    // Only port 0 can be mapped on the keyboard
+    if (data.engine == "keyboard" && data.pad.port != 0) {
+        return false;
+    }
+    // The following drivers don't need to be mapped
+    if (data.engine == "tas") {
+        return false;
+    }
+    if (data.engine == "touch") {
+        return false;
+    }
+    if (data.engine == "touch_from_button") {
+        return false;
+    }
+    if (data.engine == "analog_from_button") {
+        return false;
+    }
+    return true;
+}
+
 } // namespace InputCommon
diff --git a/src/input_common/input_mapping.h b/src/input_common/input_mapping.h
index 2622dba70c9dc6a69d408c49b2778e1becbb4583..44eb8ad9a3692280d53c6630c7d3a9fee2c5097c 100644
--- a/src/input_common/input_mapping.h
+++ b/src/input_common/input_mapping.h
@@ -66,6 +66,13 @@ private:
      */
     void RegisterMotion(const MappingData& data);
 
+    /**
+     * Returns true if driver can be mapped
+     * @param "data": An struct containing all the information needed to create a proper
+     * ParamPackage
+     */
+    bool IsDriverValid(const MappingData& data) const;
+
     Common::SPSCQueue<Common::ParamPackage> input_queue;
     Polling::InputType input_type{Polling::InputType::None};
     bool is_enabled{};
diff --git a/src/input_common/main.cpp b/src/input_common/main.cpp
index ae2518f53c7aff56881ba98380082949e9f9c1c7..df36a337c7fa9009bea20ce1b066f3ee411a567f 100644
--- a/src/input_common/main.cpp
+++ b/src/input_common/main.cpp
@@ -402,15 +402,6 @@ std::string GenerateKeyboardParam(int key_code) {
     return param.Serialize();
 }
 
-std::string GenerateModdifierKeyboardParam(int key_code) {
-    Common::ParamPackage param;
-    param.Set("engine", "keyboard");
-    param.Set("code", key_code);
-    param.Set("toggle", false);
-    param.Set("pad", 1);
-    return param.Serialize();
-}
-
 std::string GenerateAnalogParamFromKeys(int key_up, int key_down, int key_left, int key_right,
                                         int key_modifier, float modifier_scale) {
     Common::ParamPackage circle_pad_param{
diff --git a/src/input_common/main.h b/src/input_common/main.h
index 9ea39546588b80fe2afe0d6a0a09b085bf9f8494..a4a24d076ed46ae897ce47017e88dd16e9a55835 100644
--- a/src/input_common/main.h
+++ b/src/input_common/main.h
@@ -134,9 +134,6 @@ private:
 /// Generates a serialized param package for creating a keyboard button device.
 std::string GenerateKeyboardParam(int key_code);
 
-/// Generates a serialized param package for creating a moddifier keyboard button device.
-std::string GenerateModdifierKeyboardParam(int key_code);
-
 /// Generates a serialized param package for creating an analog device taking input from keyboard.
 std::string GenerateAnalogParamFromKeys(int key_up, int key_down, int key_left, int key_right,
                                         int key_modifier, float modifier_scale);
diff --git a/src/yuzu/bootmanager.cpp b/src/yuzu/bootmanager.cpp
index 61513a5b41b9289c0c9a96853a6f678f29207f3b..9f4d1aac37633178018c2aa89c1cf40f0dd21ad3 100644
--- a/src/yuzu/bootmanager.cpp
+++ b/src/yuzu/bootmanager.cpp
@@ -609,7 +609,7 @@ int GRenderWindow::QtKeyToSwitchKey(Qt::Key qt_key) {
         return Settings::NativeKeyboard::ZenkakuHankaku;
     // Modifier keys are handled by the modifier property
     default:
-        return 0;
+        return Settings::NativeKeyboard::None;
     }
 }
 
@@ -662,8 +662,10 @@ void GRenderWindow::keyPressEvent(QKeyEvent* event) {
         // Replace event->key() with event->nativeVirtualKey() since the second one provides raw key
         // buttons
         const auto key = QtKeyToSwitchKey(Qt::Key(event->key()));
-        input_subsystem->GetKeyboard()->SetModifiers(moddifier);
-        input_subsystem->GetKeyboard()->PressKey(key);
+        input_subsystem->GetKeyboard()->SetKeyboardModifiers(moddifier);
+        input_subsystem->GetKeyboard()->PressKeyboardKey(key);
+        // This is used for gamepads
+        input_subsystem->GetKeyboard()->PressKey(event->key());
     }
 }
 
@@ -671,8 +673,10 @@ void GRenderWindow::keyReleaseEvent(QKeyEvent* event) {
     if (!event->isAutoRepeat()) {
         const auto moddifier = QtModifierToSwitchModdifier(event->nativeModifiers());
         const auto key = QtKeyToSwitchKey(Qt::Key(event->key()));
-        input_subsystem->GetKeyboard()->SetModifiers(moddifier);
-        input_subsystem->GetKeyboard()->ReleaseKey(key);
+        input_subsystem->GetKeyboard()->SetKeyboardModifiers(moddifier);
+        input_subsystem->GetKeyboard()->ReleaseKeyboardKey(key);
+        // This is used for gamepads
+        input_subsystem->GetKeyboard()->ReleaseKey(event->key());
     }
 }
 
diff --git a/src/yuzu/configuration/config.cpp b/src/yuzu/configuration/config.cpp
index ccf274895f7ce7a505368bcb05be58cef77d5ab6..5865359fe0355beb756d436d9653cdf9baa25534 100644
--- a/src/yuzu/configuration/config.cpp
+++ b/src/yuzu/configuration/config.cpp
@@ -344,15 +344,6 @@ void Config::ReadDebugValues() {
 
 void Config::ReadKeyboardValues() {
     ReadBasicSetting(Settings::values.keyboard_enabled);
-
-    for (std::size_t i = 0; i < Settings::values.keyboard_keys.size(); ++i) {
-        Settings::values.keyboard_keys[i] = InputCommon::GenerateKeyboardParam(static_cast<int>(i));
-    }
-
-    for (std::size_t i = 0; i < Settings::values.keyboard_mods.size(); ++i) {
-        Settings::values.keyboard_mods[i] =
-            InputCommon::GenerateModdifierKeyboardParam(static_cast<int>(i));
-    }
 }
 
 void Config::ReadMouseValues() {