diff --git a/.changeset/yellow-cats-teach.md b/.changeset/yellow-cats-teach.md
new file mode 100644
index 000000000..8b7f50acf
--- /dev/null
+++ b/.changeset/yellow-cats-teach.md
@@ -0,0 +1,6 @@
+---
+"@tiptap/extension-font-family": minor
+"@tiptap/extension-text-style": minor
+---
+
+fix: #4311 - update the logic of removeEmptyTextStyle to manually handle the selection of all of the nodes within the selection to check for their marks independently to fix an issue where unsetting the font family on a selection would remove all applied text style marks from the selection as well 
diff --git a/packages/extension-font-family/src/font-family.ts b/packages/extension-font-family/src/font-family.ts
index ac57f521c..208ce15af 100644
--- a/packages/extension-font-family/src/font-family.ts
+++ b/packages/extension-font-family/src/font-family.ts
@@ -72,9 +72,15 @@ export const FontFamily = Extension.create<FontFamilyOptions>({
           .setMark('textStyle', { fontFamily })
           .run()
       },
-      unsetFontFamily: () => ({ chain }) => {
+      unsetFontFamily: () => ({ chain, tr }) => {
+        const { selection } = tr
+
+        // Retrieve all previous mark attributes applied to the current selection
+        const previousMarkAttributes = selection.$anchor.marks().map(mark => mark.attrs)
+
         return chain()
-          .setMark('textStyle', { fontFamily: null })
+          // Only remove the font family attribute from the previous mark attributes
+          .setMark('textStyle', { ...previousMarkAttributes, fontFamily: null })
           .removeEmptyTextStyle()
           .run()
       },
diff --git a/packages/extension-text-style/src/text-style.ts b/packages/extension-text-style/src/text-style.ts
index 80c0da06b..6c1bb7a1d 100644
--- a/packages/extension-text-style/src/text-style.ts
+++ b/packages/extension-text-style/src/text-style.ts
@@ -1,5 +1,4 @@
 import {
-  getMarkAttributes,
   Mark,
   mergeAttributes,
 } from '@tiptap/core'
@@ -64,15 +63,32 @@ export const TextStyle = Mark.create<TextStyleOptions>({
 
   addCommands() {
     return {
-      removeEmptyTextStyle: () => ({ state, commands }) => {
-        const attributes = getMarkAttributes(state, this.type)
-        const hasStyles = Object.entries(attributes).some(([, value]) => !!value)
+      removeEmptyTextStyle: () => ({ tr }) => {
 
-        if (hasStyles) {
-          return true
-        }
+        const { selection } = tr
 
-        return commands.unsetMark(this.name)
+        // Gather all of the nodes within the selection range.
+        // We would need to go through each node individually
+        // to check if it has any inline style attributes.
+        // Otherwise, calling commands.unsetMark(this.name)
+        // removes everything from all the nodes
+        // within the selection range.
+        tr.doc.nodesBetween(selection.from, selection.to, (node, pos) => {
+
+          // Check if it's a paragraph element, if so, skip this node as we apply
+          // the text style to inline text nodes only (span).
+          if (node.isTextblock) {
+            return true
+          }
+
+          // Check if the node has no inline style attributes.
+          if (!node.marks.some(mark => Object.values(mark.attrs).some(value => !!value))) {
+            // Proceed with the removal of the `textStyle` mark for this node only
+            tr.removeMark(pos, pos + node.nodeSize, this.type)
+          }
+        })
+
+        return true
       },
     }
   },