diff --git a/src/syntax.c b/src/syntax.c
index 4e35a6c..5ed7f77 100644
--- a/src/syntax.c
+++ b/src/syntax.c
@@ -9261,8 +9261,11 @@ set_hl_attr(
 	at_en.ae_u.cterm.fg_color = sgp->sg_cterm_fg;
 	at_en.ae_u.cterm.bg_color = sgp->sg_cterm_bg;
 # ifdef FEAT_TERMGUICOLORS
-	at_en.ae_u.cterm.fg_rgb = GUI_MCH_GET_RGB(sgp->sg_gui_fg);
-	at_en.ae_u.cterm.bg_rgb = GUI_MCH_GET_RGB(sgp->sg_gui_bg);
+	if (!gui.in_use || sgp->sg_gui_fg != INVALCOLOR)
+	    at_en.ae_u.cterm.fg_rgb = GUI_MCH_GET_RGB(sgp->sg_gui_fg);
+
+	if (!gui.in_use || sgp->sg_gui_bg != INVALCOLOR)
+	    at_en.ae_u.cterm.bg_rgb = GUI_MCH_GET_RGB(sgp->sg_gui_bg);
 # endif
 	sgp->sg_cterm_attr = get_attr_entry(&cterm_attr_table, &at_en);
     }