video: fbdev: atari: Atari 2 bpp (STe) palette bugfix

commit c8be5edbd36ceed2ff3d6b8f8e40643c3f396ea3 upstream.

The code to set the shifter STe palette registers has a long
standing operator precedence bug, manifesting as colors set
on a 2 bits per pixel frame buffer coming up with a distinctive
blue tint.

Add parentheses around the calculation of the per-color palette
data before shifting those into their respective bit field position.

This bug goes back a long way (2.4 days at the very least) so there
won't be a Fixes: tag.

Tested on ARAnyM as well on Falcon030 hardware.

Cc: stable@vger.kernel.org
Reported-by: Geert Uytterhoeven <geert@linux-m68k.org>
Link: https://lore.kernel.org/all/CAMuHMdU3ievhXxKR_xi_v3aumnYW7UNUO6qMdhgfyWTyVSsCkQ@mail.gmail.com
Tested-by: Michael Schmitz <schmitzmic@gmail.com>
Tested-by: Geert Uytterhoeven <geert@linux-m68k.org>
Signed-off-by: Michael Schmitz <schmitzmic@gmail.com>
Signed-off-by: Helge Deller <deller@gmx.de>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
This commit is contained in:
Michael Schmitz 2022-02-16 20:26:25 +13:00 committed by Greg Kroah-Hartman
parent 478154be3a
commit 51186190c4

View File

@ -1692,8 +1692,8 @@ static int falcon_setcolreg(unsigned int regno, unsigned int red,
((blue & 0xfc00) >> 8));
if (regno < 16) {
shifter_tt.color_reg[regno] =
(((red & 0xe000) >> 13) | ((red & 0x1000) >> 12) << 8) |
(((green & 0xe000) >> 13) | ((green & 0x1000) >> 12) << 4) |
((((red & 0xe000) >> 13) | ((red & 0x1000) >> 12)) << 8) |
((((green & 0xe000) >> 13) | ((green & 0x1000) >> 12)) << 4) |
((blue & 0xe000) >> 13) | ((blue & 0x1000) >> 12);
((u32 *)info->pseudo_palette)[regno] = ((red & 0xf800) |
((green & 0xfc00) >> 5) |
@ -1980,8 +1980,8 @@ static int stste_setcolreg(unsigned int regno, unsigned int red,
green >>= 12;
if (ATARIHW_PRESENT(EXTD_SHIFTER))
shifter_tt.color_reg[regno] =
(((red & 0xe) >> 1) | ((red & 1) << 3) << 8) |
(((green & 0xe) >> 1) | ((green & 1) << 3) << 4) |
((((red & 0xe) >> 1) | ((red & 1) << 3)) << 8) |
((((green & 0xe) >> 1) | ((green & 1) << 3)) << 4) |
((blue & 0xe) >> 1) | ((blue & 1) << 3);
else
shifter_tt.color_reg[regno] =