vga/igfx: fix integer overflow in datam calculation (from qu7uux)

data[mn] and link[mn] are 24-bit values.
in the expression 'm = (n * ((freq * bpp)/8)) / (lsclk * lanes)',
uvlongs are used to prevent integer overflow, but since freq, bpp, lsclk and
lanes are all ints, the cast to uvlong does not happen until it's too late,
getting a wrong value.
instead, use u32int for m and n, and use casts where necessary.

example of bad calculation:
freq = 141400000
lsclk = 270000000
lanes = 2
bpp = 18
→ 0x7f3ee1ca6 (correct value: 0x4b69d0)

1 files changed, 3 insertions, 3 deletions

diff --git a/sys/src/cmd/aux/vga/igfx.c b/sys/src/cmd/aux/vga/igfx.c
index d39b92db4..9ea9e1442 100644
--- a/sys/src/cmd/aux/vga/igfx.c
+++ b/sys/src/cmd/aux/vga/igfx.c
@@ -692,16 +692,16 @@ initdpll(Igfx *igfx, int x, int freq, int port)
 static void
 initdatalinkmn(Trans *t, int freq, int lsclk, int lanes, int tu, int bpp)
 {
-	uvlong m, n;
+	u32int m, n;
 
 	n = 0x800000;
-	m = (n * ((freq * bpp)/8)) / (lsclk * lanes);
+	m = (n * (((uvlong)freq * bpp)/8)) / ((uvlong)lsclk * lanes);
 
 	t->dm[0].v = (tu-1)<<25 | m;
 	t->dn[0].v = n;
 
 	n = 0x80000;
-	m = (n * freq) / lsclk;
+	m = ((uvlong)n * freq) / lsclk;
 
 	t->lm[0].v = m;
 	t->ln[0].v = n;