Můžete to udělat pomocí proměnných.
select t.*
from (select t.*,
(@rn := if(@s = subject, @rn + 1,
if(@s := subject, 1, 1)
)
) as rn
from t cross join
(select @rn := 0, @s := '') params
order by subject, marks desc
) t
where rn <= 3
order by t.subject, t.rn;