Skupinu hledáte podle ' ' klauzule, nikoli vnořené výběry!
SELECT tag.name, tag.id, COUNT(tag_map.tag_id) as cnt
FROM tag
INNER JOIN tag_map
ON (tag.id = tag_map.tag_id)
INNER JOIN image
ON tag_map.image_id = image.id
INNER JOIN imagefile
on image.id = imagefile.image_id
WHERE imagefile.type = 'smallthumb'
GROUP BY tag.name HAVING COUNT(tag_map.tag_id) > 0
ORDER BY cnt DESC