User-agent: * Disallow: /404error/ Disallow: /css/ Disallow: /ssi/ Disallow: /images/ Disallow: /gc/ # This will prevent robots from entering the dev, homepage and test areas