# /robots.txt # this covers most common folder names # adjust as needed per site # User-agent: Baiduspider Disallow: / # User-agent: * Disallow: /library/ Disallow: /images/ Disallow: /common/ Disallow: /admin/ Disallow: /download/ Disallow: /photos/ Disallow: /navigation/ Disallow: /admin/