# robots.txt file # # allow all spiders # disallow /admin/ User-agent: * Disallow: /admin/ Disallow: /clients/ Disallow: /engineers/ Disallow: /shopping-cart/