## Disallow crawlers first. and then it will crawl the rest.

## Example
# Block all crawlers for /admin
User-agent: *
Disallow: /admin

#Block all crawlers for /accounts
User-agent: *
Disallow: /accounts

#Block all crawlers for /shop/**
User-agent: *
Disallow: /shop/*/**

# Allow all crawlers
User-agent: *
Allow: /