User-agent: *
Crawl-Delay: 10
Disallow: /dump/
Disallow: *.htm
Disallow: *.html