Sitemap: <http://www.littledane.com/sitemap.xml>

# Block all robots from tmp and logs directories
User-agent: *
Disallow: /tmp/
Disallow: /logs # for directories and files called logs
Noindex: /logs

# Block all robots from .php and .js and .csv and .json files
User-agent: *
Disallow: /*.php$
Disallow: /*.js$
Disallow: /*.json$
Disallow: /*.csv$

Noindex: /*.php$
Noindex: /*.js$
Noindex: /*.json$
Noindex: /*.csv$


# Block all robots from specific .xml files
User-agent: *
Disallow: /blanketdetail.xml
Disallow: /customertable.xml
Disallow: /retailstoreaddress.csv
Disallow: /retailstoreaddress.csv.
Disallow: /retailaddress.xml
Disallow: /retailaddress.xsd
Disallow: /retailaddress.xsl
Noindex: /blanketdetail.xml
Noindex: /customertable.xml
Noindex: /retailaddress.xml
Noindex: /retailaddress.xsd
Noindex: /retailaddress.xsl
Noindex: /retailstoreaddress.csv
Noindex: /retailstoreaddress.csv.

# Allow all robots .html files
User-agent: *
Allow: /*.html$