# robots updated:  12/14/2021 Aiping Jiang

User-agent: *
Disallow: /0400-news/0200-studies-reports/1100-statistical-plans/reporting-2005/ 
Disallow: /0400-news/0200-studies-reports/1100-statistical-plans/reporting-2006/ 
Disallow: /0400-news/0200-studies-reports/1100-statistical-plans/reporting-2007/ 
Disallow: /0400-news/0200-studies-reports/1100-statistical-plans/2010/
Disallow: /0400-news/0200-studies-reports/1100-statistical-plans/2011/
Disallow: /0400-news/0200-studies-reports/1100-statistical-plans/2012/ 
Disallow: /0400-news/0200-studies-reports/1100-statistical-plans/2013/
Disallow: /0400-news/0200-studies-reports/1100-statistical-plans/2014/
Disallow: /0400-news/0200-studies-reports/1100-statistical-plans/2015/
Disallow: /0400-news/0200-studies-reports/1100-statistical-plans/2016/
Disallow: /0400-news/0200-studies-reports/1100-statistical-plans/2017/
Disallow: /0400-news/0200-studies-reports/1100-statistical-plans/2018/
Disallow: /0400-news/0200-studies-reports/1100-statistical-plans/2019/
Disallow: /0400-news/0200-studies-reports/1100-statistical-plans/2020/
Disallow: /0400-news/0200-studies-reports/1100-statistical-plans/2021/
Disallow: /0400-news/0200-studies-reports/1100-statistical-plans/reporting-2008/
Disallow: /0400-news/0200-studies-reports/1100-statistical-plans/reporting-2009/
Disallow: /0100-consumers/
Disallow: /0100-consumers/0030-licensee-info/0031-surplus-lines/lasli.cfm
Disallow: /0100-consumers/0030-licensee-info/0031-surplus-lines/
Disallow: /loader.cfm
Disallow: /login.cfm
Disallow: /logout.cfm
Disallow: /upload.cfm
Disallow: /createpage.htm
Disallow: /createpage.cfm
Disallow: /index.htm
Disallow: /customcf/handler404.cfm
Disallow: /0300-fraud/0100-fraud-division-overview/25-wc-conv/upload/January2015.pdf
Disallow: /commonspot/
Disallow: /0300-fraud/0100-fraud-division-overview/ifab/CrossTraining/
Disallow: /0250-insurers/0300-insurers/0100-applications/rsb-forms/2020/2020-3-submissions/
Disallow: /rss/

# prohibit NAICSpider crowling the site
User-agent: NAICSpider
User-agent: NAICSpider+ 
Disallow: /

# prohibit Bingbot crowling the site
User-agent: Bingbot
Disallow: /customcf/handler404.cfm

# prohibit Bingbot crowling the site
User-agent: Blekkobot
Disallow: /0250-insurers/0300-insurers/0100-applications/rsb-forms/

# prohibit MegaIndex.ru bot crowling the site
User-agent: MegaIndex.ru
User-agent: MegaIndex.ru+
disallow: /

User-agent: MegaIndex.ru/2.0
User-agent: MegaIndex.ru/2.0+
disallow: /

# prohibit botify bot crowling the site
User-agent: botify
disallow: /

# prohibit Yandex bot crowling the site
User-agent: YandexBot
disallow: /

# Do not allow Python-urllib to access any part of your site
User-agent: Python-urllib
Disallow: /

# prohibit Semrush bot crowling the site
User-agent: SemrushBot
Disallow: /

# prohibit PaperLiBot crowling the site
User-agent: PaperLiBot
Disallow: /

# prohibit Monsidobot crowling the site
User-agent: Monsidobot
Disallow: /

# prohibit bingbot crowling the site
User-agent: bingbot
Disallow: /
Disallow: /0250-insurers/0300-insurers/0100-applications/rsb-forms/2020/2020-3-submissions/
Disallow: /0250-insurers/0300-insurers/0100-applications/rsb-forms/2020/2020-8-submissions/