User-agent: sogou spider
Disallow: /

User-agent: ccbot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: RFCrawler-Mobile
Disallow: /

User-agent: twiceler
Disallow: /

User-agent: *
Disallow: /cgi-bin/
Disallow: /assets/