User-agent: *
User-agent: wget
User-agent: webzip
User-agent: scooter

Allow: /
Disallow: /files/