User-agent: Googlebot
Disallow: /cgi-bin/
Disallow: /search
Disallow: /search.py
Disallow: /yourmessages
Disallow: /events/events.py
Disallow: /tools/
Disallow: /cdslib/
Disallow: /publiline.py
Disallow: /yourbaskets/add
Disallow: /highslide/
Disallow: /ckeditor/
Disallow: /stats/
Disallow: /oai2d/
Disallow: /oai2d.py
Disallow: /record/*/linkbacks/sendtrackback
Disallow: /admin/
Disallow: /video/

User-agent: Googlebot-Image
Disallow: /cgi-bin/
Disallow: /search
Disallow: /search.py
Disallow: /yourmessages
Disallow: /events/events.py
Disallow: /tools/
Disallow: /cdslib/
Disallow: /publiline.py
Disallow: /yourbaskets/add
Disallow: /highslide/
Disallow: /ckeditor/
Disallow: /stats/
Disallow: /oai2d/
Disallow: /oai2d.py
Disallow: /record/*/linkbacks/sendtrackback
Disallow: /admin/
Disallow: /video/

User-Agent: bingbot
Disallow: /cgi-bin/
Disallow: /search
Disallow: /search.py
Disallow: /yourmessages
Disallow: /events/events.py
Disallow: /tools/
Disallow: /cdslib/
Disallow: /publiline.py
Disallow: /yourbaskets/add
Disallow: /highslide/
Disallow: /ckeditor/
Disallow: /stats/
Disallow: /oai2d/
Disallow: /oai2d.py
Disallow: /admin/
Disallow: /video/

User-Agent: Twitterbot
Disallow: /cgi-bin/
Disallow: /search
Disallow: /search.py
Disallow: /yourmessages
Disallow: /events/events.py
Disallow: /tools/
Disallow: /cdslib/
Disallow: /publiline.py
Disallow: /yourbaskets/add
Disallow: /highslide/
Disallow: /ckeditor/
Disallow: /stats/
Disallow: /oai2d/
Disallow: /oai2d.py
Disallow: /admin/
Disallow: /video/
Allow: /images/

User-agent: *
Disallow: /cgi-bin/
Disallow: /search
Disallow: /search.py
Disallow: /record
Disallow: /record/
Disallow: /yourmessages
Disallow: /events/events.py
Disallow: /tools/
Disallow: /cdslib/
Disallow: /publiline.py
Disallow: /yourbaskets/add
Disallow: /highslide/
Disallow: /ckeditor/
Disallow: /stats/
Disallow: /oai2d/
Disallow: /oai2d.py
Disallow: /admin/
Disallow: /video/
Disallow: /images/

# disabled because the BibExport task is not exporting sitemaps
# if re-enabled, change web server config to serve these files
#Sitemap: http://cds.cern.ch/sitemap_index.xml.gz