# 2006-12-21 # Manitobia.ca robots.txt # whikloj@cc.umanitobia.ca # 2007-11-01 # Disallow newspapers as they are only images and slow the system down. # 2008-09-18 # Disallow the search pages as Google seems to like sending random search # terms User-agent: * Crawl-delay: 10 Disallow: /documents Disallow: /digitalobjects Disallow: /tags Disallow: /trunk Disallow: /originalsite Disallow: /branches Disallow: /schema Disallow: /index Disallow: /index_backup Disallow: /index_backup2 Disallow: /images_old Disallow: /cocoon/launch/en/newspapers/* Disallow: /cocoon/launch/fr/newspapers/* Disallow: /cocoon/launch/en/newspaper_search_result.xml Disallow: /cocoon/launch/fr/newspaper_search_result.xml Disallow: /cocoon/launch/en/search_result.xml Disallow: /cocoon/launch/fr/search_result.xml