# robots.txt for http://clinicaltrials.gov

User-Agent: *
Disallow: /cs/                  # obsolete
Disallow: /c/                   # obsolete
Disallow: /ct/                  # obsolete

Disallow: /ct2/archive/         # leaving this site to archive page
Disallow: /ct2/bye/             # leaving this site to 3rd party page
Disallow: /ct2/helpdesk/        # helpdesk form
Disallow: /ct2/results/rss.xml  # rss feed of current search results

# crawlers are advised to avoid our browse hierarchy because
#   it finds all studies many times, with slightly different URL's
#   and term highlighting.  Much better if crawlers find the studies
#   using the crawler link:  /ct2/crawl