User-agent: *
Host: https://toldot.com
Disallow: /main/$
Disallow: /admin/
Disallow: /cgi-bin/
Disallow: /gallery/
Disallow: /includes/
Disallow: /jfamily/adam/adam-ask/?link=*
Disallow: /jfamily/children/children-ask/?link=*
Disallow: /lists/
Disallow: /media/
Disallow: /money/payment/*
Disallow: /netcat/
Disallow: /pages/
Disallow: /peredelka/
Disallow: /rus_gmach.php
Disallow: /test/
Disallow: /torah/
Disallow: /work-zone/
Disallow: *template=111
Disallow: *template=168
Disallow: /send-email/
Disallow: /search/
Disallow: /gsearch/
Disallow: /blogs/*/?tag=*
Disallow: /blogs/abre/*

User-agent: heritrix
Disallow: /
User-agent: archive.org_bot
Disallow: /
User-agent: GPTBot
Disallow: /

Sitemap: https://toldot.com/rss/sitemaps/themes/?from=0&length=50000
Sitemap: https://toldot.com/rss/sitemaps/cemeteries/?from=0&length=100
Sitemap: https://toldot.com/rss/sitemaps/surnames/?from=0&length=100
Sitemap: https://toldot.com/rss/sitemaps/audio/?from=0&length=100
Sitemap: https://toldot.com/rss/sitemaps/video/?from=0&length=100
Sitemap: https://toldot.com/rss/sitemaps/articles/?from=0&length=100
Sitemap: https://toldot.com/rss/sitemaps/voprosy/?from=0&length=100
Sitemap: https://toldot.com/rss/sitemaps/photo/?from=0&length=100
Sitemap: https://toldot.com/rss/sitemaps/blogs/?from=0&length=100