User-agent: * Disallow: /*/search Disallow: /*/record Disallow: /*/checkout Disallow: /*/help-center Disallow: /*/contact-us # Block newspapers for specific languages Allow: /en/newspapers/* Disallow: /fr/newspapers/* Disallow: /es/newspapers/* Disallow: /da/newspapers/* Disallow: /nl/newspapers/* Disallow: /it/newspapers/* Disallow: /de/newspapers/* Disallow: /no/newspapers/* Disallow: /pb/newspapers/* Disallow: /fi/newspapers/* Disallow: /sv/newspapers/* Disallow: /cs/newspapers/* # Below are blocks for specific user-agents User-agent: Sosospider Disallow: / User-agent: Baiduspider Disallow: / User-agent: YodaoBot Disallow: / User-agent: Yeti Disallow: / User-agent: PiplBot Disallow: / User-agent: MJ12bot Disallow: / User-agent: FatBot Disallow: / User-agent: SeznamBot Disallow: / User-agent: AhrefsBot Disallow: / User-agent: Sogou Disallow: / User-agent: Bytespider Disallow: / User-agent: dotbot Disallow: / User-agent: Mail.Ru Disallow: / Sitemap: https://www.oldnews.com/sitemap.xml.gz