Start a sitemap and a robots.txt.

Author: JulienPalard

build_docs.py

@@ -48,6 +48,8 @@
 import sys
 from datetime import datetime
 
+import jinja2
+
 HERE = Path(__file__).resolve().parent
 
 try:
@@ -80,6 +82,10 @@ def __init__(self, name, branch, status, sphinx_version=DEFAULT_SPHINX_VERSION):
         self.status = status
         self.sphinx_version = sphinx_version
 
+    @property
+    def changefreq(self):
+        return {"EOL": "never", "security-fixes": "yearly"}.get(self.status, "daily")
+
     @property
     def url(self):
         return "https://docs.python.org/{}/".format(self.name)
@@ -484,6 +490,22 @@ def build_venv(build_root, version):
     return venv_path
 
 
+def build_robots_txt(www_root):
+    with open(HERE / "templates" / "robots.txt") as robots_txt_template_file:
+        with open(os.path.join(www_root, "robots.txt"), "w") as robots_txt_file:
+            template = jinja2.Template(robots_txt_template_file.read())
+            robots_txt_file.write(
+                template.render(languages=LANGUAGES, versions=VERSIONS)
+            )
+
+
+def build_sitemap(www_root):
+    with open(HERE / "templates" / "sitemap.xml") as sitemap_template_file:
+        with open(os.path.join(www_root, "sitemap.xml"), "w") as sitemap_file:
+            template = jinja2.Template(sitemap_template_file.read())
+            sitemap_file.write(template.render(languages=LANGUAGES, versions=VERSIONS))
+
+
 def copy_build_to_webroot(
     build_root,
     version,
@@ -768,6 +790,8 @@ def main():
                 )
                 if sentry_sdk:
                     sentry_sdk.capture_exception(err)
+    build_sitemap(args.www_root)
+    build_robots_txt(args.www_root)
 
 
 if __name__ == "__main__":

requirements.in

@@ -1 +1,2 @@
 sentry-sdk
+jinja2

templates/robots.txt

@@ -0,0 +1,22 @@
+Sitemap: https://docs.python.org/sitemap.xml
+
+# Prevent development and old documentation from showing up in search results.
+User-agent: *
+Disallow: /dev
+Disallow: /release
+
+# Disallow EOL versions
+Disallow: /2/
+Disallow: /2.0/
+Disallow: /2.1/
+Disallow: /2.2/
+Disallow: /2.3/
+Disallow: /2.4/
+Disallow: /2.5/
+Disallow: /2.6/
+Disallow: /2.7/
+Disallow: /3.0/
+Disallow: /3.1/
+Disallow: /3.2/
+Disallow: /3.3/
+Disallow: /3.4/

templates/sitemap.xml

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
+        xmlns:xhtml="http://www.w3.org/1999/xhtml">
+  {% for version in versions %}
+  <url>
+    <loc>https://docs.python.org/{{ version.name }}/</loc>
+    {% for language in languages -%}
+    <xhtml:link rel="alternate" hreflang="{{ language.iso639_tag.replace('_', '-') }}" href="https://docs.python.org/{{ language.tag }}/{{ version.name }}/"/>
+    {% endfor -%}
+    <changefreq>{{ version.changefreq }}</changefreq>
+  </url>
+  {% endfor %}
+  <url>
+    <loc>https://docs.python.org/3/</loc>
+    {% for language in languages -%}
+    <xhtml:link rel="alternate" hreflang="{{ language.iso639_tag.replace('_', '-') }}" href="https://docs.python.org/{{ language.tag }}/3/"/>
+    {% endfor -%}
+    <changefreq>daily}</changefreq>
+  </url>
+
+  <url>
+    <loc>https://docs.python.org/dev/</loc>
+    {% for language in languages -%}
+    <xhtml:link rel="alternate" hreflang="{{ language.iso639_tag.replace('_', '-') }}" href="https://docs.python.org/{{ language.tag }}/dev/"/>
+    {% endfor -%}
+    <changefreq>daily</changefreq>
+  </url>
+</urlset>