diff --git a/build_docs.py b/build_docs.py index 4771518..fb61a47 100755 --- a/build_docs.py +++ b/build_docs.py @@ -48,6 +48,8 @@ import sys from datetime import datetime +import jinja2 + HERE = Path(__file__).resolve().parent try: @@ -80,6 +82,10 @@ def __init__(self, name, branch, status, sphinx_version=DEFAULT_SPHINX_VERSION): self.status = status self.sphinx_version = sphinx_version + @property + def changefreq(self): + return {"EOL": "never", "security-fixes": "yearly"}.get(self.status, "daily") + @property def url(self): return "https://docs.python.org/{}/".format(self.name) @@ -484,6 +490,24 @@ def build_venv(build_root, version): return venv_path +def build_robots_txt(www_root): + with open(HERE / "templates" / "robots.txt") as robots_txt_template_file: + with open(os.path.join(www_root, "robots.txt"), "w") as robots_txt_file: + template = jinja2.Template(robots_txt_template_file.read()) + robots_txt_file.write( + template.render(languages=LANGUAGES, versions=VERSIONS) + "\n" + ) + + +def build_sitemap(www_root): + with open(HERE / "templates" / "sitemap.xml") as sitemap_template_file: + with open(os.path.join(www_root, "sitemap.xml"), "w") as sitemap_file: + template = jinja2.Template(sitemap_template_file.read()) + sitemap_file.write( + template.render(languages=LANGUAGES, versions=VERSIONS) + "\n" + ) + + def copy_build_to_webroot( build_root, version, @@ -768,6 +792,8 @@ def main(): ) if sentry_sdk: sentry_sdk.capture_exception(err) + build_sitemap(args.www_root) + build_robots_txt(args.www_root) if __name__ == "__main__": diff --git a/requirements.in b/requirements.in index 9898221..b8b6a68 100644 --- a/requirements.in +++ b/requirements.in @@ -1 +1,2 @@ sentry-sdk +jinja2 diff --git a/templates/robots.txt b/templates/robots.txt new file mode 100644 index 0000000..c52e054 --- /dev/null +++ b/templates/robots.txt @@ -0,0 +1,22 @@ +Sitemap: https://docs.python.org/sitemap.xml + +# Prevent development and old documentation from showing up in search results. +User-agent: * +Disallow: /dev +Disallow: /release + +# Disallow EOL versions +Disallow: /2/ +Disallow: /2.0/ +Disallow: /2.1/ +Disallow: /2.2/ +Disallow: /2.3/ +Disallow: /2.4/ +Disallow: /2.5/ +Disallow: /2.6/ +Disallow: /2.7/ +Disallow: /3.0/ +Disallow: /3.1/ +Disallow: /3.2/ +Disallow: /3.3/ +Disallow: /3.4/ diff --git a/templates/sitemap.xml b/templates/sitemap.xml new file mode 100644 index 0000000..84487aa --- /dev/null +++ b/templates/sitemap.xml @@ -0,0 +1,22 @@ + + + {% for version in versions %} + {%- if version.status != "EOL" %} + + https://docs.python.org/{{ version.name }}/ + {% for language in languages -%} + + {% endfor -%} + {{ version.changefreq }} + + {% endif -%} + {% endfor %} + + https://docs.python.org/3/ + {% for language in languages -%} + + {% endfor -%} + daily + +