Add pa11y testing and reporting (#294)

Co-authored-by: Chris Holdgraf <[email protected]>

Author: bollwyvl

.github/workflows/tests.yml

@@ -69,6 +69,7 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
+
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v1
       with:
@@ -120,14 +121,21 @@ jobs:
       HOST: 127.0.0.1
       # the base url
       URL: http://127.0.0.1:8000
+      PA11Y_BUILD: /tmp/pa11y/pa11y-${{ github.run_number }}
 
     steps:
     - uses: actions/checkout@v2
+
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v1
       with:
         python-version: ${{ matrix.python-version }}
 
+    - name: Set up Node/yarn
+      uses: actions/setup-node@v1
+      with:
+        node-version: '10.x'
+
     - name: Cache python wheels
       uses: actions/cache@v2
       with:
@@ -138,10 +146,18 @@ jobs:
           ${{ runner.os }}-pip-${{ matrix.python-version }}-
           ${{ runner.os }}-pip-
 
+    - name: Cache node_modules
+      uses: actions/cache@v2
+      with:
+        path: 'node_modules'
+        key: |
+          ${{ runner.os }}-node-modules-${{ hashFiles('yarn.lock') }}
+
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip wheel setuptools
         python -m pip install -e .[coverage]
+        yarn --frozen-lockfile
 
     # Build the docs
     - name: Build docs to store
@@ -159,6 +175,7 @@ jobs:
     # TODO: use the hosted API with a secret? would allow for comparison over time...
     - name: Make folder for Lighthouse reports
       run: mkdir -p /tmp/lighthouse/lighthouse-${{ github.run_number }}
+
     - name: Run Lighthouse on Site
       id: lighthouse
       uses: foo-software/[email protected]
@@ -168,16 +185,13 @@ jobs:
           ${{ env.URL }}/index.html,
           ${{ env.URL }}/demo/api.html,
           ${{ env.URL }}/demo/demo.html,
-          ${{ env.URL }}/demo/example_pandas.html
+          ${{ env.URL }}/demo/example_pandas.html,
+          ${{ env.URL }}/user_guide/accessibility.html
         outputDirectory: /tmp/lighthouse/lighthouse-${{ github.run_number }}
         verbose: true
 
-    # Store the audit
-    - name: Upload Lighthouse Reports
-      uses: actions/upload-artifact@v2
-      with:
-        name: Lighthouse Report ${{ github.run_number }}
-        path: /tmp/lighthouse
+    - name: Run the accessibility audit
+      run: python docs/a11y.py --no-serve
 
     # Check the audit for threshold values
     # TODO: write this someplace after a PR is merged, and load?
@@ -189,6 +203,16 @@ jobs:
         minBestPracticesScore: "85"
         minPerformanceScore: "10"
         minSeoScore: "80"
+      if: always()
+
+    - name: Publish Audit reports
+      uses: actions/upload-artifact@v2
+      with:
+        name: Pa11y and Lighthouse ${{ github.run_number }}
+        path: |
+          /tmp/pa11y
+          /tmp/lighthouse
+      if: always()
 
   publish:
 

.gitignore

@@ -104,4 +104,5 @@ envs/
 
 node_modules/
 .vscode
+.yarn-packages
 .idea

.yarnrc

@@ -0,0 +1,4 @@
+yarn-offline-mirror "./.yarn-packages"
+yarn-offline-mirror-pruning true
+ignore-optional true
+prefer-offline true

docs/a11y-roadmap.txt

@@ -0,0 +1,9 @@
+# These are the pa11y error codes we'd like to fix. PRs welcome!
+
+WCAG2AA.Principle1.Guideline1_3.1_3_1.H39.3.LayoutTable
+WCAG2AA.Principle1.Guideline1_3.1_3_1.H43,H63
+WCAG2AA.Principle1.Guideline1_3.1_3_1.H43.HeadersRequired
+WCAG2AA.Principle1.Guideline1_4.1_4_3.G18.Fail
+WCAG2AA.Principle3.Guideline3_2.3_2_2.H32.2
+WCAG2AA.Principle4.Guideline4_1.4_1_2.H91.A.EmptyNoId
+WCAG2AA.Principle4.Guideline4_1.4_1_2.H91.A.NoContent

docs/a11y.py

@@ -0,0 +1,146 @@
+""" use pa11y-ci to generate an accessibility report
+"""
+import json
+import os
+import shutil
+import subprocess
+import sys
+import time
+from collections import defaultdict
+from pathlib import Path
+from urllib.error import URLError
+from urllib.request import urlopen
+from yaml import safe_dump
+
+HERE = Path(__file__).parent.resolve()
+ROOT = HERE.parent
+BUILD = HERE / "_build"
+PA11Y_BUILD = Path(os.environ.get("PA11Y_BUILD", BUILD / "pa11y"))
+PA11Y_JSON = PA11Y_BUILD / "pa11y-ci-results.json"
+PA11Y_ROADMAP = HERE / "a11y-roadmap.txt"
+YARN = [shutil.which("yarn"), "--silent"]
+SITEMAP = "http://127.0.0.1:8000/sitemap.xml"
+REPORT_INDEX_URL = (PA11Y_BUILD / "index.html").as_uri()
+
+
+def clean():
+    if PA11Y_BUILD.exists():
+        shutil.rmtree(PA11Y_BUILD)
+    PA11Y_BUILD.mkdir(parents=True)
+
+
+def serve():
+    """start the local server"""
+    server = subprocess.Popen(
+        [sys.executable, HERE / "serve.py"],
+    )
+    ready = 0
+    retries = 10
+    while retries and not ready:
+        retries -= 1
+        try:
+            time.sleep(1)
+            ready = urlopen(SITEMAP)
+        except URLError:
+            pass
+
+    assert ready, "server did not start in 10 seconds"
+
+    return server
+
+
+def audit():
+    """run audit, generating a raw JSON report"""
+    audit_rc = subprocess.call(
+        f"yarn --silent pa11y-ci --json --sitemap {SITEMAP} > {PA11Y_JSON}",
+        shell=True,
+        cwd=ROOT,
+    )
+
+    return audit_rc
+
+
+def report():
+    """generate HTML report from raw JSON"""
+    subprocess.call(
+        [
+            *YARN,
+            "pa11y-ci-reporter-html",
+            "--source",
+            PA11Y_JSON,
+            "--destination",
+            PA11Y_BUILD,
+        ],
+        cwd=ROOT,
+    )
+
+
+def summary():
+    """generate a summary and return the number of errors not ignored explicitly"""
+    report = dict(
+        HTML=REPORT_INDEX_URL, Roadmap=str(PA11Y_ROADMAP), JSON=str(PA11Y_JSON)
+    )
+
+    pa11y_json = json.loads(PA11Y_JSON.read_text())
+    pa11y_roadmap = [
+        line.split("#")[0].strip()
+        for line in PA11Y_ROADMAP.read_text().splitlines()
+        if line.strip() and not line.strip().startswith("#")
+    ]
+
+    error_codes = defaultdict(lambda: 0)
+
+    for page, results in pa11y_json["results"].items():
+        for result in results:
+            if "code" in result:
+                error_codes[result["code"]] += 1
+
+    roadmap_counts = {}
+    not_roadmap_counts = {}
+
+    for code, count in sorted(error_codes.items()):
+        code_on_roadmap = code in pa11y_roadmap
+        if code_on_roadmap:
+            roadmap_counts[code] = count
+        else:
+            not_roadmap_counts[code] = count
+
+    report.update(
+        {
+            "total errors": pa11y_json["errors"],
+            "on roadmap": roadmap_counts,
+            "not on roadmap": not_roadmap_counts,
+        }
+    )
+
+    nrc = sum(not_roadmap_counts.values())
+    report["passed"] = nrc == 0
+    report_str = safe_dump(report, default_flow_style=False)
+
+    if os.environ.get("CI") and nrc:
+        print("""::error ::{}""".format(report_str.replace("\n", "%0A")))
+    else:
+        print(report_str)
+
+    return nrc
+
+
+def main(no_serve=True):
+    """start the server (if needed), then audit, report, and clean up"""
+    clean()
+    server = None
+    try:
+        if not no_serve:
+            server = serve()
+        audit()
+        report()
+    finally:
+        server and server.terminate()
+
+    error_count = summary()
+
+    return error_count
+
+
+if __name__ == "__main__":
+    sys.exit(main(no_serve="--no-serve" in sys.argv))

docs/changelog.rst

@@ -3,3 +3,7 @@ Changelog
 *********
 
 See the `GitHub Releases <https://github.com/pydata/pydata-sphinx-theme/releases>`_ for the changelog of each release.
+
+.. meta::
+    :description lang=en:
+        The historical releases for pydata-sphinx-theme.

docs/conf.py

@@ -10,7 +10,8 @@
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
-# import os
+import os
+
 # import sys
 # sys.path.insert(0, os.path.abspath('.'))
 
@@ -39,6 +40,19 @@
     "jupyter_sphinx",
 ]
 
+# -- Internationalization ------------------------------------------------
+# specifying the natural language populates some key tags
+language = "en"
+
+# ReadTheDocs has its own way of generating sitemaps, etc.
+if not os.environ.get("READTHEDOCS"):
+    extensions += ["sphinx_sitemap"]
+
+    # -- Sitemap -------------------------------------------------------------
+    html_baseurl = os.environ.get("SITEMAP_URL_BASE", "http://127.0.0.1:8000/")
+    sitemap_locales = [None]
+    sitemap_url_scheme = "{link}"
+
 autosummary_generate = True
 
 # Add any paths that contain templates here, relative to this directory.
@@ -99,7 +113,7 @@
 html_static_path = ["_static"]
 
 
-# -- Auto-convert markdown pages to demo --------------------------------------
+# -- Auto-convert markdown pages to demo -------------------------------------
 import recommonmark
 from recommonmark.transform import AutoStructify
 

docs/contributing.rst

@@ -164,7 +164,7 @@ To preview the frontend assets, from the root of this repo, run:
 
     yarn build:dev
 
-This launches a development server at http://localhost:1919. When working
+This launches a development server at http://127.0.0.1:1919. When working
 on the theme, saving changes to any of:
 
 - ``src/js/index.js``
@@ -331,8 +331,66 @@ Note that if needed, you can skip these checks with:
     git commit --no-verify
 
 
+Finding accessibility problems
+==============================
+
+The accessibility checking tools can find a number of common HTML patterns which
+assistive technology can't help users understand.
+
+In addition to `Lighthouse <https://developers.google.com/web/tools/lighthouse>`__
+in CI, the ``pa11y`` stack is installed as part of the development environment.
+
+The key components are:
+
+- `pa11y <https://github.com/pa11y/pa11y>`__ which uses a headless browser to analyze
+  an HTML page with a configurable set of rules based on publish standards
+- `Pa11y-CI <https://github.com/pa11y/pa11y-ci>`__ runs ``pa11y`` on multiple pages
+- `pa11y-reporter-html <https://github.com/pa11y/pa11y-reporter-html>`__ generates
+  some nice HTML reports, suitable for review
+
+.. Note::
+
+    Presently, the *default* ``pa11y`` ruleset, ``WCAG2AA`` is used, a subset of
+    the `Web Content Accessibility Guidelines <https://www.w3.org/TR/WCAG21>`__.
+    The `Quick Reference <https://www.w3.org/WAI/WCAG21/quickref>`__ may provide
+    lighter reading.
+
+To run the accessibility problem finder locally:
+
+.. code-block:: bash
+
+    yarn build:production
+    cd docs
+    make html
+    python a11y.py
+
+The output of the last command includes:
+
+- a short summary of the current state of the accessibility rules we are trying to maintain
+- local paths to JSON and HTML reports which contain all of the issues found
+
+
+Fixing accessibility errors
+---------------------------
+
+Start by checking for issues on the
+`accessibility roadmap <https://github.com/pandas-dev/pydata-sphinx-theme/blob/master/docs/a11y-roadmap.txt>`__.
+These are issues which are currently flagged by the toolset, but that have not yet
+been fixed. If that file is empty (or just comments), hooray!
+
+To start working on one of the accessibility roadmap items, comment out one of the
+lines in `docs/a11y-roadmap.txt`, and re-run the audit to establish a baseline.
+
+Then, fix the issue in either the HTML templates, CSS, or python code, and re-run
+the audit until it is fixed.
+
+
 Make a release
 ==============
 
 This theme uses GitHub tags and releases to automatically push new releases to
 PyPI. For information on this process, see `the release checklist <https://github.com/pydata/pydata-sphinx-theme/wiki/Release-checklist#release-instructions>`_.
+
+.. meta::
+    :description lang=en:
+        How to become a contributor to the pydata-sphinx-theme.

docs/demo/api.rst

@@ -154,3 +154,8 @@ Data
     Lorem ipsum dolor sit amet, consectetur adipiscing elit. Fusce congue elit eu hendrerit mattis.
 
 Some data link :data:`Data_item_1`.
+
+
+.. meta::
+    :description lang=en:
+        An example of API documentation with pydata-sphinx-theme.