diff --git a/.gitignore b/.gitignore index 55ce56513..bff0aa0d1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ .ipynb_checkpoints/ -docs/_build/ -docs/rst-tutorials/ +build/ +tutorials/rst-tutorials/ IPython-* *.fits *.pdf @@ -8,4 +8,4 @@ IPython-* *.jpg *.pyc *.tex -exec*.ipynb \ No newline at end of file +exec*.ipynb diff --git a/.rtd-environment.yml b/.rtd-environment.yml index 07c029fce..5425ce182 100644 --- a/.rtd-environment.yml +++ b/.rtd-environment.yml @@ -14,4 +14,4 @@ dependencies: - jupyter=1.0 - notebook=5.0 - pip: - - git+https://github.com/jupyter/nbconvert + - git+https://github.com/jupyter/nbconvert # this is really >=5.4, but at the time of adding this that wasn't yet available diff --git a/README.rst b/README.rst index 0ca2839e5..1fd269a12 100644 --- a/README.rst +++ b/README.rst @@ -23,3 +23,14 @@ The tutorials are initially empty of any output. You can run them by pressing See the conda environment file or pip requirements file for a list of dependencies. + +## Building the tutorial web pages + +To build all of the tutorials in the form they appear on the web site, you just +do the sphinx build as for a Python package:: + + >>> python setup.py build_docs + +For more information on this and other ways to execute the tutorials, see the +"Documentation on tutorials infrastructure" section of the generated +sphinx docs. diff --git a/scripts/check_env.py b/scripts/check_env.py index 6955d7477..4c1d6c10e 100644 --- a/scripts/check_env.py +++ b/scripts/check_env.py @@ -39,52 +39,31 @@ from astropy import log as logger -def check_environment(tutorial=None): +def check_environment(tutorials_base_path, tutorial=None): error = False warnings = False # 'enter' checks if the tutorial name passed by user exists in the repository enter = False logger.info("Running Environment Tests...") - _orig_path = os.getcwd() - tutorials_base = os.path.join(_orig_path, 'tutorials') - for tutorial_name in os.listdir(tutorials_base): + for tutorial_name in os.listdir(tutorials_base_path): + print(tutorial_name,'ARG') if (tutorial_name == tutorial or tutorial is None): enter = True - tutorial_path = os.path.join( - tutorials_base, - tutorial_name) # Path to tutorial folder + tutorialreq_path = os.path.join(tutorials_base_path, tutorial_name, + 'requirements.txt') # Path to tutorial folder try: - with open(tutorial_path + "/requirements.json") as data_file: - # Import data from json file - data = json.load(data_file) + with open(tutorialreq_path) as req_file: # Check for all the packages to be imported - for pkgname in data: - # Check for versioning - for subinfo in data[pkgname]: - if subinfo == 'min_version': - if(not astropy.utils.introspection.minversion(pkgname, data[pkgname][subinfo])): - logger.error( - "Package " + - pkgname + - " is either missing or is out of date to run Tutorial: " + - tutorial_name) - error = True - #Package is Missing - elif subinfo == 'pref_version': - if(not astropy.utils.introspection.minversion(pkgname, data[pkgname][subinfo])): - logger.warning( - "Please upgrade Package " + - pkgname + - " to version " + - data[pkgname][subinfo] + - " for Tutorial: " + - tutorial_name) - warnings = True - # Out of date package is present + for line in req_file: + line_strip = line.strip() + astropy.utils.introspection.minversion(line_strip, '') + logger.info('Sucessfully checked requirements for ' + '"{}".'.format(tutorial_name)) + except IOError: - error = True - logger.error( - "Environment Check Failed ! requirements.json not found") + warnings = True + logger.warn('requirements.txt not found for "{}" - couldn\'t do' + ' environment check.'.format(tutorial_name)) if not enter: logger.error( @@ -112,6 +91,10 @@ def check_environment(tutorial=None): help="A regular expression to select the names of the " "notebooks to be processed. If not given, all " "notebooks will be used.") + parser.add_argument("tutorial_base_path", default='tutorials/notebooks', + nargs='?', + help="The path to the root of the tutorial " + "directories.") args = parser.parse_args() - check_environment(args.nameregex) # name passed to check_env function + check_environment(args.tutorial_base_path, args.nameregex) # name passed to check_env function diff --git a/scripts/convert.py b/scripts/convert.py index b54e6f69f..b33c3cbf1 100644 --- a/scripts/convert.py +++ b/scripts/convert.py @@ -15,7 +15,7 @@ class NBConverter(object): def __init__(self, nb_path, output_path=None, template_file=None, - overwrite=False): + overwrite=False, kernel_name=None): self.nb_path = path.abspath(nb_path) fn = path.basename(self.nb_path) self.path_only = path.dirname(self.nb_path) @@ -41,6 +41,11 @@ def __init__(self, nb_path, output_path=None, template_file=None, logger.info('Processing notebook {0} (in {1})'.format(fn, self.path_only)) + if kernel_name is None: + self.kernel_name = ExecutePreprocessor.kernel_name.default_value + else: + self.kernel_name = kernel_name + def execute(self, write=True): """ Execute the specified notebook file, and optionally write out the @@ -66,8 +71,11 @@ def execute(self, write=True): return self._executed_nb_path # Execute the notebook - logger.debug('Executing notebook...') - executor = ExecutePreprocessor(timeout=900, kernel_name='python3') + logger.debug('Executing notebook using kernel ' + '"{}"...'.format(self.kernel_name)) + executor = ExecutePreprocessor(timeout=900, + kernel_name=self.kernel_name) + with open(self.nb_path) as f: nb = nbformat.read(f, as_version=IPYTHON_VERSION) @@ -197,7 +205,8 @@ def process_notebooks(nbfile_or_path, exec_only=False, **kwargs): help='Re-run and overwrite any existing executed ' 'notebook or RST files.') - parser.add_argument('nbfile_or_path', default=None, + parser.add_argument('nbfile_or_path', default='tutorials/notebooks/', + nargs='?', help='Path to a specific notebook file, or the ' 'top-level path to a directory containing ' 'notebook files to process.') @@ -212,6 +221,11 @@ def process_notebooks(nbfile_or_path, exec_only=False, **kwargs): 'converted files will be in the same path as the ' 'source notebooks.') + parser.add_argument('--kernel-name', default='python3', dest='kernel_name', + help='The name of the kernel to run the notebooks with.' + ' Must be an available kernel from "jupyter ' + 'kernelspec list".') + args = parser.parse_args() # Set logger level based on verbose flags @@ -241,4 +255,4 @@ def process_notebooks(nbfile_or_path, exec_only=False, **kwargs): process_notebooks(args.nbfile_or_path, exec_only=args.exec_only, output_path=output_path, template_file=template_file, - overwrite=args.overwrite) + overwrite=args.overwrite, kernel_name=args.kernel_name) diff --git a/setup.cfg b/setup.cfg index 5696202cc..dd6d37fe2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,20 +1,20 @@ [build_sphinx] -source-dir = docs -build-dir = docs/_build +source-dir = tutorials +build-dir = build/sphinx all_files = 1 [build_docs] -source-dir = docs -build-dir = docs/_build +source-dir = tutorials +build-dir = build/sphinx all_files = 1 [upload_docs] -upload-dir = docs/_build/html +upload-dir = build/sphinx/html show-response = 1 [pytest] minversion = 2.2 -norecursedirs = build docs/_build +norecursedirs = build build/sphinx doctest_plus = enabled [ah_bootstrap] diff --git a/docs/Makefile b/tutorials/Makefile similarity index 100% rename from docs/Makefile rename to tutorials/Makefile diff --git a/docs/_static/astropy.tpl b/tutorials/astropy.tpl similarity index 100% rename from docs/_static/astropy.tpl rename to tutorials/astropy.tpl diff --git a/docs/conf.py b/tutorials/conf.py similarity index 92% rename from docs/conf.py rename to tutorials/conf.py index 01ad17dbd..797fe7835 100644 --- a/docs/conf.py +++ b/tutorials/conf.py @@ -33,7 +33,7 @@ # -- General configuration ---------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. -# needs_sphinx = '1.2' +needs_sphinx = '1.4' # needed for suppress_warnings # To perform a Sphinx version check that needs to be more specific than # major.minor, call `check_sphinx_version("x.y.z")` here. @@ -81,7 +81,8 @@ html_theme_options = { 'logotext1': 'astro', # white, semi-bold 'logotext2': 'py', # orange, light - 'logotext3': ':tutorials' # white, light + 'logotext3': ':tutorials', # white, light + 'nosidebar': True } # Add any paths that contain custom themes here, relative to this directory. @@ -141,8 +142,12 @@ sys.path.insert(1, _scripts_path) from convert import process_notebooks -nb_tutorials_path = os.path.join(_root, 'docs', '_static', 'tutorials') -template_path = os.path.join(_root, 'docs', '_static', 'astropy.tpl') -rst_output_path = os.path.join(_root, 'docs', 'rst-tutorials') +nb_tutorials_path = os.path.join(_root, 'tutorials', 'notebooks') +template_path = os.path.join(_root, 'tutorials', 'astropy.tpl') +rst_output_path = os.path.join(_root, 'tutorials', 'rst-tutorials') process_notebooks(nb_tutorials_path, output_path=rst_output_path, template_file=template_path) + + +suppress_warnings = ['image.nonlocal_uri'] +html_static_path = ['notebooks'] diff --git a/docs/dev.rst b/tutorials/dev.rst similarity index 86% rename from docs/dev.rst rename to tutorials/dev.rst index 1323b0be6..e0ae6702d 100644 --- a/docs/dev.rst +++ b/tutorials/dev.rst @@ -1,11 +1,11 @@ -Documentation for developers -============================ +Documentation on tutorials infrastructure +========================================= Overview -------- Tutorials are written as Jupyter notebooks on the ``master`` branch of the -``astropy/astropy-tutorials`` repository in ``docs/_static/tutorials/``. These +``astropy/astropy-tutorials`` repository in ``tutorials/notebooks/``. These notebook files do not contain output in order to simplify version-controlling the files. @@ -37,7 +37,7 @@ Tutorials directory structure ----------------------------- The notebook files must be written as a single Jupyter notebook in a directory -within the ``docs/_static/tutorials`` directory. The name of the notebook must +within the ``tutorials/notebooks`` directory. The name of the notebook must be the same as the subdirectory name. This is just needed for auto-generating links to the source notebooks from the generated RST pages. @@ -48,7 +48,7 @@ You can use the custom nbconvert script in the astropy-tutorials repository to test that the tutorials all execute correctly. From the top-level repository path:: - python scripts/convert.py docs/_static/tutorials -v --exec-only + python scripts/convert.py tutorials/notebooks -v --exec-only Running the convert script with the flag ``--exec-only`` will just execute the notebooks and won't generate RST files. If you have already run the notebooks @@ -58,16 +58,16 @@ executed. The ``-v`` flag just tells the script to output more "verbose" messages, which you may or may not want. The above command will execute all notebooks in any subdirectory of the -``docs/_static/tutorials`` path. If you want to just execute a single notebook, +``tutorials/notebooks`` path. If you want to just execute a single notebook, you can specify the path to a single notebook file, e.g.:: - python scripts/convert.py docs/_static/tutorials/coordinates/coordinates.ipynb -v --exec-only + python scripts/convert.py tutorials/notebooks/coordinates/coordinates.ipynb -v --exec-only You can also do this when running and generating RST files, which can be useful when writing a new tutorial to make sure it renders in RST properly. To do this, just remove the ``--exec-only`` flag:: - python scripts/convert.py docs/_static/tutorials/coordinates/coordinates.ipynb -v + python scripts/convert.py tutorials/notebooks/coordinates/coordinates.ipynb -v Releases -------- @@ -84,4 +84,3 @@ Marking a cell with an intentional error Edit the cell metadata of the cell in which you would like to raise an exception and add the following to the top-level JSON: ``"tags": ["raises-exception"]`` This tag is recognized by the latest (master) version of nbconvert. - diff --git a/docs/index.rst b/tutorials/index.rst similarity index 100% rename from docs/index.rst rename to tutorials/index.rst diff --git a/docs/make.bat b/tutorials/make.bat similarity index 100% rename from docs/make.bat rename to tutorials/make.bat diff --git a/docs/_static/tutorials/FITS-header/FITS-header.ipynb b/tutorials/notebooks/FITS-header/FITS-header.ipynb similarity index 100% rename from docs/_static/tutorials/FITS-header/FITS-header.ipynb rename to tutorials/notebooks/FITS-header/FITS-header.ipynb diff --git a/docs/_static/tutorials/FITS-header/input_file.fits b/tutorials/notebooks/FITS-header/input_file.fits similarity index 100% rename from docs/_static/tutorials/FITS-header/input_file.fits rename to tutorials/notebooks/FITS-header/input_file.fits diff --git a/docs/_static/tutorials/FITS-header/requirements.txt b/tutorials/notebooks/FITS-header/requirements.txt similarity index 100% rename from docs/_static/tutorials/FITS-header/requirements.txt rename to tutorials/notebooks/FITS-header/requirements.txt diff --git a/docs/_static/tutorials/FITS-images/FITS-images.ipynb b/tutorials/notebooks/FITS-images/FITS-images.ipynb similarity index 100% rename from docs/_static/tutorials/FITS-images/FITS-images.ipynb rename to tutorials/notebooks/FITS-images/FITS-images.ipynb diff --git a/docs/_static/tutorials/FITS-images/requirements.txt b/tutorials/notebooks/FITS-images/requirements.txt similarity index 100% rename from docs/_static/tutorials/FITS-images/requirements.txt rename to tutorials/notebooks/FITS-images/requirements.txt diff --git a/docs/_static/tutorials/FITS-tables/FITS-tables.ipynb b/tutorials/notebooks/FITS-tables/FITS-tables.ipynb similarity index 100% rename from docs/_static/tutorials/FITS-tables/FITS-tables.ipynb rename to tutorials/notebooks/FITS-tables/FITS-tables.ipynb diff --git a/docs/_static/tutorials/FITS-tables/requirements.txt b/tutorials/notebooks/FITS-tables/requirements.txt similarity index 100% rename from docs/_static/tutorials/FITS-tables/requirements.txt rename to tutorials/notebooks/FITS-tables/requirements.txt diff --git a/docs/_static/tutorials/UVES/CaII-1.png b/tutorials/notebooks/UVES/CaII-1.png similarity index 100% rename from docs/_static/tutorials/UVES/CaII-1.png rename to tutorials/notebooks/UVES/CaII-1.png diff --git a/docs/_static/tutorials/UVES/CaII-2.png b/tutorials/notebooks/UVES/CaII-2.png similarity index 100% rename from docs/_static/tutorials/UVES/CaII-2.png rename to tutorials/notebooks/UVES/CaII-2.png diff --git a/docs/_static/tutorials/UVES/CaII-3.png b/tutorials/notebooks/UVES/CaII-3.png similarity index 100% rename from docs/_static/tutorials/UVES/CaII-3.png rename to tutorials/notebooks/UVES/CaII-3.png diff --git a/docs/_static/tutorials/UVES/CaII-lines-all.png b/tutorials/notebooks/UVES/CaII-lines-all.png similarity index 100% rename from docs/_static/tutorials/UVES/CaII-lines-all.png rename to tutorials/notebooks/UVES/CaII-lines-all.png diff --git a/docs/_static/tutorials/UVES/CaII-lines-one.png b/tutorials/notebooks/UVES/CaII-lines-one.png similarity index 100% rename from docs/_static/tutorials/UVES/CaII-lines-one.png rename to tutorials/notebooks/UVES/CaII-lines-one.png diff --git a/docs/_static/tutorials/UVES/CaII.png b/tutorials/notebooks/UVES/CaII.png similarity index 100% rename from docs/_static/tutorials/UVES/CaII.png rename to tutorials/notebooks/UVES/CaII.png diff --git a/docs/_static/tutorials/UVES/UVES.ipynb b/tutorials/notebooks/UVES/UVES.ipynb similarity index 100% rename from docs/_static/tutorials/UVES/UVES.ipynb rename to tutorials/notebooks/UVES/UVES.ipynb diff --git a/docs/_static/tutorials/UVES/requirements.txt b/tutorials/notebooks/UVES/requirements.txt similarity index 100% rename from docs/_static/tutorials/UVES/requirements.txt rename to tutorials/notebooks/UVES/requirements.txt diff --git a/docs/_static/tutorials/coordinates/HCG7_2MASS.tbl b/tutorials/notebooks/coordinates/HCG7_2MASS.tbl similarity index 100% rename from docs/_static/tutorials/coordinates/HCG7_2MASS.tbl rename to tutorials/notebooks/coordinates/HCG7_2MASS.tbl diff --git a/docs/_static/tutorials/coordinates/HCG7_SDSS_cutout.jpg b/tutorials/notebooks/coordinates/HCG7_SDSS_cutout.jpg similarity index 100% rename from docs/_static/tutorials/coordinates/HCG7_SDSS_cutout.jpg rename to tutorials/notebooks/coordinates/HCG7_SDSS_cutout.jpg diff --git a/docs/_static/tutorials/coordinates/HCG7_SDSS_photo.dat b/tutorials/notebooks/coordinates/HCG7_SDSS_photo.dat similarity index 100% rename from docs/_static/tutorials/coordinates/HCG7_SDSS_photo.dat rename to tutorials/notebooks/coordinates/HCG7_SDSS_photo.dat diff --git a/docs/_static/tutorials/coordinates/coordinates.ipynb b/tutorials/notebooks/coordinates/coordinates.ipynb similarity index 100% rename from docs/_static/tutorials/coordinates/coordinates.ipynb rename to tutorials/notebooks/coordinates/coordinates.ipynb diff --git a/docs/_static/tutorials/coordinates/requirements.txt b/tutorials/notebooks/coordinates/requirements.txt similarity index 100% rename from docs/_static/tutorials/coordinates/requirements.txt rename to tutorials/notebooks/coordinates/requirements.txt diff --git a/docs/_static/tutorials/plot-catalog/Young-Objects-Compilation.csv b/tutorials/notebooks/plot-catalog/Young-Objects-Compilation.csv similarity index 100% rename from docs/_static/tutorials/plot-catalog/Young-Objects-Compilation.csv rename to tutorials/notebooks/plot-catalog/Young-Objects-Compilation.csv diff --git a/docs/_static/tutorials/plot-catalog/plot-catalog.ipynb b/tutorials/notebooks/plot-catalog/plot-catalog.ipynb similarity index 100% rename from docs/_static/tutorials/plot-catalog/plot-catalog.ipynb rename to tutorials/notebooks/plot-catalog/plot-catalog.ipynb diff --git a/docs/_static/tutorials/plot-catalog/requirements.txt b/tutorials/notebooks/plot-catalog/requirements.txt similarity index 100% rename from docs/_static/tutorials/plot-catalog/requirements.txt rename to tutorials/notebooks/plot-catalog/requirements.txt diff --git a/docs/_static/tutorials/plot-catalog/simple_table.csv b/tutorials/notebooks/plot-catalog/simple_table.csv similarity index 100% rename from docs/_static/tutorials/plot-catalog/simple_table.csv rename to tutorials/notebooks/plot-catalog/simple_table.csv diff --git a/docs/_static/tutorials/quantities/quantities.ipynb b/tutorials/notebooks/quantities/quantities.ipynb similarity index 100% rename from docs/_static/tutorials/quantities/quantities.ipynb rename to tutorials/notebooks/quantities/quantities.ipynb diff --git a/docs/_static/tutorials/quantities/requirements.txt b/tutorials/notebooks/quantities/requirements.txt similarity index 100% rename from docs/_static/tutorials/quantities/requirements.txt rename to tutorials/notebooks/quantities/requirements.txt diff --git a/docs/_static/tutorials/redshift-plot/ang_dist.png b/tutorials/notebooks/redshift-plot/ang_dist.png similarity index 100% rename from docs/_static/tutorials/redshift-plot/ang_dist.png rename to tutorials/notebooks/redshift-plot/ang_dist.png diff --git a/docs/_static/tutorials/redshift-plot/redshift-plot.ipynb b/tutorials/notebooks/redshift-plot/redshift-plot.ipynb similarity index 100% rename from docs/_static/tutorials/redshift-plot/redshift-plot.ipynb rename to tutorials/notebooks/redshift-plot/redshift-plot.ipynb diff --git a/docs/_static/tutorials/redshift-plot/requirements.txt b/tutorials/notebooks/redshift-plot/requirements.txt similarity index 100% rename from docs/_static/tutorials/redshift-plot/requirements.txt rename to tutorials/notebooks/redshift-plot/requirements.txt diff --git a/docs/_static/tutorials/vo/README.md b/tutorials/notebooks/vo/README.md similarity index 100% rename from docs/_static/tutorials/vo/README.md rename to tutorials/notebooks/vo/README.md diff --git a/docs/_static/tutorials/vo/conesearch.ipynb b/tutorials/notebooks/vo/conesearch.ipynb similarity index 100% rename from docs/_static/tutorials/vo/conesearch.ipynb rename to tutorials/notebooks/vo/conesearch.ipynb diff --git a/tutorials/notebooks/vo/requirements.txt b/tutorials/notebooks/vo/requirements.txt new file mode 100644 index 000000000..30317e981 --- /dev/null +++ b/tutorials/notebooks/vo/requirements.txt @@ -0,0 +1 @@ +astroquery