Adding celery scaffold to the project.

Adding celery_scaffold for use with flask based projects. It provides a
way to configure celery for use, using the flask configuration files. It
also provides a celery_app and a flask_app that can be used in your
project.

A base_scaffold was pulled out due to the celery worker assuming any 'app'
attribute is of type Celery. The original app_scaffold is available to
provide backward compatibility. It leverages the new base_scaffold and
sets the 'app' attribute to flask_app to ensure existing use cases are
handled.

Signed-off-by: Jason Joyce <[email protected]>

Author: fuzzball81

.github/workflows/test.yml

@@ -21,7 +21,7 @@ jobs:
             pip install tox tox-gh-actions
         - name: Run Tox
           # Run tox using the version of Python in `PATH`
-          run: tox -epy
+          run: tox
     flake8:
       runs-on: ubuntu-latest
       steps:

README.md

@@ -129,6 +129,16 @@ example:
         },
     })
 
+### Using CeleryScaffold
+
+This class has all of the same support as the above AppScaffold and takes
+the same parameters. Each CeleryScaffold instance has a flask_app and celery_app
+attribute that can be used in your project
+
+    celery_scaffold = CeleryScaffold(name=__name__, config=config)
+    flask_app = celery_scaffold.flask_app
+    celery_app = celery_scaffold.celery_app
+
 ### Using the parse_input method
 
 This method is used to validate incoming data against a pydantic model. A

setup.cfg

@@ -23,6 +23,7 @@ package_dir =
     = src
 packages = find:
 install_requires =
+    celery
     flask
     pydantic
     toolchest

src/flask_container_scaffold/app_scaffold.py

@@ -1,28 +1,26 @@
-import os
+from flask_container_scaffold.base_scaffold import BaseScaffold
 
-from flask import Flask
 
-from flask_container_scaffold.app_configurator import AppConfigurator
-
-
-class AppScaffold(object):
+class AppScaffold(BaseScaffold):
 
     def __init__(self, app=None,
                  name=__name__, config=None,
                  settings_required=False,
                  instance_path=None,
                  instance_relative_config=True):
         """
-        This class provides a way to dynamically configure a Flask application.
+        This class provides compatibility with older versions of scaffold that
+        expect an instance with an 'app' attribute.
 
         :param obj app: An existing Flask application, if passed, otherwise we
             will create a new one
         :param str name: The name of the application, defaults to __name__.
         :param dict config: A dict of configuration details. This can include
             standard Flask configuration keys, like 'TESTING', or
-            'CUSTOM_SETTINGS' (which can be a string referencing a file with custom
-            configuration, or a dictionary containing any values your application
-            may need) to make them available to the application during runtime
+            'CUSTOM_SETTINGS' (which can be a string referencing a file with
+            custom configuration, or a dictionary containing any values your
+            application may need) to make them available to the application
+            during runtime
         :param bool settings_required: Whether your app requires certain
             settings be specified in a settings.cfg file
         :param str instance_path: Passthrough parameter to flask. An
@@ -35,64 +33,6 @@ def __init__(self, app=None,
             the application root.
 
         """
-        # TODO: Consider taking **kwargs here, so we can automatically support
-        # all params the flask object takes, and just pass them through.  Keep
-        # the ones we already have, as they are needed for the current code to
-        # work.
-        Flask.jinja_options = dict(Flask.jinja_options, trim_blocks=True,
-                                   lstrip_blocks=True)
-        self.app = (app or
-                    Flask(name,
-                          instance_relative_config=instance_relative_config,
-                          instance_path=instance_path))
-        self.config = config
-        self.silent = not settings_required
-        self.relative = instance_relative_config
-        self._init_app()
-
-    def _init_app(self):
-        self._load_flask_settings()
-        self._load_custom_settings()
-
-    def _load_flask_settings(self):
-        """
-        This loads the 'core' settings, ie, anything you could set directly
-        on a Flask app. These can be specified in the following order, each
-        overriding the last, if specified:
-        - via config mapping
-        - via Flask settings.cfg file
-        - via environment variable 'FLASK_SETTINGS'
-        """
-        config_not_loaded = True
-        if self.config is not None:
-            # load the config if passed in
-            self.app.config.from_mapping(self.config)
-            config_not_loaded = False
-        # load the instance config, if it exists and/or is required
-        try:
-            self.app.config.from_pyfile('settings.cfg', silent=self.silent)
-            config_not_loaded = False
-        except Exception:
-            config_not_loaded = True
-        # Load any additional config specified in the FLASK_SETTINGS file,
-        # if it exists. We only want to fail in the case where settings are
-        # required by the app.
-        if ((config_not_loaded and not self.silent) or
-                os.environ.get('FLASK_SETTINGS')):
-            self.app.config.from_envvar('FLASK_SETTINGS')
-
-    def _load_custom_settings(self):
-        """
-        Load any custom configuration for the app from:
-        - app.config['CUSTOM_SETTINGS']
-        - environment variable 'CUSTOM_SETTINGS'
-        """
-        configurator = AppConfigurator(self.app, self.relative)
-        if self.app.config.get('CUSTOM_SETTINGS') is not None:
-            # load the config if passed in
-            custom = self.app.config.get('CUSTOM_SETTINGS')
-            configurator.parse(custom)
-        # Next, load from override file, if specified
-        if os.environ.get('CUSTOM_SETTINGS') is not None:
-            custom = os.environ.get('CUSTOM_SETTINGS')
-            configurator.parse(custom)
+        super().__init__(app, name, config, settings_required,
+                         instance_path, instance_relative_config)
+        self.app = app or self.flask_app

src/flask_container_scaffold/base_scaffold.py

@@ -0,0 +1,102 @@
+import os
+
+from flask import Flask
+
+from flask_container_scaffold.app_configurator import AppConfigurator
+
+
+class BaseScaffold(object):
+
+    def __init__(self, app=None,
+                 name=__name__, config=None,
+                 settings_required=False,
+                 instance_path=None,
+                 instance_relative_config=True):
+        """
+        This base class provides a way to dynamically configure a Flask
+        application.
+
+        :param obj app: An existing Flask application, if passed, otherwise we
+            will create a new one
+        :param str name: The name of the application, defaults to __name__.
+        :param dict config: A dict of configuration details. This can include
+            standard Flask configuration keys, like 'TESTING', or
+            'CUSTOM_SETTINGS' (which can be a string referencing a file with
+            custom configuration, or a dictionary containing any values your
+            application may need) to make them available to the application
+            during runtime
+        :param bool settings_required: Whether your app requires certain
+            settings be specified in a settings.cfg file
+        :param str instance_path: Passthrough parameter to flask. An
+            alternative instance path for the application. By default
+            the folder 'instance' next to the package or module is
+            assumed to be the instance path.
+        :param bool instance_relative_config: Passthrough parameter to flask.
+            If set to True relative filenames for loading the config
+            are assumed to be relative to the instance path instead of
+            the application root.
+
+        """
+        # TODO: Consider taking **kwargs here, so we can automatically support
+        # all params the flask object takes, and just pass them through.  Keep
+        # the ones we already have, as they are needed for the current code to
+        # work.
+        Flask.jinja_options = dict(Flask.jinja_options, trim_blocks=True,
+                                   lstrip_blocks=True)
+        self.flask_app = app or Flask(
+            name,
+            instance_relative_config=instance_relative_config,
+            instance_path=instance_path,
+        )
+        self.config = config
+        self.silent = not settings_required
+        self.relative = instance_relative_config
+        self._init_app()
+
+    def _init_app(self):
+        self._load_flask_settings()
+        self._load_custom_settings()
+
+    def _load_flask_settings(self):
+        """
+        This loads the 'core' settings, ie, anything you could set directly
+        on a Flask app. These can be specified in the following order, each
+        overriding the last, if specified:
+        - via config mapping
+        - via Flask settings.cfg file
+        - via environment variable 'FLASK_SETTINGS'
+        """
+        config_not_loaded = True
+        if self.config is not None:
+            # load the config if passed in
+            self.flask_app.config.from_mapping(self.config)
+            config_not_loaded = False
+        # load the instance config, if it exists and/or is required
+        try:
+            self.flask_app.config.from_pyfile('settings.cfg',
+                                              silent=self.silent)
+            config_not_loaded = False
+        except Exception:
+            config_not_loaded = True
+        # Load any additional config specified in the FLASK_SETTINGS file,
+        # if it exists. We only want to fail in the case where settings are
+        # required by the app.
+        if ((config_not_loaded and not self.silent) or
+                os.environ.get('FLASK_SETTINGS')):
+            self.flask_app.config.from_envvar('FLASK_SETTINGS')
+
+    def _load_custom_settings(self):
+        """
+        Load any custom configuration for the app from:
+        - app.config['CUSTOM_SETTINGS']
+        - environment variable 'CUSTOM_SETTINGS'
+        """
+        configurator = AppConfigurator(self.flask_app, self.relative)
+        if self.flask_app.config.get('CUSTOM_SETTINGS') is not None:
+            # load the config if passed in
+            custom = self.flask_app.config.get('CUSTOM_SETTINGS')
+            configurator.parse(custom)
+        # Next, load from override file, if specified
+        if os.environ.get('CUSTOM_SETTINGS') is not None:
+            custom = os.environ.get('CUSTOM_SETTINGS')
+            configurator.parse(custom)

src/flask_container_scaffold/celery_scaffold.py

@@ -0,0 +1,42 @@
+from celery import Celery
+
+from flask_container_scaffold.base_scaffold import BaseScaffold
+
+
+class CeleryScaffold(BaseScaffold):
+
+    def __init__(self, flask_app=None, name=__name__, config=None,
+                 settings_required=False,
+                 instance_path=None,
+                 instance_relative_config=True):
+        """
+        This class provides both a flask 'app' and a celery 'app' that has been
+        configured via flask.
+
+        :param obj flask_app: An existing Flask application, if passed,
+            otherwise we will create a new one using BaseScaffold.
+        :param str name: The name of the application, defaults to __name__.
+        :param dict config: A dict of configuration details. This can include
+            standard Flask configuration keys, like 'TESTING', or
+            'CUSTOM_SETTINGS' (which can be a string referencing a file with
+            custom configuration, or a dictionary containing any values your
+            application may need) to make them available to the application
+            during runtime
+        :param bool settings_required: Whether your app requires certain
+            settings be specified in a settings.cfg file
+        :param str instance_path: Passthrough parameter to flask. An
+            alternative instance path for the application. By default
+            the folder 'instance' next to the package or module is
+            assumed to be the instance path.
+        :param bool instance_relative_config: Passthrough parameter to flask.
+            If set to True relative filenames for loading the config
+            are assumed to be relative to the instance path instead of
+            the application root.
+
+        """
+        super().__init__(flask_app, name, config, settings_required,
+                         instance_path, instance_relative_config)
+        self.flask_app = flask_app or self.flask_app
+        self.celery_app = Celery(self.flask_app.name)
+        self.celery_app.config_from_object(self.flask_app.config.get("CELERY"))
+        self.celery_app.set_default()

tests/unit/test_celery.py

@@ -0,0 +1,51 @@
+import pytest
+
+from celery import Celery
+from flask import Flask
+
+from flask_container_scaffold.celery_scaffold import CeleryScaffold
+
+
+def test_celery_flask_empty_config():
+    """
+    GIVEN an instance of CeleryScaffold with an empty config
+    WHEN we try to create the app
+    THEN we get a celery app and a flask app
+    """
+    scaffold = CeleryScaffold()
+    assert scaffold.flask_app is not None
+    assert isinstance(scaffold.flask_app, Flask)
+    assert scaffold.celery_app is not None
+    assert isinstance(scaffold.celery_app, Celery)
+
+
+def test_celery_broker_set():
+    """
+    GIVEN an instance of CeleryScaffold
+    AND a config with a broker url
+    WHEN we create the app
+    THEN we get a celery app with a broker url matching the config
+    """
+    config = {'CELERY': {'broker': 'pyamqp://'}}
+    scaffold = CeleryScaffold(config=config)
+    app = scaffold.celery_app
+    assert app is not None
+    assert isinstance(app, Celery)
+    assert config['CELERY']['broker'] == app.conf.find_value_for_key('broker')
+
+
+def test_celery_bad_config():
+    """
+    GIVEN an instance of CeleryScaffold
+    AND a config with a bad config item
+    WHEN we create the app
+    THEN we get a celery app
+    AND the config doesn't have the bad item
+    """
+    config = {'CELERY': {'bad_config_item': 'my_bad_config'}}
+    scaffold = CeleryScaffold(config=config)
+    app = scaffold.celery_app
+    assert app is not None
+    assert isinstance(app, Celery)
+    with pytest.raises(KeyError):
+        app.conf.find_value_for_key('bad_config_item')

tox.ini

@@ -3,12 +3,21 @@ envlist =
     py{37,38,39,310,311}
     flake8
 
+[gh-actions]
+python =
+    3.7: py37
+    3.8: py38
+    3.9: py39
+    3.10: py310
+    3.11: py311, flake8
+
 [testenv]
 passenv=HOME
 sitepackages = False
 deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
        -r{toxinidir}/dist-requirements.txt
+       py37: importlib-metadata<5.0
 commands =
     pytest --cov-report=term-missing --cov=src tests