From 08a283af5efc999d4f1e29275317a5c0c05c6f7b Mon Sep 17 00:00:00 2001 From: pgjones Date: Tue, 8 Mar 2022 21:40:48 +0000 Subject: [PATCH 1/3] Allow loading of environment variables into the config This new method will pick out any environment variables with a certain prefix and place them into the config named without the prefix. This makes it easy to use environment variables to configure the app as is now more popular than when Flask started. The prefix should ensure that the environment isn't polluted and the config isn't polluted by environment variables. I've followed the dynaconf convention of trying to parse the environment variable and then falling back to the raw value if parsing fails. --- docs/config.rst | 47 ++++++++++++++++++++++---------------------- src/flask/config.py | 46 +++++++++++++++++++++++++++++++++++++++++++ tests/test_config.py | 24 ++++++++++++++++++++++ 3 files changed, 94 insertions(+), 23 deletions(-) diff --git a/docs/config.rst b/docs/config.rst index 0b86674d..9e66e01e 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -515,9 +515,11 @@ Or from a JSON file: Configuring from Environment Variables -------------------------------------- -In addition to pointing to configuration files using environment variables, you -may find it useful (or necessary) to control your configuration values directly -from the environment. +In addition to pointing to configuration files using environment +variables, you may find it useful (or necessary) to control your +configuration values directly from the environment. Flask can be +instructed to load all environment variables starting with a specific +prefix into the config using :meth:`~flask.Config.from_prefixed_env`. Environment variables can be set in the shell before starting the server: @@ -527,8 +529,8 @@ Environment variables can be set in the shell before starting the server: .. code-block:: text - $ export SECRET_KEY="5f352379324c22463451387a0aec5d2f" - $ export MAIL_ENABLED=false + $ export FLASK_SECRET_KEY="5f352379324c22463451387a0aec5d2f" + $ export FLASK_MAIL_ENABLED=false $ flask run * Running on http://127.0.0.1:5000/ @@ -536,8 +538,8 @@ Environment variables can be set in the shell before starting the server: .. code-block:: text - $ set -x SECRET_KEY "5f352379324c22463451387a0aec5d2f" - $ set -x MAIL_ENABLED false + $ set -x FLASK_SECRET_KEY "5f352379324c22463451387a0aec5d2f" + $ set -x FLASK_MAIL_ENABLED false $ flask run * Running on http://127.0.0.1:5000/ @@ -545,8 +547,8 @@ Environment variables can be set in the shell before starting the server: .. code-block:: text - > set SECRET_KEY="5f352379324c22463451387a0aec5d2f" - > set MAIL_ENABLED=false + > set FLASK_SECRET_KEY="5f352379324c22463451387a0aec5d2f" + > set FLASK_MAIL_ENABLED=false > flask run * Running on http://127.0.0.1:5000/ @@ -554,27 +556,26 @@ Environment variables can be set in the shell before starting the server: .. code-block:: text - > $env:SECRET_KEY = "5f352379324c22463451387a0aec5d2f" - > $env:MAIL_ENABLED = "false" + > $env:FLASK_SECRET_KEY = "5f352379324c22463451387a0aec5d2f" + > $env:FLASK_MAIL_ENABLED = "false" > flask run * Running on http://127.0.0.1:5000/ -While this approach is straightforward to use, it is important to remember that -environment variables are strings -- they are not automatically deserialized -into Python types. +The variables can then be loaded and accessed via the config with a +key equal to the environment variable name without the prefix i.e. -Here is an example of a configuration file that uses environment variables:: +.. code-block:: python - import os + app.config.from_prefixed_env() + app.config["SECRET_KEY"] # Is "5f352379324c22463451387a0aec5d2f" - _mail_enabled = os.environ.get("MAIL_ENABLED", default="true") - MAIL_ENABLED = _mail_enabled.lower() in {"1", "t", "true"} - - SECRET_KEY = os.environ.get("SECRET_KEY") - - if not SECRET_KEY: - raise ValueError("No SECRET_KEY set for Flask application") +The prefix is ``FLASK_`` by default, however it is an configurable via +the ``prefix`` argument of :meth:`~flask.Config.from_prefixed_env`. +Whilst the value of any environment variable is a string, it will be +parsed before being placed into the flask config. By default the +parsing is done by json.loads, however this is configurable via the +``loads`` argument of :meth:`~flask.Config.from_prefixed_env`. Notice that any value besides an empty string will be interpreted as a boolean ``True`` value in Python, which requires care if an environment explicitly sets diff --git a/src/flask/config.py b/src/flask/config.py index 9657edc8..c02db272 100644 --- a/src/flask/config.py +++ b/src/flask/config.py @@ -1,4 +1,5 @@ import errno +import json import os import types import typing as t @@ -6,6 +7,13 @@ import typing as t from werkzeug.utils import import_string +def _json_loads(raw: t.Union[str, bytes]) -> t.Any: + try: + return json.loads(raw) + except json.JSONDecodeError: + return raw + + class ConfigAttribute: """Makes an attribute forward to the config""" @@ -97,6 +105,44 @@ class Config(dict): ) return self.from_pyfile(rv, silent=silent) + def from_prefixed_env( + self, + prefix: str = "FLASK_", + *, + loads: t.Callable[[t.Union[str, bytes]], t.Any] = _json_loads, + ) -> bool: + """Updates the config from environment variables with the prefix. + + Calling this method will result in every environment variable + starting with **prefix** being placed into the configuration + without the **prefix**. The prefix is configurable as an + argument. Note that this method updates the existing config. + + For example if there is an environment variable + ``FLASK_SECRET_KEY`` with value ``secretly`` and the prefix is + ``FLASK_`` the config will contain the key ``SECRET_KEY`` with + the value ``secretly`` after calling this method. + + The value of the environment variable will be passed to the + **loads** parameter before being placed into the config. By + default **loads** utilises the stdlib json.loads to parse the + value, falling back to the value itself on parsing error. + + :param loads: A callable that takes a str (or bytes) returns + the parsed value. + :return: Always returns ``True``. + + .. versionadded:: 2.1.0 + + """ + mapping = {} + for raw_key, value in os.environ.items(): + if raw_key.startswith(prefix): + key = raw_key[len(prefix) :] # Use removeprefix with Python 3.9 + mapping[key] = loads(value) + + return self.from_mapping(mapping) + def from_pyfile(self, filename: str, silent: bool = False) -> bool: """Updates the values in the config from a Python file. This function behaves as if the file was imported as module with the diff --git a/tests/test_config.py b/tests/test_config.py index a3cd3d25..e2d5cd63 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -38,6 +38,30 @@ def test_config_from_file(): common_object_test(app) +def test_config_from_prefixed_env(monkeypatch): + app = flask.Flask(__name__) + monkeypatch.setenv("FLASK_A", "A value") + monkeypatch.setenv("FLASK_B", "true") + monkeypatch.setenv("FLASK_C", "1") + monkeypatch.setenv("FLASK_D", "1.2") + monkeypatch.setenv("NOT_FLASK_A", "Another value") + app.config.from_prefixed_env() + assert app.config["A"] == "A value" + assert app.config["B"] is True + assert app.config["C"] == 1 + assert app.config["D"] == 1.2 + assert "Another value" not in app.config.items() + + +def test_config_from_custom_prefixed_env(monkeypatch): + app = flask.Flask(__name__) + monkeypatch.setenv("FLASK_A", "A value") + monkeypatch.setenv("NOT_FLASK_A", "Another value") + app.config.from_prefixed_env("NOT_FLASK_") + assert app.config["A"] == "Another value" + assert "A value" not in app.config.items() + + def test_config_from_mapping(): app = flask.Flask(__name__) app.config.from_mapping({"SECRET_KEY": "config", "TEST_KEY": "foo"}) From 4eb5e9455b2a8f15f3dc5065a47f8e43ac47e539 Mon Sep 17 00:00:00 2001 From: David Lord Date: Fri, 25 Mar 2022 11:00:32 -0700 Subject: [PATCH 2/3] more from_prefixed_env features * support nested dict access with "__" separator * don't specify separator in prefix * catch exceptions for any loads function --- CHANGES.rst | 4 +++ docs/config.rst | 50 +++++++++++++++++--------- src/flask/config.py | 84 +++++++++++++++++++++++++++++--------------- tests/test_config.py | 60 ++++++++++++++++++++++--------- 4 files changed, 136 insertions(+), 62 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index e471d6a8..bab6690b 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -55,6 +55,10 @@ Unreleased - From Werkzeug, for redirect responses the ``Location`` header URL will remain relative, and exclude the scheme and domain, by default. :pr:`4496` +- Add ``Config.from_prefixed_env()`` to load config values from + environment variables that start with ``FLASK_`` or another prefix. + This parses values as JSON by default, and allows setting keys in + nested dicts. :pr:`4479` Version 2.0.3 diff --git a/docs/config.rst b/docs/config.rst index 9e66e01e..c25bf83f 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -521,7 +521,8 @@ configuration values directly from the environment. Flask can be instructed to load all environment variables starting with a specific prefix into the config using :meth:`~flask.Config.from_prefixed_env`. -Environment variables can be set in the shell before starting the server: +Environment variables can be set in the shell before starting the +server: .. tabs:: @@ -561,30 +562,43 @@ Environment variables can be set in the shell before starting the server: > flask run * Running on http://127.0.0.1:5000/ -The variables can then be loaded and accessed via the config with a -key equal to the environment variable name without the prefix i.e. +The variables can then be loaded and accessed via the config with a key +equal to the environment variable name without the prefix i.e. .. code-block:: python app.config.from_prefixed_env() app.config["SECRET_KEY"] # Is "5f352379324c22463451387a0aec5d2f" -The prefix is ``FLASK_`` by default, however it is an configurable via -the ``prefix`` argument of :meth:`~flask.Config.from_prefixed_env`. +The prefix is ``FLASK_`` by default. This is configurable via the +``prefix`` argument of :meth:`~flask.Config.from_prefixed_env`. -Whilst the value of any environment variable is a string, it will be -parsed before being placed into the flask config. By default the -parsing is done by json.loads, however this is configurable via the -``loads`` argument of :meth:`~flask.Config.from_prefixed_env`. +Values will be parsed to attempt to convert them to a more specific type +than strings. By default :func:`json.loads` is used, so any valid JSON +value is possible, including lists and dicts. This is configurable via +the ``loads`` argument of :meth:`~flask.Config.from_prefixed_env`. -Notice that any value besides an empty string will be interpreted as a boolean -``True`` value in Python, which requires care if an environment explicitly sets -values intended to be ``False``. +When adding a boolean value with the default JSON parsing, only "true" +and "false", lowercase, are valid values. Keep in mind that any +non-empty string is considered ``True`` by Python. -Make sure to load the configuration very early on, so that extensions have the -ability to access the configuration when starting up. There are other methods -on the config object as well to load from individual files. For a complete -reference, read the :class:`~flask.Config` class documentation. +It is possible to set keys in nested dictionaries by separating the +keys with double underscore (``__``). Any intermediate keys that don't +exist on the parent dict will be initialized to an empty dict. + +.. code-block:: text + + $ export FLASK_MYAPI__credentials__username=user123 + +.. code-block:: python + + app.config["MYAPI"]["credentials"]["username"] # Is "user123" + +For even more config loading features, including merging, try a +dedicated library such as Dynaconf_, which includes integration with +Flask. + +.. _Dynaconf: https://www.dynaconf.com/ Configuration Best Practices @@ -604,6 +618,10 @@ that experience: limit yourself to request-only accesses to the configuration you can reconfigure the object later on as needed. +3. Make sure to load the configuration very early on, so that + extensions can access the configuration when calling ``init_app``. + + .. _config-dev-prod: Development / Production diff --git a/src/flask/config.py b/src/flask/config.py index c02db272..a266ea1d 100644 --- a/src/flask/config.py +++ b/src/flask/config.py @@ -78,7 +78,7 @@ class Config(dict): """ def __init__(self, root_path: str, defaults: t.Optional[dict] = None) -> None: - dict.__init__(self, defaults or {}) + super().__init__(defaults or {}) self.root_path = root_path def from_envvar(self, variable_name: str, silent: bool = False) -> bool: @@ -106,42 +106,68 @@ class Config(dict): return self.from_pyfile(rv, silent=silent) def from_prefixed_env( - self, - prefix: str = "FLASK_", - *, - loads: t.Callable[[t.Union[str, bytes]], t.Any] = _json_loads, + self, prefix: str = "FLASK", *, loads: t.Callable[[str], t.Any] = json.loads ) -> bool: - """Updates the config from environment variables with the prefix. + """Load any environment variables that start with ``FLASK_``, + dropping the prefix from the env key for the config key. Values + are passed through a loading function to attempt to convert them + to more specific types than strings. - Calling this method will result in every environment variable - starting with **prefix** being placed into the configuration - without the **prefix**. The prefix is configurable as an - argument. Note that this method updates the existing config. + Keys are loaded in :func:`sorted` order. - For example if there is an environment variable - ``FLASK_SECRET_KEY`` with value ``secretly`` and the prefix is - ``FLASK_`` the config will contain the key ``SECRET_KEY`` with - the value ``secretly`` after calling this method. + The default loading function attempts to parse values as any + valid JSON type, including dicts and lists. - The value of the environment variable will be passed to the - **loads** parameter before being placed into the config. By - default **loads** utilises the stdlib json.loads to parse the - value, falling back to the value itself on parsing error. + Specific items in nested dicts can be set by separating the + keys with double underscores (``__``). If an intermediate key + doesn't exist, it will be initialized to an empty dict. - :param loads: A callable that takes a str (or bytes) returns - the parsed value. - :return: Always returns ``True``. - - .. versionadded:: 2.1.0 + :param prefix: Load env vars that start with this prefix, + separated with an underscore (``_``). + :param loads: Pass each string value to this function and use + the returned value as the config value. If any error is + raised it is ignored and the value remains a string. The + default is :func:`json.loads`. + .. versionadded:: 2.1 """ - mapping = {} - for raw_key, value in os.environ.items(): - if raw_key.startswith(prefix): - key = raw_key[len(prefix) :] # Use removeprefix with Python 3.9 - mapping[key] = loads(value) + prefix = f"{prefix}_" + len_prefix = len(prefix) - return self.from_mapping(mapping) + for key in sorted(os.environ): + if not key.startswith(prefix): + continue + + value = os.environ[key] + + try: + value = loads(value) + except Exception: + # Keep the value as a string if loading failed. + pass + + # Change to key.removeprefix(prefix) on Python >= 3.9. + key = key[len_prefix:] + + if "__" not in key: + # A non-nested key, set directly. + self[key] = value + continue + + # Traverse nested dictionaries with keys separated by "__". + current = self + *parts, tail = key.split("__") + + for part in parts: + # If an intermediate dict does not exist, create it. + if part not in current: + current[part] = {} + + current = current[part] + + current[tail] = value + + return True def from_pyfile(self, filename: str, silent: bool = False) -> bool: """Updates the values in the config from a Python file. This function diff --git a/tests/test_config.py b/tests/test_config.py index e2d5cd63..22aacd36 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -38,28 +38,54 @@ def test_config_from_file(): common_object_test(app) -def test_config_from_prefixed_env(monkeypatch): +def test_from_prefixed_env(monkeypatch): + monkeypatch.setenv("FLASK_STRING", "value") + monkeypatch.setenv("FLASK_BOOL", "true") + monkeypatch.setenv("FLASK_INT", "1") + monkeypatch.setenv("FLASK_FLOAT", "1.2") + monkeypatch.setenv("FLASK_LIST", "[1, 2]") + monkeypatch.setenv("FLASK_DICT", '{"k": "v"}') + monkeypatch.setenv("NOT_FLASK_OTHER", "other") + app = flask.Flask(__name__) - monkeypatch.setenv("FLASK_A", "A value") - monkeypatch.setenv("FLASK_B", "true") - monkeypatch.setenv("FLASK_C", "1") - monkeypatch.setenv("FLASK_D", "1.2") - monkeypatch.setenv("NOT_FLASK_A", "Another value") app.config.from_prefixed_env() - assert app.config["A"] == "A value" - assert app.config["B"] is True - assert app.config["C"] == 1 - assert app.config["D"] == 1.2 - assert "Another value" not in app.config.items() + + assert app.config["STRING"] == "value" + assert app.config["BOOL"] is True + assert app.config["INT"] == 1 + assert app.config["FLOAT"] == 1.2 + assert app.config["LIST"] == [1, 2] + assert app.config["DICT"] == {"k": "v"} + assert "OTHER" not in app.config -def test_config_from_custom_prefixed_env(monkeypatch): +def test_from_prefixed_env_custom_prefix(monkeypatch): + monkeypatch.setenv("FLASK_A", "a") + monkeypatch.setenv("NOT_FLASK_A", "b") + app = flask.Flask(__name__) - monkeypatch.setenv("FLASK_A", "A value") - monkeypatch.setenv("NOT_FLASK_A", "Another value") - app.config.from_prefixed_env("NOT_FLASK_") - assert app.config["A"] == "Another value" - assert "A value" not in app.config.items() + app.config.from_prefixed_env("NOT_FLASK") + + assert app.config["A"] == "b" + + +def test_from_prefixed_env_nested(monkeypatch): + monkeypatch.setenv("FLASK_EXIST__ok", "other") + monkeypatch.setenv("FLASK_EXIST__inner__ik", "2") + monkeypatch.setenv("FLASK_EXIST__new__more", '{"k": false}') + monkeypatch.setenv("FLASK_NEW__K", "v") + + app = flask.Flask(__name__) + app.config["EXIST"] = {"ok": "value", "flag": True, "inner": {"ik": 1}} + app.config.from_prefixed_env() + + assert app.config["EXIST"] == { + "ok": "other", + "flag": True, + "inner": {"ik": 2}, + "new": {"more": {"k": False}}, + } + assert app.config["NEW"] == {"K": "v"} def test_config_from_mapping(): From e75d575361841404e0b1f89c19dc7de6b325cc67 Mon Sep 17 00:00:00 2001 From: David Lord Date: Fri, 25 Mar 2022 12:05:07 -0700 Subject: [PATCH 3/3] windows env vars are uppercase --- docs/config.rst | 9 ++++++--- tests/test_config.py | 24 ++++++++++++++++++------ 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/docs/config.rst b/docs/config.rst index c25bf83f..7a5e4da1 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -594,9 +594,12 @@ exist on the parent dict will be initialized to an empty dict. app.config["MYAPI"]["credentials"]["username"] # Is "user123" -For even more config loading features, including merging, try a -dedicated library such as Dynaconf_, which includes integration with -Flask. +On Windows, environment variable keys are always uppercase, therefore +the above example would end up as ``MYAPI__CREDENTIALS__USERNAME``. + +For even more config loading features, including merging and +case-insensitive Windows support, try a dedicated library such as +Dynaconf_, which includes integration with Flask. .. _Dynaconf: https://www.dynaconf.com/ diff --git a/tests/test_config.py b/tests/test_config.py index 22aacd36..bbe4f1e2 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -79,12 +79,24 @@ def test_from_prefixed_env_nested(monkeypatch): app.config["EXIST"] = {"ok": "value", "flag": True, "inner": {"ik": 1}} app.config.from_prefixed_env() - assert app.config["EXIST"] == { - "ok": "other", - "flag": True, - "inner": {"ik": 2}, - "new": {"more": {"k": False}}, - } + if os.name != "nt": + assert app.config["EXIST"] == { + "ok": "other", + "flag": True, + "inner": {"ik": 2}, + "new": {"more": {"k": False}}, + } + else: + # Windows env var keys are always uppercase. + assert app.config["EXIST"] == { + "ok": "value", + "OK": "other", + "flag": True, + "inner": {"ik": 1}, + "INNER": {"IK": 2}, + "NEW": {"MORE": {"k": False}}, + } + assert app.config["NEW"] == {"K": "v"}