From c43edfc7c08b8dbe883761bec14fa75ee7db1bf3 Mon Sep 17 00:00:00 2001 From: David Lord Date: Tue, 7 Apr 2020 07:00:15 -0700 Subject: [PATCH 1/3] remove simplejson - remove encoding detection backport, json.loads supports it directly - use str.translate instead of multiple str.replace --- CHANGES.rst | 3 ++ docs/api.rst | 19 ++------ docs/installation.rst | 4 -- src/flask/json/__init__.py | 89 +++++++++----------------------------- tests/test_helpers.py | 21 --------- tox.ini | 4 +- 6 files changed, 27 insertions(+), 113 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 6a6db80f..a332e406 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -19,6 +19,9 @@ Unreleased 200 OK and an empty file. :issue:`3358` - When using ad-hoc certificates, check for the cryptography library instead of PyOpenSSL. :pr:`3492` +- JSON support no longer uses simplejson if it's installed. To use + another JSON module, override ``app.json_encoder`` and + ``json_decoder``. :issue:`3555` Version 1.1.2 diff --git a/docs/api.rst b/docs/api.rst index 801a65bd..97c1c7a1 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -284,22 +284,9 @@ JSON Support .. module:: flask.json -Flask uses ``simplejson`` for the JSON implementation. Since simplejson -is provided by both the standard library as well as extension, Flask will -try simplejson first and then fall back to the stdlib json module. On top -of that it will delegate access to the current application's JSON encoders -and decoders for easier customization. - -So for starters instead of doing:: - - try: - import simplejson as json - except ImportError: - import json - -You can instead just do this:: - - from flask import json +Flask uses the built-in :mod:`json` module for the JSON implementation. +It will delegate access to the current application's JSON encoders and +decoders for easier customization. For usage examples, read the :mod:`json` documentation in the standard library. The following extensions are by default applied to the stdlib's diff --git a/docs/installation.rst b/docs/installation.rst index c99d82cf..e02e111e 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -39,16 +39,12 @@ These distributions will not be installed automatically. Flask will detect and use them if you install them. * `Blinker`_ provides support for :doc:`signals`. -* `SimpleJSON`_ is a fast JSON implementation that is compatible with - Python's ``json`` module. It is preferred for JSON operations if it is - installed. * `python-dotenv`_ enables support for :ref:`dotenv` when running ``flask`` commands. * `Watchdog`_ provides a faster, more efficient reloader for the development server. .. _Blinker: https://pythonhosted.org/blinker/ -.. _SimpleJSON: https://simplejson.readthedocs.io/ .. _python-dotenv: https://github.com/theskumar/python-dotenv#readme .. _watchdog: https://pythonhosted.org/watchdog/ diff --git a/src/flask/json/__init__.py b/src/flask/json/__init__.py index 5c698ef0..0c284aee 100644 --- a/src/flask/json/__init__.py +++ b/src/flask/json/__init__.py @@ -1,10 +1,9 @@ -import codecs import io +import json as _json import uuid from datetime import date from datetime import datetime -from itsdangerous import json as _json from jinja2 import Markup from werkzeug.http import http_date @@ -17,10 +16,6 @@ except ImportError: # Python < 3.7 dataclasses = None -# Figure out if simplejson escapes slashes. This behavior was changed -# from one version to another without reason. -_slash_escape = "\\/" not in _json.dumps("/") - __all__ = [ "dump", @@ -93,7 +88,7 @@ class JSONEncoder(_json.JSONEncoder): class JSONDecoder(_json.JSONDecoder): """The default JSON decoder. This one does not change the behavior from - the default simplejson decoder. Consult the :mod:`json` documentation + the default decoder. Consult the :mod:`json` documentation for more information. This decoder is not only used for the load functions of this module but also :attr:`~flask.Request`. """ @@ -133,49 +128,6 @@ def _load_arg_defaults(kwargs, app=None): kwargs.setdefault("cls", JSONDecoder) -def detect_encoding(data): - """Detect which UTF codec was used to encode the given bytes. - - The latest JSON standard (:rfc:`8259`) suggests that only UTF-8 is - accepted. Older documents allowed 8, 16, or 32. 16 and 32 can be big - or little endian. Some editors or libraries may prepend a BOM. - - :param data: Bytes in unknown UTF encoding. - :return: UTF encoding name - """ - head = data[:4] - - if head[:3] == codecs.BOM_UTF8: - return "utf-8-sig" - - if b"\x00" not in head: - return "utf-8" - - if head in (codecs.BOM_UTF32_BE, codecs.BOM_UTF32_LE): - return "utf-32" - - if head[:2] in (codecs.BOM_UTF16_BE, codecs.BOM_UTF16_LE): - return "utf-16" - - if len(head) == 4: - if head[:3] == b"\x00\x00\x00": - return "utf-32-be" - - if head[::2] == b"\x00\x00": - return "utf-16-be" - - if head[1:] == b"\x00\x00\x00": - return "utf-32-le" - - if head[1::2] == b"\x00\x00": - return "utf-16-le" - - if len(head) == 2: - return "utf-16-be" if head.startswith(b"\x00") else "utf-16-le" - - return "utf-8" - - def dumps(obj, app=None, **kwargs): """Serialize ``obj`` to a JSON-formatted string. If there is an app context pushed, use the current app's configured encoder @@ -183,8 +135,7 @@ def dumps(obj, app=None, **kwargs): :class:`JSONEncoder`. Takes the same arguments as the built-in :func:`json.dumps`, and - does some extra configuration based on the application. If the - simplejson package is installed, it is preferred. + does some extra configuration based on the application. :param obj: Object to serialize to JSON. :param app: App instance to use to configure the JSON encoder. @@ -200,8 +151,10 @@ def dumps(obj, app=None, **kwargs): _dump_arg_defaults(kwargs, app=app) encoding = kwargs.pop("encoding", None) rv = _json.dumps(obj, **kwargs) + if encoding is not None and isinstance(rv, str): rv = rv.encode(encoding) + return rv @@ -209,8 +162,10 @@ def dump(obj, fp, app=None, **kwargs): """Like :func:`dumps` but writes into a file object.""" _dump_arg_defaults(kwargs, app=app) encoding = kwargs.pop("encoding", None) + if encoding is not None: fp = _wrap_writer_for_text(fp, encoding) + _json.dump(obj, fp, **kwargs) @@ -221,8 +176,7 @@ def loads(s, app=None, **kwargs): default :class:`JSONDecoder`. Takes the same arguments as the built-in :func:`json.loads`, and - does some extra configuration based on the application. If the - simplejson package is installed, it is preferred. + does some extra configuration based on the application. :param s: JSON string to deserialize. :param app: App instance to use to configure the JSON decoder. @@ -236,21 +190,27 @@ def loads(s, app=None, **kwargs): context for configuration. """ _load_arg_defaults(kwargs, app=app) - if isinstance(s, bytes): - encoding = kwargs.pop("encoding", None) - if encoding is None: - encoding = detect_encoding(s) + encoding = kwargs.pop("encoding", None) + + if encoding is not None and isinstance(s, bytes): s = s.decode(encoding) + return _json.loads(s, **kwargs) def load(fp, app=None, **kwargs): """Like :func:`loads` but reads from a file object.""" _load_arg_defaults(kwargs, app=app) - fp = _wrap_reader_for_text(fp, kwargs.pop("encoding", None) or "utf-8") + encoding = kwargs.pop("encoding", None) + fp = _wrap_reader_for_text(fp, encoding or "utf-8") return _json.load(fp, **kwargs) +_htmlsafe_map = str.maketrans( + {"<": "\\u003c", ">": "\\u003e", "&": "\\u0026", "'": "\\u0027"} +) + + def htmlsafe_dumps(obj, **kwargs): """Works exactly like :func:`dumps` but is safe for use in ``") assert rv == '"\\u003c/script\\u003e"' - assert type(rv) is str rv = render('{{ ""|tojson }}') assert rv == '"\\u003c/script\\u003e"' rv = render('{{ "<\0/script>"|tojson }}') From 756902cca1bab981fbec0ea759d93239d9a711f0 Mon Sep 17 00:00:00 2001 From: David Lord Date: Tue, 7 Apr 2020 12:33:00 -0700 Subject: [PATCH 3/3] update json docs --- CHANGES.rst | 6 +- docs/api.rst | 42 +++---- src/flask/json/__init__.py | 217 ++++++++++++++++++++----------------- src/flask/json/tag.py | 8 +- 4 files changed, 140 insertions(+), 133 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index a332e406..23bc99ec 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -6,6 +6,9 @@ Version 2.0.0 Unreleased - Drop support for Python 2 and 3.5. +- JSON support no longer uses simplejson. To use another JSON module, + override ``app.json_encoder`` and ``json_decoder``. :issue:`3555` +- The ``encoding`` option to JSON functions is deprecated. :pr:`3562` - Add :meth:`sessions.SessionInterface.get_cookie_name` to allow setting the session cookie name dynamically. :pr:`3369` - Add :meth:`Config.from_file` to load config using arbitrary file @@ -19,9 +22,6 @@ Unreleased 200 OK and an empty file. :issue:`3358` - When using ad-hoc certificates, check for the cryptography library instead of PyOpenSSL. :pr:`3492` -- JSON support no longer uses simplejson if it's installed. To use - another JSON module, override ``app.json_encoder`` and - ``json_decoder``. :issue:`3555` Version 1.1.2 diff --git a/docs/api.rst b/docs/api.rst index 97c1c7a1..351d9b87 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -279,45 +279,34 @@ Message Flashing .. autofunction:: get_flashed_messages + JSON Support ------------ .. module:: flask.json -Flask uses the built-in :mod:`json` module for the JSON implementation. -It will delegate access to the current application's JSON encoders and -decoders for easier customization. +Flask uses the built-in :mod:`json` module for handling JSON. It will +use the current blueprint's or application's JSON encoder and decoder +for easier customization. By default it handles some extra data types: -For usage examples, read the :mod:`json` documentation in the standard -library. The following extensions are by default applied to the stdlib's -JSON module: +- :class:`datetime.datetime` and :class:`datetime.date` are serialized + to :rfc:`822` strings. This is the same as the HTTP date format. +- :class:`uuid.UUID` is serialized to a string. +- :class:`dataclasses.dataclass` is passed to + :func:`dataclasses.asdict`. +- :class:`~markupsafe.Markup` (or any object with a ``__html__`` + method) will call the ``__html__`` method to get a string. -1. ``datetime`` objects are serialized as :rfc:`822` strings. -2. Any object with an ``__html__`` method (like :class:`~flask.Markup`) - will have that method called and then the return value is serialized - as string. - -The :func:`~htmlsafe_dumps` function of this json module is also available -as a filter called ``|tojson`` in Jinja2. Note that in versions of Flask prior -to Flask 0.10, you must disable escaping with ``|safe`` if you intend to use -``|tojson`` output inside ``script`` tags. In Flask 0.10 and above, this -happens automatically (but it's harmless to include ``|safe`` anyway). +:func:`~htmlsafe_dumps` is also available as the ``|tojson`` template +filter. The filter marks the output with ``|safe`` so it can be used +inside ``script`` tags. .. sourcecode:: html+jinja -.. admonition:: Auto-Sort JSON Keys - - The configuration variable :data:`JSON_SORT_KEYS` can be set to - ``False`` to stop Flask from auto-sorting keys. By default sorting - is enabled and outside of the app context sorting is turned on. - - Notice that disabling key sorting can cause issues when using - content based HTTP caches and Python's hash randomization feature. - .. autofunction:: jsonify .. autofunction:: dumps @@ -336,6 +325,7 @@ happens automatically (but it's harmless to include ``|safe`` anyway). .. automodule:: flask.json.tag + Template Rendering ------------------ diff --git a/src/flask/json/__init__.py b/src/flask/json/__init__.py index 3e9c21a6..6ef22e26 100644 --- a/src/flask/json/__init__.py +++ b/src/flask/json/__init__.py @@ -19,33 +19,27 @@ except ImportError: class JSONEncoder(_json.JSONEncoder): - """The default Flask JSON encoder. This one extends the default - encoder by also supporting ``datetime``, ``UUID``, ``dataclasses``, - and ``Markup`` objects. + """The default JSON encoder. Handles extra types compared to the + built-in :class:`json.JSONEncoder`. - ``datetime`` objects are serialized as RFC 822 datetime strings. - This is the same as the HTTP date format. + - :class:`datetime.datetime` and :class:`datetime.date` are + serialized to :rfc:`822` strings. This is the same as the HTTP + date format. + - :class:`uuid.UUID` is serialized to a string. + - :class:`dataclasses.dataclass` is passed to + :func:`dataclasses.asdict`. + - :class:`~markupsafe.Markup` (or any object with a ``__html__`` + method) will call the ``__html__`` method to get a string. - In order to support more data types, override the :meth:`default` - method. + Assign a subclass of this to :attr:`flask.Flask.json_encoder` or + :attr:`flask.Blueprint.json_encoder` to override the default. """ def default(self, o): - """Implement this method in a subclass such that it returns a - serializable object for ``o``, or calls the base implementation (to - raise a :exc:`TypeError`). - - For example, to support arbitrary iterators, you could implement - default like this:: - - def default(self, o): - try: - iterable = iter(o) - except TypeError: - pass - else: - return list(iterable) - return JSONEncoder.default(self, o) + """Convert ``o`` to a JSON serializable type. See + :meth:`json.JSONEncoder.default`. Python does not support + overriding how basic types like ``str`` or ``list`` are + serialized, they are handled before this method. """ if isinstance(o, datetime): return http_date(o.utctimetuple()) @@ -61,10 +55,13 @@ class JSONEncoder(_json.JSONEncoder): class JSONDecoder(_json.JSONDecoder): - """The default JSON decoder. This one does not change the behavior from - the default decoder. Consult the :mod:`json` documentation - for more information. This decoder is not only used for the load - functions of this module but also :attr:`~flask.Request`. + """The default JSON decoder. + + This does not change any behavior from the built-in + :class:`json.JSONDecoder`. + + Assign a subclass of this to :attr:`flask.Flask.json_decoder` or + :attr:`flask.Blueprint.json_decoder` to override the default. """ @@ -98,22 +95,20 @@ def _load_arg_defaults(kwargs, app=None): def dumps(obj, app=None, **kwargs): - """Serialize ``obj`` to a JSON-formatted string. If there is an - app context pushed, use the current app's configured encoder - (:attr:`~flask.Flask.json_encoder`), or fall back to the default - :class:`JSONEncoder`. + """Serialize an object to a string of JSON. - Takes the same arguments as the built-in :func:`json.dumps`, and - does some extra configuration based on the application. + Takes the same arguments as the built-in :func:`json.dumps`, with + some defaults from application configuration. :param obj: Object to serialize to JSON. - :param app: App instance to use to configure the JSON encoder. - Uses ``current_app`` if not given, and falls back to the default - encoder when not in an app context. - :param kwargs: Extra arguments passed to :func:`json.dumps`. + :param app: Use this app's config instead of the active app context + or defaults. + :param kwargs: Extra arguments passed to func:`json.dumps`. + + .. versionchanged:: 2.0 + ``encoding`` is deprecated and will be removed in 2.1. .. versionchanged:: 1.0.3 - ``app`` can be passed directly, rather than requiring an app context for configuration. """ @@ -135,7 +130,21 @@ def dumps(obj, app=None, **kwargs): def dump(obj, fp, app=None, **kwargs): - """Like :func:`dumps` but writes into a file object.""" + """Serialize an object to JSON written to a file object. + + Takes the same arguments as the built-in :func:`json.dump`, with + some defaults from application configuration. + + :param obj: Object to serialize to JSON. + :param fp: File object to write JSON to. + :param app: Use this app's config instead of the active app context + or defaults. + :param kwargs: Extra arguments passed to func:`json.dump`. + + .. versionchanged:: 2.0 + Writing to a binary file, and the ``encoding`` argument, is + deprecated and will be removed in 2.1. + """ _dump_arg_defaults(kwargs, app=app) encoding = kwargs.pop("encoding", None) show_warning = encoding is not None @@ -158,22 +167,21 @@ def dump(obj, fp, app=None, **kwargs): def loads(s, app=None, **kwargs): - """Deserialize an object from a JSON-formatted string ``s``. If - there is an app context pushed, use the current app's configured - decoder (:attr:`~flask.Flask.json_decoder`), or fall back to the - default :class:`JSONDecoder`. + """Deserialize an object from a string of JSON. - Takes the same arguments as the built-in :func:`json.loads`, and - does some extra configuration based on the application. + Takes the same arguments as the built-in :func:`json.loads`, with + some defaults from application configuration. :param s: JSON string to deserialize. - :param app: App instance to use to configure the JSON decoder. - Uses ``current_app`` if not given, and falls back to the default - encoder when not in an app context. - :param kwargs: Extra arguments passed to :func:`json.dumps`. + :param app: Use this app's config instead of the active app context + or defaults. + :param kwargs: Extra arguments passed to func:`json.dump`. + + .. versionchanged:: 2.0 + ``encoding`` is deprecated and will be removed in 2.1. The data + must be a string or UTF-8 bytes. .. versionchanged:: 1.0.3 - ``app`` can be passed directly, rather than requiring an app context for configuration. """ @@ -195,7 +203,20 @@ def loads(s, app=None, **kwargs): def load(fp, app=None, **kwargs): - """Like :func:`loads` but reads from a file object.""" + """Deserialize an object from JSON read from a file object. + + Takes the same arguments as the built-in :func:`json.load`, with + some defaults from application configuration. + + :param fp: File object to read JSON from. + :param app: Use this app's config instead of the active app context + or defaults. + :param kwargs: Extra arguments passed to func:`json.load`. + + .. versionchanged:: 2.0 + ``encoding`` is deprecated and will be removed in 2.1. The file + must be text mode, or binary mode with UTF-8 bytes. + """ _load_arg_defaults(kwargs, app=app) encoding = kwargs.pop("encoding", None) @@ -219,84 +240,80 @@ _htmlsafe_map = str.maketrans( def htmlsafe_dumps(obj, **kwargs): - """Works exactly like :func:`dumps` but is safe for use in ``