Binary files 2.1.0~rc1-3/build_helpers/bin/antlr-4.8-complete.jar and 2.2.2-1/build_helpers/bin/antlr-4.8-complete.jar differ Binary files 2.1.0~rc1-3/build_helpers/bin/antlr-4.9.3-complete.jar and 2.2.2-1/build_helpers/bin/antlr-4.9.3-complete.jar differ diff -pruN 2.1.0~rc1-3/build_helpers/build_helpers.py 2.2.2-1/build_helpers/build_helpers.py --- 2.1.0~rc1-3/build_helpers/build_helpers.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/build_helpers/build_helpers.py 2022-05-27 21:36:40.000000000 +0000 @@ -10,7 +10,7 @@ from pathlib import Path from typing import List, Optional from setuptools import Command -from setuptools.command import build_py, develop, sdist # type: ignore +from setuptools.command import build_py, develop, sdist class ANTLRCommand(Command): # type: ignore # pragma: no cover @@ -30,7 +30,7 @@ class ANTLRCommand(Command): # type: ig command = [ "java", "-jar", - str(build_dir / "bin" / "antlr-4.8-complete.jar"), + str(build_dir / "bin" / "antlr-4.9.3-complete.jar"), "-Dlanguage=Python3", "-o", str(project_root / "omegaconf" / "grammar" / "gen"), @@ -53,7 +53,7 @@ class ANTLRCommand(Command): # type: ig pass -class BuildPyCommand(build_py.build_py): # type: ignore # pragma: no cover +class BuildPyCommand(build_py.build_py): # pragma: no cover def run(self) -> None: if not self.dry_run: self.run_command("clean") @@ -79,7 +79,6 @@ class CleanCommand(Command): # type: ig "^omegaconf\\.egg-info$", "\\.eggs$", "^\\.mypy_cache$", - "^\\.nox$", "^\\.pytest_cache$", ".*/__pycache__$", "^__pycache__$", @@ -107,16 +106,16 @@ class CleanCommand(Command): # type: ig pass -class DevelopCommand(develop.develop): # type: ignore # pragma: no cover - def run(self) -> None: +class DevelopCommand(develop.develop): # pragma: no cover + def run(self) -> None: # type: ignore if not self.dry_run: run_antlr(self) develop.develop.run(self) -class SDistCommand(sdist.sdist): # type: ignore # pragma: no cover +class SDistCommand(sdist.sdist): # pragma: no cover def run(self) -> None: - if not self.dry_run: + if not self.dry_run: # type: ignore self.run_command("clean") run_antlr(self) sdist.sdist.run(self) diff -pruN 2.1.0~rc1-3/.circleci/config.yml 2.2.2-1/.circleci/config.yml --- 2.1.0~rc1-3/.circleci/config.yml 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/.circleci/config.yml 2022-05-27 21:36:40.000000000 +0000 @@ -38,4 +38,4 @@ workflows: - test_linux: matrix: parameters: - py_version: ["3.6", "3.7", "3.8", "3.9"] + py_version: ["3.6", "3.7", "3.8", "3.9", "3.10"] diff -pruN 2.1.0~rc1-3/CITATION.cff 2.2.2-1/CITATION.cff --- 2.1.0~rc1-3/CITATION.cff 1970-01-01 00:00:00.000000000 +0000 +++ 2.2.2-1/CITATION.cff 2022-05-27 21:36:40.000000000 +0000 @@ -0,0 +1,17 @@ +# file: CITATION.cff +cff-version: 1.2.0 +message: "If you use this software, please cite it as below." +authors: +- family-names: "Yadan" + given-names: "Omry" + orcid: "https://orcid.org/0000-0002-1871-7216" +- family-names: "Sommer-Simpson" + given-names: "Jasha" + orcid: "https://orcid.org/0000-0002-1397-6454" +- family-names: "Delalleau" + given-names: "Olivier" + orcid: "https://orcid.org/0000-0002-0610-7226" +title: "omegaconf" +#doi: 10.5281/zenodo.1234 +date-released: 2019-11-19 +url: "https://github.com/omry/omegaconf" diff -pruN 2.1.0~rc1-3/CONTRIBUTING.md 2.2.2-1/CONTRIBUTING.md --- 2.1.0~rc1-3/CONTRIBUTING.md 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/CONTRIBUTING.md 2022-05-27 21:36:40.000000000 +0000 @@ -18,8 +18,8 @@ pre-commit will verify your code lints c OmegaConf is compatible with Python 3.6.4 and newer. Unfortunately Mac comes with older versions. One way to install multiple Python versions on Mac to to use pyenv. -The instructions [here](https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/MAC_SETUP.md) -will provide full details. It shows how to use pyenv on mac to install multiple versions of Python and have +The instructions [here](https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/MAC_SETUP.md) +will provide full details. It shows how to use pyenv on mac to install multiple versions of Python and have pyenv make specific versions available in specific directories automatically. This plays well with Conda, which supports a single Python version. Pyenv will provide the versions not installed by Conda (which are used when running nox). @@ -35,37 +35,42 @@ Sessions defined in /home/omry/dev/omega * omegaconf-3.7 * omegaconf-3.8 * omegaconf-3.9 +* omegaconf-3.10 * docs * coverage-3.6 * coverage-3.7 * coverage-3.8 * coverage-3.9 +* coverage-3.10 * lint-3.6 * lint-3.7 * lint-3.8 * lint-3.9 +* lint-3.10 * test_jupyter_notebook-3.6 * test_jupyter_notebook-3.7 * test_jupyter_notebook-3.8 * test_jupyter_notebook-3.9 +* test_jupyter_notebook-3.10 ``` + To run a specific session use `-s`, for example `nox -s lint` will run linting OmegaConf is formatted with black, to format your code automatically use `black .` -Imports are sorted using isort, use `isort .` to sort all imports prior to pushing. +Imports are sorted using isort, use `isort .` to sort all imports prior to pushing. To build the docs execute `nox -s docs` or `make`(inside docs folder). Make gives you different options, for example, you can build the docs as html files with `make html`. Once the docs are built you can open `index.html` in the build directory to view the generated docs with your browser. ### Modifying Jupyter notebook In order to change the Jupyter notebook you first need to open it with `jupyter notebook`. -Change the cell you want and then, execute it so the expected output is shown. -Note that the output after you execute the cell is saved as expected ouput for further +Change the cell you want and then, execute it so the expected output is shown. +Note that the output after you execute the cell is saved as expected ouput for further testing. -In case that the in[number] of cells aren't in order you should go to the +In case that the in[number] of cells aren't in order you should go to the kernel in the toolbar and restart it. diff -pruN 2.1.0~rc1-3/debian/changelog 2.2.2-1/debian/changelog --- 2.1.0~rc1-3/debian/changelog 2022-05-31 08:45:28.000000000 +0000 +++ 2.2.2-1/debian/changelog 2022-08-02 15:45:50.000000000 +0000 @@ -1,3 +1,13 @@ +python-omegaconf (2.2.2-1) unstable; urgency=medium + + * New upstream release. + * Run unit tests at build time. + * Add autopkgtest. + * Do not package /usr/lib/python3/dist-packages/pydevd_plugins/__init__.py + already in pydevd (Closes: #1016510). + + -- Thomas Goirand Tue, 02 Aug 2022 17:45:50 +0200 + python-omegaconf (2.1.0~rc1-3) unstable; urgency=medium * Defines http_proxy=127.0.0.1:9 to avoid internet trafic. diff -pruN 2.1.0~rc1-3/debian/control 2.2.2-1/debian/control --- 2.1.0~rc1-3/debian/control 2022-05-31 08:45:28.000000000 +0000 +++ 2.2.2-1/debian/control 2022-08-02 15:45:50.000000000 +0000 @@ -13,7 +13,10 @@ Build-Depends: Build-Depends-Indep: default-jre, python3-antlr4, + python3-pydevd, + python3-pytest, python3-pytest-runner, + python3-pytest-mock, python3-yaml, Standards-Version: 4.5.1 Vcs-Browser: https://salsa.debian.org/openstack-team/python/python-omegaconf diff -pruN 2.1.0~rc1-3/debian/rules 2.2.2-1/debian/rules --- 2.1.0~rc1-3/debian/rules 2022-05-31 08:45:28.000000000 +0000 +++ 2.2.2-1/debian/rules 2022-08-02 15:45:50.000000000 +0000 @@ -22,14 +22,16 @@ override_dh_auto_build: override_dh_auto_install: pkgos-dh_auto_install --no-py2 --in-tmp - ifeq (,$(findstring nocheck, $(DEB_BUILD_OPTIONS))) - echo "No tests for now: missing pydevd package in Debian" + for i in `py3versions -rv` ; do \ + python$$i -m pytest tests -v ; \ + done endif +override_dh_install: + dh_install + rm $(CURDIR)/debian/python3-omegaconf/usr/lib/python3/dist-packages/pydevd_plugins/__init__.py + rm $(CURDIR)/debian/python3-omegaconf/usr/lib/python3/dist-packages/pydevd_plugins/extensions/__init__.py override_dh_auto_test: echo "Do nothing..." - - - diff -pruN 2.1.0~rc1-3/debian/tests/control 2.2.2-1/debian/tests/control --- 2.1.0~rc1-3/debian/tests/control 1970-01-01 00:00:00.000000000 +0000 +++ 2.2.2-1/debian/tests/control 2022-08-02 15:45:50.000000000 +0000 @@ -0,0 +1,4 @@ +Tests: unittests +Depends: + @builddeps@, +Restrictions: allow-stderr needs-root diff -pruN 2.1.0~rc1-3/debian/tests/unittests 2.2.2-1/debian/tests/unittests --- 2.1.0~rc1-3/debian/tests/unittests 1970-01-01 00:00:00.000000000 +0000 +++ 2.2.2-1/debian/tests/unittests 2022-08-02 15:45:50.000000000 +0000 @@ -0,0 +1,14 @@ +#!/bin/sh + +set -e +set -x + +CWD=$(pwd) +PYTHON3S=$(py3versions -vs) +for i in ${PYTHON3S} ; do + python${i} setup.py install --install-layout=deb --root ${CWD}/debian/tmp + PYTHONPATH=${CWD}/debian/tmp/usr/lib/python3/dist-packages \ + http_proxy=127.0.0.1:9 https_proxy=127.0.0.9:9 \ + HTTP_PROXY=127.0.0.1:9 HTTPS_PROXY=127.0.0.1:9 \ + PYTHON=python${i} python${i} -m pytest tests -v +done diff -pruN 2.1.0~rc1-3/docs/source/custom_resolvers.rst 2.2.2-1/docs/source/custom_resolvers.rst --- 2.1.0~rc1-3/docs/source/custom_resolvers.rst 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/docs/source/custom_resolvers.rst 2022-05-27 21:36:40.000000000 +0000 @@ -134,7 +134,7 @@ This is in contrast to the sum we define >>> def sum2(a, b, *, _parent_): ... return _parent_.get(a, 0) + _parent_.get(b, 0) - >>> OmegaConf.register_new_resolver("sum2", sum2, use_cache=False) + >>> OmegaConf.register_new_resolver("sum2", sum2) >>> cfg = OmegaConf.create( ... { ... "node": { @@ -156,7 +156,7 @@ Built-in resolvers .. _oc.env: -oc.env +oc.env ^^^^^^ Access to environment variables is supported using ``oc.env``: @@ -176,7 +176,7 @@ Input YAML file: You can specify a default value to use in case the environment variable is not set. In such a case, the default value is converted to a string using ``str(default)``, -unless it is ``null`` (representing Python ``None``) - in which case ``None`` is returned. +unless it is ``null`` (representing Python ``None``) - in which case ``None`` is returned. The following example falls back to default passwords when ``DB_PASSWORD`` is not defined: @@ -221,7 +221,7 @@ oc.create ... "dict_config_env": "${oc.create:${oc.env:YAML_ENV}}", ... } ... ) - >>> os.environ["YAML_ENV"] = "A: 10\nb: 20\nC: ${.A}" + >>> os.environ["YAML_ENV"] = "A: 10\nb: 20\nC: ${.A}" >>> show(cfg.plain_dict) # `make_dict` returns a Python dict type: dict, value: {'a': 10} >>> show(cfg.dict_config) # `oc.create` converts it to DictConfig @@ -246,11 +246,11 @@ It takes two parameters: .. doctest:: >>> conf = OmegaConf.create({ - ... "rusty_key": "${oc.deprecated:shiny_key}", + ... "rusty_key": "${oc.deprecated:shiny_key}", ... "custom_msg": "${oc.deprecated:shiny_key, 'Use $NEW_KEY'}", ... "shiny_key": 10 ... }) - >>> # Accessing rusty_key will issue a deprecation warning + >>> # Accessing rusty_key will issue a deprecation warning >>> # and return the new value automatically >>> warning = "'rusty_key' is deprecated. Change your" \ ... " code and config to use 'shiny_key'" @@ -264,13 +264,14 @@ It takes two parameters: oc.decode ^^^^^^^^^ -With ``oc.decode``, strings can be converted into their corresponding data types using the OmegaConf grammar. -This grammar recognizes typical data types like ``bool``, ``int``, ``float``, ``dict`` and ``list``, -e.g. ``"true"``, ``"1"``, ``"1e-3"``, ``"{a: b}"``, ``"[a, b, c]"``. +With ``oc.decode``, strings can be converted into their corresponding data types using +the :ref:`"element" parser rule of the OmegaConf grammar`. +This grammar recognizes typical data types like ``bool``, ``int``, ``float``, ``bytes``, ``dict`` and ``list``, +e.g. ``"true"``, ``"1"``, ``"1e-3"``, ``b"123"``, ``"{a: b}"``, ``"[a, b, c]"``. Note that: -- In general input strings provided to ``oc.decode`` should be quoted, since only a subset of the characters is allowed in unquoted strings. +- In most cases input strings provided to ``oc.decode`` should be quoted, since only a subset of the characters is allowed in unquoted strings. - ``None`` (written as ``null`` in the grammar) is the only valid non-string input to ``oc.decode`` (returning ``None`` in that case). This resolver can be useful for instance to parse environment variables: @@ -348,7 +349,7 @@ Another scenario where ``oc.select`` can ... "with_default": "${oc.select:missing,default value}", ... } ... ) - ... + ... >>> print(cfg.interpolation) Traceback (most recent call last): ... @@ -391,3 +392,67 @@ as interpolations), and they return a `` >>> show(cfg.ips) type: ListConfig, value: ['${workers.node3}', '${workers.node7}'] >>> assert cfg.ips == ["10.0.0.2", "10.0.0.9"] + +.. _clearing_resolvers: + +Clearing/removing resolvers +--------------------------- + +.. _clear_resolvers: + +clear_resolvers +^^^^^^^^^^^^^^^ + +Use ``OmegaConf.clear_resolvers()`` to remove all resolvers except the built-in resolvers (like ``oc.env`` etc). + +.. code-block:: python + + def clear_resolvers() -> None + +In the following example, first we register a new custom resolver ``str.lower``, and then clear all +custom resolvers. + +.. doctest:: + + >>> # register a new resolver: str.lower + >>> OmegaConf.register_new_resolver( + ... name='str.lower', + ... resolver=lambda x: str(x).lower(), + ... ) + >>> # check if resolver exists (after adding, before removal) + >>> OmegaConf.has_resolver("str.lower") + True + >>> # clear all custom-resolvers + >>> OmegaConf.clear_resolvers() + >>> # check if resolver exists (after removal) + >>> OmegaConf.has_resolver("str.lower") + False + >>> # default resolvers are not affected + >>> OmegaConf.has_resolver("oc.env") + True + +.. _clear_resolver: + +clear_resolver +^^^^^^^^^^^^^^ + +Use ``OmegaConf.clear_resolver()`` to remove a single resolver (including built-in resolvers). + +.. code-block:: python + + def clear_resolver(name: str) -> bool + + +``OmegaConf.clear_resolver()`` returns True if the resolver was found and removed, and False otherwise. + +Here is an example. + +.. doctest:: + + >>> OmegaConf.has_resolver("oc.env") + True + >>> # This will remove the default resolver: oc.env + >>> OmegaConf.clear_resolver("oc.env") + True + >>> OmegaConf.has_resolver("oc.env") + False diff -pruN 2.1.0~rc1-3/docs/source/grammar.rst 2.2.2-1/docs/source/grammar.rst --- 2.1.0~rc1-3/docs/source/grammar.rst 1970-01-01 00:00:00.000000000 +0000 +++ 2.2.2-1/docs/source/grammar.rst 2022-05-27 21:36:40.000000000 +0000 @@ -0,0 +1,246 @@ +.. testsetup:: * + + from omegaconf import OmegaConf + +The OmegaConf grammar +--------------------- + +OmegaConf uses an `ANTLR `_-based grammar to parse string expressions, +where the `lexer rules `_ +rules define the tokens used by the `parser rules `_. +Currently this grammar's main usage is in the parsing of :ref:`interpolations`, detailed below. + + +.. _interpolation-strings: + +Interpolation strings +^^^^^^^^^^^^^^^^^^^^^ + +An interpolation string is any string containing the ``${`` character sequence (denoting the start of an interpolation), +and is parsed using the ``text`` rule of the grammar: + + .. code-block:: antlr + + text: (interpolation | + ANY_STR | ESC | ESC_INTER | TOP_ESC | QUOTED_ESC)+; + +Such a string can either be a single interpolation, or the concatenation of multiple fragments +that can either be interpolations or regular strings +(with a special handling of escaped characters, see :ref:`escaping-in-interpolation-strings` below). +These are all examples of interpolation strings: + + - ``${foo.bar}`` + - ``https://${host}:${port}`` + - ``Hello ${name}`` + - ``${a}${oc.env:B}${c}`` + + +Interpolation types +^^^^^^^^^^^^^^^^^^^ + +An ``interpolation`` as found in the rule above can either be a :ref:`config-node-interpolation` +(e.g., ``${host}``) or a call to a :ref:`resolver` (e.g., ``${oc.env:B}``). +This is reflected in the following parser rules: + + .. code-block:: antlr + + interpolation: interpolationNode | interpolationResolver; + + interpolationNode: + INTER_OPEN // ${ + DOT* + (configKey | BRACKET_OPEN configKey BRACKET_CLOSE) + (DOT configKey | BRACKET_OPEN configKey BRACKET_CLOSE)* + INTER_CLOSE; // } + + interpolationResolver: + INTER_OPEN // ${ + resolverName COLON sequence? + BRACE_CLOSE; // } + +The following are all valid examples of config node interpolations according to the ``interpolationNode`` rule +(note in particular that it supports both dot and bracket notations to access child nodes): + + - ``${host}`` + - ``${.sibling}`` + - ``${..uncle.cousin}`` + - ``${some_list[3]}`` + - ``${some_deep_dict[key1][subkey2].subsubkey3}`` + +Here are also examples of resolver calls from the ``interpolationResolver`` rule: + + - ``${oc.env:B}`` + - ``${my_resolver_without_args:}`` + - ``${oc.select: missing, default}`` + +Resolver arguments must be provided in a comma-separated list as per the following +``sequence`` parser rule: + + .. code-block:: antlr + + sequence: (element (COMMA element?)*) | (COMMA element?)+; + +*Note that this rule currently supports empty arguments to preserve backward compatibility +with OmegaConf 2.0, but this has been deprecated (see* `#572 `_ *).* + + +.. _element-types: + +Element types +^^^^^^^^^^^^^ + +As seen in the ``sequence`` rule above, each resolver argument is parsed by an ``element`` rule, +which currently supports four main types of arguments: + + .. code-block:: antlr + + element: + quotedValue + | listContainer + | dictContainer + | primitive + ; + +A ``quotedValue`` is a quoted string that may contain basically anything in-between either double or single quotes +(including interpolations, which will be resolved at evaluation time). +For instance: + + - ``"Hello World!"`` + - ``'Hello ${name}!'`` + - ``"I ${can: ${nest}, ${interpolations}, 'and quotes'}"`` + +The ``quotedValue`` parser rule is formally defined as: + + .. code-block:: antlr + + quotedValue: + (QUOTE_OPEN_SINGLE | QUOTE_OPEN_DOUBLE) + text? + MATCHING_QUOTE_CLOSE; + + +``listContainer`` and ``dictContainer`` are respectively lists and dictionaries, using a familiar syntax: + + - List examples: ``[]``, ``[1, 2, 3]``, ``[${a}, ${oc.env:B}, c]`` + - Dict examples: ``{}``, ``{a: 1, b: 2}``, ``{a: ${a}, b: ${oc.env:B}}`` + +Their corresponding parser rules are: + + .. code-block:: antlr + + listContainer: BRACKET_OPEN sequence? BRACKET_CLOSE; + dictContainer: BRACE_OPEN + (dictKeyValuePair (COMMA dictKeyValuePair)*)? + BRACE_CLOSE; + +Regarding dictionaries, note that although values can be any ``element``, keys are more +restricted, and in particular quoted strings and interpolations are currently *not* allowed as +dictionary keys (see the definition of ``dictKey`` in the `grammar `_). + +Finally, a ``primitive`` is everything else that is allowed, including in particular (see the `full grammar `_ +for details): + + - Unquoted strings (that support only a subset of characters, contrary to quoted ones): ``foo``, ``foo_bar``, ``hello world 123`` + - Integer numbers: ``123``, ``-5``, ``+1_000_000`` + - Floating point numbers (with special case-independent keywords for infinity and NaN): ``0.1``, ``1e-3``, ``inf``, ``-INF``, ``nan`` + - Other special keywords (also case-independent): ``null``, ``true``, ``false``, ``NULL``, ``True``, ``fAlSe``. + **IMPORTANT**: ``None`` is *not* a special keyword and will be parsed as an unquoted string, you must + use the ``null`` keyword instead (as in YAML). + - Interpolations (thus allowing for nested interpolations) + + +Escaped characters +^^^^^^^^^^^^^^^^^^ + +Some characters need to be escaped, with varying escaping requirements depending on the situation. +In general, however, you can use the following rule of thumb: +*you only need to escape characters that otherwise have a special meaning in the current context*. + +.. _escaping-in-interpolation-strings: + +Escaping in interpolation strings ++++++++++++++++++++++++++++++++++ + +In order to define fields whose value is an interpolation-like string, interpolations can be escaped with ``\${``. +For instance: + +.. doctest:: + + >>> c = OmegaConf.create({"path": r"\${dir}", "dir": "tmp"}) + >>> print(c.path) # does *not* interpolate into the `dir` node + ${dir} + +If you actually want to follow a ``\`` with a resolved interpolation, this backslash +needs to be escaped into ``\\`` to differentiate it from an escaped interpolation: + +.. doctest:: + + >>> c = OmegaConf.create({"path": r"C:\\${dir}", "dir": "tmp"}) + >>> print(c.path) # *does* interpolate into the `dir` node + C:\tmp + +Note that we use Python raw strings here to make code +more readable -- otherwise all ``\`` characters would need be duplicated due to how Python handles +escaping in regular string literals. + +Finally, since the ``\`` character has no special meaning unless followed by ``${``, +it does *not* need to be escaped anywhere else: + +.. doctest:: + + >>> c = OmegaConf.create({"path": r"C:\foo_${dir}", "dir": "tmp"}) + >>> print(c.path) # a single \ is preserved... + C:\foo_tmp + >>> c = OmegaConf.create({"path": r"C:\\foo_${dir}", "dir": "tmp"}) + >>> print(c.path) # ... and multiple \\ too (no escape sequence) + C:\\foo_tmp + +Escaping in unquoted strings +++++++++++++++++++++++++++++ + +Unquoted strings can be found in a number of contexts, including dictionary keys/values, +list elements, etc. As a result, the escape sequences are used for some +special characters +(``\\``, ``\[``, ``\]``, ``\{``, ``\}``, ``\(``, ``\)``, ``\:``, ``\=``, ``\,``), +for instance: + + - ``C\:\\$\{dir\}`` resolves to the string ``"C:\${dir}"`` + - ``\[a\, b\, c\]`` resolves to the string ``"[a, b, c]"`` + +In addition, leading and trailing whitespaces must be escaped in unquoted strings +if we do not want them to be stripped (while inner whitespaces are always preserved): + +.. doctest:: + + >>> c = OmegaConf.create({"esc": r"${oc.decode: \ hi u \ }"}) + >>> c.esc # one leading whitespace and two trailing ones + ' hi u ' + >>> # Tabs are handled similarly (NB: r-strings can't be used below) + >>> c = OmegaConf.create({"esc": "${oc.decode:\t\\\thi u\t\\\t\t}"}) + >>> c.esc # one leading tab and two trailing ones + '\thi u\t\t' + +Escaping in unquoted strings can lead to hard-to-read expressions, and it is recommended +to switch to quoted strings instead of relying heavily on the above escape sequences. + +Escaping in quoted strings +++++++++++++++++++++++++++ + +As can be seen from the definition of the ``quotedValue`` parser rule above, quoted strings +are just ``text`` fragments surrounded by quotes, and are thus very similar to :ref:`interpolation-strings`. +As a result, the ``\${`` escape sequence can also be used to escape interpolations +in quoted strings (as described in :ref:`escaping-in-interpolation-strings`): + + - ``"\${dir}"`` resolves to the string ``"${dir}"`` + - ``"C:\\${dir}"`` resolves to the string ``"C:\"`` + +However, one key difference with interpolation strings is that quotes of the same type +as the enclosing quotes must be escaped, unless they are within a nested interpolation. +For instance: + + - ``'\'Hi you\', I said'`` resolves to the string ``"'Hi you', I said"`` + - ``"'Hi ${concat: 'y', "o", u}', I said"`` also resolves to the string ``"'Hi you', I said"`` + if ``concat`` is a :doc:`custom resolver` concatenating its inputs. + The main point to pay attention to in this example is that the quoted strings ``'y'`` and + ``"o"`` found within the resolver interpolation ``${concat: ...}`` do *not* need to be + escaped, regardless of existing quotes outside of this interpolation. \ No newline at end of file diff -pruN 2.1.0~rc1-3/docs/source/index.rst 2.2.2-1/docs/source/index.rst --- 2.1.0~rc1-3/docs/source/index.rst 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/docs/source/index.rst 2022-05-27 21:36:40.000000000 +0000 @@ -13,6 +13,7 @@ OmegaConf also offers runtime type safet usage custom_resolvers structured_config + grammar diff -pruN 2.1.0~rc1-3/docs/source/structured_config.rst 2.2.2-1/docs/source/structured_config.rst --- 2.1.0~rc1-3/docs/source/structured_config.rst 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/docs/source/structured_config.rst 2022-05-27 21:36:40.000000000 +0000 @@ -1,4 +1,4 @@ -.. _structured_config: +.. _structured_configs: .. testsetup:: * @@ -6,17 +6,18 @@ from enum import Enum from dataclasses import dataclass, field import os + import pathlib from pytest import raises from typing import Dict, Any import sys os.environ['USER'] = 'omry' -Structured config ------------------ +Structured configs +------------------ Structured configs are used to create OmegaConf configuration object with runtime type safety. In addition, they can be used with tools like mypy or your IDE for static type checking. -Two types of structures classes that are supported: dataclasses and attr classes. +Two types of structures classes are supported: dataclasses and attr classes. - `dataclasses `_ are standard as of Python 3.7 or newer and are available in Python 3.6 via the `dataclasses` pip package. - `attrs `_ Offset slightly cleaner syntax in some cases but depends on the attrs pip package. @@ -27,14 +28,33 @@ Basic usage involves passing in a struct the values and types specified in the input. At runtine, OmegaConf will validate modifications to the created config object against the schema specified in the input class. + +Currently, type hints supported in OmegaConf’s structured configs include: + - primitive types (``int``, ``float``, ``bool``, ``str``, ``Path``) and enum types + (user-defined subclasses of ``enum.Enum``). See the :ref:`simple_types` section below. + - unions of primitive/enum types, e.g. ``Union[float, bool, MyEnum]``. + See :ref:`union_types` below. + - structured config fields (i.e. MyConfig.x can have type hint MySubConfig). + See the :ref:`nesting_structured_configs` section below. + - dict and list types: ``typing.Dict[K, V]`` or ``typing.List[V]``, where K is + primitive or enum, and where V is any of the above (including nested dicts + or lists, e.g. ``Dict[str, List[int]]``). + See the :ref:`lists` and :ref:`dictionaries` sections below. + - optional types (any of the above can be wrapped in a ``typing.Optional[...]`` + annotation). See :ref:`other_special_features` below. + +.. _simple_types: + Simple types ^^^^^^^^^^^^ Simple types include - - int: numeric integers - - float: numeric floating point values - - bool: boolean values (True, False, On, Off etc) - - str: Any string - - Enums: User defined enums + - ``int``: numeric integers + - ``float``: numeric floating point values + - ``bool``: boolean values (True, False, On, Off etc) + - ``str``: any string + - ``bytes``: an immutable sequence of numbers in [0, 255] + - ``pathlib.Path``: filesystem paths as represented by python's standard library ``pathlib`` + - ``Enums``: User defined enums The following class defines fields with all simple types: @@ -51,6 +71,8 @@ The following class defines fields with ... is_awesome: bool = True ... height: Height = Height.SHORT ... description: str = "text" + ... data: bytes = b"bin_data" + ... path: pathlib.Path = pathlib.Path("hello.txt") You can create a config based on the SimpleTypes class itself or an instance of it. Those would be equivalent by default, but the Object variant allows you to set the values of specific @@ -72,6 +94,10 @@ fields during construction. is_awesome: true height: TALL description: text + data: !!binary | + YmluX2RhdGE= + path: !!python/object/apply:pathlib.PosixPath + - hello.txt The resulting object is a regular OmegaConf ``DictConfig``, except that it will utilize the type information in the input class/object @@ -142,10 +168,16 @@ Runtime validation and conversion works >>> conf.height = "TALL" >>> assert conf.height == Height.TALL + >>> # This works too + >>> conf.height = "Height.TALL" + >>> assert conf.height == Height.TALL + >>> # The ordinal of Height.TALL is 1 >>> conf.height = 1 >>> assert conf.height == Height.TALL +.. _nesting_structured_configs: + Nesting structured configs ^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -199,18 +231,16 @@ You can assign subclasses: >>> assert conf.manager.duper == True -Containers ----------- - -Python container types are fully supported in Structured configs. +.. _lists: Lists ^^^^^ Structured Config fields annotated with ``typing.List`` or ``typing.Tuple`` can hold any type -supported by OmegaConf (``int``, ``float``. ``bool``, ``str``, ``Enum`` or Structured configs). +supported by OmegaConf (``int``, ``float``. ``bool``, ``str``, ``bytes``, ``pathlib.Path``, ``Enum`` or Structured configs). .. doctest:: + >>> from dataclasses import dataclass, field >>> from typing import List, Tuple >>> @dataclass ... class User: @@ -218,8 +248,8 @@ supported by OmegaConf (``int``, ``float >>> @dataclass ... class ListsExample: - ... # Typed list can hold Any, int, float, bool, str and Enums as well - ... # as arbitrary Structured configs + ... # Typed list can hold Any, int, float, bool, str, + ... # bytes, pathlib.Path and Enums as well as arbitrary Structured configs. ... ints: List[int] = field(default_factory=lambda: [10, 20, 30]) ... bools: Tuple[bool, bool] = field(default_factory=lambda: (True, False)) ... users: List[User] = field(default_factory=lambda: [User(name="omry")]) @@ -242,20 +272,21 @@ In the example below, the OmegaConf obje >>> with raises(ValidationError): ... conf.users.append(10) +.. _dictionaries: + Dictionaries ^^^^^^^^^^^^ Dictionaries are supported via annotation of structured config fields with ``typing.Dict``. -Keys must be typed as one of ``str``, ``int``, ``Enum``, ``float``, or ``bool``. Values can -be any of the types supported by OmegaConf (``Any``, ``int``, ``float``, ``bool``, ``str`` and ``Enum`` as well -as arbitrary Structured configs) +Keys must be typed as one of ``str``, ``int``, ``Enum``, ``float``, ``bytes``, or ``bool``. Values can +be any of the types supported by OmegaConf (``Any``, ``int``, ``float``, ``bool``, ``bytes``, +``pathlib.Path``, ``str`` and ``Enum`` as well as arbitrary Structured configs) .. doctest:: + >>> from dataclasses import dataclass, field >>> from typing import Dict >>> @dataclass ... class DictExample: - ... # Typed dict keys are strings; values can be typed as Any, int, float, bool, str and Enums or - ... # arbitrary Structured configs ... ints: Dict[str, int] = field(default_factory=lambda: {"a": 10, "b": 20, "c": 30}) ... bools: Dict[str, bool] = field(default_factory=lambda: {"Uno": True, "Zoro": False}) ... users: Dict[str, User] = field(default_factory=lambda: {"omry": User(name="omry")}) @@ -275,8 +306,118 @@ Like with Lists, the types of values con >>> with raises(ValidationError): ... conf.users["Joe"] = 10 -Misc ----- +.. _nested_dict_and_list_annotations: + +Nested dict and list annotations +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Dict and List annotations can be nested flexibly: + +.. doctest:: + + >>> @dataclass + ... class NestedContainers: + ... dict_of_dict: Dict[str, Dict[str, int]] + ... list_of_list: List[List[int]] = field(default_factory=lambda: [[123]]) + ... dict_of_list: Dict[str, List[int]] = MISSING + ... list_of_dict: List[Dict[str, int]] = MISSING + ... + ... + >>> cfg = OmegaConf.structured(NestedContainers(dict_of_dict={"foo": {"bar": 123}})) + >>> print(OmegaConf.to_yaml(cfg)) + dict_of_dict: + foo: + bar: 123 + list_of_list: + - - 123 + dict_of_list: ??? + list_of_dict: ??? + + >>> with raises(ValidationError): + ... cfg.list_of_dict = [["whoops"]] # not a list of dicts + +.. _union_types: + +Unions +^^^^^^ + +You can use `typing.Union `_ +to annotate unions of :ref:`simple_types`. + +.. doctest:: + + >>> from typing import Union + >>> + >>> @dataclass + ... class HasUnion: + ... u: Union[float, bool] = 10.1 + ... + >>> cfg = OmegaConf.structured(HasUnion) + >>> assert cfg.u == 10.1 + >>> cfg.u = True # ok + >>> cfg.u = b"binary" # bytes not compatible with union + Traceback (most recent call last): + ... + omegaconf.errors.ValidationError: Cannot assign 'b'binary'' of type 'bytes' to Union[float, bool] + full_key: u + object_type=HasUnion + >>> OmegaConf.structured(HasUnion("abc")) # str not compatible + Traceback (most recent call last): + ... + omegaconf.errors.ValidationError: Cannot assign 'abc' of type 'str' to Union[float, bool] + full_key: u + object_type=None + +If any argument of a ``Union`` type hint is ``Optional``, the *whole* +union is considered optional. For example, OmegaConf treats all four of the +following type hints as equivalent: + +- ``Optional[Union[int, str]]`` +- ``Union[Optional[int], str]`` +- ``Union[int, str, None]`` +- ``Union[int, str, type(None)]`` + +Ordinarily, assignment to a structured config field results in coercion of the +assigned value to the field's type. For example, assigning an integer to a +field typed as ``str`` results in the integer being coverted to a string: + +.. doctest:: + + >>> @dataclass + ... class HasStr: + ... s: str + ... + >>> cfg = OmegaConf.structured(HasStr) + >>> cfg.s = 10.1 + >>> assert cfg.s == "10.1" # The assigned value has been converted to a string + +When dealing with ``Union`` types, however, conversion is disabled so as to +avoid ambiguity. Values assigned to a union-typed field of a structured config +must precisely match one of the types in the ``Union`` annotation: + +.. doctest:: + + >>> @dataclass + ... class StrOrInt: + ... u: Union[str, float] + ... + >>> cfg = OmegaConf.structured(StrOrInt) + >>> cfg.u = 10.1 + >>> assert cfg.u == 10.1 # The assigned value remains a `float`. + >>> cfg.u = "10.1" + >>> assert cfg.u == "10.1" # The assigned value remains a `str`. + >>> cfg.u = 123 # Conversion from `int` to `float` does not occur. + Traceback (most recent call last): + ... + omegaconf.errors.ValidationError: Value '123' of type 'int' is incompatible with type hint 'Union[str, float]' + full_key: u + object_type=StrOrInt + + +.. _other_special_features: + +Other special features +^^^^^^^^^^^^^^^^^^^^^^ OmegaConf supports field modifiers such as ``MISSING`` and ``Optional``. .. doctest:: @@ -289,11 +430,17 @@ OmegaConf supports field modifiers such ... num: int = 10 ... optional_num: Optional[int] = 10 ... another_num: int = MISSING + ... optional_dict: Optional[Dict[str, int]] = None + ... list_optional: List[Optional[int]] = field(default_factory=lambda: [10, MISSING, None]) >>> conf: Modifiers = OmegaConf.structured(Modifiers) +Note for Python3.6 users: :ref:`pickling ` +structured configs with complex type annotations, such as dict-of-list or +list-of-optional, is not supported. + Mandatory missing values -^^^^^^^^^^^^^^^^^^^^^^^^ +++++++++++++++++++++++++ Fields assigned the constant ``MISSING`` do not have a value and the value must be set prior to accessing the field. Otherwise a ``MissingMandatoryValue`` exception is raised. @@ -307,7 +454,7 @@ Otherwise a ``MissingMandatoryValue`` ex Optional fields -^^^^^^^^^^^^^^^ ++++++++++++++++ .. doctest:: @@ -317,11 +464,12 @@ Optional fields >>> conf.optional_num = None >>> assert conf.optional_num is None + >>> assert conf.list_optional[2] is None Interpolations -^^^^^^^^^^^^^^ +++++++++++++++ :ref:`interpolation` works normally with Structured configs, but static type checkers may object to you assigning a string to another type. To work around this, use the special functions ``omegaconf.SI`` and ``omegaconf.II`` described below. @@ -380,13 +528,15 @@ Note however that this validation step i >>> assert cfg.some_dict == 0 # type mismatch, but no error -Frozen -^^^^^^ +Frozen classes +++++++++++++++ Frozen dataclasses and attr classes are supported via OmegaConf :ref:`read-only-flag`, which makes the entire config node and all if it's child nodes read-only. .. doctest:: + >>> from dataclasses import dataclass, field + >>> from typing import List >>> @dataclass(frozen=True) ... class FrozenClass: ... x: int = 10 @@ -404,7 +554,7 @@ The read-only flag is recursive: ... conf.list[0] = 20 Merging with other configs ----------------------------- +^^^^^^^^^^^^^^^^^^^^^^^^^^ Once an OmegaConf object is created, it can be merged with others regardless of its source. OmegaConf configs created from Structured configs contains type information that is enforced at runtime. diff -pruN 2.1.0~rc1-3/docs/source/usage.rst 2.2.2-1/docs/source/usage.rst --- 2.1.0~rc1-3/docs/source/usage.rst 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/docs/source/usage.rst 2022-05-27 21:36:40.000000000 +0000 @@ -64,13 +64,13 @@ Here is an example of various supported ... BLUE = 2 >>> >>> conf = OmegaConf.create( - ... {"key": "str", 123: "int", True: "bool", 3.14: "float", Color.RED: "Color"} + ... {"key": "str", 123: "int", True: "bool", 3.14: "float", Color.RED: "Color", b"123": "bytes"} ... ) >>> >>> print(conf) - {'key': 'str', 123: 'int', True: 'bool', 3.14: 'float', : 'Color'} + {'key': 'str', 123: 'int', True: 'bool', 3.14: 'float', : 'Color', b'123': 'bytes'} -OmegaConf supports ``str``, ``int``, ``bool``, ``float`` and Enums as dictionary key types. +OmegaConf supports ``str``, ``int``, ``bool``, ``float`` ``bytes``, and ``Enum`` as dictionary key types. From a list ^^^^^^^^^^^ @@ -292,6 +292,8 @@ Save/Load YAML file Note that this does not retain type information. +.. _save_and_load_pickle_file: + Save/Load pickle file ^^^^^^^^^^^^^^^^^^^^^ Use pickle to save and load while retaining the type information. @@ -307,6 +309,12 @@ Note that the saved file may be incompat ... loaded = pickle.load(fp) ... assert conf == loaded +Note for Python3.6 users: due to limitations in pickling support, +:ref:`structured configs ` with complex type hints (such as +:ref:`nested container types ` or +:ref:`containers with optional element types `) cannot +be pickled using Python3.6. + .. _interpolation: @@ -315,6 +323,8 @@ Variable interpolation OmegaConf supports variable interpolation. Interpolations are evaluated lazily on access. +.. _config-node-interpolation: + Config node interpolation ^^^^^^^^^^^^^^^^^^^^^^^^^ The interpolated variable can be the path to another node in the configuration, and in that case @@ -383,6 +393,8 @@ Interpolated nodes can be any node in th >>> (cfg.player.height, cfg.player.weight) (180, 75) +.. _resolvers: + Resolvers ^^^^^^^^^ Add new interpolation types by registering resolvers using ``OmegaConf.register_new_resolver()``. @@ -544,6 +556,7 @@ You can temporarily remove the struct fl .. doctest:: loaded + >>> from omegaconf import open_dict >>> conf = OmegaConf.create({"a": {"aa": 10, "bb": 20}}) >>> OmegaConf.set_struct(conf, True) >>> with open_dict(conf): @@ -572,11 +585,36 @@ If ``resolve`` is set to ``True``, inter type: dict, value: {'foo': 'bar', 'foo2': 'bar'} +Using ``throw_on_missing`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +You can control how missing values are handled by ``OmegaConf.to_container()`` +using the ``throw_on_missing`` keyword argument. + +.. doctest:: + + >>> conf = OmegaConf.create({"foo": "bar", "missing": "???"}) + >>> has_missing = OmegaConf.to_container(conf, throw_on_missing=False) + >>> show(has_missing) + type: dict, value: {'foo': 'bar', 'missing': '???'} + >>> OmegaConf.to_container(conf, throw_on_missing=True) + Traceback (most recent call last): + ... + omegaconf.errors.MissingMandatoryValue: Missing mandatory value: missing + full_key: missing + object_type=dict + + +By default, ``throw_on_missing=False``. +Setting ``throw_on_missing=True`` can be useful if you want your program to +fail fast when there are missing values in the config. + + + Using ``structured_config_mode`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ You can customize the treatment of ``OmegaConf.to_container()`` for Structured Config nodes using the ``structured_config_mode`` option. -By default, Structured Config nodes are converted to plain dict. +The default, ``structured_config_mode=SCMode.DICT``, converts Structured Config nodes to plain dict. Using ``structured_config_mode=SCMode.DICT_CONFIG`` causes such nodes to remain as ``DictConfig``, allowing attribute style access on the resulting node. @@ -621,7 +659,7 @@ Note that here, ``container["structured_ ``MyConfig``. The call ``OmegaConf.to_object(conf)`` is equivalent to -``OmegaConf.to_container(conf, resolve=True, +``OmegaConf.to_container(conf, resolve=True, throw_on_missing=True, structured_config_mode=SCMode.INSTANTIATE)``. OmegaConf.resolve @@ -782,6 +820,25 @@ and ``OmegaConf.is_list(cfg)`` is equiva >>> assert not OmegaConf.is_dict(l) +OmegaConf.missing_keys +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +``OmegaConf.missing_keys(cfg)`` returns a set of missing keys present in the input ``cfg``. +Each missing key is represented as a ``str``, using a dotlist style. +This utility function can be used after creating a config object, after merging sources and so on, +to check for missing mandatory fields and aid in creating a proper error message. + +.. doctest:: + + >>> missings = OmegaConf.missing_keys({ + ... "foo": {"bar": "???"}, + ... "missing": "???", + ... "list": ["a", None, "???"] + ... }) + >>> assert missings == {'list[2]', 'foo.bar', 'missing'} + +The function raises a `ValueError` on input not representing a config. + + Debugger integration -------------------- diff -pruN 2.1.0~rc1-3/.flake8 2.2.2-1/.flake8 --- 2.1.0~rc1-3/.flake8 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/.flake8 2022-05-27 21:36:40.000000000 +0000 @@ -1,5 +1,5 @@ [flake8] -exclude = .git,.nox,.tox,omegaconf/grammar/gen +exclude = .git,.nox,.tox,omegaconf/grammar/gen,build max-line-length = 119 select = E,F,W,C ignore=W503,E203 diff -pruN 2.1.0~rc1-3/MANIFEST.in 2.2.2-1/MANIFEST.in --- 2.1.0~rc1-3/MANIFEST.in 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/MANIFEST.in 2022-05-27 21:36:40.000000000 +0000 @@ -1,3 +1,5 @@ include LICENSE include requirements/base.txt -recursive-include tests *.py +recursive-include tests *.py *.pickle +recursive-include build_helpers *.py *.jar +recursive-include omegaconf *.g4 diff -pruN 2.1.0~rc1-3/news/934.bugfix 2.2.2-1/news/934.bugfix --- 2.1.0~rc1-3/news/934.bugfix 1970-01-01 00:00:00.000000000 +0000 +++ 2.2.2-1/news/934.bugfix 2022-05-27 21:36:40.000000000 +0000 @@ -0,0 +1 @@ +Revert an accidental behavior change where implicit conversion from `Path` to `str` was disallowed. diff -pruN 2.1.0~rc1-3/NEWS.md 2.2.2-1/NEWS.md --- 2.1.0~rc1-3/NEWS.md 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/NEWS.md 2022-05-27 21:36:40.000000000 +0000 @@ -1,3 +1,65 @@ +## 2.2.2 (2022-05-26) +### Bug Fixes + +- Revert an accidental behavior change where implicit conversion from `Path` to `str` was disallowed. ([#934](https://github.com/omry/omegaconf/issues/934)) +- Revert a behavior change where namedtuples and list subclasses were coerced to ListConfig. ([#939](https://github.com/omry/omegaconf/issues/939)) +- Fix a bug where the `oc.dict.values` resolver failed when passed a relative dotpath ([#942](https://github.com/omry/omegaconf/issues/942)) + + +## 2.2.1 (2022-05-17) +OmegaConf 2.2 is a major release. The most significant area of improvement in +2.2 is support for more flexible type hints in structured configs. In addition, +OmegaConf now natively supports two new primitive types, `bytes` and `pathlib.Path`. + +### Features + +- Support unions of primitive types in structured config type hints (`typing.Union`) ([#144](https://github.com/omry/omegaconf/issues/144)) +- Support nested container type hints in structured configs, e.g. dict-of-dict and list-of-list ([#427](https://github.com/omry/omegaconf/issues/427)) +- Improve support for optional element types in structured config container type hints (`typing.Optional`) ([#460](https://github.com/omry/omegaconf/issues/460)) +- Add support for `bytes`-typed values ([#844](https://github.com/omry/omegaconf/issues/844)) +- Add support for `pathlib.Path`-typed values ([#97](https://github.com/omry/omegaconf/issues/97)) +- `ListConfig` now implements slice assignment ([#736](https://github.com/omry/omegaconf/issues/736)) +- Enable adding a `ListConfig` to a `list` via the `ListConfig.__radd__` dunder method ([#849](https://github.com/omry/omegaconf/issues/849)) +- Add `OmegaConf.missing_keys()`, a method that returns the missing keys in a config object ([#720](https://github.com/omry/omegaconf/issues/720)) +- Add `OmegaConf.clear_resolver()`, a method to remove interpolation resolvers by name ([#769](https://github.com/omry/omegaconf/issues/769)) +- Enable the use of a pipe symbol `|` in unquoted strings in OmegaConf interpolations ([#799](https://github.com/omry/omegaconf/issues/799)) + +### Bug Fixes + +- `OmegaConf.to_object` now works properly with structured configs that have `init=False` fields ([#789](https://github.com/omry/omegaconf/issues/789)) +- Fix bugs related to creation of structured configs from dataclasses having fields with a default_factory ([#831](https://github.com/omry/omegaconf/issues/831)) +- Fix default value initialization for structured configs created from subclasses of dataclasses ([#817](https://github.com/omry/omegaconf/issues/817)) + +### API changes and deprecations + +- Removed support for `OmegaConf.is_none(cfg, "key")`. Please use `cfg.key is None` instead. ([#547](https://github.com/omry/omegaconf/issues/547)) +- Removed support for `${env}` interpolations. `${oc.env}` should be used instead. ([#573](https://github.com/omry/omegaconf/issues/573)) +- Removed `OmegaConf.get_resolver()`. Please use `OmegaConf.has_resolver()` instead. ([#608](https://github.com/omry/omegaconf/issues/608)) +- Removed support for `OmegaConf.is_optional()`. ([#698](https://github.com/omry/omegaconf/issues/698)) +- Improved error message when assigning an invalid value to int or float config nodes ([#743](https://github.com/omry/omegaconf/issues/743)) +- To conform with the `MutableMapping` API, the `DictConfig.items` method now returns an object of type `ItemsView`, and `DictConfig.keys` will now always return a `KeysView` ([#848](https://github.com/omry/omegaconf/issues/848)) + + +## 2.1.1 (2021-08-17) +### Features + +- Add a throw_on_missing keyword argument to the signature of OmegaConf.to_container, which controls whether MissingMandatoryValue exceptions are raised. ([#501](https://github.com/omry/omegaconf/issues/501)) + +### Miscellaneous changes + +- Update pyyaml dependency specification for compatibility with PEP440 ([#758](https://github.com/omry/omegaconf/issues/758)) +- Fix a packaging issue (missing sdist dependency) ([#772](https://github.com/omry/omegaconf/issues/772)) + + +## 2.1.0 (2021-06-07) + + +### Bug Fixes + +- `ListConfig.append()` now copies input config nodes ([#601](https://github.com/omry/omegaconf/issues/601)) +- Fix loading of OmegaConf 2.0 pickled configs ([#718](https://github.com/omry/omegaconf/issues/718)) + + ## 2.1.0.rc1 (2021-05-12) OmegaConf 2.1 is a major release introducing substantial new features, and introducing some incompatible changes. The biggest area of improvement in 2.1 is interpolations and resolvers. In addition - OmegaConf containers are now diff -pruN 2.1.0~rc1-3/noxfile.py 2.2.2-1/noxfile.py --- 2.1.0~rc1-3/noxfile.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/noxfile.py 2022-05-27 21:36:40.000000000 +0000 @@ -3,34 +3,34 @@ import os import nox -DEFAULT_PYTHON_VERSIONS = ["3.6", "3.7", "3.8", "3.9"] +DEFAULT_PYTHON_VERSIONS = ["3.6", "3.7", "3.8", "3.9", "3.10"] PYTHON_VERSIONS = os.environ.get( "NOX_PYTHON_VERSIONS", ",".join(DEFAULT_PYTHON_VERSIONS) ).split(",") -def deps(session, editable_installl): +def deps(session, editable_install): session.install("--upgrade", "setuptools", "pip") - extra_flags = ["-e"] if editable_installl else [] + extra_flags = ["-e"] if editable_install else [] session.install("-r", "requirements/dev.txt", *extra_flags, ".", silent=True) @nox.session(python=PYTHON_VERSIONS) def omegaconf(session): - deps(session, editable_installl=False) # ensure we test the regular install + deps(session, editable_install=False) # ensure we test the regular install session.run("pytest") @nox.session(python=PYTHON_VERSIONS) def benchmark(session): - deps(session, editable_installl=True) + deps(session, editable_install=True) session.run("pytest", "benchmark/benchmark.py") @nox.session def docs(session): - deps(session, editable_installl=True) + deps(session, editable_install=True) session.chdir("docs") session.run("sphinx-build", "-W", "-b", "doctest", "source", "build") session.run("sphinx-build", "-W", "-b", "html", "source", "build") @@ -41,9 +41,9 @@ def coverage(session): # For coverage, we must use the editable installation because # `coverage run -m pytest` prepends `sys.path` with "." (the current # folder), so that the local code will be used in tests even if we set - # `editable_installl=False`. This would cause problems due to potentially + # `editable_install=False`. This would cause problems due to potentially # missing the generated grammar files. - deps(session, editable_installl=True) + deps(session, editable_install=True) session.run("coverage", "erase") session.run("coverage", "run", "--append", "-m", "pytest", silent=True) session.run("coverage", "report", "--fail-under=100") @@ -55,8 +55,10 @@ def coverage(session): @nox.session(python=PYTHON_VERSIONS) def lint(session): - deps(session, editable_installl=True) - session.run("mypy", ".", "--strict", silent=True) + deps(session, editable_install=True) + session.run( + "mypy", ".", "--strict", "--install-types", "--non-interactive", silent=True + ) session.run("isort", ".", "--check", silent=True) session.run("black", "--check", ".", silent=True) session.run("flake8") @@ -70,6 +72,16 @@ def test_jupyter_notebook(session): session.python, ",".join(DEFAULT_PYTHON_VERSIONS) ) ) - deps(session, editable_installl=False) - session.install("jupyter", "nbval") - session.run("pytest", "--nbval", "docs/notebook/Tutorial.ipynb", silent=True) + deps(session, editable_install=False) + # pytest pinned due to https://github.com/computationalmodelling/nbval/issues/180 + session.install("jupyter", "nbval", "pytest<7.0.0") + # Ignore deprecation warnings raised by jupyter_client in Python 3.10 + # https://github.com/jupyter/jupyter_client/issues/713 + extra_flags = ["-Wignore::ResourceWarning"] + if session.python == "3.10": + extra_flags.append( + "-Wdefault:There is no current event loop:DeprecationWarning" + ) + session.run( + "pytest", "--nbval", "docs/notebook/Tutorial.ipynb", *extra_flags, silent=True + ) diff -pruN 2.1.0~rc1-3/omegaconf/basecontainer.py 2.2.2-1/omegaconf/basecontainer.py --- 2.1.0~rc1-3/omegaconf/basecontainer.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/omegaconf/basecontainer.py 2022-05-27 21:36:40.000000000 +0000 @@ -2,33 +2,49 @@ import copy import sys from abc import ABC, abstractmethod from enum import Enum -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Tuple, Union import yaml from ._utils import ( _DEFAULT_MARKER_, + ValueKind, _ensure_container, _get_value, _is_interpolation, - _is_missing_literal, _is_missing_value, _is_none, + _is_special, _resolve_optional, - get_ref_type, get_structured_config_data, + get_type_hint, + get_value_kind, get_yaml_loader, is_container_annotation, is_dict_annotation, is_list_annotation, is_primitive_dict, - is_primitive_type, + is_primitive_type_annotation, is_structured_config, + is_tuple_annotation, + is_union_annotation, +) +from .base import ( + Box, + Container, + ContainerMetadata, + DictKeyType, + Node, + SCMode, + UnionNode, ) -from .base import Container, ContainerMetadata, DictKeyType, Node, SCMode from .errors import ( ConfigCycleDetectedException, + ConfigTypeError, + InterpolationResolutionError, + KeyValidationError, MissingMandatoryValue, + OmegaConfBaseException, ReadonlyConfigError, ValidationError, ) @@ -38,12 +54,34 @@ if TYPE_CHECKING: class BaseContainer(Container, ABC): - # static - _resolvers: Dict[str, Any] = {} + _resolvers: ClassVar[Dict[str, Any]] = {} - def __init__(self, parent: Optional["Container"], metadata: ContainerMetadata): + def __init__(self, parent: Optional[Box], metadata: ContainerMetadata): + if not (parent is None or isinstance(parent, Box)): + raise ConfigTypeError("Parent type is not omegaconf.Box") super().__init__(parent=parent, metadata=metadata) - self.__dict__["_content"] = None + + def _get_child( + self, + key: Any, + validate_access: bool = True, + validate_key: bool = True, + throw_on_missing_value: bool = False, + throw_on_missing_key: bool = False, + ) -> Union[Optional[Node], List[Optional[Node]]]: + """Like _get_node, passing through to the nearest concrete Node.""" + child = self._get_node( + key=key, + validate_access=validate_access, + validate_key=validate_key, + throw_on_missing_value=throw_on_missing_value, + throw_on_missing_key=throw_on_missing_key, + ) + if isinstance(child, UnionNode) and not _is_special(child): + value = child._value() + assert isinstance(value, Node) and not isinstance(value, UnionNode) + child = value + return child def _resolve_with_default( self, @@ -94,6 +132,12 @@ class BaseContainer(Container, ABC): dict_copy["_metadata"].ref_type = List else: assert False + if sys.version_info < (3, 7): # pragma: no cover + element_type = self._metadata.element_type + if is_union_annotation(element_type): + raise OmegaConfBaseException( + "Serializing structured configs with `Union` element type requires python >= 3.7" + ) return dict_copy # Support pickle @@ -103,7 +147,19 @@ class BaseContainer(Container, ABC): if isinstance(self, DictConfig): key_type = d["_metadata"].key_type + + # backward compatibility to load OmegaConf 2.0 configs + if key_type is None: + key_type = Any + d["_metadata"].key_type = key_type + element_type = d["_metadata"].element_type + + # backward compatibility to load OmegaConf 2.0 configs + if element_type is None: + element_type = Any + d["_metadata"].element_type = element_type + ref_type = d["_metadata"].ref_type if is_container_annotation(ref_type): if is_generic_dict(ref_type): @@ -162,6 +218,7 @@ class BaseContainer(Container, ABC): def _to_content( conf: Container, resolve: bool, + throw_on_missing: bool, enum_to_str: bool = False, structured_config_mode: SCMode = SCMode.DICT, ) -> Union[None, Any, str, Dict[DictKeyType, Any], List[Any]]: @@ -174,18 +231,54 @@ class BaseContainer(Container, ABC): return value - assert isinstance(conf, Container) + def get_node_value(key: Union[DictKeyType, int]) -> Any: + try: + node = conf._get_child(key, throw_on_missing_value=throw_on_missing) + except MissingMandatoryValue as e: + conf._format_and_raise(key=key, value=None, cause=e) + assert isinstance(node, Node) + if resolve: + try: + node = node._dereference_node() + except InterpolationResolutionError as e: + conf._format_and_raise(key=key, value=None, cause=e) + + if isinstance(node, Container): + value = BaseContainer._to_content( + node, + resolve=resolve, + throw_on_missing=throw_on_missing, + enum_to_str=enum_to_str, + structured_config_mode=structured_config_mode, + ) + else: + value = convert(node) + return value + if conf._is_none(): return None - elif conf._is_interpolation() and not resolve: + elif conf._is_missing(): + if throw_on_missing: + conf._format_and_raise( + key=None, + value=None, + cause=MissingMandatoryValue("Missing mandatory value"), + ) + else: + return MISSING + elif not resolve and conf._is_interpolation(): inter = conf._value() assert isinstance(inter, str) return inter - elif conf._is_missing(): - return MISSING - elif isinstance(conf, DictConfig): + + if resolve: + _conf = conf._dereference_node() + assert isinstance(_conf, Container) + conf = _conf + + if isinstance(conf, DictConfig): if ( - conf._metadata.object_type is not None + conf._metadata.object_type not in (dict, None) and structured_config_mode == SCMode.DICT_CONFIG ): return conf @@ -194,45 +287,20 @@ class BaseContainer(Container, ABC): ): return conf._to_object() - retdict: Dict[str, Any] = {} + retdict: Dict[DictKeyType, Any] = {} for key in conf.keys(): - node = conf._get_node(key) - assert isinstance(node, Node) - if resolve: - node = node._dereference_node() - + value = get_node_value(key) if enum_to_str and isinstance(key, Enum): key = f"{key.name}" - if isinstance(node, Container): - retdict[key] = BaseContainer._to_content( - node, - resolve=resolve, - enum_to_str=enum_to_str, - structured_config_mode=structured_config_mode, - ) - else: - retdict[key] = convert(node) + retdict[key] = value return retdict elif isinstance(conf, ListConfig): retlist: List[Any] = [] for index in range(len(conf)): - node = conf._get_node(index) - assert isinstance(node, Node) - if resolve: - node = node._dereference_node() - if isinstance(node, Container): - item = BaseContainer._to_content( - node, - resolve=resolve, - enum_to_str=enum_to_str, - structured_config_mode=structured_config_mode, - ) - retlist.append(item) - else: - retlist.append(convert(node)) + item = get_node_value(index) + retlist.append(item) return retlist - assert False @staticmethod @@ -243,7 +311,7 @@ class BaseContainer(Container, ABC): assert isinstance(dest, DictConfig) assert isinstance(src, DictConfig) src_type = src._metadata.object_type - src_ref_type = get_ref_type(src) + src_ref_type = get_type_hint(src) assert src_ref_type is not None # If source DictConfig is: @@ -262,7 +330,7 @@ class BaseContainer(Container, ABC): if rt is not Any: if is_dict_annotation(rt): val = {} - elif is_list_annotation(rt): + elif is_list_annotation(rt) or is_tuple_annotation(rt): val = [] else: val = rt @@ -287,23 +355,25 @@ class BaseContainer(Container, ABC): if (dest._is_interpolation() or dest._is_missing()) and not src._is_missing(): expand(dest) - src_items = src.items_ex(resolve=False) if not src._is_missing() else [] - for key, src_value in src_items: + src_items = list(src) if not src._is_missing() else [] + for key in src_items: src_node = src._get_node(key, validate_access=False) dest_node = dest._get_node(key, validate_access=False) - assert src_node is None or isinstance(src_node, Node) + assert isinstance(src_node, Node) assert dest_node is None or isinstance(dest_node, Node) + src_value = _get_value(src_node) + + src_vk = get_value_kind(src_node) + src_node_missing = src_vk is ValueKind.MANDATORY_MISSING if isinstance(dest_node, DictConfig): dest_node._validate_merge(value=src_node) - missing_src_value = _is_missing_value(src_value) - if ( isinstance(dest_node, Container) and dest_node._is_none() - and not missing_src_value - and not _is_none(src_value, resolve=True) + and not src_node_missing + and not _is_none(src_node, resolve=True) ): expand(dest_node) @@ -313,29 +383,26 @@ class BaseContainer(Container, ABC): dest[key] = target_node dest_node = dest._get_node(key) - if ( - dest_node is None - and is_structured_config(dest._metadata.element_type) - and not missing_src_value - ): + is_optional, et = _resolve_optional(dest._metadata.element_type) + if dest_node is None and is_structured_config(et) and not src_node_missing: # merging into a new node. Use element_type as a base - dest[key] = DictConfig(content=dest._metadata.element_type, parent=dest) + dest[key] = DictConfig( + et, parent=dest, ref_type=et, is_optional=is_optional + ) dest_node = dest._get_node(key) if dest_node is not None: if isinstance(dest_node, BaseContainer): - if isinstance(src_value, BaseContainer): - dest_node._merge_with(src_value) - elif not missing_src_value: - dest.__setitem__(key, src_value) + if isinstance(src_node, BaseContainer): + dest_node._merge_with(src_node) + elif not src_node_missing: + dest.__setitem__(key, src_node) else: - if isinstance(src_value, BaseContainer): - dest.__setitem__(key, src_value) + if isinstance(src_node, BaseContainer): + dest.__setitem__(key, src_node) else: - assert isinstance(dest_node, ValueNode) - assert isinstance(src_node, ValueNode) - # Compare to literal missing, ignoring interpolation - src_node_missing = _is_missing_literal(src_value) + assert isinstance(dest_node, (ValueNode, UnionNode)) + assert isinstance(src_node, (ValueNode, UnionNode)) try: if isinstance(dest_node, AnyNode): if src_node_missing: @@ -391,9 +458,9 @@ class BaseContainer(Container, ABC): temp_target.__dict__["_metadata"] = copy.deepcopy( dest.__dict__["_metadata"] ) - et = dest._metadata.element_type + is_optional, et = _resolve_optional(dest._metadata.element_type) if is_structured_config(et): - prototype = OmegaConf.structured(et) + prototype = DictConfig(et, ref_type=et, is_optional=is_optional) for item in src._iter_ex(resolve=False): if isinstance(item, DictConfig): item = OmegaConf.merge(prototype, item) @@ -457,13 +524,11 @@ class BaseContainer(Container, ABC): Changes the value of the node key with the desired value. If the node key doesn't exist it creates a new one. """ - from omegaconf.omegaconf import _maybe_wrap - from .nodes import AnyNode, ValueNode if isinstance(value, Node): do_deepcopy = not self._get_flag("no_deepcopy_set_nodes") - if not do_deepcopy and isinstance(value, Container): + if not do_deepcopy and isinstance(value, Box): # if value is from the same config, perform a deepcopy no matter what. if self._get_root() is value._get_root(): do_deepcopy = True @@ -484,82 +549,90 @@ class BaseContainer(Container, ABC): if self._get_flag("readonly"): raise ReadonlyConfigError("Cannot change read-only config container") - input_config = isinstance(value, Container) + input_is_node = isinstance(value, Node) target_node_ref = self._get_node(key) - special_value = value is None or _is_missing_value(value) + assert target_node_ref is None or isinstance(target_node_ref, Node) - input_node = isinstance(value, ValueNode) - if isinstance(self.__dict__["_content"], dict): - target_node = key in self.__dict__["_content"] and isinstance( - target_node_ref, ValueNode - ) - - elif isinstance(self.__dict__["_content"], list): - target_node = isinstance(target_node_ref, ValueNode) - # We use set_value if: - # 1. Target node is a container and the value is MISSING or None - # 2. Target node is a container and has an explicit ref_type - # 3. If the target is a NodeValue then it should set his value. - # Furthermore if it's an AnyNode it should wrap when the input is - # a container and set when the input is an compatible type(primitive type). - should_set_value = target_node_ref is not None and ( - ( - isinstance(target_node_ref, Container) - and (special_value or target_node_ref._has_ref_type()) - ) - or (target_node and not isinstance(target_node_ref, AnyNode)) - or (isinstance(target_node_ref, AnyNode) and is_primitive_type(value)) + input_is_typed_vnode = isinstance(value, ValueNode) and not isinstance( + value, AnyNode ) - def wrap(key: Any, val: Any) -> Node: - is_optional = True + def get_target_type_hint(val: Any) -> Any: if not is_structured_config(val): - ref_type = self._metadata.element_type + type_hint = self._metadata.element_type else: target = self._get_node(key) if target is None: - if is_structured_config(val): - ref_type = self._metadata.element_type + type_hint = self._metadata.element_type else: assert isinstance(target, Node) - is_optional = target._is_optional() - ref_type = target._metadata.ref_type - return _maybe_wrap( - ref_type=ref_type, - key=key, - value=val, - is_optional=is_optional, - parent=self, - ) + type_hint = target._metadata.type_hint + return type_hint - def assign(value_key: Any, val: ValueNode) -> None: + target_type_hint = get_target_type_hint(value) + _, target_ref_type = _resolve_optional(target_type_hint) + + def assign(value_key: Any, val: Node) -> None: assert val._get_parent() is None v = val v._set_parent(self) v._set_key(value_key) + _deep_update_type_hint(node=v, type_hint=self._metadata.element_type) self.__dict__["_content"][value_key] = v - if input_node and target_node: - # both nodes, replace existing node with new one + if input_is_typed_vnode and not is_union_annotation(target_ref_type): assign(key, value) - elif not input_node and target_node: - # input is not node, can be primitive or config - if should_set_value: - self.__dict__["_content"][key]._set_value(value) - elif input_config: - assign(key, value) - else: - self.__dict__["_content"][key] = wrap(key, value) - elif input_node and not target_node: - # target must be config, replace target with input node - assign(key, value) - elif not input_node and not target_node: + else: + # input is not a ValueNode, can be primitive or box + + special_value = _is_special(value) + # We use the `Node._set_value` method if the target node exists and: + # 1. the target has an explicit ref_type, or + # 2. the target is an AnyNode and the input is a primitive type. + should_set_value = target_node_ref is not None and ( + target_node_ref._has_ref_type() + or ( + isinstance(target_node_ref, AnyNode) + and is_primitive_type_annotation(value) + ) + ) if should_set_value: + if special_value and isinstance(value, Node): + value = value._value() self.__dict__["_content"][key]._set_value(value) - elif input_config: - assign(key, value) + elif input_is_node: + if ( + special_value + and ( + is_container_annotation(target_ref_type) + or is_structured_config(target_ref_type) + ) + or is_primitive_type_annotation(target_ref_type) + or is_union_annotation(target_ref_type) + ): + value = _get_value(value) + self._wrap_value_and_set(key, value, target_type_hint) + else: + assign(key, value) else: - self.__dict__["_content"][key] = wrap(key, value) + self._wrap_value_and_set(key, value, target_type_hint) + + def _wrap_value_and_set(self, key: Any, val: Any, type_hint: Any) -> None: + from omegaconf.omegaconf import _maybe_wrap + + is_optional, ref_type = _resolve_optional(type_hint) + + try: + wrapped = _maybe_wrap( + ref_type=ref_type, + key=key, + value=val, + is_optional=is_optional, + parent=self, + ) + except ValidationError as e: + self._format_and_raise(key=key, value=val, cause=e) + self.__dict__["_content"][key] = wrapped @staticmethod def _item_eq( @@ -568,8 +641,8 @@ class BaseContainer(Container, ABC): c2: Container, k2: Union[DictKeyType, int], ) -> bool: - v1 = c1._get_node(k1) - v2 = c2._get_node(k2) + v1 = c1._get_child(k1) + v2 = c2._get_child(k2) assert v1 is not None and v2 is not None assert isinstance(v1, Node) @@ -636,7 +709,7 @@ class BaseContainer(Container, ABC): from .listconfig import ListConfig from .omegaconf import _select_one - if not isinstance(key, (int, str, Enum, float, bool, slice, type(None))): + if not isinstance(key, (int, str, Enum, float, bool, slice, bytes, type(None))): return "" def _slice_to_str(x: slice) -> str: @@ -645,14 +718,24 @@ class BaseContainer(Container, ABC): else: return f"{x.start}:{x.stop}" - def prepand(full_key: str, parent_type: Any, cur_type: Any, key: Any) -> str: + def prepand( + full_key: str, + parent_type: Any, + cur_type: Any, + key: Optional[Union[DictKeyType, int, slice]], + ) -> str: + if key is None: + return full_key + if isinstance(key, slice): key = _slice_to_str(key) elif isinstance(key, Enum): key = key.name - elif isinstance(key, (int, float, bool)): + else: key = str(key) + assert isinstance(key, str) + if issubclass(parent_type, ListConfig): if full_key != "": if issubclass(cur_type, ListConfig): @@ -697,7 +780,7 @@ class BaseContainer(Container, ABC): cur = cur._get_parent() if id(cur) in memo: raise ConfigCycleDetectedException( - f"Cycle when iterating over parents of key `{key}`" + f"Cycle when iterating over parents of key `{key!s}`" ) memo.add(id(cur)) assert cur is not None @@ -725,12 +808,107 @@ def _create_structured_with_missing_fiel return cfg -def _update_types(node: Node, ref_type: type, object_type: Optional[type]) -> None: +def _update_types(node: Node, ref_type: Any, object_type: Optional[type]) -> None: if object_type is not None and not is_primitive_dict(object_type): node._metadata.object_type = object_type if node._metadata.ref_type is Any: - new_is_optional, new_ref_type = _resolve_optional(ref_type) - if new_ref_type is not Any: - node._metadata.ref_type = new_ref_type - node._metadata.optional = new_is_optional + _deep_update_type_hint(node, ref_type) + + +def _deep_update_type_hint(node: Node, type_hint: Any) -> None: + """Ensure node is compatible with type_hint, mutating if necessary.""" + from omegaconf import DictConfig, ListConfig + + from ._utils import get_dict_key_value_types, get_list_element_type + + if type_hint is Any: + return + + _shallow_validate_type_hint(node, type_hint) + + new_is_optional, new_ref_type = _resolve_optional(type_hint) + node._metadata.ref_type = new_ref_type + node._metadata.optional = new_is_optional + + if is_list_annotation(new_ref_type) and isinstance(node, ListConfig): + new_element_type = get_list_element_type(new_ref_type) + node._metadata.element_type = new_element_type + if not _is_special(node): + for i in range(len(node)): + _deep_update_subnode(node, i, new_element_type) + + if is_dict_annotation(new_ref_type) and isinstance(node, DictConfig): + new_key_type, new_element_type = get_dict_key_value_types(new_ref_type) + node._metadata.key_type = new_key_type + node._metadata.element_type = new_element_type + if not _is_special(node): + for key in node: + if new_key_type is not Any and not isinstance(key, new_key_type): + raise KeyValidationError( + f"Key {key!r} ({type(key).__name__}) is incompatible" + + f" with key type hint '{new_key_type.__name__}'" + ) + _deep_update_subnode(node, key, new_element_type) + + +def _deep_update_subnode(node: BaseContainer, key: Any, value_type_hint: Any) -> None: + """Get node[key] and ensure it is compatible with value_type_hint, mutating if necessary.""" + subnode = node._get_node(key) + assert isinstance(subnode, Node) + if _is_special(subnode): + # Ensure special values are wrapped in a Node subclass that + # is compatible with the type hint. + node._wrap_value_and_set(key, subnode._value(), value_type_hint) + subnode = node._get_node(key) + assert isinstance(subnode, Node) + _deep_update_type_hint(subnode, value_type_hint) + + +def _shallow_validate_type_hint(node: Node, type_hint: Any) -> None: + """Error if node's type, content and metadata are not compatible with type_hint.""" + from omegaconf import DictConfig, ListConfig, ValueNode + + is_optional, ref_type = _resolve_optional(type_hint) + + vk = get_value_kind(node) + + if node._is_none(): + if not is_optional: + value = _get_value(node) + raise ValidationError( + f"Value {value!r} ({type(value).__name__})" + + f" is incompatible with type hint '{ref_type.__name__}'" + ) + return + elif vk in (ValueKind.MANDATORY_MISSING, ValueKind.INTERPOLATION): + return + elif vk == ValueKind.VALUE: + if is_primitive_type_annotation(ref_type) and isinstance(node, ValueNode): + value = node._value() + if not isinstance(value, ref_type): + raise ValidationError( + f"Value {value!r} ({type(value).__name__})" + + f" is incompatible with type hint '{ref_type.__name__}'" + ) + elif is_structured_config(ref_type) and isinstance(node, DictConfig): + return + elif is_dict_annotation(ref_type) and isinstance(node, DictConfig): + return + elif is_list_annotation(ref_type) and isinstance(node, ListConfig): + return + else: + if isinstance(node, ValueNode): + value = node._value() + raise ValidationError( + f"Value {value!r} ({type(value).__name__})" + + f" is incompatible with type hint '{ref_type}'" + ) + else: + raise ValidationError( + f"'{type(node).__name__}' is incompatible" + + f" with type hint '{ref_type}'" + ) + + else: + assert False diff -pruN 2.1.0~rc1-3/omegaconf/base.py 2.2.2-1/omegaconf/base.py --- 2.1.0~rc1-3/omegaconf/base.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/omegaconf/base.py 2022-05-27 21:36:40.000000000 +0000 @@ -10,12 +10,18 @@ from antlr4 import ParserRuleContext from ._utils import ( _DEFAULT_MARKER_, + NoneType, ValueKind, _get_value, + _is_interpolation, _is_missing_value, + _is_special, format_and_raise, get_value_kind, + is_union_annotation, + is_valid_value_annotation, split_key, + type_str, ) from .errors import ( ConfigKeyError, @@ -32,7 +38,7 @@ from .grammar.gen.OmegaConfGrammarParser from .grammar_parser import parse from .grammar_visitor import GrammarVisitor -DictKeyType = Union[str, int, Enum, float, bool] +DictKeyType = Union[str, bytes, int, Enum, float, bool] @dataclass @@ -62,6 +68,17 @@ class Metadata: if self.flags is None: self.flags = {} + @property + def type_hint(self) -> Union[Type[Any], Any]: + """Compute `type_hint` from `self.optional` and `self.ref_type`""" + # For compatibility with pickled OmegaConf objects created using older + # versions of OmegaConf, we store `ref_type` and `object_type` + # separately (rather than storing `type_hint` directly). + if self.optional: + return Optional[self.ref_type] + else: + return self.ref_type + @dataclass class ContainerMetadata(Metadata): @@ -73,7 +90,10 @@ class ContainerMetadata(Metadata): self.ref_type = Any assert self.key_type is Any or isinstance(self.key_type, type) if self.element_type is not None: - assert self.element_type is Any or isinstance(self.element_type, type) + if not is_valid_value_annotation(self.element_type): + raise ValidationError( + f"Unsupported value type: '{type_str(self.element_type, include_module_name=True)}'" + ) if self.flags is None: self.flags = {} @@ -82,10 +102,10 @@ class ContainerMetadata(Metadata): class Node(ABC): _metadata: Metadata - _parent: Optional["Container"] + _parent: Optional["Box"] _flags_cache: Optional[Dict[str, Optional[bool]]] - def __init__(self, parent: Optional["Container"], metadata: Metadata): + def __init__(self, parent: Optional["Box"], metadata: Metadata): self.__dict__["_metadata"] = metadata self.__dict__["_parent"] = parent self.__dict__["_flags_cache"] = None @@ -100,19 +120,35 @@ class Node(ABC): self.__dict__.update(state_dict) self.__dict__["_flags_cache"] = None - def _set_parent(self, parent: Optional["Container"]) -> None: - assert parent is None or isinstance(parent, Container) + def _set_parent(self, parent: Optional["Box"]) -> None: + assert parent is None or isinstance(parent, Box) self.__dict__["_parent"] = parent self._invalidate_flags_cache() def _invalidate_flags_cache(self) -> None: self.__dict__["_flags_cache"] = None - def _get_parent(self) -> Optional["Container"]: + def _get_parent(self) -> Optional["Box"]: parent = self.__dict__["_parent"] - assert parent is None or isinstance(parent, Container) + assert parent is None or isinstance(parent, Box) return parent + def _get_parent_container(self) -> Optional["Container"]: + """ + Like _get_parent, but returns the grandparent + in the case where `self` is wrapped by a UnionNode. + """ + parent = self.__dict__["_parent"] + assert parent is None or isinstance(parent, Box) + + if isinstance(parent, UnionNode): + grandparent = parent.__dict__["_parent"] + assert grandparent is None or isinstance(grandparent, Container) + return grandparent + else: + assert parent is None or isinstance(parent, Container) + return parent + def _set_flag( self, flags: Union[List[str], str], @@ -185,13 +221,18 @@ class Node(ABC): return parent._get_flag(flag) def _format_and_raise( - self, key: Any, value: Any, cause: Exception, type_override: Any = None + self, + key: Any, + value: Any, + cause: Exception, + msg: Optional[str] = None, + type_override: Any = None, ) -> None: format_and_raise( node=self, key=key, value=value, - msg=str(cause), + msg=str(cause) if msg is None else msg, cause=cause, type_override=type_override, ) @@ -224,7 +265,7 @@ class Node(ABC): if not self._is_interpolation(): return self - parent = self._get_parent() + parent = self._get_parent_container() if parent is None: if throw_on_resolution_failure: raise InterpolationResolutionError( @@ -243,14 +284,15 @@ class Node(ABC): ) def _get_root(self) -> "Container": - root: Optional[Container] = self._get_parent() + root: Optional[Box] = self._get_parent() if root is None: assert isinstance(self, Container) return self - assert root is not None and isinstance(root, Container) + assert root is not None and isinstance(root, Box) while root._get_parent() is not None: root = root._get_parent() - assert root is not None and isinstance(root, Container) + assert root is not None and isinstance(root, Box) + assert root is not None and isinstance(root, Container) return root def _is_missing(self) -> bool: @@ -307,18 +349,82 @@ class Node(ABC): self._metadata.flags_root = flags_root self._invalidate_flags_cache() + def _has_ref_type(self) -> bool: + return self._metadata.ref_type is not Any + + +class Box(Node): + """ + Base class for nodes that can contain other nodes. + Concrete subclasses include DictConfig, ListConfig, and UnionNode. + """ + + _content: Any + + def __init__(self, parent: Optional["Box"], metadata: Metadata): + super().__init__(parent=parent, metadata=metadata) + self.__dict__["_content"] = None + + def __copy__(self) -> Any: + # real shallow copy is impossible because of the reference to the parent. + return copy.deepcopy(self) + + def _re_parent(self) -> None: + from .dictconfig import DictConfig + from .listconfig import ListConfig + + # update parents of first level Config nodes to self + + if isinstance(self, DictConfig): + content = self.__dict__["_content"] + if isinstance(content, dict): + for _key, value in self.__dict__["_content"].items(): + if value is not None: + value._set_parent(self) + if isinstance(value, Box): + value._re_parent() + elif isinstance(self, ListConfig): + content = self.__dict__["_content"] + if isinstance(content, list): + for item in self.__dict__["_content"]: + if item is not None: + item._set_parent(self) + if isinstance(item, Box): + item._re_parent() + elif isinstance(self, UnionNode): + content = self.__dict__["_content"] + if isinstance(content, Node): + content._set_parent(self) + if isinstance(content, Box): # pragma: no cover + # No coverage here as support for containers inside + # UnionNode is not yet implemented + content._re_parent() -class Container(Node): + +class Container(Box): """ Container tagging interface """ _metadata: ContainerMetadata + @abstractmethod + def _get_child( + self, + key: Any, + validate_access: bool = True, + validate_key: bool = True, + throw_on_missing_value: bool = False, + throw_on_missing_key: bool = False, + ) -> Union[Optional[Node], List[Optional[Node]]]: + ... + + @abstractmethod def _get_node( self, key: Any, validate_access: bool = True, + validate_key: bool = True, throw_on_missing_value: bool = False, throw_on_missing_key: bool = False, ) -> Union[Optional[Node], List[Optional[Node]]]: @@ -340,10 +446,6 @@ class Container(Node): def __getitem__(self, key_or_index: Any) -> Any: ... - def __copy__(self) -> Any: - # real shallow copy is impossible because of the reference to the parent. - return copy.deepcopy(self) - def _resolve_key_and_root(self, key: str) -> Tuple["Container", str]: orig = key if not key.startswith("."): @@ -356,7 +458,7 @@ class Container(Node): key = key[1:] if not key.startswith("."): break - root = root._get_parent() + root = root._get_parent_container() if root is None: raise ConfigKeyError(f"Error resolving key '{orig}'") @@ -480,7 +582,7 @@ class Container(Node): try: resolved = self.resolve_parse_tree( - parse_tree=parse_tree, node=value, key=key, parent=parent, memo=memo + parse_tree=parse_tree, node=value, key=key, memo=memo ) except InterpolationResolutionError: if throw_on_resolution_failure: @@ -520,6 +622,7 @@ class Container(Node): key=key, value=res_value, cause=e, + msg=f"While dereferencing interpolation '{value}': {e}", type_override=InterpolationValidationError, ) return None @@ -628,7 +731,6 @@ class Container(Node): node: Node, memo: Optional[Set[int]] = None, key: Optional[Any] = None, - parent: Optional["Container"] = None, ) -> Any: """ Resolve a given parse tree into its value. @@ -668,30 +770,6 @@ class Container(Node): f"{type(exc).__name__} raised while resolving interpolation: {exc}" ).with_traceback(sys.exc_info()[2]) - def _re_parent(self) -> None: - from .dictconfig import DictConfig - from .listconfig import ListConfig - - # update parents of first level Config nodes to self - - if isinstance(self, Container): - if isinstance(self, DictConfig): - content = self.__dict__["_content"] - if isinstance(content, dict): - for _key, value in self.__dict__["_content"].items(): - if value is not None: - value._set_parent(self) - if isinstance(value, Container): - value._re_parent() - elif isinstance(self, ListConfig): - content = self.__dict__["_content"] - if isinstance(content, list): - for item in self.__dict__["_content"]: - if item is not None: - item._set_parent(self) - if isinstance(item, Container): - item._re_parent() - def _invalidate_flags_cache(self) -> None: from .dictconfig import DictConfig from .listconfig import ListConfig @@ -711,11 +789,174 @@ class Container(Node): for item in self.__dict__["_content"]: item._invalidate_flags_cache() - def _has_ref_type(self) -> bool: - return self._metadata.ref_type is not Any - class SCMode(Enum): DICT = 1 # Convert to plain dict DICT_CONFIG = 2 # Keep as OmegaConf DictConfig INSTANTIATE = 3 # Create a dataclass or attrs class instance + + +class UnionNode(Box): + """ + This class handles Union type hints. The `_content` attribute is either a + child node that is compatible with the given Union ref_type, or it is a + special value (None or MISSING or interpolation). + + Much of the logic for e.g. value assignment and type validation is + delegated to the child node. As such, UnionNode functions as a + "pass-through" node. User apps and downstream libraries should not need to + know about UnionNode (assuming they only use OmegaConf's public API). + """ + + _parent: Optional[Container] + _content: Union[Node, None, str] + + def __init__( + self, + content: Any, + ref_type: Any, + is_optional: bool = True, + key: Any = None, + parent: Optional[Box] = None, + ) -> None: + try: + if not is_union_annotation(ref_type): # pragma: no cover + msg = ( + f"UnionNode got unexpected ref_type {ref_type}. Please file a bug" + + " report at https://github.com/omry/omegaconf/issues" + ) + raise AssertionError(msg) + if not isinstance(parent, (Container, NoneType)): + raise ConfigTypeError("Parent type is not omegaconf.Container") + super().__init__( + parent=parent, + metadata=Metadata( + ref_type=ref_type, + object_type=None, + optional=is_optional, + key=key, + flags={"convert": False}, + ), + ) + self._set_value(content) + except Exception as ex: + format_and_raise(node=None, key=key, value=content, msg=str(ex), cause=ex) + + def _get_full_key(self, key: Optional[Union[DictKeyType, int]]) -> str: + parent = self._get_parent() + if parent is None: + if self._metadata.key is None: + return "" + else: + return str(self._metadata.key) + else: + return parent._get_full_key(self._metadata.key) + + def __eq__(self, other: Any) -> bool: + content = self.__dict__["_content"] + if isinstance(content, Node): + ret = content.__eq__(other) + elif isinstance(other, Node): + ret = other.__eq__(content) + else: + ret = content.__eq__(other) + assert isinstance(ret, (bool, type(NotImplemented))) + return ret + + def __ne__(self, other: Any) -> bool: + x = self.__eq__(other) + if x is NotImplemented: + return NotImplemented + return not x + + def __hash__(self) -> int: + return hash(self.__dict__["_content"]) + + def _value(self) -> Union[Node, None, str]: + content = self.__dict__["_content"] + assert isinstance(content, (Node, NoneType, str)) + return content + + def _set_value(self, value: Any, flags: Optional[Dict[str, bool]] = None) -> None: + previous_content = self.__dict__["_content"] + previous_metadata = self.__dict__["_metadata"] + try: + self._set_value_impl(value, flags) + except Exception as e: + self.__dict__["_content"] = previous_content + self.__dict__["_metadata"] = previous_metadata + raise e + + def _set_value_impl( + self, value: Any, flags: Optional[Dict[str, bool]] = None + ) -> None: + from omegaconf.omegaconf import _node_wrap + + ref_type = self._metadata.ref_type + type_hint = self._metadata.type_hint + + value = _get_value(value) + if _is_special(value): + assert isinstance(value, (str, NoneType)) + if value is None: + if not self._is_optional(): + raise ValidationError( + f"Value '$VALUE' is incompatible with type hint '{type_str(type_hint)}'" + ) + self.__dict__["_content"] = value + elif isinstance(value, Container): + raise ValidationError( + f"Cannot assign container '$VALUE' of type '$VALUE_TYPE' to {type_str(type_hint)}" + ) + else: + for candidate_ref_type in ref_type.__args__: + try: + self.__dict__["_content"] = _node_wrap( + value=value, + ref_type=candidate_ref_type, + is_optional=False, + key=None, + parent=self, + ) + break + except ValidationError: + continue + else: + raise ValidationError( + f"Value '$VALUE' of type '$VALUE_TYPE' is incompatible with type hint '{type_str(type_hint)}'" + ) + + def _is_optional(self) -> bool: + return self.__dict__["_metadata"].optional is True + + def _is_interpolation(self) -> bool: + return _is_interpolation(self.__dict__["_content"]) + + def __str__(self) -> str: + return str(self.__dict__["_content"]) + + def __repr__(self) -> str: + return repr(self.__dict__["_content"]) + + def __deepcopy__(self, memo: Dict[int, Any]) -> "UnionNode": + res = object.__new__(type(self)) + for key, value in self.__dict__.items(): + if key not in ("_content", "_parent"): + res.__dict__[key] = copy.deepcopy(value, memo=memo) + + src_content = self.__dict__["_content"] + if isinstance(src_content, Node): + old_parent = src_content.__dict__["_parent"] + try: + src_content.__dict__["_parent"] = None + content_copy = copy.deepcopy(src_content, memo=memo) + content_copy.__dict__["_parent"] = res + finally: + src_content.__dict__["_parent"] = old_parent + else: + # None and strings can be assigned as is + content_copy = src_content + + res.__dict__["_content"] = content_copy + res.__dict__["_parent"] = self.__dict__["_parent"] + return res diff -pruN 2.1.0~rc1-3/omegaconf/dictconfig.py 2.2.2-1/omegaconf/dictconfig.py --- 2.1.0~rc1-3/omegaconf/dictconfig.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/omegaconf/dictconfig.py 2022-05-27 21:36:40.000000000 +0000 @@ -1,11 +1,12 @@ import copy from enum import Enum from typing import ( - AbstractSet, Any, Dict, + ItemsView, Iterable, Iterator, + KeysView, List, MutableMapping, Optional, @@ -23,10 +24,11 @@ from ._utils import ( _is_missing_literal, _is_missing_value, _is_none, + _resolve_optional, _valid_dict_key_annotation_type, format_and_raise, get_structured_config_data, - get_structured_config_field_names, + get_structured_config_init_field_names, get_type_of, get_value_kind, is_container_annotation, @@ -35,9 +37,8 @@ from ._utils import ( is_structured_config, is_structured_config_frozen, type_str, - valid_value_annotation_type, ) -from .base import Container, ContainerMetadata, DictKeyType, Node, SCMode +from .base import Box, Container, ContainerMetadata, DictKeyType, Node from .basecontainer import BaseContainer from .errors import ( ConfigAttributeError, @@ -60,9 +61,9 @@ class DictConfig(BaseContainer, MutableM def __init__( self, - content: Union[Dict[DictKeyType, Any], Any], + content: Union[Dict[DictKeyType, Any], "DictConfig", Any], key: Any = None, - parent: Optional[Container] = None, + parent: Optional[Box] = None, ref_type: Union[Any, Type[Any]] = Any, key_type: Union[Any, Type[Any]] = Any, element_type: Union[Any, Type[Any]] = Any, @@ -79,16 +80,12 @@ class DictConfig(BaseContainer, MutableM key=key, optional=is_optional, ref_type=ref_type, - object_type=None, + object_type=dict, key_type=key_type, element_type=element_type, flags=flags, ), ) - if not valid_value_annotation_type( - element_type - ) and not is_structured_config(element_type): - raise ValidationError(f"Unsupported value type: {element_type}") if not _valid_dict_key_annotation_type(key_type): raise KeyValidationError(f"Unsupported key type {key_type}") @@ -104,6 +101,7 @@ class DictConfig(BaseContainer, MutableM if isinstance(content, DictConfig): metadata = copy.deepcopy(content._metadata) metadata.key = key + metadata.ref_type = ref_type metadata.optional = is_optional metadata.element_type = element_type metadata.key_type = key_type @@ -159,7 +157,7 @@ class DictConfig(BaseContainer, MutableM return if is_typed or is_struct: if is_typed: - assert self._metadata.object_type is not None + assert self._metadata.object_type not in (dict, None) msg = f"Key '{key}' not in '{self._metadata.object_type.__name__}'" else: msg = f"Key '{key}' is not in struct" @@ -173,7 +171,9 @@ class DictConfig(BaseContainer, MutableM vk = get_value_kind(value) if vk == ValueKind.INTERPOLATION: return - self._validate_non_optional(key, value) + if _is_none(value): + self._validate_non_optional(key, value) + return if vk == ValueKind.MANDATORY_MISSING or value is None: return @@ -219,7 +219,7 @@ class DictConfig(BaseContainer, MutableM dest_obj_type = OmegaConf.get_type(dest) src_obj_type = OmegaConf.get_type(src) - if dest._is_missing() and src._metadata.object_type is not None: + if dest._is_missing() and src._metadata.object_type not in (dict, None): self._validate_set(key=None, value=_get_value(src)) if src._is_missing(): @@ -240,25 +240,27 @@ class DictConfig(BaseContainer, MutableM ) raise ValidationError(msg) - def _validate_non_optional(self, key: Any, value: Any) -> None: + def _validate_non_optional(self, key: Optional[DictKeyType], value: Any) -> None: if _is_none(value, resolve=True, throw_on_resolution_failure=False): + if key is not None: child = self._get_node(key) if child is not None: assert isinstance(child, Node) - if not child._is_optional(): - self._format_and_raise( - key=key, - value=value, - cause=ValidationError("child '$FULL_KEY' is not Optional"), - ) - else: - if not self._is_optional(): - self._format_and_raise( - key=None, - value=value, - cause=ValidationError("field '$FULL_KEY' is not Optional"), + field_is_optional = child._is_optional() + else: + field_is_optional, _ = _resolve_optional( + self._metadata.element_type ) + else: + field_is_optional = self._is_optional() + + if not field_is_optional: + self._format_and_raise( + key=key, + value=value, + cause=ValidationError("field '$FULL_KEY' is not Optional"), + ) def _raise_invalid_value( self, value: Any, value_type: Any, target_type: Any @@ -285,7 +287,7 @@ class DictConfig(BaseContainer, MutableM # assert hash(0) == hash(False) # assert hash(1) == hash(True) return bool(key) - elif key_type in (str, int, float, bool): # primitive type + elif key_type in (str, bytes, int, float, bool): # primitive type if not isinstance(key, key_type): raise KeyValidationError( f"Key $KEY ($KEY_TYPE) is incompatible with ({key_type.__name__})" @@ -348,7 +350,9 @@ class DictConfig(BaseContainer, MutableM raise AttributeError() try: - return self._get_impl(key=key, default_value=_DEFAULT_MARKER_) + return self._get_impl( + key=key, default_value=_DEFAULT_MARKER_, validate_key=False + ) except ConfigKeyError as e: self._format_and_raise( key=key, value=None, cause=e, type_override=ConfigAttributeError @@ -433,9 +437,13 @@ class DictConfig(BaseContainer, MutableM except KeyValidationError as e: self._format_and_raise(key=key, value=None, cause=e) - def _get_impl(self, key: DictKeyType, default_value: Any) -> Any: + def _get_impl( + self, key: DictKeyType, default_value: Any, validate_key: bool = True + ) -> Any: try: - node = self._get_node(key=key, throw_on_missing_key=True) + node = self._get_child( + key=key, throw_on_missing_key=True, validate_key=validate_key + ) except (ConfigAttributeError, ConfigKeyError): if default_value is not _DEFAULT_MARKER_: return default_value @@ -450,16 +458,20 @@ class DictConfig(BaseContainer, MutableM self, key: DictKeyType, validate_access: bool = True, + validate_key: bool = True, throw_on_missing_value: bool = False, throw_on_missing_key: bool = False, ) -> Union[Optional[Node], List[Optional[Node]]]: try: key = self._validate_and_normalize_key(key) except KeyValidationError: - if validate_access: + if validate_access and validate_key: raise else: - return None + if throw_on_missing_key: + raise ConfigAttributeError + else: + return None if validate_access: self._validate_get(key) @@ -467,9 +479,9 @@ class DictConfig(BaseContainer, MutableM value: Optional[Node] = self.__dict__["_content"].get(key) if value is None: if throw_on_missing_key: - raise ConfigKeyError(f"Missing key {key}") + raise ConfigKeyError(f"Missing key {key!s}") elif throw_on_missing_value and value._is_missing(): - raise MissingMandatoryValue("Missing mandatory value") + raise MissingMandatoryValue("Missing mandatory value: $KEY") return value def pop(self, key: DictKeyType, default: Any = _DEFAULT_MARKER_) -> Any: @@ -483,7 +495,7 @@ class DictConfig(BaseContainer, MutableM f"{type_str(self._metadata.object_type)} (DictConfig) does not support pop" ) key = self._validate_and_normalize_key(key) - node = self._get_node(key=key, validate_access=False) + node = self._get_child(key=key, validate_access=False) if node is not None: assert isinstance(node, Node) value = self._resolve_with_default( @@ -498,16 +510,20 @@ class DictConfig(BaseContainer, MutableM else: full = self._get_full_key(key=key) if full != key: - raise ConfigKeyError(f"Key not found: '{key}' (path: '{full}')") + raise ConfigKeyError( + f"Key not found: '{key!s}' (path: '{full}')" + ) else: - raise ConfigKeyError(f"Key not found: '{key}'") + raise ConfigKeyError(f"Key not found: '{key!s}'") except Exception as e: self._format_and_raise(key=key, value=None, cause=e) - def keys(self) -> Any: + def keys(self) -> KeysView[DictKeyType]: if self._is_missing() or self._is_interpolation() or self._is_none(): - return list() - return self.__dict__["_content"].keys() + return {}.keys() + ret = self.__dict__["_content"].keys() + assert isinstance(ret, KeysView) + return ret def __contains__(self, key: object) -> bool: """ @@ -523,7 +539,7 @@ class DictConfig(BaseContainer, MutableM return False try: - node = self._get_node(key) + node = self._get_child(key) assert node is None or isinstance(node, Node) except (KeyError, AttributeError): node = None @@ -544,8 +560,8 @@ class DictConfig(BaseContainer, MutableM def __iter__(self) -> Iterator[DictKeyType]: return iter(self.keys()) - def items(self) -> AbstractSet[Tuple[DictKeyType, Any]]: - return self.items_ex(resolve=True, keys=None) + def items(self) -> ItemsView[DictKeyType, Any]: + return dict(self.items_ex(resolve=True, keys=None)).items() def setdefault(self, key: DictKeyType, default: Any = None) -> Any: if key in self: @@ -557,7 +573,7 @@ class DictConfig(BaseContainer, MutableM def items_ex( self, resolve: bool = True, keys: Optional[Sequence[DictKeyType]] = None - ) -> AbstractSet[Tuple[DictKeyType, Any]]: + ) -> List[Tuple[DictKeyType, Any]]: items: List[Tuple[DictKeyType, Any]] = [] if self._is_none(): @@ -579,10 +595,7 @@ class DictConfig(BaseContainer, MutableM if keys is None or key in keys: items.append((key, value)) - # For some reason items wants to return a Set, but if the values are not - # hashable this is a problem. We use a list instead. most use cases should just - # be iterating on pairs anyway. - return items # type: ignore + return items def __eq__(self, other: Any) -> bool: if other is None: @@ -592,6 +605,8 @@ class DictConfig(BaseContainer, MutableM return DictConfig._dict_conf_eq(self, other) if isinstance(other, DictConfig): return DictConfig._dict_conf_eq(self, other) + if self._is_missing(): + return _is_missing_literal(other) return NotImplemented def __ne__(self, other: Any) -> bool: @@ -665,16 +680,17 @@ class DictConfig(BaseContainer, MutableM self._metadata.object_type = get_type_of(value) elif isinstance(value, DictConfig): - self.__dict__["_metadata"] = copy.deepcopy(value._metadata) self._metadata.flags = copy.deepcopy(flags) with flag_override(self, ["struct", "readonly"], False): for k, v in value.__dict__["_content"].items(): self.__setitem__(k, v) + self._metadata.object_type = value._metadata.object_type elif isinstance(value, dict): with flag_override(self, ["struct", "readonly"], False): for k, v in value.items(): self.__setitem__(k, v) + self._metadata.object_type = dict else: # pragma: no cover msg = f"Unsupported value type: {value}" @@ -709,29 +725,27 @@ class DictConfig(BaseContainer, MutableM """ Instantiate an instance of `self._metadata.object_type`. This requires `self` to be a structured config. - Nested subconfigs are converted to_container with resolve=True. + Nested subconfigs are converted by calling `OmegaConf.to_object`. """ + from omegaconf import OmegaConf + object_type = self._metadata.object_type assert is_structured_config(object_type) - object_type_field_names = set(get_structured_config_field_names(object_type)) + init_field_names = set(get_structured_config_init_field_names(object_type)) - field_items: Dict[str, Any] = {} - nonfield_items: Dict[str, Any] = {} + init_field_items: Dict[str, Any] = {} + non_init_field_items: Dict[str, Any] = {} for k in self.keys(): - node = self._get_node(k) + assert isinstance(k, str) + node = self._get_child(k) assert isinstance(node, Node) - node = node._dereference_node() - if isinstance(node, Container): - v = BaseContainer._to_content( - node, - resolve=True, - enum_to_str=False, - structured_config_mode=SCMode.INSTANTIATE, - ) - else: - v = node._value() - - if _is_missing_literal(v): + try: + node = node._dereference_node() + except InterpolationResolutionError as e: + self._format_and_raise(key=k, value=None, cause=e) + if node._is_missing(): + if k not in init_field_names: + continue # MISSING is ignored for init=False fields self._format_and_raise( key=k, value=None, @@ -739,12 +753,17 @@ class DictConfig(BaseContainer, MutableM "Structured config of type `$OBJECT_TYPE` has missing mandatory value: $KEY" ), ) - if k in object_type_field_names: - field_items[k] = v + if isinstance(node, Container): + v = OmegaConf.to_object(node) + else: + v = node._value() + + if k in init_field_names: + init_field_items[k] = v else: - nonfield_items[k] = v + non_init_field_items[k] = v - result = object_type(**field_items) - for k, v in nonfield_items.items(): + result = object_type(**init_field_items) + for k, v in non_init_field_items.items(): setattr(result, k, v) return result diff -pruN 2.1.0~rc1-3/omegaconf/grammar/OmegaConfGrammarLexer.g4 2.2.2-1/omegaconf/grammar/OmegaConfGrammarLexer.g4 --- 2.1.0~rc1-3/omegaconf/grammar/OmegaConfGrammarLexer.g4 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/omegaconf/grammar/OmegaConfGrammarLexer.g4 2022-05-27 21:36:40.000000000 +0000 @@ -65,7 +65,7 @@ BOOL: NULL: [Nn][Uu][Ll][Ll]; -UNQUOTED_CHAR: [/\-\\+.$%*@?]; // other characters allowed in unquoted strings +UNQUOTED_CHAR: [/\-\\+.$%*@?|]; // other characters allowed in unquoted strings ID: (CHAR|'_') (CHAR|DIGIT|'_')*; ESC: (ESC_BACKSLASH | '\\(' | '\\)' | '\\[' | '\\]' | '\\{' | '\\}' | '\\:' | '\\=' | '\\,' | '\\ ' | '\\\t')+; diff -pruN 2.1.0~rc1-3/omegaconf/grammar/OmegaConfGrammarParser.g4 2.2.2-1/omegaconf/grammar/OmegaConfGrammarParser.g4 --- 2.1.0~rc1-3/omegaconf/grammar/OmegaConfGrammarParser.g4 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/omegaconf/grammar/OmegaConfGrammarParser.g4 2022-05-27 21:36:40.000000000 +0000 @@ -10,7 +10,8 @@ // - Keep up-to-date the comments in the visitor (in `grammar_visitor.py`) // that contain grammar excerpts (within each `visit...()` method). // -// - Remember to update the documentation (including the tutorial notebook) +// - Remember to update the documentation (including the tutorial notebook as +// well as grammar.rst) parser grammar OmegaConfGrammarParser; options {tokenVocab = OmegaConfGrammarLexer;} @@ -70,7 +71,7 @@ primitive: | INT // 0, 10, -20, 1_000_000 | FLOAT // 3.14, -20.0, 1e-1, -10e3 | BOOL // true, TrUe, false, False - | UNQUOTED_CHAR // /, -, \, +, ., $, %, *, @ + | UNQUOTED_CHAR // /, -, \, +, ., $, %, *, @, ?, | | COLON // : | ESC // \\, \(, \), \[, \], \{, \}, \:, \=, \ , \\t, \, | WS // whitespaces @@ -84,7 +85,7 @@ dictKey: | INT // 0, 10, -20, 1_000_000 | FLOAT // 3.14, -20.0, 1e-1, -10e3 | BOOL // true, TrUe, false, False - | UNQUOTED_CHAR // /, -, \, +, ., $, %, *, @ + | UNQUOTED_CHAR // /, -, \, +, ., $, %, *, @, ?, | | ESC // \\, \(, \), \[, \], \{, \}, \:, \=, \ , \\t, \, | WS // whitespaces )+; \ No newline at end of file diff -pruN 2.1.0~rc1-3/omegaconf/grammar_parser.py 2.2.2-1/omegaconf/grammar_parser.py --- 2.1.0~rc1-3/omegaconf/grammar_parser.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/omegaconf/grammar_parser.py 2022-05-27 21:36:40.000000000 +0000 @@ -1,4 +1,5 @@ import re +import threading from typing import Any from antlr4 import CommonTokenStream, InputStream, ParserRuleContext @@ -14,7 +15,8 @@ from .grammar_visitor import ( # type: ) # Used to cache grammar objects to avoid re-creating them on each call to `parse()`. -_grammar_cache = None +# We use a per-thread cache to make it thread-safe. +_grammar_cache = threading.local() # Build regex pattern to efficiently identify typical interpolations. # See test `test_match_simple_interpolation_pattern` for examples. @@ -25,7 +27,7 @@ _node_path = f"(\\.)*({_key_maybe_bracke _node_inter = f"\\${{\\s*{_node_path}\\s*}}" # node interpolation ${foo.bar} _id = "[a-zA-Z_]\\w*" # foo, foo_bar, abc123 _resolver_name = f"({_id}(\\.{_id})*)?" # foo, ns.bar3, ns_1.ns_2.b0z -_arg = "[a-zA-Z_0-9/\\-\\+.$%*@]+" # string representing a resolver argument +_arg = r"[a-zA-Z_0-9/\-\+.$%*@?|]+" # string representing a resolver argument _args = f"{_arg}(\\s*,\\s*{_arg})*" # list of resolver arguments _resolver_inter = f"\\${{\\s*{_resolver_name}\\s*:\\s*{_args}?\\s*}}" # ${foo:bar} _inter = f"({_node_inter}|{_resolver_inter})" # any kind of interpolation @@ -33,6 +35,9 @@ _outer = "([^$]|\\$(?!{))+" # any chara SIMPLE_INTERPOLATION_PATTERN = re.compile( f"({_outer})?({_inter}({_outer})?)+$", flags=re.ASCII ) +# NOTE: SIMPLE_INTERPOLATION_PATTERN must not generate false positive matches: +# it must not accept anything that isn't a valid interpolation (per the +# interpolation grammar defined in `omegaconf/grammar/*.g4`). class OmegaConfErrorListener(ErrorListener): # type: ignore @@ -94,19 +99,18 @@ def parse( """ Parse interpolated string `value` (and return the parse tree). """ - global _grammar_cache - l_mode = getattr(OmegaConfGrammarLexer, lexer_mode) istream = InputStream(value) - if _grammar_cache is None: + cached = getattr(_grammar_cache, "data", None) + if cached is None: error_listener = OmegaConfErrorListener() lexer = OmegaConfGrammarLexer(istream) lexer.removeErrorListeners() lexer.addErrorListener(error_listener) lexer.mode(l_mode) - tokens = CommonTokenStream(lexer) - parser = OmegaConfGrammarParser(tokens) + token_stream = CommonTokenStream(lexer) + parser = OmegaConfGrammarParser(token_stream) parser.removeErrorListeners() parser.addErrorListener(error_listener) @@ -115,13 +119,17 @@ def parse( # from antlr4 import PredictionMode # parser._interp.predictionMode = PredictionMode.SLL - _grammar_cache = lexer, tokens, parser + # Note that although the input stream `istream` is implicitly cached within + # the lexer, it will be replaced by a new input next time the lexer is re-used. + _grammar_cache.data = lexer, token_stream, parser else: - lexer, tokens, parser = _grammar_cache + lexer, token_stream, parser = cached + # Replace the old input stream with the new one. lexer.inputStream = istream + # Initialize the lexer / token stream / parser to process the new input. lexer.mode(l_mode) - tokens.setTokenSource(lexer) + token_stream.setTokenSource(lexer) parser.reset() try: diff -pruN 2.1.0~rc1-3/omegaconf/_impl.py 2.2.2-1/omegaconf/_impl.py --- 2.1.0~rc1-3/omegaconf/_impl.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/omegaconf/_impl.py 2022-05-27 21:36:40.000000000 +0000 @@ -7,7 +7,7 @@ from ._utils import _DEFAULT_MARKER_, _g def _resolve_container_value(cfg: Container, key: Any) -> None: - node = cfg._get_node(key) + node = cfg._get_child(key) assert isinstance(node, Node) if node._is_interpolation(): try: @@ -20,7 +20,7 @@ def _resolve_container_value(cfg: Contai if isinstance(resolved, Container) and isinstance(node, ValueNode): cfg[key] = resolved else: - node._set_value(resolved._value()) + node._set_value(_get_value(resolved)) else: _resolve(node) diff -pruN 2.1.0~rc1-3/omegaconf/__init__.py 2.2.2-1/omegaconf/__init__.py --- 2.1.0~rc1-3/omegaconf/__init__.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/omegaconf/__init__.py 2022-05-27 21:36:40.000000000 +0000 @@ -1,4 +1,4 @@ -from .base import Container, DictKeyType, Node, SCMode +from .base import Container, DictKeyType, Node, SCMode, UnionNode from .dictconfig import DictConfig from .errors import ( KeyValidationError, @@ -11,9 +11,11 @@ from .listconfig import ListConfig from .nodes import ( AnyNode, BooleanNode, + BytesNode, EnumNode, FloatNode, IntegerNode, + PathNode, StringNode, ValueNode, ) @@ -37,6 +39,7 @@ __all__ = [ "UnsupportedValueType", "KeyValidationError", "Container", + "UnionNode", "ListConfig", "DictConfig", "DictKeyType", @@ -51,6 +54,8 @@ __all__ = [ "AnyNode", "IntegerNode", "StringNode", + "BytesNode", + "PathNode", "BooleanNode", "EnumNode", "FloatNode", diff -pruN 2.1.0~rc1-3/omegaconf/listconfig.py 2.2.2-1/omegaconf/listconfig.py --- 2.1.0~rc1-3/omegaconf/listconfig.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/omegaconf/listconfig.py 2022-05-27 21:36:40.000000000 +0000 @@ -16,17 +16,17 @@ from typing import ( from ._utils import ( ValueKind, + _is_missing_literal, _is_none, - _is_optional, + _resolve_optional, format_and_raise, get_value_kind, is_int, is_primitive_list, is_structured_config, type_str, - valid_value_annotation_type, ) -from .base import Container, ContainerMetadata, Node +from .base import Box, ContainerMetadata, Node from .basecontainer import BaseContainer from .errors import ( ConfigAttributeError, @@ -45,9 +45,9 @@ class ListConfig(BaseContainer, MutableS def __init__( self, - content: Union[List[Any], Tuple[Any, ...], str, None], + content: Union[List[Any], Tuple[Any, ...], "ListConfig", str, None], key: Any = None, - parent: Optional[Container] = None, + parent: Optional[Box] = None, element_type: Union[Type[Any], Any] = Any, is_optional: bool = True, ref_type: Union[Type[Any], Any] = Any, @@ -69,12 +69,14 @@ class ListConfig(BaseContainer, MutableS flags=flags, ), ) - if not (valid_value_annotation_type(self._metadata.element_type)): - raise ValidationError( - f"Unsupported value type: {self._metadata.element_type}" - ) - self.__dict__["_content"] = None + if isinstance(content, ListConfig): + metadata = copy.deepcopy(content._metadata) + metadata.key = key + metadata.ref_type = ref_type + metadata.optional = is_optional + metadata.element_type = element_type + self.__dict__["_metadata"] = metadata self._set_value(value=content, flags=flags) except Exception as ex: format_and_raise(node=None, key=key, value=None, cause=ex, msg=str(ex)) @@ -102,11 +104,15 @@ class ListConfig(BaseContainer, MutableS "$FULL_KEY is not optional and cannot be assigned None" ) - target_type = self._metadata.element_type - value_type = OmegaConf.get_type(value) - if is_structured_config(target_type): - if ( - target_type is not None + vk = get_value_kind(value) + if vk == ValueKind.MANDATORY_MISSING: + return + else: + is_optional, target_type = _resolve_optional(self._metadata.element_type) + value_type = OmegaConf.get_type(value) + + if (value_type is None and not is_optional) or ( + is_structured_config(target_type) and value_type is not None and not issubclass(value_type, target_type) ): @@ -241,26 +247,51 @@ class ListConfig(BaseContainer, MutableS def __setitem__(self, index: Union[int, slice], value: Any) -> None: try: - self._set_at_index(index, value) + if isinstance(index, slice): + _ = iter(value) # check iterable + self_indices = index.indices(len(self)) + indexes = range(*self_indices) + + # Ensure lengths match for extended slice assignment + if index.step not in (None, 1): + if len(indexes) != len(value): + raise ValueError( + f"attempt to assign sequence of size {len(value)}" + f" to extended slice of size {len(indexes)}" + ) + + # Initialize insertion offsets for empty slices + if len(indexes) == 0: + curr_index = self_indices[0] - 1 + val_i = -1 + + # Delete and optionally replace non empty slices + only_removed = 0 + for val_i, i in enumerate(indexes): + curr_index = i - only_removed + del self[curr_index] + if val_i < len(value): + self.insert(curr_index, value[val_i]) + else: + only_removed += 1 + + # Insert any remaining input items + for val_i in range(val_i + 1, len(value)): + curr_index += 1 + self.insert(curr_index, value[val_i]) + else: + self._set_at_index(index, value) except Exception as e: self._format_and_raise(key=index, value=value, cause=e) def append(self, item: Any) -> None: + content = self.__dict__["_content"] + index = len(content) + content.append(None) try: - from omegaconf.omegaconf import _maybe_wrap - - index = len(self) - self._validate_set(key=index, value=item) - - node = _maybe_wrap( - ref_type=self.__dict__["_metadata"].element_type, - key=index, - value=item, - is_optional=_is_optional(item), - parent=self, - ) - self.__dict__["_content"].append(node) + self._set_item_impl(index, item) except Exception as e: + del content[index] self._format_and_raise(key=index, value=item, cause=e) assert False @@ -288,11 +319,12 @@ class ListConfig(BaseContainer, MutableS assert isinstance(self.__dict__["_content"], list) # insert place holder self.__dict__["_content"].insert(index, None) + is_optional, ref_type = _resolve_optional(self._metadata.element_type) node = _maybe_wrap( - ref_type=self.__dict__["_metadata"].element_type, + ref_type=ref_type, key=index, value=item, - is_optional=_is_optional(item), + is_optional=is_optional, parent=self, ) self._validate_set(key=index, value=node) @@ -365,6 +397,7 @@ class ListConfig(BaseContainer, MutableS self, key: Union[int, slice], validate_access: bool = True, + validate_key: bool = True, throw_on_missing_value: bool = False, throw_on_missing_key: bool = False, ) -> Union[Optional[Node], List[Optional[Node]]]: @@ -389,7 +422,7 @@ class ListConfig(BaseContainer, MutableS else: assert isinstance(value, Node) if throw_on_missing_value and value._is_missing(): - raise MissingMandatoryValue("Missing mandatory value") + raise MissingMandatoryValue("Missing mandatory value: $KEY") return value except (IndexError, TypeError, MissingMandatoryValue, KeyValidationError) as e: if isinstance(e, MissingMandatoryValue) and throw_on_missing_value: @@ -427,7 +460,7 @@ class ListConfig(BaseContainer, MutableS raise MissingMandatoryValue("Cannot pop from a missing ListConfig") assert isinstance(self.__dict__["_content"], list) - node = self._get_node(index) + node = self._get_child(index) assert isinstance(node, Node) ret = self._resolve_with_default(key=index, value=node, default_value=None) del self.__dict__["_content"][index] @@ -476,6 +509,8 @@ class ListConfig(BaseContainer, MutableS return ListConfig._list_eq(self, other) if other is None or isinstance(other, ListConfig): return ListConfig._list_eq(self, other) + if self._is_missing(): + return _is_missing_literal(other) return NotImplemented def __ne__(self, other: Any) -> bool: @@ -538,6 +573,13 @@ class ListConfig(BaseContainer, MutableS res.extend(other) return res + def __radd__(self, other: Union[List[Any], "ListConfig"]) -> "ListConfig": + # res is sharing this list's parent to allow interpolation to work as expected + res = ListConfig(parent=self._get_parent(), content=[]) + res.extend(other) + res.extend(self) + return res + def __iadd__(self, other: Iterable[Any]) -> "ListConfig": self.extend(other) return self @@ -562,9 +604,11 @@ class ListConfig(BaseContainer, MutableS def _set_value(self, value: Any, flags: Optional[Dict[str, bool]] = None) -> None: try: previous_content = self.__dict__["_content"] + previous_metadata = self.__dict__["_metadata"] self._set_value_impl(value, flags) except Exception as e: self.__dict__["_content"] = previous_content + self.__dict__["_metadata"] = previous_metadata raise e def _set_value_impl( @@ -576,16 +620,19 @@ class ListConfig(BaseContainer, MutableS flags = {} vk = get_value_kind(value, strict_interpolation_validation=True) - if _is_none(value, resolve=True): + if _is_none(value): if not self._is_optional(): raise ValidationError( "Non optional ListConfig cannot be constructed from None" ) self.__dict__["_content"] = None + self._metadata.object_type = None elif vk is ValueKind.MANDATORY_MISSING: self.__dict__["_content"] = MISSING + self._metadata.object_type = None elif vk == ValueKind.INTERPOLATION: self.__dict__["_content"] = value + self._metadata.object_type = None else: if not (is_primitive_list(value) or isinstance(value, ListConfig)): type_ = type(value) @@ -594,7 +641,6 @@ class ListConfig(BaseContainer, MutableS self.__dict__["_content"] = [] if isinstance(value, ListConfig): - self.__dict__["_metadata"] = copy.deepcopy(value._metadata) self._metadata.flags = copy.deepcopy(flags) # disable struct and readonly for the construction phase # retaining other flags like allow_objects. The real flags are restored at the end of this function @@ -605,6 +651,7 @@ class ListConfig(BaseContainer, MutableS with flag_override(self, ["struct", "readonly"], False): for item in value: self.append(item) + self._metadata.object_type = list @staticmethod def _list_eq(l1: Optional["ListConfig"], l2: Optional["ListConfig"]) -> bool: diff -pruN 2.1.0~rc1-3/omegaconf/nodes.py 2.2.2-1/omegaconf/nodes.py --- 2.1.0~rc1-3/omegaconf/nodes.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/omegaconf/nodes.py 2022-05-27 21:36:40.000000000 +0000 @@ -3,6 +3,7 @@ import math import sys from abc import abstractmethod from enum import Enum +from pathlib import Path from typing import Any, Dict, Optional, Type, Union from omegaconf._utils import ( @@ -11,15 +12,16 @@ from omegaconf._utils import ( get_type_of, get_value_kind, is_primitive_container, + type_str, ) -from omegaconf.base import Container, DictKeyType, Metadata, Node +from omegaconf.base import Box, DictKeyType, Metadata, Node from omegaconf.errors import ReadonlyConfigError, UnsupportedValueType, ValidationError class ValueNode(Node): _val: Any - def __init__(self, parent: Optional[Container], value: Any, metadata: Metadata): + def __init__(self, parent: Optional[Box], value: Any, metadata: Metadata): from omegaconf import read_write super().__init__(parent=parent, metadata=metadata) @@ -43,6 +45,14 @@ class ValueNode(Node): else: self._val = self.validate_and_convert(value) + def _strict_validate_type(self, value: Any) -> None: + ref_type = self._metadata.ref_type + if isinstance(ref_type, type) and type(value) is not ref_type: + type_hint = type_str(self._metadata.type_hint) + raise ValidationError( + f"Value '$VALUE' of type '$VALUE_TYPE' is incompatible with type hint '{type_hint}'" + ) + def validate_and_convert(self, value: Any) -> Any: """ Validates input and converts to canonical form @@ -52,9 +62,18 @@ class ValueNode(Node): if value is None: if self._is_optional(): return None - raise ValidationError("Non optional field cannot be assigned None") - # Subclasses can assume that `value` is not None in `_validate_and_convert_impl()`. - return self._validate_and_convert_impl(value) + ref_type_str = type_str(self._metadata.ref_type) + raise ValidationError( + f"Incompatible value '{value}' for field of type '{ref_type_str}'" + ) + + # Subclasses can assume that `value` is not None in + # `_validate_and_convert_impl()` and in `_strict_validate_type()`. + if self._get_flag("convert") is False: + self._strict_validate_type(value) + return value + else: + return self._validate_and_convert_impl(value) @abstractmethod def _validate_and_convert_impl(self, value: Any) -> Any: @@ -110,7 +129,7 @@ class AnyNode(ValueNode): self, value: Any = None, key: Any = None, - parent: Optional[Container] = None, + parent: Optional[Box] = None, flags: Optional[Dict[str, bool]] = None, ): super().__init__( @@ -122,13 +141,15 @@ class AnyNode(ValueNode): ) def _validate_and_convert_impl(self, value: Any) -> Any: - from ._utils import is_primitive_type + from ._utils import is_primitive_type_annotation # allow_objects is internal and not an official API. use at your own risk. # Please be aware that this support is subject to change without notice. # If this is deemed useful and supportable it may become an official API. - if self._get_flag("allow_objects") is not True and not is_primitive_type(value): + if self._get_flag( + "allow_objects" + ) is not True and not is_primitive_type_annotation(value): t = get_type_of(value) raise UnsupportedValueType( f"Value '{t.__name__}' is not a supported primitive type" @@ -146,7 +167,7 @@ class StringNode(ValueNode): self, value: Any = None, key: Any = None, - parent: Optional[Container] = None, + parent: Optional[Box] = None, is_optional: bool = True, flags: Optional[Dict[str, bool]] = None, ): @@ -165,7 +186,11 @@ class StringNode(ValueNode): def _validate_and_convert_impl(self, value: Any) -> str: from omegaconf import OmegaConf - if OmegaConf.is_config(value) or is_primitive_container(value): + if ( + OmegaConf.is_config(value) + or is_primitive_container(value) + or isinstance(value, bytes) + ): raise ValidationError("Cannot convert '$VALUE_TYPE' to string: '$VALUE'") return str(value) @@ -175,12 +200,53 @@ class StringNode(ValueNode): return res +class PathNode(ValueNode): + def __init__( + self, + value: Any = None, + key: Any = None, + parent: Optional[Box] = None, + is_optional: bool = True, + flags: Optional[Dict[str, bool]] = None, + ): + super().__init__( + parent=parent, + value=value, + metadata=Metadata( + key=key, + optional=is_optional, + ref_type=Path, + object_type=Path, + flags=flags, + ), + ) + + def _strict_validate_type(self, value: Any) -> None: + if not isinstance(value, Path): + raise ValidationError( + "Value '$VALUE' of type '$VALUE_TYPE' is not an instance of 'pathlib.Path'" + ) + + def _validate_and_convert_impl(self, value: Any) -> Path: + if not isinstance(value, (str, Path)): + raise ValidationError( + "Value '$VALUE' of type '$VALUE_TYPE' could not be converted to Path" + ) + + return Path(value) + + def __deepcopy__(self, memo: Dict[int, Any]) -> "PathNode": + res = PathNode() + self._deepcopy_impl(res, memo) + return res + + class IntegerNode(ValueNode): def __init__( self, value: Any = None, key: Any = None, - parent: Optional[Container] = None, + parent: Optional[Box] = None, is_optional: bool = True, flags: Optional[Dict[str, bool]] = None, ): @@ -203,7 +269,9 @@ class IntegerNode(ValueNode): else: raise ValueError() except ValueError: - raise ValidationError("Value '$VALUE' could not be converted to Integer") + raise ValidationError( + "Value '$VALUE' of type '$VALUE_TYPE' could not be converted to Integer" + ) return val def __deepcopy__(self, memo: Dict[int, Any]) -> "IntegerNode": @@ -212,12 +280,46 @@ class IntegerNode(ValueNode): return res +class BytesNode(ValueNode): + def __init__( + self, + value: Any = None, + key: Any = None, + parent: Optional[Box] = None, + is_optional: bool = True, + flags: Optional[Dict[str, bool]] = None, + ): + super().__init__( + parent=parent, + value=value, + metadata=Metadata( + key=key, + optional=is_optional, + ref_type=bytes, + object_type=bytes, + flags=flags, + ), + ) + + def _validate_and_convert_impl(self, value: Any) -> bytes: + if not isinstance(value, bytes): + raise ValidationError( + "Value '$VALUE' of type '$VALUE_TYPE' is not of type 'bytes'" + ) + return value + + def __deepcopy__(self, memo: Dict[int, Any]) -> "BytesNode": + res = BytesNode() + self._deepcopy_impl(res, memo) + return res + + class FloatNode(ValueNode): def __init__( self, value: Any = None, key: Any = None, - parent: Optional[Container] = None, + parent: Optional[Box] = None, is_optional: bool = True, flags: Optional[Dict[str, bool]] = None, ): @@ -240,7 +342,9 @@ class FloatNode(ValueNode): else: raise ValueError() except ValueError: - raise ValidationError("Value '$VALUE' could not be converted to Float") + raise ValidationError( + "Value '$VALUE' of type '$VALUE_TYPE' could not be converted to Float" + ) def __eq__(self, other: Any) -> bool: if isinstance(other, ValueNode): @@ -271,7 +375,7 @@ class BooleanNode(ValueNode): self, value: Any = None, key: Any = None, - parent: Optional[Container] = None, + parent: Optional[Box] = None, is_optional: bool = True, flags: Optional[Dict[str, bool]] = None, ): @@ -328,7 +432,7 @@ class EnumNode(ValueNode): # lgtm [py/m enum_type: Type[Enum], value: Optional[Union[Enum, str]] = None, key: Any = None, - parent: Optional[Container] = None, + parent: Optional[Box] = None, is_optional: bool = True, flags: Optional[Dict[str, bool]] = None, ): @@ -352,6 +456,14 @@ class EnumNode(ValueNode): # lgtm [py/m ), ) + def _strict_validate_type(self, value: Any) -> None: + ref_type = self._metadata.ref_type + if not isinstance(value, ref_type): + type_hint = type_str(self._metadata.type_hint) + raise ValidationError( + f"Value '$VALUE' of type '$VALUE_TYPE' is incompatible with type hint '{type_hint}'" + ) + def _validate_and_convert_impl(self, value: Any) -> Enum: return self.validate_and_convert_to_enum(enum_type=self.enum_type, value=value) @@ -401,7 +513,7 @@ class InterpolationResultNode(ValueNode) self, value: Any, key: Any = None, - parent: Optional[Container] = None, + parent: Optional[Box] = None, flags: Optional[Dict[str, bool]] = None, ): super().__init__( diff -pruN 2.1.0~rc1-3/omegaconf/omegaconf.py 2.2.2-1/omegaconf/omegaconf.py --- 2.1.0~rc1-3/omegaconf/omegaconf.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/omegaconf/omegaconf.py 2022-05-27 21:36:40.000000000 +0000 @@ -16,8 +16,10 @@ from typing import ( Callable, Dict, Generator, + Iterable, List, Optional, + Set, Tuple, Type, Union, @@ -30,7 +32,7 @@ from . import DictConfig, DictKeyType, L from ._utils import ( _DEFAULT_MARKER_, _ensure_container, - _is_none, + _get_value, format_and_raise, get_dict_key_value_types, get_list_element_type, @@ -46,11 +48,12 @@ from ._utils import ( is_primitive_list, is_structured_config, is_tuple_annotation, + is_union_annotation, nullcontext, split_key, type_str, ) -from .base import Container, Node, SCMode +from .base import Box, Container, Node, SCMode, UnionNode from .basecontainer import BaseContainer from .errors import ( MissingMandatoryValue, @@ -61,9 +64,11 @@ from .errors import ( from .nodes import ( AnyNode, BooleanNode, + BytesNode, EnumNode, FloatNode, IntegerNode, + PathNode, StringNode, ValueNode, ) @@ -92,7 +97,7 @@ def SI(interpolation: str) -> Any: def register_default_resolvers() -> None: - from omegaconf.resolvers import env, oc + from omegaconf.resolvers import oc OmegaConf.register_new_resolver("oc.create", oc.create) OmegaConf.register_new_resolver("oc.decode", oc.decode) @@ -101,7 +106,6 @@ def register_default_resolvers() -> None OmegaConf.register_new_resolver("oc.select", oc.select) OmegaConf.register_new_resolver("oc.dict.keys", oc.dict.keys) OmegaConf.register_new_resolver("oc.dict.values", oc.dict.values) - OmegaConf.legacy_register_resolver("env", env) class OmegaConf: @@ -445,28 +449,29 @@ class OmegaConf: def has_resolver(cls, name: str) -> bool: return cls._get_resolver(name) is not None - # DEPRECATED: remove in 2.2 - @classmethod - def get_resolver( - cls, - name: str, - ) -> Optional[ - Callable[[Container, Container, Node, Tuple[Any, ...], Tuple[str, ...]], Any] - ]: - warnings.warn( - "`OmegaConf.get_resolver()` is deprecated (see https://github.com/omry/omegaconf/issues/608)", - UserWarning, - stacklevel=2, - ) - - return cls._get_resolver(name) - # noinspection PyProtectedMember @staticmethod def clear_resolvers() -> None: BaseContainer._resolvers = {} register_default_resolvers() + @classmethod + def clear_resolver(cls, name: str) -> bool: + """Clear(remove) any resolver only if it exists. Returns a bool: True if resolver is removed and False if not removed. + + .. warning: + This method can remove deafult resolvers as well. + + :param name: Name of the resolver. + :return: A bool (``True`` if resolver is removed, ``False`` if not found before removing). + """ + if cls.has_resolver(name): + BaseContainer._resolvers.pop(name) + return True + else: + # return False if resolver does not exist + return False + @staticmethod def get_cache(conf: BaseContainer) -> Dict[str, Any]: return conf._metadata.resolver_cache @@ -526,13 +531,16 @@ class OmegaConf: cfg: Any, *, resolve: bool = False, + throw_on_missing: bool = False, enum_to_str: bool = False, structured_config_mode: SCMode = SCMode.DICT, - ) -> Union[Dict[DictKeyType, Any], List[Any], None, str]: + ) -> Union[Dict[DictKeyType, Any], List[Any], None, str, Any]: """ Resursively converts an OmegaConf config to a primitive container (dict or list). :param cfg: the config to convert :param resolve: True to resolve all values + :param throw_on_missing: When True, raise MissingMandatoryValue if any missing values are present. + When False (the default), replace missing values with the string "???" in the output container. :param enum_to_str: True to convert Enum keys and values to strings :param structured_config_mode: Specify how Structured Configs (DictConfigs backed by a dataclass) are handled. By default (`structured_config_mode=SCMode.DICT`) structured configs are converted to plain dicts. @@ -550,6 +558,7 @@ class OmegaConf: return BaseContainer._to_content( cfg, resolve=resolve, + throw_on_missing=throw_on_missing, enum_to_str=enum_to_str, structured_config_mode=structured_config_mode, ) @@ -561,7 +570,8 @@ class OmegaConf: Any DictConfig objects backed by dataclasses or attrs classes are instantiated as instances of those backing classes. - This is an alias for OmegaConf.to_container(..., resolve=True, structured_config_mode=SCMode.INSTANTIATE) + This is an alias for OmegaConf.to_container(..., resolve=True, throw_on_missing=True, + structured_config_mode=SCMode.INSTANTIATE) :param cfg: the config to convert :return: A dict or a list or dataclass representing this config. @@ -569,6 +579,7 @@ class OmegaConf: return OmegaConf.to_container( cfg=cfg, resolve=True, + throw_on_missing=True, enum_to_str=False, structured_config_mode=SCMode.INSTANTIATE, ) @@ -577,7 +588,7 @@ class OmegaConf: def is_missing(cfg: Any, key: DictKeyType) -> bool: assert isinstance(cfg, Container) try: - node = cfg._get_node(key) + node = cfg._get_child(key) if node is None: return False assert isinstance(node, Node) @@ -585,40 +596,11 @@ class OmegaConf: except (UnsupportedInterpolationType, KeyError, AttributeError): return False - # DEPRECATED: remove in 2.2 - @staticmethod - def is_optional(obj: Any, key: Optional[Union[int, str]] = None) -> bool: - warnings.warn( - "`OmegaConf.is_optional()` is deprecated, see https://github.com/omry/omegaconf/issues/698", - stacklevel=2, - ) - if key is not None: - assert isinstance(obj, Container) - obj = obj._get_node(key) - if isinstance(obj, Node): - return obj._is_optional() - else: - return True - - # DEPRECATED: remove in 2.2 - @staticmethod - def is_none(obj: Any, key: Optional[Union[int, DictKeyType]] = None) -> bool: - warnings.warn( - "`OmegaConf.is_none()` is deprecated, see https://github.com/omry/omegaconf/issues/547", - stacklevel=2, - ) - - if key is not None: - assert isinstance(obj, Container) - obj = obj._get_node(key) - - return _is_none(obj, resolve=True, throw_on_resolution_failure=False) - @staticmethod def is_interpolation(node: Any, key: Optional[Union[int, str]] = None) -> bool: if key is not None: assert isinstance(node, Container) - target = node._get_node(key) + target = node._get_child(key) else: target = node if target is not None: @@ -647,7 +629,7 @@ class OmegaConf: @staticmethod def get_type(obj: Any, key: Optional[str] = None) -> Optional[Type[Any]]: if key is not None: - c = obj._get_node(key) + c = obj._get_child(key) else: c = obj return OmegaConf._get_obj_type(c) @@ -733,7 +715,7 @@ class OmegaConf: with ctx: if merge and (OmegaConf.is_config(value) or is_primitive_container(value)): assert isinstance(root, BaseContainer) - node = root._get_node(last_key) + node = root._get_child(last_key) if OmegaConf.is_config(node): assert isinstance(node, BaseContainer) node.merge_with(value) @@ -785,6 +767,34 @@ class OmegaConf: ) omegaconf._impl._resolve(cfg) + @staticmethod + def missing_keys(cfg: Any) -> Set[str]: + """ + Returns a set of missing keys in a dotlist style. + :param cfg: An `OmegaConf.Container`, + or a convertible object via `OmegaConf.create` (dict, list, ...). + :return: set of strings of the missing keys. + :raises ValueError: On input not representing a config. + """ + cfg = _ensure_container(cfg) + missings: Set[str] = set() + + def gather(_cfg: Container) -> None: + itr: Iterable[Any] + if isinstance(_cfg, ListConfig): + itr = range(len(_cfg)) + else: + itr = _cfg + + for key in itr: + if OmegaConf.is_missing(_cfg, key): + missings.add(_cfg._get_full_key(key)) + elif OmegaConf.is_config(_cfg[key]): + gather(_cfg[key]) + + gather(cfg) + return missings + # === private === # @staticmethod @@ -818,29 +828,46 @@ class OmegaConf: or obj is None ): if isinstance(obj, DictConfig): - key_type = obj._metadata.key_type - element_type = obj._metadata.element_type + return DictConfig( + content=obj, + parent=parent, + ref_type=obj._metadata.ref_type, + is_optional=obj._metadata.optional, + key_type=obj._metadata.key_type, + element_type=obj._metadata.element_type, + flags=flags, + ) else: obj_type = OmegaConf.get_type(obj) key_type, element_type = get_dict_key_value_types(obj_type) - return DictConfig( - content=obj, - parent=parent, - ref_type=Any, - key_type=key_type, - element_type=element_type, - flags=flags, - ) + return DictConfig( + content=obj, + parent=parent, + key_type=key_type, + element_type=element_type, + flags=flags, + ) elif is_primitive_list(obj) or OmegaConf.is_list(obj): - obj_type = OmegaConf.get_type(obj) - element_type = get_list_element_type(obj_type) - return ListConfig( - element_type=element_type, - ref_type=Any, - content=obj, - parent=parent, - flags=flags, - ) + if isinstance(obj, ListConfig): + return ListConfig( + content=obj, + parent=parent, + element_type=obj._metadata.element_type, + ref_type=obj._metadata.ref_type, + is_optional=obj._metadata.optional, + flags=flags, + ) + else: + obj_type = OmegaConf.get_type(obj) + element_type = get_list_element_type(obj_type) + return ListConfig( + content=obj, + parent=parent, + element_type=element_type, + ref_type=Any, + is_optional=True, + flags=flags, + ) else: if isinstance(obj, type): raise ValidationError( @@ -875,6 +902,8 @@ class OmegaConf: return list elif isinstance(c, ValueNode): return type(c._value()) + elif isinstance(c, UnionNode): + return type(_get_value(c)) elif isinstance(c, dict): return dict elif isinstance(c, (list, tuple)): @@ -907,7 +936,6 @@ def flag_override( names: Union[List[str], str], values: Union[List[Optional[bool]], Optional[bool]], ) -> Generator[Node, None, None]: - if isinstance(names, str): names = [names] if values is None or isinstance(values, bool): @@ -946,33 +974,28 @@ def open_dict(config: Container) -> Gene def _node_wrap( - type_: Any, - parent: Optional[BaseContainer], + parent: Optional[Box], is_optional: bool, value: Any, key: Any, ref_type: Any = Any, ) -> Node: node: Node - is_dict = is_primitive_dict(value) or is_dict_annotation(type_) - is_list = ( - type(value) in (list, tuple) - or is_list_annotation(type_) - or is_tuple_annotation(type_) - ) - if is_dict: - key_type, element_type = get_dict_key_value_types(type_) + if is_dict_annotation(ref_type) or (is_primitive_dict(value) and ref_type is Any): + key_type, element_type = get_dict_key_value_types(ref_type) node = DictConfig( content=value, key=key, parent=parent, - ref_type=type_, + ref_type=ref_type, is_optional=is_optional, key_type=key_type, element_type=element_type, ) - elif is_list: - element_type = get_list_element_type(type_) + elif (is_list_annotation(ref_type) or is_tuple_annotation(ref_type)) or ( + type(value) in (list, tuple) and ref_type is Any + ): + element_type = get_list_element_type(ref_type) node = ListConfig( content=value, key=key, @@ -981,10 +1004,10 @@ def _node_wrap( element_type=element_type, ref_type=ref_type, ) - elif is_structured_config(type_) or is_structured_config(value): + elif is_structured_config(ref_type) or is_structured_config(value): key_type, element_type = get_dict_key_value_types(value) node = DictConfig( - ref_type=type_, + ref_type=ref_type, is_optional=is_optional, content=value, key=key, @@ -992,29 +1015,41 @@ def _node_wrap( key_type=key_type, element_type=element_type, ) - elif type_ == Any or type_ is None: + elif is_union_annotation(ref_type): + node = UnionNode( + content=value, + ref_type=ref_type, + is_optional=is_optional, + key=key, + parent=parent, + ) + elif ref_type == Any or ref_type is None: node = AnyNode(value=value, key=key, parent=parent) - elif issubclass(type_, Enum): + elif issubclass(ref_type, Enum): node = EnumNode( - enum_type=type_, + enum_type=ref_type, value=value, key=key, parent=parent, is_optional=is_optional, ) - elif type_ == int: + elif ref_type == int: node = IntegerNode(value=value, key=key, parent=parent, is_optional=is_optional) - elif type_ == float: + elif ref_type == float: node = FloatNode(value=value, key=key, parent=parent, is_optional=is_optional) - elif type_ == bool: + elif ref_type == bool: node = BooleanNode(value=value, key=key, parent=parent, is_optional=is_optional) - elif type_ == str: + elif ref_type == str: node = StringNode(value=value, key=key, parent=parent, is_optional=is_optional) + elif ref_type == bytes: + node = BytesNode(value=value, key=key, parent=parent, is_optional=is_optional) + elif ref_type == pathlib.Path: + node = PathNode(value=value, key=key, parent=parent, is_optional=is_optional) else: if parent is not None and parent._get_flag("allow_objects") is True: node = AnyNode(value=value, key=key, parent=parent) else: - raise ValidationError(f"Unexpected object type: {type_str(type_)}") + raise ValidationError(f"Unexpected object type: {type_str(ref_type)}") return node @@ -1033,12 +1068,11 @@ def _maybe_wrap( return value else: return _node_wrap( - type_=ref_type, + ref_type=ref_type, parent=parent, is_optional=is_optional, value=value, key=key, - ref_type=ref_type, ) @@ -1055,7 +1089,7 @@ def _select_one( if isinstance(c, DictConfig): assert isinstance(ret_key, str) - val = c._get_node(ret_key, validate_access=False) + val = c._get_child(ret_key, validate_access=False) elif isinstance(c, ListConfig): assert isinstance(ret_key, str) if not is_int(ret_key): @@ -1070,7 +1104,7 @@ def _select_one( if ret_key < 0 or ret_key + 1 > len(c): val = None else: - val = c._get_node(ret_key) + val = c._get_child(ret_key) else: assert False diff -pruN 2.1.0~rc1-3/omegaconf/resolvers/__init__.py 2.2.2-1/omegaconf/resolvers/__init__.py --- 2.1.0~rc1-3/omegaconf/resolvers/__init__.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/omegaconf/resolvers/__init__.py 2022-05-27 21:36:40.000000000 +0000 @@ -1,28 +1,5 @@ -import os -import warnings -from typing import Any, Optional - -from omegaconf._utils import decode_primitive -from omegaconf.errors import ValidationError from omegaconf.resolvers import oc - -# DEPRECATED: remove in 2.2 -def env(key: str, default: Optional[str] = None) -> Any: - warnings.warn( - "The `env` resolver is deprecated, see https://github.com/omry/omegaconf/issues/573" - ) - - try: - return decode_primitive(os.environ[key]) - except KeyError: - if default is not None: - return decode_primitive(default) - else: - raise ValidationError(f"Environment variable '{key}' not found") - - __all__ = [ - "env", "oc", ] diff -pruN 2.1.0~rc1-3/omegaconf/resolvers/oc/dict.py 2.2.2-1/omegaconf/resolvers/oc/dict.py --- 2.1.0~rc1-3/omegaconf/resolvers/oc/dict.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/omegaconf/resolvers/oc/dict.py 2022-05-27 21:36:40.000000000 +0000 @@ -35,8 +35,10 @@ def values(key: str, _root_: BaseContain assert isinstance(content, dict) ret = ListConfig([]) + if key.startswith("."): + key = f".{key}" # extra dot to compensate for extra level of nesting within ret ListConfig for k in content: - ref_node = AnyNode(f"${{{key}.{k}}}") + ref_node = AnyNode(f"${{{key}.{k!s}}}") ret.append(ref_node) # Finalize result by setting proper type and parent. diff -pruN 2.1.0~rc1-3/omegaconf/_utils.py 2.2.2-1/omegaconf/_utils.py --- 2.1.0~rc1-3/omegaconf/_utils.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/omegaconf/_utils.py 2022-05-27 21:36:40.000000000 +0000 @@ -1,23 +1,15 @@ import copy import os +import pathlib import re import string import sys +import types import warnings from contextlib import contextmanager from enum import Enum from textwrap import dedent -from typing import ( - Any, - Dict, - Iterator, - List, - Optional, - Tuple, - Type, - Union, - get_type_hints, -) +from typing import Any, Dict, Iterator, List, Optional, Tuple, Type, Union import yaml @@ -43,6 +35,16 @@ try: except ImportError: # pragma: no cover attr = None # type: ignore # pragma: no cover +NoneType: Type[None] = type(None) + +BUILTIN_VALUE_TYPES: Tuple[Type[Any], ...] = ( + int, + float, + bool, + str, + bytes, + NoneType, +) # Regexprs to match key paths like: a.b, a[b], ..a[c].d, etc. # We begin by matching the head (in these examples: a, a, ..a). @@ -161,6 +163,20 @@ def get_yaml_loader() -> Any: ] for key, resolvers in loader.yaml_implicit_resolvers.items() } + + loader.add_constructor( + "tag:yaml.org,2002:python/object/apply:pathlib.Path", + lambda loader, node: pathlib.Path(*loader.construct_sequence(node)), + ) + loader.add_constructor( + "tag:yaml.org,2002:python/object/apply:pathlib.PosixPath", + lambda loader, node: pathlib.PosixPath(*loader.construct_sequence(node)), + ) + loader.add_constructor( + "tag:yaml.org,2002:python/object/apply:pathlib.WindowsPath", + lambda loader, node: pathlib.WindowsPath(*loader.construct_sequence(node)), + ) + return loader @@ -176,19 +192,35 @@ def _get_class(path: str) -> type: return klass -def _is_union(type_: Any) -> bool: +def is_union_annotation(type_: Any) -> bool: + if sys.version_info >= (3, 10): # pragma: no cover + if isinstance(type_, types.UnionType): + return True return getattr(type_, "__origin__", None) is Union def _resolve_optional(type_: Any) -> Tuple[bool, Any]: """Check whether `type_` is equivalent to `typing.Optional[T]` for some T.""" - if getattr(type_, "__origin__", None) is Union: + if is_union_annotation(type_): args = type_.__args__ - if len(args) == 2 and args[1] == type(None): # noqa E721 - return True, args[0] + if NoneType in args: + optional = True + args = tuple(a for a in args if a is not NoneType) + else: + optional = False + if len(args) == 1: + return optional, args[0] + elif len(args) >= 2: + return optional, Union[args] + else: + assert False + if type_ is Any: return True, Any + if type_ in (None, NoneType): + return True, NoneType + return False, type_ @@ -199,13 +231,8 @@ def _is_optional(obj: Any, key: Optional if key is not None: assert isinstance(obj, Container) obj = obj._get_node(key) - if isinstance(obj, Node): - return obj._is_optional() - else: - # In case `obj` is not a Node, treat it as optional by default. - # This is used in `ListConfig.append` and `ListConfig.insert` - # where the appended/inserted value might or might not be a Node. - return True + assert isinstance(obj, Node) + return obj._is_optional() def _resolve_forward(type_: Type[Any], module: str) -> Type[Any]: @@ -273,10 +300,14 @@ def extract_dict_subclass_data(obj: Any, return None -def get_attr_class_field_names(obj: Any) -> List[str]: +def get_attr_class_init_field_names(obj: Any) -> List[str]: is_type = isinstance(obj, type) obj_type = obj if is_type else type(obj) - return list(attr.fields_dict(obj_type)) + return [ + fieldname + for fieldname, attribute in attr.fields_dict(obj_type).items() + if attribute.init + ] def get_attr_data(obj: Any, allow_objects: Optional[bool] = None) -> Dict[str, Any]: @@ -301,9 +332,9 @@ def get_attr_data(obj: Any, allow_object value = attrib.default if value == attr.NOTHING: value = MISSING - if _is_union(type_): + if is_union_annotation(type_) and not is_supported_union_annotation(type_): e = ConfigValueError( - f"Union types are not supported:\n{name}: {type_str(type_)}" + f"Unions of containers are not supported:\n{name}: {type_str(type_)}" ) format_and_raise(node=None, key=None, value=value, cause=e, msg=str(e)) @@ -326,17 +357,20 @@ def get_attr_data(obj: Any, allow_object return d -def get_dataclass_field_names(obj: Any) -> List[str]: - return [field.name for field in dataclasses.fields(obj)] +def get_dataclass_init_field_names(obj: Any) -> List[str]: + return [field.name for field in dataclasses.fields(obj) if field.init] def get_dataclass_data( obj: Any, allow_objects: Optional[bool] = None ) -> Dict[str, Any]: + from typing import get_type_hints + from omegaconf.omegaconf import MISSING, OmegaConf, _maybe_wrap flags = {"allow_objects": allow_objects} if allow_objects is not None else {} d = {} + is_type = isinstance(obj, type) obj_type = get_type_of(obj) dummy_parent = OmegaConf.create({}, flags=flags) dummy_parent._metadata.object_type = obj_type @@ -345,20 +379,22 @@ def get_dataclass_data( name = field.name is_optional, type_ = _resolve_optional(resolved_hints[field.name]) type_ = _resolve_forward(type_, obj.__module__) + has_default = field.default != dataclasses.MISSING + has_default_factory = field.default_factory != dataclasses.MISSING # type: ignore - if hasattr(obj, name): + if not is_type: value = getattr(obj, name) - if value == dataclasses.MISSING: - value = MISSING else: - if field.default_factory == dataclasses.MISSING: # type: ignore - value = MISSING - else: + if has_default: + value = field.default + elif has_default_factory: value = field.default_factory() # type: ignore + else: + value = MISSING - if _is_union(type_): + if is_union_annotation(type_) and not is_supported_union_annotation(type_): e = ConfigValueError( - f"Union types are not supported:\n{name}: {type_str(type_)}" + f"Unions of containers are not supported:\n{name}: {type_str(type_)}" ) format_and_raise(node=None, key=None, value=value, cause=e, msg=str(e)) try: @@ -429,11 +465,11 @@ def is_structured_config_frozen(obj: Any return False -def get_structured_config_field_names(obj: Any) -> List[str]: +def get_structured_config_init_field_names(obj: Any) -> List[str]: if is_dataclass(obj): - return get_dataclass_field_names(obj) + return get_dataclass_init_field_names(obj) elif is_attr_class(obj): - return get_attr_class_field_names(obj) + return get_attr_class_init_field_names(obj) else: raise ValueError(f"Unsupported type: {type(obj).__name__}") @@ -508,29 +544,48 @@ def get_value_kind( if _is_missing_value(value): return ValueKind.MANDATORY_MISSING - value = _get_value(value) + if _is_interpolation(value, strict_interpolation_validation): + return ValueKind.INTERPOLATION + + return ValueKind.VALUE + + +def _is_interpolation(v: Any, strict_interpolation_validation: bool = False) -> bool: + from omegaconf import Node + + if isinstance(v, Node): + v = v._value() + + if isinstance(v, str) and _is_interpolation_string( + v, strict_interpolation_validation + ): + return True + return False + +def _is_interpolation_string(value: str, strict_interpolation_validation: bool) -> bool: # We identify potential interpolations by the presence of "${" in the string. # Note that escaped interpolations (ex: "esc: \${bar}") are identified as # interpolations: this is intended, since they must be processed as interpolations # for the string to be properly un-escaped. # Keep in mind that invalid interpolations will only be detected when # `strict_interpolation_validation` is True. - if isinstance(value, str) and "${" in value: + if "${" in value: if strict_interpolation_validation: # First try the cheap regex matching that detects common interpolations. if SIMPLE_INTERPOLATION_PATTERN.match(value) is None: # If no match, do the more expensive grammar parsing to detect errors. parse(value) - return ValueKind.INTERPOLATION - else: - return ValueKind.VALUE + return True + return False -# DEPRECATED: remove in 2.2 -def is_bool(st: str) -> bool: - st = str.lower(st) - return st == "true" or st == "false" +def _is_special(value: Any) -> bool: + """Special values are None, MISSING, and interpolation.""" + return _is_none(value) or get_value_kind(value) in ( + ValueKind.MANDATORY_MISSING, + ValueKind.INTERPOLATION, + ) def is_float(st: str) -> bool: @@ -549,20 +604,6 @@ def is_int(st: str) -> bool: return False -# DEPRECATED: remove in 2.2 -def decode_primitive(s: str) -> Any: - if is_bool(s): - return str.lower(s) == "true" - - if is_int(s): - return int(s) - - if is_float(s): - return float(s) - - return s - - def is_primitive_list(obj: Any) -> bool: from .base import Container @@ -576,11 +617,12 @@ def is_primitive_dict(obj: Any) -> bool: def is_dict_annotation(type_: Any) -> bool: origin = getattr(type_, "__origin__", None) - if sys.version_info < (3, 7, 0): - return origin is Dict or type_ is Dict # pragma: no cover + # type_dict is a bit hard to detect. + # this support is tentative, if it eventually causes issues in other areas it may be dropped. + if sys.version_info < (3, 7, 0): # pragma: no cover + typed_dict = hasattr(type_, "__base__") and type_.__base__ == Dict + return origin is Dict or type_ is Dict or typed_dict else: # pragma: no cover - # type_dict is a bit hard to detect. - # this support is tentative, if it eventually causes issues in other areas it may be dropped. typed_dict = hasattr(type_, "__base__") and type_.__base__ == dict return origin is dict or typed_dict @@ -601,6 +643,14 @@ def is_tuple_annotation(type_: Any) -> b return origin is tuple # pragma: no cover +def is_supported_union_annotation(obj: Any) -> bool: + """Currently only primitive types are supported in Unions, e.g. Union[int, str]""" + if not is_union_annotation(obj): + return False + args = obj.__args__ + return all(is_primitive_type_annotation(arg) for arg in args) + + def is_dict_subclass(type_: Any) -> bool: return type_ is not None and isinstance(type_, type) and issubclass(type_, Dict) @@ -645,8 +695,15 @@ def get_dict_key_value_types(ref_type: A return key_type, element_type -def valid_value_annotation_type(type_: Any) -> bool: - return type_ is Any or is_primitive_type(type_) or is_structured_config(type_) +def is_valid_value_annotation(type_: Any) -> bool: + _, type_ = _resolve_optional(type_) + return ( + type_ is Any + or is_primitive_type_annotation(type_) + or is_structured_config(type_) + or is_container_annotation(type_) + or is_supported_union_annotation(type_) + ) def _valid_dict_key_annotation_type(type_: Any) -> bool: @@ -655,24 +712,13 @@ def _valid_dict_key_annotation_type(type return type_ is None or type_ is Any or issubclass(type_, DictKeyType.__args__) # type: ignore -def is_primitive_type(type_: Any) -> bool: +def is_primitive_type_annotation(type_: Any) -> bool: type_ = get_type_of(type_) - return issubclass(type_, Enum) or type_ in (int, float, bool, str, type(None)) - - -def _is_interpolation(v: Any, strict_interpolation_validation: bool = False) -> bool: - if isinstance(v, str): - ret = ( - get_value_kind(v, strict_interpolation_validation) - == ValueKind.INTERPOLATION - ) - assert isinstance(ret, bool) - return ret - return False + return issubclass(type_, (Enum, pathlib.Path)) or type_ in BUILTIN_VALUE_TYPES def _get_value(value: Any) -> Any: - from .base import Container + from .base import Container, UnionNode from .nodes import ValueNode if isinstance(value, ValueNode): @@ -681,12 +727,18 @@ def _get_value(value: Any) -> Any: boxed = value._value() if boxed is None or _is_missing_literal(boxed) or _is_interpolation(boxed): return boxed + elif isinstance(value, UnionNode): + boxed = value._value() + if boxed is None or _is_missing_literal(boxed) or _is_interpolation(boxed): + return boxed + else: + return _get_value(boxed) # pass through value of boxed node # return primitives and regular OmegaConf Containers as is return value -def get_ref_type(obj: Any, key: Any = None) -> Optional[Type[Any]]: +def get_type_hint(obj: Any, key: Any = None) -> Optional[Type[Any]]: from omegaconf import Container, Node if isinstance(obj, Container): @@ -716,7 +768,7 @@ def _raise(ex: Exception, cause: Excepti ex.__cause__ = cause else: ex.__cause__ = None - raise ex.with_traceback(sys.exc_info()[2]) # set end OC_CAUSE=1 for full backtrace + raise ex.with_traceback(sys.exc_info()[2]) # set env var OC_CAUSE=1 for full trace def format_and_raise( @@ -765,7 +817,7 @@ def format_and_raise( object_type = OmegaConf.get_type(node) object_type_str = type_str(object_type) - ref_type = get_ref_type(node) + ref_type = get_type_hint(node) ref_type_str = type_str(ref_type) msg = string.Template(msg).safe_substitute( @@ -774,7 +826,7 @@ def format_and_raise( KEY=key, FULL_KEY=full_key, VALUE=value, - VALUE_TYPE=f"{type(value).__name__}", + VALUE_TYPE=type_str(type(value), include_module_name=True), KEY_TYPE=f"{type(key).__name__}", ) @@ -821,10 +873,10 @@ def format_and_raise( _raise(ex, cause) -def type_str(t: Any) -> str: +def type_str(t: Any, include_module_name: bool = False) -> str: is_optional, t = _resolve_optional(t) - if t is None: - return type(t).__name__ + if t is NoneType: + return str(t.__name__) if t is Any: return "Any" if t is ...: @@ -848,16 +900,31 @@ def type_str(t: Any) -> str: else: if t._name is None: if t.__origin__ is not None: - name = type_str(t.__origin__) + name = type_str( + t.__origin__, include_module_name=include_module_name + ) else: name = str(t._name) args = getattr(t, "__args__", None) if args is not None: - args = ", ".join([type_str(t) for t in t.__args__]) + args = ", ".join( + [type_str(t, include_module_name=include_module_name) for t in t.__args__] + ) ret = f"{name}[{args}]" else: ret = name + if include_module_name: + if ( + hasattr(t, "__module__") + and t.__module__ != "builtins" + and t.__module__ != "typing" + and not t.__module__.startswith("omegaconf.") + ): + module_prefix = str(t.__module__) + "." + else: + module_prefix = "" + ret = module_prefix + ret if is_optional: return f"Optional[{ret}]" else: @@ -917,7 +984,7 @@ def split_key(key: str) -> List[str]: This is similar to `key.split(".")` but also works with the getitem syntax: "a.b" -> ["a", "b"] - "a[b]" -> ["a, "b"] + "a[b]" -> ["a", "b"] ".a.b[c].d" -> ["", "a", "b", "c", "d"] "[a].b" -> ["a", "b"] """ diff -pruN 2.1.0~rc1-3/omegaconf/version.py 2.2.2-1/omegaconf/version.py --- 2.1.0~rc1-3/omegaconf/version.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/omegaconf/version.py 2022-05-27 21:36:40.000000000 +0000 @@ -1,6 +1,6 @@ import sys # pragma: no cover -__version__ = "2.1.0.rc1" +__version__ = "2.2.2" msg = """OmegaConf 2.0 and above is compatible with Python 3.6 and newer. You have the following options: diff -pruN 2.1.0~rc1-3/.pre-commit-config.yaml 2.2.2-1/.pre-commit-config.yaml --- 2.1.0~rc1-3/.pre-commit-config.yaml 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/.pre-commit-config.yaml 2022-05-27 21:36:40.000000000 +0000 @@ -17,7 +17,7 @@ repos: additional_dependencies: [-e, 'git+git://github.com/pycqa/pyflakes.git@1911c20#egg=pyflakes'] - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.790 + rev: v0.931 hooks: - id: mypy args: [--strict] diff -pruN 2.1.0~rc1-3/pydevd_plugins/extensions/__init__.py 2.2.2-1/pydevd_plugins/extensions/__init__.py --- 2.1.0~rc1-3/pydevd_plugins/extensions/__init__.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/pydevd_plugins/extensions/__init__.py 2022-05-27 21:36:40.000000000 +0000 @@ -3,4 +3,4 @@ try: except ImportError: # pragma: no cover import pkgutil - __path__ = pkgutil.extend_path(__path__, __name__) # type: ignore + __path__ = pkgutil.extend_path(__path__, __name__) diff -pruN 2.1.0~rc1-3/pydevd_plugins/__init__.py 2.2.2-1/pydevd_plugins/__init__.py --- 2.1.0~rc1-3/pydevd_plugins/__init__.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/pydevd_plugins/__init__.py 2022-05-27 21:36:40.000000000 +0000 @@ -3,4 +3,4 @@ try: except ImportError: # pragma: no cover import pkgutil - __path__ = pkgutil.extend_path(__path__, __name__) # type: ignore + __path__ = pkgutil.extend_path(__path__, __name__) diff -pruN 2.1.0~rc1-3/pyproject.toml 2.2.2-1/pyproject.toml --- 2.1.0~rc1-3/pyproject.toml 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/pyproject.toml 2022-05-27 21:36:40.000000000 +0000 @@ -5,8 +5,9 @@ exclude = ''' \.eggs # exclude a few common directories in the | \.git # root of the project | \.mypy_cache - | \omegaconf/grammar/gen + | omegaconf/grammar/gen | \.nox + | build ) ) ''' diff -pruN 2.1.0~rc1-3/README.md 2.2.2-1/README.md --- 2.1.0~rc1-3/README.md 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/README.md 2022-05-27 21:36:40.000000000 +0000 @@ -3,33 +3,38 @@ | --- | --- | | Project | [![PyPI version](https://badge.fury.io/py/omegaconf.svg)](https://badge.fury.io/py/omegaconf)[![Downloads](https://pepy.tech/badge/omegaconf/month)](https://pepy.tech/project/omegaconf?versions=1.4.*&versions=2.0.*&versions=2.1.*)![PyPI - Python Version](https://img.shields.io/pypi/pyversions/omegaconf.svg) | | Code quality| [![CircleCI](https://img.shields.io/circleci/build/github/omry/omegaconf?logo=s&token=5de2f8dc2a0dd78438520575431aa533150806e3)](https://circleci.com/gh/omry/omegaconf)[![Coverage Status](https://coveralls.io/repos/github/omry/omegaconf/badge.svg)](https://coveralls.io/github/omry/omegaconf)[![Total alerts](https://img.shields.io/lgtm/alerts/g/omry/omegaconf.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/omry/omegaconf/alerts/)[![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/omry/omegaconf.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/omry/omegaconf/context:python)| -| Docs and support |[![Documentation Status](https://readthedocs.org/projects/omegaconf/badge/?version=2.0_branch)](https://omegaconf.readthedocs.io/en/2.0_branch/)[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/omry/omegaconf/master?filepath=docs%2Fnotebook%2FTutorial.ipynb)[![](https://img.shields.io/badge/zulip-join_chat-brightgreen.svg)](https://hydra-framework.zulipchat.com)| +| Docs and support |[![Documentation Status](https://readthedocs.org/projects/omegaconf/badge/?version=2.0_branch)](https://omegaconf.readthedocs.io/en/2.1_branch/)[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/omry/omegaconf/master?filepath=docs%2Fnotebook%2FTutorial.ipynb)[![](https://img.shields.io/badge/zulip-join_chat-brightgreen.svg)](https://hydra-framework.zulipchat.com)| OmegaConf is a hierarchical configuration system, with support for merging configurations from multiple sources (YAML config files, dataclasses/objects and CLI arguments) providing a consistent API regardless of how the configuration was created. ## Releases -### Stable (2.0) -OmegaConf 2.0 stable version. -* [What's new](https://github.com/omry/omegaconf/releases/tag/v2.0.0) -* [Documentation](https://omegaconf.readthedocs.io/en/2.0_branch/) -* [Slides](https://docs.google.com/presentation/d/e/2PACX-1vT_UIV7hCnquIbLUm4NnkUpXvPEh33IKiUEvPRF850WKA8opOlZOszjKdZ3tPmf8u7hGNP6HpqS-NT5/pub?start=false&loop=false&delayms=3000) -* [Source code](https://github.com/omry/omegaconf/tree/2.0_branch) +### Stable (2.2) +OmegaConf 2.2 is the current stable version. +* [What's new](https://github.com/omry/omegaconf/releases/tag/v2.2.1) +* [Source code](https://github.com/omry/omegaconf/tree/2.2_branch) Install with `pip install --upgrade omegaconf` -### Dev release (2.1) -OmegaConf 2.1 dev release is in good shape and is approaching a release candidate. -Users are encouraged to try it out. - -* [Documentation](https://omegaconf.readthedocs.io/en/latest/) -* [Source code](https://github.com/omry/omegaconf/tree/master) -* [Milestone progress](https://github.com/omry/omegaconf/milestone/3) +### Previous release (2.1) +OmegaConf 2.1 is the current stable version. +* [What's new](https://github.com/omry/omegaconf/releases/tag/v2.1.1) +* [Documentation](https://omegaconf.readthedocs.io/en/2.1_branch/) +* [Slides](https://docs.google.com/presentation/d/e/2PACX-1vT_UIV7hCnquIbLUm4NnkUpXvPEh33IKiUEvPRF850WKA8opOlZOszjKdZ3tPmf8u7hGNP6HpqS-NT5/pub?start=false&loop=false&delayms=3000) +* [Source code](https://github.com/omry/omegaconf/tree/2.1_branch) + +Install with `pip install omegaconf==2.1` -Install with `pip install --upgrade omegaconf --pre` +### Previous release (2.0) + +* [What's new](https://github.com/omry/omegaconf/releases/tag/v2.0.0) +* [Documentation](https://omegaconf.readthedocs.io/en/2.0_branch/) +* [Slides](https://docs.google.com/presentation/d/e/2PACX-1vT_UIV7hCnquIbLUm4NnkUpXvPEh33IKiUEvPRF850WKA8opOlZOszjKdZ3tPmf8u7hGNP6HpqS-NT5/pub?start=false&loop=false&delayms=3000) +* [Source code](https://github.com/omry/omegaconf/tree/2.0_branch) +Install with `pip install omegaconf==2.0.6` ## Live tutorial Run the live tutorial: [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/omry/omegaconf/master?filepath=docs%2Fnotebook%2FTutorial.ipynb) diff -pruN 2.1.0~rc1-3/requirements/base.txt 2.2.2-1/requirements/base.txt --- 2.1.0~rc1-3/requirements/base.txt 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/requirements/base.txt 2022-05-27 21:36:40.000000000 +0000 @@ -1,4 +1,4 @@ -antlr4-python3-runtime==4.8 -PyYAML>=5.1.* +antlr4-python3-runtime==4.9.* +PyYAML>=5.1.0 # Use dataclasses backport for Python 3.6. dataclasses;python_version=='3.6' diff -pruN 2.1.0~rc1-3/requirements/dev.txt 2.2.2-1/requirements/dev.txt --- 2.1.0~rc1-3/requirements/dev.txt 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/requirements/dev.txt 2022-05-27 21:36:40.000000000 +0000 @@ -4,7 +4,7 @@ build coveralls flake8 isort~=5.0 -mypy==0.790 +mypy nox pre-commit pyflakes diff -pruN 2.1.0~rc1-3/setup.cfg 2.2.2-1/setup.cfg --- 2.1.0~rc1-3/setup.cfg 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/setup.cfg 2022-05-27 21:36:40.000000000 +0000 @@ -4,6 +4,7 @@ test=pytest [mypy] python_version = 3.6 mypy_path=.stubs +exclude = build/ [mypy-antlr4.*] ignore_missing_imports = True diff -pruN 2.1.0~rc1-3/setup.py 2.2.2-1/setup.py --- 2.1.0~rc1-3/setup.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/setup.py 2022-05-27 21:36:40.000000000 +0000 @@ -4,7 +4,7 @@ OmegaConf setup Instructions: # Build: rm -rf dist/ omegaconf.egg-info/ - python setup.py sdist bdist_wheel + python -m build # Upload: twine upload dist/* """ @@ -46,8 +46,6 @@ with open("README.md", "r") as fh: description="A flexible configuration library", long_description=LONG_DESC, long_description_content_type="text/markdown", - setup_requires=["pytest-runner"], - tests_require=["pytest"], url="https://github.com/omry/omegaconf", keywords="yaml configuration config", packages=[ @@ -65,6 +63,7 @@ with open("README.md", "r") as fh: "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", "License :: OSI Approved :: BSD License", "Operating System :: OS Independent", ], Binary files 2.1.0~rc1-3/tests/data/2.0.6.pickle and 2.2.2-1/tests/data/2.0.6.pickle differ Binary files 2.1.0~rc1-3/tests/data/2.1.0.rc1.pickle and 2.2.2-1/tests/data/2.1.0.rc1.pickle differ diff -pruN 2.1.0~rc1-3/tests/data/load.py 2.2.2-1/tests/data/load.py --- 2.1.0~rc1-3/tests/data/load.py 1970-01-01 00:00:00.000000000 +0000 +++ 2.2.2-1/tests/data/load.py 2022-05-27 21:36:40.000000000 +0000 @@ -0,0 +1,8 @@ +import pickle +import sys + +from omegaconf import OmegaConf + +with open(f"{sys.argv[1]}.pickle", mode="rb") as fp: + cfg = pickle.load(fp) + assert cfg == OmegaConf.create({"a": [{"b": 10}]}) diff -pruN 2.1.0~rc1-3/tests/data/save.py 2.2.2-1/tests/data/save.py --- 2.1.0~rc1-3/tests/data/save.py 1970-01-01 00:00:00.000000000 +0000 +++ 2.2.2-1/tests/data/save.py 2022-05-27 21:36:40.000000000 +0000 @@ -0,0 +1,8 @@ +import pickle + +from omegaconf import OmegaConf, __version__ + +cfg = OmegaConf.create({"a": [{"b": 10}]}) + +with open(f"{__version__}.pickle", mode="wb") as fp: + pickle.dump(cfg, fp) diff -pruN 2.1.0~rc1-3/tests/examples/dataclass_postponed_annotations.py 2.2.2-1/tests/examples/dataclass_postponed_annotations.py --- 2.1.0~rc1-3/tests/examples/dataclass_postponed_annotations.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/tests/examples/dataclass_postponed_annotations.py 2022-05-27 21:36:40.000000000 +0000 @@ -4,6 +4,7 @@ from __future__ import annotations # ty from dataclasses import dataclass, fields from enum import Enum +from pathlib import Path from pytest import raises @@ -22,6 +23,8 @@ class SimpleTypes: is_awesome: bool = True height: "Height" = Height.SHORT # test forward ref description: str = "text" + data: bytes = b"bin_data" + path: Path = Path("hello.txt") def simple_types_class() -> None: @@ -34,6 +37,8 @@ def simple_types_class() -> None: conf = OmegaConf.structured(SimpleTypes) assert conf.num == 10 assert conf.pi == 3.1415 + assert conf.data == b"bin_data" + assert conf.path == Path("hello.txt") assert conf.is_awesome is True assert conf.height == Height.SHORT assert conf.description == "text" diff -pruN 2.1.0~rc1-3/tests/examples/test_dataclass_example.py 2.2.2-1/tests/examples/test_dataclass_example.py --- 2.1.0~rc1-3/tests/examples/test_dataclass_example.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/tests/examples/test_dataclass_example.py 2022-05-27 21:36:40.000000000 +0000 @@ -26,6 +26,7 @@ class SimpleTypes: is_awesome: bool = True height: Height = Height.SHORT description: str = "text" + data: bytes = b"bin_data" def test_simple_types_class() -> None: @@ -36,6 +37,7 @@ def test_simple_types_class() -> None: assert conf.is_awesome is True assert conf.height == Height.SHORT assert conf.description == "text" + assert conf.data == b"bin_data" def test_static_typing() -> None: @@ -57,6 +59,7 @@ def test_simple_types_obj() -> None: assert conf.is_awesome is True assert conf.height == Height.SHORT assert conf.description == "text" + assert conf.data == b"bin_data" def test_conversions() -> None: @@ -73,6 +76,24 @@ def test_conversions() -> None: # ValidationError: "one" cannot be converted to an integer conf.num = "one" # type: ignore + conf.description = "abc" # ok, type matches + assert conf.description == "abc" + # ok, the int 20 is converted to the string "20" + conf.description = 20 # type: ignore + assert conf.description == "20" + with raises(ValidationError): + # bytes are not automatically converted to strings + conf.description = b"binary" # type: ignore + + assert conf.data == b"bin_data" + conf.data = b"def" # assignment ok, type matches + with raises(ValidationError): + # ValidationError: "text" cannot be converted to bytes + conf.data = "text" # type: ignore + with raises(ValidationError): + # ValidationError: 1234 cannot be converted to bytes + conf.data = 1234 # type: ignore + # booleans can take many forms for expected, values in { True: ["on", "yes", "true", True, "1"], @@ -175,10 +196,10 @@ manager: @dataclass class Lists: # List with Any as type can take any primitive type OmegaConf supports: - # int, float, bool, str and Enums as well as Any (which is the same as not having a specified type). + # int, float, bool, str, bytes and Enums as well as Any (which is the same as not having a specified type). untyped_list: List[Any] = field(default_factory=lambda: [1, "foo", True]) - # typed lists can hold int, bool, str, float or enums. + # typed lists can hold int, bool, str, bytes, float or enums. int_list: List[int] = field(default_factory=lambda: [10, 20, 30]) @@ -201,7 +222,7 @@ def test_typed_list_runtime_validation() @dataclass class Dicts: # Key must be a string or Enum, value can be any primitive type OmegaConf supports: - # int, float, bool, str and Enums as well as Any (which is the same as not having a specified type). + # int, float, bool, str, bytes and Enums as well as Any (which is the same as not having a specified type). untyped_dict: Dict[str, Any] = field( default_factory=lambda: {"foo": True, "bar": 100} ) diff -pruN 2.1.0~rc1-3/tests/__init__.py 2.2.2-1/tests/__init__.py --- 2.1.0~rc1-3/tests/__init__.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/tests/__init__.py 2022-05-27 21:36:40.000000000 +0000 @@ -1,7 +1,7 @@ import re from dataclasses import dataclass, field from enum import Enum -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, List, NamedTuple, Optional, Tuple, Union import attr from pytest import warns @@ -78,6 +78,11 @@ class Users: @dataclass +class OptionalUsers: + name2user: Dict[str, Optional[User]] = field(default_factory=dict) + + +@dataclass class ConfWithMissingDict: dict: Dict[str, Any] = MISSING @@ -100,6 +105,27 @@ class ConcretePlugin(Plugin): @dataclass +class NestedInterpolationToMissing: + @dataclass + class BazParams: + baz: str = "${..name}" + + subcfg: BazParams = BazParams() + name: str = MISSING + + +@dataclass +class StructuredInterpolationKeyError: + name: str = "${bar}" + + +@dataclass +class StructuredInterpolationValidationError: + x: Optional[int] = None + y: int = II(".x") + + +@dataclass class StructuredWithMissing: num: int = MISSING opt_num: Optional[int] = MISSING @@ -116,7 +142,7 @@ class StructuredWithMissing: @dataclass class UnionError: - x: Union[int, str] = 10 + x: Union[int, List[str]] = 10 @dataclass @@ -195,6 +221,17 @@ class SubscriptedList: @dataclass +class SubscriptedListOpt: + opt_list: Optional[List[int]] = field(default_factory=lambda: [1, 2]) + list_opt: List[Optional[int]] = field(default_factory=lambda: [1, 2, None]) + + +@dataclass +class ListOfAny: + list: List[Any] + + +@dataclass class UntypedDict: dict: Dict = field(default_factory=lambda: {"foo": "var"}) # type: ignore opt_dict: Optional[Dict] = None # type: ignore @@ -207,6 +244,20 @@ class SubscriptedDict: dict_int: Dict[int, int] = field(default_factory=lambda: {123: 4}) dict_float: Dict[float, int] = field(default_factory=lambda: {123.45: 4}) dict_bool: Dict[bool, int] = field(default_factory=lambda: {True: 4, False: 5}) + dict_bytes: Dict[bytes, int] = field(default_factory=lambda: {b"binary": 4}) + + +@dataclass +class SubscriptedDictOpt: + opt_dict: Optional[Dict[str, int]] = field(default_factory=lambda: {"foo": 4}) + dict_opt: Dict[str, Optional[int]] = field( + default_factory=lambda: {"foo": 4, "bar": None} + ) + + +@dataclass +class DictOfAny: + dict: Dict[Any, Any] @dataclass @@ -224,6 +275,41 @@ class Str2Int(Dict[str, int]): pass +class DictSubclass(Dict[Any, Any]): + pass + + +class ListSubclass(List[Any]): + pass + + +class Shape(NamedTuple): + channels: int + height: int + width: int + + +@dataclass +class OptTuple: + x: Optional[Tuple[int, ...]] = None + + +@dataclass +class NestedContainers: + dict_of_dict: Dict[str, Dict[str, int]] = field( + default_factory=lambda: {"foo": {"bar": 123}} + ) + list_of_list: List[List[int]] = field(default_factory=lambda: [[123]]) + dict_of_list: Dict[str, List[int]] = field(default_factory=lambda: {"foo": [123]}) + list_of_dict: List[Dict[str, int]] = field(default_factory=lambda: [{"bar": 123}]) + + +@dataclass +class UnionAnnotations: + ubf: Union[bool, float] = True + oubf: Optional[Union[bool, float]] = None + + def warns_dict_subclass_deprecated(dict_subclass: Any) -> Any: return warns( UserWarning, diff -pruN 2.1.0~rc1-3/tests/interpolation/built_in_resolvers/test_legacy_env.py 2.2.2-1/tests/interpolation/built_in_resolvers/test_legacy_env.py --- 2.1.0~rc1-3/tests/interpolation/built_in_resolvers/test_legacy_env.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/tests/interpolation/built_in_resolvers/test_legacy_env.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,55 +0,0 @@ -import re -from typing import Any - -from pytest import mark, warns - -from omegaconf import OmegaConf - - -def test_legacy_env_is_cached(monkeypatch: Any) -> None: - monkeypatch.setenv("FOOBAR", "1234") - c = OmegaConf.create({"foobar": "${env:FOOBAR}"}) - with warns(UserWarning): - before = c.foobar - monkeypatch.setenv("FOOBAR", "3456") - assert c.foobar == before - - -@mark.parametrize( - "value,expected", - [ - # bool - ("false", False), - ("true", True), - # int - ("10", 10), - ("-10", -10), - # float - ("10.0", 10.0), - ("-10.0", -10.0), - # strings - ("off", "off"), - ("no", "no"), - ("on", "on"), - ("yes", "yes"), - (">1234", ">1234"), - (":1234", ":1234"), - ("/1234", "/1234"), - # yaml strings are not getting parsed by the env resolver - ("foo: bar", "foo: bar"), - ("foo: \n - bar\n - baz", "foo: \n - bar\n - baz"), - # more advanced uses of the grammar - ("ab \\{foo} cd", "ab \\{foo} cd"), - ("ab \\\\{foo} cd", "ab \\\\{foo} cd"), - (" 1 2 3 ", " 1 2 3 "), - ("\t[1, 2, 3]\t", "\t[1, 2, 3]\t"), - (" {a: b}\t ", " {a: b}\t "), - ], -) -def test_legacy_env_values_are_typed( - monkeypatch: Any, value: Any, expected: Any -) -> None: - monkeypatch.setenv("MYKEY", value) - c = OmegaConf.create({"my_key": "${env:MYKEY}"}) - with warns(UserWarning, match=re.escape("The `env` resolver is deprecated")): - assert c.my_key == expected diff -pruN 2.1.0~rc1-3/tests/interpolation/built_in_resolvers/test_oc_dict.py 2.2.2-1/tests/interpolation/built_in_resolvers/test_oc_dict.py --- 2.1.0~rc1-3/tests/interpolation/built_in_resolvers/test_oc_dict.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/tests/interpolation/built_in_resolvers/test_oc_dict.py 2022-05-27 21:36:40.000000000 +0000 @@ -261,6 +261,36 @@ def test_readonly_parent(cfg: Any, expec ("cfg", "expected"), [ param( + {"outer": {"x": "${oc.dict.values:.y}", "y": {"a": 1}}}, + [1], + id="values_inter", + ), + param( + {"outer": {"x": "${oc.dict.keys:.y}", "y": {"a": 1}}}, + ["a"], + id="keys_inter", + ), + param( + {"outer": {"x": "${oc.dict.values:..y}"}, "y": {"a": 1}}, + [1], + id="parent_values_inter", + ), + param( + {"outer": {"x": "${oc.dict.keys:..y}"}, "y": {"a": 1}}, + ["a"], + id="parent_keys_inter", + ), + ], +) +def test_relative_path(cfg: Any, expected: Any) -> None: + cfg = OmegaConf.create(cfg) + assert cfg.outer.x == expected + + +@mark.parametrize( + ("cfg", "expected"), + [ + param( {"x": "${sum:${oc.dict.values:y}}", "y": {"one": 1, "two": 2}}, 3, id="values", diff -pruN 2.1.0~rc1-3/tests/interpolation/built_in_resolvers/test_oc_env.py 2.2.2-1/tests/interpolation/built_in_resolvers/test_oc_env.py --- 2.1.0~rc1-3/tests/interpolation/built_in_resolvers/test_oc_env.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/tests/interpolation/built_in_resolvers/test_oc_env.py 2022-05-27 21:36:40.000000000 +0000 @@ -8,13 +8,12 @@ from omegaconf._utils import _ensure_con from omegaconf.errors import InterpolationResolutionError -@mark.parametrize("env_func", ["env", "oc.env"]) class TestEnvInterpolation: @mark.parametrize( ("cfg", "env_name", "env_val", "key", "expected"), [ param( - {"path": "/test/${${env_func}:foo}"}, + {"path": "/test/${oc.env:foo}"}, "foo", "1234", "path", @@ -22,7 +21,7 @@ class TestEnvInterpolation: id="simple", ), param( - {"path": "/test/${${env_func}:not_found,ZZZ}"}, + {"path": "/test/${oc.env:not_found,ZZZ}"}, None, None, "path", @@ -30,7 +29,7 @@ class TestEnvInterpolation: id="not_found_with_default", ), param( - {"path": "/test/${${env_func}:not_found,a/b}"}, + {"path": "/test/${oc.env:not_found,a/b}"}, None, None, "path", @@ -41,10 +40,7 @@ class TestEnvInterpolation: ) def test_env_interpolation( self, - # DEPRECATED: remove in 2.2 with the legacy env resolver - recwarn: Any, monkeypatch: Any, - env_func: str, cfg: Any, env_name: Optional[str], env_val: str, @@ -54,7 +50,6 @@ class TestEnvInterpolation: if env_name is not None: monkeypatch.setenv(env_name, env_val) - cfg["env_func"] = env_func # allows choosing which env resolver to use cfg = OmegaConf.create(cfg) assert OmegaConf.select(cfg, key) == expected @@ -63,7 +58,7 @@ class TestEnvInterpolation: ("cfg", "key", "expected"), [ param( - {"path": "/test/${${env_func}:not_found}"}, + {"path": "/test/${oc.env:not_found}"}, "path", raises( InterpolationResolutionError, @@ -75,14 +70,10 @@ class TestEnvInterpolation: ) def test_env_interpolation_error( self, - # DEPRECATED: remove in 2.2 with the legacy env resolver - recwarn: Any, - env_func: str, cfg: Any, key: str, expected: Any, ) -> None: - cfg["env_func"] = env_func # allows choosing which env resolver to use cfg = _ensure_container(cfg) with expected: diff -pruN 2.1.0~rc1-3/tests/interpolation/test_custom_resolvers.py 2.2.2-1/tests/interpolation/test_custom_resolvers.py --- 2.1.0~rc1-3/tests/interpolation/test_custom_resolvers.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/tests/interpolation/test_custom_resolvers.py 2022-05-27 21:36:40.000000000 +0000 @@ -100,13 +100,6 @@ def test_clear_resolvers_and_has_resolve assert not OmegaConf.has_resolver("foo") -def test_get_resolver_deprecation() -> None: - with warns( - UserWarning, match=re.escape("https://github.com/omry/omegaconf/issues/608") - ): - assert OmegaConf.get_resolver("foo") is None - - def test_register_resolver_1(restore_resolvers: Any) -> None: OmegaConf.register_new_resolver("plus_10", lambda x: x + 10) c = OmegaConf.create( diff -pruN 2.1.0~rc1-3/tests/interpolation/test_interpolation.py 2.2.2-1/tests/interpolation/test_interpolation.py --- 2.1.0~rc1-3/tests/interpolation/test_interpolation.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/tests/interpolation/test_interpolation.py 2022-05-27 21:36:40.000000000 +0000 @@ -1,5 +1,6 @@ import copy import re +from pathlib import Path from textwrap import dedent from typing import Any, Tuple @@ -24,7 +25,14 @@ from omegaconf.errors import Interpolati from omegaconf.errors import InterpolationResolutionError as IRE from omegaconf.errors import InterpolationValidationError, ReadonlyConfigError from omegaconf.nodes import InterpolationResultNode -from tests import MissingDict, MissingList, StructuredWithMissing, SubscriptedList, User +from tests import ( + Color, + MissingDict, + MissingList, + StructuredWithMissing, + SubscriptedList, + User, +) from tests.interpolation import dereference_node # file deepcode ignore CopyPasteError: @@ -133,6 +141,9 @@ def test_indirect_interpolation2() -> No param({"a": "${b}", "b": True, "s": "foo_${b}"}, id="bool"), param({"a": "${b}", "b": 10, "s": "foo_${b}"}, id="int"), param({"a": "${b}", "b": 3.14, "s": "foo_${b}"}, id="float"), + param({"a": "${b}", "b": Color.RED, "s": "foo_${b}"}, id="enum"), + param({"a": "${b}", "b": b"binary", "s": "foo_${b}"}, id="bytes"), + param({"a": "${b}", "b": Path("hello.txt"), "s": "foo_${b}"}, id="path"), ], ) def test_type_inherit_type(cfg: Any) -> None: @@ -332,7 +343,7 @@ def test_interpolation_type_validated_ok match=re.escape( dedent( """\ - Value 'seven' could not be converted to Integer + Value 'seven' of type 'str' could not be converted to Integer full_key: age """ ) @@ -345,7 +356,9 @@ def test_interpolation_type_validated_ok "age", raises( InterpolationValidationError, - match=re.escape("'Bond' could not be converted to Integer"), + match=re.escape( + "'Bond' of type 'str' could not be converted to Integer" + ), ), id="type_mismatch_node_interpolation", ), @@ -354,7 +367,9 @@ def test_interpolation_type_validated_ok "num", raises( InterpolationValidationError, - match=re.escape("Non optional field cannot be assigned None"), + match=re.escape( + "While dereferencing interpolation '${opt_num}': Incompatible value 'None' for field of type 'int'" + ), ), id="non_optional_node_interpolation", ), diff -pruN 2.1.0~rc1-3/tests/structured_conf/data/attr_classes.py 2.2.2-1/tests/structured_conf/data/attr_classes.py --- 2.1.0~rc1-3/tests/structured_conf/data/attr_classes.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/tests/structured_conf/data/attr_classes.py 2022-05-27 21:36:40.000000000 +0000 @@ -1,11 +1,12 @@ import sys +from pathlib import Path from typing import Any, Dict, List, Optional, Tuple, Union import attr from pytest import importorskip from omegaconf import II, MISSING, SI -from tests import Color +from tests import Color, Enum1 if sys.version_info >= (3, 8): # pragma: no cover from typing import TypedDict @@ -72,6 +73,12 @@ class OptionalUser: @attr.s(auto_attribs=True) +class InterpolationToUser: + user: User = User("Bond", 7) + admin: User = II("user") + + +@attr.s(auto_attribs=True) class AnyTypeConfig: with_default: Any = "Can get any type at runtime" null_default: Any = None @@ -155,6 +162,36 @@ class FloatConfig: @attr.s(auto_attribs=True) +class BytesConfig: + # with default value + with_default: bytes = b"binary" + + # default is None + null_default: Optional[bytes] = None + + # explicit no default + mandatory_missing: bytes = MISSING + + # interpolation, will inherit the type and value of `with_default' + interpolation: bytes = II("with_default") + + +@attr.s(auto_attribs=True) +class PathConfig: + # with default value + with_default: Path = Path("hello.txt") + + # default is None + null_default: Optional[Path] = None + + # explicit no default + mandatory_missing: Path = MISSING + + # interpolation, will inherit the type and value of `with_default' + interpolation: Path = II("with_default") + + +@attr.s(auto_attribs=True) class EnumConfig: # with default value with_default: Color = Color.BLUE @@ -407,10 +444,20 @@ class DictOfObjects: @attr.s(auto_attribs=True) +class DictOfObjectsMissing: + users: Dict[str, User] = {"moe": MISSING} + + +@attr.s(auto_attribs=True) class ListOfObjects: users: List[User] = [User(name="Joe", age=18)] +@attr.s(auto_attribs=True) +class ListOfObjectsMissing: + users: List[User] = [MISSING] + + class DictSubclass: @attr.s(auto_attribs=True) class Str2Str(Dict[str, str]): @@ -527,7 +574,7 @@ class NestedWithNone: @attr.s(auto_attribs=True) class UnionError: - x: Union[int, str] = 10 + x: Union[int, List[str]] = 10 @attr.s(auto_attribs=True) @@ -556,3 +603,184 @@ class UntypedList: class UntypedDict: dict: Dict = {"foo": "var"} # type: ignore opt_dict: Optional[Dict] = None # type: ignore + + +class StructuredSubclass: + @attr.s(auto_attribs=True) + class ParentInts: + int1: int + int2: int + int3: int = attr.NOTHING # type: ignore + int4: int = MISSING + + @attr.s(auto_attribs=True) + class ChildInts(ParentInts): + int2: int = 5 + int3: int = 10 + int4: int = 15 + + @attr.s(auto_attribs=True) + class ParentContainers: + list1: List[int] = MISSING + list2: List[int] = [5, 6] + dict: Dict[str, Any] = MISSING + + @attr.s(auto_attribs=True) + class ChildContainers(ParentContainers): + list1: List[int] = [1, 2, 3] + dict: Dict[str, Any] = {"a": 5, "b": 6} + + @attr.s(auto_attribs=True) + class ParentNoDefaultFactory: + no_default_to_list: Any + int_to_list: Any = 1 + + @attr.s(auto_attribs=True) + class ChildWithDefaultFactory(ParentNoDefaultFactory): + no_default_to_list: Any = ["hi"] + int_to_list: Any = ["hi"] + + +@attr.s(auto_attribs=True) +class HasInitFalseFields: + post_initialized: str = attr.field(init=False) + without_default: str = attr.field(init=False) + with_default: str = attr.field(init=False, default="default") + + def __attrs_post_init__(self) -> None: + self.post_initialized = "set_by_post_init" + + +class NestedContainers: + @attr.s(auto_attribs=True) + class ListOfLists: + lls: List[List[str]] + llx: List[List[User]] + llla: List[List[List[Any]]] + lloli: List[List[Optional[List[int]]]] + lls_default: List[List[str]] = [[], ["abc", "def", 123, MISSING], MISSING] # type: ignore + lolx_default: List[Optional[List[User]]] = [ + [], + [User(), User(age=7, name="Bond"), MISSING], + MISSING, + ] + + @attr.s(auto_attribs=True) + class DictOfDicts: + dsdsi: Dict[str, Dict[str, int]] + dsdbi: Dict[str, Dict[bool, int]] + dsdsx: Dict[str, Dict[str, User]] + odsdsi_default: Optional[Dict[str, Dict[str, int]]] = { + "dsi1": {}, + "dsi2": {"s1": 1, "s2": "123", "s3": MISSING}, # type: ignore + "dsi3": MISSING, + } + dsdsx_default: Dict[str, Dict[str, User]] = { + "dsx1": {}, + "dsx2": { + "s1": User(), + "s2": User(age=7, name="Bond"), + "s3": MISSING, + }, + "dsx3": MISSING, + } + + @attr.s(auto_attribs=True) + class ListsAndDicts: + lldsi: List[List[Dict[str, int]]] + ldaos: List[Dict[Any, Optional[str]]] + dedsle: Dict[Color, Dict[str, List[Enum1]]] + dsolx: Dict[str, Optional[List[User]]] + oldfox: Optional[List[Dict[float, Optional[User]]]] + dedsle_default: Dict[Color, Dict[str, List[Enum1]]] = { + Color.RED: {"a": [Enum1.FOO, Enum1.BAR]}, + Color.GREEN: {"b": []}, + Color.BLUE: {}, + } + + @attr.s(auto_attribs=True) + class WithDefault: + dsolx_default: Dict[str, Optional[List[User]]] = {"lx": [User()], "n": None} + + +class UnionsOfPrimitveTypes: + @attr.s(auto_attribs=True) + class Simple: + uis: Union[int, str] + ubc: Union[bool, Color] + uxf: Union[bytes, float] + ouis: Optional[Union[int, str]] + uois: Union[Optional[int], str] + uisn: Union[int, str, None] + uisN: Union[int, str, type(None)] # type: ignore + + @attr.s(auto_attribs=True) + class WithDefaults: + uis: Union[int, str] = "abc" + ubc1: Union[bool, Color] = True + ubc2: Union[bool, Color] = Color.RED + uxf: Union[bytes, float] = 1.2 + ouis: Optional[Union[int, str]] = None + uisn: Union[int, str, None] = 123 + uisN: Union[int, str, type(None)] = "abc" # type: ignore + + @attr.s(auto_attribs=True) + class WithExplicitMissing: + uis_missing: Union[int, str] = MISSING + + @attr.s(auto_attribs=True) + class WithBadDefaults1: + uis: Union[int, str] = None # type: ignore + + @attr.s(auto_attribs=True) + class WithBadDefaults2: + ubc: Union[bool, Color] = "abc" # type: ignore + + @attr.s(auto_attribs=True) + class WithBadDefaults3: + uxf: Union[bytes, float] = True + + @attr.s(auto_attribs=True) + class WithBadDefaults4: + oufb: Optional[Union[float, bool]] = Color.RED # type: ignore + + @attr.s(auto_attribs=True) + class ContainersOfUnions: + lubc: List[Union[bool, Color]] + dsubf: Dict[str, Union[bool, float]] + dsoubf: Dict[str, Optional[Union[bool, float]]] + lubc_with_default: List[Union[bool, Color]] = [True, Color.RED] + dsubf_with_default: Dict[str, Union[bool, float]] = {"abc": True, "xyz": 1.2} + + @attr.s(auto_attribs=True) + class InterpolationFromUnion: + ubi: Union[bool, int] + oubi: Optional[Union[bool, int]] + an_int: int = 123 + a_string: str = "abc" + missing: int = MISSING + none: Optional[int] = None + ubi_with_default: Union[bool, int] = II("an_int") + oubi_with_default: Optional[Union[bool, int]] = II("none") + + @attr.s(auto_attribs=True) + class InterpolationToUnion: + a_float: float = II("ufs") + bad_int_interp: bool = II("ufs") + ufs: Union[float, str] = 10.1 + + @attr.s(auto_attribs=True) + class BadInterpolationFromUnion: + a_float: float = 10.1 + ubi: Union[bool, int] = II("a_float") + + if sys.version_info >= (3, 10): + + @attr.s(auto_attribs=True) + class SupportPEP604: + """https://peps.python.org/pep-0604/""" + + uis: int | str + ouis: Optional[int | str] + uisn: int | str | None = None + uis_with_default: int | str = 123 diff -pruN 2.1.0~rc1-3/tests/structured_conf/data/dataclasses.py 2.2.2-1/tests/structured_conf/data/dataclasses.py --- 2.1.0~rc1-3/tests/structured_conf/data/dataclasses.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/tests/structured_conf/data/dataclasses.py 2022-05-27 21:36:40.000000000 +0000 @@ -1,12 +1,13 @@ import dataclasses import sys from dataclasses import dataclass, field +from pathlib import Path from typing import Any, Dict, List, Optional, Tuple, Union from pytest import importorskip from omegaconf import II, MISSING, SI -from tests import Color +from tests import Color, Enum1 if sys.version_info >= (3, 8): # pragma: no cover from typing import TypedDict @@ -73,6 +74,12 @@ class OptionalUser: @dataclass +class InterpolationToUser: + user: User = User("Bond", 7) + admin: User = II("user") + + +@dataclass class AnyTypeConfig: with_default: Any = "Can get any type at runtime" null_default: Any = None @@ -156,6 +163,36 @@ class FloatConfig: @dataclass +class BytesConfig: + # with default value + with_default: bytes = b"binary" + + # default is None + null_default: Optional[bytes] = None + + # explicit no default + mandatory_missing: bytes = MISSING + + # interpolation, will inherit the type and value of `with_default' + interpolation: bytes = II("with_default") + + +@dataclass +class PathConfig: + # with default value + with_default: Path = Path("hello.txt") + + # default is None + null_default: Optional[Path] = None + + # explicit no default + mandatory_missing: Path = MISSING + + # interpolation, will inherit the type and value of `with_default' + interpolation: Path = II("with_default") + + +@dataclass class EnumConfig: # with default value with_default: Color = Color.BLUE @@ -428,10 +465,20 @@ class DictOfObjects: @dataclass +class DictOfObjectsMissing: + users: Dict[str, User] = field(default_factory=lambda: {"moe": MISSING}) + + +@dataclass class ListOfObjects: users: List[User] = field(default_factory=lambda: [User(name="Joe", age=18)]) +@dataclass +class ListOfObjectsMissing: + users: List[User] = field(default_factory=lambda: [MISSING]) + + class DictSubclass: @dataclass class Str2Str(Dict[str, str]): @@ -548,7 +595,7 @@ class NestedWithNone: @dataclass class UnionError: - x: Union[int, str] = 10 + x: Union[int, List[str]] = 10 @dataclass @@ -577,3 +624,200 @@ class UntypedList: class UntypedDict: dict: Dict = field(default_factory=lambda: {"foo": "var"}) # type: ignore opt_dict: Optional[Dict] = None # type: ignore + + +class StructuredSubclass: + @dataclass + class ParentInts: + int1: int + int2: int + int3: int = dataclasses.MISSING # type: ignore + int4: int = MISSING + + @dataclass + class ChildInts(ParentInts): + int2: int = 5 + int3: int = 10 + int4: int = 15 + + @dataclass + class ParentContainers: + list1: List[int] = MISSING + list2: List[int] = field(default_factory=lambda: [5, 6]) + dict: Dict[str, Any] = MISSING + + @dataclass + class ChildContainers(ParentContainers): + list1: List[int] = field(default_factory=lambda: [1, 2, 3]) + dict: Dict[str, Any] = field(default_factory=lambda: {"a": 5, "b": 6}) + + @dataclass + class ParentNoDefaultFactory: + no_default_to_list: Any + int_to_list: Any = 1 + + @dataclass + class ChildWithDefaultFactory(ParentNoDefaultFactory): + no_default_to_list: Any = field(default_factory=lambda: ["hi"]) + int_to_list: Any = field(default_factory=lambda: ["hi"]) + + +@dataclass +class HasInitFalseFields: + post_initialized: str = field(init=False) + without_default: str = field(init=False) + with_default: str = field(init=False, default="default") + + def __post_init__(self) -> None: + self.post_initialized = "set_by_post_init" + + +class NestedContainers: + @dataclass + class ListOfLists: + lls: List[List[str]] + llx: List[List[User]] + llla: List[List[List[Any]]] + lloli: List[List[Optional[List[int]]]] + lls_default: List[List[str]] = field( + default_factory=lambda: [[], ["abc", "def", 123, MISSING], MISSING] # type: ignore + ) + lolx_default: List[Optional[List[User]]] = field( + default_factory=lambda: [ + [], + [User(), User(age=7, name="Bond"), MISSING], + MISSING, + ] + ) + + @dataclass + class DictOfDicts: + dsdsi: Dict[str, Dict[str, int]] + dsdbi: Dict[str, Dict[bool, int]] + dsdsx: Dict[str, Dict[str, User]] + odsdsi_default: Optional[Dict[str, Dict[str, int]]] = field( + default_factory=lambda: { + "dsi1": {}, + "dsi2": {"s1": 1, "s2": "123", "s3": MISSING}, + "dsi3": MISSING, + } + ) + dsdsx_default: Dict[str, Dict[str, User]] = field( + default_factory=lambda: { + "dsx1": {}, + "dsx2": { + "s1": User(), + "s2": User(age=7, name="Bond"), + "s3": MISSING, + }, + "dsx3": MISSING, + } + ) + + @dataclass + class ListsAndDicts: + lldsi: List[List[Dict[str, int]]] + ldaos: List[Dict[Any, Optional[str]]] + dedsle: Dict[Color, Dict[str, List[Enum1]]] + dsolx: Dict[str, Optional[List[User]]] + oldfox: Optional[List[Dict[float, Optional[User]]]] + dedsle_default: Dict[Color, Dict[str, List[Enum1]]] = field( + default_factory=lambda: { + Color.RED: {"a": [Enum1.FOO, Enum1.BAR]}, + Color.GREEN: {"b": []}, + Color.BLUE: {}, + } + ) + + @dataclass + class WithDefault: + dsolx_default: Dict[str, Optional[List[User]]] = field( + default_factory=lambda: {"lx": [User()], "n": None} + ) + + +class UnionsOfPrimitveTypes: + @dataclass + class Simple: + uis: Union[int, str] + ubc: Union[bool, Color] + uxf: Union[bytes, float] + ouis: Optional[Union[int, str]] + uois: Union[Optional[int], str] + uisn: Union[int, str, None] + uisN: Union[int, str, type(None)] # type: ignore + + @dataclass + class WithDefaults: + uis: Union[int, str] = "abc" + ubc1: Union[bool, Color] = True + ubc2: Union[bool, Color] = Color.RED + uxf: Union[bytes, float] = 1.2 + ouis: Optional[Union[int, str]] = None + uisn: Union[int, str, None] = 123 + uisN: Union[int, str, type(None)] = "abc" # type: ignore + + @dataclass + class WithExplicitMissing: + uis_missing: Union[int, str] = MISSING + + @dataclass + class WithBadDefaults1: + uis: Union[int, str] = None # type: ignore + + @dataclass + class WithBadDefaults2: + ubc: Union[bool, Color] = "abc" # type: ignore + + @dataclass + class WithBadDefaults3: + uxf: Union[bytes, float] = True + + @dataclass + class WithBadDefaults4: + oufb: Optional[Union[float, bool]] = Color.RED # type: ignore + + @dataclass + class ContainersOfUnions: + lubc: List[Union[bool, Color]] + dsubf: Dict[str, Union[bool, float]] + dsoubf: Dict[str, Optional[Union[bool, float]]] + lubc_with_default: List[Union[bool, Color]] = field( + default_factory=lambda: [True, Color.RED] + ) + dsubf_with_default: Dict[str, Union[bool, float]] = field( + default_factory=lambda: {"abc": True, "xyz": 1.2} + ) + + @dataclass + class InterpolationFromUnion: + ubi: Union[bool, int] + oubi: Optional[Union[bool, int]] + an_int: int = 123 + a_string: str = "abc" + missing: int = MISSING + none: Optional[int] = None + ubi_with_default: Union[bool, int] = II("an_int") + oubi_with_default: Optional[Union[bool, int]] = II("none") + + @dataclass + class InterpolationToUnion: + a_float: float = II("ufs") + bad_int_interp: bool = II("ufs") + ufs: Union[float, str] = 10.1 + + @dataclass + class BadInterpolationFromUnion: + a_float: float = 10.1 + ubi: Union[bool, int] = II("a_float") + + if sys.version_info >= (3, 10): + + @dataclass + class SupportPEP604: + """https://peps.python.org/pep-0604/""" + + uis: int | str + ouis: Optional[int | str] + uisn: int | str | None = None + uis_with_default: int | str = 123 diff -pruN 2.1.0~rc1-3/tests/structured_conf/test_structured_basic.py 2.2.2-1/tests/structured_conf/test_structured_basic.py --- 2.1.0~rc1-3/tests/structured_conf/test_structured_basic.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/tests/structured_conf/test_structured_basic.py 2022-05-27 21:36:40.000000000 +0000 @@ -13,7 +13,7 @@ from omegaconf import ( _utils, flag_override, ) -from omegaconf._utils import _is_optional, get_ref_type +from omegaconf._utils import _is_optional, get_type_hint from omegaconf.errors import ConfigKeyError, UnsupportedValueType from tests import IllegalType @@ -51,7 +51,9 @@ class TestStructured: def test_error_on_creation_with_bad_value_type(self, module: Any) -> None: with raises( ValidationError, - match=re.escape("Value 'seven' could not be converted to Integer"), + match=re.escape( + "Value 'seven' of type 'str' could not be converted to Integer" + ), ): OmegaConf.structured(module.User(age="seven")) @@ -104,7 +106,7 @@ class TestStructured: def test_error_message(self, module: Any) -> None: cfg = OmegaConf.structured(module.StructuredOptional) - msg = re.escape("child 'not_optional' is not Optional") + msg = re.escape("field 'not_optional' is not Optional") with raises(ValidationError, match=msg): cfg.not_optional = None @@ -136,7 +138,7 @@ class TestStructured: def test_get_type(self, module: Any) -> None: cfg1 = OmegaConf.create(module.LinkedList) assert OmegaConf.get_type(cfg1) == module.LinkedList - assert _utils.get_ref_type(cfg1, "next") == Optional[module.LinkedList] + assert _utils.get_type_hint(cfg1, "next") == Optional[module.LinkedList] assert OmegaConf.get_type(cfg1, "next") is None assert cfg1.next is None @@ -144,42 +146,42 @@ class TestStructured: cfg2 = OmegaConf.create(module.MissingTest.Missing1) assert OmegaConf.is_missing(cfg2, "head") - assert _utils.get_ref_type(cfg2, "head") == module.LinkedList + assert _utils.get_type_hint(cfg2, "head") == module.LinkedList assert OmegaConf.get_type(cfg2, "head") is None - def test_merge_structured_onto_dict(self, module: Any) -> None: + def test_merge_structured_into_dict(self, module: Any) -> None: c1 = OmegaConf.create({"name": 7}) c2 = OmegaConf.merge(c1, module.User) assert c1 == {"name": 7} # type of name becomes str assert c2 == {"name": "7", "age": "???"} - def test_merge_structured_onto_dict_nested(self, module: Any) -> None: + def test_merge_structured_into_dict_nested(self, module: Any) -> None: c1 = OmegaConf.create({"user": {"name": 7}}) c2 = OmegaConf.merge(c1, module.MissingUserField) assert c1 == {"user": {"name": 7}} # type of name becomes str assert c2 == {"user": {"name": "7", "age": "???"}} assert isinstance(c2, DictConfig) - assert get_ref_type(c2, "user") == module.User + assert get_type_hint(c2, "user") == module.User - def test_merge_structured_onto_dict_nested2(self, module: Any) -> None: + def test_merge_structured_into_dict_nested2(self, module: Any) -> None: c1 = OmegaConf.create({"user": {"name": IntegerNode(value=7)}}) c2 = OmegaConf.merge(c1, module.MissingUserField) assert c1 == {"user": {"name": 7}} # type of name remains int assert c2 == {"user": {"name": 7, "age": "???"}} assert isinstance(c2, DictConfig) - assert get_ref_type(c2, "user") == module.User + assert get_type_hint(c2, "user") == module.User - def test_merge_structured_onto_dict_nested3(self, module: Any) -> None: + def test_merge_structured_into_dict_nested3(self, module: Any) -> None: c1 = OmegaConf.create({"user": {"name": "alice"}}) c2 = OmegaConf.merge(c1, module.MissingUserWithDefaultNameField) assert c1 == {"user": {"name": "alice"}} # name is not changed assert c2 == {"user": {"name": "alice", "age": "???"}} assert isinstance(c2, DictConfig) - assert get_ref_type(c2, "user") == module.UserWithDefaultName + assert get_type_hint(c2, "user") == module.UserWithDefaultName def test_merge_missing_object_onto_typed_dictconfig(self, module: Any) -> None: c1 = OmegaConf.structured(module.DictOfObjects) @@ -200,11 +202,11 @@ class TestStructured: assert c2.foo.user is None assert c2.foo._get_node("user")._metadata.ref_type == module.User - def test_merge_optional_structured_onto_dict(self, module: Any) -> None: + def test_merge_optional_structured_into_dict(self, module: Any) -> None: c1 = OmegaConf.create({"user": {"name": "bob"}}) c2 = OmegaConf.merge(c1, module.OptionalUser(module.User(name="alice"))) assert c2.user.name == "alice" - assert get_ref_type(c2, "user") == Optional[module.User] + assert get_type_hint(c2, "user") == Optional[module.User] assert isinstance(c2, DictConfig) c2_user = c2._get_node("user") assert isinstance(c2_user, Node) @@ -226,9 +228,17 @@ class TestStructured: src.user_3 = None c2 = OmegaConf.merge(c1, src) assert c2.user_2.name == "bob" - assert get_ref_type(c2, "user_2") == Any + assert get_type_hint(c2, "user_2") == Any assert c2.user_3 is None - assert get_ref_type(c2, "user_3") == Any + assert get_type_hint(c2, "user_3") == Any + + @mark.parametrize("resolve", [True, False]) + def test_interpolation_to_structured(self, module: Any, resolve: bool) -> None: + cfg = OmegaConf.create(module.InterpolationToUser) + if resolve: + OmegaConf.resolve(cfg) + assert OmegaConf.get_type(cfg.admin) is module.User + assert cfg.admin == {"name": "Bond", "age": 7} class TestMissing: def test_missing1(self, module: Any) -> None: @@ -252,24 +262,24 @@ class TestStructured: cfg = OmegaConf.create(module.PluginHolder) assert _is_optional(cfg, "none") - assert _utils.get_ref_type(cfg, "none") == Optional[module.Plugin] + assert _utils.get_type_hint(cfg, "none") == Optional[module.Plugin] assert OmegaConf.get_type(cfg, "none") is None assert not _is_optional(cfg, "missing") - assert _utils.get_ref_type(cfg, "missing") == module.Plugin + assert _utils.get_type_hint(cfg, "missing") == module.Plugin assert OmegaConf.get_type(cfg, "missing") is None assert not _is_optional(cfg, "plugin") - assert _utils.get_ref_type(cfg, "plugin") == module.Plugin + assert _utils.get_type_hint(cfg, "plugin") == module.Plugin assert OmegaConf.get_type(cfg, "plugin") == module.Plugin cfg.plugin = module.ConcretePlugin() assert not _is_optional(cfg, "plugin") - assert _utils.get_ref_type(cfg, "plugin") == module.Plugin + assert _utils.get_type_hint(cfg, "plugin") == module.Plugin assert OmegaConf.get_type(cfg, "plugin") == module.ConcretePlugin assert not _is_optional(cfg, "plugin2") - assert _utils.get_ref_type(cfg, "plugin2") == module.Plugin + assert _utils.get_type_hint(cfg, "plugin2") == module.Plugin assert OmegaConf.get_type(cfg, "plugin2") == module.ConcretePlugin def test_plugin_merge(self, module: Any) -> None: diff -pruN 2.1.0~rc1-3/tests/structured_conf/test_structured_config.py 2.2.2-1/tests/structured_conf/test_structured_config.py --- 2.1.0~rc1-3/tests/structured_conf/test_structured_config.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/tests/structured_conf/test_structured_config.py 2022-05-27 21:36:40.000000000 +0000 @@ -1,12 +1,19 @@ +import dataclasses +import inspect +import pathlib +import re import sys from importlib import import_module -from typing import Any, Dict, List, Optional +from pathlib import Path +from typing import Any, Callable, Dict, List, Optional, Union +from _pytest.python_api import RaisesContext from pytest import fixture, mark, param, raises from omegaconf import ( MISSING, AnyNode, + Container, DictConfig, KeyValidationError, ListConfig, @@ -16,8 +23,8 @@ from omegaconf import ( ValidationError, _utils, ) -from omegaconf.errors import ConfigKeyError -from tests import Color, User, warns_dict_subclass_deprecated +from omegaconf.errors import ConfigKeyError, InterpolationToMissingValueError +from tests import Color, Enum1, User, warns_dict_subclass_deprecated @fixture( @@ -46,17 +53,43 @@ class EnumConfigAssignments: (2, Color.GREEN), (3, Color.BLUE), ] - illegal = ["foo", True, False, 4, 1.0] + illegal = ["foo", True, b"RED", False, 4, 1.0, Path("hello.txt")] class IntegersConfigAssignments: - legal = [("10", 10), ("-10", -10), 100, 0, 1, 1] - illegal = ["foo", 1.0, float("inf"), float("nan"), Color.BLUE] + legal = [("10", 10), ("-10", -10), 100, 0, 1] + illegal = [ + "foo", + 1.0, + float("inf"), + b"123", + float("nan"), + Color.BLUE, + True, + Path("hello.txt"), + ] class StringConfigAssignments: - legal = ["10", "-10", "foo", "", (Color.BLUE, "Color.BLUE")] - illegal: Any = [] + legal = [ + "10", + "-10", + "foo", + "", + (Color.BLUE, "Color.BLUE"), + (Path("hello.txt"), "hello.txt"), + ] + illegal = [b"binary"] + + +class BytesConfigAssignments: + legal = [b"binary"] + illegal = ["foo", 10, Color.BLUE, 10.1, True, Path("hello.txt")] + + +class PathConfigAssignments: + legal = [Path("hello.txt"), ("hello.txt", Path("hello.txt"))] + illegal = [10, Color.BLUE, 10.1, True, b"binary"] class FloatConfigAssignments: @@ -68,7 +101,7 @@ class FloatConfigAssignments: ("10.2", 10.2), ("10e-3", 10e-3), ] - illegal = ["foo", True, False, Color.BLUE] + illegal = ["foo", True, False, b"10.1", Color.BLUE, Path("hello.txt")] class BoolConfigAssignments: @@ -88,11 +121,11 @@ class BoolConfigAssignments: ("0", False), (0, False), ] - illegal = [100.0, Color.BLUE] + illegal = [100.0, b"binary", Color.BLUE, Path("hello.txt")] class AnyTypeConfigAssignments: - legal = [True, False, 10, 10.0, "foobar", Color.BLUE] + legal = [True, False, 10, 10.0, b"binary", "foobar", Color.BLUE, Path("hello.txt")] illegal: Any = [] @@ -102,7 +135,7 @@ class TestConfigs: cfg = OmegaConf.structured(module.NestedWithNone) assert cfg == {"plugin": None} assert OmegaConf.get_type(cfg, "plugin") is None - assert _utils.get_ref_type(cfg, "plugin") == Optional[module.Plugin] + assert _utils.get_type_hint(cfg, "plugin") == Optional[module.Plugin] def test_nested_config(self, module: Any) -> None: def validate(cfg: DictConfig) -> None: @@ -202,8 +235,15 @@ class TestConfigs: conf1 = OmegaConf.structured(module.ConfigWithList) validate(conf1) - conf1 = OmegaConf.structured(module.ConfigWithList()) - validate(conf1) + conf2 = OmegaConf.structured(module.ConfigWithList()) + validate(conf2) + + def test_config_with_list_nondefault_values(self, module: Any) -> None: + conf1 = OmegaConf.structured(module.ConfigWithList(list1=[4, 5, 6])) + assert conf1.list1 == [4, 5, 6] + + conf2 = OmegaConf.structured(module.ConfigWithList(list1=MISSING)) + assert OmegaConf.is_missing(conf2, "list1") def test_assignment_to_nested_structured_config(self, module: Any) -> None: conf = OmegaConf.structured(module.NestedConfig) @@ -226,8 +266,16 @@ class TestConfigs: conf1 = OmegaConf.structured(module.ConfigWithDict) validate(conf1) - conf1 = OmegaConf.structured(module.ConfigWithDict()) - validate(conf1) + + conf2 = OmegaConf.structured(module.ConfigWithDict()) + validate(conf2) + + def test_config_with_dict_nondefault_values(self, module: Any) -> None: + conf1 = OmegaConf.structured(module.ConfigWithDict(dict1={"baz": "qux"})) + assert conf1.dict1 == {"baz": "qux"} + + conf2 = OmegaConf.structured(module.ConfigWithDict(dict1=MISSING)) + assert OmegaConf.is_missing(conf2, "dict1") def test_structured_config_struct_behavior(self, module: Any) -> None: def validate(cfg: DictConfig) -> None: @@ -257,12 +305,16 @@ class TestConfigs: ("BoolConfig", BoolConfigAssignments, {}), ("IntegersConfig", IntegersConfigAssignments, {}), ("FloatConfig", FloatConfigAssignments, {}), + ("BytesConfig", BytesConfigAssignments, {}), + ("PathConfig", PathConfigAssignments, {}), ("StringConfig", StringConfigAssignments, {}), ("EnumConfig", EnumConfigAssignments, {}), # Use instance to build config ("BoolConfig", BoolConfigAssignments, {"with_default": False}), ("IntegersConfig", IntegersConfigAssignments, {"with_default": 42}), ("FloatConfig", FloatConfigAssignments, {"with_default": 42.0}), + ("BytesConfig", BytesConfigAssignments, {"with_default": b"bin"}), + ("PathConfig", PathConfigAssignments, {"with_default": Path("file.txt")}), ("StringConfig", StringConfigAssignments, {"with_default": "fooooooo"}), ("EnumConfig", EnumConfigAssignments, {"with_default": Color.BLUE}), ("AnyTypeConfig", AnyTypeConfigAssignments, {}), @@ -496,11 +548,11 @@ class TestConfigs: def test_merge_with_subclass_into_missing(self, module: Any) -> None: base = OmegaConf.structured(module.PluginHolder) - assert _utils.get_ref_type(base, "missing") == module.Plugin + assert _utils.get_type_hint(base, "missing") == module.Plugin assert OmegaConf.get_type(base, "missing") is None res = OmegaConf.merge(base, {"missing": module.Plugin}) assert OmegaConf.get_type(res) == module.PluginHolder - assert _utils.get_ref_type(base, "missing") == module.Plugin + assert _utils.get_type_hint(base, "missing") == module.Plugin assert OmegaConf.get_type(res, "missing") == module.Plugin def test_merged_with_nons_subclass(self, module: Any) -> None: @@ -746,6 +798,20 @@ class TestConfigs: with raises(ValidationError): dct.fail = "fail" + def test_dict_of_objects_missing(self, module: Any) -> None: + conf = OmegaConf.structured(module.DictOfObjectsMissing) + dct = conf.users + + assert OmegaConf.is_missing(dct, "moe") + + dct.miss = MISSING + assert OmegaConf.is_missing(dct, "miss") + + def test_assign_dict_of_objects(self, module: Any) -> None: + conf = OmegaConf.structured(module.DictOfObjects) + conf.users = {"poe": module.User(name="Poe", age=8), "miss": MISSING} + assert conf.users == {"poe": {"name": "Poe", "age": 8}, "miss": "???"} + def test_list_of_objects(self, module: Any) -> None: conf = OmegaConf.structured(module.ListOfObjects) assert conf.users[0].age == 18 @@ -758,6 +824,19 @@ class TestConfigs: with raises(ValidationError): conf.users.append("fail") + def test_list_of_objects_missing(self, module: Any) -> None: + conf = OmegaConf.structured(module.ListOfObjectsMissing) + + assert OmegaConf.is_missing(conf.users, 0) + + conf.users.append(MISSING) + assert OmegaConf.is_missing(conf.users, 1) + + def test_assign_list_of_objects(self, module: Any) -> None: + conf = OmegaConf.structured(module.ListOfObjects) + conf.users = [module.User(name="Poe", age=8), MISSING] + assert conf.users == [{"name": "Poe", "age": 8}, "???"] + def test_promote_api(self, module: Any) -> None: conf = OmegaConf.create(module.AnyTypeConfig) conf._promote(None) @@ -787,6 +866,22 @@ class TestConfigs: assert OmegaConf.get_type(conf) == module.BoolConfig assert conf.with_default is False + def test_promote_to_dataclass(self, module: Any) -> None: + @dataclasses.dataclass + class Foo: + foo: pathlib.Path + bar: str + qub: int = 5 + + x = DictConfig({"foo": "hello.txt", "bar": "hello.txt"}) + assert isinstance(x.foo, str) + assert isinstance(x.bar, str) + + x._promote(Foo) + assert isinstance(x.foo, pathlib.Path) + assert isinstance(x.bar, str) + assert x.qub == 5 + def test_set_key_with_with_dataclass(self, module: Any) -> None: cfg = OmegaConf.create({"foo": [1, 2]}) cfg.foo = module.ListClass() @@ -808,10 +903,7 @@ class TestConfigs: 3.1415, ["foo", True, 1.2], User(), - param( - [None], - marks=mark.xfail, # https://github.com/omry/omegaconf/issues/579 - ), + param([None]), ], ) def test_assign_wrong_type_to_list(self, module: Any, value: Any) -> None: @@ -825,10 +917,7 @@ class TestConfigs: @mark.parametrize( "value", [ - param( - None, - marks=mark.xfail, # https://github.com/omry/omegaconf/issues/579 - ), + param(None), True, "str", 3.1415, @@ -849,10 +938,7 @@ class TestConfigs: 3.1415, ["foo", True, 1.2], {"foo": True}, - param( - {"foo": None}, - marks=mark.xfail, # expected failure, https://github.com/omry/omegaconf/issues/579 - ), + param({"foo": None}), User(age=1, name="foo"), {"user": User(age=1, name="foo")}, ListConfig(content=[1, 2], ref_type=List[int], element_type=int), @@ -883,15 +969,15 @@ class TestConfigs: def test_create_untyped_dict(self, module: Any) -> None: cfg = OmegaConf.structured(module.UntypedDict) - assert _utils.get_ref_type(cfg, "dict") == Dict[Any, Any] - assert _utils.get_ref_type(cfg, "opt_dict") == Optional[Dict[Any, Any]] + assert _utils.get_type_hint(cfg, "dict") == Dict[Any, Any] + assert _utils.get_type_hint(cfg, "opt_dict") == Optional[Dict[Any, Any]] assert cfg.dict == {"foo": "var"} assert cfg.opt_dict is None def test_create_untyped_list(self, module: Any) -> None: cfg = OmegaConf.structured(module.UntypedList) - assert _utils.get_ref_type(cfg, "list") == List[Any] - assert _utils.get_ref_type(cfg, "opt_list") == Optional[List[Any]] + assert _utils.get_type_hint(cfg, "list") == List[Any] + assert _utils.get_type_hint(cfg, "opt_list") == Optional[List[Any]] assert cfg.list == [1, 2] assert cfg.opt_list is None @@ -917,15 +1003,8 @@ def validate_frozen_impl(conf: DictConfi ret.user.age = 20 -def test_attr_frozen() -> None: - from tests.structured_conf.data.attr_classes import FrozenClass - - validate_frozen_impl(OmegaConf.structured(FrozenClass)) - validate_frozen_impl(OmegaConf.structured(FrozenClass())) - - -def test_dataclass_frozen() -> None: - from tests.structured_conf.data.dataclasses import FrozenClass +def test_frozen(module: Any) -> None: + FrozenClass = module.FrozenClass validate_frozen_impl(OmegaConf.structured(FrozenClass)) validate_frozen_impl(OmegaConf.structured(FrozenClass())) @@ -960,7 +1039,7 @@ class TestDictSubclass: with warns_dict_subclass_deprecated(module.DictSubclass.Str2Str): cfg = OmegaConf.create({"foo": module.DictSubclass.Str2Str}) assert OmegaConf.get_type(cfg.foo) == module.DictSubclass.Str2Str - assert _utils.get_ref_type(cfg.foo) == Any + assert _utils.get_type_hint(cfg.foo) == Any cfg.foo.hello = "world" assert cfg.foo.hello == "world" @@ -994,7 +1073,7 @@ class TestDictSubclass: with warns_dict_subclass_deprecated(module.DictSubclass.Int2Str): cfg = OmegaConf.create({"foo": module.DictSubclass.Int2Str}) assert OmegaConf.get_type(cfg.foo) == module.DictSubclass.Int2Str - assert _utils.get_ref_type(cfg.foo) == Any + assert _utils.get_type_hint(cfg.foo) == Any cfg.foo[10] = "ten" assert cfg.foo[10] == "ten" @@ -1179,3 +1258,1017 @@ class TestConfigs2: c3 = OmegaConf.merge(c1, c2) with raises(ValidationError): c3.missing.append("xx") + + +class TestStructredConfigInheritance: + def test_leaf_node_inheritance(self, module: Any) -> None: + parent = OmegaConf.structured(module.StructuredSubclass.ParentInts) + child = OmegaConf.structured(module.StructuredSubclass.ChildInts) + + assert OmegaConf.is_missing(parent, "int1") + assert OmegaConf.is_missing(child, "int1") + + assert OmegaConf.is_missing(parent, "int2") + assert child.int2 == 5 + + assert OmegaConf.is_missing(parent, "int3") + assert child.int3 == 10 + + assert OmegaConf.is_missing(parent, "int4") + assert child.int4 == 15 + + def test_container_inheritance(self, module: Any) -> None: + parent = OmegaConf.structured(module.StructuredSubclass.ParentContainers) + child = OmegaConf.structured(module.StructuredSubclass.ChildContainers) + + assert OmegaConf.is_missing(parent, "list1") + assert child.list1 == [1, 2, 3] + + assert parent.list2 == [5, 6] + assert child.list2 == [5, 6] + + assert OmegaConf.is_missing(parent, "dict") + assert child.dict == {"a": 5, "b": 6} + + @mark.parametrize( + "create_fn", + [ + param(lambda cls: OmegaConf.structured(cls), id="create_from_class"), + param(lambda cls: OmegaConf.structured(cls()), id="create_from_instance"), + ], + ) + def test_subclass_using_default_factory( + self, module: Any, create_fn: Callable[[Any], DictConfig] + ) -> None: + """ + When a structured config field has a default and a subclass defines a + default_factory for the same field, ensure that the DictConfig created + from the subclass uses the subclass' default_factory (not the parent + class' default). + """ + cfg = create_fn(module.StructuredSubclass.ChildWithDefaultFactory) + assert cfg.no_default_to_list == ["hi"] + assert cfg.int_to_list == ["hi"] + + +class TestNestedContainers: + @mark.parametrize( + "class_name", + [ + "ListOfLists", + "DictOfDicts", + "ListsAndDicts", + "WithDefault", + ], + ) + def test_instantiation(self, module: Any, class_name: str) -> None: + cls = getattr(module.NestedContainers, class_name) + OmegaConf.structured(cls) + + @mark.parametrize( + "class_name, keys, expected_optional, expected_ref_type", + [ + param("ListOfLists", "lls", False, List[List[str]], id="lls"), + param( + "ListOfLists", + "llx", + False, + lambda module: List[List[module.User]], # type: ignore + id="llx", + ), + param("ListOfLists", "llla", False, List[List[List[Any]]], id="llla"), + param( + "ListOfLists", + "lloli", + False, + List[List[Optional[List[int]]]], + id="lloli", + ), + param( + "ListOfLists", + "lloli", + False, + List[List[Optional[List[int]]]], + id="lloli", + ), + param( + "ListOfLists", "lls_default", False, List[List[str]], id="lls_default" + ), + param( + "ListOfLists", "lls_default", False, List[List[str]], id="lls_default" + ), + param( + "ListOfLists", ["lls_default", 0], False, List[str], id="lls_default-0" + ), + param( + "ListOfLists", ["lls_default", 1], False, List[str], id="lls_default-1" + ), + param( + "ListOfLists", ["lls_default", 2], False, List[str], id="lls_default-2" + ), + param( + "ListOfLists", + "lolx_default", + False, + lambda module: List[Optional[List[module.User]]], # type: ignore + id="lolx_default", + ), + param( + "ListOfLists", + ["lolx_default", 0], + True, + lambda module: List[module.User], # type: ignore + id="lolx_default-0", + ), + param( + "ListOfLists", + ["lolx_default", 1], + True, + lambda module: List[module.User], # type: ignore + id="lolx_default-1", + ), + param( + "ListOfLists", + ["lolx_default", 2], + True, + lambda module: List[module.User], # type: ignore + id="lolx_default-2", + ), + param("DictOfDicts", "dsdsi", False, Dict[str, Dict[str, int]], id="dsdsi"), + param( + "DictOfDicts", + ["odsdsi_default", "dsi1"], + False, + Dict[str, int], + id="odsdsi_default-dsi1", + ), + param( + "DictOfDicts", + ["odsdsi_default", "dsi2"], + False, + Dict[str, int], + id="odsdsi_default-dsi2", + ), + param( + "DictOfDicts", + ["odsdsi_default", "dsi3"], + False, + Dict[str, int], + id="odsdsi_default-dsi3", + ), + param( + "DictOfDicts", + ["dsdsx_default", "dsx1"], + False, + lambda module: Dict[str, module.User], # type: ignore + id="dsdsx_default-dsx1", + ), + param( + "DictOfDicts", + ["dsdsx_default", "dsx2"], + False, + lambda module: Dict[str, module.User], # type: ignore + id="dsdsx_default-dsx2", + ), + param( + "DictOfDicts", + ["dsdsx_default", "dsx2", "s1"], + False, + lambda module: module.User, + id="dsdsx_default-dsx2-s1", + ), + param( + "DictOfDicts", + ["dsdsx_default", "dsx2", "s2"], + False, + lambda module: module.User, + id="dsdsx_default-dsx2-s2", + ), + param( + "DictOfDicts", + ["dsdsx_default", "dsx2", "s3"], + False, + lambda module: module.User, + id="dsdsx_default-dsx2-s3", + ), + param( + "DictOfDicts", + ["dsdsx_default", "dsx3"], + False, + lambda module: Dict[str, module.User], # type: ignore + id="dsdsx_default-dsx3", + ), + param( + "ListsAndDicts", "lldsi", False, List[List[Dict[str, int]]], id="lldsi" + ), + param( + "ListsAndDicts", + "oldfox", + True, + lambda module: List[Dict[float, Optional[module.User]]], # type: ignore + id="oldfox", + ), + param( + "ListsAndDicts", + "oldfox", + True, + lambda module: List[Dict[float, Optional[module.User]]], # type: ignore + id="oldfox", + ), + param( + "ListsAndDicts", + ["dedsle_default", Color.RED], + False, + Dict[str, List[Enum1]], + id="dedsle_default-RED", + ), + param( + "WithDefault", + "dsolx_default", + False, + lambda module: Dict[str, Optional[List[module.User]]], # type: ignore + id="dsolx_default", + ), + param( + "WithDefault", + ["dsolx_default", "lx"], + True, + lambda module: List[module.User], # type: ignore + id="dsolx_default-lx", + ), + param( + "WithDefault", + ["dsolx_default", "lx", 0], + False, + lambda module: module.User, + id="dsolx_default-lx-0", + ), + ], + ) + def test_metadata( + self, + module: Any, + class_name: str, + keys: Any, + expected_optional: bool, + expected_ref_type: Any, + ) -> None: + cls = getattr(module.NestedContainers, class_name) + node = OmegaConf.structured(cls) + + if not isinstance(keys, list): + keys = [keys] + for key in keys: + node = node._get_node(key) + + if inspect.isfunction(expected_ref_type): + expected_ref_type = expected_ref_type(module) + + assert node._metadata.optional == expected_optional + assert node._metadata.ref_type == expected_ref_type + + if _utils.is_dict_annotation(expected_ref_type): + expected_key_type, expected_element_type = _utils.get_dict_key_value_types( + expected_ref_type + ) + assert node._metadata.key_type == expected_key_type + assert node._metadata.element_type == expected_element_type + + if _utils.is_list_annotation(expected_ref_type): + expected_element_type = _utils.get_list_element_type(expected_ref_type) + assert node._metadata.element_type == expected_element_type + + @mark.parametrize( + "class_name, key, value", + [ + param("ListOfLists", "lls", [["a", "b"], ["c"]], id="lls"), + param("ListOfLists", "lls", [], id="lls-empty"), + param("ListOfLists", "lls", [[]], id="lls-list-of-empty"), + param( + "ListOfLists", + "lls_default", + [["a", "b"], ["c"]], + id="lls_default", + ), + param( + "ListOfLists", + "llx", + lambda module: [[module.User("Bond", 7)]], + id="llx", + ), + param( + "ListOfLists", + "lolx_default", + lambda module: [[module.User("Bond", 7)]], + id="lolx_default", + ), + param("DictOfDicts", "dsdsi", {"abc": {"xyz": 123}}, id="dsdsi"), + param("DictOfDicts", "dsdbi", {"abc": {True: 456}}, id="dsdbi"), + param( + "DictOfDicts", + "dsdsx", + lambda module: {"abc": {"xyz": module.User("Bond", 7)}}, + id="dsdsx", + ), + ], + ) + def test_legal_assignment( + self, module: Any, class_name: str, key: str, value: Any + ) -> None: + cls = getattr(module.NestedContainers, class_name) + cfg = OmegaConf.structured(cls) + if inspect.isfunction(value): + value = value(module) + + cfg[key] = value + + assert cfg[key] == value + + @mark.parametrize( + "class_name, key, value, expected", + [ + param( + "ListOfLists", + "lls", + [["123", 456]], + [["123", "456"]], + id="lls-conversion-from-int", + ), + param("ListOfLists", "lls", [["123", 456]], [["123", "456"]], id="lls"), + param("ListOfLists", "llla", [[["123", 456]]], [[["123", 456]]], id="llla"), + param("ListOfLists", "lloli", [[["123", 456]]], [[[123, 456]]], id="lloli"), + param( + "DictOfDicts", + "dsdbi", + {"abc": {True: "456"}}, + {"abc": {True: 456}}, + id="dsdbi", + ), + ], + ) + def test_assignment_conversion( + self, module: Any, class_name: str, key: str, value: Any, expected: Any + ) -> None: + cls = getattr(module.NestedContainers, class_name) + cfg = OmegaConf.structured(cls) + if inspect.isfunction(value): + value = value(module) + + cfg[key] = value + + assert cfg[key] == expected + + @mark.parametrize( + "class_name, key, value, err_type", + [ + param( + "ListOfLists", + "lloli", + [[["abc"]]], + ValidationError, + id="assign-llls-to-lloli", + ), + param( + "ListOfLists", + "llx", + [[{"name": "Bond", "age": 7}]], + ValidationError, + id="assign-lld-to-llx", + ), + param( + "DictOfDicts", + "dsdbi", + {123: {True: 456}}, + KeyValidationError, + id="assign-didbi-to-dsdbi", + ), + param( + "DictOfDicts", + "dsdbi", + {"abc": {"True": 456}}, + KeyValidationError, + id="assign-dsdsi-to-dsdbi", + ), + ], + ) + def test_illegal_assignment( + self, module: Any, class_name: str, key: str, value: Any, err_type: Any + ) -> None: + cls = getattr(module.NestedContainers, class_name) + cfg = OmegaConf.structured(cls) + + with raises(err_type): + cfg[key] = value + + @mark.parametrize( + "class_name, keys, expected", + [ + param("ListOfLists", ["lls"], MISSING, id="lls-missing"), + param("ListOfLists", ["lls_default", 0], [], id="lls_default-empty-list"), + param("ListOfLists", ["lls_default", 1, 0], "abc", id="lls_default-str"), + param( + "ListOfLists", + ["lls_default", 1, 2], + "123", + id="lls_default-int-converted", + ), + param( + "ListOfLists", + ["lls_default", 1, 3], + MISSING, + id="lls_default-missing-nested", + ), + param("ListOfLists", ["lls_default", 2], MISSING, id="lls_default-missing"), + param("DictOfDicts", ["dsdsi"], MISSING, id="dsdsi-missing"), + param( + "DictOfDicts", ["dsdsx_default", "dsx1"], {}, id="dsdsx_default-empty" + ), + param( + "DictOfDicts", + ["dsdsx_default", "dsx2", "s1"], + {"name": "???", "age": "???"}, + id="dsdsx_default-user-missing-data", + ), + param( + "DictOfDicts", + ["dsdsx_default", "dsx2", "s2"], + {"name": "Bond", "age": 7}, + id="dsdsx_default-user", + ), + param( + "DictOfDicts", + ["dsdsx_default", "dsx2", "s3"], + MISSING, + id="dsdsx_default-missing-user", + ), + param( + "DictOfDicts", + ["odsdsi_default", "dsi2", "s2"], + 123, + id="dsdsi-str-converted-to-int", + ), + ], + ) + def test_default_values( + self, module: Any, class_name: str, keys: Any, expected: Any + ) -> None: + cls = getattr(module.NestedContainers, class_name) + node = OmegaConf.structured(cls) + + if not isinstance(keys, list): + keys = [keys] + for key in keys: + node = node._get_node(key) + + if expected is MISSING: + assert node._is_missing() + else: + assert node == expected + + @mark.parametrize( + "class_name, keys, value, is_legal", + [ + param("WithDefault", "dsolx_default", None, False, id="dsolx=none-illegal"), + param( + "WithDefault", ["dsolx_default", "lx"], None, True, id="olx=none-legal" + ), + param( + "WithDefault", "dsolx_default", {"s": None}, True, id="dsolx=dn-legal" + ), + param( + "WithDefault", + ["dsolx_default", "lx", 0], + None, + False, + id="x=none-illegal", + ), + param("DictOfDicts", "odsdsi_default", None, True, id="odsdsi=none-legal"), + param("DictOfDicts", "dsdsx", None, False, id="dsdsx=none-illegal"), + param( + "DictOfDicts", + ["odsdsi_default", "dsi1"], + None, + False, + id="dsi=none-illegal", + ), + param("DictOfDicts", "dsdsx", {"s": None}, False, id="dsdsx=dsn-illegal"), + param("ListOfLists", "lloli", None, False, id="lloli=n-illegal"), + param("ListOfLists", "lloli", [None], False, id="lloli=ln-illegal"), + param("ListOfLists", "lloli", [[None]], True, id="lloli=lln-legal"), + param("ListOfLists", "lloli", [[[None]]], False, id="lloli=llln-illegal"), + param("ListOfLists", ["lolx_default"], None, False, id="lolx=n-illegal"), + param("ListOfLists", ["lolx_default", 1], None, True, id="olx=n-legal"), + param( + "ListOfLists", ["lolx_default", 1, 0], None, False, id="lx=n-illegal" + ), + ], + ) + def test_assign_none( + self, module: Any, class_name: str, keys: Any, value: Any, is_legal: bool + ) -> None: + cls = getattr(module.NestedContainers, class_name) + node = OmegaConf.structured(cls) + + if not isinstance(keys, list): + keys = [keys] + for key in keys[:-1]: + node = node[key] + last_key = keys[-1] + + if is_legal: + node[last_key] = value + assert node[last_key] == value + else: + with raises(ValidationError): + node[last_key] = value + + +class TestUnionsOfPrimitiveTypes: + @mark.parametrize( + "class_name, key, expected_type_hint, expected_val", + [ + param("Simple", "uis", Union[int, str], MISSING, id="simple-uis"), + param("Simple", "ubc", Union[bool, Color], MISSING, id="simple-ubc"), + param("Simple", "uxf", Union[bytes, float], MISSING, id="simple-uxf"), + param( + "Simple", "ouis", Optional[Union[int, str]], MISSING, id="simple-ouis" + ), + param("Simple", "uisn", Union[int, str, None], MISSING, id="simple-uisn"), + param( + "Simple", "uisN", Union[int, str, type(None)], MISSING, id="simple-uisN" + ), + param("WithDefaults", "uis", Union[int, str], "abc", id="defaults-uis"), + param("WithDefaults", "ubc1", Union[bool, Color], True, id="defaults-ubc1"), + param( + "WithDefaults", + "ubc2", + Union[bool, Color], + Color.RED, + id="defaults-ubc2", + ), + param("WithDefaults", "uxf", Union[bytes, float], 1.2, id="defaults-uxf"), + param( + "WithDefaults", + "ouis", + Optional[Union[int, str]], + None, + id="defaults-ouis", + ), + param( + "WithDefaults", "uisn", Union[int, str, None], 123, id="defaults-uisn" + ), + param( + "WithDefaults", + "uisN", + Union[int, str, type(None)], + "abc", + id="defaults-uisN", + ), + param( + "WithExplicitMissing", + "uis_missing", + Union[int, str], + MISSING, + id="uis_missing", + ), + param( + "ContainersOfUnions", + "lubc", + List[Union[bool, Color]], + MISSING, + id="lubc", + ), + param( + "ContainersOfUnions", + "dsubf", + Dict[str, Union[bool, float]], + MISSING, + id="dsubf", + ), + param( + "ContainersOfUnions", + "lubc_with_default", + List[Union[bool, Color]], + [True, Color.RED], + id="lubc_with_default", + ), + param( + "ContainersOfUnions", + "dsubf_with_default", + Dict[str, Union[bool, float]], + {"abc": True, "xyz": 1.2}, + id="dsubf_with_default", + ), + param( + "InterpolationFromUnion", + "ubi_with_default", + Union[bool, int], + "${an_int}", + id="iterp-from-union", + ), + param( + "InterpolationFromUnion", + "ubi_with_default", + Union[bool, int], + 123, + id="iterp-from-union-resolved", + ), + param( + "InterpolationToUnion", + "a_float", + float, + 10.1, + id="iterp-to-union-resolved", + ), + ], + ) + def test_union_instantiation( + self, + module: Any, + class_name: str, + key: str, + expected_type_hint: Any, + expected_val: Any, + ) -> None: + class_ = getattr(module.UnionsOfPrimitveTypes, class_name) + cfg = OmegaConf.structured(class_) + + assert _utils.get_type_hint(cfg, key) == expected_type_hint + + vk = _utils.get_value_kind(expected_val) + + if vk is _utils.ValueKind.VALUE: + assert cfg[key] == expected_val + if _utils.is_primitive_type_annotation(type(expected_val)): + assert type(cfg[key]) == type(expected_val) + else: + assert isinstance(cfg[key], Container) + + elif vk is _utils.ValueKind.MANDATORY_MISSING: + assert OmegaConf.is_missing(cfg, key) + assert cfg._get_node(key)._value() == expected_val + + elif vk is _utils.ValueKind.INTERPOLATION: + assert OmegaConf.is_interpolation(cfg, key) + assert cfg._get_node(key)._value() == expected_val + + @mark.parametrize( + "class_name, expected_err", + [ + param( + "WithBadDefaults1", + re.escape( + "Value 'None' is incompatible with type hint 'Union[int, str]" + ), + id="assign-none-to-uis", + ), + param( + "WithBadDefaults2", + re.escape( + "Value 'abc' of type 'str' is incompatible with type hint 'Union[bool, Color]'" + ), + id="assign-str-to-ubc", + ), + param( + "WithBadDefaults3", + re.escape( + "Value 'True' of type 'bool' is incompatible with type hint 'Union[bytes, float]'" + ), + id="assign-bool-to-uxf", + ), + param( + "WithBadDefaults4", + re.escape( + "Value 'Color.RED' of type 'tests.Color' is incompatible" + + " with type hint 'Optional[Union[" + ) + + r"(bool, float)|(float, bool)\]\]'", + id="assign-enum-to-oufb", + ), + ], + ) + def test_union_instantiation_with_bad_defaults( + self, module: Any, class_name: str, expected_err: str + ) -> None: + class_ = getattr(module.UnionsOfPrimitveTypes, class_name) + with raises(ValidationError, match=expected_err): + OmegaConf.structured(class_) + + @mark.parametrize( + "class_name, key, value, expected", + [ + param("Simple", "uis", 123, 123, id="simple-uis-int"), + param("Simple", "uis", "123", "123", id="simple-uis-int_string"), + param("Simple", "uis", "abc", "abc", id="simple-uis-str"), + param("Simple", "uis", None, raises(ValidationError), id="simple-uis-none"), + param("Simple", "uis", MISSING, MISSING, id="simple-uis-missing"), + param("Simple", "uis", "${interp}", "${interp}", id="simple-uis-interp"), + param("Simple", "ubc", True, True, id="simple-ubc-bool"), + param("Simple", "ubc", Color.RED, Color.RED, id="simple-ubc-color"), + param( + "Simple", + "ubc", + "RED", + raises(ValidationError), + id="simple-ubc-color_str", + ), + param( + "Simple", + "ubc", + "a_string", + raises(ValidationError), + id="simple-ubc-str", + ), + param("Simple", "ubc", None, raises(ValidationError), id="simple-ubc-none"), + param("Simple", "ubc", MISSING, MISSING, id="simple-ubc-missing"), + param("Simple", "ubc", "${interp}", "${interp}", id="simple-ubc-interp"), + param("Simple", "ouis", None, None, id="simple-ouis-none"), + param("WithDefaults", "uis", 123, 123, id="with_defaults-uis-int"), + param( + "WithDefaults", "uis", "123", "123", id="with_defaults-uis-int_string" + ), + param("WithDefaults", "uis", "abc", "abc", id="with_defaults-uis-str"), + param( + "WithDefaults", + "uis", + None, + raises(ValidationError), + id="with_defaults-uis-none", + ), + param( + "WithDefaults", "uis", MISSING, MISSING, id="with_defaults-uis-missing" + ), + param( + "WithDefaults", + "uis", + "${interp}", + "${interp}", + id="with_defaults-uis-interp", + ), + param("WithDefaults", "ubc1", True, True, id="with_defaults-ubc-bool"), + param( + "WithDefaults", + "ubc1", + Color.RED, + Color.RED, + id="with_defaults-ubc-color", + ), + param( + "WithDefaults", + "ubc1", + "RED", + raises(ValidationError), + id="with_defaults-ubc-color_str", + ), + param( + "WithDefaults", + "ubc1", + "a_string", + raises(ValidationError), + id="with_defaults-ubc-str", + ), + param( + "WithDefaults", + "ubc1", + None, + raises(ValidationError), + id="with_defaults-ubc-none", + ), + param( + "WithDefaults", "ubc1", MISSING, MISSING, id="with_defaults-ubc-missing" + ), + param( + "WithDefaults", + "ubc1", + "${interp}", + "${interp}", + id="with_defaults-ubc-interp", + ), + param("WithDefaults", "ouis", None, None, id="with_defaults-ouis-none"), + param("ContainersOfUnions", "lubc", MISSING, MISSING, id="lubc-missing"), + param( + "ContainersOfUnions", + "lubc", + None, + raises(ValidationError), + id="lubc-none", + ), + param( + "ContainersOfUnions", "lubc", "${interp}", "${interp}", id="lubc-interp" + ), + param("ContainersOfUnions", "lubc", [], [], id="lubc-list-empty"), + param( + "ContainersOfUnions", + "lubc", + [Color.GREEN], + [Color.GREEN], + id="lubc-list-enum", + ), + param( + "ContainersOfUnions", + "lubc", + ["GREEN"], + raises(ValidationError), + id="lubc-list-enum_str", + ), + param( + "ContainersOfUnions", + "lubc", + ["abc"], + raises(ValidationError), + id="lubc-list-str", + ), + param( + "ContainersOfUnions", + "lubc", + [None], + raises(ValidationError), + id="lubc-list-none", + ), + param( + "ContainersOfUnions", + "lubc", + [MISSING], + [MISSING], + id="lubc-list-missing", + ), + param( + "ContainersOfUnions", + "lubc", + ["${interp}"], + ["${interp}"], + id="lubc-list-interp", + ), + param( + "ContainersOfUnions", + "dsubf", + {"bool": True, "float": 10.1}, + {"bool": True, "float": 10.1}, + id="dsubf", + ), + param( + "ContainersOfUnions", + "dsubf", + {"float-str": "10.1"}, + raises(ValidationError), + id="dsubf-float-str", + ), + param( + "ContainersOfUnions", + "dsubf", + {"str": "abc"}, + raises(ValidationError), + id="dsubf-dict-str", + ), + param( + "ContainersOfUnions", + "dsubf", + {"none": None}, + raises(ValidationError), + id="dsubf-dict-none", + ), + param( + "ContainersOfUnions", + "dsubf", + {"missing": MISSING}, + {"missing": MISSING}, + id="dsubf-dict-missing", + ), + param( + "ContainersOfUnions", + "dsubf", + {"interp": "${interp}"}, + {"interp": "${interp}"}, + id="dsubf-dict-interp", + ), + param( + "ContainersOfUnions", + "dsoubf", + {"none": None}, + {"none": None}, + id="dsoubf-dict-none", + ), + ], + ) + def test_assignment_to_union( + self, module: Any, class_name: str, key: str, value: Any, expected: Any + ) -> None: + class_ = getattr(module.UnionsOfPrimitveTypes, class_name) + cfg = OmegaConf.structured(class_) + + if isinstance(expected, RaisesContext): + with expected: + cfg[key] = value + + else: + cfg[key] = value + + vk = _utils.get_value_kind(expected) + + if vk is _utils.ValueKind.VALUE: + assert cfg[key] == expected + + elif vk is _utils.ValueKind.MANDATORY_MISSING: + assert OmegaConf.is_missing(cfg, key) + assert _utils._get_value(cfg._get_node(key)) == expected + + elif vk is _utils.ValueKind.INTERPOLATION: + assert OmegaConf.is_interpolation(cfg, key) + assert _utils._get_value(cfg._get_node(key)) == expected + + @mark.parametrize( + "key, value, expected", + [ + param("ubi", "${an_int}", 123, id="interp-to-int"), + param( + "ubi", + "${none}", + raises(ValidationError), + id="interp-to-none-err", + marks=mark.xfail(reason="interpolations from unions are not validated"), + ), + param( + "ubi", + "${a_string}", + raises(ValidationError), + id="interp-to-str-err", + marks=mark.xfail(reason="interpolations from unions are not validated"), + ), + param( + "ubi", + "${missing}", + raises(InterpolationToMissingValueError), + id="interp-to-missing", + ), + param("oubi", "${none}", None, id="interp-to-none"), + ], + ) + @mark.parametrize("overwrite_default", [True, False]) + def test_interpolation_from_union( + self, module: Any, key: str, overwrite_default: bool, value: Any, expected: Any + ) -> None: + class_ = module.UnionsOfPrimitveTypes.InterpolationFromUnion + cfg = OmegaConf.structured(class_) + + if overwrite_default: + key += "_with_default" + + cfg[key] = value + + assert _utils._get_value(cfg._get_node(key)) == value + + if isinstance(expected, RaisesContext): + with expected: + cfg[key] + else: + assert cfg[key] == expected + + def test_resolve_union_interpolation(self, module: Any) -> None: + class_ = module.UnionsOfPrimitveTypes.InterpolationFromUnion + cfg = OmegaConf.structured(class_) + assert OmegaConf.is_interpolation(cfg, "ubi_with_default") + assert OmegaConf.is_interpolation(cfg, "oubi_with_default") + OmegaConf.resolve(cfg) + assert not OmegaConf.is_interpolation(cfg, "ubi_with_default") + assert not OmegaConf.is_interpolation(cfg, "oubi_with_default") + + def test_resolve_union_interpolation_error(self, module: Any) -> None: + class_ = module.UnionsOfPrimitveTypes.BadInterpolationFromUnion + cfg = OmegaConf.structured(class_) + assert OmegaConf.is_interpolation(cfg, "ubi") + with raises(ValidationError): + OmegaConf.resolve(cfg) + + @mark.parametrize( + "key, expected", + [ + param("a_float", 10.1, id="interp-to-float"), + param("bad_int_interp", raises(ValidationError), id="bad-int-interp"), + ], + ) + def test_interpolation_to_union(self, module: Any, key: str, expected: Any) -> None: + class_ = module.UnionsOfPrimitveTypes.InterpolationToUnion + cfg = OmegaConf.structured(class_) + + if isinstance(expected, RaisesContext): + with expected: + cfg[key] + else: + assert cfg[key] == expected + + @mark.skipif(sys.version_info < (3, 10), reason="requires Python 3.10 or newer") + def test_support_pep_604(self, module: Any) -> None: + class_ = module.UnionsOfPrimitveTypes.SupportPEP604 + cfg = OmegaConf.structured(class_) + assert _utils.get_type_hint(cfg, "uis") == Union[int, str] + assert _utils.get_type_hint(cfg, "ouis") == Optional[Union[int, str]] + assert _utils.get_type_hint(cfg, "uisn") == Optional[Union[int, str]] + assert _utils.get_type_hint(cfg, "uis_with_default") == Union[int, str] + assert cfg.uisn is None + assert cfg.uis_with_default == 123 + + def test_assign_path_to_string_typed_field(self, module: Any) -> None: + cfg = OmegaConf.create(module.StringConfig) + cfg.null_default = Path("hello.txt") + assert isinstance(cfg.null_default, str) + assert cfg.null_default == "hello.txt" diff -pruN 2.1.0~rc1-3/tests/test_base_config.py 2.2.2-1/tests/test_base_config.py --- 2.1.0~rc1-3/tests/test_base_config.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/tests/test_base_config.py 2022-05-27 21:36:40.000000000 +0000 @@ -1,5 +1,5 @@ import copy -from typing import Any, Dict, Union +from typing import Any, Dict, List, Optional, Union from pytest import mark, param, raises @@ -7,19 +7,32 @@ from omegaconf import ( AnyNode, Container, DictConfig, + DictKeyType, IntegerNode, ListConfig, OmegaConf, ReadonlyConfigError, StringNode, + UnionNode, ValidationError, flag_override, open_dict, read_write, ) -from omegaconf._utils import nullcontext +from omegaconf._utils import _ensure_container, nullcontext from omegaconf.errors import ConfigAttributeError, ConfigKeyError, MissingMandatoryValue -from tests import StructuredWithMissing +from tests import ( + ConcretePlugin, + Group, + OptionalUsers, + StructuredWithMissing, + SubscriptedDict, + SubscriptedDictOpt, + SubscriptedList, + SubscriptedListOpt, + User, + Users, +) @mark.parametrize( @@ -494,6 +507,12 @@ class TestParentAfterCopy: assert nc._get_parent() is cfg assert nc._get_node(0)._get_parent() is nc + def test_union_copy(self, copy_func: Any) -> None: + cfg = OmegaConf.create({"a": UnionNode(10.0, Union[float, bool])}) + nc = copy_func(cfg._get_node("a")) + assert nc._get_parent() is cfg + assert nc._value()._get_parent() is nc + def test_omegaconf_init_not_implemented() -> None: with raises(NotImplementedError): @@ -587,3 +606,135 @@ def test_flags_root() -> None: cfg.a._set_flags_root(True) assert cfg.a._get_flag_no_cache("flag") is None + + +@mark.parametrize( + "cls,key,assignment,error", + [ + param(SubscriptedList, "list", [None], True, id="list_elt"), + param(SubscriptedList, "list", [0, 1, None], True, id="list_elt_partial"), + param(SubscriptedDict, "dict_str", {"key": None}, True, id="dict_elt"), + param( + SubscriptedDict, + "dict_str", + {"key_valid": 123, "key_invalid": None}, + True, + id="dict_elt_partial", + ), + param(SubscriptedList, "list", None, True, id="list"), + param(SubscriptedDict, "dict_str", None, True, id="dict"), + param(SubscriptedListOpt, "opt_list", [None], True, id="opt_list_elt"), + param(SubscriptedDictOpt, "opt_dict", {"key": None}, True, id="opt_dict_elt"), + param(SubscriptedListOpt, "opt_list", None, False, id="opt_list"), + param(SubscriptedDictOpt, "opt_dict", None, False, id="opt_dict"), + param(SubscriptedListOpt, "list_opt", [None], False, id="list_opt_elt"), + param(SubscriptedDictOpt, "dict_opt", {"key": None}, False, id="dict_opt_elt"), + param(SubscriptedListOpt, "list_opt", None, True, id="list_opt"), + param(SubscriptedDictOpt, "dict_opt", None, True, id="dict_opt"), + param( + ListConfig([None], element_type=Optional[User]), + 0, + User("Bond", 7), + False, + id="set_optional_user", + ), + param( + ListConfig([User], element_type=User), + 0, + None, + True, + id="illegal_set_user_to_none", + ), + ], +) +def test_optional_assign(cls: Any, key: str, assignment: Any, error: bool) -> None: + cfg = OmegaConf.structured(cls) + if error: + with raises(ValidationError): + cfg[key] = assignment + else: + cfg[key] = assignment + assert cfg[key] == assignment + + +@mark.parametrize( + "src,keys,ref_type,is_optional", + [ + param(Group, ["admin"], User, True, id="opt_user"), + param( + ConcretePlugin, + ["params"], + ConcretePlugin.FoobarParams, + False, + id="nested_structured_conf", + ), + param( + OmegaConf.structured(Users({"user007": User("Bond", 7)})).name2user, + ["user007"], + User, + False, + id="structured_dict_of_user", + ), + param( + DictConfig({"a": 123}, element_type=int), ["a"], int, False, id="dict_int" + ), + param( + DictConfig({"a": 123}, element_type=Optional[int]), + ["a"], + int, + True, + id="dict_opt_int", + ), + param(DictConfig({"a": 123}), ["a"], Any, True, id="dict_any"), + param( + OmegaConf.merge(Users, {"name2user": {"joe": User("joe")}}), + ["name2user", "joe"], + User, + False, + id="dict:merge_into_new_user_node", + ), + param( + OmegaConf.merge(OptionalUsers, {"name2user": {"joe": User("joe")}}), + ["name2user", "joe"], + User, + True, + id="dict:merge_into_new_optional_user_node", + ), + param( + OmegaConf.merge(ListConfig([], element_type=User), [User(name="joe")]), + [0], + User, + False, + id="list:merge_into_new_user_node", + ), + param( + OmegaConf.merge( + ListConfig([], element_type=Optional[User]), [User(name="joe")] + ), + [0], + User, + True, + id="list:merge_into_new_optional_user_node", + ), + param(SubscriptedDictOpt, ["opt_dict"], Dict[str, int], True, id="opt_dict"), + param(SubscriptedListOpt, ["opt_list"], List[int], True, id="opt_list"), + param( + SubscriptedDictOpt, + ["dict_opt"], + Dict[str, Optional[int]], + False, + id="opt_dict", + ), + param( + SubscriptedListOpt, ["list_opt"], List[Optional[int]], False, id="opt_dict" + ), + ], +) +def test_assignment_optional_behavior( + src: Any, keys: List[DictKeyType], ref_type: Any, is_optional: bool +) -> None: + cfg = _ensure_container(src) + for k in keys: + cfg = cfg._get_node(k) + assert cfg._is_optional() == is_optional + assert cfg._metadata.ref_type == ref_type diff -pruN 2.1.0~rc1-3/tests/test_basic_ops_dict.py 2.2.2-1/tests/test_basic_ops_dict.py --- 2.1.0~rc1-3/tests/test_basic_ops_dict.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/tests/test_basic_ops_dict.py 2022-05-27 21:36:40.000000000 +0000 @@ -8,6 +8,7 @@ from pytest import mark, param, raises from omegaconf import ( MISSING, + AnyNode, DictConfig, DictKeyType, ListConfig, @@ -35,6 +36,7 @@ from tests import ( IllegalType, Plugin, StructuredWithMissing, + SubscriptedDict, User, ) @@ -96,6 +98,7 @@ def test_delattr(cfg: Any, struct: bool) "key,match", [ param("a", "a", id="str"), + param(b"binary", "binary", id="bytes"), param(1, "1", id="int"), param(123.45, "123.45", id="float"), param(True, "True", id="bool-T"), @@ -147,6 +150,7 @@ class TestDictKeyTypes: "src,key,expected", [ ({"a": 10, "b": 11}, "a", {"b": 11}), + ({b"abc": 10, b"def": 11}, b"abc", {b"def": 11}), ({1: "a", 2: "b"}, 1, {2: "b"}), ({123.45: "a", 67.89: "b"}, 67.89, {123.45: "a"}), ({True: "a", False: "b"}, False, {True: "a"}), @@ -460,6 +464,21 @@ def test_iterate_dict_with_interpolation "default", id="enum_key_with_default", ), + # bytes keys + param( + {b"123": "a", b"42": "b"}, + b"42", + "__NO_DEFAULT__", + "b", + id="bytes_key_no_default", + ), + param( + {b"123": "a", b"42": "b"}, + "not found", + None, + None, + id="bytes_key_with_default", + ), # other key types param( {123.45: "a", 67.89: "b"}, @@ -539,6 +558,7 @@ def test_dict_structured_mode_pop() -> N [ # key not found ({"a": 1, "b": 2}, "not_found", raises(KeyError)), + ({b"abc": 1, b"def": 2}, b"ghi", raises(KeyError)), ({1: "a", 2: "b"}, 3, raises(KeyError)), ({123.45: "a", 67.89: "b"}, 10.11, raises(KeyError)), ({True: "a"}, False, raises(KeyError)), @@ -629,6 +649,13 @@ def test_dict_pop_error(cfg: Dict[Any, A ({True: "a", False: {}}, 1, True), ({True: "a", False: {}}, None, False), ({True: "a", False: "???"}, False, False), + # bytes key type + ({b"1": "a", b"2": {}}, b"1", True), + ({b"1": "a", b"2": {}}, b"2", True), + ({b"1": "a", b"2": {}}, b"3", False), + ({b"1": "a", b"2": "???"}, b"2", False), + ({b"1": "a", b"2": "???"}, None, False), + ({b"1": "a", b"2": "???"}, "1", False), ], ) def test_in_dict(conf: Any, key: str, expected: Any) -> None: @@ -870,6 +897,13 @@ def test_hasattr() -> None: assert not hasattr(cfg, "buz") +def test_typed_hasattr() -> None: + cfg = OmegaConf.structured(SubscriptedDict) + assert hasattr(cfg.dict_enum, "foo") is False + with raises(AttributeError): + cfg.dict_int.foo + + def test_struct_mode_missing_key_getitem() -> None: cfg = OmegaConf.create({"foo": "bar"}) OmegaConf.set_struct(cfg, True) @@ -919,7 +953,7 @@ def test_get_type() -> None: ], ) def test_get_ref_type(cfg: Any, expected_ref_type: Any) -> None: - assert _utils.get_ref_type(cfg.plugin) == expected_ref_type + assert _utils.get_type_hint(cfg.plugin) == expected_ref_type def test_get_ref_type_with_conflict() -> None: @@ -928,11 +962,11 @@ def test_get_ref_type_with_conflict() -> ) assert OmegaConf.get_type(cfg.user) == User - assert _utils.get_ref_type(cfg.user) == Any + assert _utils.get_type_hint(cfg.user) == Any # Interpolation inherits both type and ref type from the target assert OmegaConf.get_type(cfg.inter) == User - assert _utils.get_ref_type(cfg.inter) == Any + assert _utils.get_type_hint(cfg.inter) == Any def test_is_missing() -> None: @@ -963,12 +997,12 @@ def test_assign_to_reftype_none_or_any(r @mark.parametrize( "ref_type,assign", [ - (Plugin, None), - (Plugin, Plugin), - (Plugin, Plugin()), - (Plugin, ConcretePlugin), - (Plugin, ConcretePlugin()), - (ConcretePlugin, None), + param(Plugin, None, id="plugin_none"), + param(Plugin, Plugin, id="plugin_plugin"), + param(Plugin, Plugin(), id="plugin_plugin()"), + param(Plugin, ConcretePlugin, id="plugin_concrete"), + param(Plugin, ConcretePlugin(), id="plugin_concrete()"), + param(ConcretePlugin, None, id="concrete_none"), param(ConcretePlugin, ConcretePlugin, id="subclass=subclass_obj"), param(ConcretePlugin, ConcretePlugin(), id="subclass=subclass_obj"), ], @@ -976,17 +1010,17 @@ def test_assign_to_reftype_none_or_any(r class TestAssignAndMergeIntoReftypePlugin: def _test_assign(self, ref_type: Any, value: Any, assign: Any) -> None: cfg = OmegaConf.create({"foo": DictConfig(ref_type=ref_type, content=value)}) - assert _utils.get_ref_type(cfg, "foo") == Optional[ref_type] + assert _utils.get_type_hint(cfg, "foo") == Optional[ref_type] cfg.foo = assign assert cfg.foo == assign - assert _utils.get_ref_type(cfg, "foo") == Optional[ref_type] + assert _utils.get_type_hint(cfg, "foo") == Optional[ref_type] def _test_merge(self, ref_type: Any, value: Any, assign: Any) -> None: cfg = OmegaConf.create({"foo": DictConfig(ref_type=ref_type, content=value)}) cfg2 = OmegaConf.merge(cfg, {"foo": assign}) assert isinstance(cfg2, DictConfig) assert cfg2.foo == assign - assert _utils.get_ref_type(cfg2, "foo") == Optional[ref_type] + assert _utils.get_type_hint(cfg2, "foo") == Optional[ref_type] def test_assign_to_reftype_plugin1(self, ref_type: Any, assign: Any) -> None: self._test_assign(ref_type, ref_type, assign) @@ -1133,3 +1167,17 @@ def test_dictconfig_creation_with_parent parent._set_flag(flag, True) cfg = DictConfig(data, parent=parent) assert cfg == data + + +@mark.parametrize( + "node", + [ + param(AnyNode("hello"), id="any"), + param(DictConfig({}), id="dict"), + param(ListConfig([]), id="list"), + ], +) +def test_node_copy_on_set(node: Any) -> None: + cfg = OmegaConf.create({}) + cfg.a = node + assert cfg.__dict__["_content"]["a"] is not node diff -pruN 2.1.0~rc1-3/tests/test_basic_ops_list.py 2.2.2-1/tests/test_basic_ops_list.py --- 2.1.0~rc1-3/tests/test_basic_ops_list.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/tests/test_basic_ops_list.py 2022-05-27 21:36:40.000000000 +0000 @@ -1,12 +1,15 @@ # -*- coding: utf-8 -*- import re +from pathlib import Path from textwrap import dedent -from typing import Any, List, Optional +from typing import Any, Callable, List, MutableSequence, Optional, Union +from _pytest.python_api import RaisesContext from pytest import mark, param, raises from omegaconf import MISSING, AnyNode, DictConfig, ListConfig, OmegaConf, flag_override -from omegaconf._utils import nullcontext +from omegaconf._utils import _ensure_container, nullcontext +from omegaconf.base import Node from omegaconf.errors import ( ConfigTypeError, InterpolationKeyError, @@ -324,67 +327,52 @@ def test_list_append() -> None: @mark.parametrize( - "lc,element,expected", + "lc,element,err", [ param( ListConfig(content=[], element_type=int), "foo", - raises( - ValidationError, - match=re.escape("Value 'foo' could not be converted to Integer"), - ), + "Value 'foo' of type 'str' could not be converted to Integer", id="append_str_to_list[int]", ), param( ListConfig(content=[], element_type=Color), "foo", - raises( - ValidationError, - match=re.escape( - "Invalid value 'foo', expected one of [RED, GREEN, BLUE]" - ), - ), + "Invalid value 'foo', expected one of [RED, GREEN, BLUE]", id="append_str_to_list[Color]", ), param( ListConfig(content=[], element_type=User), "foo", - raises( - ValidationError, - match=re.escape( - "Invalid type assigned: str is not a subclass of User. value: foo" - ), - ), + "Invalid type assigned: str is not a subclass of User. value: foo", id="append_str_to_list[User]", ), param( ListConfig(content=[], element_type=User), {"name": "Bond", "age": 7}, - raises( - ValidationError, - match=re.escape( - "Invalid type assigned: dict is not a subclass of User. value: {'name': 'Bond', 'age': 7}" - ), - ), + "Invalid type assigned: dict is not a subclass of User. value: {'name': 'Bond', 'age': 7}", id="list:convert_dict_to_user", ), param( ListConfig(content=[], element_type=User), {}, - raises( - ValidationError, - match=re.escape( - "Invalid type assigned: dict is not a subclass of User. value: {}" - ), - ), + "Invalid type assigned: dict is not a subclass of User. value: {}", id="list:convert_empty_dict_to_user", ), + param( + ListConfig(content=[], element_type=List[int]), + 123, + "Invalid value assigned: int is not a ListConfig, list or tuple.", + ), + param( + ListConfig(content=[], element_type=List[int]), + None, + "Invalid type assigned: NoneType is not a subclass of List[int]", + ), ], ) -def test_append_invalid_element_type( - lc: ListConfig, element: Any, expected: Any -) -> None: - with expected: +def test_append_invalid_element_type(lc: ListConfig, element: Any, err: Any) -> None: + with raises(ValidationError, match=re.escape(err)): lc.append(element) @@ -404,10 +392,28 @@ def test_append_invalid_element_type( id="list:convert_str_to_float", ), param( + ListConfig(content=[], element_type=str), + 10, + "10", + id="list:convert_int_to_str", + ), + param( + ListConfig(content=[], element_type=bool), + "yes", + True, + id="list:convert_str_to_bool", + ), + param( ListConfig(content=[], element_type=Color), "RED", Color.RED, - id="list:convert_str_to_float", + id="list:convert_str_to_enum", + ), + param( + ListConfig(content=[], element_type=Path), + "hello.txt", + Path("hello.txt"), + id="list:convert_str_to_path", ), ], ) @@ -445,6 +451,86 @@ def validate_list_keys(c: Any) -> None: @mark.parametrize( + "cfg, value, expected, expected_ref_type", + [ + param( + ListConfig(element_type=int, content=[]), + 123, + [123], + int, + id="typed_list", + ), + param( + ListConfig(element_type=int, content=[]), + None, + ValidationError, + None, + id="typed_list_append_none", + ), + param( + ListConfig(element_type=Optional[int], content=[]), + 123, + [123], + int, + id="optional_typed_list", + ), + param( + ListConfig(element_type=Optional[int], content=[]), + None, + [None], + int, + id="optional_typed_list_append_none", + ), + param( + ListConfig(element_type=User, content=[]), + User(name="bond"), + [User(name="bond")], + User, + id="user_list", + ), + param( + ListConfig(element_type=User, content=[]), + None, + ValidationError, + None, + id="user_list_append_none", + ), + param( + ListConfig(element_type=Optional[User], content=[]), + User(name="bond"), + [User(name="bond")], + User, + id="optional_user_list", + ), + param( + ListConfig(element_type=Optional[User], content=[]), + None, + [None], + User, + id="optional_user_list_append_none", + ), + ], +) +def test_append_to_typed( + cfg: ListConfig, + value: Any, + expected: Any, + expected_ref_type: type, +) -> None: + cfg = _ensure_container(cfg) + if isinstance(expected, type): + with raises(expected): + cfg.append(value) + else: + cfg.append(value) + assert cfg == expected + node = cfg._get_node(-1) + assert isinstance(node, Node) + assert node._metadata.ref_type == expected_ref_type + validate_list_keys(cfg) + + +@mark.parametrize( "input_, index, value, expected, expected_node_type, expectation", [ (["a", "b", "c"], 1, 100, ["a", 100, "b", "c"], AnyNode, None), @@ -473,6 +559,24 @@ def validate_list_keys(c: Any) -> None: None, ValidationError, ), + param( + ListConfig(element_type=int, content=[]), + 0, + 123, + [123], + IntegerNode, + None, + id="typed_list", + ), + param( + ListConfig(element_type=int, content=[]), + 0, + None, + None, + None, + ValidationError, + id="typed_list_insert_none", + ), ], ) def test_insert( @@ -639,11 +743,28 @@ def test_hash() -> None: ], ) class TestListAdd: + @mark.parametrize( + "left_listconfig, right_listconfig", + [ + param(True, True, id="listconfig_plus_listconfig"), + param(True, False, id="listconfig_plus_list"), + param(False, True, id="list_plus_listconfig"), + ], + ) def test_list_plus( - self, in_list1: List[Any], in_list2: List[Any], in_expected: List[Any] + self, + in_list1: List[Any], + in_list2: List[Any], + in_expected: List[Any], + left_listconfig: bool, + right_listconfig: bool, ) -> None: - list1 = OmegaConf.create(in_list1) - list2 = OmegaConf.create(in_list2) + list1: Union[List[Any], ListConfig] = ( + OmegaConf.create(in_list1) if left_listconfig else in_list1 + ) + list2: Union[List[Any], ListConfig] = ( + OmegaConf.create(in_list2) if right_listconfig else in_list2 + ) expected = OmegaConf.create(in_expected) ret = list1 + list2 assert ret == expected @@ -664,6 +785,12 @@ def test_deep_add() -> None: assert lst == [1, 2, "xx", 10, 20] +def test_deep_radd() -> None: + cfg = OmegaConf.create({"foo": [1, 2, "${bar}"], "bar": "xx"}) + lst = [10, 20] + cfg.foo + assert lst == [10, 20, 1, 2, "xx"] + + def test_set_with_invalid_key() -> None: cfg = OmegaConf.create([1, 2, 3]) with raises(KeyValidationError): @@ -710,6 +837,143 @@ def test_getitem_slice(sli: slice) -> No @mark.parametrize( + "constructor", + [OmegaConf.create, list, lambda lst: OmegaConf.create({"foo": lst}).foo], +) +@mark.parametrize( + "lst, idx, value, expected", + [ + param( + ["a", "b", "c", "d"], + slice(1, 3), + ["x", "y"], + ["a", "x", "y", "d"], + id="same-number-of-elements", + ), + param( + ["a", "x", "y", "d"], + slice(1, 3), + ["x", "y", "z"], + ["a", "x", "y", "z", "d"], + id="extra-elements", + ), + param( + ["a", "x", "y", "z", "d"], + slice(1, 1), + ["b"], + ["a", "b", "x", "y", "z", "d"], + id="insert only", + ), + param( + ["a", "b", "x", "y", "z", "d"], + slice(1, 1), + [], + ["a", "b", "x", "y", "z", "d"], + id="nop", + ), + param( + ["a", "b", "x", "y", "z", "d"], + slice(1, 3), + [], + ["a", "y", "z", "d"], + id="less-elements", + ), + param( + ["a", "y", "z", "d"], + slice(1, 2, 1), + ["b"], + ["a", "b", "z", "d"], + id="extended-slice", + ), + param( + ["a", "b", "c", "d"], + slice(1, 3, 1), + ["x", "y"], + ["a", "x", "y", "d"], + id="extended-slice2", + ), + param( + ["a", "b", "z", "d"], + slice(0, 3, 2), + ["a", "c"], + ["a", "b", "c", "d"], + id="extended-slice-disjoint", + ), + param( + ["a", "b", "c", "d"], + slice(1, 3), + 1, + raises(TypeError), + id="non-iterable-input", + ), + param( + ["a", "b", "c", "d"], + slice(1, 3, 1), + ["x", "y", "z"], + ["a", "x", "y", "z", "d"], + id="extended-slice-length-mismatch", + ), + param( + ["a", "b", "c", "d", "e", "f"], + slice(1, 5, 2), + ["x", "y", "z"], + raises(ValueError), + id="extended-slice-length-mismatch2", + ), + param( + ["a", "b", "c", "d", "e", "f"], + slice(-1, -3, -1), + ["F", "E"], + ["a", "b", "c", "d", "E", "F"], + id="extended-slice-reverse", + ), + param( + ["a", "b", "c", "d", "e", "g"], + slice(-1, -3, None), + ["f"], + ["a", "b", "c", "d", "e", "f", "g"], + id="slice-reverse-insert", + ), + param( + ["a", "b", "c", "r", "r", "e"], + slice(-3, -1, None), + ["d"], + ["a", "b", "c", "d", "e"], + id="slice-reverse-replace", + ), + param( + ["c", "d"], + slice(-10, -10, None), + ["a", "b"], + ["a", "b", "c", "d"], + id="slice-reverse-insert-underflow", + ), + param( + ["a", "b"], + slice(10, 10, None), + ["c", "d"], + ["a", "b", "c", "d"], + id="slice-reverse-insert-overflow", + ), + ], +) +def test_setitem_slice( + lst: List[Any], + idx: slice, + value: Union[List[Any], Any], + expected: Union[List[Any], RaisesContext[Any]], + constructor: Callable[[List[Any]], MutableSequence[Any]], +) -> None: + cfg = constructor(lst) + if isinstance(expected, list): + cfg[idx] = value + assert cfg == expected + else: + with expected: + cfg[idx] = value + + +@mark.parametrize( "lst,idx,expected", [ (OmegaConf.create([1, 2]), 0, 1), @@ -770,3 +1034,51 @@ def test_listconfig_creation_with_parent d = [1, 2, 3] cfg = ListConfig(d, parent=parent) assert cfg == d + + +@mark.parametrize( + "node", + [ + param(AnyNode("hello"), id="any"), + param(DictConfig({}), id="dict"), + param(ListConfig([]), id="list"), + ], +) +def test_node_copy_on_append(node: Any) -> None: + cfg = OmegaConf.create([]) + cfg.append(node) + assert cfg.__dict__["_content"][0] is not node + + +@mark.parametrize( + "cfg,key,value,error", + [ + param( + ListConfig([], element_type=Optional[User]), + 0, + "foo", + True, + id="structured:set_optional_to_bad_type", + ), + param( + ListConfig([], element_type=int), + 0, + None, + True, + id="set_to_none_raises", + ), + param( + ListConfig([], element_type=Optional[int]), + 0, + None, + False, + id="optional_set_to_none", + ), + ], +) +def test_validate_set(cfg: ListConfig, key: int, value: Any, error: bool) -> None: + if error: + with raises(ValidationError): + cfg._validate_set(key, value) + else: + cfg._validate_set(key, value) diff -pruN 2.1.0~rc1-3/tests/test_compare_dictconfig_vs_dict.py 2.2.2-1/tests/test_compare_dictconfig_vs_dict.py --- 2.1.0~rc1-3/tests/test_compare_dictconfig_vs_dict.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/tests/test_compare_dictconfig_vs_dict.py 2022-05-27 21:36:40.000000000 +0000 @@ -32,6 +32,7 @@ from tests import Enum1 @fixture( params=[ "str", + b"abc", 1, 3.1415, True, @@ -59,6 +60,7 @@ def struct_mode(request: Any) -> Optiona "data", [ param({"a": 10}, id="str"), + param({b"abc": 10}, id="bytes"), param({1: "a"}, id="int"), param({123.45: "a"}, id="float"), param({True: "a"}, id="bool"), @@ -207,7 +209,13 @@ def cfg_typed( @mark.parametrize( "cfg_key_type,data", - [(str, {"a": 10}), (int, {1: "a"}), (float, {123.45: "a"}), (bool, {True: "a"})], + [ + (str, {"a": 10}), + (bytes, {b"abc": "a"}), + (int, {1: "a"}), + (float, {123.45: "a"}), + (bool, {True: "a"}), + ], ) class TestPrimitiveTypeDunderMethods: """Compare DictConfig with python dict in the case where key_type is a primitive type.""" diff -pruN 2.1.0~rc1-3/tests/test_config_eq.py 2.2.2-1/tests/test_config_eq.py --- 2.1.0~rc1-3/tests/test_config_eq.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/tests/test_config_eq.py 2022-05-27 21:36:40.000000000 +0000 @@ -100,6 +100,20 @@ def test_eq(i1: Any, i2: Any) -> None: @mark.parametrize( + "cfg,other", + [ + param(DictConfig("???"), "???", id="missing_dictconfig"), + param(ListConfig("???"), "???", id="missing_listconfig"), + ], +) +def test_missing_container_string_eq(cfg: Any, other: Any) -> None: + assert cfg == other + assert other == cfg + assert not (cfg != other) + assert not (other != cfg) + + +@mark.parametrize( "input1, input2", [ # Dicts diff -pruN 2.1.0~rc1-3/tests/test_create.py 2.2.2-1/tests/test_create.py --- 2.1.0~rc1-3/tests/test_create.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/tests/test_create.py 2022-05-27 21:36:40.000000000 +0000 @@ -1,6 +1,9 @@ """Testing for OmegaConf""" +import platform import re import sys +from collections.abc import Sequence +from pathlib import Path from textwrap import dedent from typing import Any, Dict, List, Optional @@ -8,8 +11,18 @@ import yaml from pytest import mark, param, raises from omegaconf import DictConfig, ListConfig, OmegaConf -from omegaconf.errors import UnsupportedValueType -from tests import ConcretePlugin, IllegalType, NonCopyableIllegalType, Plugin +from omegaconf.errors import UnsupportedValueType, ValidationError +from tests import ( + ConcretePlugin, + DictOfAny, + DictSubclass, + IllegalType, + ListOfAny, + ListSubclass, + NonCopyableIllegalType, + Plugin, + Shape, +) @mark.parametrize( @@ -47,6 +60,7 @@ from tests import ConcretePlugin, Illega (OmegaConf.create([]), []), (OmegaConf.create({"foo": OmegaConf.create([])}), {"foo": []}), (OmegaConf.create([OmegaConf.create({})]), [{}]), + (OmegaConf.create({"foo": Path("bar")}), {"foo": Path("bar")}), ], ) def test_create_value(input_: Any, expected: Any) -> None: @@ -112,6 +126,48 @@ def test_create_allow_objects_non_copyab @mark.parametrize( "input_", [ + param(Shape(10, 2, 3), id="shape"), + param(ListSubclass((1, 2, 3)), id="list_subclass"), + param(DictSubclass({"key": "value"}), id="dict_subclass"), + ], +) +class TestCreationWithCustomClass: + def test_top_level(self, input_: Any) -> None: + if isinstance(input_, Sequence): + cfg = OmegaConf.create(input_) # type: ignore + assert isinstance(cfg, ListConfig) + else: + with raises(ValidationError): + OmegaConf.create(input_) + + def test_nested(self, input_: Any) -> None: + with raises(UnsupportedValueType): + OmegaConf.create({"foo": input_}) + + def test_nested_allow_objects(self, input_: Any) -> None: + cfg = OmegaConf.create({"foo": input_}, flags={"allow_objects": True}) + assert isinstance(cfg.foo, type(input_)) + + def test_structured_conf(self, input_: Any) -> None: + if isinstance(input_, Sequence): + cfg = OmegaConf.structured(ListOfAny(input_)) # type: ignore + assert isinstance(cfg.list, ListConfig) + else: + cfg = OmegaConf.structured(DictOfAny(input_)) + assert isinstance(cfg.dict, DictConfig) + + def test_direct_creation_of_listconfig_or_dictconfig(self, input_: Any) -> None: + if isinstance(input_, Sequence): + cfg = ListConfig(input_) # type: ignore + assert isinstance(cfg, ListConfig) + else: + cfg = DictConfig(input_) # type: ignore + assert isinstance(cfg, DictConfig) + + +@mark.parametrize( + "input_", + [ param({"foo": "bar"}, id="dict"), param([1, 2, 3], id="list"), ], @@ -282,17 +338,17 @@ def test_create_unmodified_loader() -> N def test_create_untyped_list() -> None: - from omegaconf._utils import get_ref_type + from omegaconf._utils import get_type_hint cfg = ListConfig(ref_type=List, content=[]) - assert get_ref_type(cfg) == Optional[List] + assert get_type_hint(cfg) == Optional[List] def test_create_untyped_dict() -> None: - from omegaconf._utils import get_ref_type + from omegaconf._utils import get_type_hint cfg = DictConfig(ref_type=Dict, content={}) - assert get_ref_type(cfg) == Optional[Dict] + assert get_type_hint(cfg) == Optional[Dict] @mark.parametrize( @@ -300,19 +356,19 @@ def test_create_untyped_dict() -> None: [ dedent( """\ - a: - b: 1 - c: 2 - b: 3 - """ + a: + b: 1 + c: 2 + b: 3 + """ ), dedent( """\ - a: - b: 1 - a: - b: 2 - """ + a: + b: 1 + a: + b: 2 + """ ), ], ) @@ -325,22 +381,53 @@ def test_yaml_merge() -> None: cfg = OmegaConf.create( dedent( """\ - a: &A - x: 1 - b: &B - y: 2 - c: - <<: *A - <<: *B - x: 3 - z: 1 - """ + a: &A + x: 1 + b: &B + y: 2 + c: + <<: *A + <<: *B + x: 3 + z: 1 + """ ) ) assert cfg == {"a": {"x": 1}, "b": {"y": 2}, "c": {"x": 3, "y": 2, "z": 1}} @mark.parametrize( + "path_type", + [ + param("Path", id="path"), + param( + "PosixPath", + marks=mark.skipif( + platform.system() == "Windows", reason="requires posix path support" + ), + id="posixpath", + ), + param( + "WindowsPath", + marks=mark.skipif( + platform.system() != "Windows", reason="requires windows" + ), + id="windowspath", + ), + ], +) +def test_create_path(path_type: str) -> None: + yaml_document = dedent( + """\ + foo: !!python/object/apply:pathlib.{} + - hello.txt + """ + ) + yaml_document = yaml_document.format(path_type) + assert OmegaConf.create(yaml_document) == yaml.unsafe_load(yaml_document) + + +@mark.parametrize( "data", [ param("", id="empty"), diff -pruN 2.1.0~rc1-3/tests/test_errors.py 2.2.2-1/tests/test_errors.py --- 2.1.0~rc1-3/tests/test_errors.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/tests/test_errors.py 2022-05-27 21:36:40.000000000 +0000 @@ -1,18 +1,24 @@ import re from dataclasses import dataclass from enum import Enum +from pathlib import Path from textwrap import dedent -from typing import Any, Dict, List, Optional, Type +from typing import Any, Dict, List, Optional, Type, Union from pytest import mark, param, raises import tests from omegaconf import ( + BytesNode, DictConfig, + FloatNode, IntegerNode, ListConfig, OmegaConf, + PathNode, ReadonlyConfigError, + StringNode, + UnionNode, UnsupportedValueType, ValidationError, ) @@ -26,6 +32,7 @@ from omegaconf.errors import ( InterpolationKeyError, InterpolationResolutionError, InterpolationToMissingValueError, + InterpolationValidationError, KeyValidationError, MissingMandatoryValue, OmegaConfBaseException, @@ -36,9 +43,12 @@ from tests import ( ConcretePlugin, IllegalType, Module, + NestedInterpolationToMissing, Package, Plugin, Str2Int, + StructuredInterpolationKeyError, + StructuredInterpolationValidationError, StructuredWithBadDict, StructuredWithBadList, StructuredWithMissing, @@ -64,6 +74,7 @@ class NotOptionalA: class Expected: exception_type: Type[Exception] msg: str + msg_is_regex: bool = False # "Low level exceptions" are thrown from internal APIs are are not populating all the fields low_level: bool = False @@ -97,7 +108,7 @@ class Expected: if self.key is None: self.full_key = "" else: - if isinstance(self.key, (str, int, Enum, float, bool, slice)): + if isinstance(self.key, (str, bytes, int, Enum, float, bool, slice)): self.full_key = self.key else: self.full_key = "" @@ -113,7 +124,7 @@ params = [ create=lambda: OmegaConf.structured(StructuredWithMissing), op=lambda cfg: OmegaConf.update(cfg, "num", "hello"), exception_type=ValidationError, - msg="Value 'hello' could not be converted to Integer", + msg="Value 'hello' of type 'str' could not be converted to Integer", parent_node=lambda cfg: cfg, child_node=lambda cfg: cfg._get_node("num"), object_type=StructuredWithMissing, @@ -126,7 +137,7 @@ params = [ create=lambda: OmegaConf.structured(StructuredWithMissing), op=lambda cfg: OmegaConf.update(cfg, "num", None), exception_type=ValidationError, - msg="child 'num' is not Optional", + msg="field 'num' is not Optional", parent_node=lambda cfg: cfg, child_node=lambda cfg: cfg._get_node("num"), object_type=StructuredWithMissing, @@ -309,6 +320,21 @@ params = [ ), id="dict,accessing_missing_nested_interpolation", ), + param( + Expected( + create=lambda: OmegaConf.structured(StructuredInterpolationValidationError), + op=lambda cfg: getattr(cfg, "y"), + exception_type=InterpolationValidationError, + object_type=StructuredInterpolationValidationError, + msg=( + "While dereferencing interpolation '${.x}': " + "Incompatible value 'None' for field of type 'int'" + ), + key="y", + child_node=lambda cfg: cfg._get_node("y"), + ), + id="dict,non_optional_field_with_interpolation_to_none", + ), # setattr param( Expected( @@ -350,7 +376,7 @@ params = [ ), op=lambda cfg: setattr(cfg, "foo", None), exception_type=ValidationError, - msg="child 'foo' is not Optional", + msg="field 'foo' is not Optional", key="foo", full_key="foo", child_node=lambda cfg: cfg.foo, @@ -362,7 +388,7 @@ params = [ create=lambda: OmegaConf.structured(ConcretePlugin), op=lambda cfg: cfg.params.__setattr__("foo", "bar"), exception_type=ValidationError, - msg="Value 'bar' could not be converted to Integer", + msg="Value 'bar' of type 'str' could not be converted to Integer", key="foo", full_key="params.foo", object_type=ConcretePlugin.FoobarParams, @@ -484,7 +510,7 @@ params = [ create=lambda: OmegaConf.structured(ConcretePlugin), op=lambda cfg: OmegaConf.merge(cfg, {"params": {"foo": "bar"}}), exception_type=ValidationError, - msg="Value 'bar' could not be converted to Integer", + msg="Value 'bar' of type 'str' could not be converted to Integer", key="foo", full_key="params.foo", object_type=ConcretePlugin.FoobarParams, @@ -587,26 +613,37 @@ params = [ ), id="dict[bool,Any]:mistyped_key", ), + param( + Expected( + create=lambda: DictConfig({}, key_type=bytes), + op=lambda cfg: cfg.get("foo"), + exception_type=KeyValidationError, + msg="Key foo (str) is incompatible with (bytes)", + key="foo", + full_key="foo", + ), + id="dict[bool,Any]:mistyped_key", + ), # dict:create param( Expected( create=lambda: None, op=lambda _: OmegaConf.structured(NotOptionalInt), exception_type=ValidationError, - msg="Non optional field cannot be assigned None", + msg="Incompatible value 'None' for field of type 'int'", key="foo", full_key="foo", parent_node=lambda _: {}, object_type=NotOptionalInt, ), - id="dict:create_none_optional_with_none", + id="dict:create_non_optional_with_none", ), param( Expected( create=lambda: None, op=lambda _: OmegaConf.structured(NotOptionalInt), exception_type=ValidationError, - msg="Non optional field cannot be assigned None", + msg="Incompatible value 'None' for field of type 'int'", key="foo", full_key="foo", parent_node=lambda _: {}, @@ -630,6 +667,16 @@ params = [ ), param( Expected( + create=lambda: DictConfig({}, element_type=str), + op=lambda cfg: OmegaConf.merge(cfg, {"foo": None}), + exception_type=ValidationError, + key="foo", + msg="field 'foo' is not Optional", + ), + id="dict:merge_none_into_not_optional_element_type", + ), + param( + Expected( create=lambda: None, op=lambda cfg: OmegaConf.structured(IllegalType), exception_type=ValidationError, @@ -646,13 +693,147 @@ params = [ ConcretePlugin(params=ConcretePlugin.FoobarParams(foo="x")) # type: ignore ), exception_type=ValidationError, - msg="Value 'x' could not be converted to Integer", + msg="Value 'x' of type 'str' could not be converted to Integer", key="foo", full_key="foo", parent_node=lambda _: {}, object_type=ConcretePlugin.FoobarParams, ), - id="structured:create_with_invalid_value", + id="structured:create_with_invalid_value,int", + ), + param( + Expected( + create=lambda: DictConfig({"bar": FloatNode(123.456)}), + op=lambda cfg: cfg.__setattr__("bar", "x"), + exception_type=ValidationError, + msg="Value 'x' of type 'str' could not be converted to Float", + key="bar", + full_key="bar", + child_node=lambda cfg: cfg._get_node("bar"), + ), + id="typed_DictConfig:assign_with_invalid_value,str_to_float", + ), + param( + Expected( + create=lambda: DictConfig({"bar": FloatNode(123.456)}), + op=lambda cfg: cfg.__setattr__("bar", Path("hello.txt")), + exception_type=ValidationError, + msg="Value 'hello.txt' of type 'pathlib.(Posix|Windows)Path' could not be converted to Float", + msg_is_regex=True, + key="bar", + full_key="bar", + child_node=lambda cfg: cfg._get_node("bar"), + ), + id="typed_DictConfig:assign_with_invalid_value,path_to_float", + ), + param( + Expected( + create=lambda: DictConfig({"bar": BytesNode(b"binary")}), + op=lambda cfg: cfg.__setattr__("bar", 123.4), + exception_type=ValidationError, + msg="Value '123.4' of type 'float' is not of type 'bytes'", + key="bar", + full_key="bar", + child_node=lambda cfg: cfg._get_node("bar"), + ), + id="typed_DictConfig:assign_with_invalid_value,string_to_bytes", + ), + param( + Expected( + create=lambda: DictConfig( + {"bar": BytesNode(b"binary", flags={"convert": False})} + ), + op=lambda cfg: cfg.__setattr__("bar", 123.4), + exception_type=ValidationError, + msg="Value '123.4' of type 'float' is incompatible with type hint 'Optional[bytes]'", + key="bar", + full_key="bar", + child_node=lambda cfg: cfg._get_node("bar"), + ), + id="typed_DictConfig:assign_with_invalid_value,string_to_bytes,no_convert", + ), + param( + Expected( + create=lambda: DictConfig({"bar": PathNode(Path("hello.txt"))}), + op=lambda cfg: cfg.__setattr__("bar", 123.4), + exception_type=ValidationError, + msg="Value '123.4' of type 'float' could not be converted to Path", + key="bar", + full_key="bar", + child_node=lambda cfg: cfg._get_node("bar"), + ), + id="typed_DictConfig:assign_with_invalid_value,string_to_path", + ), + param( + Expected( + create=lambda: DictConfig( + {"bar": PathNode(Path("hello.txt"), flags={"convert": False})} + ), + op=lambda cfg: cfg.__setattr__("bar", 123.4), + exception_type=ValidationError, + msg="Value '123.4' of type 'float' is not an instance of 'pathlib.Path'", + key="bar", + full_key="bar", + child_node=lambda cfg: cfg._get_node("bar"), + ), + id="typed_DictConfig:assign_with_invalid_value,string_to_path,no_convert", + ), + param( + Expected( + create=lambda: DictConfig({"bar": StringNode("abc123")}), + op=lambda cfg: cfg.__setattr__("bar", b"binary"), + exception_type=ValidationError, + msg=r"Cannot convert 'bytes' to string: 'b'binary''", + key="bar", + full_key="bar", + child_node=lambda cfg: cfg._get_node("bar"), + ), + id="typed_DictConfig:assign_with_invalid_value,bytes_to_string", + ), + param( + Expected( + create=lambda: DictConfig( + {"bar": StringNode("abc123")}, flags={"convert": False} + ), + op=lambda cfg: cfg.__setattr__("bar", b"binary"), + exception_type=ValidationError, + msg=r"Value 'b'binary'' of type 'bytes' is incompatible with type hint 'Optional[str]'", + key="bar", + full_key="bar", + child_node=lambda cfg: cfg._get_node("bar"), + ), + id="typed_DictConfig:assign_with_invalid_value,bytes_to_string,parent_no_convert", + ), + param( + Expected( + create=lambda: DictConfig({"bar": FloatNode(123.456)}), + op=lambda cfg: cfg.__setattr__("bar", Color.BLUE), + exception_type=ValidationError, + msg="Value 'Color.BLUE' of type 'tests.Color' could not be converted to Float", + key="bar", + full_key="bar", + child_node=lambda cfg: cfg._get_node("bar"), + ), + id="typed_DictConfig:assign_with_invalid_value,full_module_in_error", + ), + param( + Expected( + create=lambda: DictConfig( + {"foo": {"bar": UnionNode(123.456, Union[bool, float])}} + ), + op=lambda cfg: cfg.foo.__setattr__("bar", "abc"), + exception_type=ValidationError, + msg=re.escape( + "Value 'abc' of type 'str' is incompatible with type hint 'Optional[Union[" + ) + + r"(bool, float)|(float, bool)\]\]'", + msg_is_regex=True, + key="bar", + full_key="foo.bar", + parent_node=lambda cfg: cfg.foo, + child_node=lambda cfg: cfg.foo._get_node("bar"), + ), + id="typed_DictConfig:assign_with_invalid_value-string_to_union[bool-float]", ), param( Expected( @@ -668,9 +849,18 @@ params = [ param( Expected( create=lambda: None, + op=lambda _: DictConfig({}, element_type=IllegalType), + exception_type=ValidationError, + msg="Unsupported value type: 'tests.IllegalType'", + ), + id="structured:create_with_unsupported_element_type", + ), + param( + Expected( + create=lambda: None, op=lambda cfg: OmegaConf.structured(UnionError), exception_type=ValueError, - msg="Union types are not supported:\nx: Union[int, str]", + msg="Unions of containers are not supported:\nx: Union[int, List[str]]", num_lines=3, ), id="structured:create_with_union_error", @@ -694,7 +884,7 @@ params = [ ), op=lambda cfg: cfg.__setitem__("baz", "fail"), exception_type=ValidationError, - msg="Value 'fail' could not be converted to Integer", + msg="Value 'fail' of type 'str' could not be converted to Integer", key="baz", ), id="DictConfig[str,int]:assigned_str_value", @@ -1102,7 +1292,7 @@ params = [ create=lambda: ListConfig(element_type=int, content=[1, 2, 3]), op=lambda cfg: cfg.__setitem__(0, "foo"), exception_type=ValidationError, - msg="Value 'foo' could not be converted to Integer", + msg="Value 'foo' of type 'str' could not be converted to Integer", key=0, full_key="[0]", child_node=lambda cfg: cfg[0], @@ -1117,7 +1307,7 @@ params = [ ), op=lambda cfg: cfg.__setitem__(0, "foo"), exception_type=ValidationError, - msg="Value 'foo' could not be converted to Integer", + msg="Value 'foo' of type 'str' could not be converted to Integer", key=0, full_key="[0]", child_node=lambda cfg: cfg[0], @@ -1288,6 +1478,97 @@ params = [ ), id="to_object:structured-missing-field", ), + param( + Expected( + create=lambda: OmegaConf.structured(NestedInterpolationToMissing), + op=lambda cfg: OmegaConf.to_object(cfg), + exception_type=InterpolationToMissingValueError, + msg=( + "MissingMandatoryValue while resolving interpolation: " + "Missing mandatory value: name" + ), + key="baz", + full_key="subcfg.baz", + object_type=NestedInterpolationToMissing.BazParams, + parent_node=lambda cfg: cfg.subcfg, + child_node=lambda cfg: cfg.subcfg._get_node("baz"), + num_lines=3, + ), + id="to_object:structured,throw_on_missing_interpolation", + ), + param( + Expected( + create=lambda: OmegaConf.structured(StructuredInterpolationKeyError), + op=lambda cfg: OmegaConf.to_object(cfg), + exception_type=InterpolationKeyError, + key="name", + msg=("Interpolation key 'bar' not found"), + child_node=lambda cfg: cfg._get_node("name"), + ), + id="to_object:structured,throw_on_interpolation_key_error", + ), + # to_container throw_on_missing + param( + Expected( + create=lambda: OmegaConf.create( + {"subcfg": {"x": "${missing_val}"}, "missing_val": "???"} + ), + op=lambda cfg: OmegaConf.to_container( + cfg, resolve=True, throw_on_missing=True + ), + exception_type=InterpolationToMissingValueError, + msg=( + "MissingMandatoryValue while resolving interpolation: " + "Missing mandatory value: missing_val" + ), + key="x", + full_key="subcfg.x", + parent_node=lambda cfg: cfg.subcfg, + child_node=lambda cfg: cfg.subcfg._get_node("x"), + ), + id="to_container:throw_on_missing_interpolation", + ), + param( + Expected( + create=lambda: DictConfig("???"), + op=lambda cfg: OmegaConf.to_container(cfg, throw_on_missing=True), + exception_type=MissingMandatoryValue, + msg="Missing mandatory value", + ), + id="to_container:throw_on_missing,dict", + ), + param( + Expected( + create=lambda: ListConfig("???"), + op=lambda cfg: OmegaConf.to_container(cfg, throw_on_missing=True), + exception_type=MissingMandatoryValue, + msg="Missing mandatory value", + ), + id="to_container:throw_on_missing,list", + ), + param( + Expected( + create=lambda: DictConfig({"a": "???"}), + op=lambda cfg: OmegaConf.to_container(cfg, throw_on_missing=True), + exception_type=MissingMandatoryValue, + msg="Missing mandatory value: a", + key="a", + child_node=lambda cfg: cfg._get_node("a"), + ), + id="to_container:throw_on_missing,dict_value", + ), + param( + Expected( + create=lambda: ListConfig(["???"]), + op=lambda cfg: OmegaConf.to_container(cfg, throw_on_missing=True), + exception_type=MissingMandatoryValue, + msg="Missing mandatory value: 0", + key=0, + full_key="[0]", + child_node=lambda cfg: cfg._get_node(0), + ), + id="to_container:throw_on_missing,list_item", + ), ] @@ -1308,8 +1589,12 @@ def test_errors(expected: Expected, monk monkeypatch.setenv("OC_CAUSE", "0") cfg = expected.create() expected.finalize(cfg) - msg = expected.msg - with raises(expected.exception_type, match=re.escape(msg)) as einfo: + if expected.msg_is_regex: + match = expected.msg + else: + match = re.escape(expected.msg) + + with raises(expected.exception_type, match=match) as einfo: try: expected.op(cfg) except Exception as e: @@ -1378,11 +1663,11 @@ def test_resolver_error(restore_resolver c = OmegaConf.create({"div_by_zero": "${div:1,0}"}) expected_msg = dedent( """\ - ZeroDivisionError raised while resolving interpolation: float division by zero + ZeroDivisionError raised while resolving interpolation: float division( by zero)? full_key: div_by_zero object_type=dict""" ) - with raises(InterpolationResolutionError, match=re.escape(expected_msg)): + with raises(InterpolationResolutionError, match=expected_msg): c.div_by_zero @@ -1401,6 +1686,33 @@ def test_parse_error_on_creation(create_ create_func(arg) +@mark.parametrize( + ["create_func", "obj"], + [ + param(DictConfig, {"zz": 10}, id="dict"), + param(DictConfig, {}, id="dict_empty"), + param(DictConfig, User, id="structured"), + param(ListConfig, ["zz"], id="list"), + param(ListConfig, [], id="list_empty"), + param(OmegaConf.create, {"zz": 10}, id="create"), + ], +) +def test_parent_type_error_on_creation(create_func: Any, obj: Any) -> None: + with raises(ConfigTypeError, match=re.escape("Parent type is not omegaconf.Box")): + create_func(obj, parent={"a"}) # bad parent + + +def test_union_must_not_be_parent_of_union() -> None: + bad_parent = UnionNode(123, Union[int, str]) + with raises( + ConfigTypeError, match=re.escape("Parent type is not omegaconf.Container") + ): + UnionNode(456, Union[int, str], parent=bad_parent) + + good_parent = DictConfig({}) + UnionNode(456, Union[int, str], parent=good_parent) # ok + + def test_cycle_when_iterating_over_parents() -> None: c = OmegaConf.create({"x": {}}) x_node = c._get_node("x") @@ -1446,7 +1758,7 @@ def test_dict_subclass_error() -> None: src["bar"] = "qux" # type: ignore with raises( ValidationError, - match=re.escape("Value 'qux' could not be converted to Integer"), + match=re.escape("Value 'qux' of type 'str' could not be converted to Integer"), ) as einfo: with warns_dict_subclass_deprecated(Str2Int): OmegaConf.structured(src) diff -pruN 2.1.0~rc1-3/tests/test_get_full_key.py 2.2.2-1/tests/test_get_full_key.py --- 2.1.0~rc1-3/tests/test_get_full_key.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/tests/test_get_full_key.py 2022-05-27 21:36:40.000000000 +0000 @@ -1,8 +1,8 @@ -from typing import Any +from typing import Any, Union from pytest import mark, param -from omegaconf import DictConfig, IntegerNode, OmegaConf +from omegaconf import DictConfig, IntegerNode, OmegaConf, UnionNode from tests import Color @@ -92,6 +92,11 @@ from tests import Color # slice ([1, 2, 3], "", slice(0, 1), "[0:1]"), ([1, 2, 3], "", slice(0, 1, 2), "[0:1:2]"), + # union + ({"foo": UnionNode(123, Union[int, str])}, "", "foo", "foo"), + ([UnionNode(123, Union[int, str])], "", "0", "[0]"), + ({"foo": {"bar": UnionNode(123, Union[int, str])}}, "foo", "bar", "foo.bar"), + ({"foo": {"bar": UnionNode(123, Union[int, str])}}, "foo.bar", None, "foo.bar"), ], ) def test_get_full_key_from_config( @@ -115,3 +120,13 @@ def test_value_node_get_full_key() -> No assert node._get_full_key(None) == "" node = IntegerNode(key="foo", value=10) assert node._get_full_key(None) == "foo" + + +def test_union_node_get_full_key() -> None: + cfg = OmegaConf.create({"foo": UnionNode(10, Union[int, str])}) + assert cfg._get_node("foo")._get_full_key(None) == "foo" # type: ignore + + node = UnionNode(10, Union[int, str]) + assert node._get_full_key(None) == "" + node = UnionNode(10, Union[int, str], key="foo") + assert node._get_full_key(None) == "foo" diff -pruN 2.1.0~rc1-3/tests/test_grammar.py 2.2.2-1/tests/test_grammar.py --- 2.1.0~rc1-3/tests/test_grammar.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/tests/test_grammar.py 2022-05-27 21:36:40.000000000 +0000 @@ -1,5 +1,7 @@ import math import re +import threading +import time from typing import Any, Callable, List, Optional, Set, Tuple import antlr4 @@ -27,6 +29,7 @@ TAB = "\t" # to be used in raw strings, # Characters that are not allowed by the grammar in config key names. INVALID_CHARS_IN_KEY_NAMES = r"""\{}()[].:"' """ +UNQUOTED_SPECIAL = r"/-\+.$%*@?|" # special characters allowed in unquoted strings # A fixed config that may be used (but not modified!) by tests. BASE_TEST_CFG = OmegaConf.create( @@ -35,6 +38,7 @@ BASE_TEST_CFG = OmegaConf.create( "str": "hi", "int": 123, "float": 1.2, + "bytes": b"binary", "dict": {"a": 0, "b": {"c": 1}}, "list": [x - 1 for x in range(11)], "null": None, @@ -104,7 +108,11 @@ PARAMS_SINGLE_ELEMENT_NO_INTERPOLATION: ("float_minus_nan", "-nan", math.nan), # Unquoted strings. # Note: raw strings do not allow trailing \, adding a space and stripping it. - ("str_legal", r" a/-\+.$*@?\\ ".strip(), r" a/-\+.$*@?\ ".strip()), + ( + "str_legal", + (r" a" + UNQUOTED_SPECIAL + r"\\ ").strip(), + (r" a" + UNQUOTED_SPECIAL + r"\ ").strip(), + ), ("str_illegal_1", "a,=b", GrammarParseError), ("str_illegal_2", f"{chr(200)}", GrammarParseError), ("str_illegal_3", f"{chr(129299)}", GrammarParseError), @@ -114,7 +122,7 @@ PARAMS_SINGLE_ELEMENT_NO_INTERPOLATION: ("str_ws_1", "hello world", "hello world"), ("str_ws_2", "a b\tc \t\t d", "a b\tc \t\t d"), ("str_esc_ws_1", r"\ hello\ world\ ", " hello world "), - ("str_esc_ws_2", fr"\ \{TAB}\{TAB}", f" {TAB}{TAB}"), + ("str_esc_ws_2", rf"\ \{TAB}\{TAB}", f" {TAB}{TAB}"), ("str_esc_comma", r"hello\, world", "hello, world"), ("str_esc_colon", r"a\:b", "a:b"), ("str_esc_equal", r"a\=b", "a=b"), @@ -126,8 +134,8 @@ PARAMS_SINGLE_ELEMENT_NO_INTERPOLATION: ("str_esc_illegal_1", r"\#", GrammarParseError), ("str_esc_illegal_2", r""" \'\" """.strip(), GrammarParseError), # Quoted strings. - ("str_quoted_single", "'!@#$%^&*()[]:.,\"'", '!@#$%^&*()[]:.,"'), - ("str_quoted_double", '"!@#$%^&*()[]:.,\'"', "!@#$%^&*()[]:.,'"), + ("str_quoted_single", "'!@#$%^&*|()[]:.,\"'", '!@#$%^&*|()[]:.,"'), + ("str_quoted_double", '"!@#$%^&*|()[]:.,\'"', "!@#$%^&*|()[]:.,'"), ("str_quoted_outer_ws_single", "' a \t'", " a \t"), ("str_quoted_outer_ws_double", '" a \t"', " a \t"), ("str_quoted_int", "'123'", "123"), @@ -179,8 +187,10 @@ PARAMS_SINGLE_ELEMENT_NO_INTERPOLATION: ), ( "dict_unquoted_key", - fr"{{a0-null-1-3.14-NaN- {TAB}-true-False-/\+.$%*@\(\)\[\]\{{\}}\:\=\ \{TAB}\,:0}}", - {fr"a0-null-1-3.14-NaN- {TAB}-true-False-/\+.$%*@()[]{{}}:= {TAB},": 0}, + rf"{{a0-null-1-3.14-NaN- {TAB}-true-False-{UNQUOTED_SPECIAL}\(\)\[\]\{{\}}\:\=\ \{TAB}\,:0}}", + { + rf"a0-null-1-3.14-NaN- {TAB}-true-False-{UNQUOTED_SPECIAL}()[]{{}}:= {TAB},": 0 + }, ), ( "dict_quoted", @@ -362,7 +372,11 @@ PARAMS_CONFIG_VALUE = [ ("str_top_middle_quote_double", 'I"d like ${str}', 'I"d like hi'), ("str_top_middle_quotes_single", "I like '${str}'", "I like 'hi'"), ("str_top_middle_quotes_double", 'I like "${str}"', 'I like "hi"'), - ("str_top_any_char", r"${str} !@\#$%^&*})][({,/?;", r"hi !@\#$%^&*})][({,/?;"), + ( + "str_top_any_char", + r"${str} " + UNQUOTED_SPECIAL + r"^!#&})][({,;", + r"hi " + UNQUOTED_SPECIAL + r"^!#&})][({,;", + ), ("str_top_esc_inter", r"Esc: \${str}", "Esc: ${str}"), ("str_top_esc_inter_wrong_1", r"Wrong: $\{str\}", r"Wrong: $\{str\}"), ("str_top_esc_inter_wrong_2", r"Wrong: \${str\}", r"Wrong: ${str\}"), @@ -582,7 +596,6 @@ class TestOmegaConfGrammar: # grammer tests, but `resolve_parse_tree()` requires it). node=AnyNode(None, parent=cfg), key=None, - parent=cfg, ) ) @@ -600,7 +613,7 @@ class TestOmegaConfGrammar: "$ ${foo} ${bar} ${boz} $", "${foo:bar}", "${foo : bar, baz, boz}", - "${foo:bar,0,a-b+c*d/$.%@}", + "${foo:bar,0,a-b+c*d/$.%@?|}", r"\${foo}", "${foo.bar:boz}", "${$foo.bar$.x$y}", @@ -647,6 +660,7 @@ class TestMatchSimpleInterpolationPatter ("${ns . f:var}", False), ("${$foo:bar}", False), ("${.foo:bar}", False), + (r"${foo:\}", False), # Valid according to the grammar but not matched by the regex. ("${foo.${bar}}", True), ("${foo:${bar}}", True), @@ -727,7 +741,7 @@ def test_parse_interpolation(inter: Any, def test_custom_resolver_param_supported_chars() -> None: - supported_chars = r"abc123_/:-\+.$%*@" + supported_chars = r"abc123_:" + UNQUOTED_SPECIAL c = OmegaConf.create({"dir1": "${copy:" + supported_chars + "}"}) OmegaConf.register_new_resolver("copy", lambda x: x) @@ -767,3 +781,43 @@ def test_invalid_chars_in_interpolation( # Other invalid characters should be detected at creation time. with raises(GrammarParseError): create() + + +def test_grammar_cache_is_thread_safe() -> None: + """ + This test ensures that we can parse strings across multiple threads in parallel. + + Besides ensuring that the parsing does not hang nor crash, we also verify that + the lexer used in each thread is different. + """ + n_threads = 10 + lexer_ids = [] + stop = threading.Event() + + def check_cache_lexer_id() -> None: + # Parse a dummy string to make sure the grammar cache is populated + # (this also checks that multiple threads can parse in parallel). + grammar_parser.parse("foo") + # Keep track of the ID of the cached lexer. + lexer_ids.append(id(grammar_parser._grammar_cache.data[0])) + # Wait until we are done. + while not stop.is_set(): + time.sleep(0.1) + + # Launch threads. + threads = [] + for i in range(n_threads): + threads.append(threading.Thread(target=check_cache_lexer_id)) + threads[-1].start() + + # Wait until all threads have reported their lexer ID. + while len(lexer_ids) < n_threads: + time.sleep(0.1) + + # Terminate threads. + stop.set() + for thread in threads: + thread.join() + + # Check that each thread used a unique lexer. + assert len(set(lexer_ids)) == n_threads diff -pruN 2.1.0~rc1-3/tests/test_matrix.py 2.2.2-1/tests/test_matrix.py --- 2.1.0~rc1-3/tests/test_matrix.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/tests/test_matrix.py 2022-05-27 21:36:40.000000000 +0000 @@ -1,18 +1,22 @@ import copy import re -from typing import Any, Optional +from pathlib import Path +from typing import Any, Optional, Union -from pytest import mark, raises, warns +from pytest import mark, raises from omegaconf import ( BooleanNode, + BytesNode, DictConfig, EnumNode, FloatNode, IntegerNode, ListConfig, OmegaConf, + PathNode, StringNode, + UnionNode, ValidationError, ValueNode, ) @@ -46,21 +50,21 @@ def verify( assert cfg.get(key) == exp assert OmegaConf.is_missing(cfg, key) == missing - with warns(UserWarning): - assert OmegaConf.is_none(cfg, key) == none_public assert _is_optional(cfg, key) == opt assert OmegaConf.is_interpolation(cfg, key) == inter -# for each type Node type: int, bool, str, float, Color (enum) and User (@dataclass), DictConfig, ListConfig +# for each type Node type: int, bool, str, bytes, float, Color (enum) and User (@dataclass), DictConfig, ListConfig # for each MISSING, None, Optional and interpolation: @mark.parametrize( "node_type, values", [ (BooleanNode, [True, False]), + (BytesNode, [b"binary"]), (FloatNode, [3.1415]), (IntegerNode, [42]), (StringNode, ["hello"]), + (PathNode, [Path("hello.txt")]), # EnumNode ( lambda value, is_optional, key=None: EnumNode( @@ -68,6 +72,13 @@ def verify( ), [Color.RED], ), + # UnionNode + ( + lambda value, is_optional, key=None: UnionNode( + value, ref_type=Union[bool, float], is_optional=is_optional, key=key + ), + [True, False, 10.0], + ), # DictConfig ( lambda value, is_optional, key=None: DictConfig( @@ -92,10 +103,13 @@ def verify( ], ids=( "BooleanNode", + "BytesNode", "FloatNode", "IntegerNode", "StringNode", + "PathNode", "EnumNode", + "UnionNode", "DictConfig", "ListConfig", "dataclass", @@ -111,7 +125,7 @@ class TestNodeTypesMatrix: data = {"node": node} cfg = OmegaConf.create(obj=data) verify(cfg, "node", none=False, opt=False, missing=False, inter=False) - msg = "child 'node' is not Optional" + msg = "field 'node' is not Optional" with raises(ValidationError, match=re.escape(msg)): cfg.node = None diff -pruN 2.1.0~rc1-3/tests/test_merge.py 2.2.2-1/tests/test_merge.py --- 2.1.0~rc1-3/tests/test_merge.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/tests/test_merge.py 2022-05-27 21:36:40.000000000 +0000 @@ -1,18 +1,40 @@ import copy +import re import sys -from typing import Any, Dict, List, MutableMapping, MutableSequence, Tuple, Union +from textwrap import dedent +from typing import ( + Any, + Dict, + List, + MutableMapping, + MutableSequence, + Optional, + Tuple, + Union, +) +from _pytest.python_api import RaisesContext from pytest import mark, param, raises from omegaconf import ( MISSING, DictConfig, + FloatNode, ListConfig, OmegaConf, ReadonlyConfigError, + UnionNode, ValidationError, ) -from omegaconf._utils import is_structured_config +from omegaconf._utils import ( + ValueKind, + _ensure_container, + _get_value, + get_type_hint, + get_value_kind, + is_structured_config, +) +from omegaconf.base import Node from omegaconf.errors import ConfigKeyError, UnsupportedValueType from omegaconf.nodes import IntegerNode from tests import ( @@ -28,6 +50,8 @@ from tests import ( InterpolationList, MissingDict, MissingList, + OptionalUsers, + OptTuple, Package, Plugin, User, @@ -67,6 +91,41 @@ from tests import ( param(({"a": 1}, {"a": IntegerNode(10)}), {"a": IntegerNode(10)}), param(({"a": IntegerNode(10)}, {"a": 1}), {"a": 1}), param(({"a": IntegerNode(10)}, {"a": 1}), {"a": IntegerNode(1)}), + param( + ({"a": 1.0}, {"a": UnionNode(10.1, Union[float, bool])}), + {"a": 10.1}, + id="dict_merge_union_into_float", + ), + param( + ({"a": "abc"}, {"a": UnionNode(10.1, Union[float, bool])}), + {"a": 10.1}, + id="dict_merge_union_into_str", + ), + param( + ({"a": FloatNode(1.0)}, {"a": UnionNode(10.1, Union[float, bool])}), + {"a": 10.1}, + id="dict_merge_union_into_typed_float", + ), + param( + ({"a": FloatNode(1.0)}, {"a": UnionNode(True, Union[float, bool])}), + raises(ValidationError), + id="dict_merge_union_bool_into_typed_float", + ), + param( + ({"a": IntegerNode(1)}, {"a": UnionNode(10.1, Union[float, bool])}), + raises(ValidationError), + id="dict_merge_union_into_typed_int", + ), + param( + ({"a": UnionNode(10.1, Union[float, bool])}, {"a": 1}), + raises(ValidationError), + id="dict_merge_int_into_union-err", + ), + param( + ({"a": UnionNode(10.1, Union[float, bool])}, {"a": 1.0}), + {"a": 1.0}, + id="dict_merge_float_into_union", + ), param(({"a": "???"}, {"a": {}}), {"a": {}}, id="dict_merge_into_missing"), param( ({"a": "???"}, {"a": {"b": 10}}), @@ -218,6 +277,16 @@ from tests import ( id="users_merge_with_missing_age", ), param( + [OptionalUsers, {"name2user": {"joe": {"name": "joe"}}}], + {"name2user": {"joe": {"name": "joe", "age": MISSING}}}, + id="optionalusers_merge_with_missing_age", + ), + param( + [OptionalUsers, {"name2user": {"joe": None}}], + {"name2user": {"joe": None}}, + id="optionalusers_merge_with_none", + ), + param( [ConfWithMissingDict, {"dict": {"foo": "bar"}}], {"dict": {"foo": "bar"}}, id="conf_missing_dict", @@ -250,6 +319,11 @@ from tests import ( raises(ConfigKeyError), id="merge_unknown_key_into_structured_node", ), + param( + [OptTuple, {"x": [1, 2]}], + {"x": [1, 2]}, + id="merge_list_into_optional_tuple_none", + ), # DictConfig with element_type of Structured Config param( ( @@ -277,7 +351,7 @@ from tests import ( ), param( ( - DictConfig({"user007": None}, element_type=User), + DictConfig({"user007": None}, element_type=Optional[User]), {"user007": {"age": 99}}, ), {"user007": {"name": "???", "age": 99}}, @@ -359,6 +433,682 @@ def test_merge( merge_function(*configs) +@mark.parametrize( + "inputs,expected,ref_type,is_optional", + [ + param( + (DictConfig(content={"foo": "bar"}, element_type=str), {"foo": "qux"}), + {"foo": "qux"}, + str, + False, + id="str", + ), + param( + (DictConfig(content={"foo": "bar"}, element_type=str), {"foo": None}), + raises( + ValidationError, + match="Incompatible value 'None' for field of type 'str'", + ), + None, + None, + id="str_none", + ), + param( + (DictConfig(content={"foo": "bar"}, element_type=str), {"foo": MISSING}), + {"foo": "bar"}, + str, + False, + id="str_missing", + ), + param( + ( + DictConfig(content={"foo": "bar"}, element_type=Optional[str]), + {"foo": "qux"}, + ), + {"foo": "qux"}, + str, + True, + id="optional_str", + ), + param( + ( + DictConfig(content={"foo": "bar"}, element_type=Optional[str]), + {"foo": None}, + ), + {"foo": None}, + str, + True, + id="optional_str_none", + ), + param( + ( + DictConfig(content={"foo": "bar"}, element_type=Optional[str]), + {"foo": MISSING}, + ), + {"foo": "bar"}, + str, + True, + id="optional_str_missing", + ), + param( + (DictConfig(content={}, element_type=str), {"foo": "qux"}), + {"foo": "qux"}, + str, + False, + id="new_str", + ), + param( + (DictConfig(content={}, element_type=str), {"foo": None}), + raises( + ValidationError, + match="field 'foo' is not Optional", + ), + None, + None, + id="new_str_none", + ), + param( + (DictConfig(content={}, element_type=str), {"foo": MISSING}), + {"foo": MISSING}, + str, + False, + id="new_str_missing", + ), + param( + (DictConfig(content={}, element_type=Optional[str]), {"foo": "qux"}), + {"foo": "qux"}, + str, + True, + id="new_optional_str", + ), + param( + (DictConfig(content={}, element_type=Optional[str]), {"foo": None}), + {"foo": None}, + str, + True, + id="new_optional_str_none", + ), + param( + (DictConfig(content={}, element_type=Optional[str]), {"foo": MISSING}), + {"foo": MISSING}, + str, + True, + id="new_optional_str_missing", + ), + param( + (DictConfig(content={"foo": MISSING}, element_type=str), {"foo": "qux"}), + {"foo": "qux"}, + str, + False, + id="missing_str", + ), + param( + (DictConfig(content={"foo": MISSING}, element_type=str), {"foo": None}), + raises( + ValidationError, + match="Incompatible value 'None' for field of type 'str'", + ), + None, + None, + id="missing_str_none", + ), + param( + (DictConfig(content={"foo": MISSING}, element_type=str), {"foo": MISSING}), + {"foo": MISSING}, + str, + False, + id="missing_str_missing", + ), + param( + ( + DictConfig(content={"foo": MISSING}, element_type=Optional[str]), + {"foo": "qux"}, + ), + {"foo": "qux"}, + str, + True, + id="missing_optional_str", + ), + param( + ( + DictConfig(content={"foo": MISSING}, element_type=Optional[str]), + {"foo": None}, + ), + {"foo": None}, + str, + True, + id="missing_optional_str_none", + ), + param( + ( + DictConfig(content={"foo": MISSING}, element_type=Optional[str]), + {"foo": MISSING}, + ), + {"foo": MISSING}, + str, + True, + id="missing_optional_str_missing", + ), + param( + ( + DictConfig(content={"foo": User("Bond")}, element_type=User), + {"foo": User("007")}, + ), + {"foo": User("007")}, + User, + False, + id="user", + ), + param( + ( + DictConfig(content={"foo": User("Bond")}, element_type=User), + {"foo": None}, + ), + raises( + ValidationError, + match=re.escape( + dedent( + """\ + field 'foo' is not Optional + full_key: foo + reference_type=User + object_type=User""" + ) + ), + ), + None, + None, + id="user_none", + ), + param( + ( + DictConfig(content={"foo": User("Bond")}, element_type=User), + {"foo": MISSING}, + ), + {"foo": User("Bond")}, + User, + False, + id="user_missing", + ), + param( + ( + DictConfig(content={"foo": User("Bond")}, element_type=Optional[User]), + {"foo": User("007")}, + ), + {"foo": User("007")}, + User, + True, + id="optional_user", + ), + param( + ( + DictConfig(content={"foo": User("Bond")}, element_type=Optional[User]), + {"foo": None}, + ), + {"foo": None}, + User, + True, + id="optional_user_none", + ), + param( + ( + DictConfig(content={"foo": User("Bond")}, element_type=Optional[User]), + {"foo": MISSING}, + ), + {"foo": User("Bond")}, + User, + True, + id="optional_user_missing", + ), + param( + (DictConfig(content={}, element_type=User), {"foo": User("Bond")}), + {"foo": User("Bond")}, + User, + False, + id="new_user", + ), + param( + (DictConfig(content={}, element_type=User), {"foo": None}), + raises( + ValidationError, + match=re.escape( + dedent( + """\ + field 'foo' is not Optional + full_key: foo + object_type=dict""" + ) + ), + ), + None, + None, + id="new_user_none", + ), + param( + (DictConfig(content={}, element_type=User), {"foo": MISSING}), + {"foo": MISSING}, + User, + False, + id="new_user_missing", + ), + param( + ( + DictConfig(content={}, element_type=Optional[User]), + {"foo": User("Bond")}, + ), + {"foo": User("Bond")}, + User, + True, + id="new_optional_user", + ), + param( + (DictConfig(content={}, element_type=Optional[User]), {"foo": None}), + {"foo": None}, + User, + True, + id="new_optional_user_none", + ), + param( + (DictConfig(content={}, element_type=Optional[User]), {"foo": MISSING}), + {"foo": MISSING}, + User, + True, + id="new_optional_user_missing", + ), + param( + ( + DictConfig(content={"foo": MISSING}, element_type=User), + {"foo": User("Bond")}, + ), + {"foo": User("Bond")}, + User, + False, + id="missing_user", + ), + param( + (DictConfig(content={"foo": MISSING}, element_type=User), {"foo": None}), + raises( + ValidationError, + match=re.escape( + dedent( + """\ + field 'foo' is not Optional + full_key: foo + reference_type=User + object_type=NoneType""" + ) + ), + ), + None, + None, + id="missing_user_none", + ), + param( + (DictConfig(content={"foo": MISSING}, element_type=User), {"foo": MISSING}), + {"foo": MISSING}, + User, + False, + id="missing_user_missing", + ), + param( + ( + DictConfig(content={"foo": MISSING}, element_type=Optional[User]), + {"foo": User("Bond")}, + ), + {"foo": User("Bond")}, + User, + True, + id="missing_optional_user", + ), + param( + ( + DictConfig(content={"foo": MISSING}, element_type=Optional[User]), + {"foo": None}, + ), + {"foo": None}, + User, + True, + id="missing_optional_user_none", + ), + param( + ( + DictConfig(content={"foo": MISSING}, element_type=Optional[User]), + {"foo": MISSING}, + ), + {"foo": MISSING}, + User, + True, + id="missing_optional_user_missing", + ), + ], +) +def test_optional_element_type_merge( + inputs: Any, expected: Any, ref_type: Any, is_optional: bool +) -> None: + configs = [_ensure_container(c) for c in inputs] + if isinstance(expected, RaisesContext): + with expected: + OmegaConf.merge(*configs) + else: + cfg = OmegaConf.merge(*configs) + assert cfg == expected + + assert isinstance(cfg, DictConfig) + node = cfg._get_node("foo") + assert isinstance(node, Node) + assert node._is_optional() == is_optional + assert node._metadata.ref_type == ref_type + + +@mark.parametrize( + "inputs,expected,type_hint", + [ + param( + ( + {"foo": UnionNode(10.1, Union[float, bool], is_optional=False)}, + {"foo": 20.2}, + ), + {"foo": 20.2}, + Union[float, bool], + id="merge-any-into-union", + ), + param( + ( + {"foo": 20.2}, + {"foo": UnionNode(10.1, Union[float, bool], is_optional=False)}, + ), + {"foo": 10.1}, + Union[float, bool], + id="merge-union-into-any", + ), + param( + ( + {"foo": UnionNode(10.1, Union[float, bool], is_optional=False)}, + {"foo": True}, + ), + {"foo": True}, + Union[float, bool], + id="merge-different-object-type-into-union", + ), + param( + ( + {"foo": True}, + {"foo": UnionNode(10.1, Union[float, bool], is_optional=False)}, + ), + {"foo": 10.1}, + Union[float, bool], + id="merge-union-into-different-object-type", + ), + param( + ( + {"foo": UnionNode(10.1, Union[float, bool], is_optional=False)}, + {"foo": "abc"}, + ), + raises(ValidationError), + None, + id="merge-any-into-union-incompatible_type", + ), + param( + ( + {"foo": "abc"}, + {"foo": UnionNode(10.1, Union[float, bool], is_optional=False)}, + ), + {"foo": 10.1}, + Union[float, bool], + id="merge-union-into-any-incompatible_type", + ), + param( + ( + {"foo": UnionNode(10.1, Union[float, bool], is_optional=False)}, + {"foo": UnionNode(True, Union[float, bool], is_optional=True)}, + ), + {"foo": True}, + Union[float, bool], + id="merge-two-unions", + ), + param( + ( + {"foo": UnionNode(10.1, Union[float, bool], is_optional=True)}, + {"foo": UnionNode(True, Union[float, bool], is_optional=False)}, + ), + {"foo": True}, + Optional[Union[float, bool]], + id="merge-two-unions-lhs-optional", + ), + param( + ( + {"foo": UnionNode(10.1, Union[float, bool])}, + {"foo": {"bar": "baz"}}, + ), + raises(ValidationError), + None, + id="bad-merge-dict", + ), + param( + ( + {"foo": UnionNode(10.1, Union[float, bool])}, + {"foo": [123]}, + ), + raises(ValidationError), + None, + id="bad-merge-list", + ), + param( + ( + {"foo": UnionNode(10.1, Union[float, bool])}, + {"foo": User("bond", 7)}, + ), + raises(ValidationError), + None, + id="bad-merge-structured-into-union", + ), + param( + ( + DictConfig({"foo": 10.1}, element_type=Union[float, bool]), + {"foo": User("bond", 7)}, + ), + raises(ValidationError), + None, + id="bad-merge-structured-into-union_elt_type", + ), + param( + ( + {"foo": UnionNode(10.1, Union[float, bool])}, + DictConfig({"foo": User("bond", 7)}, element_type=User), + ), + raises(ValidationError), + None, + id="bad-merge-structured_element_type-into-union", + ), + param( + ( + DictConfig({"foo": 10.1}, element_type=Union[float, bool]), + DictConfig({"foo": User("bond", 7)}, element_type=User), + ), + raises(ValidationError), + None, + id="bad-merge-structured_element_type-into-union_elt_type", + ), + param( + ( + {"foo": User("bond", 7)}, + {"foo": UnionNode(10.1, Union[float, bool])}, + ), + raises(ValidationError), + None, + id="bad-merge-union-into-structured", + ), + param( + ( + DictConfig({"foo": User("bond", 7)}, element_type=User), + {"foo": UnionNode(10.1, Union[float, bool])}, + ), + raises(ValidationError), + None, + id="bad-merge-union-into-structured_element_type", + ), + param( + ( + {"foo": User("bond", 7)}, + DictConfig({"foo": 10.1}, element_type=Union[float, bool]), + ), + raises(ValidationError), + None, + id="bad-merge-union_element_type-into-structured", + ), + param( + ( + DictConfig({"foo": User("bond", 7)}, element_type=User), + DictConfig({"foo": 10.1}, element_type=Union[float, bool]), + ), + raises(ValidationError), + None, + id="bad-merge-union_element_type-into-structured_element_type", + ), + param( + ( + {"foo": UnionNode(10.1, Union[float, bool], is_optional=False)}, + {"foo": None}, + ), + raises(ValidationError), + None, + id="bad-merge-none", + ), + param( + ( + {"foo": UnionNode(10.1, Union[float, bool], is_optional=True)}, + {"foo": None}, + ), + {"foo": None}, + Optional[Union[float, bool]], + id="merge-none-into-union", + ), + param( + ( + {"foo": None}, + {"foo": UnionNode(10.1, Union[float, bool], is_optional=True)}, + ), + {"foo": 10.1}, + Optional[Union[float, bool]], + id="merge-union-into-none", + ), + ], +) +def test_union_merge(inputs: Any, expected: Any, type_hint: Any) -> None: + configs = [_ensure_container(c) for c in inputs] + if isinstance(expected, RaisesContext): + with expected: + OmegaConf.merge(*configs) + else: + merged = OmegaConf.merge(*configs) + assert merged == expected + + assert isinstance(merged, DictConfig) + node = merged._get_node("foo") + assert isinstance(node, Node) + assert get_type_hint(node) == type_hint + + +@mark.parametrize( + "lhs", + [ + param({"foo": 10.1}, id="10.1"), + param({"foo": "abc"}, id="abc"), + param({"foo": True}, id="True"), + ], +) +@mark.parametrize( + "rhs", + [ + param({"foo": 10.1}, id="10.1"), + param({"foo": "abc"}, id="abc"), + param({"foo": True}, id="True"), + ], +) +def test_union_merge_matrix( + lhs: Any, + rhs: Any, +) -> None: + lhs = _ensure_container(lhs) + rhs = _ensure_container(rhs) + lnode = lhs._get_node("foo") + rnode = rhs._get_node("foo") + lvalue = _get_value(lnode) + rvalue = _get_value(rnode) + # lvk = get_value_kind(lnode) + rvk = get_value_kind(rnode) + rmissing = rvk is ValueKind.MANDATORY_MISSING + + can_merge = True + if can_merge: + merged = OmegaConf.merge(lhs, rhs) + if rmissing: + assert merged == {"foo": lvalue} + else: + assert merged == {"foo": rvalue} + + else: + with raises(ValidationError): + OmegaConf.merge(lhs, rhs) + + +@mark.parametrize( + "r_val", + [ + param(20.2, id="20.2"), + param(MISSING, id="missing"), + param(None, id="none"), + param("${interp}", id="interp"), + ], +) +@mark.parametrize( + "r_element_type", + [ + param(Any, id="any"), + param(Optional[float], id="float"), # X + param(Optional[Union[float, bytes]], id="union"), + param(Optional[Union[str, float]], id="different_union"), + ], +) +@mark.parametrize( + "l_val", + [ + param(10.1, id="10.1"), + param(MISSING, id="missing"), + param(None, id="none"), + param("${interp}", id="interp"), + param("NO_LVAL", id="no_lval"), + ], +) +@mark.parametrize( + "l_element_type", + [ + param(Any, id="any"), + param(Optional[float], id="float"), + param(Optional[Union[float, bytes]], id="union"), # X + ], +) +def test_union_merge_special( + r_val: Any, l_val: Any, r_element_type: Any, l_element_type: Any +) -> None: + left = DictConfig( + {"foo": l_val} if l_val != "NO_LVAL" else {}, element_type=l_element_type + ) + right = DictConfig({"foo": r_val}, element_type=r_element_type) + + merged = OmegaConf.merge(left, right) + + if l_val == "NO_LVAL" or r_val != MISSING: + assert merged == right + else: + assert merged == left + + if l_element_type is not Any: + assert get_type_hint(merged, "foo") == l_element_type + else: + assert get_type_hint(merged, "foo") == r_element_type + + def test_merge_error_retains_type() -> None: cfg = OmegaConf.structured(ConcretePlugin) with raises(ValidationError): diff -pruN 2.1.0~rc1-3/tests/test_nested_containers.py 2.2.2-1/tests/test_nested_containers.py --- 2.1.0~rc1-3/tests/test_nested_containers.py 1970-01-01 00:00:00.000000000 +0000 +++ 2.2.2-1/tests/test_nested_containers.py 2022-05-27 21:36:40.000000000 +0000 @@ -0,0 +1,1431 @@ +import copy +import re +from typing import Any, Dict, List, Optional, Union + +from pytest import mark, param, raises + +from omegaconf import ( + MISSING, + Container, + DictConfig, + IntegerNode, + KeyValidationError, + ListConfig, + Node, + OmegaConf, + ValidationError, +) +from omegaconf._utils import ( + ValueKind, + _ensure_container, + _resolve_optional, + get_value_kind, + is_dict_annotation, + is_list_annotation, + is_structured_config, +) +from tests import ConcretePlugin, Plugin + + +def check_node_metadata( + node: Container, + type_hint: Any, + key_type: Any, + elt_type: Any, + obj_type: Any, +) -> None: + value_optional, value_ref_type = _resolve_optional(type_hint) + assert node._metadata.optional == value_optional + assert node._metadata.ref_type == value_ref_type + assert node._metadata.key_type == key_type + assert node._metadata.element_type == elt_type + assert node._metadata.object_type == obj_type + + if is_dict_annotation(value_ref_type) or is_structured_config(value_ref_type): + assert isinstance(node, DictConfig) + elif is_list_annotation(value_ref_type): + assert isinstance(node, ListConfig) + + +def check_subnode( + cfg: Container, + key: Any, + value: Any, + type_hint: Any, + key_type: Any, + elt_type: Any, + obj_type: Any, +) -> None: + "Validate that `cfg[key] == value` and that subnode `cfg._get_node(key)._metadata` is correct." + node = cfg._get_node(key) + assert isinstance(node, (ListConfig, DictConfig)) + vk = get_value_kind(node) + + if vk in (ValueKind.MANDATORY_MISSING, ValueKind.INTERPOLATION): + if isinstance(value, Node): + value = value._value() + assert node._value() == value + else: + assert cfg[key] == value + + check_node_metadata(node, type_hint, key_type, elt_type, obj_type) + + +@mark.parametrize( + "cfg, type_hint, key_type, elt_type, obj_type", + [ + param( + ListConfig([[[456]]], element_type=List[List[int]]), + List[List[int]], + int, + List[int], + list, + id="list-list-list", + ), + param( + ListConfig([{"foo": {"bar": 456}}], element_type=Dict[str, Dict[str, int]]), + Dict[str, Dict[str, int]], + str, + Dict[str, int], + dict, + id="list-dict-dict", + ), + param( + ListConfig([[123], None], element_type=Optional[List[int]]), + Optional[List[int]], + int, + int, + list, + id="list-optional-list", + ), + param( + ListConfig([[123], [None]], element_type=List[Optional[int]]), + List[Optional[int]], + int, + Optional[int], + list, + id="list-list-optional", + ), + param( + ListConfig([{"bar": 456}, None], element_type=Optional[Dict[str, int]]), + Optional[Dict[str, int]], + str, + int, + dict, + id="list-optional-dict", + ), + param( + ListConfig( + [{"foo": 456}, {"bar": None}], element_type=Dict[str, Optional[int]] + ), + Dict[str, Optional[int]], + str, + Optional[int], + dict, + id="list-dict-optional", + ), + param( + DictConfig({"foo": [[456]]}, element_type=List[List[int]]), + List[List[int]], + int, + List[int], + list, + id="dict-list-list", + ), + param( + DictConfig( + {"foo": {"bar": {"baz": 456}}}, element_type=Dict[str, Dict[str, int]] + ), + Dict[str, Dict[str, int]], + str, + Dict[str, int], + dict, + id="dict-dict-dict", + ), + param( + DictConfig({"foo": [123], "bar": None}, element_type=Optional[List[int]]), + Optional[List[int]], + int, + int, + list, + id="dict-optional-list", + ), + param( + DictConfig({"foo": [123], "bar": [None]}, element_type=List[Optional[int]]), + List[Optional[int]], + int, + Optional[int], + list, + id="dict-list-optional", + ), + param( + DictConfig( + {"foo": {"bar": 456}, "baz": None}, + element_type=Optional[Dict[str, int]], + ), + Optional[Dict[str, int]], + str, + int, + dict, + id="dict-optional-dict", + ), + param( + DictConfig( + {"foo": {"bar": 456}, "baz": {"qux": None}}, + element_type=Dict[str, Optional[int]], + ), + Dict[str, Optional[int]], + str, + Optional[int], + dict, + id="dict-dict-optional", + ), + param( + DictConfig( + {"foo": {"bar": ConcretePlugin()}}, element_type=Dict[str, Plugin] + ), + Dict[str, Plugin], + str, + Plugin, + dict, + id="dict-of-plugin", + ), + param( + DictConfig({"foo": [ConcretePlugin()]}, element_type=List[Plugin]), + List[Plugin], + int, + Plugin, + list, + id="list-of-plugin", + ), + ], +) +def test_container_nested_element( + cfg: Union[DictConfig, ListConfig], + type_hint: Any, + key_type: Any, + elt_type: Any, + obj_type: Any, +) -> None: + """Ensure metadata and contents of container-typed subnode are correct""" + cfg = copy.deepcopy(cfg) + keys: Any = range(len(cfg)) if isinstance(cfg, ListConfig) else cfg.keys() + for key in keys: + value = cfg[key] + check_subnode( + cfg, + key, + value, + type_hint, + key_type, + elt_type, + obj_type if value is not None else None, + ) + + +@mark.parametrize( + "cfg, value, type_hint, key_type, elt_type, obj_type", + [ + param( + ListConfig([[[456]]], element_type=List[List[int]]), + [[123]], + List[List[int]], + int, + List[int], + list, + id="assign-to-list-element", + ), + param( + ListConfig([{"foo": {"bar": 456}}], element_type=Dict[str, Dict[str, int]]), + {"baz": {"qux": 123}}, + Dict[str, Dict[str, int]], + str, + Dict[str, int], + dict, + id="assign-to-dict-element", + ), + param( + ListConfig([[123], None], element_type=Optional[List[int]]), + [456], + Optional[List[int]], + int, + int, + list, + id="assign-list-to-optional-list", + ), + param( + ListConfig([{"foo": 456}, None], element_type=Optional[Dict[str, int]]), + {"bar": 123}, + Optional[Dict[str, int]], + str, + int, + dict, + id="assign-dict-to-optional-dict", + ), + param( + ListConfig([[123], [None]], element_type=List[Optional[int]]), + [456], + List[Optional[int]], + int, + Optional[int], + list, + id="assign-list-to-list-optional", + ), + param( + ListConfig([[123], [None]], element_type=List[Optional[int]]), + [None], + List[Optional[int]], + int, + Optional[int], + list, + id="assign-list-none-to-list-optional", + ), + param( + ListConfig( + [{"foo": 456}, {"bar": None}], element_type=Dict[str, Optional[int]] + ), + {"baz": 123}, + Dict[str, Optional[int]], + str, + Optional[int], + dict, + id="assign-dict-to-dict-optional", + ), + param( + ListConfig( + [{"foo": 456}, {"bar": None}], element_type=Dict[str, Optional[int]] + ), + {"baz": None}, + Dict[str, Optional[int]], + str, + Optional[int], + dict, + id="assign-dict-none-to-dict-optional", + ), + param( + ListConfig([{"foo": ConcretePlugin()}], element_type=Dict[str, Plugin]), + {"bar": ConcretePlugin()}, + Dict[str, Plugin], + str, + Plugin, + dict, + id="assign-dict-plugin", + ), + param( + ListConfig([[ConcretePlugin()]], element_type=List[Plugin]), + [ConcretePlugin()], + List[Plugin], + int, + Plugin, + list, + id="assign-list-plugin", + ), + ], +) +@mark.parametrize( + "ensure_container", + [ + param(True, id="container"), + param(False, id="no_container"), + ], +) +def test_list_assign_to_container_typed_element( + cfg: ListConfig, + value: Any, + type_hint: Any, + key_type: Any, + elt_type: Any, + obj_type: Any, + ensure_container: bool, +) -> None: + cfg = copy.deepcopy(cfg) + if ensure_container: + value = _ensure_container(value) + + n = len(cfg) + for idx in range(n): + cfg[idx] = value + check_subnode(cfg, idx, value, type_hint, key_type, elt_type, obj_type) + + cfg.append(value) + check_subnode(cfg, n, value, type_hint, key_type, elt_type, obj_type) + + +@mark.parametrize( + "cfg, type_hint, key_type, elt_type", + [ + param( + ListConfig([[123], None], element_type=Optional[List[int]]), + Optional[List[int]], + int, + int, + id="assign-to-optional-list", + ), + param( + ListConfig([{"bar": 456}, None], element_type=Optional[Dict[str, int]]), + Optional[Dict[str, int]], + str, + int, + id="assign-to-optional-dict", + ), + param( + ListConfig([[ConcretePlugin()], None], element_type=Optional[List[Plugin]]), + Optional[List[Plugin]], + int, + Plugin, + id="assign-to-optional-plugin-list", + ), + param( + ListConfig( + [{"bar": ConcretePlugin()}, None], + element_type=Optional[Dict[str, Plugin]], + ), + Optional[Dict[str, Plugin]], + str, + Plugin, + id="assign-to-optional-plugin-dict", + ), + ], +) +@mark.parametrize( + "value", + [ + param(None, id="none"), + param(MISSING, id="missing"), + param("${interp}", id="interp"), + ], +) +def test_list_assign_to_container_typed_element_special( + cfg: ListConfig, + value: Any, + type_hint: Any, + key_type: Any, + elt_type: Any, +) -> None: + cfg = copy.deepcopy(cfg) + + n = len(cfg) + for idx in range(n): + cfg[idx] = value + check_subnode(cfg, idx, value, type_hint, key_type, elt_type, None) + + cfg.append(value) + check_subnode(cfg, n, value, type_hint, key_type, elt_type, None) + + +@mark.parametrize( + "ensure_container", + [ + param(True, id="container"), + param(False, id="no_container"), + ], +) +@mark.parametrize( + "cfg, value, type_hint, key_type, elt_type, obj_type", + [ + param( + DictConfig({"foo": [[456]]}, element_type=List[List[int]]), + [[123]], + List[List[int]], + int, + List[int], + list, + id="assign-to-list-element", + ), + param( + DictConfig( + {"foo": {"bar": {"baz": 456}}}, element_type=Dict[str, Dict[str, int]] + ), + {"qux": {"frob": 123}}, + Dict[str, Dict[str, int]], + str, + Dict[str, int], + dict, + id="assign-to-dict-element", + ), + param( + DictConfig({"foo": [123], "bar": None}, element_type=Optional[List[int]]), + [456], + Optional[List[int]], + int, + int, + list, + id="assign-list-to-optional-list", + ), + param( + DictConfig( + {"foo": {"bar": 456}, "baz": None}, + element_type=Optional[Dict[str, int]], + ), + {"qux": 123}, + Optional[Dict[str, int]], + str, + int, + dict, + id="assign-dict-to-optional-dict", + ), + param( + DictConfig({"foo": [123], "bar": [None]}, element_type=List[Optional[int]]), + [456], + List[Optional[int]], + int, + Optional[int], + list, + id="assign-list-to-list-optional", + ), + param( + DictConfig({"foo": [123], "bar": [None]}, element_type=List[Optional[int]]), + [None], + List[Optional[int]], + int, + Optional[int], + list, + id="assign-list-none-to-list-optional", + ), + param( + DictConfig( + {"foo": {"bar": 456}, "baz": {"qux": None}}, + element_type=Dict[str, Optional[int]], + ), + {"frob": 123}, + Dict[str, Optional[int]], + str, + Optional[int], + dict, + id="assign-dict-to-dict-optional", + ), + param( + DictConfig( + {"foo": {"bar": 456}, "baz": {"qux": None}}, + element_type=Dict[str, Optional[int]], + ), + {"frob": None}, + Dict[str, Optional[int]], + str, + Optional[int], + dict, + id="assign-dict-none-to-dict-optional", + ), + param( + DictConfig({"foo": [ConcretePlugin()]}, element_type=List[Plugin]), + [ConcretePlugin()], + List[Plugin], + int, + Plugin, + list, + id="assign-to-list-of-plugins", + ), + param( + DictConfig( + {"foo": {"bar": ConcretePlugin()}}, element_type=Dict[str, Plugin] + ), + {"baz": ConcretePlugin()}, + Dict[str, Plugin], + str, + Plugin, + dict, + id="assign-to-dict-of-plugins", + ), + param( + DictConfig({"key": []}, element_type=List[int]), + DictConfig("${interp}"), + List[int], + int, + int, + None, + id="coerce-dictconfig-interp-to-listconfig", + ), + param( + DictConfig({"key": {}}, element_type=Dict[str, int]), + ListConfig("${interp}"), + Dict[str, int], + str, + int, + None, + id="coerce-listconfig-interp-to-dictconfig", + ), + param( + DictConfig({"key": []}, element_type=List[int]), + DictConfig("${interp}", ref_type=Dict[str, int]), + List[int], + int, + int, + None, + id="coerce-dictconfig-interp_with_ref-to-listconfig", + ), + param( + DictConfig({"key": {}}, element_type=Dict[str, int]), + ListConfig("${interp}", ref_type=List[int]), + Dict[str, int], + str, + int, + None, + id="coerce-listconfig-interp_with_ref-to-dictconfig", + ), + param( + DictConfig({"key": []}, element_type=List[int]), + DictConfig(MISSING), + List[int], + int, + int, + None, + id="coerce-dictconfig-missing-to-listconfig", + ), + param( + DictConfig({"key": {}}, element_type=Dict[str, int]), + ListConfig(MISSING), + Dict[str, int], + str, + int, + None, + id="coerce-listconfig-missing-to-dictconfig", + ), + param( + DictConfig({"key": []}, element_type=List[int]), + DictConfig(MISSING, ref_type=Optional[Dict[str, int]]), + List[int], + int, + int, + None, + id="coerce-dictconfig-missing_with_ref-to-listconfig", + ), + param( + DictConfig({"key": {}}, element_type=Dict[str, int]), + ListConfig(MISSING, ref_type=Optional[List[int]]), + Dict[str, int], + str, + int, + None, + id="coerce-listconfig-missing_with_ref-to-dictconfig", + ), + param( + DictConfig({"key": []}, element_type=Optional[List[int]]), + DictConfig(None), + Optional[List[int]], + int, + int, + None, + id="coerce-dictconfig-none-to-listconfig", + ), + param( + DictConfig({"key": {}}, element_type=Optional[Dict[str, int]]), + ListConfig(None), + Optional[Dict[str, int]], + str, + int, + None, + id="coerce-listconfig-none-to-dictconfig", + ), + param( + DictConfig({"key": []}, element_type=Optional[List[int]]), + DictConfig(None, ref_type=Optional[Dict[str, int]]), + Optional[List[int]], + int, + int, + None, + id="coerce-dictconfig-none_with_ref-to-listconfig", + ), + param( + DictConfig({"key": {}}, element_type=Optional[Dict[str, int]]), + ListConfig(None, ref_type=Optional[List[int]]), + Optional[Dict[str, int]], + str, + int, + None, + id="coerce-listconfig-none_with_ref-to-dictconfig", + ), + ], +) +def test_dict_assign_to_container_typed_element( + cfg: DictConfig, + value: Any, + type_hint: Any, + key_type: Any, + elt_type: Any, + obj_type: Any, + ensure_container: bool, +) -> None: + cfg = copy.deepcopy(cfg) + if ensure_container: + value = _ensure_container(value) + + for key in cfg: + cfg[key] = value + check_subnode(cfg, key, value, type_hint, key_type, elt_type, obj_type) + + cfg["_new_key"] = value + check_subnode(cfg, "_new_key", value, type_hint, key_type, elt_type, obj_type) + + +@mark.parametrize( + "dc,value", + [ + param(DictConfig({"key": 123}, element_type=int), 456, id="int"), + param(DictConfig({"key": [123]}, element_type=List[int]), [456], id="list"), + param( + DictConfig({"key": {"foo": 123}}, element_type=Dict[str, int]), + {"baz": 456}, + id="dict", + ), + ], +) +@mark.parametrize("overwrite_preexisting_key", [True, False]) +def test_setitem_valid_element_type( + dc: DictConfig, value: Any, overwrite_preexisting_key: bool +) -> None: + dc = copy.deepcopy(dc) + if not overwrite_preexisting_key: + del dc["key"] + dc["key"] = value + assert dc["key"] == value + + +@mark.parametrize( + "cfg, type_hint, key_type, elt_type", + [ + param( + DictConfig({"foo": [123], "bar": None}, element_type=Optional[List[int]]), + Optional[List[int]], + int, + int, + id="assign-to-optional-list", + ), + param( + DictConfig( + {"foo": {"bar": 456}, "baz": None}, + element_type=Optional[Dict[str, int]], + ), + Optional[Dict[str, int]], + str, + int, + id="assign-to-optional-dict", + ), + ], +) +@mark.parametrize( + "value", + [ + param(None, id="none"), + param(MISSING, id="missing"), + param("${interp}", id="interp"), + ], +) +def test_dict_assign_to_container_typed_element_special( + cfg: DictConfig, + value: Any, + type_hint: Any, + key_type: Any, + elt_type: Any, +) -> None: + cfg = copy.deepcopy(cfg) + + for key in cfg: + cfg[key] = value + check_subnode(cfg, key, value, type_hint, key_type, elt_type, None) + + cfg["_new_key"] = value + check_subnode(cfg, "_new_key", value, type_hint, key_type, elt_type, None) + + +@mark.parametrize( + "ensure_container", + [ + param(True, id="container"), + param(False, id="no_container"), + ], +) +@mark.parametrize( + "overwrite_preexisting_key", + [ + param(True, id="overwrite"), + param(False, id="no_overwrite"), + ], +) +@mark.parametrize( + "dc,value,err_msg", + [ + param( + DictConfig({"key": 123}, element_type=int), + "foo", + re.escape("Value 'foo' of type 'str' could not be converted to Integer"), + id="assign-str-to-int", + ), + param( + DictConfig({"key": [123]}, element_type=List[int]), + "foo", + re.escape("Invalid value assigned: str is not a ListConfig, list or tuple"), + id="assign-str-to-list[int]", + ), + param( + DictConfig({"key": {"foo": 123}}, element_type=Dict[str, int]), + "bar", + re.escape("Cannot assign str to Dict[str, int]"), + id="assign-str-to-list[int]", + ), + param( + DictConfig({"key": [123]}, element_type=List[int]), + None, + re.escape("field 'key' is not Optional"), + id="assign-none-to-list[int]", + ), + param( + DictConfig({"key": [123]}, element_type=List[int]), + 456, + re.escape("Invalid value assigned: int is not a ListConfig, list or tuple"), + id="assign-int-to-list[int]", + ), + param( + DictConfig({"key": [123]}, element_type=List[int]), + ["foo"], + r"(Value 'foo' of type 'str' could not be converted to Integer)" + + r"|(Value 'foo' \(str\) is incompatible with type hint 'int')", + id="assign-list[str]-to-list[int]", + ), + param( + DictConfig({"key": [123]}, element_type=List[int]), + [None], + r"(Invalid type assigned: NoneType is not a subclass of int)" + + r"|(Incompatible value 'None' for field of type 'int')", + id="assign-list[none]-to-list[int]", + ), + param( + DictConfig({"key": [123]}, element_type=List[int]), + {"baz": 456}, + r"(Invalid value assigned: dict is not a ListConfig, list or tuple)" + + r"|(Invalid value assigned: DictConfig is not a ListConfig, list or tuple)" + + r"|(Invalid value assigned: dict does not match type hint typing\.List\[int\])" + + r"|('DictConfig' is incompatible with type hint 'typing\.List\[int\]')", + id="assign-dict[str-int]-to-list[int]]", + ), + param( + DictConfig({"key": {"key2": 123}}, element_type=Dict[str, int]), + {"key2": "foo"}, + r"(Value 'foo' \(str\) is incompatible with type hint 'int')" + + r"|(Value 'foo' of type 'str' could not be converted to Integer)", + id="assign_dict[str_str]_to_dict[str_int]", + ), + param( + DictConfig({"key": {"key2": 123}}, element_type=Dict[str, int]), + [], + r"(Cannot assign list to Dict\[str, int\])" + + r"|('ListConfig' is incompatible with type hint 'typing.Dict\[str, int\]')", + id="assign_list_to_dict[str_int]", + ), + param( + DictConfig({"key": [[123]]}, element_type=List[List[int]]), + [[456.789]], + r"(Value 456\.789 \(float\) is incompatible with type hint 'int')" + + r"|(Value '456\.789' of type 'float' could not be converted to Integer)", + id="assign-list[list[float]]-to-list[list[int]]", + ), + param( + DictConfig({"key": [[123]]}, element_type=List[List[int]]), + [[None]], + r"(Invalid type assigned: NoneType is not a subclass of int)" + + r"|(Incompatible value 'None' for field of type 'int')", + id="assign-list[list[none]]-to-list[list[int]]", + ), + param( + DictConfig({"key": [[123]]}, element_type=List[List[int]]), + [[IntegerNode(None)]], + r"(Value None \(NoneType\) is incompatible with type hint 'int')" + + r"|(Incompatible value 'None' for field of type 'int')", + id="assign-list[list[typed-none]]-to-list[list[int]]", + ), + param( + DictConfig({"key": [[123.456]]}, element_type=List[List[float]]), + [[IntegerNode(789)]], + re.escape("Value 789 (int) is incompatible with type hint 'float'"), + id="assign-list[list[typed-int]]-to-list[list[float]]", + ), + param( + DictConfig( + {"key": {"foo": {"bar": 123}}}, element_type=Dict[str, Dict[str, int]] + ), + {"foo": {"bar": 456.789}}, + r"(Value 456\.789 \(float\) is incompatible with type hint 'int')" + + r"|(Value '456\.789' of type 'float' could not be converted to Integer)", + id="assign-dict[str-[dict[str-float]]]-to-dict[str[dict[str-int]]]", + ), + param( + DictConfig( + {"key": {"foo": {"bar": 123}}}, element_type=Dict[str, Dict[str, int]] + ), + {"foo": {"bar2": 456.789}}, + r"(Value 456\.789 \(float\) is incompatible with type hint 'int')" + + r"|(Value '456\.789' of type 'float' could not be converted to Integer)", + id="assign-dict[str-[dict[str-float]]]-to-dict[str[dict[str-int]]]-2", + ), + param( + DictConfig( + {"key": {"foo": {"bar": 123}}}, element_type=Dict[str, Dict[str, int]] + ), + {"foo": {123: 456}}, + r"(Key 123 \(int\) is incompatible with \(str\))" + + r"|(Key 123 \(int\) is incompatible with key type hint 'str')", + id="assign-dict[str_[dict[int_int]]]-to-dict[str[dict[str_int]]]", + ), + param( + DictConfig( + {"key": {"foo": {"bar": 123}}}, element_type=Dict[str, Dict[str, int]] + ), + {"foo": {456: 789}}, + r"(Key 456 \(int\) is incompatible with \(str\))" + + r"|(Key 456 \(int\) is incompatible with key type hint 'str')", + id="assign-dict[str_[dict[int-int]]]-to-dict[str[dict[str_int]]]", + ), + param( + DictConfig( + {"key": {"foo": {"bar": 123}}}, element_type=Dict[str, Dict[str, int]] + ), + {"foo": {456: IntegerNode(None)}}, + r"(Key 456 \(int\) is incompatible with \(str\))" + + r"|(Key 456 \(int\) is incompatible with key type hint 'str')", + id="assign-dict[str_[dict[int-typed_none]]]-to-dict[str[dict[str_int]]]", + ), + param( + DictConfig( + {"key": {"foo": {"bar": 123.456}}}, + element_type=Dict[str, Dict[str, float]], + ), + {"foo": {"bar": IntegerNode(789)}}, + re.escape("Value 789 (int) is incompatible with type hint 'float'"), + id="assign-dict[str-[dict[int_typed-int]]]-to-dict[str[dict[str-float]]]", + ), + param( + DictConfig( + {"key": {"foo": {"bar": 123.456}}}, + element_type=Dict[str, Dict[str, float]], + ), + {"foo": {"bar2": IntegerNode(789)}}, + re.escape("Value 789 (int) is incompatible with type hint 'float'"), + id="assign-dict[str-[dict[int_typed-int]]]-to-dict[str[dict[str_float]]]-2", + ), + ], +) +def test_dict_setitem_invalid_element_type( + dc: DictConfig, + value: Any, + err_msg: str, + ensure_container: bool, + overwrite_preexisting_key: bool, +) -> None: + dc_orig = dc + dc = copy.deepcopy(dc) + + if ensure_container: + if isinstance(value, (dict, list)): + value = _ensure_container(value) + else: + return # skip + + if overwrite_preexisting_key: + with raises((ValidationError, KeyValidationError), match=err_msg): + dc["key"] = value + assert dc == dc_orig + else: + del dc["key"] + with raises((ValidationError, KeyValidationError), match=err_msg): + dc["key"] = value + assert dc == {} + + +@mark.parametrize( + "lc,index,value,err_msg", + [ + param( + ListConfig([123], element_type=int), + 0, + "foo", + "Value 'foo' of type 'str' could not be converted to Integer", + id="assign_str_to_int", + ), + param( + ListConfig([123], element_type=int), + 0, + None, + re.escape("[0] is not optional and cannot be assigned None"), + id="assign_none_to_int", + ), + param( + ListConfig([[123]], element_type=List[int]), + 0, + "foo", + "Invalid value assigned: str is not a ListConfig, list or tuple", + id="assign_str_to_list[int]", + ), + param( + ListConfig([{"key": 123}], element_type=Dict[str, int]), + 0, + "foo", + re.escape("Cannot assign str to Dict[str, int]"), + id="assign_str_to_dict[str, int]", + ), + param( + ListConfig([[123]], element_type=List[int]), + 0, + None, + re.escape("[0] is not optional and cannot be assigned None"), + id="assign_none_to_list[int]", + ), + param( + ListConfig([[123]], element_type=List[int]), + 0, + 456, + "Invalid value assigned: int is not a ListConfig, list or tuple", + id="assign_int_to_list[int]", + ), + param( + ListConfig([[123]], element_type=List[int]), + 0, + ["foo"], + "Value 'foo' of type 'str' could not be converted to Integer", + id="assign_list[str]_to_list[int]", + ), + param( + ListConfig([[123]], element_type=List[int]), + 0, + [None], + re.escape("Invalid type assigned: NoneType is not a subclass of int"), + id="assign_list[none]_to_list[int]", + ), + param( + ListConfig([[123]], element_type=List[int]), + 0, + {"baz": 456}, + "Invalid value assigned: dict", + id="assign_dict[str,int]_to_list[int]]", + ), + param( + ListConfig([{"key": 123}], element_type=Dict[str, int]), + 0, + {"key2": "foo"}, + "Value 'foo' of type 'str' could not be converted to Integer", + id="assign_dict[str,str]_to_dict[str,int]", + ), + param( + ListConfig([{"key2": 123}], element_type=Dict[str, int]), + 0, + {"key2": "foo"}, + "Value 'foo' of type 'str' could not be converted to Integer", + id="assign_dict[str,str]_to_dict[str,int]", + ), + param( + ListConfig([], element_type=int), + None, + "foo", + "Value 'foo' of type 'str' could not be converted to Integer", + id="append_str_to_int", + ), + param( + ListConfig([], element_type=int), + None, + None, + "Invalid type assigned: NoneType is not a subclass of int", + id="append_none_to_int", + ), + param( + ListConfig([], element_type=List[int]), + None, + "foo", + "Invalid value assigned: str is not a ListConfig, list or tuple", + id="append_str_to_list[int]", + ), + param( + ListConfig([], element_type=Dict[str, int]), + None, + "foo", + re.escape("Cannot assign str to Dict[str, int]"), + id="append_str_to_dict[str, int]", + ), + param( + ListConfig([], element_type=List[int]), + None, + None, + re.escape("Invalid type assigned: NoneType is not a subclass of List[int]"), + id="append_none_to_list[int]", + ), + param( + ListConfig([], element_type=List[int]), + None, + 456, + "Invalid value assigned: int is not a ListConfig, list or tuple", + id="append_int_to_list[int]", + ), + param( + ListConfig([], element_type=List[int]), + None, + ["foo"], + "Value 'foo' of type 'str' could not be converted to Integer", + id="append_list[str]_to_list[int]", + ), + param( + ListConfig([], element_type=List[int]), + None, + [None], + re.escape("Invalid type assigned: NoneType is not a subclass of int"), + id="append_list[none]_to_list[int]", + ), + param( + ListConfig([], element_type=List[int]), + None, + {"baz": 456}, + "Invalid value assigned: dict", + id="append_dict[str,int]_to_list[int]]", + ), + param( + ListConfig([], element_type=Dict[str, int]), + None, + {"key2": "foo"}, + "Value 'foo' of type 'str' could not be converted to Integer", + id="append_dict[str,str]_to_dict[str,int]", + ), + param( + ListConfig([], element_type=Dict[str, int]), + None, + {"key2": "foo"}, + "Value 'foo' of type 'str' could not be converted to Integer", + id="append_dict[str,str]_to_dict[str,int]", + ), + param( + ListConfig( + [{"key2": ConcretePlugin()}], element_type=Dict[str, ConcretePlugin] + ), + 0, + {"key": Plugin()}, + "Invalid type assigned: Plugin is not a subclass of ConcretePlugin", + id="append_dict[str,str]_to_dict[str,int]", + ), + param( + ListConfig([[ConcretePlugin()]], element_type=List[ConcretePlugin]), + 0, + [Plugin()], + "Invalid type assigned: Plugin is not a subclass of ConcretePlugin", + id="append_dict[str,str]_to_dict[str,int]", + ), + param( + ListConfig([], element_type=Dict[str, ConcretePlugin]), + None, + {"key": Plugin()}, + "Invalid type assigned: Plugin is not a subclass of ConcretePlugin", + id="append_dict[str,str]_to_dict[str,int]", + ), + param( + ListConfig([], element_type=List[ConcretePlugin]), + None, + [Plugin()], + "Invalid type assigned: Plugin is not a subclass of ConcretePlugin", + id="append_dict[str,str]_to_dict[str,int]", + ), + ], +) +def test_list_setitem_invalid_element_type( + lc: ListConfig, + index: Optional[int], + value: Any, + err_msg: str, +) -> None: + lc_orig = lc + lc = copy.deepcopy(lc) + with raises(ValidationError, match=err_msg): + if index is None: + lc.append(value) + else: + lc[index] = value + assert lc == lc_orig + + +@mark.parametrize( + "dc1, dc2, value, type_hint, key_type, elt_type, obj_type", + [ + param( + DictConfig({"key": {"key2": Plugin()}}, element_type=Dict[str, Plugin]), + DictConfig({"key": {"key2": ConcretePlugin()}}, element_type=Any), + ConcretePlugin(), + Plugin, + Any, + Any, + ConcretePlugin, + id="any-plugin-into-typed-plugin", + ), + param( + DictConfig({"key": {"key2": Plugin()}}, element_type=Any), + DictConfig( + {"key": {"key2": ConcretePlugin()}}, element_type=Dict[str, Plugin] + ), + ConcretePlugin(), + Plugin, + Any, + Any, + ConcretePlugin, + id="typed-plugin-into-any-plugin", + ), + param( + DictConfig({"key": {"key2": Plugin()}}, element_type=Dict[str, Plugin]), + DictConfig( + {"key": {"key2": ConcretePlugin()}}, + element_type=Dict[str, ConcretePlugin], + ), + ConcretePlugin(), + Plugin, + Any, + Any, + ConcretePlugin, + id="typed-concrete-plugin-into-typed-plugin", + ), + param( + DictConfig({"key": {"key2": {}}}), + DictConfig({"key": {"key2": Plugin()}}, element_type=Dict[str, Plugin]), + Plugin(), + Plugin, + Any, + Any, + Plugin, + id="typed-plugin-into-any", + ), + ], +) +def test_merge_nested_dict_promotion( + dc1: DictConfig, + dc2: DictConfig, + value: Any, + type_hint: Any, + key_type: Any, + elt_type: Any, + obj_type: Any, +) -> None: + cfg = OmegaConf.merge(dc1, dc2) + check_subnode( + cfg.key, + key="key2", + value=value, + type_hint=type_hint, + key_type=key_type, + elt_type=elt_type, + obj_type=obj_type, + ) + + +@mark.parametrize( + "configs, keys, value, type_hint, key_type, elt_type, obj_type", + [ + param( + [ + DictConfig({}, element_type=Dict[str, List[int]]), + DictConfig({"foo": {"bar": "${interp}"}}, element_type=Dict[str, Any]), + ], + ["foo", "bar"], + "${interp}", + List[int], + int, + int, + None, + id="merge-interp-into-list", + ), + param( + [ + DictConfig({}, element_type=Dict[str, Optional[List[int]]]), + DictConfig({"foo": {"bar": None}}, element_type=Dict[str, Any]), + ], + ["foo", "bar"], + None, + Optional[List[int]], + int, + int, + None, + id="merge-none-into-list", + ), + param( + [ + DictConfig({}, element_type=Dict[str, Dict[str, int]]), + DictConfig({"foo": {"bar": "${interp}"}}, element_type=Dict[str, Any]), + ], + ["foo", "bar"], + "${interp}", + Dict[str, int], + str, + int, + None, + id="merge-interp-into-dict", + ), + param( + [ + DictConfig({}, element_type=Dict[str, Optional[Dict[str, int]]]), + DictConfig({"foo": {"bar": None}}, element_type=Dict[str, Any]), + ], + ["foo", "bar"], + None, + Optional[Dict[str, int]], + str, + int, + None, + id="merge-none-into-dict", + ), + ], +) +def test_merge_nested( + configs: List[Any], + keys: List[Any], + value: Any, + type_hint: Any, + key_type: Any, + elt_type: Any, + obj_type: Any, +) -> None: + """Ensure metadata and contents of container-typed subnode are correct""" + cfg = OmegaConf.merge(*configs) + for key in keys[:-1]: + cfg = cfg._get_node(key) # type: ignore + key = keys[-1] + check_subnode( + cfg, + key, + value, + type_hint, + key_type, + elt_type, + obj_type, + ) + + +@mark.parametrize( + "dc1, dc2, value, type_hint, key_type, elt_type, obj_type", + [ + param( + DictConfig({"key": {}}), + DictConfig({"key": "${interp}"}, element_type=Dict[str, int]), + "${interp}", + Dict[str, int], + str, + int, + None, + id="dict-interp-into-any", + ), + param( + DictConfig({"key": {}}), + DictConfig({"key": None}, element_type=Optional[Dict[str, int]]), + None, + Optional[Dict[str, int]], + str, + int, + None, + id="none-interp-into-any", + ), + param( + DictConfig({"key": {"foo": 123}}, element_type=Dict[str, Any]), + DictConfig({"key": {"bar": 456.789}}, element_type=Dict[str, float]), + {"foo": 123, "bar": 456.789}, + Dict[str, Any], + str, + Any, + dict, + id="dict[str,float]-into-dict[str,any]", + ), + param( + DictConfig({"key": {}}, element_type=Dict[str, int]), + DictConfig({"key": "${interp}"}), + "${interp}", + Dict[str, int], + str, + int, + None, + id="interp-into-dict", + ), + param( + DictConfig({"key": []}), + DictConfig({"key": "${interp}"}, element_type=List[int]), + "${interp}", + Any, + int, + Any, + None, + id="list-interp-into-any", + ), + param( + DictConfig({"key": []}, element_type=List[int]), + DictConfig({"key": "${interp}"}), + "${interp}", + List[int], + int, + int, + None, + id="any-interp-into-list-int", + ), + param( + DictConfig({"key": []}, element_type=List[float]), + DictConfig({"key": ["${interp}"]}, element_type=List[int]), + ["${interp}"], + List[float], + int, + float, + list, + id="any-interp_list-into-list-list-int", + ), + ], +) +def test_merge_interpolation_with_container_type( + dc1: DictConfig, + dc2: DictConfig, + value: Any, + type_hint: Any, + key_type: Any, + elt_type: Any, + obj_type: Any, +) -> None: + cfg = OmegaConf.merge(dc1, dc2) + check_subnode( + cfg, + key="key", + value=value, + type_hint=type_hint, + key_type=key_type, + elt_type=elt_type, + obj_type=obj_type, + ) + + +def test_merge_nested_list_promotion() -> None: + dc1 = DictConfig({"key": [Plugin]}, element_type=List[Plugin]) + dc2 = DictConfig({"key": [ConcretePlugin]}) + cfg = OmegaConf.merge(dc1, dc2) + check_subnode( + cfg.key, + key=0, + value=ConcretePlugin(), + type_hint=Plugin, + key_type=Any, + elt_type=Any, + obj_type=ConcretePlugin, + ) + + +@mark.parametrize( + "configs, err_msg", + [ + param( + [DictConfig({}, element_type=int), {"foo": "abc"}], + "Value 'abc' of type 'str' could not be converted to Integer", + ), + param( + [DictConfig({}, element_type=Dict[str, int]), {"foo": 123}], + "Value 123 (int) is incompatible with type hint 'typing.Dict[str, int]'", + id="merge-int-into-dict", + ), + param( + [ + DictConfig({}, element_type=Dict[str, Dict[str, int]]), + DictConfig( + {"foo": {"bar": None}}, element_type=Dict[str, Optional[int]] + ), + ], + "field 'foo.bar' is not Optional", + id="merge-none_typed-into-int", + ), + ], +) +def test_merge_bad_element_type(configs: Any, err_msg: Any) -> None: + with raises( + ValidationError, + match=re.escape(err_msg), + ): + OmegaConf.merge(*configs) diff -pruN 2.1.0~rc1-3/tests/test_nodes.py 2.2.2-1/tests/test_nodes.py --- 2.1.0~rc1-3/tests/test_nodes.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/tests/test_nodes.py 2022-05-27 21:36:40.000000000 +0000 @@ -1,14 +1,18 @@ import copy import functools import re +import sys from enum import Enum -from typing import Any, Dict, Tuple, Type +from functools import partial +from pathlib import Path +from typing import Any, Callable, Dict, Tuple, Type, Union -from pytest import mark, raises +from pytest import fixture, mark, param, raises from omegaconf import ( AnyNode, BooleanNode, + BytesNode, DictConfig, EnumNode, FloatNode, @@ -16,16 +20,23 @@ from omegaconf import ( ListConfig, Node, OmegaConf, + PathNode, StringNode, + UnionNode, ValueNode, ) +from omegaconf._utils import BUILTIN_VALUE_TYPES, type_str from omegaconf.errors import ( InterpolationToMissingValueError, UnsupportedValueType, ValidationError, ) from omegaconf.nodes import InterpolationResultNode -from tests import Color, IllegalType, User +from tests import Color, Enum1, IllegalType, User + + +def _build_union(content: Any) -> UnionNode: + return UnionNode(content=content, ref_type=Union[(Enum, *BUILTIN_VALUE_TYPES)]) # testing valid conversions @@ -35,6 +46,7 @@ from tests import Color, IllegalType, Us # string (StringNode, "abc", "abc"), (StringNode, 100, "100"), + (StringNode, Color.RED, "Color.RED"), # integer (IntegerNode, 10, 10), (IntegerNode, "10", 10), @@ -48,6 +60,9 @@ from tests import Color, IllegalType, Us (FloatNode, 10.1, 10.1), (FloatNode, "10.2", 10.2), (FloatNode, "10e-3", 10e-3), + # bytes + (BytesNode, b"binary", b"binary"), + (BytesNode, b"\xf0\xf1\xf2", b"\xf0\xf1\xf2"), # bool true (BooleanNode, True, True), (BooleanNode, "Y", True), @@ -69,6 +84,7 @@ from tests import Color, IllegalType, Us (AnyNode, 3, 3), (AnyNode, 3.14, 3.14), (AnyNode, False, False), + (AnyNode, b"\xf0\xf1\xf2", b"\xf0\xf1\xf2"), (AnyNode, Color.RED, Color.RED), (AnyNode, None, None), # Enum node @@ -76,6 +92,27 @@ from tests import Color, IllegalType, Us (lambda v: EnumNode(enum_type=Color, value=v), "Color.RED", Color.RED), (lambda v: EnumNode(enum_type=Color, value=v), "RED", Color.RED), (lambda v: EnumNode(enum_type=Color, value=v), 1, Color.RED), + # Path node + (PathNode, "hello.txt", Path("hello.txt")), + (PathNode, Path("hello.txt"), Path("hello.txt")), + # Union node + param(_build_union, "abc", "abc", id="union-str"), + param(_build_union, 10, 10, id="union-int"), + param(_build_union, 10.1, 10.1, id="union-float"), + param(_build_union, float("inf"), float("inf"), id="union-inf"), + param(_build_union, b"binary\xf0\xf1", b"binary\xf0\xf1", id="union-bytes"), + param( + _build_union, + True, + True, + marks=mark.skipif( + sys.version_info < (3, 7), + reason="python3.6 treats Union[int, bool] as equivalent to Union[int]", + ), + id="union-bool", + ), + param(_build_union, None, None, id="union-none"), + param(_build_union, Color.RED, Color.RED, id="union-enum"), ], ) def test_valid_inputs(type_: type, input_: Any, output_: Any) -> None: @@ -85,6 +122,7 @@ def test_valid_inputs(type_: type, input assert not (node != output_) assert not (node != node) assert str(node) == str(output_) + assert repr(node) == repr(output_) # testing invalid conversions @@ -92,20 +130,32 @@ def test_valid_inputs(type_: type, input "type_,input_", [ (IntegerNode, "abc"), + (IntegerNode, "-abc"), (IntegerNode, 10.1), (IntegerNode, "-1132c"), + (IntegerNode, Color.RED), + (IntegerNode, b"123"), (FloatNode, "abc"), - (IntegerNode, "-abc"), + (FloatNode, Color.RED), + (FloatNode, b"10.1"), + (BytesNode, "abc"), + (BytesNode, 23), + (BytesNode, Color.RED), + (BytesNode, 3.14), + (BytesNode, True), (BooleanNode, "Nope"), (BooleanNode, "Yup"), - (StringNode, [1, 2]), - (StringNode, ListConfig([1, 2])), - (StringNode, {"foo": "var"}), - (FloatNode, DictConfig({"foo": "var"})), + (BooleanNode, Color.RED), + (BooleanNode, b"True"), (IntegerNode, [1, 2]), (IntegerNode, ListConfig([1, 2])), (IntegerNode, {"foo": "var"}), + (IntegerNode, b"10"), (IntegerNode, DictConfig({"foo": "var"})), + (BytesNode, [1, 2]), + (BytesNode, ListConfig([1, 2])), + (BytesNode, {"foo": "var"}), + (BytesNode, DictConfig({"foo": "var"})), (BooleanNode, [1, 2]), (BooleanNode, ListConfig([1, 2])), (BooleanNode, {"foo": "var"}), @@ -114,11 +164,30 @@ def test_valid_inputs(type_: type, input (FloatNode, ListConfig([1, 2])), (FloatNode, {"foo": "var"}), (FloatNode, DictConfig({"foo": "var"})), + (StringNode, [1, 2]), + (StringNode, ListConfig([1, 2])), + (StringNode, {"foo": "var"}), + (StringNode, b"\xf0\xf1\xf2"), + (FloatNode, DictConfig({"foo": "var"})), (AnyNode, [1, 2]), (AnyNode, ListConfig([1, 2])), (AnyNode, {"foo": "var"}), (AnyNode, DictConfig({"foo": "var"})), (AnyNode, IllegalType()), + (partial(EnumNode, Color), "Color.TYPO"), + (partial(EnumNode, Color), "TYPO"), + (partial(EnumNode, Color), Enum1.FOO), + (partial(EnumNode, Color), "Enum1.RED"), + (partial(EnumNode, Color), 1000000), + (partial(EnumNode, Color), 1.0), + (partial(EnumNode, Color), b"binary"), + (partial(EnumNode, Color), True), + (partial(EnumNode, Color), [1, 2]), + (partial(EnumNode, Color), {"foo": "bar"}), + (partial(EnumNode, Color), ListConfig([1, 2])), + (partial(EnumNode, Color), DictConfig({"foo": "bar"})), + (PathNode, 1.0), + (PathNode, ["hello.txt"]), ], ) def test_invalid_inputs(type_: type, input_: Any) -> None: @@ -131,6 +200,141 @@ def test_invalid_inputs(type_: type, inp @mark.parametrize( + "optional", [param(True, id="optional"), param(False, id="not_optional")] +) +@mark.parametrize( + "input_", + [ + param("???", id="missing"), + param("${interp}", id="interp"), + param(None, id="none"), + ], +) +@mark.parametrize( + "flags", [param(None, id="convert"), param({"convert": False}, id="no_convert")] +) +class TestValueNodeSpecial: + @mark.parametrize( + "type_", + [ + param(IntegerNode, id="int"), + param(FloatNode, id="float"), + param(BytesNode, id="bytes"), + param(BooleanNode, id="bool"), + param(StringNode, id="str"), + param(partial(EnumNode, Color), id="enum"), + param(PathNode, id="path"), + ], + ) + def test_creation_special( + self, type_: Callable[..., Node], input_: Any, optional: bool, flags: Any + ) -> None: + if input_ is None and not optional: + with raises(ValidationError): + type_(input_, is_optional=optional, flags=flags) + else: + node = type_(input_, is_optional=optional, flags=flags) + assert node._value() == input_ + + @mark.parametrize( + "builds_node", + [ + param(partial(IntegerNode, 123), id="int"), + param(partial(FloatNode, 10.1), id="float"), + param(partial(BytesNode, b"binary"), id="bytes"), + param(partial(BooleanNode, True), id="bool"), + param(partial(StringNode, "abc"), id="str"), + param(partial(EnumNode, Color, value=Color.RED), id="enum"), + param(partial(PathNode, Path("hello.txt")), id="path"), + ], + ) + def test_set_value_special( + self, builds_node: Callable[..., Node], input_: Any, optional: bool, flags: Any + ) -> None: + node = builds_node(is_optional=optional, flags=flags) + if input_ is None and not optional: + with raises(ValidationError): + node._set_value(input_) + else: + node._set_value(input_) + assert node._value() == input_ + + +@mark.parametrize( + "input_", + [ + param(123, id="123"), + param(10.1, id="10.1"), + param(b"binary", id="binary"), + param(True, id="true"), + param("abc", id="abc"), + param("RED", id="red_str"), + param("123", id="123_str"), + param("10.1", id="10.1_str"), + param(Color.RED, id="Color.RED"), + param(Enum1.FOO, id="Enum1.FOO"), + param(Path("hello.txt"), id="path"), + param(object(), id="object"), + ], +) +@mark.parametrize( + "type_, legal_type", + [ + param(IntegerNode, int, id="integer_node"), + param(FloatNode, float, id="float_node"), + param(BytesNode, bytes, id="bytes_node"), + param(BooleanNode, bool, id="boolean_node"), + param(StringNode, str, id="string_node"), + param(partial(EnumNode, Color), Color, id="enum_node"), + param(partial(EnumNode, Enum), lambda t: issubclass(t, Enum), id="enum_node"), + param(PathNode, lambda t: issubclass(t, Path), id="path_node"), + ], +) +class TestNodeConvertFalse: + @fixture + def input_is_compatible_with_node( + self, input_: Any, legal_type: Union[Type[Any], Callable[[Type[Any]], bool]] + ) -> bool: + input_type = type(input_) + return ( + input_type is legal_type + if isinstance(legal_type, type) + else legal_type(input_type) + ) + + @mark.parametrize( + "optional", [param(True, id="optional"), param(False, id="not_optional")] + ) + def test_instantiate_with_value( + self, + type_: Callable[..., ValueNode], + input_: Any, + input_is_compatible_with_node: bool, + optional: bool, + ) -> None: + if input_is_compatible_with_node: + node = type_(input_, is_optional=optional, flags={"convert": False}) + assert node._value() == input_ + else: + with raises(ValidationError): + type_(input_, is_optional=optional, flags={"convert": False}) + + def test_set_value( + self, + type_: Callable[..., ValueNode], + input_: Any, + input_is_compatible_with_node: bool, + ) -> None: + node = type_(flags={"convert": False}) + if input_is_compatible_with_node: + node._set_value(input_, flags={"convert": False}) + assert node._value() == input_ + else: + with raises(ValidationError): + node._set_value(input_, flags={"convert": False}) + + +@mark.parametrize( "input_, expected_type", [ ({}, DictConfig), @@ -140,6 +344,8 @@ def test_invalid_inputs(type_: type, inp (True, AnyNode), (False, AnyNode), ("str", AnyNode), + (b"\xf0\xf1\xf2", AnyNode), + (Path("hello.txt"), AnyNode), ], ) def test_assigned_value_node_type(input_: type, expected_type: Any) -> None: @@ -233,6 +439,8 @@ def test_merge_validation_error(c1: Dict (BooleanNode, True, "invalid"), (AnyNode, "aaa", None), (StringNode, "blah", None), + (BytesNode, b"foobar", None), + (PathNode, Path("hello.txt"), None), ], ) def test_accepts_mandatory_missing( @@ -258,65 +466,139 @@ def test_accepts_mandatory_missing( conf.foo = invalid_value -class Enum1(Enum): - FOO = 1 - BAR = 2 - - -class Enum2(Enum): - NOT_FOO = 1 - NOT_BAR = 2 - - @mark.parametrize( - "type_", [BooleanNode, EnumNode, FloatNode, IntegerNode, StringNode, AnyNode] + "type_", + [ + BooleanNode, + BytesNode, + PathNode, + EnumNode, + FloatNode, + IntegerNode, + StringNode, + AnyNode, + ], ) @mark.parametrize( "values, success_map", [ - ( - # True aliases - (True, "Y", "true", "yes", "on"), + param( + # Integers + (1, 10, -10), { - "BooleanNode": True, # noqa F601 - "StringNode": str, # noqa F601 - "AnyNode": copy.copy, # noqa F601 + "BooleanNode": True, + "IntegerNode": int, + "FloatNode": float, + "StringNode": str, + "AnyNode": copy.copy, }, + id="integers", ), - ( - ("1", 1, 10, -10), + param( + # Integer strings + ("1",), { - "BooleanNode": True, # noqa F601 - "IntegerNode": int, # noqa F601 - "FloatNode": float, # noqa F601 - "StringNode": str, # noqa F601 - "AnyNode": copy.copy, # noqa F601 + "BooleanNode": True, + "IntegerNode": int, + "FloatNode": float, + "StringNode": str, + "AnyNode": copy.copy, + "PathNode": Path, }, + id="integer-strings", ), - ( - # Floaty things - ("1.0", 1.0, float("inf"), float("-inf"), "10e-3", 10e-3), + param( + # Floats + (1.0, float("inf"), float("-inf"), 10e-3), {"FloatNode": float, "StringNode": str, "AnyNode": copy.copy}, + id="floats", ), - ( - # False aliases - (False, "N", "false", "no", "off"), + param( + # Floaty strings + ("1.0", "10e-3"), { - "BooleanNode": False, # noqa F601 - "StringNode": str, # noqa F601 - "AnyNode": copy.copy, # noqa F601 + "FloatNode": float, + "StringNode": str, + "AnyNode": copy.copy, + "PathNode": Path, }, + id="floaty-strings", ), - ( - # Falsy integers - ("0", 0), + param( + # Booleans + (True, False), + { + "BooleanNode": copy.copy, + "StringNode": str, + "AnyNode": copy.copy, + }, + id="booleans", + ), + param( + # Trueish strings + ("Y", "true", "yes", "on"), + { + "BooleanNode": True, + "StringNode": str, + "AnyNode": copy.copy, + "PathNode": Path, + }, + id="trueish-strings", + ), + param( + # Falsey strings + ("N", "false", "no", "off"), { - "BooleanNode": False, # noqa F601 - "IntegerNode": 0, # noqa F601 - "FloatNode": 0.0, # noqa F601 - "StringNode": str, # noqa F601 - "AnyNode": copy.copy, # noqa F601 + "BooleanNode": False, + "StringNode": str, + "AnyNode": copy.copy, + "PathNode": Path, }, + id="falsey-strings", + ), + param( + # Falsy integer + (0,), + { + "BooleanNode": False, + "IntegerNode": 0, + "FloatNode": 0.0, + "StringNode": str, + "AnyNode": copy.copy, + }, + id="falsey-integer", + ), + param( + # Falsy integer string + ("0",), + { + "BooleanNode": False, + "IntegerNode": 0, + "FloatNode": 0.0, + "StringNode": str, + "AnyNode": copy.copy, + "PathNode": Path, + }, + id="falsey-integer-string", + ), + param( + # Binary data + (b"binary",), + { + "BytesNode": b"binary", + "AnyNode": copy.copy, + }, + id="binary-data", + ), + param( + # pathlib.Path data + (Path("hello.txt"),), + { + "PathNode": copy.copy, + "AnyNode": copy.copy, + "StringNode": str, + }, + id="path-data", ), ], ) @@ -345,6 +627,8 @@ def test_legal_assignment( (IntegerNode(), "foo"), (BooleanNode(), "foo"), (FloatNode(), "foo"), + (BytesNode(), "foo"), + (PathNode(), 123), (EnumNode(enum_type=Enum1), "foo"), ], ) @@ -354,7 +638,17 @@ def test_illegal_assignment(node: ValueN @mark.parametrize( - "node_type", [BooleanNode, EnumNode, FloatNode, IntegerNode, StringNode, AnyNode] + "node_type", + [ + BooleanNode, + BytesNode, + PathNode, + EnumNode, + FloatNode, + IntegerNode, + StringNode, + AnyNode, + ], ) @mark.parametrize( "enum_type, values, success_map", @@ -387,74 +681,18 @@ def test_legal_assignment_enum( node_type(enum_type) -class DummyEnum(Enum): - FOO = 1 - - -@mark.parametrize("is_optional", [True, False]) -@mark.parametrize( - "ref_type, type_, value, expected_type", - [ - (Any, Any, 10, AnyNode), - (DummyEnum, DummyEnum, DummyEnum.FOO, EnumNode), - (int, int, 42, IntegerNode), - (float, float, 3.1415, FloatNode), - (bool, bool, True, BooleanNode), - (str, str, "foo", StringNode), - ], -) -def test_node_wrap( - ref_type: type, type_: type, is_optional: bool, value: Any, expected_type: Any -) -> None: - from omegaconf.omegaconf import _node_wrap - - ret = _node_wrap( - ref_type=Any, - type_=type_, - value=value, - is_optional=is_optional, - parent=None, - key=None, - ) - assert ret._metadata.ref_type == ref_type - assert type(ret) == expected_type - assert ret == value - - if is_optional: - ret = _node_wrap( - ref_type=Any, - type_=type_, - value=None, - is_optional=is_optional, - parent=None, - key=None, - ) - assert type(ret) == expected_type - # noinspection PyComparisonWithNone - assert ret == None # noqa E711 - - -def test_node_wrap_illegal_type() -> None: - class UserClass: - pass - - from omegaconf.omegaconf import _node_wrap - - with raises(ValidationError): - _node_wrap( - type_=UserClass, value=UserClass(), is_optional=False, parent=None, key=None - ) - - @mark.parametrize( "obj", [ StringNode(), StringNode(value="foo"), StringNode(value="foo", is_optional=False), + BytesNode(value=b"\xf0\xf1\xf2"), + PathNode(value=Path("hello.txt")), BooleanNode(value=True), IntegerNode(value=10), FloatNode(value=10.0), + UnionNode(10.0, Union[float, bool]), OmegaConf.create({}), OmegaConf.create([]), OmegaConf.create({"foo": "foo"}), @@ -475,15 +713,37 @@ def test_deepcopy(obj: Any) -> None: (StringNode(), None, True), (StringNode(), 100, False), (StringNode("foo"), "foo", True), + (StringNode("100"), 100, False), + (StringNode("???"), "???", True), + (StringNode(None), None, True), + (StringNode("abc"), None, False), + (StringNode("${interp}"), "${interp}", True), + (StringNode("${interp}"), "${different_interp}", False), (IntegerNode(), 1, False), (IntegerNode(1), 1, True), + (IntegerNode(1), "1", False), + (IntegerNode(1), "foo", False), (IntegerNode(1), "foo", False), + (IntegerNode("???"), "???", True), + (IntegerNode(None), None, True), + (IntegerNode("${interp}"), "${interp}", True), + (IntegerNode("${interp}"), "${different_interp}", False), (FloatNode(), 1, False), (FloatNode(), None, True), (FloatNode(1.0), None, False), (FloatNode(1.0), 1.0, True), (FloatNode(1), 1, True), + (FloatNode(1), "1", False), + (FloatNode(1.0), "1.0", False), (FloatNode(1.0), "foo", False), + (FloatNode("???"), "???", True), + (BytesNode(), None, True), + (BytesNode(), b"binary", False), + (BytesNode(b"binary"), b"binary", True), + (PathNode(), None, True), + (PathNode(), Path("hello.txt"), False), + (PathNode(Path("hello.txt")), Path("hello.txt"), True), + (PathNode(Path("hello.txt")), "hello.txt", False), (BooleanNode(), True, False), (BooleanNode(), False, False), (BooleanNode(), None, True), @@ -491,6 +751,9 @@ def test_deepcopy(obj: Any) -> None: (BooleanNode(True), False, False), (BooleanNode(False), False, True), (AnyNode(value=1), AnyNode(value=1), True), + (AnyNode(value=1), 1, True), + (AnyNode(value=1), AnyNode(value="1"), False), + (AnyNode(value=1), "1", False), (EnumNode(enum_type=Enum1), Enum1.BAR, False), (EnumNode(enum_type=Enum1), EnumNode(Enum1), True), (EnumNode(enum_type=Enum1), "nope", False), @@ -499,12 +762,24 @@ def test_deepcopy(obj: Any) -> None: EnumNode(enum_type=Enum1, value=Enum1.BAR), True, ), + ( + EnumNode(enum_type=Enum1, value=Enum1.BAR), + EnumNode(enum_type=Enum1, value=Enum1.FOO), + False, + ), (EnumNode(enum_type=Enum1, value=Enum1.BAR), Enum1.BAR, True), + (EnumNode(enum_type=Enum1, value=Enum1.BAR), Enum1.FOO, False), + (EnumNode(enum_type=Enum1, value=Enum1.BAR), "Enum1.BAR", False), + (EnumNode(enum_type=Enum1, value="???"), "???", True), + (EnumNode(enum_type=Enum1, value=None), Enum1.BAR, False), + (EnumNode(enum_type=Enum1, value="${interp}"), "${interp}", True), + (EnumNode(enum_type=Enum1, value="${interp}"), "${different_interp}", False), (InterpolationResultNode("foo"), "foo", True), (InterpolationResultNode("${foo}"), "${foo}", True), (InterpolationResultNode("${foo"), "${foo", True), (InterpolationResultNode(None), None, True), (InterpolationResultNode(1), 1, True), + (InterpolationResultNode(1), "1", False), (InterpolationResultNode(1.0), 1.0, True), (InterpolationResultNode(True), True, True), (InterpolationResultNode(Color.RED), Color.RED, True), @@ -512,6 +787,53 @@ def test_deepcopy(obj: Any) -> None: (InterpolationResultNode([0, None, True]), [0, None, True], True), (InterpolationResultNode("foo"), 100, False), (InterpolationResultNode(100), "foo", False), + (InterpolationResultNode("???"), "???", True), + (InterpolationResultNode(None), None, True), + (InterpolationResultNode(None), 100, False), + (UnionNode(100, Union[int, bytes]), UnionNode(100, Union[int, bytes]), True), + (UnionNode(100, Union[int, bytes]), 100, True), + (UnionNode(100, Union[int, bytes]), IntegerNode(100), True), + (UnionNode(100, Union[int, bytes]), AnyNode(100), True), + ( + UnionNode("???", Union[int, bytes]), + UnionNode("???", Union[int, bytes]), + True, + ), + (UnionNode("???", Union[int, bytes]), "???", True), + (UnionNode("???", Union[int, bytes]), IntegerNode("???"), True), + (UnionNode("???", Union[int, bytes]), AnyNode("???"), True), + (UnionNode(None, Union[int, bytes]), UnionNode(None, Union[int, bytes]), True), + (UnionNode(None, Union[int, bytes]), None, True), + (UnionNode(None, Union[int, bytes]), IntegerNode(None), True), + (UnionNode(None, Union[int, bytes]), AnyNode(None), True), + ( + UnionNode("${interp}", Union[int, bytes]), + UnionNode("${interp}", Union[int, bytes]), + True, + ), + (UnionNode("${interp}", Union[int, bytes]), "${interp}", True), + (UnionNode("${interp}", Union[int, bytes]), IntegerNode("${interp}"), True), + (UnionNode("${interp}", Union[int, bytes]), AnyNode("${interp}"), True), + (UnionNode(100, Union[int, bytes]), UnionNode(999, Union[int, bytes]), False), + (UnionNode(100, Union[int, bytes]), 999, False), + (UnionNode(100, Union[int, bytes]), IntegerNode(999), False), + (UnionNode(100, Union[int, bytes]), AnyNode(999), False), + (UnionNode("???", Union[int, bytes]), UnionNode(999, Union[int, bytes]), False), + (UnionNode("???", Union[int, bytes]), 999, False), + (UnionNode("???", Union[int, bytes]), IntegerNode(999), False), + (UnionNode("???", Union[int, bytes]), AnyNode(999), False), + (UnionNode(None, Union[int, bytes]), UnionNode(999, Union[int, bytes]), False), + (UnionNode(None, Union[int, bytes]), 999, False), + (UnionNode(None, Union[int, bytes]), IntegerNode(999), False), + (UnionNode(None, Union[int, bytes]), AnyNode(999), False), + ( + UnionNode("${interp}", Union[int, bytes]), + UnionNode(999, Union[int, bytes]), + False, + ), + (UnionNode("${interp}", Union[int, bytes]), 999, False), + (UnionNode("${interp}", Union[int, bytes]), IntegerNode(999), False), + (UnionNode("${interp}", Union[int, bytes]), AnyNode(999), False), ], ) def test_eq(node: ValueNode, value: Any, expected: Any) -> None: @@ -525,7 +847,9 @@ def test_eq(node: ValueNode, value: Any, assert (node.__hash__() == value.__hash__()) == expected -@mark.parametrize("value", [1, 3.14, True, None, Enum1.FOO]) +@mark.parametrize( + "value", ["a_str", 1, 3.14, True, None, Enum1.FOO, b"binary", Path("hello.txt")] +) def test_set_anynode_with_primitive_type(value: Any) -> None: cfg = OmegaConf.create({"a": 5}) a_before = cfg._get_node("a") @@ -593,6 +917,8 @@ def test_dereference_missing() -> None: IntegerNode, FloatNode, BooleanNode, + BytesNode, + PathNode, lambda val, is_optional: EnumNode( enum_type=Color, value=val, is_optional=is_optional ), @@ -600,8 +926,12 @@ def test_dereference_missing() -> None: ) def test_validate_and_convert_none(make_func: Any) -> None: node = make_func("???", is_optional=False) + ref_type_str = type_str(node._metadata.ref_type) with raises( - ValidationError, match=re.escape("Non optional field cannot be assigned None") + ValidationError, + match=re.escape( + f"Incompatible value 'None' for field of type '{ref_type_str}'" + ), ): node.validate_and_convert(None) @@ -629,11 +959,13 @@ def test_dereference_interpolation_to_mi [ AnyNode, BooleanNode, + BytesNode, functools.partial(EnumNode, enum_type=Color), FloatNode, IntegerNode, InterpolationResultNode, StringNode, + PathNode, ], ) def test_set_flags_in_init(type_: Any, flags: Dict[str, bool]) -> None: diff -pruN 2.1.0~rc1-3/tests/test_omegaconf.py 2.2.2-1/tests/test_omegaconf.py --- 2.1.0~rc1-3/tests/test_omegaconf.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/tests/test_omegaconf.py 2022-05-27 21:36:40.000000000 +0000 @@ -1,11 +1,14 @@ -import re -from typing import Any +import pathlib +import platform +from pathlib import Path +from typing import Any, Union -from pytest import mark, param, raises, warns +from pytest import mark, param, raises from omegaconf import ( MISSING, BooleanNode, + BytesNode, DictConfig, EnumNode, FloatNode, @@ -13,7 +16,9 @@ from omegaconf import ( ListConfig, MissingMandatoryValue, OmegaConf, + PathNode, StringNode, + UnionNode, ) from omegaconf._utils import _is_none, nullcontext from omegaconf.errors import ( @@ -157,6 +162,8 @@ def test_is_missing_resets() -> None: (10, False), (True, False), (bool, False), + (Path, False), + (Path("hello.txt"), False), (StringNode("foo"), False), (ConcretePlugin, False), (ConcretePlugin(), False), @@ -180,6 +187,8 @@ def test_is_config(cfg: Any, expected: b (10, False), (True, False), (bool, False), + (Path, False), + (Path("hello.txt"), False), (StringNode("foo"), False), (ConcretePlugin, False), (ConcretePlugin(), False), @@ -203,6 +212,8 @@ def test_is_list(cfg: Any, expected: boo (10, False), (True, False), (bool, False), + (Path, False), + (Path("hello.txt"), False), (StringNode("foo"), False), (ConcretePlugin, False), (ConcretePlugin(), False), @@ -216,29 +227,20 @@ def test_is_dict(cfg: Any, expected: boo assert OmegaConf.is_dict(cfg) == expected -def test_is_optional_deprecation() -> None: - with warns(UserWarning, match=re.escape("`OmegaConf.is_optional()` is deprecated")): - OmegaConf.is_optional("xxx") - - -def test_coverage_for_deprecated_OmegaConf_is_optional() -> None: - cfg = OmegaConf.create({"node": 123}) - with warns(UserWarning): - assert OmegaConf.is_optional(cfg) - with warns(UserWarning): - assert OmegaConf.is_optional(cfg, "node") - with warns(UserWarning): - assert OmegaConf.is_optional("not_a_node") - - @mark.parametrize("is_none", [True, False]) @mark.parametrize( "fac", [ (lambda none: StringNode(value="foo" if not none else None, is_optional=True)), (lambda none: IntegerNode(value=10 if not none else None, is_optional=True)), - (lambda none: FloatNode(value=10 if not none else None, is_optional=True)), + (lambda none: FloatNode(value=10.0 if not none else None, is_optional=True)), (lambda none: BooleanNode(value=True if not none else None, is_optional=True)), + (lambda none: BytesNode(value=b"123" if not none else None, is_optional=True)), + ( + lambda none: PathNode( + value=Path("hello.txt") if not none else None, is_optional=True + ) + ), ( lambda none: EnumNode( enum_type=Color, @@ -284,20 +286,12 @@ def test_is_none(fac: Any, is_none: bool ) def test_is_none_interpolation(cfg: Any, key: str, is_none: bool) -> None: cfg = OmegaConf.create(cfg) - with warns(UserWarning): - assert OmegaConf.is_none(cfg, key) == is_none check = _is_none( cfg._get_node(key), resolve=True, throw_on_resolution_failure=False ) assert check == is_none -def test_is_none_invalid_node() -> None: - cfg = OmegaConf.create({}) - with warns(UserWarning): - assert OmegaConf.is_none(cfg, "invalid") - - @mark.parametrize( "fac", [ @@ -322,6 +316,16 @@ def test_is_none_invalid_node() -> None: ) ), ( + lambda inter: BytesNode( + value=b"123" if inter is None else inter, is_optional=True + ) + ), + ( + lambda inter: PathNode( + value="hello.txt" if inter is None else inter, is_optional=True + ) + ), + ( lambda inter: EnumNode( enum_type=Color, value=Color.RED if inter is None else inter, @@ -351,6 +355,8 @@ def test_is_none_invalid_node() -> None: "IntegerNode", "FloatNode", "BooleanNode", + "BytesNode", + "PathNode", "EnumNode", "ListConfig", "DictConfig", @@ -378,6 +384,17 @@ def test_is_interpolation(fac: Any) -> A ({"foo": 10}, int), ({"foo": 10.0}, float), ({"foo": True}, bool), + ({"foo": b"123"}, bytes), + ({"foo": UnionNode(10.0, Union[float, bytes])}, float), + ({"foo": UnionNode(None, Union[float, bytes])}, type(None)), + ({"foo": FloatNode(10.0)}, float), + ({"foo": FloatNode(None)}, type(None)), + ( + {"foo": Path("hello.txt")}, + pathlib.WindowsPath + if platform.system() == "Windows" + else pathlib.PosixPath, + ), ({"foo": "bar"}, str), ({"foo": None}, type(None)), ({"foo": ConcretePlugin()}, ConcretePlugin), @@ -399,6 +416,13 @@ def test_get_type(cfg: Any, type_: Any) (10, int), (10.0, float), (True, bool), + (b"123", bytes), + ( + Path("hello.txt"), + pathlib.WindowsPath + if platform.system() == "Windows" + else pathlib.PosixPath, + ), ("foo", str), (DictConfig(content={}), dict), (ListConfig(content=[]), list), @@ -518,3 +542,131 @@ def test_resolve(cfg: Any, expected: Any def test_resolve_invalid_input() -> None: with raises(ValueError): OmegaConf.resolve("aaa") # type: ignore + + +@mark.parametrize( + "cfg, expected", + [ + # dict: + ({"a": 10, "b": {"c": "???", "d": "..."}}, {"b.c"}), + ( + { + "a": "???", + "b": { + "foo": "bar", + "bar": "???", + "more": {"missing": "???", "available": "yes"}, + }, + Color.GREEN: {"tint": "???", "default": Color.BLUE}, + }, + {"a", "b.bar", "b.more.missing", "GREEN.tint"}, + ), + ( + {"a": "a", "b": {"foo": "bar", "bar": "foo"}}, + set(), + ), + ( + {"foo": "bar", "bar": "???", "more": {"foo": "???", "bar": "foo"}}, + {"bar", "more.foo"}, + ), + # list: + (["???", "foo", "bar", "???", 77], {"[0]", "[3]"}), + (["", "foo", "bar"], set()), + (["foo", "bar", "???"], {"[2]"}), + (["foo", "???", ["???", "bar"]], {"[1]", "[2][0]"}), + # mixing: + ( + [ + "???", + "foo", + { + "a": True, + "b": "???", + "c": ["???", None], + "d": {"e": "???", "f": "fff", "g": [True, "???"]}, + }, + "???", + 77, + ], + {"[0]", "[2].b", "[2].c[0]", "[2].d.e", "[2].d.g[1]", "[3]"}, + ), + ( + { + "list": [ + 0, + DictConfig({"foo": "???", "bar": None}), + "???", + ["???", 3, False], + ], + "x": "y", + "y": "???", + }, + {"list[1].foo", "list[2]", "list[3][0]", "y"}, + ), + ({Color.RED: ["???", {"missing": "???"}]}, {"RED[0]", "RED[1].missing"}), + # with DictConfig and ListConfig: + ( + DictConfig( + { + "foo": "???", + "list": ["???", 1], + "bar": {"missing": "???", "more": "yes"}, + } + ), + {"foo", "list[0]", "bar.missing"}, + ), + ( + ListConfig( + ["???", "yes", "???", [0, 1, "???"], {"missing": "???", "more": ""}], + ), + {"[0]", "[2]", "[3][2]", "[4].missing"}, + ), + ], +) +def test_missing_keys(cfg: Any, expected: Any) -> None: + assert OmegaConf.missing_keys(cfg) == expected + + +@mark.parametrize("cfg", [float, int]) +def test_missing_keys_invalid_input(cfg: Any) -> None: + with raises(ValueError): + OmegaConf.missing_keys(cfg) + + +@mark.parametrize( + ("register_resolver_params", "name", "expected"), + [ + param( + dict( + name="iamnew", + resolver=lambda x: str(x).lower(), + use_cache=False, + replace=False, + ), + "iamnew", + dict(pre_clear=True, result=True), + id="remove-new-custom-resolver", + ), + param( + dict(), + "oc.env", + dict(pre_clear=True, result=True), + id="remove-default-resolver", + ), + param( + dict(), + "idonotexist", + dict(pre_clear=False, result=False), + id="remove-nonexistent-resolver", + ), + ], +) +def test_clear_resolver( + restore_resolvers: Any, register_resolver_params: Any, name: str, expected: Any +) -> None: + if register_resolver_params: + OmegaConf.register_new_resolver(**register_resolver_params) + assert expected["pre_clear"] == OmegaConf.has_resolver(name) + + assert OmegaConf.clear_resolver(name) == expected["result"] + assert not OmegaConf.has_resolver(name) diff -pruN 2.1.0~rc1-3/tests/test_pydev_resolver_plugin.py 2.2.2-1/tests/test_pydev_resolver_plugin.py --- 2.1.0~rc1-3/tests/test_pydev_resolver_plugin.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/tests/test_pydev_resolver_plugin.py 2022-05-27 21:36:40.000000000 +0000 @@ -6,6 +6,7 @@ from pytest import fixture, mark, param from omegaconf import ( AnyNode, BooleanNode, + BytesNode, Container, DictConfig, EnumNode, @@ -14,6 +15,7 @@ from omegaconf import ( ListConfig, Node, OmegaConf, + PathNode, StringNode, ValueNode, ) @@ -39,6 +41,8 @@ def resolver() -> Any: param(IntegerNode(10), {}, id="int:10"), param(FloatNode(3.14), {}, id="float:3.14"), param(BooleanNode(True), {}, id="bool:True"), + param(BytesNode(b"binary"), {}, id="bytes:binary"), + param(PathNode("hello.txt"), {}, id="path:hello.txt"), param(EnumNode(enum_type=Color, value=Color.RED), {}, id="Color:Color.RED"), # nodes are never returning a dictionary param(AnyNode("${foo}", parent=DictConfig({"foo": 10})), {}, id="any:inter_10"), @@ -230,6 +234,8 @@ def test_get_dictionary_listconfig( (FloatNode, True), (StringNode, True), (BooleanNode, True), + (BytesNode, True), + (PathNode, True), (EnumNode, True), # not covering some other things. (builtins.int, False), diff -pruN 2.1.0~rc1-3/tests/test_serialization.py 2.2.2-1/tests/test_serialization.py --- 2.1.0~rc1-3/tests/test_serialization.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/tests/test_serialization.py 2022-05-27 21:36:40.000000000 +0000 @@ -1,22 +1,31 @@ # -*- coding: utf-8 -*- +import copy import io import os import pathlib import pickle +import re +import sys import tempfile from pathlib import Path -from typing import Any, Dict, List, Optional, Type +from typing import Any, Callable, Dict, List, Optional, Type, Union from pytest import mark, param, raises -from omegaconf import MISSING, DictConfig, ListConfig, OmegaConf -from omegaconf._utils import get_ref_type +from omegaconf import MISSING, DictConfig, ListConfig, Node, OmegaConf, UnionNode +from omegaconf._utils import get_type_hint +from omegaconf.base import Box +from omegaconf.errors import OmegaConfBaseException from tests import ( Color, + NestedContainers, PersonA, PersonD, SubscriptedDict, + SubscriptedDictOpt, SubscriptedList, + SubscriptedListOpt, + UnionAnnotations, UntypedDict, UntypedList, ) @@ -29,7 +38,7 @@ def save_load_from_file(conf: Any, resol with tempfile.NamedTemporaryFile( mode="wt", delete=False, encoding="utf-8" ) as fp: - OmegaConf.save(conf, fp.file, resolve=resolve) # type: ignore + OmegaConf.save(conf, fp.file, resolve=resolve) with io.open(os.path.abspath(fp.name), "rt", encoding="utf-8") as handle: c2 = OmegaConf.load(handle) assert c2 == expected @@ -65,7 +74,7 @@ def test_load_from_invalid() -> None: ({"foo": 10, "bar": "${foo}"}, False, None, str), ({"foo": 10, "bar": "${foo}"}, False, None, pathlib.Path), ({"foo": 10, "bar": "${foo}"}, False, {"foo": 10, "bar": 10}, str), - ([u"שלום"], False, None, str), + (["שלום"], False, None, str), ], ) class TestSaveLoad: @@ -133,7 +142,7 @@ def test_pickle(obj: Any) -> None: fp.seek(0) c1 = pickle.load(fp) assert c == c1 - assert get_ref_type(c1) == Any + assert get_type_hint(c1) == Any assert c1._metadata.element_type is Any assert c1._metadata.optional is True if isinstance(c, DictConfig): @@ -153,24 +162,110 @@ def test_load_empty_file(tmpdir: str) -> @mark.parametrize( "input_,node,element_type,key_type,optional,ref_type", [ - (UntypedList, "list", Any, Any, False, List[Any]), - (UntypedList, "opt_list", Any, Any, True, Optional[List[Any]]), - (UntypedDict, "dict", Any, Any, False, Dict[Any, Any]), - ( + param(UntypedList, "list", Any, Any, False, List[Any], id="list_untyped"), + param( + UntypedList, + "opt_list", + Any, + Any, + True, + Optional[List[Any]], + id="opt_list_untyped", + ), + param(UntypedDict, "dict", Any, Any, False, Dict[Any, Any], id="dict_untyped"), + param( UntypedDict, "opt_dict", Any, Any, True, Optional[Dict[Any, Any]], + id="opt_dict_untyped", ), - (SubscriptedDict, "dict_str", int, str, False, Dict[str, int]), - (SubscriptedDict, "dict_int", int, int, False, Dict[int, int]), - (SubscriptedDict, "dict_bool", int, bool, False, Dict[bool, int]), - (SubscriptedDict, "dict_float", int, float, False, Dict[float, int]), - (SubscriptedDict, "dict_enum", int, Color, False, Dict[Color, int]), - (SubscriptedList, "list", int, Any, False, List[int]), - ( + param( + SubscriptedDict, "dict_str", int, str, False, Dict[str, int], id="dict_str" + ), + param( + SubscriptedDict, + "dict_bytes", + int, + bytes, + False, + Dict[bytes, int], + id="dict_bytes", + ), + param( + SubscriptedDict, "dict_int", int, int, False, Dict[int, int], id="dict_int" + ), + param( + SubscriptedDict, + "dict_bool", + int, + bool, + False, + Dict[bool, int], + id="dict_bool", + ), + param( + SubscriptedDict, + "dict_float", + int, + float, + False, + Dict[float, int], + id="dict_float", + ), + param( + SubscriptedDict, + "dict_enum", + int, + Color, + False, + Dict[Color, int], + id="dict_enum", + ), + param(SubscriptedList, "list", int, Any, False, List[int], id="list_int"), + param( + SubscriptedDictOpt, + "opt_dict", + int, + str, + True, + Optional[Dict[str, int]], + marks=mark.skipif(sys.version_info < (3, 7), reason="requires Python 3.7"), + id="opt_dict", + ), + param( + SubscriptedDictOpt, + "dict_opt", + Optional[int], + str, + False, + Dict[str, Optional[int]], + marks=mark.skipif(sys.version_info < (3, 7), reason="requires Python 3.7"), + id="dict_opt", + ), + param( + SubscriptedListOpt, + "opt_list", + int, + str, + True, + Optional[List[int]], + marks=mark.skipif(sys.version_info < (3, 7), reason="requires Python 3.7"), + id="opt_list", + ), + param( + SubscriptedListOpt, + "list_opt", + Optional[int], + str, + False, + List[Optional[int]], + marks=mark.skipif(sys.version_info < (3, 7), reason="requires Python 3.7"), + id="list_opt", + ), + param( DictConfig( content={"a": "foo"}, ref_type=Dict[str, str], @@ -191,11 +286,51 @@ def test_load_empty_file(tmpdir: str) -> True, Optional[List[int]], ), + param( + NestedContainers, + "dict_of_dict", + Dict[str, int], + str, + False, + Dict[str, Dict[str, int]], + marks=mark.skipif(sys.version_info < (3, 7), reason="requires Python 3.7"), + id="dict-of-dict", + ), + param( + NestedContainers, + "list_of_list", + List[int], + int, + False, + List[List[int]], + marks=mark.skipif(sys.version_info < (3, 7), reason="requires Python 3.7"), + id="list-of-list", + ), + param( + NestedContainers, + "dict_of_list", + List[int], + str, + False, + Dict[str, List[int]], + marks=mark.skipif(sys.version_info < (3, 7), reason="requires Python 3.7"), + id="dict-of-list", + ), + param( + NestedContainers, + "list_of_dict", + Dict[str, int], + int, + False, + List[Dict[str, int]], + marks=mark.skipif(sys.version_info < (3, 7), reason="requires Python 3.7"), + id="list-of-dict", + ), ], ) def test_pickle_untyped( input_: Any, - node: str, + node: Optional[str], optional: bool, element_type: Any, key_type: Any, @@ -203,32 +338,42 @@ def test_pickle_untyped( ) -> None: cfg = OmegaConf.structured(input_) with tempfile.TemporaryFile() as fp: - import pickle - pickle.dump(cfg, fp) fp.flush() fp.seek(0) cfg2 = pickle.load(fp) - def get_node(cfg: Any, key: str) -> Any: + def get_node(cfg: Any, key: Optional[str]) -> Any: if key is None: return cfg else: return cfg._get_node(key) assert cfg == cfg2 - assert get_ref_type(get_node(cfg2, node)) == ref_type + assert get_type_hint(get_node(cfg2, node)) == ref_type assert get_node(cfg2, node)._metadata.element_type == element_type assert get_node(cfg2, node)._metadata.optional == optional if isinstance(input_, DictConfig): assert get_node(cfg2, node)._metadata.key_type == key_type +@mark.skipif(sys.version_info < (3, 7), reason="requires python3.7 or newer") +@mark.parametrize("key", ["ubf", "oubf"]) +def test_pickle_union_node(key: str) -> None: + cfg = OmegaConf.structured(UnionAnnotations) + pickled = pickle.dumps(cfg) + cfg2 = pickle.loads(pickled) + node = cfg._get_node(key) + node2 = cfg2._get_node(key) + + assert cfg == cfg2 + assert node._metadata == node2._metadata + assert node2._parent == cfg2 + + def test_pickle_missing() -> None: cfg = DictConfig(content=MISSING) with tempfile.TemporaryFile() as fp: - import pickle - pickle.dump(cfg, fp) fp.flush() fp.seek(0) @@ -239,8 +384,6 @@ def test_pickle_missing() -> None: def test_pickle_none() -> None: cfg = DictConfig(content=None) with tempfile.TemporaryFile() as fp: - import pickle - pickle.dump(cfg, fp) fp.flush() fp.seek(0) @@ -257,3 +400,60 @@ def test_pickle_flags_consistency() -> N cfg2._set_flag("test", None) assert cfg2._get_flag("test") is None assert cfg2._get_node("a")._get_flag("test") is None + + +@mark.parametrize( + "version", + [ + "2.0.6", + "2.1.0.rc1", + ], +) +def test_pickle_backward_compatibility(version: str) -> None: + path = Path(__file__).parent / "data" / f"{version}.pickle" + with open(path, mode="rb") as fp: + cfg = pickle.load(fp) + assert cfg == OmegaConf.create({"a": [{"b": 10}]}) + + +@mark.skipif(sys.version_info >= (3, 7), reason="requires python3.6") +def test_python36_pickle_optional() -> None: + cfg = OmegaConf.structured(SubscriptedDictOpt) + with raises( + OmegaConfBaseException, + match=re.escape( + "Serializing structured configs with `Union` element type requires python >= 3.7" + ), + ): + pickle.dumps(cfg) + + +@mark.parametrize( + "copy_fn", + [ + param(lambda obj: copy.deepcopy(obj), id="deepcopy"), + param(lambda obj: copy.copy(obj), id="copy"), + param(lambda obj: pickle.loads(pickle.dumps(obj)), id="pickle"), + ], +) +@mark.parametrize( + "box, get_child", + [ + param( + UnionNode(10.0, Union[float, bool]), + lambda cfg: cfg._value(), + marks=mark.skipif( + sys.version_info < (3, 7), reason="requires python3.7 or newer" + ), + id="union", + ), + param(DictConfig({"foo": "bar"}), lambda cfg: cfg._get_node("foo"), id="dict"), + param(ListConfig(["bar"]), lambda cfg: cfg._get_node(0), id="list"), + ], +) +def test_copy_preserves_parent_of_child( + box: Box, get_child: Callable[[Box], Node], copy_fn: Callable[[Box], Box] +) -> None: + assert get_child(box)._parent is box + cp = copy_fn(box) + assert get_child(cp)._parent is cp diff -pruN 2.1.0~rc1-3/tests/test_to_container.py 2.2.2-1/tests/test_to_container.py --- 2.1.0~rc1-3/tests/test_to_container.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/tests/test_to_container.py 2022-05-27 21:36:40.000000000 +0000 @@ -13,8 +13,21 @@ from omegaconf import ( SCMode, open_dict, ) -from omegaconf.errors import InterpolationResolutionError -from tests import Color, User, warns_dict_subclass_deprecated +from omegaconf._utils import _ensure_container +from omegaconf.errors import ( + InterpolationKeyError, + InterpolationResolutionError, + InterpolationToMissingValueError, +) +from tests import ( + B, + Color, + NestedInterpolationToMissing, + StructuredInterpolationKeyError, + User, + Users, + warns_dict_subclass_deprecated, +) @mark.parametrize( @@ -24,7 +37,7 @@ from tests import Color, User, warns_dic param([1, 2, {"a": 3}], id="dict_in_list"), param([1, 2, [10, 20]], id="list_in_list"), param({"b": {"b": 10}}, id="dict_in_dict"), - param({"b": [False, 1, "2", 3.0, Color.RED]}, id="list_in_dict"), + param({"b": [False, 1, "2", 3.0, Color.RED, b"binary"]}, id="list_in_dict"), param({"b": DictConfig(content=None)}, id="none_dictconfig"), param({"b": ListConfig(content=None)}, id="none_listconfig"), param({"b": DictConfig(content="???")}, id="missing_dictconfig"), @@ -40,10 +53,11 @@ def test_to_container_returns_primitives for _k, v in item.items(): assert_container_with_primitives(v) else: - assert isinstance(item, (int, float, str, bool, type(None), Enum)) + assert isinstance(item, (int, float, str, bytes, bool, type(None), Enum)) c = OmegaConf.create(input_) res = OmegaConf.to_container(c, resolve=True) + assert isinstance(res, (dict, list)) assert_container_with_primitives(res) @@ -129,6 +143,7 @@ def test_scmode( else: ret = OmegaConf.to_container(cfg, structured_config_mode=structured_config_mode) assert ret == expected + assert isinstance(ret, (dict, list)) assert isinstance(ret[key], expected_value_type) @@ -170,6 +185,14 @@ def test_scmode( id="dict_inter_dictconfig", ), param( + OmegaConf.create( + {"foo": DictConfig(content="${bar}"), "bar": {"a": 0}} + )._get_node("foo"), + "${bar}", + {"a": 0}, + id="toplevel_dict_inter", + ), + param( {"foo": ListConfig(content="???")}, {"foo": "???"}, None, @@ -187,6 +210,14 @@ def test_scmode( {"foo": 10, "bar": 10}, id="dict_inter_listconfig", ), + param( + OmegaConf.create( + {"foo": ListConfig(content="${bar}"), "bar": ["a"]} + )._get_node("foo"), + "${bar}", + ["a"], + id="toplevel_list_inter", + ), ], ) def test_to_container(src: Any, expected: Any, expected_with_resolve: Any) -> None: @@ -194,7 +225,7 @@ def test_to_container(src: Any, expected expected = src if expected_with_resolve is None: expected_with_resolve = expected - cfg = OmegaConf.create(src) + cfg = _ensure_container(src) container = OmegaConf.to_container(cfg) assert container == expected container = OmegaConf.to_container(cfg, structured_config_mode=SCMode.INSTANTIATE) @@ -220,22 +251,6 @@ def test_string_interpolation_with_reado } -@mark.parametrize( - "src,expected", - [ - param(DictConfig(content="${bar}"), "${bar}", id="DictConfig"), - param( - OmegaConf.create({"foo": DictConfig(content="${bar}")}), - {"foo": "${bar}"}, - id="nested_DictConfig", - ), - ], -) -def test_to_container_missing_inter_no_resolve(src: Any, expected: Any) -> None: - res = OmegaConf.to_container(src, resolve=False) - assert res == expected - - class TestInstantiateStructuredConfigs: @fixture( params=[ @@ -315,6 +330,22 @@ class TestInstantiateStructuredConfigs: assert nested.default_value.additional == 20 assert nested.user_provided_default.mandatory_missing == 456 + def test_unions_with_defaults_to_container(self, module: Any) -> None: + cfg = OmegaConf.structured(module.UnionsOfPrimitveTypes.WithDefaults) + obj: Any = OmegaConf.to_container(cfg) + assert obj["uis"] == "abc" + assert obj["ubc1"] is True + assert obj["ubc2"] == Color.RED + assert obj["ouis"] is None + + def test_unions_with_defaults_to_object(self, module: Any) -> None: + cfg = OmegaConf.structured(module.UnionsOfPrimitveTypes.WithDefaults) + obj: Any = OmegaConf.to_object(cfg) + assert obj.uis == "abc" + assert obj.ubc1 is True + assert obj.ubc2 == Color.RED + assert obj.ouis is None + def test_nested_object_with_missing(self, module: Any) -> None: with raises(MissingMandatoryValue): self.round_trip_to_object(module.NestedConfig) @@ -417,6 +448,42 @@ class TestInstantiateStructuredConfigs: assert type(user) is module.User assert user.extra_field == 123 + def test_init_false_with_default(self, module: Any) -> None: + cfg = OmegaConf.structured(module.HasInitFalseFields) + assert cfg.with_default == "default" + data = self.round_trip_to_object(cfg) + assert data.with_default == "default" + + def test_init_false_with_default_overridden(self, module: Any) -> None: + cfg = OmegaConf.structured(module.HasInitFalseFields) + cfg.with_default = "default_overridden" + data = self.round_trip_to_object(cfg) + assert data.with_default == "default_overridden" + + def test_init_false_without_default(self, module: Any) -> None: + cfg = OmegaConf.structured(module.HasInitFalseFields) + assert OmegaConf.is_missing(cfg, "without_default") + data = self.round_trip_to_object(cfg) + assert not hasattr(data, "without_default") + + def test_init_false_without_default_overridden(self, module: Any) -> None: + cfg = OmegaConf.structured(module.HasInitFalseFields) + cfg.with_default = "default_overridden" + data = self.round_trip_to_object(cfg) + assert data.with_default == "default_overridden" + + def test_init_false_post_initialized(self, module: Any) -> None: + cfg = OmegaConf.structured(module.HasInitFalseFields) + assert OmegaConf.is_missing(cfg, "post_initialized") + data = self.round_trip_to_object(cfg) + assert data.post_initialized == "set_by_post_init" + + def test_init_false_post_initialized_overridden(self, module: Any) -> None: + cfg = OmegaConf.structured(module.HasInitFalseFields) + cfg.post_initialized = "overridden" + data = self.round_trip_to_object(cfg) + assert data.post_initialized == "overridden" + class TestEnumToStr: """Test the `enum_to_str` argument to the `OmegaConf.to_container function`""" @@ -462,3 +529,129 @@ class TestEnumToStr: cfg = OmegaConf.create(src) container: List[Any] = OmegaConf.to_container(cfg, enum_to_str=enum_to_str) # type: ignore assert container == [expected] + + +@mark.parametrize( + "cfg", + [ + # to_container: throw_on_missing + param(DictConfig("???"), id="dict:missing"), + param(DictConfig({"a": "???"}), id="dict:missing_value"), + param(DictConfig({"a": {"b": "???"}}), id="dict:nested"), + param(ListConfig("???"), id="list:missing"), + param(ListConfig(["???"]), id="list:missing_elt"), + param(ListConfig(["abc", ["???"]]), id="list:nested"), + param(OmegaConf.structured(B), id="structured:missing_field"), + param( + OmegaConf.structured(Users({"Bond": "???"})), # type: ignore + id="structured:missing_in_dict_field", + ), + ], +) +class TestThrowOnMissing: + """Tests the `throw_on_missing` arugment to OmegaConf.to_container""" + + @mark.parametrize( + "op", + [ + param( + lambda cfg: OmegaConf.to_container(cfg, throw_on_missing=True), + id="to_container", + ), + param(OmegaConf.to_object, id="to_object"), + ], + ) + def test_throw_on_missing_raises(self, op: Any, cfg: Any) -> None: + with raises(MissingMandatoryValue): + op(cfg) + + def test_no_throw_on_missing(self, cfg: Any) -> None: + assert OmegaConf.to_container(cfg, throw_on_missing=False) == cfg + + +@mark.parametrize( + "cfg,expected,exception_type", + [ + param( + OmegaConf.create({"foo": "${bar}"}), + {"foo": "${bar}"}, + InterpolationKeyError, + id="interp_key_error", + ), + param( + OmegaConf.create({"subcfg": {"x": "${missing}"}, "missing": "???"}).subcfg, + {"x": "${missing}"}, + InterpolationToMissingValueError, + id="interp_to_missing_in_dict", + ), + param( + OmegaConf.create({"subcfg": ["${missing}"], "missing": "???"}).subcfg, + ["${missing}"], + InterpolationToMissingValueError, + id="interp_to_missing_in_list", + ), + param( + OmegaConf.structured(NestedInterpolationToMissing).subcfg, + {"baz": "${..name}"}, + InterpolationToMissingValueError, + id="interp_to_missing_in_structured", + ), + param( + OmegaConf.structured(StructuredInterpolationKeyError), + {"name": "${bar}"}, + InterpolationKeyError, + id="interp_key_error_in_structured", + ), + param( + DictConfig(content="${bar}"), + "${bar}", + InterpolationResolutionError, + id="dictconfig_interp_key_error", + ), + param( + ListConfig(content="${bar}"), + "${bar}", + InterpolationResolutionError, + id="dictconfig_interp_key_error", + ), + param( + OmegaConf.create({"foo": DictConfig(content="${bar}")}), + {"foo": "${bar}"}, + InterpolationKeyError, + id="dictconfig_interp_key_error_in_dict", + ), + ], +) +class TestResolveBadInterpolation: + @mark.parametrize( + "op", + [ + param( + lambda cfg: OmegaConf.to_container( + cfg, resolve=True, throw_on_missing=True + ), + id="throw_on_missing", + ), + param( + lambda cfg: OmegaConf.to_container( + cfg, resolve=True, throw_on_missing=False + ), + id="no_throw_on_missing", + ), + param(OmegaConf.to_object, id="to_object"), + ], + ) + def test_resolve_bad_interpolation_raises( + self, op: Any, cfg: Any, expected: Any, exception_type: Any + ) -> None: + with raises(exception_type): + op(cfg) + + @mark.parametrize("throw_on_missing", [True, False]) + def test_resolve_bad_interpolation_no_resolve( + self, cfg: Any, expected: Any, exception_type: Any, throw_on_missing: bool + ) -> None: + ret = OmegaConf.to_container( + cfg, resolve=False, throw_on_missing=throw_on_missing + ) + assert ret == expected diff -pruN 2.1.0~rc1-3/tests/test_to_yaml.py 2.2.2-1/tests/test_to_yaml.py --- 2.1.0~rc1-3/tests/test_to_yaml.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/tests/test_to_yaml.py 2022-05-27 21:36:40.000000000 +0000 @@ -1,6 +1,9 @@ +import platform +from pathlib import Path from textwrap import dedent from typing import Any +import yaml from pytest import mark from omegaconf import DictConfig, EnumNode, ListConfig, OmegaConf, _utils @@ -13,6 +16,13 @@ from tests import Enum1, User (["item1", "item2", {"key3": "value3"}], "- item1\n- item2\n- key3: value3\n"), ({"hello": "world", "list": [1, 2]}, "hello: world\nlist:\n- 1\n- 2\n"), ({"abc": "str key"}, "abc: str key\n"), + ({b"abc": "bytes key"}, "? !!binary |\n YWJj\n: bytes key\n"), + ( + {"path_value": Path("hello.txt")}, + "path_value: !!python/object/apply:pathlib.WindowsPath\n- hello.txt\n" + if platform.system() == "Windows" + else "path_value: !!python/object/apply:pathlib.PosixPath\n- hello.txt\n", + ), ({123: "int key"}, "123: int key\n"), ({123.45: "float key"}, "123.45: float key\n"), ({True: "bool key", False: "another"}, "true: bool key\nfalse: another\n"), @@ -20,8 +30,10 @@ from tests import Enum1, User ) def test_to_yaml(input_: Any, expected: str) -> None: c = OmegaConf.create(input_) - assert expected == OmegaConf.to_yaml(c) - assert OmegaConf.create(OmegaConf.to_yaml(c)) == c + as_yaml = OmegaConf.to_yaml(c) + assert as_yaml == expected + assert OmegaConf.create(as_yaml) == c + assert yaml.unsafe_load(as_yaml) == c @mark.parametrize( @@ -41,6 +53,7 @@ def test_to_yaml_unicode(input_: Any, ex "input_, expected, type_", [ (["1", 1], "- '1'\n- 1\n", int), + (["1", b"1"], "- '1'\n- !!binary |\n MQ==\n", bytes), (["10e2", "1.0", 1.0], "- '10e2'\n- '1.0'\n- 1.0\n", float), (_utils.YAML_BOOL_TYPES, None, bool), ], @@ -63,6 +76,7 @@ def test_to_yaml_string_primitive_types_ "input_, expected, type_", [ ({"b": "1", "a": 1}, "b: '1'\na: 1\n", int), + ({"b": "1", "a": b"1"}, "b: '1'\na: !!binary |\n MQ==\n", bytes), ({"b": "10e2", "a": "1.0", "c": 1.0}, "b: '10e2'\na: '1.0'\nc: 1.0\n", float), (_utils.YAML_BOOL_TYPES, None, bool), ], @@ -146,8 +160,8 @@ def test_to_yaml_with_enum_key() -> None def test_structured_configs(user: User) -> None: expected = dedent( """\ - name: Bond - age: 7 - """ + name: Bond + age: 7 + """ ) assert OmegaConf.to_yaml(user) == expected diff -pruN 2.1.0~rc1-3/tests/test_unions.py 2.2.2-1/tests/test_unions.py --- 2.1.0~rc1-3/tests/test_unions.py 1970-01-01 00:00:00.000000000 +0000 +++ 2.2.2-1/tests/test_unions.py 2022-05-27 21:36:40.000000000 +0000 @@ -0,0 +1,100 @@ +from pathlib import Path +from typing import Any, Union + +from pytest import mark, param, raises + +from omegaconf import OmegaConf, UnionNode, ValidationError +from omegaconf._utils import _get_value +from tests import Color + + +@mark.parametrize( + "union_args", + [ + param((int, float), id="int_float"), + param((float, bool), id="float_bool"), + param((bool, str), id="bool_str"), + param((str, bytes), id="str_bytes"), + param((bytes, Color), id="bytes_color"), + param((Color, int), id="color_int"), + ], +) +@mark.parametrize( + "input_", + [ + param(123, id="123"), + param(10.1, id="10.1"), + param(b"binary", id="binary"), + param(True, id="true"), + param("abc", id="abc"), + param("RED", id="red_str"), + param("123", id="123_str"), + param("10.1", id="10.1_str"), + param(Color.RED, id="Color.RED"), + param(Path("hello.txt"), id="path"), + param(object(), id="object"), + ], +) +class TestUnionNode: + def test_creation(self, input_: Any, union_args: Any) -> None: + ref_type = Union[union_args] # type: ignore + legal = type(input_) in union_args + if legal: + node = UnionNode(input_, ref_type) + assert _get_value(node) == input_ + else: + with raises(ValidationError): + UnionNode(input_, ref_type) + + def test_set_value(self, input_: Any, union_args: Any) -> None: + ref_type = Union[union_args] # type: ignore + legal = type(input_) in union_args + node = UnionNode(None, ref_type) + if legal: + node._set_value(input_) + assert _get_value(node) == input_ + else: + with raises(ValidationError): + node._set_value(input_) + + +@mark.parametrize( + "optional", [param(True, id="optional"), param(False, id="not_optional")] +) +@mark.parametrize( + "input_", + [ + param("???", id="missing"), + param("${interp}", id="interp"), + param(None, id="none"), + ], +) +class TestUnionNodeSpecial: + def test_creation_special(self, input_: Any, optional: bool) -> None: + if input_ is None and not optional: + with raises(ValidationError): + UnionNode(input_, Union[int, str], is_optional=optional) + else: + node = UnionNode(input_, Union[int, str], is_optional=optional) + assert node._value() == input_ + + def test_set_value_special(self, input_: Any, optional: bool) -> None: + node = UnionNode(123, Union[int, str], is_optional=optional) + if input_ is None and not optional: + with raises(ValidationError): + node._set_value(input_) + else: + node._set_value(input_) + assert node._value() == input_ + + +def test_get_parent_container() -> None: + cfg = OmegaConf.create({"foo": UnionNode(123, Union[int, str]), "bar": "baz"}) + + unode = cfg._get_node("foo") + nested_node = unode._value() # type: ignore + any_node = cfg._get_node("bar") + + assert unode._get_parent_container() is cfg # type: ignore + assert nested_node._get_parent_container() is cfg + assert any_node._get_parent_container() is cfg # type: ignore diff -pruN 2.1.0~rc1-3/tests/test_update.py 2.2.2-1/tests/test_update.py --- 2.1.0~rc1-3/tests/test_update.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/tests/test_update.py 2022-05-27 21:36:40.000000000 +0000 @@ -199,7 +199,7 @@ def test_update_force_add(cfg: Any, key: cfg = _ensure_container(cfg) OmegaConf.set_struct(cfg, True) - with raises((ConfigAttributeError, ConfigKeyError)): # type: ignore + with raises((ConfigAttributeError, ConfigKeyError)): OmegaConf.update(cfg, key, value, force_add=False) OmegaConf.update(cfg, key, value, force_add=True) diff -pruN 2.1.0~rc1-3/tests/test_utils.py 2.2.2-1/tests/test_utils.py --- 2.1.0~rc1-3/tests/test_utils.py 2021-05-13 00:32:41.000000000 +0000 +++ 2.2.2-1/tests/test_utils.py 2022-05-27 21:36:40.000000000 +0000 @@ -1,19 +1,30 @@ import re +import sys from dataclasses import dataclass, field from enum import Enum -from typing import Any, Dict, List, Optional, Tuple, Union +from pathlib import Path +from typing import Any, Callable, Dict, List, Optional, Tuple, Union import attr from pytest import mark, param, raises -from omegaconf import DictConfig, ListConfig, Node, OmegaConf, _utils +from omegaconf import DictConfig, ListConfig, Node, OmegaConf, UnionNode, _utils from omegaconf._utils import ( Marker, + NoneType, _ensure_container, _get_value, _is_optional, + _resolve_optional, + get_dict_key_value_types, + get_list_element_type, is_dict_annotation, is_list_annotation, + is_primitive_dict, + is_primitive_list, + is_supported_union_annotation, + is_tuple_annotation, + is_union_annotation, nullcontext, split_key, ) @@ -21,13 +32,74 @@ from omegaconf.errors import Unsupported from omegaconf.nodes import ( AnyNode, BooleanNode, + BytesNode, EnumNode, FloatNode, IntegerNode, + PathNode, StringNode, ) from omegaconf.omegaconf import _node_wrap -from tests import Color, ConcretePlugin, Dataframe, IllegalType, Plugin, User +from tests import ( + Color, + ConcretePlugin, + Dataframe, + DictSubclass, + IllegalType, + ListSubclass, + Plugin, + Shape, + Str2Int, + UnionAnnotations, + User, +) + + +class DummyEnum(Enum): + FOO = 1 + + +@mark.parametrize("is_optional", [True, False]) +@mark.parametrize( + "ref_type, value, expected_type", + [ + (Any, 10, AnyNode), + (DummyEnum, DummyEnum.FOO, EnumNode), + (int, 42, IntegerNode), + (bytes, b"\xf0\xf1\xf2", BytesNode), + (Path, Path("hello.txt"), PathNode), + (float, 3.1415, FloatNode), + (bool, True, BooleanNode), + (str, "foo", StringNode), + ], +) +def test_node_wrap( + ref_type: type, is_optional: bool, value: Any, expected_type: Any +) -> None: + from omegaconf.omegaconf import _node_wrap + + ret = _node_wrap( + ref_type=ref_type, + value=value, + is_optional=is_optional, + parent=None, + key=None, + ) + assert ret._metadata.ref_type == ref_type + assert type(ret) == expected_type + assert ret == value + + if is_optional: + ret = _node_wrap( + ref_type=ref_type, + value=None, + is_optional=is_optional, + parent=None, + key=None, + ) + assert type(ret) == expected_type + # noinspection PyComparisonWithNone + assert ret == None # noqa E711 @mark.parametrize( @@ -35,26 +107,37 @@ from tests import Color, ConcretePlugin, [ # Any param(Any, "foo", AnyNode("foo"), id="any"), + param(Any, b"binary", AnyNode(b"binary"), id="any"), + param(Any, Path("hello.txt"), AnyNode(Path("hello.txt")), id="any"), param(Any, True, AnyNode(True), id="any"), param(Any, 1, AnyNode(1), id="any"), param(Any, 1.0, AnyNode(1.0), id="any"), + param(Any, b"123", AnyNode(b"123"), id="any"), param(Any, Color.RED, AnyNode(Color.RED), id="any"), param(Any, {}, DictConfig(content={}), id="any_as_dict"), param(Any, [], ListConfig(content=[]), id="any_as_list"), # int param(int, "foo", ValidationError, id="int"), + param(int, b"binary", ValidationError, id="int"), + param(int, Path("hello.txt"), ValidationError, id="int"), param(int, True, ValidationError, id="int"), param(int, 1, IntegerNode(1), id="int"), param(int, 1.0, ValidationError, id="int"), param(int, Color.RED, ValidationError, id="int"), + param(int, b"123", ValidationError, id="int"), # float param(float, "foo", ValidationError, id="float"), + param(float, b"binary", ValidationError, id="float"), + param(float, Path("hello.txt"), ValidationError, id="float"), param(float, True, ValidationError, id="float"), param(float, 1, FloatNode(1), id="float"), param(float, 1.0, FloatNode(1.0), id="float"), param(float, Color.RED, ValidationError, id="float"), + param(float, b"123", ValidationError, id="float"), # bool param(bool, "foo", ValidationError, id="bool"), + param(bool, b"binary", ValidationError, id="bool"), + param(bool, Path("hello.txt"), ValidationError, id="bool"), param(bool, True, BooleanNode(True), id="bool"), param(bool, 1, BooleanNode(True), id="bool"), param(bool, 0, BooleanNode(False), id="bool"), @@ -64,12 +147,31 @@ from tests import Color, ConcretePlugin, param(bool, "false", BooleanNode(False), id="bool"), param(bool, "on", BooleanNode(True), id="bool"), param(bool, "off", BooleanNode(False), id="bool"), + param(bool, b"123", ValidationError, id="bool"), # str param(str, "foo", StringNode("foo"), id="str"), + param(str, b"binary", ValidationError, id="str"), + param(str, Path("hello.txt"), StringNode("hello.txt"), id="str"), param(str, True, StringNode("True"), id="str"), param(str, 1, StringNode("1"), id="str"), param(str, 1.0, StringNode("1.0"), id="str"), param(str, Color.RED, StringNode("Color.RED"), id="str"), + # bytes + param(bytes, "foo", ValidationError, id="bytes"), + param(bytes, b"binary", BytesNode(b"binary"), id="bytes"), + param(bytes, Path("hello.txt"), ValidationError, id="bytes"), + param(bytes, True, ValidationError, id="bytes"), + param(bytes, 1, ValidationError, id="bytes"), + param(bytes, 1.0, ValidationError, id="bytes"), + param(bytes, Color.RED, ValidationError, id="bytes"), + # Path + param(Path, "foo", PathNode("foo"), id="path"), + param(Path, b"binary", ValidationError, id="path"), + param(Path, Path("hello.txt"), PathNode("hello.txt"), id="path"), + param(Path, True, ValidationError, id="path"), + param(Path, 1, ValidationError, id="path"), + param(Path, 1.0, ValidationError, id="path"), + param(Path, Color.RED, ValidationError, id="path"), # Color param(Color, "foo", ValidationError, id="Color"), param(Color, True, ValidationError, id="Color"), @@ -80,23 +182,55 @@ from tests import Color, ConcretePlugin, param( Color, "Color.RED", EnumNode(enum_type=Color, value=Color.RED), id="Color" ), + param(Color, b"123", ValidationError, id="Color"), + param(Color, Path("hello.txt"), ValidationError, id="Color"), + param( + Color, "Color.RED", EnumNode(enum_type=Color, value=Color.RED), id="Color" + ), # bad type param(IllegalType, "nope", ValidationError, id="bad_type"), + param(IllegalType, [1, 2, 3], ValidationError, id="list_bad_type"), + param(IllegalType, {"foo": "bar"}, ValidationError, id="dict_bad_type"), # DictConfig param( - dict, {"foo": "bar"}, DictConfig(content={"foo": "bar"}), id="DictConfig" + Dict[Any, Any], + {"foo": "bar"}, + DictConfig(content={"foo": "bar"}), + id="DictConfig", ), + param(List[Any], {"foo": "bar"}, ValidationError, id="dict_to_list"), + param(List[int], {"foo": "bar"}, ValidationError, id="dict_to_list[int]"), + param( + Any, + {"foo": "bar"}, + DictConfig(content={"foo": "bar"}), + id="dict_to_any", + ), + param(Plugin, {"foo": "bar"}, ValidationError, id="dict_to_plugin"), + # Structured Config param(Plugin, Plugin(), DictConfig(content=Plugin()), id="DictConfig[Plugin]"), + param(Any, Plugin(), DictConfig(content=Plugin()), id="plugin_to_any"), + param( + Dict[str, int], + Plugin(), + ValidationError, + id="plugin_to_dict[str, int]", + ), + param(List[Any], Plugin(), ValidationError, id="plugin_to_list"), + param(List[int], Plugin(), ValidationError, id="plugin_to_list[int]"), # ListConfig - param(list, [1, 2, 3], ListConfig(content=[1, 2, 3]), id="ListConfig"), + param(List[Any], [1, 2, 3], ListConfig(content=[1, 2, 3]), id="ListConfig"), + param(Dict[Any, Any], [1, 2, 3], ValidationError, id="list_to_dict"), + param(Dict[str, int], [1, 2, 3], ValidationError, id="list_to_dict[str-int]"), + param(Any, [1, 2, 3], ListConfig(content=[1, 2, 3]), id="list_to_any"), ], ) -def test_node_wrap(target_type: Any, value: Any, expected: Any) -> None: +def test_node_wrap2(target_type: Any, value: Any, expected: Any) -> None: from omegaconf.omegaconf import _maybe_wrap if isinstance(expected, Node): res = _node_wrap( - type_=target_type, key="foo", value=value, is_optional=False, parent=None + ref_type=target_type, key="foo", value=value, is_optional=False, parent=None ) assert type(res) == type(expected) assert res == expected @@ -112,6 +246,22 @@ def test_node_wrap(target_type: Any, val ) +def test_node_wrap_illegal_type() -> None: + class UserClass: + pass + + from omegaconf.omegaconf import _node_wrap + + with raises(ValidationError): + _node_wrap( + ref_type=UserClass, + value=UserClass(), + is_optional=False, + parent=None, + key=None, + ) + + class _TestEnum(Enum): A = 1 B = 2 @@ -122,10 +272,13 @@ class _TestDataclass: x: int = 10 s: str = "foo" b: bool = True + p: Path = Path("hello.txt") + d: bytes = b"123" f: float = 3.14 e: _TestEnum = _TestEnum.A list1: List[int] = field(default_factory=list) dict1: Dict[str, int] = field(default_factory=dict) + init_false: str = field(init=False, default="foo") @attr.s(auto_attribs=True) @@ -133,10 +286,13 @@ class _TestAttrsClass: x: int = 10 s: str = "foo" b: bool = True + p: Path = Path("hello.txt") + d: bytes = b"123" f: float = 3.14 e: _TestEnum = _TestEnum.A list1: List[int] = [] dict1: Dict[str, int] = {} + init_false: str = attr.field(init=False, default="foo") @dataclass @@ -160,18 +316,32 @@ class _TestUserClass: (float, True), (bool, True), (str, True), + (bytes, True), + (Path, True), (Any, True), (_TestEnum, True), (_TestUserClass, False), # Nesting structured configs in contain (_TestAttrsClass, True), (_TestDataclass, True), + # container annotations + (List[int], True), + (Dict[str, int], True), + # optional and union + (Optional[int], True), + (Union[int, str], True), + (Union[int, List[str]], False), + (Union[int, Dict[int, str]], False), + (Union[int, _TestEnum], True), + (Union[int, _TestAttrsClass], False), + (Union[int, _TestDataclass], False), + (Union[int, _TestUserClass], False), ], ) -def test_valid_value_annotation_type(type_: type, expected: bool) -> None: - from omegaconf._utils import valid_value_annotation_type +def test_is_valid_value_annotation(type_: type, expected: bool) -> None: + from omegaconf._utils import is_valid_value_annotation - assert valid_value_annotation_type(type_) == expected + assert is_valid_value_annotation(type_) == expected class TestGetStructuredConfigInfo: @@ -184,6 +354,8 @@ class TestGetStructuredConfigInfo: assert d["x"] == 10 assert d["s"] == "foo" assert d["b"] == bool(True) + assert d["p"] == Path("hello.txt") + assert d["d"] == b"123" assert d["f"] == 3.14 assert d["e"] == _TestEnum.A assert d["list1"] == [] @@ -198,12 +370,12 @@ class TestGetStructuredConfigInfo: [_TestDataclass, _TestDataclass(), _TestAttrsClass, _TestAttrsClass()], ) def test_get_structured_config_field_names(self, test_cls_or_obj: Any) -> None: - field_names = _utils.get_structured_config_field_names(test_cls_or_obj) - assert field_names == ["x", "s", "b", "f", "e", "list1", "dict1"] + field_names = _utils.get_structured_config_init_field_names(test_cls_or_obj) + assert field_names == ["x", "s", "b", "p", "d", "f", "e", "list1", "dict1"] def test_get_structured_config_field_names_throws_ValueError(self) -> None: with raises(ValueError): - _utils.get_structured_config_field_names("invalid") + _utils.get_structured_config_init_field_names("invalid") @mark.parametrize( @@ -265,18 +437,55 @@ class Dataclass: ("foo", _utils.ValueKind.VALUE), (1, _utils.ValueKind.VALUE), (1.0, _utils.ValueKind.VALUE), + (b"123", _utils.ValueKind.VALUE), + (Path("hello.txt"), _utils.ValueKind.VALUE), (True, _utils.ValueKind.VALUE), (False, _utils.ValueKind.VALUE), (Color.GREEN, _utils.ValueKind.VALUE), (Dataclass, _utils.ValueKind.VALUE), (Dataframe(), _utils.ValueKind.VALUE), + (IntegerNode(123), _utils.ValueKind.VALUE), + (DictConfig({}), _utils.ValueKind.VALUE), + (ListConfig([]), _utils.ValueKind.VALUE), + (AnyNode(123), _utils.ValueKind.VALUE), + (UnionNode(123, Union[int, str]), _utils.ValueKind.VALUE), ("???", _utils.ValueKind.MANDATORY_MISSING), + (IntegerNode("???"), _utils.ValueKind.MANDATORY_MISSING), + (DictConfig("???"), _utils.ValueKind.MANDATORY_MISSING), + (ListConfig("???"), _utils.ValueKind.MANDATORY_MISSING), + (AnyNode("???"), _utils.ValueKind.MANDATORY_MISSING), + (UnionNode("???", Union[int, str]), _utils.ValueKind.MANDATORY_MISSING), ("${foo.bar}", _utils.ValueKind.INTERPOLATION), ("ftp://${host}/path", _utils.ValueKind.INTERPOLATION), ("${func:foo}", _utils.ValueKind.INTERPOLATION), ("${func:a/b}", _utils.ValueKind.INTERPOLATION), ("${func:c:\\a\\b}", _utils.ValueKind.INTERPOLATION), ("${func:c:\\a\\b}", _utils.ValueKind.INTERPOLATION), + param( + IntegerNode("${func:c:\\a\\b}"), + _utils.ValueKind.INTERPOLATION, + id="integernode-interp", + ), + param( + DictConfig("${func:c:\\a\\b}"), + _utils.ValueKind.INTERPOLATION, + id="dictconfig-interp", + ), + param( + ListConfig("${func:c:\\a\\b}"), + _utils.ValueKind.INTERPOLATION, + id="listconfig-interp", + ), + param( + AnyNode("${func:c:\\a\\b}"), + _utils.ValueKind.INTERPOLATION, + id="anynode-interp", + ), + param( + UnionNode("${func:c:\\a\\b}", Union[int, str]), + _utils.ValueKind.INTERPOLATION, + id="unionnode-interp", + ), ], ) def test_value_kind(value: Any, kind: _utils.ValueKind) -> None: @@ -289,17 +498,24 @@ def test_re_parent() -> None: assert cfg1._get_node("str")._get_parent() == cfg1 # type: ignore assert cfg1._get_node("list")._get_parent() == cfg1 # type: ignore assert cfg1.list._get_node(0)._get_parent() == cfg1.list + unode1 = cfg1._get_node("union") + assert unode1._get_parent() == cfg1 # type: ignore + assert unode1._value()._get_parent() == unode1 # type: ignore cfg = OmegaConf.create({}) assert isinstance(cfg, DictConfig) cfg.str = StringNode("str") cfg.list = [1] + cfg.union = UnionNode(123, Union[int, str]) validate(cfg) cfg._get_node("str")._set_parent(None) # type: ignore cfg._get_node("list")._set_parent(None) # type: ignore cfg.list._get_node(0)._set_parent(None) # type:ignore + unode = cfg._get_node("union") + unode._set_parent(None) # type: ignore + unode._value()._set_parent(None) # type: ignore # noinspection PyProtectedMember cfg._re_parent() validate(cfg) @@ -351,8 +567,10 @@ def test_get_key_value_types( (int, True), (float, True), (bool, True), + (bytes, True), (str, True), - (type(None), True), + (Path, True), + (NoneType, True), (Color, True), (list, False), (ListConfig, False), @@ -360,75 +578,165 @@ def test_get_key_value_types( (DictConfig, False), ], ) -def test_is_primitive_type(type_: Any, is_primitive: bool) -> None: - assert _utils.is_primitive_type(type_) == is_primitive +def test_is_primitive_type_annotation(type_: Any, is_primitive: bool) -> None: + assert _utils.is_primitive_type_annotation(type_) == is_primitive @mark.parametrize("optional", [False, True]) @mark.parametrize( - "type_, expected", + "type_, include_module_name, expected", [ - (int, "int"), - (bool, "bool"), - (float, "float"), - (str, "str"), - (Color, "Color"), - (DictConfig, "DictConfig"), - (ListConfig, "ListConfig"), - (Dict[str, str], "Dict[str, str]"), - (Dict[Color, int], "Dict[Color, int]"), - (Dict[str, Plugin], "Dict[str, Plugin]"), - (Dict[str, List[Plugin]], "Dict[str, List[Plugin]]"), - (List[str], "List[str]"), - (List[Color], "List[Color]"), - (List[Dict[str, Color]], "List[Dict[str, Color]]"), - (Tuple[str], "Tuple[str]"), - (Tuple[str, int], "Tuple[str, int]"), - (Tuple[float, ...], "Tuple[float, ...]"), + (int, False, "int"), + (int, True, "int"), + (bool, False, "bool"), + (bool, True, "bool"), + (bytes, False, "bytes"), + (bytes, True, "bytes"), + (float, False, "float"), + (float, True, "float"), + (str, False, "str"), + (str, True, "str"), + (Path, False, "Path"), + (Path, True, "pathlib.Path"), + (Color, False, "Color"), + (Color, True, "tests.Color"), + (DictConfig, False, "DictConfig"), + (DictConfig, True, "DictConfig"), + (ListConfig, False, "ListConfig"), + (ListConfig, True, "ListConfig"), + (Dict[str, str], False, "Dict[str, str]"), + (Dict[str, str], True, "Dict[str, str]"), + (Dict[Color, int], False, "Dict[Color, int]"), + (Dict[Color, int], True, "Dict[tests.Color, int]"), + (Dict[str, Plugin], False, "Dict[str, Plugin]"), + (Dict[str, Plugin], True, "Dict[str, tests.Plugin]"), + (Dict[str, List[Plugin]], False, "Dict[str, List[Plugin]]"), + (Dict[str, List[Plugin]], True, "Dict[str, List[tests.Plugin]]"), + (dict, False, "dict"), + (dict, True, "dict"), + (List[str], False, "List[str]"), + (List[str], True, "List[str]"), + (List[Color], False, "List[Color]"), + (List[Color], True, "List[tests.Color]"), + (List[Dict[str, Color]], False, "List[Dict[str, Color]]"), + (List[Dict[str, Color]], True, "List[Dict[str, tests.Color]]"), + (list, False, "list"), + (list, True, "list"), + (Tuple[str], False, "Tuple[str]"), + (Tuple[str], True, "Tuple[str]"), + (Tuple[str, int], False, "Tuple[str, int]"), + (Tuple[str, int], True, "Tuple[str, int]"), + (Tuple[float, ...], False, "Tuple[float, ...]"), + (Tuple[float, ...], True, "Tuple[float, ...]"), + (tuple, False, "tuple"), + (tuple, True, "tuple"), + (Union[str, int, Color], False, "Union[str, int, Color]"), + (Union[str, int, Color], True, "Union[str, int, tests.Color]"), + (Union[int], False, "int"), + (Union[int], True, "int"), ], ) -def test_type_str(type_: Any, expected: str, optional: bool) -> None: +def test_type_str( + type_: Any, include_module_name: bool, expected: str, optional: bool +) -> None: if optional: - assert _utils.type_str(Optional[type_]) == f"Optional[{expected}]" + assert ( + _utils.type_str(Optional[type_], include_module_name=include_module_name) + == f"Optional[{expected}]" + ) else: - assert _utils.type_str(type_) == expected + assert ( + _utils.type_str(type_, include_module_name=include_module_name) == expected + ) def test_type_str_ellipsis() -> None: assert _utils.type_str(...) == "..." -def test_type_str_none() -> None: - assert _utils.type_str(None) == "NoneType" - - @mark.parametrize( "type_, expected", [ - (Optional[int], "Optional[int]"), - (Union[str, int, Color], "Union[str, int, Color]"), - (Optional[Union[int]], "Optional[int]"), - (Optional[Union[int, str]], "Union[int, str, NoneType]"), + param(None, "NoneType", id="none"), + param(NoneType, "NoneType", id="nonetype"), + (Union[float, bool, None], "Optional[Union[float, bool]]"), + (Union[float, bool, NoneType], "Optional[Union[float, bool]]"), ], ) -def test_type_str_union(type_: Any, expected: str) -> None: +def test_type_str_nonetype(type_: Any, expected: str) -> None: assert _utils.type_str(type_) == expected @mark.parametrize( + "obj, expected", + [ + param([], True, id="list"), + param([1], True, id="list1"), + param((), True, id="tuple"), + param((1,), True, id="tuple1"), + param({}, False, id="dict"), + param(ListSubclass(), True, id="list_subclass"), + param(Shape(10, 2, 3), True, id="namedtuple"), + ], +) +def test_is_primitive_list(obj: Any, expected: bool) -> None: + assert is_primitive_list(obj) == expected + + +@mark.parametrize( + "obj, expected", + [ + param({}, True, id="dict"), + param({1: 2}, True, id="dict1"), + param([], False, id="list"), + param((), False, id="tuple"), + ], +) +def test_is_primitive_dict(obj: Any, expected: bool) -> None: + assert is_primitive_dict(obj) == expected + + +@mark.parametrize( + "obj", + [ + param(DictConfig({}), id="dictconfig"), + param(ListConfig([]), id="listconfig"), + param(DictSubclass(), id="dict_subclass"), + param(Str2Int(), id="dict_subclass_dataclass"), + param(User, id="user"), + param(User("bond", 7), id="user"), + ], +) +class TestIsPrimitiveContainerNegative: + def test_is_primitive_list(self, obj: Any) -> None: + assert not is_primitive_list(obj) + + def test_is_primitive_dict(self, obj: Any) -> None: + assert not is_primitive_dict(obj) + + +@mark.parametrize( "type_, expected", [ (Dict[str, int], True), (Dict[str, float], True), + (Dict[bytes, bytes], True), (Dict[IllegalType, bool], True), (Dict[str, IllegalType], True), (Dict[int, Color], True), (Dict[Plugin, Plugin], True), (Dict[IllegalType, int], True), (Dict, True), + (Str2Int, True), + (Str2Int(), False), + (User, False), + (User(), False), (List, False), (dict, False), (DictConfig, False), + (Any, False), + (None, False), + (NoneType, False), ], ) def test_is_dict_annotation(type_: Any, expected: Any) -> Any: @@ -441,6 +749,7 @@ def test_is_dict_annotation(type_: Any, (List[int], True), (List[float], True), (List[bool], True), + (List[bytes], True), (List[str], True), (List[Color], True), (List[Plugin], True), @@ -457,12 +766,88 @@ def test_is_list_annotation(type_: Any, @mark.parametrize( + "type_, expected", + [ + (Tuple[int], True), + (Tuple[float], True), + (Tuple[bool], True), + (Tuple[str], True), + (Tuple[Color], True), + (Tuple[Plugin], True), + (Tuple[IllegalType], True), + (Dict, False), + (List, False), + (Tuple, True), + (list, False), + (dict, False), + (tuple, False), + (Any, False), + (int, False), + (User, False), + (None, False), + (NoneType, False), + ], +) +def test_is_tuple_annotation(type_: Any, expected: Any) -> Any: + assert is_tuple_annotation(type_=type_) == expected + + +@mark.parametrize( + "input_, expected", + [ + (Union[int, str], True), + (Union[int, List[str]], True), + (Optional[Union[int, str]], True), + (Union[int, None], True), + (Optional[int], True), + (Any, False), + (int, False), + (User, False), + (None, False), + (NoneType, False), + ], +) +def test_is_union_annotation(input_: Any, expected: bool) -> None: + assert is_union_annotation(input_) == expected + + +@mark.skipif(sys.version_info < (3, 10), reason="requires Python 3.10 or newer") +def test_is_union_annotation_PEP604() -> None: + if sys.version_info >= (3, 10): # this if-statement is for mypy's benefit + assert is_union_annotation(int | str) + + +@mark.parametrize( + "input_, expected", + [ + (Union[int, str], True), + (Union[int, List[str]], False), + (Union[int, Dict[str, int]], False), + (Union[int, User], False), + (Optional[Union[int, str]], True), + (Union[int, None], True), + (Optional[int], True), + (Any, False), + (int, False), + (User, False), + (None, False), + (NoneType, False), + ], +) +def test_is_supported_union_annotation(input_: Any, expected: bool) -> None: + assert is_supported_union_annotation(input_) == expected + + +@mark.parametrize( "obj, expected", [ # Unwrapped values param(10, Any, id="int"), param(10.0, Any, id="float"), param(True, Any, id="bool"), + param(Color.RED, Any, id="enum"), + param(b"binary", Any, id="bytes"), + param(Path("hello.txt"), Any, id="path"), param("bar", Any, id="str"), param(None, Any, id="NoneType"), param({}, Any, id="dict"), @@ -475,6 +860,7 @@ def test_is_list_annotation(type_: Any, param(FloatNode(10.0), Optional[float], id="FloatNode"), param(BooleanNode(True), Optional[bool], id="BooleanNode"), param(StringNode("bar"), Optional[str], id="StringNode"), + param(BytesNode(b"binary"), Optional[bytes], id="BooleanNode"), param( EnumNode(enum_type=Color, value=Color.RED), Optional[Color], @@ -485,6 +871,7 @@ def test_is_list_annotation(type_: Any, param(FloatNode(10.0, is_optional=False), float, id="FloatNode"), param(BooleanNode(True, is_optional=False), bool, id="BooleanNode"), param(StringNode("bar", is_optional=False), str, id="StringNode"), + param(BytesNode(b"binary", is_optional=False), bytes, id="BytesNode"), param( EnumNode(enum_type=Color, value=Color.RED, is_optional=False), Color, @@ -551,7 +938,7 @@ def test_is_list_annotation(type_: Any, ], ) def test_get_ref_type(obj: Any, expected: Any) -> None: - assert _utils.get_ref_type(obj) == expected + assert _utils.get_type_hint(obj) == expected @mark.parametrize( @@ -561,16 +948,25 @@ def test_get_ref_type(obj: Any, expected param(User, "name", str, id="User.name"), param(User, "age", int, id="User.age"), param({"user": User}, "user", Any, id="user"), + param( + OmegaConf.structured(UnionAnnotations), "ubf", Union[bool, float], id="ubf" + ), + param( + OmegaConf.structured(UnionAnnotations), + "oubf", + Optional[Union[bool, float]], + id="oubf", + ), ], ) def test_get_node_ref_type(obj: Any, key: str, expected: Any) -> None: cfg = OmegaConf.create(obj) - assert _utils.get_ref_type(cfg, key) == expected + assert _utils.get_type_hint(cfg, key) == expected def test_get_ref_type_error() -> None: with raises(ValueError): - _utils.get_ref_type(AnyNode(), "foo") + _utils.get_type_hint(AnyNode(), "foo") @mark.parametrize( @@ -584,9 +980,10 @@ def test_get_ref_type_error() -> None: ) def test_get_value_basic(value: Any) -> None: val_node = _node_wrap( - value=value, type_=Any, parent=None, is_optional=True, key=None + value=value, ref_type=Any, parent=None, is_optional=True, key=None ) - assert _get_value(val_node) == value + result = _get_value(val_node) + assert result == value @mark.parametrize( @@ -599,6 +996,36 @@ def test_get_value_container(content: An assert _get_value(cfg) == content +@mark.parametrize( + "node, expected", + [ + param(AnyNode(123), 123, id="anynode"), + param(IntegerNode(123), 123, id="integernode"), + param(ListConfig([1, 2, 3]), ListConfig([1, 2, 3]), id="listconfig"), + param(123, 123, id="int"), + param("${a}", "${a}", id="raw-interp"), + param(DictConfig("${a}"), "${a}", id="dict-interp"), + param(AnyNode("${a}"), "${a}", id="any-interp"), + param(IntegerNode("${a}"), "${a}", id="int-interp"), + param(UnionNode("${a}", Union[int, str]), "${a}", id="union-interp"), + param(DictConfig("???"), "???", id="dict-missing"), + param(AnyNode("???"), "???", id="any-missing"), + param(IntegerNode("???"), "???", id="int-missing"), + param(UnionNode("???", Union[int, str]), "???", id="union-missing"), + param(DictConfig(None), None, id="dict-none"), + param(AnyNode(None), None, id="any-none"), + param(IntegerNode(None), None, id="int-none"), + param(UnionNode(None, Union[int, str]), None, id="union-none"), + param(DictConfig({"foo": "bar"}), DictConfig({"foo": "bar"}), id="dictconfig"), + param(UnionNode(123, Union[int, str]), 123, id="union[int]"), + ], +) +def test_get_value_of_node_subclass(node: Node, expected: Any) -> None: + result = _get_value(node) + assert result == expected + assert type(result) == type(expected) + + def test_ensure_container_raises_ValueError() -> None: """Some values cannot be converted to a container. On these inputs, _ensure_container should raise a ValueError.""" @@ -666,7 +1093,7 @@ def test_nullcontext() -> None: ), ( lambda is_optional, missing: FloatNode( - value=10 if not missing else "???", is_optional=is_optional + value=10.0 if not missing else "???", is_optional=is_optional ) ), ( @@ -675,6 +1102,11 @@ def test_nullcontext() -> None: ) ), ( + lambda is_optional, missing: BytesNode( + value=b"binary" if not missing else "???", is_optional=is_optional + ) + ), + ( lambda is_optional, missing: EnumNode( enum_type=Color, value=Color.RED if not missing else "???", @@ -699,6 +1131,13 @@ def test_nullcontext() -> None: is_optional=is_optional, ) ), + ( + lambda is_optional, missing: UnionNode( + ref_type=Union[int, str], + content=123 if not missing else "???", + is_optional=is_optional, + ) + ), ], ) def test_is_optional(fac: Any, is_optional: bool) -> None: @@ -713,3 +1152,238 @@ def test_is_optional(fac: Any, is_option cfg = OmegaConf.create({"node": obj}) assert _is_optional(cfg, "node") == is_optional + + +@mark.parametrize( + "type_", + [ + param(lambda val=123: val, id="passthrough"), + param(lambda val=123: AnyNode(val), id="any_node"), + param(lambda val=123: IntegerNode(val), id="integer_node"), + param(lambda val={}: DictConfig(val), id="dict_config"), + param(lambda val=[]: ListConfig(val), id="list_config"), + param(lambda val=123: UnionNode(val, Union[int, str]), id="union_node"), + ], +) +class TestIndicators: + @mark.parametrize( + "input_, expected", + [ + param("???", True, id="missing"), + param("${interp}", False, id="interp"), + param(None, False, id="none"), + param("DEFAULT", False, id="default"), + ], + ) + def test_is_missing( + self, type_: Callable[..., Any], input_: Any, expected: bool + ) -> None: + value = type_(input_) if input_ != "DEFAULT" else type_() + assert _utils._is_missing_value(value) == expected + + @mark.parametrize( + "input_, expected", + [ + param("???", False, id="missing"), + param("${interp}", True, id="interp"), + param(None, False, id="none"), + param("DEFAULT", False, id="default"), + ], + ) + def test_is_interpolation( + self, type_: Callable[..., Any], input_: Any, expected: bool + ) -> None: + value = type_(input_) if input_ != "DEFAULT" else type_() + assert _utils._is_interpolation(value) == expected + + @mark.parametrize( + "input_, expected", + [ + param("???", False, id="missing"), + param("${interp}", False, id="interp"), + param(None, True, id="none"), + param("DEFAULT", False, id="default"), + ], + ) + def test_is_none( + self, type_: Callable[..., Any], input_: Any, expected: bool + ) -> None: + value = type_(input_) if input_ != "DEFAULT" else type_() + assert _utils._is_none(value) == expected + + @mark.parametrize( + "input_, expected", + [ + param("???", True, id="missing"), + param("${interp}", True, id="interp"), + param(None, True, id="none"), + param("DEFAULT", False, id="default"), + ], + ) + def test_is_special( + self, type_: Callable[..., Any], input_: Any, expected: bool + ) -> None: + value = type_(input_) if input_ != "DEFAULT" else type_() + assert _utils._is_special(value) == expected + + +@mark.parametrize( + "ref_type, expected_key_type, expected_element_type", + [ + param(Dict, Any, Any, id="any"), + param(Dict[Any, Any], Any, Any, id="any_explicit"), + param(Dict[int, float], int, float, id="int_float"), + param(Dict[Color, User], Color, User, id="color_user"), + param(Dict[str, List[int]], str, List[int], id="list"), + param(Dict[str, Dict[int, float]], str, Dict[int, float], id="dict"), + ], +) +def test_get_dict_key_value_types( + ref_type: Any, expected_key_type: Any, expected_element_type: Any +) -> None: + key_type, element_type = get_dict_key_value_types(ref_type) + assert key_type == expected_key_type + assert element_type == expected_element_type + + +@mark.parametrize( + "ref_type, expected_element_type", + [ + param(List, Any, id="any"), + param(List[Any], Any, id="any_explicit"), + param(List[int], int, id="int"), + param(List[User], User, id="user"), + param(List[List[int]], List[int], id="list"), + param(List[Dict[int, float]], Dict[int, float], id="dict"), + ], +) +def test_get_list_element_type(ref_type: Any, expected_element_type: Any) -> None: + assert get_list_element_type(ref_type) == expected_element_type + + +@mark.parametrize( + "type_, expected_optional, expected_type", + [ + param(int, False, int, id="int"), + param(Any, True, Any, id="any"), + param(Color, False, Color, id="color"), + param(Optional[str], True, str, id="str"), + param(Optional[Any], True, Any, id="o[any]"), + param(Union[int, str], False, Union[int, str], id="int-str"), + param(Union[str, int], False, Union[str, int], id="str-int"), + param(Dict[str, int], False, Dict[str, int], id="dict[str,int]"), + param(Dict, False, Dict, id="dict"), + param(Dict[Any, Any], False, Dict[Any, Any], id="dict[any,any]"), + param(Optional[Dict[str, int]], True, Dict[str, int], id="o[dict[str,int]]"), + param(Optional[Dict], True, Dict, id="dict"), + param( + Dict[str, Optional[int]], + False, + Dict[str, Optional[int]], + id="dict[str,o[int]]", + ), + param(Union[int, None], True, int, id="int-none"), + param(Union[int, NoneType], True, int, id="int-nonetype"), + param(Union[Optional[int], None], True, int, id="o[int]-none"), + param(Union[Any, None], True, Any, id="any-none"), + param(Union[None, int], True, int, id="none-int"), + param(Union[None, None], True, NoneType, id="none-none"), + param(Union[None, NoneType], True, NoneType, id="none-nonetype"), + param(Union[None, Optional[None]], True, NoneType, id="none-o[none]"), + param(None, True, NoneType, id="none"), + param(NoneType, True, NoneType, id="nonetype"), + param(Union[int, int], False, int, id="int-int"), + param(Union[int, Optional[int]], True, int, id="int-o[int]"), + param(Union[int], False, int, id="u[int]"), + param(Union[Optional[int]], True, int, id="u[o[int]]"), + param(Optional[Union[int]], True, int, id="o[u[int]]"), + param(Union[int, Optional[str]], True, Union[int, str], id="int-o[str]"), + param(Optional[Optional[int]], True, int, id="o[o[int]]"), + param(Optional[Optional[Any]], True, Any, id="o[o[any]]"), + param(User, False, User, id="user"), + param(Optional[User], True, User, id="o[user]"), + param(Union[User, int], False, Union[User, int], id="user-int"), + param(Optional[Union[User, int]], True, Union[User, int], id="o[user-int]"), + param(Union[Optional[User], int], True, Union[User, int], id="o[user]-int"), + param(Union[User, Optional[int]], True, Union[User, int], id="user-o[int]"), + param(Optional[Union[int, str]], True, Union[int, str], id="o[u[int-str]]"), + param(Union[Optional[int], str], True, Union[int, str], id="u[o[int]-str]]"), + param( + Optional[Union[Optional[int], str]], + True, + Union[int, str], + id="o[u[o[int]-str]]]", + ), + param( + Union[Optional[int], Optional[str]], + True, + Union[int, str], + id="u[o[int]-o[str]]]", + ), + param(Union[int, str, None], True, Union[int, str], id="u[int-str-none]"), + param(Union[int, str, None], True, Union[int, str], id="u[int-str-nonetype]"), + param( + Union[User, Union[int, str]], + False, + Union[User, int, str], + id="user-[int-str]", + ), + param( + Union[User, NoneType, Union[int, str]], + True, + Union[User, int, str], + id="user-nonetype-[int-str]", + ), + param( + Union[User, None, Union[int, str]], + True, + Union[User, int, str], + id="user-none-[int-str]", + ), + param( + Union[User, None, Union[Optional[int], str]], + True, + Union[User, int, str], + id="user-none-[o[int]-str]", + ), + param( + Union[User, Union[Optional[int], str]], + True, + Union[User, int, str], + id="user-none-[o[int]-str]", + ), + param( + Union[float, bool, None], True, Union[float, bool], id="u[float-bool-none]" + ), + param( + Union[float, bool, NoneType], + True, + Union[float, bool], + id="u[float-bool-nonetype]", + ), + ], +) +def test_resolve_optional( + type_: Any, expected_optional: bool, expected_type: Any +) -> None: + resolved_optional, resolved_type = _resolve_optional(type_) + assert resolved_optional == expected_optional + assert resolved_type == expected_type + + +@mark.skipif(sys.version_info < (3, 10), reason="requires Python 3.10 or newer") +def test_resolve_optional_support_pep_604() -> None: + if sys.version_info >= (3, 10): # this if-statement is for mypy's benefit + assert _resolve_optional(int | str) == (False, Union[int, str]) + assert _resolve_optional(Optional[int | str]) == (True, Union[int, str]) + assert _resolve_optional(int | Optional[str]) == (True, Union[int, str]) + assert _resolve_optional(int | Union[str, float]) == ( + False, + Union[int, str, float], + ) + assert _resolve_optional(int | Union[str, Optional[float]]) == ( + True, + Union[int, str, float], + ) + assert _resolve_optional(int | str | None) == (True, Union[int, str]) + assert _resolve_optional(int | str | NoneType) == (True, Union[int, str])