From d5ce672c42df7552023569df78b5cda5a87e0a72 Mon Sep 17 00:00:00 2001 From: bogdandm Date: Sat, 20 Apr 2019 12:07:46 +0300 Subject: [PATCH 01/12] Generate post_init converters for StringSerializable types in dataclasses; Change GenericModelCodeGenerator.decorators signature --- json_to_models/models/attr.py | 16 +----- json_to_models/models/base.py | 21 +++++--- json_to_models/models/dataclasses.py | 81 +++++++++++++++++++++------- json_to_models/utils.py | 37 +++++++++++++ test/test_etc.py | 40 +++++++++++++- testing_tools/real_apis/f1.py | 4 +- 6 files changed, 157 insertions(+), 42 deletions(-) diff --git a/json_to_models/models/attr.py b/json_to_models/models/attr.py index b4378fb..c601cbf 100644 --- a/json_to_models/models/attr.py +++ b/json_to_models/models/attr.py @@ -29,21 +29,9 @@ def __init__(self, model: ModelMeta, meta=False, attrs_kwargs: dict = None, **kw self.no_meta = not meta self.attrs_kwargs = attrs_kwargs or {} - def generate(self, nested_classes: List[str] = None) -> Tuple[ImportPathList, str]: - """ - :param nested_classes: list of strings that contains classes code - :return: list of import data, class code - """ - imports, code = super().generate(nested_classes) - imports.append(('attr', None)) - return imports, code - @property - def decorators(self) -> List[str]: - """ - :return: List of decorators code (without @) - """ - return [self.ATTRS.render(kwargs=self.attrs_kwargs)] + def decorators(self) -> Tuple[ImportPathList, List[str]]: + return [('attr', None)], [self.ATTRS.render(kwargs=self.attrs_kwargs)] def field_data(self, name: str, meta: MetaData, optional: bool) -> Tuple[ImportPathList, dict]: """ diff --git a/json_to_models/models/base.py b/json_to_models/models/base.py index af78bfe..fa682a0 100644 --- a/json_to_models/models/base.py +++ b/json_to_models/models/base.py @@ -8,6 +8,7 @@ from . import INDENT, ModelsStructureType, OBJECTS_DELIMITER, indent, sort_fields from ..dynamic_typing import AbsoluteModelRef, ImportPathList, MetaData, ModelMeta, compile_imports, metadata_to_typing +from ..utils import cached_classmethod METADATA_FIELD_NAME = "J2M_ORIGINAL_FIELD" KWAGRS_TEMPLATE = "{% for key, value in kwargs.items() %}" \ @@ -18,6 +19,7 @@ keywords_set = set(keyword.kwlist) ones = ['', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine'] + def template(pattern: str, indent: str = INDENT) -> Template: """ Remove indent from triple-quotes string and return jinja2.Template instance @@ -56,6 +58,9 @@ class {{ name }}: {%- else %} pass {%- endif -%} + {%- if extra %} + {{ extra }} + {%- endif -%} """) FIELD: Template = template("{{name}}: {{type}}{% if body %} = {{ body }}{% endif %}") @@ -63,29 +68,31 @@ class {{ name }}: def __init__(self, model: ModelMeta, **kwargs): self.model = model - def generate(self, nested_classes: List[str] = None) -> Tuple[ImportPathList, str]: + def generate(self, nested_classes: List[str] = None, extra: str = "") -> Tuple[ImportPathList, str]: """ :param nested_classes: list of strings that contains classes code :return: list of import data, class code """ imports, fields = self.fields + decorator_imports, decorators = self.decorators data = { - "decorators": self.decorators, + "decorators": decorators, "name": self.model.name, - "fields": fields + "fields": fields, + "extra": extra } if nested_classes: data["nested"] = [indent(s) for s in nested_classes] return imports, self.BODY.render(**data) @property - def decorators(self) -> List[str]: + def decorators(self) -> Tuple[ImportPathList, List[str]]: """ - :return: List of decorators code (without @) + :return: List of imports and List of decorators code (without @) """ - return [] + return [], [] - @classmethod + @cached_classmethod def convert_field_name(cls, name): if name in keywords_set: name += "_" diff --git a/json_to_models/models/dataclasses.py b/json_to_models/models/dataclasses.py index 579038e..8d2c363 100644 --- a/json_to_models/models/dataclasses.py +++ b/json_to_models/models/dataclasses.py @@ -11,11 +11,60 @@ ) +def f(self): + for name in ('',): + t = self.__annotations__[name] + setattr(self, name, t.to_internal_value(getattr(self, name))) + + +def dataclass_post_init_converters(str_fields: List[str]): + """ + Method factory. Return post_init method to convert string into StringSerializable types + To override generated __post_init__ you can call it directly: + + >>> def __post_init__(self): + ... dataclass_post_init_converters(['a', 'b'])(self) + + :param str_fields: names of StringSerializable fields + :return: __post_init__ method + """ + + def __post_init__(self): + for name in (str_fields): + t = self.__annotations__[name] + setattr(self, name, t.to_internal_value(getattr(self, name))) + + return __post_init__ + + +def convert_strings(str_fields: List[str]): + """ + Decorator factory. Set up `__post_init__` method to convert strings fields values into StringSerializable types + + :param str_fields: names of StringSerializable fields + :return: Class decorator + """ + + def decorator(cls): + if hasattr(cls, '__post__init__'): + old_fn = cls.__post__init__ + + def __post__init__(self, *args, **kwargs): + dataclass_post_init_converters(str_fields)(self) + old_fn(self, *args, **kwargs) + + setattr(cls, '__post_init__', __post__init__) + else: + setattr(cls, '__post_init__', dataclass_post_init_converters(str_fields)) + + return cls + + return decorator + + class DataclassModelCodeGenerator(GenericModelCodeGenerator): - DC_DECORATOR = template("dataclass" - "{% if kwargs %}" - f"({KWAGRS_TEMPLATE})" - "{% endif %}") + DC_DECORATOR = template(f"dataclass{{% if kwargs %}}({KWAGRS_TEMPLATE}){{% endif %}}") + DC_CONVERT_DECORATOR = template("convert_strings({{ str_fields }})") DC_FIELD = template(f"field({KWAGRS_TEMPLATE})") def __init__(self, model: ModelMeta, meta=False, post_init_converters=False, dataclass_kwargs: dict = None, @@ -32,21 +81,17 @@ def __init__(self, model: ModelMeta, meta=False, post_init_converters=False, dat self.no_meta = not meta self.dataclass_kwargs = dataclass_kwargs or {} - def generate(self, nested_classes: List[str] = None) -> Tuple[ImportPathList, str]: - """ - :param nested_classes: list of strings that contains classes code - :return: list of import data, class code - """ - imports, code = super().generate(nested_classes) - imports.append(('dataclasses', ['dataclass, field'])) - return imports, code - @property - def decorators(self) -> List[str]: - """ - :return: List of decorators code (without @) - """ - return [self.DC_DECORATOR.render(kwargs=self.dataclass_kwargs)] + def decorators(self) -> Tuple[ImportPathList, List[str]]: + imports = [('dataclasses', ['dataclass', 'field'])] + decorators = [self.DC_DECORATOR.render(kwargs=self.dataclass_kwargs)] + if self.post_init_converters: + str_fields = [self.convert_field_name(name) for name, t in self.model.type.items() + if isclass(t) and issubclass(t, StringSerializable)] + if str_fields: + imports.append(('json_to_models.models.dataclasses', ['dataclass_post_init_converters'])) + decorators.append(self.DC_CONVERT_DECORATOR.render(str_fields=str_fields)) + return imports, decorators def field_data(self, name: str, meta: MetaData, optional: bool) -> Tuple[ImportPathList, dict]: """ diff --git a/json_to_models/utils.py b/json_to_models/utils.py index 5a9b76f..5b096a0 100644 --- a/json_to_models/utils.py +++ b/json_to_models/utils.py @@ -95,3 +95,40 @@ def decorator(fn): return convert_args(fn, *args_converters, **kwargs_converters) return decorator + + +def cached_method(func: Callable): + """ + Decorator to cache method return values + """ + null = object() + + @wraps(func) + def cached_fn(self, *args): + if getattr(self, '__cache__', None) is None: + setattr(self, '__cache__', {}) + value = self.__cache__.get(args, null) + if value is null: + value = func(self, *args) + self.__cache__[args] = value + return value + + return cached_fn + + +def cached_classmethod(func: Callable): + """ + Decorator to cache classmethod return values + """ + cache = {} + null = object() + + @wraps(func) + def cached_fn(cls, *args): + value = cache.get(args, null) + if value is null: + value = func(cls, *args) + cache[args] = value + return value + + return classmethod(cached_fn) diff --git a/test/test_etc.py b/test/test_etc.py index 074def8..9144e76 100644 --- a/test/test_etc.py +++ b/test/test_etc.py @@ -3,7 +3,8 @@ import pytest from inflection import singularize -from json_to_models.utils import Index, convert_args_decorator, distinct_words, json_format +from json_to_models.utils import (Index, cached_classmethod, cached_method, convert_args_decorator, distinct_words, + json_format) test_distinct_words_data = [ pytest.param(['test', 'foo', 'bar'], {'test', 'foo', 'bar'}), @@ -63,3 +64,40 @@ def test_convert_args_decorator(): assert f('1', b='1.5') == 2.5 a = A("2.3", "7.5") assert a.value == 9.8 + + +def test_cached_methods(): + class A: + x = [] + + def __init__(self): + self.y = [] + + @cached_method + def f(self, a): + self.y.append(a) + return a + + @cached_classmethod + def g(cls, a): + cls.x.append(a) + return a + + A.g('a') + A.g('a') + a = A() + A.g('b') + A.g('a') + a.f('b') + a.f('b') + a.f('a') + a.f('a') + b = A() + a.f('b') + b.f('a') + b.g('c') + + assert A.x == ['a', 'b', 'c'] + assert a.y == ['b', 'a'] + assert b.y == ['a'] + assert a.x == ['a', 'b', 'c'] diff --git a/testing_tools/real_apis/f1.py b/testing_tools/real_apis/f1.py index fad35c0..e5056ac 100644 --- a/testing_tools/real_apis/f1.py +++ b/testing_tools/real_apis/f1.py @@ -8,8 +8,8 @@ from json_to_models.dynamic_typing import register_datetime_classes from json_to_models.generator import MetadataGenerator from json_to_models.models import compose_models -from json_to_models.models.attr import AttrsModelCodeGenerator from json_to_models.models.base import generate_code +from json_to_models.models.dataclasses import DataclassModelCodeGenerator from json_to_models.registry import ModelRegistry from json_to_models.utils import json_format from testing_tools.pprint_meta_data import pretty_format_meta @@ -61,7 +61,7 @@ def main(): print('\n', json_format([structure[0], {str(a): str(b) for a, b in structure[1].items()}])) print("=" * 20) - print(generate_code(structure, AttrsModelCodeGenerator)) + print(generate_code(structure, DataclassModelCodeGenerator, class_generator_kwargs={"post_init_converters": True})) if __name__ == '__main__': From 6f5203a7e0e79959974eed7e10014733354149e4 Mon Sep 17 00:00:00 2001 From: bogdandm Date: Sat, 20 Apr 2019 12:13:57 +0300 Subject: [PATCH 02/12] Fix import path merging --- json_to_models/models/attr.py | 5 +---- json_to_models/models/base.py | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/json_to_models/models/attr.py b/json_to_models/models/attr.py index c601cbf..a574577 100644 --- a/json_to_models/models/attr.py +++ b/json_to_models/models/attr.py @@ -12,10 +12,7 @@ class AttrsModelCodeGenerator(GenericModelCodeGenerator): - ATTRS = template("attr.s" - "{% if kwargs %}" - f"({KWAGRS_TEMPLATE})" - "{% endif %}") + ATTRS = template(f"attr.s{{% if kwargs %}}({KWAGRS_TEMPLATE}){{% endif %}}") ATTRIB = template(f"attr.ib({KWAGRS_TEMPLATE})") def __init__(self, model: ModelMeta, meta=False, attrs_kwargs: dict = None, **kwargs): diff --git a/json_to_models/models/base.py b/json_to_models/models/base.py index fa682a0..3dd82da 100644 --- a/json_to_models/models/base.py +++ b/json_to_models/models/base.py @@ -83,7 +83,7 @@ def generate(self, nested_classes: List[str] = None, extra: str = "") -> Tuple[I } if nested_classes: data["nested"] = [indent(s) for s in nested_classes] - return imports, self.BODY.render(**data) + return [*imports, *decorator_imports], self.BODY.render(**data) @property def decorators(self) -> Tuple[ImportPathList, List[str]]: From 3c7cc8a62ee95276228c2f4546d06674d647b613 Mon Sep 17 00:00:00 2001 From: bogdandm Date: Sat, 20 Apr 2019 13:27:34 +0300 Subject: [PATCH 03/12] Cleanup and fixes --- TODO.md | 2 ++ .../dynamic_typing/string_serializable.py | 3 +++ json_to_models/models/dataclasses.py | 24 ++++++++----------- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/TODO.md b/TODO.md index 0dcc836..8f17ba2 100644 --- a/TODO.md +++ b/TODO.md @@ -14,6 +14,8 @@ - [X] attrs - [X] dataclasses - [ ] post_init converters for StringSerializable types + - [ ] Nested converters + - [ ] Cli argument - [ ] generate from_json/to_json converters - [ ] Model class -> Meta format converter - [ ] attrs diff --git a/json_to_models/dynamic_typing/string_serializable.py b/json_to_models/dynamic_typing/string_serializable.py index face266..b36d684 100644 --- a/json_to_models/dynamic_typing/string_serializable.py +++ b/json_to_models/dynamic_typing/string_serializable.py @@ -36,6 +36,9 @@ def to_typing_code(cls) -> Tuple[ImportPathList, str]: cls_name = cls.__name__ return [('json_to_models.dynamic_typing', cls_name)], cls_name + def __iter__(self): + return () + T_StringSerializable = Type[StringSerializable] diff --git a/json_to_models/models/dataclasses.py b/json_to_models/models/dataclasses.py index 8d2c363..ff7f524 100644 --- a/json_to_models/models/dataclasses.py +++ b/json_to_models/models/dataclasses.py @@ -1,5 +1,5 @@ from inspect import isclass -from typing import List, Tuple +from typing import Callable, List, Tuple from .base import GenericModelCodeGenerator, KWAGRS_TEMPLATE, METADATA_FIELD_NAME, sort_kwargs, template from ..dynamic_typing import DDict, DList, DOptional, ImportPathList, MetaData, ModelMeta, StringSerializable @@ -11,12 +11,6 @@ ) -def f(self): - for name in ('',): - t = self.__annotations__[name] - setattr(self, name, t.to_internal_value(getattr(self, name))) - - def dataclass_post_init_converters(str_fields: List[str]): """ Method factory. Return post_init method to convert string into StringSerializable types @@ -37,7 +31,7 @@ def __post_init__(self): return __post_init__ -def convert_strings(str_fields: List[str]): +def convert_strings(str_fields: List[str]) -> Callable[[type], type]: """ Decorator factory. Set up `__post_init__` method to convert strings fields values into StringSerializable types @@ -45,15 +39,15 @@ def convert_strings(str_fields: List[str]): :return: Class decorator """ - def decorator(cls): - if hasattr(cls, '__post__init__'): - old_fn = cls.__post__init__ + def decorator(cls: type) -> type: + if hasattr(cls, '__post_init__'): + old_fn = cls.__post_init__ - def __post__init__(self, *args, **kwargs): + def __post_init__(self, *args, **kwargs): dataclass_post_init_converters(str_fields)(self) old_fn(self, *args, **kwargs) - setattr(cls, '__post_init__', __post__init__) + setattr(cls, '__post_init__', __post_init__) else: setattr(cls, '__post_init__', dataclass_post_init_converters(str_fields)) @@ -85,12 +79,14 @@ def __init__(self, model: ModelMeta, meta=False, post_init_converters=False, dat def decorators(self) -> Tuple[ImportPathList, List[str]]: imports = [('dataclasses', ['dataclass', 'field'])] decorators = [self.DC_DECORATOR.render(kwargs=self.dataclass_kwargs)] + if self.post_init_converters: str_fields = [self.convert_field_name(name) for name, t in self.model.type.items() if isclass(t) and issubclass(t, StringSerializable)] if str_fields: - imports.append(('json_to_models.models.dataclasses', ['dataclass_post_init_converters'])) + imports.append(('json_to_models.models.dataclasses', ['convert_strings'])) decorators.append(self.DC_CONVERT_DECORATOR.render(str_fields=str_fields)) + return imports, decorators def field_data(self, name: str, meta: MetaData, optional: bool) -> Tuple[ImportPathList, dict]: From 5945cab3d35b0df436523fe42ebf1b112a403a64 Mon Sep 17 00:00:00 2001 From: bogdandm Date: Sat, 20 Apr 2019 19:27:03 +0300 Subject: [PATCH 04/12] Generate StringSerializable fields path --- .../dynamic_typing/string_serializable.py | 2 +- json_to_models/models/dataclasses.py | 72 ++++++++++++++-- .../test_dataclasses_generation.py | 84 +++++++++++++++++-- 3 files changed, 145 insertions(+), 13 deletions(-) diff --git a/json_to_models/dynamic_typing/string_serializable.py b/json_to_models/dynamic_typing/string_serializable.py index b36d684..400ccf5 100644 --- a/json_to_models/dynamic_typing/string_serializable.py +++ b/json_to_models/dynamic_typing/string_serializable.py @@ -37,7 +37,7 @@ def to_typing_code(cls) -> Tuple[ImportPathList, str]: return [('json_to_models.dynamic_typing', cls_name)], cls_name def __iter__(self): - return () + return iter(()) T_StringSerializable = Type[StringSerializable] diff --git a/json_to_models/models/dataclasses.py b/json_to_models/models/dataclasses.py index ff7f524..8700880 100644 --- a/json_to_models/models/dataclasses.py +++ b/json_to_models/models/dataclasses.py @@ -1,8 +1,10 @@ +import logging from inspect import isclass from typing import Callable, List, Tuple from .base import GenericModelCodeGenerator, KWAGRS_TEMPLATE, METADATA_FIELD_NAME, sort_kwargs, template -from ..dynamic_typing import DDict, DList, DOptional, ImportPathList, MetaData, ModelMeta, StringSerializable +from ..dynamic_typing import (BaseType, DDict, DList, DOptional, DUnion, ImportPathList, MetaData, ModelMeta, + StringSerializable) DEFAULT_ORDER = ( ("default", "default_factory"), @@ -31,11 +33,23 @@ def __post_init__(self): return __post_init__ -def convert_strings(str_fields: List[str]) -> Callable[[type], type]: +def convert_strings(str_field_paths: List[str]) -> Callable[[type], type]: """ Decorator factory. Set up `__post_init__` method to convert strings fields values into StringSerializable types - :param str_fields: names of StringSerializable fields + If field contains complex data type path should be consist of field name and dotted list of tokens: + + * `S` - string component + * `U` - Union or Optional + * `L` - List + * `D` - Dict + + So if field `'bar'` has type `Optional[List[List[IntString]]]` field path would be `'bar#U.L.L.S'` + + ! If type is too complex i.e. Union[List[IntString], List[List[IntString]]] + you can't specify field path and such field would be ignored + + :param str_field_paths: Paths of StringSerializable fields (field name or field name + typing path) :return: Class decorator """ @@ -44,12 +58,12 @@ def decorator(cls: type) -> type: old_fn = cls.__post_init__ def __post_init__(self, *args, **kwargs): - dataclass_post_init_converters(str_fields)(self) + dataclass_post_init_converters(str_field_paths)(self) old_fn(self, *args, **kwargs) setattr(cls, '__post_init__', __post_init__) else: - setattr(cls, '__post_init__', dataclass_post_init_converters(str_fields)) + setattr(cls, '__post_init__', dataclass_post_init_converters(str_field_paths)) return cls @@ -75,14 +89,58 @@ def __init__(self, model: ModelMeta, meta=False, post_init_converters=False, dat self.no_meta = not meta self.dataclass_kwargs = dataclass_kwargs or {} + def get_string_field_paths(self) -> List[str]: + # `S` - string component + # `U` - Union or Optional + # `L` - List + # `D` - Dict + str_fields = [] + for name, t in self.model.type.items(): + name = self.convert_field_name(name) + + # Walk through nested types + paths: List[List[str]] = [] + tokens: List[Tuple[MetaData, List[str]]] = [(t, ['#'])] + while tokens: + tmp_type, path = tokens.pop() + if isclass(tmp_type): + if issubclass(tmp_type, StringSerializable): + paths.append(path + ['S']) + elif isinstance(tmp_type, BaseType): + cls = type(tmp_type) + if cls in (DUnion, DOptional): + token = 'U' + elif cls is DList: + token = 'L' + elif cls is DDict: + token = 'D' + else: + raise TypeError(f"Unsupported meta-type for converter path {cls}") + + for nested_type in tmp_type: + tokens.append((nested_type, path + [token])) + paths: List[str] = ["".join(p[1:]) for p in paths] + if not paths: + continue + if len(paths) > 1: + logging.warning(f"Could generate (string->StringSerializable) converter for type {t} (field {name})") + continue + + path = paths.pop() + if path == 'S': + str_fields.append(name) + else: + str_fields.append(f'{name}#{".".join(path)}') + + return str_fields + @property def decorators(self) -> Tuple[ImportPathList, List[str]]: imports = [('dataclasses', ['dataclass', 'field'])] decorators = [self.DC_DECORATOR.render(kwargs=self.dataclass_kwargs)] if self.post_init_converters: - str_fields = [self.convert_field_name(name) for name, t in self.model.type.items() - if isclass(t) and issubclass(t, StringSerializable)] + str_fields = self.get_string_field_paths() if str_fields: imports.append(('json_to_models.models.dataclasses', ['convert_strings'])) decorators.append(self.DC_CONVERT_DECORATOR.render(str_fields=str_fields)) diff --git a/test/test_code_generation/test_dataclasses_generation.py b/test/test_code_generation/test_dataclasses_generation.py index 94a1054..9a7cda2 100644 --- a/test/test_code_generation/test_dataclasses_generation.py +++ b/test/test_code_generation/test_dataclasses_generation.py @@ -2,10 +2,12 @@ import pytest -from json_to_models.dynamic_typing import (DDict, DList, DOptional, FloatString, IntString, ModelMeta, compile_imports) +from json_to_models.dynamic_typing import (DDict, DList, DOptional, DUnion, FloatString, IntString, ModelMeta, + compile_imports) from json_to_models.models import sort_fields from json_to_models.models.base import METADATA_FIELD_NAME, generate_code -from json_to_models.models.dataclasses import DataclassModelCodeGenerator +from json_to_models.models.dataclasses import (DataclassModelCodeGenerator, convert_strings, + dataclass_post_init_converters) from test.test_code_generation.test_models_code_generator import model_factory, trim @@ -97,13 +99,15 @@ class Test: "type": "Dict[str, int]" } }, - "generated": trim(f""" + "generated": trim(""" from dataclasses import dataclass, field from json_to_models.dynamic_typing import FloatString, IntString + from json_to_models.models.dataclasses import convert_strings from typing import Dict, List, Optional @dataclass + @convert_strings(['bar#U.S', 'qwerty']) class Test: foo: int qwerty: FloatString @@ -112,6 +116,33 @@ class Test: bar: Optional[IntString] = None asdfg: Optional[int] = None """) + }, + "converters": { + "model": ("Test", { + "a": int, + "b": IntString, + "c": DOptional(FloatString), + "d": DList(DList(DList(IntString))), + "e": DDict(IntString), + "u": DUnion(DDict(IntString), DList(DList(IntString))), + }), + "generated": trim(""" + from dataclasses import dataclass, field + from json_to_models.dynamic_typing import FloatString, IntString + from json_to_models.models.dataclasses import convert_strings + from typing import Dict, List, Optional, Union + + + @dataclass + @convert_strings(['b', 'c#U.S', 'd#L.L.L.S', 'e#D.S']) + class Test: + a: int + b: IntString + d: List[List[List[IntString]]] + e: Dict[str, IntString] + u: Union[Dict[str, IntString], List[List[IntString]]] + c: Optional[FloatString] = None + """) } } @@ -152,6 +183,49 @@ def test_fields_dc(value: ModelMeta, expected: dict): @pytest.mark.parametrize("value,expected", test_data_unzip["generated"]) def test_generated_dc(value: ModelMeta, expected: str): - generated = generate_code(([{"model": value, "nested": []}], {}), DataclassModelCodeGenerator, - class_generator_kwargs={'meta': True}) + generated = generate_code( + ([{"model": value, "nested": []}], {}), + DataclassModelCodeGenerator, + class_generator_kwargs={'meta': True, 'post_init_converters': True} + ) assert generated.rstrip() == expected, generated + + +def test_dataclass_post_init_converters(): + from dataclasses import dataclass + + @dataclass + class A: + x: IntString + y: FloatString + + __post_init__ = dataclass_post_init_converters(['x', 'y']) + + a = A('1', '1.1') + assert type(a.x) is IntString + assert type(a.y) is FloatString + + +def test_convert_strings_decorator(): + from dataclasses import dataclass + + @dataclass + @convert_strings(['x', 'y']) + class A: + x: IntString + y: FloatString + + @dataclass + @convert_strings(['x', 'y']) + class B: + x: IntString + y: FloatString + + def __post_init__(self): + self.x *= 2 + + a = A('1', '1.1') + b = B('1', '1.1') + assert type(a.x) is IntString + assert type(a.y) is FloatString + assert b.x == 2 From 230d4cd23892e47fecb5bf061791c09941889909 Mon Sep 17 00:00:00 2001 From: bogdandm Date: Sat, 20 Apr 2019 21:32:01 +0300 Subject: [PATCH 05/12] Disable converters generation of Union fields; Implement complex types conversion at convert_strings decorator; --- TODO.md | 3 +- json_to_models/models/dataclasses.py | 77 +++++++++++++++---- .../test_dataclasses_generation.py | 25 +++++- 3 files changed, 86 insertions(+), 19 deletions(-) diff --git a/TODO.md b/TODO.md index 8f17ba2..45216fd 100644 --- a/TODO.md +++ b/TODO.md @@ -12,9 +12,10 @@ - [X] typing code generation - [ ] (Maybe in future) Extract to another module (by serializers for each dynamic typing class) - [X] attrs + - complex StringSerializable converters (based on dataclass post_init converter) - [X] dataclasses - [ ] post_init converters for StringSerializable types - - [ ] Nested converters + - [X] Nested converters - [ ] Cli argument - [ ] generate from_json/to_json converters - [ ] Model class -> Meta format converter diff --git a/json_to_models/models/dataclasses.py b/json_to_models/models/dataclasses.py index 8700880..3b1fdc7 100644 --- a/json_to_models/models/dataclasses.py +++ b/json_to_models/models/dataclasses.py @@ -1,9 +1,8 @@ -import logging from inspect import isclass -from typing import Callable, List, Tuple +from typing import Any, Callable, List, Tuple from .base import GenericModelCodeGenerator, KWAGRS_TEMPLATE, METADATA_FIELD_NAME, sort_kwargs, template -from ..dynamic_typing import (BaseType, DDict, DList, DOptional, DUnion, ImportPathList, MetaData, ModelMeta, +from ..dynamic_typing import (BaseType, DDict, DList, DOptional, DUnion, ImportPathList, MetaData, ModelMeta, ModelPtr, StringSerializable) DEFAULT_ORDER = ( @@ -13,6 +12,39 @@ ) +def _process_string_field_value(path: List[str], value: Any, current_type: Any, optional=False) -> Any: + token, *path = path + if token == 'S': + try: + value = current_type.to_internal_value(value) + except ValueError as e: + if not optional: + raise e + finally: + return value + elif token == 'O': + return _process_string_field_value( + path=path, + value=value, + current_type=current_type.__args__[0], + optional=True + ) + elif token == 'L': + t = current_type.__args__[0] + return [ + _process_string_field_value(path, item, current_type=t, optional=optional) + for item in value + ] + elif token == 'D': + t = current_type.__args__[1] + return { + key: _process_string_field_value(path, item, current_type=t, optional=optional) + for key, item in value.items() + } + else: + raise ValueError(f"Unknown token {token}") + + def dataclass_post_init_converters(str_fields: List[str]): """ Method factory. Return post_init method to convert string into StringSerializable types @@ -26,9 +58,23 @@ def dataclass_post_init_converters(str_fields: List[str]): """ def __post_init__(self): - for name in (str_fields): - t = self.__annotations__[name] - setattr(self, name, t.to_internal_value(getattr(self, name))) + # `S` - string component + # `O` - Optional + # `L` - List + # `D` - Dict + for name in str_fields: + if '#' in name: + name, path_str = name.split('#') + path: List[str] = path_str.split('.') + else: + path = ['S'] + + new_value = _process_string_field_value( + path=path, + value=getattr(self, name), + current_type=self.__annotations__[name] + ) + setattr(self, name, new_value) return __post_init__ @@ -40,11 +86,11 @@ def convert_strings(str_field_paths: List[str]) -> Callable[[type], type]: If field contains complex data type path should be consist of field name and dotted list of tokens: * `S` - string component - * `U` - Union or Optional + * `O` - Optional * `L` - List * `D` - Dict - So if field `'bar'` has type `Optional[List[List[IntString]]]` field path would be `'bar#U.L.L.S'` + So if field `'bar'` has type `Optional[List[List[IntString]]]` field path would be `'bar#O.L.L.S'` ! If type is too complex i.e. Union[List[IntString], List[List[IntString]]] you can't specify field path and such field would be ignored @@ -91,7 +137,7 @@ def __init__(self, model: ModelMeta, meta=False, post_init_converters=False, dat def get_string_field_paths(self) -> List[str]: # `S` - string component - # `U` - Union or Optional + # `O` - Optional # `L` - List # `D` - Dict str_fields = [] @@ -108,22 +154,23 @@ def get_string_field_paths(self) -> List[str]: paths.append(path + ['S']) elif isinstance(tmp_type, BaseType): cls = type(tmp_type) - if cls in (DUnion, DOptional): - token = 'U' + if cls is DOptional: + token = 'O' elif cls is DList: token = 'L' elif cls is DDict: token = 'D' + elif cls in (DUnion, ModelPtr): + # We could not resolve Union + paths = [] + break else: raise TypeError(f"Unsupported meta-type for converter path {cls}") for nested_type in tmp_type: tokens.append((nested_type, path + [token])) paths: List[str] = ["".join(p[1:]) for p in paths] - if not paths: - continue - if len(paths) > 1: - logging.warning(f"Could generate (string->StringSerializable) converter for type {t} (field {name})") + if len(paths) != 1: continue path = paths.pop() diff --git a/test/test_code_generation/test_dataclasses_generation.py b/test/test_code_generation/test_dataclasses_generation.py index 9a7cda2..d5de716 100644 --- a/test/test_code_generation/test_dataclasses_generation.py +++ b/test/test_code_generation/test_dataclasses_generation.py @@ -1,4 +1,4 @@ -from typing import Dict, List +from typing import Dict, List, Optional import pytest @@ -107,7 +107,7 @@ class Test: @dataclass - @convert_strings(['bar#U.S', 'qwerty']) + @convert_strings(['bar#O.S', 'qwerty']) class Test: foo: int qwerty: FloatString @@ -134,7 +134,7 @@ class Test: @dataclass - @convert_strings(['b', 'c#U.S', 'd#L.L.L.S', 'e#D.S']) + @convert_strings(['b', 'c#O.S', 'd#L.L.L.S', 'e#D.S']) class Test: a: int b: IntString @@ -229,3 +229,22 @@ def __post_init__(self): assert type(a.x) is IntString assert type(a.y) is FloatString assert b.x == 2 + + +def test_convert_complex_data(): + from dataclasses import dataclass + + @dataclass + @convert_strings(['x', 'y#L.S', 'z#D.S', 'a#O.S', 'b#O.L.D.L.S']) + class A: + x: IntString + y: List[IntString] + z: Dict[str, IntString] + a: Optional[IntString] + b: Optional[List[Dict[str, List[IntString]]]] + + a = A('1', '1234', {'s': '2', 'w': '3'}, None, + [{'a': ['1', '2']}, {'b': ['3', '2']}]) + + assert a == A(1, [1, 2, 3, 4], {'s': 2, 'w': 3}, None, + [{'a': [1, 2]}, {'b': [3, 2]}]) From a4a0a0fe8279b2fc789469c11976f08790c22dc1 Mon Sep 17 00:00:00 2001 From: bogdandm Date: Sat, 20 Apr 2019 21:41:18 +0300 Subject: [PATCH 06/12] Fix minor warning --- json_to_models/models/dataclasses.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/json_to_models/models/dataclasses.py b/json_to_models/models/dataclasses.py index 3b1fdc7..55ba23c 100644 --- a/json_to_models/models/dataclasses.py +++ b/json_to_models/models/dataclasses.py @@ -20,8 +20,7 @@ def _process_string_field_value(path: List[str], value: Any, current_type: Any, except ValueError as e: if not optional: raise e - finally: - return value + return value elif token == 'O': return _process_string_field_value( path=path, From 5f083089b3faf9847c069b5584ddbbcdaa35b314 Mon Sep 17 00:00:00 2001 From: bogdandm Date: Wed, 24 Apr 2019 15:10:15 +0300 Subject: [PATCH 07/12] Fix optional type conversion --- json_to_models/models/dataclasses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/json_to_models/models/dataclasses.py b/json_to_models/models/dataclasses.py index 55ba23c..456f042 100644 --- a/json_to_models/models/dataclasses.py +++ b/json_to_models/models/dataclasses.py @@ -17,7 +17,7 @@ def _process_string_field_value(path: List[str], value: Any, current_type: Any, if token == 'S': try: value = current_type.to_internal_value(value) - except ValueError as e: + except (ValueError, TypeError) as e: if not optional: raise e return value From 94e49e5df61adf061a1f123d5a0b6ded1f8ca37c Mon Sep 17 00:00:00 2001 From: bogdandm Date: Wed, 24 Apr 2019 16:15:07 +0300 Subject: [PATCH 08/12] Global `models` module refactoring --- json_to_models/cli.py | 3 +- json_to_models/models/__init__.py | 226 +----------------- json_to_models/models/base.py | 64 +++-- json_to_models/models/dataclasses.py | 183 ++------------ json_to_models/models/string_converters.py | 183 ++++++++++++++ json_to_models/models/structure.py | 154 ++++++++++++ json_to_models/models/utils.py | 73 ++++++ .../test_attrs_generation.py | 2 +- .../test_dataclasses_generation.py | 66 +---- .../test_models_code_generator.py | 3 +- .../test_models_composition.py | 3 +- .../test_string_converters.py | 77 ++++++ testing_tools/real_apis/f1.py | 2 +- testing_tools/real_apis/large_data_set.py | 2 +- .../real_apis/large_data_set_github_online.py | 2 +- testing_tools/real_apis/pathofexile.py | 2 +- testing_tools/real_apis/randomapis.py | 2 +- 17 files changed, 574 insertions(+), 473 deletions(-) create mode 100644 json_to_models/models/string_converters.py create mode 100644 json_to_models/models/structure.py create mode 100644 json_to_models/models/utils.py create mode 100644 test/test_code_generation/test_string_converters.py diff --git a/json_to_models/cli.py b/json_to_models/cli.py index 201df53..aaf8e19 100644 --- a/json_to_models/cli.py +++ b/json_to_models/cli.py @@ -12,10 +12,11 @@ import json_to_models from json_to_models.dynamic_typing import ModelMeta, register_datetime_classes from json_to_models.generator import MetadataGenerator -from json_to_models.models import ModelsStructureType, compose_models, compose_models_flat +from json_to_models.models import ModelsStructureType from json_to_models.models.attr import AttrsModelCodeGenerator from json_to_models.models.base import GenericModelCodeGenerator, generate_code from json_to_models.models.dataclasses import DataclassModelCodeGenerator +from json_to_models.models.structure import compose_models, compose_models_flat from json_to_models.registry import ( ModelCmp, ModelFieldsEquals, ModelFieldsNumberMatch, ModelFieldsPercentMatch, ModelRegistry ) diff --git a/json_to_models/models/__init__.py b/json_to_models/models/__init__.py index be56da4..748542a 100644 --- a/json_to_models/models/__init__.py +++ b/json_to_models/models/__init__.py @@ -1,229 +1,15 @@ -from collections import defaultdict -from typing import Dict, Generic, Iterable, List, Set, Tuple, TypeVar, Union +from enum import Enum +from typing import Dict, List, Tuple -from ..dynamic_typing import DOptional, ModelMeta, ModelPtr +from ..dynamic_typing import ModelMeta Index = str ModelsStructureType = Tuple[List[dict], Dict[ModelMeta, ModelMeta]] -T = TypeVar('T') - - -class ListEx(list, Generic[T]): - """ - Extended list with shortcut methods - """ - - def safe_index(self, value: T): - try: - return self.index(value) - except ValueError: - return None - - def _safe_indexes(self, *values: T): - return [i for i in map(self.safe_index, values) if i is not None] - - def insert_before(self, value: T, *before: T): - ix = self._safe_indexes(*before) - if not ix: - raise ValueError - pos = min(ix) - self.insert(pos, value) - return pos - - def insert_after(self, value: T, *after: T): - ix = self._safe_indexes(*after) - if not ix: - raise ValueError - pos = max(ix) + 1 - self.insert(pos, value) - return pos - - -class PositionsDict(defaultdict): - # Dict contains mapping Index -> position, where position is list index to insert nested element of Index - INC = object() - - def __init__(self, default_factory=int, **kwargs): - super().__init__(default_factory, **kwargs) - - def update_position(self, key: str, value: Union[object, int]): - """ - Shift all elements which are placed after updated one - - :param key: Index or "root" - :param value: Could be position or PositionsDict.INC to perform quick increment (x+=1) - :return: - """ - if value is self.INC: - value = self[key] + 1 - if key in self: - old_value = self[key] - delta = value - old_value - else: - old_value = value - delta = 1 - for k, v in self.items(): - if k != key and v >= old_value: - self[k] += delta - self[key] = value - - -def compose_models(models_map: Dict[str, ModelMeta]) -> ModelsStructureType: - """ - Generate nested sorted models structure for internal usage. - - :return: List of root models data, Map(child model -> root model) for absolute ref generation - """ - root_models = ListEx() - root_nested_ix = 0 - structure_hash_table: Dict[Index, dict] = { - key: { - "model": model, - "nested": ListEx(), - "roots": list(extract_root(model)), # Indexes of root level models - } for key, model in models_map.items() - } - # TODO: Test path_injections - path_injections: Dict[ModelMeta, ModelMeta] = {} - - for key, model in models_map.items(): - pointers = list(filter_pointers(model)) - has_root_pointers = len(pointers) != len(model.pointers) - if not pointers: - # Root level model - if not has_root_pointers: - raise Exception(f'Model {model.name} has no pointers') - root_models.append(structure_hash_table[key]) - else: - parents = {ptr.parent.index for ptr in pointers} - struct = structure_hash_table[key] - # Model is using by other models - if has_root_pointers or len(parents) > 1 and len(struct["roots"]) > 1: - # Model is using by different root models - try: - root_models.insert_before( - struct, - *(structure_hash_table[parent_key] for parent_key in struct["roots"]) - ) - except ValueError: - root_models.insert(root_nested_ix, struct) - root_nested_ix += 1 - elif len(parents) > 1 and len(struct["roots"]) == 1: - # Model is using by single root model - parent = structure_hash_table[struct["roots"][0]] - parent["nested"].insert(0, struct) - path_injections[struct["model"]] = parent["model"] - else: - # Model is using by only one model - parent = structure_hash_table[next(iter(parents))] - struct = structure_hash_table[key] - parent["nested"].append(struct) - - return root_models, path_injections - - -def compose_models_flat(models_map: Dict[Index, ModelMeta]) -> ModelsStructureType: - """ - Generate flat sorted (by nesting level, ASC) models structure for internal usage. - - :param models_map: Mapping (model index -> model meta instance). - :return: List of root models data, Map(child model -> root model) for absolute ref generation - """ - root_models = ListEx() - positions: PositionsDict[Index, int] = PositionsDict() - top_level_models: Set[Index] = set() - structure_hash_table: Dict[Index, dict] = { - key: { - "model": model, - "nested": ListEx(), - "roots": list(extract_root(model)), # Indexes of root level models - } for key, model in models_map.items() - } - - for key, model in models_map.items(): - pointers = list(filter_pointers(model)) - has_root_pointers = len(pointers) != len(model.pointers) - if not pointers: - # Root level model - if not has_root_pointers: - raise Exception(f'Model {model.name} has no pointers') - root_models.insert(positions["root"], structure_hash_table[key]) - top_level_models.add(key) - positions.update_position("root", PositionsDict.INC) - else: - parents = {ptr.parent.index for ptr in pointers} - struct = structure_hash_table[key] - # Model is using by other models - if has_root_pointers or len(parents) > 1 and len(struct["roots"]) >= 1: - # Model is using by different root models - if parents & top_level_models: - parents.add("root") - parents_positions = {positions[parent_key] for parent_key in parents - if parent_key in positions} - parents_joined = "#".join(sorted(parents)) - if parents_joined in positions: - parents_positions.add(positions[parents_joined]) - pos = max(parents_positions) if parents_positions else len(root_models) - positions.update_position(parents_joined, pos + 1) - else: - # Model is using by only one model - parent = next(iter(parents)) - pos = positions.get(parent, len(root_models)) - positions.update_position(parent, pos + 1) - positions.update_position(key, pos + 1) - root_models.insert(pos, struct) - - return root_models, {} - - -def filter_pointers(model: ModelMeta) -> Iterable[ModelPtr]: - """ - Return iterator over pointers with not None parent - """ - return (ptr for ptr in model.pointers if ptr.parent) - - -def extract_root(model: ModelMeta) -> Set[Index]: - """ - Return set of indexes of root models that are use given ``model`` directly or through another nested model. - """ - seen: Set[Index] = set() - nodes: List[ModelPtr] = list(filter_pointers(model)) - roots: Set[Index] = set() - while nodes: - node = nodes.pop() - seen.add(node.type.index) - filtered = list(filter_pointers(node.parent)) - nodes.extend(ptr for ptr in filtered if ptr.type.index not in seen) - if not filtered: - roots.add(node.parent.index) - return roots - - -def sort_fields(model_meta: ModelMeta) -> Tuple[List[str], List[str]]: - """ - Split fields into required and optional groups - - :return: two list of fields names: required fields, optional fields - """ - fields = model_meta.type - required = [] - optional = [] - for key, meta in fields.items(): - if isinstance(meta, DOptional): - optional.append(key) - else: - required.append(key) - return required, optional - - INDENT = " " * 4 OBJECTS_DELIMITER = "\n" * 3 # 2 blank lines -def indent(string: str, lvl: int = 1, indent: str = INDENT) -> str: - """ - Indent all lines of string by ``indent * lvl`` - """ - return "\n".join(indent * lvl + line for line in string.split("\n")) +class ClassType(Enum): + Dataclass = "dataclass" + Attrs = "attrs" diff --git a/json_to_models/models/base.py b/json_to_models/models/base.py index 3dd82da..afeca35 100644 --- a/json_to_models/models/base.py +++ b/json_to_models/models/base.py @@ -6,8 +6,12 @@ from jinja2 import Template from unidecode import unidecode -from . import INDENT, ModelsStructureType, OBJECTS_DELIMITER, indent, sort_fields -from ..dynamic_typing import AbsoluteModelRef, ImportPathList, MetaData, ModelMeta, compile_imports, metadata_to_typing +from . import INDENT, ModelsStructureType, OBJECTS_DELIMITER +from .string_converters import get_string_field_paths +from .structure import sort_fields +from .utils import indent +from ..dynamic_typing import (AbsoluteModelRef, ImportPathList, MetaData, + ModelMeta, compile_imports, metadata_to_typing) from ..utils import cached_classmethod METADATA_FIELD_NAME = "J2M_ORIGINAL_FIELD" @@ -63,10 +67,24 @@ class {{ name }}: {%- endif -%} """) + STR_CONVERT_DECORATOR = template("convert_strings({{ str_fields }}{%% if kwargs %%}, %s{%% endif %%})" + % KWAGRS_TEMPLATE) FIELD: Template = template("{{name}}: {{type}}{% if body %} = {{ body }}{% endif %}") - def __init__(self, model: ModelMeta, **kwargs): + def __init__(self, model: ModelMeta, post_init_converters=False, **kwargs): self.model = model + self.post_init_converters = post_init_converters + + @cached_classmethod + def convert_field_name(cls, name): + if name in keywords_set: + name += "_" + name = unidecode(name) + name = re.sub(r"\W", "", name) + if not ('a' <= name[0].lower() <= 'z'): + if '0' <= name[0] <= '9': + name = ones[int(name[0])] + "_" + name[1:] + return inflection.underscore(name) def generate(self, nested_classes: List[str] = None, extra: str = "") -> Tuple[ImportPathList, str]: """ @@ -90,18 +108,17 @@ def decorators(self) -> Tuple[ImportPathList, List[str]]: """ :return: List of imports and List of decorators code (without @) """ - return [], [] - - @cached_classmethod - def convert_field_name(cls, name): - if name in keywords_set: - name += "_" - name = unidecode(name) - name = re.sub(r"\W", "", name) - if not ('a' <= name[0].lower() <= 'z'): - if '0' <= name[0] <= '9': - name = ones[int(name[0])] + "_" + name[1:] - return inflection.underscore(name) + imports, decorators = [], [] + if self.post_init_converters: + str_fields = self.string_field_paths + decorator_imports, decorator_kwargs = self.convert_strings_kwargs + if str_fields and decorator_kwargs: + imports.extend([ + *decorator_imports, + ('json_to_models.models.string_converters', ['convert_strings']), + ]) + decorators.append(self.STR_CONVERT_DECORATOR.render(str_fields=str_fields, kwargs=decorator_kwargs)) + return imports, decorators def field_data(self, name: str, meta: MetaData, optional: bool) -> Tuple[ImportPathList, dict]: """ @@ -137,6 +154,23 @@ def fields(self) -> Tuple[ImportPathList, List[str]]: strings.append(self.FIELD.render(**data)) return imports, strings + @property + def string_field_paths(self) -> List[str]: + """ + Get paths for convert_strings function + """ + return [self.convert_field_name(name) + ('#' + '.'.join(path) if path else '') + for name, path in get_string_field_paths(self.model)] + + @property + def convert_strings_kwargs(self) -> Tuple[ImportPathList, dict]: + """ + Override it to enable generation of string types converters + + :return: Imports and Dict with kw-arguments for `json_to_models.models.string_converters.convert_strings` decorator. + """ + return [], {} + def _generate_code( structure: List[dict], diff --git a/json_to_models/models/dataclasses.py b/json_to_models/models/dataclasses.py index 456f042..12ffcf2 100644 --- a/json_to_models/models/dataclasses.py +++ b/json_to_models/models/dataclasses.py @@ -1,9 +1,8 @@ from inspect import isclass -from typing import Any, Callable, List, Tuple +from typing import List, Tuple from .base import GenericModelCodeGenerator, KWAGRS_TEMPLATE, METADATA_FIELD_NAME, sort_kwargs, template -from ..dynamic_typing import (BaseType, DDict, DList, DOptional, DUnion, ImportPathList, MetaData, ModelMeta, ModelPtr, - StringSerializable) +from ..dynamic_typing import (DDict, DList, DOptional, ImportPathList, MetaData, ModelMeta, StringSerializable) DEFAULT_ORDER = ( ("default", "default_factory"), @@ -12,116 +11,11 @@ ) -def _process_string_field_value(path: List[str], value: Any, current_type: Any, optional=False) -> Any: - token, *path = path - if token == 'S': - try: - value = current_type.to_internal_value(value) - except (ValueError, TypeError) as e: - if not optional: - raise e - return value - elif token == 'O': - return _process_string_field_value( - path=path, - value=value, - current_type=current_type.__args__[0], - optional=True - ) - elif token == 'L': - t = current_type.__args__[0] - return [ - _process_string_field_value(path, item, current_type=t, optional=optional) - for item in value - ] - elif token == 'D': - t = current_type.__args__[1] - return { - key: _process_string_field_value(path, item, current_type=t, optional=optional) - for key, item in value.items() - } - else: - raise ValueError(f"Unknown token {token}") - - -def dataclass_post_init_converters(str_fields: List[str]): - """ - Method factory. Return post_init method to convert string into StringSerializable types - To override generated __post_init__ you can call it directly: - - >>> def __post_init__(self): - ... dataclass_post_init_converters(['a', 'b'])(self) - - :param str_fields: names of StringSerializable fields - :return: __post_init__ method - """ - - def __post_init__(self): - # `S` - string component - # `O` - Optional - # `L` - List - # `D` - Dict - for name in str_fields: - if '#' in name: - name, path_str = name.split('#') - path: List[str] = path_str.split('.') - else: - path = ['S'] - - new_value = _process_string_field_value( - path=path, - value=getattr(self, name), - current_type=self.__annotations__[name] - ) - setattr(self, name, new_value) - - return __post_init__ - - -def convert_strings(str_field_paths: List[str]) -> Callable[[type], type]: - """ - Decorator factory. Set up `__post_init__` method to convert strings fields values into StringSerializable types - - If field contains complex data type path should be consist of field name and dotted list of tokens: - - * `S` - string component - * `O` - Optional - * `L` - List - * `D` - Dict - - So if field `'bar'` has type `Optional[List[List[IntString]]]` field path would be `'bar#O.L.L.S'` - - ! If type is too complex i.e. Union[List[IntString], List[List[IntString]]] - you can't specify field path and such field would be ignored - - :param str_field_paths: Paths of StringSerializable fields (field name or field name + typing path) - :return: Class decorator - """ - - def decorator(cls: type) -> type: - if hasattr(cls, '__post_init__'): - old_fn = cls.__post_init__ - - def __post_init__(self, *args, **kwargs): - dataclass_post_init_converters(str_field_paths)(self) - old_fn(self, *args, **kwargs) - - setattr(cls, '__post_init__', __post_init__) - else: - setattr(cls, '__post_init__', dataclass_post_init_converters(str_field_paths)) - - return cls - - return decorator - - class DataclassModelCodeGenerator(GenericModelCodeGenerator): DC_DECORATOR = template(f"dataclass{{% if kwargs %}}({KWAGRS_TEMPLATE}){{% endif %}}") - DC_CONVERT_DECORATOR = template("convert_strings({{ str_fields }})") DC_FIELD = template(f"field({KWAGRS_TEMPLATE})") - def __init__(self, model: ModelMeta, meta=False, post_init_converters=False, dataclass_kwargs: dict = None, - **kwargs): + def __init__(self, model: ModelMeta, meta=False, post_init_converters=False, dataclass_kwargs: dict = None): """ :param model: ModelMeta instance :param meta: Enable generation of metadata as attrib argument @@ -129,68 +23,15 @@ def __init__(self, model: ModelMeta, meta=False, post_init_converters=False, dat :param dataclass_kwargs: kwargs for @dataclass() decorators :param kwargs: """ - super().__init__(model, **kwargs) - self.post_init_converters = post_init_converters + super().__init__(model, post_init_converters) self.no_meta = not meta self.dataclass_kwargs = dataclass_kwargs or {} - def get_string_field_paths(self) -> List[str]: - # `S` - string component - # `O` - Optional - # `L` - List - # `D` - Dict - str_fields = [] - for name, t in self.model.type.items(): - name = self.convert_field_name(name) - - # Walk through nested types - paths: List[List[str]] = [] - tokens: List[Tuple[MetaData, List[str]]] = [(t, ['#'])] - while tokens: - tmp_type, path = tokens.pop() - if isclass(tmp_type): - if issubclass(tmp_type, StringSerializable): - paths.append(path + ['S']) - elif isinstance(tmp_type, BaseType): - cls = type(tmp_type) - if cls is DOptional: - token = 'O' - elif cls is DList: - token = 'L' - elif cls is DDict: - token = 'D' - elif cls in (DUnion, ModelPtr): - # We could not resolve Union - paths = [] - break - else: - raise TypeError(f"Unsupported meta-type for converter path {cls}") - - for nested_type in tmp_type: - tokens.append((nested_type, path + [token])) - paths: List[str] = ["".join(p[1:]) for p in paths] - if len(paths) != 1: - continue - - path = paths.pop() - if path == 'S': - str_fields.append(name) - else: - str_fields.append(f'{name}#{".".join(path)}') - - return str_fields - @property def decorators(self) -> Tuple[ImportPathList, List[str]]: - imports = [('dataclasses', ['dataclass', 'field'])] - decorators = [self.DC_DECORATOR.render(kwargs=self.dataclass_kwargs)] - - if self.post_init_converters: - str_fields = self.get_string_field_paths() - if str_fields: - imports.append(('json_to_models.models.dataclasses', ['convert_strings'])) - decorators.append(self.DC_CONVERT_DECORATOR.render(str_fields=str_fields)) - + imports, decorators = super().decorators + imports.append(('dataclasses', ['dataclass', 'field'])) + decorators.insert(0, self.DC_DECORATOR.render(kwargs=self.dataclass_kwargs)) return imports, decorators def field_data(self, name: str, meta: MetaData, optional: bool) -> Tuple[ImportPathList, dict]: @@ -224,3 +65,13 @@ def field_data(self, name: str, meta: MetaData, optional: bool) -> Tuple[ImportP elif body_kwargs: data["body"] = self.DC_FIELD.render(kwargs=sort_kwargs(body_kwargs, DEFAULT_ORDER)) return imports, data + + @property + def convert_strings_kwargs(self) -> Tuple[ImportPathList, dict]: + """ + :return: Imports and Dict with kw-arguments for `json_to_models.models.string_converters.convert_strings` decorator. + """ + imports, kwargs = super().convert_strings_kwargs + imports.append(('json_to_models.models', ['ClassType'])) + kwargs["class_type"] = 'ClassType.Dataclass' + return imports, kwargs diff --git a/json_to_models/models/string_converters.py b/json_to_models/models/string_converters.py new file mode 100644 index 0000000..20e86c8 --- /dev/null +++ b/json_to_models/models/string_converters.py @@ -0,0 +1,183 @@ +from functools import wraps +from inspect import isclass +from typing import Any, Callable, List, Optional, Tuple + +from . import ClassType +from ..dynamic_typing import (BaseType, DDict, DList, DOptional, DUnion, MetaData, ModelMeta, ModelPtr, + StringSerializable) + + +def convert_strings(str_field_paths: List[str], class_type: Optional[ClassType] = None, + method: Optional[str] = None) -> Callable[[type], type]: + """ + Decorator factory. Set up post-init method to convert strings fields values into StringSerializable types + + If field contains complex data type path should be consist of field name and dotted list of tokens: + + * `S` - string component + * `O` - Optional + * `L` - List + * `D` - Dict + + So if field `'bar'` has type `Optional[List[List[IntString]]]` field path would be `'bar#O.L.L.S'` + + ! If type is too complex i.e. Union[List[IntString], List[List[IntString]]] + you can't specify field path and such field would be ignored + + To specify name of post-init method you should provide it by class_type argument or directly by method argument: + + >>> convert_strings([...], class_type=ClassType.Attrs) + + is equivalent of + + >>> convert_strings([...], method="__attrs_post_init__") + + :param str_field_paths: Paths of StringSerializable fields (field name or field name + typing path) + :param class_type: attrs | dataclass - type of decorated class + :param method: post-init method name + :return: Class decorator + """ + method = { + ClassType.Attrs: '__attrs_post_init__', + ClassType.Dataclass: '__post_init__', + None: method + }.get(class_type) + + def decorator(cls: type) -> type: + if hasattr(cls, method): + old_fn = getattr(cls, method) + + @wraps(old_fn) + def __post_init__(self, *args, **kwargs): + post_init_converters(str_field_paths)(self) + old_fn(self, *args, **kwargs) + + setattr(cls, method, __post_init__) + else: + fn = post_init_converters(str_field_paths) + fn.__name__ = method + setattr(cls, method, fn) + + return cls + + return decorator + + +def post_init_converters(str_fields: List[str], wrap_fn=None): + """ + Method factory. Return post_init method to convert string into StringSerializable types + To override generated __post_init__ you can call it directly: + + >>> def __post_init__(self): + ... post_init_converters(['a', 'b'])(self) + + :param str_fields: names of StringSerializable fields + :return: __post_init__ method + """ + + def __post_init__(self): + # `S` - string component + # `O` - Optional + # `L` - List + # `D` - Dict + for name in str_fields: + if '#' in name: + name, path_str = name.split('#') + path: List[str] = path_str.split('.') + else: + path = ['S'] + + new_value = _process_string_field_value( + path=path, + value=getattr(self, name), + current_type=self.__annotations__[name] + ) + setattr(self, name, new_value) + + if wrap_fn: + __post_init__ = wraps(wrap_fn)(__post_init__) + + return __post_init__ + + +def _process_string_field_value(path: List[str], value: Any, current_type: Any, optional=False) -> Any: + token, *path = path + if token == 'S': + try: + value = current_type.to_internal_value(value) + except (ValueError, TypeError) as e: + if not optional: + raise e + return value + elif token == 'O': + return _process_string_field_value( + path=path, + value=value, + current_type=current_type.__args__[0], + optional=True + ) + elif token == 'L': + t = current_type.__args__[0] + return [ + _process_string_field_value(path, item, current_type=t, optional=optional) + for item in value + ] + elif token == 'D': + t = current_type.__args__[1] + return { + key: _process_string_field_value(path, item, current_type=t, optional=optional) + for key, item in value.items() + } + else: + raise ValueError(f"Unknown token {token}") + + +def get_string_field_paths(model: ModelMeta) -> List[Tuple[str, List[str]]]: + """ + Return paths for convert_strings function of given model + + :return: Paths with raw names + """ + # `S` - string component + # `O` - Optional + # `L` - List + # `D` - Dict + str_fields: List[Tuple[str, List[str]]] = [] + for name, t in model.type.items(): + + # Walk through nested types + paths: List[List[str]] = [] + tokens: List[Tuple[MetaData, List[str]]] = [(t, ['#'])] + while tokens: + tmp_type, path = tokens.pop() + if isclass(tmp_type): + if issubclass(tmp_type, StringSerializable): + paths.append(path + ['S']) + elif isinstance(tmp_type, BaseType): + cls = type(tmp_type) + if cls is DOptional: + token = 'O' + elif cls is DList: + token = 'L' + elif cls is DDict: + token = 'D' + elif cls in (DUnion, ModelPtr): + # We could not resolve Union + paths = [] + break + else: + raise TypeError(f"Unsupported meta-type for converter path {cls}") + + for nested_type in tmp_type: + tokens.append((nested_type, path + [token])) + paths: List[str] = ["".join(p[1:]) for p in paths] + if len(paths) != 1: + continue + + path = paths.pop() + if path == 'S': + str_fields.append((name, [])) + else: + str_fields.append((name, path)) + + return str_fields diff --git a/json_to_models/models/structure.py b/json_to_models/models/structure.py new file mode 100644 index 0000000..cdddd69 --- /dev/null +++ b/json_to_models/models/structure.py @@ -0,0 +1,154 @@ +from typing import Dict, Iterable, List, Set, Tuple + +from . import Index, ModelsStructureType +from .utils import ListEx, PositionsDict +from ..dynamic_typing import DOptional, ModelMeta, ModelPtr + + +def compose_models(models_map: Dict[str, ModelMeta]) -> ModelsStructureType: + """ + Generate nested sorted models structure for internal usage. + + :return: List of root models data, Map(child model -> root model) for absolute ref generation + """ + root_models = ListEx() + root_nested_ix = 0 + structure_hash_table: Dict[Index, dict] = { + key: { + "model": model, + "nested": ListEx(), + "roots": list(extract_root(model)), # Indexes of root level models + } for key, model in models_map.items() + } + # TODO: Test path_injections + path_injections: Dict[ModelMeta, ModelMeta] = {} + + for key, model in models_map.items(): + pointers = list(filter_pointers(model)) + has_root_pointers = len(pointers) != len(model.pointers) + if not pointers: + # Root level model + if not has_root_pointers: + raise Exception(f'Model {model.name} has no pointers') + root_models.append(structure_hash_table[key]) + else: + parents = {ptr.parent.index for ptr in pointers} + struct = structure_hash_table[key] + # Model is using by other models + if has_root_pointers or len(parents) > 1 and len(struct["roots"]) > 1: + # Model is using by different root models + try: + root_models.insert_before( + struct, + *(structure_hash_table[parent_key] for parent_key in struct["roots"]) + ) + except ValueError: + root_models.insert(root_nested_ix, struct) + root_nested_ix += 1 + elif len(parents) > 1 and len(struct["roots"]) == 1: + # Model is using by single root model + parent = structure_hash_table[struct["roots"][0]] + parent["nested"].insert(0, struct) + path_injections[struct["model"]] = parent["model"] + else: + # Model is using by only one model + parent = structure_hash_table[next(iter(parents))] + struct = structure_hash_table[key] + parent["nested"].append(struct) + + return root_models, path_injections + + +def compose_models_flat(models_map: Dict[Index, ModelMeta]) -> ModelsStructureType: + """ + Generate flat sorted (by nesting level, ASC) models structure for internal usage. + + :param models_map: Mapping (model index -> model meta instance). + :return: List of root models data, Map(child model -> root model) for absolute ref generation + """ + root_models = ListEx() + positions: PositionsDict[Index, int] = PositionsDict() + top_level_models: Set[Index] = set() + structure_hash_table: Dict[Index, dict] = { + key: { + "model": model, + "nested": ListEx(), + "roots": list(extract_root(model)), # Indexes of root level models + } for key, model in models_map.items() + } + + for key, model in models_map.items(): + pointers = list(filter_pointers(model)) + has_root_pointers = len(pointers) != len(model.pointers) + if not pointers: + # Root level model + if not has_root_pointers: + raise Exception(f'Model {model.name} has no pointers') + root_models.insert(positions["root"], structure_hash_table[key]) + top_level_models.add(key) + positions.update_position("root", PositionsDict.INC) + else: + parents = {ptr.parent.index for ptr in pointers} + struct = structure_hash_table[key] + # Model is using by other models + if has_root_pointers or len(parents) > 1 and len(struct["roots"]) >= 1: + # Model is using by different root models + if parents & top_level_models: + parents.add("root") + parents_positions = {positions[parent_key] for parent_key in parents + if parent_key in positions} + parents_joined = "#".join(sorted(parents)) + if parents_joined in positions: + parents_positions.add(positions[parents_joined]) + pos = max(parents_positions) if parents_positions else len(root_models) + positions.update_position(parents_joined, pos + 1) + else: + # Model is using by only one model + parent = next(iter(parents)) + pos = positions.get(parent, len(root_models)) + positions.update_position(parent, pos + 1) + positions.update_position(key, pos + 1) + root_models.insert(pos, struct) + + return root_models, {} + + +def filter_pointers(model: ModelMeta) -> Iterable[ModelPtr]: + """ + Return iterator over pointers with not None parent + """ + return (ptr for ptr in model.pointers if ptr.parent) + + +def extract_root(model: ModelMeta) -> Set[Index]: + """ + Return set of indexes of root models that are use given `model` directly or through another nested model. + """ + seen: Set[Index] = set() + nodes: List[ModelPtr] = list(filter_pointers(model)) + roots: Set[Index] = set() + while nodes: + node = nodes.pop() + seen.add(node.type.index) + filtered = list(filter_pointers(node.parent)) + nodes.extend(ptr for ptr in filtered if ptr.type.index not in seen) + if not filtered: + roots.add(node.parent.index) + return roots + + +def sort_fields(model_meta: ModelMeta) -> Tuple[List[str], List[str]]: + """ + Split fields into required and optional groups + + :return: two list of fields names: required fields, optional fields + """ + fields = model_meta.type + required = [] + optional = [] + for key, meta in fields.items(): + if isinstance(meta, DOptional): + optional.append(key) + else: + required.append(key) + return required, optional diff --git a/json_to_models/models/utils.py b/json_to_models/models/utils.py new file mode 100644 index 0000000..c80bd37 --- /dev/null +++ b/json_to_models/models/utils.py @@ -0,0 +1,73 @@ +from collections import defaultdict +from typing import Generic, TypeVar, Union + +from . import INDENT + +T = TypeVar('T') + + +class ListEx(list, Generic[T]): + """ + Extended list with shortcut methods + """ + + def safe_index(self, value: T): + try: + return self.index(value) + except ValueError: + return None + + def _safe_indexes(self, *values: T): + return [i for i in map(self.safe_index, values) if i is not None] + + def insert_before(self, value: T, *before: T): + ix = self._safe_indexes(*before) + if not ix: + raise ValueError + pos = min(ix) + self.insert(pos, value) + return pos + + def insert_after(self, value: T, *after: T): + ix = self._safe_indexes(*after) + if not ix: + raise ValueError + pos = max(ix) + 1 + self.insert(pos, value) + return pos + + +class PositionsDict(defaultdict): + # Dict contains mapping Index -> position, where position is list index to insert nested element of Index + INC = object() + + def __init__(self, default_factory=int, **kwargs): + super().__init__(default_factory, **kwargs) + + def update_position(self, key: str, value: Union[object, int]): + """ + Shift all elements which are placed after updated one + + :param key: Index or "root" + :param value: Could be position or PositionsDict.INC to perform quick increment (x+=1) + :return: + """ + if value is self.INC: + value = self[key] + 1 + if key in self: + old_value = self[key] + delta = value - old_value + else: + old_value = value + delta = 1 + for k, v in self.items(): + if k != key and v >= old_value: + self[k] += delta + self[key] = value + + +def indent(string: str, lvl: int = 1, indent: str = INDENT) -> str: + """ + Indent all lines of string by ``indent * lvl`` + """ + return "\n".join(indent * lvl + line for line in string.split("\n")) diff --git a/test/test_code_generation/test_attrs_generation.py b/test/test_code_generation/test_attrs_generation.py index 5d641f4..f9beaa9 100644 --- a/test/test_code_generation/test_attrs_generation.py +++ b/test/test_code_generation/test_attrs_generation.py @@ -3,9 +3,9 @@ import pytest from json_to_models.dynamic_typing import (DDict, DList, DOptional, FloatString, IntString, ModelMeta, compile_imports) -from json_to_models.models import sort_fields from json_to_models.models.attr import AttrsModelCodeGenerator, DEFAULT_ORDER from json_to_models.models.base import METADATA_FIELD_NAME, generate_code, sort_kwargs +from json_to_models.models.structure import sort_fields from test.test_code_generation.test_models_code_generator import model_factory, trim diff --git a/test/test_code_generation/test_dataclasses_generation.py b/test/test_code_generation/test_dataclasses_generation.py index d5de716..78a1262 100644 --- a/test/test_code_generation/test_dataclasses_generation.py +++ b/test/test_code_generation/test_dataclasses_generation.py @@ -1,13 +1,12 @@ -from typing import Dict, List, Optional +from typing import Dict, List import pytest from json_to_models.dynamic_typing import (DDict, DList, DOptional, DUnion, FloatString, IntString, ModelMeta, compile_imports) -from json_to_models.models import sort_fields from json_to_models.models.base import METADATA_FIELD_NAME, generate_code -from json_to_models.models.dataclasses import (DataclassModelCodeGenerator, convert_strings, - dataclass_post_init_converters) +from json_to_models.models.dataclasses import (DataclassModelCodeGenerator) +from json_to_models.models.structure import sort_fields from test.test_code_generation.test_models_code_generator import model_factory, trim @@ -189,62 +188,3 @@ def test_generated_dc(value: ModelMeta, expected: str): class_generator_kwargs={'meta': True, 'post_init_converters': True} ) assert generated.rstrip() == expected, generated - - -def test_dataclass_post_init_converters(): - from dataclasses import dataclass - - @dataclass - class A: - x: IntString - y: FloatString - - __post_init__ = dataclass_post_init_converters(['x', 'y']) - - a = A('1', '1.1') - assert type(a.x) is IntString - assert type(a.y) is FloatString - - -def test_convert_strings_decorator(): - from dataclasses import dataclass - - @dataclass - @convert_strings(['x', 'y']) - class A: - x: IntString - y: FloatString - - @dataclass - @convert_strings(['x', 'y']) - class B: - x: IntString - y: FloatString - - def __post_init__(self): - self.x *= 2 - - a = A('1', '1.1') - b = B('1', '1.1') - assert type(a.x) is IntString - assert type(a.y) is FloatString - assert b.x == 2 - - -def test_convert_complex_data(): - from dataclasses import dataclass - - @dataclass - @convert_strings(['x', 'y#L.S', 'z#D.S', 'a#O.S', 'b#O.L.D.L.S']) - class A: - x: IntString - y: List[IntString] - z: Dict[str, IntString] - a: Optional[IntString] - b: Optional[List[Dict[str, List[IntString]]]] - - a = A('1', '1234', {'s': '2', 'w': '3'}, None, - [{'a': ['1', '2']}, {'b': ['3', '2']}]) - - assert a == A(1, [1, 2, 3, 4], {'s': 2, 'w': 3}, None, - [{'a': [1, 2]}, {'b': [3, 2]}]) diff --git a/test/test_code_generation/test_models_code_generator.py b/test/test_code_generation/test_models_code_generator.py index a2d5d1b..405e7fb 100644 --- a/test/test_code_generation/test_models_code_generator.py +++ b/test/test_code_generation/test_models_code_generator.py @@ -4,8 +4,9 @@ from json_to_models.dynamic_typing import (AbsoluteModelRef, DDict, DList, DOptional, IntString, ModelMeta, ModelPtr, Unknown, compile_imports) -from json_to_models.models import indent, sort_fields from json_to_models.models.base import GenericModelCodeGenerator, generate_code +from json_to_models.models.structure import sort_fields +from json_to_models.models.utils import indent # Data structure: # (string, indent lvl, indent string) diff --git a/test/test_code_generation/test_models_composition.py b/test/test_code_generation/test_models_composition.py index 76ddc12..a7b5ab4 100644 --- a/test/test_code_generation/test_models_composition.py +++ b/test/test_code_generation/test_models_composition.py @@ -4,7 +4,8 @@ from json_to_models.dynamic_typing import ModelMeta from json_to_models.generator import MetadataGenerator -from json_to_models.models import ListEx, compose_models, compose_models_flat, extract_root +from json_to_models.models.structure import compose_models, compose_models_flat, extract_root +from json_to_models.models.utils import ListEx from json_to_models.registry import ModelRegistry diff --git a/test/test_code_generation/test_string_converters.py b/test/test_code_generation/test_string_converters.py new file mode 100644 index 0000000..86a31d0 --- /dev/null +++ b/test/test_code_generation/test_string_converters.py @@ -0,0 +1,77 @@ +from typing import Dict, List, Optional + +import attr + +from json_to_models.dynamic_typing import FloatString, IntString +from json_to_models.models import ClassType +from json_to_models.models.string_converters import convert_strings, post_init_converters + + +def test_post_init_converters(): + from dataclasses import dataclass + + @dataclass + class A: + x: IntString + y: FloatString + + __post_init__ = post_init_converters(['x', 'y']) + + a = A('1', '1.1') + assert type(a.x) is IntString + assert type(a.y) is FloatString + + @attr.s + class A: + x: IntString = attr.ib() + y: FloatString = attr.ib() + + __attrs_post_init__ = post_init_converters(['x', 'y']) + + a = A('1', '1.1') + assert type(a.x) is IntString + assert type(a.y) is FloatString + + +def test_convert_strings_decorator(): + from dataclasses import dataclass + + @dataclass + @convert_strings(['x', 'y'], class_type=ClassType.Dataclass) + class A: + x: IntString + y: FloatString + + @dataclass + @convert_strings(['x', 'y'], class_type=ClassType.Dataclass) + class B: + x: IntString + y: FloatString + + def __post_init__(self): + self.x *= 2 + + a = A('1', '1.1') + b = B('1', '1.1') + assert type(a.x) is IntString + assert type(a.y) is FloatString + assert b.x == 2 + + +def test_convert_complex_data(): + from dataclasses import dataclass + + @dataclass + @convert_strings(['x', 'y#L.S', 'z#D.S', 'a#O.S', 'b#O.L.D.L.S'], class_type=ClassType.Dataclass) + class A: + x: IntString + y: List[IntString] + z: Dict[str, IntString] + a: Optional[IntString] + b: Optional[List[Dict[str, List[IntString]]]] + + a = A('1', '1234', {'s': '2', 'w': '3'}, None, + [{'a': ['1', '2']}, {'b': ['3', '2']}]) + + assert a == A(1, [1, 2, 3, 4], {'s': 2, 'w': 3}, None, + [{'a': [1, 2]}, {'b': [3, 2]}]) diff --git a/testing_tools/real_apis/f1.py b/testing_tools/real_apis/f1.py index e5056ac..e31e15d 100644 --- a/testing_tools/real_apis/f1.py +++ b/testing_tools/real_apis/f1.py @@ -7,9 +7,9 @@ from json_to_models.dynamic_typing import register_datetime_classes from json_to_models.generator import MetadataGenerator -from json_to_models.models import compose_models from json_to_models.models.base import generate_code from json_to_models.models.dataclasses import DataclassModelCodeGenerator +from json_to_models.models.structure import compose_models from json_to_models.registry import ModelRegistry from json_to_models.utils import json_format from testing_tools.pprint_meta_data import pretty_format_meta diff --git a/testing_tools/real_apis/large_data_set.py b/testing_tools/real_apis/large_data_set.py index 101cd1a..0336ad0 100644 --- a/testing_tools/real_apis/large_data_set.py +++ b/testing_tools/real_apis/large_data_set.py @@ -3,10 +3,10 @@ from pathlib import Path from json_to_models.generator import MetadataGenerator -from json_to_models.models import compose_models, compose_models_flat from json_to_models.models.attr import AttrsModelCodeGenerator from json_to_models.models.base import generate_code from json_to_models.models.dataclasses import DataclassModelCodeGenerator +from json_to_models.models.structure import compose_models, compose_models_flat from json_to_models.registry import ModelRegistry diff --git a/testing_tools/real_apis/large_data_set_github_online.py b/testing_tools/real_apis/large_data_set_github_online.py index db4ed39..e3307b3 100644 --- a/testing_tools/real_apis/large_data_set_github_online.py +++ b/testing_tools/real_apis/large_data_set_github_online.py @@ -9,7 +9,7 @@ except ImportError: tqdm = lambda x, **kwargs: x from json_to_models.generator import MetadataGenerator -from json_to_models.models import compose_models +from json_to_models.models.structure import compose_models from json_to_models.models.attr import AttrsModelCodeGenerator from json_to_models.models.base import generate_code from json_to_models.registry import ModelRegistry diff --git a/testing_tools/real_apis/pathofexile.py b/testing_tools/real_apis/pathofexile.py index ea6335c..b3b53be 100644 --- a/testing_tools/real_apis/pathofexile.py +++ b/testing_tools/real_apis/pathofexile.py @@ -6,9 +6,9 @@ import requests from json_to_models.generator import MetadataGenerator -from json_to_models.models import compose_models_flat from json_to_models.models.attr import AttrsModelCodeGenerator from json_to_models.models.base import generate_code +from json_to_models.models.structure import compose_models_flat from json_to_models.registry import ModelRegistry from json_to_models.utils import json_format from testing_tools.pprint_meta_data import pretty_format_meta diff --git a/testing_tools/real_apis/randomapis.py b/testing_tools/real_apis/randomapis.py index cf7d2b2..d942db8 100644 --- a/testing_tools/real_apis/randomapis.py +++ b/testing_tools/real_apis/randomapis.py @@ -7,9 +7,9 @@ import requests from json_to_models.generator import MetadataGenerator -from json_to_models.models import compose_models from json_to_models.models.attr import AttrsModelCodeGenerator from json_to_models.models.base import generate_code +from json_to_models.models.structure import compose_models from json_to_models.registry import ModelRegistry from testing_tools.real_apis import dump_response From b02197cf44e6a85cd2e58ee73be386598d5f8a72 Mon Sep 17 00:00:00 2001 From: bogdandm Date: Wed, 24 Apr 2019 16:17:14 +0300 Subject: [PATCH 09/12] Update tests --- .../test_dataclasses_generation.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test/test_code_generation/test_dataclasses_generation.py b/test/test_code_generation/test_dataclasses_generation.py index 78a1262..ddbdd59 100644 --- a/test/test_code_generation/test_dataclasses_generation.py +++ b/test/test_code_generation/test_dataclasses_generation.py @@ -101,12 +101,13 @@ class Test: "generated": trim(""" from dataclasses import dataclass, field from json_to_models.dynamic_typing import FloatString, IntString - from json_to_models.models.dataclasses import convert_strings + from json_to_models.models import ClassType + from json_to_models.models.string_converters import convert_strings from typing import Dict, List, Optional @dataclass - @convert_strings(['bar#O.S', 'qwerty']) + @convert_strings(['bar#O.S', 'qwerty'], class_type=ClassType.Dataclass) class Test: foo: int qwerty: FloatString @@ -128,12 +129,13 @@ class Test: "generated": trim(""" from dataclasses import dataclass, field from json_to_models.dynamic_typing import FloatString, IntString - from json_to_models.models.dataclasses import convert_strings + from json_to_models.models import ClassType + from json_to_models.models.string_converters import convert_strings from typing import Dict, List, Optional, Union @dataclass - @convert_strings(['b', 'c#O.S', 'd#L.L.L.S', 'e#D.S']) + @convert_strings(['b', 'c#O.S', 'd#L.L.L.S', 'e#D.S'], class_type=ClassType.Dataclass) class Test: a: int b: IntString From a46865d0a6413c9410ec2899858026903568655e Mon Sep 17 00:00:00 2001 From: bogdandm Date: Wed, 24 Apr 2019 16:29:45 +0300 Subject: [PATCH 10/12] Attrs converters --- json_to_models/models/attr.py | 19 +++++++-- .../test_attrs_generation.py | 41 ++++++++++++++++--- .../test_string_converters.py | 15 +++++++ 3 files changed, 67 insertions(+), 8 deletions(-) diff --git a/json_to_models/models/attr.py b/json_to_models/models/attr.py index a574577..7cb02ed 100644 --- a/json_to_models/models/attr.py +++ b/json_to_models/models/attr.py @@ -28,7 +28,10 @@ def __init__(self, model: ModelMeta, meta=False, attrs_kwargs: dict = None, **kw @property def decorators(self) -> Tuple[ImportPathList, List[str]]: - return [('attr', None)], [self.ATTRS.render(kwargs=self.attrs_kwargs)] + imports, decorators = super().decorators + imports.append(('attr', None)) + decorators.insert(0, self.ATTRS.render(kwargs=self.attrs_kwargs)) + return imports, decorators def field_data(self, name: str, meta: MetaData, optional: bool) -> Tuple[ImportPathList, dict]: """ @@ -49,13 +52,23 @@ def field_data(self, name: str, meta: MetaData, optional: bool) -> Tuple[ImportP body_kwargs["factory"] = "dict" else: body_kwargs["default"] = "None" - if isclass(meta.type) and issubclass(meta.type, StringSerializable): + if isclass(meta.type) and issubclass(meta.type, StringSerializable) and not self.post_init_converters: body_kwargs["converter"] = f"optional({meta.type.__name__})" imports.append(("attr.converter", "optional")) - elif isclass(meta) and issubclass(meta, StringSerializable): + elif isclass(meta) and issubclass(meta, StringSerializable) and not self.post_init_converters: body_kwargs["converter"] = meta.__name__ if not self.no_meta and name != data["name"]: body_kwargs["metadata"] = {METADATA_FIELD_NAME: name} data["body"] = self.ATTRIB.render(kwargs=sort_kwargs(body_kwargs, DEFAULT_ORDER)) return imports, data + + @property + def convert_strings_kwargs(self) -> Tuple[ImportPathList, dict]: + """ + :return: Imports and Dict with kw-arguments for `json_to_models.models.string_converters.convert_strings` decorator. + """ + imports, kwargs = super().convert_strings_kwargs + imports.append(('json_to_models.models', ['ClassType'])) + kwargs["class_type"] = 'ClassType.Attrs' + return imports, kwargs diff --git a/test/test_code_generation/test_attrs_generation.py b/test/test_code_generation/test_attrs_generation.py index f9beaa9..c807ab2 100644 --- a/test/test_code_generation/test_attrs_generation.py +++ b/test/test_code_generation/test_attrs_generation.py @@ -2,7 +2,8 @@ import pytest -from json_to_models.dynamic_typing import (DDict, DList, DOptional, FloatString, IntString, ModelMeta, compile_imports) +from json_to_models.dynamic_typing import (DDict, DList, DOptional, DUnion, FloatString, IntString, ModelMeta, + compile_imports) from json_to_models.models.attr import AttrsModelCodeGenerator, DEFAULT_ORDER from json_to_models.models.base import METADATA_FIELD_NAME, generate_code, sort_kwargs from json_to_models.models.structure import sort_fields @@ -142,23 +143,53 @@ class Test: }, "generated": trim(f""" import attr - from attr.converter import optional from json_to_models.dynamic_typing import FloatString, IntString + from json_to_models.models import ClassType + from json_to_models.models.string_converters import convert_strings from typing import Dict, List, Optional @attr.s + @convert_strings(['bar#O.S', 'qwerty'], class_type=ClassType.Attrs) class Test: foo: int = attr.ib() - qwerty: FloatString = attr.ib(converter=FloatString) + qwerty: FloatString = attr.ib() dict: Dict[str, int] = attr.ib() not_: bool = attr.ib({field_meta('not')}) one_day: int = attr.ib({field_meta('1day')}) den_nedeli: str = attr.ib({field_meta('день_недели')}) baz: Optional[List[List[str]]] = attr.ib(factory=list) - bar: Optional[IntString] = attr.ib(default=None, converter=optional(IntString)) + bar: Optional[IntString] = attr.ib(default=None) asdfg: Optional[int] = attr.ib(default=None) """) + }, + "converters": { + "model": ("Test", { + "a": int, + "b": IntString, + "c": DOptional(FloatString), + "d": DList(DList(DList(IntString))), + "e": DDict(IntString), + "u": DUnion(DDict(IntString), DList(DList(IntString))), + }), + "generated": trim(""" + import attr + from json_to_models.dynamic_typing import FloatString, IntString + from json_to_models.models import ClassType + from json_to_models.models.string_converters import convert_strings + from typing import Dict, List, Optional, Union + + + @attr.s + @convert_strings(['b', 'c#O.S', 'd#L.L.L.S', 'e#D.S'], class_type=ClassType.Attrs) + class Test: + a: int = attr.ib() + b: IntString = attr.ib() + d: List[List[List[IntString]]] = attr.ib() + e: Dict[str, IntString] = attr.ib() + u: Union[Dict[str, IntString], List[List[IntString]]] = attr.ib() + c: Optional[FloatString] = attr.ib(default=None) + """) } } @@ -200,5 +231,5 @@ def test_fields_attr(value: ModelMeta, expected: dict): @pytest.mark.parametrize("value,expected", test_data_unzip["generated"]) def test_generated_attr(value: ModelMeta, expected: str): generated = generate_code(([{"model": value, "nested": []}], {}), AttrsModelCodeGenerator, - class_generator_kwargs={'meta': True}) + class_generator_kwargs={'meta': True, 'post_init_converters': True}) assert generated.rstrip() == expected, generated diff --git a/test/test_code_generation/test_string_converters.py b/test/test_code_generation/test_string_converters.py index 86a31d0..8700601 100644 --- a/test/test_code_generation/test_string_converters.py +++ b/test/test_code_generation/test_string_converters.py @@ -75,3 +75,18 @@ class A: assert a == A(1, [1, 2, 3, 4], {'s': 2, 'w': 3}, None, [{'a': [1, 2]}, {'b': [3, 2]}]) + + @attr.s + @convert_strings(['x', 'y#L.S', 'z#D.S', 'a#O.S', 'b#O.L.D.L.S'], class_type=ClassType.Attrs) + class A: + x: IntString = attr.ib() + y: List[IntString] = attr.ib() + z: Dict[str, IntString] = attr.ib() + a: Optional[IntString] = attr.ib(default=None) + b: Optional[List[Dict[str, List[IntString]]]] = attr.ib(default=None) + + a = A('1', '1234', {'s': '2', 'w': '3'}, None, + [{'a': ['1', '2']}, {'b': ['3', '2']}]) + + assert a == A(1, [1, 2, 3, 4], {'s': 2, 'w': 3}, None, + [{'a': [1, 2]}, {'b': [3, 2]}]) From 883b380b64513580b7ad7f63c79453af13fa43e1 Mon Sep 17 00:00:00 2001 From: bogdandm Date: Wed, 24 Apr 2019 16:35:53 +0300 Subject: [PATCH 11/12] Reorder args in help --- json_to_models/cli.py | 50 +++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/json_to_models/cli.py b/json_to_models/cli.py index aaf8e19..25f6230 100644 --- a/json_to_models/cli.py +++ b/json_to_models/cli.py @@ -214,6 +214,21 @@ def _create_argparser(cls) -> argparse.ArgumentParser: "I.e. for file that contains dict {\"a\": {\"b\": [model_data, ...]}} you should\n" "pass 'a.b' as .\n\n" ) + parser.add_argument( + "-f", "--framework", + default="base", + choices=list(cls.MODEL_GENERATOR_MAPPING.keys()) + ["custom"], + help="Model framework for which python code is generated.\n" + "'base' (default) mean no framework so code will be generated without any decorators\n" + "and additional meta-data.\n" + "If you pass 'custom' you should specify --code-generator argument\n\n" + ) + parser.add_argument( + "-s", "--structure", + default="nested", + choices=list(cls.STRUCTURE_FN_MAPPING.keys()), + help="Models composition style. By default nested models become nested Python classes.\n\n" + ) parser.add_argument( "--datetime", action="store_true", @@ -221,20 +236,6 @@ def _create_argparser(cls) -> argparse.ArgumentParser: "Warn.: This can lead to 6-7 times slowdown on large datasets.\n" " Be sure that you really need this option.\n\n" ) - parser.add_argument( - "--dict-keys-regex", "--dkr", - nargs="+", metavar="RegEx", - help="List of regular expressions (Python syntax).\n" - "If all keys of some dict are match one of them\n" - "then this dict will be marked as dict field but not nested model.\n" - "Note: ^ and $ tokens will be added automatically but you have to\n" - "escape other special characters manually.\n" - ) - parser.add_argument( - "--dict-keys-fields", "--dkf", - nargs="+", metavar="FIELD NAME", - help="List of model fields names that will be marked as dict fields\n\n" - ) default_percent = f"{ModelFieldsPercentMatch.DEFAULT * 100:.0f}" default_number = f"{ModelFieldsNumberMatch.DEFAULT:.0f}" @@ -253,19 +254,18 @@ def _create_argparser(cls) -> argparse.ArgumentParser: "'exact' - two models should have exact same field names to merge.\n\n" ) parser.add_argument( - "-s", "--structure", - default="nested", - choices=list(cls.STRUCTURE_FN_MAPPING.keys()), - help="Models composition style. By default nested models become nested Python classes.\n\n" + "--dict-keys-regex", "--dkr", + nargs="+", metavar="RegEx", + help="List of regular expressions (Python syntax).\n" + "If all keys of some dict are match one of them\n" + "then this dict will be marked as dict field but not nested model.\n" + "Note: ^ and $ tokens will be added automatically but you have to\n" + "escape other special characters manually.\n" ) parser.add_argument( - "-f", "--framework", - default="base", - choices=list(cls.MODEL_GENERATOR_MAPPING.keys()) + ["custom"], - help="Model framework for which python code is generated.\n" - "'base' (default) mean no framework so code will be generated without any decorators\n" - "and additional meta-data.\n" - "If you pass 'custom' you should specify --code-generator argument\n\n" + "--dict-keys-fields", "--dkf", + nargs="+", metavar="FIELD NAME", + help="List of model fields names that will be marked as dict fields\n\n" ) parser.add_argument( "--code-generator", From aca90a86c982b65c01f8364c2866ce7fb734922e Mon Sep 17 00:00:00 2001 From: bogdandm Date: Wed, 24 Apr 2019 17:03:33 +0300 Subject: [PATCH 12/12] Add cli argument for string converters; Tests and fixes; --- TODO.md | 7 +++---- json_to_models/cli.py | 13 ++++++++++--- json_to_models/models/attr.py | 5 +++-- json_to_models/models/base.py | 2 +- json_to_models/models/string_converters.py | 3 +++ test/test_cli/test_script.py | 5 +++++ 6 files changed, 25 insertions(+), 10 deletions(-) diff --git a/TODO.md b/TODO.md index 45216fd..202f2f7 100644 --- a/TODO.md +++ b/TODO.md @@ -13,10 +13,9 @@ - [ ] (Maybe in future) Extract to another module (by serializers for each dynamic typing class) - [X] attrs - complex StringSerializable converters (based on dataclass post_init converter) + - [x] post_init converters for StringSerializable types - [X] dataclasses - - [ ] post_init converters for StringSerializable types - - [X] Nested converters - - [ ] Cli argument + - [x] post_init converters for StringSerializable types - [ ] generate from_json/to_json converters - [ ] Model class -> Meta format converter - [ ] attrs @@ -26,7 +25,7 @@ - [ ] dataclasses - [ ] Decorator to mark class as exclude from models merge - Other features - - [ ] Nesting models generation + - [X] Nesting models generation - [X] Cascade (default) - [X] Flat - [ ] OptionalFieldsPolicy diff --git a/json_to_models/cli.py b/json_to_models/cli.py index 25f6230..8144e90 100644 --- a/json_to_models/cli.py +++ b/json_to_models/cli.py @@ -45,9 +45,10 @@ class Cli: } def __init__(self): - self.initialize = False + self.initialized = False self.models_data: Dict[str, Iterable[dict]] = {} # -m/-l self.enable_datetime: bool = False # --datetime + self.strings_converters: bool = False # --strings-converters self.merge_policy: List[ModelCmp] = [] # --merge self.structure_fn: STRUCTURE_FN_TYPE = None # -s self.model_generator: Type[GenericModelCodeGenerator] = None # -f & --code-generator @@ -75,6 +76,7 @@ def parse_args(self, args: List[str] = None): for model_name, lookup, path in namespace.list or () ] self.enable_datetime = namespace.datetime + self.strings_converters = namespace.strings_converters merge_policy = [m.split("_") if "_" in m else m for m in namespace.merge] structure = namespace.structure framework = namespace.framework @@ -172,7 +174,7 @@ def set_args(self, merge_policy: List[Union[List[str], str]], m = importlib.import_module(module) self.model_generator = getattr(m, cls) - self.model_generator_kwargs = {} + self.model_generator_kwargs = {} if not self.strings_converters else {'post_init_converters': True} if code_generator_kwargs_raw: for item in code_generator_kwargs_raw: if item[0] == '"': @@ -185,7 +187,7 @@ def set_args(self, merge_policy: List[Union[List[str], str]], self.dict_keys_regex = [re.compile(rf"^{r}$") for r in dict_keys_regex] if dict_keys_regex else () self.dict_keys_fields = dict_keys_fields or () - self.initialize = True + self.initialized = True @classmethod def _create_argparser(cls) -> argparse.ArgumentParser: @@ -236,6 +238,11 @@ def _create_argparser(cls) -> argparse.ArgumentParser: "Warn.: This can lead to 6-7 times slowdown on large datasets.\n" " Be sure that you really need this option.\n\n" ) + parser.add_argument( + "--strings-converters", + action="store_true", + help="Enable generation of string types converters (i.e. IsoDatetimeString or BooleanString).\n\n" + ) default_percent = f"{ModelFieldsPercentMatch.DEFAULT * 100:.0f}" default_number = f"{ModelFieldsNumberMatch.DEFAULT:.0f}" diff --git a/json_to_models/models/attr.py b/json_to_models/models/attr.py index 7cb02ed..75bc84b 100644 --- a/json_to_models/models/attr.py +++ b/json_to_models/models/attr.py @@ -15,14 +15,15 @@ class AttrsModelCodeGenerator(GenericModelCodeGenerator): ATTRS = template(f"attr.s{{% if kwargs %}}({KWAGRS_TEMPLATE}){{% endif %}}") ATTRIB = template(f"attr.ib({KWAGRS_TEMPLATE})") - def __init__(self, model: ModelMeta, meta=False, attrs_kwargs: dict = None, **kwargs): + def __init__(self, model: ModelMeta, meta=False, post_init_converters=False, attrs_kwargs: dict = None): """ :param model: ModelMeta instance :param meta: Enable generation of metadata as attrib argument + :param post_init_converters: Enable generation of type converters in __post_init__ methods :param attrs_kwargs: kwargs for @attr.s() decorators :param kwargs: """ - super().__init__(model, **kwargs) + super().__init__(model, post_init_converters) self.no_meta = not meta self.attrs_kwargs = attrs_kwargs or {} diff --git a/json_to_models/models/base.py b/json_to_models/models/base.py index afeca35..7b983a0 100644 --- a/json_to_models/models/base.py +++ b/json_to_models/models/base.py @@ -71,7 +71,7 @@ class {{ name }}: % KWAGRS_TEMPLATE) FIELD: Template = template("{{name}}: {{type}}{% if body %} = {{ body }}{% endif %}") - def __init__(self, model: ModelMeta, post_init_converters=False, **kwargs): + def __init__(self, model: ModelMeta, post_init_converters=False): self.model = model self.post_init_converters = post_init_converters diff --git a/json_to_models/models/string_converters.py b/json_to_models/models/string_converters.py index 20e86c8..04326b7 100644 --- a/json_to_models/models/string_converters.py +++ b/json_to_models/models/string_converters.py @@ -5,6 +5,7 @@ from . import ClassType from ..dynamic_typing import (BaseType, DDict, DList, DOptional, DUnion, MetaData, ModelMeta, ModelPtr, StringSerializable) +from ..dynamic_typing.base import NoneType def convert_strings(str_field_paths: List[str], class_type: Optional[ClassType] = None, @@ -165,6 +166,8 @@ def get_string_field_paths(model: ModelMeta) -> List[Tuple[str, List[str]]]: # We could not resolve Union paths = [] break + elif cls is NoneType: + continue else: raise TypeError(f"Unsupported meta-type for converter path {cls}") diff --git a/test/test_cli/test_script.py b/test/test_cli/test_script.py index f674b13..87e7e0b 100644 --- a/test/test_cli/test_script.py +++ b/test/test_cli/test_script.py @@ -70,6 +70,11 @@ def test_help(): id="gists_merge_policy"), pytest.param(f"""{executable} -m Gist "{tmp_path / '*.gist'}" --dkf files --merge exact""", id="gists_no_merge"), + pytest.param(f"""{executable} -m Gist "{tmp_path / '*.gist'}" --dkf files --datetime --strings-converters""", + id="gists_strings_converters"), + + pytest.param(f"""{executable} -l User - "{test_data_path / 'users.json'}" --strings-converters""", + id="users_strings_converters"), ]