Coverage for src/loman/serialization/profile.py: 100%

1"""Serialization profiles: how much of a saved computation is readable text. 

2 

3A profile and a container are independent choices. The profile decides how a 

4*value* is encoded --- inline JSON numbers, or bytes written out of line. The 

5container decides where those bytes *land* --- one text document, a zip, or a 

6directory. Collapsing the two into a single setting looks tidier and immediately 

7fails on real cases: a readable manifest inside a zip is genuinely useful, and 

8``dumps()`` returning a string can only ever be readable-plus-single-document. 

9 

10Only one combination is impossible: the efficient profile has nowhere to put 

11blobs in a single JSON document, and base64-inlining them would inflate by a 

12third and force a read-all. That case raises and points at ``container="zip"``. 

13""" 

14 

15from __future__ import annotations 

16 

17import fnmatch 

18from typing import Any 

19 

20import attrs 

21 

22from .compression import DEFAULT_COMPRESSION 

23 

24# Below this, a separate container member --- its own entry, filename and seek 

25# --- costs more than the JSON it saves, and keeping small values inline is what 

26# preserves "open the manifest and read it". 8 KiB is about a 1024-element 

27# float64 array. 

28DEFAULT_INLINE_MAX_BYTES = 8 * 1024 

29 

30 

31@attrs.frozen 

32class SerializationProfile: 

33 """How values are encoded for one save. 

34 

35 :ivar name: Identifier recorded in the manifest. 

36 :ivar inline_max_bytes: Values estimated at or below this many bytes stay 

37 inline. ``None`` keeps everything inline, whatever the container. 

38 :ivar overrides: Selector-to-settings map, letting one save treat some nodes 

39 differently. A selector is a node-key glob (``"market_data/**"``) or a 

40 tag (``"tag:raw"``). 

41 """ 

42 

43 name: str 

44 inline_max_bytes: int | None = None 

45 #: ``"none"``, or a codec and optional level such as ``"zstd:1"`` or 

46 #: ``"zlib:6"``. Named, never inferred: a saved file should compress the way 

47 #: you asked, not the way something guessed. Whichever of the compressed and 

48 #: raw payloads is smaller is what gets stored. 

49 compression: str = "none" 

50 dedupe: str = "identity" 

51 checksums: bool = False 

52 frame_encoding: str = "npy" 

53 overrides: dict[str, dict[str, Any]] = attrs.field(factory=dict) 

54 

55 def wants_blob(self, nbytes: int | None) -> bool: 

56 """Return whether a value of *nbytes* should be written out of line. 

57 

58 A transformer that cannot estimate its size passes ``None`` and is taken 

59 at its word that the value is worth storing out of line. 

60 """ 

61 if self.inline_max_bytes is None: 

62 return False 

63 if nbytes is None: 

64 return True 

65 return nbytes > self.inline_max_bytes 

66 

67 def settings_for(self, node: str | None, tags: frozenset[str] = frozenset()) -> dict[str, Any]: 

68 """Return the override settings that apply to *node*. 

69 

70 Later matches win, so a more specific selector listed after a general one 

71 takes precedence --- the order the overrides were written in. 

72 """ 

73 settings: dict[str, Any] = {} 

74 for selector, values in self.overrides.items(): 

75 if _selector_matches(selector, node, tags): 

76 settings.update(values) 

77 return settings 

78 

79 

80def _selector_matches(selector: str, node: str | None, tags: frozenset[str]) -> bool: 

81 """Return whether *selector* applies to a node with *tags*.""" 

82 if selector.startswith("tag:"): 

83 return selector[4:] in tags 

84 if node is None: 

85 return False 

86 # fnmatch treats "*" as matching separators too, so "a/**" and "a/*" both 

87 # match nested keys. Node keys are shallow in practice; exactness here would 

88 # mean a path-glob implementation for no observed benefit. 

89 return fnmatch.fnmatchcase(node, selector) 

90 

91 

92#: Everything inline: the file is JSON you can open and read end to end. 

93READABLE = SerializationProfile(name="readable", inline_max_bytes=None) 

94 

95#: Large values out of line as binary blobs; small ones stay inline, so the 

96#: manifest still describes every value's shape without decoding anything. 

97#: Compressed with zstd at level 1, which on measured data gives around 9x on a 

98#: realistic price series and rejects incompressible data at about 1 GB/s. A 

99#: blob that does not shrink is stored raw, so the cost of the attempt is 

100#: bounded and nothing is ever stored larger than it started. 

101EFFICIENT = SerializationProfile( 

102 name="efficient", 

103 inline_max_bytes=DEFAULT_INLINE_MAX_BYTES, 

104 compression=DEFAULT_COMPRESSION, 

105) 

106 

107_BY_NAME = {p.name: p for p in (READABLE, EFFICIENT)} 

108 

109 

110def resolve_profile(profile: str | SerializationProfile | None) -> SerializationProfile: 

111 """Return a :class:`SerializationProfile` from a name, an instance or ``None``.""" 

112 if profile is None: 

113 return EFFICIENT 

114 if isinstance(profile, SerializationProfile): 

115 return profile 

116 try: 

117 return _BY_NAME[profile] 

118 except KeyError: 

119 msg = f"Unknown profile {profile!r}; expected one of {sorted(_BY_NAME)} or a SerializationProfile" 

120 raise ValueError(msg) from None