cengal.data_manipulation.remote_objects.versions.v_0_fast.remote_objects
1#!/usr/bin/env python 2# coding=utf-8 3 4# Copyright © 2012-2024 ButenkoMS. All rights reserved. Contacts: <gtalk@butenkoms.space> 5# 6# Licensed under the Apache License, Version 2.0 (the "License"); 7# you may not use this file except in compliance with the License. 8# You may obtain a copy of the License at 9# 10# http://www.apache.org/licenses/LICENSE-2.0 11# 12# Unless required by applicable law or agreed to in writing, software 13# distributed under the License is distributed on an "AS IS" BASIS, 14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15# See the License for the specific language governing permissions and 16# limitations under the License. 17 18 19__all__ = [ 20 'default_serializable_data_types', 21 'known_types', 22 'known_data_types', 23 'known_container_types', 24 'DataType', 25 'data_type', 26 'data_type_by_type', 27 'ClassInfoFields', 28 'ObjectInfoFields', 29 'CanNotAdjustToSerializableError', 30 'CanNotAdjustFromSerializableError', 31 'RemoteObjectsManager', 32] 33 34 35from cengal.introspection.inspect import ( 36 entity_module_importable_str_and_owning_names_path, 37 entity_by_name_module_importable_str_and_owning_names_path, 38 filled_slot_names_with_values_gen, 39 is_callable, is_async, 40 is_setable_data_descriptor, 41) 42from cengal.code_flow_control.smart_values import ResultExistence 43from cengal.code_flow_control.gc import DisableGC 44from cengal.data_generation.id_generator import IDGenerator, GeneratorType 45from enum import IntEnum 46from collections.abc import MutableMapping, MutableSequence, MutableSet 47from struct import pack, unpack 48from inspect import getattr_static 49from typing import Any, Dict, Optional, Callable, Set, Type, Tuple, List, FrozenSet 50 51 52""" 53Module Docstring 54Docstrings: http://www.python.org/dev/peps/pep-0257/ 55""" 56 57__author__ = "ButenkoMS <gtalk@butenkoms.space>" 58__copyright__ = "Copyright © 2012-2024 ButenkoMS. All rights reserved. Contacts: <gtalk@butenkoms.space>" 59__credits__ = ["ButenkoMS <gtalk@butenkoms.space>", ] 60__license__ = "Apache License, Version 2.0" 61__version__ = "4.4.1" 62__maintainer__ = "ButenkoMS <gtalk@butenkoms.space>" 63__email__ = "gtalk@butenkoms.space" 64# __status__ = "Prototype" 65__status__ = "Development" 66# __status__ = "Production" 67 68 69default_serializable_data_types: Set[Type] = { 70 int, float, complex, str, bytes, bytearray, bool, type(None), list, tuple, set, frozenset, dict 71} 72 73 74known_types: Set[Type] = { 75 int, float, complex, str, bytes, bytearray, bool, type(None), list, tuple, set, frozenset, dict, 76 slice, 77} 78 79 80known_data_types: Set[Type] = { 81 int, float, str, bytes, bytearray, bool, type(None), slice, 82} 83 84 85known_container_types: Set[Type] = { 86 complex, list, tuple, set, frozenset, dict, 87} 88 89 90class DataType(IntEnum): 91 class_ = 0 # type is not in known_types and not in serializable_data_types 92 int_ = 1 93 float_ = 2 94 complex_ = 3 95 str_ = 4 96 bytes_ = 5 97 bytearray_ = 6 98 bool_ = 7 99 none_ = 8 100 list_ = 9 101 tuple_ = 10 102 set_ = 11 103 frozenset_ = 12 104 dict_ = 13 105 slice_ = 14 106 unknown_serializable = 15 # type is in serializable_data_types but not in known_types 107 108 109data_type: Dict[DataType, Type] = { 110 DataType.class_: object, 111 DataType.int_: int, 112 DataType.float_: float, 113 DataType.complex_: complex, 114 DataType.str_: str, 115 DataType.bytes_: bytes, 116 DataType.bytearray_: bytearray, 117 DataType.bool_: bool, 118 DataType.none_: type(None), 119 DataType.list_: list, 120 DataType.tuple_: tuple, 121 DataType.set_: set, 122 DataType.frozenset_: frozenset, 123 DataType.dict_: dict, 124 DataType.slice_: slice, 125 DataType.unknown_serializable: None, 126} 127 128 129data_type_by_type: Dict[Type, DataType] = { 130 object: DataType.class_, 131 int: DataType.int_, 132 float: DataType.float_, 133 complex: DataType.complex_, 134 str: DataType.str_, 135 bytes: DataType.bytes_, 136 bytearray: DataType.bytearray_, 137 bool: DataType.bool_, 138 type(None): DataType.none_, 139 list: DataType.list_, 140 tuple: DataType.tuple_, 141 set: DataType.set_, 142 frozenset: DataType.frozenset_, 143 dict: DataType.dict_, 144 slice: DataType.slice_, 145 None: DataType.unknown_serializable, 146} 147 148 149class ClassInfoFields(IntEnum): 150 class_id = 0 151 class_name = 1 152 module_importable_str = 2 153 owning_names_path = 3 154 155 156class ObjectInfoFields(IntEnum): 157 object_id = 0 # (type: int) 158 type_id = 1 # (type: DataType) 159 object_ = 2 # (Optional), (type: Any). Link to object itself if `(type(obj) in serializable_data_types)`. Not used otherwise. 160 class_id = 3 # (Optional), (type: int). Used if `type_id == DataType.class_` 161 clonable_slots = 4 # (Optional), (type: Tuple[Tuple[str, Any]]). Used if `type_id == DataType.class_`. Holds ID's (object_id) of slots objects. 162 clonable_dict_items = 5 # (Optional), (type: Tuple[Tuple[str, Any]]). Used if `type_id == DataType.class_`. Holds ID's (object_id) of value objects. 163 # contained_items = 6 # (Optional), (type: Union[Tuple, List, Set, FrozenSet, Dict]). Used if `type_id in {DataType.list_, DataType.tuple_, DataType.set_, DataType.frozenset_, DataType.dict_, DataType.slice_}`. Holds ID's (object_id) of contained items (for bothe keys and values in the case of Dict). 164 contained_mapping = 6 # (Optional), (type: Union[Tuple, List, Set, FrozenSet, Dict]). Used if `type_id in {DataType.list_, DataType.tuple_, DataType.set_, DataType.frozenset_, DataType.dict_, DataType.slice_}`. Holds ID's (object_id) of contained items (for bothe keys and values in the case of Dict). 165 contained_sequence = 7 # (Optional), (type: Union[Tuple, List, Set, FrozenSet, Dict]). Used if `type_id in {DataType.list_, DataType.tuple_, DataType.set_, DataType.frozenset_, DataType.dict_, DataType.slice_}`. Holds ID's (object_id) of contained items (for bothe keys and values in the case of Dict). 166 contained_set = 8 # (Optional), (type: Union[Tuple, List, Set, FrozenSet, Dict]). Used if `type_id in {DataType.list_, DataType.tuple_, DataType.set_, DataType.frozenset_, DataType.dict_, DataType.slice_}`. Holds ID's (object_id) of contained items (for bothe keys and values in the case of Dict). 167 168 169class CanNotAdjustToSerializableError(Exception): 170 pass 171 172 173class CanNotAdjustFromSerializableError(Exception): 174 pass 175 176 177class RemoteObjectsManager: 178 def __init__(self, 179 on_new_class_handler: Optional[Callable] = None, 180 on_new_obj_info_handler: Optional[Callable] = None, 181 # classess_db: Optional[Dict[int, Type]] = None, 182 objects_db: Optional[Dict[int, Any]] = None, 183 classes_id_gen: Optional[Callable] = None, 184 objects_id_gen: Optional[Callable] = None, 185 serializable_data_types: Optional[Set[Type]] = None, 186 ) -> None: 187 # self.classess_db: Dict[int, Type] = dict() if classess_db is None else classess_db 188 self.classes_id_gen: IDGenerator = IDGenerator() if classes_id_gen is None else classes_id_gen 189 self.objects_db: Dict[int, Any] = dict() if objects_db is None else objects_db 190 self.objects_id_gen: IDGenerator = IDGenerator() if objects_id_gen is None else objects_id_gen 191 self.serializable_data_types: Set[Type] = default_serializable_data_types if serializable_data_types is None else serializable_data_types 192 self.serializable_any: bool = not self.serializable_data_types 193 self.serializable_int: bool = (int in self.serializable_data_types) or self.serializable_any 194 self.serializable_float: bool = (float in self.serializable_data_types) or self.serializable_any 195 self.serializable_complex: bool = (complex in self.serializable_data_types) or self.serializable_any 196 self.serializable_str: bool = (str in self.serializable_data_types) or self.serializable_any 197 self.serializable_bytes: bool = (bytes in self.serializable_data_types) or self.serializable_any 198 self.serializable_bytearray: bool = (bytearray in self.serializable_data_types) or self.serializable_any 199 self.serializable_bool: bool = (bool in self.serializable_data_types) or self.serializable_any 200 self.serializable_none: bool = (type(None) in self.serializable_data_types) or self.serializable_any 201 self.serializable_list: bool = (list in self.serializable_data_types) or self.serializable_any 202 self.serializable_tuple: bool = (tuple in self.serializable_data_types) or self.serializable_any 203 self.serializable_set: bool = (set in self.serializable_data_types) or self.serializable_any 204 self.serializable_frozenset: bool = (frozenset in self.serializable_data_types) or self.serializable_any 205 self.serializable_dict: bool = (dict in self.serializable_data_types) or self.serializable_any 206 self.serializable_slice: bool = (slice in self.serializable_data_types) or self.serializable_any 207 self.known_classes: Dict[Type, int] = dict() 208 self.known_classes_by_id: Dict[int, Type] = dict() 209 self.known_classes_info: Dict[Tuple, int] = dict() 210 self.known_classes_info_by_id: Dict[int, Tuple] = dict() 211 self.on_new_class_handler: Optional[Callable] = on_new_class_handler 212 self.on_new_obj_info_handler: Optional[Callable] = on_new_obj_info_handler 213 self.objects_ids: Dict[int, int] = dict() # (Key: object_id, Value: id(obj)) 214 self.objects_ids_by_id: Dict[int, int] = dict() # (Key: id(obj), Value: object_id) 215 216 def del_object_by_id(self, id_: int) -> None: 217 if id_ in self.objects_ids_by_id: 218 object_id = self.objects_ids_by_id[id_] 219 self.objects_ids_by_id.pop(id_, None) 220 self.objects_ids.pop(object_id, None) 221 self.objects_db.pop(object_id, None) 222 223 dobi = del_object_by_id 224 225 def del_object_by_object_id(self, object_id: int) -> None: 226 if object_id in self.objects_ids: 227 id_ = self.objects_ids[object_id] 228 self.objects_ids_by_id.pop(id_, None) 229 self.objects_ids.pop(object_id, None) 230 self.objects_db.pop(object_id, None) 231 232 doboi = del_object_by_object_id 233 234 def adjust_to_serializable(self, type_id: DataType, obj: Any) -> Any: 235 if DataType.int_ == type_id: 236 return obj 237 elif DataType.float_ == type_id: 238 return obj 239 elif DataType.complex_ == type_id: 240 if self.serializable_complex: 241 return obj 242 elif self.serializable_bytes: 243 return pack('=dd', obj.real, obj.imag) 244 elif self.serializable_bytearray: 245 return bytearray(pack('=dd', obj.real, obj.imag)) 246 elif self.serializable_str: 247 return str(obj) 248 elif self.serializable_float: 249 if self.serializable_tuple: 250 return (obj.real, obj.imag) 251 elif self.serializable_list: 252 return [obj.real, obj.imag] 253 else: 254 pass 255 else: 256 pass 257 elif DataType.str_ == type_id: 258 return obj 259 elif DataType.bytes_ == type_id: 260 if self.serializable_bytes: 261 return obj 262 elif self.serializable_bytearray: 263 return bytearray(obj) 264 elif self.serializable_str: 265 return obj.hex() 266 elif self.serializable_tuple: 267 if self.serializable_int: 268 return tuple(int(c) for c in obj) 269 if self.serializable_float: 270 return tuple(float(int(c)) for c in obj) 271 else: 272 pass 273 elif self.serializable_list: 274 if self.serializable_int: 275 return [int(c) for c in obj] 276 if self.serializable_float: 277 return [float(int(c)) for c in obj] 278 else: 279 pass 280 else: 281 pass 282 elif DataType.bytearray_ == type_id: 283 if self.serializable_bytearray: 284 return obj 285 elif self.serializable_bytes: 286 return bytes(obj) 287 elif self.serializable_str: 288 return obj.hex() 289 elif self.serializable_tuple: 290 if self.serializable_int: 291 return tuple(int(c) for c in obj) 292 if self.serializable_float: 293 return tuple(float(int(c)) for c in obj) 294 else: 295 pass 296 elif self.serializable_list: 297 if self.serializable_int: 298 return [int(c) for c in obj] 299 if self.serializable_float: 300 return [float(int(c)) for c in obj] 301 else: 302 pass 303 else: 304 pass 305 elif DataType.bool_ == type_id: 306 return obj 307 elif DataType.none_ == type_id: 308 return obj 309 elif DataType.list_ == type_id: 310 if self.serializable_list: 311 return obj 312 elif self.serializable_tuple: 313 return tuple(obj) 314 elif self.serializable_dict: 315 return dict({index: item for index, item in enumerate(obj)}) 316 else: 317 pass 318 elif DataType.tuple_ == type_id: 319 if self.serializable_tuple: 320 return obj 321 elif self.serializable_list: 322 return list(obj) 323 elif self.serializable_dict: 324 return dict({index: item for index, item in enumerate(obj)}) 325 else: 326 pass 327 elif DataType.set_ == type_id: 328 if self.serializable_set: 329 return obj 330 elif self.serializable_frozenset: 331 return frozenset(obj) 332 elif self.serializable_tuple: 333 return tuple(obj) 334 elif self.serializable_list: 335 return list(obj) 336 elif self.serializable_dict: 337 return dict({k: None for k in obj}) 338 else: 339 pass 340 elif DataType.frozenset_ == type_id: 341 if self.serializable_frozenset: 342 return obj 343 elif self.serializable_set: 344 return set(obj) 345 elif self.serializable_tuple: 346 return tuple(obj) 347 elif self.serializable_list: 348 return list(obj) 349 elif self.serializable_dict: 350 return dict({k: None for k in obj}) 351 else: 352 pass 353 elif DataType.dict_ == type_id: 354 return obj 355 elif DataType.slice_ == type_id: 356 if self.serializable_slice: 357 return obj 358 elif self.serializable_tuple: 359 return (obj.start, obj.stop, obj.step) 360 elif self.serializable_list: 361 return [obj.start, obj.stop, obj.step] 362 elif self.serializable_dict: 363 return {0: obj.start, 1: obj.stop, 2: obj.step} 364 else: 365 pass 366 else: 367 raise RuntimeError('Unknown type_id') 368 369 raise CanNotAdjustToSerializableError(f'Can not adjust to serializable. Type: {type_id}, obj: {obj}') 370 371 ats = adjust_to_serializable 372 373 def adjust_from_serializable(self, type_id: DataType, obj: Any) -> Any: 374 if DataType.int_ == type_id: 375 return obj 376 elif DataType.float_ == type_id: 377 return obj 378 elif DataType.complex_ == type_id: 379 if self.serializable_complex: 380 return obj 381 elif self.serializable_bytes: 382 return complex(*unpack('=dd', obj)) 383 elif self.serializable_bytearray: 384 return complex(*unpack('=dd', bytes(obj))) 385 elif self.serializable_str: 386 return complex(*obj) 387 elif self.serializable_float: 388 if self.serializable_tuple: 389 return complex(*obj) 390 elif self.serializable_list: 391 return complex(*obj) 392 else: 393 pass 394 else: 395 pass 396 elif DataType.str_ == type_id: 397 return obj 398 elif DataType.bytes_ == type_id: 399 if self.serializable_bytes: 400 return obj 401 elif self.serializable_bytearray: 402 return bytes(obj) 403 elif self.serializable_str: 404 return bytes.fromhex(obj) 405 elif self.serializable_tuple: 406 if self.serializable_int: 407 return b''.join(item.to_bytes(1, 'little') for item in obj) 408 if self.serializable_float: 409 return b''.join((int(round(item))).to_bytes(1, 'little') for item in obj) 410 else: 411 pass 412 elif self.serializable_list: 413 if self.serializable_int: 414 return b''.join(item.to_bytes(1, 'little') for item in obj) 415 if self.serializable_float: 416 return b''.join((int(round(item))).to_bytes(1, 'little') for item in obj) 417 else: 418 pass 419 else: 420 pass 421 elif DataType.bytearray_ == type_id: 422 if self.serializable_bytearray: 423 return obj 424 elif self.serializable_bytes: 425 return bytearray(obj) 426 elif self.serializable_str: 427 return bytearray(bytes.fromhex(obj)) 428 elif self.serializable_tuple: 429 if self.serializable_int: 430 return bytearray(b''.join(item.to_bytes(1, 'little') for item in obj)) 431 if self.serializable_float: 432 return bytearray(b''.join((int(round(item))).to_bytes(1, 'little') for item in obj)) 433 else: 434 pass 435 elif self.serializable_list: 436 if self.serializable_int: 437 return bytearray(b''.join(item.to_bytes(1, 'little') for item in obj)) 438 if self.serializable_float: 439 return bytearray(b''.join((int(round(item))).to_bytes(1, 'little') for item in obj)) 440 else: 441 pass 442 else: 443 pass 444 elif DataType.bool_ == type_id: 445 return obj 446 elif DataType.none_ == type_id: 447 return None 448 elif DataType.list_ == type_id: 449 if self.serializable_list: 450 return obj 451 elif self.serializable_tuple: 452 return list(obj) 453 elif self.serializable_dict: 454 return [value for key, value in sorted(obj.items(), key=lambda x: x[0])] 455 else: 456 pass 457 elif DataType.tuple_ == type_id: 458 if self.serializable_tuple: 459 return obj 460 elif self.serializable_list: 461 return tuple(obj) 462 elif self.serializable_dict: 463 return tuple(value for key, value in sorted(obj.items(), key=lambda x: x[0])) 464 else: 465 pass 466 elif DataType.set_ == type_id: 467 if self.serializable_set: 468 return obj 469 elif self.serializable_frozenset: 470 return set(obj) 471 elif self.serializable_tuple: 472 return set(obj) 473 elif self.serializable_list: 474 return set(obj) 475 elif self.serializable_dict: 476 return set(obj.keys()) 477 else: 478 pass 479 elif DataType.frozenset_ == type_id: 480 if self.serializable_frozenset: 481 return obj 482 elif self.serializable_set: 483 return frozenset(obj) 484 elif self.serializable_tuple: 485 return frozenset(obj) 486 elif self.serializable_list: 487 return frozenset(obj) 488 elif self.serializable_dict: 489 return frozenset(obj.keys()) 490 else: 491 pass 492 elif DataType.dict_ == type_id: 493 return obj 494 elif DataType.slice_ == type_id: 495 if self.serializable_slice: 496 return obj 497 elif self.serializable_tuple: 498 return slice(*obj) 499 elif self.serializable_list: 500 return slice(*obj) 501 elif self.serializable_dict: 502 return slice(obj[0], obj[1], obj[2]) 503 else: 504 pass 505 else: 506 raise RuntimeError('Unknown type_id') 507 508 raise CanNotAdjustFromSerializableError(f'Can not adjust from serializable. Type: {type_id}, obj: {obj}') 509 510 afs = adjust_from_serializable 511 512 # def is_replicatable_object_attribute(self, attribute: Any) -> bool: 513 # if is_setable_data_descriptor(attribute): 514 # data = attribute.__get__(None, None) 515 # elif is_callable(attribute): 516 # return False 517 # else: 518 # return True 519 520 def serialize_container(self, type_id: DataType, obj: Any) -> Any: 521 if DataType.list_ == type_id: 522 new_obj = list() 523 for item in obj: 524 result: ResultExistence[int] = self.serialize_impl(item) 525 if result: 526 new_obj.append(result.value) 527 elif DataType.tuple_ == type_id: 528 new_obj = list() 529 for item in obj: 530 result: ResultExistence[int] = self.serialize_impl(item) 531 if result: 532 new_obj.append(result.value) 533 534 new_obj = tuple(new_obj) 535 elif DataType.set_ == type_id: 536 new_obj = set() 537 for item in obj: 538 result: ResultExistence[int] = self.serialize_impl(item) 539 if result: 540 new_obj.add(result.value) 541 elif DataType.frozenset_ == type_id: 542 new_obj = set() 543 for item in obj: 544 result: ResultExistence[int] = self.serialize_impl(item) 545 if result: 546 new_obj.add(result.value) 547 548 new_obj = frozenset(new_obj) 549 elif DataType.dict_ == type_id: 550 new_obj = dict() 551 for key, value in obj.items(): 552 key_result: ResultExistence[int] = self.serialize_impl(key) 553 value_result: ResultExistence[int] = self.serialize_impl(value) 554 if key_result and value_result: 555 new_obj[key_result.value] = value_result.value 556 557 return new_obj 558 559 sc = serialize_container 560 561 def serialize_impl(self, obj: Any, ignore_empty_classes: bool = False) -> ResultExistence[int]: 562 result_exists: bool = True 563 id_: int = id(obj) 564 obj_type = type(obj) 565 if (int != obj_type) and (id_ in self.objects_ids_by_id): 566 new_object: bool = False 567 object_id: int = self.objects_ids_by_id[id_] 568 else: 569 # int object must always produce new object_id because first 256 ints are persistent across Python sessions and 570 # this can cause issues within users of current module. For example within `cengal/hardware/memory/shared_memory` 571 # which changes int values inline instead of producing new objects. 572 new_object = True 573 object_id = self.objects_id_gen() 574 self.objects_ids[object_id] = id_ 575 self.objects_db[object_id] = obj 576 577 if not new_object: 578 return ResultExistence[int](result_exists, object_id) 579 580 if self.serializable_any or (obj_type in self.serializable_data_types): 581 serializable: bool = True 582 type_id: DataType = data_type_by_type.get(obj_type, DataType.unknown_serializable) 583 else: 584 serializable = False 585 type_id = data_type_by_type.get(obj_type, DataType.class_) 586 587 if obj_type in known_container_types: 588 known_container: bool = True 589 else: 590 known_container = False 591 592 class_info: Dict[int, Any] = None 593 known_data: bool = None 594 class_id: int = None 595 is_new_class: bool = None 596 class_name: str = None 597 new_obj_slots: List[Tuple[str, Any]] = None 598 new_obj_dict: Dict[str, Any] = None 599 obj_mapping: Dict = None 600 obj_sequence: List = None 601 obj_set: Set = None 602 if serializable: 603 if known_container: 604 object_info = ( 605 object_id, 606 type_id.value, 607 self.sc(type_id, obj), 608 ) 609 else: 610 object_info = ( 611 object_id, 612 type_id.value, 613 obj, 614 ) 615 else: 616 if obj_type in known_data_types: 617 known_data = True 618 else: 619 known_data = False 620 621 if DataType.class_ == type_id: 622 new_obj_slots = list() 623 for slot_name, slot_value in filled_slot_names_with_values_gen(obj): 624 adjusted_slot_name = slot_name 625 item_result: ResultExistence[int] = self.serialize_impl(slot_value) 626 if item_result: 627 if self.serializable_tuple: 628 new_obj_slots.append((adjusted_slot_name, item_result.value)) 629 elif self.serializable_list: 630 new_obj_slots.append([adjusted_slot_name, item_result.value]) 631 else: 632 new_obj_slots.append(self.ats(DataType.tuple_, (adjusted_slot_name, item_result.value))) 633 634 if new_obj_slots: 635 if self.serializable_list: 636 pass 637 elif self.serializable_tuple: 638 new_obj_slots = tuple(new_obj_slots) 639 else: 640 new_obj_slots = self.ats(DataType.list_, new_obj_slots) 641 642 new_obj_dict = dict() 643 if hasattr(obj, '__dict__'): 644 for key, value in obj.__dict__.items(): 645 # raw_value = getattr_static(obj, key) 646 # if hasattr(raw_value, '__get__') and (not hasattr(raw_value, '__set__')): 647 # # if not setable descriptor 648 # continue 649 650 adjusted_key = key 651 652 value_result: ResultExistence[int] = self.serialize_impl(value) 653 if value_result: 654 new_obj_dict[adjusted_key] = value_result.value 655 else: 656 continue 657 658 obj_mapping = dict() 659 obj_sequence = list() 660 obj_set = list() 661 if isinstance(obj, MutableMapping): 662 for key, value in obj.items(): 663 key_result: ResultExistence[int] = self.serialize_impl(value) 664 if not key_result: 665 continue 666 667 value_result: ResultExistence[int] = self.serialize_impl(value) 668 if value_result: 669 obj_mapping[key_result.value] = value_result.value 670 else: 671 continue 672 elif isinstance(obj, MutableSequence): 673 for item in obj: 674 item_result: ResultExistence[int] = self.serialize_impl(item) 675 if item_result: 676 obj_sequence.append(item_result.value) 677 else: 678 continue 679 680 if obj_sequence: 681 if self.serializable_list: 682 pass 683 else: 684 obj_sequence = self.ats(DataType.list_, obj_sequence) 685 elif isinstance(obj, MutableSet): 686 for item in obj: 687 item_result: ResultExistence[int] = self.serialize_impl(item) 688 if item_result: 689 obj_set.append(item_result.value) 690 else: 691 continue 692 693 if obj_set: 694 if self.serializable_set: 695 pass 696 else: 697 obj_set = self.ats(DataType.list_, obj_set) 698 else: 699 pass 700 701 if ignore_empty_classes: 702 result_exists = new_obj_slots or new_obj_dict or obj_mapping or obj_sequence or obj_set 703 704 if result_exists: 705 if obj_type in self.known_classes: 706 is_new_class = False 707 class_id = self.known_classes[obj_type] 708 else: 709 class_info = dict() 710 is_new_class = True 711 class_name = obj_type.__name__ 712 class_id = self.classes_id_gen() 713 self.known_classes[obj_type] = class_id 714 self.known_classes_by_id[class_id] = obj_type 715 module_importable_str, owning_names_path = entity_module_importable_str_and_owning_names_path(obj) 716 module_importable_str_and_owning_names_path = (class_name, module_importable_str, tuple(owning_names_path)) 717 self.known_classes_info[module_importable_str_and_owning_names_path] = class_id 718 self.known_classes_info_by_id[class_id] = module_importable_str_and_owning_names_path 719 720 class_info = ( 721 class_id, 722 class_name, 723 module_importable_str, 724 owning_names_path, 725 ) 726 727 if self.on_new_class_handler: 728 self.on_new_class_handler( 729 class_info, 730 obj_type, 731 class_id, 732 class_name, 733 module_importable_str, 734 owning_names_path, 735 module_importable_str_and_owning_names_path 736 ) 737 738 # will be later serialized much faster than without `if else None` 739 object_info = ( 740 object_id, 741 type_id.value, 742 0, 743 class_id, 744 new_obj_slots if new_obj_slots else 0, 745 new_obj_dict if new_obj_dict else 0, 746 obj_mapping if obj_mapping else 0, 747 obj_sequence if obj_sequence else 0, 748 obj_set if obj_set else 0, 749 ) 750 elif known_data: 751 object_info = ( 752 object_id, 753 type_id.value, 754 self.ats(type_id, obj), 755 ) 756 elif known_container: 757 object_info = ( 758 object_id, 759 type_id.value, 760 self.ats(type_id, self.sc(type_id, obj)), 761 ) 762 else: 763 raise RuntimeError('Unknown type_id') 764 765 if result_exists: 766 if self.on_new_obj_info_handler: 767 self.on_new_obj_info_handler( 768 object_info, 769 obj, 770 object_id, 771 type_id, 772 serializable, 773 known_container, 774 known_data, 775 class_id, 776 is_new_class, 777 new_obj_slots, 778 new_obj_dict, 779 obj_mapping, 780 obj_sequence, 781 obj_set, 782 ) 783 784 return ResultExistence[int](result_exists, object_id) 785 786 si = serialize_impl 787 788 def serialize(self, obj: Any, ignore_empty_classes: bool = False) -> ResultExistence[int]: 789 with DisableGC(): 790 return self.serialize_impl(obj, ignore_empty_classes) 791 792 s = serialize 793 794 def deserialize_class_impl(self, class_info: Tuple) -> ResultExistence[Tuple[int, Type]]: 795 class_id: int = class_info[ClassInfoFields.class_id.value] 796 if class_id in self.known_classes_by_id: 797 return ResultExistence[Tuple[int, Type]](True, (class_id, self.known_classes_by_id[class_id])) 798 else: 799 class_name: str = class_info[ClassInfoFields.class_name.value] 800 module_importable_str: str = class_info[ClassInfoFields.module_importable_str.value] 801 owning_names_path: List[str] = self.afs(DataType.list_, class_info[ClassInfoFields.owning_names_path.value]) 802 obj_class: Type = entity_by_name_module_importable_str_and_owning_names_path(class_name, module_importable_str, owning_names_path) 803 self.known_classes_by_id[class_id] = obj_class 804 self.known_classes[obj_class] = class_id 805 class_info_tuple: Tuple = (class_name, module_importable_str, tuple(owning_names_path)) 806 self.known_classes_info[class_info_tuple] = class_id 807 self.known_classes_info_by_id[class_id] = class_info_tuple 808 return ResultExistence[Tuple[int, Type]](True, (class_id, obj_class)) 809 810 dcli = deserialize_class_impl 811 812 def deserialize_class(self, class_info: Tuple) -> ResultExistence[Tuple[int, Type]]: 813 with DisableGC(): 814 return self.deserialize_class_impl(class_info) 815 816 dcl = deserialize_class 817 818 def deserialize_container_impl(self, type_id: DataType, obj: Any) -> Any: 819 new_obj = obj 820 if DataType.list_ == type_id: 821 new_obj = list() 822 for item_id in obj: 823 new_obj.append(self.objects_db[item_id]) 824 elif DataType.tuple_ == type_id: 825 new_obj = list() 826 for item_id in obj: 827 new_obj.append(self.objects_db[item_id]) 828 829 new_obj = tuple(new_obj) 830 elif DataType.set_ == type_id: 831 new_obj = set() 832 for item_id in obj: 833 new_obj.add(self.objects_db[item_id]) 834 elif DataType.frozenset_ == type_id: 835 new_obj = set() 836 for item_id in obj: 837 new_obj.add(self.objects_db[item_id]) 838 839 new_obj = frozenset(new_obj) 840 elif DataType.dict_ == type_id: 841 new_obj = dict() 842 for key_id, value_id in obj.items(): 843 new_obj[self.objects_db[int(key_id)]] = self.objects_db[value_id] 844 845 return new_obj 846 847 dcoi = deserialize_container_impl 848 849 def deserialize_container(self, type_id: DataType, obj: Any) -> Any: 850 with DisableGC(): 851 return self.deserialize_container_impl(type_id, obj) 852 853 dco = deserialize_container 854 855 def deserialize_obj_impl(self, obj_info: Tuple) -> ResultExistence[Tuple[int, Any]]: 856 object_id: int = obj_info[ObjectInfoFields.object_id.value] 857 if object_id in self.objects_db: 858 return ResultExistence[Tuple[int, Any]](True, (object_id, self.objects_db[object_id])) 859 860 type_id: DataType = DataType(obj_info[ObjectInfoFields.type_id.value]) 861 862 obj_type = data_type[type_id] 863 serializable: bool = self.serializable_any or (obj_type in self.serializable_data_types) 864 known_container: bool = obj_type in known_container_types 865 known_data: bool = None 866 867 if serializable: 868 if known_container: 869 obj = self.dcoi(type_id, obj_info[ObjectInfoFields.object_.value]) 870 else: 871 obj = obj_info[ObjectInfoFields.object_.value] 872 else: 873 known_data = obj_type in known_data_types 874 875 if DataType.class_ == type_id: 876 class_id: int = obj_info[ObjectInfoFields.class_id.value] 877 obj_class: Type = self.known_classes_by_id[class_id] 878 obj: Any = obj_class.__new__(obj_class) 879 880 if obj_info[ObjectInfoFields.clonable_slots.value]: 881 clonable_slots = self.afs(DataType.list_, obj_info[ObjectInfoFields.clonable_slots.value]) 882 for slot_name, slot_id in clonable_slots: 883 child_obj = self.objects_db[slot_id] 884 setattr(obj, slot_name, child_obj) 885 886 if obj_info[ObjectInfoFields.clonable_dict_items.value]: 887 clonable_dict_items = obj_info[ObjectInfoFields.clonable_dict_items.value] 888 for key, value_id in clonable_dict_items.items(): 889 child_obj = self.objects_db[value_id] 890 setattr(obj, key, child_obj) 891 892 if obj_info[ObjectInfoFields.contained_mapping.value]: 893 contained_mapping = obj_info[ObjectInfoFields.contained_mapping.value] 894 for key_id, value_id in contained_mapping.items(): 895 child_key = self.objects_db[int(key_id)] 896 child_value = self.objects_db[value_id] 897 obj[child_key] = child_value 898 899 if obj_info[ObjectInfoFields.contained_sequence.value]: 900 contained_sequence = self.afs(DataType.list_, obj_info[ObjectInfoFields.contained_sequence.value]) 901 for item_id in contained_sequence: 902 child_item = self.objects_db[item_id] 903 obj.append(child_item) 904 905 if obj_info[ObjectInfoFields.contained_set.value]: 906 contained_set = self.afs(DataType.list_, obj_info[ObjectInfoFields.contained_set.value]) 907 for item_id in contained_set: 908 child_item = self.objects_db[item_id] 909 obj.add(child_item) 910 elif known_data: 911 obj = self.afs(type_id, obj_info[ObjectInfoFields.object_.value]) 912 elif known_container: 913 obj = self.dcoi(type_id, self.afs(type_id, obj_info[ObjectInfoFields.object_.value])) 914 else: 915 raise RuntimeError('Unknown type_id') 916 917 self.objects_db[object_id] = obj 918 id_: int = id(obj) 919 self.objects_ids[object_id] = id_ 920 self.objects_ids_by_id[id_] = object_id 921 return ResultExistence[Tuple[int, Any]](True, (object_id, obj)) 922 923 doi = deserialize_obj_impl 924 925 def deserialize_obj(self, obj_info: Tuple) -> ResultExistence[Tuple[int, Any]]: 926 with DisableGC(): 927 return self.deserialize_obj_impl(obj_info) 928 929 do = deserialize_obj
default_serializable_data_types: Set[Type] =
{<class 'int'>, <class 'complex'>, <class 'NoneType'>, <class 'list'>, <class 'frozenset'>, <class 'bytes'>, <class 'str'>, <class 'tuple'>, <class 'dict'>, <class 'float'>, <class 'set'>, <class 'bool'>, <class 'bytearray'>}
known_types: Set[Type] =
{<class 'int'>, <class 'complex'>, <class 'NoneType'>, <class 'list'>, <class 'frozenset'>, <class 'bytes'>, <class 'str'>, <class 'tuple'>, <class 'dict'>, <class 'float'>, <class 'set'>, <class 'slice'>, <class 'bool'>, <class 'bytearray'>}
known_data_types: Set[Type] =
{<class 'int'>, <class 'NoneType'>, <class 'bytes'>, <class 'str'>, <class 'float'>, <class 'slice'>, <class 'bool'>, <class 'bytearray'>}
known_container_types: Set[Type] =
{<class 'complex'>, <class 'list'>, <class 'tuple'>, <class 'dict'>, <class 'set'>, <class 'frozenset'>}
class
DataType(enum.IntEnum):
91class DataType(IntEnum): 92 class_ = 0 # type is not in known_types and not in serializable_data_types 93 int_ = 1 94 float_ = 2 95 complex_ = 3 96 str_ = 4 97 bytes_ = 5 98 bytearray_ = 6 99 bool_ = 7 100 none_ = 8 101 list_ = 9 102 tuple_ = 10 103 set_ = 11 104 frozenset_ = 12 105 dict_ = 13 106 slice_ = 14 107 unknown_serializable = 15 # type is in serializable_data_types but not in known_types
An enumeration.
class_ =
<DataType.class_: 0>
int_ =
<DataType.int_: 1>
float_ =
<DataType.float_: 2>
complex_ =
<DataType.complex_: 3>
str_ =
<DataType.str_: 4>
bytes_ =
<DataType.bytes_: 5>
bytearray_ =
<DataType.bytearray_: 6>
bool_ =
<DataType.bool_: 7>
none_ =
<DataType.none_: 8>
list_ =
<DataType.list_: 9>
tuple_ =
<DataType.tuple_: 10>
set_ =
<DataType.set_: 11>
frozenset_ =
<DataType.frozenset_: 12>
dict_ =
<DataType.dict_: 13>
slice_ =
<DataType.slice_: 14>
unknown_serializable =
<DataType.unknown_serializable: 15>
Inherited Members
- enum.Enum
- name
- value
- builtins.int
- conjugate
- bit_length
- to_bytes
- from_bytes
- as_integer_ratio
- real
- imag
- numerator
- denominator
data_type: Dict[DataType, Type] =
{<DataType.class_: 0>: <class 'object'>, <DataType.int_: 1>: <class 'int'>, <DataType.float_: 2>: <class 'float'>, <DataType.complex_: 3>: <class 'complex'>, <DataType.str_: 4>: <class 'str'>, <DataType.bytes_: 5>: <class 'bytes'>, <DataType.bytearray_: 6>: <class 'bytearray'>, <DataType.bool_: 7>: <class 'bool'>, <DataType.none_: 8>: <class 'NoneType'>, <DataType.list_: 9>: <class 'list'>, <DataType.tuple_: 10>: <class 'tuple'>, <DataType.set_: 11>: <class 'set'>, <DataType.frozenset_: 12>: <class 'frozenset'>, <DataType.dict_: 13>: <class 'dict'>, <DataType.slice_: 14>: <class 'slice'>, <DataType.unknown_serializable: 15>: None}
data_type_by_type: Dict[Type, DataType] =
{<class 'object'>: <DataType.class_: 0>, <class 'int'>: <DataType.int_: 1>, <class 'float'>: <DataType.float_: 2>, <class 'complex'>: <DataType.complex_: 3>, <class 'str'>: <DataType.str_: 4>, <class 'bytes'>: <DataType.bytes_: 5>, <class 'bytearray'>: <DataType.bytearray_: 6>, <class 'bool'>: <DataType.bool_: 7>, <class 'NoneType'>: <DataType.none_: 8>, <class 'list'>: <DataType.list_: 9>, <class 'tuple'>: <DataType.tuple_: 10>, <class 'set'>: <DataType.set_: 11>, <class 'frozenset'>: <DataType.frozenset_: 12>, <class 'dict'>: <DataType.dict_: 13>, <class 'slice'>: <DataType.slice_: 14>, None: <DataType.unknown_serializable: 15>}
class
ClassInfoFields(enum.IntEnum):
150class ClassInfoFields(IntEnum): 151 class_id = 0 152 class_name = 1 153 module_importable_str = 2 154 owning_names_path = 3
An enumeration.
class_id =
<ClassInfoFields.class_id: 0>
class_name =
<ClassInfoFields.class_name: 1>
module_importable_str =
<ClassInfoFields.module_importable_str: 2>
owning_names_path =
<ClassInfoFields.owning_names_path: 3>
Inherited Members
- enum.Enum
- name
- value
- builtins.int
- conjugate
- bit_length
- to_bytes
- from_bytes
- as_integer_ratio
- real
- imag
- numerator
- denominator
class
ObjectInfoFields(enum.IntEnum):
157class ObjectInfoFields(IntEnum): 158 object_id = 0 # (type: int) 159 type_id = 1 # (type: DataType) 160 object_ = 2 # (Optional), (type: Any). Link to object itself if `(type(obj) in serializable_data_types)`. Not used otherwise. 161 class_id = 3 # (Optional), (type: int). Used if `type_id == DataType.class_` 162 clonable_slots = 4 # (Optional), (type: Tuple[Tuple[str, Any]]). Used if `type_id == DataType.class_`. Holds ID's (object_id) of slots objects. 163 clonable_dict_items = 5 # (Optional), (type: Tuple[Tuple[str, Any]]). Used if `type_id == DataType.class_`. Holds ID's (object_id) of value objects. 164 # contained_items = 6 # (Optional), (type: Union[Tuple, List, Set, FrozenSet, Dict]). Used if `type_id in {DataType.list_, DataType.tuple_, DataType.set_, DataType.frozenset_, DataType.dict_, DataType.slice_}`. Holds ID's (object_id) of contained items (for bothe keys and values in the case of Dict). 165 contained_mapping = 6 # (Optional), (type: Union[Tuple, List, Set, FrozenSet, Dict]). Used if `type_id in {DataType.list_, DataType.tuple_, DataType.set_, DataType.frozenset_, DataType.dict_, DataType.slice_}`. Holds ID's (object_id) of contained items (for bothe keys and values in the case of Dict). 166 contained_sequence = 7 # (Optional), (type: Union[Tuple, List, Set, FrozenSet, Dict]). Used if `type_id in {DataType.list_, DataType.tuple_, DataType.set_, DataType.frozenset_, DataType.dict_, DataType.slice_}`. Holds ID's (object_id) of contained items (for bothe keys and values in the case of Dict). 167 contained_set = 8 # (Optional), (type: Union[Tuple, List, Set, FrozenSet, Dict]). Used if `type_id in {DataType.list_, DataType.tuple_, DataType.set_, DataType.frozenset_, DataType.dict_, DataType.slice_}`. Holds ID's (object_id) of contained items (for bothe keys and values in the case of Dict).
An enumeration.
object_id =
<ObjectInfoFields.object_id: 0>
type_id =
<ObjectInfoFields.type_id: 1>
object_ =
<ObjectInfoFields.object_: 2>
class_id =
<ObjectInfoFields.class_id: 3>
clonable_slots =
<ObjectInfoFields.clonable_slots: 4>
clonable_dict_items =
<ObjectInfoFields.clonable_dict_items: 5>
contained_mapping =
<ObjectInfoFields.contained_mapping: 6>
contained_sequence =
<ObjectInfoFields.contained_sequence: 7>
contained_set =
<ObjectInfoFields.contained_set: 8>
Inherited Members
- enum.Enum
- name
- value
- builtins.int
- conjugate
- bit_length
- to_bytes
- from_bytes
- as_integer_ratio
- real
- imag
- numerator
- denominator
class
CanNotAdjustToSerializableError(builtins.Exception):
Common base class for all non-exit exceptions.
Inherited Members
- builtins.Exception
- Exception
- builtins.BaseException
- with_traceback
- args
class
CanNotAdjustFromSerializableError(builtins.Exception):
Common base class for all non-exit exceptions.
Inherited Members
- builtins.Exception
- Exception
- builtins.BaseException
- with_traceback
- args
class
RemoteObjectsManager:
178class RemoteObjectsManager: 179 def __init__(self, 180 on_new_class_handler: Optional[Callable] = None, 181 on_new_obj_info_handler: Optional[Callable] = None, 182 # classess_db: Optional[Dict[int, Type]] = None, 183 objects_db: Optional[Dict[int, Any]] = None, 184 classes_id_gen: Optional[Callable] = None, 185 objects_id_gen: Optional[Callable] = None, 186 serializable_data_types: Optional[Set[Type]] = None, 187 ) -> None: 188 # self.classess_db: Dict[int, Type] = dict() if classess_db is None else classess_db 189 self.classes_id_gen: IDGenerator = IDGenerator() if classes_id_gen is None else classes_id_gen 190 self.objects_db: Dict[int, Any] = dict() if objects_db is None else objects_db 191 self.objects_id_gen: IDGenerator = IDGenerator() if objects_id_gen is None else objects_id_gen 192 self.serializable_data_types: Set[Type] = default_serializable_data_types if serializable_data_types is None else serializable_data_types 193 self.serializable_any: bool = not self.serializable_data_types 194 self.serializable_int: bool = (int in self.serializable_data_types) or self.serializable_any 195 self.serializable_float: bool = (float in self.serializable_data_types) or self.serializable_any 196 self.serializable_complex: bool = (complex in self.serializable_data_types) or self.serializable_any 197 self.serializable_str: bool = (str in self.serializable_data_types) or self.serializable_any 198 self.serializable_bytes: bool = (bytes in self.serializable_data_types) or self.serializable_any 199 self.serializable_bytearray: bool = (bytearray in self.serializable_data_types) or self.serializable_any 200 self.serializable_bool: bool = (bool in self.serializable_data_types) or self.serializable_any 201 self.serializable_none: bool = (type(None) in self.serializable_data_types) or self.serializable_any 202 self.serializable_list: bool = (list in self.serializable_data_types) or self.serializable_any 203 self.serializable_tuple: bool = (tuple in self.serializable_data_types) or self.serializable_any 204 self.serializable_set: bool = (set in self.serializable_data_types) or self.serializable_any 205 self.serializable_frozenset: bool = (frozenset in self.serializable_data_types) or self.serializable_any 206 self.serializable_dict: bool = (dict in self.serializable_data_types) or self.serializable_any 207 self.serializable_slice: bool = (slice in self.serializable_data_types) or self.serializable_any 208 self.known_classes: Dict[Type, int] = dict() 209 self.known_classes_by_id: Dict[int, Type] = dict() 210 self.known_classes_info: Dict[Tuple, int] = dict() 211 self.known_classes_info_by_id: Dict[int, Tuple] = dict() 212 self.on_new_class_handler: Optional[Callable] = on_new_class_handler 213 self.on_new_obj_info_handler: Optional[Callable] = on_new_obj_info_handler 214 self.objects_ids: Dict[int, int] = dict() # (Key: object_id, Value: id(obj)) 215 self.objects_ids_by_id: Dict[int, int] = dict() # (Key: id(obj), Value: object_id) 216 217 def del_object_by_id(self, id_: int) -> None: 218 if id_ in self.objects_ids_by_id: 219 object_id = self.objects_ids_by_id[id_] 220 self.objects_ids_by_id.pop(id_, None) 221 self.objects_ids.pop(object_id, None) 222 self.objects_db.pop(object_id, None) 223 224 dobi = del_object_by_id 225 226 def del_object_by_object_id(self, object_id: int) -> None: 227 if object_id in self.objects_ids: 228 id_ = self.objects_ids[object_id] 229 self.objects_ids_by_id.pop(id_, None) 230 self.objects_ids.pop(object_id, None) 231 self.objects_db.pop(object_id, None) 232 233 doboi = del_object_by_object_id 234 235 def adjust_to_serializable(self, type_id: DataType, obj: Any) -> Any: 236 if DataType.int_ == type_id: 237 return obj 238 elif DataType.float_ == type_id: 239 return obj 240 elif DataType.complex_ == type_id: 241 if self.serializable_complex: 242 return obj 243 elif self.serializable_bytes: 244 return pack('=dd', obj.real, obj.imag) 245 elif self.serializable_bytearray: 246 return bytearray(pack('=dd', obj.real, obj.imag)) 247 elif self.serializable_str: 248 return str(obj) 249 elif self.serializable_float: 250 if self.serializable_tuple: 251 return (obj.real, obj.imag) 252 elif self.serializable_list: 253 return [obj.real, obj.imag] 254 else: 255 pass 256 else: 257 pass 258 elif DataType.str_ == type_id: 259 return obj 260 elif DataType.bytes_ == type_id: 261 if self.serializable_bytes: 262 return obj 263 elif self.serializable_bytearray: 264 return bytearray(obj) 265 elif self.serializable_str: 266 return obj.hex() 267 elif self.serializable_tuple: 268 if self.serializable_int: 269 return tuple(int(c) for c in obj) 270 if self.serializable_float: 271 return tuple(float(int(c)) for c in obj) 272 else: 273 pass 274 elif self.serializable_list: 275 if self.serializable_int: 276 return [int(c) for c in obj] 277 if self.serializable_float: 278 return [float(int(c)) for c in obj] 279 else: 280 pass 281 else: 282 pass 283 elif DataType.bytearray_ == type_id: 284 if self.serializable_bytearray: 285 return obj 286 elif self.serializable_bytes: 287 return bytes(obj) 288 elif self.serializable_str: 289 return obj.hex() 290 elif self.serializable_tuple: 291 if self.serializable_int: 292 return tuple(int(c) for c in obj) 293 if self.serializable_float: 294 return tuple(float(int(c)) for c in obj) 295 else: 296 pass 297 elif self.serializable_list: 298 if self.serializable_int: 299 return [int(c) for c in obj] 300 if self.serializable_float: 301 return [float(int(c)) for c in obj] 302 else: 303 pass 304 else: 305 pass 306 elif DataType.bool_ == type_id: 307 return obj 308 elif DataType.none_ == type_id: 309 return obj 310 elif DataType.list_ == type_id: 311 if self.serializable_list: 312 return obj 313 elif self.serializable_tuple: 314 return tuple(obj) 315 elif self.serializable_dict: 316 return dict({index: item for index, item in enumerate(obj)}) 317 else: 318 pass 319 elif DataType.tuple_ == type_id: 320 if self.serializable_tuple: 321 return obj 322 elif self.serializable_list: 323 return list(obj) 324 elif self.serializable_dict: 325 return dict({index: item for index, item in enumerate(obj)}) 326 else: 327 pass 328 elif DataType.set_ == type_id: 329 if self.serializable_set: 330 return obj 331 elif self.serializable_frozenset: 332 return frozenset(obj) 333 elif self.serializable_tuple: 334 return tuple(obj) 335 elif self.serializable_list: 336 return list(obj) 337 elif self.serializable_dict: 338 return dict({k: None for k in obj}) 339 else: 340 pass 341 elif DataType.frozenset_ == type_id: 342 if self.serializable_frozenset: 343 return obj 344 elif self.serializable_set: 345 return set(obj) 346 elif self.serializable_tuple: 347 return tuple(obj) 348 elif self.serializable_list: 349 return list(obj) 350 elif self.serializable_dict: 351 return dict({k: None for k in obj}) 352 else: 353 pass 354 elif DataType.dict_ == type_id: 355 return obj 356 elif DataType.slice_ == type_id: 357 if self.serializable_slice: 358 return obj 359 elif self.serializable_tuple: 360 return (obj.start, obj.stop, obj.step) 361 elif self.serializable_list: 362 return [obj.start, obj.stop, obj.step] 363 elif self.serializable_dict: 364 return {0: obj.start, 1: obj.stop, 2: obj.step} 365 else: 366 pass 367 else: 368 raise RuntimeError('Unknown type_id') 369 370 raise CanNotAdjustToSerializableError(f'Can not adjust to serializable. Type: {type_id}, obj: {obj}') 371 372 ats = adjust_to_serializable 373 374 def adjust_from_serializable(self, type_id: DataType, obj: Any) -> Any: 375 if DataType.int_ == type_id: 376 return obj 377 elif DataType.float_ == type_id: 378 return obj 379 elif DataType.complex_ == type_id: 380 if self.serializable_complex: 381 return obj 382 elif self.serializable_bytes: 383 return complex(*unpack('=dd', obj)) 384 elif self.serializable_bytearray: 385 return complex(*unpack('=dd', bytes(obj))) 386 elif self.serializable_str: 387 return complex(*obj) 388 elif self.serializable_float: 389 if self.serializable_tuple: 390 return complex(*obj) 391 elif self.serializable_list: 392 return complex(*obj) 393 else: 394 pass 395 else: 396 pass 397 elif DataType.str_ == type_id: 398 return obj 399 elif DataType.bytes_ == type_id: 400 if self.serializable_bytes: 401 return obj 402 elif self.serializable_bytearray: 403 return bytes(obj) 404 elif self.serializable_str: 405 return bytes.fromhex(obj) 406 elif self.serializable_tuple: 407 if self.serializable_int: 408 return b''.join(item.to_bytes(1, 'little') for item in obj) 409 if self.serializable_float: 410 return b''.join((int(round(item))).to_bytes(1, 'little') for item in obj) 411 else: 412 pass 413 elif self.serializable_list: 414 if self.serializable_int: 415 return b''.join(item.to_bytes(1, 'little') for item in obj) 416 if self.serializable_float: 417 return b''.join((int(round(item))).to_bytes(1, 'little') for item in obj) 418 else: 419 pass 420 else: 421 pass 422 elif DataType.bytearray_ == type_id: 423 if self.serializable_bytearray: 424 return obj 425 elif self.serializable_bytes: 426 return bytearray(obj) 427 elif self.serializable_str: 428 return bytearray(bytes.fromhex(obj)) 429 elif self.serializable_tuple: 430 if self.serializable_int: 431 return bytearray(b''.join(item.to_bytes(1, 'little') for item in obj)) 432 if self.serializable_float: 433 return bytearray(b''.join((int(round(item))).to_bytes(1, 'little') for item in obj)) 434 else: 435 pass 436 elif self.serializable_list: 437 if self.serializable_int: 438 return bytearray(b''.join(item.to_bytes(1, 'little') for item in obj)) 439 if self.serializable_float: 440 return bytearray(b''.join((int(round(item))).to_bytes(1, 'little') for item in obj)) 441 else: 442 pass 443 else: 444 pass 445 elif DataType.bool_ == type_id: 446 return obj 447 elif DataType.none_ == type_id: 448 return None 449 elif DataType.list_ == type_id: 450 if self.serializable_list: 451 return obj 452 elif self.serializable_tuple: 453 return list(obj) 454 elif self.serializable_dict: 455 return [value for key, value in sorted(obj.items(), key=lambda x: x[0])] 456 else: 457 pass 458 elif DataType.tuple_ == type_id: 459 if self.serializable_tuple: 460 return obj 461 elif self.serializable_list: 462 return tuple(obj) 463 elif self.serializable_dict: 464 return tuple(value for key, value in sorted(obj.items(), key=lambda x: x[0])) 465 else: 466 pass 467 elif DataType.set_ == type_id: 468 if self.serializable_set: 469 return obj 470 elif self.serializable_frozenset: 471 return set(obj) 472 elif self.serializable_tuple: 473 return set(obj) 474 elif self.serializable_list: 475 return set(obj) 476 elif self.serializable_dict: 477 return set(obj.keys()) 478 else: 479 pass 480 elif DataType.frozenset_ == type_id: 481 if self.serializable_frozenset: 482 return obj 483 elif self.serializable_set: 484 return frozenset(obj) 485 elif self.serializable_tuple: 486 return frozenset(obj) 487 elif self.serializable_list: 488 return frozenset(obj) 489 elif self.serializable_dict: 490 return frozenset(obj.keys()) 491 else: 492 pass 493 elif DataType.dict_ == type_id: 494 return obj 495 elif DataType.slice_ == type_id: 496 if self.serializable_slice: 497 return obj 498 elif self.serializable_tuple: 499 return slice(*obj) 500 elif self.serializable_list: 501 return slice(*obj) 502 elif self.serializable_dict: 503 return slice(obj[0], obj[1], obj[2]) 504 else: 505 pass 506 else: 507 raise RuntimeError('Unknown type_id') 508 509 raise CanNotAdjustFromSerializableError(f'Can not adjust from serializable. Type: {type_id}, obj: {obj}') 510 511 afs = adjust_from_serializable 512 513 # def is_replicatable_object_attribute(self, attribute: Any) -> bool: 514 # if is_setable_data_descriptor(attribute): 515 # data = attribute.__get__(None, None) 516 # elif is_callable(attribute): 517 # return False 518 # else: 519 # return True 520 521 def serialize_container(self, type_id: DataType, obj: Any) -> Any: 522 if DataType.list_ == type_id: 523 new_obj = list() 524 for item in obj: 525 result: ResultExistence[int] = self.serialize_impl(item) 526 if result: 527 new_obj.append(result.value) 528 elif DataType.tuple_ == type_id: 529 new_obj = list() 530 for item in obj: 531 result: ResultExistence[int] = self.serialize_impl(item) 532 if result: 533 new_obj.append(result.value) 534 535 new_obj = tuple(new_obj) 536 elif DataType.set_ == type_id: 537 new_obj = set() 538 for item in obj: 539 result: ResultExistence[int] = self.serialize_impl(item) 540 if result: 541 new_obj.add(result.value) 542 elif DataType.frozenset_ == type_id: 543 new_obj = set() 544 for item in obj: 545 result: ResultExistence[int] = self.serialize_impl(item) 546 if result: 547 new_obj.add(result.value) 548 549 new_obj = frozenset(new_obj) 550 elif DataType.dict_ == type_id: 551 new_obj = dict() 552 for key, value in obj.items(): 553 key_result: ResultExistence[int] = self.serialize_impl(key) 554 value_result: ResultExistence[int] = self.serialize_impl(value) 555 if key_result and value_result: 556 new_obj[key_result.value] = value_result.value 557 558 return new_obj 559 560 sc = serialize_container 561 562 def serialize_impl(self, obj: Any, ignore_empty_classes: bool = False) -> ResultExistence[int]: 563 result_exists: bool = True 564 id_: int = id(obj) 565 obj_type = type(obj) 566 if (int != obj_type) and (id_ in self.objects_ids_by_id): 567 new_object: bool = False 568 object_id: int = self.objects_ids_by_id[id_] 569 else: 570 # int object must always produce new object_id because first 256 ints are persistent across Python sessions and 571 # this can cause issues within users of current module. For example within `cengal/hardware/memory/shared_memory` 572 # which changes int values inline instead of producing new objects. 573 new_object = True 574 object_id = self.objects_id_gen() 575 self.objects_ids[object_id] = id_ 576 self.objects_db[object_id] = obj 577 578 if not new_object: 579 return ResultExistence[int](result_exists, object_id) 580 581 if self.serializable_any or (obj_type in self.serializable_data_types): 582 serializable: bool = True 583 type_id: DataType = data_type_by_type.get(obj_type, DataType.unknown_serializable) 584 else: 585 serializable = False 586 type_id = data_type_by_type.get(obj_type, DataType.class_) 587 588 if obj_type in known_container_types: 589 known_container: bool = True 590 else: 591 known_container = False 592 593 class_info: Dict[int, Any] = None 594 known_data: bool = None 595 class_id: int = None 596 is_new_class: bool = None 597 class_name: str = None 598 new_obj_slots: List[Tuple[str, Any]] = None 599 new_obj_dict: Dict[str, Any] = None 600 obj_mapping: Dict = None 601 obj_sequence: List = None 602 obj_set: Set = None 603 if serializable: 604 if known_container: 605 object_info = ( 606 object_id, 607 type_id.value, 608 self.sc(type_id, obj), 609 ) 610 else: 611 object_info = ( 612 object_id, 613 type_id.value, 614 obj, 615 ) 616 else: 617 if obj_type in known_data_types: 618 known_data = True 619 else: 620 known_data = False 621 622 if DataType.class_ == type_id: 623 new_obj_slots = list() 624 for slot_name, slot_value in filled_slot_names_with_values_gen(obj): 625 adjusted_slot_name = slot_name 626 item_result: ResultExistence[int] = self.serialize_impl(slot_value) 627 if item_result: 628 if self.serializable_tuple: 629 new_obj_slots.append((adjusted_slot_name, item_result.value)) 630 elif self.serializable_list: 631 new_obj_slots.append([adjusted_slot_name, item_result.value]) 632 else: 633 new_obj_slots.append(self.ats(DataType.tuple_, (adjusted_slot_name, item_result.value))) 634 635 if new_obj_slots: 636 if self.serializable_list: 637 pass 638 elif self.serializable_tuple: 639 new_obj_slots = tuple(new_obj_slots) 640 else: 641 new_obj_slots = self.ats(DataType.list_, new_obj_slots) 642 643 new_obj_dict = dict() 644 if hasattr(obj, '__dict__'): 645 for key, value in obj.__dict__.items(): 646 # raw_value = getattr_static(obj, key) 647 # if hasattr(raw_value, '__get__') and (not hasattr(raw_value, '__set__')): 648 # # if not setable descriptor 649 # continue 650 651 adjusted_key = key 652 653 value_result: ResultExistence[int] = self.serialize_impl(value) 654 if value_result: 655 new_obj_dict[adjusted_key] = value_result.value 656 else: 657 continue 658 659 obj_mapping = dict() 660 obj_sequence = list() 661 obj_set = list() 662 if isinstance(obj, MutableMapping): 663 for key, value in obj.items(): 664 key_result: ResultExistence[int] = self.serialize_impl(value) 665 if not key_result: 666 continue 667 668 value_result: ResultExistence[int] = self.serialize_impl(value) 669 if value_result: 670 obj_mapping[key_result.value] = value_result.value 671 else: 672 continue 673 elif isinstance(obj, MutableSequence): 674 for item in obj: 675 item_result: ResultExistence[int] = self.serialize_impl(item) 676 if item_result: 677 obj_sequence.append(item_result.value) 678 else: 679 continue 680 681 if obj_sequence: 682 if self.serializable_list: 683 pass 684 else: 685 obj_sequence = self.ats(DataType.list_, obj_sequence) 686 elif isinstance(obj, MutableSet): 687 for item in obj: 688 item_result: ResultExistence[int] = self.serialize_impl(item) 689 if item_result: 690 obj_set.append(item_result.value) 691 else: 692 continue 693 694 if obj_set: 695 if self.serializable_set: 696 pass 697 else: 698 obj_set = self.ats(DataType.list_, obj_set) 699 else: 700 pass 701 702 if ignore_empty_classes: 703 result_exists = new_obj_slots or new_obj_dict or obj_mapping or obj_sequence or obj_set 704 705 if result_exists: 706 if obj_type in self.known_classes: 707 is_new_class = False 708 class_id = self.known_classes[obj_type] 709 else: 710 class_info = dict() 711 is_new_class = True 712 class_name = obj_type.__name__ 713 class_id = self.classes_id_gen() 714 self.known_classes[obj_type] = class_id 715 self.known_classes_by_id[class_id] = obj_type 716 module_importable_str, owning_names_path = entity_module_importable_str_and_owning_names_path(obj) 717 module_importable_str_and_owning_names_path = (class_name, module_importable_str, tuple(owning_names_path)) 718 self.known_classes_info[module_importable_str_and_owning_names_path] = class_id 719 self.known_classes_info_by_id[class_id] = module_importable_str_and_owning_names_path 720 721 class_info = ( 722 class_id, 723 class_name, 724 module_importable_str, 725 owning_names_path, 726 ) 727 728 if self.on_new_class_handler: 729 self.on_new_class_handler( 730 class_info, 731 obj_type, 732 class_id, 733 class_name, 734 module_importable_str, 735 owning_names_path, 736 module_importable_str_and_owning_names_path 737 ) 738 739 # will be later serialized much faster than without `if else None` 740 object_info = ( 741 object_id, 742 type_id.value, 743 0, 744 class_id, 745 new_obj_slots if new_obj_slots else 0, 746 new_obj_dict if new_obj_dict else 0, 747 obj_mapping if obj_mapping else 0, 748 obj_sequence if obj_sequence else 0, 749 obj_set if obj_set else 0, 750 ) 751 elif known_data: 752 object_info = ( 753 object_id, 754 type_id.value, 755 self.ats(type_id, obj), 756 ) 757 elif known_container: 758 object_info = ( 759 object_id, 760 type_id.value, 761 self.ats(type_id, self.sc(type_id, obj)), 762 ) 763 else: 764 raise RuntimeError('Unknown type_id') 765 766 if result_exists: 767 if self.on_new_obj_info_handler: 768 self.on_new_obj_info_handler( 769 object_info, 770 obj, 771 object_id, 772 type_id, 773 serializable, 774 known_container, 775 known_data, 776 class_id, 777 is_new_class, 778 new_obj_slots, 779 new_obj_dict, 780 obj_mapping, 781 obj_sequence, 782 obj_set, 783 ) 784 785 return ResultExistence[int](result_exists, object_id) 786 787 si = serialize_impl 788 789 def serialize(self, obj: Any, ignore_empty_classes: bool = False) -> ResultExistence[int]: 790 with DisableGC(): 791 return self.serialize_impl(obj, ignore_empty_classes) 792 793 s = serialize 794 795 def deserialize_class_impl(self, class_info: Tuple) -> ResultExistence[Tuple[int, Type]]: 796 class_id: int = class_info[ClassInfoFields.class_id.value] 797 if class_id in self.known_classes_by_id: 798 return ResultExistence[Tuple[int, Type]](True, (class_id, self.known_classes_by_id[class_id])) 799 else: 800 class_name: str = class_info[ClassInfoFields.class_name.value] 801 module_importable_str: str = class_info[ClassInfoFields.module_importable_str.value] 802 owning_names_path: List[str] = self.afs(DataType.list_, class_info[ClassInfoFields.owning_names_path.value]) 803 obj_class: Type = entity_by_name_module_importable_str_and_owning_names_path(class_name, module_importable_str, owning_names_path) 804 self.known_classes_by_id[class_id] = obj_class 805 self.known_classes[obj_class] = class_id 806 class_info_tuple: Tuple = (class_name, module_importable_str, tuple(owning_names_path)) 807 self.known_classes_info[class_info_tuple] = class_id 808 self.known_classes_info_by_id[class_id] = class_info_tuple 809 return ResultExistence[Tuple[int, Type]](True, (class_id, obj_class)) 810 811 dcli = deserialize_class_impl 812 813 def deserialize_class(self, class_info: Tuple) -> ResultExistence[Tuple[int, Type]]: 814 with DisableGC(): 815 return self.deserialize_class_impl(class_info) 816 817 dcl = deserialize_class 818 819 def deserialize_container_impl(self, type_id: DataType, obj: Any) -> Any: 820 new_obj = obj 821 if DataType.list_ == type_id: 822 new_obj = list() 823 for item_id in obj: 824 new_obj.append(self.objects_db[item_id]) 825 elif DataType.tuple_ == type_id: 826 new_obj = list() 827 for item_id in obj: 828 new_obj.append(self.objects_db[item_id]) 829 830 new_obj = tuple(new_obj) 831 elif DataType.set_ == type_id: 832 new_obj = set() 833 for item_id in obj: 834 new_obj.add(self.objects_db[item_id]) 835 elif DataType.frozenset_ == type_id: 836 new_obj = set() 837 for item_id in obj: 838 new_obj.add(self.objects_db[item_id]) 839 840 new_obj = frozenset(new_obj) 841 elif DataType.dict_ == type_id: 842 new_obj = dict() 843 for key_id, value_id in obj.items(): 844 new_obj[self.objects_db[int(key_id)]] = self.objects_db[value_id] 845 846 return new_obj 847 848 dcoi = deserialize_container_impl 849 850 def deserialize_container(self, type_id: DataType, obj: Any) -> Any: 851 with DisableGC(): 852 return self.deserialize_container_impl(type_id, obj) 853 854 dco = deserialize_container 855 856 def deserialize_obj_impl(self, obj_info: Tuple) -> ResultExistence[Tuple[int, Any]]: 857 object_id: int = obj_info[ObjectInfoFields.object_id.value] 858 if object_id in self.objects_db: 859 return ResultExistence[Tuple[int, Any]](True, (object_id, self.objects_db[object_id])) 860 861 type_id: DataType = DataType(obj_info[ObjectInfoFields.type_id.value]) 862 863 obj_type = data_type[type_id] 864 serializable: bool = self.serializable_any or (obj_type in self.serializable_data_types) 865 known_container: bool = obj_type in known_container_types 866 known_data: bool = None 867 868 if serializable: 869 if known_container: 870 obj = self.dcoi(type_id, obj_info[ObjectInfoFields.object_.value]) 871 else: 872 obj = obj_info[ObjectInfoFields.object_.value] 873 else: 874 known_data = obj_type in known_data_types 875 876 if DataType.class_ == type_id: 877 class_id: int = obj_info[ObjectInfoFields.class_id.value] 878 obj_class: Type = self.known_classes_by_id[class_id] 879 obj: Any = obj_class.__new__(obj_class) 880 881 if obj_info[ObjectInfoFields.clonable_slots.value]: 882 clonable_slots = self.afs(DataType.list_, obj_info[ObjectInfoFields.clonable_slots.value]) 883 for slot_name, slot_id in clonable_slots: 884 child_obj = self.objects_db[slot_id] 885 setattr(obj, slot_name, child_obj) 886 887 if obj_info[ObjectInfoFields.clonable_dict_items.value]: 888 clonable_dict_items = obj_info[ObjectInfoFields.clonable_dict_items.value] 889 for key, value_id in clonable_dict_items.items(): 890 child_obj = self.objects_db[value_id] 891 setattr(obj, key, child_obj) 892 893 if obj_info[ObjectInfoFields.contained_mapping.value]: 894 contained_mapping = obj_info[ObjectInfoFields.contained_mapping.value] 895 for key_id, value_id in contained_mapping.items(): 896 child_key = self.objects_db[int(key_id)] 897 child_value = self.objects_db[value_id] 898 obj[child_key] = child_value 899 900 if obj_info[ObjectInfoFields.contained_sequence.value]: 901 contained_sequence = self.afs(DataType.list_, obj_info[ObjectInfoFields.contained_sequence.value]) 902 for item_id in contained_sequence: 903 child_item = self.objects_db[item_id] 904 obj.append(child_item) 905 906 if obj_info[ObjectInfoFields.contained_set.value]: 907 contained_set = self.afs(DataType.list_, obj_info[ObjectInfoFields.contained_set.value]) 908 for item_id in contained_set: 909 child_item = self.objects_db[item_id] 910 obj.add(child_item) 911 elif known_data: 912 obj = self.afs(type_id, obj_info[ObjectInfoFields.object_.value]) 913 elif known_container: 914 obj = self.dcoi(type_id, self.afs(type_id, obj_info[ObjectInfoFields.object_.value])) 915 else: 916 raise RuntimeError('Unknown type_id') 917 918 self.objects_db[object_id] = obj 919 id_: int = id(obj) 920 self.objects_ids[object_id] = id_ 921 self.objects_ids_by_id[id_] = object_id 922 return ResultExistence[Tuple[int, Any]](True, (object_id, obj)) 923 924 doi = deserialize_obj_impl 925 926 def deserialize_obj(self, obj_info: Tuple) -> ResultExistence[Tuple[int, Any]]: 927 with DisableGC(): 928 return self.deserialize_obj_impl(obj_info) 929 930 do = deserialize_obj
RemoteObjectsManager( on_new_class_handler: Union[Callable, NoneType] = None, on_new_obj_info_handler: Union[Callable, NoneType] = None, objects_db: Union[Dict[int, Any], NoneType] = None, classes_id_gen: Union[Callable, NoneType] = None, objects_id_gen: Union[Callable, NoneType] = None, serializable_data_types: Union[Set[Type], NoneType] = None)
179 def __init__(self, 180 on_new_class_handler: Optional[Callable] = None, 181 on_new_obj_info_handler: Optional[Callable] = None, 182 # classess_db: Optional[Dict[int, Type]] = None, 183 objects_db: Optional[Dict[int, Any]] = None, 184 classes_id_gen: Optional[Callable] = None, 185 objects_id_gen: Optional[Callable] = None, 186 serializable_data_types: Optional[Set[Type]] = None, 187 ) -> None: 188 # self.classess_db: Dict[int, Type] = dict() if classess_db is None else classess_db 189 self.classes_id_gen: IDGenerator = IDGenerator() if classes_id_gen is None else classes_id_gen 190 self.objects_db: Dict[int, Any] = dict() if objects_db is None else objects_db 191 self.objects_id_gen: IDGenerator = IDGenerator() if objects_id_gen is None else objects_id_gen 192 self.serializable_data_types: Set[Type] = default_serializable_data_types if serializable_data_types is None else serializable_data_types 193 self.serializable_any: bool = not self.serializable_data_types 194 self.serializable_int: bool = (int in self.serializable_data_types) or self.serializable_any 195 self.serializable_float: bool = (float in self.serializable_data_types) or self.serializable_any 196 self.serializable_complex: bool = (complex in self.serializable_data_types) or self.serializable_any 197 self.serializable_str: bool = (str in self.serializable_data_types) or self.serializable_any 198 self.serializable_bytes: bool = (bytes in self.serializable_data_types) or self.serializable_any 199 self.serializable_bytearray: bool = (bytearray in self.serializable_data_types) or self.serializable_any 200 self.serializable_bool: bool = (bool in self.serializable_data_types) or self.serializable_any 201 self.serializable_none: bool = (type(None) in self.serializable_data_types) or self.serializable_any 202 self.serializable_list: bool = (list in self.serializable_data_types) or self.serializable_any 203 self.serializable_tuple: bool = (tuple in self.serializable_data_types) or self.serializable_any 204 self.serializable_set: bool = (set in self.serializable_data_types) or self.serializable_any 205 self.serializable_frozenset: bool = (frozenset in self.serializable_data_types) or self.serializable_any 206 self.serializable_dict: bool = (dict in self.serializable_data_types) or self.serializable_any 207 self.serializable_slice: bool = (slice in self.serializable_data_types) or self.serializable_any 208 self.known_classes: Dict[Type, int] = dict() 209 self.known_classes_by_id: Dict[int, Type] = dict() 210 self.known_classes_info: Dict[Tuple, int] = dict() 211 self.known_classes_info_by_id: Dict[int, Tuple] = dict() 212 self.on_new_class_handler: Optional[Callable] = on_new_class_handler 213 self.on_new_obj_info_handler: Optional[Callable] = on_new_obj_info_handler 214 self.objects_ids: Dict[int, int] = dict() # (Key: object_id, Value: id(obj)) 215 self.objects_ids_by_id: Dict[int, int] = dict() # (Key: id(obj), Value: object_id)
235 def adjust_to_serializable(self, type_id: DataType, obj: Any) -> Any: 236 if DataType.int_ == type_id: 237 return obj 238 elif DataType.float_ == type_id: 239 return obj 240 elif DataType.complex_ == type_id: 241 if self.serializable_complex: 242 return obj 243 elif self.serializable_bytes: 244 return pack('=dd', obj.real, obj.imag) 245 elif self.serializable_bytearray: 246 return bytearray(pack('=dd', obj.real, obj.imag)) 247 elif self.serializable_str: 248 return str(obj) 249 elif self.serializable_float: 250 if self.serializable_tuple: 251 return (obj.real, obj.imag) 252 elif self.serializable_list: 253 return [obj.real, obj.imag] 254 else: 255 pass 256 else: 257 pass 258 elif DataType.str_ == type_id: 259 return obj 260 elif DataType.bytes_ == type_id: 261 if self.serializable_bytes: 262 return obj 263 elif self.serializable_bytearray: 264 return bytearray(obj) 265 elif self.serializable_str: 266 return obj.hex() 267 elif self.serializable_tuple: 268 if self.serializable_int: 269 return tuple(int(c) for c in obj) 270 if self.serializable_float: 271 return tuple(float(int(c)) for c in obj) 272 else: 273 pass 274 elif self.serializable_list: 275 if self.serializable_int: 276 return [int(c) for c in obj] 277 if self.serializable_float: 278 return [float(int(c)) for c in obj] 279 else: 280 pass 281 else: 282 pass 283 elif DataType.bytearray_ == type_id: 284 if self.serializable_bytearray: 285 return obj 286 elif self.serializable_bytes: 287 return bytes(obj) 288 elif self.serializable_str: 289 return obj.hex() 290 elif self.serializable_tuple: 291 if self.serializable_int: 292 return tuple(int(c) for c in obj) 293 if self.serializable_float: 294 return tuple(float(int(c)) for c in obj) 295 else: 296 pass 297 elif self.serializable_list: 298 if self.serializable_int: 299 return [int(c) for c in obj] 300 if self.serializable_float: 301 return [float(int(c)) for c in obj] 302 else: 303 pass 304 else: 305 pass 306 elif DataType.bool_ == type_id: 307 return obj 308 elif DataType.none_ == type_id: 309 return obj 310 elif DataType.list_ == type_id: 311 if self.serializable_list: 312 return obj 313 elif self.serializable_tuple: 314 return tuple(obj) 315 elif self.serializable_dict: 316 return dict({index: item for index, item in enumerate(obj)}) 317 else: 318 pass 319 elif DataType.tuple_ == type_id: 320 if self.serializable_tuple: 321 return obj 322 elif self.serializable_list: 323 return list(obj) 324 elif self.serializable_dict: 325 return dict({index: item for index, item in enumerate(obj)}) 326 else: 327 pass 328 elif DataType.set_ == type_id: 329 if self.serializable_set: 330 return obj 331 elif self.serializable_frozenset: 332 return frozenset(obj) 333 elif self.serializable_tuple: 334 return tuple(obj) 335 elif self.serializable_list: 336 return list(obj) 337 elif self.serializable_dict: 338 return dict({k: None for k in obj}) 339 else: 340 pass 341 elif DataType.frozenset_ == type_id: 342 if self.serializable_frozenset: 343 return obj 344 elif self.serializable_set: 345 return set(obj) 346 elif self.serializable_tuple: 347 return tuple(obj) 348 elif self.serializable_list: 349 return list(obj) 350 elif self.serializable_dict: 351 return dict({k: None for k in obj}) 352 else: 353 pass 354 elif DataType.dict_ == type_id: 355 return obj 356 elif DataType.slice_ == type_id: 357 if self.serializable_slice: 358 return obj 359 elif self.serializable_tuple: 360 return (obj.start, obj.stop, obj.step) 361 elif self.serializable_list: 362 return [obj.start, obj.stop, obj.step] 363 elif self.serializable_dict: 364 return {0: obj.start, 1: obj.stop, 2: obj.step} 365 else: 366 pass 367 else: 368 raise RuntimeError('Unknown type_id') 369 370 raise CanNotAdjustToSerializableError(f'Can not adjust to serializable. Type: {type_id}, obj: {obj}')
235 def adjust_to_serializable(self, type_id: DataType, obj: Any) -> Any: 236 if DataType.int_ == type_id: 237 return obj 238 elif DataType.float_ == type_id: 239 return obj 240 elif DataType.complex_ == type_id: 241 if self.serializable_complex: 242 return obj 243 elif self.serializable_bytes: 244 return pack('=dd', obj.real, obj.imag) 245 elif self.serializable_bytearray: 246 return bytearray(pack('=dd', obj.real, obj.imag)) 247 elif self.serializable_str: 248 return str(obj) 249 elif self.serializable_float: 250 if self.serializable_tuple: 251 return (obj.real, obj.imag) 252 elif self.serializable_list: 253 return [obj.real, obj.imag] 254 else: 255 pass 256 else: 257 pass 258 elif DataType.str_ == type_id: 259 return obj 260 elif DataType.bytes_ == type_id: 261 if self.serializable_bytes: 262 return obj 263 elif self.serializable_bytearray: 264 return bytearray(obj) 265 elif self.serializable_str: 266 return obj.hex() 267 elif self.serializable_tuple: 268 if self.serializable_int: 269 return tuple(int(c) for c in obj) 270 if self.serializable_float: 271 return tuple(float(int(c)) for c in obj) 272 else: 273 pass 274 elif self.serializable_list: 275 if self.serializable_int: 276 return [int(c) for c in obj] 277 if self.serializable_float: 278 return [float(int(c)) for c in obj] 279 else: 280 pass 281 else: 282 pass 283 elif DataType.bytearray_ == type_id: 284 if self.serializable_bytearray: 285 return obj 286 elif self.serializable_bytes: 287 return bytes(obj) 288 elif self.serializable_str: 289 return obj.hex() 290 elif self.serializable_tuple: 291 if self.serializable_int: 292 return tuple(int(c) for c in obj) 293 if self.serializable_float: 294 return tuple(float(int(c)) for c in obj) 295 else: 296 pass 297 elif self.serializable_list: 298 if self.serializable_int: 299 return [int(c) for c in obj] 300 if self.serializable_float: 301 return [float(int(c)) for c in obj] 302 else: 303 pass 304 else: 305 pass 306 elif DataType.bool_ == type_id: 307 return obj 308 elif DataType.none_ == type_id: 309 return obj 310 elif DataType.list_ == type_id: 311 if self.serializable_list: 312 return obj 313 elif self.serializable_tuple: 314 return tuple(obj) 315 elif self.serializable_dict: 316 return dict({index: item for index, item in enumerate(obj)}) 317 else: 318 pass 319 elif DataType.tuple_ == type_id: 320 if self.serializable_tuple: 321 return obj 322 elif self.serializable_list: 323 return list(obj) 324 elif self.serializable_dict: 325 return dict({index: item for index, item in enumerate(obj)}) 326 else: 327 pass 328 elif DataType.set_ == type_id: 329 if self.serializable_set: 330 return obj 331 elif self.serializable_frozenset: 332 return frozenset(obj) 333 elif self.serializable_tuple: 334 return tuple(obj) 335 elif self.serializable_list: 336 return list(obj) 337 elif self.serializable_dict: 338 return dict({k: None for k in obj}) 339 else: 340 pass 341 elif DataType.frozenset_ == type_id: 342 if self.serializable_frozenset: 343 return obj 344 elif self.serializable_set: 345 return set(obj) 346 elif self.serializable_tuple: 347 return tuple(obj) 348 elif self.serializable_list: 349 return list(obj) 350 elif self.serializable_dict: 351 return dict({k: None for k in obj}) 352 else: 353 pass 354 elif DataType.dict_ == type_id: 355 return obj 356 elif DataType.slice_ == type_id: 357 if self.serializable_slice: 358 return obj 359 elif self.serializable_tuple: 360 return (obj.start, obj.stop, obj.step) 361 elif self.serializable_list: 362 return [obj.start, obj.stop, obj.step] 363 elif self.serializable_dict: 364 return {0: obj.start, 1: obj.stop, 2: obj.step} 365 else: 366 pass 367 else: 368 raise RuntimeError('Unknown type_id') 369 370 raise CanNotAdjustToSerializableError(f'Can not adjust to serializable. Type: {type_id}, obj: {obj}')
374 def adjust_from_serializable(self, type_id: DataType, obj: Any) -> Any: 375 if DataType.int_ == type_id: 376 return obj 377 elif DataType.float_ == type_id: 378 return obj 379 elif DataType.complex_ == type_id: 380 if self.serializable_complex: 381 return obj 382 elif self.serializable_bytes: 383 return complex(*unpack('=dd', obj)) 384 elif self.serializable_bytearray: 385 return complex(*unpack('=dd', bytes(obj))) 386 elif self.serializable_str: 387 return complex(*obj) 388 elif self.serializable_float: 389 if self.serializable_tuple: 390 return complex(*obj) 391 elif self.serializable_list: 392 return complex(*obj) 393 else: 394 pass 395 else: 396 pass 397 elif DataType.str_ == type_id: 398 return obj 399 elif DataType.bytes_ == type_id: 400 if self.serializable_bytes: 401 return obj 402 elif self.serializable_bytearray: 403 return bytes(obj) 404 elif self.serializable_str: 405 return bytes.fromhex(obj) 406 elif self.serializable_tuple: 407 if self.serializable_int: 408 return b''.join(item.to_bytes(1, 'little') for item in obj) 409 if self.serializable_float: 410 return b''.join((int(round(item))).to_bytes(1, 'little') for item in obj) 411 else: 412 pass 413 elif self.serializable_list: 414 if self.serializable_int: 415 return b''.join(item.to_bytes(1, 'little') for item in obj) 416 if self.serializable_float: 417 return b''.join((int(round(item))).to_bytes(1, 'little') for item in obj) 418 else: 419 pass 420 else: 421 pass 422 elif DataType.bytearray_ == type_id: 423 if self.serializable_bytearray: 424 return obj 425 elif self.serializable_bytes: 426 return bytearray(obj) 427 elif self.serializable_str: 428 return bytearray(bytes.fromhex(obj)) 429 elif self.serializable_tuple: 430 if self.serializable_int: 431 return bytearray(b''.join(item.to_bytes(1, 'little') for item in obj)) 432 if self.serializable_float: 433 return bytearray(b''.join((int(round(item))).to_bytes(1, 'little') for item in obj)) 434 else: 435 pass 436 elif self.serializable_list: 437 if self.serializable_int: 438 return bytearray(b''.join(item.to_bytes(1, 'little') for item in obj)) 439 if self.serializable_float: 440 return bytearray(b''.join((int(round(item))).to_bytes(1, 'little') for item in obj)) 441 else: 442 pass 443 else: 444 pass 445 elif DataType.bool_ == type_id: 446 return obj 447 elif DataType.none_ == type_id: 448 return None 449 elif DataType.list_ == type_id: 450 if self.serializable_list: 451 return obj 452 elif self.serializable_tuple: 453 return list(obj) 454 elif self.serializable_dict: 455 return [value for key, value in sorted(obj.items(), key=lambda x: x[0])] 456 else: 457 pass 458 elif DataType.tuple_ == type_id: 459 if self.serializable_tuple: 460 return obj 461 elif self.serializable_list: 462 return tuple(obj) 463 elif self.serializable_dict: 464 return tuple(value for key, value in sorted(obj.items(), key=lambda x: x[0])) 465 else: 466 pass 467 elif DataType.set_ == type_id: 468 if self.serializable_set: 469 return obj 470 elif self.serializable_frozenset: 471 return set(obj) 472 elif self.serializable_tuple: 473 return set(obj) 474 elif self.serializable_list: 475 return set(obj) 476 elif self.serializable_dict: 477 return set(obj.keys()) 478 else: 479 pass 480 elif DataType.frozenset_ == type_id: 481 if self.serializable_frozenset: 482 return obj 483 elif self.serializable_set: 484 return frozenset(obj) 485 elif self.serializable_tuple: 486 return frozenset(obj) 487 elif self.serializable_list: 488 return frozenset(obj) 489 elif self.serializable_dict: 490 return frozenset(obj.keys()) 491 else: 492 pass 493 elif DataType.dict_ == type_id: 494 return obj 495 elif DataType.slice_ == type_id: 496 if self.serializable_slice: 497 return obj 498 elif self.serializable_tuple: 499 return slice(*obj) 500 elif self.serializable_list: 501 return slice(*obj) 502 elif self.serializable_dict: 503 return slice(obj[0], obj[1], obj[2]) 504 else: 505 pass 506 else: 507 raise RuntimeError('Unknown type_id') 508 509 raise CanNotAdjustFromSerializableError(f'Can not adjust from serializable. Type: {type_id}, obj: {obj}')
374 def adjust_from_serializable(self, type_id: DataType, obj: Any) -> Any: 375 if DataType.int_ == type_id: 376 return obj 377 elif DataType.float_ == type_id: 378 return obj 379 elif DataType.complex_ == type_id: 380 if self.serializable_complex: 381 return obj 382 elif self.serializable_bytes: 383 return complex(*unpack('=dd', obj)) 384 elif self.serializable_bytearray: 385 return complex(*unpack('=dd', bytes(obj))) 386 elif self.serializable_str: 387 return complex(*obj) 388 elif self.serializable_float: 389 if self.serializable_tuple: 390 return complex(*obj) 391 elif self.serializable_list: 392 return complex(*obj) 393 else: 394 pass 395 else: 396 pass 397 elif DataType.str_ == type_id: 398 return obj 399 elif DataType.bytes_ == type_id: 400 if self.serializable_bytes: 401 return obj 402 elif self.serializable_bytearray: 403 return bytes(obj) 404 elif self.serializable_str: 405 return bytes.fromhex(obj) 406 elif self.serializable_tuple: 407 if self.serializable_int: 408 return b''.join(item.to_bytes(1, 'little') for item in obj) 409 if self.serializable_float: 410 return b''.join((int(round(item))).to_bytes(1, 'little') for item in obj) 411 else: 412 pass 413 elif self.serializable_list: 414 if self.serializable_int: 415 return b''.join(item.to_bytes(1, 'little') for item in obj) 416 if self.serializable_float: 417 return b''.join((int(round(item))).to_bytes(1, 'little') for item in obj) 418 else: 419 pass 420 else: 421 pass 422 elif DataType.bytearray_ == type_id: 423 if self.serializable_bytearray: 424 return obj 425 elif self.serializable_bytes: 426 return bytearray(obj) 427 elif self.serializable_str: 428 return bytearray(bytes.fromhex(obj)) 429 elif self.serializable_tuple: 430 if self.serializable_int: 431 return bytearray(b''.join(item.to_bytes(1, 'little') for item in obj)) 432 if self.serializable_float: 433 return bytearray(b''.join((int(round(item))).to_bytes(1, 'little') for item in obj)) 434 else: 435 pass 436 elif self.serializable_list: 437 if self.serializable_int: 438 return bytearray(b''.join(item.to_bytes(1, 'little') for item in obj)) 439 if self.serializable_float: 440 return bytearray(b''.join((int(round(item))).to_bytes(1, 'little') for item in obj)) 441 else: 442 pass 443 else: 444 pass 445 elif DataType.bool_ == type_id: 446 return obj 447 elif DataType.none_ == type_id: 448 return None 449 elif DataType.list_ == type_id: 450 if self.serializable_list: 451 return obj 452 elif self.serializable_tuple: 453 return list(obj) 454 elif self.serializable_dict: 455 return [value for key, value in sorted(obj.items(), key=lambda x: x[0])] 456 else: 457 pass 458 elif DataType.tuple_ == type_id: 459 if self.serializable_tuple: 460 return obj 461 elif self.serializable_list: 462 return tuple(obj) 463 elif self.serializable_dict: 464 return tuple(value for key, value in sorted(obj.items(), key=lambda x: x[0])) 465 else: 466 pass 467 elif DataType.set_ == type_id: 468 if self.serializable_set: 469 return obj 470 elif self.serializable_frozenset: 471 return set(obj) 472 elif self.serializable_tuple: 473 return set(obj) 474 elif self.serializable_list: 475 return set(obj) 476 elif self.serializable_dict: 477 return set(obj.keys()) 478 else: 479 pass 480 elif DataType.frozenset_ == type_id: 481 if self.serializable_frozenset: 482 return obj 483 elif self.serializable_set: 484 return frozenset(obj) 485 elif self.serializable_tuple: 486 return frozenset(obj) 487 elif self.serializable_list: 488 return frozenset(obj) 489 elif self.serializable_dict: 490 return frozenset(obj.keys()) 491 else: 492 pass 493 elif DataType.dict_ == type_id: 494 return obj 495 elif DataType.slice_ == type_id: 496 if self.serializable_slice: 497 return obj 498 elif self.serializable_tuple: 499 return slice(*obj) 500 elif self.serializable_list: 501 return slice(*obj) 502 elif self.serializable_dict: 503 return slice(obj[0], obj[1], obj[2]) 504 else: 505 pass 506 else: 507 raise RuntimeError('Unknown type_id') 508 509 raise CanNotAdjustFromSerializableError(f'Can not adjust from serializable. Type: {type_id}, obj: {obj}')
521 def serialize_container(self, type_id: DataType, obj: Any) -> Any: 522 if DataType.list_ == type_id: 523 new_obj = list() 524 for item in obj: 525 result: ResultExistence[int] = self.serialize_impl(item) 526 if result: 527 new_obj.append(result.value) 528 elif DataType.tuple_ == type_id: 529 new_obj = list() 530 for item in obj: 531 result: ResultExistence[int] = self.serialize_impl(item) 532 if result: 533 new_obj.append(result.value) 534 535 new_obj = tuple(new_obj) 536 elif DataType.set_ == type_id: 537 new_obj = set() 538 for item in obj: 539 result: ResultExistence[int] = self.serialize_impl(item) 540 if result: 541 new_obj.add(result.value) 542 elif DataType.frozenset_ == type_id: 543 new_obj = set() 544 for item in obj: 545 result: ResultExistence[int] = self.serialize_impl(item) 546 if result: 547 new_obj.add(result.value) 548 549 new_obj = frozenset(new_obj) 550 elif DataType.dict_ == type_id: 551 new_obj = dict() 552 for key, value in obj.items(): 553 key_result: ResultExistence[int] = self.serialize_impl(key) 554 value_result: ResultExistence[int] = self.serialize_impl(value) 555 if key_result and value_result: 556 new_obj[key_result.value] = value_result.value 557 558 return new_obj
521 def serialize_container(self, type_id: DataType, obj: Any) -> Any: 522 if DataType.list_ == type_id: 523 new_obj = list() 524 for item in obj: 525 result: ResultExistence[int] = self.serialize_impl(item) 526 if result: 527 new_obj.append(result.value) 528 elif DataType.tuple_ == type_id: 529 new_obj = list() 530 for item in obj: 531 result: ResultExistence[int] = self.serialize_impl(item) 532 if result: 533 new_obj.append(result.value) 534 535 new_obj = tuple(new_obj) 536 elif DataType.set_ == type_id: 537 new_obj = set() 538 for item in obj: 539 result: ResultExistence[int] = self.serialize_impl(item) 540 if result: 541 new_obj.add(result.value) 542 elif DataType.frozenset_ == type_id: 543 new_obj = set() 544 for item in obj: 545 result: ResultExistence[int] = self.serialize_impl(item) 546 if result: 547 new_obj.add(result.value) 548 549 new_obj = frozenset(new_obj) 550 elif DataType.dict_ == type_id: 551 new_obj = dict() 552 for key, value in obj.items(): 553 key_result: ResultExistence[int] = self.serialize_impl(key) 554 value_result: ResultExistence[int] = self.serialize_impl(value) 555 if key_result and value_result: 556 new_obj[key_result.value] = value_result.value 557 558 return new_obj
def
serialize_impl( self, obj: Any, ignore_empty_classes: bool = False) -> cengal.code_flow_control.smart_values.versions.v_2.smart_values.ResultExistence[int]:
562 def serialize_impl(self, obj: Any, ignore_empty_classes: bool = False) -> ResultExistence[int]: 563 result_exists: bool = True 564 id_: int = id(obj) 565 obj_type = type(obj) 566 if (int != obj_type) and (id_ in self.objects_ids_by_id): 567 new_object: bool = False 568 object_id: int = self.objects_ids_by_id[id_] 569 else: 570 # int object must always produce new object_id because first 256 ints are persistent across Python sessions and 571 # this can cause issues within users of current module. For example within `cengal/hardware/memory/shared_memory` 572 # which changes int values inline instead of producing new objects. 573 new_object = True 574 object_id = self.objects_id_gen() 575 self.objects_ids[object_id] = id_ 576 self.objects_db[object_id] = obj 577 578 if not new_object: 579 return ResultExistence[int](result_exists, object_id) 580 581 if self.serializable_any or (obj_type in self.serializable_data_types): 582 serializable: bool = True 583 type_id: DataType = data_type_by_type.get(obj_type, DataType.unknown_serializable) 584 else: 585 serializable = False 586 type_id = data_type_by_type.get(obj_type, DataType.class_) 587 588 if obj_type in known_container_types: 589 known_container: bool = True 590 else: 591 known_container = False 592 593 class_info: Dict[int, Any] = None 594 known_data: bool = None 595 class_id: int = None 596 is_new_class: bool = None 597 class_name: str = None 598 new_obj_slots: List[Tuple[str, Any]] = None 599 new_obj_dict: Dict[str, Any] = None 600 obj_mapping: Dict = None 601 obj_sequence: List = None 602 obj_set: Set = None 603 if serializable: 604 if known_container: 605 object_info = ( 606 object_id, 607 type_id.value, 608 self.sc(type_id, obj), 609 ) 610 else: 611 object_info = ( 612 object_id, 613 type_id.value, 614 obj, 615 ) 616 else: 617 if obj_type in known_data_types: 618 known_data = True 619 else: 620 known_data = False 621 622 if DataType.class_ == type_id: 623 new_obj_slots = list() 624 for slot_name, slot_value in filled_slot_names_with_values_gen(obj): 625 adjusted_slot_name = slot_name 626 item_result: ResultExistence[int] = self.serialize_impl(slot_value) 627 if item_result: 628 if self.serializable_tuple: 629 new_obj_slots.append((adjusted_slot_name, item_result.value)) 630 elif self.serializable_list: 631 new_obj_slots.append([adjusted_slot_name, item_result.value]) 632 else: 633 new_obj_slots.append(self.ats(DataType.tuple_, (adjusted_slot_name, item_result.value))) 634 635 if new_obj_slots: 636 if self.serializable_list: 637 pass 638 elif self.serializable_tuple: 639 new_obj_slots = tuple(new_obj_slots) 640 else: 641 new_obj_slots = self.ats(DataType.list_, new_obj_slots) 642 643 new_obj_dict = dict() 644 if hasattr(obj, '__dict__'): 645 for key, value in obj.__dict__.items(): 646 # raw_value = getattr_static(obj, key) 647 # if hasattr(raw_value, '__get__') and (not hasattr(raw_value, '__set__')): 648 # # if not setable descriptor 649 # continue 650 651 adjusted_key = key 652 653 value_result: ResultExistence[int] = self.serialize_impl(value) 654 if value_result: 655 new_obj_dict[adjusted_key] = value_result.value 656 else: 657 continue 658 659 obj_mapping = dict() 660 obj_sequence = list() 661 obj_set = list() 662 if isinstance(obj, MutableMapping): 663 for key, value in obj.items(): 664 key_result: ResultExistence[int] = self.serialize_impl(value) 665 if not key_result: 666 continue 667 668 value_result: ResultExistence[int] = self.serialize_impl(value) 669 if value_result: 670 obj_mapping[key_result.value] = value_result.value 671 else: 672 continue 673 elif isinstance(obj, MutableSequence): 674 for item in obj: 675 item_result: ResultExistence[int] = self.serialize_impl(item) 676 if item_result: 677 obj_sequence.append(item_result.value) 678 else: 679 continue 680 681 if obj_sequence: 682 if self.serializable_list: 683 pass 684 else: 685 obj_sequence = self.ats(DataType.list_, obj_sequence) 686 elif isinstance(obj, MutableSet): 687 for item in obj: 688 item_result: ResultExistence[int] = self.serialize_impl(item) 689 if item_result: 690 obj_set.append(item_result.value) 691 else: 692 continue 693 694 if obj_set: 695 if self.serializable_set: 696 pass 697 else: 698 obj_set = self.ats(DataType.list_, obj_set) 699 else: 700 pass 701 702 if ignore_empty_classes: 703 result_exists = new_obj_slots or new_obj_dict or obj_mapping or obj_sequence or obj_set 704 705 if result_exists: 706 if obj_type in self.known_classes: 707 is_new_class = False 708 class_id = self.known_classes[obj_type] 709 else: 710 class_info = dict() 711 is_new_class = True 712 class_name = obj_type.__name__ 713 class_id = self.classes_id_gen() 714 self.known_classes[obj_type] = class_id 715 self.known_classes_by_id[class_id] = obj_type 716 module_importable_str, owning_names_path = entity_module_importable_str_and_owning_names_path(obj) 717 module_importable_str_and_owning_names_path = (class_name, module_importable_str, tuple(owning_names_path)) 718 self.known_classes_info[module_importable_str_and_owning_names_path] = class_id 719 self.known_classes_info_by_id[class_id] = module_importable_str_and_owning_names_path 720 721 class_info = ( 722 class_id, 723 class_name, 724 module_importable_str, 725 owning_names_path, 726 ) 727 728 if self.on_new_class_handler: 729 self.on_new_class_handler( 730 class_info, 731 obj_type, 732 class_id, 733 class_name, 734 module_importable_str, 735 owning_names_path, 736 module_importable_str_and_owning_names_path 737 ) 738 739 # will be later serialized much faster than without `if else None` 740 object_info = ( 741 object_id, 742 type_id.value, 743 0, 744 class_id, 745 new_obj_slots if new_obj_slots else 0, 746 new_obj_dict if new_obj_dict else 0, 747 obj_mapping if obj_mapping else 0, 748 obj_sequence if obj_sequence else 0, 749 obj_set if obj_set else 0, 750 ) 751 elif known_data: 752 object_info = ( 753 object_id, 754 type_id.value, 755 self.ats(type_id, obj), 756 ) 757 elif known_container: 758 object_info = ( 759 object_id, 760 type_id.value, 761 self.ats(type_id, self.sc(type_id, obj)), 762 ) 763 else: 764 raise RuntimeError('Unknown type_id') 765 766 if result_exists: 767 if self.on_new_obj_info_handler: 768 self.on_new_obj_info_handler( 769 object_info, 770 obj, 771 object_id, 772 type_id, 773 serializable, 774 known_container, 775 known_data, 776 class_id, 777 is_new_class, 778 new_obj_slots, 779 new_obj_dict, 780 obj_mapping, 781 obj_sequence, 782 obj_set, 783 ) 784 785 return ResultExistence[int](result_exists, object_id)
def
si( self, obj: Any, ignore_empty_classes: bool = False) -> cengal.code_flow_control.smart_values.versions.v_2.smart_values.ResultExistence[int]:
562 def serialize_impl(self, obj: Any, ignore_empty_classes: bool = False) -> ResultExistence[int]: 563 result_exists: bool = True 564 id_: int = id(obj) 565 obj_type = type(obj) 566 if (int != obj_type) and (id_ in self.objects_ids_by_id): 567 new_object: bool = False 568 object_id: int = self.objects_ids_by_id[id_] 569 else: 570 # int object must always produce new object_id because first 256 ints are persistent across Python sessions and 571 # this can cause issues within users of current module. For example within `cengal/hardware/memory/shared_memory` 572 # which changes int values inline instead of producing new objects. 573 new_object = True 574 object_id = self.objects_id_gen() 575 self.objects_ids[object_id] = id_ 576 self.objects_db[object_id] = obj 577 578 if not new_object: 579 return ResultExistence[int](result_exists, object_id) 580 581 if self.serializable_any or (obj_type in self.serializable_data_types): 582 serializable: bool = True 583 type_id: DataType = data_type_by_type.get(obj_type, DataType.unknown_serializable) 584 else: 585 serializable = False 586 type_id = data_type_by_type.get(obj_type, DataType.class_) 587 588 if obj_type in known_container_types: 589 known_container: bool = True 590 else: 591 known_container = False 592 593 class_info: Dict[int, Any] = None 594 known_data: bool = None 595 class_id: int = None 596 is_new_class: bool = None 597 class_name: str = None 598 new_obj_slots: List[Tuple[str, Any]] = None 599 new_obj_dict: Dict[str, Any] = None 600 obj_mapping: Dict = None 601 obj_sequence: List = None 602 obj_set: Set = None 603 if serializable: 604 if known_container: 605 object_info = ( 606 object_id, 607 type_id.value, 608 self.sc(type_id, obj), 609 ) 610 else: 611 object_info = ( 612 object_id, 613 type_id.value, 614 obj, 615 ) 616 else: 617 if obj_type in known_data_types: 618 known_data = True 619 else: 620 known_data = False 621 622 if DataType.class_ == type_id: 623 new_obj_slots = list() 624 for slot_name, slot_value in filled_slot_names_with_values_gen(obj): 625 adjusted_slot_name = slot_name 626 item_result: ResultExistence[int] = self.serialize_impl(slot_value) 627 if item_result: 628 if self.serializable_tuple: 629 new_obj_slots.append((adjusted_slot_name, item_result.value)) 630 elif self.serializable_list: 631 new_obj_slots.append([adjusted_slot_name, item_result.value]) 632 else: 633 new_obj_slots.append(self.ats(DataType.tuple_, (adjusted_slot_name, item_result.value))) 634 635 if new_obj_slots: 636 if self.serializable_list: 637 pass 638 elif self.serializable_tuple: 639 new_obj_slots = tuple(new_obj_slots) 640 else: 641 new_obj_slots = self.ats(DataType.list_, new_obj_slots) 642 643 new_obj_dict = dict() 644 if hasattr(obj, '__dict__'): 645 for key, value in obj.__dict__.items(): 646 # raw_value = getattr_static(obj, key) 647 # if hasattr(raw_value, '__get__') and (not hasattr(raw_value, '__set__')): 648 # # if not setable descriptor 649 # continue 650 651 adjusted_key = key 652 653 value_result: ResultExistence[int] = self.serialize_impl(value) 654 if value_result: 655 new_obj_dict[adjusted_key] = value_result.value 656 else: 657 continue 658 659 obj_mapping = dict() 660 obj_sequence = list() 661 obj_set = list() 662 if isinstance(obj, MutableMapping): 663 for key, value in obj.items(): 664 key_result: ResultExistence[int] = self.serialize_impl(value) 665 if not key_result: 666 continue 667 668 value_result: ResultExistence[int] = self.serialize_impl(value) 669 if value_result: 670 obj_mapping[key_result.value] = value_result.value 671 else: 672 continue 673 elif isinstance(obj, MutableSequence): 674 for item in obj: 675 item_result: ResultExistence[int] = self.serialize_impl(item) 676 if item_result: 677 obj_sequence.append(item_result.value) 678 else: 679 continue 680 681 if obj_sequence: 682 if self.serializable_list: 683 pass 684 else: 685 obj_sequence = self.ats(DataType.list_, obj_sequence) 686 elif isinstance(obj, MutableSet): 687 for item in obj: 688 item_result: ResultExistence[int] = self.serialize_impl(item) 689 if item_result: 690 obj_set.append(item_result.value) 691 else: 692 continue 693 694 if obj_set: 695 if self.serializable_set: 696 pass 697 else: 698 obj_set = self.ats(DataType.list_, obj_set) 699 else: 700 pass 701 702 if ignore_empty_classes: 703 result_exists = new_obj_slots or new_obj_dict or obj_mapping or obj_sequence or obj_set 704 705 if result_exists: 706 if obj_type in self.known_classes: 707 is_new_class = False 708 class_id = self.known_classes[obj_type] 709 else: 710 class_info = dict() 711 is_new_class = True 712 class_name = obj_type.__name__ 713 class_id = self.classes_id_gen() 714 self.known_classes[obj_type] = class_id 715 self.known_classes_by_id[class_id] = obj_type 716 module_importable_str, owning_names_path = entity_module_importable_str_and_owning_names_path(obj) 717 module_importable_str_and_owning_names_path = (class_name, module_importable_str, tuple(owning_names_path)) 718 self.known_classes_info[module_importable_str_and_owning_names_path] = class_id 719 self.known_classes_info_by_id[class_id] = module_importable_str_and_owning_names_path 720 721 class_info = ( 722 class_id, 723 class_name, 724 module_importable_str, 725 owning_names_path, 726 ) 727 728 if self.on_new_class_handler: 729 self.on_new_class_handler( 730 class_info, 731 obj_type, 732 class_id, 733 class_name, 734 module_importable_str, 735 owning_names_path, 736 module_importable_str_and_owning_names_path 737 ) 738 739 # will be later serialized much faster than without `if else None` 740 object_info = ( 741 object_id, 742 type_id.value, 743 0, 744 class_id, 745 new_obj_slots if new_obj_slots else 0, 746 new_obj_dict if new_obj_dict else 0, 747 obj_mapping if obj_mapping else 0, 748 obj_sequence if obj_sequence else 0, 749 obj_set if obj_set else 0, 750 ) 751 elif known_data: 752 object_info = ( 753 object_id, 754 type_id.value, 755 self.ats(type_id, obj), 756 ) 757 elif known_container: 758 object_info = ( 759 object_id, 760 type_id.value, 761 self.ats(type_id, self.sc(type_id, obj)), 762 ) 763 else: 764 raise RuntimeError('Unknown type_id') 765 766 if result_exists: 767 if self.on_new_obj_info_handler: 768 self.on_new_obj_info_handler( 769 object_info, 770 obj, 771 object_id, 772 type_id, 773 serializable, 774 known_container, 775 known_data, 776 class_id, 777 is_new_class, 778 new_obj_slots, 779 new_obj_dict, 780 obj_mapping, 781 obj_sequence, 782 obj_set, 783 ) 784 785 return ResultExistence[int](result_exists, object_id)
def
serialize( self, obj: Any, ignore_empty_classes: bool = False) -> cengal.code_flow_control.smart_values.versions.v_2.smart_values.ResultExistence[int]:
def
s( self, obj: Any, ignore_empty_classes: bool = False) -> cengal.code_flow_control.smart_values.versions.v_2.smart_values.ResultExistence[int]:
def
deserialize_class_impl( self, class_info: Tuple) -> cengal.code_flow_control.smart_values.versions.v_2.smart_values.ResultExistence[typing.Tuple[int, typing.Type]]:
795 def deserialize_class_impl(self, class_info: Tuple) -> ResultExistence[Tuple[int, Type]]: 796 class_id: int = class_info[ClassInfoFields.class_id.value] 797 if class_id in self.known_classes_by_id: 798 return ResultExistence[Tuple[int, Type]](True, (class_id, self.known_classes_by_id[class_id])) 799 else: 800 class_name: str = class_info[ClassInfoFields.class_name.value] 801 module_importable_str: str = class_info[ClassInfoFields.module_importable_str.value] 802 owning_names_path: List[str] = self.afs(DataType.list_, class_info[ClassInfoFields.owning_names_path.value]) 803 obj_class: Type = entity_by_name_module_importable_str_and_owning_names_path(class_name, module_importable_str, owning_names_path) 804 self.known_classes_by_id[class_id] = obj_class 805 self.known_classes[obj_class] = class_id 806 class_info_tuple: Tuple = (class_name, module_importable_str, tuple(owning_names_path)) 807 self.known_classes_info[class_info_tuple] = class_id 808 self.known_classes_info_by_id[class_id] = class_info_tuple 809 return ResultExistence[Tuple[int, Type]](True, (class_id, obj_class))
def
dcli( self, class_info: Tuple) -> cengal.code_flow_control.smart_values.versions.v_2.smart_values.ResultExistence[typing.Tuple[int, typing.Type]]:
795 def deserialize_class_impl(self, class_info: Tuple) -> ResultExistence[Tuple[int, Type]]: 796 class_id: int = class_info[ClassInfoFields.class_id.value] 797 if class_id in self.known_classes_by_id: 798 return ResultExistence[Tuple[int, Type]](True, (class_id, self.known_classes_by_id[class_id])) 799 else: 800 class_name: str = class_info[ClassInfoFields.class_name.value] 801 module_importable_str: str = class_info[ClassInfoFields.module_importable_str.value] 802 owning_names_path: List[str] = self.afs(DataType.list_, class_info[ClassInfoFields.owning_names_path.value]) 803 obj_class: Type = entity_by_name_module_importable_str_and_owning_names_path(class_name, module_importable_str, owning_names_path) 804 self.known_classes_by_id[class_id] = obj_class 805 self.known_classes[obj_class] = class_id 806 class_info_tuple: Tuple = (class_name, module_importable_str, tuple(owning_names_path)) 807 self.known_classes_info[class_info_tuple] = class_id 808 self.known_classes_info_by_id[class_id] = class_info_tuple 809 return ResultExistence[Tuple[int, Type]](True, (class_id, obj_class))
def
deserialize_class( self, class_info: Tuple) -> cengal.code_flow_control.smart_values.versions.v_2.smart_values.ResultExistence[typing.Tuple[int, typing.Type]]:
def
dcl( self, class_info: Tuple) -> cengal.code_flow_control.smart_values.versions.v_2.smart_values.ResultExistence[typing.Tuple[int, typing.Type]]:
819 def deserialize_container_impl(self, type_id: DataType, obj: Any) -> Any: 820 new_obj = obj 821 if DataType.list_ == type_id: 822 new_obj = list() 823 for item_id in obj: 824 new_obj.append(self.objects_db[item_id]) 825 elif DataType.tuple_ == type_id: 826 new_obj = list() 827 for item_id in obj: 828 new_obj.append(self.objects_db[item_id]) 829 830 new_obj = tuple(new_obj) 831 elif DataType.set_ == type_id: 832 new_obj = set() 833 for item_id in obj: 834 new_obj.add(self.objects_db[item_id]) 835 elif DataType.frozenset_ == type_id: 836 new_obj = set() 837 for item_id in obj: 838 new_obj.add(self.objects_db[item_id]) 839 840 new_obj = frozenset(new_obj) 841 elif DataType.dict_ == type_id: 842 new_obj = dict() 843 for key_id, value_id in obj.items(): 844 new_obj[self.objects_db[int(key_id)]] = self.objects_db[value_id] 845 846 return new_obj
819 def deserialize_container_impl(self, type_id: DataType, obj: Any) -> Any: 820 new_obj = obj 821 if DataType.list_ == type_id: 822 new_obj = list() 823 for item_id in obj: 824 new_obj.append(self.objects_db[item_id]) 825 elif DataType.tuple_ == type_id: 826 new_obj = list() 827 for item_id in obj: 828 new_obj.append(self.objects_db[item_id]) 829 830 new_obj = tuple(new_obj) 831 elif DataType.set_ == type_id: 832 new_obj = set() 833 for item_id in obj: 834 new_obj.add(self.objects_db[item_id]) 835 elif DataType.frozenset_ == type_id: 836 new_obj = set() 837 for item_id in obj: 838 new_obj.add(self.objects_db[item_id]) 839 840 new_obj = frozenset(new_obj) 841 elif DataType.dict_ == type_id: 842 new_obj = dict() 843 for key_id, value_id in obj.items(): 844 new_obj[self.objects_db[int(key_id)]] = self.objects_db[value_id] 845 846 return new_obj
def
deserialize_obj_impl( self, obj_info: Tuple) -> cengal.code_flow_control.smart_values.versions.v_2.smart_values.ResultExistence[typing.Tuple[int, typing.Any]]:
856 def deserialize_obj_impl(self, obj_info: Tuple) -> ResultExistence[Tuple[int, Any]]: 857 object_id: int = obj_info[ObjectInfoFields.object_id.value] 858 if object_id in self.objects_db: 859 return ResultExistence[Tuple[int, Any]](True, (object_id, self.objects_db[object_id])) 860 861 type_id: DataType = DataType(obj_info[ObjectInfoFields.type_id.value]) 862 863 obj_type = data_type[type_id] 864 serializable: bool = self.serializable_any or (obj_type in self.serializable_data_types) 865 known_container: bool = obj_type in known_container_types 866 known_data: bool = None 867 868 if serializable: 869 if known_container: 870 obj = self.dcoi(type_id, obj_info[ObjectInfoFields.object_.value]) 871 else: 872 obj = obj_info[ObjectInfoFields.object_.value] 873 else: 874 known_data = obj_type in known_data_types 875 876 if DataType.class_ == type_id: 877 class_id: int = obj_info[ObjectInfoFields.class_id.value] 878 obj_class: Type = self.known_classes_by_id[class_id] 879 obj: Any = obj_class.__new__(obj_class) 880 881 if obj_info[ObjectInfoFields.clonable_slots.value]: 882 clonable_slots = self.afs(DataType.list_, obj_info[ObjectInfoFields.clonable_slots.value]) 883 for slot_name, slot_id in clonable_slots: 884 child_obj = self.objects_db[slot_id] 885 setattr(obj, slot_name, child_obj) 886 887 if obj_info[ObjectInfoFields.clonable_dict_items.value]: 888 clonable_dict_items = obj_info[ObjectInfoFields.clonable_dict_items.value] 889 for key, value_id in clonable_dict_items.items(): 890 child_obj = self.objects_db[value_id] 891 setattr(obj, key, child_obj) 892 893 if obj_info[ObjectInfoFields.contained_mapping.value]: 894 contained_mapping = obj_info[ObjectInfoFields.contained_mapping.value] 895 for key_id, value_id in contained_mapping.items(): 896 child_key = self.objects_db[int(key_id)] 897 child_value = self.objects_db[value_id] 898 obj[child_key] = child_value 899 900 if obj_info[ObjectInfoFields.contained_sequence.value]: 901 contained_sequence = self.afs(DataType.list_, obj_info[ObjectInfoFields.contained_sequence.value]) 902 for item_id in contained_sequence: 903 child_item = self.objects_db[item_id] 904 obj.append(child_item) 905 906 if obj_info[ObjectInfoFields.contained_set.value]: 907 contained_set = self.afs(DataType.list_, obj_info[ObjectInfoFields.contained_set.value]) 908 for item_id in contained_set: 909 child_item = self.objects_db[item_id] 910 obj.add(child_item) 911 elif known_data: 912 obj = self.afs(type_id, obj_info[ObjectInfoFields.object_.value]) 913 elif known_container: 914 obj = self.dcoi(type_id, self.afs(type_id, obj_info[ObjectInfoFields.object_.value])) 915 else: 916 raise RuntimeError('Unknown type_id') 917 918 self.objects_db[object_id] = obj 919 id_: int = id(obj) 920 self.objects_ids[object_id] = id_ 921 self.objects_ids_by_id[id_] = object_id 922 return ResultExistence[Tuple[int, Any]](True, (object_id, obj))
def
doi( self, obj_info: Tuple) -> cengal.code_flow_control.smart_values.versions.v_2.smart_values.ResultExistence[typing.Tuple[int, typing.Any]]:
856 def deserialize_obj_impl(self, obj_info: Tuple) -> ResultExistence[Tuple[int, Any]]: 857 object_id: int = obj_info[ObjectInfoFields.object_id.value] 858 if object_id in self.objects_db: 859 return ResultExistence[Tuple[int, Any]](True, (object_id, self.objects_db[object_id])) 860 861 type_id: DataType = DataType(obj_info[ObjectInfoFields.type_id.value]) 862 863 obj_type = data_type[type_id] 864 serializable: bool = self.serializable_any or (obj_type in self.serializable_data_types) 865 known_container: bool = obj_type in known_container_types 866 known_data: bool = None 867 868 if serializable: 869 if known_container: 870 obj = self.dcoi(type_id, obj_info[ObjectInfoFields.object_.value]) 871 else: 872 obj = obj_info[ObjectInfoFields.object_.value] 873 else: 874 known_data = obj_type in known_data_types 875 876 if DataType.class_ == type_id: 877 class_id: int = obj_info[ObjectInfoFields.class_id.value] 878 obj_class: Type = self.known_classes_by_id[class_id] 879 obj: Any = obj_class.__new__(obj_class) 880 881 if obj_info[ObjectInfoFields.clonable_slots.value]: 882 clonable_slots = self.afs(DataType.list_, obj_info[ObjectInfoFields.clonable_slots.value]) 883 for slot_name, slot_id in clonable_slots: 884 child_obj = self.objects_db[slot_id] 885 setattr(obj, slot_name, child_obj) 886 887 if obj_info[ObjectInfoFields.clonable_dict_items.value]: 888 clonable_dict_items = obj_info[ObjectInfoFields.clonable_dict_items.value] 889 for key, value_id in clonable_dict_items.items(): 890 child_obj = self.objects_db[value_id] 891 setattr(obj, key, child_obj) 892 893 if obj_info[ObjectInfoFields.contained_mapping.value]: 894 contained_mapping = obj_info[ObjectInfoFields.contained_mapping.value] 895 for key_id, value_id in contained_mapping.items(): 896 child_key = self.objects_db[int(key_id)] 897 child_value = self.objects_db[value_id] 898 obj[child_key] = child_value 899 900 if obj_info[ObjectInfoFields.contained_sequence.value]: 901 contained_sequence = self.afs(DataType.list_, obj_info[ObjectInfoFields.contained_sequence.value]) 902 for item_id in contained_sequence: 903 child_item = self.objects_db[item_id] 904 obj.append(child_item) 905 906 if obj_info[ObjectInfoFields.contained_set.value]: 907 contained_set = self.afs(DataType.list_, obj_info[ObjectInfoFields.contained_set.value]) 908 for item_id in contained_set: 909 child_item = self.objects_db[item_id] 910 obj.add(child_item) 911 elif known_data: 912 obj = self.afs(type_id, obj_info[ObjectInfoFields.object_.value]) 913 elif known_container: 914 obj = self.dcoi(type_id, self.afs(type_id, obj_info[ObjectInfoFields.object_.value])) 915 else: 916 raise RuntimeError('Unknown type_id') 917 918 self.objects_db[object_id] = obj 919 id_: int = id(obj) 920 self.objects_ids[object_id] = id_ 921 self.objects_ids_by_id[id_] = object_id 922 return ResultExistence[Tuple[int, Any]](True, (object_id, obj))