cengal.data_manipulation.remote_objects.versions.v_0_optimized.remote_objects
1#!/usr/bin/env python 2# coding=utf-8 3 4# Copyright © 2012-2024 ButenkoMS. All rights reserved. Contacts: <gtalk@butenkoms.space> 5# 6# Licensed under the Apache License, Version 2.0 (the "License"); 7# you may not use this file except in compliance with the License. 8# You may obtain a copy of the License at 9# 10# http://www.apache.org/licenses/LICENSE-2.0 11# 12# Unless required by applicable law or agreed to in writing, software 13# distributed under the License is distributed on an "AS IS" BASIS, 14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15# See the License for the specific language governing permissions and 16# limitations under the License. 17 18 19__all__ = [ 20 'default_serializable_data_types', 21 'known_types', 22 'known_data_types', 23 'known_container_types', 24 'DataType', 25 'data_type', 26 'data_type_by_type', 27 'ClassInfoFields', 28 'ObjectInfoFields', 29 'CanNotAdjustToSerializableError', 30 'CanNotAdjustFromSerializableError', 31 'RemoteObjectsManager', 32] 33 34 35from cengal.introspection.inspect import ( 36 entity_module_importable_str_and_owning_names_path, 37 entity_by_name_module_importable_str_and_owning_names_path, 38 filled_slot_names_with_values_gen, 39 is_callable, is_async, 40 is_setable_data_descriptor, 41) 42from cengal.code_flow_control.smart_values import ResultExistence 43from cengal.code_flow_control.gc import DisableGC 44from cengal.data_generation.id_generator import IDGenerator, GeneratorType 45from enum import IntEnum 46from collections.abc import MutableMapping, MutableSequence, MutableSet 47from struct import pack, unpack 48from inspect import getattr_static 49from typing import Any, Dict, Optional, Callable, Set, Type, Tuple, List, FrozenSet 50 51 52""" 53Module Docstring 54Docstrings: http://www.python.org/dev/peps/pep-0257/ 55""" 56 57__author__ = "ButenkoMS <gtalk@butenkoms.space>" 58__copyright__ = "Copyright © 2012-2024 ButenkoMS. All rights reserved. Contacts: <gtalk@butenkoms.space>" 59__credits__ = ["ButenkoMS <gtalk@butenkoms.space>", ] 60__license__ = "Apache License, Version 2.0" 61__version__ = "4.4.1" 62__maintainer__ = "ButenkoMS <gtalk@butenkoms.space>" 63__email__ = "gtalk@butenkoms.space" 64# __status__ = "Prototype" 65__status__ = "Development" 66# __status__ = "Production" 67 68 69default_serializable_data_types: Set[Type] = { 70 int, float, complex, str, bytes, bytearray, bool, type(None), list, tuple, set, frozenset, dict 71} 72 73 74known_types: Set[Type] = { 75 int, float, complex, str, bytes, bytearray, bool, type(None), list, tuple, set, frozenset, dict, 76 slice, 77} 78 79 80known_data_types: Set[Type] = { 81 int, float, str, bytes, bytearray, bool, type(None), slice, 82} 83 84 85known_container_types: Set[Type] = { 86 complex, list, tuple, set, frozenset, dict, 87} 88 89 90class DataType(IntEnum): 91 class_ = 0 # type is not in known_types and not in serializable_data_types 92 int_ = 1 93 float_ = 2 94 complex_ = 3 95 str_ = 4 96 bytes_ = 5 97 bytearray_ = 6 98 bool_ = 7 99 none_ = 8 100 list_ = 9 101 tuple_ = 10 102 set_ = 11 103 frozenset_ = 12 104 dict_ = 13 105 slice_ = 14 106 unknown_serializable = 15 # type is in serializable_data_types but not in known_types 107 108 109data_type: Dict[DataType, Type] = { 110 0: object, 111 1: int, 112 2: float, 113 3: complex, 114 4: str, 115 5: bytes, 116 6: bytearray, 117 7: bool, 118 8: type(None), 119 9: list, 120 10: tuple, 121 11: set, 122 12: frozenset, 123 13: dict, 124 14: slice, 125 15: None, 126} 127 128 129data_type_by_type: Dict[Type, DataType] = { 130 object: 0, 131 int: 1, 132 float: 2, 133 complex: 3, 134 str: 4, 135 bytes: 5, 136 bytearray: 6, 137 bool: 7, 138 type(None): 8, 139 list: 9, 140 tuple: 10, 141 set: 11, 142 frozenset: 12, 143 dict: 13, 144 slice: 14, 145 None: 15, 146} 147 148 149class ClassInfoFields(IntEnum): 150 class_id = 0 151 class_name = 1 152 module_importable_str = 2 153 owning_names_path = 3 154 155 156class ObjectInfoFields(IntEnum): 157 object_id = 0 # (type: int) 158 type_id = 1 # (type: DataType) 159 object_ = 2 # (Optional), (type: Any). Link to object itself if `(type(obj) in serializable_data_types)`. Not used otherwise. 160 class_id = 3 # (Optional), (type: int). Used if `type_id == 0` 161 clonable_slots = 4 # (Optional), (type: Tuple[Tuple[str, Any]]). Used if `type_id == 0`. Holds ID's (object_id) of slots objects. 162 clonable_dict_items = 5 # (Optional), (type: Tuple[Tuple[str, Any]]). Used if `type_id == 0`. Holds ID's (object_id) of value objects. 163 # contained_items = 6 # (Optional), (type: Union[Tuple, List, Set, FrozenSet, Dict]). Used if `type_id in {9, 10, 11, 12, 13, 14}`. Holds ID's (object_id) of contained items (for bothe keys and values in the case of Dict). 164 contained_mapping = 6 # (Optional), (type: Union[Tuple, List, Set, FrozenSet, Dict]). Used if `type_id in {9, 10, 11, 12, 13, 14}`. Holds ID's (object_id) of contained items (for bothe keys and values in the case of Dict). 165 contained_sequence = 7 # (Optional), (type: Union[Tuple, List, Set, FrozenSet, Dict]). Used if `type_id in {9, 10, 11, 12, 13, 14}`. Holds ID's (object_id) of contained items (for bothe keys and values in the case of Dict). 166 contained_set = 8 # (Optional), (type: Union[Tuple, List, Set, FrozenSet, Dict]). Used if `type_id in {9, 10, 11, 12, 13, 14}`. Holds ID's (object_id) of contained items (for bothe keys and values in the case of Dict). 167 168 169class CanNotAdjustToSerializableError(Exception): 170 pass 171 172 173class CanNotAdjustFromSerializableError(Exception): 174 pass 175 176 177class RemoteObjectsManager: 178 __slots__ = ( 179 'classes_id_gen', 180 'objects_db', 181 'objects_id_gen', 182 'serializable_data_types', 183 'serializable_any', 184 'serializable_int', 185 'serializable_float', 186 'serializable_complex', 187 'serializable_str', 188 'serializable_bytes', 189 'serializable_bytearray', 190 'serializable_bool', 191 'serializable_none', 192 'serializable_list', 193 'serializable_tuple', 194 'serializable_set', 195 'serializable_frozenset', 196 'serializable_dict', 197 'serializable_slice', 198 'known_classes', 199 'known_classes_by_id', 200 'known_classes_info', 201 'known_classes_info_by_id', 202 'on_new_class_handler', 203 'on_new_obj_info_handler', 204 'objects_ids', 205 'objects_ids_by_id', 206 ) 207 208 def __init__(self, 209 on_new_class_handler: Optional[Callable] = None, 210 on_new_obj_info_handler: Optional[Callable] = None, 211 # classess_db: Optional[Dict[int, Type]] = None, 212 objects_db: Optional[Dict[int, Any]] = None, 213 classes_id_gen: Optional[Callable] = None, 214 objects_id_gen: Optional[Callable] = None, 215 serializable_data_types: Optional[Set[Type]] = None, 216 ) -> None: 217 # self.classess_db: Dict[int, Type] = dict() if classess_db is None else classess_db 218 self.classes_id_gen: IDGenerator = IDGenerator() if classes_id_gen is None else classes_id_gen 219 self.objects_db: Dict[int, Any] = dict() if objects_db is None else objects_db 220 self.objects_id_gen: IDGenerator = IDGenerator() if objects_id_gen is None else objects_id_gen 221 self.serializable_data_types: Set[Type] = default_serializable_data_types if serializable_data_types is None else serializable_data_types 222 self.serializable_any: bool = not self.serializable_data_types 223 self.serializable_int: bool = (int in self.serializable_data_types) or self.serializable_any 224 self.serializable_float: bool = (float in self.serializable_data_types) or self.serializable_any 225 self.serializable_complex: bool = (complex in self.serializable_data_types) or self.serializable_any 226 self.serializable_str: bool = (str in self.serializable_data_types) or self.serializable_any 227 self.serializable_bytes: bool = (bytes in self.serializable_data_types) or self.serializable_any 228 self.serializable_bytearray: bool = (bytearray in self.serializable_data_types) or self.serializable_any 229 self.serializable_bool: bool = (bool in self.serializable_data_types) or self.serializable_any 230 self.serializable_none: bool = (type(None) in self.serializable_data_types) or self.serializable_any 231 self.serializable_list: bool = (list in self.serializable_data_types) or self.serializable_any 232 self.serializable_tuple: bool = (tuple in self.serializable_data_types) or self.serializable_any 233 self.serializable_set: bool = (set in self.serializable_data_types) or self.serializable_any 234 self.serializable_frozenset: bool = (frozenset in self.serializable_data_types) or self.serializable_any 235 self.serializable_dict: bool = (dict in self.serializable_data_types) or self.serializable_any 236 self.serializable_slice: bool = (slice in self.serializable_data_types) or self.serializable_any 237 self.known_classes: Dict[Type, int] = dict() 238 self.known_classes_by_id: Dict[int, Type] = dict() 239 self.known_classes_info: Dict[Tuple, int] = dict() 240 self.known_classes_info_by_id: Dict[int, Tuple] = dict() 241 self.on_new_class_handler: Optional[Callable] = on_new_class_handler 242 self.on_new_obj_info_handler: Optional[Callable] = on_new_obj_info_handler 243 self.objects_ids: Dict[int, int] = dict() # (Key: object_id, Value: id(obj)) 244 self.objects_ids_by_id: Dict[int, int] = dict() # (Key: id(obj), Value: object_id) 245 246 def del_object_by_id(self, id_: int) -> None: 247 if id_ in self.objects_ids_by_id: 248 object_id = self.objects_ids_by_id[id_] 249 self.objects_ids_by_id.pop(id_, None) 250 self.objects_ids.pop(object_id, None) 251 self.objects_db.pop(object_id, None) 252 253 dobi = del_object_by_id 254 255 def del_object_by_object_id(self, object_id: int) -> None: 256 if object_id in self.objects_ids: 257 id_ = self.objects_ids[object_id] 258 self.objects_ids_by_id.pop(id_, None) 259 self.objects_ids.pop(object_id, None) 260 self.objects_db.pop(object_id, None) 261 262 doboi = del_object_by_object_id 263 264 def adjust_to_serializable(self, type_id: DataType, obj: Any) -> Any: 265 if 1 == type_id: 266 if self.serializable_int: 267 return obj 268 elif self.serializable_bytes: 269 return obj.to_bytes(8, 'little') 270 elif self.serializable_bytearray: 271 return bytearray(obj.to_bytes(8, 'little')) 272 elif self.serializable_str: 273 return str(obj) 274 else: 275 pass 276 elif 2 == type_id: 277 if self.serializable_float: 278 return obj 279 elif self.serializable_bytes: 280 return pack('=d', obj) 281 elif self.serializable_bytearray: 282 return bytearray(pack('=d', obj)) 283 elif self.serializable_str: 284 return str(obj) 285 else: 286 pass 287 elif 3 == type_id: 288 if self.serializable_complex: 289 return obj 290 elif self.serializable_bytes: 291 return pack('=dd', obj.real, obj.imag) 292 elif self.serializable_bytearray: 293 return bytearray(pack('=dd', obj.real, obj.imag)) 294 elif self.serializable_str: 295 return str(obj) 296 elif self.serializable_float: 297 if self.serializable_tuple: 298 return (obj.real, obj.imag) 299 elif self.serializable_list: 300 return [obj.real, obj.imag] 301 else: 302 pass 303 else: 304 pass 305 elif 4 == type_id: 306 if self.serializable_str: 307 return obj 308 elif self.serializable_bytes: 309 return obj.encode('utf-8') 310 elif self.serializable_bytearray: 311 return bytearray(obj.encode('utf-8')) 312 elif self.serializable_int: 313 if self.serializable_tuple: 314 return tuple(ord(c) for c in obj) 315 elif self.serializable_list: 316 return [ord(c) for c in obj] 317 else: 318 pass 319 elif self.serializable_float: 320 if self.serializable_tuple: 321 return tuple(float(ord(c)) for c in obj) 322 elif self.serializable_list: 323 return [float(ord(c)) for c in obj] 324 else: 325 pass 326 else: 327 pass 328 elif 5 == type_id: 329 if self.serializable_bytes: 330 return obj 331 elif self.serializable_bytearray: 332 return bytearray(obj) 333 elif self.serializable_str: 334 return obj.hex() 335 elif self.serializable_tuple: 336 if self.serializable_int: 337 return tuple(int(c) for c in obj) 338 if self.serializable_float: 339 return tuple(float(int(c)) for c in obj) 340 else: 341 pass 342 elif self.serializable_list: 343 if self.serializable_int: 344 return [int(c) for c in obj] 345 if self.serializable_float: 346 return [float(int(c)) for c in obj] 347 else: 348 pass 349 else: 350 pass 351 elif 6 == type_id: 352 if self.serializable_bytearray: 353 return obj 354 elif self.serializable_bytes: 355 return bytes(obj) 356 elif self.serializable_str: 357 return obj.hex() 358 elif self.serializable_tuple: 359 if self.serializable_int: 360 return tuple(int(c) for c in obj) 361 if self.serializable_float: 362 return tuple(float(int(c)) for c in obj) 363 else: 364 pass 365 elif self.serializable_list: 366 if self.serializable_int: 367 return [int(c) for c in obj] 368 if self.serializable_float: 369 return [float(int(c)) for c in obj] 370 else: 371 pass 372 else: 373 pass 374 elif 7 == type_id: 375 if self.serializable_bool: 376 return obj 377 elif self.serializable_int: 378 return int(obj) 379 elif self.serializable_bytes: 380 return obj.to_bytes(1, 'little') 381 elif self.serializable_bytearray: 382 return bytearray(obj.to_bytes(1, 'little')) 383 elif self.serializable_str: 384 return str(obj) 385 elif self.serializable_float: 386 return float(int(obj)) 387 else: 388 pass 389 elif 8 == type_id: 390 if self.serializable_none: 391 return obj 392 elif self.serializable_bool: 393 return False 394 elif self.serializable_int: 395 return 0 396 elif self.serializable_bytes: 397 return b'\x00' 398 elif self.serializable_bytearray: 399 return bytearray(b'\x00') 400 elif self.serializable_str: 401 return '' 402 elif self.serializable_float: 403 return 0.0 404 else: 405 pass 406 elif 9 == type_id: 407 if self.serializable_list: 408 return obj 409 elif self.serializable_tuple: 410 return tuple(obj) 411 elif self.serializable_dict: 412 return dict({index: item for index, item in enumerate(obj)}) 413 else: 414 pass 415 elif 10 == type_id: 416 if self.serializable_tuple: 417 return obj 418 elif self.serializable_list: 419 return list(obj) 420 elif self.serializable_dict: 421 return dict({index: item for index, item in enumerate(obj)}) 422 else: 423 pass 424 elif 11 == type_id: 425 if self.serializable_set: 426 return obj 427 elif self.serializable_frozenset: 428 return frozenset(obj) 429 elif self.serializable_tuple: 430 return tuple(obj) 431 elif self.serializable_list: 432 return list(obj) 433 elif self.serializable_dict: 434 return dict({k: None for k in obj}) 435 else: 436 pass 437 elif 12 == type_id: 438 if self.serializable_frozenset: 439 return obj 440 elif self.serializable_set: 441 return set(obj) 442 elif self.serializable_tuple: 443 return tuple(obj) 444 elif self.serializable_list: 445 return list(obj) 446 elif self.serializable_dict: 447 return dict({k: None for k in obj}) 448 else: 449 pass 450 elif 13 == type_id: 451 if self.serializable_dict: 452 return obj 453 elif self.serializable_tuple: 454 return tuple(obj.items()) 455 elif self.serializable_list: 456 return list(list(pair) for pair in obj.items()) 457 else: 458 pass 459 elif 14 == type_id: 460 if self.serializable_slice: 461 return obj 462 elif self.serializable_tuple: 463 return (obj.start, obj.stop, obj.step) 464 elif self.serializable_list: 465 return [obj.start, obj.stop, obj.step] 466 elif self.serializable_dict: 467 return {0: obj.start, 1: obj.stop, 2: obj.step} 468 else: 469 pass 470 else: 471 raise RuntimeError('Unknown type_id') 472 473 raise CanNotAdjustToSerializableError(f'Can not adjust to serializable. Type: {type_id}, obj: {obj}') 474 475 ats = adjust_to_serializable 476 477 def adjust_from_serializable(self, type_id: DataType, obj: Any) -> Any: 478 if 1 == type_id: 479 if self.serializable_int: 480 return obj 481 elif self.serializable_bytes: 482 return int.from_bytes(obj, 'little') 483 elif self.serializable_bytearray: 484 return int.from_bytes(bytes(obj), 'little') 485 elif self.serializable_str: 486 return int(obj) 487 else: 488 pass 489 elif 2 == type_id: 490 if self.serializable_float: 491 return obj 492 elif self.serializable_bytes: 493 return unpack('=d', obj)[0] 494 elif self.serializable_bytearray: 495 return unpack('=d', bytes(obj))[0] 496 elif self.serializable_str: 497 return float(obj) 498 else: 499 pass 500 elif 3 == type_id: 501 if self.serializable_complex: 502 return obj 503 elif self.serializable_bytes: 504 return complex(*unpack('=dd', obj)) 505 elif self.serializable_bytearray: 506 return complex(*unpack('=dd', bytes(obj))) 507 elif self.serializable_str: 508 return complex(*obj) 509 elif self.serializable_float: 510 if self.serializable_tuple: 511 return complex(*obj) 512 elif self.serializable_list: 513 return complex(*obj) 514 else: 515 pass 516 else: 517 pass 518 elif 4 == type_id: 519 if self.serializable_str: 520 return obj 521 elif self.serializable_bytes: 522 return obj.decode('utf-8') 523 elif self.serializable_bytearray: 524 return bytes(obj).decode('utf-8') 525 elif self.serializable_int: 526 if self.serializable_tuple: 527 return ''.join(chr(c) for c in obj) 528 elif self.serializable_list: 529 return ''.join(chr(c) for c in obj) 530 else: 531 pass 532 elif self.serializable_float: 533 if self.serializable_tuple: 534 return ''.join(chr(int(round(c))) for c in obj) 535 elif self.serializable_list: 536 return ''.join(chr(int(round(c))) for c in obj) 537 else: 538 pass 539 else: 540 pass 541 elif 5 == type_id: 542 if self.serializable_bytes: 543 return obj 544 elif self.serializable_bytearray: 545 return bytes(obj) 546 elif self.serializable_str: 547 return bytes.fromhex(obj) 548 elif self.serializable_tuple: 549 if self.serializable_int: 550 return b''.join(item.to_bytes(1, 'little') for item in obj) 551 if self.serializable_float: 552 return b''.join((int(round(item))).to_bytes(1, 'little') for item in obj) 553 else: 554 pass 555 elif self.serializable_list: 556 if self.serializable_int: 557 return b''.join(item.to_bytes(1, 'little') for item in obj) 558 if self.serializable_float: 559 return b''.join((int(round(item))).to_bytes(1, 'little') for item in obj) 560 else: 561 pass 562 else: 563 pass 564 elif 6 == type_id: 565 if self.serializable_bytearray: 566 return obj 567 elif self.serializable_bytes: 568 return bytearray(obj) 569 elif self.serializable_str: 570 return bytearray(bytes.fromhex(obj)) 571 elif self.serializable_tuple: 572 if self.serializable_int: 573 return bytearray(b''.join(item.to_bytes(1, 'little') for item in obj)) 574 if self.serializable_float: 575 return bytearray(b''.join((int(round(item))).to_bytes(1, 'little') for item in obj)) 576 else: 577 pass 578 elif self.serializable_list: 579 if self.serializable_int: 580 return bytearray(b''.join(item.to_bytes(1, 'little') for item in obj)) 581 if self.serializable_float: 582 return bytearray(b''.join((int(round(item))).to_bytes(1, 'little') for item in obj)) 583 else: 584 pass 585 else: 586 pass 587 elif 7 == type_id: 588 if self.serializable_bool: 589 return obj 590 elif self.serializable_int: 591 return bool(obj) 592 elif self.serializable_bytes: 593 return bool(int.from_bytes(obj, 'little')) 594 elif self.serializable_bytearray: 595 return bool(int.from_bytes(bytes(obj), 'little')) 596 elif self.serializable_str: 597 return bool(obj) 598 elif self.serializable_float: 599 return bool(int(round(obj))) 600 else: 601 pass 602 elif 8 == type_id: 603 return None 604 elif 9 == type_id: 605 if self.serializable_list: 606 return obj 607 elif self.serializable_tuple: 608 return list(obj) 609 elif self.serializable_dict: 610 return [value for key, value in sorted(obj.items(), key=lambda x: x[0])] 611 else: 612 pass 613 elif 10 == type_id: 614 if self.serializable_tuple: 615 return obj 616 elif self.serializable_list: 617 return tuple(obj) 618 elif self.serializable_dict: 619 return tuple(value for key, value in sorted(obj.items(), key=lambda x: x[0])) 620 else: 621 pass 622 elif 11 == type_id: 623 if self.serializable_set: 624 return obj 625 elif self.serializable_frozenset: 626 return set(obj) 627 elif self.serializable_tuple: 628 return set(obj) 629 elif self.serializable_list: 630 return set(obj) 631 elif self.serializable_dict: 632 return set(obj.keys()) 633 else: 634 pass 635 elif 12 == type_id: 636 if self.serializable_frozenset: 637 return obj 638 elif self.serializable_set: 639 return frozenset(obj) 640 elif self.serializable_tuple: 641 return frozenset(obj) 642 elif self.serializable_list: 643 return frozenset(obj) 644 elif self.serializable_dict: 645 return frozenset(obj.keys()) 646 else: 647 pass 648 elif 13 == type_id: 649 if self.serializable_dict: 650 return obj 651 elif self.serializable_tuple: 652 return dict(obj) 653 elif self.serializable_list: 654 return dict(obj) 655 else: 656 pass 657 elif 14 == type_id: 658 if self.serializable_slice: 659 return obj 660 elif self.serializable_tuple: 661 return slice(*obj) 662 elif self.serializable_list: 663 return slice(*obj) 664 elif self.serializable_dict: 665 return slice(obj[0], obj[1], obj[2]) 666 else: 667 pass 668 else: 669 raise RuntimeError('Unknown type_id') 670 671 raise CanNotAdjustFromSerializableError(f'Can not adjust from serializable. Type: {type_id}, obj: {obj}') 672 673 afs = adjust_from_serializable 674 675 # def is_replicatable_object_attribute(self, attribute: Any) -> bool: 676 # if is_setable_data_descriptor(attribute): 677 # data = attribute.__get__(None, None) 678 # elif is_callable(attribute): 679 # return False 680 # else: 681 # return True 682 683 def serialize_container(self, type_id: DataType, obj: Any) -> Any: 684 if 9 == type_id: 685 new_obj = list() 686 for item in obj: 687 exists, value = self.serialize_impl(item) 688 if exists: 689 new_obj.append(value) 690 elif 10 == type_id: 691 new_obj = list() 692 for item in obj: 693 exists, value = self.serialize_impl(item) 694 if exists: 695 new_obj.append(value) 696 697 new_obj = tuple(new_obj) 698 elif 11 == type_id: 699 new_obj = set() 700 for item in obj: 701 exists, value = self.serialize_impl(item) 702 if exists: 703 new_obj.add(value) 704 elif 12 == type_id: 705 new_obj = set() 706 for item in obj: 707 exists, value = self.serialize_impl(item) 708 if exists: 709 new_obj.add(value) 710 711 new_obj = frozenset(new_obj) 712 elif 13 == type_id: 713 new_obj = dict() 714 for key, value in obj.items(): 715 key_exists, key_value = self.serialize_impl(key) 716 value_exists, value_value = self.serialize_impl(value) 717 if key_exists and value_exists: 718 new_obj[key_value] = value_value 719 720 return new_obj 721 722 sc = serialize_container 723 724 def serialize_impl(self, obj: Any, ignore_empty_classes: bool = False) -> Tuple[bool, int]: 725 result_exists: bool = True 726 id_: int = id(obj) 727 obj_type = type(obj) 728 if (int != obj_type) and (id_ in self.objects_ids_by_id): 729 new_object: bool = False 730 object_id: int = self.objects_ids_by_id[id_] 731 else: 732 # int object must always produce new object_id because first 256 ints are persistent across Python sessions and 733 # this can cause issues within users of current module. For example within `cengal/hardware/memory/shared_memory` 734 # which changes int values inline instead of producing new objects. 735 new_object = True 736 object_id = self.objects_id_gen() 737 self.objects_ids[object_id] = id_ 738 self.objects_db[object_id] = obj 739 740 if not new_object: 741 return ResultExistence[int](result_exists, object_id) 742 743 if self.serializable_any or (obj_type in self.serializable_data_types): 744 serializable: bool = True 745 type_id: int = data_type_by_type.get(obj_type, 15) 746 else: 747 serializable = False 748 type_id = data_type_by_type.get(obj_type, 0) 749 750 if obj_type in known_container_types: 751 known_container: bool = True 752 else: 753 known_container = False 754 755 class_info: Dict[int, Any] = None 756 known_data: bool = None 757 class_id: int = None 758 is_new_class: bool = None 759 class_name: str = None 760 new_obj_slots: List[Tuple[str, Any]] = None 761 new_obj_dict: Dict[str, Any] = None 762 obj_mapping: Dict = None 763 obj_sequence: List = None 764 obj_set: Set = None 765 if serializable: 766 if known_container: 767 object_info = ( 768 self.ats(1, object_id), 769 self.ats(1, type_id), 770 self.sc(type_id, obj), 771 ) 772 else: 773 object_info = ( 774 self.ats(1, object_id), 775 self.ats(1, type_id), 776 obj, 777 ) 778 else: 779 if obj_type in known_data_types: 780 known_data = True 781 else: 782 known_data = False 783 784 if 0 == type_id: 785 new_obj_slots = list() 786 for slot_name, slot_value in filled_slot_names_with_values_gen(obj): 787 # if not hasattr(slot_value, '__set__'): 788 # # if not setable descriptor 789 # continue 790 791 if self.serializable_str: 792 adjusted_slot_name = slot_name 793 else: 794 adjusted_slot_name = self.ats(4, slot_name) 795 796 item_exists, item_value = self.serialize_impl(slot_value) 797 if item_exists: 798 if self.serializable_tuple: 799 new_obj_slots.append((adjusted_slot_name, item_value)) 800 elif self.serializable_list: 801 new_obj_slots.append([adjusted_slot_name, item_value]) 802 else: 803 new_obj_slots.append(self.ats(10, (adjusted_slot_name, item_value))) 804 805 if new_obj_slots: 806 if self.serializable_list: 807 pass 808 elif self.serializable_tuple: 809 new_obj_slots = tuple(new_obj_slots) 810 else: 811 new_obj_slots = self.ats(9, new_obj_slots) 812 813 new_obj_dict = dict() 814 if hasattr(obj, '__dict__'): 815 for key, value in obj.__dict__.items(): 816 # raw_value = getattr_static(obj, key) 817 # if hasattr(raw_value, '__get__') and (not hasattr(raw_value, '__set__')): 818 # # if not setable descriptor 819 # continue 820 821 if self.serializable_str: 822 adjusted_key = key 823 else: 824 adjusted_key = self.ats(4, key) 825 826 value_exists, value_value = self.serialize_impl(value) 827 if value_exists: 828 new_obj_dict[adjusted_key] = value_value 829 else: 830 continue 831 832 if new_obj_dict: 833 if self.serializable_dict: 834 pass 835 else: 836 new_obj_dict = self.ats(13, new_obj_dict) 837 838 obj_mapping = dict() 839 obj_sequence = list() 840 obj_set = set() 841 if isinstance(obj, MutableMapping): 842 for key, value in obj.items(): 843 key_exists, key_value = self.serialize_impl(value) 844 if not key_exists: 845 continue 846 847 value_exists, value_value = self.serialize_impl(value) 848 if value_exists: 849 obj_mapping[key_value] = value_value 850 else: 851 continue 852 853 if obj_mapping: 854 if self.serializable_dict: 855 pass 856 else: 857 obj_mapping = self.ats(13, obj_mapping) 858 elif isinstance(obj, MutableSequence): 859 for item in obj: 860 item_exists, item_value = self.serialize_impl(item) 861 if item_exists: 862 obj_sequence.append(item_value) 863 else: 864 continue 865 866 if obj_sequence: 867 if self.serializable_list: 868 pass 869 else: 870 obj_sequence = self.ats(9, obj_sequence) 871 elif isinstance(obj, MutableSet): 872 for item in obj: 873 item_exists, item_value = self.serialize_impl(item) 874 if item_exists: 875 obj_set.add(item_value) 876 else: 877 continue 878 879 if obj_set: 880 if self.serializable_set: 881 pass 882 else: 883 obj_set = self.ats(11, obj_set) 884 else: 885 pass 886 887 if ignore_empty_classes: 888 result_exists = new_obj_slots or new_obj_dict or obj_mapping or obj_sequence or obj_set 889 890 if obj_type in self.known_classes: 891 is_new_class = False 892 class_id = self.known_classes[obj_type] 893 else: 894 is_new_class = True 895 class_name = obj_type.__name__ 896 class_id = self.classes_id_gen() 897 self.known_classes[obj_type] = class_id 898 self.known_classes_by_id[class_id] = obj_type 899 module_importable_str, owning_names_path = entity_module_importable_str_and_owning_names_path(obj) 900 adjusted_owning_names_path: List[str] = [self.ats(4, name) for name in owning_names_path] 901 module_importable_str_and_owning_names_path = (class_name, module_importable_str, tuple(owning_names_path)) 902 self.known_classes_info[module_importable_str_and_owning_names_path] = class_id 903 self.known_classes_info_by_id[class_id] = module_importable_str_and_owning_names_path 904 905 if result_exists: 906 class_info = ( 907 self.ats(1, class_id), 908 self.ats(4, class_name), 909 self.ats(4, module_importable_str), 910 self.ats(9, adjusted_owning_names_path), 911 ) 912 913 if self.serializable_tuple: 914 adjusted_class_info = class_info 915 else: 916 adjusted_class_info = self.ats(10, class_info) 917 918 if self.on_new_class_handler: 919 self.on_new_class_handler( 920 adjusted_class_info, 921 obj_type, 922 class_id, 923 class_name, 924 module_importable_str, 925 owning_names_path, 926 module_importable_str_and_owning_names_path 927 ) 928 929 # will be later serialized much faster than without `if else None` 930 object_info = ( 931 self.ats(1, object_id), 932 self.ats(1, type_id), 933 self.ats(8, None), 934 self.ats(1, class_id), 935 new_obj_slots if new_obj_slots else self.ats(9, list()), 936 new_obj_dict if new_obj_dict else self.ats(13, dict()), 937 obj_mapping if obj_mapping else self.ats(13, dict()), 938 obj_sequence if obj_sequence else self.ats(9, list()), 939 obj_set if obj_set else self.ats(11, set()), 940 ) 941 elif known_data: 942 object_info = ( 943 self.ats(1, object_id), 944 self.ats(1, type_id), 945 self.ats(type_id, obj), 946 ) 947 elif known_container: 948 object_info = ( 949 self.ats(1, object_id), 950 self.ats(1, type_id), 951 self.ats(type_id, self.sc(type_id, obj)), 952 ) 953 else: 954 raise RuntimeError('Unknown type_id') 955 956 if self.serializable_tuple: 957 adjusted_object_info = object_info 958 else: 959 adjusted_object_info = self.ats(10, object_info) 960 961 if result_exists: 962 if self.on_new_obj_info_handler: 963 self.on_new_obj_info_handler( 964 adjusted_object_info, 965 obj, 966 object_id, 967 type_id, 968 serializable, 969 known_container, 970 known_data, 971 class_id, 972 is_new_class, 973 new_obj_slots, 974 new_obj_dict, 975 obj_mapping, 976 obj_sequence, 977 obj_set, 978 ) 979 980 return result_exists, object_id 981 982 si = serialize_impl 983 984 def serialize(self, obj: Any, ignore_empty_classes: bool = False) -> Tuple[bool, int]: 985 with DisableGC(): 986 return self.serialize_impl(obj, ignore_empty_classes) 987 988 s = serialize 989 990 def deserialize_class_impl(self, class_info: Tuple) -> Tuple[bool, int, Type]: 991 if self.serializable_tuple: 992 pass 993 else: 994 class_info = self.afs(10, class_info) 995 996 class_id: int = self.afs(1, class_info[0]) 997 if class_id in self.known_classes_by_id: 998 return ResultExistence[Tuple[int, Type]](True, (class_id, self.known_classes_by_id[class_id])) 999 else: 1000 class_name: str = self.afs(4, class_info[1]) 1001 module_importable_str: str = self.afs(4, class_info[2]) 1002 owning_names_path: List[str] = self.afs(9, class_info[3]) 1003 adjusted_owning_names_path: List[str] = [self.afs(4, name) for name in owning_names_path] 1004 obj_class: Type = entity_by_name_module_importable_str_and_owning_names_path(class_name, module_importable_str, adjusted_owning_names_path) 1005 self.known_classes_by_id[class_id] = obj_class 1006 self.known_classes[obj_class] = class_id 1007 class_info_tuple: Tuple = (class_name, module_importable_str, tuple(owning_names_path)) 1008 self.known_classes_info[class_info_tuple] = class_id 1009 self.known_classes_info_by_id[class_id] = class_info_tuple 1010 return True, class_id, obj_class 1011 1012 dcli = deserialize_class_impl 1013 1014 def deserialize_class(self, class_info: Tuple) -> Tuple[bool, int, Type]: 1015 with DisableGC(): 1016 return self.deserialize_class_impl(class_info) 1017 1018 dcl = deserialize_class 1019 1020 def deserialize_container_impl(self, type_id: DataType, obj: Any) -> Any: 1021 new_obj = obj 1022 if 9 == type_id: 1023 new_obj = list() 1024 for item_id in obj: 1025 new_obj.append(self.objects_db[self.afs(1, item_id)]) 1026 elif 10 == type_id: 1027 new_obj = list() 1028 for item_id in obj: 1029 new_obj.append(self.objects_db[self.afs(1, item_id)]) 1030 1031 new_obj = tuple(new_obj) 1032 elif 11 == type_id: 1033 new_obj = set() 1034 for item_id in obj: 1035 new_obj.add(self.objects_db[self.afs(1, item_id)]) 1036 elif 12 == type_id: 1037 new_obj = set() 1038 for item_id in obj: 1039 new_obj.add(self.objects_db[self.afs(1, item_id)]) 1040 1041 new_obj = frozenset(new_obj) 1042 elif 13 == type_id: 1043 new_obj = dict() 1044 for key_id, value_id in obj.items(): 1045 new_obj[self.objects_db[int(self.afs(1, key_id))]] = self.objects_db[self.afs(1, value_id)] 1046 1047 return new_obj 1048 1049 dcoi = deserialize_container_impl 1050 1051 def deserialize_container(self, type_id: DataType, obj: Any) -> Any: 1052 with DisableGC(): 1053 return self.deserialize_container_impl(type_id, obj) 1054 1055 dco = deserialize_container 1056 1057 def deserialize_obj_impl(self, obj_info: Tuple) -> Tuple[bool, int, Any]: 1058 if self.serializable_tuple: 1059 pass 1060 else: 1061 obj_info = self.afs(10, obj_info) 1062 1063 object_id: int = self.afs(1, obj_info[0]) 1064 if object_id in self.objects_db: 1065 return ResultExistence[Tuple[int, Any]](True, (object_id, self.objects_db[object_id])) 1066 1067 type_id: int = self.afs(1, obj_info[1]) 1068 1069 obj_type = data_type[type_id] 1070 serializable: bool = self.serializable_any or (obj_type in self.serializable_data_types) 1071 known_container: bool = obj_type in known_container_types 1072 known_data: bool = None 1073 1074 if serializable: 1075 if known_container: 1076 obj = self.dcoi(type_id, obj_info[2]) 1077 else: 1078 obj = obj_info[2] 1079 else: 1080 known_data = obj_type in known_data_types 1081 1082 if 0 == type_id: 1083 class_id: int = self.afs(1, obj_info[3]) 1084 obj_class: Type = self.known_classes_by_id[class_id] 1085 obj: Any = obj_class.__new__(obj_class) 1086 1087 if obj_info[4]: 1088 clonable_slots = self.afs(9, obj_info[4]) 1089 for slot_name, slot_id in clonable_slots: 1090 slot_name = self.afs(4, slot_name) 1091 slot_id = self.afs(1, slot_id) 1092 child_obj = self.objects_db[slot_id] 1093 setattr(obj, slot_name, child_obj) 1094 1095 if obj_info[5]: 1096 clonable_dict_items = self.afs(13, obj_info[5]) 1097 for key, value_id in clonable_dict_items.items(): 1098 key = self.afs(4, key) 1099 value_id = self.afs(1, value_id) 1100 child_obj = self.objects_db[value_id] 1101 setattr(obj, key, child_obj) 1102 1103 if obj_info[6]: 1104 contained_mapping = self.afs(13, obj_info[6]) 1105 for key_id, value_id in contained_mapping.items(): 1106 key_id = self.afs(1, key_id) 1107 value_id = self.afs(1, value_id) 1108 child_key = self.objects_db[int(key_id)] 1109 child_value = self.objects_db[value_id] 1110 obj[child_key] = child_value 1111 1112 if obj_info[7]: 1113 contained_sequence = self.afs(9, obj_info[7]) 1114 for item_id in contained_sequence: 1115 item_id = self.afs(1, item_id) 1116 child_item = self.objects_db[item_id] 1117 obj.append(child_item) 1118 1119 if obj_info[8]: 1120 contained_set = self.afs(11, obj_info[8]) 1121 for item_id in contained_set: 1122 item_id = self.afs(1, item_id) 1123 child_item = self.objects_db[item_id] 1124 obj.add(child_item) 1125 elif known_data: 1126 obj = self.afs(type_id, obj_info[2]) 1127 elif known_container: 1128 obj = self.dcoi(type_id, self.afs(type_id, obj_info[2])) 1129 else: 1130 raise RuntimeError('Unknown type_id') 1131 1132 self.objects_db[object_id] = obj 1133 id_: int = id(obj) 1134 self.objects_ids[object_id] = id_ 1135 self.objects_ids_by_id[id_] = object_id 1136 return True, object_id, obj 1137 1138 doi = deserialize_obj_impl 1139 1140 def deserialize_obj(self, obj_info: Tuple) -> Tuple[bool, int, Any]: 1141 with DisableGC(): 1142 return self.deserialize_obj_impl(obj_info) 1143 1144 do = deserialize_obj
default_serializable_data_types: Set[Type] =
{<class 'int'>, <class 'complex'>, <class 'NoneType'>, <class 'list'>, <class 'frozenset'>, <class 'bytes'>, <class 'str'>, <class 'tuple'>, <class 'dict'>, <class 'float'>, <class 'set'>, <class 'bool'>, <class 'bytearray'>}
known_types: Set[Type] =
{<class 'int'>, <class 'complex'>, <class 'NoneType'>, <class 'list'>, <class 'frozenset'>, <class 'bytes'>, <class 'str'>, <class 'tuple'>, <class 'dict'>, <class 'float'>, <class 'set'>, <class 'slice'>, <class 'bool'>, <class 'bytearray'>}
known_data_types: Set[Type] =
{<class 'int'>, <class 'NoneType'>, <class 'bytes'>, <class 'str'>, <class 'float'>, <class 'slice'>, <class 'bool'>, <class 'bytearray'>}
known_container_types: Set[Type] =
{<class 'complex'>, <class 'list'>, <class 'tuple'>, <class 'dict'>, <class 'set'>, <class 'frozenset'>}
class
DataType(enum.IntEnum):
91class DataType(IntEnum): 92 class_ = 0 # type is not in known_types and not in serializable_data_types 93 int_ = 1 94 float_ = 2 95 complex_ = 3 96 str_ = 4 97 bytes_ = 5 98 bytearray_ = 6 99 bool_ = 7 100 none_ = 8 101 list_ = 9 102 tuple_ = 10 103 set_ = 11 104 frozenset_ = 12 105 dict_ = 13 106 slice_ = 14 107 unknown_serializable = 15 # type is in serializable_data_types but not in known_types
An enumeration.
class_ =
<DataType.class_: 0>
int_ =
<DataType.int_: 1>
float_ =
<DataType.float_: 2>
complex_ =
<DataType.complex_: 3>
str_ =
<DataType.str_: 4>
bytes_ =
<DataType.bytes_: 5>
bytearray_ =
<DataType.bytearray_: 6>
bool_ =
<DataType.bool_: 7>
none_ =
<DataType.none_: 8>
list_ =
<DataType.list_: 9>
tuple_ =
<DataType.tuple_: 10>
set_ =
<DataType.set_: 11>
frozenset_ =
<DataType.frozenset_: 12>
dict_ =
<DataType.dict_: 13>
slice_ =
<DataType.slice_: 14>
unknown_serializable =
<DataType.unknown_serializable: 15>
Inherited Members
- enum.Enum
- name
- value
- builtins.int
- conjugate
- bit_length
- to_bytes
- from_bytes
- as_integer_ratio
- real
- imag
- numerator
- denominator
data_type: Dict[DataType, Type] =
{0: <class 'object'>, 1: <class 'int'>, 2: <class 'float'>, 3: <class 'complex'>, 4: <class 'str'>, 5: <class 'bytes'>, 6: <class 'bytearray'>, 7: <class 'bool'>, 8: <class 'NoneType'>, 9: <class 'list'>, 10: <class 'tuple'>, 11: <class 'set'>, 12: <class 'frozenset'>, 13: <class 'dict'>, 14: <class 'slice'>, 15: None}
data_type_by_type: Dict[Type, DataType] =
{<class 'object'>: 0, <class 'int'>: 1, <class 'float'>: 2, <class 'complex'>: 3, <class 'str'>: 4, <class 'bytes'>: 5, <class 'bytearray'>: 6, <class 'bool'>: 7, <class 'NoneType'>: 8, <class 'list'>: 9, <class 'tuple'>: 10, <class 'set'>: 11, <class 'frozenset'>: 12, <class 'dict'>: 13, <class 'slice'>: 14, None: 15}
class
ClassInfoFields(enum.IntEnum):
150class ClassInfoFields(IntEnum): 151 class_id = 0 152 class_name = 1 153 module_importable_str = 2 154 owning_names_path = 3
An enumeration.
class_id =
<ClassInfoFields.class_id: 0>
class_name =
<ClassInfoFields.class_name: 1>
module_importable_str =
<ClassInfoFields.module_importable_str: 2>
owning_names_path =
<ClassInfoFields.owning_names_path: 3>
Inherited Members
- enum.Enum
- name
- value
- builtins.int
- conjugate
- bit_length
- to_bytes
- from_bytes
- as_integer_ratio
- real
- imag
- numerator
- denominator
class
ObjectInfoFields(enum.IntEnum):
157class ObjectInfoFields(IntEnum): 158 object_id = 0 # (type: int) 159 type_id = 1 # (type: DataType) 160 object_ = 2 # (Optional), (type: Any). Link to object itself if `(type(obj) in serializable_data_types)`. Not used otherwise. 161 class_id = 3 # (Optional), (type: int). Used if `type_id == 0` 162 clonable_slots = 4 # (Optional), (type: Tuple[Tuple[str, Any]]). Used if `type_id == 0`. Holds ID's (object_id) of slots objects. 163 clonable_dict_items = 5 # (Optional), (type: Tuple[Tuple[str, Any]]). Used if `type_id == 0`. Holds ID's (object_id) of value objects. 164 # contained_items = 6 # (Optional), (type: Union[Tuple, List, Set, FrozenSet, Dict]). Used if `type_id in {9, 10, 11, 12, 13, 14}`. Holds ID's (object_id) of contained items (for bothe keys and values in the case of Dict). 165 contained_mapping = 6 # (Optional), (type: Union[Tuple, List, Set, FrozenSet, Dict]). Used if `type_id in {9, 10, 11, 12, 13, 14}`. Holds ID's (object_id) of contained items (for bothe keys and values in the case of Dict). 166 contained_sequence = 7 # (Optional), (type: Union[Tuple, List, Set, FrozenSet, Dict]). Used if `type_id in {9, 10, 11, 12, 13, 14}`. Holds ID's (object_id) of contained items (for bothe keys and values in the case of Dict). 167 contained_set = 8 # (Optional), (type: Union[Tuple, List, Set, FrozenSet, Dict]). Used if `type_id in {9, 10, 11, 12, 13, 14}`. Holds ID's (object_id) of contained items (for bothe keys and values in the case of Dict).
An enumeration.
object_id =
<ObjectInfoFields.object_id: 0>
type_id =
<ObjectInfoFields.type_id: 1>
object_ =
<ObjectInfoFields.object_: 2>
class_id =
<ObjectInfoFields.class_id: 3>
clonable_slots =
<ObjectInfoFields.clonable_slots: 4>
clonable_dict_items =
<ObjectInfoFields.clonable_dict_items: 5>
contained_mapping =
<ObjectInfoFields.contained_mapping: 6>
contained_sequence =
<ObjectInfoFields.contained_sequence: 7>
contained_set =
<ObjectInfoFields.contained_set: 8>
Inherited Members
- enum.Enum
- name
- value
- builtins.int
- conjugate
- bit_length
- to_bytes
- from_bytes
- as_integer_ratio
- real
- imag
- numerator
- denominator
class
CanNotAdjustToSerializableError(builtins.Exception):
Common base class for all non-exit exceptions.
Inherited Members
- builtins.Exception
- Exception
- builtins.BaseException
- with_traceback
- args
class
CanNotAdjustFromSerializableError(builtins.Exception):
Common base class for all non-exit exceptions.
Inherited Members
- builtins.Exception
- Exception
- builtins.BaseException
- with_traceback
- args
class
RemoteObjectsManager:
178class RemoteObjectsManager: 179 __slots__ = ( 180 'classes_id_gen', 181 'objects_db', 182 'objects_id_gen', 183 'serializable_data_types', 184 'serializable_any', 185 'serializable_int', 186 'serializable_float', 187 'serializable_complex', 188 'serializable_str', 189 'serializable_bytes', 190 'serializable_bytearray', 191 'serializable_bool', 192 'serializable_none', 193 'serializable_list', 194 'serializable_tuple', 195 'serializable_set', 196 'serializable_frozenset', 197 'serializable_dict', 198 'serializable_slice', 199 'known_classes', 200 'known_classes_by_id', 201 'known_classes_info', 202 'known_classes_info_by_id', 203 'on_new_class_handler', 204 'on_new_obj_info_handler', 205 'objects_ids', 206 'objects_ids_by_id', 207 ) 208 209 def __init__(self, 210 on_new_class_handler: Optional[Callable] = None, 211 on_new_obj_info_handler: Optional[Callable] = None, 212 # classess_db: Optional[Dict[int, Type]] = None, 213 objects_db: Optional[Dict[int, Any]] = None, 214 classes_id_gen: Optional[Callable] = None, 215 objects_id_gen: Optional[Callable] = None, 216 serializable_data_types: Optional[Set[Type]] = None, 217 ) -> None: 218 # self.classess_db: Dict[int, Type] = dict() if classess_db is None else classess_db 219 self.classes_id_gen: IDGenerator = IDGenerator() if classes_id_gen is None else classes_id_gen 220 self.objects_db: Dict[int, Any] = dict() if objects_db is None else objects_db 221 self.objects_id_gen: IDGenerator = IDGenerator() if objects_id_gen is None else objects_id_gen 222 self.serializable_data_types: Set[Type] = default_serializable_data_types if serializable_data_types is None else serializable_data_types 223 self.serializable_any: bool = not self.serializable_data_types 224 self.serializable_int: bool = (int in self.serializable_data_types) or self.serializable_any 225 self.serializable_float: bool = (float in self.serializable_data_types) or self.serializable_any 226 self.serializable_complex: bool = (complex in self.serializable_data_types) or self.serializable_any 227 self.serializable_str: bool = (str in self.serializable_data_types) or self.serializable_any 228 self.serializable_bytes: bool = (bytes in self.serializable_data_types) or self.serializable_any 229 self.serializable_bytearray: bool = (bytearray in self.serializable_data_types) or self.serializable_any 230 self.serializable_bool: bool = (bool in self.serializable_data_types) or self.serializable_any 231 self.serializable_none: bool = (type(None) in self.serializable_data_types) or self.serializable_any 232 self.serializable_list: bool = (list in self.serializable_data_types) or self.serializable_any 233 self.serializable_tuple: bool = (tuple in self.serializable_data_types) or self.serializable_any 234 self.serializable_set: bool = (set in self.serializable_data_types) or self.serializable_any 235 self.serializable_frozenset: bool = (frozenset in self.serializable_data_types) or self.serializable_any 236 self.serializable_dict: bool = (dict in self.serializable_data_types) or self.serializable_any 237 self.serializable_slice: bool = (slice in self.serializable_data_types) or self.serializable_any 238 self.known_classes: Dict[Type, int] = dict() 239 self.known_classes_by_id: Dict[int, Type] = dict() 240 self.known_classes_info: Dict[Tuple, int] = dict() 241 self.known_classes_info_by_id: Dict[int, Tuple] = dict() 242 self.on_new_class_handler: Optional[Callable] = on_new_class_handler 243 self.on_new_obj_info_handler: Optional[Callable] = on_new_obj_info_handler 244 self.objects_ids: Dict[int, int] = dict() # (Key: object_id, Value: id(obj)) 245 self.objects_ids_by_id: Dict[int, int] = dict() # (Key: id(obj), Value: object_id) 246 247 def del_object_by_id(self, id_: int) -> None: 248 if id_ in self.objects_ids_by_id: 249 object_id = self.objects_ids_by_id[id_] 250 self.objects_ids_by_id.pop(id_, None) 251 self.objects_ids.pop(object_id, None) 252 self.objects_db.pop(object_id, None) 253 254 dobi = del_object_by_id 255 256 def del_object_by_object_id(self, object_id: int) -> None: 257 if object_id in self.objects_ids: 258 id_ = self.objects_ids[object_id] 259 self.objects_ids_by_id.pop(id_, None) 260 self.objects_ids.pop(object_id, None) 261 self.objects_db.pop(object_id, None) 262 263 doboi = del_object_by_object_id 264 265 def adjust_to_serializable(self, type_id: DataType, obj: Any) -> Any: 266 if 1 == type_id: 267 if self.serializable_int: 268 return obj 269 elif self.serializable_bytes: 270 return obj.to_bytes(8, 'little') 271 elif self.serializable_bytearray: 272 return bytearray(obj.to_bytes(8, 'little')) 273 elif self.serializable_str: 274 return str(obj) 275 else: 276 pass 277 elif 2 == type_id: 278 if self.serializable_float: 279 return obj 280 elif self.serializable_bytes: 281 return pack('=d', obj) 282 elif self.serializable_bytearray: 283 return bytearray(pack('=d', obj)) 284 elif self.serializable_str: 285 return str(obj) 286 else: 287 pass 288 elif 3 == type_id: 289 if self.serializable_complex: 290 return obj 291 elif self.serializable_bytes: 292 return pack('=dd', obj.real, obj.imag) 293 elif self.serializable_bytearray: 294 return bytearray(pack('=dd', obj.real, obj.imag)) 295 elif self.serializable_str: 296 return str(obj) 297 elif self.serializable_float: 298 if self.serializable_tuple: 299 return (obj.real, obj.imag) 300 elif self.serializable_list: 301 return [obj.real, obj.imag] 302 else: 303 pass 304 else: 305 pass 306 elif 4 == type_id: 307 if self.serializable_str: 308 return obj 309 elif self.serializable_bytes: 310 return obj.encode('utf-8') 311 elif self.serializable_bytearray: 312 return bytearray(obj.encode('utf-8')) 313 elif self.serializable_int: 314 if self.serializable_tuple: 315 return tuple(ord(c) for c in obj) 316 elif self.serializable_list: 317 return [ord(c) for c in obj] 318 else: 319 pass 320 elif self.serializable_float: 321 if self.serializable_tuple: 322 return tuple(float(ord(c)) for c in obj) 323 elif self.serializable_list: 324 return [float(ord(c)) for c in obj] 325 else: 326 pass 327 else: 328 pass 329 elif 5 == type_id: 330 if self.serializable_bytes: 331 return obj 332 elif self.serializable_bytearray: 333 return bytearray(obj) 334 elif self.serializable_str: 335 return obj.hex() 336 elif self.serializable_tuple: 337 if self.serializable_int: 338 return tuple(int(c) for c in obj) 339 if self.serializable_float: 340 return tuple(float(int(c)) for c in obj) 341 else: 342 pass 343 elif self.serializable_list: 344 if self.serializable_int: 345 return [int(c) for c in obj] 346 if self.serializable_float: 347 return [float(int(c)) for c in obj] 348 else: 349 pass 350 else: 351 pass 352 elif 6 == type_id: 353 if self.serializable_bytearray: 354 return obj 355 elif self.serializable_bytes: 356 return bytes(obj) 357 elif self.serializable_str: 358 return obj.hex() 359 elif self.serializable_tuple: 360 if self.serializable_int: 361 return tuple(int(c) for c in obj) 362 if self.serializable_float: 363 return tuple(float(int(c)) for c in obj) 364 else: 365 pass 366 elif self.serializable_list: 367 if self.serializable_int: 368 return [int(c) for c in obj] 369 if self.serializable_float: 370 return [float(int(c)) for c in obj] 371 else: 372 pass 373 else: 374 pass 375 elif 7 == type_id: 376 if self.serializable_bool: 377 return obj 378 elif self.serializable_int: 379 return int(obj) 380 elif self.serializable_bytes: 381 return obj.to_bytes(1, 'little') 382 elif self.serializable_bytearray: 383 return bytearray(obj.to_bytes(1, 'little')) 384 elif self.serializable_str: 385 return str(obj) 386 elif self.serializable_float: 387 return float(int(obj)) 388 else: 389 pass 390 elif 8 == type_id: 391 if self.serializable_none: 392 return obj 393 elif self.serializable_bool: 394 return False 395 elif self.serializable_int: 396 return 0 397 elif self.serializable_bytes: 398 return b'\x00' 399 elif self.serializable_bytearray: 400 return bytearray(b'\x00') 401 elif self.serializable_str: 402 return '' 403 elif self.serializable_float: 404 return 0.0 405 else: 406 pass 407 elif 9 == type_id: 408 if self.serializable_list: 409 return obj 410 elif self.serializable_tuple: 411 return tuple(obj) 412 elif self.serializable_dict: 413 return dict({index: item for index, item in enumerate(obj)}) 414 else: 415 pass 416 elif 10 == type_id: 417 if self.serializable_tuple: 418 return obj 419 elif self.serializable_list: 420 return list(obj) 421 elif self.serializable_dict: 422 return dict({index: item for index, item in enumerate(obj)}) 423 else: 424 pass 425 elif 11 == type_id: 426 if self.serializable_set: 427 return obj 428 elif self.serializable_frozenset: 429 return frozenset(obj) 430 elif self.serializable_tuple: 431 return tuple(obj) 432 elif self.serializable_list: 433 return list(obj) 434 elif self.serializable_dict: 435 return dict({k: None for k in obj}) 436 else: 437 pass 438 elif 12 == type_id: 439 if self.serializable_frozenset: 440 return obj 441 elif self.serializable_set: 442 return set(obj) 443 elif self.serializable_tuple: 444 return tuple(obj) 445 elif self.serializable_list: 446 return list(obj) 447 elif self.serializable_dict: 448 return dict({k: None for k in obj}) 449 else: 450 pass 451 elif 13 == type_id: 452 if self.serializable_dict: 453 return obj 454 elif self.serializable_tuple: 455 return tuple(obj.items()) 456 elif self.serializable_list: 457 return list(list(pair) for pair in obj.items()) 458 else: 459 pass 460 elif 14 == type_id: 461 if self.serializable_slice: 462 return obj 463 elif self.serializable_tuple: 464 return (obj.start, obj.stop, obj.step) 465 elif self.serializable_list: 466 return [obj.start, obj.stop, obj.step] 467 elif self.serializable_dict: 468 return {0: obj.start, 1: obj.stop, 2: obj.step} 469 else: 470 pass 471 else: 472 raise RuntimeError('Unknown type_id') 473 474 raise CanNotAdjustToSerializableError(f'Can not adjust to serializable. Type: {type_id}, obj: {obj}') 475 476 ats = adjust_to_serializable 477 478 def adjust_from_serializable(self, type_id: DataType, obj: Any) -> Any: 479 if 1 == type_id: 480 if self.serializable_int: 481 return obj 482 elif self.serializable_bytes: 483 return int.from_bytes(obj, 'little') 484 elif self.serializable_bytearray: 485 return int.from_bytes(bytes(obj), 'little') 486 elif self.serializable_str: 487 return int(obj) 488 else: 489 pass 490 elif 2 == type_id: 491 if self.serializable_float: 492 return obj 493 elif self.serializable_bytes: 494 return unpack('=d', obj)[0] 495 elif self.serializable_bytearray: 496 return unpack('=d', bytes(obj))[0] 497 elif self.serializable_str: 498 return float(obj) 499 else: 500 pass 501 elif 3 == type_id: 502 if self.serializable_complex: 503 return obj 504 elif self.serializable_bytes: 505 return complex(*unpack('=dd', obj)) 506 elif self.serializable_bytearray: 507 return complex(*unpack('=dd', bytes(obj))) 508 elif self.serializable_str: 509 return complex(*obj) 510 elif self.serializable_float: 511 if self.serializable_tuple: 512 return complex(*obj) 513 elif self.serializable_list: 514 return complex(*obj) 515 else: 516 pass 517 else: 518 pass 519 elif 4 == type_id: 520 if self.serializable_str: 521 return obj 522 elif self.serializable_bytes: 523 return obj.decode('utf-8') 524 elif self.serializable_bytearray: 525 return bytes(obj).decode('utf-8') 526 elif self.serializable_int: 527 if self.serializable_tuple: 528 return ''.join(chr(c) for c in obj) 529 elif self.serializable_list: 530 return ''.join(chr(c) for c in obj) 531 else: 532 pass 533 elif self.serializable_float: 534 if self.serializable_tuple: 535 return ''.join(chr(int(round(c))) for c in obj) 536 elif self.serializable_list: 537 return ''.join(chr(int(round(c))) for c in obj) 538 else: 539 pass 540 else: 541 pass 542 elif 5 == type_id: 543 if self.serializable_bytes: 544 return obj 545 elif self.serializable_bytearray: 546 return bytes(obj) 547 elif self.serializable_str: 548 return bytes.fromhex(obj) 549 elif self.serializable_tuple: 550 if self.serializable_int: 551 return b''.join(item.to_bytes(1, 'little') for item in obj) 552 if self.serializable_float: 553 return b''.join((int(round(item))).to_bytes(1, 'little') for item in obj) 554 else: 555 pass 556 elif self.serializable_list: 557 if self.serializable_int: 558 return b''.join(item.to_bytes(1, 'little') for item in obj) 559 if self.serializable_float: 560 return b''.join((int(round(item))).to_bytes(1, 'little') for item in obj) 561 else: 562 pass 563 else: 564 pass 565 elif 6 == type_id: 566 if self.serializable_bytearray: 567 return obj 568 elif self.serializable_bytes: 569 return bytearray(obj) 570 elif self.serializable_str: 571 return bytearray(bytes.fromhex(obj)) 572 elif self.serializable_tuple: 573 if self.serializable_int: 574 return bytearray(b''.join(item.to_bytes(1, 'little') for item in obj)) 575 if self.serializable_float: 576 return bytearray(b''.join((int(round(item))).to_bytes(1, 'little') for item in obj)) 577 else: 578 pass 579 elif self.serializable_list: 580 if self.serializable_int: 581 return bytearray(b''.join(item.to_bytes(1, 'little') for item in obj)) 582 if self.serializable_float: 583 return bytearray(b''.join((int(round(item))).to_bytes(1, 'little') for item in obj)) 584 else: 585 pass 586 else: 587 pass 588 elif 7 == type_id: 589 if self.serializable_bool: 590 return obj 591 elif self.serializable_int: 592 return bool(obj) 593 elif self.serializable_bytes: 594 return bool(int.from_bytes(obj, 'little')) 595 elif self.serializable_bytearray: 596 return bool(int.from_bytes(bytes(obj), 'little')) 597 elif self.serializable_str: 598 return bool(obj) 599 elif self.serializable_float: 600 return bool(int(round(obj))) 601 else: 602 pass 603 elif 8 == type_id: 604 return None 605 elif 9 == type_id: 606 if self.serializable_list: 607 return obj 608 elif self.serializable_tuple: 609 return list(obj) 610 elif self.serializable_dict: 611 return [value for key, value in sorted(obj.items(), key=lambda x: x[0])] 612 else: 613 pass 614 elif 10 == type_id: 615 if self.serializable_tuple: 616 return obj 617 elif self.serializable_list: 618 return tuple(obj) 619 elif self.serializable_dict: 620 return tuple(value for key, value in sorted(obj.items(), key=lambda x: x[0])) 621 else: 622 pass 623 elif 11 == type_id: 624 if self.serializable_set: 625 return obj 626 elif self.serializable_frozenset: 627 return set(obj) 628 elif self.serializable_tuple: 629 return set(obj) 630 elif self.serializable_list: 631 return set(obj) 632 elif self.serializable_dict: 633 return set(obj.keys()) 634 else: 635 pass 636 elif 12 == type_id: 637 if self.serializable_frozenset: 638 return obj 639 elif self.serializable_set: 640 return frozenset(obj) 641 elif self.serializable_tuple: 642 return frozenset(obj) 643 elif self.serializable_list: 644 return frozenset(obj) 645 elif self.serializable_dict: 646 return frozenset(obj.keys()) 647 else: 648 pass 649 elif 13 == type_id: 650 if self.serializable_dict: 651 return obj 652 elif self.serializable_tuple: 653 return dict(obj) 654 elif self.serializable_list: 655 return dict(obj) 656 else: 657 pass 658 elif 14 == type_id: 659 if self.serializable_slice: 660 return obj 661 elif self.serializable_tuple: 662 return slice(*obj) 663 elif self.serializable_list: 664 return slice(*obj) 665 elif self.serializable_dict: 666 return slice(obj[0], obj[1], obj[2]) 667 else: 668 pass 669 else: 670 raise RuntimeError('Unknown type_id') 671 672 raise CanNotAdjustFromSerializableError(f'Can not adjust from serializable. Type: {type_id}, obj: {obj}') 673 674 afs = adjust_from_serializable 675 676 # def is_replicatable_object_attribute(self, attribute: Any) -> bool: 677 # if is_setable_data_descriptor(attribute): 678 # data = attribute.__get__(None, None) 679 # elif is_callable(attribute): 680 # return False 681 # else: 682 # return True 683 684 def serialize_container(self, type_id: DataType, obj: Any) -> Any: 685 if 9 == type_id: 686 new_obj = list() 687 for item in obj: 688 exists, value = self.serialize_impl(item) 689 if exists: 690 new_obj.append(value) 691 elif 10 == type_id: 692 new_obj = list() 693 for item in obj: 694 exists, value = self.serialize_impl(item) 695 if exists: 696 new_obj.append(value) 697 698 new_obj = tuple(new_obj) 699 elif 11 == type_id: 700 new_obj = set() 701 for item in obj: 702 exists, value = self.serialize_impl(item) 703 if exists: 704 new_obj.add(value) 705 elif 12 == type_id: 706 new_obj = set() 707 for item in obj: 708 exists, value = self.serialize_impl(item) 709 if exists: 710 new_obj.add(value) 711 712 new_obj = frozenset(new_obj) 713 elif 13 == type_id: 714 new_obj = dict() 715 for key, value in obj.items(): 716 key_exists, key_value = self.serialize_impl(key) 717 value_exists, value_value = self.serialize_impl(value) 718 if key_exists and value_exists: 719 new_obj[key_value] = value_value 720 721 return new_obj 722 723 sc = serialize_container 724 725 def serialize_impl(self, obj: Any, ignore_empty_classes: bool = False) -> Tuple[bool, int]: 726 result_exists: bool = True 727 id_: int = id(obj) 728 obj_type = type(obj) 729 if (int != obj_type) and (id_ in self.objects_ids_by_id): 730 new_object: bool = False 731 object_id: int = self.objects_ids_by_id[id_] 732 else: 733 # int object must always produce new object_id because first 256 ints are persistent across Python sessions and 734 # this can cause issues within users of current module. For example within `cengal/hardware/memory/shared_memory` 735 # which changes int values inline instead of producing new objects. 736 new_object = True 737 object_id = self.objects_id_gen() 738 self.objects_ids[object_id] = id_ 739 self.objects_db[object_id] = obj 740 741 if not new_object: 742 return ResultExistence[int](result_exists, object_id) 743 744 if self.serializable_any or (obj_type in self.serializable_data_types): 745 serializable: bool = True 746 type_id: int = data_type_by_type.get(obj_type, 15) 747 else: 748 serializable = False 749 type_id = data_type_by_type.get(obj_type, 0) 750 751 if obj_type in known_container_types: 752 known_container: bool = True 753 else: 754 known_container = False 755 756 class_info: Dict[int, Any] = None 757 known_data: bool = None 758 class_id: int = None 759 is_new_class: bool = None 760 class_name: str = None 761 new_obj_slots: List[Tuple[str, Any]] = None 762 new_obj_dict: Dict[str, Any] = None 763 obj_mapping: Dict = None 764 obj_sequence: List = None 765 obj_set: Set = None 766 if serializable: 767 if known_container: 768 object_info = ( 769 self.ats(1, object_id), 770 self.ats(1, type_id), 771 self.sc(type_id, obj), 772 ) 773 else: 774 object_info = ( 775 self.ats(1, object_id), 776 self.ats(1, type_id), 777 obj, 778 ) 779 else: 780 if obj_type in known_data_types: 781 known_data = True 782 else: 783 known_data = False 784 785 if 0 == type_id: 786 new_obj_slots = list() 787 for slot_name, slot_value in filled_slot_names_with_values_gen(obj): 788 # if not hasattr(slot_value, '__set__'): 789 # # if not setable descriptor 790 # continue 791 792 if self.serializable_str: 793 adjusted_slot_name = slot_name 794 else: 795 adjusted_slot_name = self.ats(4, slot_name) 796 797 item_exists, item_value = self.serialize_impl(slot_value) 798 if item_exists: 799 if self.serializable_tuple: 800 new_obj_slots.append((adjusted_slot_name, item_value)) 801 elif self.serializable_list: 802 new_obj_slots.append([adjusted_slot_name, item_value]) 803 else: 804 new_obj_slots.append(self.ats(10, (adjusted_slot_name, item_value))) 805 806 if new_obj_slots: 807 if self.serializable_list: 808 pass 809 elif self.serializable_tuple: 810 new_obj_slots = tuple(new_obj_slots) 811 else: 812 new_obj_slots = self.ats(9, new_obj_slots) 813 814 new_obj_dict = dict() 815 if hasattr(obj, '__dict__'): 816 for key, value in obj.__dict__.items(): 817 # raw_value = getattr_static(obj, key) 818 # if hasattr(raw_value, '__get__') and (not hasattr(raw_value, '__set__')): 819 # # if not setable descriptor 820 # continue 821 822 if self.serializable_str: 823 adjusted_key = key 824 else: 825 adjusted_key = self.ats(4, key) 826 827 value_exists, value_value = self.serialize_impl(value) 828 if value_exists: 829 new_obj_dict[adjusted_key] = value_value 830 else: 831 continue 832 833 if new_obj_dict: 834 if self.serializable_dict: 835 pass 836 else: 837 new_obj_dict = self.ats(13, new_obj_dict) 838 839 obj_mapping = dict() 840 obj_sequence = list() 841 obj_set = set() 842 if isinstance(obj, MutableMapping): 843 for key, value in obj.items(): 844 key_exists, key_value = self.serialize_impl(value) 845 if not key_exists: 846 continue 847 848 value_exists, value_value = self.serialize_impl(value) 849 if value_exists: 850 obj_mapping[key_value] = value_value 851 else: 852 continue 853 854 if obj_mapping: 855 if self.serializable_dict: 856 pass 857 else: 858 obj_mapping = self.ats(13, obj_mapping) 859 elif isinstance(obj, MutableSequence): 860 for item in obj: 861 item_exists, item_value = self.serialize_impl(item) 862 if item_exists: 863 obj_sequence.append(item_value) 864 else: 865 continue 866 867 if obj_sequence: 868 if self.serializable_list: 869 pass 870 else: 871 obj_sequence = self.ats(9, obj_sequence) 872 elif isinstance(obj, MutableSet): 873 for item in obj: 874 item_exists, item_value = self.serialize_impl(item) 875 if item_exists: 876 obj_set.add(item_value) 877 else: 878 continue 879 880 if obj_set: 881 if self.serializable_set: 882 pass 883 else: 884 obj_set = self.ats(11, obj_set) 885 else: 886 pass 887 888 if ignore_empty_classes: 889 result_exists = new_obj_slots or new_obj_dict or obj_mapping or obj_sequence or obj_set 890 891 if obj_type in self.known_classes: 892 is_new_class = False 893 class_id = self.known_classes[obj_type] 894 else: 895 is_new_class = True 896 class_name = obj_type.__name__ 897 class_id = self.classes_id_gen() 898 self.known_classes[obj_type] = class_id 899 self.known_classes_by_id[class_id] = obj_type 900 module_importable_str, owning_names_path = entity_module_importable_str_and_owning_names_path(obj) 901 adjusted_owning_names_path: List[str] = [self.ats(4, name) for name in owning_names_path] 902 module_importable_str_and_owning_names_path = (class_name, module_importable_str, tuple(owning_names_path)) 903 self.known_classes_info[module_importable_str_and_owning_names_path] = class_id 904 self.known_classes_info_by_id[class_id] = module_importable_str_and_owning_names_path 905 906 if result_exists: 907 class_info = ( 908 self.ats(1, class_id), 909 self.ats(4, class_name), 910 self.ats(4, module_importable_str), 911 self.ats(9, adjusted_owning_names_path), 912 ) 913 914 if self.serializable_tuple: 915 adjusted_class_info = class_info 916 else: 917 adjusted_class_info = self.ats(10, class_info) 918 919 if self.on_new_class_handler: 920 self.on_new_class_handler( 921 adjusted_class_info, 922 obj_type, 923 class_id, 924 class_name, 925 module_importable_str, 926 owning_names_path, 927 module_importable_str_and_owning_names_path 928 ) 929 930 # will be later serialized much faster than without `if else None` 931 object_info = ( 932 self.ats(1, object_id), 933 self.ats(1, type_id), 934 self.ats(8, None), 935 self.ats(1, class_id), 936 new_obj_slots if new_obj_slots else self.ats(9, list()), 937 new_obj_dict if new_obj_dict else self.ats(13, dict()), 938 obj_mapping if obj_mapping else self.ats(13, dict()), 939 obj_sequence if obj_sequence else self.ats(9, list()), 940 obj_set if obj_set else self.ats(11, set()), 941 ) 942 elif known_data: 943 object_info = ( 944 self.ats(1, object_id), 945 self.ats(1, type_id), 946 self.ats(type_id, obj), 947 ) 948 elif known_container: 949 object_info = ( 950 self.ats(1, object_id), 951 self.ats(1, type_id), 952 self.ats(type_id, self.sc(type_id, obj)), 953 ) 954 else: 955 raise RuntimeError('Unknown type_id') 956 957 if self.serializable_tuple: 958 adjusted_object_info = object_info 959 else: 960 adjusted_object_info = self.ats(10, object_info) 961 962 if result_exists: 963 if self.on_new_obj_info_handler: 964 self.on_new_obj_info_handler( 965 adjusted_object_info, 966 obj, 967 object_id, 968 type_id, 969 serializable, 970 known_container, 971 known_data, 972 class_id, 973 is_new_class, 974 new_obj_slots, 975 new_obj_dict, 976 obj_mapping, 977 obj_sequence, 978 obj_set, 979 ) 980 981 return result_exists, object_id 982 983 si = serialize_impl 984 985 def serialize(self, obj: Any, ignore_empty_classes: bool = False) -> Tuple[bool, int]: 986 with DisableGC(): 987 return self.serialize_impl(obj, ignore_empty_classes) 988 989 s = serialize 990 991 def deserialize_class_impl(self, class_info: Tuple) -> Tuple[bool, int, Type]: 992 if self.serializable_tuple: 993 pass 994 else: 995 class_info = self.afs(10, class_info) 996 997 class_id: int = self.afs(1, class_info[0]) 998 if class_id in self.known_classes_by_id: 999 return ResultExistence[Tuple[int, Type]](True, (class_id, self.known_classes_by_id[class_id])) 1000 else: 1001 class_name: str = self.afs(4, class_info[1]) 1002 module_importable_str: str = self.afs(4, class_info[2]) 1003 owning_names_path: List[str] = self.afs(9, class_info[3]) 1004 adjusted_owning_names_path: List[str] = [self.afs(4, name) for name in owning_names_path] 1005 obj_class: Type = entity_by_name_module_importable_str_and_owning_names_path(class_name, module_importable_str, adjusted_owning_names_path) 1006 self.known_classes_by_id[class_id] = obj_class 1007 self.known_classes[obj_class] = class_id 1008 class_info_tuple: Tuple = (class_name, module_importable_str, tuple(owning_names_path)) 1009 self.known_classes_info[class_info_tuple] = class_id 1010 self.known_classes_info_by_id[class_id] = class_info_tuple 1011 return True, class_id, obj_class 1012 1013 dcli = deserialize_class_impl 1014 1015 def deserialize_class(self, class_info: Tuple) -> Tuple[bool, int, Type]: 1016 with DisableGC(): 1017 return self.deserialize_class_impl(class_info) 1018 1019 dcl = deserialize_class 1020 1021 def deserialize_container_impl(self, type_id: DataType, obj: Any) -> Any: 1022 new_obj = obj 1023 if 9 == type_id: 1024 new_obj = list() 1025 for item_id in obj: 1026 new_obj.append(self.objects_db[self.afs(1, item_id)]) 1027 elif 10 == type_id: 1028 new_obj = list() 1029 for item_id in obj: 1030 new_obj.append(self.objects_db[self.afs(1, item_id)]) 1031 1032 new_obj = tuple(new_obj) 1033 elif 11 == type_id: 1034 new_obj = set() 1035 for item_id in obj: 1036 new_obj.add(self.objects_db[self.afs(1, item_id)]) 1037 elif 12 == type_id: 1038 new_obj = set() 1039 for item_id in obj: 1040 new_obj.add(self.objects_db[self.afs(1, item_id)]) 1041 1042 new_obj = frozenset(new_obj) 1043 elif 13 == type_id: 1044 new_obj = dict() 1045 for key_id, value_id in obj.items(): 1046 new_obj[self.objects_db[int(self.afs(1, key_id))]] = self.objects_db[self.afs(1, value_id)] 1047 1048 return new_obj 1049 1050 dcoi = deserialize_container_impl 1051 1052 def deserialize_container(self, type_id: DataType, obj: Any) -> Any: 1053 with DisableGC(): 1054 return self.deserialize_container_impl(type_id, obj) 1055 1056 dco = deserialize_container 1057 1058 def deserialize_obj_impl(self, obj_info: Tuple) -> Tuple[bool, int, Any]: 1059 if self.serializable_tuple: 1060 pass 1061 else: 1062 obj_info = self.afs(10, obj_info) 1063 1064 object_id: int = self.afs(1, obj_info[0]) 1065 if object_id in self.objects_db: 1066 return ResultExistence[Tuple[int, Any]](True, (object_id, self.objects_db[object_id])) 1067 1068 type_id: int = self.afs(1, obj_info[1]) 1069 1070 obj_type = data_type[type_id] 1071 serializable: bool = self.serializable_any or (obj_type in self.serializable_data_types) 1072 known_container: bool = obj_type in known_container_types 1073 known_data: bool = None 1074 1075 if serializable: 1076 if known_container: 1077 obj = self.dcoi(type_id, obj_info[2]) 1078 else: 1079 obj = obj_info[2] 1080 else: 1081 known_data = obj_type in known_data_types 1082 1083 if 0 == type_id: 1084 class_id: int = self.afs(1, obj_info[3]) 1085 obj_class: Type = self.known_classes_by_id[class_id] 1086 obj: Any = obj_class.__new__(obj_class) 1087 1088 if obj_info[4]: 1089 clonable_slots = self.afs(9, obj_info[4]) 1090 for slot_name, slot_id in clonable_slots: 1091 slot_name = self.afs(4, slot_name) 1092 slot_id = self.afs(1, slot_id) 1093 child_obj = self.objects_db[slot_id] 1094 setattr(obj, slot_name, child_obj) 1095 1096 if obj_info[5]: 1097 clonable_dict_items = self.afs(13, obj_info[5]) 1098 for key, value_id in clonable_dict_items.items(): 1099 key = self.afs(4, key) 1100 value_id = self.afs(1, value_id) 1101 child_obj = self.objects_db[value_id] 1102 setattr(obj, key, child_obj) 1103 1104 if obj_info[6]: 1105 contained_mapping = self.afs(13, obj_info[6]) 1106 for key_id, value_id in contained_mapping.items(): 1107 key_id = self.afs(1, key_id) 1108 value_id = self.afs(1, value_id) 1109 child_key = self.objects_db[int(key_id)] 1110 child_value = self.objects_db[value_id] 1111 obj[child_key] = child_value 1112 1113 if obj_info[7]: 1114 contained_sequence = self.afs(9, obj_info[7]) 1115 for item_id in contained_sequence: 1116 item_id = self.afs(1, item_id) 1117 child_item = self.objects_db[item_id] 1118 obj.append(child_item) 1119 1120 if obj_info[8]: 1121 contained_set = self.afs(11, obj_info[8]) 1122 for item_id in contained_set: 1123 item_id = self.afs(1, item_id) 1124 child_item = self.objects_db[item_id] 1125 obj.add(child_item) 1126 elif known_data: 1127 obj = self.afs(type_id, obj_info[2]) 1128 elif known_container: 1129 obj = self.dcoi(type_id, self.afs(type_id, obj_info[2])) 1130 else: 1131 raise RuntimeError('Unknown type_id') 1132 1133 self.objects_db[object_id] = obj 1134 id_: int = id(obj) 1135 self.objects_ids[object_id] = id_ 1136 self.objects_ids_by_id[id_] = object_id 1137 return True, object_id, obj 1138 1139 doi = deserialize_obj_impl 1140 1141 def deserialize_obj(self, obj_info: Tuple) -> Tuple[bool, int, Any]: 1142 with DisableGC(): 1143 return self.deserialize_obj_impl(obj_info) 1144 1145 do = deserialize_obj
RemoteObjectsManager( on_new_class_handler: Union[Callable, NoneType] = None, on_new_obj_info_handler: Union[Callable, NoneType] = None, objects_db: Union[Dict[int, Any], NoneType] = None, classes_id_gen: Union[Callable, NoneType] = None, objects_id_gen: Union[Callable, NoneType] = None, serializable_data_types: Union[Set[Type], NoneType] = None)
209 def __init__(self, 210 on_new_class_handler: Optional[Callable] = None, 211 on_new_obj_info_handler: Optional[Callable] = None, 212 # classess_db: Optional[Dict[int, Type]] = None, 213 objects_db: Optional[Dict[int, Any]] = None, 214 classes_id_gen: Optional[Callable] = None, 215 objects_id_gen: Optional[Callable] = None, 216 serializable_data_types: Optional[Set[Type]] = None, 217 ) -> None: 218 # self.classess_db: Dict[int, Type] = dict() if classess_db is None else classess_db 219 self.classes_id_gen: IDGenerator = IDGenerator() if classes_id_gen is None else classes_id_gen 220 self.objects_db: Dict[int, Any] = dict() if objects_db is None else objects_db 221 self.objects_id_gen: IDGenerator = IDGenerator() if objects_id_gen is None else objects_id_gen 222 self.serializable_data_types: Set[Type] = default_serializable_data_types if serializable_data_types is None else serializable_data_types 223 self.serializable_any: bool = not self.serializable_data_types 224 self.serializable_int: bool = (int in self.serializable_data_types) or self.serializable_any 225 self.serializable_float: bool = (float in self.serializable_data_types) or self.serializable_any 226 self.serializable_complex: bool = (complex in self.serializable_data_types) or self.serializable_any 227 self.serializable_str: bool = (str in self.serializable_data_types) or self.serializable_any 228 self.serializable_bytes: bool = (bytes in self.serializable_data_types) or self.serializable_any 229 self.serializable_bytearray: bool = (bytearray in self.serializable_data_types) or self.serializable_any 230 self.serializable_bool: bool = (bool in self.serializable_data_types) or self.serializable_any 231 self.serializable_none: bool = (type(None) in self.serializable_data_types) or self.serializable_any 232 self.serializable_list: bool = (list in self.serializable_data_types) or self.serializable_any 233 self.serializable_tuple: bool = (tuple in self.serializable_data_types) or self.serializable_any 234 self.serializable_set: bool = (set in self.serializable_data_types) or self.serializable_any 235 self.serializable_frozenset: bool = (frozenset in self.serializable_data_types) or self.serializable_any 236 self.serializable_dict: bool = (dict in self.serializable_data_types) or self.serializable_any 237 self.serializable_slice: bool = (slice in self.serializable_data_types) or self.serializable_any 238 self.known_classes: Dict[Type, int] = dict() 239 self.known_classes_by_id: Dict[int, Type] = dict() 240 self.known_classes_info: Dict[Tuple, int] = dict() 241 self.known_classes_info_by_id: Dict[int, Tuple] = dict() 242 self.on_new_class_handler: Optional[Callable] = on_new_class_handler 243 self.on_new_obj_info_handler: Optional[Callable] = on_new_obj_info_handler 244 self.objects_ids: Dict[int, int] = dict() # (Key: object_id, Value: id(obj)) 245 self.objects_ids_by_id: Dict[int, int] = dict() # (Key: id(obj), Value: object_id)
265 def adjust_to_serializable(self, type_id: DataType, obj: Any) -> Any: 266 if 1 == type_id: 267 if self.serializable_int: 268 return obj 269 elif self.serializable_bytes: 270 return obj.to_bytes(8, 'little') 271 elif self.serializable_bytearray: 272 return bytearray(obj.to_bytes(8, 'little')) 273 elif self.serializable_str: 274 return str(obj) 275 else: 276 pass 277 elif 2 == type_id: 278 if self.serializable_float: 279 return obj 280 elif self.serializable_bytes: 281 return pack('=d', obj) 282 elif self.serializable_bytearray: 283 return bytearray(pack('=d', obj)) 284 elif self.serializable_str: 285 return str(obj) 286 else: 287 pass 288 elif 3 == type_id: 289 if self.serializable_complex: 290 return obj 291 elif self.serializable_bytes: 292 return pack('=dd', obj.real, obj.imag) 293 elif self.serializable_bytearray: 294 return bytearray(pack('=dd', obj.real, obj.imag)) 295 elif self.serializable_str: 296 return str(obj) 297 elif self.serializable_float: 298 if self.serializable_tuple: 299 return (obj.real, obj.imag) 300 elif self.serializable_list: 301 return [obj.real, obj.imag] 302 else: 303 pass 304 else: 305 pass 306 elif 4 == type_id: 307 if self.serializable_str: 308 return obj 309 elif self.serializable_bytes: 310 return obj.encode('utf-8') 311 elif self.serializable_bytearray: 312 return bytearray(obj.encode('utf-8')) 313 elif self.serializable_int: 314 if self.serializable_tuple: 315 return tuple(ord(c) for c in obj) 316 elif self.serializable_list: 317 return [ord(c) for c in obj] 318 else: 319 pass 320 elif self.serializable_float: 321 if self.serializable_tuple: 322 return tuple(float(ord(c)) for c in obj) 323 elif self.serializable_list: 324 return [float(ord(c)) for c in obj] 325 else: 326 pass 327 else: 328 pass 329 elif 5 == type_id: 330 if self.serializable_bytes: 331 return obj 332 elif self.serializable_bytearray: 333 return bytearray(obj) 334 elif self.serializable_str: 335 return obj.hex() 336 elif self.serializable_tuple: 337 if self.serializable_int: 338 return tuple(int(c) for c in obj) 339 if self.serializable_float: 340 return tuple(float(int(c)) for c in obj) 341 else: 342 pass 343 elif self.serializable_list: 344 if self.serializable_int: 345 return [int(c) for c in obj] 346 if self.serializable_float: 347 return [float(int(c)) for c in obj] 348 else: 349 pass 350 else: 351 pass 352 elif 6 == type_id: 353 if self.serializable_bytearray: 354 return obj 355 elif self.serializable_bytes: 356 return bytes(obj) 357 elif self.serializable_str: 358 return obj.hex() 359 elif self.serializable_tuple: 360 if self.serializable_int: 361 return tuple(int(c) for c in obj) 362 if self.serializable_float: 363 return tuple(float(int(c)) for c in obj) 364 else: 365 pass 366 elif self.serializable_list: 367 if self.serializable_int: 368 return [int(c) for c in obj] 369 if self.serializable_float: 370 return [float(int(c)) for c in obj] 371 else: 372 pass 373 else: 374 pass 375 elif 7 == type_id: 376 if self.serializable_bool: 377 return obj 378 elif self.serializable_int: 379 return int(obj) 380 elif self.serializable_bytes: 381 return obj.to_bytes(1, 'little') 382 elif self.serializable_bytearray: 383 return bytearray(obj.to_bytes(1, 'little')) 384 elif self.serializable_str: 385 return str(obj) 386 elif self.serializable_float: 387 return float(int(obj)) 388 else: 389 pass 390 elif 8 == type_id: 391 if self.serializable_none: 392 return obj 393 elif self.serializable_bool: 394 return False 395 elif self.serializable_int: 396 return 0 397 elif self.serializable_bytes: 398 return b'\x00' 399 elif self.serializable_bytearray: 400 return bytearray(b'\x00') 401 elif self.serializable_str: 402 return '' 403 elif self.serializable_float: 404 return 0.0 405 else: 406 pass 407 elif 9 == type_id: 408 if self.serializable_list: 409 return obj 410 elif self.serializable_tuple: 411 return tuple(obj) 412 elif self.serializable_dict: 413 return dict({index: item for index, item in enumerate(obj)}) 414 else: 415 pass 416 elif 10 == type_id: 417 if self.serializable_tuple: 418 return obj 419 elif self.serializable_list: 420 return list(obj) 421 elif self.serializable_dict: 422 return dict({index: item for index, item in enumerate(obj)}) 423 else: 424 pass 425 elif 11 == type_id: 426 if self.serializable_set: 427 return obj 428 elif self.serializable_frozenset: 429 return frozenset(obj) 430 elif self.serializable_tuple: 431 return tuple(obj) 432 elif self.serializable_list: 433 return list(obj) 434 elif self.serializable_dict: 435 return dict({k: None for k in obj}) 436 else: 437 pass 438 elif 12 == type_id: 439 if self.serializable_frozenset: 440 return obj 441 elif self.serializable_set: 442 return set(obj) 443 elif self.serializable_tuple: 444 return tuple(obj) 445 elif self.serializable_list: 446 return list(obj) 447 elif self.serializable_dict: 448 return dict({k: None for k in obj}) 449 else: 450 pass 451 elif 13 == type_id: 452 if self.serializable_dict: 453 return obj 454 elif self.serializable_tuple: 455 return tuple(obj.items()) 456 elif self.serializable_list: 457 return list(list(pair) for pair in obj.items()) 458 else: 459 pass 460 elif 14 == type_id: 461 if self.serializable_slice: 462 return obj 463 elif self.serializable_tuple: 464 return (obj.start, obj.stop, obj.step) 465 elif self.serializable_list: 466 return [obj.start, obj.stop, obj.step] 467 elif self.serializable_dict: 468 return {0: obj.start, 1: obj.stop, 2: obj.step} 469 else: 470 pass 471 else: 472 raise RuntimeError('Unknown type_id') 473 474 raise CanNotAdjustToSerializableError(f'Can not adjust to serializable. Type: {type_id}, obj: {obj}')
265 def adjust_to_serializable(self, type_id: DataType, obj: Any) -> Any: 266 if 1 == type_id: 267 if self.serializable_int: 268 return obj 269 elif self.serializable_bytes: 270 return obj.to_bytes(8, 'little') 271 elif self.serializable_bytearray: 272 return bytearray(obj.to_bytes(8, 'little')) 273 elif self.serializable_str: 274 return str(obj) 275 else: 276 pass 277 elif 2 == type_id: 278 if self.serializable_float: 279 return obj 280 elif self.serializable_bytes: 281 return pack('=d', obj) 282 elif self.serializable_bytearray: 283 return bytearray(pack('=d', obj)) 284 elif self.serializable_str: 285 return str(obj) 286 else: 287 pass 288 elif 3 == type_id: 289 if self.serializable_complex: 290 return obj 291 elif self.serializable_bytes: 292 return pack('=dd', obj.real, obj.imag) 293 elif self.serializable_bytearray: 294 return bytearray(pack('=dd', obj.real, obj.imag)) 295 elif self.serializable_str: 296 return str(obj) 297 elif self.serializable_float: 298 if self.serializable_tuple: 299 return (obj.real, obj.imag) 300 elif self.serializable_list: 301 return [obj.real, obj.imag] 302 else: 303 pass 304 else: 305 pass 306 elif 4 == type_id: 307 if self.serializable_str: 308 return obj 309 elif self.serializable_bytes: 310 return obj.encode('utf-8') 311 elif self.serializable_bytearray: 312 return bytearray(obj.encode('utf-8')) 313 elif self.serializable_int: 314 if self.serializable_tuple: 315 return tuple(ord(c) for c in obj) 316 elif self.serializable_list: 317 return [ord(c) for c in obj] 318 else: 319 pass 320 elif self.serializable_float: 321 if self.serializable_tuple: 322 return tuple(float(ord(c)) for c in obj) 323 elif self.serializable_list: 324 return [float(ord(c)) for c in obj] 325 else: 326 pass 327 else: 328 pass 329 elif 5 == type_id: 330 if self.serializable_bytes: 331 return obj 332 elif self.serializable_bytearray: 333 return bytearray(obj) 334 elif self.serializable_str: 335 return obj.hex() 336 elif self.serializable_tuple: 337 if self.serializable_int: 338 return tuple(int(c) for c in obj) 339 if self.serializable_float: 340 return tuple(float(int(c)) for c in obj) 341 else: 342 pass 343 elif self.serializable_list: 344 if self.serializable_int: 345 return [int(c) for c in obj] 346 if self.serializable_float: 347 return [float(int(c)) for c in obj] 348 else: 349 pass 350 else: 351 pass 352 elif 6 == type_id: 353 if self.serializable_bytearray: 354 return obj 355 elif self.serializable_bytes: 356 return bytes(obj) 357 elif self.serializable_str: 358 return obj.hex() 359 elif self.serializable_tuple: 360 if self.serializable_int: 361 return tuple(int(c) for c in obj) 362 if self.serializable_float: 363 return tuple(float(int(c)) for c in obj) 364 else: 365 pass 366 elif self.serializable_list: 367 if self.serializable_int: 368 return [int(c) for c in obj] 369 if self.serializable_float: 370 return [float(int(c)) for c in obj] 371 else: 372 pass 373 else: 374 pass 375 elif 7 == type_id: 376 if self.serializable_bool: 377 return obj 378 elif self.serializable_int: 379 return int(obj) 380 elif self.serializable_bytes: 381 return obj.to_bytes(1, 'little') 382 elif self.serializable_bytearray: 383 return bytearray(obj.to_bytes(1, 'little')) 384 elif self.serializable_str: 385 return str(obj) 386 elif self.serializable_float: 387 return float(int(obj)) 388 else: 389 pass 390 elif 8 == type_id: 391 if self.serializable_none: 392 return obj 393 elif self.serializable_bool: 394 return False 395 elif self.serializable_int: 396 return 0 397 elif self.serializable_bytes: 398 return b'\x00' 399 elif self.serializable_bytearray: 400 return bytearray(b'\x00') 401 elif self.serializable_str: 402 return '' 403 elif self.serializable_float: 404 return 0.0 405 else: 406 pass 407 elif 9 == type_id: 408 if self.serializable_list: 409 return obj 410 elif self.serializable_tuple: 411 return tuple(obj) 412 elif self.serializable_dict: 413 return dict({index: item for index, item in enumerate(obj)}) 414 else: 415 pass 416 elif 10 == type_id: 417 if self.serializable_tuple: 418 return obj 419 elif self.serializable_list: 420 return list(obj) 421 elif self.serializable_dict: 422 return dict({index: item for index, item in enumerate(obj)}) 423 else: 424 pass 425 elif 11 == type_id: 426 if self.serializable_set: 427 return obj 428 elif self.serializable_frozenset: 429 return frozenset(obj) 430 elif self.serializable_tuple: 431 return tuple(obj) 432 elif self.serializable_list: 433 return list(obj) 434 elif self.serializable_dict: 435 return dict({k: None for k in obj}) 436 else: 437 pass 438 elif 12 == type_id: 439 if self.serializable_frozenset: 440 return obj 441 elif self.serializable_set: 442 return set(obj) 443 elif self.serializable_tuple: 444 return tuple(obj) 445 elif self.serializable_list: 446 return list(obj) 447 elif self.serializable_dict: 448 return dict({k: None for k in obj}) 449 else: 450 pass 451 elif 13 == type_id: 452 if self.serializable_dict: 453 return obj 454 elif self.serializable_tuple: 455 return tuple(obj.items()) 456 elif self.serializable_list: 457 return list(list(pair) for pair in obj.items()) 458 else: 459 pass 460 elif 14 == type_id: 461 if self.serializable_slice: 462 return obj 463 elif self.serializable_tuple: 464 return (obj.start, obj.stop, obj.step) 465 elif self.serializable_list: 466 return [obj.start, obj.stop, obj.step] 467 elif self.serializable_dict: 468 return {0: obj.start, 1: obj.stop, 2: obj.step} 469 else: 470 pass 471 else: 472 raise RuntimeError('Unknown type_id') 473 474 raise CanNotAdjustToSerializableError(f'Can not adjust to serializable. Type: {type_id}, obj: {obj}')
478 def adjust_from_serializable(self, type_id: DataType, obj: Any) -> Any: 479 if 1 == type_id: 480 if self.serializable_int: 481 return obj 482 elif self.serializable_bytes: 483 return int.from_bytes(obj, 'little') 484 elif self.serializable_bytearray: 485 return int.from_bytes(bytes(obj), 'little') 486 elif self.serializable_str: 487 return int(obj) 488 else: 489 pass 490 elif 2 == type_id: 491 if self.serializable_float: 492 return obj 493 elif self.serializable_bytes: 494 return unpack('=d', obj)[0] 495 elif self.serializable_bytearray: 496 return unpack('=d', bytes(obj))[0] 497 elif self.serializable_str: 498 return float(obj) 499 else: 500 pass 501 elif 3 == type_id: 502 if self.serializable_complex: 503 return obj 504 elif self.serializable_bytes: 505 return complex(*unpack('=dd', obj)) 506 elif self.serializable_bytearray: 507 return complex(*unpack('=dd', bytes(obj))) 508 elif self.serializable_str: 509 return complex(*obj) 510 elif self.serializable_float: 511 if self.serializable_tuple: 512 return complex(*obj) 513 elif self.serializable_list: 514 return complex(*obj) 515 else: 516 pass 517 else: 518 pass 519 elif 4 == type_id: 520 if self.serializable_str: 521 return obj 522 elif self.serializable_bytes: 523 return obj.decode('utf-8') 524 elif self.serializable_bytearray: 525 return bytes(obj).decode('utf-8') 526 elif self.serializable_int: 527 if self.serializable_tuple: 528 return ''.join(chr(c) for c in obj) 529 elif self.serializable_list: 530 return ''.join(chr(c) for c in obj) 531 else: 532 pass 533 elif self.serializable_float: 534 if self.serializable_tuple: 535 return ''.join(chr(int(round(c))) for c in obj) 536 elif self.serializable_list: 537 return ''.join(chr(int(round(c))) for c in obj) 538 else: 539 pass 540 else: 541 pass 542 elif 5 == type_id: 543 if self.serializable_bytes: 544 return obj 545 elif self.serializable_bytearray: 546 return bytes(obj) 547 elif self.serializable_str: 548 return bytes.fromhex(obj) 549 elif self.serializable_tuple: 550 if self.serializable_int: 551 return b''.join(item.to_bytes(1, 'little') for item in obj) 552 if self.serializable_float: 553 return b''.join((int(round(item))).to_bytes(1, 'little') for item in obj) 554 else: 555 pass 556 elif self.serializable_list: 557 if self.serializable_int: 558 return b''.join(item.to_bytes(1, 'little') for item in obj) 559 if self.serializable_float: 560 return b''.join((int(round(item))).to_bytes(1, 'little') for item in obj) 561 else: 562 pass 563 else: 564 pass 565 elif 6 == type_id: 566 if self.serializable_bytearray: 567 return obj 568 elif self.serializable_bytes: 569 return bytearray(obj) 570 elif self.serializable_str: 571 return bytearray(bytes.fromhex(obj)) 572 elif self.serializable_tuple: 573 if self.serializable_int: 574 return bytearray(b''.join(item.to_bytes(1, 'little') for item in obj)) 575 if self.serializable_float: 576 return bytearray(b''.join((int(round(item))).to_bytes(1, 'little') for item in obj)) 577 else: 578 pass 579 elif self.serializable_list: 580 if self.serializable_int: 581 return bytearray(b''.join(item.to_bytes(1, 'little') for item in obj)) 582 if self.serializable_float: 583 return bytearray(b''.join((int(round(item))).to_bytes(1, 'little') for item in obj)) 584 else: 585 pass 586 else: 587 pass 588 elif 7 == type_id: 589 if self.serializable_bool: 590 return obj 591 elif self.serializable_int: 592 return bool(obj) 593 elif self.serializable_bytes: 594 return bool(int.from_bytes(obj, 'little')) 595 elif self.serializable_bytearray: 596 return bool(int.from_bytes(bytes(obj), 'little')) 597 elif self.serializable_str: 598 return bool(obj) 599 elif self.serializable_float: 600 return bool(int(round(obj))) 601 else: 602 pass 603 elif 8 == type_id: 604 return None 605 elif 9 == type_id: 606 if self.serializable_list: 607 return obj 608 elif self.serializable_tuple: 609 return list(obj) 610 elif self.serializable_dict: 611 return [value for key, value in sorted(obj.items(), key=lambda x: x[0])] 612 else: 613 pass 614 elif 10 == type_id: 615 if self.serializable_tuple: 616 return obj 617 elif self.serializable_list: 618 return tuple(obj) 619 elif self.serializable_dict: 620 return tuple(value for key, value in sorted(obj.items(), key=lambda x: x[0])) 621 else: 622 pass 623 elif 11 == type_id: 624 if self.serializable_set: 625 return obj 626 elif self.serializable_frozenset: 627 return set(obj) 628 elif self.serializable_tuple: 629 return set(obj) 630 elif self.serializable_list: 631 return set(obj) 632 elif self.serializable_dict: 633 return set(obj.keys()) 634 else: 635 pass 636 elif 12 == type_id: 637 if self.serializable_frozenset: 638 return obj 639 elif self.serializable_set: 640 return frozenset(obj) 641 elif self.serializable_tuple: 642 return frozenset(obj) 643 elif self.serializable_list: 644 return frozenset(obj) 645 elif self.serializable_dict: 646 return frozenset(obj.keys()) 647 else: 648 pass 649 elif 13 == type_id: 650 if self.serializable_dict: 651 return obj 652 elif self.serializable_tuple: 653 return dict(obj) 654 elif self.serializable_list: 655 return dict(obj) 656 else: 657 pass 658 elif 14 == type_id: 659 if self.serializable_slice: 660 return obj 661 elif self.serializable_tuple: 662 return slice(*obj) 663 elif self.serializable_list: 664 return slice(*obj) 665 elif self.serializable_dict: 666 return slice(obj[0], obj[1], obj[2]) 667 else: 668 pass 669 else: 670 raise RuntimeError('Unknown type_id') 671 672 raise CanNotAdjustFromSerializableError(f'Can not adjust from serializable. Type: {type_id}, obj: {obj}')
478 def adjust_from_serializable(self, type_id: DataType, obj: Any) -> Any: 479 if 1 == type_id: 480 if self.serializable_int: 481 return obj 482 elif self.serializable_bytes: 483 return int.from_bytes(obj, 'little') 484 elif self.serializable_bytearray: 485 return int.from_bytes(bytes(obj), 'little') 486 elif self.serializable_str: 487 return int(obj) 488 else: 489 pass 490 elif 2 == type_id: 491 if self.serializable_float: 492 return obj 493 elif self.serializable_bytes: 494 return unpack('=d', obj)[0] 495 elif self.serializable_bytearray: 496 return unpack('=d', bytes(obj))[0] 497 elif self.serializable_str: 498 return float(obj) 499 else: 500 pass 501 elif 3 == type_id: 502 if self.serializable_complex: 503 return obj 504 elif self.serializable_bytes: 505 return complex(*unpack('=dd', obj)) 506 elif self.serializable_bytearray: 507 return complex(*unpack('=dd', bytes(obj))) 508 elif self.serializable_str: 509 return complex(*obj) 510 elif self.serializable_float: 511 if self.serializable_tuple: 512 return complex(*obj) 513 elif self.serializable_list: 514 return complex(*obj) 515 else: 516 pass 517 else: 518 pass 519 elif 4 == type_id: 520 if self.serializable_str: 521 return obj 522 elif self.serializable_bytes: 523 return obj.decode('utf-8') 524 elif self.serializable_bytearray: 525 return bytes(obj).decode('utf-8') 526 elif self.serializable_int: 527 if self.serializable_tuple: 528 return ''.join(chr(c) for c in obj) 529 elif self.serializable_list: 530 return ''.join(chr(c) for c in obj) 531 else: 532 pass 533 elif self.serializable_float: 534 if self.serializable_tuple: 535 return ''.join(chr(int(round(c))) for c in obj) 536 elif self.serializable_list: 537 return ''.join(chr(int(round(c))) for c in obj) 538 else: 539 pass 540 else: 541 pass 542 elif 5 == type_id: 543 if self.serializable_bytes: 544 return obj 545 elif self.serializable_bytearray: 546 return bytes(obj) 547 elif self.serializable_str: 548 return bytes.fromhex(obj) 549 elif self.serializable_tuple: 550 if self.serializable_int: 551 return b''.join(item.to_bytes(1, 'little') for item in obj) 552 if self.serializable_float: 553 return b''.join((int(round(item))).to_bytes(1, 'little') for item in obj) 554 else: 555 pass 556 elif self.serializable_list: 557 if self.serializable_int: 558 return b''.join(item.to_bytes(1, 'little') for item in obj) 559 if self.serializable_float: 560 return b''.join((int(round(item))).to_bytes(1, 'little') for item in obj) 561 else: 562 pass 563 else: 564 pass 565 elif 6 == type_id: 566 if self.serializable_bytearray: 567 return obj 568 elif self.serializable_bytes: 569 return bytearray(obj) 570 elif self.serializable_str: 571 return bytearray(bytes.fromhex(obj)) 572 elif self.serializable_tuple: 573 if self.serializable_int: 574 return bytearray(b''.join(item.to_bytes(1, 'little') for item in obj)) 575 if self.serializable_float: 576 return bytearray(b''.join((int(round(item))).to_bytes(1, 'little') for item in obj)) 577 else: 578 pass 579 elif self.serializable_list: 580 if self.serializable_int: 581 return bytearray(b''.join(item.to_bytes(1, 'little') for item in obj)) 582 if self.serializable_float: 583 return bytearray(b''.join((int(round(item))).to_bytes(1, 'little') for item in obj)) 584 else: 585 pass 586 else: 587 pass 588 elif 7 == type_id: 589 if self.serializable_bool: 590 return obj 591 elif self.serializable_int: 592 return bool(obj) 593 elif self.serializable_bytes: 594 return bool(int.from_bytes(obj, 'little')) 595 elif self.serializable_bytearray: 596 return bool(int.from_bytes(bytes(obj), 'little')) 597 elif self.serializable_str: 598 return bool(obj) 599 elif self.serializable_float: 600 return bool(int(round(obj))) 601 else: 602 pass 603 elif 8 == type_id: 604 return None 605 elif 9 == type_id: 606 if self.serializable_list: 607 return obj 608 elif self.serializable_tuple: 609 return list(obj) 610 elif self.serializable_dict: 611 return [value for key, value in sorted(obj.items(), key=lambda x: x[0])] 612 else: 613 pass 614 elif 10 == type_id: 615 if self.serializable_tuple: 616 return obj 617 elif self.serializable_list: 618 return tuple(obj) 619 elif self.serializable_dict: 620 return tuple(value for key, value in sorted(obj.items(), key=lambda x: x[0])) 621 else: 622 pass 623 elif 11 == type_id: 624 if self.serializable_set: 625 return obj 626 elif self.serializable_frozenset: 627 return set(obj) 628 elif self.serializable_tuple: 629 return set(obj) 630 elif self.serializable_list: 631 return set(obj) 632 elif self.serializable_dict: 633 return set(obj.keys()) 634 else: 635 pass 636 elif 12 == type_id: 637 if self.serializable_frozenset: 638 return obj 639 elif self.serializable_set: 640 return frozenset(obj) 641 elif self.serializable_tuple: 642 return frozenset(obj) 643 elif self.serializable_list: 644 return frozenset(obj) 645 elif self.serializable_dict: 646 return frozenset(obj.keys()) 647 else: 648 pass 649 elif 13 == type_id: 650 if self.serializable_dict: 651 return obj 652 elif self.serializable_tuple: 653 return dict(obj) 654 elif self.serializable_list: 655 return dict(obj) 656 else: 657 pass 658 elif 14 == type_id: 659 if self.serializable_slice: 660 return obj 661 elif self.serializable_tuple: 662 return slice(*obj) 663 elif self.serializable_list: 664 return slice(*obj) 665 elif self.serializable_dict: 666 return slice(obj[0], obj[1], obj[2]) 667 else: 668 pass 669 else: 670 raise RuntimeError('Unknown type_id') 671 672 raise CanNotAdjustFromSerializableError(f'Can not adjust from serializable. Type: {type_id}, obj: {obj}')
684 def serialize_container(self, type_id: DataType, obj: Any) -> Any: 685 if 9 == type_id: 686 new_obj = list() 687 for item in obj: 688 exists, value = self.serialize_impl(item) 689 if exists: 690 new_obj.append(value) 691 elif 10 == type_id: 692 new_obj = list() 693 for item in obj: 694 exists, value = self.serialize_impl(item) 695 if exists: 696 new_obj.append(value) 697 698 new_obj = tuple(new_obj) 699 elif 11 == type_id: 700 new_obj = set() 701 for item in obj: 702 exists, value = self.serialize_impl(item) 703 if exists: 704 new_obj.add(value) 705 elif 12 == type_id: 706 new_obj = set() 707 for item in obj: 708 exists, value = self.serialize_impl(item) 709 if exists: 710 new_obj.add(value) 711 712 new_obj = frozenset(new_obj) 713 elif 13 == type_id: 714 new_obj = dict() 715 for key, value in obj.items(): 716 key_exists, key_value = self.serialize_impl(key) 717 value_exists, value_value = self.serialize_impl(value) 718 if key_exists and value_exists: 719 new_obj[key_value] = value_value 720 721 return new_obj
684 def serialize_container(self, type_id: DataType, obj: Any) -> Any: 685 if 9 == type_id: 686 new_obj = list() 687 for item in obj: 688 exists, value = self.serialize_impl(item) 689 if exists: 690 new_obj.append(value) 691 elif 10 == type_id: 692 new_obj = list() 693 for item in obj: 694 exists, value = self.serialize_impl(item) 695 if exists: 696 new_obj.append(value) 697 698 new_obj = tuple(new_obj) 699 elif 11 == type_id: 700 new_obj = set() 701 for item in obj: 702 exists, value = self.serialize_impl(item) 703 if exists: 704 new_obj.add(value) 705 elif 12 == type_id: 706 new_obj = set() 707 for item in obj: 708 exists, value = self.serialize_impl(item) 709 if exists: 710 new_obj.add(value) 711 712 new_obj = frozenset(new_obj) 713 elif 13 == type_id: 714 new_obj = dict() 715 for key, value in obj.items(): 716 key_exists, key_value = self.serialize_impl(key) 717 value_exists, value_value = self.serialize_impl(value) 718 if key_exists and value_exists: 719 new_obj[key_value] = value_value 720 721 return new_obj
def
serialize_impl(self, obj: Any, ignore_empty_classes: bool = False) -> Tuple[bool, int]:
725 def serialize_impl(self, obj: Any, ignore_empty_classes: bool = False) -> Tuple[bool, int]: 726 result_exists: bool = True 727 id_: int = id(obj) 728 obj_type = type(obj) 729 if (int != obj_type) and (id_ in self.objects_ids_by_id): 730 new_object: bool = False 731 object_id: int = self.objects_ids_by_id[id_] 732 else: 733 # int object must always produce new object_id because first 256 ints are persistent across Python sessions and 734 # this can cause issues within users of current module. For example within `cengal/hardware/memory/shared_memory` 735 # which changes int values inline instead of producing new objects. 736 new_object = True 737 object_id = self.objects_id_gen() 738 self.objects_ids[object_id] = id_ 739 self.objects_db[object_id] = obj 740 741 if not new_object: 742 return ResultExistence[int](result_exists, object_id) 743 744 if self.serializable_any or (obj_type in self.serializable_data_types): 745 serializable: bool = True 746 type_id: int = data_type_by_type.get(obj_type, 15) 747 else: 748 serializable = False 749 type_id = data_type_by_type.get(obj_type, 0) 750 751 if obj_type in known_container_types: 752 known_container: bool = True 753 else: 754 known_container = False 755 756 class_info: Dict[int, Any] = None 757 known_data: bool = None 758 class_id: int = None 759 is_new_class: bool = None 760 class_name: str = None 761 new_obj_slots: List[Tuple[str, Any]] = None 762 new_obj_dict: Dict[str, Any] = None 763 obj_mapping: Dict = None 764 obj_sequence: List = None 765 obj_set: Set = None 766 if serializable: 767 if known_container: 768 object_info = ( 769 self.ats(1, object_id), 770 self.ats(1, type_id), 771 self.sc(type_id, obj), 772 ) 773 else: 774 object_info = ( 775 self.ats(1, object_id), 776 self.ats(1, type_id), 777 obj, 778 ) 779 else: 780 if obj_type in known_data_types: 781 known_data = True 782 else: 783 known_data = False 784 785 if 0 == type_id: 786 new_obj_slots = list() 787 for slot_name, slot_value in filled_slot_names_with_values_gen(obj): 788 # if not hasattr(slot_value, '__set__'): 789 # # if not setable descriptor 790 # continue 791 792 if self.serializable_str: 793 adjusted_slot_name = slot_name 794 else: 795 adjusted_slot_name = self.ats(4, slot_name) 796 797 item_exists, item_value = self.serialize_impl(slot_value) 798 if item_exists: 799 if self.serializable_tuple: 800 new_obj_slots.append((adjusted_slot_name, item_value)) 801 elif self.serializable_list: 802 new_obj_slots.append([adjusted_slot_name, item_value]) 803 else: 804 new_obj_slots.append(self.ats(10, (adjusted_slot_name, item_value))) 805 806 if new_obj_slots: 807 if self.serializable_list: 808 pass 809 elif self.serializable_tuple: 810 new_obj_slots = tuple(new_obj_slots) 811 else: 812 new_obj_slots = self.ats(9, new_obj_slots) 813 814 new_obj_dict = dict() 815 if hasattr(obj, '__dict__'): 816 for key, value in obj.__dict__.items(): 817 # raw_value = getattr_static(obj, key) 818 # if hasattr(raw_value, '__get__') and (not hasattr(raw_value, '__set__')): 819 # # if not setable descriptor 820 # continue 821 822 if self.serializable_str: 823 adjusted_key = key 824 else: 825 adjusted_key = self.ats(4, key) 826 827 value_exists, value_value = self.serialize_impl(value) 828 if value_exists: 829 new_obj_dict[adjusted_key] = value_value 830 else: 831 continue 832 833 if new_obj_dict: 834 if self.serializable_dict: 835 pass 836 else: 837 new_obj_dict = self.ats(13, new_obj_dict) 838 839 obj_mapping = dict() 840 obj_sequence = list() 841 obj_set = set() 842 if isinstance(obj, MutableMapping): 843 for key, value in obj.items(): 844 key_exists, key_value = self.serialize_impl(value) 845 if not key_exists: 846 continue 847 848 value_exists, value_value = self.serialize_impl(value) 849 if value_exists: 850 obj_mapping[key_value] = value_value 851 else: 852 continue 853 854 if obj_mapping: 855 if self.serializable_dict: 856 pass 857 else: 858 obj_mapping = self.ats(13, obj_mapping) 859 elif isinstance(obj, MutableSequence): 860 for item in obj: 861 item_exists, item_value = self.serialize_impl(item) 862 if item_exists: 863 obj_sequence.append(item_value) 864 else: 865 continue 866 867 if obj_sequence: 868 if self.serializable_list: 869 pass 870 else: 871 obj_sequence = self.ats(9, obj_sequence) 872 elif isinstance(obj, MutableSet): 873 for item in obj: 874 item_exists, item_value = self.serialize_impl(item) 875 if item_exists: 876 obj_set.add(item_value) 877 else: 878 continue 879 880 if obj_set: 881 if self.serializable_set: 882 pass 883 else: 884 obj_set = self.ats(11, obj_set) 885 else: 886 pass 887 888 if ignore_empty_classes: 889 result_exists = new_obj_slots or new_obj_dict or obj_mapping or obj_sequence or obj_set 890 891 if obj_type in self.known_classes: 892 is_new_class = False 893 class_id = self.known_classes[obj_type] 894 else: 895 is_new_class = True 896 class_name = obj_type.__name__ 897 class_id = self.classes_id_gen() 898 self.known_classes[obj_type] = class_id 899 self.known_classes_by_id[class_id] = obj_type 900 module_importable_str, owning_names_path = entity_module_importable_str_and_owning_names_path(obj) 901 adjusted_owning_names_path: List[str] = [self.ats(4, name) for name in owning_names_path] 902 module_importable_str_and_owning_names_path = (class_name, module_importable_str, tuple(owning_names_path)) 903 self.known_classes_info[module_importable_str_and_owning_names_path] = class_id 904 self.known_classes_info_by_id[class_id] = module_importable_str_and_owning_names_path 905 906 if result_exists: 907 class_info = ( 908 self.ats(1, class_id), 909 self.ats(4, class_name), 910 self.ats(4, module_importable_str), 911 self.ats(9, adjusted_owning_names_path), 912 ) 913 914 if self.serializable_tuple: 915 adjusted_class_info = class_info 916 else: 917 adjusted_class_info = self.ats(10, class_info) 918 919 if self.on_new_class_handler: 920 self.on_new_class_handler( 921 adjusted_class_info, 922 obj_type, 923 class_id, 924 class_name, 925 module_importable_str, 926 owning_names_path, 927 module_importable_str_and_owning_names_path 928 ) 929 930 # will be later serialized much faster than without `if else None` 931 object_info = ( 932 self.ats(1, object_id), 933 self.ats(1, type_id), 934 self.ats(8, None), 935 self.ats(1, class_id), 936 new_obj_slots if new_obj_slots else self.ats(9, list()), 937 new_obj_dict if new_obj_dict else self.ats(13, dict()), 938 obj_mapping if obj_mapping else self.ats(13, dict()), 939 obj_sequence if obj_sequence else self.ats(9, list()), 940 obj_set if obj_set else self.ats(11, set()), 941 ) 942 elif known_data: 943 object_info = ( 944 self.ats(1, object_id), 945 self.ats(1, type_id), 946 self.ats(type_id, obj), 947 ) 948 elif known_container: 949 object_info = ( 950 self.ats(1, object_id), 951 self.ats(1, type_id), 952 self.ats(type_id, self.sc(type_id, obj)), 953 ) 954 else: 955 raise RuntimeError('Unknown type_id') 956 957 if self.serializable_tuple: 958 adjusted_object_info = object_info 959 else: 960 adjusted_object_info = self.ats(10, object_info) 961 962 if result_exists: 963 if self.on_new_obj_info_handler: 964 self.on_new_obj_info_handler( 965 adjusted_object_info, 966 obj, 967 object_id, 968 type_id, 969 serializable, 970 known_container, 971 known_data, 972 class_id, 973 is_new_class, 974 new_obj_slots, 975 new_obj_dict, 976 obj_mapping, 977 obj_sequence, 978 obj_set, 979 ) 980 981 return result_exists, object_id
def
si(self, obj: Any, ignore_empty_classes: bool = False) -> Tuple[bool, int]:
725 def serialize_impl(self, obj: Any, ignore_empty_classes: bool = False) -> Tuple[bool, int]: 726 result_exists: bool = True 727 id_: int = id(obj) 728 obj_type = type(obj) 729 if (int != obj_type) and (id_ in self.objects_ids_by_id): 730 new_object: bool = False 731 object_id: int = self.objects_ids_by_id[id_] 732 else: 733 # int object must always produce new object_id because first 256 ints are persistent across Python sessions and 734 # this can cause issues within users of current module. For example within `cengal/hardware/memory/shared_memory` 735 # which changes int values inline instead of producing new objects. 736 new_object = True 737 object_id = self.objects_id_gen() 738 self.objects_ids[object_id] = id_ 739 self.objects_db[object_id] = obj 740 741 if not new_object: 742 return ResultExistence[int](result_exists, object_id) 743 744 if self.serializable_any or (obj_type in self.serializable_data_types): 745 serializable: bool = True 746 type_id: int = data_type_by_type.get(obj_type, 15) 747 else: 748 serializable = False 749 type_id = data_type_by_type.get(obj_type, 0) 750 751 if obj_type in known_container_types: 752 known_container: bool = True 753 else: 754 known_container = False 755 756 class_info: Dict[int, Any] = None 757 known_data: bool = None 758 class_id: int = None 759 is_new_class: bool = None 760 class_name: str = None 761 new_obj_slots: List[Tuple[str, Any]] = None 762 new_obj_dict: Dict[str, Any] = None 763 obj_mapping: Dict = None 764 obj_sequence: List = None 765 obj_set: Set = None 766 if serializable: 767 if known_container: 768 object_info = ( 769 self.ats(1, object_id), 770 self.ats(1, type_id), 771 self.sc(type_id, obj), 772 ) 773 else: 774 object_info = ( 775 self.ats(1, object_id), 776 self.ats(1, type_id), 777 obj, 778 ) 779 else: 780 if obj_type in known_data_types: 781 known_data = True 782 else: 783 known_data = False 784 785 if 0 == type_id: 786 new_obj_slots = list() 787 for slot_name, slot_value in filled_slot_names_with_values_gen(obj): 788 # if not hasattr(slot_value, '__set__'): 789 # # if not setable descriptor 790 # continue 791 792 if self.serializable_str: 793 adjusted_slot_name = slot_name 794 else: 795 adjusted_slot_name = self.ats(4, slot_name) 796 797 item_exists, item_value = self.serialize_impl(slot_value) 798 if item_exists: 799 if self.serializable_tuple: 800 new_obj_slots.append((adjusted_slot_name, item_value)) 801 elif self.serializable_list: 802 new_obj_slots.append([adjusted_slot_name, item_value]) 803 else: 804 new_obj_slots.append(self.ats(10, (adjusted_slot_name, item_value))) 805 806 if new_obj_slots: 807 if self.serializable_list: 808 pass 809 elif self.serializable_tuple: 810 new_obj_slots = tuple(new_obj_slots) 811 else: 812 new_obj_slots = self.ats(9, new_obj_slots) 813 814 new_obj_dict = dict() 815 if hasattr(obj, '__dict__'): 816 for key, value in obj.__dict__.items(): 817 # raw_value = getattr_static(obj, key) 818 # if hasattr(raw_value, '__get__') and (not hasattr(raw_value, '__set__')): 819 # # if not setable descriptor 820 # continue 821 822 if self.serializable_str: 823 adjusted_key = key 824 else: 825 adjusted_key = self.ats(4, key) 826 827 value_exists, value_value = self.serialize_impl(value) 828 if value_exists: 829 new_obj_dict[adjusted_key] = value_value 830 else: 831 continue 832 833 if new_obj_dict: 834 if self.serializable_dict: 835 pass 836 else: 837 new_obj_dict = self.ats(13, new_obj_dict) 838 839 obj_mapping = dict() 840 obj_sequence = list() 841 obj_set = set() 842 if isinstance(obj, MutableMapping): 843 for key, value in obj.items(): 844 key_exists, key_value = self.serialize_impl(value) 845 if not key_exists: 846 continue 847 848 value_exists, value_value = self.serialize_impl(value) 849 if value_exists: 850 obj_mapping[key_value] = value_value 851 else: 852 continue 853 854 if obj_mapping: 855 if self.serializable_dict: 856 pass 857 else: 858 obj_mapping = self.ats(13, obj_mapping) 859 elif isinstance(obj, MutableSequence): 860 for item in obj: 861 item_exists, item_value = self.serialize_impl(item) 862 if item_exists: 863 obj_sequence.append(item_value) 864 else: 865 continue 866 867 if obj_sequence: 868 if self.serializable_list: 869 pass 870 else: 871 obj_sequence = self.ats(9, obj_sequence) 872 elif isinstance(obj, MutableSet): 873 for item in obj: 874 item_exists, item_value = self.serialize_impl(item) 875 if item_exists: 876 obj_set.add(item_value) 877 else: 878 continue 879 880 if obj_set: 881 if self.serializable_set: 882 pass 883 else: 884 obj_set = self.ats(11, obj_set) 885 else: 886 pass 887 888 if ignore_empty_classes: 889 result_exists = new_obj_slots or new_obj_dict or obj_mapping or obj_sequence or obj_set 890 891 if obj_type in self.known_classes: 892 is_new_class = False 893 class_id = self.known_classes[obj_type] 894 else: 895 is_new_class = True 896 class_name = obj_type.__name__ 897 class_id = self.classes_id_gen() 898 self.known_classes[obj_type] = class_id 899 self.known_classes_by_id[class_id] = obj_type 900 module_importable_str, owning_names_path = entity_module_importable_str_and_owning_names_path(obj) 901 adjusted_owning_names_path: List[str] = [self.ats(4, name) for name in owning_names_path] 902 module_importable_str_and_owning_names_path = (class_name, module_importable_str, tuple(owning_names_path)) 903 self.known_classes_info[module_importable_str_and_owning_names_path] = class_id 904 self.known_classes_info_by_id[class_id] = module_importable_str_and_owning_names_path 905 906 if result_exists: 907 class_info = ( 908 self.ats(1, class_id), 909 self.ats(4, class_name), 910 self.ats(4, module_importable_str), 911 self.ats(9, adjusted_owning_names_path), 912 ) 913 914 if self.serializable_tuple: 915 adjusted_class_info = class_info 916 else: 917 adjusted_class_info = self.ats(10, class_info) 918 919 if self.on_new_class_handler: 920 self.on_new_class_handler( 921 adjusted_class_info, 922 obj_type, 923 class_id, 924 class_name, 925 module_importable_str, 926 owning_names_path, 927 module_importable_str_and_owning_names_path 928 ) 929 930 # will be later serialized much faster than without `if else None` 931 object_info = ( 932 self.ats(1, object_id), 933 self.ats(1, type_id), 934 self.ats(8, None), 935 self.ats(1, class_id), 936 new_obj_slots if new_obj_slots else self.ats(9, list()), 937 new_obj_dict if new_obj_dict else self.ats(13, dict()), 938 obj_mapping if obj_mapping else self.ats(13, dict()), 939 obj_sequence if obj_sequence else self.ats(9, list()), 940 obj_set if obj_set else self.ats(11, set()), 941 ) 942 elif known_data: 943 object_info = ( 944 self.ats(1, object_id), 945 self.ats(1, type_id), 946 self.ats(type_id, obj), 947 ) 948 elif known_container: 949 object_info = ( 950 self.ats(1, object_id), 951 self.ats(1, type_id), 952 self.ats(type_id, self.sc(type_id, obj)), 953 ) 954 else: 955 raise RuntimeError('Unknown type_id') 956 957 if self.serializable_tuple: 958 adjusted_object_info = object_info 959 else: 960 adjusted_object_info = self.ats(10, object_info) 961 962 if result_exists: 963 if self.on_new_obj_info_handler: 964 self.on_new_obj_info_handler( 965 adjusted_object_info, 966 obj, 967 object_id, 968 type_id, 969 serializable, 970 known_container, 971 known_data, 972 class_id, 973 is_new_class, 974 new_obj_slots, 975 new_obj_dict, 976 obj_mapping, 977 obj_sequence, 978 obj_set, 979 ) 980 981 return result_exists, object_id
def
deserialize_class_impl(self, class_info: Tuple) -> Tuple[bool, int, Type]:
991 def deserialize_class_impl(self, class_info: Tuple) -> Tuple[bool, int, Type]: 992 if self.serializable_tuple: 993 pass 994 else: 995 class_info = self.afs(10, class_info) 996 997 class_id: int = self.afs(1, class_info[0]) 998 if class_id in self.known_classes_by_id: 999 return ResultExistence[Tuple[int, Type]](True, (class_id, self.known_classes_by_id[class_id])) 1000 else: 1001 class_name: str = self.afs(4, class_info[1]) 1002 module_importable_str: str = self.afs(4, class_info[2]) 1003 owning_names_path: List[str] = self.afs(9, class_info[3]) 1004 adjusted_owning_names_path: List[str] = [self.afs(4, name) for name in owning_names_path] 1005 obj_class: Type = entity_by_name_module_importable_str_and_owning_names_path(class_name, module_importable_str, adjusted_owning_names_path) 1006 self.known_classes_by_id[class_id] = obj_class 1007 self.known_classes[obj_class] = class_id 1008 class_info_tuple: Tuple = (class_name, module_importable_str, tuple(owning_names_path)) 1009 self.known_classes_info[class_info_tuple] = class_id 1010 self.known_classes_info_by_id[class_id] = class_info_tuple 1011 return True, class_id, obj_class
def
dcli(self, class_info: Tuple) -> Tuple[bool, int, Type]:
991 def deserialize_class_impl(self, class_info: Tuple) -> Tuple[bool, int, Type]: 992 if self.serializable_tuple: 993 pass 994 else: 995 class_info = self.afs(10, class_info) 996 997 class_id: int = self.afs(1, class_info[0]) 998 if class_id in self.known_classes_by_id: 999 return ResultExistence[Tuple[int, Type]](True, (class_id, self.known_classes_by_id[class_id])) 1000 else: 1001 class_name: str = self.afs(4, class_info[1]) 1002 module_importable_str: str = self.afs(4, class_info[2]) 1003 owning_names_path: List[str] = self.afs(9, class_info[3]) 1004 adjusted_owning_names_path: List[str] = [self.afs(4, name) for name in owning_names_path] 1005 obj_class: Type = entity_by_name_module_importable_str_and_owning_names_path(class_name, module_importable_str, adjusted_owning_names_path) 1006 self.known_classes_by_id[class_id] = obj_class 1007 self.known_classes[obj_class] = class_id 1008 class_info_tuple: Tuple = (class_name, module_importable_str, tuple(owning_names_path)) 1009 self.known_classes_info[class_info_tuple] = class_id 1010 self.known_classes_info_by_id[class_id] = class_info_tuple 1011 return True, class_id, obj_class
1021 def deserialize_container_impl(self, type_id: DataType, obj: Any) -> Any: 1022 new_obj = obj 1023 if 9 == type_id: 1024 new_obj = list() 1025 for item_id in obj: 1026 new_obj.append(self.objects_db[self.afs(1, item_id)]) 1027 elif 10 == type_id: 1028 new_obj = list() 1029 for item_id in obj: 1030 new_obj.append(self.objects_db[self.afs(1, item_id)]) 1031 1032 new_obj = tuple(new_obj) 1033 elif 11 == type_id: 1034 new_obj = set() 1035 for item_id in obj: 1036 new_obj.add(self.objects_db[self.afs(1, item_id)]) 1037 elif 12 == type_id: 1038 new_obj = set() 1039 for item_id in obj: 1040 new_obj.add(self.objects_db[self.afs(1, item_id)]) 1041 1042 new_obj = frozenset(new_obj) 1043 elif 13 == type_id: 1044 new_obj = dict() 1045 for key_id, value_id in obj.items(): 1046 new_obj[self.objects_db[int(self.afs(1, key_id))]] = self.objects_db[self.afs(1, value_id)] 1047 1048 return new_obj
1021 def deserialize_container_impl(self, type_id: DataType, obj: Any) -> Any: 1022 new_obj = obj 1023 if 9 == type_id: 1024 new_obj = list() 1025 for item_id in obj: 1026 new_obj.append(self.objects_db[self.afs(1, item_id)]) 1027 elif 10 == type_id: 1028 new_obj = list() 1029 for item_id in obj: 1030 new_obj.append(self.objects_db[self.afs(1, item_id)]) 1031 1032 new_obj = tuple(new_obj) 1033 elif 11 == type_id: 1034 new_obj = set() 1035 for item_id in obj: 1036 new_obj.add(self.objects_db[self.afs(1, item_id)]) 1037 elif 12 == type_id: 1038 new_obj = set() 1039 for item_id in obj: 1040 new_obj.add(self.objects_db[self.afs(1, item_id)]) 1041 1042 new_obj = frozenset(new_obj) 1043 elif 13 == type_id: 1044 new_obj = dict() 1045 for key_id, value_id in obj.items(): 1046 new_obj[self.objects_db[int(self.afs(1, key_id))]] = self.objects_db[self.afs(1, value_id)] 1047 1048 return new_obj
def
deserialize_obj_impl(self, obj_info: Tuple) -> Tuple[bool, int, Any]:
1058 def deserialize_obj_impl(self, obj_info: Tuple) -> Tuple[bool, int, Any]: 1059 if self.serializable_tuple: 1060 pass 1061 else: 1062 obj_info = self.afs(10, obj_info) 1063 1064 object_id: int = self.afs(1, obj_info[0]) 1065 if object_id in self.objects_db: 1066 return ResultExistence[Tuple[int, Any]](True, (object_id, self.objects_db[object_id])) 1067 1068 type_id: int = self.afs(1, obj_info[1]) 1069 1070 obj_type = data_type[type_id] 1071 serializable: bool = self.serializable_any or (obj_type in self.serializable_data_types) 1072 known_container: bool = obj_type in known_container_types 1073 known_data: bool = None 1074 1075 if serializable: 1076 if known_container: 1077 obj = self.dcoi(type_id, obj_info[2]) 1078 else: 1079 obj = obj_info[2] 1080 else: 1081 known_data = obj_type in known_data_types 1082 1083 if 0 == type_id: 1084 class_id: int = self.afs(1, obj_info[3]) 1085 obj_class: Type = self.known_classes_by_id[class_id] 1086 obj: Any = obj_class.__new__(obj_class) 1087 1088 if obj_info[4]: 1089 clonable_slots = self.afs(9, obj_info[4]) 1090 for slot_name, slot_id in clonable_slots: 1091 slot_name = self.afs(4, slot_name) 1092 slot_id = self.afs(1, slot_id) 1093 child_obj = self.objects_db[slot_id] 1094 setattr(obj, slot_name, child_obj) 1095 1096 if obj_info[5]: 1097 clonable_dict_items = self.afs(13, obj_info[5]) 1098 for key, value_id in clonable_dict_items.items(): 1099 key = self.afs(4, key) 1100 value_id = self.afs(1, value_id) 1101 child_obj = self.objects_db[value_id] 1102 setattr(obj, key, child_obj) 1103 1104 if obj_info[6]: 1105 contained_mapping = self.afs(13, obj_info[6]) 1106 for key_id, value_id in contained_mapping.items(): 1107 key_id = self.afs(1, key_id) 1108 value_id = self.afs(1, value_id) 1109 child_key = self.objects_db[int(key_id)] 1110 child_value = self.objects_db[value_id] 1111 obj[child_key] = child_value 1112 1113 if obj_info[7]: 1114 contained_sequence = self.afs(9, obj_info[7]) 1115 for item_id in contained_sequence: 1116 item_id = self.afs(1, item_id) 1117 child_item = self.objects_db[item_id] 1118 obj.append(child_item) 1119 1120 if obj_info[8]: 1121 contained_set = self.afs(11, obj_info[8]) 1122 for item_id in contained_set: 1123 item_id = self.afs(1, item_id) 1124 child_item = self.objects_db[item_id] 1125 obj.add(child_item) 1126 elif known_data: 1127 obj = self.afs(type_id, obj_info[2]) 1128 elif known_container: 1129 obj = self.dcoi(type_id, self.afs(type_id, obj_info[2])) 1130 else: 1131 raise RuntimeError('Unknown type_id') 1132 1133 self.objects_db[object_id] = obj 1134 id_: int = id(obj) 1135 self.objects_ids[object_id] = id_ 1136 self.objects_ids_by_id[id_] = object_id 1137 return True, object_id, obj
def
doi(self, obj_info: Tuple) -> Tuple[bool, int, Any]:
1058 def deserialize_obj_impl(self, obj_info: Tuple) -> Tuple[bool, int, Any]: 1059 if self.serializable_tuple: 1060 pass 1061 else: 1062 obj_info = self.afs(10, obj_info) 1063 1064 object_id: int = self.afs(1, obj_info[0]) 1065 if object_id in self.objects_db: 1066 return ResultExistence[Tuple[int, Any]](True, (object_id, self.objects_db[object_id])) 1067 1068 type_id: int = self.afs(1, obj_info[1]) 1069 1070 obj_type = data_type[type_id] 1071 serializable: bool = self.serializable_any or (obj_type in self.serializable_data_types) 1072 known_container: bool = obj_type in known_container_types 1073 known_data: bool = None 1074 1075 if serializable: 1076 if known_container: 1077 obj = self.dcoi(type_id, obj_info[2]) 1078 else: 1079 obj = obj_info[2] 1080 else: 1081 known_data = obj_type in known_data_types 1082 1083 if 0 == type_id: 1084 class_id: int = self.afs(1, obj_info[3]) 1085 obj_class: Type = self.known_classes_by_id[class_id] 1086 obj: Any = obj_class.__new__(obj_class) 1087 1088 if obj_info[4]: 1089 clonable_slots = self.afs(9, obj_info[4]) 1090 for slot_name, slot_id in clonable_slots: 1091 slot_name = self.afs(4, slot_name) 1092 slot_id = self.afs(1, slot_id) 1093 child_obj = self.objects_db[slot_id] 1094 setattr(obj, slot_name, child_obj) 1095 1096 if obj_info[5]: 1097 clonable_dict_items = self.afs(13, obj_info[5]) 1098 for key, value_id in clonable_dict_items.items(): 1099 key = self.afs(4, key) 1100 value_id = self.afs(1, value_id) 1101 child_obj = self.objects_db[value_id] 1102 setattr(obj, key, child_obj) 1103 1104 if obj_info[6]: 1105 contained_mapping = self.afs(13, obj_info[6]) 1106 for key_id, value_id in contained_mapping.items(): 1107 key_id = self.afs(1, key_id) 1108 value_id = self.afs(1, value_id) 1109 child_key = self.objects_db[int(key_id)] 1110 child_value = self.objects_db[value_id] 1111 obj[child_key] = child_value 1112 1113 if obj_info[7]: 1114 contained_sequence = self.afs(9, obj_info[7]) 1115 for item_id in contained_sequence: 1116 item_id = self.afs(1, item_id) 1117 child_item = self.objects_db[item_id] 1118 obj.append(child_item) 1119 1120 if obj_info[8]: 1121 contained_set = self.afs(11, obj_info[8]) 1122 for item_id in contained_set: 1123 item_id = self.afs(1, item_id) 1124 child_item = self.objects_db[item_id] 1125 obj.add(child_item) 1126 elif known_data: 1127 obj = self.afs(type_id, obj_info[2]) 1128 elif known_container: 1129 obj = self.dcoi(type_id, self.afs(type_id, obj_info[2])) 1130 else: 1131 raise RuntimeError('Unknown type_id') 1132 1133 self.objects_db[object_id] = obj 1134 id_: int = id(obj) 1135 self.objects_ids[object_id] = id_ 1136 self.objects_ids_by_id[id_] = object_id 1137 return True, object_id, obj