cengal.data_manipulation.remote_objects.versions.v_0_fast_optimized.remote_objects

View Source

   1#!/usr/bin/env python
   2# coding=utf-8
   3
   4# Copyright © 2012-2024 ButenkoMS. All rights reserved. Contacts: <gtalk@butenkoms.space>
   5# 
   6# Licensed under the Apache License, Version 2.0 (the "License");
   7# you may not use this file except in compliance with the License.
   8# You may obtain a copy of the License at
   9# 
  10#     http://www.apache.org/licenses/LICENSE-2.0
  11# 
  12# Unless required by applicable law or agreed to in writing, software
  13# distributed under the License is distributed on an "AS IS" BASIS,
  14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15# See the License for the specific language governing permissions and
  16# limitations under the License.
  17
  18
  19__all__ = [
  20    'default_serializable_data_types',
  21    'known_types',
  22    'known_data_types',
  23    'known_container_types',
  24    'DataType',
  25    'data_type',
  26    'data_type_by_type',
  27    'ClassInfoFields',
  28    'ObjectInfoFields',
  29    'CanNotAdjustToSerializableError',
  30    'CanNotAdjustFromSerializableError',
  31    'RemoteObjectsManager',
  32]
  33
  34
  35from cengal.introspection.inspect import (
  36    entity_module_importable_str_and_owning_names_path,
  37    entity_by_name_module_importable_str_and_owning_names_path,
  38    filled_slot_names_with_values_gen,
  39    is_callable, is_async,
  40    is_setable_data_descriptor,
  41)
  42from cengal.code_flow_control.smart_values import ResultExistence
  43from cengal.code_flow_control.gc import DisableGC
  44from cengal.data_generation.id_generator import IDGenerator, GeneratorType
  45from enum import IntEnum
  46from collections.abc import MutableMapping, MutableSequence, MutableSet
  47from struct import pack, unpack
  48from inspect import getattr_static
  49from typing import Any, Dict, Optional, Callable, Set, Type, Tuple, List, FrozenSet
  50
  51
  52"""
  53Module Docstring
  54Docstrings: http://www.python.org/dev/peps/pep-0257/
  55"""
  56
  57__author__ = "ButenkoMS <gtalk@butenkoms.space>"
  58__copyright__ = "Copyright © 2012-2024 ButenkoMS. All rights reserved. Contacts: <gtalk@butenkoms.space>"
  59__credits__ = ["ButenkoMS <gtalk@butenkoms.space>", ]
  60__license__ = "Apache License, Version 2.0"
  61__version__ = "4.4.1"
  62__maintainer__ = "ButenkoMS <gtalk@butenkoms.space>"
  63__email__ = "gtalk@butenkoms.space"
  64# __status__ = "Prototype"
  65__status__ = "Development"
  66# __status__ = "Production"
  67
  68
  69default_serializable_data_types: Set[Type] = {
  70    int, float, complex, str, bytes, bytearray, bool, type(None), list, tuple, set, frozenset, dict
  71}
  72
  73
  74known_types: Set[Type] = {
  75    int, float, complex, str, bytes, bytearray, bool, type(None), list, tuple, set, frozenset, dict,
  76    slice, 
  77}
  78
  79
  80known_data_types: Set[Type] = {
  81    int, float, str, bytes, bytearray, bool, type(None), slice,
  82}
  83
  84
  85known_container_types: Set[Type] = {
  86    complex, list, tuple, set, frozenset, dict,
  87}
  88
  89
  90class DataType(IntEnum):
  91    class_ = 0  # type is not in known_types and not in serializable_data_types
  92    int_ = 1
  93    float_ = 2
  94    complex_ = 3
  95    str_ = 4
  96    bytes_ = 5
  97    bytearray_ = 6
  98    bool_ = 7
  99    none_ = 8
 100    list_ = 9
 101    tuple_ = 10
 102    set_ = 11
 103    frozenset_ = 12
 104    dict_ = 13
 105    slice_ = 14
 106    unknown_serializable = 15  # type is in serializable_data_types but not in known_types
 107
 108
 109data_type: Dict[DataType, Type] = {
 110    0: object,
 111    1: int,
 112    2: float,
 113    3: complex,
 114    4: str,
 115    5: bytes,
 116    6: bytearray,
 117    7: bool,
 118    8: type(None),
 119    9: list,
 120    10: tuple,
 121    11: set,
 122    12: frozenset,
 123    13: dict,
 124    14: slice,
 125    15: None,
 126}
 127
 128
 129data_type_by_type: Dict[Type, DataType] = {
 130    object: 0,
 131    int: 1,
 132    float: 2,
 133    complex: 3,
 134    str: 4,
 135    bytes: 5,
 136    bytearray: 6,
 137    bool: 7,
 138    type(None): 8,
 139    list: 9,
 140    tuple: 10,
 141    set: 11,
 142    frozenset: 12,
 143    dict: 13,
 144    slice: 14,
 145    None: 15,
 146}
 147
 148
 149class ClassInfoFields(IntEnum):
 150    class_id = 0
 151    class_name = 1
 152    module_importable_str = 2
 153    owning_names_path = 3
 154
 155
 156class ObjectInfoFields(IntEnum):
 157    object_id = 0  # (type: int)
 158    type_id = 1  # (type: DataType)
 159    object_ = 2  # (Optional), (type: Any). Link to object itself if `(type(obj) in serializable_data_types)`. Not used otherwise.
 160    class_id = 3  # (Optional), (type: int). Used if `type_id == 0`
 161    clonable_slots = 4  # (Optional), (type: Tuple[Tuple[str, Any]]). Used if `type_id == 0`. Holds ID's (object_id) of slots objects.
 162    clonable_dict_items = 5  # (Optional), (type: Tuple[Tuple[str, Any]]). Used if `type_id == 0`. Holds ID's (object_id) of value objects.
 163    # contained_items = 6  # (Optional), (type: Union[Tuple, List, Set, FrozenSet, Dict]). Used if `type_id in {9, 10, 11, 12, 13, 14}`. Holds ID's (object_id) of contained items (for bothe keys and values in the case of Dict).
 164    contained_mapping = 6  # (Optional), (type: Union[Tuple, List, Set, FrozenSet, Dict]). Used if `type_id in {9, 10, 11, 12, 13, 14}`. Holds ID's (object_id) of contained items (for bothe keys and values in the case of Dict).
 165    contained_sequence = 7  # (Optional), (type: Union[Tuple, List, Set, FrozenSet, Dict]). Used if `type_id in {9, 10, 11, 12, 13, 14}`. Holds ID's (object_id) of contained items (for bothe keys and values in the case of Dict).
 166    contained_set = 8  # (Optional), (type: Union[Tuple, List, Set, FrozenSet, Dict]). Used if `type_id in {9, 10, 11, 12, 13, 14}`. Holds ID's (object_id) of contained items (for bothe keys and values in the case of Dict).
 167
 168
 169class CanNotAdjustToSerializableError(Exception):
 170    pass
 171
 172
 173class CanNotAdjustFromSerializableError(Exception):
 174    pass
 175
 176
 177class RemoteObjectsManager:
 178    def __init__(self, 
 179                 on_new_class_handler: Optional[Callable] = None,
 180                 on_new_obj_info_handler: Optional[Callable] = None,
 181                 objects_db: Optional[Dict[int, Any]] = None, 
 182                 serializable_data_types: Optional[Set[Type]] = None,
 183                 ) -> None:
 184        # self.classess_db: Dict[int, Type] = dict() if classess_db is None else classess_db
 185        self.classes_id_gen: int = 0
 186        self.objects_db: Dict[int, Any] = dict() if objects_db is None else objects_db
 187        self.objects_id_gen: int = 0
 188        self.serializable_data_types: Set[Type] = default_serializable_data_types if serializable_data_types is None else serializable_data_types
 189        self.serializable_any: bool = not self.serializable_data_types
 190        self.serializable_int: bool = (int in self.serializable_data_types) or self.serializable_any
 191        self.serializable_float: bool = (float in self.serializable_data_types) or self.serializable_any
 192        self.serializable_complex: bool = (complex in self.serializable_data_types) or self.serializable_any
 193        self.serializable_str: bool = (str in self.serializable_data_types) or self.serializable_any
 194        self.serializable_bytes: bool = (bytes in self.serializable_data_types) or self.serializable_any
 195        self.serializable_bytearray: bool = (bytearray in self.serializable_data_types) or self.serializable_any
 196        self.serializable_bool: bool = (bool in self.serializable_data_types) or self.serializable_any
 197        self.serializable_none: bool = (type(None) in self.serializable_data_types) or self.serializable_any
 198        self.serializable_list: bool = (list in self.serializable_data_types) or self.serializable_any
 199        self.serializable_tuple: bool = (tuple in self.serializable_data_types) or self.serializable_any
 200        self.serializable_set: bool = (set in self.serializable_data_types) or self.serializable_any
 201        self.serializable_frozenset: bool = (frozenset in self.serializable_data_types) or self.serializable_any
 202        self.serializable_dict: bool = (dict in self.serializable_data_types) or self.serializable_any
 203        self.serializable_slice: bool = (slice in self.serializable_data_types) or self.serializable_any
 204        self.known_classes: Dict[Type, int] = dict()
 205        self.known_classes_by_id: Dict[int, Type] = dict()
 206        self.known_classes_info: Dict[Tuple, int] = dict()
 207        self.known_classes_info_by_id: Dict[int, Tuple] = dict()
 208        self.on_new_class_handler: Optional[Callable] = on_new_class_handler
 209        self.on_new_obj_info_handler: Optional[Callable] = on_new_obj_info_handler
 210        self.objects_ids: Dict[int, int] = dict()  # (Key: object_id, Value: id(obj))
 211        self.objects_ids_by_id: Dict[int, int] = dict()  # (Key: id(obj), Value: object_id)
 212
 213    def del_object_by_id(self, id_: int) -> None:
 214        if id_ in self.objects_ids_by_id:
 215            object_id = self.objects_ids_by_id[id_]
 216            self.objects_ids_by_id.pop(id_, None)
 217            self.objects_ids.pop(object_id, None)
 218            self.objects_db.pop(object_id, None)
 219    
 220    dobi = del_object_by_id
 221    
 222    def del_object_by_object_id(self, object_id: int) -> None:
 223        if object_id in self.objects_ids:
 224            id_ = self.objects_ids[object_id]
 225            self.objects_ids_by_id.pop(id_, None)
 226            self.objects_ids.pop(object_id, None)
 227            self.objects_db.pop(object_id, None)
 228    
 229    doboi = del_object_by_object_id
 230    
 231    def adjust_to_serializable(self, type_id: DataType, obj: Any,
 232                                # id=id,
 233                                # int=int,
 234                                # float=float,
 235                                # complex=complex,
 236                                # str=str,
 237                                # bytes=bytes,
 238                                # bytearray=bytearray,
 239                                # tuple=tuple,
 240                                # list=list,
 241                                # set=set,
 242                                # frozenset=frozenset,
 243                                # dict=dict,
 244                                # slice=slice,
 245                                # enumerate=enumerate,
 246                                # sorted=sorted,
 247                                # isinstance=isinstance,
 248                                # hasattr=hasattr,
 249                                # getattr=getattr,
 250                                # setattr=setattr,
 251                                int=int,
 252                                float=float,
 253                                str=str,
 254                                bytes=bytes,
 255                                bytearray=bytearray,
 256                                tuple=tuple,
 257                                list=list,
 258                                set=set,
 259                                frozenset=frozenset,
 260                                dict=dict,
 261                                enumerate=enumerate,
 262                               ) -> Any:
 263        # serialisable_any: bool = self.serializable_any
 264        # serialisable_int: bool = self.serializable_int
 265        # serialisable_float: bool = self.serializable_float
 266        # serialisable_complex: bool = self.serializable_complex
 267        # serialisable_str: bool = self.serializable_str
 268        # serialisable_bytes: bool = self.serializable_bytes
 269        # serialisable_bytearray: bool = self.serializable_bytearray
 270        # serialisable_bool: bool = self.serializable_bool
 271        # serialisable_none: bool = self.serializable_none
 272        # serialisable_list: bool = self.serializable_list
 273        # serialisable_tuple: bool = self.serializable_tuple
 274        # serialisable_set: bool = self.serializable_set
 275        # serialisable_frozenset: bool = self.serializable_frozenset
 276        # serialisable_dict: bool = self.serializable_dict
 277        # serialisable_slice: bool = self.serializable_slice
 278
 279        if 1 == type_id:
 280            return obj
 281        elif 2 == type_id:
 282            return obj
 283        elif 3 == type_id:
 284            if self.serializable_complex:
 285                return obj
 286            elif self.serializable_bytes:
 287                return pack('=dd', obj.real, obj.imag)
 288            elif self.serializable_bytearray:
 289                return bytearray(pack('=dd', obj.real, obj.imag))
 290            elif self.serializable_str:
 291                return str(obj)
 292            elif self.serializable_float:
 293                if self.serializable_tuple:
 294                    return (obj.real, obj.imag)
 295                elif self.serializable_list:
 296                    return [obj.real, obj.imag]
 297                else:
 298                    pass
 299            else:
 300                pass
 301        elif 4 == type_id:
 302            return obj
 303        elif 5 == type_id:
 304            if self.serializable_bytes:
 305                return obj
 306            elif self.serializable_bytearray:
 307                return bytearray(obj)
 308            elif self.serializable_str:
 309                return obj.hex()
 310            elif self.serializable_tuple:
 311                if self.serializable_int:
 312                    return  tuple(int(c) for c in obj)
 313                if self.serializable_float:
 314                    return  tuple(float(int(c)) for c in obj)
 315                else:
 316                    pass
 317            elif self.serializable_list:
 318                if self.serializable_int:
 319                    return  [int(c) for c in obj]
 320                if self.serializable_float:
 321                    return  [float(int(c)) for c in obj]
 322                else:
 323                    pass
 324            else:
 325                pass
 326        elif 6 == type_id:
 327            if self.serializable_bytearray:
 328                return obj
 329            elif self.serializable_bytes:
 330                return bytes(obj)
 331            elif self.serializable_str:
 332                return obj.hex()
 333            elif self.serializable_tuple:
 334                if self.serializable_int:
 335                    return  tuple(int(c) for c in obj)
 336                if self.serializable_float:
 337                    return  tuple(float(int(c)) for c in obj)
 338                else:
 339                    pass
 340            elif self.serializable_list:
 341                if self.serializable_int:
 342                    return  [int(c) for c in obj]
 343                if self.serializable_float:
 344                    return  [float(int(c)) for c in obj]
 345                else:
 346                    pass
 347            else:
 348                pass
 349        elif 7 == type_id:
 350            return obj
 351        elif 8 == type_id:
 352            return obj
 353        elif 9 == type_id:
 354            if self.serializable_list:
 355                return obj
 356            elif self.serializable_tuple:
 357                return tuple(obj)
 358            elif self.serializable_dict:
 359                return dict({index: item for index, item in enumerate(obj)})
 360            else:
 361                pass
 362        elif 10 == type_id:
 363            if self.serializable_tuple:
 364                return obj
 365            elif self.serializable_list:
 366                return list(obj)
 367            elif self.serializable_dict:
 368                return dict({index: item for index, item in enumerate(obj)})
 369            else:
 370                pass
 371        elif 11 == type_id:
 372            if self.serializable_set:
 373                return obj
 374            elif self.serializable_frozenset:
 375                return frozenset(obj)
 376            elif self.serializable_tuple:
 377                return tuple(obj)
 378            elif self.serializable_list:
 379                return list(obj)
 380            elif self.serializable_dict:
 381                return dict({k: None for k in obj})
 382            else:
 383                pass
 384        elif 12 == type_id:
 385            if self.serializable_frozenset:
 386                return obj
 387            elif self.serializable_set:
 388                return set(obj)
 389            elif self.serializable_tuple:
 390                return tuple(obj)
 391            elif self.serializable_list:
 392                return list(obj)
 393            elif self.serializable_dict:
 394                return dict({k: None for k in obj})
 395            else:
 396                pass
 397        elif 13 == type_id:
 398            return obj
 399        elif 14 == type_id:
 400            if self.serializable_slice:
 401                return obj
 402            elif self.serializable_tuple:
 403                return (obj.start, obj.stop, obj.step)
 404            elif self.serializable_list:
 405                return [obj.start, obj.stop, obj.step]
 406            elif self.serializable_dict:
 407                return {0: obj.start, 1: obj.stop, 2: obj.step}
 408            else:
 409                pass
 410        else:
 411            raise RuntimeError('Unknown type_id')
 412        
 413        raise CanNotAdjustToSerializableError(f'Can not adjust to serializable. Type: {type_id}, obj: {obj}')
 414
 415    ats = adjust_to_serializable
 416
 417    def adjust_from_serializable(self, type_id: DataType, obj: Any,
 418                                int=int,
 419                                complex=complex,
 420                                bytes=bytes,
 421                                bytearray=bytearray,
 422                                tuple=tuple,
 423                                list=list,
 424                                set=set,
 425                                frozenset=frozenset,
 426                                slice=slice,
 427                                sorted=sorted,
 428                                 ) -> Any:
 429        # serialisable_any: bool = self.serializable_any
 430        # serialisable_int: bool = self.serializable_int
 431        # serialisable_float: bool = self.serializable_float
 432        # serialisable_complex: bool = self.serializable_complex
 433        # serialisable_str: bool = self.serializable_str
 434        # serialisable_bytes: bool = self.serializable_bytes
 435        # serialisable_bytearray: bool = self.serializable_bytearray
 436        # serialisable_bool: bool = self.serializable_bool
 437        # serialisable_none: bool = self.serializable_none
 438        # serialisable_list: bool = self.serializable_list
 439        # serialisable_tuple: bool = self.serializable_tuple
 440        # serialisable_set: bool = self.serializable_set
 441        # serialisable_frozenset: bool = self.serializable_frozenset
 442        # serialisable_dict: bool = self.serializable_dict
 443        # serialisable_slice: bool = self.serializable_slice
 444
 445        if 1 == type_id:
 446            return obj
 447        elif 2 == type_id:
 448            return obj
 449        elif 3 == type_id:
 450            if self.serializable_complex:
 451                return obj
 452            elif self.serializable_bytes:
 453                return complex(*unpack('=dd', obj))
 454            elif self.serializable_bytearray:
 455                return complex(*unpack('=dd', bytes(obj)))
 456            elif self.serializable_str:
 457                return complex(*obj)
 458            elif self.serializable_float:
 459                if self.serializable_tuple:
 460                    return complex(*obj)
 461                elif self.serializable_list:
 462                    return complex(*obj)
 463                else:
 464                    pass
 465            else:
 466                pass
 467        elif 4 == type_id:
 468            return obj
 469        elif 5 == type_id:
 470            if self.serializable_bytes:
 471                return obj
 472            elif self.serializable_bytearray:
 473                return bytes(obj)
 474            elif self.serializable_str:
 475                return bytes.fromhex(obj)
 476            elif self.serializable_tuple:
 477                if self.serializable_int:
 478                    return  b''.join(item.to_bytes(1, 'little') for item in obj)
 479                if self.serializable_float:
 480                    return  b''.join((int(round(item))).to_bytes(1, 'little') for item in obj)
 481                else:
 482                    pass
 483            elif self.serializable_list:
 484                if self.serializable_int:
 485                    return  b''.join(item.to_bytes(1, 'little') for item in obj)
 486                if self.serializable_float:
 487                    return  b''.join((int(round(item))).to_bytes(1, 'little') for item in obj)
 488                else:
 489                    pass
 490            else:
 491                pass
 492        elif 6 == type_id:
 493            if self.serializable_bytearray:
 494                return obj
 495            elif self.serializable_bytes:
 496                return bytearray(obj)
 497            elif self.serializable_str:
 498                return bytearray(bytes.fromhex(obj))
 499            elif self.serializable_tuple:
 500                if self.serializable_int:
 501                    return  bytearray(b''.join(item.to_bytes(1, 'little') for item in obj))
 502                if self.serializable_float:
 503                    return  bytearray(b''.join((int(round(item))).to_bytes(1, 'little') for item in obj))
 504                else:
 505                    pass
 506            elif self.serializable_list:
 507                if self.serializable_int:
 508                    return  bytearray(b''.join(item.to_bytes(1, 'little') for item in obj))
 509                if self.serializable_float:
 510                    return  bytearray(b''.join((int(round(item))).to_bytes(1, 'little') for item in obj))
 511                else:
 512                    pass
 513            else:
 514                pass
 515        elif 7 == type_id:
 516            return obj
 517        elif 8 == type_id:
 518            return None
 519        elif 9 == type_id:
 520            if self.serializable_list:
 521                return obj
 522            elif self.serializable_tuple:
 523                return list(obj)
 524            elif self.serializable_dict:
 525                return [value for key, value in sorted(obj.items(), key=lambda x: x[0])]
 526            else:
 527                pass
 528        elif 10 == type_id:
 529            if self.serializable_tuple:
 530                return obj
 531            elif self.serializable_list:
 532                return tuple(obj)
 533            elif self.serializable_dict:
 534                return tuple(value for key, value in sorted(obj.items(), key=lambda x: x[0]))
 535            else:
 536                pass
 537        elif 11 == type_id:
 538            if self.serializable_set:
 539                return obj
 540            elif self.serializable_frozenset:
 541                return set(obj)
 542            elif self.serializable_tuple:
 543                return set(obj)
 544            elif self.serializable_list:
 545                return set(obj)
 546            elif self.serializable_dict:
 547                return set(obj.keys())
 548            else:
 549                pass
 550        elif 12 == type_id:
 551            if self.serializable_frozenset:
 552                return obj
 553            elif self.serializable_set:
 554                return frozenset(obj)
 555            elif self.serializable_tuple:
 556                return frozenset(obj)
 557            elif self.serializable_list:
 558                return frozenset(obj)
 559            elif self.serializable_dict:
 560                return frozenset(obj.keys())
 561            else:
 562                pass
 563        elif 13 == type_id:
 564            return obj
 565        elif 14 == type_id:
 566            if self.serializable_slice:
 567                return obj
 568            elif self.serializable_tuple:
 569                return slice(*obj)
 570            elif self.serializable_list:
 571                return slice(*obj)
 572            elif self.serializable_dict:
 573                return slice(obj[0], obj[1], obj[2])
 574            else:
 575                pass
 576        else:
 577            raise RuntimeError('Unknown type_id')
 578        
 579        raise CanNotAdjustFromSerializableError(f'Can not adjust from serializable. Type: {type_id}, obj: {obj}')
 580    
 581    afs = adjust_from_serializable
 582    
 583    # def is_replicatable_object_attribute(self, attribute: Any) -> bool:
 584    #     if is_setable_data_descriptor(attribute):
 585    #         data = attribute.__get__(None, None)
 586    #     elif is_callable(attribute):
 587    #         return False
 588    #     else:
 589    #         return True
 590
 591    def serialize_container(self, type_id: DataType, obj: Any,
 592                                tuple=tuple,
 593                                list=list,
 594                                set=set,
 595                                frozenset=frozenset,
 596                                dict=dict,
 597                            ) -> Any:
 598        if 9 == type_id:
 599            new_obj = list()
 600            for item in obj:
 601                exists, value = self.serialize_impl(item)
 602                if exists:
 603                    new_obj.append(value)
 604            
 605            return new_obj
 606        elif 10 == type_id:
 607            new_obj = list()
 608            for item in obj:
 609                exists, value = self.serialize_impl(item)
 610                if exists:
 611                    new_obj.append(value)
 612            
 613            new_obj = tuple(new_obj)
 614            return new_obj
 615        elif 11 == type_id:
 616            new_obj = set()
 617            for item in obj:
 618                exists, value = self.serialize_impl(item)
 619                if exists:
 620                    new_obj.add(value)
 621            
 622            return new_obj
 623        elif 12 == type_id:
 624            new_obj = set()
 625            for item in obj:
 626                exists, value = self.serialize_impl(item)
 627                if exists:
 628                    new_obj.add(value)
 629                
 630            new_obj = frozenset(new_obj)
 631            return new_obj
 632        elif 13 == type_id:
 633            new_obj = dict()
 634            for key, value in obj.items():
 635                key_exists, key_value = self.serialize_impl(key)
 636                value_exists, value_value = self.serialize_impl(value)
 637                if key_exists and value_exists:
 638                    new_obj[key_value] = value_value
 639            
 640            return new_obj
 641        else:
 642            return obj
 643    
 644    sc = serialize_container
 645
 646    def serialize_impl(self, obj: Any, ignore_empty_classes: bool = False,
 647                        known_data_types=known_data_types,
 648                        known_container_types=known_container_types,
 649                        data_type_by_type=data_type_by_type,
 650                        id=id,
 651                        int=int,
 652                        str=str,
 653                        tuple=tuple,
 654                        list=list,
 655                        dict=dict,
 656                        isinstance=isinstance,
 657                        hasattr=hasattr,
 658                       ) -> Tuple[bool, int]:
 659        result_exists: bool = True
 660        id_: int = id(obj)
 661        obj_type = type(obj)
 662        if (int != obj_type) and (id_ in self.objects_ids_by_id):
 663            new_object: bool = False
 664            object_id: int = self.objects_ids_by_id[id_]
 665        else:
 666            # int object must always produce new object_id because first 256 ints are persistent across Python sessions and
 667            # this can cause issues within users of current module. For example within `cengal/hardware/memory/shared_memory`
 668            # which changes int values inline instead of producing new objects.
 669            new_object = True
 670            object_id = self.objects_id_gen
 671            self.objects_id_gen += 1
 672            self.objects_ids[object_id] = id_
 673            self.objects_db[object_id] = obj
 674        
 675        if not new_object:
 676            return result_exists, object_id
 677        
 678        if  self.serializable_any or (obj_type in self.serializable_data_types):
 679            serializable: bool = True
 680            type_id: int = data_type_by_type.get(obj_type, 15)
 681        else:
 682            serializable = False
 683            type_id = data_type_by_type.get(obj_type, 0)
 684        
 685        known_container: bool = obj_type in known_container_types
 686        
 687        class_info: Dict[ClassInfoFields, Any] = None
 688        known_data: bool = None
 689        class_id: int = None
 690        is_new_class: bool = None
 691        class_name: str = None
 692        new_obj_slots: List[Tuple[str, Any]] = None
 693        new_obj_dict: Dict[str, Any] = None
 694        obj_mapping: Dict = None
 695        obj_sequence: List = None
 696        obj_set: Set = None
 697        if serializable:
 698            if known_container:
 699                object_info = (
 700                    object_id,
 701                    type_id,
 702                    self.sc(type_id, obj),
 703                )
 704            else:
 705                object_info = (
 706                    object_id,
 707                    type_id,
 708                    obj,
 709                )
 710        else:
 711            known_data = obj_type in known_data_types
 712            
 713            if 0 == type_id:
 714                new_obj_slots = list()
 715                for slot_name, slot_value in filled_slot_names_with_values_gen(obj):
 716                    adjusted_slot_name = slot_name
 717                    exists, value = self.serialize_impl(slot_value)
 718                    if exists:
 719                        if self.serializable_tuple:
 720                            new_obj_slots.append((adjusted_slot_name, value))
 721                        elif self.serializable_list:
 722                            new_obj_slots.append([adjusted_slot_name, value])
 723                        else:
 724                            new_obj_slots.append(self.ats(10, (adjusted_slot_name, value)))
 725                
 726                if new_obj_slots:
 727                    if self.serializable_list:
 728                        pass
 729                    elif self.serializable_tuple:
 730                        new_obj_slots = tuple(new_obj_slots)
 731                    else:
 732                        new_obj_slots = self.ats(9, new_obj_slots)
 733                
 734                # new_obj_dict = dict()
 735                if hasattr(obj, '__dict__'):
 736                    new_obj_dict = {key: self.serialize_impl(value)[1] for key, value in obj.__dict__.items()}
 737                    # for key, value in obj.__dict__.items():
 738                    #     # raw_value = getattr_static(obj, key)
 739                    #     # if hasattr(raw_value, '__get__') and (not hasattr(raw_value, '__set__')):
 740                    #     #     # if not setable descriptor
 741                    #     #     continue
 742
 743                    #     exists, value = self.serialize_impl(value)
 744                    #     if exists:
 745                    #         new_obj_dict[key] = value
 746                    #     else:
 747                    #         continue
 748                else:
 749                    new_obj_dict = dict()
 750                
 751                if isinstance(obj, MutableMapping):
 752                    obj_sequence = None
 753                    obj_set = None
 754                    obj_mapping = {self.serialize_impl(key)[1]: self.serialize_impl(value)[1] for key, value in obj.items()}
 755                    # for key, value in obj.items():
 756                    #     key_exists, key_value = self.serialize_impl(key)
 757                    #     if not key_exists:
 758                    #         continue
 759
 760                    #     value_exists, value_value = self.serialize_impl(value)
 761                    #     if value_exists:
 762                    #         obj_mapping[key_value] = value_value
 763                    #     else:
 764                    #         continue
 765                elif isinstance(obj, MutableSequence):
 766                    obj_mapping = None
 767                    obj_set = None
 768                    obj_sequence = [self.serialize_impl(item)[1] for item in obj]
 769                    obj_sequence = obj_sequence if self.serializable_list else self.ats(9, obj_sequence)
 770                    # for item in obj:
 771                    #     exists, value = self.serialize_impl(item)
 772                    #     if exists:
 773                    #         obj_sequence.append(value)
 774                    #     else:
 775                    #         continue
 776                    
 777                    # if obj_sequence:
 778                    #     if self.serializable_list:
 779                    #         pass
 780                    #     else:
 781                    #         obj_sequence = self.ats(9, obj_sequence)
 782                elif isinstance(obj, MutableSet):
 783                    obj_mapping = None
 784                    obj_sequence = None
 785                    obj_set = [self.serialize_impl(item)[1] for item in obj]
 786                    obj_set = obj_set if self.serializable_list else self.ats(9, obj_set)
 787                    # for item in obj:
 788                    #     exists, value = self.serialize_impl(item)
 789                    #     if exists:
 790                    #         obj_set.append(value)
 791                    #     else:
 792                    #         continue
 793                    
 794                    # if obj_set:
 795                    #     if self.serializable_set:
 796                    #         pass
 797                    #     else:
 798                    #         obj_set = self.ats(9, obj_set)
 799                else:
 800                    pass
 801
 802                if ignore_empty_classes:
 803                    result_exists = new_obj_slots or new_obj_dict or obj_mapping or obj_sequence or obj_set
 804                
 805                if obj_type in self.known_classes:
 806                    is_new_class = False
 807                    class_id = self.known_classes[obj_type]
 808                else:
 809                    is_new_class = True
 810                    class_name = obj_type.__name__
 811                    class_id = self.classes_id_gen
 812                    self.classes_id_gen += 1
 813                    self.known_classes[obj_type] = class_id
 814                    self.known_classes_by_id[class_id] = obj_type
 815                    module_importable_str, owning_names_path = entity_module_importable_str_and_owning_names_path(obj)
 816                    module_importable_str_and_owning_names_path = (class_name, module_importable_str, tuple(owning_names_path))
 817                    self.known_classes_info[module_importable_str_and_owning_names_path] = class_id
 818                    self.known_classes_info_by_id[class_id] = module_importable_str_and_owning_names_path
 819
 820                    if result_exists:
 821                        class_info = (
 822                            class_id,
 823                            class_name,
 824                            module_importable_str,
 825                            owning_names_path,
 826                        )
 827                        
 828                        if self.on_new_class_handler:
 829                            self.on_new_class_handler(
 830                                class_info,
 831                                obj_type,
 832                                class_id,
 833                                class_name,
 834                                module_importable_str,
 835                                owning_names_path,
 836                                module_importable_str_and_owning_names_path
 837                            )
 838
 839                # will be later serialized much faster than without `if else None`
 840                object_info = (
 841                    object_id,
 842                    type_id,
 843                    0,
 844                    class_id,
 845                    new_obj_slots if new_obj_slots else 0,
 846                    new_obj_dict if new_obj_dict else 0,
 847                    obj_mapping if obj_mapping else 0,
 848                    obj_sequence if obj_sequence else 0,
 849                    obj_set if obj_set else 0,
 850                )
 851            elif known_data:
 852                object_info = (
 853                    object_id,
 854                    type_id,
 855                    self.ats(type_id, obj),
 856                )
 857            elif known_container:
 858                object_info = (
 859                    object_id,
 860                    type_id,
 861                    self.ats(type_id, self.sc(type_id, obj)),
 862                )
 863            else:
 864                raise RuntimeError('Unknown type_id')
 865
 866        if result_exists:
 867            if self.on_new_obj_info_handler:
 868                self.on_new_obj_info_handler(
 869                    object_info,
 870                    obj,
 871                    object_id,
 872                    type_id,
 873                    serializable,
 874                    known_container,
 875                    known_data,
 876                    class_id,
 877                    is_new_class,
 878                    new_obj_slots,
 879                    new_obj_dict,
 880                    obj_mapping,
 881                    obj_sequence,
 882                    obj_set,
 883                )
 884        
 885        return result_exists, object_id
 886    
 887    si = serialize_impl
 888
 889    def serialize(self, obj: Any, ignore_empty_classes: bool = False,
 890                  DisableGC=DisableGC,
 891                  ) -> Tuple[bool, int]:
 892        with DisableGC():
 893            return self.serialize_impl(obj, ignore_empty_classes)
 894    
 895    s = serialize
 896    
 897    def deserialize_class_impl(self, class_info: Tuple,
 898                                int=int,
 899                                str=str,
 900                                tuple=tuple,
 901                               ) -> Tuple[bool, int, Type]:
 902        class_id: int = class_info[0]
 903        if class_id in self.known_classes_by_id:
 904            return True, class_id, self.known_classes_by_id[class_id]
 905        else:
 906            class_name: str = class_info[1]
 907            module_importable_str: str = class_info[2]
 908            owning_names_path: List[str] = self.afs(9, class_info[3])
 909            obj_class: Type = entity_by_name_module_importable_str_and_owning_names_path(class_name, module_importable_str, owning_names_path)
 910            self.known_classes_by_id[class_id] = obj_class
 911            self.known_classes[obj_class] = class_id
 912            class_info_tuple: Tuple = (class_name, module_importable_str, tuple(owning_names_path)) 
 913            self.known_classes_info[class_info_tuple] = class_id
 914            self.known_classes_info_by_id[class_id] = class_info_tuple
 915            return True, class_id, obj_class
 916    
 917    dcli = deserialize_class_impl
 918    
 919    def deserialize_class(self, class_info: Dict,
 920                          DisableGC=DisableGC,
 921                          ) -> Tuple[bool, int, Type]:
 922        with DisableGC():
 923            return self.deserialize_class_impl(class_info)
 924    
 925    dcl = deserialize_class
 926
 927    def deserialize_container_impl(self, type_id: DataType, obj: Any,
 928                                    tuple=tuple,
 929                                    list=list,
 930                                    set=set,
 931                                    frozenset=frozenset,
 932                                    dict=dict,
 933                                   ) -> Any:
 934        if 9 == type_id:
 935            new_obj = list()
 936            for item_id in obj:
 937                new_obj.append(self.objects_db[item_id])
 938            
 939            return new_obj
 940        elif 10 == type_id:
 941            new_obj = list()
 942            for item_id in obj:
 943                new_obj.append(self.objects_db[item_id])
 944            
 945            new_obj = tuple(new_obj)
 946            return new_obj
 947        elif 11 == type_id:
 948            new_obj = set()
 949            for item_id in obj:
 950                new_obj.add(self.objects_db[item_id])
 951            
 952            return new_obj
 953        elif 12 == type_id:
 954            new_obj = set()
 955            for item_id in obj:
 956                new_obj.add(self.objects_db[item_id])
 957                
 958            new_obj = frozenset(new_obj)
 959            return new_obj
 960        elif 13 == type_id:
 961            new_obj = dict()
 962            for key_id, value_id in obj.items():
 963                new_obj[self.objects_db[int(key_id)]] = self.objects_db[value_id]
 964            
 965            return new_obj
 966        else:
 967            return obj
 968    
 969    dcoi = deserialize_container_impl
 970
 971    def deserialize_container(self, type_id: DataType, obj: Any,
 972                              DisableGC=DisableGC,
 973                              ) -> Any:
 974        with DisableGC():
 975            return self.deserialize_container_impl(type_id, obj)
 976    
 977    dco = deserialize_container
 978
 979    def deserialize_obj_impl(self, obj_info: Tuple,
 980                                id=id,
 981                                int=int,
 982                                setattr=setattr,
 983                             ) -> Tuple[bool, int, Any]:
 984        object_id: int = obj_info[0]
 985        if object_id in self.objects_db:
 986            return True, object_id, self.objects_db[object_id]
 987        
 988        type_id: int = obj_info[1]
 989        
 990        obj_type = data_type[type_id] 
 991        serializable: bool = self.serializable_any or (obj_type in self.serializable_data_types)
 992        known_container: bool = obj_type in known_container_types
 993        known_data: bool = None
 994        
 995        if serializable:
 996            if known_container:
 997                obj = self.dcoi(type_id, obj_info[2])
 998            else:
 999                obj = obj_info[2]
1000        else:
1001            known_data = obj_type in known_data_types
1002            
1003            if 0 == type_id:
1004                class_id: int = obj_info[3]
1005                obj_class: Type = self.known_classes_by_id[class_id]
1006                obj: Any = obj_class.__new__(obj_class)
1007
1008                if obj_info[4]:
1009                    clonable_slots = self.afs(9, obj_info[4])
1010                    for slot_name, slot_id in clonable_slots:
1011                        child_obj = self.objects_db[slot_id]
1012                        setattr(obj, slot_name, child_obj)
1013                
1014                if obj_info[5]:
1015                    clonable_dict_items = obj_info[5]
1016                    for key, value_id in clonable_dict_items.items():
1017                        child_obj = self.objects_db[value_id]
1018                        setattr(obj, key, child_obj)
1019                
1020                if obj_info[6]:
1021                    contained_mapping = obj_info[6]
1022                    for key_id, value_id in contained_mapping.items():
1023                        child_key = self.objects_db[int(key_id)]
1024                        child_value = self.objects_db[value_id]
1025                        obj[child_key] = child_value
1026                
1027                if obj_info[7]:
1028                    contained_sequence = self.afs(9, obj_info[7])
1029                    for item_id in contained_sequence:
1030                        child_item = self.objects_db[item_id]
1031                        obj.append(child_item)
1032                
1033                if obj_info[8]:
1034                    contained_set = self.afs(9, obj_info[8])
1035                    for item_id in contained_set:
1036                        child_item = self.objects_db[item_id]
1037                        obj.add(child_item)
1038            elif known_data:
1039                obj = self.afs(type_id, obj_info[2])
1040            elif known_container:
1041                obj = self.dcoi(type_id, self.afs(type_id, obj_info[2]))
1042            else:
1043                raise RuntimeError('Unknown type_id')
1044        
1045        self.objects_db[object_id] = obj
1046        id_: int = id(obj)
1047        self.objects_ids[object_id] = id_
1048        self.objects_ids_by_id[id_] = object_id
1049        return True, object_id, obj
1050    
1051    doi = deserialize_obj_impl
1052
1053    def deserialize_obj(self, obj_info: Tuple,
1054                        DisableGC=DisableGC,
1055                        ) -> Tuple[bool, int, Any]:
1056        with DisableGC():
1057            return self.deserialize_obj_impl(obj_info)
1058    
1059    do = deserialize_obj

default_serializable_data_types: Set[Type] = {<class 'int'>, <class 'complex'>, <class 'NoneType'>, <class 'list'>, <class 'frozenset'>, <class 'bytes'>, <class 'str'>, <class 'tuple'>, <class 'dict'>, <class 'float'>, <class 'set'>, <class 'bool'>, <class 'bytearray'>}

known_types: Set[Type] = {<class 'int'>, <class 'complex'>, <class 'NoneType'>, <class 'list'>, <class 'frozenset'>, <class 'bytes'>, <class 'str'>, <class 'tuple'>, <class 'dict'>, <class 'float'>, <class 'set'>, <class 'slice'>, <class 'bool'>, <class 'bytearray'>}

known_data_types: Set[Type] = {<class 'int'>, <class 'NoneType'>, <class 'bytes'>, <class 'str'>, <class 'float'>, <class 'slice'>, <class 'bool'>, <class 'bytearray'>}

known_container_types: Set[Type] = {<class 'complex'>, <class 'list'>, <class 'tuple'>, <class 'dict'>, <class 'set'>, <class 'frozenset'>}

class DataType(enum.IntEnum): View Source

 91class DataType(IntEnum):
 92    class_ = 0  # type is not in known_types and not in serializable_data_types
 93    int_ = 1
 94    float_ = 2
 95    complex_ = 3
 96    str_ = 4
 97    bytes_ = 5
 98    bytearray_ = 6
 99    bool_ = 7
100    none_ = 8
101    list_ = 9
102    tuple_ = 10
103    set_ = 11
104    frozenset_ = 12
105    dict_ = 13
106    slice_ = 14
107    unknown_serializable = 15  # type is in serializable_data_types but not in known_types

An enumeration.

class_ = <DataType.class_: 0>

int_ = <DataType.int_: 1>

float_ = <DataType.float_: 2>

complex_ = <DataType.complex_: 3>

str_ = <DataType.str_: 4>

bytes_ = <DataType.bytes_: 5>

bytearray_ = <DataType.bytearray_: 6>

bool_ = <DataType.bool_: 7>

none_ = <DataType.none_: 8>

list_ = <DataType.list_: 9>

tuple_ = <DataType.tuple_: 10>

set_ = <DataType.set_: 11>

frozenset_ = <DataType.frozenset_: 12>

dict_ = <DataType.dict_: 13>

slice_ = <DataType.slice_: 14>

unknown_serializable = <DataType.unknown_serializable: 15>

Inherited Members

enum.Enum: name; value
builtins.int: conjugate; bit_length; to_bytes; from_bytes; as_integer_ratio; real; imag; numerator; denominator

data_type: Dict[DataType, Type] = {0: <class 'object'>, 1: <class 'int'>, 2: <class 'float'>, 3: <class 'complex'>, 4: <class 'str'>, 5: <class 'bytes'>, 6: <class 'bytearray'>, 7: <class 'bool'>, 8: <class 'NoneType'>, 9: <class 'list'>, 10: <class 'tuple'>, 11: <class 'set'>, 12: <class 'frozenset'>, 13: <class 'dict'>, 14: <class 'slice'>, 15: None}

data_type_by_type: Dict[Type, DataType] = {<class 'object'>: 0, <class 'int'>: 1, <class 'float'>: 2, <class 'complex'>: 3, <class 'str'>: 4, <class 'bytes'>: 5, <class 'bytearray'>: 6, <class 'bool'>: 7, <class 'NoneType'>: 8, <class 'list'>: 9, <class 'tuple'>: 10, <class 'set'>: 11, <class 'frozenset'>: 12, <class 'dict'>: 13, <class 'slice'>: 14, None: 15}

class ClassInfoFields(enum.IntEnum): View Source

150class ClassInfoFields(IntEnum):
151    class_id = 0
152    class_name = 1
153    module_importable_str = 2
154    owning_names_path = 3

An enumeration.

class_id = <ClassInfoFields.class_id: 0>

class_name = <ClassInfoFields.class_name: 1>

module_importable_str = <ClassInfoFields.module_importable_str: 2>

owning_names_path = <ClassInfoFields.owning_names_path: 3>

Inherited Members

enum.Enum: name; value
builtins.int: conjugate; bit_length; to_bytes; from_bytes; as_integer_ratio; real; imag; numerator; denominator

class ObjectInfoFields(enum.IntEnum): View Source

157class ObjectInfoFields(IntEnum):
158    object_id = 0  # (type: int)
159    type_id = 1  # (type: DataType)
160    object_ = 2  # (Optional), (type: Any). Link to object itself if `(type(obj) in serializable_data_types)`. Not used otherwise.
161    class_id = 3  # (Optional), (type: int). Used if `type_id == 0`
162    clonable_slots = 4  # (Optional), (type: Tuple[Tuple[str, Any]]). Used if `type_id == 0`. Holds ID's (object_id) of slots objects.
163    clonable_dict_items = 5  # (Optional), (type: Tuple[Tuple[str, Any]]). Used if `type_id == 0`. Holds ID's (object_id) of value objects.
164    # contained_items = 6  # (Optional), (type: Union[Tuple, List, Set, FrozenSet, Dict]). Used if `type_id in {9, 10, 11, 12, 13, 14}`. Holds ID's (object_id) of contained items (for bothe keys and values in the case of Dict).
165    contained_mapping = 6  # (Optional), (type: Union[Tuple, List, Set, FrozenSet, Dict]). Used if `type_id in {9, 10, 11, 12, 13, 14}`. Holds ID's (object_id) of contained items (for bothe keys and values in the case of Dict).
166    contained_sequence = 7  # (Optional), (type: Union[Tuple, List, Set, FrozenSet, Dict]). Used if `type_id in {9, 10, 11, 12, 13, 14}`. Holds ID's (object_id) of contained items (for bothe keys and values in the case of Dict).
167    contained_set = 8  # (Optional), (type: Union[Tuple, List, Set, FrozenSet, Dict]). Used if `type_id in {9, 10, 11, 12, 13, 14}`. Holds ID's (object_id) of contained items (for bothe keys and values in the case of Dict).

An enumeration.

object_id = <ObjectInfoFields.object_id: 0>

type_id = <ObjectInfoFields.type_id: 1>

object_ = <ObjectInfoFields.object_: 2>

class_id = <ObjectInfoFields.class_id: 3>

clonable_slots = <ObjectInfoFields.clonable_slots: 4>

clonable_dict_items = <ObjectInfoFields.clonable_dict_items: 5>

contained_mapping = <ObjectInfoFields.contained_mapping: 6>

contained_sequence = <ObjectInfoFields.contained_sequence: 7>

contained_set = <ObjectInfoFields.contained_set: 8>

Inherited Members

enum.Enum: name; value
builtins.int: conjugate; bit_length; to_bytes; from_bytes; as_integer_ratio; real; imag; numerator; denominator

class CanNotAdjustToSerializableError(builtins.Exception): View Source

170class CanNotAdjustToSerializableError(Exception):
171    pass

Common base class for all non-exit exceptions.

Inherited Members

builtins.Exception: Exception
builtins.BaseException: with_traceback; args

class CanNotAdjustFromSerializableError(builtins.Exception): View Source

174class CanNotAdjustFromSerializableError(Exception):
175    pass

Common base class for all non-exit exceptions.

Inherited Members

builtins.Exception: Exception
builtins.BaseException: with_traceback; args