cengal.data_manipulation.remote_objects.versions.v_0_fast_optimized_numba.remote_objects

View Source

   1#!/usr/bin/env python
   2# coding=utf-8
   3
   4# Copyright © 2012-2024 ButenkoMS. All rights reserved. Contacts: <gtalk@butenkoms.space>
   5# 
   6# Licensed under the Apache License, Version 2.0 (the "License");
   7# you may not use this file except in compliance with the License.
   8# You may obtain a copy of the License at
   9# 
  10#     http://www.apache.org/licenses/LICENSE-2.0
  11# 
  12# Unless required by applicable law or agreed to in writing, software
  13# distributed under the License is distributed on an "AS IS" BASIS,
  14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15# See the License for the specific language governing permissions and
  16# limitations under the License.
  17
  18
  19__all__ = [
  20    'default_serializable_data_types',
  21    'known_types',
  22    'known_data_types',
  23    'known_container_types',
  24    'DataType',
  25    'data_type',
  26    'data_type_by_type',
  27    'ClassInfoFields',
  28    'ObjectInfoFields',
  29    'CanNotAdjustToSerializableError',
  30    'CanNotAdjustFromSerializableError',
  31    'RemoteObjectsManager',
  32]
  33
  34
  35from cengal.introspection.inspect import (
  36    entity_module_importable_str_and_owning_names_path,
  37    entity_by_name_module_importable_str_and_owning_names_path,
  38    filled_slot_names_with_values_gen,
  39    is_callable, is_async,
  40    is_setable_data_descriptor,
  41)
  42from cengal.code_flow_control.smart_values import ResultExistence
  43from cengal.code_flow_control.gc import DisableGC
  44from cengal.data_generation.id_generator import IDGenerator, GeneratorType
  45from enum import IntEnum
  46from collections.abc import MutableMapping, MutableSequence, MutableSet
  47from struct import pack, unpack
  48from inspect import getattr_static
  49from typing import Any, Dict, Optional, Callable, Set, Type, Tuple, List, FrozenSet
  50from numba import jit, jit_module
  51from numba import int32, int64, float32, boolean, void, int64, float64
  52from numba.experimental import jitclass
  53from numba.types import ListType as NumbaTypedListType, DictType as NumbaTypedDictType
  54from numba.typed import List as NumbaTypedList, Dict as NumbaTypedDict
  55from numba import types as nb_types
  56
  57
  58"""
  59Module Docstring
  60Docstrings: http://www.python.org/dev/peps/pep-0257/
  61"""
  62
  63__author__ = "ButenkoMS <gtalk@butenkoms.space>"
  64__copyright__ = "Copyright © 2012-2024 ButenkoMS. All rights reserved. Contacts: <gtalk@butenkoms.space>"
  65__credits__ = ["ButenkoMS <gtalk@butenkoms.space>", ]
  66__license__ = "Apache License, Version 2.0"
  67__version__ = "4.4.1"
  68__maintainer__ = "ButenkoMS <gtalk@butenkoms.space>"
  69__email__ = "gtalk@butenkoms.space"
  70# __status__ = "Prototype"
  71__status__ = "Development"
  72# __status__ = "Production"
  73
  74
  75default_serializable_data_types: Set[Type] = {
  76    int, float, complex, str, bytes, bytearray, bool, type(None), list, tuple, set, frozenset, dict
  77}
  78
  79
  80known_types: Set[Type] = {
  81    int, float, complex, str, bytes, bytearray, bool, type(None), list, tuple, set, frozenset, dict,
  82    slice, 
  83}
  84
  85
  86known_data_types: Set[Type] = {
  87    int, float, str, bytes, bytearray, bool, type(None), slice,
  88}
  89
  90
  91known_container_types: Set[Type] = {
  92    complex, list, tuple, set, frozenset, dict,
  93}
  94
  95
  96class DataType(IntEnum):
  97    class_ = 0  # type is not in known_types and not in serializable_data_types
  98    int_ = 1
  99    float_ = 2
 100    complex_ = 3
 101    str_ = 4
 102    bytes_ = 5
 103    bytearray_ = 6
 104    bool_ = 7
 105    none_ = 8
 106    list_ = 9
 107    tuple_ = 10
 108    set_ = 11
 109    frozenset_ = 12
 110    dict_ = 13
 111    slice_ = 14
 112    unknown_serializable = 15  # type is in serializable_data_types but not in known_types
 113
 114
 115data_type: Dict[DataType, Type] = {
 116    0: object,
 117    1: int,
 118    2: float,
 119    3: complex,
 120    4: str,
 121    5: bytes,
 122    6: bytearray,
 123    7: bool,
 124    8: type(None),
 125    9: list,
 126    10: tuple,
 127    11: set,
 128    12: frozenset,
 129    13: dict,
 130    14: slice,
 131    15: None,
 132}
 133
 134
 135data_type_by_type: Dict[Type, DataType] = {
 136    object: 0,
 137    int: 1,
 138    float: 2,
 139    complex: 3,
 140    str: 4,
 141    bytes: 5,
 142    bytearray: 6,
 143    bool: 7,
 144    type(None): 8,
 145    list: 9,
 146    tuple: 10,
 147    set: 11,
 148    frozenset: 12,
 149    dict: 13,
 150    slice: 14,
 151    None: 15,
 152}
 153
 154
 155class ClassInfoFields(IntEnum):
 156    class_id = 0
 157    class_name = 1
 158    module_importable_str = 2
 159    owning_names_path = 3
 160
 161
 162class ObjectInfoFields(IntEnum):
 163    object_id = 0  # (type: int)
 164    type_id = 1  # (type: DataType)
 165    object_ = 2  # (Optional), (type: Any). Link to object itself if `(type(obj) in serializable_data_types)`. Not used otherwise.
 166    class_id = 3  # (Optional), (type: int). Used if `type_id == 0`
 167    clonable_slots = 4  # (Optional), (type: Tuple[Tuple[str, Any]]). Used if `type_id == 0`. Holds ID's (object_id) of slots objects.
 168    clonable_dict_items = 5  # (Optional), (type: Tuple[Tuple[str, Any]]). Used if `type_id == 0`. Holds ID's (object_id) of value objects.
 169    # contained_items = 6  # (Optional), (type: Union[Tuple, List, Set, FrozenSet, Dict]). Used if `type_id in {9, 10, 11, 12, 13, 14}`. Holds ID's (object_id) of contained items (for bothe keys and values in the case of Dict).
 170    contained_mapping = 6  # (Optional), (type: Union[Tuple, List, Set, FrozenSet, Dict]). Used if `type_id in {9, 10, 11, 12, 13, 14}`. Holds ID's (object_id) of contained items (for bothe keys and values in the case of Dict).
 171    contained_sequence = 7  # (Optional), (type: Union[Tuple, List, Set, FrozenSet, Dict]). Used if `type_id in {9, 10, 11, 12, 13, 14}`. Holds ID's (object_id) of contained items (for bothe keys and values in the case of Dict).
 172    contained_set = 8  # (Optional), (type: Union[Tuple, List, Set, FrozenSet, Dict]). Used if `type_id in {9, 10, 11, 12, 13, 14}`. Holds ID's (object_id) of contained items (for bothe keys and values in the case of Dict).
 173
 174
 175class CanNotAdjustToSerializableError(Exception):
 176    pass
 177
 178
 179class CanNotAdjustFromSerializableError(Exception):
 180    pass
 181
 182objects_db__kv_ty = (int64, nb_types.Any)
 183known_classes__kv_ty = (nb_types.Any, int32)
 184known_classes_by_id__kv_ty = (int32, nb_types.Any)
 185known_classes_info__kv_ty = (nb_types.Any, int32)
 186known_classes_info_by_id__kv_ty = (int32, nb_types.Any)
 187objects_ids__kv_ty = (int64, int64)
 188objects_ids_by_id__kv_ty = (int64, int64)
 189
 190remote_objects_manager_spec = [
 191    ('classes_id_gen', int32),
 192    ('objects_db', NumbaTypedDictType(*objects_db__kv_ty)),
 193    ('objects_id_gen', int64),
 194    ('serializable_data_types', NumbaTypedListType(nb_types.Any)),
 195    ('serializable_any', boolean),
 196    ('serializable_int', boolean),
 197    ('serializable_float', boolean),
 198    ('serializable_complex', boolean),
 199    ('serializable_str', boolean),
 200    ('serializable_bytes', boolean),
 201    ('serializable_bytearray', boolean),
 202    ('serializable_bool', boolean),
 203    ('serializable_none', boolean),
 204    ('serializable_list', boolean),
 205    ('serializable_tuple', boolean),
 206    ('serializable_set', boolean),
 207    ('serializable_frozenset', boolean),
 208    ('serializable_dict', boolean),
 209    ('serializable_slice', boolean),
 210    ('known_classes', NumbaTypedDictType(*known_classes__kv_ty)),
 211    ('known_classes_by_id', NumbaTypedDictType(*known_classes_by_id__kv_ty)),
 212    ('known_classes_info', NumbaTypedDictType(*known_classes_info__kv_ty)),
 213    ('known_classes_info_by_id', NumbaTypedDictType(*known_classes_info_by_id__kv_ty)),
 214    ('on_new_class_handler', nb_types.Optional(nb_types.Any)),
 215    ('on_new_obj_info_handler', nb_types.Optional(nb_types.Any)),
 216    ('objects_ids', NumbaTypedDictType(*objects_ids__kv_ty)),
 217    ('objects_ids_by_id', NumbaTypedDictType(*objects_ids_by_id__kv_ty)),
 218]
 219
 220@jitclass(remote_objects_manager_spec)
 221# @jitclass(nopython=False)
 222class RemoteObjectsManager:
 223    def __init__(self, 
 224                 on_new_class_handler: Optional[Callable] = None,
 225                 on_new_obj_info_handler: Optional[Callable] = None,
 226                 serializable_data_types: Optional[Set[Type]] = None,
 227                 ) -> None:
 228        # self.classess_db: Dict[int, Type] = dict() if classess_db is None else classess_db
 229        self.classes_id_gen: int = 0
 230        self.objects_db: Dict[int, Any] = dict() if objects_db is None else objects_db
 231        self.objects_id_gen: int = 0
 232        self.serializable_data_types: Set[Type] = default_serializable_data_types if serializable_data_types is None else serializable_data_types
 233        self.serializable_any: bool = not self.serializable_data_types
 234        self.serializable_int: bool = (int in self.serializable_data_types) or self.serializable_any
 235        self.serializable_float: bool = (float in self.serializable_data_types) or self.serializable_any
 236        self.serializable_complex: bool = (complex in self.serializable_data_types) or self.serializable_any
 237        self.serializable_str: bool = (str in self.serializable_data_types) or self.serializable_any
 238        self.serializable_bytes: bool = (bytes in self.serializable_data_types) or self.serializable_any
 239        self.serializable_bytearray: bool = (bytearray in self.serializable_data_types) or self.serializable_any
 240        self.serializable_bool: bool = (bool in self.serializable_data_types) or self.serializable_any
 241        self.serializable_none: bool = (type(None) in self.serializable_data_types) or self.serializable_any
 242        self.serializable_list: bool = (list in self.serializable_data_types) or self.serializable_any
 243        self.serializable_tuple: bool = (tuple in self.serializable_data_types) or self.serializable_any
 244        self.serializable_set: bool = (set in self.serializable_data_types) or self.serializable_any
 245        self.serializable_frozenset: bool = (frozenset in self.serializable_data_types) or self.serializable_any
 246        self.serializable_dict: bool = (dict in self.serializable_data_types) or self.serializable_any
 247        self.serializable_slice: bool = (slice in self.serializable_data_types) or self.serializable_any
 248        self.known_classes: Dict[Type, int] = dict()
 249        self.known_classes_by_id: Dict[int, Type] = dict()
 250        self.known_classes_info: Dict[Tuple, int] = dict()
 251        self.known_classes_info_by_id: Dict[int, Tuple] = dict()
 252        self.on_new_class_handler: Optional[Callable] = on_new_class_handler
 253        self.on_new_obj_info_handler: Optional[Callable] = on_new_obj_info_handler
 254        self.objects_ids: Dict[int, int] = dict()  # (Key: object_id, Value: id(obj))
 255        self.objects_ids_by_id: Dict[int, int] = dict()  # (Key: id(obj), Value: object_id)
 256
 257    def del_object_by_id(self, id_: int) -> None:
 258        if id_ in self.objects_ids_by_id:
 259            object_id = self.objects_ids_by_id[id_]
 260            self.objects_ids_by_id.pop(id_, None)
 261            self.objects_ids.pop(object_id, None)
 262            self.objects_db.pop(object_id, None)
 263    
 264    dobi = del_object_by_id
 265    
 266    def del_object_by_object_id(self, object_id: int) -> None:
 267        if object_id in self.objects_ids:
 268            id_ = self.objects_ids[object_id]
 269            self.objects_ids_by_id.pop(id_, None)
 270            self.objects_ids.pop(object_id, None)
 271            self.objects_db.pop(object_id, None)
 272    
 273    doboi = del_object_by_object_id
 274    
 275    # @profile
 276    def adjust_to_serializable(self, type_id: DataType, obj: Any,
 277                                # id=id,
 278                                # int=int,
 279                                # float=float,
 280                                # complex=complex,
 281                                # str=str,
 282                                # bytes=bytes,
 283                                # bytearray=bytearray,
 284                                # tuple=tuple,
 285                                # list=list,
 286                                # set=set,
 287                                # frozenset=frozenset,
 288                                # dict=dict,
 289                                # slice=slice,
 290                                # enumerate=enumerate,
 291                                # sorted=sorted,
 292                                # isinstance=isinstance,
 293                                # hasattr=hasattr,
 294                                # getattr=getattr,
 295                                # setattr=setattr,
 296                                int=int,
 297                                float=float,
 298                                str=str,
 299                                bytes=bytes,
 300                                bytearray=bytearray,
 301                                tuple=tuple,
 302                                list=list,
 303                                set=set,
 304                                frozenset=frozenset,
 305                                dict=dict,
 306                                enumerate=enumerate,
 307                               ) -> Any:
 308        # serialisable_any: bool = self.serializable_any
 309        # serialisable_int: bool = self.serializable_int
 310        # serialisable_float: bool = self.serializable_float
 311        # serialisable_complex: bool = self.serializable_complex
 312        # serialisable_str: bool = self.serializable_str
 313        # serialisable_bytes: bool = self.serializable_bytes
 314        # serialisable_bytearray: bool = self.serializable_bytearray
 315        # serialisable_bool: bool = self.serializable_bool
 316        # serialisable_none: bool = self.serializable_none
 317        # serialisable_list: bool = self.serializable_list
 318        # serialisable_tuple: bool = self.serializable_tuple
 319        # serialisable_set: bool = self.serializable_set
 320        # serialisable_frozenset: bool = self.serializable_frozenset
 321        # serialisable_dict: bool = self.serializable_dict
 322        # serialisable_slice: bool = self.serializable_slice
 323
 324        if 1 == type_id:
 325            return obj
 326        elif 2 == type_id:
 327            return obj
 328        elif 3 == type_id:
 329            if self.serializable_complex:
 330                return obj
 331            elif self.serializable_bytes:
 332                return pack('=dd', obj.real, obj.imag)
 333            elif self.serializable_bytearray:
 334                return bytearray(pack('=dd', obj.real, obj.imag))
 335            elif self.serializable_str:
 336                return str(obj)
 337            elif self.serializable_float:
 338                if self.serializable_tuple:
 339                    return (obj.real, obj.imag)
 340                elif self.serializable_list:
 341                    return [obj.real, obj.imag]
 342                else:
 343                    pass
 344            else:
 345                pass
 346        elif 4 == type_id:
 347            return obj
 348        elif 5 == type_id:
 349            if self.serializable_bytes:
 350                return obj
 351            elif self.serializable_bytearray:
 352                return bytearray(obj)
 353            elif self.serializable_str:
 354                return obj.hex()
 355            elif self.serializable_tuple:
 356                if self.serializable_int:
 357                    return  tuple(int(c) for c in obj)
 358                if self.serializable_float:
 359                    return  tuple(float(int(c)) for c in obj)
 360                else:
 361                    pass
 362            elif self.serializable_list:
 363                if self.serializable_int:
 364                    return  [int(c) for c in obj]
 365                if self.serializable_float:
 366                    return  [float(int(c)) for c in obj]
 367                else:
 368                    pass
 369            else:
 370                pass
 371        elif 6 == type_id:
 372            if self.serializable_bytearray:
 373                return obj
 374            elif self.serializable_bytes:
 375                return bytes(obj)
 376            elif self.serializable_str:
 377                return obj.hex()
 378            elif self.serializable_tuple:
 379                if self.serializable_int:
 380                    return  tuple(int(c) for c in obj)
 381                if self.serializable_float:
 382                    return  tuple(float(int(c)) for c in obj)
 383                else:
 384                    pass
 385            elif self.serializable_list:
 386                if self.serializable_int:
 387                    return  [int(c) for c in obj]
 388                if self.serializable_float:
 389                    return  [float(int(c)) for c in obj]
 390                else:
 391                    pass
 392            else:
 393                pass
 394        elif 7 == type_id:
 395            return obj
 396        elif 8 == type_id:
 397            return obj
 398        elif 9 == type_id:
 399            if self.serializable_list:
 400                return obj
 401            elif self.serializable_tuple:
 402                return tuple(obj)
 403            elif self.serializable_dict:
 404                return dict({index: item for index, item in enumerate(obj)})
 405            else:
 406                pass
 407        elif 10 == type_id:
 408            if self.serializable_tuple:
 409                return obj
 410            elif self.serializable_list:
 411                return list(obj)
 412            elif self.serializable_dict:
 413                return dict({index: item for index, item in enumerate(obj)})
 414            else:
 415                pass
 416        elif 11 == type_id:
 417            if self.serializable_set:
 418                return obj
 419            elif self.serializable_frozenset:
 420                return frozenset(obj)
 421            elif self.serializable_tuple:
 422                return tuple(obj)
 423            elif self.serializable_list:
 424                return list(obj)
 425            elif self.serializable_dict:
 426                return dict({k: None for k in obj})
 427            else:
 428                pass
 429        elif 12 == type_id:
 430            if self.serializable_frozenset:
 431                return obj
 432            elif self.serializable_set:
 433                return set(obj)
 434            elif self.serializable_tuple:
 435                return tuple(obj)
 436            elif self.serializable_list:
 437                return list(obj)
 438            elif self.serializable_dict:
 439                return dict({k: None for k in obj})
 440            else:
 441                pass
 442        elif 13 == type_id:
 443            return obj
 444        elif 14 == type_id:
 445            if self.serializable_slice:
 446                return obj
 447            elif self.serializable_tuple:
 448                return (obj.start, obj.stop, obj.step)
 449            elif self.serializable_list:
 450                return [obj.start, obj.stop, obj.step]
 451            elif self.serializable_dict:
 452                return {0: obj.start, 1: obj.stop, 2: obj.step}
 453            else:
 454                pass
 455        else:
 456            raise RuntimeError('Unknown type_id')
 457        
 458        raise CanNotAdjustToSerializableError(f'Can not adjust to serializable. Type: {type_id}, obj: {obj}')
 459
 460    ats = adjust_to_serializable
 461
 462    # @profile
 463    def adjust_from_serializable(self, type_id: DataType, obj: Any,
 464                                int=int,
 465                                complex=complex,
 466                                bytes=bytes,
 467                                bytearray=bytearray,
 468                                tuple=tuple,
 469                                list=list,
 470                                set=set,
 471                                frozenset=frozenset,
 472                                slice=slice,
 473                                sorted=sorted,
 474                                 ) -> Any:
 475        # serialisable_any: bool = self.serializable_any
 476        # serialisable_int: bool = self.serializable_int
 477        # serialisable_float: bool = self.serializable_float
 478        # serialisable_complex: bool = self.serializable_complex
 479        # serialisable_str: bool = self.serializable_str
 480        # serialisable_bytes: bool = self.serializable_bytes
 481        # serialisable_bytearray: bool = self.serializable_bytearray
 482        # serialisable_bool: bool = self.serializable_bool
 483        # serialisable_none: bool = self.serializable_none
 484        # serialisable_list: bool = self.serializable_list
 485        # serialisable_tuple: bool = self.serializable_tuple
 486        # serialisable_set: bool = self.serializable_set
 487        # serialisable_frozenset: bool = self.serializable_frozenset
 488        # serialisable_dict: bool = self.serializable_dict
 489        # serialisable_slice: bool = self.serializable_slice
 490
 491        if 1 == type_id:
 492            return obj
 493        elif 2 == type_id:
 494            return obj
 495        elif 3 == type_id:
 496            if self.serializable_complex:
 497                return obj
 498            elif self.serializable_bytes:
 499                return complex(*unpack('=dd', obj))
 500            elif self.serializable_bytearray:
 501                return complex(*unpack('=dd', bytes(obj)))
 502            elif self.serializable_str:
 503                return complex(*obj)
 504            elif self.serializable_float:
 505                if self.serializable_tuple:
 506                    return complex(*obj)
 507                elif self.serializable_list:
 508                    return complex(*obj)
 509                else:
 510                    pass
 511            else:
 512                pass
 513        elif 4 == type_id:
 514            return obj
 515        elif 5 == type_id:
 516            if self.serializable_bytes:
 517                return obj
 518            elif self.serializable_bytearray:
 519                return bytes(obj)
 520            elif self.serializable_str:
 521                return bytes.fromhex(obj)
 522            elif self.serializable_tuple:
 523                if self.serializable_int:
 524                    return  b''.join(item.to_bytes(1, 'little') for item in obj)
 525                if self.serializable_float:
 526                    return  b''.join((int(round(item))).to_bytes(1, 'little') for item in obj)
 527                else:
 528                    pass
 529            elif self.serializable_list:
 530                if self.serializable_int:
 531                    return  b''.join(item.to_bytes(1, 'little') for item in obj)
 532                if self.serializable_float:
 533                    return  b''.join((int(round(item))).to_bytes(1, 'little') for item in obj)
 534                else:
 535                    pass
 536            else:
 537                pass
 538        elif 6 == type_id:
 539            if self.serializable_bytearray:
 540                return obj
 541            elif self.serializable_bytes:
 542                return bytearray(obj)
 543            elif self.serializable_str:
 544                return bytearray(bytes.fromhex(obj))
 545            elif self.serializable_tuple:
 546                if self.serializable_int:
 547                    return  bytearray(b''.join(item.to_bytes(1, 'little') for item in obj))
 548                if self.serializable_float:
 549                    return  bytearray(b''.join((int(round(item))).to_bytes(1, 'little') for item in obj))
 550                else:
 551                    pass
 552            elif self.serializable_list:
 553                if self.serializable_int:
 554                    return  bytearray(b''.join(item.to_bytes(1, 'little') for item in obj))
 555                if self.serializable_float:
 556                    return  bytearray(b''.join((int(round(item))).to_bytes(1, 'little') for item in obj))
 557                else:
 558                    pass
 559            else:
 560                pass
 561        elif 7 == type_id:
 562            return obj
 563        elif 8 == type_id:
 564            return None
 565        elif 9 == type_id:
 566            if self.serializable_list:
 567                return obj
 568            elif self.serializable_tuple:
 569                return list(obj)
 570            elif self.serializable_dict:
 571                return [value for key, value in sorted(obj.items(), key=lambda x: x[0])]
 572            else:
 573                pass
 574        elif 10 == type_id:
 575            if self.serializable_tuple:
 576                return obj
 577            elif self.serializable_list:
 578                return tuple(obj)
 579            elif self.serializable_dict:
 580                return tuple(value for key, value in sorted(obj.items(), key=lambda x: x[0]))
 581            else:
 582                pass
 583        elif 11 == type_id:
 584            if self.serializable_set:
 585                return obj
 586            elif self.serializable_frozenset:
 587                return set(obj)
 588            elif self.serializable_tuple:
 589                return set(obj)
 590            elif self.serializable_list:
 591                return set(obj)
 592            elif self.serializable_dict:
 593                return set(obj.keys())
 594            else:
 595                pass
 596        elif 12 == type_id:
 597            if self.serializable_frozenset:
 598                return obj
 599            elif self.serializable_set:
 600                return frozenset(obj)
 601            elif self.serializable_tuple:
 602                return frozenset(obj)
 603            elif self.serializable_list:
 604                return frozenset(obj)
 605            elif self.serializable_dict:
 606                return frozenset(obj.keys())
 607            else:
 608                pass
 609        elif 13 == type_id:
 610            return obj
 611        elif 14 == type_id:
 612            if self.serializable_slice:
 613                return obj
 614            elif self.serializable_tuple:
 615                return slice(*obj)
 616            elif self.serializable_list:
 617                return slice(*obj)
 618            elif self.serializable_dict:
 619                return slice(obj[0], obj[1], obj[2])
 620            else:
 621                pass
 622        else:
 623            raise RuntimeError('Unknown type_id')
 624        
 625        raise CanNotAdjustFromSerializableError(f'Can not adjust from serializable. Type: {type_id}, obj: {obj}')
 626    
 627    afs = adjust_from_serializable
 628    
 629    # def is_replicatable_object_attribute(self, attribute: Any) -> bool:
 630    #     if is_setable_data_descriptor(attribute):
 631    #         data = attribute.__get__(None, None)
 632    #     elif is_callable(attribute):
 633    #         return False
 634    #     else:
 635    #         return True
 636
 637    # @profile
 638    def serialize_container(self, type_id: DataType, obj: Any,
 639                                tuple=tuple,
 640                                list=list,
 641                                set=set,
 642                                frozenset=frozenset,
 643                                dict=dict,
 644                            ) -> Any:
 645        if 9 == type_id:
 646            new_obj = list()
 647            for item in obj:
 648                exists, value = self.serialize_impl(item)
 649                if exists:
 650                    new_obj.append(value)
 651            
 652            return new_obj
 653        elif 10 == type_id:
 654            new_obj = list()
 655            for item in obj:
 656                exists, value = self.serialize_impl(item)
 657                if exists:
 658                    new_obj.append(value)
 659            
 660            new_obj = tuple(new_obj)
 661            return new_obj
 662        elif 11 == type_id:
 663            new_obj = set()
 664            for item in obj:
 665                exists, value = self.serialize_impl(item)
 666                if exists:
 667                    new_obj.add(value)
 668            
 669            return new_obj
 670        elif 12 == type_id:
 671            new_obj = set()
 672            for item in obj:
 673                exists, value = self.serialize_impl(item)
 674                if exists:
 675                    new_obj.add(value)
 676                
 677            new_obj = frozenset(new_obj)
 678            return new_obj
 679        elif 13 == type_id:
 680            new_obj = dict()
 681            for key, value in obj.items():
 682                key_exists, key_value = self.serialize_impl(key)
 683                value_exists, value_value = self.serialize_impl(value)
 684                if key_exists and value_exists:
 685                    new_obj[key_value] = value_value
 686            
 687            return new_obj
 688        else:
 689            return obj
 690    
 691    sc = serialize_container
 692
 693    # @profile
 694    def serialize_impl(self, obj: Any, ignore_empty_classes: bool = False,
 695                        known_data_types=known_data_types,
 696                        known_container_types=known_container_types,
 697                        data_type_by_type=data_type_by_type,
 698                        id=id,
 699                        int=int,
 700                        str=str,
 701                        tuple=tuple,
 702                        list=list,
 703                        dict=dict,
 704                        isinstance=isinstance,
 705                        hasattr=hasattr,
 706                       ) -> Tuple[bool, int]:
 707        result_exists: bool = True
 708        id_: int = id(obj)
 709        obj_type = type(obj)
 710        if (int != obj_type) and (id_ in self.objects_ids_by_id):
 711            new_object: bool = False
 712            object_id: int = self.objects_ids_by_id[id_]
 713        else:
 714            # int object must always produce new object_id because first 256 ints are persistent across Python sessions and
 715            # this can cause issues within users of current module. For example within `cengal/hardware/memory/shared_memory`
 716            # which changes int values inline instead of producing new objects.
 717            new_object = True
 718            object_id = self.objects_id_gen
 719            self.objects_id_gen += 1
 720            self.objects_ids[object_id] = id_
 721            self.objects_db[object_id] = obj
 722        
 723        if not new_object:
 724            return result_exists, object_id
 725        
 726        if  self.serializable_any or (obj_type in self.serializable_data_types):
 727            serializable: bool = True
 728            type_id: int = data_type_by_type.get(obj_type, 15)
 729        else:
 730            serializable = False
 731            type_id = data_type_by_type.get(obj_type, 0)
 732        
 733        known_container: bool = obj_type in known_container_types
 734        
 735        class_info: Dict[ClassInfoFields, Any] = None
 736        known_data: bool = None
 737        class_id: int = None
 738        is_new_class: bool = None
 739        class_name: str = None
 740        new_obj_slots: List[Tuple[str, Any]] = None
 741        new_obj_dict: Dict[str, Any] = None
 742        obj_mapping: Dict = None
 743        obj_sequence: List = None
 744        obj_set: Set = None
 745        if serializable:
 746            if known_container:
 747                object_info = (
 748                    object_id,
 749                    type_id,
 750                    self.sc(type_id, obj),
 751                )
 752            else:
 753                object_info = (
 754                    object_id,
 755                    type_id,
 756                    obj,
 757                )
 758        else:
 759            known_data = obj_type in known_data_types
 760            
 761            if 0 == type_id:
 762                new_obj_slots = list()
 763                for slot_name, slot_value in filled_slot_names_with_values_gen(obj):
 764                    adjusted_slot_name = slot_name
 765                    exists, value = self.serialize_impl(slot_value)
 766                    if exists:
 767                        if self.serializable_tuple:
 768                            new_obj_slots.append((adjusted_slot_name, value))
 769                        elif self.serializable_list:
 770                            new_obj_slots.append([adjusted_slot_name, value])
 771                        else:
 772                            new_obj_slots.append(self.ats(10, (adjusted_slot_name, value)))
 773                
 774                if new_obj_slots:
 775                    if self.serializable_list:
 776                        pass
 777                    elif self.serializable_tuple:
 778                        new_obj_slots = tuple(new_obj_slots)
 779                    else:
 780                        new_obj_slots = self.ats(9, new_obj_slots)
 781                
 782                # new_obj_dict = dict()
 783                if hasattr(obj, '__dict__'):
 784                    new_obj_dict = {key: self.serialize_impl(value)[1] for key, value in obj.__dict__.items()}
 785                    # for key, value in obj.__dict__.items():
 786                    #     # raw_value = getattr_static(obj, key)
 787                    #     # if hasattr(raw_value, '__get__') and (not hasattr(raw_value, '__set__')):
 788                    #     #     # if not setable descriptor
 789                    #     #     continue
 790
 791                    #     exists, value = self.serialize_impl(value)
 792                    #     if exists:
 793                    #         new_obj_dict[key] = value
 794                    #     else:
 795                    #         continue
 796                else:
 797                    new_obj_dict = dict()
 798                
 799                if isinstance(obj, MutableMapping):
 800                    obj_sequence = None
 801                    obj_set = None
 802                    obj_mapping = {self.serialize_impl(key)[1]: self.serialize_impl(value)[1] for key, value in obj.items()}
 803                    # for key, value in obj.items():
 804                    #     key_exists, key_value = self.serialize_impl(key)
 805                    #     if not key_exists:
 806                    #         continue
 807
 808                    #     value_exists, value_value = self.serialize_impl(value)
 809                    #     if value_exists:
 810                    #         obj_mapping[key_value] = value_value
 811                    #     else:
 812                    #         continue
 813                elif isinstance(obj, MutableSequence):
 814                    obj_mapping = None
 815                    obj_set = None
 816                    obj_sequence = [self.serialize_impl(item)[1] for item in obj]
 817                    obj_sequence = obj_sequence if self.serializable_list else self.ats(9, obj_sequence)
 818                    # for item in obj:
 819                    #     exists, value = self.serialize_impl(item)
 820                    #     if exists:
 821                    #         obj_sequence.append(value)
 822                    #     else:
 823                    #         continue
 824                    
 825                    # if obj_sequence:
 826                    #     if self.serializable_list:
 827                    #         pass
 828                    #     else:
 829                    #         obj_sequence = self.ats(9, obj_sequence)
 830                elif isinstance(obj, MutableSet):
 831                    obj_mapping = None
 832                    obj_sequence = None
 833                    obj_set = [self.serialize_impl(item)[1] for item in obj]
 834                    obj_set = obj_set if self.serializable_list else self.ats(9, obj_set)
 835                    # for item in obj:
 836                    #     exists, value = self.serialize_impl(item)
 837                    #     if exists:
 838                    #         obj_set.append(value)
 839                    #     else:
 840                    #         continue
 841                    
 842                    # if obj_set:
 843                    #     if self.serializable_set:
 844                    #         pass
 845                    #     else:
 846                    #         obj_set = self.ats(9, obj_set)
 847                else:
 848                    pass
 849
 850                if ignore_empty_classes:
 851                    result_exists = new_obj_slots or new_obj_dict or obj_mapping or obj_sequence or obj_set
 852                
 853                if obj_type in self.known_classes:
 854                    is_new_class = False
 855                    class_id = self.known_classes[obj_type]
 856                else:
 857                    is_new_class = True
 858                    class_name = obj_type.__name__
 859                    class_id = self.classes_id_gen
 860                    self.classes_id_gen += 1
 861                    self.known_classes[obj_type] = class_id
 862                    self.known_classes_by_id[class_id] = obj_type
 863                    module_importable_str, owning_names_path = entity_module_importable_str_and_owning_names_path(obj)
 864                    module_importable_str_and_owning_names_path = (class_name, module_importable_str, tuple(owning_names_path))
 865                    self.known_classes_info[module_importable_str_and_owning_names_path] = class_id
 866                    self.known_classes_info_by_id[class_id] = module_importable_str_and_owning_names_path
 867
 868                    if result_exists:
 869                        class_info = (
 870                            class_id,
 871                            class_name,
 872                            module_importable_str,
 873                            owning_names_path,
 874                        )
 875                        
 876                        if self.on_new_class_handler:
 877                            self.on_new_class_handler(
 878                                class_info,
 879                                obj_type,
 880                                class_id,
 881                                class_name,
 882                                module_importable_str,
 883                                owning_names_path,
 884                                module_importable_str_and_owning_names_path
 885                            )
 886
 887                # will be later serialized much faster than without `if else None`
 888                object_info = (
 889                    object_id,
 890                    type_id,
 891                    0,
 892                    class_id,
 893                    new_obj_slots if new_obj_slots else 0,
 894                    new_obj_dict if new_obj_dict else 0,
 895                    obj_mapping if obj_mapping else 0,
 896                    obj_sequence if obj_sequence else 0,
 897                    obj_set if obj_set else 0,
 898                )
 899            elif known_data:
 900                object_info = (
 901                    object_id,
 902                    type_id,
 903                    self.ats(type_id, obj),
 904                )
 905            elif known_container:
 906                object_info = (
 907                    object_id,
 908                    type_id,
 909                    self.ats(type_id, self.sc(type_id, obj)),
 910                )
 911            else:
 912                raise RuntimeError('Unknown type_id')
 913
 914        if result_exists:
 915            if self.on_new_obj_info_handler:
 916                self.on_new_obj_info_handler(
 917                    object_info,
 918                    obj,
 919                    object_id,
 920                    type_id,
 921                    serializable,
 922                    known_container,
 923                    known_data,
 924                    class_id,
 925                    is_new_class,
 926                    new_obj_slots,
 927                    new_obj_dict,
 928                    obj_mapping,
 929                    obj_sequence,
 930                    obj_set,
 931                )
 932        
 933        return result_exists, object_id
 934    
 935    si = serialize_impl
 936
 937    # @profile
 938    def serialize(self, obj: Any, ignore_empty_classes: bool = False,
 939                  DisableGC=DisableGC,
 940                  ) -> Tuple[bool, int]:
 941        with DisableGC():
 942            return self.serialize_impl(obj, ignore_empty_classes)
 943    
 944    s = serialize
 945    
 946    # @profile
 947    def deserialize_class_impl(self, class_info: Tuple,
 948                                int=int,
 949                                str=str,
 950                                tuple=tuple,
 951                               ) -> Tuple[bool, int, Type]:
 952        class_id: int = class_info[0]
 953        if class_id in self.known_classes_by_id:
 954            return True, class_id, self.known_classes_by_id[class_id]
 955        else:
 956            class_name: str = class_info[1]
 957            module_importable_str: str = class_info[2]
 958            owning_names_path: List[str] = self.afs(9, class_info[3])
 959            obj_class: Type = entity_by_name_module_importable_str_and_owning_names_path(class_name, module_importable_str, owning_names_path)
 960            self.known_classes_by_id[class_id] = obj_class
 961            self.known_classes[obj_class] = class_id
 962            class_info_tuple: Tuple = (class_name, module_importable_str, tuple(owning_names_path)) 
 963            self.known_classes_info[class_info_tuple] = class_id
 964            self.known_classes_info_by_id[class_id] = class_info_tuple
 965            return True, class_id, obj_class
 966    
 967    dcli = deserialize_class_impl
 968    
 969    # @profile
 970    def deserialize_class(self, class_info: Dict,
 971                          DisableGC=DisableGC,
 972                          ) -> Tuple[bool, int, Type]:
 973        with DisableGC():
 974            return self.deserialize_class_impl(class_info)
 975    
 976    dcl = deserialize_class
 977
 978    # @profile
 979    def deserialize_container_impl(self, type_id: DataType, obj: Any,
 980                                    tuple=tuple,
 981                                    list=list,
 982                                    set=set,
 983                                    frozenset=frozenset,
 984                                    dict=dict,
 985                                   ) -> Any:
 986        if 9 == type_id:
 987            new_obj = list()
 988            for item_id in obj:
 989                new_obj.append(self.objects_db[item_id])
 990            
 991            return new_obj
 992        elif 10 == type_id:
 993            new_obj = list()
 994            for item_id in obj:
 995                new_obj.append(self.objects_db[item_id])
 996            
 997            new_obj = tuple(new_obj)
 998            return new_obj
 999        elif 11 == type_id:
1000            new_obj = set()
1001            for item_id in obj:
1002                new_obj.add(self.objects_db[item_id])
1003            
1004            return new_obj
1005        elif 12 == type_id:
1006            new_obj = set()
1007            for item_id in obj:
1008                new_obj.add(self.objects_db[item_id])
1009                
1010            new_obj = frozenset(new_obj)
1011            return new_obj
1012        elif 13 == type_id:
1013            new_obj = dict()
1014            for key_id, value_id in obj.items():
1015                new_obj[self.objects_db[int(key_id)]] = self.objects_db[value_id]
1016            
1017            return new_obj
1018        else:
1019            return obj
1020    
1021    dcoi = deserialize_container_impl
1022
1023    # @profile
1024    def deserialize_container(self, type_id: DataType, obj: Any,
1025                              DisableGC=DisableGC,
1026                              ) -> Any:
1027        with DisableGC():
1028            return self.deserialize_container_impl(type_id, obj)
1029    
1030    dco = deserialize_container
1031
1032    # @profile
1033    def deserialize_obj_impl(self, obj_info: Tuple,
1034                                id=id,
1035                                int=int,
1036                                setattr=setattr,
1037                             ) -> Tuple[bool, int, Any]:
1038        object_id: int = obj_info[0]
1039        if object_id in self.objects_db:
1040            return True, object_id, self.objects_db[object_id]
1041        
1042        type_id: int = obj_info[1]
1043        
1044        obj_type = data_type[type_id] 
1045        serializable: bool = self.serializable_any or (obj_type in self.serializable_data_types)
1046        known_container: bool = obj_type in known_container_types
1047        known_data: bool = None
1048        
1049        if serializable:
1050            if known_container:
1051                obj = self.dcoi(type_id, obj_info[2])
1052            else:
1053                obj = obj_info[2]
1054        else:
1055            known_data = obj_type in known_data_types
1056            
1057            if 0 == type_id:
1058                class_id: int = obj_info[3]
1059                obj_class: Type = self.known_classes_by_id[class_id]
1060                obj: Any = obj_class.__new__(obj_class)
1061
1062                if obj_info[4]:
1063                    clonable_slots = self.afs(9, obj_info[4])
1064                    for slot_name, slot_id in clonable_slots:
1065                        child_obj = self.objects_db[slot_id]
1066                        setattr(obj, slot_name, child_obj)
1067                
1068                if obj_info[5]:
1069                    clonable_dict_items = obj_info[5]
1070                    for key, value_id in clonable_dict_items.items():
1071                        child_obj = self.objects_db[value_id]
1072                        setattr(obj, key, child_obj)
1073                
1074                if obj_info[6]:
1075                    contained_mapping = obj_info[6]
1076                    for key_id, value_id in contained_mapping.items():
1077                        child_key = self.objects_db[int(key_id)]
1078                        child_value = self.objects_db[value_id]
1079                        obj[child_key] = child_value
1080                
1081                if obj_info[7]:
1082                    contained_sequence = self.afs(9, obj_info[7])
1083                    for item_id in contained_sequence:
1084                        child_item = self.objects_db[item_id]
1085                        obj.append(child_item)
1086                
1087                if obj_info[8]:
1088                    contained_set = self.afs(9, obj_info[8])
1089                    for item_id in contained_set:
1090                        child_item = self.objects_db[item_id]
1091                        obj.add(child_item)
1092            elif known_data:
1093                obj = self.afs(type_id, obj_info[2])
1094            elif known_container:
1095                obj = self.dcoi(type_id, self.afs(type_id, obj_info[2]))
1096            else:
1097                raise RuntimeError('Unknown type_id')
1098        
1099        self.objects_db[object_id] = obj
1100        id_: int = id(obj)
1101        self.objects_ids[object_id] = id_
1102        self.objects_ids_by_id[id_] = object_id
1103        return True, object_id, obj
1104    
1105    doi = deserialize_obj_impl
1106
1107    def deserialize_obj(self, obj_info: Tuple,
1108                        DisableGC=DisableGC,
1109                        ) -> Tuple[bool, int, Any]:
1110        with DisableGC():
1111            return self.deserialize_obj_impl(obj_info)
1112    
1113    do = deserialize_obj
1114
1115
1116jit_module()

default_serializable_data_types: Set[Type] = {<class 'int'>, <class 'complex'>, <class 'NoneType'>, <class 'list'>, <class 'frozenset'>, <class 'bytes'>, <class 'str'>, <class 'tuple'>, <class 'dict'>, <class 'float'>, <class 'set'>, <class 'bool'>, <class 'bytearray'>}

known_types: Set[Type] = {<class 'int'>, <class 'complex'>, <class 'NoneType'>, <class 'list'>, <class 'frozenset'>, <class 'bytes'>, <class 'str'>, <class 'tuple'>, <class 'dict'>, <class 'float'>, <class 'set'>, <class 'slice'>, <class 'bool'>, <class 'bytearray'>}

known_data_types: Set[Type] = {<class 'int'>, <class 'NoneType'>, <class 'bytes'>, <class 'str'>, <class 'float'>, <class 'slice'>, <class 'bool'>, <class 'bytearray'>}

known_container_types: Set[Type] = {<class 'complex'>, <class 'list'>, <class 'tuple'>, <class 'dict'>, <class 'set'>, <class 'frozenset'>}

class DataType(enum.IntEnum): View Source

 97class DataType(IntEnum):
 98    class_ = 0  # type is not in known_types and not in serializable_data_types
 99    int_ = 1
100    float_ = 2
101    complex_ = 3
102    str_ = 4
103    bytes_ = 5
104    bytearray_ = 6
105    bool_ = 7
106    none_ = 8
107    list_ = 9
108    tuple_ = 10
109    set_ = 11
110    frozenset_ = 12
111    dict_ = 13
112    slice_ = 14
113    unknown_serializable = 15  # type is in serializable_data_types but not in known_types

An enumeration.

class_ = <DataType.class_: 0>

int_ = <DataType.int_: 1>

float_ = <DataType.float_: 2>

complex_ = <DataType.complex_: 3>

str_ = <DataType.str_: 4>

bytes_ = <DataType.bytes_: 5>

bytearray_ = <DataType.bytearray_: 6>

bool_ = <DataType.bool_: 7>

none_ = <DataType.none_: 8>

list_ = <DataType.list_: 9>

tuple_ = <DataType.tuple_: 10>

set_ = <DataType.set_: 11>

frozenset_ = <DataType.frozenset_: 12>

dict_ = <DataType.dict_: 13>

slice_ = <DataType.slice_: 14>

unknown_serializable = <DataType.unknown_serializable: 15>

Inherited Members

enum.Enum: name; value
builtins.int: conjugate; bit_length; to_bytes; from_bytes; as_integer_ratio; real; imag; numerator; denominator

data_type: Dict[DataType, Type] = {0: <class 'object'>, 1: <class 'int'>, 2: <class 'float'>, 3: <class 'complex'>, 4: <class 'str'>, 5: <class 'bytes'>, 6: <class 'bytearray'>, 7: <class 'bool'>, 8: <class 'NoneType'>, 9: <class 'list'>, 10: <class 'tuple'>, 11: <class 'set'>, 12: <class 'frozenset'>, 13: <class 'dict'>, 14: <class 'slice'>, 15: None}

data_type_by_type: Dict[Type, DataType] = {<class 'object'>: 0, <class 'int'>: 1, <class 'float'>: 2, <class 'complex'>: 3, <class 'str'>: 4, <class 'bytes'>: 5, <class 'bytearray'>: 6, <class 'bool'>: 7, <class 'NoneType'>: 8, <class 'list'>: 9, <class 'tuple'>: 10, <class 'set'>: 11, <class 'frozenset'>: 12, <class 'dict'>: 13, <class 'slice'>: 14, None: 15}

class ClassInfoFields(enum.IntEnum): View Source

156class ClassInfoFields(IntEnum):
157    class_id = 0
158    class_name = 1
159    module_importable_str = 2
160    owning_names_path = 3

An enumeration.

class_id = <ClassInfoFields.class_id: 0>

class_name = <ClassInfoFields.class_name: 1>

module_importable_str = <ClassInfoFields.module_importable_str: 2>

owning_names_path = <ClassInfoFields.owning_names_path: 3>

Inherited Members

enum.Enum: name; value
builtins.int: conjugate; bit_length; to_bytes; from_bytes; as_integer_ratio; real; imag; numerator; denominator

class ObjectInfoFields(enum.IntEnum): View Source

163class ObjectInfoFields(IntEnum):
164    object_id = 0  # (type: int)
165    type_id = 1  # (type: DataType)
166    object_ = 2  # (Optional), (type: Any). Link to object itself if `(type(obj) in serializable_data_types)`. Not used otherwise.
167    class_id = 3  # (Optional), (type: int). Used if `type_id == 0`
168    clonable_slots = 4  # (Optional), (type: Tuple[Tuple[str, Any]]). Used if `type_id == 0`. Holds ID's (object_id) of slots objects.
169    clonable_dict_items = 5  # (Optional), (type: Tuple[Tuple[str, Any]]). Used if `type_id == 0`. Holds ID's (object_id) of value objects.
170    # contained_items = 6  # (Optional), (type: Union[Tuple, List, Set, FrozenSet, Dict]). Used if `type_id in {9, 10, 11, 12, 13, 14}`. Holds ID's (object_id) of contained items (for bothe keys and values in the case of Dict).
171    contained_mapping = 6  # (Optional), (type: Union[Tuple, List, Set, FrozenSet, Dict]). Used if `type_id in {9, 10, 11, 12, 13, 14}`. Holds ID's (object_id) of contained items (for bothe keys and values in the case of Dict).
172    contained_sequence = 7  # (Optional), (type: Union[Tuple, List, Set, FrozenSet, Dict]). Used if `type_id in {9, 10, 11, 12, 13, 14}`. Holds ID's (object_id) of contained items (for bothe keys and values in the case of Dict).
173    contained_set = 8  # (Optional), (type: Union[Tuple, List, Set, FrozenSet, Dict]). Used if `type_id in {9, 10, 11, 12, 13, 14}`. Holds ID's (object_id) of contained items (for bothe keys and values in the case of Dict).

An enumeration.

object_id = <ObjectInfoFields.object_id: 0>

type_id = <ObjectInfoFields.type_id: 1>

object_ = <ObjectInfoFields.object_: 2>

class_id = <ObjectInfoFields.class_id: 3>

clonable_slots = <ObjectInfoFields.clonable_slots: 4>

clonable_dict_items = <ObjectInfoFields.clonable_dict_items: 5>

contained_mapping = <ObjectInfoFields.contained_mapping: 6>

contained_sequence = <ObjectInfoFields.contained_sequence: 7>

contained_set = <ObjectInfoFields.contained_set: 8>

Inherited Members

enum.Enum: name; value
builtins.int: conjugate; bit_length; to_bytes; from_bytes; as_integer_ratio; real; imag; numerator; denominator

class CanNotAdjustToSerializableError(builtins.Exception): View Source

176class CanNotAdjustToSerializableError(Exception):
177    pass

Common base class for all non-exit exceptions.

Inherited Members

builtins.Exception: Exception
builtins.BaseException: with_traceback; args

class CanNotAdjustFromSerializableError(builtins.Exception): View Source

180class CanNotAdjustFromSerializableError(Exception):
181    pass

Common base class for all non-exit exceptions.

Inherited Members

builtins.Exception: Exception
builtins.BaseException: with_traceback; args