cengal.data_manipulation.remote_objects.versions.v_0_fast.remote_objects

View Source

  1#!/usr/bin/env python
  2# coding=utf-8
  3
  4# Copyright © 2012-2024 ButenkoMS. All rights reserved. Contacts: <gtalk@butenkoms.space>
  5# 
  6# Licensed under the Apache License, Version 2.0 (the "License");
  7# you may not use this file except in compliance with the License.
  8# You may obtain a copy of the License at
  9# 
 10#     http://www.apache.org/licenses/LICENSE-2.0
 11# 
 12# Unless required by applicable law or agreed to in writing, software
 13# distributed under the License is distributed on an "AS IS" BASIS,
 14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 15# See the License for the specific language governing permissions and
 16# limitations under the License.
 17
 18
 19__all__ = [
 20    'default_serializable_data_types',
 21    'known_types',
 22    'known_data_types',
 23    'known_container_types',
 24    'DataType',
 25    'data_type',
 26    'data_type_by_type',
 27    'ClassInfoFields',
 28    'ObjectInfoFields',
 29    'CanNotAdjustToSerializableError',
 30    'CanNotAdjustFromSerializableError',
 31    'RemoteObjectsManager',
 32]
 33
 34
 35from cengal.introspection.inspect import (
 36    entity_module_importable_str_and_owning_names_path,
 37    entity_by_name_module_importable_str_and_owning_names_path,
 38    filled_slot_names_with_values_gen,
 39    is_callable, is_async,
 40    is_setable_data_descriptor,
 41)
 42from cengal.code_flow_control.smart_values import ResultExistence
 43from cengal.code_flow_control.gc import DisableGC
 44from cengal.data_generation.id_generator import IDGenerator, GeneratorType
 45from enum import IntEnum
 46from collections.abc import MutableMapping, MutableSequence, MutableSet
 47from struct import pack, unpack
 48from inspect import getattr_static
 49from typing import Any, Dict, Optional, Callable, Set, Type, Tuple, List, FrozenSet
 50
 51
 52"""
 53Module Docstring
 54Docstrings: http://www.python.org/dev/peps/pep-0257/
 55"""
 56
 57__author__ = "ButenkoMS <gtalk@butenkoms.space>"
 58__copyright__ = "Copyright © 2012-2024 ButenkoMS. All rights reserved. Contacts: <gtalk@butenkoms.space>"
 59__credits__ = ["ButenkoMS <gtalk@butenkoms.space>", ]
 60__license__ = "Apache License, Version 2.0"
 61__version__ = "4.4.1"
 62__maintainer__ = "ButenkoMS <gtalk@butenkoms.space>"
 63__email__ = "gtalk@butenkoms.space"
 64# __status__ = "Prototype"
 65__status__ = "Development"
 66# __status__ = "Production"
 67
 68
 69default_serializable_data_types: Set[Type] = {
 70    int, float, complex, str, bytes, bytearray, bool, type(None), list, tuple, set, frozenset, dict
 71}
 72
 73
 74known_types: Set[Type] = {
 75    int, float, complex, str, bytes, bytearray, bool, type(None), list, tuple, set, frozenset, dict,
 76    slice, 
 77}
 78
 79
 80known_data_types: Set[Type] = {
 81    int, float, str, bytes, bytearray, bool, type(None), slice,
 82}
 83
 84
 85known_container_types: Set[Type] = {
 86    complex, list, tuple, set, frozenset, dict,
 87}
 88
 89
 90class DataType(IntEnum):
 91    class_ = 0  # type is not in known_types and not in serializable_data_types
 92    int_ = 1
 93    float_ = 2
 94    complex_ = 3
 95    str_ = 4
 96    bytes_ = 5
 97    bytearray_ = 6
 98    bool_ = 7
 99    none_ = 8
100    list_ = 9
101    tuple_ = 10
102    set_ = 11
103    frozenset_ = 12
104    dict_ = 13
105    slice_ = 14
106    unknown_serializable = 15  # type is in serializable_data_types but not in known_types
107
108
109data_type: Dict[DataType, Type] = {
110    DataType.class_: object,
111    DataType.int_: int,
112    DataType.float_: float,
113    DataType.complex_: complex,
114    DataType.str_: str,
115    DataType.bytes_: bytes,
116    DataType.bytearray_: bytearray,
117    DataType.bool_: bool,
118    DataType.none_: type(None),
119    DataType.list_: list,
120    DataType.tuple_: tuple,
121    DataType.set_: set,
122    DataType.frozenset_: frozenset,
123    DataType.dict_: dict,
124    DataType.slice_: slice,
125    DataType.unknown_serializable: None,
126}
127
128
129data_type_by_type: Dict[Type, DataType] = {
130    object: DataType.class_,
131    int: DataType.int_,
132    float: DataType.float_,
133    complex: DataType.complex_,
134    str: DataType.str_,
135    bytes: DataType.bytes_,
136    bytearray: DataType.bytearray_,
137    bool: DataType.bool_,
138    type(None): DataType.none_,
139    list: DataType.list_,
140    tuple: DataType.tuple_,
141    set: DataType.set_,
142    frozenset: DataType.frozenset_,
143    dict: DataType.dict_,
144    slice: DataType.slice_,
145    None: DataType.unknown_serializable,
146}
147
148
149class ClassInfoFields(IntEnum):
150    class_id = 0
151    class_name = 1
152    module_importable_str = 2
153    owning_names_path = 3
154
155
156class ObjectInfoFields(IntEnum):
157    object_id = 0  # (type: int)
158    type_id = 1  # (type: DataType)
159    object_ = 2  # (Optional), (type: Any). Link to object itself if `(type(obj) in serializable_data_types)`. Not used otherwise.
160    class_id = 3  # (Optional), (type: int). Used if `type_id == DataType.class_`
161    clonable_slots = 4  # (Optional), (type: Tuple[Tuple[str, Any]]). Used if `type_id == DataType.class_`. Holds ID's (object_id) of slots objects.
162    clonable_dict_items = 5  # (Optional), (type: Tuple[Tuple[str, Any]]). Used if `type_id == DataType.class_`. Holds ID's (object_id) of value objects.
163    # contained_items = 6  # (Optional), (type: Union[Tuple, List, Set, FrozenSet, Dict]). Used if `type_id in {DataType.list_, DataType.tuple_, DataType.set_, DataType.frozenset_, DataType.dict_, DataType.slice_}`. Holds ID's (object_id) of contained items (for bothe keys and values in the case of Dict).
164    contained_mapping = 6  # (Optional), (type: Union[Tuple, List, Set, FrozenSet, Dict]). Used if `type_id in {DataType.list_, DataType.tuple_, DataType.set_, DataType.frozenset_, DataType.dict_, DataType.slice_}`. Holds ID's (object_id) of contained items (for bothe keys and values in the case of Dict).
165    contained_sequence = 7  # (Optional), (type: Union[Tuple, List, Set, FrozenSet, Dict]). Used if `type_id in {DataType.list_, DataType.tuple_, DataType.set_, DataType.frozenset_, DataType.dict_, DataType.slice_}`. Holds ID's (object_id) of contained items (for bothe keys and values in the case of Dict).
166    contained_set = 8  # (Optional), (type: Union[Tuple, List, Set, FrozenSet, Dict]). Used if `type_id in {DataType.list_, DataType.tuple_, DataType.set_, DataType.frozenset_, DataType.dict_, DataType.slice_}`. Holds ID's (object_id) of contained items (for bothe keys and values in the case of Dict).
167
168
169class CanNotAdjustToSerializableError(Exception):
170    pass
171
172
173class CanNotAdjustFromSerializableError(Exception):
174    pass
175
176
177class RemoteObjectsManager:
178    def __init__(self, 
179                 on_new_class_handler: Optional[Callable] = None,
180                 on_new_obj_info_handler: Optional[Callable] = None,
181                 # classess_db: Optional[Dict[int, Type]] = None, 
182                 objects_db: Optional[Dict[int, Any]] = None, 
183                 classes_id_gen: Optional[Callable] = None,
184                 objects_id_gen: Optional[Callable] = None,
185                 serializable_data_types: Optional[Set[Type]] = None,
186                 ) -> None:
187        # self.classess_db: Dict[int, Type] = dict() if classess_db is None else classess_db
188        self.classes_id_gen: IDGenerator = IDGenerator() if classes_id_gen is None else classes_id_gen
189        self.objects_db: Dict[int, Any] = dict() if objects_db is None else objects_db
190        self.objects_id_gen: IDGenerator = IDGenerator() if objects_id_gen is None else objects_id_gen
191        self.serializable_data_types: Set[Type] = default_serializable_data_types if serializable_data_types is None else serializable_data_types
192        self.serializable_any: bool = not self.serializable_data_types
193        self.serializable_int: bool = (int in self.serializable_data_types) or self.serializable_any
194        self.serializable_float: bool = (float in self.serializable_data_types) or self.serializable_any
195        self.serializable_complex: bool = (complex in self.serializable_data_types) or self.serializable_any
196        self.serializable_str: bool = (str in self.serializable_data_types) or self.serializable_any
197        self.serializable_bytes: bool = (bytes in self.serializable_data_types) or self.serializable_any
198        self.serializable_bytearray: bool = (bytearray in self.serializable_data_types) or self.serializable_any
199        self.serializable_bool: bool = (bool in self.serializable_data_types) or self.serializable_any
200        self.serializable_none: bool = (type(None) in self.serializable_data_types) or self.serializable_any
201        self.serializable_list: bool = (list in self.serializable_data_types) or self.serializable_any
202        self.serializable_tuple: bool = (tuple in self.serializable_data_types) or self.serializable_any
203        self.serializable_set: bool = (set in self.serializable_data_types) or self.serializable_any
204        self.serializable_frozenset: bool = (frozenset in self.serializable_data_types) or self.serializable_any
205        self.serializable_dict: bool = (dict in self.serializable_data_types) or self.serializable_any
206        self.serializable_slice: bool = (slice in self.serializable_data_types) or self.serializable_any
207        self.known_classes: Dict[Type, int] = dict()
208        self.known_classes_by_id: Dict[int, Type] = dict()
209        self.known_classes_info: Dict[Tuple, int] = dict()
210        self.known_classes_info_by_id: Dict[int, Tuple] = dict()
211        self.on_new_class_handler: Optional[Callable] = on_new_class_handler
212        self.on_new_obj_info_handler: Optional[Callable] = on_new_obj_info_handler
213        self.objects_ids: Dict[int, int] = dict()  # (Key: object_id, Value: id(obj))
214        self.objects_ids_by_id: Dict[int, int] = dict()  # (Key: id(obj), Value: object_id)
215
216    def del_object_by_id(self, id_: int) -> None:
217        if id_ in self.objects_ids_by_id:
218            object_id = self.objects_ids_by_id[id_]
219            self.objects_ids_by_id.pop(id_, None)
220            self.objects_ids.pop(object_id, None)
221            self.objects_db.pop(object_id, None)
222    
223    dobi = del_object_by_id
224    
225    def del_object_by_object_id(self, object_id: int) -> None:
226        if object_id in self.objects_ids:
227            id_ = self.objects_ids[object_id]
228            self.objects_ids_by_id.pop(id_, None)
229            self.objects_ids.pop(object_id, None)
230            self.objects_db.pop(object_id, None)
231    
232    doboi = del_object_by_object_id
233    
234    def adjust_to_serializable(self, type_id: DataType, obj: Any) -> Any:
235        if DataType.int_ == type_id:
236            return obj
237        elif DataType.float_ == type_id:
238            return obj
239        elif DataType.complex_ == type_id:
240            if self.serializable_complex:
241                return obj
242            elif self.serializable_bytes:
243                return pack('=dd', obj.real, obj.imag)
244            elif self.serializable_bytearray:
245                return bytearray(pack('=dd', obj.real, obj.imag))
246            elif self.serializable_str:
247                return str(obj)
248            elif self.serializable_float:
249                if self.serializable_tuple:
250                    return (obj.real, obj.imag)
251                elif self.serializable_list:
252                    return [obj.real, obj.imag]
253                else:
254                    pass
255            else:
256                pass
257        elif DataType.str_ == type_id:
258            return obj
259        elif DataType.bytes_ == type_id:
260            if self.serializable_bytes:
261                return obj
262            elif self.serializable_bytearray:
263                return bytearray(obj)
264            elif self.serializable_str:
265                return obj.hex()
266            elif self.serializable_tuple:
267                if self.serializable_int:
268                    return  tuple(int(c) for c in obj)
269                if self.serializable_float:
270                    return  tuple(float(int(c)) for c in obj)
271                else:
272                    pass
273            elif self.serializable_list:
274                if self.serializable_int:
275                    return  [int(c) for c in obj]
276                if self.serializable_float:
277                    return  [float(int(c)) for c in obj]
278                else:
279                    pass
280            else:
281                pass
282        elif DataType.bytearray_ == type_id:
283            if self.serializable_bytearray:
284                return obj
285            elif self.serializable_bytes:
286                return bytes(obj)
287            elif self.serializable_str:
288                return obj.hex()
289            elif self.serializable_tuple:
290                if self.serializable_int:
291                    return  tuple(int(c) for c in obj)
292                if self.serializable_float:
293                    return  tuple(float(int(c)) for c in obj)
294                else:
295                    pass
296            elif self.serializable_list:
297                if self.serializable_int:
298                    return  [int(c) for c in obj]
299                if self.serializable_float:
300                    return  [float(int(c)) for c in obj]
301                else:
302                    pass
303            else:
304                pass
305        elif DataType.bool_ == type_id:
306            return obj
307        elif DataType.none_ == type_id:
308            return obj
309        elif DataType.list_ == type_id:
310            if self.serializable_list:
311                return obj
312            elif self.serializable_tuple:
313                return tuple(obj)
314            elif self.serializable_dict:
315                return dict({index: item for index, item in enumerate(obj)})
316            else:
317                pass
318        elif DataType.tuple_ == type_id:
319            if self.serializable_tuple:
320                return obj
321            elif self.serializable_list:
322                return list(obj)
323            elif self.serializable_dict:
324                return dict({index: item for index, item in enumerate(obj)})
325            else:
326                pass
327        elif DataType.set_ == type_id:
328            if self.serializable_set:
329                return obj
330            elif self.serializable_frozenset:
331                return frozenset(obj)
332            elif self.serializable_tuple:
333                return tuple(obj)
334            elif self.serializable_list:
335                return list(obj)
336            elif self.serializable_dict:
337                return dict({k: None for k in obj})
338            else:
339                pass
340        elif DataType.frozenset_ == type_id:
341            if self.serializable_frozenset:
342                return obj
343            elif self.serializable_set:
344                return set(obj)
345            elif self.serializable_tuple:
346                return tuple(obj)
347            elif self.serializable_list:
348                return list(obj)
349            elif self.serializable_dict:
350                return dict({k: None for k in obj})
351            else:
352                pass
353        elif DataType.dict_ == type_id:
354            return obj
355        elif DataType.slice_ == type_id:
356            if self.serializable_slice:
357                return obj
358            elif self.serializable_tuple:
359                return (obj.start, obj.stop, obj.step)
360            elif self.serializable_list:
361                return [obj.start, obj.stop, obj.step]
362            elif self.serializable_dict:
363                return {0: obj.start, 1: obj.stop, 2: obj.step}
364            else:
365                pass
366        else:
367            raise RuntimeError('Unknown type_id')
368        
369        raise CanNotAdjustToSerializableError(f'Can not adjust to serializable. Type: {type_id}, obj: {obj}')
370
371    ats = adjust_to_serializable
372
373    def adjust_from_serializable(self, type_id: DataType, obj: Any) -> Any:
374        if DataType.int_ == type_id:
375            return obj
376        elif DataType.float_ == type_id:
377            return obj
378        elif DataType.complex_ == type_id:
379            if self.serializable_complex:
380                return obj
381            elif self.serializable_bytes:
382                return complex(*unpack('=dd', obj))
383            elif self.serializable_bytearray:
384                return complex(*unpack('=dd', bytes(obj)))
385            elif self.serializable_str:
386                return complex(*obj)
387            elif self.serializable_float:
388                if self.serializable_tuple:
389                    return complex(*obj)
390                elif self.serializable_list:
391                    return complex(*obj)
392                else:
393                    pass
394            else:
395                pass
396        elif DataType.str_ == type_id:
397            return obj
398        elif DataType.bytes_ == type_id:
399            if self.serializable_bytes:
400                return obj
401            elif self.serializable_bytearray:
402                return bytes(obj)
403            elif self.serializable_str:
404                return bytes.fromhex(obj)
405            elif self.serializable_tuple:
406                if self.serializable_int:
407                    return  b''.join(item.to_bytes(1, 'little') for item in obj)
408                if self.serializable_float:
409                    return  b''.join((int(round(item))).to_bytes(1, 'little') for item in obj)
410                else:
411                    pass
412            elif self.serializable_list:
413                if self.serializable_int:
414                    return  b''.join(item.to_bytes(1, 'little') for item in obj)
415                if self.serializable_float:
416                    return  b''.join((int(round(item))).to_bytes(1, 'little') for item in obj)
417                else:
418                    pass
419            else:
420                pass
421        elif DataType.bytearray_ == type_id:
422            if self.serializable_bytearray:
423                return obj
424            elif self.serializable_bytes:
425                return bytearray(obj)
426            elif self.serializable_str:
427                return bytearray(bytes.fromhex(obj))
428            elif self.serializable_tuple:
429                if self.serializable_int:
430                    return  bytearray(b''.join(item.to_bytes(1, 'little') for item in obj))
431                if self.serializable_float:
432                    return  bytearray(b''.join((int(round(item))).to_bytes(1, 'little') for item in obj))
433                else:
434                    pass
435            elif self.serializable_list:
436                if self.serializable_int:
437                    return  bytearray(b''.join(item.to_bytes(1, 'little') for item in obj))
438                if self.serializable_float:
439                    return  bytearray(b''.join((int(round(item))).to_bytes(1, 'little') for item in obj))
440                else:
441                    pass
442            else:
443                pass
444        elif DataType.bool_ == type_id:
445            return obj
446        elif DataType.none_ == type_id:
447            return None
448        elif DataType.list_ == type_id:
449            if self.serializable_list:
450                return obj
451            elif self.serializable_tuple:
452                return list(obj)
453            elif self.serializable_dict:
454                return [value for key, value in sorted(obj.items(), key=lambda x: x[0])]
455            else:
456                pass
457        elif DataType.tuple_ == type_id:
458            if self.serializable_tuple:
459                return obj
460            elif self.serializable_list:
461                return tuple(obj)
462            elif self.serializable_dict:
463                return tuple(value for key, value in sorted(obj.items(), key=lambda x: x[0]))
464            else:
465                pass
466        elif DataType.set_ == type_id:
467            if self.serializable_set:
468                return obj
469            elif self.serializable_frozenset:
470                return set(obj)
471            elif self.serializable_tuple:
472                return set(obj)
473            elif self.serializable_list:
474                return set(obj)
475            elif self.serializable_dict:
476                return set(obj.keys())
477            else:
478                pass
479        elif DataType.frozenset_ == type_id:
480            if self.serializable_frozenset:
481                return obj
482            elif self.serializable_set:
483                return frozenset(obj)
484            elif self.serializable_tuple:
485                return frozenset(obj)
486            elif self.serializable_list:
487                return frozenset(obj)
488            elif self.serializable_dict:
489                return frozenset(obj.keys())
490            else:
491                pass
492        elif DataType.dict_ == type_id:
493            return obj
494        elif DataType.slice_ == type_id:
495            if self.serializable_slice:
496                return obj
497            elif self.serializable_tuple:
498                return slice(*obj)
499            elif self.serializable_list:
500                return slice(*obj)
501            elif self.serializable_dict:
502                return slice(obj[0], obj[1], obj[2])
503            else:
504                pass
505        else:
506            raise RuntimeError('Unknown type_id')
507        
508        raise CanNotAdjustFromSerializableError(f'Can not adjust from serializable. Type: {type_id}, obj: {obj}')
509    
510    afs = adjust_from_serializable
511    
512    # def is_replicatable_object_attribute(self, attribute: Any) -> bool:
513    #     if is_setable_data_descriptor(attribute):
514    #         data = attribute.__get__(None, None)
515    #     elif is_callable(attribute):
516    #         return False
517    #     else:
518    #         return True
519
520    def serialize_container(self, type_id: DataType, obj: Any) -> Any:
521        if DataType.list_ == type_id:
522            new_obj = list()
523            for item in obj:
524                result: ResultExistence[int] = self.serialize_impl(item)
525                if result:
526                    new_obj.append(result.value)
527        elif DataType.tuple_ == type_id:
528            new_obj = list()
529            for item in obj:
530                result: ResultExistence[int] = self.serialize_impl(item)
531                if result:
532                    new_obj.append(result.value)
533            
534            new_obj = tuple(new_obj)
535        elif DataType.set_ == type_id:
536            new_obj = set()
537            for item in obj:
538                result: ResultExistence[int] = self.serialize_impl(item)
539                if result:
540                    new_obj.add(result.value)
541        elif DataType.frozenset_ == type_id:
542            new_obj = set()
543            for item in obj:
544                result: ResultExistence[int] = self.serialize_impl(item)
545                if result:
546                    new_obj.add(result.value)
547                
548            new_obj = frozenset(new_obj)
549        elif DataType.dict_ == type_id:
550            new_obj = dict()
551            for key, value in obj.items():
552                key_result: ResultExistence[int] = self.serialize_impl(key)
553                value_result: ResultExistence[int] = self.serialize_impl(value)
554                if key_result and value_result:
555                    new_obj[key_result.value] = value_result.value
556        
557        return new_obj
558    
559    sc = serialize_container
560
561    def serialize_impl(self, obj: Any, ignore_empty_classes: bool = False) -> ResultExistence[int]:
562        result_exists: bool = True
563        id_: int = id(obj)
564        obj_type = type(obj)
565        if (int != obj_type) and (id_ in self.objects_ids_by_id):
566            new_object: bool = False
567            object_id: int = self.objects_ids_by_id[id_]
568        else:
569            # int object must always produce new object_id because first 256 ints are persistent across Python sessions and
570            # this can cause issues within users of current module. For example within `cengal/hardware/memory/shared_memory`
571            # which changes int values inline instead of producing new objects.
572            new_object = True
573            object_id = self.objects_id_gen()
574            self.objects_ids[object_id] = id_
575            self.objects_db[object_id] = obj
576        
577        if not new_object:
578            return ResultExistence[int](result_exists, object_id)
579        
580        if  self.serializable_any or (obj_type in self.serializable_data_types):
581            serializable: bool = True
582            type_id: DataType = data_type_by_type.get(obj_type, DataType.unknown_serializable)
583        else:
584            serializable = False
585            type_id = data_type_by_type.get(obj_type, DataType.class_)
586        
587        if obj_type in known_container_types:
588            known_container: bool = True
589        else:
590            known_container = False
591        
592        class_info: Dict[int, Any] = None
593        known_data: bool = None
594        class_id: int = None
595        is_new_class: bool = None
596        class_name: str = None
597        new_obj_slots: List[Tuple[str, Any]] = None
598        new_obj_dict: Dict[str, Any] = None
599        obj_mapping: Dict = None
600        obj_sequence: List = None
601        obj_set: Set = None
602        if serializable:
603            if known_container:
604                object_info = (
605                    object_id,
606                    type_id.value,
607                    self.sc(type_id, obj),
608                )
609            else:
610                object_info = (
611                    object_id,
612                    type_id.value,
613                    obj,
614                )
615        else:
616            if obj_type in known_data_types:
617                known_data = True
618            else:
619                known_data = False
620            
621            if DataType.class_ == type_id:
622                new_obj_slots = list()
623                for slot_name, slot_value in filled_slot_names_with_values_gen(obj):
624                    adjusted_slot_name = slot_name
625                    item_result: ResultExistence[int] = self.serialize_impl(slot_value)
626                    if item_result:
627                        if self.serializable_tuple:
628                            new_obj_slots.append((adjusted_slot_name, item_result.value))
629                        elif self.serializable_list:
630                            new_obj_slots.append([adjusted_slot_name, item_result.value])
631                        else:
632                            new_obj_slots.append(self.ats(DataType.tuple_, (adjusted_slot_name, item_result.value)))
633                
634                if new_obj_slots:
635                    if self.serializable_list:
636                        pass
637                    elif self.serializable_tuple:
638                        new_obj_slots = tuple(new_obj_slots)
639                    else:
640                        new_obj_slots = self.ats(DataType.list_, new_obj_slots)
641                
642                new_obj_dict = dict()
643                if hasattr(obj, '__dict__'):
644                    for key, value in obj.__dict__.items():
645                        # raw_value = getattr_static(obj, key)
646                        # if hasattr(raw_value, '__get__') and (not hasattr(raw_value, '__set__')):
647                        #     # if not setable descriptor
648                        #     continue
649
650                        adjusted_key = key
651
652                        value_result: ResultExistence[int] = self.serialize_impl(value)
653                        if value_result:
654                            new_obj_dict[adjusted_key] = value_result.value
655                        else:
656                            continue
657                
658                obj_mapping = dict()
659                obj_sequence = list()
660                obj_set = list()
661                if isinstance(obj, MutableMapping):
662                    for key, value in obj.items():
663                        key_result: ResultExistence[int] = self.serialize_impl(value)
664                        if not key_result:
665                            continue
666
667                        value_result: ResultExistence[int] = self.serialize_impl(value)
668                        if value_result:
669                            obj_mapping[key_result.value] = value_result.value
670                        else:
671                            continue
672                elif isinstance(obj, MutableSequence):
673                    for item in obj:
674                        item_result: ResultExistence[int] = self.serialize_impl(item)
675                        if item_result:
676                            obj_sequence.append(item_result.value)
677                        else:
678                            continue
679                    
680                    if obj_sequence:
681                        if self.serializable_list:
682                            pass
683                        else:
684                            obj_sequence = self.ats(DataType.list_, obj_sequence)
685                elif isinstance(obj, MutableSet):
686                    for item in obj:
687                        item_result: ResultExistence[int] = self.serialize_impl(item)
688                        if item_result:
689                            obj_set.append(item_result.value)
690                        else:
691                            continue
692                    
693                    if obj_set:
694                        if self.serializable_set:
695                            pass
696                        else:
697                            obj_set = self.ats(DataType.list_, obj_set)
698                else:
699                    pass
700
701                if ignore_empty_classes:
702                    result_exists = new_obj_slots or new_obj_dict or obj_mapping or obj_sequence or obj_set
703                
704                if result_exists:
705                    if obj_type in self.known_classes:
706                        is_new_class = False
707                        class_id = self.known_classes[obj_type]
708                    else:
709                        class_info = dict()
710                        is_new_class = True
711                        class_name = obj_type.__name__
712                        class_id = self.classes_id_gen()
713                        self.known_classes[obj_type] = class_id
714                        self.known_classes_by_id[class_id] = obj_type
715                        module_importable_str, owning_names_path = entity_module_importable_str_and_owning_names_path(obj)
716                        module_importable_str_and_owning_names_path = (class_name, module_importable_str, tuple(owning_names_path))
717                        self.known_classes_info[module_importable_str_and_owning_names_path] = class_id
718                        self.known_classes_info_by_id[class_id] = module_importable_str_and_owning_names_path
719
720                        class_info = (
721                            class_id,
722                            class_name,
723                            module_importable_str,
724                            owning_names_path,
725                        )
726                        
727                        if self.on_new_class_handler:
728                            self.on_new_class_handler(
729                                class_info,
730                                obj_type,
731                                class_id,
732                                class_name,
733                                module_importable_str,
734                                owning_names_path,
735                                module_importable_str_and_owning_names_path
736                            )
737
738                # will be later serialized much faster than without `if else None`
739                object_info = (
740                    object_id,
741                    type_id.value,
742                    0,
743                    class_id,
744                    new_obj_slots if new_obj_slots else 0,
745                    new_obj_dict if new_obj_dict else 0,
746                    obj_mapping if obj_mapping else 0,
747                    obj_sequence if obj_sequence else 0,
748                    obj_set if obj_set else 0,
749                )
750            elif known_data:
751                object_info = (
752                    object_id,
753                    type_id.value,
754                    self.ats(type_id, obj),
755                )
756            elif known_container:
757                object_info = (
758                    object_id,
759                    type_id.value,
760                    self.ats(type_id, self.sc(type_id, obj)),
761                )
762            else:
763                raise RuntimeError('Unknown type_id')
764
765        if result_exists:
766            if self.on_new_obj_info_handler:
767                self.on_new_obj_info_handler(
768                    object_info,
769                    obj,
770                    object_id,
771                    type_id,
772                    serializable,
773                    known_container,
774                    known_data,
775                    class_id,
776                    is_new_class,
777                    new_obj_slots,
778                    new_obj_dict,
779                    obj_mapping,
780                    obj_sequence,
781                    obj_set,
782                )
783        
784        return ResultExistence[int](result_exists, object_id)
785    
786    si = serialize_impl
787
788    def serialize(self, obj: Any, ignore_empty_classes: bool = False) -> ResultExistence[int]:
789        with DisableGC():
790            return self.serialize_impl(obj, ignore_empty_classes)
791    
792    s = serialize
793    
794    def deserialize_class_impl(self, class_info: Tuple) -> ResultExistence[Tuple[int, Type]]:
795        class_id: int = class_info[ClassInfoFields.class_id.value]
796        if class_id in self.known_classes_by_id:
797            return ResultExistence[Tuple[int, Type]](True, (class_id, self.known_classes_by_id[class_id]))
798        else:
799            class_name: str = class_info[ClassInfoFields.class_name.value]
800            module_importable_str: str = class_info[ClassInfoFields.module_importable_str.value]
801            owning_names_path: List[str] = self.afs(DataType.list_, class_info[ClassInfoFields.owning_names_path.value])
802            obj_class: Type = entity_by_name_module_importable_str_and_owning_names_path(class_name, module_importable_str, owning_names_path)
803            self.known_classes_by_id[class_id] = obj_class
804            self.known_classes[obj_class] = class_id
805            class_info_tuple: Tuple = (class_name, module_importable_str, tuple(owning_names_path)) 
806            self.known_classes_info[class_info_tuple] = class_id
807            self.known_classes_info_by_id[class_id] = class_info_tuple
808            return ResultExistence[Tuple[int, Type]](True, (class_id, obj_class))
809    
810    dcli = deserialize_class_impl
811    
812    def deserialize_class(self, class_info: Tuple) -> ResultExistence[Tuple[int, Type]]:
813        with DisableGC():
814            return self.deserialize_class_impl(class_info)
815    
816    dcl = deserialize_class
817
818    def deserialize_container_impl(self, type_id: DataType, obj: Any) -> Any:
819        new_obj = obj
820        if DataType.list_ == type_id:
821            new_obj = list()
822            for item_id in obj:
823                new_obj.append(self.objects_db[item_id])
824        elif DataType.tuple_ == type_id:
825            new_obj = list()
826            for item_id in obj:
827                new_obj.append(self.objects_db[item_id])
828            
829            new_obj = tuple(new_obj)
830        elif DataType.set_ == type_id:
831            new_obj = set()
832            for item_id in obj:
833                new_obj.add(self.objects_db[item_id])
834        elif DataType.frozenset_ == type_id:
835            new_obj = set()
836            for item_id in obj:
837                new_obj.add(self.objects_db[item_id])
838                
839            new_obj = frozenset(new_obj)
840        elif DataType.dict_ == type_id:
841            new_obj = dict()
842            for key_id, value_id in obj.items():
843                new_obj[self.objects_db[int(key_id)]] = self.objects_db[value_id]
844        
845        return new_obj
846    
847    dcoi = deserialize_container_impl
848
849    def deserialize_container(self, type_id: DataType, obj: Any) -> Any:
850        with DisableGC():
851            return self.deserialize_container_impl(type_id, obj)
852    
853    dco = deserialize_container
854
855    def deserialize_obj_impl(self, obj_info: Tuple) -> ResultExistence[Tuple[int, Any]]:
856        object_id: int = obj_info[ObjectInfoFields.object_id.value]
857        if object_id in self.objects_db:
858            return ResultExistence[Tuple[int, Any]](True, (object_id, self.objects_db[object_id]))
859        
860        type_id: DataType = DataType(obj_info[ObjectInfoFields.type_id.value])
861        
862        obj_type = data_type[type_id] 
863        serializable: bool = self.serializable_any or (obj_type in self.serializable_data_types)
864        known_container: bool = obj_type in known_container_types
865        known_data: bool = None
866        
867        if serializable:
868            if known_container:
869                obj = self.dcoi(type_id, obj_info[ObjectInfoFields.object_.value])
870            else:
871                obj = obj_info[ObjectInfoFields.object_.value]
872        else:
873            known_data = obj_type in known_data_types
874            
875            if DataType.class_ == type_id:
876                class_id: int = obj_info[ObjectInfoFields.class_id.value]
877                obj_class: Type = self.known_classes_by_id[class_id]
878                obj: Any = obj_class.__new__(obj_class)
879
880                if obj_info[ObjectInfoFields.clonable_slots.value]:
881                    clonable_slots = self.afs(DataType.list_, obj_info[ObjectInfoFields.clonable_slots.value])
882                    for slot_name, slot_id in clonable_slots:
883                        child_obj = self.objects_db[slot_id]
884                        setattr(obj, slot_name, child_obj)
885                
886                if obj_info[ObjectInfoFields.clonable_dict_items.value]:
887                    clonable_dict_items = obj_info[ObjectInfoFields.clonable_dict_items.value]
888                    for key, value_id in clonable_dict_items.items():
889                        child_obj = self.objects_db[value_id]
890                        setattr(obj, key, child_obj)
891                
892                if obj_info[ObjectInfoFields.contained_mapping.value]:
893                    contained_mapping = obj_info[ObjectInfoFields.contained_mapping.value]
894                    for key_id, value_id in contained_mapping.items():
895                        child_key = self.objects_db[int(key_id)]
896                        child_value = self.objects_db[value_id]
897                        obj[child_key] = child_value
898                
899                if obj_info[ObjectInfoFields.contained_sequence.value]:
900                    contained_sequence = self.afs(DataType.list_, obj_info[ObjectInfoFields.contained_sequence.value])
901                    for item_id in contained_sequence:
902                        child_item = self.objects_db[item_id]
903                        obj.append(child_item)
904                
905                if obj_info[ObjectInfoFields.contained_set.value]:
906                    contained_set = self.afs(DataType.list_, obj_info[ObjectInfoFields.contained_set.value])
907                    for item_id in contained_set:
908                        child_item = self.objects_db[item_id]
909                        obj.add(child_item)
910            elif known_data:
911                obj = self.afs(type_id, obj_info[ObjectInfoFields.object_.value])
912            elif known_container:
913                obj = self.dcoi(type_id, self.afs(type_id, obj_info[ObjectInfoFields.object_.value]))
914            else:
915                raise RuntimeError('Unknown type_id')
916        
917        self.objects_db[object_id] = obj
918        id_: int = id(obj)
919        self.objects_ids[object_id] = id_
920        self.objects_ids_by_id[id_] = object_id
921        return ResultExistence[Tuple[int, Any]](True, (object_id, obj))
922    
923    doi = deserialize_obj_impl
924
925    def deserialize_obj(self, obj_info: Tuple) -> ResultExistence[Tuple[int, Any]]:
926        with DisableGC():
927            return self.deserialize_obj_impl(obj_info)
928    
929    do = deserialize_obj

default_serializable_data_types: Set[Type] = {<class 'int'>, <class 'complex'>, <class 'NoneType'>, <class 'list'>, <class 'frozenset'>, <class 'bytes'>, <class 'str'>, <class 'tuple'>, <class 'dict'>, <class 'float'>, <class 'set'>, <class 'bool'>, <class 'bytearray'>}

known_types: Set[Type] = {<class 'int'>, <class 'complex'>, <class 'NoneType'>, <class 'list'>, <class 'frozenset'>, <class 'bytes'>, <class 'str'>, <class 'tuple'>, <class 'dict'>, <class 'float'>, <class 'set'>, <class 'slice'>, <class 'bool'>, <class 'bytearray'>}

known_data_types: Set[Type] = {<class 'int'>, <class 'NoneType'>, <class 'bytes'>, <class 'str'>, <class 'float'>, <class 'slice'>, <class 'bool'>, <class 'bytearray'>}

known_container_types: Set[Type] = {<class 'complex'>, <class 'list'>, <class 'tuple'>, <class 'dict'>, <class 'set'>, <class 'frozenset'>}

class DataType(enum.IntEnum): View Source

 91class DataType(IntEnum):
 92    class_ = 0  # type is not in known_types and not in serializable_data_types
 93    int_ = 1
 94    float_ = 2
 95    complex_ = 3
 96    str_ = 4
 97    bytes_ = 5
 98    bytearray_ = 6
 99    bool_ = 7
100    none_ = 8
101    list_ = 9
102    tuple_ = 10
103    set_ = 11
104    frozenset_ = 12
105    dict_ = 13
106    slice_ = 14
107    unknown_serializable = 15  # type is in serializable_data_types but not in known_types

An enumeration.

class_ = <DataType.class_: 0>

int_ = <DataType.int_: 1>

float_ = <DataType.float_: 2>

complex_ = <DataType.complex_: 3>

str_ = <DataType.str_: 4>

bytes_ = <DataType.bytes_: 5>

bytearray_ = <DataType.bytearray_: 6>

bool_ = <DataType.bool_: 7>

none_ = <DataType.none_: 8>

list_ = <DataType.list_: 9>

tuple_ = <DataType.tuple_: 10>

set_ = <DataType.set_: 11>

frozenset_ = <DataType.frozenset_: 12>

dict_ = <DataType.dict_: 13>

slice_ = <DataType.slice_: 14>

unknown_serializable = <DataType.unknown_serializable: 15>

Inherited Members

enum.Enum: name; value
builtins.int: conjugate; bit_length; to_bytes; from_bytes; as_integer_ratio; real; imag; numerator; denominator

data_type: Dict[DataType, Type] = {<DataType.class_: 0>: <class 'object'>, <DataType.int_: 1>: <class 'int'>, <DataType.float_: 2>: <class 'float'>, <DataType.complex_: 3>: <class 'complex'>, <DataType.str_: 4>: <class 'str'>, <DataType.bytes_: 5>: <class 'bytes'>, <DataType.bytearray_: 6>: <class 'bytearray'>, <DataType.bool_: 7>: <class 'bool'>, <DataType.none_: 8>: <class 'NoneType'>, <DataType.list_: 9>: <class 'list'>, <DataType.tuple_: 10>: <class 'tuple'>, <DataType.set_: 11>: <class 'set'>, <DataType.frozenset_: 12>: <class 'frozenset'>, <DataType.dict_: 13>: <class 'dict'>, <DataType.slice_: 14>: <class 'slice'>, <DataType.unknown_serializable: 15>: None}

data_type_by_type: Dict[Type, DataType] = {<class 'object'>: <DataType.class_: 0>, <class 'int'>: <DataType.int_: 1>, <class 'float'>: <DataType.float_: 2>, <class 'complex'>: <DataType.complex_: 3>, <class 'str'>: <DataType.str_: 4>, <class 'bytes'>: <DataType.bytes_: 5>, <class 'bytearray'>: <DataType.bytearray_: 6>, <class 'bool'>: <DataType.bool_: 7>, <class 'NoneType'>: <DataType.none_: 8>, <class 'list'>: <DataType.list_: 9>, <class 'tuple'>: <DataType.tuple_: 10>, <class 'set'>: <DataType.set_: 11>, <class 'frozenset'>: <DataType.frozenset_: 12>, <class 'dict'>: <DataType.dict_: 13>, <class 'slice'>: <DataType.slice_: 14>, None: <DataType.unknown_serializable: 15>}

class ClassInfoFields(enum.IntEnum): View Source

150class ClassInfoFields(IntEnum):
151    class_id = 0
152    class_name = 1
153    module_importable_str = 2
154    owning_names_path = 3

An enumeration.

class_id = <ClassInfoFields.class_id: 0>

class_name = <ClassInfoFields.class_name: 1>

module_importable_str = <ClassInfoFields.module_importable_str: 2>

owning_names_path = <ClassInfoFields.owning_names_path: 3>

Inherited Members

enum.Enum: name; value
builtins.int: conjugate; bit_length; to_bytes; from_bytes; as_integer_ratio; real; imag; numerator; denominator

class ObjectInfoFields(enum.IntEnum): View Source

157class ObjectInfoFields(IntEnum):
158    object_id = 0  # (type: int)
159    type_id = 1  # (type: DataType)
160    object_ = 2  # (Optional), (type: Any). Link to object itself if `(type(obj) in serializable_data_types)`. Not used otherwise.
161    class_id = 3  # (Optional), (type: int). Used if `type_id == DataType.class_`
162    clonable_slots = 4  # (Optional), (type: Tuple[Tuple[str, Any]]). Used if `type_id == DataType.class_`. Holds ID's (object_id) of slots objects.
163    clonable_dict_items = 5  # (Optional), (type: Tuple[Tuple[str, Any]]). Used if `type_id == DataType.class_`. Holds ID's (object_id) of value objects.
164    # contained_items = 6  # (Optional), (type: Union[Tuple, List, Set, FrozenSet, Dict]). Used if `type_id in {DataType.list_, DataType.tuple_, DataType.set_, DataType.frozenset_, DataType.dict_, DataType.slice_}`. Holds ID's (object_id) of contained items (for bothe keys and values in the case of Dict).
165    contained_mapping = 6  # (Optional), (type: Union[Tuple, List, Set, FrozenSet, Dict]). Used if `type_id in {DataType.list_, DataType.tuple_, DataType.set_, DataType.frozenset_, DataType.dict_, DataType.slice_}`. Holds ID's (object_id) of contained items (for bothe keys and values in the case of Dict).
166    contained_sequence = 7  # (Optional), (type: Union[Tuple, List, Set, FrozenSet, Dict]). Used if `type_id in {DataType.list_, DataType.tuple_, DataType.set_, DataType.frozenset_, DataType.dict_, DataType.slice_}`. Holds ID's (object_id) of contained items (for bothe keys and values in the case of Dict).
167    contained_set = 8  # (Optional), (type: Union[Tuple, List, Set, FrozenSet, Dict]). Used if `type_id in {DataType.list_, DataType.tuple_, DataType.set_, DataType.frozenset_, DataType.dict_, DataType.slice_}`. Holds ID's (object_id) of contained items (for bothe keys and values in the case of Dict).

An enumeration.

object_id = <ObjectInfoFields.object_id: 0>

type_id = <ObjectInfoFields.type_id: 1>

object_ = <ObjectInfoFields.object_: 2>

class_id = <ObjectInfoFields.class_id: 3>

clonable_slots = <ObjectInfoFields.clonable_slots: 4>

clonable_dict_items = <ObjectInfoFields.clonable_dict_items: 5>

contained_mapping = <ObjectInfoFields.contained_mapping: 6>

contained_sequence = <ObjectInfoFields.contained_sequence: 7>

contained_set = <ObjectInfoFields.contained_set: 8>

Inherited Members

enum.Enum: name; value
builtins.int: conjugate; bit_length; to_bytes; from_bytes; as_integer_ratio; real; imag; numerator; denominator

class CanNotAdjustToSerializableError(builtins.Exception): View Source

170class CanNotAdjustToSerializableError(Exception):
171    pass

Common base class for all non-exit exceptions.

Inherited Members

builtins.Exception: Exception
builtins.BaseException: with_traceback; args

class CanNotAdjustFromSerializableError(builtins.Exception): View Source

174class CanNotAdjustFromSerializableError(Exception):
175    pass

Common base class for all non-exit exceptions.

Inherited Members

builtins.Exception: Exception
builtins.BaseException: with_traceback; args