cengal.hardware.memory.shared_memory.versions.v_0.shared_memory
Module Docstring Docstrings: http://www.python.org/dev/peps/pep-0257/
1#!/usr/bin/env python 2# coding=utf-8 3 4# Copyright © 2012-2024 ButenkoMS. All rights reserved. Contacts: <gtalk@butenkoms.space> 5# 6# Licensed under the Apache License, Version 2.0 (the "License"); 7# you may not use this file except in compliance with the License. 8# You may obtain a copy of the License at 9# 10# http://www.apache.org/licenses/LICENSE-2.0 11# 12# Unless required by applicable law or agreed to in writing, software 13# distributed under the License is distributed on an "AS IS" BASIS, 14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15# See the License for the specific language governing permissions and 16# limitations under the License. 17 18 19# __all__ = ['SharedMemory', 'QueueType', 'Offset', 'Size', 'SharedMemoryError', 20# 'WrongObjectTypeError', 'NoMessagesInQueueError', 21# 'nearest_size', 'nsize', 'TBase', 'IList', 'codec_by_type', 'get_in_line', 'wait_my_turn'] 22 23 24""" 25Module Docstring 26Docstrings: http://www.python.org/dev/peps/pep-0257/ 27""" 28 29__author__ = "ButenkoMS <gtalk@butenkoms.space>" 30__copyright__ = "Copyright © 2012-2024 ButenkoMS. All rights reserved. Contacts: <gtalk@butenkoms.space>" 31__credits__ = ["ButenkoMS <gtalk@butenkoms.space>", ] 32__license__ = "Apache License, Version 2.0" 33__version__ = "4.4.1" 34__maintainer__ = "ButenkoMS <gtalk@butenkoms.space>" 35__email__ = "gtalk@butenkoms.space" 36# __status__ = "Prototype" 37__status__ = "Development" 38# __status__ = "Production" 39 40 41from enum import IntEnum 42from multiprocessing.shared_memory import SharedMemory as MultiprocessingSharedMemory 43from array import array 44from inspect import isclass, ismodule 45import pickle 46import ctypes 47from contextlib import contextmanager 48 49import numpy as np 50 51from cengal.introspection.inspect import is_callable, is_descriptor, is_async 52from cengal.math.numbers import RationalNumber 53from cengal.hardware.memory.barriers import full_memory_barrier, mm_pause 54from cengal.time_management.cpu_clock import cpu_clock 55from cengal.time_management.high_precision_sync_sleep import hps_sleep 56from cengal.introspection.inspect import pdi 57 58# from .compilable import write_uint64 as write_uint64_c, read_uint64 as read_uint64_c, write_int64, read_int64, write_double, read_double, zero_memory 59from .compilable import write_uint64, read_uint64, read_uint8, write_int64, read_int64, write_double, read_double, zero_memory, list__get_item, list__set_item 60 61from typing import Any, Tuple, Optional, List, Dict, Type, Union, Sequence 62 63 64current_shared_memory_instance: 'SharedMemory' = None 65 66 67# def write_uint64(base_address: int, offset: int, value: int): 68# if current_shared_memory_instance is not None: 69# if 460 <= offset <= 564: 70# print('write_uint64: offset_to_be_monitored: offset: {}, value: {}'.format(offset, value)) 71 72# write_uint64_cython(base_address, offset, value) 73 74 75# def write_uint64(base_address: int, offset: int, value: int): 76# if current_shared_memory_instance is None: 77# return write_uint64_c(base_address, offset, value) 78# else: 79# return current_shared_memory_instance.write_uint64(offset, value) 80 81# def read_uint64(base_address: int, offset: int) -> int: 82# if current_shared_memory_instance is None: 83# return read_uint64_c(base_address, offset) 84# else: 85# return current_shared_memory_instance.read_uint64(offset) 86 87 88class QueueType(IntEnum): 89 fifo = 0 90 lifo = 1 91 92 93class ObjectType(IntEnum): 94 tfree_memory = 0 95 tmessage = 1 96 tnone = 2 97 tbool = 3 98 tint = 4 99 tfloat = 5 100 tcomplex = 6 101 tstr = 7 102 tbytes = 8 103 tbytearray = 9 104 ttuple = 10 105 tlist = 11 106 tdict = 12 107 tset = 13 108 tclass = 14 109 tpickable = 15 110 tinternal_list = 16 111 112 113class SysValuesOffsets(IntEnum): 114 data_start_offset = 0 115 data_size = 1 116 data_end_offset = 2 117 free_memory_search_start = 3 118 first_message_offset = 4 119 last_message_offset = 5 120 creator_in_charge = 6 121 consumer_in_charge = 7 122 creator_wants_to_be_in_charge = 8 123 consumer_wants_to_be_in_charge = 9 124 creator_ready = 10 125 consumer_ready = 11 126 127 128Offset = int 129Size = int 130minimal_memory_block_size = 8 131block_size = minimal_memory_block_size 132bs = block_size 133 134 135class SharedMemoryError(Exception): 136 pass 137 138 139class FreeMemoryChunkNotFoundError(SharedMemoryError): 140 pass 141 142 143class ObjBufferIsSmallerThanRequestedNumpyArrayError(SharedMemoryError): 144 pass 145 146 147class WrongObjectTypeError(SharedMemoryError): 148 pass 149 150 151class NoMessagesInQueueError(SharedMemoryError): 152 pass 153 154 155def nearest_size(size: Size) -> Size: 156 return ((size // bs) * bs + bs) if size % bs else size 157 158 159nsize = nearest_size 160 161 162class BaseIObject: 163 pass 164 165 166# TODO: add next fields: obj_id (simple int index; need to identify object in shared memory); ref_count (simple int counter; need to count references to object. Howerver this field can be moved to shared memory dict with all objects properties like ref_count, etc.) 167class BaseObjOffsets(IntEnum): 168 obj_type = 0 169 obj_size = 1 170 171 172class TBase: 173 def map_to_shared_memory(self, shared_memory: 'SharedMemory', obj: Any) -> Tuple[Any, Offset, Size]: 174 raise NotImplementedError 175 176 def init_from_shared_memory(self, shared_memory: 'SharedMemory', offset: Offset) -> Any: 177 raise NotImplementedError 178 179 def destroy(self, shared_memory: 'SharedMemory', offset: Offset): 180 raise NotImplementedError 181 182 def buffer(self, shared_memory: 'SharedMemory', offset: Offset) -> memoryview: 183 raise NotImplementedError 184 185 def buffer_2(self, shared_memory: 'SharedMemory', offset: Offset) -> Tuple[int, int]: 186 raise NotImplementedError 187 188 189# ====================================================================================================================== 190# === None ===================================================================================================== 191 192 193class TNone: 194 def map_to_shared_memory(self, shared_memory: 'SharedMemory', obj: None) -> Tuple[None, Offset, Size]: 195 offset, real_size = shared_memory.malloc(ObjectType.tnone, 0) 196 return obj, offset, real_size 197 198 def init_from_shared_memory(self, shared_memory: 'SharedMemory', offset: Offset) -> None: 199 if ObjectType.tnone != read_uint64(shared_memory.base_address, offset): 200 raise WrongObjectTypeError 201 202 return None 203 204 def destroy(self, shared_memory: 'SharedMemory', offset: Offset): 205 shared_memory.free(offset) 206 207 208# ====================================================================================================================== 209# === Int ===================================================================================================== 210 211 212class IntOffsets(IntEnum): 213 data = 0 214 215 216class TInt: 217 def map_to_shared_memory(self, shared_memory: 'SharedMemory', obj: int) -> Tuple[int, Offset, Size]: 218 offset, real_size = shared_memory.malloc(ObjectType.tint, bs * len(IntOffsets)) 219 write_int64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * IntOffsets.data, obj) 220 return obj, offset, real_size 221 222 def init_from_shared_memory(self, shared_memory: 'SharedMemory', offset: Offset) -> int: 223 if ObjectType.tint != read_uint64(shared_memory.base_address, offset + bs * BaseObjOffsets.obj_type): 224 raise WrongObjectTypeError 225 226 return read_int64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * IntOffsets.data) 227 228 def destroy(self, shared_memory: 'SharedMemory', offset: Offset): 229 shared_memory.free(offset) 230 231 232# ====================================================================================================================== 233# === Bool ===================================================================================================== 234 235 236class BoolOffsets(IntEnum): 237 data = 0 238 239 240class TBool: 241 def map_to_shared_memory(self, shared_memory: 'SharedMemory', obj: bool) -> Tuple[bool, Offset, Size]: 242 offset, real_size = shared_memory.malloc(ObjectType.tbool, bs * len(BoolOffsets)) 243 write_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * BoolOffsets.data, int(obj)) 244 return obj, offset, real_size 245 246 def init_from_shared_memory(self, shared_memory: 'SharedMemory', offset: Offset) -> bool: 247 if ObjectType.tbool != read_uint64(shared_memory.base_address, offset + bs * BaseObjOffsets.obj_type): 248 raise WrongObjectTypeError 249 250 return bool(read_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * BoolOffsets.data)) 251 252 def destroy(self, shared_memory: 'SharedMemory', offset: Offset): 253 shared_memory.free(offset) 254 255 256# ====================================================================================================================== 257# === Float ===================================================================================================== 258 259 260class FloatOffsets(IntEnum): 261 data = 0 262 263 264class TFloat: 265 def map_to_shared_memory(self, shared_memory: 'SharedMemory', obj: float) -> Tuple[float, Offset, Size]: 266 offset, real_size = shared_memory.malloc(ObjectType.tfloat, bs * len(FloatOffsets)) 267 write_double(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * FloatOffsets.data, obj) 268 return obj, offset, real_size 269 270 def init_from_shared_memory(self, shared_memory: 'SharedMemory', offset: Offset) -> float: 271 if ObjectType.tfloat != read_uint64(shared_memory.base_address, offset): 272 raise WrongObjectTypeError 273 274 return read_double(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * FloatOffsets.data) 275 276 def destroy(self, shared_memory: 'SharedMemory', offset: Offset): 277 shared_memory.free(offset) 278 279 280# ====================================================================================================================== 281# === Bytes ===================================================================================================== 282 283 284class BytesOffsets(IntEnum): 285 data_size = 0 286 data = 1 287 288 289class TBytes: 290 def map_to_shared_memory(self, shared_memory: 'SharedMemory', obj: bytes) -> Tuple[bytes, Offset, Size]: 291 data_size = len(obj) 292 # offset, real_size = shared_memory.malloc(ObjectType.tbytes, bs * len(BytesOffsets) + bs * data_size) 293 offset, real_size = shared_memory.malloc(ObjectType.tbytes, bs * len(BytesOffsets) + data_size) 294 write_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * BytesOffsets.data_size, data_size) 295 data_offset = offset + bs * len(BaseObjOffsets) + bs * BytesOffsets.data 296 shared_memory._shared_memory.buf[data_offset:data_offset + data_size] = obj 297 return obj, offset, real_size 298 299 def init_from_shared_memory(self, shared_memory: 'SharedMemory', offset: Offset) -> bytes: 300 if ObjectType.tbytes != read_uint64(shared_memory.base_address, offset + bs * BaseObjOffsets.obj_type): 301 raise WrongObjectTypeError 302 303 data_size = read_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * BytesOffsets.data_size) 304 if data_size: 305 data_offset = offset + bs * len(BaseObjOffsets) + bs * BytesOffsets.data 306 obj = bytes(shared_memory._shared_memory.buf[data_offset:data_offset + data_size]) 307 return obj 308 else: 309 return bytes() 310 311 def destroy(self, shared_memory: 'SharedMemory', offset: Offset): 312 shared_memory.free(offset) 313 314 def buffer(self, shared_memory: 'SharedMemory', offset: Offset) -> memoryview: 315 if ObjectType.tbytes != read_uint64(shared_memory.base_address, offset + bs * BaseObjOffsets.obj_type): 316 raise WrongObjectTypeError 317 318 data_size = read_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * BytesOffsets.data_size) 319 data_offset = offset + bs * len(BaseObjOffsets) + bs * BytesOffsets.data 320 return shared_memory._shared_memory.buf[data_offset:data_offset + data_size] 321 322 def buffer_2(self, shared_memory: 'SharedMemory', offset: Offset) -> Tuple[int, int]: 323 if ObjectType.tbytes != read_uint64(shared_memory.base_address, offset + bs * BaseObjOffsets.obj_type): 324 raise WrongObjectTypeError 325 326 data_size = read_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * BytesOffsets.data_size) 327 data_offset = offset + bs * len(BaseObjOffsets) + bs * BytesOffsets.data 328 return data_offset, data_size 329 330 331# class TBytes: 332# def map_to_shared_memory(self, shared_memory: 'SharedMemory', obj: bytes) -> Tuple[bytes, Offset, Size]: 333# data_size = len(obj) 334# if 0 == data_size: 335# allocated_data_size = 1 336# else: 337# allocated_data_size = data_size 338 339# # offset, real_size = shared_memory.malloc(ObjectType.tbytes, bs * (len(BytesOffsets) - 1) + bs * allocated_data_size) 340# offset, real_size = shared_memory.malloc(ObjectType.tbytes, bs * (len(BytesOffsets) - 1) + allocated_data_size) 341# shared_memory.print_mem(offset, 100, f'TBytes.map_to_shared_memory 0: offset: {offset}, real_size: {real_size}') 342# write_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * BytesOffsets.data_size, data_size) 343# shared_memory.print_mem(offset, 100, f'TBytes.map_to_shared_memory 1: offset: {offset}, real_size: {real_size}') 344# data_offset = offset + bs * len(BaseObjOffsets) + bs * BytesOffsets.data 345# if data_size: 346# try: 347# shared_memory._shared_memory.buf[data_offset:data_offset + data_size] = obj 348# except ValueError: 349# print(len(shared_memory._shared_memory.buf[data_offset:data_offset + data_size]), shared_memory._shared_memory.buf[data_offset:data_offset + data_size]) 350# print(len(obj), obj) 351# raise 352 353# shared_memory.print_mem(offset, 100, f'TBytes.map_to_shared_memory 2: offset: {offset}, real_size: {real_size}, data_size: {data_size}, data_offset: {data_offset}') 354 355# return obj, offset, real_size 356 357# def init_from_shared_memory(self, shared_memory: 'SharedMemory', offset: Offset) -> bytes: 358# if ObjectType.tbytes != read_uint64(shared_memory.base_address, offset + bs * BaseObjOffsets.obj_type): 359# raise WrongObjectTypeError 360 361# data_size = read_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * BytesOffsets.data_size) 362# data_offset = offset + bs * len(BaseObjOffsets) + bs * BytesOffsets.data 363# shared_memory.print_mem(offset, 100, f'TBytes.init_from_shared_memory 0: offset: {offset}, data_size: {data_size}, data_offset: {data_offset}') 364# if data_size: 365# obj = bytes(shared_memory._shared_memory.buf[data_offset:data_offset + data_size]) 366# else: 367# obj = b'' 368 369# return obj 370 371# def destroy(self, shared_memory: 'SharedMemory', offset: Offset): 372# shared_memory.free(offset) 373 374 375# ====================================================================================================================== 376# === Bytearray ===================================================================================================== 377 378 379class BytearrayOffsets(IntEnum): 380 data_size = 0 381 data = 1 382 383 384class TBytearray: 385 def map_to_shared_memory(self, shared_memory: 'SharedMemory', obj: bytearray) -> Tuple[bytearray, Offset, Size]: 386 data = bytes(obj) 387 data_size = len(data) 388 # offset, real_size = shared_memory.malloc(ObjectType.tbytearray, bs * len(BytearrayOffsets) + bs * data_size) 389 offset, real_size = shared_memory.malloc(ObjectType.tbytearray, bs * len(BytearrayOffsets) + data_size) 390 write_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * BytearrayOffsets.data_size, data_size) 391 data_offset = offset + bs * len(BaseObjOffsets) + bs * BytearrayOffsets.data 392 shared_memory._shared_memory.buf[data_offset:data_offset + data_size] = data 393 return obj, offset, real_size 394 395 def init_from_shared_memory(self, shared_memory: 'SharedMemory', offset: Offset) -> bytearray: 396 if ObjectType.tbytearray != read_uint64(shared_memory.base_address, offset + bs * BaseObjOffsets.obj_type): 397 raise WrongObjectTypeError 398 399 data_size = read_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * BytearrayOffsets.data_size) 400 if data_size: 401 data_offset = offset + bs * len(BaseObjOffsets) + bs * BytearrayOffsets.data 402 data = bytes(shared_memory._shared_memory.buf[data_offset:data_offset + data_size]) 403 return bytearray(data) 404 else: 405 return bytearray(bytes()) 406 407 def destroy(self, shared_memory: 'SharedMemory', offset: Offset): 408 shared_memory.free(offset) 409 410 def buffer(self, shared_memory: 'SharedMemory', offset: Offset) -> memoryview: 411 if ObjectType.tbytearray != read_uint64(shared_memory.base_address, offset + bs * BaseObjOffsets.obj_type): 412 raise WrongObjectTypeError 413 414 data_size = read_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * BytearrayOffsets.data_size) 415 data_offset = offset + bs * len(BaseObjOffsets) + bs * BytearrayOffsets.data 416 return shared_memory._shared_memory.buf[data_offset:data_offset + data_size] 417 418 def buffer_2(self, shared_memory: 'SharedMemory', offset: Offset) -> Tuple[int, int]: 419 if ObjectType.tbytearray != read_uint64(shared_memory.base_address, offset + bs * BaseObjOffsets.obj_type): 420 raise WrongObjectTypeError 421 422 data_size = read_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * BytearrayOffsets.data_size) 423 data_offset = offset + bs * len(BaseObjOffsets) + bs * BytearrayOffsets.data 424 return data_offset, data_size 425 426 427# ====================================================================================================================== 428# === Str ===================================================================================================== 429 430 431class StrOffsets(IntEnum): 432 data_size = 0 433 data = 1 434 435 436class TStr: 437 def map_to_shared_memory(self, shared_memory: 'SharedMemory', obj: str) -> Tuple[str, Offset, Size]: 438 data = str.encode(obj) 439 data_size = len(data) 440 # offset, real_size = shared_memory.malloc(ObjectType.tstr, bs * len(StrOffsets) + bs * data_size) 441 offset, real_size = shared_memory.malloc(ObjectType.tstr, bs * len(StrOffsets) + data_size) 442 write_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * StrOffsets.data_size, data_size) 443 data_offset = offset + bs * len(BaseObjOffsets) + bs * StrOffsets.data 444 shared_memory._shared_memory.buf[data_offset:data_offset + data_size] = data 445 return obj, offset, real_size 446 447 def init_from_shared_memory(self, shared_memory: 'SharedMemory', offset: Offset) -> str: 448 if ObjectType.tstr != read_uint64(shared_memory.base_address, offset + bs * BaseObjOffsets.obj_type): 449 raise WrongObjectTypeError 450 451 data_size = read_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * StrOffsets.data_size) 452 if data_size: 453 data_offset = offset + bs * len(BaseObjOffsets) + bs * StrOffsets.data 454 data = bytes(shared_memory._shared_memory.buf[data_offset:data_offset + data_size]) 455 return data.decode() 456 else: 457 return str() 458 459 def destroy(self, shared_memory: 'SharedMemory', offset: Offset): 460 shared_memory.free(offset) 461 462 def buffer(self, shared_memory: 'SharedMemory', offset: Offset) -> memoryview: 463 if ObjectType.tstr != read_uint64(shared_memory.base_address, offset + bs * BaseObjOffsets.obj_type): 464 raise WrongObjectTypeError 465 466 data_size = read_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * StrOffsets.data_size) 467 data_offset = offset + bs * len(BaseObjOffsets) + bs * StrOffsets.data 468 return shared_memory._shared_memory.buf[data_offset:data_offset + data_size] 469 470 def buffer_2(self, shared_memory: 'SharedMemory', offset: Offset) -> Tuple[int, int]: 471 if ObjectType.tstr != read_uint64(shared_memory.base_address, offset + bs * BaseObjOffsets.obj_type): 472 raise WrongObjectTypeError 473 474 data_size = read_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * StrOffsets.data_size) 475 data_offset = offset + bs * len(BaseObjOffsets) + bs * StrOffsets.data 476 return data_offset, data_size 477 478 479# ====================================================================================================================== 480# === ListTrue ===================================================================================================== 481 482 483class InternalListTrueOffsets(IntEnum): 484 capacity = 0 485 size = 1 486 487 488def malloc_tinternal_list_true(shared_memory: 'SharedMemory', size: Size, capacity: Size = None) -> Tuple[Offset, Size]: 489 capacity = (size << 1 if size else 16) if capacity is None else capacity 490 datas_sys_part_size = 8 * len(InternalListTrueOffsets) 491 offset, real_size = shared_memory.malloc(ObjectType.tinternal_list, datas_sys_part_size + 8 * capacity) 492 data_offset = offset + datas_sys_part_size 493 write_uint64(shared_memory.base_address, data_offset + 8 * InternalListTrueOffsets.capacity, capacity) 494 write_uint64(shared_memory.base_address, data_offset + 8 * InternalListTrueOffsets.size, size) 495 return offset, real_size 496 497 498def realloc_tinternal_list_true(shared_memory: 'SharedMemory', offset: Offset, desired_size: int = None, new_capacity: int = None, loop_allowed: bool = True, zero_mem: bool = True) -> Tuple[Offset, Size]: 499 datas_sys_part_size = 8 * len(InternalListTrueOffsets) 500 data_offset = offset + datas_sys_part_size 501 capacity = read_uint64(shared_memory.base_address, data_offset + 8 * InternalListTrueOffsets.capacity) 502 size = read_uint64(shared_memory.base_address, data_offset + 8 * InternalListTrueOffsets.size) 503 new_list_capacity = capacity << 1 if new_capacity is None else new_capacity 504 if new_capacity is None: 505 if desired_size is None: 506 new_list_capacity = capacity << 1 if capacity else 16 507 else: 508 new_list_capacity = desired_size << 1 if desired_size else 16 509 else: 510 new_list_capacity = new_capacity 511 512 if new_list_capacity < size: 513 new_list_capacity = size 514 515 new_offset, new_real_size = shared_memory.realloc(offset, datas_sys_part_size + 8 * new_list_capacity, loop_allowed, zero_mem) 516 data_offset = new_offset + datas_sys_part_size 517 write_uint64(shared_memory.base_address, data_offset + 8 * InternalListTrueOffsets.capacity, new_list_capacity) 518 return new_offset, new_real_size 519 520 521class IListTrue(BaseIObject, list): 522 def __init__(self, shared_memory: 'SharedMemory', offset: Offset = None, obj: List = None) -> None: 523 self._shared_memory = shared_memory 524 self._base_address = shared_memory.base_address 525 if offset is None: 526 offset, real_size = shared_memory.malloc(ObjectType.tlist, 8) 527 self._offset = offset 528 self._offset__data = offset + 8 * len(BaseObjOffsets) 529 self._offset__pointer_to_internal_list = self._offset__data 530 531 if obj is None: 532 obj = list() 533 534 data_len = len(obj) 535 capacity_len = data_len << 1 if data_len else 16 536 internal_list_offset, data_tuple_real_size = malloc_tinternal_list(shared_memory, data_len, capacity_len) 537 self._pointer_to_internal_list = internal_list_offset 538 for i, item in enumerate(obj): 539 item_mapped_obj, item_offset, item_size = shared_memory.put_obj(item) 540 write_uint64(self._base_address, self._item_offset(i), item_offset) 541 else: 542 self._offset = offset 543 self._offset__data = offset + 8 * len(BaseObjOffsets) 544 self._offset__pointer_to_internal_list = self._offset__data 545 546 def raw_to_bytes(self, bytes_num: int) -> bytes: 547 start_index = self._pointer_to_internal_list 548 return self._shared_memory.read_mem(start_index, bytes_num) 549 # return bytes(self._shared_memory._shared_memory.buf[start_index : start_index + bytes_num]) 550 551 @property 552 def _obj_size(self): 553 return read_uint64(self._base_address, self._offset + 8 * BaseObjOffsets.obj_size) 554 555 @property 556 def _pointer_to_internal_list(self): 557 return read_uint64(self._base_address, self._offset__pointer_to_internal_list) 558 559 @_pointer_to_internal_list.setter 560 def _pointer_to_internal_list(self, value: Offset): 561 write_uint64(self._base_address, self._offset__pointer_to_internal_list, value) 562 563 @property 564 def _list_len(self): 565 return read_uint64(self._base_address, self._pointer_to_internal_list + 8 * len(BaseObjOffsets) + 8 * InternalListTrueOffsets.size) 566 567 @_list_len.setter 568 def _list_len(self, value: int): 569 write_uint64(self._base_address, self._pointer_to_internal_list + 8 * len(BaseObjOffsets) + 8 * InternalListTrueOffsets.size, value) 570 571 @property 572 def _list_capacity(self): 573 return read_uint64(self._base_address, self._pointer_to_internal_list + 8 * len(BaseObjOffsets) + 8 * InternalListTrueOffsets.capacity) 574 575 def _item_offset(self, key: int) -> Offset: 576 return self._pointer_to_internal_list + 8 * len(BaseObjOffsets) + 8 * len(InternalListTrueOffsets) + key * 8 577 578 def __len__(self) -> int: 579 return self._list_len 580 581 def get_children_offsets(self) -> List[Offset]: 582 return [read_uint64(self._base_address, self._item_offset(i)) for i in range(self._list_len)] 583 584 def __getitem__(self, key: Union[int, slice]) -> Union[Any, List]: 585 if isinstance(key, int): 586 if key < 0: 587 key += len(self) 588 if key < 0 or key >= len(self): 589 raise IndexError 590 591 item_offset = read_uint64(self._base_address, self._item_offset(key)) 592 return self._shared_memory.get_obj(item_offset) 593 elif isinstance(key, slice): 594 if key.step is not None: 595 raise NotImplementedError 596 597 if key.start is None: 598 start = 0 599 elif key.start < 0: 600 start = key.start + len(self) 601 else: 602 start = key.start 603 604 if key.stop is None: 605 stop = len(self) 606 elif key.stop < 0: 607 stop = key.stop + len(self) 608 else: 609 stop = key.stop 610 611 if start < 0 or start >= len(self) or stop < 0 or stop > len(self) or start >= stop: 612 raise IndexError 613 614 result_list = list() 615 for i in range(start, stop): 616 item_offset = read_uint64(self._base_address, self._item_offset(i)) 617 result_list.append(self._shared_memory.get_obj(item_offset)) 618 return result_list 619 else: 620 raise TypeError 621 622 def __setitem__(self, key: Union[int, slice], value: Union[Any, Sequence]) -> Any: 623 if isinstance(key, int): 624 if key < 0: 625 key += len(self) 626 if key < 0 or key >= len(self): 627 raise IndexError 628 629 item_mapped_obj, item_offset, item_size = self._shared_memory.put_obj(value) 630 write_uint64(self._base_address, self._item_offset(key), item_offset) 631 elif isinstance(key, slice): 632 if key.step is not None: 633 raise NotImplementedError 634 635 if key.start is None: 636 start = 0 637 elif key.start < 0: 638 start = key.start + len(self) 639 else: 640 start = key.start 641 642 if key.stop is None: 643 stop = len(self) 644 elif key.stop < 0: 645 stop = key.stop + len(self) 646 else: 647 stop = key.stop 648 649 if start < 0 or start >= len(self) or stop < 0 or stop > len(self) or start >= stop: 650 raise IndexError 651 652 for i in range(start, stop): 653 item_mapped_obj, item_offset, item_size = self._shared_memory.put_obj(value[i - start]) 654 write_uint64(self._base_address, self._item_offset(i), item_offset) 655 else: 656 raise TypeError 657 658 def __delitem__(self, key: Union[int, slice]) -> None: 659 if isinstance(key, int): 660 if key < 0: 661 key += len(self) 662 if key < 0 or key >= len(self): 663 raise IndexError 664 665 for i in range(key + 1, len(self)): 666 item_offset = read_uint64(self._base_address, self._item_offset(i)) 667 self._shared_memory.free(item_offset) 668 write_uint64(self._base_address, self._item_offset(i - 1), item_offset) 669 670 self._list_len -= 1 671 elif isinstance(key, slice): 672 if key.step is not None: 673 raise NotImplementedError 674 675 if key.start is None: 676 start = 0 677 elif key.start < 0: 678 start = key.start + len(self) 679 else: 680 start = key.start 681 682 if key.stop is None: 683 stop = len(self) 684 elif key.stop < 0: 685 stop = key.stop + len(self) 686 else: 687 stop = key.stop 688 689 if start < 0 or start >= len(self) or stop < 0 or stop > len(self) or start >= stop: 690 raise IndexError 691 692 for i in range(start, stop): 693 item_offset = read_uint64(self._base_address, self._item_offset(i)) 694 self._shared_memory.free(item_offset) 695 696 del_items_num = stop - start 697 698 for i in range(stop, len(self)): 699 item_offset = read_uint64(self._base_address, self._item_offset(i)) 700 write_uint64(self._base_address, self._item_offset(i - del_items_num), item_offset) 701 702 self._list_len -= del_items_num 703 else: 704 raise TypeError 705 706 def append(self, item: Any) -> None: 707 if self._list_len > self._list_capacity: 708 self._pointer_to_internal_list, result_size = realloc_tinternal_list(self._shared_memory, self._pointer_to_internal_list) 709 710 item_mapped_obj, item_offset, item_size = self._shared_memory.put_obj(item) 711 write_uint64(self._base_address, self._item_offset(self._list_len), item_offset) 712 self._list_len += 1 713 714 def extend(self, items: Sequence) -> None: 715 items_num = len(items) 716 if self._list_len + items_num > self._list_capacity: 717 self._pointer_to_internal_list, result_size = realloc_tinternal_list(self._shared_memory, self._pointer_to_internal_list, self._list_len + items_num) 718 719 for i, item in enumerate(items): 720 item_mapped_obj, item_offset, item_size = self._shared_memory.put_obj(item) 721 write_uint64(self._base_address, self._item_offset(self._list_len + i), item_offset) 722 723 self._list_len += items_num 724 725 def insert(self, index: int, item: Any) -> None: 726 if index < 0: 727 index += len(self) 728 if index < 0 or index > len(self): 729 raise IndexError 730 731 if self._list_len > self._list_capacity: 732 self._pointer_to_internal_list, result_size = realloc_tinternal_list(self._shared_memory, self._pointer_to_internal_list) 733 734 for i in range(self._list_len, index, -1): 735 item_offset = read_uint64(self._base_address, self._item_offset(i - 1)) 736 write_uint64(self._base_address, self._item_offset(i), item_offset) 737 738 item_mapped_obj, item_offset, item_size = self._shared_memory.put_obj(item) 739 write_uint64(self._base_address, self._item_offset(index), item_offset) 740 self._list_len += 1 741 742 def pop(self, index: int = -1) -> Any: 743 if index < 0: 744 index += len(self) 745 if index < 0 or index >= len(self): 746 raise IndexError 747 748 item_offset = read_uint64(self._base_address, self._item_offset(index)) 749 result = self._shared_memory.get_obj(item_offset) 750 751 for i in range(index + 1, len(self)): 752 item_offset = read_uint64(self._base_address, self._item_offset(i)) 753 write_uint64(self._base_address, self._item_offset(i - 1), item_offset) 754 755 self._list_len -= 1 756 return result 757 758 def remove(self, item: Any) -> None: 759 for i in range(len(self)): 760 item_offset = read_uint64(self._base_address, self._item_offset(i)) 761 if item_offset == item._offset: 762 for j in range(i + 1, len(self)): 763 item_offset = read_uint64(self._base_address, self._item_offset(j)) 764 write_uint64(self._base_address, self._item_offset(j - 1), item_offset) 765 766 self._list_len -= 1 767 return 768 769 raise ValueError 770 771 def clear(self) -> None: 772 for i in range(len(self)): 773 item_offset = read_uint64(self._base_address, self._item_offset(i)) 774 self._shared_memory.free(item_offset) 775 776 self._list_len = 0 777 778 def __iter__(self): 779 return IListIterator(self) 780 781 def __reversed__(self): 782 return IListReversedIterator(self) 783 784 def __contains__(self, item: Any) -> bool: 785 for i in range(len(self)): 786 item_offset = read_uint64(self._base_address, self._item_offset(i)) 787 if item_offset == item._offset: 788 return True 789 790 return False 791 792 def index(self, item: Any, start: int = 0, stop: int = None) -> int: 793 if stop is None: 794 stop = len(self) 795 796 for i in range(start, stop): 797 item_offset = read_uint64(self._base_address, self._item_offset(i)) 798 if item_offset == item._offset: 799 return i 800 801 raise ValueError 802 803 def count(self, item: Any) -> int: 804 result = 0 805 for i in range(len(self)): 806 item_offset = read_uint64(self._base_address, self._item_offset(i)) 807 if item_offset == item._offset: 808 result += 1 809 810 return result 811 812 def reverse(self) -> None: 813 for i in range(len(self) // 2): 814 item_offset = read_uint64(self._base_address, self._item_offset(i)) 815 write_uint64(self._base_address, self._item_offset(i), read_uint64(self._base_address, self._item_offset(len(self) - i - 1))) 816 write_uint64(self._base_address, self._item_offset(len(self) - i - 1), item_offset) 817 818 def sort(self, key: Any = None, reverse: bool = False) -> None: 819 raise NotImplementedError 820 821 def copy(self) -> 'IList': 822 result = IList(self._shared_memory) 823 result.extend(self) 824 return result 825 826 def __add__(self, other: Sequence) -> 'IList': 827 result = IList(self._shared_memory) 828 result.extend(self) 829 result.extend(other) 830 return result 831 832 def __iadd__(self, other: Sequence) -> 'IList': 833 self.extend(other) 834 return self 835 836 def __mul__(self, other: int) -> 'IList': 837 result = IList(self._shared_memory) 838 for i in range(other): 839 result.extend(self) 840 841 return result 842 843 def __imul__(self, other: int) -> 'IList': 844 my_copy: IList = self.copy() 845 for i in range(other): 846 self.extend(my_copy) 847 848 return self 849 850 def __rmul__(self, other: int) -> 'IList': 851 return self.__mul__(other) 852 853 def __eq__(self, other: Sequence) -> bool: 854 if len(self) != len(other): 855 return False 856 857 for i in range(len(self)): 858 if self[i] != other[i]: 859 return False 860 861 return True 862 863 def __ne__(self, other: Sequence) -> bool: 864 return not self.__eq__(other) 865 866 def __lt__(self, other: Sequence) -> bool: 867 for i in range(len(self)): 868 if self[i] >= other[i]: 869 return False 870 871 return True 872 873 def __le__(self, other: Sequence) -> bool: 874 for i in range(len(self)): 875 if self[i] > other[i]: 876 return False 877 878 return True 879 880 def __gt__(self, other: Sequence) -> bool: 881 for i in range(len(self)): 882 if self[i] <= other[i]: 883 return False 884 885 return True 886 887 def __ge__(self, other: Sequence) -> bool: 888 for i in range(len(self)): 889 if self[i] < other[i]: 890 return False 891 892 return True 893 894 def __repr__(self) -> str: 895 return f'IList({list(self)})' 896 897 def __str__(self) -> str: 898 return f'IList({list(self)})' 899 900 def __hash__(self) -> int: 901 return hash(tuple(self)) 902 903 def __sizeof__(self) -> int: 904 return read_uint64(self._base_address, self._offset + 8 * BaseObjOffsets.obj_size) + read_uint64(self._base_address, self._pointer_to_internal_list, 8 * BaseObjOffsets.obj_size) 905 906 def export(self) -> list: 907 return list(self) 908 909 # def __del__(self) -> None: 910 # self._shared_memory.free(self._pointer_to_internal_list) 911 # self._shared_memory.free(self._offset) 912 913 914# ====================================================================================================================== 915# === InternalList ===================================================================================================== 916 917 918class InternalListOffsets(IntEnum): 919 capacity = 0 920 size = 1 921 922 923class InternalListFieldOffsets(IntEnum): 924 field_type = 0 925 offset_or_data = 1 926 927 928def malloc_tinternal_list(shared_memory: 'SharedMemory', size: Size, capacity: Size = None) -> Tuple[Offset, Size]: 929 capacity = (size << 1 if size else 16) if capacity is None else capacity 930 offset, real_size = shared_memory.malloc(ObjectType.tinternal_list, bs * len(BaseObjOffsets) + bs * len(InternalListOffsets) + capacity * bs * len(InternalListFieldOffsets)) 931 sys_data_offset = offset + bs * len(BaseObjOffsets) 932 write_uint64(shared_memory.base_address, sys_data_offset + bs * InternalListOffsets.capacity, capacity) 933 write_uint64(shared_memory.base_address, sys_data_offset + bs * InternalListOffsets.size, size) 934 return offset, real_size 935 936 937def realloc_tinternal_list(shared_memory: 'SharedMemory', offset: Offset, desired_size: int = None, new_capacity: int = None, loop_allowed: bool = True, zero_mem: bool = True) -> Tuple[Offset, Size]: 938 sys_data_offset = offset + bs * len(BaseObjOffsets) 939 capacity = read_uint64(shared_memory.base_address, sys_data_offset + bs * InternalListOffsets.capacity) 940 size = read_uint64(shared_memory.base_address, sys_data_offset + bs * InternalListOffsets.size) 941 new_list_capacity = capacity << 1 if new_capacity is None else new_capacity 942 if new_capacity is None: 943 if desired_size is None: 944 new_list_capacity = capacity << 1 if capacity else 16 945 else: 946 new_list_capacity = desired_size << 1 if desired_size else 16 947 else: 948 new_list_capacity = new_capacity 949 950 if new_list_capacity < size: 951 new_list_capacity = size 952 953 new_offset, new_real_size = shared_memory.realloc( 954 offset, 955 bs * len(BaseObjOffsets) + bs * len(InternalListOffsets) + capacity * bs * len(InternalListFieldOffsets), 956 loop_allowed, 957 zero_mem 958 ) 959 new_sys_data_offset = new_offset + bs * len(BaseObjOffsets) 960 write_uint64(shared_memory.base_address, new_sys_data_offset + bs * InternalListOffsets.capacity, new_list_capacity) 961 return new_offset, new_real_size 962 963 964def uint64_to_bytes(int_data: int) -> bytes: 965 """ 966 For a 64 bit unsigned int in little endian 967 :param int_data: 968 :return: bytes(); len == 8 969 """ 970 from struct import pack 971 result = pack('<B', int_data) 972 return result 973 974 975def uint8_to_bytes(int_data: int) -> bytes: 976 """ 977 For a 64 bit unsigned int in little endian 978 :param int_data: 979 :return: bytes(); len == 8 980 """ 981 from struct import pack 982 result = pack('<Q', int_data) 983 return result 984 985 986# ====================================================================================================================== 987# === List ===================================================================================================== 988 989 990class ListOffsets(IntEnum): 991 internal_list_offset = 0 992 993 994class IList(BaseIObject, list): 995 def __init__(self, shared_memory: 'SharedMemory', offset: Offset = None, obj: List = None) -> None: 996 self._shared_memory = shared_memory 997 self._base_address = shared_memory.base_address 998 if offset is None: 999 offset, real_size = shared_memory.malloc(ObjectType.tlist, bs * len(ListOffsets)) 1000 self._offset = offset 1001 self._offset__data = offset + bs * len(BaseObjOffsets) 1002 self._offset__pointer_to_internal_list = self._offset__data + bs * ListOffsets.internal_list_offset 1003 1004 if obj is None: 1005 obj = list() 1006 1007 data_len = len(obj) 1008 internal_list_offset, data_tuple_real_size = malloc_tinternal_list(shared_memory, data_len) 1009 self._pointer_to_internal_list = internal_list_offset 1010 for i, item in enumerate(obj): 1011 # print(self.get_children_offsets()) 1012 # # print(self.raw_to_list(slice(0, None))) 1013 # print(self.raw_to_bytes(200)) 1014 self._write_item(i, item) 1015 # print(self.get_children_offsets()) 1016 # # print(self.raw_to_list(slice(0, None))) 1017 # print(self.raw_to_bytes(200)) 1018 1019 # print(self.get_children_offsets()) 1020 # # print(self.raw_to_list(slice(0, None))) 1021 # print(self.raw_to_bytes(200)) 1022 # print('=======================') 1023 else: 1024 self._offset = offset 1025 self._offset__data = offset + bs * len(BaseObjOffsets) 1026 self._offset__pointer_to_internal_list = self._offset__data 1027 1028 def raw_to_list(self, key) -> List[bytes]: 1029 if isinstance(key, int): 1030 if key < 0: 1031 key += len(self) 1032 if key < 0 or key >= len(self): 1033 raise IndexError 1034 1035 item_offset = self._read_item_offset_or_data(key) 1036 return [uint64_to_bytes(item_offset)] 1037 elif isinstance(key, slice): 1038 if key.step is not None: 1039 raise NotImplementedError 1040 1041 if key.start is None: 1042 start = 0 1043 elif key.start < 0: 1044 start = key.start + len(self) 1045 else: 1046 start = key.start 1047 1048 if key.stop is None: 1049 stop = len(self) 1050 elif key.stop < 0: 1051 stop = key.stop + len(self) 1052 else: 1053 stop = key.stop 1054 1055 if start < 0 or start >= len(self) or stop < 0 or stop > len(self) or start >= stop: 1056 raise IndexError 1057 1058 result_list = list() 1059 for i in range(start, stop): 1060 item_offset = self._read_item_offset_or_data(i) 1061 result_list.append(uint64_to_bytes(item_offset)) 1062 1063 return result_list 1064 1065 def raw_to_bytes(self, bytes_num: int) -> bytes: 1066 start_index = self._pointer_to_internal_list 1067 return self._shared_memory.read_mem(start_index, bytes_num) 1068 # return bytes(self._shared_memory._shared_memory.buf[start_index : start_index + bytes_num]) 1069 1070 @property 1071 def _obj_size(self): 1072 return read_uint64(self._base_address, self._offset + bs * BaseObjOffsets.obj_size) 1073 1074 @property 1075 def _pointer_to_internal_list(self): 1076 return read_uint64(self._base_address, self._offset__pointer_to_internal_list) 1077 1078 @_pointer_to_internal_list.setter 1079 def _pointer_to_internal_list(self, value: Offset): 1080 write_uint64(self._base_address, self._offset__pointer_to_internal_list, value) 1081 1082 @property 1083 def _list_len(self): 1084 return read_uint64(self._base_address, self._pointer_to_internal_list + bs * len(BaseObjOffsets) + bs * InternalListOffsets.size) 1085 1086 @_list_len.setter 1087 def _list_len(self, value: int): 1088 write_uint64(self._base_address, self._pointer_to_internal_list + bs * len(BaseObjOffsets) + bs * InternalListOffsets.size, value) 1089 1090 @property 1091 def _list_capacity(self): 1092 return read_uint64(self._base_address, self._pointer_to_internal_list + bs * len(BaseObjOffsets) + bs * InternalListOffsets.capacity) 1093 1094 def _item_offset(self, key: int) -> Offset: 1095 return self._pointer_to_internal_list + bs * len(BaseObjOffsets) + bs * len(InternalListOffsets) + key * bs * len(InternalListFieldOffsets) 1096 1097 def _item_type_offset(self, key: int) -> Offset: 1098 # from os import getpid 1099 result = self._pointer_to_internal_list + bs * len(BaseObjOffsets) + bs * len(InternalListOffsets) + key * bs * len(InternalListFieldOffsets) + bs * InternalListFieldOffsets.field_type 1100 # add_0 = bs * len(BaseObjOffsets) 1101 # add_1 = bs * len(InternalListOffsets) 1102 # add_2 = key * bs * len(InternalListFieldOffsets) 1103 # add_3 = bs * InternalListFieldOffsets.field_type 1104 # print(f'PID: {getpid()}. [{add_0},{add_1},{add_2},{add_3}],{add_0 + add_1 + add_2 + add_3},{self._pointer_to_internal_list}: item_type_offset: {key}:{result}') 1105 return result 1106 1107 def _item_value_offset(self, key: int) -> Offset: 1108 # from os import getpid 1109 result = self._pointer_to_internal_list + bs * len(BaseObjOffsets) + bs * len(InternalListOffsets) + key * bs * len(InternalListFieldOffsets) + bs * InternalListFieldOffsets.offset_or_data 1110 # print(f'PID: {getpid()}. {bs * len(BaseObjOffsets) + bs * len(InternalListOffsets) + key * bs * len(InternalListFieldOffsets) + bs * InternalListFieldOffsets.offset_or_data},{self._pointer_to_internal_list}: item_value_offset: {key}:{result}') 1111 return result 1112 1113 def _read_item_type(self, key: int) -> int: 1114 return read_uint64(self._base_address, self._item_type_offset(key)) 1115 1116 def _write_item_type(self, key: int, item_type: int) -> None: 1117 write_uint64(self._base_address, self._item_type_offset(key), item_type) 1118 1119 def _read_item_offset_or_data(self, key: int) -> Union[Offset, int]: 1120 return read_uint64(self._base_address, self._item_value_offset(key)) 1121 1122 def _write_item_offset_or_data(self, key: int, offset_or_data: Union[Offset, int]) -> None: 1123 write_uint64(self._base_address, self._item_value_offset(key), offset_or_data) 1124 1125 def _determine_obj_type(self, obj: Any) -> int: 1126 if isinstance(obj, int): 1127 return 1 1128 elif isinstance(obj, float): 1129 return 2 1130 elif isinstance(obj, bool): 1131 return 3 1132 else: 1133 return 0 1134 1135 def _determine_obj_offset(self, obj: Any) -> Optional[Offset]: 1136 if isinstance(obj, BaseIObject): 1137 return obj._offset 1138 else: 1139 return None 1140 1141 def _compare_item_to_obj_fast(self, key: int, obj: Any, obj_type: int, obj_offset) -> bool: 1142 result: bool = False 1143 item_type = self._read_item_type(key) 1144 if item_type == obj_type: 1145 if item_type == 0: 1146 if obj_offset is None: 1147 if self._read_item_value(key, item_type) == obj: 1148 result = True 1149 else: 1150 if self._read_item_offset_or_data(key) == obj_offset: 1151 result = True 1152 elif item_type == 1: 1153 if self._read_item_offset_or_data(key) == obj: 1154 result = True 1155 elif item_type == 2: 1156 if self._read_item_offset_or_data(key) == obj: 1157 result = True 1158 elif item_type == 3: 1159 if self._read_item_offset_or_data(key) == obj: 1160 result = True 1161 else: 1162 raise ValueError 1163 1164 return result 1165 1166 def _compare_item_to_obj(self, key: int, obj: Any) -> bool: 1167 obj_type = self._determine_obj_type(obj) 1168 obj_offset = self._determine_obj_offset(obj) 1169 return self._compare_item_to_obj_fast(key, obj, obj_type, obj_offset) 1170 1171 def _read_item_value(self, key: int, item_type: int) -> Any: 1172 if item_type == 0: 1173 item_offset = read_uint64(self._base_address, self._item_value_offset(key)) 1174 return self._shared_memory.get_obj(item_offset) 1175 elif item_type == 1: 1176 return read_int64(self._base_address, self._item_value_offset(key)) 1177 elif item_type == 2: 1178 return read_double(self._base_address, self._item_value_offset(key)) 1179 elif item_type == 3: 1180 return bool(read_uint64(self._base_address, self._item_value_offset(key))) 1181 else: 1182 raise ValueError 1183 1184 def _write_item_value(self, key: int, item_type: int, value: Any) -> None: 1185 if item_type == 0: 1186 item_mapped_obj, item_offset, item_size = self._shared_memory.put_obj(value) 1187 write_uint64(self._base_address, self._item_value_offset(key), item_offset) 1188 elif item_type == 1: 1189 write_int64(self._base_address, self._item_value_offset(key), value) 1190 elif item_type == 2: 1191 write_double(self._base_address, self._item_value_offset(key), value) 1192 elif item_type == 3: 1193 write_uint64(self._base_address, self._item_value_offset(key), int(value)) 1194 else: 1195 raise ValueError 1196 1197 def _free_item_value(self, key: int, item_type: int) -> None: 1198 if item_type == 0: 1199 item_offset = read_uint64(self._base_address, self._item_value_offset(key)) 1200 self._shared_memory.free(item_offset) 1201 elif item_type == 1: 1202 pass 1203 elif item_type == 2: 1204 pass 1205 elif item_type == 3: 1206 pass 1207 else: 1208 raise ValueError 1209 1210 def _read_item_type_and_value(self, key: int) -> Tuple[int, Any]: 1211 item_type = self._read_item_type(key) 1212 return item_type, self._read_item_value(key, item_type) 1213 1214 def _write_item_value_and_get_type(self, key: int, value: Any) -> int: 1215 if isinstance(value, int): 1216 write_uint64(self._base_address, self._item_value_offset(key), value) 1217 return 1 1218 elif isinstance(value, float): 1219 write_double(self._base_address, self._item_value_offset(key), value) 1220 return 2 1221 elif isinstance(value, bool): 1222 write_uint64(self._base_address, self._item_value_offset(key), int(value)) 1223 return 3 1224 else: 1225 item_mapped_obj, item_offset, item_size = self._shared_memory.put_obj(value) 1226 write_uint64(self._base_address, self._item_value_offset(key), item_offset) 1227 return 0 1228 1229 def _free_item_value_and_get_type(self, key: int) -> int: 1230 item_type = self._read_item_type(key) 1231 self._free_item_value(key, item_type) 1232 return item_type 1233 1234 def _read_item(self, key: int) -> Any: 1235 item_type = self._read_item_type(key) 1236 return self._read_item_value(key, item_type) 1237 1238 def _write_item(self, key: int, value: Any) -> None: 1239 item_type = self._write_item_value_and_get_type(key, value) 1240 self._write_item_type(key, item_type) 1241 1242 def _free_item(self, key: int) -> None: 1243 item_type = self._read_item_type(key) 1244 self._free_item_value(key, item_type) 1245 1246 def _move_item(self, src_key: int, dst_key: int) -> None: 1247 self._write_item_type(dst_key, self._read_item_type(src_key)) 1248 self._write_item_offset_or_data(dst_key, self._read_item_offset_or_data(src_key)) 1249 1250 def _swap_items(self, key1: int, key2: int) -> None: 1251 item_type1 = self._read_item_type(key1) 1252 item_offset_or_data1 = self._read_item_offset_or_data(key1) 1253 self._write_item_type(key1, self._read_item_type(key2)) 1254 self._write_item_type(key2, item_type1) 1255 self._write_item_offset_or_data(key1, self._read_item_offset_or_data(key2)) 1256 self._write_item_offset_or_data(key2, item_offset_or_data1) 1257 1258 def __len__(self) -> int: 1259 return self._list_len 1260 1261 def get_children_data_or_offsets(self) -> List[Offset]: 1262 return [self._read_item_offset_or_data(i) for i in range(self._list_len)] 1263 1264 def get_children_offsets(self): 1265 return self.get_children_data_or_offsets() 1266 1267 def __getitem__(self, key: Union[int, slice]) -> Union[Any, List]: 1268 if isinstance(key, int): 1269 return list__get_item(key, self._base_address, self._offset__pointer_to_internal_list, self._shared_memory.get_obj) 1270 1271 # base_address = self._base_address 1272 # offset__pointer_to_internal_list = self._offset__pointer_to_internal_list 1273 # pointer_to_internal_list = read_uint64(base_address, offset__pointer_to_internal_list) 1274 # self_len = read_uint64(base_address, pointer_to_internal_list + 24) 1275 1276 # if key < 0: 1277 # key += self_len 1278 1279 # if key < 0 or key >= self_len: 1280 # raise IndexError 1281 1282 # item_type_offset = pointer_to_internal_list + 32 + key * 16 1283 # item_value_offset = pointer_to_internal_list + 40 + key * 16 1284 # item_type = read_uint64(base_address, item_type_offset) 1285 # if item_type == 1: 1286 # return read_int64(base_address, item_value_offset) 1287 # elif item_type == 2: 1288 # return read_double(base_address, item_value_offset) 1289 # elif item_type == 3: 1290 # return bool(read_uint64(base_address, item_value_offset)) 1291 # elif item_type == 0: 1292 # item_offset = read_uint64(base_address, item_value_offset) 1293 # return self._shared_memory.get_obj(item_offset) 1294 # else: 1295 # raise ValueError 1296 1297 # # return self._read_item(key) 1298 elif isinstance(key, slice): 1299 if key.step is not None: 1300 raise NotImplementedError 1301 1302 if key.start is None: 1303 start = 0 1304 elif key.start < 0: 1305 start = key.start + len(self) 1306 else: 1307 start = key.start 1308 1309 if key.stop is None: 1310 stop = len(self) 1311 elif key.stop < 0: 1312 stop = key.stop + len(self) 1313 else: 1314 stop = key.stop 1315 1316 if start < 0 or start >= len(self) or stop < 0 or stop > len(self) or start >= stop: 1317 raise IndexError 1318 1319 result_list = list() 1320 # performance improvement instead of using self._read_item(i) 1321 base_address = self._base_address 1322 offset__pointer_to_internal_list = self._offset__pointer_to_internal_list 1323 pointer_to_internal_list = read_uint64(base_address, offset__pointer_to_internal_list) 1324 item_type_offset = pointer_to_internal_list + 32 + i * 16 1325 item_value_offset = pointer_to_internal_list + 40 + i * 16 1326 for i in range(start, stop): 1327 # result_list.append(self._read_item(i)) 1328 1329 # performance improvement instead of using self._read_item(i) 1330 item_type = read_uint64(base_address, item_type_offset) 1331 if item_type == 1: 1332 result_list.append(read_int64(base_address, item_value_offset)) 1333 elif item_type == 2: 1334 result_list.append(read_double(base_address, item_value_offset)) 1335 elif item_type == 3: 1336 result_list.append(bool(read_uint64(base_address, item_value_offset))) 1337 elif item_type == 0: 1338 item_offset = read_uint64(base_address, item_value_offset) 1339 result_list.append(self._shared_memory.get_obj(item_offset)) 1340 else: 1341 raise ValueError 1342 1343 return result_list 1344 else: 1345 raise TypeError 1346 1347 def __setitem__(self, key: Union[int, slice], value: Union[Any, Sequence]) -> Any: 1348 if isinstance(key, int): 1349 list__set_item(key, value, self._base_address, self._offset__pointer_to_internal_list, self._shared_memory.put_obj) 1350 1351 # base_address = self._base_address 1352 # offset__pointer_to_internal_list = self._offset__pointer_to_internal_list 1353 # pointer_to_internal_list = read_uint64(base_address, offset__pointer_to_internal_list) 1354 # self_len = read_uint64(base_address, pointer_to_internal_list + 24) 1355 1356 # if key < 0: 1357 # key += self_len 1358 1359 # if key < 0 or key >= self_len: 1360 # raise IndexError 1361 1362 # item_type_offset = pointer_to_internal_list + 32 + key * 16 1363 # item_value_offset = pointer_to_internal_list + 40 + key * 16 1364 # if isinstance(value, int): 1365 # write_int64(base_address, item_value_offset, value) 1366 # item_type = 1 1367 # elif isinstance(value, float): 1368 # write_double(base_address, item_value_offset, value) 1369 # item_type = 2 1370 # elif isinstance(value, bool): 1371 # write_uint64(base_address, item_value_offset, int(value)) 1372 # item_type = 3 1373 # else: 1374 # item_mapped_obj, item_offset, item_size = self._shared_memory.put_obj(value) 1375 # write_uint64(base_address, item_value_offset, item_offset) 1376 # item_type = 0 1377 1378 # write_uint64(base_address, item_type_offset, item_type) 1379 1380 # # self._write_item(key, value) 1381 elif isinstance(key, slice): 1382 if key.step is not None: 1383 raise NotImplementedError 1384 1385 if key.start is None: 1386 start = 0 1387 elif key.start < 0: 1388 start = key.start + len(self) 1389 else: 1390 start = key.start 1391 1392 if key.stop is None: 1393 stop = len(self) 1394 elif key.stop < 0: 1395 stop = key.stop + len(self) 1396 else: 1397 stop = key.stop 1398 1399 if start < 0 or start >= len(self) or stop < 0 or stop > len(self) or start >= stop: 1400 raise IndexError 1401 1402 # performance improvement instead of using self._write_item(i, item) 1403 base_address = self._base_address 1404 offset__pointer_to_internal_list = self._offset__pointer_to_internal_list 1405 pointer_to_internal_list = read_uint64(base_address, offset__pointer_to_internal_list) 1406 item_type_offset = pointer_to_internal_list + 32 + i * 16 1407 item_value_offset = pointer_to_internal_list + 40 + i * 16 1408 for i in range(start, stop): 1409 item = value[i - start] 1410 # self._write_item(i, item) 1411 1412 # performance improvement instead of using self._write_item(i, item) 1413 if isinstance(item, int): 1414 write_int64(base_address, item_value_offset, item) 1415 item_type = 1 1416 elif isinstance(item, float): 1417 write_double(base_address, item_value_offset, item) 1418 item_type = 2 1419 elif isinstance(item, bool): 1420 write_uint64(base_address, item_value_offset, int(item)) 1421 item_type = 3 1422 else: 1423 item_mapped_obj, item_offset, item_size = self._shared_memory.put_obj(item) 1424 write_uint64(base_address, item_value_offset, item_offset) 1425 item_type = 0 1426 1427 write_uint64(base_address, item_type_offset, item_type) 1428 else: 1429 raise TypeError 1430 1431 def __delitem__(self, key: Union[int, slice]) -> None: 1432 if isinstance(key, int): 1433 if key < 0: 1434 key += len(self) 1435 if key < 0 or key >= len(self): 1436 raise IndexError 1437 1438 self._free_item(key) 1439 1440 for i in range(key + 1, len(self)): 1441 self._move_item(i, i - 1) 1442 1443 self._list_len -= 1 1444 elif isinstance(key, slice): 1445 if key.step is not None: 1446 raise NotImplementedError 1447 1448 if key.start is None: 1449 start = 0 1450 elif key.start < 0: 1451 start = key.start + len(self) 1452 else: 1453 start = key.start 1454 1455 if key.stop is None: 1456 stop = len(self) 1457 elif key.stop < 0: 1458 stop = key.stop + len(self) 1459 else: 1460 stop = key.stop 1461 1462 if start < 0 or start >= len(self) or stop < 0 or stop > len(self) or start >= stop: 1463 raise IndexError 1464 1465 for i in range(start, stop): 1466 self._free_item(i) 1467 1468 del_items_num = stop - start 1469 1470 for i in range(stop, len(self)): 1471 self._move_item(i, i - del_items_num) 1472 1473 self._list_len -= del_items_num 1474 else: 1475 raise TypeError 1476 1477 def append(self, item: Any) -> None: 1478 if self._list_len > self._list_capacity: 1479 self._pointer_to_internal_list, result_size = realloc_tinternal_list(self._shared_memory, self._pointer_to_internal_list) 1480 1481 self._list_len += 1 1482 self.__setitem__(self._list_len - 1, item) 1483 1484 def extend(self, items: Sequence) -> None: 1485 items_num = len(items) 1486 if self._list_len + items_num > self._list_capacity: 1487 self._pointer_to_internal_list, result_size = realloc_tinternal_list(self._shared_memory, self._pointer_to_internal_list, self._list_len + items_num) 1488 1489 original_list_len = self._list_len 1490 self._list_len += items_num 1491 for i, item in enumerate(items): 1492 self.__setitem__(original_list_len + i, item) 1493 1494 1495 def insert(self, index: int, item: Any) -> None: 1496 if index < 0: 1497 index += len(self) 1498 if index < 0 or index > len(self): 1499 raise IndexError 1500 1501 if self._list_len > self._list_capacity: 1502 # self._shared_memory.print_mem(self._pointer_to_internal_list, 200, 'before realloc. {}') 1503 # self.print_internal_list('before realloc. {}') 1504 self._pointer_to_internal_list, result_size = realloc_tinternal_list(self._shared_memory, self._pointer_to_internal_list) 1505 # self._shared_memory.print_mem(self._pointer_to_internal_list, 200, 'after realloc. {}') 1506 # self.print_internal_list('after realloc. {}') 1507 1508 # self.print_internal_list('before inserting {}') 1509 self._list_len += 1 1510 # self.print_internal_list('before inserting but after +1 {}') 1511 for i in range(self._list_len - 1, index, -1): 1512 self._move_item(i - 1, i) 1513 # self._shared_memory.print_mem(self._pointer_to_internal_list, 200, f'after self._move_item({i - 1, i}). {{}}') 1514 # self.print_internal_list(f'after self._move_item({i - 1, i}). {{}}') 1515 1516 self.__setitem__(index, item) 1517 # self._shared_memory.print_mem(self._pointer_to_internal_list, 200, 'after inserting. {}') 1518 # self.print_internal_list('after inserting. {}') 1519 1520 def print_internal_list(self, text: str = None, additional_cells: int = 0): 1521 internal_list = self._shared_memory.read_mem(self._pointer_to_internal_list, bs * len(BaseObjOffsets) + bs * len(InternalListOffsets) + self._list_len * bs * len(InternalListFieldOffsets) + additional_cells * bs * len(InternalListFieldOffsets)) 1522 print('--- internal list -------------') 1523 if text: 1524 print(text.format(self._pointer_to_internal_list)) 1525 print('------') 1526 1527 index = 0 1528 print(f'{index},{self._pointer_to_internal_list + index}:', internal_list[index:index + bs]) 1529 index += bs 1530 print(f'{index},{self._pointer_to_internal_list + index}:', internal_list[index:index + bs]) 1531 index += bs 1532 print('---') 1533 print(f'{index},{self._pointer_to_internal_list + index}:', internal_list[index:index + bs]) 1534 index += bs 1535 print(f'{index},{self._pointer_to_internal_list + index}:', internal_list[index:index + bs]) 1536 index += bs 1537 print('---') 1538 for i in range(self._list_len): 1539 print(f'{index},{self._pointer_to_internal_list + index}:', internal_list[index:index + bs * 2]) 1540 index += bs * 2 1541 1542 if additional_cells: 1543 print('------') 1544 for i in range(additional_cells): 1545 print(f'{index},{self._pointer_to_internal_list + index}:', internal_list[index:index + bs]) 1546 index += bs * 2 1547 print('-------------------------------') 1548 print() 1549 1550 def pop(self, index: int = -1) -> Any: 1551 if index < 0: 1552 index += len(self) 1553 if index < 0 or index >= len(self): 1554 raise IndexError 1555 1556 result = self.__getitem__(index) 1557 1558 for i in range(index + 1, len(self)): 1559 self._move_item(i, i - 1) 1560 1561 self._list_len -= 1 1562 return result 1563 1564 def remove(self, obj: Any) -> None: 1565 obj_type = self._determine_obj_type(obj) 1566 obj_offset = self._determine_obj_offset(obj) 1567 found_in_index = None 1568 for i in range(len(self)): 1569 if self._compare_item_to_obj_fast(i, obj, obj_type, obj_offset): 1570 found_in_index = i 1571 break 1572 1573 if found_in_index is None: 1574 raise ValueError 1575 else: 1576 self.__delitem__(found_in_index) 1577 1578 def clear(self) -> None: 1579 for i in range(len(self)): 1580 self._free_item(i) 1581 1582 self._list_len = 0 1583 1584 def __iter__(self): 1585 return IListIterator(self) 1586 1587 def __reversed__(self): 1588 return IListReversedIterator(self) 1589 1590 def __contains__(self, obj: Any) -> bool: 1591 obj_type = self._determine_obj_type(obj) 1592 obj_offset = self._determine_obj_offset(obj) 1593 found_in_index = None 1594 for i in range(len(self)): 1595 if self._compare_item_to_obj_fast(i, obj, obj_type, obj_offset): 1596 found_in_index = i 1597 break 1598 1599 if found_in_index is None: 1600 return False 1601 else: 1602 return True 1603 1604 def index(self, obj: Any, start: int = 0, stop: int = None) -> int: 1605 if stop is None: 1606 stop = len(self) 1607 1608 obj_type = self._determine_obj_type(obj) 1609 obj_offset = self._determine_obj_offset(obj) 1610 found_in_index = None 1611 for i in range(start, stop): 1612 if self._compare_item_to_obj_fast(i, obj, obj_type, obj_offset): 1613 found_in_index = i 1614 break 1615 1616 if found_in_index is None: 1617 raise ValueError 1618 else: 1619 return found_in_index 1620 1621 def count(self, obj: Any) -> int: 1622 obj_type = self._determine_obj_type(obj) 1623 obj_offset = self._determine_obj_offset(obj) 1624 result = 0 1625 found_in_index = None 1626 for i in range(len(self)): 1627 if self._compare_item_to_obj_fast(i, obj, obj_type, obj_offset): 1628 found_in_index = i 1629 result += 1 1630 1631 return result 1632 1633 def reverse(self) -> None: 1634 my_len = len(self) 1635 for i in range(my_len // 2): 1636 self._swap_items(i, my_len - i - 1) 1637 1638 def sort(self, key: Any = None, reverse: bool = False) -> None: 1639 raise NotImplementedError 1640 1641 def copy(self) -> 'IList': 1642 result = IList(self._shared_memory) 1643 result.extend(self) 1644 return result 1645 1646 def __add__(self, other: Sequence) -> 'IList': 1647 result = IList(self._shared_memory) 1648 result.extend(self) 1649 result.extend(other) 1650 return result 1651 1652 def __iadd__(self, other: Sequence) -> 'IList': 1653 self.extend(other) 1654 return self 1655 1656 def __mul__(self, other: int) -> 'IList': 1657 result = IList(self._shared_memory) 1658 for i in range(other): 1659 result.extend(self) 1660 1661 return result 1662 1663 def __imul__(self, other: int) -> 'IList': 1664 my_copy: IList = self.copy() 1665 for i in range(other): 1666 self.extend(my_copy) 1667 1668 return self 1669 1670 def __rmul__(self, other: int) -> 'IList': 1671 return self.__mul__(other) 1672 1673 def __eq__(self, other: Sequence) -> bool: 1674 if len(self) != len(other): 1675 return False 1676 1677 for i in range(len(self)): 1678 if self[i] != other[i]: 1679 return False 1680 1681 return True 1682 1683 def __ne__(self, other: Sequence) -> bool: 1684 return not self.__eq__(other) 1685 1686 def __lt__(self, other: Sequence) -> bool: 1687 for i in range(len(self)): 1688 if self[i] >= other[i]: 1689 return False 1690 1691 return True 1692 1693 def __le__(self, other: Sequence) -> bool: 1694 for i in range(len(self)): 1695 if self[i] > other[i]: 1696 return False 1697 1698 return True 1699 1700 def __gt__(self, other: Sequence) -> bool: 1701 for i in range(len(self)): 1702 if self[i] <= other[i]: 1703 return False 1704 1705 return True 1706 1707 def __ge__(self, other: Sequence) -> bool: 1708 for i in range(len(self)): 1709 if self[i] < other[i]: 1710 return False 1711 1712 return True 1713 1714 def __repr__(self) -> str: 1715 return f'IList({list(self)})' 1716 1717 def __str__(self) -> str: 1718 return f'IList({list(self)})' 1719 1720 def __hash__(self) -> int: 1721 return hash(tuple(self)) 1722 1723 def __sizeof__(self) -> int: 1724 return bs * len(BaseObjOffsets) + read_uint64(self._base_address, self._offset + bs * BaseObjOffsets.obj_size) + bs * len(BaseObjOffsets) + read_uint64(self._base_address, self._pointer_to_internal_list, bs * BaseObjOffsets.obj_size) 1725 1726 def export(self) -> list: 1727 return list(self) 1728 1729 # def __del__(self) -> None: 1730 # self._shared_memory.free(self._pointer_to_internal_list) 1731 # self._shared_memory.free(self._offset) 1732 1733 1734# IList = IListTrue 1735 1736 1737class IListIterator: 1738 def __init__(self, ilist: IList) -> None: 1739 self._ilist = ilist 1740 self._index = 0 1741 1742 def __next__(self): 1743 if self._index < len(self._ilist): 1744 # self._ilist.print_internal_list(f'ListIterator[{self._index}]. {{}}') 1745 result = self._ilist[self._index] 1746 self._index += 1 1747 return result 1748 else: 1749 raise StopIteration 1750 1751 def __iter__(self): 1752 return self 1753 1754 1755class IListReversedIterator: 1756 def __init__(self, ilist: IList) -> None: 1757 self._ilist = ilist 1758 self._index = len(ilist) - 1 1759 1760 def __next__(self): 1761 if self._index >= 0: 1762 result = self._ilist[self._index] 1763 self._index -= 1 1764 return result 1765 else: 1766 raise StopIteration 1767 1768 def __iter__(self): 1769 return self 1770 1771 1772class TList: 1773 def map_to_shared_memory(self, shared_memory: 'SharedMemory', obj: list) -> Tuple[list, Offset, Size]: 1774 obj = IList(shared_memory, obj=obj) 1775 return obj, obj._offset, obj._obj_size 1776 1777 def init_from_shared_memory(self, shared_memory: 'SharedMemory', offset: Offset) -> None: 1778 if ObjectType.tlist != read_uint64(shared_memory.base_address, offset): 1779 raise WrongObjectTypeError 1780 1781 return IList(shared_memory, offset) 1782 1783 def destroy(self, shared_memory: 'SharedMemory', offset: Offset): 1784 shared_memory.free(offset) 1785 1786 1787# ====================================================================================================================== 1788# === Tuple ============================================================================================================ 1789 1790 1791class TupleOffsets(IntEnum): 1792 size = 0 1793 1794 1795class TupleFieldOffsets(IntEnum): 1796 item_offset = 0 1797 1798 1799class TTuple: 1800 def map_to_shared_memory(self, shared_memory: 'SharedMemory', obj: tuple) -> Tuple[tuple, Offset, Size]: 1801 offset, real_size = shared_memory.malloc(ObjectType.ttuple, bs * len(TupleOffsets) + len(obj) * bs * len(TupleFieldOffsets)) 1802 if (1, [2, 3]) == obj: 1803 shared_memory.offset_to_be_monitored = offset 1804 1805 write_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * TupleOffsets.size, len(obj)) 1806 for i, item in enumerate(obj): 1807 item_mapped_obj, item_offset, item_size = shared_memory.put_obj(item) 1808 write_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * len(TupleOffsets) + i * bs * len(TupleFieldOffsets), item_offset) 1809 1810 return obj, offset, real_size 1811 1812 def init_from_shared_memory(self, shared_memory: 'SharedMemory', offset: Offset) -> None: 1813 if ObjectType.ttuple != read_uint64(shared_memory.base_address, offset): 1814 raise WrongObjectTypeError 1815 1816 result_list = list() 1817 size = read_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * TupleOffsets.size) 1818 for i in range(size): 1819 item_offset = read_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * len(TupleOffsets) + i * bs * len(TupleFieldOffsets)) 1820 result_list.append(shared_memory.get_obj(item_offset)) 1821 1822 return tuple(result_list) 1823 1824 def destroy(self, shared_memory: 'SharedMemory', offset: Offset): 1825 shared_memory.free(offset) 1826 1827 1828# ====================================================================================================================== 1829# === Dict ============================================================================================================= 1830 1831 1832class DictOffsets(IntEnum): 1833 data_tuple_offset = 0 1834 1835 1836class TDict: 1837 def map_to_shared_memory(self, shared_memory: 'SharedMemory', obj: dict) -> Tuple[dict, Offset, Size]: 1838 offset, real_size = shared_memory.malloc(ObjectType.tdict, bs * len(DictOffsets)) 1839 item_mapped_obj, item_offset, item_size = shared_memory.put_obj(tuple(obj.items())) 1840 write_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * DictOffsets.data_tuple_offset, item_offset) 1841 return obj, offset, real_size 1842 1843 def init_from_shared_memory(self, shared_memory: 'SharedMemory', offset: Offset) -> None: 1844 if ObjectType.tdict != read_uint64(shared_memory.base_address, offset): 1845 raise WrongObjectTypeError 1846 1847 item_offset = read_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * DictOffsets.data_tuple_offset) 1848 result_tuple = shared_memory.get_obj(item_offset) 1849 return dict(result_tuple) 1850 1851 def destroy(self, shared_memory: 'SharedMemory', offset: Offset): 1852 shared_memory.free(offset) 1853 1854 1855codec_by_type: Dict[ObjectType, TBase] = { 1856 ObjectType.tnone: TNone(), 1857 ObjectType.tint: TInt(), 1858 ObjectType.tbool: TBool(), 1859 ObjectType.tfloat: TFloat(), 1860 ObjectType.tbytes: TBytes(), 1861 ObjectType.tbytearray: TBytearray(), 1862 ObjectType.tstr: TStr(), 1863 ObjectType.tlist: TList(), 1864 ObjectType.ttuple: TTuple(), 1865 ObjectType.tdict: TDict(), 1866} 1867obj_type_map: Dict[Type, ObjectType] = { 1868} 1869 1870 1871# ====================================================================================================================== 1872# === Message ========================================================================================================== 1873 1874 1875class MessageOffsets(IntEnum): 1876 previous_message_offset = 0 1877 next_message_offset = 1 1878 item_offset = 2 1879 1880 1881class SharedMemory: 1882 def __init__(self, name: str, create: bool = False, size: int = 0, queue_type: QueueType = QueueType.fifo, zero_mem: bool = True): 1883 global current_shared_memory_instance 1884 current_shared_memory_instance = self 1885 self.offset_to_be_monitored: Offset = None 1886 self._malloc_time: float = 0.0 1887 self._realloc_time: float = 0.0 1888 self._shared_memory: MultiprocessingSharedMemory = MultiprocessingSharedMemory(name=name, create=create, size=size) 1889 self.base_address = ctypes.addressof(ctypes.c_char.from_buffer(self._shared_memory.buf)) 1890 self.sys_values_offset = 0 1891 # if create: 1892 # print(f'Creator: {self.base_address=}') 1893 # else: 1894 # print(f'Consumer: {self.base_address=}') 1895 1896 self._name: str = name 1897 self._create: bool = create 1898 self._size: int = size 1899 self._queue_type: QueueType = queue_type 1900 self._zero_mem: bool = zero_mem 1901 self._last_message_offset: Offset = None 1902 1903 self._shared_memory_bytearray = bytearray(self._shared_memory.buf) 1904 1905 sys_arr_length = len(SysValuesOffsets) 1906 arr_byte_size = sys_arr_length * bs 1907 self.log_arr = np.ndarray((500,), dtype=np.uint64, buffer=self._shared_memory.buf) 1908 self.sys_arr = np.ndarray((sys_arr_length,), dtype=np.uint64, buffer=self._shared_memory.buf) 1909 write_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.data_start_offset, sys_arr_length * bs) 1910 write_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.data_size, self._size - arr_byte_size) 1911 write_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.data_end_offset, self._size) 1912 write_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.free_memory_search_start, sys_arr_length * bs) 1913 write_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.first_message_offset, 0) 1914 write_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.last_message_offset, 0) 1915 write_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.creator_in_charge, 0) 1916 write_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.consumer_in_charge, 0) 1917 write_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.creator_wants_to_be_in_charge, 0) 1918 write_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.consumer_wants_to_be_in_charge, 0) 1919 write_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.creator_ready, 0) 1920 write_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.consumer_ready, 0) 1921 # print(bytes(self._shared_memory.buf[0:120])) 1922 1923 self.free_memory_search_start = self.read_free_memory_search_start() 1924 data_size: int = self.get_data_size() 1925 if self._create and self._zero_mem: 1926 zero_memory(self.base_address, self.free_memory_search_start, data_size) 1927 1928 write_uint64(self.base_address, self.free_memory_search_start + bs * BaseObjOffsets.obj_type, ObjectType.tfree_memory.value) 1929 write_uint64(self.base_address, self.free_memory_search_start + bs * BaseObjOffsets.obj_size, data_size - bs * len(BaseObjOffsets)) 1930 1931 if self._create: 1932 write_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.creator_ready, 1) 1933 else: 1934 write_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.consumer_ready, 1) 1935 1936 # print(bytes(self._shared_memory.buf[0:120])) 1937 self.get_data_end_offset() 1938 1939 def read_mem(self, offset: Offset, size: Size) -> List[int]: 1940 result = list() 1941 for i in range(size): 1942 result.append(read_uint8(self.base_address, offset + i)) 1943 1944 return result 1945 1946 def print_mem(self, offset: Offset, size: Size, text: str = None): 1947 result = list() 1948 for i in range(size): 1949 result.append(read_uint8(self.base_address, offset + i)) 1950 1951 if text: 1952 print(f'{text.format(offset)}: {result}') 1953 else: 1954 print(f'{result}') 1955 1956 def set_creator_ready(self): 1957 write_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.creator_ready, 1) 1958 1959 def set_consumer_ready(self): 1960 write_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.consumer_ready, 1) 1961 1962 def get_creator_ready(self): 1963 return read_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.creator_ready) 1964 1965 def get_consumer_ready(self): 1966 return read_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.consumer_ready) 1967 1968 def wait_creator_ready(self, time_limit: Optional[RationalNumber] = None, periodic_sleep_time: Optional[RationalNumber] = 0.000000001): 1969 start_time = cpu_clock() 1970 while not read_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.creator_ready): 1971 if time_limit is not None: 1972 if (cpu_clock() - start_time) > time_limit: 1973 return False 1974 1975 if periodic_sleep_time is None: 1976 mm_pause() 1977 else: 1978 hps_sleep(periodic_sleep_time) 1979 1980 def wait_consumer_ready(self, time_limit: Optional[RationalNumber] = None, periodic_sleep_time: Optional[RationalNumber] = 0.000000001): 1981 start_time = cpu_clock() 1982 while not read_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.consumer_ready): 1983 if time_limit is not None: 1984 if (cpu_clock() - start_time) > time_limit: 1985 return False 1986 1987 if periodic_sleep_time is None: 1988 mm_pause() 1989 else: 1990 hps_sleep(periodic_sleep_time) 1991 1992 def creator_in_charge(self) -> bool: 1993 return read_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.creator_in_charge) 1994 1995 def consumer_in_charge(self) -> bool: 1996 return read_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.consumer_in_charge) 1997 1998 def creator_wants_to_be_in_charge(self) -> bool: 1999 return read_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.creator_wants_to_be_in_charge) 2000 2001 def consumer_wants_to_be_in_charge(self) -> bool: 2002 return read_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.consumer_wants_to_be_in_charge) 2003 2004 def read_free_memory_search_start(self) -> int: 2005 # return self.get_data_start_offset() 2006 return read_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.free_memory_search_start) 2007 2008 def update_free_memory_search_start(self) -> int: 2009 self.free_memory_search_start = self.read_free_memory_search_start() 2010 2011 def get_free_memory_search_start(self) -> int: 2012 # self.update_free_memory_search_start() 2013 return self.free_memory_search_start 2014 2015 def write_free_memory_search_start(self, offset: Offset) -> int: 2016 # return 2017 if ((self.get_data_end_offset() - bs * len(BaseObjOffsets)) < offset) or (offset < self.get_data_start_offset()): 2018 offset = self.get_data_start_offset() 2019 2020 write_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.free_memory_search_start, offset) 2021 2022 def commit_free_memory_search_start(self): 2023 self.write_free_memory_search_start(self.free_memory_search_start) 2024 2025 def set_free_memory_search_start(self, offset: Offset) -> int: 2026 # return 2027 if ((self.get_data_end_offset() - bs * len(BaseObjOffsets)) < offset) or (offset < self.get_data_start_offset()): 2028 offset = self.get_data_start_offset() 2029 2030 self.free_memory_search_start = offset 2031 # self.commit_free_memory_search_start() 2032 2033 def get_last_message_offset(self) -> Optional[Offset]: 2034 return read_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.last_message_offset) 2035 2036 def set_last_message_offset(self, offset: Offset): 2037 write_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.last_message_offset, offset) 2038 2039 def get_first_message_offset(self) -> Optional[Offset]: 2040 return read_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.first_message_offset) 2041 2042 def set_first_message_offset(self, offset: Offset): 2043 write_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.first_message_offset, offset) 2044 2045 def get_data_start_offset(self) -> Offset: 2046 return read_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.data_start_offset) 2047 2048 def get_data_size(self) -> Size: 2049 return read_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.data_size) 2050 2051 def get_data_end_offset(self) -> Offset: 2052 result = read_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.data_end_offset) 2053 if result != len(self._shared_memory.buf): 2054 print(result, len(self._shared_memory.buf)) 2055 2056 return result 2057 2058 # def read_uint64(self, offset: Offset) -> int: 2059 # return read_uint64(self.base_address, offset) 2060 2061 # def write_uint64(self, offset: Offset, value: int): 2062 # write_uint64(self.base_address, offset, value) 2063 2064 def read_uint64(self, offset: Offset) -> int: 2065 return int.from_bytes(self._shared_memory.buf[offset:offset + 8], byteorder='little', signed=False) 2066 2067 def write_uint64(self, offset: Offset, value: int): 2068 self._shared_memory.buf[offset:offset + 8] = value.to_bytes(8, byteorder='little', signed=False) 2069 2070 # def read_uint32(self, offset: Offset) -> int: 2071 # return int.from_bytes(self._shared_memory.buf[offset:offset + 4], byteorder='little', signed=False) 2072 2073 # def write_uint32(self, offset: Offset, value: int): 2074 # self._shared_memory.buf[offset:offset + 4] = value.to_bytes(4, byteorder='little', signed=False) 2075 2076 # def read_uint16(self, offset: Offset) -> int: 2077 # return int.from_bytes(self._shared_memory.buf[offset:offset + 2], byteorder='little', signed=False) 2078 2079 # def write_uint16(self, offset: Offset, value: int): 2080 # self._shared_memory.buf[offset:offset + 2] = value.to_bytes(2, byteorder='little', signed=False) 2081 2082 # def read_uint8(self, offset: Offset) -> int: 2083 # return int.from_bytes(self._shared_memory.buf[offset:offset + 1], byteorder='little', signed=False) 2084 2085 # def write_uint8(self, offset: Offset, value: int): 2086 # self._shared_memory.buf[offset:offset + 1] = value.to_bytes(1, byteorder='little', signed=False) 2087 2088 # def read_int64(self, offset: Offset) -> int: 2089 # return int.from_bytes(self._shared_memory.buf[offset:offset + 8], byteorder='little', signed=True) 2090 2091 # def write_int64(self, offset: Offset, value: int): 2092 # self._shared_memory.buf[offset:offset + 8] = value.to_bytes(8, byteorder='little', signed=True) 2093 2094 # def read_int32(self, offset: Offset) -> int: 2095 # return int.from_bytes(self._shared_memory.buf[offset:offset + 4], byteorder='little', signed=True) 2096 2097 # def write_int32(self, offset: Offset, value: int): 2098 # self._shared_memory.buf[offset:offset + 4] = value.to_bytes(4, byteorder='little', signed=True) 2099 2100 # def read_int16(self, offset: Offset) -> int: 2101 # return int.from_bytes(self._shared_memory.buf[offset:offset + 2], byteorder='little', signed=True) 2102 2103 # def write_int16(self, offset: Offset, value: int): 2104 # self._shared_memory.buf[offset:offset + 2] = value.to_bytes(2, byteorder='little', signed=True) 2105 2106 # def read_int8(self, offset: Offset) -> int: 2107 # return int.from_bytes(self._shared_memory.buf[offset:offset + 1], byteorder='little', signed=True) 2108 2109 # def write_int8(self, offset: Offset, value: int): 2110 # self._shared_memory.buf[offset:offset + 1] = value.to_bytes(1, byteorder='little', signed=True) 2111 2112 # def read_float(self, offset: Offset) -> float: 2113 # return float.from_bytes(self._shared_memory.buf[offset:offset + 4], byteorder='little', signed=False) 2114 2115 # def write_float(self, offset: Offset, value: float): 2116 # self._shared_memory.buf[offset:offset + 4] = value.to_bytes(4, byteorder='little', signed=False) 2117 2118 # def read_double(self, offset: Offset) -> float: 2119 # return float.from_bytes(self._shared_memory.buf[offset:offset + 8], byteorder='little', signed=False) 2120 2121 # def write_double(self, offset: Offset, value: float): 2122 # self._shared_memory.buf[offset:offset + 8] = value.to_bytes(8, byteorder='little', signed=False) 2123 2124 # def read_complex(self, offset: Offset) -> complex: 2125 # return complex.from_bytes(self._shared_memory.buf[offset:offset + 16], byteorder='little', signed=False) 2126 2127 # def write_complex(self, offset: Offset, value: complex): 2128 # self._shared_memory.buf[offset:offset + 16] = value.to_bytes(16, byteorder='little', signed=False) 2129 2130 # def read_bool(self, offset: Offset) -> bool: 2131 # return bool.from_bytes(self._shared_memory.buf[offset:offset + 1], byteorder='little', signed=False) 2132 2133 # def write_bool(self, offset: Offset, value: bool): 2134 # self._shared_memory.buf[offset:offset + 1] = value.to_bytes(1, byteorder='little', signed=False) 2135 2136 # def read_str(self, offset: Offset) -> str: 2137 # size = read_uint64(self.base_address, offset) 2138 # return self._shared_memory.buf[offset + 8:offset + 8 + size].decode() 2139 2140 # def read_str_2(self, offset: Offset, size: Size) -> str: 2141 # return self._shared_memory.buf[offset + 8:offset + 8 + size].decode() 2142 2143 # def write_str(self, offset: Offset, value: str): 2144 # size = len(value) 2145 # write_uint64(self.base_address, offset, size) 2146 # self._shared_memory.buf[offset + 8:offset + 8 + size] = value.encode() 2147 2148 # def read_bytes(self, offset: Offset) -> bytes: 2149 # size = read_uint64(self.base_address, offset) 2150 # return self._shared_memory.buf[offset + 8:offset + 8 + size] 2151 2152 # def read_bytes_2(self, offset: Offset, size: Size) -> bytes: 2153 # return self._shared_memory.buf[offset + 8:offset + 8 + size] 2154 2155 # def write_bytes(self, offset: Offset, value: bytes): 2156 # size = len(value) 2157 # write_uint64(self.base_address, offset, size) 2158 # self._shared_memory.buf[offset + 8:offset + 8 + size] = value 2159 2160 # def read_bytearray(self, offset: Offset) -> bytearray: 2161 # size = read_uint64(self.base_address, offset) 2162 # return bytearray(self._shared_memory.buf[offset + 8:offset + 8 + size]) 2163 2164 # def read_bytearray_2(self, offset: Offset, size: Size) -> bytearray: 2165 # return bytearray(self._shared_memory.buf[offset + 8:offset + 8 + size]) 2166 2167 # def write_bytearray(self, offset: Offset, value: bytearray): 2168 # size = len(value) 2169 # write_uint64(self.base_address, offset, size) 2170 # self._shared_memory.buf[offset + 8:offset + 8 + size] = value 2171 2172 # def read_tuple(self, offset: Offset) -> tuple: 2173 # size = read_uint64(self.base_address, offset) 2174 # return tuple(self._shared_memory.buf[offset + 8:offset + 8 + size]) 2175 2176 # def write_tuple(self, offset: Offset, value: tuple): 2177 # size = len(value) 2178 # write_uint64(self.base_address, offset, size) 2179 # self._shared_memory.buf[offset + 8:offset + 8 + size] = value 2180 2181 # def read_list(self, offset: Offset) -> list: 2182 # size = read_uint64(self.base_address, offset) 2183 # return list(self._shared_memory.buf[offset + 8:offset + 8 + size]) 2184 2185 # def write_list(self, offset: Offset, value: list): 2186 # size = len(value) 2187 # write_uint64(self.base_address, offset, size) 2188 # self._shared_memory.buf[offset + 8:offset + 8 + size] = value 2189 2190 # def read_dict(self, offset: Offset) -> dict: 2191 # size = read_uint64(self.base_address, offset) 2192 # return dict(self._shared_memory.buf[offset + 8:offset + 8 + size]) 2193 2194 # def write_dict(self, offset: Offset, value: dict): 2195 # size = len(value) 2196 # write_uint64(self.base_address, offset, size) 2197 # self._shared_memory.buf[offset + 8:offset + 8 + size] = value 2198 2199 # def read_set(self, offset: Offset) -> set: 2200 # size = read_uint64(self.base_address, offset) 2201 # return set(self._shared_memory.buf[offset + 8:offset + 8 + size]) 2202 2203 # def write_set(self, offset: Offset, value: set): 2204 # size = len(value) 2205 # write_uint64(self.base_address, offset, size) 2206 # self._shared_memory.buf[offset + 8:offset + 8 + size] = value 2207 2208 # def read_pickable(self, offset: Offset) -> Any: 2209 # size = read_uint64(self.base_address, offset) 2210 # return pickle.loads(self._shared_memory.buf[offset + 8:offset + 8 + size]) 2211 2212 # def write_pickable(self, offset: Offset, value: Any): 2213 # value_bytes = pickle.dumps(value) 2214 # size = len(value_bytes) 2215 # write_uint64(self.base_address, offset, size) 2216 # self._shared_memory.buf[offset + 8:offset + 8 + size] = value_bytes 2217 2218 # ---------------------------- 2219 2220 def read_obj_type_and_size(self, offset: Offset) -> Tuple[ObjectType, Size]: 2221 obj_type = ObjectType(read_uint64(self.base_address, offset + bs * BaseObjOffsets.obj_type)) 2222 size = read_uint64(self.base_address, offset + bs * BaseObjOffsets.obj_size) 2223 return obj_type, size 2224 2225 def write_obj_type_and_size(self, offset: Offset, obj_type: ObjectType, size: Size): 2226 write_uint64(self.base_address, offset + bs * BaseObjOffsets.obj_type, obj_type.value) 2227 write_uint64(self.base_address, offset + bs * BaseObjOffsets.obj_size, size) 2228 return offset + bs * len(BaseObjOffsets) 2229 2230 # ---------------------------- 2231 2232 def test_free_memory_blocks(self, offset: Offset, desired_size: Size) -> Tuple[bool, Size, Offset]: 2233 adjusted_size = desired_size 2234 initial_offset = offset 2235 sum_size = 0 2236 max_viable_offset = self.get_data_end_offset() - bs * len(BaseObjOffsets) 2237 last_found_obj_offset = None 2238 last_found_obj_size = None 2239 while True: 2240 last_found_obj_offset = offset 2241 try: 2242 obj_type = ObjectType(read_uint64(self.base_address, offset)) 2243 except ValueError: 2244 print(f'Error: {offset=}, {desired_size=}, {sum_size=}') 2245 2246 size = read_uint64(self.base_address, offset + bs * BaseObjOffsets.obj_size) 2247 if size % bs: 2248 print('WRONG SIZE') 2249 2250 last_found_obj_size = bs * len(BaseObjOffsets) + size 2251 next_block_offset = last_found_obj_offset + last_found_obj_size 2252 if next_block_offset > self.get_data_end_offset(): 2253 print(f'{next_block_offset=}, {self.get_data_end_offset()=}, {len(self._shared_memory.buf)=}') 2254 return False, adjusted_size, None, None, next_block_offset 2255 2256 if obj_type is not ObjectType.tfree_memory: 2257 return False, adjusted_size, None, None, next_block_offset 2258 2259 sum_size = next_block_offset - initial_offset 2260 2261 if sum_size == desired_size: 2262 return True, adjusted_size, None, None, next_block_offset 2263 2264 if sum_size > desired_size: 2265 new_next_block_offset = initial_offset + desired_size 2266 new_next_block_size = last_found_obj_size - (new_next_block_offset - last_found_obj_offset) 2267 if new_next_block_size < bs * len(BaseObjOffsets): 2268 adjusted_size = desired_size + new_next_block_size 2269 return True, adjusted_size, None, None, next_block_offset 2270 else: 2271 return True, adjusted_size, new_next_block_offset, new_next_block_size, new_next_block_offset 2272 2273 offset = last_found_obj_offset + last_found_obj_size 2274 if offset > max_viable_offset: 2275 return False, adjusted_size, None, None, next_block_offset 2276 2277 def combine_free_memory_blocks(self, free_mem_block_offset: Offset, size: Size, last_free_block_offset: Offset, last_free_block_new_size: Size, next_block_offset: Offset, mark_block: bool = False) -> Tuple[Size, Offset]: 2278 if mark_block: 2279 self.write_obj_type_and_size(free_mem_block_offset, ObjectType.tfree_memory, size - bs * len(BaseObjOffsets)) 2280 2281 if last_free_block_offset is not None: 2282 if last_free_block_new_size - bs * len(BaseObjOffsets) < 0: 2283 print(f'Error: {last_free_block_new_size=}') 2284 2285 self.write_obj_type_and_size(last_free_block_offset, ObjectType.tfree_memory, last_free_block_new_size - bs * len(BaseObjOffsets)) 2286 2287 # self.set_free_memory_search_start(next_block_offset) 2288 2289 # ---------------------------- 2290 2291 def malloc(self, obj_type: ObjectType, size: Size, loop_allowed: bool = True, zero_mem: bool = False) -> Tuple[Optional[Offset], Size]: 2292 start_time = cpu_clock() 2293 try: 2294 size += bs * len(BaseObjOffsets) 2295 size = nearest_size(size) 2296 adjusted_size = size 2297 initial_start_offset = self.get_free_memory_search_start() 2298 search_end_offset = self.get_data_end_offset() - bs * len(BaseObjOffsets) 2299 start_offset = initial_start_offset 2300 free_mem_block_offset: Offset = None 2301 last_free_block_offset: Offset = None 2302 last_free_block_new_size: Size = None 2303 found: bool = False 2304 sum_size: Size = 0 2305 while (not found) and (start_offset <= search_end_offset): 2306 free_mem_block_offset = start_offset 2307 found, adjusted_size, last_free_block_offset, last_free_block_new_size, next_block_offset = self.test_free_memory_blocks(start_offset, size) 2308 start_offset = next_block_offset 2309 2310 if (not found) and loop_allowed: 2311 start_offset = self.get_data_start_offset() 2312 search_end_offset = initial_start_offset - bs * len(BaseObjOffsets) 2313 while (not found) and (start_offset <= search_end_offset): 2314 free_mem_block_offset = start_offset 2315 found, adjusted_size, last_free_block_offset, last_free_block_new_size, next_block_offset = self.test_free_memory_blocks(start_offset, size) 2316 start_offset = next_block_offset 2317 2318 if not found: 2319 raise FreeMemoryChunkNotFoundError(obj_type, size, loop_allowed, zero_mem) 2320 2321 self.combine_free_memory_blocks(free_mem_block_offset, adjusted_size, last_free_block_offset, last_free_block_new_size, next_block_offset) 2322 obj_size = adjusted_size - bs * len(BaseObjOffsets) 2323 self.write_obj_type_and_size(free_mem_block_offset, obj_type, obj_size) 2324 if zero_mem: 2325 # print(f'Zeroing memory 1: {free_mem_block_offset=}, {result_size=}') 2326 # hps_sleep(0.01) 2327 zero_memory(self.base_address, free_mem_block_offset + bs * len(BaseObjOffsets), obj_size) 2328 2329 if free_mem_block_offset % bs: 2330 print(f'Error: {free_mem_block_offset=}, {obj_size=}') 2331 2332 2333 self.set_free_memory_search_start(free_mem_block_offset) 2334 return free_mem_block_offset, obj_size 2335 finally: 2336 self._malloc_time += cpu_clock() - start_time 2337 2338 def zero_memory(self, offset: Offset, size: Size): 2339 # print(f'Zeroing memory 1: [{self.base_address + offset}:{self.base_address + offset + size}], {size=}') 2340 self._shared_memory_bytearray[offset:offset + size] = bytearray(size) 2341 2342 def calloc(self, obj_type: ObjectType, size: Size, num: int, loop_allowed: bool = True, zero_mem: bool = True) -> Tuple[Optional[Offset], Size]: 2343 return self.malloc(obj_type, size * num, loop_allowed, zero_mem) 2344 2345 def realloc(self, obj_offset: Offset, new_size: int, loop_allowed: bool = True, zero_mem: bool = True) -> Tuple[Optional[Offset], Size]: 2346 start_time: float = cpu_clock() 2347 internal_malloc_time: float = 0.0 2348 try: 2349 new_size += bs * len(BaseObjOffsets) 2350 new_size = nearest_size(new_size) 2351 result_offset: Offset = None 2352 result_obj_size: Size = 0 2353 original_obj_size = read_uint64(self.base_address, obj_offset + bs * BaseObjOffsets.obj_size) 2354 size = original_obj_size + bs * len(BaseObjOffsets) 2355 next_obj_offset = obj_offset + size 2356 free_mem_block_offset = next_obj_offset 2357 dsize = new_size - size 2358 found, additional_adjusted_size, last_free_block_offset, last_free_block_new_size, next_block_offset = self.test_free_memory_blocks(free_mem_block_offset, dsize) 2359 if found: 2360 self.combine_free_memory_blocks(free_mem_block_offset, additional_adjusted_size, last_free_block_offset, last_free_block_new_size, next_block_offset) 2361 if zero_mem: 2362 # print(f'Zeroing memory 3: {free_mem_block_offset=}, {result_size=}') 2363 # hps_sleep(0.01) 2364 zero_memory(self.base_address, free_mem_block_offset, dsize) 2365 2366 result_obj_size = new_size - bs * len(BaseObjOffsets) 2367 write_uint64(self.base_address, obj_offset + bs * BaseObjOffsets.obj_size, result_obj_size) 2368 self.set_free_memory_search_start(obj_offset) 2369 result_offset = obj_offset 2370 else: 2371 internal_malloc_start_time: float = cpu_clock() 2372 new_offset, result_obj_size = self.malloc(ObjectType(read_uint64(self.base_address, obj_offset + bs * BaseObjOffsets.obj_type)), new_size, loop_allowed) 2373 internal_malloc_time += cpu_clock() - internal_malloc_start_time 2374 if new_offset is None: 2375 return None, 0 2376 2377 self._shared_memory.buf[new_offset + bs * len(BaseObjOffsets):new_offset + bs * len(BaseObjOffsets) + size] = self._shared_memory.buf[obj_offset + bs * len(BaseObjOffsets):obj_offset + bs * len(BaseObjOffsets) + size] 2378 if zero_mem: 2379 # print(f'Zeroing memory 4: {new_offset=}, {new_size=}') 2380 # hps_sleep(0.01) 2381 zero_memory(self.base_address, new_offset + bs * len(BaseObjOffsets) + original_obj_size, result_obj_size - original_obj_size) 2382 2383 self.free(obj_offset) 2384 result_offset = new_offset 2385 2386 return result_offset, result_obj_size 2387 finally: 2388 self._realloc_time += cpu_clock() - start_time - internal_malloc_time 2389 2390 def free(self, offset: Offset) -> bool: 2391 write_uint64(self.base_address, offset, ObjectType.tfree_memory.value) 2392 return True 2393 2394 # ---------------------------- 2395 2396 def put_obj(self, obj: Any): 2397 obj_type = self._get_obj_type(obj) 2398 codec = codec_by_type[obj_type] 2399 mapped_obj, offset, size = codec.map_to_shared_memory(self, obj) 2400 return mapped_obj, offset, size 2401 2402 def get_obj(self, offset: int) -> Any: 2403 # print(f'get_obj: {offset=}') 2404 obj_type = ObjectType(read_uint64(self.base_address, offset)) 2405 if obj_type is ObjectType.tfree_memory: 2406 # self.print_mem(offset - 32, 96, 'get_obj [offset - 32: offset + 64]. {}') 2407 raise RuntimeError 2408 2409 codec = codec_by_type[obj_type] 2410 return codec.init_from_shared_memory(self, offset) 2411 2412 def get_obj_buffer(self, offset: int) -> memoryview: 2413 # print(f'get_obj: {offset=}') 2414 obj_type = ObjectType(read_uint64(self.base_address, offset)) 2415 if obj_type is ObjectType.tfree_memory: 2416 # self.print_mem(offset - 32, 96, 'get_obj [offset - 32: offset + 64]. {}') 2417 raise RuntimeError 2418 2419 codec = codec_by_type[obj_type] 2420 return codec.buffer(self, offset) 2421 2422 def get_obj_buffer_2(self, offset: int) -> Tuple[int, int]: 2423 # print(f'get_obj: {offset=}') 2424 obj_type = ObjectType(read_uint64(self.base_address, offset)) 2425 if obj_type is ObjectType.tfree_memory: 2426 # self.print_mem(offset - 32, 96, 'get_obj [offset - 32: offset + 64]. {}') 2427 raise RuntimeError 2428 2429 codec = codec_by_type[obj_type] 2430 return codec.buffer_2(self, offset) 2431 2432 def destroy_obj(self, offset: int): 2433 obj_type = ObjectType(read_uint64(self.base_address, offset)) 2434 codec = codec_by_type[obj_type] 2435 return codec.destroy(self, offset) 2436 2437 # ---------------------------- 2438 2439 def map_object(self, obj: Any) -> Any: 2440 # self.update_free_memory_search_start() 2441 mapped_obj, offset, size = self.put_obj(obj) 2442 # self.commit_free_memory_search_start() 2443 return mapped_obj 2444 2445 def get_object(self, offset: Offset) -> Any: 2446 return self.get_obj(offset) 2447 2448 def destroy_object(self, offset: Offset) -> Any: 2449 return self.destroy_obj(offset) 2450 2451 # ---------------------------- 2452 2453 def write_message(self, obj: Any): 2454 # self.update_free_memory_search_start() 2455 message_offset, message_real_size = self.malloc(ObjectType.tmessage, bs * len(MessageOffsets)) 2456 try: 2457 mapped_obj, offset, size = self.put_obj(obj) 2458 # self.commit_free_memory_search_start() 2459 last_message_offset: Offset = self.get_last_message_offset() 2460 if last_message_offset: 2461 write_uint64(self.base_address, last_message_offset + bs * len(BaseObjOffsets) + bs * MessageOffsets.next_message_offset, message_offset) 2462 else: 2463 self.set_first_message_offset(message_offset) 2464 2465 write_uint64(self.base_address, message_offset + bs * len(BaseObjOffsets) + bs * MessageOffsets.previous_message_offset, last_message_offset) 2466 write_uint64(self.base_address, message_offset + bs * len(BaseObjOffsets) + bs * MessageOffsets.next_message_offset, 0) 2467 write_uint64(self.base_address, message_offset + bs * len(BaseObjOffsets) + bs * MessageOffsets.item_offset, offset) 2468 self.set_last_message_offset(message_offset) 2469 except: 2470 self.free(message_offset) 2471 raise 2472 2473 return mapped_obj, offset, message_offset 2474 2475 def put_message(self, obj: Any): 2476 mapped_obj, offset, message_offset = self.write_message(obj) 2477 return mapped_obj 2478 2479 def put_message_2(self, obj: Any): 2480 mapped_obj, offset, message_offset = self.write_message(obj) 2481 return mapped_obj, offset 2482 2483 def has_messages(self) -> bool: 2484 return self.get_last_message_offset() != 0 2485 2486 def read_message_info(self, queue_type: QueueType = QueueType.fifo) -> Tuple[Any, Optional[Offset], Optional[Offset]]: 2487 # print(0) 2488 if QueueType.fifo == queue_type: 2489 message_offset = self.get_first_message_offset() 2490 # print(f'0.0| {message_offset=}') 2491 if not message_offset: 2492 return None, None, None 2493 2494 next_message_offset = read_uint64(self.base_address, message_offset + bs * len(BaseObjOffsets) + bs * MessageOffsets.next_message_offset) 2495 self.set_first_message_offset(next_message_offset) 2496 if next_message_offset: 2497 write_uint64(self.base_address, next_message_offset + bs * len(BaseObjOffsets) + bs * MessageOffsets.previous_message_offset, 0) 2498 else: 2499 self.set_last_message_offset(0) 2500 else: 2501 message_offset = self.get_last_message_offset() 2502 # print(f'0.1| {message_offset=}') 2503 if not message_offset: 2504 return None, None, None 2505 2506 prev_message_offset = read_uint64(self.base_address, message_offset + bs * len(BaseObjOffsets) + bs * MessageOffsets.previous_message_offset) 2507 self.set_last_message_offset(prev_message_offset) 2508 if prev_message_offset: 2509 write_uint64(self.base_address, prev_message_offset + bs * len(BaseObjOffsets) + bs * MessageOffsets.next_message_offset, 0) 2510 else: 2511 self.set_first_message_offset(0) 2512 2513 # print(1) 2514 obj_offset = read_uint64(self.base_address, message_offset + bs * len(BaseObjOffsets) + bs * MessageOffsets.item_offset) 2515 # print(2) 2516 if not obj_offset: 2517 return None, None, message_offset 2518 2519 # print(3) 2520 obj = self.get_obj(obj_offset) 2521 # print(4) 2522 return obj, obj_offset, message_offset 2523 2524 def destroy_message(self, message_offset: Offset): 2525 if not message_offset: 2526 return 2527 2528 # obj_offset = read_uint64(self.base_address, message_offset + bs * len(BaseObjOffsets) + bs * MessageOffsets.item_offset) 2529 # if obj_offset: 2530 # self.destroy_obj(obj_offset) 2531 2532 # self.destroy_obj(message_offset) 2533 2534 self.free(message_offset) 2535 2536 def read_message(self, queue_type: QueueType = QueueType.fifo) -> Any: 2537 obj, obj_offset, message_offset = self.read_message_info(queue_type) 2538 if message_offset: 2539 return obj 2540 else: 2541 raise NoMessagesInQueueError 2542 2543 def read_message_2(self, queue_type: QueueType = QueueType.fifo) -> Tuple[Any, int]: 2544 obj, obj_offset, message_offset = self.read_message_info(queue_type) 2545 if message_offset: 2546 return obj, obj_offset 2547 else: 2548 raise NoMessagesInQueueError 2549 2550 def take_message(self, queue_type: QueueType = QueueType.fifo) -> Any: 2551 obj, obj_offset, message_offset = self.read_message_info(queue_type) 2552 if message_offset: 2553 self.destroy_message(message_offset) 2554 else: 2555 raise NoMessagesInQueueError 2556 2557 return obj 2558 2559 def take_message_2(self, queue_type: QueueType = QueueType.fifo) -> Tuple[Any, int]: 2560 obj, obj_offset, message_offset = self.read_message_info(queue_type) 2561 if message_offset: 2562 self.destroy_message(message_offset) 2563 else: 2564 raise NoMessagesInQueueError 2565 2566 return obj, obj_offset 2567 2568 def get_message(self, default = None, queue_type: QueueType = QueueType.fifo) -> Any: 2569 obj, obj_offset, message_offset = self.read_message_info(queue_type) 2570 if message_offset: 2571 return obj 2572 else: 2573 return default 2574 2575 def get_message_2(self, default = None, queue_type: QueueType = QueueType.fifo) -> Tuple[Any, Optional[int]]: 2576 obj, obj_offset, message_offset = self.read_message_info(queue_type) 2577 if message_offset: 2578 return obj, obj_offset 2579 else: 2580 return default, None 2581 2582 def pop_message(self, default = None, queue_type: QueueType = QueueType.fifo) -> Any: 2583 obj, obj_offset, message_offset = self.read_message_info(queue_type) 2584 if message_offset: 2585 self.destroy_message(message_offset) 2586 else: 2587 obj = default 2588 2589 return obj 2590 2591 def pop_message_2(self, default = None, queue_type: QueueType = QueueType.fifo) -> Tuple[Any, Optional[int]]: 2592 obj, obj_offset, message_offset = self.read_message_info(queue_type) 2593 if message_offset: 2594 self.destroy_message(message_offset) 2595 else: 2596 obj = default 2597 obj_offset = None 2598 2599 return obj, obj_offset 2600 2601 # ---------------------------- 2602 2603 def get_in_line(self) -> bool: 2604 if self._create: 2605 write_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.creator_in_charge, 0) 2606 write_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.creator_wants_to_be_in_charge, 1) 2607 full_memory_barrier() 2608 if self.consumer_in_charge(): 2609 return False 2610 else: 2611 write_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.creator_in_charge, 1) 2612 full_memory_barrier() 2613 write_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.creator_wants_to_be_in_charge, 0) 2614 full_memory_barrier() 2615 self.update_free_memory_search_start() 2616 if self.consumer_in_charge(): 2617 write_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.creator_in_charge, 0) 2618 full_memory_barrier() 2619 write_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.creator_wants_to_be_in_charge, 1) 2620 full_memory_barrier() 2621 return False 2622 2623 return True 2624 else: 2625 write_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.consumer_in_charge, 0) 2626 write_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.consumer_wants_to_be_in_charge, 1) 2627 full_memory_barrier() 2628 if self.creator_in_charge(): 2629 return False 2630 else: 2631 write_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.consumer_in_charge, 1) 2632 full_memory_barrier() 2633 write_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.consumer_wants_to_be_in_charge, 0) 2634 full_memory_barrier() 2635 self.update_free_memory_search_start() 2636 if self.creator_in_charge(): 2637 write_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.consumer_in_charge, 0) 2638 full_memory_barrier() 2639 write_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.consumer_wants_to_be_in_charge, 1) 2640 full_memory_barrier() 2641 return False 2642 2643 return True 2644 2645 def release(self): 2646 self.commit_free_memory_search_start() 2647 if self._create: 2648 write_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.creator_in_charge, 0) 2649 write_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.creator_wants_to_be_in_charge, 0) 2650 full_memory_barrier() 2651 else: 2652 write_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.consumer_in_charge, 0) 2653 write_uint64(self.base_address, self.sys_values_offset + bs * SysValuesOffsets.consumer_wants_to_be_in_charge, 0) 2654 full_memory_barrier() 2655 2656 def wait_my_turn(self, time_limit: Optional[RationalNumber] = None, periodic_sleep_time: Optional[RationalNumber] = 0.000000001) -> bool: 2657 start_time = cpu_clock() 2658 while not self.get_in_line(): 2659 if time_limit is not None: 2660 if (cpu_clock() - start_time) > time_limit: 2661 return False 2662 2663 if periodic_sleep_time is None: 2664 mm_pause() 2665 else: 2666 hps_sleep(periodic_sleep_time) 2667 2668 return True 2669 2670 # ---------------------------- 2671 2672 def wait_for_messages(self, time_limit: Optional[RationalNumber] = None, periodic_sleep_time: Optional[RationalNumber] = 0.000000001) -> bool: 2673 start_time = cpu_clock() 2674 has_messages = False 2675 while not has_messages: 2676 if time_limit is not None: 2677 if (cpu_clock() - start_time) > time_limit: 2678 return False 2679 2680 if periodic_sleep_time is None: 2681 mm_pause() 2682 else: 2683 hps_sleep(periodic_sleep_time) 2684 2685 with wait_my_turn(self): 2686 has_messages = self.has_messages() 2687 2688 return True 2689 2690 # ---------------------------- 2691 2692 @staticmethod 2693 def _get_obj_type(obj: Any) -> ObjectType: 2694 obj_type = type(obj) 2695 if obj is None: 2696 obj_type_atom: ObjectType = ObjectType.tnone 2697 elif obj_type is bool: 2698 obj_type_atom = ObjectType.tbool 2699 elif obj_type is int: 2700 obj_type_atom = ObjectType.tint 2701 elif obj_type is float: 2702 obj_type_atom = ObjectType.tfloat 2703 elif obj_type is complex: 2704 obj_type_atom = ObjectType.tcomplex 2705 elif obj_type is str: 2706 obj_type_atom = ObjectType.tstr 2707 elif obj_type is bytes: 2708 obj_type_atom = ObjectType.tbytes 2709 elif obj_type is bytearray: 2710 obj_type_atom = ObjectType.tbytearray 2711 elif obj_type is tuple: 2712 obj_type_atom = ObjectType.ttuple 2713 elif obj_type is list: 2714 obj_type_atom = ObjectType.tlist 2715 elif obj_type is dict: 2716 obj_type_atom = ObjectType.tdict 2717 elif obj_type is set: 2718 obj_type_atom = ObjectType.tset 2719 elif obj_type in obj_type_map: 2720 obj_type_atom = obj_type_map[obj_type] 2721 else: 2722 obj_type_atom = ObjectType.tpickable 2723 2724 return obj_type_atom 2725 2726 2727@contextmanager 2728def get_in_line(shared_memory: SharedMemory): 2729 shared_memory.get_in_line() 2730 try: 2731 yield 2732 finally: 2733 shared_memory.release() 2734 2735 2736@contextmanager 2737def wait_my_turn(shared_memory: SharedMemory, time_limit: Optional[RationalNumber] = None, periodic_sleep_time: Optional[RationalNumber] = 0.000000001): 2738 shared_memory.wait_my_turn(time_limit, periodic_sleep_time) 2739 try: 2740 yield 2741 finally: 2742 shared_memory.release() 2743 2744 2745def numpy_array_memory_size(np_shape, np_dtype): 2746 num_elements = np.prod(np_shape) 2747 element_size = np.dtype(np_dtype).itemsize 2748 memory_size_bytes = num_elements * element_size 2749 return memory_size_bytes 2750 2751 2752def numpy_array_made_from_pointer_memory_size(np_shape, ctypes_type): 2753 num_elements = np.prod(np_shape) 2754 element_size = ctypes.sizeof(ctypes_type) 2755 memory_size_bytes = num_elements * element_size 2756 return memory_size_bytes 2757 2758 2759def make_numpy_array_from_obj_offset(shared_memory: SharedMemory, offset: int, np_shape, ctypes_type = None) -> Any: 2760 if ctypes_type is None: 2761 ctypes_type = ctypes.c_uint8 2762 2763 data_offset, data_size = shared_memory.get_obj_buffer_2(offset) 2764 num_elements = np.prod(np_shape) 2765 np_array_size = num_elements * ctypes.sizeof(ctypes_type) 2766 if data_size < np_array_size: 2767 raise ObjBufferIsSmallerThanRequestedNumpyArrayError(data_size, np_array_size) 2768 2769 data_address = shared_memory.base_address + data_offset 2770 void_ptr = ctypes.c_void_p(data_address) 2771 # actual_ptr = ctypes.cast(void_ptr, ctypes.POINTER(ctypes_type * num_elements)) 2772 actual_ptr = ctypes.cast(void_ptr, ctypes.POINTER(ctypes_type)) 2773 return np.ctypeslib.as_array(actual_ptr, shape=np_shape)
class
QueueType(enum.IntEnum):
An enumeration.
fifo =
<QueueType.fifo: 0>
lifo =
<QueueType.lifo: 1>
Inherited Members
- enum.Enum
- name
- value
- builtins.int
- conjugate
- bit_length
- to_bytes
- from_bytes
- as_integer_ratio
- real
- imag
- numerator
- denominator
class
ObjectType(enum.IntEnum):
94class ObjectType(IntEnum): 95 tfree_memory = 0 96 tmessage = 1 97 tnone = 2 98 tbool = 3 99 tint = 4 100 tfloat = 5 101 tcomplex = 6 102 tstr = 7 103 tbytes = 8 104 tbytearray = 9 105 ttuple = 10 106 tlist = 11 107 tdict = 12 108 tset = 13 109 tclass = 14 110 tpickable = 15 111 tinternal_list = 16
An enumeration.
tfree_memory =
<ObjectType.tfree_memory: 0>
tmessage =
<ObjectType.tmessage: 1>
tnone =
<ObjectType.tnone: 2>
tbool =
<ObjectType.tbool: 3>
tint =
<ObjectType.tint: 4>
tfloat =
<ObjectType.tfloat: 5>
tcomplex =
<ObjectType.tcomplex: 6>
tstr =
<ObjectType.tstr: 7>
tbytes =
<ObjectType.tbytes: 8>
tbytearray =
<ObjectType.tbytearray: 9>
ttuple =
<ObjectType.ttuple: 10>
tlist =
<ObjectType.tlist: 11>
tdict =
<ObjectType.tdict: 12>
tset =
<ObjectType.tset: 13>
tclass =
<ObjectType.tclass: 14>
tpickable =
<ObjectType.tpickable: 15>
tinternal_list =
<ObjectType.tinternal_list: 16>
Inherited Members
- enum.Enum
- name
- value
- builtins.int
- conjugate
- bit_length
- to_bytes
- from_bytes
- as_integer_ratio
- real
- imag
- numerator
- denominator
class
SysValuesOffsets(enum.IntEnum):
114class SysValuesOffsets(IntEnum): 115 data_start_offset = 0 116 data_size = 1 117 data_end_offset = 2 118 free_memory_search_start = 3 119 first_message_offset = 4 120 last_message_offset = 5 121 creator_in_charge = 6 122 consumer_in_charge = 7 123 creator_wants_to_be_in_charge = 8 124 consumer_wants_to_be_in_charge = 9 125 creator_ready = 10 126 consumer_ready = 11
An enumeration.
data_start_offset =
<SysValuesOffsets.data_start_offset: 0>
data_size =
<SysValuesOffsets.data_size: 1>
data_end_offset =
<SysValuesOffsets.data_end_offset: 2>
free_memory_search_start =
<SysValuesOffsets.free_memory_search_start: 3>
first_message_offset =
<SysValuesOffsets.first_message_offset: 4>
last_message_offset =
<SysValuesOffsets.last_message_offset: 5>
creator_in_charge =
<SysValuesOffsets.creator_in_charge: 6>
consumer_in_charge =
<SysValuesOffsets.consumer_in_charge: 7>
creator_wants_to_be_in_charge =
<SysValuesOffsets.creator_wants_to_be_in_charge: 8>
consumer_wants_to_be_in_charge =
<SysValuesOffsets.consumer_wants_to_be_in_charge: 9>
creator_ready =
<SysValuesOffsets.creator_ready: 10>
consumer_ready =
<SysValuesOffsets.consumer_ready: 11>
Inherited Members
- enum.Enum
- name
- value
- builtins.int
- conjugate
- bit_length
- to_bytes
- from_bytes
- as_integer_ratio
- real
- imag
- numerator
- denominator
Offset =
<class 'int'>
Size =
<class 'int'>
minimal_memory_block_size =
8
block_size =
8
bs =
8
Common base class for all non-exit exceptions.
Inherited Members
- builtins.Exception
- Exception
- builtins.BaseException
- with_traceback
- args
Common base class for all non-exit exceptions.
Inherited Members
- builtins.Exception
- Exception
- builtins.BaseException
- with_traceback
- args
Common base class for all non-exit exceptions.
Inherited Members
- builtins.Exception
- Exception
- builtins.BaseException
- with_traceback
- args
Common base class for all non-exit exceptions.
Inherited Members
- builtins.Exception
- Exception
- builtins.BaseException
- with_traceback
- args
def
nearest_size(size: int) -> int:
def
nsize(size: int) -> int:
class
BaseIObject:
class
BaseObjOffsets(enum.IntEnum):
An enumeration.
obj_type =
<BaseObjOffsets.obj_type: 0>
obj_size =
<BaseObjOffsets.obj_size: 1>
Inherited Members
- enum.Enum
- name
- value
- builtins.int
- conjugate
- bit_length
- to_bytes
- from_bytes
- as_integer_ratio
- real
- imag
- numerator
- denominator
class
TBase:
173class TBase: 174 def map_to_shared_memory(self, shared_memory: 'SharedMemory', obj: Any) -> Tuple[Any, Offset, Size]: 175 raise NotImplementedError 176 177 def init_from_shared_memory(self, shared_memory: 'SharedMemory', offset: Offset) -> Any: 178 raise NotImplementedError 179 180 def destroy(self, shared_memory: 'SharedMemory', offset: Offset): 181 raise NotImplementedError 182 183 def buffer(self, shared_memory: 'SharedMemory', offset: Offset) -> memoryview: 184 raise NotImplementedError 185 186 def buffer_2(self, shared_memory: 'SharedMemory', offset: Offset) -> Tuple[int, int]: 187 raise NotImplementedError
class
TNone:
194class TNone: 195 def map_to_shared_memory(self, shared_memory: 'SharedMemory', obj: None) -> Tuple[None, Offset, Size]: 196 offset, real_size = shared_memory.malloc(ObjectType.tnone, 0) 197 return obj, offset, real_size 198 199 def init_from_shared_memory(self, shared_memory: 'SharedMemory', offset: Offset) -> None: 200 if ObjectType.tnone != read_uint64(shared_memory.base_address, offset): 201 raise WrongObjectTypeError 202 203 return None 204 205 def destroy(self, shared_memory: 'SharedMemory', offset: Offset): 206 shared_memory.free(offset)
class
IntOffsets(enum.IntEnum):
An enumeration.
data =
<IntOffsets.data: 0>
Inherited Members
- enum.Enum
- name
- value
- builtins.int
- conjugate
- bit_length
- to_bytes
- from_bytes
- as_integer_ratio
- real
- imag
- numerator
- denominator
class
TInt:
217class TInt: 218 def map_to_shared_memory(self, shared_memory: 'SharedMemory', obj: int) -> Tuple[int, Offset, Size]: 219 offset, real_size = shared_memory.malloc(ObjectType.tint, bs * len(IntOffsets)) 220 write_int64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * IntOffsets.data, obj) 221 return obj, offset, real_size 222 223 def init_from_shared_memory(self, shared_memory: 'SharedMemory', offset: Offset) -> int: 224 if ObjectType.tint != read_uint64(shared_memory.base_address, offset + bs * BaseObjOffsets.obj_type): 225 raise WrongObjectTypeError 226 227 return read_int64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * IntOffsets.data) 228 229 def destroy(self, shared_memory: 'SharedMemory', offset: Offset): 230 shared_memory.free(offset)
class
BoolOffsets(enum.IntEnum):
An enumeration.
data =
<BoolOffsets.data: 0>
Inherited Members
- enum.Enum
- name
- value
- builtins.int
- conjugate
- bit_length
- to_bytes
- from_bytes
- as_integer_ratio
- real
- imag
- numerator
- denominator
class
TBool:
241class TBool: 242 def map_to_shared_memory(self, shared_memory: 'SharedMemory', obj: bool) -> Tuple[bool, Offset, Size]: 243 offset, real_size = shared_memory.malloc(ObjectType.tbool, bs * len(BoolOffsets)) 244 write_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * BoolOffsets.data, int(obj)) 245 return obj, offset, real_size 246 247 def init_from_shared_memory(self, shared_memory: 'SharedMemory', offset: Offset) -> bool: 248 if ObjectType.tbool != read_uint64(shared_memory.base_address, offset + bs * BaseObjOffsets.obj_type): 249 raise WrongObjectTypeError 250 251 return bool(read_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * BoolOffsets.data)) 252 253 def destroy(self, shared_memory: 'SharedMemory', offset: Offset): 254 shared_memory.free(offset)
class
FloatOffsets(enum.IntEnum):
An enumeration.
data =
<FloatOffsets.data: 0>
Inherited Members
- enum.Enum
- name
- value
- builtins.int
- conjugate
- bit_length
- to_bytes
- from_bytes
- as_integer_ratio
- real
- imag
- numerator
- denominator
class
TFloat:
265class TFloat: 266 def map_to_shared_memory(self, shared_memory: 'SharedMemory', obj: float) -> Tuple[float, Offset, Size]: 267 offset, real_size = shared_memory.malloc(ObjectType.tfloat, bs * len(FloatOffsets)) 268 write_double(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * FloatOffsets.data, obj) 269 return obj, offset, real_size 270 271 def init_from_shared_memory(self, shared_memory: 'SharedMemory', offset: Offset) -> float: 272 if ObjectType.tfloat != read_uint64(shared_memory.base_address, offset): 273 raise WrongObjectTypeError 274 275 return read_double(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * FloatOffsets.data) 276 277 def destroy(self, shared_memory: 'SharedMemory', offset: Offset): 278 shared_memory.free(offset)
class
BytesOffsets(enum.IntEnum):
An enumeration.
data_size =
<BytesOffsets.data_size: 0>
data =
<BytesOffsets.data: 1>
Inherited Members
- enum.Enum
- name
- value
- builtins.int
- conjugate
- bit_length
- to_bytes
- from_bytes
- as_integer_ratio
- real
- imag
- numerator
- denominator
class
TBytes:
290class TBytes: 291 def map_to_shared_memory(self, shared_memory: 'SharedMemory', obj: bytes) -> Tuple[bytes, Offset, Size]: 292 data_size = len(obj) 293 # offset, real_size = shared_memory.malloc(ObjectType.tbytes, bs * len(BytesOffsets) + bs * data_size) 294 offset, real_size = shared_memory.malloc(ObjectType.tbytes, bs * len(BytesOffsets) + data_size) 295 write_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * BytesOffsets.data_size, data_size) 296 data_offset = offset + bs * len(BaseObjOffsets) + bs * BytesOffsets.data 297 shared_memory._shared_memory.buf[data_offset:data_offset + data_size] = obj 298 return obj, offset, real_size 299 300 def init_from_shared_memory(self, shared_memory: 'SharedMemory', offset: Offset) -> bytes: 301 if ObjectType.tbytes != read_uint64(shared_memory.base_address, offset + bs * BaseObjOffsets.obj_type): 302 raise WrongObjectTypeError 303 304 data_size = read_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * BytesOffsets.data_size) 305 if data_size: 306 data_offset = offset + bs * len(BaseObjOffsets) + bs * BytesOffsets.data 307 obj = bytes(shared_memory._shared_memory.buf[data_offset:data_offset + data_size]) 308 return obj 309 else: 310 return bytes() 311 312 def destroy(self, shared_memory: 'SharedMemory', offset: Offset): 313 shared_memory.free(offset) 314 315 def buffer(self, shared_memory: 'SharedMemory', offset: Offset) -> memoryview: 316 if ObjectType.tbytes != read_uint64(shared_memory.base_address, offset + bs * BaseObjOffsets.obj_type): 317 raise WrongObjectTypeError 318 319 data_size = read_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * BytesOffsets.data_size) 320 data_offset = offset + bs * len(BaseObjOffsets) + bs * BytesOffsets.data 321 return shared_memory._shared_memory.buf[data_offset:data_offset + data_size] 322 323 def buffer_2(self, shared_memory: 'SharedMemory', offset: Offset) -> Tuple[int, int]: 324 if ObjectType.tbytes != read_uint64(shared_memory.base_address, offset + bs * BaseObjOffsets.obj_type): 325 raise WrongObjectTypeError 326 327 data_size = read_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * BytesOffsets.data_size) 328 data_offset = offset + bs * len(BaseObjOffsets) + bs * BytesOffsets.data 329 return data_offset, data_size
315 def buffer(self, shared_memory: 'SharedMemory', offset: Offset) -> memoryview: 316 if ObjectType.tbytes != read_uint64(shared_memory.base_address, offset + bs * BaseObjOffsets.obj_type): 317 raise WrongObjectTypeError 318 319 data_size = read_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * BytesOffsets.data_size) 320 data_offset = offset + bs * len(BaseObjOffsets) + bs * BytesOffsets.data 321 return shared_memory._shared_memory.buf[data_offset:data_offset + data_size]
323 def buffer_2(self, shared_memory: 'SharedMemory', offset: Offset) -> Tuple[int, int]: 324 if ObjectType.tbytes != read_uint64(shared_memory.base_address, offset + bs * BaseObjOffsets.obj_type): 325 raise WrongObjectTypeError 326 327 data_size = read_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * BytesOffsets.data_size) 328 data_offset = offset + bs * len(BaseObjOffsets) + bs * BytesOffsets.data 329 return data_offset, data_size
class
BytearrayOffsets(enum.IntEnum):
An enumeration.
data_size =
<BytearrayOffsets.data_size: 0>
data =
<BytearrayOffsets.data: 1>
Inherited Members
- enum.Enum
- name
- value
- builtins.int
- conjugate
- bit_length
- to_bytes
- from_bytes
- as_integer_ratio
- real
- imag
- numerator
- denominator
class
TBytearray:
385class TBytearray: 386 def map_to_shared_memory(self, shared_memory: 'SharedMemory', obj: bytearray) -> Tuple[bytearray, Offset, Size]: 387 data = bytes(obj) 388 data_size = len(data) 389 # offset, real_size = shared_memory.malloc(ObjectType.tbytearray, bs * len(BytearrayOffsets) + bs * data_size) 390 offset, real_size = shared_memory.malloc(ObjectType.tbytearray, bs * len(BytearrayOffsets) + data_size) 391 write_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * BytearrayOffsets.data_size, data_size) 392 data_offset = offset + bs * len(BaseObjOffsets) + bs * BytearrayOffsets.data 393 shared_memory._shared_memory.buf[data_offset:data_offset + data_size] = data 394 return obj, offset, real_size 395 396 def init_from_shared_memory(self, shared_memory: 'SharedMemory', offset: Offset) -> bytearray: 397 if ObjectType.tbytearray != read_uint64(shared_memory.base_address, offset + bs * BaseObjOffsets.obj_type): 398 raise WrongObjectTypeError 399 400 data_size = read_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * BytearrayOffsets.data_size) 401 if data_size: 402 data_offset = offset + bs * len(BaseObjOffsets) + bs * BytearrayOffsets.data 403 data = bytes(shared_memory._shared_memory.buf[data_offset:data_offset + data_size]) 404 return bytearray(data) 405 else: 406 return bytearray(bytes()) 407 408 def destroy(self, shared_memory: 'SharedMemory', offset: Offset): 409 shared_memory.free(offset) 410 411 def buffer(self, shared_memory: 'SharedMemory', offset: Offset) -> memoryview: 412 if ObjectType.tbytearray != read_uint64(shared_memory.base_address, offset + bs * BaseObjOffsets.obj_type): 413 raise WrongObjectTypeError 414 415 data_size = read_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * BytearrayOffsets.data_size) 416 data_offset = offset + bs * len(BaseObjOffsets) + bs * BytearrayOffsets.data 417 return shared_memory._shared_memory.buf[data_offset:data_offset + data_size] 418 419 def buffer_2(self, shared_memory: 'SharedMemory', offset: Offset) -> Tuple[int, int]: 420 if ObjectType.tbytearray != read_uint64(shared_memory.base_address, offset + bs * BaseObjOffsets.obj_type): 421 raise WrongObjectTypeError 422 423 data_size = read_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * BytearrayOffsets.data_size) 424 data_offset = offset + bs * len(BaseObjOffsets) + bs * BytearrayOffsets.data 425 return data_offset, data_size
411 def buffer(self, shared_memory: 'SharedMemory', offset: Offset) -> memoryview: 412 if ObjectType.tbytearray != read_uint64(shared_memory.base_address, offset + bs * BaseObjOffsets.obj_type): 413 raise WrongObjectTypeError 414 415 data_size = read_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * BytearrayOffsets.data_size) 416 data_offset = offset + bs * len(BaseObjOffsets) + bs * BytearrayOffsets.data 417 return shared_memory._shared_memory.buf[data_offset:data_offset + data_size]
419 def buffer_2(self, shared_memory: 'SharedMemory', offset: Offset) -> Tuple[int, int]: 420 if ObjectType.tbytearray != read_uint64(shared_memory.base_address, offset + bs * BaseObjOffsets.obj_type): 421 raise WrongObjectTypeError 422 423 data_size = read_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * BytearrayOffsets.data_size) 424 data_offset = offset + bs * len(BaseObjOffsets) + bs * BytearrayOffsets.data 425 return data_offset, data_size
class
StrOffsets(enum.IntEnum):
An enumeration.
data_size =
<StrOffsets.data_size: 0>
data =
<StrOffsets.data: 1>
Inherited Members
- enum.Enum
- name
- value
- builtins.int
- conjugate
- bit_length
- to_bytes
- from_bytes
- as_integer_ratio
- real
- imag
- numerator
- denominator
class
TStr:
437class TStr: 438 def map_to_shared_memory(self, shared_memory: 'SharedMemory', obj: str) -> Tuple[str, Offset, Size]: 439 data = str.encode(obj) 440 data_size = len(data) 441 # offset, real_size = shared_memory.malloc(ObjectType.tstr, bs * len(StrOffsets) + bs * data_size) 442 offset, real_size = shared_memory.malloc(ObjectType.tstr, bs * len(StrOffsets) + data_size) 443 write_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * StrOffsets.data_size, data_size) 444 data_offset = offset + bs * len(BaseObjOffsets) + bs * StrOffsets.data 445 shared_memory._shared_memory.buf[data_offset:data_offset + data_size] = data 446 return obj, offset, real_size 447 448 def init_from_shared_memory(self, shared_memory: 'SharedMemory', offset: Offset) -> str: 449 if ObjectType.tstr != read_uint64(shared_memory.base_address, offset + bs * BaseObjOffsets.obj_type): 450 raise WrongObjectTypeError 451 452 data_size = read_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * StrOffsets.data_size) 453 if data_size: 454 data_offset = offset + bs * len(BaseObjOffsets) + bs * StrOffsets.data 455 data = bytes(shared_memory._shared_memory.buf[data_offset:data_offset + data_size]) 456 return data.decode() 457 else: 458 return str() 459 460 def destroy(self, shared_memory: 'SharedMemory', offset: Offset): 461 shared_memory.free(offset) 462 463 def buffer(self, shared_memory: 'SharedMemory', offset: Offset) -> memoryview: 464 if ObjectType.tstr != read_uint64(shared_memory.base_address, offset + bs * BaseObjOffsets.obj_type): 465 raise WrongObjectTypeError 466 467 data_size = read_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * StrOffsets.data_size) 468 data_offset = offset + bs * len(BaseObjOffsets) + bs * StrOffsets.data 469 return shared_memory._shared_memory.buf[data_offset:data_offset + data_size] 470 471 def buffer_2(self, shared_memory: 'SharedMemory', offset: Offset) -> Tuple[int, int]: 472 if ObjectType.tstr != read_uint64(shared_memory.base_address, offset + bs * BaseObjOffsets.obj_type): 473 raise WrongObjectTypeError 474 475 data_size = read_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * StrOffsets.data_size) 476 data_offset = offset + bs * len(BaseObjOffsets) + bs * StrOffsets.data 477 return data_offset, data_size
463 def buffer(self, shared_memory: 'SharedMemory', offset: Offset) -> memoryview: 464 if ObjectType.tstr != read_uint64(shared_memory.base_address, offset + bs * BaseObjOffsets.obj_type): 465 raise WrongObjectTypeError 466 467 data_size = read_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * StrOffsets.data_size) 468 data_offset = offset + bs * len(BaseObjOffsets) + bs * StrOffsets.data 469 return shared_memory._shared_memory.buf[data_offset:data_offset + data_size]
471 def buffer_2(self, shared_memory: 'SharedMemory', offset: Offset) -> Tuple[int, int]: 472 if ObjectType.tstr != read_uint64(shared_memory.base_address, offset + bs * BaseObjOffsets.obj_type): 473 raise WrongObjectTypeError 474 475 data_size = read_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * StrOffsets.data_size) 476 data_offset = offset + bs * len(BaseObjOffsets) + bs * StrOffsets.data 477 return data_offset, data_size
class
InternalListTrueOffsets(enum.IntEnum):
An enumeration.
capacity =
<InternalListTrueOffsets.capacity: 0>
size =
<InternalListTrueOffsets.size: 1>
Inherited Members
- enum.Enum
- name
- value
- builtins.int
- conjugate
- bit_length
- to_bytes
- from_bytes
- as_integer_ratio
- real
- imag
- numerator
- denominator
def
malloc_tinternal_list_true( shared_memory: SharedMemory, size: int, capacity: int = None) -> Tuple[int, int]:
489def malloc_tinternal_list_true(shared_memory: 'SharedMemory', size: Size, capacity: Size = None) -> Tuple[Offset, Size]: 490 capacity = (size << 1 if size else 16) if capacity is None else capacity 491 datas_sys_part_size = 8 * len(InternalListTrueOffsets) 492 offset, real_size = shared_memory.malloc(ObjectType.tinternal_list, datas_sys_part_size + 8 * capacity) 493 data_offset = offset + datas_sys_part_size 494 write_uint64(shared_memory.base_address, data_offset + 8 * InternalListTrueOffsets.capacity, capacity) 495 write_uint64(shared_memory.base_address, data_offset + 8 * InternalListTrueOffsets.size, size) 496 return offset, real_size
def
realloc_tinternal_list_true( shared_memory: SharedMemory, offset: int, desired_size: int = None, new_capacity: int = None, loop_allowed: bool = True, zero_mem: bool = True) -> Tuple[int, int]:
499def realloc_tinternal_list_true(shared_memory: 'SharedMemory', offset: Offset, desired_size: int = None, new_capacity: int = None, loop_allowed: bool = True, zero_mem: bool = True) -> Tuple[Offset, Size]: 500 datas_sys_part_size = 8 * len(InternalListTrueOffsets) 501 data_offset = offset + datas_sys_part_size 502 capacity = read_uint64(shared_memory.base_address, data_offset + 8 * InternalListTrueOffsets.capacity) 503 size = read_uint64(shared_memory.base_address, data_offset + 8 * InternalListTrueOffsets.size) 504 new_list_capacity = capacity << 1 if new_capacity is None else new_capacity 505 if new_capacity is None: 506 if desired_size is None: 507 new_list_capacity = capacity << 1 if capacity else 16 508 else: 509 new_list_capacity = desired_size << 1 if desired_size else 16 510 else: 511 new_list_capacity = new_capacity 512 513 if new_list_capacity < size: 514 new_list_capacity = size 515 516 new_offset, new_real_size = shared_memory.realloc(offset, datas_sys_part_size + 8 * new_list_capacity, loop_allowed, zero_mem) 517 data_offset = new_offset + datas_sys_part_size 518 write_uint64(shared_memory.base_address, data_offset + 8 * InternalListTrueOffsets.capacity, new_list_capacity) 519 return new_offset, new_real_size
522class IListTrue(BaseIObject, list): 523 def __init__(self, shared_memory: 'SharedMemory', offset: Offset = None, obj: List = None) -> None: 524 self._shared_memory = shared_memory 525 self._base_address = shared_memory.base_address 526 if offset is None: 527 offset, real_size = shared_memory.malloc(ObjectType.tlist, 8) 528 self._offset = offset 529 self._offset__data = offset + 8 * len(BaseObjOffsets) 530 self._offset__pointer_to_internal_list = self._offset__data 531 532 if obj is None: 533 obj = list() 534 535 data_len = len(obj) 536 capacity_len = data_len << 1 if data_len else 16 537 internal_list_offset, data_tuple_real_size = malloc_tinternal_list(shared_memory, data_len, capacity_len) 538 self._pointer_to_internal_list = internal_list_offset 539 for i, item in enumerate(obj): 540 item_mapped_obj, item_offset, item_size = shared_memory.put_obj(item) 541 write_uint64(self._base_address, self._item_offset(i), item_offset) 542 else: 543 self._offset = offset 544 self._offset__data = offset + 8 * len(BaseObjOffsets) 545 self._offset__pointer_to_internal_list = self._offset__data 546 547 def raw_to_bytes(self, bytes_num: int) -> bytes: 548 start_index = self._pointer_to_internal_list 549 return self._shared_memory.read_mem(start_index, bytes_num) 550 # return bytes(self._shared_memory._shared_memory.buf[start_index : start_index + bytes_num]) 551 552 @property 553 def _obj_size(self): 554 return read_uint64(self._base_address, self._offset + 8 * BaseObjOffsets.obj_size) 555 556 @property 557 def _pointer_to_internal_list(self): 558 return read_uint64(self._base_address, self._offset__pointer_to_internal_list) 559 560 @_pointer_to_internal_list.setter 561 def _pointer_to_internal_list(self, value: Offset): 562 write_uint64(self._base_address, self._offset__pointer_to_internal_list, value) 563 564 @property 565 def _list_len(self): 566 return read_uint64(self._base_address, self._pointer_to_internal_list + 8 * len(BaseObjOffsets) + 8 * InternalListTrueOffsets.size) 567 568 @_list_len.setter 569 def _list_len(self, value: int): 570 write_uint64(self._base_address, self._pointer_to_internal_list + 8 * len(BaseObjOffsets) + 8 * InternalListTrueOffsets.size, value) 571 572 @property 573 def _list_capacity(self): 574 return read_uint64(self._base_address, self._pointer_to_internal_list + 8 * len(BaseObjOffsets) + 8 * InternalListTrueOffsets.capacity) 575 576 def _item_offset(self, key: int) -> Offset: 577 return self._pointer_to_internal_list + 8 * len(BaseObjOffsets) + 8 * len(InternalListTrueOffsets) + key * 8 578 579 def __len__(self) -> int: 580 return self._list_len 581 582 def get_children_offsets(self) -> List[Offset]: 583 return [read_uint64(self._base_address, self._item_offset(i)) for i in range(self._list_len)] 584 585 def __getitem__(self, key: Union[int, slice]) -> Union[Any, List]: 586 if isinstance(key, int): 587 if key < 0: 588 key += len(self) 589 if key < 0 or key >= len(self): 590 raise IndexError 591 592 item_offset = read_uint64(self._base_address, self._item_offset(key)) 593 return self._shared_memory.get_obj(item_offset) 594 elif isinstance(key, slice): 595 if key.step is not None: 596 raise NotImplementedError 597 598 if key.start is None: 599 start = 0 600 elif key.start < 0: 601 start = key.start + len(self) 602 else: 603 start = key.start 604 605 if key.stop is None: 606 stop = len(self) 607 elif key.stop < 0: 608 stop = key.stop + len(self) 609 else: 610 stop = key.stop 611 612 if start < 0 or start >= len(self) or stop < 0 or stop > len(self) or start >= stop: 613 raise IndexError 614 615 result_list = list() 616 for i in range(start, stop): 617 item_offset = read_uint64(self._base_address, self._item_offset(i)) 618 result_list.append(self._shared_memory.get_obj(item_offset)) 619 return result_list 620 else: 621 raise TypeError 622 623 def __setitem__(self, key: Union[int, slice], value: Union[Any, Sequence]) -> Any: 624 if isinstance(key, int): 625 if key < 0: 626 key += len(self) 627 if key < 0 or key >= len(self): 628 raise IndexError 629 630 item_mapped_obj, item_offset, item_size = self._shared_memory.put_obj(value) 631 write_uint64(self._base_address, self._item_offset(key), item_offset) 632 elif isinstance(key, slice): 633 if key.step is not None: 634 raise NotImplementedError 635 636 if key.start is None: 637 start = 0 638 elif key.start < 0: 639 start = key.start + len(self) 640 else: 641 start = key.start 642 643 if key.stop is None: 644 stop = len(self) 645 elif key.stop < 0: 646 stop = key.stop + len(self) 647 else: 648 stop = key.stop 649 650 if start < 0 or start >= len(self) or stop < 0 or stop > len(self) or start >= stop: 651 raise IndexError 652 653 for i in range(start, stop): 654 item_mapped_obj, item_offset, item_size = self._shared_memory.put_obj(value[i - start]) 655 write_uint64(self._base_address, self._item_offset(i), item_offset) 656 else: 657 raise TypeError 658 659 def __delitem__(self, key: Union[int, slice]) -> None: 660 if isinstance(key, int): 661 if key < 0: 662 key += len(self) 663 if key < 0 or key >= len(self): 664 raise IndexError 665 666 for i in range(key + 1, len(self)): 667 item_offset = read_uint64(self._base_address, self._item_offset(i)) 668 self._shared_memory.free(item_offset) 669 write_uint64(self._base_address, self._item_offset(i - 1), item_offset) 670 671 self._list_len -= 1 672 elif isinstance(key, slice): 673 if key.step is not None: 674 raise NotImplementedError 675 676 if key.start is None: 677 start = 0 678 elif key.start < 0: 679 start = key.start + len(self) 680 else: 681 start = key.start 682 683 if key.stop is None: 684 stop = len(self) 685 elif key.stop < 0: 686 stop = key.stop + len(self) 687 else: 688 stop = key.stop 689 690 if start < 0 or start >= len(self) or stop < 0 or stop > len(self) or start >= stop: 691 raise IndexError 692 693 for i in range(start, stop): 694 item_offset = read_uint64(self._base_address, self._item_offset(i)) 695 self._shared_memory.free(item_offset) 696 697 del_items_num = stop - start 698 699 for i in range(stop, len(self)): 700 item_offset = read_uint64(self._base_address, self._item_offset(i)) 701 write_uint64(self._base_address, self._item_offset(i - del_items_num), item_offset) 702 703 self._list_len -= del_items_num 704 else: 705 raise TypeError 706 707 def append(self, item: Any) -> None: 708 if self._list_len > self._list_capacity: 709 self._pointer_to_internal_list, result_size = realloc_tinternal_list(self._shared_memory, self._pointer_to_internal_list) 710 711 item_mapped_obj, item_offset, item_size = self._shared_memory.put_obj(item) 712 write_uint64(self._base_address, self._item_offset(self._list_len), item_offset) 713 self._list_len += 1 714 715 def extend(self, items: Sequence) -> None: 716 items_num = len(items) 717 if self._list_len + items_num > self._list_capacity: 718 self._pointer_to_internal_list, result_size = realloc_tinternal_list(self._shared_memory, self._pointer_to_internal_list, self._list_len + items_num) 719 720 for i, item in enumerate(items): 721 item_mapped_obj, item_offset, item_size = self._shared_memory.put_obj(item) 722 write_uint64(self._base_address, self._item_offset(self._list_len + i), item_offset) 723 724 self._list_len += items_num 725 726 def insert(self, index: int, item: Any) -> None: 727 if index < 0: 728 index += len(self) 729 if index < 0 or index > len(self): 730 raise IndexError 731 732 if self._list_len > self._list_capacity: 733 self._pointer_to_internal_list, result_size = realloc_tinternal_list(self._shared_memory, self._pointer_to_internal_list) 734 735 for i in range(self._list_len, index, -1): 736 item_offset = read_uint64(self._base_address, self._item_offset(i - 1)) 737 write_uint64(self._base_address, self._item_offset(i), item_offset) 738 739 item_mapped_obj, item_offset, item_size = self._shared_memory.put_obj(item) 740 write_uint64(self._base_address, self._item_offset(index), item_offset) 741 self._list_len += 1 742 743 def pop(self, index: int = -1) -> Any: 744 if index < 0: 745 index += len(self) 746 if index < 0 or index >= len(self): 747 raise IndexError 748 749 item_offset = read_uint64(self._base_address, self._item_offset(index)) 750 result = self._shared_memory.get_obj(item_offset) 751 752 for i in range(index + 1, len(self)): 753 item_offset = read_uint64(self._base_address, self._item_offset(i)) 754 write_uint64(self._base_address, self._item_offset(i - 1), item_offset) 755 756 self._list_len -= 1 757 return result 758 759 def remove(self, item: Any) -> None: 760 for i in range(len(self)): 761 item_offset = read_uint64(self._base_address, self._item_offset(i)) 762 if item_offset == item._offset: 763 for j in range(i + 1, len(self)): 764 item_offset = read_uint64(self._base_address, self._item_offset(j)) 765 write_uint64(self._base_address, self._item_offset(j - 1), item_offset) 766 767 self._list_len -= 1 768 return 769 770 raise ValueError 771 772 def clear(self) -> None: 773 for i in range(len(self)): 774 item_offset = read_uint64(self._base_address, self._item_offset(i)) 775 self._shared_memory.free(item_offset) 776 777 self._list_len = 0 778 779 def __iter__(self): 780 return IListIterator(self) 781 782 def __reversed__(self): 783 return IListReversedIterator(self) 784 785 def __contains__(self, item: Any) -> bool: 786 for i in range(len(self)): 787 item_offset = read_uint64(self._base_address, self._item_offset(i)) 788 if item_offset == item._offset: 789 return True 790 791 return False 792 793 def index(self, item: Any, start: int = 0, stop: int = None) -> int: 794 if stop is None: 795 stop = len(self) 796 797 for i in range(start, stop): 798 item_offset = read_uint64(self._base_address, self._item_offset(i)) 799 if item_offset == item._offset: 800 return i 801 802 raise ValueError 803 804 def count(self, item: Any) -> int: 805 result = 0 806 for i in range(len(self)): 807 item_offset = read_uint64(self._base_address, self._item_offset(i)) 808 if item_offset == item._offset: 809 result += 1 810 811 return result 812 813 def reverse(self) -> None: 814 for i in range(len(self) // 2): 815 item_offset = read_uint64(self._base_address, self._item_offset(i)) 816 write_uint64(self._base_address, self._item_offset(i), read_uint64(self._base_address, self._item_offset(len(self) - i - 1))) 817 write_uint64(self._base_address, self._item_offset(len(self) - i - 1), item_offset) 818 819 def sort(self, key: Any = None, reverse: bool = False) -> None: 820 raise NotImplementedError 821 822 def copy(self) -> 'IList': 823 result = IList(self._shared_memory) 824 result.extend(self) 825 return result 826 827 def __add__(self, other: Sequence) -> 'IList': 828 result = IList(self._shared_memory) 829 result.extend(self) 830 result.extend(other) 831 return result 832 833 def __iadd__(self, other: Sequence) -> 'IList': 834 self.extend(other) 835 return self 836 837 def __mul__(self, other: int) -> 'IList': 838 result = IList(self._shared_memory) 839 for i in range(other): 840 result.extend(self) 841 842 return result 843 844 def __imul__(self, other: int) -> 'IList': 845 my_copy: IList = self.copy() 846 for i in range(other): 847 self.extend(my_copy) 848 849 return self 850 851 def __rmul__(self, other: int) -> 'IList': 852 return self.__mul__(other) 853 854 def __eq__(self, other: Sequence) -> bool: 855 if len(self) != len(other): 856 return False 857 858 for i in range(len(self)): 859 if self[i] != other[i]: 860 return False 861 862 return True 863 864 def __ne__(self, other: Sequence) -> bool: 865 return not self.__eq__(other) 866 867 def __lt__(self, other: Sequence) -> bool: 868 for i in range(len(self)): 869 if self[i] >= other[i]: 870 return False 871 872 return True 873 874 def __le__(self, other: Sequence) -> bool: 875 for i in range(len(self)): 876 if self[i] > other[i]: 877 return False 878 879 return True 880 881 def __gt__(self, other: Sequence) -> bool: 882 for i in range(len(self)): 883 if self[i] <= other[i]: 884 return False 885 886 return True 887 888 def __ge__(self, other: Sequence) -> bool: 889 for i in range(len(self)): 890 if self[i] < other[i]: 891 return False 892 893 return True 894 895 def __repr__(self) -> str: 896 return f'IList({list(self)})' 897 898 def __str__(self) -> str: 899 return f'IList({list(self)})' 900 901 def __hash__(self) -> int: 902 return hash(tuple(self)) 903 904 def __sizeof__(self) -> int: 905 return read_uint64(self._base_address, self._offset + 8 * BaseObjOffsets.obj_size) + read_uint64(self._base_address, self._pointer_to_internal_list, 8 * BaseObjOffsets.obj_size) 906 907 def export(self) -> list: 908 return list(self) 909 910 # def __del__(self) -> None: 911 # self._shared_memory.free(self._pointer_to_internal_list) 912 # self._shared_memory.free(self._offset)
Built-in mutable sequence.
If no argument is given, the constructor creates a new empty list. The argument must be an iterable if specified.
IListTrue( shared_memory: SharedMemory, offset: int = None, obj: typing.List = None)
523 def __init__(self, shared_memory: 'SharedMemory', offset: Offset = None, obj: List = None) -> None: 524 self._shared_memory = shared_memory 525 self._base_address = shared_memory.base_address 526 if offset is None: 527 offset, real_size = shared_memory.malloc(ObjectType.tlist, 8) 528 self._offset = offset 529 self._offset__data = offset + 8 * len(BaseObjOffsets) 530 self._offset__pointer_to_internal_list = self._offset__data 531 532 if obj is None: 533 obj = list() 534 535 data_len = len(obj) 536 capacity_len = data_len << 1 if data_len else 16 537 internal_list_offset, data_tuple_real_size = malloc_tinternal_list(shared_memory, data_len, capacity_len) 538 self._pointer_to_internal_list = internal_list_offset 539 for i, item in enumerate(obj): 540 item_mapped_obj, item_offset, item_size = shared_memory.put_obj(item) 541 write_uint64(self._base_address, self._item_offset(i), item_offset) 542 else: 543 self._offset = offset 544 self._offset__data = offset + 8 * len(BaseObjOffsets) 545 self._offset__pointer_to_internal_list = self._offset__data
def
append(self, item: typing.Any) -> None:
707 def append(self, item: Any) -> None: 708 if self._list_len > self._list_capacity: 709 self._pointer_to_internal_list, result_size = realloc_tinternal_list(self._shared_memory, self._pointer_to_internal_list) 710 711 item_mapped_obj, item_offset, item_size = self._shared_memory.put_obj(item) 712 write_uint64(self._base_address, self._item_offset(self._list_len), item_offset) 713 self._list_len += 1
Append object to the end of the list.
def
extend(self, items: typing.Sequence) -> None:
715 def extend(self, items: Sequence) -> None: 716 items_num = len(items) 717 if self._list_len + items_num > self._list_capacity: 718 self._pointer_to_internal_list, result_size = realloc_tinternal_list(self._shared_memory, self._pointer_to_internal_list, self._list_len + items_num) 719 720 for i, item in enumerate(items): 721 item_mapped_obj, item_offset, item_size = self._shared_memory.put_obj(item) 722 write_uint64(self._base_address, self._item_offset(self._list_len + i), item_offset) 723 724 self._list_len += items_num
Extend list by appending elements from the iterable.
def
insert(self, index: int, item: typing.Any) -> None:
726 def insert(self, index: int, item: Any) -> None: 727 if index < 0: 728 index += len(self) 729 if index < 0 or index > len(self): 730 raise IndexError 731 732 if self._list_len > self._list_capacity: 733 self._pointer_to_internal_list, result_size = realloc_tinternal_list(self._shared_memory, self._pointer_to_internal_list) 734 735 for i in range(self._list_len, index, -1): 736 item_offset = read_uint64(self._base_address, self._item_offset(i - 1)) 737 write_uint64(self._base_address, self._item_offset(i), item_offset) 738 739 item_mapped_obj, item_offset, item_size = self._shared_memory.put_obj(item) 740 write_uint64(self._base_address, self._item_offset(index), item_offset) 741 self._list_len += 1
Insert object before index.
def
pop(self, index: int = -1) -> Any:
743 def pop(self, index: int = -1) -> Any: 744 if index < 0: 745 index += len(self) 746 if index < 0 or index >= len(self): 747 raise IndexError 748 749 item_offset = read_uint64(self._base_address, self._item_offset(index)) 750 result = self._shared_memory.get_obj(item_offset) 751 752 for i in range(index + 1, len(self)): 753 item_offset = read_uint64(self._base_address, self._item_offset(i)) 754 write_uint64(self._base_address, self._item_offset(i - 1), item_offset) 755 756 self._list_len -= 1 757 return result
Remove and return item at index (default last).
Raises IndexError if list is empty or index is out of range.
def
remove(self, item: typing.Any) -> None:
759 def remove(self, item: Any) -> None: 760 for i in range(len(self)): 761 item_offset = read_uint64(self._base_address, self._item_offset(i)) 762 if item_offset == item._offset: 763 for j in range(i + 1, len(self)): 764 item_offset = read_uint64(self._base_address, self._item_offset(j)) 765 write_uint64(self._base_address, self._item_offset(j - 1), item_offset) 766 767 self._list_len -= 1 768 return 769 770 raise ValueError
Remove first occurrence of value.
Raises ValueError if the value is not present.
def
clear(self) -> None:
772 def clear(self) -> None: 773 for i in range(len(self)): 774 item_offset = read_uint64(self._base_address, self._item_offset(i)) 775 self._shared_memory.free(item_offset) 776 777 self._list_len = 0
Remove all items from list.
def
index(self, item: typing.Any, start: int = 0, stop: int = None) -> int:
793 def index(self, item: Any, start: int = 0, stop: int = None) -> int: 794 if stop is None: 795 stop = len(self) 796 797 for i in range(start, stop): 798 item_offset = read_uint64(self._base_address, self._item_offset(i)) 799 if item_offset == item._offset: 800 return i 801 802 raise ValueError
Return first index of value.
Raises ValueError if the value is not present.
def
count(self, item: typing.Any) -> int:
804 def count(self, item: Any) -> int: 805 result = 0 806 for i in range(len(self)): 807 item_offset = read_uint64(self._base_address, self._item_offset(i)) 808 if item_offset == item._offset: 809 result += 1 810 811 return result
Return number of occurrences of value.
def
reverse(self) -> None:
813 def reverse(self) -> None: 814 for i in range(len(self) // 2): 815 item_offset = read_uint64(self._base_address, self._item_offset(i)) 816 write_uint64(self._base_address, self._item_offset(i), read_uint64(self._base_address, self._item_offset(len(self) - i - 1))) 817 write_uint64(self._base_address, self._item_offset(len(self) - i - 1), item_offset)
Reverse IN PLACE.
def
sort(self, key: typing.Any = None, reverse: bool = False) -> None:
Sort the list in ascending order and return None.
The sort is in-place (i.e. the list itself is modified) and stable (i.e. the order of two equal elements is maintained).
If a key function is given, apply it once to each list item and sort them, ascending or descending, according to their function values.
The reverse flag can be set to sort in descending order.
class
InternalListOffsets(enum.IntEnum):
An enumeration.
capacity =
<InternalListOffsets.capacity: 0>
size =
<InternalListOffsets.size: 1>
Inherited Members
- enum.Enum
- name
- value
- builtins.int
- conjugate
- bit_length
- to_bytes
- from_bytes
- as_integer_ratio
- real
- imag
- numerator
- denominator
class
InternalListFieldOffsets(enum.IntEnum):
An enumeration.
field_type =
<InternalListFieldOffsets.field_type: 0>
offset_or_data =
<InternalListFieldOffsets.offset_or_data: 1>
Inherited Members
- enum.Enum
- name
- value
- builtins.int
- conjugate
- bit_length
- to_bytes
- from_bytes
- as_integer_ratio
- real
- imag
- numerator
- denominator
def
malloc_tinternal_list( shared_memory: SharedMemory, size: int, capacity: int = None) -> Tuple[int, int]:
929def malloc_tinternal_list(shared_memory: 'SharedMemory', size: Size, capacity: Size = None) -> Tuple[Offset, Size]: 930 capacity = (size << 1 if size else 16) if capacity is None else capacity 931 offset, real_size = shared_memory.malloc(ObjectType.tinternal_list, bs * len(BaseObjOffsets) + bs * len(InternalListOffsets) + capacity * bs * len(InternalListFieldOffsets)) 932 sys_data_offset = offset + bs * len(BaseObjOffsets) 933 write_uint64(shared_memory.base_address, sys_data_offset + bs * InternalListOffsets.capacity, capacity) 934 write_uint64(shared_memory.base_address, sys_data_offset + bs * InternalListOffsets.size, size) 935 return offset, real_size
def
realloc_tinternal_list( shared_memory: SharedMemory, offset: int, desired_size: int = None, new_capacity: int = None, loop_allowed: bool = True, zero_mem: bool = True) -> Tuple[int, int]:
938def realloc_tinternal_list(shared_memory: 'SharedMemory', offset: Offset, desired_size: int = None, new_capacity: int = None, loop_allowed: bool = True, zero_mem: bool = True) -> Tuple[Offset, Size]: 939 sys_data_offset = offset + bs * len(BaseObjOffsets) 940 capacity = read_uint64(shared_memory.base_address, sys_data_offset + bs * InternalListOffsets.capacity) 941 size = read_uint64(shared_memory.base_address, sys_data_offset + bs * InternalListOffsets.size) 942 new_list_capacity = capacity << 1 if new_capacity is None else new_capacity 943 if new_capacity is None: 944 if desired_size is None: 945 new_list_capacity = capacity << 1 if capacity else 16 946 else: 947 new_list_capacity = desired_size << 1 if desired_size else 16 948 else: 949 new_list_capacity = new_capacity 950 951 if new_list_capacity < size: 952 new_list_capacity = size 953 954 new_offset, new_real_size = shared_memory.realloc( 955 offset, 956 bs * len(BaseObjOffsets) + bs * len(InternalListOffsets) + capacity * bs * len(InternalListFieldOffsets), 957 loop_allowed, 958 zero_mem 959 ) 960 new_sys_data_offset = new_offset + bs * len(BaseObjOffsets) 961 write_uint64(shared_memory.base_address, new_sys_data_offset + bs * InternalListOffsets.capacity, new_list_capacity) 962 return new_offset, new_real_size
def
uint64_to_bytes(int_data: int) -> bytes:
965def uint64_to_bytes(int_data: int) -> bytes: 966 """ 967 For a 64 bit unsigned int in little endian 968 :param int_data: 969 :return: bytes(); len == 8 970 """ 971 from struct import pack 972 result = pack('<B', int_data) 973 return result
For a 64 bit unsigned int in little endian :param int_data: :return: bytes(); len == 8
def
uint8_to_bytes(int_data: int) -> bytes:
976def uint8_to_bytes(int_data: int) -> bytes: 977 """ 978 For a 64 bit unsigned int in little endian 979 :param int_data: 980 :return: bytes(); len == 8 981 """ 982 from struct import pack 983 result = pack('<Q', int_data) 984 return result
For a 64 bit unsigned int in little endian :param int_data: :return: bytes(); len == 8
class
ListOffsets(enum.IntEnum):
An enumeration.
internal_list_offset =
<ListOffsets.internal_list_offset: 0>
Inherited Members
- enum.Enum
- name
- value
- builtins.int
- conjugate
- bit_length
- to_bytes
- from_bytes
- as_integer_ratio
- real
- imag
- numerator
- denominator
995class IList(BaseIObject, list): 996 def __init__(self, shared_memory: 'SharedMemory', offset: Offset = None, obj: List = None) -> None: 997 self._shared_memory = shared_memory 998 self._base_address = shared_memory.base_address 999 if offset is None: 1000 offset, real_size = shared_memory.malloc(ObjectType.tlist, bs * len(ListOffsets)) 1001 self._offset = offset 1002 self._offset__data = offset + bs * len(BaseObjOffsets) 1003 self._offset__pointer_to_internal_list = self._offset__data + bs * ListOffsets.internal_list_offset 1004 1005 if obj is None: 1006 obj = list() 1007 1008 data_len = len(obj) 1009 internal_list_offset, data_tuple_real_size = malloc_tinternal_list(shared_memory, data_len) 1010 self._pointer_to_internal_list = internal_list_offset 1011 for i, item in enumerate(obj): 1012 # print(self.get_children_offsets()) 1013 # # print(self.raw_to_list(slice(0, None))) 1014 # print(self.raw_to_bytes(200)) 1015 self._write_item(i, item) 1016 # print(self.get_children_offsets()) 1017 # # print(self.raw_to_list(slice(0, None))) 1018 # print(self.raw_to_bytes(200)) 1019 1020 # print(self.get_children_offsets()) 1021 # # print(self.raw_to_list(slice(0, None))) 1022 # print(self.raw_to_bytes(200)) 1023 # print('=======================') 1024 else: 1025 self._offset = offset 1026 self._offset__data = offset + bs * len(BaseObjOffsets) 1027 self._offset__pointer_to_internal_list = self._offset__data 1028 1029 def raw_to_list(self, key) -> List[bytes]: 1030 if isinstance(key, int): 1031 if key < 0: 1032 key += len(self) 1033 if key < 0 or key >= len(self): 1034 raise IndexError 1035 1036 item_offset = self._read_item_offset_or_data(key) 1037 return [uint64_to_bytes(item_offset)] 1038 elif isinstance(key, slice): 1039 if key.step is not None: 1040 raise NotImplementedError 1041 1042 if key.start is None: 1043 start = 0 1044 elif key.start < 0: 1045 start = key.start + len(self) 1046 else: 1047 start = key.start 1048 1049 if key.stop is None: 1050 stop = len(self) 1051 elif key.stop < 0: 1052 stop = key.stop + len(self) 1053 else: 1054 stop = key.stop 1055 1056 if start < 0 or start >= len(self) or stop < 0 or stop > len(self) or start >= stop: 1057 raise IndexError 1058 1059 result_list = list() 1060 for i in range(start, stop): 1061 item_offset = self._read_item_offset_or_data(i) 1062 result_list.append(uint64_to_bytes(item_offset)) 1063 1064 return result_list 1065 1066 def raw_to_bytes(self, bytes_num: int) -> bytes: 1067 start_index = self._pointer_to_internal_list 1068 return self._shared_memory.read_mem(start_index, bytes_num) 1069 # return bytes(self._shared_memory._shared_memory.buf[start_index : start_index + bytes_num]) 1070 1071 @property 1072 def _obj_size(self): 1073 return read_uint64(self._base_address, self._offset + bs * BaseObjOffsets.obj_size) 1074 1075 @property 1076 def _pointer_to_internal_list(self): 1077 return read_uint64(self._base_address, self._offset__pointer_to_internal_list) 1078 1079 @_pointer_to_internal_list.setter 1080 def _pointer_to_internal_list(self, value: Offset): 1081 write_uint64(self._base_address, self._offset__pointer_to_internal_list, value) 1082 1083 @property 1084 def _list_len(self): 1085 return read_uint64(self._base_address, self._pointer_to_internal_list + bs * len(BaseObjOffsets) + bs * InternalListOffsets.size) 1086 1087 @_list_len.setter 1088 def _list_len(self, value: int): 1089 write_uint64(self._base_address, self._pointer_to_internal_list + bs * len(BaseObjOffsets) + bs * InternalListOffsets.size, value) 1090 1091 @property 1092 def _list_capacity(self): 1093 return read_uint64(self._base_address, self._pointer_to_internal_list + bs * len(BaseObjOffsets) + bs * InternalListOffsets.capacity) 1094 1095 def _item_offset(self, key: int) -> Offset: 1096 return self._pointer_to_internal_list + bs * len(BaseObjOffsets) + bs * len(InternalListOffsets) + key * bs * len(InternalListFieldOffsets) 1097 1098 def _item_type_offset(self, key: int) -> Offset: 1099 # from os import getpid 1100 result = self._pointer_to_internal_list + bs * len(BaseObjOffsets) + bs * len(InternalListOffsets) + key * bs * len(InternalListFieldOffsets) + bs * InternalListFieldOffsets.field_type 1101 # add_0 = bs * len(BaseObjOffsets) 1102 # add_1 = bs * len(InternalListOffsets) 1103 # add_2 = key * bs * len(InternalListFieldOffsets) 1104 # add_3 = bs * InternalListFieldOffsets.field_type 1105 # print(f'PID: {getpid()}. [{add_0},{add_1},{add_2},{add_3}],{add_0 + add_1 + add_2 + add_3},{self._pointer_to_internal_list}: item_type_offset: {key}:{result}') 1106 return result 1107 1108 def _item_value_offset(self, key: int) -> Offset: 1109 # from os import getpid 1110 result = self._pointer_to_internal_list + bs * len(BaseObjOffsets) + bs * len(InternalListOffsets) + key * bs * len(InternalListFieldOffsets) + bs * InternalListFieldOffsets.offset_or_data 1111 # print(f'PID: {getpid()}. {bs * len(BaseObjOffsets) + bs * len(InternalListOffsets) + key * bs * len(InternalListFieldOffsets) + bs * InternalListFieldOffsets.offset_or_data},{self._pointer_to_internal_list}: item_value_offset: {key}:{result}') 1112 return result 1113 1114 def _read_item_type(self, key: int) -> int: 1115 return read_uint64(self._base_address, self._item_type_offset(key)) 1116 1117 def _write_item_type(self, key: int, item_type: int) -> None: 1118 write_uint64(self._base_address, self._item_type_offset(key), item_type) 1119 1120 def _read_item_offset_or_data(self, key: int) -> Union[Offset, int]: 1121 return read_uint64(self._base_address, self._item_value_offset(key)) 1122 1123 def _write_item_offset_or_data(self, key: int, offset_or_data: Union[Offset, int]) -> None: 1124 write_uint64(self._base_address, self._item_value_offset(key), offset_or_data) 1125 1126 def _determine_obj_type(self, obj: Any) -> int: 1127 if isinstance(obj, int): 1128 return 1 1129 elif isinstance(obj, float): 1130 return 2 1131 elif isinstance(obj, bool): 1132 return 3 1133 else: 1134 return 0 1135 1136 def _determine_obj_offset(self, obj: Any) -> Optional[Offset]: 1137 if isinstance(obj, BaseIObject): 1138 return obj._offset 1139 else: 1140 return None 1141 1142 def _compare_item_to_obj_fast(self, key: int, obj: Any, obj_type: int, obj_offset) -> bool: 1143 result: bool = False 1144 item_type = self._read_item_type(key) 1145 if item_type == obj_type: 1146 if item_type == 0: 1147 if obj_offset is None: 1148 if self._read_item_value(key, item_type) == obj: 1149 result = True 1150 else: 1151 if self._read_item_offset_or_data(key) == obj_offset: 1152 result = True 1153 elif item_type == 1: 1154 if self._read_item_offset_or_data(key) == obj: 1155 result = True 1156 elif item_type == 2: 1157 if self._read_item_offset_or_data(key) == obj: 1158 result = True 1159 elif item_type == 3: 1160 if self._read_item_offset_or_data(key) == obj: 1161 result = True 1162 else: 1163 raise ValueError 1164 1165 return result 1166 1167 def _compare_item_to_obj(self, key: int, obj: Any) -> bool: 1168 obj_type = self._determine_obj_type(obj) 1169 obj_offset = self._determine_obj_offset(obj) 1170 return self._compare_item_to_obj_fast(key, obj, obj_type, obj_offset) 1171 1172 def _read_item_value(self, key: int, item_type: int) -> Any: 1173 if item_type == 0: 1174 item_offset = read_uint64(self._base_address, self._item_value_offset(key)) 1175 return self._shared_memory.get_obj(item_offset) 1176 elif item_type == 1: 1177 return read_int64(self._base_address, self._item_value_offset(key)) 1178 elif item_type == 2: 1179 return read_double(self._base_address, self._item_value_offset(key)) 1180 elif item_type == 3: 1181 return bool(read_uint64(self._base_address, self._item_value_offset(key))) 1182 else: 1183 raise ValueError 1184 1185 def _write_item_value(self, key: int, item_type: int, value: Any) -> None: 1186 if item_type == 0: 1187 item_mapped_obj, item_offset, item_size = self._shared_memory.put_obj(value) 1188 write_uint64(self._base_address, self._item_value_offset(key), item_offset) 1189 elif item_type == 1: 1190 write_int64(self._base_address, self._item_value_offset(key), value) 1191 elif item_type == 2: 1192 write_double(self._base_address, self._item_value_offset(key), value) 1193 elif item_type == 3: 1194 write_uint64(self._base_address, self._item_value_offset(key), int(value)) 1195 else: 1196 raise ValueError 1197 1198 def _free_item_value(self, key: int, item_type: int) -> None: 1199 if item_type == 0: 1200 item_offset = read_uint64(self._base_address, self._item_value_offset(key)) 1201 self._shared_memory.free(item_offset) 1202 elif item_type == 1: 1203 pass 1204 elif item_type == 2: 1205 pass 1206 elif item_type == 3: 1207 pass 1208 else: 1209 raise ValueError 1210 1211 def _read_item_type_and_value(self, key: int) -> Tuple[int, Any]: 1212 item_type = self._read_item_type(key) 1213 return item_type, self._read_item_value(key, item_type) 1214 1215 def _write_item_value_and_get_type(self, key: int, value: Any) -> int: 1216 if isinstance(value, int): 1217 write_uint64(self._base_address, self._item_value_offset(key), value) 1218 return 1 1219 elif isinstance(value, float): 1220 write_double(self._base_address, self._item_value_offset(key), value) 1221 return 2 1222 elif isinstance(value, bool): 1223 write_uint64(self._base_address, self._item_value_offset(key), int(value)) 1224 return 3 1225 else: 1226 item_mapped_obj, item_offset, item_size = self._shared_memory.put_obj(value) 1227 write_uint64(self._base_address, self._item_value_offset(key), item_offset) 1228 return 0 1229 1230 def _free_item_value_and_get_type(self, key: int) -> int: 1231 item_type = self._read_item_type(key) 1232 self._free_item_value(key, item_type) 1233 return item_type 1234 1235 def _read_item(self, key: int) -> Any: 1236 item_type = self._read_item_type(key) 1237 return self._read_item_value(key, item_type) 1238 1239 def _write_item(self, key: int, value: Any) -> None: 1240 item_type = self._write_item_value_and_get_type(key, value) 1241 self._write_item_type(key, item_type) 1242 1243 def _free_item(self, key: int) -> None: 1244 item_type = self._read_item_type(key) 1245 self._free_item_value(key, item_type) 1246 1247 def _move_item(self, src_key: int, dst_key: int) -> None: 1248 self._write_item_type(dst_key, self._read_item_type(src_key)) 1249 self._write_item_offset_or_data(dst_key, self._read_item_offset_or_data(src_key)) 1250 1251 def _swap_items(self, key1: int, key2: int) -> None: 1252 item_type1 = self._read_item_type(key1) 1253 item_offset_or_data1 = self._read_item_offset_or_data(key1) 1254 self._write_item_type(key1, self._read_item_type(key2)) 1255 self._write_item_type(key2, item_type1) 1256 self._write_item_offset_or_data(key1, self._read_item_offset_or_data(key2)) 1257 self._write_item_offset_or_data(key2, item_offset_or_data1) 1258 1259 def __len__(self) -> int: 1260 return self._list_len 1261 1262 def get_children_data_or_offsets(self) -> List[Offset]: 1263 return [self._read_item_offset_or_data(i) for i in range(self._list_len)] 1264 1265 def get_children_offsets(self): 1266 return self.get_children_data_or_offsets() 1267 1268 def __getitem__(self, key: Union[int, slice]) -> Union[Any, List]: 1269 if isinstance(key, int): 1270 return list__get_item(key, self._base_address, self._offset__pointer_to_internal_list, self._shared_memory.get_obj) 1271 1272 # base_address = self._base_address 1273 # offset__pointer_to_internal_list = self._offset__pointer_to_internal_list 1274 # pointer_to_internal_list = read_uint64(base_address, offset__pointer_to_internal_list) 1275 # self_len = read_uint64(base_address, pointer_to_internal_list + 24) 1276 1277 # if key < 0: 1278 # key += self_len 1279 1280 # if key < 0 or key >= self_len: 1281 # raise IndexError 1282 1283 # item_type_offset = pointer_to_internal_list + 32 + key * 16 1284 # item_value_offset = pointer_to_internal_list + 40 + key * 16 1285 # item_type = read_uint64(base_address, item_type_offset) 1286 # if item_type == 1: 1287 # return read_int64(base_address, item_value_offset) 1288 # elif item_type == 2: 1289 # return read_double(base_address, item_value_offset) 1290 # elif item_type == 3: 1291 # return bool(read_uint64(base_address, item_value_offset)) 1292 # elif item_type == 0: 1293 # item_offset = read_uint64(base_address, item_value_offset) 1294 # return self._shared_memory.get_obj(item_offset) 1295 # else: 1296 # raise ValueError 1297 1298 # # return self._read_item(key) 1299 elif isinstance(key, slice): 1300 if key.step is not None: 1301 raise NotImplementedError 1302 1303 if key.start is None: 1304 start = 0 1305 elif key.start < 0: 1306 start = key.start + len(self) 1307 else: 1308 start = key.start 1309 1310 if key.stop is None: 1311 stop = len(self) 1312 elif key.stop < 0: 1313 stop = key.stop + len(self) 1314 else: 1315 stop = key.stop 1316 1317 if start < 0 or start >= len(self) or stop < 0 or stop > len(self) or start >= stop: 1318 raise IndexError 1319 1320 result_list = list() 1321 # performance improvement instead of using self._read_item(i) 1322 base_address = self._base_address 1323 offset__pointer_to_internal_list = self._offset__pointer_to_internal_list 1324 pointer_to_internal_list = read_uint64(base_address, offset__pointer_to_internal_list) 1325 item_type_offset = pointer_to_internal_list + 32 + i * 16 1326 item_value_offset = pointer_to_internal_list + 40 + i * 16 1327 for i in range(start, stop): 1328 # result_list.append(self._read_item(i)) 1329 1330 # performance improvement instead of using self._read_item(i) 1331 item_type = read_uint64(base_address, item_type_offset) 1332 if item_type == 1: 1333 result_list.append(read_int64(base_address, item_value_offset)) 1334 elif item_type == 2: 1335 result_list.append(read_double(base_address, item_value_offset)) 1336 elif item_type == 3: 1337 result_list.append(bool(read_uint64(base_address, item_value_offset))) 1338 elif item_type == 0: 1339 item_offset = read_uint64(base_address, item_value_offset) 1340 result_list.append(self._shared_memory.get_obj(item_offset)) 1341 else: 1342 raise ValueError 1343 1344 return result_list 1345 else: 1346 raise TypeError 1347 1348 def __setitem__(self, key: Union[int, slice], value: Union[Any, Sequence]) -> Any: 1349 if isinstance(key, int): 1350 list__set_item(key, value, self._base_address, self._offset__pointer_to_internal_list, self._shared_memory.put_obj) 1351 1352 # base_address = self._base_address 1353 # offset__pointer_to_internal_list = self._offset__pointer_to_internal_list 1354 # pointer_to_internal_list = read_uint64(base_address, offset__pointer_to_internal_list) 1355 # self_len = read_uint64(base_address, pointer_to_internal_list + 24) 1356 1357 # if key < 0: 1358 # key += self_len 1359 1360 # if key < 0 or key >= self_len: 1361 # raise IndexError 1362 1363 # item_type_offset = pointer_to_internal_list + 32 + key * 16 1364 # item_value_offset = pointer_to_internal_list + 40 + key * 16 1365 # if isinstance(value, int): 1366 # write_int64(base_address, item_value_offset, value) 1367 # item_type = 1 1368 # elif isinstance(value, float): 1369 # write_double(base_address, item_value_offset, value) 1370 # item_type = 2 1371 # elif isinstance(value, bool): 1372 # write_uint64(base_address, item_value_offset, int(value)) 1373 # item_type = 3 1374 # else: 1375 # item_mapped_obj, item_offset, item_size = self._shared_memory.put_obj(value) 1376 # write_uint64(base_address, item_value_offset, item_offset) 1377 # item_type = 0 1378 1379 # write_uint64(base_address, item_type_offset, item_type) 1380 1381 # # self._write_item(key, value) 1382 elif isinstance(key, slice): 1383 if key.step is not None: 1384 raise NotImplementedError 1385 1386 if key.start is None: 1387 start = 0 1388 elif key.start < 0: 1389 start = key.start + len(self) 1390 else: 1391 start = key.start 1392 1393 if key.stop is None: 1394 stop = len(self) 1395 elif key.stop < 0: 1396 stop = key.stop + len(self) 1397 else: 1398 stop = key.stop 1399 1400 if start < 0 or start >= len(self) or stop < 0 or stop > len(self) or start >= stop: 1401 raise IndexError 1402 1403 # performance improvement instead of using self._write_item(i, item) 1404 base_address = self._base_address 1405 offset__pointer_to_internal_list = self._offset__pointer_to_internal_list 1406 pointer_to_internal_list = read_uint64(base_address, offset__pointer_to_internal_list) 1407 item_type_offset = pointer_to_internal_list + 32 + i * 16 1408 item_value_offset = pointer_to_internal_list + 40 + i * 16 1409 for i in range(start, stop): 1410 item = value[i - start] 1411 # self._write_item(i, item) 1412 1413 # performance improvement instead of using self._write_item(i, item) 1414 if isinstance(item, int): 1415 write_int64(base_address, item_value_offset, item) 1416 item_type = 1 1417 elif isinstance(item, float): 1418 write_double(base_address, item_value_offset, item) 1419 item_type = 2 1420 elif isinstance(item, bool): 1421 write_uint64(base_address, item_value_offset, int(item)) 1422 item_type = 3 1423 else: 1424 item_mapped_obj, item_offset, item_size = self._shared_memory.put_obj(item) 1425 write_uint64(base_address, item_value_offset, item_offset) 1426 item_type = 0 1427 1428 write_uint64(base_address, item_type_offset, item_type) 1429 else: 1430 raise TypeError 1431 1432 def __delitem__(self, key: Union[int, slice]) -> None: 1433 if isinstance(key, int): 1434 if key < 0: 1435 key += len(self) 1436 if key < 0 or key >= len(self): 1437 raise IndexError 1438 1439 self._free_item(key) 1440 1441 for i in range(key + 1, len(self)): 1442 self._move_item(i, i - 1) 1443 1444 self._list_len -= 1 1445 elif isinstance(key, slice): 1446 if key.step is not None: 1447 raise NotImplementedError 1448 1449 if key.start is None: 1450 start = 0 1451 elif key.start < 0: 1452 start = key.start + len(self) 1453 else: 1454 start = key.start 1455 1456 if key.stop is None: 1457 stop = len(self) 1458 elif key.stop < 0: 1459 stop = key.stop + len(self) 1460 else: 1461 stop = key.stop 1462 1463 if start < 0 or start >= len(self) or stop < 0 or stop > len(self) or start >= stop: 1464 raise IndexError 1465 1466 for i in range(start, stop): 1467 self._free_item(i) 1468 1469 del_items_num = stop - start 1470 1471 for i in range(stop, len(self)): 1472 self._move_item(i, i - del_items_num) 1473 1474 self._list_len -= del_items_num 1475 else: 1476 raise TypeError 1477 1478 def append(self, item: Any) -> None: 1479 if self._list_len > self._list_capacity: 1480 self._pointer_to_internal_list, result_size = realloc_tinternal_list(self._shared_memory, self._pointer_to_internal_list) 1481 1482 self._list_len += 1 1483 self.__setitem__(self._list_len - 1, item) 1484 1485 def extend(self, items: Sequence) -> None: 1486 items_num = len(items) 1487 if self._list_len + items_num > self._list_capacity: 1488 self._pointer_to_internal_list, result_size = realloc_tinternal_list(self._shared_memory, self._pointer_to_internal_list, self._list_len + items_num) 1489 1490 original_list_len = self._list_len 1491 self._list_len += items_num 1492 for i, item in enumerate(items): 1493 self.__setitem__(original_list_len + i, item) 1494 1495 1496 def insert(self, index: int, item: Any) -> None: 1497 if index < 0: 1498 index += len(self) 1499 if index < 0 or index > len(self): 1500 raise IndexError 1501 1502 if self._list_len > self._list_capacity: 1503 # self._shared_memory.print_mem(self._pointer_to_internal_list, 200, 'before realloc. {}') 1504 # self.print_internal_list('before realloc. {}') 1505 self._pointer_to_internal_list, result_size = realloc_tinternal_list(self._shared_memory, self._pointer_to_internal_list) 1506 # self._shared_memory.print_mem(self._pointer_to_internal_list, 200, 'after realloc. {}') 1507 # self.print_internal_list('after realloc. {}') 1508 1509 # self.print_internal_list('before inserting {}') 1510 self._list_len += 1 1511 # self.print_internal_list('before inserting but after +1 {}') 1512 for i in range(self._list_len - 1, index, -1): 1513 self._move_item(i - 1, i) 1514 # self._shared_memory.print_mem(self._pointer_to_internal_list, 200, f'after self._move_item({i - 1, i}). {{}}') 1515 # self.print_internal_list(f'after self._move_item({i - 1, i}). {{}}') 1516 1517 self.__setitem__(index, item) 1518 # self._shared_memory.print_mem(self._pointer_to_internal_list, 200, 'after inserting. {}') 1519 # self.print_internal_list('after inserting. {}') 1520 1521 def print_internal_list(self, text: str = None, additional_cells: int = 0): 1522 internal_list = self._shared_memory.read_mem(self._pointer_to_internal_list, bs * len(BaseObjOffsets) + bs * len(InternalListOffsets) + self._list_len * bs * len(InternalListFieldOffsets) + additional_cells * bs * len(InternalListFieldOffsets)) 1523 print('--- internal list -------------') 1524 if text: 1525 print(text.format(self._pointer_to_internal_list)) 1526 print('------') 1527 1528 index = 0 1529 print(f'{index},{self._pointer_to_internal_list + index}:', internal_list[index:index + bs]) 1530 index += bs 1531 print(f'{index},{self._pointer_to_internal_list + index}:', internal_list[index:index + bs]) 1532 index += bs 1533 print('---') 1534 print(f'{index},{self._pointer_to_internal_list + index}:', internal_list[index:index + bs]) 1535 index += bs 1536 print(f'{index},{self._pointer_to_internal_list + index}:', internal_list[index:index + bs]) 1537 index += bs 1538 print('---') 1539 for i in range(self._list_len): 1540 print(f'{index},{self._pointer_to_internal_list + index}:', internal_list[index:index + bs * 2]) 1541 index += bs * 2 1542 1543 if additional_cells: 1544 print('------') 1545 for i in range(additional_cells): 1546 print(f'{index},{self._pointer_to_internal_list + index}:', internal_list[index:index + bs]) 1547 index += bs * 2 1548 print('-------------------------------') 1549 print() 1550 1551 def pop(self, index: int = -1) -> Any: 1552 if index < 0: 1553 index += len(self) 1554 if index < 0 or index >= len(self): 1555 raise IndexError 1556 1557 result = self.__getitem__(index) 1558 1559 for i in range(index + 1, len(self)): 1560 self._move_item(i, i - 1) 1561 1562 self._list_len -= 1 1563 return result 1564 1565 def remove(self, obj: Any) -> None: 1566 obj_type = self._determine_obj_type(obj) 1567 obj_offset = self._determine_obj_offset(obj) 1568 found_in_index = None 1569 for i in range(len(self)): 1570 if self._compare_item_to_obj_fast(i, obj, obj_type, obj_offset): 1571 found_in_index = i 1572 break 1573 1574 if found_in_index is None: 1575 raise ValueError 1576 else: 1577 self.__delitem__(found_in_index) 1578 1579 def clear(self) -> None: 1580 for i in range(len(self)): 1581 self._free_item(i) 1582 1583 self._list_len = 0 1584 1585 def __iter__(self): 1586 return IListIterator(self) 1587 1588 def __reversed__(self): 1589 return IListReversedIterator(self) 1590 1591 def __contains__(self, obj: Any) -> bool: 1592 obj_type = self._determine_obj_type(obj) 1593 obj_offset = self._determine_obj_offset(obj) 1594 found_in_index = None 1595 for i in range(len(self)): 1596 if self._compare_item_to_obj_fast(i, obj, obj_type, obj_offset): 1597 found_in_index = i 1598 break 1599 1600 if found_in_index is None: 1601 return False 1602 else: 1603 return True 1604 1605 def index(self, obj: Any, start: int = 0, stop: int = None) -> int: 1606 if stop is None: 1607 stop = len(self) 1608 1609 obj_type = self._determine_obj_type(obj) 1610 obj_offset = self._determine_obj_offset(obj) 1611 found_in_index = None 1612 for i in range(start, stop): 1613 if self._compare_item_to_obj_fast(i, obj, obj_type, obj_offset): 1614 found_in_index = i 1615 break 1616 1617 if found_in_index is None: 1618 raise ValueError 1619 else: 1620 return found_in_index 1621 1622 def count(self, obj: Any) -> int: 1623 obj_type = self._determine_obj_type(obj) 1624 obj_offset = self._determine_obj_offset(obj) 1625 result = 0 1626 found_in_index = None 1627 for i in range(len(self)): 1628 if self._compare_item_to_obj_fast(i, obj, obj_type, obj_offset): 1629 found_in_index = i 1630 result += 1 1631 1632 return result 1633 1634 def reverse(self) -> None: 1635 my_len = len(self) 1636 for i in range(my_len // 2): 1637 self._swap_items(i, my_len - i - 1) 1638 1639 def sort(self, key: Any = None, reverse: bool = False) -> None: 1640 raise NotImplementedError 1641 1642 def copy(self) -> 'IList': 1643 result = IList(self._shared_memory) 1644 result.extend(self) 1645 return result 1646 1647 def __add__(self, other: Sequence) -> 'IList': 1648 result = IList(self._shared_memory) 1649 result.extend(self) 1650 result.extend(other) 1651 return result 1652 1653 def __iadd__(self, other: Sequence) -> 'IList': 1654 self.extend(other) 1655 return self 1656 1657 def __mul__(self, other: int) -> 'IList': 1658 result = IList(self._shared_memory) 1659 for i in range(other): 1660 result.extend(self) 1661 1662 return result 1663 1664 def __imul__(self, other: int) -> 'IList': 1665 my_copy: IList = self.copy() 1666 for i in range(other): 1667 self.extend(my_copy) 1668 1669 return self 1670 1671 def __rmul__(self, other: int) -> 'IList': 1672 return self.__mul__(other) 1673 1674 def __eq__(self, other: Sequence) -> bool: 1675 if len(self) != len(other): 1676 return False 1677 1678 for i in range(len(self)): 1679 if self[i] != other[i]: 1680 return False 1681 1682 return True 1683 1684 def __ne__(self, other: Sequence) -> bool: 1685 return not self.__eq__(other) 1686 1687 def __lt__(self, other: Sequence) -> bool: 1688 for i in range(len(self)): 1689 if self[i] >= other[i]: 1690 return False 1691 1692 return True 1693 1694 def __le__(self, other: Sequence) -> bool: 1695 for i in range(len(self)): 1696 if self[i] > other[i]: 1697 return False 1698 1699 return True 1700 1701 def __gt__(self, other: Sequence) -> bool: 1702 for i in range(len(self)): 1703 if self[i] <= other[i]: 1704 return False 1705 1706 return True 1707 1708 def __ge__(self, other: Sequence) -> bool: 1709 for i in range(len(self)): 1710 if self[i] < other[i]: 1711 return False 1712 1713 return True 1714 1715 def __repr__(self) -> str: 1716 return f'IList({list(self)})' 1717 1718 def __str__(self) -> str: 1719 return f'IList({list(self)})' 1720 1721 def __hash__(self) -> int: 1722 return hash(tuple(self)) 1723 1724 def __sizeof__(self) -> int: 1725 return bs * len(BaseObjOffsets) + read_uint64(self._base_address, self._offset + bs * BaseObjOffsets.obj_size) + bs * len(BaseObjOffsets) + read_uint64(self._base_address, self._pointer_to_internal_list, bs * BaseObjOffsets.obj_size) 1726 1727 def export(self) -> list: 1728 return list(self) 1729 1730 # def __del__(self) -> None: 1731 # self._shared_memory.free(self._pointer_to_internal_list) 1732 # self._shared_memory.free(self._offset)
Built-in mutable sequence.
If no argument is given, the constructor creates a new empty list. The argument must be an iterable if specified.
IList( shared_memory: SharedMemory, offset: int = None, obj: typing.List = None)
996 def __init__(self, shared_memory: 'SharedMemory', offset: Offset = None, obj: List = None) -> None: 997 self._shared_memory = shared_memory 998 self._base_address = shared_memory.base_address 999 if offset is None: 1000 offset, real_size = shared_memory.malloc(ObjectType.tlist, bs * len(ListOffsets)) 1001 self._offset = offset 1002 self._offset__data = offset + bs * len(BaseObjOffsets) 1003 self._offset__pointer_to_internal_list = self._offset__data + bs * ListOffsets.internal_list_offset 1004 1005 if obj is None: 1006 obj = list() 1007 1008 data_len = len(obj) 1009 internal_list_offset, data_tuple_real_size = malloc_tinternal_list(shared_memory, data_len) 1010 self._pointer_to_internal_list = internal_list_offset 1011 for i, item in enumerate(obj): 1012 # print(self.get_children_offsets()) 1013 # # print(self.raw_to_list(slice(0, None))) 1014 # print(self.raw_to_bytes(200)) 1015 self._write_item(i, item) 1016 # print(self.get_children_offsets()) 1017 # # print(self.raw_to_list(slice(0, None))) 1018 # print(self.raw_to_bytes(200)) 1019 1020 # print(self.get_children_offsets()) 1021 # # print(self.raw_to_list(slice(0, None))) 1022 # print(self.raw_to_bytes(200)) 1023 # print('=======================') 1024 else: 1025 self._offset = offset 1026 self._offset__data = offset + bs * len(BaseObjOffsets) 1027 self._offset__pointer_to_internal_list = self._offset__data
def
raw_to_list(self, key) -> List[bytes]:
1029 def raw_to_list(self, key) -> List[bytes]: 1030 if isinstance(key, int): 1031 if key < 0: 1032 key += len(self) 1033 if key < 0 or key >= len(self): 1034 raise IndexError 1035 1036 item_offset = self._read_item_offset_or_data(key) 1037 return [uint64_to_bytes(item_offset)] 1038 elif isinstance(key, slice): 1039 if key.step is not None: 1040 raise NotImplementedError 1041 1042 if key.start is None: 1043 start = 0 1044 elif key.start < 0: 1045 start = key.start + len(self) 1046 else: 1047 start = key.start 1048 1049 if key.stop is None: 1050 stop = len(self) 1051 elif key.stop < 0: 1052 stop = key.stop + len(self) 1053 else: 1054 stop = key.stop 1055 1056 if start < 0 or start >= len(self) or stop < 0 or stop > len(self) or start >= stop: 1057 raise IndexError 1058 1059 result_list = list() 1060 for i in range(start, stop): 1061 item_offset = self._read_item_offset_or_data(i) 1062 result_list.append(uint64_to_bytes(item_offset)) 1063 1064 return result_list
def
append(self, item: typing.Any) -> None:
1478 def append(self, item: Any) -> None: 1479 if self._list_len > self._list_capacity: 1480 self._pointer_to_internal_list, result_size = realloc_tinternal_list(self._shared_memory, self._pointer_to_internal_list) 1481 1482 self._list_len += 1 1483 self.__setitem__(self._list_len - 1, item)
Append object to the end of the list.
def
extend(self, items: typing.Sequence) -> None:
1485 def extend(self, items: Sequence) -> None: 1486 items_num = len(items) 1487 if self._list_len + items_num > self._list_capacity: 1488 self._pointer_to_internal_list, result_size = realloc_tinternal_list(self._shared_memory, self._pointer_to_internal_list, self._list_len + items_num) 1489 1490 original_list_len = self._list_len 1491 self._list_len += items_num 1492 for i, item in enumerate(items): 1493 self.__setitem__(original_list_len + i, item)
Extend list by appending elements from the iterable.
def
insert(self, index: int, item: typing.Any) -> None:
1496 def insert(self, index: int, item: Any) -> None: 1497 if index < 0: 1498 index += len(self) 1499 if index < 0 or index > len(self): 1500 raise IndexError 1501 1502 if self._list_len > self._list_capacity: 1503 # self._shared_memory.print_mem(self._pointer_to_internal_list, 200, 'before realloc. {}') 1504 # self.print_internal_list('before realloc. {}') 1505 self._pointer_to_internal_list, result_size = realloc_tinternal_list(self._shared_memory, self._pointer_to_internal_list) 1506 # self._shared_memory.print_mem(self._pointer_to_internal_list, 200, 'after realloc. {}') 1507 # self.print_internal_list('after realloc. {}') 1508 1509 # self.print_internal_list('before inserting {}') 1510 self._list_len += 1 1511 # self.print_internal_list('before inserting but after +1 {}') 1512 for i in range(self._list_len - 1, index, -1): 1513 self._move_item(i - 1, i) 1514 # self._shared_memory.print_mem(self._pointer_to_internal_list, 200, f'after self._move_item({i - 1, i}). {{}}') 1515 # self.print_internal_list(f'after self._move_item({i - 1, i}). {{}}') 1516 1517 self.__setitem__(index, item) 1518 # self._shared_memory.print_mem(self._pointer_to_internal_list, 200, 'after inserting. {}') 1519 # self.print_internal_list('after inserting. {}')
Insert object before index.
def
print_internal_list(self, text: str = None, additional_cells: int = 0):
1521 def print_internal_list(self, text: str = None, additional_cells: int = 0): 1522 internal_list = self._shared_memory.read_mem(self._pointer_to_internal_list, bs * len(BaseObjOffsets) + bs * len(InternalListOffsets) + self._list_len * bs * len(InternalListFieldOffsets) + additional_cells * bs * len(InternalListFieldOffsets)) 1523 print('--- internal list -------------') 1524 if text: 1525 print(text.format(self._pointer_to_internal_list)) 1526 print('------') 1527 1528 index = 0 1529 print(f'{index},{self._pointer_to_internal_list + index}:', internal_list[index:index + bs]) 1530 index += bs 1531 print(f'{index},{self._pointer_to_internal_list + index}:', internal_list[index:index + bs]) 1532 index += bs 1533 print('---') 1534 print(f'{index},{self._pointer_to_internal_list + index}:', internal_list[index:index + bs]) 1535 index += bs 1536 print(f'{index},{self._pointer_to_internal_list + index}:', internal_list[index:index + bs]) 1537 index += bs 1538 print('---') 1539 for i in range(self._list_len): 1540 print(f'{index},{self._pointer_to_internal_list + index}:', internal_list[index:index + bs * 2]) 1541 index += bs * 2 1542 1543 if additional_cells: 1544 print('------') 1545 for i in range(additional_cells): 1546 print(f'{index},{self._pointer_to_internal_list + index}:', internal_list[index:index + bs]) 1547 index += bs * 2 1548 print('-------------------------------') 1549 print()
def
pop(self, index: int = -1) -> Any:
1551 def pop(self, index: int = -1) -> Any: 1552 if index < 0: 1553 index += len(self) 1554 if index < 0 or index >= len(self): 1555 raise IndexError 1556 1557 result = self.__getitem__(index) 1558 1559 for i in range(index + 1, len(self)): 1560 self._move_item(i, i - 1) 1561 1562 self._list_len -= 1 1563 return result
Remove and return item at index (default last).
Raises IndexError if list is empty or index is out of range.
def
remove(self, obj: typing.Any) -> None:
1565 def remove(self, obj: Any) -> None: 1566 obj_type = self._determine_obj_type(obj) 1567 obj_offset = self._determine_obj_offset(obj) 1568 found_in_index = None 1569 for i in range(len(self)): 1570 if self._compare_item_to_obj_fast(i, obj, obj_type, obj_offset): 1571 found_in_index = i 1572 break 1573 1574 if found_in_index is None: 1575 raise ValueError 1576 else: 1577 self.__delitem__(found_in_index)
Remove first occurrence of value.
Raises ValueError if the value is not present.
def
clear(self) -> None:
1579 def clear(self) -> None: 1580 for i in range(len(self)): 1581 self._free_item(i) 1582 1583 self._list_len = 0
Remove all items from list.
def
index(self, obj: typing.Any, start: int = 0, stop: int = None) -> int:
1605 def index(self, obj: Any, start: int = 0, stop: int = None) -> int: 1606 if stop is None: 1607 stop = len(self) 1608 1609 obj_type = self._determine_obj_type(obj) 1610 obj_offset = self._determine_obj_offset(obj) 1611 found_in_index = None 1612 for i in range(start, stop): 1613 if self._compare_item_to_obj_fast(i, obj, obj_type, obj_offset): 1614 found_in_index = i 1615 break 1616 1617 if found_in_index is None: 1618 raise ValueError 1619 else: 1620 return found_in_index
Return first index of value.
Raises ValueError if the value is not present.
def
count(self, obj: typing.Any) -> int:
1622 def count(self, obj: Any) -> int: 1623 obj_type = self._determine_obj_type(obj) 1624 obj_offset = self._determine_obj_offset(obj) 1625 result = 0 1626 found_in_index = None 1627 for i in range(len(self)): 1628 if self._compare_item_to_obj_fast(i, obj, obj_type, obj_offset): 1629 found_in_index = i 1630 result += 1 1631 1632 return result
Return number of occurrences of value.
def
reverse(self) -> None:
1634 def reverse(self) -> None: 1635 my_len = len(self) 1636 for i in range(my_len // 2): 1637 self._swap_items(i, my_len - i - 1)
Reverse IN PLACE.
def
sort(self, key: typing.Any = None, reverse: bool = False) -> None:
Sort the list in ascending order and return None.
The sort is in-place (i.e. the list itself is modified) and stable (i.e. the order of two equal elements is maintained).
If a key function is given, apply it once to each list item and sort them, ascending or descending, according to their function values.
The reverse flag can be set to sort in descending order.
class
IListIterator:
1738class IListIterator: 1739 def __init__(self, ilist: IList) -> None: 1740 self._ilist = ilist 1741 self._index = 0 1742 1743 def __next__(self): 1744 if self._index < len(self._ilist): 1745 # self._ilist.print_internal_list(f'ListIterator[{self._index}]. {{}}') 1746 result = self._ilist[self._index] 1747 self._index += 1 1748 return result 1749 else: 1750 raise StopIteration 1751 1752 def __iter__(self): 1753 return self
class
IListReversedIterator:
1756class IListReversedIterator: 1757 def __init__(self, ilist: IList) -> None: 1758 self._ilist = ilist 1759 self._index = len(ilist) - 1 1760 1761 def __next__(self): 1762 if self._index >= 0: 1763 result = self._ilist[self._index] 1764 self._index -= 1 1765 return result 1766 else: 1767 raise StopIteration 1768 1769 def __iter__(self): 1770 return self
class
TList:
1773class TList: 1774 def map_to_shared_memory(self, shared_memory: 'SharedMemory', obj: list) -> Tuple[list, Offset, Size]: 1775 obj = IList(shared_memory, obj=obj) 1776 return obj, obj._offset, obj._obj_size 1777 1778 def init_from_shared_memory(self, shared_memory: 'SharedMemory', offset: Offset) -> None: 1779 if ObjectType.tlist != read_uint64(shared_memory.base_address, offset): 1780 raise WrongObjectTypeError 1781 1782 return IList(shared_memory, offset) 1783 1784 def destroy(self, shared_memory: 'SharedMemory', offset: Offset): 1785 shared_memory.free(offset)
class
TupleOffsets(enum.IntEnum):
An enumeration.
size =
<TupleOffsets.size: 0>
Inherited Members
- enum.Enum
- name
- value
- builtins.int
- conjugate
- bit_length
- to_bytes
- from_bytes
- as_integer_ratio
- real
- imag
- numerator
- denominator
class
TupleFieldOffsets(enum.IntEnum):
An enumeration.
item_offset =
<TupleFieldOffsets.item_offset: 0>
Inherited Members
- enum.Enum
- name
- value
- builtins.int
- conjugate
- bit_length
- to_bytes
- from_bytes
- as_integer_ratio
- real
- imag
- numerator
- denominator
class
TTuple:
1800class TTuple: 1801 def map_to_shared_memory(self, shared_memory: 'SharedMemory', obj: tuple) -> Tuple[tuple, Offset, Size]: 1802 offset, real_size = shared_memory.malloc(ObjectType.ttuple, bs * len(TupleOffsets) + len(obj) * bs * len(TupleFieldOffsets)) 1803 if (1, [2, 3]) == obj: 1804 shared_memory.offset_to_be_monitored = offset 1805 1806 write_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * TupleOffsets.size, len(obj)) 1807 for i, item in enumerate(obj): 1808 item_mapped_obj, item_offset, item_size = shared_memory.put_obj(item) 1809 write_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * len(TupleOffsets) + i * bs * len(TupleFieldOffsets), item_offset) 1810 1811 return obj, offset, real_size 1812 1813 def init_from_shared_memory(self, shared_memory: 'SharedMemory', offset: Offset) -> None: 1814 if ObjectType.ttuple != read_uint64(shared_memory.base_address, offset): 1815 raise WrongObjectTypeError 1816 1817 result_list = list() 1818 size = read_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * TupleOffsets.size) 1819 for i in range(size): 1820 item_offset = read_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * len(TupleOffsets) + i * bs * len(TupleFieldOffsets)) 1821 result_list.append(shared_memory.get_obj(item_offset)) 1822 1823 return tuple(result_list) 1824 1825 def destroy(self, shared_memory: 'SharedMemory', offset: Offset): 1826 shared_memory.free(offset)
class
DictOffsets(enum.IntEnum):
An enumeration.
data_tuple_offset =
<DictOffsets.data_tuple_offset: 0>
Inherited Members
- enum.Enum
- name
- value
- builtins.int
- conjugate
- bit_length
- to_bytes
- from_bytes
- as_integer_ratio
- real
- imag
- numerator
- denominator
class
TDict:
1837class TDict: 1838 def map_to_shared_memory(self, shared_memory: 'SharedMemory', obj: dict) -> Tuple[dict, Offset, Size]: 1839 offset, real_size = shared_memory.malloc(ObjectType.tdict, bs * len(DictOffsets)) 1840 item_mapped_obj, item_offset, item_size = shared_memory.put_obj(tuple(obj.items())) 1841 write_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * DictOffsets.data_tuple_offset, item_offset) 1842 return obj, offset, real_size 1843 1844 def init_from_shared_memory(self, shared_memory: 'SharedMemory', offset: Offset) -> None: 1845 if ObjectType.tdict != read_uint64(shared_memory.base_address, offset): 1846 raise WrongObjectTypeError 1847 1848 item_offset = read_uint64(shared_memory.base_address, offset + bs * len(BaseObjOffsets) + bs * DictOffsets.data_tuple_offset) 1849 result_tuple = shared_memory.get_obj(item_offset) 1850 return dict(result_tuple) 1851 1852 def destroy(self, shared_memory: 'SharedMemory', offset: Offset): 1853 shared_memory.free(offset)
codec_by_type: Dict[ObjectType, TBase] =
{<ObjectType.tnone: 2>: <TNone object>, <ObjectType.tint: 4>: <TInt object>, <ObjectType.tbool: 3>: <TBool object>, <ObjectType.tfloat: 5>: <TFloat object>, <ObjectType.tbytes: 8>: <TBytes object>, <ObjectType.tbytearray: 9>: <TBytearray object>, <ObjectType.tstr: 7>: <TStr object>, <ObjectType.tlist: 11>: <TList object>, <ObjectType.ttuple: 10>: <TTuple object>, <ObjectType.tdict: 12>: <TDict object>}
class
MessageOffsets(enum.IntEnum):
1876class MessageOffsets(IntEnum): 1877 previous_message_offset = 0 1878 next_message_offset = 1 1879 item_offset = 2
An enumeration.
previous_message_offset =
<MessageOffsets.previous_message_offset: 0>
next_message_offset =
<MessageOffsets.next_message_offset: 1>
item_offset =
<MessageOffsets.item_offset: 2>
Inherited Members
- enum.Enum
- name
- value
- builtins.int
- conjugate
- bit_length
- to_bytes
- from_bytes
- as_integer_ratio
- real
- imag
- numerator
- denominator
@contextmanager
def
wait_my_turn( shared_memory: SharedMemory, time_limit: typing.Union[int, float, NoneType] = None, periodic_sleep_time: typing.Union[int, float, NoneType] = 1e-09):
2737@contextmanager 2738def wait_my_turn(shared_memory: SharedMemory, time_limit: Optional[RationalNumber] = None, periodic_sleep_time: Optional[RationalNumber] = 0.000000001): 2739 shared_memory.wait_my_turn(time_limit, periodic_sleep_time) 2740 try: 2741 yield 2742 finally: 2743 shared_memory.release()
def
numpy_array_memory_size(np_shape, np_dtype):
def
numpy_array_made_from_pointer_memory_size(np_shape, ctypes_type):
def
make_numpy_array_from_obj_offset( shared_memory: SharedMemory, offset: int, np_shape, ctypes_type=None) -> Any:
2760def make_numpy_array_from_obj_offset(shared_memory: SharedMemory, offset: int, np_shape, ctypes_type = None) -> Any: 2761 if ctypes_type is None: 2762 ctypes_type = ctypes.c_uint8 2763 2764 data_offset, data_size = shared_memory.get_obj_buffer_2(offset) 2765 num_elements = np.prod(np_shape) 2766 np_array_size = num_elements * ctypes.sizeof(ctypes_type) 2767 if data_size < np_array_size: 2768 raise ObjBufferIsSmallerThanRequestedNumpyArrayError(data_size, np_array_size) 2769 2770 data_address = shared_memory.base_address + data_offset 2771 void_ptr = ctypes.c_void_p(data_address) 2772 # actual_ptr = ctypes.cast(void_ptr, ctypes.POINTER(ctypes_type * num_elements)) 2773 actual_ptr = ctypes.cast(void_ptr, ctypes.POINTER(ctypes_type)) 2774 return np.ctypeslib.as_array(actual_ptr, shape=np_shape)