cengal.data_containers.dynamic_list_of_pieces.versions.v_1.dynamic_list_of_pieces__python
1#!/usr/bin/env python 2# coding=utf-8 3 4# Copyright © 2012-2024 ButenkoMS. All rights reserved. Contacts: <gtalk@butenkoms.space> 5# 6# Licensed under the Apache License, Version 2.0 (the "License"); 7# you may not use this file except in compliance with the License. 8# You may obtain a copy of the License at 9# 10# http://www.apache.org/licenses/LICENSE-2.0 11# 12# Unless required by applicable law or agreed to in writing, software 13# distributed under the License is distributed on an "AS IS" BASIS, 14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15# See the License for the specific language governing permissions and 16# limitations under the License. 17 18from collections import deque 19from cengal.code_flow_control.none_or import none_or 20from cengal.code_flow_control.smart_values import ValueExistence 21from cengal.data_containers.fast_fifo import FIFODequeWithLengthControl 22from typing import Optional, Tuple, List 23 24""" 25Динамический кусочный массив данных. Заполняется отдельными массивами байт (например принятых по TCP во время 26ассинхронного режима работы сокета), но функционирует так, как буд-то это цельный непрерывный массив. 27Дает значительный выигрыш в скорости по сравнению с обновлением встроенного контейнера list() за счет гораздо 28меньшего количества аллокаций и копирований памяти. 29""" 30 31__author__ = "ButenkoMS <gtalk@butenkoms.space>" 32__copyright__ = "Copyright © 2012-2024 ButenkoMS. All rights reserved. Contacts: <gtalk@butenkoms.space>" 33__credits__ = ["ButenkoMS <gtalk@butenkoms.space>", ] 34__license__ = "Apache License, Version 2.0" 35__version__ = "4.4.1" 36__maintainer__ = "ButenkoMS <gtalk@butenkoms.space>" 37__email__ = "gtalk@butenkoms.space" 38# __status__ = "Prototype" 39__status__ = "Development" 40# __status__ = "Production" 41 42 43# print("Just a lowly interpreted script.") 44 45 46class DynamicListOfPieces: 47 def __init__(self, joiner=None, on_hold_limit=None): 48 self._init_param__joiner = joiner 49 self._init_param__on_hold_limit = on_hold_limit 50 51 self._joiner = none_or(joiner, b'') 52 self._data = list() 53 self._data_length = 0 54 55 self._offset_limit = on_hold_limit or 1000 56 self._useful_size = 0 57 self._real_size = 0 58 59 def add_piece_of_data(self, piece_of_data): 60 self._data.append(piece_of_data) 61 piece_of_data_len = len(piece_of_data) 62 self._data_length += piece_of_data_len 63 self._useful_size += 1 64 self._real_size += 1 65 66 def size(self): 67 return self._data_length 68 69 def qnt(self): 70 return self._useful_size 71 72 # @profile 73 def get_data(self, size): 74 """ 75 :param size: 76 :return: data or None (if size > self._data_length) 77 """ 78 if size > self._data_length: 79 return None 80 elif size <= 0: 81 # return self._joiner.join(list()) # this is 1.6 times more faster than copy.copy(self._joiner) 82 return self._joiner 83 84 result_data_length = 0 85 pieces_qnt = delta_range = self._real_size - self._useful_size 86 87 if size < len(self._data[pieces_qnt]): 88 result = self._data[pieces_qnt][:size] 89 self._data_length -= size 90 self._data[pieces_qnt] = self._data[pieces_qnt][size:] 91 return result 92 elif size == len(self._data[pieces_qnt]): 93 result = self._data[pieces_qnt] 94 self._data_length -= size 95 pieces_qnt += 1 96 self._useful_size -= 1 97 if pieces_qnt >= self._offset_limit: 98 self._real_size = self._useful_size 99 self._data = self._data[pieces_qnt:] 100 return result 101 else: 102 for piece_num in range(delta_range, self._real_size): 103 piece = self._data[piece_num] 104 piece_length = len(piece) 105 if (result_data_length + piece_length) <= size: 106 result_data_length += piece_length 107 pieces_qnt += 1 108 else: 109 break 110 111 result_list = list() 112 if size > result_data_length: 113 splittable_piece = self._data[pieces_qnt] 114 # if (len(splittable_piece) > 4096 * 1024**2) and (type(splittable_piece) is not memoryview): 115 # splittable_piece = memoryview(splittable_piece) 116 needed = size - result_data_length 117 118 last_current_data_part = splittable_piece[:needed] 119 first_undecoded_data_part = splittable_piece[needed:] 120 121 # result_list.append(last_current_data_part) 122 self._data[pieces_qnt] = first_undecoded_data_part 123 124 result_list = self._data[delta_range:pieces_qnt] 125 result_list.append(last_current_data_part) 126 127 else: 128 result_list = self._data[delta_range:pieces_qnt] 129 130 self._useful_size = self._real_size - pieces_qnt 131 if pieces_qnt >= self._offset_limit: 132 self._real_size = self._useful_size 133 self._data = self._data[pieces_qnt:] 134 135 self._data_length -= size 136 137 # result = self._joiner.join(result_list) 138 # 139 # if len(result) > 4096: 140 # result = memoryview(result) 141 # 142 # return result 143 144 return memoryview(self._joiner.join(result_list)) 145 146 # @profile 147 def read_data(self, size): 148 """ 149 :param size: 150 :return: data or None (if size > self._data_length) 151 """ 152 if size > self._data_length: 153 return None 154 elif size <= 0: 155 # return self._joiner.join(list()) # this is 1.6 times more faster than copy.copy(self._joiner) 156 return self._joiner 157 158 result_data_length = 0 159 pieces_qnt = delta_range = self._real_size - self._useful_size 160 161 if size < len(self._data[pieces_qnt]): 162 return self._data[pieces_qnt][:size] 163 elif size == len(self._data[pieces_qnt]): 164 return self._data[pieces_qnt] 165 else: 166 for piece_num in range(delta_range, self._real_size): 167 piece = self._data[piece_num] 168 piece_length = len(piece) 169 if (result_data_length + piece_length) <= size: 170 result_data_length += piece_length 171 pieces_qnt += 1 172 else: 173 break 174 175 result_list = list() 176 if size > result_data_length: 177 splittable_piece = self._data[pieces_qnt] 178 needed = size - result_data_length 179 180 result_list = self._data[delta_range:pieces_qnt] 181 result_list.append(splittable_piece[:needed]) 182 183 else: 184 result_list = self._data[delta_range:pieces_qnt] 185 186 # result = self._joiner.join(result_list) 187 # return result 188 189 return self._joiner.join(result_list) 190 191 def remove(self): 192 pass 193 194 def __copy__(self): 195 return DynamicListOfPieces(self._init_param__joiner, 196 self._init_param__on_hold_limit) 197 198 199class DynamicListOfPiecesWithLengthControl(DynamicListOfPieces): 200 def __init__(self, joiner=None, on_hold_limit=None, on_hold_data_size_limit=None, 201 external_data_length: ValueExistence=None, 202 external_deletable_data_full_size: ValueExistence=None): 203 super(DynamicListOfPiecesWithLengthControl, self).__init__(joiner, on_hold_limit) 204 205 self._removed = False 206 207 self._init_param__on_hold_data_size_limit = on_hold_data_size_limit 208 self._init_param__external_data_length = external_data_length 209 self._init_param__external_deletable_data_full_size = external_deletable_data_full_size 210 211 self._on_hold_data_size_limit = on_hold_data_size_limit or 1024**2 212 self._deletable_data_full_size = 0 213 214 self._external_data_length = external_data_length or ValueExistence(False, 0) 215 self._external_deletable_data_full_size = external_deletable_data_full_size or ValueExistence(False, 0) 216 217 def add_piece_of_data(self, piece_of_data): 218 self._data.append(piece_of_data) 219 piece_of_data_len = len(piece_of_data) 220 self._data_length += piece_of_data_len 221 self._external_data_length.value += piece_of_data_len 222 self._useful_size += 1 223 self._real_size += 1 224 225 # @profile 226 def get_data(self, size): 227 """ 228 :param size: 229 :return: data or None (if size > self._data_length) 230 """ 231 if size > self._data_length: 232 return None 233 elif size <= 0: 234 # return self._joiner.join(list()) # this is 1.6 times more faster than copy.copy(self._joiner) 235 return self._joiner 236 237 result_data_length = 0 238 pieces_qnt = delta_range = self._real_size - self._useful_size 239 240 first_piece_len = len(self._data[pieces_qnt]) 241 if size < first_piece_len: 242 result = self._data[pieces_qnt][:size] 243 self._data_length -= size 244 self._external_data_length.value -= size 245 self._deletable_data_full_size += size 246 self._external_deletable_data_full_size.value += size 247 self._data[pieces_qnt] = self._data[pieces_qnt][size:] 248 return result 249 elif size == first_piece_len: 250 result = self._data[pieces_qnt] 251 self._data_length -= size 252 self._external_data_length.value -= size 253 self._deletable_data_full_size += size 254 self._external_deletable_data_full_size.value += size 255 pieces_qnt += 1 256 self._useful_size -= 1 257 if (pieces_qnt >= self._offset_limit) or (self._deletable_data_full_size > self._on_hold_data_size_limit): 258 self._external_deletable_data_full_size.value -= self._deletable_data_full_size 259 self._deletable_data_full_size = 0 260 self._real_size = self._useful_size 261 self._data = self._data[pieces_qnt:] 262 return result 263 else: 264 for piece_num in range(delta_range, self._real_size): 265 piece = self._data[piece_num] 266 piece_length = len(piece) 267 if (result_data_length + piece_length) <= size: 268 result_data_length += piece_length 269 pieces_qnt += 1 270 else: 271 break 272 273 result_list = list() 274 if size > result_data_length: 275 splittable_piece = self._data[pieces_qnt] 276 # if (len(splittable_piece) > 4096 * 1024**2) and (type(splittable_piece) is not memoryview): 277 # splittable_piece = memoryview(splittable_piece) 278 needed = size - result_data_length 279 280 last_current_data_part = splittable_piece[:needed] 281 first_undecoded_data_part = splittable_piece[needed:] 282 283 # result_list.append(last_current_data_part) 284 self._data[pieces_qnt] = first_undecoded_data_part 285 286 result_list = self._data[delta_range:pieces_qnt] 287 result_list.append(last_current_data_part) 288 289 else: 290 result_list = self._data[delta_range:pieces_qnt] 291 292 self._deletable_data_full_size += size 293 self._external_deletable_data_full_size.value += size 294 self._useful_size = self._real_size - pieces_qnt 295 if pieces_qnt >= self._offset_limit: 296 self._external_deletable_data_full_size.value -= self._deletable_data_full_size 297 self._deletable_data_full_size = 0 298 self._real_size = self._useful_size 299 self._data = self._data[pieces_qnt:] 300 301 self._data_length -= size 302 self._external_data_length.value -= size 303 304 # result = self._joiner.join(result_list) 305 # 306 # if len(result) > 4096: 307 # result = memoryview(result) 308 # 309 # return result 310 311 return memoryview(self._joiner.join(result_list)) 312 313 def remove(self): 314 if not self._removed: 315 self._external_data_length.value -= self._data_length 316 self._external_deletable_data_full_size.value -= self._deletable_data_full_size 317 self._removed = True 318 319 def __copy__(self): 320 return DynamicListOfPiecesWithLengthControl(self._init_param__joiner, 321 self._init_param__on_hold_limit, 322 self._init_param__on_hold_data_size_limit, 323 self._init_param__external_data_length, 324 self._init_param__external_deletable_data_full_size) 325 326 def __del__(self): 327 self.remove() 328 329 330class DynamicListOfPiecesDequeWithLengthControl: 331 def __init__(self, joiner=None, external_data_length: ValueExistence=None): 332 self._removed = False 333 334 self._init_param__joiner = joiner 335 self._init_param__external_data_length = external_data_length 336 337 self._joiner = none_or(joiner, b'') 338 self._data = deque() 339 self._data_length = 0 340 341 self._real_size = 0 342 343 self._external_data_length = external_data_length or ValueExistence(False, 0) 344 345 def size(self): 346 return self._data_length 347 348 def qnt(self): 349 return self._real_size 350 351 def add_piece_of_data(self, piece_of_data): 352 self._data.append(piece_of_data) 353 piece_of_data_len = len(piece_of_data) 354 self._data_length += piece_of_data_len 355 self._external_data_length.value += piece_of_data_len 356 self._real_size += 1 357 358 # @profile 359 def get_data(self, size): 360 """ 361 :param size: 362 :return: data or None (if size > self._data_length) 363 """ 364 if size > self._data_length: 365 return None 366 elif size <= 0: 367 # return self._joiner.join(list()) # this is 1.6 times more faster than copy.copy(self._joiner) 368 return self._joiner 369 370 first_piece_len = len(self._data[0]) 371 if size < first_piece_len: 372 current_piece = memoryview(self._data[0]) 373 result = current_piece[:size] 374 self._data[0] = current_piece[size:] 375 self._data_length -= size 376 self._external_data_length.value -= size 377 return result 378 elif size == first_piece_len: 379 self._data_length -= size 380 self._external_data_length.value -= size 381 self._real_size -= 1 382 # return memoryview(self._data.popleft()) 383 return self._data.popleft() 384 else: 385 result_data = deque() 386 result_size = 0 387 388 while self._real_size and (result_size < size): 389 another_piece = self._data.popleft() 390 another_piece_len = len(another_piece) 391 result_data.append(another_piece) 392 self._data_length -= another_piece_len 393 self._external_data_length.value -= another_piece_len 394 self._real_size -= 1 395 result_size += another_piece_len 396 if (result_size < size) and ((len(self._data[0]) + result_size) > size): 397 break 398 399 if result_size < size: 400 delta_size = size - result_size 401 current_piece = memoryview(self._data[0]) 402 result_data.append(current_piece[:delta_size]) 403 self._data[0] = current_piece[delta_size:] 404 self._data_length -= delta_size 405 self._external_data_length.value -= delta_size 406 407 # return memoryview(self._joiner.join(result_data)) 408 return self._joiner.join(result_data) 409 410 def read_data(self, size): 411 """ 412 :param size: 413 :return: data or None (if size > self._data_length) 414 """ 415 if size > self._data_length: 416 return None 417 elif size <= 0: 418 # return self._joiner.join(list()) # this is 1.6 times more faster than copy.copy(self._joiner) 419 return self._joiner 420 421 first_piece_len = len(self._data[0]) 422 if size < first_piece_len: 423 current_piece = memoryview(self._data[0]) 424 return current_piece[:size] 425 elif size == first_piece_len: 426 return memoryview(self._data[0]) 427 else: 428 result_data = deque() 429 result_size = 0 430 431 index = 0 432 while result_size < size: 433 another_piece = self._data[index] 434 index += 1 435 another_piece_len = len(another_piece) 436 result_data.append(another_piece) 437 result_size += another_piece_len 438 if (result_size < size) and ((len(self._data[index]) + result_size) > size): 439 break 440 441 if result_size < size: 442 delta_size = size - result_size 443 current_piece = memoryview(self._data[index]) 444 result_data.append(current_piece[:delta_size]) 445 446 return self._joiner.join(result_data) 447 448 # @profile 449 def get_data_pieces(self, size) -> List[bytes]: 450 """ 451 :param size: 452 :return: data or None (if size > self._data_length) 453 """ 454 if size > self._data_length: 455 return None 456 elif size <= 0: 457 # return self._joiner.join(list()) # this is 1.6 times more faster than copy.copy(self._joiner) 458 return [self._joiner] 459 460 first_piece_len = len(self._data[0]) 461 if size <= first_piece_len: 462 self._data_length -= first_piece_len 463 self._external_data_length.value -= first_piece_len 464 self._real_size -= 1 465 return [self._data.popleft()] 466 else: 467 result_data = deque() 468 result_size = 0 469 470 while self._real_size and (result_size < size): 471 another_piece = self._data.popleft() 472 another_piece_len = len(another_piece) 473 result_data.append(another_piece) 474 self._data_length -= another_piece_len 475 self._external_data_length.value -= another_piece_len 476 self._real_size -= 1 477 result_size += another_piece_len 478 479 return result_data 480 481 def read_data_pieces(self, size): 482 """ 483 :param size: 484 :return: data or None (if size > self._data_length) 485 """ 486 if size > self._data_length: 487 return None 488 elif size <= 0: 489 # return self._joiner.join(list()) # this is 1.6 times more faster than copy.copy(self._joiner) 490 return [self._joiner] 491 492 first_piece_len = len(self._data[0]) 493 if size <= first_piece_len: 494 return [memoryview(self._data[0])] 495 else: 496 result_data = deque() 497 result_size = 0 498 499 index = 0 500 while result_size < size: 501 another_piece = memoryview(self._data[index]) 502 index += 1 503 another_piece_len = len(another_piece) 504 result_data.append(another_piece) 505 result_size += another_piece_len 506 507 return result_data 508 509 def get_data_at_least(self, size): 510 return self._joiner.join(self.get_data_pieces(size)) 511 512 def read_data_at_least(self, size): 513 return self._joiner.join(self.read_data_pieces(size)) 514 515 def get_data_nearly(self, size): 516 return self._joiner.join(self.get_data_pieces(min(size, self.size()))) 517 518 def read_data_nearly(self, size): 519 return self._joiner.join(self.read_data_pieces(min(size, self.size()))) 520 521 def item_by_index(self, index: int) -> Tuple[int, int]: 522 if index >= self._data_length: 523 return -1, -1 524 525 offset = 0 526 for item_index, item in enumerate(self._data): 527 item_len = len(item) 528 offset += item_len 529 if index < offset: 530 index_within_item = item_len - (offset - index) 531 return item_index, index_within_item 532 533 def startswith(self, 534 __prefix, 535 __start = None, 536 __end = None 537 ) -> bool: 538 if not self._data_length: 539 return False 540 541 if isinstance(__prefix, tuple): 542 prefix_len = max((len(item) for item in __prefix)) 543 else: 544 prefix_len = len(__prefix) 545 546 start_index = __start or 0 547 prefix_end_index = start_index + prefix_len 548 if __end is None: 549 end_index = prefix_end_index 550 else: 551 end_index = __end 552 553 end_index = min(end_index, prefix_end_index) 554 if end_index < start_index: 555 return False 556 557 start_item_index, index_within_start_item = self.item_by_index(start_index) 558 if -1 == start_item_index: 559 return False 560 561 end_item_index, index_within_end_item = self.item_by_index(end_index) 562 if -1 == end_item_index: 563 end_item_index = self._real_size -1 564 index_within_end_item = len(self._data[end_item_index]) - 1 565 566 return self._joiner.join(self._data[start_item_index: end_item_index]).startswith( 567 __prefix, index_within_start_item, index_within_end_item) 568 569 def find( 570 self, __sub, __start = None, __end = None 571 ) -> int: 572 if not self._data_length: 573 return False 574 575 start_index = __start or 0 576 end_index = __end or (self._data_length - 1) 577 578 start_item_index, index_within_start_item = self.item_by_index(start_index) 579 if -1 == start_item_index: 580 return False 581 582 end_item_index, index_within_end_item = self.item_by_index(end_index) 583 if -1 == end_item_index: 584 end_item_index = self._real_size -1 585 index_within_end_item = len(self._data[end_item_index]) - 1 586 587 return self._joiner.join(self._data[start_item_index: end_item_index]).find( 588 __sub, index_within_start_item, index_within_end_item) 589 590 def clear(self): 591 self.get_data_pieces(self._real_size) 592 593 def remove(self): 594 if not self._removed: 595 self._external_data_length.value -= self._data_length 596 self._removed = True 597 598 def __copy__(self): 599 return DynamicListOfPiecesDequeWithLengthControl( 600 self._init_param__joiner, self._init_param__external_data_length) 601 602 def __del__(self): 603 self.remove() 604 605 606# TODO: implement in Cython: 607class DynamicListOfPiecesMemoryviewDequeWithLengthControl: 608 def __init__(self, joiner=None, external_data_length: Optional[ValueExistence] = None, mem_buffer_size: Optional[ValueExistence] = None, output_fifo: Optional[FIFODequeWithLengthControl] = None): 609 self._removed = False 610 611 self._init_param__joiner = joiner 612 self._init_param__external_data_length = external_data_length 613 self._init_param__mem_buffer_size = mem_buffer_size 614 615 self._joiner = none_or(joiner, b'') 616 self._data = deque() 617 self._data_length = 0 618 619 self._real_size = 0 620 621 self._external_data_length = external_data_length or ValueExistence(False, 0) 622 623 self.mem_buffer_size = mem_buffer_size or ValueExistence(False, 1024) 624 self.mem_buffer = memoryview(bytearray(self.mem_buffer_size.value)) 625 self._external_data_length.value += self.mem_buffer_size.value 626 627 self.mem_buffer_offset = 0 628 self.mem_buffer_nbytes = 0 629 self.mem_buffer_diff = 0 630 self.mem_buffer_message_nbytes = 0 631 632 self.output_data = output_fifo or FIFODequeWithLengthControl(self._external_data_length) 633 634 def _allocate_new_mem_buffer(self): 635 self._external_data_length.value -= len(self.mem_buffer.obj) 636 self.mem_buffer = memoryview(bytearray(self.mem_buffer_size.value)) 637 self._external_data_length.value += self.mem_buffer_size.value 638 639 def size(self): 640 return self._data_length 641 642 def qnt(self): 643 return self._real_size 644 645 def add_piece_of_data(self, piece_of_data): 646 raise NotImplemented() 647 648 def _add_piece_of_data(self, piece_of_data): 649 self._data.append(piece_of_data) 650 piece_of_data_len = len(piece_of_data) 651 self._data_length += piece_of_data_len 652 self._external_data_length.value += piece_of_data_len 653 self._real_size += 1 654 655 def piece_of_data_had_been_added(self, nbytes): 656 self.mem_buffer = self.mem_buffer[nbytes:] 657 self.mem_buffer_nbytes += nbytes 658 self.mem_buffer_diff += nbytes 659 660 if not self.mem_buffer: 661 if self.mem_buffer_diff: 662 self._add_piece_of_data(memoryview(self.mem_buffer.obj)[self.mem_buffer_offset:]) 663 self._allocate_new_mem_buffer() 664 665 # @profile 666 def get_data(self, size): 667 """ 668 :param size: 669 :return: data or None (if size > self._data_length) 670 """ 671 if size > self._data_length: 672 return None 673 elif size <= 0: 674 # return self._joiner.join(list()) # this is 1.6 times more faster than copy.copy(self._joiner) 675 return self._joiner 676 677 first_piece_len = len(self._data[0]) 678 if size < first_piece_len: 679 current_piece = memoryview(self._data[0]) 680 result = current_piece[:size] 681 self._data[0] = current_piece[size:] 682 self._data_length -= size 683 self._external_data_length.value -= size 684 return result 685 elif size == first_piece_len: 686 self._data_length -= size 687 self._external_data_length.value -= size 688 self._real_size -= 1 689 # return memoryview(self._data.popleft()) 690 return self._data.popleft() 691 else: 692 result_data = deque() 693 result_size = 0 694 695 while self._real_size and (result_size < size): 696 another_piece = self._data.popleft() 697 another_piece_len = len(another_piece) 698 result_data.append(another_piece) 699 self._data_length -= another_piece_len 700 self._external_data_length.value -= another_piece_len 701 self._real_size -= 1 702 result_size += another_piece_len 703 if (result_size < size) and ((len(self._data[0]) + result_size) > size): 704 break 705 706 if result_size < size: 707 delta_size = size - result_size 708 current_piece = memoryview(self._data[0]) 709 result_data.append(current_piece[:delta_size]) 710 self._data[0] = current_piece[delta_size:] 711 self._data_length -= delta_size 712 self._external_data_length.value -= delta_size 713 714 # return memoryview(self._joiner.join(result_data)) 715 return self._joiner.join(result_data) 716 717 def read_data(self, size): 718 raise NotImplemented() 719 720 def remove(self): 721 if not self._removed: 722 self._external_data_length.value -= self._data_length 723 self._external_data_length.value -= len(self.mem_buffer.obj) 724 self._removed = True 725 726 def __copy__(self): 727 return DynamicListOfPiecesDequeWithLengthControl( 728 self._init_param__joiner, self._init_param__external_data_length) 729 730 def __del__(self): 731 self.remove() 732 733 734def get_num_pieces_with_full_size_lesser_than(iter_object, full_size): 735 index = -1 736 if iter_object: 737 result_size = 0 738 index = 0 739 for piece in iter_object: 740 piece_len = len(piece) 741 if (result_size + piece_len) > full_size: 742 break 743 else: 744 result_size += piece_len 745 index += 1 746 return index 747 748 749def get_num_pieces_with_full_size_lesser_than_with_offset(iter_object, full_size, offset): 750 pieces_qnt = -1 751 if iter_object: 752 object_size = len(iter_object) 753 result_data_length = 0 754 pieces_qnt = 0 755 for piece_num in range(offset, object_size): 756 piece = iter_object[piece_num] 757 piece_length = len(piece) 758 if (result_data_length + piece_length) > full_size: 759 break 760 else: 761 result_data_length += piece_length 762 pieces_qnt += 1 763 return pieces_qnt
class
DynamicListOfPieces:
47class DynamicListOfPieces: 48 def __init__(self, joiner=None, on_hold_limit=None): 49 self._init_param__joiner = joiner 50 self._init_param__on_hold_limit = on_hold_limit 51 52 self._joiner = none_or(joiner, b'') 53 self._data = list() 54 self._data_length = 0 55 56 self._offset_limit = on_hold_limit or 1000 57 self._useful_size = 0 58 self._real_size = 0 59 60 def add_piece_of_data(self, piece_of_data): 61 self._data.append(piece_of_data) 62 piece_of_data_len = len(piece_of_data) 63 self._data_length += piece_of_data_len 64 self._useful_size += 1 65 self._real_size += 1 66 67 def size(self): 68 return self._data_length 69 70 def qnt(self): 71 return self._useful_size 72 73 # @profile 74 def get_data(self, size): 75 """ 76 :param size: 77 :return: data or None (if size > self._data_length) 78 """ 79 if size > self._data_length: 80 return None 81 elif size <= 0: 82 # return self._joiner.join(list()) # this is 1.6 times more faster than copy.copy(self._joiner) 83 return self._joiner 84 85 result_data_length = 0 86 pieces_qnt = delta_range = self._real_size - self._useful_size 87 88 if size < len(self._data[pieces_qnt]): 89 result = self._data[pieces_qnt][:size] 90 self._data_length -= size 91 self._data[pieces_qnt] = self._data[pieces_qnt][size:] 92 return result 93 elif size == len(self._data[pieces_qnt]): 94 result = self._data[pieces_qnt] 95 self._data_length -= size 96 pieces_qnt += 1 97 self._useful_size -= 1 98 if pieces_qnt >= self._offset_limit: 99 self._real_size = self._useful_size 100 self._data = self._data[pieces_qnt:] 101 return result 102 else: 103 for piece_num in range(delta_range, self._real_size): 104 piece = self._data[piece_num] 105 piece_length = len(piece) 106 if (result_data_length + piece_length) <= size: 107 result_data_length += piece_length 108 pieces_qnt += 1 109 else: 110 break 111 112 result_list = list() 113 if size > result_data_length: 114 splittable_piece = self._data[pieces_qnt] 115 # if (len(splittable_piece) > 4096 * 1024**2) and (type(splittable_piece) is not memoryview): 116 # splittable_piece = memoryview(splittable_piece) 117 needed = size - result_data_length 118 119 last_current_data_part = splittable_piece[:needed] 120 first_undecoded_data_part = splittable_piece[needed:] 121 122 # result_list.append(last_current_data_part) 123 self._data[pieces_qnt] = first_undecoded_data_part 124 125 result_list = self._data[delta_range:pieces_qnt] 126 result_list.append(last_current_data_part) 127 128 else: 129 result_list = self._data[delta_range:pieces_qnt] 130 131 self._useful_size = self._real_size - pieces_qnt 132 if pieces_qnt >= self._offset_limit: 133 self._real_size = self._useful_size 134 self._data = self._data[pieces_qnt:] 135 136 self._data_length -= size 137 138 # result = self._joiner.join(result_list) 139 # 140 # if len(result) > 4096: 141 # result = memoryview(result) 142 # 143 # return result 144 145 return memoryview(self._joiner.join(result_list)) 146 147 # @profile 148 def read_data(self, size): 149 """ 150 :param size: 151 :return: data or None (if size > self._data_length) 152 """ 153 if size > self._data_length: 154 return None 155 elif size <= 0: 156 # return self._joiner.join(list()) # this is 1.6 times more faster than copy.copy(self._joiner) 157 return self._joiner 158 159 result_data_length = 0 160 pieces_qnt = delta_range = self._real_size - self._useful_size 161 162 if size < len(self._data[pieces_qnt]): 163 return self._data[pieces_qnt][:size] 164 elif size == len(self._data[pieces_qnt]): 165 return self._data[pieces_qnt] 166 else: 167 for piece_num in range(delta_range, self._real_size): 168 piece = self._data[piece_num] 169 piece_length = len(piece) 170 if (result_data_length + piece_length) <= size: 171 result_data_length += piece_length 172 pieces_qnt += 1 173 else: 174 break 175 176 result_list = list() 177 if size > result_data_length: 178 splittable_piece = self._data[pieces_qnt] 179 needed = size - result_data_length 180 181 result_list = self._data[delta_range:pieces_qnt] 182 result_list.append(splittable_piece[:needed]) 183 184 else: 185 result_list = self._data[delta_range:pieces_qnt] 186 187 # result = self._joiner.join(result_list) 188 # return result 189 190 return self._joiner.join(result_list) 191 192 def remove(self): 193 pass 194 195 def __copy__(self): 196 return DynamicListOfPieces(self._init_param__joiner, 197 self._init_param__on_hold_limit)
DynamicListOfPieces(joiner=None, on_hold_limit=None)
48 def __init__(self, joiner=None, on_hold_limit=None): 49 self._init_param__joiner = joiner 50 self._init_param__on_hold_limit = on_hold_limit 51 52 self._joiner = none_or(joiner, b'') 53 self._data = list() 54 self._data_length = 0 55 56 self._offset_limit = on_hold_limit or 1000 57 self._useful_size = 0 58 self._real_size = 0
def
get_data(self, size):
74 def get_data(self, size): 75 """ 76 :param size: 77 :return: data or None (if size > self._data_length) 78 """ 79 if size > self._data_length: 80 return None 81 elif size <= 0: 82 # return self._joiner.join(list()) # this is 1.6 times more faster than copy.copy(self._joiner) 83 return self._joiner 84 85 result_data_length = 0 86 pieces_qnt = delta_range = self._real_size - self._useful_size 87 88 if size < len(self._data[pieces_qnt]): 89 result = self._data[pieces_qnt][:size] 90 self._data_length -= size 91 self._data[pieces_qnt] = self._data[pieces_qnt][size:] 92 return result 93 elif size == len(self._data[pieces_qnt]): 94 result = self._data[pieces_qnt] 95 self._data_length -= size 96 pieces_qnt += 1 97 self._useful_size -= 1 98 if pieces_qnt >= self._offset_limit: 99 self._real_size = self._useful_size 100 self._data = self._data[pieces_qnt:] 101 return result 102 else: 103 for piece_num in range(delta_range, self._real_size): 104 piece = self._data[piece_num] 105 piece_length = len(piece) 106 if (result_data_length + piece_length) <= size: 107 result_data_length += piece_length 108 pieces_qnt += 1 109 else: 110 break 111 112 result_list = list() 113 if size > result_data_length: 114 splittable_piece = self._data[pieces_qnt] 115 # if (len(splittable_piece) > 4096 * 1024**2) and (type(splittable_piece) is not memoryview): 116 # splittable_piece = memoryview(splittable_piece) 117 needed = size - result_data_length 118 119 last_current_data_part = splittable_piece[:needed] 120 first_undecoded_data_part = splittable_piece[needed:] 121 122 # result_list.append(last_current_data_part) 123 self._data[pieces_qnt] = first_undecoded_data_part 124 125 result_list = self._data[delta_range:pieces_qnt] 126 result_list.append(last_current_data_part) 127 128 else: 129 result_list = self._data[delta_range:pieces_qnt] 130 131 self._useful_size = self._real_size - pieces_qnt 132 if pieces_qnt >= self._offset_limit: 133 self._real_size = self._useful_size 134 self._data = self._data[pieces_qnt:] 135 136 self._data_length -= size 137 138 # result = self._joiner.join(result_list) 139 # 140 # if len(result) > 4096: 141 # result = memoryview(result) 142 # 143 # return result 144 145 return memoryview(self._joiner.join(result_list))
:param size: :return: data or None (if size > self._data_length)
def
read_data(self, size):
148 def read_data(self, size): 149 """ 150 :param size: 151 :return: data or None (if size > self._data_length) 152 """ 153 if size > self._data_length: 154 return None 155 elif size <= 0: 156 # return self._joiner.join(list()) # this is 1.6 times more faster than copy.copy(self._joiner) 157 return self._joiner 158 159 result_data_length = 0 160 pieces_qnt = delta_range = self._real_size - self._useful_size 161 162 if size < len(self._data[pieces_qnt]): 163 return self._data[pieces_qnt][:size] 164 elif size == len(self._data[pieces_qnt]): 165 return self._data[pieces_qnt] 166 else: 167 for piece_num in range(delta_range, self._real_size): 168 piece = self._data[piece_num] 169 piece_length = len(piece) 170 if (result_data_length + piece_length) <= size: 171 result_data_length += piece_length 172 pieces_qnt += 1 173 else: 174 break 175 176 result_list = list() 177 if size > result_data_length: 178 splittable_piece = self._data[pieces_qnt] 179 needed = size - result_data_length 180 181 result_list = self._data[delta_range:pieces_qnt] 182 result_list.append(splittable_piece[:needed]) 183 184 else: 185 result_list = self._data[delta_range:pieces_qnt] 186 187 # result = self._joiner.join(result_list) 188 # return result 189 190 return self._joiner.join(result_list)
:param size: :return: data or None (if size > self._data_length)
200class DynamicListOfPiecesWithLengthControl(DynamicListOfPieces): 201 def __init__(self, joiner=None, on_hold_limit=None, on_hold_data_size_limit=None, 202 external_data_length: ValueExistence=None, 203 external_deletable_data_full_size: ValueExistence=None): 204 super(DynamicListOfPiecesWithLengthControl, self).__init__(joiner, on_hold_limit) 205 206 self._removed = False 207 208 self._init_param__on_hold_data_size_limit = on_hold_data_size_limit 209 self._init_param__external_data_length = external_data_length 210 self._init_param__external_deletable_data_full_size = external_deletable_data_full_size 211 212 self._on_hold_data_size_limit = on_hold_data_size_limit or 1024**2 213 self._deletable_data_full_size = 0 214 215 self._external_data_length = external_data_length or ValueExistence(False, 0) 216 self._external_deletable_data_full_size = external_deletable_data_full_size or ValueExistence(False, 0) 217 218 def add_piece_of_data(self, piece_of_data): 219 self._data.append(piece_of_data) 220 piece_of_data_len = len(piece_of_data) 221 self._data_length += piece_of_data_len 222 self._external_data_length.value += piece_of_data_len 223 self._useful_size += 1 224 self._real_size += 1 225 226 # @profile 227 def get_data(self, size): 228 """ 229 :param size: 230 :return: data or None (if size > self._data_length) 231 """ 232 if size > self._data_length: 233 return None 234 elif size <= 0: 235 # return self._joiner.join(list()) # this is 1.6 times more faster than copy.copy(self._joiner) 236 return self._joiner 237 238 result_data_length = 0 239 pieces_qnt = delta_range = self._real_size - self._useful_size 240 241 first_piece_len = len(self._data[pieces_qnt]) 242 if size < first_piece_len: 243 result = self._data[pieces_qnt][:size] 244 self._data_length -= size 245 self._external_data_length.value -= size 246 self._deletable_data_full_size += size 247 self._external_deletable_data_full_size.value += size 248 self._data[pieces_qnt] = self._data[pieces_qnt][size:] 249 return result 250 elif size == first_piece_len: 251 result = self._data[pieces_qnt] 252 self._data_length -= size 253 self._external_data_length.value -= size 254 self._deletable_data_full_size += size 255 self._external_deletable_data_full_size.value += size 256 pieces_qnt += 1 257 self._useful_size -= 1 258 if (pieces_qnt >= self._offset_limit) or (self._deletable_data_full_size > self._on_hold_data_size_limit): 259 self._external_deletable_data_full_size.value -= self._deletable_data_full_size 260 self._deletable_data_full_size = 0 261 self._real_size = self._useful_size 262 self._data = self._data[pieces_qnt:] 263 return result 264 else: 265 for piece_num in range(delta_range, self._real_size): 266 piece = self._data[piece_num] 267 piece_length = len(piece) 268 if (result_data_length + piece_length) <= size: 269 result_data_length += piece_length 270 pieces_qnt += 1 271 else: 272 break 273 274 result_list = list() 275 if size > result_data_length: 276 splittable_piece = self._data[pieces_qnt] 277 # if (len(splittable_piece) > 4096 * 1024**2) and (type(splittable_piece) is not memoryview): 278 # splittable_piece = memoryview(splittable_piece) 279 needed = size - result_data_length 280 281 last_current_data_part = splittable_piece[:needed] 282 first_undecoded_data_part = splittable_piece[needed:] 283 284 # result_list.append(last_current_data_part) 285 self._data[pieces_qnt] = first_undecoded_data_part 286 287 result_list = self._data[delta_range:pieces_qnt] 288 result_list.append(last_current_data_part) 289 290 else: 291 result_list = self._data[delta_range:pieces_qnt] 292 293 self._deletable_data_full_size += size 294 self._external_deletable_data_full_size.value += size 295 self._useful_size = self._real_size - pieces_qnt 296 if pieces_qnt >= self._offset_limit: 297 self._external_deletable_data_full_size.value -= self._deletable_data_full_size 298 self._deletable_data_full_size = 0 299 self._real_size = self._useful_size 300 self._data = self._data[pieces_qnt:] 301 302 self._data_length -= size 303 self._external_data_length.value -= size 304 305 # result = self._joiner.join(result_list) 306 # 307 # if len(result) > 4096: 308 # result = memoryview(result) 309 # 310 # return result 311 312 return memoryview(self._joiner.join(result_list)) 313 314 def remove(self): 315 if not self._removed: 316 self._external_data_length.value -= self._data_length 317 self._external_deletable_data_full_size.value -= self._deletable_data_full_size 318 self._removed = True 319 320 def __copy__(self): 321 return DynamicListOfPiecesWithLengthControl(self._init_param__joiner, 322 self._init_param__on_hold_limit, 323 self._init_param__on_hold_data_size_limit, 324 self._init_param__external_data_length, 325 self._init_param__external_deletable_data_full_size) 326 327 def __del__(self): 328 self.remove()
DynamicListOfPiecesWithLengthControl( joiner=None, on_hold_limit=None, on_hold_data_size_limit=None, external_data_length: cengal.code_flow_control.smart_values.versions.v_2.smart_values.ValueExistence = None, external_deletable_data_full_size: cengal.code_flow_control.smart_values.versions.v_2.smart_values.ValueExistence = None)
201 def __init__(self, joiner=None, on_hold_limit=None, on_hold_data_size_limit=None, 202 external_data_length: ValueExistence=None, 203 external_deletable_data_full_size: ValueExistence=None): 204 super(DynamicListOfPiecesWithLengthControl, self).__init__(joiner, on_hold_limit) 205 206 self._removed = False 207 208 self._init_param__on_hold_data_size_limit = on_hold_data_size_limit 209 self._init_param__external_data_length = external_data_length 210 self._init_param__external_deletable_data_full_size = external_deletable_data_full_size 211 212 self._on_hold_data_size_limit = on_hold_data_size_limit or 1024**2 213 self._deletable_data_full_size = 0 214 215 self._external_data_length = external_data_length or ValueExistence(False, 0) 216 self._external_deletable_data_full_size = external_deletable_data_full_size or ValueExistence(False, 0)
def
get_data(self, size):
227 def get_data(self, size): 228 """ 229 :param size: 230 :return: data or None (if size > self._data_length) 231 """ 232 if size > self._data_length: 233 return None 234 elif size <= 0: 235 # return self._joiner.join(list()) # this is 1.6 times more faster than copy.copy(self._joiner) 236 return self._joiner 237 238 result_data_length = 0 239 pieces_qnt = delta_range = self._real_size - self._useful_size 240 241 first_piece_len = len(self._data[pieces_qnt]) 242 if size < first_piece_len: 243 result = self._data[pieces_qnt][:size] 244 self._data_length -= size 245 self._external_data_length.value -= size 246 self._deletable_data_full_size += size 247 self._external_deletable_data_full_size.value += size 248 self._data[pieces_qnt] = self._data[pieces_qnt][size:] 249 return result 250 elif size == first_piece_len: 251 result = self._data[pieces_qnt] 252 self._data_length -= size 253 self._external_data_length.value -= size 254 self._deletable_data_full_size += size 255 self._external_deletable_data_full_size.value += size 256 pieces_qnt += 1 257 self._useful_size -= 1 258 if (pieces_qnt >= self._offset_limit) or (self._deletable_data_full_size > self._on_hold_data_size_limit): 259 self._external_deletable_data_full_size.value -= self._deletable_data_full_size 260 self._deletable_data_full_size = 0 261 self._real_size = self._useful_size 262 self._data = self._data[pieces_qnt:] 263 return result 264 else: 265 for piece_num in range(delta_range, self._real_size): 266 piece = self._data[piece_num] 267 piece_length = len(piece) 268 if (result_data_length + piece_length) <= size: 269 result_data_length += piece_length 270 pieces_qnt += 1 271 else: 272 break 273 274 result_list = list() 275 if size > result_data_length: 276 splittable_piece = self._data[pieces_qnt] 277 # if (len(splittable_piece) > 4096 * 1024**2) and (type(splittable_piece) is not memoryview): 278 # splittable_piece = memoryview(splittable_piece) 279 needed = size - result_data_length 280 281 last_current_data_part = splittable_piece[:needed] 282 first_undecoded_data_part = splittable_piece[needed:] 283 284 # result_list.append(last_current_data_part) 285 self._data[pieces_qnt] = first_undecoded_data_part 286 287 result_list = self._data[delta_range:pieces_qnt] 288 result_list.append(last_current_data_part) 289 290 else: 291 result_list = self._data[delta_range:pieces_qnt] 292 293 self._deletable_data_full_size += size 294 self._external_deletable_data_full_size.value += size 295 self._useful_size = self._real_size - pieces_qnt 296 if pieces_qnt >= self._offset_limit: 297 self._external_deletable_data_full_size.value -= self._deletable_data_full_size 298 self._deletable_data_full_size = 0 299 self._real_size = self._useful_size 300 self._data = self._data[pieces_qnt:] 301 302 self._data_length -= size 303 self._external_data_length.value -= size 304 305 # result = self._joiner.join(result_list) 306 # 307 # if len(result) > 4096: 308 # result = memoryview(result) 309 # 310 # return result 311 312 return memoryview(self._joiner.join(result_list))
:param size: :return: data or None (if size > self._data_length)
Inherited Members
class
DynamicListOfPiecesDequeWithLengthControl:
331class DynamicListOfPiecesDequeWithLengthControl: 332 def __init__(self, joiner=None, external_data_length: ValueExistence=None): 333 self._removed = False 334 335 self._init_param__joiner = joiner 336 self._init_param__external_data_length = external_data_length 337 338 self._joiner = none_or(joiner, b'') 339 self._data = deque() 340 self._data_length = 0 341 342 self._real_size = 0 343 344 self._external_data_length = external_data_length or ValueExistence(False, 0) 345 346 def size(self): 347 return self._data_length 348 349 def qnt(self): 350 return self._real_size 351 352 def add_piece_of_data(self, piece_of_data): 353 self._data.append(piece_of_data) 354 piece_of_data_len = len(piece_of_data) 355 self._data_length += piece_of_data_len 356 self._external_data_length.value += piece_of_data_len 357 self._real_size += 1 358 359 # @profile 360 def get_data(self, size): 361 """ 362 :param size: 363 :return: data or None (if size > self._data_length) 364 """ 365 if size > self._data_length: 366 return None 367 elif size <= 0: 368 # return self._joiner.join(list()) # this is 1.6 times more faster than copy.copy(self._joiner) 369 return self._joiner 370 371 first_piece_len = len(self._data[0]) 372 if size < first_piece_len: 373 current_piece = memoryview(self._data[0]) 374 result = current_piece[:size] 375 self._data[0] = current_piece[size:] 376 self._data_length -= size 377 self._external_data_length.value -= size 378 return result 379 elif size == first_piece_len: 380 self._data_length -= size 381 self._external_data_length.value -= size 382 self._real_size -= 1 383 # return memoryview(self._data.popleft()) 384 return self._data.popleft() 385 else: 386 result_data = deque() 387 result_size = 0 388 389 while self._real_size and (result_size < size): 390 another_piece = self._data.popleft() 391 another_piece_len = len(another_piece) 392 result_data.append(another_piece) 393 self._data_length -= another_piece_len 394 self._external_data_length.value -= another_piece_len 395 self._real_size -= 1 396 result_size += another_piece_len 397 if (result_size < size) and ((len(self._data[0]) + result_size) > size): 398 break 399 400 if result_size < size: 401 delta_size = size - result_size 402 current_piece = memoryview(self._data[0]) 403 result_data.append(current_piece[:delta_size]) 404 self._data[0] = current_piece[delta_size:] 405 self._data_length -= delta_size 406 self._external_data_length.value -= delta_size 407 408 # return memoryview(self._joiner.join(result_data)) 409 return self._joiner.join(result_data) 410 411 def read_data(self, size): 412 """ 413 :param size: 414 :return: data or None (if size > self._data_length) 415 """ 416 if size > self._data_length: 417 return None 418 elif size <= 0: 419 # return self._joiner.join(list()) # this is 1.6 times more faster than copy.copy(self._joiner) 420 return self._joiner 421 422 first_piece_len = len(self._data[0]) 423 if size < first_piece_len: 424 current_piece = memoryview(self._data[0]) 425 return current_piece[:size] 426 elif size == first_piece_len: 427 return memoryview(self._data[0]) 428 else: 429 result_data = deque() 430 result_size = 0 431 432 index = 0 433 while result_size < size: 434 another_piece = self._data[index] 435 index += 1 436 another_piece_len = len(another_piece) 437 result_data.append(another_piece) 438 result_size += another_piece_len 439 if (result_size < size) and ((len(self._data[index]) + result_size) > size): 440 break 441 442 if result_size < size: 443 delta_size = size - result_size 444 current_piece = memoryview(self._data[index]) 445 result_data.append(current_piece[:delta_size]) 446 447 return self._joiner.join(result_data) 448 449 # @profile 450 def get_data_pieces(self, size) -> List[bytes]: 451 """ 452 :param size: 453 :return: data or None (if size > self._data_length) 454 """ 455 if size > self._data_length: 456 return None 457 elif size <= 0: 458 # return self._joiner.join(list()) # this is 1.6 times more faster than copy.copy(self._joiner) 459 return [self._joiner] 460 461 first_piece_len = len(self._data[0]) 462 if size <= first_piece_len: 463 self._data_length -= first_piece_len 464 self._external_data_length.value -= first_piece_len 465 self._real_size -= 1 466 return [self._data.popleft()] 467 else: 468 result_data = deque() 469 result_size = 0 470 471 while self._real_size and (result_size < size): 472 another_piece = self._data.popleft() 473 another_piece_len = len(another_piece) 474 result_data.append(another_piece) 475 self._data_length -= another_piece_len 476 self._external_data_length.value -= another_piece_len 477 self._real_size -= 1 478 result_size += another_piece_len 479 480 return result_data 481 482 def read_data_pieces(self, size): 483 """ 484 :param size: 485 :return: data or None (if size > self._data_length) 486 """ 487 if size > self._data_length: 488 return None 489 elif size <= 0: 490 # return self._joiner.join(list()) # this is 1.6 times more faster than copy.copy(self._joiner) 491 return [self._joiner] 492 493 first_piece_len = len(self._data[0]) 494 if size <= first_piece_len: 495 return [memoryview(self._data[0])] 496 else: 497 result_data = deque() 498 result_size = 0 499 500 index = 0 501 while result_size < size: 502 another_piece = memoryview(self._data[index]) 503 index += 1 504 another_piece_len = len(another_piece) 505 result_data.append(another_piece) 506 result_size += another_piece_len 507 508 return result_data 509 510 def get_data_at_least(self, size): 511 return self._joiner.join(self.get_data_pieces(size)) 512 513 def read_data_at_least(self, size): 514 return self._joiner.join(self.read_data_pieces(size)) 515 516 def get_data_nearly(self, size): 517 return self._joiner.join(self.get_data_pieces(min(size, self.size()))) 518 519 def read_data_nearly(self, size): 520 return self._joiner.join(self.read_data_pieces(min(size, self.size()))) 521 522 def item_by_index(self, index: int) -> Tuple[int, int]: 523 if index >= self._data_length: 524 return -1, -1 525 526 offset = 0 527 for item_index, item in enumerate(self._data): 528 item_len = len(item) 529 offset += item_len 530 if index < offset: 531 index_within_item = item_len - (offset - index) 532 return item_index, index_within_item 533 534 def startswith(self, 535 __prefix, 536 __start = None, 537 __end = None 538 ) -> bool: 539 if not self._data_length: 540 return False 541 542 if isinstance(__prefix, tuple): 543 prefix_len = max((len(item) for item in __prefix)) 544 else: 545 prefix_len = len(__prefix) 546 547 start_index = __start or 0 548 prefix_end_index = start_index + prefix_len 549 if __end is None: 550 end_index = prefix_end_index 551 else: 552 end_index = __end 553 554 end_index = min(end_index, prefix_end_index) 555 if end_index < start_index: 556 return False 557 558 start_item_index, index_within_start_item = self.item_by_index(start_index) 559 if -1 == start_item_index: 560 return False 561 562 end_item_index, index_within_end_item = self.item_by_index(end_index) 563 if -1 == end_item_index: 564 end_item_index = self._real_size -1 565 index_within_end_item = len(self._data[end_item_index]) - 1 566 567 return self._joiner.join(self._data[start_item_index: end_item_index]).startswith( 568 __prefix, index_within_start_item, index_within_end_item) 569 570 def find( 571 self, __sub, __start = None, __end = None 572 ) -> int: 573 if not self._data_length: 574 return False 575 576 start_index = __start or 0 577 end_index = __end or (self._data_length - 1) 578 579 start_item_index, index_within_start_item = self.item_by_index(start_index) 580 if -1 == start_item_index: 581 return False 582 583 end_item_index, index_within_end_item = self.item_by_index(end_index) 584 if -1 == end_item_index: 585 end_item_index = self._real_size -1 586 index_within_end_item = len(self._data[end_item_index]) - 1 587 588 return self._joiner.join(self._data[start_item_index: end_item_index]).find( 589 __sub, index_within_start_item, index_within_end_item) 590 591 def clear(self): 592 self.get_data_pieces(self._real_size) 593 594 def remove(self): 595 if not self._removed: 596 self._external_data_length.value -= self._data_length 597 self._removed = True 598 599 def __copy__(self): 600 return DynamicListOfPiecesDequeWithLengthControl( 601 self._init_param__joiner, self._init_param__external_data_length) 602 603 def __del__(self): 604 self.remove()
DynamicListOfPiecesDequeWithLengthControl( joiner=None, external_data_length: cengal.code_flow_control.smart_values.versions.v_2.smart_values.ValueExistence = None)
332 def __init__(self, joiner=None, external_data_length: ValueExistence=None): 333 self._removed = False 334 335 self._init_param__joiner = joiner 336 self._init_param__external_data_length = external_data_length 337 338 self._joiner = none_or(joiner, b'') 339 self._data = deque() 340 self._data_length = 0 341 342 self._real_size = 0 343 344 self._external_data_length = external_data_length or ValueExistence(False, 0)
def
get_data(self, size):
360 def get_data(self, size): 361 """ 362 :param size: 363 :return: data or None (if size > self._data_length) 364 """ 365 if size > self._data_length: 366 return None 367 elif size <= 0: 368 # return self._joiner.join(list()) # this is 1.6 times more faster than copy.copy(self._joiner) 369 return self._joiner 370 371 first_piece_len = len(self._data[0]) 372 if size < first_piece_len: 373 current_piece = memoryview(self._data[0]) 374 result = current_piece[:size] 375 self._data[0] = current_piece[size:] 376 self._data_length -= size 377 self._external_data_length.value -= size 378 return result 379 elif size == first_piece_len: 380 self._data_length -= size 381 self._external_data_length.value -= size 382 self._real_size -= 1 383 # return memoryview(self._data.popleft()) 384 return self._data.popleft() 385 else: 386 result_data = deque() 387 result_size = 0 388 389 while self._real_size and (result_size < size): 390 another_piece = self._data.popleft() 391 another_piece_len = len(another_piece) 392 result_data.append(another_piece) 393 self._data_length -= another_piece_len 394 self._external_data_length.value -= another_piece_len 395 self._real_size -= 1 396 result_size += another_piece_len 397 if (result_size < size) and ((len(self._data[0]) + result_size) > size): 398 break 399 400 if result_size < size: 401 delta_size = size - result_size 402 current_piece = memoryview(self._data[0]) 403 result_data.append(current_piece[:delta_size]) 404 self._data[0] = current_piece[delta_size:] 405 self._data_length -= delta_size 406 self._external_data_length.value -= delta_size 407 408 # return memoryview(self._joiner.join(result_data)) 409 return self._joiner.join(result_data)
:param size: :return: data or None (if size > self._data_length)
def
read_data(self, size):
411 def read_data(self, size): 412 """ 413 :param size: 414 :return: data or None (if size > self._data_length) 415 """ 416 if size > self._data_length: 417 return None 418 elif size <= 0: 419 # return self._joiner.join(list()) # this is 1.6 times more faster than copy.copy(self._joiner) 420 return self._joiner 421 422 first_piece_len = len(self._data[0]) 423 if size < first_piece_len: 424 current_piece = memoryview(self._data[0]) 425 return current_piece[:size] 426 elif size == first_piece_len: 427 return memoryview(self._data[0]) 428 else: 429 result_data = deque() 430 result_size = 0 431 432 index = 0 433 while result_size < size: 434 another_piece = self._data[index] 435 index += 1 436 another_piece_len = len(another_piece) 437 result_data.append(another_piece) 438 result_size += another_piece_len 439 if (result_size < size) and ((len(self._data[index]) + result_size) > size): 440 break 441 442 if result_size < size: 443 delta_size = size - result_size 444 current_piece = memoryview(self._data[index]) 445 result_data.append(current_piece[:delta_size]) 446 447 return self._joiner.join(result_data)
:param size: :return: data or None (if size > self._data_length)
def
get_data_pieces(self, size) -> List[bytes]:
450 def get_data_pieces(self, size) -> List[bytes]: 451 """ 452 :param size: 453 :return: data or None (if size > self._data_length) 454 """ 455 if size > self._data_length: 456 return None 457 elif size <= 0: 458 # return self._joiner.join(list()) # this is 1.6 times more faster than copy.copy(self._joiner) 459 return [self._joiner] 460 461 first_piece_len = len(self._data[0]) 462 if size <= first_piece_len: 463 self._data_length -= first_piece_len 464 self._external_data_length.value -= first_piece_len 465 self._real_size -= 1 466 return [self._data.popleft()] 467 else: 468 result_data = deque() 469 result_size = 0 470 471 while self._real_size and (result_size < size): 472 another_piece = self._data.popleft() 473 another_piece_len = len(another_piece) 474 result_data.append(another_piece) 475 self._data_length -= another_piece_len 476 self._external_data_length.value -= another_piece_len 477 self._real_size -= 1 478 result_size += another_piece_len 479 480 return result_data
:param size: :return: data or None (if size > self._data_length)
def
read_data_pieces(self, size):
482 def read_data_pieces(self, size): 483 """ 484 :param size: 485 :return: data or None (if size > self._data_length) 486 """ 487 if size > self._data_length: 488 return None 489 elif size <= 0: 490 # return self._joiner.join(list()) # this is 1.6 times more faster than copy.copy(self._joiner) 491 return [self._joiner] 492 493 first_piece_len = len(self._data[0]) 494 if size <= first_piece_len: 495 return [memoryview(self._data[0])] 496 else: 497 result_data = deque() 498 result_size = 0 499 500 index = 0 501 while result_size < size: 502 another_piece = memoryview(self._data[index]) 503 index += 1 504 another_piece_len = len(another_piece) 505 result_data.append(another_piece) 506 result_size += another_piece_len 507 508 return result_data
:param size: :return: data or None (if size > self._data_length)
def
item_by_index(self, index: int) -> Tuple[int, int]:
522 def item_by_index(self, index: int) -> Tuple[int, int]: 523 if index >= self._data_length: 524 return -1, -1 525 526 offset = 0 527 for item_index, item in enumerate(self._data): 528 item_len = len(item) 529 offset += item_len 530 if index < offset: 531 index_within_item = item_len - (offset - index) 532 return item_index, index_within_item
def
startswith( self, _DynamicListOfPiecesDequeWithLengthControl__prefix, _DynamicListOfPiecesDequeWithLengthControl__start=None, _DynamicListOfPiecesDequeWithLengthControl__end=None) -> bool:
534 def startswith(self, 535 __prefix, 536 __start = None, 537 __end = None 538 ) -> bool: 539 if not self._data_length: 540 return False 541 542 if isinstance(__prefix, tuple): 543 prefix_len = max((len(item) for item in __prefix)) 544 else: 545 prefix_len = len(__prefix) 546 547 start_index = __start or 0 548 prefix_end_index = start_index + prefix_len 549 if __end is None: 550 end_index = prefix_end_index 551 else: 552 end_index = __end 553 554 end_index = min(end_index, prefix_end_index) 555 if end_index < start_index: 556 return False 557 558 start_item_index, index_within_start_item = self.item_by_index(start_index) 559 if -1 == start_item_index: 560 return False 561 562 end_item_index, index_within_end_item = self.item_by_index(end_index) 563 if -1 == end_item_index: 564 end_item_index = self._real_size -1 565 index_within_end_item = len(self._data[end_item_index]) - 1 566 567 return self._joiner.join(self._data[start_item_index: end_item_index]).startswith( 568 __prefix, index_within_start_item, index_within_end_item)
def
find( self, _DynamicListOfPiecesDequeWithLengthControl__sub, _DynamicListOfPiecesDequeWithLengthControl__start=None, _DynamicListOfPiecesDequeWithLengthControl__end=None) -> int:
570 def find( 571 self, __sub, __start = None, __end = None 572 ) -> int: 573 if not self._data_length: 574 return False 575 576 start_index = __start or 0 577 end_index = __end or (self._data_length - 1) 578 579 start_item_index, index_within_start_item = self.item_by_index(start_index) 580 if -1 == start_item_index: 581 return False 582 583 end_item_index, index_within_end_item = self.item_by_index(end_index) 584 if -1 == end_item_index: 585 end_item_index = self._real_size -1 586 index_within_end_item = len(self._data[end_item_index]) - 1 587 588 return self._joiner.join(self._data[start_item_index: end_item_index]).find( 589 __sub, index_within_start_item, index_within_end_item)
class
DynamicListOfPiecesMemoryviewDequeWithLengthControl:
608class DynamicListOfPiecesMemoryviewDequeWithLengthControl: 609 def __init__(self, joiner=None, external_data_length: Optional[ValueExistence] = None, mem_buffer_size: Optional[ValueExistence] = None, output_fifo: Optional[FIFODequeWithLengthControl] = None): 610 self._removed = False 611 612 self._init_param__joiner = joiner 613 self._init_param__external_data_length = external_data_length 614 self._init_param__mem_buffer_size = mem_buffer_size 615 616 self._joiner = none_or(joiner, b'') 617 self._data = deque() 618 self._data_length = 0 619 620 self._real_size = 0 621 622 self._external_data_length = external_data_length or ValueExistence(False, 0) 623 624 self.mem_buffer_size = mem_buffer_size or ValueExistence(False, 1024) 625 self.mem_buffer = memoryview(bytearray(self.mem_buffer_size.value)) 626 self._external_data_length.value += self.mem_buffer_size.value 627 628 self.mem_buffer_offset = 0 629 self.mem_buffer_nbytes = 0 630 self.mem_buffer_diff = 0 631 self.mem_buffer_message_nbytes = 0 632 633 self.output_data = output_fifo or FIFODequeWithLengthControl(self._external_data_length) 634 635 def _allocate_new_mem_buffer(self): 636 self._external_data_length.value -= len(self.mem_buffer.obj) 637 self.mem_buffer = memoryview(bytearray(self.mem_buffer_size.value)) 638 self._external_data_length.value += self.mem_buffer_size.value 639 640 def size(self): 641 return self._data_length 642 643 def qnt(self): 644 return self._real_size 645 646 def add_piece_of_data(self, piece_of_data): 647 raise NotImplemented() 648 649 def _add_piece_of_data(self, piece_of_data): 650 self._data.append(piece_of_data) 651 piece_of_data_len = len(piece_of_data) 652 self._data_length += piece_of_data_len 653 self._external_data_length.value += piece_of_data_len 654 self._real_size += 1 655 656 def piece_of_data_had_been_added(self, nbytes): 657 self.mem_buffer = self.mem_buffer[nbytes:] 658 self.mem_buffer_nbytes += nbytes 659 self.mem_buffer_diff += nbytes 660 661 if not self.mem_buffer: 662 if self.mem_buffer_diff: 663 self._add_piece_of_data(memoryview(self.mem_buffer.obj)[self.mem_buffer_offset:]) 664 self._allocate_new_mem_buffer() 665 666 # @profile 667 def get_data(self, size): 668 """ 669 :param size: 670 :return: data or None (if size > self._data_length) 671 """ 672 if size > self._data_length: 673 return None 674 elif size <= 0: 675 # return self._joiner.join(list()) # this is 1.6 times more faster than copy.copy(self._joiner) 676 return self._joiner 677 678 first_piece_len = len(self._data[0]) 679 if size < first_piece_len: 680 current_piece = memoryview(self._data[0]) 681 result = current_piece[:size] 682 self._data[0] = current_piece[size:] 683 self._data_length -= size 684 self._external_data_length.value -= size 685 return result 686 elif size == first_piece_len: 687 self._data_length -= size 688 self._external_data_length.value -= size 689 self._real_size -= 1 690 # return memoryview(self._data.popleft()) 691 return self._data.popleft() 692 else: 693 result_data = deque() 694 result_size = 0 695 696 while self._real_size and (result_size < size): 697 another_piece = self._data.popleft() 698 another_piece_len = len(another_piece) 699 result_data.append(another_piece) 700 self._data_length -= another_piece_len 701 self._external_data_length.value -= another_piece_len 702 self._real_size -= 1 703 result_size += another_piece_len 704 if (result_size < size) and ((len(self._data[0]) + result_size) > size): 705 break 706 707 if result_size < size: 708 delta_size = size - result_size 709 current_piece = memoryview(self._data[0]) 710 result_data.append(current_piece[:delta_size]) 711 self._data[0] = current_piece[delta_size:] 712 self._data_length -= delta_size 713 self._external_data_length.value -= delta_size 714 715 # return memoryview(self._joiner.join(result_data)) 716 return self._joiner.join(result_data) 717 718 def read_data(self, size): 719 raise NotImplemented() 720 721 def remove(self): 722 if not self._removed: 723 self._external_data_length.value -= self._data_length 724 self._external_data_length.value -= len(self.mem_buffer.obj) 725 self._removed = True 726 727 def __copy__(self): 728 return DynamicListOfPiecesDequeWithLengthControl( 729 self._init_param__joiner, self._init_param__external_data_length) 730 731 def __del__(self): 732 self.remove()
DynamicListOfPiecesMemoryviewDequeWithLengthControl( joiner=None, external_data_length: typing.Union[cengal.code_flow_control.smart_values.versions.v_2.smart_values.ValueExistence, NoneType] = None, mem_buffer_size: typing.Union[cengal.code_flow_control.smart_values.versions.v_2.smart_values.ValueExistence, NoneType] = None, output_fifo: typing.Union[cengal.data_containers.fast_fifo.versions.v_1.fast_fifo.FIFODequeWithLengthControl, NoneType] = None)
609 def __init__(self, joiner=None, external_data_length: Optional[ValueExistence] = None, mem_buffer_size: Optional[ValueExistence] = None, output_fifo: Optional[FIFODequeWithLengthControl] = None): 610 self._removed = False 611 612 self._init_param__joiner = joiner 613 self._init_param__external_data_length = external_data_length 614 self._init_param__mem_buffer_size = mem_buffer_size 615 616 self._joiner = none_or(joiner, b'') 617 self._data = deque() 618 self._data_length = 0 619 620 self._real_size = 0 621 622 self._external_data_length = external_data_length or ValueExistence(False, 0) 623 624 self.mem_buffer_size = mem_buffer_size or ValueExistence(False, 1024) 625 self.mem_buffer = memoryview(bytearray(self.mem_buffer_size.value)) 626 self._external_data_length.value += self.mem_buffer_size.value 627 628 self.mem_buffer_offset = 0 629 self.mem_buffer_nbytes = 0 630 self.mem_buffer_diff = 0 631 self.mem_buffer_message_nbytes = 0 632 633 self.output_data = output_fifo or FIFODequeWithLengthControl(self._external_data_length)
def
piece_of_data_had_been_added(self, nbytes):
656 def piece_of_data_had_been_added(self, nbytes): 657 self.mem_buffer = self.mem_buffer[nbytes:] 658 self.mem_buffer_nbytes += nbytes 659 self.mem_buffer_diff += nbytes 660 661 if not self.mem_buffer: 662 if self.mem_buffer_diff: 663 self._add_piece_of_data(memoryview(self.mem_buffer.obj)[self.mem_buffer_offset:]) 664 self._allocate_new_mem_buffer()
def
get_data(self, size):
667 def get_data(self, size): 668 """ 669 :param size: 670 :return: data or None (if size > self._data_length) 671 """ 672 if size > self._data_length: 673 return None 674 elif size <= 0: 675 # return self._joiner.join(list()) # this is 1.6 times more faster than copy.copy(self._joiner) 676 return self._joiner 677 678 first_piece_len = len(self._data[0]) 679 if size < first_piece_len: 680 current_piece = memoryview(self._data[0]) 681 result = current_piece[:size] 682 self._data[0] = current_piece[size:] 683 self._data_length -= size 684 self._external_data_length.value -= size 685 return result 686 elif size == first_piece_len: 687 self._data_length -= size 688 self._external_data_length.value -= size 689 self._real_size -= 1 690 # return memoryview(self._data.popleft()) 691 return self._data.popleft() 692 else: 693 result_data = deque() 694 result_size = 0 695 696 while self._real_size and (result_size < size): 697 another_piece = self._data.popleft() 698 another_piece_len = len(another_piece) 699 result_data.append(another_piece) 700 self._data_length -= another_piece_len 701 self._external_data_length.value -= another_piece_len 702 self._real_size -= 1 703 result_size += another_piece_len 704 if (result_size < size) and ((len(self._data[0]) + result_size) > size): 705 break 706 707 if result_size < size: 708 delta_size = size - result_size 709 current_piece = memoryview(self._data[0]) 710 result_data.append(current_piece[:delta_size]) 711 self._data[0] = current_piece[delta_size:] 712 self._data_length -= delta_size 713 self._external_data_length.value -= delta_size 714 715 # return memoryview(self._joiner.join(result_data)) 716 return self._joiner.join(result_data)
:param size: :return: data or None (if size > self._data_length)
def
get_num_pieces_with_full_size_lesser_than(iter_object, full_size):
735def get_num_pieces_with_full_size_lesser_than(iter_object, full_size): 736 index = -1 737 if iter_object: 738 result_size = 0 739 index = 0 740 for piece in iter_object: 741 piece_len = len(piece) 742 if (result_size + piece_len) > full_size: 743 break 744 else: 745 result_size += piece_len 746 index += 1 747 return index
def
get_num_pieces_with_full_size_lesser_than_with_offset(iter_object, full_size, offset):
750def get_num_pieces_with_full_size_lesser_than_with_offset(iter_object, full_size, offset): 751 pieces_qnt = -1 752 if iter_object: 753 object_size = len(iter_object) 754 result_data_length = 0 755 pieces_qnt = 0 756 for piece_num in range(offset, object_size): 757 piece = iter_object[piece_num] 758 piece_length = len(piece) 759 if (result_data_length + piece_length) > full_size: 760 break 761 else: 762 result_data_length += piece_length 763 pieces_qnt += 1 764 return pieces_qnt