cengal.data_containers.dynamic_tag_tree.versions.v_1.TagDB

Module Docstring Docstrings: http://www.python.org/dev/peps/pep-0257/

View Source

  1#!/usr/bin/env python
  2# coding=utf-8
  3
  4# Copyright © 2012-2024 ButenkoMS. All rights reserved. Contacts: <gtalk@butenkoms.space>
  5# 
  6# Licensed under the Apache License, Version 2.0 (the "License");
  7# you may not use this file except in compliance with the License.
  8# You may obtain a copy of the License at
  9# 
 10#     http://www.apache.org/licenses/LICENSE-2.0
 11# 
 12# Unless required by applicable law or agreed to in writing, software
 13# distributed under the License is distributed on an "AS IS" BASIS,
 14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 15# See the License for the specific language governing permissions and
 16# limitations under the License.
 17
 18
 19"""
 20Module Docstring
 21Docstrings: http://www.python.org/dev/peps/pep-0257/
 22"""
 23
 24
 25__author__ = "ButenkoMS <gtalk@butenkoms.space>"
 26__copyright__ = "Copyright © 2012-2024 ButenkoMS. All rights reserved. Contacts: <gtalk@butenkoms.space>"
 27__credits__ = ["ButenkoMS <gtalk@butenkoms.space>", ]
 28__license__ = "Apache License, Version 2.0"
 29__version__ = "4.4.1"
 30__maintainer__ = "ButenkoMS <gtalk@butenkoms.space>"
 31__email__ = "gtalk@butenkoms.space"
 32# __status__ = "Prototype"
 33__status__ = "Development"
 34# __status__ = "Production"
 35
 36
 37__author__ = 'Mikhail Butenko <gtalk@mikhail-butenko.in.ua>'
 38
 39from cengal.data_generation.id_generator import IDGenerator
 40from contextlib import contextmanager
 41from typing import Hashable, Tuple, Dict, Any
 42from cengal.data_containers.compound_dict_management.standard_library.key__hashable__to__value__set import AddToCompoundDict__Set
 43from cengal.data_containers.compound_dict_management.standard_library.key_counter import KeyCounter
 44from cengal.parallel_execution.coroutines.coro_standard_services.loop_yield import gly, CoroPriority
 45from cengal.parallel_execution.coroutines.coro_scheduler import EntityStatsMixin
 46
 47
 48SMART_TREE_TYPE = 0  # smart tree. Умное дерево тегов: сеть отображенная на древо. Возвращает только список
 49    # непосредственных подтегов текущего пути, но не их подтеги; возвращает элементы текущего пути, но не элементы
 50    # из подпутей
 51SMART_TREE_TYPE_WITH_INTERNAL_MENU = 1   # smart tree with internal menu. В древо встроено меню, позволяющее прямо из
 52    # древа производить смену типа вывода: SMART_TREE_TYPE, FULL_TREE_TYPE и PLAIN_PSEUDO_TREE_TYPE. На каждый тип
 53    # вывода будет доступен подтег/подпапка, внутри когорого уже будет нормальное древо элементов, но уже выбранного
 54    # типа
 55FULL_TREE_TYPE = 2  # full tree with all tags - with repeats and without filtering. Список айтемов - как у
 56    # SMART_TREE_TYPE, но при этом список тегов - как у PLAIN_PSEUDO_TREE_TYPE
 57PLAIN_PSEUDO_TREE_TYPE = 3   # plain tags and items set (will show all tags, subtags and items of
 58    # current hm... dir - current tag set). Показывает все теги и подтеги единым списком - как у примитивных теговых
 59    # файловых систем; показывает все элементы текущего пути + все элементы всех под-путей
 60
 61USUAL_TREE_TYPE = PLAIN_PSEUDO_TREE_TYPE
 62
 63_ROOT_TAG = r'k{1+vdcY#m8t-4m9`)G2\b]/O\'Rzqyr@FEO~%./nGPzl)[^q 0RS!.bCPh ?fag{8~{SGj;Ss3U85Q-:'
 64
 65
 66class ToManyIdenticalItemsOnTheGivenTagPathError(Exception):
 67    pass
 68
 69
 70class UnknownTreeTypeError(Exception):
 71    pass
 72
 73
 74class LockableMixin:
 75    @property
 76    def lock(self) -> bool:
 77        raise NotImplementedError
 78    
 79    @lock.setter
 80    def lock(self, value: bool) -> bool:
 81        raise NotImplementedError
 82
 83
 84@contextmanager
 85def obj_locker(obj: LockableMixin):
 86    obj.lock = True
 87    try:
 88        yield obj.lock
 89    finally:
 90        obj.lock = False
 91
 92
 93class Example(LockableMixin):
 94    def __init__(self) -> None:
 95        super().__init__()
 96        self.lock: bool = False
 97    
 98    def write_coroutine(self):
 99        with obj_locker(self):
100            pass
101    
102    def read_coroutine(self):
103        if not self.lock:
104            pass
105
106
107class TagDB(EntityStatsMixin):
108    def __init__(self, default_priority: CoroPriority = CoroPriority.normal):
109        self.default_priority: CoroPriority = default_priority
110
111        self.itemsID = IDGenerator()
112
113        self.itemIDsForItem = {}  # key - item hash; data - set of itemIDs
114
115        self.itemsSet = {}  # key - ItemID; data - binItem
116        # TODO: заменить список тегов на хеш единожды сохраненного списка тегов
117        self.itemWithTags = {}  # key - ItemID; data - sorted common TagsTuple's hash
118
119        # TODO: убрать tagsNumPerItemID из кода. Заменить этот список itemID - на список hashOfTheTagHashTuple
120        self.tagsNumPerItemID = {}  # key - number of tags in this ItemID group; data - set of itemIDs which are have
121            # needed number of tags
122
123        self.tagsSet = {}  # key - tag hash; data - binTag
124        self.tagWithItems = {}  # key - Tag hash; data - set of itemIDs
125        self.tagsQnt = {}    # key - Tag hash; data - quantity of the items with this tag
126
127
128        self.commonTagSets = {}  # key - sorted common TagsTuple's hash; data - sorted TagsTuple
129        self.itemsOnTheCommonTagSets = {}   # key - sorted common TagsTuple's hash; data - set of itemIDs
130        self.tagsQntPerCommonTagSet = {}    # key - number of tags; data - set of TagsTuple hashes
131        self.setOfTagGroupQnt = set()   # {tagQntInGroup1, tagQntInGroup2, ..., tagQntInGroupN} where
132            # each Group is an key of the self.tagsQntPerCommonTagSet
133
134        # TODO: добавить словарь tagHashTuplesIDs вида [tagHashTuple.__hash__():tagHashTuple]
135        # где tagHashSet - это {tagHash_1, tagHash_2, ..., tagHash_3}
136        # TODO: добавить словарь itemsOnThePath вида [tagHashTuple.__hash__():{itemID_1, itemID_2, ..., itemID_3}]
137
138        # TODO: добавить словарь вида [tagQnt:{itemID_1, itemID_2, ..., itemID_3}] и вычитывать это из него
139        # и/или
140        # TODO: добавить словарь вида [tagQnt:{hashOfTheTagHashTuple_1, hashOfTheTagHashTuple_2, ...
141        # , hashOfTheTagHashTuple_N}] и вычитывать это из него, а потом уже и из каждого tagHashSet
142        # где hashOfTheTagHashTuple - это tagHashTuple.__hash__()
143
144    def get_entity_stats(self, stats_level: 'EntityStatsMixin.StatsLevel' = EntityStatsMixin.StatsLevel.debug) -> Tuple[str, Dict[str, Any]]:
145        return type(self).__name__, {
146            'items num': len(self.itemsSet),
147            'tags num': len(self.tagsSet),
148        }
149
150    def get_root_tag(self):
151        return str(_ROOT_TAG)
152
153    def add_tag(self, binTag):
154        tagHash = binTag.__hash__()
155        self.tagsSet[tagHash] = binTag
156        if tagHash not in self.tagsSet:
157            self.tagsQnt[tagHash] = 0
158        if tagHash not in self.tagWithItems:
159            self.tagWithItems[tagHash] = set()
160
161    def remove_tag(self, binTag):
162        # will try to delete given tag. If there is at least one item with this tag, than function will fail
163        # and will return False; otherwise it will delete given tag and will return True.
164        functionResult = False
165        tagHash = binTag.__hash__()
166        if tagHash in self.tagsSet:
167            if tagHash in self.tagsQnt:
168                if self.tagsQnt[tagHash] == 0:
169                    del self.tagsSet[tagHash]
170                    del self.tagsQnt[tagHash]
171                    functionResult = True
172                else:
173                    functionResult = False
174            else:
175                del self.tagsSet[tagHash]
176                functionResult = True
177
178        if functionResult:
179            if tagHash in self.tagWithItems:
180                del self.tagWithItems[tagHash]
181
182        return functionResult
183
184    def add_item(self, binItem, binTags):
185        # will add new item and return it's dynamic ID or None object If this Item already exist on the given tag path
186        # Or will raise an exception if we already have more than one binItem (another item that is identical to
187        # the given binItem)  on this tag path
188        binTags = set(binTags)
189
190        if self.get_root_tag() not in binTags:
191            binTags.add(self.get_root_tag())
192
193        # may raise an exception in this place. Nope - from now it will be not
194        if self.get_itemID_from_item_and_tags(binTags, binItem) is not None:
195            return None
196
197        itemID = self.itemsID()
198        self.itemsSet[itemID] = binItem
199
200        itemHash = binItem.__hash__()
201        if itemHash in self.itemIDsForItem:
202            IDsSet = self.itemIDsForItem[itemHash]
203            IDsSet.add(itemID)
204            # self.itemIDsForItem[itemHash] = IDsSet
205        else:
206            self.itemIDsForItem[itemHash] = {itemID}
207
208        tagQnt = len(binTags)
209        if tagQnt in self.tagsNumPerItemID:
210            itemIDsSet = self.tagsNumPerItemID[tagQnt]
211            itemIDsSet.add(itemID)
212            # self.tagsNumPerItemID[tagQnt] = itemIDsSet
213        else:
214            self.tagsNumPerItemID[tagQnt] = {itemID}
215
216        binTagHashes = set()
217
218        for tag in binTags:
219            self.add_tag(tag)
220            tagHash = tag.__hash__()
221            binTagHashes.add(tagHash)
222            setOfItems = self.tagWithItems[tagHash]
223            if itemID not in setOfItems:
224                setOfItems.add(itemID)
225                if tagHash in self.tagsQnt:
226                    self.tagsQnt[tagHash] += 1
227                else:
228                    self.tagsQnt[tagHash] = 1
229            self.tagWithItems[tagHash] = setOfItems
230
231        sortedTagTuple = tuple(self.sort_tag_hash_list_by_hash(binTagHashes))
232        hashOfTheSortedTagTuple = sortedTagTuple.__hash__()
233        self.itemWithTags[itemID] = hashOfTheSortedTagTuple
234
235        self.commonTagSets[hashOfTheSortedTagTuple] = sortedTagTuple
236
237        if hashOfTheSortedTagTuple in self.itemsOnTheCommonTagSets:
238            itemIDsSet = self.itemsOnTheCommonTagSets[hashOfTheSortedTagTuple]
239            itemIDsSet.add(itemID)
240            # self.itemsOnTheCommonTagSets[tagQnt] = itemIDsSet
241        else:
242            self.itemsOnTheCommonTagSets[hashOfTheSortedTagTuple] = {itemID}
243
244        lenOfTheSortedTagTuple = len(sortedTagTuple)
245        if lenOfTheSortedTagTuple in self.tagsQntPerCommonTagSet:
246            itemIDsSet = self.tagsQntPerCommonTagSet[lenOfTheSortedTagTuple]
247            itemIDsSet.add(hashOfTheSortedTagTuple)
248            # self.tagsQntPerCommonTagSet[lenOfTheSortedTagTuple] = itemIDsSet
249        else:
250            self.tagsQntPerCommonTagSet[lenOfTheSortedTagTuple] = {hashOfTheSortedTagTuple}
251            self.setOfTagGroupQnt.add(lenOfTheSortedTagTuple)
252
253        return itemID
254
255    def remove_item_by_itemID(self, itemID):
256        ly = gly(self.default_priority)
257
258        if itemID in self.itemsSet:
259            itemHash = self.itemsSet[itemID].__hash__()
260            del self.itemsSet[itemID]
261
262            if itemHash in self.itemIDsForItem:
263                IDsSet = self.itemIDsForItem[itemHash]
264                IDsSet.difference_update({itemID})
265                # self.itemIDsForItem[itemHash] = IDsSet
266                if len(IDsSet) < 1:
267                    del self.itemIDsForItem[itemHash]
268
269        if itemID in self.itemWithTags:
270            commonTagTupleHash = self.itemWithTags[itemID]
271            tagTuple = self.commonTagSets[commonTagTupleHash]
272            numberOfTags = len(tagTuple)
273
274            if commonTagTupleHash in self.itemsOnTheCommonTagSets:
275                IDsSet = self.itemsOnTheCommonTagSets[commonTagTupleHash]
276                IDsSet.difference_update({itemID})
277                # self.itemsOnTheCommonTagSets[commonTagTupleHash] = IDsSet
278                if len(IDsSet) < 1:
279                    del self.itemsOnTheCommonTagSets[commonTagTupleHash]
280                    del self.commonTagSets[commonTagTupleHash]
281                    if numberOfTags in self.tagsQntPerCommonTagSet:
282                        setOfTagTuplesHashes = self.tagsQntPerCommonTagSet[numberOfTags]
283                        setOfTagTuplesHashes.difference_update({numberOfTags})
284                        # self.tagsQntPerCommonTagSet[numberOfTags] = setOfTagTuplesHashes
285                        if len(setOfTagTuplesHashes) < 1:
286                            del self.tagsQntPerCommonTagSet[numberOfTags]
287                            self.setOfTagGroupQnt.difference_update({numberOfTags})
288
289            del self.itemWithTags[itemID]
290
291            setOfTagHashes = set(tagTuple)
292
293            tagsQnt = len(setOfTagHashes)
294            if tagsQnt in self.tagsNumPerItemID:
295                IDsSet = self.tagsNumPerItemID[tagsQnt]
296                IDsSet.difference_update({itemID})
297                # self.tagsNumPerItemID[tagsQnt] = IDsSet
298                if len(IDsSet) < 1:
299                    del self.tagsNumPerItemID[tagsQnt]
300
301            for tagHash in setOfTagHashes:
302                ly()
303                if tagHash in self.tagsQnt:
304                    tagsQuantity = self.tagsQnt[tagHash]
305                    tagsQuantity -= 1
306                    if tagsQuantity < 1:
307                        tagsQuantity = 0
308                    self.tagsQnt[tagHash] = tagsQuantity
309                    if tagsQuantity < 1:
310                        del self.tagsQnt[tagHash]
311                        del self.tagsSet[tagHash]
312                if tagHash in self.tagWithItems:
313                    IDsSet = self.tagWithItems[tagHash]
314                    IDsSet.difference_update({itemID})
315                    if len(IDsSet) < 1:
316                        del self.tagWithItems[tagHash]
317                    # self.tagWithItems[tagHash] = IDsSet
318
319        self.itemsID.remove_id(itemID)
320
321    def remove_item(self, binTags, binItem):
322        # will return ItemId for deleted item or None object if Item is not exist
323        # Or will raise an exception if we already have more than one binItem (another item that is identical to
324        # the given binItem) on this tag path
325        binTags = set(binTags)
326        if self.get_root_tag() not in binTags:
327            binTags.add(self.get_root_tag())
328        itemID = self.get_itemID_from_item_and_tags(binTags, binItem)
329        if itemID is not None:
330            self.remove_item_by_itemID(itemID)
331        return itemID
332
333    def __OLD__get_itemID_from_item_and_tags(self, binTags, binItem):
334        if self.get_root_tag() not in binTags:
335            binTags.append(self.get_root_tag())
336        potentialIDs = set(self.get_potential_itemIDs_from_item(binItem))
337        itemIDsSet = set(self.get_itemIDs_from_tags(binTags, SMART_TREE_TYPE))
338        resultItemIDsList = potentialIDs & itemIDsSet
339        if len(resultItemIDsList) == 0:
340            return None
341        elif len(resultItemIDsList) == 1:
342            resultItemID = resultItemIDsList.pop()  # we have assume that we'll have only one item in intersection
343                # between potential IDs and Items that have (and have only) given tag list (without another tags in the
344                # path to this items). We need to check it in the adding new item to the given tag path.
345            return resultItemID
346        elif len(resultItemIDsList) > 1:
347            raise ToManyIdenticalItemsOnTheGivenTagPathError()
348
349    def get_itemID_from_item_and_tags(self, binTags, binItem):
350        ly = gly(self.default_priority)
351
352        binTags = set(binTags)
353
354        if self.get_root_tag() not in binTags:
355            binTags.add(self.get_root_tag())
356
357        potentialIDs = self.get_potential_itemIDs_from_item(binItem)
358        setOfBinTagsHashes = set()
359        for tag in binTags:
360            ly()
361            setOfBinTagsHashes.add(tag.__hash__())
362        for itemID in potentialIDs:
363            ly()
364            currentItemTagsSet = self.get_tagsHashes_from_single_item(itemID, isWithoutRootHash=False)
365            if setOfBinTagsHashes == currentItemTagsSet:
366                return itemID
367        return None
368
369    def tag_hash_list_2_tag_list(self, tagHashList):
370        ly = gly(self.default_priority)
371
372        tagList = list()
373        for tagHash in tagHashList:
374            ly()
375            tagList.append(self.tagsSet[tagHash])
376        return tagList
377
378    def get_item_and_tags_from_itemID(self, itemID):
379        commonTagTupleHash = self.itemWithTags[itemID]
380        tagSet = set(self.commonTagSets[commonTagTupleHash]) - {self.get_root_tag().__hash__()}
381        sortedTagHashList = self.sort_tag_hash_list_by_qnt(tagSet)
382        result = (self.itemsSet[itemID], self.tag_hash_list_2_tag_list(sortedTagHashList))
383        return result
384
385    # @profile
386    def get_top_tag_hash_list_by_qnt(self, tagHashSet, local_tags_qnt=None):
387        ly = gly(self.default_priority)
388
389        tagsQnt = self.tagsQnt
390        if local_tags_qnt is not None:
391            tagsQnt = local_tags_qnt
392        tag_hash_set = set(tagHashSet)
393        tag_by_qnt = dict()
394        tag_by_qnt__filler = AddToCompoundDict__Set(tag_by_qnt)
395        biggest_qnt = 0
396        for tag_hash in tag_hash_set:
397            ly()
398            qnt = tagsQnt[tag_hash]
399            if qnt > biggest_qnt:
400                biggest_qnt = qnt
401            # if qnt not in tag_by_qnt:
402            #     tag_by_qnt[qnt] = set()
403            # tag_by_qnt[qnt].add(tag_hash)
404            tag_by_qnt__filler.add(qnt, tag_hash)
405
406        result = None
407        if len(tag_by_qnt) > 0:
408            # biggest_qnt = max(tag_by_qnt)
409            result = tuple(tag_by_qnt[biggest_qnt])
410        else:
411            result = tuple()
412        return result
413
414    def sort_tag_hash_list_by_qnt(self, tagHashSet):
415        # will return sorted tag list - not sorted tag hash list
416        ly = gly(self.default_priority)
417
418        tagHashSet = set(tagHashSet)
419        rawTagList = list()
420        for tagHash in tagHashSet:
421            ly()
422            tagWithWeight = (tagHash, self.tagsQnt[tagHash])
423            rawTagList.append(tagWithWeight)
424        return self.sort_raw_tag_list(rawTagList)
425
426    def sort_tag_list_by_qnt(self, binTags):
427        ly = gly(self.default_priority)
428
429        binTags = set(binTags)
430        rawTagList = list()
431        for tag in binTags:
432            ly()
433            tagHash = tag.__hash__()
434            tagWithWeight = (tag, self.tagsQnt[tagHash])
435            rawTagList.append(tagWithWeight)
436        return self.sort_raw_tag_list(rawTagList)
437
438    def sort_tag_hash_list_by_hash(self, tagHashSet):
439        # will return sorted tag list - not sorted tag hash list
440        ly = gly(self.default_priority)
441
442        tagHashSet = set(tagHashSet)
443        rawTagList = list()
444        for tagHash in tagHashSet:
445            ly()
446            tagWithWeight = (tagHash, tagHash)
447            rawTagList.append(tagWithWeight)
448        return self.sort_raw_tag_list(rawTagList)
449
450    def sort_tag_list_by_hash(self, binTags):
451        ly = gly(self.default_priority)
452
453        binTags = set(binTags)
454        rawTagList = list()
455        for tag in binTags:
456            ly()
457            tagHash = tag.__hash__()
458            tagWithWeight = (tag, tagHash)
459            rawTagList.append(tagWithWeight)
460        return self.sort_raw_tag_list(rawTagList)
461
462    def sort_raw_tag_list(self, rawTagList):
463        # will return sorted tag list
464        ly = gly(self.default_priority)
465
466        rawTagList = sorted(rawTagList, key=lambda tagAndWeight: tagAndWeight[1], reverse=True)
467        tagList = list()
468        for rawTag in rawTagList:
469            ly()
470            tagList.append(rawTag[0])
471        return tagList
472
473    def get_itemIDs_from_tags(self, binTags, treeType=USUAL_TREE_TYPE,
474                              isAlsoNeedSetOfAllInternalItemIDsForThisSetOfTags=False):
475        # TODO: исправить ошибку: SMART_TREE_TYPE: возвращает не только список файлов в текущей директории, но и из
476        # непосредственных подпапок данной папки
477
478        # treeType - type of the graph tree representation: show all tags with replies (pure representation);
479        # show only relevant tags; etc.
480        # return set of itemIDs
481        ly = gly(self.default_priority)
482
483        binTags = set(binTags)
484
485        if self.get_root_tag() not in binTags:
486            binTags.add(self.get_root_tag())
487
488        tagHashSet = set()
489        for binTag in binTags:
490            ly()
491            tagHashSet.add(binTag.__hash__())
492
493        # PLAIN_PSEUDO_TREE_TYPE
494        interceptionOfItemsWithTags = set()
495        if (treeType == PLAIN_PSEUDO_TREE_TYPE) or isAlsoNeedSetOfAllInternalItemIDsForThisSetOfTags:
496            binTagsQnt = len(tagHashSet)
497            commonTagGroupHashSet = set()
498            tagSet = set()
499            itemIDSet = set()
500            binTagHashTuple = tuple(self.sort_tag_hash_list_by_hash(tagHashSet))
501            hashOfTheBinTagHashTuple = binTagHashTuple.__hash__()
502            if hashOfTheBinTagHashTuple in self.itemsOnTheCommonTagSets:
503                itemIDSet = self.itemsOnTheCommonTagSets[hashOfTheBinTagHashTuple]
504            for commonTagQnt in self.setOfTagGroupQnt:
505                ly()
506                if commonTagQnt > binTagsQnt:
507                    setOfTheCommonTagGroupHashes = self.tagsQntPerCommonTagSet[commonTagQnt]
508                    commonTagGroupHashSet.update(setOfTheCommonTagGroupHashes)
509            for commonTagGroupHash in commonTagGroupHashSet:
510                ly()
511                commonTagHashTuple = self.commonTagSets[commonTagGroupHash]
512                commonTagHashSet = set(commonTagHashTuple)
513                if tagHashSet.issubset(commonTagHashSet):
514                    itemIDSet.update(self.itemsOnTheCommonTagSets[commonTagGroupHash])
515                # # if len(tagHashSet & commonTagHashSet) == len(tagHashSet):
516                # res_set = tagHashSet.intersection(commonTagHashSet)
517                # if len(res_set) == binTagsQnt:
518                #     itemIDSet = itemIDSet | self.itemsOnTheCommonTagSets[commonTagGroupHash]
519            interceptionOfItemsWithTags = itemIDSet
520
521            # isFirstHash = True
522            # for tag in binTags:
523            #     tagHash = tag.__hash__()
524            #     if tagHash in self.tagWithItems:
525            #         if isFirstHash:
526            #             interceptionOfItemsWithTags = self.tagWithItems[tagHash]
527            #             isFirstHash = False
528            #         else:
529            #             itemsWithTag = self.tagWithItems[tagHash]
530            #             interceptionOfItemsWithTags = interceptionOfItemsWithTags & itemsWithTag
531            #     else:
532            #         # TODO: произвести такую же провеку в get_items_from_tags() и build_smart_tree()
533            #         if isAlsoNeedSetOfAllInternalItemIDsForThisSetOfTags:
534            #             result = (set(), set())
535            #             return result
536            #         else:
537            #             return set()
538
539        resultItemIDSet = interceptionOfItemsWithTags
540        setOfAllInternalItemIDsForThisSetOfTags = interceptionOfItemsWithTags
541
542        # SMART_TREE_TYPE or FULL_TREE_TYPE
543        if (treeType == SMART_TREE_TYPE) or (treeType == FULL_TREE_TYPE):
544            resultItemIDSet = set()
545            binTagHashTuple = self.sort_tag_hash_list_by_hash(tagHashSet)
546            binTagHashTuple = tuple(binTagHashTuple)
547            hashOfTheBinTagHashTuple = binTagHashTuple.__hash__()
548            if hashOfTheBinTagHashTuple in self.itemsOnTheCommonTagSets:
549                resultItemIDSet = self.itemsOnTheCommonTagSets[hashOfTheBinTagHashTuple]
550
551            # filteredItemIDsSet = set()
552            # tagQnt = len(binTags)
553            # # for itemID in setOfAllInternalItemIDsForThisSetOfTags:
554            # #     if len(self.itemWithTags[itemID]) == tagQnt:
555            # #         # _TODO: добавить словарь вида [tagQnt:{itemID_1, itemID_2, ..., itemID_3}]
556            # #         # и вычитывать это из него
557            # #         # и/или
558            # #         # _TODO: добавить словарь вида [tagQnt:{hashOfTheTagHashSet_1, hashOfTheTagHashSet_2, ...
559            # #         # , hashOfTheTagHashSet_3}] и вычитывать это из него, а потом уже и из каждого tagHashSet
560            # #         # где hashOfTheTagHashSet - это tagHashSet.__hash__()
561            # #         filteredItemIDsSet.add(itemID)
562            # if tagQnt in self.tagsNumPerItemID:
563            #     filteredItemIDsSet = setOfAllInternalItemIDsForThisSetOfTags & self.tagsNumPerItemID[tagQnt]
564            #
565            # resultItemIDSet = set()
566            # tagHashSet = set()
567            # for binTag in binTags:
568            #     tagHashSet.add(binTag.__hash__())
569            # for itemID in filteredItemIDsSet:
570            #     commonTagTupleHash = self.itemWithTags[itemID]
571            #     tagSet = set(self.commonTagSets[commonTagTupleHash])
572            #     if tagSet == tagHashSet:
573            #         # _TODO: добавить словарь tagHashTuplesIDs вида [tagHashTuple.__hash__():tagHashTuple]
574            #         # где tagHashSet - это {tagHash_1, tagHash_2, ..., tagHash_3}
575            #         # _TODO: добавить словарь itemsOnThePath вида [tagHashTuple.__hash__():{itemID_1, itemID_2, ...
576            #         # , itemID_3}]
577            #         resultItemIDSet.add(itemID)
578        elif treeType == PLAIN_PSEUDO_TREE_TYPE:
579            # already implemented (see bellow). Don't touch this code!
580            pass
581        else:
582            raise UnknownTreeTypeError()
583
584        if isAlsoNeedSetOfAllInternalItemIDsForThisSetOfTags:
585            result = (set(resultItemIDSet), set(setOfAllInternalItemIDsForThisSetOfTags))
586            return result
587        else:
588            return set(resultItemIDSet)
589
590    def get_items_from_tags(self, binTags, treeType=USUAL_TREE_TYPE,
591                            isAlsoNeedSetOfAllInternalItemIDsForThisSetOfTags=False):
592        # treeType - type of the graph tree representation: show all tags with replies (pure representation);
593        # show only relevant tags; etc.
594        # return set of itemIDs
595        ly = gly(self.default_priority)
596
597        binTags = set(binTags)
598        itemIDsSet = self.get_itemIDs_from_tags(binTags, treeType=treeType,
599                                                isAlsoNeedSetOfAllInternalItemIDsForThisSetOfTags=
600                                                isAlsoNeedSetOfAllInternalItemIDsForThisSetOfTags)
601        if isAlsoNeedSetOfAllInternalItemIDsForThisSetOfTags:
602            itemSet = set()
603            for itemID in itemIDsSet[0]:
604                ly()
605                itemSet.add(self.itemsSet[itemID])
606            result = (tuple(itemSet), tuple(itemIDsSet[1]))  # result == (usual items set, additional set of all
607                # internal itemIDs)
608            return result
609        else:
610            itemSet = set()
611            for itemID in itemIDsSet:
612                ly()
613                itemSet.add(self.itemsSet[itemID])
614            return tuple(itemSet)
615
616    def get_tagHashes_from_tags(self, binTags, treeType=USUAL_TREE_TYPE,
617                                prePreparedSetOfAllInternalItemIDsForThisSetOfTags=None):
618        # where "itemIDsSet" is externally given "get_itemIDs_from_tags(binTags, treeType=PLAIN_PSEUDO_TREE_TYPE)"
619        # so "itemIDsSet" is a set of the all items inside the "folder" binTags (including items from "subfolders")
620        # treeType - the same as in the "get_items_from_tags()" method
621        # return set of itemIDs
622        # prePreparedSetOfAllInternalItemIDsForThisSetOfTags can be generated by:
623        #   a) get_itemIDs_from_tags(..., treeType=PLAIN_PSEUDO_TREE_TYPE)
624        #   a) get_itemIDs_from_tags(..., isAlsoNeedSetOfAllInternalItemIDsForThisSetOfTags = True)
625        #   c) get_items_from_tags(..., isAlsoNeedSetOfAllInternalItemIDsForThisSetOfTags = True)
626        #   d) get_items_from_tags(..., treeType=PLAIN_PSEUDO_TREE_TYPE) || BUT: it'll return item set - not itemID
627        #       set
628        ly = gly(self.default_priority)
629
630        binTags = set(binTags)
631
632        resultTagHashSet = set()
633
634        setOfAllInternalItemIDs = set()
635        if prePreparedSetOfAllInternalItemIDsForThisSetOfTags is None:
636            setOfAllInternalItemIDs = self.get_itemIDs_from_tags(binTags, treeType=PLAIN_PSEUDO_TREE_TYPE)
637        else:
638            setOfAllInternalItemIDs = prePreparedSetOfAllInternalItemIDsForThisSetOfTags
639
640        if (treeType == PLAIN_PSEUDO_TREE_TYPE) or (treeType == FULL_TREE_TYPE):
641            binTagHashes = set()
642            for tag in binTags:
643                ly()
644                binTagHashes.add(tag.__hash__())
645            tagHashSet = set()
646            for itemID in setOfAllInternalItemIDs:
647                ly()
648                if itemID in self.itemWithTags:
649                    commonTagTupleHash = self.itemWithTags[itemID]
650                    tagSet = set(self.commonTagSets[commonTagTupleHash])
651                    tagHashSet.update(tagSet)
652            resultTagHashSet = tagHashSet - binTagHashes
653        elif treeType == SMART_TREE_TYPE:
654            # smartTree = self.build_smart_tree(binTags, prePreparedSetOfAllInternalItemIDs=setOfAllInternalItemIDs)
655            smartTree = self.build_smart_tree(binTags, prePreparedSetOfAllInternalItemIDs=setOfAllInternalItemIDs,
656                                              zeroSliceOnly=True)
657            if 0 in smartTree:
658                resultTagHashSet = smartTree[0]
659            # filteredItemIDsList = list()
660            # tagQnt = len(binTags)
661            # for itemID in listOfAllInternalItemIDs:
662            #     if len(self.itemWithTags[itemID]) == (tagQnt + 1):
663            #         filteredItemIDsList.append(itemID)
664            #
665            # tagHashSet = set()
666            # for itemID in filteredItemIDsList:
667            #     tagHashSet.update(set(self.itemWithTags[itemID]))
668            # givenTagHashes = set()
669            # for tag in binTags:
670            #     givenTagHashes.add(tag.__hash__())
671            # tagHashSet.difference_update(givenTagHashes)
672            # ##resultTagHashList = list(tagHashSet)
673            # # если остановиться тут - то мы увидим не все папки: мы не увидим папки непосредственно в которых есть
674            # # только другие подпапки, но ни одного файла.
675            # #
676            # # значит далее мы должны исключить все файлы, которые имеют только что найденные теги, и начать строить
677            # # древо тегов для оставшихся
678            # #
679            # # а далее - повторить все это в цикле, увеличив при проверке кол-во тегов еще раз на единицу (и используя
680            # # уже оставшийся после отсеивания набор файлов). В итоге кол-во итераций зависит не от количества файлов,
681            # # а от максимальной фактически имеющейся вложенности файлов внутри тегов-каталогов
682        else:
683            raise UnknownTreeTypeError()
684
685        sortedTagHashList = self.sort_tag_hash_list_by_qnt(resultTagHashSet - {self.get_root_tag().__hash__()})
686        return list(sortedTagHashList)
687
688    def get_tags_from_tags(self, binTags, treeType=USUAL_TREE_TYPE,
689                           prePreparedSetOfAllInternalItemIDsForThisSetOfTags=None):
690        result = self.get_tagHashes_from_tags(binTags, treeType=treeType,
691                                              prePreparedSetOfAllInternalItemIDsForThisSetOfTags=
692                                              prePreparedSetOfAllInternalItemIDsForThisSetOfTags)
693        return tuple(self.tag_hash_list_2_tag_list(result))
694
695    def build_smart_tree(self, startingBinTags, prePreparedSetOfAllInternalItemIDs=None, zeroSliceOnly=False):
696        ly = gly(self.default_priority)
697
698        startingBinTags = set(startingBinTags)
699
700        if self.get_root_tag() not in startingBinTags:
701            startingBinTags.add(self.get_root_tag())
702
703        startingTagHashes = set()
704        for tag in startingBinTags:
705            ly()
706            startingTagHashes.add(tag.__hash__())
707
708        if prePreparedSetOfAllInternalItemIDs is None:
709            setOfAllInternalItemIDs = self.get_itemIDs_from_tags(startingBinTags, treeType=PLAIN_PSEUDO_TREE_TYPE)
710        else:
711            setOfAllInternalItemIDs = prePreparedSetOfAllInternalItemIDs
712
713        smartTree = {0: set()}
714        smartTree__filler = AddToCompoundDict__Set(smartTree)
715        local_tags_qnt = dict()
716        local_tags_qnt__filler = KeyCounter(local_tags_qnt)
717        for itemID in setOfAllInternalItemIDs:
718            ly()
719            commonTagTupleHash = self.itemWithTags[itemID]
720            tagSet = set(self.commonTagSets[commonTagTupleHash])
721            setOfTags = tagSet
722            setOfTags = setOfTags - startingTagHashes
723            for tag_hash in setOfTags:
724                ly()
725                # if tag_hash not in local_tags_qnt:
726                #     local_tags_qnt[tag_hash] = 0
727                # local_tags_qnt[tag_hash] += 1
728                local_tags_qnt__filler.add(tag_hash)
729
730        for itemID in setOfAllInternalItemIDs:
731            ly()
732            commonTagTupleHash = self.itemWithTags[itemID]
733            tagSet = set(self.commonTagSets[commonTagTupleHash])
734            setOfTags = tagSet
735            setOfTags = setOfTags - startingTagHashes
736            listOfTagHashes = None
737            if zeroSliceOnly:
738                listOfTagHashes = self.get_top_tag_hash_list_by_qnt(setOfTags, local_tags_qnt)
739            else:
740                listOfTagHashes = self.sort_tag_hash_list_by_qnt(setOfTags)
741
742            lastTagHash = None
743            lastTagHashQnt = None
744            treeLevel = 0
745            for tagHash in listOfTagHashes:
746                ly()
747                # currentTagHashQnt = self.tagsQnt[tagHash]
748                currentTagHashQnt = local_tags_qnt[tagHash]
749                if (lastTagHash is None) or (currentTagHashQnt == lastTagHashQnt):
750                    pass
751                else:
752                    treeLevel += 1
753                    lastTagHash = None
754                    lastTagHashQnt = None
755                # if treeLevel not in smartTree:
756                #     smartTree[treeLevel] = set()
757                # # tagsSetOnTheLevel = smartTree[treeLevel]
758                # # tagsSetOnTheLevel.add(tagHash)
759                # # smartTree[treeLevel] = tagsSetOnTheLevel
760                # smartTree[treeLevel].add(tagHash)
761                smartTree__filler.add(treeLevel, tagHash)
762                lastTagHash = tagHash
763                lastTagHashQnt = currentTagHashQnt
764        return dict(smartTree)
765
766    def build_smart_tree_2(self, startingBinTags, prePreparedSetOfAllInternalItemIDs=None, zeroSliceOnly=False):
767        ly = gly(self.default_priority)
768
769        startingBinTags = set(startingBinTags)
770
771        if self.get_root_tag() not in startingBinTags:
772            startingBinTags.add(self.get_root_tag())
773
774        startingTagHashes = set()
775        for tag in startingBinTags:
776            ly()
777            startingTagHashes.add(tag.__hash__())
778
779        if prePreparedSetOfAllInternalItemIDs is None:
780            setOfAllInternalItemIDs = self.get_itemIDs_from_tags(startingBinTags, treeType=PLAIN_PSEUDO_TREE_TYPE)
781        else:
782            setOfAllInternalItemIDs = prePreparedSetOfAllInternalItemIDs
783
784        smartTree = {0: set()}
785        smartTree__filler = AddToCompoundDict__Set(smartTree)
786        local_tags_qnt = dict()
787        local_tags_qnt__filler = KeyCounter(local_tags_qnt)
788        for itemID in setOfAllInternalItemIDs:
789            ly()
790            commonTagTupleHash = self.itemWithTags[itemID]
791            tagSet = set(self.commonTagSets[commonTagTupleHash])
792            setOfTags = tagSet
793            setOfTags = setOfTags - startingTagHashes
794            for tag_hash in setOfTags:
795                ly()
796                # if tag_hash not in local_tags_qnt:
797                #     local_tags_qnt[tag_hash] = 0
798                # local_tags_qnt[tag_hash] += 1
799                local_tags_qnt__filler.add(tag_hash)
800
801        for itemID in setOfAllInternalItemIDs:
802            ly()
803            commonTagTupleHash = self.itemWithTags[itemID]
804            tagSet = set(self.commonTagSets[commonTagTupleHash])
805            setOfTags = tagSet
806            setOfTags = setOfTags - startingTagHashes
807            listOfTagHashes = None
808            if zeroSliceOnly:
809                listOfTagHashes = self.get_top_tag_hash_list_by_qnt(setOfTags, local_tags_qnt)
810            else:
811                listOfTagHashes = self.sort_tag_hash_list_by_qnt(setOfTags)
812
813            lastTagHash = None
814            lastTagHashQnt = None
815            treeLevel = 0
816            for tagHash in listOfTagHashes:
817                ly()
818                # currentTagHashQnt = self.tagsQnt[tagHash]
819                currentTagHashQnt = local_tags_qnt[tagHash]
820                if (lastTagHash is None) or (currentTagHashQnt == lastTagHashQnt):
821                    pass
822                else:
823                    treeLevel += 1
824                    lastTagHash = None
825                    lastTagHashQnt = None
826                # if treeLevel not in smartTree:
827                #     smartTree[treeLevel] = set()
828                # # tagsSetOnTheLevel = smartTree[treeLevel]
829                # # tagsSetOnTheLevel.add(tagHash)
830                # # smartTree[treeLevel] = tagsSetOnTheLevel
831                # smartTree[treeLevel].add(tagHash)
832                smartTree__filler.add(treeLevel, tagHash)
833                lastTagHash = tagHash
834                lastTagHashQnt = currentTagHashQnt
835        return dict(smartTree)
836
837    def get_all_from_tags(self, binTags, treeType=USUAL_TREE_TYPE):
838        binTags = set(binTags)
839        items = self.get_items_from_tags(binTags, treeType=treeType,
840                                         isAlsoNeedSetOfAllInternalItemIDsForThisSetOfTags=True)
841        if len(items[1]) > 0:
842            tags = self.get_tags_from_tags(binTags, treeType=treeType,
843                                           prePreparedSetOfAllInternalItemIDsForThisSetOfTags=items[1])
844            result = (tuple(tags), set(items[0]))
845            return result
846        else:
847            result = (tuple(), set())
848            return result
849
850    def get_tagsHashes_from_single_item(self, itemID, isWithoutRootHash=True):
851        if itemID in self.itemWithTags:
852            commonTagTupleHash = self.itemWithTags[itemID]
853            tagSet = set(self.commonTagSets[commonTagTupleHash])
854            if isWithoutRootHash:
855                return set(tagSet - {self.get_root_tag().__hash__()})
856            else:
857                return set(tagSet)
858        else:
859            return set()
860
861    def get_potential_itemIDs_from_item(self, binItem):
862        itemHash = binItem.__hash__()
863        if itemHash in self.itemIDsForItem:
864            return set(self.itemIDsForItem[itemHash])
865        else:
866            return set()
867
868    def is_smart_redirection_for_a_tag_path_reduction_needed(self, binTags):
869        ly = gly(self.default_priority)
870
871        binTags = set(binTags)
872
873        if self.get_root_tag() not in binTags:
874            binTags.add(self.get_root_tag())
875
876        tagHashSet = set()
877        for binTag in binTags:
878            ly()
879            tagHashSet.add(binTag.__hash__())
880
881        binTagsQnt = len(tagHashSet)
882        commonTagGroupHashSet = set()
883        # setOfLenOfTheCommonTagHashSetForChecking = set()
884        setOfTheTagsIntersection = None
885        for commonTagQnt in self.setOfTagGroupQnt:
886            ly()
887            if commonTagQnt > binTagsQnt:
888                setOfTheCommonTagGroupHashes = self.tagsQntPerCommonTagSet[commonTagQnt]
889                commonTagGroupHashSet.update(setOfTheCommonTagGroupHashes)
890        for commonTagGroupHash in commonTagGroupHashSet:
891            ly()
892            commonTagHashTuple = self.commonTagSets[commonTagGroupHash]
893            commonTagHashSet = set(commonTagHashTuple)
894            if len(tagHashSet & commonTagHashSet) == len(tagHashSet):
895                if tagHashSet != commonTagHashSet:
896                    if setOfTheTagsIntersection is None:
897                        setOfTheTagsIntersection = commonTagHashSet
898                    else:
899                        setOfTheTagsIntersection = setOfTheTagsIntersection & commonTagHashSet
900        #         if tagHashSet != commonTagHashSet:
901        #             setOfLenOfTheCommonTagHashSetForChecking.add(len(commonTagHashSet))
902        # minimalTagPath = min(setOfLenOfTheCommonTagHashSetForChecking)
903        # pathDiff = minimalTagPath - len(tagHashSet)
904        # if pathDiff > 0:
905        if setOfTheTagsIntersection is None:
906            return list()
907
908        setOfTheTagsForAReduction = setOfTheTagsIntersection - tagHashSet
909
910        sortedTagHashList = self.sort_tag_hash_list_by_qnt(setOfTheTagsForAReduction - {self.get_root_tag().__hash__()})
911        return list(sortedTagHashList)
912
913    def get_tags_for_a_smart_redirection(self, binTags):
914        result = self.is_smart_redirection_for_a_tag_path_reduction_needed(binTags)
915        return tuple(self.tag_hash_list_2_tag_list(result))

SMART_TREE_TYPE = 0

SMART_TREE_TYPE_WITH_INTERNAL_MENU = 1

FULL_TREE_TYPE = 2

PLAIN_PSEUDO_TREE_TYPE = 3

USUAL_TREE_TYPE = 3

class ToManyIdenticalItemsOnTheGivenTagPathError(builtins.Exception): View Source

67class ToManyIdenticalItemsOnTheGivenTagPathError(Exception):
68    pass

Common base class for all non-exit exceptions.

Inherited Members

builtins.Exception: Exception
builtins.BaseException: with_traceback; args

class UnknownTreeTypeError(builtins.Exception): View Source

71class UnknownTreeTypeError(Exception):
72    pass

Common base class for all non-exit exceptions.

Inherited Members

builtins.Exception: Exception
builtins.BaseException: with_traceback; args

class LockableMixin: View Source

75class LockableMixin:
76    @property
77    def lock(self) -> bool:
78        raise NotImplementedError
79    
80    @lock.setter
81    def lock(self, value: bool) -> bool:
82        raise NotImplementedError

lock: bool View Source

76    @property
77    def lock(self) -> bool:
78        raise NotImplementedError

@contextmanager

def obj_locker( obj: LockableMixin): View Source

85@contextmanager
86def obj_locker(obj: LockableMixin):
87    obj.lock = True
88    try:
89        yield obj.lock
90    finally:
91        obj.lock = False

class Example(LockableMixin): View Source

 94class Example(LockableMixin):
 95    def __init__(self) -> None:
 96        super().__init__()
 97        self.lock: bool = False
 98    
 99    def write_coroutine(self):
100        with obj_locker(self):
101            pass
102    
103    def read_coroutine(self):
104        if not self.lock:
105            pass

lock: bool View Source

76    @property
77    def lock(self) -> bool:
78        raise NotImplementedError

def write_coroutine(self): View Source

 99    def write_coroutine(self):
100        with obj_locker(self):
101            pass

def read_coroutine(self): View Source

103    def read_coroutine(self):
104        if not self.lock:
105            pass