cengal.data_containers.dynamic_tag_tree.versions.v_0.TagDB

Module Docstring Docstrings: http://www.python.org/dev/peps/pep-0257/

View Source

  1#!/usr/bin/env python
  2# coding=utf-8
  3
  4# Copyright © 2012-2024 ButenkoMS. All rights reserved. Contacts: <gtalk@butenkoms.space>
  5# 
  6# Licensed under the Apache License, Version 2.0 (the "License");
  7# you may not use this file except in compliance with the License.
  8# You may obtain a copy of the License at
  9# 
 10#     http://www.apache.org/licenses/LICENSE-2.0
 11# 
 12# Unless required by applicable law or agreed to in writing, software
 13# distributed under the License is distributed on an "AS IS" BASIS,
 14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 15# See the License for the specific language governing permissions and
 16# limitations under the License.
 17
 18
 19"""
 20Module Docstring
 21Docstrings: http://www.python.org/dev/peps/pep-0257/
 22"""
 23
 24
 25__author__ = "ButenkoMS <gtalk@butenkoms.space>"
 26__copyright__ = "Copyright © 2012-2024 ButenkoMS. All rights reserved. Contacts: <gtalk@butenkoms.space>"
 27__credits__ = ["ButenkoMS <gtalk@butenkoms.space>", ]
 28__license__ = "Apache License, Version 2.0"
 29__version__ = "4.4.1"
 30__maintainer__ = "ButenkoMS <gtalk@butenkoms.space>"
 31__email__ = "gtalk@butenkoms.space"
 32# __status__ = "Prototype"
 33__status__ = "Development"
 34# __status__ = "Production"
 35
 36
 37__author__ = 'Mikhail Butenko <gtalk@mikhail-butenko.in.ua>'
 38
 39from cengal.data_generation.id_generator import IDGenerator
 40# from help_tools import AddToCompoundDict, ResultExistence
 41from cengal.code_flow_control.smart_values.versions.v_0 import ResultExistence
 42from cengal.data_containers.compound_dict_management import AddToCompoundDict
 43
 44
 45SMART_TREE_TYPE = 0  # smart tree. Умное дерево тегов: сеть отображенная на древо. Возвращает только список
 46    # непосредственных подтегов текущего пути, но не их подтеги; возвращает элементы текущего пути, но не элементы
 47    # из подпутей
 48SMART_TREE_TYPE_WITH_INTERNAL_MENU = 1   # smart tree with internal menu. В древо встроено меню, позволяющее прямо из
 49    # древа производить смену типа вывода: SMART_TREE_TYPE, FULL_TREE_TYPE и PLAIN_PSEUDO_TREE_TYPE. На каждый тип
 50    # вывода будет доступен подтег/подпапка, внутри когорого уже будет нормальное древо элементов, но уже выбранного
 51    # типа
 52FULL_TREE_TYPE = 2  # full tree with all tags - with repeats and without filtering. Список айтемов - как у
 53    # SMART_TREE_TYPE, но при этом список тегов - как у PLAIN_PSEUDO_TREE_TYPE
 54PLAIN_PSEUDO_TREE_TYPE = 3   # plain tags and items set (will show all tags, subtags and items of
 55    # current hm... dir - current tag set). Показывает все теги и подтеги единым списком - как у примитивных теговых
 56    # файловых систем; показывает все элементы текущего пути + все элементы всех под-путей
 57
 58USUAL_TREE_TYPE = PLAIN_PSEUDO_TREE_TYPE
 59
 60_ROOT_TAG = r'k{1+vdcY#m8t-4m9`)G2\b]/O\'Rzqyr@FEO~%./nGPzl)[^q 0RS!.bCPh ?fag{8~{SGj;Ss3U85Q-:'
 61
 62
 63class ToManyIdenticalItemsOnTheGivenTagPathError(Exception):
 64    pass
 65
 66
 67class UnknownTreeTypeError(Exception):
 68    pass
 69
 70
 71class AddToCompoundDict__Set(AddToCompoundDict):
 72    def __init__(self, original_dict):
 73        super().__init__(
 74            original_dict,
 75            lambda: set(),
 76            lambda original_dict, key, value: original_dict[key].add(value)
 77        )
 78
 79
 80class TagDB:
 81
 82    def __init__(self):
 83        super().__init__()
 84
 85        self.itemsID = IDGenerator()
 86
 87        self.itemIDsForItem = {}  # key - item hash; data - set of itemIDs
 88
 89        self.itemsSet = {}  # key - ItemID; data - binItem
 90        # TODO: заменить список тегов на хеш единожды сохраненного списка тегов
 91        self.itemWithTags = {}  # key - ItemID; data - sorted common TagsTuple's hash
 92
 93        # TODO: убрать tagsNumPerItemID из кода. Заменить этот список itemID - на список hashOfTheTagHashTuple
 94        self.tagsNumPerItemID = {}  # key - number of tags in this ItemID group; data - set of itemIDs which are have
 95            # needed number of tags
 96
 97        self.tagsSet = {}  # key - tag hash; data - binTag
 98        self.tagWithItems = {}  # key - Tag hash; data - set of itemIDs
 99        self.tagsQnt = {}    # key - Tag hash; data - quantity of the items with this tag
100
101
102        self.commonTagSets = {}  # key - sorted common TagsTuple's hash; data - sorted TagsTuple
103        self.itemsOnTheCommonTagSets = {}   # key - sorted common TagsTuple's hash; data - set of itemIDs
104        self.tagsQntPerCommonTagSet = {}    # key - number of tags; data - set of TagsTuple hashes
105        self.setOfTagGroupQnt = set()   # {tagQntInGroup1, tagQntInGroup2, ..., tagQntInGroupN} where
106            # each Group is an key of the self.tagsQntPerCommonTagSet
107
108        # TODO: добавить словарь tagHashTuplesIDs вида [tagHashTuple.__hash__():tagHashTuple]
109        # где tagHashSet - это {tagHash_1, tagHash_2, ..., tagHash_3}
110        # TODO: добавить словарь itemsOnThePath вида [tagHashTuple.__hash__():{itemID_1, itemID_2, ..., itemID_3}]
111
112        # TODO: добавить словарь вида [tagQnt:{itemID_1, itemID_2, ..., itemID_3}] и вычитывать это из него
113        # и/или
114        # TODO: добавить словарь вида [tagQnt:{hashOfTheTagHashTuple_1, hashOfTheTagHashTuple_2, ...
115        # , hashOfTheTagHashTuple_N}] и вычитывать это из него, а потом уже и из каждого tagHashSet
116        # где hashOfTheTagHashTuple - это tagHashTuple.__hash__()
117
118    def get_root_tag(self):
119        return str(_ROOT_TAG)
120
121    def add_tag(self, binTag):
122        tagHash = binTag.__hash__()
123        self.tagsSet[tagHash] = binTag
124        if tagHash not in self.tagsSet:
125            self.tagsQnt[tagHash] = 0
126        if tagHash not in self.tagWithItems:
127            self.tagWithItems[tagHash] = set()
128
129    def remove_tag(self, binTag):
130        # will try to delete given tag. If there is at least one item with this tag, than function will fail
131        # and will return False; otherwise it will delete given tag and will return True.
132        functionResult = False
133        tagHash = binTag.__hash__()
134        if tagHash in self.tagsSet:
135            if tagHash in self.tagsQnt:
136                if self.tagsQnt[tagHash] == 0:
137                    del self.tagsSet[tagHash]
138                    del self.tagsQnt[tagHash]
139                    functionResult = True
140                else:
141                    functionResult = False
142            else:
143                del self.tagsSet[tagHash]
144                functionResult = True
145
146        if functionResult:
147            if tagHash in self.tagWithItems:
148                del self.tagWithItems[tagHash]
149
150        return functionResult
151
152    def add_item(self, binItem, binTags):
153        # will add new item and return it's dynamic ID or None object If this Item already exist on the given tag path
154        # Or will raise an exception if we already have more than one binItem (another item that is identical to
155        # the given binItem)  on this tag path
156
157        binTags = set(binTags)
158
159        if self.get_root_tag() not in binTags:
160            binTags.add(self.get_root_tag())
161
162        # may raise an exception in this place. Nope - from now it will be not
163        if self.get_itemID_from_item_and_tags(binTags, binItem) is not None:
164            return None
165
166        itemID = self.itemsID()
167        self.itemsSet[itemID] = binItem
168
169        itemHash = binItem.__hash__()
170        if itemHash in self.itemIDsForItem:
171            IDsSet = self.itemIDsForItem[itemHash]
172            IDsSet.add(itemID)
173            # self.itemIDsForItem[itemHash] = IDsSet
174        else:
175            self.itemIDsForItem[itemHash] = {itemID}
176
177        tagQnt = len(binTags)
178        if tagQnt in self.tagsNumPerItemID:
179            itemIDsSet = self.tagsNumPerItemID[tagQnt]
180            itemIDsSet.add(itemID)
181            # self.tagsNumPerItemID[tagQnt] = itemIDsSet
182        else:
183            self.tagsNumPerItemID[tagQnt] = {itemID}
184
185        binTagHashes = set()
186
187        for tag in binTags:
188            self.add_tag(tag)
189            tagHash = tag.__hash__()
190            binTagHashes.add(tagHash)
191            setOfItems = self.tagWithItems[tagHash]
192            if itemID not in setOfItems:
193                setOfItems.add(itemID)
194                if tagHash in self.tagsQnt:
195                    self.tagsQnt[tagHash] += 1
196                else:
197                    self.tagsQnt[tagHash] = 1
198            self.tagWithItems[tagHash] = setOfItems
199
200        sortedTagTuple = tuple(self.sort_tag_hash_list_by_hash(binTagHashes))
201        hashOfTheSortedTagTuple = sortedTagTuple.__hash__()
202        self.itemWithTags[itemID] = hashOfTheSortedTagTuple
203
204        self.commonTagSets[hashOfTheSortedTagTuple] = sortedTagTuple
205
206        if hashOfTheSortedTagTuple in self.itemsOnTheCommonTagSets:
207            itemIDsSet = self.itemsOnTheCommonTagSets[hashOfTheSortedTagTuple]
208            itemIDsSet.add(itemID)
209            # self.itemsOnTheCommonTagSets[tagQnt] = itemIDsSet
210        else:
211            self.itemsOnTheCommonTagSets[hashOfTheSortedTagTuple] = {itemID}
212
213        lenOfTheSortedTagTuple = len(sortedTagTuple)
214        if lenOfTheSortedTagTuple in self.tagsQntPerCommonTagSet:
215            itemIDsSet = self.tagsQntPerCommonTagSet[lenOfTheSortedTagTuple]
216            itemIDsSet.add(hashOfTheSortedTagTuple)
217            # self.tagsQntPerCommonTagSet[lenOfTheSortedTagTuple] = itemIDsSet
218        else:
219            self.tagsQntPerCommonTagSet[lenOfTheSortedTagTuple] = {hashOfTheSortedTagTuple}
220            self.setOfTagGroupQnt.add(lenOfTheSortedTagTuple)
221
222        return itemID
223
224    def remove_item_by_itemID(self, itemID):
225        if itemID in self.itemsSet:
226            itemHash = self.itemsSet[itemID].__hash__()
227            del self.itemsSet[itemID]
228
229            if itemHash in self.itemIDsForItem:
230                IDsSet = self.itemIDsForItem[itemHash]
231                IDsSet.difference_update({itemID})
232                # self.itemIDsForItem[itemHash] = IDsSet
233                if len(IDsSet) < 1:
234                    del self.itemIDsForItem[itemHash]
235
236        if itemID in self.itemWithTags:
237            commonTagTupleHash = self.itemWithTags[itemID]
238            tagTuple = self.commonTagSets[commonTagTupleHash]
239            numberOfTags = len(tagTuple)
240
241            if commonTagTupleHash in self.itemsOnTheCommonTagSets:
242                IDsSet = self.itemsOnTheCommonTagSets[commonTagTupleHash]
243                IDsSet.difference_update({itemID})
244                # self.itemsOnTheCommonTagSets[commonTagTupleHash] = IDsSet
245                if len(IDsSet) < 1:
246                    del self.itemsOnTheCommonTagSets[commonTagTupleHash]
247                    del self.commonTagSets[commonTagTupleHash]
248                    if numberOfTags in self.tagsQntPerCommonTagSet:
249                        setOfTagTuplesHashes = self.tagsQntPerCommonTagSet[numberOfTags]
250                        setOfTagTuplesHashes.difference_update({numberOfTags})
251                        # self.tagsQntPerCommonTagSet[numberOfTags] = setOfTagTuplesHashes
252                        if len(setOfTagTuplesHashes) < 1:
253                            del self.tagsQntPerCommonTagSet[numberOfTags]
254                            self.setOfTagGroupQnt.difference_update({numberOfTags})
255
256            del self.itemWithTags[itemID]
257
258            setOfTagHashes = set(tagTuple)
259
260            tagsQnt = len(setOfTagHashes)
261            if tagsQnt in self.tagsNumPerItemID:
262                IDsSet = self.tagsNumPerItemID[tagsQnt]
263                IDsSet.difference_update({itemID})
264                # self.tagsNumPerItemID[tagsQnt] = IDsSet
265                if len(IDsSet) < 1:
266                    del self.tagsNumPerItemID[tagsQnt]
267
268            for tagHash in setOfTagHashes:
269                if tagHash in self.tagsQnt:
270                    tagsQuantity = self.tagsQnt[tagHash]
271                    tagsQuantity -= 1
272                    if tagsQuantity < 1:
273                        tagsQuantity = 0
274                    self.tagsQnt[tagHash] = tagsQuantity
275                    if tagsQuantity < 1:
276                        del self.tagsQnt[tagHash]
277                        del self.tagsSet[tagHash]
278                if tagHash in self.tagWithItems:
279                    IDsSet = self.tagWithItems[tagHash]
280                    IDsSet.difference_update({itemID})
281                    if len(IDsSet) < 1:
282                        del self.tagWithItems[tagHash]
283                    # self.tagWithItems[tagHash] = IDsSet
284
285        self.itemsID.remove_id(itemID)
286
287    def remove_item(self, binTags, binItem):
288        # will return ItemId for deleted item or None object if Item is not exist
289        # Or will raise an exception if we already have more than one binItem (another item that is identical to
290        # the given binItem) on this tag path
291        binTags = set(binTags)
292        if self.get_root_tag() not in binTags:
293            binTags.add(self.get_root_tag())
294        itemID = self.get_itemID_from_item_and_tags(binTags, binItem)
295        if itemID is not None:
296            self.remove_item_by_itemID(itemID)
297        return itemID
298
299    def __OLD__get_itemID_from_item_and_tags(self, binTags, binItem):
300        if self.get_root_tag() not in binTags:
301            binTags.append(self.get_root_tag())
302        potentialIDs = set(self.get_potential_itemIDs_from_item(binItem))
303        itemIDsSet = set(self.get_itemIDs_from_tags(binTags, SMART_TREE_TYPE))
304        resultItemIDsList = potentialIDs & itemIDsSet
305        if len(resultItemIDsList) == 0:
306            return None
307        elif len(resultItemIDsList) == 1:
308            resultItemID = resultItemIDsList.pop()  # we have assume that we'll have only one item in intersection
309                # between potential IDs and Items that have (and have only) given tag list (without another tags in the
310                # path to this items). We need to check it in the adding new item to the given tag path.
311            return resultItemID
312        elif len(resultItemIDsList) > 1:
313            raise ToManyIdenticalItemsOnTheGivenTagPathError()
314
315    def get_itemID_from_item_and_tags(self, binTags, binItem):
316        binTags = set(binTags)
317
318        if self.get_root_tag() not in binTags:
319            binTags.add(self.get_root_tag())
320
321        potentialIDs = self.get_potential_itemIDs_from_item(binItem)
322        setOfBinTagsHashes = set()
323        for tag in binTags:
324            setOfBinTagsHashes.add(tag.__hash__())
325        for itemID in potentialIDs:
326            currentItemTagsSet = self.get_tagsHashes_from_single_item(itemID, isWithoutRootHash=False)
327            if setOfBinTagsHashes == currentItemTagsSet:
328                return itemID
329        return None
330
331    def tag_hash_list_2_tag_list(self, tagHashList):
332        tagList = list()
333        for tagHash in tagHashList:
334            tagList.append(self.tagsSet[tagHash])
335        return tagList
336
337    def get_item_and_tags_from_itemID(self, itemID):
338        commonTagTupleHash = self.itemWithTags[itemID]
339        tagSet = set(self.commonTagSets[commonTagTupleHash]) - {self.get_root_tag().__hash__()}
340        sortedTagHashList = self.sort_tag_hash_list_by_qnt(tagSet)
341        result = (self.itemsSet[itemID], self.tag_hash_list_2_tag_list(sortedTagHashList))
342        return result
343
344    # @profile
345    def get_top_tag_hash_list_by_qnt(self, tagHashSet, local_tags_qnt=None):
346        tagsQnt = self.tagsQnt
347        if local_tags_qnt is not None:
348            tagsQnt = local_tags_qnt
349        tag_hash_set = set(tagHashSet)
350        tag_by_qnt = dict()
351        tag_by_qnt__filler = AddToCompoundDict__Set(tag_by_qnt)
352        biggest_qnt = 0
353        for tag_hash in tag_hash_set:
354            qnt = tagsQnt[tag_hash]
355            if qnt > biggest_qnt:
356                biggest_qnt = qnt
357            # if qnt not in tag_by_qnt:
358            #     tag_by_qnt[qnt] = set()
359            # tag_by_qnt[qnt].add(tag_hash)
360            tag_by_qnt__filler.add(qnt, tag_hash)
361
362        result = None
363        if len(tag_by_qnt) > 0:
364            # biggest_qnt = max(tag_by_qnt)
365            result = tuple(tag_by_qnt[biggest_qnt])
366        else:
367            result = tuple()
368        return result
369
370    def sort_tag_hash_list_by_qnt(self, tagHashSet):
371        # will return sorted tag list - not sorted tag hash list
372        tagHashSet = set(tagHashSet)
373        rawTagList = list()
374        for tagHash in tagHashSet:
375            tagWithWeight = (tagHash, self.tagsQnt[tagHash])
376            rawTagList.append(tagWithWeight)
377        return self.sort_raw_tag_list(rawTagList)
378
379    def sort_tag_list_by_qnt(self, binTags):
380        binTags = set(binTags)
381        rawTagList = list()
382        for tag in binTags:
383            tagHash = tag.__hash__()
384            tagWithWeight = (tag, self.tagsQnt[tagHash])
385            rawTagList.append(tagWithWeight)
386        return self.sort_raw_tag_list(rawTagList)
387
388    def sort_tag_hash_list_by_hash(self, tagHashSet):
389        # will return sorted tag list - not sorted tag hash list
390        tagHashSet = set(tagHashSet)
391        rawTagList = list()
392        for tagHash in tagHashSet:
393            tagWithWeight = (tagHash, tagHash)
394            rawTagList.append(tagWithWeight)
395        return self.sort_raw_tag_list(rawTagList)
396
397    def sort_tag_list_by_hash(self, binTags):
398        binTags = set(binTags)
399        rawTagList = list()
400        for tag in binTags:
401            tagHash = tag.__hash__()
402            tagWithWeight = (tag, tagHash)
403            rawTagList.append(tagWithWeight)
404        return self.sort_raw_tag_list(rawTagList)
405
406    def sort_raw_tag_list(self, rawTagList):
407        # will return sorted tag list
408        rawTagList = sorted(rawTagList, key=lambda tagAndWeight: tagAndWeight[1], reverse=True)
409        tagList = list()
410        for rawTag in rawTagList:
411            tagList.append(rawTag[0])
412        return tagList
413
414    def get_itemIDs_from_tags(self, binTags, treeType=USUAL_TREE_TYPE,
415                              isAlsoNeedSetOfAllInternalItemIDsForThisSetOfTags=False):
416        # TODO: исправить ошибку: SMART_TREE_TYPE: возвращает не только список файлов в текущей директории, но и из
417        # непосредственных подпапок данной папки
418
419        # treeType - type of the graph tree representation: show all tags with replies (pure representation);
420        # show only relevant tags; etc.
421        # return set of itemIDs
422
423        binTags = set(binTags)
424
425        if self.get_root_tag() not in binTags:
426            binTags.add(self.get_root_tag())
427
428        tagHashSet = set()
429        for binTag in binTags:
430            tagHashSet.add(binTag.__hash__())
431
432        # PLAIN_PSEUDO_TREE_TYPE
433        interceptionOfItemsWithTags = set()
434        if (treeType == PLAIN_PSEUDO_TREE_TYPE) or isAlsoNeedSetOfAllInternalItemIDsForThisSetOfTags:
435            binTagsQnt = len(tagHashSet)
436            commonTagGroupHashSet = set()
437            tagSet = set()
438            itemIDSet = set()
439            binTagHashTuple = tuple(self.sort_tag_hash_list_by_hash(tagHashSet))
440            hashOfTheBinTagHashTuple = binTagHashTuple.__hash__()
441            if hashOfTheBinTagHashTuple in self.itemsOnTheCommonTagSets:
442                itemIDSet = self.itemsOnTheCommonTagSets[hashOfTheBinTagHashTuple]
443            for commonTagQnt in self.setOfTagGroupQnt:
444                if commonTagQnt > binTagsQnt:
445                    setOfTheCommonTagGroupHashes = self.tagsQntPerCommonTagSet[commonTagQnt]
446                    commonTagGroupHashSet.update(setOfTheCommonTagGroupHashes)
447            for commonTagGroupHash in commonTagGroupHashSet:
448                commonTagHashTuple = self.commonTagSets[commonTagGroupHash]
449                commonTagHashSet = set(commonTagHashTuple)
450                if tagHashSet.issubset(commonTagHashSet):
451                    itemIDSet.update(self.itemsOnTheCommonTagSets[commonTagGroupHash])
452                # # if len(tagHashSet & commonTagHashSet) == len(tagHashSet):
453                # res_set = tagHashSet.intersection(commonTagHashSet)
454                # if len(res_set) == binTagsQnt:
455                #     itemIDSet = itemIDSet | self.itemsOnTheCommonTagSets[commonTagGroupHash]
456            interceptionOfItemsWithTags = itemIDSet
457
458            # isFirstHash = True
459            # for tag in binTags:
460            #     tagHash = tag.__hash__()
461            #     if tagHash in self.tagWithItems:
462            #         if isFirstHash:
463            #             interceptionOfItemsWithTags = self.tagWithItems[tagHash]
464            #             isFirstHash = False
465            #         else:
466            #             itemsWithTag = self.tagWithItems[tagHash]
467            #             interceptionOfItemsWithTags = interceptionOfItemsWithTags & itemsWithTag
468            #     else:
469            #         # TODO: произвести такую же провеку в get_items_from_tags() и build_smart_tree()
470            #         if isAlsoNeedSetOfAllInternalItemIDsForThisSetOfTags:
471            #             result = (set(), set())
472            #             return result
473            #         else:
474            #             return set()
475
476        resultItemIDSet = interceptionOfItemsWithTags
477        setOfAllInternalItemIDsForThisSetOfTags = interceptionOfItemsWithTags
478
479        # SMART_TREE_TYPE or FULL_TREE_TYPE
480        if (treeType == SMART_TREE_TYPE) or (treeType == FULL_TREE_TYPE):
481            resultItemIDSet = set()
482            binTagHashTuple = self.sort_tag_hash_list_by_hash(tagHashSet)
483            binTagHashTuple = tuple(binTagHashTuple)
484            hashOfTheBinTagHashTuple = binTagHashTuple.__hash__()
485            if hashOfTheBinTagHashTuple in self.itemsOnTheCommonTagSets:
486                resultItemIDSet = self.itemsOnTheCommonTagSets[hashOfTheBinTagHashTuple]
487
488            # filteredItemIDsSet = set()
489            # tagQnt = len(binTags)
490            # # for itemID in setOfAllInternalItemIDsForThisSetOfTags:
491            # #     if len(self.itemWithTags[itemID]) == tagQnt:
492            # #         # _TODO: добавить словарь вида [tagQnt:{itemID_1, itemID_2, ..., itemID_3}]
493            # #         # и вычитывать это из него
494            # #         # и/или
495            # #         # _TODO: добавить словарь вида [tagQnt:{hashOfTheTagHashSet_1, hashOfTheTagHashSet_2, ...
496            # #         # , hashOfTheTagHashSet_3}] и вычитывать это из него, а потом уже и из каждого tagHashSet
497            # #         # где hashOfTheTagHashSet - это tagHashSet.__hash__()
498            # #         filteredItemIDsSet.add(itemID)
499            # if tagQnt in self.tagsNumPerItemID:
500            #     filteredItemIDsSet = setOfAllInternalItemIDsForThisSetOfTags & self.tagsNumPerItemID[tagQnt]
501            #
502            # resultItemIDSet = set()
503            # tagHashSet = set()
504            # for binTag in binTags:
505            #     tagHashSet.add(binTag.__hash__())
506            # for itemID in filteredItemIDsSet:
507            #     commonTagTupleHash = self.itemWithTags[itemID]
508            #     tagSet = set(self.commonTagSets[commonTagTupleHash])
509            #     if tagSet == tagHashSet:
510            #         # _TODO: добавить словарь tagHashTuplesIDs вида [tagHashTuple.__hash__():tagHashTuple]
511            #         # где tagHashSet - это {tagHash_1, tagHash_2, ..., tagHash_3}
512            #         # _TODO: добавить словарь itemsOnThePath вида [tagHashTuple.__hash__():{itemID_1, itemID_2, ...
513            #         # , itemID_3}]
514            #         resultItemIDSet.add(itemID)
515        elif treeType == PLAIN_PSEUDO_TREE_TYPE:
516            # already implemented (see bellow). Don't touch this code!
517            pass
518        else:
519            raise UnknownTreeTypeError()
520
521        if isAlsoNeedSetOfAllInternalItemIDsForThisSetOfTags:
522            result = (set(resultItemIDSet), set(setOfAllInternalItemIDsForThisSetOfTags))
523            return result
524        else:
525            return set(resultItemIDSet)
526
527    def get_items_from_tags(self, binTags, treeType=USUAL_TREE_TYPE,
528                            isAlsoNeedSetOfAllInternalItemIDsForThisSetOfTags=False):
529        # treeType - type of the graph tree representation: show all tags with replies (pure representation);
530        # show only relevant tags; etc.
531        # return set of itemIDs
532        binTags = set(binTags)
533        itemIDsSet = self.get_itemIDs_from_tags(binTags, treeType=treeType,
534                                                isAlsoNeedSetOfAllInternalItemIDsForThisSetOfTags=
535                                                isAlsoNeedSetOfAllInternalItemIDsForThisSetOfTags)
536        if isAlsoNeedSetOfAllInternalItemIDsForThisSetOfTags:
537            itemSet = set()
538            for itemID in itemIDsSet[0]:
539                itemSet.add(self.itemsSet[itemID])
540            result = (tuple(itemSet), tuple(itemIDsSet[1]))  # result == (usual items set, additional set of all
541                # internal itemIDs)
542            return result
543        else:
544            itemSet = set()
545            for itemID in itemIDsSet:
546                itemSet.add(self.itemsSet[itemID])
547            return tuple(itemSet)
548
549    def get_tagHashes_from_tags(self, binTags, treeType=USUAL_TREE_TYPE,
550                                prePreparedSetOfAllInternalItemIDsForThisSetOfTags=None):
551        # where "itemIDsSet" is externally given "get_itemIDs_from_tags(binTags, treeType=PLAIN_PSEUDO_TREE_TYPE)"
552        # so "itemIDsSet" is a set of the all items inside the "folder" binTags (including items from "subfolders")
553        # treeType - the same as in the "get_items_from_tags()" method
554        # return set of itemIDs
555        # prePreparedSetOfAllInternalItemIDsForThisSetOfTags can be generated by:
556        #   a) get_itemIDs_from_tags(..., treeType=PLAIN_PSEUDO_TREE_TYPE)
557        #   a) get_itemIDs_from_tags(..., isAlsoNeedSetOfAllInternalItemIDsForThisSetOfTags = True)
558        #   c) get_items_from_tags(..., isAlsoNeedSetOfAllInternalItemIDsForThisSetOfTags = True)
559        #   d) get_items_from_tags(..., treeType=PLAIN_PSEUDO_TREE_TYPE) || BUT: it'll return item set - not itemID
560        #       set
561
562        binTags = set(binTags)
563
564        resultTagHashSet = set()
565
566        setOfAllInternalItemIDs = set()
567        if prePreparedSetOfAllInternalItemIDsForThisSetOfTags is None:
568            setOfAllInternalItemIDs = self.get_itemIDs_from_tags(binTags, treeType=PLAIN_PSEUDO_TREE_TYPE)
569        else:
570            setOfAllInternalItemIDs = prePreparedSetOfAllInternalItemIDsForThisSetOfTags
571
572        if (treeType == PLAIN_PSEUDO_TREE_TYPE) or (treeType == FULL_TREE_TYPE):
573            binTagHashes = set()
574            for tag in binTags:
575                binTagHashes.add(tag.__hash__())
576            tagHashSet = set()
577            for itemID in setOfAllInternalItemIDs:
578                if itemID in self.itemWithTags:
579                    commonTagTupleHash = self.itemWithTags[itemID]
580                    tagSet = set(self.commonTagSets[commonTagTupleHash])
581                    tagHashSet.update(tagSet)
582            resultTagHashSet = tagHashSet - binTagHashes
583        elif treeType == SMART_TREE_TYPE:
584            # smartTree = self.build_smart_tree(binTags, prePreparedSetOfAllInternalItemIDs=setOfAllInternalItemIDs)
585            smartTree = self.build_smart_tree(binTags, prePreparedSetOfAllInternalItemIDs=setOfAllInternalItemIDs,
586                                              zeroSliceOnly=True)
587            if 0 in smartTree:
588                resultTagHashSet = smartTree[0]
589            # filteredItemIDsList = list()
590            # tagQnt = len(binTags)
591            # for itemID in listOfAllInternalItemIDs:
592            #     if len(self.itemWithTags[itemID]) == (tagQnt + 1):
593            #         filteredItemIDsList.append(itemID)
594            #
595            # tagHashSet = set()
596            # for itemID in filteredItemIDsList:
597            #     tagHashSet.update(set(self.itemWithTags[itemID]))
598            # givenTagHashes = set()
599            # for tag in binTags:
600            #     givenTagHashes.add(tag.__hash__())
601            # tagHashSet.difference_update(givenTagHashes)
602            # ##resultTagHashList = list(tagHashSet)
603            # # если остановиться тут - то мы увидим не все папки: мы не увидим папки непосредственно в которых есть
604            # # только другие подпапки, но ни одного файла.
605            # #
606            # # значит далее мы должны исключить все файлы, которые имеют только что найденные теги, и начать строить
607            # # древо тегов для оставшихся
608            # #
609            # # а далее - повторить все это в цикле, увеличив при проверке кол-во тегов еще раз на единицу (и используя
610            # # уже оставшийся после отсеивания набор файлов). В итоге кол-во итераций зависит не от количества файлов,
611            # # а от максимальной фактически имеющейся вложенности файлов внутри тегов-каталогов
612        else:
613            raise UnknownTreeTypeError()
614
615        sortedTagHashList = self.sort_tag_hash_list_by_qnt(resultTagHashSet - {self.get_root_tag().__hash__()})
616        return list(sortedTagHashList)
617
618    def get_tags_from_tags(self, binTags, treeType=USUAL_TREE_TYPE,
619                           prePreparedSetOfAllInternalItemIDsForThisSetOfTags=None):
620        result = self.get_tagHashes_from_tags(binTags, treeType=treeType,
621                                              prePreparedSetOfAllInternalItemIDsForThisSetOfTags=
622                                              prePreparedSetOfAllInternalItemIDsForThisSetOfTags)
623        return tuple(self.tag_hash_list_2_tag_list(result))
624
625    def build_smart_tree(self, startingBinTags, prePreparedSetOfAllInternalItemIDs=None, zeroSliceOnly=False):
626        startingBinTags = set(startingBinTags)
627
628        if self.get_root_tag() not in startingBinTags:
629            startingBinTags.add(self.get_root_tag())
630
631        startingTagHashes = set()
632        for tag in startingBinTags:
633            startingTagHashes.add(tag.__hash__())
634
635        if prePreparedSetOfAllInternalItemIDs is None:
636            setOfAllInternalItemIDs = self.get_itemIDs_from_tags(startingBinTags, treeType=PLAIN_PSEUDO_TREE_TYPE)
637        else:
638            setOfAllInternalItemIDs = prePreparedSetOfAllInternalItemIDs
639
640        smartTree = {0: set()}
641        smartTree__filler = AddToCompoundDict__Set(smartTree)
642        local_tags_qnt = dict()
643        local_tags_qnt__filler = AddToCompoundDict(
644            local_tags_qnt,
645            lambda: 0,
646            lambda container, key, value: (True, container[key] + 1)
647        )
648        for itemID in setOfAllInternalItemIDs:
649            commonTagTupleHash = self.itemWithTags[itemID]
650            tagSet = set(self.commonTagSets[commonTagTupleHash])
651            setOfTags = tagSet
652            setOfTags = setOfTags - startingTagHashes
653            for tag_hash in setOfTags:
654                # if tag_hash not in local_tags_qnt:
655                #     local_tags_qnt[tag_hash] = 0
656                # local_tags_qnt[tag_hash] += 1
657                local_tags_qnt__filler.add(tag_hash)
658
659        for itemID in setOfAllInternalItemIDs:
660            commonTagTupleHash = self.itemWithTags[itemID]
661            tagSet = set(self.commonTagSets[commonTagTupleHash])
662            setOfTags = tagSet
663            setOfTags = setOfTags - startingTagHashes
664            listOfTagHashes = None
665            if zeroSliceOnly:
666                listOfTagHashes = self.get_top_tag_hash_list_by_qnt(setOfTags, local_tags_qnt)
667            else:
668                listOfTagHashes = self.sort_tag_hash_list_by_qnt(setOfTags)
669
670            lastTagHash = None
671            lastTagHashQnt = None
672            treeLevel = 0
673            for tagHash in listOfTagHashes:
674                # currentTagHashQnt = self.tagsQnt[tagHash]
675                currentTagHashQnt = local_tags_qnt[tagHash]
676                if (lastTagHash is None) or (currentTagHashQnt == lastTagHashQnt):
677                    pass
678                else:
679                    treeLevel += 1
680                    lastTagHash = None
681                    lastTagHashQnt = None
682                # if treeLevel not in smartTree:
683                #     smartTree[treeLevel] = set()
684                # # tagsSetOnTheLevel = smartTree[treeLevel]
685                # # tagsSetOnTheLevel.add(tagHash)
686                # # smartTree[treeLevel] = tagsSetOnTheLevel
687                # smartTree[treeLevel].add(tagHash)
688                smartTree__filler.add(treeLevel, tagHash)
689                lastTagHash = tagHash
690                lastTagHashQnt = currentTagHashQnt
691        return dict(smartTree)
692
693    def build_smart_tree_2(self, startingBinTags, prePreparedSetOfAllInternalItemIDs=None, zeroSliceOnly=False):
694        startingBinTags = set(startingBinTags)
695
696        if self.get_root_tag() not in startingBinTags:
697            startingBinTags.add(self.get_root_tag())
698
699        startingTagHashes = set()
700        for tag in startingBinTags:
701            startingTagHashes.add(tag.__hash__())
702
703        if prePreparedSetOfAllInternalItemIDs is None:
704            setOfAllInternalItemIDs = self.get_itemIDs_from_tags(startingBinTags, treeType=PLAIN_PSEUDO_TREE_TYPE)
705        else:
706            setOfAllInternalItemIDs = prePreparedSetOfAllInternalItemIDs
707
708        smartTree = {0: set()}
709        smartTree__filler = AddToCompoundDict__Set(smartTree)
710        local_tags_qnt = dict()
711        local_tags_qnt__filler = AddToCompoundDict(
712            local_tags_qnt,
713            lambda: 0,
714            lambda container, key, value: (True, container[key] + 1)
715        )
716        for itemID in setOfAllInternalItemIDs:
717            commonTagTupleHash = self.itemWithTags[itemID]
718            tagSet = set(self.commonTagSets[commonTagTupleHash])
719            setOfTags = tagSet
720            setOfTags = setOfTags - startingTagHashes
721            for tag_hash in setOfTags:
722                # if tag_hash not in local_tags_qnt:
723                #     local_tags_qnt[tag_hash] = 0
724                # local_tags_qnt[tag_hash] += 1
725                local_tags_qnt__filler.add(tag_hash)
726
727        for itemID in setOfAllInternalItemIDs:
728            commonTagTupleHash = self.itemWithTags[itemID]
729            tagSet = set(self.commonTagSets[commonTagTupleHash])
730            setOfTags = tagSet
731            setOfTags = setOfTags - startingTagHashes
732            listOfTagHashes = None
733            if zeroSliceOnly:
734                listOfTagHashes = self.get_top_tag_hash_list_by_qnt(setOfTags, local_tags_qnt)
735            else:
736                listOfTagHashes = self.sort_tag_hash_list_by_qnt(setOfTags)
737
738            lastTagHash = None
739            lastTagHashQnt = None
740            treeLevel = 0
741            for tagHash in listOfTagHashes:
742                # currentTagHashQnt = self.tagsQnt[tagHash]
743                currentTagHashQnt = local_tags_qnt[tagHash]
744                if (lastTagHash is None) or (currentTagHashQnt == lastTagHashQnt):
745                    pass
746                else:
747                    treeLevel += 1
748                    lastTagHash = None
749                    lastTagHashQnt = None
750                # if treeLevel not in smartTree:
751                #     smartTree[treeLevel] = set()
752                # # tagsSetOnTheLevel = smartTree[treeLevel]
753                # # tagsSetOnTheLevel.add(tagHash)
754                # # smartTree[treeLevel] = tagsSetOnTheLevel
755                # smartTree[treeLevel].add(tagHash)
756                smartTree__filler.add(treeLevel, tagHash)
757                lastTagHash = tagHash
758                lastTagHashQnt = currentTagHashQnt
759        return dict(smartTree)
760
761    def get_all_from_tags(self, binTags, treeType=USUAL_TREE_TYPE):
762        binTags = set(binTags)
763        items = self.get_items_from_tags(binTags, treeType=treeType,
764                                         isAlsoNeedSetOfAllInternalItemIDsForThisSetOfTags=True)
765        if len(items[1]) > 0:
766            tags = self.get_tags_from_tags(binTags, treeType=treeType,
767                                           prePreparedSetOfAllInternalItemIDsForThisSetOfTags=items[1])
768            result = (tuple(tags), set(items[0]))
769            return result
770        else:
771            result = (tuple(), set())
772            return result
773
774    def get_tagsHashes_from_single_item(self, itemID, isWithoutRootHash=True):
775        if itemID in self.itemWithTags:
776            commonTagTupleHash = self.itemWithTags[itemID]
777            tagSet = set(self.commonTagSets[commonTagTupleHash])
778            if isWithoutRootHash:
779                return set(tagSet - {self.get_root_tag().__hash__()})
780            else:
781                return set(tagSet)
782        else:
783            return set()
784
785    def get_potential_itemIDs_from_item(self, binItem):
786        itemHash = binItem.__hash__()
787        if itemHash in self.itemIDsForItem:
788            return set(self.itemIDsForItem[itemHash])
789        else:
790            return set()
791
792    def is_smart_redirection_for_a_tag_path_reduction_needed(self, binTags):
793        binTags = set(binTags)
794
795        if self.get_root_tag() not in binTags:
796            binTags.add(self.get_root_tag())
797
798        tagHashSet = set()
799        for binTag in binTags:
800            tagHashSet.add(binTag.__hash__())
801
802        binTagsQnt = len(tagHashSet)
803        commonTagGroupHashSet = set()
804        # setOfLenOfTheCommonTagHashSetForChecking = set()
805        setOfTheTagsIntersection = None
806        for commonTagQnt in self.setOfTagGroupQnt:
807            if commonTagQnt > binTagsQnt:
808                setOfTheCommonTagGroupHashes = self.tagsQntPerCommonTagSet[commonTagQnt]
809                commonTagGroupHashSet.update(setOfTheCommonTagGroupHashes)
810        for commonTagGroupHash in commonTagGroupHashSet:
811            commonTagHashTuple = self.commonTagSets[commonTagGroupHash]
812            commonTagHashSet = set(commonTagHashTuple)
813            if len(tagHashSet & commonTagHashSet) == len(tagHashSet):
814                if tagHashSet != commonTagHashSet:
815                    if setOfTheTagsIntersection is None:
816                        setOfTheTagsIntersection = commonTagHashSet
817                    else:
818                        setOfTheTagsIntersection = setOfTheTagsIntersection & commonTagHashSet
819        #         if tagHashSet != commonTagHashSet:
820        #             setOfLenOfTheCommonTagHashSetForChecking.add(len(commonTagHashSet))
821        # minimalTagPath = min(setOfLenOfTheCommonTagHashSetForChecking)
822        # pathDiff = minimalTagPath - len(tagHashSet)
823        # if pathDiff > 0:
824        if setOfTheTagsIntersection is None:
825            return list()
826
827        setOfTheTagsForAReduction = setOfTheTagsIntersection - tagHashSet
828
829        sortedTagHashList = self.sort_tag_hash_list_by_qnt(setOfTheTagsForAReduction - {self.get_root_tag().__hash__()})
830        return list(sortedTagHashList)
831
832    def get_tags_for_a_smart_redirection(self, binTags):
833        result = self.is_smart_redirection_for_a_tag_path_reduction_needed(binTags)
834        return tuple(self.tag_hash_list_2_tag_list(result))

SMART_TREE_TYPE = 0

SMART_TREE_TYPE_WITH_INTERNAL_MENU = 1

FULL_TREE_TYPE = 2

PLAIN_PSEUDO_TREE_TYPE = 3

USUAL_TREE_TYPE = 3

class ToManyIdenticalItemsOnTheGivenTagPathError(builtins.Exception): View Source

64class ToManyIdenticalItemsOnTheGivenTagPathError(Exception):
65    pass

Common base class for all non-exit exceptions.

Inherited Members

builtins.Exception: Exception
builtins.BaseException: with_traceback; args

class UnknownTreeTypeError(builtins.Exception): View Source

68class UnknownTreeTypeError(Exception):
69    pass

Common base class for all non-exit exceptions.

Inherited Members

builtins.Exception: Exception
builtins.BaseException: with_traceback; args

class AddToCompoundDict__Set(cengal.data_containers.compound_dict_management.manager.versions.v_1.manager.AddToCompoundDict): View Source

72class AddToCompoundDict__Set(AddToCompoundDict):
73    def __init__(self, original_dict):
74        super().__init__(
75            original_dict,
76            lambda: set(),
77            lambda original_dict, key, value: original_dict[key].add(value)
78        )

AddToCompoundDict__Set(original_dict) View Source

73    def __init__(self, original_dict):
74        super().__init__(
75            original_dict,
76            lambda: set(),
77            lambda original_dict, key, value: original_dict[key].add(value)
78        )

:param original_dict: :param default_value: functor. list(); {1:set(), 2:[set(), set(), list()]}; etc. :param mediator: functor. original_dict[index].add(y), original_dict[index] += y, etc. Should return ValueExistence(True, ...) or None/nothing :return:

Inherited Members

cengal.data_containers.compound_dict_management.manager.versions.v_1.manager.AddToCompoundDict: original_dict; add