cengal.data_containers.dynamic_tag_tree.versions.v_1.tag_db_interface

Module Docstring Docstrings: http://www.python.org/dev/peps/pep-0257/

  1#!/usr/bin/env python
  2# coding=utf-8
  3
  4# Copyright © 2012-2024 ButenkoMS. All rights reserved. Contacts: <gtalk@butenkoms.space>
  5# 
  6# Licensed under the Apache License, Version 2.0 (the "License");
  7# you may not use this file except in compliance with the License.
  8# You may obtain a copy of the License at
  9# 
 10#     http://www.apache.org/licenses/LICENSE-2.0
 11# 
 12# Unless required by applicable law or agreed to in writing, software
 13# distributed under the License is distributed on an "AS IS" BASIS,
 14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 15# See the License for the specific language governing permissions and
 16# limitations under the License.
 17
 18
 19"""
 20Module Docstring
 21Docstrings: http://www.python.org/dev/peps/pep-0257/
 22"""
 23
 24
 25__author__ = "ButenkoMS <gtalk@butenkoms.space>"
 26__copyright__ = "Copyright © 2012-2024 ButenkoMS. All rights reserved. Contacts: <gtalk@butenkoms.space>"
 27__credits__ = ["ButenkoMS <gtalk@butenkoms.space>", ]
 28__license__ = "Apache License, Version 2.0"
 29__version__ = "4.4.1"
 30__maintainer__ = "ButenkoMS <gtalk@butenkoms.space>"
 31__email__ = "gtalk@butenkoms.space"
 32# __status__ = "Prototype"
 33__status__ = "Development"
 34# __status__ = "Production"
 35
 36
 37__all__ = ['tag_db_interface', 'SMART_TREE_TYPE', 'SMART_TREE_TYPE_WITH_INTERNAL_MENU', 'FULL_TREE_TYPE', 'PLAIN_PSEUDO_TREE_TYPE', 'USUAL_TREE_TYPE', 'ToManyIdenticalItemsOnTheGivenTagPathError', 'UnknownTreeTypeError', ]
 38__author__ = 'Mikhail Butenko <gtalk@mikhail-butenko.in.ua>'
 39
 40
 41# import TagDB
 42# from TagDB import USUAL_TREE_TYPE as USUAL_TREE_TYPE
 43# from TagDB import SMART_TREE_TYPE as SMART_TREE_TYPE
 44# from TagDB import FULL_TREE_TYPE as FULL_TREE_TYPE
 45# from TagDB import PLAIN_PSEUDO_TREE_TYPE as PLAIN_PSEUDO_TREE_TYPE
 46from .TagDB import *
 47import copy
 48from cengal.web_tools.request_cache import RequestCache
 49import time
 50from cengal.parallel_execution.coroutines.coro_standard_services.loop_yield import gly, CoroPriority
 51from cengal.parallel_execution.coroutines.coro_standard_services.read_write_locker import grwl, RWOperation
 52from cengal.parallel_execution.coroutines.coro_scheduler import EntityStatsMixin
 53from cengal.data_manipulation.tree_traversal import *
 54
 55
 56class tag_db_interface(EntityStatsMixin):
 57    def __init__(self, max_writers_in_progress: int, max_readers_in_progress: int, lock_entity_id: Hashable = None, default_priority: CoroPriority = CoroPriority.normal):
 58        self.default_priority: CoroPriority = default_priority
 59        self.__db = TagDB(default_priority)
 60        self.cache = RequestCache(300000, default_priority=default_priority)
 61        self.lock_entity_id: Hashable = lock_entity_id or type(self)
 62        self.max_writers_in_progress: int = max_writers_in_progress
 63        self.max_readers_in_progress: int = max_readers_in_progress
 64        self.last_fill_cache_result_time: float = None
 65        self.last_fill_cache_tags = None
 66        self.last_tested_fill_cache_result_time: float = None
 67        self.last_tested_fill_cache_tags = None
 68        self.fill_cache_iterations: int = 0
 69        self.fill_cache_iterations_per_second: float = None
 70        self.fill_cache_last_time_measurement: float = None
 71        self.fill_cache_last_time_measurement_iterations: int = 0
 72
 73    def get_entity_stats(self, stats_level: 'EntityStatsMixin.StatsLevel' = EntityStatsMixin.StatsLevel.debug) -> Tuple[str, Dict[str, Any]]:
 74        tag_db_name, tag_db_stats = self.__db.get_entity_stats(stats_level)
 75        return type(self).__name__, {
 76            'interface': {
 77                'cache size': len(self.cache._requestsAndData),
 78                'last fill cache result time': self.last_fill_cache_result_time,
 79                'last fill cache tags': self.last_fill_cache_tags,
 80                'last tested fill cache result time': self.last_tested_fill_cache_result_time,
 81                'last tested fill cache tags': self.last_tested_fill_cache_tags,
 82                'fill cache iterations': self.fill_cache_iterations,
 83                'fill cache iterations per second': self.fill_cache_iterations_per_second,
 84            },
 85            'db': {
 86                tag_db_name: tag_db_stats,
 87            }
 88        }
 89    
 90    def _glock(self):
 91        return grwl(self.lock_entity_id, self.max_writers_in_progress, self.max_readers_in_progress, False)
 92    
 93    def _wlock(self):
 94        return self._glock()(RWOperation.write)
 95    
 96    def _rlock(self):
 97        return self._glock()(RWOperation.read)
 98
 99    def get_db(self):
100        return self.__db
101    
102    def new_empty(self) -> 'tag_db_interface':
103        return tag_db_interface(
104            self.max_writers_in_progress,
105            self.max_readers_in_progress,
106            self.lock_entity_id,
107            self.default_priority
108            )
109
110    def add_item(self, binItem, binTags):
111        with self._wlock():
112            binItem = copy.deepcopy(binItem)
113            binTags = copy.deepcopy(binTags)
114
115            result = self.__db.add_item(binItem, binTags)
116
117            result = copy.deepcopy(result)
118            self.cache.clear()
119            return result
120
121    def remove_item(self, binTags, binItem):
122        with self._wlock():
123            binTags = copy.deepcopy(binTags)
124            binItem = copy.deepcopy(binItem)
125
126            result = self.__db.remove_item(binTags, binItem)
127
128            result = copy.deepcopy(result)
129            self.cache.clear()
130            return result
131
132    def get_items(self, binTags, treeType=USUAL_TREE_TYPE):
133        with self._rlock():
134            binTags = copy.deepcopy(binTags)
135            treeType = copy.deepcopy(treeType)
136
137            # request = ('get_items', tuple(self.__db.sort_tag_list_by_hash(binTags)), treeType)
138            request = ('get_items', frozenset(binTags), treeType)
139            result = self.cache.try_to_get_data_for_request(request)
140            if result is None:
141                
142                self.cache.put_new_request(request, result)
143
144            result = copy.deepcopy(result)
145            return result
146
147    def get_tags(self, binTags, treeType=USUAL_TREE_TYPE):
148        with self._rlock():
149            binTags = copy.deepcopy(binTags)
150            treeType = copy.deepcopy(treeType)
151
152            # request = ('get_tags', tuple(self.__db.sort_tag_list_by_hash(binTags)), treeType)
153            request = ('get_tags', frozenset(binTags), treeType)
154            result = self.cache.try_to_get_data_for_request(request)
155            if result is None:
156                result = self.__db.get_tags_from_tags(binTags, treeType=treeType)
157                self.cache.put_new_request(request, result)
158
159            result = copy.deepcopy(result)
160            return result
161
162    def get_all(self, binTags, treeType=USUAL_TREE_TYPE):
163        with self._rlock():
164            binTags = copy.deepcopy(binTags)
165            treeType = copy.deepcopy(treeType)
166
167            # request = ('get_all', tuple(self.__db.sort_tag_list_by_hash(binTags)), treeType)
168            request = ('get_all', frozenset(binTags), treeType)
169            result = self.cache.try_to_get_data_for_request(request)
170            if result is None:
171                result = self.__db.get_all_from_tags(binTags, treeType=treeType)
172                self.cache.put_new_request(request, result)
173
174            result = copy.deepcopy(result)
175            return result
176
177    def get_tags_for_a_smart_redirection(self, binTags):
178        with self._rlock():
179            binTags = copy.deepcopy(binTags)
180
181            # request = ('is_smart_redirection_for_a_tag_path_reduction_needed'
182            #            , tuple(self.__db.sort_tag_list_by_hash(binTags)))
183            request = ('is_smart_redirection_for_a_tag_path_reduction_needed', frozenset(binTags))
184            result = self.cache.try_to_get_data_for_request(request)
185            if result is None:
186                self.__db.get_tags_for_a_smart_redirection(binTags)
187                self.cache.put_new_request(request, result)
188
189            result = copy.deepcopy(result)
190            return result
191
192    def fill_cache(self, max_answer_time_in_seconds=0.5, path=list()):
193        ly = gly(self.default_priority)
194        
195        current_path = list(path)
196        startTime = time.perf_counter()
197        self.last_tested_fill_cache_tags = current_path
198
199        tags_and_items_inside_curent_path = self.get_all(current_path, SMART_TREE_TYPE)
200        internal_tags = tags_and_items_inside_curent_path[0]
201        self.get_items(current_path, PLAIN_PSEUDO_TREE_TYPE)
202        
203        self.fill_cache_iterations += 1
204
205        endTime = time.perf_counter()
206        resultTime = endTime - startTime
207        self.last_tested_fill_cache_result_time = resultTime
208        if resultTime >= max_answer_time_in_seconds:
209            self.last_fill_cache_tags = current_path
210            self.last_fill_cache_result_time = resultTime
211            for additional_tag in internal_tags:
212                ly()
213                new_path = current_path + [additional_tag]
214                self.fill_cache(max_answer_time_in_seconds, new_path)
215
216    def fill_web_cache(self, depth: int, max_answer_time_in_seconds=0.5, path=None):
217        ly = gly(self.default_priority)
218        ly()
219
220        path = path or set()
221        current_path = list(path)
222        startTime = time.perf_counter()
223
224        result, is_cached = self.web_browse_to(current_path, True)
225        optimized_path, tags, items = result
226        current_path = list(optimized_path)
227        self.last_tested_fill_cache_tags = current_path
228        
229        self.fill_cache_iterations += 1
230        depth -= 1
231        
232        if self.fill_cache_last_time_measurement_iterations == 0:
233            self.fill_cache_last_time_measurement_iterations = self.fill_cache_iterations
234        
235        if self.fill_cache_last_time_measurement is None:
236            self.fill_cache_last_time_measurement = time.perf_counter()
237        else:
238            dtime = time.perf_counter() - self.fill_cache_last_time_measurement
239            if dtime >= 1.0:
240                self.fill_cache_last_time_measurement = time.perf_counter()
241                iterations = self.fill_cache_iterations - self.fill_cache_last_time_measurement_iterations
242                self.fill_cache_iterations_per_second = iterations / dtime
243                self.fill_cache_last_time_measurement_iterations = self.fill_cache_iterations
244
245        endTime = time.perf_counter()
246        resultTime = endTime - startTime
247        self.last_tested_fill_cache_result_time = resultTime
248        # if (depth > 0) and (is_cached or (resultTime >= max_answer_time_in_seconds)):
249        # if (depth > 0) and (resultTime >= max_answer_time_in_seconds):
250        if depth > 0:
251            self.last_fill_cache_tags = current_path
252            self.last_fill_cache_result_time = resultTime
253            for additional_tag in tags:
254                new_path = current_path + [additional_tag]
255                self.fill_web_cache(depth, max_answer_time_in_seconds, new_path)
256    
257    def clear_cache(self):
258        self.cache.clear()
259        self.fill_cache_iterations = 0
260    
261    def web_browse_to(self, binTags, return_is_cached_result: bool = False, use_cache: bool = True):
262        with self._rlock():
263            binTags = copy.deepcopy(binTags)
264            additional_tags = self.get_tags_for_a_smart_redirection(binTags)
265            if additional_tags is not None:
266                optimized_path = set(binTags) | set(additional_tags)
267            else:
268                optimized_path = binTags
269
270            request = ('web_browse_to', frozenset(optimized_path))
271            result = None
272            if use_cache:
273                result = self.cache.try_to_get_data_for_request(request)
274                is_cached_result = True
275            
276            if result is None:
277                is_cached_result = False
278                items, items_ids = self.__db.get_items_from_tags(optimized_path, treeType=PLAIN_PSEUDO_TREE_TYPE, isAlsoNeedSetOfAllInternalItemIDsForThisSetOfTags=True)
279                tags = self.__db.get_tags_from_tags(optimized_path, treeType=SMART_TREE_TYPE, prePreparedSetOfAllInternalItemIDsForThisSetOfTags=items_ids)
280                result = (optimized_path, tags, items)
281                if use_cache:
282                    self.cache.put_new_request(request, result)
283
284            result = copy.deepcopy(result)
285            if return_is_cached_result:
286                return result, is_cached_result
287            else:
288                return result
289    
290    def fs_browse_to(self, binTags):
291        with self._rlock():
292            binTags = copy.deepcopy(binTags)
293
294            request = ('web_browse_to', frozenset(binTags))
295            result = self.cache.try_to_get_data_for_request(request)
296            if result is None:
297                optimized_path = set(binTags) | set(self.__db.get_tags_for_a_smart_redirection(binTags))
298                items = self.__db.get_items_from_tags(optimized_path, treeType=SMART_TREE_TYPE)
299                tags = self.__db.get_tags_from_tags(optimized_path, treeType=SMART_TREE_TYPE)
300                result = (optimized_path, tags, items)
301                self.cache.put_new_request(request, result)
302
303            result = copy.deepcopy(result)
304            return result
class tag_db_interface(cengal.parallel_execution.coroutines.coro_scheduler.versions.v_0.coro_scheduler.EntityStatsMixin): 
 57class tag_db_interface(EntityStatsMixin):
 58    def __init__(self, max_writers_in_progress: int, max_readers_in_progress: int, lock_entity_id: Hashable = None, default_priority: CoroPriority = CoroPriority.normal):
 59        self.default_priority: CoroPriority = default_priority
 60        self.__db = TagDB(default_priority)
 61        self.cache = RequestCache(300000, default_priority=default_priority)
 62        self.lock_entity_id: Hashable = lock_entity_id or type(self)
 63        self.max_writers_in_progress: int = max_writers_in_progress
 64        self.max_readers_in_progress: int = max_readers_in_progress
 65        self.last_fill_cache_result_time: float = None
 66        self.last_fill_cache_tags = None
 67        self.last_tested_fill_cache_result_time: float = None
 68        self.last_tested_fill_cache_tags = None
 69        self.fill_cache_iterations: int = 0
 70        self.fill_cache_iterations_per_second: float = None
 71        self.fill_cache_last_time_measurement: float = None
 72        self.fill_cache_last_time_measurement_iterations: int = 0
 73
 74    def get_entity_stats(self, stats_level: 'EntityStatsMixin.StatsLevel' = EntityStatsMixin.StatsLevel.debug) -> Tuple[str, Dict[str, Any]]:
 75        tag_db_name, tag_db_stats = self.__db.get_entity_stats(stats_level)
 76        return type(self).__name__, {
 77            'interface': {
 78                'cache size': len(self.cache._requestsAndData),
 79                'last fill cache result time': self.last_fill_cache_result_time,
 80                'last fill cache tags': self.last_fill_cache_tags,
 81                'last tested fill cache result time': self.last_tested_fill_cache_result_time,
 82                'last tested fill cache tags': self.last_tested_fill_cache_tags,
 83                'fill cache iterations': self.fill_cache_iterations,
 84                'fill cache iterations per second': self.fill_cache_iterations_per_second,
 85            },
 86            'db': {
 87                tag_db_name: tag_db_stats,
 88            }
 89        }
 90    
 91    def _glock(self):
 92        return grwl(self.lock_entity_id, self.max_writers_in_progress, self.max_readers_in_progress, False)
 93    
 94    def _wlock(self):
 95        return self._glock()(RWOperation.write)
 96    
 97    def _rlock(self):
 98        return self._glock()(RWOperation.read)
 99
100    def get_db(self):
101        return self.__db
102    
103    def new_empty(self) -> 'tag_db_interface':
104        return tag_db_interface(
105            self.max_writers_in_progress,
106            self.max_readers_in_progress,
107            self.lock_entity_id,
108            self.default_priority
109            )
110
111    def add_item(self, binItem, binTags):
112        with self._wlock():
113            binItem = copy.deepcopy(binItem)
114            binTags = copy.deepcopy(binTags)
115
116            result = self.__db.add_item(binItem, binTags)
117
118            result = copy.deepcopy(result)
119            self.cache.clear()
120            return result
121
122    def remove_item(self, binTags, binItem):
123        with self._wlock():
124            binTags = copy.deepcopy(binTags)
125            binItem = copy.deepcopy(binItem)
126
127            result = self.__db.remove_item(binTags, binItem)
128
129            result = copy.deepcopy(result)
130            self.cache.clear()
131            return result
132
133    def get_items(self, binTags, treeType=USUAL_TREE_TYPE):
134        with self._rlock():
135            binTags = copy.deepcopy(binTags)
136            treeType = copy.deepcopy(treeType)
137
138            # request = ('get_items', tuple(self.__db.sort_tag_list_by_hash(binTags)), treeType)
139            request = ('get_items', frozenset(binTags), treeType)
140            result = self.cache.try_to_get_data_for_request(request)
141            if result is None:
142                
143                self.cache.put_new_request(request, result)
144
145            result = copy.deepcopy(result)
146            return result
147
148    def get_tags(self, binTags, treeType=USUAL_TREE_TYPE):
149        with self._rlock():
150            binTags = copy.deepcopy(binTags)
151            treeType = copy.deepcopy(treeType)
152
153            # request = ('get_tags', tuple(self.__db.sort_tag_list_by_hash(binTags)), treeType)
154            request = ('get_tags', frozenset(binTags), treeType)
155            result = self.cache.try_to_get_data_for_request(request)
156            if result is None:
157                result = self.__db.get_tags_from_tags(binTags, treeType=treeType)
158                self.cache.put_new_request(request, result)
159
160            result = copy.deepcopy(result)
161            return result
162
163    def get_all(self, binTags, treeType=USUAL_TREE_TYPE):
164        with self._rlock():
165            binTags = copy.deepcopy(binTags)
166            treeType = copy.deepcopy(treeType)
167
168            # request = ('get_all', tuple(self.__db.sort_tag_list_by_hash(binTags)), treeType)
169            request = ('get_all', frozenset(binTags), treeType)
170            result = self.cache.try_to_get_data_for_request(request)
171            if result is None:
172                result = self.__db.get_all_from_tags(binTags, treeType=treeType)
173                self.cache.put_new_request(request, result)
174
175            result = copy.deepcopy(result)
176            return result
177
178    def get_tags_for_a_smart_redirection(self, binTags):
179        with self._rlock():
180            binTags = copy.deepcopy(binTags)
181
182            # request = ('is_smart_redirection_for_a_tag_path_reduction_needed'
183            #            , tuple(self.__db.sort_tag_list_by_hash(binTags)))
184            request = ('is_smart_redirection_for_a_tag_path_reduction_needed', frozenset(binTags))
185            result = self.cache.try_to_get_data_for_request(request)
186            if result is None:
187                self.__db.get_tags_for_a_smart_redirection(binTags)
188                self.cache.put_new_request(request, result)
189
190            result = copy.deepcopy(result)
191            return result
192
193    def fill_cache(self, max_answer_time_in_seconds=0.5, path=list()):
194        ly = gly(self.default_priority)
195        
196        current_path = list(path)
197        startTime = time.perf_counter()
198        self.last_tested_fill_cache_tags = current_path
199
200        tags_and_items_inside_curent_path = self.get_all(current_path, SMART_TREE_TYPE)
201        internal_tags = tags_and_items_inside_curent_path[0]
202        self.get_items(current_path, PLAIN_PSEUDO_TREE_TYPE)
203        
204        self.fill_cache_iterations += 1
205
206        endTime = time.perf_counter()
207        resultTime = endTime - startTime
208        self.last_tested_fill_cache_result_time = resultTime
209        if resultTime >= max_answer_time_in_seconds:
210            self.last_fill_cache_tags = current_path
211            self.last_fill_cache_result_time = resultTime
212            for additional_tag in internal_tags:
213                ly()
214                new_path = current_path + [additional_tag]
215                self.fill_cache(max_answer_time_in_seconds, new_path)
216
217    def fill_web_cache(self, depth: int, max_answer_time_in_seconds=0.5, path=None):
218        ly = gly(self.default_priority)
219        ly()
220
221        path = path or set()
222        current_path = list(path)
223        startTime = time.perf_counter()
224
225        result, is_cached = self.web_browse_to(current_path, True)
226        optimized_path, tags, items = result
227        current_path = list(optimized_path)
228        self.last_tested_fill_cache_tags = current_path
229        
230        self.fill_cache_iterations += 1
231        depth -= 1
232        
233        if self.fill_cache_last_time_measurement_iterations == 0:
234            self.fill_cache_last_time_measurement_iterations = self.fill_cache_iterations
235        
236        if self.fill_cache_last_time_measurement is None:
237            self.fill_cache_last_time_measurement = time.perf_counter()
238        else:
239            dtime = time.perf_counter() - self.fill_cache_last_time_measurement
240            if dtime >= 1.0:
241                self.fill_cache_last_time_measurement = time.perf_counter()
242                iterations = self.fill_cache_iterations - self.fill_cache_last_time_measurement_iterations
243                self.fill_cache_iterations_per_second = iterations / dtime
244                self.fill_cache_last_time_measurement_iterations = self.fill_cache_iterations
245
246        endTime = time.perf_counter()
247        resultTime = endTime - startTime
248        self.last_tested_fill_cache_result_time = resultTime
249        # if (depth > 0) and (is_cached or (resultTime >= max_answer_time_in_seconds)):
250        # if (depth > 0) and (resultTime >= max_answer_time_in_seconds):
251        if depth > 0:
252            self.last_fill_cache_tags = current_path
253            self.last_fill_cache_result_time = resultTime
254            for additional_tag in tags:
255                new_path = current_path + [additional_tag]
256                self.fill_web_cache(depth, max_answer_time_in_seconds, new_path)
257    
258    def clear_cache(self):
259        self.cache.clear()
260        self.fill_cache_iterations = 0
261    
262    def web_browse_to(self, binTags, return_is_cached_result: bool = False, use_cache: bool = True):
263        with self._rlock():
264            binTags = copy.deepcopy(binTags)
265            additional_tags = self.get_tags_for_a_smart_redirection(binTags)
266            if additional_tags is not None:
267                optimized_path = set(binTags) | set(additional_tags)
268            else:
269                optimized_path = binTags
270
271            request = ('web_browse_to', frozenset(optimized_path))
272            result = None
273            if use_cache:
274                result = self.cache.try_to_get_data_for_request(request)
275                is_cached_result = True
276            
277            if result is None:
278                is_cached_result = False
279                items, items_ids = self.__db.get_items_from_tags(optimized_path, treeType=PLAIN_PSEUDO_TREE_TYPE, isAlsoNeedSetOfAllInternalItemIDsForThisSetOfTags=True)
280                tags = self.__db.get_tags_from_tags(optimized_path, treeType=SMART_TREE_TYPE, prePreparedSetOfAllInternalItemIDsForThisSetOfTags=items_ids)
281                result = (optimized_path, tags, items)
282                if use_cache:
283                    self.cache.put_new_request(request, result)
284
285            result = copy.deepcopy(result)
286            if return_is_cached_result:
287                return result, is_cached_result
288            else:
289                return result
290    
291    def fs_browse_to(self, binTags):
292        with self._rlock():
293            binTags = copy.deepcopy(binTags)
294
295            request = ('web_browse_to', frozenset(binTags))
296            result = self.cache.try_to_get_data_for_request(request)
297            if result is None:
298                optimized_path = set(binTags) | set(self.__db.get_tags_for_a_smart_redirection(binTags))
299                items = self.__db.get_items_from_tags(optimized_path, treeType=SMART_TREE_TYPE)
300                tags = self.__db.get_tags_from_tags(optimized_path, treeType=SMART_TREE_TYPE)
301                result = (optimized_path, tags, items)
302                self.cache.put_new_request(request, result)
303
304            result = copy.deepcopy(result)
305            return result
tag_db_interface( max_writers_in_progress: int, max_readers_in_progress: int, lock_entity_id: Hashable = None, default_priority: cengal.parallel_execution.coroutines.coro_standard_services.loop_yield.versions.v_0.loop_yield.CoroPriority = <CoroPriority.normal: 1>) 
58    def __init__(self, max_writers_in_progress: int, max_readers_in_progress: int, lock_entity_id: Hashable = None, default_priority: CoroPriority = CoroPriority.normal):
59        self.default_priority: CoroPriority = default_priority
60        self.__db = TagDB(default_priority)
61        self.cache = RequestCache(300000, default_priority=default_priority)
62        self.lock_entity_id: Hashable = lock_entity_id or type(self)
63        self.max_writers_in_progress: int = max_writers_in_progress
64        self.max_readers_in_progress: int = max_readers_in_progress
65        self.last_fill_cache_result_time: float = None
66        self.last_fill_cache_tags = None
67        self.last_tested_fill_cache_result_time: float = None
68        self.last_tested_fill_cache_tags = None
69        self.fill_cache_iterations: int = 0
70        self.fill_cache_iterations_per_second: float = None
71        self.fill_cache_last_time_measurement: float = None
72        self.fill_cache_last_time_measurement_iterations: int = 0
default_priority: cengal.parallel_execution.coroutines.coro_standard_services.loop_yield.versions.v_0.loop_yield.CoroPriority
cache
lock_entity_id: Hashable
max_writers_in_progress: int
max_readers_in_progress: int
last_fill_cache_result_time: float
last_fill_cache_tags
last_tested_fill_cache_result_time: float
last_tested_fill_cache_tags
fill_cache_iterations: int
fill_cache_iterations_per_second: float
fill_cache_last_time_measurement: float
fill_cache_last_time_measurement_iterations: int
def get_entity_stats( self, stats_level: cengal.parallel_execution.coroutines.coro_scheduler.versions.v_0.coro_scheduler.EntityStatsMixin.StatsLevel = <StatsLevel.debug: 1>) -> Tuple[str, Dict[str, Any]]: 
74    def get_entity_stats(self, stats_level: 'EntityStatsMixin.StatsLevel' = EntityStatsMixin.StatsLevel.debug) -> Tuple[str, Dict[str, Any]]:
75        tag_db_name, tag_db_stats = self.__db.get_entity_stats(stats_level)
76        return type(self).__name__, {
77            'interface': {
78                'cache size': len(self.cache._requestsAndData),
79                'last fill cache result time': self.last_fill_cache_result_time,
80                'last fill cache tags': self.last_fill_cache_tags,
81                'last tested fill cache result time': self.last_tested_fill_cache_result_time,
82                'last tested fill cache tags': self.last_tested_fill_cache_tags,
83                'fill cache iterations': self.fill_cache_iterations,
84                'fill cache iterations per second': self.fill_cache_iterations_per_second,
85            },
86            'db': {
87                tag_db_name: tag_db_stats,
88            }
89        }
def get_db(self): 
100    def get_db(self):
101        return self.__db
def new_empty( self) -> tag_db_interface: 
103    def new_empty(self) -> 'tag_db_interface':
104        return tag_db_interface(
105            self.max_writers_in_progress,
106            self.max_readers_in_progress,
107            self.lock_entity_id,
108            self.default_priority
109            )
def add_item(self, binItem, binTags): 
111    def add_item(self, binItem, binTags):
112        with self._wlock():
113            binItem = copy.deepcopy(binItem)
114            binTags = copy.deepcopy(binTags)
115
116            result = self.__db.add_item(binItem, binTags)
117
118            result = copy.deepcopy(result)
119            self.cache.clear()
120            return result
def remove_item(self, binTags, binItem): 
122    def remove_item(self, binTags, binItem):
123        with self._wlock():
124            binTags = copy.deepcopy(binTags)
125            binItem = copy.deepcopy(binItem)
126
127            result = self.__db.remove_item(binTags, binItem)
128
129            result = copy.deepcopy(result)
130            self.cache.clear()
131            return result
def get_items(self, binTags, treeType=3): 
133    def get_items(self, binTags, treeType=USUAL_TREE_TYPE):
134        with self._rlock():
135            binTags = copy.deepcopy(binTags)
136            treeType = copy.deepcopy(treeType)
137
138            # request = ('get_items', tuple(self.__db.sort_tag_list_by_hash(binTags)), treeType)
139            request = ('get_items', frozenset(binTags), treeType)
140            result = self.cache.try_to_get_data_for_request(request)
141            if result is None:
142                
143                self.cache.put_new_request(request, result)
144
145            result = copy.deepcopy(result)
146            return result
def get_tags(self, binTags, treeType=3): 
148    def get_tags(self, binTags, treeType=USUAL_TREE_TYPE):
149        with self._rlock():
150            binTags = copy.deepcopy(binTags)
151            treeType = copy.deepcopy(treeType)
152
153            # request = ('get_tags', tuple(self.__db.sort_tag_list_by_hash(binTags)), treeType)
154            request = ('get_tags', frozenset(binTags), treeType)
155            result = self.cache.try_to_get_data_for_request(request)
156            if result is None:
157                result = self.__db.get_tags_from_tags(binTags, treeType=treeType)
158                self.cache.put_new_request(request, result)
159
160            result = copy.deepcopy(result)
161            return result
def get_all(self, binTags, treeType=3): 
163    def get_all(self, binTags, treeType=USUAL_TREE_TYPE):
164        with self._rlock():
165            binTags = copy.deepcopy(binTags)
166            treeType = copy.deepcopy(treeType)
167
168            # request = ('get_all', tuple(self.__db.sort_tag_list_by_hash(binTags)), treeType)
169            request = ('get_all', frozenset(binTags), treeType)
170            result = self.cache.try_to_get_data_for_request(request)
171            if result is None:
172                result = self.__db.get_all_from_tags(binTags, treeType=treeType)
173                self.cache.put_new_request(request, result)
174
175            result = copy.deepcopy(result)
176            return result
def get_tags_for_a_smart_redirection(self, binTags): 
178    def get_tags_for_a_smart_redirection(self, binTags):
179        with self._rlock():
180            binTags = copy.deepcopy(binTags)
181
182            # request = ('is_smart_redirection_for_a_tag_path_reduction_needed'
183            #            , tuple(self.__db.sort_tag_list_by_hash(binTags)))
184            request = ('is_smart_redirection_for_a_tag_path_reduction_needed', frozenset(binTags))
185            result = self.cache.try_to_get_data_for_request(request)
186            if result is None:
187                self.__db.get_tags_for_a_smart_redirection(binTags)
188                self.cache.put_new_request(request, result)
189
190            result = copy.deepcopy(result)
191            return result
def fill_cache(self, max_answer_time_in_seconds=0.5, path=[]): 
193    def fill_cache(self, max_answer_time_in_seconds=0.5, path=list()):
194        ly = gly(self.default_priority)
195        
196        current_path = list(path)
197        startTime = time.perf_counter()
198        self.last_tested_fill_cache_tags = current_path
199
200        tags_and_items_inside_curent_path = self.get_all(current_path, SMART_TREE_TYPE)
201        internal_tags = tags_and_items_inside_curent_path[0]
202        self.get_items(current_path, PLAIN_PSEUDO_TREE_TYPE)
203        
204        self.fill_cache_iterations += 1
205
206        endTime = time.perf_counter()
207        resultTime = endTime - startTime
208        self.last_tested_fill_cache_result_time = resultTime
209        if resultTime >= max_answer_time_in_seconds:
210            self.last_fill_cache_tags = current_path
211            self.last_fill_cache_result_time = resultTime
212            for additional_tag in internal_tags:
213                ly()
214                new_path = current_path + [additional_tag]
215                self.fill_cache(max_answer_time_in_seconds, new_path)
def fill_web_cache(self, depth: int, max_answer_time_in_seconds=0.5, path=None): 
217    def fill_web_cache(self, depth: int, max_answer_time_in_seconds=0.5, path=None):
218        ly = gly(self.default_priority)
219        ly()
220
221        path = path or set()
222        current_path = list(path)
223        startTime = time.perf_counter()
224
225        result, is_cached = self.web_browse_to(current_path, True)
226        optimized_path, tags, items = result
227        current_path = list(optimized_path)
228        self.last_tested_fill_cache_tags = current_path
229        
230        self.fill_cache_iterations += 1
231        depth -= 1
232        
233        if self.fill_cache_last_time_measurement_iterations == 0:
234            self.fill_cache_last_time_measurement_iterations = self.fill_cache_iterations
235        
236        if self.fill_cache_last_time_measurement is None:
237            self.fill_cache_last_time_measurement = time.perf_counter()
238        else:
239            dtime = time.perf_counter() - self.fill_cache_last_time_measurement
240            if dtime >= 1.0:
241                self.fill_cache_last_time_measurement = time.perf_counter()
242                iterations = self.fill_cache_iterations - self.fill_cache_last_time_measurement_iterations
243                self.fill_cache_iterations_per_second = iterations / dtime
244                self.fill_cache_last_time_measurement_iterations = self.fill_cache_iterations
245
246        endTime = time.perf_counter()
247        resultTime = endTime - startTime
248        self.last_tested_fill_cache_result_time = resultTime
249        # if (depth > 0) and (is_cached or (resultTime >= max_answer_time_in_seconds)):
250        # if (depth > 0) and (resultTime >= max_answer_time_in_seconds):
251        if depth > 0:
252            self.last_fill_cache_tags = current_path
253            self.last_fill_cache_result_time = resultTime
254            for additional_tag in tags:
255                new_path = current_path + [additional_tag]
256                self.fill_web_cache(depth, max_answer_time_in_seconds, new_path)
def clear_cache(self): 
258    def clear_cache(self):
259        self.cache.clear()
260        self.fill_cache_iterations = 0
def web_browse_to( self, binTags, return_is_cached_result: bool = False, use_cache: bool = True): 
262    def web_browse_to(self, binTags, return_is_cached_result: bool = False, use_cache: bool = True):
263        with self._rlock():
264            binTags = copy.deepcopy(binTags)
265            additional_tags = self.get_tags_for_a_smart_redirection(binTags)
266            if additional_tags is not None:
267                optimized_path = set(binTags) | set(additional_tags)
268            else:
269                optimized_path = binTags
270
271            request = ('web_browse_to', frozenset(optimized_path))
272            result = None
273            if use_cache:
274                result = self.cache.try_to_get_data_for_request(request)
275                is_cached_result = True
276            
277            if result is None:
278                is_cached_result = False
279                items, items_ids = self.__db.get_items_from_tags(optimized_path, treeType=PLAIN_PSEUDO_TREE_TYPE, isAlsoNeedSetOfAllInternalItemIDsForThisSetOfTags=True)
280                tags = self.__db.get_tags_from_tags(optimized_path, treeType=SMART_TREE_TYPE, prePreparedSetOfAllInternalItemIDsForThisSetOfTags=items_ids)
281                result = (optimized_path, tags, items)
282                if use_cache:
283                    self.cache.put_new_request(request, result)
284
285            result = copy.deepcopy(result)
286            if return_is_cached_result:
287                return result, is_cached_result
288            else:
289                return result
def fs_browse_to(self, binTags): 
291    def fs_browse_to(self, binTags):
292        with self._rlock():
293            binTags = copy.deepcopy(binTags)
294
295            request = ('web_browse_to', frozenset(binTags))
296            result = self.cache.try_to_get_data_for_request(request)
297            if result is None:
298                optimized_path = set(binTags) | set(self.__db.get_tags_for_a_smart_redirection(binTags))
299                items = self.__db.get_items_from_tags(optimized_path, treeType=SMART_TREE_TYPE)
300                tags = self.__db.get_tags_from_tags(optimized_path, treeType=SMART_TREE_TYPE)
301                result = (optimized_path, tags, items)
302                self.cache.put_new_request(request, result)
303
304            result = copy.deepcopy(result)
305            return result
Inherited Members
cengal.parallel_execution.coroutines.coro_scheduler.versions.v_0.coro_scheduler.EntityStatsMixin
StatsLevel
SMART_TREE_TYPE = 0
SMART_TREE_TYPE_WITH_INTERNAL_MENU = 1
FULL_TREE_TYPE = 2
PLAIN_PSEUDO_TREE_TYPE = 3
USUAL_TREE_TYPE = 3
class ToManyIdenticalItemsOnTheGivenTagPathError(builtins.Exception): 
67class ToManyIdenticalItemsOnTheGivenTagPathError(Exception):
68    pass

Common base class for all non-exit exceptions.

Inherited Members
builtins.Exception
Exception
builtins.BaseException
with_traceback
args
class UnknownTreeTypeError(builtins.Exception): 
71class UnknownTreeTypeError(Exception):
72    pass

Common base class for all non-exit exceptions.

Inherited Members
builtins.Exception
Exception
builtins.BaseException
with_traceback
args