cengal.web_tools.request_cache.versions.v_0.request_cache

View Source

  1#!/usr/bin/env python
  2# coding=utf-8
  3
  4# Copyright © 2012-2024 ButenkoMS. All rights reserved. Contacts: <gtalk@butenkoms.space>
  5# 
  6# Licensed under the Apache License, Version 2.0 (the "License");
  7# you may not use this file except in compliance with the License.
  8# You may obtain a copy of the License at
  9# 
 10#     http://www.apache.org/licenses/LICENSE-2.0
 11# 
 12# Unless required by applicable law or agreed to in writing, software
 13# distributed under the License is distributed on an "AS IS" BASIS,
 14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 15# See the License for the specific language governing permissions and
 16# limitations under the License.
 17
 18import time
 19import json
 20from cengal.parallel_execution.coroutines.coro_standard_services.loop_yield import gly, CoroPriority
 21
 22"""
 23Module Docstring
 24Docstrings: http://www.python.org/dev/peps/pep-0257/
 25"""
 26
 27__author__ = "ButenkoMS <gtalk@butenkoms.space>"
 28__copyright__ = "Copyright © 2012-2024 ButenkoMS. All rights reserved. Contacts: <gtalk@butenkoms.space>"
 29__credits__ = ["ButenkoMS <gtalk@butenkoms.space>", ]
 30__license__ = "Apache License, Version 2.0"
 31__version__ = "4.4.1"
 32__maintainer__ = "ButenkoMS <gtalk@butenkoms.space>"
 33__email__ = "gtalk@butenkoms.space"
 34# __status__ = "Prototype"
 35__status__ = "Development"
 36# __status__ = "Production"
 37
 38
 39class RequestCache:
 40    # TODO: add GC which will clean lists every X seconds (really - every Y hours)
 41
 42    def __init__(self, itemsQntLimit, timeLimitInSeconds=None, clock_function=None, default_priority: CoroPriority = CoroPriority.normal):
 43        # timeLimitInSeconds:
 44        # - number - limit in seconds (remember that the accuracy of the server clock is
 45        #   approximately 1 second);
 46        # - 0 (zero) - cache is not used (try_to_get_data_for_request() will return 'None'
 47        #   on every request);
 48        # - None - cache is trying to be permanent (time limit is not used at all)
 49        super().__init__()
 50        self.default_priority: CoroPriority = default_priority
 51        self._clock_function = clock_function or time.perf_counter
 52        self._itemsQntLimit = itemsQntLimit
 53        self._timeLimitInSeconds = timeLimitInSeconds
 54        self._requestsAndData = {}   # key - request; data - (data, Server Time of last change). Server Time of last
 55            #  change must be less or equal to self._itemsQntLimit
 56        self._requestsHistory = []   # [request0, request1, ..., requestN]
 57        self.isWasChanged = False
 58
 59    def put_new_request(self, request, data):
 60        self._move_request_to_the_end_of_history(request)
 61        if request in self._requestsAndData:
 62            dataAndTime = self._requestsAndData[request]
 63            is_was_changed = False
 64            if isinstance(data, type(dataAndTime[0])):
 65                if data != dataAndTime[0]:
 66                    is_was_changed = True
 67            else:
 68                is_was_changed = True
 69
 70            if is_was_changed:
 71                self._requestsAndData[request] = (data, self._clock_function())
 72                self.isWasChanged = True
 73        else:
 74            if len(self._requestsAndData) >= self._itemsQntLimit:
 75                forDeleting = self._requestsHistory[0]
 76                if forDeleting in self._requestsAndData:
 77                    will_be_deleted = self._requestsAndData[forDeleting]
 78                    del self._requestsAndData[forDeleting]
 79                    return will_be_deleted
 80                del self._requestsHistory[0]
 81            self._requestsAndData[request] = (data, self._clock_function())
 82            self.isWasChanged = True
 83
 84    def put_new_request_or_renew_it(self, request, data):
 85        self._move_request_to_the_end_of_history(request)
 86        if request in self._requestsAndData:
 87            dataAndTime = self._requestsAndData[request]
 88            self._requestsAndData[request] = (data, self._clock_function())
 89            self.isWasChanged = True
 90        else:
 91            if len(self._requestsAndData) >= self._itemsQntLimit:
 92                forDeleting = self._requestsHistory[0]
 93                if forDeleting in self._requestsAndData:
 94                    will_be_deleted = self._requestsAndData[forDeleting]
 95                    del self._requestsAndData[forDeleting]
 96                    return will_be_deleted
 97                del self._requestsHistory[0]
 98            self._requestsAndData[request] = (data, self._clock_function())
 99            self.isWasChanged = True
100
101    def try_to_get_raw_data_for_request(self, request):
102        if request in self._requestsAndData:
103            dataAndTime = self._requestsAndData[request]
104            return dataAndTime[0]
105        else:
106            return None
107
108    def try_to_get_raw_data_with_time_for_request(self, request):
109        if request in self._requestsAndData:
110            dataAndTime = self._requestsAndData[request]
111            return dataAndTime
112        else:
113            return None
114
115    def try_to_get_data_for_request(self, request):
116        if request in self._requestsAndData:
117            dataAndTime = self._requestsAndData[request]
118            lastChangingTime = dataAndTime[1]
119            tLimit = self._timeLimitInSeconds
120            if (tLimit is not None) and ((self._clock_function() - lastChangingTime) >= tLimit):
121                del self._requestsAndData[request]
122                if request in self._requestsHistory:
123                    self._requestsHistory.remove(request)
124                return None
125            else:
126                self._move_request_to_the_end_of_history(request)
127                return dataAndTime[0]
128        else:
129            return None
130
131    def try_to_get_data_for_request_and_renew_it(self, request):
132        if request in self._requestsAndData:
133            dataAndTime = self._requestsAndData[request]
134            lastChangingTime = dataAndTime[1]
135            tLimit = self._timeLimitInSeconds
136            if (tLimit is not None) and ((self._clock_function() - lastChangingTime) >= tLimit):
137                del self._requestsAndData[request]
138                if request in self._requestsHistory:
139                    self._requestsHistory.remove(request)
140                return None
141            else:
142                self._move_request_to_the_end_of_history(request)
143                self._requestsAndData[request] = (dataAndTime[0], self._clock_function())
144                self.isWasChanged = True
145                return dataAndTime[0]
146        else:
147            return None
148
149    def try_to_remove_request(self, request):
150        if request in self._requestsAndData:
151            del self._requestsAndData[request]
152        if request in self._requestsHistory:
153            self._requestsHistory.remove(request)
154
155    def _move_request_to_the_end_of_history(self, request):
156        if request in self._requestsAndData:
157            if request in self._requestsHistory:
158                self._requestsHistory.remove(request)
159        self._requestsHistory.append(request)
160
161    def update(self, anotherRequestCache):
162        if type(anotherRequestCache) == RequestCache:
163            self._itemsQntLimit += anotherRequestCache._itemsQntLimit
164            self._requestsAndData.update(anotherRequestCache._requestsAndData)
165            self.isWasChanged = True
166            # Do not do this:
167            # self._requestsHistory += anotherRequestCache._requestsHistory
168
169    def clear(self):
170        self._requestsAndData = type(self._requestsAndData)()
171        # self._requestsHistory.clear()  # Python 3.3+ only so can't be used under PyPy yet.
172        self._requestsHistory = type(self._requestsHistory)()
173        self.isWasChanged = True
174
175    def get_state(self):
176        reqAndDat = []
177        for item in self._requestsAndData.items():
178            reqAndDat.append(item)
179        data = (self._itemsQntLimit
180                , self._timeLimitInSeconds
181                , reqAndDat
182                , self._requestsHistory)
183        return json.dumps(data)
184
185    def set_state(self, state):
186        data = json.loads(state)
187        self._itemsQntLimit = data[0]
188        self._timeLimitInSeconds = data[1]
189        for item in data[2]:
190            self._requestsAndData[item[0]] = item[1]
191        self._requestsHistory = data[3]
192        self.isWasChanged = True

class RequestCache: View Source

 40class RequestCache:
 41    # TODO: add GC which will clean lists every X seconds (really - every Y hours)
 42
 43    def __init__(self, itemsQntLimit, timeLimitInSeconds=None, clock_function=None, default_priority: CoroPriority = CoroPriority.normal):
 44        # timeLimitInSeconds:
 45        # - number - limit in seconds (remember that the accuracy of the server clock is
 46        #   approximately 1 second);
 47        # - 0 (zero) - cache is not used (try_to_get_data_for_request() will return 'None'
 48        #   on every request);
 49        # - None - cache is trying to be permanent (time limit is not used at all)
 50        super().__init__()
 51        self.default_priority: CoroPriority = default_priority
 52        self._clock_function = clock_function or time.perf_counter
 53        self._itemsQntLimit = itemsQntLimit
 54        self._timeLimitInSeconds = timeLimitInSeconds
 55        self._requestsAndData = {}   # key - request; data - (data, Server Time of last change). Server Time of last
 56            #  change must be less or equal to self._itemsQntLimit
 57        self._requestsHistory = []   # [request0, request1, ..., requestN]
 58        self.isWasChanged = False
 59
 60    def put_new_request(self, request, data):
 61        self._move_request_to_the_end_of_history(request)
 62        if request in self._requestsAndData:
 63            dataAndTime = self._requestsAndData[request]
 64            is_was_changed = False
 65            if isinstance(data, type(dataAndTime[0])):
 66                if data != dataAndTime[0]:
 67                    is_was_changed = True
 68            else:
 69                is_was_changed = True
 70
 71            if is_was_changed:
 72                self._requestsAndData[request] = (data, self._clock_function())
 73                self.isWasChanged = True
 74        else:
 75            if len(self._requestsAndData) >= self._itemsQntLimit:
 76                forDeleting = self._requestsHistory[0]
 77                if forDeleting in self._requestsAndData:
 78                    will_be_deleted = self._requestsAndData[forDeleting]
 79                    del self._requestsAndData[forDeleting]
 80                    return will_be_deleted
 81                del self._requestsHistory[0]
 82            self._requestsAndData[request] = (data, self._clock_function())
 83            self.isWasChanged = True
 84
 85    def put_new_request_or_renew_it(self, request, data):
 86        self._move_request_to_the_end_of_history(request)
 87        if request in self._requestsAndData:
 88            dataAndTime = self._requestsAndData[request]
 89            self._requestsAndData[request] = (data, self._clock_function())
 90            self.isWasChanged = True
 91        else:
 92            if len(self._requestsAndData) >= self._itemsQntLimit:
 93                forDeleting = self._requestsHistory[0]
 94                if forDeleting in self._requestsAndData:
 95                    will_be_deleted = self._requestsAndData[forDeleting]
 96                    del self._requestsAndData[forDeleting]
 97                    return will_be_deleted
 98                del self._requestsHistory[0]
 99            self._requestsAndData[request] = (data, self._clock_function())
100            self.isWasChanged = True
101
102    def try_to_get_raw_data_for_request(self, request):
103        if request in self._requestsAndData:
104            dataAndTime = self._requestsAndData[request]
105            return dataAndTime[0]
106        else:
107            return None
108
109    def try_to_get_raw_data_with_time_for_request(self, request):
110        if request in self._requestsAndData:
111            dataAndTime = self._requestsAndData[request]
112            return dataAndTime
113        else:
114            return None
115
116    def try_to_get_data_for_request(self, request):
117        if request in self._requestsAndData:
118            dataAndTime = self._requestsAndData[request]
119            lastChangingTime = dataAndTime[1]
120            tLimit = self._timeLimitInSeconds
121            if (tLimit is not None) and ((self._clock_function() - lastChangingTime) >= tLimit):
122                del self._requestsAndData[request]
123                if request in self._requestsHistory:
124                    self._requestsHistory.remove(request)
125                return None
126            else:
127                self._move_request_to_the_end_of_history(request)
128                return dataAndTime[0]
129        else:
130            return None
131
132    def try_to_get_data_for_request_and_renew_it(self, request):
133        if request in self._requestsAndData:
134            dataAndTime = self._requestsAndData[request]
135            lastChangingTime = dataAndTime[1]
136            tLimit = self._timeLimitInSeconds
137            if (tLimit is not None) and ((self._clock_function() - lastChangingTime) >= tLimit):
138                del self._requestsAndData[request]
139                if request in self._requestsHistory:
140                    self._requestsHistory.remove(request)
141                return None
142            else:
143                self._move_request_to_the_end_of_history(request)
144                self._requestsAndData[request] = (dataAndTime[0], self._clock_function())
145                self.isWasChanged = True
146                return dataAndTime[0]
147        else:
148            return None
149
150    def try_to_remove_request(self, request):
151        if request in self._requestsAndData:
152            del self._requestsAndData[request]
153        if request in self._requestsHistory:
154            self._requestsHistory.remove(request)
155
156    def _move_request_to_the_end_of_history(self, request):
157        if request in self._requestsAndData:
158            if request in self._requestsHistory:
159                self._requestsHistory.remove(request)
160        self._requestsHistory.append(request)
161
162    def update(self, anotherRequestCache):
163        if type(anotherRequestCache) == RequestCache:
164            self._itemsQntLimit += anotherRequestCache._itemsQntLimit
165            self._requestsAndData.update(anotherRequestCache._requestsAndData)
166            self.isWasChanged = True
167            # Do not do this:
168            # self._requestsHistory += anotherRequestCache._requestsHistory
169
170    def clear(self):
171        self._requestsAndData = type(self._requestsAndData)()
172        # self._requestsHistory.clear()  # Python 3.3+ only so can't be used under PyPy yet.
173        self._requestsHistory = type(self._requestsHistory)()
174        self.isWasChanged = True
175
176    def get_state(self):
177        reqAndDat = []
178        for item in self._requestsAndData.items():
179            reqAndDat.append(item)
180        data = (self._itemsQntLimit
181                , self._timeLimitInSeconds
182                , reqAndDat
183                , self._requestsHistory)
184        return json.dumps(data)
185
186    def set_state(self, state):
187        data = json.loads(state)
188        self._itemsQntLimit = data[0]
189        self._timeLimitInSeconds = data[1]
190        for item in data[2]:
191            self._requestsAndData[item[0]] = item[1]
192        self._requestsHistory = data[3]
193        self.isWasChanged = True

RequestCache( itemsQntLimit, timeLimitInSeconds=None, clock_function=None, default_priority: cengal.parallel_execution.coroutines.coro_standard_services.loop_yield.versions.v_0.loop_yield.CoroPriority = <CoroPriority.normal: 1>) View Source

43    def __init__(self, itemsQntLimit, timeLimitInSeconds=None, clock_function=None, default_priority: CoroPriority = CoroPriority.normal):
44        # timeLimitInSeconds:
45        # - number - limit in seconds (remember that the accuracy of the server clock is
46        #   approximately 1 second);
47        # - 0 (zero) - cache is not used (try_to_get_data_for_request() will return 'None'
48        #   on every request);
49        # - None - cache is trying to be permanent (time limit is not used at all)
50        super().__init__()
51        self.default_priority: CoroPriority = default_priority
52        self._clock_function = clock_function or time.perf_counter
53        self._itemsQntLimit = itemsQntLimit
54        self._timeLimitInSeconds = timeLimitInSeconds
55        self._requestsAndData = {}   # key - request; data - (data, Server Time of last change). Server Time of last
56            #  change must be less or equal to self._itemsQntLimit
57        self._requestsHistory = []   # [request0, request1, ..., requestN]
58        self.isWasChanged = False

default_priority: cengal.parallel_execution.coroutines.coro_standard_services.loop_yield.versions.v_0.loop_yield.CoroPriority

isWasChanged

def put_new_request(self, request, data): View Source

60    def put_new_request(self, request, data):
61        self._move_request_to_the_end_of_history(request)
62        if request in self._requestsAndData:
63            dataAndTime = self._requestsAndData[request]
64            is_was_changed = False
65            if isinstance(data, type(dataAndTime[0])):
66                if data != dataAndTime[0]:
67                    is_was_changed = True
68            else:
69                is_was_changed = True
70
71            if is_was_changed:
72                self._requestsAndData[request] = (data, self._clock_function())
73                self.isWasChanged = True
74        else:
75            if len(self._requestsAndData) >= self._itemsQntLimit:
76                forDeleting = self._requestsHistory[0]
77                if forDeleting in self._requestsAndData:
78                    will_be_deleted = self._requestsAndData[forDeleting]
79                    del self._requestsAndData[forDeleting]
80                    return will_be_deleted
81                del self._requestsHistory[0]
82            self._requestsAndData[request] = (data, self._clock_function())
83            self.isWasChanged = True

def put_new_request_or_renew_it(self, request, data): View Source

 85    def put_new_request_or_renew_it(self, request, data):
 86        self._move_request_to_the_end_of_history(request)
 87        if request in self._requestsAndData:
 88            dataAndTime = self._requestsAndData[request]
 89            self._requestsAndData[request] = (data, self._clock_function())
 90            self.isWasChanged = True
 91        else:
 92            if len(self._requestsAndData) >= self._itemsQntLimit:
 93                forDeleting = self._requestsHistory[0]
 94                if forDeleting in self._requestsAndData:
 95                    will_be_deleted = self._requestsAndData[forDeleting]
 96                    del self._requestsAndData[forDeleting]
 97                    return will_be_deleted
 98                del self._requestsHistory[0]
 99            self._requestsAndData[request] = (data, self._clock_function())
100            self.isWasChanged = True

def try_to_get_raw_data_for_request(self, request): View Source

102    def try_to_get_raw_data_for_request(self, request):
103        if request in self._requestsAndData:
104            dataAndTime = self._requestsAndData[request]
105            return dataAndTime[0]
106        else:
107            return None

def try_to_get_raw_data_with_time_for_request(self, request): View Source

109    def try_to_get_raw_data_with_time_for_request(self, request):
110        if request in self._requestsAndData:
111            dataAndTime = self._requestsAndData[request]
112            return dataAndTime
113        else:
114            return None

def try_to_get_data_for_request(self, request): View Source

116    def try_to_get_data_for_request(self, request):
117        if request in self._requestsAndData:
118            dataAndTime = self._requestsAndData[request]
119            lastChangingTime = dataAndTime[1]
120            tLimit = self._timeLimitInSeconds
121            if (tLimit is not None) and ((self._clock_function() - lastChangingTime) >= tLimit):
122                del self._requestsAndData[request]
123                if request in self._requestsHistory:
124                    self._requestsHistory.remove(request)
125                return None
126            else:
127                self._move_request_to_the_end_of_history(request)
128                return dataAndTime[0]
129        else:
130            return None

def try_to_get_data_for_request_and_renew_it(self, request): View Source

132    def try_to_get_data_for_request_and_renew_it(self, request):
133        if request in self._requestsAndData:
134            dataAndTime = self._requestsAndData[request]
135            lastChangingTime = dataAndTime[1]
136            tLimit = self._timeLimitInSeconds
137            if (tLimit is not None) and ((self._clock_function() - lastChangingTime) >= tLimit):
138                del self._requestsAndData[request]
139                if request in self._requestsHistory:
140                    self._requestsHistory.remove(request)
141                return None
142            else:
143                self._move_request_to_the_end_of_history(request)
144                self._requestsAndData[request] = (dataAndTime[0], self._clock_function())
145                self.isWasChanged = True
146                return dataAndTime[0]
147        else:
148            return None

def try_to_remove_request(self, request): View Source

150    def try_to_remove_request(self, request):
151        if request in self._requestsAndData:
152            del self._requestsAndData[request]
153        if request in self._requestsHistory:
154            self._requestsHistory.remove(request)

def update(self, anotherRequestCache): View Source

162    def update(self, anotherRequestCache):
163        if type(anotherRequestCache) == RequestCache:
164            self._itemsQntLimit += anotherRequestCache._itemsQntLimit
165            self._requestsAndData.update(anotherRequestCache._requestsAndData)
166            self.isWasChanged = True
167            # Do not do this:
168            # self._requestsHistory += anotherRequestCache._requestsHistory

def clear(self): View Source

170    def clear(self):
171        self._requestsAndData = type(self._requestsAndData)()
172        # self._requestsHistory.clear()  # Python 3.3+ only so can't be used under PyPy yet.
173        self._requestsHistory = type(self._requestsHistory)()
174        self.isWasChanged = True

def get_state(self): View Source

176    def get_state(self):
177        reqAndDat = []
178        for item in self._requestsAndData.items():
179            reqAndDat.append(item)
180        data = (self._itemsQntLimit
181                , self._timeLimitInSeconds
182                , reqAndDat
183                , self._requestsHistory)
184        return json.dumps(data)

def set_state(self, state): View Source

186    def set_state(self, state):
187        data = json.loads(state)
188        self._itemsQntLimit = data[0]
189        self._timeLimitInSeconds = data[1]
190        for item in data[2]:
191            self._requestsAndData[item[0]] = item[1]
192        self._requestsHistory = data[3]
193        self.isWasChanged = True