Commit 28c7c630 authored by Gerion Entrup's avatar Gerion Entrup
Browse files

fetcher: limit cache

- add fetcher_cache_length: maximum cache size
- add fetcher_cache_age: maximum cache age

The cleanup action for maximum cache age is not triggered yet.
parent 6fbe4ab1
import collections
import logging
import musicbrainzngs
import time
import logging
import settings
class Fetcher:
......@@ -12,22 +15,39 @@ class Fetcher:
self.logger = logging.getLogger('collector.fetcher')
musicbrainzngs.set_useragent("brainzfs", "0.1-alpha",
"https://git.finf.uni-hannover.de/Chrysops/brainzfs")
self.cache = {'recording': {},
'release-group': {},
'artist': {},
'release': {}}
self.time = 0
self._cache = {'recording': {},
'release-group': {},
'artist': {},
'release': {}}
self._cachedates = collections.deque()
self._time = 0
# for debugging
# from offline import CACHE
# self.cache = CACHE
# self._cache = CACHE
def _time_ms(self):
return int(time.time() * 1000)
def _delay(self):
def time_ms():
return int(time.time() * 1000)
diff = time_ms() - self.time
diff = self._time_ms() - self._time
if (diff < 1000):
time.sleep((1000 - diff) / 1000)
self.time = time_ms()
self._time = self._time_ms()
def _cache_append(self, tablename, mbid, result):
self._cache[tablename][mbid] = result
self._cachedates.append((self._time_ms, tablename, mbid))
if len(self._cachedates) > settings.fetcher_cache_length:
d_time, tablename, mbid = self._cachedates.popleft()
del(self._cache[tablename][mbid])
def clean_cache(self):
threshold = self._time_ms - settings.fetcher_cache_age * 1000
d_time, tablename, mbid = self._cachedates[0]
while d_time < threshold:
self._cachedates.popleft()
del(self._cache[tablename][mbid])
d_time, tablename, mbid = self._cachedates[0]
def get_table_by_id(self, mbid, tablename):
"""fetch the data from musicbrainz. Asked the cache in transparent
......@@ -46,15 +66,15 @@ class Fetcher:
'release-groups'])['release']
}
if mbid not in self.cache[tablename]:
if mbid not in self._cache[tablename]:
# delay to follow musicbrainz rules
self._delay()
# request actual data
result = methods[tablename](mbid)
self.cache[tablename][mbid] = result
self._cache_append(tablename, mbid, result)
self.logger.debug("asked web for {}".format(tablename))
else:
result = self.cache[tablename][mbid]
result = self._cache[tablename][mbid]
self.logger.debug("asked fetcher for {}".format(tablename))
return result
default_mode_dir = 0o555
default_mode_file = 0o444
fetcher_cache_length = 10000
fetcher_cache_age = 172800 # 48 hours
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment