Commit 7d13cede authored by Gerion Entrup's avatar Gerion Entrup
Browse files

translator/sqlfuse: some cleanup and logging

parent e35fa3e0
......@@ -67,11 +67,11 @@ class Main:
self._init_database(arg.database)
logger.info("database initialized")
#collector = Collector(self.session)
#collector.start()
collector = Collector(self.session)
collector.start()
#walker = Walker(self.session, arg.sourcedir)
#walker.start()
walker = Walker(self.session, arg.sourcedir)
walker.start()
#from mbdata.models import Recording, Release, ReleaseGroup, Track, Artist, ArtistCredit, ArtistCreditName, Medium
#from sqlalchemy.sql import select
......
import llfuse
from sqlfuse.model import FuseDir, Special
import logging
class Data:
def __init__(self, session, fs):
self.logger = logging.getLogger('sqlfuse.data')
self._cache = llfuse.ROOT_INODE * [None]
self._session = session
self._tree = FuseDir(None, llfuse.ROOT_INODE, None, "/", self,
root=True)
self._cache.append(self._tree)
self._create_tree(fs)
self.logger.debug("created tree")
self.logger.debug(self.print_tree())
def _create_tree(self, fs):
dtree = [{}, None]
......@@ -29,25 +33,28 @@ class Data:
queue = [(dtree[0], self._tree)]
for (dnode, node) in queue:
for filename in dnode:
t = dnode[filename]
# if self defined
if t[1] is not None:
if t[1].fuse_io:
node.add_fuseio(
t[1].get(node, self._session, filename))
else:
ndir = node.add_special(
Special(t[1].query, t[1].functions, t[1].where,
self._session, node, filename, self,
t[1].fileattr))
# if leaf
if len(t[0]) > 0:
if t[1] is None:
ndir = FuseDir(None, -1, node, filename, self)
if not node.is_special():
self.pin_inode(ndir)
node.add_fuseio(ndir, filename)
queue.append((t[0], ndir))
(subdnode, qpath) = dnode[filename]
nnode = self._create_tree_node(node, qpath, filename)
if len(subdnode) > 0:
queue.append((subdnode, nnode))
def _create_tree_node(self, node, qpath, filename):
# if self defined
if qpath is not None:
if qpath.fuse_io:
ndir = node.add_fuseio(
qpath.get(node, self._session, filename))
else:
ndir = node.add_special(
Special(qpath.query, qpath.functions, qpath.where,
self._session, node, filename, self,
qpath.fileattr))
else:
ndir = FuseDir(None, -1, node, filename, self)
if not node.is_special():
self.pin_inode(ndir)
node.add_fuseio(ndir, filename)
return ndir
def get_file(result):
pass
......@@ -64,12 +71,14 @@ class Data:
self._cache.append(fuseio)
def print_tree(self):
res = ''
queue = [('/', self._tree, 0)]
while len(queue) > 0:
(name, elem, indent) = queue.pop()
print(''.join([' ' * indent, name, ': ', elem.__repr__()]))
res += ''.join([' ' * indent, name, ': ', elem.__repr__(), '\n'])
for (name, child) in elem.get_children().items():
queue.append((name, child, indent + 2))
for special in elem.get_specials():
queue.append(
('special', special, indent + 2))
return res
import re
import logging
from sqlalchemy.orm.attributes import InstrumentedAttribute
class Functions:
def __init__(self, *args):
self.logger = logging.getLogger('sqlfuse.functions')
self._fltt_re = re.compile('([^A-Z])([A-Z])')
self._name = []
......@@ -20,10 +22,9 @@ class Functions:
self._name.append((False, attr_name))
else:
raise Exception("Type must be str or InstrumentedAttribute")
print(self._name_regex)
print(self._name)
print(self._regex_groups)
self._name_regex = re.compile(self._name_regex + '$')
self.logger.debug(
"generated regex: {}".format(self._name_regex.pattern))
@classmethod
def construct_with_functions(cls, gen_name, parse_name):
......@@ -50,6 +51,7 @@ class Functions:
return ''.join(name)
def parse_name(self, name, query):
self.logger.debug("parse name: ", name)
res = self._name_regex.match(name)
if res is None:
return None
......
......@@ -3,6 +3,7 @@ import time
import os
import stat
import copy
import logging
import settings
......@@ -129,6 +130,7 @@ class Special(Node):
self._session = session
self._where = where
self._fileattr = fileattr
self.logger = logging.getLogger('sqlfuse.special')
def __repr__(self):
return ''.join(["Special(query=", repr(self._query),
......@@ -143,11 +145,13 @@ class Special(Node):
def query_name(self, qname):
query = self._functions.parse_name(qname, self._query)
self.logger.debug("query by name '{}': {}".format(qname, str(query)))
if query is None:
return None
return self._session.execute(query).first()
def query_all(self):
self.logger.debug("query all: {}".format(str(self._query)))
return self._session.execute(self._query)
def is_special(self):
......@@ -157,15 +161,17 @@ class Special(Node):
return self._fileattr(row)
def get_obj_fname(self, row):
# from utils import debug_breakpoint
# debug_breakpoint()
return self._functions.gen_name(row)
def apply_where(self, placeholder, obj):
# from utils import debug_breakpoint
# debug_breakpoint()
for pwhere in self._where:
if pwhere.ph_obj == placeholder:
if self.logger.isEnabledFor(logging.DEBUG):
self.logger.debug("apply where clause with function '{}', \
attribute {}, object items: {}".format('.'.join([str(pwhere.function.__self__),
pwhere.function.__name__]),
pwhere.attribute,
obj.items()))
try:
attr = getattr(obj, pwhere.attribute)
except AttributeError:
......@@ -259,8 +265,6 @@ class FuseDir(FuseIO, Node):
for special in self._specials:
nlen = len(self._sorted_filenames)
res = special.query_all()
# from utils import debug_breakpoint
# debug_breakpoint()
for row in res:
name, f = self.construct_from_query(special, row)
if name is not None and offset <= nlen:
......
......@@ -2,6 +2,7 @@ import llfuse
import os
import sqlfuse
import threading
import logging
from mbdata.models import Recording, Release, ReleaseGroup, Track, Artist, ArtistCredit, ArtistCreditName, Medium
from sqlalchemy import distinct
......@@ -14,6 +15,7 @@ class Translator(threading.Thread):
"""
def __init__(self, session_fac, mountpoint):
super().__init__(target=self)
self.logger = logging.getLogger('translator')
self._session = session_fac()
self._filesystem = sqlfuse.init(self._session, self._create_fs())
self._mountpoint = mountpoint
......@@ -94,9 +96,6 @@ class Translator(threading.Thread):
#FPath("/log.txt", Translator._log, is_dir=False)]
return fs
def _result_to_string(result):
return result.name
def run(self):
self.mount(self._mountpoint)
......@@ -107,12 +106,15 @@ class Translator(threading.Thread):
fuse_options.add('fsname=brainzfs')
llfuse.init(self._filesystem, self._mountpoint, fuse_options)
try:
self.logger.info("starting llfuse main loop")
llfuse.main(workers=1)
except:
self.logger.error("llfuse main loop closed unexpected")
llfuse.close(unmount=False)
raise
llfuse.close()
self.logger.info("llfuse main loop closed")
# TODO fix
self._session.commit()
self._session.close()
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment