Commit a36593d2 authored by Gerion Entrup's avatar Gerion Entrup
Browse files

Merge branch 'master' into settingswriter

parents d91675fa 108d5ccd
......@@ -18,18 +18,32 @@ from collector import Collector
from translator import Translator
from schedu import Schedu
version = "0.1-alpha"
schema_version = 1
def _init_database(dbfile):
def _init_database(dbfile, migrate=False):
"""Initialize the database. Currently only sqlite
Arguments:
dbfile -- the path of the database
"""
logger = logging.getLogger('main')
database = 'sqlite:///'
engine = create_engine(database + dbfile,
connect_args={'check_same_thread':False},
#echo = True,
poolclass=StaticPool)
db_version = model.get_version(engine)
if db_version > 0 and db_version < schema_version:
if migrate:
model.migrate(engine, db_version, schema_version)
else:
logger.error('Your database is too old for this version of the program. Please apply the "--migrate" flag to migrate the database to the new version. This process is not invertable.')
sys.exit(1)
if schema_version < db_version:
logger.error('Your database was created by a newer version of this program and is incompatible with this version. Please update your program.')
sys.exit(1)
model.init_database(engine)
session_factory = sessionmaker(bind=engine)
session = scoped_session(session_factory)
......@@ -37,6 +51,7 @@ def _init_database(dbfile):
#dirty workaroud for sqlite non concurrent mode in python
#remove it once python supports fully concurrency in sqlite
session = sqlitequeue.get_Session(session)
model.write_version(session, schema_version)
return session
......@@ -55,7 +70,6 @@ def check(opt, f):
def main(args):
"""Main method. Parses cmdline arguments and starts the program."""
version = "0.1-alpha"
parser = argparse.ArgumentParser(add_help=True,
description="Create a directory structure of music with" \
"symlinks based on musicbrainz data. Setting commandline options is only temporary. Change the config file to set option permanently.")
......@@ -67,6 +81,7 @@ def main(args):
help="change to loglevel to info")
parser.add_argument('--database', '-d', default="music.db", help="path of the database")
parser.add_argument('--logfile', '-l', help="log into a logfile")
parser.add_argument('--migrate', '-m', action="store_true", default=False, help="migrate to a newer database schema. This cannot be undone.")
# options from settings
for opt, value in settings.get_options():
......@@ -89,9 +104,6 @@ def main(args):
datefmt='%y-%m-%d %H:%M')
logger = logging.getLogger('main')
print(arg.force_clean)
print(arg.scan_interval)
# set settings options
for opt, value in settings.get_options():
v = getattr(arg, opt)
......@@ -99,7 +111,7 @@ def main(args):
setattr(settings, opt, v)
logger.info("initializing database")
session = _init_database(arg.database)
session = _init_database(arg.database, migrate=arg.migrate)
logger.info("database initialized")
collector = Collector(session)
......
import sqlalchemy
from sqlalchemy import Column, Integer, String
from sqlalchemy.sql import text
from mbdata.utils import patch_model_schemas, NO_SCHEMAS
from mbdata.models import Base, Recording
from utils import session_scope
class BrainzFS(Base):
__tablename__ = 'brainzfs'
id = Column(Integer, primary_key=True)
schemaversion = Column(Integer, nullable=False)
programversion = Column(Integer, nullable=False)
def patch_recording():
......@@ -19,11 +22,39 @@ def patch_recording():
# store the mbid written in the file
setattr(Recording, 'fgid', Column(String, nullable=False))
def init_database(engine):
patch_model_schemas(NO_SCHEMAS)
patch_recording()
Base.metadata.create_all(engine)
def get_version(engine):
conn = engine.connect()
try:
res = conn.execute(text("SELECT schemaversion FROM brainzfs")).first()
if res == None:
res = 0
else:
res = res[0]
except sqlalchemy.exc.OperationalError:
res = 0
conn.close()
return res
def write_version(session, version):
with session_scope(session) as session:
res = session.query(BrainzFS).first()
if res != None:
res.schemaversion = version
else:
v = BrainzFS(schemaversion=version)
session.add(v)
def migrate(engine, db_version, schema_version):
print("migrate from ", db_version, "to", schema_version)
pass
......@@ -2,25 +2,7 @@ import logging
import sqlalchemy
from retrieval import fetcher
from contextlib import contextmanager
@contextmanager
def session_scope(Session):
"""Provide a transactional scope around a series of operations.
Arguments:
Session -- a SQLAlchemy session class (not an instance)
"""
session = Session()
try:
yield session
session.commit()
except:
session.rollback()
raise
finally:
session.close()
from utils import session_scope
class Retrieval():
......
......@@ -80,7 +80,7 @@ mk_opt('fetcher_cache_age', 172800,
'Define the maximum age of a fetcher cache entry (time in seconds).',
after_default='(48 hours)')
mk_opt('directory_prefix', '/mnt/tmpmount/',
mk_opt('directory_prefix', '/',
'Paths of music files will be prefixed with this path and stored relativly to it.')
mk_opt('force_clean', False,
"Automated cleaning won't remove more than half of the files at once. Set this to true to change this.")
......
......@@ -4,9 +4,9 @@ This module gets a description of a filesystem and returns a object derived
from llfuse.Operations, that could be used for mounting with llfuse.
The description of the filesystem is a list of *Path (namely QPath and FPath)
object, that will be used to generate the filesystem.
objects, that will be used to generate the filesystem.
To give an example. Assume you want the following filesystem:
To give an example: Assume you want the following filesystem:
/
|- Dir1
| \- <q1>
......@@ -30,9 +30,8 @@ In this case use a Where object like Where(function, "<q1>", 'some_attribute').
The queries have to be sqlalchemy.sql.select objects.
To use this module construct a filesystem like the above one and give it to the
module with the init function. The init functions returns then a valid
Operations object.
To use this module pass a filesystem like the above datastructure to the
init function. The init function then returns a valid Operations object.
"""
import sqlfuse.data
import sqlfuse.operations
......@@ -40,14 +39,15 @@ import sqlfuse.operations
from sqlfuse.path import FPath, QPath
from sqlfuse.where import Where
from sqlfuse.functions import Functions
from sqlfuse.model import Generic
__all__ = ['FPath', 'QPath', 'Where', 'Functions', 'init']
__all__ = ['FPath', 'QPath', 'Where', 'Functions', 'init', 'Generic']
def init(session, fs):
"""Take a fs pseudofilesystem and returns an llfuse.Operations object.
The the module description for details about the pseudofilesystem.
Read the module description for details about the pseudofilesystem.
Arguments:
session -- A sqlalchemy session.
......
......@@ -58,9 +58,9 @@ class Data:
"""
# if self defined
if qpath is not None:
if qpath.fuse_io:
ndir = node.add_fuseio(
qpath.get(node, self._session, filename))
if qpath.is_generic:
ndir = node.add_special(
qpath.generic(self._session, node, filename, self))
else:
ndir = node.add_special(
Special(qpath.query, qpath.functions, qpath.where,
......
......@@ -222,7 +222,7 @@ class Special(Node):
return self._session.execute(self._query)
def is_special(self):
"""Indicates whether this object is special. Returns always false."""
"""Indicates whether this object is special. Returns always true."""
return True
def get_fileattr(self, row):
......@@ -237,6 +237,7 @@ class Special(Node):
"""Apply a where clause, where placeholder is replaced with object."""
for pwhere in self._where:
if pwhere.ph_obj == placeholder:
# if is for faster execution in non debug mode
if self.logger.isEnabledFor(logging.DEBUG):
self.logger.debug("apply where clause with function '{}', \
attribute {}, object items: {}".format('.'.join([str(pwhere.function.__self__),
......@@ -252,6 +253,61 @@ attribute {}, object items: {}".format('.'.join([str(pwhere.function.__self__),
self._query = self._query.where(pwhere.function(attr))
class Generic(Node):
"""Represent a generic node that can hold everything and evolve to real
nodes.
"""
def __init__(self, session, parent, name, data):
"""Construct a function node.
Arguments:
session -- A SQLAlchemy session.
parent -- Superclass argument, see Node.
name -- Superclass argument, see Node.
data -- Superclass argument, see Node.
"""
super().__init__(parent, name, data)
self._session = session
self.logger = logging.getLogger('sqlfuse.generic')
def __repr__(self):
return ''.join(["Generic(session=", repr(self._session),
", parent=", self._parent.get_name(),
", name=", self._name,
", data=", repr(self._data),
")"])
def query_name(self, filename):
"""Asks for an identifier of the given filename.
The function returns a valid identifier if a result exists or None
otherwise.
"""
raise NotImplementedError
def query_all(self):
"""Get an iterable object for all identifiers of the Generic."""
raise NotImplementedError
def is_special(self):
"""Indicates whether this object is special. Returns always true."""
return True
def get_obj_fname(self, ident):
"""Return the generated filename out of an identifier."""
raise NotImplementedError
def apply_where(self, placeholder, obj):
"""Specify the Generic, where placeholder is replaced with object."""
raise NotImplementedError
def get_fuseio(self,ident):
"""Return a valid FuseIO object based on the identifier."""
raise NotImplementedError
class FuseDir(FuseIO, Node):
"""Represents a directory in the filesystem tree."""
......@@ -292,23 +348,33 @@ class FuseDir(FuseIO, Node):
special -- The special object.
row -- The row of the query result.
"""
# check if object with filename already in children
name = special.get_obj_fname(row)
if name in self._children:
return None, None
if special.is_leaf():
# construct the new FuseIO object
if isinstance(special, Generic):
f = special.get_fuseio(row)
elif special.is_leaf():
(gid, path) = special.get_fileattr(row)
f = FuseFile(None, -1, self, gid, path, self._data)
else:
f = FuseDir(None, -1, self, name, self._data)
# copy all children from the special object to the new FuseDir
# and apply the Where to all child special objects with the
# now known real value
for (fname, child) in special.get_children().items():
f.add_fuseio(child.deepcopy_and_register(special.get_name(),
row),
fname)
for s_special in special.get_specials():
s_special = copy.copy(s_special)
s_special.apply_where(special.get_name(), row)
f.add_special(s_special)
# copy all specials from the special object to the new FuseDir
# and apply the Where with the now known real value
for child_special in special.get_specials():
child_special = copy.copy(child_special)
child_special.apply_where(special.get_name(), row)
f.add_special(child_special)
# pin and add the object
self._data.pin_inode(f)
self.add_fuseio(f, name)
return name, f
......@@ -317,7 +383,7 @@ class FuseDir(FuseIO, Node):
"""Constructs a deepcopy of the directory and assign an inode.
If the directory or a subdirectory contains special nodes with
where clauses, this where clause are applied via apply_where().
where clauses, this where clauses are applied via apply_where().
Arguments:
placeholder -- The placeholder argument for apply_where().
......@@ -337,7 +403,7 @@ class FuseDir(FuseIO, Node):
return pcopy
def query_name(self, filename):
"""Query the directory contents after a given filename.
"""Query the directory contents with a given filename.
This models more less the behaviour of the lookup syscall. If the
directory contains special objects, the relevant queries are executed.
......@@ -367,7 +433,7 @@ class FuseDir(FuseIO, Node):
return fuseio
def opendir(self):
"""Will be called by Operations before readdir(), not function yet."""
"""Will be called by Operations before readdir(), no function yet."""
pass
def readdir(self, offset):
......@@ -399,7 +465,7 @@ class FuseFile(FuseIO, Node):
mode -- The read/write/execute rights of the directory.
inode -- Superclass argument, see FuseIO.
parent -- Superclass argument, see Node.
gid -- The unique gid of the file.
gid -- The unique gid (identifier) of the file.
path -- The filepath of the real file.
data -- Superclass argument, see Node.
"""
......
class QPath:
"""Represents a container class for special objects"""
fuse_io = False
is_generic = False
def generic_fileattr(row):
"""Give the gid of path attribute of the requested row.
......@@ -45,7 +45,8 @@ class QPath:
class FPath:
"""TODO"""
fuse_io = True
is_generic = True
def get(parent, session, name):
pass
def __init__(self, path, generic):
self.path = path
self.generic = generic
......@@ -4,6 +4,25 @@ import pprint
from code import InteractiveConsole
from inspect import currentframe
from contextlib import contextmanager
@contextmanager
def session_scope(Session):
"""Provide a transactional scope around a series of operations.
Arguments:
Session -- a SQLAlchemy session class (not an instance)
"""
session = Session()
try:
yield session
session.commit()
except:
session.rollback()
raise
finally:
session.close()
def get_path(relative_path):
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment