Merge branch 'master' into settingswriter

brainzfs.py

@@ -18,18 +18,32 @@ from collector import Collector
 from translator import Translator
 from schedu import Schedu
 
+version = "0.1-alpha"
+schema_version = 1
 
-def _init_database(dbfile):
+
+def _init_database(dbfile, migrate=False):
     """Initialize the database. Currently only sqlite
 
     Arguments:
     dbfile -- the path of the database
     """
+    logger = logging.getLogger('main')
     database = 'sqlite:///'
     engine = create_engine(database + dbfile,
                            connect_args={'check_same_thread':False},
                            #echo = True,
                            poolclass=StaticPool)
+    db_version = model.get_version(engine)
+    if db_version > 0 and db_version < schema_version:
+        if migrate:
+            model.migrate(engine, db_version, schema_version)
+        else:
+            logger.error('Your database is too old for this version of the program. Please apply the "--migrate" flag to migrate the database to the new version. This process is not invertable.')
+            sys.exit(1)
+    if schema_version < db_version:
+        logger.error('Your database was created by a newer version of this program and is incompatible with this version. Please update your program.')
+        sys.exit(1)
     model.init_database(engine)
     session_factory = sessionmaker(bind=engine)
     session = scoped_session(session_factory)
@@ -37,6 +51,7 @@ def _init_database(dbfile):
         #dirty workaroud for sqlite non concurrent mode in python
         #remove it once python supports fully concurrency in sqlite
         session = sqlitequeue.get_Session(session)
+    model.write_version(session, schema_version)
     return session
 
 
@@ -55,7 +70,6 @@ def check(opt, f):
 
 def main(args):
     """Main method. Parses cmdline arguments and starts the program."""
-    version = "0.1-alpha"
     parser = argparse.ArgumentParser(add_help=True,
                                      description="Create a directory structure of music with" \
                                                  "symlinks based on musicbrainz data. Setting commandline options is only temporary. Change the config file to set option permanently.")
@@ -67,6 +81,7 @@ def main(args):
                         help="change to loglevel to info")
     parser.add_argument('--database', '-d', default="music.db", help="path of the database")
     parser.add_argument('--logfile', '-l', help="log into a logfile")
+    parser.add_argument('--migrate', '-m', action="store_true", default=False, help="migrate to a newer database schema. This cannot be undone.")
 
     # options from settings
     for opt, value in settings.get_options():
@@ -89,9 +104,6 @@ def main(args):
                         datefmt='%y-%m-%d %H:%M')
     logger = logging.getLogger('main')
 
-    print(arg.force_clean)
-    print(arg.scan_interval)
-
     # set settings options
     for opt, value in settings.get_options():
         v = getattr(arg, opt)
@@ -99,7 +111,7 @@ def main(args):
             setattr(settings, opt, v)
 
     logger.info("initializing database")
-    session = _init_database(arg.database)
+    session = _init_database(arg.database, migrate=arg.migrate)
     logger.info("database initialized")
 
     collector = Collector(session)

model.py

+import sqlalchemy
+
 from sqlalchemy import Column, Integer, String
+from sqlalchemy.sql import text
 from mbdata.utils import patch_model_schemas, NO_SCHEMAS
 from mbdata.models import Base, Recording
+from utils import session_scope
 
 class BrainzFS(Base):
     __tablename__ = 'brainzfs'
 
     id = Column(Integer, primary_key=True)
     schemaversion = Column(Integer, nullable=False)
-    programversion = Column(Integer, nullable=False)
 
 
 def patch_recording():
@@ -19,11 +22,39 @@ def patch_recording():
     # store the mbid written in the file
     setattr(Recording, 'fgid', Column(String, nullable=False))
 
+
 def init_database(engine):
     patch_model_schemas(NO_SCHEMAS)
     patch_recording()
     Base.metadata.create_all(engine)
 
 
+def get_version(engine):
+    conn = engine.connect()
+
+    try:
+        res = conn.execute(text("SELECT schemaversion FROM brainzfs")).first()
+        if res == None:
+            res = 0
+        else:
+            res = res[0]
+    except sqlalchemy.exc.OperationalError:
+        res = 0
+
+    conn.close()
+    return res
+
+
+def write_version(session, version):
+    with session_scope(session) as session:
+        res = session.query(BrainzFS).first()
+        if res != None:
+            res.schemaversion = version
+        else:
+            v = BrainzFS(schemaversion=version)
+            session.add(v)
 
 
+def migrate(engine, db_version, schema_version):
+    print("migrate from ", db_version, "to", schema_version)
+    pass

retrieval/retrieval.py

@@ -2,25 +2,7 @@ import logging
 import sqlalchemy
 
 from retrieval import fetcher
-from contextlib import contextmanager
-
-
-@contextmanager
-def session_scope(Session):
-    """Provide a transactional scope around a series of operations.
-
-    Arguments:
-    Session -- a SQLAlchemy session class (not an instance)
-    """
-    session = Session()
-    try:
-        yield session
-        session.commit()
-    except:
-        session.rollback()
-        raise
-    finally:
-        session.close()
+from utils import session_scope
 
 
 class Retrieval():

settings/__init__.py

@@ -80,7 +80,7 @@ mk_opt('fetcher_cache_age', 172800,
        'Define the maximum age of a fetcher cache entry (time in seconds).',
        after_default='(48 hours)')
 
-mk_opt('directory_prefix', '/mnt/tmpmount/',
+mk_opt('directory_prefix', '/',
        'Paths of music files will be prefixed with this path and stored relativly to it.')
 mk_opt('force_clean', False,
        "Automated cleaning won't remove more than half of the files at once. Set this to true to change this.")

sqlfuse/__init__.py

@@ -4,9 +4,9 @@ This module gets a description of a filesystem and returns a object derived
 from llfuse.Operations, that could be used for mounting with llfuse.
 
 The description of the filesystem is a list of *Path (namely QPath and FPath)
-object, that will be used to generate the filesystem.
+objects, that will be used to generate the filesystem.
 
-To give an example. Assume you want the following filesystem:
+To give an example: Assume you want the following filesystem:
 /
 |- Dir1
 |   \- <q1>
@@ -30,9 +30,8 @@ In this case use a Where object like Where(function, "<q1>", 'some_attribute').
 
 The queries have to be sqlalchemy.sql.select objects.
 
-To use this module construct a filesystem like the above one and give it to the
-module with the init function. The init functions returns then a valid
-Operations object.
+To use this module pass a filesystem like the above datastructure to the
+init function. The init function then returns a valid Operations object.
 """
 import sqlfuse.data
 import sqlfuse.operations
@@ -40,14 +39,15 @@ import sqlfuse.operations
 from sqlfuse.path import FPath, QPath
 from sqlfuse.where import Where
 from sqlfuse.functions import Functions
+from sqlfuse.model import Generic
 
-__all__ = ['FPath', 'QPath', 'Where', 'Functions', 'init']
+__all__ = ['FPath', 'QPath', 'Where', 'Functions', 'init', 'Generic']
 
 
 def init(session, fs):
     """Take a fs pseudofilesystem and returns an llfuse.Operations object.
 
-    The the module description for details about the pseudofilesystem.
+    Read the module description for details about the pseudofilesystem.
 
     Arguments:
     session -- A sqlalchemy session.

sqlfuse/data.py

@@ -58,9 +58,9 @@ class Data:
         """
         # if self defined
         if qpath is not None:
-            if qpath.fuse_io:
-                ndir = node.add_fuseio(
-                    qpath.get(node, self._session, filename))
+            if qpath.is_generic:
+                ndir = node.add_special(
+                    qpath.generic(self._session, node, filename, self))
             else:
                 ndir = node.add_special(
                     Special(qpath.query, qpath.functions, qpath.where,

sqlfuse/model.py

@@ -222,7 +222,7 @@ class Special(Node):
         return self._session.execute(self._query)
 
     def is_special(self):
-        """Indicates whether this object is special. Returns always false."""
+        """Indicates whether this object is special. Returns always true."""
         return True
 
     def get_fileattr(self, row):
@@ -237,6 +237,7 @@ class Special(Node):
         """Apply a where clause, where placeholder is replaced with object."""
         for pwhere in self._where:
             if pwhere.ph_obj == placeholder:
+                # if is for faster execution in non debug mode
                 if self.logger.isEnabledFor(logging.DEBUG):
                     self.logger.debug("apply where clause with function '{}', \
 attribute {}, object items: {}".format('.'.join([str(pwhere.function.__self__),
@@ -252,6 +253,61 @@ attribute {}, object items: {}".format('.'.join([str(pwhere.function.__self__),
                 self._query = self._query.where(pwhere.function(attr))
 
 
+class Generic(Node):
+    """Represent a generic node that can hold everything and evolve to real
+    nodes.
+    """
+
+    def __init__(self, session, parent, name, data):
+        """Construct a function node.
+
+        Arguments:
+        session   -- A SQLAlchemy session.
+        parent    -- Superclass argument, see Node.
+        name      -- Superclass argument, see Node.
+        data      -- Superclass argument, see Node.
+        """
+
+        super().__init__(parent, name, data)
+        self._session = session
+        self.logger = logging.getLogger('sqlfuse.generic')
+
+    def __repr__(self):
+        return ''.join(["Generic(session=", repr(self._session),
+                        ", parent=",        self._parent.get_name(),
+                        ", name=",          self._name,
+                        ", data=",          repr(self._data),
+                        ")"])
+
+    def query_name(self, filename):
+        """Asks for an identifier of the given filename.
+
+        The function returns a valid identifier if a result exists or None
+        otherwise.
+        """
+        raise NotImplementedError
+
+    def query_all(self):
+        """Get an iterable object for all identifiers of the Generic."""
+        raise NotImplementedError
+
+    def is_special(self):
+        """Indicates whether this object is special. Returns always true."""
+        return True
+
+    def get_obj_fname(self, ident):
+        """Return the generated filename out of an identifier."""
+        raise NotImplementedError
+
+    def apply_where(self, placeholder, obj):
+        """Specify the Generic, where placeholder is replaced with object."""
+        raise NotImplementedError
+
+    def get_fuseio(self,ident):
+        """Return a valid FuseIO object based on the identifier."""
+        raise NotImplementedError
+
+
 class FuseDir(FuseIO, Node):
     """Represents a directory in the filesystem tree."""
 
@@ -292,23 +348,33 @@ class FuseDir(FuseIO, Node):
         special -- The special object.
         row     -- The row of the query result.
         """
+        # check if object with filename already in children
         name = special.get_obj_fname(row)
         if name in self._children:
             return None, None
 
-        if special.is_leaf():
+        # construct the new FuseIO object
+        if isinstance(special, Generic):
+            f = special.get_fuseio(row)
+        elif special.is_leaf():
             (gid, path) = special.get_fileattr(row)
             f = FuseFile(None, -1, self, gid, path, self._data)
         else:
             f = FuseDir(None, -1, self, name, self._data)
+            # copy all children from the special object to the new FuseDir
+            # and apply the Where to all child special objects with the
+            # now known real value
             for (fname, child) in special.get_children().items():
                 f.add_fuseio(child.deepcopy_and_register(special.get_name(),
                                                          row),
                              fname)
-            for s_special in special.get_specials():
-                s_special = copy.copy(s_special)
-                s_special.apply_where(special.get_name(), row)
-                f.add_special(s_special)
+            # copy all specials from the special object to the new FuseDir
+            # and apply the Where with the now known real value
+            for child_special in special.get_specials():
+                child_special = copy.copy(child_special)
+                child_special.apply_where(special.get_name(), row)
+                f.add_special(child_special)
+        # pin and add the object
         self._data.pin_inode(f)
         self.add_fuseio(f, name)
         return name, f
@@ -317,7 +383,7 @@ class FuseDir(FuseIO, Node):
         """Constructs a deepcopy of the directory and assign an inode.
 
         If the directory or a subdirectory contains special nodes with
-        where clauses, this where clause are applied via apply_where().
+        where clauses, this where clauses are applied via apply_where().
 
         Arguments:
         placeholder -- The placeholder argument for apply_where().
@@ -337,7 +403,7 @@ class FuseDir(FuseIO, Node):
         return pcopy
 
     def query_name(self, filename):
-        """Query the directory contents after a given filename.
+        """Query the directory contents with a given filename.
 
         This models more less the behaviour of the lookup syscall. If the
         directory contains special objects, the relevant queries are executed.
@@ -367,7 +433,7 @@ class FuseDir(FuseIO, Node):
         return fuseio
 
     def opendir(self):
-        """Will be called by Operations before readdir(), not function yet."""
+        """Will be called by Operations before readdir(), no function yet."""
         pass
 
     def readdir(self, offset):
@@ -399,7 +465,7 @@ class FuseFile(FuseIO, Node):
         mode   -- The read/write/execute rights of the directory.
         inode  -- Superclass argument, see FuseIO.
         parent -- Superclass argument, see Node.
-        gid    -- The unique gid of the file.
+        gid    -- The unique gid (identifier) of the file.
         path   -- The filepath of the real file.
         data   -- Superclass argument, see Node.
         """

sqlfuse/path.py

 class QPath:
     """Represents a container class for special objects"""
 
-    fuse_io = False
+    is_generic = False
 
     def generic_fileattr(row):
         """Give the gid of path attribute of the requested row.
@@ -45,7 +45,8 @@ class QPath:
 
 class FPath:
     """TODO"""
-    fuse_io = True
+    is_generic = True
 
-    def get(parent, session, name):
-        pass
+    def __init__(self, path, generic):
+        self.path = path
+        self.generic = generic

utils.py

@@ -4,6 +4,25 @@ import pprint
 
 from code import InteractiveConsole
 from inspect import currentframe
+from contextlib import contextmanager
+
+
+@contextmanager
+def session_scope(Session):
+    """Provide a transactional scope around a series of operations.
+
+    Arguments:
+    Session -- a SQLAlchemy session class (not an instance)
+    """
+    session = Session()
+    try:
+        yield session
+        session.commit()
+    except:
+        session.rollback()
+        raise
+    finally:
+        session.close()
 
 
 def get_path(relative_path):