Skip to content
GitLab
Commit 1c8991cd authored by Gerion Entrup's avatar Gerion Entrup
Browse files

sqlfuse: add documentation

parent 0d4d6e2a
Hide whitespace changes
Inline Side-by-side
sqlfuse/__init__.py
View file @ 1c8991cd
"""Handles the SQL to llfuse filesystem translation.
This module gets a description of a filesystem and returns a object derived
from llfuse.Operations, that could be used for mounting with llfuse.
The description of the filesystem is a list of *Path (namely QPath and FPath)
object, that will be used to generate the filesystem.
To give an example. Assume you want the following filesystem:
/
|- Dir1
| \- <q1>
| \- Dir3
| \- <q3>
\- Dir2
\- <q2>
Dir1, Dir2 and Dir3 are directories and <q1>, <q2> and <q3> are query object,
that means there are queries that will be executed to build virtual files.
Than you translate the above filesystem in something like:
[QPath("/Dir1/<q1>", query1, functions1),
QPath("/Dir1/<q1>/Dir3/<q3>", query3, functions2, where=[Where(...)]),
QPath("/Dir2/<q2>", , query2, functions2)]
query1 is the query that builds the q1 objects, function1 is the query that
generates the filenames for the q1 objects. The same holds for the other
queries and functions. Assume the query of q3 is somewhat dependend of q1.
In this case use a Where object like Where(function, "<q1>", 'some_attribute').
The queries have to be sqlalchemy.sql.select objects.
To use this module construct a filesystem like the above one and give it to the
module with the init function. The init functions returns then a valid
Operations object.
"""
import sqlfuse.data
import sqlfuse.operations
......@@ -5,9 +41,17 @@ from sqlfuse.path import FPath, QPath
from sqlfuse.where import Where
from sqlfuse.functions import Functions
__all__ = ['FPath', 'QPath', 'Where', 'Functions']
__all__ = ['FPath', 'QPath', 'Where', 'Functions', 'init']
def init(session, fs):
"""Take a fs pseudofilesystem and returns an llfuse.Operations object.
The the module description for details about the pseudofilesystem.
Arguments:
session -- A sqlalchemy session.
fs -- A pseudofilesystem.
"""
f_data = sqlfuse.data.Data(session, fs)
return sqlfuse.operations.Operations(f_data)
sqlfuse/data.py
View file @ 1c8991cd
......@@ -4,7 +4,16 @@ import logging
class Data:
"""Manages the building of the filetree and the allocated inodes"""
def __init__(self, session, fs):
"""Construct a Data object.
Arguments:
session -- a sqlalchemy session
fs -- a list of *Path objects, that describe a filesystem
"""
self.logger = logging.getLogger('sqlfuse.data')
self._cache = llfuse.ROOT_INODE * [None]
self._session = session
......@@ -16,6 +25,7 @@ class Data:
self.logger.debug(self.print_tree())
def _create_tree(self, fs):
"""Creates a tree based on the description filesystem 'fs'"""
dtree = [{}, None]
# build tree as dict
for cpath in fs:
......@@ -39,6 +49,13 @@ class Data:
queue.append((subdnode, nnode))
def _create_tree_node(self, node, qpath, filename):
"""Creates an individual node of the filesystem.
Arguments:
node -- the parent node
qpath -- the path object that holds the attributes and functions
filename -- the filename of the new node
"""
# if self defined
if qpath is not None:
if qpath.fuse_io:
......@@ -56,21 +73,21 @@ class Data:
node.add_fuseio(ndir, filename)
return ndir
def get_file(result):
pass
def get(self, inode):
"""Returns the FuseIO object that belongs to the given inode."""
try:
return self._cache[inode]
except KeyError:
return None
def pin_inode(self, fuseio):
"""Find a free inode and assign it to the FuseIO object."""
ino = len(self._cache)
fuseio.set_inode(ino)
self._cache.append(fuseio)
def print_tree(self):
"""Returns a string representation of the filesystem tree."""
res = ''
queue = [('/', self._tree, 0)]
while len(queue) > 0:
......
sqlfuse/functions.py
View file @ 1c8991cd
......@@ -4,7 +4,26 @@ from sqlalchemy.orm.attributes import InstrumentedAttribute
class Functions:
"""Holds the translation functions between table attributes and filenames.
Construct a Functions with Functions() if you want the functions
autogenerated.
Use Functions.construct_with_functions if you want take self defined
functions.
"""
def __init__(self, *args):
"""Builds a Functions object.
This function excepts a variable amount of arguments of type
str and type InstrumentedAttribute.
The arguments are meant for defining the way a filename is
generated, e.g. the arguments (Artist.name, " - ", Recording.name)
leads to the filename "<Artist.name> - <Recording.name>", where
<Artist.name> and <Recording.name> are replaced with the real values.
"""
self.logger = logging.getLogger('sqlfuse.functions')
self._fltt_re = re.compile('([^A-Z])([A-Z])')
......@@ -28,15 +47,33 @@ class Functions:
@classmethod
def construct_with_functions(cls, gen_name, parse_name):
"""Build a Functions object with predefined functions.
Arguments:
gen_name -- this function will be called if a filename has to be
generated
parse_name -- this function will be called if a filename has to be
parsed
Look into the class documentation for the arguments of the functions.
"""
n_cls = cls()
n_cls.gen_name = gen_name
n_cls.parse_name = parse_name
return n_cls
def _flatten(self, attr):
"""Try to emulate the apply_labels function of sqlalchemy."""
return self._fltt_re.sub(r'\1_\2', str(attr).replace('.', '_')).lower()
def gen_name(self, row):
"""Generates a filename out of an table row.
Arguments:
row -- One row of an sqlalchemy query.
The function is expected to return of filename of type str.
"""
name = []
for (is_str, elem) in self._name:
if is_str:
......@@ -51,6 +88,15 @@ class Functions:
return ''.join(name)
def parse_name(self, name, query):
"""Parses a filename and apply the result to the query.
Arguments:
name -- the filename, that has to be parsed
query -- the query, that needs to be modified
The function is expected to return a modified query if the filename
matches a table row or return None otherwise.
"""
self.logger.debug("parse name: ", name)
res = self._name_regex.match(name)
if res is None:
......
sqlfuse/model.py
View file @ 1c8991cd
......@@ -9,7 +9,20 @@ import settings
class FuseIO():
"""Represents a folder or a file.
The main meaning of this object is to hold a combined class for the
attribute and inode handling.
"""
def __init__(self, mode, inode, attr=None):
"""Construct a FuseIO object.
Arguments:
mode -- the mode of the file
inode -- the inode of the file
attr -- a complete set of attributes, overwrite the mode parameter
"""
self._inode = inode
if attr:
self._attr = attr
......@@ -17,6 +30,7 @@ class FuseIO():
self._attr = self._construct_attr(mode)
def _construct_attr(self, mode):
"""Returns an EntryAttributes object with defaults and given mode."""
attr = llfuse.EntryAttributes()
attr.st_mode = mode
attr.st_nlink = 1
......@@ -40,15 +54,18 @@ class FuseIO():
return attr
def get_inode(self):
"""Return the inode."""
if self._inode == -1:
raise Exception("Special File")
else:
return self._inode
def set_inode(self, inode):
"""Set the inode."""
self._inode = inode
def get_attr(self):
"""Return the attributes."""
if self._inode == -1:
if hasattr(self, "_name"):
raise Exception(self._name + ": Special File")
......@@ -57,10 +74,12 @@ class FuseIO():
return self._attr
def add_nlink(self):
"""Add an nlink to the attributes."""
self._attr.st_nlink += 1
def deepcopy_and_register(self, placeholder, obj):
""" Make a generic deepcopy and register an inode.
The placeholder and obj parameter make no sense here and are
for signature compability with FuseDir.
"""
......@@ -75,11 +94,24 @@ class FuseIO():
return pcopy
def is_special(self):
"""Indicates whether this object is special. Returns always false."""
return False
class Node:
"""Represents a node in the filesystem tree."""
def __init__(self, parent, name, data, root=False):
"""Constructs a node.
Arguments:
parent -- The parent node.
name -- The filename or virtual name of the node.
data -- The Data object, that mananges the inodes.
Keyword arguments:
root -- Indicates whether the node is the root node. (default: False)
"""
self._parent = parent
if root:
self._parent = self
......@@ -89,22 +121,27 @@ class Node:
self._specials = []
def get_parent(self):
"""Returns the parent of the node."""
return self._parent
def get_root(self):
"""Returns the root node of the tree/filesystem."""
parent = self
while parent is not parent.get_parent():
parent = parent.get_parent()
return parent
def get_name(self):
"""Returns the name of the node."""
return self._name
def add_special(self, special):
"""Adds a Special object to the node."""
self._specials.append(special)
return special
def add_fuseio(self, fuseio, filename):
"""Adds a FuseIO object to the node."""
if filename in self._children:
raise Exception("File with filename {} already exists".format(
filename))
......@@ -112,18 +149,40 @@ class Node:
return fuseio
def is_leaf(self):
"""Returns whether the node is a leaf or not."""
return len(self._children) == 0 and len(self._specials) == 0
def get_children(self):
"""Returns the FuseIO children objects as dict.
The dict has the filename as key and the object as value.
"""
return self._children
def get_specials(self):
"""Returns the Special children objects as list."""
return self._specials
class Special(Node):
"""Represent a node that holds a query and can evolve to real nodes."""
def __init__(self, query, functions, where, session,
parent, name, data, fileattr):
"""Construct a special node.
Arguments:
query -- The query that is needed to construct the real nodes.
functions -- A Functions object to generate and parse filenames.
where -- A list of Where objects, to specify the query.
session -- A SQLAlchemy session.
parent -- Superclass argument, see Node.
name -- Superclass argument, see Node.
data -- Superclass argument, see Node.
fileattr -- A function that extract gid and path for FuseFile
generation.
"""
super().__init__(parent, name, data)
self._query = query
self._functions = functions
......@@ -144,6 +203,11 @@ class Special(Node):
")"])
def query_name(self, qname):
"""Execute the query with the given qname.
The function returns the first result if results exists or None
otherwise.
"""
query = self._functions.parse_name(qname, self._query)
self.logger.debug("query by name '{}': {}".format(qname, str(query)))
if query is None:
......@@ -151,19 +215,24 @@ class Special(Node):
return self._session.execute(query).first()
def query_all(self):
"""Execute the query and return the result."""
self.logger.debug("query all: {}".format(str(self._query)))
return self._session.execute(self._query)
def is_special(self):
"""Indicates whether this object is special. Returns always false."""
return True
def get_fileattr(self, row):
"""Return the gid and path attributes of a row."""
return self._fileattr(row)
def get_obj_fname(self, row):
"""Return the generated filename out of a row."""
return self._functions.gen_name(row)
def apply_where(self, placeholder, obj):
"""Apply a where clause, where placeholder is replaced with object."""
for pwhere in self._where:
if pwhere.ph_obj == placeholder:
if self.logger.isEnabledFor(logging.DEBUG):
......@@ -182,7 +251,21 @@ attribute {}, object items: {}".format('.'.join([str(pwhere.function.__self__),
class FuseDir(FuseIO, Node):
"""Represents a directory in the filesystem tree."""
def __init__(self, mode, inode, parent, name, data, root=False):
"""Constructs a directory node.
Arguments:
mode -- The read/write/execute rights of the directory.
inode -- Superclass argument, see FuseIO.
parent -- Superclass argument, see Node.
name -- Superclass argument, see Node.
data -- Superclass argument, see Node.
Keyword arguments:
root -- Superclass argument, see Node
"""
if mode is None:
mode = settings.default_mode_dir
FuseIO.__init__(self, stat.S_IFDIR | mode, inode)
......@@ -200,6 +283,13 @@ class FuseDir(FuseIO, Node):
")"])
def construct_from_query(self, special, row):
"""Construct a FuseIO child based on a query result from a
special object.
Arguments:
special -- The special object.
row -- The row of the query result.
"""
name = special.get_obj_fname(row)
if name in self._children:
return None, None
......@@ -222,6 +312,15 @@ class FuseDir(FuseIO, Node):
return name, f
def deepcopy_and_register(self, placeholder, obj):
"""Constructs a deepcopy of the directory and assign an inode.
If the directory or a subdirectory contains special nodes with
where clauses, this where clause are applied via apply_where().
Arguments:
placeholder -- The placeholder argument for apply_where().
obj -- The obj argument for apply_where().
"""
pcopy = FuseIO.deepcopy_and_register(self, placeholder, obj)
pcopy._children = {}
pcopy._sorted_filenames = []
......@@ -236,6 +335,14 @@ class FuseDir(FuseIO, Node):
return pcopy
def query_name(self, filename):
"""Query the directory contents after a given filename.
This models more less the behaviour of the lookup syscall. If the
directory contains special objects, the relevant queries are executed.
Arguments:
filename -- the filename that will be queried.
"""
if filename == '.':
return self.get_attr()
if filename == '..':
......@@ -249,6 +356,7 @@ class FuseDir(FuseIO, Node):
return f.get_attr()
def add_fuseio(self, fuseio, filename):
"""Adds a FuseIO object with a given filename to the directory."""
Node.add_fuseio(self, fuseio, filename)
self._sorted_filenames.append(filename)
fuseio.add_nlink()
......@@ -257,9 +365,13 @@ class FuseDir(FuseIO, Node):
return fuseio
def opendir(self):
"""Will be called by Operations before readdir(), not function yet."""
pass
def readdir(self, offset):
"""Read the contents of the directory up from offset. Yields the
result.
"""
for child in self._sorted_filenames[offset:]:
yield (child, self._children[child].get_attr())
for special in self._specials:
......@@ -272,7 +384,23 @@ class FuseDir(FuseIO, Node):
class FuseFile(FuseIO, Node):
"""Represents a file node and a leaf in the filesystem tree.
The file is always linked to a real file in a real directory
and opens and read this file, if it is requested.
"""
def __init__(self, mode, inode, parent, gid, path, data):
"""Constructs a file node.
Arguments:
mode -- The read/write/execute rights of the directory.
inode -- Superclass argument, see FuseIO.
parent -- Superclass argument, see Node.
gid -- The unique gid of the file.
path -- The filepath of the real file.
data -- Superclass argument, see Node.
"""
if mode is None:
mode = settings.default_mode_file
FuseIO.__init__(self, stat.S_IFREG | mode, inode)
......@@ -290,20 +418,26 @@ class FuseFile(FuseIO, Node):
")"])
def open(self):
"""Opens the file."""
self._file = open(self._path, 'rb')
def close(self):
"""Closes the file."""
self._file.close()
def get_name(self):
"""Returns the name (gid) of the file."""
return str(self._gid)
def read(self, offset, size):
"""Read size bytes up from offset of the file."""
self._file.seek(offset)
return self._file.read(size)
def add_fuseio(self, fuseio, filename):
"""This method should never be called."""
raise Exception("FuseFile has to be a leaf.")
def add_special(self, special):
"""This method should never be called."""
raise Exception("FuseFile has to be a leaf.")
sqlfuse/path.py
View file @ 1c8991cd
class QPath:
"""Represents a container class for special objects"""
fuse_io = False
def generic_fileattr(row):
"""Give the gid of path attribute of the requested row.
This function returns the attributes of recording. This
should be the common case so it is some kind of generic
way for that.
"""
return row.recording_gid, row.recording_path
def __init__(self, path, query, functions,
where=[], fileattr=generic_fileattr,
mode=None):
"""Construct the object.
Arguments:
path -- the path where the object is located in the filesystem.
query -- the sqlquery that has to be called to populate the last
element in the path.
functions -- a Functions object that handles the creation and parsing
of the filename
Keyword arguments:
where -- list of Where objects, that are applied to the query
(default [])
fileattr -- a function that returns the attributes for gid and path of
a file (default generic_fileattr)
mode -- the mode of the directory (default None)
The path is expected to be a string with the scheme:
"/directory1/directory2/<directory_with_query>/<files_with_query>"
"""
self.path = path
self.query = query.apply_labels()
self.functions = functions
......@@ -16,6 +44,7 @@ class QPath:
class FPath:
"""TODO"""
fuse_io = True
def get(parent, session, name):
......
sqlfuse/where.py
View file @ 1c8991cd
class Where:
"""Represents a where query."""
def __init__(self, function, ph_obj, attribute):
"""Constructs a where query.
Arguments:
function -- the function of a table that should be used.
ph_obj -- the placeholder name of the comparator object.
attribute -- the attribute name of the placeholder object.
Example:
Assume you want to do the query '.where(Artist.gid = <artist>.gid)',
then you construct this object with
Where(Artist.gid.__eq__, '<artist>', 'gid')
"""
self.function = function
self.ph_obj = ph_obj
self.attribute = attribute
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment