Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Gerion Entrup
brainzfs
Commits
1c8991cd
Commit
1c8991cd
authored
Aug 12, 2016
by
Gerion Entrup
Browse files
sqlfuse: add documentation
parent
0d4d6e2a
Changes
6
Hide whitespace changes
Inline
Side-by-side
sqlfuse/__init__.py
View file @
1c8991cd
"""Handles the SQL to llfuse filesystem translation.
This module gets a description of a filesystem and returns a object derived
from llfuse.Operations, that could be used for mounting with llfuse.
The description of the filesystem is a list of *Path (namely QPath and FPath)
object, that will be used to generate the filesystem.
To give an example. Assume you want the following filesystem:
/
|- Dir1
| \- <q1>
| \- Dir3
| \- <q3>
\- Dir2
\- <q2>
Dir1, Dir2 and Dir3 are directories and <q1>, <q2> and <q3> are query object,
that means there are queries that will be executed to build virtual files.
Than you translate the above filesystem in something like:
[QPath("/Dir1/<q1>", query1, functions1),
QPath("/Dir1/<q1>/Dir3/<q3>", query3, functions2, where=[Where(...)]),
QPath("/Dir2/<q2>", , query2, functions2)]
query1 is the query that builds the q1 objects, function1 is the query that
generates the filenames for the q1 objects. The same holds for the other
queries and functions. Assume the query of q3 is somewhat dependend of q1.
In this case use a Where object like Where(function, "<q1>", 'some_attribute').
The queries have to be sqlalchemy.sql.select objects.
To use this module construct a filesystem like the above one and give it to the
module with the init function. The init functions returns then a valid
Operations object.
"""
import
sqlfuse.data
import
sqlfuse.operations
...
...
@@ -5,9 +41,17 @@ from sqlfuse.path import FPath, QPath
from
sqlfuse.where
import
Where
from
sqlfuse.functions
import
Functions
__all__
=
[
'FPath'
,
'QPath'
,
'Where'
,
'Functions'
]
__all__
=
[
'FPath'
,
'QPath'
,
'Where'
,
'Functions'
,
'init'
]
def
init
(
session
,
fs
):
"""Take a fs pseudofilesystem and returns an llfuse.Operations object.
The the module description for details about the pseudofilesystem.
Arguments:
session -- A sqlalchemy session.
fs -- A pseudofilesystem.
"""
f_data
=
sqlfuse
.
data
.
Data
(
session
,
fs
)
return
sqlfuse
.
operations
.
Operations
(
f_data
)
sqlfuse/data.py
View file @
1c8991cd
...
...
@@ -4,7 +4,16 @@ import logging
class
Data
:
"""Manages the building of the filetree and the allocated inodes"""
def
__init__
(
self
,
session
,
fs
):
"""Construct a Data object.
Arguments:
session -- a sqlalchemy session
fs -- a list of *Path objects, that describe a filesystem
"""
self
.
logger
=
logging
.
getLogger
(
'sqlfuse.data'
)
self
.
_cache
=
llfuse
.
ROOT_INODE
*
[
None
]
self
.
_session
=
session
...
...
@@ -16,6 +25,7 @@ class Data:
self
.
logger
.
debug
(
self
.
print_tree
())
def
_create_tree
(
self
,
fs
):
"""Creates a tree based on the description filesystem 'fs'"""
dtree
=
[{},
None
]
# build tree as dict
for
cpath
in
fs
:
...
...
@@ -39,6 +49,13 @@ class Data:
queue
.
append
((
subdnode
,
nnode
))
def
_create_tree_node
(
self
,
node
,
qpath
,
filename
):
"""Creates an individual node of the filesystem.
Arguments:
node -- the parent node
qpath -- the path object that holds the attributes and functions
filename -- the filename of the new node
"""
# if self defined
if
qpath
is
not
None
:
if
qpath
.
fuse_io
:
...
...
@@ -56,21 +73,21 @@ class Data:
node
.
add_fuseio
(
ndir
,
filename
)
return
ndir
def
get_file
(
result
):
pass
def
get
(
self
,
inode
):
"""Returns the FuseIO object that belongs to the given inode."""
try
:
return
self
.
_cache
[
inode
]
except
KeyError
:
return
None
def
pin_inode
(
self
,
fuseio
):
"""Find a free inode and assign it to the FuseIO object."""
ino
=
len
(
self
.
_cache
)
fuseio
.
set_inode
(
ino
)
self
.
_cache
.
append
(
fuseio
)
def
print_tree
(
self
):
"""Returns a string representation of the filesystem tree."""
res
=
''
queue
=
[(
'/'
,
self
.
_tree
,
0
)]
while
len
(
queue
)
>
0
:
...
...
sqlfuse/functions.py
View file @
1c8991cd
...
...
@@ -4,7 +4,26 @@ from sqlalchemy.orm.attributes import InstrumentedAttribute
class
Functions
:
"""Holds the translation functions between table attributes and filenames.
Construct a Functions with Functions() if you want the functions
autogenerated.
Use Functions.construct_with_functions if you want take self defined
functions.
"""
def
__init__
(
self
,
*
args
):
"""Builds a Functions object.
This function excepts a variable amount of arguments of type
str and type InstrumentedAttribute.
The arguments are meant for defining the way a filename is
generated, e.g. the arguments (Artist.name, " - ", Recording.name)
leads to the filename "<Artist.name> - <Recording.name>", where
<Artist.name> and <Recording.name> are replaced with the real values.
"""
self
.
logger
=
logging
.
getLogger
(
'sqlfuse.functions'
)
self
.
_fltt_re
=
re
.
compile
(
'([^A-Z])([A-Z])'
)
...
...
@@ -28,15 +47,33 @@ class Functions:
@
classmethod
def
construct_with_functions
(
cls
,
gen_name
,
parse_name
):
"""Build a Functions object with predefined functions.
Arguments:
gen_name -- this function will be called if a filename has to be
generated
parse_name -- this function will be called if a filename has to be
parsed
Look into the class documentation for the arguments of the functions.
"""
n_cls
=
cls
()
n_cls
.
gen_name
=
gen_name
n_cls
.
parse_name
=
parse_name
return
n_cls
def
_flatten
(
self
,
attr
):
"""Try to emulate the apply_labels function of sqlalchemy."""
return
self
.
_fltt_re
.
sub
(
r
'\1_\2'
,
str
(
attr
).
replace
(
'.'
,
'_'
)).
lower
()
def
gen_name
(
self
,
row
):
"""Generates a filename out of an table row.
Arguments:
row -- One row of an sqlalchemy query.
The function is expected to return of filename of type str.
"""
name
=
[]
for
(
is_str
,
elem
)
in
self
.
_name
:
if
is_str
:
...
...
@@ -51,6 +88,15 @@ class Functions:
return
''
.
join
(
name
)
def
parse_name
(
self
,
name
,
query
):
"""Parses a filename and apply the result to the query.
Arguments:
name -- the filename, that has to be parsed
query -- the query, that needs to be modified
The function is expected to return a modified query if the filename
matches a table row or return None otherwise.
"""
self
.
logger
.
debug
(
"parse name: "
,
name
)
res
=
self
.
_name_regex
.
match
(
name
)
if
res
is
None
:
...
...
sqlfuse/model.py
View file @
1c8991cd
...
...
@@ -9,7 +9,20 @@ import settings
class
FuseIO
():
"""Represents a folder or a file.
The main meaning of this object is to hold a combined class for the
attribute and inode handling.
"""
def
__init__
(
self
,
mode
,
inode
,
attr
=
None
):
"""Construct a FuseIO object.
Arguments:
mode -- the mode of the file
inode -- the inode of the file
attr -- a complete set of attributes, overwrite the mode parameter
"""
self
.
_inode
=
inode
if
attr
:
self
.
_attr
=
attr
...
...
@@ -17,6 +30,7 @@ class FuseIO():
self
.
_attr
=
self
.
_construct_attr
(
mode
)
def
_construct_attr
(
self
,
mode
):
"""Returns an EntryAttributes object with defaults and given mode."""
attr
=
llfuse
.
EntryAttributes
()
attr
.
st_mode
=
mode
attr
.
st_nlink
=
1
...
...
@@ -40,15 +54,18 @@ class FuseIO():
return
attr
def
get_inode
(
self
):
"""Return the inode."""
if
self
.
_inode
==
-
1
:
raise
Exception
(
"Special File"
)
else
:
return
self
.
_inode
def
set_inode
(
self
,
inode
):
"""Set the inode."""
self
.
_inode
=
inode
def
get_attr
(
self
):
"""Return the attributes."""
if
self
.
_inode
==
-
1
:
if
hasattr
(
self
,
"_name"
):
raise
Exception
(
self
.
_name
+
": Special File"
)
...
...
@@ -57,10 +74,12 @@ class FuseIO():
return
self
.
_attr
def
add_nlink
(
self
):
"""Add an nlink to the attributes."""
self
.
_attr
.
st_nlink
+=
1
def
deepcopy_and_register
(
self
,
placeholder
,
obj
):
""" Make a generic deepcopy and register an inode.
The placeholder and obj parameter make no sense here and are
for signature compability with FuseDir.
"""
...
...
@@ -75,11 +94,24 @@ class FuseIO():
return
pcopy
def
is_special
(
self
):
"""Indicates whether this object is special. Returns always false."""
return
False
class
Node
:
"""Represents a node in the filesystem tree."""
def
__init__
(
self
,
parent
,
name
,
data
,
root
=
False
):
"""Constructs a node.
Arguments:
parent -- The parent node.
name -- The filename or virtual name of the node.
data -- The Data object, that mananges the inodes.
Keyword arguments:
root -- Indicates whether the node is the root node. (default: False)
"""
self
.
_parent
=
parent
if
root
:
self
.
_parent
=
self
...
...
@@ -89,22 +121,27 @@ class Node:
self
.
_specials
=
[]
def
get_parent
(
self
):
"""Returns the parent of the node."""
return
self
.
_parent
def
get_root
(
self
):
"""Returns the root node of the tree/filesystem."""
parent
=
self
while
parent
is
not
parent
.
get_parent
():
parent
=
parent
.
get_parent
()
return
parent
def
get_name
(
self
):
"""Returns the name of the node."""
return
self
.
_name
def
add_special
(
self
,
special
):
"""Adds a Special object to the node."""
self
.
_specials
.
append
(
special
)
return
special
def
add_fuseio
(
self
,
fuseio
,
filename
):
"""Adds a FuseIO object to the node."""
if
filename
in
self
.
_children
:
raise
Exception
(
"File with filename {} already exists"
.
format
(
filename
))
...
...
@@ -112,18 +149,40 @@ class Node:
return
fuseio
def
is_leaf
(
self
):
"""Returns whether the node is a leaf or not."""
return
len
(
self
.
_children
)
==
0
and
len
(
self
.
_specials
)
==
0
def
get_children
(
self
):
"""Returns the FuseIO children objects as dict.
The dict has the filename as key and the object as value.
"""
return
self
.
_children
def
get_specials
(
self
):
"""Returns the Special children objects as list."""
return
self
.
_specials
class
Special
(
Node
):
"""Represent a node that holds a query and can evolve to real nodes."""
def
__init__
(
self
,
query
,
functions
,
where
,
session
,
parent
,
name
,
data
,
fileattr
):
"""Construct a special node.
Arguments:
query -- The query that is needed to construct the real nodes.
functions -- A Functions object to generate and parse filenames.
where -- A list of Where objects, to specify the query.
session -- A SQLAlchemy session.
parent -- Superclass argument, see Node.
name -- Superclass argument, see Node.
data -- Superclass argument, see Node.
fileattr -- A function that extract gid and path for FuseFile
generation.
"""
super
().
__init__
(
parent
,
name
,
data
)
self
.
_query
=
query
self
.
_functions
=
functions
...
...
@@ -144,6 +203,11 @@ class Special(Node):
")"
])
def
query_name
(
self
,
qname
):
"""Execute the query with the given qname.
The function returns the first result if results exists or None
otherwise.
"""
query
=
self
.
_functions
.
parse_name
(
qname
,
self
.
_query
)
self
.
logger
.
debug
(
"query by name '{}': {}"
.
format
(
qname
,
str
(
query
)))
if
query
is
None
:
...
...
@@ -151,19 +215,24 @@ class Special(Node):
return
self
.
_session
.
execute
(
query
).
first
()
def
query_all
(
self
):
"""Execute the query and return the result."""
self
.
logger
.
debug
(
"query all: {}"
.
format
(
str
(
self
.
_query
)))
return
self
.
_session
.
execute
(
self
.
_query
)
def
is_special
(
self
):
"""Indicates whether this object is special. Returns always false."""
return
True
def
get_fileattr
(
self
,
row
):
"""Return the gid and path attributes of a row."""
return
self
.
_fileattr
(
row
)
def
get_obj_fname
(
self
,
row
):
"""Return the generated filename out of a row."""
return
self
.
_functions
.
gen_name
(
row
)
def
apply_where
(
self
,
placeholder
,
obj
):
"""Apply a where clause, where placeholder is replaced with object."""
for
pwhere
in
self
.
_where
:
if
pwhere
.
ph_obj
==
placeholder
:
if
self
.
logger
.
isEnabledFor
(
logging
.
DEBUG
):
...
...
@@ -182,7 +251,21 @@ attribute {}, object items: {}".format('.'.join([str(pwhere.function.__self__),
class
FuseDir
(
FuseIO
,
Node
):
"""Represents a directory in the filesystem tree."""
def
__init__
(
self
,
mode
,
inode
,
parent
,
name
,
data
,
root
=
False
):
"""Constructs a directory node.
Arguments:
mode -- The read/write/execute rights of the directory.
inode -- Superclass argument, see FuseIO.
parent -- Superclass argument, see Node.
name -- Superclass argument, see Node.
data -- Superclass argument, see Node.
Keyword arguments:
root -- Superclass argument, see Node
"""
if
mode
is
None
:
mode
=
settings
.
default_mode_dir
FuseIO
.
__init__
(
self
,
stat
.
S_IFDIR
|
mode
,
inode
)
...
...
@@ -200,6 +283,13 @@ class FuseDir(FuseIO, Node):
")"
])
def
construct_from_query
(
self
,
special
,
row
):
"""Construct a FuseIO child based on a query result from a
special object.
Arguments:
special -- The special object.
row -- The row of the query result.
"""
name
=
special
.
get_obj_fname
(
row
)
if
name
in
self
.
_children
:
return
None
,
None
...
...
@@ -222,6 +312,15 @@ class FuseDir(FuseIO, Node):
return
name
,
f
def
deepcopy_and_register
(
self
,
placeholder
,
obj
):
"""Constructs a deepcopy of the directory and assign an inode.
If the directory or a subdirectory contains special nodes with
where clauses, this where clause are applied via apply_where().
Arguments:
placeholder -- The placeholder argument for apply_where().
obj -- The obj argument for apply_where().
"""
pcopy
=
FuseIO
.
deepcopy_and_register
(
self
,
placeholder
,
obj
)
pcopy
.
_children
=
{}
pcopy
.
_sorted_filenames
=
[]
...
...
@@ -236,6 +335,14 @@ class FuseDir(FuseIO, Node):
return
pcopy
def
query_name
(
self
,
filename
):
"""Query the directory contents after a given filename.
This models more less the behaviour of the lookup syscall. If the
directory contains special objects, the relevant queries are executed.
Arguments:
filename -- the filename that will be queried.
"""
if
filename
==
'.'
:
return
self
.
get_attr
()
if
filename
==
'..'
:
...
...
@@ -249,6 +356,7 @@ class FuseDir(FuseIO, Node):
return
f
.
get_attr
()
def
add_fuseio
(
self
,
fuseio
,
filename
):
"""Adds a FuseIO object with a given filename to the directory."""
Node
.
add_fuseio
(
self
,
fuseio
,
filename
)
self
.
_sorted_filenames
.
append
(
filename
)
fuseio
.
add_nlink
()
...
...
@@ -257,9 +365,13 @@ class FuseDir(FuseIO, Node):
return
fuseio
def
opendir
(
self
):
"""Will be called by Operations before readdir(), not function yet."""
pass
def
readdir
(
self
,
offset
):
"""Read the contents of the directory up from offset. Yields the
result.
"""
for
child
in
self
.
_sorted_filenames
[
offset
:]:
yield
(
child
,
self
.
_children
[
child
].
get_attr
())
for
special
in
self
.
_specials
:
...
...
@@ -272,7 +384,23 @@ class FuseDir(FuseIO, Node):
class
FuseFile
(
FuseIO
,
Node
):
"""Represents a file node and a leaf in the filesystem tree.
The file is always linked to a real file in a real directory
and opens and read this file, if it is requested.
"""
def
__init__
(
self
,
mode
,
inode
,
parent
,
gid
,
path
,
data
):
"""Constructs a file node.
Arguments:
mode -- The read/write/execute rights of the directory.
inode -- Superclass argument, see FuseIO.
parent -- Superclass argument, see Node.
gid -- The unique gid of the file.
path -- The filepath of the real file.
data -- Superclass argument, see Node.
"""
if
mode
is
None
:
mode
=
settings
.
default_mode_file
FuseIO
.
__init__
(
self
,
stat
.
S_IFREG
|
mode
,
inode
)
...
...
@@ -290,20 +418,26 @@ class FuseFile(FuseIO, Node):
")"
])
def
open
(
self
):
"""Opens the file."""
self
.
_file
=
open
(
self
.
_path
,
'rb'
)
def
close
(
self
):
"""Closes the file."""
self
.
_file
.
close
()
def
get_name
(
self
):
"""Returns the name (gid) of the file."""
return
str
(
self
.
_gid
)
def
read
(
self
,
offset
,
size
):
"""Read size bytes up from offset of the file."""
self
.
_file
.
seek
(
offset
)
return
self
.
_file
.
read
(
size
)
def
add_fuseio
(
self
,
fuseio
,
filename
):
"""This method should never be called."""
raise
Exception
(
"FuseFile has to be a leaf."
)
def
add_special
(
self
,
special
):
"""This method should never be called."""
raise
Exception
(
"FuseFile has to be a leaf."
)
sqlfuse/path.py
View file @
1c8991cd
class
QPath
:
"""Represents a container class for special objects"""
fuse_io
=
False
def
generic_fileattr
(
row
):
"""Give the gid of path attribute of the requested row.
This function returns the attributes of recording. This
should be the common case so it is some kind of generic
way for that.
"""
return
row
.
recording_gid
,
row
.
recording_path
def
__init__
(
self
,
path
,
query
,
functions
,
where
=
[],
fileattr
=
generic_fileattr
,
mode
=
None
):
"""Construct the object.
Arguments:
path -- the path where the object is located in the filesystem.
query -- the sqlquery that has to be called to populate the last
element in the path.
functions -- a Functions object that handles the creation and parsing
of the filename
Keyword arguments:
where -- list of Where objects, that are applied to the query
(default [])
fileattr -- a function that returns the attributes for gid and path of
a file (default generic_fileattr)
mode -- the mode of the directory (default None)
The path is expected to be a string with the scheme:
"/directory1/directory2/<directory_with_query>/<files_with_query>"
"""
self
.
path
=
path
self
.
query
=
query
.
apply_labels
()
self
.
functions
=
functions
...
...
@@ -16,6 +44,7 @@ class QPath:
class
FPath
:
"""TODO"""
fuse_io
=
True
def
get
(
parent
,
session
,
name
):
...
...
sqlfuse/where.py
View file @
1c8991cd
class
Where
:
"""Represents a where query."""
def
__init__
(
self
,
function
,
ph_obj
,
attribute
):
"""Constructs a where query.
Arguments:
function -- the function of a table that should be used.
ph_obj -- the placeholder name of the comparator object.
attribute -- the attribute name of the placeholder object.
Example:
Assume you want to do the query '.where(Artist.gid = <artist>.gid)',
then you construct this object with
Where(Artist.gid.__eq__, '<artist>', 'gid')
"""
self
.
function
=
function
self
.
ph_obj
=
ph_obj
self
.
attribute
=
attribute
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment