migrate from local mbdata to global mbdata

brainzfs.py

@@ -15,8 +15,9 @@ from sqlalchemy.pool import StaticPool
 from collector import Collector
 from translator import Translator
 from walker import Walker
-from mbdata.models import Base
-from mbdata import patch_model_schemas, NO_SCHEMAS
+from mbdata.utils import patch_model_schemas, NO_SCHEMAS
+from model import patch_recording, Base
+
 
 class Main:
     """Main class, that starts all other parts of the program"""
@@ -33,6 +34,7 @@ class Main:
                                #echo = True,
                                poolclass=StaticPool)
         patch_model_schemas(NO_SCHEMAS)
+        patch_recording()
         Base.metadata.create_all(engine)
         session_factory = sessionmaker(bind=engine)
         session = scoped_session(session_factory)
@@ -73,7 +75,8 @@ class Main:
         walker = Walker(self.session, arg.sourcedir)
         walker.start()
 
-        #from mbdata.models import Recording, Release, ReleaseGroup, Track, Artist, ArtistCredit, ArtistCreditName, Medium
+        #from model import Recording
+        #from mbdata.models import Release, ReleaseGroup, Track, Artist, ArtistCredit, ArtistCreditName, Medium
         #from sqlalchemy.sql import select
         #se = self.session()
         #from utils import debug_breakpoint

collector.py

@@ -11,7 +11,8 @@ from sqlalchemy.orm import Session
 from musicbrainzngs import WebServiceError
 from collections import namedtuple
 
-from mbdata.models import Recording, ArtistCredit, ArtistCreditName, Artist, Release, ReleaseGroup, Medium, Track, MediumFormat
+from model import Recording
+from mbdata.models import ArtistCredit, ArtistCreditName, Artist, Release, ReleaseGroup, Medium, Track, MediumFormat
 from utils import pairwise
 from fetcher import Fetcher
 

mbdata/LICENSE

-Copyright (c) 2013 Lukas Lalinsky
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-

mbdata/README.md

-MusicBrainz Database Tools
-==========================
-
-SQLAlchemy Models
------------------
-
-This is a flattened version of the MusicBrainz Database Tools from Lukáš Lalinský.
-See https://bitbucket.org/lalinsky/mbdata for the original.
-
-The code is modified to run with Python 3.
-
-If you are developing a Python application that needs access to the MusicBrainz
-data, you can use the `mbdata.models` module to get [SQLAlchemy](http://www.sqlalchemy.org/)
-models mapped to the MusicBrainz database tables.
-
-All tables from the MusicBrainz database are mapped, all foreign keys have one-way
-relationships set up and some models, where it's essential to access their
-related models, have two-way relationships (collections) set up.
-
-In order to work with the relationships efficiently, you should use the appropriate
-kind of [eager loading](http://docs.sqlalchemy.org/en/rel_0_8/orm/loading.html).
-
-Example:
-
-    #!python
-    >>> from sqlalchemy import create_engine
-    >>> from sqlalchemy.orm import sessionmaker
-    >>> from mbdata.models import Artist
-    >>> engine = create_engine('postgresql://musicbrainz:musicbrainz@127.0.0.1/musicbrainz', echo=True)
-    >>> Session = sessionmaker(bind=engine)
-    >>> session = Session()
-    >>> artist = session.query(Artist).filter_by(gid='b10bbbfc-cf9e-42e0-be17-e2c3e1d2600d').first()
-    >>> print artist.name
-

mbdata/__init__.py

-# Copyright (C) 2013 Lukas Lalinsky
-# Distributed under the MIT license, see the LICENSE file for details.
-
-from sqlalchemy.orm import class_mapper, defer
-
-
-NO_SCHEMAS = {
-    'musicbrainz': None,
-    'cover_art_archive': None,
-    'wikidocs': None,
-    'statistics': None,
-    'documentation': None,
-}
-
-SINGLE_MUSICBRAINZ_SCHEMA = {
-    'musicbrainz': 'musicbrainz',
-    'cover_art_archive': 'musicbrainz',
-    'wikidocs': 'musicbrainz',
-    'statistics': 'musicbrainz',
-    'documentation': 'musicbrainz',
-}
-
-
-def patch_model_schemas(mapping):
-    """Update mbdata.models to use different schema names
-
-    The function accepts a dictionary with schema name mapping
-    and updates the schema for all MusicBrainz tables.
-
-    If you want to use the default schema:
-
-    >>> patch_model_schemas(NO_SCHEMAS)
-
-    If you have just one 'musicbrainz' schema:
-
-    >>> patch_model_schemas(SINGLE_MUSICBRAINZ_SCHEMA)
-
-    """
-    from mbdata.models import Base
-
-    for table in Base.metadata.sorted_tables:
-        if table.schema is None:
-            continue
-        table.schema = mapping.get(table.schema, table.schema)
-
-
-def defer_everything_but(entity, *cols):
-    m = class_mapper(entity)
-    return [defer(k) for k in
-            set(p.key for p in m.iterate_properties
-                if hasattr(p, 'columns')).difference(cols)]
-
-
-def get_something_by_gid(query, redirect_model, gid):
-    artist = query.filter_by(gid=gid).first()
-    if artist is None:
-        subquery = query.session.query(redirect_model.redirect_id).\
-            filter_by(gid=gid)
-        artist = query.filter(redirect_model.redirect.property.primaryjoin.left.in_(subquery)).first()
-    return artist
-

mbdata/types.py

-# Copyright (C) 2013 Lukas Lalinsky
-# Distributed under the MIT license, see the LICENSE file for details.
-
-import re
-from sqlalchemy.ext.compiler import compiles
-from sqlalchemy.dialects.postgresql import UUID
-from sqlalchemy.types import UserDefinedType
-from sqlalchemy.sql.expression import ClauseElement, TextClause
-
-
-@compiles(UUID, 'sqlite')
-def visit_uuid_sqlite(element, compiler, **kwargs):
-    return 'CHAR(32)'
-
-
-class regexp(ClauseElement):
-    def __init__(self, column, pattern):
-        self.column = column
-        self.pattern = TextClause(pattern)
-
-
-@compiles(regexp)
-def visit_regexp(element, compiler, **kwargs):
-    return '{0} REGEXP {1}'.format(
-        compiler.process(element.column),
-        compiler.process(element.pattern))
-
-
-@compiles(regexp, 'postgresql')
-def visit_regexp_postgresql(element, compiler, **kwargs):
-    return '{0} ~ {1}'.format(
-        compiler.process(element.column),
-        compiler.process(element.pattern))
-
-
-class PartialDate(object):
-
-    __slots__ = ('year', 'month', 'day')
-
-    def __init__(self, year=None, month=None, day=None):
-        self.year = year
-        self.month = month
-        self.day = day
-
-    def __composite_values__(self):
-        return self.year, self.month, self.day
-
-    def __iter__(self):
-        yield self.year
-        yield self.month
-        yield self.day
-
-    def __repr__(self):
-        return "{0.__class__.__name__}(year={0.year}, month={0.month}, day={0.day})".format(self)
-
-    def __eq__(self, other):
-        return isinstance(other, PartialDate) and \
-            other.year == self.year and \
-            other.month == self.month and \
-            other.day == self.day
-
-    def __ne__(self, other):
-        return not self.__eq__(other)
-
-    def __bool__(self):
-        return bool(self.year or self.month or self.day)
-
-
-class Point(UserDefinedType):
-
-    # pylint: disable=W0223
-    # pylint: disable=R0201
-
-    def get_col_spec(self):
-        return 'POINT'
-
-    def bind_processor(self, dialect):
-        def process(value):
-            if value is None:
-                return None
-            return '({0},{1})'.format(value[0], value[1])
-        return process
-
-    def result_processor(self, dialect, coltype):
-        def process(value):
-            if value is None:
-                return None
-            match = re.match(r'\((-?[0-9.]+),(-?[0-9.]+)\)', value)
-            return tuple([float(x) for x in match.groups()])
-        return process
-
-
-class Cube(UserDefinedType):
-
-    # pylint: disable=W0223
-    # pylint: disable=R0201
-
-    def get_col_spec(self):
-        return 'CUBE'
-
-    def bind_processor(self, dialect):
-        def process(value):
-            if value is None:
-                return None
-            return value
-        return process
-
-    def result_processor(self, dialect, coltype):
-        def process(value):
-            if value is None:
-                return None
-            return value
-        return process
-

model.py

+from sqlalchemy import Column, Integer, String
+from mbdata.models import Base, Recording
+
+
+def patch_recording():
+    setattr(Recording, 'path', Column(String, nullable=False))
+    setattr(Recording, 'ftype', Column(String, nullable=False))
+
+
+class BrainzFS(Base):
+    __tablename__ = 'brainzfs'
+
+    id = Column(Integer, primary_key=True)
+    schemaversion = Column(Integer, nullable=False)
+    programversion = Column(Integer, nullable=False)

requirements.txt

@@ -2,3 +2,4 @@ SQLAlchemy==0.9.8
 musicbrainzngs==0.5
 mutagen==1.27
 llfuse==0.40
+mbdata

translator.py

@@ -4,7 +4,8 @@ import sqlfuse
 import threading
 import logging
 
-from mbdata.models import Recording, Release, ReleaseGroup, Track, Artist, ArtistCredit, ArtistCreditName, Medium
+from model import Recording
+from mbdata.models import Release, ReleaseGroup, Track, Artist, ArtistCredit, ArtistCreditName, Medium
 from sqlalchemy import distinct
 from sqlalchemy.sql import select, join
 from sqlfuse import QPath, FPath, Where, Functions

walker.py

@@ -5,7 +5,7 @@ import logging
 
 from sqlalchemy.orm import Session
 
-from mbdata.models import Recording
+from model import Recording
 from utils import pairwise
 from collector import Paths, NEW, NewData