Commit 6fbe4ab1 authored by Gerion Entrup's avatar Gerion Entrup
Browse files

migrate from local mbdata to global mbdata

parent b235d186
......@@ -15,8 +15,9 @@ from sqlalchemy.pool import StaticPool
from collector import Collector
from translator import Translator
from walker import Walker
from mbdata.models import Base
from mbdata import patch_model_schemas, NO_SCHEMAS
from mbdata.utils import patch_model_schemas, NO_SCHEMAS
from model import patch_recording, Base
class Main:
"""Main class, that starts all other parts of the program"""
......@@ -33,6 +34,7 @@ class Main:
#echo = True,
poolclass=StaticPool)
patch_model_schemas(NO_SCHEMAS)
patch_recording()
Base.metadata.create_all(engine)
session_factory = sessionmaker(bind=engine)
session = scoped_session(session_factory)
......@@ -73,7 +75,8 @@ class Main:
walker = Walker(self.session, arg.sourcedir)
walker.start()
#from mbdata.models import Recording, Release, ReleaseGroup, Track, Artist, ArtistCredit, ArtistCreditName, Medium
#from model import Recording
#from mbdata.models import Release, ReleaseGroup, Track, Artist, ArtistCredit, ArtistCreditName, Medium
#from sqlalchemy.sql import select
#se = self.session()
#from utils import debug_breakpoint
......
......@@ -11,7 +11,8 @@ from sqlalchemy.orm import Session
from musicbrainzngs import WebServiceError
from collections import namedtuple
from mbdata.models import Recording, ArtistCredit, ArtistCreditName, Artist, Release, ReleaseGroup, Medium, Track, MediumFormat
from model import Recording
from mbdata.models import ArtistCredit, ArtistCreditName, Artist, Release, ReleaseGroup, Medium, Track, MediumFormat
from utils import pairwise
from fetcher import Fetcher
......
Copyright (c) 2013 Lukas Lalinsky
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
MusicBrainz Database Tools
==========================
SQLAlchemy Models
-----------------
This is a flattened version of the MusicBrainz Database Tools from Lukáš Lalinský.
See https://bitbucket.org/lalinsky/mbdata for the original.
The code is modified to run with Python 3.
If you are developing a Python application that needs access to the MusicBrainz
data, you can use the `mbdata.models` module to get [SQLAlchemy](http://www.sqlalchemy.org/)
models mapped to the MusicBrainz database tables.
All tables from the MusicBrainz database are mapped, all foreign keys have one-way
relationships set up and some models, where it's essential to access their
related models, have two-way relationships (collections) set up.
In order to work with the relationships efficiently, you should use the appropriate
kind of [eager loading](http://docs.sqlalchemy.org/en/rel_0_8/orm/loading.html).
Example:
#!python
>>> from sqlalchemy import create_engine
>>> from sqlalchemy.orm import sessionmaker
>>> from mbdata.models import Artist
>>> engine = create_engine('postgresql://musicbrainz:musicbrainz@127.0.0.1/musicbrainz', echo=True)
>>> Session = sessionmaker(bind=engine)
>>> session = Session()
>>> artist = session.query(Artist).filter_by(gid='b10bbbfc-cf9e-42e0-be17-e2c3e1d2600d').first()
>>> print artist.name
# Copyright (C) 2013 Lukas Lalinsky
# Distributed under the MIT license, see the LICENSE file for details.
from sqlalchemy.orm import class_mapper, defer
NO_SCHEMAS = {
'musicbrainz': None,
'cover_art_archive': None,
'wikidocs': None,
'statistics': None,
'documentation': None,
}
SINGLE_MUSICBRAINZ_SCHEMA = {
'musicbrainz': 'musicbrainz',
'cover_art_archive': 'musicbrainz',
'wikidocs': 'musicbrainz',
'statistics': 'musicbrainz',
'documentation': 'musicbrainz',
}
def patch_model_schemas(mapping):
"""Update mbdata.models to use different schema names
The function accepts a dictionary with schema name mapping
and updates the schema for all MusicBrainz tables.
If you want to use the default schema:
>>> patch_model_schemas(NO_SCHEMAS)
If you have just one 'musicbrainz' schema:
>>> patch_model_schemas(SINGLE_MUSICBRAINZ_SCHEMA)
"""
from mbdata.models import Base
for table in Base.metadata.sorted_tables:
if table.schema is None:
continue
table.schema = mapping.get(table.schema, table.schema)
def defer_everything_but(entity, *cols):
m = class_mapper(entity)
return [defer(k) for k in
set(p.key for p in m.iterate_properties
if hasattr(p, 'columns')).difference(cols)]
def get_something_by_gid(query, redirect_model, gid):
artist = query.filter_by(gid=gid).first()
if artist is None:
subquery = query.session.query(redirect_model.redirect_id).\
filter_by(gid=gid)
artist = query.filter(redirect_model.redirect.property.primaryjoin.left.in_(subquery)).first()
return artist
This diff is collapsed.
# Copyright (C) 2013 Lukas Lalinsky
# Distributed under the MIT license, see the LICENSE file for details.
import re
from sqlalchemy.ext.compiler import compiles
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.types import UserDefinedType
from sqlalchemy.sql.expression import ClauseElement, TextClause
@compiles(UUID, 'sqlite')
def visit_uuid_sqlite(element, compiler, **kwargs):
return 'CHAR(32)'
class regexp(ClauseElement):
def __init__(self, column, pattern):
self.column = column
self.pattern = TextClause(pattern)
@compiles(regexp)
def visit_regexp(element, compiler, **kwargs):
return '{0} REGEXP {1}'.format(
compiler.process(element.column),
compiler.process(element.pattern))
@compiles(regexp, 'postgresql')
def visit_regexp_postgresql(element, compiler, **kwargs):
return '{0} ~ {1}'.format(
compiler.process(element.column),
compiler.process(element.pattern))
class PartialDate(object):
__slots__ = ('year', 'month', 'day')
def __init__(self, year=None, month=None, day=None):
self.year = year
self.month = month
self.day = day
def __composite_values__(self):
return self.year, self.month, self.day
def __iter__(self):
yield self.year
yield self.month
yield self.day
def __repr__(self):
return "{0.__class__.__name__}(year={0.year}, month={0.month}, day={0.day})".format(self)
def __eq__(self, other):
return isinstance(other, PartialDate) and \
other.year == self.year and \
other.month == self.month and \
other.day == self.day
def __ne__(self, other):
return not self.__eq__(other)
def __bool__(self):
return bool(self.year or self.month or self.day)
class Point(UserDefinedType):
# pylint: disable=W0223
# pylint: disable=R0201
def get_col_spec(self):
return 'POINT'
def bind_processor(self, dialect):
def process(value):
if value is None:
return None
return '({0},{1})'.format(value[0], value[1])
return process
def result_processor(self, dialect, coltype):
def process(value):
if value is None:
return None
match = re.match(r'\((-?[0-9.]+),(-?[0-9.]+)\)', value)
return tuple([float(x) for x in match.groups()])
return process
class Cube(UserDefinedType):
# pylint: disable=W0223
# pylint: disable=R0201
def get_col_spec(self):
return 'CUBE'
def bind_processor(self, dialect):
def process(value):
if value is None:
return None
return value
return process
def result_processor(self, dialect, coltype):
def process(value):
if value is None:
return None
return value
return process
from sqlalchemy import Column, Integer, String
from mbdata.models import Base, Recording
def patch_recording():
setattr(Recording, 'path', Column(String, nullable=False))
setattr(Recording, 'ftype', Column(String, nullable=False))
class BrainzFS(Base):
__tablename__ = 'brainzfs'
id = Column(Integer, primary_key=True)
schemaversion = Column(Integer, nullable=False)
programversion = Column(Integer, nullable=False)
......@@ -2,3 +2,4 @@ SQLAlchemy==0.9.8
musicbrainzngs==0.5
mutagen==1.27
llfuse==0.40
mbdata
......@@ -4,7 +4,8 @@ import sqlfuse
import threading
import logging
from mbdata.models import Recording, Release, ReleaseGroup, Track, Artist, ArtistCredit, ArtistCreditName, Medium
from model import Recording
from mbdata.models import Release, ReleaseGroup, Track, Artist, ArtistCredit, ArtistCreditName, Medium
from sqlalchemy import distinct
from sqlalchemy.sql import select, join
from sqlfuse import QPath, FPath, Where, Functions
......
......@@ -5,7 +5,7 @@ import logging
from sqlalchemy.orm import Session
from mbdata.models import Recording
from model import Recording
from utils import pairwise
from collector import Paths, NEW, NewData
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment