Commit 1bba597d authored by Gerion Entrup's avatar Gerion Entrup
Browse files

retrieval: replace retrieval.Table with the ORM objects of mbdata

parent 40283c93
...@@ -7,7 +7,7 @@ import musicbrainzngs ...@@ -7,7 +7,7 @@ import musicbrainzngs
from mbdata.models import ArtistCredit, ArtistCreditName, Artist, Release, ReleaseGroup, Medium, Track, MediumFormat from mbdata.models import ArtistCredit, ArtistCreditName, Artist, Release, ReleaseGroup, Medium, Track, MediumFormat
from retrieval import Entity, Filter, Table from retrieval import Entity, Filter
from model import Recording from model import Recording
from utils import pairwise from utils import pairwise
...@@ -39,7 +39,7 @@ class Collector(threading.Thread): ...@@ -39,7 +39,7 @@ class Collector(threading.Thread):
self._logger.info("Adding file " + self._logger.info("Adding file " +
"{} to the database.".format(path)) "{} to the database.".format(path))
try: try:
retrieval.create(Table.recording, mbid, path) retrieval.create(Recording, mbid, path)
retrieval.commit() retrieval.commit()
except musicbrainzngs.WebServiceError as exc: except musicbrainzngs.WebServiceError as exc:
self._logger.error("Could not connect to Musicbrainz. " self._logger.error("Could not connect to Musicbrainz. "
...@@ -51,10 +51,10 @@ class Collector(threading.Thread): ...@@ -51,10 +51,10 @@ class Collector(threading.Thread):
# non nullable attributes # non nullable attributes
release.gid = result['id'] release.gid = result['id']
release.name = result['title'] release.name = result['title']
release.artist_credit = retrieval.create(Table.artist_credit, release.artist_credit = retrieval.create(ArtistCredit,
result['artist-credit'], result['artist-credit'],
result['artist-credit-phrase']) result['artist-credit-phrase'])
release.release_group = retrieval.create(Table.release_group, release.release_group = retrieval.create(ReleaseGroup,
result['release-group']['id']) result['release-group']['id'])
# nullable attributes # nullable attributes
...@@ -63,7 +63,7 @@ class Collector(threading.Thread): ...@@ -63,7 +63,7 @@ class Collector(threading.Thread):
# extended mapping # extended mapping
for medium in result['medium-list']: for medium in result['medium-list']:
retrieval.create(Table.medium, release, medium['position'], medium) retrieval.create(Medium, release, medium['position'], medium)
def _recording_mapping(recording, result, mbid, path): def _recording_mapping(recording, result, mbid, path):
# non nullable attributes # non nullable attributes
...@@ -72,7 +72,7 @@ class Collector(threading.Thread): ...@@ -72,7 +72,7 @@ class Collector(threading.Thread):
recording.fgid = mbid recording.fgid = mbid
recording.path = path recording.path = path
recording.ftype = os.path.splitext(path)[1][1:] recording.ftype = os.path.splitext(path)[1][1:]
recording.artist_credit = retrieval.create(Table.artist_credit, recording.artist_credit = retrieval.create(ArtistCredit,
result['artist-credit'], result['artist-credit'],
result['artist-credit-phrase']) result['artist-credit-phrase'])
...@@ -82,17 +82,17 @@ class Collector(threading.Thread): ...@@ -82,17 +82,17 @@ class Collector(threading.Thread):
# extended mapping # extended mapping
for releasedata in result['release-list']: for releasedata in result['release-list']:
release = retrieval.create(Table.release, releasedata['id']) release = retrieval.create(Release, releasedata['id'])
# Find track in release. # Find track in release.
# This is clearly a workaround and only works efficient # This is clearly a workaround and only works efficient
# because of caching. Correct way would be to fetch all tracks # because of caching. Correct way would be to fetch all tracks
# directly, but the musicbrainz api offers no way to do this. # directly, but the musicbrainz api offers no way to do this.
mediumlist = retrieval.get_table_by_id( mediumlist = retrieval.get_table_by_id(
releasedata['id'], Table.release)['medium-list'] releasedata['id'], Release)['medium-list']
for medium in mediumlist: for medium in mediumlist:
for track in medium['track-list']: for track in medium['track-list']:
if track['recording']['id'] == result['id']: if track['recording']['id'] == result['id']:
retrieval.create(Table.track, track['id'], release, retrieval.create(Track, track['id'], release,
recording, medium, track) recording, medium, track)
def _artist_mapping(artist, result, mbid): def _artist_mapping(artist, result, mbid):
...@@ -104,7 +104,7 @@ class Collector(threading.Thread): ...@@ -104,7 +104,7 @@ class Collector(threading.Thread):
def _release_group_mapping(release_group, result, mbid): def _release_group_mapping(release_group, result, mbid):
release_group.gid = result['id'] release_group.gid = result['id']
release_group.name = result['title'] release_group.name = result['title']
release_group.artist_credit = retrieval.create(Table.artist_credit, release_group.artist_credit = retrieval.create(ArtistCredit,
result['artist-credit'], result['artist-credit'],
result['artist-credit-phrase']) result['artist-credit-phrase'])
...@@ -120,7 +120,7 @@ class Collector(threading.Thread): ...@@ -120,7 +120,7 @@ class Collector(threading.Thread):
# extended mapping # extended mapping
for idx, (data, joinphrase) in enumerate(pairwise(acresult + [''])): for idx, (data, joinphrase) in enumerate(pairwise(acresult + [''])):
data['joinphrase'] = joinphrase data['joinphrase'] = joinphrase
retrieval.create(Table.artist_credit_name, retrieval.create(ArtistCreditName,
artist_credit, artist_credit,
idx + 1, # SQL IDs begin with 1 idx + 1, # SQL IDs begin with 1
data) data)
...@@ -132,7 +132,7 @@ class Collector(threading.Thread): ...@@ -132,7 +132,7 @@ class Collector(threading.Thread):
# non nullable attributes # non nullable attributes
acn.artist_credit = artist_credit acn.artist_credit = artist_credit
acn.position = position acn.position = position
acn.artist = retrieval.create(Table.artist, data['artist']['id']) acn.artist = retrieval.create(Artist, data['artist']['id'])
acn.name = data['name'] if 'name' in data else data['artist']['name'] acn.name = data['name'] if 'name' in data else data['artist']['name']
acn.join_phrase = data['joinphrase'] acn.join_phrase = data['joinphrase']
...@@ -142,7 +142,7 @@ class Collector(threading.Thread): ...@@ -142,7 +142,7 @@ class Collector(threading.Thread):
medium.position = medium_data['position'] medium.position = medium_data['position']
medium.track_count = medium_data['track-count'] medium.track_count = medium_data['track-count']
if 'format' in medium_data: if 'format' in medium_data:
medium.format = retrieval.create(Table.medium_format, medium.format = retrieval.create(MediumFormat,
medium_data['format']) medium_data['format'])
def _medium_format_mapping(medium_format, name): def _medium_format_mapping(medium_format, name):
...@@ -157,7 +157,7 @@ class Collector(threading.Thread): ...@@ -157,7 +157,7 @@ class Collector(threading.Thread):
track.number = data['number'] track.number = data['number']
track.recording = recording track.recording = recording
track.name = recording.name track.name = recording.name
track.medium = retrieval.create(Table.medium, release, track.medium = retrieval.create(Medium, release,
medium_data['position'], medium_data) medium_data['position'], medium_data)
track.artist_credit = recording.artist_credit track.artist_credit = recording.artist_credit
track.artist_credit.ref_count += 1 track.artist_credit.ref_count += 1
...@@ -165,54 +165,44 @@ class Collector(threading.Thread): ...@@ -165,54 +165,44 @@ class Collector(threading.Thread):
track.length = data['length'] track.length = data['length']
_structure = { _structure = {
Table.release: Entity(parameter=('mbid',), Release: Entity(parameter=('mbid',),
mb_class=Release, query_filter=Filter(gid='mbid'),
query_filter=Filter(gid='mbid'), web='mbid',
web='mbid', mapping=_release_mapping),
mapping=_release_mapping), Recording: Entity(parameter=('mbid', 'path'),
Table.recording: Entity(parameter=('mbid', 'path'), query_filter=Filter(gid='mbid'),
mb_class=Recording, web='mbid',
query_filter=Filter(gid='mbid'), mapping=_recording_mapping),
web='mbid', Artist: Entity(parameter=('mbid',),
mapping=_recording_mapping), query_filter=Filter(gid='mbid'),
Table.artist: Entity(parameter=('mbid',), web='mbid',
mb_class=Artist, mapping=_artist_mapping),
ReleaseGroup: Entity(parameter=('mbid',),
query_filter=Filter(gid='mbid'), query_filter=Filter(gid='mbid'),
web='mbid', web='mbid',
mapping=_artist_mapping), mapping=_release_group_mapping),
Table.release_group: Entity(parameter=('mbid',), ArtistCredit: Entity(parameter=('acresult', 'acphrase'),
mb_class=ReleaseGroup, query_filter=Filter(name='acphrase'),
query_filter=Filter(gid='mbid'),
web='mbid',
mapping=_release_group_mapping),
Table.artist_credit: Entity(parameter=('acresult', 'acphrase'),
mb_class=ArtistCredit,
query_filter=Filter(name='acphrase'),
web=None,
mapping=_artist_credit_mapping,
reverse_mapping=_artist_credit_r_mapping),
Table.artist_credit_name: Entity(parameter=('artist_credit',
'position',
'data'),
mb_class=ArtistCreditName,
query_filter=Filter(artist_credit='artist_credit',
position='position'),
web=None,
mapping=_artist_credit_name_mapping),
Table.medium: Entity(parameter=('release', 'position', 'medium_data'),
mb_class=Medium,
query_filter=Filter(release='release',
position='position'),
web=None, web=None,
mapping=_medium_mapping), mapping=_artist_credit_mapping,
Table.medium_format: Entity(parameter=('name',), reverse_mapping=_artist_credit_r_mapping),
mb_class=MediumFormat, ArtistCreditName: Entity(parameter=('artist_credit', 'position',
query_filter=Filter(name='name'), 'data'),
web=None, query_filter=Filter(artist_credit='artist_credit',
mapping=_medium_format_mapping), position='position'),
Table.track: Entity(parameter=('mbid', 'release', 'recording', web=None,
'medium_data', 'data'), mapping=_artist_credit_name_mapping),
mb_class=Track, Medium: Entity(parameter=('release', 'position', 'medium_data'),
query_filter=Filter(gid='mbid'), query_filter=Filter(release='release',
web=None, position='position'),
mapping=_track_mapping)} web=None,
mapping=_medium_mapping),
MediumFormat: Entity(parameter=('name',),
query_filter=Filter(name='name'),
web=None,
mapping=_medium_format_mapping),
Track: Entity(parameter=('mbid', 'release', 'recording',
'medium_data', 'data'),
query_filter=Filter(gid='mbid'),
web=None,
mapping=_track_mapping)}
...@@ -6,7 +6,7 @@ and web retrieval tasks in a generic way. ...@@ -6,7 +6,7 @@ and web retrieval tasks in a generic way.
It was written mainly to avoid doubling code and provide a clean interface. It was written mainly to avoid doubling code and provide a clean interface.
The structure has to be a dict with an entry of Table as key and an Entity The structure has to be a dict with an ORM table class as key and an Entity
as value. See the documentation of Entity for furthor information. as value. See the documentation of Entity for furthor information.
To use the module, the first call must be init() with the the structure and To use the module, the first call must be init() with the the structure and
...@@ -14,12 +14,11 @@ a session class (not an instance). Then several create() calls could be made. ...@@ -14,12 +14,11 @@ a session class (not an instance). Then several create() calls could be made.
The last action is commit(), to commit all to the database. The last action is commit(), to commit all to the database.
""" """
from retrieval.fetcher import get_table_by_id from retrieval.fetcher import get_table_by_id
from retrieval.entity import Entity, Filter, Table from retrieval.entity import Entity, Filter
from retrieval.helper import map_quality, fake_id from retrieval.helper import map_quality, fake_id
from retrieval.retrieval import Retrieval from retrieval.retrieval import Retrieval
__all__ = ['get_table_by_id', 'Table', 'Entity', 'Filter', __all__ = ['get_table_by_id', 'Entity', 'Filter', 'create', 'init', 'commit',
'create', 'init', 'commit',
'map_quality', 'fake_id'] 'map_quality', 'fake_id']
_retrieval = False _retrieval = False
...@@ -45,8 +44,8 @@ def create(table, *args): ...@@ -45,8 +44,8 @@ def create(table, *args):
"""Create a table object (row in the database). """Create a table object (row in the database).
Arguments: Arguments:
table -- The type of the table entry that should be created. Has to be of table -- The type of the table entry that should be created. Has to be an
type Table. ORM table class.
*args -- All arguments necessary for the specific table. *args -- All arguments necessary for the specific table.
This function return a valid table object. If the object is in the cache This function return a valid table object. If the object is in the cache
......
import enum
from collections import namedtuple from collections import namedtuple
...@@ -8,8 +6,8 @@ def dummy(*args, **kwargs): ...@@ -8,8 +6,8 @@ def dummy(*args, **kwargs):
pass pass
Entity_ = namedtuple('Entity', ['parameter', 'mb_class', 'query_filter', Entity_ = namedtuple('Entity', ['parameter', 'query_filter', 'web',
'web', 'mapping', 'reverse_mapping']) 'mapping', 'reverse_mapping'])
# set reverse_mapping default to dummy and make it optional # set reverse_mapping default to dummy and make it optional
Entity_.__new__.__defaults__ = (dummy, ) Entity_.__new__.__defaults__ = (dummy, )
...@@ -21,7 +19,6 @@ class Entity(Entity_): ...@@ -21,7 +19,6 @@ class Entity(Entity_):
Attributes: Attributes:
parameter -- tuple of strings, that code the arguments later parameter -- tuple of strings, that code the arguments later
commited. commited.
mb_class -- The ORM Table class, that is worked with.
query_filter -- The Filter object necessary for the checking SQL query. query_filter -- The Filter object necessary for the checking SQL query.
web -- The codification of the ID attribute for the web query. web -- The codification of the ID attribute for the web query.
Must be existant in parameter, too. Must be existant in parameter, too.
...@@ -61,16 +58,3 @@ class Filter(): ...@@ -61,16 +58,3 @@ class Filter():
filter. filter.
""" """
return self._kwargs.copy() return self._kwargs.copy()
class Table(enum.Enum):
"""All supported tables."""
artist = 1
artist_credit = 2
artist_credit_name = 3
medium = 4
medium_format = 5
recording = 6
release = 7
release_group = 8
track = 9
...@@ -6,14 +6,14 @@ import threading ...@@ -6,14 +6,14 @@ import threading
import settings import settings
from retrieval.entity import Table from mbdata.models import Recording, ReleaseGroup, Artist, Release
""" Fetches the musicdata and caches them. Could only exist once. """ Fetches the musicdata and caches them. Could only exist once.
Use it with get_table_by_id() and clean_cache(). Use it with get_table_by_id() and clean_cache().
""" """
web = [Table.recording, Table.release_group, Table.artist, Table.release] web = [Recording, ReleaseGroup, Artist, Release]
musicbrainzngs.set_useragent("brainzfs", "0.1-alpha", musicbrainzngs.set_useragent("brainzfs", "0.1-alpha",
"https://git.finf.uni-hannover.de/Chrysops/brainzfs") "https://git.finf.uni-hannover.de/Chrysops/brainzfs")
...@@ -58,10 +58,10 @@ def _get_release(mbid): ...@@ -58,10 +58,10 @@ def _get_release(mbid):
'recordings', 'recordings',
'release-groups'])['release'] 'release-groups'])['release']
_methods = {Table.recording: _get_recording, _methods = {Recording: _get_recording,
Table.release_group: _get_release_group, ReleaseGroup: _get_release_group,
Table.artist: _get_artist, Artist: _get_artist,
Table.release: _get_release} Release: _get_release}
def _time_ms(): def _time_ms():
...@@ -111,7 +111,7 @@ def get_table_by_id(mbid, tablename): ...@@ -111,7 +111,7 @@ def get_table_by_id(mbid, tablename):
Arguments: Arguments:
mbid -- The id that should be fetched. mbid -- The id that should be fetched.
tablename -- An entry of retrieval.Table to specify the table. tablename -- An ORM table class to specify the table.
""" """
if tablename not in web: if tablename not in web:
raise("Error: no Web lookup possible for " + str(tablename)) raise("Error: no Web lookup possible for " + str(tablename))
......
...@@ -64,12 +64,12 @@ class Retrieval(): ...@@ -64,12 +64,12 @@ class Retrieval():
# search the database if object exists # search the database if object exists
elif self._check_state(kwargs.values()): elif self._check_state(kwargs.values()):
with session_scope(self._Session) as session: with session_scope(self._Session) as session:
rs = session.query(entity.mb_class).filter_by(**kwargs).first() rs = session.query(table).filter_by(**kwargs).first()
if rs is not None: if rs is not None:
entity.reverse_mapping(rs, *args) entity.reverse_mapping(rs, *args)
return rs return rs
obj = entity.mb_class() obj = table()
# fetch the data from web # fetch the data from web
if entity.web is not None: if entity.web is not None:
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment