Commit b9e887c1 authored by Gerion Entrup's avatar Gerion Entrup
Browse files

try to implement threading, doesn't work with sqlite atm

parent cec95af1
......@@ -5,7 +5,7 @@ import sys
import os.path
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy.orm import sessionmaker, scoped_session
from collector import Collector
from translator import Translator
......@@ -19,8 +19,9 @@ class Main:
#engine = create_engine('sqlite://')#, echo=True)
patch_model_schemas(NO_SCHEMAS)
Base.metadata.create_all(engine)
Session = sessionmaker(bind=engine)
self.session = Session()
session_factory = sessionmaker(bind=engine)
Session = scoped_session(session_factory)
self.session = Session
def main(self, args):
......@@ -36,14 +37,14 @@ class Main:
self.init_database(arg.database)
collector = Collector(self.session)
collector.walk(arg.sourcedir)
collector = Collector(self.session, arg.sourcedir)
collector.start()
translator = Translator(self.session)
translator.mount(arg.mountpoint)
translator = Translator(self.session, arg.mountpoint)
translator.start()
self.session.commit()
self.session.close()
collector.join()
translator.join()
if __name__ == "__main__":
Main().main(sys.argv)
......@@ -2,6 +2,7 @@ import os, sys, pprint
import mutagen
import musicbrainzngs
import itertools
import threading
from sqlalchemy import and_
from sqlalchemy.orm import Session
......@@ -10,17 +11,19 @@ from mbdata.models import Recording, ArtistCredit, ArtistCreditName, Artist, Rel
from utils import pairwise
from fetcher import Fetcher
class Collector:
class Collector(threading.Thread):
"""
Collects Tags and write them to the database.
"""
def __init__(self, session):
self.session = session
def __init__(self, session_fac, sourcedir):
super().__init__(target=self)
self._session = session_fac()
self.fetcher = Fetcher()
self._path = sourcedir
def walk(self, path):
for root, dirs, files in os.walk(path):
def run(self):
for root, dirs, files in os.walk(self._path):
for file in files:
mbid = None
filepath = os.path.join(root, file)
......@@ -37,6 +40,8 @@ class Collector:
mbid = self.get_mbid(mut)
if mbid is not None:
self.fetch_recording(mbid, os.path.abspath(filepath))
self._session.commit()
self._session.close()
def get_mbid(self, mut):
for key in mut:
......@@ -58,16 +63,16 @@ class Collector:
return None
def create_artist_credit(self, acresult, acphrase):
ac = self.session.query(ArtistCredit).filter_by(name=acphrase).first()
ac = self._session.query(ArtistCredit).filter_by(name=acphrase).first()
if ac is None:
ac = ArtistCredit()
ac.name = acphrase
acns = self.create_artist_credit_name(acresult, ac)
ac.artist_count = len(acns)
self.session.add(ac)
self._session.add(ac)
for acn in acns:
self.session.add(acn)
self._session.add(acn)
return ac
def create_artist_credit_name(self, acresult, artistcredit):
......@@ -93,24 +98,24 @@ class Collector:
medium.format = self.create_medium_format(med['format'])
mediums.append(medium)
self.session.add(medium)
self._session.add(medium)
return mediums
def create_medium_format(self, name):
mediumformat = self.session.query(MediumFormat).filter_by(name=name).first()
mediumformat = self._session.query(MediumFormat).filter_by(name=name).first()
if mediumformat is None:
mediumformat = MediumFormat()
mediumformat.name = name
self.session.add(mediumformat)
self._session.add(mediumformat)
return mediumformat
def create_track(self, trackdata, release, recording, mediumdata):
track = self.session.query(Recording).filter_by(gid=trackdata['id']).first()
track = self._session.query(Recording).filter_by(gid=trackdata['id']).first()
if track is None:
track = Track()
track.gid = trackdata['id']
track.recording = recording
track.medium = self.session.query(Medium).filter(and_(Medium.position == mediumdata['position'],
track.medium = self._session.query(Medium).filter(and_(Medium.position == mediumdata['position'],
Medium.release == release)).one()
track.artist_credit = recording.artist_credit
track.position = trackdata['position']
......@@ -119,9 +124,8 @@ class Collector:
track.length = trackdata['length']
def fetch_recording(self, mbid, path):
recording = self.session.query(Recording).filter_by(gid=mbid).first()
recording = self._session.query(Recording).filter_by(gid=mbid).first()
if recording is None:
result = self.fetcher.get_table_by_id(mbid, 'recording')
......@@ -134,7 +138,7 @@ class Collector:
recording.length = result['length']
recording.artist_credit = self.create_artist_credit(result['artist-credit'], result['artist-credit-phrase'])
self.session.add(recording)
self._session.add(recording)
#extended mapping
for releasedata in result['release-list']:
......@@ -148,11 +152,12 @@ class Collector:
for track in medium['track-list']:
if track['recording']['id'] == mbid:
self.create_track(track, release, recording, medium)
self._session.commit()
return recording
def fetch_release_group(self, mbid):
rg = self.session.query(ReleaseGroup).filter_by(gid=mbid).first()
rg = self._session.query(ReleaseGroup).filter_by(gid=mbid).first()
if rg is None:
result = self.fetcher.get_table_by_id(mbid, 'release-group')
......@@ -162,12 +167,12 @@ class Collector:
rg.name = result['title']
rg.artist_credit = self.create_artist_credit(result['artist-credit'], result['artist-credit-phrase'])
self.session.add(rg)
self._session.add(rg)
return rg
def fetch_artist(self, mbid):
artist = self.session.query(Artist).filter_by(gid=mbid).first()
artist = self._session.query(Artist).filter_by(gid=mbid).first()
if artist is None:
result = self.fetcher.get_table_by_id(mbid, 'artist')
......@@ -177,11 +182,11 @@ class Collector:
artist.sort_name = result['sort-name']
#add to db
self.session.add(artist)
self._session.add(artist)
return artist
def fetch_release(self, mbid):
release = self.session.query(Artist).filter_by(gid=mbid).first()
release = self._session.query(Artist).filter_by(gid=mbid).first()
if release is None:
result = self.fetcher.get_table_by_id(mbid, 'release')
......@@ -192,7 +197,7 @@ class Collector:
release.artist_credit = self.create_artist_credit(result['artist-credit'], result['artist-credit-phrase'])
release.release_group = self.fetch_release_group(result['release-group']['id'])
self.session.add(release)
self._session.add(release)
#extended mapping
self.create_medium(result['medium-list'], release)
......
import hlfuse, os
import threading
from mbdata.models import Recording
class Translator():
class Translator(threading.Thread):
"""
Manages the mapping between filesystem and database.
"""
def __init__(self, session):
self._root = self._create_fs(session)
def __init__(self, session_fac, mountpoint):
super().__init__(target=self)
self._session = session_fac()
self._root = self._create_fs(self._session)
self._mountpoint = mountpoint
def _create_fs(self, session):
root = hlfuse.FuseDir(0o555, root=True)
......@@ -18,8 +22,14 @@ class Translator():
return root
def run(self):
self.mount(self._mountpoint)
def mount(self, mountpoint):
hlfuse.init(mountpoint, self._root)
#TODO fix
self._session.commit()
self._session.close()
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment