From 11523174905d7b5abea0a852367191e98fe3aedf Mon Sep 17 00:00:00 2001 From: Daan Sprenkels Date: Fri, 3 Nov 2017 18:09:49 +0100 Subject: [PATCH] Add import scripts from SQL db --- .../management/commands/importhistory.py | 59 ++++++++++++++----- .../management/commands/importsongs.py | 48 +++++++++++---- .../management/commands/importusers.py | 50 ++++++++++++---- 3 files changed, 115 insertions(+), 42 deletions(-) diff --git a/marietje/marietje/management/commands/importhistory.py b/marietje/marietje/management/commands/importhistory.py index 34d4af8..e52a5ad 100644 --- a/marietje/marietje/management/commands/importhistory.py +++ b/marietje/marietje/management/commands/importhistory.py @@ -5,30 +5,57 @@ from songs.models import Song from queues.models import PlaylistSong from ...utils import get_first_queue +QUEUEDB_FORMAT = b'RequestID TrackID RequestedBy Played\n' + class Command(BaseCommand): help = 'Import history from a JSON file exported from a previous version of Marietje.' def add_arguments(self, parser): - parser.add_argument('json_file') + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument('--json_file') + group.add_argument('--tsv_file') def handle(self, *args, **options): - filename = options['json_file'] - User = get_user_model() + if options['json_file']: + filename = options['json_file'] + User = get_user_model() - with open(filename) as fp: - for line in fp: - request = json.loads(line) + with open(filename) as fp: + for line in fp: + request = json.loads(line) - # Ignore random queue and requests without media (?). - if request['b'] is None or request['m'] is None: - continue + # Ignore random queue and requests without media (?). + if request['b'] is None or request['m'] is None: + continue + + song = Song.objects.filter(old_id=request['m']['$oid']).first() + user = User.objects.filter(username=request['b']).first() + playlist = get_first_queue().playlist + if song is None or user is None or playlist is None: + continue + playlist_song = PlaylistSong(playlist=playlist, song=song, user=user, state=2, order=0) + playlist_song.save() + + if options['tsv_file']: + filename = options['tsv_file'] + User = get_user_model() + + with open(filename, 'rb') as fp: + line = fp.readline() + assert line == QUEUEDB_FORMAT, "invalid file header: {}".format(line) + for line in fp: + # RequestID TrackID RequestedBy Played + import_req = line.strip(b'\n').split(b'\t') + song = Song.objects.filter(old_id=import_req[1]).first() + user = User.objects.filter(username=import_req[3]).first() + if song is None or user is None or playlist is None: + continue + playlist_song = PlaylistSong( + playlist=get_first_queue().playlist, + song=song, user=user, + state=2, # already played + order=0) + playlist_song.save() - song = Song.objects.filter(old_id=request['m']['$oid']).first() - user = User.objects.filter(username=request['b']).first() - playlist = get_first_queue().playlist - if song is None or user is None or playlist is None: - continue - playlist_song = PlaylistSong(playlist=playlist, song=song, user=user, state=2, order=0) - playlist_song.save() print('Done importing history.') diff --git a/marietje/marietje/management/commands/importsongs.py b/marietje/marietje/management/commands/importsongs.py index 815fa8c..742bc85 100644 --- a/marietje/marietje/management/commands/importsongs.py +++ b/marietje/marietje/management/commands/importsongs.py @@ -3,25 +3,47 @@ from django.core.management.base import BaseCommand from django.contrib.auth import get_user_model from songs.models import Song +TRACKSDB_FORMAT = b'TrackID Artist Title Genre length Filename Filesize UploadedBy UploadedTimestamp Lounge Deleted trackGain trackPeak\n' + class Command(BaseCommand): help = 'Import songs from a JSON file exported from a previous version of Marietje.' def add_arguments(self, parser): - parser.add_argument('json_file') + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument('--json_file') + group.add_argument('--tsv_file') def handle(self, *args, **options): - filename = options['json_file'] - User = get_user_model() + if options['json_file']: + filename = options['json_file'] + User = get_user_model() + + with open(filename) as fp: + for line in fp: + import_song = json.loads(line) + user = User.objects.filter(username=import_song['ub']).first() + song = Song(user=user, artist=import_song['a'].strip(), title=import_song['t'].strip(), duration=import_song['l'], + hash=import_song['k'], old_id=import_song['_id']['$oid']) + song.save() + + if options['tsv_file']: + filename = options['tsv_file'] + User = get_user_model() + + with open(filename, 'rb') as fp: + line = fp.readline() + assert line == TRACKSDB_FORMAT, "invalid file header: {}".format(line) + for line in fp: + import_song = line.strip(b'\n').split(b'\t') + user = User.objects.filter(username=import_song[7]).first() + song = Song(user=user, + artist=import_song[1].strip(), + title=import_song[2].strip(), + duration=float(import_song[4]), + hash=import_song[5].strip(), + deleted=bool(int(import_song[10])), + old_id=import_song[0]) + song.save() - with open(filename) as fp: - for line in fp: - import_song = json.loads(line) - if 'd' in import_song and import_song['d']: - # Skip deleted songs. - continue - user = User.objects.filter(username=import_song['ub']).first() - song = Song(user=user, artist=import_song['a'].strip(), title=import_song['t'].strip(), duration=import_song['l'], - hash=import_song['k'], old_id=import_song['_id']['$oid']) - song.save() print('Done importing songs.') diff --git a/marietje/marietje/management/commands/importusers.py b/marietje/marietje/management/commands/importusers.py index 1b5da16..3bbc956 100644 --- a/marietje/marietje/management/commands/importusers.py +++ b/marietje/marietje/management/commands/importusers.py @@ -3,25 +3,49 @@ from django.core.management.base import BaseCommand from django.contrib.auth import get_user_model from ...utils import get_first_queue +USERDB_FORMAT = b'UserID username Fullname password Level Studie\n' + class Command(BaseCommand): help = 'Import users from a JSON file exported from a previous version of Marietje.' def add_arguments(self, parser): - parser.add_argument('json_file') + group = parser.add_mutually_exclusive_group() + group.add_argument('--json_file') + group.add_argument('--tsv_file') def handle(self, *args, **options): - filename = options['json_file'] - User = get_user_model() + if options['json_file']: + filename = options['json_file'] + User = get_user_model() + + with open(filename) as fp: + for line in fp: + import_user = json.loads(line) + user = User() + user.username = import_user['_id'].lower().strip() + user.name = import_user['n'].strip() + user.email = user.username + '@science.ru.nl' + user.password = 'md5$$' + import_user['p'] + user.queue = get_first_queue() + user.save() + + if options['tsv_file']: + filename = options['tsv_file'] + User = get_user_model() + + with open(filename, 'rb') as fp: + line = fp.readline() + assert line == USERDB_FORMAT, 'bad 1st line: {}'.format(line) + for line in fp: + import_user = line.strip(b'\n').split(b'\t') + username = import_user[1].decode('utf-8', errors='replace').lower().strip() + user, created = User.objects.get_or_create(username=username) + user.name = import_user[2].decode('utf-8', errors='replace').strip() + user.email = user.username + '@science.ru.nl' + user.password = 'md5$$' + import_user[3].decode('utf-8', errors='strict') + user.queue = get_first_queue() + user.study = import_user[5].decode('utf-8', errors='replace').strip() + user.save() - with open(filename) as fp: - for line in fp: - import_user = json.loads(line) - user = User() - user.username = import_user['_id'].lower().strip() - user.name = import_user['n'].strip() - user.email = user.username + '@science.ru.nl' - user.password = 'md5$$' + import_user['p'] - user.queue = get_first_queue() - user.save() print('Done importing users.')