Add import scripts from SQL db

marietje/marietje/management/commands/importhistory.py

@@ -5,30 +5,57 @@ from songs.models import Song
 from queues.models import PlaylistSong
 from ...utils import get_first_queue
 
+QUEUEDB_FORMAT = b'RequestID	TrackID	RequestedBy	Played\n'
+
 
 class Command(BaseCommand):
     help = 'Import history from a JSON file exported from a previous version of Marietje.'
 
     def add_arguments(self, parser):
-        parser.add_argument('json_file')
+        group = parser.add_mutually_exclusive_group(required=True)
+        group.add_argument('--json_file')
+        group.add_argument('--tsv_file')
 
     def handle(self, *args, **options):
-        filename = options['json_file']
+        if options['json_file']:
-        User = get_user_model()
+            filename = options['json_file']
+            User = get_user_model()
 
-        with open(filename) as fp:
+            with open(filename) as fp:
-            for line in fp:
+                for line in fp:
-                request = json.loads(line)
+                    request = json.loads(line)
 
-                # Ignore random queue and requests without media (?).
+                    # Ignore random queue and requests without media (?).
-                if request['b'] is None or request['m'] is None:
+                    if request['b'] is None or request['m'] is None:
-                    continue
+                        continue
+
+                    song = Song.objects.filter(old_id=request['m']['$oid']).first()
+                    user = User.objects.filter(username=request['b']).first()
+                    playlist = get_first_queue().playlist
+                    if song is None or user is None or playlist is None:
+                        continue
+                    playlist_song = PlaylistSong(playlist=playlist, song=song, user=user, state=2, order=0)
+                    playlist_song.save()
+
+        if options['tsv_file']:
+            filename = options['tsv_file']
+            User = get_user_model()
+
+            with open(filename, 'rb') as fp:
+                line = fp.readline()
+                assert line == QUEUEDB_FORMAT, "invalid file header: {}".format(line)
+                for line in fp:
+                    # RequestID	TrackID	RequestedBy	Played
+                    import_req = line.strip(b'\n').split(b'\t')
+                    song = Song.objects.filter(old_id=import_req[1]).first()
+                    user = User.objects.filter(username=import_req[3]).first()
+                    if song is None or user is None or playlist is None:
+                        continue
+                    playlist_song = PlaylistSong(
+                        playlist=get_first_queue().playlist,
+                        song=song, user=user,
+                        state=2, # already played
+                        order=0)
+                    playlist_song.save()
 
-                song = Song.objects.filter(old_id=request['m']['$oid']).first()
-                user = User.objects.filter(username=request['b']).first()
-                playlist = get_first_queue().playlist
-                if song is None or user is None or playlist is None:
-                    continue
-                playlist_song = PlaylistSong(playlist=playlist, song=song, user=user, state=2, order=0)
-                playlist_song.save()
         print('Done importing history.')

marietje/marietje/management/commands/importsongs.py

@@ -3,25 +3,47 @@ from django.core.management.base import BaseCommand
 from django.contrib.auth import get_user_model
 from songs.models import Song
 
+TRACKSDB_FORMAT = b'TrackID	Artist	Title	Genre	length	Filename	Filesize	UploadedBy	UploadedTimestamp	Lounge	Deleted	trackGain	trackPeak\n'
+
 
 class Command(BaseCommand):
     help = 'Import songs from a JSON file exported from a previous version of Marietje.'
 
     def add_arguments(self, parser):
-        parser.add_argument('json_file')
+        group = parser.add_mutually_exclusive_group(required=True)
+        group.add_argument('--json_file')
+        group.add_argument('--tsv_file')
 
     def handle(self, *args, **options):
-        filename = options['json_file']
+        if options['json_file']:
-        User = get_user_model()
+            filename = options['json_file']
+            User = get_user_model()
+
+            with open(filename) as fp:
+                for line in fp:
+                    import_song = json.loads(line)
+                    user = User.objects.filter(username=import_song['ub']).first()
+                    song = Song(user=user, artist=import_song['a'].strip(), title=import_song['t'].strip(), duration=import_song['l'],
+                                hash=import_song['k'], old_id=import_song['_id']['$oid'])
+                    song.save()
+
+        if options['tsv_file']:
+            filename = options['tsv_file']
+            User = get_user_model()
+
+            with open(filename, 'rb') as fp:
+                line = fp.readline()
+                assert line == TRACKSDB_FORMAT, "invalid file header: {}".format(line)
+                for line in fp:
+                    import_song = line.strip(b'\n').split(b'\t')
+                    user = User.objects.filter(username=import_song[7]).first()
+                    song = Song(user=user,
+                                artist=import_song[1].strip(),
+                                title=import_song[2].strip(),
+                                duration=float(import_song[4]),
+                                hash=import_song[5].strip(),
+                                deleted=bool(int(import_song[10])),
+                                old_id=import_song[0])
+                    song.save()
 
-        with open(filename) as fp:
-            for line in fp:
-                import_song = json.loads(line)
-                if 'd' in import_song and import_song['d']:
-                    # Skip deleted songs.
-                    continue
-                user = User.objects.filter(username=import_song['ub']).first()
-                song = Song(user=user, artist=import_song['a'].strip(), title=import_song['t'].strip(), duration=import_song['l'],
-                            hash=import_song['k'], old_id=import_song['_id']['$oid'])
-                song.save()
         print('Done importing songs.')

marietje/marietje/management/commands/importusers.py

@@ -3,25 +3,49 @@ from django.core.management.base import BaseCommand
 from django.contrib.auth import get_user_model
 from ...utils import get_first_queue
 
+USERDB_FORMAT = b'UserID	username	Fullname	password	Level	Studie\n'
+
 
 class Command(BaseCommand):
     help = 'Import users from a JSON file exported from a previous version of Marietje.'
 
     def add_arguments(self, parser):
-        parser.add_argument('json_file')
+        group = parser.add_mutually_exclusive_group()
+        group.add_argument('--json_file')
+        group.add_argument('--tsv_file')
 
     def handle(self, *args, **options):
-        filename = options['json_file']
+        if options['json_file']:
-        User = get_user_model()
+            filename = options['json_file']
+            User = get_user_model()
+
+            with open(filename) as fp:
+                for line in fp:
+                    import_user = json.loads(line)
+                    user = User()
+                    user.username = import_user['_id'].lower().strip()
+                    user.name = import_user['n'].strip()
+                    user.email = user.username + '@science.ru.nl'
+                    user.password = 'md5$$' + import_user['p']
+                    user.queue = get_first_queue()
+                    user.save()
+
+        if options['tsv_file']:
+            filename = options['tsv_file']
+            User = get_user_model()
+
+            with open(filename, 'rb') as fp:
+                line = fp.readline()
+                assert line == USERDB_FORMAT, 'bad 1st line: {}'.format(line)
+                for line in fp:
+                    import_user = line.strip(b'\n').split(b'\t')
+                    username = import_user[1].decode('utf-8', errors='replace').lower().strip()
+                    user, created = User.objects.get_or_create(username=username)
+                    user.name = import_user[2].decode('utf-8', errors='replace').strip()
+                    user.email = user.username + '@science.ru.nl'
+                    user.password = 'md5$$' + import_user[3].decode('utf-8', errors='strict')
+                    user.queue = get_first_queue()
+                    user.study = import_user[5].decode('utf-8', errors='replace').strip()
+                    user.save()
 
-        with open(filename) as fp:
-            for line in fp:
-                import_user = json.loads(line)
-                user = User()
-                user.username = import_user['_id'].lower().strip()
-                user.name = import_user['n'].strip()
-                user.email = user.username + '@science.ru.nl'
-                user.password = 'md5$$' + import_user['p']
-                user.queue = get_first_queue()
-                user.save()
         print('Done importing users.')