From a7fe50b61d9dc891c82f3e0ff0da02499fdd9bf3 Mon Sep 17 00:00:00 2001
From: Daan Sprenkels <hello@dsprenkels.com>
Date: Fri, 3 Nov 2017 18:31:34 +0100
Subject: [PATCH] Tweak decoding in import scripts

---
 .../marietje/management/commands/importhistory.py     |  6 +++++-
 marietje/marietje/management/commands/importsongs.py  | 11 ++++++++---
 marietje/marietje/management/commands/importusers.py  |  6 +++---
 3 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/marietje/marietje/management/commands/importhistory.py b/marietje/marietje/management/commands/importhistory.py
index e52a5ad..fe68831 100644
--- a/marietje/marietje/management/commands/importhistory.py
+++ b/marietje/marietje/management/commands/importhistory.py
@@ -56,6 +56,10 @@ class Command(BaseCommand):
                         song=song, user=user,
                         state=2, # already played
                         order=0)
-                    playlist_song.save()
+                    try:
+                        playlist_song.save()
+                    except OperationalError as e:
+                        print('Error: {}'.format(e))
+                        continue
 
         print('Done importing history.')
diff --git a/marietje/marietje/management/commands/importsongs.py b/marietje/marietje/management/commands/importsongs.py
index 742bc85..143f098 100644
--- a/marietje/marietje/management/commands/importsongs.py
+++ b/marietje/marietje/management/commands/importsongs.py
@@ -1,6 +1,7 @@
 import json
 from django.core.management.base import BaseCommand
 from django.contrib.auth import get_user_model
+from django.db.utils import OperationalError
 from songs.models import Song
 
 TRACKSDB_FORMAT = b'TrackID	Artist	Title	Genre	length	Filename	Filesize	UploadedBy	UploadedTimestamp	Lounge	Deleted	trackGain	trackPeak\n'
@@ -38,12 +39,16 @@ class Command(BaseCommand):
                     import_song = line.strip(b'\n').split(b'\t')
                     user = User.objects.filter(username=import_song[7]).first()
                     song = Song(user=user,
-                                artist=import_song[1].strip(),
-                                title=import_song[2].strip(),
+                                artist=import_song[1].decode('utf-8', errors='ignore').strip(),
+                                title=import_song[2].decode('utf-8', errors='ignore').strip(),
                                 duration=float(import_song[4]),
                                 hash=import_song[5].strip(),
                                 deleted=bool(int(import_song[10])),
                                 old_id=import_song[0])
-                    song.save()
+                    try:
+                        song.save()
+                    except OperationalError as e:
+                        print('Error: {}'.format(e))
+                        continue
 
         print('Done importing songs.')
diff --git a/marietje/marietje/management/commands/importusers.py b/marietje/marietje/management/commands/importusers.py
index 3bbc956..d61caeb 100644
--- a/marietje/marietje/management/commands/importusers.py
+++ b/marietje/marietje/management/commands/importusers.py
@@ -39,13 +39,13 @@ class Command(BaseCommand):
                 assert line == USERDB_FORMAT, 'bad 1st line: {}'.format(line)
                 for line in fp:
                     import_user = line.strip(b'\n').split(b'\t')
-                    username = import_user[1].decode('utf-8', errors='replace').lower().strip()
+                    username = import_user[1].decode('utf-8', errors='strict').lower().strip()
                     user, created = User.objects.get_or_create(username=username)
-                    user.name = import_user[2].decode('utf-8', errors='replace').strip()
+                    user.name = import_user[2].decode('utf-8', errors='ignore').strip()
                     user.email = user.username + '@science.ru.nl'
                     user.password = 'md5$$' + import_user[3].decode('utf-8', errors='strict')
                     user.queue = get_first_queue()
-                    user.study = import_user[5].decode('utf-8', errors='replace').strip()
+                    user.study = import_user[5].decode('utf-8', errors='ignore').strip()
                     user.save()
 
         print('Done importing users.')