From a2937bad710cff7edbc714246211bc86bfefdb38 Mon Sep 17 00:00:00 2001 From: oslomp Date: Thu, 10 Jan 2019 20:34:36 +0100 Subject: [PATCH 01/10] added usercache --- marietje/stats/utils.py | 4 ++-- marietje/stats/views.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/marietje/stats/utils.py b/marietje/stats/utils.py index a4cae4b..9cd17e7 100644 --- a/marietje/stats/utils.py +++ b/marietje/stats/utils.py @@ -23,8 +23,8 @@ def recache_user_stats(): for user in users: new_stats = user_stats(user['id']) cacheloc = 'userstats_{}'.format(user['id']) - caches['default'].delete(cacheloc) - caches['default'].set(cacheloc, new_stats, 48 * 3600) + caches['userstats'].delete(cacheloc) + caches['userstats'].set(cacheloc, new_stats, 48 * 3600) return new_stats def to_days(time): diff --git a/marietje/stats/views.py b/marietje/stats/views.py index 232de2a..c3acf33 100644 --- a/marietje/stats/views.py +++ b/marietje/stats/views.py @@ -20,7 +20,7 @@ def stats(request): return render(request, 'stats/stats.html', data, status=status) def user_stats(request): - stats = caches['default'].get('userstats_{}'.format(request.user.id)) + stats = caches['userstats'].get('userstats_{}'.format(request.user.id)) status = 503 current_age = None current_age_text = None From e719771d1c002b49e249fda8c0f0e713823294f8 Mon Sep 17 00:00:00 2001 From: oslomp Date: Thu, 10 Jan 2019 20:36:24 +0100 Subject: [PATCH 02/10] added cache to settings file --- marietje/marietje/settings.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/marietje/marietje/settings.py b/marietje/marietje/settings.py index fa3a15f..5d83da5 100644 --- a/marietje/marietje/settings.py +++ b/marietje/marietje/settings.py @@ -102,6 +102,11 @@ CACHES = { 'LOCATION': '/var/tmp/MarietjeDjango_cache/song_search', 'OPTIONS': { 'MAX_ENTRIES': 1000 }, }, + 'userstats': { + 'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', + 'LOCATION': '/var/tmp/MarietjeDjango_cache/default', + 'OPTIONS': { 'MAX_ENTRIES': 1500 }, + }, } AUTH_USER_MODEL = 'marietje.User' From 264ec991e5718f6bd4cd8685938062486e219968 Mon Sep 17 00:00:00 2001 From: oslomp Date: Thu, 10 Jan 2019 20:48:52 +0100 Subject: [PATCH 03/10] changed Most played uploads to exclude your requests, and some small fixes --- marietje/stats/templates/stats/user.html | 15 +++++++++++---- marietje/stats/utils.py | 15 +++++++-------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/marietje/stats/templates/stats/user.html b/marietje/stats/templates/stats/user.html index 9f23409..f0bff7d 100644 --- a/marietje/stats/templates/stats/user.html +++ b/marietje/stats/templates/stats/user.html @@ -8,17 +8,22 @@

User Statistics

+ {% if not stats %} +
+ Stats unavailable :( +
+ {% else %} {% if current_age_text %}
{{ current_age_text }} {% endif %} +

Total uploads: {{ stats.total_uploads }}

-

Total requests: {{ stats.total_requests }}

Unique requests: {{ stats.unique_requests }} ({% widthratio stats.unique_requests stats.total_requests 100 %}%)

-

Total requested uploads: {{stats.total_played_uploads}}

Most played songs

+

Total: {{ stats.total_requests }}

Top {{ stats.stats_top_count }}:

@@ -45,7 +50,8 @@
-

Most played uploads

+

Uploads requested by others

+

Total: {{stats.total_played_uploads}}

Top {{ stats.stats_top_count }}:

@@ -87,7 +93,7 @@ - + {% endfor %} @@ -95,5 +101,6 @@ + {% endif %} {% endblock %} \ No newline at end of file diff --git a/marietje/stats/utils.py b/marietje/stats/utils.py index 9cd17e7..79a7542 100644 --- a/marietje/stats/utils.py +++ b/marietje/stats/utils.py @@ -60,7 +60,7 @@ def compute_stats(): 'user__id', 'user__name').annotate( total_requests=Count('id', distinct=True), unique_requests=Count('song__id', distinct=True)).order_by( - '-total_requests')[:settings.STATS_TOP_COUNT] + '-unique_requests')[:settings.STATS_TOP_COUNT] total_unique_requests = PlaylistSong.objects.filter(state=2).exclude( Q(user_id=None) @@ -145,16 +145,15 @@ def user_stats(request): '-total')[:settings.STATS_TOP_COUNT] most_played_uploads = PlaylistSong.objects.filter( - state=2, song_id__in=Song.objects.filter(user__id=request)).exclude( - Q(user_id=None) - | Q(user_id__in=settings.STATS_REQUEST_IGNORE_USER_IDS)).values( - 'pk').values( + state=2, song_id__in=Song.objects.filter(user__id=request)).exclude(Q(user__id=None)|Q(user__id=request)).values( 'song__artist', 'song__title').annotate(total=Count('id')).order_by( '-total', 'song__artist', 'song__title')[:settings.STATS_TOP_COUNT] - - total_played_uploads = most_played_uploads.aggregate(newtotal=Sum('total')) + most_played = list(most_played_uploads) + total_played_uploads = 0 + for x in most_played: + total_played_uploads += x['total'] return { 'last_updated': last_updated, @@ -165,6 +164,6 @@ def user_stats(request): 'most_played_uploaders': list(most_played_uploaders), 'most_played_uploads': list(most_played_uploads), 'stats_top_count': settings.STATS_TOP_COUNT, - 'total_played_uploads': total_played_uploads['newtotal'], + 'total_played_uploads': total_played_uploads, } From 6a86cf1c0a0f1feb901a2994bd35689e837b0bf9 Mon Sep 17 00:00:00 2001 From: oslomp Date: Thu, 10 Jan 2019 20:51:54 +0100 Subject: [PATCH 04/10] split uploads in 2 columns --- marietje/stats/templates/stats/user.html | 9 ++++++--- marietje/stats/utils.py | 8 ++++++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/marietje/stats/templates/stats/user.html b/marietje/stats/templates/stats/user.html index f0bff7d..d6a877c 100644 --- a/marietje/stats/templates/stats/user.html +++ b/marietje/stats/templates/stats/user.html @@ -50,8 +50,9 @@
-

Uploads requested by others

-

Total: {{stats.total_played_uploads}}

+

Uploads requested

+

Total played by you: {{stats.total_played_user_uploads}}

+

Total played by others: {{stats.total_played_uploads}}

Top {{ stats.stats_top_count }}:

{{ forloop.counter }} {{ stat.song__user__name }}{{ stat.total }} ({% widthratio stat.total total_requests 100 %}%){{ stat.total }} ({% widthratio stat.total stats.total_requests 100 %}%)
@@ -60,7 +61,8 @@ - + + @@ -70,6 +72,7 @@ + {% endfor %} diff --git a/marietje/stats/utils.py b/marietje/stats/utils.py index 79a7542..08730c5 100644 --- a/marietje/stats/utils.py +++ b/marietje/stats/utils.py @@ -145,15 +145,18 @@ def user_stats(request): '-total')[:settings.STATS_TOP_COUNT] most_played_uploads = PlaylistSong.objects.filter( - state=2, song_id__in=Song.objects.filter(user__id=request)).exclude(Q(user__id=None)|Q(user__id=request)).values( + state=2, song_id__in=Song.objects.filter(user__id=request)).exclude(user__id=None).values( 'song__artist', - 'song__title').annotate(total=Count('id')).order_by( + 'song__title').annotate(total=Count('id', filter=~Q(user__id=request)), user_total=Count('id', filter=Q(user__id=request))).order_by( '-total', 'song__artist', 'song__title')[:settings.STATS_TOP_COUNT] + most_played = list(most_played_uploads) total_played_uploads = 0 + total_played_user_uploads = 0 for x in most_played: total_played_uploads += x['total'] + total_played_user_uploads += x['user_total'] return { 'last_updated': last_updated, @@ -165,5 +168,6 @@ def user_stats(request): 'most_played_uploads': list(most_played_uploads), 'stats_top_count': settings.STATS_TOP_COUNT, 'total_played_uploads': total_played_uploads, + 'total_played_user_uploads': total_played_user_uploads, } From 274949c51970f19f872cb0e3ef53ceed9c250296 Mon Sep 17 00:00:00 2001 From: oslomp Date: Thu, 10 Jan 2019 20:53:11 +0100 Subject: [PATCH 05/10] added average song length to stats --- marietje/stats/templates/stats/stats.html | 6 ++++-- marietje/stats/utils.py | 9 ++++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/marietje/stats/templates/stats/stats.html b/marietje/stats/templates/stats/stats.html index 4262d7e..ae73f43 100644 --- a/marietje/stats/templates/stats/stats.html +++ b/marietje/stats/templates/stats/stats.html @@ -76,7 +76,8 @@ - + + @@ -84,7 +85,8 @@ - + + {% endfor %} diff --git a/marietje/stats/utils.py b/marietje/stats/utils.py index 08730c5..258acfb 100644 --- a/marietje/stats/utils.py +++ b/marietje/stats/utils.py @@ -27,9 +27,12 @@ def recache_user_stats(): caches['userstats'].set(cacheloc, new_stats, 48 * 3600) return new_stats -def to_days(time): +def time_convert(time): for tr in time: tr['duration'] = str(round(tr['total'] / 86400, 2)) + ' days' + avg_dur_sec = tr['avg_dur']%60 + avg_dur_min = int((tr['avg_dur']-avg_dur_sec)/60) + tr['avg_dur'] = '{}:{}'.format(avg_dur_min, avg_dur_sec) return time def compute_stats(): @@ -84,7 +87,7 @@ def compute_stats(): time_requested = PlaylistSong.objects.filter(state=2).exclude( Q(user_id=None) | Q(user_id__in=settings.STATS_REQUEST_IGNORE_USER_IDS)).values( - 'user__id', 'user__name').annotate(total=Sum('song__duration')).order_by( + 'user__id', 'user__name').annotate(total=Sum('song__duration'), avg_dur=Sum('song__duration')/Count('id')).order_by( '-total')[:settings.STATS_TOP_COUNT] total_time_requested = PlaylistSong.objects.all().filter(state=2).exclude( @@ -108,7 +111,7 @@ def compute_stats(): 'total_unique_requests': total_unique_requests, 'most_played_songs': list(most_played_songs), 'most_played_songs_14_days': list(most_played_songs_14_days), - 'time_requested': to_days(list(time_requested)), + 'time_requested': time_convert(list(time_requested)), 'total_time_requested': str(round(float(total_time_requested['total']) / 86400, 2)) + ' days', 'stats_top_count': settings.STATS_TOP_COUNT, 'most_requested_uploaders': list(most_requested_uploaders), From 1cf43783167fd6b23544cdf0c3bc3208d337d957 Mon Sep 17 00:00:00 2001 From: oslomp Date: Thu, 10 Jan 2019 20:56:37 +0100 Subject: [PATCH 06/10] added descriptions to stats and userstats --- marietje/stats/templates/stats/stats.html | 33 ++++--- marietje/stats/templates/stats/user.html | 15 ++-- marietje/stats/utils.py | 102 +++++++++++++++++----- 3 files changed, 106 insertions(+), 44 deletions(-) diff --git a/marietje/stats/templates/stats/stats.html b/marietje/stats/templates/stats/stats.html index ae73f43..f6ed1f6 100644 --- a/marietje/stats/templates/stats/stats.html +++ b/marietje/stats/templates/stats/stats.html @@ -18,8 +18,8 @@

Uploads

-

Total: {{ stats.total_uploads }}

-

Top {{ stats.stats_top_count }}:

+

In total {{ stats.total_uploads }} songs have been uploaded. + These are the {{ stats.stats_top_count }} people who have uploaded the most.

# Artist Title# RequestsOthersYou
{{ stat.song__artist }} {{ stat.song__title }} {{ stat.total }}{{ stat.user_total }}
# UserDurationDurationAverage
{{ forloop.counter }} {{ stat.user__name }}{{ stat.duration }}{{ stat.duration }}{{stat.avg_dur}}
@@ -43,8 +43,8 @@

Requests

-

Total: {{ stats.total_requests }}

-

Top {{ stats.stats_top_count }}:

+

In total {{ stats.total_requests }} songs have been requested. + These are the {{ stats.stats_top_count }} people who have requested the most songs.

@@ -68,8 +68,9 @@

Time requested

-

Total: {{stats.total_time_requested}}

-

Top {{ stats.stats_top_count }}:

+

In total {{ stats.total_time_requested }} of music have been requested, with an + average song length of {{ stats.total_average }}. + These are the {{ stats.stats_top_count }} people with the longest total time queued

@@ -95,8 +96,9 @@

Unique requests

-

Total: {{stats.total_unique_requests.total}}

-

Top {{ stats.stats_top_count }}:

+

In total {{stats.total_unique_requests.total}} different songs + have been requested. The {{ stats.stats_top_count }} people that have requested the largest number of + different songs are shown below.

@@ -120,7 +122,7 @@

Most played songs

-

Top {{ stats.stats_top_count }}:

+

These are the {{ stats.stats_top_count }} most played songs ever.

@@ -146,22 +148,25 @@

Most played uploaders

-

Top {{ stats.stats_top_count }}:

+

The left column shows the {{ stats.stats_top_count }} people whose songs are requested most often by other people + people. The right column shows how many times that person has queued his own songs.

- + + {% for stat in stats.most_requested_uploaders %} - - + + + {% endfor %} @@ -170,7 +175,7 @@

Most played songs last 14 days

-

Top {{ stats.stats_top_count }}:

+

These songs are played the {{ stats.stats_top_count }} most in the last two weeks.

# User# Songs# Others# Own
{{ forloop.counter }}{{ stat.song__user__name }}{{ stat.total }} ({% widthratio stat.total stats.total_requests 100 %}%){{ stat.name }}{{ stat.total }}{{ stat.own_total}}
diff --git a/marietje/stats/templates/stats/user.html b/marietje/stats/templates/stats/user.html index d6a877c..0b373a5 100644 --- a/marietje/stats/templates/stats/user.html +++ b/marietje/stats/templates/stats/user.html @@ -19,11 +19,11 @@ {% endif %} -

Total uploads: {{ stats.total_uploads }}

-

Unique requests: {{ stats.unique_requests }} ({% widthratio stats.unique_requests stats.total_requests 100 %}%)

Most played songs

-

Total: {{ stats.total_requests }}

+

You have requested {{ stats.unique_requests }} different + songs a total of {{ stats.total_requests }} times. This + means {% widthratio stats.unique_requests stats.total_requests 100 %}% of your requests have been unique.

Top {{ stats.stats_top_count }}:

@@ -51,8 +51,11 @@

Uploads requested

-

Total played by you: {{stats.total_played_user_uploads}}

-

Total played by others: {{stats.total_played_uploads}}

+

You have uploaded a total of {{stats.total_uploads }} songs. The left column + shows how many times these have been requested by other people. The right column shows + how many times you requested your own songs. In total your songs + have been queued {{stats.total_played_uploads }} times by others and + {{stats.total_played_user_uploads }} by yourself.

Top {{ stats.stats_top_count }}:

@@ -81,7 +84,7 @@

Most played uploaders

-

Top {{ stats.stats_top_count }}:

+

The people whose songs you have queued the most are:

diff --git a/marietje/stats/utils.py b/marietje/stats/utils.py index 258acfb..8c94525 100644 --- a/marietje/stats/utils.py +++ b/marietje/stats/utils.py @@ -16,9 +16,11 @@ def recache_stats(): caches['default'].delete('stats') caches['default'].set('stats', new_stats, 2 * 3600) return new_stats - + + def recache_user_stats(): - users = User.objects.exclude(Q(id=None) + users = User.objects.exclude( + Q(id=None) | Q(id__in=settings.STATS_REQUEST_IGNORE_USER_IDS)).values('id') for user in users: new_stats = user_stats(user['id']) @@ -27,13 +29,51 @@ def recache_user_stats(): caches['userstats'].set(cacheloc, new_stats, 48 * 3600) return new_stats + def time_convert(time): - for tr in time: - tr['duration'] = str(round(tr['total'] / 86400, 2)) + ' days' - avg_dur_sec = tr['avg_dur']%60 - avg_dur_min = int((tr['avg_dur']-avg_dur_sec)/60) - tr['avg_dur'] = '{}:{}'.format(avg_dur_min, avg_dur_sec) - return time + try: + for tr in time: + tr['duration'] = str(round(tr['total'] / 86400, 2)) + ' days' + avg_dur_sec = tr['avg_dur'] % 60 + avg_dur_min = int((tr['avg_dur'] - avg_dur_sec) / 60) + tr['avg_dur'] = '{}:{}'.format(avg_dur_min, avg_dur_sec) + return time + except: + avg_dur_sec = round(time % 60) + avg_dur_min = int(round((time - avg_dur_sec) / 60)) + return ('{} minutes and {} seconds'.format(avg_dur_min, avg_dur_sec)) + + +def calculate_uploaders(requests_uploader, most_requested_uploaders): + for x in requests_uploader: + a = len(most_requested_uploaders) + b = 0 + while b <= a: + if b == a: + adding_list_item(most_requested_uploaders, x) + elif x['song__user__id'] == most_requested_uploaders[b]['id']: + if x['song__user__name'] == x['user__name']: + most_requested_uploaders[b]['own_total'] = x['total'] + else: + most_requested_uploaders[b]['total'] += x['total'] + break + b += 1 + return + + +def adding_list_item(list, x): + list.append({ + 'id': x['song__user__id'], + 'name': x['song__user__name'], + 'total': 0, + 'own_total': 0 + }) + if x['song__user__id'] == x['user__id']: + list[-1]['own_total']: x['total'] + else: + list[-1]['_total']: x['total'] + return + def compute_stats(): # We want to grab the time now, because otherwise we would be reporting a minute too late @@ -78,28 +118,41 @@ def compute_stats(): '-total', 'song__artist')[:settings.STATS_TOP_COUNT] most_played_songs_14_days = PlaylistSong.objects.filter( - state=2, played_at__gte=timezone.now() - - timedelta(days=14)).exclude(user_id=None).values( - 'song__artist', - 'song__title').annotate(total=Count('id')).order_by( - '-total', 'song__artist')[:settings.STATS_TOP_COUNT] + state=2, played_at__gte=timezone.now() - timedelta(days=14)).exclude( + user_id=None).values( + 'song__artist', + 'song__title').annotate(total=Count('id')).order_by( + '-total', 'song__artist')[:settings.STATS_TOP_COUNT] time_requested = PlaylistSong.objects.filter(state=2).exclude( Q(user_id=None) | Q(user_id__in=settings.STATS_REQUEST_IGNORE_USER_IDS)).values( - 'user__id', 'user__name').annotate(total=Sum('song__duration'), avg_dur=Sum('song__duration')/Count('id')).order_by( - '-total')[:settings.STATS_TOP_COUNT] + 'user__id', 'user__name').annotate( + total=Sum('song__duration'), + avg_dur=Sum('song__duration') / + Count('id')).order_by('-total')[:settings.STATS_TOP_COUNT] total_time_requested = PlaylistSong.objects.all().filter(state=2).exclude( Q(user_id=None) | Q(user_id__in=settings.STATS_REQUEST_IGNORE_USER_IDS)).aggregate( total=Sum('song__duration')) - most_requested_uploaders = PlaylistSong.objects.filter(state=2).exclude( + total_time_overall = 0 + count = 0 + for x in list(time_requested): + total_time_overall += x['avg_dur'] + count += 1 + total_average = total_time_overall / count + total_average = time_convert(total_average) + + requests_uploader = PlaylistSong.objects.filter(state=2).exclude( Q(user_id=None) | Q(user_id__in=settings.STATS_REQUEST_IGNORE_USER_IDS)).values( - 'song__user__id', 'song__user__name').annotate(total=Count( - 'song__user__id')).order_by('-total')[:settings.STATS_TOP_COUNT] + 'song__user__id', 'song__user__name', 'user__name', + 'user__id').annotate(total=Count('song__user__name')) + + most_requested_uploaders = [] + calculate_uploaders(list(requests_uploader), most_requested_uploaders) return { 'last_updated': last_updated, @@ -117,6 +170,7 @@ def compute_stats(): 'most_requested_uploaders': list(most_requested_uploaders), } + def user_stats(request): last_updated = datetime.now() @@ -148,11 +202,12 @@ def user_stats(request): '-total')[:settings.STATS_TOP_COUNT] most_played_uploads = PlaylistSong.objects.filter( - state=2, song_id__in=Song.objects.filter(user__id=request)).exclude(user__id=None).values( - 'song__artist', - 'song__title').annotate(total=Count('id', filter=~Q(user__id=request)), user_total=Count('id', filter=Q(user__id=request))).order_by( - '-total', 'song__artist', - 'song__title')[:settings.STATS_TOP_COUNT] + state=2, song_id__in=Song.objects.filter(user__id=request)).exclude( + user__id=None).values('song__artist', 'song__title').annotate( + total=Count('id', filter=~Q(user__id=request)), + user_total=Count('id', filter=Q(user__id=request))).order_by( + '-total', 'song__artist', + 'song__title')[:settings.STATS_TOP_COUNT] most_played = list(most_played_uploads) total_played_uploads = 0 @@ -173,4 +228,3 @@ def user_stats(request): 'total_played_uploads': total_played_uploads, 'total_played_user_uploads': total_played_user_uploads, } - From 00d0a32ece4603b71847096410d240bfa07a497c Mon Sep 17 00:00:00 2001 From: oslomp Date: Thu, 10 Jan 2019 21:01:15 +0100 Subject: [PATCH 07/10] general clean-up --- marietje/stats/utils.py | 74 ++++++++++++++++++----------------------- marietje/stats/views.py | 2 -- 2 files changed, 32 insertions(+), 44 deletions(-) diff --git a/marietje/stats/utils.py b/marietje/stats/utils.py index 8c94525..5a276b7 100644 --- a/marietje/stats/utils.py +++ b/marietje/stats/utils.py @@ -1,9 +1,8 @@ -from datetime import datetime, timedelta, time +from datetime import datetime, timedelta from django.core.cache import caches from django.conf import settings from django.db.models import Count, Q, Sum -from django.shortcuts import render from django.utils import timezone from queues.models import PlaylistSong @@ -30,49 +29,34 @@ def recache_user_stats(): return new_stats -def time_convert(time): - try: - for tr in time: - tr['duration'] = str(round(tr['total'] / 86400, 2)) + ' days' - avg_dur_sec = tr['avg_dur'] % 60 - avg_dur_min = int((tr['avg_dur'] - avg_dur_sec) / 60) - tr['avg_dur'] = '{}:{}'.format(avg_dur_min, avg_dur_sec) - return time - except: - avg_dur_sec = round(time % 60) - avg_dur_min = int(round((time - avg_dur_sec) / 60)) - return ('{} minutes and {} seconds'.format(avg_dur_min, avg_dur_sec)) - - -def calculate_uploaders(requests_uploader, most_requested_uploaders): - for x in requests_uploader: +def best_uploaders_list(requests_uploader, most_requested_uploaders): + for requests in requests_uploader: a = len(most_requested_uploaders) b = 0 while b <= a: if b == a: - adding_list_item(most_requested_uploaders, x) - elif x['song__user__id'] == most_requested_uploaders[b]['id']: - if x['song__user__name'] == x['user__name']: - most_requested_uploaders[b]['own_total'] = x['total'] + adding_list_item(most_requested_uploaders, requests) + elif requests['song__user__id'] == most_requested_uploaders[b]['id']: + if requests['song__user__name'] == requests['user__name']: + most_requested_uploaders[b]['own_total'] = requests['total'] else: - most_requested_uploaders[b]['total'] += x['total'] + most_requested_uploaders[b]['total'] += requests['total'] break b += 1 - return -def adding_list_item(list, x): - list.append({ - 'id': x['song__user__id'], - 'name': x['song__user__name'], +def adding_list_item(most_requested_list, requests): + # adds a single item to the best_uploaders list + most_requested_list.append({ + 'id': requests['song__user__id'], + 'name': requests['song__user__name'], 'total': 0, 'own_total': 0 }) - if x['song__user__id'] == x['user__id']: - list[-1]['own_total']: x['total'] + if requests['song__user__id'] == requests['user__id']: + most_requested_list[-1]['own_total']: requests['total'] else: - list[-1]['_total']: x['total'] - return + most_requested_list[-1]['_total']: requests['total'] def compute_stats(): @@ -137,13 +121,10 @@ def compute_stats(): | Q(user_id__in=settings.STATS_REQUEST_IGNORE_USER_IDS)).aggregate( total=Sum('song__duration')) - total_time_overall = 0 - count = 0 - for x in list(time_requested): - total_time_overall += x['avg_dur'] - count += 1 - total_average = total_time_overall / count - total_average = time_convert(total_average) + total_time_overall = sum(x['avg_dur'] for x in time_requested) + total_average = total_time_overall / len(time_requested) + avg_dur_min, avg_dur_sec = divmod(total_average, 60) + total_average = '{} minutes and {} seconds'.format(avg_dur_min, avg_dur_sec) requests_uploader = PlaylistSong.objects.filter(state=2).exclude( Q(user_id=None) @@ -152,7 +133,15 @@ def compute_stats(): 'user__id').annotate(total=Count('song__user__name')) most_requested_uploaders = [] - calculate_uploaders(list(requests_uploader), most_requested_uploaders) + best_uploaders_list(list(requests_uploader), most_requested_uploaders) + + for time in list(time_requested): + # converts total time and average time in respectively days and minutes, seconds + time['duration'] = str(round(time['total'] / 86400, 2)) + ' days' + avg_dur_min, avg_dur_sec = divmod(time['avg_dur'], 60) + if avg_dur_sec < 10: + avg_dur_sec = '0' + str(avg_dur_sec) + time['avg_dur'] = '{}:{}'.format(avg_dur_min, avg_dur_sec) return { 'last_updated': last_updated, @@ -164,10 +153,11 @@ def compute_stats(): 'total_unique_requests': total_unique_requests, 'most_played_songs': list(most_played_songs), 'most_played_songs_14_days': list(most_played_songs_14_days), - 'time_requested': time_convert(list(time_requested)), + 'time_requested': time_requested, 'total_time_requested': str(round(float(total_time_requested['total']) / 86400, 2)) + ' days', 'stats_top_count': settings.STATS_TOP_COUNT, 'most_requested_uploaders': list(most_requested_uploaders), + 'total_average': total_average, } @@ -227,4 +217,4 @@ def user_stats(request): 'stats_top_count': settings.STATS_TOP_COUNT, 'total_played_uploads': total_played_uploads, 'total_played_user_uploads': total_played_user_uploads, - } + } \ No newline at end of file diff --git a/marietje/stats/views.py b/marietje/stats/views.py index c3acf33..ad60a29 100644 --- a/marietje/stats/views.py +++ b/marietje/stats/views.py @@ -2,8 +2,6 @@ from datetime import datetime, timedelta from django.core.cache import caches from django.shortcuts import render -from stats.utils import user_stats -from django.http import JsonResponse, HttpResponseForbidden def stats(request): From 12ad8135bd7766eb58946f49337917558552c478 Mon Sep 17 00:00:00 2001 From: oslomp Date: Wed, 16 Jan 2019 21:49:55 +0100 Subject: [PATCH 08/10] fix query not being listed --- marietje/stats/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/marietje/stats/utils.py b/marietje/stats/utils.py index 5a276b7..8038090 100644 --- a/marietje/stats/utils.py +++ b/marietje/stats/utils.py @@ -121,7 +121,7 @@ def compute_stats(): | Q(user_id__in=settings.STATS_REQUEST_IGNORE_USER_IDS)).aggregate( total=Sum('song__duration')) - total_time_overall = sum(x['avg_dur'] for x in time_requested) + total_time_overall = sum(x['avg_dur'] for x in list(time_requested)) total_average = total_time_overall / len(time_requested) avg_dur_min, avg_dur_sec = divmod(total_average, 60) total_average = '{} minutes and {} seconds'.format(avg_dur_min, avg_dur_sec) From 1b4108c5d04a62025a6dd2a5766bcd60bbe936a1 Mon Sep 17 00:00:00 2001 From: oslomp Date: Wed, 30 Jan 2019 16:47:43 +0100 Subject: [PATCH 09/10] fix indentations --- marietje/stats/utils.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/marietje/stats/utils.py b/marietje/stats/utils.py index c24357d..0541b89 100644 --- a/marietje/stats/utils.py +++ b/marietje/stats/utils.py @@ -18,8 +18,9 @@ def recache_stats(): def recache_user_stats(): - users = User.objects.exclude(Q(id=None) - | Q(id__in=settings.STATS_REQUEST_IGNORE_USER_IDS)).values('id') + users = User.objects.exclude( + Q(id=None) + | Q(id__in=settings.STATS_REQUEST_IGNORE_USER_IDS)).values('id') for user in users: new_stats = user_stats(user['id']) cacheloc = 'userstats_{}'.format(user['id']) @@ -35,9 +36,11 @@ def best_uploaders_list(requests_uploader, most_requested_uploaders): while b <= a: if b == a: adding_list_item(most_requested_uploaders, requests) - elif requests['song__user__id'] == most_requested_uploaders[b]['id']: + elif requests[ + 'song__user__id'] == most_requested_uploaders[b]['id']: if requests['song__user__name'] == requests['user__name']: - most_requested_uploaders[b]['own_total'] = requests['total'] + most_requested_uploaders[b][ + 'own_total'] = requests['total'] else: most_requested_uploaders[b]['total'] += requests['total'] break @@ -53,9 +56,9 @@ def adding_list_item(most_requested_list, requests): 'own_total': 0 }) if requests['song__user__id'] == requests['user__id']: - most_requested_list[-1]['own_total']: requests['total'] + most_requested_list[-1]['own_total'] = requests['total'] else: - most_requested_list[-1]['_total']: requests['total'] + most_requested_list[-1]['total'] = requests['total'] def compute_stats(): @@ -124,7 +127,8 @@ def compute_stats(): total_time_overall = sum(x['avg_dur'] for x in list(time_requested)) total_average = total_time_overall / len(time_requested) avg_dur_min, avg_dur_sec = divmod(total_average, 60) - total_average = '{} minutes and {} seconds'.format(avg_dur_min, avg_dur_sec) + total_average = '{} minutes and {} seconds'.format( + avg_dur_min, avg_dur_sec) requests_uploader = PlaylistSong.objects.filter(state=2).exclude( Q(user_id=None) @@ -159,7 +163,8 @@ def compute_stats(): 'most_played_songs': list(most_played_songs), 'most_played_songs_14_days': list(most_played_songs_14_days), 'time_requested': time_requested, - 'total_time_requested': str(round(float(total_time_requested['total']) / 86400, 2)) + ' days', + 'total_time_requested': str(round(float( + total_time_requested['total']) / 86400, 2)) + ' days', 'stats_top_count': settings.STATS_TOP_COUNT, 'most_requested_uploaders': list(most_requested_uploaders), 'total_average': total_average, @@ -222,4 +227,4 @@ def user_stats(request): 'stats_top_count': settings.STATS_TOP_COUNT, 'total_played_uploads': total_played_uploads, 'total_played_user_uploads': total_played_user_uploads, - } \ No newline at end of file + } From 6dc36f2092aa1cf1033879567e91219f46e93306 Mon Sep 17 00:00:00 2001 From: oslomp Date: Mon, 4 Feb 2019 19:53:20 +0100 Subject: [PATCH 10/10] pylint fixes --- marietje/marietje/settings.py | 2 +- marietje/stats/utils.py | 52 +++++++++++++++++------------------ 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/marietje/marietje/settings.py b/marietje/marietje/settings.py index 3109ead..4745f32 100644 --- a/marietje/marietje/settings.py +++ b/marietje/marietje/settings.py @@ -105,7 +105,7 @@ CACHES = { 'userstats': { 'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', 'LOCATION': '/var/tmp/MarietjeDjango_cache/default', - 'OPTIONS': { 'MAX_ENTRIES': 1500 }, + 'OPTIONS': {'MAX_ENTRIES': 1500}, }, } diff --git a/marietje/stats/utils.py b/marietje/stats/utils.py index 0541b89..5555ea1 100644 --- a/marietje/stats/utils.py +++ b/marietje/stats/utils.py @@ -64,26 +64,27 @@ def adding_list_item(most_requested_list, requests): def compute_stats(): # We want to grab the time now, because otherwise we would be reporting a minute too late last_updated = datetime.now() + stats = {} - total_uploads = Song.objects.filter(deleted=False).exclude( + stats['total_uploads'] = Song.objects.filter(deleted=False).exclude( user_id=None).count() - upload_stats = Song.objects.filter(deleted=False).exclude( + stats['upload_stats'] = Song.objects.filter(deleted=False).exclude( user_id=None).values( 'user__id', 'user__name').annotate(total=Count('id')).order_by( '-total', 'user__name')[:settings.STATS_TOP_COUNT] - total_requests = PlaylistSong.objects.filter(state=2).exclude( + stats['total_requests'] = PlaylistSong.objects.filter(state=2).exclude( Q(user_id=None) | Q(user_id__in=settings.STATS_REQUEST_IGNORE_USER_IDS)).count() - request_stats = PlaylistSong.objects.filter(state=2).exclude( + stats['request_stats'] = PlaylistSong.objects.filter(state=2).exclude( Q(user_id=None) | Q(user_id__in=settings.STATS_REQUEST_IGNORE_USER_IDS)).values( 'user__id', 'user__name').annotate(total=Count('id')).order_by( '-total', 'user__name')[:settings.STATS_TOP_COUNT] - unique_request_stats = PlaylistSong.objects.filter(state=2).exclude( + stats['unique_request_stats'] = PlaylistSong.objects.filter(state=2).exclude( Q(user_id=None) | Q(user_id__in=settings.STATS_REQUEST_IGNORE_USER_IDS)).values( 'user__id', 'user__name').annotate( @@ -91,27 +92,26 @@ def compute_stats(): unique_requests=Count('song__id', distinct=True)).order_by( '-unique_requests')[:settings.STATS_TOP_COUNT] - total_unique_requests = PlaylistSong.objects.filter(state=2).exclude( + stats['total_unique_requests'] = PlaylistSong.objects.filter(state=2).exclude( Q(user_id=None) | Q(user_id__in=settings.STATS_REQUEST_IGNORE_USER_IDS)).aggregate( total=Count('song__id', distinct=True)) - most_played_songs = PlaylistSong.objects.filter(state=2).exclude( + stats['most_played_songs'] = PlaylistSong.objects.filter(state=2).exclude( Q(user_id=None) | Q(user_id__in=settings.STATS_REQUEST_IGNORE_USER_IDS)).values( 'song__artist', 'song__title').annotate(total=Count('id')).order_by( '-total', 'song__artist')[:settings.STATS_TOP_COUNT] - most_played_songs_14_days = PlaylistSong.objects.filter( - + stats['most_played_songs_14_days'] = PlaylistSong.objects.filter( state=2, played_at__gte=timezone.now() - timedelta(days=14)).exclude(user_id=None).values( 'song__artist', 'song__title').annotate(total=Count('id')).order_by( '-total', 'song__artist')[:settings.STATS_TOP_COUNT] - time_requested = PlaylistSong.objects.filter(state=2).exclude( + stats['time_requested'] = PlaylistSong.objects.filter(state=2).exclude( Q(user_id=None) | Q(user_id__in=settings.STATS_REQUEST_IGNORE_USER_IDS)).values( 'user__id', 'user__name').annotate( @@ -119,16 +119,16 @@ def compute_stats(): avg_dur=Sum('song__duration') / Count('id')).order_by('-total')[:settings.STATS_TOP_COUNT] - total_time_requested = PlaylistSong.objects.all().filter(state=2).exclude( + stats['total_time_requested'] = PlaylistSong.objects.all().filter(state=2).exclude( Q(user_id=None) | Q(user_id__in=settings.STATS_REQUEST_IGNORE_USER_IDS)).aggregate( total=Sum('song__duration')) - total_time_overall = sum(x['avg_dur'] for x in list(time_requested)) - total_average = total_time_overall / len(time_requested) + total_time_overall = sum(x['avg_dur'] for x in list(stats['time_requested'])) + total_average = total_time_overall / len(stats['time_requested']) avg_dur_min, avg_dur_sec = divmod(total_average, 60) total_average = '{} minutes and {} seconds'.format( - avg_dur_min, avg_dur_sec) + avg_dur_min, avg_dur_sec) requests_uploader = PlaylistSong.objects.filter(state=2).exclude( Q(user_id=None) @@ -139,7 +139,7 @@ def compute_stats(): most_requested_uploaders = [] best_uploaders_list(list(requests_uploader), most_requested_uploaders) - for time in list(time_requested): + for time in list(stats['time_requested']): # converts total time and average time in respectively days and minutes, seconds time['duration'] = str(round(time['total'] / 86400, 2)) + ' days' avg_dur_min, avg_dur_sec = divmod(time['avg_dur'], 60) @@ -148,23 +148,23 @@ def compute_stats(): time['avg_dur'] = '{}:{}'.format(avg_dur_min, avg_dur_sec) # Convert requested time to days - time_requested = list(time_requested) + time_requested = list(stats['time_requested']) for tr in time_requested: tr['duration'] = str(round(tr['total'] / 86400, 2)) + ' days' return { 'last_updated': last_updated, - 'total_uploads': total_uploads, - 'upload_stats': list(upload_stats), - 'total_requests': total_requests, - 'request_stats': list(request_stats), - 'unique_request_stats': list(unique_request_stats), - 'total_unique_requests': total_unique_requests, - 'most_played_songs': list(most_played_songs), - 'most_played_songs_14_days': list(most_played_songs_14_days), - 'time_requested': time_requested, + 'total_uploads': stats['total_uploads'], + 'upload_stats': list(stats['upload_stats']), + 'total_requests': stats['total_requests'], + 'request_stats': list(stats['request_stats']), + 'unique_request_stats': list(stats['unique_request_stats']), + 'total_unique_requests': stats['total_unique_requests'], + 'most_played_songs': list(stats['most_played_songs']), + 'most_played_songs_14_days': list(stats['most_played_songs_14_days']), + 'time_requested': stats['time_requested'], 'total_time_requested': str(round(float( - total_time_requested['total']) / 86400, 2)) + ' days', + stats['total_time_requested']['total']) / 86400, 2)) + ' days', 'stats_top_count': settings.STATS_TOP_COUNT, 'most_requested_uploaders': list(most_requested_uploaders), 'total_average': total_average,