diff --git a/backend/analytics/admin.py b/backend/analytics/admin.py index e78600f..1e2f00e 100644 --- a/backend/analytics/admin.py +++ b/backend/analytics/admin.py @@ -1,247 +1,11 @@ -import json -import time - -import redis -from django.apps import apps -from django.conf import settings -from django.contrib import admin -from django.contrib.humanize.templatetags.humanize import intcomma -from django.core.serializers.json import DjangoJSONEncoder -from django.db import connection -from django.db.models import Count, Q -from django_celery_beat.models import PeriodicTask -from gamedata.models import GameFIOPlayerData, GamePlanet +from django.contrib import admin, messages +from django.db.models import JSONField +from django.shortcuts import redirect +from django_json_widget.widgets import JSONEditorWidget from unfold.admin import ModelAdmin +from unfold.decorators import action -from analytics.models import AppStatistic - - -def dashboard_index(request, context): - - def kpi_data(): - return ( - AppStatistic.objects.order_by('-date') - .values( - 'user_count', - 'users_active_today', - 'users_active_30d', - 'user_count_delta', - 'plan_count', - 'empire_count', - 'cx_count', - 'plan_count_delta', - 'empire_count_delta', - 'cx_count_delta', - ) - .first() - ) - - chart_data = list( - AppStatistic.objects.order_by('-date')[:30].values( - 'date', - 'user_count', - 'users_active_today', - 'user_count_delta', - 'plan_count_delta', - 'empire_count_delta', - 'cx_count_delta', - ) - ) - chart_data.reverse() - - def get_automation_status_data(): - - return { - 'planet': GamePlanet.objects.aggregate( - total_count=Count('pk'), - ok_count=Count('pk', filter=Q(automation_refresh_status='ok')), - retrying_count=Count('pk', filter=Q(automation_refresh_status='retrying')), - failed_count=Count('pk', filter=Q(automation_refresh_status='failed')), - ), - 'fio_userdata': GameFIOPlayerData.objects.aggregate( - total_count=Count('pk'), - ok_count=Count('pk', filter=Q(automation_refresh_status='ok')), - retrying_count=Count('pk', filter=Q(automation_refresh_status='retrying')), - failed_count=Count('pk', filter=Q(automation_refresh_status='failed')), - ), - } - - def get_redis_stats(): - try: - r = redis.from_url(settings.CACHES['default']['LOCATION']) - info = r.info() - - # sse metrics - stats_key = 'stream:active_connections' - # prune stale sesions - r.zremrangebyscore(stats_key, 0, time.time() - 30) - # get active users - sse_users = r.zcard(stats_key) - - hits = info.get('keyspace_hits', 0) - misses = info.get('keyspace_misses', 0) - total_reqs = hits + misses - hit_rate = f'{(hits / total_reqs * 100):.1f}%' if total_reqs > 0 else 'N/A' - - return { - 'usage': info.get('used_memory_human', '0B'), - 'hit_rate': hit_rate, - 'active_stream_users': sse_users, - 'active_connections': info.get('connected_clients', 0), - 'blocked_clients': info.get('blocked_clients', 0), - 'fragmentation': info.get('mem_fragmentation_ratio', 0), - 'status': 'Healthy' if info.get('evicted_keys', 0) == 0 else 'Memory Pressure', - } - except Exception: - return None - - def get_postgres_perf_stats(): - with connection.cursor() as cursor: - cursor.execute(""" - SELECT - (SELECT count(*) FROM pg_stat_activity) as active_conns, - (SELECT sum(xact_commit) FROM pg_stat_database) as total_commits, - (SELECT pg_size_pretty(sum(pg_total_relation_size(quote_ident(schemaname) || '.' - || quote_ident(relname)))) - FROM pg_stat_user_tables) as total_data_size, - (SELECT - round(sum(heap_blks_hit) * 100.0 / NULLIF(sum(heap_blks_hit) + sum(heap_blks_read), 0), 2) - FROM pg_statio_user_tables) as cache_hit_rate, - (SELECT sum(n_dead_tup) FROM pg_stat_user_tables) as dead_tuples - """) - - row = cursor.fetchone() - - return { - 'active_connections': row[0] or 0, - 'total_commits': row[1] or 0, - 'db_size': row[2] or '0B', - 'cache_hit_rate': f'{row[3] or 100}%', - 'dead_tuples': row[4] or 0, - 'needs_vacuum': row[4] > 50000 if row[4] else False, - } - - def get_system_stats(): - count_apps = ['user', 'gamedata', 'planning', 'analytics'] - data = {'models': [], 'total_db_size': '0B', 'total_records': 0} - - with connection.cursor() as cursor: - cursor.execute('SELECT pg_size_pretty(pg_database_size(current_database()))') - data['total_db_size'] = cursor.fetchone()[0] - - for model in apps.get_models(): - if model._meta.app_label in count_apps: - table_name = model._meta.db_table - - cursor.execute( - """ - SELECT - pg_size_pretty(pg_total_relation_size(%s)), - reltuples::bigint - FROM pg_class - WHERE relname = %s - """, - [table_name, table_name], - ) - - result = cursor.fetchone() - size = result[0] if result else '0B' - - estimate = result[1] if result and result[1] > 0 else model.objects.count() - data['total_records'] += estimate - - data['models'].append( - { - 'name': model._meta.verbose_name, - 'model_name': model._meta.model_name, - 'app': model._meta.app_label, - 'count': estimate, - 'size': size, - } - ) - - data['models'] = sorted(data['models'], key=lambda x: x['count'], reverse=True) - return data - - def get_task_data(): - tasks = PeriodicTask.objects.all().values('name', 'enabled', 'last_run_at', 'total_run_count') - - for task in tasks: - if not task['enabled']: - task['status_color'] = '#999999' - task['status_label'] = 'PAUSED' - else: - task['status_color'] = '#28a745' - task['status_label'] = 'RUNNING' - - return tasks - - pg_stats = get_postgres_perf_stats() - redis_stats = get_redis_stats() - system_stats = get_system_stats() - model_rows = [[r['name'], r['app'], intcomma(r['count']), r['size']] for r in system_stats['models']] - task_data = get_task_data() - automation_status_data = get_automation_status_data() - - # combine all datapoints - context.update( - { - 'kpi_data': kpi_data(), - 'model_counts': system_stats, - 'automation_status_data': automation_status_data, - 'task_data': task_data, - 'redis_stats': redis_stats, - 'postgres_stats': pg_stats, - 'chart_data': json.dumps(chart_data, cls=DjangoJSONEncoder), - 'database_table': { - 'rows': [ - ['Active Connections', pg_stats['active_connections']], - ['Database Size', pg_stats['db_size']], - ['Cache Hit-Rate', pg_stats['cache_hit_rate']], - ['Total Commits', intcomma(pg_stats['total_commits'])], - ['Dead Tuples', pg_stats['dead_tuples']], - ['Needs Vacuum', pg_stats['needs_vacuum']], - ], - }, - 'redis_table': { - 'rows': [ - ['Active Connections', redis_stats['active_connections']], - ['Stream Connections', redis_stats['active_stream_users']], - ['Usage', redis_stats['usage']], - ['Hit Rate', redis_stats['hit_rate']], - ['Blocked Clients', redis_stats['blocked_clients']], - ['Fragmentation', redis_stats['fragmentation']], - ['Status', redis_stats['status']], - ], - }, - 'models_table': {'headers': ['Model', 'App', 'Record Count', 'Table Size'], 'rows': model_rows}, - 'task_table': { - 'headers': ['Task Name', 'Status', 'Last Run', 'Total Runs'], - 'rows': [ - [t['name'], t['status_label'], t['last_run_at'], intcomma(t['total_run_count'])] for t in task_data - ], - }, - 'automation_table': { - 'headers': ['Model', 'Total', 'Retrying', 'Failed'], - 'rows': [ - [ - 'Planets', - intcomma(automation_status_data['planet']['total_count']), - intcomma(automation_status_data['planet']['retrying_count']), - intcomma(automation_status_data['planet']['failed_count']), - ], - [ - 'FIO Userdata', - intcomma(automation_status_data['fio_userdata']['total_count']), - intcomma(automation_status_data['fio_userdata']['retrying_count']), - intcomma(automation_status_data['fio_userdata']['failed_count']), - ], - ], - }, - } - ) - - return context +from analytics.models import AnalyticsPlanAggregate, AppStatistic @admin.register(AppStatistic) @@ -257,3 +21,34 @@ class AppStatisticAdmin(ModelAdmin): ] search_fields = ['date'] readonly_fields = ['last_updated'] + + +@admin.register(AnalyticsPlanAggregate) +class AnalyticsPlanAggregateAdmin(ModelAdmin): + list_display = ['planet_natural_id', 'total_plans_analyzed', 'last_updated'] + search_fields = ['planet_natural_id'] + ordering = ['-last_updated'] + + formfield_overrides = { + JSONField: {'widget': JSONEditorWidget}, + } + + actions_list = ['action_aggregate_all'] + + @action(description='Run Aggregator', url_path='analytics-aggregate-all') + def action_aggregate_all(self, request): + + from analytics.services.PlanInsightAggregatorService import PlanInsightAggregatorService + + try: + aggregator = PlanInsightAggregatorService() + processed, deleted = aggregator.aggregate_all_plans() + self.message_user( + request, + f'Aggregates updated! Processed: {processed}, deleted {deleted}.', + messages.SUCCESS, + ) + except Exception: + self.message_user(request, 'Error processing aggregates.', messages.ERROR) + + return redirect('../') diff --git a/backend/analytics/api/serializer.py b/backend/analytics/api/serializer.py new file mode 100644 index 0000000..2ae8289 --- /dev/null +++ b/backend/analytics/api/serializer.py @@ -0,0 +1,13 @@ +from analytics.models import AnalyticsPlanAggregate +from rest_framework import serializers + + +class AnalyticsPlanAggregateSerializer(serializers.ModelSerializer): + status = serializers.SerializerMethodField() + + class Meta: + model = AnalyticsPlanAggregate + fields = ['status', 'planet_natural_id', 'total_plans_analyzed', 'insights_data', 'last_updated'] + + def get_status(self, obj) -> str: + return 'success' diff --git a/backend/analytics/api/urls.py b/backend/analytics/api/urls.py new file mode 100644 index 0000000..39758f4 --- /dev/null +++ b/backend/analytics/api/urls.py @@ -0,0 +1,11 @@ +from analytics.api.viewsets import AnalyticsPlanAggregateViewSet +from django.urls import path + +app_name = 'analytics' +urlpatterns = [ + path( + 'planet_insights//', + AnalyticsPlanAggregateViewSet.as_view({'get': 'retrieve'}), + name='planet-insight-detail', + ), +] diff --git a/backend/analytics/api/viewsets.py b/backend/analytics/api/viewsets.py new file mode 100644 index 0000000..9b64891 --- /dev/null +++ b/backend/analytics/api/viewsets.py @@ -0,0 +1,38 @@ +from analytics.api.serializer import AnalyticsPlanAggregateSerializer +from analytics.models import AnalyticsPlanAggregate +from django.http import Http404 +from drf_spectacular.utils import extend_schema +from gamedata.models.game_planet import GamePlanet +from rest_framework import status, viewsets +from rest_framework.exceptions import NotFound +from rest_framework.response import Response + + +class AnalyticsPlanAggregateViewSet(viewsets.ReadOnlyModelViewSet): + queryset = AnalyticsPlanAggregate.objects.all() + lookup_field = 'planet_natural_id' + serializer_class = AnalyticsPlanAggregateSerializer + + @extend_schema(auth=[], summary='Fetch planning insights for planet') + def retrieve(self, request, *args, **kwargs): + planet_id = kwargs.get('planet_natural_id') + + if not GamePlanet.objects.filter(planet_natural_id=planet_id).exists(): + return Response({'detail': 'Planet not found.'}, status=status.HTTP_404_NOT_FOUND) + + try: + # try to get aggregate + instance = self.get_object() + serializer = self.get_serializer(instance) + return Response(serializer.data) + except (AnalyticsPlanAggregate.DoesNotExist, Http404, NotFound): + # return a 200 OK, but without any data + return Response( + { + 'status': 'below_threshold', + 'planet_natural_id': planet_id, + 'total_plans_analyzed': 0, + 'aggregated_data': None, + }, + status=status.HTTP_200_OK, + ) diff --git a/backend/analytics/migrations/0004_analyticsplanaggregate.py b/backend/analytics/migrations/0004_analyticsplanaggregate.py new file mode 100644 index 0000000..5ae4890 --- /dev/null +++ b/backend/analytics/migrations/0004_analyticsplanaggregate.py @@ -0,0 +1,29 @@ +# Generated by Django 6.0.4 on 2026-04-15 09:03 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('analytics', '0003_appstatistic_cx_count_delta_and_more'), + ] + + operations = [ + migrations.CreateModel( + name='AnalyticsPlanAggregate', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('planet_natural_id', models.CharField(db_index=True, max_length=7, unique=True)), + ('total_plans_analyzed', models.PositiveIntegerField(default=0)), + ('insights_data', models.JSONField(default=dict)), + ('last_updated', models.DateTimeField(auto_now=True)), + ], + options={ + 'verbose_name': 'Plan Aggregate', + 'verbose_name_plural': 'Plan Aggregates', + 'db_table': 'prunplanner_statistics_plan_aggregate', + 'ordering': ['-last_updated'], + }, + ), + ] diff --git a/backend/analytics/models/__init__.py b/backend/analytics/models/__init__.py index 07015a3..e5cc281 100644 --- a/backend/analytics/models/__init__.py +++ b/backend/analytics/models/__init__.py @@ -1 +1,2 @@ from .app_statistics import AppStatistic +from .plan_aggregates import AnalyticsPlanAggregate diff --git a/backend/analytics/models/plan_aggregates.py b/backend/analytics/models/plan_aggregates.py new file mode 100644 index 0000000..89db71f --- /dev/null +++ b/backend/analytics/models/plan_aggregates.py @@ -0,0 +1,19 @@ +from django.db import models + + +class AnalyticsPlanAggregate(models.Model): + planet_natural_id = models.CharField(max_length=7, unique=True, db_index=True) + total_plans_analyzed = models.PositiveIntegerField(default=0) + + insights_data = models.JSONField(default=dict) + + last_updated = models.DateTimeField(auto_now=True) + + class Meta: + db_table = 'prunplanner_statistics_plan_aggregate' + verbose_name = 'Plan Aggregate' + verbose_name_plural = 'Plan Aggregates' + ordering = ['-last_updated'] + + def __str__(self) -> str: + return f'Insights for {self.planet_natural_id} (n={self.total_plans_analyzed})' diff --git a/backend/analytics/services/PlanInsightAggregatorService.py b/backend/analytics/services/PlanInsightAggregatorService.py new file mode 100644 index 0000000..61eb9e0 --- /dev/null +++ b/backend/analytics/services/PlanInsightAggregatorService.py @@ -0,0 +1,163 @@ +from collections import Counter, defaultdict +from datetime import timedelta + +from analytics.models import AnalyticsPlanAggregate +from django.db.models import Count, Value +from django.db.models.functions import Concat +from django.utils import timezone +from gamedata.models import GameBuilding, GameRecipe +from planning.models import PlanningPlan + + +class PlanInsightAggregatorService: + MIN_PLANS_THRESHOLD = 15 + USER_ACTIVITY_DAYS = 90 + PLAN_STALENESS_DAYS = 180 + BUILDING_USAGE_CUTOFF = 20 + RECIPE_USAGE_CUTOFF = 10 + + def __init__(self): + # cache valid recipes for validation + self.valid_recipes = set( + GameRecipe.objects.annotate(full_id=Concat('building_ticker', Value('#'), 'recipe_name')).values_list( + 'full_id', flat=True + ) + ) + self.valid_buildings = set(GameBuilding.objects.values_list('building_ticker', flat=True).distinct()) + + def aggregate_all_plans(self) -> tuple[int, int]: + login_cutoff = timezone.now() - timedelta(days=self.USER_ACTIVITY_DAYS) + modified_cutoff = timezone.now() - timedelta(days=self.PLAN_STALENESS_DAYS) + + active_planets = ( + PlanningPlan.objects.filter(user__last_login__gte=login_cutoff, modified_at__gte=modified_cutoff) + .values('planet_natural_id') + .annotate(total=Count('uuid')) + .filter(total__gte=self.MIN_PLANS_THRESHOLD) + .values_list('planet_natural_id', flat=True) + ) + + processed_ids = [] + + # process all active plans + for planet_id in active_planets: + result_id = self.process_planet(planet_id) + if result_id: + processed_ids.append(result_id) + + # clean up insights for planets not processed, i.e. stale ones + deleted_count, _ = AnalyticsPlanAggregate.objects.exclude(planet_natural_id__in=processed_ids).delete() + + return len(processed_ids), deleted_count + + def process_planet(self, planet_natural_id: str) -> str | None: + + login_cutoff = timezone.now() - timedelta(days=self.USER_ACTIVITY_DAYS) + modified_cutoff = timezone.now() - timedelta(days=self.PLAN_STALENESS_DAYS) + + plans = PlanningPlan.objects.filter( + planet_natural_id=planet_natural_id, user__last_login__gte=login_cutoff, modified_at__gte=modified_cutoff + ).iterator(chunk_size=500) + + total_valid_plans = 0 + experts_total = Counter() + building_presence = Counter() + recipe_distribution = defaultdict(Counter) + + for plan in plans: + total_valid_plans += 1 + data = plan.plan_data + + # experts + for exp in data.get('experts', []): + if exp.get('amount', 0) > 0: + experts_total[exp['type']] += exp['amount'] + + # buildings and recipes + plan_buildings = data.get('buildings', []) + prod_count = 0 + seen_in_this_plan = set() + + for b in plan_buildings: + b_code = b.get('name') + + # invalid / not-existing-anymore building + if b_code not in self.valid_buildings: + continue + + prod_count += b.get('amount', 0) + seen_in_this_plan.add(b_code) + + for r in b.get('active_recipes', []): + r_id = r.get('recipeid') + + # recipe must still be valid / existing + if r_id in self.valid_recipes: + recipe_distribution[b_code][r_id] += 1 + + for b_code in seen_in_this_plan: + building_presence[b_code] += 1 + + if total_valid_plans < self.MIN_PLANS_THRESHOLD: + return None + + insights_payload = { + 'expert_distribution': self._get_expert_distribution(experts_total), + 'recipe_distribution': self._get_recipe_distribution(recipe_distribution), + 'building_distribution': self._get_building_distribution(building_presence, total_valid_plans), + } + + AnalyticsPlanAggregate.objects.update_or_create( + planet_natural_id=planet_natural_id, + defaults={'insights_data': insights_payload, 'total_plans_analyzed': total_valid_plans}, + ) + + return planet_natural_id + + def _get_expert_distribution(self, expert_totals: Counter) -> list: + + total_points = sum(expert_totals.values()) + + if total_points == 0: + return [] + + expert_split = [] + + for expert_type, amount in expert_totals.items(): + percentage = (amount / total_points) * 100 + + if percentage > 0: + expert_split.append({'type': expert_type.replace('_', ' ').title(), 'percentage': round(percentage, 2)}) + + return sorted(expert_split, key=lambda x: x['percentage'], reverse=True) + + def _get_building_distribution(self, building_presence: Counter, total_plans: int) -> list: + + builds = [] + for ticker, count in building_presence.items(): + percentage = (count / total_plans) * 100 + if percentage >= self.BUILDING_USAGE_CUTOFF: + builds.append({'ticker': ticker, 'percentage': round(percentage, 2)}) + + return sorted(builds, key=lambda x: x['percentage'], reverse=True) + + def _get_recipe_distribution(self, recipe_distribution: dict) -> dict: + + deep_dive = {} + + for ticker, recipes in recipe_distribution.items(): + total_recipe_runs = sum(recipes.values()) + if total_recipe_runs == 0: + continue + + # top 5 most used recipes for this building + top_three = [] + for rid, count in recipes.most_common(5): + percentage = round(count / total_recipe_runs * 100, 2) + if percentage >= self.RECIPE_USAGE_CUTOFF: + top_three.append({'recipe_id': rid, 'percentage': percentage}) + + if top_three: + deep_dive[ticker] = top_three + + return deep_dive diff --git a/backend/analytics/tasks.py b/backend/analytics/tasks.py index b41b1b9..f99b41a 100644 --- a/backend/analytics/tasks.py +++ b/backend/analytics/tasks.py @@ -21,6 +21,7 @@ from user.models import User from analytics.models import AppStatistic +from analytics.services.PlanInsightAggregatorService import PlanInsightAggregatorService logger = structlog.get_logger(__name__) @@ -94,3 +95,21 @@ def update_daily_stats(): except Exception as exc: log.error('Failed to update daily statistics', exc_info=exc) return f'Failed to update stats at {now.date()}' + + +@shared_task(name='analytics_update_plan_insight_aggregates') +def analytics_update_plan_insight_aggregates(): + structlog.contextvars.bind_contextvars( + task_category='analytics_update_plan_insight_aggregates', + ) + + log = logger.bind(name='analytics_update_plan_insight_aggregates') + + try: + aggregator = PlanInsightAggregatorService() + processed, deleted = aggregator.aggregate_all_plans() + + log.info('completed', processed=processed, deleted=deleted) + + except Exception as exc: + log.error('exception', exc_info=exc) diff --git a/backend/api/urls.py b/backend/api/urls.py index c9b826b..d806174 100644 --- a/backend/api/urls.py +++ b/backend/api/urls.py @@ -20,4 +20,5 @@ path('user/', include('user.api.urls', namespace='user')), path('data/', include('gamedata.api.urls', namespace='data')), path('planning/', include('planning.api.urls', namespace='planning')), + path('analytics/', include('analytics.api.urls', namespace='analytics')), ]