Modify activity log database to partition table

modules/weko-logging/setup.py

@@ -65,6 +65,9 @@
     include_package_data=True,
     platforms="any",
     entry_points={
+        "flask.commands": [
+            "logging = weko_logging.cli:logging",
+        ],
         "invenio_base.apps": [
             "weko_logging_fs = weko_logging.fs:WekoLoggingFS",
             "weko_logging_user_activity = weko_logging.audit:WekoLoggingUserActivity",

modules/weko-logging/weko_logging/cli.py

@@ -0,0 +1,51 @@
+
+import click
+from datetime import datetime
+from flask.cli import with_appcontext
+from invenio_db import db
+
+@click.group()
+def logging():
+    """Logging CLI group."""
+    pass
+
+@logging.group()
+def partition():
+    """Partition management commands."""
+    pass
+
+@partition.command('create')
+@click.argument('year', nargs=1,type=int)
+@click.argument('month', default=0,type=int)
+@with_appcontext
+def _partition_create(year, month):
+    """Create partition table for user activity logs.
+
+    YEAR: Year of the partition table.
+    MONTH: Month of the partition table. If 0, create for all months in the year.
+    """
+    from weko_logging.models import make_user_activity_logs_partition_table,\
+        get_user_activity_logs_partition_tables
+
+    (sm, em) = (1, 12) if month==0 else (month, month)
+
+    try:
+        d = datetime(year, sm, 1)
+    except Exception as e:
+        click.secho(e, fg='red')
+        return
+
+    try:
+        tables = get_user_activity_logs_partition_tables()
+
+        for m in range(sm, em+1):
+            tablename = make_user_activity_logs_partition_table(year, m)
+            if tablename in tables:
+                click.secho('Table {} is already exist.'.format(tablename), fg='yellow')
+            else:
+                click.secho('Creating partitioning table {}.'.format(tablename), fg='green')
+                db.metadata.tables[tablename].create(bind=db.engine, checkfirst=True)
+        db.session.commit()
+    except Exception as e:
+        db.session.rollback()
+        click.secho(str(e), fg='red')

modules/weko-logging/weko_logging/models.py

@@ -9,17 +9,20 @@
 
 from datetime import datetime, timezone
 
-from sqlalchemy import Sequence
+from dateutil.relativedelta import relativedelta
+from sqlalchemy import Sequence, event
 from sqlalchemy.dialects import mysql, postgresql
 from sqlalchemy_utils.types import JSONType
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.sql.ddl import DDL
 
 from invenio_accounts.models import User
 from invenio_communities.models import Community
 from invenio_db import db
 
 group_id_seq = Sequence('user_activity_log_group_id_seq', metadata=db.metadata)
 
-class UserActivityLog(db.Model):
+class _UserActivityLogBase(db.Model):
     """User activity log model."""
 
     __tablename__ = 'user_activity_logs'
@@ -35,29 +38,34 @@ class UserActivityLog(db.Model):
         db.DateTime().with_variant(mysql.DATETIME(fsp=6), 'mysql'),
         nullable=False,
         default=lambda: datetime.now(timezone.utc),
+        primary_key=True
     )
     """Date and time of the log entry."""
 
-    user_id = db.Column(
-        db.Integer(),
-        db.ForeignKey(
-            User.id,
-            name='fk_user_activity_active_user_id',
-            ondelete='SET NULL'
-        ),
-        nullable=True
-    )
+    @declared_attr
+    def user_id(cls):
+        return db.Column(
+            db.Integer(),
+            db.ForeignKey(
+                User.id,
+                name='fk_user_activity_active_user_id',
+                ondelete='SET NULL'
+            ),
+            nullable=True
+        )
     """User ID of the user who performed the action."""
 
-    community_id = db.Column(
-        db.String(100),
-        db.ForeignKey(
-            Community.id,
-            name='fk_user_activity_community_id',
-            ondelete='SET NULL'
-        ),
-        nullable=True
-    )
+    @declared_attr
+    def community_id(cls):
+        return db.Column(
+            db.String(100),
+            db.ForeignKey(
+                Community.id,
+                name='fk_user_activity_community_id',
+                ondelete='SET NULL'
+            ),
+            nullable=True
+        )
     """Community ID of the community where the action was performed."""
 
     log_group_id = db.Column(
@@ -117,3 +125,55 @@ def get_log_group_sequence(cls, session):
             session = db.session
         next_id = session.execute(group_id_seq)
         return next_id
+
+class UserActivityLog(db.Model,_UserActivityLogBase):
+    """User activity log model."""
+
+    __tablename__ = 'user_activity_logs'
+    __table_args__ = (
+        db.UniqueConstraint('date', name='uq_date'),
+        { "postgresql_partition_by": 'RANGE (date)' }
+    )
+
+def get_user_activity_logs_partition_tables():
+    """Get the partition table for the user_activity_logs table
+
+    Returns:
+        list: List of partition table names.
+    """
+
+    query = "select tablename from pg_tables where tablename like 'user_activity_logs_partition_%'"
+    tables = db.session.execute(query).fetchall()
+
+    return [a[0] for a in tables]
+
+def make_user_activity_logs_partition_table(year, month):
+    """Create a new partition table for user_activity_logs for a specific year and month.
+
+    Args:
+        year (int): The year for the partition.
+        month (int): The month for the partition.
+    Returns:
+        str: The name of the created partition table.
+    """
+    start_date = datetime(year, month, 1, 0, 0, 0)
+    end_date = start_date + relativedelta(months=1)
+    suffix = '_' + start_date.strftime('%Y%m')
+    tablename = UserActivityLog.__tablename__ + suffix
+
+    NewPartitionTable = type('UserActivityLogPartition' + suffix,
+                             (db.Model,_UserActivityLogBase),
+                             {"__tablename__": tablename})
+    NewPartitionTable.__table__.add_is_dependent_on(UserActivityLog.__table__)
+
+    alter_table = \
+        "ALTER TABLE " + UserActivityLog.__tablename__ + " ATTACH PARTITION " + \
+        tablename + \
+        " FOR VALUES FROM ('{}') TO ('{}');".format(start_date.strftime('%Y-%m-%d'),
+                                                    end_date.strftime('%Y-%m-%d'))
+
+    event.listen(NewPartitionTable.__table__,
+                 "after_create",
+                 DDL(alter_table))
+
+    return tablename

postgresql/ddl/W2025-29.sql

@@ -161,13 +161,88 @@ CREATE TABLE IF NOT EXISTS user_activity_logs (
     log_group_id INTEGER,
     log JSONB NOT NULL,
     remarks TEXT,
-    CONSTRAINT pk_user_activity_logs PRIMARY KEY (id),
+    CONSTRAINT pk_user_activity_logs PRIMARY KEY (id, date),
+    CONSTRAINT uq_date UNIQUE (date),
     CONSTRAINT fk_user_activity_active_user_id
         FOREIGN KEY (user_id) REFERENCES accounts_user(id) ON DELETE SET NULL,
     CONSTRAINT fk_user_activity_community_id
         FOREIGN KEY (community_id) REFERENCES communities_community(id) ON DELETE SET NULL
-);
+) PARTITION BY RANGE (date);
 CREATE SEQUENCE IF NOT EXISTS user_activity_log_group_id_seq;
+-- 2026-01～2026-12
+CREATE TABLE user_activity_logs_202601 (LIKE user_activity_logs INCLUDING ALL);
+ALTER TABLE user_activity_logs ATTACH PARTITION user_activity_logs_202601 FOR VALUES FROM ('2026-01-01') TO ('2026-02-01');
+
+CREATE TABLE user_activity_logs_202602 (LIKE user_activity_logs INCLUDING ALL);
+ALTER TABLE user_activity_logs ATTACH PARTITION user_activity_logs_202602 FOR VALUES FROM ('2026-02-01') TO ('2026-03-01');
+
+CREATE TABLE user_activity_logs_202603 (LIKE user_activity_logs INCLUDING ALL);
+ALTER TABLE user_activity_logs ATTACH PARTITION user_activity_logs_202603 FOR VALUES FROM ('2026-03-01') TO ('2026-04-01');
+
+CREATE TABLE user_activity_logs_202604 (LIKE user_activity_logs INCLUDING ALL);
+ALTER TABLE user_activity_logs ATTACH PARTITION user_activity_logs_202604 FOR VALUES FROM ('2026-04-01') TO ('2026-05-01');
+
+CREATE TABLE user_activity_logs_202605 (LIKE user_activity_logs INCLUDING ALL);
+ALTER TABLE user_activity_logs ATTACH PARTITION user_activity_logs_202605 FOR VALUES FROM ('2026-05-01') TO ('2026-06-01');
+
+CREATE TABLE user_activity_logs_202606 (LIKE user_activity_logs INCLUDING ALL);
+ALTER TABLE user_activity_logs ATTACH PARTITION user_activity_logs_202606 FOR VALUES FROM ('2026-06-01') TO ('2026-07-01');
+
+CREATE TABLE user_activity_logs_202607 (LIKE user_activity_logs INCLUDING ALL);
+ALTER TABLE user_activity_logs ATTACH PARTITION user_activity_logs_202607 FOR VALUES FROM ('2026-07-01') TO ('2026-08-01');
+
+CREATE TABLE user_activity_logs_202608 (LIKE user_activity_logs INCLUDING ALL);
+ALTER TABLE user_activity_logs ATTACH PARTITION user_activity_logs_202608 FOR VALUES FROM ('2026-08-01') TO ('2026-09-01');
+
+CREATE TABLE user_activity_logs_202609 (LIKE user_activity_logs INCLUDING ALL);
+ALTER TABLE user_activity_logs ATTACH PARTITION user_activity_logs_202609 FOR VALUES FROM ('2026-09-01') TO ('2026-10-01');
+
+CREATE TABLE user_activity_logs_202610 (LIKE user_activity_logs INCLUDING ALL);
+ALTER TABLE user_activity_logs ATTACH PARTITION user_activity_logs_202610 FOR VALUES FROM ('2026-10-01') TO ('2026-11-01');
+
+CREATE TABLE user_activity_logs_202611 (LIKE user_activity_logs INCLUDING ALL);
+ALTER TABLE user_activity_logs ATTACH PARTITION user_activity_logs_202611 FOR VALUES FROM ('2026-11-01') TO ('2026-12-01');
+
+CREATE TABLE user_activity_logs_202612 (LIKE user_activity_logs INCLUDING ALL);
+ALTER TABLE user_activity_logs ATTACH PARTITION user_activity_logs_202612 FOR VALUES FROM ('2026-12-01') TO ('2027-01-01');
+
+-- 2027-01～2027-12
+CREATE TABLE user_activity_logs_202701 (LIKE user_activity_logs INCLUDING ALL);
+ALTER TABLE user_activity_logs ATTACH PARTITION user_activity_logs_202701 FOR VALUES FROM ('2027-01-01') TO ('2027-02-01');
+
+CREATE TABLE user_activity_logs_202702 (LIKE user_activity_logs INCLUDING ALL);
+ALTER TABLE user_activity_logs ATTACH PARTITION user_activity_logs_202702 FOR VALUES FROM ('2027-02-01') TO ('2027-03-01');
+
+CREATE TABLE user_activity_logs_202703 (LIKE user_activity_logs INCLUDING ALL);
+ALTER TABLE user_activity_logs ATTACH PARTITION user_activity_logs_202703 FOR VALUES FROM ('2027-03-01') TO ('2027-04-01');
+
+CREATE TABLE user_activity_logs_202704 (LIKE user_activity_logs INCLUDING ALL);
+ALTER TABLE user_activity_logs ATTACH PARTITION user_activity_logs_202704 FOR VALUES FROM ('2027-04-01') TO ('2027-05-01');
+
+CREATE TABLE user_activity_logs_202705 (LIKE user_activity_logs INCLUDING ALL);
+ALTER TABLE user_activity_logs ATTACH PARTITION user_activity_logs_202705 FOR VALUES FROM ('2027-05-01') TO ('2027-06-01');
+
+CREATE TABLE user_activity_logs_202706 (LIKE user_activity_logs INCLUDING ALL);
+ALTER TABLE user_activity_logs ATTACH PARTITION user_activity_logs_202706 FOR VALUES FROM ('2027-06-01') TO ('2027-07-01');
+
+CREATE TABLE user_activity_logs_202707 (LIKE user_activity_logs INCLUDING ALL);
+ALTER TABLE user_activity_logs ATTACH PARTITION user_activity_logs_202707 FOR VALUES FROM ('2027-07-01') TO ('2027-08-01');
+
+CREATE TABLE user_activity_logs_202708 (LIKE user_activity_logs INCLUDING ALL);
+ALTER TABLE user_activity_logs ATTACH PARTITION user_activity_logs_202708 FOR VALUES FROM ('2027-08-01') TO ('2027-09-01');
+
+CREATE TABLE user_activity_logs_202709 (LIKE user_activity_logs INCLUDING ALL);
+ALTER TABLE user_activity_logs ATTACH PARTITION user_activity_logs_202709 FOR VALUES FROM ('2027-09-01') TO ('2027-10-01');
+
+CREATE TABLE user_activity_logs_202710 (LIKE user_activity_logs INCLUDING ALL);
+ALTER TABLE user_activity_logs ATTACH PARTITION user_activity_logs_202710 FOR VALUES FROM ('2027-10-01') TO ('2027-11-01');
+
+CREATE TABLE user_activity_logs_202711 (LIKE user_activity_logs INCLUDING ALL);
+ALTER TABLE user_activity_logs ATTACH PARTITION user_activity_logs_202711 FOR VALUES FROM ('2027-11-01') TO ('2027-12-01');
+
+CREATE TABLE user_activity_logs_202712 (LIKE user_activity_logs INCLUDING ALL);
+ALTER TABLE user_activity_logs ATTACH PARTITION user_activity_logs_202712 FOR VALUES FROM ('2027-12-01') TO ('2028-01-01');
+
 
 -- modules/weko-notifications
 RAISE NOTICE 'Migration for module weko-notifications';

scripts/populate-instance.sh

@@ -127,6 +127,8 @@ ${INVENIO_WEB_INSTANCE} db init
 ${INVENIO_WEB_INSTANCE} db create -v
 ${INVENIO_WEB_INSTANCE} stats partition create $(date +%Y)
 ${INVENIO_WEB_INSTANCE} stats partition create $(date -d 'year' +%Y)
+${INVENIO_WEB_INSTANCE} logging partition create $(date +%Y)
+${INVENIO_WEB_INSTANCE} logging partition create $(date -d 'year' +%Y)
 # sphinxdoc-create-database-end
 
 # sphinxdoc-index-initialisation-begin