From 5fc46815fdea919ea823ffdc9cdc13571b32054a Mon Sep 17 00:00:00 2001 From: David Henne Date: Wed, 28 Apr 2021 15:39:12 -0500 Subject: [PATCH 1/8] Initial process worker and queue idea to replace threading in audit log. --- castle/cms/audit.py | 30 ++++++++++++++++-------------- castle/cms/utils/__init__.py | 1 + castle/cms/utils/workerprocess.py | 21 +++++++++++++++++++++ 3 files changed, 38 insertions(+), 14 deletions(-) create mode 100644 castle/cms/utils/workerprocess.py diff --git a/castle/cms/audit.py b/castle/cms/audit.py index c49281376..4341de832 100644 --- a/castle/cms/audit.py +++ b/castle/cms/audit.py @@ -1,4 +1,3 @@ -import threading from datetime import datetime import transaction @@ -6,8 +5,9 @@ IMetaTileEditedEvent, ITrashEmptiedEvent) from castle.cms.interfaces import ITrashed -from castle.cms.utils import ESConnectionFactoryFactory +from castle.cms.utils import ESConnectionFactoryFactory, Worker from elasticsearch import TransportError +from multiprocessing import Queue from plone import api from plone.app.iterate.interfaces import (IAfterCheckinEvent, ICancelCheckoutEvent, ICheckoutEvent, @@ -257,18 +257,20 @@ def record(success, recorder, site_path, conn): custom_index_value = None data = recorder() - thread = threading.Thread( - target=_record, - args=( - conn, - site_path, - data, - ), - kwargs={ - "es_custom_index_name_enabled": es_custom_index_name_enabled, - "custom_index_value": custom_index_value, - }) - thread.start() + + process_data = {} + process_data['target'] = _record + #! conn -> function factory cannot be pickled when put into request_queue... + #! PicklingError: Can't pickle : attribute lookup __builtin__.function failed + process_data['args'] = (conn, site_path, data,) + process_data['kwargs'] = { + "es_custom_index_name_enabled": es_custom_index_name_enabled, + "custom_index_value": custom_index_value, + } + + request_queue = Queue() + Worker(request_queue).start() + request_queue.put(process_data) def event(obj, event=None): diff --git a/castle/cms/utils/__init__.py b/castle/cms/utils/__init__.py index 2a2b5b7c0..d4acd08ea 100644 --- a/castle/cms/utils/__init__.py +++ b/castle/cms/utils/__init__.py @@ -40,3 +40,4 @@ from .security import is_backend # noqa from .text import truncate_text # noqa from .text import truncate_text as truncateText # noqa +from .workerprocess import Worker # noqa diff --git a/castle/cms/utils/workerprocess.py b/castle/cms/utils/workerprocess.py new file mode 100644 index 000000000..33b3c8be9 --- /dev/null +++ b/castle/cms/utils/workerprocess.py @@ -0,0 +1,21 @@ +from multiprocessing import Process + +import logging +logger = logging.getLogger('castle.cms') + + +class Worker(Process): + def __init__(self, queue): + super(Worker, self).__init__() + self.queue = queue + + def run(self): + logger.info('Worker started') + for process_data in iter(self.queue.get, None): + process = Process( + target=process_data['target'], + args=process_data['args'], + kwargs=process_data['kwargs'] + ) + process.start() + process.join() From 31f7399d017756362cf8b379df7747a08ce0d2d9 Mon Sep 17 00:00:00 2001 From: David Henne Date: Fri, 30 Apr 2021 15:30:24 -0500 Subject: [PATCH 2/8] Experimenting with diskcache deque to replace threading --- castle/cms/audit.py | 20 +++++++++----------- castle/cms/utils/__init__.py | 2 +- castle/cms/utils/diskcacheutil.py | 10 ++++++++++ castle/cms/utils/workerprocess.py | 21 --------------------- setup.py | 1 + versions.cfg | 1 + 6 files changed, 22 insertions(+), 33 deletions(-) create mode 100644 castle/cms/utils/diskcacheutil.py delete mode 100644 castle/cms/utils/workerprocess.py diff --git a/castle/cms/audit.py b/castle/cms/audit.py index 4341de832..4ab95c498 100644 --- a/castle/cms/audit.py +++ b/castle/cms/audit.py @@ -5,9 +5,8 @@ IMetaTileEditedEvent, ITrashEmptiedEvent) from castle.cms.interfaces import ITrashed -from castle.cms.utils import ESConnectionFactoryFactory, Worker +from castle.cms.utils import ESConnectionFactoryFactory from elasticsearch import TransportError -from multiprocessing import Queue from plone import api from plone.app.iterate.interfaces import (IAfterCheckinEvent, ICancelCheckoutEvent, ICheckoutEvent, @@ -258,19 +257,18 @@ def record(success, recorder, site_path, conn): data = recorder() - process_data = {} - process_data['target'] = _record - #! conn -> function factory cannot be pickled when put into request_queue... - #! PicklingError: Can't pickle : attribute lookup __builtin__.function failed - process_data['args'] = (conn, site_path, data,) - process_data['kwargs'] = { + from castle.cms.utils import DequeUtil + + kwargs = { "es_custom_index_name_enabled": es_custom_index_name_enabled, "custom_index_value": custom_index_value, } - request_queue = Queue() - Worker(request_queue).start() - request_queue.put(process_data) + deque_util = DequeUtil() + if data not in deque_util.deque: + deque_util.add_to_deque(data) + for i in range(len(deque_util.deque)): + _record(conn, site_path, deque_util.deque[i], kwargs) def event(obj, event=None): diff --git a/castle/cms/utils/__init__.py b/castle/cms/utils/__init__.py index d4acd08ea..d9efa96ff 100644 --- a/castle/cms/utils/__init__.py +++ b/castle/cms/utils/__init__.py @@ -18,6 +18,7 @@ from .content import is_max_paste_items # noqa from .content import is_mosaic_edit_form # noqa from .content import recursive_create_path # noqa +from .diskcacheutil import DequeUtil # noqa from .mail import get_email_from_address # noqa from .mail import send_email # noqa from .misc import get_ip # noqa @@ -40,4 +41,3 @@ from .security import is_backend # noqa from .text import truncate_text # noqa from .text import truncate_text as truncateText # noqa -from .workerprocess import Worker # noqa diff --git a/castle/cms/utils/diskcacheutil.py b/castle/cms/utils/diskcacheutil.py new file mode 100644 index 000000000..5e405026c --- /dev/null +++ b/castle/cms/utils/diskcacheutil.py @@ -0,0 +1,10 @@ +from diskcache import Deque + + +class DequeUtil(): + + def __init__(self): + self.deque = Deque() + + def add_to_deque(self, data): + self.deque.append(data) diff --git a/castle/cms/utils/workerprocess.py b/castle/cms/utils/workerprocess.py deleted file mode 100644 index 33b3c8be9..000000000 --- a/castle/cms/utils/workerprocess.py +++ /dev/null @@ -1,21 +0,0 @@ -from multiprocessing import Process - -import logging -logger = logging.getLogger('castle.cms') - - -class Worker(Process): - def __init__(self, queue): - super(Worker, self).__init__() - self.queue = queue - - def run(self): - logger.info('Worker started') - for process_data in iter(self.queue.get, None): - process = Process( - target=process_data['target'], - args=process_data['args'], - kwargs=process_data['kwargs'] - ) - process.start() - process.join() diff --git a/setup.py b/setup.py index eb1646f1a..fb43f2aef 100644 --- a/setup.py +++ b/setup.py @@ -82,6 +82,7 @@ def read(*rnames): 'tendo', 'pylru', 'sqlalchemy', + 'diskcache', # misc 'z3c.unconfigure', diff --git a/versions.cfg b/versions.cfg index a89c9ca93..73e9dc00d 100644 --- a/versions.cfg +++ b/versions.cfg @@ -96,6 +96,7 @@ futures = 3.3.0 jmespath = 0.9.4 s3transfer = 0.2.1 pylru = 1.2.0 +diskcache = 4.1.0 # testing Products.PrintingMailHost = 0.8 From ca8fe32a3c6a03bad93fd437f048140cb98ba0ab Mon Sep 17 00:00:00 2001 From: David Henne Date: Tue, 4 May 2021 14:13:47 -0500 Subject: [PATCH 3/8] create local disk cache, add audit data, and only remove after successful recording --- castle/cms/audit.py | 25 +++++++++++++++++-------- castle/cms/utils/__init__.py | 1 - castle/cms/utils/diskcacheutil.py | 10 ---------- 3 files changed, 17 insertions(+), 19 deletions(-) delete mode 100644 castle/cms/utils/diskcacheutil.py diff --git a/castle/cms/audit.py b/castle/cms/audit.py index 4ab95c498..cc790b998 100644 --- a/castle/cms/audit.py +++ b/castle/cms/audit.py @@ -6,6 +6,7 @@ ITrashEmptiedEvent) from castle.cms.interfaces import ITrashed from castle.cms.utils import ESConnectionFactoryFactory +from diskcache import Cache from elasticsearch import TransportError from plone import api from plone.app.iterate.interfaces import (IAfterCheckinEvent, @@ -256,19 +257,27 @@ def record(success, recorder, site_path, conn): custom_index_value = None data = recorder() - - from castle.cms.utils import DequeUtil - kwargs = { "es_custom_index_name_enabled": es_custom_index_name_enabled, "custom_index_value": custom_index_value, } - deque_util = DequeUtil() - if data not in deque_util.deque: - deque_util.add_to_deque(data) - for i in range(len(deque_util.deque)): - _record(conn, site_path, deque_util.deque[i], kwargs) + cache = Cache('cache/auditcache') + + with Cache(cache.directory) as reference: + reference.set(data, data) + + import pdb; pdb.set_trace() + + for key in reference: + try: + _record(conn, site_path, reference[key], **kwargs) + del reference[key] + except: + import logging + logger = logging.getLogger('castle.cms') + logger.warning('audit record failed') + pass def event(obj, event=None): diff --git a/castle/cms/utils/__init__.py b/castle/cms/utils/__init__.py index d9efa96ff..2a2b5b7c0 100644 --- a/castle/cms/utils/__init__.py +++ b/castle/cms/utils/__init__.py @@ -18,7 +18,6 @@ from .content import is_max_paste_items # noqa from .content import is_mosaic_edit_form # noqa from .content import recursive_create_path # noqa -from .diskcacheutil import DequeUtil # noqa from .mail import get_email_from_address # noqa from .mail import send_email # noqa from .misc import get_ip # noqa diff --git a/castle/cms/utils/diskcacheutil.py b/castle/cms/utils/diskcacheutil.py deleted file mode 100644 index 5e405026c..000000000 --- a/castle/cms/utils/diskcacheutil.py +++ /dev/null @@ -1,10 +0,0 @@ -from diskcache import Deque - - -class DequeUtil(): - - def __init__(self): - self.deque = Deque() - - def add_to_deque(self, data): - self.deque.append(data) From 9b2973ff9e8c825aedd8db5b8bb6193c3c4b65a2 Mon Sep 17 00:00:00 2001 From: David Henne Date: Thu, 6 May 2021 14:23:33 -0500 Subject: [PATCH 4/8] Starrted on script to check for entries in disk cache and record them. --- cache/auditcache/cache.db | Bin 0 -> 32768 bytes castle/cms/_scripts/events-audit.py | 41 +++++++++++++++++++ castle/cms/audit.py | 61 ++++++++++------------------ castle/cms/constants.py | 1 + 4 files changed, 63 insertions(+), 40 deletions(-) create mode 100644 cache/auditcache/cache.db create mode 100644 castle/cms/_scripts/events-audit.py diff --git a/cache/auditcache/cache.db b/cache/auditcache/cache.db new file mode 100644 index 0000000000000000000000000000000000000000..0d9eda0cd182bab576eaaf5dc08f0350d8655495 GIT binary patch literal 32768 zcmeI4Pi))P8Nex#k|_V-bKN+yn`L<}P|RLz{jsdrQ86I?!=&C+xs^pE0R|8Z^(@h* zL`i&%>tY$2E<5Pb+kl*U*=3g=df27M_Am^?4!aBkb_>w0?Xo|>hHi&p*ds;SOe-e_ zniw(g5uhIV`1tXAzxVM8-}m_5)|(}R61;77n;OM=Zj6gYxhoiR9LKX=WP7j;u>-+B zaq}a=g~y~km%jFkQS={fIQbEWI_OtYU3zi!qtV*P2g6?uzdGcwY}kMR5C8%|00;m9 zAOHl8Md0r8aN=AcCBB=XU87zn-7P|?(W=|3ZM;p?8ZikaYnt95YZZB=D&uNp-dFt?wOCF*t^tgeO}g~S4bU&=*J7i*C0DB7umM7~IH~~_#l*QKW;?HZwE=X? zY0={tPq^UIwVvFgX5&4`x0}VpW7R$e^1j;7g89*ESL5;2d@5oatr~gDGYHkDR+p&M zX!hSw(SJoZ%Iort{<{|{a%c`Shxg51kHu4&RAetCmAut9!eSEx#D>a*B_wJQpjH`%U~O1O{T7d@Ma##3wO zBb?V}cE@Bsy{b9X^3K&jkeUmQTtD9P?L8;{$e&~V5kVm!00e*l5C8%|00;m9AOHk_ z01yBIK;StiFv|2BlRl0~52W8qzmYyh_tB5gF4{t?=u`A-ltvd&OnQhue9o%@>Hz^D z00e*l5C8%|00;m9AOHk_!093&J><^i_?ltws!gM%y2I$+=SR;2{FC{#kFq7Usly2|0^hVceS&lu~+vN)pk6hc(cNw>K*P@ngnNe=Cx5FPn zbqGx`|NoO1N}%te&(Lqt_oe?y|CGK!s`LqZAN>?r=$Ggv>5r_UumJ%e00e*l5C8%| z00;m9AOHk_01!9{0_Z+BIUq9dUW(5Jj&jJ`7Cw{s3(*75UHGfWPp~N%Z)LHu;|(5n|dch3)h%46YJ!WsTzGQ+AHEa4 zGouLZEM!G|-KrVe2B|4=>(&m@se%jH{PIF!LDvhpTZ?+GaCv(>pIchWZxfQgoL#)6 zBy5MdDBb-V=b3hE4XsrtxNNbt!nXSPMZCRJZ|s`8JBr}2r4E#sR%2tfir_BZpuDt@ zUC7O63-j4a)!b6CP$(`er?a_2cKMPbv^CmL#xrZ0O--_p(VUt==S{1w2z0+ql!R`z zC}~k;xJxi#~_RxcNG%vKlDOSyb*F?WK^ zF*g3;&(~RVcf00e*l5C8%|00;m9 VAOHkTHG!#Ms)y_Mj|3A9{tq?Q4}bsw literal 0 HcmV?d00001 diff --git a/castle/cms/_scripts/events-audit.py b/castle/cms/_scripts/events-audit.py new file mode 100644 index 000000000..77a0ddb55 --- /dev/null +++ b/castle/cms/_scripts/events-audit.py @@ -0,0 +1,41 @@ +import argparse +import time +import logging + +from castle.cms.audit import _record +from castle.cms.constants import AUDIT_CACHE_DIRECTORY +from castle.cms.utils import ESConnectionFactoryFactory +from diskcache import Cache +from elasticsearch import TransportError +from plone import api + + +logger = logging.getLogger(__name__) + +parser = argparse.ArgumentParser( + description='...') +parser.add_argument('--site-id', dest='site_id', default='Plone') +args, _ = parser.parse_known_args() + +site = app[args.site_id] +registry = site.portal_registry +conn_factory = ESConnectionFactoryFactory(registry) + + +while (True): + cache = Cache(AUDIT_CACHE_DIRECTORY) + if len(cache) == 0: + cache.close() + logger.warn('sleeping for 5 seconds') + time.sleep(5) + else: + with Cache(cache.directory) as reference: + for key in reference: + args = (conn_factory, reference[key]['site_path'], key) + kwargs = reference[key]['kwargs'] + try: + logger.warn('Audit Record: %s' % key) + _record(*args, **kwargs) + del reference[key] + except: + logger.error('could not add record to audit log') diff --git a/castle/cms/audit.py b/castle/cms/audit.py index cc790b998..688e5c124 100644 --- a/castle/cms/audit.py +++ b/castle/cms/audit.py @@ -1,11 +1,10 @@ from datetime import datetime -import transaction +from castle.cms.constants import AUDIT_CACHE_DIRECTORY from castle.cms.events import (ICacheInvalidatedEvent, IMetaTileEditedEvent, ITrashEmptiedEvent) from castle.cms.interfaces import ITrashed -from castle.cms.utils import ESConnectionFactoryFactory from diskcache import Cache from elasticsearch import TransportError from plone import api @@ -244,42 +243,6 @@ def _record(conn_factory, site_path, data, es_custom_index_name_enabled=False, c raise ex -def record(success, recorder, site_path, conn): - if not success: - return - if recorder.valid: - try: - es_custom_index_name_enabled = api.portal.get_registry_record( - 'castle.es_index_enabled', default=False) - custom_index_value = api.portal.get_registry_record('castle.es_index', default=None) - except Exception: - es_custom_index_name_enabled = False - custom_index_value = None - - data = recorder() - kwargs = { - "es_custom_index_name_enabled": es_custom_index_name_enabled, - "custom_index_value": custom_index_value, - } - - cache = Cache('cache/auditcache') - - with Cache(cache.directory) as reference: - reference.set(data, data) - - import pdb; pdb.set_trace() - - for key in reference: - try: - _record(conn, site_path, reference[key], **kwargs) - del reference[key] - except: - import logging - logger = logging.getLogger('castle.cms') - logger.warning('audit record failed') - pass - - def event(obj, event=None): if event is None: @@ -313,7 +276,25 @@ def event(obj, event=None): recorder = audit_data.get_recorder(event, obj) site_path = '/'.join(api.portal.get().getPhysicalPath()) - transaction.get().addAfterCommitHook(record, args=( - recorder, site_path, ESConnectionFactoryFactory(registry))) + if recorder.valid: + try: + es_custom_index_name_enabled = api.portal.get_registry_record( + 'castle.es_index_enabled', default=False) + custom_index_value = api.portal.get_registry_record('castle.es_index', default=None) + except Exception: + es_custom_index_name_enabled = False + custom_index_value = None + + data = recorder() + kwargs = { + "es_custom_index_name_enabled": es_custom_index_name_enabled, + "custom_index_value": custom_index_value, + } + + cache = Cache(AUDIT_CACHE_DIRECTORY) + data_dict = {'site_path': site_path, 'kwargs': kwargs} + + with Cache(cache.directory) as reference: + reference.set(data, data_dict) else: pass diff --git a/castle/cms/constants.py b/castle/cms/constants.py index 22983571b..e5a0087a5 100644 --- a/castle/cms/constants.py +++ b/castle/cms/constants.py @@ -28,3 +28,4 @@ VALID_CSS_FONT_SIZE_PATTERN = compile("^\s*\d+\s*({})\s*$".format( # noqa:W605 '|'.join(ABSOLUTE_FONT_UNITS) )) +AUDIT_CACHE_DIRECTORY = 'cache/auditcache' From 13b34a0e560910cc0c5c62da1924032ed3096158 Mon Sep 17 00:00:00 2001 From: David Henne Date: Mon, 10 May 2021 15:48:20 -0500 Subject: [PATCH 5/8] gitignore cache folder --- .gitignore | 1 + cache/auditcache/cache.db | Bin 32768 -> 0 bytes 2 files changed, 1 insertion(+) delete mode 100644 cache/auditcache/cache.db diff --git a/.gitignore b/.gitignore index 0f0ca546d..09a772984 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,4 @@ export/ /package.json /watch-run.py /watchable-grunt.js +cache/ diff --git a/cache/auditcache/cache.db b/cache/auditcache/cache.db deleted file mode 100644 index 0d9eda0cd182bab576eaaf5dc08f0350d8655495..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 32768 zcmeI4Pi))P8Nex#k|_V-bKN+yn`L<}P|RLz{jsdrQ86I?!=&C+xs^pE0R|8Z^(@h* zL`i&%>tY$2E<5Pb+kl*U*=3g=df27M_Am^?4!aBkb_>w0?Xo|>hHi&p*ds;SOe-e_ zniw(g5uhIV`1tXAzxVM8-}m_5)|(}R61;77n;OM=Zj6gYxhoiR9LKX=WP7j;u>-+B zaq}a=g~y~km%jFkQS={fIQbEWI_OtYU3zi!qtV*P2g6?uzdGcwY}kMR5C8%|00;m9 zAOHl8Md0r8aN=AcCBB=XU87zn-7P|?(W=|3ZM;p?8ZikaYnt95YZZB=D&uNp-dFt?wOCF*t^tgeO}g~S4bU&=*J7i*C0DB7umM7~IH~~_#l*QKW;?HZwE=X? zY0={tPq^UIwVvFgX5&4`x0}VpW7R$e^1j;7g89*ESL5;2d@5oatr~gDGYHkDR+p&M zX!hSw(SJoZ%Iort{<{|{a%c`Shxg51kHu4&RAetCmAut9!eSEx#D>a*B_wJQpjH`%U~O1O{T7d@Ma##3wO zBb?V}cE@Bsy{b9X^3K&jkeUmQTtD9P?L8;{$e&~V5kVm!00e*l5C8%|00;m9AOHk_ z01yBIK;StiFv|2BlRl0~52W8qzmYyh_tB5gF4{t?=u`A-ltvd&OnQhue9o%@>Hz^D z00e*l5C8%|00;m9AOHk_!093&J><^i_?ltws!gM%y2I$+=SR;2{FC{#kFq7Usly2|0^hVceS&lu~+vN)pk6hc(cNw>K*P@ngnNe=Cx5FPn zbqGx`|NoO1N}%te&(Lqt_oe?y|CGK!s`LqZAN>?r=$Ggv>5r_UumJ%e00e*l5C8%| z00;m9AOHk_01!9{0_Z+BIUq9dUW(5Jj&jJ`7Cw{s3(*75UHGfWPp~N%Z)LHu;|(5n|dch3)h%46YJ!WsTzGQ+AHEa4 zGouLZEM!G|-KrVe2B|4=>(&m@se%jH{PIF!LDvhpTZ?+GaCv(>pIchWZxfQgoL#)6 zBy5MdDBb-V=b3hE4XsrtxNNbt!nXSPMZCRJZ|s`8JBr}2r4E#sR%2tfir_BZpuDt@ zUC7O63-j4a)!b6CP$(`er?a_2cKMPbv^CmL#xrZ0O--_p(VUt==S{1w2z0+ql!R`z zC}~k;xJxi#~_RxcNG%vKlDOSyb*F?WK^ zF*g3;&(~RVcf00e*l5C8%|00;m9 VAOHkTHG!#Ms)y_Mj|3A9{tq?Q4}bsw From 7d400022b38ad7e5d998a28e53880d5465421f20 Mon Sep 17 00:00:00 2001 From: David Henne Date: Mon, 10 May 2021 16:11:11 -0500 Subject: [PATCH 6/8] minor script tweaks --- castle/cms/_scripts/events-audit.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/castle/cms/_scripts/events-audit.py b/castle/cms/_scripts/events-audit.py index 77a0ddb55..d1725e871 100644 --- a/castle/cms/_scripts/events-audit.py +++ b/castle/cms/_scripts/events-audit.py @@ -1,13 +1,12 @@ import argparse -import time import logging +import os +import time from castle.cms.audit import _record from castle.cms.constants import AUDIT_CACHE_DIRECTORY from castle.cms.utils import ESConnectionFactoryFactory from diskcache import Cache -from elasticsearch import TransportError -from plone import api logger = logging.getLogger(__name__) @@ -15,15 +14,16 @@ parser = argparse.ArgumentParser( description='...') parser.add_argument('--site-id', dest='site_id', default='Plone') +parser.add_argument('--cache-dir', dest='cache_dir', default=AUDIT_CACHE_DIRECTORY) args, _ = parser.parse_known_args() site = app[args.site_id] +cache_dir = os.path.relpath(args.cache_dir) registry = site.portal_registry conn_factory = ESConnectionFactoryFactory(registry) - while (True): - cache = Cache(AUDIT_CACHE_DIRECTORY) + cache = Cache(cache_dir) if len(cache) == 0: cache.close() logger.warn('sleeping for 5 seconds') From 14062359dc2b63e4779a4f1403cb26441b65f16f Mon Sep 17 00:00:00 2001 From: David Henne Date: Mon, 17 May 2021 09:41:04 -0500 Subject: [PATCH 7/8] save obj in cache by UUID if exists. Start on audit tests. --- castle/cms/_scripts/events-audit.py | 2 +- castle/cms/audit.py | 7 ++++-- castle/cms/tests/test_audit.py | 33 +++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 3 deletions(-) create mode 100644 castle/cms/tests/test_audit.py diff --git a/castle/cms/_scripts/events-audit.py b/castle/cms/_scripts/events-audit.py index d1725e871..3faa79521 100644 --- a/castle/cms/_scripts/events-audit.py +++ b/castle/cms/_scripts/events-audit.py @@ -31,7 +31,7 @@ else: with Cache(cache.directory) as reference: for key in reference: - args = (conn_factory, reference[key]['site_path'], key) + args = (conn_factory, reference[key]['site_path'], reference[key]['data']) kwargs = reference[key]['kwargs'] try: logger.warn('Audit Record: %s' % key) diff --git a/castle/cms/audit.py b/castle/cms/audit.py index 60a91a602..4d1ebbc8a 100644 --- a/castle/cms/audit.py +++ b/castle/cms/audit.py @@ -287,9 +287,12 @@ def event(obj, event=None): } cache = Cache(AUDIT_CACHE_DIRECTORY) - data_dict = {'site_path': site_path, 'kwargs': kwargs} + data_dict = {'site_path': site_path, 'data': data, 'kwargs': kwargs} with Cache(cache.directory) as reference: - reference.set(data, data_dict) + if obj: + reference.set(data['object'], data_dict) + else: + reference.set(data, data_dict) else: pass diff --git a/castle/cms/tests/test_audit.py b/castle/cms/tests/test_audit.py new file mode 100644 index 000000000..50ba4f5e2 --- /dev/null +++ b/castle/cms/tests/test_audit.py @@ -0,0 +1,33 @@ +import unittest + +from castle.cms.constants import AUDIT_CACHE_DIRECTORY +from castle.cms.testing import CASTLE_PLONE_INTEGRATION_TESTING +from diskcache import Cache +from plone import api +from plone.app.testing import TEST_USER_ID +from plone.app.testing import TEST_USER_NAME +from plone.app.testing import login +from plone.app.testing import setRoles + + +class TestAudit(unittest.TestCase): + + layer = CASTLE_PLONE_INTEGRATION_TESTING + + def setUp(self): + self.portal = self.layer['portal'] + self.request = self.layer['request'] + login(self.portal, TEST_USER_NAME) + setRoles(self.portal, TEST_USER_ID, ('Member', 'Manager')) + + def test_cache_object(self): + obj = api.content.create(type='Document', id='doc1', + container=self.portal) + api.portal.set_registry_record( + 'collective.elasticsearch.interfaces.IElasticSettings.enabled', True) + api.content.transition(obj=obj, to_state='published') + obj.reindexObject() + obj_id = getattr(obj, '_plone.uuid') + cache = Cache(AUDIT_CACHE_DIRECTORY) + self.assertTrue(obj_id in cache) + cache.clear() From c3744f41b09e352bfc10c3d080f2a664e2baaa06 Mon Sep 17 00:00:00 2001 From: David Henne Date: Wed, 19 May 2021 12:06:23 -0500 Subject: [PATCH 8/8] added a couple tests for caching function --- castle/cms/_scripts/events-audit.py | 4 +--- castle/cms/tests/test_audit.py | 18 ++++++++++++++++-- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/castle/cms/_scripts/events-audit.py b/castle/cms/_scripts/events-audit.py index 3faa79521..6271c0bd2 100644 --- a/castle/cms/_scripts/events-audit.py +++ b/castle/cms/_scripts/events-audit.py @@ -26,15 +26,13 @@ cache = Cache(cache_dir) if len(cache) == 0: cache.close() - logger.warn('sleeping for 5 seconds') - time.sleep(5) + time.sleep(30) else: with Cache(cache.directory) as reference: for key in reference: args = (conn_factory, reference[key]['site_path'], reference[key]['data']) kwargs = reference[key]['kwargs'] try: - logger.warn('Audit Record: %s' % key) _record(*args, **kwargs) del reference[key] except: diff --git a/castle/cms/tests/test_audit.py b/castle/cms/tests/test_audit.py index 50ba4f5e2..d78274460 100644 --- a/castle/cms/tests/test_audit.py +++ b/castle/cms/tests/test_audit.py @@ -19,15 +19,29 @@ def setUp(self): self.request = self.layer['request'] login(self.portal, TEST_USER_NAME) setRoles(self.portal, TEST_USER_ID, ('Member', 'Manager')) + api.portal.set_registry_record( + 'collective.elasticsearch.interfaces.IElasticSettings.enabled', True) def test_cache_object(self): obj = api.content.create(type='Document', id='doc1', container=self.portal) - api.portal.set_registry_record( - 'collective.elasticsearch.interfaces.IElasticSettings.enabled', True) api.content.transition(obj=obj, to_state='published') obj.reindexObject() obj_id = getattr(obj, '_plone.uuid') cache = Cache(AUDIT_CACHE_DIRECTORY) self.assertTrue(obj_id in cache) cache.clear() + + def test_es_custom_index(self): + obj = api.content.create(type='Document', id='doc1', + container=self.portal) + api.portal.set_registry_record('castle.es_index_enabled', True) + api.portal.set_registry_record('castle.es_index', u'test-index') + api.content.transition(obj=obj, to_state='published') + obj.reindexObject() + obj_id = getattr(obj, '_plone.uuid') + cache = Cache(AUDIT_CACHE_DIRECTORY) + self.assertTrue(obj_id in cache) + self.assertTrue(cache[obj_id]['kwargs']['es_custom_index_name_enabled']) + self.assertEqual(cache[obj_id]['kwargs']['custom_index_value'], u'test-index') + cache.clear()